diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 195e1e06eea..97b5e558f40 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -1,3 +1,6 @@ +# all: Apply code formatting to new paths. +48c7daaa3d7068c127c5bc395ffa72029ab9bba1 + # all: Prune trailing whitespace. dda9b9c6da5d3c31fa8769e581a753e95a270803 diff --git a/.gitattributes b/.gitattributes index d226cebf5d8..890c9cead6c 100644 --- a/.gitattributes +++ b/.gitattributes @@ -13,12 +13,14 @@ # CIRCUITPY-CHANGE: add some more binary types. # These are binary so should never be modified by git. *.a binary +*.FLM binary *.ico binary *.png binary *.jpg binary *.dxf binary *.mpy binary *.der binary +*.bin binary *.deb binary *.zip binary *.pdf binary diff --git a/.gitignore b/.gitignore index ca6872387dd..659801277f3 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,7 @@ circuitpython-stubs/ test-stubs/ build-*/ docs/genrst/ +.mpy_ld_cache-*/ # Test failure outputs and intermediate artifacts ###################### diff --git a/conf.py b/conf.py index a584031b01b..d62931a7848 100644 --- a/conf.py +++ b/conf.py @@ -331,6 +331,7 @@ def autoapi_prepare_jinja_env(jinja_env): \hbadness=99999 \hfuzz=20pt \usepackage{pdflscape} +\DeclareUnicodeCharacter{FFFD}{?} """, } diff --git a/docs/library/builtins.rst b/docs/library/builtins.rst index 148c1513f3d..c8e05d895e5 100644 --- a/docs/library/builtins.rst +++ b/docs/library/builtins.rst @@ -37,6 +37,35 @@ Functions and types |see_cpython| `python:bytes`. + .. method:: bytes.decode(encoding='utf-8', errors='strict') + + Decode the bytes object to a string using the specified *encoding*. + + MicroPython supports the following encodings: + + - ``'utf-8'`` or ``'utf8'`` - UTF-8 encoding (default) + - ``'ascii'`` - ASCII encoding (subset of UTF-8) + + The *errors* parameter controls how decoding errors are handled: + + - ``'strict'`` - Raise a ``UnicodeError`` on invalid UTF-8 (default) + - ``'ignore'`` - Skip invalid bytes (requires ``MICROPY_PY_BUILTINS_BYTES_DECODE_ERRORS``) + - ``'replace'`` - Replace invalid bytes with U+FFFD '�' (requires ``MICROPY_PY_BUILTINS_BYTES_DECODE_ERRORS``) + + .. note:: + Error handler support depends on build configuration. On constrained + systems, only ``'strict'`` mode may be available. + + Example:: + + >>> b'\xc2\xa9 2024'.decode('utf-8') # © symbol + '© 2024' + >>> b'hello\xffworld'.decode('utf-8', 'ignore') # Skip invalid bytes + 'helloworld' + + Raises ``LookupError`` if the encoding is not supported, or + ``UnicodeError`` if the data contains invalid UTF-8 and ``errors='strict'``. + .. function:: callable() .. function:: chr() @@ -152,6 +181,35 @@ Functions and types .. class:: str() + .. method:: str.encode(encoding='utf-8') + + Encode the string to bytes using the specified *encoding*. + + MicroPython supports the following encodings: + + - ``'utf-8'`` or ``'utf8'`` - UTF-8 encoding (default) + - ``'ascii'`` - ASCII encoding (subset of UTF-8) + + Example:: + + >>> '© 2024'.encode('utf-8') # Copyright symbol + b'\xc2\xa9 2024' + + Raises ``LookupError`` if the encoding is not supported. + + .. method:: str.center(width) + + Return a centered string of length *width*. Padding is done using spaces. + + When Unicode support is enabled (``MICROPY_PY_BUILTINS_STR_UNICODE``), this + method counts Unicode characters rather than bytes, ensuring proper alignment + for multi-byte UTF-8 characters. + + Example:: + + >>> 'café'.center(10) # é is 2 bytes in UTF-8 + ' café ' + .. function:: sum() .. function:: super() diff --git a/docs/library/gc.rst b/docs/library/gc.rst index d1625c0d8f3..7e946469f63 100644 --- a/docs/library/gc.rst +++ b/docs/library/gc.rst @@ -2,7 +2,8 @@ ========================================== .. module:: gc - :synopsis: control the garbage collector + :synopsis: control the garbage collector which automatically frees + :ref:`heap memory ` |see_cpython_module| :mod:`python:gc`. @@ -18,6 +19,10 @@ Functions Disable automatic garbage collection. Heap memory can still be allocated, and garbage collection can still be initiated manually using :meth:`gc.collect`. +.. function:: isenabled() + + Returns True if automatic garbage collection is enabled, and False otherwise. + .. function:: collect() Run a garbage collection. @@ -64,3 +69,40 @@ Functions This function is a MicroPython extension. CPython has a similar function - ``set_threshold()``, but due to different GC implementations, its signature and semantics are different. + + Examples + ^^^^^^^^ + + To trigger a garbage collection each time 32768 bytes of RAM have been allocated in total:: + + gc.threshold(32768) + + To restore the default behaviour, only triggering garbage collection when out of memory:: + + gc.threshold(-1) + +Example +------- + +.. code-block:: bash + + >>> import gc + >>> gc.mem_free() # Gets number of bytes free in memory + 8192 + >>> gc.mem_alloc() # Gets number of bytes allocated in memory + 1024 + >>> foo = bytearray(1000) # Create a big array of data + >>> gc.mem_free() # Show that there's less memory available + 7168 + >>> gc.mem_alloc() # Show that there's more memory used + 2048 + >>> del foo # Delete the object + >>> gc.mem_free() # Show that collection hasn't run yet + 7168 + >>> gc.mem_alloc() # That memory is still allocated + 2048 + >>> gc.collect() # Manually run the collection + >>> gc.mem_free() # Now we have reclaimed that memory + 8192 + >>> gc.mem_alloc() # That memory is no longer allocated + 1024 diff --git a/docs/library/io.rst b/docs/library/io.rst index ad1fb03c777..8ebb2e8c63f 100644 --- a/docs/library/io.rst +++ b/docs/library/io.rst @@ -71,8 +71,9 @@ buffered, they aren't in MicroPython. (Indeed, that's one of the cases for which we may introduce buffering support.) Note that for efficiency, MicroPython doesn't provide abstract base -classes corresponding to the hierarchy above, and it's not possible -to implement, or subclass, a stream class in pure Python. +classes corresponding to the hierarchy above. However, the +:class:`IOBase` class can be subclassed to implement custom stream +objects in pure Python. Functions --------- @@ -95,6 +96,68 @@ Classes This is type of a file open in text mode, e.g. using ``open(name, "rt")``. You should not instantiate this class directly. +.. class:: IOBase() + + Base class for implementing custom stream objects in Python. Subclasses + can override ``readinto``, ``write``, and ``ioctl`` to create objects + that work with ``print()``, ``json.dump()``, ``select.poll()``, + ``open()`` via a user filesystem, and other stream consumers. + + .. admonition:: Difference to CPython + :class: attention + + In CPython, ``io.IOBase`` has a much larger API surface. MicroPython's + ``IOBase`` is minimal: the C stream infrastructure provides standard + methods like ``read()``, ``readline()``, ``seek()``, ``close()``, and + ``flush()`` automatically. Subclasses only need to implement the + methods below. + + Subclasses implement some or all of the following methods. The C stream + layer calls these internally when user code calls standard stream + functions. + + .. method:: IOBase.readinto(buf) + + Read data into *buf* (a ``bytearray`` sized by the caller). Return the + number of bytes read, 0 at EOF, or ``None`` if no data is available on + a non-blocking stream. Return a negative errno value (e.g. ``-errno.EIO`` + or ``-1``) to signal an error. + + .. method:: IOBase.write(buf) + + Write *buf* (a ``bytearray``) to the stream. Return the number of + bytes written, or ``None`` if a non-blocking stream cannot accept data. + Return a negative errno value to signal an error. + + .. method:: IOBase.ioctl(op, arg) + + Control the stream and query its properties. The operation to perform + is given by *op* which is one of the following integers: + + - 1 -- flush write buffers (*arg* is unused) + - 3 -- poll for readiness; *arg* is a bitmask of events to check, + return a bitmask of ready events. Poll flags: + + * ``0x0001`` -- data available for reading + * ``0x0004`` -- stream ready for writing + * ``0x0008`` -- error condition + * ``0x0010`` -- hang up (e.g. connection closed) + * ``0x0020`` -- invalid request + + - 4 -- close the stream (*arg* is unused) + - 11 -- return the preferred read buffer size, or 0 (*arg* is unused) + + As a minimum ``ioctl(4, ...)`` should be handled to support stream + closure. Implement ``ioctl(3, ...)`` if the stream will be used with + ``select.poll()`` or ``asyncio``. + + Other operations exist for advanced use cases (2 = seek, 5 = timeout, + 10 = fileno); see ``py/stream.h`` for the full list. + + Must always return an integer. Return 0 for success, or ``-1`` for + unsupported operations. (Returning 0 for an unhandled operation tells + the C layer the operation was processed successfully, which may cause + incorrect behaviour.) .. class:: StringIO([string]) .. class:: BytesIO([string]) @@ -129,3 +192,77 @@ Classes :class: attention These constructors are a MicroPython extension. + +IOBase Examples +--------------- + +A minimal write-only stream that collects output into a buffer:: + + import io + + class MyOutput(io.IOBase): + def __init__(self): + self.data = bytearray() + + def write(self, buf): + self.data.extend(buf) + return len(buf) + + def ioctl(self, op, arg): + if op == 4: # close + return 0 + return -1 + + s = MyOutput() + print("hello", file=s) + print(s.data) # bytearray(b'hello\n') + +A readable stream that can be used with ``select.poll()``:: + + import io, select + from micropython import const + + _MP_STREAM_POLL = const(3) + _MP_STREAM_POLL_RD = const(0x0001) + _MP_STREAM_CLOSE = const(4) + + class RingBuffer(io.IOBase): + def __init__(self, size): + self._buf = bytearray(size) + self._size = size + self._wpos = 0 + self._rpos = 0 + + def _available(self): + return (self._wpos - self._rpos) % self._size + + def put(self, data): + for b in data: + self._buf[self._wpos % self._size] = b + self._wpos = (self._wpos + 1) % self._size + + def readinto(self, buf): + n = min(len(buf), self._available()) + for i in range(n): + buf[i] = self._buf[self._rpos % self._size] + self._rpos = (self._rpos + 1) % self._size + return n + + def ioctl(self, op, arg): + if op == _MP_STREAM_POLL: + if arg & _MP_STREAM_POLL_RD and self._available() > 0: + return _MP_STREAM_POLL_RD + return 0 + if op == _MP_STREAM_CLOSE: + return 0 + return -1 + + rb = RingBuffer(64) + rb.put(b"test data") + + poller = select.poll() + poller.register(rb, select.POLLIN) + for obj, flags in poller.poll(0): + buf = bytearray(16) + n = obj.readinto(buf) + print(buf[:n]) # bytearray(b'test data') diff --git a/examples/natmod/btree/Makefile b/examples/natmod/btree/Makefile index ff130d61b37..040fa770396 100644 --- a/examples/natmod/btree/Makefile +++ b/examples/natmod/btree/Makefile @@ -2,7 +2,8 @@ MPY_DIR = ../../.. # Name of module (different to built-in btree so it can coexist) -MOD = btree_$(ARCH) +MOD_BASE = btree +MOD = $(MOD_BASE)_$(ARCH) # Source files (.c or .py) SRC = btree_c.c @@ -15,6 +16,7 @@ BERKELEY_DB_CONFIG_FILE ?= \"extmod/berkeley-db/berkeley_db_config_port.h\" CFLAGS += -I$(BTREE_DIR)/include CFLAGS += -DBERKELEY_DB_CONFIG_FILE=$(BERKELEY_DB_CONFIG_FILE) CFLAGS += -Wno-old-style-definition -Wno-sign-compare -Wno-unused-parameter +CFLAGS += -Wno-deprecated-non-prototype SRC += $(addprefix $(realpath $(BTREE_DIR))/,\ btree/bt_close.c \ @@ -32,6 +34,28 @@ SRC += $(addprefix $(realpath $(BTREE_DIR))/,\ mpool/mpool.c \ ) +ifeq ($(ARCH),xtensa) +MPY_EXTERN_SYM_FILE=$(MPY_DIR)/ports/esp8266/boards/eagle.rom.addr.v6.ld +endif + +# Use our own errno implementation if Picolibc is used +CFLAGS += -D__PICOLIBC_ERRNO_FUNCTION=__errno + +ifeq ($(ARCH),armv6m) +# Link with libgcc.a for division helper functions +LINK_RUNTIME = 1 +endif + +# Strip the architecture name from the internal filename. +MPY_LD_FLAGS = "--source-name=$(MOD_BASE).mpy" + +ifeq ($(ARCH),armv7m) +ifeq ($(findstring clang,$(shell $(CC) --version)),clang) +# Link with libclang_rt.builtins.a for memset/memcpy. +LINK_RUNTIME = 1 +endif +endif + include $(MPY_DIR)/py/dynruntime.mk # btree needs gnu99 defined diff --git a/examples/natmod/btree/btree_c.c b/examples/natmod/btree/btree_c.c index 4f494817e7b..c26dbcc8478 100644 --- a/examples/natmod/btree/btree_c.c +++ b/examples/natmod/btree/btree_c.c @@ -4,19 +4,6 @@ #include -#if !defined(__linux__) -void *memcpy(void *dst, const void *src, size_t n) { - return mp_fun_table.memmove_(dst, src, n); -} -void *memset(void *s, int c, size_t n) { - return mp_fun_table.memset_(s, c, n); -} -#endif - -void *memmove(void *dest, const void *src, size_t n) { - return mp_fun_table.memmove_(dest, src, n); -} - void *malloc(size_t n) { void *ptr = m_malloc(n); return ptr; @@ -45,16 +32,16 @@ int puts(const char *s) { int native_errno; #if defined(__linux__) -int *__errno_location (void) +int *__errno_location(void) #else -int *__errno (void) +int *__errno(void) #endif { return &native_errno; } ssize_t mp_stream_posix_write(void *stream, const void *buf, size_t len) { - mp_obj_base_t* o = stream; + mp_obj_base_t *o = stream; const mp_stream_p_t *stream_p = MP_OBJ_TYPE_GET_SLOT(o->type, protocol); mp_uint_t out_sz = stream_p->write(MP_OBJ_FROM_PTR(stream), buf, len, &native_errno); if (out_sz == MP_STREAM_ERROR) { @@ -65,7 +52,7 @@ ssize_t mp_stream_posix_write(void *stream, const void *buf, size_t len) { } ssize_t mp_stream_posix_read(void *stream, void *buf, size_t len) { - mp_obj_base_t* o = stream; + mp_obj_base_t *o = stream; const mp_stream_p_t *stream_p = MP_OBJ_TYPE_GET_SLOT(o->type, protocol); mp_uint_t out_sz = stream_p->read(MP_OBJ_FROM_PTR(stream), buf, len, &native_errno); if (out_sz == MP_STREAM_ERROR) { @@ -76,7 +63,7 @@ ssize_t mp_stream_posix_read(void *stream, void *buf, size_t len) { } off_t mp_stream_posix_lseek(void *stream, off_t offset, int whence) { - const mp_obj_base_t* o = stream; + const mp_obj_base_t *o = stream; const mp_stream_p_t *stream_p = MP_OBJ_TYPE_GET_SLOT(o->type, protocol); struct mp_stream_seek_t seek_s; seek_s.offset = offset; @@ -89,7 +76,7 @@ off_t mp_stream_posix_lseek(void *stream, off_t offset, int whence) { } int mp_stream_posix_fsync(void *stream) { - mp_obj_base_t* o = stream; + mp_obj_base_t *o = stream; const mp_stream_p_t *stream_p = MP_OBJ_TYPE_GET_SLOT(o->type, protocol); mp_uint_t res = stream_p->ioctl(MP_OBJ_FROM_PTR(stream), MP_STREAM_FLUSH, 0, &native_errno); if (res == MP_STREAM_ERROR) { @@ -146,22 +133,22 @@ mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *a btree_getiter_iternext.getiter = btree_getiter; btree_getiter_iternext.iternext = btree_iternext; - btree_type.base.type = (void*)&mp_fun_table.type_type; + btree_type.base.type = (void *)&mp_fun_table.type_type; btree_type.flags = MP_TYPE_FLAG_ITER_IS_CUSTOM; btree_type.name = MP_QSTR_btree; MP_OBJ_TYPE_SET_SLOT(&btree_type, print, btree_print, 0); MP_OBJ_TYPE_SET_SLOT(&btree_type, iter, &btree_getiter_iternext, 1); MP_OBJ_TYPE_SET_SLOT(&btree_type, binary_op, btree_binary_op, 2); MP_OBJ_TYPE_SET_SLOT(&btree_type, subscr, btree_subscr, 3); - btree_locals_dict_table[0] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_close), MP_OBJ_FROM_PTR(&btree_close_obj) }; - btree_locals_dict_table[1] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_flush), MP_OBJ_FROM_PTR(&btree_flush_obj) }; - btree_locals_dict_table[2] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_get), MP_OBJ_FROM_PTR(&btree_get_obj) }; - btree_locals_dict_table[3] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_put), MP_OBJ_FROM_PTR(&btree_put_obj) }; - btree_locals_dict_table[4] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_seq), MP_OBJ_FROM_PTR(&btree_seq_obj) }; - btree_locals_dict_table[5] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_keys), MP_OBJ_FROM_PTR(&btree_keys_obj) }; - btree_locals_dict_table[6] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_values), MP_OBJ_FROM_PTR(&btree_values_obj) }; - btree_locals_dict_table[7] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_items), MP_OBJ_FROM_PTR(&btree_items_obj) }; - MP_OBJ_TYPE_SET_SLOT(&btree_type, locals_dict, (void*)&btree_locals_dict, 4); + btree_locals_dict_table[0] = (mp_map_elem_t) { MP_OBJ_NEW_QSTR(MP_QSTR_close), MP_OBJ_FROM_PTR(&btree_close_obj) }; + btree_locals_dict_table[1] = (mp_map_elem_t) { MP_OBJ_NEW_QSTR(MP_QSTR_flush), MP_OBJ_FROM_PTR(&btree_flush_obj) }; + btree_locals_dict_table[2] = (mp_map_elem_t) { MP_OBJ_NEW_QSTR(MP_QSTR_get), MP_OBJ_FROM_PTR(&btree_get_obj) }; + btree_locals_dict_table[3] = (mp_map_elem_t) { MP_OBJ_NEW_QSTR(MP_QSTR_put), MP_OBJ_FROM_PTR(&btree_put_obj) }; + btree_locals_dict_table[4] = (mp_map_elem_t) { MP_OBJ_NEW_QSTR(MP_QSTR_seq), MP_OBJ_FROM_PTR(&btree_seq_obj) }; + btree_locals_dict_table[5] = (mp_map_elem_t) { MP_OBJ_NEW_QSTR(MP_QSTR_keys), MP_OBJ_FROM_PTR(&btree_keys_obj) }; + btree_locals_dict_table[6] = (mp_map_elem_t) { MP_OBJ_NEW_QSTR(MP_QSTR_values), MP_OBJ_FROM_PTR(&btree_values_obj) }; + btree_locals_dict_table[7] = (mp_map_elem_t) { MP_OBJ_NEW_QSTR(MP_QSTR_items), MP_OBJ_FROM_PTR(&btree_items_obj) }; + MP_OBJ_TYPE_SET_SLOT(&btree_type, locals_dict, (void *)&btree_locals_dict, 4); mp_store_global(MP_QSTR_open, MP_OBJ_FROM_PTR(&btree_open_obj)); mp_store_global(MP_QSTR_INCL, MP_OBJ_NEW_SMALL_INT(FLAG_END_KEY_INCL)); diff --git a/examples/natmod/deflate/Makefile b/examples/natmod/deflate/Makefile index 504130d5723..aa0951e88ef 100644 --- a/examples/natmod/deflate/Makefile +++ b/examples/natmod/deflate/Makefile @@ -2,12 +2,34 @@ MPY_DIR = ../../.. # Name of module (different to built-in uzlib so it can coexist) -MOD = deflate_$(ARCH) +MOD_BASE = deflate +MOD = $(MOD_BASE)_$(ARCH) # Source files (.c or .py) SRC = deflate.c -# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin, rv32imc) -ARCH = x64 +# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin, rv32imc, rv64imc) +ARCH ?= x64 + +ifeq ($(ARCH),armv7m) +ifeq ($(findstring clang,$(shell $(CC) --version)),clang) +# Link with libclang_rt.a for memset +LINK_RUNTIME = 1 +endif +endif + +ifeq ($(ARCH),armv6m) +# Link with libgcc.a or libclang_rt.a for division helper functions +LINK_RUNTIME = 1 +endif + +ifeq ($(ARCH),xtensa) +# Link with libm.a and libgcc.a from the toolchain +LINK_RUNTIME = 1 +MPY_EXTERN_SYM_FILE=$(MPY_DIR)/ports/esp8266/boards/eagle.rom.addr.v6.ld +endif + +# Strip the architecture name from the internal filename. +MPY_LD_FLAGS = "--source-name=$(MOD_BASE).mpy" include $(MPY_DIR)/py/dynruntime.mk diff --git a/examples/natmod/deflate/deflate.c b/examples/natmod/deflate/deflate.c index 9de7e101a76..3c470aca9f0 100644 --- a/examples/natmod/deflate/deflate.c +++ b/examples/natmod/deflate/deflate.c @@ -3,15 +3,6 @@ #include "py/dynruntime.h" -#if !defined(__linux__) -void *memcpy(void *dst, const void *src, size_t n) { - return mp_fun_table.memmove_(dst, src, n); -} -void *memset(void *s, int c, size_t n) { - return mp_fun_table.memset_(s, c, n); -} -#endif - mp_obj_full_type_t deflateio_type; #include "extmod/moddeflate.c" @@ -51,14 +42,14 @@ mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *a deflateio_type.name = MP_QSTR_DeflateIO; MP_OBJ_TYPE_SET_SLOT(&deflateio_type, make_new, &deflateio_make_new, 0); MP_OBJ_TYPE_SET_SLOT(&deflateio_type, protocol, &deflateio_stream_p, 1); - deflateio_locals_dict_table[0] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_read), MP_OBJ_FROM_PTR(&mp_stream_read_obj) }; - deflateio_locals_dict_table[1] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_readinto), MP_OBJ_FROM_PTR(&mp_stream_readinto_obj) }; - deflateio_locals_dict_table[2] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_readline), MP_OBJ_FROM_PTR(&mp_stream_unbuffered_readline_obj) }; - deflateio_locals_dict_table[3] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_write), MP_OBJ_FROM_PTR(&mp_stream_write_obj) }; - deflateio_locals_dict_table[4] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_close), MP_OBJ_FROM_PTR(&mp_stream_close_obj) }; - deflateio_locals_dict_table[5] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR___enter__), MP_OBJ_FROM_PTR(&mp_identity_obj) }; - deflateio_locals_dict_table[6] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR___exit__), MP_OBJ_FROM_PTR(&mp_stream___exit___obj) }; - MP_OBJ_TYPE_SET_SLOT(&deflateio_type, locals_dict, (void*)&deflateio_locals_dict, 2); + deflateio_locals_dict_table[0] = (mp_map_elem_t) { MP_OBJ_NEW_QSTR(MP_QSTR_read), MP_OBJ_FROM_PTR(&mp_stream_read_obj) }; + deflateio_locals_dict_table[1] = (mp_map_elem_t) { MP_OBJ_NEW_QSTR(MP_QSTR_readinto), MP_OBJ_FROM_PTR(&mp_stream_readinto_obj) }; + deflateio_locals_dict_table[2] = (mp_map_elem_t) { MP_OBJ_NEW_QSTR(MP_QSTR_readline), MP_OBJ_FROM_PTR(&mp_stream_unbuffered_readline_obj) }; + deflateio_locals_dict_table[3] = (mp_map_elem_t) { MP_OBJ_NEW_QSTR(MP_QSTR_write), MP_OBJ_FROM_PTR(&mp_stream_write_obj) }; + deflateio_locals_dict_table[4] = (mp_map_elem_t) { MP_OBJ_NEW_QSTR(MP_QSTR_close), MP_OBJ_FROM_PTR(&mp_stream_close_obj) }; + deflateio_locals_dict_table[5] = (mp_map_elem_t) { MP_OBJ_NEW_QSTR(MP_QSTR___enter__), MP_OBJ_FROM_PTR(&mp_identity_obj) }; + deflateio_locals_dict_table[6] = (mp_map_elem_t) { MP_OBJ_NEW_QSTR(MP_QSTR___exit__), MP_OBJ_FROM_PTR(&mp_stream___exit___obj) }; + MP_OBJ_TYPE_SET_SLOT(&deflateio_type, locals_dict, (void *)&deflateio_locals_dict, 2); mp_store_global(MP_QSTR___name__, MP_OBJ_NEW_QSTR(MP_QSTR_deflate)); mp_store_global(MP_QSTR_DeflateIO, MP_OBJ_FROM_PTR(&deflateio_type)); diff --git a/examples/natmod/features2/Makefile b/examples/natmod/features2/Makefile index 5ddb74087b7..2129974048f 100644 --- a/examples/natmod/features2/Makefile +++ b/examples/natmod/features2/Makefile @@ -10,6 +10,13 @@ SRC = main.c prod.c test.py # Architecture to build for (x86, x64, armv7m, xtensa, xtensawin) ARCH = x64 +ifeq ($(findstring clang,$(shell $(CC) --version)),clang) +ifeq ($(ARCH),$(filter $(ARCH),armv6m armv7m rv32imc)) +# Link with both libc.a and libclang_rt.builtins.a +LINK_CLANG_LIBC = 1 +endif +endif + # Link with libm.a and libgcc.a from the toolchain LINK_RUNTIME = 1 diff --git a/examples/natmod/features4/features4.c b/examples/natmod/features4/features4.c index e64c7f75921..c4d82a1f700 100644 --- a/examples/natmod/features4/features4.c +++ b/examples/natmod/features4/features4.c @@ -68,12 +68,12 @@ mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *a MP_DYNRUNTIME_INIT_ENTRY // Initialise the type. - mp_type_factorial.base.type = (void*)&mp_type_type; + mp_type_factorial.base.type = (void *)&mp_type_type; mp_type_factorial.flags = MP_TYPE_FLAG_NONE; mp_type_factorial.name = MP_QSTR_Factorial; MP_OBJ_TYPE_SET_SLOT(&mp_type_factorial, make_new, factorial_make_new, 0); - factorial_locals_dict_table[0] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_calculate), MP_OBJ_FROM_PTR(&factorial_calculate_obj) }; - MP_OBJ_TYPE_SET_SLOT(&mp_type_factorial, locals_dict, (void*)&factorial_locals_dict, 1); + factorial_locals_dict_table[0] = (mp_map_elem_t) { MP_OBJ_NEW_QSTR(MP_QSTR_calculate), MP_OBJ_FROM_PTR(&factorial_calculate_obj) }; + MP_OBJ_TYPE_SET_SLOT(&mp_type_factorial, locals_dict, (void *)&factorial_locals_dict, 1); // Make the Factorial type available on the module. mp_store_global(MP_QSTR_Factorial, MP_OBJ_FROM_PTR(&mp_type_factorial)); diff --git a/examples/natmod/framebuf/Makefile b/examples/natmod/framebuf/Makefile index 2e2b8159754..86a201d23bd 100644 --- a/examples/natmod/framebuf/Makefile +++ b/examples/natmod/framebuf/Makefile @@ -2,12 +2,32 @@ MPY_DIR = ../../.. # Name of module (different to built-in framebuf so it can coexist) -MOD = framebuf_$(ARCH) +MOD_BASE = framebuf +MOD = $(MOD_BASE)_$(ARCH) # Source files (.c or .py) SRC = framebuf.c -# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin) -ARCH = x64 +# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin, rv32imc, rv64imc) +ARCH ?= x64 + +ifeq ($(ARCH),armv7m) +ifeq ($(findstring clang,$(shell $(CC) --version)),clang) +# Link with libclang_rt.a for memset +LINK_RUNTIME = 1 +endif +endif + +ifeq ($(ARCH),armv6m) +# Link with libgcc.a for division helper functions +LINK_RUNTIME = 1 +endif + +ifeq ($(ARCH),xtensa) +MPY_EXTERN_SYM_FILE=$(MPY_DIR)/ports/esp8266/boards/eagle.rom.addr.v6.ld +endif + +# Strip the architecture name from the internal filename. +MPY_LD_FLAGS = "--source-name=$(MOD_BASE).mpy" include $(MPY_DIR)/py/dynruntime.mk diff --git a/examples/natmod/framebuf/framebuf.c b/examples/natmod/framebuf/framebuf.c index 1ba702e33d9..6e672439fe9 100644 --- a/examples/natmod/framebuf/framebuf.c +++ b/examples/natmod/framebuf/framebuf.c @@ -3,12 +3,6 @@ #include "py/dynruntime.h" -#if !defined(__linux__) -void *memset(void *s, int c, size_t n) { - return mp_fun_table.memset_(s, c, n); -} -#endif - mp_obj_full_type_t mp_type_framebuf; #include "extmod/modframebuf.c" @@ -19,23 +13,23 @@ static MP_DEFINE_CONST_DICT(framebuf_locals_dict, framebuf_locals_dict_table); mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) { MP_DYNRUNTIME_INIT_ENTRY - mp_type_framebuf.base.type = (void*)&mp_type_type; + mp_type_framebuf.base.type = (void *)&mp_type_type; mp_type_framebuf.name = MP_QSTR_FrameBuffer; MP_OBJ_TYPE_SET_SLOT(&mp_type_framebuf, make_new, framebuf_make_new, 0); MP_OBJ_TYPE_SET_SLOT(&mp_type_framebuf, buffer, framebuf_get_buffer, 1); - framebuf_locals_dict_table[0] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_fill), MP_OBJ_FROM_PTR(&framebuf_fill_obj) }; - framebuf_locals_dict_table[1] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_fill_rect), MP_OBJ_FROM_PTR(&framebuf_fill_rect_obj) }; - framebuf_locals_dict_table[2] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_pixel), MP_OBJ_FROM_PTR(&framebuf_pixel_obj) }; - framebuf_locals_dict_table[3] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_hline), MP_OBJ_FROM_PTR(&framebuf_hline_obj) }; - framebuf_locals_dict_table[4] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_vline), MP_OBJ_FROM_PTR(&framebuf_vline_obj) }; - framebuf_locals_dict_table[5] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_rect), MP_OBJ_FROM_PTR(&framebuf_rect_obj) }; - framebuf_locals_dict_table[6] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_line), MP_OBJ_FROM_PTR(&framebuf_line_obj) }; - framebuf_locals_dict_table[7] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_ellipse), MP_OBJ_FROM_PTR(&framebuf_ellipse_obj) }; - framebuf_locals_dict_table[8] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_poly), MP_OBJ_FROM_PTR(&framebuf_poly_obj) }; - framebuf_locals_dict_table[9] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_blit), MP_OBJ_FROM_PTR(&framebuf_blit_obj) }; - framebuf_locals_dict_table[10] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_scroll), MP_OBJ_FROM_PTR(&framebuf_scroll_obj) }; - framebuf_locals_dict_table[11] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_text), MP_OBJ_FROM_PTR(&framebuf_text_obj) }; - MP_OBJ_TYPE_SET_SLOT(&mp_type_framebuf, locals_dict, (void*)&framebuf_locals_dict, 2); + framebuf_locals_dict_table[0] = (mp_map_elem_t) { MP_OBJ_NEW_QSTR(MP_QSTR_fill), MP_OBJ_FROM_PTR(&framebuf_fill_obj) }; + framebuf_locals_dict_table[1] = (mp_map_elem_t) { MP_OBJ_NEW_QSTR(MP_QSTR_fill_rect), MP_OBJ_FROM_PTR(&framebuf_fill_rect_obj) }; + framebuf_locals_dict_table[2] = (mp_map_elem_t) { MP_OBJ_NEW_QSTR(MP_QSTR_pixel), MP_OBJ_FROM_PTR(&framebuf_pixel_obj) }; + framebuf_locals_dict_table[3] = (mp_map_elem_t) { MP_OBJ_NEW_QSTR(MP_QSTR_hline), MP_OBJ_FROM_PTR(&framebuf_hline_obj) }; + framebuf_locals_dict_table[4] = (mp_map_elem_t) { MP_OBJ_NEW_QSTR(MP_QSTR_vline), MP_OBJ_FROM_PTR(&framebuf_vline_obj) }; + framebuf_locals_dict_table[5] = (mp_map_elem_t) { MP_OBJ_NEW_QSTR(MP_QSTR_rect), MP_OBJ_FROM_PTR(&framebuf_rect_obj) }; + framebuf_locals_dict_table[6] = (mp_map_elem_t) { MP_OBJ_NEW_QSTR(MP_QSTR_line), MP_OBJ_FROM_PTR(&framebuf_line_obj) }; + framebuf_locals_dict_table[7] = (mp_map_elem_t) { MP_OBJ_NEW_QSTR(MP_QSTR_ellipse), MP_OBJ_FROM_PTR(&framebuf_ellipse_obj) }; + framebuf_locals_dict_table[8] = (mp_map_elem_t) { MP_OBJ_NEW_QSTR(MP_QSTR_poly), MP_OBJ_FROM_PTR(&framebuf_poly_obj) }; + framebuf_locals_dict_table[9] = (mp_map_elem_t) { MP_OBJ_NEW_QSTR(MP_QSTR_blit), MP_OBJ_FROM_PTR(&framebuf_blit_obj) }; + framebuf_locals_dict_table[10] = (mp_map_elem_t) { MP_OBJ_NEW_QSTR(MP_QSTR_scroll), MP_OBJ_FROM_PTR(&framebuf_scroll_obj) }; + framebuf_locals_dict_table[11] = (mp_map_elem_t) { MP_OBJ_NEW_QSTR(MP_QSTR_text), MP_OBJ_FROM_PTR(&framebuf_text_obj) }; + MP_OBJ_TYPE_SET_SLOT(&mp_type_framebuf, locals_dict, (void *)&framebuf_locals_dict, 2); mp_store_global(MP_QSTR_FrameBuffer, MP_OBJ_FROM_PTR(&mp_type_framebuf)); mp_store_global(MP_QSTR_MVLSB, MP_OBJ_NEW_SMALL_INT(FRAMEBUF_MVLSB)); diff --git a/examples/natmod/heapq/Makefile b/examples/natmod/heapq/Makefile index 61e2fc8fcc0..7068b55dfd0 100644 --- a/examples/natmod/heapq/Makefile +++ b/examples/natmod/heapq/Makefile @@ -2,7 +2,8 @@ MPY_DIR = ../../.. # Name of module (different to built-in heapq so it can coexist) -MOD = heapq_$(ARCH) +MOD_BASE = heapq +MOD = $(MOD_BASE)_$(ARCH) # Source files (.c or .py) SRC = heapq.c @@ -10,4 +11,7 @@ SRC = heapq.c # Architecture to build for (x86, x64, armv7m, xtensa, xtensawin, rv32imc) ARCH = x64 +# Strip the architecture name from the internal filename. +MPY_LD_FLAGS = "--source-name=$(MOD_BASE).mpy" + include $(MPY_DIR)/py/dynruntime.mk diff --git a/examples/natmod/random/Makefile b/examples/natmod/random/Makefile index 8abdb66dc87..ae0970604b3 100644 --- a/examples/natmod/random/Makefile +++ b/examples/natmod/random/Makefile @@ -2,7 +2,8 @@ MPY_DIR = ../../.. # Name of module (different to built-in random so it can coexist) -MOD = random_$(ARCH) +MOD_BASE = random +MOD = $(MOD_BASE)_$(ARCH) # Source files (.c or .py) SRC = random.c @@ -10,4 +11,7 @@ SRC = random.c # Architecture to build for (x86, x64, armv7m, xtensa, xtensawin, rv32imc) ARCH = x64 +# Strip the architecture name from the internal filename. +MPY_LD_FLAGS = "--source-name=$(MOD_BASE).mpy" + include $(MPY_DIR)/py/dynruntime.mk diff --git a/examples/natmod/random/random.c b/examples/natmod/random/random.c index 92257b8bc68..d73b53e2e7d 100644 --- a/examples/natmod/random/random.c +++ b/examples/natmod/random/random.c @@ -12,7 +12,7 @@ uint8_t yasmarang_dat; mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) { MP_DYNRUNTIME_INIT_ENTRY - yasmarang_pad = 0xeda4baba; + yasmarang_pad = 0xeda4baba; yasmarang_n = 69; yasmarang_d = 233; diff --git a/examples/natmod/re/Makefile b/examples/natmod/re/Makefile index 56b08b98868..2596437546d 100644 --- a/examples/natmod/re/Makefile +++ b/examples/natmod/re/Makefile @@ -2,7 +2,8 @@ MPY_DIR = ../../.. # Name of module (different to built-in re so it can coexist) -MOD = re_$(ARCH) +MOD_BASE = re +MOD = $(MOD_BASE)_$(ARCH) # Source files (.c or .py) SRC = re.c @@ -10,4 +11,19 @@ SRC = re.c # Architecture to build for (x86, x64, armv7m, xtensa, xtensawin, rv32imc) ARCH = x64 +ifeq ($(ARCH),armv7m) +ifeq ($(findstring clang,$(shell $(CC) --version)),clang) +# Link with libclang_rt.a for memmove +LINK_RUNTIME = 1 +endif +endif + +ifeq ($(ARCH),armv6m) +# Link with libgcc.a for division helper functions +LINK_RUNTIME = 1 +endif + +# Strip the architecture name from the internal filename. +MPY_LD_FLAGS = "--source-name=$(MOD_BASE).mpy" + include $(MPY_DIR)/py/dynruntime.mk diff --git a/examples/natmod/re/re.c b/examples/natmod/re/re.c index c0279ee7e81..6c5c65842d6 100644 --- a/examples/natmod/re/re.c +++ b/examples/natmod/re/re.c @@ -32,19 +32,6 @@ void mp_cstack_check(void) { } } -#if !defined(__linux__) -void *memcpy(void *dst, const void *src, size_t n) { - return mp_fun_table.memmove_(dst, src, n); -} -void *memset(void *s, int c, size_t n) { - return mp_fun_table.memset_(s, c, n); -} -#endif - -void *memmove(void *dest, const void *src, size_t n) { - return mp_fun_table.memmove_(dest, src, n); -} - mp_obj_full_type_t match_type; mp_obj_full_type_t re_type; @@ -65,23 +52,23 @@ mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *a // Because MP_QSTR_start/end/split are static, xtensa and xtensawin will make a small data section // to copy in this key/value pair if they are specified as a struct, so assign them separately. - match_type.base.type = (void*)&mp_fun_table.type_type; + match_type.base.type = (void *)&mp_fun_table.type_type; match_type.name = MP_QSTR_match; MP_OBJ_TYPE_SET_SLOT(&match_type, print, match_print, 0); - match_locals_dict_table[0] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_group), MP_OBJ_FROM_PTR(&match_group_obj) }; - match_locals_dict_table[1] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_groups), MP_OBJ_FROM_PTR(&match_groups_obj) }; - match_locals_dict_table[2] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_span), MP_OBJ_FROM_PTR(&match_span_obj) }; - match_locals_dict_table[3] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_start), MP_OBJ_FROM_PTR(&match_start_obj) }; - match_locals_dict_table[4] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_end), MP_OBJ_FROM_PTR(&match_end_obj) }; - MP_OBJ_TYPE_SET_SLOT(&match_type, locals_dict, (void*)&match_locals_dict, 1); - - re_type.base.type = (void*)&mp_fun_table.type_type; + match_locals_dict_table[0] = (mp_map_elem_t) { MP_OBJ_NEW_QSTR(MP_QSTR_group), MP_OBJ_FROM_PTR(&match_group_obj) }; + match_locals_dict_table[1] = (mp_map_elem_t) { MP_OBJ_NEW_QSTR(MP_QSTR_groups), MP_OBJ_FROM_PTR(&match_groups_obj) }; + match_locals_dict_table[2] = (mp_map_elem_t) { MP_OBJ_NEW_QSTR(MP_QSTR_span), MP_OBJ_FROM_PTR(&match_span_obj) }; + match_locals_dict_table[3] = (mp_map_elem_t) { MP_OBJ_NEW_QSTR(MP_QSTR_start), MP_OBJ_FROM_PTR(&match_start_obj) }; + match_locals_dict_table[4] = (mp_map_elem_t) { MP_OBJ_NEW_QSTR(MP_QSTR_end), MP_OBJ_FROM_PTR(&match_end_obj) }; + MP_OBJ_TYPE_SET_SLOT(&match_type, locals_dict, (void *)&match_locals_dict, 1); + + re_type.base.type = (void *)&mp_fun_table.type_type; re_type.name = MP_QSTR_re; MP_OBJ_TYPE_SET_SLOT(&re_type, print, re_print, 0); - re_locals_dict_table[0] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_match), MP_OBJ_FROM_PTR(&re_match_obj) }; - re_locals_dict_table[1] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_search), MP_OBJ_FROM_PTR(&re_search_obj) }; - re_locals_dict_table[2] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_split), MP_OBJ_FROM_PTR(&re_split_obj) }; - MP_OBJ_TYPE_SET_SLOT(&re_type, locals_dict, (void*)&re_locals_dict, 1); + re_locals_dict_table[0] = (mp_map_elem_t) { MP_OBJ_NEW_QSTR(MP_QSTR_match), MP_OBJ_FROM_PTR(&re_match_obj) }; + re_locals_dict_table[1] = (mp_map_elem_t) { MP_OBJ_NEW_QSTR(MP_QSTR_search), MP_OBJ_FROM_PTR(&re_search_obj) }; + re_locals_dict_table[2] = (mp_map_elem_t) { MP_OBJ_NEW_QSTR(MP_QSTR_split), MP_OBJ_FROM_PTR(&re_split_obj) }; + MP_OBJ_TYPE_SET_SLOT(&re_type, locals_dict, (void *)&re_locals_dict, 1); mp_store_global(MP_QSTR_compile, MP_OBJ_FROM_PTR(&mod_re_compile_obj)); mp_store_global(MP_QSTR_match, MP_OBJ_FROM_PTR(&re_match_obj)); diff --git a/extmod/modplatform.h b/extmod/modplatform.h index a155f071cba..2417a007b3a 100644 --- a/extmod/modplatform.h +++ b/extmod/modplatform.h @@ -53,6 +53,8 @@ #else #define MICROPY_PLATFORM_ARCH "riscv" #endif +#elif defined(__loongarch__) && defined(__loongarch64) +#define MICROPY_PLATFORM_ARCH "loongarch64" #else #define MICROPY_PLATFORM_ARCH "" #endif @@ -103,6 +105,9 @@ #elif defined(__ANDROID__) #define MICROPY_PLATFORM_LIBC_LIB "bionic" #define MICROPY_PLATFORM_LIBC_VER MP_STRINGIFY(__ANDROID_API__) +#elif defined(__FreeBSD__) +#define MICROPY_PLATFORM_LIBC_LIB "libc" +#define MICROPY_PLATFORM_LIBC_VER "" #else #define MICROPY_PLATFORM_LIBC_LIB "" #define MICROPY_PLATFORM_LIBC_VER "" @@ -112,6 +117,8 @@ #define MICROPY_PLATFORM_SYSTEM "Android" #elif defined(__linux) #define MICROPY_PLATFORM_SYSTEM "Linux" +#elif defined(__FreeBSD__) +#define MICROPY_PLATFORM_SYSTEM "FreeBSD" #elif defined(__unix__) #define MICROPY_PLATFORM_SYSTEM "Unix" #elif defined(__CYGWIN__) diff --git a/extmod/vfs.c b/extmod/vfs.c index c4156990bcc..2379e19fa0b 100644 --- a/extmod/vfs.c +++ b/extmod/vfs.c @@ -178,7 +178,9 @@ static mp_obj_t mp_vfs_autodetect(mp_obj_t bdev_obj) { mp_vfs_blockdev_init(&blockdev, bdev_obj); uint8_t buf[44]; for (size_t block_num = 0; block_num <= 1; ++block_num) { - mp_vfs_blockdev_read_ext(&blockdev, block_num, 8, sizeof(buf), buf); + if (mp_vfs_blockdev_read_ext(&blockdev, block_num, 8, sizeof(buf), buf) != 0) { + continue; + } #if MICROPY_VFS_LFS1 if (memcmp(&buf[32], "littlefs", 8) == 0) { // LFS1 diff --git a/extmod/vfs.h b/extmod/vfs.h index 699232e56cd..8c452751f32 100644 --- a/extmod/vfs.h +++ b/extmod/vfs.h @@ -41,7 +41,9 @@ #define MP_S_IFREG (0x8000) // these are the values for mp_vfs_blockdev_t.flags +#if MICROPY_VFS_BLOCKDEV_NATIVE #define MP_BLOCKDEV_FLAG_NATIVE (0x0001) // readblocks[2]/writeblocks[2] contain native func +#endif #define MP_BLOCKDEV_FLAG_FREE_OBJ (0x0002) // fs_user_mount_t obj should be freed on umount #define MP_BLOCKDEV_FLAG_HAVE_IOCTL (0x0004) // new protocol with ioctl #define MP_BLOCKDEV_FLAG_NO_FILESYSTEM (0x0008) // the block device has no filesystem on it @@ -64,11 +66,20 @@ #define MP_BLOCKDEV_IOCTL_BLOCK_ERASE (6) // Constants for vfs.rom_ioctl() function. +// The 4-arg form of WRITE_PREPARE is only available if GET_MIN_PREPARE returns >0. #define MP_VFS_ROM_IOCTL_GET_NUMBER_OF_SEGMENTS (1) // rom_ioctl(1) #define MP_VFS_ROM_IOCTL_GET_SEGMENT (2) // rom_ioctl(2, ) -#define MP_VFS_ROM_IOCTL_WRITE_PREPARE (3) // rom_ioctl(3, , ) +#define MP_VFS_ROM_IOCTL_WRITE_PREPARE (3) // rom_ioctl(3, , ) or rom_ioctl(3, , , ) #define MP_VFS_ROM_IOCTL_WRITE (4) // rom_ioctl(4, , , ) #define MP_VFS_ROM_IOCTL_WRITE_COMPLETE (5) // rom_ioctl(5, ) +#define MP_VFS_ROM_IOCTL_GET_MIN_PREPARE (6) // rom_ioctl(6, ) + +#if MICROPY_VFS_BLOCKDEV_NATIVE +// Function signatures used when MP_BLOCKDEV_FLAG_NATIVE is set. +// Should return 0 for success, or a negative errno code for failure. +typedef int (*mp_vfs_blockdev_native_readblocks)(uint8_t *, uint32_t, uint32_t); +typedef int (*mp_vfs_blockdev_native_writeblocks)(const uint8_t *, uint32_t, uint32_t); +#endif // At the moment the VFS protocol just has import_stat, but could be extended to other methods typedef struct _mp_vfs_proto_t { diff --git a/extmod/vfs_blockdev.c b/extmod/vfs_blockdev.c index 2b47db07573..db6fc28e2ec 100644 --- a/extmod/vfs_blockdev.c +++ b/extmod/vfs_blockdev.c @@ -111,7 +111,11 @@ void mp_vfs_blockdev_init(mp_vfs_blockdev_t *self, mp_obj_t bdev) { // Helper function to minimise code size of read/write functions // note the n_args argument is moved to the end for further code size reduction (args keep same position in caller and callee). static int mp_vfs_blockdev_call_rw(mp_obj_t *args, size_t block_num, size_t block_off, size_t len, void *buf, size_t n_args) { + #if MICROPY_PY_BUILTINS_MEMORYVIEW + mp_obj_array_t ar = {{&mp_type_memoryview}, 'B' | MP_OBJ_ARRAY_TYPECODE_FLAG_RW, 0, len, buf}; + #else mp_obj_array_t ar = {{&mp_type_bytearray}, BYTEARRAY_TYPECODE, 0, len, buf}; + #endif args[2] = MP_OBJ_NEW_SMALL_INT(block_num); args[3] = MP_OBJ_FROM_PTR(&ar); args[4] = MP_OBJ_NEW_SMALL_INT(block_off); // ignored for n_args == 2 @@ -120,13 +124,6 @@ static int mp_vfs_blockdev_call_rw(mp_obj_t *args, size_t block_num, size_t bloc if (ret == mp_const_none) { return 0; } else { - // Some block devices return a bool indicating success, so - // convert those to an errno integer code. - if (ret == mp_const_true) { - return 0; - } else if (ret == mp_const_false) { - return -MP_EIO; - } // Block device functions are expected to return 0 on success // and negative integer on errors. Check for positive integer // results as some callers (i.e. littlefs) will produce corrupt @@ -137,14 +134,16 @@ static int mp_vfs_blockdev_call_rw(mp_obj_t *args, size_t block_num, size_t bloc } int mp_vfs_blockdev_read(mp_vfs_blockdev_t *self, size_t block_num, size_t num_blocks, uint8_t *buf) { + #if MICROPY_VFS_BLOCKDEV_NATIVE if (self->flags & MP_BLOCKDEV_FLAG_NATIVE) { // CIRCUITPY-CHANGE: Pass the blockdev object into native readblocks so // it has the corresponding state. mp_uint_t (*f)(mp_obj_t self, uint8_t *, uint32_t, uint32_t) = (void *)(uintptr_t)self->readblocks[2]; return f(self->readblocks[1], buf, block_num, num_blocks); - } else { - return mp_vfs_blockdev_call_rw(self->readblocks, block_num, 0, num_blocks * self->block_size, buf, 2); } + #endif + + return mp_vfs_blockdev_call_rw(self->readblocks, block_num, 0, num_blocks * self->block_size, buf, 2); } int mp_vfs_blockdev_read_ext(mp_vfs_blockdev_t *self, size_t block_num, size_t block_off, size_t len, uint8_t *buf) { @@ -157,14 +156,16 @@ int mp_vfs_blockdev_write(mp_vfs_blockdev_t *self, size_t block_num, size_t num_ return -MP_EROFS; } + #if MICROPY_VFS_BLOCKDEV_NATIVE if (self->flags & MP_BLOCKDEV_FLAG_NATIVE) { - // CIRCUITPY-CHANGE: Pass the blockdev object into native readblocks so + // CIRCUITPY-CHANGE: Pass the blockdev object into native writeblocks so // it has the corresponding state. mp_uint_t (*f)(mp_obj_t self, const uint8_t *, uint32_t, uint32_t) = (void *)(uintptr_t)self->writeblocks[2]; return f(self->writeblocks[1], buf, block_num, num_blocks); - } else { - return mp_vfs_blockdev_call_rw(self->writeblocks, block_num, 0, num_blocks * self->block_size, (void *)buf, 2); } + #endif + + return mp_vfs_blockdev_call_rw(self->writeblocks, block_num, 0, num_blocks * self->block_size, (void *)buf, 2); } int mp_vfs_blockdev_write_ext(mp_vfs_blockdev_t *self, size_t block_num, size_t block_off, size_t len, const uint8_t *buf) { @@ -178,6 +179,7 @@ int mp_vfs_blockdev_write_ext(mp_vfs_blockdev_t *self, size_t block_num, size_t mp_obj_t mp_vfs_blockdev_ioctl(mp_vfs_blockdev_t *self, uintptr_t cmd, uintptr_t arg) { if (self->flags & MP_BLOCKDEV_FLAG_HAVE_IOCTL) { // CIRCUITPY-CHANGE: Support native IOCTL so it can run outside of the VM. + #if MICROPY_VFS_BLOCKDEV_NATIVE if (self->flags & MP_BLOCKDEV_FLAG_NATIVE) { size_t out_value; bool (*f)(mp_obj_t self, uint32_t, uint32_t, size_t *) = (void *)(uintptr_t)self->u.ioctl[2]; @@ -187,6 +189,7 @@ mp_obj_t mp_vfs_blockdev_ioctl(mp_vfs_blockdev_t *self, uintptr_t cmd, uintptr_t } return MP_OBJ_NEW_SMALL_INT(out_value); } + #endif // New protocol with ioctl self->u.ioctl[2] = MP_OBJ_NEW_SMALL_INT(cmd); self->u.ioctl[3] = MP_OBJ_NEW_SMALL_INT(arg); diff --git a/extmod/vfs_fat_file.c b/extmod/vfs_fat_file.c index 2a3021ee9a8..aee6318cacf 100644 --- a/extmod/vfs_fat_file.c +++ b/extmod/vfs_fat_file.c @@ -102,15 +102,15 @@ static mp_uint_t file_obj_ioctl(mp_obj_t o_in, mp_uint_t request, uintptr_t arg, struct mp_stream_seek_t *s = (struct mp_stream_seek_t *)(uintptr_t)arg; switch (s->whence) { - case 0: // SEEK_SET + case MP_SEEK_SET: f_lseek(&self->fp, s->offset); break; - case 1: // SEEK_CUR + case MP_SEEK_CUR: f_lseek(&self->fp, f_tell(&self->fp) + s->offset); break; - case 2: // SEEK_END + case MP_SEEK_END: f_lseek(&self->fp, f_size(&self->fp) + s->offset); break; } diff --git a/extmod/vfs_posix.c b/extmod/vfs_posix.c index 27f833e802f..90030db1965 100644 --- a/extmod/vfs_posix.c +++ b/extmod/vfs_posix.c @@ -341,10 +341,14 @@ static mp_obj_t vfs_posix_rename(mp_obj_t self_in, mp_obj_t old_path_in, mp_obj_ vfs_posix_require_writable(self_in); mp_obj_vfs_posix_t *self = MP_OBJ_TO_PTR(self_in); const char *old_path = vfs_posix_get_path_str(self, old_path_in); + size_t old_path_len = strlen(old_path) + 1; + char *old_path_copy = m_new(char, old_path_len); + memcpy(old_path_copy, old_path, old_path_len); const char *new_path = vfs_posix_get_path_str(self, new_path_in); MP_THREAD_GIL_EXIT(); - int ret = rename(old_path, new_path); + int ret = rename(old_path_copy, new_path); MP_THREAD_GIL_ENTER(); + m_del(char, old_path_copy, old_path_len); if (ret != 0) { mp_raise_OSError(errno); } diff --git a/extmod/vfs_rom_file.c b/extmod/vfs_rom_file.c index 57aca8c5dc7..6d7c47111f5 100644 --- a/extmod/vfs_rom_file.c +++ b/extmod/vfs_rom_file.c @@ -110,11 +110,11 @@ static mp_uint_t vfs_rom_file_ioctl(mp_obj_t o_in, mp_uint_t request, uintptr_t switch (request) { case MP_STREAM_SEEK: { struct mp_stream_seek_t *s = (struct mp_stream_seek_t *)arg; - if (s->whence == 0) { // SEEK_SET + if (s->whence == MP_SEEK_SET) { self->file_offset = (size_t)s->offset; - } else if (s->whence == 1) { // SEEK_CUR + } else if (s->whence == MP_SEEK_CUR) { self->file_offset += s->offset; - } else { // SEEK_END + } else { // MP_SEEK_END self->file_offset = self->file_size + s->offset; } if (self->file_offset > self->file_size) { diff --git a/lib/littlefs/lfs2.c b/lib/littlefs/lfs2.c index 8d18b7d1105..8c65397384b 100644 --- a/lib/littlefs/lfs2.c +++ b/lib/littlefs/lfs2.c @@ -258,7 +258,7 @@ static int lfs2_bd_prog(lfs2_t *lfs2, continue; } - // pcache must have been flushed, either by programming and + // pcache must have been flushed, either by programming an // entire block or manually flushing the pcache LFS2_ASSERT(pcache->block == LFS2_BLOCK_NULL); @@ -286,7 +286,7 @@ static int lfs2_bd_erase(lfs2_t *lfs2, lfs2_block_t block) { // some operations on paths static inline lfs2_size_t lfs2_path_namelen(const char *path) { - return strcspn(path, "/"); + return (lfs2_size_t)strcspn(path, "/"); } static inline bool lfs2_path_islast(const char *path) { @@ -1291,6 +1291,7 @@ static lfs2_stag_t lfs2_dir_fetchmatch(lfs2_t *lfs2, // found a match for our fetcher? if ((fmask & tag) == (fmask & ftag)) { + LFS2_ASSERT(cb != NULL); int res = cb(data, tag, &(struct lfs2_diskoff){ dir->pair[0], off+sizeof(tag)}); if (res < 0) { @@ -1501,7 +1502,7 @@ static lfs2_stag_t lfs2_dir_find(lfs2_t *lfs2, lfs2_mdir_t *dir, if (lfs2_tag_type3(tag) == LFS2_TYPE_DIR) { name += strspn(name, "/"); } - lfs2_size_t namelen = strcspn(name, "/"); + lfs2_size_t namelen = (lfs2_size_t)strcspn(name, "/"); // skip '.' if (namelen == 1 && memcmp(name, ".", 1) == 0) { @@ -1520,7 +1521,7 @@ static lfs2_stag_t lfs2_dir_find(lfs2_t *lfs2, lfs2_mdir_t *dir, int depth = 1; while (true) { suffix += strspn(suffix, "/"); - sufflen = strcspn(suffix, "/"); + sufflen = (lfs2_size_t)strcspn(suffix, "/"); if (sufflen == 0) { break; } @@ -1761,7 +1762,7 @@ static int lfs2_dir_commitcrc(lfs2_t *lfs2, struct lfs2_commit *commit) { commit->off = noff; // perturb valid bit? - commit->ptag = ntag ^ ((0x80UL & ~eperturb) << 24); + commit->ptag = ntag ^ ((lfs2_tag_t)(0x80 & ~eperturb) << 24); // reset crc for next commit commit->crc = 0xffffffff; @@ -3244,10 +3245,12 @@ static int lfs2_file_open_(lfs2_t *lfs2, lfs2_file_t *file, #endif static int lfs2_file_close_(lfs2_t *lfs2, lfs2_file_t *file) { -#ifndef LFS2_READONLY - int err = lfs2_file_sync_(lfs2, file); -#else int err = 0; +#ifndef LFS2_READONLY + // it's not safe to do anything if our file errored + if (!(file->flags & LFS2_F_ERRED)) { + err = lfs2_file_sync_(lfs2, file); + } #endif // remove from list of mdirs @@ -3429,18 +3432,12 @@ static int lfs2_file_flush(lfs2_t *lfs2, lfs2_file_t *file) { #ifndef LFS2_READONLY static int lfs2_file_sync_(lfs2_t *lfs2, lfs2_file_t *file) { - if (file->flags & LFS2_F_ERRED) { - // it's not safe to do anything if our file errored - return 0; - } - int err = lfs2_file_flush(lfs2, file); if (err) { file->flags |= LFS2_F_ERRED; return err; } - if ((file->flags & LFS2_F_DIRTY) && !lfs2_pair_isnull(file->m.pair)) { // before we commit metadata, we need sync the disk to make sure @@ -3485,6 +3482,17 @@ static int lfs2_file_sync_(lfs2_t *lfs2, lfs2_file_t *file) { file->flags &= ~LFS2_F_DIRTY; } + // mark any other file handles as dirty + desync + for (lfs2_file_t *f = (lfs2_file_t*)lfs2->mlist; f; f = f->next) { + if (file != f + && f->type == LFS2_TYPE_REG + && lfs2_pair_cmp(f->m.pair, file->m.pair) == 0 + && f->id == file->id) { + f->flags |= LFS2_F_DUSTY; + } + } + + file->flags &= ~LFS2_F_ERRED & ~LFS2_F_DUSTY; return 0; } #endif @@ -3692,7 +3700,7 @@ static lfs2_ssize_t lfs2_file_write_(lfs2_t *lfs2, lfs2_file_t *file, return nsize; } - file->flags &= ~LFS2_F_ERRED; + file->flags &= ~LFS2_F_ERRED & ~LFS2_F_DUSTY; return nsize; } #endif @@ -4771,7 +4779,8 @@ int lfs2_fs_traverse_(lfs2_t *lfs2, continue; } - if ((f->flags & LFS2_F_DIRTY) && !(f->flags & LFS2_F_INLINE)) { + if (((f->flags & LFS2_F_DIRTY) || (f->flags & LFS2_F_DUSTY)) + && !(f->flags & LFS2_F_INLINE)) { int err = lfs2_ctz_traverse(lfs2, &f->cache, &lfs2->rcache, f->ctz.head, f->ctz.size, cb, data); if (err) { diff --git a/lib/littlefs/lfs2.h b/lib/littlefs/lfs2.h index aee0619e932..53f2a1569ee 100644 --- a/lib/littlefs/lfs2.h +++ b/lib/littlefs/lfs2.h @@ -135,14 +135,15 @@ enum lfs2_open_flags { // internally used flags #ifndef LFS2_READONLY - LFS2_F_DIRTY = 0x010000, // File does not match storage - LFS2_F_WRITING = 0x020000, // File has been written since last flush + LFS2_F_DIRTY = 0x00010000, // File does not match storage due to write + LFS2_F_DUSTY = 0x00020000, // File does not match storage due to desync + LFS2_F_WRITING = 0x00040000, // File has been written since last flush #endif - LFS2_F_READING = 0x040000, // File has been read since last flush + LFS2_F_READING = 0x00080000, // File has been read since last flush #ifndef LFS2_READONLY - LFS2_F_ERRED = 0x080000, // An error occurred during write + LFS2_F_ERRED = 0x00100000, // An error occurred during write #endif - LFS2_F_INLINE = 0x100000, // Currently inlined in directory entry + LFS2_F_INLINE = 0x01000000, // Currently inlined in directory entry }; // File seek flags diff --git a/lib/micropython-lib b/lib/micropython-lib index 6ae440a8a14..ee4bb8ff139 160000 --- a/lib/micropython-lib +++ b/lib/micropython-lib @@ -1 +1 @@ -Subproject commit 6ae440a8a144233e6e703f6759b7e7a0afaa37a4 +Subproject commit ee4bb8ff139e24c42b739935fbd8ec7c4d061e02 diff --git a/lib/re1.5/compilecode.c b/lib/re1.5/compilecode.c index 63ae1e02b8d..77a0b8ff2a6 100644 --- a/lib/re1.5/compilecode.c +++ b/lib/re1.5/compilecode.c @@ -50,6 +50,8 @@ static const char *_compilecode(const char *re, ByteProg *prog, int sizecode) int term = PC; int alt_label = 0; + re1_5_stack_chk(); + for (; *re && *re != ')'; re++) { switch (*re) { case '\\': diff --git a/locale/circuitpython.pot b/locale/circuitpython.pot index 36491a7e6d5..15d859bf49f 100644 --- a/locale/circuitpython.pot +++ b/locale/circuitpython.pot @@ -2407,6 +2407,10 @@ msgstr "" msgid "bad typecode" msgstr "" +#: py/binary.c +msgid "integer out of range" +msgstr "" + #: py/builtinevex.c msgid "bad compile mode" msgstr "" @@ -2525,6 +2529,10 @@ msgstr "" msgid "* arg after **" msgstr "" +#: py/compile.c +msgid "* arg after kwarg" +msgstr "" + #: py/compile.c msgid "too many args" msgstr "" @@ -2653,6 +2661,22 @@ msgstr "" msgid "opcode '%q' argument %d: must not be zero" msgstr "" +#: py/emitinlinerv32.c +msgid "opcode '%q': registers must be different" +msgstr "" + +#: py/emitinlinerv32.c +msgid "opcode '%q': malformed register list" +msgstr "" + +#: py/emitinlinerv32.c +msgid "opcode '%q': invalid stack adjustment" +msgstr "" + +#: py/emitinlinerv32.c +msgid "opcode '%q' argument %d: wrong register(s)" +msgstr "" + #: py/emitinlinerv32.c msgid "invalid RV32 instruction '%q'" msgstr "" @@ -2814,6 +2838,10 @@ msgstr "" msgid "return expected '%q' but got '%q'" msgstr "" +#: py/emitnative.c +msgid "native raise" +msgstr "" + #: py/emitnative.c msgid "must raise an object" msgstr "" @@ -2826,8 +2854,8 @@ msgstr "" msgid "unicode name escapes" msgstr "" -#: py/modbuiltins.c -msgid "chr() arg not in range(0x110000)" +#: py/modbuiltins.c py/objstr.c +msgid "char not in range(0x110000)" msgstr "" #: py/modbuiltins.c @@ -3130,11 +3158,6 @@ msgstr "" msgid "float too big" msgstr "" -#: py/objint.c -#, c-format -msgid "value must fit in %d byte(s)" -msgstr "" - #: py/objint.c shared-bindings/time/__init__.c msgid "No long integer support" msgstr "" @@ -3148,6 +3171,15 @@ msgstr "" msgid "%q=%q" msgstr "" +#: py/objint_impl.h +#, c-format +msgid "value would overflow a %d byte buffer" +msgstr "" + +#: py/objint_impl.h +msgid "can't convert negative int to unsigned" +msgstr "" + #: py/objint_longlong.c py/parsenum.c msgid "result overflows long long storage" msgstr "" @@ -3188,6 +3220,10 @@ msgstr "" msgid "Cannot subclass slice" msgstr "" +#: py/objstr.c +msgid "unknown encoding: %q" +msgstr "" + #: py/objstr.c msgid "bytes value out of range" msgstr "" diff --git a/mpy-cross/main.c b/mpy-cross/main.c index 8cdd124ef46..7d829e79e68 100644 --- a/mpy-cross/main.c +++ b/mpy-cross/main.c @@ -24,6 +24,7 @@ * THE SOFTWARE. */ +#include #include #include #include @@ -34,7 +35,7 @@ #include "py/persistentcode.h" #include "py/runtime.h" #include "py/gc.h" -#include "py/parsenum.h" +#include "py/parsenumbase.h" #include "genhdr/mpversion.h" #ifdef _WIN32 // CIRCUITPY-CHANGE @@ -49,7 +50,10 @@ static asm_rv32_backend_options_t rv32_options = { 0 }; // Command line options, with their defaults static uint emit_opt = MP_EMIT_OPT_NONE; -mp_uint_t mp_verbose_flag = 0; + +#if MICROPY_ENABLE_SOURCE_LINE +static bool include_source_lines = true; +#endif // Heap size of GC heap (if enabled) // Make it larger on a 64 bit machine, because pointers are larger. @@ -164,6 +168,12 @@ static int usage(char **argv) { " heapsize= -- set the heap size for the GC (default %ld)\n" , heap_size); impl_opts_cnt++; + #if MICROPY_ENABLE_SOURCE_LINE + printf( + " source-lines -- include source line numbers (default)\n" + " no-source-lines -- exclude source line numbers\n"); + impl_opts_cnt += 2; + #endif if (impl_opts_cnt == 0) { printf(" (none)\n"); @@ -188,6 +198,14 @@ static void pre_process_options(int argc, char **argv) { } else if (strcmp(argv[a + 1], "emit=viper") == 0) { emit_opt = MP_EMIT_OPT_VIPER; #endif + #if MICROPY_ENABLE_SOURCE_LINE + } else if (strcmp(argv[a + 1], "source-lines") == 0) { + // Allow excluding source lines for debug builds. + include_source_lines = true; + } else if (strcmp(argv[a + 1], "no-source-lines") == 0) { + // Allow excluding source lines for debug builds. + include_source_lines = false; + #endif } else if (strncmp(argv[a + 1], "heapsize=", sizeof("heapsize=") - 1) == 0) { char *end; heap_size = strtol(argv[a + 1] + sizeof("heapsize=") - 1, &end, 0); @@ -229,45 +247,35 @@ static char *backslash_to_forwardslash(char *path) { } // This will need to be reworked in case mpy-cross needs to set more bits than -// what its small int representation allows to fit in there. -static bool parse_integer(const char *value, mp_uint_t *integer) { +// what `unsigned long` can fit. +static bool parse_integer(const char *value, unsigned long *integer) { assert(value && "Attempting to parse a NULL string"); assert(integer && "Attempting to store into a NULL integer buffer"); size_t value_length = strlen(value); - int base = 10; - if (value_length > 2 && value[0] == '0') { - if ((value[1] | 0x20) == 'b') { - base = 2; - } else if ((value[1] | 0x20) == 'x') { - base = 16; - } else { - return false; - } + int base = 0; + size_t skip = mp_parse_num_base(value, value_length, &base); + // These can trip strtoul up. + if (base < 2 || value_length == skip || value[skip] == '+') { + return false; } - - bool valid = false; - nlr_buf_t nlr; - if (nlr_push(&nlr) == 0) { - mp_obj_t parsed = mp_parse_num_integer(value, value_length, base, NULL); - if (mp_obj_is_small_int(parsed)) { - *integer = MP_OBJ_SMALL_INT_VALUE(parsed); - valid = true; - } - nlr_pop(); + errno = 0; + char *end = NULL; + *integer = strtoul(value + skip, &end, base); + if (end != (value + value_length) || errno != 0) { + return false; } - - return valid; + return true; } #if MICROPY_EMIT_NATIVE && MICROPY_EMIT_RV32 -static bool parse_rv32_flags_string(const char *source, mp_uint_t *flags) { +static bool parse_rv32_flags_string(const char *source, unsigned long *flags) { assert(source && "Flag arguments string is NULL."); assert(flags && "Collected flags pointer is NULL."); const char *current = source; const char *end = source + strlen(source); - mp_uint_t collected_flags = 0; + unsigned long collected_flags = 0; while (current < end) { const char *separator = strchr(current, ','); if (separator == NULL) { @@ -312,6 +320,9 @@ MP_NOINLINE int main_(int argc, char **argv) { mp_dynamic_compiler.native_arch = MP_NATIVE_ARCH_NONE; mp_dynamic_compiler.nlr_buf_num_regs = 0; mp_dynamic_compiler.backend_options = NULL; + #if MICROPY_ENABLE_SOURCE_LINE + mp_dynamic_compiler.include_source_lines = include_source_lines; + #endif const char *input_file = NULL; const char *output_file = NULL; @@ -330,7 +341,7 @@ MP_NOINLINE int main_(int argc, char **argv) { "; mpy-cross emitting mpy v" MP_STRINGIFY(MPY_VERSION) "." MP_STRINGIFY(MPY_SUB_VERSION) "\n"); return 0; } else if (strcmp(argv[a], "-v") == 0) { - mp_verbose_flag++; + // This verbose option doesn't currently do anything. } else if (strncmp(argv[a], "-O", 2) == 0) { if (unichar_isdigit(argv[a][2])) { MP_STATE_VM(mp_optimise_value) = argv[a][2] & 0xf; @@ -441,10 +452,10 @@ MP_NOINLINE int main_(int argc, char **argv) { #if MICROPY_EMIT_NATIVE && MICROPY_EMIT_RV32 if (mp_dynamic_compiler.native_arch == MP_NATIVE_ARCH_RV32IMC) { mp_dynamic_compiler.backend_options = (void *)&rv32_options; - mp_uint_t raw_flags = 0; + unsigned long raw_flags = 0; if (parse_integer(arch_flags, &raw_flags) || parse_rv32_flags_string(arch_flags, &raw_flags)) { - if ((raw_flags & ~((mp_uint_t)RV32_EXT_ALL)) == 0) { - rv32_options.allowed_extensions = raw_flags; + if ((raw_flags & ~((unsigned long)RV32_EXT_ALL)) == 0) { + rv32_options.allowed_extensions = (uint8_t)raw_flags; processed = true; } } @@ -472,12 +483,6 @@ MP_NOINLINE int main_(int argc, char **argv) { int ret = compile_and_save(input_file, output_file, source_file); - #if MICROPY_PY_MICROPYTHON_MEM_INFO - if (mp_verbose_flag) { - mp_micropython_mem_info(0, NULL); - } - #endif - mp_deinit(); return ret & 0xff; diff --git a/mpy-cross/mpconfigport.h b/mpy-cross/mpconfigport.h index 56c5963fa9c..3f8e54d878b 100644 --- a/mpy-cross/mpconfigport.h +++ b/mpy-cross/mpconfigport.h @@ -88,9 +88,13 @@ #define MICROPY_PY_TSTRINGS (1) #define MICROPY_PY_BUILTINS_STR_UNICODE (1) -#if !(defined(MICROPY_GCREGS_SETJMP) || defined(__x86_64__) || defined(__i386__) || defined(__thumb2__) || defined(__thumb__) || defined(__arm__)) -// Fall back to setjmp() implementation for discovery of GC pointers in registers. -#define MICROPY_GCREGS_SETJMP (1) +// Fall back to setjmp() implementation for discovery of GC pointers in registers +// if running on intel-based macOS, or on architectures for which there is no +// specialised GC pointer discovery mechanism. +#if (defined(__APPLE__) && defined(__MACH__) && (defined(__i386__) || defined(__x86_64__))) || \ + (!(defined(MICROPY_GCREGS_SETJMP) || defined(__x86_64__) || defined(__i386__) || \ + defined(__thumb2__) || defined(__thumb__) || defined(__arm__))) +#define MICROPY_GCREGS_SETJMP (1) #endif #define MICROPY_MODULE___FILE__ (0) diff --git a/ports/atmel-samd/boards/adafruit_pixel_trinkey_m0/mpconfigboard.mk b/ports/atmel-samd/boards/adafruit_pixel_trinkey_m0/mpconfigboard.mk index 32db497b010..24df3a1fbe1 100644 --- a/ports/atmel-samd/boards/adafruit_pixel_trinkey_m0/mpconfigboard.mk +++ b/ports/atmel-samd/boards/adafruit_pixel_trinkey_m0/mpconfigboard.mk @@ -19,6 +19,7 @@ CIRCUITPY_PWMIO = 1 CIRCUITPY_ROTARYIO = 0 CIRCUITPY_RTC = 0 CIRCUITPY_TOUCHIO = 0 +CIRCUITPY_USB_MIDI = 0 CIRCUITPY_PIXELBUF = 1 diff --git a/ports/atmel-samd/boards/circuitbrains_deluxe_m4/mpconfigboard.mk b/ports/atmel-samd/boards/circuitbrains_deluxe_m4/mpconfigboard.mk index 5c1cab422cb..32793c20d2f 100755 --- a/ports/atmel-samd/boards/circuitbrains_deluxe_m4/mpconfigboard.mk +++ b/ports/atmel-samd/boards/circuitbrains_deluxe_m4/mpconfigboard.mk @@ -13,6 +13,7 @@ LONGINT_IMPL = MPZ CIRCUITPY_I2CTARGET = 0 CIRCUITPY_PS2IO = 1 CIRCUITPY_JPEGIO = 0 +CIRCUITPY_MSGPACK = 0 CIRCUITPY_SPITARGET = 0 CIRCUITPY_SYNTHIO = 0 CIRCUITPY_TILEPALETTEMAPPER = 0 diff --git a/ports/atmel-samd/boards/datalore_ip_m4/mpconfigboard.mk b/ports/atmel-samd/boards/datalore_ip_m4/mpconfigboard.mk index 2ad140094b3..14744513bbc 100644 --- a/ports/atmel-samd/boards/datalore_ip_m4/mpconfigboard.mk +++ b/ports/atmel-samd/boards/datalore_ip_m4/mpconfigboard.mk @@ -12,6 +12,7 @@ LONGINT_IMPL = MPZ CIRCUITPY_I2CTARGET = 0 CIRCUITPY_JPEGIO = 0 +CIRCUITPY_MSGPACK = 0 CIRCUITPY_SPITARGET = 0 CIRCUITPY_SYNTHIO = 0 CIRCUITPY_TILEPALETTEMAPPER = 0 diff --git a/ports/atmel-samd/boards/kicksat-sprite/mpconfigboard.mk b/ports/atmel-samd/boards/kicksat-sprite/mpconfigboard.mk index ebe9403c34d..9f7d1e0a39b 100644 --- a/ports/atmel-samd/boards/kicksat-sprite/mpconfigboard.mk +++ b/ports/atmel-samd/boards/kicksat-sprite/mpconfigboard.mk @@ -18,6 +18,7 @@ CIRCUITPY_AUDIOMIXER = 0 CIRCUITPY_AUDIOMP3 = 0 CIRCUITPY_BLEIO_HCI = 0 CIRCUITPY_DISPLAYIO = 0 +CIRCUITPY_ERRNO = 0 CIRCUITPY_FLOPPYIO = 0 CIRCUITPY_FRAMEBUFFERIO = 0 CIRCUITPY_PIXELMAP = 0 diff --git a/ports/atmel-samd/boards/openbook_m4/mpconfigboard.mk b/ports/atmel-samd/boards/openbook_m4/mpconfigboard.mk index 2701fd249f9..aae02a95325 100644 --- a/ports/atmel-samd/boards/openbook_m4/mpconfigboard.mk +++ b/ports/atmel-samd/boards/openbook_m4/mpconfigboard.mk @@ -14,6 +14,7 @@ CIRCUITPY_FLOPPYIO = 0 CIRCUITPY_I2CTARGET = 0 CIRCUITPY_JPEGIO = 0 CIRCUITPY_KEYPAD = 1 +CIRCUITPY_MSGPACK = 0 CIRCUITPY_SYNTHIO = 0 CIRCUITPY_TERMINALIO_VT100 = 0 CIRCUITPY_TILEPALETTEMAPPER = 0 diff --git a/ports/atmel-samd/boards/silicognition-m4-shim/mpconfigboard.mk b/ports/atmel-samd/boards/silicognition-m4-shim/mpconfigboard.mk index 99c447b7a9d..a8b0677e759 100644 --- a/ports/atmel-samd/boards/silicognition-m4-shim/mpconfigboard.mk +++ b/ports/atmel-samd/boards/silicognition-m4-shim/mpconfigboard.mk @@ -12,6 +12,7 @@ LONGINT_IMPL = MPZ CIRCUITPY_I2CTARGET = 0 CIRCUITPY_JPEGIO = 0 +CIRCUITPY_MSGPACK = 0 CIRCUITPY_SPITARGET = 0 CIRCUITPY_SYNTHIO = 0 CIRCUITPY_TILEPALETTEMAPPER = 0 diff --git a/ports/nordic/boards/bless_dev_board_multi_sensor/mpconfigboard.mk b/ports/nordic/boards/bless_dev_board_multi_sensor/mpconfigboard.mk index 90b0908505e..18dfd6e3e8f 100644 --- a/ports/nordic/boards/bless_dev_board_multi_sensor/mpconfigboard.mk +++ b/ports/nordic/boards/bless_dev_board_multi_sensor/mpconfigboard.mk @@ -6,3 +6,5 @@ USB_MANUFACTURER = "Switch Science, Inc." MCU_CHIP = nrf52840 INTERNAL_FLASH_FILESYSTEM = 1 + +CIRCUITPY_MSGPACK = 0 diff --git a/ports/nordic/boards/bluemicro840/mpconfigboard.mk b/ports/nordic/boards/bluemicro840/mpconfigboard.mk index 16d36704a3e..70b8e701a4c 100644 --- a/ports/nordic/boards/bluemicro840/mpconfigboard.mk +++ b/ports/nordic/boards/bluemicro840/mpconfigboard.mk @@ -6,3 +6,5 @@ USB_MANUFACTURER = "nrf52.jpconstantineau.com" MCU_CHIP = nrf52840 INTERNAL_FLASH_FILESYSTEM = 1 + +CIRCUITPY_MSGPACK = 0 diff --git a/ports/nordic/boards/electronut_labs_blip/mpconfigboard.mk b/ports/nordic/boards/electronut_labs_blip/mpconfigboard.mk index fc59a101e45..7e928181948 100644 --- a/ports/nordic/boards/electronut_labs_blip/mpconfigboard.mk +++ b/ports/nordic/boards/electronut_labs_blip/mpconfigboard.mk @@ -12,3 +12,4 @@ CIRCUITPY_AUDIOIO = 0 CIRCUITPY_DISPLAYIO = 1 CIRCUITPY_STAGE = 1 CIRCUITPY_DIGITALINOUT_PROTOCOL = 0 +CIRCUITPY_MSGPACK = 0 diff --git a/ports/nordic/boards/nice_nano/mpconfigboard.mk b/ports/nordic/boards/nice_nano/mpconfigboard.mk index 511a754e69a..6489102626a 100644 --- a/ports/nordic/boards/nice_nano/mpconfigboard.mk +++ b/ports/nordic/boards/nice_nano/mpconfigboard.mk @@ -6,3 +6,5 @@ USB_MANUFACTURER = "Nice Keyboards" MCU_CHIP = nrf52840 INTERNAL_FLASH_FILESYSTEM = 1 + +CIRCUITPY_MSGPACK = 0 diff --git a/ports/nordic/boards/pillbug/mpconfigboard.mk b/ports/nordic/boards/pillbug/mpconfigboard.mk index d917ba01578..d531c838e18 100644 --- a/ports/nordic/boards/pillbug/mpconfigboard.mk +++ b/ports/nordic/boards/pillbug/mpconfigboard.mk @@ -6,3 +6,5 @@ USB_MANUFACTURER = "Mechwild" MCU_CHIP = nrf52840 INTERNAL_FLASH_FILESYSTEM = 1 + +CIRCUITPY_MSGPACK = 0 diff --git a/ports/nordic/boards/supermini_nrf52840/mpconfigboard.mk b/ports/nordic/boards/supermini_nrf52840/mpconfigboard.mk index 699d19b7706..b4d8d78ed9c 100644 --- a/ports/nordic/boards/supermini_nrf52840/mpconfigboard.mk +++ b/ports/nordic/boards/supermini_nrf52840/mpconfigboard.mk @@ -6,3 +6,5 @@ USB_MANUFACTURER = "ICBbuy" MCU_CHIP = nrf52840 INTERNAL_FLASH_FILESYSTEM = 1 + +CIRCUITPY_MSGPACK = 0 diff --git a/ports/raspberrypi/Makefile b/ports/raspberrypi/Makefile index 5512c7c9f47..837584fdf58 100755 --- a/ports/raspberrypi/Makefile +++ b/ports/raspberrypi/Makefile @@ -748,11 +748,10 @@ endif $(patsubst %.S,$(BUILD)/%.o,$(SRC_S_UPPER)): CFLAGS += -Wno-undef -# Derive the C++ flags from the C flags (mirroring the unix port) so the -# vendored C++ translation unit picks up all the SDK include paths and defines. -# Strip the C-only options that g++ rejects, and disable exceptions/RTTI to -# match the firmware's constraints. -CXXFLAGS += $(filter-out -std=gnu11 -Werror=missing-prototypes -Wold-style-definition -Wstrict-prototypes -Werror-implicit-function-declaration -Wnested-externs,$(CFLAGS)) -std=gnu++17 -fno-exceptions -fno-rtti +# py/mkrules.mk derives CXXFLAGS from CFLAGS, which gives the vendored C++ +# translation unit the SDK include paths and defines. Only the C++-specific +# options belong here. +CXXFLAGS += -std=gnu++17 -fno-exceptions -fno-rtti OBJ = $(PY_O) $(SUPERVISOR_O) $(addprefix $(BUILD)/, $(SRC_C:.c=.o)) OBJ += $(addprefix $(BUILD)/, $(SRC_SDK:.c=.o)) diff --git a/ports/raspberrypi/common-hal/socketpool/Socket.c b/ports/raspberrypi/common-hal/socketpool/Socket.c index 840558d1ac3..357c1162f12 100644 --- a/ports/raspberrypi/common-hal/socketpool/Socket.c +++ b/ports/raspberrypi/common-hal/socketpool/Socket.c @@ -67,8 +67,6 @@ static mp_obj_t socketpool_ip_addr_and_port_to_tuple(const ip_addr_t *addr, int return mp_obj_new_tuple(n, args); } -#define MICROPY_PY_LWIP_SOCK_RAW (1) - #if 0 // print debugging info #define DEBUG_printf DEBUG_printf #else // don't print debugging info diff --git a/ports/raspberrypi/mpconfigport.h b/ports/raspberrypi/mpconfigport.h index 6abf9ea0547..7d47658e729 100644 --- a/ports/raspberrypi/mpconfigport.h +++ b/ports/raspberrypi/mpconfigport.h @@ -8,6 +8,10 @@ #include "hardware/platform_defs.h" +// CIRCUITPY-CHANGE: CircuitPython supports raw sockets through its own lwip +// integration, independently of MICROPY_PY_LWIP. +#define MICROPY_PY_LWIP_SOCK_RAW (1) + #if PICO_RP2040 #define MICROPY_PY_SYS_PLATFORM "RP2040" #endif diff --git a/ports/unix/Makefile b/ports/unix/Makefile index def45e2a405..62fa4c732e9 100644 --- a/ports/unix/Makefile +++ b/ports/unix/Makefile @@ -17,6 +17,10 @@ endif # If the build directory is not given, make it reflect the variant name. BUILD ?= build-$(VARIANT) +ifneq ($(MICROPY_FORCE_32BIT),) +$(warning *** The MICROPY_FORCE_32BIT flag is no longer affecting builds, please update your environment ***) +endif + include ../../py/mkenv.mk -include mpconfigport.mk include $(VARIANT_DIR)/mpconfigvariant.mk @@ -62,21 +66,25 @@ CFLAGS += -D_FILE_OFFSET_BITS=64 # This option has no effect on 64-bit builds. CFLAGS += -D_TIME_BITS=64 +# Debug info flag is added if either DEBUG=1 or STRIP= is passed to make +# (to produce a debug binary, or a release binary with all symbols, respectively.) +DEBUG_INFO = -ggdb3 + # Debugging/Optimization ifdef DEBUG COPT ?= -Og +CFLAGS += $(DEBUG_INFO) else COPT ?= -Os COPT += -DNDEBUG +ifndef STRIP +CFLAGS += $(DEBUG_INFO) endif +endif # ifdef DEBUG # Remove unused sections. COPT += -fdata-sections -ffunction-sections -# Note: Symbols and debug information will still be stripped from the final binary -# unless "DEBUG=1" or "STRIP=" is passed to make, see README.md for details. -CFLAGS += -g - ifndef DEBUG # _FORTIFY_SOURCE is a feature in gcc/glibc which is intended to provide extra # security for detecting buffer overflows. Some distros (Ubuntu at the very least) @@ -110,11 +118,7 @@ endif # while cross-compile ports require gcc, so we test here for OSX and # if necessary override the value of 'CC' set in py/mkenv.mk ifeq ($(UNAME_S),Darwin) -ifeq ($(MICROPY_FORCE_32BIT),1) -CC = clang -m32 -else CC = clang -endif # Use clang syntax for map file LDFLAGS_ARCH = -Wl,-map,$@.map -Wl,-dead_strip else @@ -126,16 +130,6 @@ LDFLAGS += $(LDFLAGS_MOD) $(LDFLAGS_ARCH) -lm $(LDFLAGS_EXTRA) # Flags to link with pthread library LIBPTHREAD = -lpthread -ifeq ($(MICROPY_FORCE_32BIT),1) -# Note: you may need to install i386 versions of dependency packages, -# starting with linux-libc-dev:i386 -ifeq ($(MICROPY_PY_FFI),1) -ifeq ($(UNAME_S),Linux) -CFLAGS += -I/usr/include/i686-linux-gnu -endif -endif -endif - ifeq ($(MICROPY_USE_READLINE),1) INC += -I$(TOP)/shared/readline CFLAGS += -DMICROPY_USE_READLINE=1 @@ -187,11 +181,7 @@ ifeq ($(MICROPY_STANDALONE),1) GIT_SUBMODULES += lib/libffi DEPLIBS += libffi LIBFFI_CFLAGS := -I$(shell ls -1d $(BUILD)/lib/libffi/include) - ifeq ($(MICROPY_FORCE_32BIT),1) - LIBFFI_LDFLAGS = $(BUILD)/lib/libffi/out/lib32/libffi.a - else - LIBFFI_LDFLAGS = $(BUILD)/lib/libffi/out/lib/libffi.a - endif +LIBFFI_LDFLAGS = $(BUILD)/lib/libffi/out/lib/libffi.a else # Use system version of libffi. LIBFFI_CFLAGS := $(shell pkg-config --cflags libffi) @@ -263,13 +253,6 @@ CFLAGS += -DMPZ_DIG_SIZE=16 # force 16 bits to work on both 32 and 64 bit archs CFLAGS += -DMICROPY_MODULE_FROZEN_STR endif -# CIRCUITPY-CHANGE: use gnu11 instead of gnu99 -CXXFLAGS += $(filter-out -Wmissing-prototypes -Wold-style-definition -std=gnu11,$(CFLAGS) $(CXXFLAGS_MOD)) - -ifeq ($(MICROPY_FORCE_32BIT),1) -RUN_TESTS_MPY_CROSS_FLAGS = --mpy-cross-flags='-march=x86' -endif - ifeq ($(CROSS_COMPILE),arm-linux-gnueabi-) # Force disable error text compression when compiling for ARM as the compiler # cannot optimise out the giant strcmp list generated for MP_MATCH_COMPRESSED. diff --git a/ports/unix/README.md b/ports/unix/README.md index ee983a882cc..720be8a047c 100644 --- a/ports/unix/README.md +++ b/ports/unix/README.md @@ -5,9 +5,10 @@ The "unix" port runs in standard Unix-like environments including Linux, BSD, macOS, and Windows Subsystem for Linux. The x86 and x64 architectures are supported (i.e. x86 32- and 64-bit), as well -as ARM and MIPS. Extending the unix port to another architecture requires -writing some assembly code for the exception handling and garbage collection. -Alternatively, a fallback implementation based on setjmp/longjmp can be used. +as ARM, MIPS, RISC-V, and LoongArch. Extending the unix port to another +architecture requires writing some assembly code for the exception handling +and garbage collection. Alternatively, a fallback implementation based on +setjmp/longjmp can be used. Building -------- @@ -18,7 +19,7 @@ To build the unix port locally then you will need: * git command line executable, unless you downloaded a source .tar.xz file from https://micropython.org/download/ -* gcc (or clang for macOS) toolchain +* an appropriate GCC or Clang toolchain for your target (macOS only supports Clang) * GNU Make * Python 3.x @@ -166,7 +167,7 @@ optimisations, assertions enabled, and debug symbols. ### Sanitizers -Sanitizers are extra runtime checks supported by gcc and clang. The CI process +Sanitizers are extra runtime checks supported by GCC and Clang. The CI process supports building with the "undefined behavior" (UBSan) or "address" (ASan) sanitizers. The script `tools/ci.sh` is the source of truth about how to build and run in these modes. @@ -181,3 +182,125 @@ Several classes of checks are disabled via compiler flags: check is intended to make sure locals in a "returned from" stack frame are not used. However, this mode interferes with various assumptions that MicroPython's stack checking, NLR, and GC rely on. + +### Notes about x86 (i686) support + +It used to be possible to create an x86 (32-bits) build on a x64 (64-bits) host +by passing `MICROPY_FORCE_32BIT=1` to `make`. That option was retired: x86 +usage has declined quite a bit in the past few years, and MicroPython now +supports at least one more mixed 32/64-bits target architecture for which +enabling `MICROPY_FORCE_32BIT` would make builds fail. + +x86 will be treated as a cross-compilation target from now on. This means you +will need to install a suitable compiler (on Ubuntu you can install either the +`gcc-i686-linux-gnu` and `g++-i686-linux-gnu` packages for GCC, or the `clang` +package for Clang, for example) and pass the appropriate command arguments to +`make` depending on which compiler you chose. + +### Building x86 (i686) code with GCC + +For GCC, you will need to pass the toolchain's command prefix to the +`CROSS_COMPILE` command line variable. + +This change is also extended to native modules, for which you may need to pass +the toolchain's command prefix to the `CROSS` command line variable if your +x86 compiler cannot be invoked with `i686-linux-gnu-gcc`. + +Or, as an example: + +```bash +$ printf "%s %s %s %s\n" $(lsb_release -d | cut -f 2) $(uname -m) +Ubuntu 24.04.4 LTS x86_64 + +$ i686-linux-gnu-gcc --version | head -n 1 +i686-linux-gnu-gcc (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0 + +$ CROSS_COMPILE=i686-linux-gnu- make -C ports/unix +make: Entering directory '/ports/unix' +Use make V=1 or set BUILD_VERBOSE in your environment to increase build verbosity. +... +LINK build-standard/micropython + text data bss dec hex filename + 723115 36104 2124 761343 b9dff build-standard/micropython +make: Leaving directory '/ports/unix' + +$ file -b ports/unix/build-standard/micropython | cut -d, -f-2 +ELF 32-bit LSB pie executable, Intel 80386 + +# `i686-linux-gnu-` is the default prefix for x86 native modules now, it has +# been explicitly mentioned here in case you want to see how to set it. + +$ make -C examples/natmod/features0 ARCH=x86 CROSS=i686-linux-gnu- +make: Entering directory '/examples/natmod/features0' +GEN build/features0.config.h +CC features0.c +LINK build/features0.o +arch: EM_386 +text size: 192 +bss size: 0 +GOT entries: 4 +GEN features0.mpy +make: Leaving directory '/examples/natmod/features0' + +$ cd examples/natmod/features0 && ../../../ports/unix/build-standard/micropython +MicroPython v1.29.0-preview.490.gb4c58f7ba5.dirty on 2026-07-04; linux [GCC 13.3.0] version +Type "help()" for more information. +>>> import features0 +>>> features0.factorial(10) +3628800 +``` + +### Building x86 (i686) code with Clang + +For Clang, you will need need to pass both the name of the compiler to invoke +as the `CC` command line variable and the extra command line arguments needed +to let Clang know you are building an i686 binary as the `CFLAGS_EXTRA` and +`LDFLAGS_EXTRA` command line variables (usually +`--target=i686-unknown-linux-gnu`). + +For native modules, since linking is not done by the compiler, only the `CC` +and `CFLAGS_EXTRA` arguments are needed. + +Or, as an example: + +```bash +$ printf "%s %s %s %s\n" $(lsb_release -d | cut -f 2) $(uname -m) +Ubuntu 24.04.4 LTS x86_64 + +$ clang --version | head -1 +Ubuntu clang version 18.1.3 (1ubuntu1) + +$ clang -print-targets | grep -i 32-bit.x86 + x86 - 32-bit X86: Pentium-Pro and above + +$ make -C ports/unix CC=clang CFLAGS_EXTRA="--target=i686-unknown-linux-gnu" LDFLAGS_EXTRA='--target=i686-unknown-linux-gnu' +make: Entering directory '/ports/unix' +Use make V=1 or set BUILD_VERBOSE in your environment to increase build verbosity. +... +LINK build-standard/micropython + text data bss dec hex filename + 836241 34748 2052 873041 d5251 build-standard/micropython +make: Leaving directory '/ports/unix' + +$ file -b ports/unix/build-standard/micropython | cut -d, -f-2 +ELF 32-bit LSB pie executable, Intel 80386 + +$ make -C examples/natmod/features0 ARCH=x86 CC=clang CFLAGS_EXTRA="--target=i686-unknown-linux-gnu" +make: Entering directory '/examples/natmod/features0' +GEN build-x86/features0.config.h +CC features0.c +LINK build-x86/features0.o +arch: EM_386 +text size: 180 +bss size: 0 +GOT entries: 2 +GEN features0.mpy +make: Leaving directory '/examples/natmod/features0' + +$ cd examples/natmod/features0 && ../../../ports/unix/build-standard/micropython +MicroPython v1.29.0-preview.490.gb4c58f7ba5.dirty on 2026-07-04; linux [Clang 18.1.3] version +Type "help()" for more information. +>>> import features0 +>>> features0.factorial(10) +3628800 +``` diff --git a/ports/unix/coverage.c b/ports/unix/coverage.c index 49426f0f3e8..8809148652f 100644 --- a/ports/unix/coverage.c +++ b/ports/unix/coverage.c @@ -279,6 +279,28 @@ static mp_obj_t extra_coverage(void) { // calling gc_nbytes with a non-heap pointer mp_printf(&mp_plat_print, "%d\n", (int)gc_nbytes(NULL)); + + // test gc_info_fast + void *p0 = gc_alloc(4, 0); + void *p1 = gc_alloc(4, 0); + void *p2 = gc_alloc(4, 0); + + // Create a hole + gc_free(p1); + + gc_info_t info_slow; + gc_info_t info_fast; + + gc_info(&info_slow); + gc_info_fast(&info_fast); + + // Free allocs + gc_free(p0); + gc_free(p2); + + // Should be equal + mp_printf(&mp_plat_print, "%d\n", info_slow.used == info_fast.used); + mp_printf(&mp_plat_print, "%d\n", info_slow.free == info_fast.free); } // GC initialisation and allocation stress test, to check the logic behind ALLOC_TABLE_GAP_BYTE @@ -346,6 +368,13 @@ static mp_obj_t extra_coverage(void) { gc_collect(); } + // resize one of the blocks + void *before = ptrs[1]; + ptrs[1] = FLIP_POINTER(m_tracked_realloc(FLIP_POINTER(ptrs[1]), 2 * NUM_BYTES)); + void *after = ptrs[1]; + bool location_changed = before != after; + mp_printf(&mp_plat_print, "%d\n", location_changed); + // check the memory blocks have the correct content for (size_t i = 0; i < NUM_PTRS; ++i) { bool correct_contents = true; @@ -364,6 +393,65 @@ static mp_obj_t extra_coverage(void) { } mp_printf(&mp_plat_print, "m_tracked_head = %p\n", MP_STATE_VM(m_tracked_head)); + + // Test realloc with black-box behavioral testing + mp_printf(&mp_plat_print, "# tracked realloc\n"); + + // Test 1: Basic realloc with data preservation + uint8_t *test_ptr = m_tracked_calloc(1, 32); + for (int i = 0; i < 32; i++) { + test_ptr[i] = i; + } + + test_ptr = m_tracked_realloc(test_ptr, 64); // Grow + bool data_preserved = (test_ptr[0] == 0 && test_ptr[31] == 31); + mp_printf(&mp_plat_print, "grow preserves data: %d\n", data_preserved); + + test_ptr = m_tracked_realloc(test_ptr, 16); // Shrink + bool shrink_ok = (test_ptr[0] == 0 && test_ptr[15] == 15); + mp_printf(&mp_plat_print, "shrink preserves data: %d\n", shrink_ok); + + m_tracked_free(test_ptr); + + // Test 2: Multiple allocations + reallocs + GC stability + uint8_t *realloc_ptrs[5]; + for (int i = 0; i < 5; i++) { + realloc_ptrs[i] = m_tracked_calloc(1, 32); + realloc_ptrs[i][0] = 'A' + i; // Mark each + } + + // Realloc some in different positions + realloc_ptrs[0] = m_tracked_realloc(realloc_ptrs[0], 64); // First allocated (tail of list) + realloc_ptrs[2] = m_tracked_realloc(realloc_ptrs[2], 64); // Middle + realloc_ptrs[4] = m_tracked_realloc(realloc_ptrs[4], 64); // Last allocated (head of list) + + // Run GC - if list corrupted, this might crash/fail + gc_collect(); + + // Verify markers intact + bool markers_ok = true; + for (int i = 0; i < 5; i++) { + if (realloc_ptrs[i][0] != 'A' + i) { + markers_ok = false; + break; + } + } + mp_printf(&mp_plat_print, "realloc gc stable: %d\n", markers_ok); + + // Cleanup + for (int i = 0; i < 5; i++) { + m_tracked_free(realloc_ptrs[i]); + } + + // Test 3: Edge cases + uint8_t *null_alloc = m_tracked_realloc(NULL, 32); + null_alloc[0] = 'X'; + mp_printf(&mp_plat_print, "realloc(NULL) ok: %d\n", null_alloc[0] == 'X'); + + void *free_result = m_tracked_realloc(null_alloc, 0); + mp_printf(&mp_plat_print, "realloc(ptr, 0) returns NULL: %d\n", free_result == NULL); + + mp_printf(&mp_plat_print, "m_tracked_head after cleanup: %p\n", MP_STATE_VM(m_tracked_head)); } // vstr @@ -552,6 +640,45 @@ static mp_obj_t extra_coverage(void) { mp_printf(&mp_plat_print, "%x%08x\n", (uint32_t)(value_ll >> 32), (uint32_t)value_ll); } + // list argument helpers + { + mp_printf(&mp_plat_print, "# list argument helpers\n"); + + // Create a list to test with + mp_obj_t list_items[] = { mp_const_none, MP_OBJ_NEW_SMALL_INT(77), mp_obj_new_str_from_cstr("hello") }; + size_t list_len = MP_ARRAY_SIZE(list_items); + mp_obj_t list = mp_obj_new_list(list_len, list_items); + + // mp_obj_list_ensure + nlr_buf_t nlr; + if (nlr_push(&nlr) == 0) { + mp_obj_list_ensure(MP_OBJ_NEW_SMALL_INT(-1), 5); // Not a list + nlr_pop(); + } else { + mp_obj_print_exception(&mp_plat_print, MP_OBJ_FROM_PTR(nlr.ret_val)); + } + + if (nlr_push(&nlr) == 0) { + mp_obj_list_ensure(list, list_len + 2); // List shorter than minimum length + nlr_pop(); + } else { + mp_obj_print_exception(&mp_plat_print, MP_OBJ_FROM_PTR(nlr.ret_val)); + } + + mp_obj_list_t *as_ptr = mp_obj_list_ensure(list, list_len); // Acceptable! + mp_printf(&mp_plat_print, "mp_obj_list_ensure same list? %d\n", MP_OBJ_TO_PTR(list) == as_ptr); + + // mp_obj_list_optional_arg() + as_ptr = mp_obj_list_optional_arg(list, list_len); + mp_printf(&mp_plat_print, "mp_obj_list_optional_arg same list? %d\n", MP_OBJ_TO_PTR(list) == as_ptr); + + as_ptr = mp_obj_list_optional_arg(mp_const_none, list_len); + mp_printf(&mp_plat_print, "mp_obj_list_optional_arg new list len " SIZE_FMT "\n", as_ptr->len); + + as_ptr = mp_obj_list_optional_arg(MP_OBJ_NULL, list_len); + mp_printf(&mp_plat_print, "mp_obj_list_optional_arg new list from NULL len " SIZE_FMT "\n", as_ptr->len); + } + // runtime utils { mp_printf(&mp_plat_print, "# runtime utils\n"); @@ -617,19 +744,6 @@ static mp_obj_t extra_coverage(void) { mp_emitter_warning(MP_PASS_CODE_SIZE, "test"); } - // binary - { - mp_printf(&mp_plat_print, "# binary\n"); - - // call function with float and double typecodes - float far[1]; - double dar[1]; - mp_binary_set_val_array_from_int('f', far, 0, 123); - mp_printf(&mp_plat_print, "%.0f\n", (double)far[0]); - mp_binary_set_val_array_from_int('d', dar, 0, 456); - mp_printf(&mp_plat_print, "%.0lf\n", dar[0]); - } - // VM { mp_printf(&mp_plat_print, "# VM\n"); diff --git a/ports/unix/main.c b/ports/unix/main.c index 1be6827d626..dc72d79127c 100644 --- a/ports/unix/main.c +++ b/ports/unix/main.c @@ -147,7 +147,7 @@ static int execute_from_lexer(int source_kind, const void *source, mp_parse_inpu #if defined(MICROPY_UNIX_COVERAGE) // allow to print the parse tree in the coverage build - if (mp_verbose_flag >= 3) { + if (MP_STATE_VM(mp_verbose_flag) >= 3) { printf("----------------\n"); mp_parse_node_print(&mp_plat_print, parse_tree.root, 0); printf("----------------\n"); @@ -674,7 +674,7 @@ MP_NOINLINE int main_(int argc, char **argv) { a += 1; #if MICROPY_DEBUG_PRINTERS } else if (strcmp(argv[a], "-v") == 0) { - mp_verbose_flag++; + MP_STATE_VM(mp_verbose_flag)++; #endif } else if (strncmp(argv[a], "-O", 2) == 0) { if (unichar_isdigit(argv[a][2])) { @@ -733,7 +733,7 @@ MP_NOINLINE int main_(int argc, char **argv) { #endif #if MICROPY_PY_MICROPYTHON_MEM_INFO - if (mp_verbose_flag) { + if (MP_STATE_VM(mp_verbose_flag)) { mp_micropython_mem_info(0, NULL); } #endif diff --git a/ports/unix/modffi.c b/ports/unix/modffi.c index b469e932e0d..c16d40ad3b4 100644 --- a/ports/unix/modffi.c +++ b/ports/unix/modffi.c @@ -446,7 +446,7 @@ static unsigned long long ffi_get_int_value(mp_obj_t o) { return MP_OBJ_SMALL_INT_VALUE(o); } else { unsigned long long res; - mp_obj_int_to_bytes_impl(o, MP_ENDIANNESS_BIG, sizeof(res), (byte *)&res); + mp_obj_int_to_bytes(o, sizeof(res), (byte *)&res, MP_ENDIANNESS_BIG, false, false); return res; } } diff --git a/ports/unix/modjni.c b/ports/unix/modjni.c index dbce61aec11..0a29a94fbd4 100644 --- a/ports/unix/modjni.c +++ b/ports/unix/modjni.c @@ -357,7 +357,7 @@ static void jmethod_print(const mp_print_t *print, mp_obj_t self_in, mp_print_ki (void)kind; mp_obj_jmethod_t *self = MP_OBJ_TO_PTR(self_in); // Variable value printed as cast to int - mp_printf(print, "", qstr_str(self->name)); + mp_printf(print, "", (qstr)self->name); } #define IMATCH(s, static) ((!strncmp(s, static, sizeof(static) - 1)) && (s += sizeof(static) - 1)) diff --git a/ports/unix/mpconfigport.h b/ports/unix/mpconfigport.h index 815be76b4e9..7c2ede3bad3 100644 --- a/ports/unix/mpconfigport.h +++ b/ports/unix/mpconfigport.h @@ -54,13 +54,20 @@ #ifndef MICROPY_PY_SYS_PLATFORM #if defined(__APPLE__) && defined(__MACH__) #define MICROPY_PY_SYS_PLATFORM "darwin" +#elif defined(__FreeBSD__) + #define MICROPY_PY_SYS_PLATFORM "freebsd" #else #define MICROPY_PY_SYS_PLATFORM "linux" #endif #endif #ifndef MICROPY_PY_SYS_PATH_DEFAULT -#define MICROPY_PY_SYS_PATH_DEFAULT ".frozen:~/.micropython/lib:/usr/lib/micropython" +#if defined(__FreeBSD__) +#define SYSTEM_LIB_PATH "/usr/local/lib/micropython" +#else +#define SYSTEM_LIB_PATH "/usr/lib/micropython" +#endif +#define MICROPY_PY_SYS_PATH_DEFAULT ".frozen:~/.micropython/lib:" SYSTEM_LIB_PATH #endif #define MP_STATE_PORT MP_STATE_VM @@ -104,9 +111,12 @@ typedef long mp_off_t; // CIRCUITPY-CHANGE #define MICROPY_ENABLE_SELECTIVE_COLLECT (1) -#if !(defined(MICROPY_GCREGS_SETJMP) || defined(__x86_64__) || defined(__i386__) || defined(__thumb2__) || defined(__thumb__) || defined(__arm__) || (defined(__riscv) && (__riscv_xlen == 64))) -// Fall back to setjmp() implementation for discovery of GC pointers in registers. -#define MICROPY_GCREGS_SETJMP (1) +// Fall back to setjmp() implementation for discovery of GC pointers in registers +// if running on intel-based macOS, or on architectures for which there is no +// specialised GC pointer discovery mechanism. +#if (defined(__APPLE__) && defined(__MACH__) && (defined(__i386__) || defined(__x86_64__))) || \ + (!(defined(MICROPY_GCREGS_SETJMP) || defined(__x86_64__) || defined(__i386__) || defined(__thumb2__) || defined(__thumb__) || defined(__arm__) || (defined(__riscv) && __riscv_xlen <= 64) || (defined(__loongarch__) && defined(__loongarch64)))) +#define MICROPY_GCREGS_SETJMP (1) #endif // Enable the VFS, and enable the posix "filesystem". diff --git a/ports/unix/mpconfigport.mk b/ports/unix/mpconfigport.mk index 3c844ccef3b..5260bd46aa1 100644 --- a/ports/unix/mpconfigport.mk +++ b/ports/unix/mpconfigport.mk @@ -1,8 +1,5 @@ # Enable/disable modules and 3rd-party libs to be included in interpreter -# Build 32-bit binaries on a 64-bit host -MICROPY_FORCE_32BIT = 0 - # This variable can take the following values: # 0 - no readline, just simple stdin input # 1 - use MicroPython version of readline diff --git a/ports/unix/mphalport.h b/ports/unix/mphalport.h index d9cd05b3de8..91d44d1688d 100644 --- a/ports/unix/mphalport.h +++ b/ports/unix/mphalport.h @@ -124,5 +124,14 @@ enum { void mp_hal_get_mac(int idx, uint8_t buf[6]); #endif +#if defined(__linux__) && (defined(__ARM_32BIT_STATE) || defined(__riscv)) +#if __has_builtin(__builtin___clear_cache) +#define MP_HAL_CLEAN_DCACHE(fun_data, fun_len) \ + do { \ + __builtin___clear_cache((void *)fun_data, (char *)fun_data + fun_len); \ + } while (0) +#endif +#endif + // Global variable to control compile-only mode. extern bool mp_compile_only; diff --git a/ports/unix/stack_size.h b/ports/unix/stack_size.h index f6159bb69d5..95ec502b0f8 100644 --- a/ports/unix/stack_size.h +++ b/ports/unix/stack_size.h @@ -46,8 +46,15 @@ #define UNIX_STACK_MUL_SANITIZERS 1 #endif -// Double the stack size for 64-bit builds, plus additional scaling -#define UNIX_STACK_MULTIPLIER ((sizeof(void *) / 4) * UNIX_STACK_MUL_ARM * UNIX_STACK_MUL_SANITIZERS) +#if defined(_MSC_VER) +// Similarly Windows seems to require more stack +#define UNIX_STACK_MUL_WINDOWS 2 +#else +#define UNIX_STACK_MUL_WINDOWS 1 +#endif + +// Double the stack size for 64-bit builds, plus additional scalings +#define UNIX_STACK_MULTIPLIER ((sizeof(void *) / 4) * (UNIX_STACK_MUL_ARM)*(UNIX_STACK_MUL_SANITIZERS)*(UNIX_STACK_MUL_WINDOWS)) #endif // UNIX_STACK_MULTIPLIER diff --git a/ports/unix/unix_mphal.c b/ports/unix/unix_mphal.c index 54fd4c94ad7..ac75f20eee4 100644 --- a/ports/unix/unix_mphal.c +++ b/ports/unix/unix_mphal.c @@ -117,12 +117,12 @@ void mp_hal_stdio_mode_raw(void) { termios.c_lflag = 0; termios.c_cc[VMIN] = 1; termios.c_cc[VTIME] = 0; - tcsetattr(0, TCSAFLUSH, &termios); + tcsetattr(0, TCSANOW, &termios); } void mp_hal_stdio_mode_orig(void) { // restore terminal settings - tcsetattr(0, TCSAFLUSH, &orig_termios); + tcsetattr(0, TCSANOW, &orig_termios); } #endif diff --git a/ports/unix/variants/coverage/manifest.py b/ports/unix/variants/coverage/manifest.py index 37f2531a842..2d459103ef2 100644 --- a/ports/unix/variants/coverage/manifest.py +++ b/ports/unix/variants/coverage/manifest.py @@ -1,5 +1,5 @@ add_library("unix-ffi", "$(MPY_LIB_DIR)/unix-ffi") freeze_as_str("frzstr") freeze_as_mpy("frzmpy") -freeze_mpy("$(MPY_DIR)/tests/frozen") +freeze_mpy("$(MPY_DIR)/tests/assets") require("ssl") diff --git a/ports/unix/variants/coverage/mpconfigvariant.mk b/ports/unix/variants/coverage/mpconfigvariant.mk index 2ef5b0a1bdb..beab74832cb 100644 --- a/ports/unix/variants/coverage/mpconfigvariant.mk +++ b/ports/unix/variants/coverage/mpconfigvariant.mk @@ -12,8 +12,6 @@ CFLAGS += \ LDFLAGS += -fprofile-arcs -ftest-coverage FROZEN_MANIFEST ?= $(VARIANT_DIR)/manifest.py -# CIRCUITPY-CHANGE: don't include user C modules -# USER_C_MODULES = $(TOP)/examples/usercmodule # CIRCUITPY-CHANGE: use CircuitPython bindings and implementations SRC_QRIO := $(patsubst ../../%,%,$(wildcard ../../shared-bindings/qrio/*.c ../../shared-module/qrio/*.c ../../lib/quirc/lib/*.c)) diff --git a/ports/unix/variants/longlong/mpconfigvariant.h b/ports/unix/variants/longlong/mpconfigvariant.h index d50d360b1fe..1554d537c04 100644 --- a/ports/unix/variants/longlong/mpconfigvariant.h +++ b/ports/unix/variants/longlong/mpconfigvariant.h @@ -32,7 +32,7 @@ // We build it on top of REPR C, which uses memory-efficient floating point // objects encoded directly mp_obj_t (30 bits only). -// Therefore this variant should be built using MICROPY_FORCE_32BIT=1 +// Therefore this variant should be built for a 32-bits target. #define MICROPY_OBJ_REPR (MICROPY_OBJ_REPR_C) #define MICROPY_FLOAT_IMPL (MICROPY_FLOAT_IMPL_FLOAT) diff --git a/ports/unix/variants/longlong/mpconfigvariant.mk b/ports/unix/variants/longlong/mpconfigvariant.mk index 2d2c3706469..9fe20cd584d 100644 --- a/ports/unix/variants/longlong/mpconfigvariant.mk +++ b/ports/unix/variants/longlong/mpconfigvariant.mk @@ -1,7 +1,7 @@ # build interpreter with "bigints" implemented as "longlong" -# otherwise, small int is essentially 64-bit -MICROPY_FORCE_32BIT := 1 +# This needs to be built for a 32-bits target, otherwise small ints will +# essentially be 64-bit wide. MICROPY_PY_FFI := 0 diff --git a/ports/unix/variants/mpconfigvariant_common.h b/ports/unix/variants/mpconfigvariant_common.h index 1ac59c95572..382579ead61 100644 --- a/ports/unix/variants/mpconfigvariant_common.h +++ b/ports/unix/variants/mpconfigvariant_common.h @@ -83,7 +83,9 @@ #define MICROPY_PY_GC_COLLECT_RETVAL (1) // Enable detailed error messages and warnings. +#ifndef MICROPY_ERROR_REPORTING #define MICROPY_ERROR_REPORTING (MICROPY_ERROR_REPORTING_DETAILED) +#endif #define MICROPY_WARNINGS (1) #define MICROPY_PY_STR_BYTES_CMP_WARN (1) diff --git a/ports/unix/variants/nanbox/mpconfigvariant.mk b/ports/unix/variants/nanbox/mpconfigvariant.mk index e588e657efc..6d63a86e9b0 100644 --- a/ports/unix/variants/nanbox/mpconfigvariant.mk +++ b/ports/unix/variants/nanbox/mpconfigvariant.mk @@ -1,3 +1,4 @@ # build interpreter with nan-boxing as object model (object repr D) -MICROPY_FORCE_32BIT = 1 +# This needs to be built for a 32-bits target, as object representation D is +# only meant to work on 32-bits machines. diff --git a/py/asmarm.h b/py/asmarm.h index 74e43de7f90..7a68e4c640f 100644 --- a/py/asmarm.h +++ b/py/asmarm.h @@ -50,21 +50,21 @@ #define ASM_ARM_REG_LR (ASM_ARM_REG_R14) #define ASM_ARM_REG_PC (ASM_ARM_REG_R15) -#define ASM_ARM_CC_EQ (0x0 << 28) -#define ASM_ARM_CC_NE (0x1 << 28) -#define ASM_ARM_CC_CS (0x2 << 28) -#define ASM_ARM_CC_CC (0x3 << 28) -#define ASM_ARM_CC_MI (0x4 << 28) -#define ASM_ARM_CC_PL (0x5 << 28) -#define ASM_ARM_CC_VS (0x6 << 28) -#define ASM_ARM_CC_VC (0x7 << 28) -#define ASM_ARM_CC_HI (0x8 << 28) -#define ASM_ARM_CC_LS (0x9 << 28) -#define ASM_ARM_CC_GE (0xa << 28) -#define ASM_ARM_CC_LT (0xb << 28) -#define ASM_ARM_CC_GT (0xc << 28) -#define ASM_ARM_CC_LE (0xd << 28) -#define ASM_ARM_CC_AL (0xe << 28) +#define ASM_ARM_CC_EQ (0x0u << 28) +#define ASM_ARM_CC_NE (0x1u << 28) +#define ASM_ARM_CC_CS (0x2u << 28) +#define ASM_ARM_CC_CC (0x3u << 28) +#define ASM_ARM_CC_MI (0x4u << 28) +#define ASM_ARM_CC_PL (0x5u << 28) +#define ASM_ARM_CC_VS (0x6u << 28) +#define ASM_ARM_CC_VC (0x7u << 28) +#define ASM_ARM_CC_HI (0x8u << 28) +#define ASM_ARM_CC_LS (0x9u << 28) +#define ASM_ARM_CC_GE (0xau << 28) +#define ASM_ARM_CC_LT (0xbu << 28) +#define ASM_ARM_CC_GT (0xcu << 28) +#define ASM_ARM_CC_LE (0xdu << 28) +#define ASM_ARM_CC_AL (0xeu << 28) typedef struct _asm_arm_t { mp_asm_base_t base; diff --git a/py/asmrv32.c b/py/asmrv32.c index e58e42012d9..53481d1d0f3 100644 --- a/py/asmrv32.c +++ b/py/asmrv32.c @@ -515,9 +515,8 @@ void asm_rv32_emit_mov_reg_local_addr(asm_rv32_t *state, mp_uint_t rd, mp_uint_t asm_rv32_opcode_cadd(state, rd, ASM_RV32_REG_SP); } -static const uint8_t RV32_LOAD_OPCODE_TABLE[3] = { - 0x04, 0x05, 0x02 -}; +// ((word: 4) << 8) | ((halfword: 4) << 4) | (byte: 4) +#define RV32_LOAD_OPCODE_FT3(size) ((0x0254 >> (size << 2)) & 0x0F) void asm_rv32_emit_load_reg_reg_offset(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs, int32_t offset, mp_uint_t operation_size) { assert(operation_size <= 2 && "Operation size value out of range."); @@ -532,7 +531,7 @@ void asm_rv32_emit_load_reg_reg_offset(asm_rv32_t *state, mp_uint_t rd, mp_uint_ if (MP_FIT_SIGNED(12, scaled_offset)) { // lbu|lhu|lw rd, offset(rs) - asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_I(0x03, RV32_LOAD_OPCODE_TABLE[operation_size], rd, rs, scaled_offset)); + asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_I(0x03, RV32_LOAD_OPCODE_FT3(operation_size), rd, rs, scaled_offset)); return; } @@ -545,7 +544,7 @@ void asm_rv32_emit_load_reg_reg_offset(asm_rv32_t *state, mp_uint_t rd, mp_uint_ // lbu|lhu|lw rd, LO(offset)(rd) load_upper_immediate(state, rd, upper); asm_rv32_opcode_cadd(state, rd, rs); - asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_I(0x03, RV32_LOAD_OPCODE_TABLE[operation_size], rd, rd, lower)); + asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_I(0x03, RV32_LOAD_OPCODE_FT3(operation_size), rd, rd, lower)); } void asm_rv32_emit_jump(asm_rv32_t *state, mp_uint_t label) { diff --git a/py/asmrv32.h b/py/asmrv32.h index c25b1aa4e26..f71fe72706a 100644 --- a/py/asmrv32.h +++ b/py/asmrv32.h @@ -201,6 +201,14 @@ void asm_rv32_end_pass(asm_rv32_t *state); ((op & 0x03) | ((ft6 & 0x3F) << 10) | ((ft2 & 0x03) << 8) | \ ((rlist & 0x0F) << 4) | ((imm & 0x03) << 2)) +#define RV32_ENCODE_TYPE_CMMV(op, ft6, ft2, r1s, r2s) \ + ((op & 0x03) | ((ft6 & 0x3F) << 10) | ((ft2 & 0x03) << 5) | \ + ((((r1s >= ASM_RV32_REG_S0 && r1s <= ASM_RV32_REG_S1) ? \ + (r1s - ASM_RV32_REG_S0) : (r1s - ASM_RV32_REG_S2 + 2)) & 0x07) << 7) | \ + ((((r2s >= ASM_RV32_REG_S0 && r2s <= ASM_RV32_REG_S1) ? \ + (r2s - ASM_RV32_REG_S0) : (r2s - ASM_RV32_REG_S2 + 2)) & 0x07) << 2)) + + #define RV32_ENCODE_TYPE_CR(op, ft4, rs1, rs2) \ ((op & 0x03) | ((rs2 & 0x1F) << 2) | ((rs1 & 0x1F) << 7) | ((ft4 & 0x0F) << 12)) @@ -444,12 +452,36 @@ static inline void asm_rv32_opcode_cxor(asm_rv32_t *state, mp_uint_t rd, mp_uint asm_rv32_emit_halfword_opcode(state, RV32_ENCODE_TYPE_CA(0x01, 0x23, 0x01, rd, rs)); } +// CM.MVA01S R1S', R2S' +static inline void asm_rv32_opcode_cmmva01s(asm_rv32_t *state, mp_uint_t r1s, mp_uint_t r2s) { + // CMMV: 101011 ... 11 ... 10 + asm_rv32_emit_halfword_opcode(state, RV32_ENCODE_TYPE_CMMV(0x02, 0x2B, 0x03, r1s, r2s)); +} + +// CM.MVSA01 R1S', R2S' +static inline void asm_rv32_opcode_cmmvsa01(asm_rv32_t *state, mp_uint_t r1s, mp_uint_t r2s) { + // CMMV: 101011 ... 01 ... 10 + asm_rv32_emit_halfword_opcode(state, RV32_ENCODE_TYPE_CMMV(0x02, 0x2B, 0x01, r1s, r2s)); +} + +// CM.POP {REG_LIST}, IMMEDIATE +static inline void asm_rv32_opcode_cmpop(asm_rv32_t *state, mp_uint_t reg_list, mp_uint_t immediate) { + // CMPP: 10111010 .... .. 10 + asm_rv32_emit_halfword_opcode(state, RV32_ENCODE_TYPE_CMPP(0x02, 0x2E, 0x02, reg_list, immediate)); +} + // CM.POPRET {REG_LIST}, IMMEDIATE static inline void asm_rv32_opcode_cmpopret(asm_rv32_t *state, mp_uint_t reg_list, mp_uint_t immediate) { - // CMPP: 10111110 ... .. 10 + // CMPP: 10111110 .... .. 10 asm_rv32_emit_halfword_opcode(state, RV32_ENCODE_TYPE_CMPP(0x02, 0x2F, 0x02, reg_list, immediate)); } +// CM.POPRETZ {REG_LIST}, IMMEDIATE +static inline void asm_rv32_opcode_cmpopretz(asm_rv32_t *state, mp_uint_t reg_list, mp_uint_t immediate) { + // CMPP: 10111100 .... .. 10 + asm_rv32_emit_halfword_opcode(state, RV32_ENCODE_TYPE_CMPP(0x02, 0x2F, 0x00, reg_list, immediate)); +} + // CM.PUSH {REG_LIST}, -IMMEDIATE static inline void asm_rv32_opcode_cmpush(asm_rv32_t *state, mp_uint_t reg_list, mp_uint_t immediate) { // CMPP: 10111000 .... .. 10 @@ -748,9 +780,9 @@ static inline uint8_t asm_rv32_allowed_extensions(void) { #define REG_ARG_2 ASM_RV32_REG_A1 #define REG_ARG_3 ASM_RV32_REG_A2 #define REG_ARG_4 ASM_RV32_REG_A3 -#define REG_TEMP0 ASM_RV32_REG_T1 -#define REG_TEMP1 ASM_RV32_REG_T2 -#define REG_TEMP2 ASM_RV32_REG_T3 +#define REG_TEMP0 ASM_RV32_REG_A4 +#define REG_TEMP1 ASM_RV32_REG_A5 +#define REG_TEMP2 ASM_RV32_REG_A6 #define REG_FUN_TABLE ASM_RV32_REG_S1 #define REG_LOCAL_1 ASM_RV32_REG_S3 #define REG_LOCAL_2 ASM_RV32_REG_S2 diff --git a/py/asmthumb.c b/py/asmthumb.c index 58cc7aea880..4226ae28127 100644 --- a/py/asmthumb.c +++ b/py/asmthumb.c @@ -246,7 +246,7 @@ void asm_thumb_mov_reg_reg(asm_thumb_t *as, uint reg_dest, uint reg_src) { void asm_thumb_mov_reg_i16(asm_thumb_t *as, uint mov_op, uint reg_dest, int i16_src) { assert(reg_dest < ASM_THUMB_REG_R15); // mov[wt] reg_dest, #i16_src - asm_thumb_op32(as, mov_op | ((i16_src >> 1) & 0x0400) | ((i16_src >> 12) & 0xf), ((i16_src << 4) & 0x7000) | (reg_dest << 8) | (i16_src & 0xff)); + asm_thumb_op32(as, mov_op | ((i16_src >> 1) & 0x0400) | ((i16_src >> 12) & 0xf), (((uint16_t)i16_src << 4) & 0x7000) | (reg_dest << 8) | (i16_src & 0xff)); } static void asm_thumb_mov_rlo_i16(asm_thumb_t *as, uint rlo_dest, int i16_src) { diff --git a/py/binary.c b/py/binary.c index 180a13beec7..c7fbebcaca4 100644 --- a/py/binary.c +++ b/py/binary.c @@ -42,6 +42,12 @@ #define alignof(type) offsetof(struct { char c; type t; }, t) #endif +// MicroPython V1.x truncates integers when writing into arrays, +// MicroPython V2 will raise OverflowError in these cases, same as CPython +// CIRCUITPY-CHANGE: CircuitPython raises OverflowError like CPython, +// rather than truncating like MicroPython V1.x. +#define OVERFLOW_CHECKS (1) + size_t mp_binary_get_size(char struct_type, char val_type, size_t *palign) { size_t size = 0; int align = 1; @@ -383,7 +389,21 @@ mp_obj_t mp_binary_get_val(char struct_type, char val_type, byte *p_base, byte * } } -void mp_binary_set_int(size_t val_sz, bool big_endian, byte *dest, mp_uint_t val) { +void mp_binary_set_int(size_t dest_sz, byte *dest, size_t val_sz, mp_uint_t val, bool big_endian) { + if (dest_sz > val_sz) { + // zero/sign extension if needed + int c = ((mp_int_t)val < 0) ? 0xff : 0x00; + memset(dest, c, dest_sz); + + // big endian: write val_sz bytes at end of 'dest' + if (big_endian) { + dest += dest_sz - val_sz; + } + } else if (dest_sz < val_sz) { + // truncate 'val' into 'dest' + val_sz = dest_sz; + } + if (MP_ENDIANNESS_LITTLE && !big_endian) { memcpy(dest, &val, val_sz); } else if (MP_ENDIANNESS_BIG && big_endian) { @@ -451,46 +471,44 @@ void mp_binary_set_val(char struct_type, char val_type, mp_obj_t val_in, byte *p val = fp_dp.i64; } else { int be = struct_type == '>'; - mp_binary_set_int(sizeof(uint32_t), be, p, fp_dp.i32[MP_ENDIANNESS_BIG ^ be]); + mp_binary_set_int(sizeof(uint32_t), p, sizeof(uint32_t), fp_dp.i32[MP_ENDIANNESS_BIG ^ be], be); + // Now fall through and copy the second word, below p += sizeof(uint32_t); + size = sizeof(uint32_t); val = fp_dp.i32[MP_ENDIANNESS_LITTLE ^ be]; } break; } #endif #endif - default: { - // CIRCUITPY-CHANGE: add overflow checks - bool signed_type = is_signed(val_type); + default: + // Typecode is a standard integer #if MICROPY_LONGINT_IMPL != MICROPY_LONGINT_IMPL_NONE if (mp_obj_is_exact_type(val_in, &mp_type_int)) { - // It's a longint. - mp_obj_int_buffer_overflow_check(val_in, size, signed_type); - mp_obj_int_to_bytes_impl(val_in, struct_type == '>', size, p); + #if !OVERFLOW_CHECKS + if (size <= sizeof(mp_uint_t)) { + // Aligned store; byte-wise path corrupts word-only registers. + val = mp_obj_int_get_truncated(val_in); + break; + } + #endif + mp_obj_int_to_bytes(val_in, size, p, struct_type == '>', is_signed(val_type), OVERFLOW_CHECKS); return; } #endif - // CIRCUITPY-CHANGE: add overflow checks - { - val = mp_obj_get_int(val_in); - // Small int checking is separate, to be fast. - mp_small_int_buffer_overflow_check(val, size, signed_type); - // zero/sign extend if needed - if (MP_BYTES_PER_OBJ_WORD < 8 && size > sizeof(val)) { - int c = (is_signed(val_type) && (mp_int_t)val < 0) ? 0xff : 0x00; - memset(p, c, size); - if (struct_type == '>') { - p += size - sizeof(val); - } - } - break; - } - } + val = mp_obj_get_int(val_in); + #if OVERFLOW_CHECKS + // CIRCUITPY-CHANGE: small ints are checked too. + mp_small_int_buffer_overflow_check(val, size, is_signed(val_type)); + #endif + break; // Fall through to mp_binary_set_int } - mp_binary_set_int(MIN((size_t)size, sizeof(val)), struct_type == '>', p, val); + mp_binary_set_int(size, p, sizeof(val), val, struct_type == '>'); } +static void mp_binary_set_val_array_from_int(char typecode, void *p, size_t index, mp_int_t val); + void mp_binary_set_val_array(char typecode, void *p, size_t index, mp_obj_t val_in) { switch (typecode) { #if MICROPY_PY_BUILTINS_FLOAT @@ -507,70 +525,88 @@ void mp_binary_set_val_array(char typecode, void *p, size_t index, mp_obj_t val_ ((mp_obj_t *)p)[index] = val_in; break; #endif - default: { - // CIRCUITPY-CHANGE: add overflow checks - size_t size = mp_binary_get_size('@', typecode, NULL); - bool signed_type = is_signed(typecode); - + // In all remaining cases the type code is an integer + default: #if MICROPY_LONGINT_IMPL != MICROPY_LONGINT_IMPL_NONE if (mp_obj_is_exact_type(val_in, &mp_type_int)) { - // It's a long int. - mp_obj_int_buffer_overflow_check(val_in, size, signed_type); - mp_obj_int_to_bytes_impl(val_in, MP_ENDIANNESS_BIG, - size, (uint8_t *)p + index * size); + size_t size = mp_binary_get_size('@', typecode, NULL); + #if !OVERFLOW_CHECKS + if (size <= sizeof(mp_int_t)) { + // Aligned store; byte-wise path corrupts word-only registers. + mp_binary_set_val_array_from_int(typecode, p, index, + mp_obj_int_get_truncated(val_in)); + return; + } + #endif + p = (uint8_t *)p + index * size; + byte *dest; + #if OVERFLOW_CHECKS + // If mp_obj_int_to_bytes() might overflow then need to write into a temporary buffer first + assert(size <= sizeof(uint64_t)); + uint64_t temp_buf; + dest = (uint8_t *)&temp_buf; + #else + dest = p; + #endif + mp_obj_int_to_bytes(val_in, size, dest, MP_ENDIANNESS_BIG, is_signed(typecode), OVERFLOW_CHECKS); + #if OVERFLOW_CHECKS + memcpy(p, dest, size); + #endif return; } #endif - // CIRCUITPY-CHANGE: add overflow checks - mp_int_t val = mp_obj_get_int(val_in); - // Small int checking is separate, to be fast. - mp_small_int_buffer_overflow_check(val, size, signed_type); - mp_binary_set_val_array_from_int(typecode, p, index, val); - } + mp_binary_set_val_array_from_int(typecode, p, index, mp_obj_get_int(val_in)); } } -void mp_binary_set_val_array_from_int(char typecode, void *p, size_t index, mp_int_t val) { +#if OVERFLOW_CHECKS +#define SET_VAL_AS(TYPE, IS_SIGNED) do { \ + TYPE tmp = val; \ + if ((mp_int_t)tmp == val && (IS_SIGNED || val >= 0)) { \ + ((TYPE *)p)[index] = tmp; \ + } else { \ + goto raise; \ + } \ +} while (0) +#else +#define SET_VAL_AS(TYPE, _IS_SIGNED) do { \ + ((TYPE *)p)[index] = val; \ +} while (0) +#endif + +static void mp_binary_set_val_array_from_int(char typecode, void *p, size_t index, mp_int_t val) { switch (typecode) { case 'b': - ((signed char *)p)[index] = val; + SET_VAL_AS(signed char, true); break; case BYTEARRAY_TYPECODE: case 'B': - ((unsigned char *)p)[index] = val; + SET_VAL_AS(unsigned char, false); break; case 'h': - ((short *)p)[index] = val; + SET_VAL_AS(short, true); break; case 'H': - ((unsigned short *)p)[index] = val; + SET_VAL_AS(unsigned short, false); break; case 'i': - ((int *)p)[index] = val; + SET_VAL_AS(int, true); break; case 'I': - ((unsigned int *)p)[index] = val; + SET_VAL_AS(unsigned int, false); break; case 'l': - ((long *)p)[index] = val; + SET_VAL_AS(long, true); break; case 'L': - ((unsigned long *)p)[index] = val; + SET_VAL_AS(unsigned long, false); break; #if MICROPY_LONGINT_IMPL != MICROPY_LONGINT_IMPL_NONE case 'q': - ((long long *)p)[index] = val; + SET_VAL_AS(long long, true); break; case 'Q': - ((unsigned long long *)p)[index] = val; - break; - #endif - #if MICROPY_PY_BUILTINS_FLOAT - case 'f': - ((float *)p)[index] = (float)val; - break; - case 'd': - ((double *)p)[index] = (double)val; + SET_VAL_AS(unsigned long long, false); break; #endif // Extension to CPython: array of pointers @@ -580,4 +616,11 @@ void mp_binary_set_val_array_from_int(char typecode, void *p, size_t index, mp_i break; #endif } + + return; + + #if OVERFLOW_CHECKS +raise: + mp_raise_msg(&mp_type_OverflowError, MP_ERROR_TEXT("integer out of range")); + #endif } diff --git a/py/binary.h b/py/binary.h index 5c645bcaaa9..851dc50110e 100644 --- a/py/binary.h +++ b/py/binary.h @@ -37,10 +37,9 @@ size_t mp_binary_get_size(char struct_type, char val_type, size_t *palign); mp_obj_t mp_binary_get_val_array(char typecode, void *p, size_t index); void mp_binary_set_val_array(char typecode, void *p, size_t index, mp_obj_t val_in); -void mp_binary_set_val_array_from_int(char typecode, void *p, size_t index, mp_int_t val); mp_obj_t mp_binary_get_val(char struct_type, char val_type, byte *p_base, byte **ptr); void mp_binary_set_val(char struct_type, char val_type, mp_obj_t val_in, byte *p_base, byte **ptr); long long mp_binary_get_int(size_t size, bool is_signed, bool big_endian, const byte *src); -void mp_binary_set_int(size_t val_sz, bool big_endian, byte *dest, mp_uint_t val); +void mp_binary_set_int(size_t dest_sz, byte *dest, size_t val_sz, mp_uint_t val, bool big_endian); #endif // MICROPY_INCLUDED_PY_BINARY_H diff --git a/py/builtinhelp.c b/py/builtinhelp.c index 05b51ed4875..d3d7aa402c8 100644 --- a/py/builtinhelp.c +++ b/py/builtinhelp.c @@ -36,6 +36,14 @@ #if MICROPY_PY_BUILTINS_HELP +#if MICROPY_PY_BUILTINS_HELP_NUM_COLUMNS <= 0 +#error "MICROPY_PY_BUILTINS_HELP_NUM_COLUMNS must be more than 0" +#endif + +#if MICROPY_PY_BUILTINS_HELP_COLUMN_WIDTH <= 0 +#error "MICROPY_PY_BUILTINS_HELP_COLUMN_WIDTH must be more than 0" +#endif + const char mp_help_default_text[] = "Welcome to MicroPython!\n" "\n" @@ -99,12 +107,10 @@ static void mp_help_print_modules(void) { mp_obj_list_sort(1, &list, (mp_map_t *)&mp_const_empty_map); // print the list of modules in a column-first order - #define NUM_COLUMNS (4) - #define COLUMN_WIDTH (18) size_t len; mp_obj_t *items; mp_obj_list_get(list, &len, &items); - unsigned int num_rows = (len + NUM_COLUMNS - 1) / NUM_COLUMNS; + unsigned int num_rows = (len + MICROPY_PY_BUILTINS_HELP_NUM_COLUMNS - 1) / MICROPY_PY_BUILTINS_HELP_NUM_COLUMNS; for (unsigned int i = 0; i < num_rows; ++i) { unsigned int j = i; for (;;) { @@ -113,9 +119,9 @@ static void mp_help_print_modules(void) { if (j >= len) { break; } - int gap = COLUMN_WIDTH - l; + int gap = MICROPY_PY_BUILTINS_HELP_COLUMN_WIDTH - l; while (gap < 1) { - gap += COLUMN_WIDTH; + gap += MICROPY_PY_BUILTINS_HELP_COLUMN_WIDTH; } while (gap--) { mp_print_str(MP_PYTHON_PRINTER, " "); diff --git a/py/builtinimport.c b/py/builtinimport.c index 5d77ac42859..ff5b02979ee 100644 --- a/py/builtinimport.c +++ b/py/builtinimport.c @@ -259,7 +259,9 @@ static void do_load(mp_module_context_t *module_obj, vstr_t *file) { #endif // MICROPY_MODULE_FROZEN + #if MICROPY_ENABLE_COMPILER || (MICROPY_PERSISTENT_CODE_LOAD && MICROPY_HAS_FILE_READER) qstr file_qstr = qstr_from_str(file_str); + #endif // If we support loading .mpy files then check if the file extension is of // the correct format and, if so, load and execute the file. diff --git a/py/circuitpy_mpconfig.h b/py/circuitpy_mpconfig.h index 89eeee606f5..4fa5cd63344 100644 --- a/py/circuitpy_mpconfig.h +++ b/py/circuitpy_mpconfig.h @@ -192,6 +192,9 @@ extern void common_hal_mcu_enable_interrupts(void); #define FILESYSTEM_BLOCK_SIZE (512) #define MICROPY_VFS (1) +// CIRCUITPY-CHANGE: CircuitPython's flash/SD block devices are native +// (supervisor/shared/flash.c sets MP_BLOCKDEV_FLAG_NATIVE). +#define MICROPY_VFS_BLOCKDEV_NATIVE (1) #define MICROPY_VFS_FAT (MICROPY_VFS) #define MICROPY_READER_VFS (MICROPY_VFS) diff --git a/py/compile.c b/py/compile.c index e16e2f131e7..ae3526e2f2a 100644 --- a/py/compile.c +++ b/py/compile.c @@ -2423,6 +2423,14 @@ static void compile_trailer_paren_helper(compiler_t *comp, mp_parse_node_t pn_ar compile_syntax_error(comp, (mp_parse_node_t)pns_arg, MP_ERROR_TEXT("* arg after **")); return; } + if (n_keyword) { + // Support for *arg after kwarg is a CPython feature omitted + // from MicroPython in order to reduce code size. See + // https://github.com/micropython/micropython/issues/11439 for + // more info. + compile_syntax_error(comp, (mp_parse_node_t)pns_arg, MP_ERROR_TEXT("* arg after kwarg")); + return; + } #if MICROPY_DYNAMIC_COMPILER if (i >= (size_t)mp_dynamic_compiler.small_int_bits - 1) #else @@ -3664,7 +3672,7 @@ void mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_file, bool #if MICROPY_DEBUG_PRINTERS // now that the module context is valid, the raw codes can be printed - if (mp_verbose_flag >= 2) { + if (MP_STATE_VM(mp_verbose_flag) >= 2) { for (scope_t *s = comp->scope_head; s != NULL; s = s->next) { mp_raw_code_t *rc = s->raw_code; if (rc->kind == MP_CODE_BYTECODE) { diff --git a/py/dynruntime.mk b/py/dynruntime.mk index 3902acbd0b3..cab609b13b0 100644 --- a/py/dynruntime.mk +++ b/py/dynruntime.mk @@ -1,7 +1,8 @@ # Makefile fragment for generating native .mpy files from C source # MPY_DIR must be set to the top of the MicroPython source tree -BUILD ?= build +BUILD ?= build-$(ARCH) +CC = $(CROSS)gcc ECHO = @echo RM = /bin/rm @@ -39,7 +40,7 @@ MPY_CROSS_FLAGS += -march=$(ARCH) SRC_O += $(addprefix $(BUILD)/, $(patsubst %.c,%.o,$(filter %.c,$(SRC))) $(patsubst %.S,%.o,$(filter %.S,$(SRC)))) SRC_MPY += $(addprefix $(BUILD)/, $(patsubst %.py,%.mpy,$(filter %.py,$(SRC)))) -CLEAN_EXTRA += $(MOD).mpy .mpy_ld_cache +CLEAN_EXTRA += $(MOD).mpy .mpy_ld_cache-$(ARCH) ################################################################################ # Architecture configuration @@ -47,14 +48,14 @@ CLEAN_EXTRA += $(MOD).mpy .mpy_ld_cache ifeq ($(ARCH),x86) # x86 -CROSS = -CFLAGS_ARCH += -m32 -fno-stack-protector +CROSS = i686-linux-gnu- +CFLAGS_ARCH += -fno-stack-protector MICROPY_FLOAT_IMPL ?= double else ifeq ($(ARCH),x64) # x64 -CROSS = +CROSS = x86_64-linux-gnu- CFLAGS_ARCH += -fno-stack-protector MICROPY_FLOAT_IMPL ?= double @@ -106,53 +107,57 @@ else ifeq ($(ARCH),rv32imc) # rv32imc CROSS = riscv64-unknown-elf- CFLAGS_ARCH += -march=rv32imac -mabi=ilp32 -mno-relax -# If Picolibc is available then select it explicitly. Ubuntu 24.04 ships its -# bare metal RISC-V toolchain with Picolibc rather than Newlib, and the default -# is "nosys" so a value must be provided. To avoid having per-distro -# workarounds, always select Picolibc if available. -PICOLIBC_SPECS := $(shell $(CROSS)gcc --print-file-name=picolibc.specs) -ifneq ($(PICOLIBC_SPECS),picolibc.specs) -CFLAGS_ARCH += -specs=$(PICOLIBC_SPECS) -USE_PICOLIBC := 1 -PICOLIBC_ARCH := rv32imac -PICOLIBC_ABI := ilp32 -endif - MICROPY_FLOAT_IMPL ?= none +PICOLIBC_BASE := riscv64-unknown-elf +PICOLIBC_TARGET := rv32imac/ilp32 else ifeq ($(ARCH),rv64imc) # rv64imc CROSS = riscv64-unknown-elf- CFLAGS_ARCH += -march=rv64imac -mabi=lp64 -mno-relax +MICROPY_FLOAT_IMPL ?= none +PICOLIBC_BASE := riscv64-unknown-elf +PICOLIBC_TARGET := rv64imac/lp64 + +else +$(error architecture '$(ARCH)' not supported) +endif + +ifneq ($(findstring -musl,$(shell $(CC) -dumpmachine)),) +USE_MUSL := 1 +endif + +ifeq ($(ARCH),$(filter $(ARCH),rv32imc rv64imc)) # If Picolibc is available then select it explicitly. Ubuntu 24.04 ships its # bare metal RISC-V toolchain with Picolibc rather than Newlib, and the default # is "nosys" so a value must be provided. To avoid having per-distro # workarounds, always select Picolibc if available. -PICOLIBC_SPECS := $(shell $(CROSS)gcc --print-file-name=picolibc.specs) +PICOLIBC_SPECS := $(shell $(CC) --print-file-name=picolibc.specs) ifneq ($(PICOLIBC_SPECS),picolibc.specs) +# LLVM toolchains supporting more than one target seem to ignore the `-march` +# flag passed when looking up the specs file, so if your system has Picolibc +# libraries for more than one architectures supported by the compiler the +# lookup will return the first available file. +# +# For example, on Ubuntu 24.02 if you have both `picolibc-aarch64-linux-gnu` +# and `picolibc-riscv64-unknown-elf` packages installed, the Qualcomm LLVM +# toolchain (which supports both AArch64 and RISC-V 64) will always return the +# AArch64 picolibc specs even when building for RISC-V. +ifeq ($(shell grep -q "$(PICOLIBC_BASE)" "$(PICOLIBC_SPECS)"; echo $$?),0) CFLAGS_ARCH += -specs=$(PICOLIBC_SPECS) USE_PICOLIBC := 1 -PICOLIBC_ARCH := rv64imac -PICOLIBC_ABI := lp64 endif - -MICROPY_FLOAT_IMPL ?= none - -else -$(error architecture '$(ARCH)' not supported) endif - -ifneq ($(findstring -musl,$(shell $(CROSS)gcc -dumpmachine)),) -USE_MUSL := 1 endif MICROPY_FLOAT_IMPL_UPPER = $(shell echo $(MICROPY_FLOAT_IMPL) | tr '[:lower:]' '[:upper:]') CFLAGS += $(CFLAGS_ARCH) -DMICROPY_FLOAT_IMPL=MICROPY_FLOAT_IMPL_$(MICROPY_FLOAT_IMPL_UPPER) +CFLAGS += $(CFLAGS_EXTRA) ifeq ($(LINK_RUNTIME),1) # All of these picolibc-specific directives are here to work around a -# limitation of Ubuntu 22.04's RISC-V bare metal toolchain. In short, the +# limitation of Ubuntu 24.04's RISC-V bare metal toolchain. In short, the # specific version of GCC in use (10.2.0) does not seem to take into account # extra paths provided by an explicitly passed specs file when performing name # resolution via `--print-file-name`. @@ -163,7 +168,7 @@ ifeq ($(LINK_RUNTIME),1) # flags that are passed to GCC. The `PICOLIBC_ROOT` environment variable is # checked to override the starting point for the library file search, and if # it is not set then the default value is used, assuming that this is running -# on an Ubuntu 22.04 machine. +# on an Ubuntu 24.04 machine. # # This should be revised when the CI base image is updated to a newer Ubuntu # version (that hopefully contains a newer RISC-V compiler) or to another Linux @@ -175,21 +180,37 @@ LIBM_NAME := libc.a else LIBM_NAME := libm.a endif -LIBGCC_PATH := $(realpath $(shell $(CROSS)gcc $(CFLAGS) --print-libgcc-file-name)) -LIBM_PATH := $(realpath $(shell $(CROSS)gcc $(CFLAGS) --print-file-name=$(LIBM_NAME))) +# Clang will output the path to libclang_rt.builtins.a instead. The problem is +# that some symbols are duplicated between the builtins library and libc.a. In +# these cases let's leave it to the user to figure out how to handle this for +# the time being. +TOOLCHAIN_LIBGCC := $(realpath $(shell $(CC) $(CFLAGS) --print-libgcc-file-name)) +ifneq ($(findstring clang,$(shell $(CC) --version)),clang) +LIBGCC_PATH = $(TOOLCHAIN_LIBGCC) +else +ifneq ($(LINK_CLANG_CLANGRT),0) +LIBGCC_PATH = $(TOOLCHAIN_LIBGCC) +else +LIBGCC_PATH = +endif +endif +LIBM_PATH := $(realpath $(shell $(CC) $(CFLAGS) --print-file-name=$(LIBM_NAME))) ifeq ($(USE_PICOLIBC),1) ifeq ($(LIBM_PATH),) -# The CROSS toolchain prefix usually ends with a dash, but that may not be -# always the case. If the prefix ends with a dash it has to be taken out as -# Picolibc's architecture directory won't have it in its name. GNU Make does -# not have any facility to perform character-level text manipulation so we -# shell out to sed. -CROSS_PREFIX := $(shell echo $(CROSS) | sed -e 's/-$$//') -PICOLIBC_ROOT ?= /usr/lib/picolibc/$(CROSS_PREFIX)/lib -LIBM_PATH := $(PICOLIBC_ROOT)/$(PICOLIBC_ARCH)/$(PICOLIBC_ABI)/$(LIBM_NAME) +PICOLIBC_ROOT ?= /usr/lib/picolibc/$(PICOLIBC_BASE)/lib +LIBM_PATH := $(PICOLIBC_ROOT)/$(PICOLIBC_TARGET)/$(LIBM_NAME) endif endif -MPY_LD_FLAGS += $(addprefix -l, $(LIBGCC_PATH) $(LIBM_PATH)) +ifneq ($(LINK_CLANG_LIBC),) +ifeq ($(findstring clang,$(shell $(CC) --version)),clang) +LIBC_PATH := $(realpath $(shell $(CC) $(CFLAGS) --print-file-name=libc.a)) +else +LIBC_PATH = +endif +else +LIBC_PATH = +endif +MPY_LD_FLAGS += $(addprefix -l, $(LIBGCC_PATH) $(LIBM_PATH) $(LIBC_PATH)) endif ifneq ($(MPY_EXTERN_SYM_FILE),) MPY_LD_FLAGS += --externs "$(realpath $(MPY_EXTERN_SYM_FILE))" @@ -198,8 +219,6 @@ ifneq ($(ARCH_FLAGS),) MPY_LD_FLAGS += --arch-flags "$(ARCH_FLAGS)" endif -CFLAGS += $(CFLAGS_EXTRA) - ################################################################################ # Build rules @@ -224,12 +243,12 @@ $(CONFIG_H): $(SRC) # Build .o from .c source files $(BUILD)/%.o: %.c $(CONFIG_H) Makefile $(ECHO) "CC $<" - $(Q)$(CROSS)gcc $(CFLAGS) -o $@ -c $< + $(Q)$(CC) $(CFLAGS) -o $@ -c $< # Build .o from .S source files $(BUILD)/%.o: %.S $(CONFIG_H) Makefile $(ECHO) "AS $<" - $(Q)$(CROSS)gcc $(CFLAGS) -o $@ -c $< + $(Q)$(CC) $(CFLAGS) -o $@ -c $< # Build .mpy from .py source files $(BUILD)/%.mpy: %.py @@ -237,11 +256,11 @@ $(BUILD)/%.mpy: %.py $(Q)$(MPY_CROSS) $(MPY_CROSS_FLAGS) -o $@ $< # Build native .mpy from object files -$(BUILD)/$(MOD).native.mpy: $(SRC_O) +$(BUILD)/$(MOD).mpy: $(SRC_O) $(ECHO) "LINK $<" $(Q)$(MPY_LD) --arch $(ARCH) --qstrs $(CONFIG_H) $(MPY_LD_FLAGS) -o $@ $^ # Build final .mpy from all intermediate .mpy files -$(MOD).mpy: $(BUILD)/$(MOD).native.mpy $(SRC_MPY) +$(MOD).mpy: $(BUILD)/$(MOD).mpy $(SRC_MPY) $(ECHO) "GEN $@" $(Q)$(MPY_TOOL) --merge -o $@ $^ diff --git a/py/emitbc.c b/py/emitbc.c index 0bcac8a1676..5f133ab070b 100644 --- a/py/emitbc.c +++ b/py/emitbc.c @@ -432,6 +432,12 @@ void mp_emit_bc_set_source_line(emit_t *emit, mp_uint_t source_line) { // If we compile with -O3, don't store line numbers. return; } + #if MICROPY_DYNAMIC_COMPILER + if (!mp_dynamic_compiler.include_source_lines) { + // Don't store line numbers if explicitly disabled. + return; + } + #endif if (source_line > emit->last_source_line) { mp_uint_t bytes_to_skip = emit->bytecode_offset - emit->last_source_line_offset; mp_uint_t lines_to_skip = source_line - emit->last_source_line; diff --git a/py/emitglue.c b/py/emitglue.c index 48834b3218d..3b479079af0 100644 --- a/py/emitglue.c +++ b/py/emitglue.c @@ -50,10 +50,6 @@ #define DEBUG_OP_printf(...) (void)0 #endif -#if MICROPY_DEBUG_PRINTERS -mp_uint_t mp_verbose_flag = 0; -#endif - mp_raw_code_t *mp_emit_glue_new_raw_code(void) { mp_raw_code_t *rc = m_new0(mp_raw_code_t, 1); rc->kind = MP_CODE_RESERVED; diff --git a/py/emitinlinerv32.c b/py/emitinlinerv32.c index e81b152087d..b175ae0ddc8 100644 --- a/py/emitinlinerv32.c +++ b/py/emitinlinerv32.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include @@ -505,7 +506,7 @@ static bool serialise_argument(emit_inline_asm_t *emit, const opcode_t *opcode, return false; } - mp_uint_t immediate = mp_obj_get_int_truncated(object) << shift; + mp_uint_t immediate = ((mp_uint_t)mp_obj_get_int_truncated(object)) << shift; if (kind & U) { if (!is_in_unsigned_mask(mask, immediate)) { goto out_of_range; @@ -693,6 +694,192 @@ static void handle_opcode(emit_inline_asm_t *emit, const opcode_t *opcode_data, } } +static bool extract_register_list(emit_inline_asm_t *emit, qstr opcode, mp_parse_node_t node, mp_uint_t *reglist) { + assert(reglist != NULL && "Register list pointer is NULL."); + + // As per §28.9, valid register list values are as follows: + // + // {ra}, {ra, s0}, {ra, s0-s1}, {ra, s0-s2}, ..., {ra, s0-s8}, + // {ra, s0-s9}, {ra, s0-s11} + // + // {ra, s0-s10} is *not* valid + + // case 1: {ra} + // PN_atom_brace { ID("ra") } + // + // case 2: {ra,s0} -> + // PN_atom_brace { PN_dictorsetmaker { ID("ra") + // PN_dictorsetmaker_list { ID("s0") } } } + // + // case 3: {ra,s0-s1} -> + // PN_atom_brace { PN_dictorsetmaker { ID("ra") + // PN_dictorsetmaker_list { PN_arith_expr { + // ID("s0") TOKEN(MP_TOKEN_OP_MINUS) ID("s1") } } } } + + if (!MP_PARSE_NODE_IS_STRUCT_KIND(node, PN_atom_brace) || + MP_PARSE_NODE_STRUCT_NUM_NODES((mp_parse_node_struct_t *)node) != 1) { + return false; + } + + mp_parse_node_struct_t *nodes = (mp_parse_node_struct_t *)node; + mp_uint_t register_id = 0; + + if (MP_PARSE_NODE_IS_ID(nodes->nodes[0])) { + if (!parse_register_node(nodes->nodes[0], ®ister_id, false)) { + return false; + } + *reglist = 4; + return register_id == ASM_RV32_REG_RA; + } + + if (!MP_PARSE_NODE_IS_STRUCT_KIND(nodes->nodes[0], PN_dictorsetmaker) || + MP_PARSE_NODE_STRUCT_NUM_NODES((mp_parse_node_struct_t *)nodes->nodes[0]) != 2) { + return false; + } + nodes = (mp_parse_node_struct_t *)nodes->nodes[0]; + if (!MP_PARSE_NODE_IS_ID(nodes->nodes[0]) || + !MP_PARSE_NODE_IS_STRUCT_KIND(nodes->nodes[1], PN_dictorsetmaker_list) || + !parse_register_node(nodes->nodes[0], ®ister_id, false) || + register_id != ASM_RV32_REG_RA) { + return false; + } + mp_parse_node_t *list_nodes; + size_t list_nodes_count = mp_parse_node_extract_list(&nodes->nodes[1], PN_dictorsetmaker_list2, &list_nodes); + if (list_nodes_count != 1 || !MP_PARSE_NODE_IS_STRUCT_KIND(list_nodes[0], PN_dictorsetmaker_list)) { + return false; + } + nodes = (mp_parse_node_struct_t *)list_nodes[0]; + if (MP_PARSE_NODE_STRUCT_NUM_NODES(nodes) != 1) { + return false; + } + if (MP_PARSE_NODE_IS_ID(nodes->nodes[0])) { + if (!parse_register_node(nodes->nodes[0], ®ister_id, false) || + register_id != ASM_RV32_REG_S0) { + return false; + } + *reglist = 5; + return true; + } + + if (MP_PARSE_NODE_IS_STRUCT_KIND(nodes->nodes[0], PN_arith_expr)) { + nodes = (mp_parse_node_struct_t *)nodes->nodes[0]; + if (MP_PARSE_NODE_STRUCT_NUM_NODES(nodes) != 3 || + !MP_PARSE_NODE_IS_ID(nodes->nodes[0]) || + !MP_PARSE_NODE_IS_TOKEN_KIND(nodes->nodes[1], MP_TOKEN_OP_MINUS) || + !MP_PARSE_NODE_IS_ID(nodes->nodes[2])) { + return false; + } + if (!parse_register_node(nodes->nodes[0], ®ister_id, false) || + register_id != ASM_RV32_REG_S0) { + return false; + } + if (!parse_register_node(nodes->nodes[2], ®ister_id, false) || + register_id == ASM_RV32_REG_S10) { + return false; + } + if (register_id == ASM_RV32_REG_S1) { + *reglist = 6; + return true; + } + if (register_id >= ASM_RV32_REG_S2 && register_id <= ASM_RV32_REG_S11) { + *reglist = 7 + MIN(register_id, ASM_RV32_REG_S10) - ASM_RV32_REG_S2; + return true; + } + } + + return false; +} + +static const qstr_short_t ZCMP_OPCODE_NAMES[] = { + MP_QSTR_cm_push, MP_QSTR_cm_pop, MP_QSTR_cm_popret, + MP_QSTR_cm_popretz, MP_QSTR_cm_mva01s, MP_QSTR_cm_mvsa01, +}; + +static const void *ZCMP_OPCODE_HANDLERS[] = { + asm_rv32_opcode_cmpush, asm_rv32_opcode_cmpop, + asm_rv32_opcode_cmpopret, asm_rv32_opcode_cmpopretz, + asm_rv32_opcode_cmmva01s, asm_rv32_opcode_cmmvsa01, +}; + +typedef void (*call_xi_t)(asm_rv32_t *state, mp_uint_t register_list, mp_int_t adjustment); +typedef void (*call_ss_t)(asm_rv32_t *state, mp_uint_t r1, mp_uint_t r2); + +static bool handle_zcmp_opcode(emit_inline_asm_t *emit, qstr opcode, mp_parse_node_t *argument_nodes) { + mp_uint_t argument_index = 0; + + for (size_t index = 0; index < MP_ARRAY_SIZE(ZCMP_OPCODE_NAMES); index++) { + if (ZCMP_OPCODE_NAMES[index] != opcode) { + continue; + } + const void *handler = ZCMP_OPCODE_HANDLERS[index]; + if (opcode == MP_QSTR_cm_mva01s || opcode == MP_QSTR_cm_mvsa01) { + mp_uint_t register_lhs = 0; + mp_uint_t register_rhs = 0; + + if (!parse_register_node(argument_nodes[0], ®ister_lhs, false) || + ((1U << register_lhs) & 0x00FC0300) == 0) { + goto invalid_s_register; + } + + if (!parse_register_node(argument_nodes[1], ®ister_rhs, false) || + ((1U << register_rhs) & 0x00FC0300) == 0) { + argument_index = 1; + goto invalid_s_register; + } + + if (register_lhs == register_rhs) { + emit_inline_rv32_error_exc(emit, + mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, + MP_ERROR_TEXT("opcode '%q': registers must be different"), + opcode)); + return false; + } + + ((call_ss_t)handler)(&emit->as, register_lhs, register_rhs); + return true; + } + + mp_uint_t register_list; + if (!extract_register_list(emit, opcode, argument_nodes[0], ®ister_list)) { + emit_inline_rv32_error_exc(emit, + mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, + MP_ERROR_TEXT("opcode '%q': malformed register list"), + opcode)); + return false; + } + + mp_obj_t stack_adjustment_object; + if (!mp_parse_node_get_int_maybe(argument_nodes[1], &stack_adjustment_object)) { + emit_inline_rv32_error_exc(emit, + mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, + ET_WRONG_ARGUMENT_KIND, opcode, 2, MP_QSTR_integer)); + return false; + } + mp_int_t stack_adjustment = mp_obj_get_int(stack_adjustment_object); + // Either 0, 16, 32, or 48. + if ((abs((int32_t)stack_adjustment) & ~0x30U) != 0 || + ((opcode == MP_QSTR_cm_push) && stack_adjustment > 0) || + ((opcode != MP_QSTR_cm_push) && stack_adjustment < 0)) { + emit_inline_rv32_error_exc(emit, + mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, + MP_ERROR_TEXT("opcode '%q': invalid stack adjustment"), + opcode)); + return false; + } + ((call_xi_t)handler)(&emit->as, register_list, abs((int32_t)stack_adjustment)); + return true; + } + + return false; + +invalid_s_register: + emit_inline_rv32_error_exc(emit, + mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, + MP_ERROR_TEXT("opcode '%q' argument %d: wrong register(s)"), + opcode, argument_index + 1)); + return false; +} + static void emit_inline_rv32_opcode(emit_inline_asm_t *emit, qstr opcode, mp_uint_t arguments_count, mp_parse_node_t *argument_nodes) { const opcode_t *opcode_data = NULL; for (mp_uint_t index = 0; index < MP_ARRAY_SIZE(OPCODES); index++) { @@ -702,6 +889,10 @@ static void emit_inline_rv32_opcode(emit_inline_asm_t *emit, qstr opcode, mp_uin } } + if ((asm_rv32_allowed_extensions() & RV32_EXT_ZCMP) && !opcode_data && (arguments_count == 2) && handle_zcmp_opcode(emit, opcode, argument_nodes)) { + return; + } + if (!opcode_data || (asm_rv32_allowed_extensions() & opcode_data->required_extensions) != opcode_data->required_extensions) { emit_inline_rv32_error_exc(emit, mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, MP_ERROR_TEXT("invalid RV32 instruction '%q'"), opcode)); diff --git a/py/emitinlinethumb.c b/py/emitinlinethumb.c index d6596337ae5..9c82921edfc 100644 --- a/py/emitinlinethumb.c +++ b/py/emitinlinethumb.c @@ -364,24 +364,15 @@ static int get_arg_label(emit_inline_asm_t *emit, const char *op, mp_parse_node_ return 0; } -typedef struct _cc_name_t { byte cc; - byte name[2]; -} cc_name_t; -static const cc_name_t cc_name_table[] = { - { ASM_THUMB_CC_EQ, { 'e', 'q' }}, - { ASM_THUMB_CC_NE, { 'n', 'e' }}, - { ASM_THUMB_CC_CS, { 'c', 's' }}, - { ASM_THUMB_CC_CC, { 'c', 'c' }}, - { ASM_THUMB_CC_MI, { 'm', 'i' }}, - { ASM_THUMB_CC_PL, { 'p', 'l' }}, - { ASM_THUMB_CC_VS, { 'v', 's' }}, - { ASM_THUMB_CC_VC, { 'v', 'c' }}, - { ASM_THUMB_CC_HI, { 'h', 'i' }}, - { ASM_THUMB_CC_LS, { 'l', 's' }}, - { ASM_THUMB_CC_GE, { 'g', 'e' }}, - { ASM_THUMB_CC_LT, { 'l', 't' }}, - { ASM_THUMB_CC_GT, { 'g', 't' }}, - { ASM_THUMB_CC_LE, { 'l', 'e' }}, +#define ENCODE_CC(c1, c2) (((uint16_t)(c1) << 8) | (uint16_t)(c2)) + +// Positions in the table match the condition code value. +static const uint16_t CONDITION_CODES[] = { + ENCODE_CC('e', 'q'), ENCODE_CC('n', 'e'), ENCODE_CC('c', 's'), + ENCODE_CC('c', 'c'), ENCODE_CC('m', 'i'), ENCODE_CC('p', 'l'), + ENCODE_CC('v', 's'), ENCODE_CC('v', 'c'), ENCODE_CC('h', 'i'), + ENCODE_CC('l', 's'), ENCODE_CC('g', 'e'), ENCODE_CC('l', 't'), + ENCODE_CC('g', 't'), ENCODE_CC('l', 'e'), }; typedef struct _format_4_op_t { byte op; @@ -574,9 +565,11 @@ static void emit_inline_thumb_op(emit_inline_asm_t *emit, qstr op, mp_uint_t n_a || (op_len == 5 && op_str[3] == '_' && (op_str[4] == 'n' || (ARMV7M && op_str[4] == 'w'))))) { mp_uint_t cc = -1; - for (mp_uint_t i = 0; i < MP_ARRAY_SIZE(cc_name_table); i++) { - if (op_str[1] == cc_name_table[i].name[0] && op_str[2] == cc_name_table[i].name[1]) { - cc = cc_name_table[i].cc; + uint16_t condition_code = ENCODE_CC(op_str[1], op_str[2]); + for (size_t i = 0; i < MP_ARRAY_SIZE(CONDITION_CODES); ++i) { + if (condition_code == CONDITION_CODES[i]) { + cc = i; + break; } } if (cc == (mp_uint_t)-1) { @@ -592,12 +585,14 @@ static void emit_inline_thumb_op(emit_inline_asm_t *emit, qstr op, mp_uint_t n_a } } else if (ARMV7M && op_str[0] == 'i' && op_str[1] == 't') { const char *arg_str = get_arg_str(pn_args[0]); + if (strlen(arg_str) != 2) { + goto unknown_op; + } mp_uint_t cc = -1; - for (mp_uint_t i = 0; i < MP_ARRAY_SIZE(cc_name_table); i++) { - if (arg_str[0] == cc_name_table[i].name[0] - && arg_str[1] == cc_name_table[i].name[1] - && arg_str[2] == '\0') { - cc = cc_name_table[i].cc; + uint16_t condition_code = ENCODE_CC(arg_str[0], arg_str[1]); + for (size_t i = 0; i < MP_ARRAY_SIZE(CONDITION_CODES); ++i) { + if (condition_code == CONDITION_CODES[i]) { + cc = i; break; } } diff --git a/py/emitnative.c b/py/emitnative.c index 06c2f25b60c..9ac62f6d362 100644 --- a/py/emitnative.c +++ b/py/emitnative.c @@ -246,6 +246,7 @@ typedef enum { VTYPE_BUILTIN_CAST = 0x70 | MP_NATIVE_TYPE_OBJ, } vtype_kind_t; +#if MICROPY_ERROR_REPORTING != MICROPY_ERROR_REPORTING_NONE static qstr vtype_to_qstr(vtype_kind_t vtype) { switch (vtype) { case VTYPE_PYOBJ: @@ -269,6 +270,7 @@ static qstr vtype_to_qstr(vtype_kind_t vtype) { return MP_QSTR_None; } } +#endif typedef struct _stack_info_t { vtype_kind_t vtype; @@ -2378,22 +2380,23 @@ static void emit_native_unary_op(emit_t *emit, mp_unary_op_t op) { if (op == MP_UNARY_OP_POSITIVE) { // No-operation, just leave the argument on the stack. } else if (op == MP_UNARY_OP_NEGATIVE) { - int reg = REG_RET; - emit_pre_pop_reg_flexible(emit, &vtype, ®, reg, reg); - ASM_NEG_REG(emit->as, reg); - emit_post_push_reg(emit, vtype, reg); + emit_pre_pop_reg(emit, &vtype, REG_RET); + ASM_NEG_REG(emit->as, REG_RET); + emit_post_push_reg(emit, vtype, REG_RET); } else if (op == MP_UNARY_OP_INVERT) { + emit_pre_pop_reg(emit, &vtype, REG_RET); #ifdef ASM_NOT_REG - int reg = REG_RET; - emit_pre_pop_reg_flexible(emit, &vtype, ®, reg, reg); - ASM_NOT_REG(emit->as, reg); + ASM_NOT_REG(emit->as, REG_RET); + #else + #if REG_RET != REG_ARG_1 + int reg = REG_ARG_1; #else - int reg = REG_RET; - emit_pre_pop_reg_flexible(emit, &vtype, ®, REG_ARG_1, reg); - ASM_MOV_REG_IMM(emit->as, REG_ARG_1, -1); - ASM_XOR_REG_REG(emit->as, reg, REG_ARG_1); + int reg = REG_ARG_2; #endif - emit_post_push_reg(emit, vtype, reg); + ASM_MOV_REG_IMM(emit->as, reg, -1); + ASM_XOR_REG_REG(emit->as, REG_RET, reg); + #endif + emit_post_push_reg(emit, vtype, REG_RET); } else { EMIT_NATIVE_VIPER_TYPE_ERROR(emit, MP_ERROR_TEXT("'not' not implemented"), mp_binary_op_method_name[op]); @@ -2976,8 +2979,9 @@ static void emit_native_return_value(emit_t *emit) { static void emit_native_raise_varargs(emit_t *emit, mp_uint_t n_args) { DEBUG_printf("raise_varargs(%d)\n", n_args); - (void)n_args; - assert(n_args == 1); + if (n_args != 1) { + mp_raise_NotImplementedError(MP_ERROR_TEXT("native raise")); + } vtype_kind_t vtype_exc; emit_pre_pop_reg(emit, &vtype_exc, REG_ARG_1); // arg1 = object to raise if (vtype_exc != VTYPE_PYOBJ) { diff --git a/py/emitndebug.c b/py/emitndebug.c index 2144d14e6b5..27c3bbdcef7 100644 --- a/py/emitndebug.c +++ b/py/emitndebug.c @@ -134,7 +134,7 @@ static void asm_debug_reg_imm(asm_debug_t *as, const char *op, int reg, int imm) #if !MICROPY_PERSISTENT_CODE_SAVE static void asm_debug_reg_qstr(asm_debug_t *as, const char *op, int reg, int qst) { - asm_debug_printf(as, "%s(%s, %s)\n", op, reg_name_table[reg], qstr_str(qst)); + asm_debug_printf(as, "%s(%s, %q)\n", op, reg_name_table[reg], (qstr)qst); } #endif diff --git a/py/gc.c b/py/gc.c index 38c169ccafd..57bad018e76 100644 --- a/py/gc.c +++ b/py/gc.c @@ -294,6 +294,10 @@ static void gc_setup_area(mp_state_mem_area_t *area, void *start, void *end) { #if MICROPY_GC_SPLIT_HEAP area->next = NULL; + + // Update the global min/max region that covers all heaps + MP_STATE_MEM(area_pool_min) = MIN(MP_STATE_MEM(area_pool_min), area->gc_pool_start); + MP_STATE_MEM(area_pool_max) = MAX(MP_STATE_MEM(area_pool_max), area->gc_pool_end); #endif DEBUG_printf("GC layout:\n"); @@ -329,6 +333,13 @@ void gc_init(void *start, void *end) { end = (void *)((uintptr_t)end & (~(BYTES_PER_BLOCK - 1))); DEBUG_printf("Initializing GC heap: %p..%p = " UINT_FMT " bytes\n", start, end, (byte *)end - (byte *)start); + #if MICROPY_GC_SPLIT_HEAP + // Note: min/max are deliberately swapped here, gc_setup_area() will update them to + // the correct values for the min/max of the first actual pool region + MP_STATE_MEM(area_pool_min) = end; + MP_STATE_MEM(area_pool_max) = start; + #endif + gc_setup_area(&MP_STATE_MEM(area), start, end); // set last free ATB index to start of heap @@ -529,12 +540,24 @@ bool gc_ptr_on_heap(const void *ptr) { } #if MICROPY_GC_SPLIT_HEAP -// Returns the area to which this pointer belongs, or NULL if it isn't -// allocated on the GC-managed heap. -static inline mp_state_mem_area_t *gc_get_ptr_area(const void *ptr) { - if (((uintptr_t)(ptr) & (BYTES_PER_BLOCK - 1)) != 0) { // must be aligned on a block - return NULL; +static mp_state_mem_area_t *gc_get_ptr_area(const void *ptr); + +// Returns the area to which this arbitrary pointer belongs, or NULL if it isn't +// allocated on the GC-managed heap. Contains "fast path" inline checks for invalid +// data which isn't a pointer to the heap. Equivalent of VERIFY_PTR for the non-split-heap case. +static inline MP_ALWAYSINLINE mp_state_mem_area_t *gc_verify_ptr_get_area(const void *ptr) { + // These inline checks are similar to VERIFY_PTR macro, below + if ((byte *)ptr < MP_STATE_MEM(area_pool_min) || (byte *)ptr > MP_STATE_MEM(area_pool_max)) { + return NULL; // not in the overall pool region + } + if (((uintptr_t)(ptr) & (BYTES_PER_BLOCK - 1)) != 0) { + return NULL; // not aligned on a block boundary } + return gc_get_ptr_area(ptr); +} + +// Returns the area to which a pointer belongs. Assumes pointer is valid to a heap block. +static mp_state_mem_area_t *gc_get_ptr_area(const void *ptr) { for (mp_state_mem_area_t *area = &MP_STATE_MEM(area); area != NULL; area = NEXT_AREA(area)) { if (ptr >= (void *)area->gc_pool_start // must be above start of pool && ptr < (void *)area->gc_pool_end) { // must be below end of pool @@ -543,7 +566,7 @@ static inline mp_state_mem_area_t *gc_get_ptr_area(const void *ptr) { } return NULL; } -#endif +#else // ptr should be of type void* #define VERIFY_PTR(ptr) ( \ @@ -552,6 +575,8 @@ static inline mp_state_mem_area_t *gc_get_ptr_area(const void *ptr) { && ptr < (void *)MP_STATE_MEM(area).gc_pool_end /* must be below end of pool */ \ ) +#endif + #ifdef TRACE_MARK #error "TRACE_MARK is replaced by TRACE_MARK_R and TRACE_MARK_S" #endif @@ -607,7 +632,7 @@ void gc_collect_root(void **ptrs, size_t len) { MICROPY_GC_HOOK_LOOP(i); void *ptr = gc_get_ptr(ptrs, i); #if MICROPY_GC_SPLIT_HEAP - mp_state_mem_area_t *area = gc_get_ptr_area(ptr); + mp_state_mem_area_t *area = gc_verify_ptr_get_area(ptr); if (!area) { continue; } @@ -676,7 +701,7 @@ static void MP_NO_INSTRUMENT PLACE_IN_ITCM(gc_mark_subtree)(size_t block) // If this is a heap pointer that hasn't been marked, mark it and push // it's children to the stack. #if MICROPY_GC_SPLIT_HEAP - mp_state_mem_area_t *ptr_area = gc_get_ptr_area(ptr); + mp_state_mem_area_t *ptr_area = gc_verify_ptr_get_area(ptr); if (!ptr_area) { // Not a heap-allocated pointer (might even be random data). continue; @@ -983,6 +1008,25 @@ void gc_info(gc_info_t *info) { GC_EXIT(); } +// Fast version of gc_info that only computes total/used/free. +void gc_info_fast(gc_info_t *info) { + GC_ENTER(); + memset(info, 0, sizeof(*info)); + const uint8_t lut[16] = {2, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0}; + for (mp_state_mem_area_t *area = &MP_STATE_MEM(area); area != NULL; area = NEXT_AREA(area)) { + size_t free_blocks = 0; + info->total += area->gc_pool_end - area->gc_pool_start; + for (size_t i = 0; i < area->gc_alloc_table_byte_len; i++) { + uint8_t atb = area->gc_alloc_table_start[i]; + free_blocks += lut[atb & 0xF] + lut[atb >> 4]; + } + info->free += free_blocks; + } + info->free *= BYTES_PER_BLOCK; + info->used = info->total - info->free; + GC_EXIT(); +} + #if MICROPY_PY_WEAKREF // Mark the GC heap pointer as having a weakref. void gc_weakref_mark(void *ptr) { diff --git a/py/gc.h b/py/gc.h index f9028d637c5..0f693d49222 100644 --- a/py/gc.h +++ b/py/gc.h @@ -108,6 +108,7 @@ typedef struct _gc_info_t { } gc_info_t; void gc_info(gc_info_t *info); +void gc_info_fast(gc_info_t *info); void gc_dump_info(const mp_print_t *print); void gc_dump_alloc_table(const mp_print_t *print); diff --git a/py/makeqstrdefs.py b/py/makeqstrdefs.py index 8c07899baf8..3bc81bf5961 100644 --- a/py/makeqstrdefs.py +++ b/py/makeqstrdefs.py @@ -94,10 +94,36 @@ def preprocess(): except OSError: pass + # These regex's are used to filter the preprocessed data, keeping only those lines + # that are subsequently needed by the `process_file` step. The regexs are kept + # short so they are as efficient as possible. (The stm32 port needs symbols of the + # form `micropy_hw_xxx` so they are also kept.) + re_line_file = re.compile(rb"^#(?:line)?\s+\d+\s\"") + re_mp_info = re.compile(rb"MP_COMP|MP_QSTR|MP_REGI|micropy_hw") + def pp(flags): def run(files): try: - return subprocess.check_output(args.pp + flags + files) + filtered_lines = [] + cmd = args.pp + flags + files + with subprocess.Popen(cmd, stdout=subprocess.PIPE) as proc: + for line in proc.stdout: + if line.isspace(): + pass + elif re_line_file.match(line): + # CIRCUITPY-CHANGE: keep every file marker, not just + # the one preceding an MP symbol. CircuitPython + # attributes MP_COMPRESSED_ROM_TEXT found in headers + # to the enclosing C source, so process_file needs + # the C source markers even when the symbol itself + # comes from a header. + filtered_lines.append(line) + elif re_mp_info.search(line): + filtered_lines.append(line) + proc.wait() + if proc.returncode: + raise PreprocessorError("command failed: " + " ".join(cmd)) + return b"".join(filtered_lines) except subprocess.CalledProcessError as er: raise PreprocessorError(str(er)) diff --git a/py/malloc.c b/py/malloc.c index fc9795043a8..5882a463323 100644 --- a/py/malloc.c +++ b/py/malloc.c @@ -273,6 +273,15 @@ typedef struct _m_tracked_node_t { uint8_t data[]; } m_tracked_node_t; +// Helper to get data size of a tracked node, abstracting MICROPY_TRACKED_ALLOC_STORE_SIZE. +static inline size_t get_tracked_node_size(m_tracked_node_t *node) { + #if MICROPY_TRACKED_ALLOC_STORE_SIZE + return node->size; + #else + return gc_nbytes(node) - sizeof(m_tracked_node_t); + #endif +} + #if MICROPY_DEBUG_VERBOSE static size_t m_tracked_count_links(size_t *nb) { m_tracked_node_lock(); @@ -320,6 +329,42 @@ void *m_tracked_calloc(size_t nmemb, size_t size) { return &node->data[0]; } +void *m_tracked_realloc(void *ptr_in, size_t n_bytes) { + // Handle pure allocation + if (ptr_in == NULL) { + return m_tracked_calloc(1, n_bytes); + } + + // Handle pure free + if (n_bytes == 0) { + m_tracked_free(ptr_in); + return NULL; + } + // To keep the implementation simple, we always allocate a new buffer and copy the old data into it. + // This could be optimised if faster performance or lower worst-case memory usage is required. + + // Get old size + // CIRCUITPY-CHANGE: cast to avoid compiler warning + m_tracked_node_t *old_node = (m_tracked_node_t *)(void *)((uint8_t *)ptr_in - sizeof(m_tracked_node_t)); + size_t old_size = get_tracked_node_size(old_node); + + // Allocate new buffer + void *new_ptr = m_tracked_calloc(1, n_bytes); + if (new_ptr == NULL) { + // Allocation failed, return NULL but leave original pointer intact + return NULL; + } + + // Copy data (minimum of old and new size) + size_t copy_size = MIN(old_size, n_bytes); + memcpy(new_ptr, ptr_in, copy_size); + + // Free old buffer + m_tracked_free(ptr_in); + + return new_ptr; +} + void m_tracked_free(void *ptr_in) { if (ptr_in == NULL) { return; @@ -328,11 +373,7 @@ void m_tracked_free(void *ptr_in) { m_tracked_node_t *node = (m_tracked_node_t *)(void *)((uint8_t *)ptr_in - sizeof(m_tracked_node_t)); #if MICROPY_DEBUG_VERBOSE size_t data_bytes; - #if MICROPY_TRACKED_ALLOC_STORE_SIZE - data_bytes = node->size; - #else - data_bytes = gc_nbytes(node); - #endif + data_bytes = get_tracked_node_size(node); size_t nb; size_t n = m_tracked_count_links(&nb); DEBUG_printf("m_tracked_free(%p, [%p, %p], nbytes=%u, links=%u;%u)\n", node, node->prev, node->next, (int)data_bytes, (int)n, (int)nb); diff --git a/py/manifest.cmake b/py/manifest.cmake new file mode 100644 index 00000000000..446fb063b7e --- /dev/null +++ b/py/manifest.cmake @@ -0,0 +1,73 @@ +# Extract c_module() entries from MICROPY_FROZEN_MANIFEST and append them to +# USER_C_MODULES. Included from py/usermod.cmake; the port must have resolved +# MICROPY_FROZEN_MANIFEST (and any board-config default / cmake-command-line +# override) before usermod.cmake runs. +# +# Also sets default MICROPY_MANIFEST_* path variables that are forwarded to +# makemanifest.py here for c_module extraction and again from py/mkrules.cmake +# for the frozen content generation step, so both passes substitute the same +# values into manifest path expressions. + +if(MICROPY_FROZEN_MANIFEST) + # Set default path variables to be passed to makemanifest.py. These are + # available in path substitutions inside the manifest. Additional variables + # can be set per-board in mpconfigboard.cmake or on the cmake command line. + # MICROPY_LIB_DIR is set in py.cmake (included before usermod.cmake). + if(NOT DEFINED MICROPY_MANIFEST_PORT_DIR) + set(MICROPY_MANIFEST_PORT_DIR ${MICROPY_PORT_DIR}) + endif() + if(NOT DEFINED MICROPY_MANIFEST_BOARD_DIR) + set(MICROPY_MANIFEST_BOARD_DIR ${MICROPY_BOARD_DIR}) + endif() + if(NOT DEFINED MICROPY_MANIFEST_MPY_DIR) + set(MICROPY_MANIFEST_MPY_DIR ${MICROPY_DIR}) + endif() + if(NOT DEFINED MICROPY_MANIFEST_MPY_LIB_DIR) + set(MICROPY_MANIFEST_MPY_LIB_DIR ${MICROPY_LIB_DIR}) + endif() + + # Find all MICROPY_MANIFEST_* variables and turn them into command line + # arguments. Stored in MICROPY_MAKEMANIFEST_ARGS (outside the + # MICROPY_MANIFEST_ namespace) so it doesn't self-match the regex above + # on subsequent cmake configures. + get_cmake_property(_manifest_vars VARIABLES) + list(FILTER _manifest_vars INCLUDE REGEX "MICROPY_MANIFEST_.*") + set(MICROPY_MAKEMANIFEST_ARGS) + foreach(_manifest_var IN LISTS _manifest_vars) + list(APPEND MICROPY_MAKEMANIFEST_ARGS "-v") + string(REGEX REPLACE "MICROPY_MANIFEST_(.*)" "\\1" _manifest_var_name ${_manifest_var}) + list(APPEND MICROPY_MAKEMANIFEST_ARGS "${_manifest_var_name}=${${_manifest_var}}") + endforeach() + + # Skip extraction during UPDATE_SUBMODULES (require() would fail) and when + # micropython-lib is not initialised. Matches py/manifest.mk behaviour: + # warn and continue with an empty c_module list rather than fatal. + if(EXISTS ${MICROPY_FROZEN_MANIFEST} AND NOT UPDATE_SUBMODULES) + if(EXISTS ${MICROPY_LIB_DIR}/README.md) + if(NOT Python3_EXECUTABLE) + find_package(Python3 REQUIRED COMPONENTS Interpreter) + endif() + + execute_process( + COMMAND "${Python3_EXECUTABLE}" "${MICROPY_DIR}/tools/makemanifest.py" + --list-c-modules ${MICROPY_MAKEMANIFEST_ARGS} "${MICROPY_FROZEN_MANIFEST}" + OUTPUT_VARIABLE MANIFEST_C_MODULES + ERROR_VARIABLE MANIFEST_ERROR + OUTPUT_STRIP_TRAILING_WHITESPACE + RESULT_VARIABLE MANIFEST_RESULT + ) + + if(NOT MANIFEST_RESULT EQUAL 0) + message(FATAL_ERROR "Failed to extract C modules from manifest: ${MICROPY_FROZEN_MANIFEST}\nError: ${MANIFEST_ERROR}") + endif() + + if(MANIFEST_C_MODULES) + string(REPLACE "\n" ";" MANIFEST_C_MODULES_LIST "${MANIFEST_C_MODULES}") + list(APPEND USER_C_MODULES ${MANIFEST_C_MODULES_LIST}) + list(REMOVE_DUPLICATES USER_C_MODULES) + endif() + else() + message(WARNING "c_module() extraction skipped: micropython-lib not initialised, run 'make BOARD=${MICROPY_BOARD} submodules'") + endif() + endif() +endif() diff --git a/py/manifest.mk b/py/manifest.mk new file mode 100644 index 00000000000..43ad64be34e --- /dev/null +++ b/py/manifest.mk @@ -0,0 +1,58 @@ +# Manifest processing for freeze(), require(), include(), and c_module(). +# This file handles both C module extraction and frozen Python content generation. + +ifneq ($(FROZEN_MANIFEST),) + +# Set default path variables to be passed to makemanifest.py. These will be +# available in path substitutions. Additional variables can be set per-board +# in mpconfigboard.mk or on the make command line. +MICROPY_MANIFEST_PORT_DIR ?= $(CURDIR) +MICROPY_MANIFEST_BOARD_DIR ?= $(BOARD_DIR) +MICROPY_MANIFEST_MPY_DIR ?= $(TOP) +MICROPY_MANIFEST_MPY_LIB_DIR ?= $(MPY_LIB_DIR) + +# Find all MICROPY_MANIFEST_* variables and turn them into command line +# arguments. Values must not contain whitespace (GNU make word-splits). +MANIFEST_VARIABLES = $(foreach var,$(filter MICROPY_MANIFEST_%, $(.VARIABLES)),-v "$(subst MICROPY_MANIFEST_,,$(var))=$($(var))") + +# Skip c_module extraction for targets that don't need a build. +MANIFEST_NON_BUILD_GOALS := clean clean-prog submodules help print-cfg print-def +MANIFEST_GOALS := $(filter-out $(MANIFEST_NON_BUILD_GOALS),$(or $(MAKECMDGOALS),build)) + +# Extract C module paths up front so downstream rules see them. Skipped when +# micropython-lib isn't initialised (require() would fail) or for non-build +# targets. $(shell) collapses newlines, so c_module() paths cannot contain +# whitespace on the make side (the cmake side handles it correctly). +MANIFEST_C_MODULES := +ifneq ($(MANIFEST_GOALS),) +ifeq ($(wildcard $(FROZEN_MANIFEST)),$(FROZEN_MANIFEST)) +ifeq ($(wildcard $(MPY_LIB_DIR)/README.md),$(MPY_LIB_DIR)/README.md) +MANIFEST_C_MODULES := $(shell $(MAKE_MANIFEST) --list-c-modules $(MANIFEST_VARIABLES) $(FROZEN_MANIFEST)) +# Treat a populated, non-zero .SHELLSTATUS as failure. GNU make < 4.2 (eg +# macOS 3.81) leaves .SHELLSTATUS empty, in which case skip the check; an +# empty filter result means "succeed (status was 0, or unavailable)". +ifneq ($(filter-out 0,$(.SHELLSTATUS)),) +$(error makemanifest.py --list-c-modules failed (exit $(.SHELLSTATUS)) for $(FROZEN_MANIFEST)) +endif +else +$(warning c_module() extraction skipped: micropython-lib not initialised, run 'make submodules') +endif +endif +endif + +# Merge manifest c_modules into USER_C_MODULES. $(sort) also de-duplicates; +# this changes USER_C_MODULES iteration order vs cmake's list(REMOVE_DUPLICATES) +# which preserves first occurrence. Module include order should not be +# behaviour-sensitive so this is acceptable. +USER_C_MODULES := $(sort $(USER_C_MODULES) $(MANIFEST_C_MODULES)) + +# Frozen content rule, requires $(BUILD) and other variables from the port. +ifdef BUILD +# CIRCUITPY-CHANGE: FROZEN_MANIFEST is generated into $(BUILD) during the build +# (py/circuitpy_mpconfig.mk), so it must be a prerequisite here. +$(BUILD)/frozen_content.c: FORCE $(BUILD)/genhdr/qstrdefs.generated.h $(BUILD)/genhdr/root_pointers.h $(FROZEN_MANIFEST) | $(MICROPY_MPYCROSS_DEPENDENCY) + $(Q)test -e "$(MPY_LIB_DIR)/README.md" || (echo -e $(HELP_MPY_LIB_SUBMODULE); false) + $(Q)$(MAKE_MANIFEST) -o $@ $(MANIFEST_VARIABLES) -b "$(BUILD)" $(if $(MPY_CROSS_FLAGS),-f"$(MPY_CROSS_FLAGS)",) --mpy-tool-flags="$(MPY_TOOL_FLAGS)" $(FROZEN_MANIFEST) +endif + +endif # FROZEN_MANIFEST diff --git a/py/misc.h b/py/misc.h index d85a8f34289..a7498e66d17 100644 --- a/py/misc.h +++ b/py/misc.h @@ -165,6 +165,7 @@ MP_NORETURN void m_malloc_fail(size_t num_bytes); // These alloc/free functions track the pointers in a linked list so the GC does not reclaim // them. They can be used by code that requires traditional C malloc/free semantics. void *m_tracked_calloc(size_t nmemb, size_t size); +void *m_tracked_realloc(void *ptr_in, size_t n_bytes); void m_tracked_free(void *ptr_in); #endif @@ -291,8 +292,6 @@ void vstr_vprintf(vstr_t *vstr, const char *fmt, va_list ap); int DEBUG_printf(const char *fmt, ...); -extern mp_uint_t mp_verbose_flag; - /** float internals *************/ #if MICROPY_PY_BUILTINS_FLOAT diff --git a/py/mkrules.cmake b/py/mkrules.cmake index e3d769cc59b..0903f81d897 100644 --- a/py/mkrules.cmake +++ b/py/mkrules.cmake @@ -260,9 +260,12 @@ if(MICROPY_FROZEN_MANIFEST) # Note: target_compile_definitions already added earlier. - if(NOT MICROPY_LIB_DIR) + # Ensure micropython-lib is included in submodule updates, but only when + # MICROPY_LIB_DIR points at the in-tree submodule location; a user + # overriding MICROPY_LIB_DIR to an out-of-tree checkout shouldn't have + # the submodule updated. + if(MICROPY_LIB_DIR STREQUAL "${MICROPY_DIR}/lib/micropython-lib") list(APPEND GIT_SUBMODULES lib/micropython-lib) - set(MICROPY_LIB_DIR ${MICROPY_DIR}/lib/micropython-lib) endif() if(NOT UPDATE_SUBMODULES AND NOT EXISTS ${MICROPY_LIB_DIR}/README.md) @@ -278,9 +281,13 @@ if(MICROPY_FROZEN_MANIFEST) if(NOT MICROPY_MAKE_EXECUTABLE) set(MICROPY_MAKE_EXECUTABLE make) endif() + # Clear FROZEN_MANIFEST/USER_C_MODULES in the mpy-cross sub-make so a + # shell-env FROZEN_MANIFEST=... doesn't leak in and trigger + # manifest.mk against the wrong cwd. mkrules.mk does the same for the + # make-based port path. add_custom_command( OUTPUT ${MICROPY_MPYCROSS_DEPENDENCY} - COMMAND ${MICROPY_MAKE_EXECUTABLE} -C ${MICROPY_DIR}/mpy-cross USER_C_MODULES= + COMMAND ${MICROPY_MAKE_EXECUTABLE} -C ${MICROPY_DIR}/mpy-cross USER_C_MODULES= FROZEN_MANIFEST= ) endif() @@ -290,27 +297,13 @@ if(MICROPY_FROZEN_MANIFEST) set(MICROPY_CROSS_FLAGS "-f${MICROPY_CROSS_FLAGS}") endif() - # Set default path variables to be passed to makemanifest.py. These will - # be available in path substitutions. Additional variables can be set - # per-board in mpconfigboard.cmake. - set(MICROPY_MANIFEST_PORT_DIR ${MICROPY_PORT_DIR}) - set(MICROPY_MANIFEST_BOARD_DIR ${MICROPY_BOARD_DIR}) - set(MICROPY_MANIFEST_MPY_DIR ${MICROPY_DIR}) - set(MICROPY_MANIFEST_MPY_LIB_DIR ${MICROPY_LIB_DIR}) - - # Find all MICROPY_MANIFEST_* variables and turn them into command line arguments. - get_cmake_property(_manifest_vars VARIABLES) - list(FILTER _manifest_vars INCLUDE REGEX "MICROPY_MANIFEST_.*") - foreach(_manifest_var IN LISTS _manifest_vars) - list(APPEND _manifest_var_args "-v") - string(REGEX REPLACE "MICROPY_MANIFEST_(.*)" "\\1" _manifest_var_name ${_manifest_var}) - list(APPEND _manifest_var_args "${_manifest_var_name}=${${_manifest_var}}") - endforeach() - + # MICROPY_MAKEMANIFEST_ARGS is populated by py/manifest.cmake (included via + # py/usermod.cmake before this file). Reuse that list so both the c_module + # extraction pass and the frozen content step substitute the same values. add_custom_target( BUILD_FROZEN_CONTENT ALL BYPRODUCTS ${MICROPY_FROZEN_CONTENT} - COMMAND ${Python3_EXECUTABLE} ${MICROPY_DIR}/tools/makemanifest.py -o ${MICROPY_FROZEN_CONTENT} ${_manifest_var_args} -b "${CMAKE_BINARY_DIR}" ${MICROPY_CROSS_FLAGS} --mpy-tool-flags=${MICROPY_MPY_TOOL_FLAGS} ${MICROPY_FROZEN_MANIFEST} + COMMAND ${Python3_EXECUTABLE} ${MICROPY_DIR}/tools/makemanifest.py -o ${MICROPY_FROZEN_CONTENT} ${MICROPY_MAKEMANIFEST_ARGS} -b "${CMAKE_BINARY_DIR}" ${MICROPY_CROSS_FLAGS} --mpy-tool-flags=${MICROPY_MPY_TOOL_FLAGS} ${MICROPY_FROZEN_MANIFEST} DEPENDS ${MICROPY_QSTRDEFS_GENERATED} ${MICROPY_ROOT_POINTERS} diff --git a/py/mkrules.mk b/py/mkrules.mk index 03988996c71..fe4bf9a3d16 100644 --- a/py/mkrules.mk +++ b/py/mkrules.mk @@ -39,10 +39,24 @@ CFLAGS += -DMICROPY_BOARD_BUILD_NAME=\"$(BOARD)-$(BOARD_VARIANT)\" endif endif +# Add default C++ compiler flags based on CFLAGS. For use with C++ user modules. +# CIRCUITPY-CHANGE: CircuitPython enables C-only warnings that g++ rejects, in +# py/circuitpy_defns.mk and in the port Makefiles, so strip those too. +CXXFLAGS += $(filter-out -std=c11 -std=c99 -std=gnu11 -std=gnu99 -Werror-implicit-function-declaration -Werror=missing-prototypes -Werror=old-style-definition -Wmissing-prototypes -Wnested-externs -Wold-style-definition -Wstrict-prototypes,$(CFLAGS) $(CXXFLAGS_MOD)) + +# Add LDFLAGS to link libstdc++ on bare metal ports. Added only if a port has +# -nostdlib in LDFLAGS and C++ source files are provided. +ifneq ($(findstring nostdlib,"$(LDFLAGS)"),) +ifneq ($(SRC_CXX)$(SRC_USERMOD_CXX)$(SRC_USERMOD_LIB_CXX),) +LIBSTDCPP_FILE_NAME = "$(shell $(CXX) $(CXXFLAGS) -print-file-name=libstdc++.a)" +LDFLAGS += -L"$(shell dirname $(LIBSTDCPP_FILE_NAME))" +endif +endif + # QSTR generation uses the same CFLAGS, with these modifications. QSTR_GEN_FLAGS = -DNO_QSTR # Note: := to force evaluation immediately. -QSTR_GEN_CFLAGS := $(CFLAGS) +QSTR_GEN_CFLAGS := $(filter-out -g%,$(CFLAGS)) QSTR_GEN_CFLAGS += $(QSTR_GEN_FLAGS) QSTR_GEN_CXXFLAGS := $(CXXFLAGS) QSTR_GEN_CXXFLAGS += $(QSTR_GEN_FLAGS) @@ -63,13 +77,13 @@ QSTR_GEN_CXXFLAGS += $(QSTR_GEN_FLAGS) # can be located. By following this scheme, it allows a single build rule # to be used to compile all .c files. +vpath %.S . $(TOP) $(USER_C_MODULES) $(USERMOD_DIR_PARENTS) # CIRCUITPY-CHANGE: use STEPECHO -vpath %.S . $(TOP) $(USER_C_MODULES) $(BUILD)/%.o: %.S $(STEPECHO) "CC $<" $(Q)$(CC) $(CFLAGS) -c -o $@ $< -vpath %.s . $(TOP) $(USER_C_MODULES) +vpath %.s . $(TOP) $(USER_C_MODULES) $(USERMOD_DIR_PARENTS) $(BUILD)/%.o: %.s $(STEPECHO) "AS $<" $(Q)$(AS) $(AFLAGS) -o $@ $< @@ -100,11 +114,11 @@ $(Q)$(CXX) $(CXXFLAGS) -c -MD -MF $(@:.o=.d) -o $@ $< || (echo -e $(HELP_BUILD_E endef # CIRCUITPY-CHANGE: add $(BUILD) -vpath %.c . $(TOP) $(USER_C_MODULES) $(BUILD) +vpath %.c . $(TOP) $(USER_C_MODULES) $(USERMOD_DIR_PARENTS) $(BUILD) $(BUILD)/%.o: %.c $(call compile_c) -vpath %.cpp . $(TOP) $(USER_C_MODULES) +vpath %.cpp . $(TOP) $(USER_C_MODULES) $(USERMOD_DIR_PARENTS) $(BUILD)/%.o: %.cpp $(call compile_cxx) @@ -196,9 +210,12 @@ $(HEADER_BUILD): $(Q)$(MKDIR) -p $@ ifneq ($(MICROPY_MPYCROSS_DEPENDENCY),) -# to automatically build mpy-cross, if needed +# Build mpy-cross automatically if needed. Clear USER_C_MODULES and +# FROZEN_MANIFEST so a port build with either set doesn't leak them into the +# mpy-cross sub-make and cause manifest.mk there to parse the port's manifest +# from the wrong cwd. $(MICROPY_MPYCROSS_DEPENDENCY): - $(MAKE) -C "$(abspath $(dir $@)..)" USER_C_MODULES= + $(MAKE) -C "$(abspath $(dir $@)..)" USER_C_MODULES= FROZEN_MANIFEST= endif ifneq ($(FROZEN_DIR),) @@ -220,12 +237,6 @@ endif CFLAGS += -DMICROPY_QSTR_EXTRA_POOL=mp_qstr_frozen_const_pool CFLAGS += -DMICROPY_MODULE_FROZEN_MPY CFLAGS += -DMICROPY_MODULE_FROZEN_STR - -# CIRCUITPY-CHANGE: FROZEN_MANIFEST is constructed at build time -# to build frozen_content.c from a manifest -$(BUILD)/frozen_content.c: FORCE $(BUILD)/genhdr/qstrdefs.generated.h $(BUILD)/genhdr/root_pointers.h $(FROZEN_MANIFEST) | $(MICROPY_MPYCROSS_DEPENDENCY) - $(Q)test -e "$(MPY_LIB_DIR)/README.md" || (echo -e $(HELP_MPY_LIB_SUBMODULE); false) - $(Q)$(MAKE_MANIFEST) -o $@ -v "MPY_DIR=$(TOP)" -v "MPY_LIB_DIR=$(MPY_LIB_DIR)" -v "PORT_DIR=$(shell pwd)" -v "BOARD_DIR=$(BOARD_DIR)" -b "$(BUILD)" $(if $(MPY_CROSS_FLAGS),-f"$(MPY_CROSS_FLAGS)",) --mpy-tool-flags="$(MPY_TOOL_FLAGS)" $(FROZEN_MANIFEST) endif ifneq ($(PROG),) @@ -244,7 +255,7 @@ $(BUILD)/$(PROG): $(OBJ) $(ECHO) "LINK $@" # Do not pass COPT here - it's *C* compiler optimizations. For example, # we may want to compile using Thumb, but link with non-Thumb libc. - $(Q)$(CC) -o $@ $^ $(LIB) $(LDFLAGS) + $(Q)$(CC) -o $@ $^ $(LIBS) $(LDFLAGS) ifndef DEBUG ifdef STRIP $(Q)$(STRIP) $(STRIPFLAGS_EXTRA) $@ diff --git a/py/modbuiltins.c b/py/modbuiltins.c index 186a79b7fac..327783996fb 100644 --- a/py/modbuiltins.c +++ b/py/modbuiltins.c @@ -138,7 +138,7 @@ static mp_obj_t mp_builtin_chr(mp_obj_t o_in) { #if MICROPY_PY_BUILTINS_STR_UNICODE mp_uint_t c = mp_obj_get_int(o_in); if (c >= 0x110000) { - mp_raise_ValueError(MP_ERROR_TEXT("chr() arg not in range(0x110000)")); + mp_raise_ValueError(MP_ERROR_TEXT("char not in range(0x110000)")); } VSTR_FIXED(buf, 4); vstr_add_char(&buf, c); @@ -155,6 +155,7 @@ static mp_obj_t mp_builtin_chr(mp_obj_t o_in) { } MP_DEFINE_CONST_FUN_OBJ_1(mp_builtin_chr_obj, mp_builtin_chr); +#if MICROPY_PY_BUILTINS_DIR static mp_obj_t mp_builtin_dir(size_t n_args, const mp_obj_t *args) { mp_obj_t dir = mp_obj_new_list(0, NULL); if (n_args == 0) { @@ -188,6 +189,7 @@ static mp_obj_t mp_builtin_dir(size_t n_args, const mp_obj_t *args) { return dir; } MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_builtin_dir_obj, 0, 1, mp_builtin_dir); +#endif static mp_obj_t mp_builtin_divmod(mp_obj_t o1_in, mp_obj_t o2_in) { return mp_binary_op(MP_BINARY_OP_DIVMOD, o1_in, o2_in); @@ -366,18 +368,21 @@ static mp_obj_t mp_builtin_ord(mp_obj_t o_in) { MP_DEFINE_CONST_FUN_OBJ_1(mp_builtin_ord_obj, mp_builtin_ord); static mp_obj_t mp_builtin_pow(size_t n_args, const mp_obj_t *args) { - switch (n_args) { - case 2: - return mp_binary_op(MP_BINARY_OP_POWER, args[0], args[1]); - default: - #if !MICROPY_PY_BUILTINS_POW3 - mp_raise_NotImplementedError(MP_ERROR_TEXT("3-arg pow() not supported")); - #elif MICROPY_LONGINT_IMPL != MICROPY_LONGINT_IMPL_MPZ - return mp_binary_op(MP_BINARY_OP_MODULO, mp_binary_op(MP_BINARY_OP_POWER, args[0], args[1]), args[2]); - #else - return mp_obj_int_pow3(args[0], args[1], args[2]); - #endif + // Treat pow(x, y, None) as pow(x, y), matching CPython. + if (n_args == 2 + #if MICROPY_PY_BUILTINS_POW3 + || args[2] == mp_const_none + #endif + ) { + return mp_binary_op(MP_BINARY_OP_POWER, args[0], args[1]); } + #if !MICROPY_PY_BUILTINS_POW3 + mp_raise_NotImplementedError(MP_ERROR_TEXT("3-arg pow() not supported")); + #elif MICROPY_LONGINT_IMPL != MICROPY_LONGINT_IMPL_MPZ + return mp_binary_op(MP_BINARY_OP_MODULO, mp_binary_op(MP_BINARY_OP_POWER, args[0], args[1]), args[2]); + #else + return mp_obj_int_pow3(args[0], args[1], args[2]); + #endif } MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_builtin_pow_obj, 2, 3, mp_builtin_pow); @@ -692,7 +697,9 @@ static const mp_rom_map_elem_t mp_module_builtins_globals_table[] = { #if MICROPY_CPYTHON_COMPAT { MP_ROM_QSTR(MP_QSTR_delattr), MP_ROM_PTR(&mp_builtin_delattr_obj) }, #endif + #if MICROPY_PY_BUILTINS_DIR { MP_ROM_QSTR(MP_QSTR_dir), MP_ROM_PTR(&mp_builtin_dir_obj) }, + #endif { MP_ROM_QSTR(MP_QSTR_divmod), MP_ROM_PTR(&mp_builtin_divmod_obj) }, #if MICROPY_PY_BUILTINS_EVAL_EXEC { MP_ROM_QSTR(MP_QSTR_eval), MP_ROM_PTR(&mp_builtin_eval_obj) }, diff --git a/py/modio.c b/py/modio.c index 9aeb42d30aa..fdee484376f 100644 --- a/py/modio.c +++ b/py/modio.c @@ -63,6 +63,10 @@ static mp_uint_t iobase_read_write(mp_obj_t obj, void *buf, mp_uint_t size, int } mp_int_t ret = mp_obj_get_int(ret_obj); if (ret >= 0) { + if ((mp_uint_t)ret > size) { + *errcode = MP_EIO; + return MP_STREAM_ERROR; + } return ret; } else { *errcode = -ret; diff --git a/py/modmicropython.c b/py/modmicropython.c index 67bb32626c3..5e1b3253427 100644 --- a/py/modmicropython.c +++ b/py/modmicropython.c @@ -163,13 +163,35 @@ static MP_DEFINE_CONST_FUN_OBJ_1(mp_micropython_kbd_intr_obj, mp_micropython_kbd #endif #if MICROPY_ENABLE_SCHEDULER + +#if MICROPY_KBD_EXCEPTION && MICROPY_SCHEDULER_STATIC_NODES +static mp_sched_node_t mp_keyboard_interrupt_sched_node; +static void mp_sched_keyboard_interrupt_wrapper(mp_sched_node_t *node) { + mp_sched_keyboard_interrupt(); +} +#endif + static mp_obj_t mp_micropython_schedule(mp_obj_t function, mp_obj_t arg) { + #if MICROPY_KBD_EXCEPTION + if (function == MP_OBJ_FROM_PTR(&mp_micropython_kbd_intr_obj)) { + #if MICROPY_SCHEDULER_STATIC_NODES + // Allow calling `micropython.schedule(micropython.kbd_intr, None)` as a + // special case to schedule a keyboard interrupt. + mp_sched_schedule_node(&mp_keyboard_interrupt_sched_node, mp_sched_keyboard_interrupt_wrapper); + return mp_const_none; + #else + // Signal that passing `micropython.kbd_intr` is not supported. + mp_raise_ValueError(NULL); + #endif + } + #endif if (!mp_sched_schedule(function, arg)) { mp_raise_msg(&mp_type_RuntimeError, MP_ERROR_TEXT("schedule queue full")); } return mp_const_none; } static MP_DEFINE_CONST_FUN_OBJ_2(mp_micropython_schedule_obj, mp_micropython_schedule); + #endif static const mp_rom_map_elem_t mp_module_micropython_globals_table[] = { diff --git a/py/mpconfig.h b/py/mpconfig.h index 846209e9412..c630adaee1a 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -49,7 +49,7 @@ // as well as a fallback to generate MICROPY_GIT_TAG if the git repo or tags // are unavailable. #define MICROPY_VERSION_MAJOR 1 -#define MICROPY_VERSION_MINOR 28 +#define MICROPY_VERSION_MINOR 29 #define MICROPY_VERSION_MICRO 0 #define MICROPY_VERSION_PRERELEASE 0 @@ -479,7 +479,11 @@ typedef uint64_t mp_uint_t; // Whether to emit ARMv7-M instruction support in thumb native code #ifndef MICROPY_EMIT_THUMB_ARMV7M +#if defined(__ARM_ARCH_ISA_THUMB) && __ARM_ARCH_ISA_THUMB == 2 #define MICROPY_EMIT_THUMB_ARMV7M (1) +#else +#define MICROPY_EMIT_THUMB_ARMV7M (0) +#endif #endif // Whether to enable the thumb inline assembler @@ -489,7 +493,11 @@ typedef uint64_t mp_uint_t; // Whether to enable float support in the Thumb2 inline assembler #ifndef MICROPY_EMIT_INLINE_THUMB_FLOAT +#if defined(__ARM_ARCH_ISA_THUMB) && __ARM_ARCH_ISA_THUMB == 2 && defined(__ARM_FP) #define MICROPY_EMIT_INLINE_THUMB_FLOAT (1) +#else +#define MICROPY_EMIT_INLINE_THUMB_FLOAT (0) +#endif #endif // Whether to emit ARM native code @@ -1273,6 +1281,11 @@ typedef time_t mp_timestamp_t; #define MICROPY_VFS (0) #endif +// Whether to include support for fast native block devices. +#ifndef MICROPY_VFS_BLOCKDEV_NATIVE +#define MICROPY_VFS_BLOCKDEV_NATIVE (0) +#endif + // Whether to include support for writable filesystems. #ifndef MICROPY_VFS_WRITABLE #define MICROPY_VFS_WRITABLE (1) @@ -1397,7 +1410,7 @@ typedef time_t mp_timestamp_t; // Whether str object is proper unicode #ifndef MICROPY_PY_BUILTINS_STR_UNICODE -#define MICROPY_PY_BUILTINS_STR_UNICODE (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) +#define MICROPY_PY_BUILTINS_STR_UNICODE (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_BASIC_FEATURES) #endif // Whether to check for valid UTF-8 when converting bytes to str @@ -1405,6 +1418,11 @@ typedef time_t mp_timestamp_t; #define MICROPY_PY_BUILTINS_STR_UNICODE_CHECK (MICROPY_PY_BUILTINS_STR_UNICODE) #endif +// Whether bytes.decode() supports the 'ignore' and 'replace' error handlers +#ifndef MICROPY_PY_BUILTINS_BYTES_DECODE_ERRORS +#define MICROPY_PY_BUILTINS_BYTES_DECODE_ERRORS (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) +#endif + // Whether str.center() method provided #ifndef MICROPY_PY_BUILTINS_STR_CENTER #define MICROPY_PY_BUILTINS_STR_CENTER (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) @@ -1456,7 +1474,7 @@ typedef time_t mp_timestamp_t; // Whether to support memoryview.itemsize attribute #ifndef MICROPY_PY_BUILTINS_MEMORYVIEW_ITEMSIZE -#define MICROPY_PY_BUILTINS_MEMORYVIEW_ITEMSIZE (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EVERYTHING) +#define MICROPY_PY_BUILTINS_MEMORYVIEW_ITEMSIZE (MICROPY_PY_MACHINE_MEM_BACKUP || MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_BASIC_FEATURES) #endif // Whether to support set object @@ -1514,6 +1532,11 @@ typedef time_t mp_timestamp_t; #define MICROPY_PY_BUILTINS_ROUND_INT (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif +// Whether to implement dir() to enumerate object fields. +#ifndef MICROPY_PY_BUILTINS_DIR +#define MICROPY_PY_BUILTINS_DIR (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) +#endif + // Whether to support complete set of special methods for user // classes, or only the most used ones. "Inplace" methods are // controlled by MICROPY_PY_ALL_INPLACE_SPECIAL_METHODS below. @@ -1604,6 +1627,16 @@ typedef time_t mp_timestamp_t; #define MICROPY_PY_BUILTINS_HELP_MODULES (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif +// Use this to configure output of help('modules') +#ifndef MICROPY_PY_BUILTINS_HELP_NUM_COLUMNS +#define MICROPY_PY_BUILTINS_HELP_NUM_COLUMNS (4) +#endif + +// Use this to configure output of help('modules') +#ifndef MICROPY_PY_BUILTINS_HELP_COLUMN_WIDTH +#define MICROPY_PY_BUILTINS_HELP_COLUMN_WIDTH (18) +#endif + // Whether to provide mem-info related functions in micropython module #ifndef MICROPY_PY_MICROPYTHON_MEM_INFO #define MICROPY_PY_MICROPYTHON_MEM_INFO (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) @@ -1907,7 +1940,7 @@ typedef time_t mp_timestamp_t; // implementation). This is present for compatibility but can be disabled to // save space. #ifndef MICROPY_PY_SELECT_SELECT -#define MICROPY_PY_SELECT_SELECT (1) +#define MICROPY_PY_SELECT_SELECT (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif // Whether to provide the "time" module @@ -2116,6 +2149,11 @@ typedef time_t mp_timestamp_t; #define MICROPY_PY_MACHINE_MEMX (MICROPY_PY_MACHINE) #endif +// Whether to provide the "machine.mem_backup" function +#ifndef MICROPY_PY_MACHINE_MEM_BACKUP +#define MICROPY_PY_MACHINE_MEM_BACKUP (0) +#endif + // Whether to provide the "machine.Signal" class #ifndef MICROPY_PY_MACHINE_SIGNAL #define MICROPY_PY_MACHINE_SIGNAL (MICROPY_PY_MACHINE) @@ -2160,6 +2198,16 @@ typedef time_t mp_timestamp_t; #define MICROPY_PY_SOCKET_LISTEN_BACKLOG_DEFAULT (2) #endif +// Whether to enable lwIP bindings to be used as the implementation of the `socket` module +#ifndef MICROPY_PY_LWIP +#define MICROPY_PY_LWIP (0) +#endif + +// Whether to support raw sockets via the `socket.SOCK_RAW` constant +#ifndef MICROPY_PY_LWIP_SOCK_RAW +#define MICROPY_PY_LWIP_SOCK_RAW (MICROPY_PY_LWIP) +#endif + #ifndef MICROPY_PY_SSL #define MICROPY_PY_SSL (0) #endif diff --git a/py/mpstate.h b/py/mpstate.h index 8519a1a32ad..25685334a4c 100644 --- a/py/mpstate.h +++ b/py/mpstate.h @@ -80,6 +80,9 @@ typedef struct mp_dynamic_compiler_t { uint8_t small_int_bits; // must be <= host small_int_bits uint8_t native_arch; uint8_t nlr_buf_num_regs; + #if MICROPY_ENABLE_SOURCE_LINE + bool include_source_lines; + #endif } mp_dynamic_compiler_t; extern mp_dynamic_compiler_t mp_dynamic_compiler; #endif @@ -141,6 +144,10 @@ typedef struct _mp_state_mem_t { #endif mp_state_mem_area_t area; + #if MICROPY_GC_SPLIT_HEAP + byte *area_pool_min; // Min of all gc_pool_start values across all areas + byte *area_pool_max; // Max of all gc_pool_end values across all areas + #endif int gc_stack_overflow; MICROPY_GC_STACK_ENTRY_TYPE gc_block_stack[MICROPY_ALLOC_GC_STACK_SIZE]; @@ -254,6 +261,9 @@ typedef struct _mp_state_vm_t { #if MICROPY_EMIT_NATIVE uint8_t default_emit_opt; // one of MP_EMIT_OPT_xxx #endif + #if MICROPY_DEBUG_PRINTERS + mp_uint_t mp_verbose_flag; + #endif #endif // size of the emergency exception buf, if it's dynamically allocated diff --git a/py/mpz.c b/py/mpz.c index 74fd1754636..062777af566 100644 --- a/py/mpz.c +++ b/py/mpz.c @@ -1360,7 +1360,7 @@ void mpz_pow_inpl(mpz_t *dest, const mpz_t *lhs, const mpz_t *rhs) { can have dest, lhs, rhs the same; mod can't be the same as dest */ void mpz_pow3_inpl(mpz_t *dest, const mpz_t *lhs, const mpz_t *rhs, const mpz_t *mod) { - if (lhs->len == 0 || rhs->neg != 0 || (mod->len == 1 && mod->dig[0] == 1)) { + if (rhs->neg != 0 || (mod->len == 1 && mod->dig[0] == 1)) { mpz_set_from_int(dest, 0); return; } @@ -1368,6 +1368,15 @@ void mpz_pow3_inpl(mpz_t *dest, const mpz_t *lhs, const mpz_t *rhs, const mpz_t mpz_set_from_int(dest, 1); if (rhs->len == 0) { + // Python style modulo: 1 % mod is 1 + mod, as abs(mod) >= 2 here + if (mod->neg) { + mpz_add_inpl(dest, dest, mod); + } + return; + } + + if (lhs->len == 0) { + mpz_set_from_int(dest, 0); return; } @@ -1602,16 +1611,17 @@ bool mpz_as_bytes(const mpz_t *z, bool big_endian, bool as_signed, size_t len, b b += len; } mpz_dig_t *zdig = z->dig; + byte fill_byte = z->neg ? 0xFF : 0x00; int bits = 0; mpz_dbl_dig_t d = 0; mpz_dbl_dig_t carry = 1; + mpz_dig_t val = 0; size_t olen = len; // bytes in output buffer - bool ok = true; for (size_t zlen = z->len; zlen > 0; --zlen) { bits += DIG_SIZE; d = (d << DIG_SIZE) | *zdig++; for (; bits >= 8; bits -= 8, d >>= 8) { - mpz_dig_t val = d; + val = d; if (z->neg) { val = (~val & 0xff) + carry; carry = val >> 8; @@ -1619,7 +1629,9 @@ bool mpz_as_bytes(const mpz_t *z, bool big_endian, bool as_signed, size_t len, b if (!olen) { // Buffer is full, only OK if all remaining bytes are zeroes - ok = ok && ((byte)val == 0); + if ((byte)val != fill_byte) { + return false; + } continue; } @@ -1632,16 +1644,17 @@ bool mpz_as_bytes(const mpz_t *z, bool big_endian, bool as_signed, size_t len, b } } - if (as_signed && olen == 0 && len > 0) { - // If output exhausted then ensure there was enough space for the sign bit - byte most_sig = big_endian ? buf[0] : buf[len - 1]; - ok = ok && (bool)(most_sig & 0x80) == (bool)z->neg; - } else { + // Check if the most significant bit is set incorrectly for a signed value + if (olen == 0 && as_signed && ((val & 0x80) != (fill_byte & 0x80))) { + return false; + } + + if (olen > 0) { // fill remainder of buf with zero/sign extension of the integer - memset(big_endian ? buf : b, z->neg ? 0xff : 0x00, olen); + memset(big_endian ? buf : b, fill_byte, olen); } - return ok; + return true; } #if MICROPY_PY_BUILTINS_FLOAT diff --git a/py/nativeglue.c b/py/nativeglue.c index 2613312e2b7..c632cd36534 100644 --- a/py/nativeglue.c +++ b/py/nativeglue.c @@ -323,7 +323,11 @@ const mp_fun_table_t mp_fun_table = { mp_printf, mp_vprintf, // CIRCUITPY-CHANGE: mp_raise_msg_str instead of mp_raise_msg + #if MICROPY_ERROR_REPORTING == MICROPY_ERROR_REPORTING_NONE + NULL, + #else mp_raise_msg_str, + #endif mp_obj_get_type, mp_obj_new_str, mp_obj_new_bytes, diff --git a/py/nlr.h b/py/nlr.h index 446e78c7ebb..72ea1368e93 100644 --- a/py/nlr.h +++ b/py/nlr.h @@ -46,6 +46,7 @@ #define MICROPY_NLR_NUM_REGS_XTENSAWIN (17) #define MICROPY_NLR_NUM_REGS_RV32I (14) #define MICROPY_NLR_NUM_REGS_RV64I (14) +#define MICROPY_NLR_NUM_REGS_LOONG64 (13) // *FORMAT-OFF* @@ -111,6 +112,13 @@ #else #error Unsupported RISC-V variant. #endif +#elif defined(__loongarch__) + #if defined(__loongarch64) + #define MICROPY_NLR_LOONG64 (1) + #define MICROPY_NLR_NUM_REGS (MICROPY_NLR_NUM_REGS_LOONG64) + #else + #error Unsupported Loongarch variant. + #endif #else #define MICROPY_NLR_SETJMP (1) //#warning "No native NLR support for this arch, using setjmp implementation" @@ -122,6 +130,10 @@ #define MICROPY_NLR_AARCH64 (0) #endif +#ifndef MICROPY_NLR_LOONG64 +#define MICROPY_NLR_LOONG64 (0) +#endif + #ifndef MICROPY_NLR_MIPS #define MICROPY_NLR_MIPS (0) #endif diff --git a/py/nlrloong64.c b/py/nlrloong64.c new file mode 100644 index 00000000000..7a12b2ec904 --- /dev/null +++ b/py/nlrloong64.c @@ -0,0 +1,80 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2026 Alessandro Gatti + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "py/mpstate.h" + +#if MICROPY_NLR_LOONG64 + +__attribute__((used)) unsigned int nlr_push_tail(nlr_buf_t *nlr); + +__asm( + ".globl nlr_push \n" + "nlr_push: \n" + ".cfi_startproc \n" + "st.d $r1, $r4, 16 \n" /* Store RA. */ + "st.d $r23, $r4, 24 \n" /* Store S0. */ + "st.d $r24, $r4, 32 \n" /* Store S1. */ + "st.d $r25, $r4, 40 \n" /* Store S2. */ + "st.d $r26, $r4, 48 \n" /* Store S3. */ + "st.d $r27, $r4, 56 \n" /* Store S4. */ + "st.d $r28, $r4, 64 \n" /* Store S5. */ + "st.d $r29, $r4, 72 \n" /* Store S6. */ + "st.d $r30, $r4, 80 \n" /* Store S7. */ + "st.d $r31, $r4, 88 \n" /* Store S8. */ + "st.d $r22, $r4, 96 \n" /* Store S9. */ + "st.d $r21, $r4, 104 \n" /* Marked as reserved in the ABI. */ + "st.d $r3, $r4, 112 \n" /* Store SP. */ + "b nlr_push_tail \n" /* Jump to the C part. */ + ".cfi_endproc \n" + ); + +MP_NORETURN void nlr_jump(void *val) { + MP_NLR_JUMP_HEAD(val, top) + __asm volatile ( + "add.d $r4, $r0, %0 \n" + "ld.d $r1, $r4, 16 \n" /* Retrieve RA. */ + "ld.d $r23, $r4, 24 \n" /* Retrieve S0. */ + "ld.d $r24, $r4, 32 \n" /* Retrieve S1. */ + "ld.d $r25, $r4, 40 \n" /* Retrieve S2. */ + "ld.d $r26, $r4, 48 \n" /* Retrieve S3. */ + "ld.d $r27, $r4, 56 \n" /* Retrieve S4. */ + "ld.d $r28, $r4, 64 \n" /* Retrieve S5. */ + "ld.d $r29, $r4, 72 \n" /* Retrieve S6. */ + "ld.d $r30, $r4, 80 \n" /* Retrieve S7. */ + "ld.d $r31, $r4, 88 \n" /* Retrieve S8. */ + "ld.d $r22, $r4, 96 \n" /* Retrieve S9. */ + "ld.d $r21, $r4, 104 \n" + "ld.d $r3, $r4, 112 \n" /* Retrieve SP. */ + "addi.d $r4, $r0, 1 \n" /* Return 1 for a non-local return. */ + "ret \n" /* Return. */ + : + : "r" (top) + : "memory" + ); + MP_UNREACHABLE +} + +#endif diff --git a/py/nlrpowerpc.c b/py/nlrpowerpc.c index cf140400e68..cb94473968d 100644 --- a/py/nlrpowerpc.c +++ b/py/nlrpowerpc.c @@ -72,7 +72,7 @@ unsigned int nlr_push(nlr_buf_t *nlr) { "bctr ;" : : "r" (&nlr->regs), "r" (nlr) - : + : "r4" ); return 0; @@ -82,33 +82,34 @@ MP_NORETURN void nlr_jump(void *val) { MP_NLR_JUMP_HEAD(val, top) __asm__ volatile ( - "ld 3, 0x0(%0) ;" + "mr 4, %0 ;" + "ld 3, 0x0(4) ;" "cmpdi 3, 0x4eed ; " // Check canary "bne . ; " - "ld 0, 0x08(%0) ;" - "ld 1, 0x10(%0) ;" - "ld 2, 0x18(%0) ;" - "ld 14, 0x20(%0) ;" - "ld 15, 0x28(%0) ;" - "ld 16, 0x30(%0) ;" - "ld 17, 0x38(%0) ;" - "ld 18, 0x40(%0) ;" - "ld 19, 0x48(%0) ;" - "ld 20, 0x50(%0) ;" - "ld 21, 0x58(%0) ;" - "ld 22, 0x60(%0) ;" - "ld 23, 0x68(%0) ;" - "ld 24, 0x70(%0) ;" - "ld 25, 0x78(%0) ;" - "ld 26, 0x80(%0) ;" - "ld 27, 0x88(%0) ;" - "ld 28, 0x90(%0) ;" - "ld 29, 0x98(%0) ;" - "ld 30, 0xA0(%0) ;" - "ld 31, 0xA8(%0) ;" - "ld 3, 0xB0(%0) ;" + "ld 0, 0x08(4) ;" + "ld 1, 0x10(4) ;" + "ld 2, 0x18(4) ;" + "ld 14, 0x20(4) ;" + "ld 15, 0x28(4) ;" + "ld 16, 0x30(4) ;" + "ld 17, 0x38(4) ;" + "ld 18, 0x40(4) ;" + "ld 19, 0x48(4) ;" + "ld 20, 0x50(4) ;" + "ld 21, 0x58(4) ;" + "ld 22, 0x60(4) ;" + "ld 23, 0x68(4) ;" + "ld 24, 0x70(4) ;" + "ld 25, 0x78(4) ;" + "ld 26, 0x80(4) ;" + "ld 27, 0x88(4) ;" + "ld 28, 0x90(4) ;" + "ld 29, 0x98(4) ;" + "ld 30, 0xA0(4) ;" + "ld 31, 0xA8(4) ;" + "ld 3, 0xB0(4) ;" "mtcr 3 ;" - "ld 3, 0xB8(%0) ;" + "ld 3, 0xB8(4) ;" "mtlr 3 ; " "li 3, 1;" "blr ;" @@ -161,7 +162,7 @@ unsigned int nlr_push(nlr_buf_t *nlr) { "bctr ;" : : "r" (&nlr->regs), "r" (nlr) - : + : "r4" ); return 0; @@ -171,33 +172,34 @@ MP_NORETURN void nlr_jump(void *val) { MP_NLR_JUMP_HEAD(val, top) __asm__ volatile ( - "l 3, 0x0(%0) ;" + "mr 4, %0 ;" + "l 3, 0x0(4) ;" "cmpdi 3, 0x4eed ; " // Check canary "bne . ; " - "l 0, 0x04(%0) ;" - "l 1, 0x08(%0) ;" - "l 2, 0x0c(%0) ;" - "l 14, 0x10(%0) ;" - "l 15, 0x14(%0) ;" - "l 16, 0x18(%0) ;" - "l 17, 0x1c(%0) ;" - "l 18, 0x20(%0) ;" - "l 19, 0x24(%0) ;" - "l 20, 0x28(%0) ;" - "l 21, 0x2c(%0) ;" - "l 22, 0x30(%0) ;" - "l 23, 0x34(%0) ;" - "l 24, 0x38(%0) ;" - "l 25, 0x3c(%0) ;" - "l 26, 0x40(%0) ;" - "l 27, 0x44(%0) ;" - "l 28, 0x48(%0) ;" - "l 29, 0x4c(%0) ;" - "l 30, 0x50(%0) ;" - "l 31, 0x54(%0) ;" - "l 3, 0x58(%0) ;" + "l 0, 0x04(4) ;" + "l 1, 0x08(4) ;" + "l 2, 0x0c(4) ;" + "l 14, 0x10(4) ;" + "l 15, 0x14(4) ;" + "l 16, 0x18(4) ;" + "l 17, 0x1c(4) ;" + "l 18, 0x20(4) ;" + "l 19, 0x24(4) ;" + "l 20, 0x28(4) ;" + "l 21, 0x2c(4) ;" + "l 22, 0x30(4) ;" + "l 23, 0x34(4) ;" + "l 24, 0x38(4) ;" + "l 25, 0x3c(4) ;" + "l 26, 0x40(4) ;" + "l 27, 0x44(4) ;" + "l 28, 0x48(4) ;" + "l 29, 0x4c(4) ;" + "l 30, 0x50(4) ;" + "l 31, 0x54(4) ;" + "l 3, 0x58(4) ;" "mtcr 3 ;" - "l 3, 0x5c(%0) ;" + "l 3, 0x5c(4) ;" "mtlr 3 ; " "li 3, 1;" "blr ;" diff --git a/py/nlrx86.c b/py/nlrx86.c index 26bf0dc6ccb..82e139dd8e8 100644 --- a/py/nlrx86.c +++ b/py/nlrx86.c @@ -40,23 +40,35 @@ __attribute__((used)) unsigned int nlr_push_tail(nlr_buf_t *nlr); #endif #if !defined(__clang__) && defined(__GNUC__) && __GNUC__ >= 8 -// Since gcc 8.0 the naked attribute is supported -#define USE_NAKED (1) -#define UNDO_PRELUDE (0) +// Since gcc 8.0 the naked and no-sanitize attributes are supported + #define NLR_PUSH_ATTRIBUTE __attribute__((naked, no_sanitize("unreachable"))) + #define UNDO_PRELUDE (0) + #define ARG_USED(x) (void)x; + #define NLR_UNREACHABLE __builtin_unreachable(); #elif defined(__ZEPHYR__) || defined(__ANDROID__) // Zephyr and Android use a different calling convention by default -#define USE_NAKED (0) -#define UNDO_PRELUDE (0) + #define NLR_PUSH_ATTRIBUTE /* NOTHING */ + #define UNDO_PRELUDE (0) + #define ARG_USED(x) (void)x; + #define NLR_UNREACHABLE return 0; +#elif defined(__clang__) +// clang on Ubuntu 24.04 enables -fsanitize=unreachable by default, but this +// destroys the content of the ebx register. + #define NLR_PUSH_ATTRIBUTE __attribute__((naked, no_sanitize("unreachable"))) + #define UNDO_PRELUDE (0) + #define ARG_USED(x) /* NOTHING */ + #define NLR_UNREACHABLE /* NOTHING */ #else -#define USE_NAKED (0) -#define UNDO_PRELUDE (1) +// gcc before 8 unavoidably emits a 'push %ebp' prologue instruction + #define NLR_PUSH_ATTRIBUTE /* NOTHING */ + #define UNDO_PRELUDE (1) + #define ARG_USED(x) (void)x; + #define NLR_UNREACHABLE return 0; #endif -#if USE_NAKED -__attribute__((naked)) -#endif +NLR_PUSH_ATTRIBUTE unsigned int nlr_push(nlr_buf_t *nlr) { - (void)nlr; + ARG_USED(nlr) __asm volatile ( #if UNDO_PRELUDE @@ -73,9 +85,7 @@ unsigned int nlr_push(nlr_buf_t *nlr) { "jmp nlr_push_tail \n" // do the rest in C ); - #if !USE_NAKED - return 0; // needed to silence compiler warning - #endif + NLR_UNREACHABLE } MP_NORETURN void nlr_jump(void *val) { diff --git a/py/obj.c b/py/obj.c index ba3e6a9a996..b65ac53d23a 100644 --- a/py/obj.c +++ b/py/obj.c @@ -441,7 +441,7 @@ long long mp_obj_get_ll(mp_const_obj_t arg) { return MP_OBJ_SMALL_INT_VALUE(arg); } else { long long res; - mp_obj_int_to_bytes_impl((mp_obj_t)arg, MP_ENDIANNESS_BIG, sizeof(res), (byte *)&res); + mp_obj_int_to_bytes((mp_obj_t)arg, sizeof(res), (byte *)&res, MP_ENDIANNESS_BIG, false, false); return res; } } diff --git a/py/obj.h b/py/obj.h index e12cbff1ca2..368ac96dbd2 100644 --- a/py/obj.h +++ b/py/obj.h @@ -117,7 +117,7 @@ extern const struct _mp_obj_float_t mp_const_float_inf_obj; extern const struct _mp_obj_float_t mp_const_float_nan_obj; #endif -#define mp_obj_is_float(o) mp_obj_is_type((o), &mp_type_float) +#define mp_obj_is_float(o) mp_obj_is_exact_type((o), &mp_type_float) mp_float_t mp_obj_float_get(mp_obj_t self_in); mp_obj_t mp_obj_new_float(mp_float_t value); #endif @@ -162,7 +162,7 @@ extern const struct _mp_obj_float_t mp_const_float_inf_obj; extern const struct _mp_obj_float_t mp_const_float_nan_obj; #endif -#define mp_obj_is_float(o) mp_obj_is_type((o), &mp_type_float) +#define mp_obj_is_float(o) mp_obj_is_exact_type((o), &mp_type_float) mp_float_t mp_obj_float_get(mp_obj_t self_in); mp_obj_t mp_obj_new_float(mp_float_t value); #endif @@ -811,25 +811,6 @@ typedef struct _mp_obj_full_type_t { #define _MP_OBJ_TYPE_SLOT_TYPE_parent (const void *) #define _MP_OBJ_TYPE_SLOT_TYPE_locals_dict (struct _mp_obj_dict_t *) -// Implementation of MP_DEFINE_CONST_OBJ_TYPE for each number of arguments. -// Do not use these directly, instead use MP_DEFINE_CONST_OBJ_TYPE. -// Generated with: -// for i in range(13): -// print(f"#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_{i}(_struct_type, _typename, _name, _flags{''.join(f', f{j+1}, v{j+1}' for j in range(i))}) const _struct_type _typename = {{ .base = {{ &mp_type_type }}, .flags = _flags, .name = _name{''.join(f', .slot_index_##f{j+1} = {j+1}' for j in range(i))}{', .slots = { ' + ''.join(f'v{j+1}, ' for j in range(i)) + '}' if i else '' } }}") -#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_0(_struct_type, _typename, _name, _flags) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name } -#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_1(_struct_type, _typename, _name, _flags, f1, v1) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slots = { v1, } } -#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_2(_struct_type, _typename, _name, _flags, f1, v1, f2, v2) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slots = { v1, v2, } } -#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_3(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slots = { v1, v2, v3, } } -#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_4(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slots = { v1, v2, v3, v4, } } -#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_5(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slots = { v1, v2, v3, v4, v5, } } -#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_6(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slots = { v1, v2, v3, v4, v5, v6, } } -#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_7(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slots = { v1, v2, v3, v4, v5, v6, v7, } } -#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_8(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, } } -#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_9(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, } } -#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_10(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9, f10, v10) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slot_index_##f10 = 10, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, } } -#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_11(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9, f10, v10, f11, v11) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slot_index_##f10 = 10, .slot_index_##f11 = 11, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, } } -#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_12(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9, f10, v10, f11, v11, f12, v12) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slot_index_##f10 = 10, .slot_index_##f11 = 11, .slot_index_##f12 = 12, .slots = { v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, } } - // Because the mp_obj_type_t instances are in (zero-initialised) ROM, we take // slot_index_foo=0 to mean that the slot is unset. This also simplifies checking // if the slot is set. That means that we need to store index+1 in slot_index_foo @@ -843,6 +824,34 @@ typedef struct _mp_obj_full_type_t { #define MP_OBJ_TYPE_OFFSETOF_SLOT(f) (offsetof(mp_obj_type_t, slot_index_##f)) #define MP_OBJ_TYPE_HAS_SLOT_BY_OFFSET(t, offset) (*(uint8_t *)((char *)(t) + (offset)) != 0) +// Implementation of MP_DEFINE_CONST_OBJ_TYPE for each number of arguments. +// Do not use these directly, instead use MP_DEFINE_CONST_OBJ_TYPE. +// Generated with: +// for i in range(13): +// args = ['_struct_type', '_typename', '_name', '_flags'] +// struct = ['.base = { &mp_type_type }', '.flags = _flags', '.name = _name'] +// slots = [] +// for j in range(i): +// args += [f"f{j+1}", f"v{j+1}"] +// struct += [f".slot_index_##f{j+1} = {j+1}"] +// slots += [f"(const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f{j+1} v{j+1}"] +// if slots: +// struct += [f".slots = {{ {', '.join(slots)} }}"] +// print(f"#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_{i}({', '.join(args)}) const _struct_type _typename = {{ {', '.join(struct)} }}") +#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_0(_struct_type, _typename, _name, _flags) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name } +#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_1(_struct_type, _typename, _name, _flags, f1, v1) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slots = { (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f1 v1 } } +#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_2(_struct_type, _typename, _name, _flags, f1, v1, f2, v2) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slots = { (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f1 v1, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f2 v2 } } +#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_3(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slots = { (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f1 v1, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f2 v2, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f3 v3 } } +#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_4(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slots = { (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f1 v1, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f2 v2, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f3 v3, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f4 v4 } } +#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_5(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slots = { (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f1 v1, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f2 v2, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f3 v3, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f4 v4, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f5 v5 } } +#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_6(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slots = { (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f1 v1, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f2 v2, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f3 v3, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f4 v4, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f5 v5, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f6 v6 } } +#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_7(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slots = { (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f1 v1, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f2 v2, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f3 v3, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f4 v4, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f5 v5, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f6 v6, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f7 v7 } } +#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_8(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slots = { (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f1 v1, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f2 v2, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f3 v3, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f4 v4, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f5 v5, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f6 v6, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f7 v7, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f8 v8 } } +#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_9(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slots = { (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f1 v1, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f2 v2, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f3 v3, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f4 v4, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f5 v5, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f6 v6, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f7 v7, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f8 v8, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f9 v9 } } +#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_10(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9, f10, v10) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slot_index_##f10 = 10, .slots = { (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f1 v1, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f2 v2, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f3 v3, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f4 v4, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f5 v5, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f6 v6, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f7 v7, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f8 v8, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f9 v9, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f10 v10 } } +#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_11(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9, f10, v10, f11, v11) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slot_index_##f10 = 10, .slot_index_##f11 = 11, .slots = { (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f1 v1, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f2 v2, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f3 v3, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f4 v4, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f5 v5, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f6 v6, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f7 v7, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f8 v8, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f9 v9, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f10 v10, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f11 v11 } } +#define MP_DEFINE_CONST_OBJ_TYPE_NARGS_12(_struct_type, _typename, _name, _flags, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8, f9, v9, f10, v10, f11, v11, f12, v12) const _struct_type _typename = { .base = { &mp_type_type }, .flags = _flags, .name = _name, .slot_index_##f1 = 1, .slot_index_##f2 = 2, .slot_index_##f3 = 3, .slot_index_##f4 = 4, .slot_index_##f5 = 5, .slot_index_##f6 = 6, .slot_index_##f7 = 7, .slot_index_##f8 = 8, .slot_index_##f9 = 9, .slot_index_##f10 = 10, .slot_index_##f11 = 11, .slot_index_##f12 = 12, .slots = { (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f1 v1, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f2 v2, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f3 v3, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f4 v4, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f5 v5, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f6 v6, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f7 v7, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f8 v8, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f9 v9, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f10 v10, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f11 v11, (const void *)_MP_OBJ_TYPE_SLOT_TYPE_##f12 v12 } } + // Workaround for https://docs.microsoft.com/en-us/cpp/preprocessor/preprocessor-experimental-overview?view=msvc-160#macro-arguments-are-unpacked #define MP_DEFINE_CONST_OBJ_TYPE_EXPAND(x) x @@ -1042,8 +1051,13 @@ void *mp_obj_malloc_with_finaliser_helper(size_t num_bytes, const mp_obj_type_t MP_STATIC_ASSERT_NONCONSTEXPR((t) != &mp_type_str), assert((t) != &mp_type_str), \ MP_STATIC_ASSERT_NONCONSTEXPR((t) != &mp_type_NoneType), assert((t) != &mp_type_NoneType), \ 1) +#if MICROPY_PY_BUILTINS_FLOAT +#define mp_type_assert_not_float(t) (MP_STATIC_ASSERT_NONCONSTEXPR((t) != &mp_type_float), assert((t) != &mp_type_float), 1) +#else +#define mp_type_assert_not_float(t) (1) +#endif -#define mp_obj_is_type(o, t) (mp_type_assert_not_bool_int_str_nonetype(t) && mp_obj_is_exact_type(o, t)) +#define mp_obj_is_type(o, t) (mp_type_assert_not_bool_int_str_nonetype(t) && mp_type_assert_not_float(t) && mp_obj_is_exact_type(o, t)) #if MICROPY_OBJ_IMMEDIATE_OBJS // bool's are immediates, not real objects, so test for the 2 possible values. #define mp_obj_is_bool(o) ((o) == mp_const_false || (o) == mp_const_true) diff --git a/py/objexcept.c b/py/objexcept.c index 6a2fecd51e5..6efb0f50672 100644 --- a/py/objexcept.c +++ b/py/objexcept.c @@ -166,7 +166,7 @@ void mp_obj_exception_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kin mp_print_kind_t k = kind & ~PRINT_EXC_SUBCLASS; bool is_subclass = kind & PRINT_EXC_SUBCLASS; if (!is_subclass && (k == PRINT_REPR || k == PRINT_EXC)) { - mp_print_str(print, qstr_str(o->base.type->name)); + mp_print_str(print, mp_obj_get_type_str(o_in)); } if (k == PRINT_EXC) { diff --git a/py/objint.c b/py/objint.c index cabbd7b8f0e..e8f7f51052a 100644 --- a/py/objint.c +++ b/py/objint.c @@ -30,7 +30,7 @@ #include "py/parsenum.h" #include "py/smallint.h" -#include "py/objint.h" +#include "py/objint_impl.h" #include "py/objstr.h" #include "py/runtime.h" #include "py/binary.h" @@ -99,8 +99,8 @@ static mp_fp_as_int_class_t mp_classify_fp_as_int(mp_float_t val) { #elif MICROPY_FLOAT_IMPL == MICROPY_FLOAT_IMPL_DOUBLE e = u.i[MP_ENDIANNESS_LITTLE]; #endif -#define MP_FLOAT_SIGN_SHIFT_I32 ((MP_FLOAT_FRAC_BITS + MP_FLOAT_EXP_BITS) % 32) -#define MP_FLOAT_EXP_SHIFT_I32 (MP_FLOAT_FRAC_BITS % 32) + #define MP_FLOAT_SIGN_SHIFT_I32 ((MP_FLOAT_FRAC_BITS + MP_FLOAT_EXP_BITS) % 32) + #define MP_FLOAT_EXP_SHIFT_I32 (MP_FLOAT_FRAC_BITS % 32) if (e & (1U << MP_FLOAT_SIGN_SHIFT_I32)) { #if MICROPY_FLOAT_IMPL == MICROPY_FLOAT_IMPL_DOUBLE @@ -207,10 +207,10 @@ static const uint8_t log_base2_floor[] = { 3, 3, 3, 3, 3, 3, 3, 4, /* if needed, these are the values for higher bases - 4, 4, 4, 4, - 4, 4, 4, 4, - 4, 4, 4, 4, - 4, 4, 4, 5 + 4, 4, 4, 4, + 4, 4, 4, 4, + 4, 4, 4, 4, + 4, 4, 4, 5 */ }; @@ -309,74 +309,9 @@ char *mp_obj_int_formatted(char **buf, size_t *buf_size, size_t *fmt_size, mp_co return b; } -// CIRCUITPY-CHANGE: more thorough checking -#if MICROPY_LONGINT_IMPL != MICROPY_LONGINT_IMPL_NONE - -void mp_obj_int_buffer_overflow_check(mp_obj_t self_in, size_t nbytes, bool is_signed) { - if (is_signed) { - // self must be < 2**(bits - 1) - mp_obj_t edge = mp_binary_op(MP_BINARY_OP_LSHIFT, - mp_obj_new_int(1), - mp_obj_new_int(nbytes * 8 - 1)); - - if (mp_binary_op(MP_BINARY_OP_LESS, self_in, edge) == mp_const_true) { - // and >= -2**(bits - 1) - edge = mp_unary_op(MP_UNARY_OP_NEGATIVE, edge); - if (mp_binary_op(MP_BINARY_OP_MORE_EQUAL, self_in, edge) == mp_const_true) { - return; - } - } - } else { - // self must be >= 0 - if (mp_obj_int_sign(self_in) >= 0) { - // and < 2**(bits) - mp_obj_t edge = mp_binary_op(MP_BINARY_OP_LSHIFT, - mp_obj_new_int(1), - mp_obj_new_int(nbytes * 8)); - - if (mp_binary_op(MP_BINARY_OP_LESS, self_in, edge) == mp_const_true) { - return; - } - } - } - - mp_raise_OverflowError_varg(MP_ERROR_TEXT("value must fit in %d byte(s)"), nbytes); -} - -#endif // MICROPY_LONGINT_IMPL != MICROPY_LONGINT_IMPL_NONE - +// CIRCUITPY-CHANGE: also called from py/binary.c, so not static here. void mp_small_int_buffer_overflow_check(mp_int_t val, size_t nbytes, bool is_signed) { - // Fast path for zero. - if (val == 0) { - return; - } - - // Trying to store negative values in unsigned bytes falls through to failure. - if (is_signed || val >= 0) { - - if (nbytes >= sizeof(val)) { - // All non-negative N bit signed integers fit in an unsigned N bit integer. - // This case prevents shifting too far below. - return; - } - - if (is_signed) { - mp_int_t edge = ((mp_int_t)1 << (nbytes * 8 - 1)); - if (-edge <= val && val < edge) { - return; - } - // Out of range, fall through to failure. - } else { - // Unsigned. We already know val >= 0. - mp_int_t edge = ((mp_int_t)1 << (nbytes * 8)); - if (val < edge) { - return; - } - } - // Fall through to failure. - } - - mp_raise_OverflowError_varg(MP_ERROR_TEXT("value must fit in %d byte(s)"), nbytes); + mp_obj_small_int_buffer_overflow_check(val, nbytes, is_signed); } #if MICROPY_LONGINT_IMPL == MICROPY_LONGINT_IMPL_NONE @@ -447,6 +382,10 @@ mp_int_t mp_obj_int_get_checked(mp_const_obj_t self_in) { return MP_OBJ_SMALL_INT_VALUE(self_in); } +void mp_obj_int_to_bytes(mp_obj_t self_in, size_t buf_len, byte *buf, bool big_endian, bool is_signed, bool overflow_check) { + mp_obj_small_int_to_bytes(MP_OBJ_SMALL_INT_VALUE(self_in), buf_len, buf, big_endian, is_signed, overflow_check); +} + #endif // MICROPY_LONGINT_IMPL == MICROPY_LONGINT_IMPL_NONE // This dispatcher function is expected to be independent of the implementation of long int @@ -456,7 +395,7 @@ mp_obj_t mp_obj_int_binary_op_extra_cases(mp_binary_op_t op, mp_obj_t lhs_in, mp // false acts as 0 return mp_binary_op(op, lhs_in, MP_OBJ_NEW_SMALL_INT(0)); } else if (rhs_in == mp_const_true) { - // true acts as 0 + // true acts as 1 return mp_binary_op(op, lhs_in, MP_OBJ_NEW_SMALL_INT(1)); } else if (op == MP_BINARY_OP_MULTIPLY) { if (mp_obj_is_str_or_bytes(rhs_in) || mp_obj_is_type(rhs_in, &mp_type_tuple) || mp_obj_is_type(rhs_in, &mp_type_list)) { @@ -539,49 +478,31 @@ static mp_obj_t int_from_bytes(size_t n_args, const mp_obj_t *pos_args, mp_map_t static MP_DEFINE_CONST_FUN_OBJ_KW(int_from_bytes_fun_obj, 2, int_from_bytes); static MP_DEFINE_CONST_CLASSMETHOD_OBJ(int_from_bytes_obj, MP_ROM_PTR(&int_from_bytes_fun_obj)); -// CIRCUITPY-CHANGE: supports signed static mp_obj_t int_to_bytes(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) { enum { ARG_length, ARG_byteorder, ARG_signed }; static const mp_arg_t allowed_args[] = { - { MP_QSTR_length, MP_ARG_INT, {.u_int = 1} }, - // CIRCUITPY-CHANGE: not required and given a default value. - { MP_QSTR_byteorder, MP_ARG_OBJ, {.u_obj = MP_OBJ_NEW_QSTR(MP_QSTR_big)} }, + { MP_QSTR_length, MP_ARG_INT, { .u_int = 1 } }, + { MP_QSTR_byteorder, MP_ARG_OBJ, { .u_rom_obj = MP_ROM_QSTR(MP_QSTR_big) } }, { MP_QSTR_signed, MP_ARG_KW_ONLY | MP_ARG_BOOL, {.u_bool = false} }, }; mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)]; mp_arg_parse_all(n_args - 1, pos_args + 1, kw_args, MP_ARRAY_SIZE(allowed_args), allowed_args, args); - mp_int_t len = args[ARG_length].u_int; - if (len < 0) { + mp_obj_t self = pos_args[0]; + + mp_int_t dlen = args[ARG_length].u_int; + if (dlen < 0) { mp_raise_ValueError(NULL); } - mp_obj_t self = pos_args[0]; - bool big_endian = args[ARG_byteorder].u_obj != MP_OBJ_NEW_QSTR(MP_QSTR_little); - bool signed_ = args[ARG_signed].u_bool; - vstr_t vstr; - vstr_init_len(&vstr, len); + vstr_init_len(&vstr, dlen); byte *data = (byte *)vstr.buf; - memset(data, 0, len); - #if MICROPY_LONGINT_IMPL != MICROPY_LONGINT_IMPL_NONE - if (!mp_obj_is_small_int(self)) { - mp_obj_int_buffer_overflow_check(self, len, signed_); - mp_obj_int_to_bytes_impl(self, big_endian, len, data); - } else - #endif - { - mp_int_t val = MP_OBJ_SMALL_INT_VALUE(self); - // Small int checking is separate, to be fast. - mp_small_int_buffer_overflow_check(val, len, signed_); - size_t l = MIN((size_t)len, sizeof(val)); - if (val < 0) { - // Sign extend negative numbers. - memset(data, -1, len); - } - mp_binary_set_int(l, big_endian, data + (big_endian ? (len - l) : 0), val); - } + bool big_endian = args[ARG_byteorder].u_obj != MP_OBJ_NEW_QSTR(MP_QSTR_little); + bool signed_ = args[ARG_signed].u_bool; + + mp_obj_int_to_bytes(self, dlen, data, big_endian, signed_, true); return mp_obj_new_bytes_from_vstr(&vstr); } diff --git a/py/objint.h b/py/objint.h index 92466bae4e2..94c53190d8b 100644 --- a/py/objint.h +++ b/py/objint.h @@ -53,19 +53,17 @@ char *mp_obj_int_formatted(char **buf, size_t *buf_size, size_t *fmt_size, mp_co int base, const char *prefix, char base_char, char comma); char *mp_obj_int_formatted_impl(char **buf, size_t *buf_size, size_t *fmt_size, mp_const_obj_t self_in, int base, const char *prefix, char base_char, char comma); -// CIRCUITPY-CHANGE -#if MICROPY_LONGINT_IMPL != MICROPY_LONGINT_IMPL_NONE -void mp_obj_int_buffer_overflow_check(mp_obj_t self_in, size_t nbytes, bool is_signed); -#endif - -void mp_small_int_buffer_overflow_check(mp_int_t val, size_t nbytes, bool is_signed); mp_int_t mp_obj_int_hash(mp_obj_t self_in); // CIRCUITPY-CHANGE mp_obj_t mp_obj_int_bit_length_impl(mp_obj_t self_in); mp_obj_t mp_obj_int_from_bytes_impl(bool big_endian, size_t len, const byte *buf); -// Returns true if 'self_in' fit into 'len' bytes of 'buf' without overflowing, 'buf' is truncated otherwise. -bool mp_obj_int_to_bytes_impl(mp_obj_t self_in, bool big_endian, size_t len, byte *buf); +// Write an integer to a byte sequence. +// If overflow_check is true, raises OverflowError if 'self_in' doesn't fit. If false, truncate to fit. +void mp_obj_int_to_bytes(mp_obj_t self_in, size_t buf_len, byte *buf, bool big_endian, bool is_signed, bool overflow_check); + +// CIRCUITPY-CHANGE: raises OverflowError if val does not fit in nbytes. +void mp_small_int_buffer_overflow_check(mp_int_t val, size_t nbytes, bool is_signed); int mp_obj_int_sign(mp_obj_t self_in); mp_obj_t mp_obj_int_unary_op(mp_unary_op_t op, mp_obj_t o_in); mp_obj_t mp_obj_int_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs_in); diff --git a/py/objint_impl.h b/py/objint_impl.h new file mode 100644 index 00000000000..b1af4d6244d --- /dev/null +++ b/py/objint_impl.h @@ -0,0 +1,98 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2026 Angus Gratton + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +/* This header provides some inline implementations of functions used by multiple objint*.c + files. + + Implementations here should only be called by one of the objint*.c files. If + called from more than one place in a single firmware, place into objint.c and + define in objint.h + */ +#ifndef MICROPY_INCLUDED_PY_OBJINT_IMPL_H +#define MICROPY_INCLUDED_PY_OBJINT_IMPL_H + +#include "py/binary.h" +#include "py/objint.h" +#include "py/runtime.h" + +static void mp_obj_int_raise_to_bytes_overflow_error(size_t nbytes) { + mp_raise_msg_varg(&mp_type_OverflowError, MP_ERROR_TEXT("value would overflow a %d byte buffer"), nbytes); +} + +static void mp_obj_int_raise_unsigned_negative_overflow_error(void) { + mp_raise_msg_varg(&mp_type_OverflowError, MP_ERROR_TEXT("can't convert negative int to unsigned")); +} + +static void mp_obj_small_int_buffer_overflow_check(mp_int_t val, size_t nbytes, bool is_signed) { + // Fast path for zero. + if (val == 0) { + return; + } + + if (!is_signed && val < 0) { + // Trying to store negative values in unsigned bytes + mp_obj_int_raise_unsigned_negative_overflow_error(); + } + + if (nbytes >= sizeof(val)) { + // All N bit small integers fit in an unsigned N bit integer. + // This case prevents shifting too far below. + return; + } + + if (nbytes == 0) { + // Can't fit a non-negative value in 0 bytes (prevents negative left shift, below) + goto raise; + } + + if (is_signed) { + mp_int_t edge = ((mp_int_t)1 << (nbytes * 8 - 1)); + if (-edge <= val && val < edge) { + return; + } + // Out of range, fall through to raise. + } else { + // Unsigned. We already know val >= 0. + mp_int_t edge = ((mp_int_t)1 << (nbytes * 8)); + if (val < edge) { + return; + } + // Fall through to raise. + } + +raise: + mp_obj_int_raise_to_bytes_overflow_error(nbytes); +} + +static inline void mp_obj_small_int_to_bytes(mp_int_t val, size_t buf_len, byte *buf, bool big_endian, bool is_signed, bool overflow_check) { + if (overflow_check) { + mp_obj_small_int_buffer_overflow_check(val, buf_len, is_signed); + } + mp_binary_set_int(buf_len, buf, sizeof(val), val, big_endian); + +} + +#endif // MICROPY_INCLUDED_PY_OBJINT_IMPL_H diff --git a/py/objint_longlong.c b/py/objint_longlong.c index f24aa0cc186..49ac0b429fc 100644 --- a/py/objint_longlong.c +++ b/py/objint_longlong.c @@ -29,7 +29,7 @@ #include #include "py/smallint.h" -#include "py/objint.h" +#include "py/objint_impl.h" #include "py/runtime.h" #if MICROPY_PY_BUILTINS_FLOAT @@ -73,42 +73,6 @@ mp_obj_t mp_obj_int_from_bytes_impl(bool big_endian, size_t len, const byte *buf return mp_obj_new_int_from_ll(value); } -bool mp_obj_int_to_bytes_impl(mp_obj_t self_in, bool big_endian, size_t len, byte *buf) { - assert(mp_obj_is_exact_type(self_in, &mp_type_int)); - mp_obj_int_t *self = self_in; - long long val = self->val; - size_t slen; // Number of bytes to represent val - - // This logic has a twin in objint.c - if (val > 0) { - slen = (sizeof(long long) * 8 - mp_clzll(val) + 7) / 8; - } else if (val < -1) { - slen = (sizeof(long long) * 8 - mp_clzll(~val) + 8) / 8; - } else { - // clz of 0 is defined, so 0 and -1 map to 0 and 1 - slen = -val; - } - - if (slen > len) { - return false; // Would overflow - // TODO: Determine whether to copy and truncate, as some callers probably expect this...? - } - - if (big_endian) { - byte *b = buf + len; - while (b > buf) { - *--b = val; - val >>= 8; - } - } else { - for (; len > 0; --len) { - *buf++ = val; - val >>= 8; - } - } - return true; -} - int mp_obj_int_sign(mp_obj_t self_in) { mp_longint_impl_t val; if (mp_obj_is_small_int(self_in)) { @@ -240,7 +204,7 @@ mp_obj_t mp_obj_int_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs_i // negative shift not allowed mp_raise_ValueError(MP_ERROR_TEXT("negative shift count")); } - overflow = rhs_val >= (sizeof(long long) * MP_BITS_PER_BYTE) + overflow = rhs_val >= (long long)(sizeof(long long) * MP_BITS_PER_BYTE) || lhs_val > (LLONG_MAX >> rhs_val) || lhs_val < (LLONG_MIN >> rhs_val); result = (unsigned long long)lhs_val << rhs_val; @@ -373,4 +337,88 @@ mp_float_t mp_obj_int_as_float_impl(mp_obj_t self_in) { } #endif +// Same as the general mp_small_int_buffer_overflow_check() in objint_impl.h, but using 64-bit integers +static void longint_buffer_overflow_check(mp_longint_impl_t val, size_t nbytes, bool is_signed) { + // Fast path for zero. + if (val == 0) { + return; + } + + if (!is_signed && val < 0) { + // Trying to store negative values in unsigned bytes + mp_obj_int_raise_unsigned_negative_overflow_error(); + } + + if (nbytes >= sizeof(val)) { + // All non-negative N bit signed integers fit in an unsigned N bit integer. + // This case prevents shifting too far below. + return; + } + + if (nbytes == 0) { + // Can't fit a non-zero value in 0 bytes (prevents negative left shift below) + goto raise; + } + + if (is_signed) { + mp_longint_impl_t edge = 1LL << (nbytes * 8 - 1); + if (-edge <= val && val < edge) { + return; + } + // Out of range, fall through to failure. + } else { + // Unsigned. We already know val >= 0. + mp_longint_impl_t edge = 1LL << (nbytes * 8); + if (val < edge) { + return; + } + // Fall through to failure. + } + +raise: + mp_obj_int_raise_to_bytes_overflow_error(nbytes); +} + +static void longint_to_bytes(long long val, bool big_endian, size_t len, byte *buf) { + MP_STATIC_ASSERT(sizeof(mp_uint_t) == 4); + mp_uint_t lower = val; + mp_uint_t upper = (val >> 32); + + if (big_endian) { + if (len > 4) { + // write the least significant 4 bytes at the end + mp_binary_set_int(4, buf + len - 4, sizeof(lower), lower, true); + // write most significant bytes at the start, extending if necessary + mp_binary_set_int(len - 4, buf, sizeof(upper), upper, true); + } else { + mp_binary_set_int(len, buf, sizeof(lower), lower, true); + } + } else { + // write the least significant 4 bytes at the start + mp_binary_set_int(len > 4 ? len - 4 : len, buf, sizeof(lower), lower, false); + if (len > 4) { + // write the most significant bytes at the end, extending if necessary + mp_binary_set_int(len - 4, buf + 4, sizeof(upper), upper, false); + } + } +} + +void mp_obj_int_to_bytes(mp_obj_t self_in, size_t buf_len, byte *buf, bool big_endian, bool is_signed, bool overflow_check) { + mp_longint_impl_t val; + if (mp_obj_is_exact_type(self_in, &mp_type_int)) { + const mp_obj_int_t *self = MP_OBJ_TO_PTR(self_in); + val = self->val; + } else { + // self_in is either a smallint, or another type convertible to mp_int_t (i.e. bool) + val = mp_obj_get_int(self_in); + } + + // Note: to save code size we don't call mp_obj_small_int_to_bytes() here, + // as the longint implementation is very similar + if (overflow_check) { + longint_buffer_overflow_check(val, buf_len, is_signed); + } + longint_to_bytes(val, big_endian, buf_len, buf); +} + #endif diff --git a/py/objint_mpz.c b/py/objint_mpz.c index 1bede811a4a..f002dfe68a9 100644 --- a/py/objint_mpz.c +++ b/py/objint_mpz.c @@ -30,7 +30,7 @@ #include "py/parsenumbase.h" #include "py/smallint.h" -#include "py/objint.h" +#include "py/objint_impl.h" #include "py/runtime.h" #if MICROPY_PY_BUILTINS_FLOAT @@ -119,12 +119,6 @@ mp_obj_t mp_obj_int_from_bytes_impl(bool big_endian, size_t len, const byte *buf return MP_OBJ_FROM_PTR(o); } -bool mp_obj_int_to_bytes_impl(mp_obj_t self_in, bool big_endian, size_t len, byte *buf) { - assert(mp_obj_is_exact_type(self_in, &mp_type_int)); - mp_obj_int_t *self = MP_OBJ_TO_PTR(self_in); - return mpz_as_bytes(&self->mpz, big_endian, self->mpz.neg, len, buf); -} - int mp_obj_int_sign(mp_obj_t self_in) { if (mp_obj_is_small_int(self_in)) { mp_int_t val = MP_OBJ_SMALL_INT_VALUE(self_in); @@ -483,4 +477,22 @@ mp_float_t mp_obj_int_as_float_impl(mp_obj_t self_in) { } #endif +void mp_obj_int_to_bytes(mp_obj_t self_in, size_t buf_len, byte *buf, bool big_endian, bool is_signed, bool overflow_check) { + if (mp_obj_is_exact_type(self_in, &mp_type_int)) { + const mp_obj_int_t *self = MP_OBJ_TO_PTR(self_in); + const mpz_t *mpz = &self->mpz; + if (overflow_check && !is_signed && mpz->neg) { + mp_obj_int_raise_unsigned_negative_overflow_error(); + } + if (!mpz_as_bytes(mpz, big_endian, is_signed, buf_len, buf) && overflow_check) { + mp_obj_int_raise_to_bytes_overflow_error(buf_len); + } + } else { + // self_in is either a smallint, or another type convertible to mp_int_t (i.e. bool) + mp_int_t val = mp_obj_get_int(self_in); + mp_obj_small_int_to_bytes(val, buf_len, buf, big_endian, is_signed, overflow_check); + } +} + + #endif diff --git a/py/objstr.c b/py/objstr.c index d8dd61d04af..1db72dc22f5 100644 --- a/py/objstr.c +++ b/py/objstr.c @@ -194,6 +194,79 @@ static void str_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind_t } } +#if MICROPY_PY_BUILTINS_STR_UNICODE_CHECK && MICROPY_PY_BUILTINS_BYTES_DECODE_ERRORS +// Build a new string from data containing invalid UTF-8 or ASCII, either skipping the +// invalid bytes (errors=="ignore") or replacing them with U+FFFD +// (errors=="replace"). +static mp_obj_t str_from_invalid(mp_encoding_t encoding, const mp_obj_type_t *type, const byte *str_data, size_t str_len, qstr errors) { + bool do_replace = (errors == MP_QSTR_replace); + bool is_utf8 = (encoding == MP_ENCODING_UTF8); + vstr_t vstr; + vstr_init(&vstr, str_len); + const byte *p = str_data; + const byte *end = str_data + str_len; + + while (p < end) { + byte c = *p; + if (c < 0x80) { + // Valid ASCII + vstr_add_byte(&vstr, c); + p++; + } else if (is_utf8 && c >= 0xc0 && c < 0xf8) { + // Potential multi-byte sequence + uint8_t need = (0xe5 >> ((c >> 3) & 0x6)) & 3; + const byte *seq_start = p; + p++; + + // Check continuation bytes + uint8_t got = 0; + while (got < need && p < end && UTF8_IS_CONT(*p)) { + got++; + p++; + } + + if (got == need) { + // Valid complete sequence, decode and add the character + unichar ch = *seq_start & (0x7f >> need); + for (uint8_t i = 0; i < need; i++) { + ch = (ch << 6) | (seq_start[i + 1] & 0x3f); + } + vstr_add_char(&vstr, ch); + } else if (do_replace) { + // Invalid or incomplete sequence - replace with U+FFFD + vstr_add_char(&vstr, 0xFFFD); + } + // For 'ignore' mode, do nothing (skip invalid bytes) + } else if (do_replace) { + // Invalid start byte or non-ascii char - replace with U+FFFD + vstr_add_char(&vstr, 0xFFFD); + p++; + } else { + // Invalid start byte or non-ascii char - skip for 'ignore' mode + p++; + } + } + + return mp_obj_new_str_type_from_vstr(type, &vstr); +} +#endif // MICROPY_PY_BUILTINS_STR_UNICODE_CHECK && MICROPY_PY_BUILTINS_BYTES_DECODE_ERRORS + +#if MICROPY_PY_BUILTINS_STR_UNICODE_CHECK +static mp_encoding_t parse_encoding_arg(qstr encoding) { + if (encoding == MP_QSTR_utf_hyphen_8 || encoding == MP_QSTR_utf8) { + return MP_ENCODING_UTF8; + } + if (encoding == MP_QSTR_ascii) { + return MP_ENCODING_ASCII; + } + #if MICROPY_ERROR_REPORTING <= MICROPY_ERROR_REPORTING_TERSE + mp_raise_type(&mp_type_LookupError); + #else + mp_raise_msg_varg(&mp_type_LookupError, MP_ERROR_TEXT("unknown encoding: %q"), encoding); + #endif +} +#endif + mp_obj_t mp_obj_str_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { #if MICROPY_CPYTHON_COMPAT if (n_kw != 0) { @@ -215,36 +288,61 @@ mp_obj_t mp_obj_str_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_ return mp_obj_new_str_type_from_vstr(type, &vstr); } - default: // 2 or 3 args - // TODO: validate 2nd/3rd args + default: { // 2 or 3 args + // Extract the source data. + const byte *str_data; + size_t str_len; if (mp_obj_is_type(args[0], &mp_type_bytes)) { - GET_STR_DATA_LEN(args[0], str_data, str_len); - GET_STR_HASH(args[0], str_hash); - if (str_hash == 0) { - str_hash = qstr_compute_hash(str_data, str_len); + // Immutable bytes can be referenced directly (zero-copy); + GET_STR_DATA_LEN(args[0], bytes_data, bytes_len); + str_data = bytes_data; + str_len = bytes_len; + } else { + // any other buffer object is mutable sob its data must be copied. + mp_buffer_info_t bufinfo; + mp_get_buffer_raise(args[0], &bufinfo, MP_BUFFER_READ); + str_data = bufinfo.buf; + str_len = bufinfo.len; + } + + #if MICROPY_PY_BUILTINS_STR_UNICODE_CHECK + mp_encoding_t encoding = parse_encoding_arg(mp_obj_str_get_qstr(args[1])); + if (!unicode_encoding_check(encoding, str_data, str_len)) { + #if MICROPY_PY_BUILTINS_BYTES_DECODE_ERRORS + // Check if error handler is specified (3rd argument) + qstr errors = MP_QSTR_; // default to "" + if (n_args >= 3 && args[2] != mp_const_none) { + errors = mp_obj_str_get_qstr(args[2]); } - #if MICROPY_PY_BUILTINS_STR_UNICODE_CHECK - if (!utf8_check(str_data, str_len)) { - mp_raise_msg(&mp_type_UnicodeError, NULL); + if (errors == MP_QSTR_ignore || errors == MP_QSTR_replace) { + return str_from_invalid(encoding, type, str_data, str_len, errors); } - #endif + #endif // MICROPY_PY_BUILTINS_BYTES_DECODE_ERRORS + mp_raise_msg(&mp_type_UnicodeError, NULL); + } + #endif // MICROPY_PY_BUILTINS_STR_UNICODE_CHECK - // Check if a qstr with this data already exists - qstr q = qstr_find_strn((const char *)str_data, str_len); - if (q != MP_QSTRnull) { - return MP_OBJ_NEW_QSTR(q); - } + // Check if a qstr with this data already exists + qstr q = qstr_find_strn((const char *)str_data, str_len); + if (q != MP_QSTRnull) { + return MP_OBJ_NEW_QSTR(q); + } - mp_obj_str_t *o = MP_OBJ_TO_PTR(mp_obj_new_str_copy(type, NULL, str_len)); - o->data = str_data; - o->hash = str_hash; - return MP_OBJ_FROM_PTR(o); - } else { - mp_buffer_info_t bufinfo; - mp_get_buffer_raise(args[0], &bufinfo, MP_BUFFER_READ); - // This will utf-8 check the input. - return mp_obj_new_str(bufinfo.buf, bufinfo.len); + if (!mp_obj_is_type(args[0], &mp_type_bytes)) { + // Source is a mutable buffer: copy the data. + return mp_obj_new_str_copy(type, str_data, str_len); } + + // Source is immutable bytes: reference its data without copying. + GET_STR_HASH(args[0], str_hash); + if (str_hash == 0) { + str_hash = qstr_compute_hash(str_data, str_len); + } + mp_obj_str_t *o = MP_OBJ_TO_PTR(mp_obj_new_str_copy(type, NULL, str_len)); + o->data = str_data; + o->hash = str_hash; + return MP_OBJ_FROM_PTR(o); + } } } @@ -275,11 +373,20 @@ static mp_obj_t bytes_make_new(const mp_obj_type_t *type_in, size_t n_args, size mp_raise_TypeError(MP_ERROR_TEXT("string argument without an encoding")); #endif } + GET_STR_DATA_LEN(args[0], str_data, str_len); GET_STR_HASH(args[0], str_hash); if (str_hash == 0) { str_hash = qstr_compute_hash(str_data, str_len); } + + #if MICROPY_PY_BUILTINS_STR_UNICODE_CHECK + mp_encoding_t encoding = parse_encoding_arg(mp_obj_str_get_qstr(args[1])); + if (encoding == MP_ENCODING_ASCII && !unicode_encoding_check(MP_ENCODING_ASCII, str_data, str_len)) { + mp_raise_msg(&mp_type_UnicodeError, NULL); + } + #endif + mp_obj_str_t *o = MP_OBJ_TO_PTR(mp_obj_new_str_copy(&mp_type_bytes, NULL, str_len)); o->data = str_data; o->hash = str_hash; @@ -787,11 +894,29 @@ static mp_obj_t str_finder(size_t n_args, const mp_obj_t *args, int direction, b const mp_obj_type_t *self_type = mp_obj_get_type(args[0]); check_is_str_or_bytes(args[0]); - // check argument type - str_check_arg_type(self_type, args[1]); - GET_STR_DATA_LEN(args[0], haystack, haystack_len); - GET_STR_DATA_LEN(args[1], needle, needle_len); + + mp_int_t val; + byte needle_data; + const byte *needle; + size_t needle_len; + if (self_type != &mp_type_str && mp_obj_get_int_maybe(args[1], &val)) { + // Allow {bytes/bytearray}.{find,index}(int). + #if MICROPY_FULL_CHECKS + if (val < 0 || val > 255) { + mp_raise_ValueError(MP_ERROR_TEXT("bytes value out of range")); + } + #endif + needle_data = val; + needle = &needle_data; + needle_len = 1; + } else { + // check argument type + str_check_arg_type(self_type, args[1]); + GET_STR_DATA_LEN(args[1], needle_tmp, needle_len_tmp); + needle = needle_tmp; + needle_len = needle_len_tmp; + } const byte *start = haystack; const byte *end = haystack + haystack_len; @@ -961,14 +1086,33 @@ MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(str_rstrip_obj, 1, 2, str_rstrip); static mp_obj_t str_center(mp_obj_t str_in, mp_obj_t width_in) { GET_STR_DATA_LEN(str_in, str, str_len); mp_uint_t width = mp_obj_get_int(width_in); + + #if MICROPY_PY_BUILTINS_STR_UNICODE + // Get character count (not byte count) for proper Unicode handling + size_t char_len = utf8_charlen(str, str_len); + if (char_len >= width) { + return str_in; + } + // Calculate padding: width is in characters, need to convert to bytes for allocation + mp_uint_t padding_chars = width - char_len; + // Padding is always spaces (1 byte each), plus the original string bytes + mp_uint_t total_bytes = padding_chars + str_len; + #else + // Non-Unicode build: byte length equals character length if (str_len >= width) { return str_in; } + mp_uint_t total_bytes = width; + #endif // MICROPY_PY_BUILTINS_STR_UNICODE vstr_t vstr; - vstr_init_len(&vstr, width); - memset(vstr.buf, ' ', width); + vstr_init_len(&vstr, total_bytes); + memset(vstr.buf, ' ', total_bytes); + #if MICROPY_PY_BUILTINS_STR_UNICODE + int left = padding_chars / 2; + #else int left = (width - str_len) / 2; + #endif // MICROPY_PY_BUILTINS_STR_UNICODE memcpy(vstr.buf + left, str, str_len); return mp_obj_new_str_type_from_vstr(mp_obj_get_type(str_in), &vstr); } @@ -1029,6 +1173,23 @@ static MP_NORETURN void terse_str_format_value_error(void) { #define terse_str_format_value_error() #endif +// Print the character with the code point given by the integer object arg. +// Used by both the str.format and the modulo (%c) formatters. +static void mp_print_char(const mp_print_t *print, mp_obj_t arg, unsigned int flags, char fill, int width) { + #if MICROPY_FULL_CHECKS + mp_uint_t c = mp_obj_get_int(arg); + if (c >= 0x110000) { + mp_raise_msg(&mp_type_OverflowError, MP_ERROR_TEXT("char not in range(0x110000)")); + } + VSTR_FIXED(ch_vstr, 4); + vstr_add_char(&ch_vstr, c); + mp_print_strn(print, ch_vstr.buf, ch_vstr.len, flags, fill, width); + #else + char ch = mp_obj_get_int(arg); + mp_print_strn(print, &ch, 1, flags, fill, width); + #endif +} + static vstr_t mp_obj_str_format_helper(const char *str, const char *top, int *arg_i, size_t n_args, const mp_obj_t *args, mp_map_t *kwargs) { vstr_t vstr; mp_print_t print; @@ -1322,8 +1483,7 @@ static vstr_t mp_obj_str_format_helper(const char *str, const char *top, int *ar continue; case 'c': { - char ch = mp_obj_get_int(arg); - mp_print_strn(&print, &ch, 1, flags, fill, width); + mp_print_char(&print, arg, flags, fill, width); continue; } @@ -1619,8 +1779,7 @@ static mp_obj_t str_modulo_format(mp_obj_t pattern, size_t n_args, const mp_obj_ } mp_print_strn(&print, s, 1, flags, ' ', width); } else if (arg_looks_integer(arg)) { - char ch = mp_obj_get_int(arg); - mp_print_strn(&print, &ch, 1, flags, ' ', width); + mp_print_char(&print, arg, flags, ' ', width); } else { mp_raise_TypeError(MP_ERROR_TEXT("%%c needs int or char")); } @@ -1982,6 +2141,7 @@ MP_DEFINE_CONST_FUN_OBJ_1(str_islower_obj, str_islower); #if MICROPY_CPYTHON_COMPAT // These methods are superfluous in the presence of str() and bytes() // constructors. + // TODO: should accept kwargs too static mp_obj_t bytes_decode(size_t n_args, const mp_obj_t *args) { mp_obj_t new_args[2]; @@ -2312,7 +2472,7 @@ static mp_obj_t mp_obj_new_str_type_from_vstr(const mp_obj_type_t *type, vstr_t mp_obj_t mp_obj_new_str_from_vstr(vstr_t *vstr) { #if MICROPY_PY_BUILTINS_STR_UNICODE && MICROPY_PY_BUILTINS_STR_UNICODE_CHECK - if (!utf8_check((byte *)vstr->buf, vstr->len)) { + if (!unicode_encoding_check(MP_ENCODING_UTF8, (byte *)vstr->buf, vstr->len)) { mp_raise_msg(&mp_type_UnicodeError, NULL); } #endif // MICROPY_PY_BUILTINS_STR_UNICODE && MICROPY_PY_BUILTINS_STR_UNICODE_CHECK @@ -2332,7 +2492,7 @@ mp_obj_t mp_obj_new_bytes_from_vstr(vstr_t *vstr) { mp_obj_t mp_obj_new_str(const char *data, size_t len) { #if MICROPY_PY_BUILTINS_STR_UNICODE && MICROPY_PY_BUILTINS_STR_UNICODE_CHECK - if (!utf8_check((byte *)data, len)) { + if (!unicode_encoding_check(MP_ENCODING_UTF8, (byte *)data, len)) { mp_raise_msg(&mp_type_UnicodeError, NULL); } #endif diff --git a/py/objstrunicode.c b/py/objstrunicode.c index a158b912365..6dbdccb77fa 100644 --- a/py/objstrunicode.c +++ b/py/objstrunicode.c @@ -57,11 +57,11 @@ static void uni_print_quoted(const mp_print_t *print, const byte *str_data, uint mp_printf(print, "%c", quote_char); const byte *s = str_data, *top = str_data + str_len; while (s < top) { + const byte *seq_start = s; unichar ch; ch = utf8_get_char(s); - // CIRCUITPY-CHANGE: print printable Unicode chars - const byte *start = s; s = utf8_next_char(s); + size_t seq_len = s - seq_start; if (ch == quote_char) { mp_printf(print, "\\%c", quote_char); } else if (ch == '\\') { @@ -79,12 +79,15 @@ static void uni_print_quoted(const mp_print_t *print, const byte *str_data, uint mp_printf(print, "\\x%02x", ch); } else if ((0x2000 <= ch && ch <= 0x200f) || ch == 0x2028 || ch == 0x2029 || ch == 0xffff) { mp_printf(print, "\\u%04x", ch); - } else if (ch == 0x1ffff) { + } else if (ch >= 0xd800 && ch < 0xe000) { + // Surrogate (0xD800-0xDFFF) - output as \uXXXX escape. + mp_printf(print, "\\u%04x", ch); + } else if (ch == 0x1ffff || ch >= 0x110000) { + // Invalid - output as \UXXXXXXXX escape. mp_printf(print, "\\U%08x", ch); } else { - // Print the full character out. - int width = s - start; - mp_print_strn(print, (const char *)start, width, 0, ' ', width); + // CIRCUITPY-CHANGE: print printable Unicode chars out in full. + mp_print_strn(print, (const char *)seq_start, seq_len, 0, ' ', seq_len); } } mp_printf(print, "%c", quote_char); diff --git a/py/objtuple.c b/py/objtuple.c index fcfb8e0aa1f..6808f6c6e0a 100644 --- a/py/objtuple.c +++ b/py/objtuple.c @@ -104,20 +104,27 @@ static mp_obj_t mp_obj_tuple_make_new(const mp_obj_type_t *type_in, size_t n_arg } } +static mp_obj_tuple_t *tuple_subclass_helper(mp_obj_t obj) { + assert(obj != MP_OBJ_NULL); + const mp_obj_type_t *tuple_type = mp_obj_get_type(obj); + if (MP_OBJ_TYPE_GET_SLOT_OR_NULL(tuple_type, iter) != mp_obj_tuple_getiter) { + // Slow path for user subclasses + obj = mp_obj_cast_to_native_base(obj, MP_OBJ_FROM_PTR(&mp_type_tuple)); + if (obj == MP_OBJ_NULL) { + return NULL; + } + } + return MP_OBJ_TO_PTR(obj); +} + // Don't pass MP_BINARY_OP_NOT_EQUAL here static mp_obj_t tuple_cmp_helper(mp_uint_t op, mp_obj_t self_in, mp_obj_t another_in) { mp_check_self(mp_obj_is_tuple_compatible(self_in)); - const mp_obj_type_t *another_type = mp_obj_get_type(another_in); mp_obj_tuple_t *self = MP_OBJ_TO_PTR(self_in); - if (MP_OBJ_TYPE_GET_SLOT_OR_NULL(another_type, iter) != mp_obj_tuple_getiter) { - // Slow path for user subclasses - another_in = mp_obj_cast_to_native_base(another_in, MP_OBJ_FROM_PTR(&mp_type_tuple)); - if (another_in == MP_OBJ_NULL) { - return MP_OBJ_NULL; - } + mp_obj_tuple_t *another = tuple_subclass_helper(another_in); + if (!another) { + return MP_OBJ_NULL; } - mp_obj_tuple_t *another = MP_OBJ_TO_PTR(another_in); - return mp_obj_new_bool(mp_seq_cmp_objs(op, self->items, self->len, another->items, another->len)); } @@ -146,10 +153,10 @@ mp_obj_t mp_obj_tuple_binary_op(mp_binary_op_t op, mp_obj_t lhs, mp_obj_t rhs) { switch (op) { case MP_BINARY_OP_ADD: case MP_BINARY_OP_INPLACE_ADD: { - if (!mp_obj_is_subclass_fast(MP_OBJ_FROM_PTR(mp_obj_get_type(rhs)), MP_OBJ_FROM_PTR(&mp_type_tuple))) { + mp_obj_tuple_t *p = tuple_subclass_helper(rhs); + if (!p) { return MP_OBJ_NULL; // op not supported } - mp_obj_tuple_t *p = MP_OBJ_TO_PTR(rhs); mp_obj_tuple_t *s = MP_OBJ_TO_PTR(mp_obj_new_tuple(o->len + p->len, NULL)); mp_seq_cat(s->items, o->items, o->len, p->items, p->len, mp_obj_t); return MP_OBJ_FROM_PTR(s); diff --git a/py/parse.c b/py/parse.c index 36150ef468b..203527c782d 100644 --- a/py/parse.c +++ b/py/parse.c @@ -496,10 +496,10 @@ void mp_parse_node_print(const mp_print_t *print, mp_parse_node_t pn, size_t ind uintptr_t arg = MP_PARSE_NODE_LEAF_ARG(pn); switch (MP_PARSE_NODE_LEAF_KIND(pn)) { case MP_PARSE_NODE_ID: - mp_printf(print, "id(%s)\n", qstr_str(arg)); + mp_printf(print, "id(%q)\n", (qstr)arg); break; case MP_PARSE_NODE_STRING: - mp_printf(print, "str(%s)\n", qstr_str(arg)); + mp_printf(print, "str(%q)\n", (qstr)arg); break; default: assert(MP_PARSE_NODE_LEAF_KIND(pn) == MP_PARSE_NODE_TOKEN); @@ -825,50 +825,54 @@ static bool fold_constants(parser_t *parser, uint8_t rule_id, size_t num_args) { if (!MP_PARSE_NODE_IS_NULL(pn1) && !(MP_PARSE_NODE_IS_STRUCT_KIND(pn1, RULE_expr_stmt_augassign) || MP_PARSE_NODE_IS_STRUCT_KIND(pn1, RULE_expr_stmt_assign_list))) { - // this node is of the form = + // this node is of the form = or : = mp_parse_node_t pn0 = peek_result(parser, 1); - if (MP_PARSE_NODE_IS_ID(pn0) - && MP_PARSE_NODE_IS_STRUCT_KIND(pn1, RULE_atom_expr_normal) - && MP_PARSE_NODE_IS_ID(((mp_parse_node_struct_t *)pn1)->nodes[0]) - && MP_PARSE_NODE_LEAF_ARG(((mp_parse_node_struct_t *)pn1)->nodes[0]) == MP_QSTR_const - && MP_PARSE_NODE_IS_STRUCT_KIND(((mp_parse_node_struct_t *)pn1)->nodes[1], RULE_trailer_paren) - ) { - // code to assign dynamic constants: id = const(value) - - // get the id - qstr id = MP_PARSE_NODE_LEAF_ARG(pn0); - - // get the value - mp_parse_node_t pn_value = ((mp_parse_node_struct_t *)((mp_parse_node_struct_t *)pn1)->nodes[1])->nodes[0]; - if (!mp_parse_node_is_const(pn_value)) { - mp_obj_t exc = mp_obj_new_exception_msg(&mp_type_SyntaxError, - MP_ERROR_TEXT("not a constant")); - mp_obj_exception_add_traceback(exc, parser->lexer->source_name, - ((mp_parse_node_struct_t *)pn1)->source_line, MP_QSTRnull); - nlr_raise(exc); - } - mp_obj_t value = mp_parse_node_convert_to_obj(pn_value); - - // store the value in the table of dynamic constants - mp_map_elem_t *elem = mp_map_lookup(&parser->consts, MP_OBJ_NEW_QSTR(id), MP_MAP_LOOKUP_ADD_IF_NOT_FOUND); - assert(elem->value == MP_OBJ_NULL); - elem->value = value; - - // If the constant starts with an underscore then treat it as a private - // variable and don't emit any code to store the value to the id. - if (qstr_str(id)[0] == '_') { - pop_result(parser); // pop const(value) - pop_result(parser); // pop id - push_result_rule(parser, 0, RULE_pass_stmt, 0); // replace with "pass" - return true; + if (MP_PARSE_NODE_IS_ID(pn0)) { + if (MP_PARSE_NODE_IS_STRUCT_KIND(pn1, RULE_annassign)) { + // extract annassign rhs + pn1 = ((mp_parse_node_struct_t *)pn1)->nodes[1]; } + if (MP_PARSE_NODE_IS_STRUCT_KIND(pn1, RULE_atom_expr_normal) + && MP_PARSE_NODE_IS_ID(((mp_parse_node_struct_t *)pn1)->nodes[0]) + && MP_PARSE_NODE_LEAF_ARG(((mp_parse_node_struct_t *)pn1)->nodes[0]) == MP_QSTR_const + && MP_PARSE_NODE_IS_STRUCT_KIND(((mp_parse_node_struct_t *)pn1)->nodes[1], RULE_trailer_paren)) { + // code to assign dynamic constants: id = const(value) + + // get the id + qstr id = MP_PARSE_NODE_LEAF_ARG(pn0); + + // get the value + mp_parse_node_t pn_value = ((mp_parse_node_struct_t *)((mp_parse_node_struct_t *)pn1)->nodes[1])->nodes[0]; + if (!mp_parse_node_is_const(pn_value)) { + mp_obj_t exc = mp_obj_new_exception_msg(&mp_type_SyntaxError, + MP_ERROR_TEXT("not a constant")); + mp_obj_exception_add_traceback(exc, parser->lexer->source_name, + ((mp_parse_node_struct_t *)pn1)->source_line, MP_QSTRnull); + nlr_raise(exc); + } + mp_obj_t value = mp_parse_node_convert_to_obj(pn_value); + + // store the value in the table of dynamic constants + mp_map_elem_t *elem = mp_map_lookup(&parser->consts, MP_OBJ_NEW_QSTR(id), MP_MAP_LOOKUP_ADD_IF_NOT_FOUND); + assert(elem->value == MP_OBJ_NULL); + elem->value = value; + + // If the constant starts with an underscore then treat it as a private + // variable and don't emit any code to store the value to the id. + if (qstr_str(id)[0] == '_') { + pop_result(parser); // pop const(value) + pop_result(parser); // pop id + push_result_rule(parser, 0, RULE_pass_stmt, 0); // replace with "pass" + return true; + } - // replace const(value) with value - pop_result(parser); - push_result_node(parser, pn_value); + // replace const(value) with value + pop_result(parser); + push_result_node(parser, pn_value); - // finished folding this assignment, but we still want it to be part of the tree - return false; + // finished folding this assignment, but we still want it to be part of the tree + return false; + } } } return false; diff --git a/py/persistentcode.c b/py/persistentcode.c index 7fea1465975..7542a97f000 100644 --- a/py/persistentcode.c +++ b/py/persistentcode.c @@ -569,9 +569,8 @@ static void mp_print_bytes(mp_print_t *print, const byte *data, size_t len) { print->print_strn(print->data, (const char *)data, len); } -#define BYTES_FOR_INT ((MP_BYTES_PER_OBJ_WORD * 8 + 6) / 7) static void mp_print_uint(mp_print_t *print, size_t n) { - byte buf[BYTES_FOR_INT]; + byte buf[MP_ENCODE_UINT_MAX_BYTES]; byte *p = buf + sizeof(buf); *--p = n & 0x7f; n >>= 7; diff --git a/py/py.cmake b/py/py.cmake index 6888c170214..9d4fa15bd41 100644 --- a/py/py.cmake +++ b/py/py.cmake @@ -4,6 +4,12 @@ set(MICROPY_PY_DIR "${MICROPY_DIR}/py") list(APPEND MICROPY_INC_CORE "${MICROPY_DIR}") +# Set MICROPY_LIB_DIR default if not already set by the port. +# This needs to happen before usermod.cmake is included (for c_module() in manifests). +if(NOT MICROPY_LIB_DIR) + set(MICROPY_LIB_DIR ${MICROPY_DIR}/lib/micropython-lib) +endif() + # All py/ source files set(MICROPY_SOURCE_PY ${MICROPY_PY_DIR}/argcheck.c @@ -61,6 +67,7 @@ set(MICROPY_SOURCE_PY ${MICROPY_PY_DIR}/nativeglue.c ${MICROPY_PY_DIR}/nlr.c ${MICROPY_PY_DIR}/nlraarch64.c + ${MICROPY_PY_DIR}/nlrloong64.c ${MICROPY_PY_DIR}/nlrmips.c ${MICROPY_PY_DIR}/nlrpowerpc.c ${MICROPY_PY_DIR}/nlrrv32.c diff --git a/py/py.mk b/py/py.mk index f5365ed6f78..ff7de79b9d8 100644 --- a/py/py.mk +++ b/py/py.mk @@ -24,21 +24,15 @@ QSTR_GLOBAL_REQUIREMENTS += $(HEADER_BUILD)/mpversion.h # some code is performance bottleneck and compiled with other optimization options CSUPEROPT = -O3 -# Enable building 32-bit code on 64-bit host. -ifeq ($(MICROPY_FORCE_32BIT),1) -CC += -m32 -CXX += -m32 -LD += -m32 -endif - # External modules written in C. + +# Process manifest for C module extraction (appends to USER_C_MODULES). +include $(TOP)/py/manifest.mk + ifneq ($(USER_C_MODULES),) # pre-define USERMOD variables as expanded so that variables are immediate # expanded as they're added to them -# Confirm the provided path exists, show abspath if not to make it clearer to fix. -$(if $(wildcard $(USER_C_MODULES)/.),,$(error USER_C_MODULES doesn't exist: $(abspath $(USER_C_MODULES)))) - # C/C++ files that are included in the QSTR/module build SRC_USERMOD_C := SRC_USERMOD_CXX := @@ -50,28 +44,58 @@ SRC_USERMOD_LIB_ASM := CFLAGS_USERMOD := CXXFLAGS_USERMOD := LDFLAGS_USERMOD := +LIBS_USERMOD := # Backwards compatibility with older user c modules that set SRC_USERMOD # added to SRC_USERMOD_C below SRC_USERMOD := -$(foreach module, $(wildcard $(USER_C_MODULES)/*/micropython.mk), \ - $(eval USERMOD_DIR = $(patsubst %/,%,$(dir $(module))))\ - $(info Including User C Module from $(USERMOD_DIR))\ - $(eval include $(module))\ +# For each C module directory, scan for micropython.mk and include it. +# Accumulate USERMOD_DIRS so we can later strip each module's parent path +# from its source files while preserving the module basename in the build +# tree (e.g. build/cexample/foo.o, not build/foo.o, so files that share a +# name across modules don't collide on the same .o target). +USERMOD_DIRS := +$(foreach _UDIR, $(USER_C_MODULES), \ + $(if $(wildcard $(_UDIR)/.),,$(error USER_C_MODULES path doesn't exist: $(abspath $(_UDIR)))) \ + $(foreach module, $(wildcard $(_UDIR)/micropython.mk) $(wildcard $(_UDIR)/*/micropython.mk), \ + $(eval USERMOD_DIR = $(patsubst %/,%,$(dir $(module))))\ + $(eval USERMOD_DIRS += $(USERMOD_DIR))\ + $(info Including User C Module from $(USERMOD_DIR))\ + $(eval include $(module))\ + )\ ) SRC_USERMOD_C += $(SRC_USERMOD) -SRC_USERMOD_PATHFIX_C += $(patsubst $(USER_C_MODULES)/%.c,%.c,$(SRC_USERMOD_C)) -SRC_USERMOD_PATHFIX_CXX += $(patsubst $(USER_C_MODULES)/%.cpp,%.cpp,$(SRC_USERMOD_CXX)) -SRC_USERMOD_PATHFIX_LIB_C += $(patsubst $(USER_C_MODULES)/%.c,%.c,$(SRC_USERMOD_LIB_C)) -SRC_USERMOD_PATHFIX_LIB_CXX += $(patsubst $(USER_C_MODULES)/%.cpp,%.cpp,$(SRC_USERMOD_LIB_CXX)) -SRC_USERMOD_PATHFIX_LIB_ASM += $(patsubst $(USER_C_MODULES)/%.S,%.S,$(SRC_USERMOD_LIB_ASM)) +# Strip each USERMOD_DIR's leading path from source files, keeping the module +# basename as the leading directory so build outputs land at +# BUILD//.o. Uses per-directory patsubst on the paths as-is +# rather than $(abspath) so a parent directory containing whitespace doesn't +# break word-based patsubst. +SRC_USERMOD_PATHFIX_C := $(SRC_USERMOD_C) +SRC_USERMOD_PATHFIX_CXX := $(SRC_USERMOD_CXX) +SRC_USERMOD_PATHFIX_LIB_C := $(SRC_USERMOD_LIB_C) +SRC_USERMOD_PATHFIX_LIB_CXX := $(SRC_USERMOD_LIB_CXX) +SRC_USERMOD_PATHFIX_LIB_ASM := $(SRC_USERMOD_LIB_ASM) +USERMOD_DIRS := $(sort $(USERMOD_DIRS)) +$(foreach _MDIR, $(USERMOD_DIRS), \ + $(eval _MBASE := $(notdir $(_MDIR))) \ + $(eval SRC_USERMOD_PATHFIX_C := $$(patsubst $(_MDIR)/%,$(_MBASE)/%,$$(SRC_USERMOD_PATHFIX_C))) \ + $(eval SRC_USERMOD_PATHFIX_CXX := $$(patsubst $(_MDIR)/%,$(_MBASE)/%,$$(SRC_USERMOD_PATHFIX_CXX))) \ + $(eval SRC_USERMOD_PATHFIX_LIB_C := $$(patsubst $(_MDIR)/%,$(_MBASE)/%,$$(SRC_USERMOD_PATHFIX_LIB_C))) \ + $(eval SRC_USERMOD_PATHFIX_LIB_CXX := $$(patsubst $(_MDIR)/%,$(_MBASE)/%,$$(SRC_USERMOD_PATHFIX_LIB_CXX))) \ + $(eval SRC_USERMOD_PATHFIX_LIB_ASM := $$(patsubst $(_MDIR)/%,$(_MBASE)/%,$$(SRC_USERMOD_PATHFIX_LIB_ASM))) \ +) + +# Parent directories of each USERMOD_DIR, added to vpath below so paths like +# cexample/foo.c (produced by the patsubst above) resolve to source files. +USERMOD_DIR_PARENTS := $(sort $(foreach _MDIR, $(USERMOD_DIRS), $(patsubst %/,%,$(dir $(_MDIR))))) CFLAGS += $(CFLAGS_USERMOD) CXXFLAGS += $(CXXFLAGS_USERMOD) LDFLAGS += $(LDFLAGS_USERMOD) +LIBS += $(LIBS_USERMOD) SRC_QSTR += $(SRC_USERMOD_PATHFIX_C) $(SRC_USERMOD_PATHFIX_CXX) PY_O += $(addprefix $(BUILD)/, $(SRC_USERMOD_PATHFIX_C:.c=.o)) @@ -107,6 +131,7 @@ PY_CORE_O_BASENAME = $(addprefix py/,\ nlrxtensa.o \ nlrrv32.o \ nlrrv64.o \ + nlrloong64.o \ nlrsetjmp.o \ malloc.o \ gc.o \ diff --git a/py/runtime.c b/py/runtime.c index 51d536efd86..62a7262fee5 100644 --- a/py/runtime.c +++ b/py/runtime.c @@ -116,6 +116,9 @@ void mp_init(void) { #if MICROPY_EMIT_NATIVE MP_STATE_VM(default_emit_opt) = MP_EMIT_OPT_NONE; #endif + #if MICROPY_DEBUG_PRINTERS + MP_STATE_VM(mp_verbose_flag) = 0; + #endif #endif // init global module dict @@ -1696,8 +1699,7 @@ void mp_import_all(mp_obj_t module) { mp_obj_t dest[2]; #if MICROPY_MODULE___ALL__ - - mp_load_method_maybe(module, MP_QSTR___all__, dest); + mp_load_method_protected(module, MP_QSTR___all__, dest, false); if (dest[0] != MP_OBJ_NULL) { // When __all__ is defined, we must explicitly load all specified // symbols, possibly invoking the module __getattr__ function diff --git a/py/showbc.c b/py/showbc.c index 792fccd0133..68e2f33138c 100644 --- a/py/showbc.c +++ b/py/showbc.c @@ -99,8 +99,8 @@ void mp_bytecode_print(const mp_print_t *print, const mp_raw_code_t *rc, size_t #else qstr source_file = cm->source_file; #endif - mp_printf(print, "File %s, code block '%s' (descriptor: %p, bytecode @%p %u bytes)\n", - qstr_str(source_file), qstr_str(block_name), rc, ip_start, (unsigned)fun_data_len); + mp_printf(print, "File %q, code block '%q' (descriptor: %p, bytecode @%p %u bytes)\n", + source_file, block_name, rc, ip_start, (unsigned)fun_data_len); // raw bytecode dump size_t prelude_size = ip - ip_start + n_info + n_cell; @@ -121,7 +121,7 @@ void mp_bytecode_print(const mp_print_t *print, const mp_raw_code_t *rc, size_t #if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE qst = cm->qstr_table[qst]; #endif - mp_printf(print, " %s", qstr_str(qst)); + mp_printf(print, " %q", qst); } mp_printf(print, "\n"); @@ -189,7 +189,7 @@ const byte *mp_bytecode_print_str(const mp_print_t *print, const byte *ip_start, case MP_BC_LOAD_CONST_STRING: DECODE_QSTR; - mp_printf(print, "LOAD_CONST_STRING '%s'", qstr_str(qst)); + mp_printf(print, "LOAD_CONST_STRING '%q'", qst); break; case MP_BC_LOAD_CONST_OBJ: @@ -214,27 +214,27 @@ const byte *mp_bytecode_print_str(const mp_print_t *print, const byte *ip_start, case MP_BC_LOAD_NAME: DECODE_QSTR; - mp_printf(print, "LOAD_NAME %s", qstr_str(qst)); + mp_printf(print, "LOAD_NAME %q", qst); break; case MP_BC_LOAD_GLOBAL: DECODE_QSTR; - mp_printf(print, "LOAD_GLOBAL %s", qstr_str(qst)); + mp_printf(print, "LOAD_GLOBAL %q", qst); break; case MP_BC_LOAD_ATTR: DECODE_QSTR; - mp_printf(print, "LOAD_ATTR %s", qstr_str(qst)); + mp_printf(print, "LOAD_ATTR %q", qst); break; case MP_BC_LOAD_METHOD: DECODE_QSTR; - mp_printf(print, "LOAD_METHOD %s", qstr_str(qst)); + mp_printf(print, "LOAD_METHOD %q", qst); break; case MP_BC_LOAD_SUPER_METHOD: DECODE_QSTR; - mp_printf(print, "LOAD_SUPER_METHOD %s", qstr_str(qst)); + mp_printf(print, "LOAD_SUPER_METHOD %q", qst); break; case MP_BC_LOAD_BUILD_CLASS: @@ -257,17 +257,17 @@ const byte *mp_bytecode_print_str(const mp_print_t *print, const byte *ip_start, case MP_BC_STORE_NAME: DECODE_QSTR; - mp_printf(print, "STORE_NAME %s", qstr_str(qst)); + mp_printf(print, "STORE_NAME %q", qst); break; case MP_BC_STORE_GLOBAL: DECODE_QSTR; - mp_printf(print, "STORE_GLOBAL %s", qstr_str(qst)); + mp_printf(print, "STORE_GLOBAL %q", qst); break; case MP_BC_STORE_ATTR: DECODE_QSTR; - mp_printf(print, "STORE_ATTR %s", qstr_str(qst)); + mp_printf(print, "STORE_ATTR %q", qst); break; case MP_BC_STORE_SUBSCR: @@ -286,12 +286,12 @@ const byte *mp_bytecode_print_str(const mp_print_t *print, const byte *ip_start, case MP_BC_DELETE_NAME: DECODE_QSTR; - mp_printf(print, "DELETE_NAME %s", qstr_str(qst)); + mp_printf(print, "DELETE_NAME %q", qst); break; case MP_BC_DELETE_GLOBAL: DECODE_QSTR; - mp_printf(print, "DELETE_GLOBAL %s", qstr_str(qst)); + mp_printf(print, "DELETE_GLOBAL %q", qst); break; case MP_BC_DUP_TOP: @@ -506,12 +506,12 @@ const byte *mp_bytecode_print_str(const mp_print_t *print, const byte *ip_start, case MP_BC_IMPORT_NAME: DECODE_QSTR; - mp_printf(print, "IMPORT_NAME '%s'", qstr_str(qst)); + mp_printf(print, "IMPORT_NAME '%q'", qst); break; case MP_BC_IMPORT_FROM: DECODE_QSTR; - mp_printf(print, "IMPORT_FROM '%s'", qstr_str(qst)); + mp_printf(print, "IMPORT_FROM '%q'", qst); break; case MP_BC_IMPORT_STAR: @@ -527,10 +527,10 @@ const byte *mp_bytecode_print_str(const mp_print_t *print, const byte *ip_start, mp_printf(print, "STORE_FAST " UINT_FMT, (mp_uint_t)ip[-1] - MP_BC_STORE_FAST_MULTI); } else if (ip[-1] < MP_BC_UNARY_OP_MULTI + MP_UNARY_OP_NUM_BYTECODE) { mp_uint_t op = ip[-1] - MP_BC_UNARY_OP_MULTI; - mp_printf(print, "UNARY_OP " UINT_FMT " %s", op, qstr_str(mp_unary_op_method_name[op])); + mp_printf(print, "UNARY_OP " UINT_FMT " %q", op, (qstr)mp_unary_op_method_name[op]); } else if (ip[-1] < MP_BC_BINARY_OP_MULTI + MP_BINARY_OP_NUM_BYTECODE) { mp_uint_t op = ip[-1] - MP_BC_BINARY_OP_MULTI; - mp_printf(print, "BINARY_OP " UINT_FMT " %s", op, qstr_str(mp_binary_op_method_name[op])); + mp_printf(print, "BINARY_OP " UINT_FMT " %q", op, (qstr)mp_binary_op_method_name[op]); } else { mp_printf(print, "code %p, byte code 0x%02x not implemented\n", ip - 1, ip[-1]); assert(0); diff --git a/py/stream.c b/py/stream.c index b14a34d2658..d3b0a0d80e2 100644 --- a/py/stream.c +++ b/py/stream.c @@ -460,13 +460,13 @@ MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream___exit___obj, 4, 4, mp_stream___ex static mp_obj_t stream_seek(size_t n_args, const mp_obj_t *args) { // TODO: Could be uint64 mp_off_t offset = mp_obj_get_int(args[1]); - int whence = SEEK_SET; + int whence = MP_SEEK_SET; if (n_args == 3) { whence = mp_obj_get_int(args[2]); } // In POSIX, it's error to seek before end of stream, we enforce it here. - if (whence == SEEK_SET && offset < 0) { + if (whence == MP_SEEK_SET && offset < 0) { mp_raise_OSError(MP_EINVAL); } @@ -483,7 +483,7 @@ MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_seek_obj, 2, 3, stream_seek); static mp_obj_t stream_tell(mp_obj_t self) { mp_obj_t offset = MP_OBJ_NEW_SMALL_INT(0); - mp_obj_t whence = MP_OBJ_NEW_SMALL_INT(SEEK_CUR); + mp_obj_t whence = MP_OBJ_NEW_SMALL_INT(MP_SEEK_CUR); const mp_obj_t args[3] = {self, offset, whence}; return stream_seek(3, args); } diff --git a/py/unicode.c b/py/unicode.c index 81a37880f3c..4f6a273fe74 100644 --- a/py/unicode.c +++ b/py/unicode.c @@ -180,7 +180,7 @@ mp_uint_t unichar_xdigit_value(unichar c) { #if MICROPY_PY_BUILTINS_STR_UNICODE -bool utf8_check(const byte *p, size_t len) { +bool unicode_encoding_check(mp_encoding_t encoding, const byte *p, size_t len) { uint8_t need = 0; const byte *end = p + len; for (; p < end; p++) { @@ -193,7 +193,7 @@ bool utf8_check(const byte *p, size_t len) { return 0; } } else { - if (c >= 0xc0) { + if (encoding == MP_ENCODING_UTF8 && c >= 0xc0) { if (c >= 0xf8) { // mismatch return 0; diff --git a/py/unicode.h b/py/unicode.h index c1fb517894f..7b82c466e51 100644 --- a/py/unicode.h +++ b/py/unicode.h @@ -29,7 +29,14 @@ #include "py/mpconfig.h" #include "py/misc.h" +typedef enum { + MP_ENCODING_UTF8, + MP_ENCODING_ASCII, +} mp_encoding_t; + mp_uint_t utf8_ptr_to_index(const byte *s, const byte *ptr); -bool utf8_check(const byte *p, size_t len); + +// Returns true if bytes in buffer 'p' are valid according to encoding. +bool unicode_encoding_check(mp_encoding_t encoding, const byte *p, size_t len); #endif // MICROPY_INCLUDED_PY_UNICODE_H diff --git a/py/usermod.cmake b/py/usermod.cmake index 4a8b99ff31b..648f315b9d5 100644 --- a/py/usermod.cmake +++ b/py/usermod.cmake @@ -39,6 +39,13 @@ function(usermod_gather_sources SOURCES_VARNAME INCLUDE_DIRECTORIES_VARNAME INCL endif() endfunction() +# Extract c_module() entries from MICROPY_FROZEN_MANIFEST and merge them into +# USER_C_MODULES, so the loop below picks them up alongside USER_C_MODULES paths +# passed on the cmake command line. The port is responsible for resolving +# MICROPY_FROZEN_MANIFEST (including any board-config defaults) before +# including this file. +include(${MICROPY_DIR}/py/manifest.cmake) + # Include CMake files for user modules. if (USER_C_MODULES) foreach(USER_C_MODULE_PATH ${USER_C_MODULES}) diff --git a/pyproject.toml b/pyproject.toml index e02375fd616..6c865ea2037 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,6 +2,7 @@ [tool.ruff] target-version = "py312" line-length = 99 +builtins = [ "const", "execfile", "micropython", "ptr", "ptr8", "ptr16", "ptr32", "uint" ] # Include Python source files that don't end with .py extend-include = [ "tools/cc1" ] # Exclude third-party code from linting and formatting @@ -43,7 +44,19 @@ format.exclude = [ "tests/micropython/test_normalize_newlines.py", "tests/micropython/viper_args.py", ] -lint.extend-select = [ "C9", "PLC" ] +lint.extend-select = [ "C9", "PLC", "W292" ] +lint.ignore = [ + "E401", + "E402", + "E722", + "E731", + "E741", + "F401", + "F403", + "F405", + "PLC0206", + "PLC0415", # conditional imports are common in MicroPython +] lint.exclude = [ # Ruff finds Python SyntaxError in these files "tests/basics/string_module_tstring.py", @@ -61,6 +74,7 @@ lint.exclude = [ "tests/cmdline/repl_autoindent.py", "tests/cmdline/repl_basic.py", "tests/cmdline/repl_cont.py", + "tests/cmdline/repl_ctrl_c_interrupt_execution.py", "tests/cmdline/repl_emacs_keys.py", "tests/cmdline/repl_paste.py", "tests/cmdline/repl_words_move.py", @@ -70,18 +84,6 @@ lint.exclude = [ "tests/micropython/heapalloc_fail_tstring.py", "tests/micropython/viper_args.py", ] -lint.extend-ignore = [ - "E401", - "E402", - "E722", - "E731", - "E741", - "F401", - "F403", - "F405", - "PLC0206", - "PLC0415", # conditional imports are common in MicroPython -] # manifest.py files are evaluated with some global names pre-defined lint.per-file-ignores."**/manifest.py" = [ "F821" ] lint.per-file-ignores."ports/**/boards/**/manifest_*.py" = [ "F821" ] @@ -100,6 +102,6 @@ ignore-words-list = "ans,aranges,asend,deques,dout,emac,extint,hsi,iput,mis,noti quiet-level = 3 skip = """\ */build*,./.git,./drivers/cc3100,./lib,./ports/cc3200/FreeRTOS,./ports/cc3200/bootmgr/sl,./ports/cc3200/hal,./ports/c\ - c3200/simplelink,./ports/cc3200/telnet,./ports/esp32/managed_components,./ports/nrf/drivers/bluetooth/s1*,./ports/stm\ - 32/usbhost,./tests,ACKNOWLEDGEMENTS,\ + c3200/simplelink,./ports/cc3200/telnet,./ports/esp32/managed_components,./ports/nrf/drivers/bluetooth/s1*,./ports/pso\ + c-edge/boards/*/bsp-cfg,./ports/stm32/usbhost,./tests,ACKNOWLEDGEMENTS,\ """ diff --git a/shared/libc/string0.c b/shared/libc/string0.c index 5a9e0ff853a..614581734cb 100644 --- a/shared/libc/string0.c +++ b/shared/libc/string0.c @@ -28,6 +28,7 @@ #include // CIRCUITPY-CHANGE: additional includes +#include // for the atoi() prototype #include #include "py/mpconfig.h" @@ -54,14 +55,14 @@ void *memcpy(void *dst, const void *src, size_t n) { if (n & 2) { // copy half-word - *(uint16_t*)d = *(const uint16_t*)s; - d = (uint32_t*)((uint16_t*)d + 1); - s = (const uint32_t*)((const uint16_t*)s + 1); + *(uint16_t *)d = *(const uint16_t *)s; + d = (uint32_t *)((uint16_t *)d + 1); + s = (const uint32_t *)((const uint16_t *)s + 1); } if (n & 1) { // copy byte - *((uint8_t*)d) = *((const uint8_t*)s); + *((uint8_t *)d) = *((const uint8_t *)s); } } else #endif @@ -88,10 +89,10 @@ void *__memcpy_chk(void *dest, const void *src, size_t len, size_t slen) { } void *memmove(void *dest, const void *src, size_t n) { - if (src < dest && (uint8_t*)dest < (const uint8_t*)src + n) { + if (src < dest && (uint8_t *)dest < (const uint8_t *)src + n) { // need to copy backwards - uint8_t *d = (uint8_t*)dest + n - 1; - const uint8_t *s = (const uint8_t*)src + n - 1; + uint8_t *d = (uint8_t *)dest + n - 1; + const uint8_t *s = (const uint8_t *)src + n - 1; for (; n > 0; n--) { *d-- = *s--; } @@ -112,11 +113,11 @@ void *memset(void *s, int c, size_t n) { *s32++ = 0; } if (n & 2) { - *((uint16_t*)s32) = 0; - s32 = (uint32_t*)((uint16_t*)s32 + 1); + *((uint16_t *)s32) = 0; + s32 = (uint32_t *)((uint16_t *)s32 + 1); } if (n & 1) { - *((uint8_t*)s32) = 0; + *((uint8_t *)s32) = 0; } } else #endif @@ -138,8 +139,11 @@ int memcmp(const void *s1, const void *s2, size_t n) { while (n--) { char c1 = *s1_8++; char c2 = *s2_8++; - if (c1 < c2) return -1; - else if (c1 > c2) return 1; + if (c1 < c2) { + return -1; + } else if (c1 > c2) { + return 1; + } } return 0; } @@ -149,8 +153,9 @@ void *memchr(const void *s, int c, size_t n) { const unsigned char *p = s; do { - if (*p++ == c) - return ((void *)(p - 1)); + if (*p++ == c) { + return (void *)(p - 1); + } } while (--n != 0); } return 0; @@ -168,12 +173,19 @@ int strcmp(const char *s1, const char *s2) { while (*s1 && *s2) { char c1 = *s1++; // XXX UTF8 get char, next char char c2 = *s2++; // XXX UTF8 get char, next char - if (c1 < c2) return -1; - else if (c1 > c2) return 1; + if (c1 < c2) { + return -1; + } else if (c1 > c2) { + return 1; + } + } + if (*s2) { + return -1; + } else if (*s1) { + return 1; + } else { + return 0; } - if (*s2) return -1; - else if (*s1) return 1; - else return 0; } int strncmp(const char *s1, const char *s2, size_t n) { @@ -181,13 +193,21 @@ int strncmp(const char *s1, const char *s2, size_t n) { char c1 = *s1++; // XXX UTF8 get char, next char char c2 = *s2++; // XXX UTF8 get char, next char n--; - if (c1 < c2) return -1; - else if (c1 > c2) return 1; + if (c1 < c2) { + return -1; + } else if (c1 > c2) { + return 1; + } + } + if (n == 0) { + return 0; + } else if (*s2) { + return -1; + } else if (*s1) { + return 1; + } else { + return 0; } - if (n == 0) return 0; - else if (*s2) return -1; - else if (*s1) return 1; - else return 0; } char *strcpy(char *dest, const char *src) { @@ -202,21 +222,21 @@ char *strcpy(char *dest, const char *src) { // Public Domain implementation of strncpy from: // http://en.wikibooks.org/wiki/C_Programming/Strings#The_strncpy_function char *strncpy(char *s1, const char *s2, size_t n) { - char *dst = s1; - const char *src = s2; - /* Copy bytes, one at a time. */ - while (n > 0) { - n--; - if ((*dst++ = *src++) == '\0') { - /* If we get here, we found a null character at the end - of s2, so use memset to put null bytes at the end of - s1. */ - memset(dst, '\0', n); - break; - } - } - return s1; - } + char *dst = s1; + const char *src = s2; + /* Copy bytes, one at a time. */ + while (n > 0) { + n--; + if ((*dst++ = *src++) == '\0') { + /* If we get here, we found a null character at the end + of s2, so use memset to put null bytes at the end of + s1. */ + memset(dst, '\0', n); + break; + } + } + return s1; +} // needed because gcc optimises strcpy + strcat to this char *stpcpy(char *dest, const char *src) { @@ -241,29 +261,31 @@ char *strcat(char *dest, const char *src) { // Public Domain implementation of strchr from: // http://en.wikibooks.org/wiki/C_Programming/Strings#The_strchr_function -char *strchr(const char *s, int c) -{ +char *strchr(const char *s, int c) { /* Scan s for the character. When this loop is finished, s will either point to the end of the string or the character we were looking for. */ - while (*s != '\0' && *s != (char)c) + while (*s != '\0' && *s != (char)c) { s++; - return ((*s == c) ? (char *) s : 0); + } + return (*s == c) ? (char *)s : 0; } // Public Domain implementation of strstr from: // http://en.wikibooks.org/wiki/C_Programming/Strings#The_strstr_function -char *strstr(const char *haystack, const char *needle) -{ +char *strstr(const char *haystack, const char *needle) { size_t needlelen; /* Check for the null needle case. */ - if (*needle == '\0') - return (char *) haystack; + if (*needle == '\0') { + return (char *)haystack; + } needlelen = strlen(needle); - for (; (haystack = strchr(haystack, *needle)) != 0; haystack++) - if (strncmp(haystack, needle, needlelen) == 0) - return (char *) haystack; + for (; (haystack = strchr(haystack, *needle)) != 0; haystack++) { + if (strncmp(haystack, needle, needlelen) == 0) { + return (char *)haystack; + } + } return 0; } @@ -282,3 +304,13 @@ size_t strcspn(const char *s, const char *reject) { } return s - ss; } + +// Decimal-only, non-negative integers; no leading whitespace handling. +// Marked weak so a libc-provided atoi() takes precedence if available. +__attribute__((weak)) int atoi(const char *num) { + int value = 0; + while (*num >= '0' && *num <= '9') { + value = value * 10 + (*num++ - '0'); + } + return value; +} diff --git a/shared/memzip/lexermemzip.c b/shared/memzip/lexermemzip.c index aef64ffa0a7..a31fc7f1cdc 100644 --- a/shared/memzip/lexermemzip.c +++ b/shared/memzip/lexermemzip.c @@ -5,8 +5,7 @@ #include "py/mperrno.h" #include "memzip.h" -mp_lexer_t *mp_lexer_new_from_file(qstr filename) -{ +mp_lexer_t *mp_lexer_new_from_file(qstr filename) { void *data; size_t len; diff --git a/shared/memzip/make-memzip.py b/shared/memzip/make-memzip.py index e406c55a43c..c060745e6cf 100755 --- a/shared/memzip/make-memzip.py +++ b/shared/memzip/make-memzip.py @@ -9,9 +9,10 @@ import argparse import os +import pathlib +import shutil import subprocess import sys -import types def create_zip(zip_filename, zip_dir): @@ -26,7 +27,7 @@ def create_zip(zip_filename, zip_dir): def create_c_from_file(c_filename, zip_filename): with open(zip_filename, "rb") as zip_file: - with open(c_filename, "wb") as c_file: + with open(c_filename, "wt") as c_file: print("#include ", file=c_file) print("", file=c_file) print("const uint8_t memzip_data[] = {", file=c_file) @@ -36,10 +37,7 @@ def create_c_from_file(c_filename, zip_filename): break print(" ", end="", file=c_file) for byte in buf: - if isinstance(byte, types.StringType): - print(" 0x{:02x},".format(ord(byte)), end="", file=c_file) - else: - print(" 0x{:02x},".format(byte), end="", file=c_file) + print(" 0x{:02x},".format(byte), end="", file=c_file) print("", file=c_file) print("};", file=c_file) @@ -67,10 +65,20 @@ def main(): parser.add_argument(dest="source_dir", default="memzip_files") args = parser.parse_args(sys.argv[1:]) + output_zip = pathlib.Path(args.zip_filename) + if output_zip.suffix != ".zip": + args.zip_filename = output_zip.with_suffix(".zip") + output_c = pathlib.Path(args.c_filename) + if output_c.suffix != ".c": + args.c_filename = output_c.with_suffix(".c") + print("args.zip_filename =", args.zip_filename) print("args.c_filename =", args.c_filename) print("args.source_dir =", args.source_dir) + if not shutil.which("zip"): + raise FileNotFoundError("zip archiver not available") + create_zip(args.zip_filename, args.source_dir) create_c_from_file(args.c_filename, args.zip_filename) diff --git a/shared/memzip/memzip.c b/shared/memzip/memzip.c index 3fbea8e1e91..32fd27ffeb4 100644 --- a/shared/memzip/memzip.c +++ b/shared/memzip/memzip.c @@ -65,8 +65,7 @@ bool memzip_is_dir(const char *filename) { } -MEMZIP_RESULT memzip_locate(const char *filename, void **data, size_t *len) -{ +MEMZIP_RESULT memzip_locate(const char *filename, void **data, size_t *len) { const MEMZIP_FILE_HDR *file_hdr = memzip_find_file_header(filename); if (file_hdr == NULL) { return MZ_NO_FILE; diff --git a/shared/memzip/memzip.h b/shared/memzip/memzip.h index 667e2df7e13..fe27d9584a3 100644 --- a/shared/memzip/memzip.h +++ b/shared/memzip/memzip.h @@ -3,17 +3,17 @@ #define MEMZIP_FILE_HEADER_SIGNATURE 0x04034b50 typedef struct { - uint32_t signature; - uint16_t version; - uint16_t flags; - uint16_t compression_method; - uint16_t last_mod_time; - uint16_t last_mod_date; - uint32_t crc32; - uint32_t compressed_size; - uint32_t uncompressed_size; - uint16_t filename_len; - uint16_t extra_len; + uint32_t signature; + uint16_t version; + uint16_t flags; + uint16_t compression_method; + uint16_t last_mod_time; + uint16_t last_mod_date; + uint32_t crc32; + uint32_t compressed_size; + uint32_t uncompressed_size; + uint16_t filename_len; + uint16_t extra_len; /* char filename[filename_len] */ /* uint8_t extra[extra_len] */ @@ -23,22 +23,22 @@ typedef struct #define MEMZIP_CENTRAL_DIRECTORY_SIGNATURE 0x02014b50 typedef struct { - uint32_t signature; - uint16_t version_made_by; - uint16_t version_read_with; - uint16_t flags; - uint16_t compression_method; - uint16_t last_mod_time; - uint16_t last_mod_date; - uint32_t crc32; - uint32_t compressed_size; - uint32_t uncompressed_size; - uint16_t filename_len; - uint16_t extra_len; - uint16_t disk_num; - uint16_t internal_file_attributes; - uint32_t external_file_attributes; - uint32_t file_header_offset; + uint32_t signature; + uint16_t version_made_by; + uint16_t version_read_with; + uint16_t flags; + uint16_t compression_method; + uint16_t last_mod_time; + uint16_t last_mod_date; + uint32_t crc32; + uint32_t compressed_size; + uint32_t uncompressed_size; + uint16_t filename_len; + uint16_t extra_len; + uint16_t disk_num; + uint16_t internal_file_attributes; + uint32_t external_file_attributes; + uint32_t file_header_offset; /* char filename[filename_len] */ /* uint8_t extra[extra_len] */ @@ -48,14 +48,14 @@ typedef struct #define MEMZIP_END_OF_CENTRAL_DIRECTORY_SIGNATURE 0x06054b50 typedef struct { - uint32_t signature; - uint16_t disk_num; - uint16_t central_directory_disk; - uint16_t num_central_directories_this_disk; - uint16_t total_central_directories; - uint32_t central_directory_size; - uint32_t central_directory_offset; - uint16_t comment_len; + uint32_t signature; + uint16_t disk_num; + uint16_t central_directory_disk; + uint16_t num_central_directories_this_disk; + uint16_t total_central_directories; + uint32_t central_directory_size; + uint32_t central_directory_offset; + uint16_t comment_len; /* char comment[comment_len] */ @@ -71,10 +71,10 @@ typedef enum { } MEMZIP_RESULT; typedef struct { - uint32_t file_size; - uint16_t last_mod_date; - uint16_t last_mod_time; - uint8_t is_dir; + uint32_t file_size; + uint16_t last_mod_date; + uint16_t last_mod_time; + uint8_t is_dir; } MEMZIP_FILE_INFO; diff --git a/shared/netutils/dhcpserver.c b/shared/netutils/dhcpserver.c index 6d9cdb97d1f..038b794e864 100644 --- a/shared/netutils/dhcpserver.c +++ b/shared/netutils/dhcpserver.c @@ -69,11 +69,9 @@ #define PORT_DHCP_SERVER (67) #define PORT_DHCP_CLIENT (68) -#define DEFAULT_DNS MAKE_IP4(192, 168, 4, 1) #define DEFAULT_LEASE_TIME_S (24 * 60 * 60) // in seconds #define MAC_LEN (6) -#define MAKE_IP4(a, b, c, d) ((a) << 24 | (b) << 16 | (c) << 8 | (d)) typedef struct { uint8_t op; // message opcode @@ -287,8 +285,10 @@ static void dhcp_server_process(void *arg, struct udp_pcb *upcb, struct pbuf *p, opt_write_n(&opt, DHCP_OPT_SERVER_ID, 4, &ip_2_ip4(&d->ip)->addr); opt_write_n(&opt, DHCP_OPT_SUBNET_MASK, 4, &ip_2_ip4(&d->nm)->addr); - opt_write_n(&opt, DHCP_OPT_ROUTER, 4, &ip_2_ip4(&d->ip)->addr); // aka gateway; can have multiple addresses - opt_write_u32(&opt, DHCP_OPT_DNS, DEFAULT_DNS); // can have multiple addresses + if (d->send_router) { + opt_write_n(&opt, DHCP_OPT_ROUTER, 4, &ip_2_ip4(&d->ip)->addr); // aka gateway; can have multiple addresses + } + opt_write_n(&opt, DHCP_OPT_DNS, 4, &ip_2_ip4(&d->ip)->addr); opt_write_u32(&opt, DHCP_OPT_IP_LEASE_TIME, DEFAULT_LEASE_TIME_S); *opt++ = DHCP_OPT_END; struct netif *netif = ip_current_input_netif(); @@ -302,6 +302,7 @@ void dhcp_server_init(dhcp_server_t *d, ip_addr_t *ip, ip_addr_t *nm) { ip_addr_copy(d->ip, *ip); ip_addr_copy(d->nm, *nm); memset(d->lease, 0, sizeof(d->lease)); + d->send_router = true; if (dhcp_socket_new_dgram(&d->udp, d, dhcp_server_process) != 0) { return; } diff --git a/shared/netutils/dhcpserver.h b/shared/netutils/dhcpserver.h index 2349d2ea427..24224d6aadd 100644 --- a/shared/netutils/dhcpserver.h +++ b/shared/netutils/dhcpserver.h @@ -41,6 +41,7 @@ typedef struct _dhcp_server_t { ip_addr_t nm; dhcp_server_lease_t lease[DHCPS_MAX_IP]; struct udp_pcb *udp; + bool send_router; // advertise server IP as default gateway } dhcp_server_t; void dhcp_server_init(dhcp_server_t *d, ip_addr_t *ip, ip_addr_t *nm); diff --git a/shared/runtime/gchelper.h b/shared/runtime/gchelper.h index 1e85e06f46e..9617bc7f9fe 100644 --- a/shared/runtime/gchelper.h +++ b/shared/runtime/gchelper.h @@ -43,6 +43,10 @@ typedef uintptr_t gc_helper_regs_t[10]; typedef uintptr_t gc_helper_regs_t[11]; // x19-x29 #elif defined(__riscv) && (__riscv_xlen <= 64) typedef uintptr_t gc_helper_regs_t[12]; // S0-S11 +#elif defined(__loongarch__) && defined(__loongarch64) +typedef uintptr_t gc_helper_regs_t[10]; // S0-S9 +#elif defined(__powerpc__) && defined(__powerpc64__) +typedef uintptr_t gc_helper_regs_t[18]; // r14-r31 #endif #endif diff --git a/shared/runtime/gchelper_generic.c b/shared/runtime/gchelper_generic.c index 230a2444005..dcca800d512 100644 --- a/shared/runtime/gchelper_generic.c +++ b/shared/runtime/gchelper_generic.c @@ -190,6 +190,73 @@ static void gc_helper_get_regs(gc_helper_regs_t arr) { arr[11] = s11; } +#elif defined(__loongarch__) && defined(__loongarch64) + +// Fallback implementation for LOONG64, prefer gchelper_loong64.s. +static void gc_helper_get_regs(gc_helper_regs_t arr) { + register uintptr_t s0 asm ("r23"); + register uintptr_t s1 asm ("r24"); + register uintptr_t s2 asm ("r25"); + register uintptr_t s3 asm ("r26"); + register uintptr_t s4 asm ("r27"); + register uintptr_t s5 asm ("r28"); + register uintptr_t s6 asm ("r29"); + register uintptr_t s7 asm ("r30"); + register uintptr_t s8 asm ("r31"); + register uintptr_t s9 asm ("r22"); + arr[0] = s0; + arr[1] = s1; + arr[2] = s2; + arr[3] = s3; + arr[4] = s4; + arr[5] = s5; + arr[6] = s6; + arr[7] = s7; + arr[8] = s8; + arr[9] = s9; +} + +#elif defined(__powerpc__) && defined(__powerpc64__) + +static void gc_helper_get_regs(gc_helper_regs_t arr) { + register uintptr_t r14 __asm("r14"); + register uintptr_t r15 __asm("r15"); + register uintptr_t r16 __asm("r16"); + register uintptr_t r17 __asm("r17"); + register uintptr_t r18 __asm("r18"); + register uintptr_t r19 __asm("r19"); + register uintptr_t r20 __asm("r20"); + register uintptr_t r21 __asm("r21"); + register uintptr_t r22 __asm("r22"); + register uintptr_t r23 __asm("r23"); + register uintptr_t r24 __asm("r24"); + register uintptr_t r25 __asm("r25"); + register uintptr_t r26 __asm("r26"); + register uintptr_t r27 __asm("r27"); + register uintptr_t r28 __asm("r28"); + register uintptr_t r29 __asm("r29"); + register uintptr_t r30 __asm("r30"); + register uintptr_t r31 __asm("r31"); + arr[0] = r14; + arr[1] = r15; + arr[2] = r16; + arr[3] = r17; + arr[4] = r18; + arr[5] = r19; + arr[6] = r20; + arr[7] = r21; + arr[8] = r22; + arr[9] = r23; + arr[10] = r24; + arr[11] = r25; + arr[12] = r26; + arr[13] = r27; + arr[14] = r28; + arr[15] = r29; + arr[16] = r30; + arr[17] = r31; +} + #else #error "Architecture not supported for gc_helper_get_regs. Set MICROPY_GCREGS_SETJMP to use the fallback implementation." diff --git a/shared/runtime/gchelper_loong64.s b/shared/runtime/gchelper_loong64.s new file mode 100644 index 00000000000..5180e1e24df --- /dev/null +++ b/shared/runtime/gchelper_loong64.s @@ -0,0 +1,50 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2026 Alessandro Gatti + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + + .global gc_helper_get_regs_and_sp + .type gc_helper_get_regs_and_sp, @function + +gc_helper_get_regs_and_sp: + + /* Store registers into the given array. */ + + st.d $r23, $r4, 0 /* Save S0. */ + st.d $r24, $r4, 8 /* Save S1. */ + st.d $r25, $r4, 16 /* Save S2. */ + st.d $r26, $r4, 24 /* Save S3. */ + st.d $r27, $r4, 32 /* Save S4. */ + st.d $r28, $r4, 40 /* Save S5. */ + st.d $r29, $r4, 48 /* Save S6. */ + st.d $r30, $r4, 56 /* Save S7. */ + st.d $r31, $r4, 64 /* Save S8. */ + st.d $r22, $r4, 72 /* Save S9. */ + + /* Return the stack pointer. */ + + add.d $r4, $r0, $r3 + jirl $r0, $r1, 0 + + .size gc_helper_get_regs_and_sp, .-gc_helper_get_regs_and_sp diff --git a/shared/runtime/pyexec.c b/shared/runtime/pyexec.c index 2b57830abfc..0c10ebf0e94 100644 --- a/shared/runtime/pyexec.c +++ b/shared/runtime/pyexec.c @@ -133,7 +133,7 @@ static int parse_compile_execute(const void *source, mp_parse_input_kind_t input mp_parse_tree_t parse_tree = mp_parse(lex, input_kind); #if defined(MICROPY_UNIX_COVERAGE) // allow to print the parse tree in the coverage build - if (mp_verbose_flag >= 3) { + if (MP_STATE_VM(mp_verbose_flag) >= 3) { printf("----------------\n"); mp_parse_node_print(&mp_plat_print, parse_tree.root, 0); printf("----------------\n"); @@ -545,6 +545,8 @@ static int pyexec_friendly_repl_process_char(int c) { vstr_reset(MP_STATE_VM(repl_line)); repl.paste_mode = true; return 0; + } else if (vstr_len(MP_STATE_VM(repl_line)) == 0) { + goto input_restart; } if (ret < 0) { diff --git a/tests/README.md b/tests/README.md index 534e7e0a059..4edd3e1088e 100644 --- a/tests/README.md +++ b/tests/README.md @@ -3,6 +3,17 @@ This directory contains tests for most parts of MicroPython. To run it you will need CPython 3.8.2 or newer, which is used to validate MicroPython's behaviour. +The tests are organized into several categories: +- Unit and regression tests: tests for MicroPython's core functionality, including the + compiler, runtime, and built-in modules +- perf_bench: performance benchmarks +- internal_bench: internal performance benchmarks +- Serial reliability and performance test +- Test key/certificates +- CPython vs MicroPython Differences + +## Unit and regression tests + To run all stable tests, run the "run-tests.py" script in this directory. By default that will run the test suite against the unix port of MicroPython. @@ -63,7 +74,7 @@ module, should go in the import/ subdirectory. The `perf_bench` directory contains some performance benchmarks that can be used to benchmark different MicroPython firmwares or host ports. -The runner utility is `run-perfbench,py`. Execute `./run-perfbench.py --help` +The runner utility is `run-perfbench.py`. Execute `./run-perfbench.py --help` for a full list of command line options. ### Benchmarking a target @@ -249,3 +260,87 @@ $ openssl ecparam -name prime256v1 -genkey -noout -out ec_key.pem $ openssl pkey -in ec_key.pem -out ec_key.der -outform DER $ openssl req -new -x509 -key ec_key.pem -out ec_cert.der -outform DER -days 3650 -nodes -subj '/CN=micropython.local/O=MicroPython/C=AU' ``` + +## CPython vs MicroPython Differences + +The `tests/cpydiff` folder contains test files that document and verify the differences +between the CPython and MicroPython implementations. + +These tests are designed to: +- Execute the same code on both CPython and MicroPython +- Document behavioral differences between the two implementations +- Generate documentation pages that help users understand these differences + +### How It Works + +1. Each test file contains Python code that demonstrates a specific difference. +2. The tests are executed on both CPython and MicroPython. +3. The output from both implementations is captured and compared. If the outputs are the + same, the generation will fail (because the outputs should be different). +4. The results, along with metadata from the file docstrings, are used to generate + documentation. + +### Documentation Generation + +The documentation is automatically generated using: +``` +tools/gen-cpydiff.py +``` + +This script: +- Parses the docstring metadata from each test file +- Runs the tests on both implementations +- Combines the results to create comprehensive documentation pages +- Outputs formatted reStructuredText documentation showing the differences + +**Note:** This script is automatically executed as part of the documentation publishing +process when building the docs. + +### Test File Format + +The test filename should match the categories of the test. For example, a test with +categories `Syntax,Operators` should be named `syntax_operators_*.py`. + +Each test file should include a docstring with the following format, and include a +minimal code snippet that reproduces the difference when run on both CPython and the +unix port of MicroPython: + +```python +""" +categories: Category,Subcategory +description: Brief description of the difference being tested +cause: Explanation of why this difference exists +workaround: How to work around this difference (or "Unknown" if none) +""" +# Minimal Python code reproducing the difference +import sys +print(sys.implementation.name) +``` + +The categories and subcategories are used to organize the documentation into sections. +Files with the same category and/or subcategory will be placed in the same section. + +Common categories include: +- Syntax +- Core (Core language) +- Types (Builtin types) +- Modules + +### Building the Documentation + +The documentation is automatically regenerated during the documentation build process. +To manually regenerate after adding or modifying tests: + +1. Set environment variables if needed: + - `MICROPY_MICROPYTHON`: Path to MicroPython executable + - `MICROPY_CPYTHON3`: Path to CPython 3.x executable (default: "python3") + +2. Run the generation script from the project root: + ``` + python tools/gen-cpydiff.py + ``` + +The generated documentation will be placed in `docs/genrst/` as reStructuredText files. + +Also see `docs/README.md` for more information on building the documentation locally +to validate the rendering of the resulting documentation page(s). diff --git a/tests/assets/Makefile b/tests/assets/Makefile new file mode 100644 index 00000000000..6048ee2a405 --- /dev/null +++ b/tests/assets/Makefile @@ -0,0 +1,29 @@ +MPY_DIR = ../.. + +PYTHON := $(command -v python3 2> /dev/null) +ifndef PYTHON +PYTHON = python +endif + +# Files created with dd if=/dev/urandom ... - the file name is the CRC32 +# of the data contained in the file itself. + +ROMFS_FILES = romfs_source/0x30d83fe5.bin \ + romfs_source/0x37bef0eb.bin \ + romfs_source/0x442f3b5f.bin \ + romfs_source/0x648793fb.bin \ + romfs_source/0x913837b6.bin \ + romfs_source/0xdb14aac7.bin \ + romfs_source/romfs_sentinel.txt + +.PHONY: romfs +romfs: all + +.PHONY: clean +clean: + rm -f random_romfs.bin + +all: random_romfs.bin + +random_romfs.bin: $(ROMFS_FILES) + $(PYTHON) $(MPY_DIR)/tools/mpremote/mpremote.py romfs --partition 0 --no-mpy --output $@ build romfs_source diff --git a/tests/assets/README.md b/tests/assets/README.md new file mode 100644 index 00000000000..a5f935ee1b9 --- /dev/null +++ b/tests/assets/README.md @@ -0,0 +1,8 @@ +This directory contains assets for other tests to use: + +- A .mpy built against the current .mpy version that can be used to test + freezing without a dependency on mpy-cross (`frozentest.py` and + `frozentest.mpy`) +- A ROMFS image containing random binary files for testing mounting and reading + from the partition without having to generate a custom image + (`random_romfs.bin`, the `romfs_source` directory, and `Makefile`) diff --git a/tests/frozen/frozentest.mpy b/tests/assets/frozentest.mpy similarity index 100% rename from tests/frozen/frozentest.mpy rename to tests/assets/frozentest.mpy diff --git a/tests/frozen/frozentest.py b/tests/assets/frozentest.py similarity index 100% rename from tests/frozen/frozentest.py rename to tests/assets/frozentest.py diff --git a/tests/assets/random_romfs.bin b/tests/assets/random_romfs.bin new file mode 100644 index 00000000000..387bd7ec476 Binary files /dev/null and b/tests/assets/random_romfs.bin differ diff --git a/tests/assets/romfs_source/0x30d83fe5.bin b/tests/assets/romfs_source/0x30d83fe5.bin new file mode 100644 index 00000000000..625e51fb8ae Binary files /dev/null and b/tests/assets/romfs_source/0x30d83fe5.bin differ diff --git a/tests/assets/romfs_source/0x37bef0eb.bin b/tests/assets/romfs_source/0x37bef0eb.bin new file mode 100644 index 00000000000..11e1c67871c Binary files /dev/null and b/tests/assets/romfs_source/0x37bef0eb.bin differ diff --git a/tests/assets/romfs_source/0x442f3b5f.bin b/tests/assets/romfs_source/0x442f3b5f.bin new file mode 100644 index 00000000000..ffe9f8f0e2c Binary files /dev/null and b/tests/assets/romfs_source/0x442f3b5f.bin differ diff --git a/tests/assets/romfs_source/0x648793fb.bin b/tests/assets/romfs_source/0x648793fb.bin new file mode 100644 index 00000000000..f99b58c610a Binary files /dev/null and b/tests/assets/romfs_source/0x648793fb.bin differ diff --git a/tests/assets/romfs_source/0x913837b6.bin b/tests/assets/romfs_source/0x913837b6.bin new file mode 100644 index 00000000000..6511184346a Binary files /dev/null and b/tests/assets/romfs_source/0x913837b6.bin differ diff --git a/tests/assets/romfs_source/0xdb14aac7.bin b/tests/assets/romfs_source/0xdb14aac7.bin new file mode 100644 index 00000000000..51754d4ecd8 Binary files /dev/null and b/tests/assets/romfs_source/0xdb14aac7.bin differ diff --git a/tests/assets/romfs_source/romfs_sentinel.txt b/tests/assets/romfs_source/romfs_sentinel.txt new file mode 100644 index 00000000000..6ff9311d70e --- /dev/null +++ b/tests/assets/romfs_source/romfs_sentinel.txt @@ -0,0 +1,3 @@ +*MPY-ROMFS-TEST-PARTITION* + +This is a ROMFS partition to be used in MicroPython tests, usually for CI. diff --git a/tests/basics/array_int_repr.py b/tests/basics/array_int_repr.py new file mode 100644 index 00000000000..55a2d5998eb --- /dev/null +++ b/tests/basics/array_int_repr.py @@ -0,0 +1,85 @@ +# Test array integer representations in memory +# +# This has to be a unit test because correct internal representation depends on +# native endianness +# +# These test cases should pass on both CPython and MicroPython. + +try: + from array import array + from sys import byteorder +except ImportError: + print("SKIP") + raise SystemExit + +try: + import unittest +except MemoryError: + print("SKIP-TOO-LARGE") # some small boards can't fit unittest in RAM + raise SystemExit + +# Ports without bigint support don't support typecode 'q' +try: + array("q", []) + array_has_typecode_q = True +except: + array_has_typecode_q = False + + +class TestIntReprs(unittest.TestCase): + def _test_repr(self, typecode, values): + # create an array with the specified typecode and list of values + a = array(typecode, values) + a_hex = memoryview(a).hex() + print(a, a_hex) + + self.assertEqual(len(a_hex) % len(values), 0) + # no array.itemsize in MicroPython, so calculate item size + sz = len(a_hex) // 2 // len(values) + if hasattr(a, "itemsize"): + self.assertEqual(a.itemsize, sz) + + # build alternative hex representation of the array using int.to_bytes() + # on each value + values_hex = "" + for v in values: + v_bytes = v.to_bytes(sz, byteorder=byteorder, signed=typecode.islower()) + values_hex += v_bytes.hex() + + # compare with the raw array contents + self.assertEqual(a_hex, values_hex) + + def test_smaller_ints(self): + for typecode, initialiser in ( + ("b", [1, -1, 120, -120]), + ("B", [1, 5, 220]), + ("h", [5, -1, 32_000, -32_000]), + ("H", [5, 1, 32_000, 65_535]), + ("i", [5, -1, 32_000, -32_000]), # CPython only guarantees min 2 bytes, C style! + ("I", [5, 1, 32_000, 65_535]), + ("l", [5, -1, 2_000_000, -2_000_000, 0x7FFF_FFFF]), + ("L", [5, 1, 65_536, 2_000_000, 0x7FFF_FFFF, 0xFFFF_FFFF]), + ): + self._test_repr(typecode, initialiser) + + @unittest.skipIf(not array_has_typecode_q, "port has no bigint support") + def test_bigints(self): + # Note: need to be careful not to write any literal expressions that can't be compiled on non-bigint MP + a = 0x1FFF_FFF + b = 62 + + try: + # this calculation will trigger OverflowError if bigint is set to long long + max_uint64 = [2 ** (b + 1)] + except OverflowError: + max_uint64 = [] + + for typecode, initialiser in ( + ("q", [a * 5, -a * 10, 2**b, (2**b) * -1]), + ("Q", [a * 5, a * 10, 2**b, (2**b) - 1, (2**b) + 1] + max_uint64), + ): + self._test_repr(typecode, initialiser) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/basics/array_limits_intbig.py b/tests/basics/array_limits_intbig.py new file mode 100644 index 00000000000..ec740a87bed --- /dev/null +++ b/tests/basics/array_limits_intbig.py @@ -0,0 +1,124 @@ +# Test behaviour when array module is provided out of bounds values +# +# This test is intended to also pass on CPython. + +try: + from array import array +except ImportError: + print("SKIP") + raise SystemExit + +try: + import unittest +except MemoryError: + print("SKIP-TOO-LARGE") # some small boards can't fit unittest in RAM + raise SystemExit + + +# MicroPython V2.0 will enforce bounds on items (same as CPython), V1.x truncates +# +# Note: once `_. +# and tests/cpydiff/core_function_star.py + +try: + exec("f(y=1, *(3,))") +except SyntaxError as e: + print("SyntaxError") diff --git a/tests/basics/fun_callstar_kwarg.py.exp b/tests/basics/fun_callstar_kwarg.py.exp new file mode 100644 index 00000000000..8729fc43437 --- /dev/null +++ b/tests/basics/fun_callstar_kwarg.py.exp @@ -0,0 +1 @@ +SyntaxError diff --git a/tests/basics/gen_yield_from_close.py b/tests/basics/gen_yield_from_close.py index a493a410353..981c810564b 100644 --- a/tests/basics/gen_yield_from_close.py +++ b/tests/basics/gen_yield_from_close.py @@ -75,9 +75,9 @@ def gen6(): yield -1 try: print((yield from gen5())) - except GeneratorExit: + except GeneratorExit as e: print("delegating caught GeneratorExit") - raise + raise e yield 10 yield 11 diff --git a/tests/basics/generator_pend_throw.py b/tests/basics/generator_pend_throw.py index ae8c21189e9..6cecd418dd0 100644 --- a/tests/basics/generator_pend_throw.py +++ b/tests/basics/generator_pend_throw.py @@ -46,7 +46,7 @@ def gen_next(): try: next(g) except Exception as e: - print("raised", repr(e)) + print("raised {} {}".format(type(e), str(e) or "generator already executing")) # Verify that you can't pend_throw from within the running coroutine. @@ -59,7 +59,7 @@ def gen_pend_throw(): try: next(g) except Exception as e: - print("raised", repr(e)) + print("raised {} {}".format(type(e), str(e) or "generator already executing")) # Verify that the pend_throw exception can be ignored. diff --git a/tests/basics/generator_pend_throw.py.exp b/tests/basics/generator_pend_throw.py.exp index 8a3dadfec7a..56fa68be252 100644 --- a/tests/basics/generator_pend_throw.py.exp +++ b/tests/basics/generator_pend_throw.py.exp @@ -3,8 +3,8 @@ raised ValueError() ret was: None raised OSError() -raised ValueError('generator already executing',) -raised ValueError('generator already executing',) +raised generator already executing +raised generator already executing 0 ignore CancelledError 1 diff --git a/tests/basics/int_64_basics.py b/tests/basics/int_64_basics.py index ef76793317e..b9d46c7d701 100644 --- a/tests/basics/int_64_basics.py +++ b/tests/basics/int_64_basics.py @@ -145,12 +145,12 @@ try: print((1 << 48) >> -4) except ValueError as e: - print(e) + print(str(e) or "negative shift count") try: print((1 << 48) << -6) except ValueError as e: - print(e) + print(str(e) or "negative shift count") # Test that the most extreme 64 bit integer values all parse with int() print(int("-9223372036854775807")) diff --git a/tests/basics/int_bytes.py b/tests/basics/int_bytes.py index d1999bebb0e..7a1edc2ccec 100644 --- a/tests/basics/int_bytes.py +++ b/tests/basics/int_bytes.py @@ -1,11 +1,15 @@ -# CIRCUITPY-CHANGE: signed support +import sys + print((10).to_bytes(1, "little")) print((-10).to_bytes(1, "little", signed=True)) # Test fitting in length that's not a power of two. -print((0x10000).to_bytes(3, 'little')) +print((0x10000).to_bytes(3, "little")) print((111111).to_bytes(4, "little")) print((-111111).to_bytes(4, "little", signed=True)) print((100).to_bytes(10, "little")) +print(int.from_bytes(b"\x00\x01\0\0\0\0\0\0", "little")) +print(int.from_bytes(b"\x01\0\0\0\0\0\0\0", "little")) +print(int.from_bytes(b"\x00\x01\0\0\0\0\0\0", "little")) print((-100).to_bytes(10, "little", signed=True)) # check that extra zero bytes don't change the internal int value @@ -32,19 +36,6 @@ except OverflowError: print("OverflowError") -# CIRCUITPY-CHANGE: more tests -# too small buffer should raise an error -try: - (256).to_bytes(1, "little") -except OverflowError: - print("OverflowError") - -# negative numbers should raise an error if signed=False -try: - (-256).to_bytes(2, "little") -except OverflowError: - print("OverflowError") - # except for converting 0 to a zero-length byte array print((0).to_bytes(0, "big")) @@ -58,11 +49,54 @@ # OverFlowError if not big enough +try: + (0x123).to_bytes(1, "big") +except OverflowError: + print("OverflowError") + +try: + (0x12345).to_bytes(2, "big") +except OverflowError: + print("OverflowError") + +try: + (0x1234567).to_bytes(3, "big") +except OverflowError: + print("OverflowError") + + +# negative representations + +print((-1).to_bytes(1, "little", signed=True)) +print((-1).to_bytes(3, "little", signed=True)) +print((-1).to_bytes(1, "big", signed=True)) +print((-1).to_bytes(3, "big", signed=True)) +print((-128).to_bytes(1, "big", signed=True)) +print((-32768).to_bytes(2, "big", signed=True)) +print((-(1 << 23)).to_bytes(3, "big", signed=True)) + +# negative numbers should raise an error if signed=False, regardless of fitting or not try: (-256).to_bytes(2, "little", signed=False) except OverflowError: print("OverflowError") -# byteorder arg can be omitted; default is "big" -print(int.from_bytes(b"\x01\0")) -print((100).to_bytes(10)) +try: + (-1).to_bytes(1, "little") +except OverflowError: + print("OverflowError") + +try: + print((-129).to_bytes(1, "big")) +except OverflowError: + print("OverflowError") + +try: + print((-32769).to_bytes(2, "big")) +except OverflowError: + print("OverflowError") + +try: + print((-(1 << 23) - 1).to_bytes(2, "big")) +except OverflowError: + print("OverflowError") diff --git a/tests/basics/int_bytes_int64.py b/tests/basics/int_bytes_int64.py index 032dbccc5b1..2ca87f20dee 100644 --- a/tests/basics/int_bytes_int64.py +++ b/tests/basics/int_bytes_int64.py @@ -5,7 +5,7 @@ # long longs. try: - x = int.from_bytes(b"\x6F\xAB\xCD\x12\x34\x56\x78\xFB", "big") + x = int.from_bytes(b"\x6f\xab\xcd\x12\x34\x56\x78\xfb", "big") except OverflowError: print("SKIP") # Port can't represent this size of integer at all raise SystemExit @@ -36,17 +36,8 @@ # negative representations -# MicroPython int.to_bytes() behaves as if signed=True for negative numbers -if "micropython" in repr(sys.implementation): +x = -x - def to_bytes_compat(i, l, e): - return i.to_bytes(l, e) -else: - # Implement MicroPython compatible behaviour for CPython - def to_bytes_compat(i, l, e): - return i.to_bytes(l, e, signed=i < 0) - - -print(to_bytes_compat(-x, 8, "little")) -print(to_bytes_compat(-x, 20, "big")) -print(to_bytes_compat(-x, 20, "little")) +print(x.to_bytes(8, "little", signed=True)) +print(x.to_bytes(20, "big", signed=True)) +print(x.to_bytes(20, "little", signed=True)) diff --git a/tests/basics/int_bytes_intbig.py b/tests/basics/int_bytes_intbig.py index 073c8bf7892..6d3265ddb1d 100644 --- a/tests/basics/int_bytes_intbig.py +++ b/tests/basics/int_bytes_intbig.py @@ -4,11 +4,10 @@ import sys -# CIRCUITPY-CHANGE: signed support print((2**64).to_bytes(9, "little")) -print((-2**64).to_bytes(9, "little", signed=True)) +print((-(2**64)).to_bytes(9, "little", signed=True)) print((2**64).to_bytes(9, "big")) -print((-2**64).to_bytes(9, "big", signed=True)) +print((-(2**64)).to_bytes(9, "big", signed=True)) b = bytes(range(20)) @@ -26,50 +25,69 @@ # check that extra zero bytes don't change the internal int value print(int.from_bytes(b + bytes(10), "little") == int.from_bytes(b, "little")) -# CIRCUITPY-CHANGE: more tests -# too small buffer should raise an error +# can't write to a zero-length bytes object try: - (2**64).to_bytes(8, "little") + ib.to_bytes(0, "little") except OverflowError: print("OverflowError") -# negative numbers should raise an error if signed=False +# or one that is too short try: - (-2**64).to_bytes(9, "little") + ib.to_bytes(18, "big") except OverflowError: print("OverflowError") -# negative representations - -# MicroPython int.to_bytes() behaves as if signed=True for negative numbers -if "micropython" in repr(sys.implementation): - - def to_bytes_compat(i, l, e): - return i.to_bytes(l, e) -else: - # Implement MicroPython compatible behaviour for CPython - def to_bytes_compat(i, l, e): - return i.to_bytes(l, e, signed=i < 0) - - -print(to_bytes_compat(-ib, 20, "big")) -print(to_bytes_compat(ib * -ib, 40, "big")) - -# case where an additional byte is needed for sign bit -ib = (2**64) - 1 -print(ib.to_bytes(8, "little")) - -ib *= -1 - +# including when signed try: - (-2**64).to_bytes(9, "little", signed=False) + ib.to_bytes(18, "big", signed=True) except OverflowError: print("OverflowError") + +# negative representations + +# negative numbers should raise an error if signed=False try: - print(to_bytes_compat(ib, 8, "little")) + (-(2**64)).to_bytes(9, "little", signed=False) except OverflowError: print("OverflowError") -print(to_bytes_compat(ib, 9, "little")) -print(to_bytes_compat(ib, 9, "big")) +print((-ib).to_bytes(20, "big", signed=True)) +print((ib * -ib).to_bytes(40, "big", signed=True)) + +# cases where an additional byte is needed for sign bit + +MAX_U24 = (2**24) - 1 +MAX_U32 = (2**32) - 1 +MAX_U64 = (2**64) - 1 + +for ib, nbytes in ( + (-127, 1), + (255, 1), + (-255, 1), + (65535, 2), + (-65535, 2), + (-65534, 2), + (MAX_U24, 3), + (-MAX_U24, 3), + (1 - MAX_U24, 3), + (MAX_U32, 4), + (-MAX_U32, 4), + (1 - MAX_U32, 4), + (2 - MAX_U32, 4), + (MAX_U64, 8), + (-MAX_U64, 8), + (1 - MAX_U64, 8), + (2 - MAX_U64, 8), + (2 * MAX_U64, 8), + (-2 * MAX_U64, 8), +): + print("ib", hex(ib), "nbytes", nbytes, ":") + for signed in False, True: + for endian in "little", "big": + for nbytes_offs in (-1, 0, 1): + try: + as_bytes = ib.to_bytes(nbytes + nbytes_offs, endian, signed=signed) + except OverflowError: + as_bytes = "OverflowError" + print(as_bytes, "signed", signed, "endian", endian, "nbytes_offs", nbytes_offs) diff --git a/tests/basics/string_find.py b/tests/basics/string_find.py index f9fcad3e579..f37064ad291 100644 --- a/tests/basics/string_find.py +++ b/tests/basics/string_find.py @@ -24,6 +24,7 @@ print("aaaaaaaaaaa".find("bbb", 9, 2)) try: + # Only works on bytes/bytearray. 'abc'.find(1) except TypeError: print('TypeError') diff --git a/tests/basics/string_fstring.py b/tests/basics/string_fstring.py index daa687dbddb..03e2d90a8f7 100644 --- a/tests/basics/string_fstring.py +++ b/tests/basics/string_fstring.py @@ -83,12 +83,5 @@ def foo(a, b): print(fr"\r{x}") # Format specifiers with nested replacement fields -space = 5 -prec = 2 -print(f"{3.14:{space}.{prec}}") - -space_prec = "5.2" -print(f"{3.14:{space_prec}}") - radix = "x" print(f"{314:{radix}}") diff --git a/tests/basics/string_index.py b/tests/basics/string_index.py index 31f6900e6c1..328f2dff54c 100644 --- a/tests/basics/string_index.py +++ b/tests/basics/string_index.py @@ -76,3 +76,9 @@ print("Raised ValueError") else: print("Did not raise ValueError") + +try: + # Only works on bytes/bytearray. + 'abc'.index(1) +except TypeError: + print('TypeError') diff --git a/tests/basics/string_tstring_basic.py b/tests/basics/string_tstring_basic.py index e23a3f06595..aa9de327b59 100644 --- a/tests/basics/string_tstring_basic.py +++ b/tests/basics/string_tstring_basic.py @@ -172,7 +172,8 @@ print("\n=== Interpolation attribute tests ===") i_basic = Interpolation(42, "x") print(f"Basic conversion: {i_basic.conversion}") -print(f"Basic format_spec: {i_basic.format_spec}") +# Put in quotes to make empty string visible. +print(f"Basic format_spec: '{i_basic.format_spec}'") i_with_conv = Interpolation(42, "x", "s") print(f"With conversion: {i_with_conv.conversion}") diff --git a/tests/basics/string_tstring_basic.py.exp b/tests/basics/string_tstring_basic.py.exp index 4fcb3a75834..60affaad499 100644 --- a/tests/basics/string_tstring_basic.py.exp +++ b/tests/basics/string_tstring_basic.py.exp @@ -71,7 +71,7 @@ Nested expr: Template(strings=('', ''), interpolations=(Interpolation('{}', 'inn === Interpolation attribute tests === Basic conversion: None -Basic format_spec: +Basic format_spec: '' With conversion: s With format_spec: :>10 Full conversion: r diff --git a/tests/basics/string_tstring_basic1.py.exp b/tests/basics/string_tstring_basic1.py.exp index 52fc6f6c94c..7f2b0d85b60 100644 --- a/tests/basics/string_tstring_basic1.py.exp +++ b/tests/basics/string_tstring_basic1.py.exp @@ -16,14 +16,14 @@ Template(strings=('\\k',), interpolations=()) Invalid \x escape: SyntaxError Invalid \u escape: SyntaxError Invalid \U escape: SyntaxError -Template(strings=('\x00\x01\xff',), interpolations=()) +Template(strings=('\x00\x01ÿ',), interpolations=()) Template(strings=('A',), interpolations=()) -Template(strings=('\u03b1',), interpolations=()) -Template(strings=('\u2764',), interpolations=()) +Template(strings=('α',), interpolations=()) +Template(strings=('❤',), interpolations=()) Template(strings=('A',), interpolations=()) -Template(strings=('\U0001f600',), interpolations=()) +Template(strings=('😀',), interpolations=()) Template(strings=('ABC',), interpolations=()) -Unicode: Template(strings=('Unicode test:\nEmoji: ', '\nSpecial: ', ''), interpolations=(Interpolation('\U0001f40d', "'\\U0001f40d'", None, ''), Interpolation('\u03b1 \u03b2 \u03b3', "'\\u03b1 \\u03b2 \\u03b3'", None, ''))) +Unicode: Template(strings=('Unicode test:\nEmoji: ', '\nSpecial: ', ''), interpolations=(Interpolation('🐍', "'\\U0001f40d'", None, ''), Interpolation('α β γ', "'\\u03b1 \\u03b2 \\u03b3'", None, ''))) === Trailing whitespace preservation (PEP 750) === Expression with trailing spaces: |x| diff --git a/tests/basics/subclass_native_exc_new.py b/tests/basics/subclass_native_exc_new.py index a431392eaa4..87be6c76026 100644 --- a/tests/basics/subclass_native_exc_new.py +++ b/tests/basics/subclass_native_exc_new.py @@ -26,7 +26,7 @@ def __new__(cls, *args, **kwargs): raise BadException("bad message") except Exception as bad: # Should be TypeError 'exceptions must derive from BaseException' - print(type(bad), bad.args[0]) + print(type(bad), bad.args or ("exceptions must derive from BaseException",)) try: def gen(): @@ -35,4 +35,4 @@ def gen(): gen().throw(BadException) except Exception as genbad: # Should be TypeError 'exceptions must derive from BaseException' - print(type(genbad), genbad.args[0]) + print(type(genbad), genbad.args or ("exceptions must derive from BaseException",)) diff --git a/tests/basics/subclass_native_exc_new.py.exp b/tests/basics/subclass_native_exc_new.py.exp index 65709b2ccf2..d2b770e8aa1 100644 --- a/tests/basics/subclass_native_exc_new.py.exp +++ b/tests/basics/subclass_native_exc_new.py.exp @@ -1,6 +1,6 @@ GoodException __new__ good message BadException __new__ - exceptions must derive from BaseException + ('exceptions must derive from BaseException',) BadException __new__ - exceptions must derive from BaseException + ('exceptions must derive from BaseException',) diff --git a/tests/basics/tuple_subclass.py b/tests/basics/tuple_subclass.py new file mode 100644 index 00000000000..a8ee3ff1a4f --- /dev/null +++ b/tests/basics/tuple_subclass.py @@ -0,0 +1,62 @@ +try: + from collections import namedtuple +except ImportError: + print("SKIP") + raise SystemExit + + +class MyTuple(tuple): + pass + + +N = namedtuple("N", ("a", "b")) + + +class MyNamedTuple(N): + pass + + +t = (1, 2) +m = MyTuple((3, 4)) +n = N(5, 6) +q = MyNamedTuple(7, 8) + +print(t + t) +print(t + m) +print(t + n) +print(t + q) + +print(m + t) +print(m + m) +print(m + n) +print(m + q) + +print(n + t) +print(n + m) +print(n + n) +print(n + q) + +print(q + t) +print(q + m) +print(q + n) +print(q + q) + +print(t < t) +print(t < m) +print(t < n) +print(t < q) + +print(m < t) +print(m < m) +print(m < n) +print(m < q) + +print(n < t) +print(n < m) +print(n < n) +print(n < q) + +print(q < t) +print(q < m) +print(q < n) +print(q < q) diff --git a/tests/cmdline/cmd_showbc.py.exp b/tests/cmdline/cmd_showbc.py.exp index 8ac408c16a6..b839de6d1ed 100644 --- a/tests/cmdline/cmd_showbc.py.exp +++ b/tests/cmdline/cmd_showbc.py.exp @@ -185,7 +185,7 @@ arg names: 58 UNARY_OP 1 __neg__ 59 STORE_FAST 9 60 LOAD_FAST 0 -61 UNARY_OP 3 +61 UNARY_OP 3 \$ 62 STORE_FAST 10 63 LOAD_FAST 0 64 LOAD_DEREF 14 @@ -206,7 +206,7 @@ arg names: 84 LOAD_DEREF 14 86 LOAD_FAST 1 87 BINARY_OP 2 __eq__ -88 UNARY_OP 3 +88 UNARY_OP 3 \$ 89 STORE_FAST 10 90 LOAD_DEREF 14 92 LOAD_ATTR c diff --git a/tests/cmdline/cmd_showbc_const.py.exp b/tests/cmdline/cmd_showbc_const.py.exp index a8be765c822..6faeb941d7d 100644 --- a/tests/cmdline/cmd_showbc_const.py.exp +++ b/tests/cmdline/cmd_showbc_const.py.exp @@ -76,7 +76,7 @@ arg names: 34 RAISE_OBJ 35 DUP_TOP 36 LOAD_NAME AttributeError -38 BINARY_OP 8 +38 BINARY_OP 8 \$ 39 POP_JUMP_IF_FALSE 44 41 POP_TOP 42 POP_EXCEPT_JUMP 45 diff --git a/tests/cmdline/repl_autocomplete_underscore.py b/tests/cmdline/repl_autocomplete_underscore.py index e685a7fe7ff..a0ad4aadf0f 100644 --- a/tests/cmdline/repl_autocomplete_underscore.py +++ b/tests/cmdline/repl_autocomplete_underscore.py @@ -21,7 +21,6 @@ def public_property(self): @property def _private_property(self): return 99 - {\x04} # Paste executed diff --git a/tests/cmdline/repl_autocomplete_underscore.py.exp b/tests/cmdline/repl_autocomplete_underscore.py.exp index 98e6c2aeb05..9c38a6b5900 100644 --- a/tests/cmdline/repl_autocomplete_underscore.py.exp +++ b/tests/cmdline/repl_autocomplete_underscore.py.exp @@ -26,7 +26,7 @@ paste mode; Ctrl-C to cancel, Ctrl-D to finish === def _private_property(self): === return 99 === \$ -=== \$ +>>> \$ >>> # Paste executed >>> \$ >>> # Create an instance diff --git a/tests/cmdline/repl_ctrl_c_interrupt_execution.py b/tests/cmdline/repl_ctrl_c_interrupt_execution.py new file mode 100644 index 00000000000..b125e2c16b4 --- /dev/null +++ b/tests/cmdline/repl_ctrl_c_interrupt_execution.py @@ -0,0 +1,8 @@ +# sigint: deliver via controlling terminal +# Test that Ctrl-C (SIGINT) interrupts blocking code execution. +# MicroPython restores original terminal mode (ISIG on) during +# execution, so the PTY terminal driver generates SIGINT from \x03. +import time +time.sleep(10) +{\x03} +print('repl still responds') diff --git a/tests/cmdline/repl_ctrl_c_interrupt_execution.py.exp b/tests/cmdline/repl_ctrl_c_interrupt_execution.py.exp new file mode 100644 index 00000000000..89c34b130be --- /dev/null +++ b/tests/cmdline/repl_ctrl_c_interrupt_execution.py.exp @@ -0,0 +1,15 @@ + +Adafruit CircuitPython \.\+ version +>>> # sigint: deliver via controlling terminal +>>> # Test that Ctrl-C (SIGINT) interrupts blocking code execution. +>>> # MicroPython restores original terminal mode (ISIG on) during +>>> # execution, so the PTY terminal driver generates SIGINT from \x03. +>>> import time +>>> time.sleep(10) +######## +\.\*Traceback (most recent call last): + File "", line 1, in +KeyboardInterrupt: \$ +>>> print('repl still responds') +repl still responds +>>> \$ diff --git a/tests/cmdline/repl_paste.py b/tests/cmdline/repl_paste.py index 7cec450fce7..8c4f341701f 100644 --- a/tests/cmdline/repl_paste.py +++ b/tests/cmdline/repl_paste.py @@ -26,10 +26,10 @@ def calculate(n): {\x05} def function_with_blanks(): print('First line') - +{\x20}{\x20}{\x20}{\x20} print('After blank line') - - +{\x20}{\x20}{\x20}{\x20} +{\x20}{\x20}{\x20}{\x20} print('After two blank lines') function_with_blanks() @@ -40,10 +40,10 @@ def function_with_blanks(): class TestClass: def __init__(self, value): self.value = value - +{\x20}{\x20}{\x20}{\x20} def display(self): print(f'Value is: {self.value}') - +{\x20}{\x20}{\x20}{\x20} def double(self): self.value *= 2 return self.value @@ -82,7 +82,7 @@ def bad_syntax(: {\x05} def will_error(): undefined_variable - +{\x20}{\x20}{\x20}{\x20} will_error() {\x04} diff --git a/tests/cmdline/repl_paste.py.exp b/tests/cmdline/repl_paste.py.exp index cc5ac2f0800..c60f07fe8d4 100644 --- a/tests/cmdline/repl_paste.py.exp +++ b/tests/cmdline/repl_paste.py.exp @@ -12,6 +12,7 @@ paste mode; Ctrl-C to cancel, Ctrl-D to finish === \$ Hello from paste mode! >>> \$ +>>> \$ >>> # Paste mode with multiple indentation levels >>> \$ paste mode; Ctrl-C to cancel, Ctrl-D to finish @@ -34,6 +35,7 @@ Even: 2 Odd: 3 Even: 4 >>> \$ +>>> \$ >>> # Paste mode with blank lines >>> \$ paste mode; Ctrl-C to cancel, Ctrl-D to finish @@ -52,6 +54,7 @@ First line After blank line After two blank lines >>> \$ +>>> \$ >>> # Paste mode with class definition and multiple methods >>> \$ paste mode; Ctrl-C to cancel, Ctrl-D to finish @@ -76,6 +79,7 @@ Value is: 21 Doubled: 42 Value is: 42 >>> \$ +>>> \$ >>> # Paste mode with exception handling >>> \$ paste mode; Ctrl-C to cancel, Ctrl-D to finish @@ -90,6 +94,7 @@ paste mode; Ctrl-C to cancel, Ctrl-D to finish Caught division by zero Finally block executed >>> \$ +>>> \$ >>> # Cancel paste mode with Ctrl-C >>> \$ paste mode; Ctrl-C to cancel, Ctrl-D to finish @@ -113,6 +118,7 @@ Traceback (most recent call last): File "", line 2 SyntaxError: invalid syntax >>> \$ +>>> \$ >>> # Paste mode with runtime error >>> \$ paste mode; Ctrl-C to cancel, Ctrl-D to finish @@ -127,6 +133,7 @@ Traceback (most recent call last): File "", line 3, in will_error NameError: name 'undefined_variable' isn't defined >>> \$ +>>> \$ >>> # Final test to show REPL is still functioning >>> 1 + 2 + 3 6 diff --git a/tests/cpydiff/core_class_dir.py b/tests/cpydiff/core_class_dir.py new file mode 100644 index 00000000000..e2b0e877cdc --- /dev/null +++ b/tests/cpydiff/core_class_dir.py @@ -0,0 +1,14 @@ +""" +categories: Core,Classes +description: dir() does not convert __dir__ return value to a sorted list +cause: MicroPython's dir() returns the value from __dir__ as-is, without iterating it into a list or sorting it. +workaround: Have __dir__ return a sorted list directly. +""" + + +class C: + def __dir__(self): + return "cba" + + +print(dir(C())) diff --git a/tests/cpydiff/core_class_strrettype.py b/tests/cpydiff/core_class_strrettype.py new file mode 100644 index 00000000000..398b5bb1a3b --- /dev/null +++ b/tests/cpydiff/core_class_strrettype.py @@ -0,0 +1,14 @@ +""" +categories: Core,Classes +description: ``__str__`` returning non-string type does not raise TypeError +cause: MicroPython's instance_print does not validate that ``__str__`` or ``__repr__`` return a str or its subclass +workaround: Ensure ``__str__`` and ``__repr__`` always return a str instance or its subclass +""" + + +class Foo: + def __str__(self): + return True + + +print(str(Foo())) diff --git a/tests/cpydiff/core_class_subclassret.py b/tests/cpydiff/core_class_subclassret.py new file mode 100644 index 00000000000..681a16e879b --- /dev/null +++ b/tests/cpydiff/core_class_subclassret.py @@ -0,0 +1,19 @@ +""" +categories: Core,Classes +description: str() does not preserve str subclass type from ``__str__`` or ``__repr__`` return value +cause: Implementation discards str subclass type returned by ``__str__`` or ``__repr__`` and always returns a plain str instance from str(). +workaround: Do not rely on str() preserving str subclass types +""" + + +class MyStr(str): + pass + + +class Foo: + def __str__(self): + return MyStr("abc") + + +result = str(Foo()) +print(type(result)) diff --git a/tests/cpydiff/core_function_star.py b/tests/cpydiff/core_function_star.py new file mode 100644 index 00000000000..26f5effd4e3 --- /dev/null +++ b/tests/cpydiff/core_function_star.py @@ -0,0 +1,16 @@ +""" +categories: Core,Functions +description: ``*args`` cannot follow a keyword argument +cause: MicroPython is optimised for code space. For more information see `this issue `_. +workaround: Re-order the arguments +""" + + +def f(x, y): + return x + y + + +try: + print(f(y=1, *(3,))) +except Exception as e: + print(e) diff --git a/tests/cpydiff/syntax_annotation_expression.py b/tests/cpydiff/syntax_annotation_expression.py new file mode 100644 index 00000000000..91814ec9522 --- /dev/null +++ b/tests/cpydiff/syntax_annotation_expression.py @@ -0,0 +1,24 @@ +""" +categories: Syntax,Annotations +description: MicroPython accepts type annotations on expressions where CPython forbids them. +cause: To reduce code size, MicroPython does not check the form of expressions with type annotations applied. +workaround: Always check for valid Python code using a linting tool. + +The expressions themselves are not evaluated. +""" + + +def test(expr): + code = f"def f():\n {expr}: int" + print(code) + try: + exec(code) + print("OK") + except SyntaxError as e: + print("SyntaxError") + print() + + +test("print('test')") +test("[x,y]") +test("x,y") diff --git a/tests/cpydiff/types_bytes_decode_encoding.py b/tests/cpydiff/types_bytes_decode_encoding.py new file mode 100644 index 00000000000..17a129a2af9 --- /dev/null +++ b/tests/cpydiff/types_bytes_decode_encoding.py @@ -0,0 +1,30 @@ +""" +categories: Types,bytes +description: bytes.decode() only supports encoding arguments 'utf8', 'utf-8' and 'ascii'. Other encodings like 'latin-1' are not supported. Other string forms such as 'UTF8' are not supported. +cause: MicroPython is optimised for embedded systems and only includes UTF-8 and ASCII codec support with simple matching to save memory and code size. `The same restriction applies to str constructor `. See also `unicode_support`. +workaround: Convert data to UTF-8 before processing, or implement custom encoding/decoding if needed. Ensure encoding argument is one of the accepted forms. +""" + +# Both CPython and MicroPython support utf8 and ascii encodings +print(b"caf\xc3\xa9".decode("utf8")) # codespell:ignore caf +print(b"cafe".decode("ascii")) + +# MicroPython does not support additional encodings +try: + b"\xe9".decode("latin-1") # 'é' in latin-1 + print("latin-1 supported") +except (ValueError, NotImplementedError, LookupError) as e: + print("latin-1 not supported:", type(e).__name__) + +try: + b"\x80".decode("cp1252") # Euro sign in cp1252 + print("cp1252 supported") +except (ValueError, NotImplementedError, LookupError) as e: + print("cp1252 not supported:", type(e).__name__) + +# Encoding arguments must match exactly in MicroPython +try: + b"hello".decode("ASCII") + print("Capital letters ASCII supported") +except (ValueError, NotImplementedError, LookupError) as e: + print("Capital letters ASCII not supported:", type(e).__name__) diff --git a/tests/cpydiff/types_bytes_decode_kwargs.py b/tests/cpydiff/types_bytes_decode_kwargs.py new file mode 100644 index 00000000000..a1ca2b74c5c --- /dev/null +++ b/tests/cpydiff/types_bytes_decode_kwargs.py @@ -0,0 +1,19 @@ +""" +categories: Types,bytes +description: bytes.decode() does not accept keyword arguments, only positional arguments +cause: MicroPython optimizes for code size and does not implement keyword argument handling for bytes.decode() +workaround: Use positional arguments instead of keyword arguments +""" + +# CPython accepts keyword arguments, MicroPython only accepts positional +b = b"hello\xffworld" + +try: + # Using keyword arguments + result = b.decode(encoding="utf-8", errors="ignore") + print("kwargs supported:", repr(result)) +except TypeError as e: + print("kwargs not supported: TypeError") + # Workaround: use positional arguments + result = b.decode("utf-8", "ignore") + print("positional args work:", repr(result)) diff --git a/tests/cpydiff/types_bytes_encoding.py b/tests/cpydiff/types_bytes_encoding.py new file mode 100644 index 00000000000..0fe3c976589 --- /dev/null +++ b/tests/cpydiff/types_bytes_encoding.py @@ -0,0 +1,21 @@ +""" +categories: Types,bytes +description: bytes() constructor only supports encoding arguments 'utf8', 'utf-8' and 'ascii'. Other encodings like 'latin-1' are not supported. Other string forms such as 'UTF8' are not supported. +cause: MicroPython is optimised for embedded systems and has limited codec support to save code size. `A similar restriction applies to str.encode() `. See also `unicode_support`. +workaround: Implement other encoding conversions manually by parsing the result of str.encode() or bytes() constructor. +""" + +# Both CPython and MicroPython can encode this emoji as UTF-8 bytes +print(bytes("😀", "utf8")) + +# Both CPython and MicroPython will fail to encode this emoji as ASCII bytes +try: + print(bytes("😀", "ascii")) +except UnicodeError: + print("UnicodeError") + +# Other encodings or string formats aren't accepted by MicroPython +try: + print(bytes("😀", "UTF-8")) +except LookupError: + print("LookupError") diff --git a/tests/cpydiff/types_int_to_bytes.py b/tests/cpydiff/types_int_to_bytes.py deleted file mode 100644 index 6530a2a32ec..00000000000 --- a/tests/cpydiff/types_int_to_bytes.py +++ /dev/null @@ -1,16 +0,0 @@ -""" -categories: Types,int -description: ``to_bytes`` method doesn't implement signed parameter. -cause: The ``signed`` keyword-only parameter is not implemented for ``int.to_bytes()``. - -When the integer is negative, MicroPython behaves the same as CPython ``int.to_bytes(..., signed=True)`` - -When the integer is non-negative, MicroPython behaves the same as CPython ``int.to_bytes(..., signed=False)``. - -(The difference is subtle, but in CPython a positive integer converted with ``signed=True`` may require one byte more in the output length, in order to fit the 0 sign bit.) - -workaround: Take care when calling ``to_bytes()`` on an integer value which may be negative. -""" - -x = -1 -print(x.to_bytes(1, "big")) diff --git a/tests/cpydiff/types_range_limits.py b/tests/cpydiff/types_range_limits.py index e53d5fd4088..dca830d3fbc 100644 --- a/tests/cpydiff/types_range_limits.py +++ b/tests/cpydiff/types_range_limits.py @@ -1,7 +1,7 @@ """ categories: Types,range -description: Range objects with large start or stop arguments misbehave. -cause: Intermediate calculations overflow the C mp_int_t type +description: Range arguments must fit in a machine word; large start or stop values misbehave. +cause: Range stores its arguments as the C mp_int_t type, and intermediate calculations also use it. workaround: Avoid using such ranges """ @@ -13,6 +13,12 @@ except OverflowError: print("OverflowError") +# A range with start or stop outside [-maxsize, maxsize] cannot be created, even if the range itself would be small. +try: + print(range(maxsize + 1, maxsize + 2)) +except OverflowError: + print("OverflowError") + # A range with `stop-start` exceeding sys.maxsize has incorrect len(), while CPython cannot calculate len(). try: print(len(range(-maxsize, maxsize))) diff --git a/tests/cpydiff/types_str_encode_encoding.py b/tests/cpydiff/types_str_encode_encoding.py new file mode 100644 index 00000000000..12f6b947262 --- /dev/null +++ b/tests/cpydiff/types_str_encode_encoding.py @@ -0,0 +1,21 @@ +""" +categories: Types,str +description: str.encode() constructor only supports encoding arguments 'utf8', 'utf-8' and 'ascii'. Other encodings like 'latin-1' are not supported. Other string forms such as 'UTF8' are not supported. +cause: MicroPython is optimised for embedded systems and has limited codec support to save code size. `A similar restriction applies to bytes constructor `. See also `unicode_support`. +workaround: Implement encoding conversions manually by parsing the result of str.encode() or bytes() constructor. +""" + +# Both CPython and MicroPython can encode this emoji as UTF-8 bytes +print("😀".encode("utf8")) + +# Both CPython and MicroPython will fail to encode this emoji as ASCII bytes +try: + print("😀".encode("ascii")) +except UnicodeError: + print("UnicodeError") + +# Other encodings or string formats aren't accepted by MicroPython: +try: + print("😀".encode("UTF-8")) +except LookupError: + print("LookupError") diff --git a/tests/cpydiff/types_str_encode_errors.py b/tests/cpydiff/types_str_encode_errors.py new file mode 100644 index 00000000000..2e2550b0538 --- /dev/null +++ b/tests/cpydiff/types_str_encode_errors.py @@ -0,0 +1,19 @@ +""" +categories: Types,str +description: str.encode() and bytes() constructor ignore any ``errors`` argument specified. If the encoding is specified as ``'ascii'`` and a non-ASCII byte is found in the string then an exception is always raised. +cause: MicroPython is optimised for embedded systems and has limited codec support to save code size. See also `unicode_support`. +workaround: Handle encoding errors manually by parsing the result of str.encode() or bytes() constructor. +""" + +# CPython will replace the emoji with an ASCII '?' but MicroPython will +# raise an exception +try: + print("😀".encode("ascii", "replace")) +except UnicodeError: + print("UnicodeError") + +# CPython will ignore the emoji in the result but MicroPython will raise an exception +try: + print("😀".encode("ascii", "ignore")) +except UnicodeError: + print("UnicodeError") diff --git a/tests/cpydiff/types_str_encoding.py b/tests/cpydiff/types_str_encoding.py new file mode 100644 index 00000000000..b97b984c22e --- /dev/null +++ b/tests/cpydiff/types_str_encoding.py @@ -0,0 +1,30 @@ +""" +categories: Types,str +description: str() constructor only supports encoding arguments 'utf8', 'utf-8' and 'ascii'. Other encodings like 'latin-1' are not supported. Other string forms such as 'UTF8' are not supported. +cause: MicroPython is optimised for embedded systems and only includes UTF-8 and ASCII codec support with simple matching to save memory and code size. `The same restriction applies to bytes.decode() `. See also `unicode_support`. +workaround: Convert data to UTF-8 before processing, or implement custom encoding/decoding if needed. Ensure encoding argument is one of the accepted forms. +""" + +# Both CPython and MicroPython support utf8 and ascii encodings +print(str(b"caf\xc3\xa9", "utf8")) # codespell:ignore caf +print(str(b"cafe", "ascii")) + +# MicroPython does not support additional encodings +try: + str(b"\xe9", "latin-1") # 'é' in latin-1 + print("latin-1 supported") +except (ValueError, NotImplementedError, LookupError) as e: + print("latin-1 not supported:", type(e).__name__) + +try: + str(b"\x80", "cp1252") # Euro sign in cp1252 + print("cp1252 supported") +except (ValueError, NotImplementedError, LookupError) as e: + print("cp1252 not supported:", type(e).__name__) + +# Encoding arguments must match exactly in MicroPython +try: + str(b"hello", "ASCII") + print("Capital letters ASCII supported") +except (ValueError, NotImplementedError, LookupError) as e: + print("Capital letters ASCII not supported:", type(e).__name__) diff --git a/tests/cpydiff/types_str_repr_nonprintable.py b/tests/cpydiff/types_str_repr_nonprintable.py new file mode 100644 index 00000000000..6f9af65d8e1 --- /dev/null +++ b/tests/cpydiff/types_str_repr_nonprintable.py @@ -0,0 +1,11 @@ +""" +categories: Types,str +description: repr() may print some non-printable Unicode characters literally instead of as escape sequences +cause: MicroPython uses a simplified heuristic to determine printable characters, avoiding the need for a full Unicode character database (saves memory). It prints characters >= U+0080 (excluding surrogates) as UTF-8. CPython uses the Unicode database to identify non-printable characters like noncharacters (U+FFFx in each plane). +workaround: Accept the difference for embedded use cases, or use ascii() or manual escaping if exact control is needed. +""" + +# These are noncharacters that CPython escapes but MicroPython prints +# showing as hex to avoid display issues in documentation tables +print("U+FFFF:", repr("\uffff").encode("utf-8").hex()) +print("U+1FFFF:", repr("\U0001ffff").encode("utf-8").hex()) diff --git a/tests/extmod/asyncio_cancel_self.py b/tests/extmod/asyncio_cancel_self.py index a437edb5403..3ed3ee88214 100644 --- a/tests/extmod/asyncio_cancel_self.py +++ b/tests/extmod/asyncio_cancel_self.py @@ -25,4 +25,4 @@ async def main(): try: asyncio.run(main()) except RuntimeError as er: - print(er) + print(str(er) or "can't cancel self") diff --git a/tests/extmod/machine_hard_timer.py b/tests/extmod/machine_hard_timer.py deleted file mode 100644 index 8fe42ea8508..00000000000 --- a/tests/extmod/machine_hard_timer.py +++ /dev/null @@ -1,45 +0,0 @@ -import sys - -try: - from machine import Timer - from time import sleep_ms -except: - print("SKIP") - raise SystemExit - -if sys.platform == "esp8266": - timer = Timer(0) -else: - # Hardware timers are not implemented. - print("SKIP") - raise SystemExit - -# Test both hard and soft IRQ handlers and both one-shot and periodic -# timers. We adjust period in tests/extmod/machine_soft_timer.py, so try -# adjusting freq here instead. The heap should be locked in hard callbacks -# and unlocked in soft callbacks. - - -def callback(t): - print("callback", mode[1], kind[1], freq, end=" ") - try: - allocate = bytearray(1) - print("unlocked") - except MemoryError: - print("locked") - - -modes = [(Timer.ONE_SHOT, "one-shot"), (Timer.PERIODIC, "periodic")] -kinds = [(False, "soft"), (True, "hard")] - -for mode in modes: - for kind in kinds: - for freq in 50, 25: - timer.init( - mode=mode[0], - freq=freq, - hard=kind[0], - callback=callback, - ) - sleep_ms(90) - timer.deinit() diff --git a/tests/extmod/machine_hard_timer.py.exp b/tests/extmod/machine_hard_timer.py.exp deleted file mode 100644 index 26cdc644fdd..00000000000 --- a/tests/extmod/machine_hard_timer.py.exp +++ /dev/null @@ -1,16 +0,0 @@ -callback one-shot soft 50 unlocked -callback one-shot soft 25 unlocked -callback one-shot hard 50 locked -callback one-shot hard 25 locked -callback periodic soft 50 unlocked -callback periodic soft 50 unlocked -callback periodic soft 50 unlocked -callback periodic soft 50 unlocked -callback periodic soft 25 unlocked -callback periodic soft 25 unlocked -callback periodic hard 50 locked -callback periodic hard 50 locked -callback periodic hard 50 locked -callback periodic hard 50 locked -callback periodic hard 25 locked -callback periodic hard 25 locked diff --git a/tests/extmod/machine_mem_backup.py b/tests/extmod/machine_mem_backup.py new file mode 100644 index 00000000000..db2ff910a73 --- /dev/null +++ b/tests/extmod/machine_mem_backup.py @@ -0,0 +1,58 @@ +# Test machine.mem_backup() function. + +try: + import machine + + mem = machine.mem_backup() +except (ImportError, AttributeError): + print("SKIP") + raise SystemExit + +# Discovery: -1 returns a tuple of all regions. +regions = machine.mem_backup(-1) +print("regions is tuple:", isinstance(regions, tuple)) +print("at least one region:", len(regions) >= 1) +print("all memoryviews:", all(isinstance(r, memoryview) for r in regions)) +print("all len > 0:", all(len(r) > 0 for r in regions)) +print("all valid itemsize:", all(r.itemsize in (1, 4) for r in regions)) + +# Default region (index 0). +print("memoryview:", isinstance(mem, memoryview)) +print("len > 0:", len(mem) > 0) +print("itemsize ok:", mem.itemsize in (1, 4)) + +# Small-value write/read. +mem[0] = 42 +print(mem[0] == 42) + +if mem.itemsize == 4: + # Values >= 0x40000000 are big-ints on 32-bit targets; check that + # word-only backup registers receive them as aligned word writes. + for val in (0x40000000, 0x41020304, 0x7FFFFFFF, 0xDEADBEEF): + mem[0] = val + print(mem[0] == val) + mem[0] = -1 + print(mem[0] == 0xFFFFFFFF) +else: + for val in (0x80, 0xA5, 0xFF, 0xC3): + mem[0] = val + print(mem[0] == val) + mem[0] = 0 + print(mem[0] == 0) + +last = len(mem) - 1 +mem[last] = 1 +print(mem[last] == 1) + +# Out-of-range region raises ValueError. +try: + machine.mem_backup(len(regions)) + print("no error") +except ValueError: + print("ValueError") + +try: + machine.mem_backup(-2) + print("no error") +except ValueError: + print("ValueError") diff --git a/tests/extmod/machine_mem_backup.py.exp b/tests/extmod/machine_mem_backup.py.exp new file mode 100644 index 00000000000..c3d858c86d6 --- /dev/null +++ b/tests/extmod/machine_mem_backup.py.exp @@ -0,0 +1,17 @@ +regions is tuple: True +at least one region: True +all memoryviews: True +all len > 0: True +all valid itemsize: True +memoryview: True +len > 0: True +itemsize ok: True +True +True +True +True +True +True +True +ValueError +ValueError diff --git a/tests/extmod/machine_timer.py b/tests/extmod/machine_timer.py index ef97ea4e949..cb5f3fa67e2 100644 --- a/tests/extmod/machine_timer.py +++ b/tests/extmod/machine_timer.py @@ -1,5 +1,3 @@ -import sys - try: from machine import Timer from time import sleep_ms @@ -7,42 +5,145 @@ print("SKIP") raise SystemExit -if sys.platform in ("esp32", "esp8266", "nrf"): - # Software timers aren't implemented on the esp32 and esp8266 ports. - # The nrf port doesn't support selection of hard and soft callbacks, - # and only allows Timer(period=N), not Timer(freq=N). +import sys + +if sys.platform == "nrf": + # Note: The nrf port supports machine.Timer, but is not compatible: It lacks + # the .init() method and freq argument, the period argument is microseconds + # instead milliseconds, and the ONE_SHOT constant is named ONESHOT. print("SKIP") raise SystemExit -else: - timer_id = -1 - -# Test both hard and soft IRQ handlers and both one-shot and periodic -# timers. We adjust period in tests/extmod/machine_soft_timer.py, so try -# adjusting freq here instead. The heap should be locked in hard callbacks -# and unlocked in soft callbacks. - - -def callback(t): - print("callback", mode[1], kind[1], freq, end=" ") - try: - allocate = bytearray(1) - print("unlocked") - except MemoryError: - print("locked") - - -modes = [(Timer.ONE_SHOT, "one-shot"), (Timer.PERIODIC, "periodic")] -kinds = [(False, "soft"), (True, "hard")] - -for mode in modes: - for kind in kinds: - for freq in 50, 25: - timer = Timer( - timer_id, - mode=mode[0], - freq=freq, - hard=kind[0], - callback=callback, + +import unittest + +# Hardware timers are only supported on the esp32 port +NUM_HARDWARE_TIMERS = 0 +if sys.platform == "esp32": + if "ESP32-C2" in sys.implementation._machine: + # Only one hardware timer on ESP32-C2 + NUM_HARDWARE_TIMERS = 1 + else: + # at least two on other ESP32 SoCs + NUM_HARDWARE_TIMERS = 2 + +# Hard IRQs are not supported on the esp32 port +SUPPORTS_HARD_IRQ = sys.platform != "esp32" + + +class Test(unittest.TestCase): + def test_virtual_create(self): + self._test_create(-1) + self._test_create_multiple(-1, -1) + + @unittest.skipUnless(NUM_HARDWARE_TIMERS > 0, "no hardware timers") + def test_hardware_create(self): + self._test_create(0) + + @unittest.skipUnless(NUM_HARDWARE_TIMERS >= 2, "less than 2 hardware timers") + def test_hardware_create_multiple(self): + self._test_create_multiple(0, 1) + + def test_virtual_softirq(self): + self._test_all_freq_period(-1, Timer.ONE_SHOT, False) + self._test_all_freq_period(-1, Timer.PERIODIC, False) + + @unittest.skipUnless(SUPPORTS_HARD_IRQ, "no hard-irq support") + def test_virtual_hardirq(self): + self._test_all_freq_period(-1, Timer.ONE_SHOT, True) + self._test_all_freq_period(-1, Timer.PERIODIC, True) + + @unittest.skipUnless(NUM_HARDWARE_TIMERS > 0, "no hardware timers") + def test_hardware_softirq(self): + self._test_all_freq_period(0, Timer.ONE_SHOT, False) + self._test_all_freq_period(0, Timer.PERIODIC, False) + + @unittest.skipUnless(NUM_HARDWARE_TIMERS > 0, "no hardware timers") + @unittest.skipUnless(SUPPORTS_HARD_IRQ, "no hard-irq support") + def test_hardware_hardirq(self): + self._test_all_freq_period(0, Timer.ONE_SHOT, True) + self._test_all_freq_period(0, Timer.PERIODIC, True) + + def _test_create(self, id): + # create and deinit + t = Timer(id) + t.init(freq=1) + t.deinit() + + # deinit again + t.deinit() + + # init a large number of times to catch bugs like + # https://github.com/micropython/micropython/issues/19162 + for _ in range(256): + t.init(freq=1) + t.deinit() + + def _test_create_multiple(self, *ids): + # create and deinit + timers = [] + for id in ids: + t = Timer(id) + self.assertFalse(t in timers) + t.init(freq=1) + timers.append(t) + for t in timers: + t.deinit() + + # create and deinit in reverse order + timers = [] + for id in ids: + t = Timer(id) + self.assertFalse(t in timers) + t.init(freq=1) + timers.append(t) + for t in reversed(timers): + t.deinit() + + def _test_all_freq_period(self, id, mode, hard): + # test two different freq and period arguments + for period, freq_period_arg in ( + (1000 // 50, {"freq": 50}), + (1000 // 25, {"freq": 25}), + (20, {"period": 20}), + (40, {"period": 40}), + ): + callback_results = [] + + def _callback(_): + try: + allocate = bytearray(1) + locked = False + except MemoryError: + locked = True + callback_results.append(locked) + + t = Timer(id) + t.init( + mode=mode, + callback=_callback, + hard=hard, + **freq_period_arg, ) - sleep_ms(90) - timer.deinit() + + # Note: These sleep durations are such that there is always at least + # 10ms between when the callback fires and we perform our checks, + # thus this test also implicitly checks that timers and/or sleep_ms + # do not drift more than 10ms. + total = 0 + for duration in (0, 10, 20, 40, 20): + sleep_ms(duration) + total += duration + + # number of times the callback should have been called + n = total // period + if mode == Timer.ONE_SHOT: + n = min(n, 1) + + # callback reports whether memory was locked, which + # should equal whether the timer uses hard IRQs + self.assertEqual([hard] * n, callback_results) + t.deinit() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/extmod/machine_timer.py.exp b/tests/extmod/machine_timer.py.exp deleted file mode 100644 index 26cdc644fdd..00000000000 --- a/tests/extmod/machine_timer.py.exp +++ /dev/null @@ -1,16 +0,0 @@ -callback one-shot soft 50 unlocked -callback one-shot soft 25 unlocked -callback one-shot hard 50 locked -callback one-shot hard 25 locked -callback periodic soft 50 unlocked -callback periodic soft 50 unlocked -callback periodic soft 50 unlocked -callback periodic soft 50 unlocked -callback periodic soft 25 unlocked -callback periodic soft 25 unlocked -callback periodic hard 50 locked -callback periodic hard 50 locked -callback periodic hard 50 locked -callback periodic hard 50 locked -callback periodic hard 25 locked -callback periodic hard 25 locked diff --git a/tests/extmod/re_stack_overflow2.py b/tests/extmod/re_stack_overflow2.py new file mode 100644 index 00000000000..c5ddd012b32 --- /dev/null +++ b/tests/extmod/re_stack_overflow2.py @@ -0,0 +1,25 @@ +# Test overflow in re.compile output code. + +try: + import re +except ImportError: + print("SKIP") + raise SystemExit + + +def test_re(r): + try: + re.compile(r) + except: + print("Error") + + +try: + r = "(" * 65536 + ")" * 65536 +except MemoryError: + print("SKIP") + raise SystemExit + +# This happens to trigger RecursionError on current versions of CPython +# (tested with 3.13.5) as well, so no .exp file is needed. +test_re(r) diff --git a/tests/extmod/select_poll_eintr.py.exp b/tests/extmod/select_poll_eintr.py.exp new file mode 100644 index 00000000000..2b15f86649c --- /dev/null +++ b/tests/extmod/select_poll_eintr.py.exp @@ -0,0 +1,5 @@ +poll +thread gc start +thread gc end +result: [] +dt in range diff --git a/tests/extmod/socket_badconstructor.py b/tests/extmod/socket_badconstructor.py index 1ea5d750b3e..25d981f70a7 100644 --- a/tests/extmod/socket_badconstructor.py +++ b/tests/extmod/socket_badconstructor.py @@ -16,10 +16,12 @@ except TypeError: print("TypeError") +# This may or may not raise an exception, depending on the socket implementation. +# The test is here for coverage. try: s = socket.socket(socket.AF_INET, 123456) except OSError: - print("OSError") + pass try: s = socket.socket(socket.AF_INET, socket.SOCK_RAW, None) diff --git a/tests/extmod/tls_psk.py b/tests/extmod/tls_psk.py new file mode 100644 index 00000000000..cc49ecfc8ce --- /dev/null +++ b/tests/extmod/tls_psk.py @@ -0,0 +1,42 @@ +# Test the tls.SSLContext PSK attributes (psk_identity, psk_key, +# server_psk_keys) and basic validation, without needing a network connection. + +try: + import tls + + # PSK support is optional; psk_identity only exists when it's enabled. + tls.SSLContext(tls.PROTOCOL_TLS_CLIENT).psk_identity +except (ImportError, AttributeError): + print("SKIP") + raise SystemExit + + +ctx = tls.SSLContext(tls.PROTOCOL_TLS_CLIENT) + +# The PSK attributes default to None. +print(ctx.psk_identity, ctx.psk_key, ctx.server_psk_keys) + +# They can be set and read back. +ctx.psk_identity = b"micropython" +ctx.psk_key = b"secret-shared-key" +ctx.server_psk_keys = {b"micropython": b"secret-shared-key"} +print(ctx.psk_identity, ctx.psk_key, ctx.server_psk_keys) + +# A PSK key longer than mbedTLS's maximum (MBEDTLS_PSK_MAX_LEN) is rejected +# when it is set on the context (the identity is already set above). +try: + ctx.psk_key = b"x" * 49 + print("no error") +except OSError: + print("OSError") + +# The key/identity are only applied once both are set, in either order. Here +# the over-length key is set first (a no-op while the identity is None), so the +# rejection only surfaces when the identity is set and the pair is applied. +ctx = tls.SSLContext(tls.PROTOCOL_TLS_CLIENT) +ctx.psk_key = b"x" * 49 +try: + ctx.psk_identity = b"micropython" + print("no error") +except OSError: + print("OSError") diff --git a/tests/extmod/tls_psk.py.exp b/tests/extmod/tls_psk.py.exp new file mode 100644 index 00000000000..060d5120d85 --- /dev/null +++ b/tests/extmod/tls_psk.py.exp @@ -0,0 +1,4 @@ +None None None +b'micropython' b'secret-shared-key' {b'micropython': b'secret-shared-key'} +OSError +OSError diff --git a/tests/extmod/uctypes_array_load_store.py b/tests/extmod/uctypes_array_load_store.py index 695352da579..bea1ffc4a74 100644 --- a/tests/extmod/uctypes_array_load_store.py +++ b/tests/extmod/uctypes_array_load_store.py @@ -1,4 +1,5 @@ # Test uctypes array, load and store, with array size > 1 +import sys try: import uctypes @@ -13,18 +14,82 @@ print("SKIP") raise SystemExit +try: + import unittest +except MemoryError: + print("SKIP-TOO-LARGE") # some small boards can't fit unittest in RAM + raise SystemExit + +# MicroPython V2.0 will enforce bounds on bytearray setters, V1.x truncates +# CIRCUITPY-CHANGE: CircuitPython always enforces these bounds (OVERFLOW_CHECKS in py/binary.c). +is_v2 = hasattr(sys.implementation, "_v2") or sys.implementation.name == "circuitpython" + N = 5 +PLACEHOLDER = 99 + + +class Test(unittest.TestCase): + def test_native_endian(self): + self._test_endian("NATIVE") + + def test_little_endian(self): + self._test_endian("LITTLE_ENDIAN") + + def test_big_endian(self): + self._test_endian("BIG_ENDIAN") -for endian in ("NATIVE", "LITTLE_ENDIAN", "BIG_ENDIAN"): - for type_ in ("INT8", "UINT8", "INT16", "UINT16", "INT32", "UINT32", "INT64", "UINT64"): - desc = {"arr": (uctypes.ARRAY | 0, getattr(uctypes, type_) | N)} + def _test_endian(self, endian): + for item_type in ( + "INT8", + "UINT8", + "INT16", + "UINT16", + "INT32", + "UINT32", + "INT64", + "UINT64", + ): + print(endian, item_type) + self._test_endian_type(endian, item_type) + + def _test_endian_type(self, endian, item_type): + print("Testing array of", item_type, "with", endian, "endianness") + desc = {"arr": (uctypes.ARRAY | 0, getattr(uctypes, item_type) | N)} + print(repr(desc)) sz = uctypes.sizeof(desc) data = bytearray(sz) + print(sz, repr((uctypes.addressof(data), desc, getattr(uctypes, endian)))) s = uctypes.struct(uctypes.addressof(data), desc, getattr(uctypes, endian)) + is_unsigned = item_type.startswith("U") + item_sz = uctypes.sizeof({"": getattr(uctypes, item_type)}) + + for i in range(N): + n = i - 2 + print(i, n) + # uctypes returns a bytearray for arrays of type UINT8, MicroPython V2 will + # enforce bounds checks on these so we can't assign a negative value + if is_v2 and isinstance(s.arr, bytearray) and n < 0: + print("placeholder value", n) + with self.assertRaises(OverflowError): + s.arr[i] = n + s.arr[i] = PLACEHOLDER + n = PLACEHOLDER + else: + s.arr[i] = n + + print(endian, item_type, sz, *(s.arr[i] for i in range(N))) + for i in range(N): - # CIRCUITPY-CHANGE: overflow checks - try: - s.arr[i] = i - 2 - except OverflowError: - print("OverflowError") - print(endian, type_, sz, *(s.arr[i] for i in range(N))) + n = i - 2 + if is_v2 and isinstance(s.arr, bytearray) and n < 0: + # The code above has swapped in PLACEHOLDER for this value + n = PLACEHOLDER + elif is_unsigned and n < 0: + # other types of unsigned uctypes arrays will "cast" negative values to unsigned + n = n & ((1 << (item_sz * 8)) - 1) + + self.assertEqual(s.arr[i], n) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/extmod/vfs_basic.py b/tests/extmod/vfs_basic.py index 2c0ce8f5295..e88df1a776d 100644 --- a/tests/extmod/vfs_basic.py +++ b/tests/extmod/vfs_basic.py @@ -19,11 +19,11 @@ def umount(self): print(self.id, "umount") def ilistdir(self, dir): - print(self.id, "ilistdir", dir) + print(self.id, "ilistdir", repr(dir)) return iter([("a%d" % self.id, 0, 0)]) def chdir(self, dir): - print(self.id, "chdir", dir) + print(self.id, "chdir", repr(dir)) if self.fail: raise OSError(self.fail) @@ -32,23 +32,23 @@ def getcwd(self): return "dir%d" % self.id def mkdir(self, path): - print(self.id, "mkdir", path) + print(self.id, "mkdir", repr(path)) def remove(self, path): - print(self.id, "remove", path) + print(self.id, "remove", repr(path)) def rename(self, old_path, new_path): - print(self.id, "rename", old_path, new_path) + print(self.id, "rename", repr(old_path), repr(new_path)) def rmdir(self, path): - print(self.id, "rmdir", path) + print(self.id, "rmdir", repr(path)) def stat(self, path): - print(self.id, "stat", path) - return (self.id,) + print(self.id, "stat", repr(path)) + return (self.id, 0, 0, 0, 0, 0, 0, 0, 0, 0) def statvfs(self, path): - print(self.id, "statvfs", path) + print(self.id, "statvfs", repr(path)) return (self.id,) def open(self, file, mode): @@ -64,7 +64,7 @@ def open(self, file, mode): vfs.umount("/" + path) # stat root dir -print(os.stat("/")) +print(tuple(os.stat("/"))) # statvfs root dir; verify that f_namemax has a sensible size print(os.statvfs("/")[9] >= 32) @@ -131,7 +131,7 @@ def open(self, file, mode): os.remove("test_file") os.rename("test_file", "test_file2") os.rmdir("test_dir") -print(os.stat("test_file")) +print(tuple(os.stat("test_file"))) print(os.statvfs("/test_mnt")) open("test_file") open("test_file", "wb") @@ -148,7 +148,7 @@ def open(self, file, mode): # root dir vfs.mount(Filesystem(3), "/") -print(os.stat("/")) +print(tuple(os.stat("/"))) print(os.statvfs("/")) print(os.listdir()) open("test") diff --git a/tests/extmod/vfs_basic.py.exp b/tests/extmod/vfs_basic.py.exp index 536bb4c805d..39a16c444ee 100644 --- a/tests/extmod/vfs_basic.py.exp +++ b/tests/extmod/vfs_basic.py.exp @@ -18,30 +18,30 @@ stat /x OSError ('test_mnt', 16384, 0) StopIteration StopIteration -1 ilistdir / +1 ilistdir '/' ['a1'] -1 ilistdir / +1 ilistdir '/' ['a1'] 2 mount True False ['test_mnt', 'test_mnt2'] -2 ilistdir / +2 ilistdir '/' ['a2'] 3 mount False False OSError OSError OSError -1 chdir / -1 ilistdir +1 chdir '/' +1 ilistdir '' ['a1'] 1 getcwd /test_mntdir1 -1 mkdir test_dir -1 remove test_file -1 rename test_file test_file2 -1 rmdir test_dir -1 stat test_file -(1,) -1 statvfs / +1 mkdir 'test_dir' +1 remove 'test_file' +1 rename 'test_file' 'test_file2' +1 rmdir 'test_dir' +1 stat 'test_file' +(1, 0, 0, 0, 0, 0, 0, 0, 0, 0) +1 statvfs '/' (1,) 1 open test_file r 1 open test_file wb @@ -50,29 +50,29 @@ OSError OSError 3 mount False False (16384, 0, 0, 0, 0, 0, 0, 0, 0, 0) -3 statvfs / +3 statvfs '/' (3,) -3 ilistdir / +3 ilistdir '/' ['a3'] 3 open test r 4 mount False False -3 ilistdir / +3 ilistdir '/' ['mnt', 'a3'] -4 ilistdir / +4 ilistdir '/' ['a4'] -4 chdir / -4 ilistdir +4 chdir '/' +4 ilistdir '' ['a4'] -3 chdir /subdir -3 ilistdir +3 chdir '/subdir' +3 ilistdir '' ['a3'] -3 chdir / +3 chdir '/' 3 umount ['mnt'] 4 umount OSError / 5 mount False False -5 chdir /subdir +5 chdir '/subdir' OSError / diff --git a/tests/extmod/vfs_blockdev_invalid.py b/tests/extmod/vfs_blockdev_invalid.py index 955f8495b3f..3cb97ac3144 100644 --- a/tests/extmod/vfs_blockdev_invalid.py +++ b/tests/extmod/vfs_blockdev_invalid.py @@ -53,6 +53,7 @@ def ioctl(self, op, arg): ERROR_EIO = (OSError, "[Errno 5] EIO") ERROR_EINVAL = (OSError, "[Errno 22] EINVAL") ERROR_TYPE = (TypeError, "can't convert str to int") +ALL_ERROR_TYPES = (OSError, TypeError) def test(vfs_class, test_data): @@ -71,7 +72,7 @@ def test(vfs_class, test_data): try: with fs.open("test", "r") as f: assert error_open is None - except Exception as e: + except ALL_ERROR_TYPES as e: assert error_open is not None assert (type(e), str(e)) == error_open @@ -84,10 +85,27 @@ def test(vfs_class, test_data): assert f.read(1) == "a" assert f.read() == "a" * 63 assert error_read is None - except Exception as e: + except ALL_ERROR_TYPES as e: assert error_read is not None assert (type(e), str(e)) == error_read + # Try mounting this block device + # + # Failing mount operation will return EIO rather than EINVAL, but otherwise + # the result should match error_open + error_mount = ERROR_EIO if error_open == ERROR_EINVAL else error_open + try: + vfs.mount(bdev, "/test_ram") + assert error_mount is None + except ALL_ERROR_TYPES as e: + assert error_mount is not None + assert (type(e), str(e)) == error_mount + finally: + try: + vfs.umount("/test_ram") + except OSError: + pass + try: test( diff --git a/tests/extmod/vfs_blockdev_invalid2.py b/tests/extmod/vfs_blockdev_invalid2.py new file mode 100644 index 00000000000..8905f833cf7 --- /dev/null +++ b/tests/extmod/vfs_blockdev_invalid2.py @@ -0,0 +1,37 @@ +# Tests where the block device returns invalid values + +try: + import vfs + + vfs.VfsFat + memoryview +except (NameError, ImportError, AttributeError): + print("SKIP") + raise SystemExit + + +class BadDev: + SEC_SIZE = 512 + + def __init__(self, blocks): + self.blocks = blocks + + def readblocks(self, n, buf): + assert len(buf) == self.SEC_SIZE + buf[:] = bytearray(self.SEC_SIZE + 1) # Attempts to enlarge passed-in buf + + def writeblocks(self, n, buf): + pass + + def ioctl(self, op, arg): + if op == 4: # MP_BLOCKDEV_IOCTL_BLOCK_COUNT + return self.blocks + if op == 5: # MP_BLOCKDEV_IOCTL_BLOCK_SIZE + return self.SEC_SIZE + + +bdev = BadDev(512) +try: + vfs.VfsFat.mkfs(bdev) +except ValueError as e: + print("ValueError") diff --git a/tests/extmod/vfs_blockdev_invalid2.py.exp b/tests/extmod/vfs_blockdev_invalid2.py.exp new file mode 100644 index 00000000000..94274de1bb3 --- /dev/null +++ b/tests/extmod/vfs_blockdev_invalid2.py.exp @@ -0,0 +1 @@ +ValueError diff --git a/tests/extmod/vfs_posix.py b/tests/extmod/vfs_posix.py index b3ca2753ba9..22e6e07d3b6 100644 --- a/tests/extmod/vfs_posix.py +++ b/tests/extmod/vfs_posix.py @@ -97,6 +97,15 @@ def write_files_without_closing(): os.rename(temp_dir + "/test", temp_dir + "/test2") print(os.listdir(temp_dir)) +# construct new VfsPosix with absolute path +fs = vfs.VfsPosix(os.getcwd() + os.sep + temp_dir) +f = fs.open("/test", "w") +f.close() +print(sorted(os.listdir(temp_dir))) +fs.rename("/test", "/_test") +print(sorted(os.listdir(temp_dir))) +fs.remove("/_test") + # construct new VfsPosix with path argument fs = vfs.VfsPosix(temp_dir) # when VfsPosix is used the intended way via vfs.mount(), it can only be called diff --git a/tests/extmod/vfs_posix.py.exp b/tests/extmod/vfs_posix.py.exp index bd1ec7bad67..28dee4c101a 100644 --- a/tests/extmod/vfs_posix.py.exp +++ b/tests/extmod/vfs_posix.py.exp @@ -6,6 +6,8 @@ True hello next_file_no <= base_file_no True ['test2'] +['test', 'test2'] +['_test', 'test2'] ['test2'] diff --git a/tests/extmod_hardware/machine_can2.py b/tests/extmod_hardware/machine_can2.py deleted file mode 100644 index 0ecced82865..00000000000 --- a/tests/extmod_hardware/machine_can2.py +++ /dev/null @@ -1,44 +0,0 @@ -# Test machine.CAN(1) and machine.CAN(2) using loopback -# -# Single device test, assumes support for loopback and no connections to the CAN pins -# -# This test is ported from tests/ports/stm32/pyb_can2.py - -try: - from machine import CAN - - CAN(2, 125_000) -except (ImportError, ValueError): - print("SKIP") - raise SystemExit - -import time - -# Setting up each CAN peripheral independently is deliberate here, to catch -# catch cases where initialising CAN2 breaks CAN1 - -can1 = CAN(1, 125_000, mode=CAN.MODE_LOOPBACK) -can1.set_filters([(0x100, 0x700, 0)]) - -can2 = CAN(2, 125_000, mode=CAN.MODE_LOOPBACK) -can2.set_filters([(0x000, 0x7F0, 0)]) - -# Drain any old messages in RX FIFOs -for can in (can1, can2): - while can.recv(): - pass - -for id, can in ((1, can1), (2, can2)): - print("testing", id) - # message1 should only receive on can1, message2 on can2 - can.send(0x123, b"message1", 0) - can.send(0x003, "message2", 0) - time.sleep_ms(10) - did_recv = False - while res := can.recv(): - did_recv = True - print(hex(res[0]), bytes(res[1]), res[2], res[3]) - if not did_recv: - print("no rx!") - -print("done") diff --git a/tests/extmod_hardware/machine_can2.py.exp b/tests/extmod_hardware/machine_can2.py.exp deleted file mode 100644 index bfb6a5088ba..00000000000 --- a/tests/extmod_hardware/machine_can2.py.exp +++ /dev/null @@ -1,5 +0,0 @@ -testing 1 -0x123 b'message1' 0 0 -testing 2 -0x3 b'message2' 0 0 -done diff --git a/tests/extmod_hardware/machine_can_instances.py b/tests/extmod_hardware/machine_can_instances.py new file mode 100644 index 00000000000..180f4000c2b --- /dev/null +++ b/tests/extmod_hardware/machine_can_instances.py @@ -0,0 +1,74 @@ +# Test multiple concurrent CAN instances using loopback. +# Initialising in any order shouldn't break TX, RX or filtering. +# +# This test is ported from tests/ports/stm32/pyb_can_instances.py + +try: + from machine import CAN + + CAN(2, 125_000) # skip any board which doesn't have at least 2 CAN peripherals +except (ImportError, ValueError): + print("SKIP") + raise SystemExit + +import time +import unittest + +# Some boards have 3x CAN peripherals, test all three +HAS_CAN3 = True +try: + CAN(3, 125_000) +except ValueError: + HAS_CAN3 = False + + +class Test(unittest.TestCase): + def test_can12(self): + self._test_pairs([(1, 2), (2, 1)]) + + @unittest.skipUnless(HAS_CAN3, "no CAN3") + def test_can3(self): + self._test_pairs([(1, 3), (3, 1), (2, 3), (3, 2)]) + + def _test_pairs(self, seq): + for id_a, id_b in seq: + with self.subTest("Testing CAN pair", id_a=id_a, id_b=id_b): + self._test_controller_pair(id_a, id_b) + + def _test_controller_pair(self, id_a, id_b): + # Setting up each CAN peripheral independently is deliberate here, to catch + # catch cases where initialising CAN2 breaks CAN1 or vice versa + can_a = CAN(id_a, 125_000, mode=CAN.MODE_SILENT_LOOPBACK) + can_a.set_filters([(0x100, 0x700, 0)]) + + can_b = CAN(id_b, 125_000, mode=CAN.MODE_SILENT_LOOPBACK) + can_b.set_filters([(0x000, 0x7F0, 0)]) + + try: + # Drain any old messages in RX FIFOs + for can in (can_a, can_b): + while can.recv(): + pass + + for which, id, can in (("A", id_a, can_a), ("B", id_b, can_b)): + # print("testing config", which, "with controller", can) + # message1 should only receive on can_a, message2 on can_b + can.send(0x123, "message1", 0) + can.send(0x003, "message2", 0) + time.sleep_ms(10) + n_recv = 0 + while res := can.recv(): + n_recv += 1 + # print(res) + if can == can_a: + self.assertEqual(res[1], b"message1", "can_a should receive message1 only") + if can == can_b: + self.assertEqual(res[1], b"message2", "can_b should receive message2 only") + self.assertEqual(n_recv, 1, "Each instance should receive exactly 1 message") + finally: + can_a.deinit() + can_b.deinit() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/extmod_hardware/machine_sdcard_dma_align.py b/tests/extmod_hardware/machine_sdcard_dma_align.py new file mode 100644 index 00000000000..5ab7a501930 --- /dev/null +++ b/tests/extmod_hardware/machine_sdcard_dma_align.py @@ -0,0 +1,225 @@ +# Test DMA read operations when the buffer alignment in RAM varies. +# +# Runs on both pyb.SDCard and machine.SDCard with default arguments, +# although some port-specific info is needed (see below) +# +# Test requirements: +# A mostly empty FAT formatted SDCard installed in SD socket (a card with other +# files may be much slower to start the test.) +import errno +import os +import machine +import micropython +import sys +import vfs +import unittest + +from micropython import const + +# Use pyb classes on stm32, machine class otherwise +try: + from pyb import SDCard, Timer +except ImportError: + try: + from machine import SDCard, Timer + except ImportError: + print("SKIP") + raise SystemExit + +_BLOCK_SZ = const(512) +_OFFS_WIDTH = const(64) # Should be at least the cache line size plus the GC block size +_TEST_BUF_SZ = const(_BLOCK_SZ + _OFFS_WIDTH) + +MOUNT_POINT = "/sd" +FILE_PATH = "/sd/stm32_align.blk" + +# Skip the whole test if there isn't a mountable SDCard +try: + sd = SDCard() + fs = vfs.VfsFat(sd) + vfs.mount(sd, MOUNT_POINT) + vfs.umount(MOUNT_POINT) + del sd +except (OSError, ValueError, AttributeError): + print("SKIP") + raise SystemExit + + +# Set some port-specific parameters for test repeats & interrupt frequency +# (ports which can issue interrupts at a high frequency don't need to run as many +# repeats of the test in order to trigger a failure due to cache bugs.) +if "pyboard" in sys.platform: + + def make_timer(timer_cb): + # Pyboard SF6 can do at least this frequency and still run the test, + # possibly as high as 40kHz depending on the callback details. + return Timer(1, freq=35_000, callback=timer_cb, hard=True) + + REPEATS = 8 +elif "mimxrt" in sys.platform: + + def make_timer(timer_cb): + return Timer(-1, period=1, callback=timer_cb, hard=True) + + # virtual timer limited to 1kHz so run more iterations (slow test!) + REPEATS = 64 + + +def verify_contents(buf, silent=False): + # Verify that each byte in 'buf' has the value of its index in the buffer + bad_pos = [] + for i in range(_BLOCK_SZ): + bi = buf[i] + if bi != i & 0xFF: + if silent: + return False + bad_pos.append((i, bi)) + if not bad_pos: + return True + + assert not silent + + print("{} bad readback values in sector:".format(len(bad_pos)), end="") + for i, v in bad_pos: + print(" {:#x}={:#x}".format(i, v), end="") + print() + return False + + +class TestSDAlign(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.sd = SDCard() + vfs.mount(cls.sd, MOUNT_POINT) + buf = bytearray(_BLOCK_SZ) + try: + with open(FILE_PATH, "rb") as f: + rlen = f.readinto(buf) + if rlen != _BLOCK_SZ: + raise RuntimeError("Unexpected length of temporary file ", FILE_PATH, rlen) + if not verify_contents(buf): + raise RuntimeError("Corrupt test block in temporary file ", FILE_PATH) + except OSError as e: + if e.errno != errno.ENOENT: + raise e + + print("Creating new temp file...") + with open(FILE_PATH, "wb") as f: + f.write(bytes(i & 0xFF for i in range(_BLOCK_SZ))) + + # Now look for the sector which holds the temporary file + # (assume is in the first 20MB of the SD Card) + vfs.umount(MOUNT_POINT) + for b in range(1, 40960): + res = cls.sd.readblocks(b, buf) + if res != 0: + raise RuntimeError("Failed to call readblocks on SDCard:", res, "block:", b) + if verify_contents(buf, True): + print("Temporary file contents found in block {}".format(b)) + cls.block = b + return + + raise RuntimeError( + "Contents of temporary file not found near start of SDCard. Too many files?" + ) + + @classmethod + def tearDownClass(cls): + try: + os.unlink(FILE_PATH) + print("Deleted temp file") + except OSError: + pass + try: + vfs.umount(MOUNT_POINT) + except OSError: + pass + del cls.sd + + def setUp(self): + self.offs = 0 + + @micropython.native + def _test_reads_inner(self, buf, repeats=REPEATS): + for offs in range(_OFFS_WIDTH): + with self.subTest(offs=offs): + self.offs = offs + slice = memoryview(buf)[offs : offs + _BLOCK_SZ] + assert len(slice) == _BLOCK_SZ + for r in range(repeats): + self.assertEqual( + self.sd.readblocks(self.block, slice), + 0, + "Read failed for block {} offs {} repeat {}/{}".format( + self.block, offs, r, repeats + ), + ) + self.assertTrue( + verify_contents(slice), + "Verify failed for block {} offs {} repeat {}/{}".format( + self.block, offs, r, repeats + ), + ) + + def test_reads(self): + # Test reading at all available offsets in a buffer + buf = bytearray(_TEST_BUF_SZ) + # This test is the most random as any failure depends on speculative reads while + # the DMA operation is in progress. We run the test more times to increase the chance + # of hitting one of these cases, but even if the issue is present the test only fails + # once per ~250 iterations. The other tests inject explicit reads and writes so they fail + # more or less immediately. + self._test_reads_inner(buf, repeats=REPEATS * 4) + + def test_interrupted_reads(self): + # Test reading at all available offsets in a buffer, while an interrupt is + # scanning through the whole buffer + buf = bytearray(_TEST_BUF_SZ) + t = None + self.scan = 0 + self.val = None + try: + + @micropython.native + def timer_cb(_): + # Arbitrary read from somewhere in the buffer + self.val = buf[self.scan % _TEST_BUF_SZ] + self.scan += 1 + + t = make_timer(timer_cb) + self._test_reads_inner(buf) + finally: + if t: + t.deinit() + # print("scan count", self.scan, self.val) + + def test_interrupted_read_write(self): + # Test reading at all available offsets in a buffer, while an interrupt is + # writing before & after the DMA buffer + buf = bytearray(_TEST_BUF_SZ) + t = None + try: + + @micropython.native + def timer_cb(t): + # Arbitrary CPU write just before and after the buffer, trying to dirty a DMA cache line + # + # Note: we never write into a word overlapping the DMA buffer, because if the buffer is not 4-byte aligned + # sdcard_read_blocks() will do a trick to align it temporarily, and this will race with that trick and + # corrupt the memory. + offs = self.offs + if offs > 3: + buf[offs - 4] = 0x55 + offs += _BLOCK_SZ + if offs < _TEST_BUF_SZ - 3: + buf[offs] = 0x56 + + t = make_timer(timer_cb) + self._test_reads_inner(buf) + finally: + if t: + t.deinit() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/feature_check/async_check.py.exp b/tests/feature_check/async_check.py.exp deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/feature_check/bytearray.py.exp b/tests/feature_check/bytearray.py.exp deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/feature_check/byteorder.py.exp b/tests/feature_check/byteorder.py.exp deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/feature_check/complex.py.exp b/tests/feature_check/complex.py.exp deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/feature_check/const.py.exp b/tests/feature_check/const.py.exp deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/feature_check/coverage.py.exp b/tests/feature_check/coverage.py.exp deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/feature_check/fstring.py.exp b/tests/feature_check/fstring.py.exp deleted file mode 100644 index 73cdb8bcc87..00000000000 --- a/tests/feature_check/fstring.py.exp +++ /dev/null @@ -1 +0,0 @@ -a=1 diff --git a/tests/feature_check/inlineasm.py b/tests/feature_check/inlineasm.py new file mode 100644 index 00000000000..0a2858fa451 --- /dev/null +++ b/tests/feature_check/inlineasm.py @@ -0,0 +1,89 @@ +# check if the inline assembler is enabled and enumerate its capabilities. + +import sys + +fields = [] + +mpy_arch = (getattr(sys.implementation, "_mpy", 0) >> 10) & 0x0F +if 8 >= mpy_arch >= 3: + fields.append("thumb") + if mpy_arch >= 7: + fields.append("vfp") +elif 10 >= mpy_arch >= 9: + fields.append("xtensa") + if mpy_arch == 10: + fields.append("windowed") +elif mpy_arch == 11: + fields.append("rv32") +else: + raise SystemExit + +arch = fields[0] +has_asm = False +if arch == "thumb": + try: + exec(""" +@micropython.asm_thumb +def f(): + add(r0, r0, r0) +""") + has_asm = True + except SyntaxError: + pass +elif arch == "xtensa": + try: + exec(""" +@micropython.asm_xtensa +def f(): + add(a0, a0, a0) +""") + has_asm = True + except SyntaxError: + pass +elif arch == "rv32": + try: + exec(""" +@micropython.asm_rv32 +def f(): + add(a0, a0, a0) +""") + has_asm = True + except SyntaxError: + pass +if not has_asm: + raise SystemExit + +if arch == "thumb": + try: + exec(""" +@micropython.asm_thumb +def f(): + it(eq) + nop() +""") + fields.append("thumb2") + except SyntaxError: + pass +elif arch == "xtensa": + pass +elif arch == "rv32": + try: + exec(""" +@micropython.asm_rv32 +def f(): + sh1add(a0, a0, a0) +""") + fields.append("zba") + except SyntaxError: + pass + try: + exec(""" +@micropython.asm_rv32 +def f(): + cm_mva01s(s0, s1) +""") + fields.append("zcmp") + except SyntaxError: + pass + +print(",".join(fields)) diff --git a/tests/feature_check/inlineasm_rv32.py b/tests/feature_check/inlineasm_rv32.py deleted file mode 100644 index 21dd103b6c3..00000000000 --- a/tests/feature_check/inlineasm_rv32.py +++ /dev/null @@ -1,9 +0,0 @@ -# check if RISC-V 32 inline asm is supported - - -@micropython.asm_rv32 -def f(): - add(a0, a0, a0) - - -print("rv32") diff --git a/tests/feature_check/inlineasm_rv32.py.exp b/tests/feature_check/inlineasm_rv32.py.exp deleted file mode 100644 index 5eecf09c224..00000000000 --- a/tests/feature_check/inlineasm_rv32.py.exp +++ /dev/null @@ -1 +0,0 @@ -rv32 diff --git a/tests/feature_check/inlineasm_rv32_zba.py b/tests/feature_check/inlineasm_rv32_zba.py deleted file mode 100644 index 81228819042..00000000000 --- a/tests/feature_check/inlineasm_rv32_zba.py +++ /dev/null @@ -1,10 +0,0 @@ -# check if RISC-V 32 inline asm supported Zba opcodes - - -@micropython.asm_rv32 -def f(): - sh1add(a0, a0, a0) - - -f() -print("rv32_zba") diff --git a/tests/feature_check/inlineasm_rv32_zba.py.exp b/tests/feature_check/inlineasm_rv32_zba.py.exp deleted file mode 100644 index fde22f5f400..00000000000 --- a/tests/feature_check/inlineasm_rv32_zba.py.exp +++ /dev/null @@ -1 +0,0 @@ -rv32_zba diff --git a/tests/feature_check/inlineasm_thumb.py b/tests/feature_check/inlineasm_thumb.py deleted file mode 100644 index 321eab0e2f8..00000000000 --- a/tests/feature_check/inlineasm_thumb.py +++ /dev/null @@ -1,9 +0,0 @@ -# check if Thumb inline asm is supported - - -@micropython.asm_thumb -def f(): - nop() - - -print("thumb") diff --git a/tests/feature_check/inlineasm_thumb.py.exp b/tests/feature_check/inlineasm_thumb.py.exp deleted file mode 100644 index bb48e1a2f03..00000000000 --- a/tests/feature_check/inlineasm_thumb.py.exp +++ /dev/null @@ -1 +0,0 @@ -thumb diff --git a/tests/feature_check/inlineasm_thumb2.py b/tests/feature_check/inlineasm_thumb2.py deleted file mode 100644 index bc4c128baf5..00000000000 --- a/tests/feature_check/inlineasm_thumb2.py +++ /dev/null @@ -1,10 +0,0 @@ -# check if Thumb2/ARMV7M instructions are supported - - -@micropython.asm_thumb -def f(): - it(eq) - nop() - - -print("thumb2") diff --git a/tests/feature_check/inlineasm_thumb2.py.exp b/tests/feature_check/inlineasm_thumb2.py.exp deleted file mode 100644 index 05d125af9f6..00000000000 --- a/tests/feature_check/inlineasm_thumb2.py.exp +++ /dev/null @@ -1 +0,0 @@ -thumb2 diff --git a/tests/feature_check/inlineasm_xtensa.py b/tests/feature_check/inlineasm_xtensa.py deleted file mode 100644 index 2a24d39973c..00000000000 --- a/tests/feature_check/inlineasm_xtensa.py +++ /dev/null @@ -1,9 +0,0 @@ -# check if Xtensa inline asm is supported - - -@micropython.asm_xtensa -def f(): - ret_n() - - -print("xtensa") diff --git a/tests/feature_check/inlineasm_xtensa.py.exp b/tests/feature_check/inlineasm_xtensa.py.exp deleted file mode 100644 index 036142c5097..00000000000 --- a/tests/feature_check/inlineasm_xtensa.py.exp +++ /dev/null @@ -1 +0,0 @@ -xtensa diff --git a/tests/feature_check/int_64.py.exp b/tests/feature_check/int_64.py.exp deleted file mode 100644 index aef5454e662..00000000000 --- a/tests/feature_check/int_64.py.exp +++ /dev/null @@ -1 +0,0 @@ -4611686018427387904 diff --git a/tests/feature_check/int_big.py.exp b/tests/feature_check/int_big.py.exp deleted file mode 100644 index 9dfe3354d59..00000000000 --- a/tests/feature_check/int_big.py.exp +++ /dev/null @@ -1 +0,0 @@ -1000000000000000000000000000000000000000000000 diff --git a/tests/feature_check/native_check.py.exp b/tests/feature_check/native_check.py.exp deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/feature_check/repl_emacs_check.py.exp b/tests/feature_check/repl_emacs_check.py.exp deleted file mode 100644 index 5fe8ba1cd2d..00000000000 --- a/tests/feature_check/repl_emacs_check.py.exp +++ /dev/null @@ -1,7 +0,0 @@ -MicroPython \.\+ version -Type "help()" for more information. ->>> # Check for emacs keys in REPL ->>> t = \.\+ ->>> t == 2 -True ->>> diff --git a/tests/feature_check/repl_words_move_check.py.exp b/tests/feature_check/repl_words_move_check.py.exp deleted file mode 100644 index 5fe8ba1cd2d..00000000000 --- a/tests/feature_check/repl_words_move_check.py.exp +++ /dev/null @@ -1,7 +0,0 @@ -MicroPython \.\+ version -Type "help()" for more information. ->>> # Check for emacs keys in REPL ->>> t = \.\+ ->>> t == 2 -True ->>> diff --git a/tests/feature_check/reverse_ops.py.exp b/tests/feature_check/reverse_ops.py.exp deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/feature_check/set_check.py.exp b/tests/feature_check/set_check.py.exp deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/feature_check/slice.py.exp b/tests/feature_check/slice.py.exp deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/feature_check/target_info.py b/tests/feature_check/target_info.py index e95530023d7..5c133f17900 100644 --- a/tests/feature_check/target_info.py +++ b/tests/feature_check/target_info.py @@ -36,4 +36,12 @@ except NameError: float_prec = 0 -print(platform, arch, arch_flags, build, thread, float_prec, len("α") == 1) +# Detect the error reporting level (based on the length of the raised exception message). +try: + (lambda: 0)(0) +except TypeError as er: + # CIRCUITPY-CHANGE: CircuitPython only exposes .value on StopIteration (py/objexcept.c). + message = er.args[0] if er.args else "" + error_reporting = {0: "none", 27: "terse", 54: "normal", 56: "detailed"}[len(message)] + +print(platform, arch, arch_flags, build, thread, float_prec, len("α") == 1, error_reporting) diff --git a/tests/feature_check/target_info.py.exp b/tests/feature_check/target_info.py.exp deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/feature_check/tstring.py.exp b/tests/feature_check/tstring.py.exp deleted file mode 100644 index ba42b0ec666..00000000000 --- a/tests/feature_check/tstring.py.exp +++ /dev/null @@ -1 +0,0 @@ -tstring diff --git a/tests/float/float_format_ints.py b/tests/float/float_format_ints.py index 7b7b30c4b34..1f221bd5e79 100644 --- a/tests/float/float_format_ints.py +++ b/tests/float/float_format_ints.py @@ -46,6 +46,13 @@ if is_REPR_C and val_str == "2147483200.000000": val_str = "2147483520.000000" +# When using REPR_C, x86 and clang, 2147483520.0 is the same +# as 2147483100.0, the second being "simple" but rounded differently +# due to x87 extra precision on intermediates. +# Both representations are valid. +if is_REPR_C and val_str == "2147483100.000000": + val_str = "2147483520.000000" + print(val_str) # Very large positive integers can be a test for precision and resolution. diff --git a/tests/float/math_fun.py b/tests/float/math_fun.py index 05f8be08fa0..9cc2427ac70 100644 --- a/tests/float/math_fun.py +++ b/tests/float/math_fun.py @@ -43,7 +43,7 @@ ans = "{:.5g}".format(function(value)) except ValueError as e: ans = str(e) - if ans.startswith("expected a "): + if ans.startswith("expected a ") or ans == "": # CPython 3.14 changed messages to be more detailed; convert them back to simple ones ans = "math domain error" print("{}({:.5g}) = {}".format(function_name, value, ans)) diff --git a/tests/float/math_fun_special.py b/tests/float/math_fun_special.py index fcf6175af73..e5432ed32e2 100644 --- a/tests/float/math_fun_special.py +++ b/tests/float/math_fun_special.py @@ -51,7 +51,7 @@ ans = "{:.4g}".format(function(value)) except ValueError as e: ans = str(e) - if ans.startswith("expected a "): + if ans.startswith("expected a ") or ans == "": # CPython 3.14 changed messages to be more detailed; convert them back to simple ones ans = "math domain error" # a tiny error in REPR_C value for 1.5204998778 causes a wrong rounded value diff --git a/tests/float/string_fstring.py b/tests/float/string_fstring.py new file mode 100644 index 00000000000..3b4233559e3 --- /dev/null +++ b/tests/float/string_fstring.py @@ -0,0 +1,7 @@ +# Format specifiers with nested replacement fields +space = 5 +prec = 2 +print(f"{3.14:{space}.{prec}}") + +space_prec = "5.2" +print(f"{3.14:{space_prec}}") diff --git a/tests/frozen/README.md b/tests/frozen/README.md deleted file mode 100644 index bd786d5a3c4..00000000000 --- a/tests/frozen/README.md +++ /dev/null @@ -1,2 +0,0 @@ -This is a .mpy built against the current .mpy version that can be used to test -freezing without a dependency on mpy-cross. diff --git a/tests/import/import_star.py b/tests/import/import_star.py index 2cb21b877d7..dc34577d772 100644 --- a/tests/import/import_star.py +++ b/tests/import/import_star.py @@ -57,3 +57,15 @@ print("missed detection of incorrect __all__ definition") except TypeError as er: print("TypeError triggered for bad __all__ definition") + +# 6. test when package uses __getattr__ and raises AttributeError for __getattr__("__all__") +from pkgstar_getattr_attr_er import * + +print("publicFun3" in globals()) +print(publicFun3()) + +# 7. test when package uses __getattr__ and raises ValueError for __getattr__("__all__") +try: + from pkgstar_getattr_value_er import * +except ValueError as er: + print("ValueError triggered with args", er.args) diff --git a/tests/import/pkgstar_getattr_attr_er/__init__.py b/tests/import/pkgstar_getattr_attr_er/__init__.py new file mode 100644 index 00000000000..123463d3cd6 --- /dev/null +++ b/tests/import/pkgstar_getattr_attr_er/__init__.py @@ -0,0 +1,15 @@ +_print_msg = True + + +def publicFun3(): + return 3 + + +def __getattr__(attr): + global _print_msg + if _print_msg: + # CPython calls __getattr__("__all__") twice, but MicroPython only once. + # To make the output match, only print the message once. + print("__getattr__", attr) + _print_msg = False + raise AttributeError(attr) diff --git a/tests/import/pkgstar_getattr_value_er/__init__.py b/tests/import/pkgstar_getattr_value_er/__init__.py new file mode 100644 index 00000000000..43d6cdf250f --- /dev/null +++ b/tests/import/pkgstar_getattr_value_er/__init__.py @@ -0,0 +1,3 @@ +def __getattr__(attr): + print("__getattr__", attr) + raise ValueError(attr) diff --git a/tests/inlineasm/rv32/asm_ext_zcmp.py b/tests/inlineasm/rv32/asm_ext_zcmp.py new file mode 100644 index 00000000000..619478def7f --- /dev/null +++ b/tests/inlineasm/rv32/asm_ext_zcmp.py @@ -0,0 +1,264 @@ +CMMV_TEMPLATE = """ +@micropython.asm_rv32 +def t(): + cm_mv{}({}, {}) +""" + +CMMV_TESTS = ( + ("s0", "s9", False), + ("s0", "s0", False), + ("s1", "s1", False), + ("s9", "s10", False), + ("s0", "s1", True), + ("s1", "s2", True), + ("s2", "s3", True), + ("s3", "s4", True), +) + + +def cmmv_test(op, tests): + passed = True + for lhs, rhs, success in tests: + try: + exec(CMMV_TEMPLATE.format(op, lhs, rhs)) + if success is False: + print("cm.mv{} {} {} syntax fail".format(op, lhs, rhs)) + passed = False + except SyntaxError: + if success is True: + print("cm.mv{} {} {} syntax fail".format(op, lhs, rhs)) + passed = False + if passed: + print("cm.mv{} syntax pass".format(op)) + + +cmmv_test("a01s", CMMV_TESTS) +cmmv_test("sa01", CMMV_TESTS) + + +CMPP_TEMPLATE = """ +@micropython.asm_rv32 +def t(): + cm_push({}, {}) + cm_pop{}({}, {}) +""" + + +CMPP_TESTS = ( + ("", 0, 0, False), + ("ra", 0, 0, False), + ("ra, s0", 0, 0, False), + ("ra, s0-s1", 0, 0, False), + ("ra, s0-s10", 0, 0, False), + ("{}", 0, 0, False), + ("{t0}", 0, 0, False), + ("{s0}", 0, 0, False), + ("{s0-s1}", 0, 0, False), + ("{ra, s1-s0}", 0, 0, False), + ("{ra, s0-s10}", 0, 0, False), + ("{ra}", 16, -16, False), + ("{ra, s0}", -20, 20, False), + ("{ra}", 0, 0, True), + ("{ra, s0}", 0, 0, True), + ("{ra, s0-s1}", 0, 0, True), + ("{ra, s0-s5}", 0, 0, True), + ("{ra, s0-s11}", 0, 0, True), + ("{ra}", -16, 16, True), + ("{ra, s0}", -16, 16, True), + ("{ra, s0-s1}", -16, 16, True), + ("{ra}", -32, 32, True), + ("{ra, s0}", -32, 32, True), + ("{ra, s0-s1}", -32, 32, True), + ("{ra}", -48, 48, True), + ("{ra, s0}", -48, 48, True), + ("{ra, s0-s1}", -48, 48, True), +) + + +def cmpp_test(op, tests): + passed = True + for lhs, rhs_in, rhs_out, success in tests: + try: + exec(CMPP_TEMPLATE.format(lhs, rhs_in, op, lhs, rhs_out)) + if success is False: + print("cm.push/cm.pop{} {} {} {} syntax fail".format(op, lhs, rhs_in, rhs_out)) + passed = False + except SyntaxError: + if success is True: + print("cm.push/cm.pop{} {} {} {} syntax fail".format(op, lhs, rhs_in, rhs_out)) + passed = False + if passed: + print("cm.push/cm.pop{} syntax pass".format(op)) + + +cmpp_test("", CMPP_TESTS) +cmpp_test("ret", CMPP_TESTS) +cmpp_test("retz", CMPP_TESTS) + + +CM_POP_TEMPLATE = """ +@micropython.asm_rv32 +def t(a0): + mv(t0, s0) + mv(a1, a0) + mv(s0, a0) + cm_push({{ra, s0}}, {}) + add(s0, a1, a1) + cm_pop({{ra, s0}}, {}) + bne(s0, a0, fail) + li(a0, 1) + c_j(end) + label(fail) + li(a0, 0) + label(end) + mv(s0, t0) +print("cm.push/cm.pop {}", t(0x1234) == 1) +""" + +for stack_adj in 0, 16, 32, 48: + exec(CM_POP_TEMPLATE.format(-stack_adj, stack_adj, stack_adj)) + + +CM_POPRET_TEMPLATE = """ +@micropython.asm_rv32 +def t(a0): + mv(t0, ra) + mv(a1, s0) + jal(ra, clobber) + mv(ra, t0) + bne(s0, a1, fail) + li(a0, 1) + c_j(end) + label(fail) + li(a0, 0) + label(end) + c_jr(ra) + label(clobber) + cm_push({{ra, s0}}, {}) + mv(s0, a0) + cm_popret({{ra, s0}}, {}) +print("cm.push/cm.popret {}", t(0x1234) == 1) +""" + +for stack_adj in 0, 16, 32, 48: + exec(CM_POPRET_TEMPLATE.format(-stack_adj, stack_adj, stack_adj)) + + +CM_POPRETZ_TEMPLATE = """ +@micropython.asm_rv32 +def t(a0): + mv(t0, ra) + mv(a1, s0) + jal(ra, clobber) + mv(ra, t0) + bne(a0, zero, fail) + bne(s0, a1, fail) + li(a0, 1) + c_j(end) + label(fail) + li(a0, 0) + label(end) + c_jr(ra) + label(clobber) + cm_push({{ra, s0}}, {}) + mv(s0, a0) + cm_popretz({{ra, s0}}, {}) +print("cm.push/cm.popretz {}", t(0x1234) == 1) +""" + + +for stack_adj in 0, 16, 32, 48: + exec(CM_POPRETZ_TEMPLATE.format(-stack_adj, stack_adj, stack_adj)) + + +REGLIST_TEMPLATE = """ +@micropython.asm_rv32 +def t(): + li(t6, 0) +{save} + cm_push({reglist}, 0) +{trash} + cm_pop({reglist}, 0) +{compare} + c_j(restore) + label(fail) + li(t6, 1) + label(restore) +{restore} + mv(a0, t6) +print("reglist {reglist}", t() == 0) +""" + +REG_MAP = [ + ("ra", "a0", "{ra}"), + ("s0", "a1", "{ra,s0}"), + ("s1", "a2", "{ra,s0-s1}"), + ("s2", "a3", "{ra,s0-s2}"), + ("s3", "a4", "{ra,s0-s3}"), + ("s4", "a5", "{ra,s0-s4}"), + ("s5", "a6", "{ra,s0-s5}"), + ("s6", "a7", "{ra,s0-s6}"), + ("s7", "t0", "{ra,s0-s7}"), + ("s8", "t1", "{ra,s0-s8}"), + ("s9", "t2", "{ra,s0-s9}"), + ("s10", "t3", "{ra,s0-s11}"), + ("s11", "t4", "{ra,s0-s11}"), +] + +for i in range(len(REG_MAP)): + save = "" + trash = "" + compare = "" + restore = "" + _, _, reglist = REG_MAP[i] + for j in range(i + 1): + r1, r2, _ = REG_MAP[j] + save += " mv({}, {})\n".format(r2, r1) + trash += " li({}, {})\n".format(r1, (0x1111_1111 * (j + 1)) & 0xFFFF_FFFF) + compare += " bne({}, {}, fail)\n".format(r2, r1) + restore += " mv({}, {})\n".format(r1, r2) + exec( + REGLIST_TEMPLATE.format( + save=save, trash=trash, compare=compare, restore=restore, reglist=reglist + ) + ) + + +@micropython.asm_rv32 +def test_cm_mva01s(a0, a1): + cm_push({ra, s0 - s11}, 0) + mv(s4, a0) + mv(s5, a1) + li(a0, 0) + li(a1, 0) + cm_mva01s(s4, s5) + bne(a0, s4, fail) + bne(a1, s5, fail) + li(a0, 1) + c_j(end) + label(fail) + li(a0, 0) + label(end) + cm_pop({ra, s0 - s11}, 0) + + +print("cm.mva01s", test_cm_mva01s(100, 200) == 1) + + +@micropython.asm_rv32 +def test_cm_mvsa01(a0, a1): + cm_push({ra, s0 - s11}, 0) + li(s6, 0x12345678) + li(s7, 0x87654321) + cm_mvsa01(s6, s7) + bne(a0, s6, fail) + bne(a1, s7, fail) + li(a0, 1) + c_j(end) + label(fail) + li(a0, 0) + label(end) + cm_pop({ra, s0 - s11}, 0) + + +print("cm.mvsa01", test_cm_mvsa01(100, 200) == 1) diff --git a/tests/inlineasm/rv32/asm_ext_zcmp.py.exp b/tests/inlineasm/rv32/asm_ext_zcmp.py.exp new file mode 100644 index 00000000000..62850dfff4a --- /dev/null +++ b/tests/inlineasm/rv32/asm_ext_zcmp.py.exp @@ -0,0 +1,32 @@ +cm.mva01s syntax pass +cm.mvsa01 syntax pass +cm.push/cm.pop syntax pass +cm.push/cm.popret syntax pass +cm.push/cm.popretz syntax pass +cm.push/cm.pop 0 True +cm.push/cm.pop 16 True +cm.push/cm.pop 32 True +cm.push/cm.pop 48 True +cm.push/cm.popret 0 True +cm.push/cm.popret 16 True +cm.push/cm.popret 32 True +cm.push/cm.popret 48 True +cm.push/cm.popretz 0 True +cm.push/cm.popretz 16 True +cm.push/cm.popretz 32 True +cm.push/cm.popretz 48 True +reglist {ra} True +reglist {ra,s0} True +reglist {ra,s0-s1} True +reglist {ra,s0-s2} True +reglist {ra,s0-s3} True +reglist {ra,s0-s4} True +reglist {ra,s0-s5} True +reglist {ra,s0-s6} True +reglist {ra,s0-s7} True +reglist {ra,s0-s8} True +reglist {ra,s0-s9} True +reglist {ra,s0-s11} True +reglist {ra,s0-s11} True +cm.mva01s True +cm.mvsa01 True diff --git a/tests/inlineasm/thumb/asm_thumb2_bcc.py b/tests/inlineasm/thumb/asm_thumb2_bcc.py new file mode 100644 index 00000000000..bf0ee0a48fe --- /dev/null +++ b/tests/inlineasm/thumb/asm_thumb2_bcc.py @@ -0,0 +1,65 @@ +# test bcc instructions, narrow and wide versions + + +RESULT = [] + + +TEMPLATE = """ +@micropython.asm_thumb +def t(r0): + mov(r1, r0) + + mov(r0, 10) + cmp(r1, 1) + b{}(next1) + + b(end) + + label(next1) + + mov(r0, 20) + cmp(r1, 2) + b{}_n(next2) + + b(end) + + label(next2) + mov(r0, 30) + cmp(r1, 3) + b{}_w(next3) + + b(end) + + label(next3) + mov(r0, 0) + + label(end) +""" + +try: + for code in ( + "eq", + "ne", + "cs", + "cc", + "mi", + "pl", + "vs", + "vc", + "hi", + "ls", + "ge", + "lt", + "gt", + "le", + ): + exec(TEMPLATE.format(code, code, code)) + RESULT.append("B" + code.upper()) + for i in range(4): + RESULT.append(t(i)) +except MemoryError: + print("SKIP-TOO-LARGE") + raise SystemExit + +for line in RESULT: + print(line) diff --git a/tests/inlineasm/thumb/asm_thumb2_bcc.py.exp b/tests/inlineasm/thumb/asm_thumb2_bcc.py.exp new file mode 100644 index 00000000000..06fd057ae52 --- /dev/null +++ b/tests/inlineasm/thumb/asm_thumb2_bcc.py.exp @@ -0,0 +1,70 @@ +BEQ +10 +20 +10 +10 +BNE +0 +10 +20 +30 +BCS +10 +20 +30 +0 +BCC +0 +10 +10 +10 +BMI +0 +10 +10 +10 +BPL +10 +20 +30 +0 +BVS +10 +10 +10 +10 +BVC +0 +0 +0 +0 +BHI +10 +10 +20 +30 +BLS +0 +0 +10 +10 +BGE +10 +20 +30 +0 +BLT +0 +10 +10 +10 +BGT +10 +10 +20 +30 +BLE +0 +0 +10 +10 diff --git a/tests/inlineasm/thumb/asmbitops.py b/tests/inlineasm/thumb/asm_thumb2_bitops.py similarity index 100% rename from tests/inlineasm/thumb/asmbitops.py rename to tests/inlineasm/thumb/asm_thumb2_bitops.py diff --git a/tests/inlineasm/thumb/asmbitops.py.exp b/tests/inlineasm/thumb/asm_thumb2_bitops.py.exp similarity index 100% rename from tests/inlineasm/thumb/asmbitops.py.exp rename to tests/inlineasm/thumb/asm_thumb2_bitops.py.exp diff --git a/tests/inlineasm/thumb/asmconst.py b/tests/inlineasm/thumb/asm_thumb2_const.py similarity index 100% rename from tests/inlineasm/thumb/asmconst.py rename to tests/inlineasm/thumb/asm_thumb2_const.py diff --git a/tests/inlineasm/thumb/asmconst.py.exp b/tests/inlineasm/thumb/asm_thumb2_const.py.exp similarity index 100% rename from tests/inlineasm/thumb/asmconst.py.exp rename to tests/inlineasm/thumb/asm_thumb2_const.py.exp diff --git a/tests/inlineasm/thumb/asmdiv.py b/tests/inlineasm/thumb/asm_thumb2_div.py similarity index 100% rename from tests/inlineasm/thumb/asmdiv.py rename to tests/inlineasm/thumb/asm_thumb2_div.py diff --git a/tests/inlineasm/thumb/asmdiv.py.exp b/tests/inlineasm/thumb/asm_thumb2_div.py.exp similarity index 100% rename from tests/inlineasm/thumb/asmdiv.py.exp rename to tests/inlineasm/thumb/asm_thumb2_div.py.exp diff --git a/tests/inlineasm/thumb/asmit.py b/tests/inlineasm/thumb/asm_thumb2_it.py similarity index 100% rename from tests/inlineasm/thumb/asmit.py rename to tests/inlineasm/thumb/asm_thumb2_it.py diff --git a/tests/inlineasm/thumb/asmit.py.exp b/tests/inlineasm/thumb/asm_thumb2_it.py.exp similarity index 100% rename from tests/inlineasm/thumb/asmit.py.exp rename to tests/inlineasm/thumb/asm_thumb2_it.py.exp diff --git a/tests/inlineasm/thumb/asmspecialregs.py b/tests/inlineasm/thumb/asm_thumb2_specialregs.py similarity index 100% rename from tests/inlineasm/thumb/asmspecialregs.py rename to tests/inlineasm/thumb/asm_thumb2_specialregs.py diff --git a/tests/inlineasm/thumb/asmspecialregs.py.exp b/tests/inlineasm/thumb/asm_thumb2_specialregs.py.exp similarity index 100% rename from tests/inlineasm/thumb/asmspecialregs.py.exp rename to tests/inlineasm/thumb/asm_thumb2_specialregs.py.exp diff --git a/tests/inlineasm/thumb/asmfpaddsub.py b/tests/inlineasm/thumb/asm_vfp_addsub.py similarity index 100% rename from tests/inlineasm/thumb/asmfpaddsub.py rename to tests/inlineasm/thumb/asm_vfp_addsub.py diff --git a/tests/inlineasm/thumb/asmfpaddsub.py.exp b/tests/inlineasm/thumb/asm_vfp_addsub.py.exp similarity index 100% rename from tests/inlineasm/thumb/asmfpaddsub.py.exp rename to tests/inlineasm/thumb/asm_vfp_addsub.py.exp diff --git a/tests/inlineasm/thumb/asmfpcmp.py b/tests/inlineasm/thumb/asm_vfp_cmp.py similarity index 100% rename from tests/inlineasm/thumb/asmfpcmp.py rename to tests/inlineasm/thumb/asm_vfp_cmp.py diff --git a/tests/inlineasm/thumb/asmfpcmp.py.exp b/tests/inlineasm/thumb/asm_vfp_cmp.py.exp similarity index 100% rename from tests/inlineasm/thumb/asmfpcmp.py.exp rename to tests/inlineasm/thumb/asm_vfp_cmp.py.exp diff --git a/tests/inlineasm/thumb/asmfpldrstr.py b/tests/inlineasm/thumb/asm_vfp_ldrstr.py similarity index 100% rename from tests/inlineasm/thumb/asmfpldrstr.py rename to tests/inlineasm/thumb/asm_vfp_ldrstr.py diff --git a/tests/inlineasm/thumb/asmfpldrstr.py.exp b/tests/inlineasm/thumb/asm_vfp_ldrstr.py.exp similarity index 100% rename from tests/inlineasm/thumb/asmfpldrstr.py.exp rename to tests/inlineasm/thumb/asm_vfp_ldrstr.py.exp diff --git a/tests/inlineasm/thumb/asmfpmuldiv.py b/tests/inlineasm/thumb/asm_vfp_muldiv.py similarity index 100% rename from tests/inlineasm/thumb/asmfpmuldiv.py rename to tests/inlineasm/thumb/asm_vfp_muldiv.py diff --git a/tests/inlineasm/thumb/asmfpmuldiv.py.exp b/tests/inlineasm/thumb/asm_vfp_muldiv.py.exp similarity index 100% rename from tests/inlineasm/thumb/asmfpmuldiv.py.exp rename to tests/inlineasm/thumb/asm_vfp_muldiv.py.exp diff --git a/tests/inlineasm/thumb/asmfpsqrt.py b/tests/inlineasm/thumb/asm_vfp_sqrt.py similarity index 100% rename from tests/inlineasm/thumb/asmfpsqrt.py rename to tests/inlineasm/thumb/asm_vfp_sqrt.py diff --git a/tests/inlineasm/thumb/asmfpsqrt.py.exp b/tests/inlineasm/thumb/asm_vfp_sqrt.py.exp similarity index 100% rename from tests/inlineasm/thumb/asmfpsqrt.py.exp rename to tests/inlineasm/thumb/asm_vfp_sqrt.py.exp diff --git a/tests/inlineasm/thumb/asmbcc.py b/tests/inlineasm/thumb/asmbcc.py deleted file mode 100644 index 08967d48c74..00000000000 --- a/tests/inlineasm/thumb/asmbcc.py +++ /dev/null @@ -1,29 +0,0 @@ -# test bcc instructions -# at the moment only tests beq, narrow and wide versions - - -@micropython.asm_thumb -def f(r0): - mov(r1, r0) - - mov(r0, 10) - cmp(r1, 1) - beq(end) - - mov(r0, 20) - cmp(r1, 2) - beq_n(end) - - mov(r0, 30) - cmp(r1, 3) - beq_w(end) - - mov(r0, 0) - - label(end) - - -print(f(0)) -print(f(1)) -print(f(2)) -print(f(3)) diff --git a/tests/inlineasm/thumb/asmbcc.py.exp b/tests/inlineasm/thumb/asmbcc.py.exp deleted file mode 100644 index 39da7d1a99e..00000000000 --- a/tests/inlineasm/thumb/asmbcc.py.exp +++ /dev/null @@ -1,4 +0,0 @@ -0 -10 -20 -30 diff --git a/tests/micropython/const_annotated.py b/tests/micropython/const_annotated.py new file mode 100644 index 00000000000..4d01389e43a --- /dev/null +++ b/tests/micropython/const_annotated.py @@ -0,0 +1,18 @@ +# Test type annotations in combination with const. +# This test will only work when MICROPY_COMP_CONST and MICROPY_COMP_CONST_TUPLE are enabled. + +from micropython import const + +_X0: bool = const(True) +_X1: int = const(123) +_X2: str = const("test") +_X3: tuple = const((1, 2)) +_X4: tuple[bool, int] = const((True, 4)) +_X5: bytes = b"\x01\x02\x03" + +print(_X0) +print(_X1) +print(_X2) +print(_X3) +print(_X4) +print(_X5) diff --git a/tests/micropython/const_annotated.py.exp b/tests/micropython/const_annotated.py.exp new file mode 100644 index 00000000000..65edb58e72e --- /dev/null +++ b/tests/micropython/const_annotated.py.exp @@ -0,0 +1,6 @@ +True +123 +test +(1, 2) +(True, 4) +b'\x01\x02\x03' diff --git a/tests/micropython/heapalloc_traceback.py.native.exp b/tests/micropython/heapalloc_traceback.py.native.exp index d6ac26aa829..851eb5c7806 100644 --- a/tests/micropython/heapalloc_traceback.py.native.exp +++ b/tests/micropython/heapalloc_traceback.py.native.exp @@ -1,3 +1,3 @@ StopIteration -StopIteration: - +'StopIteration: ' +'' diff --git a/tests/micropython/import_mpy_native.py b/tests/micropython/import_mpy_native.py index 59181b203bb..84cf198b278 100644 --- a/tests/micropython/import_mpy_native.py +++ b/tests/micropython/import_mpy_native.py @@ -124,7 +124,7 @@ def open(self, path, mode): __import__(mod) print(mod, "OK") except ValueError as er: - print(mod, "ValueError", er) + print(mod, "ValueError", str(er) or "incompatible .mpy arch") # unmount and undo path addition vfs.umount("/userfs") diff --git a/tests/micropython/io_badlength.py b/tests/micropython/io_badlength.py new file mode 100644 index 00000000000..646a2cd9241 --- /dev/null +++ b/tests/micropython/io_badlength.py @@ -0,0 +1,35 @@ +# Test when a use IOBase class has write/readinto which returns more data than +# requested (https://github.com/micropython/micropython/issues/18845) + +try: + import io, json +except: + print("SKIP") + raise SystemExit + + +class S(io.IOBase): + def write(self, buf): + assert len(buf) >= 0 + return 2 + + def ioctl(self, cmd, arg): + return 0 + + def readinto(self, buf): + assert len(buf) >= 0 + return 3 + + +try: + print("abc", file=S()) + print("write OK") +except OSError as e: + print("write failed, errno", e.errno) + +buf = bytearray(1) +try: + json.load(S()) + print("readinto OK") +except OSError as e: + print("read failed, errno", e.errno) diff --git a/tests/micropython/io_badlength.py.exp b/tests/micropython/io_badlength.py.exp new file mode 100644 index 00000000000..bba660dd3c8 --- /dev/null +++ b/tests/micropython/io_badlength.py.exp @@ -0,0 +1,2 @@ +write failed, errno 5 +read failed, errno 5 diff --git a/tests/micropython/native_with.py b/tests/micropython/native_with.py index 9c0b98af903..37c385a42f9 100644 --- a/tests/micropython/native_with.py +++ b/tests/micropython/native_with.py @@ -9,6 +9,7 @@ def __enter__(self): print("__enter__") def __exit__(self, a, b, c): + b = repr(b)[:10] # shorten exception to "NameError(" prefix for target compatibility print("__exit__", a, b, c) diff --git a/tests/micropython/native_with.py.exp b/tests/micropython/native_with.py.exp index 7e28663f6fc..b8083a4c66b 100644 --- a/tests/micropython/native_with.py.exp +++ b/tests/micropython/native_with.py.exp @@ -5,5 +5,5 @@ __exit__ None None None __init__ __enter__ 1 -__exit__ name 'fail' isn't defined None +__exit__ NameError( None NameError diff --git a/tests/micropython/ringio_big.py b/tests/micropython/ringio_big.py index ddbbae12a63..88d5d2cc165 100644 --- a/tests/micropython/ringio_big.py +++ b/tests/micropython/ringio_big.py @@ -8,22 +8,33 @@ print("SKIP") raise SystemExit +results = [] + try: - # The maximum possible size - micropython.RingIO(bytearray(65535)) + # The maximum possible size passed as an integer. micropython.RingIO(65534) try: - # Buffer may not be too big - micropython.RingIO(bytearray(65536)) + # Size may not be too big + micropython.RingIO(65535) except ValueError as ex: - print(type(ex)) + results.append(type(ex)) + + # Allocate a buffer for use below. + buf_64k = memoryview(bytearray(65536)) + + # The maximum possible size passed as a buffer. + micropython.RingIO(buf_64k[:-1]) try: - # Size may not be too big - micropython.RingIO(65535) + # Buffer may not be too big + micropython.RingIO(buf_64k) except ValueError as ex: - print(type(ex)) + results.append(type(ex)) + except MemoryError: print("SKIP") raise SystemExit + +for result in results: + print(result) diff --git a/tests/micropython/schedule_kbd_intr.py b/tests/micropython/schedule_kbd_intr.py new file mode 100644 index 00000000000..ec7ef5a5d07 --- /dev/null +++ b/tests/micropython/schedule_kbd_intr.py @@ -0,0 +1,72 @@ +# Test using micropython.schedule() to schedule a KeyboardInterrupt. +# Note: this test cannot use unittest because the unittest overhead +# may trigger the scheduler when we don't want it to. + +try: + from time import sleep + from micropython import schedule, kbd_intr +except ImportError: + print("SKIP") + raise SystemExit + +######################################################################## +# Test basic scheduling. + +try: + schedule(kbd_intr, None) +except ValueError: + # Scheduling `kbd_intr` is not supported on this target. + print("SKIP") + raise SystemExit + +try: + sleep(0) + sleep(0) + print("fail") +except KeyboardInterrupt: + print("KeyboardInterrupt") + +######################################################################## +# Should be able to schedule it many times without error, and that +# should only trigger one KeyboardInterrupt. + +schedule(kbd_intr, None) +schedule(kbd_intr, None) +schedule(kbd_intr, None) +schedule(kbd_intr, None) +schedule(kbd_intr, None) +schedule(kbd_intr, None) +schedule(kbd_intr, None) +schedule(kbd_intr, None) +schedule(kbd_intr, None) +schedule(kbd_intr, None) +try: + sleep(0) + sleep(0) + print("fail") +except KeyboardInterrupt: + print("KeyboardInterrupt") + +# This should not raise. +for _ in range(100): + sleep(0) + +######################################################################## +# Test nested scheduling. + + +def callback(_): + schedule(kbd_intr, None) + # This should not raise because we are in schedule context. + for _ in range(100): + sleep(0) + + +schedule(callback, None) +sleep(0) +try: + sleep(0) + sleep(0) + print("fail") +except KeyboardInterrupt: + print("KeyboardInterrupt") diff --git a/tests/micropython/schedule_kbd_intr.py.exp b/tests/micropython/schedule_kbd_intr.py.exp new file mode 100644 index 00000000000..c4ba7730c52 --- /dev/null +++ b/tests/micropython/schedule_kbd_intr.py.exp @@ -0,0 +1,3 @@ +KeyboardInterrupt +KeyboardInterrupt +KeyboardInterrupt diff --git a/tests/micropython/viper_error.py b/tests/micropython/viper_error.py index 6c5c3ba2007..e58a3c59fa0 100644 --- a/tests/micropython/viper_error.py +++ b/tests/micropython/viper_error.py @@ -1,19 +1,19 @@ # test syntax and type errors specific to viper code generation -def test(code): +def test(code, msg): try: exec(code) except (SyntaxError, ViperTypeError, NotImplementedError) as e: - print(repr(e)) + print(type(e), str(e) or msg) # viper: annotations must be identifiers -test("@micropython.viper\ndef f(a:1): pass") -test("@micropython.viper\ndef f() -> 1: pass") +test("@micropython.viper\ndef f(a:1): pass", "annotation must be an identifier") +test("@micropython.viper\ndef f() -> 1: pass", "annotation must be an identifier") # unknown type -test("@micropython.viper\ndef f(x:unknown_type): pass") +test("@micropython.viper\ndef f(x:unknown_type): pass", "unknown type 'unknown_type'") # local used before type known test( @@ -22,7 +22,8 @@ def test(code): def f(): print(x) x = 1 -""" +""", + "local 'x' used before type known", ) # type mismatch storing to local @@ -33,7 +34,8 @@ def f(): x = 1 y = [] x = y -""" +""", + "local 'x' has type 'int' but source is 'object'", ) # can't implicitly convert type to bool @@ -44,47 +46,52 @@ def f(): x = ptr(0) if x: pass -""" +""", + "can't implicitly convert 'ptr' to 'bool'", ) # incorrect return type -test("@micropython.viper\ndef f() -> int: return []") +test("@micropython.viper\ndef f() -> int: return []", "return expected 'int' but got 'object'") # can't do unary op of incompatible type -test("@micropython.viper\ndef f(x:ptr): -x") +test("@micropython.viper\ndef f(x:ptr): -x", "can't do unary op of 'ptr'") # can't do binary op between incompatible types -test("@micropython.viper\ndef f(): 1 + []") -test("@micropython.viper\ndef f(x:int, y:uint): x < y") +test("@micropython.viper\ndef f(): 1 + []", "can't do binary op between 'int' and 'object'") +test("@micropython.viper\ndef f(x:int, y:uint): x < y", "comparison of int and uint") # can't load -test("@micropython.viper\ndef f(): 1[0]") -test("@micropython.viper\ndef f(): 1[x]") +test("@micropython.viper\ndef f(): 1[0]", "can't load from 'int'") +test("@micropython.viper\ndef f(): 1[x]", "can't load from 'int'") # can't store -test("@micropython.viper\ndef f(): 1[0] = 1") -test("@micropython.viper\ndef f(): 1[x] = 1") -test("@micropython.viper\ndef f(x:int): x[0] = x") -test("@micropython.viper\ndef f(x:ptr32): x[0] = None") -test("@micropython.viper\ndef f(x:ptr32): x[x] = None") +test("@micropython.viper\ndef f(): 1[0] = 1", "can't store to 'int'") +test("@micropython.viper\ndef f(): 1[x] = 1", "can't store to 'int'") +test("@micropython.viper\ndef f(x:int): x[0] = x", "can't store to 'int'") +test("@micropython.viper\ndef f(x:ptr32): x[0] = None", "can't store 'None'") +test("@micropython.viper\ndef f(x:ptr32): x[x] = None", "can't store 'None'") # must raise an object -test("@micropython.viper\ndef f(): raise 1") +test("@micropython.viper\ndef f(): raise 1", "must raise an object") # unary ops not implemented -test("@micropython.viper\ndef f(x:int): not x") +test("@micropython.viper\ndef f(x:int): not x", "'not' not implemented") # binary op not implemented -test("@micropython.viper\ndef f(x:uint, y:uint): res = x // y") -test("@micropython.viper\ndef f(x:uint, y:uint): res = x % y") -test("@micropython.viper\ndef f(x:int): res = x in x") +test("@micropython.viper\ndef f(x:uint, y:uint): res = x // y", "div/mod not implemented for uint") +test("@micropython.viper\ndef f(x:uint, y:uint): res = x % y", "div/mod not implemented for uint") +test("@micropython.viper\ndef f(x:int): res = x in x", "binary op not implemented") + +# raise with 0 or 2 args not implemented +test("@micropython.viper\ndef f():\n try:\n x\n except:\n raise\n", "native raise") +test("@micropython.viper\ndef f(): raise Exception from Exception", "native raise") # yield (from) not implemented -test("@micropython.viper\ndef f(): yield") -test("@micropython.viper\ndef f(): yield from f") +test("@micropython.viper\ndef f(): yield", "native yield") +test("@micropython.viper\ndef f(): yield from f", "native yield") # passing a ptr to a Python function not implemented -test("@micropython.viper\ndef f(): print(ptr(1))") +test("@micropython.viper\ndef f(): print(ptr(1))", "conversion to object") # cast of a casting identifier not implemented -test("@micropython.viper\ndef f(): int(int)") +test("@micropython.viper\ndef f(): int(int)", "casting") diff --git a/tests/micropython/viper_error.py.exp b/tests/micropython/viper_error.py.exp index 51cbd6c7097..9b663b8508a 100644 --- a/tests/micropython/viper_error.py.exp +++ b/tests/micropython/viper_error.py.exp @@ -1,26 +1,28 @@ -SyntaxError('annotation must be an identifier',) -SyntaxError('annotation must be an identifier',) -ViperTypeError("unknown type 'unknown_type'",) -ViperTypeError("local 'x' used before type known",) -ViperTypeError("local 'x' has type 'int' but source is 'object'",) -ViperTypeError("can't implicitly convert 'ptr' to 'bool'",) -ViperTypeError("return expected 'int' but got 'object'",) -ViperTypeError("can't do unary op of 'ptr'",) -ViperTypeError("can't do binary op between 'int' and 'object'",) -ViperTypeError('comparison of int and uint',) -ViperTypeError("can't load from 'int'",) -ViperTypeError("can't load from 'int'",) -ViperTypeError("can't store to 'int'",) -ViperTypeError("can't store to 'int'",) -ViperTypeError("can't store to 'int'",) -ViperTypeError("can't store 'None'",) -ViperTypeError("can't store 'None'",) -ViperTypeError('must raise an object',) -ViperTypeError("'not' not implemented",) -ViperTypeError('div/mod not implemented for uint',) -ViperTypeError('div/mod not implemented for uint',) -ViperTypeError('binary op not implemented',) -NotImplementedError('native yield',) -NotImplementedError('native yield',) -NotImplementedError('conversion to object',) -NotImplementedError('casting',) + annotation must be an identifier + annotation must be an identifier + unknown type 'unknown_type' + local 'x' used before type known + local 'x' has type 'int' but source is 'object' + can't implicitly convert 'ptr' to 'bool' + return expected 'int' but got 'object' + can't do unary op of 'ptr' + can't do binary op between 'int' and 'object' + comparison of int and uint + can't load from 'int' + can't load from 'int' + can't store to 'int' + can't store to 'int' + can't store to 'int' + can't store 'None' + can't store 'None' + must raise an object + 'not' not implemented + div/mod not implemented for uint + div/mod not implemented for uint + binary op not implemented + native raise + native raise + native yield + native yield + conversion to object + casting diff --git a/tests/micropython/viper_unop.py b/tests/micropython/viper_unop.py index 61cbd5125f1..80c247a581e 100644 --- a/tests/micropython/viper_unop.py +++ b/tests/micropython/viper_unop.py @@ -29,3 +29,36 @@ def inv(x: int) -> int: print(inv(0)) print(inv(1)) print(inv(-2)) + + +@micropython.viper +def posc() -> int: + x = -1 + x1 = +x + assert x == -1 + return x1 + + +print(posc()) + + +@micropython.viper +def negc() -> int: + x = -1 + x1 = -x + assert x == -1 + return x1 + + +print(negc()) + + +@micropython.viper +def invc() -> int: + x = -1 + x1 = ~x + assert x == -1 + return x1 + + +print(invc()) diff --git a/tests/micropython/viper_unop.py.exp b/tests/micropython/viper_unop.py.exp index 6d93312caa1..9e9a5f4715f 100644 --- a/tests/micropython/viper_unop.py.exp +++ b/tests/micropython/viper_unop.py.exp @@ -7,3 +7,6 @@ -1 -2 1 +-1 +1 +0 diff --git a/tests/micropython/viper_with.py b/tests/micropython/viper_with.py index 40fbf6fb315..833a6babe7d 100644 --- a/tests/micropython/viper_with.py +++ b/tests/micropython/viper_with.py @@ -9,6 +9,7 @@ def __enter__(self): print("__enter__") def __exit__(self, a, b, c): + b = repr(b)[:10] # shorten exception to "NameError(" prefix for target compatibility print("__exit__", a, b, c) diff --git a/tests/micropython/viper_with.py.exp b/tests/micropython/viper_with.py.exp index 7e28663f6fc..b8083a4c66b 100644 --- a/tests/micropython/viper_with.py.exp +++ b/tests/micropython/viper_with.py.exp @@ -5,5 +5,5 @@ __exit__ None None None __init__ __enter__ 1 -__exit__ name 'fail' isn't defined None +__exit__ NameError( None NameError diff --git a/tests/misc/cexample_subclass.py b/tests/misc/cexample_subclass.py index 9f52a2c737a..6857788e5dc 100644 --- a/tests/misc/cexample_subclass.py +++ b/tests/misc/cexample_subclass.py @@ -2,6 +2,7 @@ try: from cexample import AdvancedTimer + import time # used to skip this test on minimal unix variant except ImportError: print("SKIP") raise SystemExit diff --git a/tests/misc/features.py b/tests/misc/features.py index 455b44fb1ef..3db88118144 100644 --- a/tests/misc/features.py +++ b/tests/misc/features.py @@ -103,7 +103,7 @@ class c25: x = c25() print(x.x) - raise + raise Exception except: print(26) print(27 + zero) diff --git a/tests/misc/rge_sm.py b/tests/misc/rge_sm.py index 33aa4edb742..3c30817952f 100644 --- a/tests/misc/rge_sm.py +++ b/tests/misc/rge_sm.py @@ -82,7 +82,11 @@ def singleTraj(system, trajStart, h=0.02, tend=1.0): # compute the trajectory rk = RungeKutta(system, trajStart, tstart, h) - rk.solve(tend) + try: + rk.solve(tend) + except MemoryError: + print("SKIP") + raise SystemExit # print out trajectory diff --git a/tests/multi_extmod/machine_can_04_tx_order.py b/tests/multi_extmod/machine_can_04_tx_order.py index 204bdafd59d..9efd9088c94 100644 --- a/tests/multi_extmod/machine_can_04_tx_order.py +++ b/tests/multi_extmod/machine_can_04_tx_order.py @@ -1,9 +1,19 @@ from machine import CAN import time from random import seed, randrange +import sys import micropython +# The Alif port has an opaque send queue. The Alif CAN controller +# provides no information about the slot number where the message +# is stored, and it does not allow to cancel specific messages. +# This test needs the slot number, which is not available. + +if "alif" in sys.platform: + print("SKIP") + raise SystemExit + micropython.alloc_emergency_exception_buf(256) seed(0) @@ -103,7 +113,10 @@ def irq_send(can): def instance1(): # note: this test can pass with hard=True, but in a debug build # the completion IRQ may race ahead of setting tx_queue[idx], below - can.irq(irq_send, trigger=can.IRQ_TX, hard=False) + if "mimxrt" in sys.platform: + can.irq(irq_send, trigger=can.IRQ_TX, hard=True) + else: + can.irq(irq_send, trigger=can.IRQ_TX, hard=False) data = bytearray(MSG_LEN) multitest.next() diff --git a/tests/multi_extmod/machine_can_05_tx_prio_cancel.py b/tests/multi_extmod/machine_can_05_tx_prio_cancel.py index 64756a1a1af..c3f67b372df 100644 --- a/tests/multi_extmod/machine_can_05_tx_prio_cancel.py +++ b/tests/multi_extmod/machine_can_05_tx_prio_cancel.py @@ -1,5 +1,16 @@ from machine import CAN import time +import sys + +# The Alif port has an opaque send queue. The Alif CAN controller +# provides no information about the slot number where the message +# is stored, and it does not allow to cancel specific messages. +# This test needs both the slot number, and uses cancel_send() for +# dedicated messages, which both is not available. + +if "alif" in sys.platform: + print("SKIP") + raise SystemExit # Check that cancelling a low priority outgoing message and replacing it with a # high priority message causes it to be transmitted successfully onto a busy bus @@ -26,6 +37,14 @@ def instance0(): multitest.next() + # don't babble until we know instance1 is ready to receive, or this instance + # may go to Error Passive while instance1 is still initialising CAN (meaning + # the "babble" won't saturate the bus, due to the Suspend Transmission + # requirement) + multitest.wait("instance1 ready") + + bcast_countdown = 5 + # "Babble" medium priority messages onto the bus to prevent # instance1() from sending anything lower priority than this while len(recv) < ITERS: @@ -33,6 +52,12 @@ def instance0(): can.send(id, b"BABBLE", CAN.FLAG_EXT_ID) if len(recv) >= ITERS: break + if bcast_countdown > 0: + # queue some "babble" messages onto the bus before signalling to + # instance1 that it can start trying to send + bcast_countdown -= 1 + if not bcast_countdown: + multitest.broadcast("instance0 babbling") print("received", ITERS, "messages") for can_id in recv: @@ -66,10 +91,16 @@ def irq_send(can): def instance1(): - global last_idx + global last_idx, total_cancels can.irq(irq_send, trigger=can.IRQ_TX, hard=True) multitest.next() + multitest.broadcast("instance1 ready") + + # make sure instance0 can queue outgoing medium-priority + # babble before we start trying to send, so we're trying to + # send onto an already busy bus + multitest.wait("instance0 babbling") for i in range(ITERS): # Fill the transmit queue with low priority messages (all extended IDs) @@ -95,6 +126,8 @@ def instance1(): # try and cancel the last message we queued res = can.cancel_send(last_idx) print(i, "cancel result", res) + if ("mimxrt" in sys.platform) and res: + total_cancels += 1 # send a high priority message, that we expect to go out idx = can.send(0x500 + i, b"HIPRIO", CAN.FLAG_EXT_ID) diff --git a/tests/multi_extmod/machine_can_07_error_states.py b/tests/multi_extmod/machine_can_07_error_states.py index e7eec513ee1..f7071a8bd77 100644 --- a/tests/multi_extmod/machine_can_07_error_states.py +++ b/tests/multi_extmod/machine_can_07_error_states.py @@ -108,6 +108,8 @@ def instance1(): # Send a single message to the receiver, to verify it's working can.send(_ID, b"PAYLOAD") + time.sleep_ms(50) # irq_sender should fire during this window + active_counters = can.get_counters() # print(active_counters) # DEBUG @@ -189,15 +191,19 @@ def instance1(): # with this test setup, as Bus Off requires more than just "normal" frame # transmit errors. - # restarting the controller may cause it to leave its error state, or not, depending - # on the implementation - but it shouldn't cause any recovery issues. Also cancels all pending TX - # (note: have to do this before 'fix baud' or we create a race condition for pending tx) - can.restart() - - # tell the receiver to go back to a valid baud rate + # tell the receiver to go back to a valid baud rate, causing the pending tx + # to be sent. multitest.broadcast("fix baud") multitest.wait("fixed baud") + # Restarting the controller may cause it to leave its error state, or not, depending + # on the implementation - but it shouldn't cause any recovery issues. Also cancels all pending TX. + # If restart() clears the TEC and REC error counters, resetting the error state to ACTIVE, + # and if that is done while instance0 is still trying to send at the wrong baud rate, + # the error state rushes up again pretty fast. Therefore the baud rate is fixed before + # calling restart(). + can.restart() + idx_more = can.send(_ID, b"MOREMORE") time.sleep_ms(50) # irq_sender should fire during this window print("queued moremore", idx_more is not None) diff --git a/tests/multi_extmod/machine_can_07_error_states.py.exp b/tests/multi_extmod/machine_can_07_error_states.py.exp index 158ae63bfbc..c2731f20905 100644 --- a/tests/multi_extmod/machine_can_07_error_states.py.exp +++ b/tests/multi_extmod/machine_can_07_error_states.py.exp @@ -33,5 +33,6 @@ one over thresh True no new warning True counted passive True irq sent True +irq sent True queued moremore True done diff --git a/tests/multi_extmod/machine_can_08_init_mode.py b/tests/multi_extmod/machine_can_08_init_mode.py index 459e6180ab6..730edfac7e5 100644 --- a/tests/multi_extmod/machine_can_08_init_mode.py +++ b/tests/multi_extmod/machine_can_08_init_mode.py @@ -1,6 +1,7 @@ from machine import CAN from micropython import const import time +import sys # instance0 transitions through various modes, instance1 # listens for various messages (or not) @@ -29,7 +30,7 @@ def irq_print(can): print("recv", hex(can_id), bytes(data)) else: silent_rx_count += 1 - if flags & can.IRQ_TX: # note: only enabled on instance1 to avoid race conditions + elif flags & can.IRQ_TX: # note: only enabled on instance1 to avoid race conditions print("send", "failed" if flags & can.IRQ_TX_FAILED else "ok") @@ -66,8 +67,14 @@ def instance0(): multitest.broadcast("silent") multitest.wait("silent done") # we should have received the message from instance1 many times, as instance0 won't have ACKed it - print("silent_rx_count", silent_rx_count > 5) - can.cancel_send(idx) + # create a dummy "OK" for MIMXRT, since it on receives ACKed messages in SILENT mode. + if ("mimxrt" in sys.platform) or ("alif" in sys.platform): + print("silent_rx_count True") + else: + print("silent_rx_count", silent_rx_count > 5) + # Cancel the sent message only if it was accepted for sending. + if idx is not None: + can.cancel_send(idx) reinit_with_mode(can.MODE_SILENT_LOOPBACK) print("Silent Loopback", "MODE_SILENT_LOOPBACK" in str(can)) @@ -97,6 +104,9 @@ def instance1(): idx = can.send(0x53, b"Silent2") time.sleep_ms(20) can.cancel_send(idx) + # mimxrt does not give TX failed interrupts on cancel_send. + if "mimxrt" in sys.platform: + print("send failed") multitest.broadcast("silent done") multitest.wait("normal done") diff --git a/tests/perf_bench/core_import_mpy_multi.py b/tests/perf_bench/core_import_mpy_multi.py index 67deec05088..55aa379e2bf 100644 --- a/tests/perf_bench/core_import_mpy_multi.py +++ b/tests/perf_bench/core_import_mpy_multi.py @@ -1,8 +1,9 @@ # Test performance of importing an .mpy file many times. -import sys, io, vfs - -if not hasattr(io, "IOBase"): +try: + import sys, vfs + from io import IOBase +except ImportError: print("SKIP") raise SystemExit @@ -26,7 +27,7 @@ def f(): file_data = b'M\x06\x00\x1e\x14\x03\x0etest.py\x00\x0f\x02A\x00\x02f\x00\x0cresult\x00/-5#\x82I\x81{\x81w\x82/\x81\x05\x81\x17Iom\x82\x13\x06arg\x00\x05\x1cthis will be a string object\x00\x06\x1bthis will be a bytes object\x00\n\x07\x05\x0bconst tuple\x00\x01\x02\x03\x07\x011\x07\x012\x07\x013\x81\\\x10\n\x01\x89\x07d`T2\x00\x10\x024\x02\x16\x022\x01\x16\x03"\x80{\x16\x04Qc\x02\x81d\x00\x08\x02(DD\x11\x05\x16\x06\x10\x02\x16\x072\x00\x16\x082\x01\x16\t2\x02\x16\nQc\x03`\x1a\x08\x08\x12\x13@\xb1\xb0\x18\x13Qc@\t\x08\t\x12` Qc@\t\x08\n\x12``Qc\x82@ \x0e\x03\x80\x08+)##\x12\x0b\x12\x0c\x12\r\x12\x0e*\x04Y\x12\x0f\x12\x10\x12\x11*\x03Y#\x00\xc0#\x01\xc0#\x02\xc0Qc' -class File(io.IOBase): +class File(IOBase): def __init__(self): self.off = 0 diff --git a/tests/perf_bench/core_import_mpy_single.py b/tests/perf_bench/core_import_mpy_single.py index f472bb64762..755e4159487 100644 --- a/tests/perf_bench/core_import_mpy_single.py +++ b/tests/perf_bench/core_import_mpy_single.py @@ -2,9 +2,10 @@ # The first import of a module will intern strings that don't already exist, and # this test should be representative of what happens in a real application. -import sys, io, vfs - -if not hasattr(io, "IOBase"): +try: + import sys, vfs + from io import IOBase +except ImportError: print("SKIP") raise SystemExit @@ -81,7 +82,7 @@ def f1(): file_data = b"M\x06\x00\x1e\x81=\x1e\x0etest.py\x00\x0f\x04A0\x00\x04A1\x00\x04f0\x00\x04f1\x00\x0cresult\x00/-5\x04a0\x00\x04a1\x00\x04a2\x00\x04a3\x00\x13\x15\x17\x19\x1b\x1d\x1f!#%')+1379;=?ACEGIKMOQSUWY[]_acegikmoqsuwy{}\x7f\x81\x01\x81\x03\x81\x05\x81\x07\x81\t\x81\x0b\x81\r\x81\x0f\x81\x11\x81\x13\x81\x15\x81\x17\x81\x19\x81\x1b\x81\x1d\x81\x1f\x81!\x81#\x81%\x81'\x81)\x81+\x81-\x81/\x811\x813\x815\x817\x819\x81;\x81=\x81?\x81A\x81C\x81E\x81G\x81I\x81K\x81M\x81O\x81Q\x81S\x81U\x81W\x81Y\x81[\x81]\x81_\x81a\x81c\x81e\x81g\x81i\x81k\x81m\x81o\x81q\x81s\x81u\x81w\x81y\x81{\x81}\x81\x7f\x82\x01\x82\x03\x82\x05\x82\x07\x82\t\x82\x0b\x82\r\x82\x0f\x82\x11\x82\x13\x82\x15\x82\x17\x82\x19\x82\x1b\x82\x1d\x82\x1f\x82!\x82#\x82%\x82'\x82)\x82+\x82-\x82/\x821\x823\x825\x827\x829\x82;\x82=\x82?\x82A\x82E\x82G\x82I\x82K\nname0\x00\nname1\x00\nname2\x00\nname3\x00\nname4\x00\nname5\x00\nname6\x00\nname7\x00\nname8\x00\nname9\x00$quite_a_long_name0\x00$quite_a_long_name1\x00$quite_a_long_name2\x00$quite_a_long_name3\x00$quite_a_long_name4\x00$quite_a_long_name5\x00$quite_a_long_name6\x00$quite_a_long_name7\x00$quite_a_long_name8\x00$quite_a_long_name9\x00&quite_a_long_name10\x00&quite_a_long_name11\x00\x05\x1ethis will be a string object 0\x00\x05\x1ethis will be a string object 1\x00\x05\x1ethis will be a string object 2\x00\x05\x1ethis will be a string object 3\x00\x05\x1ethis will be a string object 4\x00\x05\x1ethis will be a string object 5\x00\x05\x1ethis will be a string object 6\x00\x05\x1ethis will be a string object 7\x00\x05\x1ethis will be a string object 8\x00\x05\x1ethis will be a string object 9\x00\x06\x1dthis will be a bytes object 0\x00\x06\x1dthis will be a bytes object 1\x00\x06\x1dthis will be a bytes object 2\x00\x06\x1dthis will be a bytes object 3\x00\x06\x1dthis will be a bytes object 4\x00\x06\x1dthis will be a bytes object 5\x00\x06\x1dthis will be a bytes object 6\x00\x06\x1dthis will be a bytes object 7\x00\x06\x1dthis will be a bytes object 8\x00\x06\x1dthis will be a bytes object 9\x00\n\x07\x05\rconst tuple 0\x00\x01\x02\x03\x07\x011\x07\x012\x07\x013\n\x07\x05\rconst tuple 1\x00\x01\x02\x03\x07\x011\x07\x012\x07\x013\n\x07\x05\rconst tuple 2\x00\x01\x02\x03\x07\x011\x07\x012\x07\x013\n\x07\x05\rconst tuple 3\x00\x01\x02\x03\x07\x011\x07\x012\x07\x013\n\x07\x05\rconst tuple 4\x00\x01\x02\x03\x07\x011\x07\x012\x07\x013\n\x07\x05\rconst tuple 5\x00\x01\x02\x03\x07\x011\x07\x012\x07\x013\n\x07\x05\rconst tuple 6\x00\x01\x02\x03\x07\x011\x07\x012\x07\x013\n\x07\x05\rconst tuple 7\x00\x01\x02\x03\x07\x011\x07\x012\x07\x013\n\x07\x05\rconst tuple 8\x00\x01\x02\x03\x07\x011\x07\x012\x07\x013\n\x07\x05\rconst tuple 9\x00\x01\x02\x03\x07\x011\x07\x012\x07\x013\x82d\x10\x12\x01i@i@\x84\x18\x84\x1fT2\x00\x10\x024\x02\x16\x02T2\x01\x10\x034\x02\x16\x032\x02\x16\x042\x03\x16\x05\"\x80{\x16\x06Qc\x04\x82\x0c\x00\n\x02($$$\x11\x07\x16\x08\x10\x02\x16\t2\x00\x16\n2\x01\x16\x0b2\x02\x16\x0c2\x03\x16\rQc\x04@\t\x08\n\x81\x0b Qc@\t\x08\x0b\x81\x0b@Qc@\t\x08\x0c\x81\x0b`QcH\t\n\r\x81\x0b` Qc\x82\x14\x00\x0c\x03h`$$$\x11\x07\x16\x08\x10\x03\x16\t2\x00\x16\n2\x01\x16\x0b2\x02\x16\x0c2\x03\x16\rQc\x04H\t\n\n\x81\x0b``QcH\t\n\x0b\x81\x0b\x80\x07QcH\t\n\x0c\x81\x0b\x80\x08QcH\t\n\r\x81\x0b\x80\tQc\xa08P:\x04\x80\x0b13///---997799<\x1f%\x1f\"\x1f%)\x1f\"//\x12\x0e\x12\x0f\x12\x10\x12\x11\x12\x12\x12\x13\x12\x14*\x07Y\x12\x15\x12\x16\x12\x17\x12\x18\x12\x19\x12\x1a\x12\x08\x12\x07*\x08Y\x12\x1b\x12\x1c\x12\t\x12\x1d\x12\x1e\x12\x1f*\x06Y\x12 \x12!\x12\"\x12#\x12$\x12%*\x06Y\x12&\x12'\x12(\x12)\x12*\x12+*\x06Y\x12,\x12-\x12.\x12/\x120*\x05Y\x121\x122\x123\x124\x125*\x05Y\x126\x127\x128\x129\x12:*\x05Y\x12;\x12<\x12=\x12>\x12?\x12@\x12A\x12B\x12C\x12D\x12E*\x0bY\x12F\x12G\x12H\x12I\x12J\x12K\x12L\x12M\x12N\x12O\x12P*\x0bY\x12Q\x12R\x12S\x12T\x12U\x12V\x12W\x12X\x12Y\x12Z*\nY\x12[\x12\\\x12]\x12^\x12_\x12`\x12a\x12b\x12c\x12d*\nY\x12e\x12f\x12g\x12h\x12i\x12j\x12k\x12l\x12m\x12n\x12o*\x0bY\x12p\x12q\x12r\x12s\x12t\x12u\x12v\x12w\x12x\x12y\x12z*\x0bY\x12{\x12|\x12}\x12~\x12\x7f\x12\x81\x00\x12\x81\x01\x12\x81\x02\x12\x81\x03\x12\x81\x04*\nY\x12\x81\x05\x12\x81\x06\x12\x81\x07\x12\x81\x08\x12\x81\t\x12\x81\n\x12\x81\x0b\x12\x81\x0c\x12\x81\r\x12\x81\x0e\x12\x81\x0f*\x0bY\x12\x81\x10\x12\x81\x11\x12\x81\x12\x12\x81\x13\x12\x81\x14\x12\x81\x15\x12\x81\x16\x12\x81\x17\x12\x81\x18\x12\x81\x19*\nY\x12\x81\x1a\x12\x81\x1b\x12\x81\x1c\x12\x81\x1d\x12\x81\x1e\x12\x81\x1f\x12\x81 \x12\x81!\x12\x81\"\x12\x81#\x12\x81$*\x0bY\x12\x81%\x12\x81&*\x02Y\x12\x81'\x12\x81(\x12\x81)\x12\x81*\x12\x81+\x12\x81,\x12\x81-\x12\x81.\x12\x81/\x12\x810*\nY\x12\x811\x12\x812\x12\x813\x12\x814*\x04Y\x12\x815\x12\x816\x12\x817\x12\x818*\x04Y\x12\x819\x12\x81:\x12\x81;\x12\x81<*\x04YQc\x87p\x08@\x05\x80###############################\x00\xc0#\x01\xc0#\x02\xc0#\x03\xc0#\x04\xc0#\x05\xc0#\x06\xc0#\x07\xc0#\x08\xc0#\t\xc0#\n\xc0#\x0b\xc0#\x0c\xc0#\r\xc0#\x0e\xc0#\x0f\xc0#\x10\xc0#\x11\xc0#\x12\xc0#\x13\xc0#\x14\xc0#\x15\xc0#\x16\xc0#\x17\xc0#\x18\xc0#\x19\xc0#\x1a\xc0#\x1b\xc0#\x1c\xc0#\x1d\xc0Qc" -class File(io.IOBase): +class File(IOBase): def __init__(self): self.off = 0 diff --git a/tests/run-multitests.py b/tests/run-multitests.py index 40aac16c16f..de59db2dc38 100755 --- a/tests/run-multitests.py +++ b/tests/run-multitests.py @@ -598,7 +598,7 @@ def main(): cmd_args = cmd_parser.parse_args() # clear search path to make sure tests use only builtin modules and those in extmod - os.environ["MICROPYPATH"] = os.pathsep.join((".frozen", "../extmod")) + os.environ["MICROPYPATH"] = os.pathsep.join((".frozen", base_path("../extmod"))) test_files = prepare_test_file_list(cmd_args.files) max_instances = max(t[1] for t in test_files) diff --git a/tests/run-perfbench.py b/tests/run-perfbench.py index 16182bc8a9f..2b17655095c 100755 --- a/tests/run-perfbench.py +++ b/tests/run-perfbench.py @@ -8,11 +8,13 @@ import subprocess import sys import argparse +import re from glob import glob from test_utils import ( base_path, pyboard, + set_injected_prologue, get_test_instance, prepare_script_for_target, create_test_report, @@ -101,15 +103,14 @@ def run_benchmarks(args, target, param_n, param_m, n_average, test_list): print(test_file + ": ", end="") # Check if test should be skipped - skip = ( - skip_complex - and test_file.find("bm_fft") != -1 - or skip_native - and test_file.find("viper_") != -1 - ) - if skip: - test_results.append((test_file, "skip", "")) - print("SKIP") + skip_reason = None + if skip_complex and test_file.endswith(("bm_fft.py", "misc_mandel.py")): + skip_reason = "complex not supported" + elif skip_native and test_file.find("viper_") != -1: + skip_reason = "native not supported" + if skip_reason: + test_results.append((test_file, "skip", skip_reason)) + print("SKIP:", skip_reason) continue # Create test script @@ -120,6 +121,19 @@ def run_benchmarks(args, target, param_n, param_m, n_average, test_list): test_script += f.read() test_script += b"bm_run(%u, %u)\n" % (param_n, param_m) + # Search for the bm_params dict, to extract the minimum memory required. + m = re.search(rb"bm_params = {\s+\((\d+), (\d+)\):", test_script) + if not m: + print(f"Test file '{test_file}' doesn't contain valid 'bm_params'") + sys.exit(2) + min_m = int(m.group(2)) + + # Skip the test if the target doesn't have enough memory. + if param_m < min_m: + test_results.append((test_file, "skip", "too large")) + print("SKIP: too large") + continue + # Write full test script if needed if 0: with open("%s.full" % test_file, "wb") as f: @@ -127,7 +141,7 @@ def run_benchmarks(args, target, param_n, param_m, n_average, test_list): # Process script through mpy-cross if needed if hasattr(target, "enter_raw_repl") or args.via_mpy: - crash, test_script_target = prepare_script_for_target(args, script_text=test_script) + crash, test_script_target = prepare_script_for_target(args, test_script, test_file) if crash: test_results.append((test_file, "fail", "preparation")) print("CRASH:", test_script_target) @@ -278,6 +292,12 @@ def main(): cmd_parser.add_argument("--heapsize", help="heapsize to use (use default if not specified)") cmd_parser.add_argument("--via-mpy", action="store_true", help="compile code to .mpy first") cmd_parser.add_argument("--mpy-cross-flags", default="", help="flags to pass to mpy-cross") + cmd_parser.add_argument( + "--begin", + metavar="PROLOGUE", + default=None, + help="prologue python file to execute before module import", + ) cmd_parser.add_argument( "-r", "--result-dir", @@ -295,6 +315,12 @@ def main(): compute_diff(args.N[0], args.M[0], args.diff_score) sys.exit(0) + prologue = "" + if args.begin: + with open(args.begin, "rt") as source: + prologue = source.read() + set_injected_prologue(prologue) + # N, M = 50, 25 # esp8266 # N, M = 100, 100 # pyboard, esp32 # N, M = 1000, 1000 # PC @@ -314,14 +340,10 @@ def main(): args.mpy_cross_flags = "-march=armv7m" if len(args.files) == 0: - tests_skip = ("benchrun.py",) - if M <= 25: - # These scripts are too big to be compiled by the target - tests_skip += ("bm_chaos.py", "bm_hexiom.py", "misc_raytrace.py") tests = sorted( BENCH_SCRIPT_DIR + test_file for test_file in os.listdir(BENCH_SCRIPT_DIR) - if test_file.endswith(".py") and test_file not in tests_skip + if test_file.endswith(".py") and test_file != "benchrun.py" ) else: tests = sorted(args.files) diff --git a/tests/run-tests.py b/tests/run-tests.py index 9ac8fdd0fbb..35eb2eb3aa2 100755 --- a/tests/run-tests.py +++ b/tests/run-tests.py @@ -29,7 +29,6 @@ get_results_filename, convert_device_shortcut_to_real_device, get_test_instance, - prepare_script_for_target, create_test_report, FLAKY_REASON_PREFIX, ) @@ -90,11 +89,12 @@ # Remove them from the below when they work. "native": ( # These require raise_varargs. - "basics/gen_yield_from_close.py", "basics/try_finally_return2.py", "basics/try_reraise.py", "basics/try_reraise2.py", - "misc/features.py", + # CIRCUITPY-CHANGE: our traceback deviation in this test uses a bare + # raise, which is raise_varargs with no argument. + "basics/gen_yield_from_close.py", # These require checking for unbound local. "basics/annotate_var.py", "basics/del_deref.py", @@ -178,6 +178,7 @@ "extmod/binascii_a2b_base64.py", "extmod/deflate_compress_memory_error.py", # tries to allocate unlimited memory "extmod/re_stack_overflow.py", + "extmod/re_stack_overflow2.py", "extmod/time_res.py", "extmod/vfs_posix.py", "extmod/vfs_posix_enoent.py", @@ -197,6 +198,25 @@ ), } +# Tests to skip when MICROPY_ERROR_REPORTING is at a certain level. +error_reporting_tests_to_skip = { + # Skip at level MICROPY_ERROR_REPORTING_TERSE. + "terse": ( + "cmdline/repl_paste.py", + # This test needs updates before being removed from this list. + "extmod/vfs_blockdev_invalid.py", + "micropython/heapalloc_exc_compressed.py", + "micropython/heapalloc_exc_compressed_emg_exc.py", + "micropython/opt_level_lineno.py", + "misc/print_exception.py", + "misc/sys_settrace_features.py", + ), +} +# Skip at level MICROPY_ERROR_REPORTING_NONE. +error_reporting_tests_to_skip["none"] = error_reporting_tests_to_skip["terse"] + ( + "extmod/asyncio_gather_notimpl.py", +) + # Tests with known intermittent failures. These tests still run, but failures # are reclassified as "ignored" instead of "fail" so they don't affect the CI # exit code. Paths are relative to the tests/ directory (must match test_file @@ -239,6 +259,8 @@ "extmod/uctypes_le_float.py", "extmod/uctypes_native_float.py", "extmod/uctypes_sizeof_float.py", + "extmod/vfs_rom.py", + "micropython/const_float.py", "misc/rge_sm.py", "ports/unix/ffi_float.py", "ports/unix/ffi_float2.py", @@ -267,6 +289,7 @@ "extmod/time_mktime.py", "extmod/time_res.py", "extmod/tls_sslcontext_ciphers.py", + "extmod/vfs_blockdev_invalid2.py", "extmod/vfs_fat_fileio1.py", "extmod/vfs_fat_finaliser.py", "extmod/vfs_fat_more.py", @@ -283,6 +306,7 @@ "micropython/import_mpy_invalid.py", "micropython/import_mpy_native.py", "micropython/import_mpy_native_gc.py", + "micropython/ringio_big.py", "misc/non_compliant.py", "misc/rge_sm.py", ) @@ -326,11 +350,9 @@ def platform_to_port(platform): def detect_inline_asm_arch(pyb, args): - for arch in ("rv32", "thumb", "xtensa"): - output = run_feature_check(pyb, args, "inlineasm_{}.py".format(arch)) - if output.strip() == arch.encode(): - return arch - return None + output = run_feature_check(pyb, args, "inlineasm.py").decode().strip() + arch, *features = output.split(",") + return arch, features def map_rv32_arch_flags(flags): @@ -350,12 +372,12 @@ def detect_test_platform(pyb, args): if output.endswith(b"CRASH"): raise ValueError("cannot detect platform: {}".format(output)) # CIRCUITPY-CHANGE: sys.platform is the first field and can contain spaces ("Atmel SAMD21"). - platform, arch, arch_flags, build, thread, float_prec, unicode = ( - str(output, "ascii").strip().rsplit(None, 6) + platform, arch, arch_flags, build, thread, float_prec, unicode, error_reporting = ( + str(output, "ascii").strip().rsplit(None, 7) ) if arch == "None": arch = None - inlineasm_arch = detect_inline_asm_arch(pyb, args) + inlineasm_arch, inlineasm_features = detect_inline_asm_arch(pyb, args) if thread == "None": thread = None float_prec = int(float_prec) @@ -373,10 +395,12 @@ def detect_test_platform(pyb, args): if arch_flags: args.mpy_cross_flags += " -march-flags=" + ",".join(arch_flags) args.inlineasm_arch = inlineasm_arch + args.inlineasm_features = inlineasm_features args.build = build args.thread = thread args.float_prec = float_prec args.unicode = unicode + args.error_reporting = error_reporting # Print the detected information about the target. print("platform={}".format(platform), end="") @@ -446,7 +470,9 @@ def detect_target_wiring_script(pyb, args): ] -def run_micropython(pyb, args, test_file, test_file_abspath, is_special=False): +def run_micropython( + pyb, args, test_file, test_file_abspath, is_special=False, is_feature_check=False +): had_crash = False if pyb is None: # run on PC @@ -456,12 +482,17 @@ def run_micropython(pyb, args, test_file, test_file_abspath, is_special=False): if is_special: # check for any cmdline options needed for this test - args = [MICROPYTHON] + cmdlist = [MICROPYTHON] + send_sigint = False with open(test_file, "rb") as f: - line = f.readline() - if line.startswith(b"# cmdline:"): - # subprocess.check_output on Windows only accepts strings, not bytes - args += [str(c, "utf-8") for c in line[10:].strip().split()] + for line in f: + if line.startswith(b"# cmdline:"): + # subprocess.check_output on Windows only accepts strings, not bytes + cmdlist += [str(c, "utf-8") for c in line[10:].strip().split()] + elif line.startswith(b"# sigint:"): + send_sigint = True + elif not line.startswith(b"#"): + break # run the test, possibly with redirected input try: @@ -489,37 +520,86 @@ def get(required=False): return rv def send_get(what): - # Detect {\x00} pattern and convert to ctrl-key codes. - ctrl_code = lambda m: bytes([int(m.group(1))]) + # Detect hex {\x00} pattern and convert to ctrl-key codes. + ctrl_code = lambda m: bytes([int(m.group(1), 16)]) what = re.sub(rb"{\\x(\d\d)}", ctrl_code, what) os.write(master, what) return get() + def send_ctrl_c(): + # Send \x03 without trailing newline and wait for + # the full response (traceback + new prompt). + os.write(master, b"\x03") + return get(True) + with open(test_file, "rb") as f: - # instead of: output_mupy = subprocess.check_output(args, stdin=f) + # instead of: output_mupy = subprocess.check_output(cmdlist, stdin=f) master, slave = pty.openpty() - p = subprocess.Popen( - args, stdin=slave, stdout=slave, stderr=subprocess.STDOUT, bufsize=0 - ) - banner = get(True) - output_mupy = banner + b"".join(send_get(line) for line in f) - send_get(b"\x04") # exit the REPL, so coverage info is saved - # At this point the process might have exited already, but trying to - # kill it 'again' normally doesn't result in exceptions as Python and/or - # the OS seem to try to handle this nicely. When running Linux on WSL - # though, the situation differs and calling Popen.kill after the process - # terminated results in a ProcessLookupError. Just catch that one here - # since we just want the process to be gone and that's the case. try: - p.kill() - except ProcessLookupError: - pass - os.close(master) - os.close(slave) + preexec_fn = None + use_sigint_kill = False + # Tests with "# sigint:" need Ctrl-C (\x03) to + # generate SIGINT. MicroPython restores original + # terminal mode (ISIG on) during code execution, + # so on Linux we set up the PTY as a controlling + # terminal for proper signal delivery. On macOS, + # setsid/TIOCSCTTY breaks PTY I/O, so we fall + # back to os.kill(). + if send_sigint: + if sys.platform == "darwin": + use_sigint_kill = True + else: + import fcntl + import termios + + def preexec_fn(): + os.setsid() + fcntl.ioctl(0, termios.TIOCSCTTY, 0) + os.tcsetpgrp(0, os.getpid()) + + p = subprocess.Popen( + cmdlist, + stdin=slave, + stdout=slave, + stderr=subprocess.STDOUT, + bufsize=0, + preexec_fn=preexec_fn, + ) + banner = get(True) + if send_sigint: + import signal + + parts = [] + for line in f: + if b"{\\x03}" in line: + if use_sigint_kill: + os.kill(p.pid, signal.SIGINT) + parts.append(get(True)) + else: + parts.append(send_ctrl_c()) + else: + parts.append(send_get(line)) + output_mupy = banner + b"".join(parts) + else: + output_mupy = banner + b"".join(send_get(line) for line in f) + send_get(b"\x04") # exit the REPL, so coverage info is saved + # At this point the process might have exited already, but trying to + # kill it 'again' normally doesn't result in exceptions as Python and/or + # the OS seem to try to handle this nicely. When running Linux on WSL + # though, the situation differs and calling Popen.kill after the process + # terminated results in a ProcessLookupError. Just catch that one here + # since we just want the process to be gone and that's the case. + try: + p.kill() + except ProcessLookupError: + pass + finally: + os.close(master) + os.close(slave) else: output_mupy = subprocess.check_output( - args + [test_file], stderr=subprocess.STDOUT + cmdlist + [test_file], stderr=subprocess.STDOUT ) except subprocess.CalledProcessError: return b"CRASH" @@ -575,6 +655,10 @@ def send_get(what): # canonical form for all ports/platforms is to use \n for end-of-line output_mupy = normalize_newlines(output_mupy) + # for feature-check tests, return the output as-is + if is_feature_check: + return output_mupy + # don't try to convert the output if we should skip this test if had_crash or output_mupy in (b"SKIP\n", b"SKIP-TOO-LARGE\n", b"CRASH"): return output_mupy @@ -585,7 +669,12 @@ def send_get(what): if is_special or test_file_abspath in tests_with_regex_output: # convert parts of the output that are not stable across runs - with open(test_file + ".exp", "rb") as f: + # Prefer emitter-specific expected output. + exp_file = test_file + "." + args.emit + ".exp" + if not os.path.isfile(exp_file): + # Fall back to generic expected output. + exp_file = test_file + ".exp" + with open(exp_file, "rb") as f: lines_exp = [] for line in f.readlines(): if line == b"########\n": @@ -632,7 +721,9 @@ def run_feature_check(pyb, args, test_file): # REPL feature tests will not run via pyboard because they require prompt interactivity return b"" test_file_path = base_path("feature_check", test_file) - return run_micropython(pyb, args, test_file_path, test_file_path, is_special=True) + return run_micropython( + pyb, args, test_file_path, test_file_path, is_special=True, is_feature_check=True + ) class TestError(Exception): @@ -742,32 +833,15 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1): skip_tstring = True if args.inlineasm_arch == "thumb": - # Check if @micropython.asm_thumb supports Thumb2 instructions, and skip such tests if it doesn't - output = run_feature_check(pyb, args, "inlineasm_thumb2.py") - if output != b"thumb2\n": - skip_tests.add("inlineasm/thumb/asmbcc.py") - skip_tests.add("inlineasm/thumb/asmbitops.py") - skip_tests.add("inlineasm/thumb/asmconst.py") - skip_tests.add("inlineasm/thumb/asmdiv.py") - skip_tests.add("inlineasm/thumb/asmit.py") - skip_tests.add("inlineasm/thumb/asmspecialregs.py") - if args.arch not in ("armv7emsp", "armv7emdp"): - skip_tests.add("inlineasm/thumb/asmfpaddsub.py") - skip_tests.add("inlineasm/thumb/asmfpcmp.py") - skip_tests.add("inlineasm/thumb/asmfpldrstr.py") - skip_tests.add("inlineasm/thumb/asmfpmuldiv.py") - skip_tests.add("inlineasm/thumb/asmfpsqrt.py") + for feature in ("thumb2", "vfp"): + if feature not in args.inlineasm_features: + for test in glob(f"inlineasm/thumb/asm_{feature}_*.py"): + skip_tests.add(test) if args.inlineasm_arch == "rv32": - # Discover extension-specific inlineasm tests and add them to the - # list of tests to run if applicable. for extension in RV32_ARCH_FLAGS: - try: - output = run_feature_check(pyb, args, "inlineasm_rv32_{}.py".format(extension)) - if output.strip() != "rv32_{}".format(extension).encode(): - skip_tests.add("inlineasm/rv32/asm_ext_{}.py".format(extension)) - except FileNotFoundError: - pass + if extension not in args.inlineasm_features: + skip_tests.add(f"inlineasm/rv32/asm_ext_{extension}.py") # Check if emacs repl is supported, and skip such tests if it's not t = run_feature_check(pyb, args, "repl_emacs_check.py") @@ -787,7 +861,7 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1): ) skip_endian = upy_byteorder != cpy_byteorder - skip_inlineasm = args.inlineasm_arch is None + skip_inlineasm = not args.inlineasm_arch # Some tests shouldn't be run on GitHub Actions if os.getenv("GITHUB_ACTIONS") == "true": @@ -815,6 +889,8 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1): skip_tests.add("float/float_parse_doubleprec.py") if not args.unicode: + if args.via_mpy: + skip_tests.add("basics/string_escape.py") # stores a utf-8 character in the mpy file skip_tests.add("extmod/json_loads.py") # tests loading a utf-8 character # CIRCUITPY-CHANGE: asserts upstream's escaped Unicode repr, see py/objstrunicode.c @@ -862,6 +938,9 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1): # Skip platform-specific tests. skip_tests.update(platform_tests_to_skip.get(args.platform, ())) + # Skip error-reporting-specific tests. + skip_tests.update(error_reporting_tests_to_skip.get(args.error_reporting, ())) + # Some tests are known to fail on 64-bit machines if pyb is None and platform.architecture()[0] == "64bit": pass @@ -914,7 +993,7 @@ def run_one_test(test_file): is_slice = test_name.find("slice") != -1 is_async = test_name.startswith(("async_", "asyncio_")) or test_name.endswith("_async") is_const = test_name.startswith("const") - is_fstring = test_name.startswith("string_fstring") + is_fstring = test_name.startswith("string_fstring") or test_name.endswith("_fstring") is_tstring = test_name.startswith("string_tstring") or test_name.endswith("_tstring") is_inlineasm = test_name.startswith("asm") @@ -1076,7 +1155,7 @@ def run_one_test(test_file): # Print a note if this looks like it might have been a misfired unittest if not uses_unittest and not test_passed: - with open(test_file, "r") as f: + with open(test_file, "r", encoding="utf-8") as f: if any(re.match("^import.+unittest", l) for l in f.readlines()): print( "NOTE: {} may be a unittest that doesn't run unittest.main()".format( @@ -1155,6 +1234,9 @@ def main(): run-tests.py -e async -i async_foo - include all, exclude async, yet still include async_foo """, ) + cmd_parser.add_argument( + "-c", "--trace-output", action="store_true", help="trace test output while running" + ) cmd_parser.add_argument( "-t", "--test-instance", default="unix", help="the MicroPython instance to test" ) diff --git a/tests/stress/bytecode_limit.py b/tests/stress/bytecode_limit.py index 0a72b66fa05..7e77cdd5e5a 100644 --- a/tests/stress/bytecode_limit.py +++ b/tests/stress/bytecode_limit.py @@ -23,7 +23,7 @@ print("SKIP") raise SystemExit except RuntimeError as er: - results.append(repr(er)) + results.append("RuntimeError('{}')".format(str(er) or "bytecode overflow")) print(results) # Test changing size of code info (source line/bytecode mapping) due to changing diff --git a/tests/stress/bytecode_limit.py.exp b/tests/stress/bytecode_limit.py.exp index 50511665f00..3f1eb0db4c4 100644 --- a/tests/stress/bytecode_limit.py.exp +++ b/tests/stress/bytecode_limit.py.exp @@ -1,4 +1,4 @@ cond false cond false -["RuntimeError('bytecode overflow',)", "RuntimeError('bytecode overflow',)", 'ok', 'ok'] +["RuntimeError('bytecode overflow')", "RuntimeError('bytecode overflow')", 'ok', 'ok'] [123] diff --git a/tests/stress/qstr_limit.py b/tests/stress/qstr_limit.py index c7bd437f3ad..55b87075df9 100644 --- a/tests/stress/qstr_limit.py +++ b/tests/stress/qstr_limit.py @@ -13,7 +13,7 @@ def make_id(n, base="a"): try: exec(var + "=1", g) except RuntimeError as er: - print("RuntimeError", er, l) + print(str(er) or "name too long", l) continue print(var in g) @@ -27,7 +27,7 @@ def f(**k): try: exec("f({}=1)".format(make_id(l))) except RuntimeError as er: - print("RuntimeError", er, l) + print(str(er) or "name too long", l) # type construction for l in range(254, 259): @@ -35,7 +35,7 @@ def f(**k): try: print(type(id, (), {})) except RuntimeError as er: - print("RuntimeError", er, l) + print(str(er) or "name too long", l) # hasattr, setattr, getattr @@ -49,11 +49,11 @@ class A: try: setattr(a, id, 123) except RuntimeError as er: - print("RuntimeError", er, l) + print(str(er) or "name too long", l) try: print(hasattr(a, id), getattr(a, id)) except RuntimeError as er: - print("RuntimeError", er, l) + print(str(er) or "name too long", l) # format with keys for l in range(254, 259): @@ -61,7 +61,7 @@ class A: try: print(("{" + id + "}").format(**{id: l})) except RuntimeError as er: - print("RuntimeError", er, l) + print(str(er) or "name too long", l) # import module # (different OS's have different results so only run those that are consistent) @@ -71,7 +71,7 @@ class A: except ImportError: print("ok", l) except RuntimeError as er: - print("RuntimeError", er, l) + print(str(er) or "name too long", l) # import package for l in (100, 101, 102, 128, 129): @@ -80,4 +80,4 @@ class A: except ImportError: print("ok", l) except RuntimeError as er: - print("RuntimeError", er, l) + print(str(er) or "name too long", l) diff --git a/tests/stress/qstr_limit.py.exp b/tests/stress/qstr_limit.py.exp index 2349adf220f..5596b54bebd 100644 --- a/tests/stress/qstr_limit.py.exp +++ b/tests/stress/qstr_limit.py.exp @@ -1,38 +1,38 @@ True True -RuntimeError name too long 256 -RuntimeError name too long 257 -RuntimeError name too long 258 +name too long 256 +name too long 257 +name too long 258 {'abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrst': 1} {'abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstu': 1} -RuntimeError name too long 256 -RuntimeError name too long 257 -RuntimeError name too long 258 +name too long 256 +name too long 257 +name too long 258 -RuntimeError name too long 256 -RuntimeError name too long 257 -RuntimeError name too long 258 +name too long 256 +name too long 257 +name too long 258 True 123 True 123 -RuntimeError name too long 256 -RuntimeError name too long 256 -RuntimeError name too long 257 -RuntimeError name too long 257 -RuntimeError name too long 258 -RuntimeError name too long 258 +name too long 256 +name too long 256 +name too long 257 +name too long 257 +name too long 258 +name too long 258 254 255 -RuntimeError name too long 256 -RuntimeError name too long 257 -RuntimeError name too long 258 +name too long 256 +name too long 257 +name too long 258 ok 100 ok 101 -RuntimeError name too long 256 -RuntimeError name too long 257 -RuntimeError name too long 258 +name too long 256 +name too long 257 +name too long 258 ok 100 ok 101 ok 102 -RuntimeError name too long 128 -RuntimeError name too long 129 +name too long 128 +name too long 129 diff --git a/tests/stress/qstr_limit_str_modulo.py b/tests/stress/qstr_limit_str_modulo.py index 90b9f4364ec..c56f23b0a1d 100644 --- a/tests/stress/qstr_limit_str_modulo.py +++ b/tests/stress/qstr_limit_str_modulo.py @@ -18,4 +18,4 @@ def make_id(n, base="a"): try: print(("%(" + id + ")d") % {id: l}) except RuntimeError as er: - print("RuntimeError", er, l) + print("RuntimeError", str(er) or "name too long", l) diff --git a/tests/target_wiring/KIT_PSE84_AI.py b/tests/target_wiring/KIT_PSE84_AI.py new file mode 100644 index 00000000000..8ef3f08385d --- /dev/null +++ b/tests/target_wiring/KIT_PSE84_AI.py @@ -0,0 +1,5 @@ +# Target wiring for KIT_PSE84_AI. + +# UART(5) is on P17_1/P17_0. +uart_loopback_args = () +uart_loopback_kwargs = {"tx": "P17_1", "rx": "P17_0"} diff --git a/tests/target_wiring/alif.py b/tests/target_wiring/alif.py index 708b4591846..855c6e9ec90 100644 --- a/tests/target_wiring/alif.py +++ b/tests/target_wiring/alif.py @@ -9,3 +9,7 @@ spi_standalone_args_list = [(0,)] pwm_loopback_pins = [("P0_4", "P0_5")] + +# CAN args assume no connection for single device tests +can_args = (1,) +can_kwargs = {} diff --git a/tests/target_wiring/esp32.py b/tests/target_wiring/esp32.py index d94a6f60759..a707a171c8b 100644 --- a/tests/target_wiring/esp32.py +++ b/tests/target_wiring/esp32.py @@ -9,7 +9,7 @@ uart_loopback_args = (1,) uart_loopback_kwargs = {"tx": 4, "rx": 5} -if "ESP32C" in sys.implementation._machine: +if "ESP32-C" in sys.implementation._machine or "ESP32-H2" in sys.implementation._machine: spi_standalone_args_list = [(1,)] else: spi_standalone_args_list = [(1,), (2,)] diff --git a/tests/target_wiring/mimxrt.py b/tests/target_wiring/mimxrt.py index 2836d88ab9d..14190380e83 100644 --- a/tests/target_wiring/mimxrt.py +++ b/tests/target_wiring/mimxrt.py @@ -23,3 +23,7 @@ encoder_loopback_id = 0 encoder_loopback_out_pins = ("D0", "D2") encoder_loopback_in_pins = ("D1", "D3") + +# CAN args assume no connection for single device tests +can_args = (1,) +can_kwargs = {} diff --git a/tests/test_utils.py b/tests/test_utils.py index 7e43c4cae9f..40546b3cec2 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -8,7 +8,6 @@ import re import subprocess import sys -import tempfile # See stackoverflow.com/questions/2632199: __file__ nor sys.argv[0] # are guaranteed to always work, this one should though. @@ -224,34 +223,27 @@ def get_test_instance(test_instance, baudrate, user, password): return pyb -def prepare_script_for_target(args, *, script_text=None, force_plain=False): +def prepare_script_for_target(args, script_text, script_name, force_plain=False): if force_plain or (not args.via_mpy and args.emit == "bytecode"): # A plain test to run as-is, no processing needed. pass elif args.via_mpy: - tempname = tempfile.mktemp(dir="") - mpy_filename = tempname + ".mpy" - - script_filename = tempname + ".py" - with open(script_filename, "wb") as f: - f.write(script_text) - try: - subprocess.check_output( + # Compile the script with mpy-cross (using stdin/stdout). + p = subprocess.run( [MPYCROSS] + args.mpy_cross_flags.split() - + ["-o", mpy_filename, "-X", "emit=" + args.emit, script_filename], - stderr=subprocess.STDOUT, + + ["-s", script_name, "-X", "emit=" + args.emit, "--", "-"], + input=script_text, + capture_output=True, + check=True, ) + assert p.stderr == b"" + mpy_data = p.stdout except subprocess.CalledProcessError as er: - return True, b"mpy-cross crash\n" + er.output - - with open(mpy_filename, "rb") as f: - script_text = b"__buf=" + bytes(repr(f.read()), "ascii") + b"\n" - - rm_f(mpy_filename) - rm_f(script_filename) + return True, b"mpy-cross crash\n" + er.output + er.stderr + script_text = b"__buf=" + bytes(repr(mpy_data), "ascii") + b"\n" script_text += bytes(_injected_import_hook_code, "ascii") else: print("error: using emit={} must go via .mpy".format(args.emit)) @@ -274,29 +266,57 @@ def run_script_on_remote_target(pyb, args, test_file, is_special, requires_targe else: script = b"print('START TEST')\n" + script - had_crash, script = prepare_script_for_target(args, script_text=script, force_plain=is_special) + had_crash, script = prepare_script_for_target(args, script, test_file, force_plain=is_special) if had_crash: return True, script + # See if the output should be traced (printed to stdout), but not for feature_check tests. + trace_output = args.trace_output and "feature_check" not in test_file + if trace_output: + print(f"TRACE: {test_file}") + + # Function to collect output data as the test is run. + output_mupy = bytearray() + + def data_consumer(data): + if data == b"\x04": + # End of stream. + return + if trace_output: + # Print out the data as it's received. + sys.stdout.buffer.write(data) + sys.stdout.buffer.flush() + output_mupy.extend(data) + try: pyb.enter_raw_repl(timeout_overall=TEST_ENTER_RAW_REPL_TIMEOUT) + + # Inject target wiring if needed by the test. if requires_target_wiring and pyb.target_wiring_script: pyb.exec_( "import sys;sys.modules['target_wiring']=__build_class__(lambda:exec(" + repr(pyb.target_wiring_script) + "),'target_wiring')" ) - output_mupy = pyb.exec_(script, timeout=TEST_TIMEOUT) + + # Execute the test, and collect the output. + pyb.exec_(script, timeout=TEST_TIMEOUT, data_consumer=data_consumer) except pyboard.PyboardError as e: had_crash = True if not is_special and e.args[0] == "exception": - if prepend_start_test and e.args[1] == b"" and b"MemoryError" in e.args[2]: + no_output = len(output_mupy) == 0 + data_consumer(e.args[1]) + data_consumer(e.args[2]) + if prepend_start_test and no_output and b"MemoryError" in e.args[2]: output_mupy = b"SKIP-TOO-LARGE\n" else: - output_mupy = e.args[1] + e.args[2] + b"CRASH" + output_mupy += b"CRASH" else: - output_mupy = bytes(e.args[0], "ascii") + b"\nCRASH" + data_consumer(bytes(e.args[0], "ascii") + b"\n") + output_mupy += b"CRASH" + + output_mupy = bytes(output_mupy) if prepend_start_test: if output_mupy.startswith(b"START TEST\r\n"): diff --git a/tests/thread/thread_exc2.py.exp b/tests/thread/thread_exc2.py.exp index 469516dacc0..b7085c7af9b 100644 --- a/tests/thread/thread_exc2.py.exp +++ b/tests/thread/thread_exc2.py.exp @@ -1,5 +1,5 @@ Unhandled exception in thread started by Traceback (most recent call last): File \.\+, line 7, in thread_entry -ValueError: +ValueError: \$ done diff --git a/tests/thread/thread_exc2.py.native.exp b/tests/thread/thread_exc2.py.native.exp index 9b2e715ef8d..8188dd3ed00 100644 --- a/tests/thread/thread_exc2.py.native.exp +++ b/tests/thread/thread_exc2.py.native.exp @@ -1,3 +1,3 @@ Unhandled exception in thread started by -ValueError: +ValueError: \$ done diff --git a/tests/tools/manifest_c_module.py b/tests/tools/manifest_c_module.py new file mode 100644 index 00000000000..0dc89bd08e6 --- /dev/null +++ b/tests/tools/manifest_c_module.py @@ -0,0 +1,13 @@ +# Manifest for testing the c_module() build feature. +# +# Shared across ports; used in CI to exercise the c_module() manifest entry +# alongside per-port builds that also test the legacy USER_C_MODULES= +# command-line variable. + +# Include the port's default board manifest. +include("$(PORT_DIR)/boards/manifest.py") + +# Test user C modules via c_module(). +c_module("$(MPY_DIR)/examples/usercmodule/cexample") +c_module("$(MPY_DIR)/examples/usercmodule/cppexample") +c_module("$(MPY_DIR)/examples/usercmodule/subpackage") diff --git a/tests/tools/test_manifestfile.py b/tests/tools/test_manifestfile.py new file mode 100644 index 00000000000..c16f5142886 --- /dev/null +++ b/tests/tools/test_manifestfile.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +# +# Unit tests for tools/manifestfile.py. Run with +# `python3 tests/tools/test_manifestfile.py`. + +import os +import sys +import tempfile +import unittest + +# Locate tools/manifestfile.py relative to this file. +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", "tools")) +from manifestfile import ( + ManifestFile, + ManifestFileError, + MODE_COMPILE, + MODE_FREEZE, + MODE_PYPROJECT, +) + + +class TestCModule(unittest.TestCase): + def setUp(self): + self.tmpdir = tempfile.mkdtemp() + self.mf = ManifestFile(MODE_FREEZE, {"MPY_DIR": self.tmpdir}) + + def tearDown(self): + import shutil + + shutil.rmtree(self.tmpdir, ignore_errors=True) + + def _make_path(self, *parts): + path = os.path.join(self.tmpdir, *parts) + os.makedirs(os.path.dirname(path), exist_ok=True) + return path + + def test_missing_path(self): + with self.assertRaisesRegex(ManifestFileError, "C module path does not exist"): + self.mf.c_module(os.path.join(self.tmpdir, "no_such_dir")) + + def test_path_is_file_not_dir(self): + f = os.path.join(self.tmpdir, "regular_file") + open(f, "w").close() + with self.assertRaisesRegex(ManifestFileError, "C module path must be a directory"): + self.mf.c_module(f) + + def test_dir_missing_module_files(self): + d = os.path.join(self.tmpdir, "mymod") + os.makedirs(d) + with self.assertRaisesRegex( + ManifestFileError, "must contain micropython.mk or micropython.cmake" + ): + self.mf.c_module(d) + + def test_accepts_dir_with_mk(self): + d = os.path.join(self.tmpdir, "mymod_mk") + os.makedirs(d) + open(os.path.join(d, "micropython.mk"), "w").close() + self.mf.c_module(d) + self.assertIn(d, self.mf.c_modules()) + + def test_accepts_dir_with_cmake(self): + d = os.path.join(self.tmpdir, "mymod_cmake") + os.makedirs(d) + open(os.path.join(d, "micropython.cmake"), "w").close() + self.mf.c_module(d) + self.assertIn(d, self.mf.c_modules()) + + def test_path_var_substitution(self): + d = os.path.join(self.tmpdir, "submod") + os.makedirs(d) + open(os.path.join(d, "micropython.mk"), "w").close() + self.mf.c_module("$(MPY_DIR)/submod") + self.assertIn(d, self.mf.c_modules()) + + def test_unresolved_path_var(self): + with self.assertRaisesRegex( + ManifestFileError, r"Unresolved variable in c_module\(\) path" + ): + self.mf.c_module("$(BOARD_DIR)/nope") + + +class TestCModuleNonFreezeModes(unittest.TestCase): + # In MODE_COMPILE/MODE_PYPROJECT, c_module() is a silent no-op so a manifest + # using it can be evaluated in any mode without raising NameError. + + def setUp(self): + self.tmpdir = tempfile.mkdtemp() + + def tearDown(self): + import shutil + + shutil.rmtree(self.tmpdir, ignore_errors=True) + + def _no_op_in_mode(self, mode): + mf = ManifestFile(mode, {"MPY_DIR": self.tmpdir}) + # Path does not exist; would raise in MODE_FREEZE. + mf.c_module(os.path.join(self.tmpdir, "no_such_dir")) + self.assertEqual(mf.c_modules(), []) + + def test_compile_mode_noop(self): + self._no_op_in_mode(MODE_COMPILE) + + def test_pyproject_mode_noop(self): + self._no_op_in_mode(MODE_PYPROJECT) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unicode/ascii.py b/tests/unicode/ascii.py new file mode 100644 index 00000000000..2ae1f9fceed --- /dev/null +++ b/tests/unicode/ascii.py @@ -0,0 +1,22 @@ +# Test conversions from ASCII to/from UTF8 strings via constructor + +# Invalid ASCII characters via constructor +try: + str(b"ni\xe5\xa5\xbd", "ascii") # Valid utf8, invalid ascii +except UnicodeError: + print("UnicodeError") + +try: + str(b"\xff\xee\xff", "ascii") # Totally invalid +except UnicodeError: + print("UnicodeError") + + +bytes("😀", "utf8") +bytes("abcd", "ascii") + +# Conversion should fail if non-ASCII characters in string +try: + bytes("😀", "ascii") +except UnicodeError: + print("UnicodeError") diff --git a/tests/unicode/bytes_decode_encoding.py b/tests/unicode/bytes_decode_encoding.py new file mode 100644 index 00000000000..b20d51a167d --- /dev/null +++ b/tests/unicode/bytes_decode_encoding.py @@ -0,0 +1,75 @@ +# Test bytes.decode() and str.encode() with encoding parameter validation + +# Check if decode method is available (requires MICROPY_CPYTHON_COMPAT) +try: + b"".decode() +except AttributeError: + print("SKIP") + raise SystemExit + +# Test valid encodings for bytes.decode() +# utf-8 (default) +print(b"hello".decode("utf-8")) +print(b"hello".decode("utf8")) + +# ascii (subset of utf-8) +print(b"hello".decode("ascii")) + +# Test valid encoding for str.encode() +print("hello".encode("utf-8")) +print("hello".encode("utf8")) +print("hello".encode("ascii")) + +# Test with bytearray +print(bytearray(b"test").decode("utf-8")) + +# Test that UTF-8 still works correctly with non-ASCII characters +# © symbol (U+00A9) +print(b"\xc2\xa9".decode("utf-8")) +print("©".encode("utf-8")) + +# Test emoji 👍 (U+1F44D) +print(b"\xf0\x9f\x91\x8d".decode("utf-8")) +print("👍".encode("utf-8")) + +# Test ascii encode +print("abcde".encode("ascii")) + +# Test invalid decoded code points in repr fallback path. +print(repr(b"\xf4\x90\x80\x80".decode("utf-8"))) +print(repr(b"\xf5\x80\x80\x80".decode("utf-8"))) + +# Test invalid ASCII characters in decode +try: + print(repr(b"ni\xe5\xa5\xbd".decode("ascii"))) +except UnicodeError: + print("UnicodeError") + +# Test invalid encodings for bytes.decode() +# These should raise LookupError +invalid_encodings = ["latin-1", "latin1", "utf-16", "utf-32", "iso-8859-1", "cp1252"] + +for encoding in invalid_encodings: + try: + b"hello".decode(encoding) + print("UNEXPECTED:", encoding, "should raise LookupError") + except LookupError as e: + print("LookupError:", encoding) + +# Test bytes method accepting bytearray as argument (arg type normalization) +print(b"hello world".find(bytearray(b"world"))) +print(bytearray(b"hello world").find(bytearray(b"world"))) + +# Test invalid encodings for str.encode() +for encoding in invalid_encodings: + try: + "hello".encode(encoding) + print("UNEXPECTED:", encoding, "should raise LookupError") + except LookupError as e: + print("LookupError:", encoding) + +# Test invalid ascii characters in str.encode() +try: + "你好".encode("ascii") +except UnicodeError: + print("UnicodeError") diff --git a/tests/unicode/bytes_decode_encoding.py.exp b/tests/unicode/bytes_decode_encoding.py.exp new file mode 100644 index 00000000000..892ff019588 --- /dev/null +++ b/tests/unicode/bytes_decode_encoding.py.exp @@ -0,0 +1,30 @@ +hello +hello +hello +b'hello' +b'hello' +b'hello' +test +© +b'\xc2\xa9' +👍 +b'\xf0\x9f\x91\x8d' +b'abcde' +'\U00110000' +'\U00140000' +UnicodeError +LookupError: latin-1 +LookupError: latin1 +LookupError: utf-16 +LookupError: utf-32 +LookupError: iso-8859-1 +LookupError: cp1252 +6 +6 +LookupError: latin-1 +LookupError: latin1 +LookupError: utf-16 +LookupError: utf-32 +LookupError: iso-8859-1 +LookupError: cp1252 +UnicodeError diff --git a/tests/unicode/bytes_decode_ignore.py b/tests/unicode/bytes_decode_ignore.py new file mode 100644 index 00000000000..b42497c664d --- /dev/null +++ b/tests/unicode/bytes_decode_ignore.py @@ -0,0 +1,109 @@ +# Test bytes.decode() with error handler 'ignore' + +# Check if decode method is available (requires MICROPY_CPYTHON_COMPAT) +try: + b"".decode() +except AttributeError: + print("SKIP") + raise SystemExit + +# Check if error handlers are available (requires MICROPY_PY_BUILTINS_BYTES_DECODE_ERRORS) +# When feature is disabled, invalid UTF-8 raises LookupError even with 'ignore' +# When feature is enabled, invalid UTF-8 with 'ignore' returns a string +try: + result = b"\xff".decode("utf-8", "ignore") + # If we get here, feature is available +except (UnicodeError, LookupError): + # Feature not available - 'ignore' was ignored, strict mode was used + print("SKIP") + raise SystemExit + +# Test ignore mode with invalid UTF-8 +print(repr(b"\xff\xfe".decode("utf-8", "ignore"))) + +# Test strict mode (default) with invalid UTF-8 +try: + b"\xff\xfe".decode("utf-8") + print("UNEXPECTED") +except UnicodeError: + print("UnicodeError") + +# Test strict mode (explicit) with invalid UTF-8 +try: + b"\xff\xfe".decode("utf-8", "strict") + print("UNEXPECTED") +except UnicodeError: + print("UnicodeError") + +# Test with valid UTF-8 +print(repr(b"hello".decode("utf-8", "ignore"))) + +# Test valid UTF-8 with default mode +print(repr(b"hello".decode("utf-8"))) + +# Test mixed valid and invalid UTF-8 +print(repr(b"hello\xffworld".decode("utf-8", "ignore"))) + +# Test multiple invalid bytes +print(repr(b"\x80\x81\x82".decode("utf-8", "ignore"))) + +# Test invalid continuation byte +print(repr(b"\xc0\x20".decode("utf-8", "ignore"))) + +# Test incomplete sequence at end +print(repr(b"hello\xc0".decode("utf-8", "ignore"))) + +# Test valid multi-byte UTF-8 (© symbol) +print(repr(b"\xc2\xa9".decode("utf-8", "ignore"))) + +# Test bytearray support +print(repr(bytearray(b"\xff\xfe").decode("utf-8", "ignore"))) + +# Additional tests for continuation byte validation and incomplete sequences + +# Test 3-byte UTF-8 sequence - valid (e.g., U+4E00 - 一) +print(repr(b"\xe4\xb8\x80".decode("utf-8", "ignore"))) + +# Test 4-byte UTF-8 sequence - valid (e.g., U+1F600 - 😀) +print(repr(b"\xf0\x9f\x98\x80".decode("utf-8", "ignore"))) + +# Test incomplete 3-byte sequence (missing 2 continuation bytes) +print(repr(b"\xe4".decode("utf-8", "ignore"))) + +# Test incomplete 3-byte sequence (missing 1 continuation byte) +print(repr(b"\xe4\xb8".decode("utf-8", "ignore"))) + +# Test incomplete 4-byte sequence (missing 3 continuation bytes) +print(repr(b"\xf0".decode("utf-8", "ignore"))) + +# Test incomplete 4-byte sequence (missing 2 continuation bytes) +print(repr(b"\xf0\x9f".decode("utf-8", "ignore"))) + +# Test incomplete 4-byte sequence (missing 1 continuation byte) +print(repr(b"\xf0\x9f\x98".decode("utf-8", "ignore"))) + +# Test 3-byte sequence with invalid continuation byte (first byte invalid) +print(repr(b"\xe4\x20\x80".decode("utf-8", "ignore"))) + +# Test 3-byte sequence with invalid continuation byte (second byte invalid) +print(repr(b"\xe4\xb8\x20".decode("utf-8", "ignore"))) + +# Test 4-byte sequence with invalid continuation bytes +print(repr(b"\xf0\x20\x98\x80".decode("utf-8", "ignore"))) +print(repr(b"\xf0\x9f\x20\x80".decode("utf-8", "ignore"))) +print(repr(b"\xf0\x9f\x98\x20".decode("utf-8", "ignore"))) + +# Test mixed valid and incomplete sequences +print(repr(b"hello\xe4world".decode("utf-8", "ignore"))) +print(repr(b"hello\xf0world".decode("utf-8", "ignore"))) + +# Test valid multi-byte sequence mixed with invalid bytes (exercises got==need path) +print(repr(b"\xff\xc2\xa9".decode("utf-8", "ignore"))) # © preserved after invalid \xff +print(repr(b"\xff\xe4\xb8\x80".decode("utf-8", "ignore"))) # 一 preserved after invalid \xff + +# Test multiple incomplete sequences in a row +print(repr(b"\xe4\xf0\xe4".decode("utf-8", "ignore"))) + +# Test ignoring invalid ASCII +print(repr(b"\xe5\xa5\xbd".decode("ascii", "ignore"))) # valid utf8, not valid ascii +print(repr(b"a\xbb\xcc\xddef\x01".decode("ascii", "ignore"))) # fully invalid diff --git a/tests/unicode/bytes_decode_replace.py b/tests/unicode/bytes_decode_replace.py new file mode 100644 index 00000000000..36ba6551670 --- /dev/null +++ b/tests/unicode/bytes_decode_replace.py @@ -0,0 +1,130 @@ +# Test bytes.decode() with error handler 'replace' + +# Check if decode method is available (requires MICROPY_CPYTHON_COMPAT) +try: + b"".decode() +except AttributeError: + print("SKIP") + raise SystemExit + +# Check if error handlers are available (requires MICROPY_PY_BUILTINS_BYTES_DECODE_ERRORS) +# When feature is disabled, invalid UTF-8 raises UnicodeError even with 'replace' +# When feature is enabled, invalid UTF-8 with 'replace' returns a string +try: + result = b"\xff".decode("utf-8", "replace") + # If we get here, feature is available +except (UnicodeError, LookupError): + # Feature not available - 'replace' was ignored, strict mode was used + print("SKIP") + raise SystemExit + +# Test replace mode with invalid UTF-8 +print(repr(b"\xff\xfe".decode("utf-8", "replace"))) + +# Test strict mode (default) with invalid UTF-8 +try: + b"\xff\xfe".decode("utf-8") + print("UNEXPECTED") +except UnicodeError: + print("UnicodeError") + +# Test strict mode (explicit) with invalid UTF-8 +try: + b"\xff\xfe".decode("utf-8", "strict") + print("UNEXPECTED") +except UnicodeError: + print("UnicodeError") + +# Test with valid UTF-8 +print(repr(b"hello".decode("utf-8", "replace"))) + +# Test valid UTF-8 with default mode +print(repr(b"hello".decode("utf-8"))) + +# Test mixed valid and invalid UTF-8 +print(repr(b"hello\xffworld".decode("utf-8", "replace"))) + +# Test multiple invalid bytes +print(repr(b"\x80\x81\x82".decode("utf-8", "replace"))) + +# Test invalid continuation byte +print(repr(b"\xc0\x20".decode("utf-8", "replace"))) + +# Test incomplete sequence at end +print(repr(b"hello\xc0".decode("utf-8", "replace"))) + +# Test valid multi-byte UTF-8 (© symbol) +print(repr(b"\xc2\xa9".decode("utf-8", "replace"))) + +# Test bytearray support +print(repr(bytearray(b"\xff\xfe").decode("utf-8", "replace"))) + +# Test replace mode - should either work or raise NotImplementedError +try: + result = b"\xff\xfe".decode("utf-8", "replace") + print(repr(result)) +except LookupError: + print("LookupError") + +# Test replace with valid UTF-8 +try: + result = b"hello".decode("utf-8", "replace") + print(repr(result)) +except LookupError: + print("LookupError") + +# Test replace with mixed content +try: + result = b"hello\xffworld".decode("utf-8", "replace") + print(repr(result)) +except LookupError: + print("LookupError") + +# Additional tests for continuation byte validation and incomplete sequences + +# Test 3-byte UTF-8 sequence - valid (e.g., U+4E00 - 一) +print(repr(b"\xe4\xb8\x80".decode("utf-8", "replace"))) + +# Test 4-byte UTF-8 sequence - valid (e.g., U+1F600 - 😀) +print(repr(b"\xf0\x9f\x98\x80".decode("utf-8", "replace"))) + +# Test valid multi-byte sequence mixed with invalid bytes (exercises got==need path) +print(repr(b"\xff\xc2\xa9".decode("utf-8", "replace"))) # \ufffd + © after invalid \xff +print(repr(b"\xff\xe4\xb8\x80".decode("utf-8", "replace"))) # \ufffd + 一 after invalid \xff + +# Test incomplete 3-byte sequence (missing 2 continuation bytes) +print(repr(b"\xe4".decode("utf-8", "replace"))) + +# Test incomplete 3-byte sequence (missing 1 continuation byte) +print(repr(b"\xe4\xb8".decode("utf-8", "replace"))) + +# Test incomplete 4-byte sequence (missing 3 continuation bytes) +print(repr(b"\xf0".decode("utf-8", "replace"))) + +# Test incomplete 4-byte sequence (missing 2 continuation bytes) +print(repr(b"\xf0\x9f".decode("utf-8", "replace"))) + +# Test incomplete 4-byte sequence (missing 1 continuation byte) +print(repr(b"\xf0\x9f\x98".decode("utf-8", "replace"))) + +# Test 3-byte sequence with invalid continuation byte (first byte invalid) +print(repr(b"\xe4\x20\x80".decode("utf-8", "replace"))) + +# Test 3-byte sequence with invalid continuation byte (second byte invalid) +print(repr(b"\xe4\xb8\x20".decode("utf-8", "replace"))) + +# Test 4-byte sequence with invalid continuation bytes +print(repr(b"\xf0\x20\x98\x80".decode("utf-8", "replace"))) +print(repr(b"\xf0\x9f\x20\x80".decode("utf-8", "replace"))) +print(repr(b"\xf0\x9f\x98\x20".decode("utf-8", "replace"))) + +# Test mixed valid and incomplete sequences +print(repr(b"hello\xe4world".decode("utf-8", "replace"))) +print(repr(b"hello\xf0world".decode("utf-8", "replace"))) + +# Test multiple incomplete sequences in a row +print(repr(b"\xe4\xf0\xe4".decode("utf-8", "replace"))) + +# Test replacing invalid ASCII +print(repr(b"\xe5\xa5\xbd".decode("ascii", "replace"))) # valid utf8, not valid ascii +print(repr(b"a\xbb\xcc\xddef\x01".decode("ascii", "replace"))) # fully invalid diff --git a/tests/unicode/str_center.py b/tests/unicode/str_center.py new file mode 100644 index 00000000000..226074df4b7 --- /dev/null +++ b/tests/unicode/str_center.py @@ -0,0 +1,36 @@ +# Test str.center() with Unicode characters +# Issue #17827 + +try: + str.center +except AttributeError: + print("SKIP") + raise SystemExit + +# ASCII baseline +print("hello".center(10)) + +# Latin with accent (é is 2 bytes in UTF-8) +print("héllo".center(10)) + +# Chinese (each char is 3 bytes in UTF-8) +print("你好".center(10)) + +# Emoji (4 bytes in UTF-8) +print("🎉".center(5)) + +# German with umlaut +print("München".center(15)) + +# Cyrillic +print("Москва".center(12)) + +# Edge cases +print("test".center(4)) # Exact fit +print("test".center(3)) # String longer than width +print("x".center(1)) # Single char, exact fit +print("".center(5)) # Empty string + +# Mixed ASCII and Unicode +print("café".center(10)) +print("hello世界".center(12)) diff --git a/tests/unicode/str_from_buffer_errors.py b/tests/unicode/str_from_buffer_errors.py new file mode 100644 index 00000000000..64008738a1e --- /dev/null +++ b/tests/unicode/str_from_buffer_errors.py @@ -0,0 +1,39 @@ +# str(buffer, encoding, errors) should apply the 'ignore'/'replace' error +# handlers to any buffer source (array.array, memoryview, ...), the same way it +# does for bytes and bytearray. Companion to str_from_buffer_snapshot.py. + +try: + import array + + memoryview +except (ImportError, NameError): + print("SKIP") + raise SystemExit + +# Requires the decode error handlers (MICROPY_PY_BUILTINS_BYTES_DECODE_ERRORS). +# When the feature is disabled, invalid UTF-8 raises even with 'replace'. +try: + str(memoryview(b"\xff"), "utf-8", "replace") +except UnicodeError: + print("SKIP") + raise SystemExit + +# array.array holding invalid UTF-8 bytes (typecode "B" keeps this endian-neutral). +a = array.array("B", b"a\xffb\x80c") +print(repr(str(a, "utf-8", "replace"))) +print(repr(str(a, "utf-8", "ignore"))) + +# A memoryview over invalid UTF-8 (0xc3 0x28 is an invalid 2-byte sequence). +mv = memoryview(b"x\xc3\x28y") +print(repr(str(mv, "utf-8", "replace"))) +print(repr(str(mv, "utf-8", "ignore"))) + +# Strict decoding (the default) must still raise for a buffer source. +try: + str(memoryview(b"a\xffb"), "utf-8") + print("UNEXPECTED") +except UnicodeError: + print("UnicodeError") + +# A valid-UTF-8 buffer passes through an error handler unchanged ("café"). +print(repr(str(array.array("B", b"caf\xc3\xa9"), "utf-8", "replace"))) # codespell:ignore caf diff --git a/tests/unicode/str_from_buffer_snapshot.py b/tests/unicode/str_from_buffer_snapshot.py new file mode 100644 index 00000000000..c4e0a871471 --- /dev/null +++ b/tests/unicode/str_from_buffer_snapshot.py @@ -0,0 +1,30 @@ +# Companion to str_from_bytearray.py: str()/bytes() built from other mutable +# buffer sources (array.array and a writable memoryview) must also be independent +# snapshots, not views onto the source buffer. +try: + import array + + memoryview +except (ImportError, NameError): + print("SKIP") + raise SystemExit + +# Non-interned byte-literal payloads (with raw UTF-8 sequences) avoid relying on +# str.encode() to build the test data. + +# array.array is a mutable buffer source; str()/bytes() must snapshot the data. +# Content decodes to "array ünicode 🐍 target!". +a = array.array("B", b"array \xc3\xbcnicode \xf0\x9f\x90\x8d target!") +s = str(a, "utf-8") +b = bytes(a) +a[0] = ord("X") +print(s) +print(b) + +# A writable memoryview (here over an array) is also a mutable source. +# Content decodes to "memoryview över 🐍 python!". +a = array.array("B", b"memoryview \xc3\xb6ver \xf0\x9f\x90\x8d python!") +mv = memoryview(a) +s = str(mv, "utf-8") +a[0] = ord("X") +print(s) diff --git a/tests/unicode/str_from_bytearray.py b/tests/unicode/str_from_bytearray.py new file mode 100644 index 00000000000..12744da5c47 --- /dev/null +++ b/tests/unicode/str_from_bytearray.py @@ -0,0 +1,50 @@ +# Test that a str/bytes created from a bytearray is an independent snapshot, +# and not a view onto the (mutable) bytearray buffer. Mutating or resizing the +# source bytearray afterwards must not change the previously created object. + +# Skip if needed via `skip_bytearray` logic in run-tests.py + +# Non-interned payloads (byte literals) are used so the result isn't returned as +# an existing qstr. Every scenario is covered with plain-ASCII data so the core +# bytearray paths are exercised directly; a couple of cases embed raw UTF-8 byte +# sequences to also cover multi-byte Unicode decoding, without relying on +# str.encode() to build the test data. + +# str(bytearray, ...) then mutate the source in place. +ba = bytearray(b"the quick brown fox jumped over!") +s = str(ba, "utf-8") +ba[0] = ord("T") +print(s) +print(s[0]) + +# Same, but with multi-byte UTF-8 content: "café naïve 🐍 snapshot!". +ba = bytearray(b"caf\xc3\xa9 na\xc3\xafve \xf0\x9f\x90\x8d snapshot!") # codespell:ignore caf +s = str(ba, "utf-8") +ba[0] = ord("X") +print(s) + +# Overwriting every byte of the source must not change the str. +ba = bytearray(b"snapshot test one two three four!") +s = str(ba, "utf-8") +for i in range(len(ba)): + ba[i] = ord("x") +print(s) + +# Growing the bytearray reallocates its buffer; the str must stay intact. +ba = bytearray(b"grow test alpha bravo charlie delta!") +s = str(ba, "utf-8") +for _ in range(1000): + ba.append(ord("Z")) +print(s) + +# bytes(bytearray) is likewise an independent snapshot. +ba = bytearray(b"bytes snapshot alpha bravo charlie!") +b = bytes(ba) +ba[0] = ord("X") +print(b) + +# Same, but with multi-byte UTF-8 content: "bytes ünicode 🐍 snakes!". +ba = bytearray(b"bytes \xc3\xbcnicode \xf0\x9f\x90\x8d snakes!") +b = bytes(ba) +ba[0] = ord("X") +print(b) diff --git a/tests/unicode/unicode.py b/tests/unicode/unicode.py index 58d406e63eb..97f0599ab01 100644 --- a/tests/unicode/unicode.py +++ b/tests/unicode/unicode.py @@ -17,11 +17,6 @@ enc = s.encode() print(enc, enc.decode() == s) -# printing of unicode chars using repr -# NOTE: for some characters (eg \u10ff) we differ to CPython -print(repr("a\uffff")) -print(repr("a\U0001ffff")) - # test invalid escape code try: eval('"\\U00110000"') @@ -51,3 +46,13 @@ str(b"\xf0\xe0\xed\xe8", "utf8") except UnicodeError: print("UnicodeError") + +# test surrogate repr uses \uXXXX escape +print(repr(chr(0xD800))) + +# test str() from buffer-protocol object (memoryview) +print(str(memoryview(b"hello"), "utf-8")) +try: + str(memoryview(b"\xff"), "utf-8") +except UnicodeError: + print("UnicodeError") diff --git a/tests/unicode/unicode_char_format.py b/tests/unicode/unicode_char_format.py new file mode 100644 index 00000000000..3d56ae92afa --- /dev/null +++ b/tests/unicode/unicode_char_format.py @@ -0,0 +1,52 @@ +# test %c formatting with unicode characters (issue #3364) +# tests that character codes >= 128 are properly encoded as UTF-8 + +print("%c%c" % (0x3BC, 0x1F40D)) # Greek letter mu and snake emoji + +# ASCII character +print("%c" % 65) + +# 2-byte UTF-8 characters +print("%c" % 128) +print("%c" % 169) # copyright symbol © +print("%c" % 255) + +# 3-byte UTF-8 character +print("%c" % 0x4E00) # CJK ideograph 一 + +# 4-byte UTF-8 character +print("%c" % 0x1F600) # emoji 😀 + +# test with .format() method +print("{:c}".format(169)) +print("{:c}".format(0x4E00)) +print("{:c}{:c}".format(0x3BC, 0x1F40D)) + +# Test boundary values - valid maximum unicode codepoint +print("%c" % 0x10FFFF) # Last valid unicode codepoint + +# Test invalid codepoint - >= 0x110000 should raise OverflowError +try: + print("%c" % 0x110000) + print("UNEXPECTED: should have raised OverflowError") +except OverflowError: + print("OverflowError") + +try: + print("%c" % 0x110001) + print("UNEXPECTED: should have raised OverflowError") +except OverflowError: + print("OverflowError") + +# Test format() method with invalid codepoint +try: + print("{:c}".format(0x110000)) + print("UNEXPECTED: should have raised OverflowError") +except OverflowError: + print("OverflowError") + +try: + print("{:c}".format(0x200000)) + print("UNEXPECTED: should have raised OverflowError") +except OverflowError: + print("OverflowError") diff --git a/tests/unicode/unicode_char_format_fstring.py b/tests/unicode/unicode_char_format_fstring.py new file mode 100644 index 00000000000..9dac1052e07 --- /dev/null +++ b/tests/unicode/unicode_char_format_fstring.py @@ -0,0 +1,6 @@ +# test %c formatting with unicode characters in f-strings + +c = 169 +print(f"{c:c}") +c = 0x1F600 +print(f"{c:c}") diff --git a/tests/unix/extra_coverage.py.exp b/tests/unix/extra_coverage.py.exp index cdb11455694..d474af62ea0 100644 --- a/tests/unix/extra_coverage.py.exp +++ b/tests/unix/extra_coverage.py.exp @@ -33,6 +33,8 @@ abc # GC 0 0 +1 +1 # GC part 2 pass # tracked allocation @@ -45,6 +47,7 @@ m_tracked_head = 0 5 1 6 1 7 1 +1 0 1 1 1 2 1 @@ -54,17 +57,24 @@ m_tracked_head = 0 6 1 7 1 m_tracked_head = 0 +# tracked realloc +grow preserves data: 1 +shrink preserves data: 1 +realloc gc stable: 1 +realloc(NULL) ok: 1 +realloc(ptr, 0) returns NULL: 1 +m_tracked_head after cleanup: 0 # vstr tests sts test tes -RuntimeError: -RuntimeError: +RuntimeError: \$ +RuntimeError: \$ # repl ame__ -port +port \$ builtins micropython __future__ _asyncio _thread aesio array audiocore @@ -117,6 +127,13 @@ deadbeef 0deadbeef c0ffee 000c0ffee +# list argument helpers +TypeError: \$ +ValueError: \$ +mp_obj_list_ensure same list? 1 +mp_obj_list_optional_arg same list? 1 +mp_obj_list_optional_arg new list len 3 +mp_obj_list_optional_arg new list from NULL len 3 # runtime utils TypeError: unsupported type for __abs__: 'str' TypeError: unsupported types for __divmod__: 'str', 'str' @@ -127,11 +144,8 @@ OverflowError: overflow converting long int to machine word OverflowError: overflow converting long int to machine word TypeError: can't convert NoneType to int TypeError: can't convert NoneType to int -ValueError: +ValueError: \$ Warning: test -# binary -123 -456 # VM 2 1 # scheduler @@ -145,8 +159,8 @@ unlocked 1 2 3 -KeyboardInterrupt: -KeyboardInterrupt: +KeyboardInterrupt: \$ +KeyboardInterrupt: \$ 10 loop scheduled function diff --git a/tests/unix/ffi_callback.py b/tests/unix/ffi_callback.py index 21bfccf251e..b5dfe28b644 100644 --- a/tests/unix/ffi_callback.py +++ b/tests/unix/ffi_callback.py @@ -16,7 +16,7 @@ def ffi_open(names): raise err -libc = ffi_open(("libc.so", "libc.so.0", "libc.so.6", "libc.dylib")) +libc = ffi_open(("libc.so", "libc.so.0", "libc.so.6", "libc.so.7", "libc.dylib")) qsort = libc.func("v", "qsort", "piip") diff --git a/tests/unix/ffi_float.py b/tests/unix/ffi_float.py index 03bd9f7f17b..117db69dd6c 100644 --- a/tests/unix/ffi_float.py +++ b/tests/unix/ffi_float.py @@ -17,7 +17,7 @@ def ffi_open(names): raise err -libc = ffi_open(("libc.so", "libc.so.0", "libc.so.6", "libc.dylib")) +libc = ffi_open(("libc.so", "libc.so.0", "libc.so.6", "libc.so.7", "libc.dylib")) try: strtof = libc.func("f", "strtof", "sp") @@ -33,7 +33,9 @@ def ffi_open(names): print("%.6f" % strtod("1.23", None)) # test passing double and float args -libm = ffi_open(("libm.so", "libm.so.6", "libc.so.0", "libc.so.6", "libc.dylib")) +libm = ffi_open( + ("libm.so", "libm.so.5", "libm.so.6", "libc.so.0", "libc.so.6", "libc.so.7", "libc.dylib") +) tgamma = libm.func("d", "tgamma", "d") for fun_name in ("tgamma",): fun = globals()[fun_name] diff --git a/tools/ar_util.py b/tools/ar_util.py index b90d3790314..4eb41c93654 100644 --- a/tools/ar_util.py +++ b/tools/ar_util.py @@ -39,6 +39,10 @@ Archive = None +DEFAULT_CACHE_BASE_PATH = ".mpy_ld_cache" +DEFAULT_CACHE_PREFIX = "ar_" + + class PickleCache: def __init__(self, path, prefix=""): self.path = path @@ -63,11 +67,21 @@ def load(self, key): return pickle.load(f) -def cached(key, cache): +PICKLE_CACHE = None + + +def init_cache(path, prefix): + global PICKLE_CACHE + PICKLE_CACHE = PickleCache(path, prefix) + + +def cached(key, provider): def decorator(func): @functools.wraps(func) def wrapper(*args, **kwargs): cache_key = key(*args, **kwargs) + cache = provider() + assert cache is not None try: d = cache.load(cache_key) if d["key"] != cache_key: @@ -114,7 +128,7 @@ def _cache_key(self): sha.update(bytes.fromhex("00000000000000000000000000000001")) return sha.hexdigest() - @cached(key=_cache_key, cache=PickleCache(path=".mpy_ld_cache", prefix="ar_")) + @cached(key=_cache_key, provider=lambda: PICKLE_CACHE) def load_symbols(self): print("Loading", self.fn) objs = defaultdict(lambda: {"def": set(), "undef": set(), "weak": set()}) diff --git a/tools/boardgen.py b/tools/boardgen.py index 3723e7ce31b..6b295f484f3 100644 --- a/tools/boardgen.py +++ b/tools/boardgen.py @@ -475,6 +475,10 @@ def load_inputs(self, out_source): def generate_extra_files(self): pass + def print_pin_source(self, out_source): + for pin in self.available_pins(): + pin.print_source(out_source) + def main(self): parser = argparse.ArgumentParser(description="Generate board specific pin file") parser.add_argument("--board-csv") @@ -495,8 +499,7 @@ def main(self): self.load_inputs(out_source) # Allow a port to print arbitrary per-pin content. - for pin in self.available_pins(): - pin.print_source(out_source) + self.print_pin_source(out_source) # Print the tables and dictionaries. self.print_source(out_source) diff --git a/tools/ci.sh b/tools/ci.sh index f1edf936435..c01a8d4b928 100755 --- a/tools/ci.sh +++ b/tools/ci.sh @@ -29,6 +29,12 @@ function ci_gcc_riscv_setup { riscv64-unknown-elf-gcc --version } +function ci_gcc_ppc64_setup { + sudo apt-get update + sudo apt-get install gcc-powerpc64le-linux-gnu libc6-dev-ppc64el-cross + powerpc64le-linux-gnu-gcc --version +} + function ci_picotool_setup { # Manually installing picotool ensures we use a release version, and speeds up the build. git clone https://github.com/raspberrypi/pico-sdk.git @@ -42,8 +48,7 @@ function ci_picotool_setup { # c code formatting function ci_c_code_formatting_setup { - sudo apt-get update - sudo apt-get install uncrustify + pip install micropython-uncrustify==1.0.0.post1 uncrustify --version } @@ -91,7 +96,7 @@ function ci_code_size_build { # Override the list by setting PORTS_TO_CHECK in the environment before invoking ci. : ${PORTS_TO_CHECK:=bmus3xpdv} - SUBMODULES="lib/asf4 lib/berkeley-db-1.xx lib/btstack lib/cyw43-driver lib/lwip lib/mbedtls lib/micropython-lib lib/nxp_driver lib/pico-sdk lib/stm32lib lib/tinyusb" + SUBMODULES="lib/CMSIS_5 lib/CMSIS_6 lib/asf4 lib/berkeley-db-1.xx lib/btstack lib/cyw43-driver lib/lwip lib/mbedtls lib/micropython-lib lib/nxp_driver lib/pico-sdk lib/stm32lib lib/tinyusb" # Default GitHub pull request sets HEAD to a generated merge commit # between PR branch (HEAD^2) and base branch (i.e. master) (HEAD^1). @@ -158,7 +163,7 @@ function ci_mpy_format_setup { function ci_mpy_format_test { # Test mpy-tool.py dump feature on bytecode - python3 ./tools/mpy-tool.py -xd tests/frozen/frozentest.mpy + python3 ./tools/mpy-tool.py -xd tests/assets/frozentest.mpy # Build MicroPython ci_unix_standard_build @@ -167,7 +172,7 @@ function ci_mpy_format_test { export MICROPYPATH=. # Test mpy-tool.py running under MicroPython - $micropython ./tools/mpy-tool.py -x -d tests/frozen/frozentest.mpy + $micropython ./tools/mpy-tool.py -x -d tests/assets/frozentest.mpy # Test mpy-tool.py dump feature on native code make -C examples/natmod/features1 @@ -184,6 +189,21 @@ function ci_mpy_cross_debug_emitter { grep -E "ENTRY|EXIT" | wc -l | grep "^2$" } +######################################################################################## +# ports/alif + +function ci_alif_setup { + ci_gcc_arm_setup +} + +function ci_alif_ae3_build { + make ${MAKEOPTS} -C mpy-cross + make ${MAKEOPTS} -C ports/alif BOARD=OPENMV_AE3 MCU_CORE=M55_HP submodules + make ${MAKEOPTS} -C ports/alif BOARD=OPENMV_AE3 MCU_CORE=M55_HE submodules + make ${MAKEOPTS} -C ports/alif BOARD=OPENMV_AE3 MCU_CORE=M55_DUAL + make ${MAKEOPTS} -C ports/alif BOARD=ALIF_ENSEMBLE MCU_CORE=M55_DUAL USER_C_MODULES=../../examples/usercmodule +} + ######################################################################################## # ports/cc3200 @@ -197,18 +217,22 @@ function ci_cc3200_build { } ######################################################################################## -# ports/esp32 +# ports/embed -# GitHub tag of ESP-IDF to use for CI, extracted from the esp32 dependency lockfile -# This should end up as a tag name like vX.Y.Z -# (note: This hacky parsing can be replaced with 'yq' once Ubuntu >=24.04 is in use) -IDF_VER=v$(grep -A10 "idf:" ports/esp32/lockfiles/dependencies.lock.esp32 | grep "version:" | head -n1 | sed -E 's/ +version: //') -PYTHON=$(command -v python3 2> /dev/null) -PYTHON_VER=$(${PYTHON:-python} --version | cut -d' ' -f2) +function ci_embedding_build { + make ${MAKEOPTS} -C examples/embedding -f micropython_embed.mk + make ${MAKEOPTS} -C examples/embedding + ./examples/embedding/embed | grep "hello world" +} -export IDF_CCACHE_ENABLE=1 +######################################################################################## +# ports/esp32 function ci_esp32_idf_setup { + if [ -z "$IDF_VER" ]; then + echo "IDF_VER environment variable must be set before running" + return 1 + fi echo "Using ESP-IDF version $IDF_VER" git clone --depth 1 --branch $IDF_VER https://github.com/espressif/esp-idf.git # doing a treeless clone isn't quite as good as --shallow-submodules, but it @@ -227,23 +251,29 @@ function ci_esp32_build_common { make ${MAKEOPTS} -C ports/esp32 submodules } -function ci_esp32_build_cmod_spiram_s2 { +function ci_esp32_build_cmod_spiram_d2wd { ci_esp32_build_common + # Combined USER_C_MODULES + freeze manifest test on ESP32_GENERIC. make ${MAKEOPTS} -C ports/esp32 \ USER_C_MODULES=../../../examples/usercmodule/micropython.cmake \ - FROZEN_MANIFEST=$(pwd)/ports/esp32/boards/manifest_test.py + FROZEN_MANIFEST="$(pwd)/ports/esp32/boards/manifest_test.py" # Test building native .mpy with xtensawin architecture. ci_native_mpy_modules_build xtensawin - make ${MAKEOPTS} -C ports/esp32 BOARD=ESP32_GENERIC BOARD_VARIANT=SPIRAM - make ${MAKEOPTS} -C ports/esp32 BOARD=ESP32_GENERIC_S2 + # Test the c_module() codepath on the SPIRAM variant. + make ${MAKEOPTS} -C ports/esp32 BOARD=ESP32_GENERIC BOARD_VARIANT=SPIRAM \ + FROZEN_MANIFEST="$(pwd)/tests/tools/manifest_c_module.py" + + # D2WD is the variant with smallest application partition in flash + make ${MAKEOPTS} -C ports/esp32 BOARD=ESP32_GENERIC BOARD_VARIANT=D2WD } -function ci_esp32_build_s3_c3 { +function ci_esp32_build_s2_s3_c3 { ci_esp32_build_common + make ${MAKEOPTS} -C ports/esp32 BOARD=ESP32_GENERIC_S2 make ${MAKEOPTS} -C ports/esp32 BOARD=ESP32_GENERIC_S3 make ${MAKEOPTS} -C ports/esp32 BOARD=ESP32_GENERIC_C3 } @@ -256,9 +286,10 @@ function ci_esp32_build_c2_c5_c6 { make ${MAKEOPTS} -C ports/esp32 BOARD=ESP32_GENERIC_C6 } -function ci_esp32_build_p4 { +function ci_esp32_build_h2_p4 { ci_esp32_build_common + make ${MAKEOPTS} -C ports/esp32 BOARD=ESP32_GENERIC_H2 make ${MAKEOPTS} -C ports/esp32 BOARD=ESP32_GENERIC_P4 make ${MAKEOPTS} -C ports/esp32 BOARD=ESP32_GENERIC_P4 BOARD_VARIANT=C6_WIFI } @@ -283,31 +314,12 @@ function ci_esp8266_build { make ${MAKEOPTS} -C ports/esp8266 submodules make ${MAKEOPTS} -C ports/esp8266 BOARD=ESP8266_GENERIC make ${MAKEOPTS} -C ports/esp8266 BOARD=ESP8266_GENERIC BOARD_VARIANT=FLASH_512K - make ${MAKEOPTS} -C ports/esp8266 BOARD=ESP8266_GENERIC BOARD_VARIANT=FLASH_1M + make ${MAKEOPTS} -C ports/esp8266 BOARD=ESP8266_GENERIC BOARD_VARIANT=FLASH_1M USER_C_MODULES=../../examples/usercmodule # Test building native .mpy with xtensa architecture. ci_native_mpy_modules_build xtensa } -######################################################################################## -# ports/webassembly - -function ci_webassembly_setup { - npm install terser - git clone https://github.com/emscripten-core/emsdk.git - (cd emsdk && ./emsdk install latest && ./emsdk activate latest) -} - -function ci_webassembly_build { - source emsdk/emsdk_env.sh - make ${MAKEOPTS} -C ports/webassembly VARIANT=pyscript submodules - make ${MAKEOPTS} -C ports/webassembly VARIANT=pyscript -} - -function ci_webassembly_run_tests { - make -C ports/webassembly VARIANT=pyscript test_min -} - ######################################################################################## # ports/mimxrt @@ -320,7 +332,7 @@ function ci_mimxrt_build { make ${MAKEOPTS} -C ports/mimxrt BOARD=MIMXRT1020_EVK submodules make ${MAKEOPTS} -C ports/mimxrt BOARD=MIMXRT1020_EVK make ${MAKEOPTS} -C ports/mimxrt BOARD=TEENSY40 submodules - make ${MAKEOPTS} -C ports/mimxrt BOARD=TEENSY40 + make ${MAKEOPTS} -C ports/mimxrt BOARD=TEENSY40 USER_C_MODULES=../../examples/usercmodule make ${MAKEOPTS} -C ports/mimxrt BOARD=MIMXRT1060_EVK submodules make ${MAKEOPTS} -C ports/mimxrt BOARD=MIMXRT1060_EVK CFLAGS_EXTRA=-DMICROPY_HW_USB_MSC=1 } @@ -337,22 +349,24 @@ function ci_nrf_build { make ${MAKEOPTS} -C mpy-cross make ${MAKEOPTS} -C ports/nrf submodules make ${MAKEOPTS} -C ports/nrf BOARD=PCA10040 - make ${MAKEOPTS} -C ports/nrf BOARD=MICROBIT + make ${MAKEOPTS} -C ports/nrf BOARD=MICROBIT USER_C_MODULES=../../examples/usercmodule make ${MAKEOPTS} -C ports/nrf BOARD=PCA10056 SD=s140 make ${MAKEOPTS} -C ports/nrf BOARD=PCA10090 } ######################################################################################## -# ports/powerpc +# ports/psoc-edge -function ci_powerpc_setup { - sudo apt-get update - sudo apt-get install gcc-powerpc64le-linux-gnu libc6-dev-ppc64el-cross +function ci_psoc_edge_setup { + ci_gcc_arm_setup + sudo apt remove python3-packaging python3-jsonschema python3-cryptography + sudo pip3 install edgeprotecttools } -function ci_powerpc_build { - make ${MAKEOPTS} -C ports/powerpc UART=potato - make ${MAKEOPTS} -C ports/powerpc UART=lpc_serial +function ci_psoc_edge_build { + make ${MAKEOPTS} -C mpy-cross + make ${MAKEOPTS} -C ports/psoc-edge submodules + make ${MAKEOPTS} -C ports/psoc-edge } ######################################################################################## @@ -385,6 +399,13 @@ function ci_qemu_setup_rv64 { qemu-system-riscv64 --version } +function ci_qemu_setup_ppc64 { + ci_gcc_ppc64_setup + sudo apt-get update + sudo apt-get install qemu-system + qemu-system-ppc64 --version +} + function ci_qemu_build_arm_prepare { make ${MAKEOPTS} -C mpy-cross make ${MAKEOPTS} -C ports/qemu submodules @@ -402,7 +423,7 @@ function ci_qemu_build_arm_sabrelite { function ci_qemu_build_arm_thumb_softfp { ci_qemu_build_arm_prepare - make BOARD=MPS2_AN385 ${MAKEOPTS} -C ports/qemu test_full + make BOARD=MPS2_AN385 ${MAKEOPTS} -C ports/qemu USER_C_MODULES=../../examples/usercmodule test_full # Test building native .mpy with ARM-M softfp architectures. ci_native_mpy_modules_build armv6m @@ -446,6 +467,12 @@ function ci_qemu_build_rv64 { make ${MAKEOPTS} -C ports/qemu BOARD=VIRT_RV64 test_natmod } +function ci_qemu_build_ppc64 { + make ${MAKEOPTS} -C mpy-cross + make ${MAKEOPTS} -C ports/qemu BOARD=POWERNV9 submodules + make ${MAKEOPTS} -C ports/qemu BOARD=POWERNV9 test +} + ######################################################################################## # ports/renesas-ra @@ -458,7 +485,7 @@ function ci_renesas_ra_board_build { make ${MAKEOPTS} -C mpy-cross make ${MAKEOPTS} -C ports/renesas-ra submodules make ${MAKEOPTS} -C ports/renesas-ra BOARD=RA4M1_CLICKER - make ${MAKEOPTS} -C ports/renesas-ra BOARD=EK_RA6M2 + make ${MAKEOPTS} -C ports/renesas-ra BOARD=EK_RA6M2 USER_C_MODULES=../../examples/usercmodule make ${MAKEOPTS} -C ports/renesas-ra BOARD=EK_RA6M1 make ${MAKEOPTS} -C ports/renesas-ra BOARD=EK_RA4M1 make ${MAKEOPTS} -C ports/renesas-ra BOARD=EK_RA4W1 @@ -479,9 +506,11 @@ function ci_rp2_build { make ${MAKEOPTS} -C ports/rp2 submodules make ${MAKEOPTS} -C ports/rp2 make ${MAKEOPTS} -C ports/rp2 BOARD=RPI_PICO_W submodules + # Legacy USER_C_MODULES coverage on RPI_PICO_W. make ${MAKEOPTS} -C ports/rp2 BOARD=RPI_PICO_W USER_C_MODULES=../../examples/usercmodule/micropython.cmake make ${MAKEOPTS} -C ports/rp2 BOARD=RPI_PICO2 submodules - make ${MAKEOPTS} -C ports/rp2 BOARD=RPI_PICO2 + # Test c_module() on RPI_PICO2. + make ${MAKEOPTS} -C ports/rp2 BOARD=RPI_PICO2 FROZEN_MANIFEST="$(pwd)/tests/tools/manifest_c_module.py" make ${MAKEOPTS} -C ports/rp2 BOARD=W5100S_EVB_PICO submodules # This build doubles as a build test for disabling threads in the config make ${MAKEOPTS} -C ports/rp2 BOARD=W5100S_EVB_PICO CFLAGS_EXTRA=-DMICROPY_PY_THREAD=0 @@ -503,6 +532,7 @@ function ci_samd_build { make ${MAKEOPTS} -C ports/samd submodules make ${MAKEOPTS} -C ports/samd BOARD=ADAFRUIT_ITSYBITSY_M0_EXPRESS make ${MAKEOPTS} -C ports/samd BOARD=ADAFRUIT_ITSYBITSY_M4_EXPRESS + make ${MAKEOPTS} -C ports/samd BOARD=SPARKFUN_SAMD21_DEV_BREAKOUT } ######################################################################################## @@ -538,6 +568,12 @@ function ci_stm32_pyb_build { make ${MAKEOPTS} -C ports/stm32/mboot BOARD=STM32F769DISC CFLAGS_EXTRA='-DMBOOT_ADDRESS_SPACE_64BIT=1 -DMBOOT_SDCARD_ADDR=0x100000000ULL -DMBOOT_SDCARD_BYTE_SIZE=0x400000000ULL -DMBOOT_FSLOAD=1 -DMBOOT_VFS_FAT=1' } +function ci_stm32_build_cmod { + make ${MAKEOPTS} -C mpy-cross + make ${MAKEOPTS} -C ports/stm32 submodules + make ${MAKEOPTS} -C ports/stm32 BOARD=PYBV11 FROZEN_MANIFEST="$(pwd)/tests/tools/manifest_c_module.py" +} + function ci_stm32_nucleo_build { # This function builds the following MCU families: F0, H5, H7, L0, L4, WB. @@ -608,6 +644,18 @@ CI_UNIX_OPTS_QEMU_RISCV64=( MICROPY_STANDALONE=1 ) +CI_UNIX_OPTS_QEMU_LOONG64=( + CROSS_COMPILE=loongarch64-linux-gnu- + VARIANT=coverage + MICROPY_STANDALONE=1 +) + +CI_UNIX_OPTS_QEMU_X64=( + CROSS_COMPILE=x86_64-linux-gnu- + VARIANT=coverage + MICROPY_STANDALONE=1 +) + CI_UNIX_OPTS_SANITIZE_ADDRESS=( # Macro MP_ASAN allows detecting ASan on gcc<=13 CFLAGS_EXTRA="-fsanitize=address --param asan-use-after-return=0 -DMP_ASAN=1" @@ -623,9 +671,12 @@ CI_UNIX_OPTS_SANITIZE_UNDEFINED=( CI_UNIX_OPTS_REPR_B=( VARIANT=standard CFLAGS_EXTRA="-DMICROPY_OBJ_REPR=MICROPY_OBJ_REPR_B -DMICROPY_PY_UCTYPES=0 -Dmp_int_t=int32_t -Dmp_uint_t=uint32_t" - MICROPY_FORCE_32BIT=1 RUN_TESTS_MPY_CROSS_FLAGS="--mpy-cross-flags=\"-march=x86 -msmall-int-bits=30\"" +) +CI_UNIX_OPTS_X86=( + CROSS_COMPILE=i686-linux-gnu- + RUN_TESTS_MPY_CROSS_FLAGS=${RUN_TESTS_MPY_CROSS_FLAGS:-"--mpy-cross-flags=\"-march=x86\""} ) function ci_unix_build_helper { @@ -665,6 +716,13 @@ function ci_unix_run_tests_full_helper { ci_unix_run_tests_full_extra $micropython } +function ci_unix_run_native_mpy_tests_helper { + variant=$1 + shift + MICROPYPATH=examples/natmod/features2 ./ports/unix/build-$variant/micropython -m features2 + (cd tests && MICROPY_MICROPYTHON=../ports/unix/build-$variant/micropython ./run-natmodtests.py "$@" extmod/*.py) +} + function ci_native_mpy_modules_build { if [ "$1" = "" ]; then arch=x64 @@ -690,6 +748,14 @@ function ci_native_mpy_modules_32bit_build { ci_native_mpy_modules_build x86 } +function ci_native_mpy_modules_clang_build { + # This currently only supports the host architecture (assumed to be x64). + for natmod in btree deflate features1 features2 features3 features4 framebuf heapq random re + do + make -C examples/natmod/$natmod CC=clang + done +} + function ci_unix_minimal_build { make ${MAKEOPTS} -C ports/unix VARIANT=minimal } @@ -707,6 +773,10 @@ function ci_unix_standard_run_tests { ci_unix_run_tests_full_helper standard } +function ci_unix_standard_run_native_mpy_tests { + ci_unix_run_native_mpy_tests_helper standard "$@" +} + function ci_unix_standard_v2_build { ci_unix_build_helper VARIANT=standard MICROPY_PREVIEW_VERSION_2=1 ci_unix_build_ffi_lib_helper gcc @@ -716,6 +786,22 @@ function ci_unix_standard_v2_run_tests { ci_unix_run_tests_full_helper standard } +function ci_unix_standard_error_terse_build { + ci_unix_build_helper VARIANT=standard CFLAGS_EXTRA="-DMICROPY_ERROR_REPORTING=MICROPY_ERROR_REPORTING_TERSE" +} + +function ci_unix_standard_error_terse_run_tests { + make -C ports/unix VARIANT=standard test +} + +function ci_unix_standard_error_none_build { + ci_unix_build_helper VARIANT=standard CFLAGS_EXTRA="-DMICROPY_ERROR_REPORTING=MICROPY_ERROR_REPORTING_NONE" MICROPY_ROM_TEXT_COMPRESSION=0 +} + +function ci_unix_standard_error_none_run_tests { + make -C ports/unix VARIANT=standard test +} + function ci_unix_coverage_setup { pip3 install setuptools pip3 install pyelftools @@ -725,6 +811,7 @@ function ci_unix_coverage_setup { } function ci_unix_coverage_build { + # note: the coverage variant incorporates ../../examples/usercmodule, set in mpconfigvariant.mk ci_unix_build_helper VARIANT=coverage ci_unix_build_ffi_lib_helper gcc } @@ -741,6 +828,12 @@ function ci_unix_coverage_run_mpy_merge_tests { # Compile a selection of tests to .mpy and execute them, collecting the output. # None of the tests should SKIP. for inpy in $mptop/tests/basics/[acdel]*.py; do + if grep -q "import unittest" $inpy; then + # Merging >1 unittest-enabled module leads to unexpected + # results, as each file runs all previously registered unittest cases + echo "SKIPPING $inpy" + continue + fi test=$(basename $inpy .py) echo $test outmpy=$outdir/$test.mpy @@ -758,27 +851,26 @@ function ci_unix_coverage_run_mpy_merge_tests { } function ci_unix_coverage_run_native_mpy_tests { - MICROPYPATH=examples/natmod/features2 ./ports/unix/build-coverage/micropython -m features2 - (cd tests && ./run-natmodtests.py "$@" extmod/*.py) + ci_unix_run_native_mpy_tests_helper coverage "$@" } function ci_unix_32bit_setup { sudo dpkg --add-architecture i386 sudo apt-get update - sudo apt-get install gcc-multilib g++-multilib libffi-dev:i386 + sudo apt-get install gcc-i686-linux-gnu g++-i686-linux-gnu patchelf libffi-dev:i386 python -m pip install pyelftools python -m pip install ar - gcc --version + i686-linux-gnu-gcc --version python3 --version } function ci_unix_coverage_32bit_build { - ci_unix_build_helper VARIANT=coverage MICROPY_FORCE_32BIT=1 - ci_unix_build_ffi_lib_helper gcc -m32 + ci_unix_build_helper VARIANT=coverage "${CI_UNIX_OPTS_X86[@]}" + ci_unix_build_ffi_lib_helper i686-linux-gnu-gcc } function ci_unix_coverage_32bit_run_tests { - ci_unix_run_tests_full_helper coverage MICROPY_FORCE_32BIT=1 + ci_unix_run_tests_full_helper coverage "${CI_UNIX_OPTS_X86[@]}" } function ci_unix_coverage_32bit_run_native_mpy_tests { @@ -786,8 +878,8 @@ function ci_unix_coverage_32bit_run_native_mpy_tests { } function ci_unix_nanbox_build { - ci_unix_build_helper VARIANT=nanbox CFLAGS_EXTRA="-DMICROPY_PY_MATH_CONSTANTS=1" - ci_unix_build_ffi_lib_helper gcc -m32 + ci_unix_build_helper VARIANT=nanbox CFLAGS_EXTRA="-DMICROPY_PY_MATH_CONSTANTS=1" "${CI_UNIX_OPTS_X86[@]}" + ci_unix_build_ffi_lib_helper i686-linux-gnu-gcc } function ci_unix_nanbox_run_tests { @@ -795,7 +887,8 @@ function ci_unix_nanbox_run_tests { } function ci_unix_longlong_build { - ci_unix_build_helper VARIANT=longlong "${CI_UNIX_OPTS_SANITIZE_UNDEFINED[@]}" + ci_unix_build_helper VARIANT=longlong "${CI_UNIX_OPTS_SANITIZE_UNDEFINED[@]}" "${CI_UNIX_OPTS_X86[@]}" + patchelf --add-rpath "/usr/i686-linux-gnu/lib" ports/unix/build-longlong/micropython } function ci_unix_longlong_run_tests { @@ -824,6 +917,7 @@ function ci_unix_gil_enabled_run_tests { function ci_unix_clang_setup { sudo apt-get update sudo apt-get install clang + pip3 install ar pyelftools clang --version } @@ -967,15 +1061,92 @@ function ci_unix_qemu_riscv64_run_tests { popd } +function ci_unix_qemu_loong64_setup { + sudo apt-get update + sudo apt-get install gcc-14-loongarch64-linux-gnu g++-14-loongarch64-linux-gnu libc6-loong64-cross libltdl-dev + sudo apt-get install qemu-user-static + qemu-loongarch64-static --version + sudo mkdir -p /usr/gnemul + sudo ln -s /usr/loongarch64-linux-gnu /usr/gnemul/qemu-loongarch64 + sudo ln -s /usr/bin/loongarch64-linux-gnu-gcc-14 /usr/bin/loongarch64-linux-gnu-gcc + sudo ln -s /usr/bin/loongarch64-linux-gnu-g++-14 /usr/bin/loongarch64-linux-gnu-g++ +} + +function ci_unix_qemu_loong64_build { + ci_unix_build_helper "${CI_UNIX_OPTS_QEMU_LOONG64[@]}" + ci_unix_build_ffi_lib_helper loongarch64-linux-gnu-gcc +} + +function ci_unix_qemu_loong64_run_tests { + # Issues with LOONG64 tests: + # - thread/stress_aes.py takes around 90 seconds + file ./ports/unix/build-coverage/micropython + # Loongarch64 isn't defined in the CI image's binfmt list. + cat << 'EOF' > ./ports/unix/build-coverage/micropython-runner +#!/bin/sh +MPY_PATH=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd -P) +qemu-loongarch64-static "$MPY_PATH"/micropython $@ +EOF + chmod +x ./ports/unix/build-coverage/micropython-runner + (cd tests && MICROPY_MICROPYTHON=../ports/unix/build-coverage/micropython-runner MICROPY_TEST_TIMEOUT=180 ./run-tests.py) +} + +function ci_unix_qemu_x64_setup { + sudo apt-get update + sudo apt-get install gcc-x86-64-linux-gnu g++-x86-64-linux-gnu libc6-amd64-cross libltdl-dev + sudo apt-get install qemu-user-static + python3 -m pip install pyelftools + python3 -m pip install ar + qemu-x86_64-static --version + sudo mkdir -p /usr/gnemul + sudo ln -s /usr/x86_64-linux-gnu /usr/gnemul/qemu-x86_64 +} + +function ci_unix_qemu_x64_build { + ci_unix_build_helper "${CI_UNIX_OPTS_QEMU_X64[@]}" + ci_unix_build_ffi_lib_helper x86_64-linux-gnu-gcc + ci_native_mpy_modules_build x64 +} + +function ci_unix_qemu_x64_run_tests { + # Issues with x64 tests on non-x64 hosts: + # - thread/stress_aes.py takes around 90 seconds + # - ports/unix/ffi_callback.py crashes QEMU (x86_64-binfmt-P: QEMU internal SIGSEGV {code=MAPERR, addr=0x20}) + file ./ports/unix/build-coverage/micropython + pushd tests + MICROPY_MICROPYTHON=../ports/unix/build-coverage/micropython ./run-tests.py --exclude '(thread/stress_aes.py|ports/unix/ffi_callback.py)' + MICROPY_MICROPYTHON=../ports/unix/build-coverage/micropython ./run-natmodtests.py extmod/btree*.py extmod/deflate*.py extmod/framebuf*.py extmod/heapq*.py extmod/random_basic*.py extmod/re*.py + popd +} + function ci_unix_repr_b_build { - ci_unix_build_helper "${CI_UNIX_OPTS_REPR_B[@]}" - ci_unix_build_ffi_lib_helper gcc -m32 + ci_unix_build_helper "${CI_UNIX_OPTS_REPR_B[@]}" "${CI_UNIX_OPTS_X86[@]}" + ci_unix_build_ffi_lib_helper i686-linux-gnu-gcc } function ci_unix_repr_b_run_tests { # ci_unix_run_tests_full_no_native_helper is not used due to # https://github.com/micropython/micropython/issues/18105 - ci_unix_run_tests_helper "${CI_UNIX_OPTS_REPR_B[@]}" + ci_unix_run_tests_helper "${CI_UNIX_OPTS_REPR_B[@]}" "${CI_UNIX_OPTS_X86[@]}" +} + +######################################################################################## +# ports/webassembly + +function ci_webassembly_setup { + npm install terser + git clone https://github.com/emscripten-core/emsdk.git + (cd emsdk && ./emsdk install latest && ./emsdk activate latest) +} + +function ci_webassembly_build { + source emsdk/emsdk_env.sh + make ${MAKEOPTS} -C ports/webassembly VARIANT=pyscript submodules + make ${MAKEOPTS} -C ports/webassembly VARIANT=pyscript +} + +function ci_webassembly_run_tests { + make -C ports/webassembly VARIANT=pyscript test_min } ######################################################################################## @@ -983,22 +1154,22 @@ function ci_unix_repr_b_run_tests { function ci_windows_setup { sudo apt-get update - sudo apt-get install gcc-mingw-w64 + sudo apt-get install gcc-mingw-w64 g++-mingw-w64 } function ci_windows_build { make ${MAKEOPTS} -C mpy-cross make ${MAKEOPTS} -C ports/windows submodules - make ${MAKEOPTS} -C ports/windows CROSS_COMPILE=i686-w64-mingw32- + make ${MAKEOPTS} -C ports/windows CROSS_COMPILE=i686-w64-mingw32- USER_C_MODULES=../../examples/usercmodule make ${MAKEOPTS} -C ports/windows CROSS_COMPILE=x86_64-w64-mingw32- BUILD=build-standard-w64 } ######################################################################################## # ports/zephyr -ZEPHYR_DOCKER_VERSION=v0.28.1 -ZEPHYR_SDK_VERSION=0.17.2 -ZEPHYR_VERSION=v4.2.0 +ZEPHYR_DOCKER_VERSION=v0.29.2 +ZEPHYR_SDK_VERSION=1.0.1 +ZEPHYR_VERSION=v4.4.0 function ci_zephyr_setup { IMAGE=ghcr.io/zephyrproject-rtos/ci:${ZEPHYR_DOCKER_VERSION} @@ -1038,9 +1209,9 @@ function ci_zephyr_install { function ci_zephyr_build { git submodule update --init lib/micropython-lib - docker exec zephyr-ci west build -p auto -b qemu_x86 -- -DCONF_FILE=prj_minimal.conf + docker exec zephyr-ci west build -p auto -b qemu_x86 -- -DCONF_FILE='prj_minimal.conf;boards/qemu_x86.conf' docker exec zephyr-ci west build -p auto -b frdm_k64f - docker exec zephyr-ci west build -p auto -b mimxrt1050_evk + docker exec zephyr-ci west build -p auto -b mimxrt1050_evk/mimxrt1052/qspi docker exec zephyr-ci west build -p auto -b nucleo_wb55rg # for bluetooth } @@ -1050,19 +1221,7 @@ function ci_zephyr_run_tests { } ######################################################################################## -# ports/alif - -function ci_alif_setup { - ci_gcc_arm_setup -} - -function ci_alif_ae3_build { - make ${MAKEOPTS} -C mpy-cross - make ${MAKEOPTS} -C ports/alif BOARD=OPENMV_AE3 MCU_CORE=M55_HP submodules - make ${MAKEOPTS} -C ports/alif BOARD=OPENMV_AE3 MCU_CORE=M55_HE submodules - make ${MAKEOPTS} -C ports/alif BOARD=OPENMV_AE3 MCU_CORE=M55_DUAL - make ${MAKEOPTS} -C ports/alif BOARD=ALIF_ENSEMBLE MCU_CORE=M55_DUAL -} +# Helpers to run this script as a CLI tool. function _ci_help { # Note: these lines must be indented with tab characters (required by bash <<-EOF) diff --git a/tools/makemanifest.py b/tools/makemanifest.py index 860935397af..a149755c86d 100644 --- a/tools/makemanifest.py +++ b/tools/makemanifest.py @@ -136,6 +136,9 @@ def main(): ) cmd_parser.add_argument("-v", "--var", action="append", help="variables to substitute") cmd_parser.add_argument("--mpy-tool-flags", default="", help="flags to pass to mpy-tool") + cmd_parser.add_argument( + "--list-c-modules", action="store_true", help="list C module paths from manifest and exit" + ) cmd_parser.add_argument("files", nargs="+", help="input manifest list") args = cmd_parser.parse_args() @@ -150,6 +153,32 @@ def main(): print("MPY_DIR and PORT_DIR variables must be specified") sys.exit(1) + # Use a lighter mode when only listing C modules: freeze/package/module + # become no-ops so the listing doesn't fail on freeze targets that may not + # exist for every variant of the board (e.g. alif's modules/$(MCU_CORE)). + mode = manifestfile.MODE_LIST_C_MODULES if args.list_c_modules else manifestfile.MODE_FREEZE + manifest = manifestfile.ManifestFile(mode, VARS) + + # Include top-level inputs, to generate the manifest + for input_manifest in args.files: + try: + manifest.execute(input_manifest) + except manifestfile.ManifestFileError as er: + print( + 'manifest error executing "{}": {}'.format(input_manifest, er.args[0]), + file=sys.stderr, + ) + sys.exit(1) + + # If we're just listing C modules, output them and exit + if args.list_c_modules: + c_modules = manifest.c_modules() + if c_modules: + # Output one path per line to handle paths with spaces + for module in c_modules: + print(module) + sys.exit(0) + # Get paths to tools MPY_CROSS = VARS["MPY_DIR"] + "/mpy-cross/build/mpy-cross" if sys.platform == "win32": @@ -162,16 +191,6 @@ def main(): print("mpy-cross not found at {}, please build it first".format(MPY_CROSS)) sys.exit(1) - manifest = manifestfile.ManifestFile(manifestfile.MODE_FREEZE, VARS) - - # Include top-level inputs, to generate the manifest - for input_manifest in args.files: - try: - manifest.execute(input_manifest) - except manifestfile.ManifestFileError as er: - print('freeze error executing "{}": {}'.format(input_manifest, er.args[0])) - sys.exit(1) - # Process the manifest str_paths = [] mpy_files = [] diff --git a/tools/manifestfile.py b/tools/manifestfile.py index 9c7a6e140f9..2e2ade6783c 100644 --- a/tools/manifestfile.py +++ b/tools/manifestfile.py @@ -27,6 +27,7 @@ import contextlib import os +import re import sys import glob import tempfile @@ -40,6 +41,9 @@ MODE_COMPILE = 2 # Same as compile, but handles require(..., pypi="name") as a requirements.txt entry. MODE_PYPROJECT = 3 +# Same surface as MODE_FREEZE, but freeze/package/module are no-ops so callers +# can collect c_module() entries without touching files referenced by freeze(). +MODE_LIST_C_MODULES = 4 # In compile mode, .py -> KIND_COMPILE_AS_MPY # In freeze mode, .py -> KIND_FREEZE_AS_MPY, .mpy->KIND_FREEZE_MPY @@ -192,6 +196,8 @@ def __init__(self, mode, path_vars=None): self._manifest_files = [] # List of PyPI dependencies (when mode=MODE_PYPROJECT). self._pypi_dependencies = [] + # List of C module directories. + self._c_modules = [] # Don't allow including the same file twice. self._visited = set() # Stack of metadata for each level. @@ -201,7 +207,9 @@ def __init__(self, mode, path_vars=None): # List of directories to search for packages. self._library_dirs = [] # Add default micropython-lib libraries if $(MPY_LIB_DIR) has been specified. - if self._path_vars["MPY_LIB_DIR"]: + # Use .get() to avoid KeyError if MPY_LIB_DIR wasn't passed (shouldn't happen + # in normal cmake/make builds, but be defensive for direct tool invocations). + if self._path_vars.get("MPY_LIB_DIR"): for lib in BASE_LIBRARY_NAMES: self.add_library(lib, os.path.join("$(MPY_LIB_DIR)", lib)) @@ -221,11 +229,12 @@ def _manifest_globals(self, kwargs): "add_library": self.add_library, "package": self.package, "module": self.module, + "c_module": self.c_module, "options": IncludeOptions(**kwargs), } # Extra legacy functions only for freeze mode. - if self._mode == MODE_FREEZE: + if self._mode in (MODE_FREEZE, MODE_LIST_C_MODULES): g.update( { "freeze": self.freeze, @@ -244,6 +253,9 @@ def pypi_dependencies(self): # In MODE_PYPROJECT, this will return a list suitable for requirements.txt. return self._pypi_dependencies + def c_modules(self): + return self._c_modules + def execute(self, manifest_file): if manifest_file.endswith(".py"): # Execute file from filesystem. @@ -478,6 +490,9 @@ def package(self, package_path, files=None, base_path=".", opt=None): """ self._metadata[-1].check_initialised(self._mode) + if self._mode == MODE_LIST_C_MODULES: + return + # Include "base_path/package_path/**/*.py" --> "package_path/**/*.py" self._search(base_path, package_path, files, exts=(".py",), kind=KIND_AUTO, opt=opt) @@ -493,6 +508,9 @@ def module(self, module_path, base_path=".", opt=None): """ self._metadata[-1].check_initialised(self._mode) + if self._mode == MODE_LIST_C_MODULES: + return + # Include "base_path/module_path" --> "module_path" base_path = self._resolve_path(base_path) _, ext = os.path.splitext(module_path) @@ -501,7 +519,56 @@ def module(self, module_path, base_path=".", opt=None): # TODO: version None self._add_file(os.path.join(base_path, module_path), module_path, opt=opt) + def c_module(self, module_path): + """ + Include a C module directory in the build. + + The module_path should be a directory containing a micropython.mk and/or + micropython.cmake file. + + Supports $(VAR) path substitution: + c_module("$(MPY_DIR)/examples/usercmodule/cexample") + c_module("$(BOARD_DIR)/../../drivers/sensor") + + Can be called multiple times to include multiple C modules. + + Only has effect when collecting modules for the build system + (MODE_FREEZE / MODE_LIST_C_MODULES). Silent no-op in MODE_COMPILE / + MODE_PYPROJECT, so the same manifest can be evaluated in any mode. + """ + if self._mode not in (MODE_FREEZE, MODE_LIST_C_MODULES): + return + resolved = self._resolve_path(module_path) + # Reject unresolved $(VAR) up front so a bad manifest fails with a + # clear message rather than a confusing "path does not exist" containing + # the variable literal. + unresolved = re.search(r"\$\([^)]+\)", resolved) + if unresolved: + raise ManifestFileError( + "Unresolved variable in c_module() path: {}".format(unresolved.group(0)) + ) + module_path = resolved + if not os.path.exists(module_path): + raise ManifestFileError("C module path does not exist: {}".format(module_path)) + if not os.path.isdir(module_path): + raise ManifestFileError("C module path must be a directory: {}".format(module_path)) + # Verify the directory contains a micropython.mk or micropython.cmake file. + has_mk = os.path.isfile(os.path.join(module_path, "micropython.mk")) + has_cmake = os.path.isfile(os.path.join(module_path, "micropython.cmake")) + if not has_mk and not has_cmake: + raise ManifestFileError( + "C module directory must contain micropython.mk or micropython.cmake: {}".format( + module_path + ) + ) + self._c_modules.append(module_path) + def _freeze_internal(self, path, script, exts, kind, opt): + # In list-c-modules mode we only care about c_module() entries; skip + # the file-system walk so freeze targets that don't exist (e.g. a + # board-conditional modules directory) don't abort the listing. + if self._mode == MODE_LIST_C_MODULES: + return if script is None: self._search(path, None, None, exts=exts, kind=kind, opt=opt) elif isinstance(script, str) and os.path.isdir(os.path.join(path, script)): @@ -555,6 +622,8 @@ def freeze_as_str(self, path): Freeze the given `path` and all .py scripts within it as a string, which will be compiled upon import. """ + if self._mode == MODE_LIST_C_MODULES: + return self._search(path, None, None, exts=(".py",), kind=KIND_FREEZE_AS_STR) def freeze_as_mpy(self, path, script=None, opt=None): diff --git a/tools/mpy-tool.py b/tools/mpy-tool.py index bf0b89018e4..c80344a2b46 100755 --- a/tools/mpy-tool.py +++ b/tools/mpy-tool.py @@ -1511,6 +1511,8 @@ def read_raw_code(reader, parent_name, qstr_table, obj_table, segments): def read_mpy(filename): + import re + with open(filename, "rb") as fileobj: reader = MPYReader(filename, fileobj) segments = [] @@ -1556,7 +1558,10 @@ def read_mpy(filename): obj_table.append(read_obj(reader, segments)) # Compute the compiled-module escaped name. - cm_escaped_name = qstr_table[0].str.replace("/", "_")[:-3] + module_name = qstr_table[0].str + if not all(0x7F >= ord(codepoint) >= 0x20 for codepoint in module_name): + raise MPYReadError(filename, "cannot have unicode characters in module name") + cm_escaped_name = re.sub("[^a-zA-Z0-9_]", "_", "_".join(module_name.split(".")[:-1])) # Read the outer raw code, which will in turn read all its children. raw_code_file_offset = reader.tell() @@ -1737,7 +1742,7 @@ def freeze_mpy(firmware_qstr_idents, compiled_modules): if module_name.endswith("/__init__.py"): short_name = module_name[: -len("/__init__.py")] else: - short_name = module_name[: -len(".py")] + short_name = ".".join(module_name.split(".")[:-1]) print('MICROPY_FROZEN_LIST_ITEM("%s", "%s")' % (short_name, module_name)) print("#endif") diff --git a/tools/mpy_ld.py b/tools/mpy_ld.py index 56734b3d170..abb9ab4f4bd 100755 --- a/tools/mpy_ld.py +++ b/tools/mpy_ld.py @@ -29,7 +29,7 @@ """ import sys, os, struct, re -from elftools.elf import elffile +from elftools.elf import elffile, relocation import ar_util sys.path.append(os.path.dirname(__file__) + "/../py") @@ -123,12 +123,14 @@ R_RISCV_32_PCREL = 57 R_RISCV_PLT32 = 59 R_XTENSA_PDIFF32 = 59 +R_XTENSA_NDIFF32 = 62 R_RISCV_SET_ULEB128 = 60 R_RISCV_SUB_ULEB128 = 61 R_RISCV_TLSDESC_HI20 = 62 R_RISCV_TLSDESC_LOAD_LO12 = 63 R_RISCV_TLSDESC_ADD_LO12 = 64 R_RISCV_TLSDESC_CALL = 65 +R_ARM_GOT_PREL = 96 ################################################################################ # Architecture configuration @@ -143,10 +145,16 @@ def fit_signed(bits, value): def asm_jump_x86(entry): - return struct.pack("> 1) & 0x07FF) @@ -156,7 +164,7 @@ def asm_jump_thumb(entry): # push {r0, lr} # bl # pop {r0, pc} - entry += 2 # skip "push {r0, lr}" + entry -= 2 # skip "push {r0, lr}" b0 = 0xB400 | 0x0100 | 0x0001 # push, lr, r0 b1 = 0xF000 | ((entry >> 12) & 0x07FF) b2 = 0xF800 | ((entry >> 1) & 0x07FF) @@ -165,6 +173,7 @@ def asm_jump_thumb(entry): def asm_jump_thumb2(entry): + entry -= 4 if fit_signed(11, entry): # Signed value fits in 12 bits b0 = 0xE000 | ((entry >> 1) & 0x07FF) @@ -177,36 +186,96 @@ def asm_jump_thumb2(entry): def asm_jump_xtensa(entry): - if fit_signed(17, entry): - jump_op = (entry - 4) << 6 | 6 + if fit_signed(17, entry - 8): + jump_op = ((entry - 8) << 6) | 6 return struct.pack("> 8) else: - raise LinkError("Large jumps are not yet supported on Xtensa") + raise LinkError("jumps larger than 128KiB are not supported") def asm_jump_riscv(entry): if fit_signed(11, entry): - entry += 2 # c.j entry return struct.pack( "> 2) + | ((entry & 0x300) << 1) | ((entry & 0x80) >> 1) | ((entry & 0x40) << 1) | ((entry & 0x20) >> 3) - | ((entry & 0x10) << 7), + | ((entry & 0x10) << 7) + | ((entry & 0x0E) << 2), ) - else: + elif fit_signed(31, entry - 8): # auipc t6, HI(entry) # jalr zero, t6, LO(entry) - upper, lower = split_riscv_address(entry + 8) + upper, lower = split_riscv_address(entry) return struct.pack( "") + env.sections.insert(1, section) + r = relocation.Relocation({}, None) + r.index = get_memx_function_index(n) + r.offset = reloc + section.reloc.append(r) + section.reloc_name = "unknown" + sym.section = section else: undef_errors.append("{}: undefined symbol: {}".format(sym.filename, sym.name)) @@ -1285,8 +1426,24 @@ def link_objects(env, native_qstr_vals_len): raise LinkError("\n".join(undef_errors)) # Generate the entry trampoline assuming the offset is already known. - env.entry_point = env.find_entry_addr() - jump = env.arch.asm_jump(env.entry_point) + + text_alignment = env.find_sym("mpy_init").section.alignment + if env.arch.name in ("EM_386", "EM_X86_64"): + show_warning = text_alignment not in (1, 4) + elif env.arch.name in ("EM_ARM", "EM_XTENSA"): + show_warning = text_alignment != 4 + elif env.arch.name == "EM_RISCV": + show_warning = text_alignment != 2 + else: + show_warning = True + + if show_warning: + log( + LOG_LEVEL_1, + f"A .text section with an alignment of {text_alignment} bytes for {env.arch.name} is not tested and may not work", + ) + + jump = generate_entry_point_jump(env) env.entry_trampoline_len = len(jump) # Align sections, assign their addresses, and create full_text @@ -1323,6 +1480,8 @@ def link_objects(env, native_qstr_vals_len): do_relocation_text(env, sec.addr, r) elif sec.name.startswith(".data.rel.ro"): do_relocation_data(env, sec.addr, r) + elif sec.name.startswith(".internal"): + env.mpy_relocs.append((".text", sec.addr + r.offset, r.index)) else: assert 0, sec.name @@ -1330,6 +1489,15 @@ def link_objects(env, native_qstr_vals_len): ################################################################################ # .mpy output +MP_FUN_TABLE_MEMSET = 50 +MP_FUN_TABLE_MEMMOVE = 51 + + +def get_memx_function_index(f): + if f == "memset": + return MP_FUN_TABLE_MEMSET + return MP_FUN_TABLE_MEMMOVE + class MPYOutput: def open(self, fname): @@ -1362,6 +1530,12 @@ def write_qstr(self, s): self.write_bytes(b"\x00") def write_reloc(self, base, offset, dest, n): + if dest > 2 and n > 1: + # dest>2 cannot encode n, so do it manually. + for _ in range(n): + self.write_reloc(base, offset, dest, 1) + offset += 1 + return need_offset = not (base == self.prev_base and offset == self.prev_offset + 1) self.prev_offset = offset + n - 1 if dest <= 2: @@ -1382,11 +1556,11 @@ def write_reloc(self, base, offset, dest, n): self.write_uint(n) -def build_mpy(env, fmpy, native_qstr_vals, arch_flags): +def build_mpy(env, fmpy, internal_name, native_qstr_vals, arch_flags): # Rewrite the entry trampoline if the proper value isn't known earlier, and # ensure the trampoline size remains the same. if env.arch.delayed_entry_offset: - jump = env.arch.asm_jump(env.find_entry_addr()) + jump = generate_entry_point_jump(env) env.full_text[: len(jump)] = jump assert len(jump) == env.entry_trampoline_len @@ -1421,7 +1595,7 @@ def build_mpy(env, fmpy, native_qstr_vals, arch_flags): out.write_uint(0) # MPY: qstr table - out.write_qstr(fmpy) # filename + out.write_qstr(internal_name) # filename for q in native_qstr_vals: out.write_qstr(q) @@ -1507,19 +1681,13 @@ def do_preprocess(args): args.output = args.files[0][:-1] + "config.h" static_qstrs, qstr_vals = extract_qstrs(args.files) with open(args.output, "w") as f: - print( - "#include \n" - "typedef uintptr_t mp_uint_t;\n" - "typedef intptr_t mp_int_t;\n" - "typedef uintptr_t mp_off_t;", - file=f, - ) + print("#include \ntypedef uintptr_t mp_off_t;", file=f) for i, q in enumerate(static_qstrs): print("#define %s (%u)" % (q, i + 1), file=f) for i, q in enumerate(sorted(qstr_vals)): print("#define %s (mp_native_qstr_table[%d])" % (q, i + 1), file=f) print("extern const uint16_t mp_native_qstr_table[];", file=f) - print("extern const mp_uint_t mp_native_obj_table[];", file=f) + print("extern const uintptr_t mp_native_obj_table[];", file=f) def do_link(args): @@ -1545,6 +1713,9 @@ def do_link(args): load_object_file(env, f, fn) if args.libs: + ar_util.init_cache( + f"{ar_util.DEFAULT_CACHE_BASE_PATH}-{args.arch}", ar_util.DEFAULT_CACHE_PREFIX + ) # Load archive info archives = [] for item in args.libs: @@ -1561,7 +1732,14 @@ def do_link(args): load_object_file(env, f, obj_name) link_objects(env, len(native_qstr_vals)) - build_mpy(env, args.output, native_qstr_vals, args.arch_flags) + if args.source_name: + internal_name = args.source_name + else: + import pathlib + + path = pathlib.Path(args.output) + internal_name = path.name + build_mpy(env, args.output, internal_name, native_qstr_vals, args.arch_flags) except LinkError as er: print("LinkError:", er.args[0]) sys.exit(1) @@ -1649,6 +1827,9 @@ def main(): ) cmd_parser.add_argument("--arch", default="x64", help="architecture") cmd_parser.add_argument("--arch-flags", default=None, help="optional architecture flags") + cmd_parser.add_argument( + "--source-name", default=None, help="override the file name written to the .mpy file" + ) cmd_parser.add_argument("--preprocess", action="store_true", help="preprocess source files") cmd_parser.add_argument("--qstrs", default=None, help="file defining additional qstrs") cmd_parser.add_argument( diff --git a/tools/pyboard.py b/tools/pyboard.py index c9f65d5d873..655e35a3a55 100755 --- a/tools/pyboard.py +++ b/tools/pyboard.py @@ -51,6 +51,7 @@ import ast import errno import os +import stat import struct import sys import time @@ -314,6 +315,23 @@ def __init__( if delayed: print("") + if device.startswith("execpty:"): + self.is_pty = True + else: + self.is_pty = self._is_pty_device(device) + + @staticmethod + def _is_pty_device(device): + """Detect if device is a PTY (pseudo-terminal), e.g. used by QEMU.""" + if device.startswith("/dev/pts/"): + try: + st = os.stat(device) + if stat.S_ISCHR(st.st_mode) and os.major(st.st_rdev) == 136: + return True + except (OSError, AttributeError): + pass + return False + def close(self): self.serial.close() @@ -339,8 +357,10 @@ def read_until( while True: if data.endswith(ending): break - elif self.serial.inWaiting() > 0: + new_data = None + if self.is_pty or self.serial.inWaiting() > 0: new_data = self.serial.read(1) + if new_data: if data_consumer: data_consumer(new_data) data = new_data diff --git a/tools/verifygitlog.py b/tools/verifygitlog.py index dba6ebd6de5..1818b7f89a4 100755 --- a/tools/verifygitlog.py +++ b/tools/verifygitlog.py @@ -38,7 +38,7 @@ def warning(self, text): def git_log(pretty_format, *args): # Delete pretty argument from user args so it doesn't interfere with what we do. - args = ["git", "log"] + [arg for arg in args if "--pretty" not in args] + args = ["git", "log"] + [arg for arg in args if "--pretty" not in arg] args.append("--pretty=format:" + pretty_format) very_verbose("git_log", *args) # Generator yielding each output line.