diff --git a/.github/workflows/run-zephyr-tests.yml b/.github/workflows/run-zephyr-tests.yml index e99da28fe42..ff3e8bb1882 100644 --- a/.github/workflows/run-zephyr-tests.yml +++ b/.github/workflows/run-zephyr-tests.yml @@ -18,7 +18,7 @@ jobs: matrix: include: - shard: native_sim - boards: native_native_sim native_native_sim_asan + boards: native_native_sim native_native_sim_asan native_native_sim_lfs pytest_args: tests/ --ignore=tests/bsim - shard: nrf5340bsim boards: native_nrf5340bsim diff --git a/extmod/vfs_fat.c b/extmod/vfs_fat.c index 95ed79eb18b..636b68c5c4b 100644 --- a/extmod/vfs_fat.c +++ b/extmod/vfs_fat.c @@ -104,7 +104,7 @@ static mp_obj_t fat_vfs_make_new(const mp_obj_type_t *type, size_t n_args, size_ // CIRCUITPY-CHANGE static void verify_fs_writable(fs_user_mount_t *vfs) { - if (!filesystem_is_writable_by_python(vfs)) { + if (!filesystem_is_writable_by_python((supervisor_vfs_t *)vfs)) { mp_raise_OSError(MP_EROFS); } } @@ -430,7 +430,10 @@ static mp_obj_t vfs_fat_mount(mp_obj_t self_in, mp_obj_t readonly, mp_obj_t mkfs // CIRCUITPY-CHANGE: Use MP_BLOCKDEV_FLAG_USB_WRITABLE instead of writeblocks[0] =/!= MP_OBJ_NULL // to specify read-write. // If readonly to Python, it's writable by USB and vice versa. - filesystem_set_writable_by_usb(self, mp_obj_is_true(readonly)); + // CIRCUITPY-CHANGE: The flag helpers take a supervisor_vfs_t so they work + // for the FAT and littlefs root filesystems alike. A VfsFat is always a + // fs_user_mount_t, whose blockdev prefix matches both vfs kinds. + filesystem_set_writable_by_usb((supervisor_vfs_t *)self, mp_obj_is_true(readonly)); // check if we need to make the filesystem FRESULT res = (self->blockdev.flags & MP_BLOCKDEV_FLAG_NO_FILESYSTEM) ? FR_NO_FILESYSTEM : FR_OK; @@ -487,7 +490,7 @@ static MP_DEFINE_CONST_FUN_OBJ_3(fat_vfs_utime_obj, vfs_fat_utime); static mp_obj_t vfs_fat_getreadonly(mp_obj_t self_in) { fs_user_mount_t *self = MP_OBJ_TO_PTR(self_in); - return mp_obj_new_bool(!filesystem_is_writable_by_python(self)); + return mp_obj_new_bool(!filesystem_is_writable_by_python((supervisor_vfs_t *)self)); } static MP_DEFINE_CONST_FUN_OBJ_1(fat_vfs_getreadonly_obj, vfs_fat_getreadonly); diff --git a/extmod/vfs_fat_diskio.c b/extmod/vfs_fat_diskio.c index 591da07eb56..5d99ce24d5b 100644 --- a/extmod/vfs_fat_diskio.c +++ b/extmod/vfs_fat_diskio.c @@ -157,7 +157,7 @@ DRESULT disk_ioctl( // error initialising stat = STA_NOINIT; // CIRCUITPY-CHANGE: writability from Python check - } else if (!filesystem_is_writable_by_python(vfs)) { + } else if (!filesystem_is_writable_by_python((supervisor_vfs_t *)vfs)) { stat = STA_PROTECT; } else { stat = 0; diff --git a/extmod/vfs_fat_file.c b/extmod/vfs_fat_file.c index 2a3021ee9a8..aff6b3bde12 100644 --- a/extmod/vfs_fat_file.c +++ b/extmod/vfs_fat_file.c @@ -246,7 +246,7 @@ static mp_obj_t fat_vfs_open(mp_obj_t self_in, mp_obj_t path_in, mp_obj_t mode_i } assert(self != NULL); - if ((mode & FA_WRITE) != 0 && !filesystem_is_writable_by_python(self)) { + if ((mode & FA_WRITE) != 0 && !filesystem_is_writable_by_python((supervisor_vfs_t *)self)) { mp_raise_OSError(MP_EROFS); } diff --git a/extmod/vfs_lfs.c b/extmod/vfs_lfs.c index 19063d63066..eed6946fc26 100644 --- a/extmod/vfs_lfs.c +++ b/extmod/vfs_lfs.c @@ -26,6 +26,7 @@ #include "py/runtime.h" #include "py/mphal.h" +#include "supervisor/fatfs.h" #if MICROPY_VFS && (MICROPY_VFS_LFS1 || MICROPY_VFS_LFS2) @@ -105,15 +106,6 @@ mp_obj_t mp_vfs_lfs1_file_open(mp_obj_t self_in, mp_obj_t path_in, mp_obj_t mode // Attribute ids for lfs2_attr.type. #define LFS_ATTR_MTIME (1) // 64-bit little endian, nanoseconds since 1970/1/1 -typedef struct _mp_obj_vfs_lfs2_t { - mp_obj_base_t base; - mp_vfs_blockdev_t blockdev; - bool enable_mtime; - vstr_t cur_dir; - struct lfs2_config config; - lfs2_t lfs; -} mp_obj_vfs_lfs2_t; - typedef struct _mp_obj_vfs_lfs2_file_t { mp_obj_base_t base; mp_obj_vfs_lfs2_t *vfs; @@ -128,8 +120,10 @@ const char *mp_vfs_lfs2_make_path(mp_obj_vfs_lfs2_t *self, mp_obj_t path_in); mp_obj_t mp_vfs_lfs2_file_open(mp_obj_t self_in, mp_obj_t path_in, mp_obj_t mode_in); static void lfs_get_mtime(uint8_t buf[8]) { - // On-disk storage of timestamps uses 1970 as the Epoch, so convert from host's Epoch. - uint64_t ns = timeutils_nanoseconds_since_epoch_to_nanoseconds_since_1970(mp_hal_time_ns()); + // CIRCUITPY-CHANGE: Use the same RTC-based time that get_fattime() gives + // the FAT filesystems, so every filesystem kind stamps files alike. + // On-disk storage of timestamps is 64-bit little endian, ns since 1970/1/1. + uint64_t ns = get_fattime_ns(); // Store "ns" to "buf" in little-endian format (essentially htole64). for (size_t i = 0; i < 8; ++i) { buf[i] = ns; @@ -140,6 +134,50 @@ static void lfs_get_mtime(uint8_t buf[8]) { #include "extmod/vfs_lfsx.c" #include "extmod/vfs_lfsx_file.c" +// CIRCUITPY-CHANGE: Supervisor-facing littlefs mount. The caller prepares the +// mp_obj_vfs_lfs2_t (zeroed, with blockdev callbacks and lfs2_config geometry +// already filled in) and the port allocator for the littlefs caches. This runs +// without the VM or its GC, so no MicroPython allocations may happen here. +mp_obj_t mp_vfs_lfs2_mount_supervisor(mp_obj_vfs_lfs2_t *self, void *(*alloc)(size_t), bool format_allowed, bool *formatted_out, int *mount_err) { + struct lfs2_config *config = &self->config; + + config->block_cycles = 100; + config->cache_size = MIN(config->block_size, (4 * MAX(config->read_size, config->prog_size))); + config->lookahead_size = 32; + config->read_buffer = alloc(config->cache_size); + config->prog_buffer = alloc(config->cache_size); + config->lookahead_buffer = alloc(config->lookahead_size); + if (config->read_buffer == NULL || config->prog_buffer == NULL || config->lookahead_buffer == NULL) { + if (mount_err != NULL) { + *mount_err = LFS2_ERR_NOMEM; + } + return MP_OBJ_NULL; + } + + bool formatted = false; + int ret = lfs2_mount(&self->lfs, config); + if (ret < 0 && format_allowed) { + // Empty or corrupted. Format a fresh filesystem and try again. + ret = lfs2_format(&self->lfs, config); + if (ret >= 0) { + formatted = true; + ret = lfs2_mount(&self->lfs, config); + } + } + if (formatted_out != NULL && formatted) { + // Only ever set true: the caller may have formatted the filesystem + // itself (force reformat) and needs that to survive the mount. + *formatted_out = true; + } + if (mount_err != NULL) { + *mount_err = ret; + } + if (ret < 0) { + return MP_OBJ_NULL; + } + return MP_OBJ_FROM_PTR(self); +} + #endif // MICROPY_VFS_LFS2 #endif // MICROPY_VFS && (MICROPY_VFS_LFS1 || MICROPY_VFS_LFS2) diff --git a/extmod/vfs_lfs.h b/extmod/vfs_lfs.h index 1fdf792f1b3..3cb50b75087 100644 --- a/extmod/vfs_lfs.h +++ b/extmod/vfs_lfs.h @@ -36,4 +36,31 @@ extern const mp_obj_type_t mp_type_vfs_lfs2; extern const mp_obj_type_t mp_type_vfs_lfs2_fileio; extern const mp_obj_type_t mp_type_vfs_lfs2_textio; +// CIRCUITPY-CHANGE: Export the lfs2 VFS object so the supervisor can mount +// littlefs directly (before the VM and its GC are running) using the same +// layout the VFS methods in vfs_lfsx.c expect. +#if MICROPY_VFS_LFS2 +#include "lib/littlefs/lfs2.h" +#include "extmod/vfs.h" + +typedef struct _mp_obj_vfs_lfs2_t { + mp_obj_base_t base; + mp_vfs_blockdev_t blockdev; + bool enable_mtime; + vstr_t cur_dir; + struct lfs2_config config; + lfs2_t lfs; +} mp_obj_vfs_lfs2_t; + +// Mount (and optionally format first) a littlefs filesystem on the +// caller-prepared lfs2_config at self->config. self must be zeroed static or +// VM-heap storage; buffers are allocated via alloc(). Returns self_in on +// success or MP_OBJ_NULL on mount failure (the errno-like lfs2 return code is +// passed out via mount_err). formatted_out is set to true when a fresh +// filesystem was formatted before mounting successfully; it is never cleared, +// so the caller must initialize it to false (and may set it true itself after +// an explicit format). +mp_obj_t mp_vfs_lfs2_mount_supervisor(mp_obj_vfs_lfs2_t *self, void *(*alloc)(size_t), bool format_allowed, bool *formatted_out, int *mount_err); +#endif + #endif // MICROPY_INCLUDED_EXTMOD_VFS_LFS_H diff --git a/extmod/vfs_lfsx.c b/extmod/vfs_lfsx.c index bbdd21cfb91..20a9f5e665a 100644 --- a/extmod/vfs_lfsx.c +++ b/extmod/vfs_lfsx.c @@ -37,6 +37,7 @@ #include "py/objstr.h" #include "py/mperrno.h" #include "extmod/vfs.h" +#include "supervisor/filesystem.h" #include "shared/timeutils/timeutils.h" #if !MICROPY_ENABLE_FINALISER @@ -244,8 +245,18 @@ static mp_obj_t MP_VFS_LFSx(ilistdir_func)(size_t n_args, const mp_obj_t *args) } static MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(MP_VFS_LFSx(ilistdir_obj), 1, 2, MP_VFS_LFSx(ilistdir_func)); +static void MP_VFS_LFSx(verify_fs_writable)(MP_OBJ_VFS_LFSx * self) { + // CIRCUITPY-CHANGE: Honor the supervisor's write protection flags so that + // storage.remount(mount, readonly=True) also applies to littlefs mounts, + // matching what the FAT VFS does. + if (!filesystem_is_writable_by_python((supervisor_vfs_t *)self)) { + mp_raise_OSError(MP_EROFS); + } +} + static mp_obj_t MP_VFS_LFSx(remove)(mp_obj_t self_in, mp_obj_t path_in) { MP_OBJ_VFS_LFSx *self = MP_OBJ_TO_PTR(self_in); + MP_VFS_LFSx(verify_fs_writable)(self); const char *path = MP_VFS_LFSx(make_path)(self, path_in); int ret = LFSx_API(remove)(&self->lfs, path); if (ret < 0) { @@ -257,6 +268,7 @@ static MP_DEFINE_CONST_FUN_OBJ_2(MP_VFS_LFSx(remove_obj), MP_VFS_LFSx(remove)); static mp_obj_t MP_VFS_LFSx(rmdir)(mp_obj_t self_in, mp_obj_t path_in) { MP_OBJ_VFS_LFSx *self = MP_OBJ_TO_PTR(self_in); + MP_VFS_LFSx(verify_fs_writable)(self); const char *path = MP_VFS_LFSx(make_path)(self, path_in); int ret = LFSx_API(remove)(&self->lfs, path); if (ret < 0) { @@ -268,6 +280,7 @@ static MP_DEFINE_CONST_FUN_OBJ_2(MP_VFS_LFSx(rmdir_obj), MP_VFS_LFSx(rmdir)); static mp_obj_t MP_VFS_LFSx(rename)(mp_obj_t self_in, mp_obj_t path_old_in, mp_obj_t path_new_in) { MP_OBJ_VFS_LFSx *self = MP_OBJ_TO_PTR(self_in); + MP_VFS_LFSx(verify_fs_writable)(self); const char *path_old = MP_VFS_LFSx(make_path)(self, path_old_in); const char *path = mp_obj_str_get_str(path_new_in); vstr_t path_new; @@ -287,6 +300,7 @@ static MP_DEFINE_CONST_FUN_OBJ_3(MP_VFS_LFSx(rename_obj), MP_VFS_LFSx(rename)); static mp_obj_t MP_VFS_LFSx(mkdir)(mp_obj_t self_in, mp_obj_t path_o) { MP_OBJ_VFS_LFSx *self = MP_OBJ_TO_PTR(self_in); + MP_VFS_LFSx(verify_fs_writable)(self); const char *path = MP_VFS_LFSx(make_path)(self, path_o); int ret = LFSx_API(mkdir)(&self->lfs, path); if (ret < 0) { diff --git a/extmod/vfs_lfsx_file.c b/extmod/vfs_lfsx_file.c index a0e572d80cb..c72fcf5b59d 100644 --- a/extmod/vfs_lfsx_file.c +++ b/extmod/vfs_lfsx_file.c @@ -89,6 +89,12 @@ mp_obj_t MP_VFS_LFSx(file_open)(mp_obj_t self_in, mp_obj_t path_in, mp_obj_t mod flags = LFSx_MACRO(_O_RDONLY); } + // CIRCUITPY-CHANGE: Writes honor the supervisor's write protection flags, + // like the FAT VFS does, so storage.remount() readonly applies too. + if ((flags & LFSx_MACRO(_O_WRONLY)) != 0) { + MP_VFS_LFSx(verify_fs_writable)(self); + } + #if LFS_BUILD_VERSION == 1 MP_OBJ_VFS_LFSx_FILE *o = mp_obj_malloc_var_with_finaliser(MP_OBJ_VFS_LFSx_FILE, file_buffer, uint8_t, self->lfs.cfg->prog_size, type); #else diff --git a/main.c b/main.c index c860e739275..0d0859142d1 100644 --- a/main.c +++ b/main.c @@ -875,8 +875,8 @@ static void __attribute__ ((noinline)) run_boot_py(safe_mode_t safe_mode) { #endif // Get the base filesystem. - fs_user_mount_t *vfs = filesystem_circuitpy(); - FATFS *fs = &vfs->fatfs; + supervisor_vfs_t *vfs_root = filesystem_circuitpy(); + fs_user_mount_t *vfs = vfs_root == NULL ? NULL : &vfs_root->fat; // Allow boot.py access to CIRCUITPY, and allow writes to boot_out.txt. // We can't use the regular flags for this, because they might get modified inside boot.py. @@ -905,15 +905,17 @@ static void __attribute__ ((noinline)) run_boot_py(safe_mode_t safe_mode) { supervisor_status_bar_resume(); #endif bool write_boot_output = true; - FIL boot_output_file; - if (f_open(fs, &boot_output_file, CIRCUITPY_BOOT_OUTPUT_FILE, FA_READ) == FR_OK) { + supervisor_vfs_file_t boot_output_file; + if (supervisor_vfs_open_file(vfs_root, CIRCUITPY_BOOT_OUTPUT_FILE, SUPERVISOR_FS_OPEN_READ, 0, + &boot_output_file) == SUPERVISOR_FS_OK) { char *file_contents = m_new(char, boot_text.alloc); - UINT chars_read; - if (f_read(&boot_output_file, file_contents, 1 + boot_text.len, &chars_read) == FR_OK) { + size_t chars_read; + if (supervisor_vfs_read_file(&boot_output_file, file_contents, 1 + boot_text.len, &chars_read) == + SUPERVISOR_FS_OK) { write_boot_output = (chars_read != boot_text.len) || (memcmp(boot_text.buf, file_contents, chars_read) != 0); } - // no need to f_close the file + supervisor_vfs_close_file(&boot_output_file); } if (write_boot_output) { @@ -921,11 +923,14 @@ static void __attribute__ ((noinline)) run_boot_py(safe_mode_t safe_mode) { // in case power is momentary or will fail shortly due to, say a low, battery. mp_hal_delay_ms(1000); - f_open(fs, &boot_output_file, CIRCUITPY_BOOT_OUTPUT_FILE, FA_WRITE | FA_CREATE_ALWAYS); - UINT chars_written; - f_write(&boot_output_file, boot_text.buf, boot_text.len, &chars_written); - f_close(&boot_output_file); - filesystem_flush(); + if (supervisor_vfs_open_file(vfs_root, CIRCUITPY_BOOT_OUTPUT_FILE, + SUPERVISOR_FS_OPEN_WRITE | SUPERVISOR_FS_OPEN_CREATE | SUPERVISOR_FS_OPEN_TRUNCATE, 0, + &boot_output_file) == SUPERVISOR_FS_OK) { + size_t chars_written; + supervisor_vfs_write_file(&boot_output_file, boot_text.buf, boot_text.len, &chars_written); + supervisor_vfs_close_file(&boot_output_file); + filesystem_flush(); + } } #endif diff --git a/ports/espressif/boards/mixgo_ce_serial/board.c b/ports/espressif/boards/mixgo_ce_serial/board.c index e83f3b69679..e0d9f9530f2 100644 --- a/ports/espressif/boards/mixgo_ce_serial/board.c +++ b/ports/espressif/boards/mixgo_ce_serial/board.c @@ -16,8 +16,8 @@ void board_init(void) { mp_import_stat_t stat_b = mp_import_stat("boot.py"); if (stat_b != MP_IMPORT_STAT_FILE) { - fs_user_mount_t *fs_mount = filesystem_circuitpy(); - FATFS *fatfs = &fs_mount->fatfs; + supervisor_vfs_t *fs_mount = filesystem_circuitpy(); + FATFS *fatfs = &fs_mount->fat.fatfs; FIL fs; UINT char_written = 0; const byte buffer[] = "#Serial port upload mode\nimport storage\nstorage.remount(\"/\", False)\nstorage.disable_usb_drive()\n"; diff --git a/ports/espressif/boards/yoto_mini_2024/board.c b/ports/espressif/boards/yoto_mini_2024/board.c index fb6a96a957d..aa331e51c2f 100644 --- a/ports/espressif/boards/yoto_mini_2024/board.c +++ b/ports/espressif/boards/yoto_mini_2024/board.c @@ -191,8 +191,8 @@ void board_init(void) { } common_hal_sdioio_sdcard_never_reset(&sdmmc); - filesystem_set_concurrent_write_protection(vfs, true); - filesystem_set_writable_by_usb(vfs, false); + filesystem_set_concurrent_write_protection((supervisor_vfs_t *)vfs, true); + filesystem_set_writable_by_usb((supervisor_vfs_t *)vfs, false); mp_vfs_mount_t *sdcard_vfs = &_sdcard_vfs; sdcard_vfs->str = "/sd"; diff --git a/ports/espressif/boards/yoto_player_v3/board.c b/ports/espressif/boards/yoto_player_v3/board.c index fd6ee40b9f3..a2f943b0d18 100644 --- a/ports/espressif/boards/yoto_player_v3/board.c +++ b/ports/espressif/boards/yoto_player_v3/board.c @@ -136,8 +136,8 @@ void board_init(void) { } common_hal_sdioio_sdcard_never_reset(&sdmmc); - filesystem_set_concurrent_write_protection(vfs, true); - filesystem_set_writable_by_usb(vfs, false); + filesystem_set_concurrent_write_protection((supervisor_vfs_t *)vfs, true); + filesystem_set_writable_by_usb((supervisor_vfs_t *)vfs, false); mp_vfs_mount_t *sdcard_vfs = &_sdcard_vfs; sdcard_vfs->str = "/sd"; diff --git a/ports/espressif/supervisor/internal_flash.c b/ports/espressif/supervisor/internal_flash.c index a18d3fe0a0f..4b3836188a1 100644 --- a/ports/espressif/supervisor/internal_flash.c +++ b/ports/espressif/supervisor/internal_flash.c @@ -43,8 +43,8 @@ static uint32_t _cache_lba = 0xffffffff; #define SECSIZE(fs) ((fs)->ssize) #endif // FF_MAX_SS == FF_MIN_SS static DWORD fatfs_bytes(void) { - fs_user_mount_t *fs_mount = filesystem_circuitpy(); - FATFS *fatfs = &fs_mount->fatfs; + supervisor_vfs_t *fs_mount = filesystem_circuitpy(); + FATFS *fatfs = &fs_mount->fat.fatfs; return (fatfs->csize * SECSIZE(fatfs)) * (fatfs->n_fatent - 2); } static bool storage_extended = true; diff --git a/ports/zephyr-cp/Makefile b/ports/zephyr-cp/Makefile index 1fc5040fefa..8a20c9f1f80 100644 --- a/ports/zephyr-cp/Makefile +++ b/ports/zephyr-cp/Makefile @@ -127,7 +127,7 @@ clean-sim: # Every board the test suite uses: native sim (non-asan + asan) and the bsim # boards (tests/bsim/conftest.py parametrizes over both). Zephyr samples for # bsim tests are built on demand by the zephyr_sample fixture. -TEST_BOARDS := native_native_sim native_native_sim_asan native_nrf5340bsim native_nrf54lm20bsim +TEST_BOARDS := native_native_sim native_native_sim_asan native_native_sim_lfs native_nrf5340bsim native_nrf54lm20bsim # Delegate to a sub-make with BOARD set so the per-board build rule (and its # shield args + bsim prep) applies. The targets are phony: the west builds are diff --git a/ports/zephyr-cp/boards/README.md b/ports/zephyr-cp/boards/README.md new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/ports/zephyr-cp/boards/README.md @@ -0,0 +1 @@ + diff --git a/ports/zephyr-cp/boards/board_aliases.cmake b/ports/zephyr-cp/boards/board_aliases.cmake index e1faa7c3888..87362bae0a7 100644 --- a/ports/zephyr-cp/boards/board_aliases.cmake +++ b/ports/zephyr-cp/boards/board_aliases.cmake @@ -36,6 +36,7 @@ cp_board_alias(renesas_ek_ra8d1 ek_ra8d1) cp_board_alias(renesas_da14695_dk_usb da14695_dk_usb) cp_board_alias(native_native_sim native_sim/native) cp_board_alias(native_native_sim_asan native_sim/native) +cp_board_alias(native_native_sim_lfs native_sim/native) cp_board_alias(native_nrf5340bsim nrf5340bsim/nrf5340/cpuapp) cp_board_alias(native_nrf54lm20bsim nrf54lm20bsim/nrf54lm20a/cpuapp) cp_board_alias(nordic_nrf54l15dk nrf54l15dk/nrf54l15/cpuapp) diff --git a/ports/zephyr-cp/boards/native/native_sim_lfs/autogen_board_info.toml b/ports/zephyr-cp/boards/native/native_sim_lfs/autogen_board_info.toml new file mode 100644 index 00000000000..e19260e3203 --- /dev/null +++ b/ports/zephyr-cp/boards/native/native_sim_lfs/autogen_board_info.toml @@ -0,0 +1,131 @@ +# This file is autogenerated when a board is built. Do not edit. Do commit it to git. Other scripts use its info. +name = "zephyr Native simulator - littlefs filesystem" + +[modules] +__future__ = true +_bleio = false +_eve = false +_pew = false +_pixelmap = false +_stage = false +adafruit_bus_device = true +adafruit_pixelbuf = false +aesio = true +alarm = false +analogbufio = false +analogio = false +atexit = false +audiobusio = true # Zephyr board has audiobusio +audiocore = true # Zephyr board has audiobusio +audiodelays = true # Zephyr board has audiobusio +audiofilewriter = false +audiofilters = true # Zephyr board has audiobusio +audiofreeverb = true # Zephyr board has audiobusio +audioi2sin = false +audioio = false +audiomixer = true # Zephyr board has audiobusio +audiomp3 = true # Zephyr board has audiobusio +audiopwmio = false +audiospeed = false +aurora_epaper = false +bitbangio = false +bitmapfilter = true # Zephyr board has busio +bitmaptools = true # Zephyr board has busio +bitops = false +board = true # Always enabled +busdisplay = true # Zephyr board has busio +busio = true # Zephyr board has busio +camera = false +canio = false +codeop = false +countio = false +digitalio = true +displayio = true # Zephyr board has displayio +dotclockframebuffer = false +dualbank = false +emmcio = false +epaperdisplay = true # Zephyr board has busio +floppyio = false +fontio = true # Zephyr board has busio +fourwire = true # Zephyr board has busio +framebufferio = true # Zephyr board has busio +frequencyio = false +getpass = true +gifio = true # Zephyr board has busio +gnss = false +hashlib = true # Zephyr networking enabled +hostnetwork = true # Zephyr board has hostnetwork +i2cdisplaybus = true # Zephyr board has busio +i2cioexpander = false +i2ctarget = false +imagecapture = false +ipaddress = true # Zephyr networking enabled +is31fl3741 = false +jpegio = true # Zephyr board has busio +keypad = false +keypad_demux = false +locale = false +lvfontio = true # Zephyr board has busio +math = true +max3421e = false +mcp4822 = false +mdns = false +memorymap = false +memorymonitor = false +microcontroller = true +mipidsi = false +msgpack = true +neopixel_write = false +nvm = true # Zephyr board has nvm +onewireio = false +os = true +paralleldisplaybus = false +picogame = false +ps2io = false +pulseio = false +pwmio = false +qrio = false +qspibus = false +rainbowio = true +random = true +rclcpy = false +rgbmatrix = false +rotaryio = true # Zephyr board has rotaryio +rtc = false +sdcardio = true # Zephyr board has busio +sdioio = false +sharpdisplay = true # Zephyr board has busio +socketpool = true # Zephyr networking enabled +spitarget = false +ssl = false +storage = true +struct = true +supervisor = true +synthio = true # Zephyr board has audiobusio +terminalio = true # Zephyr board has busio +tilepalettemapper = true # Zephyr board has busio +time = true +touchio = false +traceback = true +uheap = false +usb = false +usb_audio = false +usb_cdc = false +usb_hid = false +usb_host = false +usb_midi = false +usb_video = false +ustack = false +vectorio = true # Zephyr board has busio +warnings = true +watchdog = false +wifi = false +zephyr_display = true # Zephyr board has zephyr_display +zephyr_kernel = false +zlib = true +# extmod modules shared with MicroPython +asyncio = true +binascii = true +json = true +re = true +select = true diff --git a/ports/zephyr-cp/boards/native/native_sim_lfs/board.conf b/ports/zephyr-cp/boards/native/native_sim_lfs/board.conf new file mode 100644 index 00000000000..a550d70dacc --- /dev/null +++ b/ports/zephyr-cp/boards/native/native_sim_lfs/board.conf @@ -0,0 +1,48 @@ +# No Bluetooth hardware on native_sim +CONFIG_BT=n + +CONFIG_EMUL=y +CONFIG_GPIO=y +CONFIG_NATIVE_SIM_SLOWDOWN_TO_REAL_TIME=n + +# So we can test safe mode +CONFIG_NATIVE_SIM_REBOOT=y + +CONFIG_TRACING=y +CONFIG_TRACING_PERFETTO=y +CONFIG_TRACING_SYNC=y +CONFIG_TRACING_BACKEND_POSIX=y +CONFIG_TRACING_GPIO=y + +# I2C emulation for testing +CONFIG_I2C_EMUL=y + +# Display emulation for display/terminal golden tests. +CONFIG_DISPLAY=y +CONFIG_SDL_DISPLAY=y +# Don't require hardware acceleration so the software renderer can be used. +CONFIG_SDL_DISPLAY_USE_HARDWARE_ACCELERATOR=n + +# EEPROM emulation for testing +CONFIG_EEPROM=y +CONFIG_EEPROM_AT24=y +CONFIG_EEPROM_AT2X_EMUL=y + +# I2S SDL emulation for audio testing +CONFIG_I2S_SDL=y + +CONFIG_NETWORKING=y +CONFIG_NET_IPV4=y +CONFIG_NET_TCP=y +CONFIG_NET_SOCKETS=y +CONFIG_ETH_NATIVE_TAP=n +CONFIG_NET_DRIVERS=y +CONFIG_NET_SOCKETS_OFFLOAD=y +CONFIG_NET_NATIVE_OFFLOADED_SOCKETS=y +CONFIG_HEAP_MEM_POOL_SIZE=1024 + +CONFIG_NET_LOG=y + +CONFIG_MBEDTLS=y +CONFIG_PSA_WANT_ALG_SHA_1=y +CONFIG_PSA_WANT_ALG_SHA_256=y diff --git a/ports/zephyr-cp/boards/native/native_sim_lfs/board.overlay b/ports/zephyr-cp/boards/native/native_sim_lfs/board.overlay new file mode 100644 index 00000000000..72c4a260dcb --- /dev/null +++ b/ports/zephyr-cp/boards/native/native_sim_lfs/board.overlay @@ -0,0 +1,71 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * Device tree overlay for CircuitPython on native_sim (littlefs filesystem). + * Same as the native_sim overlay, but the user filesystem partition node is + * named littlefs_partition to match the supervisor's littlefs build. + */ + +#include + +/* The mcuboot fork's layout (dts/native/native_sim.dtsi) is applied to this + * image first and names the filesystem node fatfs_partition. Replace the whole + * partition table so the filesystem node is littlefs_partition; the supervisor + * opens the filesystem by that nodelabel. The geometry must match the fork's + * layout: filesystem, then 8 KB storage, then 8 KB nvm. */ + +&flash0 { + /delete-node/ partitions; + + partitions { + #address-cells = <1>; + #size-cells = <1>; + + littlefs_partition: partition@0 { + compatible = "zephyr,mapped-partition"; + label = "filesystem"; + reg = <0x00000000 DT_SIZE_K(2032)>; + }; + + storage_partition: partition@1fc000 { + compatible = "zephyr,mapped-partition"; + label = "storage"; + reg = <0x001fc000 DT_SIZE_K(8)>; + }; + + nvm_partition: partition@1fe000 { + compatible = "zephyr,mapped-partition"; + label = "nvm"; + reg = <0x001fe000 0x00002000>; + }; + }; +}; + +/ { + sram0: memory@20000000 { + device_type = "memory"; + compatible = "zephyr,memory-region", "mmio-sram"; + reg = <0x20000000 DT_SIZE_M(1)>; + zephyr,memory-region = "SRAM"; + }; + + chosen { + zephyr,sram = &sram0; + /delete-property/ zephyr,flash; + /delete-property/ zephyr,code-partition; + }; +}; + +/* Add emulated I2C devices for testing */ +&i2c0 { + at24_eeprom: eeprom@50 { + compatible = "atmel,at24"; + reg = <0x50>; + size = <256>; + pagesize = <8>; + address-width = <8>; + timeout = <5>; + }; +}; + +#include "../../../app.overlay" diff --git a/ports/zephyr-cp/boards/native/native_sim_lfs/circuitpython.toml b/ports/zephyr-cp/boards/native/native_sim_lfs/circuitpython.toml new file mode 100644 index 00000000000..f98422e04c7 --- /dev/null +++ b/ports/zephyr-cp/boards/native/native_sim_lfs/circuitpython.toml @@ -0,0 +1,2 @@ +NAME = "Native simulator - littlefs filesystem" +CIRCUITPY_BUILD_EXTENSIONS = ["exe"] diff --git a/ports/zephyr-cp/cptools/build_circuitpython.py b/ports/zephyr-cp/cptools/build_circuitpython.py index 8594c245c81..f75c58443ce 100644 --- a/ports/zephyr-cp/cptools/build_circuitpython.py +++ b/ports/zephyr-cp/cptools/build_circuitpython.py @@ -434,6 +434,16 @@ async def build_circuitpython(): # noqa: C901 autogen_board_info_fn = mpconfigboard_fn.parent / "autogen_board_info.toml" + # Filesystem type is a compile-time choice made by the partition layout: + # zephyr2cp reports littlefs when the devicetree has a littlefs_partition + # node, otherwise the supervisor mounts FAT. + filesystem_littlefs = board_info.get("littlefs", False) + circuitpython_flags.append( + f"-DCIRCUITPY_FILESYSTEM_LITTLEFS={1 if filesystem_littlefs else 0}" + ) + if filesystem_littlefs: + circuitpython_flags.append("-DMICROPY_VFS_LFS2=1") + creator_id = mpconfigboard.get("CIRCUITPY_CREATOR_ID", mpconfigboard.get("USB_VID", 0x1209)) creation_id = mpconfigboard.get("CIRCUITPY_CREATION_ID", mpconfigboard.get("USB_PID", 0x000C)) circuitpython_flags.append(f"-DCIRCUITPY_CREATOR_ID=0x{creator_id:08x}") @@ -501,6 +511,16 @@ async def build_circuitpython(): # noqa: C901 supervisor_source.extend(top.glob("supervisor/shared/web_workflow/*.c")) usb_ok = board_info.get("usb_device", False) + if usb_ok and filesystem_littlefs: + # USB MSC can only expose a FAT block image; a littlefs CIRCUITPY has + # no MSC-compatible layout and USB MSC of it would corrupt the drive. + # Rename the filesystem partition to fatfs_partition in the board's + # Adaboot layout dtsi, or remove the USB device from the devicetree. + raise SystemExit( + f"{board}: littlefs CIRCUITPY filesystem is incompatible with USB. " + "Rename the littlefs_partition node to fatfs_partition in the board's " + "layout dtsi, or disable the USB device controller in the devicetree." + ) circuitpython_flags.append(f"-DCIRCUITPY_USB_DEVICE={1 if usb_ok else 0}") if usb_ok: @@ -670,6 +690,23 @@ async def build_circuitpython(): # noqa: C901 ) source_files = supervisor_source + hal_source + ["extmod/vfs.c"] + if filesystem_littlefs: + source_files.extend( + ( + "extmod/vfs_lfs.c", + "lib/littlefs/lfs2.c", + "lib/littlefs/lfs2_util.c", + ) + ) + circuitpython_flags.extend( + ( + "-DLFS2_NO_MALLOC", + "-DLFS2_NO_DEBUG", + "-DLFS2_NO_WARN", + "-DLFS2_NO_ERROR", + "-DLFS2_NO_ASSERT", + ) + ) if ulab_enabled: source_files.extend(sorted((top / "extmod" / "ulab" / "code").rglob("*.c"))) assembly_files = [] diff --git a/ports/zephyr-cp/cptools/zephyr2cp.py b/ports/zephyr-cp/cptools/zephyr2cp.py index cc40ccfda85..a21fcdb4d56 100644 --- a/ports/zephyr-cp/cptools/zephyr2cp.py +++ b/ports/zephyr-cp/cptools/zephyr2cp.py @@ -1081,4 +1081,9 @@ def zephyr_dts_to_cp_board(board_id, portdir, builddir, zephyrbuilddir, mpconfig nvm_node = device_tree.label2node.get("nvm_partition") board_info["nvm"] = nvm_node is not None + # The user filesystem type is a compile-time choice made by the partition + # layout: a littlefs_partition node (named for littlefs in the Adaboot + # fork's layout dtsi) mounts littlefs; everything else mounts FAT. + board_info["littlefs"] = device_tree.label2node.get("littlefs_partition") is not None + return board_info diff --git a/ports/zephyr-cp/supervisor/flash.c b/ports/zephyr-cp/supervisor/flash.c index a13bb3cffcb..202258bd715 100644 --- a/ports/zephyr-cp/supervisor/flash.c +++ b/ports/zephyr-cp/supervisor/flash.c @@ -20,7 +20,20 @@ #include #include +// The filesystem partition node label comes from the Adaboot fork's layout +// dtsi (and the native simulator layout). Prefer the explicit fatfs/littlefs +// names, then fall back to the generic circuitpy_partition name the native +// simulators (and other boards) use, and finally to runtime discovery of an +// unpartitioned flash device. +#if FIXED_PARTITION_EXISTS(littlefs_partition) +#define CIRCUITPY_PARTITION littlefs_partition +#elif FIXED_PARTITION_EXISTS(fatfs_partition) +#define CIRCUITPY_PARTITION fatfs_partition +#elif FIXED_PARTITION_EXISTS(filesystem_partition) +#define CIRCUITPY_PARTITION filesystem_partition +#elif FIXED_PARTITION_EXISTS(circuitpy_partition) #define CIRCUITPY_PARTITION circuitpy_partition +#endif static const struct flash_area *filesystem_area = NULL; #if !FIXED_PARTITION_EXISTS(CIRCUITPY_PARTITION) @@ -534,8 +547,11 @@ static bool _flash_write_block(const uint8_t *data, uint32_t block) { // Flush the cache if we're moving onto a different page. if (_current_page_address != page_address) { // Check to see if we'd write to an erased block and aren't writing to - // our cache. In that case we can write directly. - if (block_erased(address)) { + // our cache. In that case we can write directly. Do not take this fast + // path when the target block is part of the currently cached page: the + // cache copy would go stale and later reads would return the old data. + if (block_erased(address) && + !(_current_page_address == page_address && (_row_flags[block_index] & ROW_LOADED))) { return write_flash(address, data, FILESYSTEM_BLOCK_SIZE); } if (_current_page_address != NO_PAGE_LOADED) { diff --git a/ports/zephyr-cp/supervisor/usb.c b/ports/zephyr-cp/supervisor/usb.c index 6d3bf3286dd..6fee2e064cc 100644 --- a/ports/zephyr-cp/supervisor/usb.c +++ b/ports/zephyr-cp/supervisor/usb.c @@ -102,8 +102,11 @@ int _zephyr_disk_init(struct disk_info *disk) { return 0; } +// We don't check whether the filesystem is littlefs below because we know these +// are APIs used by USB MSC and we enforce it to be FAT at build time. int _zephyr_disk_status(struct disk_info *disk) { - fs_user_mount_t *root = filesystem_circuitpy(); + supervisor_vfs_t *root_vfs = filesystem_circuitpy(); + fs_user_mount_t *root = root_vfs == NULL ? NULL : &root_vfs->fat; int lun = 0; if (root == NULL) { printk("Status: No media\n"); @@ -114,7 +117,7 @@ int _zephyr_disk_status(struct disk_info *disk) { return DISK_STATUS_WR_PROTECT; } // Lock the blockdev once we say we're writable. - if (!locked[lun] && !blockdev_lock(root)) { + if (!locked[lun] && !blockdev_lock((supervisor_vfs_t *)root)) { printk("Status: Locked\n"); return DISK_STATUS_WR_PROTECT; } @@ -123,7 +126,8 @@ int _zephyr_disk_status(struct disk_info *disk) { } int _zephyr_disk_read(struct disk_info *disk, uint8_t *data_buf, uint32_t start_sector, uint32_t num_sector) { - fs_user_mount_t *root = filesystem_circuitpy(); + supervisor_vfs_t *root_vfs = filesystem_circuitpy(); + fs_user_mount_t *root = root_vfs == NULL ? NULL : &root_vfs->fat; uint32_t disk_block_count; disk_ioctl(root, GET_SECTOR_COUNT, &disk_block_count); @@ -136,7 +140,8 @@ int _zephyr_disk_read(struct disk_info *disk, uint8_t *data_buf, uint32_t start_ } int _zephyr_disk_write(struct disk_info *disk, const uint8_t *data_buf, uint32_t start_sector, uint32_t num_sector) { - fs_user_mount_t *root = filesystem_circuitpy(); + supervisor_vfs_t *root_vfs = filesystem_circuitpy(); + fs_user_mount_t *root = root_vfs == NULL ? NULL : &root_vfs->fat; int lun = 0; autoreload_suspend(AUTORELOAD_SUSPEND_USB); disk_write(root, data_buf, start_sector, num_sector); @@ -169,7 +174,8 @@ int _zephyr_disk_write(struct disk_info *disk, const uint8_t *data_buf, uint32_t int _zephyr_disk_ioctl(struct disk_info *disk, uint8_t cmd, void *buff) { - fs_user_mount_t *root = filesystem_circuitpy(); + supervisor_vfs_t *root_vfs = filesystem_circuitpy(); + fs_user_mount_t *root = root_vfs == NULL ? NULL : &root_vfs->fat; int lun = 0; switch (cmd) { case DISK_IOCTL_GET_SECTOR_COUNT: diff --git a/ports/zephyr-cp/tests/conftest.py b/ports/zephyr-cp/tests/conftest.py index 6a5f9de60d7..369b1eb43e2 100644 --- a/ports/zephyr-cp/tests/conftest.py +++ b/ports/zephyr-cp/tests/conftest.py @@ -5,6 +5,7 @@ import logging import os +import posixpath import subprocess from pathlib import Path @@ -77,12 +78,44 @@ def pytest_configure(config): "markers", "flash_config(erase_block_size=N, total_size=N): override flash simulator parameters", ) + config.addinivalue_line( + "markers", + "fat_filesystem_only: skip the test on littlefs filesystem boards", + ) ZEPHYR_CP = Path(__file__).parent.parent BUILD_DIR = ZEPHYR_CP / "build-native_native_sim" BINARY = BUILD_DIR / "zephyr-cp/zephyr/zephyr.exe" +# Boards parameterized by the filesystem tests. native_native_sim mounts a +# FAT filesystem; native_native_sim_lfs mounts littlefs. +FILESYSTEM_SIMULATORS = ["native_native_sim", "native_native_sim_lfs"] + +# Boards whose flash image is a FAT drive that can be populated from the host +# with mtools. +FAT_FILESYSTEM_BOARDS = { + "native_native_sim", + "native_native_sim_asan", + "native_nrf5340bsim", + "native_nrf54lm20bsim", +} + +# Boards whose flash image holds a littlefs filesystem. Their images are +# populated from the host with littlefs-python, using geometry that matches +# the board overlay and supervisor/shared/filesystem.c: FILESYSTEM_BLOCK_SIZE +# (512) byte blocks over the filesystem partition, with the partition's last +# erase page left out because the supervisor flash cache uses it as its +# scratch page (see supervisor_flash_get_block_count()). +DEFAULT_ERASE_PAGE_SIZE = 4096 +LITTLEFS_FILESYSTEM_BOARDS = { + "native_native_sim_lfs": { + "partition_size": 2032 * 1024, + "erase_page_size": DEFAULT_ERASE_PAGE_SIZE, + "block_size": 512, + }, +} + def _iter_uart_tx_slices(trace_file: Path) -> list[tuple[int, int, str, str]]: """Return UART TX slices as (timestamp_ns, duration_ns, text, device_name).""" @@ -149,21 +182,87 @@ def log_uart_trace_output(trace_file: Path) -> None: ) -# Native_sim boards each test runs against: the non-asan default and the -# asan-enabled build, so memory errors fail tests. -NATIVE_BOARDS = ["native_native_sim", "native_native_sim_asan"] +# Native_sim boards each test runs against: the FAT default, the littlefs +# build, and the asan-enabled build, so filesystem and memory errors fail +# tests. +NATIVE_BOARDS = [ + "native_native_sim", + "native_native_sim_lfs", + "native_native_sim_asan", +] @pytest.fixture(params=NATIVE_BOARDS) def board(request): - """Parametrized over both native_sim builds (non-asan and asan). + board_marker = request.node.get_closest_marker("circuitpython_board") + if board_marker is not None: + return board_marker.args[0] + # Support indirect parametrization, e.g. + # @pytest.mark.parametrize("board", FILESYSTEM_SIMULATORS, indirect=True) + if hasattr(request, "param"): + return request.param + return "native_native_sim" + + +def _lfs_geometry(board, erase_page_size): + """Return (block_size, block_count) matching what CircuitPython mounts.""" + geometry = LITTLEFS_FILESYSTEM_BOARDS[board] + block_size = geometry["block_size"] + page_size = max(erase_page_size, block_size) + # The last erase page of the partition is the supervisor flash cache's + # scratch page, so littlefs only gets the blocks before it. + last_page_start = ((geometry["partition_size"] - block_size) // page_size) * page_size + return block_size, last_page_start // block_size + + +def _populate_lfs_drive(flash, files, board, erase_page_size): + """Copy files onto a littlefs flash image with littlefs-python.""" + from littlefs import LittleFS + from littlefs.context import UserContextFile + + block_size, block_count = _lfs_geometry(board, erase_page_size) + # The fresh image is all erased bytes, so the initial mount fails and + # LittleFS formats it. This matches what CircuitPython does on a blank + # flash, minus the boot files, which are added below. + fs = LittleFS( + context=UserContextFile(str(flash)), + block_size=block_size, + block_count=block_count, + ) + for name, content in files.items(): + parent = posixpath.dirname(name) + if parent: + fs.makedirs(parent, exist_ok=True) + mode = "wb" if isinstance(content, bytes) else "wt" + with fs.open(f"/{name}", mode) as f: + f.write(content) + fs.context.close() + + +def read_lfs_file_from_flash(flash_file, path, board, erase_page_size): + """Extract a file from the littlefs filesystem in the flash image.""" + from littlefs import LittleFS + from littlefs.context import UserContextFile + + block_size, block_count = _lfs_geometry(board, erase_page_size) + fs = LittleFS( + context=UserContextFile(str(flash_file)), + block_size=block_size, + block_count=block_count, + ) + with fs.open(f"/{path}", "rb") as f: + content = f.read() + fs.context.close() + return content.decode() - The bsim conftest overrides this fixture with its own bsim boards. - """ - board = request.node.get_closest_marker("circuitpython_board") - if board is not None: - return board.args[0] - return request.param + +@pytest.fixture(autouse=True) +def _skip_fat_filesystem_only_tests(request, board): + """Honor the ``fat_filesystem_only`` marker on littlefs boards.""" + if request.node.get_closest_marker("fat_filesystem_only") is None: + return + if board in LITTLEFS_FILESYSTEM_BOARDS: + pytest.skip("test requires the FAT filesystem build") @pytest.fixture @@ -212,6 +311,48 @@ def sim_id(request) -> str: return request.node.nodeid.replace("/", "_") +def _populate_drive(flash, files, board, tmp_path, index, erase_page_size): + """Copy files onto the flash image with the host tool matching the board's + filesystem, if any were requested.""" + if files is None: + return + if board in LITTLEFS_FILESYSTEM_BOARDS: + _populate_lfs_drive(flash, files, board, erase_page_size) + else: + _populate_fat_drive(flash, files, board, tmp_path, index) + + +def _populate_fat_drive(flash, files, board, tmp_path, index): + """Copy files onto a FAT flash image with mtools.""" + if board not in FAT_FILESYSTEM_BOARDS: + pytest.skip( + f"cannot preload files onto {board}: only FAT filesystem images " + "can be built from the host" + ) + subprocess.run(["mformat", "-i", str(flash), "::"], check=True) + tmp_drive = tmp_path / f"drive{index}" + tmp_drive.mkdir(exist_ok=True) + + fat_dirs_created = set() + for name, content in files.items(): + src = tmp_drive / name + src.parent.mkdir(parents=True, exist_ok=True) + if isinstance(content, bytes): + src.write_bytes(content) + else: + src.write_text(content) + # Create parent directories on the FAT image. + fat_dir = Path(name).parent + for fat_part in [*reversed(fat_dir.parents), fat_dir]: + if fat_part == Path("."): + continue + fat_path = "::" + str(fat_part) + if fat_path not in fat_dirs_created: + subprocess.run(["mmd", "-i", str(flash), fat_path], check=True) + fat_dirs_created.add(fat_path) + subprocess.run(["mcopy", "-i", str(flash), str(src), f"::{name}"], check=True) + + @pytest.fixture def circuitpython(request, board, sim_id, native_sim_binary, native_sim_env, tmp_path): """Run CircuitPython with given code string and return PTY output.""" @@ -290,6 +431,11 @@ def circuitpython(request, board, sim_id, native_sim_binary, native_sim_env, tmp flash_total_size = flash_config_marker.kwargs.get("total_size", flash_total_size) flash_erase_block_size = flash_config_marker.kwargs.get("erase_block_size", None) flash_write_block_size = flash_config_marker.kwargs.get("write_block_size", None) + # The littlefs image geometry must match the erase page size the + # simulator will run with, which the flash_config marker can override. + populate_erase_page_size = ( + flash_erase_block_size if flash_erase_block_size is not None else DEFAULT_ERASE_PAGE_SIZE + ) procs = [] for i in range(instance_count): @@ -298,29 +444,7 @@ def circuitpython(request, board, sim_id, native_sim_binary, native_sim_env, tmp files = None if len(drives[i][1].args) == 1: files = drives[i][1].args[0] - if files is not None: - subprocess.run(["mformat", "-i", str(flash), "::"], check=True) - tmp_drive = tmp_path / f"drive{i}" - tmp_drive.mkdir(exist_ok=True) - - fat_dirs_created = set() - for name, content in files.items(): - src = tmp_drive / name - src.parent.mkdir(parents=True, exist_ok=True) - if isinstance(content, bytes): - src.write_bytes(content) - else: - src.write_text(content) - # Create parent directories on the FAT image. - fat_dir = Path(name).parent - for fat_part in [*reversed(fat_dir.parents), fat_dir]: - if fat_part == Path("."): - continue - fat_path = "::" + str(fat_part) - if fat_path not in fat_dirs_created: - subprocess.run(["mmd", "-i", str(flash), fat_path], check=True) - fat_dirs_created.add(fat_path) - subprocess.run(["mcopy", "-i", str(flash), str(src), f"::{name}"], check=True) + _populate_drive(flash, files, board, tmp_path, i, populate_erase_page_size) trace_file = tmp_path / f"trace-{i}.perfetto" diff --git a/ports/zephyr-cp/tests/test_filesystem.py b/ports/zephyr-cp/tests/test_filesystem.py new file mode 100644 index 00000000000..2e149dcfe99 --- /dev/null +++ b/ports/zephyr-cp/tests/test_filesystem.py @@ -0,0 +1,144 @@ +# SPDX-FileCopyrightText: 2026 Scott Shawcroft for Adafruit Industries +# SPDX-License-Identifier: MIT + +"""Filesystem tests that run against every filesystem the port supports. + +Each test is parameterized over the two native_sim builds so the same +expectations are checked against the FAT filesystem (``native_native_sim``) +and littlefs (``native_native_sim_lfs``). These tests boot from an erased +flash image so they see the filesystem that CircuitPython creates itself, +not one preloaded from the host (see ``conftest`` for that). +""" + +import time + +import pytest + +from .conftest import FILESYSTEM_SIMULATORS + + +def _enter_repl(circuitpython): + """Run the default code.py, then enter the REPL from the reload prompt.""" + circuitpython.serial.wait_for("Press any key to enter the REPL") + circuitpython.serial.write("\r") + circuitpython.serial.wait_for(">>>") + + +def _repl(circuitpython, line): + """Send one line to the REPL and give it time to echo and run.""" + # The native sim console needs CR, not LF, to submit the line. + circuitpython.serial.write(line.rstrip("\n") + "\r") + time.sleep(0.5) + + +@pytest.mark.parametrize("board", FILESYSTEM_SIMULATORS, indirect=True) +@pytest.mark.circuitpy_drive(None) +@pytest.mark.duration(30) +@pytest.mark.port_resets(6) +def test_fresh_filesystem_defaults(circuitpython): + """An erased flash image gets a filesystem with the default files.""" + _enter_repl(circuitpython) + + _repl(circuitpython, "import os\n") + _repl(circuitpython, "entries = os.listdir('/')\n") + _repl(circuitpython, "print('code.py:', 'code.py' in entries)\n") + circuitpython.serial.wait_for("code.py: True") + _repl(circuitpython, "print('lib:', 'lib' in entries)\n") + circuitpython.serial.wait_for("lib: True") + + +@pytest.mark.parametrize("board", FILESYSTEM_SIMULATORS, indirect=True) +@pytest.mark.circuitpy_drive(None) +@pytest.mark.duration(60) +@pytest.mark.port_resets(6) +def test_filesystem_write_read_delete(circuitpython): + """Files survive write, close, reopen and delete.""" + _enter_repl(circuitpython) + + _repl(circuitpython, "with open('/fs_test.txt', 'w') as f:\n") + _repl(circuitpython, " f.write('hello filesystem')\n") + _repl(circuitpython, "\n") + _repl(circuitpython, "print('read:', open('/fs_test.txt').read())\n") + circuitpython.serial.wait_for("read: hello filesystem") + + _repl(circuitpython, "import os\n") + _repl(circuitpython, "print('exists:', 'fs_test.txt' in os.listdir('/'))\n") + circuitpython.serial.wait_for("exists: True") + + _repl(circuitpython, "os.remove('/fs_test.txt')\n") + _repl(circuitpython, "print('exists:', 'fs_test.txt' in os.listdir('/'))\n") + circuitpython.serial.wait_for("exists: False") + + +@pytest.mark.parametrize("board", FILESYSTEM_SIMULATORS, indirect=True) +@pytest.mark.circuitpy_drive(None) +@pytest.mark.duration(60) +@pytest.mark.port_resets(6) +def test_filesystem_mkdir_rename_stat(circuitpython): + """Directories, renames and os.stat work.""" + _enter_repl(circuitpython) + + _repl(circuitpython, "import os\n") + _repl(circuitpython, "os.mkdir('/subdir')\n") + _repl(circuitpython, "with open('/moved.txt', 'w') as f:\n") + _repl(circuitpython, " f.write('moved')\n") + _repl(circuitpython, "\n") + _repl(circuitpython, "os.rename('/moved.txt', '/subdir/renamed.txt')\n") + _repl(circuitpython, "print('dir:', 'subdir' in os.listdir('/'))\n") + circuitpython.serial.wait_for("dir: True") + _repl(circuitpython, "print('renamed:', open('/subdir/renamed.txt').read())\n") + circuitpython.serial.wait_for("renamed: moved") + _repl(circuitpython, "print('stat size:', os.stat('/subdir/renamed.txt')[6])\n") + circuitpython.serial.wait_for("stat size: 5") + + +@pytest.mark.parametrize("board", FILESYSTEM_SIMULATORS, indirect=True) +@pytest.mark.circuitpy_drive(None) +@pytest.mark.duration(60) +@pytest.mark.port_resets(6) +def test_storage_remount(circuitpython): + """storage.remount('/') toggles writability on the root filesystem.""" + _enter_repl(circuitpython) + + _repl(circuitpython, "import storage\n") + _repl(circuitpython, "storage.remount('/', readonly=False)\n") + _repl(circuitpython, "with open('/remount.txt', 'w') as f:\n") + _repl(circuitpython, " f.write('writable')\n") + _repl(circuitpython, "\n") + _repl(circuitpython, "print('read:', open('/remount.txt').read())\n") + circuitpython.serial.wait_for("read: writable") + + _repl(circuitpython, "storage.remount('/', readonly=True)\n") + _repl(circuitpython, "try:\n") + _repl(circuitpython, " with open('/remount2.txt', 'w') as f:\n") + _repl(circuitpython, " f.write('nope')\n") + _repl(circuitpython, " print('unexpected: write succeeded')\n") + _repl(circuitpython, "except OSError as e:\n") + _repl(circuitpython, " print('caught OSError:', e.errno)\n") + _repl(circuitpython, "\n") + circuitpython.serial.wait_for("caught OSError:") + + _repl(circuitpython, "import os\n") + _repl(circuitpython, "print('blocked:', 'remount2.txt' in os.listdir('/'))\n") + circuitpython.serial.wait_for("blocked: False") + + +@pytest.mark.parametrize("board", FILESYSTEM_SIMULATORS, indirect=True) +@pytest.mark.circuitpy_drive(None) +@pytest.mark.duration(120) +@pytest.mark.port_resets(8) +def test_filesystem_persists_across_reset(circuitpython): + """Files written before microcontroller.reset() survive the reboot.""" + _enter_repl(circuitpython) + + _repl(circuitpython, "with open('/persist.txt', 'w') as f:\n") + _repl(circuitpython, " f.write('still here')\n") + _repl(circuitpython, "\n") + _repl(circuitpython, "import microcontroller\n") + _repl(circuitpython, "microcontroller.reset()\n") + + assert circuitpython.reconnect_serial(timeout=30), "simulator did not reboot" + + _enter_repl(circuitpython) + _repl(circuitpython, "print('persisted:', open('/persist.txt').read())\n") + circuitpython.serial.wait_for("persisted: still here") diff --git a/ports/zephyr-cp/tests/test_flash.py b/ports/zephyr-cp/tests/test_flash.py index e2e5f4de14f..9af4c95ab14 100644 --- a/ports/zephyr-cp/tests/test_flash.py +++ b/ports/zephyr-cp/tests/test_flash.py @@ -7,9 +7,17 @@ import pytest +from .conftest import ( + DEFAULT_ERASE_PAGE_SIZE, + LITTLEFS_FILESYSTEM_BOARDS, + read_lfs_file_from_flash, +) -def read_file_from_flash(flash_file, path): - """Extract a file from the FAT filesystem in the flash image.""" + +def read_file_from_flash(flash_file, path, board, erase_page_size=DEFAULT_ERASE_PAGE_SIZE): + """Extract a file from the flash image using the board's filesystem tool.""" + if board in LITTLEFS_FILESYSTEM_BOARDS: + return read_lfs_file_from_flash(flash_file, path, board, erase_page_size) result = subprocess.run( ["mcopy", "-i", str(flash_file), f"::{path}", "-"], capture_output=True, @@ -34,40 +42,44 @@ def read_file_from_flash(flash_file, path): @pytest.mark.circuitpy_drive({"code.py": WRITE_READ_CODE}) -def test_flash_default_erase_size(circuitpython): +def test_flash_default_erase_size(circuitpython, board): """Test filesystem write/read with default 4KB erase blocks.""" circuitpython.wait_until_done() output = circuitpython.serial.all_output assert "content: hello flash" in output assert "done" in output - content = read_file_from_flash(circuitpython.flash_file, "test.txt") + content = read_file_from_flash(circuitpython.flash_file, "test.txt", board) assert content == "hello flash" @pytest.mark.circuitpy_drive({"code.py": WRITE_READ_CODE}) @pytest.mark.flash_config(erase_block_size=65536) -def test_flash_64k_erase_blocks(circuitpython): +def test_flash_64k_erase_blocks(circuitpython, board): """Test filesystem write/read with 64KB erase blocks (128 blocks per page).""" circuitpython.wait_until_done() output = circuitpython.serial.all_output assert "content: hello flash" in output assert "done" in output - content = read_file_from_flash(circuitpython.flash_file, "test.txt") + content = read_file_from_flash( + circuitpython.flash_file, "test.txt", board, erase_page_size=65536 + ) assert content == "hello flash" @pytest.mark.circuitpy_drive({"code.py": WRITE_READ_CODE}) @pytest.mark.flash_config(erase_block_size=262144, total_size=4 * 1024 * 1024) -def test_flash_256k_erase_blocks(circuitpython): +def test_flash_256k_erase_blocks(circuitpython, board): """Test filesystem write/read with 256KB erase blocks (like RA8D1 OSPI).""" circuitpython.wait_until_done() output = circuitpython.serial.all_output assert "content: hello flash" in output assert "done" in output - content = read_file_from_flash(circuitpython.flash_file, "test.txt") + content = read_file_from_flash( + circuitpython.flash_file, "test.txt", board, erase_page_size=262144 + ) assert content == "hello flash" @@ -85,14 +97,19 @@ def test_flash_256k_erase_blocks(circuitpython): @pytest.mark.circuitpy_drive({"code.py": MULTI_FILE_CODE}) @pytest.mark.flash_config(erase_block_size=262144, total_size=4 * 1024 * 1024) -def test_flash_256k_multi_file(circuitpython): +def test_flash_256k_multi_file(circuitpython, board): """Test multiple file writes with 256KB erase blocks to exercise cache flushing.""" circuitpython.wait_until_done() output = circuitpython.serial.all_output assert "multi_file_ok" in output for i in range(5): - content = read_file_from_flash(circuitpython.flash_file, f"file{i}.txt") + content = read_file_from_flash( + circuitpython.flash_file, + f"file{i}.txt", + board, + erase_page_size=262144, + ) assert content == f"data{i}" * 100, f"file{i}.txt mismatch" @@ -109,9 +126,9 @@ def test_flash_256k_multi_file(circuitpython): storage.remount("/", readonly=True) storage.remount("/", readonly=False) -# Now write a small new file. This updates FAT metadata and directory -# entries that share an erase page with original.txt, exercising the -# read-modify-write cycle on the cached page. +# Now write a small new file. This updates filesystem metadata and +# directory entries that share an erase page with original.txt, +# exercising the read-modify-write cycle on the cached page. with open("/small.txt", "w") as f: f.write("tiny") @@ -128,7 +145,7 @@ def test_flash_256k_multi_file(circuitpython): @pytest.mark.circuitpy_drive({"code.py": EXISTING_DATA_CODE}) @pytest.mark.flash_config(erase_block_size=262144, total_size=4 * 1024 * 1024) -def test_flash_256k_existing_data_survives(circuitpython): +def test_flash_256k_existing_data_survives(circuitpython, board): """Test that existing data in an erase page survives when new data is written. With 256KB erase blocks (512 blocks per page), writing to any block in @@ -140,10 +157,14 @@ def test_flash_256k_existing_data_survives(circuitpython): assert "existing_data_ok" in output # Verify both files survived on the actual flash image. - original = read_file_from_flash(circuitpython.flash_file, "original.txt") + original = read_file_from_flash( + circuitpython.flash_file, "original.txt", board, erase_page_size=262144 + ) assert original == "A" * 4000, f"original.txt corrupted: got {len(original)} bytes" - small = read_file_from_flash(circuitpython.flash_file, "small.txt") + small = read_file_from_flash( + circuitpython.flash_file, "small.txt", board, erase_page_size=262144 + ) assert small == "tiny" @@ -161,11 +182,13 @@ def test_flash_256k_existing_data_survives(circuitpython): @pytest.mark.circuitpy_drive({"code.py": OVERWRITE_CODE}) @pytest.mark.flash_config(erase_block_size=262144, total_size=4 * 1024 * 1024) -def test_flash_256k_overwrite(circuitpython): +def test_flash_256k_overwrite(circuitpython, board): """Test overwriting a file with 256KB erase blocks to exercise erase-rewrite cycle.""" circuitpython.wait_until_done() output = circuitpython.serial.all_output assert "overwrite_ok" in output - content = read_file_from_flash(circuitpython.flash_file, "overwrite.txt") + content = read_file_from_flash( + circuitpython.flash_file, "overwrite.txt", board, erase_page_size=262144 + ) assert content == "second version" diff --git a/ports/zephyr-cp/tests/test_storage.py b/ports/zephyr-cp/tests/test_storage.py index 8805a582b6b..002170f3266 100644 --- a/ports/zephyr-cp/tests/test_storage.py +++ b/ports/zephyr-cp/tests/test_storage.py @@ -60,6 +60,7 @@ def test_storage_remount_readonly_blocks_writes(circuitpython): """ +@pytest.mark.fat_filesystem_only @pytest.mark.circuitpy_drive({"code.py": GETMOUNT_CODE}) def test_storage_getmount(circuitpython): """getmount('/') returns the VfsFat object for the root mount.""" @@ -131,6 +132,7 @@ def test_storage_remount_persists_across_reload(circuitpython): """ +@pytest.mark.fat_filesystem_only @pytest.mark.circuitpy_drive({"code.py": LABEL_CODE}) def test_storage_set_label(circuitpython): """VfsFat.label can be set when mounted read-write.""" diff --git a/py/circuitpy_mpconfig.h b/py/circuitpy_mpconfig.h index 89eeee606f5..c8a555566e2 100644 --- a/py/circuitpy_mpconfig.h +++ b/py/circuitpy_mpconfig.h @@ -48,8 +48,20 @@ extern void common_hal_mcu_enable_interrupts(void); #define MICROPY_PY_OS_DUPTERM (0) #define MICROPY_PYEXEC_COMPILE_ONLY (0) #define MICROPY_ROM_TEXT_COMPRESSION (0) +#ifndef MICROPY_VFS_LFS1 #define MICROPY_VFS_LFS1 (0) +#endif +#ifndef MICROPY_VFS_LFS2 #define MICROPY_VFS_LFS2 (0) +#endif + +// CIRCUITPY-CHANGE: Compile-time choice of the CIRCUITPY filesystem. When 1, +// the supervisor mounts littlefs instead of FAT on the block device returned +// by supervisor_flash_*. Ports set this (and MICROPY_VFS_LFS2) from their +// board configuration; the default remains FAT. +#ifndef CIRCUITPY_FILESYSTEM_LITTLEFS +#define CIRCUITPY_FILESYSTEM_LITTLEFS (0) +#endif // Always turn on exit code handling #define MICROPY_PYEXEC_ENABLE_EXIT_CODE_HANDLING (1) diff --git a/requirements-dev.txt b/requirements-dev.txt index a6fedc4b4af..571bc3aad71 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -47,3 +47,6 @@ tomlkit pytest pytest-rerunfailures perfetto + +# For littlefs flash image tests on the zephyr port native_simulators +littlefs-python diff --git a/shared-module/emmcio/__init__.c b/shared-module/emmcio/__init__.c index 075451a6ebc..3eaab720ea6 100644 --- a/shared-module/emmcio/__init__.c +++ b/shared-module/emmcio/__init__.c @@ -87,8 +87,8 @@ void automount_emmc(void) { } // Same as CIRCUITPY: while a host has the drive, the host owns writing. - filesystem_set_concurrent_write_protection(vfs, true); - filesystem_set_writable_by_usb(vfs, true); + filesystem_set_concurrent_write_protection((supervisor_vfs_t *)vfs, true); + filesystem_set_writable_by_usb((supervisor_vfs_t *)vfs, true); mp_vfs_mount_t *emmc_vfs = &_emmc_vfs; emmc_vfs->str = CIRCUITPY_EMMC_MOUNT_PATH; diff --git a/shared-module/lvfontio/OnDiskFont.c b/shared-module/lvfontio/OnDiskFont.c index efffed63074..f1526835702 100644 --- a/shared-module/lvfontio/OnDiskFont.c +++ b/shared-module/lvfontio/OnDiskFont.c @@ -436,9 +436,11 @@ void common_hal_lvfontio_ondiskfont_construct(lvfontio_ondiskfont_t *self, // Determine which filesystem to use based on the path const char *path_under_mount; - fs_user_mount_t *vfs = filesystem_for_path(file_path, &path_under_mount); + supervisor_vfs_t *vfs = filesystem_for_path(file_path, &path_under_mount); - if (vfs == NULL) { + if (vfs == NULL || vfs->common.base.type != &mp_fat_vfs_type) { + // Not found, or a non-FAT filesystem (littlefs builds) that f_open + // can't access. if (self->use_gc_allocator) { mp_raise_ValueError(MP_ERROR_TEXT("File not found")); } @@ -446,7 +448,7 @@ void common_hal_lvfontio_ondiskfont_construct(lvfontio_ondiskfont_t *self, } // Open the file and keep it open for the lifetime of the object - FRESULT res = f_open(&vfs->fatfs, &self->file, path_under_mount, FA_READ); + FRESULT res = f_open(&vfs->fat.fatfs, &self->file, path_under_mount, FA_READ); if (res != FR_OK) { if (self->use_gc_allocator) { diff --git a/shared-module/sdcardio/__init__.c b/shared-module/sdcardio/__init__.c index d53f7a3fabd..23a78160b26 100644 --- a/shared-module/sdcardio/__init__.c +++ b/shared-module/sdcardio/__init__.c @@ -134,8 +134,8 @@ void automount_sd_card(void) { return; } - filesystem_set_concurrent_write_protection(vfs, true); - filesystem_set_writable_by_usb(vfs, false); + filesystem_set_concurrent_write_protection((supervisor_vfs_t *)vfs, true); + filesystem_set_writable_by_usb((supervisor_vfs_t *)vfs, false); mp_vfs_mount_t *sdcard_vfs = &_sdcard_vfs; sdcard_vfs->str = "/sd"; diff --git a/shared-module/storage/__init__.c b/shared-module/storage/__init__.c index 1522136332a..4a5561c3164 100644 --- a/shared-module/storage/__init__.c +++ b/shared-module/storage/__init__.c @@ -145,8 +145,11 @@ void common_hal_storage_mount(mp_obj_t vfs_obj, const char *mount_path, bool rea fs_user_mount_t *vfs_fat = MP_OBJ_TO_PTR(vfs_obj); // Filesystem is read-only to USB if writable by CircuitPython, and vice versa. - filesystem_set_writable_by_usb(vfs_fat, readonly); - filesystem_set_concurrent_write_protection(vfs_fat, true); + // The flag helpers take a supervisor_vfs_t; a mounted VfsFat is always a + // fs_user_mount_t, whose base + blockdev prefix matches both vfs + // kinds. + filesystem_set_writable_by_usb((supervisor_vfs_t *)vfs_fat, readonly); + filesystem_set_concurrent_write_protection((supervisor_vfs_t *)vfs_fat, true); // Insert the vfs into the mount table by pushing it onto the front of the // mount table. @@ -197,29 +200,37 @@ mp_obj_t common_hal_storage_getmount(const char *mount_path) { return storage_object_from_path(mount_path); } -void common_hal_storage_remount(const char *mount_path, bool readonly, bool disable_concurrent_write_protection) { - const char *path_under_mount; - const char *abs_mount_path = common_hal_os_path_abspath(mount_path); - fs_user_mount_t *fs_usermount = filesystem_for_path(abs_mount_path, &path_under_mount); - if (path_under_mount[0] != 0 && strcmp(abs_mount_path, "/") != 0) { - mp_raise_OSError(MP_EINVAL); - } - +static void remount_vfs(supervisor_vfs_t *fs_mount, bool readonly, bool disable_concurrent_write_protection) { #if CIRCUITPY_USB_DEVICE && CIRCUITPY_USB_MSC - if (!blockdev_lock(fs_usermount)) { + if (!blockdev_lock(fs_mount)) { mp_raise_RuntimeError(MP_ERROR_TEXT("Cannot remount path when visible via USB.")); } #endif - filesystem_set_writable_by_usb(fs_usermount, readonly); - filesystem_set_concurrent_write_protection(fs_usermount, !disable_concurrent_write_protection); - blockdev_unlock(fs_usermount); + filesystem_set_writable_by_usb(fs_mount, readonly); + filesystem_set_concurrent_write_protection(fs_mount, !disable_concurrent_write_protection); + blockdev_unlock(fs_mount); #if CIRCUITPY_USB_DEVICE && CIRCUITPY_USB_MSC - usb_msc_remount(fs_usermount); + usb_msc_remount(&fs_mount->fat); #endif } +void common_hal_storage_remount(const char *mount_path, bool readonly, bool disable_concurrent_write_protection) { + const char *path_under_mount; + const char *abs_mount_path = common_hal_os_path_abspath(mount_path); + supervisor_vfs_t *fs_mount = filesystem_for_path(abs_mount_path, &path_under_mount); + if (path_under_mount[0] != 0 && strcmp(abs_mount_path, "/") != 0) { + mp_raise_OSError(MP_EINVAL); + } + if (fs_mount == NULL) { + // Nothing is mounted there (or there is no filesystem at all). + mp_raise_OSError(MP_EINVAL); + } + + remount_vfs(fs_mount, readonly, disable_concurrent_write_protection); +} + void common_hal_storage_erase_filesystem(bool extended) { #if CIRCUITPY_USB_DEVICE usb_disconnect(); diff --git a/supervisor/fatfs.h b/supervisor/fatfs.h index e212664210d..0dc1f3b6876 100644 --- a/supervisor/fatfs.h +++ b/supervisor/fatfs.h @@ -6,6 +6,14 @@ #pragma once +#include + #include "lib/oofatfs/ff.h" void override_fattime(DWORD time); + +// Current time in nanoseconds past 1970/1/1, read from the same RTC source +// that get_fattime() uses. When there is no RTC, this is the same fixed +// fallback timestamp get_fattime() falls back to, so every filesystem kind +// stamps files with the same "current time". +uint64_t get_fattime_ns(void); diff --git a/supervisor/filesystem.h b/supervisor/filesystem.h index 30cd83d4438..14231d2f2dc 100644 --- a/supervisor/filesystem.h +++ b/supervisor/filesystem.h @@ -9,9 +9,145 @@ #include #include "extmod/vfs_fat.h" +#if CIRCUITPY_FILESYSTEM_LITTLEFS +#include "extmod/vfs_lfs.h" +#endif + +// A supervisor-managed filesystem object, which is either a FAT fs_user_mount_t +// or, on littlefs builds, an mp_obj_vfs_lfs2_t. Both begin with the same +// mp_obj_base_t + mp_vfs_blockdev_t prefix, so a bare fs_user_mount_t (e.g. an +// SD card mount) can also be cast to supervisor_vfs_t and inspected through +// .common. The two kinds are told apart by base.type: &mp_fat_vfs_type vs +// &mp_type_vfs_lfs2. +typedef union _supervisor_vfs_t { + struct { + mp_obj_base_t base; + mp_vfs_blockdev_t blockdev; + } common; + fs_user_mount_t fat; + #if CIRCUITPY_FILESYSTEM_LITTLEFS + mp_obj_vfs_lfs2_t lfs2; + #endif +} supervisor_vfs_t; extern volatile bool filesystem_flush_requested; +// Supervisor-level filesystem API. These functions work on both FAT and +// littlefs mounts so that supervisor code (workflows, settings) doesn't have to +// care which one is active. + +// Error codes shared by the supervisor filesystem API. They mirror the subset +// of FatFS FRESULT values used by the workflows so call sites can be shared +// between FAT and littlefs. +typedef enum { + SUPERVISOR_FS_OK = 0, + SUPERVISOR_FS_NO_FILE, // File or directory doesn't exist. + SUPERVISOR_FS_NO_PATH, // A path component is missing (or bad). + SUPERVISOR_FS_EXIST, // File or directory already exists. + SUPERVISOR_FS_WRITE_PROTECTED, // Filesystem or media is not writable. + SUPERVISOR_FS_NO_SPACE, // No space left on the filesystem. + SUPERVISOR_FS_IO, // Any other error. +} supervisor_fs_err_t; + +// Open flags for supervisor_vfs_open_file(). +#define SUPERVISOR_FS_OPEN_READ 0x01 +#define SUPERVISOR_FS_OPEN_WRITE 0x02 +// Create the file if it doesn't exist (with WRITE). An existing file keeps its +// contents. +#define SUPERVISOR_FS_OPEN_CREATE 0x04 +// Empty an existing file on open (with WRITE|CREATE). +#define SUPERVISOR_FS_OPEN_TRUNCATE 0x08 + +// CIRCUITPY-CHANGE: littlefs mtime attribute id, matching extmod/vfs_lfs.c: +// 64-bit little endian, nanoseconds since 1970/1/1. +#ifndef LFS_ATTR_MTIME +#define LFS_ATTR_MTIME (1) +#endif + +// Handle for a file opened with supervisor_vfs_open_file(). +typedef struct _supervisor_vfs_file_t { + bool open; + bool lfs; + supervisor_vfs_t *vfs; + // FAT: timestamp to stamp the file with on close (0 = current time). + DWORD fattime; + union { + FIL fat; + #if CIRCUITPY_FILESYSTEM_LITTLEFS + struct { + lfs2_file_t lfs2; + struct lfs2_file_config cfg; + struct lfs2_attr attr; + uint8_t mtime[8]; + uint8_t buffer[FILESYSTEM_BLOCK_SIZE]; + } lfs; + #endif + } file; +} supervisor_vfs_file_t; + +// Directory handle for supervisor_vfs_opendir(). +#define SUPERVISOR_VFS_DIR_PATH_MAX (256) +typedef struct _supervisor_vfs_dir_t { + bool open; + bool lfs; + supervisor_vfs_t *vfs; + // Directory path used to build per-entry paths for littlefs, which keeps + // modification times in per-path attributes. Unused on FAT. + char path[SUPERVISOR_VFS_DIR_PATH_MAX]; + union { + FF_DIR fat; + #if CIRCUITPY_FILESYSTEM_LITTLEFS + lfs2_dir_t lfs2; + #endif + } dir; +} supervisor_vfs_dir_t; + +// Open a file on the given mount. path is relative to the mount. mtime_ns is +// the modification time to use for a newly written file, in nanoseconds past +// 1970/1/1, or 0 to use the current RTC time (the same time get_fattime() +// stamps FAT files with, including its fixed fallback when there is no RTC). On success, file must be closed with +// supervisor_vfs_close_file(). +supervisor_fs_err_t supervisor_vfs_open_file(supervisor_vfs_t *vfs, const char *path, uint32_t flags, + uint64_t mtime_ns, supervisor_vfs_file_t *file); +supervisor_fs_err_t supervisor_vfs_close_file(supervisor_vfs_file_t *file); +// Read at most len bytes into buf. On success *bytes_read holds the number of +// bytes read; it may be less than len, and 0 at end of file. +supervisor_fs_err_t supervisor_vfs_read_file(supervisor_vfs_file_t *file, void *buf, size_t len, size_t *bytes_read); +// Write len bytes. On success *bytes_written holds the number of bytes +// written; it is less than len when the filesystem is full. +supervisor_fs_err_t supervisor_vfs_write_file(supervisor_vfs_file_t *file, const void *buf, size_t len, size_t *bytes_written); +// Seek to an absolute offset from the start of the file. +supervisor_fs_err_t supervisor_vfs_seek_file(supervisor_vfs_file_t *file, size_t offset); +// Current absolute offset from the start of the file. +size_t supervisor_vfs_tell_file(supervisor_vfs_file_t *file); +// File size in bytes. +size_t supervisor_vfs_file_size(supervisor_vfs_file_t *file); +// Truncate the file to the current offset. +supervisor_fs_err_t supervisor_vfs_truncate_file(supervisor_vfs_file_t *file); + +// Stat a file or directory. All output arguments are optional (may be NULL). +// size is 0 for directories. mtime_ns is nanoseconds past the same epoch used +// for the littlefs mtime attribute (1970/1/1) and is 0 when unknown. +supervisor_fs_err_t supervisor_vfs_stat(supervisor_vfs_t *vfs, const char *path, bool *is_dir, size_t *size, uint64_t *mtime_ns); +// Create a directory. mtime_ns is used to stamp it on FAT, 0 to use the +// current RTC time. littlefs directories have no modification time. +supervisor_fs_err_t supervisor_vfs_mkdir(supervisor_vfs_t *vfs, const char *path, uint64_t mtime_ns); +supervisor_fs_err_t supervisor_vfs_rename(supervisor_vfs_t *vfs, const char *old_path, const char *new_path); +supervisor_fs_err_t supervisor_vfs_unlink(supervisor_vfs_t *vfs, const char *path); + +// Open a directory for listing. path is relative to the mount. +supervisor_fs_err_t supervisor_vfs_opendir(supervisor_vfs_t *vfs, const char *path, supervisor_vfs_dir_t *dir); +// Read the next directory entry. On success name holds a null-terminated entry +// name; at the end of the directory, name is empty. The other output arguments +// are optional (may be NULL). +supervisor_fs_err_t supervisor_vfs_readdir(supervisor_vfs_dir_t *dir, char *name, size_t name_len, bool *is_dir, size_t *size, uint64_t *mtime_ns); +supervisor_fs_err_t supervisor_vfs_rewinddir(supervisor_vfs_dir_t *dir); +supervisor_fs_err_t supervisor_vfs_closedir(supervisor_vfs_dir_t *dir); + +// Filesystem geometry in blocks. For FAT these are clusters; for littlefs, +// blocks. +supervisor_fs_err_t supervisor_vfs_statfs(supervisor_vfs_t *vfs, size_t *block_size, size_t *total_blocks, size_t *free_blocks); + void filesystem_background(void); void filesystem_tick(void); bool filesystem_init(bool create_allowed, bool force_create); @@ -19,30 +155,35 @@ void filesystem_flush(void); bool filesystem_present(void); void filesystem_set_internal_writable_by_usb(bool usb_writable); void filesystem_set_internal_concurrent_write_protection(bool concurrent_write_protection); -void filesystem_set_writable_by_usb(fs_user_mount_t *vfs, bool usb_writable); -void filesystem_set_concurrent_write_protection(fs_user_mount_t *vfs, bool concurrent_write_protection); +void filesystem_set_writable_by_usb(supervisor_vfs_t *vfs, bool usb_writable); +void filesystem_set_concurrent_write_protection(supervisor_vfs_t *vfs, bool concurrent_write_protection); void filesystem_set_ignore_write_protection(fs_user_mount_t *vfs, bool ignore_write_protection); // Whether user code can modify the filesystem. It doesn't depend on the state // of USB. Don't use this for a workflow. In workflows, grab the shared file // system lock. -bool filesystem_is_writable_by_python(fs_user_mount_t *vfs); +bool filesystem_is_writable_by_python(supervisor_vfs_t *vfs); // This controls whether USB tries to grab the underlying block device lock // during enumeration. If another workflow is modifying the filesystem when this // happens, then USB will be readonly. bool filesystem_is_writable_by_usb(fs_user_mount_t *vfs); -fs_user_mount_t *filesystem_circuitpy(void); -fs_user_mount_t *filesystem_for_path(const char *path_in, const char **path_under_mount); -bool filesystem_native_fatfs(fs_user_mount_t *fs_mount); +supervisor_vfs_t *filesystem_circuitpy(void); +supervisor_vfs_t *filesystem_for_path(const char *path_in, const char **path_under_mount); + +// Whether the supervisor-level filesystem API (above) can operate on this +// mount: FAT mounts reachable through FatFS and supervisor littlefs mounts. +// Use it to reject mounts (e.g. non-native or remote filesystems) that the +// workflows cannot access. +bool supervisor_vfs_supported(supervisor_vfs_t *fs_mount); // We have two levels of locking. filesystem_* calls grab a shared blockdev lock to allow // CircuitPython's fatfs code to edit the blocks. blockdev_* calls grab a lock to mutate blocks // directly, excluding any filesystem_* locks. -bool filesystem_lock(fs_user_mount_t *fs_mount); -void filesystem_unlock(fs_user_mount_t *fs_mount); +bool filesystem_lock(supervisor_vfs_t *fs_mount); +void filesystem_unlock(supervisor_vfs_t *fs_mount); -bool blockdev_lock(fs_user_mount_t *fs_mount); -void blockdev_unlock(fs_user_mount_t *fs_mount); +bool blockdev_lock(supervisor_vfs_t *fs_mount); +void blockdev_unlock(supervisor_vfs_t *fs_mount); diff --git a/supervisor/flash.h b/supervisor/flash.h index ca3e42735ff..cdd7eb4ca01 100644 --- a/supervisor/flash.h +++ b/supervisor/flash.h @@ -28,6 +28,7 @@ struct _fs_user_mount_t; void supervisor_flash_init_vfs(struct _fs_user_mount_t *vfs); void supervisor_flash_flush(void); void supervisor_flash_release_cache(void); +void supervisor_flash_mark_dirty(void); void supervisor_flash_set_extended(bool extended); bool supervisor_flash_get_extended(void); diff --git a/supervisor/shared/bluetooth/file_transfer.c b/supervisor/shared/bluetooth/file_transfer.c index d8b5f68ca2d..354c55a9bf1 100644 --- a/supervisor/shared/bluetooth/file_transfer.c +++ b/supervisor/shared/bluetooth/file_transfer.c @@ -14,7 +14,6 @@ #include "shared-bindings/_bleio/Service.h" #include "shared-bindings/_bleio/UUID.h" -#include "supervisor/fatfs.h" #include "supervisor/filesystem.h" #include "supervisor/shared/reload.h" #include "supervisor/shared/bluetooth/file_transfer.h" @@ -118,22 +117,15 @@ void supervisor_stop_bluetooth_file_transfer(void) { #define THIS_COMMAND 0x01 // FATFS has a two second timestamp resolution but the BLE API allows for nanosecond resolution. -// This function truncates the time the time to a resolution storable by FATFS and fills in the -// FATFS encoded version into fattime. -static uint64_t truncate_time(uint64_t input_time, DWORD *fattime) { - timeutils_struct_time_t tm; +// This function truncates the time to a resolution storable by FATFS and returns it. +static uint64_t truncate_time(uint64_t input_time) { uint64_t seconds_since_epoch = timeutils_seconds_since_epoch_from_nanoseconds_since_1970(input_time); - timeutils_seconds_since_epoch_to_struct_time(seconds_since_epoch, &tm); - uint64_t truncated_time = timeutils_nanoseconds_since_epoch_to_nanoseconds_since_1970((seconds_since_epoch / 2) * 2 * 1000000000); - - *fattime = ((tm.tm_year - 1980) << 25) | (tm.tm_mon << 21) | (tm.tm_mday << 16) | - (tm.tm_hour << 11) | (tm.tm_min << 5) | (tm.tm_sec >> 1); - return truncated_time; + return timeutils_nanoseconds_since_epoch_to_nanoseconds_since_1970((seconds_since_epoch / 2) * 2 * 1000000000); } // Used by read and write. -static FIL active_file; -static fs_user_mount_t *active_mount; +static supervisor_vfs_file_t active_file; +static supervisor_vfs_t *active_mount; static uint8_t _process_read(const uint8_t *raw_buf, size_t command_len) { struct read_command *command = (struct read_command *)raw_buf; size_t header_size = sizeof(struct read_command); @@ -158,20 +150,18 @@ static uint8_t _process_read(const uint8_t *raw_buf, size_t command_len) { const char *mount_path; active_mount = filesystem_for_path(full_path, &mount_path); - if (active_mount == NULL || !filesystem_native_fatfs(active_mount)) { + if (active_mount == NULL || !supervisor_vfs_supported(active_mount)) { response.status = STATUS_ERROR; common_hal_bleio_packet_buffer_write(&_transfer_packet_buffer, (const uint8_t *)&response, response_size, NULL, 0); return ANY_COMMAND; } - FATFS *fs = &active_mount->fatfs; - FRESULT result = f_open(fs, &active_file, mount_path, FA_READ); - if (result != FR_OK) { + if (supervisor_vfs_open_file(active_mount, mount_path, SUPERVISOR_FS_OPEN_READ, 0, &active_file) != SUPERVISOR_FS_OK) { response.status = STATUS_ERROR; common_hal_bleio_packet_buffer_write(&_transfer_packet_buffer, (const uint8_t *)&response, response_size, NULL, 0); return ANY_COMMAND; } - uint32_t total_length = f_size(&active_file); + uint32_t total_length = supervisor_vfs_file_size(&active_file); // Write out the response header. uint32_t offset = command->chunk_offset; uint32_t chunk_size = command->chunk_size; @@ -180,20 +170,20 @@ static uint8_t _process_read(const uint8_t *raw_buf, size_t command_len) { response.total_length = total_length; response.data_size = chunk_size; common_hal_bleio_packet_buffer_write(&_transfer_packet_buffer, (const uint8_t *)&response, response_size, NULL, 0); - f_lseek(&active_file, offset); + supervisor_vfs_seek_file(&active_file, offset); // Write out the chunk contents. We can do this in small pieces because PacketBuffer // will assemble them into larger packets of its own. size_t chunk_end = offset + chunk_size; while (offset < chunk_end) { size_t quantity_read; size_t read_amount = MIN(response_size, chunk_end - offset); - f_read(&active_file, data_buffer, read_amount, &quantity_read); + supervisor_vfs_read_file(&active_file, data_buffer, read_amount, &quantity_read); offset += quantity_read; // TODO: Do something if the read fails common_hal_bleio_packet_buffer_write(&_transfer_packet_buffer, data_buffer, quantity_read, NULL, 0); } if (offset >= total_length) { - f_close(&active_file); + supervisor_vfs_close_file(&active_file); return ANY_COMMAND; } return READ_PACING; @@ -206,14 +196,14 @@ static uint8_t _process_read_pacing(const uint8_t *raw_buf, size_t command_len) response.status = STATUS_OK; size_t response_size = sizeof(struct read_data); - uint32_t total_length = f_size(&active_file); + uint32_t total_length = supervisor_vfs_file_size(&active_file); // Write out the response header. uint32_t chunk_size = MIN(command->chunk_size, total_length - command->chunk_offset); response.chunk_offset = command->chunk_offset; response.total_length = total_length; response.data_size = chunk_size; common_hal_bleio_packet_buffer_write(&_transfer_packet_buffer, (const uint8_t *)&response, response_size, NULL, 0); - f_lseek(&active_file, command->chunk_offset); + supervisor_vfs_seek_file(&active_file, command->chunk_offset); // Write out the chunk contents. We can do this in small pieces because PacketBuffer // will assemble them into larger packets of its own. size_t chunk_offset = 0; @@ -221,8 +211,8 @@ static uint8_t _process_read_pacing(const uint8_t *raw_buf, size_t command_len) while (chunk_offset < chunk_size) { size_t quantity_read; size_t read_size = MIN(chunk_size - chunk_offset, sizeof(data)); - FRESULT result = f_read(&active_file, &data, read_size, &quantity_read); - if (quantity_read == 0 || result != FR_OK) { + supervisor_fs_err_t result = supervisor_vfs_read_file(&active_file, &data, read_size, &quantity_read); + if (quantity_read == 0 || result != SUPERVISOR_FS_OK) { // TODO: If we can't read everything, then the file must have been shortened. Maybe we // should return 0s to pad it out. break; @@ -231,7 +221,7 @@ static uint8_t _process_read_pacing(const uint8_t *raw_buf, size_t command_len) chunk_offset += quantity_read; } if ((chunk_offset + chunk_size) >= total_length) { - f_close(&active_file); + supervisor_vfs_close_file(&active_file); return ANY_COMMAND; } return READ_PACING; @@ -264,7 +254,7 @@ static uint8_t _process_write(const uint8_t *raw_buf, size_t command_len) { const char *mount_path; active_mount = filesystem_for_path(full_path, &mount_path); - if (active_mount == NULL || !filesystem_native_fatfs(active_mount)) { + if (active_mount == NULL || !supervisor_vfs_supported(active_mount)) { response.status = STATUS_ERROR; common_hal_bleio_packet_buffer_write(&_transfer_packet_buffer, (const uint8_t *)&response, sizeof(struct write_pacing), NULL, 0); return ANY_COMMAND; @@ -275,16 +265,13 @@ static uint8_t _process_write(const uint8_t *raw_buf, size_t command_len) { return ANY_COMMAND; } - FATFS *fs = &active_mount->fatfs; - DWORD fattime; - _truncated_time = truncate_time(command->modification_time, &fattime); - override_fattime(fattime); - FRESULT result = f_open(fs, &active_file, mount_path, FA_WRITE | FA_OPEN_ALWAYS); - if (result != FR_OK) { + _truncated_time = truncate_time(command->modification_time); + supervisor_fs_err_t result = supervisor_vfs_open_file(active_mount, mount_path, + SUPERVISOR_FS_OPEN_WRITE | SUPERVISOR_FS_OPEN_CREATE, command->modification_time, &active_file); + if (result != SUPERVISOR_FS_OK) { response.status = STATUS_ERROR; common_hal_bleio_packet_buffer_write(&_transfer_packet_buffer, (const uint8_t *)&response, sizeof(struct write_pacing), NULL, 0); filesystem_unlock(active_mount); - override_fattime(0); return ANY_COMMAND; } // Write out the pacing response. @@ -294,10 +281,9 @@ static uint8_t _process_write(const uint8_t *raw_buf, size_t command_len) { size_t chunk_size = MIN(total_write_length - offset, 512 - (offset % 512)); // Special case when truncating the file. (Deleting stuff off the end.) if (chunk_size == 0) { - f_lseek(&active_file, offset); - f_truncate(&active_file); - f_close(&active_file); - override_fattime(0); + supervisor_vfs_seek_file(&active_file, offset); + supervisor_vfs_truncate_file(&active_file); + supervisor_vfs_close_file(&active_file); filesystem_unlock(active_mount); } response.offset = offset; @@ -324,7 +310,6 @@ static uint8_t _process_write_data(const uint8_t *raw_buf, size_t command_len) { response.status = STATUS_ERROR; common_hal_bleio_packet_buffer_write(&_transfer_packet_buffer, (const uint8_t *)&response, sizeof(struct write_pacing), NULL, 0); filesystem_unlock(active_mount); - override_fattime(0); return ANY_COMMAND; } // We need to receive another packet to have the full path. @@ -332,15 +317,14 @@ static uint8_t _process_write_data(const uint8_t *raw_buf, size_t command_len) { return THIS_COMMAND; } uint32_t offset = command->offset; - f_lseek(&active_file, offset); - UINT actual; - f_write(&active_file, command->data, command->data_size, &actual); - if (actual < command->data_size) { // -1 for the null we'll write + supervisor_vfs_seek_file(&active_file, offset); + size_t actual; + supervisor_fs_err_t result = supervisor_vfs_write_file(&active_file, command->data, command->data_size, &actual); + if (actual < command->data_size || result != SUPERVISOR_FS_OK) { // -1 for the null we'll write // TODO: throw away any more packets of path. response.status = STATUS_ERROR; common_hal_bleio_packet_buffer_write(&_transfer_packet_buffer, (const uint8_t *)&response, sizeof(struct write_pacing), NULL, 0); filesystem_unlock(active_mount); - override_fattime(0); return ANY_COMMAND; } offset += command->data_size; @@ -351,9 +335,8 @@ static uint8_t _process_write_data(const uint8_t *raw_buf, size_t command_len) { response.truncated_time = _truncated_time; common_hal_bleio_packet_buffer_write(&_transfer_packet_buffer, (const uint8_t *)&response, sizeof(struct write_pacing), NULL, 0); if (total_write_length == offset) { - f_truncate(&active_file); - f_close(&active_file); - override_fattime(0); + supervisor_vfs_truncate_file(&active_file); + supervisor_vfs_close_file(&active_file); filesystem_unlock(active_mount); // Don't reload until everything is written out of the packet buffer. common_hal_bleio_packet_buffer_flush(&_transfer_packet_buffer); @@ -382,16 +365,16 @@ static uint8_t _process_delete(const uint8_t *raw_buf, size_t command_len) { char *full_path = (char *)((uint8_t *)command) + header_size; full_path[command->path_length] = '\0'; - FRESULT result = supervisor_workflow_delete_recursive(full_path); + supervisor_fs_err_t result = supervisor_workflow_delete_recursive(full_path); - if (result == FR_WRITE_PROTECTED) { + if (result == SUPERVISOR_FS_WRITE_PROTECTED) { response.status = STATUS_ERROR_READONLY; } - if (result != FR_OK) { + if (result != SUPERVISOR_FS_OK) { response.status = STATUS_ERROR; } common_hal_bleio_packet_buffer_write(&_transfer_packet_buffer, (const uint8_t *)&response, sizeof(struct delete_status), NULL, 0); - if (result == FR_OK) { + if (result == SUPERVISOR_FS_OK) { // Don't reload until everything is written out of the packet buffer. common_hal_bleio_packet_buffer_flush(&_transfer_packet_buffer); } @@ -428,14 +411,12 @@ static uint8_t _process_mkdir(const uint8_t *raw_buf, size_t command_len) { char *full_path = (char *)command->path; _terminate_path(full_path, command->path_length); - DWORD fattime; - response.truncated_time = truncate_time(command->modification_time, &fattime); - FRESULT result = supervisor_workflow_mkdir(fattime, full_path); - if (result != FR_OK) { + supervisor_fs_err_t result = supervisor_workflow_mkdir(truncate_time(command->modification_time), full_path); + if (result != SUPERVISOR_FS_OK) { response.status = STATUS_ERROR; } common_hal_bleio_packet_buffer_write(&_transfer_packet_buffer, (const uint8_t *)&response, sizeof(struct mkdir_status), NULL, 0); - if (result == FR_OK) { + if (result == SUPERVISOR_FS_OK) { // Don't reload until everything is written out of the packet buffer. common_hal_bleio_packet_buffer_flush(&_transfer_packet_buffer); } @@ -457,6 +438,9 @@ static uint8_t _process_listdir(uint8_t *raw_buf, size_t command_len) { const struct listdir_command *command = (struct listdir_command *)raw_buf; struct listdir_entry *entry = (struct listdir_entry *)raw_buf; size_t header_size = sizeof(struct listdir_command); + bool entry_is_dir; + size_t entry_size; + uint64_t entry_mtime; mp_int_t max_packet_size = common_hal_bleio_packet_buffer_get_outgoing_packet_length(&_transfer_packet_buffer); if (max_packet_size < 0) { // -1 means we're disconnected @@ -481,16 +465,16 @@ static uint8_t _process_listdir(uint8_t *raw_buf, size_t command_len) { const char *mount_path; active_mount = filesystem_for_path(full_path, &mount_path); - if (active_mount == NULL || !filesystem_native_fatfs(active_mount)) { + if (active_mount == NULL || !supervisor_vfs_supported(active_mount)) { entry->command = LISTDIR_ENTRY; entry->status = STATUS_ERROR_NO_FILE; send_listdir_entry_header(entry, max_packet_size); return ANY_COMMAND; } - FATFS *fs = &active_mount->fatfs; - FF_DIR dir; - FRESULT res = f_opendir(fs, &dir, mount_path); + supervisor_vfs_dir_t dir; + memset(&dir, 0, sizeof(dir)); + supervisor_fs_err_t res = supervisor_vfs_opendir(active_mount, mount_path, &dir); entry->command = LISTDIR_ENTRY; entry->status = STATUS_OK; @@ -499,51 +483,44 @@ static uint8_t _process_listdir(uint8_t *raw_buf, size_t command_len) { entry->entry_count = 0; entry->flags = 0; - if (res != FR_OK) { + if (res != SUPERVISOR_FS_OK) { entry->status = STATUS_ERROR_NO_FILE; send_listdir_entry_header(entry, max_packet_size); return ANY_COMMAND; } - FILINFO file_info; - res = f_readdir(&dir, &file_info); - char *fn = file_info.fname; + char fn[FF_MAX_LFN + 1]; + res = supervisor_vfs_readdir(&dir, fn, sizeof(fn), NULL, NULL, NULL); size_t total_entries = 0; - while (res == FR_OK && fn[0] != 0) { - res = f_readdir(&dir, &file_info); + while (res == SUPERVISOR_FS_OK && fn[0] != 0) { + res = supervisor_vfs_readdir(&dir, fn, sizeof(fn), NULL, NULL, NULL); total_entries += 1; } // Rewind the directory. - f_readdir(&dir, NULL); + supervisor_vfs_rewinddir(&dir); entry->entry_count = total_entries; for (size_t i = 0; i < total_entries; i++) { - res = f_readdir(&dir, &file_info); + res = supervisor_vfs_readdir(&dir, fn, sizeof(fn), &entry_is_dir, &entry_size, &entry_mtime); entry->entry_number = i; - uint64_t truncated_time = timeutils_mktime(1980 + (file_info.fdate >> 9), - (file_info.fdate >> 5) & 0xf, - file_info.fdate & 0x1f, - file_info.ftime >> 11, - (file_info.ftime >> 5) & 0x1f, - (file_info.ftime & 0x1f) * 2) * 1000000000ULL; - entry->truncated_time = truncated_time; - if ((file_info.fattrib & AM_DIR) != 0) { + entry->truncated_time = entry_mtime; + if (entry_is_dir) { entry->flags = 1; // Directory entry->file_size = 0; } else { entry->flags = 0; - entry->file_size = file_info.fsize; + entry->file_size = entry_size; } - size_t name_length = strlen(file_info.fname); + size_t name_length = strlen(fn); entry->path_length = name_length; send_listdir_entry_header(entry, max_packet_size); size_t fn_offset = 0; while (fn_offset < name_length) { size_t fn_size = MIN(name_length - fn_offset, 4); - common_hal_bleio_packet_buffer_write(&_transfer_packet_buffer, ((uint8_t *)file_info.fname) + fn_offset, fn_size, NULL, 0); + common_hal_bleio_packet_buffer_write(&_transfer_packet_buffer, ((uint8_t *)fn) + fn_offset, fn_size, NULL, 0); fn_offset += fn_size; } } - f_closedir(&dir); + supervisor_vfs_closedir(&dir); entry->path_length = 0; entry->entry_number = entry->entry_count; entry->flags = 0; @@ -577,14 +554,14 @@ static uint8_t _process_move(const uint8_t *raw_buf, size_t command_len) { char *new_path = old_path + command->old_path_length + 1; new_path[command->new_path_length] = '\0'; - FRESULT result = supervisor_workflow_move(old_path, new_path); - if (result == FR_WRITE_PROTECTED) { + supervisor_fs_err_t result = supervisor_workflow_move(old_path, new_path); + if (result == SUPERVISOR_FS_WRITE_PROTECTED) { response.status = STATUS_ERROR_READONLY; - } else if (result != FR_OK) { + } else if (result != SUPERVISOR_FS_OK) { response.status = STATUS_ERROR; } common_hal_bleio_packet_buffer_write(&_transfer_packet_buffer, (const uint8_t *)&response, sizeof(struct move_status), NULL, 0); - if (result == FR_OK) { + if (result == SUPERVISOR_FS_OK) { // Don't reload until everything is written out of the packet buffer. common_hal_bleio_packet_buffer_flush(&_transfer_packet_buffer); } @@ -678,6 +655,6 @@ void supervisor_bluetooth_file_transfer_background(void) { void supervisor_bluetooth_file_transfer_disconnected(void) { next_command = ANY_COMMAND; current_offset = 0; - f_close(&active_file); + supervisor_vfs_close_file(&active_file); autoreload_resume(AUTORELOAD_SUSPEND_BLE); } diff --git a/supervisor/shared/display.c b/supervisor/shared/display.c index 1352c76b006..798a1b16e0e 100644 --- a/supervisor/shared/display.c +++ b/supervisor/shared/display.c @@ -46,8 +46,6 @@ #if CIRCUITPY_LVFONTIO #include "shared-bindings/lvfontio/OnDiskFont.h" #include "supervisor/filesystem.h" -#include "extmod/vfs_fat.h" -#include "lib/oofatfs/ff.h" #include "supervisor/shared/serial.h" @@ -58,13 +56,13 @@ static bool check_for_custom_font(const char **font_path_out) { return false; } - fs_user_mount_t *vfs = filesystem_circuitpy(); + // CIRCUITPY-CHANGE: Check for the font through the supervisor filesystem + // API so it works on FAT and littlefs mounts. + supervisor_vfs_t *vfs = filesystem_circuitpy(); if (vfs == NULL) { return false; } - // Use FATFS directly to check if file exists - FILINFO file_info; const char *default_font_path = "/fonts/terminal.lvfontbin"; const char *font_path = default_font_path; @@ -77,8 +75,7 @@ static bool check_for_custom_font(const char **font_path_out) { } #endif - FRESULT result = f_stat(&vfs->fatfs, font_path, &file_info); - if (result == FR_OK) { + if (supervisor_vfs_stat(vfs, font_path, NULL, NULL, NULL) == SUPERVISOR_FS_OK) { if (font_path_out != NULL) { *font_path_out = font_path; } @@ -87,9 +84,8 @@ static bool check_for_custom_font(const char **font_path_out) { // If custom font path doesn't exist, use default font font_path = default_font_path; - result = f_stat(&vfs->fatfs, font_path, &file_info); - if (result == FR_OK) { + if (supervisor_vfs_stat(vfs, font_path, NULL, NULL, NULL) == SUPERVISOR_FS_OK) { if (font_path_out != NULL) { *font_path_out = font_path; } diff --git a/supervisor/shared/fatfs.c b/supervisor/shared/fatfs.c index 07ab724d7a9..ba437158f73 100644 --- a/supervisor/shared/fatfs.c +++ b/supervisor/shared/fatfs.c @@ -15,6 +15,21 @@ #include "shared/timeutils/timeutils.h" DWORD _time_override = 0; + +// 2016-09-01 16:43:35 UTC, the fallback timestamp used when there is no RTC. +#define FATTIME_FALLBACK_NS (1472748215000000000ULL) + +uint64_t get_fattime_ns(void) { + #if CIRCUITPY_RTC + timeutils_struct_time_t tm; + common_hal_rtc_get_time(&tm); + return (uint64_t)timeutils_mktime_1970(tm.tm_year, tm.tm_mon, tm.tm_mday, + tm.tm_hour, tm.tm_min, tm.tm_sec) * 1000000000ULL; + #else + return FATTIME_FALLBACK_NS; + #endif +} + DWORD get_fattime(void) { if (_time_override > 0) { return _time_override; @@ -25,6 +40,7 @@ DWORD get_fattime(void) { return ((tm.tm_year - 1980) << 25) | (tm.tm_mon << 21) | (tm.tm_mday << 16) | (tm.tm_hour << 11) | (tm.tm_min << 5) | (tm.tm_sec >> 1); #else + // Same instant as FATTIME_FALLBACK_NS: 2016-09-01 16:43:35 UTC. return ((2016 - 1980) << 25) | ((9) << 21) | ((1) << 16) | ((16) << 11) | ((43) << 5) | (35 / 2); #endif } diff --git a/supervisor/shared/filesystem.c b/supervisor/shared/filesystem.c index 4335e9afc6e..12d42da8f07 100644 --- a/supervisor/shared/filesystem.c +++ b/supervisor/shared/filesystem.c @@ -12,9 +12,18 @@ #include "py/mpstate.h" +#include "shared/timeutils/timeutils.h" +#include "supervisor/fatfs.h" #include "supervisor/flash.h" #include "supervisor/linker.h" +#if CIRCUITPY_FILESYSTEM_LITTLEFS +#include "extmod/vfs_lfs.h" +#include "lib/littlefs/lfs2.h" +#include "supervisor/port_heap.h" +#include "supervisor/shared/tick.h" +#endif + #if CIRCUITPY_SDCARDIO #include "shared-module/sdcardio/__init__.h" #endif @@ -24,7 +33,11 @@ #endif static mp_vfs_mount_t _circuitpy_vfs; -static fs_user_mount_t _circuitpy_usermount; +static supervisor_vfs_t _circuitpy_mount; + +#if CIRCUITPY_FILESYSTEM_LITTLEFS +static bool _lfs_freshly_formatted; +#endif #if CIRCUITPY_SAVES_PARTITION_SIZE > 0 static mp_vfs_mount_t _saves_vfs; @@ -34,6 +47,90 @@ static fs_user_mount_t _saves_usermount; static volatile uint32_t filesystem_flush_interval_ms = CIRCUITPY_FILESYSTEM_FLUSH_INTERVAL_MS; volatile bool filesystem_flush_requested = false; +// Mapping between FatFS and littlefs results and the supervisor filesystem +// error codes. +static supervisor_fs_err_t fat_error(FRESULT res) { + switch (res) { + case FR_OK: + return SUPERVISOR_FS_OK; + case FR_NO_FILE: + return SUPERVISOR_FS_NO_FILE; + case FR_NO_PATH: + case FR_INVALID_NAME: + return SUPERVISOR_FS_NO_PATH; + case FR_EXIST: + return SUPERVISOR_FS_EXIST; + case FR_WRITE_PROTECTED: + return SUPERVISOR_FS_WRITE_PROTECTED; + case FR_DENIED: + // FatFS uses FR_DENIED for read-only media and for no free + // cluster. We can't tell them apart, so report no space. + return SUPERVISOR_FS_NO_SPACE; + default: + return SUPERVISOR_FS_IO; + } +} + +// Convert nanoseconds past 1970/1/1 to the FAT timestamp format. +static DWORD fattime_from_ns(uint64_t ns) { + timeutils_struct_time_t tm; + uint64_t seconds = timeutils_seconds_since_epoch_from_nanoseconds_since_1970(ns); + timeutils_seconds_since_epoch_to_struct_time(seconds, &tm); + return ((tm.tm_year - 1980) << 25) | (tm.tm_mon << 21) | (tm.tm_mday << 16) | + (tm.tm_hour << 11) | (tm.tm_min << 5) | (tm.tm_sec >> 1); +} + +#if CIRCUITPY_FILESYSTEM_LITTLEFS +static supervisor_fs_err_t lfs_error(int err) { + switch (err) { + case LFS2_ERR_OK: + return SUPERVISOR_FS_OK; + case LFS2_ERR_NOENT: + return SUPERVISOR_FS_NO_FILE; + case LFS2_ERR_EXIST: + return SUPERVISOR_FS_EXIST; + case LFS2_ERR_NOTDIR: + case LFS2_ERR_ISDIR: + return SUPERVISOR_FS_NO_PATH; + case LFS2_ERR_NOSPC: + return SUPERVISOR_FS_NO_SPACE; + default: + return SUPERVISOR_FS_IO; + } +} + +// Store nanoseconds past the littlefs mtime epoch (1970/1/1) little-endian. +static void lfs_mtime_store(uint8_t buf[8], uint64_t ns) { + ns = timeutils_nanoseconds_since_epoch_to_nanoseconds_since_1970(ns); + for (size_t i = 0; i < 8; ++i) { + buf[i] = ns; + ns >>= 8; + } +} + +static uint64_t lfs_mtime_load(const uint8_t buf[8]) { + uint64_t ns = 0; + for (size_t i = 8; i > 0; --i) { + ns = ns << 8 | buf[i - 1]; + } + return timeutils_nanoseconds_since_epoch_to_nanoseconds_since_1970(ns); +} + +// Read a file or directory modification time from the littlefs mtime attribute. +static uint64_t lfs_get_path_mtime(lfs2_t *lfs, const char *path) { + uint8_t mtime[8]; + lfs2_ssize_t sz = lfs2_getattr(lfs, path, LFS_ATTR_MTIME, mtime, sizeof(mtime)); + if (sz != (lfs2_ssize_t)sizeof(mtime)) { + return 0; + } + return lfs_mtime_load(mtime); +} + +static bool is_lfs(supervisor_vfs_t *vfs) { + return vfs->common.base.type == &mp_type_vfs_lfs2; +} +#endif + void filesystem_background(void) { if (filesystem_flush_requested) { filesystem_flush_interval_ms = CIRCUITPY_FILESYSTEM_FLUSH_INTERVAL_MS; @@ -56,37 +153,177 @@ inline void filesystem_tick(void) { } } - __attribute__((unused)) // this function MAY be unused -static void make_empty_file(FATFS *fatfs, const char *path) { - FIL fp; - f_open(fatfs, &fp, path, FA_WRITE | FA_CREATE_ALWAYS); - f_close(&fp); +static bool make_empty_file(supervisor_vfs_t *vfs, const char *path) { + supervisor_vfs_file_t fp; + supervisor_fs_err_t err = supervisor_vfs_open_file(vfs, path, + SUPERVISOR_FS_OPEN_WRITE | SUPERVISOR_FS_OPEN_CREATE | SUPERVISOR_FS_OPEN_TRUNCATE, 0, &fp); + if (err != SUPERVISOR_FS_OK) { + return false; + } + return supervisor_vfs_close_file(&fp) == SUPERVISOR_FS_OK; } #if CIRCUITPY_FULL_BUILD -#define MAKE_FILE_WITH_OPTIONAL_CONTENTS(fatfs, filename, string_literal) do { \ +#define MAKE_FILE_WITH_OPTIONAL_CONTENTS(vfs, filename, string_literal) do { \ const byte buffer[] = string_literal; \ - make_file_with_contents(fatfs, filename, buffer, sizeof(buffer) - 1); \ + make_file_with_contents(vfs, filename, buffer, sizeof(buffer) - 1); \ } while (0) -static void make_file_with_contents(FATFS *fatfs, const char *filename, const byte *content, UINT size) { - FIL fs; - // Create or modify existing code.py file - f_open(fatfs, &fs, filename, FA_WRITE | FA_CREATE_ALWAYS); - f_write(&fs, content, size, &size); - f_close(&fs); +static bool make_file_with_contents(supervisor_vfs_t *vfs, const char *filename, const byte *content, UINT size) { + supervisor_vfs_file_t fp; + if (supervisor_vfs_open_file(vfs, filename, + SUPERVISOR_FS_OPEN_WRITE | SUPERVISOR_FS_OPEN_CREATE | SUPERVISOR_FS_OPEN_TRUNCATE, 0, &fp) != SUPERVISOR_FS_OK) { + return false; + } + size_t written; + supervisor_vfs_write_file(&fp, content, size, &written); + return supervisor_vfs_close_file(&fp) == SUPERVISOR_FS_OK && written == size; } #else -#define MAKE_FILE_WITH_OPTIONAL_CONTENTS(fatfs, filename, string_literal) \ - make_empty_file(fatfs, filename) +#define MAKE_FILE_WITH_OPTIONAL_CONTENTS(vfs, filename, string_literal) \ + (void)make_empty_file(vfs, filename) #endif +#if CIRCUITPY_FILESYSTEM_LITTLEFS + +// littlefs block-device adapter for the supervisor flash block device. The +// block size is FILESYSTEM_BLOCK_SIZE so lfs2 blocks map 1:1 onto supervisor +// flash blocks. Reads/writes that are not block-aligned are served through a +// read-modify-write because the supervisor block device is block granular. + +static int lfs_flash_read(const struct lfs2_config *c, lfs2_block_t block, lfs2_off_t off, void *buffer, lfs2_size_t size) { + if (off == 0 && size == FILESYSTEM_BLOCK_SIZE) { + return supervisor_flash_read_blocks(buffer, block, 1) == 0 ? LFS2_ERR_OK : LFS2_ERR_IO; + } + uint8_t tmp[FILESYSTEM_BLOCK_SIZE]; + if (supervisor_flash_read_blocks(tmp, block, 1) != 0) { + return LFS2_ERR_IO; + } + memcpy(buffer, tmp + off, size); + return LFS2_ERR_OK; +} + +static int lfs_flash_prog(const struct lfs2_config *c, lfs2_block_t block, lfs2_off_t off, const void *buffer, lfs2_size_t size) { + supervisor_flash_mark_dirty(); + if (off == 0 && size == FILESYSTEM_BLOCK_SIZE) { + return supervisor_flash_write_blocks(buffer, block, 1) == 0 ? LFS2_ERR_OK : LFS2_ERR_IO; + } + uint8_t tmp[FILESYSTEM_BLOCK_SIZE]; + if (supervisor_flash_read_blocks(tmp, block, 1) != 0) { + return LFS2_ERR_IO; + } + memcpy(tmp + off, buffer, size); + if (supervisor_flash_write_blocks(tmp, block, 1) != 0) { + return LFS2_ERR_IO; + } + return LFS2_ERR_OK; +} + +// The supervisor block device erases as part of its write path, so there is +// nothing to do here. +static int lfs_flash_erase(const struct lfs2_config *c, lfs2_block_t block) { + return LFS2_ERR_OK; +} + +static int lfs_flash_sync(const struct lfs2_config *c) { + supervisor_flash_flush(); + return LFS2_ERR_OK; +} + +static void *lfs_supervisor_alloc(size_t size) { + return port_malloc(size, false); +} + +static bool lfs_write_boot_file(const char *path, const char *contents, size_t size) { + supervisor_vfs_file_t file; + if (supervisor_vfs_open_file(&_circuitpy_mount, path, + SUPERVISOR_FS_OPEN_WRITE | SUPERVISOR_FS_OPEN_CREATE | SUPERVISOR_FS_OPEN_TRUNCATE, 0, &file) != SUPERVISOR_FS_OK) { + return false; + } + size_t written = 0; + supervisor_vfs_write_file(&file, contents, size, &written); + supervisor_fs_err_t err = supervisor_vfs_close_file(&file); + return written == size && err == SUPERVISOR_FS_OK; +} + +#endif // CIRCUITPY_FILESYSTEM_LITTLEFS + // we don't make this function static because it needs a lot of stack and we // want it to be executed without using stack within main() function bool filesystem_init(bool create_allowed, bool force_create) { + mp_vfs_mount_t *circuitpy_vfs = &_circuitpy_vfs; + circuitpy_vfs->len = 0; + + #if CIRCUITPY_FILESYSTEM_LITTLEFS + // init the vfs object. The VM (and its GC) is not running yet, so + // everything here uses static storage or the port allocator. + mp_obj_vfs_lfs2_t *circuitpy = &_circuitpy_mount.lfs2; + memset(circuitpy, 0, sizeof(*circuitpy)); + circuitpy->base.type = &mp_type_vfs_lfs2; + // cur_dir is used to join relative paths (make_path). It cannot be + // allocated on the GC heap because the VM is not running yet, so give it + // a static buffer. It is truncated back to its previous length after each + // join, so it stays short in practice. + static char lfs_cur_dir_buf[128]; + vstr_init_fixed_buf(&circuitpy->cur_dir, sizeof(lfs_cur_dir_buf), lfs_cur_dir_buf); + vstr_add_str(&circuitpy->cur_dir, "/"); + circuitpy->enable_mtime = true; + + struct lfs2_config *config = &circuitpy->config; + memset(config, 0, sizeof(*config)); + config->context = circuitpy; + config->read = lfs_flash_read; + config->prog = lfs_flash_prog; + config->erase = lfs_flash_erase; + config->sync = lfs_flash_sync; + config->block_size = supervisor_flash_get_block_size(); + // Initialize the block device (opens the partition) before asking for its + // geometry; the FAT path does the same via the blockdev INIT ioctl. + supervisor_flash_init(); + config->block_count = supervisor_flash_get_block_count(); + config->read_size = config->block_size; + config->prog_size = config->block_size; + + _lfs_freshly_formatted = false; + if (force_create) { + // Reformat requested (e.g. from safe mode). + if (lfs2_format(&circuitpy->lfs, config) != LFS2_ERR_OK) { + return false; + } + _lfs_freshly_formatted = true; + } + int mount_err; + mp_obj_t mounted = mp_vfs_lfs2_mount_supervisor(circuitpy, lfs_supervisor_alloc, + create_allowed && !force_create, &_lfs_freshly_formatted, &mount_err); + if (mounted == MP_OBJ_NULL) { + return false; + } + (void)mount_err; + + if (_lfs_freshly_formatted) { + #if CIRCUITPY_SDCARDIO + lfs2_mkdir(&circuitpy->lfs, "/sd"); + #endif + + #if CIRCUITPY_SETTINGS_TOML + // settings.toml is not read yet on littlefs filesystems; see + // supervisor/shared/settings.c. Create it empty for forward + // compatibility. + lfs_write_boot_file("/settings.toml", "", 0); + #endif + // make a sample code.py file + lfs_write_boot_file("/code.py", "print(\"Hello World!\")\n", sizeof("print(\"Hello World!\")\n") - 1); + + // create empty lib directory + lfs2_mkdir(&circuitpy->lfs, "/lib"); + + // and ensure everything is flushed + supervisor_flash_flush(); + } + #else // init the vfs object - fs_user_mount_t *circuitpy = &_circuitpy_usermount; + fs_user_mount_t *circuitpy = &_circuitpy_mount.fat; circuitpy->blockdev.flags = 0; supervisor_flash_init_vfs(circuitpy); @@ -104,9 +341,6 @@ bool filesystem_init(bool create_allowed, bool force_create) { filesystem_set_writable_by_usb(saves, false); #endif - mp_vfs_mount_t *circuitpy_vfs = &_circuitpy_vfs; - circuitpy_vfs->len = 0; - // try to mount the flash FRESULT res = f_mount(&circuitpy->fatfs); if ((res == FR_NO_FILESYSTEM && create_allowed) || force_create) { @@ -139,19 +373,19 @@ bool filesystem_init(bool create_allowed, bool force_create) { if (res != FR_OK) { return false; } - make_empty_file(&circuitpy->fatfs, "/.fseventsd/no_log"); - make_empty_file(&circuitpy->fatfs, "/.metadata_never_index"); + make_empty_file(&_circuitpy_mount, "/.fseventsd/no_log"); + make_empty_file(&_circuitpy_mount, "/.metadata_never_index"); // Prevent storing trash on all OSes. - make_empty_file(&circuitpy->fatfs, "/.Trashes"); // MacOS - make_empty_file(&circuitpy->fatfs, "/.Trash-1000"); // Linux, XDG trash spec: + make_empty_file(&_circuitpy_mount, "/.Trashes"); // MacOS + make_empty_file(&_circuitpy_mount, "/.Trash-1000"); // Linux, XDG trash spec: // https://specifications.freedesktop.org/trash-spec/trashspec-latest.html #endif #if CIRCUITPY_SDCARDIO || CIRCUITPY_SDIOIO res = f_mkdir(&circuitpy->fatfs, "/sd"); #if CIRCUITPY_FULL_BUILD - MAKE_FILE_WITH_OPTIONAL_CONTENTS(&circuitpy->fatfs, "/sd/placeholder.txt", + MAKE_FILE_WITH_OPTIONAL_CONTENTS(&_circuitpy_mount, "/sd/placeholder.txt", "SD cards mounted at /sd will hide this file from Python.\n"); #endif #endif @@ -169,7 +403,7 @@ bool filesystem_init(bool create_allowed, bool force_create) { } #if CIRCUITPY_FULL_BUILD if (res == FR_OK) { - MAKE_FILE_WITH_OPTIONAL_CONTENTS(&circuitpy->fatfs, "/saves/placeholder.txt", + MAKE_FILE_WITH_OPTIONAL_CONTENTS(&_circuitpy_mount, "/saves/placeholder.txt", "A separate filesystem mounted at /saves will hide this file from Python." " Saves are visible via USB CPSAVES.\n"); } @@ -177,10 +411,10 @@ bool filesystem_init(bool create_allowed, bool force_create) { #endif #if CIRCUITPY_SETTINGS_TOML - make_empty_file(&circuitpy->fatfs, "/settings.toml"); + make_empty_file(&_circuitpy_mount, "/settings.toml"); #endif // make a sample code.py file - MAKE_FILE_WITH_OPTIONAL_CONTENTS(&circuitpy->fatfs, "/code.py", "print(\"Hello World!\")\n"); + MAKE_FILE_WITH_OPTIONAL_CONTENTS(&_circuitpy_mount, "/code.py", "print(\"Hello World!\")\n"); // create empty lib directory res = f_mkdir(&circuitpy->fatfs, "/lib"); @@ -193,6 +427,7 @@ bool filesystem_init(bool create_allowed, bool force_create) { } else if (res != FR_OK) { return false; } + #endif // CIRCUITPY_FILESYSTEM_LITTLEFS circuitpy_vfs->str = "/"; circuitpy_vfs->len = 1; @@ -253,23 +488,21 @@ void PLACE_IN_ITCM(filesystem_flush)(void) { } void filesystem_set_internal_writable_by_usb(bool writable) { - fs_user_mount_t *vfs = &_circuitpy_usermount; - - filesystem_set_writable_by_usb(vfs, writable); + filesystem_set_writable_by_usb(&_circuitpy_mount, writable); } -void filesystem_set_writable_by_usb(fs_user_mount_t *vfs, bool usb_writable) { +void filesystem_set_writable_by_usb(supervisor_vfs_t *vfs, bool usb_writable) { if (usb_writable) { - vfs->blockdev.flags |= MP_BLOCKDEV_FLAG_USB_WRITABLE; + vfs->common.blockdev.flags |= MP_BLOCKDEV_FLAG_USB_WRITABLE; } else { - vfs->blockdev.flags &= ~MP_BLOCKDEV_FLAG_USB_WRITABLE; + vfs->common.blockdev.flags &= ~MP_BLOCKDEV_FLAG_USB_WRITABLE; } } -bool filesystem_is_writable_by_python(fs_user_mount_t *vfs) { - return ((vfs->blockdev.flags & MP_BLOCKDEV_FLAG_CONCURRENT_WRITE_PROTECTED) == 0) || - ((vfs->blockdev.flags & MP_BLOCKDEV_FLAG_USB_WRITABLE) == 0) || - ((vfs->blockdev.flags & MP_BLOCKDEV_FLAG_IGNORE_WRITE_PROTECTION) != 0); +bool filesystem_is_writable_by_python(supervisor_vfs_t *vfs) { + return ((vfs->common.blockdev.flags & MP_BLOCKDEV_FLAG_CONCURRENT_WRITE_PROTECTED) == 0) || + ((vfs->common.blockdev.flags & MP_BLOCKDEV_FLAG_USB_WRITABLE) == 0) || + ((vfs->common.blockdev.flags & MP_BLOCKDEV_FLAG_IGNORE_WRITE_PROTECTION) != 0); } bool filesystem_is_writable_by_usb(fs_user_mount_t *vfs) { @@ -279,14 +512,14 @@ bool filesystem_is_writable_by_usb(fs_user_mount_t *vfs) { } void filesystem_set_internal_concurrent_write_protection(bool concurrent_write_protection) { - filesystem_set_concurrent_write_protection(&_circuitpy_usermount, concurrent_write_protection); + filesystem_set_concurrent_write_protection(&_circuitpy_mount, concurrent_write_protection); } -void filesystem_set_concurrent_write_protection(fs_user_mount_t *vfs, bool concurrent_write_protection) { +void filesystem_set_concurrent_write_protection(supervisor_vfs_t *vfs, bool concurrent_write_protection) { if (concurrent_write_protection) { - vfs->blockdev.flags |= MP_BLOCKDEV_FLAG_CONCURRENT_WRITE_PROTECTED; + vfs->common.blockdev.flags |= MP_BLOCKDEV_FLAG_CONCURRENT_WRITE_PROTECTED; } else { - vfs->blockdev.flags &= ~MP_BLOCKDEV_FLAG_CONCURRENT_WRITE_PROTECTED; + vfs->common.blockdev.flags &= ~MP_BLOCKDEV_FLAG_CONCURRENT_WRITE_PROTECTED; } } @@ -302,24 +535,24 @@ bool filesystem_present(void) { return _circuitpy_vfs.len > 0; } -fs_user_mount_t *filesystem_circuitpy(void) { +supervisor_vfs_t *filesystem_circuitpy(void) { if (!filesystem_present()) { return NULL; } - return &_circuitpy_usermount; + return &_circuitpy_mount; } -fs_user_mount_t *filesystem_for_path(const char *path_in, const char **path_under_mount) { +supervisor_vfs_t *filesystem_for_path(const char *path_in, const char **path_under_mount) { mp_vfs_mount_t *vfs = mp_vfs_lookup_path(path_in, path_under_mount); if (vfs == MP_VFS_NONE) { return NULL; } - fs_user_mount_t *fs_mount; + supervisor_vfs_t *fs_mount; *path_under_mount = path_in; if (vfs == MP_VFS_ROOT) { fs_mount = filesystem_circuitpy(); } else { - fs_mount = MP_OBJ_TO_PTR(vfs->obj); + fs_mount = (supervisor_vfs_t *)MP_OBJ_TO_PTR(vfs->obj); // Check if the vfs name is one character long: it must be "/" in that case. // If so don't remove the mount point name. We must use an absolute path // because otherwise the path will be adjusted by os.getcwd() when it's looked up. @@ -331,15 +564,22 @@ fs_user_mount_t *filesystem_for_path(const char *path_in, const char **path_unde return fs_mount; } -bool filesystem_native_fatfs(fs_user_mount_t *fs_mount) { - return fs_mount->base.type == &mp_fat_vfs_type && (fs_mount->blockdev.flags & MP_BLOCKDEV_FLAG_NATIVE) != 0; +static bool filesystem_native_fatfs(supervisor_vfs_t *fs_mount) { + return fs_mount->common.base.type == &mp_fat_vfs_type && + (fs_mount->common.blockdev.flags & MP_BLOCKDEV_FLAG_NATIVE) != 0; } -bool filesystem_lock(fs_user_mount_t *fs_mount) { - if (fs_mount->lock_count == 0 && !blockdev_lock(fs_mount)) { +bool filesystem_lock(supervisor_vfs_t *fs_mount) { + if (fs_mount->common.base.type != &mp_fat_vfs_type) { + // littlefs mounts are never exposed through USB MSC, so there is no + // STA_PROTECT to bypass and no lock_count to track. The blockdev lock + // alone excludes other writers. + return blockdev_lock(fs_mount); + } + if (fs_mount->fat.lock_count == 0 && !blockdev_lock(fs_mount)) { return false; } - fs_mount->lock_count += 1; + fs_mount->fat.lock_count += 1; // CIRCUITPY-CHANGE: while a non-USB-MSC writer (BLE file transfer, web // workflow, storage.remount) holds the filesystem lock, allow the // FatFS f_open(FA_WRITE) path to bypass STA_PROTECT. Without this, the @@ -348,27 +588,544 @@ bool filesystem_lock(fs_user_mount_t *fs_mount) { // so even after the lock is held, f_open returns FR_WRITE_PROTECTED. // USB MSC takes the lock via blockdev_lock() directly, NOT via // filesystem_lock(), so this flag is never set on its behalf. - fs_mount->blockdev.flags |= MP_BLOCKDEV_FLAG_IGNORE_WRITE_PROTECTION; + fs_mount->common.blockdev.flags |= MP_BLOCKDEV_FLAG_IGNORE_WRITE_PROTECTION; return true; } -void filesystem_unlock(fs_user_mount_t *fs_mount) { - fs_mount->lock_count -= 1; - if (fs_mount->lock_count == 0) { +void filesystem_unlock(supervisor_vfs_t *fs_mount) { + if (fs_mount->common.base.type != &mp_fat_vfs_type) { + blockdev_unlock(fs_mount); + return; + } + fs_mount->fat.lock_count -= 1; + if (fs_mount->fat.lock_count == 0) { // CIRCUITPY-CHANGE: clear the bypass when releasing the lock. - fs_mount->blockdev.flags &= ~MP_BLOCKDEV_FLAG_IGNORE_WRITE_PROTECTION; + fs_mount->common.blockdev.flags &= ~MP_BLOCKDEV_FLAG_IGNORE_WRITE_PROTECTION; blockdev_unlock(fs_mount); } } -bool blockdev_lock(fs_user_mount_t *fs_mount) { - if ((fs_mount->blockdev.flags & MP_BLOCKDEV_FLAG_LOCKED) != 0) { +bool blockdev_lock(supervisor_vfs_t *fs_mount) { + if ((fs_mount->common.blockdev.flags & MP_BLOCKDEV_FLAG_LOCKED) != 0) { return false; } - fs_mount->blockdev.flags |= MP_BLOCKDEV_FLAG_LOCKED; + fs_mount->common.blockdev.flags |= MP_BLOCKDEV_FLAG_LOCKED; return true; } -void blockdev_unlock(fs_user_mount_t *fs_mount) { - fs_mount->blockdev.flags &= ~MP_BLOCKDEV_FLAG_LOCKED; +void blockdev_unlock(supervisor_vfs_t *fs_mount) { + fs_mount->common.blockdev.flags &= ~MP_BLOCKDEV_FLAG_LOCKED; +} + +// Supervisor-level file, directory and filesystem helpers shared between FAT +// and littlefs mounts. All of them take or return supervisor_fs_err_t; paths +// are relative to the given mount. See supervisor/filesystem.h for details. + +bool supervisor_vfs_supported(supervisor_vfs_t *vfs) { + #if CIRCUITPY_FILESYSTEM_LITTLEFS + if (is_lfs(vfs)) { + return true; + } + #endif + return filesystem_native_fatfs(vfs); +} + +supervisor_fs_err_t supervisor_vfs_open_file(supervisor_vfs_t *vfs, const char *path, uint32_t flags, + uint64_t mtime_ns, supervisor_vfs_file_t *file) { + if (vfs == NULL) { + return SUPERVISOR_FS_NO_FILE; + } + memset(file, 0, sizeof(*file)); + file->vfs = vfs; + #if CIRCUITPY_FILESYSTEM_LITTLEFS + if (is_lfs(vfs)) { + struct lfs2_file_config *cfg = &file->file.lfs.cfg; + memset(cfg, 0, sizeof(*cfg)); + cfg->buffer = file->file.lfs.buffer; + // Modification time attribute, committed when a modified file is + // closed. littlefs writes file attributes at close. + if (mtime_ns == 0) { + // No timestamp was requested. Use the same RTC-based time that FAT + // stamps files with via get_fattime(). + mtime_ns = get_fattime_ns(); + } + lfs_mtime_store(file->file.lfs.mtime, mtime_ns); + struct lfs2_attr *attr = &file->file.lfs.attr; + attr->type = LFS_ATTR_MTIME; + attr->buffer = file->file.lfs.mtime; + attr->size = sizeof(file->file.lfs.mtime); + cfg->attrs = attr; + cfg->attr_count = 1; + int lfs_flags; + if ((flags & SUPERVISOR_FS_OPEN_WRITE) != 0) { + lfs_flags = LFS2_O_WRONLY; + } else { + lfs_flags = LFS2_O_RDONLY; + } + if ((flags & SUPERVISOR_FS_OPEN_CREATE) != 0) { + lfs_flags |= LFS2_O_CREAT; + } + if ((flags & SUPERVISOR_FS_OPEN_TRUNCATE) != 0) { + lfs_flags |= LFS2_O_TRUNC; + } + int err = lfs2_file_opencfg(&vfs->lfs2.lfs, &file->file.lfs.lfs2, path, lfs_flags, cfg); + if (err < 0) { + return lfs_error(err); + } + file->open = true; + file->lfs = true; + return SUPERVISOR_FS_OK; + } + #endif + if (!filesystem_native_fatfs(vfs)) { + return SUPERVISOR_FS_IO; + } + DWORD fattime = mtime_ns != 0 ? fattime_from_ns(mtime_ns) : 0; + if (fattime != 0) { + // FatFS stamps files when they're closed, so remember the timestamp + // and apply it again in supervisor_vfs_close_file(). + override_fattime(fattime); + } + BYTE fa = 0; + if ((flags & SUPERVISOR_FS_OPEN_READ) != 0) { + fa |= FA_READ; + } + if ((flags & SUPERVISOR_FS_OPEN_WRITE) != 0) { + fa |= FA_WRITE; + } + if ((flags & SUPERVISOR_FS_OPEN_TRUNCATE) != 0) { + fa |= FA_CREATE_ALWAYS; + } else if ((flags & SUPERVISOR_FS_OPEN_CREATE) != 0) { + fa |= FA_OPEN_ALWAYS; + } + FRESULT res = f_open(&vfs->fat.fatfs, &file->file.fat, path, fa); + if (fattime != 0) { + override_fattime(0); + } + if (res != FR_OK) { + return fat_error(res); + } + file->open = true; + file->lfs = false; + file->fattime = fattime; + return SUPERVISOR_FS_OK; +} + +supervisor_fs_err_t supervisor_vfs_close_file(supervisor_vfs_file_t *file) { + if (!file->open) { + return SUPERVISOR_FS_OK; + } + file->open = false; + #if CIRCUITPY_FILESYSTEM_LITTLEFS + if (file->lfs) { + // The mtime attribute filled in at open is written out here for a + // modified file. + int err = lfs2_file_close(&file->vfs->lfs2.lfs, &file->file.lfs.lfs2); + return err < 0 ? lfs_error(err) : SUPERVISOR_FS_OK; + } + #endif + if (file->fattime != 0) { + override_fattime(file->fattime); + } + FRESULT res = f_close(&file->file.fat); + if (file->fattime != 0) { + override_fattime(0); + } + return fat_error(res); +} + +supervisor_fs_err_t supervisor_vfs_read_file(supervisor_vfs_file_t *file, void *buf, size_t len, size_t *bytes_read) { + *bytes_read = 0; + if (!file->open) { + return SUPERVISOR_FS_IO; + } + #if CIRCUITPY_FILESYSTEM_LITTLEFS + if (file->lfs) { + lfs2_ssize_t read = lfs2_file_read(&file->vfs->lfs2.lfs, &file->file.lfs.lfs2, buf, len); + if (read < 0) { + return lfs_error(read); + } + *bytes_read = read; + return SUPERVISOR_FS_OK; + } + #endif + UINT read; + FRESULT res = f_read(&file->file.fat, buf, len, &read); + *bytes_read = read; + return fat_error(res); +} + +supervisor_fs_err_t supervisor_vfs_write_file(supervisor_vfs_file_t *file, const void *buf, size_t len, size_t *bytes_written) { + *bytes_written = 0; + if (!file->open) { + return SUPERVISOR_FS_IO; + } + #if CIRCUITPY_FILESYSTEM_LITTLEFS + if (file->lfs) { + lfs2_ssize_t written = lfs2_file_write(&file->vfs->lfs2.lfs, &file->file.lfs.lfs2, buf, len); + if (written < 0) { + return lfs_error(written); + } + *bytes_written = written; + return SUPERVISOR_FS_OK; + } + #endif + UINT written; + FRESULT res = f_write(&file->file.fat, buf, len, &written); + *bytes_written = written; + return fat_error(res); +} + +supervisor_fs_err_t supervisor_vfs_seek_file(supervisor_vfs_file_t *file, size_t offset) { + if (!file->open) { + return SUPERVISOR_FS_IO; + } + #if CIRCUITPY_FILESYSTEM_LITTLEFS + if (file->lfs) { + lfs2_soff_t result = lfs2_file_seek(&file->vfs->lfs2.lfs, &file->file.lfs.lfs2, offset, LFS2_SEEK_SET); + return result < 0 ? lfs_error(result) : SUPERVISOR_FS_OK; + } + #endif + return fat_error(f_lseek(&file->file.fat, offset)); +} + +size_t supervisor_vfs_tell_file(supervisor_vfs_file_t *file) { + if (!file->open) { + return 0; + } + #if CIRCUITPY_FILESYSTEM_LITTLEFS + if (file->lfs) { + return lfs2_file_tell(&file->vfs->lfs2.lfs, &file->file.lfs.lfs2); + } + #endif + return f_tell(&file->file.fat); +} + +size_t supervisor_vfs_file_size(supervisor_vfs_file_t *file) { + if (!file->open) { + return 0; + } + #if CIRCUITPY_FILESYSTEM_LITTLEFS + if (file->lfs) { + return lfs2_file_size(&file->vfs->lfs2.lfs, &file->file.lfs.lfs2); + } + #endif + return f_size(&file->file.fat); +} + +supervisor_fs_err_t supervisor_vfs_truncate_file(supervisor_vfs_file_t *file) { + if (!file->open) { + return SUPERVISOR_FS_IO; + } + #if CIRCUITPY_FILESYSTEM_LITTLEFS + if (file->lfs) { + int err = lfs2_file_truncate(&file->vfs->lfs2.lfs, &file->file.lfs.lfs2, + lfs2_file_tell(&file->vfs->lfs2.lfs, &file->file.lfs.lfs2)); + return err < 0 ? lfs_error(err) : SUPERVISOR_FS_OK; + } + #endif + return fat_error(f_truncate(&file->file.fat)); +} + +supervisor_fs_err_t supervisor_vfs_stat(supervisor_vfs_t *vfs, const char *path, bool *is_dir, size_t *size, uint64_t *mtime_ns) { + if (is_dir != NULL) { + *is_dir = false; + } + if (size != NULL) { + *size = 0; + } + if (mtime_ns != NULL) { + *mtime_ns = 0; + } + #if CIRCUITPY_FILESYSTEM_LITTLEFS + if (is_lfs(vfs)) { + struct lfs2_info info; + int err = lfs2_stat(&vfs->lfs2.lfs, path, &info); + if (err < 0) { + return lfs_error(err); + } + bool directory = info.type == LFS2_TYPE_DIR; + if (is_dir != NULL) { + *is_dir = directory; + } + if (size != NULL && !directory) { + *size = info.size; + } + if (mtime_ns != NULL) { + *mtime_ns = lfs_get_path_mtime(&vfs->lfs2.lfs, path); + } + return SUPERVISOR_FS_OK; + } + #endif + FILINFO file_info; + FRESULT res = f_stat(&vfs->fat.fatfs, path, &file_info); + if (res != FR_OK) { + return fat_error(res); + } + bool directory = (file_info.fattrib & AM_DIR) != 0; + if (is_dir != NULL) { + *is_dir = directory; + } + if (size != NULL && !directory) { + *size = file_info.fsize; + } + if (mtime_ns != NULL) { + *mtime_ns = (uint64_t)timeutils_mktime_1970(1980 + (file_info.fdate >> 9), + (file_info.fdate >> 5) & 0xf, + file_info.fdate & 0x1f, + file_info.ftime >> 11, + (file_info.ftime >> 5) & 0x3f, + (file_info.ftime & 0x1f) * 2) * 1000000000ULL; + } + return SUPERVISOR_FS_OK; +} + +supervisor_fs_err_t supervisor_vfs_mkdir(supervisor_vfs_t *vfs, const char *path, uint64_t mtime_ns) { + #if CIRCUITPY_FILESYSTEM_LITTLEFS + if (is_lfs(vfs)) { + int err = lfs2_mkdir(&vfs->lfs2.lfs, path); + return err < 0 ? lfs_error(err) : SUPERVISOR_FS_OK; + } + #endif + if (!filesystem_native_fatfs(vfs)) { + return SUPERVISOR_FS_IO; + } + DWORD fattime = mtime_ns != 0 ? fattime_from_ns(mtime_ns) : 0; + if (fattime != 0) { + override_fattime(fattime); + } + FRESULT res = f_mkdir(&vfs->fat.fatfs, path); + if (fattime != 0) { + override_fattime(0); + } + return fat_error(res); +} + +supervisor_fs_err_t supervisor_vfs_rename(supervisor_vfs_t *vfs, const char *old_path, const char *new_path) { + #if CIRCUITPY_FILESYSTEM_LITTLEFS + if (is_lfs(vfs)) { + int err = lfs2_rename(&vfs->lfs2.lfs, old_path, new_path); + return err < 0 ? lfs_error(err) : SUPERVISOR_FS_OK; + } + #endif + if (!filesystem_native_fatfs(vfs)) { + return SUPERVISOR_FS_IO; + } + return fat_error(f_rename(&vfs->fat.fatfs, old_path, new_path)); +} + +supervisor_fs_err_t supervisor_vfs_unlink(supervisor_vfs_t *vfs, const char *path) { + #if CIRCUITPY_FILESYSTEM_LITTLEFS + if (is_lfs(vfs)) { + int err = lfs2_remove(&vfs->lfs2.lfs, path); + return err < 0 ? lfs_error(err) : SUPERVISOR_FS_OK; + } + #endif + if (!filesystem_native_fatfs(vfs)) { + return SUPERVISOR_FS_IO; + } + return fat_error(f_unlink(&vfs->fat.fatfs, path)); +} + +supervisor_fs_err_t supervisor_vfs_opendir(supervisor_vfs_t *vfs, const char *path, supervisor_vfs_dir_t *dir) { + memset(dir, 0, sizeof(*dir)); + dir->vfs = vfs; + #if CIRCUITPY_FILESYSTEM_LITTLEFS + if (is_lfs(vfs)) { + int err = lfs2_dir_open(&vfs->lfs2.lfs, &dir->dir.lfs2, path); + if (err < 0) { + return lfs_error(err); + } + dir->open = true; + dir->lfs = true; + // Store the path (without a trailing slash) for per-entry mtime + // lookups. If it doesn't fit, mtimes will simply read as 0. + size_t len = strlen(path); + while (len > 0 && path[len - 1] == '/') { + len--; + } + if (len > sizeof(dir->path) - 1) { + len = sizeof(dir->path) - 1; + } + memcpy(dir->path, path, len); + dir->path[len] = '\0'; + return SUPERVISOR_FS_OK; + } + #endif + if (!filesystem_native_fatfs(vfs)) { + return SUPERVISOR_FS_IO; + } + FRESULT res = f_opendir(&vfs->fat.fatfs, &dir->dir.fat, path); + if (res != FR_OK) { + return fat_error(res); + } + dir->open = true; + dir->lfs = false; + return SUPERVISOR_FS_OK; +} + +supervisor_fs_err_t supervisor_vfs_readdir(supervisor_vfs_dir_t *dir, char *name, size_t name_len, bool *is_dir, size_t *size, uint64_t *mtime_ns) { + if (is_dir != NULL) { + *is_dir = false; + } + if (size != NULL) { + *size = 0; + } + if (mtime_ns != NULL) { + *mtime_ns = 0; + } + if (!dir->open) { + return SUPERVISOR_FS_IO; + } + #if CIRCUITPY_FILESYSTEM_LITTLEFS + if (dir->lfs) { + struct lfs2_info info; + int res; + do { + res = lfs2_dir_read(&dir->vfs->lfs2.lfs, &dir->dir.lfs2, &info); + if (res < 0) { + return lfs_error(res); + } + if (res == 0) { + // End of the directory. + name[0] = '\0'; + return SUPERVISOR_FS_OK; + } + // littlefs lists the "." and ".." entries; FatFS doesn't, so skip + // them for parity. + } while (info.name[0] == '.' && + (info.name[1] == '\0' || (info.name[1] == '.' && info.name[2] == '\0'))); + size_t name_chars = strlen(info.name); + if (name_chars > name_len - 1) { + name_chars = name_len - 1; + } + memcpy(name, info.name, name_chars); + name[name_chars] = '\0'; + bool directory = info.type == LFS2_TYPE_DIR; + if (is_dir != NULL) { + *is_dir = directory; + } + if (size != NULL && !directory) { + *size = info.size; + } + if (mtime_ns != NULL && !directory) { + // littlefs keeps modification times as per-path attributes, so + // build the entry's full path. The stored directory path is + // temporarily extended to hold the entry name. + size_t len = strlen(dir->path); + if (len + 1 + name_chars + 1 <= sizeof(dir->path)) { + dir->path[len] = '/'; + memcpy(dir->path + len + 1, info.name, name_chars); + dir->path[len + 1 + name_chars] = '\0'; + *mtime_ns = lfs_get_path_mtime(&dir->vfs->lfs2.lfs, dir->path); + dir->path[len] = '\0'; + } + } + return SUPERVISOR_FS_OK; + } + #endif + FILINFO file_info; + FRESULT res = f_readdir(&dir->dir.fat, &file_info); + if (res != FR_OK) { + return fat_error(res); + } + if (file_info.fname[0] == '\0') { + // End of the directory. + name[0] = '\0'; + return SUPERVISOR_FS_OK; + } + size_t name_chars = strlen(file_info.fname); + if (name_chars > name_len - 1) { + name_chars = name_len - 1; + } + memcpy(name, file_info.fname, name_chars); + name[name_chars] = '\0'; + bool directory = (file_info.fattrib & AM_DIR) != 0; + if (is_dir != NULL) { + *is_dir = directory; + } + if (size != NULL && !directory) { + *size = file_info.fsize; + } + if (mtime_ns != NULL) { + *mtime_ns = (uint64_t)timeutils_mktime_1970(1980 + (file_info.fdate >> 9), + (file_info.fdate >> 5) & 0xf, + file_info.fdate & 0x1f, + file_info.ftime >> 11, + (file_info.ftime >> 5) & 0x3f, + (file_info.ftime & 0x1f) * 2) * 1000000000ULL; + } + return SUPERVISOR_FS_OK; +} + +supervisor_fs_err_t supervisor_vfs_rewinddir(supervisor_vfs_dir_t *dir) { + if (!dir->open) { + return SUPERVISOR_FS_IO; + } + #if CIRCUITPY_FILESYSTEM_LITTLEFS + if (dir->lfs) { + int err = lfs2_dir_rewind(&dir->vfs->lfs2.lfs, &dir->dir.lfs2); + return err < 0 ? lfs_error(err) : SUPERVISOR_FS_OK; + } + #endif + f_readdir(&dir->dir.fat, NULL); + return SUPERVISOR_FS_OK; +} + +supervisor_fs_err_t supervisor_vfs_closedir(supervisor_vfs_dir_t *dir) { + if (!dir->open) { + return SUPERVISOR_FS_OK; + } + dir->open = false; + #if CIRCUITPY_FILESYSTEM_LITTLEFS + if (dir->lfs) { + int err = lfs2_dir_close(&dir->vfs->lfs2.lfs, &dir->dir.lfs2); + return err < 0 ? lfs_error(err) : SUPERVISOR_FS_OK; + } + #endif + return fat_error(f_closedir(&dir->dir.fat)); +} + +supervisor_fs_err_t supervisor_vfs_statfs(supervisor_vfs_t *vfs, size_t *block_size, size_t *total_blocks, size_t *free_blocks) { + #if CIRCUITPY_FILESYSTEM_LITTLEFS + if (is_lfs(vfs)) { + struct lfs2_config *config = &vfs->lfs2.config; + if (block_size != NULL) { + *block_size = config->block_size; + } + if (total_blocks != NULL) { + *total_blocks = config->block_count; + } + if (free_blocks != NULL) { + lfs2_ssize_t used = lfs2_fs_size(&vfs->lfs2.lfs); + *free_blocks = used < 0 ? 0 : config->block_count - used; + } + return SUPERVISOR_FS_OK; + } + #endif + if (!filesystem_native_fatfs(vfs)) { + return SUPERVISOR_FS_IO; + } + FATFS *fatfs = &vfs->fat.fatfs; + DWORD free_clusters = 0; + FRESULT res = f_getfree(fatfs, &free_clusters); + if (res != FR_OK) { + return fat_error(res); + } + size_t ssize; + #if FF_MAX_SS != FF_MIN_SS + ssize = fatfs->ssize; + #else + ssize = FF_MIN_SS; + #endif + if (block_size != NULL) { + *block_size = fatfs->csize * ssize; + } + if (total_blocks != NULL) { + *total_blocks = fatfs->n_fatent - 2; + } + if (free_blocks != NULL) { + *free_blocks = free_clusters; + } + return SUPERVISOR_FS_OK; } diff --git a/supervisor/shared/flash.c b/supervisor/shared/flash.c index ccb60efe321..a75adc4091f 100644 --- a/supervisor/shared/flash.c +++ b/supervisor/shared/flash.c @@ -150,6 +150,13 @@ static mp_uint_t flash_read_blocks(mp_obj_t self_in, uint8_t *dest, uint32_t blo } static volatile bool filesystem_dirty = false; +void supervisor_flash_mark_dirty(void) { + if (!filesystem_dirty) { + // Turn on ticks so that we can flush after a period of time elapses. + supervisor_enable_tick(); + filesystem_dirty = true; + } +} static mp_uint_t flash_write_blocks(mp_obj_t self_in, const uint8_t *src, uint32_t block_num, uint32_t num_blocks) { if (block_num == 0) { @@ -159,11 +166,7 @@ static mp_uint_t flash_write_blocks(mp_obj_t self_in, const uint8_t *src, uint32 // can't write MBR, but pretend we did return 0; } else { - if (!filesystem_dirty) { - // Turn on ticks so that we can flush after a period of time elapses. - supervisor_enable_tick(); - filesystem_dirty = true; - } + supervisor_flash_mark_dirty(); block_num -= PART1_START_BLOCK; #if CIRCUITPY_SAVES_PARTITION_SIZE > 0 mp_vfs_blockdev_t *self = (mp_vfs_blockdev_t *)self_in; diff --git a/supervisor/shared/settings.c b/supervisor/shared/settings.c index 1414195411e..fe13a4e6416 100644 --- a/supervisor/shared/settings.c +++ b/supervisor/shared/settings.c @@ -26,9 +26,9 @@ #include "extmod/vfs_fat.h" #if CIRCUITPY_SETTINGS_TOML +#if defined(UNIX) typedef FIL file_arg; static bool open_file(const char *name, file_arg *file_handle) { - #if defined(UNIX) nlr_buf_t nlr; if (nlr_push(&nlr) == 0) { mp_obj_t file_obj = mp_call_function_2( @@ -41,15 +41,6 @@ static bool open_file(const char *name, file_arg *file_handle) { } else { return false; } - #else - fs_user_mount_t *fs_mount = filesystem_circuitpy(); - if (fs_mount == NULL) { - return false; - } - FATFS *fatfs = &fs_mount->fatfs; - FRESULT result = f_open(fatfs, file_handle, name, FA_READ); - return result == FR_OK; - #endif } static void close_file(file_arg *file_handle) { @@ -72,6 +63,46 @@ static uint8_t get_next_byte(FIL *file_handle) { static void seek_eof(file_arg *file_handle) { f_lseek(file_handle, f_size(file_handle)); } +#else +// Outside of the unix port, settings.toml is read through the supervisor +// filesystem API so it works on both FAT and littlefs mounts. +typedef supervisor_vfs_file_t file_arg; + +static bool open_file(const char *name, file_arg *file_handle) { + supervisor_vfs_t *fs_mount = filesystem_circuitpy(); + if (fs_mount == NULL) { + return false; + } + return supervisor_vfs_open_file(fs_mount, name, SUPERVISOR_FS_OPEN_READ, 0, file_handle) == SUPERVISOR_FS_OK; +} + +static void close_file(file_arg *file_handle) { + supervisor_vfs_close_file(file_handle); +} + +// Reads mark the end of the file by closing the handle. +static bool is_eof(file_arg *file_handle) { + return !file_handle->open; +} + +// Return 0 if there is no next character (EOF). +static uint8_t get_next_byte(file_arg *file_handle) { + uint8_t character = 0; + size_t quantity_read; + // If there's an error or quantity_read is 0, character will remain 0. + supervisor_fs_err_t err = supervisor_vfs_read_file(file_handle, &character, 1, &quantity_read); + // If we hit the end of the file (or an error), close the handle so that + // is_eof() reports the file is done. + if (err != SUPERVISOR_FS_OK || quantity_read == 0) { + close_file(file_handle); + } + return character; +} + +static void seek_eof(file_arg *file_handle) { + supervisor_vfs_seek_file(file_handle, supervisor_vfs_file_size(file_handle)); +} +#endif // For a fixed buffer, record the required size rather than throwing static void vstr_add_byte_nonstd(vstr_t *vstr, byte b) { diff --git a/supervisor/shared/usb/usb_msc_flash.c b/supervisor/shared/usb/usb_msc_flash.c index 5fcd2c8d62b..face2711b31 100644 --- a/supervisor/shared/usb/usb_msc_flash.c +++ b/supervisor/shared/usb/usb_msc_flash.c @@ -137,7 +137,8 @@ size_t usb_msc_add_descriptor(uint8_t *descriptor_buf, descriptor_counts_t *desc // We hardcode LUN -> mount mapping so that it doesn't changes with saves and // SD card appearing and disappearing. static fs_user_mount_t *get_vfs(int lun) { - fs_user_mount_t *root = filesystem_circuitpy(); + supervisor_vfs_t *root_vfs = filesystem_circuitpy(); + fs_user_mount_t *root = root_vfs == NULL ? NULL : &root_vfs->fat; if (lun == 0) { return root; } @@ -146,7 +147,8 @@ static fs_user_mount_t *get_vfs(int lun) { #ifdef SAVES_LUN if (lun == SAVES_LUN) { const char *path_under_mount; - fs_user_mount_t *saves = filesystem_for_path("/saves", &path_under_mount); + supervisor_vfs_t *saves_mount = filesystem_for_path("/saves", &path_under_mount); + fs_user_mount_t *saves = saves_mount == NULL ? NULL : &saves_mount->fat; if (saves != root && (saves->blockdev.flags & MP_BLOCKDEV_FLAG_NATIVE) != 0 && !gc_ptr_on_heap(saves)) { return saves; @@ -156,7 +158,8 @@ static fs_user_mount_t *get_vfs(int lun) { #ifdef SDCARD_LUN if (lun == SDCARD_LUN) { const char *path_under_mount; - fs_user_mount_t *sdcard = filesystem_for_path("/sd", &path_under_mount); + supervisor_vfs_t *sdcard_mount = filesystem_for_path("/sd", &path_under_mount); + fs_user_mount_t *sdcard = sdcard_mount == NULL ? NULL : &sdcard_mount->fat; // If sdcard ("/sd") is on the root filesystem, nothing has been mounted there, so don't // return it as a separate filesystem. // If the SD card was automounted at startup, then it persists across VMs and its fs_user_mount_t is @@ -177,8 +180,10 @@ static fs_user_mount_t *get_vfs(int lun) { #ifdef EMMC_LUN if (lun == EMMC_LUN) { const char *path_under_mount; + fs_user_mount_t *emmc; - fs_user_mount_t *emmc = filesystem_for_path(CIRCUITPY_EMMC_MOUNT_PATH, &path_under_mount); + supervisor_vfs_t *emmc_mount = filesystem_for_path(CIRCUITPY_EMMC_MOUNT_PATH, &path_under_mount); + emmc = emmc_mount == NULL ? NULL : &emmc_mount->fat; // Unlike the SD card there is no heap-mount case to allow: the eMMC's // drive exists only when the supervisor mounted it, and // that mount is static. A user mount made by code.py stays a Python @@ -214,7 +219,7 @@ void usb_msc_umount(void) { if (vfs == NULL) { continue; } - blockdev_unlock(vfs); + blockdev_unlock((supervisor_vfs_t *)vfs); locked[i] = false; } } @@ -296,7 +301,7 @@ bool tud_msc_is_writable_cb(uint8_t lun) { return false; } // Lock the blockdev once we say we're writable. - if (!locked[lun] && !blockdev_lock(vfs)) { + if (!locked[lun] && !blockdev_lock((supervisor_vfs_t *)vfs)) { return false; } locked[lun] = true; @@ -465,7 +470,7 @@ bool tud_msc_start_stop_cb(uint8_t lun, uint8_t power_condition, bool start, boo if (disk_ioctl(current_mount, CTRL_SYNC, NULL) != RES_OK) { return false; } else { - blockdev_unlock(current_mount); + blockdev_unlock((supervisor_vfs_t *)current_mount); ejected[lun] = true; locked[lun] = false; } diff --git a/supervisor/shared/web_workflow/web_workflow.c b/supervisor/shared/web_workflow/web_workflow.c index be352d6c0b3..829a6c04221 100644 --- a/supervisor/shared/web_workflow/web_workflow.c +++ b/supervisor/shared/web_workflow/web_workflow.c @@ -19,8 +19,6 @@ #include "py/mpstate.h" #include "shared-bindings/wifi/Radio.h" -#include "shared/timeutils/timeutils.h" -#include "supervisor/fatfs.h" #include "supervisor/filesystem.h" #include "supervisor/port.h" #include "supervisor/shared/reload.h" @@ -729,11 +727,13 @@ static void _reply_redirect(socketpool_socket_obj_t *socket, _request *request, } #endif -static void _reply_directory_json(socketpool_socket_obj_t *socket, _request *request, fs_user_mount_t *fs_mount, FF_DIR *dir, const char *request_path, const char *path) { - FILINFO file_info; - char *fn = file_info.fname; - FRESULT res = f_readdir(dir, &file_info); - if (res != FR_OK) { +static void _reply_directory_json(socketpool_socket_obj_t *socket, _request *request, supervisor_vfs_t *fs_mount, supervisor_vfs_dir_t *dir, const char *request_path, const char *path) { + char fn[FF_MAX_LFN + 1]; + bool is_dir = false; + size_t file_size = 0; + uint64_t mtime_ns = 0; + supervisor_fs_err_t res = supervisor_vfs_readdir(dir, fn, sizeof(fn), &is_dir, &file_size, &mtime_ns); + if (res != SUPERVISOR_FS_OK) { _reply_missing(socket, request); return; } @@ -743,18 +743,12 @@ static void _reply_directory_json(socketpool_socket_obj_t *socket, _request *req _send_str(socket, "\r\n"); mp_print_t _socket_print = {socket, _print_chunk}; - // Send mount info. - DWORD free_clusters = 0; - FATFS *fatfs = &fs_mount->fatfs; - f_getfree(fatfs, &free_clusters); - size_t ssize; - #if FF_MAX_SS != FF_MIN_SS - ssize = fatfs->ssize; - #else - ssize = FF_MIN_SS; - #endif - uint32_t cluster_size = fatfs->csize * ssize; - uint32_t total_clusters = fatfs->n_fatent - 2; + // Send mount info. Counts are in filesystem blocks: clusters on FAT, + // blocks on littlefs. + size_t block_size = 0; + size_t total_blocks = 0; + size_t free_blocks = 0; + supervisor_vfs_statfs(fs_mount, &block_size, &total_blocks, &free_blocks); const char *writable = "false"; // Test to see if we can grab the write lock. USB will grab the underlying @@ -768,20 +762,20 @@ static void _reply_directory_json(socketpool_socket_obj_t *socket, _request *req "{\"free\": %u, " "\"total\": %u, " "\"block_size\": %u, " - "\"writable\": %s, ", free_clusters, total_clusters, cluster_size, writable); + "\"writable\": %s, ", free_blocks, total_blocks, block_size, writable); // Send file list _send_chunk(socket, "\"files\": ["); bool first = true; - while (res == FR_OK && fn[0] != 0) { + while (res == SUPERVISOR_FS_OK && fn[0] != 0) { if (!first) { _send_chunk(socket, ","); } _send_chunks(socket, - "{\"name\": \"", file_info.fname, "\",", + "{\"name\": \"", fn, "\",", "\"directory\": ", NULL); - if ((file_info.fattrib & AM_DIR) != 0) { + if (is_dir) { _send_chunk(socket, "true"); } else { _send_chunk(socket, "false"); @@ -790,31 +784,31 @@ static void _reply_directory_json(socketpool_socket_obj_t *socket, _request *req // LittleFS. _send_chunk(socket, ", "); - uint32_t truncated_time = timeutils_mktime(1980 + (file_info.fdate >> 9), - (file_info.fdate >> 5) & 0xf, - file_info.fdate & 0x1f, - file_info.ftime >> 11, - (file_info.ftime >> 5) & 0x3f, - (file_info.ftime & 0x1f) * 2); - - // Manually append zeros to make the time nanoseconds. Support for printing 64 bit numbers - // varies across chipsets. - mp_printf(&_socket_print, "\"modified_ns\": %lu000000000, ", truncated_time); - size_t file_size = 0; - if ((file_info.fattrib & AM_DIR) == 0) { - file_size = file_info.fsize; + // mpprint doesn't support 64-bit formats on all targets, so print the + // seconds and zero-padded remainder of the nanosecond value separately. + mp_uint_t mtime_secs = (mp_uint_t)(mtime_ns / 1000000000ULL); + mp_uint_t mtime_frac = (mp_uint_t)(mtime_ns % 1000000000ULL); + if (mtime_secs == 0) { + // Avoid a leading zero, which isn't valid JSON. + mp_printf(&_socket_print, "\"modified_ns\": %u, ", mtime_frac); + } else { + mp_printf(&_socket_print, "\"modified_ns\": %u%09u, ", mtime_secs, mtime_frac); + } + if (!is_dir) { + mp_printf(&_socket_print, "\"file_size\": %d }", file_size); + } else { + _send_chunk(socket, "\"file_size\": 0 }"); } - mp_printf(&_socket_print, "\"file_size\": %d }", file_size); first = false; - res = f_readdir(dir, &file_info); + res = supervisor_vfs_readdir(dir, fn, sizeof(fn), &is_dir, &file_size, &mtime_ns); } _send_chunk(socket, "]}"); _send_chunk(socket, ""); } -static void _reply_with_file(socketpool_socket_obj_t *socket, _request *request, const char *filename, FIL *active_file) { - uint32_t total_length = f_size(active_file); +static void _reply_with_file(socketpool_socket_obj_t *socket, _request *request, const char *filename, supervisor_vfs_file_t *active_file) { + uint32_t total_length = supervisor_vfs_file_size(active_file); _send_str(socket, "HTTP/1.1 200 OK\r\n"); mp_print_t _socket_print = {socket, _print_raw}; @@ -839,7 +833,7 @@ static void _reply_with_file(socketpool_socket_obj_t *socket, _request *request, while (total_read < total_length) { uint8_t data_buffer[64]; size_t quantity_read; - f_read(active_file, data_buffer, 64, &quantity_read); + supervisor_vfs_read_file(active_file, data_buffer, 64, &quantity_read); total_read += quantity_read; // When getting near the end of the file, disable Nagle's combining algorithm so that // data is sent immediately. @@ -965,23 +959,17 @@ static void _reply_with_diskinfo_json(socketpool_socket_obj_t *socket, _request if (i > 0) { _send_chunk(socket, ","); } - fs_user_mount_t *fs = MP_OBJ_TO_PTR(vfs->obj); - // Skip non-fat and non-native block file systems. - if (fs->base.type != &mp_fat_vfs_type || (fs->blockdev.flags & MP_BLOCKDEV_FLAG_NATIVE) == 0) { + supervisor_vfs_t *fs = MP_OBJ_TO_PTR(vfs->obj); + // Skip filesystems the supervisor can't work with (non-fat and + // non-native block file systems). + if (!supervisor_vfs_supported(fs)) { vfs = vfs->next; continue; } - DWORD free_clusters = 0; - FATFS *fatfs = &fs->fatfs; - f_getfree(fatfs, &free_clusters); - size_t ssize; - #if FF_MAX_SS != FF_MIN_SS - ssize = fatfs->ssize; - #else - ssize = FF_MIN_SS; - #endif - size_t block_size = fatfs->csize * ssize; - size_t total_size = fatfs->n_fatent - 2; + size_t block_size = 0; + size_t total_size = 0; + size_t free_blocks = 0; + supervisor_vfs_statfs(fs, &block_size, &total_size, &free_blocks); const char *writable = "false"; if (filesystem_lock(fs)) { @@ -993,7 +981,7 @@ static void _reply_with_diskinfo_json(socketpool_socket_obj_t *socket, _request "\"free\": %u, " "\"total\": %u, " "\"block_size\": %u, " - "\"writable\": %s}", vfs->str, free_clusters, total_size, block_size, writable); + "\"writable\": %s}", vfs->str, free_blocks, total_size, block_size, writable); i++; vfs = vfs->next; } @@ -1003,21 +991,6 @@ static void _reply_with_diskinfo_json(socketpool_socket_obj_t *socket, _request _send_chunk(socket, ""); } - -// FATFS has a two second timestamp resolution but the BLE API allows for nanosecond resolution. -// This function truncates the time the time to a resolution storable by FATFS and fills in the -// FATFS encoded version into fattime. -static uint64_t truncate_time(uint64_t input_time, DWORD *fattime) { - timeutils_struct_time_t tm; - uint64_t seconds_since_epoch = timeutils_seconds_since_epoch_from_nanoseconds_since_1970(input_time); - timeutils_seconds_since_epoch_to_struct_time(seconds_since_epoch, &tm); - uint64_t truncated_time = timeutils_nanoseconds_since_epoch_to_nanoseconds_since_1970((seconds_since_epoch / 2) * 2 * 1000000000); - - *fattime = ((tm.tm_year - 1980) << 25) | (tm.tm_mon << 21) | (tm.tm_mday << 16) | - (tm.tm_hour << 11) | (tm.tm_min << 5) | (tm.tm_sec >> 1); - return truncated_time; -} - static void _discard_incoming(socketpool_socket_obj_t *socket, size_t amount) { size_t discarded = 0; while (discarded < amount) { @@ -1034,52 +1007,46 @@ static void _discard_incoming(socketpool_socket_obj_t *socket, size_t amount) { } } -static void _write_file_and_reply(socketpool_socket_obj_t *socket, _request *request, fs_user_mount_t *fs_mount, const TCHAR *path) { - FIL active_file; +static void _write_file_and_reply(socketpool_socket_obj_t *socket, _request *request, supervisor_vfs_t *fs_mount, const TCHAR *path) { + supervisor_vfs_file_t active_file; if (!filesystem_lock(fs_mount)) { _discard_incoming(socket, request->content_length); _reply_conflict(socket, request); return; } - if (request->timestamp_ms > 0) { - DWORD fattime; - truncate_time(request->timestamp_ms * 1000000, &fattime); - override_fattime(fattime); - } - FATFS *fs = &fs_mount->fatfs; - FRESULT result = f_open(fs, &active_file, path, FA_WRITE); + uint64_t mtime_ns = request->timestamp_ms > 0 ? (uint64_t)request->timestamp_ms * 1000000 : 0; + + supervisor_fs_err_t result = supervisor_vfs_open_file(fs_mount, path, SUPERVISOR_FS_OPEN_WRITE, mtime_ns, &active_file); bool new_file = false; size_t old_length = 0; - if (result == FR_NO_FILE) { + if (result == SUPERVISOR_FS_NO_FILE) { new_file = true; - result = f_open(fs, &active_file, path, FA_WRITE | FA_OPEN_ALWAYS); - } else if (result == FR_OK) { - old_length = f_size(&active_file); + result = supervisor_vfs_open_file(fs_mount, path, + SUPERVISOR_FS_OPEN_WRITE | SUPERVISOR_FS_OPEN_CREATE, mtime_ns, &active_file); + } else if (result == SUPERVISOR_FS_OK) { + old_length = supervisor_vfs_file_size(&active_file); } - if (result == FR_NO_PATH) { - override_fattime(0); + if (result == SUPERVISOR_FS_NO_PATH) { filesystem_unlock(fs_mount); _discard_incoming(socket, request->content_length); _reply_missing(socket, request); return; } - if (result == FR_WRITE_PROTECTED) { + if (result == SUPERVISOR_FS_WRITE_PROTECTED) { // The filesystem is held by something else with write access (most // commonly USB-MSC: the host has CIRCUITPY mounted, so CircuitPython // can't write through FatFS). Match the mkdir/move/delete paths and // reply 409 Conflict so clients can show an actionable message // ("eject CIRCUITPY / disable USB MSC") instead of a generic 500. - override_fattime(0); filesystem_unlock(fs_mount); _discard_incoming(socket, request->content_length); _reply_conflict(socket, request); return; } - if (result != FR_OK) { - override_fattime(0); + if (result != SUPERVISOR_FS_OK) { filesystem_unlock(fs_mount); _discard_incoming(socket, request->content_length); _reply_server_error(socket, request); @@ -1087,19 +1054,18 @@ static void _write_file_and_reply(socketpool_socket_obj_t *socket, _request *req } // Change the file size to start. - f_lseek(&active_file, request->content_length); - if (f_tell(&active_file) < request->content_length) { + supervisor_vfs_seek_file(&active_file, request->content_length); + if (supervisor_vfs_tell_file(&active_file) < request->content_length) { if (!new_file) { // Truncate the file back to the old length. - f_lseek(&active_file, old_length); - f_truncate(&active_file); + supervisor_vfs_seek_file(&active_file, old_length); + supervisor_vfs_truncate_file(&active_file); } - f_close(&active_file); + supervisor_vfs_close_file(&active_file); if (new_file) { - f_unlink(fs, path); + supervisor_vfs_unlink(fs_mount, path); } - override_fattime(0); filesystem_unlock(fs_mount); // Too large. if (request->expect) { @@ -1112,8 +1078,10 @@ static void _write_file_and_reply(socketpool_socket_obj_t *socket, _request *req } else if (request->expect) { _reply_continue(socket, request); } - f_truncate(&active_file); - f_rewind(&active_file); + // Truncate to content_length (current offset) and rewind so that the body + // overwrites the file from the start. + supervisor_vfs_truncate_file(&active_file); + supervisor_vfs_seek_file(&active_file, 0); size_t total_read = 0; bool error = false; @@ -1129,18 +1097,17 @@ static void _write_file_and_reply(socketpool_socket_obj_t *socket, _request *req break; } total_read += len; - UINT actual; - f_write(&active_file, bytes, len, &actual); - if (actual < (UINT)len) { + size_t actual; + supervisor_fs_err_t result = supervisor_vfs_write_file(&active_file, bytes, len, &actual); + if (actual < (size_t)len || result != SUPERVISOR_FS_OK) { error = true; break; } } - f_close(&active_file); + supervisor_vfs_close_file(&active_file); filesystem_unlock(fs_mount); - override_fattime(0); if (error) { _discard_incoming(socket, request->content_length - total_read); _reply_server_error(socket, request); @@ -1286,12 +1253,12 @@ static bool _reply(socketpool_socket_obj_t *socket, _request *request) { // Delete is almost identical for files and directories so share the // implementation. if (strcasecmp(request->method, "DELETE") == 0) { - FRESULT result = supervisor_workflow_delete_recursive(path); - if (result == FR_WRITE_PROTECTED) { + supervisor_fs_err_t result = supervisor_workflow_delete_recursive(path); + if (result == SUPERVISOR_FS_WRITE_PROTECTED) { _reply_conflict(socket, request); - } else if (result == FR_NO_PATH || result == FR_NO_FILE) { + } else if (result == SUPERVISOR_FS_NO_PATH || result == SUPERVISOR_FS_NO_FILE) { _reply_missing(socket, request); - } else if (result != FR_OK) { + } else if (result != SUPERVISOR_FS_OK) { _reply_server_error(socket, request); } else { _reply_no_content(socket, request); @@ -1306,14 +1273,14 @@ static bool _reply(socketpool_socket_obj_t *socket, _request *request) { destination[destinationlen - 1] = '\0'; } - FRESULT result = supervisor_workflow_move(path, destination); - if (result == FR_WRITE_PROTECTED) { + supervisor_fs_err_t result = supervisor_workflow_move(path, destination); + if (result == SUPERVISOR_FS_WRITE_PROTECTED) { _reply_conflict(socket, request); - } else if (result == FR_EXIST) { // File exists and won't be overwritten. + } else if (result == SUPERVISOR_FS_EXIST) { // File exists and won't be overwritten. _reply_precondition_failed(socket, request); - } else if (result == FR_NO_PATH || result == FR_NO_FILE) { // Missing higher directories or target file. + } else if (result == SUPERVISOR_FS_NO_PATH || result == SUPERVISOR_FS_NO_FILE) { // Missing higher directories or target file. _reply_missing(socket, request); - } else if (result != FR_OK) { + } else if (result != SUPERVISOR_FS_OK) { _reply_server_error(socket, request); } else { _reply_created(socket, request); @@ -1321,18 +1288,15 @@ static bool _reply(socketpool_socket_obj_t *socket, _request *request) { } return false; } else if (directory && strcasecmp(request->method, "PUT") == 0) { - DWORD fattime = 0; - if (request->timestamp_ms > 0) { - truncate_time(request->timestamp_ms * 1000000, &fattime); - } - FRESULT result = supervisor_workflow_mkdir_parents(fattime, path); - if (result == FR_WRITE_PROTECTED) { + uint64_t mtime_ns = request->timestamp_ms > 0 ? (uint64_t)request->timestamp_ms * 1000000 : 0; + supervisor_fs_err_t result = supervisor_workflow_mkdir_parents(mtime_ns, path); + if (result == SUPERVISOR_FS_WRITE_PROTECTED) { _reply_conflict(socket, request); - } else if (result == FR_EXIST) { + } else if (result == SUPERVISOR_FS_EXIST) { _reply_no_content(socket, request); - } else if (result == FR_NO_PATH) { + } else if (result == SUPERVISOR_FS_NO_PATH) { _reply_missing(socket, request); - } else if (result != FR_OK) { + } else if (result != SUPERVISOR_FS_OK) { _reply_server_error(socket, request); } else { _reply_created(socket, request); @@ -1342,43 +1306,28 @@ static bool _reply(socketpool_socket_obj_t *socket, _request *request) { } // These responses don't use helpers because they stream data in and - // out. So, share the mount lookup code. + // out. So, share the mount lookup code. filesystem_for_path handles + // both the root filesystem and additional mounts, whether they are + // FAT or littlefs. const char *path_out = NULL; - mp_vfs_mount_t *vfs = mp_vfs_lookup_path(path, &path_out); - if (vfs == MP_VFS_NONE) { + supervisor_vfs_t *fs_mount = filesystem_for_path(path, &path_out); + if (fs_mount == NULL || !supervisor_vfs_supported(fs_mount)) { _reply_missing(socket, request); return false; } - fs_user_mount_t *fs_mount; - if (vfs == MP_VFS_ROOT) { - fs_mount = filesystem_circuitpy(); - } else { - fs_mount = MP_OBJ_TO_PTR(vfs->obj); - // Skip non-fat and non-native block file systems. - if (!filesystem_native_fatfs(fs_mount)) { - _reply_missing(socket, request); - return false; - } - // Check if the vfs name is one character long: it must be "/" in that case. - // If so don't remove the mount point name. We must use an absolute path - // because otherwise the path will be adjusted by os.getcwd() when it's looked up. - if (strlen(vfs->str) != 1) { - // Remove the mount point directory name, such as "/sd". - path += strlen(vfs->str); - } - pathlen = strlen(path); + path = (char *)path_out; + pathlen = strlen(path); - } - FATFS *fs = &fs_mount->fatfs; if (directory) { if (strcasecmp(request->method, "GET") == 0) { - FF_DIR dir; - FRESULT res = f_opendir(fs, &dir, path); + supervisor_vfs_dir_t dir; + memset(&dir, 0, sizeof(dir)); + supervisor_fs_err_t res = supervisor_vfs_opendir(fs_mount, path, &dir); // Put the / back for replies. if (pathlen > 1) { path[pathlen - 1] = '/'; } - if (res != FR_OK) { + if (res != SUPERVISOR_FS_OK) { _reply_missing(socket, request); return false; } @@ -1390,20 +1339,20 @@ static bool _reply(socketpool_socket_obj_t *socket, _request *request) { _reply_missing(socket, request); } - f_closedir(&dir); + supervisor_vfs_closedir(&dir); } } else { // Dealing with a file. if (strcasecmp(request->method, "GET") == 0) { - FIL active_file; - FRESULT result = f_open(fs, &active_file, path, FA_READ); + supervisor_vfs_file_t active_file; + supervisor_fs_err_t result = supervisor_vfs_open_file(fs_mount, path, SUPERVISOR_FS_OPEN_READ, 0, &active_file); - if (result != FR_OK) { + if (result != SUPERVISOR_FS_OK) { _reply_missing(socket, request); } else { _reply_with_file(socket, request, path, &active_file); } - f_close(&active_file); + supervisor_vfs_close_file(&active_file); } else if (strcasecmp(request->method, "PUT") == 0) { _write_file_and_reply(socket, request, fs_mount, path); return true; diff --git a/supervisor/shared/workflow.c b/supervisor/shared/workflow.c index 21dd0512eb1..be68fc59fa5 100644 --- a/supervisor/shared/workflow.c +++ b/supervisor/shared/workflow.c @@ -4,12 +4,10 @@ // // SPDX-License-Identifier: MIT -#include #include "py/mpconfig.h" #include "py/mpstate.h" #include "py/stackctrl.h" #include "supervisor/background_callback.h" -#include "supervisor/fatfs.h" #include "supervisor/filesystem.h" #include "supervisor/workflow.h" #include "supervisor/shared/serial.h" @@ -121,138 +119,139 @@ void supervisor_workflow_start(void) { #endif } -FRESULT supervisor_workflow_move(const char *old_path, const char *new_path) { +supervisor_fs_err_t supervisor_workflow_move(const char *old_path, const char *new_path) { const char *old_mount_path; const char *new_mount_path; - fs_user_mount_t *active_mount = filesystem_for_path(old_path, &old_mount_path); - fs_user_mount_t *new_mount = filesystem_for_path(new_path, &new_mount_path); - if (active_mount == NULL || new_mount == NULL || active_mount != new_mount || !filesystem_native_fatfs(active_mount)) { - return FR_NO_PATH; + supervisor_vfs_t *active_mount = filesystem_for_path(old_path, &old_mount_path); + supervisor_vfs_t *new_mount = filesystem_for_path(new_path, &new_mount_path); + if (active_mount == NULL || new_mount == NULL || active_mount != new_mount || !supervisor_vfs_supported(active_mount)) { + return SUPERVISOR_FS_NO_PATH; } if (!filesystem_lock(active_mount)) { - return FR_WRITE_PROTECTED; + return SUPERVISOR_FS_WRITE_PROTECTED; } - FATFS *fs = &active_mount->fatfs; - FRESULT result = f_rename(fs, old_mount_path, new_mount_path); + supervisor_fs_err_t result = supervisor_vfs_rename(active_mount, old_mount_path, new_mount_path); filesystem_unlock(active_mount); return result; } -FRESULT supervisor_workflow_mkdir(DWORD fattime, const char *full_path) { +supervisor_fs_err_t supervisor_workflow_mkdir(uint64_t mtime_ns, const char *full_path) { const char *mount_path; - fs_user_mount_t *active_mount = filesystem_for_path(full_path, &mount_path); - if (active_mount == NULL || !filesystem_native_fatfs(active_mount)) { - return FR_NO_PATH; + supervisor_vfs_t *active_mount = filesystem_for_path(full_path, &mount_path); + if (active_mount == NULL || !supervisor_vfs_supported(active_mount)) { + return SUPERVISOR_FS_NO_PATH; } // If there is a mount on the directory, then the mount_path will be empty. if (strlen(mount_path) == 0) { - return FR_EXIST; + return SUPERVISOR_FS_EXIST; } // Check to see if the directory exists already. We don't care about writing // it if it already exists. - FATFS *fs = &active_mount->fatfs; - FILINFO file; - FRESULT result = f_stat(fs, mount_path, &file); - if (result == FR_OK) { - return FR_EXIST; + supervisor_fs_err_t result = supervisor_vfs_stat(active_mount, mount_path, NULL, NULL, NULL); + if (result == SUPERVISOR_FS_OK) { + return SUPERVISOR_FS_EXIST; } if (!filesystem_lock(active_mount)) { - return FR_WRITE_PROTECTED; + return SUPERVISOR_FS_WRITE_PROTECTED; } - override_fattime(fattime); - result = f_mkdir(fs, mount_path); - override_fattime(0); + result = supervisor_vfs_mkdir(active_mount, mount_path, mtime_ns); filesystem_unlock(active_mount); return result; } -FRESULT supervisor_workflow_mkdir_parents(DWORD fattime, char *path) { - override_fattime(fattime); - FRESULT result = FR_OK; +supervisor_fs_err_t supervisor_workflow_mkdir_parents(uint64_t mtime_ns, char *path) { + supervisor_fs_err_t result = SUPERVISOR_FS_OK; // Make parent directories. for (size_t j = 1; j < strlen(path); j++) { if (path[j] == '/') { path[j] = '\0'; - result = supervisor_workflow_mkdir(fattime, path); + result = supervisor_workflow_mkdir(mtime_ns, path); path[j] = '/'; - if (result != FR_OK && result != FR_EXIST) { + if (result != SUPERVISOR_FS_OK && result != SUPERVISOR_FS_EXIST) { break; } } } // Make the target directory. - if (result == FR_OK || result == FR_EXIST) { - result = supervisor_workflow_mkdir(fattime, path); - // This may return FR_EXIST when a file with the same name already exists. + if (result == SUPERVISOR_FS_OK || result == SUPERVISOR_FS_EXIST) { + result = supervisor_workflow_mkdir(mtime_ns, path); + // This may return SUPERVISOR_FS_EXIST when a file with the same name already exists. // FATFS does the same thing. } - override_fattime(0); return result; } -static FRESULT supervisor_workflow_delete_directory_contents(FATFS *fs, const TCHAR *path) { - FF_DIR dir; - FILINFO file_info; +static supervisor_fs_err_t supervisor_workflow_delete_directory_contents(supervisor_vfs_t *active_mount, const char *path) { // Check the stack since we're putting paths on it. if (mp_stack_usage() >= MP_STATE_THREAD(stack_limit)) { - return FR_INT_ERR; + return SUPERVISOR_FS_IO; } - FRESULT res = FR_OK; - while (res == FR_OK) { - res = f_opendir(fs, &dir, path); - if (res != FR_OK) { + supervisor_vfs_dir_t dir; + memset(&dir, 0, sizeof(dir)); + char name[FF_MAX_LFN + 1]; + supervisor_fs_err_t res = SUPERVISOR_FS_OK; + while (res == SUPERVISOR_FS_OK) { + res = supervisor_vfs_opendir(active_mount, path, &dir); + if (res != SUPERVISOR_FS_OK) { break; } - res = f_readdir(&dir, &file_info); + res = supervisor_vfs_readdir(&dir, name, sizeof(name), NULL, NULL, NULL); // We close and reopen the directory every time since we're deleting // entries and it may invalidate the directory handle. - f_closedir(&dir); - if (res != FR_OK || file_info.fname[0] == '\0') { + supervisor_vfs_closedir(&dir); + if (res != SUPERVISOR_FS_OK || name[0] == '\0') { break; } size_t pathlen = strlen(path); - size_t fnlen = strlen(file_info.fname); - TCHAR full_path[pathlen + 1 + fnlen]; + size_t fnlen = strlen(name); + char full_path[pathlen + 1 + fnlen + 1]; memcpy(full_path, path, pathlen); full_path[pathlen] = '/'; - size_t full_pathlen = pathlen + 1 + fnlen; - memcpy(full_path + pathlen + 1, file_info.fname, fnlen); - full_path[full_pathlen] = '\0'; - if ((file_info.fattrib & AM_DIR) != 0) { - res = supervisor_workflow_delete_directory_contents(fs, full_path); + memcpy(full_path + pathlen + 1, name, fnlen + 1); + if (path[pathlen - 1] == '/') { + // Trim the extra slash we put in. This happens when path is "/". + memmove(full_path + pathlen, name, fnlen + 1); } - if (res != FR_OK) { + bool is_dir = false; + res = supervisor_vfs_stat(active_mount, full_path, &is_dir, NULL, NULL); + if (res != SUPERVISOR_FS_OK) { break; } - res = f_unlink(fs, full_path); + if (is_dir) { + res = supervisor_workflow_delete_directory_contents(active_mount, full_path); + if (res != SUPERVISOR_FS_OK) { + break; + } + } + res = supervisor_vfs_unlink(active_mount, full_path); } - f_closedir(&dir); + supervisor_vfs_closedir(&dir); return res; } -FRESULT supervisor_workflow_delete_recursive(const char *full_path) { +supervisor_fs_err_t supervisor_workflow_delete_recursive(const char *full_path) { const char *mount_path; - fs_user_mount_t *active_mount = filesystem_for_path(full_path, &mount_path); - if (active_mount == NULL || !filesystem_native_fatfs(active_mount)) { - return FR_NO_PATH; + supervisor_vfs_t *active_mount = filesystem_for_path(full_path, &mount_path); + if (active_mount == NULL || !supervisor_vfs_supported(active_mount)) { + return SUPERVISOR_FS_NO_PATH; } if (!filesystem_lock(active_mount)) { - return FR_WRITE_PROTECTED; + return SUPERVISOR_FS_WRITE_PROTECTED; } - FATFS *fs = &active_mount->fatfs; - FILINFO file; - FRESULT result = f_stat(fs, mount_path, &file); - if (result == FR_OK) { - if ((file.fattrib & AM_DIR) != 0) { - result = supervisor_workflow_delete_directory_contents(fs, mount_path); + supervisor_fs_err_t result = SUPERVISOR_FS_OK; + bool is_dir = false; + result = supervisor_vfs_stat(active_mount, mount_path, &is_dir, NULL, NULL); + if (result == SUPERVISOR_FS_OK) { + if (is_dir) { + result = supervisor_workflow_delete_directory_contents(active_mount, mount_path); } - if (result == FR_OK) { - result = f_unlink(fs, mount_path); + if (result == SUPERVISOR_FS_OK) { + result = supervisor_vfs_unlink(active_mount, mount_path); } } filesystem_unlock(active_mount); diff --git a/supervisor/shared/workflow.h b/supervisor/shared/workflow.h index e03f9c00573..0698e2edebe 100644 --- a/supervisor/shared/workflow.h +++ b/supervisor/shared/workflow.h @@ -6,12 +6,16 @@ #pragma once -#include "lib/oofatfs/ff.h" +#include + +#include "supervisor/filesystem.h" extern bool supervisor_workflow_connecting(void); -// File system helpers for workflow code. -FRESULT supervisor_workflow_move(const char *old_path, const char *new_path); -FRESULT supervisor_workflow_mkdir(DWORD fattime, const char *full_path); -FRESULT supervisor_workflow_mkdir_parents(DWORD fattime, char *path); -FRESULT supervisor_workflow_delete_recursive(const char *full_path); +// File system helpers for workflow code. All of them take full paths (such as +// "/code.py"), resolve them to a mount and return supervisor_fs_err_t. Paths +// may name either FAT or littlefs mounts. +supervisor_fs_err_t supervisor_workflow_move(const char *old_path, const char *new_path); +supervisor_fs_err_t supervisor_workflow_mkdir(uint64_t mtime_ns, const char *full_path); +supervisor_fs_err_t supervisor_workflow_mkdir_parents(uint64_t mtime_ns, char *path); +supervisor_fs_err_t supervisor_workflow_delete_recursive(const char *full_path); diff --git a/supervisor/stub/filesystem.c b/supervisor/stub/filesystem.c index ce9204b95c3..7534f4e3400 100644 --- a/supervisor/stub/filesystem.c +++ b/supervisor/stub/filesystem.c @@ -4,6 +4,9 @@ // // SPDX-License-Identifier: MIT +#include +#include + #include "supervisor/filesystem.h" @@ -29,13 +32,13 @@ void filesystem_set_internal_writable_by_usb(bool writable) { return; } -void filesystem_set_writable_by_usb(fs_user_mount_t *vfs, bool usb_writable) { +void filesystem_set_writable_by_usb(supervisor_vfs_t *vfs, bool usb_writable) { (void)vfs; (void)usb_writable; return; } -bool filesystem_is_writable_by_python(fs_user_mount_t *vfs) { +bool filesystem_is_writable_by_python(supervisor_vfs_t *vfs) { (void)vfs; return true; } @@ -49,7 +52,7 @@ void filesystem_set_internal_concurrent_write_protection(bool concurrent_write_p return; } -void filesystem_set_concurrent_write_protection(fs_user_mount_t *vfs, bool concurrent_write_protection) { +void filesystem_set_concurrent_write_protection(supervisor_vfs_t *vfs, bool concurrent_write_protection) { (void)vfs; (void)concurrent_write_protection; return; @@ -58,3 +61,137 @@ void filesystem_set_concurrent_write_protection(fs_user_mount_t *vfs, bool concu bool filesystem_present(void) { return false; } + +// Without a filesystem, nothing is supported and all file operations fail. +bool supervisor_vfs_supported(supervisor_vfs_t *fs_mount) { + (void)fs_mount; + return false; +} + +supervisor_fs_err_t supervisor_vfs_open_file(supervisor_vfs_t *vfs, const char *path, uint32_t flags, + uint64_t mtime_ns, supervisor_vfs_file_t *file) { + (void)vfs; + (void)path; + (void)flags; + (void)mtime_ns; + (void)file; + return SUPERVISOR_FS_IO; +} + +supervisor_fs_err_t supervisor_vfs_close_file(supervisor_vfs_file_t *file) { + (void)file; + return SUPERVISOR_FS_OK; +} + +supervisor_fs_err_t supervisor_vfs_read_file(supervisor_vfs_file_t *file, void *buf, size_t len, size_t *bytes_read) { + (void)file; + (void)buf; + (void)len; + *bytes_read = 0; + return SUPERVISOR_FS_IO; +} + +supervisor_fs_err_t supervisor_vfs_write_file(supervisor_vfs_file_t *file, const void *buf, size_t len, size_t *bytes_written) { + (void)file; + (void)buf; + (void)len; + *bytes_written = 0; + return SUPERVISOR_FS_IO; +} + +supervisor_fs_err_t supervisor_vfs_seek_file(supervisor_vfs_file_t *file, size_t offset) { + (void)file; + (void)offset; + return SUPERVISOR_FS_IO; +} + +size_t supervisor_vfs_tell_file(supervisor_vfs_file_t *file) { + (void)file; + return 0; +} + +size_t supervisor_vfs_file_size(supervisor_vfs_file_t *file) { + (void)file; + return 0; +} + +supervisor_fs_err_t supervisor_vfs_truncate_file(supervisor_vfs_file_t *file) { + (void)file; + return SUPERVISOR_FS_IO; +} + +supervisor_fs_err_t supervisor_vfs_stat(supervisor_vfs_t *vfs, const char *path, bool *is_dir, size_t *size, uint64_t *mtime_ns) { + (void)vfs; + (void)path; + if (is_dir != NULL) { + *is_dir = false; + } + if (size != NULL) { + *size = 0; + } + if (mtime_ns != NULL) { + *mtime_ns = 0; + } + return SUPERVISOR_FS_NO_FILE; +} + +supervisor_fs_err_t supervisor_vfs_mkdir(supervisor_vfs_t *vfs, const char *path, uint64_t mtime_ns) { + (void)vfs; + (void)path; + (void)mtime_ns; + return SUPERVISOR_FS_IO; +} + +supervisor_fs_err_t supervisor_vfs_rename(supervisor_vfs_t *vfs, const char *old_path, const char *new_path) { + (void)vfs; + (void)old_path; + (void)new_path; + return SUPERVISOR_FS_IO; +} + +supervisor_fs_err_t supervisor_vfs_unlink(supervisor_vfs_t *vfs, const char *path) { + (void)vfs; + (void)path; + return SUPERVISOR_FS_IO; +} + +supervisor_fs_err_t supervisor_vfs_opendir(supervisor_vfs_t *vfs, const char *path, supervisor_vfs_dir_t *dir) { + (void)vfs; + (void)path; + (void)dir; + return SUPERVISOR_FS_NO_FILE; +} + +supervisor_fs_err_t supervisor_vfs_readdir(supervisor_vfs_dir_t *dir, char *name, size_t name_len, bool *is_dir, size_t *size, uint64_t *mtime_ns) { + (void)dir; + (void)name_len; + if (is_dir != NULL) { + *is_dir = false; + } + if (size != NULL) { + *size = 0; + } + if (mtime_ns != NULL) { + *mtime_ns = 0; + } + name[0] = '\0'; + return SUPERVISOR_FS_OK; +} + +supervisor_fs_err_t supervisor_vfs_rewinddir(supervisor_vfs_dir_t *dir) { + (void)dir; + return SUPERVISOR_FS_IO; +} + +supervisor_fs_err_t supervisor_vfs_closedir(supervisor_vfs_dir_t *dir) { + (void)dir; + return SUPERVISOR_FS_OK; +} + +supervisor_fs_err_t supervisor_vfs_statfs(supervisor_vfs_t *vfs, size_t *block_size, size_t *total_blocks, size_t *free_blocks) { + (void)vfs; + (void)block_size; + (void)total_blocks; + (void)free_blocks; + return SUPERVISOR_FS_IO; +}