diff --git a/.github/workflows/wasi.yml b/.github/workflows/wasi.yml index 5072ccc573b..ac46bb323fa 100644 --- a/.github/workflows/wasi.yml +++ b/.github/workflows/wasi.yml @@ -60,18 +60,20 @@ jobs: # Tests incompatible with WASI are annotated with # #[cfg_attr(wasi_runner, ignore)] in the test source files. # TODO: add integration tests for these tools as WASI support is extended: - # arch b2sum cat cksum cp csplit date dir dircolors fmt join + # arch b2sum cksum cp csplit date dir dircolors fmt join # ls md5sum mkdir mv nproc pathchk pr printenv ptx pwd readlink # realpath rm rmdir seq sha1sum sha224sum sha256sum sha384sum - # sha512sum shred sleep sort split tail touch tsort uname uniq - # vdir + # sha512sum shred sleep split tsort uname uniq vdir UUTESTS_BINARY_PATH="$(pwd)/target/${{ matrix.job.target }}/debug/coreutils.wasm" \ UUTESTS_WASM_RUNNER=wasmtime \ cargo test --test tests -- \ test_base32:: test_base64:: test_basenc:: test_basename:: \ + test_cat:: \ + test_cp::test_cp_arg_symlink test_cp::test_cp_preserve_timestamps \ test_comm:: test_cut:: test_dirname:: test_echo:: \ test_expand:: test_factor:: test_false:: test_fold:: \ test_head:: test_link:: test_ln:: test_nl:: test_numfmt:: \ - test_od:: test_paste:: test_printf:: test_shuf:: test_sum:: \ - test_tee:: test_tr:: test_true:: test_truncate:: \ - test_unexpand:: test_unlink:: test_wc:: test_yes:: + test_od:: test_paste:: test_printf:: test_shuf:: test_sort:: \ + test_sum:: test_tail:: test_tee:: test_touch:: test_tr:: \ + test_true:: test_truncate:: test_unexpand:: test_unlink:: test_wc:: \ + test_yes:: diff --git a/docs/src/wasi-test-gaps.md b/docs/src/wasi-test-gaps.md index 4789ef03dc9..a7f660aa4e2 100644 --- a/docs/src/wasi-test-gaps.md +++ b/docs/src/wasi-test-gaps.md @@ -1,3 +1,5 @@ + + # WASI integration test gaps Tests annotated with `#[cfg_attr(wasi_runner, ignore = "...")]` are skipped when running integration tests against a WASI binary via wasmtime. This document tracks the reasons so that gaps in WASI support are visible in one place. @@ -6,7 +8,7 @@ To find all annotated tests: `grep -rn 'wasi_runner, ignore' tests/` ## Tools not yet covered by integration tests -arch, b2sum, cat, cksum, cp, csplit, date, dir, dircolors, fmt, join, ls, md5sum, mkdir, mv, nproc, pathchk, pr, printenv, ptx, pwd, readlink, realpath, rm, rmdir, seq, sha1sum, sha224sum, sha256sum, sha384sum, sha512sum, shred, sleep, sort, split, tail, touch, tsort, uname, uniq, vdir, yes +arch, b2sum, cksum, cp, csplit, date, dir, dircolors, fmt, join, ls, md5sum, mkdir, mv, nproc, pathchk, pr, printenv, ptx, pwd, readlink, realpath, rm, rmdir, seq, sha1sum, sha224sum, sha256sum, sha384sum, sha512sum, shred, sleep, split, tsort, uname, uniq, vdir, yes ## WASI sandbox: host paths not visible @@ -32,6 +34,58 @@ WASI does not support spawning child processes. Tests that shell out to other co When stdin is a seekable file, wasmtime does not preserve the file position between the host and guest. Tests that validate stdin offset behavior after `head` reads are skipped. -## WASI: read_link on absolute paths fails under wasmtime via spawned test harness +## WASI: absolute symlink targets fail under wasmtime + +Under wasmtime, symlinks whose stored target is an absolute guest path (for example `bar -> /foo`) fail in cases that work on POSIX: `readlink bar` exits 1 and `readlink -v bar` reports `Permission denied`, while the equivalent relative symlink (`bar -> foo`) succeeds. This reproduces even when `wasmtime` is invoked directly, so it is not specific to the cargo-test harness. `realpath` and symlink-heavy `cp` paths inherit the same limitation, and individual symptom tests under this umbrella are annotated with narrower reasons describing the observed errno mismatch. + +## WASI: no Unix domain socket support + +WASI does not support Unix domain sockets. Tests that create or read from `AF_UNIX` sockets are skipped. + +## WASI: no locale data + +The WASI sandbox does not ship locale data, so `setlocale`/`LC_ALL` have no effect and sorting falls back to byte comparison. Tests that depend on locale-aware collation or month-name translation are skipped. + +## WASI: tail follow mode disabled + +`tail -f` / `tail -F` (follow mode) requires change-notification mechanisms (`inotify`, `kqueue`) and signal handling that WASI does not provide, so follow is disabled on WASI and a warning is emitted. Tests that exercise follow behavior are skipped. + +## WASI: cannot detect unsafe overwrite + +`is_unsafe_overwrite` (used by `cat` to detect input-is-output situations) is stubbed to return `false` on WASI because the required `stat` / device-and-inode comparison is not available. Tests that assert this error path are skipped. + +## WASI: pre-epoch timestamps not representable + +WASI Preview 1 `Timestamp` is a `u64` nanosecond count since the Unix epoch, so `path_filestat_set_times` (and therefore `touch -t` with a two-digit year ≥ 69) cannot express dates before 1970. Tests that set pre-epoch timestamps are skipped. + +## WASI: no timezone database + +wasi-libc does not ship tzdata, so `TZ` is not honoured and timezone-dependent validation (e.g. `touch -t` rejecting a nonexistent local time during a DST transition) does not happen. Tests that rely on this are skipped. + +## WASI: guest root is a writable preopen + +The test harness maps the per-test working directory as the guest's `/`. That makes `/` writable inside the guest, so GNU-style protections against operating on the system root (e.g. `touch /` failing) cannot be exercised. It also means guest-visible absolute paths are rooted at `/`, not at the host tempdir. Tests that compare against host `canonicalize()` results or pass host absolute paths into the guest (for example some `cp --parents`, `readlink`, `realpath`, `pwd`, and `ls` cases) need guest-aware expectations or separate coverage. Tests that assert the root-protection behavior are skipped. + +## WASI: `touch -` (stdout) unsupported + +On WASI, `touch -` returns `UnsupportedPlatformFeature` because the guest cannot reliably locate the host file backing stdout. Tests that exercise `touch -` are skipped. + +## WASI: rlimit/setrlimit not supported + +WASI has no concept of per-process resource limits, so `setrlimit` (and the `rlimit` crate that wraps it) has no effect. Tests that set `RLIMIT_NOFILE` to verify behavior under restricted file-descriptor budgets are skipped. + +## WASI: sysinfo/meminfo not available + +WASI has no `sysinfo`/`/proc/meminfo` equivalent, so features that size buffers as a percentage of system memory (e.g. `sort -S 10%`) cannot resolve the limit and fail. Tests that exercise percentage-based sizing are skipped. + +## WASI: errno/error-message mismatches + +Several error paths surface different errno values (and therefore different error messages) through wasmtime than on POSIX. Observed cases: + +- Opening a directory as a file returns `EBADF` rather than `EISDIR`. +- Redirecting a directory into stdin returns `ENOENT` rather than `EISDIR`. +- Filesystem permission errors surface as `ENOENT` rather than `EACCES`. +- Symlink-loop traversal does not reliably surface `ELOOP` ("Too many levels of symbolic links"). +- Opening a symlink-to-directory does not reliably surface `EISDIR`. -`fs::read_link` on an absolute path inside the sandbox (e.g. `/file2`) returns `EPERM` when the WASI binary is launched through `std::process::Command` from the test harness, even though the same call works when wasmtime is invoked directly. This breaks `uucore::fs::canonicalize` for symlink sources, so tests that rely on following a symlink to compute a relative path are skipped. +Tests that assert specific error text for these paths are skipped. diff --git a/src/uu/cat/src/platform/mod.rs b/src/uu/cat/src/platform/mod.rs index cde1ee96cbb..77255eff14a 100644 --- a/src/uu/cat/src/platform/mod.rs +++ b/src/uu/cat/src/platform/mod.rs @@ -9,7 +9,11 @@ pub use self::unix::is_safe_overwrite; #[cfg(windows)] pub use self::windows::is_safe_overwrite; -// WASI: no fstat-based device/inode checks available; assume safe. +// WASI: when stdout is inherited from a host file descriptor, wasmtime +// reports its fstat as all-zero (st_dev == st_ino == 0), so the dev/inode +// comparison against any input file descriptor can never match. There is +// no reliable way to detect unsafe overwrite here; assume safe rather than +// risk a spurious error. #[cfg(target_os = "wasi")] pub fn is_safe_overwrite(_input: &I, _output: &O) -> bool { true diff --git a/src/uu/cp/Cargo.toml b/src/uu/cp/Cargo.toml index 0f2a818119e..6b778010b27 100644 --- a/src/uu/cp/Cargo.toml +++ b/src/uu/cp/Cargo.toml @@ -49,6 +49,9 @@ windows-sys = { workspace = true, features = [ "Win32_Storage_FileSystem", ] } +[target.'cfg(target_os = "wasi")'.dependencies] +rustix = { workspace = true, features = ["fs"] } + [[bin]] name = "cp" path = "src/main.rs" diff --git a/src/uu/cp/src/cp.rs b/src/uu/cp/src/cp.rs index b8062b8dafe..a0ec2318fa0 100644 --- a/src/uu/cp/src/cp.rs +++ b/src/uu/cp/src/cp.rs @@ -2,8 +2,8 @@ // // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -// spell-checker:ignore (ToDO) copydir fiemap ftruncate linkgs lstat nlink nlinks pathbuf pwrite reflink strs xattrs symlinked deduplicated advcpmv nushell IRWXG IRWXO IRWXU IRWXUGO IRWXU IRWXG IRWXO IRWXUGO sflag -// spell-checker:ignore RDONLY futimens utimensat +// spell-checker:ignore (ToDO) copydir ficlone fiemap filestat ftruncate linkgs lstat nlink nlinks pathbuf pwrite reflink strs utimensat xattrs symlinked deduplicated advcpmv nushell IRWXG IRWXO IRWXU IRWXUGO IRWXU IRWXG IRWXO IRWXUGO sflag +// spell-checker:ignore RDONLY futimens use std::cmp::Ordering; use std::collections::{HashMap, HashSet}; @@ -21,6 +21,7 @@ use uucore::fsxattr::{copy_acls, copy_xattrs, copy_xattrs_skip_selinux}; use uucore::translate; use clap::{Arg, ArgAction, ArgMatches, Command, builder::ValueParser, value_parser}; +#[cfg(not(target_os = "wasi"))] use filetime::FileTime; use indicatif::{ProgressBar, ProgressStyle}; #[cfg(unix)] @@ -1368,9 +1369,9 @@ fn parse_path_args( /// Check if an error is ENOTSUP/EOPNOTSUPP (operation not supported). /// This is used to suppress xattr errors on filesystems that don't support them. fn is_enotsup_error(error: &CpError) -> bool { - #[cfg(unix)] + #[cfg(any(unix, target_os = "wasi"))] const EOPNOTSUPP: i32 = libc::EOPNOTSUPP; - #[cfg(not(unix))] + #[cfg(not(any(unix, target_os = "wasi")))] const EOPNOTSUPP: i32 = 95; match error { @@ -1780,6 +1781,66 @@ fn copy_extended_attrs(source: &Path, dest: &Path, skip_selinux: bool) -> CopyRe Ok(()) } +/// Copy the access and modification timestamps from `source_metadata` onto `dest`. +/// If `dest` is a symlink, the symlink's own timestamps are set rather than the +/// target's. +/// +/// On WASI this calls `rustix::fs::utimensat` directly because `filetime`'s +/// WASI backend panics in `from_last_{access,modification}_time`. `SystemTime` +/// values are converted to `Timespec` against `UNIX_EPOCH`, matching WASI's +/// `path_filestat_set_times` contract (unsigned nanosecond count — pre-epoch +/// source times can't be represented). +fn set_timestamps(source_metadata: &Metadata, dest: &Path) -> CopyResult<()> { + #[cfg(target_os = "wasi")] + { + use std::time::UNIX_EPOCH; + let to_timespec = |t: std::time::SystemTime| -> io::Result { + let d = t + .duration_since(UNIX_EPOCH) + .map_err(|e| io::Error::new(io::ErrorKind::Unsupported, e))?; + Ok(rustix::fs::Timespec { + tv_sec: d.as_secs() as i64, + tv_nsec: d.subsec_nanos() as i32, + }) + }; + let timestamps = rustix::fs::Timestamps { + last_access: to_timespec(source_metadata.accessed()?)?, + last_modification: to_timespec(source_metadata.modified()?)?, + }; + let flags = if dest.is_symlink() { + rustix::fs::AtFlags::SYMLINK_NOFOLLOW + } else { + rustix::fs::AtFlags::empty() + }; + rustix::fs::utimensat(rustix::fs::CWD, dest, ×tamps, flags) + .map_err(io::Error::from)?; + Ok(()) + } + + #[cfg(not(target_os = "wasi"))] + { + let atime = FileTime::from_last_access_time(source_metadata); + let mtime = FileTime::from_last_modification_time(source_metadata); + #[cfg(unix)] + let no_open = { + let ft = source_metadata.file_type(); + dest.is_symlink() + || ft.is_fifo() + || ft.is_socket() + || ft.is_char_device() + || ft.is_block_device() + }; + #[cfg(not(unix))] + let no_open = dest.is_symlink(); + if no_open { + filetime::set_symlink_file_times(dest, atime, mtime)?; + } else { + filetime::set_file_times(dest, atime, mtime)?; + } + Ok(()) + } +} + /// Copy the specified attributes from one path to another. /// If `skip_selinux_xattr` is true, the security.selinux xattr will not be copied /// (used when -Z is specified to set the default context instead). @@ -1888,32 +1949,8 @@ pub(crate) fn copy_attributes( Ok(()) })?; - handle_preserve(attributes.timestamps, || -> CopyResult<()> { - let atime = FileTime::from_last_access_time(&source_metadata); - let mtime = FileTime::from_last_modification_time(&source_metadata); - // `set_file_times` opens the destination (O_RDONLY) before calling - // futimens; opening a FIFO or device with no peer blocks forever, and a - // socket cannot be opened at all. For symlinks and these special files - // use the path-based, no-follow variant, which sets the times via - // utimensat without opening. - #[cfg(unix)] - let no_open = { - let ft = source_metadata.file_type(); - dest.is_symlink() - || ft.is_fifo() - || ft.is_socket() - || ft.is_char_device() - || ft.is_block_device() - }; - #[cfg(not(unix))] - let no_open = dest.is_symlink(); - if no_open { - filetime::set_symlink_file_times(dest, atime, mtime)?; - } else { - filetime::set_file_times(dest, atime, mtime)?; - } - - Ok(()) + handle_preserve(attributes.timestamps, || { + set_timestamps(&source_metadata, dest) })?; #[cfg(all(feature = "selinux", any(target_os = "linux", target_os = "android")))] @@ -1959,19 +1996,9 @@ pub(crate) fn copy_attributes( fn symlink_file( source: &Path, dest: &Path, - #[cfg(not(target_os = "wasi"))] symlinked_files: &mut HashSet, - #[cfg(target_os = "wasi")] _symlinked_files: &mut HashSet, + symlinked_files: &mut HashSet, ) -> CopyResult<()> { - #[cfg(target_os = "wasi")] - { - Err(CpError::IoErrContext( - io::Error::new(io::ErrorKind::Unsupported, "symlinks not supported"), - translate!("cp-error-cannot-create-symlink", - "dest" => get_filename(dest).unwrap_or("?").quote(), - "source" => get_filename(source).unwrap_or("?").quote()), - )) - } - #[cfg(not(any(windows, target_os = "wasi")))] + #[cfg(unix)] { std::os::unix::fs::symlink(source, dest).map_err(|e| { CpError::IoErrContext( @@ -1993,13 +2020,21 @@ fn symlink_file( ) })?; } - #[cfg(not(target_os = "wasi"))] + #[cfg(target_os = "wasi")] { - if let Ok(file_info) = FileInformation::from_path(dest, false) { - symlinked_files.insert(file_info); - } - Ok(()) + rustix::fs::symlink(source, dest).map_err(|e| { + CpError::IoErrContext( + io::Error::from(e), + translate!("cp-error-cannot-create-symlink", + "dest" => get_filename(dest).unwrap_or("?").quote(), + "source" => get_filename(source).unwrap_or("?").quote()), + ) + })?; } + if let Ok(file_info) = FileInformation::from_path(dest, false) { + symlinked_files.insert(file_info); + } + Ok(()) } fn context_for(src: &Path, dest: &Path) -> String { diff --git a/src/uu/sort/Cargo.toml b/src/uu/sort/Cargo.toml index 5eaf7d9bccf..470f8a1b95e 100644 --- a/src/uu/sort/Cargo.toml +++ b/src/uu/sort/Cargo.toml @@ -29,7 +29,6 @@ compare = { workspace = true } itertools = { workspace = true } memchr = { workspace = true } rand = { workspace = true } -rayon = { workspace = true } self_cell = { workspace = true } tempfile = { workspace = true } thiserror = { workspace = true } @@ -47,6 +46,9 @@ foldhash = { workspace = true } [target.'cfg(all(unix, not(any(target_os = "redox", target_os = "fuchsia", target_os = "haiku", target_os = "solaris", target_os = "illumos"))))'.dependencies] rustix = { workspace = true, features = ["system", "process"] } +[target.'cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))'.dependencies] +rayon = { workspace = true } + [target.'cfg(not(any(target_os = "redox", target_os = "wasi")))'.dependencies] ctrlc = { workspace = true } diff --git a/src/uu/sort/build.rs b/src/uu/sort/build.rs new file mode 100644 index 00000000000..e138c3eeaac --- /dev/null +++ b/src/uu/sort/build.rs @@ -0,0 +1,14 @@ +fn main() { + // Set a short alias for the WASI-without-threads configuration so that + // source files can use `#[cfg(wasi_no_threads)]` instead of the verbose + // `#[cfg(all(target_os = "wasi", not(target_feature = "atomics")))]`. + println!("cargo::rustc-check-cfg=cfg(wasi_no_threads)"); + + let target_os = std::env::var("CARGO_CFG_TARGET_OS").unwrap_or_default(); + let has_atomics = std::env::var("CARGO_CFG_TARGET_FEATURE") + .is_ok_and(|f| f.split(',').any(|feat| feat == "atomics")); + + if target_os == "wasi" && !has_atomics { + println!("cargo::rustc-cfg=wasi_no_threads"); + } +} diff --git a/src/uu/sort/src/check/mod.rs b/src/uu/sort/src/check/mod.rs new file mode 100644 index 00000000000..79226660e84 --- /dev/null +++ b/src/uu/sort/src/check/mod.rs @@ -0,0 +1,44 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +//! Check if a file is ordered. +//! +//! On most platforms this uses a multi-threaded reader. On WASI without +//! atomics, a synchronous variant is used instead. The two implementations +//! live in sibling modules and are selected via cfg at the module boundary. + +use std::cmp::Ordering; +use std::ffi::OsStr; + +use uucore::error::UResult; + +use crate::{GlobalSettings, open}; + +#[cfg(not(wasi_no_threads))] +mod threaded; +#[cfg(not(wasi_no_threads))] +use threaded as runner; + +#[cfg(wasi_no_threads)] +mod sync; +#[cfg(wasi_no_threads)] +use sync as runner; + +/// Check if the file at `path` is ordered. +pub fn check(path: &OsStr, settings: &GlobalSettings) -> UResult<()> { + let max_allowed_cmp = if settings.unique { + Ordering::Less + } else { + Ordering::Equal + }; + let file = open(path)?; + let chunk_size = if settings.buffer_size < 100 * 1024 { + settings.buffer_size + } else { + 100 * 1024 + }; + + runner::check(path, settings, max_allowed_cmp, file, chunk_size) +} diff --git a/src/uu/sort/src/check/sync.rs b/src/uu/sort/src/check/sync.rs new file mode 100644 index 00000000000..1cb338caacb --- /dev/null +++ b/src/uu/sort/src/check/sync.rs @@ -0,0 +1,98 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +//! Synchronous ordered-file check for targets without thread support. + +use std::cmp::Ordering; +use std::ffi::OsStr; +use std::io::Read; +use std::iter; + +use itertools::Itertools; +use uucore::error::UResult; + +use crate::{ + GlobalSettings, SortError, + chunks::{self, Chunk, RecycledChunk}, + compare_by, +}; + +pub(super) fn check( + path: &OsStr, + settings: &GlobalSettings, + max_allowed_cmp: Ordering, + mut file: Box, + chunk_size: usize, +) -> UResult<()> { + let separator = settings.line_ending.into(); + let mut carry_over = vec![]; + let mut prev_chunk: Option = None; + let mut spare_recycled: Option = None; + let mut line_idx = 0; + + loop { + let recycled = spare_recycled + .take() + .unwrap_or_else(|| RecycledChunk::new(chunk_size)); + + let (chunk, should_continue) = chunks::read_to_chunk( + recycled, + None, + &mut carry_over, + &mut file, + &mut iter::empty(), + separator, + settings, + )?; + + let Some(chunk) = chunk else { + break; + }; + + line_idx += 1; + if let Some(prev) = prev_chunk.take() { + let prev_last = prev.lines().last().unwrap(); + let new_first = chunk.lines().first().unwrap(); + + if compare_by( + prev_last, + new_first, + settings, + prev.line_data(), + chunk.line_data(), + ) > max_allowed_cmp + { + return Err(SortError::Disorder { + file: path.to_owned(), + line_number: line_idx, + line: String::from_utf8_lossy(new_first.line).into_owned(), + silent: settings.check_silent, + } + .into()); + } + spare_recycled = Some(prev.recycle()); + } + + for (a, b) in chunk.lines().iter().tuple_windows() { + line_idx += 1; + if compare_by(a, b, settings, chunk.line_data(), chunk.line_data()) > max_allowed_cmp { + return Err(SortError::Disorder { + file: path.to_owned(), + line_number: line_idx, + line: String::from_utf8_lossy(b.line).into_owned(), + silent: settings.check_silent, + } + .into()); + } + } + + prev_chunk = Some(chunk); + + if !should_continue { + break; + } + } + Ok(()) +} diff --git a/src/uu/sort/src/check.rs b/src/uu/sort/src/check/threaded.rs similarity index 61% rename from src/uu/sort/src/check.rs rename to src/uu/sort/src/check/threaded.rs index a826bc75507..5f2ac6e3ad8 100644 --- a/src/uu/sort/src/check.rs +++ b/src/uu/sort/src/check/threaded.rs @@ -3,38 +3,32 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -//! Check if a file is ordered +//! Multi-threaded ordered-file check: a reader thread streams chunks while +//! the main thread compares the boundary between consecutive chunks. + +use std::cmp::Ordering; +use std::ffi::OsStr; +use std::io::Read; +use std::iter; +use std::sync::mpsc::{Receiver, SyncSender, sync_channel}; +use std::thread; + +use itertools::Itertools; +use uucore::error::UResult; use crate::{ GlobalSettings, SortError, chunks::{self, Chunk, RecycledChunk}, - compare_by, open, -}; -use itertools::Itertools; -use std::{ - cmp::Ordering, - ffi::OsStr, - io::Read, - iter, - sync::mpsc::{Receiver, SyncSender, sync_channel}, - thread, + compare_by, }; -use uucore::error::UResult; -/// Check if the file at `path` is ordered. -/// -/// # Returns -/// -/// The code we should exit with. -pub fn check(path: &OsStr, settings: &GlobalSettings) -> UResult<()> { - let max_allowed_cmp = if settings.unique { - // If `unique` is enabled, the previous line must compare _less_ to the next one. - Ordering::Less - } else { - // Otherwise, the line previous line must compare _less or equal_ to the next one. - Ordering::Equal - }; - let file = open(path)?; +pub(super) fn check( + path: &OsStr, + settings: &GlobalSettings, + max_allowed_cmp: Ordering, + file: Box, + chunk_size: usize, +) -> UResult<()> { let (recycled_sender, recycled_receiver) = sync_channel(2); let (loaded_sender, loaded_receiver) = sync_channel(2); thread::spawn({ @@ -42,28 +36,17 @@ pub fn check(path: &OsStr, settings: &GlobalSettings) -> UResult<()> { move || reader(file, &recycled_receiver, &loaded_sender, &settings) }); for _ in 0..2 { - let _ = recycled_sender.send(RecycledChunk::new(if settings.buffer_size < 100 * 1024 { - // when the buffer size is smaller than 100KiB we choose it instead of the default. - // this improves testability. - settings.buffer_size - } else { - 100 * 1024 - })); + let _ = recycled_sender.send(RecycledChunk::new(chunk_size)); } let mut prev_chunk: Option = None; let mut line_idx = 0; let mut result: UResult<()> = Ok(()); - // Note that we iterate over a reference, so that `loaded_receiver` is still alive - // once we stop: `chunks::read` unwraps its `send`, so dropping our end while the - // reader thread is still going would panic it. Since we stop at the *first* - // disorder, the reader is usually still working at that point, so we shut it down - // in an orderly fashion below instead of just dropping our end. + // Keep the receiver alive after the first disorder so the reader's in-flight + // send can complete while the channel is drained below. 'outer: for chunk in &loaded_receiver { line_idx += 1; if let Some(prev_chunk) = prev_chunk.take() { - // Check if the first element of the new chunk is greater than the last - // element from the previous chunk let prev_last = prev_chunk.lines().last().unwrap(); let new_first = chunk.lines().first().unwrap(); @@ -103,11 +86,6 @@ pub fn check(path: &OsStr, settings: &GlobalSettings) -> UResult<()> { prev_chunk = Some(chunk); } - - // Stop handing out buffers, so the reader runs out of work, then drain anything it - // has already produced. This lets its in-flight `send` complete instead of failing, - // and terminates because the reader can only own the (at most two) recycled chunks - // that are still outstanding. drop(recycled_sender); while loaded_receiver.recv().is_ok() {} diff --git a/src/uu/sort/src/chunks.rs b/src/uu/sort/src/chunks.rs index 4b388dc7bbd..ff5df3a114e 100644 --- a/src/uu/sort/src/chunks.rs +++ b/src/uu/sort/src/chunks.rs @@ -9,10 +9,11 @@ #![allow(dead_code)] // Ignores non-used warning for `borrow_buffer` in `Chunk` +#[cfg(not(wasi_no_threads))] +use std::sync::mpsc::SyncSender; use std::{ io::{ErrorKind, Read}, ops::Range, - sync::mpsc::SyncSender, }; use memchr::memchr_iter; @@ -157,30 +158,13 @@ impl RecycledChunk { } } -/// Read a chunk, parse lines and send them. +/// Read a chunk from the input, parse lines, and return it directly. /// -/// No empty chunk will be sent. If we reach the end of the input, `false` is returned. -/// However, if this function returns `true`, it is not guaranteed that there is still -/// input left: If the input fits _exactly_ into a buffer, we will only notice that there's -/// nothing more to read at the next invocation. In case there is no input left, nothing will -/// be sent. -/// -/// # Arguments -/// -/// (see also `read_to_chunk` for a more detailed documentation) -/// -/// * `sender`: The sender to send the lines to the sorter. -/// * `recycled_chunk`: The recycled chunk, as returned by `Chunk::recycle`. -/// (i.e. `buffer.len()` should be equal to `buffer.capacity()`) -/// * `max_buffer_size`: How big `buffer` can be. -/// * `carry_over`: The bytes that must be carried over in between invocations. -/// * `file`: The current file. -/// * `next_files`: What `file` should be updated to next. -/// * `separator`: The line separator. -/// * `settings`: The global settings. +/// Returns `(Some(chunk), should_continue)` if data was read, or +/// `(None, false)` if the input was empty. The `should_continue` flag +/// indicates whether more data may remain. #[allow(clippy::too_many_arguments)] -pub fn read( - sender: &SyncSender, +pub fn read_to_chunk( recycled_chunk: RecycledChunk, max_buffer_size: Option, carry_over: &mut Vec, @@ -188,7 +172,7 @@ pub fn read( next_files: &mut impl Iterator>, separator: u8, settings: &GlobalSettings, -) -> UResult { +) -> UResult<(Option, bool)> { let RecycledChunk { lines, selections, @@ -202,7 +186,6 @@ pub fn read( mut buffer, } = recycled_chunk; if buffer.len() < carry_over.len() { - // Separate carry_over and copy them to avoid cost of 0 fill buffer buffer.extend_from_slice(&carry_over[buffer.len()..]); } buffer[..carry_over.len()].copy_from_slice(carry_over); @@ -218,7 +201,7 @@ pub fn read( carry_over.extend_from_slice(&buffer[read..]); if read != 0 { - let payload: UResult = Chunk::try_new(buffer, |buffer| { + let chunk: UResult = Chunk::try_new(buffer, |buffer| { let selections = unsafe { // SAFETY: It is safe to transmute to an empty vector of selections with shorter lifetime. // It was only temporarily transmuted to a Vec> to make recycling possible. @@ -254,7 +237,38 @@ pub fn read( line_count_hint, }) }); - sender.send(payload?).unwrap(); + Ok((Some(chunk?), should_continue)) + } else { + Ok((None, should_continue)) + } +} + +/// Read a chunk, parse lines and send them via channel. +/// +/// Wrapper around [`read_to_chunk`] for the threaded code path. +#[cfg(not(wasi_no_threads))] +#[allow(clippy::too_many_arguments)] +pub fn read( + sender: &SyncSender, + recycled_chunk: RecycledChunk, + max_buffer_size: Option, + carry_over: &mut Vec, + file: &mut T, + next_files: &mut impl Iterator>, + separator: u8, + settings: &GlobalSettings, +) -> UResult { + let (chunk, should_continue) = read_to_chunk( + recycled_chunk, + max_buffer_size, + carry_over, + file, + next_files, + separator, + settings, + )?; + if let Some(chunk) = chunk { + sender.send(chunk).unwrap(); } Ok(should_continue) } @@ -431,32 +445,3 @@ fn read_to_buffer( } } } - -/// Parse a buffer into a `ChunkContents` suitable for `Chunk::try_new`. -/// Used by the WASI single-threaded sort path. -#[cfg(target_os = "wasi")] -pub fn parse_into_chunk<'a>( - buffer: &'a [u8], - separator: u8, - settings: &GlobalSettings, -) -> ChunkContents<'a> { - let mut lines = Vec::new(); - let mut line_data = LineData::default(); - let mut token_buffer = Vec::new(); - let mut line_count_hint = 0; - parse_lines( - buffer, - &mut lines, - &mut line_data, - &mut token_buffer, - &mut line_count_hint, - separator, - settings, - ); - ChunkContents { - lines, - line_data, - token_buffer, - line_count_hint, - } -} diff --git a/src/uu/sort/src/ext_sort/mod.rs b/src/uu/sort/src/ext_sort/mod.rs index 099a4b72e62..f93082cfbb1 100644 --- a/src/uu/sort/src/ext_sort/mod.rs +++ b/src/uu/sort/src/ext_sort/mod.rs @@ -6,15 +6,50 @@ //! External sort: sort large inputs that may not fit in memory. //! //! On most platforms this uses a multi-threaded chunked approach with -//! temporary files. On WASI (no threads) we fall back to an in-memory sort. +//! temporary files. On WASI without atomics, a synchronous variant is used +//! instead. The two implementations live in sibling modules and are selected +//! via cfg at the module boundary. -#[cfg(not(target_os = "wasi"))] +use std::fs::File; +use std::io::Write; +use std::path::PathBuf; + +use uucore::error::UResult; + +use crate::Line; +use crate::chunks::Chunk; +use crate::merge::WriteableTmpFile; + +#[cfg(not(wasi_no_threads))] mod threaded; -#[cfg(not(target_os = "wasi"))] +#[cfg(not(wasi_no_threads))] pub use threaded::ext_sort; -#[cfg(target_os = "wasi")] -mod wasi; -#[cfg(target_os = "wasi")] -// `self::` needed to disambiguate from the `wasi` crate -pub use self::wasi::ext_sort; +#[cfg(wasi_no_threads)] +mod sync; +#[cfg(wasi_no_threads)] +pub use sync::ext_sort; + +// Note: update `test_sort::test_start_buffer` if this size is changed +// Fixed to 8 KiB (equivalent to `std::sys::io::DEFAULT_BUF_SIZE` on most targets) +pub(super) const DEFAULT_BUF_SIZE: usize = 8 * 1024; + +/// Write the lines in `chunk` to `file`, separated by `separator`. +/// `compress_prog` is used to optionally compress file contents. +pub(super) fn write( + chunk: &Chunk, + file: (File, PathBuf), + compress_prog: Option<&str>, + separator: u8, +) -> UResult { + let mut tmp_file = I::create(file, compress_prog)?; + write_lines(chunk.lines(), tmp_file.as_write(), separator); + tmp_file.finished_writing() +} + +fn write_lines(lines: &[Line], writer: &mut T, separator: u8) { + for s in lines { + writer.write_all(s.line).unwrap(); + writer.write_all(&[separator]).unwrap(); + } +} diff --git a/src/uu/sort/src/ext_sort/sync.rs b/src/uu/sort/src/ext_sort/sync.rs new file mode 100644 index 00000000000..6487fb7f721 --- /dev/null +++ b/src/uu/sort/src/ext_sort/sync.rs @@ -0,0 +1,179 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +//! Synchronous external sort for targets without thread support +//! (e.g. `wasm32-wasip1`). +//! +//! Uses the same chunked sort-write-merge strategy as the threaded version, +//! but reads and sorts each chunk sequentially on the calling thread. + +use std::cmp::Ordering; +use std::io::{Read, Write, stderr}; + +use itertools::Itertools; +use uucore::error::UResult; + +use crate::Output; +use crate::chunks::{self, Chunk, RecycledChunk}; +use crate::merge::{self, WriteablePlainTmpFile, WriteableTmpFile}; +use crate::tmp_dir::TmpDirWrapper; +use crate::{GlobalSettings, compare_by, print_sorted, sort_by}; + +use super::{DEFAULT_BUF_SIZE, write}; + +pub fn ext_sort( + files: &mut impl Iterator>>, + settings: &GlobalSettings, + output: Output, + tmp_dir: &mut TmpDirWrapper, +) -> UResult<()> { + let separator = settings.line_ending.into(); + let mut buffer_size = match settings.buffer_size { + size if size <= 512 * 1024 * 1024 => size, + size => size / 2, + }; + if !settings.buffer_size_is_explicit { + buffer_size = buffer_size.max(8 * 1024 * 1024); + } + + if settings.compress_prog.is_some() { + let _ = writeln!( + stderr(), + "sort: warning: --compress-program is ignored on this platform" + ); + } + + let mut file = files.next().unwrap()?; + let mut carry_over = vec![]; + + // Read and sort first chunk. + let (first, cont) = chunks::read_to_chunk( + RecycledChunk::new(buffer_size.min(DEFAULT_BUF_SIZE)), + Some(buffer_size), + &mut carry_over, + &mut file, + files, + separator, + settings, + )?; + let Some(mut first) = first else { + return Ok(()); // empty input + }; + first.with_dependent_mut(|_, c| sort_by(&mut c.lines, settings, &c.line_data)); + + if !cont { + // All input fits in one chunk. + return print_chunk(&first, settings, output); + } + + // Read and sort second chunk. + let (second, cont) = chunks::read_to_chunk( + RecycledChunk::new(buffer_size.min(DEFAULT_BUF_SIZE)), + Some(buffer_size), + &mut carry_over, + &mut file, + files, + separator, + settings, + )?; + let Some(mut second) = second else { + return print_chunk(&first, settings, output); + }; + second.with_dependent_mut(|_, c| sort_by(&mut c.lines, settings, &c.line_data)); + + if !cont { + // All input fits in two chunks — merge in memory. + return print_two_chunks(first, second, settings, output); + } + + // More than two chunks: write sorted chunks to temp files, then merge. + let mut tmp_files: Vec<::Closed> = vec![]; + + tmp_files.push(write::( + &first, + tmp_dir.next_file()?, + settings.compress_prog.as_deref(), + separator, + )?); + drop(first); + + tmp_files.push(write::( + &second, + tmp_dir.next_file()?, + settings.compress_prog.as_deref(), + separator, + )?); + let mut recycled = second.recycle(); + + loop { + let (chunk, cont) = chunks::read_to_chunk( + recycled, + None, + &mut carry_over, + &mut file, + files, + separator, + settings, + )?; + let Some(mut chunk) = chunk else { break }; + chunk.with_dependent_mut(|_, c| sort_by(&mut c.lines, settings, &c.line_data)); + tmp_files.push(write::( + &chunk, + tmp_dir.next_file()?, + settings.compress_prog.as_deref(), + separator, + )?); + recycled = chunk.recycle(); + if !cont { + break; + } + } + + merge::merge_with_file_limit::<_, _, WriteablePlainTmpFile>( + tmp_files.into_iter().map(merge::ClosedTmpFile::reopen), + settings, + output, + tmp_dir, + ) +} + +/// Print a single sorted chunk. +fn print_chunk(chunk: &Chunk, settings: &GlobalSettings, output: Output) -> UResult<()> { + if settings.unique { + print_sorted( + chunk.lines().iter().dedup_by(|a, b| { + compare_by(a, b, settings, chunk.line_data(), chunk.line_data()) == Ordering::Equal + }), + settings, + output, + ) + } else { + print_sorted(chunk.lines().iter(), settings, output) + } +} + +/// Merge two in-memory chunks and print. +fn print_two_chunks(a: Chunk, b: Chunk, settings: &GlobalSettings, output: Output) -> UResult<()> { + let merged_iter = a.lines().iter().map(|line| (line, &a)).merge_by( + b.lines().iter().map(|line| (line, &b)), + |(line_a, a), (line_b, b)| { + compare_by(line_a, line_b, settings, a.line_data(), b.line_data()) != Ordering::Greater + }, + ); + if settings.unique { + print_sorted( + merged_iter + .dedup_by(|(line_a, a), (line_b, b)| { + compare_by(line_a, line_b, settings, a.line_data(), b.line_data()) + == Ordering::Equal + }) + .map(|(line, _)| line), + settings, + output, + ) + } else { + print_sorted(merged_iter.map(|(line, _)| line), settings, output) + } +} diff --git a/src/uu/sort/src/ext_sort/threaded.rs b/src/uu/sort/src/ext_sort/threaded.rs index 7dd089d0fe8..d0d0d02ddb8 100644 --- a/src/uu/sort/src/ext_sort/threaded.rs +++ b/src/uu/sort/src/ext_sort/threaded.rs @@ -7,9 +7,7 @@ //! thread, and spill to temporary files when memory is exceeded. use std::cmp::Ordering; -use std::fs::File; use std::io::{Read, Write, stderr}; -use std::path::PathBuf; use std::sync::mpsc::{Receiver, SyncSender}; use std::thread; @@ -17,20 +15,12 @@ use itertools::Itertools; use uucore::error::{UResult, strip_errno}; use crate::Output; -use crate::chunks::RecycledChunk; -use crate::merge::WriteableCompressedTmpFile; -use crate::merge::WriteablePlainTmpFile; -use crate::merge::WriteableTmpFile; +use crate::chunks::{self, Chunk, RecycledChunk}; +use crate::merge::{self, WriteableCompressedTmpFile, WriteablePlainTmpFile, WriteableTmpFile}; use crate::tmp_dir::TmpDirWrapper; -use crate::{ - GlobalSettings, Line, - chunks::{self, Chunk}, - compare_by, merge, print_sorted, sort_by, -}; +use crate::{GlobalSettings, compare_by, print_sorted, sort_by}; -// Note: update `test_sort::test_start_buffer` if this size is changed -// Fixed to 8 KiB (equivalent to `std::sys::io::DEFAULT_BUF_SIZE` on most targets) -const DEFAULT_BUF_SIZE: usize = 8 * 1024; +use super::{DEFAULT_BUF_SIZE, write}; /// Sort files by using auxiliary files for storing intermediate chunks (if needed), and output the result. /// @@ -283,23 +273,3 @@ fn read_write_loop( } } } - -/// Write the lines in `chunk` to `file`, separated by `separator`. -/// `compress_prog` is used to optionally compress file contents. -fn write( - chunk: &Chunk, - file: (File, PathBuf), - compress_prog: Option<&str>, - separator: u8, -) -> UResult { - let mut tmp_file = I::create(file, compress_prog)?; - write_lines(chunk.lines(), tmp_file.as_write(), separator); - tmp_file.finished_writing() -} - -fn write_lines(lines: &[Line], writer: &mut T, separator: u8) { - for s in lines { - writer.write_all(s.line).unwrap(); - writer.write_all(&[separator]).unwrap(); - } -} diff --git a/src/uu/sort/src/ext_sort/wasi.rs b/src/uu/sort/src/ext_sort/wasi.rs deleted file mode 100644 index 50bd5f63033..00000000000 --- a/src/uu/sort/src/ext_sort/wasi.rs +++ /dev/null @@ -1,59 +0,0 @@ -// This file is part of the uutils coreutils package. -// -// For the full copyright and license information, please view the LICENSE -// file that was distributed with this source code. - -//! WASI single-threaded sort: read all input into memory, sort, and output. -//! Threads are not available on WASI, so we bypass the chunked/threaded path. - -use std::cmp::Ordering; -use std::io::Read; - -use itertools::Itertools; -use uucore::error::UResult; - -use crate::Output; -use crate::chunks::{self, Chunk}; -use crate::tmp_dir::TmpDirWrapper; -use crate::{GlobalSettings, compare_by, print_sorted, sort_by}; - -/// Sort files by reading all input into memory, sorting in a single thread, and outputting directly. -pub fn ext_sort( - files: &mut impl Iterator>>, - settings: &GlobalSettings, - output: Output, - _tmp_dir: &mut TmpDirWrapper, -) -> UResult<()> { - let separator = settings.line_ending.into(); - // Read all input into memory at once. Unlike the threaded path which uses - // chunked buffered reads, WASI has no threads so we accept the memory cost. - // Note: there is no size limit here — WASI targets are expected to handle - // moderately sized inputs; very large files may cause OOM. - let mut input = Vec::new(); - for file in files { - file?.read_to_end(&mut input)?; - } - if input.is_empty() { - return Ok(()); - } - let mut chunk = Chunk::try_new(input, |buffer| { - Ok::<_, Box>(chunks::parse_into_chunk( - buffer, separator, settings, - )) - })?; - chunk.with_dependent_mut(|_, contents| { - sort_by(&mut contents.lines, settings, &contents.line_data); - }); - if settings.unique { - print_sorted( - chunk.lines().iter().dedup_by(|a, b| { - compare_by(a, b, settings, chunk.line_data(), chunk.line_data()) == Ordering::Equal - }), - settings, - output, - )?; - } else { - print_sorted(chunk.lines().iter(), settings, output)?; - } - Ok(()) -} diff --git a/src/uu/sort/src/merge.rs b/src/uu/sort/src/merge/mod.rs similarity index 52% rename from src/uu/sort/src/merge.rs rename to src/uu/sort/src/merge/mod.rs index 8f0b5bd54ae..6f8002a404a 100644 --- a/src/uu/sort/src/merge.rs +++ b/src/uu/sort/src/merge/mod.rs @@ -4,36 +4,37 @@ // file that was distributed with this source code. //! Merge already sorted files. //! -//! We achieve performance by splitting the tasks of sorting and writing, and reading and parsing between two threads. -//! The threads communicate over channels. There's one channel per file in the direction reader -> sorter, but only -//! one channel from the sorter back to the reader. The channels to the sorter are used to send the read chunks. -//! The sorter reads the next chunk from the channel whenever it needs the next chunk after running out of lines -//! from the previous read of the file. The channel back from the sorter to the reader has two purposes: To allow the reader -//! to reuse memory allocations and to tell the reader which file to read from next. +//! On most platforms this uses a multi-threaded reader/merger setup. On WASI +//! without atomics, a synchronous variant is used instead. The two +//! implementations live in sibling modules and are selected via cfg at the +//! module boundary. use std::{ - cmp::Ordering, ffi::{OsStr, OsString}, fs::{self, File}, io::{BufWriter, Read, Write}, - iter, path::{Path, PathBuf}, process::{Child, ChildStdin, ChildStdout, Command, Stdio}, rc::Rc, - sync::mpsc::{Receiver, Sender, SyncSender, TryRecvError, channel, sync_channel}, - thread::{self, JoinHandle}, }; -use compare::Compare; -use uucore::error::{FromIo, UResult}; +use uucore::error::UResult; use crate::{ - GlobalSettings, Output, SortError, - chunks::{self, Chunk, RecycledChunk}, - compare_by, current_open_fd_count, fd_soft_limit, open, + GlobalSettings, Output, SortError, chunks::Chunk, current_open_fd_count, fd_soft_limit, open, tmp_dir::TmpDirWrapper, }; +#[cfg(not(wasi_no_threads))] +mod threaded; +#[cfg(not(wasi_no_threads))] +use threaded as runner; + +#[cfg(wasi_no_threads)] +mod sync; +#[cfg(wasi_no_threads)] +use sync as runner; + /// If the output file occurs in the input files as well, copy the contents of the output file /// and replace its occurrences in the inputs with that copy. fn replace_output_file_in_input_files( @@ -104,6 +105,17 @@ pub fn merge( let files = files .iter() .map(|file| open(file).map(|file| PlainMergeInput { inner: file })); + + if !runner::SUPPORTS_COMPRESSION && settings.compress_prog.is_some() { + let _ = writeln!( + std::io::stderr(), + "sort: warning: --compress-program is ignored on this platform" + ); + return merge_with_file_limit::<_, _, WriteablePlainTmpFile>( + files, settings, output, tmp_dir, + ); + } + if settings.compress_prog.is_none() { merge_with_file_limit::<_, _, WriteablePlainTmpFile>(files, settings, output, tmp_dir) } else { @@ -111,6 +123,24 @@ pub fn merge( } } +/// Merge and write to output, dispatching to the active runner. +fn do_merge_to_output( + files: impl Iterator>, + settings: &GlobalSettings, + output: Output, +) -> UResult<()> { + runner::merge_without_limit(files, settings)?.write_all(settings, output) +} + +/// Merge and write to a writer, dispatching to the active runner. +fn do_merge_to_writer( + files: impl Iterator>, + settings: &GlobalSettings, + out: &mut impl Write, +) -> UResult<()> { + runner::merge_without_limit(files, settings)?.write_all_to(settings, out) +} + // Merge already sorted `MergeInput`s. pub fn merge_with_file_limit< M: MergeInput + 'static, @@ -126,8 +156,7 @@ pub fn merge_with_file_limit< debug_assert!(batch_size >= 2); if files.len() <= batch_size { - let merger = merge_without_limit(files, settings); - merger?.write_all(settings, output) + do_merge_to_output(files, settings, output) } else { let mut temporary_files = vec![]; let mut batch = Vec::with_capacity(batch_size); @@ -135,23 +164,21 @@ pub fn merge_with_file_limit< batch.push(file); if batch.len() >= batch_size { assert_eq!(batch.len(), batch_size); - let merger = merge_without_limit(batch.into_iter(), settings)?; - batch = Vec::with_capacity(batch_size); + let full_batch = std::mem::replace(&mut batch, Vec::with_capacity(batch_size)); let mut tmp_file = Tmp::create(tmp_dir.next_file()?, settings.compress_prog.as_deref())?; - merger.write_all_to(settings, tmp_file.as_write())?; + do_merge_to_writer(full_batch.into_iter(), settings, tmp_file.as_write())?; temporary_files.push(tmp_file.finished_writing()?); } } // Merge any remaining files that didn't get merged in a full batch above. if !batch.is_empty() { assert!(batch.len() < batch_size); - let merger = merge_without_limit(batch.into_iter(), settings)?; let mut tmp_file = Tmp::create(tmp_dir.next_file()?, settings.compress_prog.as_deref())?; - merger.write_all_to(settings, tmp_file.as_write())?; + do_merge_to_writer(batch.into_iter(), settings, tmp_file.as_write())?; temporary_files.push(tmp_file.finished_writing()?); } merge_with_file_limit::<_, _, Tmp>( @@ -168,271 +195,20 @@ pub fn merge_with_file_limit< } } -/// Merge files without limiting how many files are concurrently open. -/// -/// It is the responsibility of the caller to ensure that `files` yields only -/// as many files as we are allowed to open concurrently. -fn merge_without_limit>>( - files: F, - settings: &GlobalSettings, -) -> UResult> { - let (request_sender, request_receiver) = channel(); - let mut reader_files = Vec::with_capacity(files.size_hint().0); - let mut loaded_receivers = Vec::with_capacity(files.size_hint().0); - for (file_number, file) in files.enumerate() { - let (sender, receiver) = sync_channel(2); - loaded_receivers.push(receiver); - reader_files.push(Some(ReaderFile { - file: file?, - sender, - carry_over: vec![], - })); - // Send the initial chunk to trigger a read for each file - request_sender - .send((file_number, RecycledChunk::new(8 * 1024))) - .unwrap(); - } - - // Send the second chunk for each file - for file_number in 0..reader_files.len() { - request_sender - .send((file_number, RecycledChunk::new(8 * 1024))) - .unwrap(); - } - - let reader_join_handle = thread::spawn({ - let settings = settings.clone(); - move || { - reader( - &request_receiver, - &mut reader_files, - &settings, - settings.line_ending.into(), - ) - } - }); - - let mut mergeable_files = vec![]; - - for (file_number, receiver) in loaded_receivers.into_iter().enumerate() { - if let Ok(chunk) = receiver.recv() { - mergeable_files.push(MergeableFile { - current_chunk: Rc::new(chunk), - file_number, - line_idx: 0, - receiver, - }); - } - } - - Ok(FileMerger { - heap: binary_heap_plus::BinaryHeap::from_vec_cmp( - mergeable_files, - FileComparator { settings }, - ), - request_sender, - prev: None, - reader_join_handle, - }) -} -/// The struct on the reader thread representing an input file -struct ReaderFile { - file: M, - sender: SyncSender, - carry_over: Vec, -} - -/// The function running on the reader thread. -fn reader( - recycled_receiver: &Receiver<(usize, RecycledChunk)>, - files: &mut [Option>], - settings: &GlobalSettings, - separator: u8, -) -> UResult<()> { - for (file_idx, recycled_chunk) in recycled_receiver { - if let Some(ReaderFile { - file, - sender, - carry_over, - }) = &mut files[file_idx] - { - let should_continue = chunks::read( - sender, - recycled_chunk, - None, - carry_over, - file.as_read(), - &mut iter::empty(), - separator, - settings, - )?; - if !should_continue { - // Remove the file from the list by replacing it with `None`. - let ReaderFile { file, .. } = files[file_idx].take().unwrap(); - // Depending on the kind of the `MergeInput`, this may delete the file: - file.finished_reading()?; - } - } - } - Ok(()) -} -/// The struct on the main thread representing an input file -pub struct MergeableFile { - current_chunk: Rc, - line_idx: usize, - receiver: Receiver, - file_number: usize, -} - /// A struct to keep track of the previous line we encountered. /// /// This is required for deduplication purposes. -struct PreviousLine { - chunk: Rc, - line_idx: usize, - file_number: usize, -} - -/// Merges files together. This is **not** an iterator because of lifetime problems. -struct FileMerger<'a> { - heap: binary_heap_plus::BinaryHeap>, - request_sender: Sender<(usize, RecycledChunk)>, - prev: Option, - reader_join_handle: JoinHandle>, -} - -impl FileMerger<'_> { - /// Write the merged contents to the output file. - fn write_all(self, settings: &GlobalSettings, output: Output) -> UResult<()> { - let mut out = output.into_write(); - self.write_all_to(settings, &mut out) - } - - fn write_all_to(mut self, settings: &GlobalSettings, out: &mut impl Write) -> UResult<()> { - let write_result = loop { - match self - .write_next(out, settings) - .map_err_context(|| "write failed".into()) - { - Ok(true) => (), - Ok(false) => break Ok(()), - Err(error) => { - // Don't return yet: we still have to shut the reader thread down in an - // orderly fashion below. Returning here would drop our receivers while - // the reader is still sending, and `chunks::read` unwraps that send. - break Err(error); - } - } - }; - - let Self { - heap, - request_sender, - reader_join_handle, - .. - } = self; - - // Stop asking for chunks, so the reader runs out of work and returns. - drop(request_sender); - // Until it does, keep draining the files it might still be sending to. We have to - // poll all of them in turn rather than draining one at a time: the reader can be - // blocked on any one channel, and blocking on a different one would deadlock. - let mut files = heap.into_vec(); - while !files.is_empty() { - files.retain(|file| { - !matches!(file.receiver.try_recv(), Err(TryRecvError::Disconnected)) - }); - thread::yield_now(); - } - - let reader_result = reader_join_handle.join().unwrap(); - // A write failure is what the user needs to hear about; the reader hitting an error - // on the way down is secondary. - write_result.and(reader_result) - } - - fn write_next( - &mut self, - writer: &mut impl Write, - settings: &GlobalSettings, - ) -> std::io::Result { - if let Some(file) = self.heap.peek() { - let prev = self.prev.replace(PreviousLine { - chunk: file.current_chunk.clone(), - line_idx: file.line_idx, - file_number: file.file_number, - }); - - file.current_chunk.with_dependent(|_, contents| { - let current_line = &contents.lines[file.line_idx]; - if settings.unique - && let Some(prev) = &prev - { - let cmp = compare_by( - &prev.chunk.lines()[prev.line_idx], - current_line, - settings, - prev.chunk.line_data(), - file.current_chunk.line_data(), - ); - if cmp == Ordering::Equal { - return Ok(()); - } - } - current_line.write(writer, settings) - })?; - - let was_last_line_for_file = file.current_chunk.lines().len() == file.line_idx + 1; - - if was_last_line_for_file { - if let Ok(next_chunk) = file.receiver.recv() { - let mut file = self.heap.peek_mut().unwrap(); - file.current_chunk = Rc::new(next_chunk); - file.line_idx = 0; - } else { - self.heap.pop(); - } - } else { - // This will cause the comparison to use a different line and the heap to readjust. - self.heap.peek_mut().unwrap().line_idx += 1; - } - - if let Some(prev) = prev - && let Ok(prev_chunk) = Rc::try_unwrap(prev.chunk) - { - // If nothing is referencing the previous chunk anymore, this means that the previous line - // was the last line of the chunk. We can recycle the chunk. - self.request_sender - .send((prev.file_number, prev_chunk.recycle())) - .ok(); - } - } - Ok(!self.heap.is_empty()) - } +pub(super) struct PreviousLine { + pub chunk: Rc, + pub line_idx: usize, + // Only the threaded merger reads this back to recycle chunks. + #[cfg_attr(wasi_no_threads, allow(dead_code))] + pub file_number: usize, } /// Compares files by their current line. -struct FileComparator<'a> { - settings: &'a GlobalSettings, -} - -impl Compare for FileComparator<'_> { - fn compare(&self, a: &MergeableFile, b: &MergeableFile) -> Ordering { - let mut cmp = compare_by( - &a.current_chunk.lines()[a.line_idx], - &b.current_chunk.lines()[b.line_idx], - self.settings, - a.current_chunk.line_data(), - b.current_chunk.line_data(), - ); - if cmp == Ordering::Equal { - // To make sorting stable, we need to consider the file number as well, - // as lines from a file with a lower number are to be considered "earlier". - cmp = a.file_number.cmp(&b.file_number); - } - // BinaryHeap is a max heap. We use it as a min heap, so we need to reverse the ordering. - cmp.reverse() - } +pub(super) struct FileComparator<'a> { + pub settings: &'a GlobalSettings, } /// Wait for the child to exit and check its exit code. diff --git a/src/uu/sort/src/merge/sync.rs b/src/uu/sort/src/merge/sync.rs new file mode 100644 index 00000000000..badbff11289 --- /dev/null +++ b/src/uu/sort/src/merge/sync.rs @@ -0,0 +1,207 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +//! Synchronous merge for targets without thread support (e.g. wasm32-wasip1). +//! +//! Reads chunks on demand from each input on the calling thread instead of +//! using a dedicated reader thread. + +use std::{cmp::Ordering, io::Write, iter, rc::Rc}; + +use compare::Compare; +use uucore::error::UResult; + +use crate::{ + GlobalSettings, Output, + chunks::{self, Chunk, RecycledChunk}, + compare_by, +}; + +use super::{FileComparator, MergeInput, PreviousLine}; + +pub(super) const SUPPORTS_COMPRESSION: bool = false; + +struct SyncReaderFile { + file: M, + carry_over: Vec, +} + +struct SyncMergeableFile { + current_chunk: Rc, + line_idx: usize, + file_number: usize, +} + +impl Compare for FileComparator<'_> { + fn compare(&self, a: &SyncMergeableFile, b: &SyncMergeableFile) -> Ordering { + let mut cmp = compare_by( + &a.current_chunk.lines()[a.line_idx], + &b.current_chunk.lines()[b.line_idx], + self.settings, + a.current_chunk.line_data(), + b.current_chunk.line_data(), + ); + if cmp == Ordering::Equal { + cmp = a.file_number.cmp(&b.file_number); + } + cmp.reverse() + } +} + +pub(super) struct SyncFileMerger<'a, M: MergeInput> { + heap: binary_heap_plus::BinaryHeap>, + readers: Vec>>, + prev: Option, + recycled: Option, + settings: &'a GlobalSettings, +} + +impl SyncFileMerger<'_, M> { + pub(super) fn write_all(self, settings: &GlobalSettings, output: Output) -> UResult<()> { + let mut out = output.into_write(); + self.write_all_to(settings, &mut out) + } + + pub(super) fn write_all_to( + mut self, + settings: &GlobalSettings, + out: &mut impl Write, + ) -> UResult<()> { + while self.write_next(out, settings)? {} + for reader in self.readers.into_iter().flatten() { + reader.file.finished_reading()?; + } + Ok(()) + } + + fn write_next(&mut self, writer: &mut impl Write, settings: &GlobalSettings) -> UResult { + if let Some(file) = self.heap.peek() { + let prev = self.prev.replace(PreviousLine { + chunk: file.current_chunk.clone(), + line_idx: file.line_idx, + file_number: file.file_number, + }); + + file.current_chunk.with_dependent(|_, contents| { + let current_line = &contents.lines[file.line_idx]; + if settings.unique + && let Some(prev) = &prev + { + let cmp = compare_by( + &prev.chunk.lines()[prev.line_idx], + current_line, + settings, + prev.chunk.line_data(), + file.current_chunk.line_data(), + ); + if cmp == Ordering::Equal { + return Ok(()); + } + } + current_line.write(writer, settings) + })?; + + let was_last = file.current_chunk.lines().len() == file.line_idx + 1; + let file_number = file.file_number; + + if was_last { + let separator = self.settings.line_ending.into(); + let recycled = self + .recycled + .take() + .unwrap_or_else(|| RecycledChunk::new(8 * 1024)); + let next_chunk = if let Some(reader) = self.readers[file_number].as_mut() { + let (chunk, should_continue) = chunks::read_to_chunk( + recycled, + None, + &mut reader.carry_over, + reader.file.as_read(), + &mut iter::empty(), + separator, + self.settings, + )?; + if !should_continue && let Some(reader) = self.readers[file_number].take() { + reader.file.finished_reading()?; + } + chunk + } else { + None + }; + + if let Some(next_chunk) = next_chunk { + let mut file = self.heap.peek_mut().unwrap(); + file.current_chunk = Rc::new(next_chunk); + file.line_idx = 0; + } else { + self.heap.pop(); + } + } else { + self.heap.peek_mut().unwrap().line_idx += 1; + } + + // Recycle the previous chunk if no other reference holds it. + if let Some(prev) = prev + && let Ok(chunk) = Rc::try_unwrap(prev.chunk) + { + self.recycled = Some(chunk.recycle()); + } + } + Ok(!self.heap.is_empty()) + } +} + +pub(super) fn merge_without_limit>>( + files: F, + settings: &GlobalSettings, +) -> UResult> { + let separator = settings.line_ending.into(); + let mut readers: Vec>> = Vec::new(); + let mut mergeable_files = Vec::new(); + + for (file_number, file) in files.enumerate() { + let mut reader = SyncReaderFile { + file: file?, + carry_over: vec![], + }; + let recycled = RecycledChunk::new(8 * 1024); + let (chunk, should_continue) = chunks::read_to_chunk( + recycled, + None, + &mut reader.carry_over, + reader.file.as_read(), + &mut iter::empty(), + separator, + settings, + )?; + + if let Some(chunk) = chunk { + mergeable_files.push(SyncMergeableFile { + current_chunk: Rc::new(chunk), + line_idx: 0, + file_number, + }); + if should_continue { + readers.push(Some(reader)); + } else { + reader.file.finished_reading()?; + readers.push(None); + } + } else { + reader.file.finished_reading()?; + readers.push(None); + } + } + + Ok(SyncFileMerger { + heap: binary_heap_plus::BinaryHeap::from_vec_cmp( + mergeable_files, + FileComparator { settings }, + ), + readers, + prev: None, + recycled: None, + settings, + }) +} diff --git a/src/uu/sort/src/merge/threaded.rs b/src/uu/sort/src/merge/threaded.rs new file mode 100644 index 00000000000..ee5a68abf7b --- /dev/null +++ b/src/uu/sort/src/merge/threaded.rs @@ -0,0 +1,277 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +//! Multi-threaded merge: a reader thread feeds chunks into the per-file +//! channels while the main thread merges the next-line heap. + +use std::{ + cmp::Ordering, + io::Write, + iter, + rc::Rc, + sync::mpsc::{Receiver, Sender, SyncSender, TryRecvError, channel, sync_channel}, + thread::{self, JoinHandle}, +}; + +use compare::Compare; +use uucore::error::{FromIo, UResult}; + +use crate::{ + GlobalSettings, Output, + chunks::{self, Chunk, RecycledChunk}, + compare_by, +}; + +use super::{FileComparator, MergeInput, PreviousLine}; + +pub(super) const SUPPORTS_COMPRESSION: bool = true; + +/// Merge files without limiting how many files are concurrently open. +/// +/// It is the responsibility of the caller to ensure that `files` yields only +/// as many files as we are allowed to open concurrently. +pub(super) fn merge_without_limit>>( + files: F, + settings: &GlobalSettings, +) -> UResult> { + let (request_sender, request_receiver) = channel(); + let mut reader_files = Vec::with_capacity(files.size_hint().0); + let mut loaded_receivers = Vec::with_capacity(files.size_hint().0); + for (file_number, file) in files.enumerate() { + let (sender, receiver) = sync_channel(2); + loaded_receivers.push(receiver); + reader_files.push(Some(ReaderFile { + file: file?, + sender, + carry_over: vec![], + })); + // Send the initial chunk to trigger a read for each file + request_sender + .send((file_number, RecycledChunk::new(8 * 1024))) + .unwrap(); + } + + // Send the second chunk for each file + for file_number in 0..reader_files.len() { + request_sender + .send((file_number, RecycledChunk::new(8 * 1024))) + .unwrap(); + } + + let reader_join_handle = thread::spawn({ + let settings = settings.clone(); + move || { + reader( + &request_receiver, + &mut reader_files, + &settings, + settings.line_ending.into(), + ) + } + }); + + let mut mergeable_files = vec![]; + + for (file_number, receiver) in loaded_receivers.into_iter().enumerate() { + if let Ok(chunk) = receiver.recv() { + mergeable_files.push(MergeableFile { + current_chunk: Rc::new(chunk), + file_number, + line_idx: 0, + receiver, + }); + } + } + + Ok(FileMerger { + heap: binary_heap_plus::BinaryHeap::from_vec_cmp( + mergeable_files, + FileComparator { settings }, + ), + request_sender, + prev: None, + reader_join_handle, + }) +} + +/// The struct on the reader thread representing an input file +struct ReaderFile { + file: M, + sender: SyncSender, + carry_over: Vec, +} + +/// The function running on the reader thread. +fn reader( + recycled_receiver: &Receiver<(usize, RecycledChunk)>, + files: &mut [Option>], + settings: &GlobalSettings, + separator: u8, +) -> UResult<()> { + for (file_idx, recycled_chunk) in recycled_receiver { + if let Some(ReaderFile { + file, + sender, + carry_over, + }) = &mut files[file_idx] + { + let should_continue = chunks::read( + sender, + recycled_chunk, + None, + carry_over, + file.as_read(), + &mut iter::empty(), + separator, + settings, + )?; + if !should_continue { + // Remove the file from the list by replacing it with `None`. + let ReaderFile { file, .. } = files[file_idx].take().unwrap(); + // Depending on the kind of the `MergeInput`, this may delete the file: + file.finished_reading()?; + } + } + } + Ok(()) +} + +/// The struct on the main thread representing an input file +pub(super) struct MergeableFile { + current_chunk: Rc, + line_idx: usize, + receiver: Receiver, + file_number: usize, +} + +/// Merges files together. This is **not** an iterator because of lifetime problems. +pub(super) struct FileMerger<'a> { + heap: binary_heap_plus::BinaryHeap>, + request_sender: Sender<(usize, RecycledChunk)>, + prev: Option, + reader_join_handle: JoinHandle>, +} + +impl FileMerger<'_> { + /// Write the merged contents to the output file. + pub(super) fn write_all(self, settings: &GlobalSettings, output: Output) -> UResult<()> { + let mut out = output.into_write(); + self.write_all_to(settings, &mut out) + } + + pub(super) fn write_all_to( + mut self, + settings: &GlobalSettings, + out: &mut impl Write, + ) -> UResult<()> { + let write_result = loop { + match self + .write_next(out, settings) + .map_err_context(|| "write failed".into()) + { + Ok(true) => (), + Ok(false) => break Ok(()), + Err(error) => break Err(error), + } + }; + + let Self { + heap, + request_sender, + reader_join_handle, + .. + } = self; + + drop(request_sender); + let mut files = heap.into_vec(); + while !files.is_empty() { + files.retain(|file| { + !matches!(file.receiver.try_recv(), Err(TryRecvError::Disconnected)) + }); + thread::yield_now(); + } + + let reader_result = reader_join_handle.join().unwrap(); + write_result.and(reader_result) + } + + fn write_next( + &mut self, + writer: &mut impl Write, + settings: &GlobalSettings, + ) -> std::io::Result { + if let Some(file) = self.heap.peek() { + let prev = self.prev.replace(PreviousLine { + chunk: file.current_chunk.clone(), + line_idx: file.line_idx, + file_number: file.file_number, + }); + + file.current_chunk.with_dependent(|_, contents| { + let current_line = &contents.lines[file.line_idx]; + if settings.unique + && let Some(prev) = &prev + { + let cmp = compare_by( + &prev.chunk.lines()[prev.line_idx], + current_line, + settings, + prev.chunk.line_data(), + file.current_chunk.line_data(), + ); + if cmp == Ordering::Equal { + return Ok(()); + } + } + current_line.write(writer, settings) + })?; + + let was_last_line_for_file = file.current_chunk.lines().len() == file.line_idx + 1; + + if was_last_line_for_file { + if let Ok(next_chunk) = file.receiver.recv() { + let mut file = self.heap.peek_mut().unwrap(); + file.current_chunk = Rc::new(next_chunk); + file.line_idx = 0; + } else { + self.heap.pop(); + } + } else { + // This will cause the comparison to use a different line and the heap to readjust. + self.heap.peek_mut().unwrap().line_idx += 1; + } + + if let Some(prev) = prev + && let Ok(prev_chunk) = Rc::try_unwrap(prev.chunk) + { + // If nothing is referencing the previous chunk anymore, this means that the previous line + // was the last line of the chunk. We can recycle the chunk. + self.request_sender + .send((prev.file_number, prev_chunk.recycle())) + .ok(); + } + } + Ok(!self.heap.is_empty()) + } +} + +impl Compare for FileComparator<'_> { + fn compare(&self, a: &MergeableFile, b: &MergeableFile) -> Ordering { + let mut cmp = compare_by( + &a.current_chunk.lines()[a.line_idx], + &b.current_chunk.lines()[b.line_idx], + self.settings, + a.current_chunk.line_data(), + b.current_chunk.line_data(), + ); + if cmp == Ordering::Equal { + // To make sorting stable, we need to consider the file number as well, + // as lines from a file with a lower number are to be considered "earlier". + cmp = a.file_number.cmp(&b.file_number); + } + // BinaryHeap is a max heap. We use it as a min heap, so we need to reverse the ordering. + cmp.reverse() + } +} diff --git a/src/uu/sort/src/parallel.rs b/src/uu/sort/src/parallel.rs new file mode 100644 index 00000000000..4adb29ad432 --- /dev/null +++ b/src/uu/sort/src/parallel.rs @@ -0,0 +1,59 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +//! Parallel-or-sequential sort helpers and thread-pool initialization. +//! +//! On targets without thread support (`wasm32-wasip1` without atomics) these +//! fall back to the sequential `[T]::sort_*` methods and a no-op pool init. +//! On every other target they use rayon's parallel sorts. + +#[cfg(not(wasi_no_threads))] +mod imp { + use std::cmp::Ordering; + use std::num::NonZero; + + use rayon::slice::ParallelSliceMut; + + pub fn sort_by(slice: &mut [T], cmp: impl Fn(&T, &T) -> Ordering + Sync) { + slice.par_sort_by(cmp); + } + + pub fn sort_unstable_by(slice: &mut [T], cmp: impl Fn(&T, &T) -> Ordering + Sync) { + slice.par_sort_unstable_by(cmp); + } + + pub fn init_thread_pool(num_threads: Option) { + let num_threads = num_threads.map_or_else( + || std::thread::available_parallelism().map_or(1, NonZero::get), + |n| n as usize, + ); + let _ = rayon::ThreadPoolBuilder::new() + .num_threads(num_threads) + .build_global(); + } +} + +#[cfg(wasi_no_threads)] +mod imp { + use std::cmp::Ordering; + + // The `Send`/`Sync` bounds mirror the parallel implementation so that call + // sites compile identically on both targets. They are stricter than the + // underlying `[T]::sort_*` methods require, but every caller in this crate + // already satisfies them. + pub fn sort_by(slice: &mut [T], cmp: impl Fn(&T, &T) -> Ordering + Sync) { + slice.sort_by(cmp); + } + + pub fn sort_unstable_by(slice: &mut [T], cmp: impl Fn(&T, &T) -> Ordering + Sync) { + slice.sort_unstable_by(cmp); + } + + pub fn init_thread_pool(_num_threads: Option) { + // No-op: there is no thread pool on this target, so --parallel is ignored. + } +} + +pub use imp::{init_thread_pool, sort_by, sort_unstable_by}; diff --git a/src/uu/sort/src/sort.rs b/src/uu/sort/src/sort.rs index ef816006b09..74bc902e887 100644 --- a/src/uu/sort/src/sort.rs +++ b/src/uu/sort/src/sort.rs @@ -17,6 +17,7 @@ mod custom_str_cmp; mod ext_sort; mod merge; mod numeric_str_cmp; +mod parallel; mod tmp_dir; use bigdecimal::BigDecimal; @@ -29,8 +30,6 @@ use foldhash::fast::FoldHasher; use foldhash::{HashMap, SharedSeed}; use numeric_str_cmp::{NumInfo, NumInfoParseSettings, human_numeric_str_cmp, numeric_str_cmp}; use rand::{RngExt as _, rng}; -#[cfg(not(target_os = "wasi"))] -use rayon::slice::ParallelSliceMut; use std::cmp::Ordering; use std::env; use std::ffi::{OsStr, OsString}; @@ -2083,17 +2082,8 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { settings.ignore_non_printing = ignore_non_printing; settings.ignore_case = ignore_case; - // WASI doesn't support threads, so we ignore the corresponding option - #[cfg(not(target_os = "wasi"))] - { - let threads = matches - .get_one::(options::PARALLEL) - .copied() - .unwrap_or_else(|| std::thread::available_parallelism().map_or(1, |n| n.get() as u64)); - let _ = rayon::ThreadPoolBuilder::new() - .num_threads(threads as usize) - .build_global(); - } + // On targets without thread support this is a no-op and --parallel is ignored. + parallel::init_thread_pool(matches.get_one::(options::PARALLEL).copied()); if let Some(size_str) = matches.get_one::(options::BUF_SIZE) { settings.buffer_size = GlobalSettings::parse_byte_count(size_str).map_err(|e| { @@ -2105,22 +2095,22 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { settings.buffer_size_is_explicit = false; } - let mut tmp_dir = TmpDirWrapper::new(matches.get_one::(options::TMP_DIR).map_or_else( - || { - // WASI does not support std::env::temp_dir() — it panics with - // "no filesystem on wasm". Use /tmp as a nominal fallback; - // the WASI ext_sort path never actually creates temp files. - #[cfg(target_os = "wasi")] - { - PathBuf::from("/tmp") - } - #[cfg(not(target_os = "wasi"))] - { - env::temp_dir() - } - }, - PathBuf::from, - )); + let mut tmp_dir = TmpDirWrapper::new( + matches + .get_one::(options::TMP_DIR) + .map(PathBuf::from) + .or_else(|| env::var_os("TMPDIR").map(PathBuf::from)) + .unwrap_or_else(|| { + #[cfg(target_os = "wasi")] + { + uucore::fs::wasi_default_tmp_dir() + } + #[cfg(not(target_os = "wasi"))] + { + env::temp_dir() + } + }), + ); settings.compress_prog = matches .get_one::(options::COMPRESS_PROG) @@ -2573,18 +2563,10 @@ fn exec( fn sort_by<'a>(unsorted: &mut Vec>, settings: &GlobalSettings, line_data: &LineData<'a>) { let cmp = |a: &Line<'a>, b: &Line<'a>| compare_by(a, b, settings, line_data, line_data); - // WASI does not support threads, so use non-parallel sort to avoid - // rayon's thread pool which triggers an unreachable trap. if settings.stable || settings.unique { - #[cfg(not(target_os = "wasi"))] - unsorted.par_sort_by(cmp); - #[cfg(target_os = "wasi")] - unsorted.sort_by(cmp); + parallel::sort_by(unsorted, cmp); } else { - #[cfg(not(target_os = "wasi"))] - unsorted.par_sort_unstable_by(cmp); - #[cfg(target_os = "wasi")] - unsorted.sort_unstable_by(cmp); + parallel::sort_unstable_by(unsorted, cmp); } } diff --git a/src/uu/tail/src/platform/mod.rs b/src/uu/tail/src/platform/mod.rs index bb77501fdcb..9c3e8f6e819 100644 --- a/src/uu/tail/src/platform/mod.rs +++ b/src/uu/tail/src/platform/mod.rs @@ -16,7 +16,22 @@ pub use self::windows::{Pid, ProcessChecker, supports_pid_checks}; // WASI has no process management; provide stubs so tail compiles. #[cfg(target_os = "wasi")] -pub type Pid = u64; +pub type Pid = u32; + +#[cfg(target_os = "wasi")] +#[allow(dead_code)] +pub struct ProcessChecker; + +#[cfg(target_os = "wasi")] +#[allow(dead_code)] +impl ProcessChecker { + pub fn new(_pid: Pid) -> Self { + Self + } + pub fn is_dead(&self) -> bool { + true + } +} #[cfg(target_os = "wasi")] pub fn supports_pid_checks(_pid: Pid) -> bool { diff --git a/src/uu/touch/Cargo.toml b/src/uu/touch/Cargo.toml index be19f567c55..1eac51334ef 100644 --- a/src/uu/touch/Cargo.toml +++ b/src/uu/touch/Cargo.toml @@ -30,6 +30,8 @@ tempfile = { workspace = true } [target.'cfg(unix)'.dependencies] libc = { workspace = true } + +[target.'cfg(any(unix, target_os = "wasi"))'.dependencies] rustix = { workspace = true, features = ["fs"] } [target.'cfg(target_os = "windows")'.dependencies] diff --git a/src/uu/touch/src/error.rs b/src/uu/touch/src/error.rs index 47823cde93d..55e598d0efd 100644 --- a/src/uu/touch/src/error.rs +++ b/src/uu/touch/src/error.rs @@ -28,6 +28,10 @@ pub enum TouchError { #[error("{}", translate!("touch-error-windows-stdout-path-failed", "code" => .0.clone()))] WindowsStdoutPathError(String), + /// A feature that is not available on the current platform + #[error("{0}")] + UnsupportedPlatformFeature(String), + /// An error encountered on a specific file #[error("{error}")] TouchFileError { diff --git a/src/uu/touch/src/touch.rs b/src/uu/touch/src/touch.rs index fc9fcc57737..fdf1625304d 100644 --- a/src/uu/touch/src/touch.rs +++ b/src/uu/touch/src/touch.rs @@ -10,9 +10,11 @@ pub mod error; use clap::builder::{PossibleValue, ValueParser}; use clap::{Arg, ArgAction, ArgGroup, ArgMatches, Command}; -#[cfg(any(not(unix), target_os = "redox"))] +use filetime::FileTime; +#[cfg(all(any(not(unix), target_os = "redox"), not(target_os = "wasi")))] use filetime::set_file_times; -use filetime::{FileTime, set_symlink_file_times}; +#[cfg(not(target_os = "wasi"))] +use filetime::set_symlink_file_times; use jiff::civil::Time; use jiff::fmt::strtime; use jiff::tz::TimeZone; @@ -708,6 +710,47 @@ fn try_futimens_via_write_fd(path: &Path, atime: FileTime, mtime: FileTime) -> s futimens(&file, ×tamps).map_err(|e| Error::from_raw_os_error(e.raw_os_error())) } +/// WASI replacement for `filetime::set_file_times`. +/// +/// The `filetime` crate has an unimplemented stub on `wasm32-wasi`. WASI +/// supports setting both atime and mtime via `utimensat`, which we reach +/// through `rustix`. +#[cfg(target_os = "wasi")] +fn set_file_times(path: &Path, atime: FileTime, mtime: FileTime) -> std::io::Result<()> { + wasi_utimensat(path, atime, mtime, false) +} + +/// WASI replacement for `filetime::set_symlink_file_times`. +#[cfg(target_os = "wasi")] +fn set_symlink_file_times(path: &Path, atime: FileTime, mtime: FileTime) -> std::io::Result<()> { + wasi_utimensat(path, atime, mtime, true) +} + +#[cfg(target_os = "wasi")] +fn wasi_utimensat( + path: &Path, + atime: FileTime, + mtime: FileTime, + no_follow: bool, +) -> std::io::Result<()> { + let timestamps = rustix::fs::Timestamps { + last_access: rustix::fs::Timespec { + tv_sec: atime.unix_seconds(), + tv_nsec: atime.nanoseconds() as _, + }, + last_modification: rustix::fs::Timespec { + tv_sec: mtime.unix_seconds(), + tv_nsec: mtime.nanoseconds() as _, + }, + }; + let flags = if no_follow { + rustix::fs::AtFlags::SYMLINK_NOFOLLOW + } else { + rustix::fs::AtFlags::empty() + }; + rustix::fs::utimensat(rustix::fs::CWD, path, ×tamps, flags).map_err(Error::from) +} + /// Get metadata of the provided path /// If `follow` is `true`, the function will try to follow symlinks. Errors if the symlink is dangling, otherwise defaults to symlink metadata. /// If `follow` is `false`, the function will return metadata of the symlink itself @@ -725,6 +768,19 @@ fn stat(path: &Path, follow: bool) -> std::io::Result<(FileTime, FileTime)> { fs::symlink_metadata(path)? }; + // `FileTime::from_last_{access,modification}_time` is unimplemented on + // `wasm32-wasi`, so go through `Metadata::{accessed, modified}` (which + // return `SystemTime`) and convert via `FileTime::from_system_time`. + #[cfg(target_os = "wasi")] + { + let atime = metadata.accessed()?; + let mtime = metadata.modified()?; + Ok(( + FileTime::from_system_time(atime), + FileTime::from_system_time(mtime), + )) + } + #[cfg(not(target_os = "wasi"))] Ok(( FileTime::from_last_access_time(&metadata), FileTime::from_last_modification_time(&metadata), @@ -888,7 +944,10 @@ fn parse_timestamp(s: &str) -> UResult { /// /// On Windows, uses `GetFinalPathNameByHandleW` to attempt to get the path /// from the stdout handle. -#[cfg_attr(not(windows), expect(clippy::unnecessary_wraps))] +#[cfg_attr( + not(any(windows, target_os = "wasi")), + expect(clippy::unnecessary_wraps) +)] fn pathbuf_from_stdout() -> Result { #[cfg(all(unix, not(target_os = "android")))] { @@ -898,6 +957,10 @@ fn pathbuf_from_stdout() -> Result { { Ok(PathBuf::from("/proc/self/fd/1")) } + #[cfg(target_os = "wasi")] + return Err(TouchError::UnsupportedPlatformFeature( + "touch - (stdout) is not supported on WASI".to_string(), + )); #[cfg(windows)] { use std::os::windows::prelude::AsRawHandle; @@ -954,10 +1017,6 @@ fn pathbuf_from_stdout() -> Result { .map_err(|e| TouchError::WindowsStdoutPathError(e.to_string()))? .into()) } - #[cfg(target_os = "wasi")] - { - Ok(PathBuf::from("/dev/stdout")) - } } #[cfg(test)] diff --git a/src/uucore/src/lib/features/fs.rs b/src/uucore/src/lib/features/fs.rs index dbbeb5cbfd9..5ccd6c0f4ff 100644 --- a/src/uucore/src/lib/features/fs.rs +++ b/src/uucore/src/lib/features/fs.rs @@ -5,7 +5,7 @@ //! Set of functions to manage regular files, special files, and links. -// spell-checker:ignore backport Ioctl absolutized +// spell-checker:ignore backport Ioctl absolutized preopen #[cfg(all(unix, not(target_os = "redox")))] pub use libc::{major, makedev, minor}; @@ -234,6 +234,26 @@ pub enum ResolveMode { Logical, } +/// WASI fallback used when neither `--tmp-dir` nor `TMPDIR` is set and +/// `env::temp_dir()` would be inapplicable. +/// +/// The WASI sandbox only exposes explicitly preopened directories, and +/// `/tmp` is not one by default. This returns `/tmp` when a host preopen +/// has made it visible as a directory, and the current directory otherwise +/// — the current directory is always accessible under a preopen mapped +/// to `/`. +/// +/// Callers on WASI should prefer `--tmp-dir` and `TMPDIR` before falling +/// back to this helper. +#[cfg(target_os = "wasi")] +pub fn wasi_default_tmp_dir() -> PathBuf { + if fs::metadata("/tmp").is_ok_and(|m| m.is_dir()) { + PathBuf::from("/tmp") + } else { + PathBuf::from(".") + } +} + /// Normalize a path by removing relative information /// For example, convert 'bar/../foo/bar.txt' => 'foo/bar.txt' /// copied from `` diff --git a/tests/by-util/test_cat.rs b/tests/by-util/test_cat.rs index e0baf5095b0..1158078e9d1 100644 --- a/tests/by-util/test_cat.rs +++ b/tests/by-util/test_cat.rs @@ -2,7 +2,7 @@ // // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -// spell-checker:ignore NOFILE nonewline cmdline +// spell-checker:ignore NOFILE nonewline cmdline setrlimit ELOOP #[cfg(any(target_os = "linux", target_os = "android"))] use rlimit::Resource; @@ -87,6 +87,7 @@ fn test_no_options_big_input() { #[test] #[cfg(unix)] +#[cfg_attr(wasi_runner, ignore = "WASI: no FIFO/mkfifo support")] fn test_fifo_symlink() { use std::io::Write; use std::thread; @@ -121,6 +122,7 @@ fn test_fifo_symlink() { // TODO(#7542): Re-enable on Android once we figure out why setting limit is broken. // #[cfg(any(target_os = "linux", target_os = "android"))] #[cfg(target_os = "linux")] +#[cfg_attr(wasi_runner, ignore = "WASI: rlimit/setrlimit not supported")] fn test_closes_file_descriptors() { // Each file creates a pipe, which has two file descriptors. // If they are not closed then five is certainly too many. @@ -138,6 +140,7 @@ fn test_closes_file_descriptors() { #[test] #[cfg(unix)] +#[cfg_attr(wasi_runner, ignore = "WASI: no pipe/signal support")] fn test_broken_pipe() { let mut cmd = new_ucmd!(); let mut child = cmd @@ -514,6 +517,7 @@ fn test_squeeze_blank_before_numbering() { /// This tests reading from Unix character devices #[test] #[cfg(unix)] +#[cfg_attr(wasi_runner, ignore = "WASI sandbox: host paths not visible")] fn test_dev_random() { #[cfg(any(target_os = "linux", target_os = "android"))] const DEV_RANDOM: &str = "/dev/urandom"; @@ -544,6 +548,7 @@ fn test_dev_random() { /// Wikipedia says there is support on Linux, FreeBSD, and `NetBSD`. #[test] #[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "netbsd"))] +#[cfg_attr(wasi_runner, ignore = "WASI sandbox: host paths not visible")] fn test_dev_full() { let mut proc = new_ucmd!() .set_stdout(Stdio::piped()) @@ -559,6 +564,7 @@ fn test_dev_full() { #[test] #[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "netbsd"))] +#[cfg_attr(wasi_runner, ignore = "WASI sandbox: host paths not visible")] fn test_dev_full_show_all() { let buf_len = 2048; let mut proc = new_ucmd!() @@ -581,6 +587,7 @@ fn test_dev_full_show_all() { // without additional flush output gets reversed. #[test] #[cfg(target_os = "linux")] +#[cfg_attr(wasi_runner, ignore = "WASI sandbox: host paths not visible")] fn test_write_fast_fallthrough_uses_flush() { const PROC_INIT_CMDLINE: &str = "/proc/1/cmdline"; let cmdline = read_to_string(PROC_INIT_CMDLINE).unwrap(); @@ -593,6 +600,7 @@ fn test_write_fast_fallthrough_uses_flush() { #[test] #[cfg(unix)] +#[cfg_attr(wasi_runner, ignore = "WASI: no Unix domain socket support")] fn test_domain_socket() { use std::os::unix::net::UnixListener; @@ -607,6 +615,7 @@ fn test_domain_socket() { } #[test] +#[cfg_attr(wasi_runner, ignore = "WASI sandbox: host paths not visible")] fn test_write_to_self_empty() { // it's ok if the input file is also the output file if it's empty let s = TestScenario::new(util_name!()); @@ -622,6 +631,7 @@ fn test_write_to_self_empty() { } #[test] +#[cfg_attr(wasi_runner, ignore = "WASI: cannot detect unsafe overwrite")] fn test_write_to_self() { let s = TestScenario::new(util_name!()); let file_path = s.fixtures.plus("first_file"); @@ -678,6 +688,7 @@ fn test_successful_write_to_read_write_self() { /// /// `cat fx fx3 1<>fx3` #[test] +#[cfg_attr(wasi_runner, ignore = "WASI: cannot detect unsafe overwrite")] fn test_failed_write_to_read_write_self() { let (at, mut ucmd) = at_and_ucmd!(); at.write("fx", "g"); @@ -703,6 +714,10 @@ fn test_failed_write_to_read_write_self() { #[test] #[cfg(unix)] #[cfg(not(target_os = "openbsd"))] +#[cfg_attr( + wasi_runner, + ignore = "WASI: symlink loop traversal does not surface ELOOP ('Too many levels of symbolic links')" +)] fn test_error_loop() { let (at, mut ucmd) = at_and_ucmd!(); at.symlink_file("2", "1"); @@ -726,6 +741,7 @@ fn test_u_ignored() { #[test] #[cfg(unix)] +#[cfg_attr(wasi_runner, ignore = "WASI: errno/error-message mismatches")] fn test_write_fast_read_error() { use std::os::unix::fs::PermissionsExt; @@ -746,6 +762,7 @@ fn test_write_fast_read_error() { #[test] #[cfg(target_os = "linux")] +#[cfg_attr(wasi_runner, ignore = "WASI: argv/filenames must be valid UTF-8")] fn test_cat_non_utf8_paths() { use std::ffi::OsStr; use std::os::unix::ffi::OsStrExt; @@ -769,7 +786,8 @@ fn test_cat_non_utf8_paths() { } #[test] -#[cfg(unix)] +#[cfg(target_os = "linux")] +#[cfg_attr(wasi_runner, ignore = "WASI: cannot detect unsafe overwrite")] fn test_appending_same_input_output() { let (at, mut ucmd) = at_and_ucmd!(); diff --git a/tests/by-util/test_comm.rs b/tests/by-util/test_comm.rs index f0b7baed291..606446eff0c 100644 --- a/tests/by-util/test_comm.rs +++ b/tests/by-util/test_comm.rs @@ -450,6 +450,7 @@ fn test_is_dir() { } #[test] +#[cfg_attr(wasi_runner, ignore = "WASI: no locale data")] fn test_sorted() { let scene = TestScenario::new(util_name!()); let at = &scene.fixtures; @@ -481,6 +482,7 @@ fn test_sorted_check_order() { } #[test] +#[cfg_attr(wasi_runner, ignore = "WASI: no locale data")] fn test_both_inputs_out_of_order() { let scene = TestScenario::new(util_name!()); let at = &scene.fixtures; @@ -500,6 +502,7 @@ fn test_both_inputs_out_of_order() { } #[test] +#[cfg_attr(wasi_runner, ignore = "WASI: no locale data")] fn test_both_inputs_out_of_order_last_pair() { let scene = TestScenario::new(util_name!()); let at = &scene.fixtures; diff --git a/tests/by-util/test_sort.rs b/tests/by-util/test_sort.rs index a762ec2cf6e..6a325dfe1e0 100644 --- a/tests/by-util/test_sort.rs +++ b/tests/by-util/test_sort.rs @@ -3,7 +3,7 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -// spell-checker:ignore (words) ints (linux) NOFILE dfgi abmon avril +// spell-checker:ignore (words) ints (linux) NOFILE dfgi abmon avril setrlimit EISDIR #![allow(clippy::cast_possible_wrap)] use std::env; @@ -36,6 +36,7 @@ fn test_helper(file_name: &str, possible_args: &[&str]) { } #[test] +#[cfg_attr(wasi_runner, ignore = "WASI: sysinfo/meminfo not available")] fn test_buffer_sizes() { #[cfg(target_os = "linux")] let buffer_sizes = ["0", "50K", "50k", "1M", "100M", "0%", "10%"]; @@ -52,7 +53,10 @@ fn test_buffer_sizes() { .stdout_is_fixture("ext_sort.expected"); } - #[cfg(not(target_pointer_width = "32"))] + // The test runner compiles for the host (often 64-bit), but the binary + // under test may be 32-bit (e.g. wasm32-wasip1), which rejects very + // large buffer sizes. + #[cfg(all(not(target_pointer_width = "32"), not(wasi_runner)))] { let buffer_sizes = ["1000G", "10T"]; for buffer_size in &buffer_sizes { @@ -92,7 +96,9 @@ fn test_invalid_buffer_size() { // A percentage can fit in a u128 while its product with the total // physical memory does not; the parser must report it as too large // rather than panicking or silently wrapping. - #[cfg(target_os = "linux")] + // The test runner is built for Linux, but the binary under test may not + // expose Linux host memory information (e.g. wasm32-wasip1). + #[cfg(all(target_os = "linux", not(wasi_runner)))] new_ucmd!() .arg("-S") .arg("340282366920938463463374607431768211455%") @@ -678,6 +684,7 @@ fn month_sort_input_expected(months: &[String]) -> (String, String) { #[test] #[cfg(unix)] +#[cfg_attr(wasi_runner, ignore = "WASI: no locale data")] fn test_month_sort_french_locale() { let locale = "fr_FR.UTF-8"; if !is_locale_available(locale) { @@ -709,6 +716,7 @@ fn test_month_sort_french_locale() { #[test] #[cfg(unix)] +#[cfg_attr(wasi_runner, ignore = "WASI: no locale data")] fn test_month_sort_hungarian_locale() { let locale = "hu_HU.UTF-8"; if !is_locale_available(locale) { @@ -740,6 +748,7 @@ fn test_month_sort_hungarian_locale() { /// E.g. "av ril" should NOT match "avril" — GNU treats it as unknown. #[test] #[cfg(unix)] +#[cfg_attr(wasi_runner, ignore = "WASI: no locale data")] fn test_month_sort_french_embedded_blanks() { let locale = "fr_FR.UTF-8"; if !is_locale_available(locale) { @@ -792,6 +801,7 @@ fn test_month_sort_french_embedded_blanks() { #[test] #[cfg(unix)] +#[cfg_attr(wasi_runner, ignore = "WASI: no locale data")] fn test_month_sort_japanese_locale() { let locale = "ja_JP.UTF-8"; if !is_locale_available(locale) { @@ -1401,6 +1411,7 @@ fn sort_empty_chunk() { #[test] #[cfg(any(target_os = "linux", target_os = "android"))] +#[cfg_attr(wasi_runner, ignore = "WASI: no subprocess spawning")] fn test_compress() { new_ucmd!() .args(&[ @@ -1417,6 +1428,7 @@ fn test_compress() { #[test] #[cfg(any(target_os = "linux", target_os = "android"))] +#[cfg_attr(wasi_runner, ignore = "WASI: no subprocess spawning")] fn test_compress_merge() { new_ucmd!() .args(&[ @@ -1440,6 +1452,7 @@ fn test_compress_merge() { #[test] #[cfg(not(target_os = "android"))] +#[cfg_attr(wasi_runner, ignore = "WASI: no subprocess spawning")] fn test_compress_fail() { let result = new_ucmd!() .args(&[ @@ -1493,6 +1506,7 @@ fn test_batch_size_invalid() { } #[test] +#[cfg_attr(wasi_runner, ignore = "WASI: rlimit/setrlimit not supported")] fn test_batch_size_too_large() { let large_batch_size = "18446744073709551616"; new_ucmd!() @@ -1529,6 +1543,7 @@ fn test_merge_batch_size() { // TODO(#7542): Re-enable on Android once we figure out why setting limit is broken. // #[cfg(any(target_os = "linux", target_os = "android"))] #[cfg(target_os = "linux")] +#[cfg_attr(wasi_runner, ignore = "WASI: rlimit/setrlimit not supported")] fn test_merge_batch_size_with_limit() { use rlimit::Resource; // Currently need... @@ -1563,6 +1578,7 @@ fn test_sigpipe_panic() { #[test] #[cfg(unix)] +#[cfg_attr(wasi_runner, ignore = "WASI: no FIFO/mkfifo support")] fn test_fifo_without_trailing_newline() { let (at, mut ucmd) = at_and_ucmd!(); at.mkfifo("FIFO"); @@ -1648,6 +1664,7 @@ fn test_verifies_files_after_keys() { #[test] #[cfg(unix)] +#[cfg_attr(wasi_runner, ignore = "WASI sandbox: host paths not visible")] fn test_verifies_input_files() { new_ucmd!() .args(&["/dev/random", "nonexistent_file"]) @@ -1679,6 +1696,7 @@ fn test_output_is_input() { #[test] #[cfg(unix)] +#[cfg_attr(wasi_runner, ignore = "WASI sandbox: host paths not visible")] fn test_output_device() { new_ucmd!() .args(&["-o", "/dev/null"]) @@ -1712,6 +1730,7 @@ fn test_wrong_args_exit_code() { #[test] #[cfg(unix)] +#[cfg_attr(wasi_runner, ignore = "WASI: no pipe/signal support")] fn test_tmp_files_deleted_on_sigint() { use rand::{RngExt as _, SeedableRng, rngs::SmallRng}; use rustix::process::{Pid, Signal, kill_process}; @@ -1909,6 +1928,10 @@ fn test_files0_from_non_utf8_name() { #[test] #[cfg(unix)] +#[cfg_attr( + wasi_runner, + ignore = "WASI: opening a directory as a file reports EBADF instead of EISDIR" +)] fn test_files0_read_error() { new_ucmd!() .args(&["--files0-from", "."]) @@ -1918,6 +1941,7 @@ fn test_files0_read_error() { #[cfg(unix)] #[test] +#[cfg_attr(wasi_runner, ignore = "WASI sandbox: host paths not visible")] // Test for GNU tests/sort/sort-files0-from.pl "empty-non-regular" fn test_files0_from_empty_non_regular() { new_ucmd!() @@ -2005,6 +2029,7 @@ fn test_files0_from_2a() { #[test] // Test for GNU tests/sort/sort-files0-from.pl "non-utf8" #[cfg(all(unix, not(target_os = "macos")))] +#[cfg_attr(wasi_runner, ignore = "WASI: argv/filenames must be valid UTF-8")] fn test_files0_from_non_utf8() { use std::os::unix::ffi::OsStringExt; let (at, mut ucmd) = at_and_ucmd!(); @@ -2993,6 +3018,7 @@ fn test_locale_collation_utf8() { } #[test] +#[cfg_attr(wasi_runner, ignore = "WASI: no locale data")] fn test_locale_interleaved_en_us_utf8() { // Test case for issue: locale-based collation support // In en_US.UTF-8, lowercase and uppercase letters should interleave @@ -3065,6 +3091,7 @@ fn test_locale_with_ignore_case_flag() { } #[test] +#[cfg_attr(wasi_runner, ignore = "WASI: no locale data")] fn test_locale_complex_utf8_sorting() { // More complex test with mixed case and special characters // In en_US.UTF-8, should respect locale collation rules @@ -3089,6 +3116,7 @@ fn test_locale_posix_sort_debug_message() { } #[test] +#[cfg_attr(wasi_runner, ignore = "WASI: no locale data")] fn test_locale_utf8_sort_debug_message() { new_ucmd!() .env("LC_ALL", "en_US.UTF-8") @@ -3100,6 +3128,7 @@ fn test_locale_utf8_sort_debug_message() { #[test] #[cfg(unix)] +#[cfg_attr(wasi_runner, ignore = "WASI: no locale data")] fn test_failed_to_set_locale_debug_message() { let result = new_ucmd!() .env("LC_ALL", "not-valid-locale") @@ -3131,6 +3160,7 @@ e f 5436 down data path1 path2 path3 path4 path5\n"; } #[test] +#[cfg_attr(wasi_runner, ignore = "WASI: no locale data")] fn test_consistent_sorting_with_i18n_collate() { // Regression test for issue #11980 // Lexicographic fallback sorting for equal sorting keys for 01 and 0_1 diff --git a/tests/by-util/test_tail.rs b/tests/by-util/test_tail.rs index ff63f3ae69a..e900f5ce718 100644 --- a/tests/by-util/test_tail.rs +++ b/tests/by-util/test_tail.rs @@ -4,7 +4,7 @@ // file that was distributed with this source code. // spell-checker:ignore (ToDO) abcdefghijklmnopqrstuvwxyz efghijklmnopqrstuvwxyz vwxyz emptyfile file siette ocho nueve diez MULT -// spell-checker:ignore (libs) kqueue +// spell-checker:ignore (libs) kqueue ELOOP EISDIR // spell-checker:ignore (jargon) tailable untailable datasame runneradmin tmpi // spell-checker:ignore (cmd) taskkill #![allow( @@ -250,6 +250,8 @@ fn test_nc_0_wo_follow2() { } #[test] +#[cfg(not(target_os = "windows"))] +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_n0_with_follow() { let (at, mut ucmd) = at_and_ucmd!(); let test_file = "test.txt"; @@ -273,6 +275,7 @@ fn test_n0_with_follow() { // TODO: Add similar test for windows #[test] #[cfg(unix)] +#[cfg_attr(wasi_runner, ignore = "WASI: errno/error-message mismatches")] fn test_permission_denied() { use std::os::unix::fs::PermissionsExt; @@ -293,6 +296,7 @@ fn test_permission_denied() { // TODO: Add similar test for windows #[test] #[cfg(unix)] +#[cfg_attr(wasi_runner, ignore = "WASI: errno/error-message mismatches")] fn test_permission_denied_multiple() { use std::os::unix::fs::PermissionsExt; @@ -341,6 +345,7 @@ fn test_follow_redirect_stdin_name_retry() { not(target_os = "freebsd"), not(target_os = "openbsd") ))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: errno/error-message mismatches")] fn test_stdin_redirect_dir() { // $ mkdir dir // $ tail < dir, $ tail - < dir @@ -372,6 +377,10 @@ fn test_stdin_redirect_dir() { // `test_stdin_redirect_dir` #[test] #[cfg(target_vendor = "apple")] +#[cfg_attr( + wasi_runner, + ignore = "WASI: directory redirected into stdin reports ENOENT rather than EISDIR" +)] fn test_stdin_redirect_dir_when_target_os_is_macos() { // $ mkdir dir // $ tail < dir, $ tail - < dir @@ -523,6 +532,8 @@ fn test_null_default() { } #[test] +#[cfg(not(target_os = "windows"))] // FIXME: test times out +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_single() { let (at, mut ucmd) = at_and_ucmd!(); @@ -553,6 +564,7 @@ fn test_follow_single() { /// Test for following when bytes are written that are not valid UTF-8. #[test] #[cfg(not(target_os = "windows"))] // FIXME: test times out +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_non_utf8_bytes() { // Tail the test file and start following it. let (at, mut ucmd) = at_and_ucmd!(); @@ -610,6 +622,8 @@ fn test_permission_denied_is_not_reported_as_not_found() { } #[test] +#[cfg(not(target_os = "windows"))] // FIXME: test times out +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_multiple() { let (at, mut ucmd) = at_and_ucmd!(); let mut child = ucmd @@ -645,6 +659,8 @@ fn test_follow_multiple() { } #[test] +#[cfg(not(target_os = "windows"))] // FIXME: test times out +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_name_multiple() { // spell-checker:disable-next-line for argument in ["--follow=name", "--follo=nam", "--f=n"] { @@ -690,6 +706,7 @@ fn test_follow_name_multiple() { } #[test] +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_multiple_untailable() { // $ tail -f DIR1 DIR2 // ==> DIR1 <== @@ -731,6 +748,7 @@ fn test_follow_stdin_pipe() { #[test] #[cfg(not(target_os = "windows"))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_invalid_pid() { new_ucmd!() .args(&["-f", "--pid=-1234"]) @@ -762,6 +780,7 @@ fn test_follow_invalid_pid() { not(target_os = "android"), not(target_os = "freebsd") ))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_with_pid() { use std::process::Command; @@ -959,6 +978,7 @@ fn test_multiple_input_files_missing() { } #[test] +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_missing() { // Ensure that --follow=name does not imply --retry. // Ensure that --follow={descriptor,name} (without --retry) does *not wait* for the @@ -1027,6 +1047,7 @@ fn test_dir() { } #[test] +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_dir_follow() { let ts = TestScenario::new(util_name!()); let at = &ts.fixtures; @@ -1046,6 +1067,7 @@ fn test_dir_follow() { } #[test] +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_dir_follow_retry() { let ts = TestScenario::new(util_name!()); let at = &ts.fixtures; @@ -1308,6 +1330,7 @@ fn test_num_with_undocumented_sign_bytes() { #[test] #[cfg(unix)] +#[cfg_attr(wasi_runner, ignore = "WASI sandbox: host paths not visible")] fn test_bytes_for_funny_unix_files() { // inspired by: gnu/tests/tail-2/tail-c.sh let ts = TestScenario::new(util_name!()); @@ -1369,6 +1392,7 @@ fn test_retry2() { not(target_os = "freebsd"), not(target_os = "openbsd") ))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_retry3() { // inspired by: gnu/tests/tail-2/retry.sh // Ensure that `tail --retry --follow=name` waits for the file to appear. @@ -1414,6 +1438,7 @@ fn test_retry3() { not(target_os = "freebsd"), not(target_os = "openbsd") ))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_retry4() { // inspired by: gnu/tests/tail-2/retry.sh // Ensure that `tail --retry --follow=descriptor` waits for the file to appear. @@ -1472,6 +1497,7 @@ fn test_retry4() { not(target_os = "freebsd"), not(target_os = "openbsd") ))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_retry5() { // inspired by: gnu/tests/tail-2/retry.sh // Ensure that `tail --follow=descriptor --retry` exits when the file appears untailable. @@ -1513,6 +1539,7 @@ fn test_retry5() { // >X #[test] #[cfg(all(not(target_os = "windows"), not(target_os = "android")))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_retry6() { // inspired by: gnu/tests/tail-2/retry.sh // Ensure that --follow=descriptor (without --retry) does *not* try @@ -1562,6 +1589,7 @@ fn test_retry6() { not(target_os = "freebsd"), not(target_os = "openbsd") ))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_retry7() { // inspired by: gnu/tests/tail-2/retry.sh // Ensure that `tail -F` retries when the file is initially untailable. @@ -1635,6 +1663,7 @@ fn test_retry7() { #[test] #[cfg(unix)] #[cfg(not(target_os = "android"))] +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_name_replaced_by_symlink_is_untailable() { let ts = TestScenario::new(util_name!()); let at = &ts.fixtures; @@ -1693,6 +1722,7 @@ fn test_follow_name_replaced_by_symlink_is_untailable() { not(target_os = "freebsd"), not(target_os = "openbsd") ))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_retry8() { // Ensure that inotify will switch to polling mode if directory // of the watched file was initially missing and later created. @@ -1762,6 +1792,7 @@ fn test_retry8() { not(target_os = "freebsd"), not(target_os = "openbsd") ))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_retry9() { // inspired by: gnu/tests/tail-2/inotify-dir-recreate.sh // Ensure that inotify will switch to polling mode if directory @@ -1844,6 +1875,7 @@ fn test_retry9() { not(target_os = "freebsd"), not(target_os = "openbsd") ))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_descriptor_vs_rename1() { // inspired by: gnu/tests/tail-2/descriptor-vs-rename.sh // $ ((rm -f A && touch A && sleep 1 && echo -n "A\n" >> A && sleep 1 && \ @@ -1908,6 +1940,7 @@ fn test_follow_descriptor_vs_rename1() { not(target_os = "freebsd"), not(target_os = "openbsd") ))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_descriptor_vs_rename2() { // Ensure the headers are correct for --verbose. // NOTE: GNU's tail does not update the header from FILE_A to FILE_C after `mv FILE_A FILE_C` @@ -1961,6 +1994,7 @@ fn test_follow_descriptor_vs_rename2() { not(target_os = "freebsd"), not(target_os = "openbsd") ))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_name_retry_headers() { // inspired by: "gnu/tests/tail-2/F-headers.sh" // Ensure tail -F distinguishes output with the @@ -2027,6 +2061,7 @@ fn test_follow_name_retry_headers() { #[test] #[cfg(all(not(target_os = "windows"), not(target_os = "android")))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_name_remove() { // This test triggers a remove event while `tail --follow=name file` is running. // ((sleep 2 && rm file &)>/dev/null 2>&1 &) ; tail --follow=name file @@ -2086,7 +2121,12 @@ fn test_follow_name_remove() { } #[test] -#[cfg(all(not(target_os = "android"), not(target_os = "freebsd")))] // FIXME: for currently not working platforms +#[cfg(all( + not(target_os = "windows"), + not(target_os = "android"), + not(target_os = "freebsd") +))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_name_truncate1() { // This test triggers a truncate event while `tail --follow=name file` is running. // $ cp file backup && head file > file && sleep 1 && cp backup file @@ -2123,7 +2163,12 @@ fn test_follow_name_truncate1() { } #[test] -#[cfg(all(not(target_os = "android"), not(target_os = "freebsd")))] // FIXME: for currently not working platforms +#[cfg(all( + not(target_os = "windows"), + not(target_os = "android"), + not(target_os = "freebsd") +))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_name_truncate2() { // This test triggers a truncate event while `tail --follow=name file` is running. // $ ((sleep 1 && echo -n "x\nx\nx\n" >> file && sleep 1 && \ @@ -2166,6 +2211,8 @@ fn test_follow_name_truncate2() { } #[test] +#[cfg(not(target_os = "windows"))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_name_truncate3() { // Opening an empty file in truncate mode should not trigger a truncate event while // `tail --follow=name file` is running. @@ -2202,6 +2249,7 @@ fn test_follow_name_truncate3() { not(target_os = "windows"), not(feature = "feat_selinux") // flaky ))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_name_truncate4() { // Truncating a file with the same content it already has should not trigger a truncate event @@ -2237,6 +2285,7 @@ fn test_follow_name_truncate4() { #[test] #[cfg(not(target_os = "windows"))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_truncate_fast() { // inspired by: "gnu/tests/tail-2/truncate.sh" // Ensure all logs are output upon file truncation @@ -2291,6 +2340,7 @@ fn test_follow_truncate_fast() { not(target_os = "freebsd"), not(target_os = "openbsd") ))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_name_move_create1() { // This test triggers a move/create event while `tail --follow=name file` is running. // ((sleep 2 && mv file backup && sleep 2 && cp backup file &)>/dev/null 2>&1 &) ; tail --follow=name file @@ -2347,6 +2397,7 @@ fn test_follow_name_move_create1() { not(target_os = "freebsd"), not(target_os = "openbsd") ))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_name_move_create2() { // inspired by: "gnu/tests/tail-2/inotify-hash-abuse.sh" // Exercise an abort-inducing flaw in inotify-enabled tail -F @@ -2427,6 +2478,7 @@ fn test_follow_name_move_create2() { not(target_os = "freebsd"), not(target_os = "openbsd") ))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_name_move1() { // This test triggers a move event while `tail --follow=name file` is running. // ((sleep 2 && mv file backup &)>/dev/null 2>&1 &) ; tail --follow=name file @@ -2489,6 +2541,7 @@ fn test_follow_name_move1() { not(target_os = "freebsd"), not(target_os = "openbsd") ))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_name_move2() { // Like test_follow_name_move1, but move to a name that's already monitored. @@ -2577,6 +2630,7 @@ fn test_follow_name_move2() { not(target_os = "freebsd"), not(target_os = "openbsd") ))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_name_move_retry1() { // Similar to test_follow_name_move1 but with `--retry` (`-F`) // This test triggers two move/rename events while `tail --follow=name --retry file` is running. @@ -2637,6 +2691,7 @@ fn test_follow_name_move_retry1() { not(target_os = "freebsd"), not(target_os = "openbsd") ))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_name_move_retry2() { // inspired by: "gnu/tests/tail-2/F-vs-rename.sh" // Similar to test_follow_name_move2 (move to a name that's already monitored) @@ -2737,6 +2792,7 @@ fn test_follow_name_move_retry2() { #[test] #[cfg(not(target_os = "windows"))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_inotify_only_regular() { // The GNU test inotify-only-regular.sh uses strace to ensure that `tail -f` // doesn't make inotify syscalls and only uses inotify for regular files or fifos. @@ -2816,6 +2872,7 @@ fn test_fifo() { not(target_os = "freebsd"), not(target_os = "openbsd") ))] +#[cfg_attr(wasi_runner, ignore = "WASI: no FIFO/mkfifo support")] fn test_fifo_with_pid() { use std::process::{Command, Stdio}; @@ -3703,6 +3760,10 @@ fn test_when_argument_file_is_a_directory() { // TODO: make this work on windows #[test] #[cfg(unix)] +#[cfg_attr( + wasi_runner, + ignore = "WASI: symlink-to-directory error path differs from POSIX (ELOOP/EISDIR not reliably surfaced)" +)] fn test_when_argument_file_is_a_symlink() { let ts = TestScenario::new(util_name!()); let at = &ts.fixtures; @@ -3736,6 +3797,10 @@ fn test_when_argument_file_is_a_symlink() { // TODO: make this work on windows #[test] #[cfg(unix)] +#[cfg_attr( + wasi_runner, + ignore = "WASI: symlink-to-directory error path differs from POSIX (ELOOP/EISDIR not reliably surfaced)" +)] fn test_when_argument_file_is_a_symlink_to_directory_then_error() { let ts = TestScenario::new(util_name!()); let at = &ts.fixtures; @@ -4058,6 +4123,7 @@ fn test_when_follow_retry_then_initial_print_of_file_is_written_to_stdout() { // TODO: Add test for the warning `--pid=PID is not supported on this system` #[test] +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_args_when_settings_check_warnings_then_shows_warnings() { let scene = TestScenario::new(util_name!()); let at = &scene.fixtures; @@ -4126,6 +4192,7 @@ fn test_args_when_settings_check_warnings_then_shows_warnings() { /// TODO: Write similar tests for windows #[test] #[cfg(target_os = "linux")] +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_args_when_settings_check_warnings_follow_indefinitely_then_warning() { let scene = TestScenario::new(util_name!()); @@ -4250,6 +4317,7 @@ fn test_args_when_settings_check_warnings_follow_indefinitely_then_warning() { #[test] #[cfg(unix)] +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_args_when_settings_check_warnings_follow_indefinitely_then_no_warning() { let scene = TestScenario::new(util_name!()); let at = &scene.fixtures; @@ -4564,6 +4632,7 @@ fn test_follow_when_file_and_symlink_are_pointing_to_same_file_and_append_data() } #[test] +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_args_when_directory_given_shorthand_big_f_together_with_retry() { let scene = TestScenario::new(util_name!()); let at = &scene.fixtures; @@ -4624,6 +4693,7 @@ fn test_args_when_directory_given_shorthand_big_f_together_with_retry() { not(target_os = "openbsd"), not(feature = "feat_selinux") // flaky ))] +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_when_files_are_pointing_to_same_relative_file_and_file_stays_same_size() { let scene = TestScenario::new(util_name!()); let at = &scene.fixtures; @@ -4930,6 +5000,7 @@ fn test_gnu_args_err() { } #[test] +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_gnu_args_f() { let scene = TestScenario::new(util_name!()); let at = &scene.fixtures; @@ -4951,6 +5022,7 @@ fn test_gnu_args_f() { #[test] #[cfg(unix)] +#[cfg_attr(wasi_runner, ignore = "WASI: argv/filenames must be valid UTF-8")] fn test_obsolete_encoding_unix() { use std::ffi::OsStr; use std::os::unix::ffi::OsStrExt; @@ -4985,6 +5057,7 @@ fn test_obsolete_encoding_windows() { #[test] #[cfg(not(target_vendor = "apple"))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_following_with_pid() { use std::process::Command; @@ -5061,6 +5134,7 @@ fn test_when_piped_input_then_no_broken_pipe() { #[test] #[cfg(unix)] +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_when_output_closed_then_no_broken_pipe() { let mut cmd = new_ucmd!(); let mut child = cmd @@ -5116,7 +5190,8 @@ fn test_failed_write_is_reported_on_seekable_input() { } #[test] -#[cfg(unix)] +#[cfg(target_os = "linux")] +#[cfg_attr(wasi_runner, ignore = "WASI sandbox: host paths not visible")] fn test_dev_zero() { new_ucmd!() .args(&["-c", "1", "/dev/zero"]) @@ -5161,6 +5236,7 @@ fn test_follow_pipe_f() { #[test] #[cfg(target_os = "linux")] +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_stdout_pipe_close() { let (at, mut ucmd) = at_and_ucmd!(); at.write("f", "line1\nline2\n"); @@ -5176,6 +5252,7 @@ fn test_follow_stdout_pipe_close() { } #[test] +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_debug_flag_with_polling() { let ts = TestScenario::new(util_name!()); let at = &ts.fixtures; @@ -5196,6 +5273,7 @@ fn test_debug_flag_with_polling() { #[test] #[cfg(target_os = "linux")] +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_debug_flag_with_inotify() { let ts = TestScenario::new(util_name!()); let at = &ts.fixtures; @@ -5213,6 +5291,7 @@ fn test_debug_flag_with_inotify() { #[test] #[cfg(target_os = "linux")] +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_dangling_symlink() { let (at, mut ucmd) = at_and_ucmd!(); at.symlink_file("target", "link"); @@ -5227,6 +5306,7 @@ fn test_follow_dangling_symlink() { #[test] #[cfg(target_os = "linux")] +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_symlink_target_change() { let (at, mut ucmd) = at_and_ucmd!(); at.write("t1", "A\n"); diff --git a/tests/by-util/test_touch.rs b/tests/by-util/test_touch.rs index 2b8c0efc3d5..4136d8e686d 100644 --- a/tests/by-util/test_touch.rs +++ b/tests/by-util/test_touch.rs @@ -2,7 +2,7 @@ // // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -// spell-checker:ignore (formats) cymdhm cymdhms datetime mdhm mdhms mktime strtime ymdhm ymdhms +// spell-checker:ignore (formats) cymdhm cymdhms datetime filestat mdhm mdhms mktime preopen strtime tzdb ymdhm ymdhms use filetime::FileTime; #[cfg(not(target_os = "freebsd"))] @@ -166,6 +166,10 @@ fn test_touch_2_digit_years_2038() { } #[test] +#[cfg_attr( + wasi_runner, + ignore = "WASI: pre-epoch timestamps not representable by path_filestat_set_times" +)] fn test_touch_2_digit_years_69() { // 69 and after are 19xx let (at, mut ucmd) = at_and_ucmd!(); @@ -623,6 +627,10 @@ fn test_touch_set_date7() { } #[test] +#[cfg_attr( + wasi_runner, + ignore = "WASI: no tzdb; TZ env var is not honoured so timezone-dependent timestamps differ" +)] fn test_touch_set_date_without_leading_zeroes() { let (at, mut ucmd) = at_and_ucmd!(); let file = "test_touch_set_date_without_leading_zeroes"; @@ -645,7 +653,11 @@ fn test_touch_set_date_without_leading_zeroes() { /// expected by the old nix-based implementation. After switching to rustix /// (which uses i64 `tv_sec` natively), this should succeed on all targets. #[test] -#[cfg(unix)] +#[cfg(target_os = "linux")] +#[cfg_attr( + wasi_runner, + ignore = "WASI: pre-epoch timestamps not representable by path_filestat_set_times" +)] fn test_touch_set_date_year_zero() { let (at, mut ucmd) = at_and_ucmd!(); let file = "test_touch_year_zero"; @@ -780,6 +792,10 @@ fn test_touch_mtime_dst_succeeds() { #[test] #[cfg(unix)] +#[cfg_attr( + wasi_runner, + ignore = "WASI: no tzdb; TZ env var is not honoured so DST validation is skipped" +)] fn test_touch_mtime_dst_fails() { let file = "test_touch_set_mtime_dst_fails"; @@ -797,6 +813,10 @@ fn test_touch_mtime_dst_fails() { #[test] #[cfg(unix)] +#[cfg_attr( + wasi_runner, + ignore = "WASI: guest root is a writable preopen, not the protected system root" +)] fn test_touch_system_fails() { let file = "/"; new_ucmd!() @@ -874,6 +894,7 @@ fn test_touch_no_such_file_error_msg() { #[test] #[cfg(not(any(target_os = "freebsd", target_os = "openbsd")))] +#[cfg_attr(wasi_runner, ignore = "WASI: touch - (stdout) is unsupported")] fn test_touch_changes_time_of_file_in_stdout() { // command like: `touch - 1< ./c` // should change the timestamp of c @@ -896,6 +917,10 @@ fn test_touch_changes_time_of_file_in_stdout() { #[test] #[cfg(unix)] +#[cfg_attr( + wasi_runner, + ignore = "WASI: filesystem permission errors surface as ENOENT rather than EACCES" +)] fn test_touch_permission_denied_error_msg() { let (at, mut ucmd) = at_and_ucmd!(); @@ -1016,6 +1041,7 @@ fn test_touch_no_dereference_dangling() { #[test] #[cfg(not(target_os = "openbsd"))] +#[cfg_attr(wasi_runner, ignore = "WASI: touch - (stdout) is unsupported")] fn test_touch_dash() { new_ucmd!().args(&["-h", "-"]).succeeds().no_output(); } @@ -1124,6 +1150,7 @@ fn test_touch_f_option() { #[test] #[cfg(target_os = "linux")] +#[cfg_attr(wasi_runner, ignore = "WASI: argv/filenames must be valid UTF-8")] fn test_touch_non_utf8_paths() { use std::ffi::OsStr; use std::os::unix::ffi::OsStrExt; @@ -1140,6 +1167,7 @@ fn test_touch_non_utf8_paths() { #[test] #[cfg(target_os = "linux")] +#[cfg_attr(wasi_runner, ignore = "WASI sandbox: host paths not visible")] fn test_touch_device_files() { let (_, mut ucmd) = at_and_ucmd!(); ucmd.args(&["/dev/null", "/dev/zero", "/dev/full", "/dev/random"]) @@ -1155,6 +1183,10 @@ fn test_touch_device_files() { // check in util/check-safe-traversal.sh. #[test] #[cfg(unix)] +#[cfg_attr( + wasi_runner, + ignore = "WASI sandbox: absolute symlink targets cannot be followed" +)] fn test_touch_does_not_truncate_symlink_target() { use std::os::unix::fs::symlink; @@ -1170,6 +1202,10 @@ fn test_touch_does_not_truncate_symlink_target() { // Touching a dangling symlink creates its target as an empty file, like GNU. #[test] #[cfg(unix)] +#[cfg_attr( + wasi_runner, + ignore = "WASI sandbox: absolute symlink targets cannot be followed" +)] fn test_touch_through_dangling_symlink_creates_target() { use std::os::unix::fs::symlink; diff --git a/util/run-wasi-integration-tests-docker.sh b/util/run-wasi-integration-tests-docker.sh new file mode 100755 index 00000000000..2a31af7d70e --- /dev/null +++ b/util/run-wasi-integration-tests-docker.sh @@ -0,0 +1,165 @@ +#!/usr/bin/env bash + +# spell-checker:ignore wasip wasmtime UUTESTS rustup + +# Run the WASI integration-test selection from .github/workflows/wasi.yml in +# an Ubuntu 24.04 container. This includes Linux-only host-test paths that a +# native macOS run does not compile. + +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: run-wasi-integration-tests-docker.sh [--source PATH] + +Run the WASI integration tests selected by PATH/.github/workflows/wasi.yml. +PATH defaults to the repository containing this script. +EOF +} + +SOURCE_DIR="" +while (($# > 0)); do + case "$1" in + --source) + if (($# < 2)); then + echo "error: --source requires a path" >&2 + exit 2 + fi + SOURCE_DIR="$2" + shift 2 + ;; + --source=*) + SOURCE_DIR="${1#*=}" + shift + ;; + -h | --help) + usage + exit 0 + ;; + *) + echo "error: unknown argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +SCRIPT_DIR="$(CDPATH='' cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +if [[ -z "${SOURCE_DIR}" ]]; then + SOURCE_DIR="$(dirname -- "${SCRIPT_DIR}")" +elif [[ ! -d "${SOURCE_DIR}" ]]; then + echo "error: source directory does not exist: ${SOURCE_DIR}" >&2 + exit 2 +else + SOURCE_DIR="$(CDPATH='' cd -- "${SOURCE_DIR}" && pwd -P)" +fi + +if [[ ! -f "${SOURCE_DIR}/Cargo.toml" || ! -f "${SOURCE_DIR}/.github/workflows/wasi.yml" ]]; then + echo "error: source directory is not a coreutils working tree: ${SOURCE_DIR}" >&2 + exit 2 +fi + +command -v docker >/dev/null 2>&1 || { + echo "error: docker not found in PATH" >&2 + exit 1 +} +docker info >/dev/null 2>&1 || { + echo "error: docker daemon not reachable" >&2 + exit 1 +} + +HOST_LOG_DIR="$(mktemp -d "${TMPDIR:-/tmp}/wasi-coreutils.XXXXXX")" +HOST_LOG="${HOST_LOG_DIR}/wasi-integration-test-output.log" + +# Report the log location on every exit path, including Docker failures. +trap 'echo; echo "Full log saved to ${HOST_LOG}"' EXIT + +# Toolchains and build artifacts use named volumes so repeat runs only fetch +# updates. The source remains read-only and is copied into the container. +docker run --rm -i \ + --volume "${SOURCE_DIR}:/src:ro" \ + --volume "${HOST_LOG_DIR}:/host-tmp" \ + --volume uutils-coreutils-wasi-cargo:/root/.cargo \ + --volume uutils-coreutils-wasi-rustup:/root/.rustup \ + --volume uutils-coreutils-wasi-wasmtime:/root/.wasmtime \ + --volume uutils-coreutils-wasi-target:/target \ + ubuntu:24.04 bash -se <<'EOF' +set -euo pipefail +export DEBIAN_FRONTEND=noninteractive + +apt-get update -qq +apt-get install -y -qq curl rsync ca-certificates build-essential pkg-config libssl-dev xz-utils >/dev/null + +if [[ ! -x /root/.cargo/bin/rustup ]]; then + curl -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain none --profile minimal >/dev/null +fi +. /root/.cargo/env +rustup toolchain install stable --profile minimal --target wasm32-wasip1 >/dev/null +rustup default stable >/dev/null + +curl -sSf https://wasmtime.dev/install.sh | bash >/dev/null +export PATH="/root/.wasmtime/bin:$PATH" + +mkdir -p /work +rsync -a --exclude=target --exclude='target-*' --exclude=.git /src/ /work/ +cd /work +export CARGO_TARGET_DIR=/target + +mapfile -t TEST_SELECTORS < <( + awk ' + /cargo test --test tests --/ { + capture = 1 + next + } + capture { + for (i = 1; i <= NF; i++) { + token = $i + sub(/\\$/, "", token) + if (token ~ /^test_[[:alnum:]_:]+$/) { + print token + } + } + if ($0 !~ /\\[[:space:]]*$/) { + exit + } + } + ' .github/workflows/wasi.yml +) + +if (("${#TEST_SELECTORS[@]}" == 0)); then + echo "error: no WASI integration-test selectors found in .github/workflows/wasi.yml" >&2 + exit 1 +fi + +LOG=/host-tmp/wasi-integration-test-output.log +: > "${LOG}" + +{ + echo "=== Tool versions ===" + rustc --version + cargo --version + wasmtime --version + printf 'Integration-test selectors (%d):' "${#TEST_SELECTORS[@]}" + printf ' %s' "${TEST_SELECTORS[@]}" + echo + + echo "=== Building WASI binary ===" + RUSTFLAGS="--cfg wasi_runner" \ + cargo build --locked --target wasm32-wasip1 --no-default-features --features feat_wasm +} 2>&1 | tee -a "${LOG}" + +# Preserve the test exit status while still writing the failure summary. +echo "=== Running WASI integration tests ===" | tee -a "${LOG}" +set +e +RUSTFLAGS="--cfg wasi_runner" \ +UUTESTS_BINARY_PATH="${CARGO_TARGET_DIR}/wasm32-wasip1/debug/coreutils.wasm" \ +UUTESTS_WASM_RUNNER=wasmtime \ + cargo test --locked --test tests -- "${TEST_SELECTORS[@]}" 2>&1 | tee -a "${LOG}" +test_status=${PIPESTATUS[0]} +set -e + +echo | tee -a "${LOG}" +echo "=== Failure summary ===" | tee -a "${LOG}" +grep -E "FAILED|^failures:|test result" "${LOG}" || echo "No failure or test-result lines found" +exit "${test_status}" +EOF