From c98baa674b7891cc4d756e65428c3140d57a439a Mon Sep 17 00:00:00 2001 From: Anthony DePasquale Date: Mon, 30 Mar 2026 00:37:03 +0200 Subject: [PATCH 01/17] Add wasm32-wasi platform support Add #[cfg(target_os = "wasi")] blocks alongside existing unix and windows platform code. No changes to existing platform behavior. Enables compilation to wasm32-wasip1 and wasm32-wasip1-threads targets for running in WASI-compatible runtimes like WasmKit and Wasmer. --- src/uu/cp/src/cp.rs | 96 ++++++++++++++++++------------- src/uu/sort/src/sort.rs | 27 ++++----- src/uu/tail/src/platform/mod.rs | 17 +++++- src/uu/touch/src/error.rs | 4 ++ src/uu/touch/src/touch.rs | 10 ++-- src/uucore/src/lib/features/fs.rs | 15 +++++ src/uucore/src/lib/lib.rs | 1 + 7 files changed, 109 insertions(+), 61 deletions(-) diff --git a/src/uu/cp/src/cp.rs b/src/uu/cp/src/cp.rs index b8062b8dafe..b61420c7dd7 100644 --- a/src/uu/cp/src/cp.rs +++ b/src/uu/cp/src/cp.rs @@ -2,6 +2,7 @@ // // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. +#![cfg_attr(target_os = "wasi", feature(wasi_ext))] // spell-checker:ignore (ToDO) copydir fiemap ftruncate linkgs lstat nlink nlinks pathbuf pwrite reflink strs xattrs symlinked deduplicated advcpmv nushell IRWXG IRWXO IRWXU IRWXUGO IRWXU IRWXG IRWXO IRWXUGO sflag // spell-checker:ignore RDONLY futimens utimensat @@ -21,6 +22,7 @@ use uucore::fsxattr::{copy_acls, copy_xattrs, copy_xattrs_skip_selinux}; use uucore::translate; use clap::{Arg, ArgAction, ArgMatches, Command, builder::ValueParser, value_parser}; +#[cfg(not(target_os = "wasi"))] use filetime::FileTime; use indicatif::{ProgressBar, ProgressStyle}; #[cfg(unix)] @@ -883,7 +885,12 @@ impl Attributes { #[cfg(unix)] ownership: Preserve::Yes { required: true }, mode: Preserve::Yes { required: true }, + // WASI: filetime panics in from_last_{access,modification}_time, + // so timestamps cannot be preserved. Mark as optional so -a works. + #[cfg(not(target_os = "wasi"))] timestamps: Preserve::Yes { required: true }, + #[cfg(target_os = "wasi")] + timestamps: Preserve::Yes { required: false }, context: { #[cfg(all(feature = "selinux", any(target_os = "linux", target_os = "android")))] { @@ -1889,31 +1896,40 @@ pub(crate) fn copy_attributes( })?; handle_preserve(attributes.timestamps, || -> CopyResult<()> { - let atime = FileTime::from_last_access_time(&source_metadata); - let mtime = FileTime::from_last_modification_time(&source_metadata); - // `set_file_times` opens the destination (O_RDONLY) before calling - // futimens; opening a FIFO or device with no peer blocks forever, and a - // socket cannot be opened at all. For symlinks and these special files - // use the path-based, no-follow variant, which sets the times via - // utimensat without opening. - #[cfg(unix)] - let no_open = { - let ft = source_metadata.file_type(); - dest.is_symlink() - || ft.is_fifo() - || ft.is_socket() - || ft.is_char_device() - || ft.is_block_device() - }; - #[cfg(not(unix))] - let no_open = dest.is_symlink(); - if no_open { - filetime::set_symlink_file_times(dest, atime, mtime)?; - } else { - filetime::set_file_times(dest, atime, mtime)?; - } + // filetime's WASI backend panics in from_last_{access,modification}_time, + // so return ENOTSUP. handle_preserve silently suppresses ENOTSUP for + // optional preservation (-a) and reports it for required (--preserve=timestamps). + #[cfg(target_os = "wasi")] + return Err(io::Error::from_raw_os_error(95).into()); // 95 = EOPNOTSUPP - Ok(()) + #[cfg(not(target_os = "wasi"))] + { + let atime = FileTime::from_last_access_time(&source_metadata); + let mtime = FileTime::from_last_modification_time(&source_metadata); + // `set_file_times` opens the destination (O_RDONLY) before calling + // futimens; opening a FIFO or device with no peer blocks forever, and a + // socket cannot be opened at all. For symlinks and these special files + // use the path-based, no-follow variant, which sets the times via + // utimensat without opening. + #[cfg(unix)] + let no_open = { + let ft = source_metadata.file_type(); + dest.is_symlink() + || ft.is_fifo() + || ft.is_socket() + || ft.is_char_device() + || ft.is_block_device() + }; + #[cfg(not(unix))] + let no_open = dest.is_symlink(); + if no_open { + filetime::set_symlink_file_times(dest, atime, mtime)?; + } else { + filetime::set_file_times(dest, atime, mtime)?; + } + + Ok(()) + } })?; #[cfg(all(feature = "selinux", any(target_os = "linux", target_os = "android")))] @@ -1959,19 +1975,9 @@ pub(crate) fn copy_attributes( fn symlink_file( source: &Path, dest: &Path, - #[cfg(not(target_os = "wasi"))] symlinked_files: &mut HashSet, - #[cfg(target_os = "wasi")] _symlinked_files: &mut HashSet, + symlinked_files: &mut HashSet, ) -> CopyResult<()> { - #[cfg(target_os = "wasi")] - { - Err(CpError::IoErrContext( - io::Error::new(io::ErrorKind::Unsupported, "symlinks not supported"), - translate!("cp-error-cannot-create-symlink", - "dest" => get_filename(dest).unwrap_or("?").quote(), - "source" => get_filename(source).unwrap_or("?").quote()), - )) - } - #[cfg(not(any(windows, target_os = "wasi")))] + #[cfg(unix)] { std::os::unix::fs::symlink(source, dest).map_err(|e| { CpError::IoErrContext( @@ -1993,13 +1999,21 @@ fn symlink_file( ) })?; } - #[cfg(not(target_os = "wasi"))] + #[cfg(target_os = "wasi")] { - if let Ok(file_info) = FileInformation::from_path(dest, false) { - symlinked_files.insert(file_info); - } - Ok(()) + std::os::wasi::fs::symlink_path(source, dest).map_err(|e| { + CpError::IoErrContext( + e, + translate!("cp-error-cannot-create-symlink", + "dest" => get_filename(dest).unwrap_or("?").quote(), + "source" => get_filename(source).unwrap_or("?").quote()), + ) + })?; } + if let Ok(file_info) = FileInformation::from_path(dest, false) { + symlinked_files.insert(file_info); + } + Ok(()) } fn context_for(src: &Path, dest: &Path) -> String { diff --git a/src/uu/sort/src/sort.rs b/src/uu/sort/src/sort.rs index ef816006b09..63ecb0dc736 100644 --- a/src/uu/sort/src/sort.rs +++ b/src/uu/sort/src/sort.rs @@ -2105,22 +2105,19 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { settings.buffer_size_is_explicit = false; } - let mut tmp_dir = TmpDirWrapper::new(matches.get_one::(options::TMP_DIR).map_or_else( - || { - // WASI does not support std::env::temp_dir() — it panics with - // "no filesystem on wasm". Use /tmp as a nominal fallback; - // the WASI ext_sort path never actually creates temp files. - #[cfg(target_os = "wasi")] - { - PathBuf::from("/tmp") - } - #[cfg(not(target_os = "wasi"))] - { + let mut tmp_dir = TmpDirWrapper::new( + matches + .get_one::(options::TMP_DIR) + .map(PathBuf::from) + .or_else(|| env::var_os("TMPDIR").map(PathBuf::from)) + .unwrap_or_else(|| { + // env::temp_dir() panics on WASI; default to /tmp + #[cfg(target_os = "wasi")] + return PathBuf::from("/tmp"); + #[cfg(not(target_os = "wasi"))] env::temp_dir() - } - }, - PathBuf::from, - )); + }), + ); settings.compress_prog = matches .get_one::(options::COMPRESS_PROG) diff --git a/src/uu/tail/src/platform/mod.rs b/src/uu/tail/src/platform/mod.rs index bb77501fdcb..9c3e8f6e819 100644 --- a/src/uu/tail/src/platform/mod.rs +++ b/src/uu/tail/src/platform/mod.rs @@ -16,7 +16,22 @@ pub use self::windows::{Pid, ProcessChecker, supports_pid_checks}; // WASI has no process management; provide stubs so tail compiles. #[cfg(target_os = "wasi")] -pub type Pid = u64; +pub type Pid = u32; + +#[cfg(target_os = "wasi")] +#[allow(dead_code)] +pub struct ProcessChecker; + +#[cfg(target_os = "wasi")] +#[allow(dead_code)] +impl ProcessChecker { + pub fn new(_pid: Pid) -> Self { + Self + } + pub fn is_dead(&self) -> bool { + true + } +} #[cfg(target_os = "wasi")] pub fn supports_pid_checks(_pid: Pid) -> bool { diff --git a/src/uu/touch/src/error.rs b/src/uu/touch/src/error.rs index 47823cde93d..55e598d0efd 100644 --- a/src/uu/touch/src/error.rs +++ b/src/uu/touch/src/error.rs @@ -28,6 +28,10 @@ pub enum TouchError { #[error("{}", translate!("touch-error-windows-stdout-path-failed", "code" => .0.clone()))] WindowsStdoutPathError(String), + /// A feature that is not available on the current platform + #[error("{0}")] + UnsupportedPlatformFeature(String), + /// An error encountered on a specific file #[error("{error}")] TouchFileError { diff --git a/src/uu/touch/src/touch.rs b/src/uu/touch/src/touch.rs index fc9fcc57737..485ec0b6696 100644 --- a/src/uu/touch/src/touch.rs +++ b/src/uu/touch/src/touch.rs @@ -898,6 +898,12 @@ fn pathbuf_from_stdout() -> Result { { Ok(PathBuf::from("/proc/self/fd/1")) } + #[cfg(target_os = "wasi")] + { + return Err(TouchError::UnsupportedPlatformFeature( + "touch - (stdout) is not supported on WASI".to_string(), + )); + } #[cfg(windows)] { use std::os::windows::prelude::AsRawHandle; @@ -954,10 +960,6 @@ fn pathbuf_from_stdout() -> Result { .map_err(|e| TouchError::WindowsStdoutPathError(e.to_string()))? .into()) } - #[cfg(target_os = "wasi")] - { - Ok(PathBuf::from("/dev/stdout")) - } } #[cfg(test)] diff --git a/src/uucore/src/lib/features/fs.rs b/src/uucore/src/lib/features/fs.rs index dbbeb5cbfd9..264d0abf4fd 100644 --- a/src/uucore/src/lib/features/fs.rs +++ b/src/uucore/src/lib/features/fs.rs @@ -71,6 +71,13 @@ impl FileInformation { Ok(Self(info)) } + /// Get information from a currently open file + #[cfg(target_os = "wasi")] + pub fn from_file(file: &fs::File) -> IOResult { + let meta = file.metadata()?; + Ok(Self(meta)) + } + /// Get information for a given path. /// /// If `path` points to a symlink and `dereference` is true, information about @@ -191,6 +198,14 @@ impl PartialEq for FileInformation { } } +#[cfg(target_os = "wasi")] +impl PartialEq for FileInformation { + fn eq(&self, other: &Self) -> bool { + use std::os::wasi::fs::MetadataExt; + self.0.dev() == other.0.dev() && self.0.ino() == other.0.ino() + } +} + impl Eq for FileInformation {} impl Hash for FileInformation { diff --git a/src/uucore/src/lib/lib.rs b/src/uucore/src/lib/lib.rs index aeacfe01fc6..a693ad869ca 100644 --- a/src/uucore/src/lib/lib.rs +++ b/src/uucore/src/lib/lib.rs @@ -2,6 +2,7 @@ // // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. +#![cfg_attr(all(target_os = "wasi", feature = "fs"), feature(wasi_ext))] //! library ~ (core/bundler file) // #![deny(missing_docs)] //TODO: enable this // From 17fbd2644cef86b43ad430b0250c0cafd5191c1c Mon Sep 17 00:00:00 2001 From: Anthony DePasquale Date: Mon, 30 Mar 2026 16:32:13 +0200 Subject: [PATCH 02/17] sort: add synchronous fallback for WASI without thread support On wasm32-wasip1 (no atomics), sort crashes because ext_sort, merge, check, and rayon all spawn threads unconditionally. Add synchronous code paths gated on cfg(all(target_os = "wasi", not(target_feature = "atomics"))) so sort works on both wasip1 (sync) and wasip1-threads (threaded). Key changes: - Extract read_to_chunk() from chunks::read() for shared use - Add synchronous ext_sort with chunked sort-write-merge flow - Add SyncFileMerger for threadless merge operations - Add synchronous check for order verification - Gate rayon par_sort with sequential fallback --- src/uu/sort/Cargo.toml | 4 +- src/uu/sort/src/check.rs | 123 +++++++++++-- src/uu/sort/src/chunks.rs | 96 +++++----- src/uu/sort/src/ext_sort/mod.rs | 11 +- src/uu/sort/src/ext_sort/threaded.rs | 185 ++++++++++++++++++- src/uu/sort/src/ext_sort/wasi.rs | 59 ------ src/uu/sort/src/merge.rs | 256 ++++++++++++++++++++++++++- src/uu/sort/src/sort.rs | 10 +- 8 files changed, 593 insertions(+), 151 deletions(-) delete mode 100644 src/uu/sort/src/ext_sort/wasi.rs diff --git a/src/uu/sort/Cargo.toml b/src/uu/sort/Cargo.toml index 5eaf7d9bccf..470f8a1b95e 100644 --- a/src/uu/sort/Cargo.toml +++ b/src/uu/sort/Cargo.toml @@ -29,7 +29,6 @@ compare = { workspace = true } itertools = { workspace = true } memchr = { workspace = true } rand = { workspace = true } -rayon = { workspace = true } self_cell = { workspace = true } tempfile = { workspace = true } thiserror = { workspace = true } @@ -47,6 +46,9 @@ foldhash = { workspace = true } [target.'cfg(all(unix, not(any(target_os = "redox", target_os = "fuchsia", target_os = "haiku", target_os = "solaris", target_os = "illumos"))))'.dependencies] rustix = { workspace = true, features = ["system", "process"] } +[target.'cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))'.dependencies] +rayon = { workspace = true } + [target.'cfg(not(any(target_os = "redox", target_os = "wasi")))'.dependencies] ctrlc = { workspace = true } diff --git a/src/uu/sort/src/check.rs b/src/uu/sort/src/check.rs index a826bc75507..b4f69c1948e 100644 --- a/src/uu/sort/src/check.rs +++ b/src/uu/sort/src/check.rs @@ -16,9 +16,11 @@ use std::{ ffi::OsStr, io::Read, iter, - sync::mpsc::{Receiver, SyncSender, sync_channel}, - thread, }; +#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] +use std::sync::mpsc::{sync_channel, SyncSender, Receiver}; +#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] +use std::thread; use uucore::error::UResult; /// Check if the file at `path` is ordered. @@ -28,13 +30,35 @@ use uucore::error::UResult; /// The code we should exit with. pub fn check(path: &OsStr, settings: &GlobalSettings) -> UResult<()> { let max_allowed_cmp = if settings.unique { - // If `unique` is enabled, the previous line must compare _less_ to the next one. Ordering::Less } else { - // Otherwise, the line previous line must compare _less or equal_ to the next one. Ordering::Equal }; let file = open(path)?; + let chunk_size = if settings.buffer_size < 100 * 1024 { + settings.buffer_size + } else { + 100 * 1024 + }; + + #[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] + { + check_threaded(path, settings, max_allowed_cmp, file, chunk_size) + } + #[cfg(all(target_os = "wasi", not(target_feature = "atomics")))] + { + check_sync(path, settings, max_allowed_cmp, file, chunk_size) + } +} + +#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] +fn check_threaded( + path: &OsStr, + settings: &GlobalSettings, + max_allowed_cmp: Ordering, + file: Box, + chunk_size: usize, +) -> UResult<()> { let (recycled_sender, recycled_receiver) = sync_channel(2); let (loaded_sender, loaded_receiver) = sync_channel(2); thread::spawn({ @@ -42,13 +66,7 @@ pub fn check(path: &OsStr, settings: &GlobalSettings) -> UResult<()> { move || reader(file, &recycled_receiver, &loaded_sender, &settings) }); for _ in 0..2 { - let _ = recycled_sender.send(RecycledChunk::new(if settings.buffer_size < 100 * 1024 { - // when the buffer size is smaller than 100KiB we choose it instead of the default. - // this improves testability. - settings.buffer_size - } else { - 100 * 1024 - })); + let _ = recycled_sender.send(RecycledChunk::new(chunk_size)); } let mut prev_chunk: Option = None; @@ -62,8 +80,6 @@ pub fn check(path: &OsStr, settings: &GlobalSettings) -> UResult<()> { 'outer: for chunk in &loaded_receiver { line_idx += 1; if let Some(prev_chunk) = prev_chunk.take() { - // Check if the first element of the new chunk is greater than the last - // element from the previous chunk let prev_last = prev_chunk.lines().last().unwrap(); let new_first = chunk.lines().first().unwrap(); @@ -115,6 +131,7 @@ pub fn check(path: &OsStr, settings: &GlobalSettings) -> UResult<()> { } /// The function running on the reader thread. +#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] fn reader( mut file: Box, receiver: &Receiver, @@ -139,3 +156,83 @@ fn reader( } Ok(()) } + +/// Synchronous check for targets without thread support. +#[cfg(all(target_os = "wasi", not(target_feature = "atomics")))] +fn check_sync( + path: &OsStr, + settings: &GlobalSettings, + max_allowed_cmp: Ordering, + mut file: Box, + chunk_size: usize, +) -> UResult<()> { + let separator = settings.line_ending.into(); + let mut carry_over = vec![]; + let mut prev_chunk: Option = None; + let mut spare_recycled: Option = None; + let mut line_idx = 0; + + loop { + let recycled = spare_recycled + .take() + .unwrap_or_else(|| RecycledChunk::new(chunk_size)); + + let (chunk, should_continue) = chunks::read_to_chunk( + recycled, + None, + &mut carry_over, + &mut file, + &mut iter::empty(), + separator, + settings, + )?; + + let Some(chunk) = chunk else { + break; + }; + + line_idx += 1; + if let Some(prev) = prev_chunk.take() { + let prev_last = prev.lines().last().unwrap(); + let new_first = chunk.lines().first().unwrap(); + + if compare_by( + prev_last, + new_first, + settings, + prev.line_data(), + chunk.line_data(), + ) > max_allowed_cmp + { + return Err(SortError::Disorder { + file: path.to_owned(), + line_number: line_idx, + line: String::from_utf8_lossy(new_first.line).into_owned(), + silent: settings.check_silent, + } + .into()); + } + spare_recycled = Some(prev.recycle()); + } + + for (a, b) in chunk.lines().iter().tuple_windows() { + line_idx += 1; + if compare_by(a, b, settings, chunk.line_data(), chunk.line_data()) > max_allowed_cmp { + return Err(SortError::Disorder { + file: path.to_owned(), + line_number: line_idx, + line: String::from_utf8_lossy(b.line).into_owned(), + silent: settings.check_silent, + } + .into()); + } + } + + prev_chunk = Some(chunk); + + if !should_continue { + break; + } + } + Ok(()) +} diff --git a/src/uu/sort/src/chunks.rs b/src/uu/sort/src/chunks.rs index 4b388dc7bbd..0c079d68b5b 100644 --- a/src/uu/sort/src/chunks.rs +++ b/src/uu/sort/src/chunks.rs @@ -12,8 +12,9 @@ use std::{ io::{ErrorKind, Read}, ops::Range, - sync::mpsc::SyncSender, }; +#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] +use std::sync::mpsc::SyncSender; use memchr::memchr_iter; use self_cell::self_cell; @@ -157,30 +158,13 @@ impl RecycledChunk { } } -/// Read a chunk, parse lines and send them. +/// Read a chunk from the input, parse lines, and return it directly. /// -/// No empty chunk will be sent. If we reach the end of the input, `false` is returned. -/// However, if this function returns `true`, it is not guaranteed that there is still -/// input left: If the input fits _exactly_ into a buffer, we will only notice that there's -/// nothing more to read at the next invocation. In case there is no input left, nothing will -/// be sent. -/// -/// # Arguments -/// -/// (see also `read_to_chunk` for a more detailed documentation) -/// -/// * `sender`: The sender to send the lines to the sorter. -/// * `recycled_chunk`: The recycled chunk, as returned by `Chunk::recycle`. -/// (i.e. `buffer.len()` should be equal to `buffer.capacity()`) -/// * `max_buffer_size`: How big `buffer` can be. -/// * `carry_over`: The bytes that must be carried over in between invocations. -/// * `file`: The current file. -/// * `next_files`: What `file` should be updated to next. -/// * `separator`: The line separator. -/// * `settings`: The global settings. +/// Returns `(Some(chunk), should_continue)` if data was read, or +/// `(None, false)` if the input was empty. The `should_continue` flag +/// indicates whether more data may remain. #[allow(clippy::too_many_arguments)] -pub fn read( - sender: &SyncSender, +pub fn read_to_chunk( recycled_chunk: RecycledChunk, max_buffer_size: Option, carry_over: &mut Vec, @@ -188,7 +172,7 @@ pub fn read( next_files: &mut impl Iterator>, separator: u8, settings: &GlobalSettings, -) -> UResult { +) -> UResult<(Option, bool)> { let RecycledChunk { lines, selections, @@ -202,7 +186,6 @@ pub fn read( mut buffer, } = recycled_chunk; if buffer.len() < carry_over.len() { - // Separate carry_over and copy them to avoid cost of 0 fill buffer buffer.extend_from_slice(&carry_over[buffer.len()..]); } buffer[..carry_over.len()].copy_from_slice(carry_over); @@ -218,7 +201,7 @@ pub fn read( carry_over.extend_from_slice(&buffer[read..]); if read != 0 { - let payload: UResult = Chunk::try_new(buffer, |buffer| { + let chunk: UResult = Chunk::try_new(buffer, |buffer| { let selections = unsafe { // SAFETY: It is safe to transmute to an empty vector of selections with shorter lifetime. // It was only temporarily transmuted to a Vec> to make recycling possible. @@ -254,7 +237,38 @@ pub fn read( line_count_hint, }) }); - sender.send(payload?).unwrap(); + Ok((Some(chunk?), should_continue)) + } else { + Ok((None, should_continue)) + } +} + +/// Read a chunk, parse lines and send them via channel. +/// +/// Wrapper around [`read_to_chunk`] for the threaded code path. +#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] +#[allow(clippy::too_many_arguments)] +pub fn read( + sender: &SyncSender, + recycled_chunk: RecycledChunk, + max_buffer_size: Option, + carry_over: &mut Vec, + file: &mut T, + next_files: &mut impl Iterator>, + separator: u8, + settings: &GlobalSettings, +) -> UResult { + let (chunk, should_continue) = read_to_chunk( + recycled_chunk, + max_buffer_size, + carry_over, + file, + next_files, + separator, + settings, + )?; + if let Some(chunk) = chunk { + sender.send(chunk).unwrap(); } Ok(should_continue) } @@ -432,31 +446,3 @@ fn read_to_buffer( } } -/// Parse a buffer into a `ChunkContents` suitable for `Chunk::try_new`. -/// Used by the WASI single-threaded sort path. -#[cfg(target_os = "wasi")] -pub fn parse_into_chunk<'a>( - buffer: &'a [u8], - separator: u8, - settings: &GlobalSettings, -) -> ChunkContents<'a> { - let mut lines = Vec::new(); - let mut line_data = LineData::default(); - let mut token_buffer = Vec::new(); - let mut line_count_hint = 0; - parse_lines( - buffer, - &mut lines, - &mut line_data, - &mut token_buffer, - &mut line_count_hint, - separator, - settings, - ); - ChunkContents { - lines, - line_data, - token_buffer, - line_count_hint, - } -} diff --git a/src/uu/sort/src/ext_sort/mod.rs b/src/uu/sort/src/ext_sort/mod.rs index 099a4b72e62..e20452f7e8c 100644 --- a/src/uu/sort/src/ext_sort/mod.rs +++ b/src/uu/sort/src/ext_sort/mod.rs @@ -6,15 +6,8 @@ //! External sort: sort large inputs that may not fit in memory. //! //! On most platforms this uses a multi-threaded chunked approach with -//! temporary files. On WASI (no threads) we fall back to an in-memory sort. +//! temporary files. On WASI without atomics, synchronous fallbacks are +//! used instead (selected via `cfg` guards inside the module). -#[cfg(not(target_os = "wasi"))] mod threaded; -#[cfg(not(target_os = "wasi"))] pub use threaded::ext_sort; - -#[cfg(target_os = "wasi")] -mod wasi; -#[cfg(target_os = "wasi")] -// `self::` needed to disambiguate from the `wasi` crate -pub use self::wasi::ext_sort; diff --git a/src/uu/sort/src/ext_sort/threaded.rs b/src/uu/sort/src/ext_sort/threaded.rs index 7dd089d0fe8..7f9fb1f4d34 100644 --- a/src/uu/sort/src/ext_sort/threaded.rs +++ b/src/uu/sort/src/ext_sort/threaded.rs @@ -10,14 +10,19 @@ use std::cmp::Ordering; use std::fs::File; use std::io::{Read, Write, stderr}; use std::path::PathBuf; +#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] use std::sync::mpsc::{Receiver, SyncSender}; +#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] use std::thread; use itertools::Itertools; -use uucore::error::{UResult, strip_errno}; +use uucore::error::UResult; +#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] +use uucore::error::strip_errno; use crate::Output; use crate::chunks::RecycledChunk; +#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] use crate::merge::WriteableCompressedTmpFile; use crate::merge::WriteablePlainTmpFile; use crate::merge::WriteableTmpFile; @@ -32,11 +37,185 @@ use crate::{ // Fixed to 8 KiB (equivalent to `std::sys::io::DEFAULT_BUF_SIZE` on most targets) const DEFAULT_BUF_SIZE: usize = 8 * 1024; +/// Synchronous sort for targets without thread support (e.g. wasm32-wasip1). +/// +/// Uses the same chunked sort-write-merge strategy as the threaded version, +/// but reads and sorts each chunk sequentially on the calling thread. +#[cfg(all(target_os = "wasi", not(target_feature = "atomics")))] +pub fn ext_sort( + files: &mut impl Iterator>>, + settings: &GlobalSettings, + output: Output, + tmp_dir: &mut TmpDirWrapper, +) -> UResult<()> { + let separator = settings.line_ending.into(); + let mut buffer_size = match settings.buffer_size { + size if size <= 512 * 1024 * 1024 => size, + size => size / 2, + }; + if !settings.buffer_size_is_explicit { + buffer_size = buffer_size.max(8 * 1024 * 1024); + } + + if settings.compress_prog.is_some() { + let _ = writeln!( + stderr(), + "sort: warning: --compress-program is ignored on this platform" + ); + } + + let mut file = files.next().unwrap()?; + let mut carry_over = vec![]; + + // Read and sort first chunk. + let (first, cont) = chunks::read_to_chunk( + RecycledChunk::new(buffer_size.min(DEFAULT_BUF_SIZE)), + Some(buffer_size), + &mut carry_over, + &mut file, + files, + separator, + settings, + )?; + let Some(mut first) = first else { + return Ok(()); // empty input + }; + first.with_dependent_mut(|_, c| sort_by(&mut c.lines, settings, &c.line_data)); + + if !cont { + // All input fits in one chunk. + return print_chunk(&first, settings, output); + } + + // Read and sort second chunk. + let (second, cont) = chunks::read_to_chunk( + RecycledChunk::new(buffer_size.min(DEFAULT_BUF_SIZE)), + Some(buffer_size), + &mut carry_over, + &mut file, + files, + separator, + settings, + )?; + let Some(mut second) = second else { + return print_chunk(&first, settings, output); + }; + second.with_dependent_mut(|_, c| sort_by(&mut c.lines, settings, &c.line_data)); + + if !cont { + // All input fits in two chunks — merge in memory. + return print_two_chunks(first, second, settings, output); + } + + // More than two chunks: write sorted chunks to temp files, then merge. + let mut tmp_files: Vec<::Closed> = vec![]; + + tmp_files.push(write::( + &first, + tmp_dir.next_file()?, + settings.compress_prog.as_deref(), + separator, + )?); + drop(first); + + tmp_files.push(write::( + &second, + tmp_dir.next_file()?, + settings.compress_prog.as_deref(), + separator, + )?); + let mut recycled = second.recycle(); + + loop { + let (chunk, cont) = chunks::read_to_chunk( + recycled, + None, + &mut carry_over, + &mut file, + files, + separator, + settings, + )?; + let Some(mut chunk) = chunk else { break }; + chunk.with_dependent_mut(|_, c| sort_by(&mut c.lines, settings, &c.line_data)); + tmp_files.push(write::( + &chunk, + tmp_dir.next_file()?, + settings.compress_prog.as_deref(), + separator, + )?); + recycled = chunk.recycle(); + if !cont { + break; + } + } + + merge::merge_with_file_limit::<_, _, WriteablePlainTmpFile>( + tmp_files.into_iter().map(merge::ClosedTmpFile::reopen), + settings, + output, + tmp_dir, + ) +} + +/// Print a single sorted chunk. +#[cfg(all(target_os = "wasi", not(target_feature = "atomics")))] +fn print_chunk( + chunk: &Chunk, + settings: &GlobalSettings, + output: Output, +) -> UResult<()> { + if settings.unique { + print_sorted( + chunk.lines().iter().dedup_by(|a, b| { + compare_by(a, b, settings, chunk.line_data(), chunk.line_data()) + == Ordering::Equal + }), + settings, + output, + ) + } else { + print_sorted(chunk.lines().iter(), settings, output) + } +} + +/// Merge two in-memory chunks and print. +#[cfg(all(target_os = "wasi", not(target_feature = "atomics")))] +fn print_two_chunks( + a: Chunk, + b: Chunk, + settings: &GlobalSettings, + output: Output, +) -> UResult<()> { + let merged_iter = a.lines().iter().map(|line| (line, &a)).merge_by( + b.lines().iter().map(|line| (line, &b)), + |(line_a, a), (line_b, b)| { + compare_by(line_a, line_b, settings, a.line_data(), b.line_data()) + != Ordering::Greater + }, + ); + if settings.unique { + print_sorted( + merged_iter + .dedup_by(|(line_a, a), (line_b, b)| { + compare_by(line_a, line_b, settings, a.line_data(), b.line_data()) + == Ordering::Equal + }) + .map(|(line, _)| line), + settings, + output, + ) + } else { + print_sorted(merged_iter.map(|(line, _)| line), settings, output) + } +} + /// Sort files by using auxiliary files for storing intermediate chunks (if needed), and output the result. /// /// Two threads cooperate: one reads input and writes temporary chunk files, /// while the other sorts each chunk in memory. Once all chunks are written, /// they are merged back together for final output. +#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] pub fn ext_sort( files: &mut impl Iterator>>, settings: &GlobalSettings, @@ -97,6 +276,7 @@ pub fn ext_sort( } } +#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] fn reader_writer< F: Iterator>>, Tmp: WriteableTmpFile + 'static, @@ -182,6 +362,7 @@ fn reader_writer< } /// The function that is executed on the sorter thread. +#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] fn sorter(receiver: &Receiver, sender: &SyncSender, settings: &GlobalSettings) { while let Ok(mut payload) = receiver.recv() { payload.with_dependent_mut(|_, contents| { @@ -196,6 +377,7 @@ fn sorter(receiver: &Receiver, sender: &SyncSender, settings: &Glo } /// Describes how we read the chunks from the input. +#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] enum ReadResult { /// The input was empty. Nothing was read. EmptyInput, @@ -207,6 +389,7 @@ enum ReadResult { WroteChunksToFile { tmp_files: Vec }, } /// The function that is executed on the reader/writer thread. +#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] fn read_write_loop( mut files: impl Iterator>>, tmp_dir: &mut TmpDirWrapper, diff --git a/src/uu/sort/src/ext_sort/wasi.rs b/src/uu/sort/src/ext_sort/wasi.rs deleted file mode 100644 index 50bd5f63033..00000000000 --- a/src/uu/sort/src/ext_sort/wasi.rs +++ /dev/null @@ -1,59 +0,0 @@ -// This file is part of the uutils coreutils package. -// -// For the full copyright and license information, please view the LICENSE -// file that was distributed with this source code. - -//! WASI single-threaded sort: read all input into memory, sort, and output. -//! Threads are not available on WASI, so we bypass the chunked/threaded path. - -use std::cmp::Ordering; -use std::io::Read; - -use itertools::Itertools; -use uucore::error::UResult; - -use crate::Output; -use crate::chunks::{self, Chunk}; -use crate::tmp_dir::TmpDirWrapper; -use crate::{GlobalSettings, compare_by, print_sorted, sort_by}; - -/// Sort files by reading all input into memory, sorting in a single thread, and outputting directly. -pub fn ext_sort( - files: &mut impl Iterator>>, - settings: &GlobalSettings, - output: Output, - _tmp_dir: &mut TmpDirWrapper, -) -> UResult<()> { - let separator = settings.line_ending.into(); - // Read all input into memory at once. Unlike the threaded path which uses - // chunked buffered reads, WASI has no threads so we accept the memory cost. - // Note: there is no size limit here — WASI targets are expected to handle - // moderately sized inputs; very large files may cause OOM. - let mut input = Vec::new(); - for file in files { - file?.read_to_end(&mut input)?; - } - if input.is_empty() { - return Ok(()); - } - let mut chunk = Chunk::try_new(input, |buffer| { - Ok::<_, Box>(chunks::parse_into_chunk( - buffer, separator, settings, - )) - })?; - chunk.with_dependent_mut(|_, contents| { - sort_by(&mut contents.lines, settings, &contents.line_data); - }); - if settings.unique { - print_sorted( - chunk.lines().iter().dedup_by(|a, b| { - compare_by(a, b, settings, chunk.line_data(), chunk.line_data()) == Ordering::Equal - }), - settings, - output, - )?; - } else { - print_sorted(chunk.lines().iter(), settings, output)?; - } - Ok(()) -} diff --git a/src/uu/sort/src/merge.rs b/src/uu/sort/src/merge.rs index 8f0b5bd54ae..1baa0097877 100644 --- a/src/uu/sort/src/merge.rs +++ b/src/uu/sort/src/merge.rs @@ -20,12 +20,17 @@ use std::{ path::{Path, PathBuf}, process::{Child, ChildStdin, ChildStdout, Command, Stdio}, rc::Rc, +}; +#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] +use std::{ sync::mpsc::{Receiver, Sender, SyncSender, TryRecvError, channel, sync_channel}, thread::{self, JoinHandle}, }; use compare::Compare; -use uucore::error::{FromIo, UResult}; +#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] +use uucore::error::FromIo; +use uucore::error::UResult; use crate::{ GlobalSettings, Output, SortError, @@ -104,6 +109,17 @@ pub fn merge( let files = files .iter() .map(|file| open(file).map(|file| PlainMergeInput { inner: file })); + #[cfg(all(target_os = "wasi", not(target_feature = "atomics")))] + if settings.compress_prog.is_some() { + let _ = writeln!( + std::io::stderr(), + "sort: warning: --compress-program is ignored on this platform" + ); + return merge_with_file_limit::<_, _, WriteablePlainTmpFile>( + files, settings, output, tmp_dir, + ); + } + if settings.compress_prog.is_none() { merge_with_file_limit::<_, _, WriteablePlainTmpFile>(files, settings, output, tmp_dir) } else { @@ -111,6 +127,30 @@ pub fn merge( } } +/// Merge and write to output — dispatches between threaded and synchronous. +fn do_merge_to_output( + files: impl Iterator>, + settings: &GlobalSettings, + output: Output, +) -> UResult<()> { + #[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] + return merge_without_limit(files, settings)?.write_all(settings, output); + #[cfg(all(target_os = "wasi", not(target_feature = "atomics")))] + return merge_without_limit_sync(files, settings)?.write_all(settings, output); +} + +/// Merge and write to a writer — dispatches between threaded and synchronous. +fn do_merge_to_writer( + files: impl Iterator>, + settings: &GlobalSettings, + out: &mut impl Write, +) -> UResult<()> { + #[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] + return merge_without_limit(files, settings)?.write_all_to(settings, out); + #[cfg(all(target_os = "wasi", not(target_feature = "atomics")))] + return merge_without_limit_sync(files, settings)?.write_all_to(settings, out); +} + // Merge already sorted `MergeInput`s. pub fn merge_with_file_limit< M: MergeInput + 'static, @@ -126,8 +166,7 @@ pub fn merge_with_file_limit< debug_assert!(batch_size >= 2); if files.len() <= batch_size { - let merger = merge_without_limit(files, settings); - merger?.write_all(settings, output) + do_merge_to_output(files, settings, output) } else { let mut temporary_files = vec![]; let mut batch = Vec::with_capacity(batch_size); @@ -135,23 +174,21 @@ pub fn merge_with_file_limit< batch.push(file); if batch.len() >= batch_size { assert_eq!(batch.len(), batch_size); - let merger = merge_without_limit(batch.into_iter(), settings)?; - batch = Vec::with_capacity(batch_size); + let full_batch = std::mem::replace(&mut batch, Vec::with_capacity(batch_size)); let mut tmp_file = Tmp::create(tmp_dir.next_file()?, settings.compress_prog.as_deref())?; - merger.write_all_to(settings, tmp_file.as_write())?; + do_merge_to_writer(full_batch.into_iter(), settings, tmp_file.as_write())?; temporary_files.push(tmp_file.finished_writing()?); } } // Merge any remaining files that didn't get merged in a full batch above. if !batch.is_empty() { assert!(batch.len() < batch_size); - let merger = merge_without_limit(batch.into_iter(), settings)?; let mut tmp_file = Tmp::create(tmp_dir.next_file()?, settings.compress_prog.as_deref())?; - merger.write_all_to(settings, tmp_file.as_write())?; + do_merge_to_writer(batch.into_iter(), settings, tmp_file.as_write())?; temporary_files.push(tmp_file.finished_writing()?); } merge_with_file_limit::<_, _, Tmp>( @@ -172,6 +209,7 @@ pub fn merge_with_file_limit< /// /// It is the responsibility of the caller to ensure that `files` yields only /// as many files as we are allowed to open concurrently. +#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] fn merge_without_limit>>( files: F, settings: &GlobalSettings, @@ -236,6 +274,7 @@ fn merge_without_limit>>( }) } /// The struct on the reader thread representing an input file +#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] struct ReaderFile { file: M, sender: SyncSender, @@ -243,6 +282,7 @@ struct ReaderFile { } /// The function running on the reader thread. +#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] fn reader( recycled_receiver: &Receiver<(usize, RecycledChunk)>, files: &mut [Option>], @@ -277,6 +317,7 @@ fn reader( Ok(()) } /// The struct on the main thread representing an input file +#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] pub struct MergeableFile { current_chunk: Rc, line_idx: usize, @@ -290,10 +331,12 @@ pub struct MergeableFile { struct PreviousLine { chunk: Rc, line_idx: usize, + #[cfg_attr(all(target_os = "wasi", not(target_feature = "atomics")), allow(dead_code))] file_number: usize, } /// Merges files together. This is **not** an iterator because of lifetime problems. +#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] struct FileMerger<'a> { heap: binary_heap_plus::BinaryHeap>, request_sender: Sender<(usize, RecycledChunk)>, @@ -301,6 +344,7 @@ struct FileMerger<'a> { reader_join_handle: JoinHandle>, } +#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] impl FileMerger<'_> { /// Write the merged contents to the output file. fn write_all(self, settings: &GlobalSettings, output: Output) -> UResult<()> { @@ -416,6 +460,7 @@ struct FileComparator<'a> { settings: &'a GlobalSettings, } +#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] impl Compare for FileComparator<'_> { fn compare(&self, a: &MergeableFile, b: &MergeableFile) -> Ordering { let mut cmp = compare_by( @@ -631,3 +676,198 @@ impl MergeInput for PlainMergeInput { &mut self.inner } } + +// --------------------------------------------------------------------------- +// Synchronous merge for targets without thread support (e.g. wasm32-wasip1). +// --------------------------------------------------------------------------- + +#[cfg(all(target_os = "wasi", not(target_feature = "atomics")))] +struct SyncReaderFile { + file: M, + carry_over: Vec, +} + +#[cfg(all(target_os = "wasi", not(target_feature = "atomics")))] +struct SyncMergeableFile { + current_chunk: Rc, + line_idx: usize, + file_number: usize, +} + +#[cfg(all(target_os = "wasi", not(target_feature = "atomics")))] +impl Compare for FileComparator<'_> { + fn compare(&self, a: &SyncMergeableFile, b: &SyncMergeableFile) -> Ordering { + let mut cmp = compare_by( + &a.current_chunk.lines()[a.line_idx], + &b.current_chunk.lines()[b.line_idx], + self.settings, + a.current_chunk.line_data(), + b.current_chunk.line_data(), + ); + if cmp == Ordering::Equal { + cmp = a.file_number.cmp(&b.file_number); + } + cmp.reverse() + } +} + +#[cfg(all(target_os = "wasi", not(target_feature = "atomics")))] +struct SyncFileMerger<'a, M: MergeInput> { + heap: binary_heap_plus::BinaryHeap>, + readers: Vec>>, + prev: Option, + recycled: Option, + settings: &'a GlobalSettings, +} + +#[cfg(all(target_os = "wasi", not(target_feature = "atomics")))] +impl SyncFileMerger<'_, M> { + fn write_all(self, settings: &GlobalSettings, output: Output) -> UResult<()> { + let mut out = output.into_write(); + self.write_all_to(settings, &mut out) + } + + fn write_all_to(mut self, settings: &GlobalSettings, out: &mut impl Write) -> UResult<()> { + while self.write_next(settings, out)? {} + for reader in self.readers.into_iter().flatten() { + reader.file.finished_reading()?; + } + Ok(()) + } + + fn write_next( + &mut self, + settings: &GlobalSettings, + out: &mut impl Write, + ) -> UResult { + if let Some(file) = self.heap.peek() { + let prev = self.prev.replace(PreviousLine { + chunk: file.current_chunk.clone(), + line_idx: file.line_idx, + file_number: file.file_number, + }); + + file.current_chunk.with_dependent(|_, contents| { + let current_line = &contents.lines[file.line_idx]; + if settings.unique { + if let Some(prev) = &prev { + let cmp = compare_by( + &prev.chunk.lines()[prev.line_idx], + current_line, + settings, + prev.chunk.line_data(), + file.current_chunk.line_data(), + ); + if cmp == Ordering::Equal { + return Ok(()); + } + } + } + current_line.print(out, settings) + })?; + + let was_last = file.current_chunk.lines().len() == file.line_idx + 1; + let file_number = file.file_number; + + if was_last { + let separator = self.settings.line_ending.into(); + let recycled = self + .recycled + .take() + .unwrap_or_else(|| RecycledChunk::new(8 * 1024)); + let next_chunk = if let Some(reader) = self.readers[file_number].as_mut() { + let (chunk, should_continue) = chunks::read_to_chunk( + recycled, + None, + &mut reader.carry_over, + reader.file.as_read(), + &mut iter::empty(), + separator, + self.settings, + )?; + if !should_continue { + if let Some(reader) = self.readers[file_number].take() { + reader.file.finished_reading()?; + } + } + chunk + } else { + None + }; + + if let Some(next_chunk) = next_chunk { + let mut file = self.heap.peek_mut().unwrap(); + file.current_chunk = Rc::new(next_chunk); + file.line_idx = 0; + } else { + self.heap.pop(); + } + } else { + self.heap.peek_mut().unwrap().line_idx += 1; + } + + // Recycle the previous chunk if no other reference holds it. + if let Some(prev) = prev { + if let Ok(chunk) = Rc::try_unwrap(prev.chunk) { + self.recycled = Some(chunk.recycle()); + } + } + } + Ok(!self.heap.is_empty()) + } +} + +#[cfg(all(target_os = "wasi", not(target_feature = "atomics")))] +fn merge_without_limit_sync<'a, M: MergeInput + 'static, F: Iterator>>( + files: F, + settings: &'a GlobalSettings, +) -> UResult> { + let separator = settings.line_ending.into(); + let mut readers: Vec>> = Vec::new(); + let mut mergeable_files = Vec::new(); + + for (file_number, file) in files.enumerate() { + let mut reader = SyncReaderFile { + file: file?, + carry_over: vec![], + }; + let recycled = RecycledChunk::new(8 * 1024); + let (chunk, should_continue) = chunks::read_to_chunk( + recycled, + None, + &mut reader.carry_over, + reader.file.as_read(), + &mut iter::empty(), + separator, + settings, + )?; + + if let Some(chunk) = chunk { + mergeable_files.push(SyncMergeableFile { + current_chunk: Rc::new(chunk), + line_idx: 0, + file_number, + }); + if should_continue { + readers.push(Some(reader)); + } else { + reader.file.finished_reading()?; + readers.push(None); + } + } else { + reader.file.finished_reading()?; + readers.push(None); + } + } + + Ok(SyncFileMerger { + heap: binary_heap_plus::BinaryHeap::from_vec_cmp( + mergeable_files, + FileComparator { settings }, + ), + readers, + prev: None, + recycled: None, + settings, + }) +} diff --git a/src/uu/sort/src/sort.rs b/src/uu/sort/src/sort.rs index 63ecb0dc736..adae45ed202 100644 --- a/src/uu/sort/src/sort.rs +++ b/src/uu/sort/src/sort.rs @@ -29,7 +29,7 @@ use foldhash::fast::FoldHasher; use foldhash::{HashMap, SharedSeed}; use numeric_str_cmp::{NumInfo, NumInfoParseSettings, human_numeric_str_cmp, numeric_str_cmp}; use rand::{RngExt as _, rng}; -#[cfg(not(target_os = "wasi"))] +#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] use rayon::slice::ParallelSliceMut; use std::cmp::Ordering; use std::env; @@ -2573,14 +2573,14 @@ fn sort_by<'a>(unsorted: &mut Vec>, settings: &GlobalSettings, line_dat // WASI does not support threads, so use non-parallel sort to avoid // rayon's thread pool which triggers an unreachable trap. if settings.stable || settings.unique { - #[cfg(not(target_os = "wasi"))] + #[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] unsorted.par_sort_by(cmp); - #[cfg(target_os = "wasi")] + #[cfg(all(target_os = "wasi", not(target_feature = "atomics")))] unsorted.sort_by(cmp); } else { - #[cfg(not(target_os = "wasi"))] + #[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] unsorted.par_sort_unstable_by(cmp); - #[cfg(target_os = "wasi")] + #[cfg(all(target_os = "wasi", not(target_feature = "atomics")))] unsorted.sort_unstable_by(cmp); } } From 98cd2617fd8fb2a8ad770d765de9cd1e815f56ae Mon Sep 17 00:00:00 2001 From: Anthony DePasquale Date: Wed, 8 Apr 2026 11:20:39 +0200 Subject: [PATCH 03/17] sort: add wasi_no_threads cfg alias to reduce predicate verbosity --- src/uu/sort/build.rs | 14 ++++++++ src/uu/sort/src/check.rs | 23 ++++++------- src/uu/sort/src/chunks.rs | 7 ++-- src/uu/sort/src/ext_sort/threaded.rs | 43 ++++++++++--------------- src/uu/sort/src/merge.rs | 48 +++++++++++++--------------- src/uu/sort/src/sort.rs | 10 +++--- 6 files changed, 69 insertions(+), 76 deletions(-) create mode 100644 src/uu/sort/build.rs diff --git a/src/uu/sort/build.rs b/src/uu/sort/build.rs new file mode 100644 index 00000000000..e138c3eeaac --- /dev/null +++ b/src/uu/sort/build.rs @@ -0,0 +1,14 @@ +fn main() { + // Set a short alias for the WASI-without-threads configuration so that + // source files can use `#[cfg(wasi_no_threads)]` instead of the verbose + // `#[cfg(all(target_os = "wasi", not(target_feature = "atomics")))]`. + println!("cargo::rustc-check-cfg=cfg(wasi_no_threads)"); + + let target_os = std::env::var("CARGO_CFG_TARGET_OS").unwrap_or_default(); + let has_atomics = std::env::var("CARGO_CFG_TARGET_FEATURE") + .is_ok_and(|f| f.split(',').any(|feat| feat == "atomics")); + + if target_os == "wasi" && !has_atomics { + println!("cargo::rustc-cfg=wasi_no_threads"); + } +} diff --git a/src/uu/sort/src/check.rs b/src/uu/sort/src/check.rs index b4f69c1948e..7569a078d73 100644 --- a/src/uu/sort/src/check.rs +++ b/src/uu/sort/src/check.rs @@ -11,16 +11,11 @@ use crate::{ compare_by, open, }; use itertools::Itertools; -use std::{ - cmp::Ordering, - ffi::OsStr, - io::Read, - iter, -}; -#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] -use std::sync::mpsc::{sync_channel, SyncSender, Receiver}; -#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] +#[cfg(not(wasi_no_threads))] +use std::sync::mpsc::{Receiver, SyncSender, sync_channel}; +#[cfg(not(wasi_no_threads))] use std::thread; +use std::{cmp::Ordering, ffi::OsStr, io::Read, iter}; use uucore::error::UResult; /// Check if the file at `path` is ordered. @@ -41,17 +36,17 @@ pub fn check(path: &OsStr, settings: &GlobalSettings) -> UResult<()> { 100 * 1024 }; - #[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] + #[cfg(not(wasi_no_threads))] { check_threaded(path, settings, max_allowed_cmp, file, chunk_size) } - #[cfg(all(target_os = "wasi", not(target_feature = "atomics")))] + #[cfg(wasi_no_threads)] { check_sync(path, settings, max_allowed_cmp, file, chunk_size) } } -#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] +#[cfg(not(wasi_no_threads))] fn check_threaded( path: &OsStr, settings: &GlobalSettings, @@ -131,7 +126,7 @@ fn check_threaded( } /// The function running on the reader thread. -#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] +#[cfg(not(wasi_no_threads))] fn reader( mut file: Box, receiver: &Receiver, @@ -158,7 +153,7 @@ fn reader( } /// Synchronous check for targets without thread support. -#[cfg(all(target_os = "wasi", not(target_feature = "atomics")))] +#[cfg(wasi_no_threads)] fn check_sync( path: &OsStr, settings: &GlobalSettings, diff --git a/src/uu/sort/src/chunks.rs b/src/uu/sort/src/chunks.rs index 0c079d68b5b..ff5df3a114e 100644 --- a/src/uu/sort/src/chunks.rs +++ b/src/uu/sort/src/chunks.rs @@ -9,12 +9,12 @@ #![allow(dead_code)] // Ignores non-used warning for `borrow_buffer` in `Chunk` +#[cfg(not(wasi_no_threads))] +use std::sync::mpsc::SyncSender; use std::{ io::{ErrorKind, Read}, ops::Range, }; -#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] -use std::sync::mpsc::SyncSender; use memchr::memchr_iter; use self_cell::self_cell; @@ -246,7 +246,7 @@ pub fn read_to_chunk( /// Read a chunk, parse lines and send them via channel. /// /// Wrapper around [`read_to_chunk`] for the threaded code path. -#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] +#[cfg(not(wasi_no_threads))] #[allow(clippy::too_many_arguments)] pub fn read( sender: &SyncSender, @@ -445,4 +445,3 @@ fn read_to_buffer( } } } - diff --git a/src/uu/sort/src/ext_sort/threaded.rs b/src/uu/sort/src/ext_sort/threaded.rs index 7f9fb1f4d34..2f24bb7a14a 100644 --- a/src/uu/sort/src/ext_sort/threaded.rs +++ b/src/uu/sort/src/ext_sort/threaded.rs @@ -10,19 +10,19 @@ use std::cmp::Ordering; use std::fs::File; use std::io::{Read, Write, stderr}; use std::path::PathBuf; -#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] +#[cfg(not(wasi_no_threads))] use std::sync::mpsc::{Receiver, SyncSender}; -#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] +#[cfg(not(wasi_no_threads))] use std::thread; use itertools::Itertools; use uucore::error::UResult; -#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] +#[cfg(not(wasi_no_threads))] use uucore::error::strip_errno; use crate::Output; use crate::chunks::RecycledChunk; -#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] +#[cfg(not(wasi_no_threads))] use crate::merge::WriteableCompressedTmpFile; use crate::merge::WriteablePlainTmpFile; use crate::merge::WriteableTmpFile; @@ -41,7 +41,7 @@ const DEFAULT_BUF_SIZE: usize = 8 * 1024; /// /// Uses the same chunked sort-write-merge strategy as the threaded version, /// but reads and sorts each chunk sequentially on the calling thread. -#[cfg(all(target_os = "wasi", not(target_feature = "atomics")))] +#[cfg(wasi_no_threads)] pub fn ext_sort( files: &mut impl Iterator>>, settings: &GlobalSettings, @@ -159,17 +159,12 @@ pub fn ext_sort( } /// Print a single sorted chunk. -#[cfg(all(target_os = "wasi", not(target_feature = "atomics")))] -fn print_chunk( - chunk: &Chunk, - settings: &GlobalSettings, - output: Output, -) -> UResult<()> { +#[cfg(wasi_no_threads)] +fn print_chunk(chunk: &Chunk, settings: &GlobalSettings, output: Output) -> UResult<()> { if settings.unique { print_sorted( chunk.lines().iter().dedup_by(|a, b| { - compare_by(a, b, settings, chunk.line_data(), chunk.line_data()) - == Ordering::Equal + compare_by(a, b, settings, chunk.line_data(), chunk.line_data()) == Ordering::Equal }), settings, output, @@ -180,18 +175,12 @@ fn print_chunk( } /// Merge two in-memory chunks and print. -#[cfg(all(target_os = "wasi", not(target_feature = "atomics")))] -fn print_two_chunks( - a: Chunk, - b: Chunk, - settings: &GlobalSettings, - output: Output, -) -> UResult<()> { +#[cfg(wasi_no_threads)] +fn print_two_chunks(a: Chunk, b: Chunk, settings: &GlobalSettings, output: Output) -> UResult<()> { let merged_iter = a.lines().iter().map(|line| (line, &a)).merge_by( b.lines().iter().map(|line| (line, &b)), |(line_a, a), (line_b, b)| { - compare_by(line_a, line_b, settings, a.line_data(), b.line_data()) - != Ordering::Greater + compare_by(line_a, line_b, settings, a.line_data(), b.line_data()) != Ordering::Greater }, ); if settings.unique { @@ -215,7 +204,7 @@ fn print_two_chunks( /// Two threads cooperate: one reads input and writes temporary chunk files, /// while the other sorts each chunk in memory. Once all chunks are written, /// they are merged back together for final output. -#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] +#[cfg(not(wasi_no_threads))] pub fn ext_sort( files: &mut impl Iterator>>, settings: &GlobalSettings, @@ -276,7 +265,7 @@ pub fn ext_sort( } } -#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] +#[cfg(not(wasi_no_threads))] fn reader_writer< F: Iterator>>, Tmp: WriteableTmpFile + 'static, @@ -362,7 +351,7 @@ fn reader_writer< } /// The function that is executed on the sorter thread. -#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] +#[cfg(not(wasi_no_threads))] fn sorter(receiver: &Receiver, sender: &SyncSender, settings: &GlobalSettings) { while let Ok(mut payload) = receiver.recv() { payload.with_dependent_mut(|_, contents| { @@ -377,7 +366,7 @@ fn sorter(receiver: &Receiver, sender: &SyncSender, settings: &Glo } /// Describes how we read the chunks from the input. -#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] +#[cfg(not(wasi_no_threads))] enum ReadResult { /// The input was empty. Nothing was read. EmptyInput, @@ -389,7 +378,7 @@ enum ReadResult { WroteChunksToFile { tmp_files: Vec }, } /// The function that is executed on the reader/writer thread. -#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] +#[cfg(not(wasi_no_threads))] fn read_write_loop( mut files: impl Iterator>>, tmp_dir: &mut TmpDirWrapper, diff --git a/src/uu/sort/src/merge.rs b/src/uu/sort/src/merge.rs index 1baa0097877..e79d92fc671 100644 --- a/src/uu/sort/src/merge.rs +++ b/src/uu/sort/src/merge.rs @@ -21,14 +21,14 @@ use std::{ process::{Child, ChildStdin, ChildStdout, Command, Stdio}, rc::Rc, }; -#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] +#[cfg(not(wasi_no_threads))] use std::{ sync::mpsc::{Receiver, Sender, SyncSender, TryRecvError, channel, sync_channel}, thread::{self, JoinHandle}, }; use compare::Compare; -#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] +#[cfg(not(wasi_no_threads))] use uucore::error::FromIo; use uucore::error::UResult; @@ -109,7 +109,7 @@ pub fn merge( let files = files .iter() .map(|file| open(file).map(|file| PlainMergeInput { inner: file })); - #[cfg(all(target_os = "wasi", not(target_feature = "atomics")))] + #[cfg(wasi_no_threads)] if settings.compress_prog.is_some() { let _ = writeln!( std::io::stderr(), @@ -133,9 +133,9 @@ fn do_merge_to_output( settings: &GlobalSettings, output: Output, ) -> UResult<()> { - #[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] + #[cfg(not(wasi_no_threads))] return merge_without_limit(files, settings)?.write_all(settings, output); - #[cfg(all(target_os = "wasi", not(target_feature = "atomics")))] + #[cfg(wasi_no_threads)] return merge_without_limit_sync(files, settings)?.write_all(settings, output); } @@ -145,9 +145,9 @@ fn do_merge_to_writer( settings: &GlobalSettings, out: &mut impl Write, ) -> UResult<()> { - #[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] + #[cfg(not(wasi_no_threads))] return merge_without_limit(files, settings)?.write_all_to(settings, out); - #[cfg(all(target_os = "wasi", not(target_feature = "atomics")))] + #[cfg(wasi_no_threads)] return merge_without_limit_sync(files, settings)?.write_all_to(settings, out); } @@ -209,7 +209,7 @@ pub fn merge_with_file_limit< /// /// It is the responsibility of the caller to ensure that `files` yields only /// as many files as we are allowed to open concurrently. -#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] +#[cfg(not(wasi_no_threads))] fn merge_without_limit>>( files: F, settings: &GlobalSettings, @@ -274,7 +274,7 @@ fn merge_without_limit>>( }) } /// The struct on the reader thread representing an input file -#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] +#[cfg(not(wasi_no_threads))] struct ReaderFile { file: M, sender: SyncSender, @@ -282,7 +282,7 @@ struct ReaderFile { } /// The function running on the reader thread. -#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] +#[cfg(not(wasi_no_threads))] fn reader( recycled_receiver: &Receiver<(usize, RecycledChunk)>, files: &mut [Option>], @@ -317,7 +317,7 @@ fn reader( Ok(()) } /// The struct on the main thread representing an input file -#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] +#[cfg(not(wasi_no_threads))] pub struct MergeableFile { current_chunk: Rc, line_idx: usize, @@ -331,12 +331,12 @@ pub struct MergeableFile { struct PreviousLine { chunk: Rc, line_idx: usize, - #[cfg_attr(all(target_os = "wasi", not(target_feature = "atomics")), allow(dead_code))] + #[cfg_attr(wasi_no_threads, allow(dead_code))] file_number: usize, } /// Merges files together. This is **not** an iterator because of lifetime problems. -#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] +#[cfg(not(wasi_no_threads))] struct FileMerger<'a> { heap: binary_heap_plus::BinaryHeap>, request_sender: Sender<(usize, RecycledChunk)>, @@ -344,7 +344,7 @@ struct FileMerger<'a> { reader_join_handle: JoinHandle>, } -#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] +#[cfg(not(wasi_no_threads))] impl FileMerger<'_> { /// Write the merged contents to the output file. fn write_all(self, settings: &GlobalSettings, output: Output) -> UResult<()> { @@ -460,7 +460,7 @@ struct FileComparator<'a> { settings: &'a GlobalSettings, } -#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] +#[cfg(not(wasi_no_threads))] impl Compare for FileComparator<'_> { fn compare(&self, a: &MergeableFile, b: &MergeableFile) -> Ordering { let mut cmp = compare_by( @@ -681,20 +681,20 @@ impl MergeInput for PlainMergeInput { // Synchronous merge for targets without thread support (e.g. wasm32-wasip1). // --------------------------------------------------------------------------- -#[cfg(all(target_os = "wasi", not(target_feature = "atomics")))] +#[cfg(wasi_no_threads)] struct SyncReaderFile { file: M, carry_over: Vec, } -#[cfg(all(target_os = "wasi", not(target_feature = "atomics")))] +#[cfg(wasi_no_threads)] struct SyncMergeableFile { current_chunk: Rc, line_idx: usize, file_number: usize, } -#[cfg(all(target_os = "wasi", not(target_feature = "atomics")))] +#[cfg(wasi_no_threads)] impl Compare for FileComparator<'_> { fn compare(&self, a: &SyncMergeableFile, b: &SyncMergeableFile) -> Ordering { let mut cmp = compare_by( @@ -711,7 +711,7 @@ impl Compare for FileComparator<'_> { } } -#[cfg(all(target_os = "wasi", not(target_feature = "atomics")))] +#[cfg(wasi_no_threads)] struct SyncFileMerger<'a, M: MergeInput> { heap: binary_heap_plus::BinaryHeap>, readers: Vec>>, @@ -720,7 +720,7 @@ struct SyncFileMerger<'a, M: MergeInput> { settings: &'a GlobalSettings, } -#[cfg(all(target_os = "wasi", not(target_feature = "atomics")))] +#[cfg(wasi_no_threads)] impl SyncFileMerger<'_, M> { fn write_all(self, settings: &GlobalSettings, output: Output) -> UResult<()> { let mut out = output.into_write(); @@ -735,11 +735,7 @@ impl SyncFileMerger<'_, M> { Ok(()) } - fn write_next( - &mut self, - settings: &GlobalSettings, - out: &mut impl Write, - ) -> UResult { + fn write_next(&mut self, settings: &GlobalSettings, out: &mut impl Write) -> UResult { if let Some(file) = self.heap.peek() { let prev = self.prev.replace(PreviousLine { chunk: file.current_chunk.clone(), @@ -817,7 +813,7 @@ impl SyncFileMerger<'_, M> { } } -#[cfg(all(target_os = "wasi", not(target_feature = "atomics")))] +#[cfg(wasi_no_threads)] fn merge_without_limit_sync<'a, M: MergeInput + 'static, F: Iterator>>( files: F, settings: &'a GlobalSettings, diff --git a/src/uu/sort/src/sort.rs b/src/uu/sort/src/sort.rs index adae45ed202..a47e708b156 100644 --- a/src/uu/sort/src/sort.rs +++ b/src/uu/sort/src/sort.rs @@ -29,7 +29,7 @@ use foldhash::fast::FoldHasher; use foldhash::{HashMap, SharedSeed}; use numeric_str_cmp::{NumInfo, NumInfoParseSettings, human_numeric_str_cmp, numeric_str_cmp}; use rand::{RngExt as _, rng}; -#[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] +#[cfg(not(wasi_no_threads))] use rayon::slice::ParallelSliceMut; use std::cmp::Ordering; use std::env; @@ -2573,14 +2573,14 @@ fn sort_by<'a>(unsorted: &mut Vec>, settings: &GlobalSettings, line_dat // WASI does not support threads, so use non-parallel sort to avoid // rayon's thread pool which triggers an unreachable trap. if settings.stable || settings.unique { - #[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] + #[cfg(not(wasi_no_threads))] unsorted.par_sort_by(cmp); - #[cfg(all(target_os = "wasi", not(target_feature = "atomics")))] + #[cfg(wasi_no_threads)] unsorted.sort_by(cmp); } else { - #[cfg(not(all(target_os = "wasi", not(target_feature = "atomics"))))] + #[cfg(not(wasi_no_threads))] unsorted.par_sort_unstable_by(cmp); - #[cfg(all(target_os = "wasi", not(target_feature = "atomics")))] + #[cfg(wasi_no_threads)] unsorted.sort_unstable_by(cmp); } } From 990ceffc94f6bc8d5123eb11c78a8d1b4187bd65 Mon Sep 17 00:00:00 2001 From: Anthony DePasquale Date: Wed, 8 Apr 2026 14:06:31 +0200 Subject: [PATCH 04/17] Replace unstable wasi_ext with stable libc calls --- src/uu/cp/src/cp.rs | 25 +++++++++++++++---------- src/uucore/src/lib/features/fs.rs | 11 +++++++---- src/uucore/src/lib/lib.rs | 1 - 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/src/uu/cp/src/cp.rs b/src/uu/cp/src/cp.rs index b61420c7dd7..2a37580d67e 100644 --- a/src/uu/cp/src/cp.rs +++ b/src/uu/cp/src/cp.rs @@ -2,8 +2,7 @@ // // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -#![cfg_attr(target_os = "wasi", feature(wasi_ext))] -// spell-checker:ignore (ToDO) copydir fiemap ftruncate linkgs lstat nlink nlinks pathbuf pwrite reflink strs xattrs symlinked deduplicated advcpmv nushell IRWXG IRWXO IRWXU IRWXUGO IRWXU IRWXG IRWXO IRWXUGO sflag +// spell-checker:ignore (ToDO) copydir ficlone fiemap ftruncate linkgs lstat nlink nlinks pathbuf pwrite reflink strs xattrs symlinked deduplicated advcpmv nushell IRWXG IRWXO IRWXU IRWXUGO IRWXU IRWXG IRWXO IRWXUGO sflag // spell-checker:ignore RDONLY futimens utimensat use std::cmp::Ordering; @@ -1375,9 +1374,9 @@ fn parse_path_args( /// Check if an error is ENOTSUP/EOPNOTSUPP (operation not supported). /// This is used to suppress xattr errors on filesystems that don't support them. fn is_enotsup_error(error: &CpError) -> bool { - #[cfg(unix)] + #[cfg(any(unix, target_os = "wasi"))] const EOPNOTSUPP: i32 = libc::EOPNOTSUPP; - #[cfg(not(unix))] + #[cfg(not(any(unix, target_os = "wasi")))] const EOPNOTSUPP: i32 = 95; match error { @@ -1900,7 +1899,7 @@ pub(crate) fn copy_attributes( // so return ENOTSUP. handle_preserve silently suppresses ENOTSUP for // optional preservation (-a) and reports it for required (--preserve=timestamps). #[cfg(target_os = "wasi")] - return Err(io::Error::from_raw_os_error(95).into()); // 95 = EOPNOTSUPP + return Err(io::Error::from_raw_os_error(libc::EOPNOTSUPP).into()); #[cfg(not(target_os = "wasi"))] { @@ -2001,14 +2000,20 @@ fn symlink_file( } #[cfg(target_os = "wasi")] { - std::os::wasi::fs::symlink_path(source, dest).map_err(|e| { - CpError::IoErrContext( - e, + use std::ffi::CString; + use std::os::wasi::ffi::OsStrExt; + let src_c = CString::new(source.as_os_str().as_bytes()) + .map_err(|e| CpError::Error(e.to_string()))?; + let dst_c = + CString::new(dest.as_os_str().as_bytes()).map_err(|e| CpError::Error(e.to_string()))?; + if unsafe { libc::symlink(src_c.as_ptr(), dst_c.as_ptr()) } != 0 { + return Err(CpError::IoErrContext( + io::Error::last_os_error(), translate!("cp-error-cannot-create-symlink", "dest" => get_filename(dest).unwrap_or("?").quote(), "source" => get_filename(source).unwrap_or("?").quote()), - ) - })?; + )); + } } if let Ok(file_info) = FileInformation::from_path(dest, false) { symlinked_files.insert(file_info); diff --git a/src/uucore/src/lib/features/fs.rs b/src/uucore/src/lib/features/fs.rs index 264d0abf4fd..bcfd02f1f7b 100644 --- a/src/uucore/src/lib/features/fs.rs +++ b/src/uucore/src/lib/features/fs.rs @@ -74,8 +74,12 @@ impl FileInformation { /// Get information from a currently open file #[cfg(target_os = "wasi")] pub fn from_file(file: &fs::File) -> IOResult { - let meta = file.metadata()?; - Ok(Self(meta)) + use std::os::fd::AsRawFd; + let mut stat: libc::stat = unsafe { std::mem::zeroed() }; + if unsafe { libc::fstat(file.as_raw_fd(), &mut stat) } != 0 { + return Err(Error::last_os_error()); + } + Ok(Self(stat)) } /// Get information for a given path. @@ -201,8 +205,7 @@ impl PartialEq for FileInformation { #[cfg(target_os = "wasi")] impl PartialEq for FileInformation { fn eq(&self, other: &Self) -> bool { - use std::os::wasi::fs::MetadataExt; - self.0.dev() == other.0.dev() && self.0.ino() == other.0.ino() + self.0.st_dev == other.0.st_dev && self.0.st_ino == other.0.st_ino } } diff --git a/src/uucore/src/lib/lib.rs b/src/uucore/src/lib/lib.rs index a693ad869ca..aeacfe01fc6 100644 --- a/src/uucore/src/lib/lib.rs +++ b/src/uucore/src/lib/lib.rs @@ -2,7 +2,6 @@ // // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -#![cfg_attr(all(target_os = "wasi", feature = "fs"), feature(wasi_ext))] //! library ~ (core/bundler file) // #![deny(missing_docs)] //TODO: enable this // From 43dd81539ce76187a1a0e0f801846a9638a4fb35 Mon Sep 17 00:00:00 2001 From: Anthony DePasquale Date: Wed, 8 Apr 2026 15:10:08 +0200 Subject: [PATCH 05/17] Fix clippy warnings for wasm32-wasip1 target --- src/uu/sort/src/merge.rs | 6 +++--- src/uu/touch/src/touch.rs | 13 +++++++------ 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/uu/sort/src/merge.rs b/src/uu/sort/src/merge.rs index e79d92fc671..6b58df5ef04 100644 --- a/src/uu/sort/src/merge.rs +++ b/src/uu/sort/src/merge.rs @@ -814,10 +814,10 @@ impl SyncFileMerger<'_, M> { } #[cfg(wasi_no_threads)] -fn merge_without_limit_sync<'a, M: MergeInput + 'static, F: Iterator>>( +fn merge_without_limit_sync>>( files: F, - settings: &'a GlobalSettings, -) -> UResult> { + settings: &GlobalSettings, +) -> UResult> { let separator = settings.line_ending.into(); let mut readers: Vec>> = Vec::new(); let mut mergeable_files = Vec::new(); diff --git a/src/uu/touch/src/touch.rs b/src/uu/touch/src/touch.rs index 485ec0b6696..016a016df9e 100644 --- a/src/uu/touch/src/touch.rs +++ b/src/uu/touch/src/touch.rs @@ -888,7 +888,10 @@ fn parse_timestamp(s: &str) -> UResult { /// /// On Windows, uses `GetFinalPathNameByHandleW` to attempt to get the path /// from the stdout handle. -#[cfg_attr(not(windows), expect(clippy::unnecessary_wraps))] +#[cfg_attr( + not(any(windows, target_os = "wasi")), + expect(clippy::unnecessary_wraps) +)] fn pathbuf_from_stdout() -> Result { #[cfg(all(unix, not(target_os = "android")))] { @@ -899,11 +902,9 @@ fn pathbuf_from_stdout() -> Result { Ok(PathBuf::from("/proc/self/fd/1")) } #[cfg(target_os = "wasi")] - { - return Err(TouchError::UnsupportedPlatformFeature( - "touch - (stdout) is not supported on WASI".to_string(), - )); - } + return Err(TouchError::UnsupportedPlatformFeature( + "touch - (stdout) is not supported on WASI".to_string(), + )); #[cfg(windows)] { use std::os::windows::prelude::AsRawHandle; From bc62e5b8677dd9f8dceab9c07d8528dc6785db42 Mon Sep 17 00:00:00 2001 From: Anthony DePasquale Date: Sun, 12 Apr 2026 23:11:14 +0200 Subject: [PATCH 06/17] Add integration tests for cat/sort/tail/touch + underlying fixes --- .github/workflows/wasi.yml | 14 ++++---- docs/src/wasi-test-gaps.md | 50 ++++++++++++++++++++++++-- src/uu/cat/src/platform/mod.rs | 6 +++- src/uu/cp/Cargo.toml | 3 ++ src/uu/cp/src/cp.rs | 55 +++++++++++++++++----------- src/uu/sort/src/sort.rs | 9 +++-- src/uu/touch/Cargo.toml | 2 ++ src/uu/touch/src/touch.rs | 60 +++++++++++++++++++++++++++++-- src/uucore/src/lib/features/fs.rs | 38 ++++++++++---------- tests/by-util/test_cat.rs | 11 ++++++ tests/by-util/test_sort.rs | 20 ++++++++++- tests/by-util/test_tail.rs | 51 ++++++++++++++++++++++++-- tests/by-util/test_touch.rs | 18 ++++++++++ 13 files changed, 280 insertions(+), 57 deletions(-) diff --git a/.github/workflows/wasi.yml b/.github/workflows/wasi.yml index 5072ccc573b..a0a73c0863f 100644 --- a/.github/workflows/wasi.yml +++ b/.github/workflows/wasi.yml @@ -60,18 +60,18 @@ jobs: # Tests incompatible with WASI are annotated with # #[cfg_attr(wasi_runner, ignore)] in the test source files. # TODO: add integration tests for these tools as WASI support is extended: - # arch b2sum cat cksum cp csplit date dir dircolors fmt join + # arch b2sum cksum cp csplit date dir dircolors fmt join # ls md5sum mkdir mv nproc pathchk pr printenv ptx pwd readlink # realpath rm rmdir seq sha1sum sha224sum sha256sum sha384sum - # sha512sum shred sleep sort split tail touch tsort uname uniq - # vdir + # sha512sum shred sleep split tsort uname uniq vdir UUTESTS_BINARY_PATH="$(pwd)/target/${{ matrix.job.target }}/debug/coreutils.wasm" \ UUTESTS_WASM_RUNNER=wasmtime \ cargo test --test tests -- \ test_base32:: test_base64:: test_basenc:: test_basename:: \ - test_comm:: test_cut:: test_dirname:: test_echo:: \ + test_cat:: test_comm:: test_cut:: test_dirname:: test_echo:: \ test_expand:: test_factor:: test_false:: test_fold:: \ test_head:: test_link:: test_ln:: test_nl:: test_numfmt:: \ - test_od:: test_paste:: test_printf:: test_shuf:: test_sum:: \ - test_tee:: test_tr:: test_true:: test_truncate:: \ - test_unexpand:: test_unlink:: test_wc:: test_yes:: + test_od:: test_paste:: test_printf:: test_shuf:: test_sort:: \ + test_sum:: test_tail:: test_tee:: test_touch:: test_tr:: \ + test_true:: test_truncate:: test_unexpand:: test_unlink:: test_wc:: \ + test_yes:: diff --git a/docs/src/wasi-test-gaps.md b/docs/src/wasi-test-gaps.md index 4789ef03dc9..af9aad321b8 100644 --- a/docs/src/wasi-test-gaps.md +++ b/docs/src/wasi-test-gaps.md @@ -6,7 +6,7 @@ To find all annotated tests: `grep -rn 'wasi_runner, ignore' tests/` ## Tools not yet covered by integration tests -arch, b2sum, cat, cksum, cp, csplit, date, dir, dircolors, fmt, join, ls, md5sum, mkdir, mv, nproc, pathchk, pr, printenv, ptx, pwd, readlink, realpath, rm, rmdir, seq, sha1sum, sha224sum, sha256sum, sha384sum, sha512sum, shred, sleep, sort, split, tail, touch, tsort, uname, uniq, vdir, yes +arch, b2sum, cksum, cp, csplit, date, dir, dircolors, fmt, join, ls, md5sum, mkdir, mv, nproc, pathchk, pr, printenv, ptx, pwd, readlink, realpath, rm, rmdir, seq, sha1sum, sha224sum, sha256sum, sha384sum, sha512sum, shred, sleep, split, tsort, uname, uniq, vdir, yes ## WASI sandbox: host paths not visible @@ -32,6 +32,50 @@ WASI does not support spawning child processes. Tests that shell out to other co When stdin is a seekable file, wasmtime does not preserve the file position between the host and guest. Tests that validate stdin offset behavior after `head` reads are skipped. -## WASI: read_link on absolute paths fails under wasmtime via spawned test harness +## WASI: read_link fails under wasmtime via spawned test harness -`fs::read_link` on an absolute path inside the sandbox (e.g. `/file2`) returns `EPERM` when the WASI binary is launched through `std::process::Command` from the test harness, even though the same call works when wasmtime is invoked directly. This breaks `uucore::fs::canonicalize` for symlink sources, so tests that rely on following a symlink to compute a relative path are skipped. +When the WASI binary is spawned via `std::process::Command` from the cargo-test harness, `fs::read_link` (and operations that follow symlinks, such as opening a symlink to a FIFO or traversing a symlink loop) can return `EPERM` on absolute paths — paths that work when wasmtime is invoked directly. Individual symptom tests skipped under this umbrella are annotated with narrower reasons describing the observed errno mismatch. + +## WASI: no Unix domain socket support + +WASI does not support Unix domain sockets. Tests that create or read from `AF_UNIX` sockets are skipped. + +## WASI: no locale data + +The WASI sandbox does not ship locale data, so `setlocale`/`LC_ALL` have no effect and sorting falls back to byte comparison. Tests that depend on locale-aware collation or month-name translation are skipped. + +## WASI: tail follow mode disabled + +`tail -f` / `tail -F` (follow mode) requires change-notification mechanisms (`inotify`, `kqueue`) and signal handling that WASI does not provide, so follow is disabled on WASI and a warning is emitted. Tests that exercise follow behaviour are skipped. + +## WASI: cannot detect unsafe overwrite + +`is_unsafe_overwrite` (used by `cat` to detect input-is-output situations) is stubbed to return `false` on WASI because the required `stat` / device-and-inode comparison is not available. Tests that assert this error path are skipped. + +## WASI: pre-epoch timestamps not representable + +WASI Preview 1 `Timestamp` is a `u64` nanosecond count since the Unix epoch, so `path_filestat_set_times` (and therefore `touch -t` with a two-digit year ≥ 69) cannot express dates before 1970. Tests that set pre-epoch timestamps are skipped. + +## WASI: no timezone database + +wasi-libc does not ship tzdata, so `TZ` is not honoured and timezone-dependent validation (e.g. `touch -t` rejecting a nonexistent local time during a DST transition) does not happen. Tests that rely on this are skipped. + +## WASI: guest root is a writable preopen + +The test harness maps the per-test working directory as the guest's `/`. That makes `/` writable inside the guest, so GNU-style protections against operating on the system root (e.g. `touch /` failing) cannot be exercised. Tests that assert these protections are skipped. + +## WASI: `touch -` (stdout) unsupported + +On WASI, `touch -` returns `UnsupportedPlatformFeature` because the guest cannot reliably locate the host file backing stdout. Tests that exercise `touch -` are skipped. + +## WASI: errno/error-message mismatches + +Several error paths surface different errno values (and therefore different error messages) through wasmtime than on POSIX. Observed cases: + +- Opening a directory as a file returns `EBADF` rather than `EISDIR`. +- Redirecting a directory into stdin returns `ENOENT` rather than `EISDIR`. +- Filesystem permission errors surface as `ENOENT` rather than `EACCES`. +- Symlink-loop traversal does not reliably surface `ELOOP` ("Too many levels of symbolic links"). +- Opening a symlink-to-directory does not reliably surface `EISDIR`. + +Tests that assert specific error text for these paths are skipped. diff --git a/src/uu/cat/src/platform/mod.rs b/src/uu/cat/src/platform/mod.rs index cde1ee96cbb..77255eff14a 100644 --- a/src/uu/cat/src/platform/mod.rs +++ b/src/uu/cat/src/platform/mod.rs @@ -9,7 +9,11 @@ pub use self::unix::is_safe_overwrite; #[cfg(windows)] pub use self::windows::is_safe_overwrite; -// WASI: no fstat-based device/inode checks available; assume safe. +// WASI: when stdout is inherited from a host file descriptor, wasmtime +// reports its fstat as all-zero (st_dev == st_ino == 0), so the dev/inode +// comparison against any input file descriptor can never match. There is +// no reliable way to detect unsafe overwrite here; assume safe rather than +// risk a spurious error. #[cfg(target_os = "wasi")] pub fn is_safe_overwrite(_input: &I, _output: &O) -> bool { true diff --git a/src/uu/cp/Cargo.toml b/src/uu/cp/Cargo.toml index 0f2a818119e..6b778010b27 100644 --- a/src/uu/cp/Cargo.toml +++ b/src/uu/cp/Cargo.toml @@ -49,6 +49,9 @@ windows-sys = { workspace = true, features = [ "Win32_Storage_FileSystem", ] } +[target.'cfg(target_os = "wasi")'.dependencies] +rustix = { workspace = true, features = ["fs"] } + [[bin]] name = "cp" path = "src/main.rs" diff --git a/src/uu/cp/src/cp.rs b/src/uu/cp/src/cp.rs index 2a37580d67e..ad6c3778d8a 100644 --- a/src/uu/cp/src/cp.rs +++ b/src/uu/cp/src/cp.rs @@ -884,12 +884,7 @@ impl Attributes { #[cfg(unix)] ownership: Preserve::Yes { required: true }, mode: Preserve::Yes { required: true }, - // WASI: filetime panics in from_last_{access,modification}_time, - // so timestamps cannot be preserved. Mark as optional so -a works. - #[cfg(not(target_os = "wasi"))] timestamps: Preserve::Yes { required: true }, - #[cfg(target_os = "wasi")] - timestamps: Preserve::Yes { required: false }, context: { #[cfg(all(feature = "selinux", any(target_os = "linux", target_os = "android")))] { @@ -1895,11 +1890,37 @@ pub(crate) fn copy_attributes( })?; handle_preserve(attributes.timestamps, || -> CopyResult<()> { - // filetime's WASI backend panics in from_last_{access,modification}_time, - // so return ENOTSUP. handle_preserve silently suppresses ENOTSUP for - // optional preservation (-a) and reports it for required (--preserve=timestamps). #[cfg(target_os = "wasi")] - return Err(io::Error::from_raw_os_error(libc::EOPNOTSUPP).into()); + { + // `filetime`'s WASI backend panics in + // `from_last_{access,modification}_time`. Reach `utimensat` directly + // through `rustix`, converting `SystemTime` → `Timespec` via + // `UNIX_EPOCH` (which matches the `path_filestat_set_times` contract). + use std::time::UNIX_EPOCH; + let to_timespec = |t: std::time::SystemTime| -> io::Result { + // Pre-epoch source times can't be represented by WASI's + // `path_filestat_set_times` (unsigned nanosecond count). + let d = t + .duration_since(UNIX_EPOCH) + .map_err(|e| io::Error::new(io::ErrorKind::Unsupported, e))?; + Ok(rustix::fs::Timespec { + tv_sec: d.as_secs() as _, + tv_nsec: d.subsec_nanos() as _, + }) + }; + let timestamps = rustix::fs::Timestamps { + last_access: to_timespec(source_metadata.accessed()?)?, + last_modification: to_timespec(source_metadata.modified()?)?, + }; + let flags = if dest.is_symlink() { + rustix::fs::AtFlags::SYMLINK_NOFOLLOW + } else { + rustix::fs::AtFlags::empty() + }; + rustix::fs::utimensat(rustix::fs::CWD, dest, ×tamps, flags) + .map_err(io::Error::from)?; + Ok(()) + } #[cfg(not(target_os = "wasi"))] { @@ -2000,20 +2021,14 @@ fn symlink_file( } #[cfg(target_os = "wasi")] { - use std::ffi::CString; - use std::os::wasi::ffi::OsStrExt; - let src_c = CString::new(source.as_os_str().as_bytes()) - .map_err(|e| CpError::Error(e.to_string()))?; - let dst_c = - CString::new(dest.as_os_str().as_bytes()).map_err(|e| CpError::Error(e.to_string()))?; - if unsafe { libc::symlink(src_c.as_ptr(), dst_c.as_ptr()) } != 0 { - return Err(CpError::IoErrContext( - io::Error::last_os_error(), + rustix::fs::symlink(source, dest).map_err(|e| { + CpError::IoErrContext( + io::Error::from(e), translate!("cp-error-cannot-create-symlink", "dest" => get_filename(dest).unwrap_or("?").quote(), "source" => get_filename(source).unwrap_or("?").quote()), - )); - } + ) + })?; } if let Ok(file_info) = FileInformation::from_path(dest, false) { symlinked_files.insert(file_info); diff --git a/src/uu/sort/src/sort.rs b/src/uu/sort/src/sort.rs index a47e708b156..3b549c38b5b 100644 --- a/src/uu/sort/src/sort.rs +++ b/src/uu/sort/src/sort.rs @@ -2111,11 +2111,14 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { .map(PathBuf::from) .or_else(|| env::var_os("TMPDIR").map(PathBuf::from)) .unwrap_or_else(|| { - // env::temp_dir() panics on WASI; default to /tmp #[cfg(target_os = "wasi")] - return PathBuf::from("/tmp"); + { + uucore::fs::wasi_default_tmp_dir() + } #[cfg(not(target_os = "wasi"))] - env::temp_dir() + { + env::temp_dir() + } }), ); diff --git a/src/uu/touch/Cargo.toml b/src/uu/touch/Cargo.toml index be19f567c55..1eac51334ef 100644 --- a/src/uu/touch/Cargo.toml +++ b/src/uu/touch/Cargo.toml @@ -30,6 +30,8 @@ tempfile = { workspace = true } [target.'cfg(unix)'.dependencies] libc = { workspace = true } + +[target.'cfg(any(unix, target_os = "wasi"))'.dependencies] rustix = { workspace = true, features = ["fs"] } [target.'cfg(target_os = "windows")'.dependencies] diff --git a/src/uu/touch/src/touch.rs b/src/uu/touch/src/touch.rs index 016a016df9e..fdf1625304d 100644 --- a/src/uu/touch/src/touch.rs +++ b/src/uu/touch/src/touch.rs @@ -10,9 +10,11 @@ pub mod error; use clap::builder::{PossibleValue, ValueParser}; use clap::{Arg, ArgAction, ArgGroup, ArgMatches, Command}; -#[cfg(any(not(unix), target_os = "redox"))] +use filetime::FileTime; +#[cfg(all(any(not(unix), target_os = "redox"), not(target_os = "wasi")))] use filetime::set_file_times; -use filetime::{FileTime, set_symlink_file_times}; +#[cfg(not(target_os = "wasi"))] +use filetime::set_symlink_file_times; use jiff::civil::Time; use jiff::fmt::strtime; use jiff::tz::TimeZone; @@ -708,6 +710,47 @@ fn try_futimens_via_write_fd(path: &Path, atime: FileTime, mtime: FileTime) -> s futimens(&file, ×tamps).map_err(|e| Error::from_raw_os_error(e.raw_os_error())) } +/// WASI replacement for `filetime::set_file_times`. +/// +/// The `filetime` crate has an unimplemented stub on `wasm32-wasi`. WASI +/// supports setting both atime and mtime via `utimensat`, which we reach +/// through `rustix`. +#[cfg(target_os = "wasi")] +fn set_file_times(path: &Path, atime: FileTime, mtime: FileTime) -> std::io::Result<()> { + wasi_utimensat(path, atime, mtime, false) +} + +/// WASI replacement for `filetime::set_symlink_file_times`. +#[cfg(target_os = "wasi")] +fn set_symlink_file_times(path: &Path, atime: FileTime, mtime: FileTime) -> std::io::Result<()> { + wasi_utimensat(path, atime, mtime, true) +} + +#[cfg(target_os = "wasi")] +fn wasi_utimensat( + path: &Path, + atime: FileTime, + mtime: FileTime, + no_follow: bool, +) -> std::io::Result<()> { + let timestamps = rustix::fs::Timestamps { + last_access: rustix::fs::Timespec { + tv_sec: atime.unix_seconds(), + tv_nsec: atime.nanoseconds() as _, + }, + last_modification: rustix::fs::Timespec { + tv_sec: mtime.unix_seconds(), + tv_nsec: mtime.nanoseconds() as _, + }, + }; + let flags = if no_follow { + rustix::fs::AtFlags::SYMLINK_NOFOLLOW + } else { + rustix::fs::AtFlags::empty() + }; + rustix::fs::utimensat(rustix::fs::CWD, path, ×tamps, flags).map_err(Error::from) +} + /// Get metadata of the provided path /// If `follow` is `true`, the function will try to follow symlinks. Errors if the symlink is dangling, otherwise defaults to symlink metadata. /// If `follow` is `false`, the function will return metadata of the symlink itself @@ -725,6 +768,19 @@ fn stat(path: &Path, follow: bool) -> std::io::Result<(FileTime, FileTime)> { fs::symlink_metadata(path)? }; + // `FileTime::from_last_{access,modification}_time` is unimplemented on + // `wasm32-wasi`, so go through `Metadata::{accessed, modified}` (which + // return `SystemTime`) and convert via `FileTime::from_system_time`. + #[cfg(target_os = "wasi")] + { + let atime = metadata.accessed()?; + let mtime = metadata.modified()?; + Ok(( + FileTime::from_system_time(atime), + FileTime::from_system_time(mtime), + )) + } + #[cfg(not(target_os = "wasi"))] Ok(( FileTime::from_last_access_time(&metadata), FileTime::from_last_modification_time(&metadata), diff --git a/src/uucore/src/lib/features/fs.rs b/src/uucore/src/lib/features/fs.rs index bcfd02f1f7b..40f8cc24c34 100644 --- a/src/uucore/src/lib/features/fs.rs +++ b/src/uucore/src/lib/features/fs.rs @@ -71,17 +71,6 @@ impl FileInformation { Ok(Self(info)) } - /// Get information from a currently open file - #[cfg(target_os = "wasi")] - pub fn from_file(file: &fs::File) -> IOResult { - use std::os::fd::AsRawFd; - let mut stat: libc::stat = unsafe { std::mem::zeroed() }; - if unsafe { libc::fstat(file.as_raw_fd(), &mut stat) } != 0 { - return Err(Error::last_os_error()); - } - Ok(Self(stat)) - } - /// Get information for a given path. /// /// If `path` points to a symlink and `dereference` is true, information about @@ -202,13 +191,6 @@ impl PartialEq for FileInformation { } } -#[cfg(target_os = "wasi")] -impl PartialEq for FileInformation { - fn eq(&self, other: &Self) -> bool { - self.0.st_dev == other.0.st_dev && self.0.st_ino == other.0.st_ino - } -} - impl Eq for FileInformation {} impl Hash for FileInformation { @@ -252,6 +234,26 @@ pub enum ResolveMode { Logical, } +/// WASI fallback used when neither `--tmp-dir` nor `TMPDIR` is set and +/// `env::temp_dir()` would be inapplicable. +/// +/// The WASI sandbox only exposes explicitly preopened directories, and +/// `/tmp` is not one by default. This returns `/tmp` when a host preopen +/// has made it visible as a directory, and the current directory otherwise +/// — the current directory is always accessible under a preopen mapped +/// to `/`. +/// +/// Callers on WASI should prefer `--tmp-dir` and `TMPDIR` before falling +/// back to this helper. +#[cfg(target_os = "wasi")] +pub fn wasi_default_tmp_dir() -> PathBuf { + if fs::metadata("/tmp").is_ok_and(|m| m.is_dir()) { + PathBuf::from("/tmp") + } else { + PathBuf::from(".") + } +} + /// Normalize a path by removing relative information /// For example, convert 'bar/../foo/bar.txt' => 'foo/bar.txt' /// copied from `` diff --git a/tests/by-util/test_cat.rs b/tests/by-util/test_cat.rs index e0baf5095b0..39973f8b5c3 100644 --- a/tests/by-util/test_cat.rs +++ b/tests/by-util/test_cat.rs @@ -87,6 +87,7 @@ fn test_no_options_big_input() { #[test] #[cfg(unix)] +#[cfg_attr(wasi_runner, ignore = "WASI: no FIFO/mkfifo support")] fn test_fifo_symlink() { use std::io::Write; use std::thread; @@ -138,6 +139,7 @@ fn test_closes_file_descriptors() { #[test] #[cfg(unix)] +#[cfg_attr(wasi_runner, ignore = "WASI: no pipe/signal support")] fn test_broken_pipe() { let mut cmd = new_ucmd!(); let mut child = cmd @@ -514,6 +516,7 @@ fn test_squeeze_blank_before_numbering() { /// This tests reading from Unix character devices #[test] #[cfg(unix)] +#[cfg_attr(wasi_runner, ignore = "WASI sandbox: host paths not visible")] fn test_dev_random() { #[cfg(any(target_os = "linux", target_os = "android"))] const DEV_RANDOM: &str = "/dev/urandom"; @@ -593,6 +596,7 @@ fn test_write_fast_fallthrough_uses_flush() { #[test] #[cfg(unix)] +#[cfg_attr(wasi_runner, ignore = "WASI: no Unix domain socket support")] fn test_domain_socket() { use std::os::unix::net::UnixListener; @@ -607,6 +611,7 @@ fn test_domain_socket() { } #[test] +#[cfg_attr(wasi_runner, ignore = "WASI sandbox: host paths not visible")] fn test_write_to_self_empty() { // it's ok if the input file is also the output file if it's empty let s = TestScenario::new(util_name!()); @@ -622,6 +627,7 @@ fn test_write_to_self_empty() { } #[test] +#[cfg_attr(wasi_runner, ignore = "WASI: cannot detect unsafe overwrite")] fn test_write_to_self() { let s = TestScenario::new(util_name!()); let file_path = s.fixtures.plus("first_file"); @@ -678,6 +684,7 @@ fn test_successful_write_to_read_write_self() { /// /// `cat fx fx3 1<>fx3` #[test] +#[cfg_attr(wasi_runner, ignore = "WASI: cannot detect unsafe overwrite")] fn test_failed_write_to_read_write_self() { let (at, mut ucmd) = at_and_ucmd!(); at.write("fx", "g"); @@ -703,6 +710,10 @@ fn test_failed_write_to_read_write_self() { #[test] #[cfg(unix)] #[cfg(not(target_os = "openbsd"))] +#[cfg_attr( + wasi_runner, + ignore = "WASI: symlink loop traversal does not surface ELOOP ('Too many levels of symbolic links')" +)] fn test_error_loop() { let (at, mut ucmd) = at_and_ucmd!(); at.symlink_file("2", "1"); diff --git a/tests/by-util/test_sort.rs b/tests/by-util/test_sort.rs index a762ec2cf6e..5b4ce2b1128 100644 --- a/tests/by-util/test_sort.rs +++ b/tests/by-util/test_sort.rs @@ -52,7 +52,10 @@ fn test_buffer_sizes() { .stdout_is_fixture("ext_sort.expected"); } - #[cfg(not(target_pointer_width = "32"))] + // The test runner compiles for the host (often 64-bit), but the binary + // under test may be 32-bit (e.g. wasm32-wasip1), which rejects very + // large buffer sizes. + #[cfg(all(not(target_pointer_width = "32"), not(wasi_runner)))] { let buffer_sizes = ["1000G", "10T"]; for buffer_size in &buffer_sizes { @@ -678,6 +681,7 @@ fn month_sort_input_expected(months: &[String]) -> (String, String) { #[test] #[cfg(unix)] +#[cfg_attr(wasi_runner, ignore = "WASI: no locale data")] fn test_month_sort_french_locale() { let locale = "fr_FR.UTF-8"; if !is_locale_available(locale) { @@ -709,6 +713,7 @@ fn test_month_sort_french_locale() { #[test] #[cfg(unix)] +#[cfg_attr(wasi_runner, ignore = "WASI: no locale data")] fn test_month_sort_hungarian_locale() { let locale = "hu_HU.UTF-8"; if !is_locale_available(locale) { @@ -740,6 +745,7 @@ fn test_month_sort_hungarian_locale() { /// E.g. "av ril" should NOT match "avril" — GNU treats it as unknown. #[test] #[cfg(unix)] +#[cfg_attr(wasi_runner, ignore = "WASI: no locale data")] fn test_month_sort_french_embedded_blanks() { let locale = "fr_FR.UTF-8"; if !is_locale_available(locale) { @@ -792,6 +798,7 @@ fn test_month_sort_french_embedded_blanks() { #[test] #[cfg(unix)] +#[cfg_attr(wasi_runner, ignore = "WASI: no locale data")] fn test_month_sort_japanese_locale() { let locale = "ja_JP.UTF-8"; if !is_locale_available(locale) { @@ -1440,6 +1447,7 @@ fn test_compress_merge() { #[test] #[cfg(not(target_os = "android"))] +#[cfg_attr(wasi_runner, ignore = "WASI: no subprocess spawning")] fn test_compress_fail() { let result = new_ucmd!() .args(&[ @@ -1648,6 +1656,7 @@ fn test_verifies_files_after_keys() { #[test] #[cfg(unix)] +#[cfg_attr(wasi_runner, ignore = "WASI sandbox: host paths not visible")] fn test_verifies_input_files() { new_ucmd!() .args(&["/dev/random", "nonexistent_file"]) @@ -1679,6 +1688,7 @@ fn test_output_is_input() { #[test] #[cfg(unix)] +#[cfg_attr(wasi_runner, ignore = "WASI sandbox: host paths not visible")] fn test_output_device() { new_ucmd!() .args(&["-o", "/dev/null"]) @@ -1712,6 +1722,7 @@ fn test_wrong_args_exit_code() { #[test] #[cfg(unix)] +#[cfg_attr(wasi_runner, ignore = "WASI: no pipe/signal support")] fn test_tmp_files_deleted_on_sigint() { use rand::{RngExt as _, SeedableRng, rngs::SmallRng}; use rustix::process::{Pid, Signal, kill_process}; @@ -1909,6 +1920,10 @@ fn test_files0_from_non_utf8_name() { #[test] #[cfg(unix)] +#[cfg_attr( + wasi_runner, + ignore = "WASI: opening a directory as a file reports EBADF instead of EISDIR" +)] fn test_files0_read_error() { new_ucmd!() .args(&["--files0-from", "."]) @@ -2993,6 +3008,7 @@ fn test_locale_collation_utf8() { } #[test] +#[cfg_attr(wasi_runner, ignore = "WASI: no locale data")] fn test_locale_interleaved_en_us_utf8() { // Test case for issue: locale-based collation support // In en_US.UTF-8, lowercase and uppercase letters should interleave @@ -3065,6 +3081,7 @@ fn test_locale_with_ignore_case_flag() { } #[test] +#[cfg_attr(wasi_runner, ignore = "WASI: no locale data")] fn test_locale_complex_utf8_sorting() { // More complex test with mixed case and special characters // In en_US.UTF-8, should respect locale collation rules @@ -3089,6 +3106,7 @@ fn test_locale_posix_sort_debug_message() { } #[test] +#[cfg_attr(wasi_runner, ignore = "WASI: no locale data")] fn test_locale_utf8_sort_debug_message() { new_ucmd!() .env("LC_ALL", "en_US.UTF-8") diff --git a/tests/by-util/test_tail.rs b/tests/by-util/test_tail.rs index ff63f3ae69a..1c95b24a5c9 100644 --- a/tests/by-util/test_tail.rs +++ b/tests/by-util/test_tail.rs @@ -250,6 +250,8 @@ fn test_nc_0_wo_follow2() { } #[test] +#[cfg(not(target_os = "windows"))] +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_n0_with_follow() { let (at, mut ucmd) = at_and_ucmd!(); let test_file = "test.txt"; @@ -372,6 +374,10 @@ fn test_stdin_redirect_dir() { // `test_stdin_redirect_dir` #[test] #[cfg(target_vendor = "apple")] +#[cfg_attr( + wasi_runner, + ignore = "WASI: directory redirected into stdin reports ENOENT rather than EISDIR" +)] fn test_stdin_redirect_dir_when_target_os_is_macos() { // $ mkdir dir // $ tail < dir, $ tail - < dir @@ -523,6 +529,8 @@ fn test_null_default() { } #[test] +#[cfg(not(target_os = "windows"))] // FIXME: test times out +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_single() { let (at, mut ucmd) = at_and_ucmd!(); @@ -553,6 +561,7 @@ fn test_follow_single() { /// Test for following when bytes are written that are not valid UTF-8. #[test] #[cfg(not(target_os = "windows"))] // FIXME: test times out +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_non_utf8_bytes() { // Tail the test file and start following it. let (at, mut ucmd) = at_and_ucmd!(); @@ -610,6 +619,8 @@ fn test_permission_denied_is_not_reported_as_not_found() { } #[test] +#[cfg(not(target_os = "windows"))] // FIXME: test times out +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_multiple() { let (at, mut ucmd) = at_and_ucmd!(); let mut child = ucmd @@ -645,6 +656,8 @@ fn test_follow_multiple() { } #[test] +#[cfg(not(target_os = "windows"))] // FIXME: test times out +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_name_multiple() { // spell-checker:disable-next-line for argument in ["--follow=name", "--follo=nam", "--f=n"] { @@ -690,6 +703,7 @@ fn test_follow_name_multiple() { } #[test] +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_multiple_untailable() { // $ tail -f DIR1 DIR2 // ==> DIR1 <== @@ -731,6 +745,7 @@ fn test_follow_stdin_pipe() { #[test] #[cfg(not(target_os = "windows"))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_invalid_pid() { new_ucmd!() .args(&["-f", "--pid=-1234"]) @@ -959,6 +974,7 @@ fn test_multiple_input_files_missing() { } #[test] +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_missing() { // Ensure that --follow=name does not imply --retry. // Ensure that --follow={descriptor,name} (without --retry) does *not wait* for the @@ -1027,6 +1043,7 @@ fn test_dir() { } #[test] +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_dir_follow() { let ts = TestScenario::new(util_name!()); let at = &ts.fixtures; @@ -1046,6 +1063,7 @@ fn test_dir_follow() { } #[test] +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_dir_follow_retry() { let ts = TestScenario::new(util_name!()); let at = &ts.fixtures; @@ -1513,6 +1531,7 @@ fn test_retry5() { // >X #[test] #[cfg(all(not(target_os = "windows"), not(target_os = "android")))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_retry6() { // inspired by: gnu/tests/tail-2/retry.sh // Ensure that --follow=descriptor (without --retry) does *not* try @@ -2027,6 +2046,7 @@ fn test_follow_name_retry_headers() { #[test] #[cfg(all(not(target_os = "windows"), not(target_os = "android")))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_name_remove() { // This test triggers a remove event while `tail --follow=name file` is running. // ((sleep 2 && rm file &)>/dev/null 2>&1 &) ; tail --follow=name file @@ -2086,7 +2106,12 @@ fn test_follow_name_remove() { } #[test] -#[cfg(all(not(target_os = "android"), not(target_os = "freebsd")))] // FIXME: for currently not working platforms +#[cfg(all( + not(target_os = "windows"), + not(target_os = "android"), + not(target_os = "freebsd") +))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_name_truncate1() { // This test triggers a truncate event while `tail --follow=name file` is running. // $ cp file backup && head file > file && sleep 1 && cp backup file @@ -2123,7 +2148,12 @@ fn test_follow_name_truncate1() { } #[test] -#[cfg(all(not(target_os = "android"), not(target_os = "freebsd")))] // FIXME: for currently not working platforms +#[cfg(all( + not(target_os = "windows"), + not(target_os = "android"), + not(target_os = "freebsd") +))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_name_truncate2() { // This test triggers a truncate event while `tail --follow=name file` is running. // $ ((sleep 1 && echo -n "x\nx\nx\n" >> file && sleep 1 && \ @@ -2166,6 +2196,8 @@ fn test_follow_name_truncate2() { } #[test] +#[cfg(not(target_os = "windows"))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_name_truncate3() { // Opening an empty file in truncate mode should not trigger a truncate event while // `tail --follow=name file` is running. @@ -2237,6 +2269,7 @@ fn test_follow_name_truncate4() { #[test] #[cfg(not(target_os = "windows"))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_truncate_fast() { // inspired by: "gnu/tests/tail-2/truncate.sh" // Ensure all logs are output upon file truncation @@ -3703,6 +3736,10 @@ fn test_when_argument_file_is_a_directory() { // TODO: make this work on windows #[test] #[cfg(unix)] +#[cfg_attr( + wasi_runner, + ignore = "WASI: symlink-to-directory error path differs from POSIX (ELOOP/EISDIR not reliably surfaced)" +)] fn test_when_argument_file_is_a_symlink() { let ts = TestScenario::new(util_name!()); let at = &ts.fixtures; @@ -3736,6 +3773,10 @@ fn test_when_argument_file_is_a_symlink() { // TODO: make this work on windows #[test] #[cfg(unix)] +#[cfg_attr( + wasi_runner, + ignore = "WASI: symlink-to-directory error path differs from POSIX (ELOOP/EISDIR not reliably surfaced)" +)] fn test_when_argument_file_is_a_symlink_to_directory_then_error() { let ts = TestScenario::new(util_name!()); let at = &ts.fixtures; @@ -4058,6 +4099,7 @@ fn test_when_follow_retry_then_initial_print_of_file_is_written_to_stdout() { // TODO: Add test for the warning `--pid=PID is not supported on this system` #[test] +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_args_when_settings_check_warnings_then_shows_warnings() { let scene = TestScenario::new(util_name!()); let at = &scene.fixtures; @@ -4250,6 +4292,7 @@ fn test_args_when_settings_check_warnings_follow_indefinitely_then_warning() { #[test] #[cfg(unix)] +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_args_when_settings_check_warnings_follow_indefinitely_then_no_warning() { let scene = TestScenario::new(util_name!()); let at = &scene.fixtures; @@ -4564,6 +4607,7 @@ fn test_follow_when_file_and_symlink_are_pointing_to_same_file_and_append_data() } #[test] +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_args_when_directory_given_shorthand_big_f_together_with_retry() { let scene = TestScenario::new(util_name!()); let at = &scene.fixtures; @@ -4951,6 +4995,7 @@ fn test_gnu_args_f() { #[test] #[cfg(unix)] +#[cfg_attr(wasi_runner, ignore = "WASI: argv/filenames must be valid UTF-8")] fn test_obsolete_encoding_unix() { use std::ffi::OsStr; use std::os::unix::ffi::OsStrExt; @@ -5061,6 +5106,7 @@ fn test_when_piped_input_then_no_broken_pipe() { #[test] #[cfg(unix)] +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_when_output_closed_then_no_broken_pipe() { let mut cmd = new_ucmd!(); let mut child = cmd @@ -5176,6 +5222,7 @@ fn test_follow_stdout_pipe_close() { } #[test] +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_debug_flag_with_polling() { let ts = TestScenario::new(util_name!()); let at = &ts.fixtures; diff --git a/tests/by-util/test_touch.rs b/tests/by-util/test_touch.rs index 2b8c0efc3d5..3bdb42fa5cf 100644 --- a/tests/by-util/test_touch.rs +++ b/tests/by-util/test_touch.rs @@ -166,6 +166,10 @@ fn test_touch_2_digit_years_2038() { } #[test] +#[cfg_attr( + wasi_runner, + ignore = "WASI: pre-epoch timestamps not representable by path_filestat_set_times" +)] fn test_touch_2_digit_years_69() { // 69 and after are 19xx let (at, mut ucmd) = at_and_ucmd!(); @@ -780,6 +784,10 @@ fn test_touch_mtime_dst_succeeds() { #[test] #[cfg(unix)] +#[cfg_attr( + wasi_runner, + ignore = "WASI: no tzdb; TZ env var is not honoured so DST validation is skipped" +)] fn test_touch_mtime_dst_fails() { let file = "test_touch_set_mtime_dst_fails"; @@ -797,6 +805,10 @@ fn test_touch_mtime_dst_fails() { #[test] #[cfg(unix)] +#[cfg_attr( + wasi_runner, + ignore = "WASI: guest root is a writable preopen, not the protected system root" +)] fn test_touch_system_fails() { let file = "/"; new_ucmd!() @@ -874,6 +886,7 @@ fn test_touch_no_such_file_error_msg() { #[test] #[cfg(not(any(target_os = "freebsd", target_os = "openbsd")))] +#[cfg_attr(wasi_runner, ignore = "WASI: touch - (stdout) is unsupported")] fn test_touch_changes_time_of_file_in_stdout() { // command like: `touch - 1< ./c` // should change the timestamp of c @@ -896,6 +909,10 @@ fn test_touch_changes_time_of_file_in_stdout() { #[test] #[cfg(unix)] +#[cfg_attr( + wasi_runner, + ignore = "WASI: filesystem permission errors surface as ENOENT rather than EACCES" +)] fn test_touch_permission_denied_error_msg() { let (at, mut ucmd) = at_and_ucmd!(); @@ -1016,6 +1033,7 @@ fn test_touch_no_dereference_dangling() { #[test] #[cfg(not(target_os = "openbsd"))] +#[cfg_attr(wasi_runner, ignore = "WASI: touch - (stdout) is unsupported")] fn test_touch_dash() { new_ucmd!().args(&["-h", "-"]).succeeds().no_output(); } From 8a511832a70e6e0c98a91b60f80297a1aad471f7 Mon Sep 17 00:00:00 2001 From: Anthony DePasquale Date: Mon, 13 Apr 2026 00:49:48 +0200 Subject: [PATCH 07/17] Annotate more WASI-incompatible tests caught on Linux --- docs/src/wasi-test-gaps.md | 8 ++++++++ tests/by-util/test_cat.rs | 9 ++++++++- tests/by-util/test_comm.rs | 3 +++ tests/by-util/test_sort.rs | 13 ++++++++++++- tests/by-util/test_tail.rs | 33 ++++++++++++++++++++++++++++++++- tests/by-util/test_touch.rs | 12 +++++++++++- 6 files changed, 74 insertions(+), 4 deletions(-) diff --git a/docs/src/wasi-test-gaps.md b/docs/src/wasi-test-gaps.md index af9aad321b8..a5fe08f1c89 100644 --- a/docs/src/wasi-test-gaps.md +++ b/docs/src/wasi-test-gaps.md @@ -68,6 +68,14 @@ The test harness maps the per-test working directory as the guest's `/`. That ma On WASI, `touch -` returns `UnsupportedPlatformFeature` because the guest cannot reliably locate the host file backing stdout. Tests that exercise `touch -` are skipped. +## WASI: rlimit/setrlimit not supported + +WASI has no concept of per-process resource limits, so `setrlimit` (and the `rlimit` crate that wraps it) has no effect. Tests that set `RLIMIT_NOFILE` to verify behaviour under restricted file-descriptor budgets are skipped. + +## WASI: sysinfo/meminfo not available + +WASI has no `sysinfo`/`/proc/meminfo` equivalent, so features that size buffers as a percentage of system memory (e.g. `sort -S 10%`) cannot resolve the limit and fail. Tests that exercise percentage-based sizing are skipped. + ## WASI: errno/error-message mismatches Several error paths surface different errno values (and therefore different error messages) through wasmtime than on POSIX. Observed cases: diff --git a/tests/by-util/test_cat.rs b/tests/by-util/test_cat.rs index 39973f8b5c3..17e923d0cff 100644 --- a/tests/by-util/test_cat.rs +++ b/tests/by-util/test_cat.rs @@ -122,6 +122,7 @@ fn test_fifo_symlink() { // TODO(#7542): Re-enable on Android once we figure out why setting limit is broken. // #[cfg(any(target_os = "linux", target_os = "android"))] #[cfg(target_os = "linux")] +#[cfg_attr(wasi_runner, ignore = "WASI: rlimit/setrlimit not supported")] fn test_closes_file_descriptors() { // Each file creates a pipe, which has two file descriptors. // If they are not closed then five is certainly too many. @@ -547,6 +548,7 @@ fn test_dev_random() { /// Wikipedia says there is support on Linux, FreeBSD, and `NetBSD`. #[test] #[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "netbsd"))] +#[cfg_attr(wasi_runner, ignore = "WASI sandbox: host paths not visible")] fn test_dev_full() { let mut proc = new_ucmd!() .set_stdout(Stdio::piped()) @@ -562,6 +564,7 @@ fn test_dev_full() { #[test] #[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "netbsd"))] +#[cfg_attr(wasi_runner, ignore = "WASI sandbox: host paths not visible")] fn test_dev_full_show_all() { let buf_len = 2048; let mut proc = new_ucmd!() @@ -584,6 +587,7 @@ fn test_dev_full_show_all() { // without additional flush output gets reversed. #[test] #[cfg(target_os = "linux")] +#[cfg_attr(wasi_runner, ignore = "WASI sandbox: host paths not visible")] fn test_write_fast_fallthrough_uses_flush() { const PROC_INIT_CMDLINE: &str = "/proc/1/cmdline"; let cmdline = read_to_string(PROC_INIT_CMDLINE).unwrap(); @@ -737,6 +741,7 @@ fn test_u_ignored() { #[test] #[cfg(unix)] +#[cfg_attr(wasi_runner, ignore = "WASI: errno/error-message mismatches")] fn test_write_fast_read_error() { use std::os::unix::fs::PermissionsExt; @@ -757,6 +762,7 @@ fn test_write_fast_read_error() { #[test] #[cfg(target_os = "linux")] +#[cfg_attr(wasi_runner, ignore = "WASI: argv/filenames must be valid UTF-8")] fn test_cat_non_utf8_paths() { use std::ffi::OsStr; use std::os::unix::ffi::OsStrExt; @@ -780,7 +786,8 @@ fn test_cat_non_utf8_paths() { } #[test] -#[cfg(unix)] +#[cfg(target_os = "linux")] +#[cfg_attr(wasi_runner, ignore = "WASI: cannot detect unsafe overwrite")] fn test_appending_same_input_output() { let (at, mut ucmd) = at_and_ucmd!(); diff --git a/tests/by-util/test_comm.rs b/tests/by-util/test_comm.rs index f0b7baed291..606446eff0c 100644 --- a/tests/by-util/test_comm.rs +++ b/tests/by-util/test_comm.rs @@ -450,6 +450,7 @@ fn test_is_dir() { } #[test] +#[cfg_attr(wasi_runner, ignore = "WASI: no locale data")] fn test_sorted() { let scene = TestScenario::new(util_name!()); let at = &scene.fixtures; @@ -481,6 +482,7 @@ fn test_sorted_check_order() { } #[test] +#[cfg_attr(wasi_runner, ignore = "WASI: no locale data")] fn test_both_inputs_out_of_order() { let scene = TestScenario::new(util_name!()); let at = &scene.fixtures; @@ -500,6 +502,7 @@ fn test_both_inputs_out_of_order() { } #[test] +#[cfg_attr(wasi_runner, ignore = "WASI: no locale data")] fn test_both_inputs_out_of_order_last_pair() { let scene = TestScenario::new(util_name!()); let at = &scene.fixtures; diff --git a/tests/by-util/test_sort.rs b/tests/by-util/test_sort.rs index 5b4ce2b1128..2aae70940d9 100644 --- a/tests/by-util/test_sort.rs +++ b/tests/by-util/test_sort.rs @@ -36,6 +36,7 @@ fn test_helper(file_name: &str, possible_args: &[&str]) { } #[test] +#[cfg_attr(wasi_runner, ignore = "WASI: sysinfo/meminfo not available")] fn test_buffer_sizes() { #[cfg(target_os = "linux")] let buffer_sizes = ["0", "50K", "50k", "1M", "100M", "0%", "10%"]; @@ -95,7 +96,9 @@ fn test_invalid_buffer_size() { // A percentage can fit in a u128 while its product with the total // physical memory does not; the parser must report it as too large // rather than panicking or silently wrapping. - #[cfg(target_os = "linux")] + // The test runner is built for Linux, but the binary under test may not + // expose Linux host memory information (e.g. wasm32-wasip1). + #[cfg(all(target_os = "linux", not(wasi_runner)))] new_ucmd!() .arg("-S") .arg("340282366920938463463374607431768211455%") @@ -1408,6 +1411,7 @@ fn sort_empty_chunk() { #[test] #[cfg(any(target_os = "linux", target_os = "android"))] +#[cfg_attr(wasi_runner, ignore = "WASI: no subprocess spawning")] fn test_compress() { new_ucmd!() .args(&[ @@ -1424,6 +1428,7 @@ fn test_compress() { #[test] #[cfg(any(target_os = "linux", target_os = "android"))] +#[cfg_attr(wasi_runner, ignore = "WASI: no subprocess spawning")] fn test_compress_merge() { new_ucmd!() .args(&[ @@ -1501,6 +1506,7 @@ fn test_batch_size_invalid() { } #[test] +#[cfg_attr(wasi_runner, ignore = "WASI: rlimit/setrlimit not supported")] fn test_batch_size_too_large() { let large_batch_size = "18446744073709551616"; new_ucmd!() @@ -1537,6 +1543,7 @@ fn test_merge_batch_size() { // TODO(#7542): Re-enable on Android once we figure out why setting limit is broken. // #[cfg(any(target_os = "linux", target_os = "android"))] #[cfg(target_os = "linux")] +#[cfg_attr(wasi_runner, ignore = "WASI: rlimit/setrlimit not supported")] fn test_merge_batch_size_with_limit() { use rlimit::Resource; // Currently need... @@ -1571,6 +1578,7 @@ fn test_sigpipe_panic() { #[test] #[cfg(unix)] +#[cfg_attr(wasi_runner, ignore = "WASI: no FIFO/mkfifo support")] fn test_fifo_without_trailing_newline() { let (at, mut ucmd) = at_and_ucmd!(); at.mkfifo("FIFO"); @@ -1933,6 +1941,7 @@ fn test_files0_read_error() { #[cfg(unix)] #[test] +#[cfg_attr(wasi_runner, ignore = "WASI sandbox: host paths not visible")] // Test for GNU tests/sort/sort-files0-from.pl "empty-non-regular" fn test_files0_from_empty_non_regular() { new_ucmd!() @@ -2020,6 +2029,7 @@ fn test_files0_from_2a() { #[test] // Test for GNU tests/sort/sort-files0-from.pl "non-utf8" #[cfg(all(unix, not(target_os = "macos")))] +#[cfg_attr(wasi_runner, ignore = "WASI: argv/filenames must be valid UTF-8")] fn test_files0_from_non_utf8() { use std::os::unix::ffi::OsStringExt; let (at, mut ucmd) = at_and_ucmd!(); @@ -3118,6 +3128,7 @@ fn test_locale_utf8_sort_debug_message() { #[test] #[cfg(unix)] +#[cfg_attr(wasi_runner, ignore = "WASI: no locale data")] fn test_failed_to_set_locale_debug_message() { let result = new_ucmd!() .env("LC_ALL", "not-valid-locale") diff --git a/tests/by-util/test_tail.rs b/tests/by-util/test_tail.rs index 1c95b24a5c9..b2d5d09b15d 100644 --- a/tests/by-util/test_tail.rs +++ b/tests/by-util/test_tail.rs @@ -275,6 +275,7 @@ fn test_n0_with_follow() { // TODO: Add similar test for windows #[test] #[cfg(unix)] +#[cfg_attr(wasi_runner, ignore = "WASI: errno/error-message mismatches")] fn test_permission_denied() { use std::os::unix::fs::PermissionsExt; @@ -295,6 +296,7 @@ fn test_permission_denied() { // TODO: Add similar test for windows #[test] #[cfg(unix)] +#[cfg_attr(wasi_runner, ignore = "WASI: errno/error-message mismatches")] fn test_permission_denied_multiple() { use std::os::unix::fs::PermissionsExt; @@ -343,6 +345,7 @@ fn test_follow_redirect_stdin_name_retry() { not(target_os = "freebsd"), not(target_os = "openbsd") ))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: errno/error-message mismatches")] fn test_stdin_redirect_dir() { // $ mkdir dir // $ tail < dir, $ tail - < dir @@ -777,6 +780,7 @@ fn test_follow_invalid_pid() { not(target_os = "android"), not(target_os = "freebsd") ))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_with_pid() { use std::process::Command; @@ -1326,6 +1330,7 @@ fn test_num_with_undocumented_sign_bytes() { #[test] #[cfg(unix)] +#[cfg_attr(wasi_runner, ignore = "WASI sandbox: host paths not visible")] fn test_bytes_for_funny_unix_files() { // inspired by: gnu/tests/tail-2/tail-c.sh let ts = TestScenario::new(util_name!()); @@ -1387,6 +1392,7 @@ fn test_retry2() { not(target_os = "freebsd"), not(target_os = "openbsd") ))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_retry3() { // inspired by: gnu/tests/tail-2/retry.sh // Ensure that `tail --retry --follow=name` waits for the file to appear. @@ -1432,6 +1438,7 @@ fn test_retry3() { not(target_os = "freebsd"), not(target_os = "openbsd") ))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_retry4() { // inspired by: gnu/tests/tail-2/retry.sh // Ensure that `tail --retry --follow=descriptor` waits for the file to appear. @@ -1490,6 +1497,7 @@ fn test_retry4() { not(target_os = "freebsd"), not(target_os = "openbsd") ))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_retry5() { // inspired by: gnu/tests/tail-2/retry.sh // Ensure that `tail --follow=descriptor --retry` exits when the file appears untailable. @@ -1581,6 +1589,7 @@ fn test_retry6() { not(target_os = "freebsd"), not(target_os = "openbsd") ))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_retry7() { // inspired by: gnu/tests/tail-2/retry.sh // Ensure that `tail -F` retries when the file is initially untailable. @@ -1654,6 +1663,7 @@ fn test_retry7() { #[test] #[cfg(unix)] #[cfg(not(target_os = "android"))] +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_name_replaced_by_symlink_is_untailable() { let ts = TestScenario::new(util_name!()); let at = &ts.fixtures; @@ -1712,6 +1722,7 @@ fn test_follow_name_replaced_by_symlink_is_untailable() { not(target_os = "freebsd"), not(target_os = "openbsd") ))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_retry8() { // Ensure that inotify will switch to polling mode if directory // of the watched file was initially missing and later created. @@ -1781,6 +1792,7 @@ fn test_retry8() { not(target_os = "freebsd"), not(target_os = "openbsd") ))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_retry9() { // inspired by: gnu/tests/tail-2/inotify-dir-recreate.sh // Ensure that inotify will switch to polling mode if directory @@ -1863,6 +1875,7 @@ fn test_retry9() { not(target_os = "freebsd"), not(target_os = "openbsd") ))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_descriptor_vs_rename1() { // inspired by: gnu/tests/tail-2/descriptor-vs-rename.sh // $ ((rm -f A && touch A && sleep 1 && echo -n "A\n" >> A && sleep 1 && \ @@ -1927,6 +1940,7 @@ fn test_follow_descriptor_vs_rename1() { not(target_os = "freebsd"), not(target_os = "openbsd") ))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_descriptor_vs_rename2() { // Ensure the headers are correct for --verbose. // NOTE: GNU's tail does not update the header from FILE_A to FILE_C after `mv FILE_A FILE_C` @@ -1980,6 +1994,7 @@ fn test_follow_descriptor_vs_rename2() { not(target_os = "freebsd"), not(target_os = "openbsd") ))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_name_retry_headers() { // inspired by: "gnu/tests/tail-2/F-headers.sh" // Ensure tail -F distinguishes output with the @@ -2234,6 +2249,7 @@ fn test_follow_name_truncate3() { not(target_os = "windows"), not(feature = "feat_selinux") // flaky ))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_name_truncate4() { // Truncating a file with the same content it already has should not trigger a truncate event @@ -2324,6 +2340,7 @@ fn test_follow_truncate_fast() { not(target_os = "freebsd"), not(target_os = "openbsd") ))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_name_move_create1() { // This test triggers a move/create event while `tail --follow=name file` is running. // ((sleep 2 && mv file backup && sleep 2 && cp backup file &)>/dev/null 2>&1 &) ; tail --follow=name file @@ -2380,6 +2397,7 @@ fn test_follow_name_move_create1() { not(target_os = "freebsd"), not(target_os = "openbsd") ))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_name_move_create2() { // inspired by: "gnu/tests/tail-2/inotify-hash-abuse.sh" // Exercise an abort-inducing flaw in inotify-enabled tail -F @@ -2460,6 +2478,7 @@ fn test_follow_name_move_create2() { not(target_os = "freebsd"), not(target_os = "openbsd") ))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_name_move1() { // This test triggers a move event while `tail --follow=name file` is running. // ((sleep 2 && mv file backup &)>/dev/null 2>&1 &) ; tail --follow=name file @@ -2522,6 +2541,7 @@ fn test_follow_name_move1() { not(target_os = "freebsd"), not(target_os = "openbsd") ))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_name_move2() { // Like test_follow_name_move1, but move to a name that's already monitored. @@ -2610,6 +2630,7 @@ fn test_follow_name_move2() { not(target_os = "freebsd"), not(target_os = "openbsd") ))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_name_move_retry1() { // Similar to test_follow_name_move1 but with `--retry` (`-F`) // This test triggers two move/rename events while `tail --follow=name --retry file` is running. @@ -2670,6 +2691,7 @@ fn test_follow_name_move_retry1() { not(target_os = "freebsd"), not(target_os = "openbsd") ))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_name_move_retry2() { // inspired by: "gnu/tests/tail-2/F-vs-rename.sh" // Similar to test_follow_name_move2 (move to a name that's already monitored) @@ -2849,6 +2871,7 @@ fn test_fifo() { not(target_os = "freebsd"), not(target_os = "openbsd") ))] +#[cfg_attr(wasi_runner, ignore = "WASI: no FIFO/mkfifo support")] fn test_fifo_with_pid() { use std::process::{Command, Stdio}; @@ -4168,6 +4191,7 @@ fn test_args_when_settings_check_warnings_then_shows_warnings() { /// TODO: Write similar tests for windows #[test] #[cfg(target_os = "linux")] +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_args_when_settings_check_warnings_follow_indefinitely_then_warning() { let scene = TestScenario::new(util_name!()); @@ -4668,6 +4692,7 @@ fn test_args_when_directory_given_shorthand_big_f_together_with_retry() { not(target_os = "openbsd"), not(feature = "feat_selinux") // flaky ))] +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_when_files_are_pointing_to_same_relative_file_and_file_stays_same_size() { let scene = TestScenario::new(util_name!()); let at = &scene.fixtures; @@ -5030,6 +5055,7 @@ fn test_obsolete_encoding_windows() { #[test] #[cfg(not(target_vendor = "apple"))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_following_with_pid() { use std::process::Command; @@ -5162,7 +5188,8 @@ fn test_failed_write_is_reported_on_seekable_input() { } #[test] -#[cfg(unix)] +#[cfg(target_os = "linux")] +#[cfg_attr(wasi_runner, ignore = "WASI sandbox: host paths not visible")] fn test_dev_zero() { new_ucmd!() .args(&["-c", "1", "/dev/zero"]) @@ -5207,6 +5234,7 @@ fn test_follow_pipe_f() { #[test] #[cfg(target_os = "linux")] +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_stdout_pipe_close() { let (at, mut ucmd) = at_and_ucmd!(); at.write("f", "line1\nline2\n"); @@ -5243,6 +5271,7 @@ fn test_debug_flag_with_polling() { #[test] #[cfg(target_os = "linux")] +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_debug_flag_with_inotify() { let ts = TestScenario::new(util_name!()); let at = &ts.fixtures; @@ -5260,6 +5289,7 @@ fn test_debug_flag_with_inotify() { #[test] #[cfg(target_os = "linux")] +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_dangling_symlink() { let (at, mut ucmd) = at_and_ucmd!(); at.symlink_file("target", "link"); @@ -5274,6 +5304,7 @@ fn test_follow_dangling_symlink() { #[test] #[cfg(target_os = "linux")] +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_symlink_target_change() { let (at, mut ucmd) = at_and_ucmd!(); at.write("t1", "A\n"); diff --git a/tests/by-util/test_touch.rs b/tests/by-util/test_touch.rs index 3bdb42fa5cf..9efd4ad129a 100644 --- a/tests/by-util/test_touch.rs +++ b/tests/by-util/test_touch.rs @@ -627,6 +627,10 @@ fn test_touch_set_date7() { } #[test] +#[cfg_attr( + wasi_runner, + ignore = "WASI: no tzdb; TZ env var is not honoured so timezone-dependent timestamps differ" +)] fn test_touch_set_date_without_leading_zeroes() { let (at, mut ucmd) = at_and_ucmd!(); let file = "test_touch_set_date_without_leading_zeroes"; @@ -649,7 +653,11 @@ fn test_touch_set_date_without_leading_zeroes() { /// expected by the old nix-based implementation. After switching to rustix /// (which uses i64 `tv_sec` natively), this should succeed on all targets. #[test] -#[cfg(unix)] +#[cfg(target_os = "linux")] +#[cfg_attr( + wasi_runner, + ignore = "WASI: pre-epoch timestamps not representable by path_filestat_set_times" +)] fn test_touch_set_date_year_zero() { let (at, mut ucmd) = at_and_ucmd!(); let file = "test_touch_year_zero"; @@ -1142,6 +1150,7 @@ fn test_touch_f_option() { #[test] #[cfg(target_os = "linux")] +#[cfg_attr(wasi_runner, ignore = "WASI: argv/filenames must be valid UTF-8")] fn test_touch_non_utf8_paths() { use std::ffi::OsStr; use std::os::unix::ffi::OsStrExt; @@ -1158,6 +1167,7 @@ fn test_touch_non_utf8_paths() { #[test] #[cfg(target_os = "linux")] +#[cfg_attr(wasi_runner, ignore = "WASI sandbox: host paths not visible")] fn test_touch_device_files() { let (_, mut ucmd) = at_and_ucmd!(); ucmd.args(&["/dev/null", "/dev/zero", "/dev/full", "/dev/random"]) From 9544216f1807dffd38fe5c7ccbc2b70ef2527bff Mon Sep 17 00:00:00 2001 From: Anthony DePasquale Date: Mon, 13 Apr 2026 01:36:21 +0200 Subject: [PATCH 08/17] Clarify WASI symlink and guest-path test gaps --- docs/src/wasi-test-gaps.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/src/wasi-test-gaps.md b/docs/src/wasi-test-gaps.md index a5fe08f1c89..9b200cf8902 100644 --- a/docs/src/wasi-test-gaps.md +++ b/docs/src/wasi-test-gaps.md @@ -32,9 +32,9 @@ WASI does not support spawning child processes. Tests that shell out to other co When stdin is a seekable file, wasmtime does not preserve the file position between the host and guest. Tests that validate stdin offset behavior after `head` reads are skipped. -## WASI: read_link fails under wasmtime via spawned test harness +## WASI: absolute symlink targets fail under wasmtime -When the WASI binary is spawned via `std::process::Command` from the cargo-test harness, `fs::read_link` (and operations that follow symlinks, such as opening a symlink to a FIFO or traversing a symlink loop) can return `EPERM` on absolute paths — paths that work when wasmtime is invoked directly. Individual symptom tests skipped under this umbrella are annotated with narrower reasons describing the observed errno mismatch. +Under wasmtime, symlinks whose stored target is an absolute guest path (for example `bar -> /foo`) fail in cases that work on POSIX: `readlink bar` exits 1 and `readlink -v bar` reports `Permission denied`, while the equivalent relative symlink (`bar -> foo`) succeeds. This reproduces even when `wasmtime` is invoked directly, so it is not specific to the cargo-test harness. `realpath` and symlink-heavy `cp` paths inherit the same limitation, and individual symptom tests under this umbrella are annotated with narrower reasons describing the observed errno mismatch. ## WASI: no Unix domain socket support @@ -62,7 +62,7 @@ wasi-libc does not ship tzdata, so `TZ` is not honoured and timezone-dependent v ## WASI: guest root is a writable preopen -The test harness maps the per-test working directory as the guest's `/`. That makes `/` writable inside the guest, so GNU-style protections against operating on the system root (e.g. `touch /` failing) cannot be exercised. Tests that assert these protections are skipped. +The test harness maps the per-test working directory as the guest's `/`. That makes `/` writable inside the guest, so GNU-style protections against operating on the system root (e.g. `touch /` failing) cannot be exercised. It also means guest-visible absolute paths are rooted at `/`, not at the host tempdir. Tests that compare against host `canonicalize()` results or pass host absolute paths into the guest (for example some `cp --parents`, `readlink`, `realpath`, `pwd`, and `ls` cases) need guest-aware expectations or separate coverage. Tests that assert the root-protection behaviour are skipped. ## WASI: `touch -` (stdout) unsupported From 5dfb20bfabcb19a241af2a4d831ed5a1793817c9 Mon Sep 17 00:00:00 2001 From: Anthony DePasquale Date: Mon, 13 Apr 2026 01:47:29 +0200 Subject: [PATCH 09/17] Fix cspell errors in CI --- docs/src/wasi-test-gaps.md | 8 +++++--- src/uu/cp/src/cp.rs | 4 ++-- src/uucore/src/lib/features/fs.rs | 2 +- tests/by-util/test_cat.rs | 2 +- tests/by-util/test_sort.rs | 2 +- tests/by-util/test_tail.rs | 2 +- tests/by-util/test_touch.rs | 2 +- 7 files changed, 12 insertions(+), 10 deletions(-) diff --git a/docs/src/wasi-test-gaps.md b/docs/src/wasi-test-gaps.md index 9b200cf8902..a7f660aa4e2 100644 --- a/docs/src/wasi-test-gaps.md +++ b/docs/src/wasi-test-gaps.md @@ -1,3 +1,5 @@ + + # WASI integration test gaps Tests annotated with `#[cfg_attr(wasi_runner, ignore = "...")]` are skipped when running integration tests against a WASI binary via wasmtime. This document tracks the reasons so that gaps in WASI support are visible in one place. @@ -46,7 +48,7 @@ The WASI sandbox does not ship locale data, so `setlocale`/`LC_ALL` have no effe ## WASI: tail follow mode disabled -`tail -f` / `tail -F` (follow mode) requires change-notification mechanisms (`inotify`, `kqueue`) and signal handling that WASI does not provide, so follow is disabled on WASI and a warning is emitted. Tests that exercise follow behaviour are skipped. +`tail -f` / `tail -F` (follow mode) requires change-notification mechanisms (`inotify`, `kqueue`) and signal handling that WASI does not provide, so follow is disabled on WASI and a warning is emitted. Tests that exercise follow behavior are skipped. ## WASI: cannot detect unsafe overwrite @@ -62,7 +64,7 @@ wasi-libc does not ship tzdata, so `TZ` is not honoured and timezone-dependent v ## WASI: guest root is a writable preopen -The test harness maps the per-test working directory as the guest's `/`. That makes `/` writable inside the guest, so GNU-style protections against operating on the system root (e.g. `touch /` failing) cannot be exercised. It also means guest-visible absolute paths are rooted at `/`, not at the host tempdir. Tests that compare against host `canonicalize()` results or pass host absolute paths into the guest (for example some `cp --parents`, `readlink`, `realpath`, `pwd`, and `ls` cases) need guest-aware expectations or separate coverage. Tests that assert the root-protection behaviour are skipped. +The test harness maps the per-test working directory as the guest's `/`. That makes `/` writable inside the guest, so GNU-style protections against operating on the system root (e.g. `touch /` failing) cannot be exercised. It also means guest-visible absolute paths are rooted at `/`, not at the host tempdir. Tests that compare against host `canonicalize()` results or pass host absolute paths into the guest (for example some `cp --parents`, `readlink`, `realpath`, `pwd`, and `ls` cases) need guest-aware expectations or separate coverage. Tests that assert the root-protection behavior are skipped. ## WASI: `touch -` (stdout) unsupported @@ -70,7 +72,7 @@ On WASI, `touch -` returns `UnsupportedPlatformFeature` because the guest cannot ## WASI: rlimit/setrlimit not supported -WASI has no concept of per-process resource limits, so `setrlimit` (and the `rlimit` crate that wraps it) has no effect. Tests that set `RLIMIT_NOFILE` to verify behaviour under restricted file-descriptor budgets are skipped. +WASI has no concept of per-process resource limits, so `setrlimit` (and the `rlimit` crate that wraps it) has no effect. Tests that set `RLIMIT_NOFILE` to verify behavior under restricted file-descriptor budgets are skipped. ## WASI: sysinfo/meminfo not available diff --git a/src/uu/cp/src/cp.rs b/src/uu/cp/src/cp.rs index ad6c3778d8a..f18db4b7c41 100644 --- a/src/uu/cp/src/cp.rs +++ b/src/uu/cp/src/cp.rs @@ -2,8 +2,8 @@ // // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -// spell-checker:ignore (ToDO) copydir ficlone fiemap ftruncate linkgs lstat nlink nlinks pathbuf pwrite reflink strs xattrs symlinked deduplicated advcpmv nushell IRWXG IRWXO IRWXU IRWXUGO IRWXU IRWXG IRWXO IRWXUGO sflag -// spell-checker:ignore RDONLY futimens utimensat +// spell-checker:ignore (ToDO) copydir ficlone fiemap filestat ftruncate linkgs lstat nlink nlinks pathbuf pwrite reflink strs utimensat xattrs symlinked deduplicated advcpmv nushell IRWXG IRWXO IRWXU IRWXUGO IRWXU IRWXG IRWXO IRWXUGO sflag +// spell-checker:ignore RDONLY futimens use std::cmp::Ordering; use std::collections::{HashMap, HashSet}; diff --git a/src/uucore/src/lib/features/fs.rs b/src/uucore/src/lib/features/fs.rs index 40f8cc24c34..5ccd6c0f4ff 100644 --- a/src/uucore/src/lib/features/fs.rs +++ b/src/uucore/src/lib/features/fs.rs @@ -5,7 +5,7 @@ //! Set of functions to manage regular files, special files, and links. -// spell-checker:ignore backport Ioctl absolutized +// spell-checker:ignore backport Ioctl absolutized preopen #[cfg(all(unix, not(target_os = "redox")))] pub use libc::{major, makedev, minor}; diff --git a/tests/by-util/test_cat.rs b/tests/by-util/test_cat.rs index 17e923d0cff..1158078e9d1 100644 --- a/tests/by-util/test_cat.rs +++ b/tests/by-util/test_cat.rs @@ -2,7 +2,7 @@ // // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -// spell-checker:ignore NOFILE nonewline cmdline +// spell-checker:ignore NOFILE nonewline cmdline setrlimit ELOOP #[cfg(any(target_os = "linux", target_os = "android"))] use rlimit::Resource; diff --git a/tests/by-util/test_sort.rs b/tests/by-util/test_sort.rs index 2aae70940d9..60bdfd24dbb 100644 --- a/tests/by-util/test_sort.rs +++ b/tests/by-util/test_sort.rs @@ -3,7 +3,7 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -// spell-checker:ignore (words) ints (linux) NOFILE dfgi abmon avril +// spell-checker:ignore (words) ints (linux) NOFILE dfgi abmon avril setrlimit EISDIR #![allow(clippy::cast_possible_wrap)] use std::env; diff --git a/tests/by-util/test_tail.rs b/tests/by-util/test_tail.rs index b2d5d09b15d..2065116e897 100644 --- a/tests/by-util/test_tail.rs +++ b/tests/by-util/test_tail.rs @@ -4,7 +4,7 @@ // file that was distributed with this source code. // spell-checker:ignore (ToDO) abcdefghijklmnopqrstuvwxyz efghijklmnopqrstuvwxyz vwxyz emptyfile file siette ocho nueve diez MULT -// spell-checker:ignore (libs) kqueue +// spell-checker:ignore (libs) kqueue ELOOP EISDIR // spell-checker:ignore (jargon) tailable untailable datasame runneradmin tmpi // spell-checker:ignore (cmd) taskkill #![allow( diff --git a/tests/by-util/test_touch.rs b/tests/by-util/test_touch.rs index 9efd4ad129a..4cc27cba528 100644 --- a/tests/by-util/test_touch.rs +++ b/tests/by-util/test_touch.rs @@ -2,7 +2,7 @@ // // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -// spell-checker:ignore (formats) cymdhm cymdhms datetime mdhm mdhms mktime strtime ymdhm ymdhms +// spell-checker:ignore (formats) cymdhm cymdhms datetime filestat mdhm mdhms mktime preopen strtime tzdb ymdhm ymdhms use filetime::FileTime; #[cfg(not(target_os = "freebsd"))] From cc9539f2d19a9600cf8fb3b57070f4a2c11f15b0 Mon Sep 17 00:00:00 2001 From: Anthony DePasquale Date: Mon, 13 Apr 2026 03:14:36 +0200 Subject: [PATCH 10/17] tail: ignore test_gnu_args_f under wasi_runner --- tests/by-util/test_tail.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/by-util/test_tail.rs b/tests/by-util/test_tail.rs index 2065116e897..9f7649fb50f 100644 --- a/tests/by-util/test_tail.rs +++ b/tests/by-util/test_tail.rs @@ -4999,6 +4999,7 @@ fn test_gnu_args_err() { } #[test] +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_gnu_args_f() { let scene = TestScenario::new(util_name!()); let at = &scene.fixtures; From 2dfc7caba2b72ac3ab9a0b34463c8823e59d9e00 Mon Sep 17 00:00:00 2001 From: Anthony DePasquale Date: Mon, 13 Apr 2026 03:58:45 +0200 Subject: [PATCH 11/17] tail: ignore test_follow_inotify_only_regular under wasi_runner --- tests/by-util/test_tail.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/by-util/test_tail.rs b/tests/by-util/test_tail.rs index 9f7649fb50f..e900f5ce718 100644 --- a/tests/by-util/test_tail.rs +++ b/tests/by-util/test_tail.rs @@ -2792,6 +2792,7 @@ fn test_follow_name_move_retry2() { #[test] #[cfg(not(target_os = "windows"))] // FIXME: for currently not working platforms +#[cfg_attr(wasi_runner, ignore = "WASI: tail follow mode disabled")] fn test_follow_inotify_only_regular() { // The GNU test inotify-only-regular.sh uses strace to ensure that `tail -f` // doesn't make inotify syscalls and only uses inotify for regular files or fifos. From 64b3e8358a209886667124d0bba8bc059925e6b2 Mon Sep 17 00:00:00 2001 From: Anthony DePasquale Date: Tue, 21 Apr 2026 01:03:01 +0200 Subject: [PATCH 12/17] sort: isolate WASI sync code into sibling modules --- src/uu/sort/src/check.rs | 233 ------- src/uu/sort/src/check/mod.rs | 44 ++ src/uu/sort/src/check/sync.rs | 98 +++ src/uu/sort/src/check/threaded.rs | 119 ++++ src/uu/sort/src/ext_sort/mod.rs | 46 +- src/uu/sort/src/ext_sort/sync.rs | 179 ++++++ src/uu/sort/src/ext_sort/threaded.rs | 212 +------ src/uu/sort/src/merge.rs | 869 --------------------------- src/uu/sort/src/merge/mod.rs | 409 +++++++++++++ src/uu/sort/src/merge/sync.rs | 207 +++++++ src/uu/sort/src/merge/threaded.rs | 277 +++++++++ src/uu/sort/src/parallel.rs | 59 ++ src/uu/sort/src/sort.rs | 28 +- 13 files changed, 1446 insertions(+), 1334 deletions(-) delete mode 100644 src/uu/sort/src/check.rs create mode 100644 src/uu/sort/src/check/mod.rs create mode 100644 src/uu/sort/src/check/sync.rs create mode 100644 src/uu/sort/src/check/threaded.rs create mode 100644 src/uu/sort/src/ext_sort/sync.rs delete mode 100644 src/uu/sort/src/merge.rs create mode 100644 src/uu/sort/src/merge/mod.rs create mode 100644 src/uu/sort/src/merge/sync.rs create mode 100644 src/uu/sort/src/merge/threaded.rs create mode 100644 src/uu/sort/src/parallel.rs diff --git a/src/uu/sort/src/check.rs b/src/uu/sort/src/check.rs deleted file mode 100644 index 7569a078d73..00000000000 --- a/src/uu/sort/src/check.rs +++ /dev/null @@ -1,233 +0,0 @@ -// This file is part of the uutils coreutils package. -// -// For the full copyright and license information, please view the LICENSE -// file that was distributed with this source code. - -//! Check if a file is ordered - -use crate::{ - GlobalSettings, SortError, - chunks::{self, Chunk, RecycledChunk}, - compare_by, open, -}; -use itertools::Itertools; -#[cfg(not(wasi_no_threads))] -use std::sync::mpsc::{Receiver, SyncSender, sync_channel}; -#[cfg(not(wasi_no_threads))] -use std::thread; -use std::{cmp::Ordering, ffi::OsStr, io::Read, iter}; -use uucore::error::UResult; - -/// Check if the file at `path` is ordered. -/// -/// # Returns -/// -/// The code we should exit with. -pub fn check(path: &OsStr, settings: &GlobalSettings) -> UResult<()> { - let max_allowed_cmp = if settings.unique { - Ordering::Less - } else { - Ordering::Equal - }; - let file = open(path)?; - let chunk_size = if settings.buffer_size < 100 * 1024 { - settings.buffer_size - } else { - 100 * 1024 - }; - - #[cfg(not(wasi_no_threads))] - { - check_threaded(path, settings, max_allowed_cmp, file, chunk_size) - } - #[cfg(wasi_no_threads)] - { - check_sync(path, settings, max_allowed_cmp, file, chunk_size) - } -} - -#[cfg(not(wasi_no_threads))] -fn check_threaded( - path: &OsStr, - settings: &GlobalSettings, - max_allowed_cmp: Ordering, - file: Box, - chunk_size: usize, -) -> UResult<()> { - let (recycled_sender, recycled_receiver) = sync_channel(2); - let (loaded_sender, loaded_receiver) = sync_channel(2); - thread::spawn({ - let settings = settings.clone(); - move || reader(file, &recycled_receiver, &loaded_sender, &settings) - }); - for _ in 0..2 { - let _ = recycled_sender.send(RecycledChunk::new(chunk_size)); - } - - let mut prev_chunk: Option = None; - let mut line_idx = 0; - let mut result: UResult<()> = Ok(()); - // Note that we iterate over a reference, so that `loaded_receiver` is still alive - // once we stop: `chunks::read` unwraps its `send`, so dropping our end while the - // reader thread is still going would panic it. Since we stop at the *first* - // disorder, the reader is usually still working at that point, so we shut it down - // in an orderly fashion below instead of just dropping our end. - 'outer: for chunk in &loaded_receiver { - line_idx += 1; - if let Some(prev_chunk) = prev_chunk.take() { - let prev_last = prev_chunk.lines().last().unwrap(); - let new_first = chunk.lines().first().unwrap(); - - if compare_by( - prev_last, - new_first, - settings, - prev_chunk.line_data(), - chunk.line_data(), - ) > max_allowed_cmp - { - result = Err(SortError::Disorder { - file: path.to_owned(), - line_number: line_idx, - line: String::from_utf8_lossy(new_first.line).into_owned(), - silent: settings.check_silent, - } - .into()); - break 'outer; - } - let _ = recycled_sender.send(prev_chunk.recycle()); - } - - for (a, b) in chunk.lines().iter().tuple_windows() { - line_idx += 1; - if compare_by(a, b, settings, chunk.line_data(), chunk.line_data()) > max_allowed_cmp { - result = Err(SortError::Disorder { - file: path.to_owned(), - line_number: line_idx, - line: String::from_utf8_lossy(b.line).into_owned(), - silent: settings.check_silent, - } - .into()); - break 'outer; - } - } - - prev_chunk = Some(chunk); - } - - // Stop handing out buffers, so the reader runs out of work, then drain anything it - // has already produced. This lets its in-flight `send` complete instead of failing, - // and terminates because the reader can only own the (at most two) recycled chunks - // that are still outstanding. - drop(recycled_sender); - while loaded_receiver.recv().is_ok() {} - - result -} - -/// The function running on the reader thread. -#[cfg(not(wasi_no_threads))] -fn reader( - mut file: Box, - receiver: &Receiver, - sender: &SyncSender, - settings: &GlobalSettings, -) -> UResult<()> { - let mut carry_over = vec![]; - for recycled_chunk in receiver { - let should_continue = chunks::read( - sender, - recycled_chunk, - None, - &mut carry_over, - &mut file, - &mut iter::empty(), - settings.line_ending.into(), - settings, - )?; - if !should_continue { - break; - } - } - Ok(()) -} - -/// Synchronous check for targets without thread support. -#[cfg(wasi_no_threads)] -fn check_sync( - path: &OsStr, - settings: &GlobalSettings, - max_allowed_cmp: Ordering, - mut file: Box, - chunk_size: usize, -) -> UResult<()> { - let separator = settings.line_ending.into(); - let mut carry_over = vec![]; - let mut prev_chunk: Option = None; - let mut spare_recycled: Option = None; - let mut line_idx = 0; - - loop { - let recycled = spare_recycled - .take() - .unwrap_or_else(|| RecycledChunk::new(chunk_size)); - - let (chunk, should_continue) = chunks::read_to_chunk( - recycled, - None, - &mut carry_over, - &mut file, - &mut iter::empty(), - separator, - settings, - )?; - - let Some(chunk) = chunk else { - break; - }; - - line_idx += 1; - if let Some(prev) = prev_chunk.take() { - let prev_last = prev.lines().last().unwrap(); - let new_first = chunk.lines().first().unwrap(); - - if compare_by( - prev_last, - new_first, - settings, - prev.line_data(), - chunk.line_data(), - ) > max_allowed_cmp - { - return Err(SortError::Disorder { - file: path.to_owned(), - line_number: line_idx, - line: String::from_utf8_lossy(new_first.line).into_owned(), - silent: settings.check_silent, - } - .into()); - } - spare_recycled = Some(prev.recycle()); - } - - for (a, b) in chunk.lines().iter().tuple_windows() { - line_idx += 1; - if compare_by(a, b, settings, chunk.line_data(), chunk.line_data()) > max_allowed_cmp { - return Err(SortError::Disorder { - file: path.to_owned(), - line_number: line_idx, - line: String::from_utf8_lossy(b.line).into_owned(), - silent: settings.check_silent, - } - .into()); - } - } - - prev_chunk = Some(chunk); - - if !should_continue { - break; - } - } - Ok(()) -} diff --git a/src/uu/sort/src/check/mod.rs b/src/uu/sort/src/check/mod.rs new file mode 100644 index 00000000000..79226660e84 --- /dev/null +++ b/src/uu/sort/src/check/mod.rs @@ -0,0 +1,44 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +//! Check if a file is ordered. +//! +//! On most platforms this uses a multi-threaded reader. On WASI without +//! atomics, a synchronous variant is used instead. The two implementations +//! live in sibling modules and are selected via cfg at the module boundary. + +use std::cmp::Ordering; +use std::ffi::OsStr; + +use uucore::error::UResult; + +use crate::{GlobalSettings, open}; + +#[cfg(not(wasi_no_threads))] +mod threaded; +#[cfg(not(wasi_no_threads))] +use threaded as runner; + +#[cfg(wasi_no_threads)] +mod sync; +#[cfg(wasi_no_threads)] +use sync as runner; + +/// Check if the file at `path` is ordered. +pub fn check(path: &OsStr, settings: &GlobalSettings) -> UResult<()> { + let max_allowed_cmp = if settings.unique { + Ordering::Less + } else { + Ordering::Equal + }; + let file = open(path)?; + let chunk_size = if settings.buffer_size < 100 * 1024 { + settings.buffer_size + } else { + 100 * 1024 + }; + + runner::check(path, settings, max_allowed_cmp, file, chunk_size) +} diff --git a/src/uu/sort/src/check/sync.rs b/src/uu/sort/src/check/sync.rs new file mode 100644 index 00000000000..1cb338caacb --- /dev/null +++ b/src/uu/sort/src/check/sync.rs @@ -0,0 +1,98 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +//! Synchronous ordered-file check for targets without thread support. + +use std::cmp::Ordering; +use std::ffi::OsStr; +use std::io::Read; +use std::iter; + +use itertools::Itertools; +use uucore::error::UResult; + +use crate::{ + GlobalSettings, SortError, + chunks::{self, Chunk, RecycledChunk}, + compare_by, +}; + +pub(super) fn check( + path: &OsStr, + settings: &GlobalSettings, + max_allowed_cmp: Ordering, + mut file: Box, + chunk_size: usize, +) -> UResult<()> { + let separator = settings.line_ending.into(); + let mut carry_over = vec![]; + let mut prev_chunk: Option = None; + let mut spare_recycled: Option = None; + let mut line_idx = 0; + + loop { + let recycled = spare_recycled + .take() + .unwrap_or_else(|| RecycledChunk::new(chunk_size)); + + let (chunk, should_continue) = chunks::read_to_chunk( + recycled, + None, + &mut carry_over, + &mut file, + &mut iter::empty(), + separator, + settings, + )?; + + let Some(chunk) = chunk else { + break; + }; + + line_idx += 1; + if let Some(prev) = prev_chunk.take() { + let prev_last = prev.lines().last().unwrap(); + let new_first = chunk.lines().first().unwrap(); + + if compare_by( + prev_last, + new_first, + settings, + prev.line_data(), + chunk.line_data(), + ) > max_allowed_cmp + { + return Err(SortError::Disorder { + file: path.to_owned(), + line_number: line_idx, + line: String::from_utf8_lossy(new_first.line).into_owned(), + silent: settings.check_silent, + } + .into()); + } + spare_recycled = Some(prev.recycle()); + } + + for (a, b) in chunk.lines().iter().tuple_windows() { + line_idx += 1; + if compare_by(a, b, settings, chunk.line_data(), chunk.line_data()) > max_allowed_cmp { + return Err(SortError::Disorder { + file: path.to_owned(), + line_number: line_idx, + line: String::from_utf8_lossy(b.line).into_owned(), + silent: settings.check_silent, + } + .into()); + } + } + + prev_chunk = Some(chunk); + + if !should_continue { + break; + } + } + Ok(()) +} diff --git a/src/uu/sort/src/check/threaded.rs b/src/uu/sort/src/check/threaded.rs new file mode 100644 index 00000000000..5f2ac6e3ad8 --- /dev/null +++ b/src/uu/sort/src/check/threaded.rs @@ -0,0 +1,119 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +//! Multi-threaded ordered-file check: a reader thread streams chunks while +//! the main thread compares the boundary between consecutive chunks. + +use std::cmp::Ordering; +use std::ffi::OsStr; +use std::io::Read; +use std::iter; +use std::sync::mpsc::{Receiver, SyncSender, sync_channel}; +use std::thread; + +use itertools::Itertools; +use uucore::error::UResult; + +use crate::{ + GlobalSettings, SortError, + chunks::{self, Chunk, RecycledChunk}, + compare_by, +}; + +pub(super) fn check( + path: &OsStr, + settings: &GlobalSettings, + max_allowed_cmp: Ordering, + file: Box, + chunk_size: usize, +) -> UResult<()> { + let (recycled_sender, recycled_receiver) = sync_channel(2); + let (loaded_sender, loaded_receiver) = sync_channel(2); + thread::spawn({ + let settings = settings.clone(); + move || reader(file, &recycled_receiver, &loaded_sender, &settings) + }); + for _ in 0..2 { + let _ = recycled_sender.send(RecycledChunk::new(chunk_size)); + } + + let mut prev_chunk: Option = None; + let mut line_idx = 0; + let mut result: UResult<()> = Ok(()); + // Keep the receiver alive after the first disorder so the reader's in-flight + // send can complete while the channel is drained below. + 'outer: for chunk in &loaded_receiver { + line_idx += 1; + if let Some(prev_chunk) = prev_chunk.take() { + let prev_last = prev_chunk.lines().last().unwrap(); + let new_first = chunk.lines().first().unwrap(); + + if compare_by( + prev_last, + new_first, + settings, + prev_chunk.line_data(), + chunk.line_data(), + ) > max_allowed_cmp + { + result = Err(SortError::Disorder { + file: path.to_owned(), + line_number: line_idx, + line: String::from_utf8_lossy(new_first.line).into_owned(), + silent: settings.check_silent, + } + .into()); + break 'outer; + } + let _ = recycled_sender.send(prev_chunk.recycle()); + } + + for (a, b) in chunk.lines().iter().tuple_windows() { + line_idx += 1; + if compare_by(a, b, settings, chunk.line_data(), chunk.line_data()) > max_allowed_cmp { + result = Err(SortError::Disorder { + file: path.to_owned(), + line_number: line_idx, + line: String::from_utf8_lossy(b.line).into_owned(), + silent: settings.check_silent, + } + .into()); + break 'outer; + } + } + + prev_chunk = Some(chunk); + } + drop(recycled_sender); + while loaded_receiver.recv().is_ok() {} + + result +} + +/// The function running on the reader thread. +fn reader( + mut file: Box, + receiver: &Receiver, + sender: &SyncSender, + settings: &GlobalSettings, +) -> UResult<()> { + let mut carry_over = vec![]; + for recycled_chunk in receiver { + let should_continue = chunks::read( + sender, + recycled_chunk, + None, + &mut carry_over, + &mut file, + &mut iter::empty(), + settings.line_ending.into(), + settings, + )?; + if !should_continue { + break; + } + } + Ok(()) +} diff --git a/src/uu/sort/src/ext_sort/mod.rs b/src/uu/sort/src/ext_sort/mod.rs index e20452f7e8c..f93082cfbb1 100644 --- a/src/uu/sort/src/ext_sort/mod.rs +++ b/src/uu/sort/src/ext_sort/mod.rs @@ -6,8 +6,50 @@ //! External sort: sort large inputs that may not fit in memory. //! //! On most platforms this uses a multi-threaded chunked approach with -//! temporary files. On WASI without atomics, synchronous fallbacks are -//! used instead (selected via `cfg` guards inside the module). +//! temporary files. On WASI without atomics, a synchronous variant is used +//! instead. The two implementations live in sibling modules and are selected +//! via cfg at the module boundary. +use std::fs::File; +use std::io::Write; +use std::path::PathBuf; + +use uucore::error::UResult; + +use crate::Line; +use crate::chunks::Chunk; +use crate::merge::WriteableTmpFile; + +#[cfg(not(wasi_no_threads))] mod threaded; +#[cfg(not(wasi_no_threads))] pub use threaded::ext_sort; + +#[cfg(wasi_no_threads)] +mod sync; +#[cfg(wasi_no_threads)] +pub use sync::ext_sort; + +// Note: update `test_sort::test_start_buffer` if this size is changed +// Fixed to 8 KiB (equivalent to `std::sys::io::DEFAULT_BUF_SIZE` on most targets) +pub(super) const DEFAULT_BUF_SIZE: usize = 8 * 1024; + +/// Write the lines in `chunk` to `file`, separated by `separator`. +/// `compress_prog` is used to optionally compress file contents. +pub(super) fn write( + chunk: &Chunk, + file: (File, PathBuf), + compress_prog: Option<&str>, + separator: u8, +) -> UResult { + let mut tmp_file = I::create(file, compress_prog)?; + write_lines(chunk.lines(), tmp_file.as_write(), separator); + tmp_file.finished_writing() +} + +fn write_lines(lines: &[Line], writer: &mut T, separator: u8) { + for s in lines { + writer.write_all(s.line).unwrap(); + writer.write_all(&[separator]).unwrap(); + } +} diff --git a/src/uu/sort/src/ext_sort/sync.rs b/src/uu/sort/src/ext_sort/sync.rs new file mode 100644 index 00000000000..6487fb7f721 --- /dev/null +++ b/src/uu/sort/src/ext_sort/sync.rs @@ -0,0 +1,179 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +//! Synchronous external sort for targets without thread support +//! (e.g. `wasm32-wasip1`). +//! +//! Uses the same chunked sort-write-merge strategy as the threaded version, +//! but reads and sorts each chunk sequentially on the calling thread. + +use std::cmp::Ordering; +use std::io::{Read, Write, stderr}; + +use itertools::Itertools; +use uucore::error::UResult; + +use crate::Output; +use crate::chunks::{self, Chunk, RecycledChunk}; +use crate::merge::{self, WriteablePlainTmpFile, WriteableTmpFile}; +use crate::tmp_dir::TmpDirWrapper; +use crate::{GlobalSettings, compare_by, print_sorted, sort_by}; + +use super::{DEFAULT_BUF_SIZE, write}; + +pub fn ext_sort( + files: &mut impl Iterator>>, + settings: &GlobalSettings, + output: Output, + tmp_dir: &mut TmpDirWrapper, +) -> UResult<()> { + let separator = settings.line_ending.into(); + let mut buffer_size = match settings.buffer_size { + size if size <= 512 * 1024 * 1024 => size, + size => size / 2, + }; + if !settings.buffer_size_is_explicit { + buffer_size = buffer_size.max(8 * 1024 * 1024); + } + + if settings.compress_prog.is_some() { + let _ = writeln!( + stderr(), + "sort: warning: --compress-program is ignored on this platform" + ); + } + + let mut file = files.next().unwrap()?; + let mut carry_over = vec![]; + + // Read and sort first chunk. + let (first, cont) = chunks::read_to_chunk( + RecycledChunk::new(buffer_size.min(DEFAULT_BUF_SIZE)), + Some(buffer_size), + &mut carry_over, + &mut file, + files, + separator, + settings, + )?; + let Some(mut first) = first else { + return Ok(()); // empty input + }; + first.with_dependent_mut(|_, c| sort_by(&mut c.lines, settings, &c.line_data)); + + if !cont { + // All input fits in one chunk. + return print_chunk(&first, settings, output); + } + + // Read and sort second chunk. + let (second, cont) = chunks::read_to_chunk( + RecycledChunk::new(buffer_size.min(DEFAULT_BUF_SIZE)), + Some(buffer_size), + &mut carry_over, + &mut file, + files, + separator, + settings, + )?; + let Some(mut second) = second else { + return print_chunk(&first, settings, output); + }; + second.with_dependent_mut(|_, c| sort_by(&mut c.lines, settings, &c.line_data)); + + if !cont { + // All input fits in two chunks — merge in memory. + return print_two_chunks(first, second, settings, output); + } + + // More than two chunks: write sorted chunks to temp files, then merge. + let mut tmp_files: Vec<::Closed> = vec![]; + + tmp_files.push(write::( + &first, + tmp_dir.next_file()?, + settings.compress_prog.as_deref(), + separator, + )?); + drop(first); + + tmp_files.push(write::( + &second, + tmp_dir.next_file()?, + settings.compress_prog.as_deref(), + separator, + )?); + let mut recycled = second.recycle(); + + loop { + let (chunk, cont) = chunks::read_to_chunk( + recycled, + None, + &mut carry_over, + &mut file, + files, + separator, + settings, + )?; + let Some(mut chunk) = chunk else { break }; + chunk.with_dependent_mut(|_, c| sort_by(&mut c.lines, settings, &c.line_data)); + tmp_files.push(write::( + &chunk, + tmp_dir.next_file()?, + settings.compress_prog.as_deref(), + separator, + )?); + recycled = chunk.recycle(); + if !cont { + break; + } + } + + merge::merge_with_file_limit::<_, _, WriteablePlainTmpFile>( + tmp_files.into_iter().map(merge::ClosedTmpFile::reopen), + settings, + output, + tmp_dir, + ) +} + +/// Print a single sorted chunk. +fn print_chunk(chunk: &Chunk, settings: &GlobalSettings, output: Output) -> UResult<()> { + if settings.unique { + print_sorted( + chunk.lines().iter().dedup_by(|a, b| { + compare_by(a, b, settings, chunk.line_data(), chunk.line_data()) == Ordering::Equal + }), + settings, + output, + ) + } else { + print_sorted(chunk.lines().iter(), settings, output) + } +} + +/// Merge two in-memory chunks and print. +fn print_two_chunks(a: Chunk, b: Chunk, settings: &GlobalSettings, output: Output) -> UResult<()> { + let merged_iter = a.lines().iter().map(|line| (line, &a)).merge_by( + b.lines().iter().map(|line| (line, &b)), + |(line_a, a), (line_b, b)| { + compare_by(line_a, line_b, settings, a.line_data(), b.line_data()) != Ordering::Greater + }, + ); + if settings.unique { + print_sorted( + merged_iter + .dedup_by(|(line_a, a), (line_b, b)| { + compare_by(line_a, line_b, settings, a.line_data(), b.line_data()) + == Ordering::Equal + }) + .map(|(line, _)| line), + settings, + output, + ) + } else { + print_sorted(merged_iter.map(|(line, _)| line), settings, output) + } +} diff --git a/src/uu/sort/src/ext_sort/threaded.rs b/src/uu/sort/src/ext_sort/threaded.rs index 2f24bb7a14a..d0d0d02ddb8 100644 --- a/src/uu/sort/src/ext_sort/threaded.rs +++ b/src/uu/sort/src/ext_sort/threaded.rs @@ -7,204 +7,26 @@ //! thread, and spill to temporary files when memory is exceeded. use std::cmp::Ordering; -use std::fs::File; use std::io::{Read, Write, stderr}; -use std::path::PathBuf; -#[cfg(not(wasi_no_threads))] use std::sync::mpsc::{Receiver, SyncSender}; -#[cfg(not(wasi_no_threads))] use std::thread; use itertools::Itertools; -use uucore::error::UResult; -#[cfg(not(wasi_no_threads))] -use uucore::error::strip_errno; +use uucore::error::{UResult, strip_errno}; use crate::Output; -use crate::chunks::RecycledChunk; -#[cfg(not(wasi_no_threads))] -use crate::merge::WriteableCompressedTmpFile; -use crate::merge::WriteablePlainTmpFile; -use crate::merge::WriteableTmpFile; +use crate::chunks::{self, Chunk, RecycledChunk}; +use crate::merge::{self, WriteableCompressedTmpFile, WriteablePlainTmpFile, WriteableTmpFile}; use crate::tmp_dir::TmpDirWrapper; -use crate::{ - GlobalSettings, Line, - chunks::{self, Chunk}, - compare_by, merge, print_sorted, sort_by, -}; +use crate::{GlobalSettings, compare_by, print_sorted, sort_by}; -// Note: update `test_sort::test_start_buffer` if this size is changed -// Fixed to 8 KiB (equivalent to `std::sys::io::DEFAULT_BUF_SIZE` on most targets) -const DEFAULT_BUF_SIZE: usize = 8 * 1024; - -/// Synchronous sort for targets without thread support (e.g. wasm32-wasip1). -/// -/// Uses the same chunked sort-write-merge strategy as the threaded version, -/// but reads and sorts each chunk sequentially on the calling thread. -#[cfg(wasi_no_threads)] -pub fn ext_sort( - files: &mut impl Iterator>>, - settings: &GlobalSettings, - output: Output, - tmp_dir: &mut TmpDirWrapper, -) -> UResult<()> { - let separator = settings.line_ending.into(); - let mut buffer_size = match settings.buffer_size { - size if size <= 512 * 1024 * 1024 => size, - size => size / 2, - }; - if !settings.buffer_size_is_explicit { - buffer_size = buffer_size.max(8 * 1024 * 1024); - } - - if settings.compress_prog.is_some() { - let _ = writeln!( - stderr(), - "sort: warning: --compress-program is ignored on this platform" - ); - } - - let mut file = files.next().unwrap()?; - let mut carry_over = vec![]; - - // Read and sort first chunk. - let (first, cont) = chunks::read_to_chunk( - RecycledChunk::new(buffer_size.min(DEFAULT_BUF_SIZE)), - Some(buffer_size), - &mut carry_over, - &mut file, - files, - separator, - settings, - )?; - let Some(mut first) = first else { - return Ok(()); // empty input - }; - first.with_dependent_mut(|_, c| sort_by(&mut c.lines, settings, &c.line_data)); - - if !cont { - // All input fits in one chunk. - return print_chunk(&first, settings, output); - } - - // Read and sort second chunk. - let (second, cont) = chunks::read_to_chunk( - RecycledChunk::new(buffer_size.min(DEFAULT_BUF_SIZE)), - Some(buffer_size), - &mut carry_over, - &mut file, - files, - separator, - settings, - )?; - let Some(mut second) = second else { - return print_chunk(&first, settings, output); - }; - second.with_dependent_mut(|_, c| sort_by(&mut c.lines, settings, &c.line_data)); - - if !cont { - // All input fits in two chunks — merge in memory. - return print_two_chunks(first, second, settings, output); - } - - // More than two chunks: write sorted chunks to temp files, then merge. - let mut tmp_files: Vec<::Closed> = vec![]; - - tmp_files.push(write::( - &first, - tmp_dir.next_file()?, - settings.compress_prog.as_deref(), - separator, - )?); - drop(first); - - tmp_files.push(write::( - &second, - tmp_dir.next_file()?, - settings.compress_prog.as_deref(), - separator, - )?); - let mut recycled = second.recycle(); - - loop { - let (chunk, cont) = chunks::read_to_chunk( - recycled, - None, - &mut carry_over, - &mut file, - files, - separator, - settings, - )?; - let Some(mut chunk) = chunk else { break }; - chunk.with_dependent_mut(|_, c| sort_by(&mut c.lines, settings, &c.line_data)); - tmp_files.push(write::( - &chunk, - tmp_dir.next_file()?, - settings.compress_prog.as_deref(), - separator, - )?); - recycled = chunk.recycle(); - if !cont { - break; - } - } - - merge::merge_with_file_limit::<_, _, WriteablePlainTmpFile>( - tmp_files.into_iter().map(merge::ClosedTmpFile::reopen), - settings, - output, - tmp_dir, - ) -} - -/// Print a single sorted chunk. -#[cfg(wasi_no_threads)] -fn print_chunk(chunk: &Chunk, settings: &GlobalSettings, output: Output) -> UResult<()> { - if settings.unique { - print_sorted( - chunk.lines().iter().dedup_by(|a, b| { - compare_by(a, b, settings, chunk.line_data(), chunk.line_data()) == Ordering::Equal - }), - settings, - output, - ) - } else { - print_sorted(chunk.lines().iter(), settings, output) - } -} - -/// Merge two in-memory chunks and print. -#[cfg(wasi_no_threads)] -fn print_two_chunks(a: Chunk, b: Chunk, settings: &GlobalSettings, output: Output) -> UResult<()> { - let merged_iter = a.lines().iter().map(|line| (line, &a)).merge_by( - b.lines().iter().map(|line| (line, &b)), - |(line_a, a), (line_b, b)| { - compare_by(line_a, line_b, settings, a.line_data(), b.line_data()) != Ordering::Greater - }, - ); - if settings.unique { - print_sorted( - merged_iter - .dedup_by(|(line_a, a), (line_b, b)| { - compare_by(line_a, line_b, settings, a.line_data(), b.line_data()) - == Ordering::Equal - }) - .map(|(line, _)| line), - settings, - output, - ) - } else { - print_sorted(merged_iter.map(|(line, _)| line), settings, output) - } -} +use super::{DEFAULT_BUF_SIZE, write}; /// Sort files by using auxiliary files for storing intermediate chunks (if needed), and output the result. /// /// Two threads cooperate: one reads input and writes temporary chunk files, /// while the other sorts each chunk in memory. Once all chunks are written, /// they are merged back together for final output. -#[cfg(not(wasi_no_threads))] pub fn ext_sort( files: &mut impl Iterator>>, settings: &GlobalSettings, @@ -265,7 +87,6 @@ pub fn ext_sort( } } -#[cfg(not(wasi_no_threads))] fn reader_writer< F: Iterator>>, Tmp: WriteableTmpFile + 'static, @@ -351,7 +172,6 @@ fn reader_writer< } /// The function that is executed on the sorter thread. -#[cfg(not(wasi_no_threads))] fn sorter(receiver: &Receiver, sender: &SyncSender, settings: &GlobalSettings) { while let Ok(mut payload) = receiver.recv() { payload.with_dependent_mut(|_, contents| { @@ -366,7 +186,6 @@ fn sorter(receiver: &Receiver, sender: &SyncSender, settings: &Glo } /// Describes how we read the chunks from the input. -#[cfg(not(wasi_no_threads))] enum ReadResult { /// The input was empty. Nothing was read. EmptyInput, @@ -378,7 +197,6 @@ enum ReadResult { WroteChunksToFile { tmp_files: Vec }, } /// The function that is executed on the reader/writer thread. -#[cfg(not(wasi_no_threads))] fn read_write_loop( mut files: impl Iterator>>, tmp_dir: &mut TmpDirWrapper, @@ -455,23 +273,3 @@ fn read_write_loop( } } } - -/// Write the lines in `chunk` to `file`, separated by `separator`. -/// `compress_prog` is used to optionally compress file contents. -fn write( - chunk: &Chunk, - file: (File, PathBuf), - compress_prog: Option<&str>, - separator: u8, -) -> UResult { - let mut tmp_file = I::create(file, compress_prog)?; - write_lines(chunk.lines(), tmp_file.as_write(), separator); - tmp_file.finished_writing() -} - -fn write_lines(lines: &[Line], writer: &mut T, separator: u8) { - for s in lines { - writer.write_all(s.line).unwrap(); - writer.write_all(&[separator]).unwrap(); - } -} diff --git a/src/uu/sort/src/merge.rs b/src/uu/sort/src/merge.rs deleted file mode 100644 index 6b58df5ef04..00000000000 --- a/src/uu/sort/src/merge.rs +++ /dev/null @@ -1,869 +0,0 @@ -// This file is part of the uutils coreutils package. -// -// For the full copyright and license information, please view the LICENSE -// file that was distributed with this source code. -//! Merge already sorted files. -//! -//! We achieve performance by splitting the tasks of sorting and writing, and reading and parsing between two threads. -//! The threads communicate over channels. There's one channel per file in the direction reader -> sorter, but only -//! one channel from the sorter back to the reader. The channels to the sorter are used to send the read chunks. -//! The sorter reads the next chunk from the channel whenever it needs the next chunk after running out of lines -//! from the previous read of the file. The channel back from the sorter to the reader has two purposes: To allow the reader -//! to reuse memory allocations and to tell the reader which file to read from next. - -use std::{ - cmp::Ordering, - ffi::{OsStr, OsString}, - fs::{self, File}, - io::{BufWriter, Read, Write}, - iter, - path::{Path, PathBuf}, - process::{Child, ChildStdin, ChildStdout, Command, Stdio}, - rc::Rc, -}; -#[cfg(not(wasi_no_threads))] -use std::{ - sync::mpsc::{Receiver, Sender, SyncSender, TryRecvError, channel, sync_channel}, - thread::{self, JoinHandle}, -}; - -use compare::Compare; -#[cfg(not(wasi_no_threads))] -use uucore::error::FromIo; -use uucore::error::UResult; - -use crate::{ - GlobalSettings, Output, SortError, - chunks::{self, Chunk, RecycledChunk}, - compare_by, current_open_fd_count, fd_soft_limit, open, - tmp_dir::TmpDirWrapper, -}; - -/// If the output file occurs in the input files as well, copy the contents of the output file -/// and replace its occurrences in the inputs with that copy. -fn replace_output_file_in_input_files( - files: &mut [OsString], - output: Option<&OsStr>, - tmp_dir: &mut TmpDirWrapper, -) -> UResult<()> { - let mut copy: Option = None; - if let Some(Ok(output_path)) = output.map(|path| Path::new(path).canonicalize()) { - for file in files { - if Path::new(file) - .canonicalize() - .is_ok_and(|file_path| file_path == output_path) - { - if let Some(copy) = © { - *file = copy.clone().into_os_string(); - } else { - let (_file, copy_path) = tmp_dir.next_file()?; - fs::copy(&output_path, ©_path) - .map_err(|error| SortError::OpenTmpFileFailed { error })?; - *file = copy_path.clone().into_os_string(); - copy = Some(copy_path); - } - } - } - } - Ok(()) -} - -/// Determine the effective merge batch size, enforcing a minimum and respecting the -/// file-descriptor soft limit after reserving stdio/output and a safety margin. -fn effective_merge_batch_size(settings: &GlobalSettings) -> usize { - const MIN_BATCH_SIZE: usize = 2; - const RESERVED_TMP_OUTPUT: usize = 1; - const RESERVED_CTRL_C: usize = 2; - const RESERVED_RANDOM_SOURCE: usize = 1; - const SAFETY_MARGIN: usize = 1; - let mut batch_size = settings.merge_batch_size.max(MIN_BATCH_SIZE); - - if let Some(limit) = fd_soft_limit() { - let open_fds = current_open_fd_count().unwrap_or(3); - let mut reserved = RESERVED_TMP_OUTPUT + RESERVED_CTRL_C + SAFETY_MARGIN; - if settings.salt.is_some() { - reserved = reserved.saturating_add(RESERVED_RANDOM_SOURCE); - } - let available_inputs = limit.saturating_sub(open_fds.saturating_add(reserved)); - if available_inputs >= MIN_BATCH_SIZE { - batch_size = batch_size.min(available_inputs); - } else { - batch_size = MIN_BATCH_SIZE; - } - } - - batch_size -} - -/// Merge pre-sorted `Box`s. -/// -/// If `settings.merge_batch_size` is greater than the length of `files`, intermediate files will be used. -/// If `settings.compress_prog` is `Some`, intermediate files will be compressed with it. -pub fn merge( - files: &mut [OsString], - settings: &GlobalSettings, - output: Output, - tmp_dir: &mut TmpDirWrapper, -) -> UResult<()> { - replace_output_file_in_input_files(files, output.as_output_name(), tmp_dir)?; - let files = files - .iter() - .map(|file| open(file).map(|file| PlainMergeInput { inner: file })); - #[cfg(wasi_no_threads)] - if settings.compress_prog.is_some() { - let _ = writeln!( - std::io::stderr(), - "sort: warning: --compress-program is ignored on this platform" - ); - return merge_with_file_limit::<_, _, WriteablePlainTmpFile>( - files, settings, output, tmp_dir, - ); - } - - if settings.compress_prog.is_none() { - merge_with_file_limit::<_, _, WriteablePlainTmpFile>(files, settings, output, tmp_dir) - } else { - merge_with_file_limit::<_, _, WriteableCompressedTmpFile>(files, settings, output, tmp_dir) - } -} - -/// Merge and write to output — dispatches between threaded and synchronous. -fn do_merge_to_output( - files: impl Iterator>, - settings: &GlobalSettings, - output: Output, -) -> UResult<()> { - #[cfg(not(wasi_no_threads))] - return merge_without_limit(files, settings)?.write_all(settings, output); - #[cfg(wasi_no_threads)] - return merge_without_limit_sync(files, settings)?.write_all(settings, output); -} - -/// Merge and write to a writer — dispatches between threaded and synchronous. -fn do_merge_to_writer( - files: impl Iterator>, - settings: &GlobalSettings, - out: &mut impl Write, -) -> UResult<()> { - #[cfg(not(wasi_no_threads))] - return merge_without_limit(files, settings)?.write_all_to(settings, out); - #[cfg(wasi_no_threads)] - return merge_without_limit_sync(files, settings)?.write_all_to(settings, out); -} - -// Merge already sorted `MergeInput`s. -pub fn merge_with_file_limit< - M: MergeInput + 'static, - F: ExactSizeIterator>, - Tmp: WriteableTmpFile + 'static, ->( - files: F, - settings: &GlobalSettings, - output: Output, - tmp_dir: &mut TmpDirWrapper, -) -> UResult<()> { - let batch_size = effective_merge_batch_size(settings); - debug_assert!(batch_size >= 2); - - if files.len() <= batch_size { - do_merge_to_output(files, settings, output) - } else { - let mut temporary_files = vec![]; - let mut batch = Vec::with_capacity(batch_size); - for file in files { - batch.push(file); - if batch.len() >= batch_size { - assert_eq!(batch.len(), batch_size); - let full_batch = std::mem::replace(&mut batch, Vec::with_capacity(batch_size)); - - let mut tmp_file = - Tmp::create(tmp_dir.next_file()?, settings.compress_prog.as_deref())?; - do_merge_to_writer(full_batch.into_iter(), settings, tmp_file.as_write())?; - temporary_files.push(tmp_file.finished_writing()?); - } - } - // Merge any remaining files that didn't get merged in a full batch above. - if !batch.is_empty() { - assert!(batch.len() < batch_size); - - let mut tmp_file = - Tmp::create(tmp_dir.next_file()?, settings.compress_prog.as_deref())?; - do_merge_to_writer(batch.into_iter(), settings, tmp_file.as_write())?; - temporary_files.push(tmp_file.finished_writing()?); - } - merge_with_file_limit::<_, _, Tmp>( - temporary_files - .into_iter() - .map(Box::new(|c: Tmp::Closed| c.reopen()) - as Box< - dyn FnMut(Tmp::Closed) -> UResult<::Reopened>, - >), - settings, - output, - tmp_dir, - ) - } -} - -/// Merge files without limiting how many files are concurrently open. -/// -/// It is the responsibility of the caller to ensure that `files` yields only -/// as many files as we are allowed to open concurrently. -#[cfg(not(wasi_no_threads))] -fn merge_without_limit>>( - files: F, - settings: &GlobalSettings, -) -> UResult> { - let (request_sender, request_receiver) = channel(); - let mut reader_files = Vec::with_capacity(files.size_hint().0); - let mut loaded_receivers = Vec::with_capacity(files.size_hint().0); - for (file_number, file) in files.enumerate() { - let (sender, receiver) = sync_channel(2); - loaded_receivers.push(receiver); - reader_files.push(Some(ReaderFile { - file: file?, - sender, - carry_over: vec![], - })); - // Send the initial chunk to trigger a read for each file - request_sender - .send((file_number, RecycledChunk::new(8 * 1024))) - .unwrap(); - } - - // Send the second chunk for each file - for file_number in 0..reader_files.len() { - request_sender - .send((file_number, RecycledChunk::new(8 * 1024))) - .unwrap(); - } - - let reader_join_handle = thread::spawn({ - let settings = settings.clone(); - move || { - reader( - &request_receiver, - &mut reader_files, - &settings, - settings.line_ending.into(), - ) - } - }); - - let mut mergeable_files = vec![]; - - for (file_number, receiver) in loaded_receivers.into_iter().enumerate() { - if let Ok(chunk) = receiver.recv() { - mergeable_files.push(MergeableFile { - current_chunk: Rc::new(chunk), - file_number, - line_idx: 0, - receiver, - }); - } - } - - Ok(FileMerger { - heap: binary_heap_plus::BinaryHeap::from_vec_cmp( - mergeable_files, - FileComparator { settings }, - ), - request_sender, - prev: None, - reader_join_handle, - }) -} -/// The struct on the reader thread representing an input file -#[cfg(not(wasi_no_threads))] -struct ReaderFile { - file: M, - sender: SyncSender, - carry_over: Vec, -} - -/// The function running on the reader thread. -#[cfg(not(wasi_no_threads))] -fn reader( - recycled_receiver: &Receiver<(usize, RecycledChunk)>, - files: &mut [Option>], - settings: &GlobalSettings, - separator: u8, -) -> UResult<()> { - for (file_idx, recycled_chunk) in recycled_receiver { - if let Some(ReaderFile { - file, - sender, - carry_over, - }) = &mut files[file_idx] - { - let should_continue = chunks::read( - sender, - recycled_chunk, - None, - carry_over, - file.as_read(), - &mut iter::empty(), - separator, - settings, - )?; - if !should_continue { - // Remove the file from the list by replacing it with `None`. - let ReaderFile { file, .. } = files[file_idx].take().unwrap(); - // Depending on the kind of the `MergeInput`, this may delete the file: - file.finished_reading()?; - } - } - } - Ok(()) -} -/// The struct on the main thread representing an input file -#[cfg(not(wasi_no_threads))] -pub struct MergeableFile { - current_chunk: Rc, - line_idx: usize, - receiver: Receiver, - file_number: usize, -} - -/// A struct to keep track of the previous line we encountered. -/// -/// This is required for deduplication purposes. -struct PreviousLine { - chunk: Rc, - line_idx: usize, - #[cfg_attr(wasi_no_threads, allow(dead_code))] - file_number: usize, -} - -/// Merges files together. This is **not** an iterator because of lifetime problems. -#[cfg(not(wasi_no_threads))] -struct FileMerger<'a> { - heap: binary_heap_plus::BinaryHeap>, - request_sender: Sender<(usize, RecycledChunk)>, - prev: Option, - reader_join_handle: JoinHandle>, -} - -#[cfg(not(wasi_no_threads))] -impl FileMerger<'_> { - /// Write the merged contents to the output file. - fn write_all(self, settings: &GlobalSettings, output: Output) -> UResult<()> { - let mut out = output.into_write(); - self.write_all_to(settings, &mut out) - } - - fn write_all_to(mut self, settings: &GlobalSettings, out: &mut impl Write) -> UResult<()> { - let write_result = loop { - match self - .write_next(out, settings) - .map_err_context(|| "write failed".into()) - { - Ok(true) => (), - Ok(false) => break Ok(()), - Err(error) => { - // Don't return yet: we still have to shut the reader thread down in an - // orderly fashion below. Returning here would drop our receivers while - // the reader is still sending, and `chunks::read` unwraps that send. - break Err(error); - } - } - }; - - let Self { - heap, - request_sender, - reader_join_handle, - .. - } = self; - - // Stop asking for chunks, so the reader runs out of work and returns. - drop(request_sender); - // Until it does, keep draining the files it might still be sending to. We have to - // poll all of them in turn rather than draining one at a time: the reader can be - // blocked on any one channel, and blocking on a different one would deadlock. - let mut files = heap.into_vec(); - while !files.is_empty() { - files.retain(|file| { - !matches!(file.receiver.try_recv(), Err(TryRecvError::Disconnected)) - }); - thread::yield_now(); - } - - let reader_result = reader_join_handle.join().unwrap(); - // A write failure is what the user needs to hear about; the reader hitting an error - // on the way down is secondary. - write_result.and(reader_result) - } - - fn write_next( - &mut self, - writer: &mut impl Write, - settings: &GlobalSettings, - ) -> std::io::Result { - if let Some(file) = self.heap.peek() { - let prev = self.prev.replace(PreviousLine { - chunk: file.current_chunk.clone(), - line_idx: file.line_idx, - file_number: file.file_number, - }); - - file.current_chunk.with_dependent(|_, contents| { - let current_line = &contents.lines[file.line_idx]; - if settings.unique - && let Some(prev) = &prev - { - let cmp = compare_by( - &prev.chunk.lines()[prev.line_idx], - current_line, - settings, - prev.chunk.line_data(), - file.current_chunk.line_data(), - ); - if cmp == Ordering::Equal { - return Ok(()); - } - } - current_line.write(writer, settings) - })?; - - let was_last_line_for_file = file.current_chunk.lines().len() == file.line_idx + 1; - - if was_last_line_for_file { - if let Ok(next_chunk) = file.receiver.recv() { - let mut file = self.heap.peek_mut().unwrap(); - file.current_chunk = Rc::new(next_chunk); - file.line_idx = 0; - } else { - self.heap.pop(); - } - } else { - // This will cause the comparison to use a different line and the heap to readjust. - self.heap.peek_mut().unwrap().line_idx += 1; - } - - if let Some(prev) = prev - && let Ok(prev_chunk) = Rc::try_unwrap(prev.chunk) - { - // If nothing is referencing the previous chunk anymore, this means that the previous line - // was the last line of the chunk. We can recycle the chunk. - self.request_sender - .send((prev.file_number, prev_chunk.recycle())) - .ok(); - } - } - Ok(!self.heap.is_empty()) - } -} - -/// Compares files by their current line. -struct FileComparator<'a> { - settings: &'a GlobalSettings, -} - -#[cfg(not(wasi_no_threads))] -impl Compare for FileComparator<'_> { - fn compare(&self, a: &MergeableFile, b: &MergeableFile) -> Ordering { - let mut cmp = compare_by( - &a.current_chunk.lines()[a.line_idx], - &b.current_chunk.lines()[b.line_idx], - self.settings, - a.current_chunk.line_data(), - b.current_chunk.line_data(), - ); - if cmp == Ordering::Equal { - // To make sorting stable, we need to consider the file number as well, - // as lines from a file with a lower number are to be considered "earlier". - cmp = a.file_number.cmp(&b.file_number); - } - // BinaryHeap is a max heap. We use it as a min heap, so we need to reverse the ordering. - cmp.reverse() - } -} - -/// Wait for the child to exit and check its exit code. -fn check_child_success(mut child: Child, program: &str) -> UResult<()> { - if matches!(child.wait().map(|e| e.code()), Ok(Some(0) | None) | Err(_)) { - Ok(()) - } else { - Err(SortError::CompressProgTerminatedAbnormally { - prog: program.to_owned(), - } - .into()) - } -} - -/// A temporary file that can be written to. -pub trait WriteableTmpFile: Sized { - type Closed: ClosedTmpFile; - type InnerWrite: Write; - fn create(file: (File, PathBuf), compress_prog: Option<&str>) -> UResult; - /// Closes the temporary file. - fn finished_writing(self) -> UResult; - fn as_write(&mut self) -> &mut Self::InnerWrite; -} -/// A temporary file that is (temporarily) closed, but can be reopened. -pub trait ClosedTmpFile { - type Reopened: MergeInput; - /// Reopens the temporary file. - fn reopen(self) -> UResult; -} -/// A pre-sorted input for merging. -pub trait MergeInput: Send { - type InnerRead: Read; - /// Cleans this `MergeInput` up. - /// Implementations may delete the backing file. - fn finished_reading(self) -> UResult<()>; - fn as_read(&mut self) -> &mut Self::InnerRead; -} - -pub struct WriteablePlainTmpFile { - path: PathBuf, - file: BufWriter, -} -pub struct ClosedPlainTmpFile { - path: PathBuf, -} -pub struct PlainTmpMergeInput { - path: PathBuf, - file: File, -} -impl WriteableTmpFile for WriteablePlainTmpFile { - type Closed = ClosedPlainTmpFile; - type InnerWrite = BufWriter; - - fn create((file, path): (File, PathBuf), _: Option<&str>) -> UResult { - Ok(Self { - file: BufWriter::new(file), - path, - }) - } - - fn finished_writing(self) -> UResult { - Ok(ClosedPlainTmpFile { path: self.path }) - } - - fn as_write(&mut self) -> &mut Self::InnerWrite { - &mut self.file - } -} -impl ClosedTmpFile for ClosedPlainTmpFile { - type Reopened = PlainTmpMergeInput; - fn reopen(self) -> UResult { - Ok(PlainTmpMergeInput { - file: File::open(&self.path).map_err(|error| SortError::OpenTmpFileFailed { error })?, - path: self.path, - }) - } -} -impl MergeInput for PlainTmpMergeInput { - type InnerRead = File; - - fn finished_reading(self) -> UResult<()> { - // we ignore failures to delete the temporary file, - // because there is a race at the end of the execution and the whole - // temporary directory might already be gone. - let _ = fs::remove_file(self.path); - Ok(()) - } - - fn as_read(&mut self) -> &mut Self::InnerRead { - &mut self.file - } -} - -pub struct WriteableCompressedTmpFile { - path: PathBuf, - compress_prog: String, - child: Child, - child_stdin: BufWriter, -} -pub struct ClosedCompressedTmpFile { - path: PathBuf, - compress_prog: String, -} -pub struct CompressedTmpMergeInput { - path: PathBuf, - compress_prog: String, - child: Child, - child_stdout: ChildStdout, -} -impl WriteableTmpFile for WriteableCompressedTmpFile { - type Closed = ClosedCompressedTmpFile; - type InnerWrite = BufWriter; - - fn create((file, path): (File, PathBuf), compress_prog: Option<&str>) -> UResult { - let compress_prog = compress_prog.unwrap(); - let mut command = Command::new(compress_prog); - command.stdin(Stdio::piped()).stdout(file); - let mut child = command - .spawn() - .map_err(|err| SortError::CompressProgExecutionFailed { - prog: compress_prog.to_owned(), - error: err, - })?; - let child_stdin = child.stdin.take().unwrap(); - Ok(Self { - path, - compress_prog: compress_prog.to_owned(), - child, - child_stdin: BufWriter::new(child_stdin), - }) - } - - fn finished_writing(self) -> UResult { - drop(self.child_stdin); - check_child_success(self.child, &self.compress_prog)?; - Ok(ClosedCompressedTmpFile { - path: self.path, - compress_prog: self.compress_prog, - }) - } - - fn as_write(&mut self) -> &mut Self::InnerWrite { - &mut self.child_stdin - } -} -impl ClosedTmpFile for ClosedCompressedTmpFile { - type Reopened = CompressedTmpMergeInput; - - fn reopen(self) -> UResult { - let mut command = Command::new(&self.compress_prog); - // mirroring what is done for ClosedPlainTmpFile - let file = - File::open(&self.path).map_err(|error| SortError::OpenTmpFileFailed { error })?; - command.stdin(file).stdout(Stdio::piped()).arg("-d"); - let mut child = command - .spawn() - .map_err(|err| SortError::CompressProgExecutionFailed { - prog: self.compress_prog.clone(), - error: err, - })?; - let child_stdout = child.stdout.take().unwrap(); - Ok(CompressedTmpMergeInput { - path: self.path, - compress_prog: self.compress_prog, - child, - child_stdout, - }) - } -} -impl MergeInput for CompressedTmpMergeInput { - type InnerRead = ChildStdout; - - fn finished_reading(self) -> UResult<()> { - // Explicitly close stdout before waiting on the child process. - #[allow(clippy::drop_non_drop)] - drop(self.child_stdout); - check_child_success(self.child, &self.compress_prog)?; - let _ = fs::remove_file(self.path); - Ok(()) - } - - fn as_read(&mut self) -> &mut Self::InnerRead { - &mut self.child_stdout - } -} - -pub struct PlainMergeInput { - inner: R, -} -impl MergeInput for PlainMergeInput { - type InnerRead = R; - fn finished_reading(self) -> UResult<()> { - Ok(()) - } - fn as_read(&mut self) -> &mut Self::InnerRead { - &mut self.inner - } -} - -// --------------------------------------------------------------------------- -// Synchronous merge for targets without thread support (e.g. wasm32-wasip1). -// --------------------------------------------------------------------------- - -#[cfg(wasi_no_threads)] -struct SyncReaderFile { - file: M, - carry_over: Vec, -} - -#[cfg(wasi_no_threads)] -struct SyncMergeableFile { - current_chunk: Rc, - line_idx: usize, - file_number: usize, -} - -#[cfg(wasi_no_threads)] -impl Compare for FileComparator<'_> { - fn compare(&self, a: &SyncMergeableFile, b: &SyncMergeableFile) -> Ordering { - let mut cmp = compare_by( - &a.current_chunk.lines()[a.line_idx], - &b.current_chunk.lines()[b.line_idx], - self.settings, - a.current_chunk.line_data(), - b.current_chunk.line_data(), - ); - if cmp == Ordering::Equal { - cmp = a.file_number.cmp(&b.file_number); - } - cmp.reverse() - } -} - -#[cfg(wasi_no_threads)] -struct SyncFileMerger<'a, M: MergeInput> { - heap: binary_heap_plus::BinaryHeap>, - readers: Vec>>, - prev: Option, - recycled: Option, - settings: &'a GlobalSettings, -} - -#[cfg(wasi_no_threads)] -impl SyncFileMerger<'_, M> { - fn write_all(self, settings: &GlobalSettings, output: Output) -> UResult<()> { - let mut out = output.into_write(); - self.write_all_to(settings, &mut out) - } - - fn write_all_to(mut self, settings: &GlobalSettings, out: &mut impl Write) -> UResult<()> { - while self.write_next(settings, out)? {} - for reader in self.readers.into_iter().flatten() { - reader.file.finished_reading()?; - } - Ok(()) - } - - fn write_next(&mut self, settings: &GlobalSettings, out: &mut impl Write) -> UResult { - if let Some(file) = self.heap.peek() { - let prev = self.prev.replace(PreviousLine { - chunk: file.current_chunk.clone(), - line_idx: file.line_idx, - file_number: file.file_number, - }); - - file.current_chunk.with_dependent(|_, contents| { - let current_line = &contents.lines[file.line_idx]; - if settings.unique { - if let Some(prev) = &prev { - let cmp = compare_by( - &prev.chunk.lines()[prev.line_idx], - current_line, - settings, - prev.chunk.line_data(), - file.current_chunk.line_data(), - ); - if cmp == Ordering::Equal { - return Ok(()); - } - } - } - current_line.print(out, settings) - })?; - - let was_last = file.current_chunk.lines().len() == file.line_idx + 1; - let file_number = file.file_number; - - if was_last { - let separator = self.settings.line_ending.into(); - let recycled = self - .recycled - .take() - .unwrap_or_else(|| RecycledChunk::new(8 * 1024)); - let next_chunk = if let Some(reader) = self.readers[file_number].as_mut() { - let (chunk, should_continue) = chunks::read_to_chunk( - recycled, - None, - &mut reader.carry_over, - reader.file.as_read(), - &mut iter::empty(), - separator, - self.settings, - )?; - if !should_continue { - if let Some(reader) = self.readers[file_number].take() { - reader.file.finished_reading()?; - } - } - chunk - } else { - None - }; - - if let Some(next_chunk) = next_chunk { - let mut file = self.heap.peek_mut().unwrap(); - file.current_chunk = Rc::new(next_chunk); - file.line_idx = 0; - } else { - self.heap.pop(); - } - } else { - self.heap.peek_mut().unwrap().line_idx += 1; - } - - // Recycle the previous chunk if no other reference holds it. - if let Some(prev) = prev { - if let Ok(chunk) = Rc::try_unwrap(prev.chunk) { - self.recycled = Some(chunk.recycle()); - } - } - } - Ok(!self.heap.is_empty()) - } -} - -#[cfg(wasi_no_threads)] -fn merge_without_limit_sync>>( - files: F, - settings: &GlobalSettings, -) -> UResult> { - let separator = settings.line_ending.into(); - let mut readers: Vec>> = Vec::new(); - let mut mergeable_files = Vec::new(); - - for (file_number, file) in files.enumerate() { - let mut reader = SyncReaderFile { - file: file?, - carry_over: vec![], - }; - let recycled = RecycledChunk::new(8 * 1024); - let (chunk, should_continue) = chunks::read_to_chunk( - recycled, - None, - &mut reader.carry_over, - reader.file.as_read(), - &mut iter::empty(), - separator, - settings, - )?; - - if let Some(chunk) = chunk { - mergeable_files.push(SyncMergeableFile { - current_chunk: Rc::new(chunk), - line_idx: 0, - file_number, - }); - if should_continue { - readers.push(Some(reader)); - } else { - reader.file.finished_reading()?; - readers.push(None); - } - } else { - reader.file.finished_reading()?; - readers.push(None); - } - } - - Ok(SyncFileMerger { - heap: binary_heap_plus::BinaryHeap::from_vec_cmp( - mergeable_files, - FileComparator { settings }, - ), - readers, - prev: None, - recycled: None, - settings, - }) -} diff --git a/src/uu/sort/src/merge/mod.rs b/src/uu/sort/src/merge/mod.rs new file mode 100644 index 00000000000..6f8002a404a --- /dev/null +++ b/src/uu/sort/src/merge/mod.rs @@ -0,0 +1,409 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. +//! Merge already sorted files. +//! +//! On most platforms this uses a multi-threaded reader/merger setup. On WASI +//! without atomics, a synchronous variant is used instead. The two +//! implementations live in sibling modules and are selected via cfg at the +//! module boundary. + +use std::{ + ffi::{OsStr, OsString}, + fs::{self, File}, + io::{BufWriter, Read, Write}, + path::{Path, PathBuf}, + process::{Child, ChildStdin, ChildStdout, Command, Stdio}, + rc::Rc, +}; + +use uucore::error::UResult; + +use crate::{ + GlobalSettings, Output, SortError, chunks::Chunk, current_open_fd_count, fd_soft_limit, open, + tmp_dir::TmpDirWrapper, +}; + +#[cfg(not(wasi_no_threads))] +mod threaded; +#[cfg(not(wasi_no_threads))] +use threaded as runner; + +#[cfg(wasi_no_threads)] +mod sync; +#[cfg(wasi_no_threads)] +use sync as runner; + +/// If the output file occurs in the input files as well, copy the contents of the output file +/// and replace its occurrences in the inputs with that copy. +fn replace_output_file_in_input_files( + files: &mut [OsString], + output: Option<&OsStr>, + tmp_dir: &mut TmpDirWrapper, +) -> UResult<()> { + let mut copy: Option = None; + if let Some(Ok(output_path)) = output.map(|path| Path::new(path).canonicalize()) { + for file in files { + if Path::new(file) + .canonicalize() + .is_ok_and(|file_path| file_path == output_path) + { + if let Some(copy) = © { + *file = copy.clone().into_os_string(); + } else { + let (_file, copy_path) = tmp_dir.next_file()?; + fs::copy(&output_path, ©_path) + .map_err(|error| SortError::OpenTmpFileFailed { error })?; + *file = copy_path.clone().into_os_string(); + copy = Some(copy_path); + } + } + } + } + Ok(()) +} + +/// Determine the effective merge batch size, enforcing a minimum and respecting the +/// file-descriptor soft limit after reserving stdio/output and a safety margin. +fn effective_merge_batch_size(settings: &GlobalSettings) -> usize { + const MIN_BATCH_SIZE: usize = 2; + const RESERVED_TMP_OUTPUT: usize = 1; + const RESERVED_CTRL_C: usize = 2; + const RESERVED_RANDOM_SOURCE: usize = 1; + const SAFETY_MARGIN: usize = 1; + let mut batch_size = settings.merge_batch_size.max(MIN_BATCH_SIZE); + + if let Some(limit) = fd_soft_limit() { + let open_fds = current_open_fd_count().unwrap_or(3); + let mut reserved = RESERVED_TMP_OUTPUT + RESERVED_CTRL_C + SAFETY_MARGIN; + if settings.salt.is_some() { + reserved = reserved.saturating_add(RESERVED_RANDOM_SOURCE); + } + let available_inputs = limit.saturating_sub(open_fds.saturating_add(reserved)); + if available_inputs >= MIN_BATCH_SIZE { + batch_size = batch_size.min(available_inputs); + } else { + batch_size = MIN_BATCH_SIZE; + } + } + + batch_size +} + +/// Merge pre-sorted `Box`s. +/// +/// If `settings.merge_batch_size` is greater than the length of `files`, intermediate files will be used. +/// If `settings.compress_prog` is `Some`, intermediate files will be compressed with it. +pub fn merge( + files: &mut [OsString], + settings: &GlobalSettings, + output: Output, + tmp_dir: &mut TmpDirWrapper, +) -> UResult<()> { + replace_output_file_in_input_files(files, output.as_output_name(), tmp_dir)?; + let files = files + .iter() + .map(|file| open(file).map(|file| PlainMergeInput { inner: file })); + + if !runner::SUPPORTS_COMPRESSION && settings.compress_prog.is_some() { + let _ = writeln!( + std::io::stderr(), + "sort: warning: --compress-program is ignored on this platform" + ); + return merge_with_file_limit::<_, _, WriteablePlainTmpFile>( + files, settings, output, tmp_dir, + ); + } + + if settings.compress_prog.is_none() { + merge_with_file_limit::<_, _, WriteablePlainTmpFile>(files, settings, output, tmp_dir) + } else { + merge_with_file_limit::<_, _, WriteableCompressedTmpFile>(files, settings, output, tmp_dir) + } +} + +/// Merge and write to output, dispatching to the active runner. +fn do_merge_to_output( + files: impl Iterator>, + settings: &GlobalSettings, + output: Output, +) -> UResult<()> { + runner::merge_without_limit(files, settings)?.write_all(settings, output) +} + +/// Merge and write to a writer, dispatching to the active runner. +fn do_merge_to_writer( + files: impl Iterator>, + settings: &GlobalSettings, + out: &mut impl Write, +) -> UResult<()> { + runner::merge_without_limit(files, settings)?.write_all_to(settings, out) +} + +// Merge already sorted `MergeInput`s. +pub fn merge_with_file_limit< + M: MergeInput + 'static, + F: ExactSizeIterator>, + Tmp: WriteableTmpFile + 'static, +>( + files: F, + settings: &GlobalSettings, + output: Output, + tmp_dir: &mut TmpDirWrapper, +) -> UResult<()> { + let batch_size = effective_merge_batch_size(settings); + debug_assert!(batch_size >= 2); + + if files.len() <= batch_size { + do_merge_to_output(files, settings, output) + } else { + let mut temporary_files = vec![]; + let mut batch = Vec::with_capacity(batch_size); + for file in files { + batch.push(file); + if batch.len() >= batch_size { + assert_eq!(batch.len(), batch_size); + let full_batch = std::mem::replace(&mut batch, Vec::with_capacity(batch_size)); + + let mut tmp_file = + Tmp::create(tmp_dir.next_file()?, settings.compress_prog.as_deref())?; + do_merge_to_writer(full_batch.into_iter(), settings, tmp_file.as_write())?; + temporary_files.push(tmp_file.finished_writing()?); + } + } + // Merge any remaining files that didn't get merged in a full batch above. + if !batch.is_empty() { + assert!(batch.len() < batch_size); + + let mut tmp_file = + Tmp::create(tmp_dir.next_file()?, settings.compress_prog.as_deref())?; + do_merge_to_writer(batch.into_iter(), settings, tmp_file.as_write())?; + temporary_files.push(tmp_file.finished_writing()?); + } + merge_with_file_limit::<_, _, Tmp>( + temporary_files + .into_iter() + .map(Box::new(|c: Tmp::Closed| c.reopen()) + as Box< + dyn FnMut(Tmp::Closed) -> UResult<::Reopened>, + >), + settings, + output, + tmp_dir, + ) + } +} + +/// A struct to keep track of the previous line we encountered. +/// +/// This is required for deduplication purposes. +pub(super) struct PreviousLine { + pub chunk: Rc, + pub line_idx: usize, + // Only the threaded merger reads this back to recycle chunks. + #[cfg_attr(wasi_no_threads, allow(dead_code))] + pub file_number: usize, +} + +/// Compares files by their current line. +pub(super) struct FileComparator<'a> { + pub settings: &'a GlobalSettings, +} + +/// Wait for the child to exit and check its exit code. +fn check_child_success(mut child: Child, program: &str) -> UResult<()> { + if matches!(child.wait().map(|e| e.code()), Ok(Some(0) | None) | Err(_)) { + Ok(()) + } else { + Err(SortError::CompressProgTerminatedAbnormally { + prog: program.to_owned(), + } + .into()) + } +} + +/// A temporary file that can be written to. +pub trait WriteableTmpFile: Sized { + type Closed: ClosedTmpFile; + type InnerWrite: Write; + fn create(file: (File, PathBuf), compress_prog: Option<&str>) -> UResult; + /// Closes the temporary file. + fn finished_writing(self) -> UResult; + fn as_write(&mut self) -> &mut Self::InnerWrite; +} +/// A temporary file that is (temporarily) closed, but can be reopened. +pub trait ClosedTmpFile { + type Reopened: MergeInput; + /// Reopens the temporary file. + fn reopen(self) -> UResult; +} +/// A pre-sorted input for merging. +pub trait MergeInput: Send { + type InnerRead: Read; + /// Cleans this `MergeInput` up. + /// Implementations may delete the backing file. + fn finished_reading(self) -> UResult<()>; + fn as_read(&mut self) -> &mut Self::InnerRead; +} + +pub struct WriteablePlainTmpFile { + path: PathBuf, + file: BufWriter, +} +pub struct ClosedPlainTmpFile { + path: PathBuf, +} +pub struct PlainTmpMergeInput { + path: PathBuf, + file: File, +} +impl WriteableTmpFile for WriteablePlainTmpFile { + type Closed = ClosedPlainTmpFile; + type InnerWrite = BufWriter; + + fn create((file, path): (File, PathBuf), _: Option<&str>) -> UResult { + Ok(Self { + file: BufWriter::new(file), + path, + }) + } + + fn finished_writing(self) -> UResult { + Ok(ClosedPlainTmpFile { path: self.path }) + } + + fn as_write(&mut self) -> &mut Self::InnerWrite { + &mut self.file + } +} +impl ClosedTmpFile for ClosedPlainTmpFile { + type Reopened = PlainTmpMergeInput; + fn reopen(self) -> UResult { + Ok(PlainTmpMergeInput { + file: File::open(&self.path).map_err(|error| SortError::OpenTmpFileFailed { error })?, + path: self.path, + }) + } +} +impl MergeInput for PlainTmpMergeInput { + type InnerRead = File; + + fn finished_reading(self) -> UResult<()> { + // we ignore failures to delete the temporary file, + // because there is a race at the end of the execution and the whole + // temporary directory might already be gone. + let _ = fs::remove_file(self.path); + Ok(()) + } + + fn as_read(&mut self) -> &mut Self::InnerRead { + &mut self.file + } +} + +pub struct WriteableCompressedTmpFile { + path: PathBuf, + compress_prog: String, + child: Child, + child_stdin: BufWriter, +} +pub struct ClosedCompressedTmpFile { + path: PathBuf, + compress_prog: String, +} +pub struct CompressedTmpMergeInput { + path: PathBuf, + compress_prog: String, + child: Child, + child_stdout: ChildStdout, +} +impl WriteableTmpFile for WriteableCompressedTmpFile { + type Closed = ClosedCompressedTmpFile; + type InnerWrite = BufWriter; + + fn create((file, path): (File, PathBuf), compress_prog: Option<&str>) -> UResult { + let compress_prog = compress_prog.unwrap(); + let mut command = Command::new(compress_prog); + command.stdin(Stdio::piped()).stdout(file); + let mut child = command + .spawn() + .map_err(|err| SortError::CompressProgExecutionFailed { + prog: compress_prog.to_owned(), + error: err, + })?; + let child_stdin = child.stdin.take().unwrap(); + Ok(Self { + path, + compress_prog: compress_prog.to_owned(), + child, + child_stdin: BufWriter::new(child_stdin), + }) + } + + fn finished_writing(self) -> UResult { + drop(self.child_stdin); + check_child_success(self.child, &self.compress_prog)?; + Ok(ClosedCompressedTmpFile { + path: self.path, + compress_prog: self.compress_prog, + }) + } + + fn as_write(&mut self) -> &mut Self::InnerWrite { + &mut self.child_stdin + } +} +impl ClosedTmpFile for ClosedCompressedTmpFile { + type Reopened = CompressedTmpMergeInput; + + fn reopen(self) -> UResult { + let mut command = Command::new(&self.compress_prog); + // mirroring what is done for ClosedPlainTmpFile + let file = + File::open(&self.path).map_err(|error| SortError::OpenTmpFileFailed { error })?; + command.stdin(file).stdout(Stdio::piped()).arg("-d"); + let mut child = command + .spawn() + .map_err(|err| SortError::CompressProgExecutionFailed { + prog: self.compress_prog.clone(), + error: err, + })?; + let child_stdout = child.stdout.take().unwrap(); + Ok(CompressedTmpMergeInput { + path: self.path, + compress_prog: self.compress_prog, + child, + child_stdout, + }) + } +} +impl MergeInput for CompressedTmpMergeInput { + type InnerRead = ChildStdout; + + fn finished_reading(self) -> UResult<()> { + // Explicitly close stdout before waiting on the child process. + #[allow(clippy::drop_non_drop)] + drop(self.child_stdout); + check_child_success(self.child, &self.compress_prog)?; + let _ = fs::remove_file(self.path); + Ok(()) + } + + fn as_read(&mut self) -> &mut Self::InnerRead { + &mut self.child_stdout + } +} + +pub struct PlainMergeInput { + inner: R, +} +impl MergeInput for PlainMergeInput { + type InnerRead = R; + fn finished_reading(self) -> UResult<()> { + Ok(()) + } + fn as_read(&mut self) -> &mut Self::InnerRead { + &mut self.inner + } +} diff --git a/src/uu/sort/src/merge/sync.rs b/src/uu/sort/src/merge/sync.rs new file mode 100644 index 00000000000..badbff11289 --- /dev/null +++ b/src/uu/sort/src/merge/sync.rs @@ -0,0 +1,207 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +//! Synchronous merge for targets without thread support (e.g. wasm32-wasip1). +//! +//! Reads chunks on demand from each input on the calling thread instead of +//! using a dedicated reader thread. + +use std::{cmp::Ordering, io::Write, iter, rc::Rc}; + +use compare::Compare; +use uucore::error::UResult; + +use crate::{ + GlobalSettings, Output, + chunks::{self, Chunk, RecycledChunk}, + compare_by, +}; + +use super::{FileComparator, MergeInput, PreviousLine}; + +pub(super) const SUPPORTS_COMPRESSION: bool = false; + +struct SyncReaderFile { + file: M, + carry_over: Vec, +} + +struct SyncMergeableFile { + current_chunk: Rc, + line_idx: usize, + file_number: usize, +} + +impl Compare for FileComparator<'_> { + fn compare(&self, a: &SyncMergeableFile, b: &SyncMergeableFile) -> Ordering { + let mut cmp = compare_by( + &a.current_chunk.lines()[a.line_idx], + &b.current_chunk.lines()[b.line_idx], + self.settings, + a.current_chunk.line_data(), + b.current_chunk.line_data(), + ); + if cmp == Ordering::Equal { + cmp = a.file_number.cmp(&b.file_number); + } + cmp.reverse() + } +} + +pub(super) struct SyncFileMerger<'a, M: MergeInput> { + heap: binary_heap_plus::BinaryHeap>, + readers: Vec>>, + prev: Option, + recycled: Option, + settings: &'a GlobalSettings, +} + +impl SyncFileMerger<'_, M> { + pub(super) fn write_all(self, settings: &GlobalSettings, output: Output) -> UResult<()> { + let mut out = output.into_write(); + self.write_all_to(settings, &mut out) + } + + pub(super) fn write_all_to( + mut self, + settings: &GlobalSettings, + out: &mut impl Write, + ) -> UResult<()> { + while self.write_next(out, settings)? {} + for reader in self.readers.into_iter().flatten() { + reader.file.finished_reading()?; + } + Ok(()) + } + + fn write_next(&mut self, writer: &mut impl Write, settings: &GlobalSettings) -> UResult { + if let Some(file) = self.heap.peek() { + let prev = self.prev.replace(PreviousLine { + chunk: file.current_chunk.clone(), + line_idx: file.line_idx, + file_number: file.file_number, + }); + + file.current_chunk.with_dependent(|_, contents| { + let current_line = &contents.lines[file.line_idx]; + if settings.unique + && let Some(prev) = &prev + { + let cmp = compare_by( + &prev.chunk.lines()[prev.line_idx], + current_line, + settings, + prev.chunk.line_data(), + file.current_chunk.line_data(), + ); + if cmp == Ordering::Equal { + return Ok(()); + } + } + current_line.write(writer, settings) + })?; + + let was_last = file.current_chunk.lines().len() == file.line_idx + 1; + let file_number = file.file_number; + + if was_last { + let separator = self.settings.line_ending.into(); + let recycled = self + .recycled + .take() + .unwrap_or_else(|| RecycledChunk::new(8 * 1024)); + let next_chunk = if let Some(reader) = self.readers[file_number].as_mut() { + let (chunk, should_continue) = chunks::read_to_chunk( + recycled, + None, + &mut reader.carry_over, + reader.file.as_read(), + &mut iter::empty(), + separator, + self.settings, + )?; + if !should_continue && let Some(reader) = self.readers[file_number].take() { + reader.file.finished_reading()?; + } + chunk + } else { + None + }; + + if let Some(next_chunk) = next_chunk { + let mut file = self.heap.peek_mut().unwrap(); + file.current_chunk = Rc::new(next_chunk); + file.line_idx = 0; + } else { + self.heap.pop(); + } + } else { + self.heap.peek_mut().unwrap().line_idx += 1; + } + + // Recycle the previous chunk if no other reference holds it. + if let Some(prev) = prev + && let Ok(chunk) = Rc::try_unwrap(prev.chunk) + { + self.recycled = Some(chunk.recycle()); + } + } + Ok(!self.heap.is_empty()) + } +} + +pub(super) fn merge_without_limit>>( + files: F, + settings: &GlobalSettings, +) -> UResult> { + let separator = settings.line_ending.into(); + let mut readers: Vec>> = Vec::new(); + let mut mergeable_files = Vec::new(); + + for (file_number, file) in files.enumerate() { + let mut reader = SyncReaderFile { + file: file?, + carry_over: vec![], + }; + let recycled = RecycledChunk::new(8 * 1024); + let (chunk, should_continue) = chunks::read_to_chunk( + recycled, + None, + &mut reader.carry_over, + reader.file.as_read(), + &mut iter::empty(), + separator, + settings, + )?; + + if let Some(chunk) = chunk { + mergeable_files.push(SyncMergeableFile { + current_chunk: Rc::new(chunk), + line_idx: 0, + file_number, + }); + if should_continue { + readers.push(Some(reader)); + } else { + reader.file.finished_reading()?; + readers.push(None); + } + } else { + reader.file.finished_reading()?; + readers.push(None); + } + } + + Ok(SyncFileMerger { + heap: binary_heap_plus::BinaryHeap::from_vec_cmp( + mergeable_files, + FileComparator { settings }, + ), + readers, + prev: None, + recycled: None, + settings, + }) +} diff --git a/src/uu/sort/src/merge/threaded.rs b/src/uu/sort/src/merge/threaded.rs new file mode 100644 index 00000000000..ee5a68abf7b --- /dev/null +++ b/src/uu/sort/src/merge/threaded.rs @@ -0,0 +1,277 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +//! Multi-threaded merge: a reader thread feeds chunks into the per-file +//! channels while the main thread merges the next-line heap. + +use std::{ + cmp::Ordering, + io::Write, + iter, + rc::Rc, + sync::mpsc::{Receiver, Sender, SyncSender, TryRecvError, channel, sync_channel}, + thread::{self, JoinHandle}, +}; + +use compare::Compare; +use uucore::error::{FromIo, UResult}; + +use crate::{ + GlobalSettings, Output, + chunks::{self, Chunk, RecycledChunk}, + compare_by, +}; + +use super::{FileComparator, MergeInput, PreviousLine}; + +pub(super) const SUPPORTS_COMPRESSION: bool = true; + +/// Merge files without limiting how many files are concurrently open. +/// +/// It is the responsibility of the caller to ensure that `files` yields only +/// as many files as we are allowed to open concurrently. +pub(super) fn merge_without_limit>>( + files: F, + settings: &GlobalSettings, +) -> UResult> { + let (request_sender, request_receiver) = channel(); + let mut reader_files = Vec::with_capacity(files.size_hint().0); + let mut loaded_receivers = Vec::with_capacity(files.size_hint().0); + for (file_number, file) in files.enumerate() { + let (sender, receiver) = sync_channel(2); + loaded_receivers.push(receiver); + reader_files.push(Some(ReaderFile { + file: file?, + sender, + carry_over: vec![], + })); + // Send the initial chunk to trigger a read for each file + request_sender + .send((file_number, RecycledChunk::new(8 * 1024))) + .unwrap(); + } + + // Send the second chunk for each file + for file_number in 0..reader_files.len() { + request_sender + .send((file_number, RecycledChunk::new(8 * 1024))) + .unwrap(); + } + + let reader_join_handle = thread::spawn({ + let settings = settings.clone(); + move || { + reader( + &request_receiver, + &mut reader_files, + &settings, + settings.line_ending.into(), + ) + } + }); + + let mut mergeable_files = vec![]; + + for (file_number, receiver) in loaded_receivers.into_iter().enumerate() { + if let Ok(chunk) = receiver.recv() { + mergeable_files.push(MergeableFile { + current_chunk: Rc::new(chunk), + file_number, + line_idx: 0, + receiver, + }); + } + } + + Ok(FileMerger { + heap: binary_heap_plus::BinaryHeap::from_vec_cmp( + mergeable_files, + FileComparator { settings }, + ), + request_sender, + prev: None, + reader_join_handle, + }) +} + +/// The struct on the reader thread representing an input file +struct ReaderFile { + file: M, + sender: SyncSender, + carry_over: Vec, +} + +/// The function running on the reader thread. +fn reader( + recycled_receiver: &Receiver<(usize, RecycledChunk)>, + files: &mut [Option>], + settings: &GlobalSettings, + separator: u8, +) -> UResult<()> { + for (file_idx, recycled_chunk) in recycled_receiver { + if let Some(ReaderFile { + file, + sender, + carry_over, + }) = &mut files[file_idx] + { + let should_continue = chunks::read( + sender, + recycled_chunk, + None, + carry_over, + file.as_read(), + &mut iter::empty(), + separator, + settings, + )?; + if !should_continue { + // Remove the file from the list by replacing it with `None`. + let ReaderFile { file, .. } = files[file_idx].take().unwrap(); + // Depending on the kind of the `MergeInput`, this may delete the file: + file.finished_reading()?; + } + } + } + Ok(()) +} + +/// The struct on the main thread representing an input file +pub(super) struct MergeableFile { + current_chunk: Rc, + line_idx: usize, + receiver: Receiver, + file_number: usize, +} + +/// Merges files together. This is **not** an iterator because of lifetime problems. +pub(super) struct FileMerger<'a> { + heap: binary_heap_plus::BinaryHeap>, + request_sender: Sender<(usize, RecycledChunk)>, + prev: Option, + reader_join_handle: JoinHandle>, +} + +impl FileMerger<'_> { + /// Write the merged contents to the output file. + pub(super) fn write_all(self, settings: &GlobalSettings, output: Output) -> UResult<()> { + let mut out = output.into_write(); + self.write_all_to(settings, &mut out) + } + + pub(super) fn write_all_to( + mut self, + settings: &GlobalSettings, + out: &mut impl Write, + ) -> UResult<()> { + let write_result = loop { + match self + .write_next(out, settings) + .map_err_context(|| "write failed".into()) + { + Ok(true) => (), + Ok(false) => break Ok(()), + Err(error) => break Err(error), + } + }; + + let Self { + heap, + request_sender, + reader_join_handle, + .. + } = self; + + drop(request_sender); + let mut files = heap.into_vec(); + while !files.is_empty() { + files.retain(|file| { + !matches!(file.receiver.try_recv(), Err(TryRecvError::Disconnected)) + }); + thread::yield_now(); + } + + let reader_result = reader_join_handle.join().unwrap(); + write_result.and(reader_result) + } + + fn write_next( + &mut self, + writer: &mut impl Write, + settings: &GlobalSettings, + ) -> std::io::Result { + if let Some(file) = self.heap.peek() { + let prev = self.prev.replace(PreviousLine { + chunk: file.current_chunk.clone(), + line_idx: file.line_idx, + file_number: file.file_number, + }); + + file.current_chunk.with_dependent(|_, contents| { + let current_line = &contents.lines[file.line_idx]; + if settings.unique + && let Some(prev) = &prev + { + let cmp = compare_by( + &prev.chunk.lines()[prev.line_idx], + current_line, + settings, + prev.chunk.line_data(), + file.current_chunk.line_data(), + ); + if cmp == Ordering::Equal { + return Ok(()); + } + } + current_line.write(writer, settings) + })?; + + let was_last_line_for_file = file.current_chunk.lines().len() == file.line_idx + 1; + + if was_last_line_for_file { + if let Ok(next_chunk) = file.receiver.recv() { + let mut file = self.heap.peek_mut().unwrap(); + file.current_chunk = Rc::new(next_chunk); + file.line_idx = 0; + } else { + self.heap.pop(); + } + } else { + // This will cause the comparison to use a different line and the heap to readjust. + self.heap.peek_mut().unwrap().line_idx += 1; + } + + if let Some(prev) = prev + && let Ok(prev_chunk) = Rc::try_unwrap(prev.chunk) + { + // If nothing is referencing the previous chunk anymore, this means that the previous line + // was the last line of the chunk. We can recycle the chunk. + self.request_sender + .send((prev.file_number, prev_chunk.recycle())) + .ok(); + } + } + Ok(!self.heap.is_empty()) + } +} + +impl Compare for FileComparator<'_> { + fn compare(&self, a: &MergeableFile, b: &MergeableFile) -> Ordering { + let mut cmp = compare_by( + &a.current_chunk.lines()[a.line_idx], + &b.current_chunk.lines()[b.line_idx], + self.settings, + a.current_chunk.line_data(), + b.current_chunk.line_data(), + ); + if cmp == Ordering::Equal { + // To make sorting stable, we need to consider the file number as well, + // as lines from a file with a lower number are to be considered "earlier". + cmp = a.file_number.cmp(&b.file_number); + } + // BinaryHeap is a max heap. We use it as a min heap, so we need to reverse the ordering. + cmp.reverse() + } +} diff --git a/src/uu/sort/src/parallel.rs b/src/uu/sort/src/parallel.rs new file mode 100644 index 00000000000..4adb29ad432 --- /dev/null +++ b/src/uu/sort/src/parallel.rs @@ -0,0 +1,59 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +//! Parallel-or-sequential sort helpers and thread-pool initialization. +//! +//! On targets without thread support (`wasm32-wasip1` without atomics) these +//! fall back to the sequential `[T]::sort_*` methods and a no-op pool init. +//! On every other target they use rayon's parallel sorts. + +#[cfg(not(wasi_no_threads))] +mod imp { + use std::cmp::Ordering; + use std::num::NonZero; + + use rayon::slice::ParallelSliceMut; + + pub fn sort_by(slice: &mut [T], cmp: impl Fn(&T, &T) -> Ordering + Sync) { + slice.par_sort_by(cmp); + } + + pub fn sort_unstable_by(slice: &mut [T], cmp: impl Fn(&T, &T) -> Ordering + Sync) { + slice.par_sort_unstable_by(cmp); + } + + pub fn init_thread_pool(num_threads: Option) { + let num_threads = num_threads.map_or_else( + || std::thread::available_parallelism().map_or(1, NonZero::get), + |n| n as usize, + ); + let _ = rayon::ThreadPoolBuilder::new() + .num_threads(num_threads) + .build_global(); + } +} + +#[cfg(wasi_no_threads)] +mod imp { + use std::cmp::Ordering; + + // The `Send`/`Sync` bounds mirror the parallel implementation so that call + // sites compile identically on both targets. They are stricter than the + // underlying `[T]::sort_*` methods require, but every caller in this crate + // already satisfies them. + pub fn sort_by(slice: &mut [T], cmp: impl Fn(&T, &T) -> Ordering + Sync) { + slice.sort_by(cmp); + } + + pub fn sort_unstable_by(slice: &mut [T], cmp: impl Fn(&T, &T) -> Ordering + Sync) { + slice.sort_unstable_by(cmp); + } + + pub fn init_thread_pool(_num_threads: Option) { + // No-op: there is no thread pool on this target, so --parallel is ignored. + } +} + +pub use imp::{init_thread_pool, sort_by, sort_unstable_by}; diff --git a/src/uu/sort/src/sort.rs b/src/uu/sort/src/sort.rs index 3b549c38b5b..74bc902e887 100644 --- a/src/uu/sort/src/sort.rs +++ b/src/uu/sort/src/sort.rs @@ -17,6 +17,7 @@ mod custom_str_cmp; mod ext_sort; mod merge; mod numeric_str_cmp; +mod parallel; mod tmp_dir; use bigdecimal::BigDecimal; @@ -29,8 +30,6 @@ use foldhash::fast::FoldHasher; use foldhash::{HashMap, SharedSeed}; use numeric_str_cmp::{NumInfo, NumInfoParseSettings, human_numeric_str_cmp, numeric_str_cmp}; use rand::{RngExt as _, rng}; -#[cfg(not(wasi_no_threads))] -use rayon::slice::ParallelSliceMut; use std::cmp::Ordering; use std::env; use std::ffi::{OsStr, OsString}; @@ -2083,17 +2082,8 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { settings.ignore_non_printing = ignore_non_printing; settings.ignore_case = ignore_case; - // WASI doesn't support threads, so we ignore the corresponding option - #[cfg(not(target_os = "wasi"))] - { - let threads = matches - .get_one::(options::PARALLEL) - .copied() - .unwrap_or_else(|| std::thread::available_parallelism().map_or(1, |n| n.get() as u64)); - let _ = rayon::ThreadPoolBuilder::new() - .num_threads(threads as usize) - .build_global(); - } + // On targets without thread support this is a no-op and --parallel is ignored. + parallel::init_thread_pool(matches.get_one::(options::PARALLEL).copied()); if let Some(size_str) = matches.get_one::(options::BUF_SIZE) { settings.buffer_size = GlobalSettings::parse_byte_count(size_str).map_err(|e| { @@ -2573,18 +2563,10 @@ fn exec( fn sort_by<'a>(unsorted: &mut Vec>, settings: &GlobalSettings, line_data: &LineData<'a>) { let cmp = |a: &Line<'a>, b: &Line<'a>| compare_by(a, b, settings, line_data, line_data); - // WASI does not support threads, so use non-parallel sort to avoid - // rayon's thread pool which triggers an unreachable trap. if settings.stable || settings.unique { - #[cfg(not(wasi_no_threads))] - unsorted.par_sort_by(cmp); - #[cfg(wasi_no_threads)] - unsorted.sort_by(cmp); + parallel::sort_by(unsorted, cmp); } else { - #[cfg(not(wasi_no_threads))] - unsorted.par_sort_unstable_by(cmp); - #[cfg(wasi_no_threads)] - unsorted.sort_unstable_by(cmp); + parallel::sort_unstable_by(unsorted, cmp); } } From 89730f3349ac9b5b1709199a537d82c262e0e3ba Mon Sep 17 00:00:00 2001 From: Anthony DePasquale Date: Tue, 21 Apr 2026 09:13:10 +0200 Subject: [PATCH 13/17] util: add run-wasi-tests-docker.sh for local Linux verification --- util/run-wasi-tests-docker.sh | 92 +++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100755 util/run-wasi-tests-docker.sh diff --git a/util/run-wasi-tests-docker.sh b/util/run-wasi-tests-docker.sh new file mode 100755 index 00000000000..dcd2266808a --- /dev/null +++ b/util/run-wasi-tests-docker.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash + +# spell-checker:ignore mktemp wasip wasmtime UUTESTS rustup + +# Run the WASI integration tests in an Ubuntu 24.04 container. Mirrors the +# "Run integration tests via wasmtime" step of .github/workflows/wasi.yml +# (the unit-test step cross-compiles to wasm and runs under wasmtime on any +# host, so macOS already covers it). Keep the selector list below in sync +# with that workflow. +# +# The gap this closes: integration tests are host-built and many are gated +# with #[cfg(not(target_vendor = "apple"))] / #[cfg(target_os = "linux")], +# so macOS silently excludes them. + +set -euo pipefail + +command -v docker >/dev/null 2>&1 || { + echo "error: docker not found in PATH" >&2 + exit 1 +} +docker info >/dev/null 2>&1 || { + echo "error: docker daemon not reachable" >&2 + exit 1 +} + +ME="${0}" +ME_resolved="$(readlink -f -- "${ME}" 2>/dev/null || python3 -c 'import os,sys;print(os.path.realpath(sys.argv[1]))' "${ME}" 2>/dev/null || true)" +if [[ -z "${ME_resolved}" || ! -f "${ME_resolved}" ]]; then + echo "error: could not resolve script path (neither 'readlink -f' nor python3 available)" >&2 + exit 1 +fi +ME_dir="$(dirname -- "${ME_resolved}")" +REPO_main_dir="$(dirname -- "${ME_dir}")" + +HOST_LOG_DIR="$(mktemp -d -t wasi-coreutils-XXXXXX)" +HOST_LOG="${HOST_LOG_DIR}/wasi-test-output.log" + +# Report the log location on every exit path (including docker failure). +trap 'echo; echo "Full log saved to ${HOST_LOG}"' EXIT + +# Source is mounted read-only; only the log dir is writable by the container. +docker run --rm -i \ + -v "${REPO_main_dir}:/src:ro" \ + -v "${HOST_LOG_DIR}:/host-tmp" \ + ubuntu:24.04 bash -se <<'EOF' +set -euo pipefail +export DEBIAN_FRONTEND=noninteractive +apt-get update -qq +apt-get install -y -qq curl rsync ca-certificates build-essential pkg-config libssl-dev xz-utils >/dev/null + +curl -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable --profile minimal --target wasm32-wasip1 >/dev/null +. "$HOME/.cargo/env" + +curl -sSf https://wasmtime.dev/install.sh | bash >/dev/null +export PATH="$HOME/.wasmtime/bin:$PATH" + +mkdir -p /work +rsync -a --exclude='target' --exclude='target-linux' --exclude='.git' /src/ /work/ +cd /work +mkdir -p target-linux +export CARGO_TARGET_DIR=/work/target-linux + +LOG=/host-tmp/wasi-test-output.log +: > "$LOG" + +echo "=== Building WASI binary ===" | tee -a "$LOG" +RUSTFLAGS="--cfg wasi_runner" \ + cargo build --target wasm32-wasip1 --no-default-features --features feat_wasm 2>&1 | tee -a "$LOG" + +# `set +e` around the pipeline so a test failure still reaches the summary. +echo "=== Running integration tests via wasmtime ===" | tee -a "$LOG" +set +e +RUSTFLAGS="--cfg wasi_runner" \ +UUTESTS_BINARY_PATH="$CARGO_TARGET_DIR/wasm32-wasip1/debug/coreutils.wasm" \ +UUTESTS_WASM_RUNNER=wasmtime \ + cargo test --test tests -- \ + test_base32:: test_base64:: test_basenc:: test_basename:: \ + test_cat:: test_comm:: test_cut:: test_dirname:: test_echo:: \ + test_expand:: test_factor:: test_false:: test_fold:: \ + test_head:: test_link:: test_nl:: test_numfmt:: \ + test_od:: test_paste:: test_printf:: test_shuf:: test_sort:: \ + test_sum:: test_tail:: test_tee:: test_touch:: test_tr:: \ + test_true:: test_truncate:: test_unexpand:: test_unlink:: test_wc:: \ + 2>&1 | tee -a "$LOG" +test_status=${PIPESTATUS[0]} +set -e + +echo "" | tee -a "$LOG" +echo "=== Failure summary (from saved log) ===" | tee -a "$LOG" +grep -E "FAILED|^failures:|test result" "$LOG" || echo "no FAILED / failures: / test result lines found" +exit "$test_status" +EOF From 5f9a8300f4d52dd66c88ac86d2729ad9848c1680 Mon Sep 17 00:00:00 2001 From: Anthony DePasquale Date: Tue, 21 Apr 2026 09:24:30 +0200 Subject: [PATCH 14/17] cp: extract WASI timestamp logic into set_timestamps function --- src/uu/cp/src/cp.rs | 123 ++++++++++++++++++++++---------------------- 1 file changed, 62 insertions(+), 61 deletions(-) diff --git a/src/uu/cp/src/cp.rs b/src/uu/cp/src/cp.rs index f18db4b7c41..a0ec2318fa0 100644 --- a/src/uu/cp/src/cp.rs +++ b/src/uu/cp/src/cp.rs @@ -1781,6 +1781,66 @@ fn copy_extended_attrs(source: &Path, dest: &Path, skip_selinux: bool) -> CopyRe Ok(()) } +/// Copy the access and modification timestamps from `source_metadata` onto `dest`. +/// If `dest` is a symlink, the symlink's own timestamps are set rather than the +/// target's. +/// +/// On WASI this calls `rustix::fs::utimensat` directly because `filetime`'s +/// WASI backend panics in `from_last_{access,modification}_time`. `SystemTime` +/// values are converted to `Timespec` against `UNIX_EPOCH`, matching WASI's +/// `path_filestat_set_times` contract (unsigned nanosecond count — pre-epoch +/// source times can't be represented). +fn set_timestamps(source_metadata: &Metadata, dest: &Path) -> CopyResult<()> { + #[cfg(target_os = "wasi")] + { + use std::time::UNIX_EPOCH; + let to_timespec = |t: std::time::SystemTime| -> io::Result { + let d = t + .duration_since(UNIX_EPOCH) + .map_err(|e| io::Error::new(io::ErrorKind::Unsupported, e))?; + Ok(rustix::fs::Timespec { + tv_sec: d.as_secs() as i64, + tv_nsec: d.subsec_nanos() as i32, + }) + }; + let timestamps = rustix::fs::Timestamps { + last_access: to_timespec(source_metadata.accessed()?)?, + last_modification: to_timespec(source_metadata.modified()?)?, + }; + let flags = if dest.is_symlink() { + rustix::fs::AtFlags::SYMLINK_NOFOLLOW + } else { + rustix::fs::AtFlags::empty() + }; + rustix::fs::utimensat(rustix::fs::CWD, dest, ×tamps, flags) + .map_err(io::Error::from)?; + Ok(()) + } + + #[cfg(not(target_os = "wasi"))] + { + let atime = FileTime::from_last_access_time(source_metadata); + let mtime = FileTime::from_last_modification_time(source_metadata); + #[cfg(unix)] + let no_open = { + let ft = source_metadata.file_type(); + dest.is_symlink() + || ft.is_fifo() + || ft.is_socket() + || ft.is_char_device() + || ft.is_block_device() + }; + #[cfg(not(unix))] + let no_open = dest.is_symlink(); + if no_open { + filetime::set_symlink_file_times(dest, atime, mtime)?; + } else { + filetime::set_file_times(dest, atime, mtime)?; + } + Ok(()) + } +} + /// Copy the specified attributes from one path to another. /// If `skip_selinux_xattr` is true, the security.selinux xattr will not be copied /// (used when -Z is specified to set the default context instead). @@ -1889,67 +1949,8 @@ pub(crate) fn copy_attributes( Ok(()) })?; - handle_preserve(attributes.timestamps, || -> CopyResult<()> { - #[cfg(target_os = "wasi")] - { - // `filetime`'s WASI backend panics in - // `from_last_{access,modification}_time`. Reach `utimensat` directly - // through `rustix`, converting `SystemTime` → `Timespec` via - // `UNIX_EPOCH` (which matches the `path_filestat_set_times` contract). - use std::time::UNIX_EPOCH; - let to_timespec = |t: std::time::SystemTime| -> io::Result { - // Pre-epoch source times can't be represented by WASI's - // `path_filestat_set_times` (unsigned nanosecond count). - let d = t - .duration_since(UNIX_EPOCH) - .map_err(|e| io::Error::new(io::ErrorKind::Unsupported, e))?; - Ok(rustix::fs::Timespec { - tv_sec: d.as_secs() as _, - tv_nsec: d.subsec_nanos() as _, - }) - }; - let timestamps = rustix::fs::Timestamps { - last_access: to_timespec(source_metadata.accessed()?)?, - last_modification: to_timespec(source_metadata.modified()?)?, - }; - let flags = if dest.is_symlink() { - rustix::fs::AtFlags::SYMLINK_NOFOLLOW - } else { - rustix::fs::AtFlags::empty() - }; - rustix::fs::utimensat(rustix::fs::CWD, dest, ×tamps, flags) - .map_err(io::Error::from)?; - Ok(()) - } - - #[cfg(not(target_os = "wasi"))] - { - let atime = FileTime::from_last_access_time(&source_metadata); - let mtime = FileTime::from_last_modification_time(&source_metadata); - // `set_file_times` opens the destination (O_RDONLY) before calling - // futimens; opening a FIFO or device with no peer blocks forever, and a - // socket cannot be opened at all. For symlinks and these special files - // use the path-based, no-follow variant, which sets the times via - // utimensat without opening. - #[cfg(unix)] - let no_open = { - let ft = source_metadata.file_type(); - dest.is_symlink() - || ft.is_fifo() - || ft.is_socket() - || ft.is_char_device() - || ft.is_block_device() - }; - #[cfg(not(unix))] - let no_open = dest.is_symlink(); - if no_open { - filetime::set_symlink_file_times(dest, atime, mtime)?; - } else { - filetime::set_file_times(dest, atime, mtime)?; - } - - Ok(()) - } + handle_preserve(attributes.timestamps, || { + set_timestamps(&source_metadata, dest) })?; #[cfg(all(feature = "selinux", any(target_os = "linux", target_os = "android")))] From 102641ef22bb378e1a87248fdf97f6c9eb4a7786 Mon Sep 17 00:00:00 2001 From: Anthony DePasquale Date: Sat, 9 May 2026 17:56:56 +0200 Subject: [PATCH 15/17] tests/sort: ignore test_consistent_sorting_with_i18n_collate on WASI --- tests/by-util/test_sort.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/by-util/test_sort.rs b/tests/by-util/test_sort.rs index 60bdfd24dbb..6a325dfe1e0 100644 --- a/tests/by-util/test_sort.rs +++ b/tests/by-util/test_sort.rs @@ -3160,6 +3160,7 @@ e f 5436 down data path1 path2 path3 path4 path5\n"; } #[test] +#[cfg_attr(wasi_runner, ignore = "WASI: no locale data")] fn test_consistent_sorting_with_i18n_collate() { // Regression test for issue #11980 // Lexicographic fallback sorting for equal sorting keys for 01 and 0_1 From 7c4fd27cb7115698ce91afef2fe65edfc9ea1470 Mon Sep 17 00:00:00 2001 From: Anthony DePasquale Date: Fri, 12 Jun 2026 00:25:36 +0200 Subject: [PATCH 16/17] tests/touch: ignore upstream symlink tests on WASI --- tests/by-util/test_touch.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/by-util/test_touch.rs b/tests/by-util/test_touch.rs index 4cc27cba528..4136d8e686d 100644 --- a/tests/by-util/test_touch.rs +++ b/tests/by-util/test_touch.rs @@ -1183,6 +1183,10 @@ fn test_touch_device_files() { // check in util/check-safe-traversal.sh. #[test] #[cfg(unix)] +#[cfg_attr( + wasi_runner, + ignore = "WASI sandbox: absolute symlink targets cannot be followed" +)] fn test_touch_does_not_truncate_symlink_target() { use std::os::unix::fs::symlink; @@ -1198,6 +1202,10 @@ fn test_touch_does_not_truncate_symlink_target() { // Touching a dangling symlink creates its target as an empty file, like GNU. #[test] #[cfg(unix)] +#[cfg_attr( + wasi_runner, + ignore = "WASI sandbox: absolute symlink targets cannot be followed" +)] fn test_touch_through_dangling_symlink_creates_target() { use std::os::unix::fs::symlink; From 7f0b7cfd0ca9b08d7e1d6b78430871c5213eb6e3 Mon Sep 17 00:00:00 2001 From: Anthony DePasquale Date: Sat, 8 Aug 2026 11:31:58 +0200 Subject: [PATCH 17/17] util: improve Docker WASI integration helper --- .github/workflows/wasi.yml | 4 +- util/run-wasi-integration-tests-docker.sh | 165 ++++++++++++++++++++++ util/run-wasi-tests-docker.sh | 92 ------------ 3 files changed, 168 insertions(+), 93 deletions(-) create mode 100755 util/run-wasi-integration-tests-docker.sh delete mode 100755 util/run-wasi-tests-docker.sh diff --git a/.github/workflows/wasi.yml b/.github/workflows/wasi.yml index a0a73c0863f..ac46bb323fa 100644 --- a/.github/workflows/wasi.yml +++ b/.github/workflows/wasi.yml @@ -68,7 +68,9 @@ jobs: UUTESTS_WASM_RUNNER=wasmtime \ cargo test --test tests -- \ test_base32:: test_base64:: test_basenc:: test_basename:: \ - test_cat:: test_comm:: test_cut:: test_dirname:: test_echo:: \ + test_cat:: \ + test_cp::test_cp_arg_symlink test_cp::test_cp_preserve_timestamps \ + test_comm:: test_cut:: test_dirname:: test_echo:: \ test_expand:: test_factor:: test_false:: test_fold:: \ test_head:: test_link:: test_ln:: test_nl:: test_numfmt:: \ test_od:: test_paste:: test_printf:: test_shuf:: test_sort:: \ diff --git a/util/run-wasi-integration-tests-docker.sh b/util/run-wasi-integration-tests-docker.sh new file mode 100755 index 00000000000..2a31af7d70e --- /dev/null +++ b/util/run-wasi-integration-tests-docker.sh @@ -0,0 +1,165 @@ +#!/usr/bin/env bash + +# spell-checker:ignore wasip wasmtime UUTESTS rustup + +# Run the WASI integration-test selection from .github/workflows/wasi.yml in +# an Ubuntu 24.04 container. This includes Linux-only host-test paths that a +# native macOS run does not compile. + +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: run-wasi-integration-tests-docker.sh [--source PATH] + +Run the WASI integration tests selected by PATH/.github/workflows/wasi.yml. +PATH defaults to the repository containing this script. +EOF +} + +SOURCE_DIR="" +while (($# > 0)); do + case "$1" in + --source) + if (($# < 2)); then + echo "error: --source requires a path" >&2 + exit 2 + fi + SOURCE_DIR="$2" + shift 2 + ;; + --source=*) + SOURCE_DIR="${1#*=}" + shift + ;; + -h | --help) + usage + exit 0 + ;; + *) + echo "error: unknown argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +SCRIPT_DIR="$(CDPATH='' cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +if [[ -z "${SOURCE_DIR}" ]]; then + SOURCE_DIR="$(dirname -- "${SCRIPT_DIR}")" +elif [[ ! -d "${SOURCE_DIR}" ]]; then + echo "error: source directory does not exist: ${SOURCE_DIR}" >&2 + exit 2 +else + SOURCE_DIR="$(CDPATH='' cd -- "${SOURCE_DIR}" && pwd -P)" +fi + +if [[ ! -f "${SOURCE_DIR}/Cargo.toml" || ! -f "${SOURCE_DIR}/.github/workflows/wasi.yml" ]]; then + echo "error: source directory is not a coreutils working tree: ${SOURCE_DIR}" >&2 + exit 2 +fi + +command -v docker >/dev/null 2>&1 || { + echo "error: docker not found in PATH" >&2 + exit 1 +} +docker info >/dev/null 2>&1 || { + echo "error: docker daemon not reachable" >&2 + exit 1 +} + +HOST_LOG_DIR="$(mktemp -d "${TMPDIR:-/tmp}/wasi-coreutils.XXXXXX")" +HOST_LOG="${HOST_LOG_DIR}/wasi-integration-test-output.log" + +# Report the log location on every exit path, including Docker failures. +trap 'echo; echo "Full log saved to ${HOST_LOG}"' EXIT + +# Toolchains and build artifacts use named volumes so repeat runs only fetch +# updates. The source remains read-only and is copied into the container. +docker run --rm -i \ + --volume "${SOURCE_DIR}:/src:ro" \ + --volume "${HOST_LOG_DIR}:/host-tmp" \ + --volume uutils-coreutils-wasi-cargo:/root/.cargo \ + --volume uutils-coreutils-wasi-rustup:/root/.rustup \ + --volume uutils-coreutils-wasi-wasmtime:/root/.wasmtime \ + --volume uutils-coreutils-wasi-target:/target \ + ubuntu:24.04 bash -se <<'EOF' +set -euo pipefail +export DEBIAN_FRONTEND=noninteractive + +apt-get update -qq +apt-get install -y -qq curl rsync ca-certificates build-essential pkg-config libssl-dev xz-utils >/dev/null + +if [[ ! -x /root/.cargo/bin/rustup ]]; then + curl -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain none --profile minimal >/dev/null +fi +. /root/.cargo/env +rustup toolchain install stable --profile minimal --target wasm32-wasip1 >/dev/null +rustup default stable >/dev/null + +curl -sSf https://wasmtime.dev/install.sh | bash >/dev/null +export PATH="/root/.wasmtime/bin:$PATH" + +mkdir -p /work +rsync -a --exclude=target --exclude='target-*' --exclude=.git /src/ /work/ +cd /work +export CARGO_TARGET_DIR=/target + +mapfile -t TEST_SELECTORS < <( + awk ' + /cargo test --test tests --/ { + capture = 1 + next + } + capture { + for (i = 1; i <= NF; i++) { + token = $i + sub(/\\$/, "", token) + if (token ~ /^test_[[:alnum:]_:]+$/) { + print token + } + } + if ($0 !~ /\\[[:space:]]*$/) { + exit + } + } + ' .github/workflows/wasi.yml +) + +if (("${#TEST_SELECTORS[@]}" == 0)); then + echo "error: no WASI integration-test selectors found in .github/workflows/wasi.yml" >&2 + exit 1 +fi + +LOG=/host-tmp/wasi-integration-test-output.log +: > "${LOG}" + +{ + echo "=== Tool versions ===" + rustc --version + cargo --version + wasmtime --version + printf 'Integration-test selectors (%d):' "${#TEST_SELECTORS[@]}" + printf ' %s' "${TEST_SELECTORS[@]}" + echo + + echo "=== Building WASI binary ===" + RUSTFLAGS="--cfg wasi_runner" \ + cargo build --locked --target wasm32-wasip1 --no-default-features --features feat_wasm +} 2>&1 | tee -a "${LOG}" + +# Preserve the test exit status while still writing the failure summary. +echo "=== Running WASI integration tests ===" | tee -a "${LOG}" +set +e +RUSTFLAGS="--cfg wasi_runner" \ +UUTESTS_BINARY_PATH="${CARGO_TARGET_DIR}/wasm32-wasip1/debug/coreutils.wasm" \ +UUTESTS_WASM_RUNNER=wasmtime \ + cargo test --locked --test tests -- "${TEST_SELECTORS[@]}" 2>&1 | tee -a "${LOG}" +test_status=${PIPESTATUS[0]} +set -e + +echo | tee -a "${LOG}" +echo "=== Failure summary ===" | tee -a "${LOG}" +grep -E "FAILED|^failures:|test result" "${LOG}" || echo "No failure or test-result lines found" +exit "${test_status}" +EOF diff --git a/util/run-wasi-tests-docker.sh b/util/run-wasi-tests-docker.sh deleted file mode 100755 index dcd2266808a..00000000000 --- a/util/run-wasi-tests-docker.sh +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env bash - -# spell-checker:ignore mktemp wasip wasmtime UUTESTS rustup - -# Run the WASI integration tests in an Ubuntu 24.04 container. Mirrors the -# "Run integration tests via wasmtime" step of .github/workflows/wasi.yml -# (the unit-test step cross-compiles to wasm and runs under wasmtime on any -# host, so macOS already covers it). Keep the selector list below in sync -# with that workflow. -# -# The gap this closes: integration tests are host-built and many are gated -# with #[cfg(not(target_vendor = "apple"))] / #[cfg(target_os = "linux")], -# so macOS silently excludes them. - -set -euo pipefail - -command -v docker >/dev/null 2>&1 || { - echo "error: docker not found in PATH" >&2 - exit 1 -} -docker info >/dev/null 2>&1 || { - echo "error: docker daemon not reachable" >&2 - exit 1 -} - -ME="${0}" -ME_resolved="$(readlink -f -- "${ME}" 2>/dev/null || python3 -c 'import os,sys;print(os.path.realpath(sys.argv[1]))' "${ME}" 2>/dev/null || true)" -if [[ -z "${ME_resolved}" || ! -f "${ME_resolved}" ]]; then - echo "error: could not resolve script path (neither 'readlink -f' nor python3 available)" >&2 - exit 1 -fi -ME_dir="$(dirname -- "${ME_resolved}")" -REPO_main_dir="$(dirname -- "${ME_dir}")" - -HOST_LOG_DIR="$(mktemp -d -t wasi-coreutils-XXXXXX)" -HOST_LOG="${HOST_LOG_DIR}/wasi-test-output.log" - -# Report the log location on every exit path (including docker failure). -trap 'echo; echo "Full log saved to ${HOST_LOG}"' EXIT - -# Source is mounted read-only; only the log dir is writable by the container. -docker run --rm -i \ - -v "${REPO_main_dir}:/src:ro" \ - -v "${HOST_LOG_DIR}:/host-tmp" \ - ubuntu:24.04 bash -se <<'EOF' -set -euo pipefail -export DEBIAN_FRONTEND=noninteractive -apt-get update -qq -apt-get install -y -qq curl rsync ca-certificates build-essential pkg-config libssl-dev xz-utils >/dev/null - -curl -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable --profile minimal --target wasm32-wasip1 >/dev/null -. "$HOME/.cargo/env" - -curl -sSf https://wasmtime.dev/install.sh | bash >/dev/null -export PATH="$HOME/.wasmtime/bin:$PATH" - -mkdir -p /work -rsync -a --exclude='target' --exclude='target-linux' --exclude='.git' /src/ /work/ -cd /work -mkdir -p target-linux -export CARGO_TARGET_DIR=/work/target-linux - -LOG=/host-tmp/wasi-test-output.log -: > "$LOG" - -echo "=== Building WASI binary ===" | tee -a "$LOG" -RUSTFLAGS="--cfg wasi_runner" \ - cargo build --target wasm32-wasip1 --no-default-features --features feat_wasm 2>&1 | tee -a "$LOG" - -# `set +e` around the pipeline so a test failure still reaches the summary. -echo "=== Running integration tests via wasmtime ===" | tee -a "$LOG" -set +e -RUSTFLAGS="--cfg wasi_runner" \ -UUTESTS_BINARY_PATH="$CARGO_TARGET_DIR/wasm32-wasip1/debug/coreutils.wasm" \ -UUTESTS_WASM_RUNNER=wasmtime \ - cargo test --test tests -- \ - test_base32:: test_base64:: test_basenc:: test_basename:: \ - test_cat:: test_comm:: test_cut:: test_dirname:: test_echo:: \ - test_expand:: test_factor:: test_false:: test_fold:: \ - test_head:: test_link:: test_nl:: test_numfmt:: \ - test_od:: test_paste:: test_printf:: test_shuf:: test_sort:: \ - test_sum:: test_tail:: test_tee:: test_touch:: test_tr:: \ - test_true:: test_truncate:: test_unexpand:: test_unlink:: test_wc:: \ - 2>&1 | tee -a "$LOG" -test_status=${PIPESTATUS[0]} -set -e - -echo "" | tee -a "$LOG" -echo "=== Failure summary (from saved log) ===" | tee -a "$LOG" -grep -E "FAILED|^failures:|test result" "$LOG" || echo "no FAILED / failures: / test result lines found" -exit "$test_status" -EOF