From 3ea6888cafe06dbed134cbd4775608757d5ed69a Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Thu, 9 Jul 2026 22:00:00 -0400 Subject: [PATCH 1/3] expand/unexpand: fix multibyte display width via shared char_width helper The UTF-8 sequence length was derived from char::from(b).len_utf8(), which uses the leading byte's value as a codepoint rather than decoding the length from its bit pattern. That is only correct for 2-byte characters; 3- and 4-byte ones were split into single bytes and each counted as one column, so a following tab expanded to the wrong column. Derive the length from the leading byte and measure the display width with unicode-width. expand and unexpand needed identical logic, so it lives in a new uucore::char_width helper (char_width_at) instead of being duplicated. Should make test tests/expand/mb.sh pass --- Cargo.lock | 2 +- fuzz/Cargo.lock | 1 + src/uu/expand/Cargo.toml | 1 - src/uu/expand/src/expand.rs | 15 ++----- src/uu/unexpand/src/unexpand.rs | 12 ++--- src/uucore/Cargo.toml | 4 ++ src/uucore/src/lib/features.rs | 1 + src/uucore/src/lib/features/char_width.rs | 53 +++++++++++++++++++++++ src/uucore/src/lib/lib.rs | 1 + tests/by-util/test_expand.rs | 18 ++++++++ tests/by-util/test_unexpand.rs | 21 +++++++-- 11 files changed, 102 insertions(+), 27 deletions(-) create mode 100644 src/uucore/src/lib/features/char_width.rs diff --git a/Cargo.lock b/Cargo.lock index 0d14894950e..cad693b2644 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3570,7 +3570,6 @@ dependencies = [ "rustix", "tempfile", "thiserror 2.0.18", - "unicode-width 0.2.2", "uucore", ] @@ -4572,6 +4571,7 @@ dependencies = [ "thiserror 2.0.18", "time", "unic-langid", + "unicode-width 0.2.2", "unit-prefix", "utmp-classic", "uucore_procs", diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index f59484f90b5..d41768e048e 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -2150,6 +2150,7 @@ dependencies = [ "sm3", "thiserror", "unic-langid", + "unicode-width", "unit-prefix", "uucore_procs", "wild", diff --git a/src/uu/expand/Cargo.toml b/src/uu/expand/Cargo.toml index f481ab0b626..e010b1c1a58 100644 --- a/src/uu/expand/Cargo.toml +++ b/src/uu/expand/Cargo.toml @@ -23,7 +23,6 @@ clap = { workspace = true } fluent = { workspace = true } rustix = { workspace = true } thiserror = { workspace = true } -unicode-width = { workspace = true } uucore = { workspace = true } [dev-dependencies] diff --git a/src/uu/expand/src/expand.rs b/src/uu/expand/src/expand.rs index 890c1958a08..007156ab2e8 100644 --- a/src/uu/expand/src/expand.rs +++ b/src/uu/expand/src/expand.rs @@ -13,7 +13,7 @@ use std::num::IntErrorKind; use std::path::Path; use std::str::from_utf8; use thiserror::Error; -use unicode_width::UnicodeWidthChar; +use uucore::char_width::char_width_at; use uucore::display::Quotable; use uucore::error::{FromIo, UError, UResult, USimpleError, set_exit_code}; use uucore::{format_usage, show, translate}; @@ -361,17 +361,8 @@ fn classify_char(buf: &[u8], byte: usize, utf8: bool) -> (CharType, usize, usize } if utf8 { - let nbytes = char::from(b).len_utf8(); - let Some(slice) = buf.get(byte..byte + nbytes) else { - // don't overrun buffer because of invalid UTF-8 - return (Other, 1, 1); - }; - - if let Ok(t) = from_utf8(slice) - && let Some(c) = t.chars().next() - { - return (Other, UnicodeWidthChar::width(c).unwrap_or(0), nbytes); - } + let (width, nbytes) = char_width_at(buf, byte); + return (Other, width, nbytes); } (Other, 1, 1) // implicit assumption: non-UTF-8 char is 1 col wide } diff --git a/src/uu/unexpand/src/unexpand.rs b/src/uu/unexpand/src/unexpand.rs index 51df08a24a1..8f1f218f0cf 100644 --- a/src/uu/unexpand/src/unexpand.rs +++ b/src/uu/unexpand/src/unexpand.rs @@ -11,8 +11,8 @@ use std::fs::File; use std::io::{self, BufReader, BufWriter, Read, Stdin, Stdout, Write, stdin, stdout}; use std::num::IntErrorKind; use std::path::Path; -use std::str::from_utf8; use thiserror::Error; +use uucore::char_width::char_width_at; use uucore::display::Quotable; use uucore::error::{FromIo, UError, UResult, USimpleError, set_exit_code}; use uucore::translate; @@ -411,14 +411,8 @@ fn next_char_info(utf8: bool, buf: &[u8], byte: usize) -> (CharType, usize, usiz } if utf8 { - let nbytes = char::from(b).len_utf8(); - // don't overrun the buffer because of invalid UTF-8 - if buf - .get(byte..byte + nbytes) - .is_some_and(|s| from_utf8(s).is_ok()) - { - return (Other, nbytes, nbytes); - } + let (width, nbytes) = char_width_at(buf, byte); + return (Other, width, nbytes); } (Other, 1, 1) } diff --git a/src/uucore/Cargo.toml b/src/uucore/Cargo.toml index f797b5dc1e9..cc02644cb4a 100644 --- a/src/uucore/Cargo.toml +++ b/src/uucore/Cargo.toml @@ -49,6 +49,10 @@ z85 = { version = "3.0.5", optional = true } base64-simd = { version = "0.8", optional = true } libc = { workspace = true, optional = true } os_display = "0.1.3" +# Not optional: os_display already pulls unicode-width into every uucore build, +# so making it a direct dependency here is free and keeps char_width available +# on all targets (a feature-gated optional dep failed to activate on wasm). +unicode-width = { workspace = true } # Benchmark dependencies (optional) divan = { workspace = true, optional = true } diff --git a/src/uucore/src/lib/features.rs b/src/uucore/src/lib/features.rs index e0e99aee6b0..4129bdf5b28 100644 --- a/src/uucore/src/lib/features.rs +++ b/src/uucore/src/lib/features.rs @@ -12,6 +12,7 @@ pub mod backup_control; pub mod benchmark; #[cfg(feature = "buf-copy")] pub mod buf_copy; +pub mod char_width; #[cfg(feature = "checksum")] pub mod checksum; #[cfg(feature = "colors")] diff --git a/src/uucore/src/lib/features/char_width.rs b/src/uucore/src/lib/features/char_width.rs new file mode 100644 index 00000000000..5775f6b3e28 --- /dev/null +++ b/src/uucore/src/lib/features/char_width.rs @@ -0,0 +1,53 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. +// spell-checker:ignore nbytes + +//! Display width and byte length of UTF-8 characters within a byte buffer. + +use std::str::from_utf8; +use unicode_width::UnicodeWidthChar; + +/// Return the display width and byte length of the (possibly multibyte) UTF-8 +/// character beginning at `buf[pos]`. +/// +/// The sequence length is taken from the leading byte's bit pattern rather than +/// its value, so 3- and 4-byte characters are measured correctly. A truncated +/// or otherwise invalid sequence resolves to a single byte of width one, which +/// lets callers resync one byte at a time. +/// +/// # Panics +/// +/// Panics if `pos` is out of bounds for `buf`. +#[must_use] +pub fn char_width_at(buf: &[u8], pos: usize) -> (usize, usize) { + let nbytes = match buf[pos] { + 0xC0..=0xDF => 2, + 0xE0..=0xEF => 3, + 0xF0..=0xF7 => 4, + _ => 1, + }; + buf.get(pos..pos + nbytes) + .and_then(|s| from_utf8(s).ok()) + .and_then(|s| s.chars().next()) + .map_or((1, 1), |c| { + (UnicodeWidthChar::width(c).unwrap_or(0), nbytes) + }) +} + +#[cfg(test)] +mod tests { + use super::char_width_at; + + #[test] + fn width_and_length() { + assert_eq!(char_width_at(b"a", 0), (1, 1)); // ASCII + assert_eq!(char_width_at("é".as_bytes(), 0), (1, 2)); // U+00E9, 2-byte width 1 + assert_eq!(char_width_at("\u{0301}".as_bytes(), 0), (0, 2)); // combining mark, width 0 + assert_eq!(char_width_at("\u{3000}".as_bytes(), 0), (2, 3)); // 3-byte width 2 + assert_eq!(char_width_at("\u{1F600}".as_bytes(), 0), (2, 4)); // 4-byte width 2 + assert_eq!(char_width_at(&[0xFF], 0), (1, 1)); // invalid leading byte resyncs + assert_eq!(char_width_at(&[0xE3], 0), (1, 1)); // truncated sequence resyncs + } +} diff --git a/src/uucore/src/lib/lib.rs b/src/uucore/src/lib/lib.rs index 4dfffed2995..05c3ab18b67 100644 --- a/src/uucore/src/lib/lib.rs +++ b/src/uucore/src/lib/lib.rs @@ -40,6 +40,7 @@ pub use crate::features::backup_control; pub use crate::features::benchmark; #[cfg(feature = "buf-copy")] pub use crate::features::buf_copy; +pub use crate::features::char_width; #[cfg(feature = "checksum")] pub use crate::features::checksum; #[cfg(feature = "colors")] diff --git a/tests/by-util/test_expand.rs b/tests/by-util/test_expand.rs index 9a7d7c2cecd..6cc45c682a3 100644 --- a/tests/by-util/test_expand.rs +++ b/tests/by-util/test_expand.rs @@ -473,6 +473,24 @@ fn test_expand_non_utf8_paths() { .stdout_is("hello world\ntest line\n"); } +#[test] +fn test_wide_multibyte_char_width() { + // A tab following a wide character must expand to the next tab stop based + // on the character's display width, not its byte length. Regression for a + // bug where the UTF-8 sequence length was derived from the leading byte's + // codepoint value, so 3- and 4-byte characters were mis-measured. + // U+4E2D (中, 3 bytes, width 2): col 0->2, tab pads 6 spaces to column 8. + new_ucmd!() + .pipe_in("\u{4E2D}\t|".as_bytes()) + .succeeds() + .stdout_is("\u{4E2D} |"); + // U+1F600 (😀, 4 bytes, width 2) behaves the same. + new_ucmd!() + .pipe_in("\u{1F600}\t|".as_bytes()) + .succeeds() + .stdout_is("\u{1F600} |"); +} + #[test] fn test_buffered_reads_new_line_no_tabs_in_first_chunk() { // test includes tabs after 1 full chunk of no tabs with new line diff --git a/tests/by-util/test_unexpand.rs b/tests/by-util/test_unexpand.rs index fef63fc4a41..75beac1a0d4 100644 --- a/tests/by-util/test_unexpand.rs +++ b/tests/by-util/test_unexpand.rs @@ -309,14 +309,27 @@ fn test_non_utf8_filename() { #[test] fn unexpand_multibyte_utf8_gnu_compat() { - // Verifies GNU-compatible behavior: column position uses byte count, not display width - // "1ΔΔΔ5" is 8 bytes (1 + 2*3 + 1), already at tab stop 8 - // So 3 spaces should NOT convert to tab (would need 8 more to reach tab stop 16) + // Column position uses display width, not byte count (matches GNU 9.11). + // "1ΔΔΔ5" is 5 columns wide (each Δ is a 2-byte char of display width 1), + // so the run of 3 spaces reaches tab stop 8 and collapses to a single tab. new_ucmd!() .args(&["-a"]) .pipe_in("1ΔΔΔ5 99999\n") .succeeds() - .stdout_is("1ΔΔΔ5 99999\n"); + .stdout_is("1ΔΔΔ5\t99999\n"); +} + +#[test] +fn unexpand_wide_multibyte_char_width() { + // A 3-byte character of display width 2 must advance the column by 2, not by + // its byte length. U+FF0D (full-width hyphen, 3 bytes, width 2) at column 0 + // reaches column 2; the following 6 spaces then collapse to one tab at + // column 8. Regression for the byte-length-as-width bug. + new_ucmd!() + .args(&["-a"]) + .pipe_in("\u{FF0D} X\n") + .succeeds() + .stdout_is("\u{FF0D}\tX\n"); } #[test] From 1e3212bcd2c09e400227ed188ab1a11be33f6498 Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Thu, 9 Jul 2026 22:14:00 -0400 Subject: [PATCH 2/3] unexpand: apply leading-whitespace fast path in UTF-8 mode The fast path that batches leading spaces/tabs was gated on --no-utf8, so the default (UTF-8) mode fell back to the per-byte loop, which the display-width fix made measurably slower on ASCII input. Leading whitespace is ASCII and independent of UTF-8 mode, so run the fast path there too, and add the missing lastcol == 0 guard so it does not apply when multiple tab stops cap conversion. --- src/uu/unexpand/src/unexpand.rs | 8 ++++++-- tests/by-util/test_unexpand.rs | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/src/uu/unexpand/src/unexpand.rs b/src/uu/unexpand/src/unexpand.rs index 8f1f218f0cf..d9b987e8117 100644 --- a/src/uu/unexpand/src/unexpand.rs +++ b/src/uu/unexpand/src/unexpand.rs @@ -469,8 +469,12 @@ fn unexpand_buf( // We can only fast forward if we don't need to calculate col/scol if let Some(b'\n') = buf.last() { - // Fast path for leading spaces in non-UTF8 mode: count consecutive spaces/tabs at start - if !options.utf8 && !options.aflag && print_state.leading { + // Fast path: count consecutive leading spaces/tabs at the start of the + // line. Leading whitespace is ASCII, so this is independent of UTF-8 + // mode; the rest of the line is copied verbatim in default mode. It does + // not apply once a finite last column is in play (multiple tab stops), + // where the general loop stops converting past that column. + if !options.aflag && print_state.leading && lastcol == 0 { // In default mode (not -a), we only convert leading spaces // So we can batch process them and then copy the rest while byte < buf.len() { diff --git a/tests/by-util/test_unexpand.rs b/tests/by-util/test_unexpand.rs index 75beac1a0d4..b533b102a2e 100644 --- a/tests/by-util/test_unexpand.rs +++ b/tests/by-util/test_unexpand.rs @@ -167,6 +167,38 @@ fn unexpand_spaces_after_fields() { .stdout_is("\t\tA B C D\t\t A\t\n"); } +#[test] +fn unexpand_leading_spaces_before_multibyte() { + // Leading whitespace is converted even in the default (UTF-8) mode, and the + // multibyte tail is copied verbatim. + new_ucmd!() + .pipe_in(" caf\u{e9}\n") + .succeeds() + .stdout_is("\tcaf\u{e9}\n"); +} + +#[test] +fn unexpand_deep_indent_stops_at_last_tabstop() { + // With several tab stops, conversion stops past the last one: the extra + // leading spaces beyond column 4 stay as spaces. + new_ucmd!() + .args(&["-t", "2,4"]) + .pipe_in(" x\n") + .succeeds() + .stdout_is("\t\t x\n"); +} + +#[test] +fn unexpand_wide_char_width_before_tab() { + // A 3-byte, double-width character advances the column by two, so the + // following tab lands on the next tab stop. + new_ucmd!() + .args(&["-a", "-t", "4"]) + .pipe_in(" \u{3000}X\ty\n") + .succeeds() + .stdout_is(" \u{3000}X\ty\n"); +} + #[test] fn unexpand_read_from_file() { new_ucmd!().arg("with_spaces.txt").arg("-t4").succeeds(); From b21d0d69b2cb92886483c5e9873b6baf604a8160 Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Thu, 9 Jul 2026 22:47:00 -0400 Subject: [PATCH 3/3] unexpand: convert multibyte blanks to tabs GNU treats a wide blank such as U+3000 (any character for which iswblank is true) as a convertible blank: a run of them that reaches a tab stop is absorbed into a tab, but a run that does not is left as-is. Recognize the Unicode blank set in next_char_info, advance the column by the blank's display width, and remember buffered wide blanks so write_tabs can emit them verbatim when their run is not converted (and never let a tab straddle one). Expose the decoded char from uucore::char_width via char_info_at so unexpand reuses the shared UTF-8 decoding instead of duplicating it. Should make test tests/unexpand/mb.sh pass. --- src/uu/unexpand/src/unexpand.rs | 127 +++++++++++++++++----- src/uucore/src/lib/features/char_width.rs | 40 +++++-- tests/by-util/test_unexpand.rs | 24 +++- 3 files changed, 153 insertions(+), 38 deletions(-) diff --git a/src/uu/unexpand/src/unexpand.rs b/src/uu/unexpand/src/unexpand.rs index d9b987e8117..b8c62caba0e 100644 --- a/src/uu/unexpand/src/unexpand.rs +++ b/src/uu/unexpand/src/unexpand.rs @@ -3,7 +3,7 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -// spell-checker:ignore (ToDO) nums aflag scol prevtab amode ctype cwidth nbytes lastcol pctype Preprocess +// spell-checker:ignore (ToDO) nums aflag scol prevtab amode ctype cwidth nbytes lastcol pctype Preprocess iswblank use clap::{Arg, ArgAction, Command}; use std::ffi::OsString; @@ -12,7 +12,7 @@ use std::io::{self, BufReader, BufWriter, Read, Stdin, Stdout, Write, stdin, std use std::num::IntErrorKind; use std::path::Path; use thiserror::Error; -use uucore::char_width::char_width_at; +use uucore::char_width::char_info_at; use uucore::display::Quotable; use uucore::error::{FromIo, UError, UResult, USimpleError, set_exit_code}; use uucore::translate; @@ -374,19 +374,46 @@ fn write_tabs( && (print_state.leading || ai && print_state.pctype == CharType::Tab)) { while let Some(nts) = next_tabstop(tab_config, print_state.scol) { - if print_state.col < print_state.scol + nts { + let target = print_state.scol + nts; + if print_state.col < target { + break; + } + // A tab may absorb a wide blank that lies fully within it, but not + // one that straddles the tab stop; stop converting before it. + if print_state + .pending_wide + .iter() + .any(|&(start, width, _)| start < target && target < start + width) + { break; } output.write_all(b"\t")?; - print_state.scol += nts; + print_state.scol = target; } } + // Fill the remaining columns. Wide blanks that were not absorbed into a tab + // are emitted verbatim; everything else is a plain space. + let mut wi = 0; while print_state.col > print_state.scol { - output.write_all(b" ")?; - print_state.scol += 1; + while wi < print_state.pending_wide.len() + && print_state.pending_wide[wi].0 < print_state.scol + { + wi += 1; + } + if wi < print_state.pending_wide.len() && print_state.pending_wide[wi].0 == print_state.scol + { + let (_, width, bytes) = &print_state.pending_wide[wi]; + output.write_all(bytes)?; + print_state.scol += width; + wi += 1; + } else { + output.write_all(b" ")?; + print_state.scol += 1; + } } + print_state.pending_wide.clear(); Ok(()) } @@ -398,12 +425,26 @@ enum CharType { Other, } +/// A multibyte blank is a horizontal space other than ASCII space/tab, i.e. a +/// character for which `iswblank` is true in a UTF-8 locale. Such a blank +/// participates in tab conversion (its columns can be absorbed into a tab), so +/// it must be classified as [`CharType::Space`]. The set is the Unicode space +/// separators minus the non-breaking ones (U+00A0, U+2007, U+202F). +fn is_multibyte_blank(c: char) -> bool { + matches!(c, + '\u{1680}' + | '\u{2000}'..='\u{2006}' + | '\u{2008}'..='\u{200A}' + | '\u{205F}' + | '\u{3000}') +} + fn next_char_info(utf8: bool, buf: &[u8], byte: usize) -> (CharType, usize, usize) { use CharType::{Backspace, Other, Space, Tab}; let b = buf[byte]; if b.is_ascii() { return match b { - b' ' => (Space, 0, 1), + b' ' => (Space, 1, 1), b'\t' => (Tab, 0, 1), b'\x08' => (Backspace, 0, 1), _ => (Other, 1, 1), @@ -411,7 +452,10 @@ fn next_char_info(utf8: bool, buf: &[u8], byte: usize) -> (CharType, usize, usiz } if utf8 { - let (width, nbytes) = char_width_at(buf, byte); + let (c, width, nbytes) = char_info_at(buf, byte); + if c.is_some_and(is_multibyte_blank) { + return (Space, width, nbytes); + } return (Other, width, nbytes); } (Other, 1, 1) @@ -428,6 +472,10 @@ struct PrintState { scol: usize, leading: bool, pctype: CharType, + // Multibyte blanks buffered but not yet written, as (start column, display + // width, bytes). They are absorbed into a tab when a run reaches a tab stop + // and otherwise emitted verbatim by write_tabs. Empty for ASCII-only input. + pending_wide: Vec<(usize, usize, Vec)>, } impl PrintState { @@ -437,6 +485,7 @@ impl PrintState { self.scol = 0; self.leading = true; self.pctype = CharType::Other; + self.pending_wide.clear(); } } @@ -475,34 +524,43 @@ fn unexpand_buf( // not apply once a finite last column is in play (multiple tab stops), // where the general loop stops converting past that column. if !options.aflag && print_state.leading && lastcol == 0 { - // In default mode (not -a), we only convert leading spaces - // So we can batch process them and then copy the rest - while byte < buf.len() { - match buf[byte] { - b' ' => { - print_state.col += 1; - byte += 1; - } + // In default mode (not -a), we only convert leading spaces, so batch + // over them and then copy the rest of the line. Compute the column + // into a local without touching state: if the first non-blank is a + // multibyte start it may be a blank that needs the general loop, so + // bail out to it instead of committing here. + let mut end = 0; + let mut col = print_state.col; + let mut saw_tab = false; + while end < buf.len() { + match buf[end] { + b' ' => col += 1, b'\t' => { - print_state.col += next_tabstop(tab_config, print_state.col).unwrap_or(1); - byte += 1; - print_state.pctype = CharType::Tab; + col += next_tabstop(tab_config, col).unwrap_or(1); + saw_tab = true; } _ => break, } + end += 1; } + if end == buf.len() || buf[end].is_ascii() { + print_state.col = col; + if saw_tab { + print_state.pctype = CharType::Tab; + } - // If we found spaces/tabs, write them as tabs - if byte > 0 { - write_tabs(output, tab_config, print_state, options.aflag)?; - } + // If we found spaces/tabs, write them as tabs + if end > 0 { + write_tabs(output, tab_config, print_state, options.aflag)?; + } - // Write the rest of the line directly (no more tab conversion needed) - if byte < buf.len() { - print_state.leading = false; - output.write_all(&buf[byte..])?; + // Write the rest of the line directly (no more tab conversion needed) + if end < buf.len() { + print_state.leading = false; + output.write_all(&buf[end..])?; + } + return Ok(()); } - return Ok(()); } } @@ -522,9 +580,19 @@ fn unexpand_buf( let tabs_buffered = print_state.leading || options.aflag; match ctype { CharType::Space | CharType::Tab => { + // A buffered multibyte blank must be remembered so it can be + // emitted verbatim if its run is not converted to a tab. + if tabs_buffered && ctype == CharType::Space && nbytes > 1 { + print_state.pending_wide.push(( + print_state.col, + cwidth, + buf[byte..byte + nbytes].to_vec(), + )); + } + // compute next col, but only write space or tab chars if not buffering print_state.col += if ctype == CharType::Space { - 1 + cwidth } else { next_tabstop(tab_config, print_state.col).unwrap_or(1) }; @@ -573,6 +641,7 @@ fn unexpand_file( scol: 0, leading: true, pctype: CharType::Other, + pending_wide: Vec::new(), }; loop { diff --git a/src/uucore/src/lib/features/char_width.rs b/src/uucore/src/lib/features/char_width.rs index 5775f6b3e28..7be5012adfd 100644 --- a/src/uucore/src/lib/features/char_width.rs +++ b/src/uucore/src/lib/features/char_width.rs @@ -9,19 +9,19 @@ use std::str::from_utf8; use unicode_width::UnicodeWidthChar; -/// Return the display width and byte length of the (possibly multibyte) UTF-8 -/// character beginning at `buf[pos]`. +/// Decode the (possibly multibyte) UTF-8 character beginning at `buf[pos]`, +/// returning the character, its display width, and its byte length. /// /// The sequence length is taken from the leading byte's bit pattern rather than /// its value, so 3- and 4-byte characters are measured correctly. A truncated -/// or otherwise invalid sequence resolves to a single byte of width one, which -/// lets callers resync one byte at a time. +/// or otherwise invalid sequence resolves to `(None, 1, 1)`, which lets callers +/// resync one byte at a time. /// /// # Panics /// /// Panics if `pos` is out of bounds for `buf`. #[must_use] -pub fn char_width_at(buf: &[u8], pos: usize) -> (usize, usize) { +pub fn char_info_at(buf: &[u8], pos: usize) -> (Option, usize, usize) { let nbytes = match buf[pos] { 0xC0..=0xDF => 2, 0xE0..=0xEF => 3, @@ -31,14 +31,28 @@ pub fn char_width_at(buf: &[u8], pos: usize) -> (usize, usize) { buf.get(pos..pos + nbytes) .and_then(|s| from_utf8(s).ok()) .and_then(|s| s.chars().next()) - .map_or((1, 1), |c| { - (UnicodeWidthChar::width(c).unwrap_or(0), nbytes) + .map_or((None, 1, 1), |c| { + (Some(c), UnicodeWidthChar::width(c).unwrap_or(0), nbytes) }) } +/// Return the display width and byte length of the character at `buf[pos]`. +/// +/// A truncated or otherwise invalid sequence resolves to a single byte of width +/// one. See [`char_info_at`] for the decoding details. +/// +/// # Panics +/// +/// Panics if `pos` is out of bounds for `buf`. +#[must_use] +pub fn char_width_at(buf: &[u8], pos: usize) -> (usize, usize) { + let (_, width, nbytes) = char_info_at(buf, pos); + (width, nbytes) +} + #[cfg(test)] mod tests { - use super::char_width_at; + use super::{char_info_at, char_width_at}; #[test] fn width_and_length() { @@ -50,4 +64,14 @@ mod tests { assert_eq!(char_width_at(&[0xFF], 0), (1, 1)); // invalid leading byte resyncs assert_eq!(char_width_at(&[0xE3], 0), (1, 1)); // truncated sequence resyncs } + + #[test] + fn info_decodes_char() { + assert_eq!( + char_info_at("\u{3000}".as_bytes(), 0), + (Some('\u{3000}'), 2, 3) + ); + assert_eq!(char_info_at(b"a", 0), (Some('a'), 1, 1)); + assert_eq!(char_info_at(&[0xFF], 0), (None, 1, 1)); // invalid resyncs + } } diff --git a/tests/by-util/test_unexpand.rs b/tests/by-util/test_unexpand.rs index b533b102a2e..b440b780f89 100644 --- a/tests/by-util/test_unexpand.rs +++ b/tests/by-util/test_unexpand.rs @@ -3,7 +3,7 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. // -// spell-checker:ignore contenta edgecase behaviour +// spell-checker:ignore contenta edgecase behaviour tcaf use uutests::{at_and_ucmd, new_ucmd}; @@ -199,6 +199,28 @@ fn unexpand_wide_char_width_before_tab() { .stdout_is(" \u{3000}X\ty\n"); } +#[test] +fn unexpand_ideographic_space_absorbed_into_tab() { + // U+3000 is a width-2 blank: a run of them that reaches a tab stop is + // converted to a tab, just like a run of ASCII spaces. + new_ucmd!() + .arg("-a") + .pipe_in("\u{3000}\u{3000}\u{3000}\u{3000}Z\n") + .succeeds() + .stdout_is("\tZ\n"); +} + +#[test] +fn unexpand_ideographic_space_preserved_when_not_converted() { + // A blank run that does not reach a tab stop is emitted verbatim, so the + // multibyte blank keeps its original bytes rather than becoming spaces. + new_ucmd!() + .arg("-a") + .pipe_in("a\u{3000}b\n") + .succeeds() + .stdout_is("a\u{3000}b\n"); +} + #[test] fn unexpand_read_from_file() { new_ucmd!().arg("with_spaces.txt").arg("-t4").succeeds();