expand/unexpand: fix display width of 3- and 4-byte UTF-8 characters

src/uu/expand/Cargo.toml

@@ -23,7 +23,6 @@ clap = { workspace = true }
 fluent = { workspace = true }
 rustix = { workspace = true }
 thiserror = { workspace = true }
-unicode-width = { workspace = true }
 uucore = { workspace = true }
 
 [dev-dependencies]

src/uu/expand/src/expand.rs

@@ -13,7 +13,7 @@ use std::num::IntErrorKind;
 use std::path::Path;
 use std::str::from_utf8;
 use thiserror::Error;
-use unicode_width::UnicodeWidthChar;
+use uucore::char_width::char_width_at;
 use uucore::display::Quotable;
 use uucore::error::{FromIo, UError, UResult, USimpleError, set_exit_code};
 use uucore::{format_usage, show, translate};
@@ -361,17 +361,8 @@ fn classify_char(buf: &[u8], byte: usize, utf8: bool) -> (CharType, usize, usize
     }
 
     if utf8 {
-        let nbytes = char::from(b).len_utf8();
-        let Some(slice) = buf.get(byte..byte + nbytes) else {
-            // don't overrun buffer because of invalid UTF-8
-            return (Other, 1, 1);
-        };
-
-        if let Ok(t) = from_utf8(slice)
-            && let Some(c) = t.chars().next()
-        {
-            return (Other, UnicodeWidthChar::width(c).unwrap_or(0), nbytes);
-        }
+        let (width, nbytes) = char_width_at(buf, byte);
+        return (Other, width, nbytes);
     }
     (Other, 1, 1) // implicit assumption: non-UTF-8 char is 1 col wide
 }

src/uu/unexpand/src/unexpand.rs

@@ -3,16 +3,16 @@
 // For the full copyright and license information, please view the LICENSE
 // file that was distributed with this source code.
 
-// spell-checker:ignore (ToDO) nums aflag scol prevtab amode ctype cwidth nbytes lastcol pctype Preprocess
+// spell-checker:ignore (ToDO) nums aflag scol prevtab amode ctype cwidth nbytes lastcol pctype Preprocess iswblank
 
 use clap::{Arg, ArgAction, Command};
 use std::ffi::OsString;
 use std::fs::File;
 use std::io::{self, BufReader, BufWriter, Read, Stdin, Stdout, Write, stdin, stdout};
 use std::num::IntErrorKind;
 use std::path::Path;
-use std::str::from_utf8;
 use thiserror::Error;
+use uucore::char_width::char_info_at;
 use uucore::display::Quotable;
 use uucore::error::{FromIo, UError, UResult, USimpleError, set_exit_code};
 use uucore::translate;
@@ -374,19 +374,46 @@ fn write_tabs(
             && (print_state.leading || ai && print_state.pctype == CharType::Tab))
     {
         while let Some(nts) = next_tabstop(tab_config, print_state.scol) {
-            if print_state.col < print_state.scol + nts {
+            let target = print_state.scol + nts;
+            if print_state.col < target {
+                break;
+            }
+            // A tab may absorb a wide blank that lies fully within it, but not
+            // one that straddles the tab stop; stop converting before it.
+            if print_state
+                .pending_wide
+                .iter()
+                .any(|&(start, width, _)| start < target && target < start + width)
+            {
                 break;
             }
 
             output.write_all(b"\t")?;
-            print_state.scol += nts;
+            print_state.scol = target;
         }
     }
 
+    // Fill the remaining columns. Wide blanks that were not absorbed into a tab
+    // are emitted verbatim; everything else is a plain space.
+    let mut wi = 0;
     while print_state.col > print_state.scol {
-        output.write_all(b" ")?;
-        print_state.scol += 1;
+        while wi < print_state.pending_wide.len()
+            && print_state.pending_wide[wi].0 < print_state.scol
+        {
+            wi += 1;
+        }
+        if wi < print_state.pending_wide.len() && print_state.pending_wide[wi].0 == print_state.scol
+        {
+            let (_, width, bytes) = &print_state.pending_wide[wi];
+            output.write_all(bytes)?;
+            print_state.scol += width;
+            wi += 1;
+        } else {
+            output.write_all(b" ")?;
+            print_state.scol += 1;
+        }
     }
+    print_state.pending_wide.clear();
     Ok(())
 }
 
@@ -398,27 +425,38 @@ enum CharType {
     Other,
 }
 
+/// A multibyte blank is a horizontal space other than ASCII space/tab, i.e. a
+/// character for which `iswblank` is true in a UTF-8 locale. Such a blank
+/// participates in tab conversion (its columns can be absorbed into a tab), so
+/// it must be classified as [`CharType::Space`]. The set is the Unicode space
+/// separators minus the non-breaking ones (U+00A0, U+2007, U+202F).
+fn is_multibyte_blank(c: char) -> bool {
+    matches!(c,
+        '\u{1680}'
+        | '\u{2000}'..='\u{2006}'
+        | '\u{2008}'..='\u{200A}'
+        | '\u{205F}'
+        | '\u{3000}')
+}
+
 fn next_char_info(utf8: bool, buf: &[u8], byte: usize) -> (CharType, usize, usize) {
     use CharType::{Backspace, Other, Space, Tab};
     let b = buf[byte];
     if b.is_ascii() {
         return match b {
-            b' ' => (Space, 0, 1),
+            b' ' => (Space, 1, 1),
             b'\t' => (Tab, 0, 1),
             b'\x08' => (Backspace, 0, 1),
             _ => (Other, 1, 1),
         };
     }
 
     if utf8 {
-        let nbytes = char::from(b).len_utf8();
-        // don't overrun the buffer because of invalid UTF-8
-        if buf
-            .get(byte..byte + nbytes)
-            .is_some_and(|s| from_utf8(s).is_ok())
-        {
-            return (Other, nbytes, nbytes);
+        let (c, width, nbytes) = char_info_at(buf, byte);
+        if c.is_some_and(is_multibyte_blank) {
+            return (Space, width, nbytes);
         }
+        return (Other, width, nbytes);
     }
     (Other, 1, 1)
 }
@@ -434,6 +472,10 @@ struct PrintState {
     scol: usize,
     leading: bool,
     pctype: CharType,
+    // Multibyte blanks buffered but not yet written, as (start column, display
+    // width, bytes). They are absorbed into a tab when a run reaches a tab stop
+    // and otherwise emitted verbatim by write_tabs. Empty for ASCII-only input.
+    pending_wide: Vec<(usize, usize, Vec<u8>)>,
 }
 
 impl PrintState {
@@ -443,6 +485,7 @@ impl PrintState {
         self.scol = 0;
         self.leading = true;
         self.pctype = CharType::Other;
+        self.pending_wide.clear();
     }
 }
 
@@ -475,36 +518,49 @@ fn unexpand_buf(
 
     // We can only fast forward if we don't need to calculate col/scol
     if let Some(b'\n') = buf.last() {
-        // Fast path for leading spaces in non-UTF8 mode: count consecutive spaces/tabs at start
-        if !options.utf8 && !options.aflag && print_state.leading {
-            // In default mode (not -a), we only convert leading spaces
-            // So we can batch process them and then copy the rest
-            while byte < buf.len() {
-                match buf[byte] {
-                    b' ' => {
-                        print_state.col += 1;
-                        byte += 1;
-                    }
+        // Fast path: count consecutive leading spaces/tabs at the start of the
+        // line. Leading whitespace is ASCII, so this is independent of UTF-8
+        // mode; the rest of the line is copied verbatim in default mode. It does
+        // not apply once a finite last column is in play (multiple tab stops),
+        // where the general loop stops converting past that column.
+        if !options.aflag && print_state.leading && lastcol == 0 {
+            // In default mode (not -a), we only convert leading spaces, so batch
+            // over them and then copy the rest of the line. Compute the column
+            // into a local without touching state: if the first non-blank is a
+            // multibyte start it may be a blank that needs the general loop, so
+            // bail out to it instead of committing here.
+            let mut end = 0;
+            let mut col = print_state.col;
+            let mut saw_tab = false;
+            while end < buf.len() {
+                match buf[end] {
+                    b' ' => col += 1,
                     b'\t' => {
-                        print_state.col += next_tabstop(tab_config, print_state.col).unwrap_or(1);
-                        byte += 1;
-                        print_state.pctype = CharType::Tab;
+                        col += next_tabstop(tab_config, col).unwrap_or(1);
+                        saw_tab = true;
                     }
                     _ => break,
                 }
+                end += 1;
             }
+            if end == buf.len() || buf[end].is_ascii() {
+                print_state.col = col;
+                if saw_tab {
+                    print_state.pctype = CharType::Tab;
+                }
 
-            // If we found spaces/tabs, write them as tabs
-            if byte > 0 {
-                write_tabs(output, tab_config, print_state, options.aflag)?;
-            }
+                // If we found spaces/tabs, write them as tabs
+                if end > 0 {
+                    write_tabs(output, tab_config, print_state, options.aflag)?;
+                }
 
-            // Write the rest of the line directly (no more tab conversion needed)
-            if byte < buf.len() {
-                print_state.leading = false;
-                output.write_all(&buf[byte..])?;
+                // Write the rest of the line directly (no more tab conversion needed)
+                if end < buf.len() {
+                    print_state.leading = false;
+                    output.write_all(&buf[end..])?;
+                }
+                return Ok(());
             }
-            return Ok(());
         }
     }
 
@@ -524,9 +580,19 @@ fn unexpand_buf(
         let tabs_buffered = print_state.leading || options.aflag;
         match ctype {
             CharType::Space | CharType::Tab => {
+                // A buffered multibyte blank must be remembered so it can be
+                // emitted verbatim if its run is not converted to a tab.
+                if tabs_buffered && ctype == CharType::Space && nbytes > 1 {
+                    print_state.pending_wide.push((
+                        print_state.col,
+                        cwidth,
+                        buf[byte..byte + nbytes].to_vec(),
+                    ));
+                }
+
                 // compute next col, but only write space or tab chars if not buffering
                 print_state.col += if ctype == CharType::Space {
-                    1
+                    cwidth
                 } else {
                     next_tabstop(tab_config, print_state.col).unwrap_or(1)
                 };
@@ -575,6 +641,7 @@ fn unexpand_file(
         scol: 0,
         leading: true,
         pctype: CharType::Other,
+        pending_wide: Vec::new(),
     };
 
     loop {

src/uucore/Cargo.toml

@@ -49,6 +49,10 @@ z85 = { version = "3.0.5", optional = true }
 base64-simd = { version = "0.8", optional = true }
 libc = { workspace = true, optional = true }
 os_display = "0.1.3"
+# Not optional: os_display already pulls unicode-width into every uucore build,
+# so making it a direct dependency here is free and keeps char_width available
+# on all targets (a feature-gated optional dep failed to activate on wasm).
+unicode-width = { workspace = true }
 
 # Benchmark dependencies (optional)
 divan = { workspace = true, optional = true }

src/uucore/src/lib/features.rs

@@ -12,6 +12,7 @@ pub mod backup_control;
 pub mod benchmark;
 #[cfg(feature = "buf-copy")]
 pub mod buf_copy;
+pub mod char_width;
 #[cfg(feature = "checksum")]
 pub mod checksum;
 #[cfg(feature = "colors")]

src/uucore/src/lib/features/char_width.rs

@@ -0,0 +1,77 @@
+// This file is part of the uutils coreutils package.
+//
+// For the full copyright and license information, please view the LICENSE
+// file that was distributed with this source code.
+// spell-checker:ignore nbytes
+
+//! Display width and byte length of UTF-8 characters within a byte buffer.
+
+use std::str::from_utf8;
+use unicode_width::UnicodeWidthChar;
+
+/// Decode the (possibly multibyte) UTF-8 character beginning at `buf[pos]`,
+/// returning the character, its display width, and its byte length.
+///
+/// The sequence length is taken from the leading byte's bit pattern rather than
+/// its value, so 3- and 4-byte characters are measured correctly. A truncated
+/// or otherwise invalid sequence resolves to `(None, 1, 1)`, which lets callers
+/// resync one byte at a time.
+///
+/// # Panics
+///
+/// Panics if `pos` is out of bounds for `buf`.
+#[must_use]
+pub fn char_info_at(buf: &[u8], pos: usize) -> (Option<char>, usize, usize) {
+    let nbytes = match buf[pos] {
+        0xC0..=0xDF => 2,
+        0xE0..=0xEF => 3,
+        0xF0..=0xF7 => 4,
+        _ => 1,
+    };
+    buf.get(pos..pos + nbytes)
+        .and_then(|s| from_utf8(s).ok())
+        .and_then(|s| s.chars().next())
+        .map_or((None, 1, 1), |c| {
+            (Some(c), UnicodeWidthChar::width(c).unwrap_or(0), nbytes)
+        })
+}
+
+/// Return the display width and byte length of the character at `buf[pos]`.
+///
+/// A truncated or otherwise invalid sequence resolves to a single byte of width
+/// one. See [`char_info_at`] for the decoding details.
+///
+/// # Panics
+///
+/// Panics if `pos` is out of bounds for `buf`.
+#[must_use]
+pub fn char_width_at(buf: &[u8], pos: usize) -> (usize, usize) {
+    let (_, width, nbytes) = char_info_at(buf, pos);
+    (width, nbytes)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{char_info_at, char_width_at};
+
+    #[test]
+    fn width_and_length() {
+        assert_eq!(char_width_at(b"a", 0), (1, 1)); // ASCII
+        assert_eq!(char_width_at("é".as_bytes(), 0), (1, 2)); // U+00E9, 2-byte width 1
+        assert_eq!(char_width_at("\u{0301}".as_bytes(), 0), (0, 2)); // combining mark, width 0
+        assert_eq!(char_width_at("\u{3000}".as_bytes(), 0), (2, 3)); // 3-byte width 2
+        assert_eq!(char_width_at("\u{1F600}".as_bytes(), 0), (2, 4)); // 4-byte width 2
+        assert_eq!(char_width_at(&[0xFF], 0), (1, 1)); // invalid leading byte resyncs
+        assert_eq!(char_width_at(&[0xE3], 0), (1, 1)); // truncated sequence resyncs
+    }
+
+    #[test]
+    fn info_decodes_char() {
+        assert_eq!(
+            char_info_at("\u{3000}".as_bytes(), 0),
+            (Some('\u{3000}'), 2, 3)
+        );
+        assert_eq!(char_info_at(b"a", 0), (Some('a'), 1, 1));
+        assert_eq!(char_info_at(&[0xFF], 0), (None, 1, 1)); // invalid resyncs
+    }
+}