fix(plan): slice task commands by byte offsets

CHANGELOG.md

@@ -1,5 +1,6 @@
 # Changelog
 
+- **Fixed** `vp run` no longer panics before running a script that contains non-ASCII characters, such as `echo こんにちは`, and no longer cuts off such commands where it displays them ([#861](https://github.com/voidzero-dev/vite-task/pull/861)).
 - **Fixed** On Linux, concurrent system calls in tracked programs, such as `pnpm dlx`, no longer cause a stack overflow ([#854](https://github.com/voidzero-dev/vite-task/pull/854)).
 - **Fixed** Cached tasks can now run scripts whose shebang names an interpreter path longer than 126 bytes. Before, the path was cut short and the script failed to start or ran the wrong program. On macOS, text after a `#` in the shebang line is now ignored, as it is when the script runs on its own ([#788](https://github.com/voidzero-dev/vite-task/pull/788)).
 - **Fixed** On macOS, tasks with automatic file-access tracking can now run Node.js single executable applications, such as pnpm 11's standalone `pnpm` binary, which previously aborted with `Assertion failed: (magic) == (kMagic)`, and `deno compile` outputs, which previously failed with `Did not find magic bytes` ([#840](https://github.com/voidzero-dev/vite-task/pull/840)).

crates/vt_plan/tests/plan_snapshots/fixtures/non_ascii_commands/package.json

@@ -0,0 +1,10 @@
+{
+  "name": "non-ascii-commands",
+  "private": true,
+  "scripts": {
+    "greet": "echo ああ",
+    "greet-long": "echo あああ",
+    "greet-and-done": "echo 你好 && FOO=é echo 🎉 done",
+    "greet-and-missing": "echo こんにちは && vt run missing"
+  }
+}

crates/vt_plan/tests/plan_snapshots/fixtures/non_ascii_commands/snapshots.toml

@@ -0,0 +1,27 @@
+# Tests that commands containing multi-byte UTF-8 characters are displayed in full.
+# A character position used as a byte position either lands inside a character
+# (`greet`) or truncates the command (`greet-long`).
+
+[fields]
+key = true
+execution_item_display = { command = true }
+
+[[plan]]
+name = "command_ending_with_multi_byte_characters"
+args = ["run", "greet"]
+
+[[plan]]
+name = "command_ending_on_a_character_boundary"
+args = ["run", "greet-long"]
+
+[[plan]]
+name = "and_list_after_multi_byte_characters"
+args = ["run", "greet-and-done"]
+
+[[plan]]
+name = "extra_args_after_multi_byte_characters"
+args = ["run", "greet", "🎉"]
+
+[[plan]]
+name = "nested_error_after_multi_byte_characters"
+args = ["run", "greet-and-missing"]

crates/vt_plan/tests/plan_snapshots/fixtures/non_ascii_commands/snapshots/query_and_list_after_multi_byte_characters.jsonc

@@ -0,0 +1,25 @@
+// run greet-and-done
+{
+  "graph": [
+    {
+      "key": [
+        "<workspace>/",
+        "greet-and-done"
+      ],
+      "node": {
+        "items": [
+          {
+            "execution_item_display": {
+              "command": "echo 你好"
+            }
+          },
+          {
+            "execution_item_display": {
+              "command": "FOO=é echo 🎉 done"
+            }
+          }
+        ]
+      }
+    }
+  ]
+}

crates/vt_plan/tests/plan_snapshots/fixtures/non_ascii_commands/snapshots/query_command_ending_on_a_character_boundary.jsonc

@@ -0,0 +1,20 @@
+// run greet-long
+{
+  "graph": [
+    {
+      "key": [
+        "<workspace>/",
+        "greet-long"
+      ],
+      "node": {
+        "items": [
+          {
+            "execution_item_display": {
+              "command": "echo あああ"
+            }
+          }
+        ]
+      }
+    }
+  ]
+}

crates/vt_plan/tests/plan_snapshots/fixtures/non_ascii_commands/snapshots/query_command_ending_with_multi_byte_characters.jsonc

@@ -0,0 +1,20 @@
+// run greet
+{
+  "graph": [
+    {
+      "key": [
+        "<workspace>/",
+        "greet"
+      ],
+      "node": {
+        "items": [
+          {
+            "execution_item_display": {
+              "command": "echo ああ"
+            }
+          }
+        ]
+      }
+    }
+  ]
+}

crates/vt_plan/tests/plan_snapshots/fixtures/non_ascii_commands/snapshots/query_extra_args_after_multi_byte_characters.jsonc

@@ -0,0 +1,20 @@
+// run greet 🎉
+{
+  "graph": [
+    {
+      "key": [
+        "<workspace>/",
+        "greet"
+      ],
+      "node": {
+        "items": [
+          {
+            "execution_item_display": {
+              "command": "echo ああ '🎉'"
+            }
+          }
+        ]
+      }
+    }
+  ]
+}

crates/vt_plan/tests/plan_snapshots/fixtures/non_ascii_commands/snapshots/query_nested_error_after_multi_byte_characters.snap

@@ -0,0 +1 @@
+Failed to plan tasks from `vt run missing` in task non-ascii-commands#greet-and-missing: Task "missing" not found

crates/vt_plan/tests/plan_snapshots/fixtures/non_ascii_commands/snapshots/task_graph.md

@@ -0,0 +1,170 @@
+# task graph
+
+```mermaid
+flowchart TD
+  task_0["<workspace>/#greet"]
+  task_1["<workspace>/#greet-and-done"]
+  task_2["<workspace>/#greet-and-missing"]
+  task_3["<workspace>/#greet-long"]
+```
+
+## `<workspace>/#greet`
+
+```json
+{
+  "task_display": {
+    "package_name": "non-ascii-commands",
+    "task_name": "greet",
+    "package_path": "<workspace>/"
+  },
+  "resolved_config": {
+    "commands": [
+      "echo ああ"
+    ],
+    "resolved_options": {
+      "cwd": "<workspace>/",
+      "cache_config": {
+        "remote_cache_allowed": true,
+        "env_config": {
+          "fingerprinted_envs": [],
+          "untracked_env": [
+            "<default untracked envs>"
+          ]
+        },
+        "input_config": {
+          "includes_auto": true,
+          "positive_globs": [],
+          "negative_globs": []
+        },
+        "output_config": {
+          "includes_auto": true,
+          "positive_globs": [],
+          "negative_globs": []
+        }
+      }
+    }
+  },
+  "source": "PackageJsonScript"
+}
+```
+
+## `<workspace>/#greet-and-done`
+
+```json
+{
+  "task_display": {
+    "package_name": "non-ascii-commands",
+    "task_name": "greet-and-done",
+    "package_path": "<workspace>/"
+  },
+  "resolved_config": {
+    "commands": [
+      "echo 你好 && FOO=é echo 🎉 done"
+    ],
+    "resolved_options": {
+      "cwd": "<workspace>/",
+      "cache_config": {
+        "remote_cache_allowed": true,
+        "env_config": {
+          "fingerprinted_envs": [],
+          "untracked_env": [
+            "<default untracked envs>"
+          ]
+        },
+        "input_config": {
+          "includes_auto": true,
+          "positive_globs": [],
+          "negative_globs": []
+        },
+        "output_config": {
+          "includes_auto": true,
+          "positive_globs": [],
+          "negative_globs": []
+        }
+      }
+    }
+  },
+  "source": "PackageJsonScript"
+}
+```
+
+## `<workspace>/#greet-and-missing`
+
+```json
+{
+  "task_display": {
+    "package_name": "non-ascii-commands",
+    "task_name": "greet-and-missing",
+    "package_path": "<workspace>/"
+  },
+  "resolved_config": {
+    "commands": [
+      "echo こんにちは && vt run missing"
+    ],
+    "resolved_options": {
+      "cwd": "<workspace>/",
+      "cache_config": {
+        "remote_cache_allowed": true,
+        "env_config": {
+          "fingerprinted_envs": [],
+          "untracked_env": [
+            "<default untracked envs>"
+          ]
+        },
+        "input_config": {
+          "includes_auto": true,
+          "positive_globs": [],
+          "negative_globs": []
+        },
+        "output_config": {
+          "includes_auto": true,
+          "positive_globs": [],
+          "negative_globs": []
+        }
+      }
+    }
+  },
+  "source": "PackageJsonScript"
+}
+```
+
+## `<workspace>/#greet-long`
+
+```json
+{
+  "task_display": {
+    "package_name": "non-ascii-commands",
+    "task_name": "greet-long",
+    "package_path": "<workspace>/"
+  },
+  "resolved_config": {
+    "commands": [
+      "echo あああ"
+    ],
+    "resolved_options": {
+      "cwd": "<workspace>/",
+      "cache_config": {
+        "remote_cache_allowed": true,
+        "env_config": {
+          "fingerprinted_envs": [],
+          "untracked_env": [
+            "<default untracked envs>"
+          ]
+        },
+        "input_config": {
+          "includes_auto": true,
+          "positive_globs": [],
+          "negative_globs": []
+        },
+        "output_config": {
+          "includes_auto": true,
+          "positive_globs": [],
+          "negative_globs": []
+        }
+      }
+    }
+  },
+  "source": "PackageJsonScript"
+}
+```
+

crates/vt_shell/src/lib.rs

@@ -92,9 +92,28 @@ fn flatten_pieces(pieces: &[WordPieceWithSource], result: &mut Str) -> Option<()
     Some(())
 }
 
-fn pipeline_to_command(pipeline: &Pipeline) -> Option<(TaskParsedCommand, Range<usize>)> {
+/// Converts a character index reported by `brush_parser` into a byte index
+/// into `source`.
+///
+/// `brush_parser::SourcePosition::index` counts characters, while the spans
+/// returned by [`try_parse_as_and_list`] slice the `&str` by bytes.
+///
+/// Remove this conversion when `brush_parser` reports byte offsets. The open
+/// <https://github.com/reubeno/brush/pull/1144> plans this change and also
+/// renames `index`, which would fail to compile here. If a release changes
+/// the unit without the rename, this conversion applies twice, and the test
+/// `command_ranges_are_byte_ranges_after_multi_byte_characters` fails.
+fn char_index_to_byte_index(source: &str, char_index: usize) -> Option<usize> {
+    source.char_indices().map(|(byte_index, _)| byte_index).chain([source.len()]).nth(char_index)
+}
+
+fn pipeline_to_command(
+    source: &str,
+    pipeline: &Pipeline,
+) -> Option<(TaskParsedCommand, Range<usize>)> {
     let location = pipeline.location()?;
-    let range = location.start.index..location.end.index;
+    let range = char_index_to_byte_index(source, location.start.index)?
+        ..char_index_to_byte_index(source, location.end.index)?;
 
     let Pipeline { timed: None, bang: false, seq } = pipeline else {
         return None;
@@ -137,6 +156,9 @@ fn pipeline_to_command(pipeline: &Pipeline) -> Option<(TaskParsedCommand, Range<
     Some((TaskParsedCommand { envs, program: unquote(program)?, args }, range))
 }
 
+/// Parses `cmd` as simple commands joined by `&&`.
+///
+/// Each command is returned with its byte range in `cmd`.
 #[must_use]
 pub fn try_parse_as_and_list(cmd: &str) -> Option<Vec<(TaskParsedCommand, Range<usize>)>> {
     let mut parser = Parser::new(cmd.as_bytes(), &PARSER_OPTIONS);
@@ -150,12 +172,12 @@ pub fn try_parse_as_and_list(cmd: &str) -> Option<Vec<(TaskParsedCommand, Range<
     };
 
     let mut commands = Vec::<(TaskParsedCommand, Range<usize>)>::new();
-    commands.push(pipeline_to_command(&and_or_list.first)?);
+    commands.push(pipeline_to_command(cmd, &and_or_list.first)?);
     for and_or in &and_or_list.additional {
         let AndOr::And(pipeline) = and_or else {
             return None;
         };
-        commands.push(pipeline_to_command(pipeline)?);
+        commands.push(pipeline_to_command(cmd, pipeline)?);
     }
     Some(commands)
 }
@@ -216,6 +238,17 @@ mod tests {
         );
     }
 
+    #[test]
+    fn command_ranges_are_byte_ranges_after_multi_byte_characters() {
+        // `あ` is 3 bytes, `é` is 2 bytes, and `🎉` is 4 bytes in UTF-8, so
+        // character and byte positions diverge after each of them.
+        let source = "echo ああ && FOO=é echo 🎉 && echo done";
+        let list = try_parse_as_and_list(source).unwrap();
+
+        let substrs = list.iter().map(|(_, range)| source.get(range.clone())).collect::<Vec<_>>();
+        assert_eq!(substrs, [Some("echo ああ"), Some("FOO=é echo 🎉"), Some("echo done")]);
+    }
+
     #[test]
     fn test_task_parsed_command_stable_env_ordering() {
         // Test that environment variables maintain stable ordering