From 9bdedb90b8e1415f1dbbaee58859e44335420779 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 15:01:09 +0000 Subject: [PATCH 1/2] fix(word): expand globs for every simple command via typed words Glob expansion lived only in `external::expand_glob_args`, which only `execute_external_with_redirects` called. `wc -c *.txt` passed `*.txt` to wc, pipeline stages and `echo` never expanded, and quoted metacharacters reached programs with the tokenizer's escape backslash (`echo '*.txt'` printed `\*.txt`). The tokenizer now also builds a typed `word::Word` that records the quoting of each piece (unquoted, single, double, backslash). `word::expand_words` runs tilde, parameter, command, arithmetic and process substitution, then brace expansion, glob expansion and quote removal on it. External commands (with or without redirections and in the background), every pipeline stage, `echo`, function calls, command substitution and process substitution use it; `Command::External`, `Command::Pipeline` and `Command::Echo` carry `Vec` arguments. Rules kept from `expand_glob_args`: no match or an invalid pattern keeps the word; a `GlobLimitExceeded` fails the command (exit 1) without running it. Builtin operands, redirection targets, assignments, `case` and `test` still use the old string encoding (documented in POSIX_COMPLIANCE.adoc). Also: `$( ... )` is read to its matching `)`, so `$((1+(2*3)))` is one substitution, and `$(echo ...)` is supported. Tests: unit tests in word.rs, a Display/tokenize round-trip property test, and end-to-end tests through the vsh binary. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01J4QvDfX8Xoq6vedAKdWBif --- CHANGELOG.adoc | 29 + docs/POSIX_COMPLIANCE.adoc | 12 + impl/rust-cli/src/commands.rs | 2 +- impl/rust-cli/src/executable.rs | 121 ++-- impl/rust-cli/src/external.rs | 107 +-- impl/rust-cli/src/lib.rs | 1 + impl/rust-cli/src/parser.rs | 210 ++++-- impl/rust-cli/src/process_sub.rs | 14 +- impl/rust-cli/src/word.rs | 668 ++++++++++++++++++ .../tests/word_expansion_e2e_tests.rs | 126 ++++ 10 files changed, 1076 insertions(+), 214 deletions(-) create mode 100644 impl/rust-cli/src/word.rs create mode 100644 impl/rust-cli/tests/word_expansion_e2e_tests.rs diff --git a/CHANGELOG.adoc b/CHANGELOG.adoc index de152145..fb3d88be 100644 --- a/CHANGELOG.adoc +++ b/CHANGELOG.adoc @@ -11,6 +11,35 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 == [Unreleased] +=== Fixed — 2026-11-18 (arguments: globs, quote removal) + +- Glob expansion now applies to the arguments of every external command, + every pipeline stage, `echo` and function calls. Before, only an external + command that also had a redirection expanded its arguments: `wc -c *.txt` + passed `*.txt` to `wc`, `ls *.txt | wc -l` and `echo *.txt` did not expand. +- Quoted arguments no longer reach the program with a backslash in front of + `*`, `?`, `[` or `{`: `echo '*.txt'` printed `\*.txt` and now prints + `*.txt`. A partly quoted word globs only its unquoted part (`'*'x.*`). +- `echo` performs command substitution and arithmetic expansion + (`echo $((2+3))` printed `$((2+3))`), and `$(echo ...)` works. +- `$((1+(2*3)))` and other substitutions containing `(` are read to their + matching `)`. +- An expansion error (such as a glob limit from #191) fails the command with + exit status 1 and does not run it. With a redirection the status was 127. + +=== Changed — 2026-11-18 (typed words) + +- New module `src/word.rs`: a `Word` keeps the quoting of each of its pieces + (unquoted, single, double, backslash), and `word::expand_words` performs + tilde, parameter, command, arithmetic and process substitution, brace + expansion, glob expansion and quote removal on it. `Command::External`, + `Command::Pipeline` and `Command::Echo` now carry `Vec` arguments. + Other commands still use the older string encoding; see + `docs/POSIX_COMPLIANCE.adoc` (Milestones 5 and 6) for what is not migrated. +- Removed `external::expand_glob_args`; its rules (no match or invalid + pattern keeps the word, a limit is an error) moved to `word.rs` with their + tests. + === Fixed — 2026-11-18 (glob expansion out of memory, #191) - `**` no longer follows symbolic links to directories, so a link back to a diff --git a/docs/POSIX_COMPLIANCE.adoc b/docs/POSIX_COMPLIANCE.adoc index e8202f26..175e144c 100644 --- a/docs/POSIX_COMPLIANCE.adoc +++ b/docs/POSIX_COMPLIANCE.adoc @@ -177,6 +177,12 @@ or literal result (#191, 2026-11-18) |Brace expansion `+{a,b,c}+` |Done |Nested braces supported |Sorted results |Done |POSIX requirement + +|Where globs expand |Partial |Arguments of external commands (with or +without redirections), every pipeline stage, `+echo+` and function calls, +through `+word::expand_words+` (2026-11-18; before, only external commands +with a redirection). Not yet: builtin operands (`+ls+`, `+rm+`, `+cp+`, ...), +`+for+` word lists, redirection targets |=== ==== Milestone 6: Quote Processing — COMPLETE @@ -194,6 +200,12 @@ quotes |Unclosed quote detection |Done | |Line continuation |Done | + +|Quote removal |Partial |Done for the arguments that go through +`+word::expand_words+` (see Milestone 5), which keeps the quoting of each +piece of a word. Builtin operands, redirection targets, assignment values, +`+case+` patterns and `+test+` arguments still receive quoted `+*?[{~+` with +a backslash in front (`+touch '*q'+` creates `+\*q+`) |=== ==== Milestone 7: Command Substitution — COMPLETE diff --git a/impl/rust-cli/src/commands.rs b/impl/rust-cli/src/commands.rs index 4f91625d..02918fca 100644 --- a/impl/rust-cli/src/commands.rs +++ b/impl/rust-cli/src/commands.rs @@ -2856,7 +2856,7 @@ mod tests { let state = ShellState::new(tmp.path().to_str().unwrap()).unwrap(); let cmd = crate::parser::Command::External { program: "git".to_string(), - args: vec!["status".to_string()], + args: vec!["status".into()], redirects: vec![], background: false, }; diff --git a/impl/rust-cli/src/executable.rs b/impl/rust-cli/src/executable.rs index aeb72a25..7148cec7 100644 --- a/impl/rust-cli/src/executable.rs +++ b/impl/rust-cli/src/executable.rs @@ -488,35 +488,36 @@ impl ExecutableCommand for Command { // Expand program name (no process substitutions in program name) let expanded_program = crate::parser::expand_with_command_sub(program, state)?; + // Word expansion: variables, substitutions, braces, globs and + // quote removal. A failed expansion (for example a glob that + // reaches a limit) fails the command without running it. + let expansion = + match crate::word::expand_words(args, state, crate::word::ProcessSubs::Allow) { + Ok(expansion) => expansion, + Err(e) => { + eprintln!("vsh: {}: {}", expanded_program, e); + return Ok(ExecutionResult::ExternalCommand { exit_code: 1 }); + } + }; + // Check if the command is a shell function before external execution. // `if let Some(...)` performs a single map lookup; the former // double-lookup plus expect shape could panic and duplicated // the lookup. if let Some(func_def) = state.functions.get(&expanded_program).cloned() { - let expanded_args: Vec = args - .iter() - .map(|a| crate::parser::expand_variables(a, state)) - .collect(); - return execute_function_call(&func_def, &expanded_args, state); + let result = execute_function_call(&func_def, &expansion.fields, state); + expansion.cleanup(); + return result; } - // Expand arguments with process substitutions - let mut all_process_subs = Vec::new(); - let mut expanded_args = Vec::new(); - - for arg in args { - let (expanded_arg, proc_subs) = - crate::parser::expand_with_process_sub(arg, state)?; - expanded_args.push(expanded_arg); - all_process_subs.extend(proc_subs); - } + let expanded_args = &expansion.fields; // Execute main command if *background { // Background execution let _job_id = external::execute_external_background( &expanded_program, - &expanded_args, + expanded_args, redirects, state, ) @@ -525,24 +526,19 @@ impl ExecutableCommand for Command { 0 // No job created }); - // Cleanup all process substitutions - for proc_sub in all_process_subs { - if let Err(e) = proc_sub.cleanup() { - eprintln!("Warning: Failed to cleanup process substitution: {}", e); - } - } + expansion.cleanup(); Ok(ExecutionResult::Success) } else { // Foreground execution let exit_code = if redirects.is_empty() { // No redirections - use simple path - external::execute_external(&expanded_program, &expanded_args) + external::execute_external(&expanded_program, expanded_args) } else { // Has redirections - use redirection-aware execution external::execute_external_with_redirects( &expanded_program, - &expanded_args, + expanded_args, redirects, state, ) @@ -552,12 +548,7 @@ impl ExecutableCommand for Command { 127 }); - // Cleanup all process substitutions - for proc_sub in all_process_subs { - if let Err(e) = proc_sub.cleanup() { - eprintln!("Warning: Failed to cleanup process substitution: {}", e); - } - } + expansion.cleanup(); Ok(ExecutionResult::ExternalCommand { exit_code }) } @@ -569,33 +560,42 @@ impl ExecutableCommand for Command { redirects, background, } => { - // Expand variables and command substitutions in all pipeline stages - let expanded_stages: Result)>> = stages - .iter() - .map(|(program, args)| { - let expanded_program = - crate::parser::expand_with_command_sub(program, state)?; - let expanded_args: Result> = args - .iter() - .map(|arg| crate::parser::expand_with_command_sub(arg, state)) - .collect(); - Ok((expanded_program, expanded_args?)) - }) - .collect(); - let expanded_stages = expanded_stages?; - - if *background && !expanded_stages.is_empty() { + if *background && !stages.is_empty() { return Err(anyhow::anyhow!( "Background pipelines are not supported yet; refusing to run them in the foreground" )); } + // Expand every stage before any stage runs, so a failed + // expansion runs no part of the pipeline. + let mut expanded_stages: Vec<(String, Vec)> = Vec::new(); + let mut process_subs = Vec::new(); + for (program, args) in stages { + let expanded_program = crate::parser::expand_with_command_sub(program, state)?; + match crate::word::expand_words(args, state, crate::word::ProcessSubs::Allow) { + Ok(expansion) => { + expanded_stages.push((expanded_program, expansion.fields)); + process_subs.extend(expansion.process_subs); + } + Err(e) => { + eprintln!("vsh: {}: {}", expanded_program, e); + return Ok(ExecutionResult::ExternalCommand { exit_code: 1 }); + } + } + } + let exit_code = external::execute_pipeline(&expanded_stages, redirects, state) .unwrap_or_else(|e| { eprintln!("Pipeline error: {}", e); 127 }); + crate::word::Expansion { + fields: Vec::new(), + process_subs, + } + .cleanup(); + Ok(ExecutionResult::ExternalCommand { exit_code }) } @@ -684,10 +684,15 @@ impl ExecutableCommand for Command { interpret_escapes, redirects, } => { - let expanded_args: Vec = args - .iter() - .map(|a| crate::parser::expand_variables(a, state)) - .collect(); + let expansion = + match crate::word::expand_words(args, state, crate::word::ProcessSubs::Allow) { + Ok(expansion) => expansion, + Err(e) => { + eprintln!("vsh: echo: {}", e); + return Ok(ExecutionResult::ExternalCommand { exit_code: 1 }); + } + }; + let expanded_args = &expansion.fields; let output = if *interpret_escapes { expanded_args @@ -719,6 +724,7 @@ impl ExecutableCommand for Command { Ok(()) })?; } + expansion.cleanup(); Ok(ExecutionResult::Success) } @@ -1291,7 +1297,7 @@ impl ExecutableCommand for Command { if args.is_empty() { program.clone() } else { - format!("{} {}", program, args.join(" ")) + format!("{} {}", program, join_words(args)) } } Command::Pipeline { stages, .. } => { @@ -1301,7 +1307,7 @@ impl ExecutableCommand for Command { if args.is_empty() { prog.clone() } else { - format!("{} {}", prog, args.join(" ")) + format!("{} {}", prog, join_words(args)) } }) .collect(); @@ -1377,7 +1383,7 @@ impl ExecutableCommand for Command { format!("{} {} {}", left.description(), op_str, right.description()) } - Command::Echo { args, .. } => format!("echo {}", args.join(" ")), + Command::Echo { args, .. } => format!("echo {}", join_words(args)), Command::True => "true".to_string(), Command::False => "false".to_string(), Command::Read { var_names, .. } => format!("read {}", var_names.join(" ")), @@ -1645,6 +1651,15 @@ fn execute_loop( Ok(last_result) } +/// Words written back as source and separated by spaces. +fn join_words(words: &[crate::word::Word]) -> String { + words + .iter() + .map(|w| w.to_string()) + .collect::>() + .join(" ") +} + /// Match a word against a case pattern (supports glob-style wildcards) fn case_pattern_matches(word: &str, pattern: &str) -> bool { if pattern == "*" { diff --git a/impl/rust-cli/src/external.rs b/impl/rust-cli/src/external.rs index 60f345a3..070e5f99 100644 --- a/impl/rust-cli/src/external.rs +++ b/impl/rust-cli/src/external.rs @@ -11,7 +11,6 @@ use std::path::PathBuf; use std::process::{Command, Stdio}; use std::time::Duration; -use crate::glob; use crate::redirection::{RedirectSetup, Redirection}; use crate::signals; use crate::state::ShellState; @@ -37,8 +36,7 @@ pub fn execute_external_with_redirects( let (stdin_cfg, stdout_cfg, stderr_cfg) = stdio_config_from_redirects(redirects, &redirect_setup, state)?; - // Expand glob patterns in arguments (Phase 6 M12) - let expanded_args = expand_glob_args(args, state)?; + // The arguments arrive already expanded, globs included (`crate::word`). // PATH lookup let executable = find_in_path(program)?; @@ -46,7 +44,7 @@ pub fn execute_external_with_redirects( // Build command with redirected stdio let mut command = Command::new(&executable); command - .args(&expanded_args) + .args(args) .stdin(stdin_cfg) .stdout(stdout_cfg) .stderr(stderr_cfg); @@ -475,80 +473,6 @@ fn stdin_from_bytes(content: &[u8], what: &str) -> Result { Ok(Stdio::from(reader)) } -/// Expand glob patterns in command arguments (Phase 6 M12) -/// -/// For each argument: -/// - If it contains glob metacharacters (*, ?, [, {), expand it -/// - If expansion finds matches, use them -/// - If expansion finds nothing, or the word is not a valid pattern, use the -/// word literally (POSIX behavior) -/// - If expansion stops at a limit, fail: the command is not run with a -/// literal or partial argument list in place of the real expansion -/// - If no metacharacters, use argument as-is -/// -/// # POSIX Behavior -/// - Brace expansion happens first: {a,b} -> [a, b] -/// - Then glob expansion: *.txt -> [file1.txt, file2.txt] -/// - Empty matches return literal: "*.xyz" if no .xyz files -/// -/// # Examples -/// ```ignore -/// expand_glob_args(&["echo", "*.txt"], state) // -> ["echo", "file1.txt", "file2.txt"] -/// expand_glob_args(&["ls", "file?.rs"], state) // -> ["ls", "file1.rs", "file2.rs"] -/// expand_glob_args(&["rm", "*.xyz"], state) // -> ["rm", "*.xyz"] (no .xyz files) -/// ``` -fn expand_glob_args(args: &[String], _state: &ShellState) -> Result> { - // Get current working directory for glob expansion - let cwd = std::env::current_dir().context("Failed to get current working directory")?; - expand_glob_args_in(args, &cwd, &glob::GlobLimits::DEFAULT) -} - -/// [`expand_glob_args`] with an explicit directory and limits. -/// -/// # Errors -/// Returns the [`glob::GlobLimitExceeded`] error of the first argument whose -/// expansion reaches a limit. -fn expand_glob_args_in( - args: &[String], - cwd: &std::path::Path, - limits: &glob::GlobLimits, -) -> Result> { - let mut expanded: Vec = Vec::new(); - - for arg in args { - // Check if argument contains glob metacharacters - if glob::contains_glob_pattern(arg) { - // First, expand braces: file{1,2}.txt -> [file1.txt, file2.txt] - let brace_expanded = glob::expand_braces(arg); - - // Then expand each brace result for glob patterns - for pattern in &brace_expanded { - // Expand glob pattern relative to current directory - match glob::expand_glob_with_limits(pattern, cwd, limits) { - Ok(expansion) if !expansion.paths.is_empty() => { - expanded.extend( - expansion - .paths - .iter() - .map(|path| path.to_string_lossy().to_string()), - ); - } - // A limit is not "no match": substituting the literal word - // would run the command on the wrong arguments. - Err(e) if e.is::() => return Err(e), - // No matches or not a valid pattern - use literal (POSIX behavior) - _ => expanded.push(pattern.clone()), - } - } - } else { - // No glob pattern - use argument as-is - expanded.push(arg.clone()); - } - } - - Ok(expanded) -} - /// Handle command exit status, converting signals to exit codes fn handle_exit_status(status: std::process::ExitStatus) -> Result { #[cfg(unix)] @@ -827,31 +751,4 @@ mod tests { let exit_code = execute_pipeline(&stages, &[], &mut state).unwrap(); assert_eq!(exit_code, 0, "Single-stage pipeline should work"); } - #[test] - fn glob_args_use_the_literal_word_when_nothing_matches() { - let dir = tempfile::TempDir::new().unwrap(); - std::fs::write(dir.path().join("a.txt"), b"").unwrap(); - let args = vec!["*.txt".to_string(), "*.none".to_string(), "[z-".to_string()]; - let out = expand_glob_args_in(&args, dir.path(), &glob::GlobLimits::DEFAULT).unwrap(); - assert_eq!(out, ["a.txt", "*.none", "[z-"]); - } - - #[test] - fn glob_args_fail_instead_of_substituting_the_literal_at_a_limit() { - let dir = tempfile::TempDir::new().unwrap(); - for i in 0..5 { - std::fs::write(dir.path().join(format!("f{i}.txt")), b"").unwrap(); - } - let limits = glob::GlobLimits { - max_matches: 4, - ..glob::GlobLimits::DEFAULT - }; - let args = vec!["-l".to_string(), "*.txt".to_string()]; - let err = expand_glob_args_in(&args, dir.path(), &limits).unwrap_err(); - let stopped = err - .downcast_ref::() - .expect("the limit error must reach the caller"); - assert_eq!(stopped.exceeded.axis, "matches held"); - assert!(err.to_string().contains("over the limit of 4"), "{err}"); - } } diff --git a/impl/rust-cli/src/lib.rs b/impl/rust-cli/src/lib.rs index fd5fa34a..e74b4b1f 100644 --- a/impl/rust-cli/src/lib.rs +++ b/impl/rust-cli/src/lib.rs @@ -40,6 +40,7 @@ pub mod state; pub mod test_command; pub mod verification; +pub mod word; /// Echidna property-based verification integration (optional feature) #[cfg(feature = "echidna")] diff --git a/impl/rust-cli/src/parser.rs b/impl/rust-cli/src/parser.rs index da677ce3..1b51f2d0 100644 --- a/impl/rust-cli/src/parser.rs +++ b/impl/rust-cli/src/parser.rs @@ -9,6 +9,7 @@ use anyhow::{anyhow, Context, Result}; use crate::redirection::Redirection; +use crate::word::{Quote, Word}; /// Quote type for a word or word part #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -82,6 +83,8 @@ struct ParameterExpansion { struct QuotedWord { parts: Vec, quote_type: QuoteType, + /// The same word with the quoting of each piece kept (see `crate::word`). + typed: Word, } impl QuotedWord { @@ -89,6 +92,7 @@ impl QuotedWord { Self { parts: Vec::new(), quote_type: QuoteType::None, + typed: Word::new(), } } @@ -102,19 +106,23 @@ impl QuotedWord { } } - fn push_variable(&mut self, name: String) { + fn push_variable(&mut self, name: String, quote: Quote) { + self.typed.push_parameter(name.clone(), false, quote); self.parts.push(WordPart::Variable(name)); } - fn push_braced_variable(&mut self, name: String) { + fn push_braced_variable(&mut self, name: String, quote: Quote) { + self.typed.push_parameter(name.clone(), true, quote); self.parts.push(WordPart::BracedVariable(name)); } - fn push_command_sub(&mut self, cmd: String) { + fn push_command_sub(&mut self, cmd: String, quote: Quote) { + self.typed.push_command_sub(cmd.clone(), quote); self.parts.push(WordPart::CommandSub(cmd)); } fn push_process_sub(&mut self, sub_type: ProcessSubType, cmd: String) { + self.typed.push_process_sub(sub_type, cmd.clone()); self.parts.push(WordPart::ProcessSub(sub_type, cmd)); } } @@ -285,7 +293,8 @@ pub enum Command { // Shell builtins /// Echo: write arguments to stdout Echo { - args: Vec, + /// Arguments after the options, with their quoting. + args: Vec, no_newline: bool, interpret_escapes: bool, redirects: Vec, @@ -335,7 +344,8 @@ pub enum Command { // External command External { program: String, - args: Vec, + /// Arguments with their quoting; expanded by [`crate::word::expand_words`]. + args: Vec, redirects: Vec, background: bool, }, @@ -345,7 +355,7 @@ pub enum Command { /// Each stage is a (program, args) pair. Intermediate stages use piped stdio. /// Final redirections apply to the last stage only. Pipeline { - stages: Vec<(String, Vec)>, + stages: Vec<(String, Vec)>, redirects: Vec, background: bool, }, @@ -599,6 +609,7 @@ fn tokenize(input: &str) -> Result> { current_literal.push('\\'); } current_literal.push(ch); + current_word.typed.push_char(ch, Quote::Backslash); quote_state = QuoteState::None; } @@ -609,6 +620,7 @@ fn tokenize(input: &str) -> Result> { quote_state = QuoteState::None; } else { current_literal.push(ch); + current_word.typed.push_char(ch, Quote::Single); } } @@ -631,27 +643,31 @@ fn tokenize(input: &str) -> Result> { current_literal.push('\\'); } current_literal.push(next_ch); + current_word.typed.push_char(next_ch, Quote::Double); } else { // Not a special char, keep backslash current_literal.push('\\'); + current_word.typed.push_char('\\', Quote::Double); } } else { current_literal.push('\\'); + current_word.typed.push_char('\\', Quote::Double); } } '$' => { // Variable expansion in double quotes push_literal!(); - parse_variable(&mut chars, &mut current_word)?; + parse_variable(&mut chars, &mut current_word, Quote::Double)?; } '`' => { // Backtick command substitution in double quotes push_literal!(); let cmd = parse_command_sub_backtick(&mut chars)?; - current_word.push_command_sub(cmd); + current_word.push_command_sub(cmd, Quote::Double); } _ => { current_literal.push(ch); + current_word.typed.push_char(ch, Quote::Double); } } } @@ -678,14 +694,14 @@ fn tokenize(input: &str) -> Result> { // Variable expansion '$' => { push_literal!(); - parse_variable(&mut chars, &mut current_word)?; + parse_variable(&mut chars, &mut current_word, Quote::None)?; } // Backtick command substitution '`' => { push_literal!(); let cmd = parse_command_sub_backtick(&mut chars)?; - current_word.push_command_sub(cmd); + current_word.push_command_sub(cmd, Quote::None); } // Whitespace: end current word @@ -766,6 +782,7 @@ fn tokenize(input: &str) -> Result> { } else { // Invalid: 2>&[not 1] current_literal.push_str("2>&"); + current_word.typed.push_str("2>&", Quote::None); } } _ => { @@ -775,6 +792,7 @@ fn tokenize(input: &str) -> Result> { } else { // Regular '2' character, part of word current_literal.push(ch); + current_word.typed.push_char(ch, Quote::None); } } @@ -816,6 +834,7 @@ fn tokenize(input: &str) -> Result> { } else { // Regular '[' character, part of word current_literal.push(ch); + current_word.typed.push_char(ch, Quote::None); } } @@ -828,12 +847,14 @@ fn tokenize(input: &str) -> Result> { } else { // Regular ']' character, part of word current_literal.push(ch); + current_word.typed.push_char(ch, Quote::None); } } // Regular character _ => { current_literal.push(ch); + current_word.typed.push_char(ch, Quote::None); } } } @@ -861,12 +882,13 @@ fn tokenize(input: &str) -> Result> { fn parse_variable( chars: &mut std::iter::Peekable, word: &mut QuotedWord, + quote: Quote, ) -> Result<()> { if chars.peek() == Some(&'(') { // Command substitution: $(cmd) chars.next(); // consume '(' let cmd = parse_command_sub_dollar(chars)?; - word.push_command_sub(cmd); + word.push_command_sub(cmd, quote); } else if chars.peek() == Some(&'{') { // Braced form: ${VAR} chars.next(); // consume '{' @@ -888,7 +910,7 @@ fn parse_variable( } } - word.push_braced_variable(var_name); + word.push_braced_variable(var_name, quote); } else if let Some(&next_ch) = chars.peek() { // Simple form: $VAR or special variables. The peek above already // bound next_ch by value; chars.next() is called purely to advance @@ -897,11 +919,11 @@ fn parse_variable( if next_ch == '?' || next_ch == '$' || next_ch == '#' { // Single-character special variable chars.next(); - word.push_variable(next_ch.to_string()); + word.push_variable(next_ch.to_string(), quote); } else if next_ch.is_ascii_digit() { // Positional parameter: $0, $1, $2, etc. chars.next(); - word.push_variable(next_ch.to_string()); + word.push_variable(next_ch.to_string(), quote); } else if next_ch.is_alphabetic() || next_ch == '_' { // Variable name let mut var_name = String::new(); @@ -915,18 +937,20 @@ fn parse_variable( } } - word.push_variable(var_name); + word.push_variable(var_name, quote); } else if next_ch == '@' || next_ch == '*' { // Special positional parameters: $@ or $* chars.next(); - word.push_variable(next_ch.to_string()); + word.push_variable(next_ch.to_string(), quote); } else { // $ not followed by variable, treat as literal word.push_literal("$".to_string()); + word.typed.push_char('$', quote); } } else { // $ at end of string word.push_literal("$".to_string()); + word.typed.push_char('$', quote); } Ok(()) @@ -935,15 +959,14 @@ fn parse_variable( /// Parse command substitution in $(cmd) form fn parse_command_sub_dollar(chars: &mut std::iter::Peekable) -> Result { let mut cmd = String::new(); - let mut depth = 1; // Track nesting depth for nested $() + let mut depth = 1; // Parenthesis depth: nested $(), $((expr)) and (...) for ch in chars.by_ref() { match ch { '(' => { - // Check if it's $( to track nested command substitution - if cmd.ends_with('$') { - depth += 1; - } + // Every `(` nests, so the `)` that closes `$((1+2))` or + // `$(echo $(pwd))` belongs to the substitution. + depth += 1; cmd.push(ch); } ')' => { @@ -1228,28 +1251,28 @@ fn parse_pipeline(tokens: &[Token]) -> Result { stages[last_stage_idx] = last_command_tokens; // Parse each stage as (program, args) - let mut parsed_stages: Vec<(String, Vec)> = Vec::new(); + let mut parsed_stages: Vec<(String, Vec)> = Vec::new(); for stage_tokens in stages { if stage_tokens.is_empty() { return Err(anyhow!("Empty pipeline stage")); } - let words: Vec = stage_tokens + let words: Vec<&QuotedWord> = stage_tokens .iter() .filter_map(|t| match t { - Token::Word(w) => Some(quoted_word_to_string(w)), + Token::Word(w) => Some(w), _ => None, }) .collect(); - if words.is_empty() { + let Some((program, args)) = words.split_first() else { return Err(anyhow!("Pipeline stage has no command")); - } - - let program = words[0].clone(); - let args = words[1..].to_vec(); - parsed_stages.push((program, args)); + }; + parsed_stages.push(( + quoted_word_to_string(program), + args.iter().map(|w| w.typed.clone()).collect(), + )); } Ok(Command::Pipeline { @@ -1719,6 +1742,7 @@ pub fn parse_command(input: &str) -> Result { // Separate command tokens from redirections let mut command_tokens: Vec = Vec::new(); + let mut command_words: Vec = Vec::new(); let mut redirects = Vec::new(); let mut background = false; @@ -1727,6 +1751,7 @@ pub fn parse_command(input: &str) -> Result { match &tokens[i] { Token::Word(w) => { command_tokens.push(quoted_word_to_string(w)); + command_words.push(w.typed.clone()); i += 1; } @@ -1852,9 +1877,10 @@ pub fn parse_command(input: &str) -> Result { // Parse command from tokens let cmd = &command_tokens[0]; let args: Vec = command_tokens[1..].to_vec(); + let words: Vec = command_words[1..].to_vec(); // Parse base command with redirections - parse_base_command(cmd, args, redirects, background) + parse_base_command(cmd, args, words, redirects, background) } /// Check if a string is a valid variable name @@ -2369,7 +2395,7 @@ fn apply_substring(value: &str, offset: i32, length: Option) -> String { /// /// `~user` is not yet supported (requires getpwnam). An escaped `\~` /// (produced by `quoted_word_to_string` for quoted `~`) stays literal. -fn expand_tilde(input: &str) -> String { +pub(crate) fn expand_tilde(input: &str) -> String { // An escaped tilde `\~` means the tilde was inside quotes — leave it. if let Some(stripped) = input.strip_prefix("\\~") { let mut s = String::with_capacity(input.len()); @@ -2628,14 +2654,12 @@ pub fn expand_command_substitution( process_cmd.stdout(Stdio::piped()); process_cmd.stderr(Stdio::null()); - // Expand variables and command substitutions in args before execution - let expanded_args: Result> = args - .iter() - .map(|arg| expand_with_command_sub(arg, state)) - .collect(); - let expanded_args = expanded_args?; + // Expand the arguments (variables, command substitution, globs) + // before execution. + let expansion = + crate::word::expand_words(args, state, crate::word::ProcessSubs::Refuse)?; - process_cmd.args(&expanded_args); + process_cmd.args(&expansion.fields); let output_result = process_cmd .output() @@ -2651,6 +2675,22 @@ pub fn expand_command_substitution( } } + Command::Echo { + args, + no_newline, + interpret_escapes: false, + redirects, + } if redirects.is_empty() => { + // echo builtin: its output is the joined, expanded arguments. + let expansion = + crate::word::expand_words(args, state, crate::word::ProcessSubs::Refuse)?; + let mut out = expansion.fields.join(" "); + if !no_newline { + out.push('\n'); + } + out + } + Command::Pwd { .. } => { // pwd builtin std::env::current_dir() @@ -3253,9 +3293,13 @@ pub fn is_incomplete_control_structure(input: &str) -> bool { block_depth > 0 } +/// `args` are the arguments in the older string encoding, which the builtins +/// still read; `words` are the same arguments with their quoting, used by +/// `echo` and external commands. The two lists have the same length. fn parse_base_command( cmd: &str, args: Vec, + words: Vec, redirects: Vec, background: bool, ) -> Result { @@ -3382,7 +3426,8 @@ fn parse_base_command( // Re-parse the inner command: first arg is the command name, rest are its args let inner_cmd = &args[0]; let inner_args = args[1..].to_vec(); - let inner = parse_base_command(inner_cmd, inner_args, vec![], false)?; + let inner_words = words[1..].to_vec(); + let inner = parse_base_command(inner_cmd, inner_args, inner_words, vec![], false)?; Ok(Command::Explain { inner: Box::new(inner), }) @@ -3541,7 +3586,7 @@ fn parse_base_command( let mut echo_args = Vec::new(); let mut skip_flags = true; - for arg in &args { + for (arg, word) in args.iter().zip(&words) { if skip_flags && arg.starts_with('-') { for ch in arg[1..].chars() { match ch { @@ -3551,14 +3596,14 @@ fn parse_base_command( _ => { // Unknown flag — treat as regular argument skip_flags = false; - echo_args.push(arg.clone()); + echo_args.push(word.clone()); break; } } } } else { skip_flags = false; - echo_args.push(arg.clone()); + echo_args.push(word.clone()); } } @@ -3729,7 +3774,7 @@ fn parse_base_command( // Everything else: external command _ => Ok(Command::External { program: cmd.to_string(), - args, + args: words, redirects, background, }), @@ -4246,7 +4291,7 @@ mod tests { match cmd { Command::External { program, args, .. } => { assert_eq!(program, "touch"); - let expanded = expand_variables(&args[0], &state); + let expanded = expand_variables(&args[0].to_string(), &state); assert_eq!(expanded, "file_Alice"); } Command::Touch { path, .. } => { @@ -5313,3 +5358,78 @@ mod semicolon_tests { assert_eq!(parts, vec!["", ""]); } } + +#[cfg(test)] +mod typed_word_tests { + use super::*; + use proptest::prelude::*; + + fn typed_words(input: &str) -> Vec { + tokenize(input) + .unwrap() + .into_iter() + .filter_map(|t| match t { + Token::Word(w) => Some(w.typed), + _ => None, + }) + .collect() + } + + fn word(parts: &[(&str, Quote)]) -> Word { + let mut w = Word::new(); + for (text, quote) in parts { + w.push_str(text, *quote); + } + w + } + + #[test] + fn the_tokenizer_records_the_quoting_of_each_piece() { + assert_eq!( + typed_words("echo '*'x\\?\"a $V\"*"), + vec![Word::literal("echo"), { + let mut w = word(&[("*", Quote::Single), ("x", Quote::None)]); + w.push_char('?', Quote::Backslash); + w.push_str("a ", Quote::Double); + w.push_parameter("V".into(), false, Quote::Double); + w.push_char('*', Quote::None); + w + }] + ); + } + + #[test] + fn arithmetic_expansion_is_one_command_substitution_piece() { + let mut expected = Word::new(); + expected.push_command_sub("(1+(2*3))".into(), Quote::None); + expected.push_char('x', Quote::None); + assert_eq!(typed_words("$((1+(2*3)))x"), vec![expected]); + } + + /// One literal piece: (text, quoting). Characters are limited to those + /// the tokenizer reads literally in each quoting. + fn piece() -> impl Strategy { + prop_oneof![ + "[a-z0-9._/*?-]{1,4}".prop_map(|s| (s, Quote::None)), + "[a-z *?$\"\\\\]{1,4}".prop_map(|s| (s, Quote::Single)), + "[a-z '*?$\"\\\\]{1,4}".prop_map(|s| (s, Quote::Double)), + "[a-z*?$'\"\\\\ ]".prop_map(|s| (s, Quote::Backslash)), + ] + } + + proptest! { + #![proptest_config(ProptestConfig::with_cases(256))] + + /// `Display` writes a word as source that tokenizes back to the + /// same word, so the quoting survives a round trip. + #[test] + fn display_then_tokenize_gives_the_same_word(pieces in prop::collection::vec(piece(), 1..6)) { + let mut w = Word::new(); + for (text, quote) in &pieces { + w.push_str(text, *quote); + } + let source = w.to_string(); + prop_assert_eq!(typed_words(&source), vec![w], "source: {}", source); + } + } +} diff --git a/impl/rust-cli/src/process_sub.rs b/impl/rust-cli/src/process_sub.rs index 7387952b..131eee30 100644 --- a/impl/rust-cli/src/process_sub.rs +++ b/impl/rust-cli/src/process_sub.rs @@ -157,11 +157,8 @@ fn start_command_with_output_redirect( let cmd_string = match cmd { Command::External { program, args, .. } => { let expanded_program = crate::parser::expand_with_command_sub(program, state)?; - let expanded_args: Result> = args - .iter() - .map(|arg| crate::parser::expand_with_command_sub(arg, state)) - .collect(); - let expanded_args = expanded_args?; + let expanded_args = + crate::word::expand_words(args, state, crate::word::ProcessSubs::Refuse)?.fields; // Build command string: prog arg1 arg2 > fifo let mut cmd = expanded_program; @@ -210,11 +207,8 @@ fn start_command_with_input_redirect( let cmd_string = match cmd { Command::External { program, args, .. } => { let expanded_program = crate::parser::expand_with_command_sub(program, state)?; - let expanded_args: Result> = args - .iter() - .map(|arg| crate::parser::expand_with_command_sub(arg, state)) - .collect(); - let expanded_args = expanded_args?; + let expanded_args = + crate::word::expand_words(args, state, crate::word::ProcessSubs::Refuse)?.fields; // Build command string let mut cmd = expanded_program; diff --git a/impl/rust-cli/src/word.rs b/impl/rust-cli/src/word.rs new file mode 100644 index 00000000..7229a79a --- /dev/null +++ b/impl/rust-cli/src/word.rs @@ -0,0 +1,668 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright (c) Jonathan D.A. Jewell +//! Typed shell words and the word-expansion stage for simple-command arguments. +//! +//! The tokenizer used to flatten each word into a `String` in which quoted +//! glob metacharacters were marked with a backslash (`'*'` became `\*`). +//! Nothing removed those backslashes again, so `echo '*.txt'` printed +//! `\*.txt`, and glob expansion could not tell a quoted `*` from an unquoted +//! one without re-parsing that encoding. +//! +//! A [`Word`] keeps the quoting of every piece instead: each [`Segment`] +//! records whether it was written unquoted, in single or double quotes, or +//! after a backslash. [`expand_words`] then runs the POSIX word-expansion +//! steps on that structure, in this order: +//! +//! 1. tilde expansion (an unquoted `~`, `~/`, `~+`, `~-` at the start), +//! parameter expansion, command substitution, arithmetic expansion and +//! process substitution; +//! 2. brace expansion (`{a,b}`) of unquoted literal text; +//! 3. pathname (glob) expansion of the characters that are not quoted; +//! 4. quote removal. +//! +//! # Rules kept from the previous expansion code +//! +//! - A pattern that matches nothing, or is not a valid pattern, stays as the +//! literal word. +//! - A pattern whose expansion reaches a [`GlobLimits`] limit fails the whole +//! expansion with the [`GlobLimitExceeded`] error; the caller must not run +//! the command. +//! +//! # Known differences from POSIX (not changed here) +//! +//! - No field splitting: an unquoted `$X` whose value is `a b` stays one +//! argument. +//! - An empty word (`''`, or an unquoted expansion that is empty) is dropped +//! by the tokenizer or kept as an empty argument, as before. +//! - Brace expansion runs after parameter expansion (bash runs it before); +//! braces that come out of an expansion are not expanded. +//! +//! Only the arguments of external commands, pipeline stages, `echo` and +//! function calls go through this stage so far. Builtin operands, redirection +//! targets, assignments, `case` patterns and `test` arguments still use the +//! older string encoding. + +use std::fmt; +use std::path::Path; + +use anyhow::{Context, Result}; + +use crate::glob::{self, GlobLimitExceeded, GlobLimits}; +use crate::parser::ProcessSubType; +use crate::process_sub::ProcessSubstitution; +use crate::state::ShellState; + +/// How a piece of a word was written in the source. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Quote { + /// Not quoted: subject to tilde, brace and glob expansion. + None, + /// Inside `'...'`. + Single, + /// Inside `"..."`. + Double, + /// A single character after an unquoted `\`. + Backslash, +} + +impl Quote { + fn is_quoted(self) -> bool { + self != Quote::None + } +} + +/// One piece of a [`Word`]. +#[derive(Debug, Clone, PartialEq)] +pub enum Segment { + /// Literal text. + Literal { text: String, quote: Quote }, + /// `$NAME` (`braced: false`) or `${EXPR}` (`braced: true`). + Parameter { + expr: String, + braced: bool, + quote: Quote, + }, + /// `$(CMD)` or `` `CMD` ``; `$((EXPR))` is stored as the command `(EXPR)`. + CommandSub { cmd: String, quote: Quote }, + /// `<(CMD)` or `>(CMD)`. + ProcessSub { kind: ProcessSubType, cmd: String }, +} + +/// A shell word with the quoting of each of its pieces. +#[derive(Debug, Clone, PartialEq, Default)] +pub struct Word { + segments: Vec, +} + +impl Word { + /// An empty word. + pub fn new() -> Self { + Self::default() + } + + /// A word that is a single unquoted literal. + /// + /// # Examples + /// ``` + /// use vsh::word::Word; + /// assert_eq!(Word::literal("a.txt").to_string(), "a.txt"); + /// ``` + pub fn literal(text: &str) -> Self { + let mut word = Self::new(); + for ch in text.chars() { + word.push_char(ch, Quote::None); + } + word + } + + /// The pieces of the word, in order. + pub fn segments(&self) -> &[Segment] { + &self.segments + } + + /// True when the word has no pieces. + pub fn is_empty(&self) -> bool { + self.segments.is_empty() + } + + /// Append one literal character, merging it into the last piece when that + /// piece is a literal with the same quoting. + pub fn push_char(&mut self, ch: char, quote: Quote) { + if quote != Quote::Backslash { + if let Some(Segment::Literal { text, quote: last }) = self.segments.last_mut() { + if *last == quote { + text.push(ch); + return; + } + } + } + self.segments.push(Segment::Literal { + text: ch.to_string(), + quote, + }); + } + + /// Append literal text. + pub fn push_str(&mut self, text: &str, quote: Quote) { + for ch in text.chars() { + self.push_char(ch, quote); + } + } + + /// Append a parameter expansion. + pub fn push_parameter(&mut self, expr: String, braced: bool, quote: Quote) { + self.segments.push(Segment::Parameter { + expr, + braced, + quote, + }); + } + + /// Append a command substitution. + pub fn push_command_sub(&mut self, cmd: String, quote: Quote) { + self.segments.push(Segment::CommandSub { cmd, quote }); + } + + /// Append a process substitution. + pub fn push_process_sub(&mut self, kind: ProcessSubType, cmd: String) { + self.segments.push(Segment::ProcessSub { kind, cmd }); + } +} + +impl From<&str> for Word { + fn from(text: &str) -> Self { + Word::literal(text) + } +} + +/// Writes the word back as shell source that parses to the same word. +impl fmt::Display for Word { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for segment in &self.segments { + match segment { + Segment::Literal { text, quote } => match quote { + Quote::None => f.write_str(text)?, + Quote::Backslash => { + for ch in text.chars() { + write!(f, "\\{ch}")?; + } + } + Quote::Single => { + // `'` cannot appear inside single quotes. + write!(f, "'{}'", text.replace('\'', "'\\''"))? + } + Quote::Double => { + f.write_str("\"")?; + for ch in text.chars() { + if matches!(ch, '"' | '$' | '`' | '\\') { + f.write_str("\\")?; + } + write!(f, "{ch}")?; + } + f.write_str("\"")?; + } + }, + Segment::Parameter { + expr, + braced, + quote, + } => { + let open = if quote.is_quoted() { "\"" } else { "" }; + if *braced { + write!(f, "{open}${{{expr}}}{open}")?; + } else { + write!(f, "{open}${expr}{open}")?; + } + } + Segment::CommandSub { cmd, quote } => { + let open = if quote.is_quoted() { "\"" } else { "" }; + write!(f, "{open}$({cmd}){open}")?; + } + Segment::ProcessSub { kind, cmd } => match kind { + ProcessSubType::Input => write!(f, "<({cmd})")?, + ProcessSubType::Output => write!(f, ">({cmd})")?, + }, + } + } + Ok(()) + } +} + +/// Compares the source form ([`fmt::Display`]) with `other`. +impl PartialEq for Word { + fn eq(&self, other: &str) -> bool { + let source = self.to_string(); + source.as_str() == other + } +} + +impl PartialEq<&str> for Word { + fn eq(&self, other: &&str) -> bool { + *self == **other + } +} + +/// Whether process substitutions may be started while expanding. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ProcessSubs { + /// Start them; the caller cleans them up after the command finishes. + Allow, + /// Fail on them, as the code paths without FIFO handling always have. + Refuse, +} + +/// The result of expanding a list of words. +pub struct Expansion { + /// The arguments, after quote removal. + pub fields: Vec, + /// Process substitutions started for these arguments. + pub process_subs: Vec, +} + +impl Expansion { + /// Wait for the process substitutions and remove their FIFOs. + pub fn cleanup(self) { + for proc_sub in self.process_subs { + if let Err(e) = proc_sub.cleanup() { + eprintln!("Warning: Failed to cleanup process substitution: {}", e); + } + } + } +} + +/// Expand words relative to the current directory under the default limits. +/// +/// # Errors +/// Fails on an expansion error (a failing command substitution, a refused +/// process substitution) or when a glob reaches a limit +/// ([`GlobLimitExceeded`], reachable with `anyhow::Error::downcast_ref`). +pub fn expand_words( + words: &[Word], + state: &mut ShellState, + process_subs: ProcessSubs, +) -> Result { + let cwd = std::env::current_dir().context("Failed to get current working directory")?; + expand_words_in(words, state, process_subs, &cwd, &GlobLimits::DEFAULT) +} + +/// [`expand_words`] with an explicit directory for relative patterns and +/// explicit glob limits. +/// +/// # Errors +/// As for [`expand_words`]. +pub fn expand_words_in( + words: &[Word], + state: &mut ShellState, + process_subs: ProcessSubs, + cwd: &Path, + limits: &GlobLimits, +) -> Result { + let mut expansion = Expansion { + fields: Vec::new(), + process_subs: Vec::new(), + }; + for word in words { + // On error the substitutions started so far are dropped, and Drop + // stops them and removes their FIFOs. + let chars = expand_segments(word, state, process_subs, &mut expansion.process_subs)?; + glob_fields(&chars, cwd, limits, &mut expansion.fields)?; + } + Ok(expansion) +} + +/// What later steps may do with a character produced by step 1. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Role { + /// Quoted text or a tilde result: taken literally. + Literal, + /// Unquoted source text: brace and glob expansion apply. + Source, + /// Result of an unquoted expansion: glob expansion applies, braces do not. + Expanded, +} + +/// Step 1: tilde, parameter, command, arithmetic and process substitution. +fn expand_segments( + word: &Word, + state: &mut ShellState, + process_subs: ProcessSubs, + started: &mut Vec, +) -> Result> { + let mut out: Vec<(char, Role)> = Vec::new(); + let push = |out: &mut Vec<(char, Role)>, text: &str, role: Role| { + out.extend(text.chars().map(|ch| (ch, role))); + }; + + for (index, segment) in word.segments.iter().enumerate() { + match segment { + Segment::Literal { text, quote } => { + if !quote.is_quoted() { + if index == 0 && text.starts_with('~') { + let (prefix, rest) = text.split_at(text.find('/').unwrap_or(text.len())); + let expanded = crate::parser::expand_tilde(prefix); + if expanded != prefix { + push(&mut out, &expanded, Role::Literal); + push(&mut out, rest, Role::Source); + continue; + } + } + push(&mut out, text, Role::Source); + } else { + push(&mut out, text, Role::Literal); + } + } + Segment::Parameter { + expr, + braced, + quote, + } => { + let source = if *braced { + format!("${{{expr}}}") + } else { + format!("${expr}") + }; + let value = crate::parser::expand_variables(&source, state); + push(&mut out, &value, expansion_role(*quote)); + } + Segment::CommandSub { cmd, quote } => { + let value = crate::parser::expand_with_command_sub(&format!("$({cmd})"), state)?; + push(&mut out, &value, expansion_role(*quote)); + } + Segment::ProcessSub { kind, cmd } => { + if process_subs == ProcessSubs::Refuse { + anyhow::bail!( + "Process substitution not yet implemented: execution requires FIFO support ({})", + Word { + segments: vec![segment.clone()] + } + ); + } + let mut proc_sub = ProcessSubstitution::create(*kind, cmd.clone(), state)?; + proc_sub.start(state)?; + push( + &mut out, + &proc_sub.fifo_path.to_string_lossy(), + Role::Literal, + ); + started.push(proc_sub); + } + } + } + Ok(out) +} + +fn expansion_role(quote: Quote) -> Role { + if quote.is_quoted() { + Role::Literal + } else { + Role::Expanded + } +} + +/// Encode step-1 output for [`glob::expand_braces`] and +/// [`glob::contains_glob_pattern`], which both read `\x` as a literal `x`. +fn encode(chars: &[(char, Role)]) -> String { + let mut out = String::with_capacity(chars.len() * 2); + for &(ch, role) in chars { + let literal = match role { + Role::Literal => true, + Role::Source => ch == '\\', + Role::Expanded => matches!(ch, '\\' | '{' | '}' | ','), + }; + if literal { + out.push('\\'); + } + out.push(ch); + } + out +} + +/// Steps 2 to 4: brace expansion, glob expansion and quote removal. +fn glob_fields( + chars: &[(char, Role)], + cwd: &Path, + limits: &GlobLimits, + fields: &mut Vec, +) -> Result<()> { + let encoded = encode(chars); + for candidate in glob::expand_braces(&encoded) { + match glob_pattern(&candidate) { + None => fields.push(remove_quotes(&candidate)), + Some(pattern) => match glob::expand_glob_with_limits(&pattern, cwd, limits) { + Ok(found) if !found.paths.is_empty() => fields.extend( + found + .paths + .iter() + .map(|path| path.to_string_lossy().into_owned()), + ), + // A limit is not "no match": substituting the literal word + // would run the command on the wrong arguments. + Err(e) if e.is::() => return Err(e), + // No match, or not a valid pattern: keep the word (POSIX). + _ => fields.push(remove_quotes(&candidate)), + }, + } + } + Ok(()) +} + +/// The pattern for the `glob` crate, or `None` when no `*`, `?` or `[` is +/// left unquoted. Quoted metacharacters are written as one-character classes +/// (`[*]`), the only escape that crate has. +fn glob_pattern(encoded: &str) -> Option { + let mut pattern = String::with_capacity(encoded.len()); + let mut active = false; + let mut chars = encoded.chars(); + while let Some(ch) = chars.next() { + if ch == '\\' { + if let Some(next) = chars.next() { + pattern.push_str(&::glob::Pattern::escape(&next.to_string())); + } + } else { + active |= matches!(ch, '*' | '?' | '['); + pattern.push(ch); + } + } + active.then_some(pattern) +} + +/// Quote removal: `\x` becomes `x`. +fn remove_quotes(encoded: &str) -> String { + let mut out = String::with_capacity(encoded.len()); + let mut chars = encoded.chars(); + while let Some(ch) = chars.next() { + if ch == '\\' { + if let Some(next) = chars.next() { + out.push(next); + } + } else { + out.push(ch); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn word(parts: &[(&str, Quote)]) -> Word { + let mut w = Word::new(); + for (text, quote) in parts { + w.push_str(text, *quote); + } + w + } + + fn fixture(names: &[&str]) -> tempfile::TempDir { + let dir = tempfile::TempDir::new().unwrap(); + for name in names { + std::fs::write(dir.path().join(name), b"").unwrap(); + } + dir + } + + fn expand_in(dir: &Path, words: &[Word]) -> Vec { + let mut state = ShellState::new(dir.to_str().unwrap()).unwrap(); + expand_words_in( + words, + &mut state, + ProcessSubs::Refuse, + dir, + &GlobLimits::DEFAULT, + ) + .unwrap() + .fields + } + + #[test] + fn unquoted_patterns_expand_and_quoted_ones_do_not() { + let dir = fixture(&["a.txt", "b.txt", "c.log"]); + let out = expand_in( + dir.path(), + &[ + Word::literal("*.txt"), + word(&[("*.txt", Quote::Single)]), + word(&[("*.txt", Quote::Double)]), + word(&[("*", Quote::Backslash), (".txt", Quote::None)]), + ], + ); + assert_eq!(out, ["a.txt", "b.txt", "*.txt", "*.txt", "*.txt"]); + } + + #[test] + fn a_partly_quoted_pattern_matches_its_quoted_part_literally() { + let dir = fixture(&["*x.txt", "ax.txt", "a.txt"]); + // `'*'x*` : a literal star followed by `x` and a wildcard. + let out = expand_in( + dir.path(), + &[word(&[("*", Quote::Single), ("x*", Quote::None)])], + ); + assert_eq!(out, ["*x.txt"]); + } + + #[test] + fn no_match_and_invalid_patterns_stay_literal() { + let dir = fixture(&["a.txt"]); + let out = expand_in( + dir.path(), + &[ + Word::literal("*.txt"), + Word::literal("*.none"), + Word::literal("[z-"), + ], + ); + assert_eq!(out, ["a.txt", "*.none", "[z-"]); + } + + #[test] + fn braces_expand_only_when_unquoted() { + let dir = fixture(&[]); + let out = expand_in( + dir.path(), + &[Word::literal("a{1,2}"), word(&[("a{1,2}", Quote::Single)])], + ); + assert_eq!(out, ["a1", "a2", "a{1,2}"]); + } + + #[test] + fn literal_backslashes_survive_quote_removal() { + let dir = fixture(&["a.txt"]); + let out = expand_in( + dir.path(), + &[ + word(&[("a\\*", Quote::Single)]), + word(&[("\\", Quote::Backslash)]), + ], + ); + assert_eq!(out, ["a\\*", "\\"]); + } + + #[test] + fn a_quoted_backslash_is_a_literal_character_in_a_pattern() { + let dir = fixture(&["a\\b.txt", "ab.txt"]); + // `'a\'*` : names that start with `a\`. + let out = expand_in( + dir.path(), + &[word(&[("a\\", Quote::Single), ("*", Quote::None)])], + ); + assert_eq!(out, ["a\\b.txt"]); + } + + #[test] + fn unquoted_parameter_values_are_globbed_and_quoted_ones_are_not() { + let dir = fixture(&["a.txt", "b.txt"]); + let mut state = ShellState::new(dir.path().to_str().unwrap()).unwrap(); + state.set_variable("P", "*.txt"); + let mut unquoted = Word::new(); + unquoted.push_parameter("P".into(), false, Quote::None); + let mut quoted = Word::new(); + quoted.push_parameter("P".into(), true, Quote::Double); + let out = expand_words_in( + &[unquoted, quoted], + &mut state, + ProcessSubs::Refuse, + dir.path(), + &GlobLimits::DEFAULT, + ) + .unwrap() + .fields; + assert_eq!(out, ["a.txt", "b.txt", "*.txt"]); + } + + #[test] + fn a_glob_limit_fails_the_expansion_instead_of_substituting_the_literal() { + let dir = fixture(&["f0.txt", "f1.txt", "f2.txt", "f3.txt", "f4.txt"]); + let mut state = ShellState::new(dir.path().to_str().unwrap()).unwrap(); + let limits = GlobLimits { + max_matches: 4, + ..GlobLimits::DEFAULT + }; + let words = [Word::literal("-l"), Word::literal("*.txt")]; + let err = expand_words_in(&words, &mut state, ProcessSubs::Refuse, dir.path(), &limits) + .err() + .expect("the limit must fail the expansion"); + let stopped = err + .downcast_ref::() + .expect("the limit error must reach the caller"); + assert_eq!(stopped.exceeded.axis, "matches held"); + assert!(err.to_string().contains("over the limit of 4"), "{err}"); + } + + #[test] + fn refused_process_substitution_is_an_error() { + let dir = fixture(&[]); + let mut state = ShellState::new(dir.path().to_str().unwrap()).unwrap(); + let mut w = Word::new(); + w.push_process_sub(ProcessSubType::Input, "true".into()); + let err = expand_words_in( + &[w], + &mut state, + ProcessSubs::Refuse, + dir.path(), + &GlobLimits::DEFAULT, + ) + .err() + .expect("refused"); + assert!(err.to_string().contains("<(true)"), "{err}"); + } + + #[test] + fn display_round_trips_quoting() { + let w = word(&[ + ("a b", Quote::Single), + ("*", Quote::Backslash), + ("x\"$", Quote::Double), + ("y", Quote::None), + ]); + assert_eq!(w.to_string(), "'a b'\\*\"x\\\"\\$\"y"); + assert_eq!(Word::literal("plain"), "plain"); + } + + #[test] + fn glob_pattern_escapes_quoted_metacharacters() { + assert_eq!(glob_pattern("\\*a*"), Some("[*]a*".to_string())); + assert_eq!(glob_pattern("\\*a"), None); + assert_eq!(glob_pattern("\\[x?"), Some("[[]x?".to_string())); + } +} diff --git a/impl/rust-cli/tests/word_expansion_e2e_tests.rs b/impl/rust-cli/tests/word_expansion_e2e_tests.rs new file mode 100644 index 00000000..bcfa282b --- /dev/null +++ b/impl/rust-cli/tests/word_expansion_e2e_tests.rs @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright (c) Jonathan D.A. Jewell +//! End-to-end tests for argument glob expansion, run through the `vsh` binary. +//! +//! Before the word-expansion stage (`vsh::word`), globs were expanded only +//! for external commands that also had a redirection: `wc -c *.txt` passed +//! `*.txt` to `wc`, pipelines and `echo` never expanded, and quoted patterns +//! reached the program with a stray backslash (`echo '*.txt'` printed +//! `\*.txt`). Each test runs `vsh -c` in a directory holding `a.txt`, +//! `b.txt` and `c.log`. + +use std::path::Path; +use std::process::{Command, Output}; + +use tempfile::TempDir; + +fn fixture() -> TempDir { + let dir = TempDir::new().unwrap(); + for name in ["a.txt", "b.txt", "c.log"] { + std::fs::write(dir.path().join(name), b"").unwrap(); + } + dir +} + +fn vsh(dir: &Path, script: &str) -> Output { + Command::new(env!("CARGO_BIN_EXE_vsh")) + .arg("-c") + .arg(script) + .current_dir(dir) + .output() + .expect("vsh runs") +} + +fn stdout(dir: &Path, script: &str) -> String { + let out = vsh(dir, script); + assert!( + out.status.success(), + "`{script}` failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8(out.stdout).unwrap() +} + +#[test] +fn external_command_without_redirect_gets_the_matches() { + let dir = fixture(); + assert_eq!( + stdout(dir.path(), "wc -c *.txt"), + "0 a.txt\n0 b.txt\n0 total\n" + ); +} + +#[test] +fn external_command_with_redirect_gets_the_matches() { + let dir = fixture(); + stdout(dir.path(), "wc -c *.txt > out.wc"); + assert_eq!( + std::fs::read_to_string(dir.path().join("out.wc")).unwrap(), + "0 a.txt\n0 b.txt\n0 total\n" + ); +} + +#[test] +fn every_pipeline_stage_gets_the_matches() { + let dir = fixture(); + assert_eq!(stdout(dir.path(), "ls *.txt | wc -l").trim(), "2"); + assert_eq!(stdout(dir.path(), "printf '%s\\n' *.log | cat"), "c.log\n"); +} + +#[test] +fn builtin_echo_gets_the_matches() { + let dir = fixture(); + assert_eq!(stdout(dir.path(), "echo *.txt"), "a.txt b.txt\n"); + assert_eq!(stdout(dir.path(), "echo -n *.log"), "c.log"); +} + +#[test] +fn quoted_patterns_stay_literal_without_a_backslash() { + let dir = fixture(); + assert_eq!(stdout(dir.path(), "echo '*.txt'"), "*.txt\n"); + assert_eq!(stdout(dir.path(), "echo \"*.txt\""), "*.txt\n"); + assert_eq!(stdout(dir.path(), "echo \\*.txt"), "*.txt\n"); + assert_eq!( + stdout(dir.path(), "printf '%s\\n' '*.txt' \"*.txt\" \\*.txt"), + "*.txt\n*.txt\n*.txt\n" + ); + assert_eq!( + stdout(dir.path(), "printf '%s\\n' '*.txt' | cat"), + "*.txt\n" + ); +} + +#[test] +fn a_pattern_that_matches_nothing_is_passed_literally() { + let dir = fixture(); + assert_eq!(stdout(dir.path(), "echo *.none"), "*.none\n"); + assert_eq!(stdout(dir.path(), "printf '%s\\n' *.none"), "*.none\n"); + assert_eq!( + stdout(dir.path(), "printf '%s\\n' *.none > out.none; cat out.none"), + "*.none\n" + ); +} + +#[test] +fn a_partly_quoted_word_globs_only_its_unquoted_part() { + let dir = fixture(); + std::fs::write(dir.path().join("*x.txt"), b"").unwrap(); + std::fs::write(dir.path().join("ax.txt"), b"").unwrap(); + assert_eq!(stdout(dir.path(), "echo '*'x.*"), "*x.txt\n"); + assert_eq!(stdout(dir.path(), "echo *x.*"), "*x.txt ax.txt\n"); +} + +#[test] +fn braces_expand_unquoted_and_stay_literal_quoted() { + let dir = fixture(); + assert_eq!(stdout(dir.path(), "echo a{1,2} '{a,b}'"), "a1 a2 {a,b}\n"); +} + +#[test] +fn function_arguments_get_the_matches() { + let dir = fixture(); + assert_eq!( + stdout(dir.path(), "f() { echo \"$#:$1:$2\"; }; f *.txt"), + "2:a.txt:b.txt\n" + ); +} From a03ea4f29860b9f05a3420ae21f2fc2ade53b003 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 15:03:40 +0000 Subject: [PATCH 2/2] refactor(word): move literal quoting out of Word's Display SonarCloud rust:S3776 flagged the cognitive complexity of Display::fmt on PR #244. The per-quoting literal case is now fmt_literal; output is unchanged (round-trip property test and Display unit test pass). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01J4QvDfX8Xoq6vedAKdWBif --- impl/rust-cli/src/word.rs | 43 +++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/impl/rust-cli/src/word.rs b/impl/rust-cli/src/word.rs index 7229a79a..98fc8fc9 100644 --- a/impl/rust-cli/src/word.rs +++ b/impl/rust-cli/src/word.rs @@ -175,33 +175,32 @@ impl From<&str> for Word { } } +/// Write one literal piece back as source with its quoting. +fn fmt_literal(f: &mut fmt::Formatter<'_>, text: &str, quote: Quote) -> fmt::Result { + match quote { + Quote::None => f.write_str(text), + Quote::Backslash => text.chars().try_for_each(|ch| write!(f, "\\{ch}")), + // `'` cannot appear inside single quotes. + Quote::Single => write!(f, "'{}'", text.replace('\'', "'\\''")), + Quote::Double => { + let mut escaped = String::with_capacity(text.len() + 2); + for ch in text.chars() { + if matches!(ch, '"' | '$' | '`' | '\\') { + escaped.push('\\'); + } + escaped.push(ch); + } + write!(f, "\"{escaped}\"") + } + } +} + /// Writes the word back as shell source that parses to the same word. impl fmt::Display for Word { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { for segment in &self.segments { match segment { - Segment::Literal { text, quote } => match quote { - Quote::None => f.write_str(text)?, - Quote::Backslash => { - for ch in text.chars() { - write!(f, "\\{ch}")?; - } - } - Quote::Single => { - // `'` cannot appear inside single quotes. - write!(f, "'{}'", text.replace('\'', "'\\''"))? - } - Quote::Double => { - f.write_str("\"")?; - for ch in text.chars() { - if matches!(ch, '"' | '$' | '`' | '\\') { - f.write_str("\\")?; - } - write!(f, "{ch}")?; - } - f.write_str("\"")?; - } - }, + Segment::Literal { text, quote } => fmt_literal(f, text, *quote)?, Segment::Parameter { expr, braced,