diff --git a/CHANGELOG.adoc b/CHANGELOG.adoc index de15214..fb3d88b 100644 --- a/CHANGELOG.adoc +++ b/CHANGELOG.adoc @@ -11,6 +11,35 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 == [Unreleased] +=== Fixed — 2026-11-18 (arguments: globs, quote removal) + +- Glob expansion now applies to the arguments of every external command, + every pipeline stage, `echo` and function calls. Before, only an external + command that also had a redirection expanded its arguments: `wc -c *.txt` + passed `*.txt` to `wc`, `ls *.txt | wc -l` and `echo *.txt` did not expand. +- Quoted arguments no longer reach the program with a backslash in front of + `*`, `?`, `[` or `{`: `echo '*.txt'` printed `\*.txt` and now prints + `*.txt`. A partly quoted word globs only its unquoted part (`'*'x.*`). +- `echo` performs command substitution and arithmetic expansion + (`echo $((2+3))` printed `$((2+3))`), and `$(echo ...)` works. +- `$((1+(2*3)))` and other substitutions containing `(` are read to their + matching `)`. +- An expansion error (such as a glob limit from #191) fails the command with + exit status 1 and does not run it. With a redirection the status was 127. + +=== Changed — 2026-11-18 (typed words) + +- New module `src/word.rs`: a `Word` keeps the quoting of each of its pieces + (unquoted, single, double, backslash), and `word::expand_words` performs + tilde, parameter, command, arithmetic and process substitution, brace + expansion, glob expansion and quote removal on it. `Command::External`, + `Command::Pipeline` and `Command::Echo` now carry `Vec` arguments. + Other commands still use the older string encoding; see + `docs/POSIX_COMPLIANCE.adoc` (Milestones 5 and 6) for what is not migrated. +- Removed `external::expand_glob_args`; its rules (no match or invalid + pattern keeps the word, a limit is an error) moved to `word.rs` with their + tests. + === Fixed — 2026-11-18 (glob expansion out of memory, #191) - `**` no longer follows symbolic links to directories, so a link back to a diff --git a/docs/POSIX_COMPLIANCE.adoc b/docs/POSIX_COMPLIANCE.adoc index e8202f2..175e144 100644 --- a/docs/POSIX_COMPLIANCE.adoc +++ b/docs/POSIX_COMPLIANCE.adoc @@ -177,6 +177,12 @@ or literal result (#191, 2026-11-18) |Brace expansion `+{a,b,c}+` |Done |Nested braces supported |Sorted results |Done |POSIX requirement + +|Where globs expand |Partial |Arguments of external commands (with or +without redirections), every pipeline stage, `+echo+` and function calls, +through `+word::expand_words+` (2026-11-18; before, only external commands +with a redirection). Not yet: builtin operands (`+ls+`, `+rm+`, `+cp+`, ...), +`+for+` word lists, redirection targets |=== ==== Milestone 6: Quote Processing — COMPLETE @@ -194,6 +200,12 @@ quotes |Unclosed quote detection |Done | |Line continuation |Done | + +|Quote removal |Partial |Done for the arguments that go through +`+word::expand_words+` (see Milestone 5), which keeps the quoting of each +piece of a word. Builtin operands, redirection targets, assignment values, +`+case+` patterns and `+test+` arguments still receive quoted `+*?[{~+` with +a backslash in front (`+touch '*q'+` creates `+\*q+`) |=== ==== Milestone 7: Command Substitution — COMPLETE diff --git a/impl/rust-cli/src/commands.rs b/impl/rust-cli/src/commands.rs index 4f91625..02918fc 100644 --- a/impl/rust-cli/src/commands.rs +++ b/impl/rust-cli/src/commands.rs @@ -2856,7 +2856,7 @@ mod tests { let state = ShellState::new(tmp.path().to_str().unwrap()).unwrap(); let cmd = crate::parser::Command::External { program: "git".to_string(), - args: vec!["status".to_string()], + args: vec!["status".into()], redirects: vec![], background: false, }; diff --git a/impl/rust-cli/src/executable.rs b/impl/rust-cli/src/executable.rs index aeb72a2..7148cec 100644 --- a/impl/rust-cli/src/executable.rs +++ b/impl/rust-cli/src/executable.rs @@ -488,35 +488,36 @@ impl ExecutableCommand for Command { // Expand program name (no process substitutions in program name) let expanded_program = crate::parser::expand_with_command_sub(program, state)?; + // Word expansion: variables, substitutions, braces, globs and + // quote removal. A failed expansion (for example a glob that + // reaches a limit) fails the command without running it. + let expansion = + match crate::word::expand_words(args, state, crate::word::ProcessSubs::Allow) { + Ok(expansion) => expansion, + Err(e) => { + eprintln!("vsh: {}: {}", expanded_program, e); + return Ok(ExecutionResult::ExternalCommand { exit_code: 1 }); + } + }; + // Check if the command is a shell function before external execution. // `if let Some(...)` performs a single map lookup; the former // double-lookup plus expect shape could panic and duplicated // the lookup. if let Some(func_def) = state.functions.get(&expanded_program).cloned() { - let expanded_args: Vec = args - .iter() - .map(|a| crate::parser::expand_variables(a, state)) - .collect(); - return execute_function_call(&func_def, &expanded_args, state); + let result = execute_function_call(&func_def, &expansion.fields, state); + expansion.cleanup(); + return result; } - // Expand arguments with process substitutions - let mut all_process_subs = Vec::new(); - let mut expanded_args = Vec::new(); - - for arg in args { - let (expanded_arg, proc_subs) = - crate::parser::expand_with_process_sub(arg, state)?; - expanded_args.push(expanded_arg); - all_process_subs.extend(proc_subs); - } + let expanded_args = &expansion.fields; // Execute main command if *background { // Background execution let _job_id = external::execute_external_background( &expanded_program, - &expanded_args, + expanded_args, redirects, state, ) @@ -525,24 +526,19 @@ impl ExecutableCommand for Command { 0 // No job created }); - // Cleanup all process substitutions - for proc_sub in all_process_subs { - if let Err(e) = proc_sub.cleanup() { - eprintln!("Warning: Failed to cleanup process substitution: {}", e); - } - } + expansion.cleanup(); Ok(ExecutionResult::Success) } else { // Foreground execution let exit_code = if redirects.is_empty() { // No redirections - use simple path - external::execute_external(&expanded_program, &expanded_args) + external::execute_external(&expanded_program, expanded_args) } else { // Has redirections - use redirection-aware execution external::execute_external_with_redirects( &expanded_program, - &expanded_args, + expanded_args, redirects, state, ) @@ -552,12 +548,7 @@ impl ExecutableCommand for Command { 127 }); - // Cleanup all process substitutions - for proc_sub in all_process_subs { - if let Err(e) = proc_sub.cleanup() { - eprintln!("Warning: Failed to cleanup process substitution: {}", e); - } - } + expansion.cleanup(); Ok(ExecutionResult::ExternalCommand { exit_code }) } @@ -569,33 +560,42 @@ impl ExecutableCommand for Command { redirects, background, } => { - // Expand variables and command substitutions in all pipeline stages - let expanded_stages: Result)>> = stages - .iter() - .map(|(program, args)| { - let expanded_program = - crate::parser::expand_with_command_sub(program, state)?; - let expanded_args: Result> = args - .iter() - .map(|arg| crate::parser::expand_with_command_sub(arg, state)) - .collect(); - Ok((expanded_program, expanded_args?)) - }) - .collect(); - let expanded_stages = expanded_stages?; - - if *background && !expanded_stages.is_empty() { + if *background && !stages.is_empty() { return Err(anyhow::anyhow!( "Background pipelines are not supported yet; refusing to run them in the foreground" )); } + // Expand every stage before any stage runs, so a failed + // expansion runs no part of the pipeline. + let mut expanded_stages: Vec<(String, Vec)> = Vec::new(); + let mut process_subs = Vec::new(); + for (program, args) in stages { + let expanded_program = crate::parser::expand_with_command_sub(program, state)?; + match crate::word::expand_words(args, state, crate::word::ProcessSubs::Allow) { + Ok(expansion) => { + expanded_stages.push((expanded_program, expansion.fields)); + process_subs.extend(expansion.process_subs); + } + Err(e) => { + eprintln!("vsh: {}: {}", expanded_program, e); + return Ok(ExecutionResult::ExternalCommand { exit_code: 1 }); + } + } + } + let exit_code = external::execute_pipeline(&expanded_stages, redirects, state) .unwrap_or_else(|e| { eprintln!("Pipeline error: {}", e); 127 }); + crate::word::Expansion { + fields: Vec::new(), + process_subs, + } + .cleanup(); + Ok(ExecutionResult::ExternalCommand { exit_code }) } @@ -684,10 +684,15 @@ impl ExecutableCommand for Command { interpret_escapes, redirects, } => { - let expanded_args: Vec = args - .iter() - .map(|a| crate::parser::expand_variables(a, state)) - .collect(); + let expansion = + match crate::word::expand_words(args, state, crate::word::ProcessSubs::Allow) { + Ok(expansion) => expansion, + Err(e) => { + eprintln!("vsh: echo: {}", e); + return Ok(ExecutionResult::ExternalCommand { exit_code: 1 }); + } + }; + let expanded_args = &expansion.fields; let output = if *interpret_escapes { expanded_args @@ -719,6 +724,7 @@ impl ExecutableCommand for Command { Ok(()) })?; } + expansion.cleanup(); Ok(ExecutionResult::Success) } @@ -1291,7 +1297,7 @@ impl ExecutableCommand for Command { if args.is_empty() { program.clone() } else { - format!("{} {}", program, args.join(" ")) + format!("{} {}", program, join_words(args)) } } Command::Pipeline { stages, .. } => { @@ -1301,7 +1307,7 @@ impl ExecutableCommand for Command { if args.is_empty() { prog.clone() } else { - format!("{} {}", prog, args.join(" ")) + format!("{} {}", prog, join_words(args)) } }) .collect(); @@ -1377,7 +1383,7 @@ impl ExecutableCommand for Command { format!("{} {} {}", left.description(), op_str, right.description()) } - Command::Echo { args, .. } => format!("echo {}", args.join(" ")), + Command::Echo { args, .. } => format!("echo {}", join_words(args)), Command::True => "true".to_string(), Command::False => "false".to_string(), Command::Read { var_names, .. } => format!("read {}", var_names.join(" ")), @@ -1645,6 +1651,15 @@ fn execute_loop( Ok(last_result) } +/// Words written back as source and separated by spaces. +fn join_words(words: &[crate::word::Word]) -> String { + words + .iter() + .map(|w| w.to_string()) + .collect::>() + .join(" ") +} + /// Match a word against a case pattern (supports glob-style wildcards) fn case_pattern_matches(word: &str, pattern: &str) -> bool { if pattern == "*" { diff --git a/impl/rust-cli/src/external.rs b/impl/rust-cli/src/external.rs index 60f345a..070e5f9 100644 --- a/impl/rust-cli/src/external.rs +++ b/impl/rust-cli/src/external.rs @@ -11,7 +11,6 @@ use std::path::PathBuf; use std::process::{Command, Stdio}; use std::time::Duration; -use crate::glob; use crate::redirection::{RedirectSetup, Redirection}; use crate::signals; use crate::state::ShellState; @@ -37,8 +36,7 @@ pub fn execute_external_with_redirects( let (stdin_cfg, stdout_cfg, stderr_cfg) = stdio_config_from_redirects(redirects, &redirect_setup, state)?; - // Expand glob patterns in arguments (Phase 6 M12) - let expanded_args = expand_glob_args(args, state)?; + // The arguments arrive already expanded, globs included (`crate::word`). // PATH lookup let executable = find_in_path(program)?; @@ -46,7 +44,7 @@ pub fn execute_external_with_redirects( // Build command with redirected stdio let mut command = Command::new(&executable); command - .args(&expanded_args) + .args(args) .stdin(stdin_cfg) .stdout(stdout_cfg) .stderr(stderr_cfg); @@ -475,80 +473,6 @@ fn stdin_from_bytes(content: &[u8], what: &str) -> Result { Ok(Stdio::from(reader)) } -/// Expand glob patterns in command arguments (Phase 6 M12) -/// -/// For each argument: -/// - If it contains glob metacharacters (*, ?, [, {), expand it -/// - If expansion finds matches, use them -/// - If expansion finds nothing, or the word is not a valid pattern, use the -/// word literally (POSIX behavior) -/// - If expansion stops at a limit, fail: the command is not run with a -/// literal or partial argument list in place of the real expansion -/// - If no metacharacters, use argument as-is -/// -/// # POSIX Behavior -/// - Brace expansion happens first: {a,b} -> [a, b] -/// - Then glob expansion: *.txt -> [file1.txt, file2.txt] -/// - Empty matches return literal: "*.xyz" if no .xyz files -/// -/// # Examples -/// ```ignore -/// expand_glob_args(&["echo", "*.txt"], state) // -> ["echo", "file1.txt", "file2.txt"] -/// expand_glob_args(&["ls", "file?.rs"], state) // -> ["ls", "file1.rs", "file2.rs"] -/// expand_glob_args(&["rm", "*.xyz"], state) // -> ["rm", "*.xyz"] (no .xyz files) -/// ``` -fn expand_glob_args(args: &[String], _state: &ShellState) -> Result> { - // Get current working directory for glob expansion - let cwd = std::env::current_dir().context("Failed to get current working directory")?; - expand_glob_args_in(args, &cwd, &glob::GlobLimits::DEFAULT) -} - -/// [`expand_glob_args`] with an explicit directory and limits. -/// -/// # Errors -/// Returns the [`glob::GlobLimitExceeded`] error of the first argument whose -/// expansion reaches a limit. -fn expand_glob_args_in( - args: &[String], - cwd: &std::path::Path, - limits: &glob::GlobLimits, -) -> Result> { - let mut expanded: Vec = Vec::new(); - - for arg in args { - // Check if argument contains glob metacharacters - if glob::contains_glob_pattern(arg) { - // First, expand braces: file{1,2}.txt -> [file1.txt, file2.txt] - let brace_expanded = glob::expand_braces(arg); - - // Then expand each brace result for glob patterns - for pattern in &brace_expanded { - // Expand glob pattern relative to current directory - match glob::expand_glob_with_limits(pattern, cwd, limits) { - Ok(expansion) if !expansion.paths.is_empty() => { - expanded.extend( - expansion - .paths - .iter() - .map(|path| path.to_string_lossy().to_string()), - ); - } - // A limit is not "no match": substituting the literal word - // would run the command on the wrong arguments. - Err(e) if e.is::() => return Err(e), - // No matches or not a valid pattern - use literal (POSIX behavior) - _ => expanded.push(pattern.clone()), - } - } - } else { - // No glob pattern - use argument as-is - expanded.push(arg.clone()); - } - } - - Ok(expanded) -} - /// Handle command exit status, converting signals to exit codes fn handle_exit_status(status: std::process::ExitStatus) -> Result { #[cfg(unix)] @@ -827,31 +751,4 @@ mod tests { let exit_code = execute_pipeline(&stages, &[], &mut state).unwrap(); assert_eq!(exit_code, 0, "Single-stage pipeline should work"); } - #[test] - fn glob_args_use_the_literal_word_when_nothing_matches() { - let dir = tempfile::TempDir::new().unwrap(); - std::fs::write(dir.path().join("a.txt"), b"").unwrap(); - let args = vec!["*.txt".to_string(), "*.none".to_string(), "[z-".to_string()]; - let out = expand_glob_args_in(&args, dir.path(), &glob::GlobLimits::DEFAULT).unwrap(); - assert_eq!(out, ["a.txt", "*.none", "[z-"]); - } - - #[test] - fn glob_args_fail_instead_of_substituting_the_literal_at_a_limit() { - let dir = tempfile::TempDir::new().unwrap(); - for i in 0..5 { - std::fs::write(dir.path().join(format!("f{i}.txt")), b"").unwrap(); - } - let limits = glob::GlobLimits { - max_matches: 4, - ..glob::GlobLimits::DEFAULT - }; - let args = vec!["-l".to_string(), "*.txt".to_string()]; - let err = expand_glob_args_in(&args, dir.path(), &limits).unwrap_err(); - let stopped = err - .downcast_ref::() - .expect("the limit error must reach the caller"); - assert_eq!(stopped.exceeded.axis, "matches held"); - assert!(err.to_string().contains("over the limit of 4"), "{err}"); - } } diff --git a/impl/rust-cli/src/lib.rs b/impl/rust-cli/src/lib.rs index fd5fa34..e74b4b1 100644 --- a/impl/rust-cli/src/lib.rs +++ b/impl/rust-cli/src/lib.rs @@ -40,6 +40,7 @@ pub mod state; pub mod test_command; pub mod verification; +pub mod word; /// Echidna property-based verification integration (optional feature) #[cfg(feature = "echidna")] diff --git a/impl/rust-cli/src/parser.rs b/impl/rust-cli/src/parser.rs index da677ce..1b51f2d 100644 --- a/impl/rust-cli/src/parser.rs +++ b/impl/rust-cli/src/parser.rs @@ -9,6 +9,7 @@ use anyhow::{anyhow, Context, Result}; use crate::redirection::Redirection; +use crate::word::{Quote, Word}; /// Quote type for a word or word part #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -82,6 +83,8 @@ struct ParameterExpansion { struct QuotedWord { parts: Vec, quote_type: QuoteType, + /// The same word with the quoting of each piece kept (see `crate::word`). + typed: Word, } impl QuotedWord { @@ -89,6 +92,7 @@ impl QuotedWord { Self { parts: Vec::new(), quote_type: QuoteType::None, + typed: Word::new(), } } @@ -102,19 +106,23 @@ impl QuotedWord { } } - fn push_variable(&mut self, name: String) { + fn push_variable(&mut self, name: String, quote: Quote) { + self.typed.push_parameter(name.clone(), false, quote); self.parts.push(WordPart::Variable(name)); } - fn push_braced_variable(&mut self, name: String) { + fn push_braced_variable(&mut self, name: String, quote: Quote) { + self.typed.push_parameter(name.clone(), true, quote); self.parts.push(WordPart::BracedVariable(name)); } - fn push_command_sub(&mut self, cmd: String) { + fn push_command_sub(&mut self, cmd: String, quote: Quote) { + self.typed.push_command_sub(cmd.clone(), quote); self.parts.push(WordPart::CommandSub(cmd)); } fn push_process_sub(&mut self, sub_type: ProcessSubType, cmd: String) { + self.typed.push_process_sub(sub_type, cmd.clone()); self.parts.push(WordPart::ProcessSub(sub_type, cmd)); } } @@ -285,7 +293,8 @@ pub enum Command { // Shell builtins /// Echo: write arguments to stdout Echo { - args: Vec, + /// Arguments after the options, with their quoting. + args: Vec, no_newline: bool, interpret_escapes: bool, redirects: Vec, @@ -335,7 +344,8 @@ pub enum Command { // External command External { program: String, - args: Vec, + /// Arguments with their quoting; expanded by [`crate::word::expand_words`]. + args: Vec, redirects: Vec, background: bool, }, @@ -345,7 +355,7 @@ pub enum Command { /// Each stage is a (program, args) pair. Intermediate stages use piped stdio. /// Final redirections apply to the last stage only. Pipeline { - stages: Vec<(String, Vec)>, + stages: Vec<(String, Vec)>, redirects: Vec, background: bool, }, @@ -599,6 +609,7 @@ fn tokenize(input: &str) -> Result> { current_literal.push('\\'); } current_literal.push(ch); + current_word.typed.push_char(ch, Quote::Backslash); quote_state = QuoteState::None; } @@ -609,6 +620,7 @@ fn tokenize(input: &str) -> Result> { quote_state = QuoteState::None; } else { current_literal.push(ch); + current_word.typed.push_char(ch, Quote::Single); } } @@ -631,27 +643,31 @@ fn tokenize(input: &str) -> Result> { current_literal.push('\\'); } current_literal.push(next_ch); + current_word.typed.push_char(next_ch, Quote::Double); } else { // Not a special char, keep backslash current_literal.push('\\'); + current_word.typed.push_char('\\', Quote::Double); } } else { current_literal.push('\\'); + current_word.typed.push_char('\\', Quote::Double); } } '$' => { // Variable expansion in double quotes push_literal!(); - parse_variable(&mut chars, &mut current_word)?; + parse_variable(&mut chars, &mut current_word, Quote::Double)?; } '`' => { // Backtick command substitution in double quotes push_literal!(); let cmd = parse_command_sub_backtick(&mut chars)?; - current_word.push_command_sub(cmd); + current_word.push_command_sub(cmd, Quote::Double); } _ => { current_literal.push(ch); + current_word.typed.push_char(ch, Quote::Double); } } } @@ -678,14 +694,14 @@ fn tokenize(input: &str) -> Result> { // Variable expansion '$' => { push_literal!(); - parse_variable(&mut chars, &mut current_word)?; + parse_variable(&mut chars, &mut current_word, Quote::None)?; } // Backtick command substitution '`' => { push_literal!(); let cmd = parse_command_sub_backtick(&mut chars)?; - current_word.push_command_sub(cmd); + current_word.push_command_sub(cmd, Quote::None); } // Whitespace: end current word @@ -766,6 +782,7 @@ fn tokenize(input: &str) -> Result> { } else { // Invalid: 2>&[not 1] current_literal.push_str("2>&"); + current_word.typed.push_str("2>&", Quote::None); } } _ => { @@ -775,6 +792,7 @@ fn tokenize(input: &str) -> Result> { } else { // Regular '2' character, part of word current_literal.push(ch); + current_word.typed.push_char(ch, Quote::None); } } @@ -816,6 +834,7 @@ fn tokenize(input: &str) -> Result> { } else { // Regular '[' character, part of word current_literal.push(ch); + current_word.typed.push_char(ch, Quote::None); } } @@ -828,12 +847,14 @@ fn tokenize(input: &str) -> Result> { } else { // Regular ']' character, part of word current_literal.push(ch); + current_word.typed.push_char(ch, Quote::None); } } // Regular character _ => { current_literal.push(ch); + current_word.typed.push_char(ch, Quote::None); } } } @@ -861,12 +882,13 @@ fn tokenize(input: &str) -> Result> { fn parse_variable( chars: &mut std::iter::Peekable, word: &mut QuotedWord, + quote: Quote, ) -> Result<()> { if chars.peek() == Some(&'(') { // Command substitution: $(cmd) chars.next(); // consume '(' let cmd = parse_command_sub_dollar(chars)?; - word.push_command_sub(cmd); + word.push_command_sub(cmd, quote); } else if chars.peek() == Some(&'{') { // Braced form: ${VAR} chars.next(); // consume '{' @@ -888,7 +910,7 @@ fn parse_variable( } } - word.push_braced_variable(var_name); + word.push_braced_variable(var_name, quote); } else if let Some(&next_ch) = chars.peek() { // Simple form: $VAR or special variables. The peek above already // bound next_ch by value; chars.next() is called purely to advance @@ -897,11 +919,11 @@ fn parse_variable( if next_ch == '?' || next_ch == '$' || next_ch == '#' { // Single-character special variable chars.next(); - word.push_variable(next_ch.to_string()); + word.push_variable(next_ch.to_string(), quote); } else if next_ch.is_ascii_digit() { // Positional parameter: $0, $1, $2, etc. chars.next(); - word.push_variable(next_ch.to_string()); + word.push_variable(next_ch.to_string(), quote); } else if next_ch.is_alphabetic() || next_ch == '_' { // Variable name let mut var_name = String::new(); @@ -915,18 +937,20 @@ fn parse_variable( } } - word.push_variable(var_name); + word.push_variable(var_name, quote); } else if next_ch == '@' || next_ch == '*' { // Special positional parameters: $@ or $* chars.next(); - word.push_variable(next_ch.to_string()); + word.push_variable(next_ch.to_string(), quote); } else { // $ not followed by variable, treat as literal word.push_literal("$".to_string()); + word.typed.push_char('$', quote); } } else { // $ at end of string word.push_literal("$".to_string()); + word.typed.push_char('$', quote); } Ok(()) @@ -935,15 +959,14 @@ fn parse_variable( /// Parse command substitution in $(cmd) form fn parse_command_sub_dollar(chars: &mut std::iter::Peekable) -> Result { let mut cmd = String::new(); - let mut depth = 1; // Track nesting depth for nested $() + let mut depth = 1; // Parenthesis depth: nested $(), $((expr)) and (...) for ch in chars.by_ref() { match ch { '(' => { - // Check if it's $( to track nested command substitution - if cmd.ends_with('$') { - depth += 1; - } + // Every `(` nests, so the `)` that closes `$((1+2))` or + // `$(echo $(pwd))` belongs to the substitution. + depth += 1; cmd.push(ch); } ')' => { @@ -1228,28 +1251,28 @@ fn parse_pipeline(tokens: &[Token]) -> Result { stages[last_stage_idx] = last_command_tokens; // Parse each stage as (program, args) - let mut parsed_stages: Vec<(String, Vec)> = Vec::new(); + let mut parsed_stages: Vec<(String, Vec)> = Vec::new(); for stage_tokens in stages { if stage_tokens.is_empty() { return Err(anyhow!("Empty pipeline stage")); } - let words: Vec = stage_tokens + let words: Vec<&QuotedWord> = stage_tokens .iter() .filter_map(|t| match t { - Token::Word(w) => Some(quoted_word_to_string(w)), + Token::Word(w) => Some(w), _ => None, }) .collect(); - if words.is_empty() { + let Some((program, args)) = words.split_first() else { return Err(anyhow!("Pipeline stage has no command")); - } - - let program = words[0].clone(); - let args = words[1..].to_vec(); - parsed_stages.push((program, args)); + }; + parsed_stages.push(( + quoted_word_to_string(program), + args.iter().map(|w| w.typed.clone()).collect(), + )); } Ok(Command::Pipeline { @@ -1719,6 +1742,7 @@ pub fn parse_command(input: &str) -> Result { // Separate command tokens from redirections let mut command_tokens: Vec = Vec::new(); + let mut command_words: Vec = Vec::new(); let mut redirects = Vec::new(); let mut background = false; @@ -1727,6 +1751,7 @@ pub fn parse_command(input: &str) -> Result { match &tokens[i] { Token::Word(w) => { command_tokens.push(quoted_word_to_string(w)); + command_words.push(w.typed.clone()); i += 1; } @@ -1852,9 +1877,10 @@ pub fn parse_command(input: &str) -> Result { // Parse command from tokens let cmd = &command_tokens[0]; let args: Vec = command_tokens[1..].to_vec(); + let words: Vec = command_words[1..].to_vec(); // Parse base command with redirections - parse_base_command(cmd, args, redirects, background) + parse_base_command(cmd, args, words, redirects, background) } /// Check if a string is a valid variable name @@ -2369,7 +2395,7 @@ fn apply_substring(value: &str, offset: i32, length: Option) -> String { /// /// `~user` is not yet supported (requires getpwnam). An escaped `\~` /// (produced by `quoted_word_to_string` for quoted `~`) stays literal. -fn expand_tilde(input: &str) -> String { +pub(crate) fn expand_tilde(input: &str) -> String { // An escaped tilde `\~` means the tilde was inside quotes — leave it. if let Some(stripped) = input.strip_prefix("\\~") { let mut s = String::with_capacity(input.len()); @@ -2628,14 +2654,12 @@ pub fn expand_command_substitution( process_cmd.stdout(Stdio::piped()); process_cmd.stderr(Stdio::null()); - // Expand variables and command substitutions in args before execution - let expanded_args: Result> = args - .iter() - .map(|arg| expand_with_command_sub(arg, state)) - .collect(); - let expanded_args = expanded_args?; + // Expand the arguments (variables, command substitution, globs) + // before execution. + let expansion = + crate::word::expand_words(args, state, crate::word::ProcessSubs::Refuse)?; - process_cmd.args(&expanded_args); + process_cmd.args(&expansion.fields); let output_result = process_cmd .output() @@ -2651,6 +2675,22 @@ pub fn expand_command_substitution( } } + Command::Echo { + args, + no_newline, + interpret_escapes: false, + redirects, + } if redirects.is_empty() => { + // echo builtin: its output is the joined, expanded arguments. + let expansion = + crate::word::expand_words(args, state, crate::word::ProcessSubs::Refuse)?; + let mut out = expansion.fields.join(" "); + if !no_newline { + out.push('\n'); + } + out + } + Command::Pwd { .. } => { // pwd builtin std::env::current_dir() @@ -3253,9 +3293,13 @@ pub fn is_incomplete_control_structure(input: &str) -> bool { block_depth > 0 } +/// `args` are the arguments in the older string encoding, which the builtins +/// still read; `words` are the same arguments with their quoting, used by +/// `echo` and external commands. The two lists have the same length. fn parse_base_command( cmd: &str, args: Vec, + words: Vec, redirects: Vec, background: bool, ) -> Result { @@ -3382,7 +3426,8 @@ fn parse_base_command( // Re-parse the inner command: first arg is the command name, rest are its args let inner_cmd = &args[0]; let inner_args = args[1..].to_vec(); - let inner = parse_base_command(inner_cmd, inner_args, vec![], false)?; + let inner_words = words[1..].to_vec(); + let inner = parse_base_command(inner_cmd, inner_args, inner_words, vec![], false)?; Ok(Command::Explain { inner: Box::new(inner), }) @@ -3541,7 +3586,7 @@ fn parse_base_command( let mut echo_args = Vec::new(); let mut skip_flags = true; - for arg in &args { + for (arg, word) in args.iter().zip(&words) { if skip_flags && arg.starts_with('-') { for ch in arg[1..].chars() { match ch { @@ -3551,14 +3596,14 @@ fn parse_base_command( _ => { // Unknown flag — treat as regular argument skip_flags = false; - echo_args.push(arg.clone()); + echo_args.push(word.clone()); break; } } } } else { skip_flags = false; - echo_args.push(arg.clone()); + echo_args.push(word.clone()); } } @@ -3729,7 +3774,7 @@ fn parse_base_command( // Everything else: external command _ => Ok(Command::External { program: cmd.to_string(), - args, + args: words, redirects, background, }), @@ -4246,7 +4291,7 @@ mod tests { match cmd { Command::External { program, args, .. } => { assert_eq!(program, "touch"); - let expanded = expand_variables(&args[0], &state); + let expanded = expand_variables(&args[0].to_string(), &state); assert_eq!(expanded, "file_Alice"); } Command::Touch { path, .. } => { @@ -5313,3 +5358,78 @@ mod semicolon_tests { assert_eq!(parts, vec!["", ""]); } } + +#[cfg(test)] +mod typed_word_tests { + use super::*; + use proptest::prelude::*; + + fn typed_words(input: &str) -> Vec { + tokenize(input) + .unwrap() + .into_iter() + .filter_map(|t| match t { + Token::Word(w) => Some(w.typed), + _ => None, + }) + .collect() + } + + fn word(parts: &[(&str, Quote)]) -> Word { + let mut w = Word::new(); + for (text, quote) in parts { + w.push_str(text, *quote); + } + w + } + + #[test] + fn the_tokenizer_records_the_quoting_of_each_piece() { + assert_eq!( + typed_words("echo '*'x\\?\"a $V\"*"), + vec![Word::literal("echo"), { + let mut w = word(&[("*", Quote::Single), ("x", Quote::None)]); + w.push_char('?', Quote::Backslash); + w.push_str("a ", Quote::Double); + w.push_parameter("V".into(), false, Quote::Double); + w.push_char('*', Quote::None); + w + }] + ); + } + + #[test] + fn arithmetic_expansion_is_one_command_substitution_piece() { + let mut expected = Word::new(); + expected.push_command_sub("(1+(2*3))".into(), Quote::None); + expected.push_char('x', Quote::None); + assert_eq!(typed_words("$((1+(2*3)))x"), vec![expected]); + } + + /// One literal piece: (text, quoting). Characters are limited to those + /// the tokenizer reads literally in each quoting. + fn piece() -> impl Strategy { + prop_oneof![ + "[a-z0-9._/*?-]{1,4}".prop_map(|s| (s, Quote::None)), + "[a-z *?$\"\\\\]{1,4}".prop_map(|s| (s, Quote::Single)), + "[a-z '*?$\"\\\\]{1,4}".prop_map(|s| (s, Quote::Double)), + "[a-z*?$'\"\\\\ ]".prop_map(|s| (s, Quote::Backslash)), + ] + } + + proptest! { + #![proptest_config(ProptestConfig::with_cases(256))] + + /// `Display` writes a word as source that tokenizes back to the + /// same word, so the quoting survives a round trip. + #[test] + fn display_then_tokenize_gives_the_same_word(pieces in prop::collection::vec(piece(), 1..6)) { + let mut w = Word::new(); + for (text, quote) in &pieces { + w.push_str(text, *quote); + } + let source = w.to_string(); + prop_assert_eq!(typed_words(&source), vec![w], "source: {}", source); + } + } +} diff --git a/impl/rust-cli/src/process_sub.rs b/impl/rust-cli/src/process_sub.rs index 7387952..131eee3 100644 --- a/impl/rust-cli/src/process_sub.rs +++ b/impl/rust-cli/src/process_sub.rs @@ -157,11 +157,8 @@ fn start_command_with_output_redirect( let cmd_string = match cmd { Command::External { program, args, .. } => { let expanded_program = crate::parser::expand_with_command_sub(program, state)?; - let expanded_args: Result> = args - .iter() - .map(|arg| crate::parser::expand_with_command_sub(arg, state)) - .collect(); - let expanded_args = expanded_args?; + let expanded_args = + crate::word::expand_words(args, state, crate::word::ProcessSubs::Refuse)?.fields; // Build command string: prog arg1 arg2 > fifo let mut cmd = expanded_program; @@ -210,11 +207,8 @@ fn start_command_with_input_redirect( let cmd_string = match cmd { Command::External { program, args, .. } => { let expanded_program = crate::parser::expand_with_command_sub(program, state)?; - let expanded_args: Result> = args - .iter() - .map(|arg| crate::parser::expand_with_command_sub(arg, state)) - .collect(); - let expanded_args = expanded_args?; + let expanded_args = + crate::word::expand_words(args, state, crate::word::ProcessSubs::Refuse)?.fields; // Build command string let mut cmd = expanded_program; diff --git a/impl/rust-cli/src/word.rs b/impl/rust-cli/src/word.rs new file mode 100644 index 0000000..98fc8fc --- /dev/null +++ b/impl/rust-cli/src/word.rs @@ -0,0 +1,667 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright (c) Jonathan D.A. Jewell +//! Typed shell words and the word-expansion stage for simple-command arguments. +//! +//! The tokenizer used to flatten each word into a `String` in which quoted +//! glob metacharacters were marked with a backslash (`'*'` became `\*`). +//! Nothing removed those backslashes again, so `echo '*.txt'` printed +//! `\*.txt`, and glob expansion could not tell a quoted `*` from an unquoted +//! one without re-parsing that encoding. +//! +//! A [`Word`] keeps the quoting of every piece instead: each [`Segment`] +//! records whether it was written unquoted, in single or double quotes, or +//! after a backslash. [`expand_words`] then runs the POSIX word-expansion +//! steps on that structure, in this order: +//! +//! 1. tilde expansion (an unquoted `~`, `~/`, `~+`, `~-` at the start), +//! parameter expansion, command substitution, arithmetic expansion and +//! process substitution; +//! 2. brace expansion (`{a,b}`) of unquoted literal text; +//! 3. pathname (glob) expansion of the characters that are not quoted; +//! 4. quote removal. +//! +//! # Rules kept from the previous expansion code +//! +//! - A pattern that matches nothing, or is not a valid pattern, stays as the +//! literal word. +//! - A pattern whose expansion reaches a [`GlobLimits`] limit fails the whole +//! expansion with the [`GlobLimitExceeded`] error; the caller must not run +//! the command. +//! +//! # Known differences from POSIX (not changed here) +//! +//! - No field splitting: an unquoted `$X` whose value is `a b` stays one +//! argument. +//! - An empty word (`''`, or an unquoted expansion that is empty) is dropped +//! by the tokenizer or kept as an empty argument, as before. +//! - Brace expansion runs after parameter expansion (bash runs it before); +//! braces that come out of an expansion are not expanded. +//! +//! Only the arguments of external commands, pipeline stages, `echo` and +//! function calls go through this stage so far. Builtin operands, redirection +//! targets, assignments, `case` patterns and `test` arguments still use the +//! older string encoding. + +use std::fmt; +use std::path::Path; + +use anyhow::{Context, Result}; + +use crate::glob::{self, GlobLimitExceeded, GlobLimits}; +use crate::parser::ProcessSubType; +use crate::process_sub::ProcessSubstitution; +use crate::state::ShellState; + +/// How a piece of a word was written in the source. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Quote { + /// Not quoted: subject to tilde, brace and glob expansion. + None, + /// Inside `'...'`. + Single, + /// Inside `"..."`. + Double, + /// A single character after an unquoted `\`. + Backslash, +} + +impl Quote { + fn is_quoted(self) -> bool { + self != Quote::None + } +} + +/// One piece of a [`Word`]. +#[derive(Debug, Clone, PartialEq)] +pub enum Segment { + /// Literal text. + Literal { text: String, quote: Quote }, + /// `$NAME` (`braced: false`) or `${EXPR}` (`braced: true`). + Parameter { + expr: String, + braced: bool, + quote: Quote, + }, + /// `$(CMD)` or `` `CMD` ``; `$((EXPR))` is stored as the command `(EXPR)`. + CommandSub { cmd: String, quote: Quote }, + /// `<(CMD)` or `>(CMD)`. + ProcessSub { kind: ProcessSubType, cmd: String }, +} + +/// A shell word with the quoting of each of its pieces. +#[derive(Debug, Clone, PartialEq, Default)] +pub struct Word { + segments: Vec, +} + +impl Word { + /// An empty word. + pub fn new() -> Self { + Self::default() + } + + /// A word that is a single unquoted literal. + /// + /// # Examples + /// ``` + /// use vsh::word::Word; + /// assert_eq!(Word::literal("a.txt").to_string(), "a.txt"); + /// ``` + pub fn literal(text: &str) -> Self { + let mut word = Self::new(); + for ch in text.chars() { + word.push_char(ch, Quote::None); + } + word + } + + /// The pieces of the word, in order. + pub fn segments(&self) -> &[Segment] { + &self.segments + } + + /// True when the word has no pieces. + pub fn is_empty(&self) -> bool { + self.segments.is_empty() + } + + /// Append one literal character, merging it into the last piece when that + /// piece is a literal with the same quoting. + pub fn push_char(&mut self, ch: char, quote: Quote) { + if quote != Quote::Backslash { + if let Some(Segment::Literal { text, quote: last }) = self.segments.last_mut() { + if *last == quote { + text.push(ch); + return; + } + } + } + self.segments.push(Segment::Literal { + text: ch.to_string(), + quote, + }); + } + + /// Append literal text. + pub fn push_str(&mut self, text: &str, quote: Quote) { + for ch in text.chars() { + self.push_char(ch, quote); + } + } + + /// Append a parameter expansion. + pub fn push_parameter(&mut self, expr: String, braced: bool, quote: Quote) { + self.segments.push(Segment::Parameter { + expr, + braced, + quote, + }); + } + + /// Append a command substitution. + pub fn push_command_sub(&mut self, cmd: String, quote: Quote) { + self.segments.push(Segment::CommandSub { cmd, quote }); + } + + /// Append a process substitution. + pub fn push_process_sub(&mut self, kind: ProcessSubType, cmd: String) { + self.segments.push(Segment::ProcessSub { kind, cmd }); + } +} + +impl From<&str> for Word { + fn from(text: &str) -> Self { + Word::literal(text) + } +} + +/// Write one literal piece back as source with its quoting. +fn fmt_literal(f: &mut fmt::Formatter<'_>, text: &str, quote: Quote) -> fmt::Result { + match quote { + Quote::None => f.write_str(text), + Quote::Backslash => text.chars().try_for_each(|ch| write!(f, "\\{ch}")), + // `'` cannot appear inside single quotes. + Quote::Single => write!(f, "'{}'", text.replace('\'', "'\\''")), + Quote::Double => { + let mut escaped = String::with_capacity(text.len() + 2); + for ch in text.chars() { + if matches!(ch, '"' | '$' | '`' | '\\') { + escaped.push('\\'); + } + escaped.push(ch); + } + write!(f, "\"{escaped}\"") + } + } +} + +/// Writes the word back as shell source that parses to the same word. +impl fmt::Display for Word { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for segment in &self.segments { + match segment { + Segment::Literal { text, quote } => fmt_literal(f, text, *quote)?, + Segment::Parameter { + expr, + braced, + quote, + } => { + let open = if quote.is_quoted() { "\"" } else { "" }; + if *braced { + write!(f, "{open}${{{expr}}}{open}")?; + } else { + write!(f, "{open}${expr}{open}")?; + } + } + Segment::CommandSub { cmd, quote } => { + let open = if quote.is_quoted() { "\"" } else { "" }; + write!(f, "{open}$({cmd}){open}")?; + } + Segment::ProcessSub { kind, cmd } => match kind { + ProcessSubType::Input => write!(f, "<({cmd})")?, + ProcessSubType::Output => write!(f, ">({cmd})")?, + }, + } + } + Ok(()) + } +} + +/// Compares the source form ([`fmt::Display`]) with `other`. +impl PartialEq for Word { + fn eq(&self, other: &str) -> bool { + let source = self.to_string(); + source.as_str() == other + } +} + +impl PartialEq<&str> for Word { + fn eq(&self, other: &&str) -> bool { + *self == **other + } +} + +/// Whether process substitutions may be started while expanding. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ProcessSubs { + /// Start them; the caller cleans them up after the command finishes. + Allow, + /// Fail on them, as the code paths without FIFO handling always have. + Refuse, +} + +/// The result of expanding a list of words. +pub struct Expansion { + /// The arguments, after quote removal. + pub fields: Vec, + /// Process substitutions started for these arguments. + pub process_subs: Vec, +} + +impl Expansion { + /// Wait for the process substitutions and remove their FIFOs. + pub fn cleanup(self) { + for proc_sub in self.process_subs { + if let Err(e) = proc_sub.cleanup() { + eprintln!("Warning: Failed to cleanup process substitution: {}", e); + } + } + } +} + +/// Expand words relative to the current directory under the default limits. +/// +/// # Errors +/// Fails on an expansion error (a failing command substitution, a refused +/// process substitution) or when a glob reaches a limit +/// ([`GlobLimitExceeded`], reachable with `anyhow::Error::downcast_ref`). +pub fn expand_words( + words: &[Word], + state: &mut ShellState, + process_subs: ProcessSubs, +) -> Result { + let cwd = std::env::current_dir().context("Failed to get current working directory")?; + expand_words_in(words, state, process_subs, &cwd, &GlobLimits::DEFAULT) +} + +/// [`expand_words`] with an explicit directory for relative patterns and +/// explicit glob limits. +/// +/// # Errors +/// As for [`expand_words`]. +pub fn expand_words_in( + words: &[Word], + state: &mut ShellState, + process_subs: ProcessSubs, + cwd: &Path, + limits: &GlobLimits, +) -> Result { + let mut expansion = Expansion { + fields: Vec::new(), + process_subs: Vec::new(), + }; + for word in words { + // On error the substitutions started so far are dropped, and Drop + // stops them and removes their FIFOs. + let chars = expand_segments(word, state, process_subs, &mut expansion.process_subs)?; + glob_fields(&chars, cwd, limits, &mut expansion.fields)?; + } + Ok(expansion) +} + +/// What later steps may do with a character produced by step 1. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Role { + /// Quoted text or a tilde result: taken literally. + Literal, + /// Unquoted source text: brace and glob expansion apply. + Source, + /// Result of an unquoted expansion: glob expansion applies, braces do not. + Expanded, +} + +/// Step 1: tilde, parameter, command, arithmetic and process substitution. +fn expand_segments( + word: &Word, + state: &mut ShellState, + process_subs: ProcessSubs, + started: &mut Vec, +) -> Result> { + let mut out: Vec<(char, Role)> = Vec::new(); + let push = |out: &mut Vec<(char, Role)>, text: &str, role: Role| { + out.extend(text.chars().map(|ch| (ch, role))); + }; + + for (index, segment) in word.segments.iter().enumerate() { + match segment { + Segment::Literal { text, quote } => { + if !quote.is_quoted() { + if index == 0 && text.starts_with('~') { + let (prefix, rest) = text.split_at(text.find('/').unwrap_or(text.len())); + let expanded = crate::parser::expand_tilde(prefix); + if expanded != prefix { + push(&mut out, &expanded, Role::Literal); + push(&mut out, rest, Role::Source); + continue; + } + } + push(&mut out, text, Role::Source); + } else { + push(&mut out, text, Role::Literal); + } + } + Segment::Parameter { + expr, + braced, + quote, + } => { + let source = if *braced { + format!("${{{expr}}}") + } else { + format!("${expr}") + }; + let value = crate::parser::expand_variables(&source, state); + push(&mut out, &value, expansion_role(*quote)); + } + Segment::CommandSub { cmd, quote } => { + let value = crate::parser::expand_with_command_sub(&format!("$({cmd})"), state)?; + push(&mut out, &value, expansion_role(*quote)); + } + Segment::ProcessSub { kind, cmd } => { + if process_subs == ProcessSubs::Refuse { + anyhow::bail!( + "Process substitution not yet implemented: execution requires FIFO support ({})", + Word { + segments: vec![segment.clone()] + } + ); + } + let mut proc_sub = ProcessSubstitution::create(*kind, cmd.clone(), state)?; + proc_sub.start(state)?; + push( + &mut out, + &proc_sub.fifo_path.to_string_lossy(), + Role::Literal, + ); + started.push(proc_sub); + } + } + } + Ok(out) +} + +fn expansion_role(quote: Quote) -> Role { + if quote.is_quoted() { + Role::Literal + } else { + Role::Expanded + } +} + +/// Encode step-1 output for [`glob::expand_braces`] and +/// [`glob::contains_glob_pattern`], which both read `\x` as a literal `x`. +fn encode(chars: &[(char, Role)]) -> String { + let mut out = String::with_capacity(chars.len() * 2); + for &(ch, role) in chars { + let literal = match role { + Role::Literal => true, + Role::Source => ch == '\\', + Role::Expanded => matches!(ch, '\\' | '{' | '}' | ','), + }; + if literal { + out.push('\\'); + } + out.push(ch); + } + out +} + +/// Steps 2 to 4: brace expansion, glob expansion and quote removal. +fn glob_fields( + chars: &[(char, Role)], + cwd: &Path, + limits: &GlobLimits, + fields: &mut Vec, +) -> Result<()> { + let encoded = encode(chars); + for candidate in glob::expand_braces(&encoded) { + match glob_pattern(&candidate) { + None => fields.push(remove_quotes(&candidate)), + Some(pattern) => match glob::expand_glob_with_limits(&pattern, cwd, limits) { + Ok(found) if !found.paths.is_empty() => fields.extend( + found + .paths + .iter() + .map(|path| path.to_string_lossy().into_owned()), + ), + // A limit is not "no match": substituting the literal word + // would run the command on the wrong arguments. + Err(e) if e.is::() => return Err(e), + // No match, or not a valid pattern: keep the word (POSIX). + _ => fields.push(remove_quotes(&candidate)), + }, + } + } + Ok(()) +} + +/// The pattern for the `glob` crate, or `None` when no `*`, `?` or `[` is +/// left unquoted. Quoted metacharacters are written as one-character classes +/// (`[*]`), the only escape that crate has. +fn glob_pattern(encoded: &str) -> Option { + let mut pattern = String::with_capacity(encoded.len()); + let mut active = false; + let mut chars = encoded.chars(); + while let Some(ch) = chars.next() { + if ch == '\\' { + if let Some(next) = chars.next() { + pattern.push_str(&::glob::Pattern::escape(&next.to_string())); + } + } else { + active |= matches!(ch, '*' | '?' | '['); + pattern.push(ch); + } + } + active.then_some(pattern) +} + +/// Quote removal: `\x` becomes `x`. +fn remove_quotes(encoded: &str) -> String { + let mut out = String::with_capacity(encoded.len()); + let mut chars = encoded.chars(); + while let Some(ch) = chars.next() { + if ch == '\\' { + if let Some(next) = chars.next() { + out.push(next); + } + } else { + out.push(ch); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn word(parts: &[(&str, Quote)]) -> Word { + let mut w = Word::new(); + for (text, quote) in parts { + w.push_str(text, *quote); + } + w + } + + fn fixture(names: &[&str]) -> tempfile::TempDir { + let dir = tempfile::TempDir::new().unwrap(); + for name in names { + std::fs::write(dir.path().join(name), b"").unwrap(); + } + dir + } + + fn expand_in(dir: &Path, words: &[Word]) -> Vec { + let mut state = ShellState::new(dir.to_str().unwrap()).unwrap(); + expand_words_in( + words, + &mut state, + ProcessSubs::Refuse, + dir, + &GlobLimits::DEFAULT, + ) + .unwrap() + .fields + } + + #[test] + fn unquoted_patterns_expand_and_quoted_ones_do_not() { + let dir = fixture(&["a.txt", "b.txt", "c.log"]); + let out = expand_in( + dir.path(), + &[ + Word::literal("*.txt"), + word(&[("*.txt", Quote::Single)]), + word(&[("*.txt", Quote::Double)]), + word(&[("*", Quote::Backslash), (".txt", Quote::None)]), + ], + ); + assert_eq!(out, ["a.txt", "b.txt", "*.txt", "*.txt", "*.txt"]); + } + + #[test] + fn a_partly_quoted_pattern_matches_its_quoted_part_literally() { + let dir = fixture(&["*x.txt", "ax.txt", "a.txt"]); + // `'*'x*` : a literal star followed by `x` and a wildcard. + let out = expand_in( + dir.path(), + &[word(&[("*", Quote::Single), ("x*", Quote::None)])], + ); + assert_eq!(out, ["*x.txt"]); + } + + #[test] + fn no_match_and_invalid_patterns_stay_literal() { + let dir = fixture(&["a.txt"]); + let out = expand_in( + dir.path(), + &[ + Word::literal("*.txt"), + Word::literal("*.none"), + Word::literal("[z-"), + ], + ); + assert_eq!(out, ["a.txt", "*.none", "[z-"]); + } + + #[test] + fn braces_expand_only_when_unquoted() { + let dir = fixture(&[]); + let out = expand_in( + dir.path(), + &[Word::literal("a{1,2}"), word(&[("a{1,2}", Quote::Single)])], + ); + assert_eq!(out, ["a1", "a2", "a{1,2}"]); + } + + #[test] + fn literal_backslashes_survive_quote_removal() { + let dir = fixture(&["a.txt"]); + let out = expand_in( + dir.path(), + &[ + word(&[("a\\*", Quote::Single)]), + word(&[("\\", Quote::Backslash)]), + ], + ); + assert_eq!(out, ["a\\*", "\\"]); + } + + #[test] + fn a_quoted_backslash_is_a_literal_character_in_a_pattern() { + let dir = fixture(&["a\\b.txt", "ab.txt"]); + // `'a\'*` : names that start with `a\`. + let out = expand_in( + dir.path(), + &[word(&[("a\\", Quote::Single), ("*", Quote::None)])], + ); + assert_eq!(out, ["a\\b.txt"]); + } + + #[test] + fn unquoted_parameter_values_are_globbed_and_quoted_ones_are_not() { + let dir = fixture(&["a.txt", "b.txt"]); + let mut state = ShellState::new(dir.path().to_str().unwrap()).unwrap(); + state.set_variable("P", "*.txt"); + let mut unquoted = Word::new(); + unquoted.push_parameter("P".into(), false, Quote::None); + let mut quoted = Word::new(); + quoted.push_parameter("P".into(), true, Quote::Double); + let out = expand_words_in( + &[unquoted, quoted], + &mut state, + ProcessSubs::Refuse, + dir.path(), + &GlobLimits::DEFAULT, + ) + .unwrap() + .fields; + assert_eq!(out, ["a.txt", "b.txt", "*.txt"]); + } + + #[test] + fn a_glob_limit_fails_the_expansion_instead_of_substituting_the_literal() { + let dir = fixture(&["f0.txt", "f1.txt", "f2.txt", "f3.txt", "f4.txt"]); + let mut state = ShellState::new(dir.path().to_str().unwrap()).unwrap(); + let limits = GlobLimits { + max_matches: 4, + ..GlobLimits::DEFAULT + }; + let words = [Word::literal("-l"), Word::literal("*.txt")]; + let err = expand_words_in(&words, &mut state, ProcessSubs::Refuse, dir.path(), &limits) + .err() + .expect("the limit must fail the expansion"); + let stopped = err + .downcast_ref::() + .expect("the limit error must reach the caller"); + assert_eq!(stopped.exceeded.axis, "matches held"); + assert!(err.to_string().contains("over the limit of 4"), "{err}"); + } + + #[test] + fn refused_process_substitution_is_an_error() { + let dir = fixture(&[]); + let mut state = ShellState::new(dir.path().to_str().unwrap()).unwrap(); + let mut w = Word::new(); + w.push_process_sub(ProcessSubType::Input, "true".into()); + let err = expand_words_in( + &[w], + &mut state, + ProcessSubs::Refuse, + dir.path(), + &GlobLimits::DEFAULT, + ) + .err() + .expect("refused"); + assert!(err.to_string().contains("<(true)"), "{err}"); + } + + #[test] + fn display_round_trips_quoting() { + let w = word(&[ + ("a b", Quote::Single), + ("*", Quote::Backslash), + ("x\"$", Quote::Double), + ("y", Quote::None), + ]); + assert_eq!(w.to_string(), "'a b'\\*\"x\\\"\\$\"y"); + assert_eq!(Word::literal("plain"), "plain"); + } + + #[test] + fn glob_pattern_escapes_quoted_metacharacters() { + assert_eq!(glob_pattern("\\*a*"), Some("[*]a*".to_string())); + assert_eq!(glob_pattern("\\*a"), None); + assert_eq!(glob_pattern("\\[x?"), Some("[[]x?".to_string())); + } +} diff --git a/impl/rust-cli/tests/word_expansion_e2e_tests.rs b/impl/rust-cli/tests/word_expansion_e2e_tests.rs new file mode 100644 index 0000000..bcfa282 --- /dev/null +++ b/impl/rust-cli/tests/word_expansion_e2e_tests.rs @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright (c) Jonathan D.A. Jewell +//! End-to-end tests for argument glob expansion, run through the `vsh` binary. +//! +//! Before the word-expansion stage (`vsh::word`), globs were expanded only +//! for external commands that also had a redirection: `wc -c *.txt` passed +//! `*.txt` to `wc`, pipelines and `echo` never expanded, and quoted patterns +//! reached the program with a stray backslash (`echo '*.txt'` printed +//! `\*.txt`). Each test runs `vsh -c` in a directory holding `a.txt`, +//! `b.txt` and `c.log`. + +use std::path::Path; +use std::process::{Command, Output}; + +use tempfile::TempDir; + +fn fixture() -> TempDir { + let dir = TempDir::new().unwrap(); + for name in ["a.txt", "b.txt", "c.log"] { + std::fs::write(dir.path().join(name), b"").unwrap(); + } + dir +} + +fn vsh(dir: &Path, script: &str) -> Output { + Command::new(env!("CARGO_BIN_EXE_vsh")) + .arg("-c") + .arg(script) + .current_dir(dir) + .output() + .expect("vsh runs") +} + +fn stdout(dir: &Path, script: &str) -> String { + let out = vsh(dir, script); + assert!( + out.status.success(), + "`{script}` failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8(out.stdout).unwrap() +} + +#[test] +fn external_command_without_redirect_gets_the_matches() { + let dir = fixture(); + assert_eq!( + stdout(dir.path(), "wc -c *.txt"), + "0 a.txt\n0 b.txt\n0 total\n" + ); +} + +#[test] +fn external_command_with_redirect_gets_the_matches() { + let dir = fixture(); + stdout(dir.path(), "wc -c *.txt > out.wc"); + assert_eq!( + std::fs::read_to_string(dir.path().join("out.wc")).unwrap(), + "0 a.txt\n0 b.txt\n0 total\n" + ); +} + +#[test] +fn every_pipeline_stage_gets_the_matches() { + let dir = fixture(); + assert_eq!(stdout(dir.path(), "ls *.txt | wc -l").trim(), "2"); + assert_eq!(stdout(dir.path(), "printf '%s\\n' *.log | cat"), "c.log\n"); +} + +#[test] +fn builtin_echo_gets_the_matches() { + let dir = fixture(); + assert_eq!(stdout(dir.path(), "echo *.txt"), "a.txt b.txt\n"); + assert_eq!(stdout(dir.path(), "echo -n *.log"), "c.log"); +} + +#[test] +fn quoted_patterns_stay_literal_without_a_backslash() { + let dir = fixture(); + assert_eq!(stdout(dir.path(), "echo '*.txt'"), "*.txt\n"); + assert_eq!(stdout(dir.path(), "echo \"*.txt\""), "*.txt\n"); + assert_eq!(stdout(dir.path(), "echo \\*.txt"), "*.txt\n"); + assert_eq!( + stdout(dir.path(), "printf '%s\\n' '*.txt' \"*.txt\" \\*.txt"), + "*.txt\n*.txt\n*.txt\n" + ); + assert_eq!( + stdout(dir.path(), "printf '%s\\n' '*.txt' | cat"), + "*.txt\n" + ); +} + +#[test] +fn a_pattern_that_matches_nothing_is_passed_literally() { + let dir = fixture(); + assert_eq!(stdout(dir.path(), "echo *.none"), "*.none\n"); + assert_eq!(stdout(dir.path(), "printf '%s\\n' *.none"), "*.none\n"); + assert_eq!( + stdout(dir.path(), "printf '%s\\n' *.none > out.none; cat out.none"), + "*.none\n" + ); +} + +#[test] +fn a_partly_quoted_word_globs_only_its_unquoted_part() { + let dir = fixture(); + std::fs::write(dir.path().join("*x.txt"), b"").unwrap(); + std::fs::write(dir.path().join("ax.txt"), b"").unwrap(); + assert_eq!(stdout(dir.path(), "echo '*'x.*"), "*x.txt\n"); + assert_eq!(stdout(dir.path(), "echo *x.*"), "*x.txt ax.txt\n"); +} + +#[test] +fn braces_expand_unquoted_and_stay_literal_quoted() { + let dir = fixture(); + assert_eq!(stdout(dir.path(), "echo a{1,2} '{a,b}'"), "a1 a2 {a,b}\n"); +} + +#[test] +fn function_arguments_get_the_matches() { + let dir = fixture(); + assert_eq!( + stdout(dir.path(), "f() { echo \"$#:$1:$2\"; }; f *.txt"), + "2:a.txt:b.txt\n" + ); +}