From da50ac41283faee8d8624ff1ee8250f668e836dc Mon Sep 17 00:00:00 2001 From: limadog9 Date: Sun, 18 Oct 2026 10:24:53 -0400 Subject: [PATCH] wc: saturate overflowing total counts Use 64-bit counters and checked accumulation for each total, reporting an error when a displayed count overflows. Saturate the metadata size sum used for column widths and cover total output modes with a sparse file regression test. Closes #11209 --- src/uu/wc/Cargo.toml | 1 - src/uu/wc/locales/en-US.ftl | 5 ++ src/uu/wc/src/count_fast.rs | 26 +++---- src/uu/wc/src/wc.rs | 44 +++++++++--- src/uu/wc/src/word_count.rs | 138 +++++++++++++++++++++++++++++++----- tests/by-util/test_wc.rs | 43 +++++++++++ 6 files changed, 217 insertions(+), 40 deletions(-) diff --git a/src/uu/wc/Cargo.toml b/src/uu/wc/Cargo.toml index be25c406c5a..4e51df730ad 100644 --- a/src/uu/wc/Cargo.toml +++ b/src/uu/wc/Cargo.toml @@ -13,7 +13,6 @@ readme.workspace = true [lib] path = "src/wc.rs" -test = false doctest = false [dependencies] diff --git a/src/uu/wc/locales/en-US.ftl b/src/uu/wc/locales/en-US.ftl index c6d2262b950..74c94802c8d 100644 --- a/src/uu/wc/locales/en-US.ftl +++ b/src/uu/wc/locales/en-US.ftl @@ -23,6 +23,11 @@ wc-error-cannot-open-for-reading = cannot open { $path } for reading wc-error-read-error = { $path }: read error wc-error-failed-to-print-result = failed to print result for { $title } wc-error-failed-to-print-total = failed to print total +wc-error-overflow = Value too large for defined data type +wc-error-total-bytes-overflow = total bytes: { wc-error-overflow } +wc-error-total-chars-overflow = total characters: { wc-error-overflow } +wc-error-total-lines-overflow = total lines: { wc-error-overflow } +wc-error-total-words-overflow = total words: { wc-error-overflow } # Decoder error messages decoder-error-invalid-byte-sequence = invalid byte sequence: { $bytes } diff --git a/src/uu/wc/src/count_fast.rs b/src/uu/wc/src/count_fast.rs index 82200fe08d9..2dc2797d1a5 100644 --- a/src/uu/wc/src/count_fast.rs +++ b/src/uu/wc/src/count_fast.rs @@ -28,9 +28,9 @@ const BUF_SIZE: usize = 64 * 1024; /// caller will fall back to a simpler method. #[inline] #[cfg(any(target_os = "linux", target_os = "android"))] -fn count_bytes_using_splice(fd: &impl rustix::fd::AsFd) -> Result { +fn count_bytes_using_splice(fd: &impl rustix::fd::AsFd) -> Result { use uucore::pipes::{MAX_ROOTLESS_PIPE_SIZE, pipe, splice}; - let null_file = uucore::pipes::dev_null().ok_or(0_usize)?; + let null_file = uucore::pipes::dev_null().ok_or(0_u64)?; let mut byte_count = 0; // no need to increase pipe size of input fd since // - sender with splice probably increased size already @@ -38,13 +38,13 @@ fn count_bytes_using_splice(fd: &impl rustix::fd::AsFd) -> Result loop { match splice(fd, &null_file, MAX_ROOTLESS_PIPE_SIZE) { Ok(0) => return Ok(byte_count), - Ok(res) => byte_count += res, + Ok(res) => byte_count += res as u64, Err(_) => break, // input is not pipe. needs additional pipe... } } let (pipe_rd, pipe_wr) = pipe::().map_err(|_| byte_count)?; while let s @ 1.. = splice(fd, &pipe_wr, MAX_ROOTLESS_PIPE_SIZE).map_err(|_| byte_count)? { - byte_count += s; + byte_count += s as u64; // pipe to null is not blocked. So this returns the same length at most cases // next splice does not hang if we discarded 1+ pages splice(&pipe_rd, &null_file, s).map_err(|_| byte_count)?; @@ -61,7 +61,7 @@ fn count_bytes_using_splice(fd: &impl rustix::fd::AsFd) -> Result /// 3. Otherwise, we just read normally, but without the overhead of counting /// other things such as lines and words. #[inline] -pub(crate) fn count_bytes_fast(handle: &mut T) -> (usize, Option) { +pub(crate) fn count_bytes_fast(handle: &mut T) -> (u64, Option) { let mut byte_count = 0; #[cfg(unix)] @@ -109,10 +109,10 @@ pub(crate) fn count_bytes_fast(handle: &mut T) -> (usize, Opti && stat.st_size > 0 { let sys_page_size = rustix::param::page_size(); - if !(stat.st_size as usize).is_multiple_of(sys_page_size) { + if !(stat.st_size as u64).is_multiple_of(sys_page_size as u64) { // regular file or file from /proc, /sys and similar pseudo-filesystems // with size that is NOT a multiple of system page size - return (stat.st_size as usize, None); + return (stat.st_size as u64, None); } else if let Some(file) = handle.inner_file() { // On some platforms `stat.st_blksize` and `stat.st_size` // are of different types: i64 vs i32 @@ -125,7 +125,7 @@ pub(crate) fn count_bytes_fast(handle: &mut T) -> (usize, Opti stat.st_size as i64 - stat.st_size as i64 % (stat.st_blksize as i64 + 1); if let Ok(n) = file.seek(SeekFrom::Start(offset as u64)) { - byte_count = n as usize; + byte_count = n; } } } @@ -143,7 +143,7 @@ pub(crate) fn count_bytes_fast(handle: &mut T) -> (usize, Opti { let attributes = metadata.file_attributes(); if (attributes & FILE_ATTRIBUTE_ARCHIVE) != 0 || (attributes & FILE_ATTRIBUTE_NORMAL) != 0 { - return (metadata.file_size() as usize, None); + return (metadata.file_size(), None); } } @@ -153,7 +153,7 @@ pub(crate) fn count_bytes_fast(handle: &mut T) -> (usize, Opti loop { match handle.read(&mut buf) { Ok(0) => return (byte_count, None), - Ok(n) => byte_count += n, + Ok(n) => byte_count += n as u64, Err(e) if e.kind() == ErrorKind::Interrupted => (), Err(e) => return (byte_count, Some(e)), } @@ -201,21 +201,21 @@ pub(crate) fn count_bytes_chars_and_lines_fast< Ok(0) => return (total, None), Ok(n) => { if COUNT_BYTES { - total.bytes += n; + total.bytes += n as u64; } if COUNT_CHARS { total.chars += if simd_allowed { bytecount::num_chars(&buf[..n]) } else { bytecount::naive_num_chars(&buf[..n]) - }; + } as u64; } if COUNT_LINES { total.lines += if simd_allowed { bytecount::count(&buf[..n], b'\n') } else { bytecount::naive_count(&buf[..n], b'\n') - }; + } as u64; } } Err(e) if e.kind() == ErrorKind::Interrupted => (), diff --git a/src/uu/wc/src/wc.rs b/src/uu/wc/src/wc.rs index 60d586c873e..68c5b86e9c8 100644 --- a/src/uu/wc/src/wc.rs +++ b/src/uu/wc/src/wc.rs @@ -33,13 +33,13 @@ use uucore::{ hardware::{HardwareFeature, HasHardwareFeatures as _, SimdPolicy}, parser::shortcut_value_parser::ShortcutValueParser, quoting_style::{self, QuotingStyle}, - show, + show, show_error, }; use crate::{ count_fast::{count_bytes_chars_and_lines_fast, count_bytes_fast}, countable::WordCountable, - word_count::WordCount, + word_count::{CountOverflow, WordCount}, }; /// The minimum character width for formatting counts when reading from stdin. @@ -575,7 +575,7 @@ fn process_chunk< >( total: &mut WordCount, text: &str, - current_len: &mut usize, + current_len: &mut u64, in_word: &mut bool, is_posixly_correct: bool, ) { @@ -606,7 +606,7 @@ fn process_chunk< *current_len += 8; } _ => { - *current_len += ch.width().unwrap_or(0); + *current_len += ch.width().unwrap_or(0) as u64; } } } @@ -617,7 +617,7 @@ fn process_chunk< total.chars += 1; } } - total.bytes += text.len(); + total.bytes += text.len() as u64; total.max_line_length = max(*current_len, total.max_line_length); } @@ -629,7 +629,7 @@ fn handle_error( ) -> Option { match error { BufReadDecoderError::InvalidByteSequence(bytes) => { - total.bytes += bytes.len(); + total.bytes += bytes.len() as u64; if !(*in_word) { *in_word = true; total.words += 1; @@ -739,7 +739,7 @@ fn compute_number_width(inputs: &Inputs, settings: &Settings) -> usize { Input::Path(path) => { if let Ok(meta) = fs::metadata(path) { if meta.is_file() { - total += meta.len(); + total = total.saturating_add(meta.len()); } else { minimum_width = MINIMUM_WIDTH; } @@ -895,6 +895,7 @@ pub(crate) fn wc_simd_allowed(policy: &SimdPolicy) -> bool { fn wc(inputs: &Inputs, settings: &Settings) -> UResult<()> { let mut total_word_count = WordCount::default(); + let mut total_overflow = CountOverflow::default(); let mut num_inputs: usize = 0; let (number_width, are_stats_visible) = match settings.total_when { @@ -973,7 +974,7 @@ fn wc(inputs: &Inputs, settings: &Settings) -> UResult<()> { continue; } }; - total_word_count += word_count; + total_word_count.accumulate(word_count, &mut total_overflow); if are_stats_visible { let maybe_title = input.to_title(); let maybe_title_str = maybe_title.as_deref(); @@ -991,6 +992,33 @@ fn wc(inputs: &Inputs, settings: &Settings) -> UResult<()> { } if settings.total_when.is_total_row_visible(num_inputs) { + for (overflowed, enabled, message) in [ + ( + total_overflow.lines, + settings.show_lines, + "wc-error-total-lines-overflow", + ), + ( + total_overflow.words, + settings.show_words, + "wc-error-total-words-overflow", + ), + ( + total_overflow.chars, + settings.show_chars, + "wc-error-total-chars-overflow", + ), + ( + total_overflow.bytes, + settings.show_bytes, + "wc-error-total-bytes-overflow", + ), + ] { + if enabled && overflowed { + show_error!("{}", translate!(message)); + uucore::error::set_exit_code(1); + } + } let wc_total_msg = translate!("wc-total"); let title = are_stats_visible.then_some(OsStr::new(&wc_total_msg)); if let Err(err) = print_stats(settings, &total_word_count, title, number_width) { diff --git a/src/uu/wc/src/word_count.rs b/src/uu/wc/src/word_count.rs index fce74d23dd8..f70283a847f 100644 --- a/src/uu/wc/src/word_count.rs +++ b/src/uu/wc/src/word_count.rs @@ -4,33 +4,135 @@ // file that was distributed with this source code. use std::cmp::max; -use std::ops::{Add, AddAssign}; #[derive(Debug, Default, Copy, Clone)] pub struct WordCount { - pub bytes: usize, - pub chars: usize, - pub lines: usize, - pub words: usize, - pub max_line_length: usize, + pub bytes: u64, + pub chars: u64, + pub lines: u64, + pub words: u64, + pub max_line_length: u64, } -impl Add for WordCount { - type Output = Self; +#[derive(Debug, Default, PartialEq, Eq)] +pub struct CountOverflow { + pub bytes: bool, + pub chars: bool, + pub lines: bool, + pub words: bool, +} - fn add(self, other: Self) -> Self { - Self { - bytes: self.bytes + other.bytes, - chars: self.chars + other.chars, - lines: self.lines + other.lines, - words: self.words + other.words, - max_line_length: max(self.max_line_length, other.max_line_length), +impl WordCount { + /// Accumulate counts, saturating each independently and retaining overflow flags. + pub fn accumulate(&mut self, other: Self, overflow: &mut CountOverflow) { + for (total, count, overflowed) in [ + (&mut self.bytes, other.bytes, &mut overflow.bytes), + (&mut self.chars, other.chars, &mut overflow.chars), + (&mut self.lines, other.lines, &mut overflow.lines), + (&mut self.words, other.words, &mut overflow.words), + ] { + if let Some(sum) = total.checked_add(count) { + *total = sum; + } else { + *total = u64::MAX; + *overflowed = true; + } } + self.max_line_length = max(self.max_line_length, other.max_line_length); } } -impl AddAssign for WordCount { - fn add_assign(&mut self, other: Self) { - *self = *self + other; +#[cfg(test)] +mod tests { + use super::{CountOverflow, WordCount}; + + #[test] + fn accumulate_saturates_each_count_independently() { + let mut total = WordCount { + bytes: u64::MAX - 1, + chars: u64::MAX - 2, + lines: u64::MAX - 3, + words: u64::MAX - 4, + max_line_length: 20, + }; + let mut overflow = CountOverflow::default(); + total.accumulate( + WordCount { + bytes: 1, + chars: 2, + lines: 3, + words: 4, + max_line_length: 10, + }, + &mut overflow, + ); + assert_eq!(overflow, CountOverflow::default()); + total.accumulate( + WordCount { + bytes: 1, + ..WordCount::default() + }, + &mut overflow, + ); + assert_eq!( + overflow, + CountOverflow { + bytes: true, + ..CountOverflow::default() + } + ); + total.accumulate( + WordCount { + chars: 1, + lines: 1, + words: 1, + ..WordCount::default() + }, + &mut overflow, + ); + assert_eq!( + overflow, + CountOverflow { + bytes: true, + chars: true, + lines: true, + words: true + } + ); + assert_eq!( + (total.bytes, total.chars, total.lines, total.words), + (u64::MAX, u64::MAX, u64::MAX, u64::MAX) + ); + assert_eq!(total.max_line_length, 20); + } + + #[test] + fn accumulate_keeps_other_counts_after_overflow() { + let mut total = WordCount { + bytes: u64::MAX, + ..WordCount::default() + }; + let mut overflow = CountOverflow::default(); + total.accumulate( + WordCount { + bytes: 1, + chars: 3, + lines: 2, + words: 1, + max_line_length: 5, + }, + &mut overflow, + ); + assert_eq!( + overflow, + CountOverflow { + bytes: true, + ..CountOverflow::default() + } + ); + assert_eq!( + (total.chars, total.lines, total.words, total.max_line_length), + (3, 2, 1, 5) + ); } } diff --git a/tests/by-util/test_wc.rs b/tests/by-util/test_wc.rs index 16fd79293f5..98c3150086c 100644 --- a/tests/by-util/test_wc.rs +++ b/tests/by-util/test_wc.rs @@ -657,6 +657,49 @@ fn test_total_only() { .stdout_is("31 313 1887\n"); } +#[test] +#[cfg(unix)] +#[cfg_attr(wasi_runner, ignore = "WASI: byte counting reads sparse file contents")] +fn test_total_bytes_overflow() { + let (at, ts) = uutests::at_and_ts!(); + let path = "sparse"; + let size = (1_u64 << 61) + 1; + // Not every filesystem supports sparse files this large. + if let Err(error) = at.make_file(path).set_len(size) { + eprintln!("Skipping test: cannot create a sparse file of size {size}: {error}"); + return; + } + let files = vec![path; 9]; + let diagnostic = "wc: total bytes: Value too large for defined data type\n"; + + ts.ucmd() + .args(&["-c", "--total=only"]) + .args(&files) + .fails_with_code(1) + .stdout_is(format!("{}\n", u64::MAX)) + .stderr_is(diagnostic); + + // This also exercises the metadata sum used to choose the column width. + for total in ["--total=auto", "--total=always"] { + ts.ucmd() + .args(&["-c", total]) + .args(&files) + .fails_with_code(1) + .stdout_is(format!( + "{}{} total\n", + format!("{size:20} {path}\n").repeat(files.len()), + u64::MAX, + )) + .stderr_is(diagnostic); + } + + ts.ucmd() + .args(&["-c", "--total=never"]) + .args(&files) + .succeeds() + .stdout_only(format!("{size:20} {path}\n").repeat(files.len())); +} + #[test] fn test_zero_length_files() { // A trailing zero is ignored, but otherwise empty file names are an error...