diff --git a/Cargo.lock b/Cargo.lock index 79a8b0e..e1dd06f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -915,7 +915,7 @@ checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" [[package]] name = "xtask" -version = "0.1.0" +version = "0.1.1" dependencies = [ "clap", "serde_json", @@ -925,7 +925,7 @@ dependencies = [ [[package]] name = "xtce-cli" -version = "0.1.0" +version = "0.1.1" dependencies = [ "clap", "xtce-codegen", @@ -935,7 +935,7 @@ dependencies = [ [[package]] name = "xtce-codegen" -version = "0.1.0" +version = "0.1.1" dependencies = [ "criterion", "prettyplease", @@ -949,7 +949,7 @@ dependencies = [ [[package]] name = "xtce-codegen-e2e" -version = "0.1.0" +version = "0.1.1" dependencies = [ "xtce-codegen", "xtce-decode", @@ -958,7 +958,7 @@ dependencies = [ [[package]] name = "xtce-decode" -version = "0.1.0" +version = "0.1.1" dependencies = [ "criterion", "proptest", @@ -968,7 +968,7 @@ dependencies = [ [[package]] name = "xtce-model" -version = "0.1.0" +version = "0.1.1" dependencies = [ "criterion", "proptest", diff --git a/crates/xtce-cli/src/decode.rs b/crates/xtce-cli/src/decode.rs index 18de97a..dde4556 100644 --- a/crates/xtce-cli/src/decode.rs +++ b/crates/xtce-cli/src/decode.rs @@ -1,6 +1,6 @@ //! `xtce decode` — run a packet stream through a definition and print the values. -use std::io::Write; +use std::io::{Read, Write}; use std::path::Path; use std::time::Instant; @@ -14,6 +14,8 @@ pub enum Format { Table, /// One JSON object per packet, one packet per line. Jsonl, + /// Comma-separated values (CSV) with packet, container, and parameter columns. + Csv, /// Nothing per packet; only the summary. Quiet, } @@ -28,6 +30,62 @@ pub struct Options<'a> { pub only: &'a [String], pub format: Format, pub show_raw: bool, + pub no_summary: bool, +} + +fn load_stream(packets: &Path) -> std::io::Result> { + if packets.as_os_str() == "-" { + let mut bytes = Vec::new(); + std::io::stdin().read_to_end(&mut bytes)?; + Ok(bytes) + } else { + std::fs::read(packets) + } +} + +fn load_definition(definition: &Path) -> Result> { + if definition.as_os_str() == "-" { + let mut xml = String::new(); + std::io::stdin().read_to_string(&mut xml)?; + Ok(XtceDb::from_xml(&xml)?) + } else { + Ok(XtceDb::from_path(definition)?) + } +} + +struct Summary { + total: usize, + decoded: usize, + unrecognised: usize, + failed: usize, + trailing: usize, + load_elapsed: std::time::Duration, + decode_elapsed: std::time::Duration, +} + +fn print_summary(out: &mut impl Write, summary: &Summary) -> std::io::Result<()> { + writeln!(out)?; + writeln!( + out, + "{} packet(s): {} decoded, {} not described, {} failed", + summary.total, summary.decoded, summary.unrecognised, summary.failed + )?; + if summary.trailing > 0 { + // Worth surfacing: the definition and the packets disagree about length, which + // usually means a container is missing entries rather than that anything is corrupt. + writeln!( + out, + "{} packet(s) had bits no entry claimed — the definition may be incomplete", + summary.trailing + )?; + } + writeln!( + out, + "load {:.2} ms, decode {:.2} ms ({:.1} packets/s)", + summary.load_elapsed.as_secs_f64() * 1e3, + summary.decode_elapsed.as_secs_f64() * 1e3, + summary.decoded as f64 / summary.decode_elapsed.as_secs_f64().max(f64::MIN_POSITIVE), + ) } /// Decodes a stream and writes the result. @@ -39,8 +97,12 @@ pub struct Options<'a> { /// not well-framed. A packet that fails to *decode* is reported and counted, not fatal: one /// malformed frame in a downlink should not end the run. pub fn run(options: &Options<'_>) -> Result<(), Box> { + if options.definition.as_os_str() == "-" && options.packets.as_os_str() == "-" { + return Err("cannot read both definition and packets from standard input".into()); + } + let load_start = Instant::now(); - let db = XtceDb::from_path(options.definition)?; + let db = load_definition(options.definition)?; let load_elapsed = load_start.elapsed(); check_only(&db, options.only)?; @@ -50,7 +112,7 @@ pub fn run(options: &Options<'_>) -> Result<(), Box> { None => Decoder::new(&db)?, }; - let stream = std::fs::read(options.packets)?; + let stream = load_stream(options.packets)?; let stdout = std::io::stdout(); let mut out = std::io::BufWriter::new(stdout.lock()); @@ -63,6 +125,14 @@ pub fn run(options: &Options<'_>) -> Result<(), Box> { // One packet buffer for the whole stream: after the first packet this allocates nothing. let mut packet = decoder.new_packet(&stream); + if options.format == Format::Csv { + if options.show_raw { + writeln!(out, "packet,container,parameter,eng,raw")?; + } else { + writeln!(out, "packet,container,parameter,eng")?; + } + } + let decode_start = Instant::now(); for (index, framed) in PacketIter::new(&stream, options.skip_header_bytes).enumerate() { if options.limit.is_some_and(|limit| total >= limit) { @@ -80,46 +150,49 @@ pub fn run(options: &Options<'_>) -> Result<(), Box> { match options.format { Format::Table => write_table(&mut out, index, &packet, options)?, Format::Jsonl => write_jsonl(&mut out, index, &packet, options)?, + Format::Csv => write_csv(&mut out, index, &packet, options)?, Format::Quiet => {} } } Err(DecodeError::UnrecognizedPacket { container, .. }) => { unrecognised += 1; - if options.format != Format::Quiet { + if options.format == Format::Table { writeln!( out, "packet {index}: not described by this definition (no inheritor of {container} matches)" )?; + } else if options.format != Format::Quiet { + eprintln!( + "packet {index}: not described by this definition (no inheritor of {container} matches)" + ); } } Err(error) => { failed += 1; - writeln!(out, "packet {index}: {error}")?; + if options.format == Format::Table { + writeln!(out, "packet {index}: {error}")?; + } else { + eprintln!("packet {index}: {error}"); + } } } } let decode_elapsed = decode_start.elapsed(); - writeln!(out)?; - writeln!( - out, - "{total} packet(s): {decoded_count} decoded, {unrecognised} not described, {failed} failed" - )?; - if trailing > 0 { - // Worth surfacing: the definition and the packets disagree about length, which - // usually means a container is missing entries rather than that anything is corrupt. - writeln!( - out, - "{trailing} packet(s) had bits no entry claimed — the definition may be incomplete" + if !options.no_summary { + print_summary( + &mut out, + &Summary { + total, + decoded: decoded_count, + unrecognised, + failed, + trailing, + load_elapsed, + decode_elapsed, + }, )?; } - writeln!( - out, - "load {:.2} ms, decode {:.2} ms ({:.1} packets/s)", - load_elapsed.as_secs_f64() * 1e3, - decode_elapsed.as_secs_f64() * 1e3, - decoded_count as f64 / decode_elapsed.as_secs_f64().max(f64::MIN_POSITIVE), - )?; out.flush()?; if failed > 0 { @@ -382,6 +455,94 @@ fn write_json_string(out: &mut impl Write, text: &str) -> std::io::Result<()> { Ok(()) } +fn write_csv( + out: &mut impl Write, + index: usize, + packet: &xtce_decode::DecodedPacket<'_, '_>, + options: &Options<'_>, +) -> std::io::Result<()> { + let db = packet.db(); + let container = db + .container(packet.container()) + .map_or("?", |container| db.name(container.name)); + for (name, value) in packet.iter_named() { + if !wanted(options.only, name) { + continue; + } + write!(out, "{index},")?; + write_csv_string(out, container)?; + write!(out, ",")?; + write_csv_string(out, name)?; + write!(out, ",")?; + write_csv_eng(out, &value.eng)?; + if options.show_raw { + write!(out, ",")?; + write_csv_raw(out, &value.raw)?; + } + writeln!(out)?; + } + Ok(()) +} + +fn write_csv_eng(out: &mut impl Write, value: &EngValue<'_, '_>) -> std::io::Result<()> { + match value { + EngValue::Unsigned(number) => write!(out, "{number}"), + EngValue::Signed(number) => write!(out, "{number}"), + EngValue::Float(number) => write_csv_float(out, *number), + EngValue::Bool(flag) => write!(out, "{flag}"), + EngValue::Label(text) => write_csv_string(out, text), + EngValue::Text(text) => write_csv_string(out, text.as_ref()), + EngValue::Bytes(bytes) => write_hex_raw(out, bytes), + } +} + +fn write_csv_raw(out: &mut impl Write, value: &RawValue<'_>) -> std::io::Result<()> { + match value { + RawValue::Unsigned(number) => write!(out, "{number}"), + RawValue::Signed(number) => write!(out, "{number}"), + RawValue::Float(number) => write_csv_float(out, *number), + RawValue::Bytes(bytes) => write_hex_raw(out, bytes), + } +} + +fn write_csv_float(out: &mut impl Write, value: f64) -> std::io::Result<()> { + if value.is_nan() { + write!(out, "NaN") + } else if value.is_infinite() { + if value.is_sign_positive() { + write!(out, "Infinity") + } else { + write!(out, "-Infinity") + } + } else { + write!(out, "{value}") + } +} + +fn write_hex_raw(out: &mut impl Write, bytes: &[u8]) -> std::io::Result<()> { + for byte in bytes { + write!(out, "{byte:02x}")?; + } + Ok(()) +} + +fn write_csv_string(out: &mut impl Write, text: &str) -> std::io::Result<()> { + if text.contains([',', '"', '\n', '\r']) { + out.write_all(b"\"")?; + for ch in text.chars() { + if ch == '"' { + out.write_all(b"\"\"")?; + } else { + let mut buf = [0u8; 4]; + out.write_all(ch.encode_utf8(&mut buf).as_bytes())?; + } + } + out.write_all(b"\"") + } else { + out.write_all(text.as_bytes()) + } +} + #[cfg(test)] mod tests { use super::{check_only, edit_distance, glob_matches, suggest, wanted}; @@ -568,4 +729,98 @@ mod tests { // Two characters apart is still two edits: only *adjacent* swaps are one. assert_eq!(edit_distance("ABCD", "CBAD"), 2); } + + #[test] + fn csv_string_escapes_only_when_needed() { + let mut out = Vec::new(); + super::write_csv_string(&mut out, "plain_identifier").unwrap(); + assert_eq!(String::from_utf8(out).unwrap(), "plain_identifier"); + + let mut out = Vec::new(); + super::write_csv_string(&mut out, "comma, separated").unwrap(); + assert_eq!(String::from_utf8(out).unwrap(), "\"comma, separated\""); + + let mut out = Vec::new(); + super::write_csv_string(&mut out, "contains \"quotes\" inside").unwrap(); + assert_eq!( + String::from_utf8(out).unwrap(), + "\"contains \"\"quotes\"\" inside\"" + ); + + let mut out = Vec::new(); + super::write_csv_string(&mut out, "line1\nline2").unwrap(); + assert_eq!(String::from_utf8(out).unwrap(), "\"line1\nline2\""); + } + + #[test] + fn csv_float_formats_special_values() { + let mut out = Vec::new(); + super::write_csv_float(&mut out, 42.5).unwrap(); + assert_eq!(String::from_utf8(out).unwrap(), "42.5"); + + let mut out = Vec::new(); + super::write_csv_float(&mut out, f64::NAN).unwrap(); + assert_eq!(String::from_utf8(out).unwrap(), "NaN"); + + let mut out = Vec::new(); + super::write_csv_float(&mut out, f64::INFINITY).unwrap(); + assert_eq!(String::from_utf8(out).unwrap(), "Infinity"); + + let mut out = Vec::new(); + super::write_csv_float(&mut out, f64::NEG_INFINITY).unwrap(); + assert_eq!(String::from_utf8(out).unwrap(), "-Infinity"); + } + + #[test] + fn csv_formatting_writes_headers_and_rows() { + let db = db(); + let decoder = xtce_decode::Decoder::new(&db).expect("decoder creates"); + let data = [0x00, 0x00, 0x00, 0x2A]; + let mut packet = decoder.new_packet(&data); + decoder.decode_into(&mut packet, &data).expect("decodes"); + + let mut out = Vec::new(); + let options = super::Options { + definition: std::path::Path::new("dummy.xml"), + packets: std::path::Path::new("dummy.bin"), + root: None, + skip_header_bytes: 0, + limit: None, + only: &[], + format: super::Format::Csv, + show_raw: false, + no_summary: false, + }; + + super::write_csv(&mut out, 0, &packet, &options).expect("writes csv"); + let text = String::from_utf8(out).expect("utf8"); + assert_eq!(text, "0,PKT,ADGPSPOSX,42\n"); + + let mut out = Vec::new(); + let options_raw = super::Options { + show_raw: true, + ..options + }; + super::write_csv(&mut out, 0, &packet, &options_raw).expect("writes csv raw"); + let text_raw = String::from_utf8(out).expect("utf8"); + assert_eq!(text_raw, "0,PKT,ADGPSPOSX,42,42\n"); + } + + #[test] + fn print_summary_writes_formatted_counts() { + let mut out = Vec::new(); + let summary = super::Summary { + total: 10, + decoded: 8, + unrecognised: 1, + failed: 1, + trailing: 2, + load_elapsed: std::time::Duration::from_millis(5), + decode_elapsed: std::time::Duration::from_millis(15), + }; + super::print_summary(&mut out, &summary).expect("prints summary"); + let text = String::from_utf8(out).expect("utf8"); + assert!(text.contains("10 packet(s): 8 decoded, 1 not described, 1 failed")); + assert!(text.contains("2 packet(s) had bits no entry claimed")); + } } diff --git a/crates/xtce-cli/src/info.rs b/crates/xtce-cli/src/info.rs index 6e5b62c..e9a49ef 100644 --- a/crates/xtce-cli/src/info.rs +++ b/crates/xtce-cli/src/info.rs @@ -9,7 +9,7 @@ pub fn report(db: &XtceDb, elapsed: Duration, verbose: bool) { let stats = db.stats(); let source = db .source() - .map_or_else(|| "".to_owned(), |path| path.display().to_string()); + .map_or_else(|| "".to_owned(), |path| path.display().to_string()); println!("{source}"); println!( @@ -97,6 +97,7 @@ pub fn report(db: &XtceDb, elapsed: Duration, verbose: bool) { if verbose { print_containers(db); + print_commands(db); } println!(); } @@ -263,3 +264,48 @@ fn print_containers(db: &XtceDb) { } } } + +fn print_commands(db: &XtceDb) { + if db.meta_commands().is_empty() { + return; + } + println!(" telecommands:"); + for command in db.meta_commands() { + let base = command + .base + .and_then(|base| db.meta_command(base)) + .map(|base| format!(" extends {}", db.name(base.name))) + .unwrap_or_default(); + let container = command + .container + .and_then(|c| db.container(c)) + .map(|c| format!(" -> {}", db.name(c.name))) + .unwrap_or_default(); + println!( + " {}{}{}{}", + db.name(command.name), + if command.is_abstract { + " (abstract)" + } else { + "" + }, + base, + container + ); + for ¶m_id in &command.arguments { + let Some(param) = db.parameter(param_id) else { + continue; + }; + let ty = db.type_of(param_id); + let bits = ty + .and_then(xtce_model::ParameterType::fixed_size_in_bits) + .map_or_else(|| "var".to_owned(), |bits| format!("{bits}b")); + println!( + " {:<28} {:<16} {}", + db.name(param.name), + ty.map_or("?", |ty| ty.kind.label()), + bits + ); + } + } +} diff --git a/crates/xtce-cli/src/main.rs b/crates/xtce-cli/src/main.rs index f1e5230..274d5ce 100644 --- a/crates/xtce-cli/src/main.rs +++ b/crates/xtce-cli/src/main.rs @@ -11,7 +11,8 @@ #![allow(clippy::doc_markdown)] #![cfg_attr(test, allow(clippy::unwrap_used, clippy::expect_used, clippy::panic))] -use std::path::PathBuf; +use std::io::Read; +use std::path::{Path, PathBuf}; use std::process::ExitCode; use std::time::Instant; @@ -32,7 +33,7 @@ struct Cli { enum Command { /// Summarise one or more XTCE files: space systems, counts, and anything not decodable. Info { - /// XTCE files to inspect. + /// XTCE files to inspect. Use '-' to read from standard input. #[arg(required = true)] files: Vec, @@ -43,10 +44,10 @@ enum Command { /// Decode a CCSDS packet stream against an XTCE definition. Decode { - /// XTCE definition. + /// XTCE definition. Use '-' to read from standard input. definition: PathBuf, - /// File of concatenated CCSDS packets. + /// File of concatenated CCSDS packets. Use '-' to read from standard input. packets: PathBuf, /// Container to start from. Defaults to CCSDSPacket, CCSDSTelemetryPacket, or the @@ -74,11 +75,15 @@ enum Command { /// Also show the raw value alongside the engineering value. #[arg(long)] raw: bool, + + /// Suppress the final summary (load and decode timing, packet counts). + #[arg(long)] + no_summary: bool, }, /// Compile a definition into a static Rust decoder. Codegen { - /// XTCE definition. + /// XTCE definition. Use '-' to read from standard input. definition: PathBuf, /// Container to start from. @@ -92,7 +97,7 @@ enum Command { /// Time loading an XTCE file. Bench { - /// XTCE file to load. + /// XTCE file to load. Use '-' to read from standard input. file: PathBuf, /// Number of load iterations to time. @@ -116,26 +121,19 @@ fn main() -> ExitCode { } } +fn load_db(path: &Path) -> Result> { + if path.as_os_str() == "-" { + let mut xml = String::new(); + std::io::stdin().read_to_string(&mut xml)?; + Ok(XtceDb::from_xml(&xml)?) + } else { + Ok(XtceDb::from_path(path)?) + } +} + fn run(cli: Cli) -> Result<(), Box> { match cli.command { - Command::Info { files, verbose } => { - let mut failures = 0; - for file in &files { - let started = Instant::now(); - match XtceDb::from_path(file) { - Ok(db) => info::report(&db, started.elapsed(), verbose), - Err(error) => { - failures += 1; - eprintln!("{}: {error}", file.display()); - } - } - } - if failures > 0 { - return Err(format!("{failures} of {} file(s) failed to load", files.len()).into()); - } - Ok(()) - } - + Command::Info { files, verbose } => run_info(&files, verbose), Command::Decode { definition, packets, @@ -145,6 +143,7 @@ fn run(cli: Cli) -> Result<(), Box> { params, format, raw, + no_summary, } => decode::run(&decode::Options { definition: &definition, packets: &packets, @@ -154,69 +153,105 @@ fn run(cli: Cli) -> Result<(), Box> { only: ¶ms, format, show_raw: raw, + no_summary, }), - Command::Codegen { definition, root, out, - } => { - let db = XtceDb::from_path(&definition)?; - let source = xtce_codegen::generate( - &db, - &xtce_codegen::Options { - root, - source_label: Some(definition.display().to_string()), - }, - )?; - match out { - Some(path) => { - std::fs::write(&path, source)?; - eprintln!("wrote {}", path.display()); - eprintln!( - "include it inside a module carrying the lint allowances generated \ - code needs; the file's header shows how" - ); - } - None => print!("{source}"), + } => run_codegen(&definition, root, out), + Command::Bench { file, iterations } => run_bench(&file, iterations), + } +} + +fn run_info(files: &[PathBuf], verbose: bool) -> Result<(), Box> { + let mut failures = 0; + for file in files { + let started = Instant::now(); + match load_db(file) { + Ok(db) => info::report(&db, started.elapsed(), verbose), + Err(error) => { + failures += 1; + eprintln!("{}: {error}", file.display()); } - Ok(()) } + } + if failures > 0 { + return Err(format!("{failures} of {} file(s) failed to load", files.len()).into()); + } + Ok(()) +} - Command::Bench { file, iterations } => { - let bytes = std::fs::metadata(&file).map_or(0, |meta| meta.len()); - let mut timings = Vec::with_capacity(iterations as usize); - let mut last = None; - for _ in 0..iterations.max(1) { - let started = Instant::now(); - let db = XtceDb::from_path(&file)?; - timings.push(started.elapsed()); - last = Some(db); - } - timings.sort_unstable(); - let median = timings.get(timings.len() / 2).copied().unwrap_or_default(); - let best = timings.first().copied().unwrap_or_default(); - - println!("{}", file.display()); - println!(" size {:>10.1} KiB", bytes as f64 / 1024.0); - println!(" load median {:>10.3} ms", median.as_secs_f64() * 1e3); - println!(" load best {:>10.3} ms", best.as_secs_f64() * 1e3); - if let Some(db) = last { - let stats = db.stats(); - println!( - " throughput {:>10.1} MiB/s", - (bytes as f64 / (1024.0 * 1024.0)) - / median.as_secs_f64().max(f64::MIN_POSITIVE) - ); - println!( - " model {} parameters, {} containers, {} interned names ({:.1} KiB)", - stats.parameters, - stats.containers, - stats.interned_names, - stats.interned_bytes as f64 / 1024.0 - ); - } - Ok(()) +fn run_codegen( + definition: &Path, + root: Option, + out: Option, +) -> Result<(), Box> { + let db = load_db(definition)?; + let source_label = if definition.as_os_str() == "-" { + Some("".to_owned()) + } else { + Some(definition.display().to_string()) + }; + let source = xtce_codegen::generate(&db, &xtce_codegen::Options { root, source_label })?; + match out { + Some(path) => { + std::fs::write(&path, source)?; + eprintln!("wrote {}", path.display()); + eprintln!( + "include it inside a module carrying the lint allowances generated \ + code needs; the file's header shows how" + ); } + None => print!("{source}"), + } + Ok(()) +} + +fn run_bench(file: &Path, iterations: u32) -> Result<(), Box> { + let (bytes, xml_data) = if file.as_os_str() == "-" { + let mut xml = String::new(); + std::io::stdin().read_to_string(&mut xml)?; + (xml.len() as u64, Some(xml)) + } else { + (std::fs::metadata(file).map_or(0, |meta| meta.len()), None) + }; + let mut timings = Vec::with_capacity(iterations as usize); + let mut last = None; + for _ in 0..iterations.max(1) { + let started = Instant::now(); + let db = match &xml_data { + Some(xml) => XtceDb::from_xml(xml)?, + None => XtceDb::from_path(file)?, + }; + timings.push(started.elapsed()); + last = Some(db); + } + timings.sort_unstable(); + let median = timings.get(timings.len() / 2).copied().unwrap_or_default(); + let best = timings.first().copied().unwrap_or_default(); + + if file.as_os_str() == "-" { + println!(""); + } else { + println!("{}", file.display()); + } + println!(" size {:>10.1} KiB", bytes as f64 / 1024.0); + println!(" load median {:>10.3} ms", median.as_secs_f64() * 1e3); + println!(" load best {:>10.3} ms", best.as_secs_f64() * 1e3); + if let Some(db) = last { + let stats = db.stats(); + println!( + " throughput {:>10.1} MiB/s", + (bytes as f64 / (1024.0 * 1024.0)) / median.as_secs_f64().max(f64::MIN_POSITIVE) + ); + println!( + " model {} parameters, {} containers, {} interned names ({:.1} KiB)", + stats.parameters, + stats.containers, + stats.interned_names, + stats.interned_bytes as f64 / 1024.0 + ); } + Ok(()) }