From bf958e46bb848b76fefd901738e18d2312d83d01 Mon Sep 17 00:00:00 2001 From: Matthew Charles Busel Date: Tue, 17 Nov 2026 23:09:19 -0500 Subject: [PATCH] rust(feat): add a MAVLink tlog replay example to sift_stream --- Cargo.lock | 75 ++++- Cargo.toml | 2 + rust/crates/sift_stream/Cargo.toml | 6 + .../sift_stream/examples/mavlink-tlog/main.rs | 308 ++++++++++++++++++ 4 files changed, 389 insertions(+), 2 deletions(-) create mode 100644 rust/crates/sift_stream/examples/mavlink-tlog/main.rs diff --git a/Cargo.lock b/Cargo.lock index 7a485da264..e359ced7a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1096,6 +1096,12 @@ dependencies = [ "libc", ] +[[package]] +name = "crc-any" +version = "2.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46db9f663dfb869b80fcf59e32d7a80fc6c464a4f6328f3f06a00f5e36d05f8c" + [[package]] name = "crc32fast" version = "1.5.2" @@ -2665,6 +2671,49 @@ dependencies = [ "rawpointer", ] +[[package]] +name = "mavlink" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc217df9ea6cf11b5b6a6e8bef71d367b44d3dae5cd464d0eb7947a48849977b" +dependencies = [ + "bitflags", + "mavlink-bindgen", + "mavlink-core", + "num-derive 0.5.1", + "num-traits", +] + +[[package]] +name = "mavlink-bindgen" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "695fe1dff0efebde67fde50fb800f4703e8188e0447ab6332f48cb3066bcf982" +dependencies = [ + "crc-any", + "proc-macro2", + "quick-xml 0.41.0", + "quote", + "regex", + "thiserror 2.0.21", +] + +[[package]] +name = "mavlink-core" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff9827ded6087bf5adbb2f4d4450c167096888c1a1619b16834fe9bfb9759d0b" +dependencies = [ + "crc-any", + "memchr", +] + +[[package]] +name = "mavlink-tlog" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b47c5b6ef0363c8fa90c66ff0dbdf97ac16ba4c79fe4c5f8aabcd3c8c71742" + [[package]] name = "mcap" version = "0.25.0" @@ -2857,6 +2906,17 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "num-derive" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4e98dc3b890f6c23a0f9d3d491a2823d0dea0fa656302a13dd225fa924112a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + [[package]] name = "num-integer" version = "0.1.47" @@ -2932,7 +2992,7 @@ dependencies = [ "itertools 0.14.0", "parking_lot", "percent-encoding", - "quick-xml", + "quick-xml 0.39.4", "rand 0.10.3", "reqwest 0.12.28", "ring", @@ -3763,7 +3823,7 @@ dependencies = [ "indexmap 2.14.2", "libc", "memmap2", - "num-derive", + "num-derive 0.4.2", "num-traits", "polars-error", "rand 0.9.5", @@ -4056,6 +4116,15 @@ dependencies = [ "serde", ] +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "memchr", +] + [[package]] name = "quinn" version = "0.11.12" @@ -5038,6 +5107,8 @@ dependencies = [ "futures-core", "hyper", "hyper-util", + "mavlink", + "mavlink-tlog", "pbjson-types", "prost", "rand 0.10.3", diff --git a/Cargo.toml b/Cargo.toml index 55faed9492..1eb7f3dca0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,6 +52,8 @@ hyper-util = { version = "0.1.20", features = ["service", "server", "tokio"] } include_dir = "0.7" indicatif = "0.18" indoc = "2.0" +mavlink = { version = "0.21", default-features = false, features = ["std", "dialect-common"] } +mavlink-tlog = "0.1.1" mcap = "0.25" mime_guess = "2" parquet = "58.3.0" diff --git a/rust/crates/sift_stream/Cargo.toml b/rust/crates/sift_stream/Cargo.toml index 6770105d77..e6af809cc0 100644 --- a/rust/crates/sift_stream/Cargo.toml +++ b/rust/crates/sift_stream/Cargo.toml @@ -50,6 +50,8 @@ async-trait = { workspace = true } criterion = { workspace = true } dotenvy = { workspace = true } hyper-util = { workspace = true } +mavlink = { workspace = true } +mavlink-tlog = { workspace = true } rand = { workspace = true } tempdir = { workspace = true } tokio = { workspace = true, features = ["rt", "rt-multi-thread", "sync", "time", "signal"] } @@ -74,6 +76,10 @@ path = "examples/auto-register/main.rs" name = "backups-only" path = "examples/backups-only/main.rs" +[[example]] +name = "mavlink-tlog" +path = "examples/mavlink-tlog/main.rs" + [[example]] name = "quick-start" path = "examples/quick-start/main.rs" diff --git a/rust/crates/sift_stream/examples/mavlink-tlog/main.rs b/rust/crates/sift_stream/examples/mavlink-tlog/main.rs new file mode 100644 index 0000000000..9ff42bd4f4 --- /dev/null +++ b/rust/crates/sift_stream/examples/mavlink-tlog/main.rs @@ -0,0 +1,308 @@ +// Replays a drone telemetry log (.tlog) into Sift at the times it was recorded. +// +// QGroundControl, Mission Planner and MAVProxy save a vehicle's MAVLink link as a .tlog: every +// frame behind the microsecond time the ground station received it. This example reads the frames +// with `mavlink-tlog`, decodes them with `mavlink`, and sends position, attitude and battery flows +// with each frame's original timestamp. A `link` flow adds the packet loss and duplicate counts +// of the vehicle's link, worked out from the MAVLink sequence numbers. +// +// SIFT_API_KEY=... SIFT_URI=... cargo run --example mavlink-tlog -- flight.tlog +// +// MAVLINK_SYSTEM_ID picks the vehicle (default 1, the ArduPilot and PX4 default). +// SIFT_ASSET_NAME names the Sift asset (default "mavlink-vehicle"). + +use mavlink::dialects::common::MavMessage; +use mavlink_tlog::{LinkStats, TlogReader}; +use sift_stream::{ + ChannelConfig, ChannelDataType, ChannelValue, Credentials, Flow, FlowConfig, + IngestionConfigEncoder, IngestionConfigForm, LiveStreamingWithBackups, RunForm, SiftStream, + SiftStreamBuilder, TimeValue, +}; +use std::{ + env, + error::Error, + fs::File, + io::{BufReader, Read}, + path::Path, + process::ExitCode, + time::{SystemTime, UNIX_EPOCH}, +}; +use tracing_subscriber::filter::EnvFilter; + +/// How often, in log time, to send the `link` flow. +const LINK_INTERVAL_US: u64 = 1_000_000; + +#[tokio::main] +async fn main() -> ExitCode { + tracing_subscriber::fmt() + .with_target(false) + .with_env_filter(EnvFilter::from_default_env()) + .init(); + + match run().await { + Ok(()) => ExitCode::SUCCESS, + Err(err) => { + eprintln!("{err}"); + ExitCode::FAILURE + } + } +} + +fn channel(name: &str, data_type: ChannelDataType, unit: &str, description: &str) -> ChannelConfig { + ChannelConfig { + name: name.into(), + data_type: data_type.into(), + unit: unit.into(), + description: description.into(), + ..Default::default() + } +} + +fn flows() -> Vec { + use ChannelDataType::{Double, Uint64}; + vec![ + FlowConfig { + name: "GLOBAL_POSITION_INT".into(), + channels: vec![ + channel("lat", Double, "deg", "latitude, WGS84"), + channel("lon", Double, "deg", "longitude, WGS84"), + channel("alt", Double, "m", "altitude above mean sea level"), + channel("relative_alt", Double, "m", "altitude above home"), + channel("vx", Double, "m/s", "ground speed north"), + channel("vy", Double, "m/s", "ground speed east"), + channel("vz", Double, "m/s", "ground speed down"), + channel("hdg", Double, "deg", "heading, empty when unknown"), + ], + }, + FlowConfig { + name: "ATTITUDE".into(), + channels: vec![ + channel("roll", Double, "rad", "roll angle"), + channel("pitch", Double, "rad", "pitch angle"), + channel("yaw", Double, "rad", "yaw angle"), + channel("rollspeed", Double, "rad/s", "roll rate"), + channel("pitchspeed", Double, "rad/s", "pitch rate"), + channel("yawspeed", Double, "rad/s", "yaw rate"), + ], + }, + FlowConfig { + name: "SYS_STATUS".into(), + channels: vec![ + channel("voltage_battery", Double, "V", "battery voltage"), + channel("current_battery", Double, "A", "battery current"), + channel("battery_remaining", Double, "%", "battery charge left"), + ], + }, + FlowConfig { + name: "link".into(), + channels: vec![ + channel( + "received", + Uint64, + "", + "frames received from the vehicle so far", + ), + channel("lost", Uint64, "", "frames lost so far, from sequence gaps"), + channel("duplicates", Uint64, "", "frames received twice so far"), + channel("loss", Double, "%", "lost / (received + lost)"), + ], + }, + ] +} + +/// Converts one decoded message to a flow. Values are converted to degrees, metres, m/s, volts, +/// amps and percent. Fields the vehicle marks as unknown are left empty. +fn to_flow(message: &MavMessage, time: TimeValue) -> Option { + let flow = match message { + MavMessage::GLOBAL_POSITION_INT(m) => { + let mut values = vec![ + ChannelValue::new("lat", f64::from(m.lat) / 1e7), + ChannelValue::new("lon", f64::from(m.lon) / 1e7), + ChannelValue::new("alt", f64::from(m.alt) / 1e3), + ChannelValue::new("relative_alt", f64::from(m.relative_alt) / 1e3), + ChannelValue::new("vx", f64::from(m.vx) / 100.0), + ChannelValue::new("vy", f64::from(m.vy) / 100.0), + ChannelValue::new("vz", f64::from(m.vz) / 100.0), + ]; + if m.hdg != u16::MAX { + values.push(ChannelValue::new("hdg", f64::from(m.hdg) / 100.0)); + } + Flow::new("GLOBAL_POSITION_INT", time, &values) + } + MavMessage::ATTITUDE(m) => Flow::new( + "ATTITUDE", + time, + &[ + ChannelValue::new("roll", f64::from(m.roll)), + ChannelValue::new("pitch", f64::from(m.pitch)), + ChannelValue::new("yaw", f64::from(m.yaw)), + ChannelValue::new("rollspeed", f64::from(m.rollspeed)), + ChannelValue::new("pitchspeed", f64::from(m.pitchspeed)), + ChannelValue::new("yawspeed", f64::from(m.yawspeed)), + ], + ), + MavMessage::SYS_STATUS(m) => { + let mut values = Vec::new(); + if m.voltage_battery != u16::MAX { + values.push(ChannelValue::new( + "voltage_battery", + f64::from(m.voltage_battery) / 1e3, + )); + } + if m.current_battery != -1 { + values.push(ChannelValue::new( + "current_battery", + f64::from(m.current_battery) / 100.0, + )); + } + if m.battery_remaining != -1 { + values.push(ChannelValue::new( + "battery_remaining", + f64::from(m.battery_remaining), + )); + } + Flow::new("SYS_STATUS", time, &values) + } + _ => return None, + }; + Some(flow) +} + +/// Loss and duplicate counts over every component of the vehicle. +fn link_flow(stats: &LinkStats, system_id: u8, time: TimeValue) -> Flow { + let (mut received, mut lost, mut duplicates) = (0u64, 0u64, 0u64); + for ((sys, _), s) in stats.sources() { + if sys == system_id { + received += s.received; + lost += s.lost; + duplicates += s.duplicates; + } + } + let loss = if received + lost == 0 { + 0.0 + } else { + 100.0 * lost as f64 / (received + lost) as f64 + }; + Flow::new( + "link", + time, + &[ + ChannelValue::new("received", received), + ChannelValue::new("lost", lost), + ChannelValue::new("duplicates", duplicates), + ChannelValue::new("loss", loss), + ], + ) +} + +async fn run() -> Result<(), Box> { + let path = env::args() + .nth(1) + .ok_or("usage: cargo run --example mavlink-tlog -- ")?; + let system_id: u8 = env::var("MAVLINK_SYSTEM_ID").map_or(Ok(1), |id| id.parse())?; + let asset_name = env::var("SIFT_ASSET_NAME").unwrap_or_else(|_| "mavlink-vehicle".into()); + + let credentials = Credentials::Config { + apikey: env::var("SIFT_API_KEY").map_err(|_| "SIFT_API_KEY is not set")?, + uri: env::var("SIFT_URI").map_err(|_| "SIFT_URI is not set")?, + }; + + let ts = SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis(); + let file_name = Path::new(&path) + .file_name() + .map_or(path.clone(), |f| f.to_string_lossy().into_owned()); + + let mut sift_stream = SiftStreamBuilder::new(credentials) + .ingestion_config(IngestionConfigForm { + asset_name: asset_name.clone(), + client_key: format!("{asset_name}-mavlink-tlog-v1"), + flows: flows(), + }) + .attach_run(RunForm { + name: format!("[{asset_name}].{file_name}.{ts}"), + client_key: format!("{asset_name}-tlog-{ts}"), + description: Some(format!("replay of {file_name}")), + tags: Some(vec!["mavlink".into(), "tlog".into()]), + metadata: None, + }) + .live_with_backups() + .build() + .await?; + + let file = BufReader::new(File::open(&path)?); + let (sent, stats) = replay(&mut sift_stream, file, system_id).await?; + + println!("{file_name}: {sent} flows sent"); + for ((sys, comp), s) in stats.sources() { + println!( + " sender {sys}/{comp}: {} received, {} lost ({:.2}%), {} duplicates", + s.received, + s.lost, + s.loss_ratio() * 100.0, + s.duplicates + ); + } + + // Gracefully terminate your stream + sift_stream.finish().await?; + Ok(()) +} + +/// Sends every frame of `system_id` in the log and returns the number of flows sent and the link +/// statistics. +async fn replay( + sift_stream: &mut SiftStream, + tlog: R, + system_id: u8, +) -> Result<(u64, LinkStats), Box> { + let mut log = TlogReader::new(tlog); + let mut stats = LinkStats::default(); + let mut next_link_us = 0; + let mut last_time = None; + let mut sent = 0; + + while let Some(record) = log.next_record()? { + if record.frame.sys_id() != system_id { + continue; + } + stats.observe(record.timestamp_us, &record.frame); + + // The time the ground station received the frame. + let time = TimeValue::try_from_timestamp_micros(i64::try_from(record.timestamp_us)?)?; + + if record.timestamp_us >= next_link_us { + sift_stream + .send(link_flow(&stats, system_id, time.clone())) + .await?; + sent += 1; + next_link_us = record.timestamp_us + LINK_INTERVAL_US; + } + last_time = Some(time.clone()); + + // Frames from messages outside the common dialect (such as ardupilotmega) are skipped. + let Ok((_, message)) = + mavlink::MavlinkReader::new(record.frame.as_bytes()).read_any_message::() + else { + continue; + }; + if let Some(flow) = to_flow(&message, time) { + sift_stream.send(flow).await?; + sent += 1; + } + } + + // Final totals at the time of the last frame. + if let Some(time) = last_time { + sift_stream.send(link_flow(&stats, system_id, time)).await?; + sent += 1; + } + + let read = log.stats(); + if read.skipped_bytes > 0 || read.bad_crc > 0 { + eprintln!( + "damaged log: {} bytes skipped, {} frames with bad checksums", + read.skipped_bytes, read.bad_crc + ); + } + Ok((sent, stats)) +}