From 4276ee0828f1e0192ea1357647a1861181dbcf8c Mon Sep 17 00:00:00 2001 From: Valentin PLANES Date: Sun, 2 Aug 2026 20:15:27 +0200 Subject: [PATCH 1/2] Count the items riding belts in the item search An item search summed every inventory in the save but nothing in transit, so a running factory under-reported by everything on its belt network -- half the iron ingots on one 54MB test save (166k of 331k), 559k items on another. Since 1.0 a belt line's items live in one ring buffer on the shared FGConveyorChainActor, each member belt owning a contiguous window of it. item_location_index now walks those windows and counts each item against the belt it physically rides -- not the chain actor, which has no footprint -- so every location the search lists stays a real placed building at a real position, and belts group next to machines and containers in the modal like any other holder. Pre-1.0 saves keep items on the belt actor itself; those count too. The ring arithmetic moves into queries::conveyor_chain_segment_window, shared with the belt tooltip's per-segment item list rather than copied. Tests: the index agrees belt-by-belt with aggregateSelectionInventory (the independent path behind the tooltip and selection panel), and the per-belt windows tile each chain's ring exactly -- no slot double-counted into two belts, none dropped. Co-Authored-By: Claude Opus 5 (1M context) --- map/static/map/finditem.js | 5 +- rust_parser/core/src/extract.rs | 101 +++++++++++++++---- rust_parser/core/src/mapdata/queries.rs | 46 ++++++--- rust_parser/core/tests/item_locations.rs | 121 +++++++++++++++++++++++ 4 files changed, 241 insertions(+), 32 deletions(-) create mode 100644 rust_parser/core/tests/item_locations.rs diff --git a/map/static/map/finditem.js b/map/static/map/finditem.js index 5b42367..97b62a6 100644 --- a/map/static/map/finditem.js +++ b/map/static/map/finditem.js @@ -8,9 +8,10 @@ // trains are searchable only as one whole-consist "Train" entry, matching // the sidebar, never as individual locomotives/freight cars). // -// Item search looks up one item across every inventory in the save (see +// Item search looks up one item across every inventory in the save -- plus +// the items riding the belts, which are stock like any other (see // sav_map_data.findItemLocations / collectItemLocationIndex, queried via -// /api/find-item) and lists everything that holds it, grouped by building +// /api/find-item) -- and lists everything that holds it, grouped by building // type (one expandable summed row per type, most first -- see // buildDisplayGroups/renderGroupedLocations), with an optional toggle to // highlight (and hide everything else) just those buildings on the map. diff --git a/rust_parser/core/src/extract.rs b/rust_parser/core/src/extract.rs index 12ae031..8787a67 100644 --- a/rust_parser/core/src/extract.rs +++ b/rust_parser/core/src/extract.rs @@ -4,6 +4,7 @@ //! (named in the doc comment); tools/diff_payload.py compares the two //! implementations and is the regression gate. +use crate::mapdata::queries::conveyor_chain_segment_window; use crate::mapdata::scan::SaveScan; use crate::store::*; use std::collections::{HashMap, HashSet}; @@ -69,10 +70,33 @@ struct ItemIndex<'a> { by_name: HashMap<&'a [u8], usize>, } -/// Exact port of sav_map_data._collectItemLocationIndex (+ its helpers -/// _inventoryComponentObjects / the stack walk). Returns -/// [(itemShortName, [(instanceName, count)])] in the same insertion order the -/// Python implementation's dict produces. +impl<'a> ItemIndex<'a> { + fn push(&mut self, short: &'a [u8], slot: usize, count: i64) { + match self.by_name.get(short) { + Some(&idx) => self.order[idx].1.push((slot, count)), + None => { + self.by_name.insert(short, self.order.len()); + self.order.push((short, vec![(slot, count)])); + } + } + } +} + +/// `countByItem[short] += n` over one object's insertion-ordered assoc list +/// (a handful of distinct items per object -- a linear scan beats a map). +fn add_item_count<'a>(counts: &mut Vec<(&'a [u8], i64)>, short: &'a [u8], n: i64) { + match counts.iter_mut().find(|(s, _)| *s == short) { + Some((_, total)) => *total += n, + None => counts.push((short, n)), + } +} + +/// Port of sav_map_data._collectItemLocationIndex (+ its helpers +/// _inventoryComponentObjects / the stack walk), extended with the items +/// riding the belt lines -- stock the Python version never counted, so an +/// item search under-reported a running factory by everything in transit. +/// Returns [(itemShortName, [(instanceName, count)])] in the same insertion +/// order the Python implementation's dict produces. pub fn item_location_index(scan: &SaveScan) -> Vec<(Vec, Vec<(Vec, i64)>)> { let store = scan.store; let data: &[u8] = &store.data; @@ -124,11 +148,7 @@ pub fn item_location_index(scan: &SaveScan) -> Vec<(Vec, Vec<(Vec, i64)> }; for stack in stacks { if let Some((item_path, n)) = stack_item(stack, data) { - let short = short_name(item_path); - match count_by_item.iter_mut().find(|(s, _)| *s == short) { - Some((_, total)) => *total += n, - None => count_by_item.push((short, n)), - } + add_item_count(&mut count_by_item, short_name(item_path), n); } } } @@ -138,20 +158,65 @@ pub fn item_location_index(scan: &SaveScan) -> Vec<(Vec, Vec<(Vec, i64)> find_prop(&object.properties, data, b"mPickupItems") { if let Some((item_path, n)) = stack_item(pl, data) { - let short = short_name(item_path); - match count_by_item.iter_mut().find(|(s, _)| *s == short) { - Some((_, total)) => *total += n, - None => count_by_item.push((short, n)), + add_item_count(&mut count_by_item, short_name(item_path), n); + } + } + + // Items riding a conveyor are real stock too -- a late-game belt + // network holds a lot of it -- so they count towards the item search + // exactly like a machine's or a container's contents. Pre-1.0 saves + // (and the chained-belt delete write-back, see editor::apply) keep a + // belt's items on the belt itself, one entity per record. + if let ActorSpecific::ConveyorBelt { items, .. } = &object.actor_specific { + for (_, item_path, _) in items { + let path = item_path.bytes(data); + if !path.is_empty() { + add_item_count(&mut count_by_item, short_name(path), 1); } } } for (short, count) in count_by_item { - match index.by_name.get(short) { - Some(&idx) => index.order[idx].1.push((slot, count)), - None => { - index.by_name.insert(short, index.order.len()); - index.order.push((short, vec![(slot, count)])); + index.push(short, slot, count); + } + + // Since 1.0 a whole belt line's items instead live in one ring buffer + // on the shared FGConveyorChainActor, each member belt owning a + // contiguous window of it. Counted against the belt the item is + // physically on -- not against the chain actor, which is an invisible + // bookkeeping object with no footprint -- so every location the search + // lists is a real placed building at a real position. Same split the + // belt tooltip's "items on this segment" and the selection panel's + // combined inventory already show. + if let ActorSpecific::ConveyorChain { + belts, items, maximum_items, chain_lead_item_index, .. + } = &object.actor_specific + { + // The member belts' windows tile `items` exactly -- no slot + // counted twice, none missed -- so a line's total across its + // belts is exactly the length of the ring. + let mut belt_counts: Vec<(&[u8], i64)> = Vec::new(); + for chain_belt in belts { + let Some((start, count)) = conveyor_chain_segment_window( + chain_belt, + *maximum_items, + *chain_lead_item_index, + ) else { + continue; + }; + let Some(&belt_slot) = slot_by_name.get(chain_belt.belt.path_name.bytes(data)) + else { + continue; + }; + belt_counts.clear(); + for (item_path, _) in items.iter().skip(start).take(count) { + let path = item_path.bytes(data); + if !path.is_empty() { + add_item_count(&mut belt_counts, short_name(path), 1); + } + } + for &(short, n) in &belt_counts { + index.push(short, belt_slot, n); } } } diff --git a/rust_parser/core/src/mapdata/queries.rs b/rust_parser/core/src/mapdata/queries.rs index 5ea0cd7..36a3b07 100644 --- a/rust_parser/core/src/mapdata/queries.rs +++ b/rust_parser/core/src/mapdata/queries.rs @@ -193,10 +193,36 @@ fn add_item( // Conveyor chain ring buffer (sav_map_data 2284-2309) // --------------------------------------------------------------------------- +/// One member belt's own contiguous window `[start, start + count)` into its +/// chain's ring-buffer item list, or None when the chain's/belt's indices say +/// there is nothing to read. Python's `%` on possibly negative operands == +/// rem_euclid. Shared by every consumer of a chain's items (the belt tooltip +/// below, the item-location index in extract.rs) so the ring arithmetic lives +/// in exactly one place. +pub fn conveyor_chain_segment_window( + chain_belt: &ChainBelt, + maximum_items: i32, + chain_lead_item_index: i32, +) -> Option<(usize, usize)> { + if maximum_items <= 0 + || chain_lead_item_index < 0 + || chain_belt.lead_item_index < 0 + || chain_belt.tail_item_index < 0 + { + return None; + } + let maximum = maximum_items as i64; + let start = (chain_belt.lead_item_index as i64 - chain_lead_item_index as i64) + .rem_euclid(maximum) as usize; + let count = (chain_belt.tail_item_index as i64 - chain_belt.lead_item_index as i64) + .rem_euclid(maximum) as usize + + 1; + Some((start, count)) +} + /// sav_map_data._conveyorChainSegmentItemPaths: this belt's own contiguous -/// slice of the chain's ring-buffer item window. Python's `%` on possibly -/// negative operands == rem_euclid; the slice `chainItems[start:start+count]` -/// clamps at the end like Python slicing. +/// slice of the chain's ring-buffer item window. The slice +/// `chainItems[start:start+count]` clamps at the end like Python slicing. pub fn conveyor_chain_segment_item_paths<'a>( // NOT tied to 'a: the returned slices borrow only `data` (StrRefs are // offsets), so an owned re-parsed chain actor can be a short-lived local. @@ -209,22 +235,18 @@ pub fn conveyor_chain_segment_item_paths<'a>( else { return Vec::new(); }; - if items.is_empty() || *maximum_items <= 0 || *chain_lead_item_index < 0 { + if items.is_empty() { return Vec::new(); } for chain_belt in belts { if chain_belt.belt.path_name.bytes(data) != belt_instance_name { continue; } - if chain_belt.lead_item_index < 0 || chain_belt.tail_item_index < 0 { + let Some((start, count)) = + conveyor_chain_segment_window(chain_belt, *maximum_items, *chain_lead_item_index) + else { return Vec::new(); - } - let maximum = *maximum_items as i64; - let start = (chain_belt.lead_item_index as i64 - *chain_lead_item_index as i64) - .rem_euclid(maximum) as usize; - let count = (chain_belt.tail_item_index as i64 - chain_belt.lead_item_index as i64) - .rem_euclid(maximum) as usize - + 1; + }; return items.iter().skip(start).take(count).map(|(path, _)| path.bytes(data)).collect(); } Vec::new() diff --git a/rust_parser/core/tests/item_locations.rs b/rust_parser/core/tests/item_locations.rs new file mode 100644 index 0000000..ce753de --- /dev/null +++ b/rust_parser/core/tests/item_locations.rs @@ -0,0 +1,121 @@ +//! Item-location index tests, focused on the items riding conveyors -- the +//! one class of stock that lives outside an inventory component and so has to +//! be gathered from the belt lines themselves (extract::item_location_index). + +use sav_core::level::parse_full_save; +use sav_core::mapdata::queries::conveyor_chain_segment_window; +use sav_core::mapdata::{self, index::MapIndex, queries}; +use sav_core::object::ClassTables; +use sav_core::store::{ActorSpecific, SaveStore}; +use std::collections::{HashMap, HashSet}; +use std::path::PathBuf; + +fn load(name: &str) -> (SaveStore, MapIndex) { + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../map/uploads").join(name); + let bytes = std::fs::read(path).expect("test save present"); + let store = parse_full_save(&bytes, &ClassTables::embedded(), None).unwrap(); + let (_payload, index) = mapdata::build_all_json(&store, None).unwrap(); + (store, index) +} + +fn belt_instance_names(store: &SaveStore, index: &MapIndex) -> Vec { + let belt_types: HashSet<&str> = + sav_core::gamedata::get().type_paths.conveyor_belts.iter().map(String::as_str).collect(); + let mut names = Vec::new(); + for type_path in index.instance_slots_by_type_path.keys() { + if belt_types.contains(type_path.as_str()) { + names.extend(index.instance_names_for_type_path(store, type_path)); + } + } + names +} + +/// The item search must agree, belt by belt, with the independent per-belt +/// path the selection panel and the belt tooltip already use -- and must +/// actually find items there, not quietly contribute nothing. +#[test] +fn belt_items_match_the_per_belt_inventory_path() { + for name in ["All_080726-163150.sav", "solo_autosave_1.sav"] { + let (store, index) = load(name); + let belts = belt_instance_names(&store, &index); + let belt_set: HashSet<&str> = belts.iter().map(String::as_str).collect(); + + // (belt, itemShortName) -> count, as the item search sees it. + let mut from_index: HashMap<(String, String), i64> = HashMap::new(); + for (short, entries) in &index.item_location_index { + let short = String::from_utf8_lossy(short).into_owned(); + for (instance_name, count) in entries { + let instance_name = String::from_utf8_lossy(instance_name).into_owned(); + if belt_set.contains(instance_name.as_str()) { + *from_index.entry((instance_name, short.clone())).or_insert(0) += count; + } + } + } + + // ... and as aggregateSelectionInventory sees it. + let mut from_selection: HashMap<(String, String), i64> = HashMap::new(); + for belt in &belts { + for row in queries::aggregate_selection_inventory(&store, &index, &[belt.as_str()]) + .as_array() + .unwrap() + { + let item = row["item"].as_str().unwrap().to_string(); + let count = row["count"].as_f64().unwrap() as i64; + *from_selection.entry((belt.clone(), item)).or_insert(0) += count; + } + } + + assert_eq!(from_index, from_selection, "{name}: belt contents disagree"); + let total: i64 = from_index.values().sum(); + assert!(total > 0, "{name}: the index found no items on any belt at all"); + } +} + +/// Since 1.0 a belt line's items live in one ring buffer on the shared chain +/// actor, split between its member belts by index arithmetic. Attributing +/// each item to the belt it rides is only correct if those windows tile the +/// ring exactly: an overlap would double-count an item into two belts, a gap +/// would drop it from the search entirely. +#[test] +fn chain_item_windows_tile_the_ring_exactly() { + for name in ["All_080726-163150.sav", "solo_autosave_1.sav"] { + let (store, _index) = load(name); + let mut chains_with_items = 0usize; + for level in &store.levels { + for object in level.parsed_objects() { + let ActorSpecific::ConveyorChain { + belts, items, maximum_items, chain_lead_item_index, .. + } = &object.actor_specific + else { + continue; + }; + if items.is_empty() { + continue; + } + chains_with_items += 1; + let mut claims = vec![0u32; items.len()]; + for chain_belt in belts { + let Some((start, count)) = + conveyor_chain_segment_window(chain_belt, *maximum_items, *chain_lead_item_index) + else { + continue; // Belt holding nothing (game writes -1 indices). + }; + assert!( + start + count <= items.len(), + "{name}: belt window {start}+{count} overruns a {}-slot ring", + items.len() + ); + for claim in claims.iter_mut().skip(start).take(count) { + *claim += 1; + } + } + assert!( + claims.iter().all(|&c| c == 1), + "{name}: ring slots claimed by {:?} belts, expected exactly one each", + claims.iter().collect::>() + ); + } + } + assert!(chains_with_items > 0, "{name}: no loaded belt lines to check"); + } +} From d766800e97690db094de78fcecd5cdbae982908b Mon Sep 17 00:00:00 2001 From: Valentin PLANES Date: Fri, 4 Sep 2026 17:27:22 +0200 Subject: [PATCH 2/2] Read the per-item state riding a conveyor A save with anything stateful parked on a belt -- a jetpack with fuel, a loaded weapon, a gas mask with a filter -- failed to load outright: object re-parse failed during index build: object at (3861, 109202): Value 1172451892 at offset 663613945 does not match the expected 0. Since save version 44 an item's slot carries an `itemState` flag and, when set, an inline record: an empty level name, the state's type path, and its length-prefixed properties. The per-belt item format already read that; the conveyor CHAIN format -- which is where every belt item actually lives since 1.0 -- asserted the flag was zero instead, so the first geared-up item on a belt turned the rest of the object into drift and took the whole save with it. Reported in #22, reproduced on a save of belts hung with jetpacks, gas masks and hazmat suits (now map/uploads/belt_item_state.sav). Both formats now skip the record through one helper and keep its span, so the chained-belt delete write-back hands the bytes back verbatim: a jetpack that survives a cut line comes back with its fuel, not empty. Chain and belt item slots become named structs rather than tuples, which is also where the chain slot's trailing float gets its real name -- it is a distance along the line, not an instance id (consecutive gallery items sat exactly 120cm apart). Verified: every map payload in the corpus is byte-identical to before, the save-index differs only by the belt items the search now counts (nothing lost, no total decreased), and the repro save loads in the app with its belt items listed in the item search. Fixes #22 Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/ci.yml | 2 +- CONTRIBUTING.md | 2 +- rust_parser/core/examples/dump_belt_items.rs | 10 +- rust_parser/core/src/editor/apply.rs | 54 ++++++-- rust_parser/core/src/extract.rs | 8 +- rust_parser/core/src/mapdata/describe.rs | 2 +- rust_parser/core/src/mapdata/queries.rs | 7 +- rust_parser/core/src/object.rs | 54 +++++--- rust_parser/core/src/store.rs | 30 +++- rust_parser/core/tests/belt_item_state.rs | 138 +++++++++++++++++++ tools/fetch_test_saves.py | 7 +- 11 files changed, 275 insertions(+), 39 deletions(-) create mode 100644 rust_parser/core/tests/belt_item_state.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7118bbb..fe057b7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,7 +38,7 @@ jobs: uses: actions/cache@v4 with: path: map/uploads - key: test-saves-v1 + key: test-saves-v2 - name: Fetch test saves run: python3 tools/fetch_test_saves.py - name: cargo test (release) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 19125b7..da59fa7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -104,7 +104,7 @@ The Rust suite reads real save files from `map/uploads/` (gitignored — saves are tens of MB). Fetch the public corpus first, then test: ```bash -py tools/fetch_test_saves.py # downloads the test-saves-v1 release assets +py tools/fetch_test_saves.py # downloads the test-saves-v2 release assets cd rust_parser cargo test -p sav_core --release # release: debug parses of 50MB saves crawl ``` diff --git a/rust_parser/core/examples/dump_belt_items.rs b/rust_parser/core/examples/dump_belt_items.rs index 1fa0d5c..6a70a1b 100644 --- a/rust_parser/core/examples/dump_belt_items.rs +++ b/rust_parser/core/examples/dump_belt_items.rs @@ -33,8 +33,14 @@ fn main() { .instance_name() .to_string(&store.data); println!("{arg}: {name} ({} items):", items.len()); - for (len, path, pos) in items.iter().take(10) { - println!(" u32={} pos={} {}", len, pos, path.to_string(&store.data)); + for item in items.iter().take(10) { + println!( + " u32={} pos={} state={} {}", + item.length, + item.position, + item.state.map_or(0, |s| s.len), + item.item_path.to_string(&store.data) + ); } } } diff --git a/rust_parser/core/src/editor/apply.rs b/rust_parser/core/src/editor/apply.rs index 3530ed5..acf2857 100644 --- a/rust_parser/core/src/editor/apply.rs +++ b/rust_parser/core/src/editor/apply.rs @@ -1089,13 +1089,15 @@ pub(crate) fn lightweight_record_bytes( /// format -- chain actors only exist in saves past that gate): [u32 0 /// InventoryItem padding][item class string][u32 0 no item state] /// [f32 position along the belt, cm]. -fn belt_item_record(path: &[u8], position: f32) -> Vec { - let mut r = Vec::with_capacity(path.len() + 17); +fn belt_item_record(path: &[u8], state: &[u8], position: f32) -> Vec { + let mut r = Vec::with_capacity(path.len() + state.len() + 17); r.extend_from_slice(&0u32.to_le_bytes()); r.extend_from_slice(&(path.len() as u32 + 1).to_le_bytes()); r.extend_from_slice(path); r.push(0); - r.extend_from_slice(&0u32.to_le_bytes()); + // itemState: the has-state flag, then that record's bytes unchanged. + r.extend_from_slice(&(!state.is_empty() as u32).to_le_bytes()); + r.extend_from_slice(state); r.extend_from_slice(&position.to_le_bytes()); r } @@ -1175,14 +1177,18 @@ fn plan_delete_actors( let count = (cb.tail_item_index as i64 - cb.lead_item_index as i64) .rem_euclid(maximum) as usize + 1; - let slots: Vec<(usize, &[u8])> = items + let slots: Vec<(usize, &[u8], &[u8])> = items .iter() .skip(start) .take(count) .enumerate() - .filter_map(|(j, (p, _))| { - let b = p.bytes(data); - (!b.is_empty() && b.is_ascii()).then_some((j, b)) + .filter_map(|(j, item)| { + let b = item.item_path.bytes(data); + // The per-item state (a jetpack's fuel, a weapon's + // magazine) rides along verbatim -- same record layout + // on a belt as in the chain's ring. + let st = item.state.map_or(&[][..], |s| s.bytes(data)); + (!b.is_empty() && b.is_ascii()).then_some((j, b, st)) }) .collect(); if slots.is_empty() { @@ -1207,9 +1213,9 @@ fn plan_delete_actors( belt_len = 100.0; } let mut records = Vec::new(); - for &(j, path) in &slots { + for &(j, path, state) in &slots { let position = (belt_len * (j as f64 + 0.5) / count as f64) as f32; - records.extend_from_slice(&belt_item_record(path, position)); + records.extend_from_slice(&belt_item_record(path, state, position)); } belt_writebacks.push(( bli, @@ -1449,6 +1455,36 @@ mod plan_apply_tests { plan } + /// A written-back belt item mirrors the slot the game itself writes: the + /// leading u32, the item class path, the itemState flag and (when the + /// item has one) its state record verbatim, then the position. Getting + /// the flag wrong here writes a save the game reads as drift -- the same + /// mistake that made stated items unreadable in the first place. + #[test] + fn belt_item_record_writes_the_item_state_slot() { + let path = b"/Game/X/Desc_Y.Desc_Y_C"; + + let plain = belt_item_record(path, &[], 12.5f32); + let mut expected = Vec::new(); + expected.extend_from_slice(&0u32.to_le_bytes()); + expected.extend_from_slice(&(path.len() as u32 + 1).to_le_bytes()); + expected.extend_from_slice(path); + expected.push(0); + expected.extend_from_slice(&0u32.to_le_bytes()); // no state + expected.extend_from_slice(&12.5f32.to_le_bytes()); + assert_eq!(plain, expected); + + // A state record rides along whole, behind a set flag. + let state = [0xEFu8; 40]; + let stated = belt_item_record(path, &state, 12.5f32); + assert_eq!(stated.len(), plain.len() + state.len()); + let flag_at = stated.len() - 4 - state.len() - 4; + assert_eq!(&stated[..flag_at], &plain[..flag_at]); + assert_eq!(&stated[flag_at..flag_at + 4], &1u32.to_le_bytes()); + assert_eq!(&stated[flag_at + 4..flag_at + 4 + state.len()], &state); + assert_eq!(&stated[stated.len() - 4..], &12.5f32.to_le_bytes()); + } + /// The streamed (compress-free-rebuild) path must produce byte-identical /// output to the in-place path for the same plan. #[test] diff --git a/rust_parser/core/src/extract.rs b/rust_parser/core/src/extract.rs index 8787a67..e056acb 100644 --- a/rust_parser/core/src/extract.rs +++ b/rust_parser/core/src/extract.rs @@ -168,8 +168,8 @@ pub fn item_location_index(scan: &SaveScan) -> Vec<(Vec, Vec<(Vec, i64)> // (and the chained-belt delete write-back, see editor::apply) keep a // belt's items on the belt itself, one entity per record. if let ActorSpecific::ConveyorBelt { items, .. } = &object.actor_specific { - for (_, item_path, _) in items { - let path = item_path.bytes(data); + for item in items { + let path = item.item_path.bytes(data); if !path.is_empty() { add_item_count(&mut count_by_item, short_name(path), 1); } @@ -209,8 +209,8 @@ pub fn item_location_index(scan: &SaveScan) -> Vec<(Vec, Vec<(Vec, i64)> continue; }; belt_counts.clear(); - for (item_path, _) in items.iter().skip(start).take(count) { - let path = item_path.bytes(data); + for item in items.iter().skip(start).take(count) { + let path = item.item_path.bytes(data); if !path.is_empty() { add_item_count(&mut belt_counts, short_name(path), 1); } diff --git a/rust_parser/core/src/mapdata/describe.rs b/rust_parser/core/src/mapdata/describe.rs index 8f7b3e3..fccb510 100644 --- a/rust_parser/core/src/mapdata/describe.rs +++ b/rust_parser/core/src/mapdata/describe.rs @@ -566,7 +566,7 @@ pub fn describe_instance(store: &SaveStore, index: &MapIndex, instance_name: &st } if belts.len() > 1 { let line_items = - counted_item_list(items.iter().map(|(path, _)| path.bytes(data))); + counted_item_list(items.iter().map(|item| item.item_path.bytes(data))); if !line_items.is_empty() { result.insert("itemsOnLine".into(), Value::Array(line_items)); result.insert("lineSegmentCount".into(), Value::from(belts.len() as i64)); diff --git a/rust_parser/core/src/mapdata/queries.rs b/rust_parser/core/src/mapdata/queries.rs index 36a3b07..a774b61 100644 --- a/rust_parser/core/src/mapdata/queries.rs +++ b/rust_parser/core/src/mapdata/queries.rs @@ -247,7 +247,12 @@ pub fn conveyor_chain_segment_item_paths<'a>( else { return Vec::new(); }; - return items.iter().skip(start).take(count).map(|(path, _)| path.bytes(data)).collect(); + return items + .iter() + .skip(start) + .take(count) + .map(|item| item.item_path.bytes(data)) + .collect(); } Vec::new() } diff --git a/rust_parser/core/src/object.rs b/rust_parser/core/src/object.rs index a12862b..5a18485 100644 --- a/rust_parser/core/src/object.rs +++ b/rust_parser/core/src/object.rs @@ -4,7 +4,7 @@ use crate::error::{perr, PResult}; use crate::properties::{parse_object_reference, parse_properties}; -use crate::reader::Cursor; +use crate::reader::{Cursor, DataRef}; use crate::store::*; use crate::version_data::parse_save_object_version_data; @@ -14,6 +14,29 @@ pub struct ClassTables { pub conveyor_belts: Vec, } +/// The optional per-item state record that follows an item's class path in a +/// conveyor item slot. Save version 44 replaced the old `itemState` +/// (levelName, pathName) reference pair with a flag plus, when set, an inline +/// record: an empty level name, the state's type path, and its +/// length-prefixed properties. Anything that carries state rides belts with +/// one attached -- a jetpack's fuel (FGJetPackItemState), a weapon's +/// magazine, a gas mask's filter -- and a save full of geared-up belts is +/// unreadable without this. +/// +/// The record is skipped, not decoded; the returned span covers it whole +/// (flag excluded) so a writer can hand the bytes back verbatim. +fn parse_item_state(c: &mut Cursor, ctx: &str) -> PResult> { + if !c.bool_u32(ctx)? { + return Ok(None); + } + let off = c.pos; + c.confirm_u32(0)?; // state ObjectReference.levelName + let _state_type = c.string()?; + let state_size = c.u32()?; + c.data_ref(state_size as usize)?; + Ok(Some(DataRef { off, len: (c.pos - off) as u32 })) +} + const GAME_MODE_STATE: [&str; 2] = [ "/Game/FactoryGame/-Shared/Blueprint/BP_GameMode.BP_GameMode_C", "/Game/FactoryGame/-Shared/Blueprint/BP_GameState.BP_GameState_C", @@ -172,21 +195,16 @@ pub fn parse_object( let v44_item_format = header_save_version >= 44 && object_game_version >= 44; for _ in 0..count { let length = c.u32()?; - let name = c.string()?; - if v44_item_format { - let has_state = c.bool_u32("ConveyorBelt.itemHasStateFlag")?; - if has_state { - c.confirm_u32(0)?; // state ObjectReference.Level - let _state_type = c.string()?; - let state_size = c.u32()?; - c.data_ref(state_size as usize)?; - } + let item_path = c.string()?; + let state = if v44_item_format { + parse_item_state(c, "ConveyorBelt.itemHasStateFlag")? } else { c.confirm_string("")?; c.confirm_string("")?; - } + None + }; let position = c.f32()?; - items.push((length, name, position)); + items.push(BeltItem { length, item_path, state, position }); } actor_specific = ActorSpecific::ConveyorBelt { items, count_field_off, end_off: c.pos }; @@ -399,12 +417,16 @@ pub fn parse_object( let chain_tail = c.i32()?; let num_items = c.u32()?; let mut items = Vec::with_capacity(c.capped_capacity(num_items as usize, 8)); + // Same slot layout as a per-belt item record, minus its + // leading length: an item class reference, then the optional + // state (chain actors only exist in 1.0+ saves, so the v44 + // format is the only one that can appear here). for _ in 0..num_items { - c.confirm_u32(0)?; + c.confirm_u32(0)?; // item class ObjectReference.levelName let item_path = c.string()?; - c.confirm_u32(0)?; - let item_instance_id = c.u32()?; - items.push((item_path, item_instance_id)); + let state = parse_item_state(c, "ConveyorChain.itemHasStateFlag")?; + let chain_offset = c.f32()?; + items.push(ChainItem { item_path, state, chain_offset }); } actor_specific = ActorSpecific::ConveyorChain { chain_actor, diff --git a/rust_parser/core/src/store.rs b/rust_parser/core/src/store.rs index c8bee48..53bf435 100644 --- a/rust_parser/core/src/store.rs +++ b/rust_parser/core/src/store.rs @@ -255,6 +255,32 @@ pub struct ChainBelt { pub tail_item_index: i32, } +/// One item riding a belt, in the per-belt record format (pre-chain saves and +/// the chained-belt delete write-back; see `ActorSpecific::ConveyorBelt`). +#[derive(Debug)] +pub struct BeltItem { + /// The record's leading u32, as the game wrote it. + pub length: u32, + pub item_path: StrRef, + /// Raw per-item state record, as on [`ChainItem`]. + pub state: Option, + /// Distance along the belt, in centimetres. + pub position: f32, +} + +/// One slot of a conveyor chain's ring buffer: the item riding it. +#[derive(Debug)] +pub struct ChainItem { + /// The item's class path (`/Game/.../Desc_IronPlate.Desc_IronPlate_C`). + pub item_path: StrRef, + /// Raw bytes of the item's per-item state record when it has one -- a + /// jetpack's fuel level, a weapon's magazine, a gas mask's filter. Kept + /// verbatim (never decoded) so a writer can copy it straight back out. + pub state: Option, + /// Distance along the chain, in centimetres. + pub chain_offset: f32, +} + #[derive(Debug)] pub enum ActorSpecific { None, @@ -264,7 +290,7 @@ pub enum ActorSpecific { /// delete engine writes records back here when a chain is deleted -- the /// game reads them when it rebuilds chains on load. ConveyorBelt { - items: Vec<(u32, StrRef, f32)>, + items: Vec, /// Offset in `SaveStore.data` of the belt's u32 item count. count_field_off: usize, /// Offset just past the last item record (insertion point). @@ -299,7 +325,7 @@ pub enum ActorSpecific { ConveyorChain { chain_actor: ObjectRef, belts: Vec, - items: Vec<(StrRef, u32)>, + items: Vec, cu32: u32, maximum_items: i32, chain_lead_item_index: i32, diff --git a/rust_parser/core/tests/belt_item_state.rs b/rust_parser/core/tests/belt_item_state.rs new file mode 100644 index 0000000..bd340b6 --- /dev/null +++ b/rust_parser/core/tests/belt_item_state.rs @@ -0,0 +1,138 @@ +//! Items with per-item state riding a conveyor: a jetpack's fuel level, a +//! weapon's magazine, a gas mask's filter. Their state record sits inline in +//! the conveyor's item slot, and a parser that assumes it away walks off the +//! end of the object and fails the whole save (issue #22) -- so a gallery of +//! gear on a belt is the exact shape these tests keep working. + +use sav_core::editor::ops::EditOp; +use sav_core::editor::{effective_body, export_sav, session}; +use sav_core::level::parse_full_save; +use sav_core::mapdata::scan::SaveScan; +use sav_core::mapdata::{self, index::MapIndex}; +use sav_core::object::ClassTables; +use sav_core::store::{ActorSpecific, SaveStore}; +use std::path::PathBuf; + +/// A save with equipment parked on a belt: jetpacks, gas masks, hazmat suits. +const SAVE: &str = "belt_item_state.sav"; + +fn load(name: &str) -> SaveStore { + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../map/uploads").join(name); + let bytes = std::fs::read(path).expect("test save present"); + parse_full_save(&bytes, &ClassTables::embedded(), None) + .unwrap_or_else(|e| panic!("{name}: parse failed: {}", e.msg)) +} + +/// Every state record on a conveyor, chain-owned or belt-owned, as raw bytes. +fn conveyor_item_states(store: &SaveStore) -> Vec<&[u8]> { + let data: &[u8] = &store.data; + let mut states = Vec::new(); + for level in &store.levels { + for object in level.parsed_objects() { + match &object.actor_specific { + ActorSpecific::ConveyorChain { items, .. } => { + states.extend(items.iter().filter_map(|i| i.state).map(|s| s.bytes(data))); + } + ActorSpecific::ConveyorBelt { items, .. } => { + states.extend(items.iter().filter_map(|i| i.state).map(|s| s.bytes(data))); + } + _ => {} + } + } + } + states +} + +/// A state record is an empty level name, the state's type path, and its +/// length-prefixed properties -- and the span we kept covers exactly that, +/// no more and no less. (Reading it back is what a writer does with these +/// bytes, so if the span is off by one the write path corrupts a save.) +fn assert_well_formed_state(state: &[u8]) { + assert!(state.len() > 12, "state record too short: {} bytes", state.len()); + assert_eq!(&state[..4], &0u32.to_le_bytes(), "state level name is not empty"); + let type_len = u32::from_le_bytes(state[4..8].try_into().unwrap()) as usize; + let type_end = 8 + type_len; + assert!(type_end + 4 <= state.len(), "state type path overruns the record"); + let type_path = String::from_utf8_lossy(&state[8..type_end - 1]).into_owned(); + assert!( + type_path.starts_with("/Script/FactoryGame.FG") || type_path.starts_with("/Game/"), + "implausible state type path {type_path:?}" + ); + let size = u32::from_le_bytes(state[type_end..type_end + 4].try_into().unwrap()) as usize; + assert_eq!( + type_end + 4 + size, + state.len(), + "state span does not end where its property block does ({type_path})" + ); +} + +/// The save parses at all -- the issue-#22 crash -- and every state record it +/// holds round-trips as the span we hand a writer. +#[test] +fn stated_items_on_conveyors_parse() { + let store = load(SAVE); + let states = conveyor_item_states(&store); + assert!(!states.is_empty(), "{SAVE} has no stated conveyor items left to gate on"); + for state in &states { + assert_well_formed_state(state); + } +} + +/// An item search must find the geared-up items too, listed against the belt +/// they ride. (They reach the index through the same path as plain items, so +/// this is really "state does not hide an item from the search".) +#[test] +fn stated_belt_items_are_searchable() { + let store = load(SAVE); + let (_payload, index): (_, MapIndex) = mapdata::build_all_json(&store, None).unwrap(); + let mut found = 0; + for (short, entries) in &index.item_location_index { + if !short.starts_with(b"BP_EquipmentDescriptor") { + continue; + } + for (instance_name, count) in entries { + if String::from_utf8_lossy(instance_name).contains("Build_Conveyor") { + assert!(*count > 0); + found += 1; + } + } + } + assert!(found > 0, "no stateful equipment found on a belt by the item search"); +} + +/// Deleting the belt that carries stated items takes the whole line with it +/// (a chain actor cannot outlive its belts), and the resulting save must +/// still export and re-parse -- the write-back path runs over these records +/// on the way out. The record layout it emits for a stated item is pinned by +/// editor::apply's belt_item_record_writes_the_item_state_slot. +#[test] +fn deleting_a_belt_of_stated_items_still_round_trips() { + let store = load(SAVE); + let tables = ClassTables::embedded(); + let data: &[u8] = &store.data; + + let mut belt = None; + for level in &store.levels { + for object in level.parsed_objects() { + let ActorSpecific::ConveyorChain { belts, items, .. } = &object.actor_specific else { + continue; + }; + if items.iter().any(|i| i.state.is_some()) { + belt = Some(belts[0].belt.path_name.to_string(data)); + break; + } + } + } + let belt = belt.expect("no chain carrying stated items"); + + let store2 = session::step(&store, &EditOp::DeleteActors { names: vec![belt.clone()] }, &tables) + .unwrap(); + assert!(!SaveScan::new(&store2).by_instance_name.contains_key(belt.as_bytes())); + + let exported = export_sav(&store2.file_header, effective_body(&store2)); + let reparsed = parse_full_save(&exported, &tables, None) + .unwrap_or_else(|e| panic!("exported save failed to re-parse: {}", e.msg)); + for state in conveyor_item_states(&reparsed) { + assert_well_formed_state(state); + } +} diff --git a/tools/fetch_test_saves.py b/tools/fetch_test_saves.py index 92f4227..0b9d04d 100644 --- a/tools/fetch_test_saves.py +++ b/tools/fetch_test_saves.py @@ -2,7 +2,7 @@ The Rust test suite (cargo test -p sav_core), tools/e2e_editor.py, and CI all read real .sav files from map/uploads/, which is gitignored (saves are tens of -MB). The corpus lives as assets on the `test-saves-v1` GitHub release -- same +MB). The corpus lives as assets on the `test-saves-v2` GitHub release -- same distribution mechanism as game_data.zip -- so a clean clone becomes testable with: @@ -18,13 +18,16 @@ REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) DEST = os.path.join(REPO, "map", "uploads") -BASE = "https://github.com/valentinps/satisfactorymap/releases/download/test-saves-v1/" +BASE = "https://github.com/valentinps/satisfactorymap/releases/download/test-saves-v2/" # name -> expected size (skip check; also catches truncated downloads) SAVES = { "All_080726-163150.sav": 263_439, "solo_autosave_1.sav": 15_068_544, "BuildITBIIIIIG_210726-231135.sav": 50_186_116, + # Equipment with per-item state parked on a belt (issue #22): jetpacks, + # gas masks, hazmat suits. Gates tests/belt_item_state.rs. + "belt_item_state.sav": 419_339, }