Read per-item state on conveyors, and count belt items in the item search

.github/workflows/ci.yml

@@ -38,7 +38,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: map/uploads
-          key: test-saves-v1
+          key: test-saves-v2
       - name: Fetch test saves
         run: python3 tools/fetch_test_saves.py
       - name: cargo test (release)

CONTRIBUTING.md

@@ -104,7 +104,7 @@ The Rust suite reads real save files from `map/uploads/` (gitignored — saves
 are tens of MB). Fetch the public corpus first, then test:
 
 ```bash
-py tools/fetch_test_saves.py     # downloads the test-saves-v1 release assets
+py tools/fetch_test_saves.py     # downloads the test-saves-v2 release assets
 cd rust_parser
 cargo test -p sav_core --release # release: debug parses of 50MB saves crawl
 ```

map/static/map/finditem.js

@@ -8,9 +8,10 @@
 // trains are searchable only as one whole-consist "Train" entry, matching
 // the sidebar, never as individual locomotives/freight cars).
 //
-// Item search looks up one item across every inventory in the save (see
+// Item search looks up one item across every inventory in the save -- plus
+// the items riding the belts, which are stock like any other (see
 // sav_map_data.findItemLocations / collectItemLocationIndex, queried via
-// /api/find-item) and lists everything that holds it, grouped by building
+// /api/find-item) -- and lists everything that holds it, grouped by building
 // type (one expandable summed row per type, most first -- see
 // buildDisplayGroups/renderGroupedLocations), with an optional toggle to
 // highlight (and hide everything else) just those buildings on the map.

rust_parser/core/examples/dump_belt_items.rs

@@ -33,8 +33,14 @@ fn main() {
                             .instance_name()
                             .to_string(&store.data);
                         println!("{arg}: {name} ({} items):", items.len());
-                        for (len, path, pos) in items.iter().take(10) {
-                            println!("  u32={} pos={} {}", len, pos, path.to_string(&store.data));
+                        for item in items.iter().take(10) {
+                            println!(
+                                "  u32={} pos={} state={} {}",
+                                item.length,
+                                item.position,
+                                item.state.map_or(0, |s| s.len),
+                                item.item_path.to_string(&store.data)
+                            );
                         }
                     }
                 }

rust_parser/core/src/editor/apply.rs

@@ -1089,13 +1089,15 @@ pub(crate) fn lightweight_record_bytes(
 /// format -- chain actors only exist in saves past that gate): [u32 0
 /// InventoryItem padding][item class string][u32 0 no item state]
 /// [f32 position along the belt, cm].
-fn belt_item_record(path: &[u8], position: f32) -> Vec<u8> {
-    let mut r = Vec::with_capacity(path.len() + 17);
+fn belt_item_record(path: &[u8], state: &[u8], position: f32) -> Vec<u8> {
+    let mut r = Vec::with_capacity(path.len() + state.len() + 17);
     r.extend_from_slice(&0u32.to_le_bytes());
     r.extend_from_slice(&(path.len() as u32 + 1).to_le_bytes());
     r.extend_from_slice(path);
     r.push(0);
-    r.extend_from_slice(&0u32.to_le_bytes());
+    // itemState: the has-state flag, then that record's bytes unchanged.
+    r.extend_from_slice(&(!state.is_empty() as u32).to_le_bytes());
+    r.extend_from_slice(state);
     r.extend_from_slice(&position.to_le_bytes());
     r
 }
@@ -1175,14 +1177,18 @@ fn plan_delete_actors(
             let count = (cb.tail_item_index as i64 - cb.lead_item_index as i64)
                 .rem_euclid(maximum) as usize
                 + 1;
-            let slots: Vec<(usize, &[u8])> = items
+            let slots: Vec<(usize, &[u8], &[u8])> = items
                 .iter()
                 .skip(start)
                 .take(count)
                 .enumerate()
-                .filter_map(|(j, (p, _))| {
-                    let b = p.bytes(data);
-                    (!b.is_empty() && b.is_ascii()).then_some((j, b))
+                .filter_map(|(j, item)| {
+                    let b = item.item_path.bytes(data);
+                    // The per-item state (a jetpack's fuel, a weapon's
+                    // magazine) rides along verbatim -- same record layout
+                    // on a belt as in the chain's ring.
+                    let st = item.state.map_or(&[][..], |s| s.bytes(data));
+                    (!b.is_empty() && b.is_ascii()).then_some((j, b, st))
                 })
                 .collect();
             if slots.is_empty() {
@@ -1207,9 +1213,9 @@ fn plan_delete_actors(
                 belt_len = 100.0;
             }
             let mut records = Vec::new();
-            for &(j, path) in &slots {
+            for &(j, path, state) in &slots {
                 let position = (belt_len * (j as f64 + 0.5) / count as f64) as f32;
-                records.extend_from_slice(&belt_item_record(path, position));
+                records.extend_from_slice(&belt_item_record(path, state, position));
             }
             belt_writebacks.push((
                 bli,
@@ -1449,6 +1455,36 @@ mod plan_apply_tests {
         plan
     }
 
+    /// A written-back belt item mirrors the slot the game itself writes: the
+    /// leading u32, the item class path, the itemState flag and (when the
+    /// item has one) its state record verbatim, then the position. Getting
+    /// the flag wrong here writes a save the game reads as drift -- the same
+    /// mistake that made stated items unreadable in the first place.
+    #[test]
+    fn belt_item_record_writes_the_item_state_slot() {
+        let path = b"/Game/X/Desc_Y.Desc_Y_C";
+
+        let plain = belt_item_record(path, &[], 12.5f32);
+        let mut expected = Vec::new();
+        expected.extend_from_slice(&0u32.to_le_bytes());
+        expected.extend_from_slice(&(path.len() as u32 + 1).to_le_bytes());
+        expected.extend_from_slice(path);
+        expected.push(0);
+        expected.extend_from_slice(&0u32.to_le_bytes()); // no state
+        expected.extend_from_slice(&12.5f32.to_le_bytes());
+        assert_eq!(plain, expected);
+
+        // A state record rides along whole, behind a set flag.
+        let state = [0xEFu8; 40];
+        let stated = belt_item_record(path, &state, 12.5f32);
+        assert_eq!(stated.len(), plain.len() + state.len());
+        let flag_at = stated.len() - 4 - state.len() - 4;
+        assert_eq!(&stated[..flag_at], &plain[..flag_at]);
+        assert_eq!(&stated[flag_at..flag_at + 4], &1u32.to_le_bytes());
+        assert_eq!(&stated[flag_at + 4..flag_at + 4 + state.len()], &state);
+        assert_eq!(&stated[stated.len() - 4..], &12.5f32.to_le_bytes());
+    }
+
     /// The streamed (compress-free-rebuild) path must produce byte-identical
     /// output to the in-place path for the same plan.
     #[test]

rust_parser/core/src/extract.rs

@@ -4,6 +4,7 @@
 //! (named in the doc comment); tools/diff_payload.py compares the two
 //! implementations and is the regression gate.
 
+use crate::mapdata::queries::conveyor_chain_segment_window;
 use crate::mapdata::scan::SaveScan;
 use crate::store::*;
 use std::collections::{HashMap, HashSet};
@@ -69,10 +70,33 @@ struct ItemIndex<'a> {
     by_name: HashMap<&'a [u8], usize>,
 }
 
-/// Exact port of sav_map_data._collectItemLocationIndex (+ its helpers
-/// _inventoryComponentObjects / the stack walk). Returns
-/// [(itemShortName, [(instanceName, count)])] in the same insertion order the
-/// Python implementation's dict produces.
+impl<'a> ItemIndex<'a> {
+    fn push(&mut self, short: &'a [u8], slot: usize, count: i64) {
+        match self.by_name.get(short) {
+            Some(&idx) => self.order[idx].1.push((slot, count)),
+            None => {
+                self.by_name.insert(short, self.order.len());
+                self.order.push((short, vec![(slot, count)]));
+            }
+        }
+    }
+}
+
+/// `countByItem[short] += n` over one object's insertion-ordered assoc list
+/// (a handful of distinct items per object -- a linear scan beats a map).
+fn add_item_count<'a>(counts: &mut Vec<(&'a [u8], i64)>, short: &'a [u8], n: i64) {
+    match counts.iter_mut().find(|(s, _)| *s == short) {
+        Some((_, total)) => *total += n,
+        None => counts.push((short, n)),
+    }
+}
+
+/// Port of sav_map_data._collectItemLocationIndex (+ its helpers
+/// _inventoryComponentObjects / the stack walk), extended with the items
+/// riding the belt lines -- stock the Python version never counted, so an
+/// item search under-reported a running factory by everything in transit.
+/// Returns [(itemShortName, [(instanceName, count)])] in the same insertion
+/// order the Python implementation's dict produces.
 pub fn item_location_index(scan: &SaveScan) -> Vec<(Vec<u8>, Vec<(Vec<u8>, i64)>)> {
     let store = scan.store;
     let data: &[u8] = &store.data;
@@ -124,11 +148,7 @@ pub fn item_location_index(scan: &SaveScan) -> Vec<(Vec<u8>, Vec<(Vec<u8>, i64)>
             };
             for stack in stacks {
                 if let Some((item_path, n)) = stack_item(stack, data) {
-                    let short = short_name(item_path);
-                    match count_by_item.iter_mut().find(|(s, _)| *s == short) {
-                        Some((_, total)) => *total += n,
-                        None => count_by_item.push((short, n)),
-                    }
+                    add_item_count(&mut count_by_item, short_name(item_path), n);
                 }
             }
         }
@@ -138,20 +158,65 @@ pub fn item_location_index(scan: &SaveScan) -> Vec<(Vec<u8>, Vec<(Vec<u8>, i64)>
             find_prop(&object.properties, data, b"mPickupItems")
         {
             if let Some((item_path, n)) = stack_item(pl, data) {
-                let short = short_name(item_path);
-                match count_by_item.iter_mut().find(|(s, _)| *s == short) {
-                    Some((_, total)) => *total += n,
-                    None => count_by_item.push((short, n)),
+                add_item_count(&mut count_by_item, short_name(item_path), n);
+            }
+        }
+
+        // Items riding a conveyor are real stock too -- a late-game belt
+        // network holds a lot of it -- so they count towards the item search
+        // exactly like a machine's or a container's contents. Pre-1.0 saves
+        // (and the chained-belt delete write-back, see editor::apply) keep a
+        // belt's items on the belt itself, one entity per record.
+        if let ActorSpecific::ConveyorBelt { items, .. } = &object.actor_specific {
+            for item in items {
+                let path = item.item_path.bytes(data);
+                if !path.is_empty() {
+                    add_item_count(&mut count_by_item, short_name(path), 1);
                 }
             }
         }
 
         for (short, count) in count_by_item {
-            match index.by_name.get(short) {
-                Some(&idx) => index.order[idx].1.push((slot, count)),
-                None => {
-                    index.by_name.insert(short, index.order.len());
-                    index.order.push((short, vec![(slot, count)]));
+            index.push(short, slot, count);
+        }
+
+        // Since 1.0 a whole belt line's items instead live in one ring buffer
+        // on the shared FGConveyorChainActor, each member belt owning a
+        // contiguous window of it. Counted against the belt the item is
+        // physically on -- not against the chain actor, which is an invisible
+        // bookkeeping object with no footprint -- so every location the search
+        // lists is a real placed building at a real position. Same split the
+        // belt tooltip's "items on this segment" and the selection panel's
+        // combined inventory already show.
+        if let ActorSpecific::ConveyorChain {
+            belts, items, maximum_items, chain_lead_item_index, ..
+        } = &object.actor_specific
+        {
+            // The member belts' windows tile `items` exactly -- no slot
+            // counted twice, none missed -- so a line's total across its
+            // belts is exactly the length of the ring.
+            let mut belt_counts: Vec<(&[u8], i64)> = Vec::new();
+            for chain_belt in belts {
+                let Some((start, count)) = conveyor_chain_segment_window(
+                    chain_belt,
+                    *maximum_items,
+                    *chain_lead_item_index,
+                ) else {
+                    continue;
+                };
+                let Some(&belt_slot) = slot_by_name.get(chain_belt.belt.path_name.bytes(data))
+                else {
+                    continue;
+                };
+                belt_counts.clear();
+                for item in items.iter().skip(start).take(count) {
+                    let path = item.item_path.bytes(data);
+                    if !path.is_empty() {
+                        add_item_count(&mut belt_counts, short_name(path), 1);
+                    }
+                }
+                for &(short, n) in &belt_counts {
+                    index.push(short, belt_slot, n);
                 }
             }
         }

rust_parser/core/src/mapdata/describe.rs

@@ -566,7 +566,7 @@ pub fn describe_instance(store: &SaveStore, index: &MapIndex, instance_name: &st
             }
             if belts.len() > 1 {
                 let line_items =
-                    counted_item_list(items.iter().map(|(path, _)| path.bytes(data)));
+                    counted_item_list(items.iter().map(|item| item.item_path.bytes(data)));
                 if !line_items.is_empty() {
                     result.insert("itemsOnLine".into(), Value::Array(line_items));
                     result.insert("lineSegmentCount".into(), Value::from(belts.len() as i64));

rust_parser/core/src/mapdata/queries.rs

@@ -193,10 +193,36 @@ fn add_item(
 // Conveyor chain ring buffer (sav_map_data 2284-2309)
 // ---------------------------------------------------------------------------
 
+/// One member belt's own contiguous window `[start, start + count)` into its
+/// chain's ring-buffer item list, or None when the chain's/belt's indices say
+/// there is nothing to read. Python's `%` on possibly negative operands ==
+/// rem_euclid. Shared by every consumer of a chain's items (the belt tooltip
+/// below, the item-location index in extract.rs) so the ring arithmetic lives
+/// in exactly one place.
+pub fn conveyor_chain_segment_window(
+    chain_belt: &ChainBelt,
+    maximum_items: i32,
+    chain_lead_item_index: i32,
+) -> Option<(usize, usize)> {
+    if maximum_items <= 0
+        || chain_lead_item_index < 0
+        || chain_belt.lead_item_index < 0
+        || chain_belt.tail_item_index < 0
+    {
+        return None;
+    }
+    let maximum = maximum_items as i64;
+    let start = (chain_belt.lead_item_index as i64 - chain_lead_item_index as i64)
+        .rem_euclid(maximum) as usize;
+    let count = (chain_belt.tail_item_index as i64 - chain_belt.lead_item_index as i64)
+        .rem_euclid(maximum) as usize
+        + 1;
+    Some((start, count))
+}
+
 /// sav_map_data._conveyorChainSegmentItemPaths: this belt's own contiguous
-/// slice of the chain's ring-buffer item window. Python's `%` on possibly
-/// negative operands == rem_euclid; the slice `chainItems[start:start+count]`
-/// clamps at the end like Python slicing.
+/// slice of the chain's ring-buffer item window. The slice
+/// `chainItems[start:start+count]` clamps at the end like Python slicing.
 pub fn conveyor_chain_segment_item_paths<'a>(
     // NOT tied to 'a: the returned slices borrow only `data` (StrRefs are
     // offsets), so an owned re-parsed chain actor can be a short-lived local.
@@ -209,23 +235,24 @@ pub fn conveyor_chain_segment_item_paths<'a>(
     else {
         return Vec::new();
     };
-    if items.is_empty() || *maximum_items <= 0 || *chain_lead_item_index < 0 {
+    if items.is_empty() {
         return Vec::new();
     }
     for chain_belt in belts {
         if chain_belt.belt.path_name.bytes(data) != belt_instance_name {
             continue;
         }
-        if chain_belt.lead_item_index < 0 || chain_belt.tail_item_index < 0 {
+        let Some((start, count)) =
+            conveyor_chain_segment_window(chain_belt, *maximum_items, *chain_lead_item_index)
+        else {
             return Vec::new();
-        }
-        let maximum = *maximum_items as i64;
-        let start = (chain_belt.lead_item_index as i64 - *chain_lead_item_index as i64)
-            .rem_euclid(maximum) as usize;
-        let count = (chain_belt.tail_item_index as i64 - chain_belt.lead_item_index as i64)
-            .rem_euclid(maximum) as usize
-            + 1;
-        return items.iter().skip(start).take(count).map(|(path, _)| path.bytes(data)).collect();
+        };
+        return items
+            .iter()
+            .skip(start)
+            .take(count)
+            .map(|item| item.item_path.bytes(data))
+            .collect();
     }
     Vec::new()
 }