From 42726b262977e1393b258a7ab32b8a8de15a3fbe Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Fri, 19 Jun 2026 11:13:38 -0400 Subject: [PATCH 01/21] chore: Enable various plugins in settings.json for enhanced functionality Signed-off-by: UncleSp1d3r --- .claude/settings.json | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.claude/settings.json b/.claude/settings.json index c72c6b7..4ee3ac9 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -1,3 +1,15 @@ { - "enabledPlugins": {} + "enabledPlugins": { + "compound-engineering@compound-engineering-plugin": true, + "ecc@ecc": true, + "rust-analyzer-lsp@claude-plugins-official": true, + "rust-cli-developer@geoffjay-claude-plugins": true, + "systems-programming@claude-code-workflows": true, + "agents-quality-security@buildwithclaude": true, + "security-guidance@claude-plugins-official": true, + "pr-review-toolkit@claude-plugins-official": true, + "comprehensive-review@claude-code-workflows": true, + "superpowers@claude-plugins-official": true, + "github@claude-plugins-official": true + } } From 2e13409f83ea7c921feee59ee58bb0ce3328c667 Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Fri, 19 Jun 2026 12:09:24 -0400 Subject: [PATCH 02/21] chore: Remove exclusion patterns from mdformat configuration Signed-off-by: UncleSp1d3r --- .mdformat.toml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/.mdformat.toml b/.mdformat.toml index 22c2e4e..dcda597 100644 --- a/.mdformat.toml +++ b/.mdformat.toml @@ -1,19 +1,4 @@ # mdformat configuration - consistent with Prettier and markdownlint -exclude = [ - "venv/**", - "**/node_modules/**", - "**/*.txt", - "**/*.mdc", - "**/*.tpl.md", - "**/CHANGELOG.md", - "target/**", - "megalinter-reports/**", - "**/*.result", - "**/*.testfile", - "**/SKILL.md", # AI stuff - ".claude/**/*", # AI stuff - ".tessl/**/*", # AI stuff -] validate = true number = true wrap = "no" From d3a415e9de7444a6df3ab72f4769a956b2b41560 Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Fri, 19 Jun 2026 12:09:39 -0400 Subject: [PATCH 03/21] feat: Implement ImportClassifier for tagged import/export symbol classification Signed-off-by: UncleSp1d3r --- ...ol-classification-pipeline-requirements.md | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 docs/brainstorms/2026-06-19-symbol-classification-pipeline-requirements.md diff --git a/docs/brainstorms/2026-06-19-symbol-classification-pipeline-requirements.md b/docs/brainstorms/2026-06-19-symbol-classification-pipeline-requirements.md new file mode 100644 index 0000000..cf856e3 --- /dev/null +++ b/docs/brainstorms/2026-06-19-symbol-classification-pipeline-requirements.md @@ -0,0 +1,98 @@ +--- +date: 2026-06-19 +topic: symbol-classification-pipeline +--- + +# Import/Export Symbol Classification Pipeline + +## Summary + +Add an `ImportClassifier` that converts parsed import/export symbols and section names from `ContainerInfo` into tagged, scored `FoundString` objects, and route it into the live extraction path so the already-defined Import/Export ranking boosts finally fire. Four new semantic tags (crypto, network, file-I/O, entry-point) are recognized via static API-name sets and the existing symbol demangler. + +--- + +## Problem Frame + +`stringy` already emits import and export names in its default output: when `include_symbols` is on (the default), `src/extraction/basic_extractor.rs:67-89` pushes each `ImportInfo`/`ExportInfo` name as a `FoundString`. But those strings are emitted bare: no `Tag::Import`/`Tag::Export`, no RVA, no originating library, and no semantic classification. The `RankingEngine` defines +60 boosts for `Tag::Import` and `Tag::Export` (`src/classification/ranking.rs:131-132`), but nothing ever applies those tags, so the boosts are dead code and symbol strings rank no higher than incidental noise. A binary analyst scanning a packed sample gets `CreateFileW` buried next to junk instead of surfaced as a tagged file-I/O import. + +The pieces to fix this already exist in isolation - the demangler, the tag enum scaffolding, the ranking boosts, the parsed symbol metadata. What is missing is the classifier that ties symbol metadata to tags, plus the wiring that makes the tagged strings the ones that actually reach output. + +--- + +## Key Decisions + +- **Replace the existing untagged symbol emission rather than add a parallel path.** `basic_extractor` already emits import/export names untagged and default-on. Adding a second, tagged emission path would produce two `FoundString`s per symbol; deduplication is by text with a nondeterministic HashMap winner, so dedup could keep the untagged copy and silently drop the tagged one, no-opping the feature. The classifier becomes the single emission point at that call site instead. The issue's `ImportClassifier` / `extract_symbol_strings` design is unchanged; only where it is called moves. + +- **Emit section names as ranked output rows.** Section names (`.text`, `.idata`) are not emitted as standalone strings today (only attached as the `section` field on content strings), so this is net-new with no duplication risk. Generic short names rely on the existing noise/min-length filter to stay out of the way. + +- **Reuse `SymbolDemangler` as-is.** It already handles both Rust and C++ (`src/classification/symbols.rs`); no new demangling logic is introduced. + +- **New tags ride the existing tag machinery.** No new CLI flags. Once the four variants are wired through `FromStr`, `Display`, serde, and the help text, `--only-tags`/`--no-tags` filter them automatically. + +--- + +## Requirements + +**Symbol classification** + +- R1. Import names emit as `FoundString`s with `source: StringSource::ImportName`, always carrying `Tag::Import`, at `confidence: 1.0`. +- R2. Imports whose name matches the crypto, network, or file-I/O API sets additionally carry `Tag::Crypto`, `Tag::Network`, or `Tag::FileIO`; imports that match none carry only `Tag::Import`. +- R3. An import's originating library populates the `section` field when present; when absent, a format-appropriate default is used (`.idata` for PE, `.dynsym` for ELF). +- R4. An import's RVA populates from `ImportInfo::address` when present (it is optional). +- R5. Export names emit as `FoundString`s with `source: StringSource::ExportName`, always carrying `Tag::Export`, with RVA from `ExportInfo::address` (always available). +- R6. Exports are run through the existing `SymbolDemangler`; successfully demangled exports additionally carry `Tag::DemangledSymbol`. +- R7. Exports whose name is a known entry point (`main`, `_start`, `DllMain`, `WinMain`, `wWinMain`) additionally carry `Tag::EntryPoint`. +- R8. Section names emit as standalone `FoundString`s with `source: StringSource::SectionData` at `confidence: 1.0`, ranked alongside other output. + +**Type system and ranking** + +- R9. Add `Tag` variants `Crypto`, `Network`, `FileIO`, `EntryPoint` with serde renames (`crypto`, `network`, `fileio`, `entry-point`), wired through `Tag::from_str`, `Display`, and every exhaustive match site, including the CLI help text that the `cli_help_lists_all_canonical_tags` test guards. +- R10. `RankingConfig` gains tag boosts: `Crypto` +50, `Network` +45, `FileIO` +35, `EntryPoint` +40. + +**Pipeline integration** + +- R11. The tagged classifier replaces the untagged import/export emission in `basic_extractor` so each symbol is emitted exactly once, already tagged; section-name emission is added to the same path. +- R12. Tagged symbol strings flow through the remaining pipeline stages (classification, ranking) with their classifier-assigned tags preserved, so the Import/Export and new semantic boosts apply to live output. + +**Tests and docs** + +- R13. `tests/classification_symbols.rs` covers import classification, export classification, API-set detection (at least 3 representative symbols per category plus a negative case), section-name extraction, and end-to-end `extract_symbol_strings` output against the existing ELF, PE, and Mach-O fixtures. +- R14. `docs/src/classification.md` is updated to reference `std::sync::LazyLock` instead of the stale `once_cell::sync::Lazy` pattern example. + +--- + +## Acceptance Examples + +- AE1. **Covers R2.** Import `CreateFileW` yields tags `Import` + `FileIO`; `connect` yields `Import` + `Network`; `EVP_EncryptInit` yields `Import` + `Crypto`; `printf` yields `Import` only. +- AE2. **Covers R6, R7.** A C++-mangled export demangles and yields `Export` + `DemangledSymbol`; export `main` yields `Export` + `EntryPoint`; export `DllMain` yields `Export` + `EntryPoint`. +- AE3. **Covers R3.** A PE import from `kernel32.dll` sets `section` to `kernel32.dll`; an ELF import with no library sets `section` to `.dynsym`. +- AE4. **Covers R11, R12.** Running `stringy` on a PE fixture (with the default `include_symbols`) shows `CreateFileW` exactly once, tagged `import` + `fileio` + - not a duplicate untagged row, and not dropped by dedup. + +--- + +## Scope Boundaries + +- New CLI flags to filter the four new tags - out of scope; the existing `--only-tags`/`--no-tags` machinery covers them. +- Redesigning deduplication or string-ordering behavior - out of scope. +- Tuning exact membership of the crypto/network/file-I/O API sets beyond the issue's seed lists - deferred to planning and implementation. + +--- + +## Dependencies / Assumptions + +- `ExportInfo::address` is a required `u64` while `ImportInfo::address` is `Option`; export RVA is always available, import RVA is conditional (R4, R5 reflect this). +- This work assumes `include_symbols` remains default-on, so the upgrade is visible in default output rather than gated behind a flag. + +--- + +## Sources / Research + +- `src/extraction/basic_extractor.rs:67-89` - existing untagged import/export emission that R11 replaces (gated on `config.include_symbols`). +- `src/extraction/traits.rs:97` - `include_symbols` defaults to `true`. +- `src/types/mod.rs` - `ImportInfo` (`name`, `library: Option`, `address: Option`, `ordinal`), `ExportInfo` (`name`, `address: u64`, `ordinal`), `ContainerInfo.imports`/`exports`, and the `Tag` / `StringSource` enums (`Import`, `Export`, `DemangledSymbol`, `ImportName`, `ExportName`, `SectionData` all present; `Crypto`/`Network`/`FileIO`/ `EntryPoint` absent). +- `src/types/found_string.rs` - `FoundString::new(text, encoding, offset, length, source)` plus `with_rva`, `with_section`, `with_tags`, `with_confidence` builders (constructor defaults `confidence: 1.0`). +- `src/classification/symbols.rs` - `SymbolDemangler` (Rust via `rustc_demangle`, C++ via `cpp_demangle`), reused directly. +- `src/classification/ranking.rs:131-132` - existing `Import`/`Export` +60 boosts that R10 extends. +- `docs/src/classification.md:77,83` - stale `once_cell::sync::Lazy` example that R14 corrects. +- GitHub issue #20 - originating specification (note: its "nothing converts imports/exports into FoundStrings" premise is corrected by R11). From f5cf28bfb575e133d4f87bdd571e18bd44bf3530 Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Fri, 19 Jun 2026 12:09:58 -0400 Subject: [PATCH 04/21] fix: Correct formatting in AI Usage Policy for clarity Signed-off-by: UncleSp1d3r --- AI_POLICY.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/AI_POLICY.md b/AI_POLICY.md index e868b08..25e6927 100644 --- a/AI_POLICY.md +++ b/AI_POLICY.md @@ -6,8 +6,7 @@ Think of AI assistance like spellcheck. It catches typos, suggests corrections, ## The Rule -**You own every line you submit.** You must be able to explain what -it does and how it interacts with the rest of the system without asking your AI to explain it back to you. +**You own every line you submit.** You must be able to explain what it does and how it interacts with the rest of the system without asking your AI to explain it back to you. Everything else follows from that. From de84fcd4aa045312379c8ef667376659f758776f Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Fri, 19 Jun 2026 12:10:04 -0400 Subject: [PATCH 05/21] chore: Update changelog for clarity and formatting improvements Signed-off-by: UncleSp1d3r --- CHANGELOG.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 88873c2..95c8422 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,12 +2,12 @@ All notable changes to this project will be documented in this file. -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), -and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] ### Added + - Pipeline orchestrator (`Pipeline::run`) with configurable stages: parse, extract, classify, rank, normalize, filter, output - `PipelineConfig`, `FilterConfig`, and `EncodingFilter` for pipeline configuration - `FilterEngine` for post-extraction string filtering (min-len, encoding, tags, top-N) @@ -39,6 +39,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Debug-build env var injection for e2e warning-path testing ### Changed + - Container parsers refactored from single files to module directories (elf/, pe/, macho/) - Repository renamed from StringyMcStringFace to Stringy - Improved YARA formatter code quality and test coverage @@ -46,6 +47,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Replaced `once_cell::sync::Lazy` with `std::sync::LazyLock` (stabilized in Rust 1.80) ### Fixed + - Rustdoc warning for IPv6 address example in documentation - ELF section name fallback no longer eagerly allocates on every iteration - Pipeline section lookups use HashMap for O(1) access instead of O(n) linear scans @@ -55,6 +57,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Demangler text clone now conditional (only for mangled-looking symbols) ### Dependencies + - Added `mmap-guard` for safe memory-mapped file I/O - Added `indicatif` for progress bars and spinners - Added `tempfile` for stdin-to-pipeline bridging @@ -72,6 +75,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Initial release with core functionality: ### Added + - ELF, PE, and Mach-O binary format detection and parsing - ASCII and UTF-8 string extraction from binary sections - Section-aware extraction with weight-based prioritization From 1a389735affa2b532a6cbe8efe23bc2b017ef5f9 Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Fri, 19 Jun 2026 12:10:24 -0400 Subject: [PATCH 06/21] feat: Implement ImportClassifier for tagged import/export symbol classification pipeline Signed-off-by: UncleSp1d3r --- ...eat-symbol-classification-pipeline-plan.md | 206 ++++++++++++++++++ 1 file changed, 206 insertions(+) create mode 100644 docs/plans/2026-06-19-001-feat-symbol-classification-pipeline-plan.md diff --git a/docs/plans/2026-06-19-001-feat-symbol-classification-pipeline-plan.md b/docs/plans/2026-06-19-001-feat-symbol-classification-pipeline-plan.md new file mode 100644 index 0000000..f601712 --- /dev/null +++ b/docs/plans/2026-06-19-001-feat-symbol-classification-pipeline-plan.md @@ -0,0 +1,206 @@ +--- +title: 'feat: Import/Export symbol classification pipeline' +type: feat +date: 2026-06-19 +origin: docs/brainstorms/2026-06-19-symbol-classification-pipeline-requirements.md +--- + +# feat: Import/Export symbol classification pipeline + +## Summary + +Add an `ImportClassifier` that converts `ContainerInfo` import/export symbols and section names into tagged, scored `FoundString`s, and route it into the extraction path as the single symbol emission point so the already-defined Import/Export ranking boosts finally fire. Four new semantic tags (crypto, network, file-I/O, entry-point) are recognized via static API-name sets and the existing demangler. + +--- + +## Problem Frame + +`stringy` already emits import and export names in default output: when `include_symbols` is on (the default), `src/extraction/basic_extractor.rs:67-89` pushes each symbol name as a bare `FoundString` -- no tags, no RVA, no library, no semantic classification. The `RankingEngine` defines +60 boosts for `Tag::Import` and `Tag::Export` (`src/classification/ranking.rs:131-132`), but nothing applies those tags, so the boosts are dead code and symbol strings rank no higher than incidental noise. The demangler, the tag scaffolding, the ranking boosts, and the parsed symbol metadata all exist in isolation; what is missing is the classifier that ties metadata to tags plus the wiring that makes the tagged strings the ones that reach output. + +--- + +## High-Level Technical Design + +The classifier becomes the single emission point for symbol strings, replacing the untagged push in the extractor. Tags accumulate down the pipeline rather than being overwritten, so classifier-assigned tags survive to ranking where the boosts apply. + +```mermaid +flowchart TB + CI[ContainerInfo: imports, exports, sections] --> IC[ImportClassifier] + IC --> PI[process_imports: Import + semantic] + IC --> PE[process_exports: Export + demangle + entry-point] + IC --> PS[process_section_names: SectionData] + PI --> ESS[extract_symbol_strings -> Vec FoundString, tagged] + PE --> ESS + PS --> ESS + ESS --> BE[basic_extractor include_symbols block: replaces untagged push] + BE --> DD[dedup: merge by text+encoding] + DD --> CS[pipeline classify_strings: APPENDS semantic tags, preserves existing] + CS --> RK[rank_strings: Import/Export/Crypto/Network/FileIO/EntryPoint boosts fire] + RK --> OUT[ranked output] +``` + +--- + +## Key Technical Decisions + +- KTD1. **Replace the untagged emission at `basic_extractor.rs:67-89`, not add a parallel path.** Dedup merges by `(text, encoding)` with a nondeterministic HashMap winner; a parallel tagged copy alongside the existing untagged one could be the copy dedup discards, silently dropping the tags. A single emission point removes the hazard. (see origin: docs/brainstorms/2026-06-19-symbol-classification-pipeline-requirements.md) +- KTD2. **Rely on the pipeline's append-only classification.** `classify_strings()` pushes each semantic tag only `if !s.tags.contains(&tag)` (`src/pipeline/mod.rs:320-326`), so classifier-assigned `Import`/`Export`/semantic tags are preserved through the pipeline's own pass. No tag-clobber guard is needed; R12 depends on this behavior. +- KTD3. **Export order: tag `Export`, then demangle, then entry-point check.** `SymbolDemangler::demangle(&mut fs)` adds `Tag::DemangledSymbol` and preserves existing tags; the entry-point name match runs against the resulting `text`. +- KTD4. **Wire the four tags as one foundational unit across all co-change sites.** Adding a variant forces updates at two compiler-enforced output formatters plus the enum/`FromStr`, both CLI `long_help` literals, and five variant-enumerating test lists; the build will not compile and CI will not pass until every site is updated, so they land together. +- KTD5. **Static API sets as `LazyLock>`.** Mirrors `src/classification/patterns/paths.rs`. Never `once_cell`. +- KTD6. **Section-name emission rides `include_symbols` (default-on).** All three symbol kinds (imports, exports, section names) are bundled behind one `extract_symbol_strings` call at the existing `include_symbols` gate, so the symbols toggle also governs section-name rows. + +--- + +## Requirements + +Carried from the origin requirements doc. + +**Symbol classification** + +- R1. Import names emit as `FoundString`s with `source: StringSource::ImportName`, always carrying `Tag::Import`, at `confidence: 1.0`. +- R2. Imports matching the crypto, network, or file-I/O API sets additionally carry `Tag::Crypto`, `Tag::Network`, or `Tag::FileIO`; non-matching imports carry only `Tag::Import`. +- R3. An import's originating library populates the `section` field when present; when absent, a format-appropriate default is used (`.idata` for PE, `.dynsym` for ELF). +- R4. An import's RVA populates from `ImportInfo::address` when present (optional). +- R5. Export names emit as `FoundString`s with `source: StringSource::ExportName`, always carrying `Tag::Export`, with RVA from `ExportInfo::address`. +- R6. Exports run through `SymbolDemangler`; successfully demangled exports additionally carry `Tag::DemangledSymbol`. +- R7. Exports whose name is a known entry point (`main`, `_start`, `DllMain`, `WinMain`, `wWinMain`) additionally carry `Tag::EntryPoint`. +- R8. Section names emit as standalone `FoundString`s with `source: StringSource::SectionData` at `confidence: 1.0`, ranked alongside other output. + +**Type system and ranking** + +- R9. Add `Tag` variants `Crypto`, `Network`, `FileIO`, `EntryPoint` with serde renames (`crypto`, `network`, `fileio`, `entry-point`), wired through `Tag::from_str`, the output display formatters, and the CLI help text. +- R10. `RankingConfig` gains tag boosts: `Crypto` +50, `Network` +45, `FileIO` +35, `EntryPoint` +40. + +**Pipeline integration** + +- R11. The tagged classifier replaces the untagged import/export emission in the extractor so each symbol is emitted once, already tagged; section-name emission is added to the same path. +- R12. Tagged symbol strings flow through the remaining pipeline stages with their classifier-assigned tags preserved, so the Import/Export and new semantic boosts apply to live output. + +**Tests and docs** + +- R13. `tests/classification_symbols.rs` covers import classification, export classification, API-set detection (at least 3 representative symbols per category plus a negative case), section-name extraction, and end-to-end `extract_symbol_strings` output against the ELF, PE, and Mach-O fixtures. +- R14. `docs/src/classification.md` is updated to reference `std::sync::LazyLock` instead of the stale `once_cell::sync::Lazy` example. + +--- + +## Acceptance Examples + +- AE1. **Covers R2.** Import `CreateFileW` yields `Import` + `FileIO`; `connect` yields `Import` + `Network`; `EVP_EncryptInit` yields `Import` + `Crypto`; `printf` yields `Import` only. +- AE2. **Covers R6, R7.** A C++-mangled export demangles and yields `Export` + `DemangledSymbol`; export `main` yields `Export` + `EntryPoint`; export `DllMain` yields `Export` + `EntryPoint`. +- AE3. **Covers R3.** A PE import from `kernel32.dll` sets `section` to `kernel32.dll`; an ELF import with no library sets `section` to `.dynsym`. +- AE4. **Covers R11, R12.** Running `stringy` on a PE fixture (default `include_symbols`) shows `CreateFileW` exactly once, tagged `import` + `fileio` -- not a duplicate untagged row, and not dropped by dedup. + +--- + +## Implementation Units + +### U1. New Tag variants and full co-change wiring + +- **Goal:** Add `Crypto`, `Network`, `FileIO`, `EntryPoint` to the `Tag` enum and thread them through every site adding a variant touches, so the crate compiles and all tag-enumerating tests pass. +- **Requirements:** R9 +- **Dependencies:** none +- **Files:** `src/types/mod.rs`, `src/output/table/formatting.rs`, `src/output/yara/mod.rs`, `src/main.rs`, `src/types/tests.rs`, `src/output/json.rs`, `tests/integration_cli.rs`, `tests/output_yara_integration.rs` +- **Approach:** Add the four variants with serde renames (`crypto`, `network`, `fileio`, `entry-point`). Add `FromStr` arms. Add display arms in `tag_to_display_string` (`src/output/table/formatting.rs:81-103`) and YARA `tag_name` (`src/output/yara/mod.rs:164-187`) -- both are compiler-enforced exhaustive matches. Add the canonical strings to both `--only-tags` and `--no-tags` `long_help` literals (`src/main.rs:108-111`, `122-125`; must be string literals, not `const`/`concat!`). Extend the variant-enumerating test lists. +- **Patterns to follow:** existing `DemangledSymbol` variant (serde rename + `FromStr` arm) in `src/types/mod.rs`; the exhaustive matches in `formatting.rs:81-103` and `yara/mod.rs:164-187`. +- **Test scenarios:** + - Covers R9. `from_str("crypto"/"network"/"fileio"/"entry-point")` return the matching variants; an unknown string still returns `Err` (extend `test_tag_from_str_all_variants`, `src/types/tests.rs:126-148`). + - Each new variant produces its display string in `tag_to_display_string` and YARA `tag_name` (extend `all_tag_variants_have_display`, `formatting.rs:224-245`, and `test_yara_all_tag_types`, `tests/output_yara_integration.rs:154-177`). + - JSON serialization yields the serde-renamed form: `Tag::FileIO` -> `"fileio"`, `Tag::EntryPoint` -> `"entry-point"` (extend `test_all_tag_types_serialize_correct_names`, `src/output/json.rs:159-181`). + - `stringy --help` lists `crypto`, `network`, `fileio`, `entry-point` under both flags (extend `cli_help_lists_all_canonical_tags`, `tests/integration_cli.rs:264-291`). +- **Verification:** `cargo build` compiles with no non-exhaustive-match errors; the extended variant-list tests pass; `--help` shows the four new tags. + +### U2. RankingConfig boosts for the new tags + +- **Goal:** Give the four new tags their ranking boosts so semantic symbols outrank noise. +- **Requirements:** R10 +- **Dependencies:** U1 +- **Files:** `src/classification/ranking.rs` +- **Approach:** Add `tag_boosts.insert(...)` for `Crypto` +50, `Network` +45, `FileIO` +35, `EntryPoint` +40 in `RankingConfig::default()`; add matching assertions in `test_default_config_values`. +- **Patterns to follow:** `ranking.rs:131-132` (Import/Export +60 inserts) and the paired `test_default_config_values` (`ranking.rs:304-349`). +- **Test scenarios:** + - Covers R10. `tag_boosts.get(&Tag::Crypto) == Some(&50)`, `Network` `&45`, `FileIO` `&35`, `EntryPoint` `&40` (extend `test_default_config_values`). + - A `FoundString` carrying `Tag::Crypto` scores at least 50 above an otherwise identical untagged string under the default config. +- **Verification:** `test_default_config_values` passes with the four new assertions; a crypto-tagged string ranks above an untagged peer. + +### U3. ImportClassifier and extract_symbol_strings + +- **Goal:** Build the classifier converting `ContainerInfo` imports/exports/section names into tagged `FoundString`s, plus the `extract_symbol_strings` entry point, and test it directly. +- **Requirements:** R1, R2, R3, R4, R5, R6, R7, R8, R13; AE1, AE2, AE3 +- **Dependencies:** U1 +- **Files:** `src/classification/imports.rs` (new), `src/classification/mod.rs`, `tests/classification_symbols.rs` (new) +- **Approach:** New `ImportClassifier` holding a `SymbolDemangler` and three `LazyLock>` API sets (crypto, network, file-I/O) seeded from the origin lists. `process_imports`: build via `FoundString::new(name, Utf8, 0, len, ImportName)` then builder chain -- tags `[Import]` plus any semantic match, `.with_section(library or .idata/.dynsym default)`, `.with_rva` when `address` is `Some`, `.with_confidence(1.0)`. `process_exports`: tags `[Export]`, RVA from the required `address`, then `demangler.demangle(&mut fs)`, then entry-point check on `fs.text`. `process_section_names`: one `FoundString` per section name, source `SectionData`, confidence 1.0. `extract_symbol_strings(&ContainerInfo)` composes all three. Export `ImportClassifier` and `extract_symbol_strings` from `src/classification/mod.rs` (`pub mod imports;` + `pub use`). +- **Technical design (directional, not a spec):** export processing order -- `new(... ExportName) -> with_tags([Export]) -> with_rva(address) -> demangle(&mut) [adds DemangledSymbol] -> if text in ENTRY_POINTS push EntryPoint`. +- **Patterns to follow:** `LazyLock` idiom in `src/classification/patterns/paths.rs:30-43` (`HashSet::from([...])`); `FoundString::new()` + builder chain only, never struct literals (GOTCHAS); `SymbolDemangler::demangle` in `src/classification/symbols.rs`; fixture loading via `get_fixture_path` in `tests/classification_integration_tests.rs`. +- **Execution note:** Implement classifier behavior test-first. +- **Test scenarios** (`tests/classification_symbols.rs`): + - Covers R1. Every emitted import carries `Tag::Import`, source `ImportName`, confidence 1.0. + - Covers AE1. `CreateFileW` -> `Import`+`FileIO`; `connect` -> `Import`+`Network`; `EVP_EncryptInit` -> `Import`+`Crypto`; `printf` -> `Import` only (at least 3 matches per category plus the negative). + - Covers AE3 / R3. PE import with library `kernel32.dll` -> section `kernel32.dll`; PE import with no library -> `.idata`; ELF import with no library -> `.dynsym`. + - Covers R4. Import with `address: Some(x)` sets `rva` to `x`; `address: None` leaves `rva` unset. + - Covers R5. Every export carries `Tag::Export`, source `ExportName`, RVA from `address`. + - Covers AE2 / R6, R7. A C++-mangled export gains `DemangledSymbol` and demangled text; `main`/`_start`/`DllMain`/`WinMain`/`wWinMain` gain `EntryPoint`; an ordinary export gains neither. + - Covers R8. Section names emit as `FoundString`s with source `SectionData`, confidence 1.0. + - Covers R13. `extract_symbol_strings` on parsed ELF/PE/Mach-O fixtures returns tagged import/export/section-name strings; JSON-form comparisons use the serde-renamed tag strings. +- **Verification:** `tests/classification_symbols.rs` passes; `extract_symbol_strings` is reachable as `stringy::classification::extract_symbol_strings`. + +### U4. Replace the untagged symbol emission in the extractor + +- **Goal:** Route `extract_symbol_strings` into the extraction path as the single symbol emission point so tagged symbols reach live output and the boosts fire. +- **Requirements:** R11, R12; AE4 +- **Dependencies:** U3 (also U1, U2 for the tags and their boosts) +- **Files:** `src/extraction/basic_extractor.rs`, affected insta snapshots (e.g. the `tests/snapshots` for `integration_flows_1_5`) +- **Approach:** In `collect_all_strings`, replace the import/export push loops at `basic_extractor.rs:67-89` with a call to `extract_symbol_strings(container_info)`, still inside the `if config.include_symbols` block (which now also governs section-name rows). Dedup then merges by text; `classify_strings` appends semantic tags without clobbering the classifier's (KTD2). Regenerate insta snapshots whose fixture output now shows tagged, reranked symbols. `test_binary.c` is unchanged, so no `just gen-fixtures` -- snapshot regeneration only. +- **Execution note:** Capture current snapshots before the change, then regenerate after, so the snapshot diff is the reviewable record of the output shift. +- **Test scenarios:** + - Covers AE4 / R11. The pipeline on a PE fixture yields exactly one row per symbol -- no duplicate untagged-plus-tagged pair. + - Covers R12. A symbol that also matches a semantic pattern carries both its classifier tags and any pipeline-appended tags after `classify_strings`. + - Covers R11. With `include_symbols = false`, no import/export or section-name rows are emitted; with the default (`true`), they are. + - Regression: existing flow/snapshot tests pass after regeneration, and symbol strings rank above incidental noise. +- **Verification:** the full `nextest` suite passes after `INSTA_UPDATE=always cargo nextest run`; a CLI run on a PE fixture shows tagged imports ranked up, one row each. + +### U5. Documentation fix + +- **Goal:** Correct the stale `once_cell` example in the classification docs. +- **Requirements:** R14 +- **Dependencies:** none +- **Files:** `docs/src/classification.md` +- **Approach:** Replace the `once_cell::sync::Lazy` usages (`docs/src/classification.md` lines 77, 80, 120) with `std::sync::LazyLock` to match the actual implementation. +- **Patterns to follow:** the `LazyLock` idiom in `src/classification/mod.rs`. +- **Test scenarios:** Test expectation: none -- documentation-only change with no behavioral surface. +- **Verification:** `docs/src/classification.md` no longer references `once_cell` and the example mirrors the `std::sync::LazyLock` pattern. + +--- + +## Scope Boundaries + +- New CLI flags to filter the four new tags -- out of scope; the existing `--only-tags`/`--no-tags` machinery covers them once the variants are wired. +- Redesigning deduplication or string-ordering behavior -- out of scope. + +### Deferred to Follow-Up Work + +- Tuning the exact membership of the crypto/network/file-I/O API sets beyond the origin seed lists -- left to implementation and later iteration. + +--- + +## Risks & Dependencies + +- **Snapshot churn.** Routing tagged symbols into default output shifts rank order, so insta snapshots that capture fixture output (notably `integration_flows_1_5`) will change and must be regenerated. This is expected; if a snapshot diff looks wrong (e.g. a symbol losing its tag), investigate before accepting rather than blanket-accepting. +- **Append-only classification (KTD2).** R12 depends on `classify_strings` pushing tags only when absent. If that behavior changes, classifier tags could be lost. +- **Field-shape asymmetry.** `ExportInfo::address` is a required `u64` while `ImportInfo::address` is `Option` -- export RVA is always set, import RVA is conditional (R4, R5 reflect this). +- **Fixtures gitignored.** `just gen-fixtures` must run before tests; `test_binary.c` is unchanged here, so only snapshot regeneration is needed, not fixture rebuild. + +--- + +## Sources / Research + +- `src/extraction/basic_extractor.rs:67-89` -- existing untagged import/export emission that U4 replaces; `src/extraction/traits.rs:97` -- `include_symbols` defaults to `true`. +- `src/pipeline/mod.rs:320-326` -- `classify_strings` appends tags only when absent (basis for KTD2 / R12). +- `src/types/mod.rs` -- `Tag` enum (serde renames; `Import`/`Export`/`DemangledSymbol` present, the four new variants absent), `ImportInfo` (`name`, `library: Option`, `address: Option`, `ordinal`), `ExportInfo` (`name`, `address: u64`, `ordinal`), `ContainerInfo.imports`/`exports`. +- `src/types/found_string.rs` -- `FoundString::new(text, encoding, offset, length, source)` plus `with_rva`, `with_section`, `with_tags`, `with_confidence` builders (constructor defaults `confidence: 1.0`, empty tags). +- `src/classification/symbols.rs` -- `SymbolDemangler` (Rust via `rustc_demangle`, C++ via `cpp_demangle`); demangle adds `DemangledSymbol` and preserves existing tags. +- `src/classification/ranking.rs:131-132` -- existing Import/Export +60 boosts; `:304-349` -- the paired `test_default_config_values`. +- `src/classification/patterns/paths.rs:30-43` -- `LazyLock>` idiom to mirror. +- Exhaustive-match and variant-list co-change sites (U1): `tag_to_display_string` (`src/output/table/formatting.rs:81-103`), YARA `tag_name` (`src/output/yara/mod.rs:164-187`), CLI `long_help` (`src/main.rs:108-111`, `122-125`), and test lists in `src/types/tests.rs:126-148`, `src/output/json.rs:159-181`, `src/output/table/formatting.rs:224-245`, `tests/output_yara_integration.rs:154-177`, `tests/integration_cli.rs:264-291`. +- `docs/src/classification.md:77,80,120` -- stale `once_cell` example that U5 fixes. +- GitHub issue #20 -- originating specification (its "nothing converts imports/exports into FoundStrings" premise is corrected: the extractor already emits them untagged, which U4 replaces). From 1795d3bf13ccc6cecf7dad7f39afcd86e5211a64 Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Fri, 19 Jun 2026 12:30:16 -0400 Subject: [PATCH 07/21] feat: Enhance ImportClassifier and normalize encoding for symbol classification pipeline Signed-off-by: UncleSp1d3r --- ...eat-symbol-classification-pipeline-plan.md | 60 +++++++++++++------ 1 file changed, 43 insertions(+), 17 deletions(-) diff --git a/docs/plans/2026-06-19-001-feat-symbol-classification-pipeline-plan.md b/docs/plans/2026-06-19-001-feat-symbol-classification-pipeline-plan.md index f601712..e4065e6 100644 --- a/docs/plans/2026-06-19-001-feat-symbol-classification-pipeline-plan.md +++ b/docs/plans/2026-06-19-001-feat-symbol-classification-pipeline-plan.md @@ -43,12 +43,15 @@ flowchart TB ## Key Technical Decisions -- KTD1. **Replace the untagged emission at `basic_extractor.rs:67-89`, not add a parallel path.** Dedup merges by `(text, encoding)` with a nondeterministic HashMap winner; a parallel tagged copy alongside the existing untagged one could be the copy dedup discards, silently dropping the tags. A single emission point removes the hazard. (see origin: docs/brainstorms/2026-06-19-symbol-classification-pipeline-requirements.md) +- KTD1. **Replace the untagged emission at `basic_extractor.rs:67-89`, not add a parallel path.** A parallel tagged copy alongside the existing untagged one would produce two `FoundString`s per symbol. Dedup keys by `(text, encoding)` and unions tags across occurrences via `merge_tags` (`src/extraction/dedup/mod.rs:237-248`), so tags are not lost -- but it adopts the `source`/`section`/`rva` metadata of a nondeterministic first occurrence, so a symbol can surface carrying the untagged copy's bare metadata. A single emission point removes that ambiguity. (see origin: docs/brainstorms/2026-06-19-symbol-classification-pipeline-requirements.md) - KTD2. **Rely on the pipeline's append-only classification.** `classify_strings()` pushes each semantic tag only `if !s.tags.contains(&tag)` (`src/pipeline/mod.rs:320-326`), so classifier-assigned `Import`/`Export`/semantic tags are preserved through the pipeline's own pass. No tag-clobber guard is needed; R12 depends on this behavior. - KTD3. **Export order: tag `Export`, then demangle, then entry-point check.** `SymbolDemangler::demangle(&mut fs)` adds `Tag::DemangledSymbol` and preserves existing tags; the entry-point name match runs against the resulting `text`. -- KTD4. **Wire the four tags as one foundational unit across all co-change sites.** Adding a variant forces updates at two compiler-enforced output formatters plus the enum/`FromStr`, both CLI `long_help` literals, and five variant-enumerating test lists; the build will not compile and CI will not pass until every site is updated, so they land together. +- KTD4. **Wire the four tags as one foundational unit across all co-change sites.** Adding a variant touches two compiler-enforced output formatters (`tag_to_display_string`, YARA `tag_name`) -- the build will not compile until those are updated -- plus the enum/`FromStr`, both CLI `long_help` literals, and five variant-enumerating test lists, which are not compiler-checked and fail at test-time instead. All land together so CI passes. - KTD5. **Static API sets as `LazyLock>`.** Mirrors `src/classification/patterns/paths.rs`. Never `once_cell`. - KTD6. **Section-name emission rides `include_symbols` (default-on).** All three symbol kinds (imports, exports, section names) are bundled behind one `extract_symbol_strings` call at the existing `include_symbols` gate, so the symbols toggle also governs section-name rows. +- KTD7. **Extraction labels ASCII content as UTF-8, not a distinct `Encoding::Ascii`.** Import-bearing sections (e.g. PE `.idata`) are byte-scanned by default; if that scan emitted `CreateFileW` as `Encoding::Ascii` while the classifier emits `Encoding::Utf8`, the `(text, encoding)` dedup key would split and AE4's "exactly once" would break. The ASCII extraction path instead emits `Encoding::Utf8` (ASCII is a UTF-8 subset), so the byte-scanned and classifier occurrences of the same name share one dedup key and merge. `Encoding::Ascii` is no longer produced as extraction output; the enum variant is retained for compatibility but unused. (U6 carries this normalization.) +- KTD8. **Section names use a dedicated `StringSource::SectionName` variant.** Reusing `StringSource::SectionData` (which means "string found in section bytes") would make section-name rows indistinguishable from byte-scanned content to downstream `source` filters and classification context. A new `StringSource::SectionName` variant keeps the two distinguishable. +- KTD9. **`--enc ascii` becomes a content filter.** With extraction no longer labeling rows `Encoding::Ascii`, `EncodingFilter::Exact(Encoding::Ascii)` would match nothing. `--enc ascii` is redefined to match rows whose text is pure-ASCII (`text.is_ascii()`), computed from content rather than stored encoding, so the documented CLI flag keeps working. `--enc utf8`/`--enc utf16*` are unchanged. --- @@ -60,12 +63,12 @@ Carried from the origin requirements doc. - R1. Import names emit as `FoundString`s with `source: StringSource::ImportName`, always carrying `Tag::Import`, at `confidence: 1.0`. - R2. Imports matching the crypto, network, or file-I/O API sets additionally carry `Tag::Crypto`, `Tag::Network`, or `Tag::FileIO`; non-matching imports carry only `Tag::Import`. -- R3. An import's originating library populates the `section` field when present; when absent, a format-appropriate default is used (`.idata` for PE, `.dynsym` for ELF). +- R3. An import's originating library populates the `section` field when present; when absent, a format-appropriate default is used (`.idata` for PE, `.dynsym` for ELF, `__LINKEDIT` for Mach-O). Mach-O imports never carry a library -- the parser leaves it `None` (`src/container/macho/mod.rs:136`) -- so Mach-O always uses the default. - R4. An import's RVA populates from `ImportInfo::address` when present (optional). - R5. Export names emit as `FoundString`s with `source: StringSource::ExportName`, always carrying `Tag::Export`, with RVA from `ExportInfo::address`. - R6. Exports run through `SymbolDemangler`; successfully demangled exports additionally carry `Tag::DemangledSymbol`. - R7. Exports whose name is a known entry point (`main`, `_start`, `DllMain`, `WinMain`, `wWinMain`) additionally carry `Tag::EntryPoint`. -- R8. Section names emit as standalone `FoundString`s with `source: StringSource::SectionData` at `confidence: 1.0`, ranked alongside other output. +- R8. Section names emit as standalone `FoundString`s with `source: StringSource::SectionName` (a new variant, distinct from `SectionData`) at `confidence: 1.0`, ranked alongside other output. **Type system and ranking** @@ -77,6 +80,11 @@ Carried from the origin requirements doc. - R11. The tagged classifier replaces the untagged import/export emission in the extractor so each symbol is emitted once, already tagged; section-name emission is added to the same path. - R12. Tagged symbol strings flow through the remaining pipeline stages with their classifier-assigned tags preserved, so the Import/Export and new semantic boosts apply to live output. +**Encoding normalization** + +- R15. The ASCII extraction path emits `Encoding::Utf8`; no extracted row carries `Encoding::Ascii`. The `Encoding::Ascii` enum variant is retained for compatibility but is not produced as output. +- R16. `--enc ascii` filters by content -- it matches rows whose text is pure-ASCII (`text.is_ascii()`), independent of stored encoding. `--enc utf8` and `--enc utf16*` are unchanged. + **Tests and docs** - R13. `tests/classification_symbols.rs` covers import classification, export classification, API-set detection (at least 3 representative symbols per category plus a negative case), section-name extraction, and end-to-end `extract_symbol_strings` output against the ELF, PE, and Mach-O fixtures. @@ -88,8 +96,8 @@ Carried from the origin requirements doc. - AE1. **Covers R2.** Import `CreateFileW` yields `Import` + `FileIO`; `connect` yields `Import` + `Network`; `EVP_EncryptInit` yields `Import` + `Crypto`; `printf` yields `Import` only. - AE2. **Covers R6, R7.** A C++-mangled export demangles and yields `Export` + `DemangledSymbol`; export `main` yields `Export` + `EntryPoint`; export `DllMain` yields `Export` + `EntryPoint`. -- AE3. **Covers R3.** A PE import from `kernel32.dll` sets `section` to `kernel32.dll`; an ELF import with no library sets `section` to `.dynsym`. -- AE4. **Covers R11, R12.** Running `stringy` on a PE fixture (default `include_symbols`) shows `CreateFileW` exactly once, tagged `import` + `fileio` -- not a duplicate untagged row, and not dropped by dedup. +- AE3. **Covers R3.** A PE import from `kernel32.dll` sets `section` to `kernel32.dll`; an ELF import with no library sets `section` to `.dynsym`; a Mach-O import (always library-less) sets `section` to `__LINKEDIT`. +- AE4. **Covers R11, R12.** Running `stringy` on a PE fixture (default `include_symbols`) shows `CreateFileW` exactly once, tagged `import` + `fileio` -- not a duplicate untagged row. Because the byte-scanned occurrence and the classifier occurrence both carry UTF-8 encoding (KTD7), dedup merges them into the single tagged row rather than splitting on encoding. --- @@ -97,11 +105,11 @@ Carried from the origin requirements doc. ### U1. New Tag variants and full co-change wiring -- **Goal:** Add `Crypto`, `Network`, `FileIO`, `EntryPoint` to the `Tag` enum and thread them through every site adding a variant touches, so the crate compiles and all tag-enumerating tests pass. -- **Requirements:** R9 +- **Goal:** Add `Crypto`, `Network`, `FileIO`, `EntryPoint` to the `Tag` enum and a `SectionName` variant to the `StringSource` enum, threading both through every site adding a variant touches, so the crate compiles and all variant-enumerating tests pass. +- **Requirements:** R9; the new `StringSource::SectionName` variant that R8 consumes - **Dependencies:** none - **Files:** `src/types/mod.rs`, `src/output/table/formatting.rs`, `src/output/yara/mod.rs`, `src/main.rs`, `src/types/tests.rs`, `src/output/json.rs`, `tests/integration_cli.rs`, `tests/output_yara_integration.rs` -- **Approach:** Add the four variants with serde renames (`crypto`, `network`, `fileio`, `entry-point`). Add `FromStr` arms. Add display arms in `tag_to_display_string` (`src/output/table/formatting.rs:81-103`) and YARA `tag_name` (`src/output/yara/mod.rs:164-187`) -- both are compiler-enforced exhaustive matches. Add the canonical strings to both `--only-tags` and `--no-tags` `long_help` literals (`src/main.rs:108-111`, `122-125`; must be string literals, not `const`/`concat!`). Extend the variant-enumerating test lists. +- **Approach:** Add the four variants with serde renames (`crypto`, `network`, `fileio`, `entry-point`). Add `FromStr` arms. Add display arms in `tag_to_display_string` (`src/output/table/formatting.rs:81-103`) and YARA `tag_name` (`src/output/yara/mod.rs:164-187`) -- both are compiler-enforced exhaustive matches. Add the canonical strings to both `--only-tags` and `--no-tags` `long_help` literals (`src/main.rs:108-111`, `122-125`; must be string literals, not `const`/`concat!`). Extend the variant-enumerating test lists. Separately, add a `StringSource::SectionName` variant to the `StringSource` enum and update its match and serialization sites by mirroring the existing `StringSource` variants (e.g. `ImportName`, `ExportName`). - **Patterns to follow:** existing `DemangledSymbol` variant (serde rename + `FromStr` arm) in `src/types/mod.rs`; the exhaustive matches in `formatting.rs:81-103` and `yara/mod.rs:164-187`. - **Test scenarios:** - Covers R9. `from_str("crypto"/"network"/"fileio"/"entry-point")` return the matching variants; an unknown string still returns `Err` (extend `test_tag_from_str_all_variants`, `src/types/tests.rs:126-148`). @@ -129,31 +137,46 @@ Carried from the origin requirements doc. - **Requirements:** R1, R2, R3, R4, R5, R6, R7, R8, R13; AE1, AE2, AE3 - **Dependencies:** U1 - **Files:** `src/classification/imports.rs` (new), `src/classification/mod.rs`, `tests/classification_symbols.rs` (new) -- **Approach:** New `ImportClassifier` holding a `SymbolDemangler` and three `LazyLock>` API sets (crypto, network, file-I/O) seeded from the origin lists. `process_imports`: build via `FoundString::new(name, Utf8, 0, len, ImportName)` then builder chain -- tags `[Import]` plus any semantic match, `.with_section(library or .idata/.dynsym default)`, `.with_rva` when `address` is `Some`, `.with_confidence(1.0)`. `process_exports`: tags `[Export]`, RVA from the required `address`, then `demangler.demangle(&mut fs)`, then entry-point check on `fs.text`. `process_section_names`: one `FoundString` per section name, source `SectionData`, confidence 1.0. `extract_symbol_strings(&ContainerInfo)` composes all three. Export `ImportClassifier` and `extract_symbol_strings` from `src/classification/mod.rs` (`pub mod imports;` + `pub use`). +- **Approach:** New `ImportClassifier` holding a `SymbolDemangler` and three `LazyLock>` API sets (crypto, network, file-I/O) seeded from the origin lists. `process_imports`: build via `FoundString::new(name, Utf8, 0, len, ImportName)` then builder chain -- tags `[Import]` plus any semantic match, `.with_section(library or .idata/.dynsym default)`, `.with_rva` when `address` is `Some`, `.with_confidence(1.0)`. `process_exports`: tags `[Export]`, RVA from the required `address`, then `demangler.demangle(&mut fs)`, then entry-point check on `fs.text`. `process_section_names`: one `FoundString` per section name, source `StringSource::SectionName` (the U1 variant), encoding `Utf8`, confidence 1.0. `extract_symbol_strings(&ContainerInfo)` composes all three. Export `ImportClassifier` and `extract_symbol_strings` from `src/classification/mod.rs` (`pub mod imports;` + `pub use`). - **Technical design (directional, not a spec):** export processing order -- `new(... ExportName) -> with_tags([Export]) -> with_rva(address) -> demangle(&mut) [adds DemangledSymbol] -> if text in ENTRY_POINTS push EntryPoint`. - **Patterns to follow:** `LazyLock` idiom in `src/classification/patterns/paths.rs:30-43` (`HashSet::from([...])`); `FoundString::new()` + builder chain only, never struct literals (GOTCHAS); `SymbolDemangler::demangle` in `src/classification/symbols.rs`; fixture loading via `get_fixture_path` in `tests/classification_integration_tests.rs`. - **Execution note:** Implement classifier behavior test-first. - **Test scenarios** (`tests/classification_symbols.rs`): - Covers R1. Every emitted import carries `Tag::Import`, source `ImportName`, confidence 1.0. - Covers AE1. `CreateFileW` -> `Import`+`FileIO`; `connect` -> `Import`+`Network`; `EVP_EncryptInit` -> `Import`+`Crypto`; `printf` -> `Import` only (at least 3 matches per category plus the negative). - - Covers AE3 / R3. PE import with library `kernel32.dll` -> section `kernel32.dll`; PE import with no library -> `.idata`; ELF import with no library -> `.dynsym`. + - Covers AE3 / R3. PE import with library `kernel32.dll` -> section `kernel32.dll`; PE import with no library -> `.idata`; ELF import with no library -> `.dynsym`; Mach-O import (always library-less) -> `__LINKEDIT`. - Covers R4. Import with `address: Some(x)` sets `rva` to `x`; `address: None` leaves `rva` unset. - Covers R5. Every export carries `Tag::Export`, source `ExportName`, RVA from `address`. - Covers AE2 / R6, R7. A C++-mangled export gains `DemangledSymbol` and demangled text; `main`/`_start`/`DllMain`/`WinMain`/`wWinMain` gain `EntryPoint`; an ordinary export gains neither. - - Covers R8. Section names emit as `FoundString`s with source `SectionData`, confidence 1.0. + - Covers R8. Section names emit as `FoundString`s with source `StringSource::SectionName`, confidence 1.0. - Covers R13. `extract_symbol_strings` on parsed ELF/PE/Mach-O fixtures returns tagged import/export/section-name strings; JSON-form comparisons use the serde-renamed tag strings. - **Verification:** `tests/classification_symbols.rs` passes; `extract_symbol_strings` is reachable as `stringy::classification::extract_symbol_strings`. +### U6. Normalize ASCII extraction to UTF-8 and redefine `--enc ascii` + +- **Goal:** Stop labeling extracted ASCII content as `Encoding::Ascii` so symbol rows merge under one UTF-8 dedup key (KTD7), and redefine `--enc ascii` as a content filter (KTD9) so the CLI flag keeps working. +- **Requirements:** R15, R16; enables AE4 +- **Dependencies:** none (independent of the tag work; U4 depends on this unit) +- **Files:** `src/extraction/ascii/extraction.rs`, `src/extraction/traits.rs`, `src/main.rs`, `src/pipeline/config.rs`, `src/pipeline/filter.rs`, `src/extraction/ascii/tests.rs`, `src/extraction/tests.rs`, `src/types/tests.rs`, affected insta snapshots +- **Approach:** Change the ASCII extraction pass (`src/extraction/ascii/extraction.rs:99,124,135`) to emit `Encoding::Utf8` instead of `Encoding::Ascii`. The `enabled_encodings` `Ascii` entry (`src/extraction/traits.rs:100`) still gates whether the ASCII byte-scan runs -- it controls scanning, not output labeling -- so the default stays as-is. Add a content-matching `EncodingFilter` variant (e.g. `AsciiContent`) in `src/pipeline/config.rs` whose `src/pipeline/filter.rs` arm matches `s.text.is_ascii()`, and map `CliEncoding::Ascii` (`src/main.rs:50`) to it instead of `EncodingFilter::Exact(Encoding::Ascii)`. Retain the `Encoding::Ascii` enum variant (no output uses it). Update tests asserting `Encoding::Ascii` on extracted strings and regenerate snapshots. +- **Execution note:** Capture current ASCII-labeled output via snapshots before the relabel, so the diff documents the encoding shift. +- **Test scenarios:** + - Covers R15. A pure-ASCII string extracted from a section is labeled `Encoding::Utf8`, not `Encoding::Ascii`. + - Covers R16. `--enc ascii` returns rows whose text is pure-ASCII (e.g. `CreateFileW`) and excludes rows whose text contains non-ASCII characters, regardless of stored encoding. + - Edge: a UTF-8 string with non-ASCII characters is excluded by `--enc ascii` but included by `--enc utf8`. + - Regression: `--enc utf8` and `--enc utf16*` filtering behavior is unchanged. +- **Verification:** no extracted row carries `Encoding::Ascii`; `--enc ascii` filters by text content; the encoding unit tests and regenerated snapshots pass. + ### U4. Replace the untagged symbol emission in the extractor - **Goal:** Route `extract_symbol_strings` into the extraction path as the single symbol emission point so tagged symbols reach live output and the boosts fire. - **Requirements:** R11, R12; AE4 -- **Dependencies:** U3 (also U1, U2 for the tags and their boosts) +- **Dependencies:** U3, U6 (also U1, U2 for the tags and their boosts) - **Files:** `src/extraction/basic_extractor.rs`, affected insta snapshots (e.g. the `tests/snapshots` for `integration_flows_1_5`) -- **Approach:** In `collect_all_strings`, replace the import/export push loops at `basic_extractor.rs:67-89` with a call to `extract_symbol_strings(container_info)`, still inside the `if config.include_symbols` block (which now also governs section-name rows). Dedup then merges by text; `classify_strings` appends semantic tags without clobbering the classifier's (KTD2). Regenerate insta snapshots whose fixture output now shows tagged, reranked symbols. `test_binary.c` is unchanged, so no `just gen-fixtures` -- snapshot regeneration only. +- **Approach:** In `collect_all_strings`, replace the import/export push loops at `basic_extractor.rs:67-89` with a call to `extract_symbol_strings(container_info)`, still inside the `if config.include_symbols` block (which now also governs section-name rows). Symbol strings are emitted as `Utf8`, and with U6 normalizing the byte-scan to UTF-8 as well (KTD7), a byte-scanned occurrence of the same name (e.g. from PE `.idata`) shares the `(text, Utf8)` dedup key and merges into the single tagged row rather than producing a separate untagged `Ascii` row; `classify_strings` then appends semantic tags without clobbering the classifier's (KTD2). Regenerate insta snapshots whose fixture output now shows tagged, reranked symbols. `test_binary.c` is unchanged, so no `just gen-fixtures` -- snapshot regeneration only. - **Execution note:** Capture current snapshots before the change, then regenerate after, so the snapshot diff is the reviewable record of the output shift. - **Test scenarios:** - - Covers AE4 / R11. The pipeline on a PE fixture yields exactly one row per symbol -- no duplicate untagged-plus-tagged pair. + - Covers AE4 / R11. The pipeline on a PE fixture yields exactly one row per symbol -- no duplicate untagged-plus-tagged pair (the byte-scanned and classifier occurrences merge under the shared UTF-8 dedup key). - Covers R12. A symbol that also matches a semantic pattern carries both its classifier tags and any pipeline-appended tags after `classify_strings`. - Covers R11. With `include_symbols = false`, no import/export or section-name rows are emitted; with the default (`true`), they are. - Regression: existing flow/snapshot tests pass after regeneration, and symbol strings rank above incidental noise. @@ -175,7 +198,8 @@ Carried from the origin requirements doc. ## Scope Boundaries - New CLI flags to filter the four new tags -- out of scope; the existing `--only-tags`/`--no-tags` machinery covers them once the variants are wired. -- Redesigning deduplication or string-ordering behavior -- out of scope. +- Redesigning deduplication internals or string-ordering remains out of scope. The one exception is encoding normalization: U6 relabels extracted ASCII content as UTF-8 and redefines `--enc ascii` as a content filter (R15, R16), which is in scope. +- Removing the `Encoding::Ascii` enum variant -- out of scope; it is retained for compatibility, just not produced as extraction output. ### Deferred to Follow-Up Work @@ -185,7 +209,8 @@ Carried from the origin requirements doc. ## Risks & Dependencies -- **Snapshot churn.** Routing tagged symbols into default output shifts rank order, so insta snapshots that capture fixture output (notably `integration_flows_1_5`) will change and must be regenerated. This is expected; if a snapshot diff looks wrong (e.g. a symbol losing its tag), investigate before accepting rather than blanket-accepting. +- **Snapshot churn (broad).** Two changes shift fixture output: routing tagged symbols into default output reranks rows, and U6 relabels every previously-`Ascii` row as `Utf8`. Insta snapshots capturing fixture output (notably `integration_flows_1_5`) will change widely and must be regenerated. Expected; if a diff looks wrong (a symbol losing its tag, a row vanishing), investigate before accepting rather than blanket-accepting. +- **CLI contract change for `--enc ascii` (R16).** The flag's meaning shifts from "stored encoding is ASCII" to "text content is pure-ASCII". Output for `--enc ascii` will differ from prior releases. Update the AGENTS.md CLI table and any user-facing docs describing `--enc`. - **Append-only classification (KTD2).** R12 depends on `classify_strings` pushing tags only when absent. If that behavior changes, classifier tags could be lost. - **Field-shape asymmetry.** `ExportInfo::address` is a required `u64` while `ImportInfo::address` is `Option` -- export RVA is always set, import RVA is conditional (R4, R5 reflect this). - **Fixtures gitignored.** `just gen-fixtures` must run before tests; `test_binary.c` is unchanged here, so only snapshot regeneration is needed, not fixture rebuild. @@ -202,5 +227,6 @@ Carried from the origin requirements doc. - `src/classification/ranking.rs:131-132` -- existing Import/Export +60 boosts; `:304-349` -- the paired `test_default_config_values`. - `src/classification/patterns/paths.rs:30-43` -- `LazyLock>` idiom to mirror. - Exhaustive-match and variant-list co-change sites (U1): `tag_to_display_string` (`src/output/table/formatting.rs:81-103`), YARA `tag_name` (`src/output/yara/mod.rs:164-187`), CLI `long_help` (`src/main.rs:108-111`, `122-125`), and test lists in `src/types/tests.rs:126-148`, `src/output/json.rs:159-181`, `src/output/table/formatting.rs:224-245`, `tests/output_yara_integration.rs:154-177`, `tests/integration_cli.rs:264-291`. +- Encoding model (U6): `Encoding` enum at `src/types/mod.rs:13` (`Ascii`, `Utf8`, `Utf16Le`, `Utf16Be`); ASCII extraction emits `Encoding::Ascii` at `src/extraction/ascii/extraction.rs:99,124,135`; default `enabled_encodings: [Ascii, Utf8]` at `src/extraction/traits.rs:100`; `CliEncoding::Ascii -> EncodingFilter::Exact(Encoding::Ascii)` at `src/main.rs:50`; the `EncodingFilter` match arm at `src/pipeline/filter.rs:44`. - `docs/src/classification.md:77,80,120` -- stale `once_cell` example that U5 fixes. - GitHub issue #20 -- originating specification (its "nothing converts imports/exports into FoundStrings" premise is corrected: the extractor already emits them untagged, which U4 replaces). From cd743593e961ac96c6cefca193101530259fc5b1 Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Fri, 19 Jun 2026 12:49:29 -0400 Subject: [PATCH 08/21] fix(lint): use sort_by_key for descending sorts The floated toolchain's clippy (1.96) flags unnecessary_sort_by on these pre-existing descending sorts. Convert to sort_by_key(Reverse(..)) -- same stable comparator, behavior-preserving -- so clippy -D warnings passes. Signed-off-by: UncleSp1d3r --- examples/basic_extraction.rs | 2 +- examples/output_formats.rs | 2 +- src/classification/ranking.rs | 2 +- src/extraction/dedup/mod.rs | 2 +- src/output/mod.rs | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/basic_extraction.rs b/examples/basic_extraction.rs index ce05ebb..85d3750 100644 --- a/examples/basic_extraction.rs +++ b/examples/basic_extraction.rs @@ -48,7 +48,7 @@ fn main() -> Result<(), Box> { // Display the top 20 strings by score let mut sorted_strings = strings.clone(); - sorted_strings.sort_by(|a, b| b.score.cmp(&a.score)); + sorted_strings.sort_by_key(|b| std::cmp::Reverse(b.score)); println!("Top strings by score:"); println!("{:-<60}", ""); diff --git a/examples/output_formats.rs b/examples/output_formats.rs index f52aaf0..56424a2 100644 --- a/examples/output_formats.rs +++ b/examples/output_formats.rs @@ -47,7 +47,7 @@ fn main() -> Result<(), Box> { // Limit to top 50 strings for demonstration let mut sorted_strings = strings; - sorted_strings.sort_by(|a, b| b.score.cmp(&a.score)); + sorted_strings.sort_by_key(|b| std::cmp::Reverse(b.score)); let top_strings: Vec<_> = sorted_strings.into_iter().take(50).collect(); // Create output metadata diff --git a/src/classification/ranking.rs b/src/classification/ranking.rs index 7a1a9e2..ca2a493 100644 --- a/src/classification/ranking.rs +++ b/src/classification/ranking.rs @@ -275,7 +275,7 @@ impl RankingEngine { /// otherwise ordering will reflect uninitialized or stale scores. This uses /// a stable sort, so the relative order of equal scores is preserved. pub fn rank_strings(&self, strings: &mut [FoundString]) { - strings.sort_by(|a, b| b.score.cmp(&a.score)); + strings.sort_by_key(|b| std::cmp::Reverse(b.score)); } } diff --git a/src/extraction/dedup/mod.rs b/src/extraction/dedup/mod.rs index c4dc0ed..6a48aee 100644 --- a/src/extraction/dedup/mod.rs +++ b/src/extraction/dedup/mod.rs @@ -217,7 +217,7 @@ pub fn deduplicate( .collect(); // Sort by combined_score descending - canonical_strings.sort_by(|a, b| b.combined_score.cmp(&a.combined_score)); + canonical_strings.sort_by_key(|b| std::cmp::Reverse(b.combined_score)); canonical_strings } diff --git a/src/output/mod.rs b/src/output/mod.rs index 5d62932..43b80dc 100644 --- a/src/output/mod.rs +++ b/src/output/mod.rs @@ -199,7 +199,7 @@ impl OutputMetadata { } } let mut sorted: Vec<(Tag, usize)> = counts.into_iter().collect(); - sorted.sort_by(|a, b| b.1.cmp(&a.1)); + sorted.sort_by_key(|b| std::cmp::Reverse(b.1)); sorted.truncate(limit); sorted } From b1973052f9072470d025398c758cf6c47fa8f994 Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Fri, 19 Jun 2026 12:49:42 -0400 Subject: [PATCH 09/21] feat(types): add Crypto/Network/FileIO/EntryPoint tags and SectionName source Add four semantic Tag variants (serde renames crypto/network/fileio/entry-point) and a StringSource::SectionName variant for section-name rows. Thread the tags through FromStr, the table and YARA display formatters, and both CLI --only-tags /--no-tags long_help literals. Extend the variant-enumerating test lists and the JSON source-serialization test; regenerate the YARA all-tags snapshot. Implements U1 (R9). Foundational unit for the symbol classification pipeline. Signed-off-by: UncleSp1d3r --- src/main.rs | 4 ++-- src/output/json.rs | 18 +++++++++++++++++- src/output/table/formatting.rs | 8 ++++++++ src/output/yara/mod.rs | 4 ++++ src/types/mod.rs | 14 ++++++++++++++ src/types/tests.rs | 4 ++++ tests/integration_cli.rs | 6 +++++- tests/output_yara_integration.rs | 4 ++++ ...t_yara_integration__yara_all_tag_types.snap | 12 ++++++++++++ 9 files changed, 70 insertions(+), 4 deletions(-) diff --git a/src/main.rs b/src/main.rs index fb4f688..f9c09f1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -108,7 +108,7 @@ struct Cli { long_help = "Include only strings with this tag. Repeat the flag for multiple tags \ (OR logic).\nValid tags: url, domain, ipv4, ipv6, filepath, regpath, guid, email, \ b64, fmt, user-agent-ish, demangled, import, export, version, manifest, resource, \ - dylib-path, rpath, rpath-var, framework-path" + dylib-path, rpath, rpath-var, framework-path, crypto, network, fileio, entry-point" )] only_tags: Vec, @@ -121,7 +121,7 @@ struct Cli { long_help = "Exclude strings with this tag. Repeat the flag for multiple tags \ (OR logic).\nValid tags: url, domain, ipv4, ipv6, filepath, regpath, guid, email, \ b64, fmt, user-agent-ish, demangled, import, export, version, manifest, resource, \ - dylib-path, rpath, rpath-var, framework-path" + dylib-path, rpath, rpath-var, framework-path, crypto, network, fileio, entry-point" )] no_tags: Vec, diff --git a/src/output/json.rs b/src/output/json.rs index cb0cca0..1ac333a 100644 --- a/src/output/json.rs +++ b/src/output/json.rs @@ -178,6 +178,10 @@ mod tests { Tag::Rpath, Tag::RpathVariable, Tag::FrameworkPath, + Tag::Crypto, + Tag::Network, + Tag::FileIO, + Tag::EntryPoint, ]; let strings = vec![make_string("tagged").with_tags(tags)]; let output = format_json(&strings, &make_metadata(1)).expect("Formatting should succeed"); @@ -211,6 +215,10 @@ mod tests { "rpath", "rpath-var", "framework-path", + "crypto", + "network", + "fileio", + "entry-point", ]; for name in expected { @@ -263,8 +271,15 @@ mod tests { 1, StringSource::DebugInfo, ), + FoundString::new( + "g".to_string(), + Encoding::Ascii, + 6, + 1, + StringSource::SectionName, + ), ]; - let output = format_json(&strings, &make_metadata(6)).expect("Formatting should succeed"); + let output = format_json(&strings, &make_metadata(7)).expect("Formatting should succeed"); let lines: Vec<&str> = output.lines().collect(); assert_eq!(parse_line(lines[0])["source"], "SectionData"); assert_eq!(parse_line(lines[1])["source"], "ImportName"); @@ -272,6 +287,7 @@ mod tests { assert_eq!(parse_line(lines[3])["source"], "ResourceString"); assert_eq!(parse_line(lines[4])["source"], "LoadCommand"); assert_eq!(parse_line(lines[5])["source"], "DebugInfo"); + assert_eq!(parse_line(lines[6])["source"], "SectionName"); } #[test] diff --git a/src/output/table/formatting.rs b/src/output/table/formatting.rs index 3891eaf..9668689 100644 --- a/src/output/table/formatting.rs +++ b/src/output/table/formatting.rs @@ -100,6 +100,10 @@ pub(crate) fn tag_to_display_string(tag: &Tag) -> String { Tag::Rpath => "rpath".to_string(), Tag::RpathVariable => "rpath-var".to_string(), Tag::FrameworkPath => "framework-path".to_string(), + Tag::Crypto => "crypto".to_string(), + Tag::Network => "network".to_string(), + Tag::FileIO => "fileio".to_string(), + Tag::EntryPoint => "entry-point".to_string(), } } @@ -243,6 +247,10 @@ mod tests { Tag::Rpath, Tag::RpathVariable, Tag::FrameworkPath, + Tag::Crypto, + Tag::Network, + Tag::FileIO, + Tag::EntryPoint, ]; for tag in all_tags { diff --git a/src/output/yara/mod.rs b/src/output/yara/mod.rs index 5043278..e42fbf7 100644 --- a/src/output/yara/mod.rs +++ b/src/output/yara/mod.rs @@ -184,6 +184,10 @@ fn tag_name(tag: &Tag) -> &'static str { Tag::Rpath => "rpath", Tag::RpathVariable => "rpath-var", Tag::FrameworkPath => "framework-path", + Tag::Crypto => "crypto", + Tag::Network => "network", + Tag::FileIO => "fileio", + Tag::EntryPoint => "entry-point", } } diff --git a/src/types/mod.rs b/src/types/mod.rs index aededfa..af2321c 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -55,6 +55,14 @@ pub enum Tag { RpathVariable, #[serde(rename = "framework-path")] FrameworkPath, + #[serde(rename = "crypto")] + Crypto, + #[serde(rename = "network")] + Network, + #[serde(rename = "fileio")] + FileIO, + #[serde(rename = "entry-point")] + EntryPoint, } impl std::str::FromStr for Tag { @@ -83,6 +91,10 @@ impl std::str::FromStr for Tag { "rpath" => Ok(Tag::Rpath), "rpath-var" => Ok(Tag::RpathVariable), "framework-path" => Ok(Tag::FrameworkPath), + "crypto" => Ok(Tag::Crypto), + "network" => Ok(Tag::Network), + "fileio" => Ok(Tag::FileIO), + "entry-point" => Ok(Tag::EntryPoint), _ => Err(format!("unknown tag: {s}")), } } @@ -112,6 +124,8 @@ pub enum SectionType { pub enum StringSource { /// String found in section data SectionData, + /// Section name emitted as a standalone string + SectionName, /// String from import table ImportName, /// String from export table diff --git a/src/types/tests.rs b/src/types/tests.rs index d1a2940..be039a3 100644 --- a/src/types/tests.rs +++ b/src/types/tests.rs @@ -145,6 +145,10 @@ fn test_tag_from_str_all_variants() { ("rpath", Tag::Rpath), ("rpath-var", Tag::RpathVariable), ("framework-path", Tag::FrameworkPath), + ("crypto", Tag::Crypto), + ("network", Tag::Network), + ("fileio", Tag::FileIO), + ("entry-point", Tag::EntryPoint), ]; for (input, expected) in cases { diff --git a/tests/integration_cli.rs b/tests/integration_cli.rs index 2b84057..28f11c5 100644 --- a/tests/integration_cli.rs +++ b/tests/integration_cli.rs @@ -287,7 +287,11 @@ fn cli_help_lists_all_canonical_tags() { .stdout(predicate::str::contains("dylib-path")) .stdout(predicate::str::contains("rpath")) .stdout(predicate::str::contains("rpath-var")) - .stdout(predicate::str::contains("framework-path")); + .stdout(predicate::str::contains("framework-path")) + .stdout(predicate::str::contains("crypto")) + .stdout(predicate::str::contains("network")) + .stdout(predicate::str::contains("fileio")) + .stdout(predicate::str::contains("entry-point")); } #[test] diff --git a/tests/output_yara_integration.rs b/tests/output_yara_integration.rs index e8cf416..272ab52 100644 --- a/tests/output_yara_integration.rs +++ b/tests/output_yara_integration.rs @@ -174,6 +174,10 @@ fn test_yara_all_tag_types() { make_string("rpath").with_tags(vec![Tag::Rpath]), make_string("rpathvar").with_tags(vec![Tag::RpathVariable]), make_string("framework").with_tags(vec![Tag::FrameworkPath]), + make_string("crypto").with_tags(vec![Tag::Crypto]), + make_string("network").with_tags(vec![Tag::Network]), + make_string("fileio").with_tags(vec![Tag::FileIO]), + make_string("entrypoint").with_tags(vec![Tag::EntryPoint]), ]; let output = format_yara(&strings, &make_metadata("tags.exe", strings.len())).unwrap(); assert_snapshot!(output); diff --git a/tests/snapshots/output_yara_integration__yara_all_tag_types.snap b/tests/snapshots/output_yara_integration__yara_all_tag_types.snap index 6b5cb58..44b8d83 100644 --- a/tests/snapshots/output_yara_integration__yara_all_tag_types.snap +++ b/tests/snapshots/output_yara_integration__yara_all_tag_types.snap @@ -39,12 +39,21 @@ rule tags_exe_strings { // tag: b64 // score: 0 $b64_1 = "b64" ascii + // tag: crypto + // score: 0 + $crypto_1 = "crypto" ascii // tag: demangled // score: 0 $demangled_1 = "demangled" ascii // tag: dylib-path // score: 0 $dylib_path_1 = "dylib" ascii + // tag: entry-point + // score: 0 + $entry_point_1 = "entrypoint" ascii + // tag: fileio + // score: 0 + $fileio_1 = "fileio" ascii // tag: filepath // score: 0 $filepath_1 = "path" ascii @@ -63,6 +72,9 @@ rule tags_exe_strings { // tag: ipv6 // score: 0 $ipv6_1 = "ipv6" ascii + // tag: network + // score: 0 + $network_1 = "network" ascii // tag: regpath // score: 0 $regpath_1 = "reg" ascii From 4559f64ae5d8d6f05d8f21c14bfadc1f32ba35f0 Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Fri, 19 Jun 2026 12:50:54 -0400 Subject: [PATCH 10/21] feat(ranking): add boosts for Crypto/Network/FileIO/EntryPoint tags Crypto +50, Network +45, FileIO +35, EntryPoint +40 in RankingConfig::default(). Extend test_default_config_values and add a test that a Crypto-tagged string outranks an otherwise identical untagged peer by at least the boost. Implements U2 (R10). Signed-off-by: UncleSp1d3r --- src/classification/ranking.rs | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/classification/ranking.rs b/src/classification/ranking.rs index ca2a493..54bc761 100644 --- a/src/classification/ranking.rs +++ b/src/classification/ranking.rs @@ -140,6 +140,10 @@ impl Default for RankingConfig { tag_boosts.insert(Tag::Base64, 10); tag_boosts.insert(Tag::FormatString, 15); tag_boosts.insert(Tag::UserAgent, 20); + tag_boosts.insert(Tag::Crypto, 50); + tag_boosts.insert(Tag::Network, 45); + tag_boosts.insert(Tag::FileIO, 35); + tag_boosts.insert(Tag::EntryPoint, 40); Self { section_weights, @@ -345,10 +349,29 @@ mod tests { assert_eq!(config.tag_boosts.get(&Tag::Base64), Some(&10)); assert_eq!(config.tag_boosts.get(&Tag::FormatString), Some(&15)); assert_eq!(config.tag_boosts.get(&Tag::UserAgent), Some(&20)); + assert_eq!(config.tag_boosts.get(&Tag::Crypto), Some(&50)); + assert_eq!(config.tag_boosts.get(&Tag::Network), Some(&45)); + assert_eq!(config.tag_boosts.get(&Tag::FileIO), Some(&35)); + assert_eq!(config.tag_boosts.get(&Tag::EntryPoint), Some(&40)); assert_eq!(config.noise_penalty_multiplier, 100); } + #[test] + fn test_crypto_tag_outranks_untagged_peer() { + let engine = RankingEngine::new(false); + let section_info = make_section_info(SectionType::StringData); + + let mut crypto = make_found_string(vec![Tag::Crypto], 1.0); + let mut untagged = make_found_string(vec![], 1.0); + engine.calculate_score(&mut crypto, Some(§ion_info)); + engine.calculate_score(&mut untagged, Some(§ion_info)); + + // The Crypto boost (+50) lifts the score at least 50 above an + // otherwise identical untagged string. + assert!(crypto.score >= untagged.score + 50); + } + #[test] fn test_score_calculation_high_value_url() { let engine = RankingEngine::new(true); From b85772f0605ff3a78ecf0130e337b5f9ada5cab3 Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Fri, 19 Jun 2026 12:55:22 -0400 Subject: [PATCH 11/21] feat(classification): add ImportClassifier and extract_symbol_strings New ImportClassifier converts ContainerInfo imports, exports, and section names into tagged FoundStrings: imports carry Tag::Import plus a semantic tag (Crypto/ Network/FileIO) from static LazyLock API sets, with section from library or a format default (.idata/.dynsym/__LINKEDIT) and RVA when present; exports carry Tag::Export with RVA, are demangled (Tag::DemangledSymbol), and gain Tag::EntryPoint for known entry points; section names emit with StringSource::SectionName. extract_symbol_strings composes all three. Symbols are emitted as Encoding::Utf8 so byte-scanned occurrences share the dedup key (KTD7). Provenance tests use hand-built ContainerInfo (KTD1). Implements U3 (R1-R8, R13; AE1-AE3). Signed-off-by: UncleSp1d3r --- src/classification/imports.rs | 249 ++++++++++++++++++++++++++++++ src/classification/mod.rs | 2 + tests/classification_symbols.rs | 261 ++++++++++++++++++++++++++++++++ 3 files changed, 512 insertions(+) create mode 100644 src/classification/imports.rs create mode 100644 tests/classification_symbols.rs diff --git a/src/classification/imports.rs b/src/classification/imports.rs new file mode 100644 index 0000000..f33c2da --- /dev/null +++ b/src/classification/imports.rs @@ -0,0 +1,249 @@ +//! Import/export symbol classification. +//! +//! Converts parsed [`ContainerInfo`] imports, exports, and section names into +//! tagged, scored [`FoundString`]s. This is the single emission point for +//! symbol strings in the extraction path: every import carries [`Tag::Import`] +//! (plus a semantic tag when its name matches a known crypto, network, or +//! file-I/O API), every export carries [`Tag::Export`] (plus +//! [`Tag::DemangledSymbol`] when demangling succeeds and [`Tag::EntryPoint`] +//! for known entry points), and each section name is emitted as a standalone +//! row with [`StringSource::SectionName`]. +//! +//! Symbol strings are emitted with [`Encoding::Utf8`] so that a byte-scanned +//! occurrence of the same name (e.g. from a PE `.idata` section) shares the +//! `(text, encoding)` deduplication key and merges into the single tagged row. + +use std::collections::HashSet; +use std::sync::LazyLock; + +use crate::classification::SymbolDemangler; +use crate::types::{ + BinaryFormat, ContainerInfo, Encoding, ExportInfo, FoundString, ImportInfo, SectionInfo, + StringSource, Tag, +}; + +/// Crypto-related API names. Matching imports gain [`Tag::Crypto`]. +static CRYPTO_APIS: LazyLock> = LazyLock::new(|| { + HashSet::from([ + "EVP_EncryptInit", + "EVP_DecryptInit", + "EVP_DigestInit", + "CryptEncrypt", + "CryptDecrypt", + "CryptAcquireContextW", + "CryptGenKey", + "BCryptEncrypt", + "BCryptDecrypt", + "AES_encrypt", + "AES_decrypt", + "SHA256_Init", + "MD5_Init", + ]) +}); + +/// Network-related API names. Matching imports gain [`Tag::Network`]. +static NETWORK_APIS: LazyLock> = LazyLock::new(|| { + HashSet::from([ + "connect", + "send", + "recv", + "socket", + "bind", + "listen", + "accept", + "getaddrinfo", + "gethostbyname", + "WSAStartup", + "WSASocketW", + "InternetOpenW", + "InternetConnectW", + "HttpSendRequestW", + ]) +}); + +/// File-I/O-related API names. Matching imports gain [`Tag::FileIO`]. +static FILEIO_APIS: LazyLock> = LazyLock::new(|| { + HashSet::from([ + "CreateFileW", + "CreateFileA", + "ReadFile", + "WriteFile", + "DeleteFileW", + "MoveFileW", + "CopyFileW", + "fopen", + "fread", + "fwrite", + "open", + "read", + "write", + ]) +}); + +/// Known program entry-point symbol names. Matching exports gain +/// [`Tag::EntryPoint`]. +static ENTRY_POINTS: LazyLock> = + LazyLock::new(|| HashSet::from(["main", "_start", "DllMain", "WinMain", "wWinMain"])); + +/// Classifies import/export symbols and section names into tagged strings. +#[derive(Debug, Default, Clone)] +pub struct ImportClassifier { + demangler: SymbolDemangler, +} + +impl ImportClassifier { + /// Create a new `ImportClassifier`. + #[must_use] + pub fn new() -> Self { + Self { + demangler: SymbolDemangler::new(), + } + } + + /// Convert import symbols into tagged `FoundString`s. + /// + /// Each import carries [`Tag::Import`] plus a semantic tag when its name + /// matches a known API set. The `section` field is populated from the + /// import's originating library when present, otherwise from a + /// format-appropriate default (see [`default_import_section`]). The RVA is + /// populated from `ImportInfo::address` when available. + #[must_use] + pub fn process_imports( + &self, + imports: &[ImportInfo], + format: BinaryFormat, + ) -> Vec { + imports + .iter() + .map(|import| { + let mut tags = vec![Tag::Import]; + if let Some(semantic) = semantic_tag(&import.name) { + tags.push(semantic); + } + + let section = import + .library + .clone() + .unwrap_or_else(|| default_import_section(format).to_string()); + + let length = import.name.len() as u32; + let mut found = FoundString::new( + import.name.clone(), + Encoding::Utf8, + 0, + length, + StringSource::ImportName, + ) + .with_tags(tags) + .with_section(section); + + if let Some(address) = import.address { + found = found.with_rva(address); + } + + found + }) + .collect() + } + + /// Convert export symbols into tagged `FoundString`s. + /// + /// Each export carries [`Tag::Export`] and an RVA from the always-present + /// `ExportInfo::address`. Exports are demangled (adding + /// [`Tag::DemangledSymbol`] on success), then checked against the known + /// entry-point names (adding [`Tag::EntryPoint`] on a match). + #[must_use] + pub fn process_exports(&self, exports: &[ExportInfo]) -> Vec { + exports + .iter() + .map(|export| { + let length = export.name.len() as u32; + let mut found = FoundString::new( + export.name.clone(), + Encoding::Utf8, + 0, + length, + StringSource::ExportName, + ) + .with_tags(vec![Tag::Export]) + .with_rva(export.address); + + // Demangle first; this may add Tag::DemangledSymbol and replace + // `text` with the demangled form (preserving existing tags). + self.demangler.demangle(&mut found); + + // Entry-point check runs against the resulting text. + if ENTRY_POINTS.contains(found.text.as_str()) { + found.tags.push(Tag::EntryPoint); + } + + found + }) + .collect() + } + + /// Convert section names into standalone `FoundString`s. + /// + /// Each section name is emitted with [`StringSource::SectionName`] so it is + /// distinguishable from byte-scanned section content downstream. + #[must_use] + pub fn process_section_names(&self, sections: &[SectionInfo]) -> Vec { + sections + .iter() + .map(|section| { + let length = section.name.len() as u32; + FoundString::new( + section.name.clone(), + Encoding::Utf8, + 0, + length, + StringSource::SectionName, + ) + }) + .collect() + } +} + +/// Return the semantic tag for an import name, if it matches a known API set. +/// +/// The crypto, network, and file-I/O sets are disjoint, so an import matches +/// at most one category. +fn semantic_tag(name: &str) -> Option { + if CRYPTO_APIS.contains(name) { + Some(Tag::Crypto) + } else if NETWORK_APIS.contains(name) { + Some(Tag::Network) + } else if FILEIO_APIS.contains(name) { + Some(Tag::FileIO) + } else { + None + } +} + +/// Format-appropriate default `section` for an import with no library name. +/// +/// Mach-O imports never carry a library (the parser leaves it `None`), so +/// Mach-O always uses this default. +fn default_import_section(format: BinaryFormat) -> &'static str { + match format { + BinaryFormat::Pe => ".idata", + BinaryFormat::Elf => ".dynsym", + BinaryFormat::MachO => "__LINKEDIT", + // Unknown formats fall back to raw scanning and carry no imports; + // use the ELF default as a neutral placeholder. + BinaryFormat::Unknown => ".dynsym", + } +} + +/// Convert all symbols and section names in a [`ContainerInfo`] into tagged +/// `FoundString`s. +/// +/// This is the single symbol emission point routed into the extraction path. +#[must_use] +pub fn extract_symbol_strings(container_info: &ContainerInfo) -> Vec { + let classifier = ImportClassifier::new(); + let mut strings = classifier.process_imports(&container_info.imports, container_info.format); + strings.extend(classifier.process_exports(&container_info.exports)); + strings.extend(classifier.process_section_names(&container_info.sections)); + strings +} diff --git a/src/classification/mod.rs b/src/classification/mod.rs index 0e682a6..9deddb0 100644 --- a/src/classification/mod.rs +++ b/src/classification/mod.rs @@ -42,10 +42,12 @@ use std::sync::LazyLock; use crate::types::{BinaryFormat, SectionType, StringContext, StringSource, Tag}; +pub mod imports; pub mod patterns; pub mod ranking; pub mod symbols; +pub use imports::{ImportClassifier, extract_symbol_strings}; pub use ranking::{RankingConfig, RankingEngine}; pub use symbols::SymbolDemangler; diff --git a/tests/classification_symbols.rs b/tests/classification_symbols.rs new file mode 100644 index 0000000..590a330 --- /dev/null +++ b/tests/classification_symbols.rs @@ -0,0 +1,261 @@ +//! Symbol classification tests for the import/export pipeline. +//! +//! Provenance assertions (section, source, rva) run against hand-built +//! `ContainerInfo` values so no byte-scan occurrence is in play -- per the +//! plan's KTD1, a symbol's source/section/rva are only deterministic when the +//! classifier is the sole emitter. + +use std::fs; + +use stringy::classification::{ImportClassifier, extract_symbol_strings}; +use stringy::container::{ContainerParser, ElfParser, MachoParser, PeParser}; +use stringy::types::{ + BinaryFormat, ExportInfo, FoundString, ImportInfo, SectionInfo, SectionType, StringSource, Tag, +}; + +fn get_fixture_path(name: &str) -> std::path::PathBuf { + std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests") + .join("fixtures") + .join(name) +} + +fn find<'a>(strings: &'a [FoundString], text: &str) -> &'a FoundString { + strings + .iter() + .find(|s| s.text == text) + .unwrap_or_else(|| panic!("expected a string with text {text:?}")) +} + +// --- Imports (R1, R2, R3, R4; AE1, AE3) -------------------------------------- + +#[test] +fn every_import_carries_import_tag_source_and_full_confidence() { + let classifier = ImportClassifier::new(); + let imports = vec![ + ImportInfo::new("printf".to_string()), + ImportInfo::new("connect".to_string()), + ImportInfo::new("CreateFileW".to_string()), + ]; + + let strings = classifier.process_imports(&imports, BinaryFormat::Pe); + + assert_eq!(strings.len(), 3); + for s in &strings { + assert!( + s.tags.contains(&Tag::Import), + "import must carry Tag::Import" + ); + assert_eq!(s.source, StringSource::ImportName); + assert_eq!(s.confidence, 1.0); + } +} + +#[test] +fn imports_gain_matching_semantic_tags() { + let classifier = ImportClassifier::new(); + let imports = vec![ + ImportInfo::new("CreateFileW".to_string()), + ImportInfo::new("connect".to_string()), + ImportInfo::new("EVP_EncryptInit".to_string()), + ImportInfo::new("printf".to_string()), + ]; + + let strings = classifier.process_imports(&imports, BinaryFormat::Pe); + + let create_file = find(&strings, "CreateFileW"); + assert!(create_file.tags.contains(&Tag::Import)); + assert!(create_file.tags.contains(&Tag::FileIO)); + + let connect = find(&strings, "connect"); + assert!(connect.tags.contains(&Tag::Import)); + assert!(connect.tags.contains(&Tag::Network)); + + let crypto = find(&strings, "EVP_EncryptInit"); + assert!(crypto.tags.contains(&Tag::Import)); + assert!(crypto.tags.contains(&Tag::Crypto)); + + // printf matches no semantic set: Import only. + let printf = find(&strings, "printf"); + assert_eq!(printf.tags, vec![Tag::Import]); +} + +#[test] +fn import_section_uses_library_then_format_default() { + let classifier = ImportClassifier::new(); + + // Library present: section is the library name. + let with_lib = classifier.process_imports( + &[ImportInfo::new("CreateFileW".to_string()).with_library("kernel32.dll".to_string())], + BinaryFormat::Pe, + ); + assert_eq!(with_lib[0].section.as_deref(), Some("kernel32.dll")); + + // No library: format-appropriate default. + let pe = classifier.process_imports(&[ImportInfo::new("f".to_string())], BinaryFormat::Pe); + assert_eq!(pe[0].section.as_deref(), Some(".idata")); + + let elf = classifier.process_imports(&[ImportInfo::new("f".to_string())], BinaryFormat::Elf); + assert_eq!(elf[0].section.as_deref(), Some(".dynsym")); + + // Mach-O imports are always library-less. + let macho = + classifier.process_imports(&[ImportInfo::new("f".to_string())], BinaryFormat::MachO); + assert_eq!(macho[0].section.as_deref(), Some("__LINKEDIT")); +} + +#[test] +fn import_rva_populates_only_when_address_present() { + let classifier = ImportClassifier::new(); + let imports = vec![ + ImportInfo::new("with_addr".to_string()).with_address(0x4010), + ImportInfo::new("no_addr".to_string()), + ]; + + let strings = classifier.process_imports(&imports, BinaryFormat::Elf); + + assert_eq!(find(&strings, "with_addr").rva, Some(0x4010)); + assert_eq!(find(&strings, "no_addr").rva, None); +} + +// --- Exports (R5, R6, R7; AE2) ----------------------------------------------- + +#[test] +fn every_export_carries_export_tag_source_and_rva() { + let classifier = ImportClassifier::new(); + let exports = vec![ + ExportInfo::new("alpha".to_string(), 0x1000), + ExportInfo::new("beta".to_string(), 0x2000), + ]; + + let strings = classifier.process_exports(&exports); + + assert_eq!(strings.len(), 2); + for s in &strings { + assert!(s.tags.contains(&Tag::Export)); + assert_eq!(s.source, StringSource::ExportName); + assert_eq!(s.confidence, 1.0); + } + assert_eq!(find(&strings, "alpha").rva, Some(0x1000)); + assert_eq!(find(&strings, "beta").rva, Some(0x2000)); +} + +#[test] +fn mangled_export_is_demangled_and_tagged() { + let classifier = ImportClassifier::new(); + let exports = vec![ExportInfo::new("_ZN3foo3barEv".to_string(), 0x3000)]; + + let strings = classifier.process_exports(&exports); + let demangled = &strings[0]; + + assert!(demangled.tags.contains(&Tag::Export)); + assert!(demangled.tags.contains(&Tag::DemangledSymbol)); + assert_eq!(demangled.original_text.as_deref(), Some("_ZN3foo3barEv")); + assert!(demangled.text.contains("foo")); + assert!(demangled.text.contains("bar")); +} + +#[test] +fn entry_point_exports_gain_entry_point_tag() { + let classifier = ImportClassifier::new(); + let exports = vec![ + ExportInfo::new("main".to_string(), 0x1000), + ExportInfo::new("DllMain".to_string(), 0x2000), + ExportInfo::new("_start".to_string(), 0x3000), + ExportInfo::new("WinMain".to_string(), 0x4000), + ExportInfo::new("wWinMain".to_string(), 0x5000), + ExportInfo::new("ordinary_export".to_string(), 0x6000), + ]; + + let strings = classifier.process_exports(&exports); + + for name in ["main", "DllMain", "_start", "WinMain", "wWinMain"] { + let s = find(&strings, name); + assert!( + s.tags.contains(&Tag::EntryPoint), + "{name} must be an entry point" + ); + } + + let ordinary = find(&strings, "ordinary_export"); + assert!(!ordinary.tags.contains(&Tag::EntryPoint)); + assert!(!ordinary.tags.contains(&Tag::DemangledSymbol)); + assert_eq!(ordinary.tags, vec![Tag::Export]); +} + +// --- Section names (R8) ------------------------------------------------------ + +#[test] +fn section_names_emit_as_section_name_source() { + let classifier = ImportClassifier::new(); + let sections = vec![ + SectionInfo::new(".text".to_string(), 0, 100, SectionType::Code, 1.0), + SectionInfo::new( + ".rodata".to_string(), + 100, + 100, + SectionType::StringData, + 1.0, + ), + ]; + + let strings = classifier.process_section_names(§ions); + + assert_eq!(strings.len(), 2); + for s in &strings { + assert_eq!(s.source, StringSource::SectionName); + assert_eq!(s.confidence, 1.0); + } + assert!(strings.iter().any(|s| s.text == ".text")); + assert!(strings.iter().any(|s| s.text == ".rodata")); +} + +// --- End-to-end against fixtures (R13) --------------------------------------- + +fn assert_symbol_contract(strings: &[FoundString]) { + // Section names are always present (every binary has sections). + assert!( + strings + .iter() + .any(|s| s.source == StringSource::SectionName), + "extract_symbol_strings must emit section-name rows" + ); + // Any import/export row carries the matching always-on tag. + for s in strings { + match s.source { + StringSource::ImportName => assert!(s.tags.contains(&Tag::Import)), + StringSource::ExportName => assert!(s.tags.contains(&Tag::Export)), + _ => {} + } + } +} + +#[test] +fn extract_symbol_strings_on_elf_fixture() { + let data = fs::read(get_fixture_path("test_binary_elf")) + .expect("read ELF fixture (run `just gen-fixtures`)"); + assert!(ElfParser::detect(&data)); + let info = ElfParser::new().parse(&data).expect("parse ELF"); + let strings = extract_symbol_strings(&info); + assert_symbol_contract(&strings); +} + +#[test] +fn extract_symbol_strings_on_pe_fixture() { + let data = fs::read(get_fixture_path("test_binary_pe.exe")) + .expect("read PE fixture (run `just gen-fixtures`)"); + assert!(PeParser::detect(&data)); + let info = PeParser::new().parse(&data).expect("parse PE"); + let strings = extract_symbol_strings(&info); + assert_symbol_contract(&strings); +} + +#[test] +fn extract_symbol_strings_on_macho_fixture() { + let data = fs::read(get_fixture_path("test_binary_macho")) + .expect("read Mach-O fixture (run `just gen-fixtures`)"); + assert!(MachoParser::detect(&data)); + let info = MachoParser::new().parse(&data).expect("parse Mach-O"); + let strings = extract_symbol_strings(&info); + assert_symbol_contract(&strings); +} From 827f601e1a634665412fbcdf329c20ce3e5d41e5 Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Fri, 19 Jun 2026 13:02:34 -0400 Subject: [PATCH 12/21] feat(extraction): label ASCII content as UTF-8 and make --enc ascii a content filter ASCII extraction now emits Encoding::Utf8 (ASCII is a UTF-8 subset) so a byte-scanned occurrence of a symbol name shares the (text, encoding) dedup key with the classifier's occurrence and merges into one row (KTD7, enables AE4). The Encoding::Ascii variant is retained but no longer produced as output (R15). Since no row is labeled Encoding::Ascii anymore, --enc ascii is redefined as a content filter (new EncodingFilter::AsciiContent): it matches rows whose text is pure-ASCII and whose encoding is not UTF-16, preserving the flag's prior narrow-strings-only meaning (R16). The enabled_encodings ASCII entry still gates whether the ASCII byte-scan runs. Implements U6 (R15, R16). Updates extracted-output encoding assertions and the flow1 JSON snapshot (Ascii -> Utf8). Signed-off-by: UncleSp1d3r --- src/extraction/ascii/extraction.rs | 8 +++--- src/extraction/ascii/tests.rs | 3 ++- src/extraction/tests.rs | 12 +++++---- src/main.rs | 2 +- src/pipeline/config.rs | 7 +++++ src/pipeline/filter.rs | 27 +++++++++++++++++++ tests/integration_extraction.rs | 9 ++++--- ...ntegration_flows_1_5__flow1_top3_json.snap | 6 ++--- tests/test_ascii_extraction.rs | 3 ++- 9 files changed, 58 insertions(+), 19 deletions(-) diff --git a/src/extraction/ascii/extraction.rs b/src/extraction/ascii/extraction.rs index f8d101e..bc4fe72 100644 --- a/src/extraction/ascii/extraction.rs +++ b/src/extraction/ascii/extraction.rs @@ -33,7 +33,7 @@ use super::{AsciiExtractionConfig, is_printable_ascii}; /// /// Vector of FoundString entries with the following metadata: /// - `text`: UTF-8 string from accumulated bytes -/// - `encoding`: `Encoding::Ascii` +/// - `encoding`: `Encoding::Utf8` (ASCII is a UTF-8 subset; see KTD7) /// - `offset`: Start position in the data slice /// - `length`: Byte count /// - `source`: `StringSource::SectionData` @@ -96,7 +96,7 @@ pub fn extract_ascii_strings(data: &[u8], config: &AsciiExtractionConfig) -> Vec let text = String::from_utf8(bytes).expect("ASCII bytes should be valid UTF-8"); strings.push(FoundString::new( text, - Encoding::Ascii, + Encoding::Utf8, start as u64, len as u32, StringSource::SectionData, @@ -121,7 +121,7 @@ pub fn extract_ascii_strings(data: &[u8], config: &AsciiExtractionConfig) -> Vec let text = String::from_utf8(bytes).expect("ASCII bytes should be valid UTF-8"); strings.push(FoundString::new( text, - Encoding::Ascii, + Encoding::Utf8, start as u64, len as u32, StringSource::SectionData, @@ -132,7 +132,7 @@ pub fn extract_ascii_strings(data: &[u8], config: &AsciiExtractionConfig) -> Vec let text = String::from_utf8(bytes).expect("ASCII bytes should be valid UTF-8"); strings.push(FoundString::new( text, - Encoding::Ascii, + Encoding::Utf8, start as u64, len as u32, StringSource::SectionData, diff --git a/src/extraction/ascii/tests.rs b/src/extraction/ascii/tests.rs index 7aebfe7..b1f5d92 100644 --- a/src/extraction/ascii/tests.rs +++ b/src/extraction/ascii/tests.rs @@ -40,7 +40,8 @@ fn test_extract_ascii_strings_basic() { assert_eq!(strings.len(), 3); assert_eq!(strings[0].text, "Hello"); assert_eq!(strings[0].offset, 0); - assert_eq!(strings[0].encoding, Encoding::Ascii); + // ASCII content is emitted as UTF-8 (KTD7); ASCII is a UTF-8 subset. + assert_eq!(strings[0].encoding, Encoding::Utf8); assert_eq!(strings[0].source, StringSource::SectionData); assert_eq!(strings[1].text, "World"); assert_eq!(strings[1].offset, 6); diff --git a/src/extraction/tests.rs b/src/extraction/tests.rs index b796ef0..614e9cf 100644 --- a/src/extraction/tests.rs +++ b/src/extraction/tests.rs @@ -36,7 +36,8 @@ fn test_basic_extractor_extract_from_section() { assert_eq!(strings[0].offset, 0); assert_eq!(strings[0].rva, Some(0x1000)); assert_eq!(strings[0].section, Some(".rodata".to_string())); - assert_eq!(strings[0].encoding, Encoding::Ascii); + // ASCII content is now labeled UTF-8 (KTD7); the variant is no longer emitted. + assert_eq!(strings[0].encoding, Encoding::Utf8); assert_eq!(strings[1].text, "Test"); assert_eq!(strings[1].offset, 12); assert_eq!(strings[1].rva, Some(0x100C)); @@ -162,13 +163,14 @@ fn test_basic_extractor_encoding_filtering() { .extract_from_section(data, §ion, &config) .unwrap(); - // Should only find ASCII strings, not UTF-8 - // Note: "Hello" and "Test" are ASCII, "\u{4e16}\u{754c}" is UTF-8 and should be filtered + // Should only find narrow strings, not the wide UTF-8 content. + // "Hello" and "Test" are ASCII (emitted as UTF-8 per KTD7); the multibyte + // "\u{4e16}\u{754c}" is not scanned because only ASCII scanning is enabled. let ascii_strings: Vec<_> = strings .iter() - .filter(|s| s.encoding == Encoding::Ascii) + .filter(|s| s.encoding == Encoding::Utf8) .collect(); - assert_eq!(ascii_strings.len(), 2, "Should find 2 ASCII strings"); + assert_eq!(ascii_strings.len(), 2, "Should find 2 narrow strings"); assert!(ascii_strings.iter().any(|s| s.text == "Hello")); assert!(ascii_strings.iter().any(|s| s.text == "Test")); // UTF-8 string "\u{4e16}\u{754c}" should be filtered out diff --git a/src/main.rs b/src/main.rs index f9c09f1..e2f9db7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -47,7 +47,7 @@ fn parse_positive_usize(s: &str) -> Result { impl From for EncodingFilter { fn from(enc: CliEncoding) -> Self { match enc { - CliEncoding::Ascii => EncodingFilter::Exact(Encoding::Ascii), + CliEncoding::Ascii => EncodingFilter::AsciiContent, CliEncoding::Utf8 => EncodingFilter::Exact(Encoding::Utf8), CliEncoding::Utf16 => EncodingFilter::Utf16Any, CliEncoding::Utf16Le => EncodingFilter::Exact(Encoding::Utf16Le), diff --git a/src/pipeline/config.rs b/src/pipeline/config.rs index a69ce8f..f3a8666 100644 --- a/src/pipeline/config.rs +++ b/src/pipeline/config.rs @@ -15,6 +15,13 @@ pub enum EncodingFilter { Exact(Encoding), /// Match both `Encoding::Utf16Le` and `Encoding::Utf16Be`. Utf16Any, + /// Match narrow strings whose text content is pure-ASCII. + /// + /// Extraction no longer labels rows `Encoding::Ascii` (ASCII content is + /// emitted as `Encoding::Utf8`), so `--enc ascii` filters by content: + /// the text is pure-ASCII and the stored encoding is not UTF-16. This + /// preserves the flag's pre-existing meaning of "narrow ASCII strings". + AsciiContent, } /// Configuration for post-extraction string filtering. diff --git a/src/pipeline/filter.rs b/src/pipeline/filter.rs index 7828bc8..d891438 100644 --- a/src/pipeline/filter.rs +++ b/src/pipeline/filter.rs @@ -45,6 +45,11 @@ impl FilterEngine { Some(EncodingFilter::Utf16Any) => { s.encoding == Encoding::Utf16Le || s.encoding == Encoding::Utf16Be } + Some(EncodingFilter::AsciiContent) => { + s.text.is_ascii() + && s.encoding != Encoding::Utf16Le + && s.encoding != Encoding::Utf16Be + } }) // 3. include-tags .filter(|s| include_set.is_empty() || s.tags.iter().any(|t| include_set.contains(t))) @@ -120,6 +125,28 @@ mod tests { assert_eq!(result[0].text, "wide"); } + #[test] + fn test_ascii_content_filter_matches_narrow_ascii_text() { + // Narrow ASCII content (now labeled Utf8 per KTD7) passes. + let mut narrow = make_string("CreateFileW", 10, 0); + narrow.encoding = Encoding::Utf8; + // UTF-8 content with non-ASCII characters is excluded. + let mut wide_utf8 = make_string("caf\u{e9}", 10, 10); + wide_utf8.encoding = Encoding::Utf8; + // A UTF-16 row that happens to decode to ASCII is still excluded, + // preserving the flag's pre-existing "narrow strings only" meaning. + let mut utf16_ascii = make_string("hello", 10, 20); + utf16_ascii.encoding = Encoding::Utf16Le; + + let config = FilterConfig::new() + .with_min_length(1) + .with_encoding(EncodingFilter::AsciiContent); + let result = FilterEngine::new().apply(vec![narrow, wide_utf8, utf16_ascii], &config); + + assert_eq!(result.len(), 1); + assert_eq!(result[0].text, "CreateFileW"); + } + #[test] fn test_encoding_utf16any_filter() { let mut s_le = make_string("le", 10, 0); diff --git a/tests/integration_extraction.rs b/tests/integration_extraction.rs index 42170f4..0305ba1 100644 --- a/tests/integration_extraction.rs +++ b/tests/integration_extraction.rs @@ -32,7 +32,8 @@ fn test_basic_extractor_ascii_strings() { assert_eq!(strings.len(), 3); assert_eq!(strings[0].text, "Hello"); - assert_eq!(strings[0].encoding, Encoding::Ascii); + // ASCII content is now labeled UTF-8 (KTD7); the variant is no longer emitted. + assert_eq!(strings[0].encoding, Encoding::Utf8); assert_eq!(strings[0].source, StringSource::SectionData); assert_eq!(strings[1].text, "World"); assert_eq!(strings[2].text, "Test123"); @@ -384,12 +385,12 @@ fn test_basic_extractor_encoding_filtering() { .extract_from_section(data, §ion, &config) .unwrap(); - // Should only find ASCII strings, not UTF-8 + // Should only find narrow strings; ASCII content is labeled UTF-8 (KTD7). assert_eq!(strings.len(), 2); assert_eq!(strings[0].text, "Hello"); - assert_eq!(strings[0].encoding, Encoding::Ascii); + assert_eq!(strings[0].encoding, Encoding::Utf8); assert_eq!(strings[1].text, "Test"); - assert_eq!(strings[1].encoding, Encoding::Ascii); + assert_eq!(strings[1].encoding, Encoding::Utf8); // UTF-8 string "世界" should be filtered out assert!(!strings.iter().any(|s| s.text.contains("世界"))); } diff --git a/tests/snapshots/integration_flows_1_5__flow1_top3_json.snap b/tests/snapshots/integration_flows_1_5__flow1_top3_json.snap index cb91372..ed73fb2 100644 --- a/tests/snapshots/integration_flows_1_5__flow1_top3_json.snap +++ b/tests/snapshots/integration_flows_1_5__flow1_top3_json.snap @@ -2,6 +2,6 @@ source: tests/integration_flows_1_5.rs expression: stdout --- -{"text":"https://github.com/EvilBit-Labs/Stringy","encoding":"Ascii","offset":5008,"rva":16782224,"section":".rodata","length":39,"tags":["Url"],"score":1050,"display_score":100,"source":"SectionData","confidence":1.0} -{"text":"/proc/self/exe","encoding":"Ascii","offset":3747,"rva":16780963,"section":".rodata","length":14,"tags":["filepath"],"score":1022,"display_score":100,"source":"SectionData","confidence":0.865} -{"text":"/usr/lib/debug","encoding":"Ascii","offset":4213,"rva":16781429,"section":".rodata","length":14,"tags":["filepath"],"score":1022,"display_score":100,"source":"SectionData","confidence":0.865} +{"text":"https://github.com/EvilBit-Labs/Stringy","encoding":"Utf8","offset":5008,"rva":16782224,"section":".rodata","length":39,"tags":["Url"],"score":1050,"display_score":100,"source":"SectionData","confidence":1.0} +{"text":"/proc/self/exe","encoding":"Utf8","offset":3747,"rva":16780963,"section":".rodata","length":14,"tags":["filepath"],"score":1022,"display_score":100,"source":"SectionData","confidence":0.865} +{"text":"/usr/lib/debug","encoding":"Utf8","offset":4213,"rva":16781429,"section":".rodata","length":14,"tags":["filepath"],"score":1022,"display_score":100,"source":"SectionData","confidence":0.865} diff --git a/tests/test_ascii_extraction.rs b/tests/test_ascii_extraction.rs index fa11d68..2b5d9a1 100644 --- a/tests/test_ascii_extraction.rs +++ b/tests/test_ascii_extraction.rs @@ -14,7 +14,8 @@ fn test_basic_extraction() { assert_eq!(strings.len(), 3); assert_eq!(strings[0].text, "Hello"); assert_eq!(strings[0].offset, 0); - assert_eq!(strings[0].encoding, Encoding::Ascii); + // ASCII content is emitted as UTF-8 (KTD7); ASCII is a UTF-8 subset. + assert_eq!(strings[0].encoding, Encoding::Utf8); assert_eq!(strings[0].source, StringSource::SectionData); assert_eq!(strings[0].confidence, 1.0); } From 872e6800d6827b65e97bf72e70266918af0dfadc Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Fri, 19 Jun 2026 13:10:28 -0400 Subject: [PATCH 13/21] feat(extraction): route tagged symbols into the extraction path Replace the untagged import/export push in collect_all_strings with a single extract_symbol_strings call inside the include_symbols gate, so symbols reach live output already tagged and the Import/Export and semantic ranking boosts fire. Section-name rows ride the same gate. Scope section-name emission to recognized formats: the unknown/raw fallback's synthetic 'raw-bytes' section is not a real section, so Unknown format skips section-name rows (keeps empty-file output empty and unknown-binary rows all attributed to 'raw-bytes'). Update symbol-property tests to the classifier contract (RVA/section/tags), assert content via SectionData source where section names now also appear, and add an Unknown-format regression test. Implements U4 (R11, R12; AE4). Signed-off-by: UncleSp1d3r --- src/classification/imports.rs | 8 +++++++- src/extraction/basic_extractor.rs | 30 ++++++++---------------------- src/extraction/tests.rs | 28 +++++++++++++++++++--------- tests/classification_symbols.rs | 27 +++++++++++++++++++++++++++ tests/integration_extraction.rs | 13 +++++++------ tests/test_deduplication.rs | 6 ++++-- 6 files changed, 72 insertions(+), 40 deletions(-) diff --git a/src/classification/imports.rs b/src/classification/imports.rs index f33c2da..e04f152 100644 --- a/src/classification/imports.rs +++ b/src/classification/imports.rs @@ -244,6 +244,12 @@ pub fn extract_symbol_strings(container_info: &ContainerInfo) -> Vec = strings + .iter() + .filter(|s| s.source == StringSource::SectionData) + .collect(); + assert_eq!(content.len(), 1); + assert_eq!(content[0].text, "RoDataTest"); } diff --git a/tests/classification_symbols.rs b/tests/classification_symbols.rs index 590a330..de3a04a 100644 --- a/tests/classification_symbols.rs +++ b/tests/classification_symbols.rs @@ -210,6 +210,33 @@ fn section_names_emit_as_section_name_source() { assert!(strings.iter().any(|s| s.text == ".rodata")); } +#[test] +fn unknown_format_skips_section_name_rows() { + // The unknown/raw fallback uses a synthetic "raw-bytes" section; its name + // carries no analytic value and must not be emitted as a standalone row. + let info = stringy::types::ContainerInfo::new( + BinaryFormat::Unknown, + vec![SectionInfo::new( + "raw-bytes".to_string(), + 0, + 10, + SectionType::Other, + 1.0, + )], + vec![], + vec![], + None, + ); + + let strings = extract_symbol_strings(&info); + assert!( + !strings + .iter() + .any(|s| s.source == StringSource::SectionName), + "Unknown format must not emit section-name rows" + ); +} + // --- End-to-end against fixtures (R13) --------------------------------------- fn assert_symbol_contract(strings: &[FoundString]) { diff --git a/tests/integration_extraction.rs b/tests/integration_extraction.rs index 0305ba1..81a1831 100644 --- a/tests/integration_extraction.rs +++ b/tests/integration_extraction.rs @@ -1,7 +1,7 @@ use std::fs; use stringy::container::{ContainerParser, ElfParser, PeParser}; use stringy::extraction::{BasicExtractor, ExtractionConfig, StringExtractor}; -use stringy::types::{Encoding, SectionType, StringSource}; +use stringy::types::{Encoding, SectionType, StringSource, Tag}; fn get_fixture_path(name: &str) -> std::path::PathBuf { std::path::Path::new(env!("CARGO_MANIFEST_DIR")) @@ -424,22 +424,23 @@ fn test_basic_extractor_include_symbols() { // Verify we found some imports/exports assert!(!import_strings.is_empty() || !export_strings.is_empty()); - // Verify import string properties + // Imports are tagged and carry a section (library name or format default). for import_str in &import_strings { assert_eq!(import_str.encoding, Encoding::Utf8); assert_eq!(import_str.offset, 0); - assert_eq!(import_str.rva, None); - assert_eq!(import_str.section, None); + assert!(import_str.section.is_some()); assert!(import_str.length > 0); + assert!(import_str.tags.contains(&Tag::Import)); } - // Verify export string properties + // Exports are tagged, always carry an RVA (address is required), no section. for export_str in &export_strings { assert_eq!(export_str.encoding, Encoding::Utf8); assert_eq!(export_str.offset, 0); - assert_eq!(export_str.rva, None); + assert!(export_str.rva.is_some()); assert_eq!(export_str.section, None); assert!(export_str.length > 0); + assert!(export_str.tags.contains(&Tag::Export)); } } diff --git a/tests/test_deduplication.rs b/tests/test_deduplication.rs index 87d945b..2289e6a 100644 --- a/tests/test_deduplication.rs +++ b/tests/test_deduplication.rs @@ -45,13 +45,15 @@ fn test_deduplication_with_basic_extractor() { assert!(strings.len() >= 3); let hello_count = strings.iter().filter(|s| s.text == "Hello").count(); assert!(hello_count >= 2, "Should have at least 2 'Hello' strings"); + let raw_count = strings.len(); // Apply deduplication let canonical = deduplicate(strings, None); - // Verify deduplication reduced count + // Verify deduplication reduced the count (duplicate content collapses; + // section-name rows are unique and remain). assert!( - canonical.len() < 6, + canonical.len() < raw_count, "Deduplication should reduce string count" ); From 641be6cdd1c2eaf165644501b84e102f1cf5d674 Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Fri, 19 Jun 2026 13:11:20 -0400 Subject: [PATCH 14/21] docs(classification): replace stale once_cell example with std::sync::LazyLock The implementation uses std::sync::LazyLock; update the classification docs prose and example to match. Implements U5 (R14). Signed-off-by: UncleSp1d3r --- docs/src/classification.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/src/classification.md b/docs/src/classification.md index 3f358d9..b4203a3 100644 --- a/docs/src/classification.md +++ b/docs/src/classification.md @@ -74,13 +74,13 @@ Raw String -> Pattern Matching -> Validation -> Tag Assignment ## Pattern Matching Engine -The semantic classifier uses cached regex patterns via `once_cell::sync::Lazy` and applies validation checks to reduce false positives. +The semantic classifier uses cached regex patterns via `std::sync::LazyLock` and applies validation checks to reduce false positives. ```rust -use once_cell::sync::Lazy; use regex::Regex; +use std::sync::LazyLock; -static GUID_REGEX: Lazy = Lazy::new(|| { +static GUID_REGEX: LazyLock = LazyLock::new(|| { Regex::new(r"^\{[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\}$") .expect("Invalid GUID regex") }); @@ -117,7 +117,7 @@ if tags.contains(&Tag::Guid) { ## Performance Notes -- Regexes are compiled once via `once_cell::sync::Lazy` and reused across calls. +- Regexes are compiled once via `std::sync::LazyLock` and reused across calls. - Minimum length checks avoid unnecessary regex work on short inputs. - The classifier is stateless and thread-safe. From 99489d6b31649c699867ffef30f96a8f282d98ab Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Fri, 19 Jun 2026 13:47:39 -0400 Subject: [PATCH 15/21] perf(filter): short-circuit AsciiContent guards before is_ascii scan Evaluate the O(1) UTF-16 encoding guards before the O(n) text.is_ascii() content scan so UTF-16 rows skip the scan. Same boolean result; tightens the config.rs comment to scope the no-longer-Ascii claim to the extraction path. From ce-simplify-code review. Signed-off-by: UncleSp1d3r --- src/pipeline/config.rs | 5 +++-- src/pipeline/filter.rs | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/pipeline/config.rs b/src/pipeline/config.rs index f3a8666..a32f561 100644 --- a/src/pipeline/config.rs +++ b/src/pipeline/config.rs @@ -17,8 +17,9 @@ pub enum EncodingFilter { Utf16Any, /// Match narrow strings whose text content is pure-ASCII. /// - /// Extraction no longer labels rows `Encoding::Ascii` (ASCII content is - /// emitted as `Encoding::Utf8`), so `--enc ascii` filters by content: + /// The ASCII extraction path no longer labels rows `Encoding::Ascii` + /// (ASCII content is emitted as `Encoding::Utf8`), so `--enc ascii` + /// filters by content: /// the text is pure-ASCII and the stored encoding is not UTF-16. This /// preserves the flag's pre-existing meaning of "narrow ASCII strings". AsciiContent, diff --git a/src/pipeline/filter.rs b/src/pipeline/filter.rs index d891438..6a16abc 100644 --- a/src/pipeline/filter.rs +++ b/src/pipeline/filter.rs @@ -46,9 +46,11 @@ impl FilterEngine { s.encoding == Encoding::Utf16Le || s.encoding == Encoding::Utf16Be } Some(EncodingFilter::AsciiContent) => { - s.text.is_ascii() - && s.encoding != Encoding::Utf16Le + // Cheap O(1) encoding guards short-circuit before the O(n) + // is_ascii() scan, so UTF-16 rows skip the content check. + s.encoding != Encoding::Utf16Le && s.encoding != Encoding::Utf16Be + && s.text.is_ascii() } }) // 3. include-tags From 26e4f7900c532e67aa2824723bd6ee45e4e6d74d Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Fri, 19 Jun 2026 13:48:21 -0400 Subject: [PATCH 16/21] fix(classification): defer export demangling to the pipeline; harden symbol rows Code review found that demangling exports inside process_exports (during extraction) ran outside the pipeline's catch_unwind guard -- a cpp_demangle panic on a malicious export name would abort the process -- and also caused raw mode to show demangled text. Demangling already happens once in the pipeline's classify_strings (under catch_unwind, skipped in raw mode), so the classifier emission was redundant. Remove it: ImportClassifier is now stateless and only tags (Export + EntryPoint); the pipeline demangles downstream, preserving AE2 end-to-end. This also lets a mangled export share a dedup key with any byte-scanned occurrence of the same raw name. (Deviates from plan KTD3, which predated the knowledge that the pipeline already demangles.) Also: skip empty section names (PE all-null name fields) so no zero-length row reaches output; document the --enc ascii content-filter contract in the AGENTS.md CLI table and GOTCHAS.md; strengthen the --enc ascii CLI test to assert content-based filtering; re-point the export test to the new contract. From ce-code-review (P1 panic-safety, P2 docs, P3 empty-name guard). Signed-off-by: UncleSp1d3r --- AGENTS.md | 26 ++++++++++---------- GOTCHAS.md | 1 + src/classification/imports.rs | 41 +++++++++++++++++-------------- tests/classification_symbols.rs | 32 +++++++++++++++++------- tests/integration_cli_coverage.rs | 23 +++++++++++++++++ 5 files changed, 83 insertions(+), 40 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 3fe6122..d92616f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -58,19 +58,19 @@ Use idiomatic `clap` derive API patterns. Push validation into clap wherever pos ### Current CLI Flags (main.rs) -| Flag | Short | Type | Notes | -| ------------- | ----- | --------------------- | ----------------------------------------------------------------------- | -| `FILE` | | positional | Input binary (use `-` for stdin) | -| `--json` | `-j` | bool | Conflicts with `--yara` | -| `--yara` | | bool | Conflicts with `--json` | -| `--only-tags` | | `Vec` | Repeatable, `value_parser = Tag::from_str` | -| `--no-tags` | | `Vec` | Repeatable, runtime overlap check with `--only-tags` | -| `--min-len` | `-m` | `Option` | Custom parser enforces >= 1 | -| `--top` | `-t` | `Option` | Custom parser enforces >= 1 | -| `--enc` | | `Option` | ascii, utf8, utf16, utf16le, utf16be | -| `--raw` | | bool | Conflicts with `--only-tags`, `--no-tags`, `--top`, `--debug`, `--yara` | -| `--summary` | | bool | Conflicts with `--json`, `--yara`; runtime TTY check | -| `--debug` | | bool | Conflicts with `--raw` | +| Flag | Short | Type | Notes | +| ------------- | ----- | --------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `FILE` | | positional | Input binary (use `-` for stdin) | +| `--json` | `-j` | bool | Conflicts with `--yara` | +| `--yara` | | bool | Conflicts with `--json` | +| `--only-tags` | | `Vec` | Repeatable, `value_parser = Tag::from_str` | +| `--no-tags` | | `Vec` | Repeatable, runtime overlap check with `--only-tags` | +| `--min-len` | `-m` | `Option` | Custom parser enforces >= 1 | +| `--top` | `-t` | `Option` | Custom parser enforces >= 1 | +| `--enc` | | `Option` | utf8, utf16, utf16le, utf16be filter by stored encoding; `ascii` is content-based (matches pure-ASCII text that is not UTF-16, since extraction labels ASCII content as UTF-8) | +| `--raw` | | bool | Conflicts with `--only-tags`, `--no-tags`, `--top`, `--debug`, `--yara` | +| `--summary` | | bool | Conflicts with `--json`, `--yara`; runtime TTY check | +| `--debug` | | bool | Conflicts with `--raw` | ### Regex Patterns diff --git a/GOTCHAS.md b/GOTCHAS.md index 8d6f971..2b48736 100644 --- a/GOTCHAS.md +++ b/GOTCHAS.md @@ -31,6 +31,7 @@ Changing default values in `ExtractionConfig::default()` requires updating asser - `--no-tags` is the canonical flag name (kebab-case). Previously was `--notags` -- update all references when touching CLI flag names - Short flags: `-j` (json), `-m` (min-len), `-t` (top). Do not add short flags for infrequent flags (--enc, --yara, --raw, --summary, --debug) - `NO_COLOR` env var disables progress spinner. The spinner is also hidden when stderr is not a TTY +- `--enc ascii` is a CONTENT filter (`EncodingFilter::AsciiContent`), not a stored-encoding match. Extraction labels ASCII content as `Encoding::Utf8` (ASCII is a UTF-8 subset), so no row is ever `Encoding::Ascii`; `--enc ascii` matches rows whose `text.is_ascii()` and whose encoding is not UTF-16. `--enc utf8`/`--enc utf16*` still match the stored encoding. JSON output no longer emits `"encoding":"Ascii"` for narrow strings -- it emits `"Utf8"` - Clap derive attributes (`long_help`, `about`, etc.) require string literals -- `const` values and `concat!` with consts do not work. The `cli_help_lists_all_canonical_tags` test in `integration_cli.rs` verifies help text stays in sync with `Tag::from_str()` ## Dependencies diff --git a/src/classification/imports.rs b/src/classification/imports.rs index e04f152..6a57d47 100644 --- a/src/classification/imports.rs +++ b/src/classification/imports.rs @@ -4,11 +4,15 @@ //! tagged, scored [`FoundString`]s. This is the single emission point for //! symbol strings in the extraction path: every import carries [`Tag::Import`] //! (plus a semantic tag when its name matches a known crypto, network, or -//! file-I/O API), every export carries [`Tag::Export`] (plus -//! [`Tag::DemangledSymbol`] when demangling succeeds and [`Tag::EntryPoint`] +//! file-I/O API), every export carries [`Tag::Export`] (plus [`Tag::EntryPoint`] //! for known entry points), and each section name is emitted as a standalone //! row with [`StringSource::SectionName`]. //! +//! Demangling is deliberately left to the pipeline's `classify_strings` (which +//! runs under `catch_unwind` and is skipped in raw mode); the classifier only +//! tags. Exports that are mangled symbols receive [`Tag::DemangledSymbol`] and +//! their demangled text there. +//! //! Symbol strings are emitted with [`Encoding::Utf8`] so that a byte-scanned //! occurrence of the same name (e.g. from a PE `.idata` section) shares the //! `(text, encoding)` deduplication key and merges into the single tagged row. @@ -16,7 +20,6 @@ use std::collections::HashSet; use std::sync::LazyLock; -use crate::classification::SymbolDemangler; use crate::types::{ BinaryFormat, ContainerInfo, Encoding, ExportInfo, FoundString, ImportInfo, SectionInfo, StringSource, Tag, @@ -86,18 +89,18 @@ static ENTRY_POINTS: LazyLock> = LazyLock::new(|| HashSet::from(["main", "_start", "DllMain", "WinMain", "wWinMain"])); /// Classifies import/export symbols and section names into tagged strings. +/// +/// Stateless: tagging is the classifier's job. Demangling is performed once by +/// the pipeline's `classify_strings` (under `catch_unwind`, and skipped in raw +/// mode), so it is intentionally not done here. #[derive(Debug, Default, Clone)] -pub struct ImportClassifier { - demangler: SymbolDemangler, -} +pub struct ImportClassifier; impl ImportClassifier { /// Create a new `ImportClassifier`. #[must_use] pub fn new() -> Self { - Self { - demangler: SymbolDemangler::new(), - } + Self } /// Convert import symbols into tagged `FoundString`s. @@ -149,9 +152,13 @@ impl ImportClassifier { /// Convert export symbols into tagged `FoundString`s. /// /// Each export carries [`Tag::Export`] and an RVA from the always-present - /// `ExportInfo::address`. Exports are demangled (adding - /// [`Tag::DemangledSymbol`] on success), then checked against the known - /// entry-point names (adding [`Tag::EntryPoint`] on a match). + /// `ExportInfo::address`, plus [`Tag::EntryPoint`] when the name is a known + /// entry point. Demangling (and the resulting [`Tag::DemangledSymbol`]) is + /// performed downstream by the pipeline's `classify_strings`, which runs + /// under `catch_unwind` and is skipped in raw mode -- so a panic in a + /// third-party demangler never aborts extraction, and raw mode shows the + /// untouched symbol name. Entry-point names are never mangled, so the check + /// runs correctly on the raw export name. #[must_use] pub fn process_exports(&self, exports: &[ExportInfo]) -> Vec { exports @@ -168,11 +175,6 @@ impl ImportClassifier { .with_tags(vec![Tag::Export]) .with_rva(export.address); - // Demangle first; this may add Tag::DemangledSymbol and replace - // `text` with the demangled form (preserving existing tags). - self.demangler.demangle(&mut found); - - // Entry-point check runs against the resulting text. if ENTRY_POINTS.contains(found.text.as_str()) { found.tags.push(Tag::EntryPoint); } @@ -185,11 +187,14 @@ impl ImportClassifier { /// Convert section names into standalone `FoundString`s. /// /// Each section name is emitted with [`StringSource::SectionName`] so it is - /// distinguishable from byte-scanned section content downstream. + /// distinguishable from byte-scanned section content downstream. Empty + /// section names (e.g. PE sections with an all-null name field) are skipped + /// so no zero-length row reaches output. #[must_use] pub fn process_section_names(&self, sections: &[SectionInfo]) -> Vec { sections .iter() + .filter(|section| !section.name.is_empty()) .map(|section| { let length = section.name.len() as u32; FoundString::new( diff --git a/tests/classification_symbols.rs b/tests/classification_symbols.rs index de3a04a..0ee1edd 100644 --- a/tests/classification_symbols.rs +++ b/tests/classification_symbols.rs @@ -7,7 +7,7 @@ use std::fs; -use stringy::classification::{ImportClassifier, extract_symbol_strings}; +use stringy::classification::{ImportClassifier, SymbolDemangler, extract_symbol_strings}; use stringy::container::{ContainerParser, ElfParser, MachoParser, PeParser}; use stringy::types::{ BinaryFormat, ExportInfo, FoundString, ImportInfo, SectionInfo, SectionType, StringSource, Tag, @@ -141,18 +141,29 @@ fn every_export_carries_export_tag_source_and_rva() { } #[test] -fn mangled_export_is_demangled_and_tagged() { +fn mangled_export_emitted_raw_for_pipeline_demangling() { + // The classifier tags exports but does NOT demangle. Demangling is the + // pipeline's job (classify_strings: under catch_unwind, skipped in raw + // mode), so a third-party demangler panic never aborts extraction and raw + // mode shows the untouched symbol name. let classifier = ImportClassifier::new(); let exports = vec![ExportInfo::new("_ZN3foo3barEv".to_string(), 0x3000)]; let strings = classifier.process_exports(&exports); - let demangled = &strings[0]; - - assert!(demangled.tags.contains(&Tag::Export)); - assert!(demangled.tags.contains(&Tag::DemangledSymbol)); - assert_eq!(demangled.original_text.as_deref(), Some("_ZN3foo3barEv")); - assert!(demangled.text.contains("foo")); - assert!(demangled.text.contains("bar")); + let export = &strings[0]; + + assert!(export.tags.contains(&Tag::Export)); + assert!(!export.tags.contains(&Tag::DemangledSymbol)); + assert_eq!(export.text, "_ZN3foo3barEv"); + assert_eq!(export.original_text, None); + + // The emitted string is in a demangleable state: the pipeline's + // SymbolDemangler produces Export + DemangledSymbol + demangled text (AE2). + let mut downstream = strings[0].clone(); + SymbolDemangler::new().demangle(&mut downstream); + assert!(downstream.tags.contains(&Tag::Export)); + assert!(downstream.tags.contains(&Tag::DemangledSymbol)); + assert!(downstream.text.contains("foo") && downstream.text.contains("bar")); } #[test] @@ -197,6 +208,8 @@ fn section_names_emit_as_section_name_source() { SectionType::StringData, 1.0, ), + // Empty-named section (e.g. PE all-null name) must be skipped. + SectionInfo::new(String::new(), 200, 10, SectionType::Other, 1.0), ]; let strings = classifier.process_section_names(§ions); @@ -205,6 +218,7 @@ fn section_names_emit_as_section_name_source() { for s in &strings { assert_eq!(s.source, StringSource::SectionName); assert_eq!(s.confidence, 1.0); + assert!(!s.text.is_empty()); } assert!(strings.iter().any(|s| s.text == ".text")); assert!(strings.iter().any(|s| s.text == ".rodata")); diff --git a/tests/integration_cli_coverage.rs b/tests/integration_cli_coverage.rs index 08376c9..bf7caa1 100644 --- a/tests/integration_cli_coverage.rs +++ b/tests/integration_cli_coverage.rs @@ -61,6 +61,29 @@ fn encoding_filter_ascii() { .success(); } +#[test] +fn encoding_filter_ascii_is_content_based() { + // --enc ascii is a content filter (R16): every emitted row's text must be + // pure-ASCII, regardless of stored encoding. Extraction labels ASCII + // content as Utf8, so this asserts content, not the encoding field. + let assert = stringy() + .args(["tests/fixtures/test_binary_elf", "--enc", "ascii", "--json"]) + .assert() + .success(); + let stdout = String::from_utf8(assert.get_output().stdout.clone()).unwrap(); + let mut rows = 0; + for line in stdout.lines().filter(|l| !l.is_empty()) { + let v: serde_json::Value = serde_json::from_str(line).expect("valid JSON line"); + let text = v["text"].as_str().expect("text field"); + assert!( + text.is_ascii(), + "--enc ascii must only emit pure-ASCII text, got: {text:?}" + ); + rows += 1; + } + assert!(rows > 0, "ELF fixture should yield some ASCII rows"); +} + #[test] fn encoding_filter_invalid() { stringy() From f59eece28b91ce557f42debcc4b565232a747c95 Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Fri, 19 Jun 2026 13:54:47 -0400 Subject: [PATCH 17/21] fix(ci): restore zig provisioning via .zig-version Commit 2d7d25a removed zig from idiomatic_version_file_enable_tools without adding it to [tools], so mise stopped reading .zig-version (0.15.2) and CI's gen-fixtures step failed with 'zig couldn't exec process'. Re-add zig to the idiomatic list so mise auto-installs the pinned version (exec_auto_install is on); using the existing .zig-version pin keeps fixture layouts and insta snapshots reproducible. Pre-existing break on main, unrelated to the feature. Signed-off-by: UncleSp1d3r --- mise.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mise.toml b/mise.toml index d633a23..5e7885b 100644 --- a/mise.toml +++ b/mise.toml @@ -43,7 +43,7 @@ uv = "latest" # Many of these settings are defaults, but we are explicit in case they change in the future and to make it clear to users what is enabled. [settings] activate_aggressive = true -idiomatic_version_file_enable_tools = [ "python", "rust", "bun" ] +idiomatic_version_file_enable_tools = [ "python", "rust", "bun", "zig" ] env_cache = true exec_auto_install = true github_attestations = true From cd614a288db8f1505f5967212bd7f337e3184bf6 Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Fri, 19 Jun 2026 14:17:53 -0400 Subject: [PATCH 18/21] feat(tools): add zig tool with latest version and update idiomatic version file settings Signed-off-by: UncleSp1d3r --- mise.lock | 40 ++++++++++++++++++++++++++++++++++++++++ mise.toml | 3 ++- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/mise.lock b/mise.lock index 9b32766..aaee89f 100644 --- a/mise.lock +++ b/mise.lock @@ -707,3 +707,43 @@ provenance = "github-attestations" checksum = "sha256:ace861f360c6de2babedc1607d0f454b6b09a820dbc8182dc15af927e4df9589" url = "https://github.com/astral-sh/uv/releases/download/0.11.21/uv-x86_64-pc-windows-msvc.zip" provenance = "github-attestations" + +[[tools.zig]] +version = "0.16.0" +backend = "core:zig" + +[tools.zig."platforms.linux-x64"] +checksum = "sha256:70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d00" +url = "https://ziglang.org/download/0.16.0/zig-x86_64-linux-0.16.0.tar.xz" + +[tools.zig."platforms.linux-x64-baseline"] +checksum = "sha256:70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d00" +url = "https://ziglang.org/download/0.16.0/zig-x86_64-linux-0.16.0.tar.xz" + +[tools.zig."platforms.linux-x64-musl"] +checksum = "sha256:70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d00" +url = "https://ziglang.org/download/0.16.0/zig-x86_64-linux-0.16.0.tar.xz" + +[tools.zig."platforms.linux-x64-musl-baseline"] +checksum = "sha256:70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d00" +url = "https://ziglang.org/download/0.16.0/zig-x86_64-linux-0.16.0.tar.xz" + +[tools.zig."platforms.macos-arm64"] +checksum = "sha256:b23d70deaa879b5c2d486ed3316f7eaa53e84acf6fc9cc747de152450d401489" +url = "https://ziglang.org/download/0.16.0/zig-aarch64-macos-0.16.0.tar.xz" + +[tools.zig."platforms.macos-x64"] +checksum = "sha256:0387557ed1877bc6a2e1802c8391953baddba76081876301c522f52977b52ba7" +url = "https://ziglang.org/download/0.16.0/zig-x86_64-macos-0.16.0.tar.xz" + +[tools.zig."platforms.macos-x64-baseline"] +checksum = "sha256:0387557ed1877bc6a2e1802c8391953baddba76081876301c522f52977b52ba7" +url = "https://ziglang.org/download/0.16.0/zig-x86_64-macos-0.16.0.tar.xz" + +[tools.zig."platforms.windows-x64"] +checksum = "sha256:68659eb5f1e4eb1437a722f1dd889c5a322c9954607f5edcf337bc3684a75a7e" +url = "https://ziglang.org/download/0.16.0/zig-x86_64-windows-0.16.0.zip" + +[tools.zig."platforms.windows-x64-baseline"] +checksum = "sha256:68659eb5f1e4eb1437a722f1dd889c5a322c9954607f5edcf337bc3684a75a7e" +url = "https://ziglang.org/download/0.16.0/zig-x86_64-windows-0.16.0.zip" diff --git a/mise.toml b/mise.toml index 5e7885b..c5256dc 100644 --- a/mise.toml +++ b/mise.toml @@ -39,11 +39,12 @@ scorecard = "5.5.0" "bun" = "latest" shellcheck = "0.11.0" uv = "latest" +zig = "latest" # Many of these settings are defaults, but we are explicit in case they change in the future and to make it clear to users what is enabled. [settings] activate_aggressive = true -idiomatic_version_file_enable_tools = [ "python", "rust", "bun", "zig" ] +idiomatic_version_file_enable_tools = [ "python", "rust", "bun" ] env_cache = true exec_auto_install = true github_attestations = true From a832026d872af392779cf95d4a62755d012e4114 Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Fri, 19 Jun 2026 14:41:55 -0400 Subject: [PATCH 19/21] fix(ci): pin zig to 0.15.2 to match fixture snapshots zig = latest pulled 0.16.0, which generates a materially different ELF fixture (81 imports vs 30) than the committed insta snapshots, which were calibrated for the .zig-version pin (0.15.2). Pin zig to 0.15.2 in [tools] so CI-generated fixtures match the snapshots and the version is reproducible across zig releases. Resolves the elf_symbol_extraction snapshot failure. Signed-off-by: UncleSp1d3r --- mise.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mise.toml b/mise.toml index c5256dc..fe5fee0 100644 --- a/mise.toml +++ b/mise.toml @@ -39,7 +39,7 @@ scorecard = "5.5.0" "bun" = "latest" shellcheck = "0.11.0" uv = "latest" -zig = "latest" +zig = "0.15.2" # Many of these settings are defaults, but we are explicit in case they change in the future and to make it clear to users what is enabled. [settings] From 3d08bb00607f733ec0920f2b9b159a0e9691e1d6 Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Fri, 19 Jun 2026 14:46:30 -0400 Subject: [PATCH 20/21] fix(ci): lock zig 0.15.2 across all platforms in mise.lock CI runs 'mise install --locked', which requires every pinned tool version in mise.lock. The lockfile still held zig 0.16.0 (from the earlier zig=latest), so pinning mise.toml to 0.15.2 failed to resolve under --locked. Relock zig to 0.15.2 for all 9 target platforms and prune the stale 0.16.0 entry so the pin installs cleanly on every CI runner. Signed-off-by: UncleSp1d3r --- mise.lock | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/mise.lock b/mise.lock index aaee89f..f869795 100644 --- a/mise.lock +++ b/mise.lock @@ -709,41 +709,41 @@ url = "https://github.com/astral-sh/uv/releases/download/0.11.21/uv-x86_64-pc-wi provenance = "github-attestations" [[tools.zig]] -version = "0.16.0" +version = "0.15.2" backend = "core:zig" [tools.zig."platforms.linux-x64"] -checksum = "sha256:70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d00" -url = "https://ziglang.org/download/0.16.0/zig-x86_64-linux-0.16.0.tar.xz" +checksum = "sha256:02aa270f183da276e5b5920b1dac44a63f1a49e55050ebde3aecc9eb82f93239" +url = "https://ziglang.org/download/0.15.2/zig-x86_64-linux-0.15.2.tar.xz" [tools.zig."platforms.linux-x64-baseline"] -checksum = "sha256:70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d00" -url = "https://ziglang.org/download/0.16.0/zig-x86_64-linux-0.16.0.tar.xz" +checksum = "sha256:02aa270f183da276e5b5920b1dac44a63f1a49e55050ebde3aecc9eb82f93239" +url = "https://ziglang.org/download/0.15.2/zig-x86_64-linux-0.15.2.tar.xz" [tools.zig."platforms.linux-x64-musl"] -checksum = "sha256:70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d00" -url = "https://ziglang.org/download/0.16.0/zig-x86_64-linux-0.16.0.tar.xz" +checksum = "sha256:02aa270f183da276e5b5920b1dac44a63f1a49e55050ebde3aecc9eb82f93239" +url = "https://ziglang.org/download/0.15.2/zig-x86_64-linux-0.15.2.tar.xz" [tools.zig."platforms.linux-x64-musl-baseline"] -checksum = "sha256:70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d00" -url = "https://ziglang.org/download/0.16.0/zig-x86_64-linux-0.16.0.tar.xz" +checksum = "sha256:02aa270f183da276e5b5920b1dac44a63f1a49e55050ebde3aecc9eb82f93239" +url = "https://ziglang.org/download/0.15.2/zig-x86_64-linux-0.15.2.tar.xz" [tools.zig."platforms.macos-arm64"] -checksum = "sha256:b23d70deaa879b5c2d486ed3316f7eaa53e84acf6fc9cc747de152450d401489" -url = "https://ziglang.org/download/0.16.0/zig-aarch64-macos-0.16.0.tar.xz" +checksum = "sha256:3cc2bab367e185cdfb27501c4b30b1b0653c28d9f73df8dc91488e66ece5fa6b" +url = "https://ziglang.org/download/0.15.2/zig-aarch64-macos-0.15.2.tar.xz" [tools.zig."platforms.macos-x64"] -checksum = "sha256:0387557ed1877bc6a2e1802c8391953baddba76081876301c522f52977b52ba7" -url = "https://ziglang.org/download/0.16.0/zig-x86_64-macos-0.16.0.tar.xz" +checksum = "sha256:375b6909fc1495d16fc2c7db9538f707456bfc3373b14ee83fdd3e22b3d43f7f" +url = "https://ziglang.org/download/0.15.2/zig-x86_64-macos-0.15.2.tar.xz" [tools.zig."platforms.macos-x64-baseline"] -checksum = "sha256:0387557ed1877bc6a2e1802c8391953baddba76081876301c522f52977b52ba7" -url = "https://ziglang.org/download/0.16.0/zig-x86_64-macos-0.16.0.tar.xz" +checksum = "sha256:375b6909fc1495d16fc2c7db9538f707456bfc3373b14ee83fdd3e22b3d43f7f" +url = "https://ziglang.org/download/0.15.2/zig-x86_64-macos-0.15.2.tar.xz" [tools.zig."platforms.windows-x64"] -checksum = "sha256:68659eb5f1e4eb1437a722f1dd889c5a322c9954607f5edcf337bc3684a75a7e" -url = "https://ziglang.org/download/0.16.0/zig-x86_64-windows-0.16.0.zip" +checksum = "sha256:3a0ed1e8799a2f8ce2a6e6290a9ff22e6906f8227865911fb7ddedc3cc14cb0c" +url = "https://ziglang.org/download/0.15.2/zig-x86_64-windows-0.15.2.zip" [tools.zig."platforms.windows-x64-baseline"] -checksum = "sha256:68659eb5f1e4eb1437a722f1dd889c5a322c9954607f5edcf337bc3684a75a7e" -url = "https://ziglang.org/download/0.16.0/zig-x86_64-windows-0.16.0.zip" +checksum = "sha256:3a0ed1e8799a2f8ce2a6e6290a9ff22e6906f8227865911fb7ddedc3cc14cb0c" +url = "https://ziglang.org/download/0.15.2/zig-x86_64-windows-0.15.2.zip" From ed016989bd25b9699d5bbb34e83ddcf4249b6f98 Mon Sep 17 00:00:00 2001 From: UncleSp1d3r Date: Fri, 19 Jun 2026 15:47:01 -0400 Subject: [PATCH 21/21] docs(review): address PR #174 review feedback - Mark StringSource #[non_exhaustive] (matches Tag) so future variants are not semver-breaking for external exhaustive matches (Copilot) - Clarify --enc help text: ascii is narrow-ASCII content filtering (CodeRabbit) - Soften imports.rs module doc: demangle adds DemangledSymbol on success; names that fail to demangle are left unchanged (Copilot) - Update origin requirements doc R3/R6/R8 to match the implementation: Mach-O __LINKEDIT default, demangle-in-pipeline note, StringSource::SectionName (Copilot) Signed-off-by: UncleSp1d3r --- ...026-06-19-symbol-classification-pipeline-requirements.md | 6 +++--- src/classification/imports.rs | 3 ++- src/main.rs | 2 +- src/types/mod.rs | 1 + 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/brainstorms/2026-06-19-symbol-classification-pipeline-requirements.md b/docs/brainstorms/2026-06-19-symbol-classification-pipeline-requirements.md index cf856e3..2fca237 100644 --- a/docs/brainstorms/2026-06-19-symbol-classification-pipeline-requirements.md +++ b/docs/brainstorms/2026-06-19-symbol-classification-pipeline-requirements.md @@ -37,12 +37,12 @@ The pieces to fix this already exist in isolation - the demangler, the tag enum - R1. Import names emit as `FoundString`s with `source: StringSource::ImportName`, always carrying `Tag::Import`, at `confidence: 1.0`. - R2. Imports whose name matches the crypto, network, or file-I/O API sets additionally carry `Tag::Crypto`, `Tag::Network`, or `Tag::FileIO`; imports that match none carry only `Tag::Import`. -- R3. An import's originating library populates the `section` field when present; when absent, a format-appropriate default is used (`.idata` for PE, `.dynsym` for ELF). +- R3. An import's originating library populates the `section` field when present; when absent, a format-appropriate default is used (`.idata` for PE, `.dynsym` for ELF, `__LINKEDIT` for Mach-O -- Mach-O imports never carry a library, so they always use the default). - R4. An import's RVA populates from `ImportInfo::address` when present (it is optional). - R5. Export names emit as `FoundString`s with `source: StringSource::ExportName`, always carrying `Tag::Export`, with RVA from `ExportInfo::address` (always available). -- R6. Exports are run through the existing `SymbolDemangler`; successfully demangled exports additionally carry `Tag::DemangledSymbol`. +- R6. Exports are run through the existing `SymbolDemangler`; successfully demangled exports additionally carry `Tag::DemangledSymbol`. (Implementation note: demangling runs in the pipeline's `classify_strings` -- under `catch_unwind` and skipped in raw mode -- not in the classifier, so a third-party demangler panic never aborts extraction.) - R7. Exports whose name is a known entry point (`main`, `_start`, `DllMain`, `WinMain`, `wWinMain`) additionally carry `Tag::EntryPoint`. -- R8. Section names emit as standalone `FoundString`s with `source: StringSource::SectionData` at `confidence: 1.0`, ranked alongside other output. +- R8. Section names emit as standalone `FoundString`s with `source: StringSource::SectionName` (a dedicated variant, distinct from `SectionData`, so section-name rows are distinguishable from byte-scanned section content) at `confidence: 1.0`, ranked alongside other output. **Type system and ranking** diff --git a/src/classification/imports.rs b/src/classification/imports.rs index 6a57d47..b7a8caa 100644 --- a/src/classification/imports.rs +++ b/src/classification/imports.rs @@ -11,7 +11,8 @@ //! Demangling is deliberately left to the pipeline's `classify_strings` (which //! runs under `catch_unwind` and is skipped in raw mode); the classifier only //! tags. Exports that are mangled symbols receive [`Tag::DemangledSymbol`] and -//! their demangled text there. +//! their demangled text there when demangling succeeds; names that fail to +//! demangle are left unchanged. //! //! Symbol strings are emitted with [`Encoding::Utf8`] so that a byte-scanned //! occurrence of the same name (e.g. from a PE `.idata` section) shares the diff --git a/src/main.rs b/src/main.rs index e2f9db7..4ec4ba4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -133,7 +133,7 @@ struct Cli { #[arg(short = 't', long, value_name = "N", value_parser = parse_positive_usize)] top: Option, - /// Filter by encoding [possible values: ascii, utf8, utf16, utf16le, utf16be] + /// Filter by encoding/content [possible values: ascii (narrow ASCII content), utf8, utf16, utf16le, utf16be] #[arg(long, value_enum, value_name = "ENCODING")] enc: Option, diff --git a/src/types/mod.rs b/src/types/mod.rs index af2321c..421ec2e 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -120,6 +120,7 @@ pub enum SectionType { } /// Source of a string within the binary +#[non_exhaustive] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum StringSource { /// String found in section data