diff --git a/.claude/settings.json b/.claude/settings.json index c72c6b7..4ee3ac9 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -1,3 +1,15 @@ { - "enabledPlugins": {} + "enabledPlugins": { + "compound-engineering@compound-engineering-plugin": true, + "ecc@ecc": true, + "rust-analyzer-lsp@claude-plugins-official": true, + "rust-cli-developer@geoffjay-claude-plugins": true, + "systems-programming@claude-code-workflows": true, + "agents-quality-security@buildwithclaude": true, + "security-guidance@claude-plugins-official": true, + "pr-review-toolkit@claude-plugins-official": true, + "comprehensive-review@claude-code-workflows": true, + "superpowers@claude-plugins-official": true, + "github@claude-plugins-official": true + } } diff --git a/.mdformat.toml b/.mdformat.toml index 22c2e4e..dcda597 100644 --- a/.mdformat.toml +++ b/.mdformat.toml @@ -1,19 +1,4 @@ # mdformat configuration - consistent with Prettier and markdownlint -exclude = [ - "venv/**", - "**/node_modules/**", - "**/*.txt", - "**/*.mdc", - "**/*.tpl.md", - "**/CHANGELOG.md", - "target/**", - "megalinter-reports/**", - "**/*.result", - "**/*.testfile", - "**/SKILL.md", # AI stuff - ".claude/**/*", # AI stuff - ".tessl/**/*", # AI stuff -] validate = true number = true wrap = "no" diff --git a/AGENTS.md b/AGENTS.md index 3fe6122..d92616f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -58,19 +58,19 @@ Use idiomatic `clap` derive API patterns. Push validation into clap wherever pos ### Current CLI Flags (main.rs) -| Flag | Short | Type | Notes | -| ------------- | ----- | --------------------- | ----------------------------------------------------------------------- | -| `FILE` | | positional | Input binary (use `-` for stdin) | -| `--json` | `-j` | bool | Conflicts with `--yara` | -| `--yara` | | bool | Conflicts with `--json` | -| `--only-tags` | | `Vec` | Repeatable, `value_parser = Tag::from_str` | -| `--no-tags` | | `Vec` | Repeatable, runtime overlap check with `--only-tags` | -| `--min-len` | `-m` | `Option` | Custom parser enforces >= 1 | -| `--top` | `-t` | `Option` | Custom parser enforces >= 1 | -| `--enc` | | `Option` | ascii, utf8, utf16, utf16le, utf16be | -| `--raw` | | bool | Conflicts with `--only-tags`, `--no-tags`, `--top`, `--debug`, `--yara` | -| `--summary` | | bool | Conflicts with `--json`, `--yara`; runtime TTY check | -| `--debug` | | bool | Conflicts with `--raw` | +| Flag | Short | Type | Notes | +| ------------- | ----- | --------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `FILE` | | positional | Input binary (use `-` for stdin) | +| `--json` | `-j` | bool | Conflicts with `--yara` | +| `--yara` | | bool | Conflicts with `--json` | +| `--only-tags` | | `Vec` | Repeatable, `value_parser = Tag::from_str` | +| `--no-tags` | | `Vec` | Repeatable, runtime overlap check with `--only-tags` | +| `--min-len` | `-m` | `Option` | Custom parser enforces >= 1 | +| `--top` | `-t` | `Option` | Custom parser enforces >= 1 | +| `--enc` | | `Option` | utf8, utf16, utf16le, utf16be filter by stored encoding; `ascii` is content-based (matches pure-ASCII text that is not UTF-16, since extraction labels ASCII content as UTF-8) | +| `--raw` | | bool | Conflicts with `--only-tags`, `--no-tags`, `--top`, `--debug`, `--yara` | +| `--summary` | | bool | Conflicts with `--json`, `--yara`; runtime TTY check | +| `--debug` | | bool | Conflicts with `--raw` | ### Regex Patterns diff --git a/AI_POLICY.md b/AI_POLICY.md index e868b08..25e6927 100644 --- a/AI_POLICY.md +++ b/AI_POLICY.md @@ -6,8 +6,7 @@ Think of AI assistance like spellcheck. It catches typos, suggests corrections, ## The Rule -**You own every line you submit.** You must be able to explain what -it does and how it interacts with the rest of the system without asking your AI to explain it back to you. +**You own every line you submit.** You must be able to explain what it does and how it interacts with the rest of the system without asking your AI to explain it back to you. Everything else follows from that. diff --git a/CHANGELOG.md b/CHANGELOG.md index 88873c2..95c8422 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,12 +2,12 @@ All notable changes to this project will be documented in this file. -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), -and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] ### Added + - Pipeline orchestrator (`Pipeline::run`) with configurable stages: parse, extract, classify, rank, normalize, filter, output - `PipelineConfig`, `FilterConfig`, and `EncodingFilter` for pipeline configuration - `FilterEngine` for post-extraction string filtering (min-len, encoding, tags, top-N) @@ -39,6 +39,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Debug-build env var injection for e2e warning-path testing ### Changed + - Container parsers refactored from single files to module directories (elf/, pe/, macho/) - Repository renamed from StringyMcStringFace to Stringy - Improved YARA formatter code quality and test coverage @@ -46,6 +47,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Replaced `once_cell::sync::Lazy` with `std::sync::LazyLock` (stabilized in Rust 1.80) ### Fixed + - Rustdoc warning for IPv6 address example in documentation - ELF section name fallback no longer eagerly allocates on every iteration - Pipeline section lookups use HashMap for O(1) access instead of O(n) linear scans @@ -55,6 +57,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Demangler text clone now conditional (only for mangled-looking symbols) ### Dependencies + - Added `mmap-guard` for safe memory-mapped file I/O - Added `indicatif` for progress bars and spinners - Added `tempfile` for stdin-to-pipeline bridging @@ -72,6 +75,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Initial release with core functionality: ### Added + - ELF, PE, and Mach-O binary format detection and parsing - ASCII and UTF-8 string extraction from binary sections - Section-aware extraction with weight-based prioritization diff --git a/GOTCHAS.md b/GOTCHAS.md index 8d6f971..2b48736 100644 --- a/GOTCHAS.md +++ b/GOTCHAS.md @@ -31,6 +31,7 @@ Changing default values in `ExtractionConfig::default()` requires updating asser - `--no-tags` is the canonical flag name (kebab-case). Previously was `--notags` -- update all references when touching CLI flag names - Short flags: `-j` (json), `-m` (min-len), `-t` (top). Do not add short flags for infrequent flags (--enc, --yara, --raw, --summary, --debug) - `NO_COLOR` env var disables progress spinner. The spinner is also hidden when stderr is not a TTY +- `--enc ascii` is a CONTENT filter (`EncodingFilter::AsciiContent`), not a stored-encoding match. Extraction labels ASCII content as `Encoding::Utf8` (ASCII is a UTF-8 subset), so no row is ever `Encoding::Ascii`; `--enc ascii` matches rows whose `text.is_ascii()` and whose encoding is not UTF-16. `--enc utf8`/`--enc utf16*` still match the stored encoding. JSON output no longer emits `"encoding":"Ascii"` for narrow strings -- it emits `"Utf8"` - Clap derive attributes (`long_help`, `about`, etc.) require string literals -- `const` values and `concat!` with consts do not work. The `cli_help_lists_all_canonical_tags` test in `integration_cli.rs` verifies help text stays in sync with `Tag::from_str()` ## Dependencies diff --git a/docs/brainstorms/2026-06-19-symbol-classification-pipeline-requirements.md b/docs/brainstorms/2026-06-19-symbol-classification-pipeline-requirements.md new file mode 100644 index 0000000..2fca237 --- /dev/null +++ b/docs/brainstorms/2026-06-19-symbol-classification-pipeline-requirements.md @@ -0,0 +1,98 @@ +--- +date: 2026-06-19 +topic: symbol-classification-pipeline +--- + +# Import/Export Symbol Classification Pipeline + +## Summary + +Add an `ImportClassifier` that converts parsed import/export symbols and section names from `ContainerInfo` into tagged, scored `FoundString` objects, and route it into the live extraction path so the already-defined Import/Export ranking boosts finally fire. Four new semantic tags (crypto, network, file-I/O, entry-point) are recognized via static API-name sets and the existing symbol demangler. + +--- + +## Problem Frame + +`stringy` already emits import and export names in its default output: when `include_symbols` is on (the default), `src/extraction/basic_extractor.rs:67-89` pushes each `ImportInfo`/`ExportInfo` name as a `FoundString`. But those strings are emitted bare: no `Tag::Import`/`Tag::Export`, no RVA, no originating library, and no semantic classification. The `RankingEngine` defines +60 boosts for `Tag::Import` and `Tag::Export` (`src/classification/ranking.rs:131-132`), but nothing ever applies those tags, so the boosts are dead code and symbol strings rank no higher than incidental noise. A binary analyst scanning a packed sample gets `CreateFileW` buried next to junk instead of surfaced as a tagged file-I/O import. + +The pieces to fix this already exist in isolation - the demangler, the tag enum scaffolding, the ranking boosts, the parsed symbol metadata. What is missing is the classifier that ties symbol metadata to tags, plus the wiring that makes the tagged strings the ones that actually reach output. + +--- + +## Key Decisions + +- **Replace the existing untagged symbol emission rather than add a parallel path.** `basic_extractor` already emits import/export names untagged and default-on. Adding a second, tagged emission path would produce two `FoundString`s per symbol; deduplication is by text with a nondeterministic HashMap winner, so dedup could keep the untagged copy and silently drop the tagged one, no-opping the feature. The classifier becomes the single emission point at that call site instead. The issue's `ImportClassifier` / `extract_symbol_strings` design is unchanged; only where it is called moves. + +- **Emit section names as ranked output rows.** Section names (`.text`, `.idata`) are not emitted as standalone strings today (only attached as the `section` field on content strings), so this is net-new with no duplication risk. Generic short names rely on the existing noise/min-length filter to stay out of the way. + +- **Reuse `SymbolDemangler` as-is.** It already handles both Rust and C++ (`src/classification/symbols.rs`); no new demangling logic is introduced. + +- **New tags ride the existing tag machinery.** No new CLI flags. Once the four variants are wired through `FromStr`, `Display`, serde, and the help text, `--only-tags`/`--no-tags` filter them automatically. + +--- + +## Requirements + +**Symbol classification** + +- R1. Import names emit as `FoundString`s with `source: StringSource::ImportName`, always carrying `Tag::Import`, at `confidence: 1.0`. +- R2. Imports whose name matches the crypto, network, or file-I/O API sets additionally carry `Tag::Crypto`, `Tag::Network`, or `Tag::FileIO`; imports that match none carry only `Tag::Import`. +- R3. An import's originating library populates the `section` field when present; when absent, a format-appropriate default is used (`.idata` for PE, `.dynsym` for ELF, `__LINKEDIT` for Mach-O -- Mach-O imports never carry a library, so they always use the default). +- R4. An import's RVA populates from `ImportInfo::address` when present (it is optional). +- R5. Export names emit as `FoundString`s with `source: StringSource::ExportName`, always carrying `Tag::Export`, with RVA from `ExportInfo::address` (always available). +- R6. Exports are run through the existing `SymbolDemangler`; successfully demangled exports additionally carry `Tag::DemangledSymbol`. (Implementation note: demangling runs in the pipeline's `classify_strings` -- under `catch_unwind` and skipped in raw mode -- not in the classifier, so a third-party demangler panic never aborts extraction.) +- R7. Exports whose name is a known entry point (`main`, `_start`, `DllMain`, `WinMain`, `wWinMain`) additionally carry `Tag::EntryPoint`. +- R8. Section names emit as standalone `FoundString`s with `source: StringSource::SectionName` (a dedicated variant, distinct from `SectionData`, so section-name rows are distinguishable from byte-scanned section content) at `confidence: 1.0`, ranked alongside other output. + +**Type system and ranking** + +- R9. Add `Tag` variants `Crypto`, `Network`, `FileIO`, `EntryPoint` with serde renames (`crypto`, `network`, `fileio`, `entry-point`), wired through `Tag::from_str`, `Display`, and every exhaustive match site, including the CLI help text that the `cli_help_lists_all_canonical_tags` test guards. +- R10. `RankingConfig` gains tag boosts: `Crypto` +50, `Network` +45, `FileIO` +35, `EntryPoint` +40. + +**Pipeline integration** + +- R11. The tagged classifier replaces the untagged import/export emission in `basic_extractor` so each symbol is emitted exactly once, already tagged; section-name emission is added to the same path. +- R12. Tagged symbol strings flow through the remaining pipeline stages (classification, ranking) with their classifier-assigned tags preserved, so the Import/Export and new semantic boosts apply to live output. + +**Tests and docs** + +- R13. `tests/classification_symbols.rs` covers import classification, export classification, API-set detection (at least 3 representative symbols per category plus a negative case), section-name extraction, and end-to-end `extract_symbol_strings` output against the existing ELF, PE, and Mach-O fixtures. +- R14. `docs/src/classification.md` is updated to reference `std::sync::LazyLock` instead of the stale `once_cell::sync::Lazy` pattern example. + +--- + +## Acceptance Examples + +- AE1. **Covers R2.** Import `CreateFileW` yields tags `Import` + `FileIO`; `connect` yields `Import` + `Network`; `EVP_EncryptInit` yields `Import` + `Crypto`; `printf` yields `Import` only. +- AE2. **Covers R6, R7.** A C++-mangled export demangles and yields `Export` + `DemangledSymbol`; export `main` yields `Export` + `EntryPoint`; export `DllMain` yields `Export` + `EntryPoint`. +- AE3. **Covers R3.** A PE import from `kernel32.dll` sets `section` to `kernel32.dll`; an ELF import with no library sets `section` to `.dynsym`. +- AE4. **Covers R11, R12.** Running `stringy` on a PE fixture (with the default `include_symbols`) shows `CreateFileW` exactly once, tagged `import` + `fileio` + - not a duplicate untagged row, and not dropped by dedup. + +--- + +## Scope Boundaries + +- New CLI flags to filter the four new tags - out of scope; the existing `--only-tags`/`--no-tags` machinery covers them. +- Redesigning deduplication or string-ordering behavior - out of scope. +- Tuning exact membership of the crypto/network/file-I/O API sets beyond the issue's seed lists - deferred to planning and implementation. + +--- + +## Dependencies / Assumptions + +- `ExportInfo::address` is a required `u64` while `ImportInfo::address` is `Option`; export RVA is always available, import RVA is conditional (R4, R5 reflect this). +- This work assumes `include_symbols` remains default-on, so the upgrade is visible in default output rather than gated behind a flag. + +--- + +## Sources / Research + +- `src/extraction/basic_extractor.rs:67-89` - existing untagged import/export emission that R11 replaces (gated on `config.include_symbols`). +- `src/extraction/traits.rs:97` - `include_symbols` defaults to `true`. +- `src/types/mod.rs` - `ImportInfo` (`name`, `library: Option`, `address: Option`, `ordinal`), `ExportInfo` (`name`, `address: u64`, `ordinal`), `ContainerInfo.imports`/`exports`, and the `Tag` / `StringSource` enums (`Import`, `Export`, `DemangledSymbol`, `ImportName`, `ExportName`, `SectionData` all present; `Crypto`/`Network`/`FileIO`/ `EntryPoint` absent). +- `src/types/found_string.rs` - `FoundString::new(text, encoding, offset, length, source)` plus `with_rva`, `with_section`, `with_tags`, `with_confidence` builders (constructor defaults `confidence: 1.0`). +- `src/classification/symbols.rs` - `SymbolDemangler` (Rust via `rustc_demangle`, C++ via `cpp_demangle`), reused directly. +- `src/classification/ranking.rs:131-132` - existing `Import`/`Export` +60 boosts that R10 extends. +- `docs/src/classification.md:77,83` - stale `once_cell::sync::Lazy` example that R14 corrects. +- GitHub issue #20 - originating specification (note: its "nothing converts imports/exports into FoundStrings" premise is corrected by R11). diff --git a/docs/plans/2026-06-19-001-feat-symbol-classification-pipeline-plan.md b/docs/plans/2026-06-19-001-feat-symbol-classification-pipeline-plan.md new file mode 100644 index 0000000..e4065e6 --- /dev/null +++ b/docs/plans/2026-06-19-001-feat-symbol-classification-pipeline-plan.md @@ -0,0 +1,232 @@ +--- +title: 'feat: Import/Export symbol classification pipeline' +type: feat +date: 2026-06-19 +origin: docs/brainstorms/2026-06-19-symbol-classification-pipeline-requirements.md +--- + +# feat: Import/Export symbol classification pipeline + +## Summary + +Add an `ImportClassifier` that converts `ContainerInfo` import/export symbols and section names into tagged, scored `FoundString`s, and route it into the extraction path as the single symbol emission point so the already-defined Import/Export ranking boosts finally fire. Four new semantic tags (crypto, network, file-I/O, entry-point) are recognized via static API-name sets and the existing demangler. + +--- + +## Problem Frame + +`stringy` already emits import and export names in default output: when `include_symbols` is on (the default), `src/extraction/basic_extractor.rs:67-89` pushes each symbol name as a bare `FoundString` -- no tags, no RVA, no library, no semantic classification. The `RankingEngine` defines +60 boosts for `Tag::Import` and `Tag::Export` (`src/classification/ranking.rs:131-132`), but nothing applies those tags, so the boosts are dead code and symbol strings rank no higher than incidental noise. The demangler, the tag scaffolding, the ranking boosts, and the parsed symbol metadata all exist in isolation; what is missing is the classifier that ties metadata to tags plus the wiring that makes the tagged strings the ones that reach output. + +--- + +## High-Level Technical Design + +The classifier becomes the single emission point for symbol strings, replacing the untagged push in the extractor. Tags accumulate down the pipeline rather than being overwritten, so classifier-assigned tags survive to ranking where the boosts apply. + +```mermaid +flowchart TB + CI[ContainerInfo: imports, exports, sections] --> IC[ImportClassifier] + IC --> PI[process_imports: Import + semantic] + IC --> PE[process_exports: Export + demangle + entry-point] + IC --> PS[process_section_names: SectionData] + PI --> ESS[extract_symbol_strings -> Vec FoundString, tagged] + PE --> ESS + PS --> ESS + ESS --> BE[basic_extractor include_symbols block: replaces untagged push] + BE --> DD[dedup: merge by text+encoding] + DD --> CS[pipeline classify_strings: APPENDS semantic tags, preserves existing] + CS --> RK[rank_strings: Import/Export/Crypto/Network/FileIO/EntryPoint boosts fire] + RK --> OUT[ranked output] +``` + +--- + +## Key Technical Decisions + +- KTD1. **Replace the untagged emission at `basic_extractor.rs:67-89`, not add a parallel path.** A parallel tagged copy alongside the existing untagged one would produce two `FoundString`s per symbol. Dedup keys by `(text, encoding)` and unions tags across occurrences via `merge_tags` (`src/extraction/dedup/mod.rs:237-248`), so tags are not lost -- but it adopts the `source`/`section`/`rva` metadata of a nondeterministic first occurrence, so a symbol can surface carrying the untagged copy's bare metadata. A single emission point removes that ambiguity. (see origin: docs/brainstorms/2026-06-19-symbol-classification-pipeline-requirements.md) +- KTD2. **Rely on the pipeline's append-only classification.** `classify_strings()` pushes each semantic tag only `if !s.tags.contains(&tag)` (`src/pipeline/mod.rs:320-326`), so classifier-assigned `Import`/`Export`/semantic tags are preserved through the pipeline's own pass. No tag-clobber guard is needed; R12 depends on this behavior. +- KTD3. **Export order: tag `Export`, then demangle, then entry-point check.** `SymbolDemangler::demangle(&mut fs)` adds `Tag::DemangledSymbol` and preserves existing tags; the entry-point name match runs against the resulting `text`. +- KTD4. **Wire the four tags as one foundational unit across all co-change sites.** Adding a variant touches two compiler-enforced output formatters (`tag_to_display_string`, YARA `tag_name`) -- the build will not compile until those are updated -- plus the enum/`FromStr`, both CLI `long_help` literals, and five variant-enumerating test lists, which are not compiler-checked and fail at test-time instead. All land together so CI passes. +- KTD5. **Static API sets as `LazyLock>`.** Mirrors `src/classification/patterns/paths.rs`. Never `once_cell`. +- KTD6. **Section-name emission rides `include_symbols` (default-on).** All three symbol kinds (imports, exports, section names) are bundled behind one `extract_symbol_strings` call at the existing `include_symbols` gate, so the symbols toggle also governs section-name rows. +- KTD7. **Extraction labels ASCII content as UTF-8, not a distinct `Encoding::Ascii`.** Import-bearing sections (e.g. PE `.idata`) are byte-scanned by default; if that scan emitted `CreateFileW` as `Encoding::Ascii` while the classifier emits `Encoding::Utf8`, the `(text, encoding)` dedup key would split and AE4's "exactly once" would break. The ASCII extraction path instead emits `Encoding::Utf8` (ASCII is a UTF-8 subset), so the byte-scanned and classifier occurrences of the same name share one dedup key and merge. `Encoding::Ascii` is no longer produced as extraction output; the enum variant is retained for compatibility but unused. (U6 carries this normalization.) +- KTD8. **Section names use a dedicated `StringSource::SectionName` variant.** Reusing `StringSource::SectionData` (which means "string found in section bytes") would make section-name rows indistinguishable from byte-scanned content to downstream `source` filters and classification context. A new `StringSource::SectionName` variant keeps the two distinguishable. +- KTD9. **`--enc ascii` becomes a content filter.** With extraction no longer labeling rows `Encoding::Ascii`, `EncodingFilter::Exact(Encoding::Ascii)` would match nothing. `--enc ascii` is redefined to match rows whose text is pure-ASCII (`text.is_ascii()`), computed from content rather than stored encoding, so the documented CLI flag keeps working. `--enc utf8`/`--enc utf16*` are unchanged. + +--- + +## Requirements + +Carried from the origin requirements doc. + +**Symbol classification** + +- R1. Import names emit as `FoundString`s with `source: StringSource::ImportName`, always carrying `Tag::Import`, at `confidence: 1.0`. +- R2. Imports matching the crypto, network, or file-I/O API sets additionally carry `Tag::Crypto`, `Tag::Network`, or `Tag::FileIO`; non-matching imports carry only `Tag::Import`. +- R3. An import's originating library populates the `section` field when present; when absent, a format-appropriate default is used (`.idata` for PE, `.dynsym` for ELF, `__LINKEDIT` for Mach-O). Mach-O imports never carry a library -- the parser leaves it `None` (`src/container/macho/mod.rs:136`) -- so Mach-O always uses the default. +- R4. An import's RVA populates from `ImportInfo::address` when present (optional). +- R5. Export names emit as `FoundString`s with `source: StringSource::ExportName`, always carrying `Tag::Export`, with RVA from `ExportInfo::address`. +- R6. Exports run through `SymbolDemangler`; successfully demangled exports additionally carry `Tag::DemangledSymbol`. +- R7. Exports whose name is a known entry point (`main`, `_start`, `DllMain`, `WinMain`, `wWinMain`) additionally carry `Tag::EntryPoint`. +- R8. Section names emit as standalone `FoundString`s with `source: StringSource::SectionName` (a new variant, distinct from `SectionData`) at `confidence: 1.0`, ranked alongside other output. + +**Type system and ranking** + +- R9. Add `Tag` variants `Crypto`, `Network`, `FileIO`, `EntryPoint` with serde renames (`crypto`, `network`, `fileio`, `entry-point`), wired through `Tag::from_str`, the output display formatters, and the CLI help text. +- R10. `RankingConfig` gains tag boosts: `Crypto` +50, `Network` +45, `FileIO` +35, `EntryPoint` +40. + +**Pipeline integration** + +- R11. The tagged classifier replaces the untagged import/export emission in the extractor so each symbol is emitted once, already tagged; section-name emission is added to the same path. +- R12. Tagged symbol strings flow through the remaining pipeline stages with their classifier-assigned tags preserved, so the Import/Export and new semantic boosts apply to live output. + +**Encoding normalization** + +- R15. The ASCII extraction path emits `Encoding::Utf8`; no extracted row carries `Encoding::Ascii`. The `Encoding::Ascii` enum variant is retained for compatibility but is not produced as output. +- R16. `--enc ascii` filters by content -- it matches rows whose text is pure-ASCII (`text.is_ascii()`), independent of stored encoding. `--enc utf8` and `--enc utf16*` are unchanged. + +**Tests and docs** + +- R13. `tests/classification_symbols.rs` covers import classification, export classification, API-set detection (at least 3 representative symbols per category plus a negative case), section-name extraction, and end-to-end `extract_symbol_strings` output against the ELF, PE, and Mach-O fixtures. +- R14. `docs/src/classification.md` is updated to reference `std::sync::LazyLock` instead of the stale `once_cell::sync::Lazy` example. + +--- + +## Acceptance Examples + +- AE1. **Covers R2.** Import `CreateFileW` yields `Import` + `FileIO`; `connect` yields `Import` + `Network`; `EVP_EncryptInit` yields `Import` + `Crypto`; `printf` yields `Import` only. +- AE2. **Covers R6, R7.** A C++-mangled export demangles and yields `Export` + `DemangledSymbol`; export `main` yields `Export` + `EntryPoint`; export `DllMain` yields `Export` + `EntryPoint`. +- AE3. **Covers R3.** A PE import from `kernel32.dll` sets `section` to `kernel32.dll`; an ELF import with no library sets `section` to `.dynsym`; a Mach-O import (always library-less) sets `section` to `__LINKEDIT`. +- AE4. **Covers R11, R12.** Running `stringy` on a PE fixture (default `include_symbols`) shows `CreateFileW` exactly once, tagged `import` + `fileio` -- not a duplicate untagged row. Because the byte-scanned occurrence and the classifier occurrence both carry UTF-8 encoding (KTD7), dedup merges them into the single tagged row rather than splitting on encoding. + +--- + +## Implementation Units + +### U1. New Tag variants and full co-change wiring + +- **Goal:** Add `Crypto`, `Network`, `FileIO`, `EntryPoint` to the `Tag` enum and a `SectionName` variant to the `StringSource` enum, threading both through every site adding a variant touches, so the crate compiles and all variant-enumerating tests pass. +- **Requirements:** R9; the new `StringSource::SectionName` variant that R8 consumes +- **Dependencies:** none +- **Files:** `src/types/mod.rs`, `src/output/table/formatting.rs`, `src/output/yara/mod.rs`, `src/main.rs`, `src/types/tests.rs`, `src/output/json.rs`, `tests/integration_cli.rs`, `tests/output_yara_integration.rs` +- **Approach:** Add the four variants with serde renames (`crypto`, `network`, `fileio`, `entry-point`). Add `FromStr` arms. Add display arms in `tag_to_display_string` (`src/output/table/formatting.rs:81-103`) and YARA `tag_name` (`src/output/yara/mod.rs:164-187`) -- both are compiler-enforced exhaustive matches. Add the canonical strings to both `--only-tags` and `--no-tags` `long_help` literals (`src/main.rs:108-111`, `122-125`; must be string literals, not `const`/`concat!`). Extend the variant-enumerating test lists. Separately, add a `StringSource::SectionName` variant to the `StringSource` enum and update its match and serialization sites by mirroring the existing `StringSource` variants (e.g. `ImportName`, `ExportName`). +- **Patterns to follow:** existing `DemangledSymbol` variant (serde rename + `FromStr` arm) in `src/types/mod.rs`; the exhaustive matches in `formatting.rs:81-103` and `yara/mod.rs:164-187`. +- **Test scenarios:** + - Covers R9. `from_str("crypto"/"network"/"fileio"/"entry-point")` return the matching variants; an unknown string still returns `Err` (extend `test_tag_from_str_all_variants`, `src/types/tests.rs:126-148`). + - Each new variant produces its display string in `tag_to_display_string` and YARA `tag_name` (extend `all_tag_variants_have_display`, `formatting.rs:224-245`, and `test_yara_all_tag_types`, `tests/output_yara_integration.rs:154-177`). + - JSON serialization yields the serde-renamed form: `Tag::FileIO` -> `"fileio"`, `Tag::EntryPoint` -> `"entry-point"` (extend `test_all_tag_types_serialize_correct_names`, `src/output/json.rs:159-181`). + - `stringy --help` lists `crypto`, `network`, `fileio`, `entry-point` under both flags (extend `cli_help_lists_all_canonical_tags`, `tests/integration_cli.rs:264-291`). +- **Verification:** `cargo build` compiles with no non-exhaustive-match errors; the extended variant-list tests pass; `--help` shows the four new tags. + +### U2. RankingConfig boosts for the new tags + +- **Goal:** Give the four new tags their ranking boosts so semantic symbols outrank noise. +- **Requirements:** R10 +- **Dependencies:** U1 +- **Files:** `src/classification/ranking.rs` +- **Approach:** Add `tag_boosts.insert(...)` for `Crypto` +50, `Network` +45, `FileIO` +35, `EntryPoint` +40 in `RankingConfig::default()`; add matching assertions in `test_default_config_values`. +- **Patterns to follow:** `ranking.rs:131-132` (Import/Export +60 inserts) and the paired `test_default_config_values` (`ranking.rs:304-349`). +- **Test scenarios:** + - Covers R10. `tag_boosts.get(&Tag::Crypto) == Some(&50)`, `Network` `&45`, `FileIO` `&35`, `EntryPoint` `&40` (extend `test_default_config_values`). + - A `FoundString` carrying `Tag::Crypto` scores at least 50 above an otherwise identical untagged string under the default config. +- **Verification:** `test_default_config_values` passes with the four new assertions; a crypto-tagged string ranks above an untagged peer. + +### U3. ImportClassifier and extract_symbol_strings + +- **Goal:** Build the classifier converting `ContainerInfo` imports/exports/section names into tagged `FoundString`s, plus the `extract_symbol_strings` entry point, and test it directly. +- **Requirements:** R1, R2, R3, R4, R5, R6, R7, R8, R13; AE1, AE2, AE3 +- **Dependencies:** U1 +- **Files:** `src/classification/imports.rs` (new), `src/classification/mod.rs`, `tests/classification_symbols.rs` (new) +- **Approach:** New `ImportClassifier` holding a `SymbolDemangler` and three `LazyLock>` API sets (crypto, network, file-I/O) seeded from the origin lists. `process_imports`: build via `FoundString::new(name, Utf8, 0, len, ImportName)` then builder chain -- tags `[Import]` plus any semantic match, `.with_section(library or .idata/.dynsym default)`, `.with_rva` when `address` is `Some`, `.with_confidence(1.0)`. `process_exports`: tags `[Export]`, RVA from the required `address`, then `demangler.demangle(&mut fs)`, then entry-point check on `fs.text`. `process_section_names`: one `FoundString` per section name, source `StringSource::SectionName` (the U1 variant), encoding `Utf8`, confidence 1.0. `extract_symbol_strings(&ContainerInfo)` composes all three. Export `ImportClassifier` and `extract_symbol_strings` from `src/classification/mod.rs` (`pub mod imports;` + `pub use`). +- **Technical design (directional, not a spec):** export processing order -- `new(... ExportName) -> with_tags([Export]) -> with_rva(address) -> demangle(&mut) [adds DemangledSymbol] -> if text in ENTRY_POINTS push EntryPoint`. +- **Patterns to follow:** `LazyLock` idiom in `src/classification/patterns/paths.rs:30-43` (`HashSet::from([...])`); `FoundString::new()` + builder chain only, never struct literals (GOTCHAS); `SymbolDemangler::demangle` in `src/classification/symbols.rs`; fixture loading via `get_fixture_path` in `tests/classification_integration_tests.rs`. +- **Execution note:** Implement classifier behavior test-first. +- **Test scenarios** (`tests/classification_symbols.rs`): + - Covers R1. Every emitted import carries `Tag::Import`, source `ImportName`, confidence 1.0. + - Covers AE1. `CreateFileW` -> `Import`+`FileIO`; `connect` -> `Import`+`Network`; `EVP_EncryptInit` -> `Import`+`Crypto`; `printf` -> `Import` only (at least 3 matches per category plus the negative). + - Covers AE3 / R3. PE import with library `kernel32.dll` -> section `kernel32.dll`; PE import with no library -> `.idata`; ELF import with no library -> `.dynsym`; Mach-O import (always library-less) -> `__LINKEDIT`. + - Covers R4. Import with `address: Some(x)` sets `rva` to `x`; `address: None` leaves `rva` unset. + - Covers R5. Every export carries `Tag::Export`, source `ExportName`, RVA from `address`. + - Covers AE2 / R6, R7. A C++-mangled export gains `DemangledSymbol` and demangled text; `main`/`_start`/`DllMain`/`WinMain`/`wWinMain` gain `EntryPoint`; an ordinary export gains neither. + - Covers R8. Section names emit as `FoundString`s with source `StringSource::SectionName`, confidence 1.0. + - Covers R13. `extract_symbol_strings` on parsed ELF/PE/Mach-O fixtures returns tagged import/export/section-name strings; JSON-form comparisons use the serde-renamed tag strings. +- **Verification:** `tests/classification_symbols.rs` passes; `extract_symbol_strings` is reachable as `stringy::classification::extract_symbol_strings`. + +### U6. Normalize ASCII extraction to UTF-8 and redefine `--enc ascii` + +- **Goal:** Stop labeling extracted ASCII content as `Encoding::Ascii` so symbol rows merge under one UTF-8 dedup key (KTD7), and redefine `--enc ascii` as a content filter (KTD9) so the CLI flag keeps working. +- **Requirements:** R15, R16; enables AE4 +- **Dependencies:** none (independent of the tag work; U4 depends on this unit) +- **Files:** `src/extraction/ascii/extraction.rs`, `src/extraction/traits.rs`, `src/main.rs`, `src/pipeline/config.rs`, `src/pipeline/filter.rs`, `src/extraction/ascii/tests.rs`, `src/extraction/tests.rs`, `src/types/tests.rs`, affected insta snapshots +- **Approach:** Change the ASCII extraction pass (`src/extraction/ascii/extraction.rs:99,124,135`) to emit `Encoding::Utf8` instead of `Encoding::Ascii`. The `enabled_encodings` `Ascii` entry (`src/extraction/traits.rs:100`) still gates whether the ASCII byte-scan runs -- it controls scanning, not output labeling -- so the default stays as-is. Add a content-matching `EncodingFilter` variant (e.g. `AsciiContent`) in `src/pipeline/config.rs` whose `src/pipeline/filter.rs` arm matches `s.text.is_ascii()`, and map `CliEncoding::Ascii` (`src/main.rs:50`) to it instead of `EncodingFilter::Exact(Encoding::Ascii)`. Retain the `Encoding::Ascii` enum variant (no output uses it). Update tests asserting `Encoding::Ascii` on extracted strings and regenerate snapshots. +- **Execution note:** Capture current ASCII-labeled output via snapshots before the relabel, so the diff documents the encoding shift. +- **Test scenarios:** + - Covers R15. A pure-ASCII string extracted from a section is labeled `Encoding::Utf8`, not `Encoding::Ascii`. + - Covers R16. `--enc ascii` returns rows whose text is pure-ASCII (e.g. `CreateFileW`) and excludes rows whose text contains non-ASCII characters, regardless of stored encoding. + - Edge: a UTF-8 string with non-ASCII characters is excluded by `--enc ascii` but included by `--enc utf8`. + - Regression: `--enc utf8` and `--enc utf16*` filtering behavior is unchanged. +- **Verification:** no extracted row carries `Encoding::Ascii`; `--enc ascii` filters by text content; the encoding unit tests and regenerated snapshots pass. + +### U4. Replace the untagged symbol emission in the extractor + +- **Goal:** Route `extract_symbol_strings` into the extraction path as the single symbol emission point so tagged symbols reach live output and the boosts fire. +- **Requirements:** R11, R12; AE4 +- **Dependencies:** U3, U6 (also U1, U2 for the tags and their boosts) +- **Files:** `src/extraction/basic_extractor.rs`, affected insta snapshots (e.g. the `tests/snapshots` for `integration_flows_1_5`) +- **Approach:** In `collect_all_strings`, replace the import/export push loops at `basic_extractor.rs:67-89` with a call to `extract_symbol_strings(container_info)`, still inside the `if config.include_symbols` block (which now also governs section-name rows). Symbol strings are emitted as `Utf8`, and with U6 normalizing the byte-scan to UTF-8 as well (KTD7), a byte-scanned occurrence of the same name (e.g. from PE `.idata`) shares the `(text, Utf8)` dedup key and merges into the single tagged row rather than producing a separate untagged `Ascii` row; `classify_strings` then appends semantic tags without clobbering the classifier's (KTD2). Regenerate insta snapshots whose fixture output now shows tagged, reranked symbols. `test_binary.c` is unchanged, so no `just gen-fixtures` -- snapshot regeneration only. +- **Execution note:** Capture current snapshots before the change, then regenerate after, so the snapshot diff is the reviewable record of the output shift. +- **Test scenarios:** + - Covers AE4 / R11. The pipeline on a PE fixture yields exactly one row per symbol -- no duplicate untagged-plus-tagged pair (the byte-scanned and classifier occurrences merge under the shared UTF-8 dedup key). + - Covers R12. A symbol that also matches a semantic pattern carries both its classifier tags and any pipeline-appended tags after `classify_strings`. + - Covers R11. With `include_symbols = false`, no import/export or section-name rows are emitted; with the default (`true`), they are. + - Regression: existing flow/snapshot tests pass after regeneration, and symbol strings rank above incidental noise. +- **Verification:** the full `nextest` suite passes after `INSTA_UPDATE=always cargo nextest run`; a CLI run on a PE fixture shows tagged imports ranked up, one row each. + +### U5. Documentation fix + +- **Goal:** Correct the stale `once_cell` example in the classification docs. +- **Requirements:** R14 +- **Dependencies:** none +- **Files:** `docs/src/classification.md` +- **Approach:** Replace the `once_cell::sync::Lazy` usages (`docs/src/classification.md` lines 77, 80, 120) with `std::sync::LazyLock` to match the actual implementation. +- **Patterns to follow:** the `LazyLock` idiom in `src/classification/mod.rs`. +- **Test scenarios:** Test expectation: none -- documentation-only change with no behavioral surface. +- **Verification:** `docs/src/classification.md` no longer references `once_cell` and the example mirrors the `std::sync::LazyLock` pattern. + +--- + +## Scope Boundaries + +- New CLI flags to filter the four new tags -- out of scope; the existing `--only-tags`/`--no-tags` machinery covers them once the variants are wired. +- Redesigning deduplication internals or string-ordering remains out of scope. The one exception is encoding normalization: U6 relabels extracted ASCII content as UTF-8 and redefines `--enc ascii` as a content filter (R15, R16), which is in scope. +- Removing the `Encoding::Ascii` enum variant -- out of scope; it is retained for compatibility, just not produced as extraction output. + +### Deferred to Follow-Up Work + +- Tuning the exact membership of the crypto/network/file-I/O API sets beyond the origin seed lists -- left to implementation and later iteration. + +--- + +## Risks & Dependencies + +- **Snapshot churn (broad).** Two changes shift fixture output: routing tagged symbols into default output reranks rows, and U6 relabels every previously-`Ascii` row as `Utf8`. Insta snapshots capturing fixture output (notably `integration_flows_1_5`) will change widely and must be regenerated. Expected; if a diff looks wrong (a symbol losing its tag, a row vanishing), investigate before accepting rather than blanket-accepting. +- **CLI contract change for `--enc ascii` (R16).** The flag's meaning shifts from "stored encoding is ASCII" to "text content is pure-ASCII". Output for `--enc ascii` will differ from prior releases. Update the AGENTS.md CLI table and any user-facing docs describing `--enc`. +- **Append-only classification (KTD2).** R12 depends on `classify_strings` pushing tags only when absent. If that behavior changes, classifier tags could be lost. +- **Field-shape asymmetry.** `ExportInfo::address` is a required `u64` while `ImportInfo::address` is `Option` -- export RVA is always set, import RVA is conditional (R4, R5 reflect this). +- **Fixtures gitignored.** `just gen-fixtures` must run before tests; `test_binary.c` is unchanged here, so only snapshot regeneration is needed, not fixture rebuild. + +--- + +## Sources / Research + +- `src/extraction/basic_extractor.rs:67-89` -- existing untagged import/export emission that U4 replaces; `src/extraction/traits.rs:97` -- `include_symbols` defaults to `true`. +- `src/pipeline/mod.rs:320-326` -- `classify_strings` appends tags only when absent (basis for KTD2 / R12). +- `src/types/mod.rs` -- `Tag` enum (serde renames; `Import`/`Export`/`DemangledSymbol` present, the four new variants absent), `ImportInfo` (`name`, `library: Option`, `address: Option`, `ordinal`), `ExportInfo` (`name`, `address: u64`, `ordinal`), `ContainerInfo.imports`/`exports`. +- `src/types/found_string.rs` -- `FoundString::new(text, encoding, offset, length, source)` plus `with_rva`, `with_section`, `with_tags`, `with_confidence` builders (constructor defaults `confidence: 1.0`, empty tags). +- `src/classification/symbols.rs` -- `SymbolDemangler` (Rust via `rustc_demangle`, C++ via `cpp_demangle`); demangle adds `DemangledSymbol` and preserves existing tags. +- `src/classification/ranking.rs:131-132` -- existing Import/Export +60 boosts; `:304-349` -- the paired `test_default_config_values`. +- `src/classification/patterns/paths.rs:30-43` -- `LazyLock>` idiom to mirror. +- Exhaustive-match and variant-list co-change sites (U1): `tag_to_display_string` (`src/output/table/formatting.rs:81-103`), YARA `tag_name` (`src/output/yara/mod.rs:164-187`), CLI `long_help` (`src/main.rs:108-111`, `122-125`), and test lists in `src/types/tests.rs:126-148`, `src/output/json.rs:159-181`, `src/output/table/formatting.rs:224-245`, `tests/output_yara_integration.rs:154-177`, `tests/integration_cli.rs:264-291`. +- Encoding model (U6): `Encoding` enum at `src/types/mod.rs:13` (`Ascii`, `Utf8`, `Utf16Le`, `Utf16Be`); ASCII extraction emits `Encoding::Ascii` at `src/extraction/ascii/extraction.rs:99,124,135`; default `enabled_encodings: [Ascii, Utf8]` at `src/extraction/traits.rs:100`; `CliEncoding::Ascii -> EncodingFilter::Exact(Encoding::Ascii)` at `src/main.rs:50`; the `EncodingFilter` match arm at `src/pipeline/filter.rs:44`. +- `docs/src/classification.md:77,80,120` -- stale `once_cell` example that U5 fixes. +- GitHub issue #20 -- originating specification (its "nothing converts imports/exports into FoundStrings" premise is corrected: the extractor already emits them untagged, which U4 replaces). diff --git a/docs/src/classification.md b/docs/src/classification.md index 3f358d9..b4203a3 100644 --- a/docs/src/classification.md +++ b/docs/src/classification.md @@ -74,13 +74,13 @@ Raw String -> Pattern Matching -> Validation -> Tag Assignment ## Pattern Matching Engine -The semantic classifier uses cached regex patterns via `once_cell::sync::Lazy` and applies validation checks to reduce false positives. +The semantic classifier uses cached regex patterns via `std::sync::LazyLock` and applies validation checks to reduce false positives. ```rust -use once_cell::sync::Lazy; use regex::Regex; +use std::sync::LazyLock; -static GUID_REGEX: Lazy = Lazy::new(|| { +static GUID_REGEX: LazyLock = LazyLock::new(|| { Regex::new(r"^\{[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\}$") .expect("Invalid GUID regex") }); @@ -117,7 +117,7 @@ if tags.contains(&Tag::Guid) { ## Performance Notes -- Regexes are compiled once via `once_cell::sync::Lazy` and reused across calls. +- Regexes are compiled once via `std::sync::LazyLock` and reused across calls. - Minimum length checks avoid unnecessary regex work on short inputs. - The classifier is stateless and thread-safe. diff --git a/examples/basic_extraction.rs b/examples/basic_extraction.rs index ce05ebb..85d3750 100644 --- a/examples/basic_extraction.rs +++ b/examples/basic_extraction.rs @@ -48,7 +48,7 @@ fn main() -> Result<(), Box> { // Display the top 20 strings by score let mut sorted_strings = strings.clone(); - sorted_strings.sort_by(|a, b| b.score.cmp(&a.score)); + sorted_strings.sort_by_key(|b| std::cmp::Reverse(b.score)); println!("Top strings by score:"); println!("{:-<60}", ""); diff --git a/examples/output_formats.rs b/examples/output_formats.rs index f52aaf0..56424a2 100644 --- a/examples/output_formats.rs +++ b/examples/output_formats.rs @@ -47,7 +47,7 @@ fn main() -> Result<(), Box> { // Limit to top 50 strings for demonstration let mut sorted_strings = strings; - sorted_strings.sort_by(|a, b| b.score.cmp(&a.score)); + sorted_strings.sort_by_key(|b| std::cmp::Reverse(b.score)); let top_strings: Vec<_> = sorted_strings.into_iter().take(50).collect(); // Create output metadata diff --git a/mise.lock b/mise.lock index 9b32766..f869795 100644 --- a/mise.lock +++ b/mise.lock @@ -707,3 +707,43 @@ provenance = "github-attestations" checksum = "sha256:ace861f360c6de2babedc1607d0f454b6b09a820dbc8182dc15af927e4df9589" url = "https://github.com/astral-sh/uv/releases/download/0.11.21/uv-x86_64-pc-windows-msvc.zip" provenance = "github-attestations" + +[[tools.zig]] +version = "0.15.2" +backend = "core:zig" + +[tools.zig."platforms.linux-x64"] +checksum = "sha256:02aa270f183da276e5b5920b1dac44a63f1a49e55050ebde3aecc9eb82f93239" +url = "https://ziglang.org/download/0.15.2/zig-x86_64-linux-0.15.2.tar.xz" + +[tools.zig."platforms.linux-x64-baseline"] +checksum = "sha256:02aa270f183da276e5b5920b1dac44a63f1a49e55050ebde3aecc9eb82f93239" +url = "https://ziglang.org/download/0.15.2/zig-x86_64-linux-0.15.2.tar.xz" + +[tools.zig."platforms.linux-x64-musl"] +checksum = "sha256:02aa270f183da276e5b5920b1dac44a63f1a49e55050ebde3aecc9eb82f93239" +url = "https://ziglang.org/download/0.15.2/zig-x86_64-linux-0.15.2.tar.xz" + +[tools.zig."platforms.linux-x64-musl-baseline"] +checksum = "sha256:02aa270f183da276e5b5920b1dac44a63f1a49e55050ebde3aecc9eb82f93239" +url = "https://ziglang.org/download/0.15.2/zig-x86_64-linux-0.15.2.tar.xz" + +[tools.zig."platforms.macos-arm64"] +checksum = "sha256:3cc2bab367e185cdfb27501c4b30b1b0653c28d9f73df8dc91488e66ece5fa6b" +url = "https://ziglang.org/download/0.15.2/zig-aarch64-macos-0.15.2.tar.xz" + +[tools.zig."platforms.macos-x64"] +checksum = "sha256:375b6909fc1495d16fc2c7db9538f707456bfc3373b14ee83fdd3e22b3d43f7f" +url = "https://ziglang.org/download/0.15.2/zig-x86_64-macos-0.15.2.tar.xz" + +[tools.zig."platforms.macos-x64-baseline"] +checksum = "sha256:375b6909fc1495d16fc2c7db9538f707456bfc3373b14ee83fdd3e22b3d43f7f" +url = "https://ziglang.org/download/0.15.2/zig-x86_64-macos-0.15.2.tar.xz" + +[tools.zig."platforms.windows-x64"] +checksum = "sha256:3a0ed1e8799a2f8ce2a6e6290a9ff22e6906f8227865911fb7ddedc3cc14cb0c" +url = "https://ziglang.org/download/0.15.2/zig-x86_64-windows-0.15.2.zip" + +[tools.zig."platforms.windows-x64-baseline"] +checksum = "sha256:3a0ed1e8799a2f8ce2a6e6290a9ff22e6906f8227865911fb7ddedc3cc14cb0c" +url = "https://ziglang.org/download/0.15.2/zig-x86_64-windows-0.15.2.zip" diff --git a/mise.toml b/mise.toml index d633a23..fe5fee0 100644 --- a/mise.toml +++ b/mise.toml @@ -39,6 +39,7 @@ scorecard = "5.5.0" "bun" = "latest" shellcheck = "0.11.0" uv = "latest" +zig = "0.15.2" # Many of these settings are defaults, but we are explicit in case they change in the future and to make it clear to users what is enabled. [settings] diff --git a/src/classification/imports.rs b/src/classification/imports.rs new file mode 100644 index 0000000..b7a8caa --- /dev/null +++ b/src/classification/imports.rs @@ -0,0 +1,261 @@ +//! Import/export symbol classification. +//! +//! Converts parsed [`ContainerInfo`] imports, exports, and section names into +//! tagged, scored [`FoundString`]s. This is the single emission point for +//! symbol strings in the extraction path: every import carries [`Tag::Import`] +//! (plus a semantic tag when its name matches a known crypto, network, or +//! file-I/O API), every export carries [`Tag::Export`] (plus [`Tag::EntryPoint`] +//! for known entry points), and each section name is emitted as a standalone +//! row with [`StringSource::SectionName`]. +//! +//! Demangling is deliberately left to the pipeline's `classify_strings` (which +//! runs under `catch_unwind` and is skipped in raw mode); the classifier only +//! tags. Exports that are mangled symbols receive [`Tag::DemangledSymbol`] and +//! their demangled text there when demangling succeeds; names that fail to +//! demangle are left unchanged. +//! +//! Symbol strings are emitted with [`Encoding::Utf8`] so that a byte-scanned +//! occurrence of the same name (e.g. from a PE `.idata` section) shares the +//! `(text, encoding)` deduplication key and merges into the single tagged row. + +use std::collections::HashSet; +use std::sync::LazyLock; + +use crate::types::{ + BinaryFormat, ContainerInfo, Encoding, ExportInfo, FoundString, ImportInfo, SectionInfo, + StringSource, Tag, +}; + +/// Crypto-related API names. Matching imports gain [`Tag::Crypto`]. +static CRYPTO_APIS: LazyLock> = LazyLock::new(|| { + HashSet::from([ + "EVP_EncryptInit", + "EVP_DecryptInit", + "EVP_DigestInit", + "CryptEncrypt", + "CryptDecrypt", + "CryptAcquireContextW", + "CryptGenKey", + "BCryptEncrypt", + "BCryptDecrypt", + "AES_encrypt", + "AES_decrypt", + "SHA256_Init", + "MD5_Init", + ]) +}); + +/// Network-related API names. Matching imports gain [`Tag::Network`]. +static NETWORK_APIS: LazyLock> = LazyLock::new(|| { + HashSet::from([ + "connect", + "send", + "recv", + "socket", + "bind", + "listen", + "accept", + "getaddrinfo", + "gethostbyname", + "WSAStartup", + "WSASocketW", + "InternetOpenW", + "InternetConnectW", + "HttpSendRequestW", + ]) +}); + +/// File-I/O-related API names. Matching imports gain [`Tag::FileIO`]. +static FILEIO_APIS: LazyLock> = LazyLock::new(|| { + HashSet::from([ + "CreateFileW", + "CreateFileA", + "ReadFile", + "WriteFile", + "DeleteFileW", + "MoveFileW", + "CopyFileW", + "fopen", + "fread", + "fwrite", + "open", + "read", + "write", + ]) +}); + +/// Known program entry-point symbol names. Matching exports gain +/// [`Tag::EntryPoint`]. +static ENTRY_POINTS: LazyLock> = + LazyLock::new(|| HashSet::from(["main", "_start", "DllMain", "WinMain", "wWinMain"])); + +/// Classifies import/export symbols and section names into tagged strings. +/// +/// Stateless: tagging is the classifier's job. Demangling is performed once by +/// the pipeline's `classify_strings` (under `catch_unwind`, and skipped in raw +/// mode), so it is intentionally not done here. +#[derive(Debug, Default, Clone)] +pub struct ImportClassifier; + +impl ImportClassifier { + /// Create a new `ImportClassifier`. + #[must_use] + pub fn new() -> Self { + Self + } + + /// Convert import symbols into tagged `FoundString`s. + /// + /// Each import carries [`Tag::Import`] plus a semantic tag when its name + /// matches a known API set. The `section` field is populated from the + /// import's originating library when present, otherwise from a + /// format-appropriate default (see [`default_import_section`]). The RVA is + /// populated from `ImportInfo::address` when available. + #[must_use] + pub fn process_imports( + &self, + imports: &[ImportInfo], + format: BinaryFormat, + ) -> Vec { + imports + .iter() + .map(|import| { + let mut tags = vec![Tag::Import]; + if let Some(semantic) = semantic_tag(&import.name) { + tags.push(semantic); + } + + let section = import + .library + .clone() + .unwrap_or_else(|| default_import_section(format).to_string()); + + let length = import.name.len() as u32; + let mut found = FoundString::new( + import.name.clone(), + Encoding::Utf8, + 0, + length, + StringSource::ImportName, + ) + .with_tags(tags) + .with_section(section); + + if let Some(address) = import.address { + found = found.with_rva(address); + } + + found + }) + .collect() + } + + /// Convert export symbols into tagged `FoundString`s. + /// + /// Each export carries [`Tag::Export`] and an RVA from the always-present + /// `ExportInfo::address`, plus [`Tag::EntryPoint`] when the name is a known + /// entry point. Demangling (and the resulting [`Tag::DemangledSymbol`]) is + /// performed downstream by the pipeline's `classify_strings`, which runs + /// under `catch_unwind` and is skipped in raw mode -- so a panic in a + /// third-party demangler never aborts extraction, and raw mode shows the + /// untouched symbol name. Entry-point names are never mangled, so the check + /// runs correctly on the raw export name. + #[must_use] + pub fn process_exports(&self, exports: &[ExportInfo]) -> Vec { + exports + .iter() + .map(|export| { + let length = export.name.len() as u32; + let mut found = FoundString::new( + export.name.clone(), + Encoding::Utf8, + 0, + length, + StringSource::ExportName, + ) + .with_tags(vec![Tag::Export]) + .with_rva(export.address); + + if ENTRY_POINTS.contains(found.text.as_str()) { + found.tags.push(Tag::EntryPoint); + } + + found + }) + .collect() + } + + /// Convert section names into standalone `FoundString`s. + /// + /// Each section name is emitted with [`StringSource::SectionName`] so it is + /// distinguishable from byte-scanned section content downstream. Empty + /// section names (e.g. PE sections with an all-null name field) are skipped + /// so no zero-length row reaches output. + #[must_use] + pub fn process_section_names(&self, sections: &[SectionInfo]) -> Vec { + sections + .iter() + .filter(|section| !section.name.is_empty()) + .map(|section| { + let length = section.name.len() as u32; + FoundString::new( + section.name.clone(), + Encoding::Utf8, + 0, + length, + StringSource::SectionName, + ) + }) + .collect() + } +} + +/// Return the semantic tag for an import name, if it matches a known API set. +/// +/// The crypto, network, and file-I/O sets are disjoint, so an import matches +/// at most one category. +fn semantic_tag(name: &str) -> Option { + if CRYPTO_APIS.contains(name) { + Some(Tag::Crypto) + } else if NETWORK_APIS.contains(name) { + Some(Tag::Network) + } else if FILEIO_APIS.contains(name) { + Some(Tag::FileIO) + } else { + None + } +} + +/// Format-appropriate default `section` for an import with no library name. +/// +/// Mach-O imports never carry a library (the parser leaves it `None`), so +/// Mach-O always uses this default. +fn default_import_section(format: BinaryFormat) -> &'static str { + match format { + BinaryFormat::Pe => ".idata", + BinaryFormat::Elf => ".dynsym", + BinaryFormat::MachO => "__LINKEDIT", + // Unknown formats fall back to raw scanning and carry no imports; + // use the ELF default as a neutral placeholder. + BinaryFormat::Unknown => ".dynsym", + } +} + +/// Convert all symbols and section names in a [`ContainerInfo`] into tagged +/// `FoundString`s. +/// +/// This is the single symbol emission point routed into the extraction path. +#[must_use] +pub fn extract_symbol_strings(container_info: &ContainerInfo) -> Vec { + let classifier = ImportClassifier::new(); + let mut strings = classifier.process_imports(&container_info.imports, container_info.format); + strings.extend(classifier.process_exports(&container_info.exports)); + // Section names are only meaningful for recognized container formats. The + // unknown/raw fallback uses a synthetic "raw-bytes" section that carries no + // analytic value (and would pollute empty-file output), so skip + // section-name rows for Unknown format. + if container_info.format != BinaryFormat::Unknown { + strings.extend(classifier.process_section_names(&container_info.sections)); + } + strings +} diff --git a/src/classification/mod.rs b/src/classification/mod.rs index 0e682a6..9deddb0 100644 --- a/src/classification/mod.rs +++ b/src/classification/mod.rs @@ -42,10 +42,12 @@ use std::sync::LazyLock; use crate::types::{BinaryFormat, SectionType, StringContext, StringSource, Tag}; +pub mod imports; pub mod patterns; pub mod ranking; pub mod symbols; +pub use imports::{ImportClassifier, extract_symbol_strings}; pub use ranking::{RankingConfig, RankingEngine}; pub use symbols::SymbolDemangler; diff --git a/src/classification/ranking.rs b/src/classification/ranking.rs index 7a1a9e2..54bc761 100644 --- a/src/classification/ranking.rs +++ b/src/classification/ranking.rs @@ -140,6 +140,10 @@ impl Default for RankingConfig { tag_boosts.insert(Tag::Base64, 10); tag_boosts.insert(Tag::FormatString, 15); tag_boosts.insert(Tag::UserAgent, 20); + tag_boosts.insert(Tag::Crypto, 50); + tag_boosts.insert(Tag::Network, 45); + tag_boosts.insert(Tag::FileIO, 35); + tag_boosts.insert(Tag::EntryPoint, 40); Self { section_weights, @@ -275,7 +279,7 @@ impl RankingEngine { /// otherwise ordering will reflect uninitialized or stale scores. This uses /// a stable sort, so the relative order of equal scores is preserved. pub fn rank_strings(&self, strings: &mut [FoundString]) { - strings.sort_by(|a, b| b.score.cmp(&a.score)); + strings.sort_by_key(|b| std::cmp::Reverse(b.score)); } } @@ -345,10 +349,29 @@ mod tests { assert_eq!(config.tag_boosts.get(&Tag::Base64), Some(&10)); assert_eq!(config.tag_boosts.get(&Tag::FormatString), Some(&15)); assert_eq!(config.tag_boosts.get(&Tag::UserAgent), Some(&20)); + assert_eq!(config.tag_boosts.get(&Tag::Crypto), Some(&50)); + assert_eq!(config.tag_boosts.get(&Tag::Network), Some(&45)); + assert_eq!(config.tag_boosts.get(&Tag::FileIO), Some(&35)); + assert_eq!(config.tag_boosts.get(&Tag::EntryPoint), Some(&40)); assert_eq!(config.noise_penalty_multiplier, 100); } + #[test] + fn test_crypto_tag_outranks_untagged_peer() { + let engine = RankingEngine::new(false); + let section_info = make_section_info(SectionType::StringData); + + let mut crypto = make_found_string(vec![Tag::Crypto], 1.0); + let mut untagged = make_found_string(vec![], 1.0); + engine.calculate_score(&mut crypto, Some(§ion_info)); + engine.calculate_score(&mut untagged, Some(§ion_info)); + + // The Crypto boost (+50) lifts the score at least 50 above an + // otherwise identical untagged string. + assert!(crypto.score >= untagged.score + 50); + } + #[test] fn test_score_calculation_high_value_url() { let engine = RankingEngine::new(true); diff --git a/src/extraction/ascii/extraction.rs b/src/extraction/ascii/extraction.rs index f8d101e..bc4fe72 100644 --- a/src/extraction/ascii/extraction.rs +++ b/src/extraction/ascii/extraction.rs @@ -33,7 +33,7 @@ use super::{AsciiExtractionConfig, is_printable_ascii}; /// /// Vector of FoundString entries with the following metadata: /// - `text`: UTF-8 string from accumulated bytes -/// - `encoding`: `Encoding::Ascii` +/// - `encoding`: `Encoding::Utf8` (ASCII is a UTF-8 subset; see KTD7) /// - `offset`: Start position in the data slice /// - `length`: Byte count /// - `source`: `StringSource::SectionData` @@ -96,7 +96,7 @@ pub fn extract_ascii_strings(data: &[u8], config: &AsciiExtractionConfig) -> Vec let text = String::from_utf8(bytes).expect("ASCII bytes should be valid UTF-8"); strings.push(FoundString::new( text, - Encoding::Ascii, + Encoding::Utf8, start as u64, len as u32, StringSource::SectionData, @@ -121,7 +121,7 @@ pub fn extract_ascii_strings(data: &[u8], config: &AsciiExtractionConfig) -> Vec let text = String::from_utf8(bytes).expect("ASCII bytes should be valid UTF-8"); strings.push(FoundString::new( text, - Encoding::Ascii, + Encoding::Utf8, start as u64, len as u32, StringSource::SectionData, @@ -132,7 +132,7 @@ pub fn extract_ascii_strings(data: &[u8], config: &AsciiExtractionConfig) -> Vec let text = String::from_utf8(bytes).expect("ASCII bytes should be valid UTF-8"); strings.push(FoundString::new( text, - Encoding::Ascii, + Encoding::Utf8, start as u64, len as u32, StringSource::SectionData, diff --git a/src/extraction/ascii/tests.rs b/src/extraction/ascii/tests.rs index 7aebfe7..b1f5d92 100644 --- a/src/extraction/ascii/tests.rs +++ b/src/extraction/ascii/tests.rs @@ -40,7 +40,8 @@ fn test_extract_ascii_strings_basic() { assert_eq!(strings.len(), 3); assert_eq!(strings[0].text, "Hello"); assert_eq!(strings[0].offset, 0); - assert_eq!(strings[0].encoding, Encoding::Ascii); + // ASCII content is emitted as UTF-8 (KTD7); ASCII is a UTF-8 subset. + assert_eq!(strings[0].encoding, Encoding::Utf8); assert_eq!(strings[0].source, StringSource::SectionData); assert_eq!(strings[1].text, "World"); assert_eq!(strings[1].offset, 6); diff --git a/src/extraction/basic_extractor.rs b/src/extraction/basic_extractor.rs index 2eb5643..e797574 100644 --- a/src/extraction/basic_extractor.rs +++ b/src/extraction/basic_extractor.rs @@ -4,6 +4,7 @@ //! and UTF-16 string extraction across binary sections, applying noise filtering, //! deduplication, and semantic enrichment. +use crate::classification::extract_symbol_strings; use crate::types::{ ContainerInfo, Encoding, FoundString, Result, SectionInfo, SectionType, StringSource, }; @@ -63,29 +64,14 @@ fn collect_all_strings( all_strings.extend(section_strings); } - // Include import/export symbols if configured + // Include import/export symbols and section names if configured. The + // classifier is the single emission point: it tags each symbol (Import/ + // Export plus semantic and entry-point tags) and emits section-name rows, + // so the ranking boosts fire on live output. Symbol names are emitted as + // UTF-8, so a byte-scanned occurrence of the same name (e.g. from PE + // .idata) shares the dedup key and merges into the single tagged row. if config.include_symbols { - for import in &container_info.imports { - let length = import.name.len() as u32; - all_strings.push(FoundString::new( - import.name.clone(), - Encoding::Utf8, - 0, - length, - StringSource::ImportName, - )); - } - - for export in &container_info.exports { - let length = export.name.len() as u32; - all_strings.push(FoundString::new( - export.name.clone(), - Encoding::Utf8, - 0, - length, - StringSource::ExportName, - )); - } + all_strings.extend(extract_symbol_strings(container_info)); } Ok(all_strings) diff --git a/src/extraction/dedup/mod.rs b/src/extraction/dedup/mod.rs index c4dc0ed..6a48aee 100644 --- a/src/extraction/dedup/mod.rs +++ b/src/extraction/dedup/mod.rs @@ -217,7 +217,7 @@ pub fn deduplicate( .collect(); // Sort by combined_score descending - canonical_strings.sort_by(|a, b| b.combined_score.cmp(&a.combined_score)); + canonical_strings.sort_by_key(|b| std::cmp::Reverse(b.combined_score)); canonical_strings } diff --git a/src/extraction/tests.rs b/src/extraction/tests.rs index b796ef0..163d87b 100644 --- a/src/extraction/tests.rs +++ b/src/extraction/tests.rs @@ -1,7 +1,7 @@ use super::*; use crate::types::{ BinaryFormat, ContainerInfo, Encoding, ExportInfo, ImportInfo, SectionInfo, SectionType, - StringSource, + StringSource, Tag, }; #[test] @@ -36,7 +36,8 @@ fn test_basic_extractor_extract_from_section() { assert_eq!(strings[0].offset, 0); assert_eq!(strings[0].rva, Some(0x1000)); assert_eq!(strings[0].section, Some(".rodata".to_string())); - assert_eq!(strings[0].encoding, Encoding::Ascii); + // ASCII content is now labeled UTF-8 (KTD7); the variant is no longer emitted. + assert_eq!(strings[0].encoding, Encoding::Utf8); assert_eq!(strings[1].text, "Test"); assert_eq!(strings[1].offset, 12); assert_eq!(strings[1].rva, Some(0x100C)); @@ -162,13 +163,14 @@ fn test_basic_extractor_encoding_filtering() { .extract_from_section(data, §ion, &config) .unwrap(); - // Should only find ASCII strings, not UTF-8 - // Note: "Hello" and "Test" are ASCII, "\u{4e16}\u{754c}" is UTF-8 and should be filtered + // Should only find narrow strings, not the wide UTF-8 content. + // "Hello" and "Test" are ASCII (emitted as UTF-8 per KTD7); the multibyte + // "\u{4e16}\u{754c}" is not scanned because only ASCII scanning is enabled. let ascii_strings: Vec<_> = strings .iter() - .filter(|s| s.encoding == Encoding::Ascii) + .filter(|s| s.encoding == Encoding::Utf8) .collect(); - assert_eq!(ascii_strings.len(), 2, "Should find 2 ASCII strings"); + assert_eq!(ascii_strings.len(), 2, "Should find 2 narrow strings"); assert!(ascii_strings.iter().any(|s| s.text == "Hello")); assert!(ascii_strings.iter().any(|s| s.text == "Test")); // UTF-8 string "\u{4e16}\u{754c}" should be filtered out @@ -271,21 +273,26 @@ fn test_basic_extractor_include_symbols() { assert!(export_strings.iter().any(|s| s.text == "main")); assert!(export_strings.iter().any(|s| s.text == "exported_function")); - // Verify import string properties + // Verify import string properties: the classifier tags imports, populates + // the RVA from the import address, and sets section to the library name. let printf_str = import_strings.iter().find(|s| s.text == "printf").unwrap(); assert_eq!(printf_str.encoding, Encoding::Utf8); assert_eq!(printf_str.offset, 0); - assert_eq!(printf_str.rva, None); - assert_eq!(printf_str.section, None); + assert_eq!(printf_str.rva, Some(0x1000)); + assert_eq!(printf_str.section, Some("libc.so.6".to_string())); assert_eq!(printf_str.length, 6); + assert!(printf_str.tags.contains(&Tag::Import)); - // Verify export string properties + // Verify export string properties: RVA from the export address, and `main` + // is recognized as an entry point. Exports carry no section. let main_str = export_strings.iter().find(|s| s.text == "main").unwrap(); assert_eq!(main_str.encoding, Encoding::Utf8); assert_eq!(main_str.offset, 0); - assert_eq!(main_str.rva, None); + assert_eq!(main_str.rva, Some(0x3000)); assert_eq!(main_str.section, None); assert_eq!(main_str.length, 4); + assert!(main_str.tags.contains(&Tag::Export)); + assert!(main_str.tags.contains(&Tag::EntryPoint)); } #[test] @@ -347,7 +354,12 @@ fn test_basic_extractor_section_filtering() { let strings = extractor.extract(data, &container_info, &config).unwrap(); - // Should only extract from data section, not code or debug - assert_eq!(strings.len(), 1); - assert_eq!(strings[0].text, "RoDataTest"); + // Section-name rows are emitted for every section (KTD6); section *content* + // is only scanned for the data section, not code or debug. + let content: Vec<_> = strings + .iter() + .filter(|s| s.source == StringSource::SectionData) + .collect(); + assert_eq!(content.len(), 1); + assert_eq!(content[0].text, "RoDataTest"); } diff --git a/src/main.rs b/src/main.rs index fb4f688..4ec4ba4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -47,7 +47,7 @@ fn parse_positive_usize(s: &str) -> Result { impl From for EncodingFilter { fn from(enc: CliEncoding) -> Self { match enc { - CliEncoding::Ascii => EncodingFilter::Exact(Encoding::Ascii), + CliEncoding::Ascii => EncodingFilter::AsciiContent, CliEncoding::Utf8 => EncodingFilter::Exact(Encoding::Utf8), CliEncoding::Utf16 => EncodingFilter::Utf16Any, CliEncoding::Utf16Le => EncodingFilter::Exact(Encoding::Utf16Le), @@ -108,7 +108,7 @@ struct Cli { long_help = "Include only strings with this tag. Repeat the flag for multiple tags \ (OR logic).\nValid tags: url, domain, ipv4, ipv6, filepath, regpath, guid, email, \ b64, fmt, user-agent-ish, demangled, import, export, version, manifest, resource, \ - dylib-path, rpath, rpath-var, framework-path" + dylib-path, rpath, rpath-var, framework-path, crypto, network, fileio, entry-point" )] only_tags: Vec, @@ -121,7 +121,7 @@ struct Cli { long_help = "Exclude strings with this tag. Repeat the flag for multiple tags \ (OR logic).\nValid tags: url, domain, ipv4, ipv6, filepath, regpath, guid, email, \ b64, fmt, user-agent-ish, demangled, import, export, version, manifest, resource, \ - dylib-path, rpath, rpath-var, framework-path" + dylib-path, rpath, rpath-var, framework-path, crypto, network, fileio, entry-point" )] no_tags: Vec, @@ -133,7 +133,7 @@ struct Cli { #[arg(short = 't', long, value_name = "N", value_parser = parse_positive_usize)] top: Option, - /// Filter by encoding [possible values: ascii, utf8, utf16, utf16le, utf16be] + /// Filter by encoding/content [possible values: ascii (narrow ASCII content), utf8, utf16, utf16le, utf16be] #[arg(long, value_enum, value_name = "ENCODING")] enc: Option, diff --git a/src/output/json.rs b/src/output/json.rs index cb0cca0..1ac333a 100644 --- a/src/output/json.rs +++ b/src/output/json.rs @@ -178,6 +178,10 @@ mod tests { Tag::Rpath, Tag::RpathVariable, Tag::FrameworkPath, + Tag::Crypto, + Tag::Network, + Tag::FileIO, + Tag::EntryPoint, ]; let strings = vec![make_string("tagged").with_tags(tags)]; let output = format_json(&strings, &make_metadata(1)).expect("Formatting should succeed"); @@ -211,6 +215,10 @@ mod tests { "rpath", "rpath-var", "framework-path", + "crypto", + "network", + "fileio", + "entry-point", ]; for name in expected { @@ -263,8 +271,15 @@ mod tests { 1, StringSource::DebugInfo, ), + FoundString::new( + "g".to_string(), + Encoding::Ascii, + 6, + 1, + StringSource::SectionName, + ), ]; - let output = format_json(&strings, &make_metadata(6)).expect("Formatting should succeed"); + let output = format_json(&strings, &make_metadata(7)).expect("Formatting should succeed"); let lines: Vec<&str> = output.lines().collect(); assert_eq!(parse_line(lines[0])["source"], "SectionData"); assert_eq!(parse_line(lines[1])["source"], "ImportName"); @@ -272,6 +287,7 @@ mod tests { assert_eq!(parse_line(lines[3])["source"], "ResourceString"); assert_eq!(parse_line(lines[4])["source"], "LoadCommand"); assert_eq!(parse_line(lines[5])["source"], "DebugInfo"); + assert_eq!(parse_line(lines[6])["source"], "SectionName"); } #[test] diff --git a/src/output/mod.rs b/src/output/mod.rs index 5d62932..43b80dc 100644 --- a/src/output/mod.rs +++ b/src/output/mod.rs @@ -199,7 +199,7 @@ impl OutputMetadata { } } let mut sorted: Vec<(Tag, usize)> = counts.into_iter().collect(); - sorted.sort_by(|a, b| b.1.cmp(&a.1)); + sorted.sort_by_key(|b| std::cmp::Reverse(b.1)); sorted.truncate(limit); sorted } diff --git a/src/output/table/formatting.rs b/src/output/table/formatting.rs index 3891eaf..9668689 100644 --- a/src/output/table/formatting.rs +++ b/src/output/table/formatting.rs @@ -100,6 +100,10 @@ pub(crate) fn tag_to_display_string(tag: &Tag) -> String { Tag::Rpath => "rpath".to_string(), Tag::RpathVariable => "rpath-var".to_string(), Tag::FrameworkPath => "framework-path".to_string(), + Tag::Crypto => "crypto".to_string(), + Tag::Network => "network".to_string(), + Tag::FileIO => "fileio".to_string(), + Tag::EntryPoint => "entry-point".to_string(), } } @@ -243,6 +247,10 @@ mod tests { Tag::Rpath, Tag::RpathVariable, Tag::FrameworkPath, + Tag::Crypto, + Tag::Network, + Tag::FileIO, + Tag::EntryPoint, ]; for tag in all_tags { diff --git a/src/output/yara/mod.rs b/src/output/yara/mod.rs index 5043278..e42fbf7 100644 --- a/src/output/yara/mod.rs +++ b/src/output/yara/mod.rs @@ -184,6 +184,10 @@ fn tag_name(tag: &Tag) -> &'static str { Tag::Rpath => "rpath", Tag::RpathVariable => "rpath-var", Tag::FrameworkPath => "framework-path", + Tag::Crypto => "crypto", + Tag::Network => "network", + Tag::FileIO => "fileio", + Tag::EntryPoint => "entry-point", } } diff --git a/src/pipeline/config.rs b/src/pipeline/config.rs index a69ce8f..a32f561 100644 --- a/src/pipeline/config.rs +++ b/src/pipeline/config.rs @@ -15,6 +15,14 @@ pub enum EncodingFilter { Exact(Encoding), /// Match both `Encoding::Utf16Le` and `Encoding::Utf16Be`. Utf16Any, + /// Match narrow strings whose text content is pure-ASCII. + /// + /// The ASCII extraction path no longer labels rows `Encoding::Ascii` + /// (ASCII content is emitted as `Encoding::Utf8`), so `--enc ascii` + /// filters by content: + /// the text is pure-ASCII and the stored encoding is not UTF-16. This + /// preserves the flag's pre-existing meaning of "narrow ASCII strings". + AsciiContent, } /// Configuration for post-extraction string filtering. diff --git a/src/pipeline/filter.rs b/src/pipeline/filter.rs index 7828bc8..6a16abc 100644 --- a/src/pipeline/filter.rs +++ b/src/pipeline/filter.rs @@ -45,6 +45,13 @@ impl FilterEngine { Some(EncodingFilter::Utf16Any) => { s.encoding == Encoding::Utf16Le || s.encoding == Encoding::Utf16Be } + Some(EncodingFilter::AsciiContent) => { + // Cheap O(1) encoding guards short-circuit before the O(n) + // is_ascii() scan, so UTF-16 rows skip the content check. + s.encoding != Encoding::Utf16Le + && s.encoding != Encoding::Utf16Be + && s.text.is_ascii() + } }) // 3. include-tags .filter(|s| include_set.is_empty() || s.tags.iter().any(|t| include_set.contains(t))) @@ -120,6 +127,28 @@ mod tests { assert_eq!(result[0].text, "wide"); } + #[test] + fn test_ascii_content_filter_matches_narrow_ascii_text() { + // Narrow ASCII content (now labeled Utf8 per KTD7) passes. + let mut narrow = make_string("CreateFileW", 10, 0); + narrow.encoding = Encoding::Utf8; + // UTF-8 content with non-ASCII characters is excluded. + let mut wide_utf8 = make_string("caf\u{e9}", 10, 10); + wide_utf8.encoding = Encoding::Utf8; + // A UTF-16 row that happens to decode to ASCII is still excluded, + // preserving the flag's pre-existing "narrow strings only" meaning. + let mut utf16_ascii = make_string("hello", 10, 20); + utf16_ascii.encoding = Encoding::Utf16Le; + + let config = FilterConfig::new() + .with_min_length(1) + .with_encoding(EncodingFilter::AsciiContent); + let result = FilterEngine::new().apply(vec![narrow, wide_utf8, utf16_ascii], &config); + + assert_eq!(result.len(), 1); + assert_eq!(result[0].text, "CreateFileW"); + } + #[test] fn test_encoding_utf16any_filter() { let mut s_le = make_string("le", 10, 0); diff --git a/src/types/mod.rs b/src/types/mod.rs index aededfa..421ec2e 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -55,6 +55,14 @@ pub enum Tag { RpathVariable, #[serde(rename = "framework-path")] FrameworkPath, + #[serde(rename = "crypto")] + Crypto, + #[serde(rename = "network")] + Network, + #[serde(rename = "fileio")] + FileIO, + #[serde(rename = "entry-point")] + EntryPoint, } impl std::str::FromStr for Tag { @@ -83,6 +91,10 @@ impl std::str::FromStr for Tag { "rpath" => Ok(Tag::Rpath), "rpath-var" => Ok(Tag::RpathVariable), "framework-path" => Ok(Tag::FrameworkPath), + "crypto" => Ok(Tag::Crypto), + "network" => Ok(Tag::Network), + "fileio" => Ok(Tag::FileIO), + "entry-point" => Ok(Tag::EntryPoint), _ => Err(format!("unknown tag: {s}")), } } @@ -108,10 +120,13 @@ pub enum SectionType { } /// Source of a string within the binary +#[non_exhaustive] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum StringSource { /// String found in section data SectionData, + /// Section name emitted as a standalone string + SectionName, /// String from import table ImportName, /// String from export table diff --git a/src/types/tests.rs b/src/types/tests.rs index d1a2940..be039a3 100644 --- a/src/types/tests.rs +++ b/src/types/tests.rs @@ -145,6 +145,10 @@ fn test_tag_from_str_all_variants() { ("rpath", Tag::Rpath), ("rpath-var", Tag::RpathVariable), ("framework-path", Tag::FrameworkPath), + ("crypto", Tag::Crypto), + ("network", Tag::Network), + ("fileio", Tag::FileIO), + ("entry-point", Tag::EntryPoint), ]; for (input, expected) in cases { diff --git a/tests/classification_symbols.rs b/tests/classification_symbols.rs new file mode 100644 index 0000000..0ee1edd --- /dev/null +++ b/tests/classification_symbols.rs @@ -0,0 +1,302 @@ +//! Symbol classification tests for the import/export pipeline. +//! +//! Provenance assertions (section, source, rva) run against hand-built +//! `ContainerInfo` values so no byte-scan occurrence is in play -- per the +//! plan's KTD1, a symbol's source/section/rva are only deterministic when the +//! classifier is the sole emitter. + +use std::fs; + +use stringy::classification::{ImportClassifier, SymbolDemangler, extract_symbol_strings}; +use stringy::container::{ContainerParser, ElfParser, MachoParser, PeParser}; +use stringy::types::{ + BinaryFormat, ExportInfo, FoundString, ImportInfo, SectionInfo, SectionType, StringSource, Tag, +}; + +fn get_fixture_path(name: &str) -> std::path::PathBuf { + std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests") + .join("fixtures") + .join(name) +} + +fn find<'a>(strings: &'a [FoundString], text: &str) -> &'a FoundString { + strings + .iter() + .find(|s| s.text == text) + .unwrap_or_else(|| panic!("expected a string with text {text:?}")) +} + +// --- Imports (R1, R2, R3, R4; AE1, AE3) -------------------------------------- + +#[test] +fn every_import_carries_import_tag_source_and_full_confidence() { + let classifier = ImportClassifier::new(); + let imports = vec![ + ImportInfo::new("printf".to_string()), + ImportInfo::new("connect".to_string()), + ImportInfo::new("CreateFileW".to_string()), + ]; + + let strings = classifier.process_imports(&imports, BinaryFormat::Pe); + + assert_eq!(strings.len(), 3); + for s in &strings { + assert!( + s.tags.contains(&Tag::Import), + "import must carry Tag::Import" + ); + assert_eq!(s.source, StringSource::ImportName); + assert_eq!(s.confidence, 1.0); + } +} + +#[test] +fn imports_gain_matching_semantic_tags() { + let classifier = ImportClassifier::new(); + let imports = vec![ + ImportInfo::new("CreateFileW".to_string()), + ImportInfo::new("connect".to_string()), + ImportInfo::new("EVP_EncryptInit".to_string()), + ImportInfo::new("printf".to_string()), + ]; + + let strings = classifier.process_imports(&imports, BinaryFormat::Pe); + + let create_file = find(&strings, "CreateFileW"); + assert!(create_file.tags.contains(&Tag::Import)); + assert!(create_file.tags.contains(&Tag::FileIO)); + + let connect = find(&strings, "connect"); + assert!(connect.tags.contains(&Tag::Import)); + assert!(connect.tags.contains(&Tag::Network)); + + let crypto = find(&strings, "EVP_EncryptInit"); + assert!(crypto.tags.contains(&Tag::Import)); + assert!(crypto.tags.contains(&Tag::Crypto)); + + // printf matches no semantic set: Import only. + let printf = find(&strings, "printf"); + assert_eq!(printf.tags, vec![Tag::Import]); +} + +#[test] +fn import_section_uses_library_then_format_default() { + let classifier = ImportClassifier::new(); + + // Library present: section is the library name. + let with_lib = classifier.process_imports( + &[ImportInfo::new("CreateFileW".to_string()).with_library("kernel32.dll".to_string())], + BinaryFormat::Pe, + ); + assert_eq!(with_lib[0].section.as_deref(), Some("kernel32.dll")); + + // No library: format-appropriate default. + let pe = classifier.process_imports(&[ImportInfo::new("f".to_string())], BinaryFormat::Pe); + assert_eq!(pe[0].section.as_deref(), Some(".idata")); + + let elf = classifier.process_imports(&[ImportInfo::new("f".to_string())], BinaryFormat::Elf); + assert_eq!(elf[0].section.as_deref(), Some(".dynsym")); + + // Mach-O imports are always library-less. + let macho = + classifier.process_imports(&[ImportInfo::new("f".to_string())], BinaryFormat::MachO); + assert_eq!(macho[0].section.as_deref(), Some("__LINKEDIT")); +} + +#[test] +fn import_rva_populates_only_when_address_present() { + let classifier = ImportClassifier::new(); + let imports = vec![ + ImportInfo::new("with_addr".to_string()).with_address(0x4010), + ImportInfo::new("no_addr".to_string()), + ]; + + let strings = classifier.process_imports(&imports, BinaryFormat::Elf); + + assert_eq!(find(&strings, "with_addr").rva, Some(0x4010)); + assert_eq!(find(&strings, "no_addr").rva, None); +} + +// --- Exports (R5, R6, R7; AE2) ----------------------------------------------- + +#[test] +fn every_export_carries_export_tag_source_and_rva() { + let classifier = ImportClassifier::new(); + let exports = vec![ + ExportInfo::new("alpha".to_string(), 0x1000), + ExportInfo::new("beta".to_string(), 0x2000), + ]; + + let strings = classifier.process_exports(&exports); + + assert_eq!(strings.len(), 2); + for s in &strings { + assert!(s.tags.contains(&Tag::Export)); + assert_eq!(s.source, StringSource::ExportName); + assert_eq!(s.confidence, 1.0); + } + assert_eq!(find(&strings, "alpha").rva, Some(0x1000)); + assert_eq!(find(&strings, "beta").rva, Some(0x2000)); +} + +#[test] +fn mangled_export_emitted_raw_for_pipeline_demangling() { + // The classifier tags exports but does NOT demangle. Demangling is the + // pipeline's job (classify_strings: under catch_unwind, skipped in raw + // mode), so a third-party demangler panic never aborts extraction and raw + // mode shows the untouched symbol name. + let classifier = ImportClassifier::new(); + let exports = vec![ExportInfo::new("_ZN3foo3barEv".to_string(), 0x3000)]; + + let strings = classifier.process_exports(&exports); + let export = &strings[0]; + + assert!(export.tags.contains(&Tag::Export)); + assert!(!export.tags.contains(&Tag::DemangledSymbol)); + assert_eq!(export.text, "_ZN3foo3barEv"); + assert_eq!(export.original_text, None); + + // The emitted string is in a demangleable state: the pipeline's + // SymbolDemangler produces Export + DemangledSymbol + demangled text (AE2). + let mut downstream = strings[0].clone(); + SymbolDemangler::new().demangle(&mut downstream); + assert!(downstream.tags.contains(&Tag::Export)); + assert!(downstream.tags.contains(&Tag::DemangledSymbol)); + assert!(downstream.text.contains("foo") && downstream.text.contains("bar")); +} + +#[test] +fn entry_point_exports_gain_entry_point_tag() { + let classifier = ImportClassifier::new(); + let exports = vec![ + ExportInfo::new("main".to_string(), 0x1000), + ExportInfo::new("DllMain".to_string(), 0x2000), + ExportInfo::new("_start".to_string(), 0x3000), + ExportInfo::new("WinMain".to_string(), 0x4000), + ExportInfo::new("wWinMain".to_string(), 0x5000), + ExportInfo::new("ordinary_export".to_string(), 0x6000), + ]; + + let strings = classifier.process_exports(&exports); + + for name in ["main", "DllMain", "_start", "WinMain", "wWinMain"] { + let s = find(&strings, name); + assert!( + s.tags.contains(&Tag::EntryPoint), + "{name} must be an entry point" + ); + } + + let ordinary = find(&strings, "ordinary_export"); + assert!(!ordinary.tags.contains(&Tag::EntryPoint)); + assert!(!ordinary.tags.contains(&Tag::DemangledSymbol)); + assert_eq!(ordinary.tags, vec![Tag::Export]); +} + +// --- Section names (R8) ------------------------------------------------------ + +#[test] +fn section_names_emit_as_section_name_source() { + let classifier = ImportClassifier::new(); + let sections = vec![ + SectionInfo::new(".text".to_string(), 0, 100, SectionType::Code, 1.0), + SectionInfo::new( + ".rodata".to_string(), + 100, + 100, + SectionType::StringData, + 1.0, + ), + // Empty-named section (e.g. PE all-null name) must be skipped. + SectionInfo::new(String::new(), 200, 10, SectionType::Other, 1.0), + ]; + + let strings = classifier.process_section_names(§ions); + + assert_eq!(strings.len(), 2); + for s in &strings { + assert_eq!(s.source, StringSource::SectionName); + assert_eq!(s.confidence, 1.0); + assert!(!s.text.is_empty()); + } + assert!(strings.iter().any(|s| s.text == ".text")); + assert!(strings.iter().any(|s| s.text == ".rodata")); +} + +#[test] +fn unknown_format_skips_section_name_rows() { + // The unknown/raw fallback uses a synthetic "raw-bytes" section; its name + // carries no analytic value and must not be emitted as a standalone row. + let info = stringy::types::ContainerInfo::new( + BinaryFormat::Unknown, + vec![SectionInfo::new( + "raw-bytes".to_string(), + 0, + 10, + SectionType::Other, + 1.0, + )], + vec![], + vec![], + None, + ); + + let strings = extract_symbol_strings(&info); + assert!( + !strings + .iter() + .any(|s| s.source == StringSource::SectionName), + "Unknown format must not emit section-name rows" + ); +} + +// --- End-to-end against fixtures (R13) --------------------------------------- + +fn assert_symbol_contract(strings: &[FoundString]) { + // Section names are always present (every binary has sections). + assert!( + strings + .iter() + .any(|s| s.source == StringSource::SectionName), + "extract_symbol_strings must emit section-name rows" + ); + // Any import/export row carries the matching always-on tag. + for s in strings { + match s.source { + StringSource::ImportName => assert!(s.tags.contains(&Tag::Import)), + StringSource::ExportName => assert!(s.tags.contains(&Tag::Export)), + _ => {} + } + } +} + +#[test] +fn extract_symbol_strings_on_elf_fixture() { + let data = fs::read(get_fixture_path("test_binary_elf")) + .expect("read ELF fixture (run `just gen-fixtures`)"); + assert!(ElfParser::detect(&data)); + let info = ElfParser::new().parse(&data).expect("parse ELF"); + let strings = extract_symbol_strings(&info); + assert_symbol_contract(&strings); +} + +#[test] +fn extract_symbol_strings_on_pe_fixture() { + let data = fs::read(get_fixture_path("test_binary_pe.exe")) + .expect("read PE fixture (run `just gen-fixtures`)"); + assert!(PeParser::detect(&data)); + let info = PeParser::new().parse(&data).expect("parse PE"); + let strings = extract_symbol_strings(&info); + assert_symbol_contract(&strings); +} + +#[test] +fn extract_symbol_strings_on_macho_fixture() { + let data = fs::read(get_fixture_path("test_binary_macho")) + .expect("read Mach-O fixture (run `just gen-fixtures`)"); + assert!(MachoParser::detect(&data)); + let info = MachoParser::new().parse(&data).expect("parse Mach-O"); + let strings = extract_symbol_strings(&info); + assert_symbol_contract(&strings); +} diff --git a/tests/integration_cli.rs b/tests/integration_cli.rs index 2b84057..28f11c5 100644 --- a/tests/integration_cli.rs +++ b/tests/integration_cli.rs @@ -287,7 +287,11 @@ fn cli_help_lists_all_canonical_tags() { .stdout(predicate::str::contains("dylib-path")) .stdout(predicate::str::contains("rpath")) .stdout(predicate::str::contains("rpath-var")) - .stdout(predicate::str::contains("framework-path")); + .stdout(predicate::str::contains("framework-path")) + .stdout(predicate::str::contains("crypto")) + .stdout(predicate::str::contains("network")) + .stdout(predicate::str::contains("fileio")) + .stdout(predicate::str::contains("entry-point")); } #[test] diff --git a/tests/integration_cli_coverage.rs b/tests/integration_cli_coverage.rs index 08376c9..bf7caa1 100644 --- a/tests/integration_cli_coverage.rs +++ b/tests/integration_cli_coverage.rs @@ -61,6 +61,29 @@ fn encoding_filter_ascii() { .success(); } +#[test] +fn encoding_filter_ascii_is_content_based() { + // --enc ascii is a content filter (R16): every emitted row's text must be + // pure-ASCII, regardless of stored encoding. Extraction labels ASCII + // content as Utf8, so this asserts content, not the encoding field. + let assert = stringy() + .args(["tests/fixtures/test_binary_elf", "--enc", "ascii", "--json"]) + .assert() + .success(); + let stdout = String::from_utf8(assert.get_output().stdout.clone()).unwrap(); + let mut rows = 0; + for line in stdout.lines().filter(|l| !l.is_empty()) { + let v: serde_json::Value = serde_json::from_str(line).expect("valid JSON line"); + let text = v["text"].as_str().expect("text field"); + assert!( + text.is_ascii(), + "--enc ascii must only emit pure-ASCII text, got: {text:?}" + ); + rows += 1; + } + assert!(rows > 0, "ELF fixture should yield some ASCII rows"); +} + #[test] fn encoding_filter_invalid() { stringy() diff --git a/tests/integration_extraction.rs b/tests/integration_extraction.rs index 42170f4..81a1831 100644 --- a/tests/integration_extraction.rs +++ b/tests/integration_extraction.rs @@ -1,7 +1,7 @@ use std::fs; use stringy::container::{ContainerParser, ElfParser, PeParser}; use stringy::extraction::{BasicExtractor, ExtractionConfig, StringExtractor}; -use stringy::types::{Encoding, SectionType, StringSource}; +use stringy::types::{Encoding, SectionType, StringSource, Tag}; fn get_fixture_path(name: &str) -> std::path::PathBuf { std::path::Path::new(env!("CARGO_MANIFEST_DIR")) @@ -32,7 +32,8 @@ fn test_basic_extractor_ascii_strings() { assert_eq!(strings.len(), 3); assert_eq!(strings[0].text, "Hello"); - assert_eq!(strings[0].encoding, Encoding::Ascii); + // ASCII content is now labeled UTF-8 (KTD7); the variant is no longer emitted. + assert_eq!(strings[0].encoding, Encoding::Utf8); assert_eq!(strings[0].source, StringSource::SectionData); assert_eq!(strings[1].text, "World"); assert_eq!(strings[2].text, "Test123"); @@ -384,12 +385,12 @@ fn test_basic_extractor_encoding_filtering() { .extract_from_section(data, §ion, &config) .unwrap(); - // Should only find ASCII strings, not UTF-8 + // Should only find narrow strings; ASCII content is labeled UTF-8 (KTD7). assert_eq!(strings.len(), 2); assert_eq!(strings[0].text, "Hello"); - assert_eq!(strings[0].encoding, Encoding::Ascii); + assert_eq!(strings[0].encoding, Encoding::Utf8); assert_eq!(strings[1].text, "Test"); - assert_eq!(strings[1].encoding, Encoding::Ascii); + assert_eq!(strings[1].encoding, Encoding::Utf8); // UTF-8 string "世界" should be filtered out assert!(!strings.iter().any(|s| s.text.contains("世界"))); } @@ -423,22 +424,23 @@ fn test_basic_extractor_include_symbols() { // Verify we found some imports/exports assert!(!import_strings.is_empty() || !export_strings.is_empty()); - // Verify import string properties + // Imports are tagged and carry a section (library name or format default). for import_str in &import_strings { assert_eq!(import_str.encoding, Encoding::Utf8); assert_eq!(import_str.offset, 0); - assert_eq!(import_str.rva, None); - assert_eq!(import_str.section, None); + assert!(import_str.section.is_some()); assert!(import_str.length > 0); + assert!(import_str.tags.contains(&Tag::Import)); } - // Verify export string properties + // Exports are tagged, always carry an RVA (address is required), no section. for export_str in &export_strings { assert_eq!(export_str.encoding, Encoding::Utf8); assert_eq!(export_str.offset, 0); - assert_eq!(export_str.rva, None); + assert!(export_str.rva.is_some()); assert_eq!(export_str.section, None); assert!(export_str.length > 0); + assert!(export_str.tags.contains(&Tag::Export)); } } diff --git a/tests/output_yara_integration.rs b/tests/output_yara_integration.rs index e8cf416..272ab52 100644 --- a/tests/output_yara_integration.rs +++ b/tests/output_yara_integration.rs @@ -174,6 +174,10 @@ fn test_yara_all_tag_types() { make_string("rpath").with_tags(vec![Tag::Rpath]), make_string("rpathvar").with_tags(vec![Tag::RpathVariable]), make_string("framework").with_tags(vec![Tag::FrameworkPath]), + make_string("crypto").with_tags(vec![Tag::Crypto]), + make_string("network").with_tags(vec![Tag::Network]), + make_string("fileio").with_tags(vec![Tag::FileIO]), + make_string("entrypoint").with_tags(vec![Tag::EntryPoint]), ]; let output = format_yara(&strings, &make_metadata("tags.exe", strings.len())).unwrap(); assert_snapshot!(output); diff --git a/tests/snapshots/integration_flows_1_5__flow1_top3_json.snap b/tests/snapshots/integration_flows_1_5__flow1_top3_json.snap index cb91372..ed73fb2 100644 --- a/tests/snapshots/integration_flows_1_5__flow1_top3_json.snap +++ b/tests/snapshots/integration_flows_1_5__flow1_top3_json.snap @@ -2,6 +2,6 @@ source: tests/integration_flows_1_5.rs expression: stdout --- -{"text":"https://github.com/EvilBit-Labs/Stringy","encoding":"Ascii","offset":5008,"rva":16782224,"section":".rodata","length":39,"tags":["Url"],"score":1050,"display_score":100,"source":"SectionData","confidence":1.0} -{"text":"/proc/self/exe","encoding":"Ascii","offset":3747,"rva":16780963,"section":".rodata","length":14,"tags":["filepath"],"score":1022,"display_score":100,"source":"SectionData","confidence":0.865} -{"text":"/usr/lib/debug","encoding":"Ascii","offset":4213,"rva":16781429,"section":".rodata","length":14,"tags":["filepath"],"score":1022,"display_score":100,"source":"SectionData","confidence":0.865} +{"text":"https://github.com/EvilBit-Labs/Stringy","encoding":"Utf8","offset":5008,"rva":16782224,"section":".rodata","length":39,"tags":["Url"],"score":1050,"display_score":100,"source":"SectionData","confidence":1.0} +{"text":"/proc/self/exe","encoding":"Utf8","offset":3747,"rva":16780963,"section":".rodata","length":14,"tags":["filepath"],"score":1022,"display_score":100,"source":"SectionData","confidence":0.865} +{"text":"/usr/lib/debug","encoding":"Utf8","offset":4213,"rva":16781429,"section":".rodata","length":14,"tags":["filepath"],"score":1022,"display_score":100,"source":"SectionData","confidence":0.865} diff --git a/tests/snapshots/output_yara_integration__yara_all_tag_types.snap b/tests/snapshots/output_yara_integration__yara_all_tag_types.snap index 6b5cb58..44b8d83 100644 --- a/tests/snapshots/output_yara_integration__yara_all_tag_types.snap +++ b/tests/snapshots/output_yara_integration__yara_all_tag_types.snap @@ -39,12 +39,21 @@ rule tags_exe_strings { // tag: b64 // score: 0 $b64_1 = "b64" ascii + // tag: crypto + // score: 0 + $crypto_1 = "crypto" ascii // tag: demangled // score: 0 $demangled_1 = "demangled" ascii // tag: dylib-path // score: 0 $dylib_path_1 = "dylib" ascii + // tag: entry-point + // score: 0 + $entry_point_1 = "entrypoint" ascii + // tag: fileio + // score: 0 + $fileio_1 = "fileio" ascii // tag: filepath // score: 0 $filepath_1 = "path" ascii @@ -63,6 +72,9 @@ rule tags_exe_strings { // tag: ipv6 // score: 0 $ipv6_1 = "ipv6" ascii + // tag: network + // score: 0 + $network_1 = "network" ascii // tag: regpath // score: 0 $regpath_1 = "reg" ascii diff --git a/tests/test_ascii_extraction.rs b/tests/test_ascii_extraction.rs index fa11d68..2b5d9a1 100644 --- a/tests/test_ascii_extraction.rs +++ b/tests/test_ascii_extraction.rs @@ -14,7 +14,8 @@ fn test_basic_extraction() { assert_eq!(strings.len(), 3); assert_eq!(strings[0].text, "Hello"); assert_eq!(strings[0].offset, 0); - assert_eq!(strings[0].encoding, Encoding::Ascii); + // ASCII content is emitted as UTF-8 (KTD7); ASCII is a UTF-8 subset. + assert_eq!(strings[0].encoding, Encoding::Utf8); assert_eq!(strings[0].source, StringSource::SectionData); assert_eq!(strings[0].confidence, 1.0); } diff --git a/tests/test_deduplication.rs b/tests/test_deduplication.rs index 87d945b..2289e6a 100644 --- a/tests/test_deduplication.rs +++ b/tests/test_deduplication.rs @@ -45,13 +45,15 @@ fn test_deduplication_with_basic_extractor() { assert!(strings.len() >= 3); let hello_count = strings.iter().filter(|s| s.text == "Hello").count(); assert!(hello_count >= 2, "Should have at least 2 'Hello' strings"); + let raw_count = strings.len(); // Apply deduplication let canonical = deduplicate(strings, None); - // Verify deduplication reduced count + // Verify deduplication reduced the count (duplicate content collapses; + // section-name rows are unique and remain). assert!( - canonical.len() < 6, + canonical.len() < raw_count, "Deduplication should reduce string count" );