Skip to content

AST node layout: shrink Inline/Block/Attr/text storage (epic bd-4x6kih4c) #1192

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
cscheid wants to merge 7 commits into
base: main
Choose a base branch
from
+4,283 −2,708
Draft

AST node layout: shrink Inline/Block/Attr/text storage (epic bd-4x6kih4c) #1192

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
bc0cf18
perf-harness: add ast-memory census tool and AST string-storage study…
Nov 12, 2026
f2878b3
docs(plans): link AST layout follow-up strands in the string-storage …
Nov 12, 2026
248de55
docs(plans): record ecow-first, Lua-heavy benchmark, and wasm32 track…
Nov 12, 2026
3c8b3d9
Box cold Inline variants: Inline is 64 bytes, down from 360 (bd-xrfun…
Nov 12, 2026
13c98d9
Merge bd-xrfun6mn: box cold Inline variants
Nov 12, 2026
d9cb400
Box cold Block variants: Block is 64 bytes, down from 824 (bd-09g8zl30)
Nov 12, 2026
f8b11e3
Merge bd-09g8zl30: box cold Block variants
Nov 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

517 changes: 517 additions & 0 deletions claude-notes/plans/2026-11-12-ast-string-storage-study.md
View file
Open in desktop

Large diffs are not rendered by default.

16 changes: 8 additions & 8 deletions crates/comrak-to-pandoc/src/block.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,11 +133,11 @@ fn convert_block<'a>(

// Raw HTML, as pandoc's commonmark reader (raw_html on) produces it.
NodeValue::HtmlBlock(html) => {
vec![Block::RawBlock(RawBlock {
vec![Block::RawBlock(Box::new(RawBlock {
format: "html".to_string(),
text: html.literal.clone(),
source_info,
})]
}))]
}

// Unsupported block types - panic as they're outside CommonMark.
Expand Down Expand Up @@ -197,13 +197,13 @@ fn convert_heading<'a>(
let ast = node.data.borrow();
let source_info = get_source_info(&ast, source_ctx);
let inlines = convert_children_to_inlines_with_source(node, source_ctx);
Block::Header(Header {
Block::Header(Box::new(Header {
level: heading.level as usize,
attr: empty_attr(),
content: inlines,
source_info,
attr_source: AttrSourceInfo::empty(),
})
}))
}

fn convert_code_block(code_block: &NodeCodeBlock, source_info: SourceInfo) -> Block {
Expand All @@ -223,12 +223,12 @@ fn convert_code_block(code_block: &NodeCodeBlock, source_info: SourceInfo) -> Bl
]
};

Block::CodeBlock(CodeBlock {
Block::CodeBlock(Box::new(CodeBlock {
attr: (String::new(), classes, LinkedHashMap::new()),
text: code_block.literal.clone(),
source_info,
attr_source: AttrSourceInfo::empty(),
})
}))
}

fn convert_list<'a>(
Expand Down Expand Up @@ -258,11 +258,11 @@ fn convert_list<'a>(
ListDelimType::Paren => ListNumberDelim::OneParen,
},
);
Block::OrderedList(OrderedList {
Block::OrderedList(Box::new(OrderedList {
attr,
content: items,
source_info,
})
}))
}
}
}
Expand Down
96 changes: 48 additions & 48 deletions crates/comrak-to-pandoc/src/compare.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -215,41 +215,41 @@ mod tests {
fn test_header_eq() {
use hashlink::LinkedHashMap;

let a = Block::Header(Header {
let a = Block::Header(Box::new(Header {
level: 1,
attr: (String::new(), vec![], LinkedHashMap::new()),
content: vec![],
source_info: dummy_source(),
attr_source: quarto_pandoc_types::AttrSourceInfo::empty(),
});
let b = Block::Header(Header {
}));
let b = Block::Header(Box::new(Header {
level: 1,
attr: (String::new(), vec![], LinkedHashMap::new()),
content: vec![],
source_info: other_source(),
attr_source: quarto_pandoc_types::AttrSourceInfo::empty(),
});
}));
assert!(block_eq(&a, &b));
}

#[test]
fn test_header_neq_different_level() {
use hashlink::LinkedHashMap;

let a = Block::Header(Header {
let a = Block::Header(Box::new(Header {
level: 1,
attr: (String::new(), vec![], LinkedHashMap::new()),
content: vec![],
source_info: dummy_source(),
attr_source: quarto_pandoc_types::AttrSourceInfo::empty(),
});
let b = Block::Header(Header {
}));
let b = Block::Header(Box::new(Header {
level: 2,
attr: (String::new(), vec![], LinkedHashMap::new()),
content: vec![],
source_info: dummy_source(),
attr_source: quarto_pandoc_types::AttrSourceInfo::empty(),
});
}));
assert!(!block_eq(&a, &b));
}

Expand Down Expand Up @@ -347,127 +347,127 @@ mod tests {
#[test]
fn test_code_inline_eq() {
use hashlink::LinkedHashMap;
let a = Inline::Code(quarto_pandoc_types::Code {
let a = Inline::Code(Box::new(quarto_pandoc_types::Code {
attr: (String::new(), vec![], LinkedHashMap::new()),
text: "code".to_string(),
source_info: dummy_source(),
attr_source: quarto_pandoc_types::AttrSourceInfo::empty(),
});
let b = Inline::Code(quarto_pandoc_types::Code {
}));
let b = Inline::Code(Box::new(quarto_pandoc_types::Code {
attr: (String::new(), vec![], LinkedHashMap::new()),
text: "code".to_string(),
source_info: other_source(),
attr_source: quarto_pandoc_types::AttrSourceInfo::empty(),
});
}));
assert!(inline_eq(&a, &b));
}

#[test]
fn test_link_eq() {
use hashlink::LinkedHashMap;
let a = Inline::Link(quarto_pandoc_types::Link {
let a = Inline::Link(Box::new(quarto_pandoc_types::Link {
attr: (String::new(), vec![], LinkedHashMap::new()),
content: vec![],
target: ("url".to_string(), "title".to_string()),
source_info: dummy_source(),
attr_source: quarto_pandoc_types::AttrSourceInfo::empty(),
target_source: quarto_pandoc_types::TargetSourceInfo::empty(),
});
let b = Inline::Link(quarto_pandoc_types::Link {
}));
let b = Inline::Link(Box::new(quarto_pandoc_types::Link {
attr: (String::new(), vec![], LinkedHashMap::new()),
content: vec![],
target: ("url".to_string(), "title".to_string()),
source_info: other_source(),
attr_source: quarto_pandoc_types::AttrSourceInfo::empty(),
target_source: quarto_pandoc_types::TargetSourceInfo::empty(),
});
}));
assert!(inline_eq(&a, &b));
}

#[test]
fn test_image_eq() {
use hashlink::LinkedHashMap;
let a = Inline::Image(quarto_pandoc_types::Image {
let a = Inline::Image(Box::new(quarto_pandoc_types::Image {
attr: (String::new(), vec![], LinkedHashMap::new()),
content: vec![],
target: ("img.png".to_string(), "alt".to_string()),
source_info: dummy_source(),
attr_source: quarto_pandoc_types::AttrSourceInfo::empty(),
target_source: quarto_pandoc_types::TargetSourceInfo::empty(),
});
let b = Inline::Image(quarto_pandoc_types::Image {
}));
let b = Inline::Image(Box::new(quarto_pandoc_types::Image {
attr: (String::new(), vec![], LinkedHashMap::new()),
content: vec![],
target: ("img.png".to_string(), "alt".to_string()),
source_info: other_source(),
attr_source: quarto_pandoc_types::AttrSourceInfo::empty(),
target_source: quarto_pandoc_types::TargetSourceInfo::empty(),
});
}));
assert!(inline_eq(&a, &b));
}

#[test]
fn test_span_eq() {
use hashlink::LinkedHashMap;
let a = Inline::Span(quarto_pandoc_types::Span {
let a = Inline::Span(Box::new(quarto_pandoc_types::Span {
attr: (String::new(), vec![], LinkedHashMap::new()),
content: vec![],
source_info: dummy_source(),
attr_source: quarto_pandoc_types::AttrSourceInfo::empty(),
});
let b = Inline::Span(quarto_pandoc_types::Span {
}));
let b = Inline::Span(Box::new(quarto_pandoc_types::Span {
attr: (String::new(), vec![], LinkedHashMap::new()),
content: vec![],
source_info: other_source(),
attr_source: quarto_pandoc_types::AttrSourceInfo::empty(),
});
}));
assert!(inline_eq(&a, &b));
}

#[test]
fn test_quoted_eq() {
let a = Inline::Quoted(quarto_pandoc_types::Quoted {
let a = Inline::Quoted(Box::new(quarto_pandoc_types::Quoted {
quote_type: quarto_pandoc_types::QuoteType::DoubleQuote,
content: vec![],
source_info: dummy_source(),
});
let b = Inline::Quoted(quarto_pandoc_types::Quoted {
}));
let b = Inline::Quoted(Box::new(quarto_pandoc_types::Quoted {
quote_type: quarto_pandoc_types::QuoteType::DoubleQuote,
content: vec![],
source_info: other_source(),
});
}));
assert!(inline_eq(&a, &b));
}

#[test]
fn test_math_eq() {
let a = Inline::Math(quarto_pandoc_types::Math {
let a = Inline::Math(Box::new(quarto_pandoc_types::Math {
math_type: quarto_pandoc_types::MathType::InlineMath,
text: "x^2".to_string(),
source_info: dummy_source(),
text_source: None,
});
let b = Inline::Math(quarto_pandoc_types::Math {
}));
let b = Inline::Math(Box::new(quarto_pandoc_types::Math {
math_type: quarto_pandoc_types::MathType::InlineMath,
text: "x^2".to_string(),
source_info: other_source(),
text_source: None,
});
}));
assert!(inline_eq(&a, &b));
}

#[test]
fn test_rawinline_eq() {
let a = Inline::RawInline(quarto_pandoc_types::RawInline {
let a = Inline::RawInline(Box::new(quarto_pandoc_types::RawInline {
format: "html".to_string(),
text: "<b>".to_string(),
source_info: dummy_source(),
});
let b = Inline::RawInline(quarto_pandoc_types::RawInline {
}));
let b = Inline::RawInline(Box::new(quarto_pandoc_types::RawInline {
format: "html".to_string(),
text: "<b>".to_string(),
source_info: other_source(),
});
}));
assert!(inline_eq(&a, &b));
}

Expand All @@ -487,7 +487,7 @@ mod tests {
#[test]
fn test_codeblock_eq() {
use hashlink::LinkedHashMap;
let a = Block::CodeBlock(quarto_pandoc_types::CodeBlock {
let a = Block::CodeBlock(Box::new(quarto_pandoc_types::CodeBlock {
attr: (
String::new(),
vec!["python".to_string()],
Expand All @@ -496,8 +496,8 @@ mod tests {
text: "print(1)".to_string(),
source_info: dummy_source(),
attr_source: quarto_pandoc_types::AttrSourceInfo::empty(),
});
let b = Block::CodeBlock(quarto_pandoc_types::CodeBlock {
}));
let b = Block::CodeBlock(Box::new(quarto_pandoc_types::CodeBlock {
attr: (
String::new(),
vec!["python".to_string()],
Expand All @@ -506,7 +506,7 @@ mod tests {
text: "print(1)".to_string(),
source_info: other_source(),
attr_source: quarto_pandoc_types::AttrSourceInfo::empty(),
});
}));
assert!(block_eq(&a, &b));
}

Expand Down Expand Up @@ -538,24 +538,24 @@ mod tests {

#[test]
fn test_orderedlist_eq() {
let a = Block::OrderedList(quarto_pandoc_types::OrderedList {
let a = Block::OrderedList(Box::new(quarto_pandoc_types::OrderedList {
attr: (
1,
quarto_pandoc_types::ListNumberStyle::Decimal,
quarto_pandoc_types::ListNumberDelim::Period,
),
content: vec![vec![]],
source_info: dummy_source(),
});
let b = Block::OrderedList(quarto_pandoc_types::OrderedList {
}));
let b = Block::OrderedList(Box::new(quarto_pandoc_types::OrderedList {
attr: (
1,
quarto_pandoc_types::ListNumberStyle::Decimal,
quarto_pandoc_types::ListNumberDelim::Period,
),
content: vec![vec![]],
source_info: other_source(),
});
}));
assert!(block_eq(&a, &b));
}

Expand All @@ -573,18 +573,18 @@ mod tests {
#[test]
fn test_div_eq() {
use hashlink::LinkedHashMap;
let a = Block::Div(quarto_pandoc_types::Div {
let a = Block::Div(Box::new(quarto_pandoc_types::Div {
attr: (String::new(), vec![], LinkedHashMap::new()),
content: vec![],
source_info: dummy_source(),
attr_source: quarto_pandoc_types::AttrSourceInfo::empty(),
});
let b = Block::Div(quarto_pandoc_types::Div {
}));
let b = Block::Div(Box::new(quarto_pandoc_types::Div {
attr: (String::new(), vec![], LinkedHashMap::new()),
content: vec![],
source_info: other_source(),
attr_source: quarto_pandoc_types::AttrSourceInfo::empty(),
});
}));
assert!(block_eq(&a, &b));
}

Expand Down
Loading