diff --git a/.changeset/linear-html-comment-seek.md b/.changeset/linear-html-comment-seek.md
new file mode 100644
index 000000000..46acdf023
--- /dev/null
+++ b/.changeset/linear-html-comment-seek.md
@@ -0,0 +1,5 @@
+---
+'@shopify/liquid-html-parser': patch
+---
+
+Find the parser's position after each HTML comment, `` (NBSP) is body text, not a close.
let end = i + 1;
- while (end < tokenCount && parser.tokenAt(end).type === TokenType.Text) end++;
+ while (parser.hasToken(end) && parser.tokenAt(end).type === TokenType.Text) end++;
if (end === i + 1) continue;
const text = source.slice(token.end, parser.tokenAt(end - 1).end);
if (text.trim().toLowerCase() === lowerName) {
diff --git a/packages/liquid-html-parser/src/document/liquid-raw.ts b/packages/liquid-html-parser/src/document/liquid-raw.ts
index 56690aad5..5e8d67d1e 100644
--- a/packages/liquid-html-parser/src/document/liquid-raw.ts
+++ b/packages/liquid-html-parser/src/document/liquid-raw.ts
@@ -93,7 +93,7 @@ export function parseLiquidInRange(
// token at each step (and re-checking against `bodyEnd`) keeps the walk
// correct across that mutation; a cached end index would point at a stale slot.
let i = parser.getPosition();
- while (i < parser.tokenCount() && parser.tokenAt(i).start < bodyEnd) {
+ while (parser.hasToken(i) && parser.tokenAt(i).start < bodyEnd) {
const token = parser.tokenAt(i);
if (
@@ -112,12 +112,12 @@ export function parseLiquidInRange(
? TokenType.LiquidVariableOutputClose
: TokenType.LiquidTagClose;
let j = i + 1;
- while (j < parser.tokenCount() && parser.tokenAt(j).type !== closeType) j++;
- const closesWithinBody = j < parser.tokenCount() && parser.tokenAt(j).end <= bodyEnd;
+ while (parser.hasToken(j) && parser.tokenAt(j).type !== closeType) j++;
+ const closesWithinBody = parser.hasToken(j) && parser.tokenAt(j).end <= bodyEnd;
if (!closesWithinBody) {
nodes.push(makeTextNode(source.slice(token.start, bodyEnd), token.start, bodyEnd, source));
- while (i < parser.tokenCount() && parser.tokenAt(i).start < bodyEnd) i++;
+ while (parser.hasToken(i) && parser.tokenAt(i).start < bodyEnd) i++;
break;
}
diff --git a/packages/liquid-html-parser/src/document/tokenizer.ts b/packages/liquid-html-parser/src/document/tokenizer.ts
index 229c6ab9b..2d9345c0f 100644
--- a/packages/liquid-html-parser/src/document/tokenizer.ts
+++ b/packages/liquid-html-parser/src/document/tokenizer.ts
@@ -63,6 +63,31 @@ export interface TokenizeOptions {
export function tokenize(source: string, options: TokenizeOptions = {}): Token[] {
const tokens: Token[] = [];
+ const tokenizer = createTokenizer(source, tokens);
+ tokenizer.restart(0, options);
+ tokenizer.fill(Infinity);
+ return tokens;
+}
+
+/**
+ * `tokenize`, run on demand. `ParserBase.resliceTokensFrom` uses it to
+ * re-tokenize the rest of a document only as far as the parser reads.
+ */
+export interface Tokenizer {
+ /**
+ * Start over at `start`, as if `source` began there; tokens keep their
+ * offsets in `source`.
+ */
+ restart(start: number, options?: TokenizeOptions): void;
+
+ /**
+ * Add tokens until there are more than `index` of them, or until the input
+ * runs out, which adds `EndOfInput`. Returns whether input remains.
+ */
+ fill(index: number): boolean;
+}
+
+export function createTokenizer(source: string, tokens: Token[]): Tokenizer {
const modeStack: Mode[] = [];
let mode = Mode.Default as Mode;
let pos = 0;
@@ -70,21 +95,12 @@ export function tokenize(source: string, options: TokenizeOptions = {}): Token[]
// Whether the pending Text run is whitespace. Only HtmlTag mode reads it.
let textIsSpace = false;
let quoteChar = '';
-
- // Resume inside a quoted attribute value when reslicing a suffix that begins
- // mid-attribute (e.g. `...{% endraw %}">` straddled by a stray `{{`). The
- // enclosing HtmlTag mode is kept beneath QuotedValue so popping on the
- // closing quote correctly emits the attribute close and trailing `>`.
- if (options.insideQuotedAttribute) {
- quoteChar = options.insideQuotedAttribute;
- modeStack.push(Mode.HtmlTag);
- mode = Mode.QuotedValue;
- } else if (options.insideHtmlTag) {
- // Resume inside the attribute list of an open tag. The empty mode stack
- // means the closing `>` pops back to `Default`, so the rest of the suffix
- // (`>ok`) tokenizes as normal document content.
- mode = Mode.HtmlTag;
- }
+ let ended = false;
+ // The last search for `}}`: where it started, and where it found one (-1 for
+ // nowhere). Restarts after raw tags can land inside the same unclosed `{{`
+ // again and again, and this keeps each from searching the rest anew.
+ let closeSearchFrom = Infinity;
+ let closeFound = -1;
function ch(offset: number): string {
const i = pos + offset;
@@ -121,17 +137,53 @@ export function tokenize(source: string, options: TokenizeOptions = {}): Token[]
if (textStart === -1) textStart = pos;
}
- // YAML frontmatter: only at position 0
- if (!options.skipFrontmatter && (match('---\n') || match('---\r\n'))) {
- const searchStart = source.indexOf('\n', 0) + 1;
- const closeIdx = source.indexOf('\n---', searchStart);
- if (closeIdx !== -1) {
- let end = closeIdx + 4; // after \n---
- // Include trailing newline/CRLF
- if (end < source.length && source[end] === '\r') end++;
- if (end < source.length && source[end] === '\n') end++;
- tokens.push({ type: TokenType.YamlFrontmatter, start: 0, end });
- pos = end;
+ /**
+ * Where the `-}}` or `}}` that closes Liquid output at `pos` begins, or the
+ * end of the input. Neither begins at `pos` itself.
+ */
+ function nextOutputClose(): number {
+ if (pos < closeSearchFrom || (closeFound !== -1 && pos > closeFound)) {
+ closeSearchFrom = pos;
+ closeFound = source.indexOf('}}', pos);
+ }
+ if (closeFound === -1) return source.length;
+ return source[closeFound - 1] === '-' ? closeFound - 1 : closeFound;
+ }
+
+ function restart(start: number, options: TokenizeOptions = {}) {
+ modeStack.length = 0;
+ mode = Mode.Default;
+ pos = start;
+ textStart = -1;
+ ended = false;
+
+ // Resume inside a quoted attribute value when reslicing a suffix that begins
+ // mid-attribute (e.g. `...{% endraw %}">` straddled by a stray `{{`). The
+ // enclosing HtmlTag mode is kept beneath QuotedValue so popping on the
+ // closing quote correctly emits the attribute close and trailing `>`.
+ if (options.insideQuotedAttribute) {
+ quoteChar = options.insideQuotedAttribute;
+ modeStack.push(Mode.HtmlTag);
+ mode = Mode.QuotedValue;
+ } else if (options.insideHtmlTag) {
+ // Resume inside the attribute list of an open tag. The empty mode stack
+ // means the closing `>` pops back to `Default`, so the rest of the suffix
+ // (`>ok`) tokenizes as normal document content.
+ mode = Mode.HtmlTag;
+ }
+
+ // YAML frontmatter: only at position 0
+ if (!options.skipFrontmatter && (match('---\n') || match('---\r\n'))) {
+ const searchStart = source.indexOf('\n', pos) + 1;
+ const closeIdx = source.indexOf('\n---', searchStart);
+ if (closeIdx !== -1) {
+ let end = closeIdx + 4; // after \n---
+ // Include trailing newline/CRLF
+ if (end < source.length && source[end] === '\r') end++;
+ if (end < source.length && source[end] === '\n') end++;
+ tokens.push({ type: TokenType.YamlFrontmatter, start: pos, end });
+ pos = end;
+ }
}
}
@@ -160,153 +212,161 @@ export function tokenize(source: string, options: TokenizeOptions = {}): Token[]
return false;
}
- while (pos < source.length) {
- switch (mode) {
- case Mode.LiquidTag: {
- if (match('-%}')) {
- emit(TokenType.LiquidTagClose, 3);
- popMode();
- } else if (match('%}')) {
- emit(TokenType.LiquidTagClose, 2);
- popMode();
- } else {
- startText();
- pos++;
- }
- break;
- }
-
- case Mode.LiquidVariableOutput: {
- if (match('-}}')) {
- emit(TokenType.LiquidVariableOutputClose, 3);
- popMode();
- } else if (match('}}')) {
- emit(TokenType.LiquidVariableOutputClose, 2);
- popMode();
- } else {
- startText();
- pos++;
+ function fill(index: number): boolean {
+ while (pos < source.length && tokens.length <= index) {
+ switch (mode) {
+ case Mode.LiquidTag: {
+ if (match('-%}')) {
+ emit(TokenType.LiquidTagClose, 3);
+ popMode();
+ } else if (match('%}')) {
+ emit(TokenType.LiquidTagClose, 2);
+ popMode();
+ } else {
+ startText();
+ pos++;
+ }
+ break;
}
- break;
- }
-
- case Mode.Default: {
- if (scanLiquidOpen()) continue;
- if (match('')) {
- emit(TokenType.HtmlCommentClose, 3);
- continue;
- }
+ case Mode.Default: {
+ if (scanLiquidOpen()) continue;
- if (match('')) {
+ emit(TokenType.HtmlCommentClose, 3);
continue;
}
- }
- if (ch(0) === '<') {
- const after = ch(1);
- if (/[a-zA-Z]/.test(after) || after === '{') {
- emit(TokenType.HtmlTagOpen, 1);
+ if (match('')) {
- emit(TokenType.HtmlSelfClose, 2);
- popMode();
- continue;
+ startText();
+ pos++;
+ break;
}
- if (ch(0) === '>') {
- emit(TokenType.HtmlTagClose, 1);
- popMode();
- continue;
- }
+ case Mode.HtmlTag: {
+ if (scanLiquidOpen()) continue;
- if (ch(0) === '=') {
- emit(TokenType.HtmlEquals, 1);
- continue;
- }
+ if (match('/>')) {
+ emit(TokenType.HtmlSelfClose, 2);
+ popMode();
+ continue;
+ }
- // Accept straight quotes and curly (smart) quotes as attribute-value
- // openers. Curly quotes get normalized to straight quotes downstream;
- // recognizing them here (HTML scope only — Liquid tokenization is
- // untouched) stops the value from splitting at interior spaces.
- if (
- ch(0) === '"' ||
- ch(0) === "'" ||
- ch(0) === '“' ||
- ch(0) === '”' ||
- ch(0) === '‘' ||
- ch(0) === '’'
- ) {
- quoteChar = ch(0);
- emit(TokenType.HtmlQuoteOpen, 1);
- pushMode(Mode.QuotedValue);
- continue;
+ if (ch(0) === '>') {
+ emit(TokenType.HtmlTagClose, 1);
+ popMode();
+ continue;
+ }
+
+ if (ch(0) === '=') {
+ emit(TokenType.HtmlEquals, 1);
+ continue;
+ }
+
+ // Accept straight quotes and curly (smart) quotes as attribute-value
+ // openers. Curly quotes get normalized to straight quotes downstream;
+ // recognizing them here (HTML scope only — Liquid tokenization is
+ // untouched) stops the value from splitting at interior spaces.
+ if (
+ ch(0) === '"' ||
+ ch(0) === "'" ||
+ ch(0) === '“' ||
+ ch(0) === '”' ||
+ ch(0) === '‘' ||
+ ch(0) === '’'
+ ) {
+ quoteChar = ch(0);
+ emit(TokenType.HtmlQuoteOpen, 1);
+ pushMode(Mode.QuotedValue);
+ continue;
+ }
+
+ // A run of whitespace inside a tag is a Text token of its own, never
+ // part of a name or an unquoted value. It plays the part of ohm's
+ // `space`, which the grammar's syntactic attribute rules skipped
+ // implicitly (`attr = "v"` is one attribute).
+ const isSpace = /\s/.test(ch(0));
+ if (textStart !== -1 && isSpace !== textIsSpace) flushText();
+ if (textStart === -1) textIsSpace = isSpace;
+ startText();
+ pos++;
+ break;
}
- // A run of whitespace inside a tag is a Text token of its own, never
- // part of a name or an unquoted value. It plays the part of ohm's
- // `space`, which the grammar's syntactic attribute rules skipped
- // implicitly (`attr = "v"` is one attribute).
- const isSpace = /\s/.test(ch(0));
- if (textStart !== -1 && isSpace !== textIsSpace) flushText();
- if (textStart === -1) textIsSpace = isSpace;
- startText();
- pos++;
- break;
- }
+ case Mode.QuotedValue: {
+ if (scanLiquidOpen()) continue;
- case Mode.QuotedValue: {
- if (scanLiquidOpen()) continue;
+ // Curly quotes are directional, so a value opened with a left curly
+ // quote closes on its right partner (and vice-versa); straight quotes
+ // close on themselves.
+ if (ch(0) === closingQuoteFor(quoteChar) || ch(0) === quoteChar) {
+ emit(TokenType.HtmlQuoteClose, 1);
+ popMode();
+ continue;
+ }
- // Curly quotes are directional, so a value opened with a left curly
- // quote closes on its right partner (and vice-versa); straight quotes
- // close on themselves.
- if (ch(0) === closingQuoteFor(quoteChar) || ch(0) === quoteChar) {
- emit(TokenType.HtmlQuoteClose, 1);
- popMode();
- continue;
+ startText();
+ pos++;
+ break;
}
- startText();
- pos++;
- break;
+ default:
+ assertNever(mode);
}
-
- default:
- assertNever(mode);
}
+ if (pos < source.length) return true;
+ if (!ended) {
+ flushText();
+ tokens.push({ type: TokenType.EndOfInput, start: source.length, end: source.length });
+ ended = true;
+ }
+ return false;
}
- flushText();
- tokens.push({ type: TokenType.EndOfInput, start: source.length, end: source.length });
- return tokens;
+ return { restart, fill };
}
enum Mode {
diff --git a/specs/parser/html.yml b/specs/parser/html.yml
index 8e5047ca9..7df09c1f6 100644
--- a/specs/parser/html.yml
+++ b/specs/parser/html.yml
@@ -4928,3 +4928,47 @@ cases:
value: " b"
position: { start: [0, 34], text: " b" }
children: []
+ - name: should parse HTML comments and raw elements side by side
+ source:
+ parse: [html]
+ ast:
+ type: Document
+ name: "#document"
+ position: { start: [0, 0], text: }
+ children:
+ - type: HtmlComment
+ body: a
+ position: { start: [0, 0], text: }
+ - type: HtmlRawNode
+ name: script
+ blockStartPosition: { start: [0, 8], text: }
+ position: { start: [0, 8], text: }
+ attributes: []
+ body:
+ type: RawMarkup
+ kind: javascript
+ value: b
+ position: { start: [0, 16], text: b }
+ nodes:
+ - type: TextNode
+ value: b
+ position: { start: [0, 16], text: b }
+ - type: HtmlComment
+ body: c
+ position: { start: [0, 26], text: }
+ - type: HtmlRawNode
+ name: style
+ blockStartPosition: { start: [0, 34], text: }
+ position: { start: [0, 34], text: }
+ attributes: []
+ body:
+ type: RawMarkup
+ kind: css
+ value: d
+ position: { start: [0, 41], text: d }
+ nodes:
+ - type: TextNode
+ value: d
+ position: { start: [0, 41], text: d }