From 1f7d7e330d4d8a1ea8ab22b61060b8b38bf43489 Mon Sep 17 00:00:00 2001 From: ydah Date: Wed, 21 Oct 2026 12:00:10 +0900 Subject: [PATCH] Ignore braces in action comments --- lib/lrama/lexer.rb | 37 +++++++++---- spec/lrama/lexer_spec.rb | 109 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 135 insertions(+), 11 deletions(-) diff --git a/lib/lrama/lexer.rb b/lib/lrama/lexer.rb index c70b91ce..a5b2a0c3 100644 --- a/lib/lrama/lexer.rb +++ b/lib/lrama/lexer.rb @@ -159,7 +159,6 @@ def lex_c_code raise "end_symbol is not set" unless end_symbol end_symbol_pattern = Regexp.new(end_symbol) - escaped_end_symbol_pattern = Regexp.new(Regexp.escape(end_symbol)) reset_first_position until @scanner.eos? do @@ -180,17 +179,33 @@ def lex_c_code when @scanner.scan(/\n/) code << @scanner.matched newline - when @scanner.scan(/".*?"/) - code << %Q(#{@scanner.matched}) - @line += @scanner.matched.count("\n") - when @scanner.scan(/'.*?'/) - code << %Q(#{@scanner.matched}) - when @scanner.scan(/[^\"'\{\}\n]+/) - code << @scanner.matched - when @scanner.scan(escaped_end_symbol_pattern) - code << @scanner.matched + when (fragment = @scanner.scan(%r{/\*|//[^\n]*|"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'}m)) + if fragment == '/*' + comment = @scanner.scan_until(%r{\*/}) + raise ParseError, location.generate_error_message("Unterminated comment") unless comment # steep:ignore UnknownConstant + + fragment << comment + end + code << fragment + @line += fragment.count("\n") + if (last_newline = fragment.b.rindex("\n")) + @head = @scanner.pos - fragment.bytesize + last_newline + 1 + end else - code << @scanner.getch + start = @scanner.pos + # Search for delimiters without retaining backtracking positions for ordinary C text. + if (length = @scanner.skip_until(%r{["'{}\n]|/[/*]})) + delimiter_size = @scanner.matched_size + @scanner.pos -= delimiter_size + if length > delimiter_size + code << @scanner.string.byteslice(start, length - delimiter_size) + else + code << @scanner.getch + end + else + code << @scanner.rest + @scanner.terminate + end end end diff --git a/spec/lrama/lexer_spec.rb b/spec/lrama/lexer_spec.rb index f06fd6cc..373b7af1 100644 --- a/spec/lrama/lexer_spec.rb +++ b/spec/lrama/lexer_spec.rb @@ -431,6 +431,58 @@ MSG end + [ + [' /* } */ { int x; } ', 1, 21], + [' /* { */ int x; ', 1, 17], + [" // don't {\n int x; ", 2, 8], + [" // }\n int x; ", 2, 8], + [" /* あ\n } */ int x; ", 2, 13], + [%q( const char *s = "\"//"; int x; ), 1, 33], + ].each do |code, last_line, last_column| + it "finds the action terminator after #{code.inspect}" do + grammar_file = Lrama::Lexer::GrammarFile.new("action.y", "{#{code}};") + lexer = Lrama::Lexer.new(grammar_file) + + expect(lexer.next_token).to eq(['{', token_class::Token.new(s_value: '{')]) + lexer.status = :c_declaration + lexer.end_symbol = '}' + token = lexer.next_token + + expect(token).to eq([:C_DECLARATION, token_class::UserCode.new(s_value: code)]) + expect(token[1].location).to eq Lrama::Lexer::Location.new(grammar_file: grammar_file, first_line: 1, first_column: 1, last_line: last_line, last_column: last_column) + lexer.status = :initial + + expect(lexer.next_token).to eq(['}', token_class::Token.new(s_value: '}')]) + expect(lexer.next_token).to eq([';', token_class::Token.new(s_value: ';')]) + expect(lexer.next_token).to be_nil + end + end + + [ + [%q( printf("\"}"); ), 1, 17], + [%q( printf("\"{"); ), 1, 17], + [%q( printf("\\\\"); { int x; } ), 1, 27], + [" printf(\"あ\\\n\\\"}\"); ", 2, 7], + ].each do |code, last_line, last_column| + it "lexes an action containing an escaped C string in #{code.inspect}" do + grammar_file = Lrama::Lexer::GrammarFile.new("escaped.y", "{#{code}};") + lexer = Lrama::Lexer.new(grammar_file) + + expect(lexer.next_token).to eq(['{', token_class::Token.new(s_value: '{')]) + lexer.status = :c_declaration + lexer.end_symbol = '}' + token = lexer.next_token + + expect(token).to eq([:C_DECLARATION, token_class::UserCode.new(s_value: code)]) + expect(token[1].location).to eq Lrama::Lexer::Location.new(grammar_file: grammar_file, first_line: 1, first_column: 1, last_line: last_line, last_column: last_column) + lexer.status = :initial + + expect(lexer.next_token).to eq(['}', token_class::Token.new(s_value: '}')]) + expect(lexer.next_token).to eq([';', token_class::Token.new(s_value: ';')]) + expect(lexer.next_token).to be_nil + end + end + context 'unexpected_c_code.y' do it do grammar_file = Lrama::Lexer::GrammarFile.new("invalid.y", "@invalid") @@ -462,6 +514,63 @@ end end + ['%}', '}', '\Z'].each do |end_symbol| + ['/*', '/*/', '/* unterminated } %}', "/* unterminated\n/* still unterminated"].each do |code| + it "rejects an unterminated C comment #{code.inspect} before #{end_symbol.inspect}" do + grammar_file = Lrama::Lexer::GrammarFile.new("comment.y", code) + lexer = Lrama::Lexer.new(grammar_file) + lexer.status = :c_declaration + lexer.end_symbol = end_symbol + + expect { lexer.next_token }.to raise_error(ParseError, /Unterminated comment/) + end + end + end + + [ + 'value / 2 % 3', + '/**/value/2;/***/', + 'value /= 2; value %= 3;', + 'value / 2/* } */%3;', + "value/2// }\n%3;", + %q(value/2; const char *s = "/* } */";), + 'value/', + 'value%', + 'value\Z', + '%}', + '/%}', + '"', + '"unterminated\\', + ].each do |code| + it "preserves C operators and comments in #{code.inspect}" do + grammar_file = Lrama::Lexer::GrammarFile.new("operators.y", code) + lexer = Lrama::Lexer.new(grammar_file) + lexer.status = :c_declaration + lexer.end_symbol = '\Z' + + expect(lexer.next_token).to eq([:C_DECLARATION, token_class::UserCode.new(s_value: code)]) + end + end + + it 'tracks action locations after multibyte comments and operators' do + code = " value/2; /* あ\n } */value%3; " + grammar_file = Lrama::Lexer::GrammarFile.new("operators.y", "{#{code}};") + lexer = Lrama::Lexer.new(grammar_file) + + expect(lexer.next_token).to eq(['{', token_class::Token.new(s_value: '{')]) + lexer.status = :c_declaration + lexer.end_symbol = '}' + token = lexer.next_token + + expect(token).to eq([:C_DECLARATION, token_class::UserCode.new(s_value: code)]) + expect(token[1].location).to eq Lrama::Lexer::Location.new(grammar_file: grammar_file, first_line: 1, first_column: 1, last_line: 2, last_column: 14) + lexer.status = :initial + + expect(lexer.next_token).to eq(['}', token_class::Token.new(s_value: '}')]) + expect(lexer.next_token).to eq([';', token_class::Token.new(s_value: ';')]) + expect(lexer.next_token).to be_nil + end + it 'lex a trailing space with newline' do grammar_file = Lrama::Lexer::GrammarFile.new("trailing_space.y", " \n%require") lexer = Lrama::Lexer.new(grammar_file)