diff --git a/Coder.Test/Languages/ReservedWordIdentifierTests.cs b/Coder.Test/Languages/ReservedWordIdentifierTests.cs new file mode 100644 index 0000000..c09fcdb --- /dev/null +++ b/Coder.Test/Languages/ReservedWordIdentifierTests.cs @@ -0,0 +1,202 @@ +// Copyright (c) 2023-2026 ktsu-dev contributors + +namespace ktsu.Coder.Test.Languages; + +using ktsu.Coder.Ast; +using ktsu.Coder.Languages; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +/// +/// Compiles and runs what the Rust and C++ generators write for names that are keywords of the +/// target: a property called Type or Default, which are ordinary C# names, and fields, +/// parameters and locals called where, virtual, match, delete, +/// loop and new. Each name has to come out escaped where it is declared and spelled the +/// same way wherever it is used, or the output either fails to parse or fails to resolve. +/// +/// +/// The tests are inconclusive rather than failing where no compiler is on the path, which is the +/// honest result: nothing was checked. +/// +[TestClass] +public class ReservedWordIdentifierTests +{ + private static readonly string[] CppCompilers = ["c++", "g++", "clang++"]; + + /// + /// The Rust driver. r# is how a Rust caller names a keyword, so it reads the escaped names + /// back exactly as a user of the generated type would. + /// + private const string RustDriver = """ + + fn main() { + let mut holder = Holder::make(); + if holder.r#where != 3 || holder.r#virtual != 4 { std::process::exit(1); } + if holder.r#type() != 1 { std::process::exit(2); } + if holder.default() != 2 { std::process::exit(3); } + if holder.total(5, 6) != 14 { std::process::exit(4); } + } + + """; + + /// + /// The C++ driver. C++ has no escape for a keyword, so an escaped name has had an underscore + /// appended, and that is the name a caller uses. + /// + private const string CppDriver = """ + #include "holder.hpp" + + int main() + { + Holder holder = Holder::make(); + if (holder.where != 3 || holder.virtual_ != 4) { return 1; } + if (holder.type() != 1) { return 2; } + if (holder.default_() != 2) { return 3; } + if (holder.total(5, 6) != 15) { return 4; } + return 0; + } + + """; + + /// + /// Tests that Rust names a keyword-named field, getter, parameter and local as raw identifiers, + /// at their declarations and where they are used. + /// + [TestMethod] + public void Rust_KeywordNamesCompileAndResolve() + { + if (ToolchainHarness.FindOnPath("--version", "rustc") is null) + { + Assert.Inconclusive("No Rust compiler on the path, so nothing was compiled."); + return; + } + + ToolchainHarness.InTemporaryDirectory(directory => + { + string program = OperatingSystem.IsWindows() ? "holder.exe" : "holder"; + string source = new RustGenerator().Generate(Types("self.where")) + RustDriver; + File.WriteAllText(Path.Combine(directory, "holder.rs"), source); + + (int built, string output) = ToolchainHarness.Run("rustc", $"--edition 2021 -o {program} holder.rs", directory); + Assert.AreEqual(0, built, $"rustc rejected the generated source:{Environment.NewLine}{output}{Environment.NewLine}{source}"); + + (int ran, _) = ToolchainHarness.Run(Path.Combine(directory, program), string.Empty, directory); + Assert.AreEqual(0, ran, $"a keyword-named member read back wrongly (driver check {ran}):{Environment.NewLine}{source}"); + }); + } + + /// + /// Tests that C++ appends an underscore to a keyword-named field, getter, parameter and local, + /// at their declarations and where they are used. + /// + [TestMethod] + public void Cpp_KeywordNamesCompileAndResolve() + { + string? compiler = ToolchainHarness.FindOnPath("--version", CppCompilers); + if (compiler is null) + { + Assert.Inconclusive("No C++ compiler on the path, so nothing was compiled."); + return; + } + + ToolchainHarness.InTemporaryDirectory(directory => + { + string program = OperatingSystem.IsWindows() ? "holder.exe" : "holder"; + string header = new CppGenerator().Generate(Types("virtual")); + File.WriteAllText(Path.Combine(directory, "holder.hpp"), header); + File.WriteAllText(Path.Combine(directory, "driver.cpp"), CppDriver); + + (int built, string output) = ToolchainHarness.Run( + compiler, + $"-std=c++20 -Wall -Wextra -pedantic -o {program} driver.cpp", + directory); + Assert.AreEqual(0, built, $"{compiler} rejected the generated header:{Environment.NewLine}{output}{Environment.NewLine}{header}"); + + (int ran, _) = ToolchainHarness.Run(Path.Combine(directory, program), string.Empty, directory); + Assert.AreEqual(0, ran, $"a keyword-named member read back wrongly (driver check {ran}):{Environment.NewLine}{header}"); + }); + } + + /// + /// Tests that a reference keeps a receiver as written and escapes only the plain names after it. + /// + [TestMethod] + public void Rust_ReferenceKeepsTheReceiverAndEscapesTheMember() + { + FunctionDeclaration read = new("read") { ReturnType = "int" }; + read.Body.Add(new ReturnStatement(new VariableReference("self.type"))); + ClassDeclaration holder = new("Holder") { Kind = TypeDeclarationKind.Struct }; + holder.Members.Add(new VariableDeclaration("type", "int")); + holder.Members.Add(read); + SourceFile file = new("holder"); + file.Members.Add(holder); + + string source = new RustGenerator().Generate(file); + + Assert.Contains("pub r#type: i32,", source, StringComparison.Ordinal, source); + Assert.Contains("self.r#type", source, StringComparison.Ordinal, source); + } + + /// + /// Tests that the path keywords, which Rust refuses as raw identifiers, are given an underscore + /// instead when they name a declaration. + /// + [TestMethod] + public void Rust_PathKeywordNamesAreSuffixed() + { + FunctionDeclaration function = new("super") { ReturnType = "int" }; + function.Parameters.Add(new Parameter("crate", "int")); + function.Body.Add(new ReturnStatement(new VariableReference("crate"))); + SourceFile file = new("paths"); + file.Members.Add(function); + + string source = new RustGenerator().Generate(file); + + Assert.Contains("fn super_(crate_: i32)", source, StringComparison.Ordinal, source); + Assert.DoesNotContain("r#super", source, StringComparison.Ordinal, source); + } + + /// + /// Builds a struct whose members are all named after a keyword of one target or the other. + /// + /// How the target's method body refers to one of the struct's own fields. + /// The file to generate. + private static SourceFile Types(string fieldReference) + { + ClassDeclaration holder = new("Holder") { Kind = TypeDeclarationKind.Struct }; + holder.Members.Add(new VariableDeclaration("where", "int")); + holder.Members.Add(new VariableDeclaration("virtual", "int")); + holder.Members.Add(new PropertyDeclaration("Type", "int") + { + GetterBody = { new ReturnStatement(new LiteralExpression(1)) }, + }); + holder.Members.Add(new PropertyDeclaration("Default", "int") + { + GetterBody = { new ReturnStatement(new LiteralExpression(2)) }, + }); + + ConstructionExpression built = new(new TypeReference("Holder")); + built.Arguments.Add(new MemberInitialiser("where", new LiteralExpression(3))); + built.Arguments.Add(new MemberInitialiser("virtual", new LiteralExpression(4))); + FunctionDeclaration make = new("make") { ReturnType = "Holder", IsStatic = true }; + make.Body.Add(new ReturnStatement(built)); + holder.Members.Add(make); + + FunctionDeclaration total = new("total") { ReturnType = "int" }; + total.Parameters.Add(new Parameter("match", "int")); + total.Parameters.Add(new Parameter("delete", "int")); + total.Body.Add(new VariableDeclaration( + "loop", + "int", + new BinaryExpression(new VariableReference("match"), BinaryOperator.Add, new VariableReference("delete")))); + total.Body.Add(new VariableDeclaration( + "new", + "int", + new BinaryExpression(new VariableReference("loop"), BinaryOperator.Add, new VariableReference(fieldReference)))); + total.Body.Add(new ReturnStatement(new VariableReference("new"))); + holder.Members.Add(total); + + SourceFile file = new("holder"); + file.Members.Add(holder); + return file; + } +} diff --git a/Coder/Languages/CFamilyGenerator.cs b/Coder/Languages/CFamilyGenerator.cs index e261d36..802759f 100644 --- a/Coder/Languages/CFamilyGenerator.cs +++ b/Coder/Languages/CFamilyGenerator.cs @@ -170,7 +170,7 @@ protected string SpellDeclarator(TypeReference type, string name) element.IsArray = false; } - string declared = name.Length == 0 ? string.Empty : $" {name}"; + string declared = name.Length == 0 ? string.Empty : $" {EscapeIdentifier(name)}"; return $"{SpellType(element)}{declared}{(type.IsArray ? "[]" : string.Empty)}"; } diff --git a/Coder/Languages/CppGenerator.cs b/Coder/Languages/CppGenerator.cs index e03ee4f..e65a86c 100644 --- a/Coder/Languages/CppGenerator.cs +++ b/Coder/Languages/CppGenerator.cs @@ -110,6 +110,32 @@ private static void WriteTemplateHead(IEnumerable parameters, Cod } } + /// + /// + /// The C++20 keywords and the alternative operator tokens (and, not and the rest), + /// which the standard reserves just as firmly. An escaped name has an underscore appended, which + /// is a rename: C++ has no way to use a keyword as a name. + /// + protected override IReadOnlySet ReservedWords { get; } = new HashSet(StringComparer.Ordinal) + { + "alignas", "alignof", "and", "and_eq", "asm", "auto", "bitand", "bitor", "bool", "break", "case", + "catch", "char", "char8_t", "char16_t", "char32_t", "class", "compl", "concept", "const", "consteval", + "constexpr", "constinit", "const_cast", "continue", "co_await", "co_return", "co_yield", "decltype", + "default", "delete", "do", "double", "dynamic_cast", "else", "enum", "explicit", "export", "extern", + "false", "float", "for", "friend", "goto", "if", "inline", "int", "long", "mutable", "namespace", + "new", "noexcept", "not", "not_eq", "nullptr", "operator", "or", "or_eq", "private", "protected", + "public", "register", "reinterpret_cast", "requires", "return", "short", "signed", "sizeof", "static", + "static_assert", "static_cast", "struct", "switch", "template", "this", "thread_local", "throw", "true", + "try", "typedef", "typeid", "typename", "union", "unsigned", "using", "virtual", "void", "volatile", + "wchar_t", "while", "xor", "xor_eq", + }; + + /// + /// + /// this and the literal words, which a reference written as text means as C++ means them. + /// + protected override bool IsReceiverWord(string word) => word is "this" or "true" or "false" or "nullptr"; + /// /// /// The standard library spells an invented accessor set_value, and that is the convention a reader of any C++ header already has. @@ -352,7 +378,7 @@ private void WriteInitialiserList(FunctionDeclaration funcDecl, CodeBlocker code } MemberInitialiser initialiser = funcDecl.Initialisers[index]; - code.Write($"{initialiser.Name}("); + code.Write($"{EscapeIdentifier(initialiser.Name ?? string.Empty)}("); if (initialiser.Value is not null) { @@ -372,7 +398,7 @@ private void WriteInitialiserList(FunctionDeclaration funcDecl, CodeBlocker code /// The declaration being emitted. /// The name of the type it belongs to, when it belongs to one. /// The name as C++ writes it. - private static string SpellFunctionName(FunctionDeclaration funcDecl, string? enclosingType) + private string SpellFunctionName(FunctionDeclaration funcDecl, string? enclosingType) { string typeName = enclosingType ?? funcDecl.Name ?? "UnnamedType"; @@ -383,7 +409,7 @@ private static string SpellFunctionName(FunctionDeclaration funcDecl, string? en FunctionKind.Operator => $"operator{funcDecl.Name}", FunctionKind.ConversionOperator => $"operator {MapToCppType(funcDecl.ReturnType ?? new TypeReference("void"))}", - _ => funcDecl.Name ?? "unnamedFunction", + _ => EscapeIdentifier(funcDecl.Name ?? "unnamedFunction"), }; } @@ -835,7 +861,7 @@ private string SpellVariableDeclarator(VariableDeclaration varDecl) return SpellDeclarator(declared, varDecl.Name); } - return $"{(varDecl.InitialValue is not null ? "auto" : "std::any")} {varDecl.Name}"; + return $"{(varDecl.InitialValue is not null ? "auto" : "std::any")} {EscapeIdentifier(varDecl.Name)}"; } /// diff --git a/Coder/Languages/LanguageGeneratorBase.cs b/Coder/Languages/LanguageGeneratorBase.cs index adb90b7..8c18bca 100644 --- a/Coder/Languages/LanguageGeneratorBase.cs +++ b/Coder/Languages/LanguageGeneratorBase.cs @@ -154,7 +154,7 @@ protected bool TryGenerateCommonNode(AstNode node, CodeBlocker code) switch (node) { case VariableReference varRef: - code.Write(varRef.Name); + code.Write(EscapeReference(varRef.Name)); return true; case LiteralExpression stringLit: @@ -1127,6 +1127,76 @@ or AstLeafNode protected static Visibility VisibilityOf(AstNode node) => node is IHasVisibility declaration ? declaration.Visibility : Visibility.Unspecified; + /// + /// Gets the words the target language reserves, which no declared name may be spelled as. + /// + /// + /// The AST is language-neutral, so a name that is ordinary in one target is a keyword in another: + /// a property called Type becomes a getter type in snake case, which Rust reserves, + /// and one called Default becomes default, which C++ does. Empty by default, so a + /// generator that declares nothing escapes nothing. + /// + protected virtual IReadOnlySet ReservedWords { get; } = new HashSet(StringComparer.Ordinal); + + /// + /// Spells a declared name so the target does not read it as one of its own keywords. + /// + /// The name as the AST holds it. + /// The name unchanged, or escaped when it is one of . + /// + /// Appends an underscore, which is the only escape most targets have: default becomes + /// default_. A target with a real escape — Rust's raw identifiers — overrides this. Every + /// site that writes a name, at its declaration and wherever it is used, goes through here or + /// through , so the two cannot come out spelled differently. + /// + protected virtual string EscapeIdentifier(string name) + { + Ensure.NotNull(name); + return ReservedWords.Contains(name) ? $"{name}_" : name; + } + + /// + /// Tells whether a word leading a reference means what the language means by it. + /// + /// The first dotted segment of a . + /// True when the word is left as written there; false when it is escaped like any other name. + /// + /// A reference is text, and self.x or this names the receiver rather than something + /// the AST declared. Such a word is a keyword, but escaping it would break the reference. + /// + protected virtual bool IsReceiverWord(string word) => false; + + /// + /// Spells a reference so any segment naming a declared member escapes as its declaration did. + /// + /// The reference's text, possibly a dotted member access such as self.type. + /// The reference with each reserved segment escaped. + /// + /// Only a segment that is exactly a reserved word changes. Anything else — a call such as + /// args.len(), an index such as steps[0], a path such as Self::CAPACITY — is + /// written as given, since a reference is text and only its plain names are the AST's. + /// + protected string EscapeReference(string reference) + { + Ensure.NotNull(reference); + + if (ReservedWords.Count == 0) + { + return reference; + } + + string[] segments = reference.Split('.'); + for (int i = 0; i < segments.Length; i++) + { + if (ReservedWords.Contains(segments[i]) && !(i == 0 && IsReceiverWord(segments[i]))) + { + segments[i] = EscapeIdentifier(segments[i]); + } + } + + return string.Join(".", segments); + } + /// /// Escapes a string literal's contents for a target language using C-style backslash escapes. /// diff --git a/Coder/Languages/RustGenerator.cs b/Coder/Languages/RustGenerator.cs index 5691f40..1b2aeb2 100644 --- a/Coder/Languages/RustGenerator.cs +++ b/Coder/Languages/RustGenerator.cs @@ -305,6 +305,45 @@ private sealed record OperatorTrait(string Name, string Method, bool ReturnsBool /// public override string FileExtension => "rs"; + /// + /// + /// The strict keywords and the ones reserved for later, from the 2018 edition on, which is the + /// first to have raw identifiers. gen is reserved by 2024 and escaped here so the output + /// reads the same under it. + /// + protected override IReadOnlySet ReservedWords { get; } = new HashSet(StringComparer.Ordinal) + { + "as", "async", "await", "break", "const", "continue", "crate", "dyn", "else", "enum", "extern", "false", + "fn", "for", "if", "impl", "in", "let", "loop", "match", "mod", "move", "mut", "pub", "ref", "return", + "self", "Self", "static", "struct", "super", "trait", "true", "type", "unsafe", "use", "where", "while", + "abstract", "become", "box", "do", "final", "gen", "macro", "override", "priv", "try", "typeof", + "unsized", "virtual", "yield", + }; + + /// + /// + /// A raw identifier, r#type, which is the keyword used as a name rather than a different + /// name, so nothing is renamed. The four path keywords cannot be raw — r#self is an error — + /// so those, alone, have an underscore appended. + /// + protected override string EscapeIdentifier(string name) + { + Ensure.NotNull(name); + + if (!ReservedWords.Contains(name)) + { + return name; + } + + return name is "self" or "Self" or "crate" or "super" ? $"{name}_" : $"r#{name}"; + } + + /// + /// + /// The receiver, the path keywords that start a path, and the literal words. + /// + protected override bool IsReceiverWord(string word) => word is "self" or "Self" or "crate" or "super" or "true" or "false"; + /// /// /// Enforced rather than conventional: rustc warns on a member name that is not snake case, so an invented one in any other convention makes the generated file noisy to build. @@ -721,7 +760,7 @@ private void WriteAssociatedField(FieldDeclaration field, CodeBlocker code) TypeReference type = field.Type ?? new TypeReference(UnknownTypeName); - code.Write($"{SpellVisibilityKeyword(field.Visibility)}const {field.Name}: {SpellStorageType(type)} = "); + code.Write($"{SpellVisibilityKeyword(field.Visibility)}const {EscapeIdentifier(field.Name ?? string.Empty)}: {SpellStorageType(type)} = "); if (type.IsArray) { @@ -745,7 +784,7 @@ private void WriteAssociatedField(FieldDeclaration field, CodeBlocker code) /// Where there is no constructor, says so instead. /// private void WriteStructMember(string? name, TypeReference? type, Visibility visibility, CodeBlocker code) => - code.WriteLine($"{SpellVisibilityKeyword(visibility)}{name}: {SpellType(type ?? new TypeReference(UnknownTypeName))},"); + code.WriteLine($"{SpellVisibilityKeyword(visibility)}{EscapeIdentifier(name ?? string.Empty)}: {SpellType(type ?? new TypeReference(UnknownTypeName))},"); /// /// Writes an interface as the trait it is. @@ -1384,11 +1423,11 @@ private bool ReturnsAValue(FunctionDeclaration funcDecl) => /// constructors, and an associated function returning Self is what every type writes /// instead. /// - private static string SpellFunctionName(FunctionDeclaration funcDecl) => funcDecl.Kind switch + private string SpellFunctionName(FunctionDeclaration funcDecl) => funcDecl.Kind switch { FunctionKind.Constructor => "new", FunctionKind.Destructor => "drop", - _ => funcDecl.Name ?? "unnamed_function", + _ => EscapeIdentifier(funcDecl.Name ?? "unnamed_function"), }; /// @@ -1612,7 +1651,7 @@ protected override void GenerateParameter(Parameter parameter, CodeBlocker code, code.Write("mut "); } - code.Write($"{parameter.Name ?? $"param{position}"}: "); + code.Write($"{EscapeIdentifier(parameter.Name ?? $"param{position}")}: "); code.Write(SpellType(parameter.Type ?? new TypeReference(UnknownTypeName))); if (parameter.IsOptional && !string.IsNullOrEmpty(parameter.DefaultValue)) @@ -1640,7 +1679,7 @@ protected override void GenerateVariableDeclaration(VariableDeclaration varDecl, RegisterStaticType(varDecl); code.Write(varDecl.IsConstant ? "let " : "let mut "); - code.Write(varDecl.Name); + code.Write(EscapeIdentifier(varDecl.Name)); bool inferred = varDecl.IsTypeInferred || varDecl.Type is null; TypeReference? annotation = !inferred ? varDecl.Type @@ -1694,7 +1733,7 @@ protected override void GenerateFieldDeclaration(FieldDeclaration field, CodeBlo // holds. That is a requirement rather than a value with something missing. if (insideTrait && field.InitialValue is null) { - code.Write($"const {field.Name}: {SpellStorageType(type)}"); + code.Write($"const {EscapeIdentifier(field.Name ?? string.Empty)}: {SpellStorageType(type)}"); EndStatement(code); return; } @@ -1709,7 +1748,7 @@ protected override void GenerateFieldDeclaration(FieldDeclaration field, CodeBlo string storage = IsAssociatedItem || field.IsConstant ? "const" : "static"; code.Write($"{visibility}{storage} "); - code.Write($"{field.Name}: {SpellStorageType(type)} = "); + code.Write($"{EscapeIdentifier(field.Name ?? string.Empty)}: {SpellStorageType(type)} = "); // A slice borrows the array literal it is written from, which is what makes a table with no // length a constant at all. @@ -1959,7 +1998,7 @@ protected override void WriteListElement(AstNode argument, CodeBlocker code) if (argument is MemberInitialiser { Value: VariableReference reference } designated && string.Equals(designated.Name, reference.Name, StringComparison.Ordinal)) { - code.Write(reference.Name); + code.Write(EscapeReference(reference.Name)); return; } @@ -1972,7 +2011,7 @@ protected override void WriteListElement(AstNode argument, CodeBlocker code) literalType = null; try { - WriteDesignator(field.Name ?? string.Empty, code); + WriteDesignator(EscapeIdentifier(field.Name ?? string.Empty), code); WriteValue(value, fieldType, code); } finally diff --git a/Coder/Languages/StandardLanguageGenerator.cs b/Coder/Languages/StandardLanguageGenerator.cs index a50bda3..a068211 100644 --- a/Coder/Languages/StandardLanguageGenerator.cs +++ b/Coder/Languages/StandardLanguageGenerator.cs @@ -1381,7 +1381,7 @@ protected virtual void WriteListElement(AstNode argument, CodeBlocker code) { if (argument is MemberInitialiser designated) { - WriteDesignator(designated.Name ?? string.Empty, code); + WriteDesignator(EscapeIdentifier(designated.Name ?? string.Empty), code); WriteListValue(designated.Value!, code); return; }