diff --git a/Cargo.lock b/Cargo.lock index d25a9e9d2d..41007b15cc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -834,6 +834,7 @@ dependencies = [ "rayon", "sqlbuild-config", "sqlbuild-core", + "sqlbuild-model-config", "sqlbuild-sqltext", "tempfile", ] diff --git a/Cargo.toml b/Cargo.toml index 97a9c2b859..d142b6768f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,10 +26,10 @@ order = [ "sqlbuild-cache", "sqlbuild-sqltext", "sqlbuild-config", + "sqlbuild-model-config", "sqlbuild-discovery", "sqlbuild-render", "sqlbuild-scopes", - "sqlbuild-model-config", "sqlbuild-attachments", "sqlbuild-analysis", "sqlbuild-rules", diff --git a/crates/sqlbuild-core/src/text/_helpers/mod.rs b/crates/sqlbuild-core/src/text/_helpers/mod.rs index a24bcdea58..7e429641a5 100644 --- a/crates/sqlbuild-core/src/text/_helpers/mod.rs +++ b/crates/sqlbuild-core/src/text/_helpers/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod cleandoc; pub(crate) mod decimal_digits; pub(crate) mod glob; +pub(crate) mod python_tables; diff --git a/crates/sqlbuild-core/src/text/_helpers/python_tables.rs b/crates/sqlbuild-core/src/text/_helpers/python_tables.rs new file mode 100644 index 0000000000..451667160f --- /dev/null +++ b/crates/sqlbuild-core/src/text/_helpers/python_tables.rs @@ -0,0 +1,688 @@ +//! Where Python's `str` predicates differ from Rust's `char` methods, from CPython 3.12's tables. + +/// Characters `char::is_alphanumeric` accepts and `str.isalnum` rejects, as inclusive ranges. +pub(crate) const NOT_PYTHON_ALNUM: [(u32, u32); 296] = [ + (0x0345, 0x0345), + (0x0363, 0x036F), + (0x05B0, 0x05BD), + (0x05BF, 0x05BF), + (0x05C1, 0x05C2), + (0x05C4, 0x05C5), + (0x05C7, 0x05C7), + (0x0610, 0x061A), + (0x064B, 0x0657), + (0x0659, 0x065F), + (0x0670, 0x0670), + (0x06D6, 0x06DC), + (0x06E1, 0x06E4), + (0x06E7, 0x06E8), + (0x06ED, 0x06ED), + (0x0711, 0x0711), + (0x0730, 0x073F), + (0x07A6, 0x07B0), + (0x0816, 0x0817), + (0x081B, 0x0823), + (0x0825, 0x0827), + (0x0829, 0x082C), + (0x088F, 0x088F), + (0x0897, 0x0897), + (0x08D4, 0x08DF), + (0x08E3, 0x08E9), + (0x08F0, 0x0903), + (0x093A, 0x093B), + (0x093E, 0x094C), + (0x094E, 0x094F), + (0x0955, 0x0957), + (0x0962, 0x0963), + (0x0981, 0x0983), + (0x09BE, 0x09C4), + (0x09C7, 0x09C8), + (0x09CB, 0x09CC), + (0x09D7, 0x09D7), + (0x09E2, 0x09E3), + (0x0A01, 0x0A03), + (0x0A3E, 0x0A42), + (0x0A47, 0x0A48), + (0x0A4B, 0x0A4C), + (0x0A51, 0x0A51), + (0x0A70, 0x0A71), + (0x0A75, 0x0A75), + (0x0A81, 0x0A83), + (0x0ABE, 0x0AC5), + (0x0AC7, 0x0AC9), + (0x0ACB, 0x0ACC), + (0x0AE2, 0x0AE3), + (0x0AFA, 0x0AFC), + (0x0B01, 0x0B03), + (0x0B3E, 0x0B44), + (0x0B47, 0x0B48), + (0x0B4B, 0x0B4C), + (0x0B56, 0x0B57), + (0x0B62, 0x0B63), + (0x0B82, 0x0B82), + (0x0BBE, 0x0BC2), + (0x0BC6, 0x0BC8), + (0x0BCA, 0x0BCC), + (0x0BD7, 0x0BD7), + (0x0C00, 0x0C04), + (0x0C3E, 0x0C44), + (0x0C46, 0x0C48), + (0x0C4A, 0x0C4C), + (0x0C55, 0x0C56), + (0x0C5C, 0x0C5C), + (0x0C62, 0x0C63), + (0x0C81, 0x0C83), + (0x0CBE, 0x0CC4), + (0x0CC6, 0x0CC8), + (0x0CCA, 0x0CCC), + (0x0CD5, 0x0CD6), + (0x0CDC, 0x0CDC), + (0x0CE2, 0x0CE3), + (0x0CF3, 0x0CF3), + (0x0D00, 0x0D03), + (0x0D3E, 0x0D44), + (0x0D46, 0x0D48), + (0x0D4A, 0x0D4C), + (0x0D57, 0x0D57), + (0x0D62, 0x0D63), + (0x0D81, 0x0D83), + (0x0DCF, 0x0DD4), + (0x0DD6, 0x0DD6), + (0x0DD8, 0x0DDF), + (0x0DF2, 0x0DF3), + (0x0E31, 0x0E31), + (0x0E34, 0x0E3A), + (0x0E4D, 0x0E4D), + (0x0EB1, 0x0EB1), + (0x0EB4, 0x0EB9), + (0x0EBB, 0x0EBC), + (0x0ECD, 0x0ECD), + (0x0F71, 0x0F83), + (0x0F8D, 0x0F97), + (0x0F99, 0x0FBC), + (0x102B, 0x1036), + (0x1038, 0x1038), + (0x103B, 0x103E), + (0x1056, 0x1059), + (0x105E, 0x1060), + (0x1062, 0x1064), + (0x1067, 0x106D), + (0x1071, 0x1074), + (0x1082, 0x108D), + (0x108F, 0x108F), + (0x109A, 0x109D), + (0x1712, 0x1713), + (0x1732, 0x1733), + (0x1752, 0x1753), + (0x1772, 0x1773), + (0x17B6, 0x17C8), + (0x1885, 0x1886), + (0x18A9, 0x18A9), + (0x1920, 0x192B), + (0x1930, 0x1938), + (0x1A17, 0x1A1B), + (0x1A55, 0x1A5E), + (0x1A61, 0x1A74), + (0x1ABF, 0x1AC0), + (0x1ACC, 0x1ACE), + (0x1B00, 0x1B04), + (0x1B35, 0x1B43), + (0x1B80, 0x1B82), + (0x1BA1, 0x1BA9), + (0x1BAC, 0x1BAD), + (0x1BE7, 0x1BF1), + (0x1C24, 0x1C36), + (0x1C89, 0x1C8A), + (0x1DD3, 0x1DF4), + (0x24B6, 0x24E9), + (0x2DE0, 0x2DFF), + (0xA674, 0xA67B), + (0xA69E, 0xA69F), + (0xA7CB, 0xA7CF), + (0xA7D2, 0xA7D2), + (0xA7D4, 0xA7D4), + (0xA7DA, 0xA7DC), + (0xA7F1, 0xA7F1), + (0xA802, 0xA802), + (0xA80B, 0xA80B), + (0xA823, 0xA827), + (0xA880, 0xA881), + (0xA8B4, 0xA8C3), + (0xA8C5, 0xA8C5), + (0xA8FF, 0xA8FF), + (0xA926, 0xA92A), + (0xA947, 0xA952), + (0xA980, 0xA983), + (0xA9B4, 0xA9BF), + (0xA9E5, 0xA9E5), + (0xAA29, 0xAA36), + (0xAA43, 0xAA43), + (0xAA4C, 0xAA4D), + (0xAA7B, 0xAA7D), + (0xAAB0, 0xAAB0), + (0xAAB2, 0xAAB4), + (0xAAB7, 0xAAB8), + (0xAABE, 0xAABE), + (0xAAEB, 0xAAEF), + (0xAAF5, 0xAAF5), + (0xABE3, 0xABEA), + (0xFB1E, 0xFB1E), + (0x10376, 0x1037A), + (0x105C0, 0x105F3), + (0x10940, 0x10959), + (0x10A01, 0x10A03), + (0x10A05, 0x10A06), + (0x10A0C, 0x10A0F), + (0x10D24, 0x10D27), + (0x10D40, 0x10D65), + (0x10D69, 0x10D69), + (0x10D6F, 0x10D85), + (0x10EAB, 0x10EAC), + (0x10EC2, 0x10EC7), + (0x10EFA, 0x10EFC), + (0x11000, 0x11002), + (0x11038, 0x11045), + (0x11073, 0x11074), + (0x11080, 0x11082), + (0x110B0, 0x110B8), + (0x110C2, 0x110C2), + (0x11100, 0x11102), + (0x11127, 0x11132), + (0x11145, 0x11146), + (0x11180, 0x11182), + (0x111B3, 0x111BF), + (0x111CE, 0x111CF), + (0x1122C, 0x11234), + (0x11237, 0x11237), + (0x1123E, 0x1123E), + (0x11241, 0x11241), + (0x112DF, 0x112E8), + (0x11300, 0x11303), + (0x1133E, 0x11344), + (0x11347, 0x11348), + (0x1134B, 0x1134C), + (0x11357, 0x11357), + (0x11362, 0x11363), + (0x11380, 0x11389), + (0x1138B, 0x1138B), + (0x1138E, 0x1138E), + (0x11390, 0x113B5), + (0x113B7, 0x113C0), + (0x113C2, 0x113C2), + (0x113C5, 0x113C5), + (0x113C7, 0x113CA), + (0x113CC, 0x113CD), + (0x113D1, 0x113D1), + (0x113D3, 0x113D3), + (0x11435, 0x11441), + (0x11443, 0x11445), + (0x114B0, 0x114C1), + (0x115AF, 0x115B5), + (0x115B8, 0x115BE), + (0x115DC, 0x115DD), + (0x11630, 0x1163E), + (0x11640, 0x11640), + (0x116AB, 0x116B5), + (0x116D0, 0x116E3), + (0x1171D, 0x1172A), + (0x1182C, 0x11838), + (0x11930, 0x11935), + (0x11937, 0x11938), + (0x1193B, 0x1193C), + (0x11940, 0x11940), + (0x11942, 0x11942), + (0x119D1, 0x119D7), + (0x119DA, 0x119DF), + (0x119E4, 0x119E4), + (0x11A01, 0x11A0A), + (0x11A35, 0x11A39), + (0x11A3B, 0x11A3E), + (0x11A51, 0x11A5B), + (0x11A8A, 0x11A97), + (0x11B60, 0x11B67), + (0x11BC0, 0x11BE0), + (0x11BF0, 0x11BF9), + (0x11C2F, 0x11C36), + (0x11C38, 0x11C3E), + (0x11C92, 0x11CA7), + (0x11CA9, 0x11CB6), + (0x11D31, 0x11D36), + (0x11D3A, 0x11D3A), + (0x11D3C, 0x11D3D), + (0x11D3F, 0x11D41), + (0x11D43, 0x11D43), + (0x11D47, 0x11D47), + (0x11D8A, 0x11D8E), + (0x11D90, 0x11D91), + (0x11D93, 0x11D96), + (0x11DB0, 0x11DDB), + (0x11DE0, 0x11DE9), + (0x11EF3, 0x11EF6), + (0x11F00, 0x11F01), + (0x11F03, 0x11F03), + (0x11F34, 0x11F3A), + (0x11F3E, 0x11F40), + (0x13460, 0x143FA), + (0x16100, 0x1612E), + (0x16130, 0x16139), + (0x16D40, 0x16D6C), + (0x16D70, 0x16D79), + (0x16EA0, 0x16EB8), + (0x16EBB, 0x16ED3), + (0x16F4F, 0x16F4F), + (0x16F51, 0x16F87), + (0x16F8F, 0x16F92), + (0x16FF0, 0x16FF6), + (0x187F8, 0x187FF), + (0x18CFF, 0x18CFF), + (0x18D09, 0x18D1E), + (0x18D80, 0x18DF2), + (0x1BC9E, 0x1BC9E), + (0x1CCF0, 0x1CCF9), + (0x1E000, 0x1E006), + (0x1E008, 0x1E018), + (0x1E01B, 0x1E021), + (0x1E023, 0x1E024), + (0x1E026, 0x1E02A), + (0x1E08F, 0x1E08F), + (0x1E5D0, 0x1E5ED), + (0x1E5F0, 0x1E5FA), + (0x1E6C0, 0x1E6DE), + (0x1E6E0, 0x1E6F5), + (0x1E6FE, 0x1E6FF), + (0x1E947, 0x1E947), + (0x1F130, 0x1F149), + (0x1F150, 0x1F169), + (0x1F170, 0x1F189), + (0x2B73A, 0x2B73F), + (0x2CEA2, 0x2CEAD), + (0x2EBF0, 0x2EE5D), + (0x323B0, 0x33479), +]; + +/// Characters `str.isdigit` accepts beyond the decimal digits, as inclusive ranges. +pub(crate) const PYTHON_NON_DECIMAL_DIGITS: [(u32, u32); 20] = [ + (0x00B2, 0x00B3), + (0x00B9, 0x00B9), + (0x1369, 0x1371), + (0x19DA, 0x19DA), + (0x2070, 0x2070), + (0x2074, 0x2079), + (0x2080, 0x2089), + (0x2460, 0x2468), + (0x2474, 0x247C), + (0x2488, 0x2490), + (0x24EA, 0x24EA), + (0x24F5, 0x24FD), + (0x24FF, 0x24FF), + (0x2776, 0x277E), + (0x2780, 0x2788), + (0x278A, 0x2792), + (0x10A40, 0x10A43), + (0x10E60, 0x10E68), + (0x11052, 0x1105A), + (0x1F100, 0x1F10A), +]; + +/// `str.casefold` of each character whose casefold differs from `char::to_lowercase`. +pub(crate) const PYTHON_CASEFOLD: [(u32, &str); 352] = [ + (0x00B5, "\u{3bc}"), + (0x00DF, "ss"), + (0x0149, "\u{2bc}n"), + (0x017F, "s"), + (0x01F0, "j\u{30c}"), + (0x0345, "\u{3b9}"), + (0x0390, "\u{3b9}\u{308}\u{301}"), + (0x03B0, "\u{3c5}\u{308}\u{301}"), + (0x03C2, "\u{3c3}"), + (0x03D0, "\u{3b2}"), + (0x03D1, "\u{3b8}"), + (0x03D5, "\u{3c6}"), + (0x03D6, "\u{3c0}"), + (0x03F0, "\u{3ba}"), + (0x03F1, "\u{3c1}"), + (0x03F5, "\u{3b5}"), + (0x0587, "\u{565}\u{582}"), + (0x13A0, "\u{13a0}"), + (0x13A1, "\u{13a1}"), + (0x13A2, "\u{13a2}"), + (0x13A3, "\u{13a3}"), + (0x13A4, "\u{13a4}"), + (0x13A5, "\u{13a5}"), + (0x13A6, "\u{13a6}"), + (0x13A7, "\u{13a7}"), + (0x13A8, "\u{13a8}"), + (0x13A9, "\u{13a9}"), + (0x13AA, "\u{13aa}"), + (0x13AB, "\u{13ab}"), + (0x13AC, "\u{13ac}"), + (0x13AD, "\u{13ad}"), + (0x13AE, "\u{13ae}"), + (0x13AF, "\u{13af}"), + (0x13B0, "\u{13b0}"), + (0x13B1, "\u{13b1}"), + (0x13B2, "\u{13b2}"), + (0x13B3, "\u{13b3}"), + (0x13B4, "\u{13b4}"), + (0x13B5, "\u{13b5}"), + (0x13B6, "\u{13b6}"), + (0x13B7, "\u{13b7}"), + (0x13B8, "\u{13b8}"), + (0x13B9, "\u{13b9}"), + (0x13BA, "\u{13ba}"), + (0x13BB, "\u{13bb}"), + (0x13BC, "\u{13bc}"), + (0x13BD, "\u{13bd}"), + (0x13BE, "\u{13be}"), + (0x13BF, "\u{13bf}"), + (0x13C0, "\u{13c0}"), + (0x13C1, "\u{13c1}"), + (0x13C2, "\u{13c2}"), + (0x13C3, "\u{13c3}"), + (0x13C4, "\u{13c4}"), + (0x13C5, "\u{13c5}"), + (0x13C6, "\u{13c6}"), + (0x13C7, "\u{13c7}"), + (0x13C8, "\u{13c8}"), + (0x13C9, "\u{13c9}"), + (0x13CA, "\u{13ca}"), + (0x13CB, "\u{13cb}"), + (0x13CC, "\u{13cc}"), + (0x13CD, "\u{13cd}"), + (0x13CE, "\u{13ce}"), + (0x13CF, "\u{13cf}"), + (0x13D0, "\u{13d0}"), + (0x13D1, "\u{13d1}"), + (0x13D2, "\u{13d2}"), + (0x13D3, "\u{13d3}"), + (0x13D4, "\u{13d4}"), + (0x13D5, "\u{13d5}"), + (0x13D6, "\u{13d6}"), + (0x13D7, "\u{13d7}"), + (0x13D8, "\u{13d8}"), + (0x13D9, "\u{13d9}"), + (0x13DA, "\u{13da}"), + (0x13DB, "\u{13db}"), + (0x13DC, "\u{13dc}"), + (0x13DD, "\u{13dd}"), + (0x13DE, "\u{13de}"), + (0x13DF, "\u{13df}"), + (0x13E0, "\u{13e0}"), + (0x13E1, "\u{13e1}"), + (0x13E2, "\u{13e2}"), + (0x13E3, "\u{13e3}"), + (0x13E4, "\u{13e4}"), + (0x13E5, "\u{13e5}"), + (0x13E6, "\u{13e6}"), + (0x13E7, "\u{13e7}"), + (0x13E8, "\u{13e8}"), + (0x13E9, "\u{13e9}"), + (0x13EA, "\u{13ea}"), + (0x13EB, "\u{13eb}"), + (0x13EC, "\u{13ec}"), + (0x13ED, "\u{13ed}"), + (0x13EE, "\u{13ee}"), + (0x13EF, "\u{13ef}"), + (0x13F0, "\u{13f0}"), + (0x13F1, "\u{13f1}"), + (0x13F2, "\u{13f2}"), + (0x13F3, "\u{13f3}"), + (0x13F4, "\u{13f4}"), + (0x13F5, "\u{13f5}"), + (0x13F8, "\u{13f0}"), + (0x13F9, "\u{13f1}"), + (0x13FA, "\u{13f2}"), + (0x13FB, "\u{13f3}"), + (0x13FC, "\u{13f4}"), + (0x13FD, "\u{13f5}"), + (0x1C80, "\u{432}"), + (0x1C81, "\u{434}"), + (0x1C82, "\u{43e}"), + (0x1C83, "\u{441}"), + (0x1C84, "\u{442}"), + (0x1C85, "\u{442}"), + (0x1C86, "\u{44a}"), + (0x1C87, "\u{463}"), + (0x1C88, "\u{a64b}"), + (0x1C89, "\u{1c89}"), + (0x1E96, "h\u{331}"), + (0x1E97, "t\u{308}"), + (0x1E98, "w\u{30a}"), + (0x1E99, "y\u{30a}"), + (0x1E9A, "a\u{2be}"), + (0x1E9B, "\u{1e61}"), + (0x1E9E, "ss"), + (0x1F50, "\u{3c5}\u{313}"), + (0x1F52, "\u{3c5}\u{313}\u{300}"), + (0x1F54, "\u{3c5}\u{313}\u{301}"), + (0x1F56, "\u{3c5}\u{313}\u{342}"), + (0x1F80, "\u{1f00}\u{3b9}"), + (0x1F81, "\u{1f01}\u{3b9}"), + (0x1F82, "\u{1f02}\u{3b9}"), + (0x1F83, "\u{1f03}\u{3b9}"), + (0x1F84, "\u{1f04}\u{3b9}"), + (0x1F85, "\u{1f05}\u{3b9}"), + (0x1F86, "\u{1f06}\u{3b9}"), + (0x1F87, "\u{1f07}\u{3b9}"), + (0x1F88, "\u{1f00}\u{3b9}"), + (0x1F89, "\u{1f01}\u{3b9}"), + (0x1F8A, "\u{1f02}\u{3b9}"), + (0x1F8B, "\u{1f03}\u{3b9}"), + (0x1F8C, "\u{1f04}\u{3b9}"), + (0x1F8D, "\u{1f05}\u{3b9}"), + (0x1F8E, "\u{1f06}\u{3b9}"), + (0x1F8F, "\u{1f07}\u{3b9}"), + (0x1F90, "\u{1f20}\u{3b9}"), + (0x1F91, "\u{1f21}\u{3b9}"), + (0x1F92, "\u{1f22}\u{3b9}"), + (0x1F93, "\u{1f23}\u{3b9}"), + (0x1F94, "\u{1f24}\u{3b9}"), + (0x1F95, "\u{1f25}\u{3b9}"), + (0x1F96, "\u{1f26}\u{3b9}"), + (0x1F97, "\u{1f27}\u{3b9}"), + (0x1F98, "\u{1f20}\u{3b9}"), + (0x1F99, "\u{1f21}\u{3b9}"), + (0x1F9A, "\u{1f22}\u{3b9}"), + (0x1F9B, "\u{1f23}\u{3b9}"), + (0x1F9C, "\u{1f24}\u{3b9}"), + (0x1F9D, "\u{1f25}\u{3b9}"), + (0x1F9E, "\u{1f26}\u{3b9}"), + (0x1F9F, "\u{1f27}\u{3b9}"), + (0x1FA0, "\u{1f60}\u{3b9}"), + (0x1FA1, "\u{1f61}\u{3b9}"), + (0x1FA2, "\u{1f62}\u{3b9}"), + (0x1FA3, "\u{1f63}\u{3b9}"), + (0x1FA4, "\u{1f64}\u{3b9}"), + (0x1FA5, "\u{1f65}\u{3b9}"), + (0x1FA6, "\u{1f66}\u{3b9}"), + (0x1FA7, "\u{1f67}\u{3b9}"), + (0x1FA8, "\u{1f60}\u{3b9}"), + (0x1FA9, "\u{1f61}\u{3b9}"), + (0x1FAA, "\u{1f62}\u{3b9}"), + (0x1FAB, "\u{1f63}\u{3b9}"), + (0x1FAC, "\u{1f64}\u{3b9}"), + (0x1FAD, "\u{1f65}\u{3b9}"), + (0x1FAE, "\u{1f66}\u{3b9}"), + (0x1FAF, "\u{1f67}\u{3b9}"), + (0x1FB2, "\u{1f70}\u{3b9}"), + (0x1FB3, "\u{3b1}\u{3b9}"), + (0x1FB4, "\u{3ac}\u{3b9}"), + (0x1FB6, "\u{3b1}\u{342}"), + (0x1FB7, "\u{3b1}\u{342}\u{3b9}"), + (0x1FBC, "\u{3b1}\u{3b9}"), + (0x1FBE, "\u{3b9}"), + (0x1FC2, "\u{1f74}\u{3b9}"), + (0x1FC3, "\u{3b7}\u{3b9}"), + (0x1FC4, "\u{3ae}\u{3b9}"), + (0x1FC6, "\u{3b7}\u{342}"), + (0x1FC7, "\u{3b7}\u{342}\u{3b9}"), + (0x1FCC, "\u{3b7}\u{3b9}"), + (0x1FD2, "\u{3b9}\u{308}\u{300}"), + (0x1FD3, "\u{3b9}\u{308}\u{301}"), + (0x1FD6, "\u{3b9}\u{342}"), + (0x1FD7, "\u{3b9}\u{308}\u{342}"), + (0x1FE2, "\u{3c5}\u{308}\u{300}"), + (0x1FE3, "\u{3c5}\u{308}\u{301}"), + (0x1FE4, "\u{3c1}\u{313}"), + (0x1FE6, "\u{3c5}\u{342}"), + (0x1FE7, "\u{3c5}\u{308}\u{342}"), + (0x1FF2, "\u{1f7c}\u{3b9}"), + (0x1FF3, "\u{3c9}\u{3b9}"), + (0x1FF4, "\u{3ce}\u{3b9}"), + (0x1FF6, "\u{3c9}\u{342}"), + (0x1FF7, "\u{3c9}\u{342}\u{3b9}"), + (0x1FFC, "\u{3c9}\u{3b9}"), + (0xA7CB, "\u{a7cb}"), + (0xA7CC, "\u{a7cc}"), + (0xA7CE, "\u{a7ce}"), + (0xA7D2, "\u{a7d2}"), + (0xA7D4, "\u{a7d4}"), + (0xA7DA, "\u{a7da}"), + (0xA7DC, "\u{a7dc}"), + (0xAB70, "\u{13a0}"), + (0xAB71, "\u{13a1}"), + (0xAB72, "\u{13a2}"), + (0xAB73, "\u{13a3}"), + (0xAB74, "\u{13a4}"), + (0xAB75, "\u{13a5}"), + (0xAB76, "\u{13a6}"), + (0xAB77, "\u{13a7}"), + (0xAB78, "\u{13a8}"), + (0xAB79, "\u{13a9}"), + (0xAB7A, "\u{13aa}"), + (0xAB7B, "\u{13ab}"), + (0xAB7C, "\u{13ac}"), + (0xAB7D, "\u{13ad}"), + (0xAB7E, "\u{13ae}"), + (0xAB7F, "\u{13af}"), + (0xAB80, "\u{13b0}"), + (0xAB81, "\u{13b1}"), + (0xAB82, "\u{13b2}"), + (0xAB83, "\u{13b3}"), + (0xAB84, "\u{13b4}"), + (0xAB85, "\u{13b5}"), + (0xAB86, "\u{13b6}"), + (0xAB87, "\u{13b7}"), + (0xAB88, "\u{13b8}"), + (0xAB89, "\u{13b9}"), + (0xAB8A, "\u{13ba}"), + (0xAB8B, "\u{13bb}"), + (0xAB8C, "\u{13bc}"), + (0xAB8D, "\u{13bd}"), + (0xAB8E, "\u{13be}"), + (0xAB8F, "\u{13bf}"), + (0xAB90, "\u{13c0}"), + (0xAB91, "\u{13c1}"), + (0xAB92, "\u{13c2}"), + (0xAB93, "\u{13c3}"), + (0xAB94, "\u{13c4}"), + (0xAB95, "\u{13c5}"), + (0xAB96, "\u{13c6}"), + (0xAB97, "\u{13c7}"), + (0xAB98, "\u{13c8}"), + (0xAB99, "\u{13c9}"), + (0xAB9A, "\u{13ca}"), + (0xAB9B, "\u{13cb}"), + (0xAB9C, "\u{13cc}"), + (0xAB9D, "\u{13cd}"), + (0xAB9E, "\u{13ce}"), + (0xAB9F, "\u{13cf}"), + (0xABA0, "\u{13d0}"), + (0xABA1, "\u{13d1}"), + (0xABA2, "\u{13d2}"), + (0xABA3, "\u{13d3}"), + (0xABA4, "\u{13d4}"), + (0xABA5, "\u{13d5}"), + (0xABA6, "\u{13d6}"), + (0xABA7, "\u{13d7}"), + (0xABA8, "\u{13d8}"), + (0xABA9, "\u{13d9}"), + (0xABAA, "\u{13da}"), + (0xABAB, "\u{13db}"), + (0xABAC, "\u{13dc}"), + (0xABAD, "\u{13dd}"), + (0xABAE, "\u{13de}"), + (0xABAF, "\u{13df}"), + (0xABB0, "\u{13e0}"), + (0xABB1, "\u{13e1}"), + (0xABB2, "\u{13e2}"), + (0xABB3, "\u{13e3}"), + (0xABB4, "\u{13e4}"), + (0xABB5, "\u{13e5}"), + (0xABB6, "\u{13e6}"), + (0xABB7, "\u{13e7}"), + (0xABB8, "\u{13e8}"), + (0xABB9, "\u{13e9}"), + (0xABBA, "\u{13ea}"), + (0xABBB, "\u{13eb}"), + (0xABBC, "\u{13ec}"), + (0xABBD, "\u{13ed}"), + (0xABBE, "\u{13ee}"), + (0xABBF, "\u{13ef}"), + (0xFB00, "ff"), + (0xFB01, "fi"), + (0xFB02, "fl"), + (0xFB03, "ffi"), + (0xFB04, "ffl"), + (0xFB05, "st"), + (0xFB06, "st"), + (0xFB13, "\u{574}\u{576}"), + (0xFB14, "\u{574}\u{565}"), + (0xFB15, "\u{574}\u{56b}"), + (0xFB16, "\u{57e}\u{576}"), + (0xFB17, "\u{574}\u{56d}"), + (0x10D50, "\u{10d50}"), + (0x10D51, "\u{10d51}"), + (0x10D52, "\u{10d52}"), + (0x10D53, "\u{10d53}"), + (0x10D54, "\u{10d54}"), + (0x10D55, "\u{10d55}"), + (0x10D56, "\u{10d56}"), + (0x10D57, "\u{10d57}"), + (0x10D58, "\u{10d58}"), + (0x10D59, "\u{10d59}"), + (0x10D5A, "\u{10d5a}"), + (0x10D5B, "\u{10d5b}"), + (0x10D5C, "\u{10d5c}"), + (0x10D5D, "\u{10d5d}"), + (0x10D5E, "\u{10d5e}"), + (0x10D5F, "\u{10d5f}"), + (0x10D60, "\u{10d60}"), + (0x10D61, "\u{10d61}"), + (0x10D62, "\u{10d62}"), + (0x10D63, "\u{10d63}"), + (0x10D64, "\u{10d64}"), + (0x10D65, "\u{10d65}"), + (0x16EA0, "\u{16ea0}"), + (0x16EA1, "\u{16ea1}"), + (0x16EA2, "\u{16ea2}"), + (0x16EA3, "\u{16ea3}"), + (0x16EA4, "\u{16ea4}"), + (0x16EA5, "\u{16ea5}"), + (0x16EA6, "\u{16ea6}"), + (0x16EA7, "\u{16ea7}"), + (0x16EA8, "\u{16ea8}"), + (0x16EA9, "\u{16ea9}"), + (0x16EAA, "\u{16eaa}"), + (0x16EAB, "\u{16eab}"), + (0x16EAC, "\u{16eac}"), + (0x16EAD, "\u{16ead}"), + (0x16EAE, "\u{16eae}"), + (0x16EAF, "\u{16eaf}"), + (0x16EB0, "\u{16eb0}"), + (0x16EB1, "\u{16eb1}"), + (0x16EB2, "\u{16eb2}"), + (0x16EB3, "\u{16eb3}"), + (0x16EB4, "\u{16eb4}"), + (0x16EB5, "\u{16eb5}"), + (0x16EB6, "\u{16eb6}"), + (0x16EB7, "\u{16eb7}"), + (0x16EB8, "\u{16eb8}"), +]; + +/// Whether `character` falls in one of the sorted inclusive `ranges`. +pub(crate) fn in_ranges(ranges: &[(u32, u32)], character: char) -> bool { + let code: u32 = u32::from(character); + let index: usize = ranges.partition_point(|(_, end)| *end < code); + ranges.get(index).is_some_and(|(start, _)| *start <= code) +} diff --git a/crates/sqlbuild-core/src/text/constants.rs b/crates/sqlbuild-core/src/text/constants.rs index f678ed5afb..4e313e1612 100644 --- a/crates/sqlbuild-core/src/text/constants.rs +++ b/crates/sqlbuild-core/src/text/constants.rs @@ -1,2 +1,5 @@ /// The code points in one run of Unicode decimal digits, valued 0 to 9. pub(crate) const DIGITS_PER_RUN: u32 = 10; + +/// The information separators Python's `str.isspace` accepts and `char::is_whitespace` does not. +pub(crate) const PYTHON_SEPARATOR_CONTROLS: std::ops::RangeInclusive = '\u{1c}'..='\u{1f}'; diff --git a/crates/sqlbuild-core/src/text/main/mod.rs b/crates/sqlbuild-core/src/text/main/mod.rs index bac686af6a..d439d53a75 100644 --- a/crates/sqlbuild-core/src/text/main/mod.rs +++ b/crates/sqlbuild-core/src/text/main/mod.rs @@ -7,5 +7,10 @@ pub mod ignorecase_key; pub mod is_decimal_digit; pub mod is_letter; pub mod is_word_char; +pub mod python_casefold; pub mod python_cleandoc; +pub mod python_isalnum; +pub mod python_isdigit; +pub mod python_isspace; pub mod python_str_repr; +pub mod python_strip; diff --git a/crates/sqlbuild-core/src/text/main/python_casefold.rs b/crates/sqlbuild-core/src/text/main/python_casefold.rs new file mode 100644 index 0000000000..9039e62a20 --- /dev/null +++ b/crates/sqlbuild-core/src/text/main/python_casefold.rs @@ -0,0 +1,17 @@ +//! Python's `str.casefold`. + +use crate::text::_helpers::python_tables::PYTHON_CASEFOLD; + +/// Python's `text.casefold()`. +#[must_use] +pub fn python_casefold(text: &str) -> String { + let mut folded: String = String::with_capacity(text.len()); + for character in text.chars() { + let code: u32 = u32::from(character); + match PYTHON_CASEFOLD.binary_search_by_key(&code, |(entry, _)| *entry) { + Ok(index) => folded.push_str(PYTHON_CASEFOLD[index].1), + Err(_) => folded.extend(character.to_lowercase()), + } + } + folded +} diff --git a/crates/sqlbuild-core/src/text/main/python_isalnum.rs b/crates/sqlbuild-core/src/text/main/python_isalnum.rs new file mode 100644 index 0000000000..3c2e088c90 --- /dev/null +++ b/crates/sqlbuild-core/src/text/main/python_isalnum.rs @@ -0,0 +1,9 @@ +//! Python's `str.isalnum` for one character. + +use crate::text::_helpers::python_tables::{NOT_PYTHON_ALNUM, in_ranges}; + +/// Whether Python's `str.isalnum` accepts `character`. +#[must_use] +pub fn python_isalnum(character: char) -> bool { + character.is_alphanumeric() && !in_ranges(&NOT_PYTHON_ALNUM, character) +} diff --git a/crates/sqlbuild-core/src/text/main/python_isdigit.rs b/crates/sqlbuild-core/src/text/main/python_isdigit.rs new file mode 100644 index 0000000000..56456c76e0 --- /dev/null +++ b/crates/sqlbuild-core/src/text/main/python_isdigit.rs @@ -0,0 +1,10 @@ +//! Python's `str.isdigit` for one character. + +use crate::text::_helpers::python_tables::{PYTHON_NON_DECIMAL_DIGITS, in_ranges}; +use crate::text::main::is_decimal_digit::is_decimal_digit; + +/// Whether Python's `str.isdigit` accepts `character`: decimal digits and digit characters. +#[must_use] +pub fn python_isdigit(character: char) -> bool { + is_decimal_digit(character) || in_ranges(&PYTHON_NON_DECIMAL_DIGITS, character) +} diff --git a/crates/sqlbuild-core/src/text/main/python_isspace.rs b/crates/sqlbuild-core/src/text/main/python_isspace.rs new file mode 100644 index 0000000000..5888db8697 --- /dev/null +++ b/crates/sqlbuild-core/src/text/main/python_isspace.rs @@ -0,0 +1,9 @@ +//! Python's `str.isspace` for one character. + +use crate::text::constants::PYTHON_SEPARATOR_CONTROLS; + +/// Whether Python's `str.isspace` accepts `character`. +#[must_use] +pub fn python_isspace(character: char) -> bool { + character.is_whitespace() || PYTHON_SEPARATOR_CONTROLS.contains(&character) +} diff --git a/crates/sqlbuild-core/src/text/main/python_strip.rs b/crates/sqlbuild-core/src/text/main/python_strip.rs new file mode 100644 index 0000000000..73f195b728 --- /dev/null +++ b/crates/sqlbuild-core/src/text/main/python_strip.rs @@ -0,0 +1,9 @@ +//! Python's `str.strip()` with no arguments. + +use crate::text::main::python_isspace::python_isspace; + +/// Python's `text.strip()`. +#[must_use] +pub fn python_strip(text: &str) -> &str { + text.trim_matches(python_isspace) +} diff --git a/crates/sqlbuild-core/src/text/tests.rs b/crates/sqlbuild-core/src/text/tests.rs index 98247d0665..58c350466d 100644 --- a/crates/sqlbuild-core/src/text/tests.rs +++ b/crates/sqlbuild-core/src/text/tests.rs @@ -1,5 +1,7 @@ #[path = "tests/test_matching.rs"] mod matching; +#[path = "tests/test_python_str.rs"] +mod python_str; #[path = "tests/test_types.rs"] mod test_types; #[path = "tests/test_text.rs"] diff --git a/crates/sqlbuild-core/src/text/tests/test_python_str.rs b/crates/sqlbuild-core/src/text/tests/test_python_str.rs new file mode 100644 index 0000000000..cd5a678ed8 --- /dev/null +++ b/crates/sqlbuild-core/src/text/tests/test_python_str.rs @@ -0,0 +1,126 @@ +use crate::text::main::python_casefold::python_casefold; +use crate::text::main::python_isalnum::python_isalnum; +use crate::text::main::python_isdigit::python_isdigit; +use crate::text::main::python_isspace::python_isspace; +use crate::text::tests::test_types::{CasefoldTestCase, CharacterClassTestCase}; + +/// Expectations recorded from CPython 3.12. +#[test] +fn given_text_when_casefolding_then_matches_python_casefold() { + let test_cases = [ + CasefoldTestCase { + description: "final sigma folds to sigma", + text: "ΟΔΟΣ", + expected_folded: "οδοσ", + }, + CasefoldTestCase { + description: "lowercase final sigma", + text: "οδος", + expected_folded: "οδοσ", + }, + CasefoldTestCase { + description: "sharp s expands", + text: "Straße", + expected_folded: "strasse", + }, + CasefoldTestCase { + description: "ligature expands", + text: "\u{fb01}le", + expected_folded: "file", + }, + CasefoldTestCase { + description: "long s", + text: "\u{17f}", + expected_folded: "s", + }, + CasefoldTestCase { + description: "micro sign", + text: "\u{b5}", + expected_folded: "\u{3bc}", + }, + ]; + for test_case in test_cases { + assert_eq!( + python_casefold(test_case.text), + test_case.expected_folded, + "{}", + test_case.description + ); + } +} + +/// Expectations recorded from CPython 3.12. +#[test] +fn given_characters_when_classifying_then_matches_python_str_predicates() { + let test_cases = [ + CharacterClassTestCase { + description: "ascii letter", + character: 'a', + expected_alnum: true, + expected_digit: false, + expected_space: false, + }, + CharacterClassTestCase { + description: "combining vowel sign", + character: '\u{93e}', + expected_alnum: false, + expected_digit: false, + expected_space: false, + }, + CharacterClassTestCase { + description: "arabic-indic digit", + character: '٣', + expected_alnum: true, + expected_digit: true, + expected_space: false, + }, + CharacterClassTestCase { + description: "circled digit", + character: '①', + expected_alnum: true, + expected_digit: true, + expected_space: false, + }, + CharacterClassTestCase { + description: "subscript digit", + character: '₃', + expected_alnum: true, + expected_digit: true, + expected_space: false, + }, + CharacterClassTestCase { + description: "vulgar fraction", + character: '½', + expected_alnum: true, + expected_digit: false, + expected_space: false, + }, + CharacterClassTestCase { + description: "unit separator", + character: '\u{1f}', + expected_alnum: false, + expected_digit: false, + expected_space: true, + }, + ]; + for test_case in test_cases { + assert_eq!( + python_isalnum(test_case.character), + test_case.expected_alnum, + "{}", + test_case.description + ); + assert_eq!( + python_isdigit(test_case.character), + test_case.expected_digit, + "{}", + test_case.description + ); + assert_eq!( + python_isspace(test_case.character), + test_case.expected_space, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/text/tests/test_types.rs b/crates/sqlbuild-core/src/text/tests/test_types.rs index a7e100e90f..3eb58cfe4b 100644 --- a/crates/sqlbuild-core/src/text/tests/test_types.rs +++ b/crates/sqlbuild-core/src/text/tests/test_types.rs @@ -45,3 +45,17 @@ pub(super) struct GlobTestCase { pub(super) name: &'static str, pub(super) expected_match: bool, } + +pub(super) struct CasefoldTestCase { + pub(super) description: &'static str, + pub(super) text: &'static str, + pub(super) expected_folded: &'static str, +} + +pub(super) struct CharacterClassTestCase { + pub(super) description: &'static str, + pub(super) character: char, + pub(super) expected_alnum: bool, + pub(super) expected_digit: bool, + pub(super) expected_space: bool, +} diff --git a/crates/sqlbuild-discovery/Cargo.toml b/crates/sqlbuild-discovery/Cargo.toml index e5b3130fff..3b1132a79f 100644 --- a/crates/sqlbuild-discovery/Cargo.toml +++ b/crates/sqlbuild-discovery/Cargo.toml @@ -11,6 +11,7 @@ rust-version.workspace = true rayon.workspace = true sqlbuild-config.workspace = true sqlbuild-core.workspace = true +sqlbuild-model-config.workspace = true sqlbuild-sqltext.workspace = true [target.'cfg(windows)'.dependencies] diff --git a/crates/sqlbuild-discovery/src/_helpers/reading.rs b/crates/sqlbuild-discovery/src/_helpers/reading.rs index bbc3dea814..36f9bc3f2f 100644 --- a/crates/sqlbuild-discovery/src/_helpers/reading.rs +++ b/crates/sqlbuild-discovery/src/_helpers/reading.rs @@ -114,7 +114,7 @@ pub(crate) fn windows_errno(code: i32) -> i32 { } /// CPython's UTF-8 decoder error: the range and reason it reports for the first bad sequence. -fn decode_failure(bytes: Vec, error: TextDecodeError) -> ReadFailure { +pub(crate) fn decode_failure(bytes: Vec, error: TextDecodeError) -> ReadFailure { let start: usize = error.valid_up_to; let (end, reason): (usize, &'static str) = match error.error_len { None => (bytes.len(), "unexpected end of data"), diff --git a/crates/sqlbuild-discovery/src/lib.rs b/crates/sqlbuild-discovery/src/lib.rs index c15f5b2b32..e68026dc37 100644 --- a/crates/sqlbuild-discovery/src/lib.rs +++ b/crates/sqlbuild-discovery/src/lib.rs @@ -9,6 +9,8 @@ pub mod declarations; pub mod key_checks; pub mod model_files; pub mod models; +pub mod project_validation; pub mod sql_tests; pub mod tree; +pub mod yaml_entries; pub mod yaml_files; diff --git a/crates/sqlbuild-discovery/src/models.rs b/crates/sqlbuild-discovery/src/models.rs index b38c79864f..f00fcbd4df 100644 --- a/crates/sqlbuild-discovery/src/models.rs +++ b/crates/sqlbuild-discovery/src/models.rs @@ -26,6 +26,10 @@ pub enum FailureKind { SqlAudit, /// `ResourceIdentityError` (D016). ResourceIdentity, + /// `DiscoveryConflictError` (D007). + Conflict, + /// `SeedDiscoveryError` (D008). + Seed, } impl FailureKind { @@ -42,6 +46,8 @@ impl FailureKind { Self::SqlHook => "sql_hook", Self::SqlAudit => "sql_audit", Self::ResourceIdentity => "resource_identity", + Self::Conflict => "conflict", + Self::Seed => "seed", } } } diff --git a/crates/sqlbuild-discovery/src/project_validation/_helpers/conflicts.rs b/crates/sqlbuild-discovery/src/project_validation/_helpers/conflicts.rs new file mode 100644 index 0000000000..b949afbcaf --- /dev/null +++ b/crates/sqlbuild-discovery/src/project_validation/_helpers/conflicts.rs @@ -0,0 +1,60 @@ +//! Shared pieces of the conflict checks: the error, name tables and the duplicate scan. + +use std::collections::HashMap; + +use crate::models::{DiscoveryFailure, FailureKind}; +use crate::project_validation::models::{NamedPath, NodeFact, ProjectFacts, ValidationFailure}; + +/// A `DiscoveryConflictError` with this message. +pub(crate) fn conflict(message: String) -> ValidationFailure { + DiscoveryFailure::new(FailureKind::Conflict, message).into() +} + +/// Every schema seed declaration with the schema file that declares it. +pub(crate) fn schema_seeds(facts: &ProjectFacts) -> Vec { + let mut seeds: Vec = Vec::new(); + for schema in &facts.schemas { + for seed in &schema.seeds { + seeds.push(named(&seed.name, &schema.path)); + } + } + seeds +} + +/// Every source entry with the source file that declares it. +pub(crate) fn source_entries(facts: &ProjectFacts) -> Vec { + let mut entries: Vec = Vec::new(); + for file in &facts.sources { + for entry in &file.entries { + entries.push(named(&entry.name, &file.path)); + } + } + entries +} + +pub(crate) fn named(name: &str, path: &str) -> NamedPath { + NamedPath { + name: name.to_owned(), + path: path.to_owned(), + } +} + +pub(crate) fn node_path(node: &NodeFact) -> NamedPath { + named(&node.name, &node.path) +} + +/// Reject the second of two entries with the same name. +pub(crate) fn unique_named( + entries: &[impl std::borrow::Borrow], + message: impl Fn(&str, &str, &str) -> String, +) -> Result<(), ValidationFailure> { + let mut seen: HashMap<&str, &str> = HashMap::new(); + for entry in entries { + let entry: &NamedPath = entry.borrow(); + if let Some(existing) = seen.get(entry.name.as_str()) { + return Err(conflict(message(&entry.name, existing, &entry.path))); + } + seen.insert(&entry.name, &entry.path); + } + Ok(()) +} diff --git a/crates/sqlbuild-discovery/src/project_validation/_helpers/csv_header.rs b/crates/sqlbuild-discovery/src/project_validation/_helpers/csv_header.rs new file mode 100644 index 0000000000..1859d7aa66 --- /dev/null +++ b/crates/sqlbuild-discovery/src/project_validation/_helpers/csv_header.rs @@ -0,0 +1,355 @@ +//! The first record Python's `csv.reader` returns for a seed file opened with `newline=""`. + +use crate::_helpers::reading::decode_failure; +use crate::project_validation::errors::HeaderError; +use crate::project_validation::models::CsvSettings; +use sqlbuild_core::text::main::decode_text::decode_text; + +/// The size of the chunks `TextIOWrapper` decodes. +const DECODE_CHUNK: usize = 8192; +const UNQUOTED_NEWLINE: &str = + "new-line character seen in unquoted field - do you need to open the file with newline=''?"; + +/// The header row, `None` for an empty file, or the error Python raises reading it. +pub(crate) fn csv_header( + bytes: &[u8], + settings: &CsvSettings, + field_limit: usize, +) -> Result, HeaderError> { + let dialect: Dialect = Dialect::new(settings, field_limit); + let settings: &Dialect = &dialect; + let mut end: usize = bytes.len().min(DECODE_CHUNK); + loop { + let text: String = decoded_prefix(bytes, end)?; + let complete: bool = end == bytes.len(); + match first_record(&text, settings, complete)? { + Some(record) => return Ok(record), + None if complete => return Ok(Vec::new()), + None => end = bytes.len().min(end + DECODE_CHUNK), + } + } +} + +/// The reader's settings as `csv.reader` receives them; Python has already accepted them. +pub(crate) struct Dialect { + delimiter: char, + quotechar: Option, + escapechar: Option, + doublequote: bool, + skipinitialspace: bool, + field_limit: usize, +} + +impl Dialect { + fn new(settings: &CsvSettings, field_limit: usize) -> Self { + let single = |value: &Option| -> Option { + let mut characters = value.as_deref()?.chars(); + match (characters.next(), characters.next()) { + (Some(character), None) => Some(character), + _ => None, + } + }; + Self { + delimiter: single(&settings.delimiter).unwrap_or(','), + quotechar: single(&settings.quotechar), + escapechar: single(&settings.escapechar), + doublequote: settings.doublequote.unwrap_or(true), + skipinitialspace: settings.skipinitialspace.unwrap_or(false), + field_limit, + } + } +} + +/// The text of `bytes[..end]`; a multi-byte character cut by `end` waits for the next chunk. +fn decoded_prefix(bytes: &[u8], end: usize) -> Result { + let prefix: &[u8] = &bytes[..end]; + match std::str::from_utf8(prefix) { + Ok(text) => Ok(text.to_owned()), + Err(error) if error.error_len().is_none() && end < bytes.len() => { + Ok(String::from_utf8_lossy(&prefix[..error.valid_up_to()]).into_owned()) + } + Err(_) => decode_text(prefix) + .map_err(|error| HeaderError::Read(decode_failure(prefix.to_vec(), error))), + } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum State { + StartRecord, + StartField, + EscapedChar, + InField, + InQuotedField, + EscapeInQuotedField, + QuoteInQuotedField, + EatCrnl, + AfterEscapedCrnl, +} + +/// One character, or the end of a line. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Input { + Char(char), + Eol, +} + +struct Parser<'a> { + settings: &'a Dialect, + state: State, + field: String, + field_len: usize, + fields: Vec, +} + +impl Parser<'_> { + fn save_field(&mut self) { + self.fields.push(std::mem::take(&mut self.field)); + self.field_len = 0; + } + + fn add(&mut self, character: char) -> Result<(), HeaderError> { + if self.field_len >= self.settings.field_limit { + return Err(HeaderError::Csv(format!( + "field larger than field limit ({})", + self.settings.field_limit + ))); + } + self.field.push(character); + self.field_len += 1; + Ok(()) + } + + fn is_quote(&self, character: char) -> bool { + self.settings.quotechar == Some(character) + } + + fn is_escape(&self, character: char) -> bool { + self.settings.escapechar == Some(character) + } + + fn delimiter(&self) -> char { + self.settings.delimiter + } + + /// CPython's `parse_process_char`. + fn process(&mut self, input: Input) -> Result<(), HeaderError> { + let newline = |input: Input| matches!(input, Input::Char('\n' | '\r')); + loop { + match self.state { + State::StartRecord => { + if input == Input::Eol { + return Ok(()); + } + if newline(input) { + self.state = State::EatCrnl; + return Ok(()); + } + self.state = State::StartField; + } + State::StartField => { + match input { + Input::Eol => { + self.save_field(); + self.state = State::StartRecord; + } + Input::Char('\n' | '\r') => { + self.save_field(); + self.state = State::EatCrnl; + } + Input::Char(character) if self.is_quote(character) => { + self.state = State::InQuotedField; + } + Input::Char(character) if self.is_escape(character) => { + self.state = State::EscapedChar; + } + Input::Char(' ') if self.settings.skipinitialspace => {} + Input::Char(character) if character == self.delimiter() => { + self.save_field(); + } + Input::Char(character) => { + self.add(character)?; + self.state = State::InField; + } + } + return Ok(()); + } + State::EscapedChar => { + match input { + Input::Char(character @ ('\n' | '\r')) => { + self.add(character)?; + self.state = State::AfterEscapedCrnl; + } + Input::Eol => { + self.add('\n')?; + self.state = State::InField; + } + Input::Char(character) => { + self.add(character)?; + self.state = State::InField; + } + } + return Ok(()); + } + State::AfterEscapedCrnl => { + if input == Input::Eol { + return Ok(()); + } + self.state = State::InField; + } + State::InField => { + match input { + Input::Eol => { + self.save_field(); + self.state = State::StartRecord; + } + Input::Char('\n' | '\r') => { + self.save_field(); + self.state = State::EatCrnl; + } + Input::Char(character) if self.is_escape(character) => { + self.state = State::EscapedChar; + } + Input::Char(character) if character == self.delimiter() => { + self.save_field(); + self.state = State::StartField; + } + Input::Char(character) => self.add(character)?, + } + return Ok(()); + } + State::InQuotedField => { + match input { + Input::Eol => {} + Input::Char(character) if self.is_escape(character) => { + self.state = State::EscapeInQuotedField; + } + Input::Char(character) if self.is_quote(character) => { + self.state = if self.settings.doublequote { + State::QuoteInQuotedField + } else { + State::InField + }; + } + Input::Char(character) => self.add(character)?, + } + return Ok(()); + } + State::EscapeInQuotedField => { + self.add(match input { + Input::Eol => '\n', + Input::Char(character) => character, + })?; + self.state = State::InQuotedField; + return Ok(()); + } + State::QuoteInQuotedField => { + match input { + Input::Char(character) if self.is_quote(character) => { + self.add(character)?; + self.state = State::InQuotedField; + } + Input::Char(character) if character == self.delimiter() => { + self.save_field(); + self.state = State::StartField; + } + Input::Eol => { + self.save_field(); + self.state = State::StartRecord; + } + Input::Char('\n' | '\r') => { + self.save_field(); + self.state = State::EatCrnl; + } + Input::Char(character) => { + self.add(character)?; + self.state = State::InField; + } + } + return Ok(()); + } + State::EatCrnl => { + return match input { + Input::Char('\n' | '\r') => Ok(()), + Input::Eol => { + self.state = State::StartRecord; + Ok(()) + } + Input::Char(_) => Err(HeaderError::Csv(UNQUOTED_NEWLINE.to_owned())), + }; + } + } + } + } +} + +/// The first record of `text`, or `None` when `text` ends before it (`complete` is false) or is empty. +fn first_record( + text: &str, + settings: &Dialect, + complete: bool, +) -> Result>, HeaderError> { + let mut parser: Parser<'_> = Parser { + settings, + state: State::StartRecord, + field: String::new(), + field_len: 0, + fields: Vec::new(), + }; + let mut lines = lines(text).peekable(); + while let Some(line) = lines.next() { + let terminated: bool = line.ends_with(['\n', '\r']); + let last: bool = lines.peek().is_none(); + if last && !complete && !terminated_line(line, terminated, text) { + return Ok(None); + } + for character in line.chars() { + parser.process(Input::Char(character))?; + } + parser.process(Input::Eol)?; + if parser.state == State::StartRecord { + return Ok(Some(parser.fields)); + } + } + if !complete { + return Ok(None); + } + if parser.field_len != 0 || parser.state == State::InQuotedField { + parser.save_field(); + return Ok(Some(parser.fields)); + } + Ok(None) +} + +/// Whether a final line is known to be whole: it ends with `\n`, or `\r` not followed later. +fn terminated_line(line: &str, terminated: bool, text: &str) -> bool { + terminated && (line.ends_with('\n') || !text.ends_with('\r')) +} + +/// The lines of `text` as `newline=""` iteration yields them, terminators included. +fn lines(text: &str) -> impl Iterator { + let mut rest: &str = text; + std::iter::from_fn(move || { + if rest.is_empty() { + return None; + } + let bytes: &[u8] = rest.as_bytes(); + let mut end: usize = bytes.len(); + for (index, byte) in bytes.iter().enumerate() { + if *byte == b'\n' { + end = index + 1; + break; + } + if *byte == b'\r' { + end = if bytes.get(index + 1) == Some(&b'\n') { + index + 2 + } else { + index + 1 + }; + break; + } + } + let (line, tail) = rest.split_at(end); + rest = tail; + Some(line) + }) +} diff --git a/crates/sqlbuild-discovery/src/project_validation/_helpers/declarations.rs b/crates/sqlbuild-discovery/src/project_validation/_helpers/declarations.rs new file mode 100644 index 0000000000..f0832fbcf4 --- /dev/null +++ b/crates/sqlbuild-discovery/src/project_validation/_helpers/declarations.rs @@ -0,0 +1,48 @@ +//! Duplicate schema declarations and logical relation names shared across resource kinds. + +use std::collections::HashMap; + +use crate::project_validation::_helpers::conflicts::{ + conflict, named, schema_seeds, source_entries, unique_named, +}; +use crate::project_validation::models::{NamedPath, ProjectFacts, ValidationFailure}; + +/// Reject duplicate schema model and seed declarations, then shared relation names. +pub(crate) fn validate_declarations(facts: &ProjectFacts) -> Result<(), ValidationFailure> { + let mut schema_models: Vec = Vec::new(); + for schema in &facts.schemas { + for name in &schema.model_names { + schema_models.push(named(name, &schema.path)); + } + } + unique_named(&schema_models, |name, existing, path| { + format!( + "Duplicate schema.yml model declaration found for '{name}' in {existing} and {path}" + ) + })?; + unique_named(&schema_seeds(facts), |name, existing, path| { + format!("Duplicate seed declaration found for '{name}' in {existing} and {path}") + })?; + unique_logical_relation_names(facts) +} + +fn unique_logical_relation_names(facts: &ProjectFacts) -> Result<(), ValidationFailure> { + let mut seen: HashMap = HashMap::new(); + for model in &facts.models { + seen.insert(model.file.name.clone(), ("model", model.file.path.clone())); + } + let sources = source_entries(facts) + .into_iter() + .map(|entry| ("source", entry)); + let seeds = schema_seeds(facts).into_iter().map(|seed| ("seed", seed)); + for (kind, NamedPath { name, path }) in sources.chain(seeds) { + if let Some((existing_kind, existing_path)) = seen.get(&name) { + return Err(conflict(format!( + "Logical relation name '{name}' is declared as both {existing_kind} in \ + {existing_path} and {kind} in {path}" + ))); + } + seen.insert(name, (kind, path)); + } + Ok(()) +} diff --git a/crates/sqlbuild-discovery/src/project_validation/_helpers/file_names.rs b/crates/sqlbuild-discovery/src/project_validation/_helpers/file_names.rs new file mode 100644 index 0000000000..c216660682 --- /dev/null +++ b/crates/sqlbuild-discovery/src/project_validation/_helpers/file_names.rs @@ -0,0 +1,37 @@ +//! Duplicate and reserved names among model, scenario and source files. + +use std::collections::HashMap; + +use crate::project_validation::_helpers::conflicts::{conflict, source_entries, unique_named}; +use crate::project_validation::constants::RESERVED_MODEL_NAMES; +use crate::project_validation::models::{ModelFact, NamedPath, ProjectFacts, ValidationFailure}; + +/// Reject reserved or duplicate model names, then duplicate scenario and source names. +pub(crate) fn validate_file_names(facts: &ProjectFacts) -> Result<(), ValidationFailure> { + unique_model_file_names(&facts.models)?; + unique_named(&facts.scenarios, |name, existing, path| { + format!("Duplicate scenario file name found for '{name}' in {existing} and {path}") + })?; + unique_named(&source_entries(facts), |name, existing, path| { + format!("Duplicate source declaration found for '{name}' in {existing} and {path}") + }) +} + +fn unique_model_file_names(models: &[ModelFact]) -> Result<(), ValidationFailure> { + let mut seen: HashMap<&str, &str> = HashMap::new(); + for model in models { + let NamedPath { name, path } = &model.file; + if RESERVED_MODEL_NAMES.contains(&name.as_str()) { + return Err(conflict(format!( + "Model name '{name}' in {path} is reserved by SQLBuild for compiled output structure" + ))); + } + if let Some(existing) = seen.get(name.as_str()) { + return Err(conflict(format!( + "Duplicate model file name found for '{name}' in {existing} and {path}" + ))); + } + seen.insert(name, path); + } + Ok(()) +} diff --git a/crates/sqlbuild-discovery/src/project_validation/_helpers/identities.rs b/crates/sqlbuild-discovery/src/project_validation/_helpers/identities.rs new file mode 100644 index 0000000000..44716a3842 --- /dev/null +++ b/crates/sqlbuild-discovery/src/project_validation/_helpers/identities.rs @@ -0,0 +1,155 @@ +//! Python's `validate_discovered_resource_identities`: canonical identities, in discovery order. + +use crate::models::{DiscoveryFailure, FailureKind}; +use crate::project_validation::models::{ + AuditFileFact, NamedPath, NodeKind, ProjectFacts, ValidationFailure, +}; +use sqlbuild_model_config::resource_identities::main::resource_identity_failure::resource_identity_failure; + +/// One identity to check: its name, the kind the error names, its path and whether it is private. +struct Candidate<'a> { + name: &'a str, + kind: &'a str, + path: &'a str, + private: bool, +} + +/// The identities to check, in the order Python's `validate_discovered_resource_identities` lists them. +struct Candidates<'a> { + list: Vec>, +} + +impl<'a> Candidates<'a> { + fn push(&mut self, name: &'a str, kind: &'a str, path: &'a str) { + self.list.push(Candidate { + name, + kind, + path, + private: false, + }); + } + + fn push_named(&mut self, entries: &'a [NamedPath], kind: &'a str) { + for entry in entries { + self.push(&entry.name, kind, &entry.path); + } + } + + /// A generic audit is named by its stem, each block naming an instance; others by the block. + fn push_audit(&mut self, audit: &'a AuditFileFact, block_name: Option<&'a str>) { + let path: &str = &audit.file.path; + if audit.generic { + self.push(&audit.file.name, "audit", path); + if let Some(instance) = block_name { + self.push(instance, "generic audit instance", path); + } + } else { + self.push( + block_or_file_name(block_name, &audit.file.name), + "audit", + path, + ); + } + } + + fn push_declarations(&mut self, declarations: &'a [NamedPath], kind: &'a str, private: bool) { + for declaration in declarations { + self.list.push(Candidate { + name: &declaration.name, + kind, + path: &declaration.path, + private, + }); + } + } +} + +/// Reject the first identity that is not canonical lowercase snake_case. +pub(crate) fn validate_identities(facts: &ProjectFacts) -> Result<(), ValidationFailure> { + for candidate in candidates(facts) { + if let Some((corrected, help)) = + resource_identity_failure(candidate.name, candidate.private) + { + return Err(ValidationFailure::Discovery(DiscoveryFailure { + kind: FailureKind::ResourceIdentity, + message: format!( + "Invalid {} identity '{}' in {}; use snake_case '{corrected}'", + candidate.kind, candidate.name, candidate.path + ), + help: Some(help), + })); + } + } + Ok(()) +} + +fn candidates(facts: &ProjectFacts) -> Vec> { + let mut candidates: Candidates<'_> = Candidates { list: Vec::new() }; + for model in &facts.models { + candidates.push(&model.file.name, "model", &model.file.path); + candidates.push_declarations(&model.local_declarations, "model-local declaration", true); + } + candidates.push_named(&facts.sql_functions, "function"); + candidates.push_named(&facts.python_functions, "function"); + candidates.push_named(&facts.sql_hooks, "SQL hook"); + for seed in &facts.seeds { + candidates.push(&seed.file.name, "seed", &seed.file.path); + } + for schema in &facts.schemas { + for name in &schema.model_names { + candidates.push(name, "schema model", &schema.path); + } + for seed in &schema.seeds { + candidates.push(&seed.name, "schema seed", &schema.path); + } + } + for file in &facts.sources { + for entry in &file.entries { + candidates.push(&entry.name, "source", &file.path); + } + } + for test in &facts.tests { + for block in &test.blocks { + candidates.push( + block_or_file_name(block.name.as_deref(), &test.file.name), + "SQL test", + &test.file.path, + ); + for case in &block.cases { + candidates.push(case, "SQL test case", &test.file.path); + } + } + } + candidates.push_named(&facts.scenarios, "scenario"); + for audit in &facts.audits { + for block_name in &audit.block_names { + candidates.push_audit(audit, block_name.as_deref()); + } + } + for file in &facts.declaration_files { + candidates.push_declarations(&file.declarations, "declaration", file.private); + } + candidates.push_named(&facts.model_schemas, "model schema"); + candidates.push_named(&facts.materializations, "materialization"); + for kind in [ + NodeKind::Loader, + NodeKind::Task, + NodeKind::Asset, + NodeKind::Check, + ] { + for node in facts.nodes.iter().filter(|node| node.kind == kind) { + candidates.push(&node.name, kind.label(), &node.path); + } + } + candidates.push_named(&facts.hook_functions, "Python hook"); + candidates.push_named(&facts.event_exporters, "event exporter"); + candidates.push_named(&facts.providers, "provider"); + candidates.list +} + +/// Python's `block.name or stem`. +pub(crate) fn block_or_file_name<'a>(block_name: Option<&'a str>, file_name: &'a str) -> &'a str { + block_name + .filter(|name| !name.is_empty()) + .unwrap_or(file_name) +} diff --git a/crates/sqlbuild-discovery/src/project_validation/_helpers/loaders.rs b/crates/sqlbuild-discovery/src/project_validation/_helpers/loaders.rs new file mode 100644 index 0000000000..47ebf99d0a --- /dev/null +++ b/crates/sqlbuild-discovery/src/project_validation/_helpers/loaders.rs @@ -0,0 +1,139 @@ +//! Loader and Python node names, dependencies, and the sources that name loaders. + +use std::collections::{HashMap, HashSet}; + +use crate::project_validation::_helpers::conflicts::{conflict, node_path, unique_named}; +use crate::project_validation::_helpers::python_nodes::{ + validate_check_dependencies, validate_loader_dependencies, validate_python_node_dependencies, +}; +use crate::project_validation::constants::DLT_LOADER_KIND; +use crate::project_validation::models::{ + NamedPath, NodeFact, NodeKind, ProjectFacts, SourceFileFact, ValidationFailure, +}; + +/// Validate loader and node names, node dependencies, then the loaders sources name. +pub(crate) fn validate_loaders(facts: &ProjectFacts) -> Result<(), ValidationFailure> { + let loaders: Vec<&NodeFact> = facts + .nodes + .iter() + .filter(|node| node.kind == NodeKind::Loader) + .collect(); + let loader_paths: Vec = loaders.iter().map(|node| node_path(node)).collect(); + unique_named(&loader_paths, |name, existing, path| { + format!("Duplicate source loader found for '{name}' in {existing} and {path}") + })?; + let node_paths: Vec = facts.nodes.iter().map(node_path).collect(); + unique_named(&node_paths, |name, existing, path| { + format!( + "Duplicate Python node found for '{name}' in {existing} and {path}; loader, \ + task, asset, and check names must be globally unique" + ) + })?; + source_loader_name_collisions(&facts.sources, &loaders)?; + validate_loader_dependencies(&facts.nodes)?; + validate_python_node_dependencies(&facts.nodes)?; + validate_check_dependencies(&facts.nodes)?; + source_loader_references(&facts.sources, &loaders)?; + terminal_loader_config_ownership(&facts.sources, &loaders) +} + +fn source_loader_name_collisions( + sources: &[SourceFileFact], + loaders: &[&NodeFact], +) -> Result<(), ValidationFailure> { + let loader_paths: HashMap<&str, &str> = loaders + .iter() + .map(|loader| (loader.name.as_str(), loader.path.as_str())) + .collect(); + for file in sources { + for entry in &file.entries { + let Some(loader_path) = loader_paths.get(entry.name.as_str()) else { + continue; + }; + if entry.managed && entry.loader.as_deref() == Some(entry.name.as_str()) { + continue; + } + return Err(conflict(format!( + "Source '{name}' in {path} conflicts with loader '{name}' in {loader_path}; \ + source and loader names must be globally unique unless the source is managed", + name = entry.name, + path = file.path, + ))); + } + } + Ok(()) +} + +/// Python's `integration_loader_name`. +fn integration_loader_name(kind: &str, source_name: &str) -> String { + if kind == DLT_LOADER_KIND { + source_name.to_owned() + } else { + format!("{kind}__{source_name}") + } +} + +fn source_loader_references( + sources: &[SourceFileFact], + loaders: &[&NodeFact], +) -> Result<(), ValidationFailure> { + let loader_names: HashSet<&str> = loaders.iter().map(|loader| loader.name.as_str()).collect(); + for file in sources { + for entry in &file.entries { + let expected: String = match &entry.integration_loader { + Some(integration) => integration_loader_name(&integration.kind, &entry.name), + None => entry.name.clone(), + }; + if entry.managed && entry.loader.as_deref() != Some(expected.as_str()) { + return Err(conflict(format!( + "Managed source '{}' in {} must use terminal loader '{expected}'", + entry.name, file.path + ))); + } + if entry.managed && !loader_names.contains(expected.as_str()) { + return Err(conflict(format!( + "Managed source '{}' in {} requires loader '{expected}'", + entry.name, file.path + ))); + } + match &entry.loader { + Some(loader) if !loader_names.contains(loader.as_str()) => { + return Err(conflict(format!( + "Source '{}' in {} references unknown loader '{loader}'", + entry.name, file.path + ))); + } + _ => {} + } + } + } + Ok(()) +} + +fn terminal_loader_config_ownership( + sources: &[SourceFileFact], + loaders: &[&NodeFact], +) -> Result<(), ValidationFailure> { + let by_name: HashMap<&str, &NodeFact> = loaders + .iter() + .map(|loader| (loader.name.as_str(), *loader)) + .collect(); + for file in sources { + for entry in &file.entries { + let Some(loader) = &entry.loader else { + continue; + }; + let configured: bool = by_name + .get(loader.as_str()) + .is_some_and(|loader| loader.decorator_config); + if configured { + return Err(conflict(format!( + "Source '{}' references loader '{loader}', but terminal source loader write \ + and schema config must be declared in source YAML, not the @loader decorator", + entry.name + ))); + } + } + } + Ok(()) +} diff --git a/crates/sqlbuild-discovery/src/project_validation/_helpers/mod.rs b/crates/sqlbuild-discovery/src/project_validation/_helpers/mod.rs new file mode 100644 index 0000000000..ddba252ca6 --- /dev/null +++ b/crates/sqlbuild-discovery/src/project_validation/_helpers/mod.rs @@ -0,0 +1,10 @@ +pub(crate) mod conflicts; +pub(crate) mod csv_header; +pub(crate) mod declarations; +pub(crate) mod file_names; +pub(crate) mod identities; +pub(crate) mod loaders; +pub(crate) mod path_defaults; +pub(crate) mod python_nodes; +pub(crate) mod resource_names; +pub(crate) mod seeds; diff --git a/crates/sqlbuild-discovery/src/project_validation/_helpers/path_defaults.rs b/crates/sqlbuild-discovery/src/project_validation/_helpers/path_defaults.rs new file mode 100644 index 0000000000..0568696b6a --- /dev/null +++ b/crates/sqlbuild-discovery/src/project_validation/_helpers/path_defaults.rs @@ -0,0 +1,77 @@ +//! Every `path_defaults` key must match at least one model path. + +use std::collections::HashSet; + +use sqlbuild_model_config::path_defaults::main::matched_path_default_keys::matched_path_default_keys; + +use crate::models::{DiscoveryFailure, FailureKind}; +use crate::project_validation::_helpers::conflicts::conflict; +use crate::project_validation::constants::{CURRENT_DIRECTORY, KNOWN_FOLDER_LIMIT, MODELS_PREFIX}; +use crate::project_validation::models::{ProjectFacts, ValidationFailure}; + +pub(crate) fn validate_path_defaults(facts: &ProjectFacts) -> Result<(), ValidationFailure> { + if facts.path_default_keys.is_empty() { + return Ok(()); + } + let model_paths: Vec = facts + .models + .iter() + .map(|model| { + let path: String = model.file.path.replace('\\', "/"); + path.strip_prefix(MODELS_PREFIX) + .map_or(path.clone(), str::to_owned) + }) + .collect(); + let mut matched: HashSet = HashSet::new(); + for model_path in &model_paths { + let keys: Vec = matched_path_default_keys(model_path, &facts.path_default_keys) + .map_err(|error| { + ValidationFailure::Discovery(DiscoveryFailure { + kind: FailureKind::Conflict, + message: error.message, + help: error.help, + }) + })?; + matched.extend(keys); + } + for key in &facts.path_default_keys { + if matched.contains(key) { + continue; + } + let mut folders: Vec = model_paths + .iter() + .map(|path| parent_folder(path)) + .filter(|folder| folder != CURRENT_DIRECTORY) + .collect::>() + .into_iter() + .collect(); + folders.sort(); + let known: String = if folders.is_empty() { + "".to_owned() + } else { + folders + .iter() + .take(KNOWN_FOLDER_LIMIT) + .cloned() + .collect::>() + .join(", ") + }; + return Err(conflict(format!( + "path_defaults['{key}'] does not match any model paths. Known model folders include: \ + {known}" + ))); + } + Ok(()) +} + +/// `str(Path(path).parent)` for a discovered POSIX relative path. +fn parent_folder(path: &str) -> String { + let parts: Vec<&str> = path + .split('/') + .filter(|part| !part.is_empty() && *part != CURRENT_DIRECTORY) + .collect(); + match parts.split_last() { + Some((_, parents)) if !parents.is_empty() => parents.join("/"), + _ => CURRENT_DIRECTORY.to_owned(), + } +} diff --git a/crates/sqlbuild-discovery/src/project_validation/_helpers/python_nodes.rs b/crates/sqlbuild-discovery/src/project_validation/_helpers/python_nodes.rs new file mode 100644 index 0000000000..5547a19d0f --- /dev/null +++ b/crates/sqlbuild-discovery/src/project_validation/_helpers/python_nodes.rs @@ -0,0 +1,217 @@ +//! Python node dependency checks: known targets, no SQL inputs on loaders, no cycles. + +use std::collections::{HashMap, HashSet}; + +use crate::project_validation::_helpers::conflicts::conflict; +use crate::project_validation::models::{DependencyFact, NodeFact, NodeKind, ValidationFailure}; + +/// `_python_node_dependency_key`: a decorated object's name, else the object itself. +#[derive(Clone, Copy, PartialEq, Eq)] +enum DependencyKey<'a> { + Name(&'a str), + Object(Option), +} + +fn dependency_key(dependency: &DependencyFact) -> Option> { + match dependency { + DependencyFact::SqlResource { .. } => None, + DependencyFact::Object { + decorated_name: Some(name), + .. + } => Some(DependencyKey::Name(name)), + DependencyFact::Object { owner, .. } => Some(DependencyKey::Object(*owner)), + } +} + +/// The loaders, tasks and assets a dependency key names, as `node_by_dependency_key` maps them. +struct ExecutableNodes<'a> { + nodes: &'a [NodeFact], + by_name: HashMap<&'a str, usize>, +} + +impl<'a> ExecutableNodes<'a> { + fn new(nodes: &'a [NodeFact]) -> Self { + let mut by_name: HashMap<&'a str, usize> = HashMap::new(); + for (index, node) in nodes.iter().enumerate() { + if node.kind != NodeKind::Check { + by_name.insert(&node.name, index); + } + } + Self { nodes, by_name } + } + + fn resolve(&self, key: DependencyKey<'_>) -> Option { + match key { + DependencyKey::Name(name) => self.by_name.get(name).copied(), + DependencyKey::Object(Some(owner)) if self.nodes[owner].kind != NodeKind::Check => { + Some(owner) + } + DependencyKey::Object(_) => None, + } + } +} + +/// `_validate_loader_dependencies`: no cycles through loader function dependencies. +pub(crate) fn validate_loader_dependencies(nodes: &[NodeFact]) -> Result<(), ValidationFailure> { + let next = |index: usize| -> Vec { + nodes[index] + .depends_on + .iter() + .filter_map(|dependency| match dependency { + DependencyFact::Object { + owner: Some(owner), .. + } if nodes[*owner].kind == NodeKind::Loader => Some(*owner), + _ => None, + }) + .collect() + }; + let roots: Vec = (0..nodes.len()) + .filter(|index| nodes[*index].kind == NodeKind::Loader) + .collect(); + find_cycle(nodes, &roots, next, "Loader dependency cycle detected") +} + +/// `_validate_python_node_dependencies` over loaders, tasks and assets. +pub(crate) fn validate_python_node_dependencies( + nodes: &[NodeFact], +) -> Result<(), ValidationFailure> { + let executable: ExecutableNodes<'_> = ExecutableNodes::new(nodes); + let roots: Vec = (0..nodes.len()) + .filter(|index| nodes[*index].kind != NodeKind::Check) + .collect(); + for index in &roots { + let node: &NodeFact = &nodes[*index]; + for dependency in &node.depends_on { + match dependency { + DependencyFact::SqlResource { name } => { + if node.kind == NodeKind::Loader { + return Err(conflict(format!( + "Loader '{}' depends on SQL resource '{name}'; loaders may depend on \ + Python loader/task/asset functions only", + node.name + ))); + } + } + DependencyFact::Object { .. } => { + let known: bool = dependency_key(dependency) + .and_then(|key| executable.resolve(key)) + .is_some(); + if known { + continue; + } + if node.kind == NodeKind::Loader { + return Err(conflict(format!( + "Loader '{}' depends on an unknown loader", + node.name + ))); + } + return Err(conflict(format!( + "Python node '{}' depends on an unknown Python node; use function \ + references to decorated loaders, tasks, or assets", + node.name + ))); + } + } + } + } + let next = |index: usize| -> Vec { + nodes[index] + .depends_on + .iter() + .filter_map(dependency_key) + .filter_map(|key| executable.resolve(key)) + .collect() + }; + find_cycle(nodes, &roots, next, "Python node dependency cycle detected") +} + +/// `_validate_check_dependencies`: checks depend on loaders, tasks and assets, never checks. +pub(crate) fn validate_check_dependencies(nodes: &[NodeFact]) -> Result<(), ValidationFailure> { + let allowed: HashSet<&str> = nodes + .iter() + .filter(|node| node.kind != NodeKind::Check) + .map(|node| node.name.as_str()) + .collect(); + let checks: HashSet<&str> = nodes + .iter() + .filter(|node| node.kind == NodeKind::Check) + .map(|node| node.name.as_str()) + .collect(); + for check in nodes.iter().filter(|node| node.kind == NodeKind::Check) { + for dependency in &check.depends_on { + let Some(key) = dependency_key(dependency) else { + continue; + }; + let name: Option<&str> = match key { + DependencyKey::Name(name) => Some(name), + DependencyKey::Object(_) => None, + }; + if name.is_some_and(|name| checks.contains(name)) { + return Err(conflict(format!( + "Check '{}' depends on another check; checks may depend on loaders, tasks, \ + assets, models, sources, and seeds only", + check.name + ))); + } + if !name.is_some_and(|name| allowed.contains(name)) { + return Err(conflict(format!( + "Check '{}' depends on an unknown Python node; use function references to \ + decorated loaders, tasks, or assets", + check.name + ))); + } + } + } + Ok(()) +} + +/// Python's recursive `visit`: depth-first from each root, sharing visited names across roots. +fn find_cycle( + nodes: &[NodeFact], + roots: &[usize], + next: impl Fn(usize) -> Vec, + label: &str, +) -> Result<(), ValidationFailure> { + let mut search: CycleSearch<'_, _> = CycleSearch { + nodes, + next, + label, + path: Vec::new(), + visited: HashSet::new(), + }; + for root in roots { + search.path.clear(); + search.visit(*root)?; + } + Ok(()) +} + +/// One depth-first cycle search: the nodes on the current path and those already cleared. +struct CycleSearch<'a, F> { + nodes: &'a [NodeFact], + next: F, + label: &'a str, + path: Vec<&'a str>, + visited: HashSet<&'a str>, +} + +impl<'a, F: Fn(usize) -> Vec> CycleSearch<'a, F> { + fn visit(&mut self, index: usize) -> Result<(), ValidationFailure> { + let name: &'a str = &self.nodes[index].name; + if self.visited.contains(name) { + return Ok(()); + } + if self.path.contains(&name) { + let mut cycle: Vec<&str> = self.path.clone(); + cycle.push(name); + return Err(conflict(format!("{}: {}", self.label, cycle.join(" -> ")))); + } + self.path.push(name); + for dependency in (self.next)(index) { + self.visit(dependency)?; + } + self.path.pop(); + self.visited.insert(name); + Ok(()) + } +} diff --git a/crates/sqlbuild-discovery/src/project_validation/_helpers/resource_names.rs b/crates/sqlbuild-discovery/src/project_validation/_helpers/resource_names.rs new file mode 100644 index 0000000000..7f2032bafa --- /dev/null +++ b/crates/sqlbuild-discovery/src/project_validation/_helpers/resource_names.rs @@ -0,0 +1,73 @@ +//! Python's `_validate_unique_project_resource_names`: one namespace across resource kinds. + +use std::collections::{HashMap, HashSet}; + +use crate::project_validation::_helpers::conflicts::{conflict, schema_seeds}; +use crate::project_validation::_helpers::identities::block_or_file_name; +use crate::project_validation::models::{NodeKind, ProjectFacts, ValidationFailure}; + +/// Reject the first resource name declared twice across models, sources, seeds and the rest. +pub(crate) fn validate_unique_project_resource_names( + facts: &ProjectFacts, +) -> Result<(), ValidationFailure> { + let mut seen: HashMap = HashMap::new(); + let mut declare = |name: &str, kind: &str, path: &str| -> Result<(), ValidationFailure> { + if let Some((existing_kind, existing_path)) = seen.get(name) { + return Err(conflict(format!( + "Project resource name '{name}' is declared as both {existing_kind} in \ + {existing_path} and {kind} in {path}; model, source, seed, function, loader, \ + task, asset, check, provider, hook, unit test, and scenario names must be \ + globally unique" + ))); + } + seen.insert(name.to_owned(), (kind.to_owned(), path.to_owned())); + Ok(()) + }; + for model in &facts.models { + declare(&model.file.name, "model", &model.file.path)?; + } + let mut managed_sources: HashSet<&str> = HashSet::new(); + for file in &facts.sources { + for entry in &file.entries { + if entry.managed { + managed_sources.insert(&entry.name); + } + declare(&entry.name, "source", &file.path)?; + } + } + for seed in schema_seeds(facts) { + declare(&seed.name, "seed", &seed.path)?; + } + for function in facts.sql_functions.iter().chain(&facts.python_functions) { + declare(&function.name, "function", &function.path)?; + } + for node in &facts.nodes { + if node.kind == NodeKind::Loader && managed_sources.contains(node.name.as_str()) { + continue; + } + declare(&node.name, node.kind.label(), &node.path)?; + } + for provider in &facts.providers { + declare(&provider.name, "provider", &provider.path)?; + } + for hook in &facts.sql_hooks { + declare(&hook.name, "sql hook", &hook.path)?; + } + for hook in &facts.hook_functions { + declare(&hook.name, "python hook", &hook.path)?; + } + for scenario in &facts.scenarios { + declare(&scenario.name, "scenario", &scenario.path)?; + } + let mut test_names: HashSet<&str> = HashSet::new(); + for test in &facts.tests { + for block in &test.blocks { + let name: &str = block_or_file_name(block.name.as_deref(), &test.file.name); + if !test_names.insert(name) { + continue; + } + declare(name, "unit test", &test.file.path)?; + } + } + Ok(()) +} diff --git a/crates/sqlbuild-discovery/src/project_validation/_helpers/seeds.rs b/crates/sqlbuild-discovery/src/project_validation/_helpers/seeds.rs new file mode 100644 index 0000000000..7f9fe73367 --- /dev/null +++ b/crates/sqlbuild-discovery/src/project_validation/_helpers/seeds.rs @@ -0,0 +1,133 @@ +//! Python's `_validate_declared_seed_files`: every declared seed has one CSV whose header matches. + +use std::collections::{HashMap, HashSet}; + +use sqlbuild_core::text::main::python_str_repr::python_str_repr; + +use crate::_helpers::reading::read_failure; +use crate::models::{DiscoveryFailure, FailureKind}; +use crate::project_validation::_helpers::csv_header::csv_header; +use crate::project_validation::errors::HeaderError; +use crate::project_validation::models::{ + ProjectFacts, SeedEntryFact, SeedFileFact, ValidationFailure, +}; + +/// `csv.field_size_limit()` before any project changes it. +const DEFAULT_FIELD_LIMIT: usize = 131_072; + +fn seed_error(message: String) -> ValidationFailure { + DiscoveryFailure::new(FailureKind::Seed, message).into() +} + +/// Reject undeclared, duplicate or mismatched seed CSVs in Python's order. +pub(crate) fn validate_declared_seed_files(facts: &ProjectFacts) -> Result<(), ValidationFailure> { + let field_limit: usize = facts.csv_field_limit.unwrap_or(DEFAULT_FIELD_LIMIT); + unique_seed_csv_names(&facts.seeds)?; + let mut by_name: HashMap<&str, Vec<&SeedFileFact>> = HashMap::new(); + for seed in &facts.seeds { + by_name.entry(&seed.file.name).or_default().push(seed); + } + let mut declared: HashSet<&str> = HashSet::new(); + for schema in &facts.schemas { + for entry in &schema.seeds { + declared.insert(&entry.name); + let matching: &[&SeedFileFact] = + by_name.get(entry.name.as_str()).map_or(&[], Vec::as_slice); + match matching { + [] => { + return Err(seed_error(format!( + "Seed declaration '{}' in {} has no matching CSV file under seeds/", + entry.name, schema.path + ))); + } + [seed] => seed_csv_header(entry, seed, field_limit)?, + _ => { + let paths: Vec<&str> = matching + .iter() + .map(|seed| seed.file.path.as_str()) + .collect(); + return Err(seed_error(format!( + "Seed declaration '{}' matches multiple CSV files: {}", + entry.name, + paths.join(", ") + ))); + } + } + } + } + for seed in &facts.seeds { + if !declared.contains(seed.file.name.as_str()) { + return Err(seed_error(format!( + "Seed CSV {} has no matching declaration for seed '{}' under seeds/**/*.yml", + seed.file.path, seed.file.name + ))); + } + } + Ok(()) +} + +fn unique_seed_csv_names(seeds: &[SeedFileFact]) -> Result<(), ValidationFailure> { + let mut seen: HashMap<&str, &str> = HashMap::new(); + for seed in seeds { + if let Some(existing) = seen.get(seed.file.name.as_str()) { + return Err(seed_error(format!( + "Duplicate seed CSV name '{}' found: {existing}, {}. Seed CSV filenames must be \ + unique under seeds/.", + seed.file.name, seed.file.path + ))); + } + seen.insert(&seed.file.name, &seed.file.path); + } + Ok(()) +} + +fn seed_csv_header( + entry: &SeedEntryFact, + seed: &SeedFileFact, + field_limit: usize, +) -> Result<(), ValidationFailure> { + let unreadable = |failure| ValidationFailure::Unreadable { + absolute_path: seed.absolute_path.clone(), + failure, + }; + let bytes: Vec = + std::fs::read(&seed.absolute_path).map_err(|error| unreadable(read_failure(&error)))?; + if let Some(error) = &entry.csv.dialect_error { + return Err(ValidationFailure::CsvDialect(error.clone())); + } + let header: Vec = match csv_header(&bytes, &entry.csv, field_limit) { + Ok(header) => header, + Err(HeaderError::Read(failure)) => return Err(unreadable(failure)), + Err(HeaderError::Csv(message)) => return Err(ValidationFailure::Csv(message)), + }; + let path: &str = &seed.file.path; + if header.is_empty() { + return Err(seed_error(format!("{path} must contain a CSV header row"))); + } + let mut seen: HashSet<&str> = HashSet::new(); + for column in &header { + if !seen.insert(column) { + return Err(seed_error(format!( + "{path} contains duplicate CSV header column '{column}'" + ))); + } + } + if header != entry.columns { + return Err(seed_error(format!( + "{path} header {} does not match declared seed columns {} for '{}'", + python_tuple_repr(&header), + python_tuple_repr(&entry.columns), + entry.name + ))); + } + Ok(()) +} + +/// Python's `repr` of a tuple of strings. +fn python_tuple_repr(items: &[String]) -> String { + let parts: Vec = items.iter().map(|item| python_str_repr(item)).collect(); + match parts.as_slice() { + [single] => format!("({single},)"), + _ => format!("({})", parts.join(", ")), + } +} diff --git a/crates/sqlbuild-discovery/src/project_validation/constants.rs b/crates/sqlbuild-discovery/src/project_validation/constants.rs new file mode 100644 index 0000000000..8ee3042fb3 --- /dev/null +++ b/crates/sqlbuild-discovery/src/project_validation/constants.rs @@ -0,0 +1,12 @@ +//! Names and limits cross-file discovery validation checks against. + +/// The loader kind whose generated loader takes the source's own name. +pub(crate) const DLT_LOADER_KIND: &str = "dlt"; +/// The prefix discovery strips from model paths before matching path defaults. +pub(crate) const MODELS_PREFIX: &str = "models/"; +/// `str(Path(...).parent)` of a path with no folder. +pub(crate) const CURRENT_DIRECTORY: &str = "."; +/// How many known model folders an unmatched path default lists. +pub(crate) const KNOWN_FOLDER_LIMIT: usize = 5; +/// Model names SQLBuild reserves for compiled output structure. +pub(crate) const RESERVED_MODEL_NAMES: [&str; 1] = ["_chain_"]; diff --git a/crates/sqlbuild-discovery/src/project_validation/errors.rs b/crates/sqlbuild-discovery/src/project_validation/errors.rs new file mode 100644 index 0000000000..1c057a217c --- /dev/null +++ b/crates/sqlbuild-discovery/src/project_validation/errors.rs @@ -0,0 +1,27 @@ +//! Errors of reading seed CSV headers the way Python's `csv` module does. + +use crate::models::ReadFailure; + +/// A `TypeError` or `ValueError` Python's `csv` module raises for a reader's settings. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct CsvDialectError { + pub class: String, + pub message: String, +} + +/// Why the header could not be read. +#[derive(Debug)] +pub(crate) enum HeaderError { + Read(ReadFailure), + /// `_csv.Error` with its message. + Csv(String), +} + +impl std::fmt::Display for HeaderError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Read(failure) => write!(formatter, "{failure:?}"), + Self::Csv(message) => formatter.write_str(message), + } + } +} diff --git a/crates/sqlbuild-discovery/src/project_validation/main/mod.rs b/crates/sqlbuild-discovery/src/project_validation/main/mod.rs new file mode 100644 index 0000000000..1fc67badd8 --- /dev/null +++ b/crates/sqlbuild-discovery/src/project_validation/main/mod.rs @@ -0,0 +1 @@ +pub mod validate_project_inputs; diff --git a/crates/sqlbuild-discovery/src/project_validation/main/validate_project_inputs.rs b/crates/sqlbuild-discovery/src/project_validation/main/validate_project_inputs.rs new file mode 100644 index 0000000000..4ed360042a --- /dev/null +++ b/crates/sqlbuild-discovery/src/project_validation/main/validate_project_inputs.rs @@ -0,0 +1,21 @@ +//! Python's `validate_discovered_inputs`: cross-file conflicts across discovered project inputs. + +use crate::project_validation::_helpers::declarations::validate_declarations; +use crate::project_validation::_helpers::file_names::validate_file_names; +use crate::project_validation::_helpers::identities::validate_identities; +use crate::project_validation::_helpers::loaders::validate_loaders; +use crate::project_validation::_helpers::path_defaults::validate_path_defaults; +use crate::project_validation::_helpers::resource_names::validate_unique_project_resource_names; +use crate::project_validation::_helpers::seeds::validate_declared_seed_files; +use crate::project_validation::models::{ProjectFacts, ValidationFailure}; + +/// Validate cross-file conflicts in Python's order; the first conflict is reported. +pub fn validate_project_inputs(facts: &ProjectFacts) -> Result<(), ValidationFailure> { + validate_identities(facts)?; + validate_file_names(facts)?; + validate_loaders(facts)?; + validate_declarations(facts)?; + validate_declared_seed_files(facts)?; + validate_unique_project_resource_names(facts)?; + validate_path_defaults(facts) +} diff --git a/crates/sqlbuild-discovery/src/project_validation/mod.rs b/crates/sqlbuild-discovery/src/project_validation/mod.rs new file mode 100644 index 0000000000..383898ca23 --- /dev/null +++ b/crates/sqlbuild-discovery/src/project_validation/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-discovery/src/project_validation/models.rs b/crates/sqlbuild-discovery/src/project_validation/models.rs new file mode 100644 index 0000000000..1a0f4fdb09 --- /dev/null +++ b/crates/sqlbuild-discovery/src/project_validation/models.rs @@ -0,0 +1,215 @@ +//! The facts cross-file discovery validation reads, in each collection's discovery order. + +use crate::models::{DiscoveryFailure, ReadFailure}; +use crate::project_validation::errors::CsvDialectError; + +/// A resource name and the project-relative path that declares it. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct NamedPath { + pub name: String, + pub path: String, +} + +/// A discovered SQL model file: its stem, path and the declarations its header owns. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ModelFact { + pub file: NamedPath, + /// `(name, declaring path)` of the header's enums, then its constants. + pub local_declarations: Vec, +} + +/// One declarative integration loader a source names. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct IntegrationLoaderFact { + pub kind: String, +} + +/// One source entry of a source YAML file. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct SourceFact { + pub name: String, + pub managed: bool, + pub loader: Option, + pub integration_loader: Option, +} + +/// A source YAML file and its entries. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct SourceFileFact { + pub path: String, + pub entries: Vec, +} + +/// The decorated Python node kinds, in the order discovery lists them. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum NodeKind { + Loader, + Task, + Asset, + Check, +} + +impl NodeKind { + /// The kind as resource-name conflicts word it. + pub fn label(self) -> &'static str { + match self { + Self::Loader => "loader", + Self::Task => "task", + Self::Asset => "asset", + Self::Check => "check", + } + } +} + +/// One dependency a Python node declares. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum DependencyFact { + /// A `SqlResourceRef` to a model, source or seed. + SqlResource { name: String }, + /// Any other dependency object. + Object { + /// The index of the discovered node whose function this object is. + owner: Option, + /// The name a loader, task, asset or check decorator records on the object. + decorated_name: Option, + }, +} + +/// A decorated Python loader, task, asset or check function. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct NodeFact { + pub kind: NodeKind, + pub name: String, + pub path: String, + /// Whether a loader's decorator sets destination, write or schema config. + pub decorator_config: bool, + pub depends_on: Vec, +} + +/// The CSV reader settings a seed declaration authors. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct CsvSettings { + pub delimiter: Option, + pub quotechar: Option, + pub escapechar: Option, + pub doublequote: Option, + pub skipinitialspace: Option, + /// The error the running Python's `csv.reader` raises for these settings, if any. + pub dialect_error: Option, +} + +/// One seed entry of a schema YAML file. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct SeedEntryFact { + pub name: String, + pub columns: Vec, + pub csv: CsvSettings, +} + +/// A schema YAML file's model and seed entries. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct SchemaFileFact { + pub path: String, + pub model_names: Vec, + pub seeds: Vec, +} + +/// A seed CSV file: its stem and project-relative path, and where to read it. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct SeedFileFact { + pub file: NamedPath, + pub absolute_path: String, +} + +/// One SQL test block and its case names. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct TestBlockFact { + pub name: Option, + pub cases: Vec, +} + +/// A SQL test file: its path, stem and blocks. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct TestFileFact { + pub file: NamedPath, + pub blocks: Vec, +} + +/// An audit file: its path and stem, whether it declares a generic audit, and its block names. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct AuditFileFact { + pub file: NamedPath, + pub generic: bool, + pub block_names: Vec>, +} + +/// An enum or constant declaration file's declarations and whether its scope is private. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ScopedDeclarationsFact { + pub private: bool, + pub declarations: Vec, +} + +/// Everything cross-file validation reads about one discovery. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ProjectFacts { + pub models: Vec, + pub sql_functions: Vec, + pub python_functions: Vec, + pub sql_hooks: Vec, + pub seeds: Vec, + pub schemas: Vec, + pub sources: Vec, + pub tests: Vec, + pub scenarios: Vec, + pub audits: Vec, + /// Enum declaration files, then constant declaration files. + pub declaration_files: Vec, + /// Every model schema declaration across the model schema files. + pub model_schemas: Vec, + pub materializations: Vec, + /// Loaders, tasks, assets, then checks. + pub nodes: Vec, + pub hook_functions: Vec, + pub event_exporters: Vec, + pub providers: Vec, + pub path_default_keys: Vec, + /// Python's `csv.field_size_limit()`, or `None` for its default. + pub csv_field_limit: Option, +} + +/// Why discovered inputs are rejected: a discovery error, or a seed CSV Python cannot read. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ValidationFailure { + Discovery(DiscoveryFailure), + Unreadable { + absolute_path: String, + failure: ReadFailure, + }, + /// Python's `csv.Error` reading a seed header, with its message. + Csv(String), + /// The settings error the running Python's `csv.reader` raises. + CsvDialect(CsvDialectError), +} + +impl From for ValidationFailure { + fn from(failure: DiscoveryFailure) -> Self { + Self::Discovery(failure) + } +} + +impl std::fmt::Display for ValidationFailure { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Discovery(failure) => { + write!(formatter, "{:?}: {}", failure.kind, failure.message) + } + Self::Unreadable { + absolute_path, + failure, + } => write!(formatter, "unreadable {absolute_path}: {failure:?}"), + Self::Csv(message) => write!(formatter, "csv: {message}"), + Self::CsvDialect(error) => write!(formatter, "{}: {}", error.class, error.message), + } + } +} diff --git a/crates/sqlbuild-discovery/src/project_validation/tests.rs b/crates/sqlbuild-discovery/src/project_validation/tests.rs new file mode 100644 index 0000000000..ff3236ffdb --- /dev/null +++ b/crates/sqlbuild-discovery/src/project_validation/tests.rs @@ -0,0 +1,8 @@ +#[path = "tests/test_csv_header.rs"] +mod csv_header; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; +#[path = "tests/test_validate_project_inputs.rs"] +mod validate_project_inputs; diff --git a/crates/sqlbuild-discovery/src/project_validation/tests/helpers.rs b/crates/sqlbuild-discovery/src/project_validation/tests/helpers.rs new file mode 100644 index 0000000000..dc141fbc95 --- /dev/null +++ b/crates/sqlbuild-discovery/src/project_validation/tests/helpers.rs @@ -0,0 +1,90 @@ +use crate::project_validation::_helpers::csv_header::csv_header; +use crate::project_validation::main::validate_project_inputs::validate_project_inputs; +use crate::project_validation::models::{ + CsvSettings, DependencyFact, ModelFact, NamedPath, NodeFact, NodeKind, ProjectFacts, + SchemaFileFact, SeedEntryFact, SeedFileFact, ValidationFailure, +}; + +/// The header the reader returns, or its error's message. +pub(super) fn read_header( + text: &str, + settings: &CsvSettings, + field_limit: usize, +) -> Result, String> { + csv_header(text.as_bytes(), settings, field_limit).map_err(|error| error.to_string()) +} + +/// Validation's outcome, the failure shown as `Kind: message`. +pub(super) fn validation_outcome(facts: &ProjectFacts) -> Result<(), String> { + validate_project_inputs(facts).map_err(|failure: ValidationFailure| failure.to_string()) +} + +/// Validation's outcome for one seed CSV holding `csv_text` declared with `columns`. +pub(super) fn seed_header_outcome(csv_text: &str, columns: &[&str]) -> Result<(), String> { + let directory = tempfile::tempdir().expect("a temporary directory is writable"); + let csv_path = directory.path().join("countries.csv"); + std::fs::write(&csv_path, csv_text).expect("the seed CSV is writable"); + let facts = ProjectFacts { + seeds: vec![SeedFileFact { + file: named("countries", "seeds/countries.csv"), + absolute_path: csv_path.to_string_lossy().into_owned(), + }], + schemas: vec![SchemaFileFact { + path: "seeds/countries.yml".to_owned(), + model_names: Vec::new(), + seeds: vec![SeedEntryFact { + name: "countries".to_owned(), + columns: columns.iter().map(|column| (*column).to_owned()).collect(), + ..SeedEntryFact::default() + }], + }], + ..ProjectFacts::default() + }; + validation_outcome(&facts) +} + +pub(super) fn named(name: &str, path: &str) -> NamedPath { + NamedPath { + name: name.to_owned(), + path: path.to_owned(), + } +} + +/// A model file at `path` named by its stem. +pub(super) fn model_at(name: &str, path: &str) -> ModelFact { + ModelFact { + file: named(name, path), + local_declarations: Vec::new(), + } +} + +pub(super) fn model(name: &str) -> ModelFact { + model_at(name, &format!("models/{name}.sql")) +} + +pub(super) fn node(kind: NodeKind, name: &str, depends_on: Vec) -> NodeFact { + NodeFact { + kind, + name: name.to_owned(), + path: format!("python/{name}.py"), + decorator_config: false, + depends_on, + } +} + +/// A dependency on the decorated function of node `owner`, named `name`. +pub(super) fn on(owner: usize, name: &str) -> DependencyFact { + DependencyFact::Object { + owner: Some(owner), + decorated_name: Some(name.to_owned()), + } +} + +/// Facts for models at `paths` with the path-default `keys`. +pub(super) fn path_default_facts(paths: &[&str], keys: &[&str]) -> ProjectFacts { + ProjectFacts { + models: paths.iter().map(|path| model_at("orders", path)).collect(), + path_default_keys: keys.iter().map(|key| (*key).to_owned()).collect(), + ..ProjectFacts::default() + } +} diff --git a/crates/sqlbuild-discovery/src/project_validation/tests/test_csv_header.rs b/crates/sqlbuild-discovery/src/project_validation/tests/test_csv_header.rs new file mode 100644 index 0000000000..f95658ba51 --- /dev/null +++ b/crates/sqlbuild-discovery/src/project_validation/tests/test_csv_header.rs @@ -0,0 +1,1264 @@ +use crate::project_validation::models::CsvSettings; +use crate::project_validation::tests::helpers::read_header; +use crate::project_validation::tests::test_types::{CsvHeaderTestCase, LIMIT, csv_error, header}; + +/// Expectations recorded from CPython 3.12's `next(csv.reader(...))`, identical on 3.13 and 3.14. +#[test] +fn given_seed_csv_text_when_reading_the_header_then_matches_python_csv_reader() { + let test_cases = [ + CsvHeaderTestCase { + description: "bare cr", + text: "id,name\r1,a\r".to_owned(), + settings: CsvSettings::default(), + field_limit: LIMIT, + expected_header: header(&["id", "name"]), + }, + CsvHeaderTestCase { + description: "no terminator", + text: "id,name".to_owned(), + settings: CsvSettings::default(), + field_limit: LIMIT, + expected_header: header(&["id", "name"]), + }, + CsvHeaderTestCase { + description: "empty", + text: "".to_owned(), + settings: CsvSettings::default(), + field_limit: LIMIT, + expected_header: header(&[]), + }, + CsvHeaderTestCase { + description: "blank first line", + text: "\nid\n".to_owned(), + settings: CsvSettings::default(), + field_limit: LIMIT, + expected_header: header(&[]), + }, + CsvHeaderTestCase { + description: "trailing delimiter", + text: "a,b,\n".to_owned(), + settings: CsvSettings::default(), + field_limit: LIMIT, + expected_header: header(&["a", "b", ""]), + }, + CsvHeaderTestCase { + description: "'id,name\\n1,a\\n' read default", + text: "id,name\n1,a\n".to_owned(), + settings: CsvSettings::default(), + field_limit: LIMIT, + expected_header: header(&["id", "name"]), + }, + CsvHeaderTestCase { + description: "'id,name\\n1,a\\n' read quoted", + text: "id,name\n1,a\n".to_owned(), + settings: CsvSettings { + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["id", "name"]), + }, + CsvHeaderTestCase { + description: "'id,name\\n1,a\\n' read quoted without doublequote", + text: "id,name\n1,a\n".to_owned(), + settings: CsvSettings { + quotechar: Some("\"".to_owned()), + doublequote: Some(false), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["id", "name"]), + }, + CsvHeaderTestCase { + description: "'id,name\\n1,a\\n' read quoted skipping initial space", + text: "id,name\n1,a\n".to_owned(), + settings: CsvSettings { + skipinitialspace: Some(true), + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["id", "name"]), + }, + CsvHeaderTestCase { + description: "'id,name\\n1,a\\n' read escaped", + text: "id,name\n1,a\n".to_owned(), + settings: CsvSettings { + escapechar: Some("\\".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["id", "name"]), + }, + CsvHeaderTestCase { + description: "'id,name\\n1,a\\n' read quoted and escaped", + text: "id,name\n1,a\n".to_owned(), + settings: CsvSettings { + escapechar: Some("\\".to_owned()), + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["id", "name"]), + }, + CsvHeaderTestCase { + description: "'id,name\\n1,a\\n' read semicolon", + text: "id,name\n1,a\n".to_owned(), + settings: CsvSettings { + delimiter: Some(";".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["id,name"]), + }, + CsvHeaderTestCase { + description: "'id,name\\r\\n' read default", + text: "id,name\r\n".to_owned(), + settings: CsvSettings::default(), + field_limit: LIMIT, + expected_header: header(&["id", "name"]), + }, + CsvHeaderTestCase { + description: "'id,name\\r\\n' read quoted", + text: "id,name\r\n".to_owned(), + settings: CsvSettings { + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["id", "name"]), + }, + CsvHeaderTestCase { + description: "'id,name\\r\\n' read quoted without doublequote", + text: "id,name\r\n".to_owned(), + settings: CsvSettings { + quotechar: Some("\"".to_owned()), + doublequote: Some(false), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["id", "name"]), + }, + CsvHeaderTestCase { + description: "'id,name\\r\\n' read quoted skipping initial space", + text: "id,name\r\n".to_owned(), + settings: CsvSettings { + skipinitialspace: Some(true), + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["id", "name"]), + }, + CsvHeaderTestCase { + description: "'id,name\\r\\n' read escaped", + text: "id,name\r\n".to_owned(), + settings: CsvSettings { + escapechar: Some("\\".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["id", "name"]), + }, + CsvHeaderTestCase { + description: "'id,name\\r\\n' read quoted and escaped", + text: "id,name\r\n".to_owned(), + settings: CsvSettings { + escapechar: Some("\\".to_owned()), + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["id", "name"]), + }, + CsvHeaderTestCase { + description: "'id,name\\r\\n' read semicolon", + text: "id,name\r\n".to_owned(), + settings: CsvSettings { + delimiter: Some(";".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["id,name"]), + }, + CsvHeaderTestCase { + description: "'\"a,x\",b\\n' read default", + text: "\"a,x\",b\n".to_owned(), + settings: CsvSettings::default(), + field_limit: LIMIT, + expected_header: header(&["\"a", "x\"", "b"]), + }, + CsvHeaderTestCase { + description: "'\"a,x\",b\\n' read quoted", + text: "\"a,x\",b\n".to_owned(), + settings: CsvSettings { + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a,x", "b"]), + }, + CsvHeaderTestCase { + description: "'\"a,x\",b\\n' read quoted without doublequote", + text: "\"a,x\",b\n".to_owned(), + settings: CsvSettings { + quotechar: Some("\"".to_owned()), + doublequote: Some(false), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a,x", "b"]), + }, + CsvHeaderTestCase { + description: "'\"a,x\",b\\n' read quoted skipping initial space", + text: "\"a,x\",b\n".to_owned(), + settings: CsvSettings { + skipinitialspace: Some(true), + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a,x", "b"]), + }, + CsvHeaderTestCase { + description: "'\"a,x\",b\\n' read escaped", + text: "\"a,x\",b\n".to_owned(), + settings: CsvSettings { + escapechar: Some("\\".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["\"a", "x\"", "b"]), + }, + CsvHeaderTestCase { + description: "'\"a,x\",b\\n' read quoted and escaped", + text: "\"a,x\",b\n".to_owned(), + settings: CsvSettings { + escapechar: Some("\\".to_owned()), + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a,x", "b"]), + }, + CsvHeaderTestCase { + description: "'\"a,x\",b\\n' read semicolon", + text: "\"a,x\",b\n".to_owned(), + settings: CsvSettings { + delimiter: Some(";".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["\"a,x\",b"]), + }, + CsvHeaderTestCase { + description: "'\"a\"\"b\",c\\n' read default", + text: "\"a\"\"b\",c\n".to_owned(), + settings: CsvSettings::default(), + field_limit: LIMIT, + expected_header: header(&["\"a\"\"b\"", "c"]), + }, + CsvHeaderTestCase { + description: "'\"a\"\"b\",c\\n' read quoted", + text: "\"a\"\"b\",c\n".to_owned(), + settings: CsvSettings { + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a\"b", "c"]), + }, + CsvHeaderTestCase { + description: "'\"a\"\"b\",c\\n' read quoted without doublequote", + text: "\"a\"\"b\",c\n".to_owned(), + settings: CsvSettings { + quotechar: Some("\"".to_owned()), + doublequote: Some(false), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a\"b\"", "c"]), + }, + CsvHeaderTestCase { + description: "'\"a\"\"b\",c\\n' read quoted skipping initial space", + text: "\"a\"\"b\",c\n".to_owned(), + settings: CsvSettings { + skipinitialspace: Some(true), + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a\"b", "c"]), + }, + CsvHeaderTestCase { + description: "'\"a\"\"b\",c\\n' read escaped", + text: "\"a\"\"b\",c\n".to_owned(), + settings: CsvSettings { + escapechar: Some("\\".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["\"a\"\"b\"", "c"]), + }, + CsvHeaderTestCase { + description: "'\"a\"\"b\",c\\n' read quoted and escaped", + text: "\"a\"\"b\",c\n".to_owned(), + settings: CsvSettings { + escapechar: Some("\\".to_owned()), + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a\"b", "c"]), + }, + CsvHeaderTestCase { + description: "'\"a\"\"b\",c\\n' read semicolon", + text: "\"a\"\"b\",c\n".to_owned(), + settings: CsvSettings { + delimiter: Some(";".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["\"a\"\"b\",c"]), + }, + CsvHeaderTestCase { + description: "'\"a\"b,c\\n' read default", + text: "\"a\"b,c\n".to_owned(), + settings: CsvSettings::default(), + field_limit: LIMIT, + expected_header: header(&["\"a\"b", "c"]), + }, + CsvHeaderTestCase { + description: "'\"a\"b,c\\n' read quoted", + text: "\"a\"b,c\n".to_owned(), + settings: CsvSettings { + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["ab", "c"]), + }, + CsvHeaderTestCase { + description: "'\"a\"b,c\\n' read quoted without doublequote", + text: "\"a\"b,c\n".to_owned(), + settings: CsvSettings { + quotechar: Some("\"".to_owned()), + doublequote: Some(false), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["ab", "c"]), + }, + CsvHeaderTestCase { + description: "'\"a\"b,c\\n' read quoted skipping initial space", + text: "\"a\"b,c\n".to_owned(), + settings: CsvSettings { + skipinitialspace: Some(true), + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["ab", "c"]), + }, + CsvHeaderTestCase { + description: "'\"a\"b,c\\n' read escaped", + text: "\"a\"b,c\n".to_owned(), + settings: CsvSettings { + escapechar: Some("\\".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["\"a\"b", "c"]), + }, + CsvHeaderTestCase { + description: "'\"a\"b,c\\n' read quoted and escaped", + text: "\"a\"b,c\n".to_owned(), + settings: CsvSettings { + escapechar: Some("\\".to_owned()), + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["ab", "c"]), + }, + CsvHeaderTestCase { + description: "'\"a\"b,c\\n' read semicolon", + text: "\"a\"b,c\n".to_owned(), + settings: CsvSettings { + delimiter: Some(";".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["\"a\"b,c"]), + }, + CsvHeaderTestCase { + description: "'\"a\\nb\",c\\n' read default", + text: "\"a\nb\",c\n".to_owned(), + settings: CsvSettings::default(), + field_limit: LIMIT, + expected_header: header(&["\"a"]), + }, + CsvHeaderTestCase { + description: "'\"a\\nb\",c\\n' read quoted", + text: "\"a\nb\",c\n".to_owned(), + settings: CsvSettings { + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a\nb", "c"]), + }, + CsvHeaderTestCase { + description: "'\"a\\nb\",c\\n' read quoted without doublequote", + text: "\"a\nb\",c\n".to_owned(), + settings: CsvSettings { + quotechar: Some("\"".to_owned()), + doublequote: Some(false), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a\nb", "c"]), + }, + CsvHeaderTestCase { + description: "'\"a\\nb\",c\\n' read quoted skipping initial space", + text: "\"a\nb\",c\n".to_owned(), + settings: CsvSettings { + skipinitialspace: Some(true), + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a\nb", "c"]), + }, + CsvHeaderTestCase { + description: "'\"a\\nb\",c\\n' read escaped", + text: "\"a\nb\",c\n".to_owned(), + settings: CsvSettings { + escapechar: Some("\\".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["\"a"]), + }, + CsvHeaderTestCase { + description: "'\"a\\nb\",c\\n' read quoted and escaped", + text: "\"a\nb\",c\n".to_owned(), + settings: CsvSettings { + escapechar: Some("\\".to_owned()), + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a\nb", "c"]), + }, + CsvHeaderTestCase { + description: "'\"a\\nb\",c\\n' read semicolon", + text: "\"a\nb\",c\n".to_owned(), + settings: CsvSettings { + delimiter: Some(";".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["\"a"]), + }, + CsvHeaderTestCase { + description: "'a, b, c\\n' read default", + text: "a, b, c\n".to_owned(), + settings: CsvSettings::default(), + field_limit: LIMIT, + expected_header: header(&["a", " b", " c"]), + }, + CsvHeaderTestCase { + description: "'a, b, c\\n' read quoted", + text: "a, b, c\n".to_owned(), + settings: CsvSettings { + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a", " b", " c"]), + }, + CsvHeaderTestCase { + description: "'a, b, c\\n' read quoted without doublequote", + text: "a, b, c\n".to_owned(), + settings: CsvSettings { + quotechar: Some("\"".to_owned()), + doublequote: Some(false), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a", " b", " c"]), + }, + CsvHeaderTestCase { + description: "'a, b, c\\n' read quoted skipping initial space", + text: "a, b, c\n".to_owned(), + settings: CsvSettings { + skipinitialspace: Some(true), + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a", "b", "c"]), + }, + CsvHeaderTestCase { + description: "'a, b, c\\n' read escaped", + text: "a, b, c\n".to_owned(), + settings: CsvSettings { + escapechar: Some("\\".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a", " b", " c"]), + }, + CsvHeaderTestCase { + description: "'a, b, c\\n' read quoted and escaped", + text: "a, b, c\n".to_owned(), + settings: CsvSettings { + escapechar: Some("\\".to_owned()), + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a", " b", " c"]), + }, + CsvHeaderTestCase { + description: "'a, b, c\\n' read semicolon", + text: "a, b, c\n".to_owned(), + settings: CsvSettings { + delimiter: Some(";".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a, b, c"]), + }, + CsvHeaderTestCase { + description: "'a\\\\,b,c\\n' read default", + text: "a\\,b,c\n".to_owned(), + settings: CsvSettings::default(), + field_limit: LIMIT, + expected_header: header(&["a\\", "b", "c"]), + }, + CsvHeaderTestCase { + description: "'a\\\\,b,c\\n' read quoted", + text: "a\\,b,c\n".to_owned(), + settings: CsvSettings { + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a\\", "b", "c"]), + }, + CsvHeaderTestCase { + description: "'a\\\\,b,c\\n' read quoted without doublequote", + text: "a\\,b,c\n".to_owned(), + settings: CsvSettings { + quotechar: Some("\"".to_owned()), + doublequote: Some(false), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a\\", "b", "c"]), + }, + CsvHeaderTestCase { + description: "'a\\\\,b,c\\n' read quoted skipping initial space", + text: "a\\,b,c\n".to_owned(), + settings: CsvSettings { + skipinitialspace: Some(true), + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a\\", "b", "c"]), + }, + CsvHeaderTestCase { + description: "'a\\\\,b,c\\n' read escaped", + text: "a\\,b,c\n".to_owned(), + settings: CsvSettings { + escapechar: Some("\\".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a,b", "c"]), + }, + CsvHeaderTestCase { + description: "'a\\\\,b,c\\n' read quoted and escaped", + text: "a\\,b,c\n".to_owned(), + settings: CsvSettings { + escapechar: Some("\\".to_owned()), + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a,b", "c"]), + }, + CsvHeaderTestCase { + description: "'a\\\\,b,c\\n' read semicolon", + text: "a\\,b,c\n".to_owned(), + settings: CsvSettings { + delimiter: Some(";".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a\\,b,c"]), + }, + CsvHeaderTestCase { + description: "'a,b,\\n' read default", + text: "a,b,\n".to_owned(), + settings: CsvSettings::default(), + field_limit: LIMIT, + expected_header: header(&["a", "b", ""]), + }, + CsvHeaderTestCase { + description: "'a,b,\\n' read quoted", + text: "a,b,\n".to_owned(), + settings: CsvSettings { + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a", "b", ""]), + }, + CsvHeaderTestCase { + description: "'a,b,\\n' read quoted without doublequote", + text: "a,b,\n".to_owned(), + settings: CsvSettings { + quotechar: Some("\"".to_owned()), + doublequote: Some(false), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a", "b", ""]), + }, + CsvHeaderTestCase { + description: "'a,b,\\n' read quoted skipping initial space", + text: "a,b,\n".to_owned(), + settings: CsvSettings { + skipinitialspace: Some(true), + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a", "b", ""]), + }, + CsvHeaderTestCase { + description: "'a,b,\\n' read escaped", + text: "a,b,\n".to_owned(), + settings: CsvSettings { + escapechar: Some("\\".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a", "b", ""]), + }, + CsvHeaderTestCase { + description: "'a,b,\\n' read quoted and escaped", + text: "a,b,\n".to_owned(), + settings: CsvSettings { + escapechar: Some("\\".to_owned()), + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a", "b", ""]), + }, + CsvHeaderTestCase { + description: "'a,b,\\n' read semicolon", + text: "a,b,\n".to_owned(), + settings: CsvSettings { + delimiter: Some(";".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a,b,"]), + }, + CsvHeaderTestCase { + description: "'\"abc' read default", + text: "\"abc".to_owned(), + settings: CsvSettings::default(), + field_limit: LIMIT, + expected_header: header(&["\"abc"]), + }, + CsvHeaderTestCase { + description: "'\"abc' read quoted", + text: "\"abc".to_owned(), + settings: CsvSettings { + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["abc"]), + }, + CsvHeaderTestCase { + description: "'\"abc' read quoted without doublequote", + text: "\"abc".to_owned(), + settings: CsvSettings { + quotechar: Some("\"".to_owned()), + doublequote: Some(false), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["abc"]), + }, + CsvHeaderTestCase { + description: "'\"abc' read quoted skipping initial space", + text: "\"abc".to_owned(), + settings: CsvSettings { + skipinitialspace: Some(true), + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["abc"]), + }, + CsvHeaderTestCase { + description: "'\"abc' read escaped", + text: "\"abc".to_owned(), + settings: CsvSettings { + escapechar: Some("\\".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["\"abc"]), + }, + CsvHeaderTestCase { + description: "'\"abc' read quoted and escaped", + text: "\"abc".to_owned(), + settings: CsvSettings { + escapechar: Some("\\".to_owned()), + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["abc"]), + }, + CsvHeaderTestCase { + description: "'\"abc' read semicolon", + text: "\"abc".to_owned(), + settings: CsvSettings { + delimiter: Some(";".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["\"abc"]), + }, + CsvHeaderTestCase { + description: "'\\ufeffid,name\\n' read default", + text: "\u{feff}id,name\n".to_owned(), + settings: CsvSettings::default(), + field_limit: LIMIT, + expected_header: header(&["\u{feff}id", "name"]), + }, + CsvHeaderTestCase { + description: "'\\ufeffid,name\\n' read quoted", + text: "\u{feff}id,name\n".to_owned(), + settings: CsvSettings { + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["\u{feff}id", "name"]), + }, + CsvHeaderTestCase { + description: "'\\ufeffid,name\\n' read quoted without doublequote", + text: "\u{feff}id,name\n".to_owned(), + settings: CsvSettings { + quotechar: Some("\"".to_owned()), + doublequote: Some(false), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["\u{feff}id", "name"]), + }, + CsvHeaderTestCase { + description: "'\\ufeffid,name\\n' read quoted skipping initial space", + text: "\u{feff}id,name\n".to_owned(), + settings: CsvSettings { + skipinitialspace: Some(true), + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["\u{feff}id", "name"]), + }, + CsvHeaderTestCase { + description: "'\\ufeffid,name\\n' read escaped", + text: "\u{feff}id,name\n".to_owned(), + settings: CsvSettings { + escapechar: Some("\\".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["\u{feff}id", "name"]), + }, + CsvHeaderTestCase { + description: "'\\ufeffid,name\\n' read quoted and escaped", + text: "\u{feff}id,name\n".to_owned(), + settings: CsvSettings { + escapechar: Some("\\".to_owned()), + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["\u{feff}id", "name"]), + }, + CsvHeaderTestCase { + description: "'\\ufeffid,name\\n' read semicolon", + text: "\u{feff}id,name\n".to_owned(), + settings: CsvSettings { + delimiter: Some(";".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["\u{feff}id,name"]), + }, + CsvHeaderTestCase { + description: "' \"a\", b\\n' read default", + text: " \"a\", b\n".to_owned(), + settings: CsvSettings::default(), + field_limit: LIMIT, + expected_header: header(&[" \"a\"", " b"]), + }, + CsvHeaderTestCase { + description: "' \"a\", b\\n' read quoted", + text: " \"a\", b\n".to_owned(), + settings: CsvSettings { + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&[" \"a\"", " b"]), + }, + CsvHeaderTestCase { + description: "' \"a\", b\\n' read quoted without doublequote", + text: " \"a\", b\n".to_owned(), + settings: CsvSettings { + quotechar: Some("\"".to_owned()), + doublequote: Some(false), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&[" \"a\"", " b"]), + }, + CsvHeaderTestCase { + description: "' \"a\", b\\n' read quoted skipping initial space", + text: " \"a\", b\n".to_owned(), + settings: CsvSettings { + skipinitialspace: Some(true), + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a", "b"]), + }, + CsvHeaderTestCase { + description: "' \"a\", b\\n' read escaped", + text: " \"a\", b\n".to_owned(), + settings: CsvSettings { + escapechar: Some("\\".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&[" \"a\"", " b"]), + }, + CsvHeaderTestCase { + description: "' \"a\", b\\n' read quoted and escaped", + text: " \"a\", b\n".to_owned(), + settings: CsvSettings { + escapechar: Some("\\".to_owned()), + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&[" \"a\"", " b"]), + }, + CsvHeaderTestCase { + description: "' \"a\", b\\n' read semicolon", + text: " \"a\", b\n".to_owned(), + settings: CsvSettings { + delimiter: Some(";".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&[" \"a\", b"]), + }, + CsvHeaderTestCase { + description: "'a\\\\\\nb,c\\n' read default", + text: "a\\\nb,c\n".to_owned(), + settings: CsvSettings::default(), + field_limit: LIMIT, + expected_header: header(&["a\\"]), + }, + CsvHeaderTestCase { + description: "'a\\\\\\nb,c\\n' read quoted", + text: "a\\\nb,c\n".to_owned(), + settings: CsvSettings { + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a\\"]), + }, + CsvHeaderTestCase { + description: "'a\\\\\\nb,c\\n' read quoted without doublequote", + text: "a\\\nb,c\n".to_owned(), + settings: CsvSettings { + quotechar: Some("\"".to_owned()), + doublequote: Some(false), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a\\"]), + }, + CsvHeaderTestCase { + description: "'a\\\\\\nb,c\\n' read quoted skipping initial space", + text: "a\\\nb,c\n".to_owned(), + settings: CsvSettings { + skipinitialspace: Some(true), + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a\\"]), + }, + CsvHeaderTestCase { + description: "'a\\\\\\nb,c\\n' read escaped", + text: "a\\\nb,c\n".to_owned(), + settings: CsvSettings { + escapechar: Some("\\".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a\nb", "c"]), + }, + CsvHeaderTestCase { + description: "'a\\\\\\nb,c\\n' read quoted and escaped", + text: "a\\\nb,c\n".to_owned(), + settings: CsvSettings { + escapechar: Some("\\".to_owned()), + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a\nb", "c"]), + }, + CsvHeaderTestCase { + description: "'a\\\\\\nb,c\\n' read semicolon", + text: "a\\\nb,c\n".to_owned(), + settings: CsvSettings { + delimiter: Some(";".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a\\"]), + }, + CsvHeaderTestCase { + description: "'\"a\" ,b\\n' read default", + text: "\"a\" ,b\n".to_owned(), + settings: CsvSettings::default(), + field_limit: LIMIT, + expected_header: header(&["\"a\" ", "b"]), + }, + CsvHeaderTestCase { + description: "'\"a\" ,b\\n' read quoted", + text: "\"a\" ,b\n".to_owned(), + settings: CsvSettings { + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a ", "b"]), + }, + CsvHeaderTestCase { + description: "'\"a\" ,b\\n' read quoted without doublequote", + text: "\"a\" ,b\n".to_owned(), + settings: CsvSettings { + quotechar: Some("\"".to_owned()), + doublequote: Some(false), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a ", "b"]), + }, + CsvHeaderTestCase { + description: "'\"a\" ,b\\n' read quoted skipping initial space", + text: "\"a\" ,b\n".to_owned(), + settings: CsvSettings { + skipinitialspace: Some(true), + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a ", "b"]), + }, + CsvHeaderTestCase { + description: "'\"a\" ,b\\n' read escaped", + text: "\"a\" ,b\n".to_owned(), + settings: CsvSettings { + escapechar: Some("\\".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["\"a\" ", "b"]), + }, + CsvHeaderTestCase { + description: "'\"a\" ,b\\n' read quoted and escaped", + text: "\"a\" ,b\n".to_owned(), + settings: CsvSettings { + escapechar: Some("\\".to_owned()), + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a ", "b"]), + }, + CsvHeaderTestCase { + description: "'\"a\" ,b\\n' read semicolon", + text: "\"a\" ,b\n".to_owned(), + settings: CsvSettings { + delimiter: Some(";".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["\"a\" ,b"]), + }, + CsvHeaderTestCase { + description: "'a,\"b\"c\\n' read default", + text: "a,\"b\"c\n".to_owned(), + settings: CsvSettings::default(), + field_limit: LIMIT, + expected_header: header(&["a", "\"b\"c"]), + }, + CsvHeaderTestCase { + description: "'a,\"b\"c\\n' read quoted", + text: "a,\"b\"c\n".to_owned(), + settings: CsvSettings { + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a", "bc"]), + }, + CsvHeaderTestCase { + description: "'a,\"b\"c\\n' read quoted without doublequote", + text: "a,\"b\"c\n".to_owned(), + settings: CsvSettings { + quotechar: Some("\"".to_owned()), + doublequote: Some(false), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a", "bc"]), + }, + CsvHeaderTestCase { + description: "'a,\"b\"c\\n' read quoted skipping initial space", + text: "a,\"b\"c\n".to_owned(), + settings: CsvSettings { + skipinitialspace: Some(true), + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a", "bc"]), + }, + CsvHeaderTestCase { + description: "'a,\"b\"c\\n' read escaped", + text: "a,\"b\"c\n".to_owned(), + settings: CsvSettings { + escapechar: Some("\\".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a", "\"b\"c"]), + }, + CsvHeaderTestCase { + description: "'a,\"b\"c\\n' read quoted and escaped", + text: "a,\"b\"c\n".to_owned(), + settings: CsvSettings { + escapechar: Some("\\".to_owned()), + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a", "bc"]), + }, + CsvHeaderTestCase { + description: "'a,\"b\"c\\n' read semicolon", + text: "a,\"b\"c\n".to_owned(), + settings: CsvSettings { + delimiter: Some(";".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a,\"b\"c"]), + }, + CsvHeaderTestCase { + description: "'\\n\\nid\\n' read default", + text: "\n\nid\n".to_owned(), + settings: CsvSettings::default(), + field_limit: LIMIT, + expected_header: header(&[]), + }, + CsvHeaderTestCase { + description: "'\\n\\nid\\n' read quoted", + text: "\n\nid\n".to_owned(), + settings: CsvSettings { + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&[]), + }, + CsvHeaderTestCase { + description: "'\\n\\nid\\n' read quoted without doublequote", + text: "\n\nid\n".to_owned(), + settings: CsvSettings { + quotechar: Some("\"".to_owned()), + doublequote: Some(false), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&[]), + }, + CsvHeaderTestCase { + description: "'\\n\\nid\\n' read quoted skipping initial space", + text: "\n\nid\n".to_owned(), + settings: CsvSettings { + skipinitialspace: Some(true), + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&[]), + }, + CsvHeaderTestCase { + description: "'\\n\\nid\\n' read escaped", + text: "\n\nid\n".to_owned(), + settings: CsvSettings { + escapechar: Some("\\".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&[]), + }, + CsvHeaderTestCase { + description: "'\\n\\nid\\n' read quoted and escaped", + text: "\n\nid\n".to_owned(), + settings: CsvSettings { + escapechar: Some("\\".to_owned()), + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&[]), + }, + CsvHeaderTestCase { + description: "'\\n\\nid\\n' read semicolon", + text: "\n\nid\n".to_owned(), + settings: CsvSettings { + delimiter: Some(";".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&[]), + }, + CsvHeaderTestCase { + description: "'a,b\\rc\\n' read default", + text: "a,b\rc\n".to_owned(), + settings: CsvSettings::default(), + field_limit: LIMIT, + expected_header: header(&["a", "b"]), + }, + CsvHeaderTestCase { + description: "'a,b\\rc\\n' read quoted", + text: "a,b\rc\n".to_owned(), + settings: CsvSettings { + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a", "b"]), + }, + CsvHeaderTestCase { + description: "'a,b\\rc\\n' read quoted without doublequote", + text: "a,b\rc\n".to_owned(), + settings: CsvSettings { + quotechar: Some("\"".to_owned()), + doublequote: Some(false), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a", "b"]), + }, + CsvHeaderTestCase { + description: "'a,b\\rc\\n' read quoted skipping initial space", + text: "a,b\rc\n".to_owned(), + settings: CsvSettings { + skipinitialspace: Some(true), + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a", "b"]), + }, + CsvHeaderTestCase { + description: "'a,b\\rc\\n' read escaped", + text: "a,b\rc\n".to_owned(), + settings: CsvSettings { + escapechar: Some("\\".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a", "b"]), + }, + CsvHeaderTestCase { + description: "'a,b\\rc\\n' read quoted and escaped", + text: "a,b\rc\n".to_owned(), + settings: CsvSettings { + escapechar: Some("\\".to_owned()), + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a", "b"]), + }, + CsvHeaderTestCase { + description: "'a,b\\rc\\n' read semicolon", + text: "a,b\rc\n".to_owned(), + settings: CsvSettings { + delimiter: Some(";".to_owned()), + ..CsvSettings::default() + }, + field_limit: LIMIT, + expected_header: header(&["a,b"]), + }, + CsvHeaderTestCase { + description: "a header longer than one decode chunk", + text: format!("{},id\n1,2\n", "c".repeat(10_000)), + settings: CsvSettings::default(), + field_limit: LIMIT, + expected_header: header(&[&"c".repeat(10_000), "id"]), + }, + CsvHeaderTestCase { + description: "a field at the limit", + text: format!("{},id\n", "c".repeat(10)), + settings: CsvSettings::default(), + field_limit: 10, + expected_header: header(&[&"c".repeat(10), "id"]), + }, + CsvHeaderTestCase { + description: "a field over the limit", + text: format!("{},id\n", "c".repeat(11)), + settings: CsvSettings::default(), + field_limit: 10, + expected_header: csv_error("field larger than field limit (10)"), + }, + ]; + for test_case in test_cases { + assert_eq!( + read_header(&test_case.text, &test_case.settings, test_case.field_limit), + test_case.expected_header, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-discovery/src/project_validation/tests/test_types.rs b/crates/sqlbuild-discovery/src/project_validation/tests/test_types.rs new file mode 100644 index 0000000000..6881934f58 --- /dev/null +++ b/crates/sqlbuild-discovery/src/project_validation/tests/test_types.rs @@ -0,0 +1,44 @@ +use crate::models::FailureKind; +use crate::project_validation::models::{CsvSettings, ProjectFacts}; + +/// Python's default `csv.field_size_limit()`. +pub(super) const LIMIT: usize = 131_072; + +/// One seed CSV text and the header Python's `csv` reader returns or the error it raises. +pub(super) struct CsvHeaderTestCase { + pub(super) description: &'static str, + pub(super) text: String, + pub(super) settings: CsvSettings, + pub(super) field_limit: usize, + pub(super) expected_header: Result, String>, +} + +/// One project's facts and the first failure Python's validation reports, if any. +pub(super) struct ValidationTestCase { + pub(super) description: &'static str, + pub(super) facts: ProjectFacts, + pub(super) expected_failure: Result<(), String>, +} + +/// One seed CSV beside a declaration of `declared_columns`, and the failure validation reports. +pub(super) struct SeedHeaderTestCase { + pub(super) description: &'static str, + pub(super) csv_text: &'static str, + pub(super) declared_columns: &'static [&'static str], + pub(super) expected_failure: Result<(), String>, +} + +/// A header the reader returns. +pub(super) fn header(columns: &[&str]) -> Result, String> { + Ok(columns.iter().map(|column| (*column).to_owned()).collect()) +} + +/// The `_csv.Error` message the reader raises. +pub(super) fn csv_error(message: &str) -> Result, String> { + Err(message.to_owned()) +} + +/// The failure `Display` names for `kind` with `message`. +pub(super) fn failure(kind: FailureKind, message: &str) -> Result<(), String> { + Err(format!("{kind:?}: {message}")) +} diff --git a/crates/sqlbuild-discovery/src/project_validation/tests/test_validate_project_inputs.rs b/crates/sqlbuild-discovery/src/project_validation/tests/test_validate_project_inputs.rs new file mode 100644 index 0000000000..c6a9d1bd52 --- /dev/null +++ b/crates/sqlbuild-discovery/src/project_validation/tests/test_validate_project_inputs.rs @@ -0,0 +1,231 @@ +use crate::models::FailureKind; +use crate::project_validation::models::{ + NodeKind, ProjectFacts, SeedFileFact, SourceFact, SourceFileFact, +}; +use crate::project_validation::tests::helpers::{ + model, model_at, named, node, on, path_default_facts, seed_header_outcome, validation_outcome, +}; +use crate::project_validation::tests::test_types::{ + SeedHeaderTestCase, ValidationTestCase, failure, +}; + +#[test] +fn given_discovered_facts_when_validating_then_reports_python_first_conflict() { + let test_cases = [ + ValidationTestCase { + description: "a valid project passes", + facts: ProjectFacts { + models: vec![model("orders"), model("customers")], + ..ProjectFacts::default() + }, + expected_failure: Ok(()), + }, + ValidationTestCase { + description: "a non-canonical model identity is rejected first", + facts: ProjectFacts { + models: vec![model("Orders"), model("_chain_")], + ..ProjectFacts::default() + }, + expected_failure: failure( + FailureKind::ResourceIdentity, + "Invalid model identity 'Orders' in models/Orders.sql; use snake_case 'orders'", + ), + }, + ValidationTestCase { + description: "a reserved model name fails its identity check first", + facts: ProjectFacts { + models: vec![model("_chain_")], + ..ProjectFacts::default() + }, + expected_failure: failure( + FailureKind::ResourceIdentity, + "Invalid model identity '_chain_' in models/_chain_.sql; use snake_case 'chain'", + ), + }, + ValidationTestCase { + description: "a source and a model share a logical relation name", + facts: ProjectFacts { + models: vec![model("orders")], + sources: vec![SourceFileFact { + path: "sources/shop.yml".to_owned(), + entries: vec![SourceFact { + name: "orders".to_owned(), + ..SourceFact::default() + }], + }], + ..ProjectFacts::default() + }, + expected_failure: failure( + FailureKind::Conflict, + "Logical relation name 'orders' is declared as both model in models/orders.sql \ + and source in sources/shop.yml", + ), + }, + ValidationTestCase { + description: "a task cycle names its path", + facts: ProjectFacts { + nodes: vec![ + node(NodeKind::Task, "extract", vec![on(1, "load")]), + node(NodeKind::Task, "load", vec![on(0, "extract")]), + ], + ..ProjectFacts::default() + }, + expected_failure: failure( + FailureKind::Conflict, + "Python node dependency cycle detected: extract -> load -> extract", + ), + }, + ValidationTestCase { + description: "a check depending on a check", + facts: ProjectFacts { + nodes: vec![ + node(NodeKind::Check, "first", Vec::new()), + node(NodeKind::Check, "second", vec![on(0, "first")]), + ], + ..ProjectFacts::default() + }, + expected_failure: failure( + FailureKind::Conflict, + "Check 'second' depends on another check; checks may depend on loaders, tasks, \ + assets, models, sources, and seeds only", + ), + }, + ValidationTestCase { + description: "an undeclared seed CSV", + facts: ProjectFacts { + seeds: vec![SeedFileFact { + file: named("countries", "seeds/countries.csv"), + absolute_path: "/missing/seeds/countries.csv".to_owned(), + }], + ..ProjectFacts::default() + }, + expected_failure: failure( + FailureKind::Seed, + "Seed CSV seeds/countries.csv has no matching declaration for seed 'countries' \ + under seeds/**/*.yml", + ), + }, + ValidationTestCase { + description: "an unmatched path default lists known folders", + facts: ProjectFacts { + models: vec![model_at("orders", "models/marts/orders.sql")], + path_default_keys: vec!["staging".to_owned()], + ..ProjectFacts::default() + }, + expected_failure: failure( + FailureKind::Conflict, + "path_defaults['staging'] does not match any model paths. Known model folders \ + include: marts", + ), + }, + ValidationTestCase { + description: "accepts Windows-style model paths that match path defaults", + facts: path_default_facts(&["models\\staging\\orders.sql"], &["staging"]), + expected_failure: Ok(()), + }, + ValidationTestCase { + description: "accepts recursive glob matching nested staging prefix", + facts: path_default_facts( + &["models/commerce/partner_feed/staging/orders.sql"], + &["commerce/**/staging"], + ), + expected_failure: Ok(()), + }, + ValidationTestCase { + description: "accepts recursive glob matching zero segments", + facts: path_default_facts( + &["models/commerce/staging/orders.sql"], + &["commerce/**/staging"], + ), + expected_failure: Ok(()), + }, + ValidationTestCase { + description: "counts broad glob as matched when a literal wins selection", + facts: path_default_facts( + &["models/commerce/partner_feed/staging/orders.sql"], + &["commerce/**/staging", "commerce/partner_feed/staging"], + ), + expected_failure: Ok(()), + }, + ValidationTestCase { + description: "rejects recursive glob matching no model paths", + facts: path_default_facts( + &["models/commerce/partner_feed/marts/orders.sql"], + &["commerce/**/staging"], + ), + expected_failure: failure( + FailureKind::Conflict, + "path_defaults['commerce/**/staging'] does not match any model paths. Known \ + model folders include: commerce/partner_feed/marts", + ), + }, + ValidationTestCase { + description: "rejects equally specific matching globs during discovery", + facts: path_default_facts( + &["models/commerce/eu/staging/orders.sql"], + &["commerce/*/staging", "commerce/eu/*"], + ), + expected_failure: failure( + FailureKind::Conflict, + "Model path 'commerce/eu/staging/orders.sql' matches equally specific \ + path_defaults keys: 'commerce/*/staging', 'commerce/eu/*'.", + ), + }, + ]; + for test_case in test_cases { + assert_eq!( + validation_outcome(&test_case.facts), + test_case.expected_failure, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_seed_csv_headers_when_validating_then_reports_python_tuple_reprs() { + let test_cases = [ + SeedHeaderTestCase { + description: "a matching header passes", + csv_text: "code,label\nGB,United Kingdom\n", + declared_columns: &["code", "label"], + expected_failure: Ok(()), + }, + SeedHeaderTestCase { + description: "an extra column shows both tuples", + csv_text: "code,label\nGB,United Kingdom\n", + declared_columns: &["code"], + expected_failure: failure( + FailureKind::Seed, + "seeds/countries.csv header ('code', 'label') does not match declared seed \ + columns ('code',) for 'countries'", + ), + }, + SeedHeaderTestCase { + description: "a duplicate header column", + csv_text: "code,code\n", + declared_columns: &["code", "code"], + expected_failure: failure( + FailureKind::Seed, + "seeds/countries.csv contains duplicate CSV header column 'code'", + ), + }, + SeedHeaderTestCase { + description: "an empty file has no header row", + csv_text: "", + declared_columns: &["code"], + expected_failure: failure( + FailureKind::Seed, + "seeds/countries.csv must contain a CSV header row", + ), + }, + ]; + for test_case in test_cases { + assert_eq!( + seed_header_outcome(test_case.csv_text, test_case.declared_columns), + test_case.expected_failure, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-discovery/src/yaml_entries/_helpers/columns.rs b/crates/sqlbuild-discovery/src/yaml_entries/_helpers/columns.rs new file mode 100644 index 0000000000..72fc1f4a9e --- /dev/null +++ b/crates/sqlbuild-discovery/src/yaml_entries/_helpers/columns.rs @@ -0,0 +1,67 @@ +//! The `columns` list of a seed or source entry. + +use sqlbuild_model_config::errors::ConfigError; +use sqlbuild_model_config::header_metadata::main::parse_audit_list::parse_audit_list; +use sqlbuild_model_config::types::{AuthoredNode, NodeKind}; + +use crate::yaml_entries::_helpers::fields::{ + Site, get, optional_bool, optional_mapping, optional_text, reject_unknown_keys, required_text, +}; +use crate::yaml_entries::models::ParsedYamlColumn; + +const NOT_NULL_AUDIT: &str = "not_null"; + +/// `_parse_columns` of a seed or source entry. +pub(crate) fn yaml_columns( + entries: &[(N, N)], + site: Site<'_>, + allowed: Option<&[&str]>, +) -> Result>, ConfigError> { + let Some(raw) = get(entries, "columns") else { + return Ok(Vec::new()); + }; + if raw.kind() != NodeKind::List { + return Err(site.error("columns must be a list")); + } + let label: String = format!("{} column", site.label); + let column_site: Site<'_> = Site { + path: site.path, + label: &label, + }; + let mut columns: Vec> = Vec::new(); + for raw_column in raw.items() { + if raw_column.kind() != NodeKind::Map { + return Err(site.error("columns must contain only mappings")); + } + let column: Vec<(N, N)> = raw_column.entries(); + if let Some(allowed) = allowed { + reject_unknown_keys(&column, allowed, column_site)?; + } + let name: N = required_text(&column, "name", column_site)?; + let nullable: Option = optional_bool(&column, "nullable", column_site)?; + let audits = parse_audit_list(get(&column, "audits"), site.path, &label, false)?; + let declares_nullable: bool = nullable + .as_ref() + .is_some_and(|value| value.kind() == NodeKind::Bool(true)); + if declares_nullable + && audits + .iter() + .any(|audit| audit.definition_name.is_text(NOT_NULL_AUDIT)) + { + return Err(ConfigError::compile(format!( + "{} column '{}' cannot set nullable = true and audit not_null", + site.path, + name.python_str() + ))); + } + columns.push(ParsedYamlColumn { + name, + column_type: optional_text(&column, "type", column_site)?, + nullable, + description: optional_text(&column, "description", column_site)?, + meta: optional_mapping(&column, "meta", column_site)?, + audits, + }); + } + Ok(columns) +} diff --git a/crates/sqlbuild-discovery/src/yaml_entries/_helpers/fields.rs b/crates/sqlbuild-discovery/src/yaml_entries/_helpers/fields.rs new file mode 100644 index 0000000000..0784da32a3 --- /dev/null +++ b/crates/sqlbuild-discovery/src/yaml_entries/_helpers/fields.rs @@ -0,0 +1,175 @@ +//! Python's authored-value field readers, with the errors their YAML callers raise. + +use std::num::FpCategory; + +use sqlbuild_core::text::main::python_strip::python_strip; +use sqlbuild_model_config::errors::ConfigError; +use sqlbuild_model_config::types::{AuthoredNode, NodeKind}; + +use crate::key_checks::main::unknown_mapping_keys::unknown_mapping_keys; + +/// Where one YAML parse reports its errors: the file path and the label the messages name. +#[derive(Clone, Copy)] +pub(crate) struct Site<'a> { + pub(crate) path: &'a str, + pub(crate) label: &'a str, +} + +impl Site<'_> { + /// The error `