Convert REGEXP_SUBSTR with a computed position, occurrence or group

coverage/translation/results-baseline.json

@@ -10257,26 +10257,26 @@
     "translation/functions/regexp-substr-all.toml#t009_null_regex_parameters": "refused",
     "translation/functions/regexp-substr-all.toml#t010_null_group_num": "refused",
     "translation/functions/regexp-substr-all.toml#t011_empty": "matching",
-    "translation/functions/regexp-substr.column-arguments.toml#t001_position_zero": "refused",
-    "translation/functions/regexp-substr.column-arguments.toml#t002_position_negative": "refused",
-    "translation/functions/regexp-substr.column-arguments.toml#t003_position_past_end": "refused",
-    "translation/functions/regexp-substr.column-arguments.toml#t004_position_far_past_end": "refused",
-    "translation/functions/regexp-substr.column-arguments.toml#t005_occurrence_zero": "refused",
-    "translation/functions/regexp-substr.column-arguments.toml#t006_occurrence_negative": "refused",
-    "translation/functions/regexp-substr.column-arguments.toml#t007_group_zero": "refused",
-    "translation/functions/regexp-substr.column-arguments.toml#t008_group_past_last": "refused",
-    "translation/functions/regexp-substr.column-arguments.toml#t009_group_negative": "refused",
-    "translation/functions/regexp-substr.column-arguments.toml#t010_anchor_after_start_position": "refused",
+    "translation/functions/regexp-substr.column-arguments.toml#t001_position_zero": "matching_error",
+    "translation/functions/regexp-substr.column-arguments.toml#t002_position_negative": "matching_error",
+    "translation/functions/regexp-substr.column-arguments.toml#t003_position_past_end": "matching",
+    "translation/functions/regexp-substr.column-arguments.toml#t004_position_far_past_end": "matching",
+    "translation/functions/regexp-substr.column-arguments.toml#t005_occurrence_zero": "matching_error",
+    "translation/functions/regexp-substr.column-arguments.toml#t006_occurrence_negative": "matching_error",
+    "translation/functions/regexp-substr.column-arguments.toml#t007_group_zero": "matching",
+    "translation/functions/regexp-substr.column-arguments.toml#t008_group_past_last": "matching",
+    "translation/functions/regexp-substr.column-arguments.toml#t009_group_negative": "matching_error",
+    "translation/functions/regexp-substr.column-arguments.toml#t010_anchor_after_start_position": "matching",
     "translation/functions/regexp-substr.toml#t001_varchar": "matching",
-    "translation/functions/regexp-substr.toml#t002_position_occurrence": "refused",
-    "translation/functions/regexp-substr.toml#t003_group": "refused",
+    "translation/functions/regexp-substr.toml#t002_position_occurrence": "matching",
+    "translation/functions/regexp-substr.toml#t003_group": "matching",
     "translation/functions/regexp-substr.toml#t004_parameters": "matching",
     "translation/functions/regexp-substr.toml#t005_null_subject": "matching",
     "translation/functions/regexp-substr.toml#t006_null_pattern": "matching",
-    "translation/functions/regexp-substr.toml#t007_null_position": "refused",
-    "translation/functions/regexp-substr.toml#t008_null_occurrence": "refused",
+    "translation/functions/regexp-substr.toml#t007_null_position": "matching",
+    "translation/functions/regexp-substr.toml#t008_null_occurrence": "matching",
     "translation/functions/regexp-substr.toml#t009_null_regex_parameters": "refused",
-    "translation/functions/regexp-substr.toml#t010_null_group_num": "refused",
+    "translation/functions/regexp-substr.toml#t010_null_group_num": "matching",
     "translation/functions/regexp-substr.toml#t011_empty": "matching",
     "translation/functions/regexp.toml#t001_anchored": "matching",
     "translation/functions/regexp.toml#t002_case": "matching",

crates/sqlquack-core/src/failures/constants.rs

@@ -151,6 +151,12 @@ pub const SQUARE_ROOT_OF_NEGATIVE: SnowflakeCode = SnowflakeCode {
     sqlstate: "22P01",
 };
 
+/// REGEXP_SUBSTR with a position or occurrence below 1, or a negative capture group.
+pub const INVALID_PARAMETER_VALUE: SnowflakeCode = SnowflakeCode {
+    errno: 100_050,
+    sqlstate: "22023",
+};
+
 /// Every Snowflake error the recordings hold, and its kind.
 pub const SNOWFLAKE_ERRORS: &[(SnowflakeCode, ErrorKind)] = &[
     (NUMBER_OUT_OF_RANGE, ErrorKind::NumericOverflow),
@@ -178,6 +184,7 @@ pub const SNOWFLAKE_ERRORS: &[(SnowflakeCode, ErrorKind)] = &[
     (HEX_DECODE_FAILED, ErrorKind::InvalidArgument),
     (SQUARE_ROOT_OF_NEGATIVE, ErrorKind::InvalidArgument),
     (UNKNOWN_TIME_ZONE, ErrorKind::InvalidArgument),
+    (INVALID_PARAMETER_VALUE, ErrorKind::InvalidArgument),
     (
         SnowflakeCode {
             errno: 1007,
@@ -263,6 +270,12 @@ pub const HEX_DECODE_FAILURES: &[FailureMode] = &[FailureMode {
     snowflake: &[HEX_DECODE_FAILED],
 }];
 
+/// How REGEXP_SUBSTR fails for a position, occurrence or capture group out of range.
+pub const REGEXP_PARAMETER_FAILURES: &[FailureMode] = &[FailureMode {
+    kind: ErrorKind::InvalidArgument,
+    snowflake: &[INVALID_PARAMETER_VALUE],
+}];
+
 /// How CHR fails for a code point no character has.
 pub const CHARACTER_CODE_FAILURES: &[FailureMode] = &[FailureMode {
     kind: ErrorKind::InvalidArgument,

crates/sqlquack-rules/src/catalogue/constants.rs

@@ -6,9 +6,9 @@ use sqlquack_core::failures::constants::{
     DIVISION_BY_ZERO, DIVISION_FAILURES, FLATTEN_FAILURES, HEX_DECODE_FAILURES,
     INVALID_UTF8_FAILURES, JSON_NOT_PARSED, KEY_EXISTS_FAILURES, LOGARITHM_FAILURES,
     NOT_AN_ARRAY_ARGUMENT, NOT_AN_ARRAY_FAILURES, NUMBER_NOT_RECOGNIZED, NUMBER_OUT_OF_RANGE,
-    NUMBER_TEXT_OUT_OF_RANGE, OBJECT_KEY_EXISTS, RANGE_STEP_FAILURES, STRING_TOO_LONG,
-    TIME_TEXT_FAILURES, TIMESTAMP_FORMAT_MISMATCH, TIMESTAMP_NOT_RECOGNIZED, VARIANT_NOT_CAST,
-    ZONE_FAILURES,
+    NUMBER_TEXT_OUT_OF_RANGE, OBJECT_KEY_EXISTS, RANGE_STEP_FAILURES, REGEXP_PARAMETER_FAILURES,
+    STRING_TOO_LONG, TIME_TEXT_FAILURES, TIMESTAMP_FORMAT_MISMATCH, TIMESTAMP_NOT_RECOGNIZED,
+    VARIANT_NOT_CAST, ZONE_FAILURES,
 };
 use sqlquack_core::failures::models::{ErrorKind, FailureMode};
 use sqlquack_snowflake::typing::constants::VARIANT_COMMON_TYPE;
@@ -1062,8 +1062,8 @@ pub const RULES: &[RuleSpec] = &[
             "fn:REGEXP_SUBSTR",
             "node:regexp_like",
         ],
-        conditions: "A literal pattern without lookarounds, group modifiers, lazy quantifiers (except +?, *? and ?? in REGEXP_SUBSTR, recorded reading as optional greedy repetitions, so as *, * and ?), POSIX classes or escapes other than \\d \\s \\w (ASCII, as recorded), their negations, escaped punctuation and, in REGEXP_SUBSTR and REGEXP_SUBSTR_ALL, \\n, \\t, \\r and, outside a bracket expression, the ASCII word boundary \\b, as recorded beside digits, underscores, punctuation and non-ASCII letters. REGEXP_LIKE matches the whole text; REGEXP_REPLACE replaces every match, or only the first, from position 1, case-sensitively or with 'i', with group references only; REGEXP_SUBSTR from position 1, any occurrence, with parameters of c, i, m, s and e and a group, which implies e (1 with e and without one, 0 without either), NULL when there is no match; REGEXP_SUBSTR_ALL the same, every match as an ARRAY of strings, empty without one. A ':' opening a bracket expression is literal, and ']' first in one is literal. REGEXP_LIKE also takes a VARCHAR pattern from a column, with or without literal parameters, relying on regexp_pattern_syntax: it runs only where the pattern is literal characters, '.', groups, alternation, plain bracket expressions, single '*', '+' or '?' and anchors at its ends, and raises otherwise.",
-        failures: &[],
+        conditions: "A literal pattern without lookarounds, group modifiers, lazy quantifiers (except +?, *? and ?? in REGEXP_SUBSTR, recorded reading as optional greedy repetitions, so as *, * and ?), POSIX classes or escapes other than \\d \\s \\w (ASCII, as recorded), their negations, escaped punctuation and, in REGEXP_SUBSTR and REGEXP_SUBSTR_ALL, \\n, \\t, \\r and, outside a bracket expression, the ASCII word boundary \\b, as recorded beside digits, underscores, punctuation and non-ASCII letters. REGEXP_LIKE matches the whole text; REGEXP_REPLACE replaces every match, or only the first, from position 1, case-sensitively or with 'i', with group references only; REGEXP_SUBSTR from position 1, any occurrence, with parameters of c, i, m, s and e and a group, which implies e (1 with e and without one, 0 without either), NULL when there is no match or the group is past the pattern's last; a position, occurrence or group from a NUMBER column is read as recorded: NULL gives NULL, a position or occurrence below 1 or a negative group raises 100050, a start position past the end gives NULL and ^ anchors at the start position (a pattern with \\b and such a position is not converted, and a pattern matching empty text raises regexp_pattern_syntax, its matches unrecorded); REGEXP_SUBSTR_ALL the same, every match as an ARRAY of strings, empty without one. A ':' opening a bracket expression is literal, and ']' first in one is literal. REGEXP_LIKE also takes a VARCHAR pattern from a column, with or without literal parameters, relying on regexp_pattern_syntax: it runs only where the pattern is literal characters, '.', groups, alternation, plain bracket expressions, single '*', '+' or '?' and anchors at its ends, and raises otherwise.",
+        failures: REGEXP_PARAMETER_FAILURES,
     },
     RuleSpec {
         id: NULL_ARITHMETIC,

crates/sqlquack-rules/src/conversion/_helpers/expressions/text_functions.rs

@@ -2,6 +2,8 @@
 
 use serde_json::Value;
 use sqlquack_core::classification::models::Construct;
+use sqlquack_core::failures::constants::INVALID_PARAMETER_VALUE;
+use sqlquack_core::failures::main::tagged_message::tagged_message;
 use sqlquack_core::refusals::main::outside_conditions::outside_conditions;
 use sqlquack_core::refusals::models::Refusal;
 use sqlquack_duckdb::blocks::main::cast::cast;
@@ -42,6 +44,13 @@ const TUPLE_KIND: &str = "tuple";
 const ILIKE_KIND: &str = "i_like";
 /// `\b`, an ASCII word boundary in both engines, as REGEXP_SUBSTR was recorded.
 const WORD_BOUNDARY_ESCAPE: char = 'b';
+/// `\b` as written in a pattern.
+const WORD_BOUNDARY: &str = "\\b";
+/// Where REGEXP_SUBSTR's parameters and group sit among the arguments after its pattern.
+const PARAMETERS_INDEX: usize = 2;
+const GROUP_INDEX: usize = 3;
+/// The start of Snowflake's message for a REGEXP_SUBSTR argument out of range.
+const INVALID_PARAMETER_MESSAGE: &str = "Invalid parameter value. Reason:";
 
 /// LPAD or RPAD: padded or cut to the length; an empty pad only cuts, as Snowflake documents.
 pub(crate) fn pad(
@@ -448,10 +457,20 @@ pub(crate) fn regexp(
         .map_err(|reason| refuse(&reason))?
         .then(|| log.rule(REGEXP_POSIX_SPACE));
     let pattern = string_literal(&pattern_text, log);
-    let mut literals: Vec<String> = Vec::with_capacity(rest.len());
+    let mut emitted: Vec<Emitted> = Vec::with_capacity(rest.len());
     for argument in rest {
-        let emitted = convert_expression(argument, site)?;
-        match emitted.literal {
+        emitted.push(convert_expression(argument, site)?);
+    }
+    let computed = emitted
+        .iter()
+        .enumerate()
+        .any(|(index, argument)| index != PARAMETERS_INDEX && argument.literal.is_none());
+    if computed && matches!(name, "REGEXP_SUBSTR") {
+        return regexp_substr_computed(&text, &pattern_text, &emitted, (construct, site));
+    }
+    let mut literals: Vec<String> = Vec::with_capacity(rest.len());
+    for argument in emitted {
+        match argument.literal {
             Some(Literal::Text(text) | Literal::Number(text)) => literals.push(text),
             None => return Err(refuse("a REGEXP argument other than a literal")),
         }
@@ -591,6 +610,118 @@ pub(crate) fn regexp(
     Ok(Emitted::new(sql, data_type))
 }
 
+/// REGEXP_SUBSTR whose position, occurrence or group is computed, read as A's receipts record.
+fn regexp_substr_computed(
+    text: &str,
+    pattern_text: &str,
+    arguments: &[Emitted],
+    (construct, site): (&Construct, Site<'_>),
+) -> Result<Emitted, Refusal> {
+    let refuse = |reason: &str| outside_conditions(construct.clone(), reason.to_owned());
+    let log = site.context.log;
+    let integer = |argument: Option<&Emitted>| match argument {
+        None => Ok("1".to_owned()),
+        Some(argument) => match argument.data_type {
+            SnowflakeType::Number { scale: 0, .. } | SnowflakeType::Null => {
+                Ok(argument.sql.clone())
+            }
+            other => Err(refuse(&format!(
+                "a REGEXP_SUBSTR position, occurrence or group of {other}"
+            ))),
+        },
+    };
+    let position = integer(arguments.first())?;
+    let occurrence = integer(arguments.get(1))?;
+    let parameters = match arguments
+        .get(PARAMETERS_INDEX)
+        .map(|argument| &argument.literal)
+    {
+        None => String::new(),
+        Some(Some(Literal::Text(parameters))) => parameters.clone(),
+        Some(_) => return Err(refuse("REGEXP_SUBSTR parameters other than a text literal")),
+    };
+    let flags = extracting_flags(&parameters)
+        .ok_or_else(|| refuse("REGEXP_SUBSTR parameters other than c, e, i, m and s"))?;
+    let group = match arguments.get(GROUP_INDEX) {
+        Some(group) => integer(Some(group))?,
+        None if parameters.contains('e') => "1".to_owned(),
+        None => "0".to_owned(),
+    };
+    let starts_later = arguments.first().is_some_and(
+        |position| !matches!(&position.literal, Some(Literal::Number(one)) if one == "1"),
+    );
+    if starts_later && pattern_text.contains(WORD_BOUNDARY) {
+        return Err(refuse(
+            "a word boundary after a start position, which no recording shows",
+        ));
+    }
+    let _parameters = (!flags.is_empty()).then(|| log.rule(REGEXP_PARAMETERS));
+    let pattern = string_literal(&format!("{flags}{}", greedy(pattern_text)), log);
+    let groups = capture_groups(pattern_text);
+    let extracted: Vec<String> = (0..=groups)
+        .map(|index| {
+            format!(
+                "WHEN {index} THEN regexp_extract_all(u, {pattern}, {index})\
+                 [CAST(least(r['o'], length(u) + 2) AS BIGINT)]"
+            )
+        })
+        .collect();
+    let raise = |reason: &str| {
+        raised(&tagged_message(
+            INVALID_PARAMETER_VALUE,
+            &format!("{INVALID_PARAMETER_MESSAGE} {reason}"),
+        ))
+    };
+    let empty_match = empty_match_guard("REGEXP_SUBSTR", &pattern, (construct, site))?;
+    Ok(Emitted::new(
+        format!(
+            "list_transform([struct_pack(t := {text}, p := {position}, o := {occurrence}, \
+             g := {group})], lambda r: CASE WHEN {empty_match} THEN CAST(NULL AS VARCHAR) \
+             WHEN r['t'] IS NULL OR r['p'] IS NULL \
+             OR r['o'] IS NULL OR r['g'] IS NULL THEN NULL \
+             WHEN r['p'] < 1 THEN CAST({} AS VARCHAR) WHEN r['o'] < 1 THEN CAST({} AS VARCHAR) \
+             WHEN r['g'] < 0 THEN CAST({} AS VARCHAR) \
+             ELSE list_transform([substr(r['t'], CAST(least(r['p'], length(r['t']) + 1) AS BIGINT))], \
+             lambda u: CASE CAST(least(r['g'], {}) AS BIGINT) {} ELSE NULL END)[1] END)[1]",
+            raise("Position must be positive"),
+            raise("Occurrence must be positive"),
+            raise("Capture group must be non-negative"),
+            groups + 1,
+            extracted.join(" ")
+        ),
+        SnowflakeType::Varchar,
+    ))
+}
+
+/// The capturing groups of a pattern `shared_pattern` accepted, which has no group modifiers.
+fn capture_groups(pattern: &str) -> usize {
+    let mut characters = pattern.chars().peekable();
+    let (mut groups, mut in_bracket) = (0, false);
+    while let Some(character) = characters.next() {
+        match character {
+            '\\' => {
+                characters.next();
+            }
+            '[' if !in_bracket => {
+                in_bracket = true;
+                characters.next_if_eq(&'^');
+                characters.next_if_eq(&']');
+            }
+            '[' if characters.peek() == Some(&':') => {
+                while let Some(class) = characters.next() {
+                    if class == ':' && characters.next_if_eq(&']').is_some() {
+                        break;
+                    }
+                }
+            }
+            ']' => in_bracket = false,
+            '(' if !in_bracket => groups += 1,
+            _ => {}
+        }
+    }
+    groups
+}
+
 /// A condition raising regexp_pattern_syntax where the pattern matches empty text; FALSE otherwise.
 fn empty_match_guard(
     name: &str,