From efa0953c331b68453740a995709248c0c34dabf9 Mon Sep 17 00:00:00 2001 From: Elie ROUDNINSKI Date: Fri, 13 Nov 2026 10:47:35 +0100 Subject: [PATCH] Support native execution contexts and composable field values Allow filters to read application-owned Rust contexts and nested collections without first constructing LhsValue fields. Use the same field-binding and reader interfaces for runtime-typed LhsValue storage, including ExecutionContext as the default and downstream field tables using LhsValueProvider. Provide composable value traits, context families, and a field-definition macro that binds typed getters during compilation. Support borrowing contexts, required and optional getters, getter attributes, field metadata, and reusable context state. Preserve Compiler hooks and hygienic access to application names in getters. Validate scheme coverage and provider types during explicit startup validation, including handwritten context families. The macro additionally validates duplicate and extra definitions and getter optionality. Select keys, indices, and [*] paths before materializing function arguments and custom collection comparisons, preserving terminal map iteration compatibility. Prepare fixed path splits during compilation and traverse concrete indices without a preliminary scan. Reuse LhsValue traversal for function results and move owned selections when flattening results. Share comparison code through typed getter callbacks and common execution helpers. Retain compact array indices and discard literal formatting and redundant selection state during compilation. Group context APIs and value interfaces by purpose, with value implementations alongside their types. Cover field readers, collection traversal, ownership, validation, and getter hygiene with tests and benchmarks. --- Cargo.lock | 7 + Cargo.toml | 1 + engine/Cargo.toml | 12 +- engine/benches/bench.rs | 5 + engine/benches/lhs_value_reader/mod.rs | 279 +++ engine/benches/reader_traversal/mod.rs | 337 +++ engine/src/ast/field_expr.rs | 212 +- engine/src/ast/function_expr.rs | 936 ++++++++- engine/src/ast/index_expr.rs | 1302 +++++++++--- engine/src/ast/logical_expr.rs | 5 +- engine/src/ast/mod.rs | 30 +- engine/src/compiler.rs | 70 +- engine/src/context/fields.rs | 1255 ++++++++++++ engine/src/context/mod.rs | 1646 +++++++++++++++ engine/src/context/state.rs | 457 +++++ engine/src/execution_context.rs | 81 +- engine/src/filter.rs | 243 ++- engine/src/functions/mod.rs | 3 +- engine/src/lhs_types/array.rs | 17 +- engine/src/lhs_types/bytes.rs | 48 + engine/src/lhs_types/map.rs | 6 +- engine/src/lhs_types/mod.rs | 889 +++++++- engine/src/lib.rs | 18 +- engine/src/rhs_types/regex/imp_real.rs | 16 + engine/src/rhs_types/regex/imp_stub.rs | 12 + engine/src/rhs_types/regex/mod.rs | 73 +- engine/src/scheme.rs | 69 + engine/src/searcher.rs | 29 +- engine/src/types.rs | 693 +------ engine/src/values/mod.rs | 1802 +++++++++++++++++ engine/tests/native_collection_comparisons.rs | 291 +++ engine/tests/native_family_validation.rs | 164 ++ engine/tests/native_getter_attributes.rs | 224 ++ engine/tests/native_getter_hygiene.rs | 128 ++ engine/tests/native_lhs_values.rs | 662 ++++++ engine/tests/native_operations.rs | 270 +++ engine/tests/reader_traversal.rs | 440 ++++ 37 files changed, 11642 insertions(+), 1090 deletions(-) create mode 100644 engine/benches/lhs_value_reader/mod.rs create mode 100644 engine/benches/reader_traversal/mod.rs create mode 100644 engine/src/context/fields.rs create mode 100644 engine/src/context/mod.rs create mode 100644 engine/src/context/state.rs create mode 100644 engine/src/values/mod.rs create mode 100644 engine/tests/native_collection_comparisons.rs create mode 100644 engine/tests/native_family_validation.rs create mode 100644 engine/tests/native_getter_attributes.rs create mode 100644 engine/tests/native_getter_hygiene.rs create mode 100644 engine/tests/native_lhs_values.rs create mode 100644 engine/tests/native_operations.rs create mode 100644 engine/tests/reader_traversal.rs diff --git a/Cargo.lock b/Cargo.lock index df973240..4f95ef27 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -142,6 +142,12 @@ version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + [[package]] name = "cast" version = "0.3.0" @@ -1226,6 +1232,7 @@ version = "0.7.0" dependencies = [ "arcstr", "backtrace", + "bytes", "cfg-if", "cidr", "criterion", diff --git a/Cargo.toml b/Cargo.toml index 1d8b2591..4dfcb6c5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,7 @@ publish = true [workspace.dependencies] arcstr = "1.2.0" backtrace = "0.3.76" +bytes = "1.12.1" cbindgen = "0.29.4" cfg-if = "1.0.4" cidr = { version = "0.2.3", features = ["serde"] } diff --git a/engine/Cargo.toml b/engine/Cargo.toml index 2add20fa..ca4d47c6 100644 --- a/engine/Cargo.toml +++ b/engine/Cargo.toml @@ -11,6 +11,12 @@ keywords = ["engine", "filter", "parser", "runtime", "wireshark"] categories = ["config", "parser-implementations"] publish.workspace = true +[features] +default = ["get-size2", "regex"] +bytes = ["dep:bytes"] +get-size2 = ["dep:get-size2"] +regex = ["dep:regex-automata"] + [lib] bench = false name = "wirefilter" @@ -22,6 +28,7 @@ name = "bench" [dependencies] arcstr.workspace = true backtrace.workspace = true +bytes = { workspace = true, optional = true } cfg-if.workspace = true cidr.workspace = true dyn-clone.workspace = true @@ -46,8 +53,3 @@ serde_json.workspace = true # This optional dependency allows us to build with `--features getrandom/wasm_js`. # For more information see: https://docs.rs/getrandom/#webassembly-support getrandom.workspace = true - -[features] -default = ["get-size2", "regex"] -get-size2 = ["dep:get-size2"] -regex = ["dep:regex-automata"] diff --git a/engine/benches/bench.rs b/engine/benches/bench.rs index e79d04db..16d02c4e 100644 --- a/engine/benches/bench.rs +++ b/engine/benches/bench.rs @@ -1,3 +1,6 @@ +mod lhs_value_reader; +mod reader_traversal; + use std::alloc::System; // Most of our usage will be via FFI as a dynamic library, so we're interested @@ -250,6 +253,8 @@ criterion_group! { bench_string_comparisons, bench_string_matches, bench_string_function_comparison, + lhs_value_reader::bench, + reader_traversal::bench, } criterion_main!(field_benchmarks); diff --git a/engine/benches/lhs_value_reader/mod.rs b/engine/benches/lhs_value_reader/mod.rs new file mode 100644 index 00000000..65e1a5a7 --- /dev/null +++ b/engine/benches/lhs_value_reader/mod.rs @@ -0,0 +1,279 @@ +//! Compare built-in and downstream field getters using the same LhsValue reader. +//! +//! `execution_context` uses the default family and internal field accessor. +//! `public_getter` uses a downstream getter and the public field accessor, which +//! checks the scheme on each read. + +use criterion::{BenchmarkId, Criterion}; +use std::hint::black_box; +use wirefilter::{ + Array, Compare, CompiledExpr, ContextFamily, DefaultCompiler, ExecutionContext, Field, + FieldBindingError, FieldConsumer, FieldGetter, FieldIndex, FunctionArgs, IdentifierExpr, + IndexExpr, LhsValue, LhsValueProvider, Map, Scheme, SchemeBuilder, SimpleFunctionArgKind, + SimpleFunctionDefinition, SimpleFunctionImpl, SimpleFunctionParam, Type, TypeMismatchError, +}; + +struct LhsFields; + +impl ContextFamily for LhsFields { + type Context<'data> = ExecutionContext<'data>; + type UserData = (); + + fn bind>( + field: Field, + consumer: B, + ) -> Result { + consumer.consume(field, LhsGetter) + } +} + +struct LhsGetter; + +impl FieldGetter for LhsGetter { + type Provider = LhsValueProvider; + + fn get<'ctx, 'data: 'ctx>( + &self, + context: &'ctx ExecutionContext<'data>, + field: &Field, + ) -> Option<&'ctx LhsValue<'ctx>> { + context.get_field_value(field.as_ref()) + } +} + +fn count<'a>(args: FunctionArgs<'_, 'a>) -> Option> { + let len = match args.next()?.ok()? { + LhsValue::Array(values) => values.len(), + LhsValue::Map(values) => values.len(), + _ => unreachable!(), + }; + Some(LhsValue::Int(i64::try_from(len).unwrap())) +} + +fn has_header<'a>(args: FunctionArgs<'_, 'a>) -> Option> { + let LhsValue::Bytes(name) = args.next()?.ok()? else { + unreachable!() + }; + let LhsValue::Map(headers) = args.next()?.ok()? else { + unreachable!() + }; + Some(LhsValue::Bool(headers.get(name.as_ref()).is_some())) +} + +fn function( + params: &[Type], + return_type: Type, + implementation: SimpleFunctionImpl, +) -> SimpleFunctionDefinition { + SimpleFunctionDefinition { + params: params + .iter() + .map(|&val_type| SimpleFunctionParam { + arg_kind: SimpleFunctionArgKind::Field, + val_type, + }) + .collect(), + opt_params: Box::new([]), + return_type, + implementation, + } +} + +fn context() -> (Scheme, ExecutionContext<'static>) { + let strings = Type::array(Type::Bytes); + let headers = Type::map(strings); + let mut builder = SchemeBuilder::new(); + for (name, ty) in [ + ("flag", Type::Bool), + ("port", Type::Int), + ("ip", Type::Ip), + ("text", Type::Bytes), + ("numbers", Type::array(Type::Int)), + ("matrix", Type::array(Type::array(Type::Int))), + ("names", strings), + ("headers", headers), + ] { + builder.add_field(name, ty).unwrap(); + } + for (name, params, output, implementation) in [ + ( + "lower", + vec![Type::Bytes], + Type::Bytes, + SimpleFunctionImpl::new(super::lowercase), + ), + ( + "count_values", + vec![strings], + Type::Int, + SimpleFunctionImpl::new(count), + ), + ( + "count_headers", + vec![headers], + Type::Int, + SimpleFunctionImpl::new(count), + ), + ( + "has_header", + vec![Type::Bytes, headers], + Type::Bool, + SimpleFunctionImpl::new(has_header), + ), + ] { + builder + .add_function(name, function(¶ms, output, implementation)) + .unwrap(); + } + let scheme = builder.build(); + let mut context = ExecutionContext::new(&scheme); + for (name, value) in [ + ("flag", LhsValue::Bool(true)), + ("port", LhsValue::Int(443)), + ("ip", LhsValue::Ip("192.0.2.1".parse().unwrap())), + ("text", LhsValue::from("example-value-0123456789")), + ( + "numbers", + Array::try_from_iter(Type::Int, 0..64_i64).unwrap().into(), + ), + ( + "matrix", + Array::try_from_iter( + Type::array(Type::Int), + (0..8).map(|row| Array::try_from_iter(Type::Int, row * 8..row * 8 + 8).unwrap()), + ) + .unwrap() + .into(), + ), + ( + "names", + Array::try_from_iter( + Type::Bytes, + (0..64).map(|index| LhsValue::from(format!("x-test-{index}").into_bytes())), + ) + .unwrap() + .into(), + ), + ( + "headers", + Map::try_from_iter( + strings, + (0..64).map(|index| { + Ok::<_, TypeMismatchError>(( + format!("x-test-{index}").into_bytes().into_boxed_slice(), + Array::try_from_iter(Type::Bytes, ["clean", "also-clean"]).unwrap(), + )) + }), + ) + .unwrap() + .into(), + ), + ] { + context + .set_field_value(scheme.get_field(name).unwrap(), value) + .unwrap(); + } + (scheme, context) +} + +pub(super) fn bench(c: &mut Criterion) { + let (scheme, context) = context(); + let mut group = c.benchmark_group("lhs_value_reader"); + for (name, expression) in [ + ("bool", "flag"), + ("integer_ordering", "port > 80"), + ("ip_membership", "ip in {192.0.2.0/24}"), + ("bytes_equal", r#"text eq "example-value-0123456789""#), + ("bytes_contains", r#"text contains "value""#), + ("array_index", "numbers[63] eq 63"), + ("array_missing", "numbers[100] eq 63"), + ("array_each", "any(numbers[*] eq 63)"), + ("nested_each", "any(matrix[*][*] eq 63)"), + ("map_index", r#"headers["x-test-0"][0] eq "clean""#), + ("map_missing", r#"headers["missing"][0] eq "clean""#), + ("map_each", r#"any(headers[*][*] eq "missing")"#), + ( + "selected_function", + r#"any(lower(headers["x-test-0"][*])[*] eq "clean")"#, + ), + ("array_function", "count_values(names) eq 64"), + ("map_function", "count_headers(headers) eq 64"), + ("map_each_function", "all(count_values(headers[*])[*] eq 2)"), + ("mapped_extra_map", "all(has_header(names[*], headers))"), + ] { + let ast = scheme.parse(expression).unwrap(); + let execution_context = ast.clone().compile(); + let public_getter = ast.compile_with_compiler(&mut DefaultCompiler::<(), LhsFields>::new()); + let expected = execution_context.execute(&context).unwrap(); + assert_eq!(expected, public_getter.execute(&context).unwrap(), "{name}"); + group.bench_function(BenchmarkId::new("execution_context", name), |b| { + b.iter(|| execution_context.execute(black_box(&context)).unwrap()) + }); + group.bench_function(BenchmarkId::new("public_getter", name), |b| { + b.iter(|| public_getter.execute(black_box(&context)).unwrap()) + }); + } + for (name, field, indexes) in [ + ("custom_map", "headers", vec![]), + ( + "custom_selected_array", + "headers", + vec![FieldIndex::MapKey("x-test-0".into())], + ), + ("custom_each_array", "matrix", vec![FieldIndex::MapEach]), + ] { + let index = IndexExpr { + identifier: IdentifierExpr::Field(scheme.get_field(field).unwrap().to_owned()), + indexes: indexes.into_boxed_slice(), + }; + let execution_context = + index + .clone() + .compile_with(&mut DefaultCompiler::<()>::new(), false, Nonempty); + let public_getter = index.compile_with( + &mut DefaultCompiler::<(), LhsFields>::new(), + false, + Nonempty, + ); + match (execution_context, public_getter) { + (CompiledExpr::One(execution_context), CompiledExpr::One(public_getter)) => { + let expected = execution_context.execute(&context); + assert_eq!(expected, public_getter.execute(&context)); + group.bench_function(BenchmarkId::new("execution_context", name), |b| { + b.iter(|| execution_context.execute(black_box(&context))) + }); + group.bench_function(BenchmarkId::new("public_getter", name), |b| { + b.iter(|| public_getter.execute(black_box(&context))) + }); + } + (CompiledExpr::Vec(execution_context), CompiledExpr::Vec(public_getter)) => { + let expected = execution_context.execute(&context); + assert_eq!(expected, public_getter.execute(&context)); + group.bench_function(BenchmarkId::new("execution_context", name), |b| { + b.iter(|| execution_context.execute(black_box(&context))) + }); + group.bench_function(BenchmarkId::new("public_getter", name), |b| { + b.iter(|| public_getter.execute(black_box(&context))) + }); + } + _ => panic!("comparison cardinality changed"), + } + } + group.finish(); +} + +struct Nonempty; + +impl> Compare<(), F> for Nonempty { + fn compare<'ctx, 'data: 'ctx>( + &self, + value: &LhsValue<'ctx>, + _: &'ctx F::Context<'data>, + ) -> bool { + match value { + LhsValue::Array(values) => !values.is_empty(), + LhsValue::Map(values) => !values.is_empty(), + _ => panic!("custom comparator expects a collection"), + } + } +} diff --git a/engine/benches/reader_traversal/mod.rs b/engine/benches/reader_traversal/mod.rs new file mode 100644 index 00000000..01e7c1b4 --- /dev/null +++ b/engine/benches/reader_traversal/mod.rs @@ -0,0 +1,337 @@ +//! Function-result traversal and native custom-collection comparison. + +use criterion::Criterion; +use std::hint::black_box; +use wirefilter::{ + Array, Compare, CompiledExpr, CompiledValueExpr, ContextFamily, DefaultCompiler, + EvaluationContext, ExecutionContext, FieldIndex, FilterValueExpr, FunctionArgs, HasScheme, + IdentifierExpr, IndexExpr, LhsValue, ListMatcher, ListRef, Scheme, SchemeBuilder, + SimpleFunctionArgKind, SimpleFunctionDefinition, SimpleFunctionImpl, SimpleFunctionParam, Type, + ValueExpr, +}; + +fn echo<'a>(args: FunctionArgs<'_, 'a>) -> Option> { + args.next()?.ok() +} + +fn own<'a>(args: FunctionArgs<'_, 'a>) -> Option> { + Some(args.next()?.ok()?.into_owned()) +} + +fn function(ty: Type, implementation: SimpleFunctionImpl) -> SimpleFunctionDefinition { + SimpleFunctionDefinition { + params: vec![SimpleFunctionParam { + arg_kind: SimpleFunctionArgKind::Field, + val_type: ty, + }] + .into_boxed_slice(), + opt_params: Box::new([]), + return_type: ty, + implementation, + } +} + +struct NativeContext { + scheme: Scheme, + number: i64, + optional_number: Option, + text: Box, + optional_text: Option>, + numbers: Vec, + matrix: Vec>, + cube: Vec>>, + flags: Vec, +} + +impl HasScheme for NativeContext { + fn scheme(&self) -> &Scheme { + &self.scheme + } +} + +impl EvaluationContext for NativeContext { + type UserData = (); + + fn get_user_data(&self) -> &() { + &() + } + + fn get_list_matcher(&self, _: ListRef<'_>) -> &dyn ListMatcher { + unreachable!("this benchmark has no lists") + } +} + +wirefilter::wirefilter_fields! { + impl NativeFields for NativeContext { + "number" => { fn(ctx: &NativeContext) -> i64 { ctx.number } }, + "optional_number" => { fn(ctx: &NativeContext) -> Option { ctx.optional_number } }, + "text" => { fn(ctx: &NativeContext) -> &str { &ctx.text } }, + "optional_text" => { fn(ctx: &NativeContext) -> Option<&str> { ctx.optional_text.as_deref() } }, + "numbers" => { fn(ctx: &NativeContext) -> &[i64] { &ctx.numbers } }, + "matrix" => { fn(ctx: &NativeContext) -> &[Vec] { &ctx.matrix } }, + "cube" => { fn(ctx: &NativeContext) -> &[Vec>] { &ctx.cube } }, + "flags" => { fn(ctx: &NativeContext) -> &[bool] { &ctx.flags } }, + } +} + +fn context() -> (Scheme, ExecutionContext<'static>, NativeContext) { + let numbers = Type::array(Type::Int); + let matrix = Type::array(numbers); + let cube = Type::array(matrix); + let flags = Type::array(Type::Bool); + let mut builder = SchemeBuilder::new(); + for (name, ty) in [ + ("number", Type::Int), + ("text", Type::Bytes), + ("numbers", numbers), + ("matrix", matrix), + ("cube", cube), + ("flags", flags), + ] { + builder.add_field(name, ty).unwrap(); + builder + .add_function( + format!("echo_{name}"), + function(ty, SimpleFunctionImpl::new(echo)), + ) + .unwrap(); + builder + .add_function( + format!("own_{name}"), + function(ty, SimpleFunctionImpl::new(own)), + ) + .unwrap(); + } + builder + .add_optional_field("optional_number", Type::Int) + .unwrap(); + builder + .add_optional_field("optional_text", Type::Bytes) + .unwrap(); + let scheme = builder.build(); + let native = NativeContext { + scheme: scheme.clone(), + number: 63, + optional_number: Some(63), + text: "target".into(), + optional_text: Some("target".into()), + numbers: (0..64).collect(), + matrix: (0..8).map(|row| (row * 8..row * 8 + 8).collect()).collect(), + cube: (0..4) + .map(|plane| { + (0..4) + .map(|row| (plane * 16 + row * 4..plane * 16 + row * 4 + 4).collect()) + .collect() + }) + .collect(), + flags: (0..64).map(|i| i == 63).collect(), + }; + NativeFields::validate_scheme(&scheme).unwrap(); + let mut context = ExecutionContext::new(&scheme); + context + .set_field_value_from_name("number", native.number) + .unwrap(); + context + .set_field_value_from_name("numbers", Array::from_iter(native.numbers.iter().copied())) + .unwrap(); + context + .set_field_value_from_name( + "matrix", + Array::try_from_iter( + numbers, + native + .matrix + .iter() + .map(|row| Array::from_iter(row.iter().copied())), + ) + .unwrap(), + ) + .unwrap(); + context + .set_field_value_from_name("flags", Array::from_iter(native.flags.iter().copied())) + .unwrap(); + (scheme, context, native) +} + +fn compile_value_selection(scheme: &Scheme, expression: &str, depth: usize) -> CompiledValueExpr { + // Parse a complete function call, then benchmark its indexing node directly. + // Top-level value parsing rejects [*], while function arguments compile this + // same node and consume its flattened result. + let ast = scheme.parse_value(expression).unwrap(); + let FilterValueExpr::Index(mut index) = ast.expression().clone() else { + unreachable!() + }; + index.indexes = vec![FieldIndex::MapEach; depth].into_boxed_slice(); + index.compile() +} + +pub(super) fn bench(c: &mut Criterion) { + let (scheme, context, mut native) = context(); + let mut group = c.benchmark_group("reader_traversal"); + for (name, expression) in [ + ("function_scalar", "echo_number(number) eq 63"), + ("function_index", "echo_numbers(numbers)[63] eq 63"), + ( + "function_flat_borrowed", + "any(echo_numbers(numbers)[*] eq 63)", + ), + ("function_flat_owned", "any(own_numbers(numbers)[*] eq 63)"), + ( + "function_nested_borrowed", + "any(echo_matrix(matrix)[*][*] eq 63)", + ), + ( + "function_nested_owned", + "any(own_matrix(matrix)[*][*] eq 63)", + ), + ("implicit_bool_array", "any(flags)"), + ("implicit_bool_function_borrowed", "any(echo_flags(flags))"), + ("implicit_bool_function_owned", "any(own_flags(flags))"), + ] { + let filter = scheme.parse(expression).unwrap().compile(); + assert!(filter.execute(&context).unwrap(), "{name}"); + group.bench_function(name, |b| { + b.iter(|| filter.execute(black_box(&context)).unwrap()) + }); + } + for (name, expression, depth, expected_type, expected_len) in [ + ( + "value_root_borrowed", + "echo_matrix(matrix)", + 0, + Type::array(Type::array(Type::Int)), + 8, + ), + ( + "value_root_owned", + "own_matrix(matrix)", + 0, + Type::array(Type::array(Type::Int)), + 8, + ), + ( + "value_flat_borrowed", + "echo_numbers(numbers)", + 1, + Type::array(Type::Int), + 64, + ), + ( + "value_nested_borrowed", + "echo_matrix(matrix)", + 2, + Type::array(Type::Int), + 64, + ), + ( + "value_nested_owned", + "own_matrix(matrix)", + 2, + Type::array(Type::Int), + 64, + ), + ] { + let value = compile_value_selection(&scheme, expression, depth); + let result = value.execute(&context).unwrap(); + assert_eq!(wirefilter::GetType::get_type(&result), expected_type); + let LhsValue::Array(array) = result else { + unreachable!() + }; + assert_eq!(array.len(), expected_len); + group.bench_function(name, |b| { + b.iter(|| value.execute(black_box(&context)).unwrap()) + }); + } + let implicit = scheme + .parse("any(flags)") + .unwrap() + .compile_with_compiler(&mut DefaultCompiler::<(), NativeFields>::new()); + assert!(implicit.execute(&native).unwrap()); + group.bench_function("implicit_bool_native", |b| { + b.iter(|| implicit.execute(black_box(&native)).unwrap()) + }); + + for (name, expression) in [ + ("native_scalar", "number eq 63"), + ("native_optional_scalar", "optional_number eq 63"), + ("native_bytes", r#"text eq "target""#), + ("native_optional_bytes", r#"optional_text eq "target""#), + ("native_array", "any(numbers[*] eq 63)"), + ("native_nested", "any(matrix[*][*] eq 63)"), + ] { + let filter = scheme + .parse(expression) + .unwrap() + .compile_with_compiler(&mut DefaultCompiler::<(), NativeFields>::new()); + assert!(filter.execute(&native).unwrap(), "{name}"); + group.bench_function(name, |b| { + b.iter(|| filter.execute(black_box(&native)).unwrap()) + }); + } + + for (name, field, indexes) in [ + ("custom_native_root", "matrix", vec![]), + ( + "custom_native_index", + "matrix", + vec![FieldIndex::ArrayIndex(3)], + ), + ("custom_native_flat", "matrix", vec![FieldIndex::MapEach]), + ( + "custom_native_nested", + "cube", + vec![FieldIndex::MapEach, FieldIndex::MapEach], + ), + ] { + let expression = IndexExpr { + identifier: IdentifierExpr::Field(scheme.get_field(field).unwrap().to_owned()), + indexes: indexes.into_boxed_slice(), + } + .compile_with( + &mut DefaultCompiler::<(), NativeFields>::new(), + false, + Nonempty, + ); + match expression { + CompiledExpr::One(expression) => { + assert!(expression.execute(&native)); + group.bench_function(name, |b| b.iter(|| expression.execute(black_box(&native)))); + } + CompiledExpr::Vec(expression) => { + assert!(!expression.execute(&native).is_empty()); + group.bench_function(name, |b| b.iter(|| expression.execute(black_box(&native)))); + } + } + } + native.optional_number = None; + native.optional_text = None; + for (name, expression) in [ + ("native_missing_scalar", "optional_number eq 63"), + ("native_missing_bytes", r#"optional_text eq "target""#), + ] { + let filter = scheme + .parse(expression) + .unwrap() + .compile_with_compiler(&mut DefaultCompiler::<(), NativeFields>::new()); + assert!(!filter.execute(&native).unwrap(), "{name}"); + group.bench_function(name, |b| { + b.iter(|| filter.execute(black_box(&native)).unwrap()) + }); + } + group.finish(); +} + +struct Nonempty; + +impl> Compare<(), F> for Nonempty { + fn compare<'ctx, 'data: 'ctx>( + &self, + value: &LhsValue<'ctx>, + _: &'ctx F::Context<'data>, + ) -> bool { + let LhsValue::Array(values) = value else { + panic!("expected array") + }; + !values.is_empty() + } +} diff --git a/engine/src/ast/field_expr.rs b/engine/src/ast/field_expr.rs index 6b45ef65..e6abc240 100644 --- a/engine/src/ast/field_expr.rs +++ b/engine/src/ast/field_expr.rs @@ -6,16 +6,14 @@ use crate::ast::index_expr::{Compare, IndexExpr}; use crate::compiler::Compiler; use crate::filter::{CompiledExpr, CompiledOneExpr, CompiledVecExpr}; use crate::lex::{Lex, LexErrorKind, LexResult, LexWith, expect, skip_space, span}; -use crate::lhs_types::TypedArray; +use crate::lhs_types::{LhsValue, TypedArray}; use crate::range_set::RangeSet; use crate::rhs_types::{BytesExpr, ExplicitIpRange, ListName, Regex, Wildcard}; use crate::scheme::{Field, Identifier, List}; use crate::searcher::{EmptySearcher, MemmemSearcher}; use crate::strict_partial_ord::StrictPartialOrd; -use crate::types::{ - ExpectedTypeList, GetType, LhsValue, LiteralSet, LiteralValue, Type, TypeMismatchError, -}; -use crate::{ExecutionContext, Scheme}; +use crate::types::{ExpectedTypeList, GetType, LiteralSet, LiteralValue, Type, TypeMismatchError}; +use crate::{ContextFamily, EvaluationContext, Scheme}; use serde::{Serialize, Serializer}; use sliceslice::MemchrSearcher; use std::cmp::Ordering; @@ -710,11 +708,18 @@ macro_rules! cast_value { }; } -impl Compare for Wildcard { +impl> Compare + for Wildcard +{ #[inline] - fn compare<'e>(&self, value: &LhsValue<'e>, _: &'e ExecutionContext<'e, U>) -> bool { + fn compare<'e, 'data: 'e>(&self, value: &LhsValue<'e>, _: &'e F::Context<'data>) -> bool { self.is_match(cast_value!(value, Bytes)) } + + #[inline] + fn compare_bytes<'e, 'data: 'e>(&self, value: &'e [u8], _: &'e F::Context<'data>) -> bool { + self.is_match(value) + } } fn compile_dynamic_comparison( @@ -724,7 +729,7 @@ fn compile_dynamic_comparison( missing_one_default: bool, missing_both_default: bool, compiler: &mut C, -) -> CompiledExpr +) -> CompiledExpr where C: Compiler, F: for<'l, 'r> Fn(&LhsValue<'l>, &LhsValue<'r>) -> bool + Copy + Send + Sync + 'static, @@ -782,7 +787,7 @@ impl Expr for ComparisonExpr { } } - fn compile_with_compiler(self, compiler: &mut C) -> CompiledExpr { + fn compile_with_compiler(self, compiler: &mut C) -> CompiledExpr { let lhs = self.lhs; let nil_not_equal_behavior = lhs.identifier.scheme().nil_not_equal_behavior(); @@ -790,15 +795,24 @@ impl Expr for ComparisonExpr { ComparisonOpExpr::IsTrue => { struct IsTrue; - impl Compare for IsTrue { + impl> Compare for IsTrue { #[inline] - fn compare<'e>( + fn compare<'e, 'data: 'e>( &self, value: &LhsValue<'e>, - _: &'e ExecutionContext<'e, U>, + _: &'e F::Context<'data>, ) -> bool { *cast_value!(value, Bool) } + + #[inline] + fn compare_bool<'e, 'data: 'e>( + &self, + value: bool, + _: &'e F::Context<'data>, + ) -> bool { + value + } } if lhs.get_type() == Type::Bool { @@ -866,46 +880,55 @@ impl Expr for ComparisonExpr { }; macro_rules! gen_ordering { - ($op:tt, $def:ident) => { + ($ordering:ident, $op:tt, $def:ident) => { match rhs { LiteralValue::Bytes(bytes) => { - struct BytesOp(BytesExpr); + struct BytesOp(Box<[u8]>); - impl Compare for BytesOp { + impl> Compare for BytesOp { #[inline] - fn compare<'e>(&self, value: &LhsValue<'e>, _: &'e ExecutionContext<'e, U>) -> bool { + fn compare<'e, 'data: 'e>(&self, value: &LhsValue<'e>, _: &'e F::Context<'data>) -> bool { cast_value!(value, Bytes).as_ref() $op self.0.as_ref() } + #[inline] + fn compare_bytes<'e, 'data: 'e>(&self, value: &'e [u8], _: &'e F::Context<'data>) -> bool { + value $op self.0.as_ref() + } } - lhs.compile_with(compiler, $def, BytesOp(bytes)) + lhs.compile_with(compiler, $def, BytesOp(bytes.into())) } LiteralValue::Int(int) => { struct IntOp(i64); - impl Compare for IntOp { + impl> Compare for IntOp { #[inline] - fn compare<'e>(&self, value: &LhsValue<'e>, _: &'e ExecutionContext<'e, U>) -> bool { + fn compare<'e, 'data: 'e>(&self, value: &LhsValue<'e>, _: &'e F::Context<'data>) -> bool { *cast_value!(value, Int) $op self.0 } + #[inline] + fn compare_int<'e, 'data: 'e>(&self, value: i64, _: &'e F::Context<'data>) -> bool { + value $op self.0 + } } lhs.compile_with(compiler, $def, IntOp(int)) } LiteralValue::Ip(ip) => { - struct IpOp { - op: OrderingOp, - ip: IpAddr, - } + struct IpOp(IpAddr); - impl Compare for IpOp { + impl> Compare for IpOp { #[inline] - fn compare<'e>(&self, value: &LhsValue<'e>, _: &'e ExecutionContext<'e, U>) -> bool { - self.op.matches_opt(cast_value!(value, Ip).strict_partial_cmp(&self.ip)) + fn compare<'e, 'data: 'e>(&self, value: &LhsValue<'e>, _: &'e F::Context<'data>) -> bool { + OrderingOp::$ordering.matches_opt(cast_value!(value, Ip).strict_partial_cmp(&self.0)) + } + #[inline] + fn compare_ip<'e, 'data: 'e>(&self, value: IpAddr, _: &'e F::Context<'data>) -> bool { + OrderingOp::$ordering.matches_opt(value.strict_partial_cmp(&self.0)) } } - lhs.compile_with(compiler, $def, IpOp { op, ip }) + lhs.compile_with(compiler, $def, IpOp(ip)) } LiteralValue::Bool(_) | LiteralValue::Array(_) | LiteralValue::Map(_) => unreachable!(), } @@ -913,12 +936,12 @@ impl Expr for ComparisonExpr { } match op { - OrderingOp::NotEqual => gen_ordering!(!=, nil_not_equal_behavior), - OrderingOp::Equal => gen_ordering!(==, false), - OrderingOp::GreaterThanEqual => gen_ordering!(>=, false), - OrderingOp::LessThanEqual => gen_ordering!(<=, false), - OrderingOp::GreaterThan => gen_ordering!(>, false), - OrderingOp::LessThan => gen_ordering!(<, false), + OrderingOp::NotEqual => gen_ordering!(NotEqual, !=, nil_not_equal_behavior), + OrderingOp::Equal => gen_ordering!(Equal, ==, false), + OrderingOp::GreaterThanEqual => gen_ordering!(GreaterThanEqual, >=, false), + OrderingOp::LessThanEqual => gen_ordering!(LessThanEqual, <=, false), + OrderingOp::GreaterThan => gen_ordering!(GreaterThan, >, false), + OrderingOp::LessThan => gen_ordering!(LessThan, <, false), } } ComparisonOpExpr::Int { @@ -941,15 +964,24 @@ impl Expr for ComparisonExpr { struct BitwiseAnd(i64); - impl Compare for BitwiseAnd { + impl> Compare for BitwiseAnd { #[inline] - fn compare<'e>( + fn compare<'e, 'data: 'e>( &self, value: &LhsValue<'e>, - _: &'e ExecutionContext<'e, U>, + _: &'e F::Context<'data>, ) -> bool { cast_value!(value, Int) & self.0 != 0 } + + #[inline] + fn compare_int<'e, 'data: 'e>( + &self, + value: i64, + _: &'e F::Context<'data>, + ) -> bool { + value & self.0 != 0 + } } lhs.compile_with(compiler, false, BitwiseAnd(rhs)) } @@ -975,28 +1007,50 @@ impl Expr for ComparisonExpr { struct ArraySearcher(Avx2Searcher<[u8; N]>); - impl Compare for ArraySearcher { + impl> Compare + for ArraySearcher + { #[inline] - fn compare<'e>( + fn compare<'e, 'data: 'e>( &self, value: &LhsValue<'e>, - _: &'e ExecutionContext<'e, U>, + _: &'e F::Context<'data>, ) -> bool { unsafe { self.0.search_in(cast_value!(value, Bytes)) } } + + #[inline] + fn compare_bytes<'e, 'data: 'e>( + &self, + value: &'e [u8], + _: &'e F::Context<'data>, + ) -> bool { + // SAFETY: This searcher is only constructed after USE_AVX2 succeeds. + unsafe { self.0.search_in(value) } + } } struct BoxSearcher(Avx2Searcher>); - impl Compare for BoxSearcher { + impl> Compare for BoxSearcher { #[inline] - fn compare<'e>( + fn compare<'e, 'data: 'e>( &self, value: &LhsValue<'e>, - _: &'e ExecutionContext<'e, U>, + _: &'e F::Context<'data>, ) -> bool { unsafe { self.0.search_in(cast_value!(value, Bytes)) } } + + #[inline] + fn compare_bytes<'e, 'data: 'e>( + &self, + value: &'e [u8], + _: &'e F::Context<'data>, + ) -> bool { + // SAFETY: This searcher is only constructed after USE_AVX2 succeeds. + unsafe { self.0.search_in(value) } + } } fn slice_to_array(slice: &[u8]) -> [u8; N] { @@ -1079,15 +1133,25 @@ impl Expr for ComparisonExpr { struct WasmSearcher(Wasm32Searcher>); - impl Compare for WasmSearcher { + impl> Compare for WasmSearcher { #[inline] - fn compare<'e>( + fn compare<'e, 'data: 'e>( &self, value: &LhsValue<'e>, - _: &'e ExecutionContext<'e, U>, + _: &'e F::Context<'data>, ) -> bool { unsafe { self.0.search_in(cast_value!(value, Bytes)) } } + + #[inline] + fn compare_bytes<'e, 'data: 'e>( + &self, + value: &'e [u8], + _: &'e F::Context<'data>, + ) -> bool { + // SAFETY: This searcher is only constructed after USE_SIMD128 succeeds. + unsafe { self.0.search_in(value) } + } } let position = rng().random_range(1..bytes.len()); @@ -1099,7 +1163,9 @@ impl Expr for ComparisonExpr { search!(MemmemSearcher::new(bytes)) } - ComparisonOpExpr::Matches(regex) => lhs.compile_with(compiler, false, regex), + ComparisonOpExpr::Matches(regex) => { + lhs.compile_with(compiler, false, regex.into_compiled()) + } ComparisonOpExpr::Wildcard(wildcard) => lhs.compile_with(compiler, false, wildcard), ComparisonOpExpr::StrictWildcard(wildcard) => { lhs.compile_with(compiler, false, wildcard) @@ -1122,18 +1188,30 @@ impl Expr for ComparisonExpr { v6: RangeSet, } - impl Compare for OneOfIp { + impl> Compare for OneOfIp { #[inline] - fn compare<'e>( + fn compare<'e, 'data: 'e>( &self, value: &LhsValue<'e>, - _: &'e ExecutionContext<'e, U>, + _: &'e F::Context<'data>, ) -> bool { match cast_value!(value, Ip) { IpAddr::V4(addr) => self.v4.contains(addr), IpAddr::V6(addr) => self.v6.contains(addr), } } + + #[inline] + fn compare_ip<'e, 'data: 'e>( + &self, + value: IpAddr, + _: &'e F::Context<'data>, + ) -> bool { + match &value { + IpAddr::V4(addr) => self.v4.contains(addr), + IpAddr::V6(addr) => self.v6.contains(addr), + } + } } lhs.compile_with(compiler, false, OneOfIp { v4, v6 }) @@ -1143,15 +1221,24 @@ impl Expr for ComparisonExpr { struct OneOfInt(RangeSet); - impl Compare for OneOfInt { + impl> Compare for OneOfInt { #[inline] - fn compare<'e>( + fn compare<'e, 'data: 'e>( &self, value: &LhsValue<'e>, - _: &'e ExecutionContext<'e, U>, + _: &'e F::Context<'data>, ) -> bool { self.0.contains(cast_value!(value, Int)) } + + #[inline] + fn compare_int<'e, 'data: 'e>( + &self, + value: i64, + _: &'e F::Context<'data>, + ) -> bool { + self.0.contains(&value) + } } lhs.compile_with(compiler, false, OneOfInt(values)) @@ -1161,15 +1248,24 @@ impl Expr for ComparisonExpr { struct Contains(BTreeSet>); - impl Compare for Contains { + impl> Compare for Contains { #[inline] - fn compare<'e>( + fn compare<'e, 'data: 'e>( &self, value: &LhsValue<'e>, - _: &'e ExecutionContext<'e, U>, + _: &'e F::Context<'data>, ) -> bool { self.0.contains(cast_value!(value, Bytes).as_ref()) } + + #[inline] + fn compare_bytes<'e, 'data: 'e>( + &self, + value: &'e [u8], + _: &'e F::Context<'data>, + ) -> bool { + self.0.contains(value) + } } lhs.compile_with(compiler, false, Contains(values)) @@ -1187,12 +1283,12 @@ impl Expr for ComparisonExpr { list: List, } - impl Compare for InList { + impl> Compare for InList { #[inline] - fn compare<'e>( + fn compare<'e, 'data: 'e>( &self, value: &LhsValue<'e>, - ctx: &'e ExecutionContext<'e, U>, + ctx: &'e F::Context<'data>, ) -> bool { ctx.get_list_matcher_unchecked(&self.list) .match_value(self.name.as_str(), value) diff --git a/engine/src/ast/function_expr.rs b/engine/src/ast/function_expr.rs index d061f370..855049c3 100644 --- a/engine/src/ast/function_expr.rs +++ b/engine/src/ast/function_expr.rs @@ -12,9 +12,9 @@ use crate::functions::{ FunctionParamError, }; use crate::lex::{Lex, LexError, LexErrorKind, LexResult, LexWith, expect, skip_space, span}; -use crate::lhs_types::Array; +use crate::lhs_types::{Array, LhsValue}; use crate::scheme::Function; -use crate::types::{GetType, LhsValue, LiteralValue, Type}; +use crate::types::{GetType, LiteralValue, Type}; use serde::Serialize; use std::hash::{Hash, Hasher}; use std::iter::once; @@ -54,9 +54,18 @@ impl ValueExpr for FunctionCallArgExpr { } } - fn compile_with_compiler(self, compiler: &mut C) -> CompiledValueExpr { + fn compile_with_compiler( + self, + compiler: &mut C, + ) -> CompiledValueExpr { match self { FunctionCallArgExpr::IndexExpr(index_expr) => compiler.compile_index_expr(index_expr), + FunctionCallArgExpr::Literal(LiteralValue::Bytes(bytes)) => { + // Formatting is only needed by the AST. Keep the payload and + // clone it on access so the result can outlive this expression. + let bytes: Box<[u8]> = bytes.into(); + CompiledValueExpr::new(move |_| LhsValue::from(bytes.clone()).into()) + } FunctionCallArgExpr::Literal(literal) => { CompiledValueExpr::new(move |_| LhsValue::from(literal.clone()).into()) } @@ -70,7 +79,8 @@ impl ValueExpr for FunctionCallArgExpr { CompiledExpr::One(expr) => { CompiledValueExpr::new(move |ctx| LhsValue::from(expr.execute(ctx)).into()) } - CompiledExpr::Vec(expr) => CompiledValueExpr::new(move |ctx| { + CompiledExpr::Vec(expr) => CompiledValueExpr::new_with_context(move |ctx| { + let ctx = ctx.context(); let result = expr.execute(ctx); LhsValue::Array(result.into()).into() }), @@ -91,8 +101,8 @@ impl FunctionCallArgExpr { /// Returns `true` if re-evaluating this argument for every mapped element /// could be costly, i.e. it is a nested function call or a logical - /// sub-expression. Literals and plain field accesses are cheap to evaluate - /// repeatedly, so memoizing them would only add overhead. + /// sub-expression. Literals and stored field accesses are cheap to evaluate + /// repeatedly. Field conversion is classified separately at binding. fn is_expensive_to_reevaluate(&self) -> bool { match self { FunctionCallArgExpr::Literal(_) => false, @@ -267,7 +277,10 @@ impl ValueExpr for FunctionCallExpr { visitor.visit_function(&self.function) } - fn compile_with_compiler(self, compiler: &mut C) -> CompiledValueExpr { + fn compile_with_compiler( + self, + compiler: &mut C, + ) -> CompiledValueExpr { let return_type = self.return_type(); let Self { @@ -282,7 +295,7 @@ impl ValueExpr for FunctionCallExpr { .compile(&mut args.iter().map(|arg| arg.into()), context); // For `map_each`, only bother evaluating the non-mapped arguments once // (instead of once per element) when at least one of them is expensive - // to re-evaluate. For trivial arguments (literals / plain field + // to re-evaluate. For trivial arguments (literals / stored field // accesses) inline re-evaluation is just as cheap and avoids a // per-call allocation. let memoize_extra_args = map_each_count > 0 @@ -290,6 +303,15 @@ impl ValueExpr for FunctionCallExpr { .iter() .skip(1) .any(FunctionCallArgExpr::is_expensive_to_reevaluate); + // Field values can require whole-container conversion, including + // when an index subsequently selects one scalar. Classify the provider + // during compilation so ordinary stored field accesses stay cheap. + let memoize_converted_extra_args = map_each_count > 0 + && !memoize_extra_args + && args.iter().skip(1).any(|arg| { + matches!(arg, FunctionCallArgExpr::IndexExpr(index) + if index.materializes_collection(compiler)) + }); let mut args = args .into_iter() .map(|arg| compiler.compile_function_call_arg_expr(arg)); @@ -340,7 +362,8 @@ impl ValueExpr for FunctionCallExpr { } if args.is_empty() { - CompiledValueExpr::new(move |ctx| { + CompiledValueExpr::new_with_context(move |ctx| { + let ctx = ctx.context(); compute( first.execute(ctx), &call, @@ -350,7 +373,8 @@ impl ValueExpr for FunctionCallExpr { ) }) } else if memoize_extra_args { - CompiledValueExpr::new(move |ctx| { + CompiledValueExpr::new_with_context(move |ctx| { + let ctx = ctx.context(); // At least one non-mapped argument is expensive to // re-evaluate, so evaluate all of them once per call and // reuse them (cheaply cloned) for every element instead of @@ -367,8 +391,31 @@ impl ValueExpr for FunctionCallExpr { |elem| ExactSizeChain::new(once(Ok(elem)), extra_args.iter().cloned()), ) }) + } else if memoize_converted_extra_args { + CompiledValueExpr::new_with_context(move |ctx| { + let ctx = ctx.context(); + // Preserve lazy argument consumption: empty mapped inputs + // and functions that ignore an argument must not read it. + // Cloning cached LhsValues retains the function interface's + // ownership contract, but conversion runs only once. + let extra_args = std::cell::OnceCell::new(); + compute(first.execute(ctx), &call, return_type, |elem| { + let values = extra_args.get_or_init(|| { + (0..args.len()) + .map(|_| std::cell::OnceCell::new()) + .collect::>() + }); + ExactSizeChain::new( + once(Ok(elem)), + args.iter() + .zip(values.iter()) + .map(|(arg, value)| value.get_or_init(|| arg.execute(ctx)).clone()), + ) + }) + }) } else { - CompiledValueExpr::new(move |ctx| { + CompiledValueExpr::new_with_context(move |ctx| { + let ctx = ctx.context(); compute( first.execute(ctx), &call, @@ -385,7 +432,8 @@ impl ValueExpr for FunctionCallExpr { } } else { let args = args.collect::>(); - CompiledValueExpr::new(move |ctx| { + CompiledValueExpr::new_with_context(move |ctx| { + let ctx = ctx.context(); match call(&mut args.iter().map(|arg| arg.execute(ctx))) { Some(value) => { debug_assert!(value.get_type() == return_type); @@ -1510,4 +1558,868 @@ mod tests { "http.headers[*]" ); } + #[test] + fn test_indexed_native_function_arguments_materialize_only_the_selection() { + use crate::values::{ArrayRead, ArrayValue, IntoLhsValue, MapRead, MapValue, Value}; + use crate::{EvaluationContext, HasScheme, ListMatcher, ListRef}; + use std::cell::{Cell, RefCell}; + use std::collections::BTreeMap; + + struct HeaderValues { + values: Vec>, + gets: Cell, + iterations: Cell, + } + impl Value for HeaderValues { + type Read = ArrayRead; + type ValueType = crate::values::Array; + } + impl ArrayValue for HeaderValues { + type Element = crate::values::Bytes; + type Item<'a> = &'a Box<[u8]>; + type Iter<'a> = std::slice::Iter<'a, Box<[u8]>>; + + fn get(&self, index: usize) -> Option> { + self.gets.set(self.gets.get() + 1); + self.values.get(index) + } + + fn iter(&self) -> Self::Iter<'_> { + self.iterations.set(self.iterations.get() + 1); + self.values.iter() + } + } + struct Headers { + entries: BTreeMap, HeaderValues>, + keys: RefCell>>, + value_iterations: Cell, + entry_iterations: Cell, + } + impl Value for Headers { + type Read = MapRead; + type ValueType = crate::values::Map>; + } + impl MapValue for Headers { + type Element = crate::values::Array; + type Item<'a> = &'a HeaderValues; + type Iter<'a> = std::collections::btree_map::Values<'a, Box<[u8]>, HeaderValues>; + + fn get(&self, key: &[u8]) -> Option> { + self.keys.borrow_mut().push(key.into()); + self.entries.get(key) + } + + fn values(&self) -> Self::Iter<'_> { + self.value_iterations.set(self.value_iterations.get() + 1); + self.entries.values() + } + + fn for_each_entry<'a>(&'a self, mut visitor: impl FnMut(&[u8], Self::Item<'a>)) { + self.entry_iterations.set(self.entry_iterations.get() + 1); + for (key, value) in &self.entries { + visitor(key, value); + } + } + } + struct Request { + scheme: Scheme, + headers: Headers, + } + impl HasScheme for Request { + fn scheme(&self) -> &Scheme { + &self.scheme + } + } + impl EvaluationContext for Request { + type UserData = (); + + fn get_user_data(&self) -> &() { + &() + } + + fn get_list_matcher(&self, _: ListRef<'_>) -> &dyn ListMatcher { + unreachable!() + } + } + crate::wirefilter_fields! { + impl HeaderFields for Request { + "http.request.headers" => { + fn(request: &Request) -> &Headers { &request.headers } + }, + } + } + fn lower_borrowed<'a>(args: FunctionArgs<'_, 'a>) -> Option> { + let LhsValue::Bytes(mut value) = args.next()?.ok()? else { + unreachable!() + }; + assert!(matches!(value, crate::Bytes::Borrowed(_))); + if value.iter().any(u8::is_ascii_uppercase) { + value.to_mut().make_ascii_lowercase(); + } + Some(value.into()) + } + let array_type = Type::array(Type::Bytes); + let map_type = Type::map(array_type); + let mut builder = crate::SchemeBuilder::new(); + builder.add_field("http.request.headers", map_type).unwrap(); + for (name, val_type, implementation) in [ + ( + "lower", + Type::Bytes, + SimpleFunctionImpl::new(lower_borrowed), + ), + ( + "echo_array", + array_type, + SimpleFunctionImpl::new(echo_function), + ), + ("echo_map", map_type, SimpleFunctionImpl::new(echo_function)), + ] { + builder + .add_function( + name, + SimpleFunctionDefinition { + params: Box::new([SimpleFunctionParam { + arg_kind: SimpleFunctionArgKind::Field, + val_type, + }]), + opt_params: Box::new([]), + return_type: val_type, + implementation, + }, + ) + .unwrap(); + } + let scheme = builder.build(); + HeaderFields::validate_scheme(&scheme).unwrap(); + let entries: BTreeMap, Vec>> = (0..128) + .map(|index| { + ( + format!("x-bench-{index}").into_bytes().into_boxed_slice(), + vec![format!("value-{index}").into_bytes().into_boxed_slice()], + ) + }) + .collect(); + let request = || Request { + scheme: scheme.clone(), + headers: Headers { + entries: entries + .iter() + .map(|(key, values)| { + ( + key.clone(), + HeaderValues { + values: values.clone(), + gets: Cell::new(0), + iterations: Cell::new(0), + }, + ) + }) + .collect(), + keys: RefCell::new(Vec::new()), + value_iterations: Cell::new(0), + entry_iterations: Cell::new(0), + }, + }; + let mut stored = crate::ExecutionContext::new(&scheme); + stored + .set_field_value_from_name("http.request.headers", (&entries).into_lhs()) + .unwrap(); + // Expected accesses per evaluation: key, array lookup, array iteration, + // and whether the function needs the complete map. + for (expression, expected, key, array_gets, array_iterations, complete_map) in [ + ( + r#"any(lower(http.request.headers["x-bench-0"][*])[*] eq "value-0")"#, + true, + Some(b"x-bench-0".as_slice()), + 0, + 1, + false, + ), + ( + r#"lower(http.request.headers["x-bench-0"][0]) eq "value-0""#, + true, + Some(b"x-bench-0".as_slice()), + 1, + 0, + false, + ), + ( + r#"lower(http.request.headers["missing"][0]) eq """#, + false, + Some(b"missing".as_slice()), + 0, + 0, + false, + ), + ( + r#"any(lower(http.request.headers["missing"][*])[*] eq "")"#, + false, + Some(b"missing".as_slice()), + 0, + 0, + false, + ), + ( + r#"lower(http.request.headers["x-bench-0"][99]) eq """#, + false, + Some(b"x-bench-0".as_slice()), + 1, + 0, + false, + ), + ( + r#"any(echo_array(http.request.headers["x-bench-0"])[*] eq "value-0")"#, + true, + Some(b"x-bench-0".as_slice()), + 0, + 1, + false, + ), + ( + r#"echo_map(http.request.headers)["x-bench-0"][0] eq "value-0""#, + true, + None, + 0, + 1, + true, + ), + ] { + let native = request(); + let filter = scheme + .parse(expression) + .unwrap() + .compile_with_compiler(&mut crate::DefaultCompiler::<(), HeaderFields>::new()); + let stored_filter = scheme.parse(expression).unwrap().compile(); + for evaluations in 1..=4 { + assert_eq!(filter.execute(&native).unwrap(), expected, "{expression}"); + assert_eq!( + stored_filter.execute(&stored).unwrap(), + expected, + "{expression}" + ); + assert_eq!( + native.headers.keys.borrow().as_slice(), + key.map_or_else(Vec::new, |key| vec![Box::<[u8]>::from(key); evaluations]), + "{expression}", + ); + assert_eq!( + native.headers.value_iterations.get(), + evaluations * usize::from(complete_map), + "{expression}" + ); + assert_eq!( + native.headers.entry_iterations.get(), + evaluations * usize::from(complete_map), + "{expression}" + ); + for (name, values) in &native.headers.entries { + let selected = name.as_ref() == b"x-bench-0"; + assert_eq!( + values.gets.get(), + evaluations * if selected { array_gets } else { 0 }, + "{expression}: {name:?}" + ); + assert_eq!( + values.iterations.get(), + evaluations + * if selected { + array_iterations + } else { + usize::from(complete_map) + }, + "{expression}: {name:?}" + ); + } + } + } + + let native = request(); + let filter = scheme + .parse_value(r#"lower(http.request.headers["x-bench-0"][0])"#) + .unwrap() + .compile_with_compiler(&mut crate::DefaultCompiler::<(), HeaderFields>::new()); + let value = filter.execute(&native).unwrap().unwrap(); + let LhsValue::Bytes(crate::Bytes::Borrowed(value)) = value else { + panic!("selected header payload should remain borrowed"); + }; + assert!(std::ptr::eq( + value, + native.headers.entries[b"x-bench-0".as_slice()].values[0].as_ref() + )); + + for (selection, array_index) in + [("[*][0]", Some(0)), ("[*][*]", None), ("[*][99]", Some(99))] + { + for empty_map in [false, true] { + let mut native = request(); + if empty_map { + native.headers.entries.clear(); + } else { + native + .headers + .entries + .get_mut(b"x-bench-1".as_slice()) + .unwrap() + .values + .clear(); + native + .headers + .entries + .get_mut(b"x-bench-2".as_slice()) + .unwrap() + .values + .push(b"second-value".as_slice().into()); + } + let entries: BTreeMap<_, _> = native + .headers + .entries + .iter() + .map(|(key, values)| (key.clone(), values.values.clone())) + .collect(); + let mut stored = crate::ExecutionContext::new(&scheme); + stored + .set_field_value_from_name("http.request.headers", (&entries).into_lhs()) + .unwrap(); + let expression = format!("lower(http.request.headers{selection})"); + let filter = scheme + .parse_value(&expression) + .unwrap() + .compile_with_compiler(&mut crate::DefaultCompiler::<(), HeaderFields>::new()); + let stored_filter = scheme.parse_value(&expression).unwrap().compile(); + let expected: Vec<_> = native + .headers + .entries + .values() + .flat_map(|values| match array_index { + Some(index) => values.values.get(index).into_iter().collect::>(), + None => values.values.iter().collect(), + }) + .collect(); + + for evaluations in 1..=4 { + let value = filter.execute(&native).unwrap().unwrap(); + assert_eq!( + value, + stored_filter.execute(&stored).unwrap().unwrap(), + "{expression}" + ); + let LhsValue::Array(values) = value else { + panic!("flattened selection should return an array"); + }; + assert_eq!(values.len(), expected.len(), "{expression}"); + for (actual, expected) in values.into_iter().zip(&expected) { + let LhsValue::Bytes(crate::Bytes::Borrowed(actual)) = actual else { + panic!("selected header payload should remain borrowed"); + }; + assert!(std::ptr::eq(actual, expected.as_ref()), "{expression}"); + } + assert!(native.headers.keys.borrow().is_empty(), "{expression}"); + assert_eq!( + native.headers.value_iterations.get(), + evaluations, + "{expression}" + ); + assert_eq!(native.headers.entry_iterations.get(), 0, "{expression}"); + for values in native.headers.entries.values() { + assert_eq!( + values.gets.get(), + evaluations * usize::from(array_index.is_some()), + "{expression}" + ); + assert_eq!( + values.iterations.get(), + evaluations * usize::from(array_index.is_none()), + "{expression}" + ); + } + } + } + } + + struct PreserveMap { + argument_hooks: usize, + index_hooks: usize, + family: std::marker::PhantomData, + } + impl PreserveMap { + fn new() -> Self { + Self { + argument_hooks: 0, + index_hooks: 0, + family: std::marker::PhantomData, + } + } + } + impl> crate::Compiler for PreserveMap { + type Family = F; + type U = (); + + fn compile_function_call_arg_expr( + &mut self, + node: FunctionCallArgExpr, + ) -> CompiledValueExpr { + self.argument_hooks += 1; + let value = self.compile_value_expr(node); + CompiledValueExpr::new_with_context(move |ctx| { + let result = value.execute(ctx.context()); + assert!(matches!(&result, Ok(LhsValue::Map(_)))); + result + }) + } + + fn compile_index_expr( + &mut self, + node: IndexExpr, + ) -> CompiledValueExpr { + let terminal_map = node.indexes.as_ref() == [FieldIndex::MapEach]; + let value = self.compile_value_expr(node); + if terminal_map { + self.index_hooks += 1; + CompiledValueExpr::new_with_context(move |ctx| { + let result = value.execute(ctx.context()); + assert!(matches!(&result, Ok(LhsValue::Map(_)))); + result + }) + } else { + value + } + } + } + + // A single terminal `[*]` preserves the map visible to compiler hooks, + // even though the mapped function consumes only its values. + for (expression, argument_hooks) in [ + ("echo_array(http.request.headers[*])", 1), + ("echo_array(echo_map(http.request.headers)[*])", 2), + ] { + let mut native_compiler = PreserveMap::::new(); + let native_filter = scheme + .parse_value(expression) + .unwrap() + .compile_with_compiler(&mut native_compiler); + let mut stored_compiler = PreserveMap::>::new(); + let stored_filter = scheme + .parse_value(expression) + .unwrap() + .compile_with_compiler(&mut stored_compiler); + assert_eq!(native_compiler.argument_hooks, argument_hooks); + assert_eq!(native_compiler.index_hooks, 1); + assert_eq!(stored_compiler.argument_hooks, argument_hooks); + assert_eq!(stored_compiler.index_hooks, 1); + let native = request(); + assert_eq!( + native_filter.execute(&native).unwrap(), + stored_filter.execute(&stored).unwrap() + ); + assert_eq!(native.headers.entry_iterations.get(), 1); + } + } + + #[test] + fn test_mapped_native_extra_arguments_convert_once_and_remain_lazy() { + use crate::values::{IntoLhsValue, MapRead, MapValue, Value}; + use crate::{EvaluationContext, HasScheme, ListMatcher, ListRef}; + use std::cell::Cell; + use std::collections::BTreeMap; + use std::sync::OnceLock; + + struct CountingMap { + entries: BTreeMap, i64>, + conversions: Cell, + } + impl Value for CountingMap { + type Read = MapRead; + type ValueType = crate::values::Map; + } + impl MapValue for CountingMap { + type Element = crate::values::Int; + type Item<'a> = &'a i64; + type Iter<'a> = std::collections::btree_map::Values<'a, Box<[u8]>, i64>; + + fn get(&self, key: &[u8]) -> Option> { + self.entries.get(key) + } + + fn values(&self) -> Self::Iter<'_> { + self.entries.values() + } + + fn for_each_entry<'a>(&'a self, mut visitor: impl FnMut(&[u8], Self::Item<'a>)) { + self.conversions.set(self.conversions.get() + 1); + for (key, value) in &self.entries { + visitor(key, value); + } + } + } + struct Request { + scheme: Scheme, + numbers: Option>, + number: i64, + boolean: bool, + bytes: Box, + ip: std::net::IpAddr, + scalar_getters: Cell, + map: OnceLock, + provide_map: bool, + map_getters: Cell, + } + impl HasScheme for Request { + fn scheme(&self) -> &Scheme { + &self.scheme + } + } + impl EvaluationContext for Request { + type UserData = (); + + fn get_user_data(&self) -> &() { + &() + } + + fn get_list_matcher(&self, _: ListRef<'_>) -> &dyn ListMatcher { + unreachable!() + } + } + crate::wirefilter_fields! { + impl NativeFields for Request { + "numbers" => { + fn(request: &Request) -> Option<&Vec> { + request.numbers.as_ref() + } + }, + "number" => { + fn(request: &Request) -> i64 { + request.scalar_getters.set(request.scalar_getters.get() + 1); + request.number + } + }, + "boolean" => { + fn(request: &Request) -> bool { + request.scalar_getters.set(request.scalar_getters.get() + 1); + request.boolean + } + }, + "bytes" => { + fn(request: &Request) -> &str { + request.scalar_getters.set(request.scalar_getters.get() + 1); + &request.bytes + } + }, + "ip" => { + fn(request: &Request) -> std::net::IpAddr { + request.scalar_getters.set(request.scalar_getters.get() + 1); + request.ip + } + }, + "map" => { + fn(request: &Request) -> &CountingMap { + request.map_getters.set(request.map_getters.get() + 1); + assert!(request.provide_map, "mandatory map accessed while unavailable"); + request.map.get_or_init(|| CountingMap { + entries: BTreeMap::from([(Box::from(b"a".as_slice()), 1)]), + conversions: Cell::new(0), + }) + } + }, + } + } + fn consume_both<'a>(args: FunctionArgs<'_, 'a>) -> Option> { + args.next()?.ok()?; + args.next()?.ok()?; + Some(true.into()) + } + fn consume_first<'a>(args: FunctionArgs<'_, 'a>) -> Option> { + args.next()?.ok()?; + Some(true.into()) + } + fn identity<'a>(args: FunctionArgs<'_, 'a>) -> Option> { + args.next()?.ok() + } + let map_type = Type::map(Type::Int); + let mut builder = crate::SchemeBuilder::new(); + builder + .add_optional_field("numbers", Type::array(Type::Int)) + .unwrap(); + builder.add_field("map", map_type).unwrap(); + for (name, ty) in [ + ("number", Type::Int), + ("boolean", Type::Bool), + ("bytes", Type::Bytes), + ("ip", Type::Ip), + ] { + builder.add_field(name, ty).unwrap(); + } + for (name, extra, implementation) in [ + ("map_arg", map_type, SimpleFunctionImpl::new(consume_both)), + ("int_arg", Type::Int, SimpleFunctionImpl::new(consume_both)), + ( + "bool_arg", + Type::Bool, + SimpleFunctionImpl::new(consume_both), + ), + ( + "bytes_arg", + Type::Bytes, + SimpleFunctionImpl::new(consume_both), + ), + ("ip_arg", Type::Ip, SimpleFunctionImpl::new(consume_both)), + ( + "ignore_map", + map_type, + SimpleFunctionImpl::new(consume_first), + ), + ] { + builder + .add_function( + name, + SimpleFunctionDefinition { + params: Box::new([ + SimpleFunctionParam { + arg_kind: SimpleFunctionArgKind::Field, + val_type: Type::Int, + }, + SimpleFunctionParam { + arg_kind: SimpleFunctionArgKind::Field, + val_type: extra, + }, + ]), + opt_params: Box::new([]), + return_type: Type::Bool, + implementation, + }, + ) + .unwrap(); + } + builder + .add_function( + "echo_map", + SimpleFunctionDefinition { + params: Box::new([SimpleFunctionParam { + arg_kind: SimpleFunctionArgKind::Field, + val_type: map_type, + }]), + opt_params: Box::new([]), + return_type: map_type, + implementation: SimpleFunctionImpl::new(identity), + }, + ) + .unwrap(); + let scheme = builder.build(); + NativeFields::validate_scheme(&scheme).unwrap(); + let request = |numbers, provide_map| Request { + scheme: scheme.clone(), + numbers, + number: 1, + boolean: true, + bytes: Box::from("value"), + ip: std::net::Ipv4Addr::LOCALHOST.into(), + scalar_getters: Cell::new(0), + map: OnceLock::new(), + provide_map, + map_getters: Cell::new(0), + }; + let compile = |expression| { + scheme + .parse(expression) + .unwrap() + .compile_with_compiler(&mut crate::DefaultCompiler::<(), NativeFields>::new()) + }; + + struct StoredReads; + impl crate::Compiler for StoredReads { + type Family = crate::ExecutionContextFamily; + type U = Cell; + + fn compile_index_expr(&mut self, node: IndexExpr) -> CompiledValueExpr { + let track = matches!(&node.identifier, IdentifierExpr::Field(field) if field.name() == "map"); + let value = self.compile_value_expr(node); + if track { + CompiledValueExpr::::new_with_context(move |ctx| { + let ctx = ctx.context(); + let reads = ctx.get_user_data(); + reads.set(reads.get() + 1); + value.execute(ctx) + }) + } else { + value + } + } + } + let map = BTreeMap::from([(Box::<[u8]>::from(b"a".as_slice()), 1i64)]); + for (expression, conversions_per_call) in [ + ("any(map_arg(numbers[*], map))", 1), + (r#"any(int_arg(numbers[*], map["a"]))"#, 0), + ] { + let native = request(Some(vec![1, 2, 3]), true); + let filter = compile(expression); + assert!(native.map.get().is_none()); + assert_eq!(native.map_getters.get(), 0); + for calls in [1, 2] { + assert!(filter.execute(&native).unwrap()); + assert_eq!(native.map_getters.get(), calls); + assert_eq!( + native.map.get().unwrap().conversions.get(), + calls * conversions_per_call + ); + } + let stored_filter = scheme + .parse(expression) + .unwrap() + .compile_with_compiler(&mut StoredReads); + let mut stored = crate::ExecutionContext::new_with(&scheme, || Cell::new(0usize)); + stored + .set_field_value_from_name( + "numbers", + Array::try_from_iter(Type::Int, [1i64, 2, 3]).unwrap(), + ) + .unwrap(); + stored + .set_field_value_from_name("map", (&map).into_lhs()) + .unwrap(); + assert!(stored_filter.execute(&stored).unwrap()); + assert_eq!( + stored.get_user_data().get(), + 3, + "stored fields keep their existing cheap access path" + ); + + for numbers in [Some(Vec::new()), None] { + for provide_map in [false, true] { + let native = request(numbers.clone(), provide_map); + assert!(!filter.execute(&native).unwrap()); + assert_eq!(native.map_getters.get(), 0); + assert!(native.map.get().is_none()); + let mut stored = + crate::ExecutionContext::new_with(&scheme, || Cell::new(0usize)); + if let Some(numbers) = &numbers { + stored + .set_field_value_from_name( + "numbers", + Array::try_from_iter(Type::Int, numbers.iter().copied()).unwrap(), + ) + .unwrap(); + } + assert!(!stored_filter.execute(&stored).unwrap()); + assert_eq!(stored.get_user_data().get(), 0); + } + } + } + struct CountingBindings { + fields: BTreeMap, usize>, + family: std::marker::PhantomData F>, + } + impl CountingBindings { + fn new() -> Self { + Self { + fields: BTreeMap::new(), + family: std::marker::PhantomData, + } + } + } + impl crate::Compiler for CountingBindings { + type Family = F; + type U = F::UserData; + + fn bind_field>( + &mut self, + field: crate::Field, + consumer: B, + ) -> Result { + *self.fields.entry(field.name().into()).or_default() += 1; + F::bind(field, consumer) + } + } + // Scalar roots only bind for value compilation. Indexed collection roots + // still need classification, even though the argument itself is scalar. + for (expression, extra_field, bindings, scalar_getters) in [ + ("any(int_arg(numbers[*], number))", "number", 1, 3), + ("any(bool_arg(numbers[*], boolean))", "boolean", 1, 3), + ("any(bytes_arg(numbers[*], bytes))", "bytes", 1, 3), + ("any(ip_arg(numbers[*], ip))", "ip", 1, 3), + (r#"any(int_arg(numbers[*], map["a"]))"#, "map", 2, 0), + // The mapped argument contributes the third binding of numbers. + ("any(int_arg(numbers[*], numbers[0]))", "numbers", 3, 0), + ] { + let native = request(Some(vec![1, 2, 3]), true); + let mut native_compiler = CountingBindings::::new(); + let native_filter = scheme + .parse(expression) + .unwrap() + .compile_with_compiler(&mut native_compiler); + assert_eq!( + native_compiler.fields[extra_field], bindings, + "{expression}" + ); + assert_eq!(native.scalar_getters.get(), 0); + let native_bindings = native_compiler.fields.clone(); + + let mut stored_compiler = CountingBindings::>::new(); + let stored_filter = scheme + .parse(expression) + .unwrap() + .compile_with_compiler(&mut stored_compiler); + assert_eq!( + stored_compiler.fields[extra_field], bindings, + "{expression}" + ); + let stored_bindings = stored_compiler.fields.clone(); + let mut stored = crate::ExecutionContext::new(&scheme); + stored + .set_field_value_from_name( + "numbers", + Array::try_from_iter(Type::Int, [1i64, 2, 3]).unwrap(), + ) + .unwrap(); + for (name, value) in [ + ("number", native.number.into()), + ("boolean", native.boolean.into()), + ("bytes", native.bytes.as_bytes().into()), + ("ip", native.ip.into()), + ("map", (&map).into_lhs()), + ] { + stored.set_field_value_from_name(name, value).unwrap(); + } + for calls in [1, 2] { + assert!(native_filter.execute(&native).unwrap(), "{expression}"); + assert!(stored_filter.execute(&stored).unwrap(), "{expression}"); + assert_eq!(native.scalar_getters.get(), calls * scalar_getters); + assert_eq!(native_compiler.fields, native_bindings); + assert_eq!(stored_compiler.fields, stored_bindings); + } + } + // FunctionArgs may leave an extra argument unconsumed even when a mapped + // call runs. Its mandatory field must remain unread in that case. + let ignored = request(Some(vec![1, 2, 3]), false); + assert!( + compile("any(ignore_map(numbers[*], map))") + .execute(&ignored) + .unwrap() + ); + assert_eq!(ignored.map_getters.get(), 0); + assert!(ignored.map.get().is_none()); + + // Preserve the established eager handling for an existing expensive + // function argument, including when the mapped input is empty. + let eager_expression = "any(map_arg(numbers[*], echo_map(map)))"; + let empty = request(Some(Vec::new()), true); + assert!(!compile(eager_expression).execute(&empty).unwrap()); + assert_eq!(empty.map_getters.get(), 1); + assert_eq!(empty.map.get().unwrap().conversions.get(), 1); + let mut stored = crate::ExecutionContext::new_with(&scheme, || Cell::new(0usize)); + stored + .set_field_value_from_name("numbers", Array::new(Type::Int)) + .unwrap(); + stored + .set_field_value_from_name("map", (&map).into_lhs()) + .unwrap(); + assert!( + !scheme + .parse(eager_expression) + .unwrap() + .compile_with_compiler(&mut StoredReads) + .execute(&stored) + .unwrap() + ); + assert_eq!(stored.get_user_data().get(), 1); + } } diff --git a/engine/src/ast/index_expr.rs b/engine/src/ast/index_expr.rs index 7c35e56a..237d4905 100644 --- a/engine/src/ast/index_expr.rs +++ b/engine/src/ast/index_expr.rs @@ -2,17 +2,23 @@ use super::ValueExpr; use super::field_expr::IdentifierExpr; use super::parse::FilterParser; use super::visitor::{Visitor, VisitorMut}; +#[cfg(test)] +use crate::ExecutionContext; +use crate::Field; use crate::compiler::Compiler; -use crate::execution_context::ExecutionContext; +use crate::context::{ + ContextFamily, ExecutionContextFamily, FieldBindingError, FieldConsumer, FieldGetter, + ProviderFamily, check_field_type, +}; use crate::filter::{CompiledExpr, CompiledOneExpr, CompiledValueExpr, CompiledVecExpr}; use crate::lex::{Lex, LexErrorKind, LexResult, LexWith, expect, skip_space, span}; -use crate::lhs_types::{Array, Map, TypedArray}; +use crate::lhs_types::{Array, LhsValue, TypedArray}; use crate::scheme::{FieldIndex, IndexAccessError}; -use crate::types::{GetType, IntoIter, LhsValue, Type}; +use crate::types::{GetType, Type}; +use crate::values::{self, FieldValueType, IntoLhsValue, LhsValueRead, Value, ValuePredicate}; use serde::ser::SerializeSeq; use serde::{Serialize, Serializer}; - -const BOOL_ARRAY: TypedArray<'_, bool> = TypedArray::new(); +use std::net::IpAddr; /// IndexExpr is an expr that destructures an index into an IdentifierExpr. /// @@ -28,15 +34,6 @@ pub struct IndexExpr { pub indexes: Box<[FieldIndex]>, } -#[allow(clippy::manual_ok_err)] -#[inline] -pub fn ok_ref(result: &Result) -> Option<&T> { - match result { - Ok(x) => Some(x), - Err(_) => None, - } -} - impl ValueExpr for IndexExpr { #[inline] fn walk<'a, V: Visitor<'a>>(&'a self, visitor: &mut V) { @@ -56,71 +53,70 @@ impl ValueExpr for IndexExpr { } } - fn compile_with_compiler(self, compiler: &mut C) -> CompiledValueExpr { + fn compile_with_compiler( + self, + compiler: &mut C, + ) -> CompiledValueExpr { let mut ty = self.get_type(); - let map_each_count = self.map_each_count(); + let count = self.map_each_count(); + let flatten = match count { + 0 => false, + 1 if self.indexes.last() == Some(&FieldIndex::MapEach) => { + // Preserve the parent container: a terminal `map[*]` evaluates + // to a Map here, and mapped-function execution consumes its values. + // Native maps could instead produce an array of values directly, + // avoiding map materialization and key copies. That would change + // the value visible to Compiler hooks wrapping this expression + // (e.g. compile_index_expr / compile_function_call_arg_expr), so + // it requires a separate compatibility decision. + ty = Type::Array(ty.into()); + false + } + _ => true, + }; let Self { identifier, indexes, } = self; - - let last = match map_each_count { - 0 => Some(indexes.len()), - 1 if indexes.last() == Some(&FieldIndex::MapEach) => { - ty = Type::Array(ty.into()); - Some(indexes.len() - 1) - } - _ => None, + // Resolve the terminal `[*]` selection during compilation so the + // closure owns only the indices it follows, without a separate end offset. + let indexes = if flatten { + indexes + } else { + simplify_indexes(indexes) }; - if last == Some(0) { - // Fast path - match identifier { - IdentifierExpr::Field(f) => CompiledValueExpr::new(move |ctx| { - ctx.get_field_value_unchecked(&f) - .map(LhsValue::as_ref) - .ok_or(ty) - }), - IdentifierExpr::FunctionCallExpr(call) => compiler.compile_function_call_expr(call), - } - } else if let Some(last) = last { - // Average path - match identifier { - IdentifierExpr::Field(f) => CompiledValueExpr::new(move |ctx| { - ctx.get_field_value_unchecked(&f) - .and_then(|value| value.get_nested(&indexes[..last])) - .map(LhsValue::as_ref) - .ok_or(ty) - }), - IdentifierExpr::FunctionCallExpr(call) => { - let call = compiler.compile_function_call_expr(call); - CompiledValueExpr::new(move |ctx| { + match identifier { + IdentifierExpr::Field(field) => compiler + .bind_field( + field, + ValueConsumer { + indexes, + ty, + flatten, + }, + ) + .expect("field binding or path preparation failed"), + IdentifierExpr::FunctionCallExpr(call) => { + let call = compiler.compile_function_call_expr(call); + match (flatten, indexes.is_empty()) { + (false, true) => call, + (false, false) => CompiledValueExpr::new_with_context(move |ctx| { + let ctx = ctx.context(); call.execute(ctx) .ok() - .and_then(|val| val.extract_nested(&indexes[..last])) + .and_then(|value| value.extract_nested(&indexes)) .ok_or(ty) - }) - } - } - } else { - let return_type = Type::Array(ty.into()); - // Slow path - match identifier { - IdentifierExpr::Field(f) => CompiledValueExpr::new(move |ctx| { - let mut iter = MapEachIterator::from_indexes(&indexes[..]); - iter.reset( - ctx.get_field_value_unchecked(&f) - .map(LhsValue::as_ref) - .ok_or(return_type)?, - ); - Ok(LhsValue::Array(Array::try_from_iter(ty, iter).unwrap())) - }), - IdentifierExpr::FunctionCallExpr(call) => { - let call = compiler.compile_function_call_expr(call); - CompiledValueExpr::new(move |ctx| { - let mut iter = MapEachIterator::from_indexes(&indexes[..]); - iter.reset(call.execute(ctx).map_err(|_| return_type)?); - Ok(LhsValue::Array(Array::try_from_iter(ty, iter).unwrap())) - }) + }), + (true, _) => { + let return_type = Type::Array(ty.into()); + CompiledValueExpr::new_with_context(move |ctx| { + let ctx = ctx.context(); + let value = call.execute(ctx).map_err(|_| return_type)?; + let mut selected = Vec::new(); + value.append_lhs_at(&indexes, &mut selected); + Ok(LhsValue::Array(Array::try_from_vec(ty, selected).unwrap())) + }) + } } } } @@ -137,19 +133,511 @@ fn simplify_indexes(indexes: Box<[FieldIndex]>) -> Box<[FieldIndex]> { } } +fn with_terminal_each(indexes: Box<[FieldIndex]>) -> Vec { + let mut indexes = indexes.into_vec(); + if indexes.last() != Some(&FieldIndex::MapEach) { + // Boolean-array comparisons can request iteration without an explicit + // [*] in the parsed expression. Other field paths already contain it. + indexes.push(FieldIndex::MapEach); + } + indexes +} + /// Interface used to implement comparison against a value. -pub trait Compare: Send + Sync + 'static { +pub trait Compare = ExecutionContextFamily>: + Send + Sync + 'static +{ /// Returns true if the value matches the comparison. - fn compare<'e>(&self, value: &LhsValue<'e>, ctx: &'e ExecutionContext<'e, U>) -> bool; + fn compare<'ctx, 'data: 'ctx>( + &self, + value: &LhsValue<'ctx>, + ctx: &'ctx F::Context<'data>, + ) -> bool; + + /// Compare native bytes. Override to operate directly on the native value. + #[inline] + fn compare_bytes<'ctx, 'data: 'ctx>( + &self, + value: &'ctx [u8], + ctx: &'ctx F::Context<'data>, + ) -> bool { + self.compare(&LhsValue::Bytes(value.into()), ctx) + } + /// Compare a native integer. + #[inline] + fn compare_int<'ctx, 'data: 'ctx>(&self, value: i64, ctx: &'ctx F::Context<'data>) -> bool { + self.compare(&LhsValue::Int(value), ctx) + } + /// Compare a native Boolean. + #[inline] + fn compare_bool<'ctx, 'data: 'ctx>(&self, value: bool, ctx: &'ctx F::Context<'data>) -> bool { + self.compare(&LhsValue::Bool(value), ctx) + } + /// Compare a native IP address. + #[inline] + fn compare_ip<'ctx, 'data: 'ctx>(&self, value: IpAddr, ctx: &'ctx F::Context<'data>) -> bool { + self.compare(&LhsValue::Ip(value), ctx) + } +} + +#[inline] +fn required(value: Option, field: &Field) -> Option { + if value.is_none() && !field.optional() { + panic!( + "Field {} was registered as mandatory but not given a value", + field.name() + ); + } + value +} + +struct MaterializationCost; + +impl FieldConsumer for MaterializationCost { + type Output = bool; + + fn consume>(self, field: Field, _: G) -> Result { + // Stored values also use this interface, but borrowing their containers + // must not introduce the memoization allocation for native conversion. + Ok(<::Value<'static, 'static> as IntoLhsValue< + 'static, + >>::MATERIALIZES_COLLECTIONS + && matches!(field.get_type(), Type::Array(_) | Type::Map(_))) + } +} + +struct ValueConsumer { + indexes: Box<[FieldIndex]>, + ty: Type, + flatten: bool, +} + +impl FieldConsumer for ValueConsumer { + type Output = CompiledValueExpr; + + fn consume>( + self, + field: Field, + getter: G, + ) -> Result { + check_field_type::<::ValueType>(&field)?; + let Self { + indexes, + ty, + flatten, + } = self; + Ok(match (flatten, indexes.is_empty()) { + (false, true) => CompiledValueExpr::new_with_context(move |ctx| { + let ctx = ctx.context(); + read_field_root(getter.get(ctx, &field), &field, ty) + }), + (false, false) => CompiledValueExpr::new_with_context(move |ctx| { + let ctx = ctx.context(); + read_field_selection(getter.get(ctx, &field), &field, &indexes, ty) + }), + (true, _) => { + let return_type = Type::Array(ty.into()); + CompiledValueExpr::new_with_context(move |ctx| { + let ctx = ctx.context(); + read_field_each(getter.get(ctx, &field), &field, &indexes, ty, return_type) + }) + } + }) + } +} + +// Single-result and vector comparisons use separate consumers so binding +// returns the concrete compiled expression without a runtime mode dispatch. +struct OneComparisonConsumer

{ + indexes: Box<[FieldIndex]>, + default: bool, + comp: P, +} + +// Select the collection fallback during compilation; the mode is not retained +// by the executable expression. Items keeps one trailing iteration implicit. +enum CollectionComparisonMode { + Items, + Path, +} + +struct ArrayComparisonConsumer

{ + indexes: Vec, + comp: P, + mode: CollectionComparisonMode, +} + +struct ContextComparison<'p, 'ctx, 'data, F: ContextFamily, P> { + comp: &'p P, + ctx: &'ctx F::Context<'data>, +} + +impl> ValuePredicate + for ContextComparison<'_, '_, '_, F, P> +{ + #[inline] + fn lhs(&self, value: &LhsValue<'_>) -> bool { + self.comp.compare(value, self.ctx) + } + + #[inline] + fn bytes(&self, value: &[u8]) -> bool { + self.comp.compare_bytes(value, self.ctx) + } + + #[inline] + fn int(&self, value: i64) -> bool { + self.comp.compare_int(value, self.ctx) + } + + #[inline] + fn boolean(&self, value: bool) -> bool { + self.comp.compare_bool(value, self.ctx) + } + + #[inline] + fn ip(&self, value: IpAddr) -> bool { + self.comp.compare_ip(value, self.ctx) + } +} + +// Share type validation and path-type calculation across getter types. +fn comparison_input_type(field: &Field, depth: usize) -> Type { + let mut input_type = field.get_type(); + for _ in 0..depth { + input_type = input_type + .next() + .expect("parsed field path must have valid types"); + } + input_type +} + +// Prepare paths independently of getter types. +enum PreparedFieldPath { + Reader(S::Path), + Collection(Box<[FieldIndex]>), +} + +fn prepare_field_path( + field: &Field, + indexes: Vec, +) -> Result, FieldBindingError> { + check_field_type::(field)?; + let input_type = comparison_input_type(field, indexes.len()); + if needs_collection_conversion::() && matches!(input_type, Type::Array(_) | Type::Map(_)) { + return Ok(PreparedFieldPath::Collection(indexes.into_boxed_slice())); + } + let path = S::compile_field_path(field.get_type(), indexes, input_type) + .map_err(|error| FieldBindingError::Unsupported(error.0))?; + Ok(PreparedFieldPath::Reader(path)) +} + +// Preserve the constant guards that prevent scalar providers from instantiating +// collection-conversion helpers. Readers supporting direct collection comparison +// also bypass those helpers. +const fn needs_collection_conversion() -> bool { + !S::DIRECT_COLLECTION_COMPARISON + && matches!(S::STATIC_TYPE, None | Some(Type::Array(_) | Type::Map(_))) +} + +// Keep the execute_* helpers separate from getter-specific closures. +// Execution depends on the context family, produced value type and comparator, +// not the getter type, so fields with the same types can share execution code +// while their closures retain concrete getter calls. Moving these bodies into +// the closures substantially increased release-build compiler memory and code +// size. Leave inlining decisions to the compiler. +fn execute_one( + ctx: &F::Context<'_>, + field: &Field, + value: Option, + path: &::Path, + comp: &P, + default: bool, +) -> bool +where + F: ContextFamily, + V: Value, + P: Compare, +{ + let Some(value) = required(value, field) else { + return default; + }; + let predicate = ContextComparison:: { comp, ctx }; + values::compare_one(&value, path, &predicate).unwrap_or(default) +} + +fn execute_vec( + ctx: &F::Context<'_>, + field: &Field, + value: Option, + path: &::Path, + comp: &P, +) -> TypedArray<'static, bool> +where + F: ContextFamily, + V: Value, + P: Compare, +{ + let mut output = TypedArray::new(); + if let Some(value) = required(value, field) { + let predicate = ContextComparison:: { comp, ctx }; + values::compare_vec(&value, path, &predicate, &mut output); + } + output +} + +fn execute_collection_one<'ctx, 'data: 'ctx, F, V, P>( + ctx: &'ctx F::Context<'data>, + field: &Field, + value: Option, + indexes: &[FieldIndex], + comp: &P, + default: bool, +) -> bool +where + F: ContextFamily, + V: IntoLhsValue<'ctx>, + P: Compare, +{ + let Some(value) = required(value, field) else { + return default; + }; + // Select on native storage before materializing the comparator's collection. + let Some(value) = value.into_lhs_at(indexes) else { + return default; + }; + comp.compare(&value, ctx) +} + +fn execute_collection_items<'ctx, 'data: 'ctx, F, V, P>( + ctx: &'ctx F::Context<'data>, + field: &Field, + value: Option, + prefix: &[FieldIndex], + comp: &P, +) -> TypedArray<'static, bool> +where + F: ContextFamily, + V: IntoLhsValue<'ctx>, + P: Compare, +{ + let mut output = TypedArray::new(); + if let Some(value) = required(value, field) { + let Some(value) = value.into_lhs_at(prefix) else { + return output; + }; + let predicate = ContextComparison:: { comp, ctx }; + LhsValueRead::compare_items(&value, &[], &predicate, &mut output); + } + output +} + +fn execute_collection_path<'ctx, 'data: 'ctx, F, V, P>( + ctx: &'ctx F::Context<'data>, + field: &Field, + value: Option, + prefix: &[FieldIndex], + rest: &[FieldIndex], + comp: &P, +) -> TypedArray<'static, bool> +where + F: ContextFamily, + V: IntoLhsValue<'ctx>, + P: Compare, +{ + let mut output = TypedArray::new(); + if let Some(value) = required(value, field) { + let Some(value) = value.into_lhs_at(prefix) else { + return output; + }; + let predicate = ContextComparison:: { comp, ctx }; + LhsValueRead::compare_vec(&value, rest, &predicate, &mut output); + } + output +} + +fn read_field_root<'ctx, V: IntoLhsValue<'ctx>>( + value: Option, + field: &Field, + ty: Type, +) -> Result, Type> { + required(value, field).map(IntoLhsValue::into_lhs).ok_or(ty) +} + +fn read_field_selection<'ctx, V: IntoLhsValue<'ctx>>( + value: Option, + field: &Field, + indexes: &[FieldIndex], + ty: Type, +) -> Result, Type> { + required(value, field) + .and_then(|value| value.into_lhs_at(indexes)) + .ok_or(ty) +} + +fn read_field_each<'ctx, V: IntoLhsValue<'ctx>>( + value: Option, + field: &Field, + indexes: &[FieldIndex], + ty: Type, + return_type: Type, +) -> Result, Type> { + let value = required(value, field).ok_or(return_type)?; + // Missing children contribute no elements; an absent root remains an error. + // Only selected values are materialized, retaining terminal map[*] behavior. + let mut values = Vec::new(); + value.append_lhs_at(indexes, &mut values); + Ok(LhsValue::Array(Array::try_from_vec(ty, values).unwrap())) +} + +// Field array comparisons share this getter-specific execution closure. +fn compile_field_vec( + field: Field, + getter: G, + path: <::ValueType as FieldValueType>::Path, + comp: P, +) -> CompiledVecExpr +where + F: ContextFamily, + G: FieldGetter, + P: Compare, +{ + CompiledVecExpr::new(move |ctx| { + execute_vec::(ctx, &field, getter.get(ctx, &field), &path, &comp) + }) +} + +impl> FieldConsumer for OneComparisonConsumer

{ + type Output = CompiledOneExpr; + + fn consume>( + self, + field: Field, + getter: G, + ) -> Result { + let Self { + indexes, + default, + comp, + } = self; + let prepared = prepare_field_path::<::ValueType>( + &field, + indexes.into_vec(), + )?; + match prepared { + PreparedFieldPath::Reader(path) => Ok(CompiledOneExpr::new(move |ctx| { + execute_one::(ctx, &field, getter.get(ctx, &field), &path, &comp, default) + })), + PreparedFieldPath::Collection(indexes) => { + // Retain the constant root-type guard so scalar providers do not + // instantiate collection conversion closures behind this boundary. + if needs_collection_conversion::<::ValueType>() { + Ok(CompiledOneExpr::new(move |ctx| { + execute_collection_one::( + ctx, + &field, + getter.get(ctx, &field), + &indexes, + &comp, + default, + ) + })) + } else { + unreachable!("provider does not require collection conversion") + } + } + } + } +} + +impl> FieldConsumer for ArrayComparisonConsumer

{ + type Output = CompiledVecExpr; + + fn consume>( + self, + field: Field, + getter: G, + ) -> Result { + let Self { + indexes, + comp, + mode, + } = self; + let prepared = + prepare_field_path::<::ValueType>(&field, indexes)?; + match prepared { + PreparedFieldPath::Reader(path) => { + Ok(compile_field_vec::(field, getter, path, comp)) + } + PreparedFieldPath::Collection(indexes) => { + // Retain the constant guard to avoid instantiating conversion + // helpers for readers that never need collection conversion. + if needs_collection_conversion::<::ValueType>() { + match mode { + CollectionComparisonMode::Items => { + let prefix = simplify_indexes(indexes); + Ok(CompiledVecExpr::new(move |ctx| { + execute_collection_items::( + ctx, + &field, + getter.get(ctx, &field), + &prefix, + &comp, + ) + })) + } + CollectionComparisonMode::Path => { + // The path is fixed for this expression. Retain one + // split offset rather than searching during execution. + let prefix_len = indexes + .iter() + .position(|index| *index == FieldIndex::MapEach) + .unwrap_or(indexes.len()); + Ok(CompiledVecExpr::new(move |ctx| { + let (prefix, rest) = indexes.split_at(prefix_len); + execute_collection_path::( + ctx, + &field, + getter.get(ctx, &field), + prefix, + rest, + &comp, + ) + })) + } + } + } else { + unreachable!("provider does not require collection conversion") + } + } + } + } } impl IndexExpr { + /// Determine conversion cost while the field's provider type is known. + pub(crate) fn materializes_collection(&self, compiler: &mut C) -> bool { + match &self.identifier { + // Custom by-value descriptors may still convert before indexing. + // Conservatively memoize collection-backed arguments even when + // borrowed native storage can select a scalar without conversion. + IdentifierExpr::Field(field) + if matches!(field.get_type(), Type::Array(_) | Type::Map(_)) => + { + compiler + .bind_field(field.clone(), MaterializationCost) + .expect("field binding or path preparation failed") + } + IdentifierExpr::Field(_) | IdentifierExpr::FunctionCallExpr(_) => false, + } + } + fn compile_one_with( self, compiler: &mut C, default: bool, - comp: impl Compare, - ) -> CompiledOneExpr { + comp: impl Compare, + ) -> CompiledOneExpr { let Self { identifier, indexes, @@ -168,115 +656,105 @@ impl IndexExpr { }) } else { CompiledOneExpr::new(move |ctx| { - ok_ref(&call.execute(ctx)) - .and_then(|val| val.get_nested(&indexes)) - .map_or( - default, - #[inline] - |val| comp.compare(val, ctx), - ) - }) - } - } - IdentifierExpr::Field(f) => { - if indexes.is_empty() { - CompiledOneExpr::new(move |ctx| { - ctx.get_field_value_unchecked(&f) - .map(|value| comp.compare(value, ctx)) - .unwrap_or(default) - }) - } else { - CompiledOneExpr::new(move |ctx| { - ctx.get_field_value_unchecked(&f) - .and_then(|value| value.get_nested(&indexes)) - .map_or( - default, - #[inline] - |val| comp.compare(val, ctx), - ) + let Ok(value) = call.execute(ctx) else { + return default; + }; + LhsValueRead::compare_one( + &value, + &indexes, + &ContextComparison:: { comp: &comp, ctx }, + ) + .unwrap_or(default) }) } } + IdentifierExpr::Field(field) => compiler + .bind_field( + field, + OneComparisonConsumer { + indexes, + default, + comp, + }, + ) + .expect("field binding or path preparation failed"), } } pub(crate) fn compile_vec_with( self, compiler: &mut C, - comp: impl Compare, - ) -> CompiledVecExpr { + comp: impl Compare, + ) -> CompiledVecExpr { let Self { identifier, indexes, } = self; - let indexes = simplify_indexes(indexes); match identifier { IdentifierExpr::FunctionCallExpr(call) => { + // The reader can iterate children without retaining a terminal + // [*] entry in function-result closures. + let indexes = simplify_indexes(indexes); let call = compiler.compile_function_call_expr(call); CompiledVecExpr::new(move |ctx| { - let comp = ∁ - ok_ref(&call.execute(ctx)) - .and_then(|val| val.get_nested(&indexes)) - .map_or( - BOOL_ARRAY, - #[inline] - |val: &LhsValue<'_>| { - TypedArray::from_iter( - val.iter().unwrap().map(|item| comp.compare(item, ctx)), - ) - }, - ) + let mut output = TypedArray::new(); + if let Ok(value) = call.execute(ctx) { + LhsValueRead::compare_items( + &value, + &indexes, + &ContextComparison:: { comp: &comp, ctx }, + &mut output, + ); + } + output }) } - IdentifierExpr::Field(f) => CompiledVecExpr::new(move |ctx| { - let comp = ∁ - ctx.get_field_value_unchecked(&f) - .and_then(|value| value.get_nested(&indexes)) - .map_or( - BOOL_ARRAY, - #[inline] - |val: &LhsValue<'_>| { - TypedArray::from_iter( - val.iter().unwrap().map(|item| comp.compare(item, ctx)), - ) - }, - ) - }), + IdentifierExpr::Field(field) => compiler + .bind_field( + field, + ArrayComparisonConsumer { + indexes: with_terminal_each(indexes), + comp, + mode: CollectionComparisonMode::Items, + }, + ) + .expect("field binding or path preparation failed"), } } pub(crate) fn compile_iter_with( self, compiler: &mut C, - comp: impl Compare, - ) -> CompiledVecExpr { + comp: impl Compare, + ) -> CompiledVecExpr { let Self { identifier, indexes, } = self; match identifier { - IdentifierExpr::Field(f) => CompiledVecExpr::new(move |ctx| { - let mut iter = MapEachIterator::from_indexes(&indexes[..]); - match ctx.get_field_value_unchecked(&f) { - Some(value) => iter.reset(value.as_ref()), - None => return TypedArray::default(), - }; - TypedArray::from_iter(iter.map(|item| comp.compare(&item, ctx))) - }), + IdentifierExpr::Field(field) => compiler + .bind_field( + field, + ArrayComparisonConsumer { + indexes: indexes.into_vec(), + comp, + mode: CollectionComparisonMode::Path, + }, + ) + .expect("field binding or path preparation failed"), IdentifierExpr::FunctionCallExpr(call) => { let call = compiler.compile_function_call_expr(call); CompiledVecExpr::new(move |ctx| { - let mut iter = MapEachIterator::from_indexes(&indexes[..]); - match call.execute(ctx) { - Ok(val) => { - iter.reset(val); - } - _ => { - return TypedArray::default(); - } + let mut output = TypedArray::new(); + if let Ok(value) = call.execute(ctx) { + LhsValueRead::compare_vec( + &value, + &indexes, + &ContextComparison:: { comp: &comp, ctx }, + &mut output, + ); } - - TypedArray::from_iter(iter.map(|item| comp.compare(&item, ctx))) + output }) } } @@ -288,8 +766,8 @@ impl IndexExpr { self, compiler: &mut C, default: bool, - comp: impl Compare, - ) -> CompiledExpr { + comp: impl Compare, + ) -> CompiledExpr { match self.map_each_count() { 0 => CompiledExpr::One(self.compile_one_with(compiler, default, comp)), 1 if self.indexes.last() == Some(&FieldIndex::MapEach) => { @@ -447,104 +925,12 @@ impl Serialize for IndexExpr { } } -enum FieldIndexIterator<'a, 'b> { - ArrayIndex(Option<(Array<'a>, u32)>), - MapKey(Option<(Map<'a>, &'b [u8])>), - MapEach(IntoIter<'a>), -} - -impl<'a, 'b> FieldIndexIterator<'a, 'b> { - fn new(val: LhsValue<'a>, idx: &'b FieldIndex) -> Result { - match idx { - FieldIndex::ArrayIndex(idx) => match val { - LhsValue::Array(arr) => Ok(Self::ArrayIndex(Some((arr, *idx)))), - _ => Err(IndexAccessError { - index: FieldIndex::ArrayIndex(*idx), - actual: val.get_type(), - }), - }, - FieldIndex::MapKey(key) => match val { - LhsValue::Map(map) => Ok(Self::MapKey(Some((map, key.as_bytes())))), - _ => Err(IndexAccessError { - index: FieldIndex::MapKey(key.clone()), - actual: val.get_type(), - }), - }, - FieldIndex::MapEach => match val { - LhsValue::Array(_) | LhsValue::Map(_) => Ok(Self::MapEach(val.into_iter())), - _ => Err(IndexAccessError { - index: FieldIndex::MapEach, - actual: val.get_type(), - }), - }, - } - } -} - -impl<'a> Iterator for FieldIndexIterator<'a, '_> { - type Item = LhsValue<'a>; - - fn next(&mut self) -> Option { - match self { - Self::ArrayIndex(opt) => opt.take().and_then(|(arr, idx)| arr.extract(idx as usize)), - Self::MapKey(opt) => opt.take().and_then(|(map, key)| map.extract(key)), - Self::MapEach(iter) => iter.next(), - } - } -} - -struct MapEachIterator<'a, 'b> { - indexes: &'b [FieldIndex], - stack: Vec>, -} - -impl<'a, 'b> MapEachIterator<'a, 'b> { - fn from_indexes(indexes: &'b [FieldIndex]) -> Self { - Self { - indexes, - stack: Vec::with_capacity(indexes.len()), - } - } - - fn reset(&mut self, val: LhsValue<'a>) { - self.stack.clear(); - let first = self.indexes.first().unwrap(); - self.stack - .push(FieldIndexIterator::new(val, first).unwrap()); - } -} - -impl<'a> Iterator for MapEachIterator<'a, '_> { - type Item = LhsValue<'a>; - - fn next(&mut self) -> Option> { - while !self.stack.is_empty() { - assert!(self.stack.len() <= self.indexes.len()); - if let Some(nxt) = self.stack.last_mut().unwrap().next() { - // Check that current iterator is a leaf iterator - if self.stack.len() == self.indexes.len() { - // Return a value if a leaf iterator returned a value - return Some(nxt); - } else { - self.stack.push( - FieldIndexIterator::new(nxt, &self.indexes[self.stack.len()]).unwrap(), - ); - } - } else { - // Last iterator is finished, remove it - self.stack.pop(); - } - } - None - } -} - #[cfg(test)] mod tests { use super::*; use crate::ast::field_expr::IdentifierExpr; use crate::{ - Array, FieldIndex, FilterParser, FunctionArgs, FunctionCallArgExpr, FunctionCallExpr, + Array, FieldIndex, FilterParser, FunctionArgs, FunctionCallArgExpr, FunctionCallExpr, Map, Scheme, SchemeBuilder, SimpleFunctionArgKind, SimpleFunctionDefinition, SimpleFunctionImpl, SimpleFunctionParam, }; @@ -673,6 +1059,99 @@ mod tests { ); } + #[test] + fn test_function_return_selection() { + let mut ctx = ExecutionContext::new(&SCHEME); + ctx.set_field_value_from_name("test", Array::from_iter(["value"])) + .unwrap(); + let array = LhsValue::Array(Array::from_iter(["value"])); + for (expression, expected) in [ + ("array(test[0])", Ok(array.clone())), + ("array(test[0])[0]", Ok(LhsValue::from("value"))), + ("array(test[0])[*]", Ok(array.clone())), + ("array2(test[0])[0]", Ok(array.clone())), + ("array2(test[0])[0][*]", Ok(array.clone())), + ("array2(test[0])[*][0]", Ok(array.clone())), + ("array2(test[0])[*][*]", Ok(array)), + ("array2(test[0])[9][*]", Err(Type::array(Type::Bytes))), + ("array2(test[0])[*][9]", Ok(Array::new(Type::Bytes).into())), + ] { + let (index, rest) = FilterParser::new(&SCHEME) + .lex_as::(expression) + .unwrap(); + assert!(rest.is_empty()); + assert_eq!(index.compile().execute(&ctx), expected, "{expression}"); + } + } + + #[test] + fn test_terminal_map_each_preserves_function_result_and_error_types() { + fn echo<'a>(args: FunctionArgs<'_, 'a>) -> Option> { + args.next()?.ok() + } + + let map_type = Type::map(Type::Bytes); + let mut builder = SchemeBuilder::new(); + for (field, function, ty) in [ + ("map", "echo_map", map_type), + ("maps", "echo_maps", Type::array(map_type)), + ] { + builder.add_optional_field(field, ty).unwrap(); + builder + .add_function( + function, + SimpleFunctionDefinition { + params: Box::new([SimpleFunctionParam { + arg_kind: SimpleFunctionArgKind::Field, + val_type: ty, + }]), + opt_params: Box::new([]), + return_type: ty, + implementation: SimpleFunctionImpl::new(echo), + }, + ) + .unwrap(); + } + let scheme = builder.build(); + let map = Map::try_from_iter::( + Type::Bytes, + [Ok((Box::from(b"key".as_slice()), "value"))], + ) + .unwrap(); + let missing = ExecutionContext::new(&scheme); + let mut present = ExecutionContext::new(&scheme); + present + .set_field_value_from_name("map", map.clone()) + .unwrap(); + present + .set_field_value_from_name( + "maps", + Array::try_from_iter(map_type, [map.clone()]).unwrap(), + ) + .unwrap(); + for (expression, missing_type) in [ + ("map[*]", Type::array(Type::Bytes)), + ("echo_map(map)[*]", map_type), + ("echo_maps(maps)[0][*]", Type::array(Type::Bytes)), + ] { + let (index, rest) = FilterParser::new(&scheme) + .lex_as::(expression) + .unwrap(); + assert!(rest.is_empty()); + let compiled = index.compile(); + assert_eq!( + compiled.execute(&present), + Ok(LhsValue::Map(map.clone())), + "{expression}" + ); + assert_eq!( + compiled.execute(&missing), + Err(missing_type), + "{expression}" + ); + } + } + #[test] fn test_function_call_with_missing_argument_then_index_access() { let expr = assert_ok!( @@ -948,10 +1427,9 @@ mod tests { for i in 0..10 { let indexes = [FieldIndex::ArrayIndex(i), FieldIndex::MapEach]; - let mut iter = MapEachIterator::from_indexes(&indexes[..]); - - iter.reset(arr.clone()); - for (j, elem) in iter.enumerate() { + let mut selected = Vec::new(); + arr.clone().append_lhs_at(&indexes, &mut selected); + for (j, elem) in selected.into_iter().enumerate() { let bytes = match elem { LhsValue::Bytes(bytes) => bytes, _ => unreachable!(), @@ -960,10 +1438,9 @@ mod tests { } let indexes = [FieldIndex::MapEach, FieldIndex::ArrayIndex(i)]; - let mut iter = MapEachIterator::from_indexes(&indexes[..]); - - iter.reset(arr.clone()); - for (j, elem) in iter.enumerate() { + let mut selected = Vec::new(); + arr.clone().append_lhs_at(&indexes, &mut selected); + for (j, elem) in selected.into_iter().enumerate() { let bytes = match elem { LhsValue::Bytes(bytes) => bytes, _ => unreachable!(), @@ -973,12 +1450,12 @@ mod tests { } let indexes = [FieldIndex::MapEach, FieldIndex::MapEach]; - let mut iter = MapEachIterator::from_indexes(&indexes[..]); + let mut selected = Vec::new(); + arr.append_lhs_at(&indexes, &mut selected); let mut i = 0; let mut j = 0; - iter.reset(arr.clone()); - for elem in iter { + for elem in selected { let bytes = match elem { LhsValue::Bytes(bytes) => bytes, _ => unreachable!(), @@ -988,4 +1465,319 @@ mod tests { i += (j == 0) as u32; } } + #[test] + fn test_native_custom_comparators_can_consume_collections() { + use crate::{DefaultCompiler, EvaluationContext, HasScheme, ListMatcher, ListRef}; + use std::collections::BTreeMap; + + struct Request { + scheme: Scheme, + rows: Vec>>, + maps: Vec, String>>, + } + impl HasScheme for Request { + fn scheme(&self) -> &Scheme { + &self.scheme + } + } + impl EvaluationContext for Request { + type UserData = (); + + fn get_user_data(&self) -> &() { + &() + } + + fn get_list_matcher(&self, _: ListRef<'_>) -> &dyn ListMatcher { + unreachable!() + } + } + crate::wirefilter_fields! { + impl NativeFields for Request { + "rows" => { + fn(request: &Request) -> &Vec>> { + &request.rows + } + }, + "maps" => { + fn(request: &Request) -> &Vec, String>> { + &request.maps + } + }, + } + } + struct Nonempty; + impl> Compare<(), F> for Nonempty { + fn compare<'ctx, 'data: 'ctx>( + &self, + value: &LhsValue<'ctx>, + _: &'ctx F::Context<'data>, + ) -> bool { + match value { + LhsValue::Array(values) => !values.is_empty(), + LhsValue::Map(values) => !values.is_empty(), + _ => panic!("custom comparator requires a complete collection"), + } + } + } + let scheme = crate::Scheme! { + rows: Array(Array(Array(Bytes))), + maps: Array(Map(Bytes)), + } + .build(); + let native = Request { + scheme: scheme.clone(), + rows: vec![vec![vec![], vec!["value".into()]], vec![]], + maps: vec![ + BTreeMap::new(), + BTreeMap::from([(b"key".as_slice().into(), "value".into())]), + ], + }; + NativeFields::validate_scheme(&scheme).unwrap(); + let mut stored = ExecutionContext::new(&scheme); + stored + .set_field_value(scheme.get_field("rows").unwrap(), (&native.rows).into_lhs()) + .unwrap(); + stored + .set_field_value(scheme.get_field("maps").unwrap(), (&native.maps).into_lhs()) + .unwrap(); + for (name, indexes) in [ + ("rows", vec![]), + ("rows", vec![FieldIndex::ArrayIndex(8)]), + ("rows", vec![FieldIndex::MapEach]), + ("rows", vec![FieldIndex::MapEach, FieldIndex::MapEach]), + ("maps", vec![FieldIndex::MapEach]), + ] { + for missing in [false, true] { + let index = IndexExpr { + identifier: IdentifierExpr::Field(scheme.get_field(name).unwrap().to_owned()), + indexes: indexes.clone().into_boxed_slice(), + }; + let expected = index.clone().compile_with( + &mut DefaultCompiler::<()>::new(), + missing, + Nonempty, + ); + let actual = index.compile_with( + &mut DefaultCompiler::<(), NativeFields>::new(), + missing, + Nonempty, + ); + match (expected, actual) { + (CompiledExpr::One(expected), CompiledExpr::One(actual)) => { + assert_eq!(expected.execute(&stored), actual.execute(&native)); + } + (CompiledExpr::Vec(expected), CompiledExpr::Vec(actual)) => { + assert_eq!(expected.execute(&stored), actual.execute(&native)); + } + _ => panic!("comparison cardinality changed"), + } + } + } + } + + #[test] + fn test_native_indexed_value_expressions_match_stored_values() { + use crate::{DefaultCompiler, EvaluationContext, HasScheme, ListMatcher, ListRef}; + use std::collections::BTreeMap; + + type Rows = BTreeMap, Vec>>>; + type Groups = Vec, Vec>>>; + struct Request { + scheme: Scheme, + rows: Option, + groups: Option, + } + impl HasScheme for Request { + fn scheme(&self) -> &Scheme { + &self.scheme + } + } + impl EvaluationContext for Request { + type UserData = (); + + fn get_user_data(&self) -> &() { + &() + } + + fn get_list_matcher(&self, _: ListRef<'_>) -> &dyn ListMatcher { + unreachable!() + } + } + crate::wirefilter_fields! { + impl NativeFields for Request { + "rows" => { + fn(request: &Request) -> Option<&Rows> { + request.rows.as_ref() + } + }, + "groups" => { + fn(request: &Request) -> Option<&Groups> { + request.groups.as_ref() + } + }, + } + } + let mut builder = SchemeBuilder::new(); + builder + .add_optional_field("rows", Type::map(Type::array(Type::array(Type::Bytes)))) + .unwrap(); + builder + .add_optional_field("groups", Type::array(Type::map(Type::array(Type::Bytes)))) + .unwrap(); + let scheme = builder.build(); + NativeFields::validate_scheme(&scheme).unwrap(); + + for present in [true, false] { + let native = Request { + scheme: scheme.clone(), + rows: present.then(|| { + BTreeMap::from([ + (Box::from(b"z".as_slice()), vec![vec!["z0".into()]]), + ( + Box::from(b"selected".as_slice()), + vec![vec!["s0".into(), "s1".into()], vec![], vec!["s2".into()]], + ), + (Box::from(b"empty".as_slice()), vec![]), + (Box::from(b"a".as_slice()), vec![vec!["a0".into()]]), + ]) + }), + groups: present.then(|| { + vec![ + BTreeMap::from([ + (Box::from(b"z".as_slice()), vec!["z1".into()]), + ( + Box::from(b"values".as_slice()), + vec!["v0".into(), "v1".into()], + ), + (Box::from(b"empty".as_slice()), vec![]), + (Box::from(b"a".as_slice()), vec!["a1".into()]), + ]), + BTreeMap::new(), + BTreeMap::from([(Box::from(b"values".as_slice()), vec!["v2".into()])]), + ] + }), + }; + let mut stored = ExecutionContext::new(&scheme); + if let Some(rows) = &native.rows { + stored + .set_field_value_from_name("rows", rows.into_lhs()) + .unwrap(); + } + if let Some(groups) = &native.groups { + stored + .set_field_value_from_name("groups", groups.into_lhs()) + .unwrap(); + } + + for expression in [ + "rows", + r#"rows["selected"]"#, + r#"rows["selected"][0]"#, + r#"rows["selected"][0][1]"#, + r#"rows["selected"][0][*]"#, + r#"rows["selected"][*]"#, + r#"rows["selected"][*][*]"#, + r#"rows["selected"][*][0]"#, + r#"rows["selected"][*][9]"#, + r#"rows["selected"][9]"#, + r#"rows["selected"][9][*]"#, + r#"rows["selected"][9][0]"#, + r#"rows["empty"][*]"#, + r#"rows["empty"][*][*]"#, + r#"rows["empty"][*][0]"#, + r#"rows["missing"]"#, + r#"rows["missing"][*]"#, + r#"rows["missing"][0][*]"#, + r#"rows["missing"][*][*]"#, + r#"rows["missing"][*][0]"#, + "rows[*]", + "rows[*][*]", + "rows[*][*][*]", + "rows[*][0][*]", + "groups", + "groups[0]", + "groups[0][*]", + "groups[0][*][*]", + "groups[0][*][0]", + r#"groups[0]["values"][*]"#, + r#"groups[0]["values"][1]"#, + r#"groups[0]["missing"][*]"#, + "groups[1][*]", + "groups[1][*][*]", + "groups[9][*]", + "groups[9][*][*]", + "groups[9][*][0]", + r#"groups[9]["values"][*]"#, + r#"groups[*]["values"][*]"#, + r#"groups[*]["values"][0]"#, + r#"groups[*]["missing"][*]"#, + "groups[*][*][*]", + ] { + let (index, rest) = FilterParser::new(&scheme) + .lex_as::(expression) + .unwrap(); + assert!(rest.is_empty(), "{expression}"); + let expected = index + .clone() + .compile_with_compiler(&mut DefaultCompiler::<()>::new()); + let actual = + index.compile_with_compiler(&mut DefaultCompiler::<(), NativeFields>::new()); + let expected = expected.execute(&stored); + let actual = actual.execute(&native); + assert_eq!(actual, expected, "{expression}, present={present}"); + if !present { + assert!(actual.is_err(), "{expression}"); + continue; + } + + // Check flattening order and the difference between a missing + // selection before `[*]` and a missing optional root. + let flat: Option<&[&str]> = match expression { + r#"rows["selected"][*][*]"# => Some(&["s0", "s1", "s2"]), + r#"rows["selected"][*][0]"# => Some(&["s0", "s2"]), + "rows[*][*][*]" => Some(&["a0", "s0", "s1", "s2", "z0"]), + "rows[*][0][*]" => Some(&["a0", "s0", "s1", "z0"]), + "groups[0][*][*]" => Some(&["a1", "v0", "v1", "z1"]), + "groups[0][*][0]" => Some(&["a1", "v0", "z1"]), + r#"groups[*]["values"][*]"# => Some(&["v0", "v1", "v2"]), + r#"groups[*]["values"][0]"# => Some(&["v0", "v2"]), + "groups[*][*][*]" => Some(&["a1", "v0", "v1", "z1", "v2"]), + r#"rows["selected"][*][9]"# + | r#"rows["empty"][*][*]"# + | r#"rows["empty"][*][0]"# + | r#"rows["missing"][*][*]"# + | r#"rows["missing"][*][0]"# + | "groups[1][*][*]" + | "groups[9][*][*]" + | "groups[9][*][0]" + | r#"groups[*]["missing"][*]"# => Some(&[]), + _ => None, + }; + if let Some(values) = flat { + assert_eq!( + actual.as_ref().unwrap(), + &LhsValue::Array(Array::from_iter(values.iter().copied())), + "{expression}", + ); + } + if expression == "groups[0][*]" { + // A single terminal `map[*]` retains its map for the + // mapped function, whose iteration follows byte-key order. + let LhsValue::Map(map) = actual.unwrap() else { + panic!("terminal map[*] must retain the map"); + }; + let values: Vec<_> = map.into_values().collect(); + assert_eq!( + values, + vec![ + LhsValue::Array(Array::from_iter(["a1"])), + LhsValue::Array(Array::new(Type::Bytes)), + LhsValue::Array(Array::from_iter(["v0", "v1"])), + LhsValue::Array(Array::from_iter(["z1"])), + ], + ); + } + } + } + } } diff --git a/engine/src/ast/logical_expr.rs b/engine/src/ast/logical_expr.rs index 1f8a0073..e10928ec 100644 --- a/engine/src/ast/logical_expr.rs +++ b/engine/src/ast/logical_expr.rs @@ -7,7 +7,8 @@ use super::visitor::{Visitor, VisitorMut}; use crate::compiler::Compiler; use crate::filter::{CompiledExpr, CompiledOneExpr, CompiledVecExpr}; use crate::lex::{Lex, LexErrorKind, LexResult, LexWith, expect, skip_space, span}; -use crate::types::{GetType, LhsValue, Type, TypeMismatchError}; +use crate::lhs_types::LhsValue; +use crate::types::{GetType, Type, TypeMismatchError}; use serde::Serialize; lex_enum!( @@ -362,7 +363,7 @@ impl Expr for LogicalExpr { } } - fn compile_with_compiler(self, compiler: &mut C) -> CompiledExpr { + fn compile_with_compiler(self, compiler: &mut C) -> CompiledExpr { match self { LogicalExpr::Comparison(op) => compiler.compile_comparison_expr(op), LogicalExpr::Parenthesized(node) => compiler.compile_logical_expr(node.expr), diff --git a/engine/src/ast/mod.rs b/engine/src/ast/mod.rs index 79d7d054..d29445b3 100644 --- a/engine/src/ast/mod.rs +++ b/engine/src/ast/mod.rs @@ -12,8 +12,9 @@ use self::visitor::{UsesListVisitor, UsesVisitor, Visitor, VisitorMut}; use crate::compiler::{Compiler, DefaultCompiler}; use crate::filter::{CompiledExpr, CompiledValueExpr, Filter, FilterValue}; use crate::lex::{LexErrorKind, LexResult, LexWith}; +use crate::lhs_types::LhsValue; use crate::scheme::{Scheme, UnknownFieldError}; -use crate::types::{GetType, LhsValue, Type, TypeMismatchError}; +use crate::types::{GetType, Type, TypeMismatchError}; use serde::Serialize; use std::fmt::{self, Debug}; @@ -26,7 +27,7 @@ pub trait Expr: /// Recursively visit all nodes in the AST using a [`VisitorMut`]. fn walk_mut<'a, V: VisitorMut<'a>>(&'a mut self, visitor: &mut V); /// Compiles current node into a [`CompiledExpr`] using [`Compiler`]. - fn compile_with_compiler(self, compiler: &mut C) -> CompiledExpr; + fn compile_with_compiler(self, compiler: &mut C) -> CompiledExpr; /// Compiles current node into a [`CompiledExpr`] using [`DefaultCompiler`]. fn compile(self) -> CompiledExpr { let mut compiler = DefaultCompiler::new(); @@ -43,7 +44,10 @@ pub trait ValueExpr: /// Recursively visit all nodes in the AST using a [`VisitorMut`]. fn walk_mut<'a, V: VisitorMut<'a>>(&'a mut self, visitor: &mut V); /// Compiles current node into a [`CompiledValueExpr`] using [`Compiler`]. - fn compile_with_compiler(self, compiler: &mut C) -> CompiledValueExpr; + fn compile_with_compiler( + self, + compiler: &mut C, + ) -> CompiledValueExpr; /// Compiles current node into a [`CompiledValueExpr`] using [`DefaultCompiler`]. fn compile(self) -> CompiledValueExpr { let mut compiler = DefaultCompiler::new(); @@ -150,7 +154,11 @@ impl FilterAst { } /// Compiles a [`FilterAst`] into a [`Filter`] using a specific [`Compiler`]. - pub fn compile_with_compiler(self, compiler: &mut C) -> Filter { + /// + /// Field definitions must be validated explicitly during application + /// initialization with [`crate::ContextFamily::validate_scheme`]. Compilation + /// does not repeat that validation. + pub fn compile_with_compiler(self, compiler: &mut C) -> Filter { match compiler.compile_logical_expr(self.op) { CompiledExpr::One(one) => Filter::new(one, self.scheme), CompiledExpr::Vec(_) => unreachable!(), @@ -221,7 +229,10 @@ impl ValueExpr for FilterValueExpr { } } - fn compile_with_compiler(self, compiler: &mut C) -> CompiledValueExpr { + fn compile_with_compiler( + self, + compiler: &mut C, + ) -> CompiledValueExpr { match self { Self::Index(expr) => compiler.compile_index_expr(expr), Self::Quantifier { op, arg } => { @@ -336,7 +347,14 @@ impl FilterValueAst { } /// Compiles a [`FilterValueAst`] into a [`FilterValue`] using a specific [`Compiler`]. - pub fn compile_with_compiler(self, compiler: &mut C) -> FilterValue { + /// + /// Field definitions must be validated explicitly during application + /// initialization with [`crate::ContextFamily::validate_scheme`]. Compilation + /// does not repeat that validation. + pub fn compile_with_compiler( + self, + compiler: &mut C, + ) -> FilterValue { FilterValue::new(compiler.compile_value_expr(self.op), self.scheme) } diff --git a/engine/src/compiler.rs b/engine/src/compiler.rs index 3b01ffad..f018940f 100644 --- a/engine/src/compiler.rs +++ b/engine/src/compiler.rs @@ -1,6 +1,7 @@ use crate::{ - ComparisonExpr, CompiledExpr, CompiledValueExpr, Expr, FunctionCallArgExpr, FunctionCallExpr, - IndexExpr, LogicalExpr, ValueExpr, + ComparisonExpr, CompiledExpr, CompiledValueExpr, ContextFamily, ExecutionContextFamily, Expr, + Field, FieldBindingError, FieldConsumer, FunctionCallArgExpr, FunctionCallExpr, IndexExpr, + LogicalExpr, ValueExpr, }; /// Trait used to drive the compilation of a [`crate::FilterAst`] into a [`crate::Filter`]. @@ -8,33 +9,57 @@ pub trait Compiler: Sized { /// The user data type passed in the [`crate::ExecutionContext`]. type U: 'static; + /// Context and concrete field bindings used by compiled expressions. + type Family: ContextFamily; + + /// Resolve a field while retaining the concrete getter type during compilation. + /// The binding takes ownership so the compiled expression can retain the field + /// without cloning it. + #[inline] + fn bind_field>( + &mut self, + field: Field, + consumer: B, + ) -> Result { + Self::Family::bind(field, consumer) + } + /// Compiles a [`Expr`] node into a [`CompiledExpr`] (boxed closure). #[inline] - fn compile_expr(&mut self, node: impl Expr) -> CompiledExpr { + fn compile_expr(&mut self, node: impl Expr) -> CompiledExpr { node.compile_with_compiler(self) } /// Compiles a [`LogicalExpr`] node into a [`CompiledExpr`] (boxed closure). #[inline] - fn compile_logical_expr(&mut self, node: LogicalExpr) -> CompiledExpr { + fn compile_logical_expr(&mut self, node: LogicalExpr) -> CompiledExpr { self.compile_expr(node) } /// Compiles a [`ComparisonExpr`] node into a [`CompiledExpr`] (boxed closure). #[inline] - fn compile_comparison_expr(&mut self, node: ComparisonExpr) -> CompiledExpr { + fn compile_comparison_expr( + &mut self, + node: ComparisonExpr, + ) -> CompiledExpr { self.compile_expr(node) } /// Compiles a [`ValueExpr`] node into a [`CompiledValueExpr`] (boxed closure). #[inline] - fn compile_value_expr(&mut self, node: impl ValueExpr) -> CompiledValueExpr { + fn compile_value_expr( + &mut self, + node: impl ValueExpr, + ) -> CompiledValueExpr { node.compile_with_compiler(self) } /// Compiles a [`FunctionCallExpr`] node into a [`CompiledValueExpr`] (boxed closure). #[inline] - fn compile_function_call_expr(&mut self, node: FunctionCallExpr) -> CompiledValueExpr { + fn compile_function_call_expr( + &mut self, + node: FunctionCallExpr, + ) -> CompiledValueExpr { self.compile_value_expr(node) } @@ -43,24 +68,38 @@ pub trait Compiler: Sized { fn compile_function_call_arg_expr( &mut self, node: FunctionCallArgExpr, - ) -> CompiledValueExpr { + ) -> CompiledValueExpr { self.compile_value_expr(node) } /// Compiles a [`IndexExpr`] node into a [`CompiledValueExpr`] (boxed closure). #[inline] - fn compile_index_expr(&mut self, node: IndexExpr) -> CompiledValueExpr { + fn compile_index_expr(&mut self, node: IndexExpr) -> CompiledValueExpr { self.compile_value_expr(node) } } /// Default compiler -#[derive(Clone, Copy, Debug)] -pub struct DefaultCompiler { - _marker: std::marker::PhantomData, +pub struct DefaultCompiler> { + _marker: std::marker::PhantomData (U, F)>, +} + +impl Copy for DefaultCompiler {} + +impl Clone for DefaultCompiler { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +impl std::fmt::Debug for DefaultCompiler { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DefaultCompiler").finish() + } } -impl Default for DefaultCompiler { +impl Default for DefaultCompiler { #[inline] fn default() -> Self { Self { @@ -69,7 +108,7 @@ impl Default for DefaultCompiler { } } -impl DefaultCompiler { +impl DefaultCompiler { /// Creates a new [`DefaultCompiler`]. #[inline] pub fn new() -> Self { @@ -77,6 +116,7 @@ impl DefaultCompiler { } } -impl Compiler for DefaultCompiler { +impl> Compiler for DefaultCompiler { + type Family = F; type U = U; } diff --git a/engine/src/context/fields.rs b/engine/src/context/fields.rs new file mode 100644 index 00000000..d88ee839 --- /dev/null +++ b/engine/src/context/fields.rs @@ -0,0 +1,1255 @@ +//! Field definitions for existing runtime contexts. + +use crate::{FieldDefinitionError, GetType, Scheme, SchemeValidationError, Type}; +use std::collections::BTreeSet; + +/// Validate complete scheme coverage and getter types without running getters. +/// +/// Public for calls generated by [`crate::wirefilter_fields!`] in other crates. +/// Applications normally use the generated family's `validate_scheme` method. +/// Each entry is the field name, its static value type (or `None` for a +/// scheme-typed stored value), and whether its getter returns `Option`. A +/// scheme-typed context must validate values on insertion; this function only +/// checks coverage and optionality for such getters. +#[doc(hidden)] +pub fn validate_field_definitions( + scheme: &Scheme, + definitions: &[(&str, Option, bool)], +) -> Result<(), SchemeValidationError> { + let mut errors = Vec::new(); + let mut names = BTreeSet::new(); + for &(name, getter_type, may_be_absent) in definitions { + if !names.insert(name) { + errors.push(FieldDefinitionError::DuplicateGetter(name.into())); + } + match scheme.get_field(name) { + Ok(field) => { + if let Some(getter_type) = getter_type + && field.get_type() != getter_type + { + errors.push(FieldDefinitionError::TypeMismatch { + name: name.into(), + scheme_type: field.get_type(), + getter_type, + }); + } + if may_be_absent && !field.optional() { + errors.push(FieldDefinitionError::OptionalGetterForMandatoryField( + name.into(), + )); + } + } + Err(_) => errors.push(FieldDefinitionError::UnknownField(name.into())), + } + } + for field in scheme.fields() { + if !names.contains(field.name()) { + errors.push(FieldDefinitionError::MissingGetter(field.name().into())); + } + } + if errors.is_empty() { + Ok(()) + } else { + Err(SchemeValidationError(errors.into_boxed_slice())) + } +} + +/// Define field getters on an existing evaluation context. +/// +/// The context implements [`crate::HasScheme`] and [`crate::EvaluationContext`] +/// separately. Each signature declares the getter's argument and return type. +/// Mandatory fields use `&T` for borrowed storage or `T` for an owned value. +/// Optional fields may additionally use `Option<&T>` or `Option` for absence. +/// Startup validation rejects an `Option` getter for a mandatory field, even if +/// its body always returns `Some`. Plain getter bodies retain their declared +/// return type, so returning `None` from one is a Rust type error. +/// Each comma-separated field entry groups its getter in braces so declarations +/// can be parsed independently without expansion depth growing with field count. +/// Plain-return getters use [`crate::RequiredFieldGetterFn`]; `Option` getters use +/// [`crate::FieldGetterFn`]. Getters sharing a provider and presence use the same +/// getter type. Field names are resolved during filter compilation, never on +/// field access. Evaluation calls the selected getter through a function pointer. +/// Attributes placed directly before `fn` apply to the function containing the +/// getter body, for both plain and optional returns. This function has no +/// attributes unless explicitly supplied. The `FieldGetter::get` adapters remain +/// `#[inline]`. Plain-return getters wrap their result in `Some` after calling the +/// function pointer; `Option` getters forward their result unchanged. Getter +/// attributes do not conditionally include or omit the field definition. +/// A getter may also declare a second parameter, `field: &Field`, to access the +/// original field's name, index, type, and optionality. Both getter forms support +/// the same return types and provider overrides. A borrowed return is tied to +/// the context parameter, independently of the field parameter's lifetime. +/// +/// In the declaration below, `RequestState` is the application's existing +/// context and `RequestFields` is a new zero-sized [`crate::ContextFamily`] +/// marker generated by this macro. The marker contains no request values and +/// is not instantiated per request. For a borrowing context, it lets compiled +/// filters name `Context<'data>` independently of any particular request lifetime. +/// An owned context could implement `ContextFamily` directly, but this macro +/// consistently generates a separate marker. +/// +/// ```ignore +/// wirefilter_fields! { +/// pub impl RequestFields for RequestState { +/// "http.request.headers.names" => { +/// fn(state: &RequestState) -> &Vec { +/// state.names.get_or_init(|| compute_names(state)) +/// } +/// }, +/// "tcp.port" => { +/// fn(state: &RequestState) -> i64 { +/// i64::from(state.port) +/// } +/// }, +/// } +/// } +/// RequestFields::validate_scheme(&scheme)?; +/// ``` +/// +/// A context borrowing request data can use `impl<'data> RequestFields for +/// RequestState<'data>`. Return types borrowing that data through a +/// descriptor can select their provider explicitly by placing +/// `provider SelectionProvider;` before the getter signature inside its group. +/// The getter then returns `Selection<'data>` or `Option>`. +/// When combining a provider override with attributes, put the attributes after +/// `provider SelectionProvider;` and immediately before `fn`. For example: +/// +/// ```ignore +/// "request.computed_value" => { +/// #[inline(never)] +/// fn(state: &RequestState) -> &ComputedValue { +/// state.computed_value() +/// } +/// }, +/// ``` +/// +/// Validation reports missing, unknown, duplicate, mistyped, and incorrectly +/// optional definitions, including unused and optional scheme fields. It does not execute getters or +/// initialize lazy values. Applications must call the generated `validate_scheme` +/// method during initialization for every scheme/family pair, before loading +/// rulesets or compiling expressions. Parsing, compilation, and execution do not +/// invoke it automatically. +#[macro_export] +macro_rules! wirefilter_fields { + ( + $(#[$meta:meta])* $visibility:vis impl<$data:lifetime> $family:ident + for $context:ident<$context_data:lifetime> { + $($name:literal => $getter:tt),* $(,)? + } + ) => { + $crate::wirefilter_fields! { + @emit [$(#[$meta])*] [$visibility] [$family] [$data] + [$context<$context_data>] [$context<'static>] + { $($name => $getter),* } + } + }; + ( + $(#[$meta:meta])* $visibility:vis impl $family:ident for $context:ty { + $($name:literal => $getter:tt),* $(,)? + } + ) => { + $crate::wirefilter_fields! { + @emit [$(#[$meta])*] [$visibility] [$family] ['__wirefilter_data] + [$context] [$context] { $($name => $getter),* } + } + }; + ( + @provider { provider $provider:ty; $($getter:tt)* } + ) => { $provider }; + ( + @optional { provider $provider:ty; $($getter:tt)* } + ) => { $crate::wirefilter_fields!(@optional { $($getter)* }) }; + ( + @optional { + $(#[$attribute:meta])* + fn($argument:ident: &$argument_type:ty $(, $field:ident: &$field_type:ty)?) + -> Option<$return_type:ty> $(provider $provider:ty)? $body:block + } + ) => { true }; + ( + @optional { + $(#[$attribute:meta])* + fn($argument:ident: &$argument_type:ty $(, $field:ident: &$field_type:ty)?) + -> $return_type:ty $body:block + } + ) => { false }; + // Keep each getter as a token-tree group so these independent helpers can + // distinguish borrowed and owned return types without parsing the field list. + ( + @provider { + $(#[$attribute:meta])* + fn($argument:ident: &$argument_type:ty $(, $field:ident: &$field_type:ty)?) + -> Option<$return_type:ty> provider $provider:ty $body:block + } + ) => { $provider }; + ( + @provider { + $(#[$attribute:meta])* + fn($argument:ident: &$argument_type:ty $(, $field:ident: &$field_type:ty)?) + -> Option<&$return_type:ty> $body:block + } + ) => { $crate::BorrowedValueProvider<$return_type> }; + ( + @provider { + $(#[$attribute:meta])* + fn($argument:ident: &$argument_type:ty $(, $field:ident: &$field_type:ty)?) + -> Option<$return_type:ty> $body:block + } + ) => { $crate::OwnedValueProvider<$return_type> }; + ( + @provider { + $(#[$attribute:meta])* + fn($argument:ident: &$argument_type:ty $(, $field:ident: &$field_type:ty)?) + -> &$return_type:ty $body:block + } + ) => { $crate::BorrowedValueProvider<$return_type> }; + ( + @provider { + $(#[$attribute:meta])* + fn($argument:ident: &$argument_type:ty $(, $field:ident: &$field_type:ty)?) + -> $return_type:ty $body:block + } + ) => { $crate::OwnedValueProvider<$return_type> }; + ( + @getter [$family:ident] [$data:lifetime] { + provider $provider:ty; + $(#[$attribute:meta])* + fn($argument:ident: &$argument_type:ty $(, $field:ident: &$field_type:ty)?) + -> Option<&$return_type:ty> $body:block + } + ) => { + $crate::wirefilter_fields! { + @impl_getter [$family] [$data] [$(#[$attribute])*] + [$argument: $argument_type] + [$($field: $field_type)?] + [&'__wirefilter_ctx $return_type] [$provider] $body + } + }; + ( + @getter [$family:ident] [$data:lifetime] { + provider $provider:ty; + $(#[$attribute:meta])* + fn($argument:ident: &$argument_type:ty $(, $field:ident: &$field_type:ty)?) + -> Option<$return_type:ty> $body:block + } + ) => { + $crate::wirefilter_fields! { + @impl_getter [$family] [$data] [$(#[$attribute])*] + [$argument: $argument_type] + [$($field: $field_type)?] + [$return_type] [$provider] $body + } + }; + ( + @getter [$family:ident] [$data:lifetime] { + provider $provider:ty; + $(#[$attribute:meta])* + fn($argument:ident: &$argument_type:ty $(, $field:ident: &$field_type:ty)?) + -> &$return_type:ty $body:block + } + ) => { + $crate::wirefilter_fields! { + @impl_required_getter [$family] [$data] [$(#[$attribute])*] + [$argument: $argument_type] + [$($field: $field_type)?] + [&'__wirefilter_ctx $return_type] [$provider] $body + } + }; + ( + @getter [$family:ident] [$data:lifetime] { + provider $provider:ty; + $(#[$attribute:meta])* + fn($argument:ident: &$argument_type:ty $(, $field:ident: &$field_type:ty)?) + -> $return_type:ty $body:block + } + ) => { + $crate::wirefilter_fields! { + @impl_required_getter [$family] [$data] [$(#[$attribute])*] + [$argument: $argument_type] + [$($field: $field_type)?] + [$return_type] [$provider] $body + } + }; + ( + @getter [$family:ident] [$data:lifetime] { + $(#[$attribute:meta])* + fn($argument:ident: &$argument_type:ty $(, $field:ident: &$field_type:ty)?) + -> Option<&$return_type:ty> provider $provider:ty $body:block + } + ) => { + $crate::wirefilter_fields! { + @impl_getter [$family] [$data] [$(#[$attribute])*] + [$argument: $argument_type] + [$($field: $field_type)?] + [&'__wirefilter_ctx $return_type] [$provider] $body + } + }; + ( + @getter [$family:ident] [$data:lifetime] { + $(#[$attribute:meta])* + fn($argument:ident: &$argument_type:ty $(, $field:ident: &$field_type:ty)?) + -> Option<$return_type:ty> provider $provider:ty $body:block + } + ) => { + $crate::wirefilter_fields! { + @impl_getter [$family] [$data] [$(#[$attribute])*] + [$argument: $argument_type] + [$($field: $field_type)?] + [$return_type] [$provider] $body + } + }; + ( + @getter [$family:ident] [$data:lifetime] { + $(#[$attribute:meta])* + fn($argument:ident: &$argument_type:ty $(, $field:ident: &$field_type:ty)?) + -> Option<&$return_type:ty> $body:block + } + ) => { + $crate::wirefilter_fields! { + @impl_getter [$family] [$data] [$(#[$attribute])*] + [$argument: $argument_type] + [$($field: $field_type)?] + [&'__wirefilter_ctx $return_type] [$crate::BorrowedValueProvider<$return_type>] $body + } + }; + ( + @getter [$family:ident] [$data:lifetime] { + $(#[$attribute:meta])* + fn($argument:ident: &$argument_type:ty $(, $field:ident: &$field_type:ty)?) + -> Option<$return_type:ty> $body:block + } + ) => { + $crate::wirefilter_fields! { + @impl_getter [$family] [$data] [$(#[$attribute])*] + [$argument: $argument_type] + [$($field: $field_type)?] + [$return_type] [$crate::OwnedValueProvider<$return_type>] $body + } + }; + ( + @getter [$family:ident] [$data:lifetime] { + $(#[$attribute:meta])* + fn($argument:ident: &$argument_type:ty $(, $field:ident: &$field_type:ty)?) + -> &$return_type:ty $body:block + } + ) => { + $crate::wirefilter_fields! { + @impl_required_getter [$family] [$data] [$(#[$attribute])*] + [$argument: $argument_type] + [$($field: $field_type)?] + [&'__wirefilter_ctx $return_type] [$crate::BorrowedValueProvider<$return_type>] $body + } + }; + ( + @getter [$family:ident] [$data:lifetime] { + $(#[$attribute:meta])* + fn($argument:ident: &$argument_type:ty $(, $field:ident: &$field_type:ty)?) + -> $return_type:ty $body:block + } + ) => { + $crate::wirefilter_fields! { + @impl_required_getter [$family] [$data] [$(#[$attribute])*] + [$argument: $argument_type] + [$($field: $field_type)?] + [$return_type] [$crate::OwnedValueProvider<$return_type>] $body + } + }; + ( + @impl_required_getter [$family:ident] [$data:lifetime] [$(#[$attribute:meta])*] + [$argument:ident: $argument_type:ty] [$($field:ident: $field_type:ty)?] + [$return_type:ty] [$provider:ty] $body:block + ) => {{ + // The parameter shadows this function's name inside the body. Reusing + // that caller-provided identifier avoids introducing a helper name that + // could shadow application items referenced by the body or return type. + $(#[$attribute])* + fn $argument<'__wirefilter_ctx, $data: '__wirefilter_ctx>( + $argument: &'__wirefilter_ctx $argument_type, + __wirefilter_field: &$crate::Field, + ) -> $return_type { + $(let $field: &$field_type = __wirefilter_field;)? + $body + } + + // Keep the callback's name outside the body function's scope. + { + fn __wirefilter_call<'__wirefilter_ctx, $data>( + context: $crate::ContextRef<'__wirefilter_ctx, $data, $family>, + field: &$crate::Field, + ) -> $crate::RequiredFieldGetterResult<'__wirefilter_ctx, $data, $provider> { + $crate::RequiredFieldGetterResult::new($argument(context.context(), field)) + } + $crate::RequiredFieldGetterFn::<$family, $provider>::new(__wirefilter_call) + } + }}; + ( + @impl_getter [$family:ident] [$data:lifetime] [$(#[$attribute:meta])*] + [$argument:ident: $argument_type:ty] [$($field:ident: $field_type:ty)?] + [$return_type:ty] [$provider:ty] $body:block + ) => {{ + // Preserve the same name isolation as the plain-return getter. + $(#[$attribute])* + fn $argument<'__wirefilter_ctx, $data: '__wirefilter_ctx>( + $argument: &'__wirefilter_ctx $argument_type, + __wirefilter_field: &$crate::Field, + ) -> Option<$return_type> { + $(let $field: &$field_type = __wirefilter_field;)? + $body + } + + { + fn __wirefilter_call<'__wirefilter_ctx, $data>( + context: $crate::ContextRef<'__wirefilter_ctx, $data, $family>, + field: &$crate::Field, + ) -> $crate::OptionalFieldGetterResult<'__wirefilter_ctx, $data, $provider> { + $crate::OptionalFieldGetterResult::new($argument(context.context(), field)) + } + $crate::FieldGetterFn::<$family, $provider>::new(__wirefilter_call) + } + }}; + ( + @emit [$($meta:tt)*] [$visibility:vis] [$family:ident] [$data:lifetime] + [$context:ty] [$static_context:ty] { $($name:literal => $getter:tt),* } + ) => { + $($meta)* + $visibility struct $family; + + impl $family { + /// Check all field definitions against the runtime scheme. + /// + /// Call during application initialization before compiling expressions. + /// This check is never invoked automatically by parsing or execution. + $visibility fn validate_scheme( + scheme: &$crate::Scheme, + ) -> Result<(), $crate::SchemeValidationError> { + ::validate_scheme(scheme) + } + } + + impl $crate::ContextFamily for $family { + type UserData = <$static_context as $crate::EvaluationContext>::UserData; + type Context<$data> = $context; + + fn validate_scheme(scheme: &$crate::Scheme) -> Result<(), $crate::SchemeValidationError> { + $crate::validate_field_definitions(scheme, &[ + $(($name, + <<$crate::wirefilter_fields!(@provider $getter) as $crate::ProviderFamily>::ValueType + as $crate::values::FieldValueType>::STATIC_TYPE, + $crate::wirefilter_fields!(@optional $getter) + ),)* + ]) + } + + fn bind>( + field: $crate::Field, + consumer: B, + ) -> Result { + #[allow(unreachable_patterns)] + match field.name() { + $($name => consumer.consume( + field, + $crate::wirefilter_fields!(@getter [$family] [$data] $getter) + ),)* + name => Err($crate::FieldBindingError::UnknownField(name.into())), + } + } + } + }; +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{EvaluationContext, HasScheme, ListMatcher, ListRef, SchemeBuilder}; + use std::sync::OnceLock; + + struct State { + scheme: Scheme, + names: Box<[Box]>, + country: Option>, + lazy: OnceLock]>>, + } + + impl HasScheme for State { + fn scheme(&self) -> &Scheme { + &self.scheme + } + } + + impl EvaluationContext for State { + type UserData = (); + + fn get_user_data(&self) -> &() { + &() + } + + fn get_list_matcher(&self, _: ListRef<'_>) -> &dyn ListMatcher { + panic!("these fields have no lists") + } + } + + wirefilter_fields! { + impl Fields for State { + "names" => { + fn(state: &State) -> &[Box] { + if state.names.is_empty() { + return &[]; + } + &state.names + } + }, + "country" => { + fn(state: &State) -> Option<&str> { state.country.as_deref() } + }, + "lazy" => { + fn(state: &State) -> &[Box] { + state.lazy.get_or_init(|| state.names.clone()) + } + }, + "count" => { + fn(state: &State) -> i64 { + if state.names.is_empty() { + return 0; + } + state.names.len() as i64 + } + }, + } + } + + wirefilter_fields! { + impl ValidationOnly for State { + "names" => { + fn(state: &State) -> &[Box] { + let _ = state; + panic!("validation must not evaluate getters") + } + }, + "country" => { + fn(state: &State) -> Option<&str> { + let _ = state; + panic!("validation must not evaluate getters") + } + }, + "lazy" => { + fn(state: &State) -> &[Box] { + let _ = state; + panic!("validation must not initialize lazy data") + } + }, + "count" => { + fn(state: &State) -> i64 { + let _ = state; + panic!("validation must not evaluate getters") + } + }, + } + } + + fn schema(fields: &[(&str, Type, bool)]) -> Scheme { + let mut builder = SchemeBuilder::new(); + for &(name, ty, optional) in fields { + if optional { + builder.add_optional_field(name, ty).unwrap(); + } else { + builder.add_field(name, ty).unwrap(); + } + } + builder.build() + } + + fn definitions() -> Vec<(&'static str, Type, bool)> { + vec![ + ("names", Type::array(Type::Bytes), false), + ("country", Type::Bytes, true), + ("lazy", Type::array(Type::Bytes), false), + ("count", Type::Int, false), + ] + } + + #[test] + fn test_complete_validation_does_not_run_getters() { + let scheme = schema(&definitions()); + assert_eq!(Fields::validate_scheme(&scheme), Ok(())); + assert_eq!(ValidationOnly::validate_scheme(&scheme), Ok(())); + } + + #[test] + fn test_existing_state_lazy_getters_and_phase_reuse() { + let scheme = schema(&definitions()); + Fields::validate_scheme(&scheme).unwrap(); + let mut compiler = crate::DefaultCompiler::<(), Fields>::new(); + let ordinary = scheme + .parse(r#"any(names[*] == "accept") and count == 1"#) + .unwrap() + .compile_with_compiler(&mut compiler); + let lazy = scheme + .parse(r#"any(lazy[*] == "accept")"#) + .unwrap() + .compile_with_compiler(&mut compiler); + let mut state = State { + scheme, + names: vec![Box::from("other")].into_boxed_slice(), + country: None, + lazy: OnceLock::new(), + }; + assert!(state.lazy.get().is_none()); + assert!(!ordinary.execute(&state).unwrap()); + state.names[0] = "accept".into(); + assert!(ordinary.execute(&state).unwrap()); + assert!(state.lazy.get().is_none()); + assert!(lazy.execute(&state).unwrap()); + let pointer = state.lazy.get().unwrap().as_ptr(); + assert!(lazy.execute(&state).unwrap()); + assert_eq!(pointer, state.lazy.get().unwrap().as_ptr()); + // The runtime owns cache invalidation when source data changes. + state.names[0] = "other".into(); + state.lazy.take(); + assert!(!lazy.execute(&state).unwrap()); + } + + struct BorrowingState<'data> { + scheme: &'data Scheme, + names: &'data [Box], + } + + impl HasScheme for BorrowingState<'_> { + fn scheme(&self) -> &Scheme { + self.scheme + } + } + + impl EvaluationContext for BorrowingState<'_> { + type UserData = (); + + fn get_user_data(&self) -> &() { + &() + } + + fn get_list_matcher(&self, _: ListRef<'_>) -> &dyn ListMatcher { + panic!("these fields have no lists") + } + } + + wirefilter_fields! { + impl<'data> BorrowingFields for BorrowingState<'data> { + "names" => { + fn(state: &BorrowingState<'data>) -> &[Box] { + state.names + } + }, + } + } + + #[test] + fn test_context_can_borrow_request_storage() { + let scheme = schema(&[("names", Type::array(Type::Bytes), false)]); + BorrowingFields::validate_scheme(&scheme).unwrap(); + let mut compiler = crate::DefaultCompiler::<(), BorrowingFields>::new(); + let filter = scheme + .parse(r#"names[0] == "accept""#) + .unwrap() + .compile_with_compiler(&mut compiler); + let names = [Box::::from("accept")]; + let state = BorrowingState { + scheme: &scheme, + names: &names, + }; + assert!(filter.execute(&state).unwrap()); + } + + struct SelectionProvider; + + impl crate::ProviderFamily for SelectionProvider { + type Storage<'data> = &'data [Box]; + type Value<'ctx, 'data> + = crate::values::SliceView<'data, Box> + where + 'data: 'ctx; + type ValueType = crate::values::Array; + + fn reborrow<'ctx, 'data: 'ctx>( + value: &'ctx Self::Storage<'data>, + ) -> Self::Value<'ctx, 'data> { + crate::values::SliceView(value) + } + } + + wirefilter_fields! { + impl<'data> SelectionFields for BorrowingState<'data> { + "names" => { + provider SelectionProvider; + fn(state: &BorrowingState<'data>, field: &crate::Field) + -> crate::values::SliceView<'data, Box> { + assert_eq!(field.as_ref(), state.scheme.get_field("names").unwrap()); + crate::values::SliceView(state.names) + } + }, + } + } + + #[test] + fn test_explicit_descriptor_provider_converts_general_values_safely() { + let scheme = schema(&[("names", Type::array(Type::Bytes), false)]); + SelectionFields::validate_scheme(&scheme).unwrap(); + let mut compiler = crate::DefaultCompiler::<(), SelectionFields>::new(); + let value = scheme + .parse_value("names") + .unwrap() + .compile_with_compiler(&mut compiler); + let names = [Box::::from("accept")]; + let state = BorrowingState { + scheme: &scheme, + names: &names, + }; + let crate::LhsValue::Array(array) = value.execute(&state).unwrap().unwrap() else { + panic!("expected an array") + }; + let crate::LhsValue::Bytes(bytes) = array.get(0).unwrap() else { + panic!("expected a byte string") + }; + assert_eq!(bytes.as_ref().as_ptr(), names[0].as_ptr()); + } + + #[test] + fn test_large_field_family_with_default_recursion_limit() { + // A complete schema larger than Rust's default macro recursion limit. + // Repetition keeps this fixture compact without changing the crate limit. + macro_rules! define_fields { + ($(($borrowed:literal, $owned:literal, $selection:literal)),* $(,)?) => { + wirefilter_fields! { + impl<'data> LargeFields for BorrowingState<'data> { + $( + $borrowed => { + fn(state: &BorrowingState<'data>) -> &[Box] { + state.names + } + }, + $owned => { + fn(state: &BorrowingState<'data>) -> i64 { + state.names.len() as i64 + } + }, + $selection => { + provider SelectionProvider; + fn(state: &BorrowingState<'data>) + -> crate::values::SliceView<'data, Box> { + crate::values::SliceView(state.names) + } + }, + )* + } + } + fn large_schema() -> Scheme { + schema(&[$( + ($borrowed, Type::array(Type::Bytes), false), + ($owned, Type::Int, false), + ($selection, Type::array(Type::Bytes), false), + )*]) + } + }; + } + + define_fields! { + ("names000", "count000", "selection000"), + ("names001", "count001", "selection001"), + ("names002", "count002", "selection002"), + ("names003", "count003", "selection003"), + ("names004", "count004", "selection004"), + ("names005", "count005", "selection005"), + ("names006", "count006", "selection006"), + ("names007", "count007", "selection007"), + ("names008", "count008", "selection008"), + ("names009", "count009", "selection009"), + ("names010", "count010", "selection010"), + ("names011", "count011", "selection011"), + ("names012", "count012", "selection012"), + ("names013", "count013", "selection013"), + ("names014", "count014", "selection014"), + ("names015", "count015", "selection015"), + ("names016", "count016", "selection016"), + ("names017", "count017", "selection017"), + ("names018", "count018", "selection018"), + ("names019", "count019", "selection019"), + ("names020", "count020", "selection020"), + ("names021", "count021", "selection021"), + ("names022", "count022", "selection022"), + ("names023", "count023", "selection023"), + ("names024", "count024", "selection024"), + ("names025", "count025", "selection025"), + ("names026", "count026", "selection026"), + ("names027", "count027", "selection027"), + ("names028", "count028", "selection028"), + ("names029", "count029", "selection029"), + ("names030", "count030", "selection030"), + ("names031", "count031", "selection031"), + ("names032", "count032", "selection032"), + ("names033", "count033", "selection033"), + ("names034", "count034", "selection034"), + ("names035", "count035", "selection035"), + ("names036", "count036", "selection036"), + ("names037", "count037", "selection037"), + ("names038", "count038", "selection038"), + ("names039", "count039", "selection039"), + ("names040", "count040", "selection040"), + ("names041", "count041", "selection041"), + ("names042", "count042", "selection042"), + ("names043", "count043", "selection043"), + ("names044", "count044", "selection044"), + ("names045", "count045", "selection045"), + ("names046", "count046", "selection046"), + ("names047", "count047", "selection047"), + ("names048", "count048", "selection048"), + ("names049", "count049", "selection049"), + ("names050", "count050", "selection050"), + ("names051", "count051", "selection051"), + ("names052", "count052", "selection052"), + ("names053", "count053", "selection053"), + ("names054", "count054", "selection054"), + ("names055", "count055", "selection055"), + ("names056", "count056", "selection056"), + ("names057", "count057", "selection057"), + ("names058", "count058", "selection058"), + ("names059", "count059", "selection059"), + ("names060", "count060", "selection060"), + ("names061", "count061", "selection061"), + ("names062", "count062", "selection062"), + ("names063", "count063", "selection063"), + ("names064", "count064", "selection064"), + ("names065", "count065", "selection065"), + ("names066", "count066", "selection066"), + ("names067", "count067", "selection067"), + ("names068", "count068", "selection068"), + ("names069", "count069", "selection069"), + ("names070", "count070", "selection070"), + ("names071", "count071", "selection071"), + ("names072", "count072", "selection072"), + ("names073", "count073", "selection073"), + ("names074", "count074", "selection074"), + ("names075", "count075", "selection075"), + ("names076", "count076", "selection076"), + ("names077", "count077", "selection077"), + ("names078", "count078", "selection078"), + ("names079", "count079", "selection079"), + ("names080", "count080", "selection080"), + ("names081", "count081", "selection081"), + ("names082", "count082", "selection082"), + ("names083", "count083", "selection083"), + ("names084", "count084", "selection084"), + ("names085", "count085", "selection085"), + ("names086", "count086", "selection086"), + ("names087", "count087", "selection087"), + ("names088", "count088", "selection088"), + ("names089", "count089", "selection089"), + ("names090", "count090", "selection090"), + ("names091", "count091", "selection091"), + ("names092", "count092", "selection092"), + ("names093", "count093", "selection093"), + ("names094", "count094", "selection094"), + ("names095", "count095", "selection095"), + ("names096", "count096", "selection096"), + ("names097", "count097", "selection097"), + ("names098", "count098", "selection098"), + ("names099", "count099", "selection099"), + } + let scheme = large_schema(); + assert_eq!(scheme.field_count(), 300); + LargeFields::validate_scheme(&scheme).unwrap(); + let names = [Box::::from("accept"), Box::::from("other")]; + let state = BorrowingState { + scheme: &scheme, + names: &names, + }; + let mut compiler = crate::DefaultCompiler::<(), LargeFields>::new(); + for index in [0, 49, 99] { + for expression in [ + format!("names{index:03}[0] == \"accept\""), + format!("count{index:03} == 2"), + format!("any(selection{index:03}[*] == \"other\")"), + ] { + let filter = scheme + .parse(&expression) + .unwrap() + .compile_with_compiler(&mut compiler); + assert!(filter.execute(&state).unwrap(), "{expression}"); + } + let value = scheme + .parse_value(&format!("selection{index:03}")) + .unwrap() + .compile_with_compiler(&mut compiler); + let crate::LhsValue::Array(array) = value.execute(&state).unwrap().unwrap() else { + panic!("expected an array"); + }; + let crate::LhsValue::Bytes(bytes) = array.get(0).unwrap() else { + panic!("expected bytes"); + }; + assert_eq!(bytes.as_ref().as_ptr(), names[0].as_ptr()); + } + } + + #[test] + fn test_plain_getter_can_call_application_value_function() { + fn value(state: &State) -> i64 { + state.names.len() as i64 + } + wirefilter_fields! { + impl HelperFields for State { + "count" => { + fn(state: &State) -> i64 { value(state) } + }, + } + } + let scheme = schema(&[("count", Type::Int, false)]); + HelperFields::validate_scheme(&scheme).unwrap(); + let filter = scheme + .parse("count == 1") + .unwrap() + .compile_with_compiler(&mut crate::DefaultCompiler::<(), HelperFields>::new()); + let state = State { + scheme, + names: vec![Box::from("accept")].into_boxed_slice(), + country: None, + lazy: OnceLock::new(), + }; + assert!(filter.execute(&state).unwrap()); + } + + #[test] + fn test_getters_with_field_metadata() { + use crate::Field; + + fn check_field(state: &State, field: &Field, name: &str) { + assert_eq!(field.as_ref(), state.scheme.get_field(name).unwrap()); + } + + wirefilter_fields! { + impl MetadataFields for State { + "names" => { + fn(state: &State, field: &Field) -> &[Box] { + check_field(state, field, "names"); + if state.names.is_empty() { + return &[]; + } + &state.names + } + }, + "count" => { + fn(state: &State, field: &crate::Field) -> i64 { + check_field(state, field, "count"); + if state.names.is_empty() { + return 0; + } + state.names.len() as i64 + } + }, + "country" => { + fn(state: &State, field: &Field) -> Option<&str> { + check_field(state, field, "country"); + state.country.as_deref() + } + }, + "country_length" => { + fn(state: &State, field: &Field) -> Option { + check_field(state, field, "country_length"); + Some(state.country.as_ref()?.len() as i64) + } + }, + } + } + + let scheme = schema(&[ + ("names", Type::array(Type::Bytes), false), + ("count", Type::Int, false), + ("country", Type::Bytes, true), + ("country_length", Type::Int, true), + ]); + MetadataFields::validate_scheme(&scheme).unwrap(); + let mut compiler = crate::DefaultCompiler::<(), MetadataFields>::new(); + let filter = scheme + .parse(r#"any(names[*] == "accept") and count == 1 and country == "GB" and country_length == 2"#) + .unwrap() + .compile_with_compiler(&mut compiler); + let names_value = scheme + .parse_value("names") + .unwrap() + .compile_with_compiler(&mut compiler); + let country_value = scheme + .parse_value("country") + .unwrap() + .compile_with_compiler(&mut compiler); + let length_value = scheme + .parse_value("country_length") + .unwrap() + .compile_with_compiler(&mut compiler); + let mut state = State { + scheme, + names: vec![Box::from("accept")].into_boxed_slice(), + country: Some("GB".into()), + lazy: OnceLock::new(), + }; + assert!(filter.execute(&state).unwrap()); + let crate::LhsValue::Array(names) = names_value.execute(&state).unwrap().unwrap() else { + panic!("expected an array"); + }; + // The result borrows the context, not the field retained by the expression. + drop(names_value); + let crate::LhsValue::Bytes(bytes) = names.get(0).unwrap() else { + panic!("expected bytes"); + }; + assert_eq!(bytes.as_ref().as_ptr(), state.names[0].as_ptr()); + assert_eq!( + country_value.execute(&state).unwrap(), + Ok(crate::LhsValue::from("GB")) + ); + assert_eq!( + length_value.execute(&state).unwrap(), + Ok(crate::LhsValue::Int(2)) + ); + state.country = None; + assert!(!filter.execute(&state).unwrap()); + assert_eq!(country_value.execute(&state).unwrap(), Err(Type::Bytes)); + assert_eq!(length_value.execute(&state).unwrap(), Err(Type::Int)); + state.names = Box::new([]); + assert!(!filter.execute(&state).unwrap()); + } + + #[test] + fn test_plain_returns_and_optional_values() { + for optional_count in [false, true] { + let mut defs = definitions(); + defs[3].2 = optional_count; + let scheme = schema(&defs); + Fields::validate_scheme(&scheme).unwrap(); + let mut compiler = crate::DefaultCompiler::<(), Fields>::new(); + let names = scheme + .parse_value("names") + .unwrap() + .compile_with_compiler(&mut compiler); + let count = scheme + .parse_value("count") + .unwrap() + .compile_with_compiler(&mut compiler); + let country = scheme + .parse_value("country") + .unwrap() + .compile_with_compiler(&mut compiler); + let mut state = State { + scheme, + names: Box::new([]), + country: None, + lazy: OnceLock::new(), + }; + // Both plain bodies take an early-return branch for empty input. + assert_eq!(count.execute(&state).unwrap(), Ok(crate::LhsValue::Int(0))); + assert_eq!(country.execute(&state).unwrap(), Err(Type::Bytes)); + let crate::LhsValue::Array(empty) = names.execute(&state).unwrap().unwrap() else { + panic!("expected a present empty array"); + }; + assert!(empty.is_empty()); + + state.names = vec![Box::from("accept")].into_boxed_slice(); + state.country = Some("GB".into()); + assert_eq!(count.execute(&state).unwrap(), Ok(crate::LhsValue::Int(1))); + assert_eq!( + country.execute(&state).unwrap(), + Ok(crate::LhsValue::from("GB")) + ); + let crate::LhsValue::Array(values) = names.execute(&state).unwrap().unwrap() else { + panic!("expected an array"); + }; + let crate::LhsValue::Bytes(bytes) = values.get(0).unwrap() else { + panic!("expected bytes"); + }; + assert_eq!(bytes.as_ref().as_ptr(), state.names[0].as_ptr()); + } + } + + #[test] + fn test_mandatory_fields_reject_optional_getter_signatures() { + wirefilter_fields! { + impl OptionalFields for State { + "borrowed" => { + fn(_state: &State) -> Option<&str> { + panic!("validation must not execute getters") + } + }, + "owned" => { + fn(_state: &State, _field: &crate::Field) -> Option { Some(42) } + }, + "explicit" => { + fn(_state: &State, _field: &crate::Field) -> Option provider crate::OwnedValueProvider { + None + } + }, + } + } + let mut defs = [ + ("borrowed", Type::Bytes, false), + ("owned", Type::Int, false), + ("explicit", Type::Int, false), + ]; + assert_eq!( + OptionalFields::validate_scheme(&schema(&defs)), + Err(SchemeValidationError( + ["borrowed", "owned", "explicit"] + .map(|name| FieldDefinitionError::OptionalGetterForMandatoryField(name.into())) + .into(), + )) + ); + for (_, _, optional) in &mut defs { + *optional = true; + } + assert_eq!(OptionalFields::validate_scheme(&schema(&defs)), Ok(())); + } + + #[test] + fn test_explicit_providers_support_plain_and_optional_returns() { + wirefilter_fields! { + impl<'data> ProviderFields for BorrowingState<'data> { + "names" => { + provider crate::BorrowedValueProvider<[Box]>; + fn(state: &BorrowingState<'data>, field: &crate::Field) -> &[Box] { + assert_eq!(field.as_ref(), state.scheme.get_field("names").unwrap()); + state.names + } + }, + "first" => { + provider crate::BorrowedValueProvider; + fn(state: &BorrowingState<'data>, field: &crate::Field) -> Option<&str> { + assert_eq!(field.as_ref(), state.scheme.get_field("first").unwrap()); + Some(state.names.first()?.as_ref()) + } + }, + "legacy" => { + fn(state: &BorrowingState<'data>, field: &crate::Field) -> Option<&str> + provider crate::BorrowedValueProvider { + assert_eq!(field.as_ref(), state.scheme.get_field("legacy").unwrap()); + Some(state.names.first()?.as_ref()) + } + }, + "selection" => { + provider SelectionProvider; + fn(state: &BorrowingState<'data>, field: &crate::Field) + -> Option>> { + assert_eq!(field.as_ref(), state.scheme.get_field("selection").unwrap()); + state.names.first()?; + Some(crate::values::SliceView(state.names)) + } + }, + } + } + let scheme = schema(&[ + ("names", Type::array(Type::Bytes), false), + ("first", Type::Bytes, true), + ("legacy", Type::Bytes, true), + ("selection", Type::array(Type::Bytes), true), + ]); + ProviderFields::validate_scheme(&scheme).unwrap(); + let mut compiler = crate::DefaultCompiler::<(), ProviderFields>::new(); + let names = [Box::::from("accept")]; + for field in ["names", "first", "legacy", "selection"] { + let value = scheme + .parse_value(field) + .unwrap() + .compile_with_compiler(&mut compiler); + let present = BorrowingState { + scheme: &scheme, + names: &names, + }; + let result = value.execute(&present).unwrap().unwrap(); + let bytes = match &result { + crate::LhsValue::Bytes(bytes) => bytes, + crate::LhsValue::Array(values) => match values.get(0).unwrap() { + crate::LhsValue::Bytes(bytes) => bytes, + _ => panic!("expected bytes"), + }, + _ => panic!("unexpected field type"), + }; + assert_eq!(bytes.as_ref().as_ptr(), names[0].as_ptr()); + let absent = BorrowingState { + scheme: &scheme, + names: &[], + }; + let result = value.execute(&absent).unwrap(); + if field == "names" { + let crate::LhsValue::Array(values) = result.unwrap() else { + panic!("expected a present empty array"); + }; + assert!(values.is_empty()); + } else { + assert_eq!(result, Err(scheme.get_field(field).unwrap().get_type())); + } + } + } + + #[test] + fn test_missing_unused_and_optional_fields() { + for optional in [false, true] { + let mut fields = definitions(); + fields.push(("unused", Type::Int, optional)); + assert_eq!( + Fields::validate_scheme(&schema(&fields)), + Err(SchemeValidationError( + vec![FieldDefinitionError::MissingGetter("unused".into())].into_boxed_slice() + )), + ); + } + } + + #[test] + fn test_unknown_getter() { + let mut fields = definitions(); + fields.remove(1); + assert_eq!( + Fields::validate_scheme(&schema(&fields)), + Err(SchemeValidationError( + vec![FieldDefinitionError::UnknownField("country".into())].into_boxed_slice() + )), + ); + } + + #[test] + fn test_nested_type_mismatch() { + let mut fields = definitions(); + fields[0].1 = Type::array(Type::Int); + assert_eq!( + Fields::validate_scheme(&schema(&fields)), + Err(SchemeValidationError( + vec![FieldDefinitionError::TypeMismatch { + name: "names".into(), + scheme_type: Type::array(Type::Int), + getter_type: Type::array(Type::Bytes), + }] + .into_boxed_slice() + )), + ); + } + + #[test] + fn test_all_registration_errors_reported_together() { + let scheme = schema(&[("field", Type::Bytes, false), ("missing", Type::Int, true)]); + assert_eq!( + validate_field_definitions( + &scheme, + &[ + ("field", Some(Type::Int), false), + ("unknown", Some(Type::Bool), false), + ("field", Some(Type::Bytes), false), + ] + ), + Err(SchemeValidationError( + vec![ + FieldDefinitionError::TypeMismatch { + name: "field".into(), + scheme_type: Type::Bytes, + getter_type: Type::Int, + }, + FieldDefinitionError::UnknownField("unknown".into()), + FieldDefinitionError::DuplicateGetter("field".into()), + FieldDefinitionError::MissingGetter("missing".into()), + ] + .into_boxed_slice() + )), + ); + } +} diff --git a/engine/src/context/mod.rs b/engine/src/context/mod.rs new file mode 100644 index 00000000..cab2151b --- /dev/null +++ b/engine/src/context/mod.rs @@ -0,0 +1,1646 @@ +//! Runtime contexts, reusable state, and statically bound field access. +//! +//! Filters read fields through typed getters bound during compilation. An +//! application can expose its Rust values directly, or retain fields as +//! [`LhsValue`]. Both representations use the same field-binding +//! and reader interfaces. Contexts may own their data or borrow it from the +//! application. +//! +//! # Context interfaces and state +//! +//! [`EvaluationContext`] exposes user data and list +//! matchers, and its [`HasScheme`] supertrait exposes the scheme. +//! These traits do not prescribe storage or define field getters. +//! [`EvaluationContextMut`] adds mutable access for +//! applications that need to update user data or list matchers. +//! +//! [`ContextState`](crate::ContextState) is an optional owner of a scheme, +//! list matchers, and user data. It contains no field values. A context can +//! borrow it and forward the evaluation-context methods to it, while getters read +//! the application's own fields. Applications that already store this state can +//! implement the interfaces directly without using `ContextState`. +//! +//! [`ContextFamily`] associates compiled expressions with +//! the application's context type, including contexts that borrow data. It also +//! binds fields to typed getters during compilation. The +//! [`wirefilter_fields!`](crate::wirefilter_fields) macro generates a family and +//! its field bindings; it does not create a context or store request values. +//! Getters read values when an expression is evaluated. +//! +//! Call [`ContextFamily::validate_scheme`] +//! during application initialization for each scheme/family pair. Its default +//! implementation binds every scheme field and checks static provider types +//! without evaluating getters. The macro additionally checks extra and duplicate +//! definitions and rejects optional getter signatures for mandatory fields. +//! Handwritten families must uphold those constraints themselves or override +//! validation. Parsing, compilation, and execution do not run validation. +//! +//! The default [`ExecutionContext`] implements the same +//! evaluation interfaces and already owns equivalent state alongside its stored +//! field values. It does not require a separate `ContextState`. +//! Its default [`ExecutionContextFamily`] binds those +//! fields through [`LhsValueProvider`], using the same +//! reader interface as application-owned Rust values. +//! +//! # Borrowing request data and reusable state +//! +//! This example borrows a request's `Vec` without constructing a +//! Wirefilter array. `RequestFields` is the family generated by the macro; +//! `RequestContext` is the value passed to the compiled filter. +//! +//! ``` +//! use wirefilter::{ +//! ContextState, DefaultCompiler, EvaluationContext, HasScheme, List, ListMatcher, ListRef, +//! Scheme, SchemeBuilder, Type, +//! }; +//! +//! struct Request { +//! names: Vec, +//! } +//! +//! struct RequestContext<'a> { +//! request: &'a Request, +//! state: &'a ContextState<()>, +//! } +//! +//! impl HasScheme for RequestContext<'_> { +//! fn scheme(&self) -> &Scheme { +//! self.state.scheme() +//! } +//! } +//! +//! impl EvaluationContext for RequestContext<'_> { +//! type UserData = (); +//! +//! fn get_user_data(&self) -> &() { +//! self.state.get_user_data() +//! } +//! +//! fn get_list_matcher(&self, list: ListRef<'_>) -> &dyn ListMatcher { +//! self.state.get_list_matcher(list) +//! } +//! +//! fn get_list_matcher_unchecked(&self, list: &List) -> &dyn ListMatcher { +//! self.state.get_list_matcher_unchecked(list) +//! } +//! } +//! +//! wirefilter::wirefilter_fields! { +//! impl<'data> RequestFields for RequestContext<'data> { +//! "request.names" => { +//! fn(ctx: &RequestContext<'data>) -> &[String] { +//! &ctx.request.names +//! } +//! }, +//! } +//! } +//! +//! # fn main() -> Result<(), Box> { +//! let mut builder = SchemeBuilder::new(); +//! builder.add_field("request.names", Type::array(Type::Bytes))?; +//! let scheme = builder.build(); +//! +//! // Validate every field definition during initialization, before compilation. +//! RequestFields::validate_scheme(&scheme)?; +//! let mut compiler = DefaultCompiler::<(), RequestFields>::new(); +//! let filter = scheme +//! .parse(r#"any(request.names[*] eq "example")"#)? +//! .compile_with_compiler(&mut compiler); +//! +//! let request = Request { +//! names: vec!["example".into()], +//! }; +//! let state = ContextState::<()>::new(&scheme); +//! +//! // Each evaluation borrows the same request and state. +//! for _ in 0..2 { +//! let context = RequestContext { +//! request: &request, +//! state: &state, +//! }; +//! assert!(filter.execute(&context)?); +//! } +//! # Ok(()) +//! # } +//! ``` +//! +//! Initialization validation checks getter coverage, types, and optionality; it +//! does not call getters. Parsing and compilation do not invoke it automatically. +//! +//! The application controls the lifetime and cleanup of reusable state. +//! [`ContextState::clear`](crate::ContextState::clear) clears list contents +//! but retains user data; any compiler-specific cleanup remains the application's +//! responsibility. See [`crate::values`] for supported storage and custom collection +//! interfaces, and [`wirefilter_fields!`](crate::wirefilter_fields) for getter forms. +//! +//! # Fields backed by `LhsValue` +//! +//! A context may also store [`LhsValue`] fields whose types are +//! chosen when the scheme is built, including arbitrary array/map nesting. +//! [`LhsValueProvider`] reads these fields through the +//! same field-binding interface. Its reader traverses and compares the existing +//! values without rebuilding containers. Function arguments borrow selected +//! values; flattening multiple selections still needs an output container. +//! +//! The default `ExecutionContext` already uses this provider. The explicit +//! binding below shows how a downstream context can implement its own table and +//! evaluation state, then provide a getter with the same return signature. The +//! example reuses `ExecutionContext` as its table to focus on field binding. +//! +//! ``` +//! use wirefilter::{ +//! ContextFamily, DefaultCompiler, ExecutionContext, Field, FieldBindingError, FieldConsumer, +//! FieldGetter, LhsValue, LhsValueProvider, SchemeBuilder, Type, +//! }; +//! +//! struct Fields; +//! struct Getter; +//! +//! impl ContextFamily for Fields { +//! type Context<'data> = ExecutionContext<'data>; +//! type UserData = (); +//! +//! fn bind>( +//! field: Field, +//! consumer: B, +//! ) -> Result { +//! consumer.consume(field, Getter) +//! } +//! } +//! +//! impl FieldGetter for Getter { +//! type Provider = LhsValueProvider; +//! +//! fn get<'ctx, 'data: 'ctx>( +//! &self, +//! context: &'ctx ExecutionContext<'data>, +//! field: &Field, +//! ) -> Option<&'ctx LhsValue<'ctx>> { +//! context.get_field_value(field.as_ref()) +//! } +//! } +//! +//! # fn main() -> Result<(), Box> { +//! let mut builder = SchemeBuilder::new(); +//! builder.add_field("port", Type::Int)?; +//! let scheme = builder.build(); +//! Fields::validate_scheme(&scheme)?; +//! let filter = scheme +//! .parse("port eq 443")? +//! .compile_with_compiler(&mut DefaultCompiler::<(), Fields>::new()); +//! let mut context = ExecutionContext::new(&scheme); +//! context.set_field_value(scheme.get_field("port")?, 443i64)?; +//! assert!(filter.execute(&context)?); +//! # Ok(()) +//! # } +//! ``` +//! +//! The field's scheme type determines comparison and traversal during filter +//! compilation. Because a getter returning `&LhsValue` has no fixed logical type, +//! startup validation cannot check which variant it returns. The context must +//! check values against the scheme when inserting them, as `ExecutionContext` +//! does. The field macro still validates getter coverage and optionality. +//! +//! [`crate::values::FieldValueType`] describes these scheme-typed field roots as +//! well as ordinary static [`crate::values::ValueType`] implementations. The +//! collection interfaces require a static `ValueType` for their elements: for +//! example, `Vec` is supported, while a runtime-typed array uses +//! `LhsValue::Array` rather than `Vec`. + +pub(crate) mod fields; +pub(crate) mod state; + +use crate::values::{FieldValueType, IntoLhsValue, LhsValueType, Value}; +use crate::{ExecutionContext, Field, GetType, LhsValue, List, ListMatcher, ListRef, Scheme, Type}; +use std::marker::PhantomData; + +/// The scheme associated with a runtime context. +pub trait HasScheme { + /// Scheme used to interpret fields and lists. + fn scheme(&self) -> &Scheme; +} + +/// State shared by expression evaluation, independently of field storage. +/// +/// Together with [`HasScheme`], this trait exposes the scheme, list matchers, +/// and user data available during evaluation. Field getters are bound separately +/// through [`ContextFamily`]. +/// +/// An application context can use its existing storage or forward these methods +/// to a borrowed [`crate::ContextState`]. [`ExecutionContext`] already implements +/// this interface alongside its stored field values. See [`crate::context`] for +/// a complete example using application-owned fields and reusable state. +pub trait EvaluationContext: HasScheme { + /// Per-evaluation state used by compiler customizations. + type UserData; + + /// State retained by the embedding runtime. + fn get_user_data(&self) -> &Self::UserData; + + /// Matcher for a list registered in this context's scheme. + fn get_list_matcher(&self, list: ListRef<'_>) -> &dyn ListMatcher; + + /// Matcher for the list registered for a Wirefilter type, if present. + #[inline] + fn get_list_matcher_from_type(&self, ty: &Type) -> Option<&dyn ListMatcher> { + let list = self.scheme().get_list(ty)?; + Some(self.get_list_matcher(list)) + } + + /// Matcher access after the outer filter has checked scheme compatibility. + /// + /// The default delegates to the checked accessor. Contexts forwarding matcher + /// storage to another context, such as [`crate::ContextState`], should + /// also forward this method to avoid repeating its scheme assertion for each + /// list comparison. The supplied list must belong to this context's scheme. + #[inline] + fn get_list_matcher_unchecked(&self, list: &List) -> &dyn ListMatcher { + self.get_list_matcher(list.as_ref()) + } +} + +/// Mutable executor state, independently of field storage. +/// +/// Extends [`EvaluationContext`] for updating list matchers and user data. Filter +/// evaluation only requires the shared interface; an application adds this +/// trait when it also needs mutable access. Both [`ExecutionContext`] and +/// [`crate::ContextState`] implement it. +pub trait EvaluationContextMut: EvaluationContext { + /// State used to initialize and clean up compiler customizations. + fn get_user_data_mut(&mut self) -> &mut Self::UserData; + + /// Mutable matcher for a list registered in this context's scheme. + fn get_list_matcher_mut(&mut self, list: ListRef<'_>) -> &mut dyn ListMatcher; + + /// Mutable matcher for the list registered for a Wirefilter type, if present. + #[inline] + fn get_list_matcher_mut_from_type(&mut self, ty: &Type) -> Option<&mut dyn ListMatcher> { + let list = self.scheme().get_list(ty)?.to_owned(); + Some(self.get_list_matcher_mut(list.as_ref())) + } +} + +/// A field-definition problem detected before filters are evaluated. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum FieldDefinitionError { + /// A scheme field failed binding or static type checking during validation. + #[error("Field {name} failed validation: {error}")] + Binding { + /// Scheme field being validated. + name: Box, + /// The binding or type-checking failure. + #[source] + error: FieldBindingError, + }, + /// A scheme field has no getter. + #[error("Missing field getter for {0}")] + MissingGetter(Box), + /// A getter names a field absent from the scheme. + #[error("Unknown field {0}")] + UnknownField(Box), + /// More than one getter names the same field. + #[error("Duplicate field getter for {0}")] + DuplicateGetter(Box), + /// A getter for a mandatory scheme field can return no value. + #[error("Field {0} is mandatory but its getter returns Option")] + OptionalGetterForMandatoryField(Box), + /// Getter and scheme types differ. + #[error("Field {name} has type {getter_type}, expected {scheme_type}")] + TypeMismatch { + /// Field name. + name: Box, + /// Type declared in the scheme. + scheme_type: Type, + /// Type returned by the getter. + getter_type: Type, + }, +} + +/// All field-definition problems found during explicit scheme validation. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[error("Invalid field definitions: {0:?}")] +pub struct SchemeValidationError(pub Box<[FieldDefinitionError]>); + +/// A failure to bind a field or prepare its access. +/// +/// Binding APIs return this error. Default scheme validation collects it in +/// [`FieldDefinitionError::Binding`]. The infallible expression compilation APIs +/// panic on a binding failure; filter execution does not return this error. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum FieldBindingError { + /// A field cannot be resolved by the context family. + #[error("Unknown field {0}")] + UnknownField(Box), + /// The getter's logical type differs from the scheme. + #[error("Field {field} has type {actual}, expected {expected}")] + TypeMismatch { + /// Field whose type differs. + field: Box, + /// Type declared in the scheme. + expected: Type, + /// Type provided by the getter. + actual: Type, + }, + /// A path is incompatible with the declared value type. + /// + /// Produced during binding or path compilation, not evaluation. Parsed paths + /// for matching scheme and getter types are expected to be compatible. + #[error("Unsupported value operation: {0}")] + Unsupported(&'static str), +} + +/// A lifetime-independent description of contexts and their field bindings. +/// +/// A separate marker can associate `Context<'data>` with a borrowing request +/// context while compiled filters retain no request lifetime. An owned context +/// type satisfying `'static` may instead implement this trait itself and use +/// `type Context<'data> = Self`. The family is used as a type parameter; compiled +/// expressions do not store an instance of it. [`crate::wirefilter_fields!`] +/// generates a separate marker for both forms of context. +/// +/// [`EvaluationContext`] describes the state available during evaluation; this +/// trait selects that context type and binds its field getters for validation +/// and compilation. It does not own the context, request data, or +/// [`crate::ContextState`]. See [`crate::context`] for a complete example +/// showing their relationship. +pub trait ContextFamily: Sized + 'static { + /// State exposed to compiler customizations. + type UserData: 'static; + + /// The runtime context, possibly retaining request references. + type Context<'data>: EvaluationContext; + + /// Resolve a field and pass its concrete getter to the consumer. + /// Called by default scheme validation and field compilation. + /// Ownership of the logical field transfers to the consumer. + fn bind>( + field: Field, + consumer: B, + ) -> Result; + + /// Validate field definitions without evaluating getters. + /// + /// Applications must call this explicitly during initialization for each + /// scheme and context family, before loading rulesets or compiling + /// expressions. A failed check is an initialization error. Parsing, + /// compilation, and evaluation do not invoke this method automatically. + /// + /// The default implementation binds every scheme field and checks its + /// provider's static type, collecting all binding failures. It constructs + /// getters through [`Self::bind`] but never calls [`FieldGetter::get`]. + /// Scheme-typed providers such as [`LhsValueProvider`] have no static type to + /// check; their storage must validate values separately. + /// + /// This cannot detect extra or duplicate getter definitions, or determine + /// whether a mandatory getter can return `None`. Handwritten families remain + /// responsible for those constraints and may override this method for + /// stronger checks. [`crate::wirefilter_fields!`] generates an override that + /// checks all definitions, including their declared optionality. + /// [`ExecutionContextFamily`] overrides this with a no-op: its field table + /// comes from the scheme and its context validates values on insertion. + fn validate_scheme(scheme: &Scheme) -> Result<(), SchemeValidationError> { + let errors = scheme + .fields() + .filter_map(|field| { + Self::bind(field.to_owned(), CheckFieldType) + .err() + .map(|error| FieldDefinitionError::Binding { + name: field.name().into(), + error, + }) + }) + .collect::>(); + if errors.is_empty() { + Ok(()) + } else { + Err(SchemeValidationError(errors)) + } + } +} + +struct CheckFieldType; + +impl FieldConsumer for CheckFieldType { + type Output = (); + + fn consume>( + self, + field: Field, + _: G, + ) -> Result { + check_field_type::<::ValueType>(&field) + } +} + +pub(crate) fn check_field_type(field: &Field) -> Result<(), FieldBindingError> { + let expected = field.get_type(); + if let Some(actual) = S::STATIC_TYPE + && actual != expected + { + return Err(FieldBindingError::TypeMismatch { + field: field.name().into(), + expected, + actual, + }); + } + Ok(()) +} + +/// A context reference whose type records that request data outlives this access. +/// +/// Higher-ranked closures over an arbitrary context family's associated type do +/// not otherwise imply this lifetime relationship. The witness lets compiled +/// value expressions return borrowed payloads, including from invariant contexts. +/// It contains only the original reference and requires no allocation. +#[repr(transparent)] +pub struct ContextRef<'ctx, 'data, F: ContextFamily> { + context: &'ctx F::Context<'data>, + lifetime: PhantomData<&'ctx &'data ()>, +} + +impl Copy for ContextRef<'_, '_, F> {} + +impl Clone for ContextRef<'_, '_, F> { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +impl<'ctx, 'data: 'ctx, F: ContextFamily> ContextRef<'ctx, 'data, F> { + /// Record the relationship between the context borrow and its retained data. + #[inline] + pub fn new(context: &'ctx F::Context<'data>) -> Self { + Self { + context, + lifetime: PhantomData, + } + } + + /// Return the original context borrow, independently of the witness's storage. + #[inline] + pub fn context(self) -> &'ctx F::Context<'data> { + self.context + } +} + +/// An operation that validates a field binding or compiles field access. +pub trait FieldConsumer: Sized { + /// Result produced by this operation, such as an executable expression. + type Output; + + /// Process a getter whose concrete type remains known. + /// `field` is the original logical field supplied to the binding operation. + /// This includes getters borrowing existing [`LhsValue`] fields through + /// [`LhsValueProvider`]. + fn consume>( + self, + field: Field, + getter: G, + ) -> Result; +} + +/// A field's representation across request and access lifetimes. +pub trait ProviderFamily: 'static { + /// Recursive Wirefilter type represented by the provider. + type ValueType: FieldValueType; + + /// Storage usable by generated contexts and custom adapters. + type Storage<'data>; + + /// A reference or selection descriptor returned by one field access. + type Value<'ctx, 'data>: Value + IntoLhsValue<'ctx> + where + 'data: 'ctx; + + /// Obtain a shorter access to retained storage. + fn reborrow<'ctx, 'data: 'ctx>(value: &'ctx Self::Storage<'data>) -> Self::Value<'ctx, 'data>; +} + +/// A provider borrowing storage whose Rust type has no request lifetime. +pub struct BorrowedValueProvider(PhantomData T>); + +impl ProviderFamily for BorrowedValueProvider { + type Storage<'data> = &'data T; + type Value<'ctx, 'data> + = &'ctx T + where + 'data: 'ctx; + type ValueType = T::ValueType; + + #[inline] + fn reborrow<'ctx, 'data: 'ctx>(value: &'ctx Self::Storage<'data>) -> Self::Value<'ctx, 'data> { + value + } +} + +/// A provider returning a value whose Rust type has no request lifetime. +/// +/// Scalar getters can return values directly. Custom by-value descriptors that +/// borrow request data instead implement [`ProviderFamily`] with those lifetimes. +pub struct OwnedValueProvider(PhantomData T>); + +impl ProviderFamily for OwnedValueProvider +where + T: Value + Clone + 'static, + for<'a> T: IntoLhsValue<'a>, +{ + type Storage<'data> = T; + type Value<'ctx, 'data> + = T + where + 'data: 'ctx; + type ValueType = T::ValueType; + + #[inline] + fn reborrow<'ctx, 'data: 'ctx>(value: &'ctx Self::Storage<'data>) -> Self::Value<'ctx, 'data> { + value.clone() + } +} + +/// A provider borrowing a stored value with request-borrowed payloads. +/// +/// The scheme supplies the logical field type. A context must validate inserted +/// values against it; this provider does not convert or copy the retained value. +/// Handwritten getters can use this provider to read an ordinary field table +/// through the field-binding interface. +pub struct LhsValueProvider; + +impl ProviderFamily for LhsValueProvider { + type Storage<'data> = LhsValue<'data>; + type Value<'ctx, 'data> + = &'ctx LhsValue<'ctx> + where + 'data: 'ctx; + type ValueType = LhsValueType; + + #[inline] + fn reborrow<'ctx, 'data: 'ctx>(value: &'ctx Self::Storage<'data>) -> Self::Value<'ctx, 'data> { + value + } +} + +/// Concrete field access captured inside an executable expression. +pub trait FieldGetter: Send + Sync + 'static { + /// The field representation. + type Provider: ProviderFamily; + + /// Read the field for this evaluation without constructing a value enum. + /// + /// `field` is the original logical field supplied to the binding operation, + /// including its scheme index, name, type, and optionality. Its borrow is + /// independent of the context borrow and cannot be retained in the result. + /// + /// A getter bound to a mandatory scheme field must return `Some`. The field + /// macro enforces this through its return signatures and explicit scheme + /// validation; handwritten implementations must uphold the same contract. + fn get<'ctx, 'data: 'ctx>( + &self, + context: &'ctx F::Context<'data>, + field: &Field, + ) -> Option<::Value<'ctx, 'data>>; +} + +/// A getter result preserving the relationship between access and request lifetimes. +/// +/// This transparent wrapper keeps the provider's associated value outside the +/// higher-ranked function pointer's return projection. The value has exactly +/// the representation of `Option>`. +// Returning the projection directly encounters rustc's higher-ranked lifetime +// limitation (rust-lang/rust#100013), despite ContextRef carrying the constraint. +#[repr(transparent)] +pub struct OptionalFieldGetterResult<'ctx, 'data: 'ctx, P: ProviderFamily> { + value: Option>, +} + +impl<'ctx, 'data: 'ctx, P: ProviderFamily> OptionalFieldGetterResult<'ctx, 'data, P> { + /// Retain the result of a getter for the context access lifetime. + pub fn new(value: Option>) -> Self { + Self { value } + } +} + +/// Field access represented by a typed function pointer. +/// +/// Fields in the same context family using the same provider share this getter +/// type, allowing consumers to share their compiled expression implementations. +/// The pointer selects the field's getter on each access; values retain +/// their concrete type and do not require an intermediate value enum or allocation. +/// Handwritten getters retaining additional state can implement [`FieldGetter`] +/// directly instead. +pub struct FieldGetterFn { + callback: for<'ctx, 'data, 'field> fn( + ContextRef<'ctx, 'data, F>, + &'field Field, + ) -> OptionalFieldGetterResult<'ctx, 'data, P>, +} + +impl FieldGetterFn { + /// Wrap a getter without allocating or retaining a request lifetime. + /// + /// The context witness records that request data outlives the access. Field + /// metadata is borrowed independently and cannot be retained in the result. + /// A mandatory field's result must contain `Some`, as for [`FieldGetter`]. + pub fn new( + callback: for<'ctx, 'data, 'field> fn( + ContextRef<'ctx, 'data, F>, + &'field Field, + ) -> OptionalFieldGetterResult<'ctx, 'data, P>, + ) -> Self { + Self { callback } + } +} + +impl FieldGetter for FieldGetterFn { + type Provider = P; + + // Keep this adapter as thin as the macro's concrete FieldGetter adapter. + #[inline] + fn get<'ctx, 'data: 'ctx>( + &self, + context: &'ctx F::Context<'data>, + field: &Field, + ) -> Option> { + (self.callback)(ContextRef::new(context), field).value + } +} + +/// A plain getter result preserving access and request lifetime relationships. +/// +/// Like [`OptionalFieldGetterResult`], this transparent wrapper avoids a higher-ranked +/// associated-type projection in the callback signature. It contains a plain +/// value; presence is added by [`RequiredFieldGetterFn::get`] after the +/// function pointer returns. +#[repr(transparent)] +pub struct RequiredFieldGetterResult<'ctx, 'data: 'ctx, P: ProviderFamily> { + value: P::Value<'ctx, 'data>, +} + +impl<'ctx, 'data: 'ctx, P: ProviderFamily> RequiredFieldGetterResult<'ctx, 'data, P> { + /// Retain an always-present value for the context access lifetime. + pub fn new(value: P::Value<'ctx, 'data>) -> Self { + Self { value } + } +} + +/// Always-present field access represented by a typed function pointer. +/// +/// Plain-return getters in the same context family using the same provider share +/// this getter type. The callback returns a plain value; this wrapper +/// constructs `Some` after calling it, so consumers can establish presence +/// independently of the selected callback. The scheme field may be mandatory or +/// optional. Getters returning `Option` use [`FieldGetterFn`] instead. +pub struct RequiredFieldGetterFn { + callback: for<'ctx, 'data, 'field> fn( + ContextRef<'ctx, 'data, F>, + &'field Field, + ) -> RequiredFieldGetterResult<'ctx, 'data, P>, +} + +impl RequiredFieldGetterFn { + /// Wrap a plain-return getter without allocation or a retained request lifetime. + /// + /// Field metadata is borrowed independently and cannot be retained in the + /// result. The callback must produce a value on every access. + pub fn new( + callback: for<'ctx, 'data, 'field> fn( + ContextRef<'ctx, 'data, F>, + &'field Field, + ) -> RequiredFieldGetterResult<'ctx, 'data, P>, + ) -> Self { + Self { callback } + } +} + +impl FieldGetter for RequiredFieldGetterFn { + type Provider = P; + + // Match the optional wrapper's thin adapter while making presence visible + // to consumers without inspecting the callback implementation. + #[inline] + fn get<'ctx, 'data: 'ctx>( + &self, + context: &'ctx F::Context<'data>, + field: &Field, + ) -> Option> { + Some((self.callback)(ContextRef::new(context), field).value) + } +} + +/// Default family for the ordinary field-table [`ExecutionContext`]. +/// +/// Fields use [`LhsValueProvider`] through the same binding interface as +/// application-owned Rust values. The context owns the field table and validates +/// inserted values against the scheme; readers borrow the retained values and +/// containers. +pub struct ExecutionContextFamily(PhantomData U>); + +impl ContextFamily for ExecutionContextFamily { + type Context<'data> = ExecutionContext<'data, U>; + type UserData = U; + + #[inline] + fn bind>( + field: Field, + consumer: B, + ) -> Result { + consumer.consume(field, ExecutionContextGetter) + } + + fn validate_scheme(_: &Scheme) -> Result<(), SchemeValidationError> { + // Field coverage comes from the scheme; values are checked on insertion. + Ok(()) + } +} + +struct ExecutionContextGetter; + +impl FieldGetter> for ExecutionContextGetter { + type Provider = LhsValueProvider; + + #[inline] + fn get<'ctx, 'data: 'ctx>( + &self, + context: &'ctx ExecutionContext<'data, U>, + field: &Field, + ) -> Option<&'ctx LhsValue<'ctx>> { + context.get_field_value_unchecked(field) + } +} + +#[cfg(test)] +mod tests { + use super::{ + ContextFamily, FieldBindingError, FieldConsumer, FieldGetter, LhsValueProvider, + SchemeValidationError, + }; + use crate::{ExecutionContext, Field, Scheme}; + + #[test] + fn test_schema_validation_is_explicit() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + static VALIDATIONS: AtomicUsize = AtomicUsize::new(0); + struct Family; + struct Getter; + impl ContextFamily for Family { + type Context<'data> = ExecutionContext<'data>; + type UserData = (); + + fn bind>( + field: Field, + consumer: B, + ) -> Result { + consumer.consume(field, Getter) + } + + fn validate_scheme(scheme: &Scheme) -> Result<(), SchemeValidationError> { + VALIDATIONS.fetch_add(1, Ordering::Relaxed); + crate::validate_field_definitions( + scheme, + &[("flag", Some(crate::Type::Bool), false)], + ) + } + } + impl FieldGetter for Getter { + type Provider = LhsValueProvider; + + fn get<'ctx, 'data: 'ctx>( + &self, + context: &'ctx ExecutionContext<'data>, + field: &Field, + ) -> Option<&'ctx crate::LhsValue<'ctx>> { + context.get_field_value_unchecked(field) + } + } + + let scheme = Scheme! { flag: Bool }.build(); + Family::validate_scheme(&scheme).unwrap(); + assert_eq!(VALIDATIONS.load(Ordering::Relaxed), 1); + + let mut compiler = crate::DefaultCompiler::<(), Family>::new(); + let mut context = ExecutionContext::new(&scheme); + context.set_field_value_from_name("flag", true).unwrap(); + for _ in 0..3 { + let filter = scheme + .parse("flag") + .unwrap() + .compile_with_compiler(&mut compiler); + let value = scheme + .parse_value("flag") + .unwrap() + .compile_with_compiler(&mut compiler); + assert!(filter.execute(&context).unwrap()); + assert_eq!( + value.execute(&context).unwrap(), + Ok(crate::LhsValue::Bool(true)) + ); + } + assert_eq!(VALIDATIONS.load(Ordering::Relaxed), 1); + } + + #[test] + fn test_custom_lhs_value_getter_uses_logical_field_metadata() { + struct Family; + struct Getter; + impl ContextFamily for Family { + type Context<'data> = ExecutionContext<'data>; + type UserData = (); + + fn bind>( + field: Field, + consumer: B, + ) -> Result { + consumer.consume(field, Getter) + } + } + impl FieldGetter for Getter { + type Provider = LhsValueProvider; + + fn get<'ctx, 'data: 'ctx>( + &self, + context: &'ctx ExecutionContext<'data>, + _: &Field, + ) -> Option<&'ctx crate::LhsValue<'ctx>> { + // Both logical fields read the same optional storage slot. + let storage = context.scheme().get_field("storage").unwrap(); + context.get_field_value(storage) + } + } + #[derive(Default)] + struct Compiler { + bindings: usize, + } + impl crate::Compiler for Compiler { + type Family = Family; + type U = (); + + fn bind_field>( + &mut self, + field: Field, + consumer: B, + ) -> Result { + self.bindings += 1; + Family::bind(field, consumer) + } + } + + let mut builder = crate::SchemeBuilder::new(); + builder.add_field("required", crate::Type::Bool).unwrap(); + builder + .add_optional_field("optional", crate::Type::Bool) + .unwrap(); + builder + .add_optional_field("storage", crate::Type::Bool) + .unwrap(); + let scheme = builder.build(); + let mut compiler = Compiler::default(); + let required = scheme + .parse("required") + .unwrap() + .compile_with_compiler(&mut compiler); + let optional = scheme + .parse("optional") + .unwrap() + .compile_with_compiler(&mut compiler); + let required_value = scheme + .parse_value("required") + .unwrap() + .compile_with_compiler(&mut compiler); + let optional_value = scheme + .parse_value("optional") + .unwrap() + .compile_with_compiler(&mut compiler); + assert_eq!(compiler.bindings, 4); + + let mut context = ExecutionContext::new(&scheme); + assert!(!optional.execute(&context).unwrap()); + assert_eq!( + optional_value.execute(&context).unwrap(), + Err(crate::Type::Bool) + ); + let operations: [&dyn Fn(); 2] = [ + &|| { + let _ = required.execute(&context); + }, + &|| { + let _ = required_value.execute(&context); + }, + ]; + for operation in operations { + let panic = + std::panic::catch_unwind(std::panic::AssertUnwindSafe(operation)).unwrap_err(); + assert_eq!( + panic.downcast_ref::().map(String::as_str), + Some("Field required was registered as mandatory but not given a value"), + ); + } + for value in [false, true] { + context.set_field_value_from_name("storage", value).unwrap(); + assert_eq!(required.execute(&context).unwrap(), value); + assert_eq!(optional.execute(&context).unwrap(), value); + assert_eq!( + required_value.execute(&context).unwrap(), + Ok(crate::LhsValue::Bool(value)) + ); + assert_eq!( + optional_value.execute(&context).unwrap(), + Ok(crate::LhsValue::Bool(value)) + ); + } + assert_eq!(compiler.bindings, 4); + } + + type Headers = std::collections::BTreeMap, Vec>>; + + struct NativeContext { + scheme: Scheme, + names: Vec>, + flags: Vec, + other_flags: Vec, + text: Option>, + other_text: Option>, + num: Option, + other_num: Option, + ip: std::net::IpAddr, + headers: Headers, + required: Option>, + lists: Box<[Box]>, + list_accesses: std::cell::Cell, + } + + impl super::HasScheme for NativeContext { + fn scheme(&self) -> &Scheme { + &self.scheme + } + } + + impl super::EvaluationContext for NativeContext { + type UserData = (); + + fn get_user_data(&self) -> &() { + &() + } + + fn get_list_matcher(&self, list: crate::ListRef<'_>) -> &dyn crate::ListMatcher { + self.list_accesses.set(self.list_accesses.get() + 1); + &*self.lists[list.index()] + } + } + + crate::wirefilter_fields! { + impl NativeFields for NativeContext { + "names" => { + fn(ctx: &NativeContext) -> &Vec> { &ctx.names } + }, + "flags" => { + fn(ctx: &NativeContext) -> &Vec { &ctx.flags } + }, + "other_flags" => { + fn(ctx: &NativeContext) -> &Vec { &ctx.other_flags } + }, + "text" => { + fn(ctx: &NativeContext) -> Option<&str> { ctx.text.as_deref() } + }, + "other_text" => { + fn(ctx: &NativeContext) -> Option<&str> { ctx.other_text.as_deref() } + }, + "num" => { + fn(ctx: &NativeContext) -> Option { ctx.num } + }, + "other_num" => { + fn(ctx: &NativeContext) -> Option { ctx.other_num } + }, + "ip" => { + fn(ctx: &NativeContext) -> std::net::IpAddr { ctx.ip } + }, + "headers" => { + fn(ctx: &NativeContext) -> &Headers { &ctx.headers } + }, + "required" => { + fn(ctx: &NativeContext) -> &str { + ctx.required.as_deref().expect("test context has its required field") + } + }, + } + } + + fn echo<'a>(args: crate::FunctionArgs<'_, 'a>) -> Option> { + args.next()?.ok() + } + + fn owned_names<'a>(args: crate::FunctionArgs<'_, 'a>) -> Option> { + args.next()?.as_ref().ok()?; + Some(crate::LhsValue::Array( + crate::Array::try_from_iter( + crate::Type::Bytes, + ["Accept", "Other"].map(|s| crate::LhsValue::from(Box::::from(s))), + ) + .unwrap(), + )) + } + + fn native_context(nil_not_equal: bool) -> NativeContext { + use crate::{ + SimpleFunctionArgKind, SimpleFunctionDefinition, SimpleFunctionImpl, + SimpleFunctionParam, Type, + }; + let array_bytes = Type::array(Type::Bytes); + let headers = Type::map(array_bytes); + let mut builder = crate::SchemeBuilder::new(); + for (name, ty, optional) in [ + ("names", array_bytes, false), + ("flags", Type::array(Type::Bool), false), + ("other_flags", Type::array(Type::Bool), false), + ("text", Type::Bytes, true), + ("other_text", Type::Bytes, true), + ("num", Type::Int, true), + ("other_num", Type::Int, true), + ("ip", Type::Ip, false), + ("headers", headers, false), + ("required", Type::Bytes, false), + ] { + if optional { + builder.add_optional_field(name, ty).unwrap(); + } else { + builder.add_field(name, ty).unwrap(); + } + } + for (name, ty) in [ + ("echo", Type::Bytes), + ("echo_names", array_bytes), + ("echo_headers", headers), + ] { + builder + .add_function( + name, + SimpleFunctionDefinition { + params: Box::new([SimpleFunctionParam { + arg_kind: SimpleFunctionArgKind::Field, + val_type: ty, + }]), + opt_params: Box::new([]), + return_type: ty, + implementation: SimpleFunctionImpl::new(echo), + }, + ) + .unwrap(); + } + builder + .add_function( + "owned_names", + SimpleFunctionDefinition { + params: Box::new([SimpleFunctionParam { + arg_kind: SimpleFunctionArgKind::Field, + val_type: Type::Bytes, + }]), + opt_params: Box::new([]), + return_type: array_bytes, + implementation: SimpleFunctionImpl::new(owned_names), + }, + ) + .unwrap(); + builder + .add_list(Type::Int, crate::AlwaysList::default()) + .unwrap(); + builder.set_nil_not_equal_behavior(nil_not_equal); + let scheme = builder.build(); + NativeFields::validate_scheme(&scheme).unwrap(); + let lists = scheme + .lists() + .map(|list| list.definition().new_matcher()) + .collect(); + NativeContext { + scheme, + names: vec!["Accept".into(), "Other".into(), "".into()], + flags: vec![true, false, true], + other_flags: vec![false], + text: Some("Accept".into()), + other_text: Some("Other".into()), + num: Some(7), + other_num: Some(2), + ip: "192.0.2.1".parse().unwrap(), + headers: [ + ( + Box::from(b"accept".as_slice()), + vec!["text/html".into(), "application/json".into()], + ), + (Box::from(b"empty".as_slice()), Vec::new()), + ] + .into_iter() + .collect(), + required: Some("ok".into()), + lists, + list_accesses: std::cell::Cell::new(0), + } + } + + fn byte_array(values: &[Box]) -> crate::LhsValue<'_> { + crate::LhsValue::Array( + crate::Array::try_from_iter( + crate::Type::Bytes, + values.iter().map(|s| crate::LhsValue::from(s.as_ref())), + ) + .unwrap(), + ) + } + + fn stored_context(ctx: &NativeContext) -> ExecutionContext<'_> { + use crate::{LhsValue, Type}; + let mut stored = ExecutionContext::new(&ctx.scheme); + for (name, value) in [ + ("names", byte_array(&ctx.names)), + ( + "flags", + crate::Array::try_from_iter(Type::Bool, ctx.flags.iter().copied()) + .unwrap() + .into(), + ), + ( + "other_flags", + crate::Array::try_from_iter(Type::Bool, ctx.other_flags.iter().copied()) + .unwrap() + .into(), + ), + ("ip", LhsValue::Ip(ctx.ip)), + ( + "headers", + crate::Map::try_from_iter( + Type::array(Type::Bytes), + ctx.headers.iter().map(|(key, values)| { + Ok::<_, crate::TypeMismatchError>((key.clone(), byte_array(values))) + }), + ) + .unwrap() + .into(), + ), + ] { + stored.set_field_value_from_name(name, value).unwrap(); + } + for (name, value) in [ + ("text", &ctx.text), + ("other_text", &ctx.other_text), + ("required", &ctx.required), + ] { + if let Some(value) = value { + stored + .set_field_value_from_name(name, value.as_ref()) + .unwrap(); + } + } + for (name, value) in [("num", ctx.num), ("other_num", ctx.other_num)] { + if let Some(value) = value { + stored.set_field_value_from_name(name, value).unwrap(); + } + } + stored + } + + fn compare_filters(ctx: &NativeContext, expressions: &[&str]) { + let stored = stored_context(ctx); + let mut compiler = crate::DefaultCompiler::<(), NativeFields>::new(); + for expression in expressions { + let ast = ctx + .scheme + .parse(expression) + .unwrap_or_else(|error| panic!("{expression}: {error}")); + let expected = ast.clone().compile().execute(&stored).unwrap(); + let actual = ast + .compile_with_compiler(&mut compiler) + .execute(ctx) + .unwrap(); + assert_eq!(actual, expected, "{expression}"); + } + } + + #[test] + fn test_comparison_bindings_use_compiler_hooks_only_during_compilation() { + struct RecordingCompiler { + fields: Vec>, + family: std::marker::PhantomData, + } + + impl> crate::Compiler for RecordingCompiler { + type Family = F; + type U = (); + + fn bind_field>( + &mut self, + field: Field, + consumer: B, + ) -> Result { + self.fields.push(field.name().into()); + F::bind(field, consumer) + } + } + + let context = native_context(true); + let stored = stored_context(&context); + let mut native_compiler = RecordingCompiler:: { + fields: Vec::new(), + family: std::marker::PhantomData, + }; + let mut stored_compiler = RecordingCompiler::> { + fields: Vec::new(), + family: std::marker::PhantomData, + }; + + for (expression, field) in [ + (r#"names[0] == "Accept""#, "names"), + (r#"any(names[*] == "Accept")"#, "names"), + (r#"any(headers[*][0] == "text/html")"#, "headers"), + (r#"any(headers[*][*] == "application/json")"#, "headers"), + ] { + let ast = context.scheme.parse(expression).unwrap(); + let native_filter = ast.clone().compile_with_compiler(&mut native_compiler); + let stored_filter = ast.compile_with_compiler(&mut stored_compiler); + assert_eq!(native_compiler.fields.len(), 1); + assert_eq!(native_compiler.fields[0].as_ref(), field); + assert_eq!(stored_compiler.fields.len(), 1); + assert_eq!(stored_compiler.fields[0].as_ref(), field); + + for _ in 0..2 { + assert!(native_filter.execute(&context).unwrap(), "{expression}"); + assert!(stored_filter.execute(&stored).unwrap(), "{expression}"); + } + assert_eq!(native_compiler.fields.len(), 1); + assert_eq!(stored_compiler.fields.len(), 1); + native_compiler.fields.clear(); + stored_compiler.fields.clear(); + } + } + + #[test] + fn test_native_complete_operator_and_collection_paths() { + let ctx = native_context(true); + compare_filters( + &ctx, + &[ + "num == 7", + "num != 7", + "num < 8", + "num <= 7", + "num > 7", + "num >= 7", + "num & 2", + "num & other_num", + "num in {1 5..9}", + "num == other_num", + r#"text == "Accept""#, + r#"text != "Accept""#, + r#"text < "Other""#, + r#"text contains "cep""#, + r#"text contains """#, + r#"text contains "a string longer than the entire field""#, + r#"text matches "^A.*t$""#, + r#"text wildcard "a*""#, + r#"text strict wildcard "a*""#, + r#"text in {"Other" "Accept"}"#, + "text == other_text", + "text > other_text", + "ip == 192.0.2.1", + "ip < 192.0.2.2", + "ip < ::1", + "ip in {192.0.2.0/24}", + "any(flags)", + "all(flags)", + "any(not flags)", + "any((flags and other_flags))", + "any((flags or other_flags))", + "all((flags xor other_flags))", + r#"any(names[*] == "Accept")"#, + r#"all(names[*] != "absent")"#, + r#"names[0] == "Accept""#, + r#"names[20] != "absent""#, + r#"any(headers[*][*] == "text/html")"#, + r#"headers["accept"][1] == "application/json""#, + r#"headers["ACCEPT"][0] == "text/html""#, + r#"all(headers["empty"][*] == "x")"#, + r#"all(headers["missing"][*] == "x")"#, + r#"any(echo(names[*])[*] == "Accept")"#, + r#"any(echo(headers[*][*])[*] == "text/html")"#, + r#"echo_names(names)[0] == "Accept""#, + r#"echo_headers(headers)["accept"][0] == "text/html""#, + r#"any(echo_headers(headers)[*][*] == "text/html")"#, + r#"owned_names(text)[1] == "Other""#, + r#"any(owned_names(text)[*] == "Accept")"#, + "num in $named_list", + ], + ); + assert!(ctx.list_accesses.get() > 0); + } + + #[test] + fn test_native_and_stored_byte_literal_formats() { + let ctx = native_context(true); + let stored = stored_context(&ctx); + for literal in [r#""Accept""#, r###"r##"Accept"##"###, "41:63:63:65:70:74"] { + for (operator, expected) in [ + ("==", true), + ("!=", false), + ("<=", true), + (">=", true), + ("<", false), + (">", false), + ("contains", true), + ] { + let expression = format!("text {operator} {literal}"); + let ast = ctx.scheme.parse(&expression).unwrap(); + let native = ast + .clone() + .compile_with_compiler(&mut crate::DefaultCompiler::<(), NativeFields>::new()); + assert_eq!(native.execute(&ctx).unwrap(), expected, "{expression}"); + assert_eq!( + ast.compile().execute(&stored).unwrap(), + expected, + "{expression}" + ); + } + } + } + + #[cfg(feature = "regex")] + #[test] + fn test_native_and_stored_regex_literal_formats() { + let ctx = native_context(true); + let stored = stored_context(&ctx); + for literal in [r#""^A.*t$""#, r###"r##"^A.*t$"##"###] { + for (lhs, expected) in [("text", true), ("other_text", false)] { + let expression = format!("{lhs} matches {literal}"); + let ast = ctx.scheme.parse(&expression).unwrap(); + let native = ast + .clone() + .compile_with_compiler(&mut crate::DefaultCompiler::<(), NativeFields>::new()); + assert_eq!(native.execute(&ctx).unwrap(), expected, "{expression}"); + assert_eq!( + ast.compile().execute(&stored).unwrap(), + expected, + "{expression}" + ); + } + } + } + + #[test] + fn test_native_and_stored_ip_literal_ordering() { + let mut ctx = native_context(true); + for (literal, values) in [ + ( + "192.0.2.2", + ["192.0.2.1", "192.0.2.2", "192.0.2.3", "2001:db8::2"], + ), + ( + "2001:db8::2", + ["2001:db8::1", "2001:db8::2", "2001:db8::3", "192.0.2.2"], + ), + ] { + for (operator, expected) in [ + ("==", [false, true, false, false]), + ("!=", [true, false, true, true]), + ("<", [true, false, false, false]), + ("<=", [true, true, false, false]), + (">", [false, false, true, false]), + (">=", [false, true, true, false]), + ] { + let expression = format!("ip {operator} {literal}"); + let ast = ctx.scheme.parse(&expression).unwrap(); + let native = ast + .clone() + .compile_with_compiler(&mut crate::DefaultCompiler::<(), NativeFields>::new()); + let stored = ast.compile(); + for (value, expected) in values.into_iter().zip(expected) { + ctx.ip = value.parse().unwrap(); + assert_eq!( + native.execute(&ctx).unwrap(), + expected, + "{value}: {expression}" + ); + assert_eq!( + stored.execute(&stored_context(&ctx)).unwrap(), + expected, + "{value}: {expression}" + ); + } + } + } + } + + #[test] + fn test_native_general_values_match_stored_values() { + let ctx = native_context(true); + let stored = stored_context(&ctx); + let mut compiler = crate::DefaultCompiler::<(), NativeFields>::new(); + for expression in [ + "text", + "names", + "headers", + "flags", + "num", + "ip", + "echo_names(names)", + "echo_headers(headers)", + "echo(names[*])", + "owned_names(text)", + "any((flags or other_flags))", + r#"headers["accept"]"#, + ] { + let ast = ctx + .scheme + .parse_value(expression) + .unwrap_or_else(|error| panic!("{expression}: {error}")); + let expected = ast.clone().compile(); + let actual = ast.compile_with_compiler(&mut compiler); + assert_eq!( + actual.execute(&ctx).unwrap(), + expected.execute(&stored).unwrap(), + "{expression}" + ); + } + } + + #[test] + fn test_native_dynamic_missing_values_follow_nil_policy() { + for nil_not_equal in [false, true] { + for (left, right) in [(true, true), (false, true), (true, false), (false, false)] { + let mut ctx = native_context(nil_not_equal); + if !left { + ctx.text = None; + ctx.num = None; + } + if !right { + ctx.other_text = None; + ctx.other_num = None; + } + compare_filters( + &ctx, + &[ + "text == other_text", + "text != other_text", + "text < other_text", + "text >= other_text", + "num == other_num", + "num != other_num", + "num < other_num", + "num >= other_num", + "num & other_num", + r#"text != "Accept""#, + "any(names[*] == other_text)", + "any(names[*] != other_text)", + r#"names[50] != "absent""#, + r#"any(echo(names[*])[*] == other_text)"#, + ], + ); + } + } + } + + #[test] + fn test_native_required_missing_and_scheme_identity() { + // Handwritten getters may violate mandatory-field presence even though + // the field macro rejects optional getters during scheme validation. + // Retain coverage for the execution-time invariant in that case. + struct MissingRequiredFields; + struct RequiredGetter; + + impl ContextFamily for MissingRequiredFields { + type Context<'data> = NativeContext; + type UserData = (); + + fn bind>( + field: Field, + consumer: B, + ) -> Result { + assert_eq!(field.name(), "required"); + consumer.consume(field, RequiredGetter) + } + } + + impl super::FieldGetter for RequiredGetter { + type Provider = super::BorrowedValueProvider; + + fn get<'ctx, 'data: 'ctx>( + &self, + context: &'ctx NativeContext, + field: &Field, + ) -> Option<&'ctx str> { + assert_eq!( + field.as_ref(), + context.scheme.get_field("required").unwrap() + ); + context.required.as_deref() + } + } + + let mut ctx = native_context(true); + let ast = ctx.scheme.parse(r#"required == "ok""#).unwrap(); + let stored_filter = ast.clone().compile(); + let native_filter = ast + .compile_with_compiler(&mut crate::DefaultCompiler::<(), MissingRequiredFields>::new()); + let different = native_context(true); + assert_eq!( + native_filter.execute(&different), + Err(crate::SchemeMismatchError) + ); + ctx.required = None; + let stored = stored_context(&ctx); + assert!( + std::panic::catch_unwind(std::panic::AssertUnwindSafe( + || stored_filter.execute(&stored) + )) + .is_err() + ); + let panic = + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| native_filter.execute(&ctx))) + .unwrap_err(); + assert_eq!( + panic.downcast_ref::().map(String::as_str), + Some("Field required was registered as mandatory but not given a value") + ); + compare_filters(&ctx, &[r#"num == 7 or required == "ok""#]); + } + + #[test] + fn test_native_value_borrows_invariant_context_for_one_access() { + use std::cell::Cell; + + struct InvariantContext<'data> { + scheme: &'data Scheme, + value: Cell<&'data str>, + } + + impl super::HasScheme for InvariantContext<'_> { + fn scheme(&self) -> &Scheme { + self.scheme + } + } + + impl super::EvaluationContext for InvariantContext<'_> { + type UserData = (); + + fn get_user_data(&self) -> &() { + &() + } + + fn get_list_matcher(&self, _: crate::ListRef<'_>) -> &dyn crate::ListMatcher { + unreachable!("the test scheme has no lists") + } + } + + crate::wirefilter_fields! { + impl<'data> InvariantFields for InvariantContext<'data> { + "value" => { + fn(ctx: &InvariantContext<'data>) -> &str { + ctx.value.get() + } + }, + } + } + + let scheme = Scheme! { value: Bytes }.build(); + InvariantFields::validate_scheme(&scheme).unwrap(); + let filter = scheme + .parse_value("value") + .unwrap() + .compile_with_compiler(&mut crate::DefaultCompiler::<(), InvariantFields>::new()); + let first = Box::::from("first"); + let second = Box::::from("second"); + let mut context = InvariantContext { + scheme: &scheme, + value: Cell::new(&first), + }; + let result = filter.execute(&context).unwrap().unwrap(); + let crate::LhsValue::Bytes(bytes) = result else { + panic!("expected bytes") + }; + assert_eq!(bytes.as_ref().as_ptr(), first.as_ptr()); + *context.value.get_mut() = &second; + assert_eq!( + filter.execute(&context).unwrap().unwrap(), + crate::LhsValue::from(second.as_ref()) + ); + assert_eq!( + std::mem::size_of::>(), + std::mem::size_of::<&InvariantContext<'_>>(), + ); + } + + #[test] + fn test_native_map_order_matches_materialized_identity_function() { + let mut context = native_context(true); + context + .headers + .insert(b"z-after".as_slice().into(), vec!["text/html".into()]); + context + .headers + .insert(b"a-before".as_slice().into(), vec!["other".into()]); + let expression = + r#"all((headers[*][*] == "text/html" xor echo_headers(headers)[*][*] == "text/html"))"#; + let stored = stored_context(&context); + let ast = context.scheme.parse(expression).unwrap(); + let stored_filter = ast.clone().compile(); + let native_filter = + ast.compile_with_compiler(&mut crate::DefaultCompiler::<(), NativeFields>::new()); + assert!(!stored_filter.execute(&stored).unwrap()); + assert!(!native_filter.execute(&context).unwrap()); + } +} diff --git a/engine/src/context/state.rs b/engine/src/context/state.rs new file mode 100644 index 00000000..0ff2c499 --- /dev/null +++ b/engine/src/context/state.rs @@ -0,0 +1,457 @@ +//! Persistent executor state independent of field storage. + +use crate::{ + EvaluationContext, EvaluationContextMut, HasScheme, List, ListMatcher, ListRef, Scheme, Type, +}; + +/// Scheme, list matchers, and compiler state retained across evaluations. +/// +/// This type has no field-value slots. The embedding runtime supplies +/// fields through its context family and can borrow this state while evaluating +/// them. Neither the state nor its list matchers must be rebuilt for each phase. +/// +/// This is an optional storage implementation of [`HasScheme`], +/// [`EvaluationContext`], and [`EvaluationContextMut`]. An application context can +/// borrow it and forward those methods while its [`crate::ContextFamily`] supplies +/// getters for application-owned field values. Applications can instead implement +/// the same interfaces using their existing state. See [`crate::context`] for a +/// complete example. +/// +/// Forward [`EvaluationContext::get_list_matcher_unchecked`] as well as the +/// checked accessor so list comparisons do not repeat the scheme check already +/// performed by the filter. +/// +/// [`crate::ExecutionContext`] already owns equivalent state together with its +/// stored field values and does not need a separate `ContextState`. The +/// application is responsible for resetting state when reusing it; [`Self::clear`] +/// clears list contents but retains user data. +#[derive(Debug, PartialEq)] +pub struct ContextState { + scheme: Scheme, + list_matchers: Box<[Box]>, + user_data: U, +} + +impl ContextState { + /// Create persistent state with default user data and the scheme's matchers. + pub fn new(scheme: &Scheme) -> Self + where + U: Default, + { + Self::new_with(scheme, U::default) + } + + /// Create persistent state with the supplied user-data initializer. + pub fn new_with(scheme: &Scheme, user_data: impl FnOnce() -> U) -> Self { + Self { + scheme: scheme.clone(), + list_matchers: scheme + .lists() + .map(|list| list.definition().new_matcher()) + .collect(), + user_data: user_data(), + } + } + + /// Scheme associated with this state. + #[inline] + pub fn scheme(&self) -> &Scheme { + &self.scheme + } + + /// Compiler-specific execution state. + #[inline] + pub fn get_user_data(&self) -> &U { + &self.user_data + } + + /// Mutable compiler-specific execution state. + #[inline] + pub fn get_user_data_mut(&mut self) -> &mut U { + &mut self.user_data + } + + /// Matcher registered for the supplied list. + #[inline] + pub fn get_list_matcher(&self, list: ListRef<'_>) -> &dyn ListMatcher { + assert!(self.scheme() == list.scheme()); + &*self.list_matchers[list.index()] + } + + /// Matcher registered for the supplied type, if any. + #[inline] + pub fn get_list_matcher_from_type(&self, ty: &Type) -> Option<&dyn ListMatcher> { + let list = self.scheme.get_list(ty)?; + Some(&*self.list_matchers[list.index()]) + } + + /// Mutable matcher registered for the supplied list. + #[inline] + pub fn get_list_matcher_mut(&mut self, list: ListRef<'_>) -> &mut dyn ListMatcher { + assert!(self.scheme() == list.scheme()); + &mut *self.list_matchers[list.index()] + } + + /// Mutable matcher registered for the supplied type, if any. + #[inline] + pub fn get_list_matcher_mut_from_type(&mut self, ty: &Type) -> Option<&mut dyn ListMatcher> { + let list = self.scheme.get_list(ty)?; + Some(&mut *self.list_matchers[list.index()]) + } + + /// Clear list contents while retaining their storage and user data. + /// + /// As with [`crate::ExecutionContext::clear`], compiler-specific user-data + /// cleanup remains the embedding runtime's responsibility. + pub fn clear(&mut self) { + self.list_matchers + .iter_mut() + .for_each(|matcher| matcher.clear()); + } +} + +impl HasScheme for ContextState { + #[inline] + fn scheme(&self) -> &Scheme { + self.scheme() + } +} + +impl EvaluationContext for ContextState { + type UserData = U; + + #[inline] + fn get_user_data(&self) -> &U { + self.get_user_data() + } + + #[inline] + fn get_list_matcher(&self, list: ListRef<'_>) -> &dyn ListMatcher { + self.get_list_matcher(list) + } + + #[inline] + fn get_list_matcher_from_type(&self, ty: &Type) -> Option<&dyn ListMatcher> { + self.get_list_matcher_from_type(ty) + } + + #[inline] + fn get_list_matcher_unchecked(&self, list: &List) -> &dyn ListMatcher { + debug_assert!(self.scheme() == list.scheme()); + &*self.list_matchers[list.index()] + } +} + +impl EvaluationContextMut for ContextState { + #[inline] + fn get_user_data_mut(&mut self) -> &mut U { + self.get_user_data_mut() + } + + #[inline] + fn get_list_matcher_mut(&mut self, list: ListRef<'_>) -> &mut dyn ListMatcher { + self.get_list_matcher_mut(list) + } + + #[inline] + fn get_list_matcher_mut_from_type(&mut self, ty: &Type) -> Option<&mut dyn ListMatcher> { + self.get_list_matcher_mut_from_type(ty) + } +} + +#[cfg(test)] +mod tests { + use super::ContextState; + use crate::{ + DefaultCompiler, EvaluationContext, EvaluationContextMut, HasScheme, LhsValue, List, + ListDefinition, ListMatcher, ListRef, Scheme, SchemeBuilder, Type, + }; + use serde::{Deserialize, Serialize}; + use std::collections::BTreeSet; + + #[derive(Debug)] + struct ValuesList; + + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)] + struct ValuesMatcher { + values: BTreeSet, + clears: usize, + } + + impl ListDefinition for ValuesList { + fn deserialize_matcher<'de>( + &self, + _: Type, + deserializer: &mut dyn erased_serde::Deserializer<'de>, + ) -> Result, erased_serde::Error> { + erased_serde::deserialize::(deserializer) + .map(|matcher| Box::new(matcher) as Box) + } + + fn new_matcher(&self) -> Box { + Box::new(ValuesMatcher::default()) + } + } + + impl ListMatcher for ValuesMatcher { + fn match_value(&self, name: &str, value: &LhsValue<'_>) -> bool { + name == "numbers" + && matches!(value, LhsValue::Int(value) if self.values.contains(value)) + } + + fn clear(&mut self) { + self.values.clear(); + self.clears += 1; + } + } + + fn scheme() -> Scheme { + let mut builder = SchemeBuilder::new(); + builder.add_field("name", Type::Bytes).unwrap(); + builder + .add_field("numbers", Type::array(Type::Int)) + .unwrap(); + builder.add_list(Type::Int, ValuesList).unwrap(); + builder.build() + } + + fn insert(context: &mut impl EvaluationContextMut, value: i64) { + context + .get_list_matcher_mut_from_type(&Type::Int) + .unwrap() + .as_any_mut() + .downcast_mut::() + .unwrap() + .values + .insert(value); + *context.get_user_data_mut() += 1; + } + + struct StateAdapter<'a>(&'a mut ContextState); + + impl HasScheme for StateAdapter<'_> { + fn scheme(&self) -> &Scheme { + self.0.scheme() + } + } + + impl EvaluationContext for StateAdapter<'_> { + type UserData = usize; + + fn get_user_data(&self) -> &usize { + self.0.get_user_data() + } + + fn get_list_matcher(&self, list: ListRef<'_>) -> &dyn ListMatcher { + self.0.get_list_matcher(list) + } + + fn get_list_matcher_unchecked(&self, list: &List) -> &dyn ListMatcher { + self.0.get_list_matcher_unchecked(list) + } + } + + impl EvaluationContextMut for StateAdapter<'_> { + fn get_user_data_mut(&mut self) -> &mut usize { + self.0.get_user_data_mut() + } + + fn get_list_matcher_mut(&mut self, list: ListRef<'_>) -> &mut dyn ListMatcher { + self.0.get_list_matcher_mut(list) + } + } + + #[test] + fn test_list_state_reuse_and_generic_mutation() { + let scheme = scheme(); + let mut state = ContextState::new_with(&scheme, || 7usize); + assert!(state.get_list_matcher_from_type(&Type::Bytes).is_none()); + assert!(state.get_list_matcher_mut_from_type(&Type::Bytes).is_none()); + insert(&mut StateAdapter(&mut state), 41); + assert_eq!(*state.get_user_data(), 8); + let matcher = state + .get_list_matcher(scheme.get_list(&Type::Int).unwrap()) + .as_any() + .downcast_ref::() + .unwrap(); + assert!(matcher.match_value("numbers", &LhsValue::Int(41))); + let address = matcher as *const ValuesMatcher; + state.clear(); + let matcher = state + .get_list_matcher_from_type(&Type::Int) + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(matcher as *const ValuesMatcher, address); + assert!(!matcher.match_value("numbers", &LhsValue::Int(41))); + assert_eq!(matcher.clears, 1); + assert_eq!(*state.get_user_data(), 8); + let adapter = StateAdapter(&mut state); + assert!(EvaluationContext::get_list_matcher_from_type(&adapter, &Type::Int).is_some()); + + let mut stored = crate::ExecutionContext::new_with(&scheme, || 7usize); + insert(&mut stored, 41); + assert_eq!(*stored.get_user_data(), 8); + assert!( + stored + .get_list_matcher_from_type(&Type::Int) + .unwrap() + .match_value("numbers", &LhsValue::Int(41)) + ); + } + + #[test] + fn test_explicit_stored_snapshot_preserves_lists_without_populating_fields() { + let scheme = scheme(); + let mut state = ContextState::new_with(&scheme, || 0usize); + insert(&mut state, 41); + let snapshot = crate::ExecutionContext::new_from_context(&state, 9usize); + assert!( + snapshot + .get_field_value(scheme.get_field("name").unwrap()) + .is_none() + ); + assert!( + snapshot + .get_field_value(scheme.get_field("numbers").unwrap()) + .is_none() + ); + state.clear(); + assert!( + !state + .get_list_matcher_from_type(&Type::Int) + .unwrap() + .match_value("numbers", &LhsValue::Int(41)) + ); + assert!( + snapshot + .get_list_matcher_from_type(&Type::Int) + .unwrap() + .match_value("numbers", &LhsValue::Int(41)) + ); + assert_eq!(*snapshot.get_user_data(), 9); + assert_eq!(*state.get_user_data(), 1); + } + + #[test] + fn test_list_access_checks_scheme_identity() { + let scheme = scheme(); + let other = self::scheme(); + let mut state = ContextState::<()>::new(&scheme); + let foreign = other.get_list(&Type::Int).unwrap(); + assert!( + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + state.get_list_matcher(foreign); + })) + .is_err() + ); + assert!( + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + state.get_list_matcher_mut(foreign); + })) + .is_err() + ); + } + + struct Request { + name: Box, + numbers: Vec, + } + + struct NativeContext<'a> { + request: &'a Request, + state: &'a ContextState, + } + + impl HasScheme for NativeContext<'_> { + fn scheme(&self) -> &Scheme { + self.state.scheme() + } + } + + impl EvaluationContext for NativeContext<'_> { + type UserData = usize; + + fn get_user_data(&self) -> &usize { + self.state.get_user_data() + } + + fn get_list_matcher(&self, list: ListRef<'_>) -> &dyn ListMatcher { + self.state.get_list_matcher(list) + } + + fn get_list_matcher_unchecked(&self, list: &List) -> &dyn ListMatcher { + self.state.get_list_matcher_unchecked(list) + } + } + + crate::wirefilter_fields! { + impl<'data> NativeFields for NativeContext<'data> { + "name" => { + fn(context: &NativeContext<'data>) -> &str { + &context.request.name + } + }, + "numbers" => { + fn(context: &NativeContext<'data>) -> &Vec { + &context.request.numbers + } + }, + } + } + + #[test] + fn test_owned_state_can_move_to_worker_and_borrow_native_fields_there() { + let scheme = scheme(); + NativeFields::validate_scheme(&scheme).unwrap(); + let mut compiler = DefaultCompiler::::new(); + let filter = scheme + .parse(r#"name == "first" and any(numbers[*] in $numbers)"#) + .unwrap() + .compile_with_compiler(&mut compiler); + let value = scheme + .parse_value("name") + .unwrap() + .compile_with_compiler(&mut compiler); + let mut state = ContextState::new_with(&scheme, || 0usize); + insert(&mut state, 41); + let request = Request { + name: "first".into(), + numbers: vec![40, 41], + }; + let caller = std::thread::current().id(); + let (mut request, state, filter) = std::thread::spawn(move || { + assert_ne!(std::thread::current().id(), caller); + let context = NativeContext { + request: &request, + state: &state, + }; + assert!(filter.execute(&context).unwrap()); + let LhsValue::Bytes(bytes) = value.execute(&context).unwrap().unwrap() else { + panic!("expected bytes"); + }; + assert_eq!(bytes.as_ref().as_ptr(), request.name.as_ptr()); + assert_eq!(*context.get_user_data(), 1); + (request, state, filter) + }) + .join() + .unwrap(); + request.name = "second".into(); + assert!( + !filter + .execute(&NativeContext { + request: &request, + state: &state, + }) + .unwrap() + ); + assert_eq!(*state.get_user_data(), 1); + assert!( + state + .get_list_matcher_from_type(&Type::Int) + .unwrap() + .match_value("numbers", &LhsValue::Int(41)) + ); + } +} diff --git a/engine/src/execution_context.rs b/engine/src/execution_context.rs index fdb9eb05..c03dcbb5 100644 --- a/engine/src/execution_context.rs +++ b/engine/src/execution_context.rs @@ -1,5 +1,6 @@ +use crate::lhs_types::{LhsValue, LhsValueSeed}; use crate::scheme::{Field, List, Scheme, SchemeMismatchError}; -use crate::types::{GetType, LhsValue, LhsValueSeed, Type, TypeMismatchError}; +use crate::types::{GetType, Type, TypeMismatchError}; use crate::{FieldRef, ListMatcher, ListRef, UnknownFieldError}; use serde::Serialize; use serde::de::{self, DeserializeSeed, Deserializer, MapAccess, SeqAccess, Visitor}; @@ -38,6 +39,17 @@ pub struct InvalidListMatcherError { /// /// It acts as a map in terms of public API, but provides a constant-time /// index-based access to values for a filter during execution. +/// +/// The default [`crate::ExecutionContextFamily`] binds its fields using +/// [`crate::LhsValueProvider`]. It borrows existing values through the same +/// reader interface used for native Rust fields, without converting the stored +/// containers. Setters validate values against the field types in the scheme. +/// +/// This type implements [`crate::EvaluationContext`] and +/// [`crate::EvaluationContextMut`], owning its scheme, list matchers, and user data +/// alongside the field values. Custom contexts can use [`crate::ContextState`] +/// to own equivalent state without field-value slots. See [`crate::context`] for +/// the context interfaces and an example. #[derive(Debug, PartialEq)] pub struct ExecutionContext<'e, U = ()> { scheme: Scheme, @@ -46,6 +58,54 @@ pub struct ExecutionContext<'e, U = ()> { user_data: U, } +impl crate::HasScheme for ExecutionContext<'_, U> { + #[inline] + fn scheme(&self) -> &Scheme { + self.scheme() + } +} + +impl crate::EvaluationContext for ExecutionContext<'_, U> { + type UserData = U; + + #[inline] + fn get_user_data(&self) -> &U { + self.get_user_data() + } + + #[inline] + fn get_list_matcher(&self, list: ListRef<'_>) -> &dyn ListMatcher { + self.get_list_matcher(list) + } + + #[inline] + fn get_list_matcher_from_type(&self, ty: &Type) -> Option<&dyn ListMatcher> { + self.get_list_matcher_from_type(ty) + } + + #[inline] + fn get_list_matcher_unchecked(&self, list: &List) -> &dyn ListMatcher { + self.get_list_matcher_unchecked(list) + } +} + +impl crate::EvaluationContextMut for ExecutionContext<'_, U> { + #[inline] + fn get_user_data_mut(&mut self) -> &mut U { + self.get_user_data_mut() + } + + #[inline] + fn get_list_matcher_mut(&mut self, list: ListRef<'_>) -> &mut dyn ListMatcher { + self.get_list_matcher_mut(list) + } + + #[inline] + fn get_list_matcher_mut_from_type(&mut self, ty: &Type) -> Option<&mut dyn ListMatcher> { + self.get_list_matcher_mut_from_type(ty) + } +} + impl<'e, U> ExecutionContext<'e, U> { /// Creates an execution context associated with a given scheme. /// @@ -72,6 +132,25 @@ impl<'e, U> ExecutionContext<'e, U> { } } + /// Create an empty stored-value snapshot with a context's scheme and lists. + /// + /// Applications can use this explicitly for consumers that require stored + /// values, such as diagnostic serialization. Field getters are not evaluated; + /// the caller chooses which fields to materialize. List matchers are cloned + /// independently so later changes to either context do not affect the other. + pub fn new_from_context(context: &impl crate::EvaluationContext, user_data: U) -> Self { + let scheme = context.scheme(); + Self { + scheme: scheme.clone(), + values: vec![None; scheme.field_count()].into(), + list_matchers: scheme + .lists() + .map(|list| dyn_clone::clone_box(context.get_list_matcher(list))) + .collect(), + user_data, + } + } + /// Returns the associated scheme. pub fn scheme(&self) -> &Scheme { &self.scheme diff --git a/engine/src/filter.rs b/engine/src/filter.rs index 510672cf..849cac75 100644 --- a/engine/src/filter.rs +++ b/engine/src/filter.rs @@ -5,19 +5,27 @@ //! their `execute` methods and aggregating results into a single boolean value //! as recursion unwinds. -use crate::execution_context::ExecutionContext; -use crate::lhs_types::TypedArray; +use crate::context::{ContextFamily, ContextRef, ExecutionContextFamily, HasScheme}; +use crate::lhs_types::{LhsValue, TypedArray}; use crate::scheme::{Scheme, SchemeMismatchError}; -use crate::types::{LhsValue, Type}; +use crate::types::Type; use std::fmt; +use std::marker::PhantomData; -type BoxedClosureToOneBool = - Box Fn(&'e ExecutionContext<'e, U>) -> bool + Sync + Send + 'static>; +type BoxedClosureToOneBool = Box< + dyn for<'ctx, 'data> Fn(&'ctx ::Context<'data>) -> bool + + Sync + + Send + + 'static, +>; /// Boxed closure for [`crate::Expr`] AST node that evaluates to a simple [`bool`]. -pub struct CompiledOneExpr(BoxedClosureToOneBool); +pub struct CompiledOneExpr = ExecutionContextFamily>( + BoxedClosureToOneBool, + PhantomData U>, +); -impl fmt::Debug for CompiledOneExpr { +impl> fmt::Debug for CompiledOneExpr { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { fmt.debug_tuple("CompiledOneExpr") .field(&((&*self.0) as *const _)) @@ -25,35 +33,44 @@ impl fmt::Debug for CompiledOneExpr { } } -impl CompiledOneExpr { +impl> CompiledOneExpr { /// Creates a compiled expression IR from a generic closure. pub fn new( - closure: impl for<'e> Fn(&'e ExecutionContext<'e, U>) -> bool + Sync + Send + 'static, + closure: impl for<'ctx, 'data> Fn(&'ctx ::Context<'data>) -> bool + + Sync + + Send + + 'static, ) -> Self { - CompiledOneExpr(Box::new(closure)) + CompiledOneExpr(Box::new(closure), PhantomData) } /// Executes the closure against a provided context with values. - pub fn execute<'e>(&self, ctx: &'e ExecutionContext<'e, U>) -> bool { + pub fn execute<'ctx, 'data: 'ctx>(&self, ctx: &'ctx F::Context<'data>) -> bool { self.0(ctx) } /// Extracts the underlying boxed closure. - pub fn into_boxed_closure(self) -> BoxedClosureToOneBool { + pub fn into_boxed_closure(self) -> BoxedClosureToOneBool { self.0 } } pub(crate) type CompiledVecExprResult = TypedArray<'static, bool>; -type BoxedClosureToVecBool = Box< - dyn for<'e> Fn(&'e ExecutionContext<'e, U>) -> CompiledVecExprResult + Sync + Send + 'static, +type BoxedClosureToVecBool = Box< + dyn for<'ctx, 'data> Fn(&'ctx ::Context<'data>) -> CompiledVecExprResult + + Sync + + Send + + 'static, >; /// Boxed closure for [`crate::Expr`] AST node that evaluates to a list of [`bool`]. -pub struct CompiledVecExpr(BoxedClosureToVecBool); +pub struct CompiledVecExpr = ExecutionContextFamily>( + BoxedClosureToVecBool, + PhantomData U>, +); -impl fmt::Debug for CompiledVecExpr { +impl> fmt::Debug for CompiledVecExpr { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { fmt.debug_tuple("CompiledVecExpr") .field(&((&*self.0) as *const _)) @@ -61,40 +78,53 @@ impl fmt::Debug for CompiledVecExpr { } } -impl CompiledVecExpr { +impl> CompiledVecExpr { /// Creates a compiled expression IR from a generic closure. pub fn new( - closure: impl for<'e> Fn(&'e ExecutionContext<'e, U>) -> CompiledVecExprResult + closure: impl for<'ctx, 'data> Fn( + &'ctx ::Context<'data>, + ) -> CompiledVecExprResult + Sync + Send + 'static, ) -> Self { - CompiledVecExpr(Box::new(closure)) + CompiledVecExpr(Box::new(closure), PhantomData) } /// Executes the closure against a provided context with values. - pub fn execute<'e>(&self, ctx: &'e ExecutionContext<'e, U>) -> CompiledVecExprResult { + pub fn execute<'ctx, 'data: 'ctx>( + &self, + ctx: &'ctx F::Context<'data>, + ) -> CompiledVecExprResult { self.0(ctx) } /// Extracts the underlying boxed closure. - pub fn into_boxed_closure(self) -> BoxedClosureToVecBool { + pub fn into_boxed_closure(self) -> BoxedClosureToVecBool { self.0 } } /// Enum of boxed closure for [`crate::Expr`] AST nodes. -#[derive(Debug)] -pub enum CompiledExpr { +pub enum CompiledExpr = ExecutionContextFamily> { /// Variant for [`crate::Expr`] AST node that evaluates to a simple [`bool`]. - One(CompiledOneExpr), + One(CompiledOneExpr), /// Variant for [`crate::Expr`] AST node that evaluates to a list of [`bool`]. - Vec(CompiledVecExpr), + Vec(CompiledVecExpr), } -impl CompiledExpr { +impl> fmt::Debug for CompiledExpr { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::One(expr) => fmt.debug_tuple("One").field(expr).finish(), + Self::Vec(expr) => fmt.debug_tuple("Vec").field(expr).finish(), + } + } +} + +impl> CompiledExpr { #[cfg(test)] - pub(crate) fn execute_one<'e>(&self, ctx: &'e ExecutionContext<'e, U>) -> bool { + pub(crate) fn execute_one<'ctx, 'data: 'ctx>(&self, ctx: &'ctx F::Context<'data>) -> bool { match self { CompiledExpr::One(one) => one.execute(ctx), CompiledExpr::Vec(_) => unreachable!(), @@ -102,9 +132,9 @@ impl CompiledExpr { } #[cfg(test)] - pub(crate) fn execute_vec<'e>( + pub(crate) fn execute_vec<'ctx, 'data: 'ctx>( &self, - ctx: &'e ExecutionContext<'e, U>, + ctx: &'ctx F::Context<'data>, ) -> CompiledVecExprResult { match self { CompiledExpr::One(_) => unreachable!(), @@ -127,14 +157,24 @@ impl From for CompiledValueResult<'_> { } } -type BoxedClosureToValue = Box< - dyn for<'e> Fn(&'e ExecutionContext<'e, U>) -> CompiledValueResult<'e> + Sync + Send + 'static, +type BoxedClosureToValue = Box< + dyn for<'ctx, 'data> Fn(ContextRef<'ctx, 'data, F>) -> CompiledValueResult<'ctx> + + Sync + + Send + + 'static, +>; + +type BoxedStoredClosureToValue = Box< + dyn for<'e> Fn(&'e crate::ExecutionContext<'e, U>) -> CompiledValueResult<'e> + Sync + Send, >; /// Boxed closure for [`crate::ValueExpr`] AST node that evaluates to an [`LhsValue`]. -pub struct CompiledValueExpr(BoxedClosureToValue); +pub struct CompiledValueExpr = ExecutionContextFamily>( + BoxedClosureToValue, + PhantomData U>, +); -impl fmt::Debug for CompiledValueExpr { +impl> fmt::Debug for CompiledValueExpr { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { fmt.debug_tuple("CompiledValueExpr") .field(&((&*self.0) as *const _)) @@ -142,28 +182,57 @@ impl fmt::Debug for CompiledValueExpr { } } -impl CompiledValueExpr { +impl> CompiledValueExpr { /// Creates a compiled expression IR from a generic closure. pub fn new( - closure: impl for<'e> Fn(&'e ExecutionContext<'e, U>) -> CompiledValueResult<'e> + closure: impl for<'ctx, 'data> Fn( + &'ctx ::Context<'data>, + ) -> CompiledValueResult<'ctx> + + Sync + + Send + + 'static, + ) -> Self { + Self::new_with_context(move |ctx| closure(ctx.context())) + } + + /// Create a value expression that can borrow context payloads. + /// + /// The callback's witness records that the context's retained data outlives + /// this evaluation. Call [`ContextRef::context`] to obtain the original + /// context reference before invoking field getters or child expressions. + pub fn new_with_context( + closure: impl for<'ctx, 'data> Fn(ContextRef<'ctx, 'data, F>) -> CompiledValueResult<'ctx> + Sync + Send + 'static, ) -> Self { - CompiledValueExpr(Box::new(closure)) + CompiledValueExpr(Box::new(closure), PhantomData) } /// Executes the closure against a provided context with values. - pub fn execute<'e>(&self, ctx: &'e ExecutionContext<'e, U>) -> CompiledValueResult<'e> { - self.0(ctx) + pub fn execute<'ctx, 'data: 'ctx>( + &self, + ctx: &'ctx F::Context<'data>, + ) -> CompiledValueResult<'ctx> { + self.0(ContextRef::new(ctx)) } - /// Extracts the underlying boxed closure. - pub fn into_boxed_closure(self) -> BoxedClosureToValue { + /// Extract the callback, retaining its context-lifetime witness. + pub fn into_boxed_context_closure(self) -> BoxedClosureToValue { self.0 } } +impl CompiledValueExpr { + /// Convert to a callback accepting the ordinary stored execution context. + /// + /// This compatibility adapter adds one boxed closure around the expression. + /// [`Self::into_boxed_context_closure`] extracts the original callback directly. + pub fn into_boxed_closure(self) -> BoxedStoredClosureToValue { + Box::new(move |ctx| self.execute(ctx)) + } +} + /// An IR for a compiled filter expression. /// /// Currently it works by creating and combining boxed untyped closures and @@ -179,12 +248,12 @@ impl CompiledValueExpr { /// In the future the underlying representation might change, but for now it /// provides the best trade-off between safety and performance of compilation /// and execution. -pub struct Filter { - root_expr: CompiledOneExpr, +pub struct Filter = ExecutionContextFamily> { + root_expr: CompiledOneExpr, scheme: Scheme, } -impl std::fmt::Debug for Filter { +impl> std::fmt::Debug for Filter { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("Filter") .field("root", &self.root_expr) @@ -193,16 +262,16 @@ impl std::fmt::Debug for Filter { } } -impl Filter { +impl> Filter { /// Creates a compiled expression IR from a generic closure. - pub(crate) fn new(root_expr: CompiledOneExpr, scheme: Scheme) -> Self { + pub(crate) fn new(root_expr: CompiledOneExpr, scheme: Scheme) -> Self { Filter { root_expr, scheme } } /// Executes a compiled filter expression against a provided context with values. - pub fn execute<'e>( + pub fn execute<'ctx, 'data: 'ctx>( &self, - ctx: &'e ExecutionContext<'e, U>, + ctx: &'ctx F::Context<'data>, ) -> Result { if ctx.scheme() == &self.scheme { Ok(self.root_expr.execute(ctx)) @@ -213,22 +282,22 @@ impl Filter { } /// An IR for a compiled value expression. -pub struct FilterValue { - root_expr: CompiledValueExpr, +pub struct FilterValue = ExecutionContextFamily> { + root_expr: CompiledValueExpr, scheme: Scheme, } -impl FilterValue { +impl> FilterValue { /// Creates a compiled expression IR from a generic closure. - pub(crate) fn new(root_expr: CompiledValueExpr, scheme: Scheme) -> Self { + pub(crate) fn new(root_expr: CompiledValueExpr, scheme: Scheme) -> Self { FilterValue { root_expr, scheme } } /// Executes a compiled value expression against a provided context with values. - pub fn execute<'e>( + pub fn execute<'ctx, 'data: 'ctx>( &self, - ctx: &'e ExecutionContext<'e, U>, - ) -> Result, Type>, SchemeMismatchError> { + ctx: &'ctx F::Context<'data>, + ) -> Result, Type>, SchemeMismatchError> { if ctx.scheme() == &self.scheme { Ok(self.root_expr.execute(ctx)) } else { @@ -260,4 +329,70 @@ mod tests { is_send::>>(); is_sync::>>(); } + + #[test] + fn test_distinct_context_and_data_lifetimes() { + use crate::{ + CompiledOneExpr, CompiledValueExpr, ContextFamily, EvaluationContext, Field, + FieldBindingError, FieldConsumer, HasScheme, LhsValue, ListMatcher, ListRef, Scheme, + }; + use std::cell::Cell; + + struct Context<'data> { + scheme: &'data Scheme, + value: Cell<&'data str>, + } + + impl HasScheme for Context<'_> { + fn scheme(&self) -> &Scheme { + self.scheme + } + } + + impl EvaluationContext for Context<'_> { + type UserData = (); + + fn get_user_data(&self) -> &() { + &() + } + + fn get_list_matcher(&self, _: ListRef<'_>) -> &dyn ListMatcher { + unreachable!("the test scheme has no lists") + } + } + + struct Family; + + impl ContextFamily for Family { + type Context<'data> = Context<'data>; + type UserData = (); + + fn bind>( + field: Field, + _: B, + ) -> Result { + Err(FieldBindingError::UnknownField(field.name().into())) + } + } + + fn assert_send_sync() {} + assert_send_sync::>(); + assert_send_sync::>(); + + let value = + CompiledValueExpr::<(), Family>::new(|ctx| Ok(LhsValue::Bytes(ctx.value.get().into()))); + let matches = CompiledOneExpr::<(), Family>::new(|ctx| ctx.value.get() == "first"); + let first = String::from("first"); + let second = String::from("second"); + let scheme = crate::SchemeBuilder::new().build(); + let mut ctx = Context { + scheme: &scheme, + value: Cell::new(&first), + }; + assert_eq!(value.execute(&ctx).unwrap(), LhsValue::from("first")); + assert!(matches.execute(&ctx)); + *ctx.value.get_mut() = &second; + assert_eq!(value.execute(&ctx).unwrap(), LhsValue::from("second")); + assert!(!matches.execute(&ctx)); + } } diff --git a/engine/src/functions/mod.rs b/engine/src/functions/mod.rs index d180b214..ccf943de 100644 --- a/engine/src/functions/mod.rs +++ b/engine/src/functions/mod.rs @@ -3,8 +3,9 @@ pub(crate) mod concat; pub use self::concat::ConcatFunction; use crate::ParserSettings; use crate::filter::CompiledValueResult; +use crate::lhs_types::LhsValue; use crate::types::{ - ExpectedType, ExpectedTypeList, GetType, LhsValue, LiteralValue, Type, TypeMismatchError, + ExpectedType, ExpectedTypeList, GetType, LiteralValue, Type, TypeMismatchError, }; use std::any::Any; use std::convert::TryFrom; diff --git a/engine/src/lhs_types/array.rs b/engine/src/lhs_types/array.rs index 341961ee..16ee2eeb 100644 --- a/engine/src/lhs_types/array.rs +++ b/engine/src/lhs_types/array.rs @@ -1,9 +1,7 @@ use super::TypedMap; use super::map::InnerMap; -use crate::lhs_types::AsRefIterator; -use crate::types::{ - CompoundType, GetType, IntoValue, LhsValue, LhsValueSeed, Type, TypeMismatchError, -}; +use crate::lhs_types::{AsRefIterator, IntoValue, LhsValue, LhsValueSeed}; +use crate::types::{CompoundType, GetType, Type, TypeMismatchError}; #[cfg(feature = "get-size2")] use get_size2::{GetSize, GetSizeTracker}; use serde::de::{self, DeserializeSeed, Deserializer, SeqAccess, Visitor}; @@ -481,6 +479,15 @@ impl<'a, V: IntoValue<'a>> TypedArray<'a, V> { } } + /// Creates an empty typed array with space for at least `capacity` values. + #[inline] + pub fn with_capacity(capacity: usize) -> Self { + Self { + array: InnerArray::Owned(Vec::with_capacity(capacity)), + _marker: std::marker::PhantomData, + } + } + #[inline] fn as_vec_ref(&self) -> &Vec> { match &self.array { @@ -608,6 +615,8 @@ impl<'a, V: IntoValue<'a>> FromIterator for TypedArray<'a, V> { } } +impl<'a, V: IntoValue<'a>> super::private::SealedIntoValue for TypedArray<'a, V> {} + impl<'a, V: IntoValue<'a>> IntoValue<'a> for TypedArray<'a, V> { const TYPE: Type = Type::Array(CompoundType::from_type(V::TYPE)); diff --git a/engine/src/lhs_types/bytes.rs b/engine/src/lhs_types/bytes.rs index 1e07dd0a..f60fd9d0 100644 --- a/engine/src/lhs_types/bytes.rs +++ b/engine/src/lhs_types/bytes.rs @@ -1,3 +1,5 @@ +use crate::LhsValue; +use crate::values::{Bytes as BytesType, BytesRead, IntoLhsValue, Value}; #[cfg(feature = "get-size2")] use get_size2::{GetSize, GetSizeTracker}; use serde::de::Visitor; @@ -25,6 +27,18 @@ impl GetSize for Bytes<'_> { } } +impl Value for Bytes<'_> { + type Read = BytesRead; + type ValueType = BytesType; +} + +impl<'a, 'data: 'a> IntoLhsValue<'a> for Bytes<'data> { + #[inline] + fn into_lhs(self) -> LhsValue<'a> { + LhsValue::Bytes(self) + } +} + impl<'a> Bytes<'a> { /// Clones self into a fully owned byte string. #[inline] @@ -254,6 +268,20 @@ impl PartialEq for Bytes<'_> { impl Eq for Bytes<'_> {} +impl PartialOrd for Bytes<'_> { + #[inline] + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for Bytes<'_> { + #[inline] + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.deref().cmp(other.deref()) + } +} + impl Hash for Bytes<'_> { #[inline] fn hash(&self, h: &mut H) { @@ -351,6 +379,26 @@ impl<'de> Deserialize<'de> for Bytes<'de> { mod tests { use super::*; + #[test] + fn test_byte_order_is_independent_of_storage() { + use std::collections::BTreeMap; + + let mut values = BTreeMap::new(); + values.insert(Bytes::from(b"b".to_vec()), 1); + values.insert(Bytes::from(&b"a"[..]), 2); + values.insert(Bytes::from(&b"a\0"[..]), 3); + values.insert(Bytes::from(vec![0xff]), 4); + values.insert(Bytes::from(b"a".to_vec()), 5); + + assert_eq!(values.len(), 4); + assert_eq!(values.get(&b"a"[..]), Some(&5)); + assert_eq!(values.get(&b"b"[..]), Some(&1)); + assert_eq!( + values.keys().map(AsRef::as_ref).collect::>(), + [&b"a"[..], &b"a\0"[..], &b"b"[..], &[0xff][..]], + ); + } + #[test] fn test_bytes_deserialize() { let bytes = serde_json::from_str::>("\"a JSON string with unicode ❤\"").unwrap(); diff --git a/engine/src/lhs_types/map.rs b/engine/src/lhs_types/map.rs index 8e9ed581..7e08bd46 100644 --- a/engine/src/lhs_types/map.rs +++ b/engine/src/lhs_types/map.rs @@ -1,8 +1,8 @@ use super::TypedArray; use super::array::InnerArray; use crate::TypeMismatchError; -use crate::lhs_types::{AsRefIterator, Bytes}; -use crate::types::{CompoundType, GetType, IntoValue, LhsValue, LhsValueSeed, Type}; +use crate::lhs_types::{AsRefIterator, Bytes, IntoValue, LhsValue, LhsValueSeed}; +use crate::types::{CompoundType, GetType, Type}; #[cfg(feature = "get-size2")] use get_size2::{GetSize, GetSizeTracker}; use serde::de::{self, DeserializeSeed, Deserializer, MapAccess, SeqAccess, Visitor}; @@ -631,6 +631,8 @@ impl<'a, V: IntoValue<'a>> FromIterator<(Box<[u8]>, V)> for TypedMap<'a, V> { } } +impl<'a, V: IntoValue<'a>> super::private::SealedIntoValue for TypedMap<'a, V> {} + impl<'a, V: IntoValue<'a>> IntoValue<'a> for TypedMap<'a, V> { const TYPE: Type = Type::Map(CompoundType::from_type(V::TYPE)); diff --git a/engine/src/lhs_types/mod.rs b/engine/src/lhs_types/mod.rs index 2f4c6c3b..68a5ecf9 100644 --- a/engine/src/lhs_types/mod.rs +++ b/engine/src/lhs_types/mod.rs @@ -1,3 +1,5 @@ +//! Values stored by execution contexts and returned by value expressions. + mod array; mod bytes; mod map; @@ -5,7 +7,519 @@ mod map; pub use self::array::{Array, ArrayIntoIter, ArrayIter, TypedArray}; pub use self::bytes::Bytes; pub use self::map::{Map, MapIter, MapValuesIntoIter, TypedMap}; -use crate::types::LhsValue; +use crate::scheme::{FieldIndex, IndexAccessError}; +use crate::strict_partial_ord::StrictPartialOrd; +use crate::types::{ + ExpectedType, GetType, LiteralValue, Type, TypeMismatchError, declare_types, + specialized_get_type, specialized_try_from, +}; +use crate::values::{FieldValueType, IntoLhsValue, ReadValue, Value, ValueError, ValuePredicate}; +#[cfg(feature = "get-size2")] +use get_size2::{GetSize, GetSizeTracker}; +use serde::de::{DeserializeSeed, Deserializer}; +use serde::{Deserialize, Serialize, Serializer}; +use std::cmp::Ordering; +use std::fmt::{self, Debug, Formatter}; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; + +macro_rules! declare_lhs_value { + ($($(# $attrs:tt)* $name:ident $([$val_ty:ty])? ( $(# $lhs_attrs:tt)* $lhs_ty:ty | $literal_ty:ty | $literal_set_ty:ty ) , )*) => { + declare_types! { + /// An LHS value provided for filter execution. + /// + /// These are passed to the [`crate::ExecutionContext`] + /// and are used by [`crate::Filter`] for execution and comparisons. + #[derive(PartialEq, Eq, Clone, Deserialize, Hash)] + #[serde(untagged)] + enum LhsValue<'a> { + $($(# $attrs)* $(# $lhs_attrs)* $name($lhs_ty),)* + } + } + + $(impl<'a> TryFrom> for $lhs_ty { + type Error = TypeMismatchError; + + fn try_from(value: LhsValue<'a>) -> Result<$lhs_ty, TypeMismatchError> { + match value { + LhsValue::$name(value) => Ok(value), + _ => Err(TypeMismatchError { + expected: specialized_try_from!($name).into(), + actual: value.get_type(), + }), + } + } + })* + + $(impl<'a> TryFrom<&'a LhsValue<'a>> for &'a $lhs_ty { + type Error = TypeMismatchError; + + fn try_from(value: &'a LhsValue<'a>) -> Result<&'a $lhs_ty, TypeMismatchError> { + match value { + LhsValue::$name(value) => Ok(value), + _ => Err(TypeMismatchError { + expected: specialized_try_from!($name).into(), + actual: value.get_type(), + }), + } + } + })* + + impl<'a> PartialOrd for LhsValue<'a> { + fn partial_cmp(&self, other: &LiteralValue) -> Option { + match (self, other) { + $((LhsValue::$name(lhs), LiteralValue::$name(rhs)) => { + lhs.strict_partial_cmp(rhs) + },)* + _ => None, + } + } + } + + }; +} + +crate::types::with_value_types!(declare_lhs_value); + +impl PartialEq<&LhsValue<'_>> for LhsValue<'_> { + fn eq(&self, other: &&LhsValue<'_>) -> bool { + self.eq(*other) + } +} + +impl StrictPartialOrd for LhsValue<'_> {} + +impl PartialEq for LhsValue<'_> { + fn eq(&self, other: &LiteralValue) -> bool { + self.strict_partial_cmp(other) == Some(Ordering::Equal) + } +} + +mod private { + use crate::Bytes; + use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; + + pub trait SealedIntoValue {} + + impl SealedIntoValue for bool {} + + impl SealedIntoValue for i8 {} + impl SealedIntoValue for u8 {} + impl SealedIntoValue for i16 {} + impl SealedIntoValue for u16 {} + impl SealedIntoValue for i32 {} + impl SealedIntoValue for u32 {} + impl SealedIntoValue for i64 {} + + impl SealedIntoValue for IpAddr {} + impl SealedIntoValue for Ipv4Addr {} + impl SealedIntoValue for Ipv6Addr {} + + impl<'a, T> SealedIntoValue for T where Bytes<'a>: From {} +} + +/// Converts a value into an `LhsValue`. +/// It is a stronger version of `Into>` in that +/// any value of the input type will *always* convert to the +/// same statically known `LhsValue` variant. +pub trait IntoValue<'a>: private::SealedIntoValue { + const TYPE: Type; + + fn into_value(self) -> LhsValue<'a>; +} + +impl<'a> IntoValue<'a> for bool { + const TYPE: Type = Type::Bool; + + #[inline] + fn into_value(self) -> LhsValue<'a> { + LhsValue::Bool(self) + } +} + +impl<'a> IntoValue<'a> for i64 { + const TYPE: Type = Type::Int; + + #[inline] + fn into_value(self) -> LhsValue<'a> { + LhsValue::Int(self) + } +} + +impl<'a> IntoValue<'a> for i32 { + const TYPE: Type = Type::Int; + + #[inline] + fn into_value(self) -> LhsValue<'a> { + LhsValue::Int(i64::from(self)) + } +} + +impl<'a> IntoValue<'a> for i16 { + const TYPE: Type = Type::Int; + + #[inline] + fn into_value(self) -> LhsValue<'a> { + LhsValue::Int(i64::from(self)) + } +} + +impl<'a> IntoValue<'a> for u16 { + const TYPE: Type = Type::Int; + + #[inline] + fn into_value(self) -> LhsValue<'a> { + LhsValue::Int(i64::from(self)) + } +} + +impl<'a> IntoValue<'a> for i8 { + const TYPE: Type = Type::Int; + + #[inline] + fn into_value(self) -> LhsValue<'a> { + LhsValue::Int(i64::from(self)) + } +} + +impl<'a> IntoValue<'a> for u8 { + const TYPE: Type = Type::Int; + + #[inline] + fn into_value(self) -> LhsValue<'a> { + LhsValue::Int(i64::from(self)) + } +} + +impl<'a> IntoValue<'a> for IpAddr { + const TYPE: Type = Type::Ip; + + #[inline] + fn into_value(self) -> LhsValue<'a> { + LhsValue::Ip(self) + } +} + +impl<'a> IntoValue<'a> for Ipv4Addr { + const TYPE: Type = Type::Ip; + + #[inline] + fn into_value(self) -> LhsValue<'a> { + LhsValue::Ip(IpAddr::V4(self)) + } +} + +impl<'a> IntoValue<'a> for Ipv6Addr { + const TYPE: Type = Type::Ip; + + #[inline] + fn into_value(self) -> LhsValue<'a> { + LhsValue::Ip(IpAddr::V6(self)) + } +} + +impl<'a, T> IntoValue<'a> for T +where + Bytes<'a>: From, +{ + const TYPE: Type = Type::Bytes; + + #[inline] + fn into_value(self) -> LhsValue<'a> { + LhsValue::Bytes(Bytes::from(self)) + } +} + +impl<'a, T: IntoValue<'a>> From for LhsValue<'a> { + #[inline] + fn from(value: T) -> Self { + value.into_value() + } +} + +// Array cannot implement `IntoValue` as the +// underlying element type is not statically +// known. +impl<'a> From> for LhsValue<'a> { + #[inline] + fn from(value: Array<'a>) -> LhsValue<'a> { + LhsValue::Array(value) + } +} + +// Map cannot implement `IntoValue` as the +// underlying element type is not statically +// known. +impl<'a> From> for LhsValue<'a> { + #[inline] + fn from(value: Map<'a>) -> LhsValue<'a> { + LhsValue::Map(value) + } +} + +impl<'a> TryFrom<&'a LhsValue<'a>> for &'a [u8] { + type Error = TypeMismatchError; + + fn try_from(value: &'a LhsValue<'_>) -> Result { + match value { + LhsValue::Bytes(value) => Ok(value), + _ => Err(TypeMismatchError { + expected: Type::Bytes.into(), + actual: value.get_type(), + }), + } + } +} + +impl<'a> From<&'a LiteralValue> for LhsValue<'a> { + fn from(literal: &'a LiteralValue) -> Self { + match literal { + LiteralValue::Ip(ip) => LhsValue::Ip(*ip), + LiteralValue::Bytes(bytes) => LhsValue::Bytes(Bytes::Borrowed(bytes)), + LiteralValue::Int(integer) => LhsValue::Int(*integer), + LiteralValue::Bool(b) => match *b {}, + LiteralValue::Array(a) => match *a {}, + LiteralValue::Map(m) => match *m {}, + } + } +} + +impl From for LhsValue<'_> { + fn from(literal: LiteralValue) -> Self { + match literal { + LiteralValue::Ip(ip) => LhsValue::Ip(ip), + LiteralValue::Bytes(bytes) => LhsValue::Bytes(Bytes::Owned(bytes.into())), + LiteralValue::Int(integer) => LhsValue::Int(integer), + LiteralValue::Bool(b) => match b {}, + LiteralValue::Array(a) => match a {}, + LiteralValue::Map(m) => match m {}, + } + } +} + +impl<'a> LhsValue<'a> { + /// Converts a reference to an LhsValue to an LhsValue with an internal + /// references + pub fn as_ref(&'a self) -> Self { + match self { + LhsValue::Ip(ip) => LhsValue::Ip(*ip), + LhsValue::Bytes(bytes) => LhsValue::Bytes(Bytes::Borrowed(bytes)), + LhsValue::Int(integer) => LhsValue::Int(*integer), + LhsValue::Bool(b) => LhsValue::Bool(*b), + LhsValue::Array(a) => LhsValue::Array(a.as_ref()), + LhsValue::Map(m) => LhsValue::Map(m.as_ref()), + } + } + + /// Converts an `LhsValue` with borrowed data to a fully owned `LhsValue`. + pub fn into_owned(self) -> LhsValue<'static> { + match self { + LhsValue::Ip(ip) => LhsValue::Ip(ip), + LhsValue::Bytes(bytes) => LhsValue::Bytes(Bytes::Owned(bytes.into_owned())), + LhsValue::Int(i) => LhsValue::Int(i), + LhsValue::Bool(b) => LhsValue::Bool(b), + LhsValue::Array(arr) => LhsValue::Array(arr.into_owned()), + LhsValue::Map(map) => LhsValue::Map(map.into_owned()), + } + } + + /// Retrieve an element from an LhsValue given a path item and a specified + /// type. + /// Returns a TypeMismatchError error if current type does not support it + /// nested element. + /// + /// Both LhsValue::Array and LhsValue::Map support nested elements. + pub(crate) fn get( + &'a self, + item: &FieldIndex, + ) -> Result>, IndexAccessError> { + match (self, item) { + (LhsValue::Array(arr), FieldIndex::ArrayIndex(idx)) => Ok(arr.get(*idx as usize)), + (_, FieldIndex::ArrayIndex(_)) => Err(IndexAccessError { + index: item.clone(), + actual: self.get_type(), + }), + (LhsValue::Map(map), FieldIndex::MapKey(key)) => Ok(map.get(key.as_bytes())), + (_, FieldIndex::MapKey(_)) => Err(IndexAccessError { + index: item.clone(), + actual: self.get_type(), + }), + (_, FieldIndex::MapEach) => Err(IndexAccessError { + index: item.clone(), + actual: self.get_type(), + }), + } + } + + #[inline] + pub(crate) fn get_nested(&'a self, indexes: &[FieldIndex]) -> Option<&'a LhsValue<'a>> { + indexes + .iter() + .try_fold(self, |value, idx| value.get(idx).unwrap()) + } + + pub(crate) fn extract( + self, + item: &FieldIndex, + ) -> Result>, IndexAccessError> { + match item { + FieldIndex::ArrayIndex(idx) => match self { + LhsValue::Array(arr) => Ok(arr.extract(*idx as usize)), + _ => Err(IndexAccessError { + index: item.clone(), + actual: self.get_type(), + }), + }, + FieldIndex::MapKey(key) => match self { + LhsValue::Map(map) => Ok(map.extract(key.as_bytes())), + _ => Err(IndexAccessError { + index: item.clone(), + actual: self.get_type(), + }), + }, + FieldIndex::MapEach => Err(IndexAccessError { + index: item.clone(), + actual: self.get_type(), + }), + } + } + + #[inline] + pub(crate) fn extract_nested(self, indexes: &[FieldIndex]) -> Option> { + indexes + .iter() + .try_fold(self, |value, idx| value.extract(idx).unwrap()) + } + + /// Returns an iterator over the Map or Array + pub(crate) fn iter(&'a self) -> Option> { + match self { + LhsValue::Array(array) => Some(Iter::IterArray(array.iter())), + LhsValue::Map(map) => Some(Iter::IterMap(map.iter())), + _ => None, + } + } +} + +impl Serialize for LhsValue<'_> { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + match self { + LhsValue::Ip(ip) => ip.serialize(serializer), + LhsValue::Bytes(bytes) => { + if let Ok(s) = std::str::from_utf8(bytes) { + serializer.serialize_str(s) + } else { + serializer.serialize_bytes(bytes) + } + } + LhsValue::Int(num) => num.serialize(serializer), + LhsValue::Bool(b) => b.serialize(serializer), + LhsValue::Array(arr) => arr.serialize(serializer), + LhsValue::Map(map) => map.serialize(serializer), + } + } +} + +pub(crate) struct LhsValueSeed<'a>(pub &'a Type); + +impl<'de> DeserializeSeed<'de> for LhsValueSeed<'_> { + type Value = LhsValue<'de>; + + fn deserialize(self, deserializer: D) -> Result + where + D: Deserializer<'de>, + { + match self.0 { + Type::Ip => Ok(LhsValue::Ip(std::net::IpAddr::deserialize(deserializer)?)), + Type::Int => Ok(LhsValue::Int(i64::deserialize(deserializer)?)), + Type::Bool => Ok(LhsValue::Bool(bool::deserialize(deserializer)?)), + Type::Bytes => Ok(LhsValue::Bytes(Bytes::deserialize(deserializer)?)), + Type::Array(ty) => Ok(LhsValue::Array({ + let mut arr = Array::new(*ty); + arr.deserialize(deserializer)?; + arr + })), + Type::Map(ty) => Ok(LhsValue::Map({ + let mut map = Map::new(*ty); + map.deserialize(deserializer)?; + map + })), + } + } +} + +pub enum IntoIter<'a> { + IntoArray(ArrayIntoIter<'a>), + IntoMap(MapValuesIntoIter<'a>), +} + +impl<'a> Iterator for IntoIter<'a> { + type Item = LhsValue<'a>; + + fn next(&mut self) -> Option> { + match self { + IntoIter::IntoArray(array) => array.next(), + IntoIter::IntoMap(map) => map.next(), + } + } + + fn size_hint(&self) -> (usize, Option) { + (self.len(), Some(self.len())) + } +} + +impl ExactSizeIterator for IntoIter<'_> { + fn len(&self) -> usize { + match self { + IntoIter::IntoArray(array) => array.len(), + IntoIter::IntoMap(map) => map.len(), + } + } +} + +impl<'a> IntoIterator for LhsValue<'a> { + type IntoIter = IntoIter<'a>; + type Item = LhsValue<'a>; + + fn into_iter(self) -> Self::IntoIter { + match self { + LhsValue::Array(array) => IntoIter::IntoArray(array.into_iter()), + LhsValue::Map(map) => IntoIter::IntoMap(map.into_values()), + _ => unreachable!(), + } + } +} + +pub(crate) enum Iter<'a> { + IterArray(ArrayIter<'a, 'a>), + IterMap(MapIter<'a, 'a>), +} + +impl<'a> Iterator for Iter<'a> { + type Item = &'a LhsValue<'a>; + + fn next(&mut self) -> Option<&'a LhsValue<'a>> { + match self { + Iter::IterArray(array) => array.next(), + Iter::IterMap(map) => map.next().map(|(_, v)| v), + } + } + + fn size_hint(&self) -> (usize, Option) { + (self.len(), Some(self.len())) + } +} + +impl ExactSizeIterator for Iter<'_> { + fn len(&self) -> usize { + match self { + Iter::IterArray(array) => array.len(), + Iter::IterMap(map) => map.len(), + } + } +} pub struct AsRefIterator<'a, T: Iterator>>(T); @@ -28,3 +542,376 @@ impl<'a, T: ExactSizeIterator>> ExactSizeIterator for As self.0.len() } } + +pub(crate) fn append_stored_lhs_at<'a>( + mut value: LhsValue<'a>, + mut indexes: &[FieldIndex], + output: &mut Vec>, +) { + while let Some((index, rest)) = indexes.split_first() { + if *index == FieldIndex::MapEach { + let values = value.into_iter(); + if rest.is_empty() { + output.extend(values); + } else { + for value in values { + append_stored_lhs_at(value, rest, output); + } + } + return; + } + let Some(selected) = value.extract(index).expect("invalid value selection path") else { + return; + }; + value = selected; + indexes = rest; + } + output.push(value); +} + +/// A stored field whose logical type is supplied by the scheme. +/// +/// Unlike a static [`super::values::ValueType`], this describes an entire field, +/// including arbitrary collection nesting, rather than a statically typed container element. +/// Contexts must check values against the declared type when inserting them, as +/// [`crate::ExecutionContext::set_field_value`] does. Startup validation cannot +/// infer the variant returned by an arbitrary getter of [`LhsValue`]. +pub struct LhsValueType; + +impl FieldValueType for LhsValueType { + type Path = Box<[FieldIndex]>; + + const DIRECT_COLLECTION_COMPARISON: bool = true; + const STATIC_TYPE: Option = None; + + fn compile_field_path( + mut field_type: Type, + path: Vec, + input_type: Type, + ) -> Result { + for step in &path { + field_type = match (field_type, step) { + (Type::Array(child), FieldIndex::ArrayIndex(_) | FieldIndex::MapEach) + | (Type::Map(child), FieldIndex::MapKey(_) | FieldIndex::MapEach) => child.into(), + _ => return Err(ValueError("index does not match the declared field type")), + }; + } + if field_type != input_type { + return Err(ValueError( + "predicate input type differs from selected value", + )); + } + Ok(path.into_boxed_slice()) + } +} + +/// Engine reader borrowing existing Wirefilter values and their containers. +pub struct LhsValueRead; + +impl LhsValueRead { + /// Compare a selected value while retaining the caller's value ownership. + pub(crate) fn compare_one( + value: &LhsValue<'_>, + path: &[FieldIndex], + predicate: &P, + ) -> Option { + value.get_nested(path).map(|value| predicate.lhs(value)) + } + + /// Compare selections from a possibly nested path. + pub(crate) fn compare_vec( + value: &LhsValue<'_>, + path: &[FieldIndex], + predicate: &P, + output: &mut TypedArray<'static, bool>, + ) { + compare_each(value, path, predicate, output); + } + + /// Compare each child of the collection selected by a concrete prefix. + /// Keeping the final iteration implicit avoids retaining another path entry. + pub(crate) fn compare_items( + value: &LhsValue<'_>, + prefix: &[FieldIndex], + predicate: &P, + output: &mut TypedArray<'static, bool>, + ) { + if let Some(value) = value.get_nested(prefix) { + compare_each(value, &[FieldIndex::MapEach], predicate, output); + } + } +} + +impl Value for LhsValue<'_> { + type Read = LhsValueRead; + type ValueType = LhsValueType; +} + +impl<'a, 'data: 'a> IntoLhsValue<'a> for LhsValue<'data> { + const MATERIALIZES_COLLECTIONS: bool = false; + + #[inline] + fn into_lhs(self) -> LhsValue<'a> { + self + } + + #[inline] + fn append_lhs_at(self, indexes: &[FieldIndex], output: &mut Vec>) { + append_stored_lhs_at(self, indexes, output); + } +} + +impl ReadValue, LhsValueType> for LhsValueRead { + const MATERIALIZES_COLLECTIONS: bool = false; + + #[inline] + fn to_lhs<'a>(value: &'a LhsValue<'_>) -> LhsValue<'a> { + value.as_ref() + } + + #[inline] + fn to_lhs_at<'a>(value: &'a LhsValue<'_>, indexes: &[FieldIndex]) -> Option> { + value.get_nested(indexes).map(LhsValue::as_ref) + } + + #[inline] + fn append_lhs_at<'a>( + value: &'a LhsValue<'_>, + indexes: &[FieldIndex], + output: &mut Vec>, + ) { + append_stored_lhs_at(value.as_ref(), indexes, output); + } + + #[inline] + fn compare_one( + value: &LhsValue<'_>, + path: &Box<[FieldIndex]>, + predicate: &P, + ) -> Option { + Self::compare_one(value, path, predicate) + } + + #[inline] + fn compare_vec( + value: &LhsValue<'_>, + path: &Box<[FieldIndex]>, + predicate: &P, + output: &mut TypedArray<'static, bool>, + ) { + Self::compare_vec(value, path, predicate, output); + } +} + +fn compare_each<'a, P: ValuePredicate + ?Sized>( + mut value: &'a LhsValue<'a>, + mut path: &[FieldIndex], + predicate: &P, + output: &mut TypedArray<'static, bool>, +) { + // Follow concrete indices while reading values, without first searching for + // [*]. Only collection iteration recurses; concrete indices add no calls. + while let Some((index, rest)) = path.split_first() { + if *index == FieldIndex::MapEach { + let children = value.iter().expect("[*] requires a collection"); + if rest.is_empty() { + if output.is_empty() { + *output = TypedArray::with_capacity(children.size_hint().0); + } + for child in children { + output.push(predicate.lhs(child)); + } + } else { + for child in children { + compare_each(child, rest, predicate, output); + } + } + return; + } + let Some(child) = value.get(index).unwrap() else { + return; + }; + value = child; + path = rest; + } + output.push(predicate.lhs(value)); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn validates_paths_against_the_scheme_type() { + let root = Type::map(Type::array(Type::Int)); + let path = vec![FieldIndex::MapKey("values".into()), FieldIndex::MapEach]; + assert_eq!( + LhsValueType::compile_field_path(root, path.clone(), Type::Int).unwrap(), + path.into_boxed_slice(), + ); + assert!(LhsValueType::compile_field_path(root, vec![], root).is_ok()); + assert!(LhsValueType::compile_field_path(root, vec![], Type::Int).is_err()); + assert!( + LhsValueType::compile_field_path( + root, + vec![FieldIndex::ArrayIndex(0)], + Type::array(Type::Int), + ) + .is_err() + ); + assert!( + LhsValueType::compile_field_path(Type::Int, vec![FieldIndex::MapEach], Type::Int,) + .is_err() + ); + assert!( + LhsValueType::compile_field_path( + Type::array(Type::Int), + vec![FieldIndex::MapKey("values".into())], + Type::Int, + ) + .is_err() + ); + } +} + +#[cfg(feature = "get-size2")] +impl GetSize for LhsValue<'_> { + fn get_heap_size_with_tracker(&self, tracker: T) -> (usize, T) { + match self { + Self::Bool(_) | Self::Int(_) | Self::Ip(_) => (0, tracker), + Self::Bytes(value) => value.get_heap_size_with_tracker(tracker), + Self::Array(value) => value.get_heap_size_with_tracker(tracker), + Self::Map(value) => value.get_heap_size_with_tracker(tracker), + } + } +} + +#[test] +fn test_lhs_value_deserialize() { + use std::str::FromStr; + + let ipv4: LhsValue<'_> = serde_json::from_str("\"127.0.0.1\"").unwrap(); + assert_eq!(ipv4, LhsValue::Ip(IpAddr::from_str("127.0.0.1").unwrap())); + + let ipv6: LhsValue<'_> = serde_json::from_str("\"::1\"").unwrap(); + assert_eq!(ipv6, LhsValue::Ip(IpAddr::from_str("::1").unwrap())); + + let bytes: LhsValue<'_> = serde_json::from_str("\"a JSON string with unicode ❤\"").unwrap(); + assert_eq!( + bytes, + LhsValue::from(&b"a JSON string with unicode \xE2\x9D\xA4"[..]) + ); + + let bytes = + serde_json::from_str::>("\"a JSON string with escaped-unicode \\u2764\"") + .unwrap(); + assert_eq!( + bytes, + LhsValue::from(&b"a JSON string with escaped-unicode \xE2\x9D\xA4"[..]) + ); + + let bytes: LhsValue<'_> = serde_json::from_str("\"1337\"").unwrap(); + assert_eq!(bytes, LhsValue::from(&b"1337"[..])); + + let integer: LhsValue<'_> = serde_json::from_str("1337").unwrap(); + assert_eq!(integer, LhsValue::Int(1337)); + + let b: LhsValue<'_> = serde_json::from_str("false").unwrap(); + assert_eq!(b, LhsValue::Bool(false)); +} + +#[test] +fn test_size_of_lhs_value() { + assert_eq!(std::mem::size_of::>(), 48); +} + +#[cfg(feature = "get-size2")] +#[test] +fn test_lhs_value_get_size() { + let stack_size = std::mem::size_of::>(); + + assert_eq!(LhsValue::Bool(false).get_size(), stack_size); + assert_eq!(LhsValue::Int(42).get_size(), stack_size); + assert_eq!( + LhsValue::Ip(IpAddr::V4(Ipv4Addr::LOCALHOST)).get_size(), + stack_size + ); + + let borrowed_bytes = LhsValue::Bytes(Bytes::Borrowed(b"borrowed")); + assert_eq!(borrowed_bytes.get_size(), stack_size); + + let owned_bytes = LhsValue::Bytes(Bytes::Owned(Box::from(&b"owned"[..]))); + assert_eq!(owned_bytes.get_size(), stack_size + b"owned".len()); +} + +#[cfg(feature = "get-size2")] +#[test] +fn test_array_lhs_value_get_size() { + let stack_size = std::mem::size_of::>(); + + let mut borrowed_values = Vec::with_capacity(4); + borrowed_values.push(LhsValue::Bytes(Bytes::Borrowed(b"borrowed"))); + let borrowed_values_capacity = borrowed_values.capacity(); + let owned_array_of_borrowed = Array::try_from_vec(Type::Bytes, borrowed_values).unwrap(); + assert_eq!( + LhsValue::Array(owned_array_of_borrowed.as_ref()).get_size(), + stack_size + ); + assert_eq!( + LhsValue::Array(owned_array_of_borrowed).get_size(), + stack_size + borrowed_values_capacity * stack_size + ); + + let mut owned_values = Vec::with_capacity(4); + owned_values.push(LhsValue::Bytes(Bytes::Owned(Box::from(&b"owned"[..])))); + let owned_values_capacity = owned_values.capacity(); + let owned_array_of_owned = Array::try_from_vec(Type::Bytes, owned_values).unwrap(); + assert_eq!( + LhsValue::Array(owned_array_of_owned.as_ref()).get_size(), + stack_size + ); + assert_eq!( + LhsValue::Array(owned_array_of_owned).get_size(), + stack_size + owned_values_capacity * stack_size + b"owned".len() + ); +} + +#[cfg(feature = "get-size2")] +#[test] +fn test_map_lhs_value_get_size() { + let stack_size = std::mem::size_of::>(); + + let owned_map_of_borrowed = Map::try_from_iter( + Type::Bytes, + [Ok::<_, TypeMismatchError>(( + Box::from(b"key".as_slice()), + Bytes::Borrowed(b"borrowed"), + ))], + ) + .unwrap(); + assert_eq!( + LhsValue::Map(owned_map_of_borrowed.as_ref()).get_size(), + stack_size + ); + assert_eq!( + LhsValue::Map(owned_map_of_borrowed).get_size(), + stack_size + std::mem::size_of::>() + b"key".len() + stack_size + ); + + let owned_map_of_owned = Map::try_from_iter( + Type::Bytes, + [Ok::<_, TypeMismatchError>(( + Box::from(b"key".as_slice()), + Bytes::Owned(Box::from(&b"owned"[..])), + ))], + ) + .unwrap(); + assert_eq!( + LhsValue::Map(owned_map_of_owned.as_ref()).get_size(), + stack_size + ); + assert_eq!( + LhsValue::Map(owned_map_of_owned).get_size(), + stack_size + std::mem::size_of::>() + b"key".len() + stack_size + b"owned".len() + ); +} diff --git a/engine/src/lib.rs b/engine/src/lib.rs index a16c005d..aa00c255 100644 --- a/engine/src/lib.rs +++ b/engine/src/lib.rs @@ -67,6 +67,7 @@ mod scheme; mod ast; mod compiler; +pub mod context; mod execution_context; mod filter; mod functions; @@ -78,6 +79,7 @@ mod rhs_types; mod searcher; mod strict_partial_ord; mod types; +pub mod values; pub use self::ast::field_expr::{ ComparisonExpr, ComparisonOpExpr, ComparisonRhs, IdentifierExpr, IntOp, IntRhs, OrderingOp, @@ -92,6 +94,16 @@ pub use self::ast::parse::{FilterParser, ParseError, ParserSettings}; pub use self::ast::visitor::{Visitor, VisitorMut}; pub use self::ast::{Expr, FilterAst, FilterValueAst, FilterValueExpr, ValueExpr}; pub use self::compiler::{Compiler, DefaultCompiler}; +// The exported field macro calls this helper from downstream crates. +#[doc(hidden)] +pub use self::context::fields::validate_field_definitions; +pub use self::context::state::ContextState; +pub use self::context::{ + BorrowedValueProvider, ContextFamily, ContextRef, EvaluationContext, EvaluationContextMut, + ExecutionContextFamily, FieldBindingError, FieldConsumer, FieldDefinitionError, FieldGetter, + FieldGetterFn, HasScheme, LhsValueProvider, OptionalFieldGetterResult, OwnedValueProvider, + ProviderFamily, RequiredFieldGetterFn, RequiredFieldGetterResult, SchemeValidationError, +}; pub use self::execution_context::{ ExecutionContext, ExecutionContextGuard, InvalidListMatcherError, SetFieldValueError, }; @@ -105,7 +117,7 @@ pub use self::functions::{ SimpleFunctionImpl, SimpleFunctionOptParam, SimpleFunctionParam, }; pub use self::lex::LexErrorKind; -pub use self::lhs_types::{Array, Bytes, Map, MapIter, TypedArray, TypedMap}; +pub use self::lhs_types::{Array, Bytes, LhsValue, Map, MapIter, TypedArray, TypedMap}; pub use self::list_matcher::{ AlwaysList, AlwaysListMatcher, ListDefinition, ListMatcher, NeverList, NeverListMatcher, }; @@ -123,6 +135,6 @@ pub use self::scheme::{ ListRef, Scheme, SchemeBuilder, SchemeMismatchError, UnknownFieldError, }; pub use self::types::{ - CompoundType, ExpectedType, ExpectedTypeList, GetType, LhsValue, LiteralSet, LiteralValue, - Type, TypeMismatchError, + CompoundType, ExpectedType, ExpectedTypeList, GetType, LiteralSet, LiteralValue, Type, + TypeMismatchError, }; diff --git a/engine/src/rhs_types/regex/imp_real.rs b/engine/src/rhs_types/regex/imp_real.rs index ec821ad8..5ab33bef 100644 --- a/engine/src/rhs_types/regex/imp_real.rs +++ b/engine/src/rhs_types/regex/imp_real.rs @@ -12,7 +12,23 @@ pub struct Regex { format: RegexFormat, } +/// Matching state retained by a compiled filter, without the source representation. +pub(crate) struct CompiledRegex(regex_automata::meta::Regex); + +impl CompiledRegex { + #[inline] + pub(super) fn is_match(&self, input: &[u8]) -> bool { + self.0.is_match(input) + } +} + impl Regex { + /// Move the existing matcher and its cache into the filter without retaining + /// the pattern and formatting used by the AST. + pub(crate) fn into_compiled(self) -> CompiledRegex { + CompiledRegex(self.regex) + } + /// Retrieves the syntax configuration that will be used to build the regex. #[inline] pub fn syntax_config() -> regex_automata::util::syntax::Config { diff --git a/engine/src/rhs_types/regex/imp_stub.rs b/engine/src/rhs_types/regex/imp_stub.rs index 117b4036..3cbf371a 100644 --- a/engine/src/rhs_types/regex/imp_stub.rs +++ b/engine/src/rhs_types/regex/imp_stub.rs @@ -10,7 +10,19 @@ pub struct Regex { format: RegexFormat, } +pub(crate) struct CompiledRegex; + +impl CompiledRegex { + pub(super) fn is_match(&self, _input: &[u8]) -> bool { + unimplemented!("Engine was built without regex support") + } +} + impl Regex { + pub(crate) fn into_compiled(self) -> CompiledRegex { + CompiledRegex + } + /// Creates a new dummy regex. pub fn new( pattern: impl Into, diff --git a/engine/src/rhs_types/regex/mod.rs b/engine/src/rhs_types/regex/mod.rs index 67b1918a..a36a6003 100644 --- a/engine/src/rhs_types/regex/mod.rs +++ b/engine/src/rhs_types/regex/mod.rs @@ -1,6 +1,6 @@ use crate::lex::{LexErrorKind, LexResult, LexWith, span}; use crate::rhs_types::bytes::lex_raw_string_as_str; -use crate::{Compare, ExecutionContext, FilterParser, LhsValue}; +use crate::{Compare, ContextFamily, FilterParser, LhsValue}; use cfg_if::cfg_if; use serde::{Serialize, Serializer}; use std::fmt::{self, Debug, Display, Formatter}; @@ -146,14 +146,34 @@ pub enum Error { Other(Box), } -impl Compare for Regex { +impl> Compare for Regex { #[inline] - fn compare<'e>(&self, value: &LhsValue<'e>, _: &'e ExecutionContext<'e, U>) -> bool { + fn compare<'e, 'data: 'e>(&self, value: &LhsValue<'e>, _: &'e F::Context<'data>) -> bool { self.is_match(match value { LhsValue::Bytes(bytes) => bytes, _ => unreachable!(), }) } + + #[inline] + fn compare_bytes<'e, 'data: 'e>(&self, value: &'e [u8], _: &'e F::Context<'data>) -> bool { + self.is_match(value) + } +} + +impl> Compare for CompiledRegex { + #[inline] + fn compare<'e, 'data: 'e>(&self, value: &LhsValue<'e>, _: &'e F::Context<'data>) -> bool { + self.is_match(match value { + LhsValue::Bytes(bytes) => bytes, + _ => unreachable!(), + }) + } + + #[inline] + fn compare_bytes<'e, 'data: 'e>(&self, value: &'e [u8], _: &'e F::Context<'data>) -> bool { + self.is_match(value) + } } #[cfg(test)] @@ -161,6 +181,53 @@ mod test { use super::*; use crate::{ParserSettings, SchemeBuilder}; + #[test] + fn test_compiled_filter_releases_regex_source() { + use crate::{ComparisonOpExpr, LogicalExpr, Type}; + use arcstr::ArcStr; + + let mut builder = SchemeBuilder::new(); + builder.add_field("text", Type::Bytes).unwrap(); + let scheme = builder.build(); + for literal in [r#""a.b""#, r###"r##"a.b"##"###] { + let expression = format!("text matches {literal}"); + let ast = scheme.parse(&expression).unwrap(); + let LogicalExpr::Comparison(comparison) = ast.expression() else { + unreachable!(); + }; + let ComparisonOpExpr::Matches(regex) = &comparison.op else { + unreachable!(); + }; + let pattern = regex.pattern().clone(); + assert_eq!(ArcStr::strong_count(&pattern), Some(2)); + let filter = ast.compile(); + assert_eq!(ArcStr::strong_count(&pattern), Some(1)); + + #[cfg(feature = "regex")] + { + let mut ctx = crate::ExecutionContext::new(&scheme); + for (value, expected) in [(b"a\xffb".as_slice(), true), (b"ac", false)] { + ctx.set_field_value_from_name("text", value).unwrap(); + assert_eq!(filter.execute(&ctx).unwrap(), expected); + } + } + drop(filter); + } + } + + #[cfg(not(feature = "regex"))] + #[test] + #[should_panic(expected = "Engine was built without regex support")] + fn test_compiled_regex_without_feature_panics_on_execution() { + let mut builder = SchemeBuilder::new(); + builder.add_field("text", crate::Type::Bytes).unwrap(); + let scheme = builder.build(); + let filter = scheme.parse(r#"text matches "a.b""#).unwrap().compile(); + let mut ctx = crate::ExecutionContext::new(&scheme); + ctx.set_field_value_from_name("text", "axb").unwrap(); + filter.execute(&ctx).unwrap(); + } + #[test] fn test() { let scheme = SchemeBuilder::new().build(); diff --git a/engine/src/scheme.rs b/engine/src/scheme.rs index 3e6b0343..0dc814a9 100644 --- a/engine/src/scheme.rs +++ b/engine/src/scheme.rs @@ -2149,3 +2149,72 @@ fn test_nil_not_equal_behavior_false() { assert_eq!(filter.execute(&ctx), Ok(false)); } + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + #[test] + fn test_filter_compilation_does_not_duplicate_field_ownership() { + let scheme = Scheme! { + number: Int, + numbers: Array(Int), + headers: Map(Bytes), + } + .build(); + let baseline = Arc::strong_count(&scheme.inner); + + for input in [ + "number == 42", + "numbers[0] == 42", + "any(numbers[*] == 42)", + "number == 42 or number == 43", + r#"headers["host"] == "example.com""#, + ] { + let ast = scheme.parse(input).unwrap(); + let parsed_owners = Arc::strong_count(&scheme.inner); + let filter = ast.compile(); + let compiled_owners = Arc::strong_count(&scheme.inner); + + // Fields own a Scheme reference. Moving the AST's fields into + // compiled getters must not retain additional field copies. + assert!( + compiled_owners <= parsed_owners, + "compiling {input:?} increased Scheme owners from {parsed_owners} to {compiled_owners}" + ); + drop(filter); + assert_eq!(Arc::strong_count(&scheme.inner), baseline, "{input:?}"); + } + } + + #[test] + fn test_value_compilation_does_not_duplicate_field_ownership() { + let scheme = Scheme! { + number: Int, + numbers: Array(Int), + headers: Map(Bytes), + } + .build(); + let baseline = Arc::strong_count(&scheme.inner); + + for input in [ + "number", + "numbers", + "numbers[0]", + r#"headers["host"]"#, + "any(numbers[*] == 42)", + ] { + let ast = scheme.parse_value(input).unwrap(); + let parsed_owners = Arc::strong_count(&scheme.inner); + let value = ast.compile(); + let compiled_owners = Arc::strong_count(&scheme.inner); + + assert!( + compiled_owners <= parsed_owners, + "compiling {input:?} increased Scheme owners from {parsed_owners} to {compiled_owners}" + ); + drop(value); + assert_eq!(Arc::strong_count(&scheme.inner), baseline, "{input:?}"); + } + } +} diff --git a/engine/src/searcher.rs b/engine/src/searcher.rs index 23846c71..24e8e3bb 100644 --- a/engine/src/searcher.rs +++ b/engine/src/searcher.rs @@ -1,12 +1,17 @@ -use crate::{Compare, ExecutionContext, LhsValue}; +use crate::{Compare, ContextFamily, LhsValue}; use memchr::memmem::{Finder, FinderBuilder}; use sliceslice::MemchrSearcher; pub struct EmptySearcher; -impl Compare for EmptySearcher { +impl> Compare for EmptySearcher { #[inline] - fn compare<'e>(&self, _: &LhsValue<'e>, _: &'e ExecutionContext<'e, U>) -> bool { + fn compare<'e, 'data: 'e>(&self, _: &LhsValue<'e>, _: &'e F::Context<'data>) -> bool { + true + } + + #[inline] + fn compare_bytes<'e, 'data: 'e>(&self, _: &'e [u8], _: &'e F::Context<'data>) -> bool { true } } @@ -20,9 +25,9 @@ impl MemmemSearcher { } } -impl Compare for MemmemSearcher { +impl> Compare for MemmemSearcher { #[inline] - fn compare<'e>(&self, value: &LhsValue<'e>, _: &'e ExecutionContext<'e, U>) -> bool { + fn compare<'e, 'data: 'e>(&self, value: &LhsValue<'e>, _: &'e F::Context<'data>) -> bool { self.0 .find(match value { LhsValue::Bytes(bytes) => bytes, @@ -30,14 +35,24 @@ impl Compare for MemmemSearcher { }) .is_some() } + + #[inline] + fn compare_bytes<'e, 'data: 'e>(&self, value: &'e [u8], _: &'e F::Context<'data>) -> bool { + self.0.find(value).is_some() + } } -impl Compare for MemchrSearcher { +impl> Compare for MemchrSearcher { #[inline] - fn compare<'e>(&self, value: &LhsValue<'e>, _: &'e ExecutionContext<'e, U>) -> bool { + fn compare<'e, 'data: 'e>(&self, value: &LhsValue<'e>, _: &'e F::Context<'data>) -> bool { self.search_in(match value { LhsValue::Bytes(bytes) => bytes, _ => unreachable!(), }) } + + #[inline] + fn compare_bytes<'e, 'data: 'e>(&self, value: &'e [u8], _: &'e F::Context<'data>) -> bool { + self.search_in(value) + } } diff --git a/engine/src/types.rs b/engine/src/types.rs index c31d65c3..7e593d7e 100644 --- a/engine/src/types.rs +++ b/engine/src/types.rs @@ -1,20 +1,15 @@ use crate::lex::{Lex, LexResult, LexWith, expect, skip_space}; -use crate::lhs_types::{Array, ArrayIntoIter, ArrayIter, Bytes, Map, MapIter, MapValuesIntoIter}; +use crate::lhs_types::{LhsValue, LhsValueSeed}; use crate::rhs_types::{ BytesExpr, IntRange, IpRange, UninhabitedArray, UninhabitedBool, UninhabitedMap, }; -use crate::scheme::{FieldIndex, IndexAccessError}; -use crate::strict_partial_ord::StrictPartialOrd; -#[cfg(feature = "get-size2")] -use get_size2::{GetSize, GetSizeTracker}; use serde::de::{DeserializeSeed, Deserializer}; use serde::{Deserialize, Serialize, Serializer}; -use std::cmp::Ordering; use std::collections::BTreeSet; use std::convert::TryFrom; use std::fmt::{self, Debug, Formatter}; use std::iter::once; -use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; +use std::net::IpAddr; use thiserror::Error; fn lex_literal_set<'i, T: Lex<'i>>(input: &'i str) -> LexResult<'i, Vec> { @@ -170,7 +165,8 @@ macro_rules! specialized_try_from { }; } -// This macro generates `Type`, `LhsValue`, `LiteralValue`, `LiteralSet`. +// This macro generates `Type`, `LiteralValue`, and `LiteralSet`. +// `lhs_types` generates `LhsValue` from the same `with_value_types` table. // // Before the parenthesis is the variant for the `Type` enum (`Type::Ip`). // First argument is the corresponding `LhsValue` variant (`LhsValue::Ip(IpAddr)`). @@ -216,46 +212,6 @@ macro_rules! declare_types { $($(# $attrs)* $name$(($val_ty))?,)* } - declare_types! { - /// An LHS value provided for filter execution. - /// - /// These are passed to the [`crate::ExecutionContext`] - /// and are used by [`crate::Filter`] for execution and comparisons. - #[derive(PartialEq, Eq, Clone, Deserialize, Hash)] - #[serde(untagged)] - enum LhsValue<'a> { - $($(# $attrs)* $(# $lhs_attrs)* $name($lhs_ty),)* - } - } - - $(impl<'a> TryFrom> for $lhs_ty { - type Error = TypeMismatchError; - - fn try_from(value: LhsValue<'a>) -> Result<$lhs_ty, TypeMismatchError> { - match value { - LhsValue::$name(value) => Ok(value), - _ => Err(TypeMismatchError { - expected: specialized_try_from!($name).into(), - actual: value.get_type(), - }), - } - } - })* - - $(impl<'a> TryFrom<&'a LhsValue<'a>> for &'a $lhs_ty { - type Error = TypeMismatchError; - - fn try_from(value: &'a LhsValue<'a>) -> Result<&'a $lhs_ty, TypeMismatchError> { - match value { - LhsValue::$name(value) => Ok(value), - _ => Err(TypeMismatchError { - expected: specialized_try_from!($name).into(), - actual: value.get_type(), - }), - } - } - })* - declare_types! { /// A literal value parsed from a filter string. #[derive(PartialEq, Eq, Clone, Hash, Serialize)] @@ -276,17 +232,6 @@ macro_rules! declare_types { } } - impl<'a> PartialOrd for LhsValue<'a> { - fn partial_cmp(&self, other: &LiteralValue) -> Option { - match (self, other) { - $((LhsValue::$name(lhs), LiteralValue::$name(rhs)) => { - lhs.strict_partial_cmp(rhs) - },)* - _ => None, - } - } - } - $(impl<'a> TryFrom for $literal_ty { type Error = TypeMismatchError; @@ -455,451 +400,6 @@ impl GetType for CompoundType { } } -impl PartialEq<&LhsValue<'_>> for LhsValue<'_> { - fn eq(&self, other: &&LhsValue<'_>) -> bool { - self.eq(*other) - } -} - -impl StrictPartialOrd for LhsValue<'_> {} - -impl PartialEq for LhsValue<'_> { - fn eq(&self, other: &LiteralValue) -> bool { - self.strict_partial_cmp(other) == Some(Ordering::Equal) - } -} - -mod private { - use super::IntoValue; - use crate::{Bytes, TypedArray, TypedMap}; - use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; - - pub trait SealedIntoValue {} - - impl SealedIntoValue for bool {} - - impl SealedIntoValue for i8 {} - impl SealedIntoValue for u8 {} - impl SealedIntoValue for i16 {} - impl SealedIntoValue for u16 {} - impl SealedIntoValue for i32 {} - impl SealedIntoValue for u32 {} - impl SealedIntoValue for i64 {} - - impl SealedIntoValue for IpAddr {} - impl SealedIntoValue for Ipv4Addr {} - impl SealedIntoValue for Ipv6Addr {} - - impl<'a, T> SealedIntoValue for T where Bytes<'a>: From {} - - impl<'a, V: IntoValue<'a>> SealedIntoValue for TypedArray<'a, V> {} - impl<'a, V: IntoValue<'a>> SealedIntoValue for TypedMap<'a, V> {} -} - -/// Converts a value into an `LhsValue`. -/// It is a stronger version of `Into>` in that -/// any value of the input type will *always* convert to the -/// same statically known `LhsValue` variant. -pub trait IntoValue<'a>: private::SealedIntoValue { - const TYPE: Type; - - fn into_value(self) -> LhsValue<'a>; -} - -impl<'a> IntoValue<'a> for bool { - const TYPE: Type = Type::Bool; - - #[inline] - fn into_value(self) -> LhsValue<'a> { - LhsValue::Bool(self) - } -} - -impl<'a> IntoValue<'a> for i64 { - const TYPE: Type = Type::Int; - - #[inline] - fn into_value(self) -> LhsValue<'a> { - LhsValue::Int(self) - } -} - -impl<'a> IntoValue<'a> for i32 { - const TYPE: Type = Type::Int; - - #[inline] - fn into_value(self) -> LhsValue<'a> { - LhsValue::Int(i64::from(self)) - } -} - -impl<'a> IntoValue<'a> for i16 { - const TYPE: Type = Type::Int; - - #[inline] - fn into_value(self) -> LhsValue<'a> { - LhsValue::Int(i64::from(self)) - } -} - -impl<'a> IntoValue<'a> for u16 { - const TYPE: Type = Type::Int; - - #[inline] - fn into_value(self) -> LhsValue<'a> { - LhsValue::Int(i64::from(self)) - } -} - -impl<'a> IntoValue<'a> for i8 { - const TYPE: Type = Type::Int; - - #[inline] - fn into_value(self) -> LhsValue<'a> { - LhsValue::Int(i64::from(self)) - } -} - -impl<'a> IntoValue<'a> for u8 { - const TYPE: Type = Type::Int; - - #[inline] - fn into_value(self) -> LhsValue<'a> { - LhsValue::Int(i64::from(self)) - } -} - -impl<'a> IntoValue<'a> for IpAddr { - const TYPE: Type = Type::Ip; - - #[inline] - fn into_value(self) -> LhsValue<'a> { - LhsValue::Ip(self) - } -} - -impl<'a> IntoValue<'a> for Ipv4Addr { - const TYPE: Type = Type::Ip; - - #[inline] - fn into_value(self) -> LhsValue<'a> { - LhsValue::Ip(IpAddr::V4(self)) - } -} - -impl<'a> IntoValue<'a> for Ipv6Addr { - const TYPE: Type = Type::Ip; - - #[inline] - fn into_value(self) -> LhsValue<'a> { - LhsValue::Ip(IpAddr::V6(self)) - } -} - -impl<'a, T> IntoValue<'a> for T -where - Bytes<'a>: From, -{ - const TYPE: Type = Type::Bytes; - - #[inline] - fn into_value(self) -> LhsValue<'a> { - LhsValue::Bytes(Bytes::from(self)) - } -} - -impl<'a, T: IntoValue<'a>> From for LhsValue<'a> { - #[inline] - fn from(value: T) -> Self { - value.into_value() - } -} - -// Array cannot implement `IntoValue` as the -// underlying element type is not statically -// known. -impl<'a> From> for LhsValue<'a> { - #[inline] - fn from(value: Array<'a>) -> LhsValue<'a> { - LhsValue::Array(value) - } -} - -// Map cannot implement `IntoValue` as the -// underlying element type is not statically -// known. -impl<'a> From> for LhsValue<'a> { - #[inline] - fn from(value: Map<'a>) -> LhsValue<'a> { - LhsValue::Map(value) - } -} - -impl<'a> TryFrom<&'a LhsValue<'a>> for &'a [u8] { - type Error = TypeMismatchError; - - fn try_from(value: &'a LhsValue<'_>) -> Result { - match value { - LhsValue::Bytes(value) => Ok(value), - _ => Err(TypeMismatchError { - expected: Type::Bytes.into(), - actual: value.get_type(), - }), - } - } -} - -impl<'a> From<&'a LiteralValue> for LhsValue<'a> { - fn from(literal: &'a LiteralValue) -> Self { - match literal { - LiteralValue::Ip(ip) => LhsValue::Ip(*ip), - LiteralValue::Bytes(bytes) => LhsValue::Bytes(Bytes::Borrowed(bytes)), - LiteralValue::Int(integer) => LhsValue::Int(*integer), - LiteralValue::Bool(b) => match *b {}, - LiteralValue::Array(a) => match *a {}, - LiteralValue::Map(m) => match *m {}, - } - } -} - -impl From for LhsValue<'_> { - fn from(literal: LiteralValue) -> Self { - match literal { - LiteralValue::Ip(ip) => LhsValue::Ip(ip), - LiteralValue::Bytes(bytes) => LhsValue::Bytes(Bytes::Owned(bytes.into())), - LiteralValue::Int(integer) => LhsValue::Int(integer), - LiteralValue::Bool(b) => match b {}, - LiteralValue::Array(a) => match a {}, - LiteralValue::Map(m) => match m {}, - } - } -} - -impl<'a> LhsValue<'a> { - /// Converts a reference to an LhsValue to an LhsValue with an internal - /// references - pub fn as_ref(&'a self) -> Self { - match self { - LhsValue::Ip(ip) => LhsValue::Ip(*ip), - LhsValue::Bytes(bytes) => LhsValue::Bytes(Bytes::Borrowed(bytes)), - LhsValue::Int(integer) => LhsValue::Int(*integer), - LhsValue::Bool(b) => LhsValue::Bool(*b), - LhsValue::Array(a) => LhsValue::Array(a.as_ref()), - LhsValue::Map(m) => LhsValue::Map(m.as_ref()), - } - } - - /// Converts an `LhsValue` with borrowed data to a fully owned `LhsValue`. - pub fn into_owned(self) -> LhsValue<'static> { - match self { - LhsValue::Ip(ip) => LhsValue::Ip(ip), - LhsValue::Bytes(bytes) => LhsValue::Bytes(Bytes::Owned(bytes.into_owned())), - LhsValue::Int(i) => LhsValue::Int(i), - LhsValue::Bool(b) => LhsValue::Bool(b), - LhsValue::Array(arr) => LhsValue::Array(arr.into_owned()), - LhsValue::Map(map) => LhsValue::Map(map.into_owned()), - } - } - - /// Retrieve an element from an LhsValue given a path item and a specified - /// type. - /// Returns a TypeMismatchError error if current type does not support it - /// nested element. - /// - /// Both LhsValue::Array and LhsValue::Map support nested elements. - pub(crate) fn get( - &'a self, - item: &FieldIndex, - ) -> Result>, IndexAccessError> { - match (self, item) { - (LhsValue::Array(arr), FieldIndex::ArrayIndex(idx)) => Ok(arr.get(*idx as usize)), - (_, FieldIndex::ArrayIndex(_)) => Err(IndexAccessError { - index: item.clone(), - actual: self.get_type(), - }), - (LhsValue::Map(map), FieldIndex::MapKey(key)) => Ok(map.get(key.as_bytes())), - (_, FieldIndex::MapKey(_)) => Err(IndexAccessError { - index: item.clone(), - actual: self.get_type(), - }), - (_, FieldIndex::MapEach) => Err(IndexAccessError { - index: item.clone(), - actual: self.get_type(), - }), - } - } - - #[inline] - pub(crate) fn get_nested(&'a self, indexes: &[FieldIndex]) -> Option<&'a LhsValue<'a>> { - indexes - .iter() - .try_fold(self, |value, idx| value.get(idx).unwrap()) - } - - pub(crate) fn extract( - self, - item: &FieldIndex, - ) -> Result>, IndexAccessError> { - match item { - FieldIndex::ArrayIndex(idx) => match self { - LhsValue::Array(arr) => Ok(arr.extract(*idx as usize)), - _ => Err(IndexAccessError { - index: item.clone(), - actual: self.get_type(), - }), - }, - FieldIndex::MapKey(key) => match self { - LhsValue::Map(map) => Ok(map.extract(key.as_bytes())), - _ => Err(IndexAccessError { - index: item.clone(), - actual: self.get_type(), - }), - }, - FieldIndex::MapEach => Err(IndexAccessError { - index: item.clone(), - actual: self.get_type(), - }), - } - } - - #[inline] - pub(crate) fn extract_nested(self, indexes: &[FieldIndex]) -> Option> { - indexes - .iter() - .try_fold(self, |value, idx| value.extract(idx).unwrap()) - } - - /// Returns an iterator over the Map or Array - pub(crate) fn iter(&'a self) -> Option> { - match self { - LhsValue::Array(array) => Some(Iter::IterArray(array.iter())), - LhsValue::Map(map) => Some(Iter::IterMap(map.iter())), - _ => None, - } - } -} - -impl Serialize for LhsValue<'_> { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - match self { - LhsValue::Ip(ip) => ip.serialize(serializer), - LhsValue::Bytes(bytes) => { - if let Ok(s) = std::str::from_utf8(bytes) { - serializer.serialize_str(s) - } else { - serializer.serialize_bytes(bytes) - } - } - LhsValue::Int(num) => num.serialize(serializer), - LhsValue::Bool(b) => b.serialize(serializer), - LhsValue::Array(arr) => arr.serialize(serializer), - LhsValue::Map(map) => map.serialize(serializer), - } - } -} - -pub(crate) struct LhsValueSeed<'a>(pub &'a Type); - -impl<'de> DeserializeSeed<'de> for LhsValueSeed<'_> { - type Value = LhsValue<'de>; - - fn deserialize(self, deserializer: D) -> Result - where - D: Deserializer<'de>, - { - match self.0 { - Type::Ip => Ok(LhsValue::Ip(std::net::IpAddr::deserialize(deserializer)?)), - Type::Int => Ok(LhsValue::Int(i64::deserialize(deserializer)?)), - Type::Bool => Ok(LhsValue::Bool(bool::deserialize(deserializer)?)), - Type::Bytes => Ok(LhsValue::Bytes(Bytes::deserialize(deserializer)?)), - Type::Array(ty) => Ok(LhsValue::Array({ - let mut arr = Array::new(*ty); - arr.deserialize(deserializer)?; - arr - })), - Type::Map(ty) => Ok(LhsValue::Map({ - let mut map = Map::new(*ty); - map.deserialize(deserializer)?; - map - })), - } - } -} - -pub enum IntoIter<'a> { - IntoArray(ArrayIntoIter<'a>), - IntoMap(MapValuesIntoIter<'a>), -} - -impl<'a> Iterator for IntoIter<'a> { - type Item = LhsValue<'a>; - - fn next(&mut self) -> Option> { - match self { - IntoIter::IntoArray(array) => array.next(), - IntoIter::IntoMap(map) => map.next(), - } - } - - fn size_hint(&self) -> (usize, Option) { - (self.len(), Some(self.len())) - } -} - -impl ExactSizeIterator for IntoIter<'_> { - fn len(&self) -> usize { - match self { - IntoIter::IntoArray(array) => array.len(), - IntoIter::IntoMap(map) => map.len(), - } - } -} - -impl<'a> IntoIterator for LhsValue<'a> { - type IntoIter = IntoIter<'a>; - type Item = LhsValue<'a>; - - fn into_iter(self) -> Self::IntoIter { - match self { - LhsValue::Array(array) => IntoIter::IntoArray(array.into_iter()), - LhsValue::Map(map) => IntoIter::IntoMap(map.into_values()), - _ => unreachable!(), - } - } -} - -pub(crate) enum Iter<'a> { - IterArray(ArrayIter<'a, 'a>), - IterMap(MapIter<'a, 'a>), -} - -impl<'a> Iterator for Iter<'a> { - type Item = &'a LhsValue<'a>; - - fn next(&mut self) -> Option<&'a LhsValue<'a>> { - match self { - Iter::IterArray(array) => array.next(), - Iter::IterMap(map) => map.next().map(|(_, v)| v), - } - } - - fn size_hint(&self) -> (usize, Option) { - (self.len(), Some(self.len())) - } -} - -impl ExactSizeIterator for Iter<'_> { - fn len(&self) -> usize { - match self { - Iter::IterArray(array) => array.len(), - Iter::IterMap(map) => map.len(), - } - } -} - #[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize, Hash, PartialOrd, Ord)] enum PrimitiveType { Bool, @@ -1039,76 +539,43 @@ impl From for Type { } } -declare_types!( - /// A boolean. - Bool(bool | UninhabitedBool | UninhabitedBool), +// Keep the variants and their storage/literal representations in one place. +// Each definition module expands its own types and implementations. +macro_rules! with_value_types { + ($declare:ident) => { + $declare! { + /// A boolean. + Bool(bool | UninhabitedBool | UninhabitedBool), - /// A 64-bit integer number. - Int(i64 | i64 | IntRange), + /// A 64-bit integer number. + Int(i64 | i64 | IntRange), - /// An IPv4 or IPv6 address. - /// - /// These are represented as a single type to allow interop comparisons. - Ip(IpAddr | IpAddr | IpRange), - - /// A raw bytes or a string field. - /// - /// These are completely interchangeable in runtime and differ only in - /// syntax representation, so we represent them as a single type. - Bytes(#[serde(borrow)] Bytes<'a> | BytesExpr | BytesExpr), + /// An IPv4 or IPv6 address. + /// + /// These are represented as a single type to allow interop comparisons. + Ip(IpAddr | IpAddr | IpRange), - /// An Array of [`Type`]. - Array[CompoundType](#[serde(skip_deserializing)] Array<'a> | UninhabitedArray | UninhabitedArray), + /// A raw bytes or a string field. + /// + /// These are completely interchangeable in runtime and differ only in + /// syntax representation, so we represent them as a single type. + Bytes(#[serde(borrow)] Bytes<'a> | BytesExpr | BytesExpr), - /// A Map of string to [`Type`]. - Map[CompoundType](#[serde(skip_deserializing)] Map<'a> | UninhabitedMap | UninhabitedMap), -); + /// An Array of [`Type`]. + Array[CompoundType](#[serde(skip_deserializing)] Array<'a> | UninhabitedArray | UninhabitedArray), -#[cfg(feature = "get-size2")] -impl GetSize for LhsValue<'_> { - fn get_heap_size_with_tracker(&self, tracker: T) -> (usize, T) { - match self { - Self::Bool(_) | Self::Int(_) | Self::Ip(_) => (0, tracker), - Self::Bytes(value) => value.get_heap_size_with_tracker(tracker), - Self::Array(value) => value.get_heap_size_with_tracker(tracker), - Self::Map(value) => value.get_heap_size_with_tracker(tracker), + /// A Map of string to [`Type`]. + Map[CompoundType](#[serde(skip_deserializing)] Map<'a> | UninhabitedMap | UninhabitedMap), } - } + }; } -#[test] -fn test_lhs_value_deserialize() { - use std::str::FromStr; - - let ipv4: LhsValue<'_> = serde_json::from_str("\"127.0.0.1\"").unwrap(); - assert_eq!(ipv4, LhsValue::Ip(IpAddr::from_str("127.0.0.1").unwrap())); - - let ipv6: LhsValue<'_> = serde_json::from_str("\"::1\"").unwrap(); - assert_eq!(ipv6, LhsValue::Ip(IpAddr::from_str("::1").unwrap())); - - let bytes: LhsValue<'_> = serde_json::from_str("\"a JSON string with unicode ❤\"").unwrap(); - assert_eq!( - bytes, - LhsValue::from(&b"a JSON string with unicode \xE2\x9D\xA4"[..]) - ); - - let bytes = - serde_json::from_str::>("\"a JSON string with escaped-unicode \\u2764\"") - .unwrap(); - assert_eq!( - bytes, - LhsValue::from(&b"a JSON string with escaped-unicode \xE2\x9D\xA4"[..]) - ); +pub(crate) use declare_types; +pub(crate) use specialized_get_type; +pub(crate) use specialized_try_from; +pub(crate) use with_value_types; - let bytes: LhsValue<'_> = serde_json::from_str("\"1337\"").unwrap(); - assert_eq!(bytes, LhsValue::from(&b"1337"[..])); - - let integer: LhsValue<'_> = serde_json::from_str("1337").unwrap(); - assert_eq!(integer, LhsValue::Int(1337)); - - let b: LhsValue<'_> = serde_json::from_str("false").unwrap(); - assert_eq!(b, LhsValue::Bool(false)); -} +with_value_types!(declare_types); #[test] fn test_type_serialize() { @@ -1185,99 +652,3 @@ fn test_type_deserialize() { Type::Array(Type::Map(Type::Bytes.into()).into()), ); } - -#[test] -fn test_size_of_lhs_value() { - assert_eq!(std::mem::size_of::>(), 48); -} - -#[cfg(feature = "get-size2")] -#[test] -fn test_lhs_value_get_size() { - let stack_size = std::mem::size_of::>(); - - assert_eq!(LhsValue::Bool(false).get_size(), stack_size); - assert_eq!(LhsValue::Int(42).get_size(), stack_size); - assert_eq!( - LhsValue::Ip(IpAddr::V4(Ipv4Addr::LOCALHOST)).get_size(), - stack_size - ); - - let borrowed_bytes = LhsValue::Bytes(Bytes::Borrowed(b"borrowed")); - assert_eq!(borrowed_bytes.get_size(), stack_size); - - let owned_bytes = LhsValue::Bytes(Bytes::Owned(Box::from(&b"owned"[..]))); - assert_eq!(owned_bytes.get_size(), stack_size + b"owned".len()); -} - -#[cfg(feature = "get-size2")] -#[test] -fn test_array_lhs_value_get_size() { - let stack_size = std::mem::size_of::>(); - - let mut borrowed_values = Vec::with_capacity(4); - borrowed_values.push(LhsValue::Bytes(Bytes::Borrowed(b"borrowed"))); - let borrowed_values_capacity = borrowed_values.capacity(); - let owned_array_of_borrowed = Array::try_from_vec(Type::Bytes, borrowed_values).unwrap(); - assert_eq!( - LhsValue::Array(owned_array_of_borrowed.as_ref()).get_size(), - stack_size - ); - assert_eq!( - LhsValue::Array(owned_array_of_borrowed).get_size(), - stack_size + borrowed_values_capacity * stack_size - ); - - let mut owned_values = Vec::with_capacity(4); - owned_values.push(LhsValue::Bytes(Bytes::Owned(Box::from(&b"owned"[..])))); - let owned_values_capacity = owned_values.capacity(); - let owned_array_of_owned = Array::try_from_vec(Type::Bytes, owned_values).unwrap(); - assert_eq!( - LhsValue::Array(owned_array_of_owned.as_ref()).get_size(), - stack_size - ); - assert_eq!( - LhsValue::Array(owned_array_of_owned).get_size(), - stack_size + owned_values_capacity * stack_size + b"owned".len() - ); -} - -#[cfg(feature = "get-size2")] -#[test] -fn test_map_lhs_value_get_size() { - let stack_size = std::mem::size_of::>(); - - let owned_map_of_borrowed = Map::try_from_iter( - Type::Bytes, - [Ok::<_, TypeMismatchError>(( - Box::from(b"key".as_slice()), - Bytes::Borrowed(b"borrowed"), - ))], - ) - .unwrap(); - assert_eq!( - LhsValue::Map(owned_map_of_borrowed.as_ref()).get_size(), - stack_size - ); - assert_eq!( - LhsValue::Map(owned_map_of_borrowed).get_size(), - stack_size + std::mem::size_of::>() + b"key".len() + stack_size - ); - - let owned_map_of_owned = Map::try_from_iter( - Type::Bytes, - [Ok::<_, TypeMismatchError>(( - Box::from(b"key".as_slice()), - Bytes::Owned(Box::from(&b"owned"[..])), - ))], - ) - .unwrap(); - assert_eq!( - LhsValue::Map(owned_map_of_owned.as_ref()).get_size(), - stack_size - ); - assert_eq!( - LhsValue::Map(owned_map_of_owned).get_size(), - stack_size + std::mem::size_of::>() + b"key".len() + stack_size + b"owned".len() - ); -} diff --git a/engine/src/values/mod.rs b/engine/src/values/mod.rs new file mode 100644 index 00000000..a6c3be83 --- /dev/null +++ b/engine/src/values/mod.rs @@ -0,0 +1,1802 @@ +//! Value interfaces, concrete collection traversal, and value conversion. +//! +//! Implement [`Value`] to associate storage with a [`FieldValueType`] and an +//! engine-provided reader. Custom containers implement [`ArrayValue`] or +//! [`MapValue`] and select [`ArrayRead`] or [`MapRead`]. Comparison execution is +//! internal: scalar comparison returns an optional boolean, while array +//! comparison appends results to an ordinary [`TypedArray`]. + +use crate::lhs_types::append_stored_lhs_at; +pub use crate::lhs_types::{LhsValueRead, LhsValueType}; +use crate::{CompoundType, FieldIndex, LhsValue, Type, TypedArray}; +use std::borrow::{Borrow, Cow}; +use std::collections::BTreeMap; +use std::marker::PhantomData; +use std::net::IpAddr; + +/// A path incompatible with the declared value type. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[error("unsupported value operation: {0}")] +pub struct ValueError(pub &'static str); + +/// A Wirefilter value type, including recursively composed containers. +/// +/// Types such as [`Bytes`] and [`Array`] describe the value independently +/// of its Rust storage and borrow lifetime. [`Self::TYPE`] is the corresponding +/// [`Type`] used by the scheme and compiler. +/// +/// Paths contain owned keys and concrete child paths. Their lifetimes are +/// independent of runtime storage or borrowed selection descriptors. +pub trait ValueType { + /// Whether this type is a scalar, for one-pass output-capacity hints. + const IS_PRIMITIVE: bool; + + /// Concrete traversal independent of a scalar predicate. + type Path: Send + Sync + 'static; + + /// Wirefilter's corresponding logical type. + const TYPE: Type; + + /// Validate a path ending at the custom predicate's declared scalar type. + fn compile_path(path: &[FieldIndex], input_type: Type) -> Result; +} + +/// Type and traversal of a field, resolved when its filter is compiled. +/// +/// Every static [`ValueType`] implements this trait automatically. [`LhsValueType`] +/// instead obtains its type from the scheme, allowing one provider to expose +/// stored values with arbitrary collection nesting. Collection elements exposed +/// through [`ArrayValue`] or [`MapValue`] still require a static [`ValueType`]; a +/// scheme-typed value is a complete field. +pub trait FieldValueType { + /// A fixed getter type, or `None` when the scheme supplies the field's type. + const STATIC_TYPE: Option; + + /// Whether the reader can compare a selected collection directly. + /// Typed collections normally convert for custom collection comparisons. + const DIRECT_COLLECTION_COMPARISON: bool = false; + + /// Compiled traversal, independent of request data and the field getter. + type Path: Send + Sync + 'static; + + /// Validate traversal from the declared field type to the comparator's input. + fn compile_field_path( + field_type: Type, + path: Vec, + input_type: Type, + ) -> Result; +} + +impl FieldValueType for S { + type Path = S::Path; + + const STATIC_TYPE: Option = Some(S::TYPE); + + fn compile_field_path( + field_type: Type, + path: Vec, + input_type: Type, + ) -> Result { + if field_type != S::TYPE { + return Err(ValueError("getter type differs from declared field type")); + } + S::compile_path(&path, input_type) + } +} + +/// A statically dispatched comparison over values and their scalar representations. +/// +/// The compiler selects the scalar comparison method from the validated path's +/// type. Implementations override that method. A compiler adapts a comparison +/// that needs the evaluation context to this interface by +/// borrowing its evaluation context; traversal does not erase the predicate or +/// construct a value enum for each element. +pub(crate) trait ValuePredicate { + /// Compare an existing stored value, including a selected collection. + fn lhs(&self, _: &LhsValue<'_>) -> bool { + panic!("predicate does not accept stored values") + } + + /// Compare a borrowed byte string. + fn bytes(&self, _: &[u8]) -> bool { + panic!("predicate does not accept byte strings") + } + + /// Compare an integer. + fn int(&self, _: i64) -> bool { + panic!("predicate does not accept integers") + } + + /// Compare an IP address. + fn ip(&self, _: IpAddr) -> bool { + panic!("predicate does not accept IP addresses") + } + + /// Compare a boolean. + fn boolean(&self, _: bool) -> bool { + panic!("predicate does not accept booleans") + } +} + +/// A value selects its logical type and an engine-provided reader. +/// +/// A custom array implements [`ArrayValue`] and selects [`ArrayRead`]; a custom +/// map implements [`MapValue`] and selects [`MapRead`]. Neither implementation +/// needs to implement comparison algorithms or construct erased value enums. +/// +/// Byte storage includes `Vec`, `Cow<[u8]>`, and [`crate::Bytes`]. The optional +/// `bytes` feature also supports `bytes::Bytes`, including inside arrays and maps. +pub trait Value { + /// Logical type, independent of the borrow lifetime of this provider. + type ValueType: FieldValueType; + + /// Engine-provided reader for the selected type and storage interface. + /// + /// Select a supplied reader such as [`BytesRead`], [`ArrayRead`], or + /// [`MapRead`]. Its execution interface is internal to Wirefilter; + /// applications customize their storage through the collection traits. + // The private bound intentionally permits selecting supported readers while + // keeping their comparison implementation inside Wirefilter. + #[allow(private_bounds)] + type Read: ReadValue; +} + +/// Convert a value into the representation used by general value expressions. +/// +/// Direct comparisons through the value reader do not require conversion. +/// Application arrays and maps materialize their container structure when passed +/// to functions or other operations requiring an [`LhsValue`]. Borrowed values +/// retain references to byte payloads. +/// +/// A descriptor returned by value may implement this trait with the default method: +/// the resulting value owns its payloads and cannot borrow the temporary descriptor. +/// Descriptors that retain longer-lived references can override it to keep those +/// payloads borrowed, as [`SliceView`] does. +pub trait IntoLhsValue<'a>: Value + Sized { + /// Whether converting a collection builds container storage. + /// + /// The compiler uses this hint to memoize converted non-mapped function + /// arguments. Borrowing existing Wirefilter containers is cheap and returns + /// `false`; application arrays and maps retain the default. This does not + /// change whether the value can be passed to a function. + const MATERIALIZES_COLLECTIONS: bool = true; + + /// Consume the value, preserving references valid for `'a` where possible. + fn into_lhs(self) -> LhsValue<'a> { + Self::Read::to_lhs(&self).into_owned() + } + + /// Consume the value and convert the selection reached by keys and indexes. + /// + /// Borrowed values select directly from their source storage and convert only + /// the selected value. + /// The default preserves custom conversion and ownership behavior by calling + /// [`Self::into_lhs`] before indexing. Borrowing descriptors can override this + /// method to select first, as [`SliceView`] does. + /// + /// Returns `None` for a missing key or index. An empty path converts the whole + /// value. Paths must contain valid array indexes and map keys, not `[*]`; + /// incompatible paths panic, as they indicate an invalid compiled expression. + #[inline] + fn into_lhs_at(self, indexes: &[FieldIndex]) -> Option> { + self.into_lhs().extract_nested(indexes) + } + + /// Consume the value and append selections reached by keys, indexes, and `[*]`. + /// + /// Missing children contribute no values. The `[*]` operator visits arrays in + /// array order and maps in the byte-key order required by [`MapValue::values`]. + /// An empty path appends the whole value, including its container structure. + /// Existing entries in `output` are preserved. + /// + /// Borrowed values select directly from their source storage and convert only + /// the selected values. + /// The default selects the fixed prefix through [`Self::into_lhs_at`], then + /// traverses the converted value, preserving custom conversion and ownership. + /// Borrowing descriptors can override this method to retain their source + /// lifetime during traversal, as [`SliceView`] does. + /// + /// Paths must be compatible with the declared type; incompatible paths panic. + #[inline] + fn append_lhs_at(self, indexes: &[FieldIndex], output: &mut Vec>) { + let prefix = indexes + .iter() + .position(|index| *index == FieldIndex::MapEach) + .unwrap_or(indexes.len()); + if let Some(value) = self.into_lhs_at(&indexes[..prefix]) { + append_stored_lhs_at(value, &indexes[prefix..], output); + } + } +} + +impl<'a, 'data: 'a, T: Value + ?Sized> IntoLhsValue<'a> for &'data T { + const MATERIALIZES_COLLECTIONS: bool = T::Read::MATERIALIZES_COLLECTIONS; + + #[inline] + fn into_lhs(self) -> LhsValue<'a> { + T::Read::to_lhs(self) + } + + #[inline] + fn into_lhs_at(self, indexes: &[FieldIndex]) -> Option> { + T::Read::to_lhs_at(self, indexes) + } + + #[inline] + fn append_lhs_at(self, indexes: &[FieldIndex], output: &mut Vec>) { + T::Read::append_lhs_at(self, indexes, output); + } +} + +/// Internal conversion and comparison operations for a value. +/// +/// The compiler chooses scalar or array evaluation before constructing the +/// closure. Both operations preserve concrete storage and predicate types. +pub(crate) trait ReadValue { + /// Cost of converting borrowed storage, used by `IntoLhsValue for &T`. + const MATERIALIZES_COLLECTIONS: bool = true; + + /// Borrow storage as an ordinary value of its declared field type. + fn to_lhs(value: &V) -> LhsValue<'_>; + + /// Convert a selection without materializing its ancestor containers. + /// Container readers consume child projections so temporary descriptors can + /// preserve their own payload lifetime through `IntoLhsValue`. + #[inline] + fn to_lhs_at<'a>(value: &'a V, indexes: &[FieldIndex]) -> Option> { + Self::to_lhs(value).extract_nested(indexes) + } + + /// Append selected values without materializing their ancestor containers. + /// Child projections are consumed so borrowed descriptors retain their source + /// lifetime instead of borrowing a temporary on the traversal stack. + #[inline] + fn append_lhs_at<'a>(value: &'a V, indexes: &[FieldIndex], output: &mut Vec>) { + assert!(indexes.is_empty(), "path continues beyond scalar"); + output.push(Self::to_lhs(value)); + } + + /// Compare the value selected by a path containing only indexes and keys. + /// A missing index or key returns `None`; `[*]` is a compiler error. + fn compare_one( + value: &V, + path: &S::Path, + predicate: &P, + ) -> Option; + + /// Append comparisons in traversal order, skipping missing indexes and keys. + /// Scalar readers append one result; container readers also handle `[*]`. + #[inline] + fn compare_vec( + value: &V, + path: &S::Path, + predicate: &P, + output: &mut TypedArray<'static, bool>, + ) { + if let Some(result) = Self::compare_one(value, path, predicate) { + output.push(result); + } + } +} + +/// Compare a scalar selection without constructing a result collector. +#[inline] +pub(crate) fn compare_one( + value: &V, + path: &::Path, + predicate: &P, +) -> Option { + V::Read::compare_one(value, path, predicate) +} + +/// Append comparisons for a possibly nested array or map selection. +#[inline] +pub(crate) fn compare_vec( + value: &V, + path: &::Path, + predicate: &P, + output: &mut TypedArray<'static, bool>, +) { + V::Read::compare_vec(value, path, predicate, output); +} + +/// A concrete array projection. Elements can be references or borrowed descriptors. +pub trait ArrayValue { + /// Stable logical type of each element. + type Element: ValueType; + + /// A projection tied to the borrow of the container. + type Item<'a>: Value + IntoLhsValue<'a> + where + Self: 'a; + + /// A concrete iterator; no trait-object iterator or exact-length scan is needed. + type Iter<'a>: Iterator> + where + Self: 'a; + + /// Retrieve a single element without erasing its type. + fn get(&self, index: usize) -> Option>; + + /// Iterate in logical array order. + fn iter(&self) -> Self::Iter<'_>; +} + +/// A byte-keyed map with concrete value projections. +pub trait MapValue { + /// Stable logical type of each map value. + type Element: ValueType; + + /// A reference or descriptor returned by value. + type Item<'a>: Value + IntoLhsValue<'a> + where + Self: 'a; + + /// A concrete iterator over map values. + type Iter<'a>: Iterator> + where + Self: 'a; + + /// Look up a key with exact byte equality, without case normalization. + fn get(&self, key: &[u8]) -> Option>; + + /// Iterate values in ascending lexicographic order of their exact byte keys. + /// + /// This must match ordinary Wirefilter maps, including after conversion for + /// function arguments. Vector logical operations combine results by position; + /// a different iteration order can change results even through an identity + /// function. Adapters over differently ordered storage must supply this order. + fn values(&self) -> Self::Iter<'_>; + + /// Visit key-value pairs when a general value expression needs a stored map. + /// + /// Keys use exact bytes and must be unique. Keys may be temporary: conversion + /// copies each key, while the returned value may borrow the map for `'a`. + fn for_each_entry<'a>(&'a self, visitor: impl FnMut(&[u8], Self::Item<'a>)); +} + +/// An array type composed with any other type. +pub struct Array(PhantomData); +/// A map type composed with any other type. +pub struct Map(PhantomData); + +/// A concrete array access followed by a concrete child path. +/// +/// This is [`ValueType::Path`] for [`Array`], with `P = S::Path`. +/// [`ValueType::compile_path`] constructs it during expression compilation; +/// [`ArrayRead`] follows it during evaluation. The path owns no request data. +/// Indices retain the parsed [`FieldIndex::ArrayIndex`] width to keep compiled +/// paths compact, and are converted to `usize` only when accessing a collection. +pub enum ArrayPath

{ + /// Select one index, then follow the child path if the element exists. + Index(u32, P), + /// Follow the child path for each element in array order. + Each(P), +} + +/// A concrete map access followed by a concrete child path. +/// +/// This is [`ValueType::Path`] for [`Map`], with `P = S::Path`. +/// [`ValueType::compile_path`] constructs it during expression compilation; +/// [`MapRead`] follows it during evaluation. Keys are owned so the path can +/// outlive both the parsed expression and any request. +pub enum MapPath

{ + /// Select an exact byte key, then follow the child path if the key exists. + Key(Box<[u8]>, P), + /// Follow the child path for each value in ascending byte-key order. + Each(P), +} + +impl ValueType for Array { + type Path = ArrayPath; + + const IS_PRIMITIVE: bool = false; + const TYPE: Type = Type::Array(CompoundType::from_type(S::TYPE)); + + fn compile_path(path: &[FieldIndex], input_type: Type) -> Result { + let (step, rest) = path + .split_first() + .ok_or(ValueError("array requires an index or [*]"))?; + match step { + FieldIndex::ArrayIndex(index) => { + Ok(ArrayPath::Index(*index, S::compile_path(rest, input_type)?)) + } + FieldIndex::MapEach => Ok(ArrayPath::Each(S::compile_path(rest, input_type)?)), + FieldIndex::MapKey(_) => Err(ValueError("map key used on array")), + } + } +} + +impl ValueType for Map { + type Path = MapPath; + + const IS_PRIMITIVE: bool = false; + const TYPE: Type = Type::Map(CompoundType::from_type(S::TYPE)); + + fn compile_path(path: &[FieldIndex], input_type: Type) -> Result { + let (step, rest) = path + .split_first() + .ok_or(ValueError("map requires a key or [*]"))?; + match step { + FieldIndex::MapKey(key) => Ok(MapPath::Key( + key.as_bytes().into(), + S::compile_path(rest, input_type)?, + )), + FieldIndex::MapEach => Ok(MapPath::Each(S::compile_path(rest, input_type)?)), + FieldIndex::ArrayIndex(_) => Err(ValueError("array index used on map")), + } + } +} + +/// Engine reader for any concrete [`ArrayValue`] implementation. +pub struct ArrayRead; + +impl ReadValue> for ArrayRead { + fn to_lhs(array: &A) -> LhsValue<'_> { + crate::Array::try_from_iter(A::Element::TYPE, array.iter().map(IntoLhsValue::into_lhs)) + .expect("array items must match their declared type") + .into() + } + + #[inline] + fn to_lhs_at<'a>(array: &'a A, indexes: &[FieldIndex]) -> Option> { + match indexes.split_first() { + None => Some(Self::to_lhs(array)), + Some((FieldIndex::ArrayIndex(index), rest)) => { + array.get(*index as usize)?.into_lhs_at(rest) + } + Some(_) => panic!("array selection requires an array index"), + } + } + + #[inline] + fn append_lhs_at<'a>(array: &'a A, indexes: &[FieldIndex], output: &mut Vec>) { + match indexes.split_first() { + None => output.push(Self::to_lhs(array)), + Some((FieldIndex::ArrayIndex(index), rest)) => { + if let Some(value) = array.get(*index as usize) { + value.append_lhs_at(rest, output); + } + } + Some((FieldIndex::MapEach, rest)) => { + let values = array.iter(); + if rest.is_empty() { + output.reserve(values.size_hint().0); + } + for value in values { + value.append_lhs_at(rest, output); + } + } + Some(_) => panic!("map key used on array"), + } + } + + #[inline] + fn compare_one( + array: &A, + path: &ArrayPath<::Path>, + predicate: &P, + ) -> Option { + match path { + ArrayPath::Index(index, child) => { + let value = array.get(*index as usize)?; + compare_one(&value, child, predicate) + } + ArrayPath::Each(_) => panic!("[*] in scalar comparison path"), + } + } + + #[inline] + fn compare_vec( + array: &A, + path: &ArrayPath<::Path>, + predicate: &P, + output: &mut TypedArray<'static, bool>, + ) { + match path { + ArrayPath::Index(index, child) => { + if let Some(value) = array.get(*index as usize) { + compare_vec(&value, child, predicate, output); + } + } + ArrayPath::Each(child) => { + let values = array.iter(); + if A::Element::IS_PRIMITIVE && output.is_empty() { + let capacity = values.size_hint().0; + if capacity != 0 { + *output = TypedArray::with_capacity(capacity); + } + } + for value in values { + compare_vec(&value, child, predicate, output); + } + } + } + } +} + +/// Engine reader for any concrete [`MapValue`] implementation. +pub struct MapRead; + +impl ReadValue> for MapRead { + fn to_lhs(map: &M) -> LhsValue<'_> { + let mut entries = Vec::with_capacity(map.values().size_hint().0); + map.for_each_entry(|key, value| { + entries.push(Ok::<_, crate::TypeMismatchError>(( + Box::<[u8]>::from(key), + value.into_lhs(), + ))); + }); + crate::Map::try_from_iter(M::Element::TYPE, entries) + .expect("map values must match their declared type") + .into() + } + + #[inline] + fn to_lhs_at<'a>(map: &'a M, indexes: &[FieldIndex]) -> Option> { + match indexes.split_first() { + None => Some(Self::to_lhs(map)), + Some((FieldIndex::MapKey(key), rest)) => map.get(key.as_bytes())?.into_lhs_at(rest), + Some(_) => panic!("map selection requires a map key"), + } + } + + #[inline] + fn append_lhs_at<'a>(map: &'a M, indexes: &[FieldIndex], output: &mut Vec>) { + match indexes.split_first() { + None => output.push(Self::to_lhs(map)), + Some((FieldIndex::MapKey(key), rest)) => { + if let Some(value) = map.get(key.as_bytes()) { + value.append_lhs_at(rest, output); + } + } + Some((FieldIndex::MapEach, rest)) => { + let values = map.values(); + if rest.is_empty() { + output.reserve(values.size_hint().0); + } + for value in values { + value.append_lhs_at(rest, output); + } + } + Some(_) => panic!("array index used on map"), + } + } + + #[inline] + fn compare_one( + map: &M, + path: &MapPath<::Path>, + predicate: &P, + ) -> Option { + match path { + MapPath::Key(key, child) => { + let value = map.get(key)?; + compare_one(&value, child, predicate) + } + MapPath::Each(_) => panic!("[*] in scalar comparison path"), + } + } + + #[inline] + fn compare_vec( + map: &M, + path: &MapPath<::Path>, + predicate: &P, + output: &mut TypedArray<'static, bool>, + ) { + match path { + MapPath::Key(key, child) => { + if let Some(value) = map.get(key) { + compare_vec(&value, child, predicate, output); + } + } + MapPath::Each(child) => { + let values = map.values(); + if M::Element::IS_PRIMITIVE && output.is_empty() { + let capacity = values.size_hint().0; + if capacity != 0 { + *output = TypedArray::with_capacity(capacity); + } + } + for value in values { + compare_vec(&value, child, predicate, output); + } + } + } + } +} + +/// Forward a borrowed provider to its concrete engine reader. +pub struct Borrowed(PhantomData); + +impl Value for &T { + type Read = Borrowed; + type ValueType = T::ValueType; +} + +impl ReadValue<&T, T::ValueType> for Borrowed { + const MATERIALIZES_COLLECTIONS: bool = T::Read::MATERIALIZES_COLLECTIONS; + + #[inline] + fn to_lhs<'a>(value: &'a &T) -> LhsValue<'a> { + T::Read::to_lhs(value) + } + + #[inline] + fn to_lhs_at<'a>(value: &'a &T, indexes: &[FieldIndex]) -> Option> { + T::Read::to_lhs_at(value, indexes) + } + + #[inline] + fn append_lhs_at<'a>(value: &'a &T, indexes: &[FieldIndex], output: &mut Vec>) { + T::Read::append_lhs_at(value, indexes, output); + } + + #[inline] + fn compare_one( + value: &&T, + path: &::Path, + predicate: &P, + ) -> Option { + compare_one::(value, path, predicate) + } + + #[inline] + fn compare_vec( + value: &&T, + path: &::Path, + predicate: &P, + output: &mut TypedArray<'static, bool>, + ) { + compare_vec::(value, path, predicate, output); + } +} + +/// Forward boxed storage to its concrete reader. +pub struct BoxRead(PhantomData); + +impl Value for Box { + type Read = BoxRead; + type ValueType = T::ValueType; +} + +impl<'a, T: Value + ?Sized> IntoLhsValue<'a> for Box { + #[inline] + fn append_lhs_at(self, indexes: &[FieldIndex], output: &mut Vec>) { + append_stored_lhs_at(self.into_lhs(), indexes, output); + } +} + +impl ReadValue, T::ValueType> for BoxRead { + const MATERIALIZES_COLLECTIONS: bool = T::Read::MATERIALIZES_COLLECTIONS; + + #[inline] + fn to_lhs(value: &Box) -> LhsValue<'_> { + T::Read::to_lhs(value) + } + + #[inline] + fn to_lhs_at<'a>(value: &'a Box, indexes: &[FieldIndex]) -> Option> { + T::Read::to_lhs_at(value, indexes) + } + + #[inline] + fn append_lhs_at<'a>( + value: &'a Box, + indexes: &[FieldIndex], + output: &mut Vec>, + ) { + T::Read::append_lhs_at(value, indexes, output); + } + + #[inline] + fn compare_one( + value: &Box, + path: &::Path, + predicate: &P, + ) -> Option { + compare_one::(value, path, predicate) + } + + #[inline] + fn compare_vec( + value: &Box, + path: &::Path, + predicate: &P, + output: &mut TypedArray<'static, bool>, + ) { + compare_vec::(value, path, predicate, output); + } +} + +impl ArrayValue for Vec +where + T::ValueType: ValueType, +{ + type Element = T::ValueType; + type Item<'a> + = &'a T + where + Self: 'a; + type Iter<'a> + = std::slice::Iter<'a, T> + where + Self: 'a; + + #[inline] + fn get(&self, index: usize) -> Option> { + self.as_slice().get(index) + } + + #[inline] + fn iter(&self) -> Self::Iter<'_> { + self.as_slice().iter() + } +} + +impl Value for Vec +where + T::ValueType: ValueType, +{ + type Read = ArrayRead; + type ValueType = Array; +} + +impl<'a, T: Value> IntoLhsValue<'a> for Vec +where + T::ValueType: ValueType, +{ + #[inline] + fn append_lhs_at(self, indexes: &[FieldIndex], output: &mut Vec>) { + append_stored_lhs_at(self.into_lhs(), indexes, output); + } +} + +impl ArrayValue for [T] +where + T::ValueType: ValueType, +{ + type Element = T::ValueType; + type Item<'a> + = &'a T + where + Self: 'a; + type Iter<'a> + = std::slice::Iter<'a, T> + where + Self: 'a; + + #[inline] + fn get(&self, index: usize) -> Option> { + <[T]>::get(self, index) + } + + #[inline] + fn iter(&self) -> Self::Iter<'_> { + <[T]>::iter(self) + } +} + +impl Value for [T] +where + T::ValueType: ValueType, +{ + type Read = ArrayRead; + type ValueType = Array; +} + +impl + Ord, V: Value> MapValue for BTreeMap +where + V::ValueType: ValueType, +{ + type Element = V::ValueType; + type Item<'a> + = &'a V + where + Self: 'a; + type Iter<'a> + = std::collections::btree_map::Values<'a, K, V> + where + Self: 'a; + + #[inline] + fn get(&self, key: &[u8]) -> Option> { + BTreeMap::get(self, key) + } + + #[inline] + fn values(&self) -> Self::Iter<'_> { + BTreeMap::values(self) + } + + #[inline] + fn for_each_entry<'a>(&'a self, mut visitor: impl FnMut(&[u8], Self::Item<'a>)) { + for (key, value) in self { + visitor(key.borrow(), value); + } + } +} + +impl + Ord, V: Value> Value for BTreeMap +where + V::ValueType: ValueType, +{ + type Read = MapRead; + type ValueType = Map; +} + +impl<'a, K: Borrow<[u8]> + Ord, V: Value> IntoLhsValue<'a> for BTreeMap +where + V::ValueType: ValueType, +{ + #[inline] + fn append_lhs_at(self, indexes: &[FieldIndex], output: &mut Vec>) { + append_stored_lhs_at(self.into_lhs(), indexes, output); + } +} + +/// A small borrowed selection descriptor that can be returned by value. +#[derive(Debug, Clone, Copy)] +pub struct SliceView<'a, T>(pub &'a [T]); + +impl ArrayValue for SliceView<'_, T> +where + T::ValueType: ValueType, +{ + type Element = T::ValueType; + type Item<'a> + = &'a T + where + Self: 'a; + type Iter<'a> + = std::slice::Iter<'a, T> + where + Self: 'a; + + #[inline] + fn get(&self, index: usize) -> Option> { + self.0.get(index) + } + + #[inline] + fn iter(&self) -> Self::Iter<'_> { + self.0.iter() + } +} + +impl Value for SliceView<'_, T> +where + T::ValueType: ValueType, +{ + type Read = ArrayRead; + type ValueType = Array; +} + +impl<'a, 'data: 'a, T: Value> IntoLhsValue<'a> for SliceView<'data, T> +where + T::ValueType: ValueType, +{ + fn into_lhs(self) -> LhsValue<'a> { + ArrayRead::to_lhs(self.0) + } + + #[inline] + fn into_lhs_at(self, indexes: &[FieldIndex]) -> Option> { + self.0.into_lhs_at(indexes) + } + + #[inline] + fn append_lhs_at(self, indexes: &[FieldIndex], output: &mut Vec>) { + self.0.append_lhs_at(indexes, output); + } +} + +/// Byte-string type; storage can be any concrete byte-slice provider. +pub struct Bytes; +/// Engine reader for a provider implementing `AsRef<[u8]>`. +pub struct BytesRead; + +fn validate_scalar_path( + path: &[FieldIndex], + input_type: Type, + scalar_type: Type, +) -> Result<(), ValueError> { + if !path.is_empty() { + Err(ValueError("path continues beyond scalar")) + } else if input_type != scalar_type { + Err(ValueError( + "predicate input type differs from selected scalar", + )) + } else { + Ok(()) + } +} + +impl ValueType for Bytes { + type Path = (); + + const IS_PRIMITIVE: bool = true; + const TYPE: Type = Type::Bytes; + + fn compile_path(path: &[FieldIndex], input_type: Type) -> Result { + validate_scalar_path(path, input_type, Type::Bytes) + } +} + +impl + ?Sized> ReadValue for BytesRead { + #[inline] + fn to_lhs(value: &T) -> LhsValue<'_> { + value.as_ref().into() + } + + #[inline] + fn compare_one(value: &T, _: &(), predicate: &P) -> Option { + Some(predicate.bytes(value.as_ref())) + } +} + +macro_rules! bytes_storage { + ($($ty:ty),* $(,)?) => {$( + impl Value for $ty { + type ValueType = Bytes; + type Read = BytesRead; + } + )*}; +} + +// `u8` deliberately has no Value implementation: Vec and [u8] are +// byte strings, while Vec and [T] compose arrays for element types implementing Value. +bytes_storage!(str, String, [u8], Vec, Cow<'_, [u8]>); + +#[cfg(feature = "bytes")] +bytes_storage!(bytes::Bytes); + +impl IntoLhsValue<'_> for String {} +impl IntoLhsValue<'_> for Vec {} + +#[cfg(feature = "bytes")] +impl IntoLhsValue<'_> for bytes::Bytes {} + +impl<'a, 'data: 'a> IntoLhsValue<'a> for Cow<'data, [u8]> { + #[inline] + fn into_lhs(self) -> LhsValue<'a> { + LhsValue::Bytes(crate::Bytes::from(self)) + } +} + +macro_rules! ordered_scalar { + ($marker:ident, $ty:ty, $variant:ident, $compare:ident, $doc:literal) => { + #[doc = $doc] + pub struct $marker; + impl ValueType for $marker { + type Path = (); + + const IS_PRIMITIVE: bool = true; + const TYPE: Type = Type::$variant; + + fn compile_path( + path: &[FieldIndex], + input_type: Type, + ) -> Result { + validate_scalar_path(path, input_type, Type::$variant) + } + } + impl Value for $ty { + type Read = $marker; + type ValueType = $marker; + } + impl ReadValue<$ty, $marker> for $marker { + #[inline] + fn to_lhs(value: &$ty) -> LhsValue<'_> { + LhsValue::$variant(*value) + } + + #[inline] + fn compare_one( + value: &$ty, + _: &(), + predicate: &P, + ) -> Option { + Some(predicate.$compare(*value)) + } + } + impl IntoLhsValue<'_> for $ty {} + }; +} + +ordered_scalar!(Int, i64, Int, int, "Integer type and its engine reader."); +ordered_scalar!(Ip, IpAddr, Ip, ip, "IP-address type and its engine reader."); + +/// Boolean type and reader for direct boolean evaluation. +pub struct Bool; + +impl ValueType for Bool { + type Path = (); + + const IS_PRIMITIVE: bool = true; + const TYPE: Type = Type::Bool; + + fn compile_path(path: &[FieldIndex], input_type: Type) -> Result { + validate_scalar_path(path, input_type, Type::Bool) + } +} + +impl Value for bool { + type Read = Bool; + type ValueType = Bool; +} + +impl ReadValue for Bool { + #[inline] + fn to_lhs(value: &bool) -> LhsValue<'_> { + LhsValue::Bool(*value) + } + + #[inline] + fn compare_one( + value: &bool, + _: &(), + predicate: &P, + ) -> Option { + Some(predicate.boolean(*value)) + } +} + +impl IntoLhsValue<'_> for bool {} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::Cell; + + fn bytes<'a>(value: &'a LhsValue<'_>) -> &'a [u8] { + match value { + LhsValue::Bytes(value) => value.as_ref(), + _ => panic!("expected bytes"), + } + } + + #[test] + fn test_bytes_materialization_preserves_payload_storage() { + let input = vec![0xff, b'a']; + let borrowed = crate::Bytes::Borrowed(&input).into_lhs(); + assert_eq!(bytes(&borrowed).as_ptr(), input.as_ptr()); + assert!(matches!( + borrowed, + LhsValue::Bytes(crate::Bytes::Borrowed(_)) + )); + + let input = input.into_boxed_slice(); + let ptr = input.as_ptr(); + let owned = crate::Bytes::Owned(input).into_lhs(); + assert_eq!(bytes(&owned).as_ptr(), ptr); + assert!(matches!(owned, LhsValue::Bytes(crate::Bytes::Owned(_)))); + + let source = vec![ + crate::Bytes::Borrowed(b"borrowed"), + crate::Bytes::from("owned".to_owned()), + ]; + let LhsValue::Array(array) = (&source).into_lhs() else { + panic!("expected an array"); + }; + for (index, original) in source.iter().enumerate() { + let value = array.get(index).unwrap(); + assert_eq!(bytes(value), original.as_ref()); + assert_eq!(bytes(value).as_ptr(), original.as_ref().as_ptr()); + assert!(matches!(value, LhsValue::Bytes(crate::Bytes::Borrowed(_)))); + } + } + + #[test] + fn test_cow_materialization_preserves_ownership() { + let input = vec![0xff, b'a']; + let borrowed = Cow::Borrowed(input.as_slice()).into_lhs(); + assert_eq!(bytes(&borrowed).as_ptr(), input.as_ptr()); + + let owned: LhsValue<'static> = Cow::<'_, [u8]>::Owned(input).into_lhs(); + assert_eq!(bytes(&owned), &[0xff, b'a']); + assert!(matches!(owned, LhsValue::Bytes(crate::Bytes::Owned(_)))); + } + + #[cfg(feature = "bytes")] + #[test] + fn test_shared_bytes_materialization_borrows_or_copies_as_needed() { + let input = bytes::Bytes::from(vec![0xff, b'a']); + let borrowed = (&input).into_lhs(); + assert_eq!(bytes(&borrowed).as_ptr(), input.as_ptr()); + assert!(matches!( + borrowed, + LhsValue::Bytes(crate::Bytes::Borrowed(_)) + )); + + // LhsValue has no shared ownership variant, so consuming the last owner + // requires an owned copy. Borrowed field getters avoid this copy. + let owned: LhsValue<'static> = input.into_lhs(); + assert_eq!(bytes(&owned), &[0xff, b'a']); + assert!(matches!(owned, LhsValue::Bytes(crate::Bytes::Owned(_)))); + } + + #[test] + fn test_nested_materialization_preserves_types_and_borrows_payloads() { + let source = BTreeMap::from([( + Box::<[u8]>::from(&b"Mixed\xff"[..]), + vec![Box::::from("first"), Box::::from("second")].into_boxed_slice(), + )]); + let value = (&source).into_lhs(); + let LhsValue::Map(map) = value else { + panic!("expected a map") + }; + assert_eq!(map.value_type(), Type::array(Type::Bytes)); + assert!(map.get(b"mixed\xff").is_none()); + let LhsValue::Array(array) = map.get(b"Mixed\xff").unwrap() else { + panic!("expected an array") + }; + assert_eq!(array.value_type(), Type::Bytes); + let source_values = source.get(&b"Mixed\xff"[..]).unwrap(); + for (index, text) in source_values.iter().enumerate() { + let actual = bytes(array.get(index).unwrap()); + assert_eq!(actual, text.as_bytes()); + assert_eq!(actual.as_ptr(), text.as_ptr()); + } + + let empty: Vec, Vec>> = Vec::new(); + let LhsValue::Array(array) = (&empty).into_lhs() else { + panic!("expected an array") + }; + assert_eq!(array.value_type(), Type::map(Type::array(Type::Int))); + assert!(array.is_empty()); + } + + struct SelectionMap(Vec); + + impl Value for SelectionMap { + type Read = MapRead; + type ValueType = Map>; + } + + impl MapValue for SelectionMap { + type Element = Array; + type Item<'a> = SliceView<'a, String>; + type Iter<'a> = std::iter::Once>; + + fn get(&self, key: &[u8]) -> Option> { + (key == b"accept").then_some(SliceView(&self.0)) + } + + fn values(&self) -> Self::Iter<'_> { + std::iter::once(SliceView(&self.0)) + } + + fn for_each_entry<'a>(&'a self, mut visitor: impl FnMut(&[u8], Self::Item<'a>)) { + visitor(b"accept", SliceView(&self.0)); + } + } + + #[test] + fn test_temporary_array_selection_can_preserve_borrowed_payloads() { + let source = SelectionMap(vec!["text/html".into()]); + // The selection descriptor is consumed here; only its payload outlives it. + let selected = source.get(b"accept").unwrap().into_lhs(); + let LhsValue::Array(array) = selected else { + panic!("expected an array") + }; + assert_eq!(bytes(array.get(0).unwrap()).as_ptr(), source.0[0].as_ptr()); + + let LhsValue::Map(map) = (&source).into_lhs() else { + panic!("expected a map") + }; + let LhsValue::Array(array) = map.get(b"accept").unwrap() else { + panic!("expected an array") + }; + assert_eq!(bytes(array.get(0).unwrap()).as_ptr(), source.0[0].as_ptr()); + } + + #[test] + fn test_indexed_materialization_preserves_borrowed_descriptors() { + let source = Box::new(SelectionMap(vec!["text/html".into()])); + let reference = &source; + let indexes = [ + FieldIndex::MapKey("accept".into()), + FieldIndex::ArrayIndex(0), + ]; + // The map creates a temporary SliceView. Its consuming conversion must + // retain the source lifetime through both indexes and reader wrappers. + for selected in [ + reference.into_lhs_at(&indexes).unwrap(), + (&reference).into_lhs_at(&indexes).unwrap(), + SliceView(source.0.as_slice()) + .into_lhs_at(&indexes[1..]) + .unwrap(), + ] { + assert!(matches!( + selected, + LhsValue::Bytes(crate::Bytes::Borrowed(_)) + )); + assert_eq!(bytes(&selected).as_ptr(), source.0[0].as_ptr()); + } + assert!( + reference + .into_lhs_at(&[FieldIndex::MapKey("missing".into())]) + .is_none() + ); + assert!( + reference + .into_lhs_at(&[ + FieldIndex::MapKey("accept".into()), + FieldIndex::ArrayIndex(1), + ]) + .is_none() + ); + } + + #[test] + fn test_map_each_materialization_preserves_borrowed_descriptors() { + let source = Box::new(SelectionMap(vec!["text/html".into(), "text/plain".into()])); + let reference = &source; + for indexes in [ + vec![FieldIndex::MapEach, FieldIndex::MapEach], + vec![FieldIndex::MapKey("accept".into()), FieldIndex::MapEach], + ] { + // Each map item is a temporary SliceView, consumed during traversal. + // Both wrapper readers must preserve its references to the source. + let mut actual = vec![LhsValue::Bool(true)]; + reference.append_lhs_at(&indexes, &mut actual); + (&reference).append_lhs_at(&indexes, &mut actual); + assert_eq!(actual[0], LhsValue::Bool(true)); + assert_eq!(actual.len(), 5); + for (value, original) in actual[1..].iter().zip(source.0.iter().cycle()) { + assert!(matches!(value, LhsValue::Bytes(crate::Bytes::Borrowed(_)))); + assert_eq!(bytes(value).as_ptr(), original.as_ptr()); + } + } + + let mut selected = Vec::new(); + reference.append_lhs_at( + &[FieldIndex::MapEach, FieldIndex::ArrayIndex(1)], + &mut selected, + ); + assert_eq!(selected.len(), 1); + assert_eq!(bytes(&selected[0]).as_ptr(), source.0[1].as_ptr()); + for indexes in [ + vec![FieldIndex::MapEach, FieldIndex::ArrayIndex(99)], + vec![FieldIndex::MapKey("missing".into()), FieldIndex::MapEach], + ] { + reference.append_lhs_at(&indexes, &mut selected); + assert_eq!(selected.len(), 1); + } + + // Stopping at a container preserves that container, rather than + // flattening beyond the specified `[*]`. + let mut containers = Vec::new(); + reference.append_lhs_at(&[FieldIndex::MapEach], &mut containers); + assert_eq!(containers.len(), 1); + let LhsValue::Array(array) = &containers[0] else { + panic!("expected the selected array"); + }; + assert_eq!(array.len(), 2); + assert_eq!(bytes(array.get(0).unwrap()).as_ptr(), source.0[0].as_ptr()); + } + + #[test] + fn test_indexed_descriptor_fallback_preserves_custom_conversion() { + struct Descriptor<'a> { + values: &'a [String], + conversions: &'a Cell, + } + impl Value for Descriptor<'_> { + type Read = ArrayRead; + type ValueType = Array; + } + impl ArrayValue for Descriptor<'_> { + type Element = Bytes; + type Item<'a> + = &'a String + where + Self: 'a; + type Iter<'a> + = std::slice::Iter<'a, String> + where + Self: 'a; + + fn get(&self, index: usize) -> Option> { + self.values.get(index) + } + + fn iter(&self) -> Self::Iter<'_> { + self.values.iter() + } + } + impl<'a> IntoLhsValue<'a> for Descriptor<'a> { + fn into_lhs(self) -> LhsValue<'a> { + self.conversions.set(self.conversions.get() + 1); + self.values.into_lhs() + } + } + + let values = vec!["borrowed".to_owned()]; + let conversions = Cell::new(0); + let selected = Descriptor { + values: &values, + conversions: &conversions, + } + .into_lhs_at(&[FieldIndex::ArrayIndex(0)]) + .unwrap(); + assert_eq!(conversions.get(), 1); + assert!(matches!( + selected, + LhsValue::Bytes(crate::Bytes::Borrowed(_)) + )); + assert_eq!(bytes(&selected).as_ptr(), values[0].as_ptr()); + + let mut flattened = Vec::new(); + Descriptor { + values: &values, + conversions: &conversions, + } + .append_lhs_at(&[FieldIndex::MapEach], &mut flattened); + assert_eq!(conversions.get(), 2); + assert_eq!(flattened.len(), 1); + assert!(matches!( + flattened[0], + LhsValue::Bytes(crate::Bytes::Borrowed(_)) + )); + assert_eq!(bytes(&flattened[0]).as_ptr(), values[0].as_ptr()); + + // The same fallback must also remain safe for a temporary owning its + // payload: after it is consumed, the selected result owns its bytes. + let selected: LhsValue<'static> = vec!["owned".to_owned()] + .into_lhs_at(&[FieldIndex::ArrayIndex(0)]) + .unwrap(); + assert!(matches!(selected, LhsValue::Bytes(crate::Bytes::Owned(_)))); + assert_eq!(bytes(&selected), b"owned"); + + let mut flattened: Vec> = Vec::new(); + vec![vec!["owned".to_owned()], vec![]].append_lhs_at( + &[FieldIndex::MapEach, FieldIndex::ArrayIndex(0)], + &mut flattened, + ); + assert_eq!(flattened.len(), 1); + assert!(matches!( + flattened[0], + LhsValue::Bytes(crate::Bytes::Owned(_)) + )); + assert_eq!(bytes(&flattened[0]), b"owned"); + } + + #[test] + fn test_map_each_fallback_preserves_custom_indexed_conversion() { + struct IndexedSelection<'a>(&'a [Vec]); + + impl Value for IndexedSelection<'_> { + type Read = ArrayRead; + type ValueType = Array>; + } + + impl ArrayValue for IndexedSelection<'_> { + type Element = Array; + type Item<'a> + = &'a Vec + where + Self: 'a; + type Iter<'a> + = std::slice::Iter<'a, Vec> + where + Self: 'a; + + fn get(&self, index: usize) -> Option> { + self.0.get(index) + } + + fn iter(&self) -> Self::Iter<'_> { + self.0.iter() + } + } + + impl<'a> IntoLhsValue<'a> for IndexedSelection<'a> { + fn into_lhs(self) -> LhsValue<'a> { + panic!("the custom indexed conversion must be used"); + } + + fn into_lhs_at(self, indexes: &[FieldIndex]) -> Option> { + self.0.into_lhs_at(indexes) + } + } + + let source = vec![vec!["first".to_owned(), "second".to_owned()]]; + let mut selected = Vec::new(); + IndexedSelection(&source).append_lhs_at( + &[FieldIndex::ArrayIndex(0), FieldIndex::MapEach], + &mut selected, + ); + assert_eq!(selected.len(), 2); + for (value, original) in selected.iter().zip(&source[0]) { + assert!(matches!(value, LhsValue::Bytes(crate::Bytes::Borrowed(_)))); + assert_eq!(bytes(value).as_ptr(), original.as_ptr()); + } + } + + #[test] + fn test_map_each_preserves_custom_child_append() { + struct AppendingSelection<'a>(&'a [String]); + + impl Value for AppendingSelection<'_> { + type Read = ArrayRead; + type ValueType = Array; + } + + impl ArrayValue for AppendingSelection<'_> { + type Element = Bytes; + type Item<'a> + = &'a String + where + Self: 'a; + type Iter<'a> + = std::slice::Iter<'a, String> + where + Self: 'a; + + fn get(&self, _: usize) -> Option> { + panic!("the custom append method must be used"); + } + + fn iter(&self) -> Self::Iter<'_> { + panic!("the custom append method must be used"); + } + } + + impl<'a> IntoLhsValue<'a> for AppendingSelection<'a> { + fn into_lhs(self) -> LhsValue<'a> { + panic!("the custom append method must be used"); + } + + fn into_lhs_at(self, _: &[FieldIndex]) -> Option> { + panic!("the custom append method must be used"); + } + + fn append_lhs_at(self, indexes: &[FieldIndex], output: &mut Vec>) { + assert_eq!(indexes, &[FieldIndex::ArrayIndex(0)]); + self.0.append_lhs_at(indexes, output); + } + } + + struct MapWithSelection(Vec); + + impl Value for MapWithSelection { + type Read = MapRead; + type ValueType = Map>; + } + + impl MapValue for MapWithSelection { + type Element = Array; + type Item<'a> = AppendingSelection<'a>; + type Iter<'a> = std::iter::Once>; + + fn get(&self, key: &[u8]) -> Option> { + (key == b"values").then_some(AppendingSelection(&self.0)) + } + + fn values(&self) -> Self::Iter<'_> { + std::iter::once(AppendingSelection(&self.0)) + } + + fn for_each_entry<'a>(&'a self, mut visitor: impl FnMut(&[u8], Self::Item<'a>)) { + visitor(b"values", AppendingSelection(&self.0)); + } + } + + let source = MapWithSelection(vec!["selected".to_owned(), "other".to_owned()]); + let mut selected = vec![LhsValue::Bool(true)]; + (&source).append_lhs_at( + &[FieldIndex::MapEach, FieldIndex::ArrayIndex(0)], + &mut selected, + ); + assert_eq!(selected.len(), 2); + assert_eq!(selected[0], LhsValue::Bool(true)); + assert!(matches!( + selected[1], + LhsValue::Bytes(crate::Bytes::Borrowed(_)) + )); + assert_eq!(bytes(&selected[1]).as_ptr(), source.0[0].as_ptr()); + } + + #[test] + fn test_owned_containers_append_selected_values_with_owned_payloads() { + fn check(value: impl IntoLhsValue<'static>, indexes: &[FieldIndex]) { + let mut selected = vec![LhsValue::Bool(true)]; + value.append_lhs_at(indexes, &mut selected); + assert_eq!(selected.len(), 2); + assert_eq!(selected[0], LhsValue::Bool(true)); + assert!(matches!( + selected[1], + LhsValue::Bytes(crate::Bytes::Owned(_)) + )); + assert_eq!(bytes(&selected[1]), b"selected"); + } + + let rows = vec![vec!["selected".to_owned()], vec![]]; + check( + vec![vec![vec!["other".to_owned()]], rows.clone()], + &[ + FieldIndex::ArrayIndex(1), + FieldIndex::MapEach, + FieldIndex::ArrayIndex(0), + ], + ); + check( + Box::new(rows.clone()), + &[FieldIndex::MapEach, FieldIndex::ArrayIndex(0)], + ); + check( + BTreeMap::from([ + (Box::<[u8]>::from(&b"wanted"[..]), rows), + ( + Box::<[u8]>::from(&b"other"[..]), + vec![vec!["other".to_owned()]], + ), + ]), + &[ + FieldIndex::MapKey("wanted".into()), + FieldIndex::MapEach, + FieldIndex::ArrayIndex(0), + ], + ); + } + + #[test] + fn test_default_descriptor_conversion_owns_temporary_payload() { + struct Temporary(Box<[u8]>); + + impl AsRef<[u8]> for Temporary { + fn as_ref(&self) -> &[u8] { + &self.0 + } + } + + impl Value for Temporary { + type Read = BytesRead; + type ValueType = Bytes; + } + + impl IntoLhsValue<'_> for Temporary {} + + let result: LhsValue<'static> = Temporary(Box::from(&b"temporary"[..])).into_lhs(); + assert_eq!(bytes(&result), b"temporary"); + } + + #[test] + fn test_owned_primitive_and_collection_conversion() { + assert_eq!(42i64.into_lhs(), LhsValue::Int(42)); + assert_eq!(true.into_lhs(), LhsValue::Bool(true)); + let ip = IpAddr::V4(std::net::Ipv4Addr::LOCALHOST); + assert_eq!(ip.into_lhs(), LhsValue::Ip(ip)); + let value: LhsValue<'static> = vec!["one".to_owned()].into_lhs(); + let LhsValue::Array(array) = value else { + panic!("expected an array") + }; + assert_eq!(bytes(array.get(0).unwrap()), b"one"); + } + + struct Matches<'a> { + needle: &'a [u8], + reads: &'a Cell, + } + + impl ValuePredicate for Matches<'_> { + fn bytes(&self, value: &[u8]) -> bool { + self.reads.set(self.reads.get() + 1); + value == self.needle + } + } + + #[test] + fn test_scalar_comparison_distinguishes_false_from_missing() { + type Input = Box, Vec>>>; + let input: Input = Box::new(vec![BTreeMap::from([( + Box::from(b"key".as_slice()), + vec!["miss".into(), "hit".into()], + )])]); + let reads = Cell::new(0); + let predicate = Matches { + needle: b"hit", + reads: &reads, + }; + // Exercise both reference and Box forwarding through nested selections. + for (outer, key, inner, expected) in [ + (0, "key", 0, Some(false)), + (0, "key", 1, Some(true)), + (0, "key", 2, None), + (0, "absent", 0, None), + (1, "key", 0, None), + (u32::MAX, "key", 0, None), + (0, "key", u32::MAX, None), + ] { + let path = ::ValueType::compile_path( + &[ + FieldIndex::ArrayIndex(outer), + FieldIndex::MapKey(key.into()), + FieldIndex::ArrayIndex(inner), + ], + Type::Bytes, + ) + .unwrap(); + assert_eq!(compare_one(&&input, &path, &predicate), expected); + } + assert_eq!(reads.get(), 2); + } + + #[test] + fn test_array_comparison_appends_in_map_order() { + type Input = Box, Vec>>; + let input: Input = Box::new(BTreeMap::from([ + (Box::from(b"c".as_slice()), vec!["hit".into()]), + ( + Box::from(b"a".as_slice()), + vec!["miss".into(), "hit".into()], + ), + (Box::from(b"b".as_slice()), vec![]), + ])); + let reads = Cell::new(0); + let predicate = Matches { + needle: b"hit", + reads: &reads, + }; + let path = ::ValueType::compile_path( + &[FieldIndex::MapEach, FieldIndex::MapEach], + Type::Bytes, + ) + .unwrap(); + let mut output = TypedArray::from_iter([true]); + compare_vec(&&input, &path, &predicate, &mut output); + assert_eq!(output, [true, false, true, true]); + assert_eq!(reads.get(), 3); + + // Scalar-valued maps also append without replacing earlier results. + let values = BTreeMap::from([(Box::<[u8]>::from(b"a".as_slice()), "miss".to_owned())]); + let path = Map::::compile_path(&[FieldIndex::MapEach], Type::Bytes).unwrap(); + compare_vec(&values, &path, &predicate, &mut output); + assert_eq!(output, [true, false, true, true, false]); + assert_eq!(reads.get(), 4); + } + + #[test] + #[should_panic(expected = "[*] in scalar comparison path")] + fn test_scalar_array_comparison_rejects_map_each() { + let path = Array::::compile_path(&[FieldIndex::MapEach], Type::Bytes).unwrap(); + compare_one( + &Vec::::new(), + &path, + &Matches { + needle: b"hit", + reads: &Cell::new(0), + }, + ); + } + + #[test] + #[should_panic(expected = "[*] in scalar comparison path")] + fn test_scalar_map_comparison_rejects_map_each() { + let path = Map::::compile_path(&[FieldIndex::MapEach], Type::Bytes).unwrap(); + compare_one( + &BTreeMap::, String>::new(), + &path, + &Matches { + needle: b"hit", + reads: &Cell::new(0), + }, + ); + } + + #[test] + fn test_custom_predicate_traverses_nested_projections_without_missing_values() { + type Input = Vec, Vec>>; + let key = Box::<[u8]>::from(&b"key"[..]); + let input: Input = vec![ + BTreeMap::from([(key.clone(), vec!["one".into(), "two".into()])]), + BTreeMap::new(), + BTreeMap::from([(key, vec!["two".into()])]), + ]; + let reads = Cell::new(0); + let predicate = Matches { + needle: b"two", + reads: &reads, + }; + let path = ::ValueType::compile_path( + &[ + FieldIndex::MapEach, + FieldIndex::MapKey("key".into()), + FieldIndex::MapEach, + ], + Type::Bytes, + ) + .unwrap(); + let mut results = TypedArray::new(); + compare_vec(&input, &path, &predicate, &mut results); + assert_eq!(results, [false, true, true]); + assert_eq!(reads.get(), 3); + + let absent = ::ValueType::compile_path( + &[ + FieldIndex::ArrayIndex(8), + FieldIndex::MapKey("key".into()), + FieldIndex::ArrayIndex(0), + ], + Type::Bytes, + ) + .unwrap(); + assert_eq!(compare_one(&input, &absent, &predicate), None); + assert_eq!(reads.get(), 3); + + for indexes in [ + [ + FieldIndex::ArrayIndex(u32::MAX), + FieldIndex::MapKey("key".into()), + FieldIndex::MapEach, + ], + [ + FieldIndex::MapEach, + FieldIndex::MapKey("key".into()), + FieldIndex::ArrayIndex(u32::MAX), + ], + ] { + let path = ::ValueType::compile_path(&indexes, Type::Bytes).unwrap(); + compare_vec(&input, &path, &predicate, &mut results); + assert_eq!(results, [false, true, true]); + assert_eq!(reads.get(), 3); + } + } + + #[test] + fn test_custom_predicate_path_validates_value_type() { + type Nested = Array>>; + let path = [ + FieldIndex::MapEach, + FieldIndex::MapKey("key".into()), + FieldIndex::MapEach, + ]; + assert!(Nested::compile_path(&path, Type::Bytes).is_ok()); + assert!(Nested::compile_path(&path, Type::Int).is_err()); + assert!(Nested::compile_path(&path[..2], Type::Bytes).is_err()); + assert!( + Array::::compile_path(&[FieldIndex::MapKey("key".into())], Type::Int).is_err() + ); + assert!(Map::::compile_path(&[FieldIndex::ArrayIndex(0)], Type::Int).is_err()); + assert!(Int::compile_path(&[FieldIndex::MapEach], Type::Int).is_err()); + assert!(Int::compile_path(&[], Type::Int).is_ok()); + assert!(Ip::compile_path(&[], Type::Ip).is_ok()); + assert!(Bool::compile_path(&[], Type::Bool).is_ok()); + assert!(Bytes::compile_path(&[], Type::array(Type::Bytes)).is_err()); + } + + #[test] + fn test_custom_predicate_receives_each_scalar_kind() { + struct IntPredicate; + impl ValuePredicate for IntPredicate { + fn int(&self, value: i64) -> bool { + value == 42 + } + } + struct IpPredicate; + impl ValuePredicate for IpPredicate { + fn ip(&self, value: IpAddr) -> bool { + value.is_loopback() + } + } + struct BoolPredicate; + impl ValuePredicate for BoolPredicate { + fn boolean(&self, value: bool) -> bool { + !value + } + } + let mut results = TypedArray::new(); + compare_vec(&42i64, &(), &IntPredicate, &mut results); + compare_vec( + &IpAddr::V4(std::net::Ipv4Addr::LOCALHOST), + &(), + &IpPredicate, + &mut results, + ); + compare_vec(&false, &(), &BoolPredicate, &mut results); + assert_eq!(results, [true, true, true]); + } +} diff --git a/engine/tests/native_collection_comparisons.rs b/engine/tests/native_collection_comparisons.rs new file mode 100644 index 00000000..d795bd91 --- /dev/null +++ b/engine/tests/native_collection_comparisons.rs @@ -0,0 +1,291 @@ +//! Custom collection comparisons select native storage before materialization. + +use std::cell::RefCell; +use std::collections::BTreeMap; +use std::sync::Arc; +use wirefilter::values::{ + self, ArrayRead, ArrayValue, IntoLhsValue, MapRead, MapValue, Value, ValueType, +}; +use wirefilter::{ + Compare, CompiledExpr, ContextFamily, ContextState, DefaultCompiler, EvaluationContext, + ExecutionContext, FieldIndex, HasScheme, IdentifierExpr, IndexExpr, LhsValue, List, + ListMatcher, ListRef, Scheme, SchemeBuilder, Type, +}; + +thread_local! { + static MATERIALIZED: RefCell> = const { RefCell::new(Vec::new()) }; +} + +struct CountedArray { + label: String, + items: Vec, +} + +impl Value for CountedArray +where + T::ValueType: ValueType, +{ + type Read = ArrayRead; + type ValueType = values::Array; +} + +impl ArrayValue for CountedArray +where + T::ValueType: ValueType, +{ + type Element = T::ValueType; + type Item<'a> + = &'a T + where + Self: 'a; + type Iter<'a> + = std::slice::Iter<'a, T> + where + Self: 'a; + + fn get(&self, index: usize) -> Option> { + self.items.get(index) + } + + fn iter(&self) -> Self::Iter<'_> { + MATERIALIZED.with_borrow_mut(|values| values.push(self.label.clone())); + self.items.iter() + } +} + +type Row = CountedArray; +type Section = CountedArray; +type Group = CountedArray

; +type Groups = BTreeMap, Group>; + +// This by-value descriptor intentionally keeps the default into_lhs_at method. +#[derive(Clone)] +struct Descriptor(Arc); + +impl Value for Descriptor { + type Read = MapRead; + type ValueType = values::Map<::ValueType>; +} + +impl IntoLhsValue<'_> for Descriptor {} + +impl MapValue for Descriptor { + type Element = ::ValueType; + type Item<'a> = &'a Group; + type Iter<'a> = std::collections::btree_map::Values<'a, Box<[u8]>, Group>; + + fn get(&self, key: &[u8]) -> Option> { + self.0.get(key) + } + + fn values(&self) -> Self::Iter<'_> { + self.0.values() + } + + fn for_each_entry<'a>(&'a self, mut visitor: impl FnMut(&[u8], Self::Item<'a>)) { + for (key, value) in self.0.iter() { + visitor(key, value); + } + } +} + +struct Context { + state: ContextState<()>, + groups: Option>, +} + +impl HasScheme for Context { + fn scheme(&self) -> &Scheme { + self.state.scheme() + } +} + +impl EvaluationContext for Context { + type UserData = (); + + fn get_user_data(&self) -> &() { + &() + } + + fn get_list_matcher(&self, list: ListRef<'_>) -> &dyn ListMatcher { + self.state.get_list_matcher(list) + } + + fn get_list_matcher_unchecked(&self, list: &List) -> &dyn ListMatcher { + EvaluationContext::get_list_matcher_unchecked(&self.state, list) + } +} + +wirefilter::wirefilter_fields! { + impl Fields for Context { + "groups" => { + fn(context: &Context) -> Option<&Groups> { + context.groups.as_deref() + } + }, + "owned" => { + fn(context: &Context) -> Option { + context.groups.clone().map(Descriptor) + } + }, + } +} + +struct Nonempty; + +impl> Compare<(), F> for Nonempty { + fn compare<'ctx, 'data: 'ctx>( + &self, + value: &LhsValue<'ctx>, + _: &'ctx F::Context<'data>, + ) -> bool { + match value { + LhsValue::Array(array) => !array.is_empty(), + LhsValue::Map(map) => !map.is_empty(), + _ => panic!("custom comparator requires a collection"), + } + } +} + +fn fixture(present: bool) -> Context { + let mut builder = SchemeBuilder::new(); + let ty = Type::map(Type::array(Type::array(Type::array(Type::Bytes)))); + builder.add_optional_field("groups", ty).unwrap(); + builder.add_optional_field("owned", ty).unwrap(); + let scheme = builder.build(); + Fields::validate_scheme(&scheme).unwrap(); + let groups = present.then(|| { + Arc::new( + ["wanted", "other"] + .into_iter() + .map(|name| { + let group = CountedArray { + label: name.into(), + items: (0..2) + .map(|section| CountedArray { + label: format!("{name}/{section}"), + items: (0..2) + .map(|row| CountedArray { + label: format!("{name}/{section}/{row}"), + items: if row == 0 { + vec!["value".into()] + } else { + vec![] + }, + }) + .collect(), + }) + .collect(), + }; + (Box::<[u8]>::from(name.as_bytes()), group) + }) + .collect(), + ) + }); + Context { + state: ContextState::new(&scheme), + groups, + } +} + +fn evaluate(native: &Context, field: &str, indexes: Vec, default: bool) -> Vec { + let scheme = native.scheme(); + let mut stored = ExecutionContext::new(scheme); + if let Some(groups) = &native.groups { + stored + .set_field_value_from_name(field, groups.as_ref().into_lhs()) + .unwrap(); + } + let index = IndexExpr { + identifier: IdentifierExpr::Field(scheme.get_field(field).unwrap().to_owned()), + indexes: indexes.into_boxed_slice(), + }; + let stored_expr = + index + .clone() + .compile_with(&mut DefaultCompiler::<()>::new(), default, Nonempty); + let native_expr = + index.compile_with(&mut DefaultCompiler::<(), Fields>::new(), default, Nonempty); + MATERIALIZED.with_borrow_mut(Vec::clear); + match (stored_expr, native_expr) { + (CompiledExpr::One(stored_expr), CompiledExpr::One(native_expr)) => { + assert_eq!(stored_expr.execute(&stored), native_expr.execute(native)); + } + (CompiledExpr::Vec(stored_expr), CompiledExpr::Vec(native_expr)) => { + assert_eq!(stored_expr.execute(&stored), native_expr.execute(native)); + } + _ => panic!("comparison cardinality changed"), + } + MATERIALIZED.with_borrow_mut(std::mem::take) +} + +#[test] +fn borrowed_comparisons_only_materialize_the_selected_subtree() { + use FieldIndex::{ArrayIndex as At, MapEach as Each, MapKey}; + let native = fixture(true); + for (path, selected) in [ + (vec![MapKey("wanted".into()), At(0), At(0)], "wanted/0/0"), + (vec![MapKey("wanted".into()), At(0), Each], "wanted/0"), + (vec![MapKey("wanted".into()), Each, At(0)], "wanted"), + (vec![MapKey("wanted".into()), Each, Each], "wanted"), + // Missing children after [*] still produce the same empty comparison vector. + (vec![MapKey("wanted".into()), Each, At(99)], "wanted"), + ] { + let materialized = evaluate(&native, "groups", path, false); + assert!(!materialized.is_empty()); + assert!( + materialized.iter().all(|name| name.starts_with(selected)), + "converted unrelated containers: {materialized:?}" + ); + } +} + +#[test] +fn missing_selections_and_optional_roots_keep_stored_comparison_semantics() { + use FieldIndex::{ArrayIndex as At, MapEach as Each, MapKey}; + let present = fixture(true); + let absent = fixture(false); + for default in [false, true] { + for path in [ + vec![MapKey("missing".into()), At(0)], + vec![MapKey("wanted".into()), At(99)], + vec![MapKey("missing".into()), Each], + vec![MapKey("missing".into()), Each, At(0)], + vec![MapKey("wanted".into()), At(99), Each], + ] { + assert!(evaluate(&present, "groups", path, default).is_empty()); + } + for path in [ + vec![MapKey("wanted".into()), At(0)], + vec![MapKey("wanted".into()), Each], + vec![MapKey("wanted".into()), Each, At(0)], + ] { + assert!(evaluate(&absent, "groups", path, default).is_empty()); + } + } +} + +#[test] +fn by_value_descriptors_keep_the_default_conversion_before_selection() { + use FieldIndex::{ArrayIndex as At, MapEach as Each, MapKey}; + let native = fixture(true); + for path in [ + vec![MapKey("wanted".into()), At(0), At(0)], + vec![MapKey("wanted".into()), At(0), Each], + vec![MapKey("wanted".into()), Each, At(0)], + ] { + let materialized = evaluate(&native, "owned", path, false); + assert!(materialized.iter().any(|name| name == "wanted")); + assert!(materialized.iter().any(|name| name == "other")); + } +} + +#[test] +fn comparisons_without_a_concrete_prefix_still_materialize_the_whole_map() { + let native = fixture(true); + for path in [vec![], vec![FieldIndex::MapEach]] { + let materialized = evaluate(&native, "groups", path, false); + assert!(materialized.iter().any(|name| name == "wanted")); + assert!(materialized.iter().any(|name| name == "other")); + } +} diff --git a/engine/tests/native_family_validation.rs b/engine/tests/native_family_validation.rs new file mode 100644 index 00000000..82c9eaf7 --- /dev/null +++ b/engine/tests/native_family_validation.rs @@ -0,0 +1,164 @@ +//! Startup validation for context families implemented without the field macro. + +use std::cell::RefCell; +use std::collections::BTreeMap; +use wirefilter::{ + BorrowedValueProvider, ContextFamily, ExecutionContext, Field, FieldBindingError, + FieldConsumer, FieldDefinitionError, FieldGetter, LhsValue, LhsValueProvider, + OwnedValueProvider, Scheme, SchemeBuilder, SchemeValidationError, Type, +}; + +thread_local! { + static BINDINGS: RefCell> = const { RefCell::new(Vec::new()) }; +} + +struct Fields; +struct IntegerGetter; +struct HeadersGetter; +struct StoredGetter; + +type Headers = BTreeMap, Vec>; + +impl ContextFamily for Fields { + type Context<'data> = ExecutionContext<'data>; + type UserData = (); + + fn bind>( + field: Field, + consumer: B, + ) -> Result { + BINDINGS.with_borrow_mut(|bindings| bindings.push(field.name().into())); + match field.name() { + "count" | "after_errors" => consumer.consume(field, IntegerGetter), + "headers" => consumer.consume(field, HeadersGetter), + "stored_scalar" | "stored_collection" => consumer.consume(field, StoredGetter), + "unsupported" => Err(FieldBindingError::Unsupported( + "application binding rejected", + )), + name => Err(FieldBindingError::UnknownField(name.into())), + } + } +} + +impl FieldGetter for IntegerGetter { + type Provider = OwnedValueProvider; + + fn get<'ctx, 'data: 'ctx>(&self, _: &'ctx ExecutionContext<'data>, _: &Field) -> Option { + panic!("scheme validation must not read integer values") + } +} + +impl FieldGetter for HeadersGetter { + type Provider = BorrowedValueProvider; + + fn get<'ctx, 'data: 'ctx>( + &self, + _: &'ctx ExecutionContext<'data>, + _: &Field, + ) -> Option<&'ctx Headers> { + panic!("scheme validation must not read header values") + } +} + +impl FieldGetter for StoredGetter { + type Provider = LhsValueProvider; + + fn get<'ctx, 'data: 'ctx>( + &self, + _: &'ctx ExecutionContext<'data>, + _: &Field, + ) -> Option<&'ctx LhsValue<'ctx>> { + panic!("scheme validation must not read stored values") + } +} + +fn validate(scheme: &Scheme) -> Result<(), SchemeValidationError> { + BINDINGS.with_borrow_mut(Vec::clear); + let result = Fields::validate_scheme(scheme); + let bindings = BINDINGS.with_borrow_mut(std::mem::take); + let expected: Vec<_> = scheme + .fields() + .map(|field| field.name().to_owned()) + .collect(); + assert_eq!( + bindings, expected, + "every scheme field must bind exactly once" + ); + result +} + +#[test] +fn default_validation_checks_scalar_and_nested_collection_types_without_reading_values() { + let mut builder = SchemeBuilder::new(); + builder.add_field("count", Type::Int).unwrap(); + builder + .add_optional_field("headers", Type::map(Type::array(Type::Bytes))) + .unwrap(); + + // No request context or field values are needed for validation. + assert_eq!(validate(&builder.build()), Ok(())); +} + +#[test] +fn default_validation_aggregates_binding_errors_for_unused_and_optional_fields() { + let mut builder = SchemeBuilder::new(); + builder.add_field("count", Type::Bytes).unwrap(); + builder + .add_field("headers", Type::map(Type::array(Type::Int))) + .unwrap(); + builder.add_field("missing", Type::Int).unwrap(); + builder + .add_optional_field("missing_optional", Type::Bool) + .unwrap(); + builder.add_field("unsupported", Type::Bool).unwrap(); + builder.add_field("after_errors", Type::Int).unwrap(); + + // No filter references these fields: startup checks the complete scheme. + let errors = validate(&builder.build()).unwrap_err(); + let expected = [ + ( + "count", + FieldBindingError::TypeMismatch { + field: "count".into(), + expected: Type::Bytes, + actual: Type::Int, + }, + ), + ( + "headers", + FieldBindingError::TypeMismatch { + field: "headers".into(), + expected: Type::map(Type::array(Type::Int)), + actual: Type::map(Type::array(Type::Bytes)), + }, + ), + ("missing", FieldBindingError::UnknownField("missing".into())), + ( + "missing_optional", + FieldBindingError::UnknownField("missing_optional".into()), + ), + ( + "unsupported", + FieldBindingError::Unsupported("application binding rejected"), + ), + ] + .into_iter() + .map(|(name, error)| FieldDefinitionError::Binding { + name: name.into(), + error, + }) + .collect::>(); + assert_eq!(errors, SchemeValidationError(expected)); +} + +#[test] +fn default_validation_accepts_runtime_typed_providers_without_reading_values() { + let mut builder = SchemeBuilder::new(); + builder.add_field("stored_scalar", Type::Ip).unwrap(); + builder + .add_optional_field("stored_collection", Type::array(Type::map(Type::Bool))) + .unwrap(); + + // LhsValueProvider has no static type; its context must check actual values. + assert_eq!(validate(&builder.build()), Ok(())); +} diff --git a/engine/tests/native_getter_attributes.rs b/engine/tests/native_getter_attributes.rs new file mode 100644 index 00000000..61a2ea42 --- /dev/null +++ b/engine/tests/native_getter_attributes.rs @@ -0,0 +1,224 @@ +#![deny(unused_variables)] + +use std::cell::Cell; +use wirefilter::values::{self, SliceView}; +use wirefilter::{ + DefaultCompiler, EvaluationContext, Field, HasScheme, LhsValue, ListMatcher, ListRef, + ProviderFamily, Scheme, SchemeBuilder, Type, +}; + +struct Request<'data> { + scheme: &'data Scheme, + names: &'data [Box], + number: Option, + accesses: Cell, +} + +#[test] +fn default_compiler_traits_do_not_require_user_data_or_family_traits() { + struct UserData; + + fn assert_compiler_traits() {} + + assert_compiler_traits::>(); + assert_compiler_traits::>(); + assert_compiler_traits::>(); +} + +impl Request<'_> { + fn record_access(&self) { + self.accesses.set(self.accesses.get() + 1); + } +} + +impl HasScheme for Request<'_> { + fn scheme(&self) -> &Scheme { + self.scheme + } +} + +impl EvaluationContext for Request<'_> { + type UserData = (); + + fn get_user_data(&self) -> &() { + &() + } + + fn get_list_matcher(&self, _: ListRef<'_>) -> &dyn ListMatcher { + unreachable!("the test scheme has no lists") + } +} + +struct NamesProvider; + +impl ProviderFamily for NamesProvider { + type Storage<'data> = &'data [Box]; + type Value<'ctx, 'data> + = SliceView<'data, Box> + where + 'data: 'ctx; + type ValueType = values::Array; + + fn reborrow<'ctx, 'data: 'ctx>(value: &'ctx Self::Storage<'data>) -> Self::Value<'ctx, 'data> { + SliceView(value) + } +} + +// The unused locals require attributes to reach the function containing each +// getter body; applying them only to the FieldGetter adapter is insufficient. +wirefilter::wirefilter_fields! { + impl<'data> RequestFields for Request<'data> { + "name" => { + provider wirefilter::BorrowedValueProvider; + #[inline(never)] + #[allow(unused_variables)] + fn(request: &Request<'data>, field: &Field) -> &str { + let intentionally_unused = (); + assert_eq!(field.name(), "name"); + request.record_access(); + if let Some(name) = request.names.first() { + return name.as_ref(); + } + "" + } + }, + "count" => { + #[inline(always)] + #[cfg_attr(test, allow(unused_variables))] + fn(request: &Request<'data>) -> i64 { + let intentionally_unused = (); + request.record_access(); + request.names.len() as i64 + } + }, + "maybe_name" => { + #[inline(never)] + #[cfg_attr(test, allow(unused_variables))] + fn(request: &Request<'data>) -> Option<&str> { + let intentionally_unused = (); + request.record_access(); + let name = request.names.first()?; + if name.is_empty() { + return None; + } + Some(name.as_ref()) + } + }, + "maybe_number" => { + #[inline(always)] + #[allow(unused_variables)] + fn(request: &Request<'data>) -> Option { + let intentionally_unused = (); + request.record_access(); + let number = request.number?; + if number < 0 { + return None; + } + Some(number) + } + }, + "selection" => { + provider NamesProvider; + #[inline(never)] + #[cfg_attr(test, allow(unused_variables))] + fn(request: &Request<'data>, field: &Field) -> Option>> { + let intentionally_unused = (); + assert_eq!(field.name(), "selection"); + request.record_access(); + request.names.first()?; + Some(SliceView(request.names)) + } + }, + } +} + +fn scheme() -> Scheme { + let mut builder = SchemeBuilder::new(); + builder.add_field("name", Type::Bytes).unwrap(); + builder.add_field("count", Type::Int).unwrap(); + builder + .add_optional_field("maybe_name", Type::Bytes) + .unwrap(); + builder + .add_optional_field("maybe_number", Type::Int) + .unwrap(); + builder + .add_optional_field("selection", Type::array(Type::Bytes)) + .unwrap(); + let scheme = builder.build(); + RequestFields::validate_scheme(&scheme).unwrap(); + scheme +} + +#[test] +fn test_attributed_getters_preserve_returns_and_optional_control_flow() { + let scheme = scheme(); + let names = [Box::::from("alpha"), Box::::from("beta")]; + let empty_name = [Box::::from("")]; + let mut compiler = DefaultCompiler::<(), RequestFields>::new(); + let filters = [ + r#"name eq "alpha""#, + "count eq 2", + r#"maybe_name eq "alpha""#, + "maybe_number eq 7", + r#"any(selection[*] eq "alpha")"#, + ] + .map(|expression| { + scheme + .parse(expression) + .unwrap() + .compile_with_compiler(&mut compiler) + }); + + for (names, number, expected) in [ + (names.as_slice(), Some(7), [true, true, true, true, true]), + (names.as_slice(), None, [true, true, true, false, true]), + (names.as_slice(), Some(-1), [true, true, true, false, true]), + ([].as_slice(), Some(7), [false, false, false, true, false]), + ( + empty_name.as_slice(), + Some(7), + [false, false, false, true, false], + ), + ] { + let request = Request { + scheme: &scheme, + names, + number, + accesses: Cell::new(0), + }; + for (index, (filter, expected)) in filters.iter().zip(expected).enumerate() { + assert_eq!(filter.execute(&request).unwrap(), expected); + assert_eq!(request.accesses.get(), index + 1); + } + } +} + +#[test] +fn test_attributed_getters_preserve_borrowed_payloads() { + let scheme = scheme(); + let names = [Box::::from("alpha")]; + let request = Request { + scheme: &scheme, + names: &names, + number: None, + accesses: Cell::new(0), + }; + let mut compiler = DefaultCompiler::<(), RequestFields>::new(); + for field in ["name", "maybe_name", "selection"] { + let filter = scheme + .parse_value(field) + .unwrap() + .compile_with_compiler(&mut compiler); + let value = filter.execute(&request).unwrap().unwrap(); + let bytes = match &value { + LhsValue::Bytes(bytes) => bytes, + LhsValue::Array(values) => match values.get(0).unwrap() { + LhsValue::Bytes(bytes) => bytes, + _ => panic!("expected array of byte strings"), + }, + _ => panic!("expected bytes or array"), + }; + assert_eq!(bytes.as_ref().as_ptr(), names[0].as_ptr(), "{field}"); + } +} diff --git a/engine/tests/native_getter_hygiene.rs b/engine/tests/native_getter_hygiene.rs new file mode 100644 index 00000000..a7cdffd2 --- /dev/null +++ b/engine/tests/native_getter_hygiene.rs @@ -0,0 +1,128 @@ +use wirefilter::{ + DefaultCompiler, EvaluationContext, Field, HasScheme, LhsValue, ListMatcher, ListRef, Scheme, + SchemeBuilder, Type, +}; + +// These names belong to the application. Generated getter helpers must not +// replace them when resolving the context type or expressions in a getter body. +struct Getter<'data> { + scheme: &'data Scheme, + number: i64, + optional: Option, + name: &'data str, +} + +impl Getter<'_> { + fn read(&self) -> i64 { + self.number + } + + fn read_optional(&self) -> Option { + self.optional + } + + fn name(&self) -> &str { + self.name + } +} + +impl HasScheme for Getter<'_> { + fn scheme(&self) -> &Scheme { + self.scheme + } +} + +impl EvaluationContext for Getter<'_> { + type UserData = (); + + fn get_user_data(&self) -> &() { + &() + } + + fn get_list_matcher(&self, _: ListRef<'_>) -> &dyn ListMatcher { + unreachable!("this test has no lists") + } +} + +fn value(context: &Getter<'_>) -> i64 { + context.number + 1 +} + +fn call(context: &Getter<'_>) -> i64 { + context.number + 2 +} + +fn __wirefilter_call(context: &Getter<'_>) -> i64 { + context.number + 3 +} + +wirefilter::wirefilter_fields! { + impl<'data> Fields for Getter<'data> { + "required" => { + fn(field: &Getter<'data>, metadata: &Field) -> i64 { + assert_eq!(metadata.name(), "required"); + Getter::read(field) + value(field) + call(field) + __wirefilter_call(field) + } + }, + "optional" => { + fn(consumer: &Getter<'data>, field: &Field) -> Option { + assert_eq!(field.name(), "optional"); + assert!(field.optional()); + let number = Getter::read_optional(consumer)?; + Some(number + value(consumer) + call(consumer) + __wirefilter_call(consumer)) + } + }, + "name" => { + fn(context: &Getter<'data>) -> &str { + Getter::name(context) + } + }, + } +} + +#[test] +fn test_getter_bodies_resolve_application_items() { + let mut builder = SchemeBuilder::new(); + builder.add_field("required", Type::Int).unwrap(); + builder.add_optional_field("optional", Type::Int).unwrap(); + builder.add_field("name", Type::Bytes).unwrap(); + let scheme = builder.build(); + Fields::validate_scheme(&scheme).unwrap(); + let mut compiler = DefaultCompiler::<(), Fields>::new(); + let values = ["required", "optional", "name"].map(|field| { + scheme + .parse_value(field) + .unwrap() + .compile_with_compiler(&mut compiler) + }); + let comparison = scheme + .parse(r#"required eq 34 and optional eq 38 and name eq "example""#) + .unwrap() + .compile_with_compiler(&mut compiler); + + for (number, optional, expected_required, expected_optional, matches) in [ + (7, Some(11), 34, Some(38), true), + (7, None, 34, None, false), + (-2, Some(3), -2, Some(3), false), + ] { + let context = Getter { + scheme: &scheme, + number, + optional, + name: "example", + }; + assert_eq!( + values[0].execute(&context).unwrap(), + Ok(LhsValue::Int(expected_required)), + ); + assert_eq!( + values[1].execute(&context).unwrap(), + expected_optional.map(LhsValue::Int).ok_or(Type::Int), + ); + assert_eq!( + values[2].execute(&context).unwrap(), + Ok(LhsValue::from("example")), + ); + assert_eq!(comparison.execute(&context).unwrap(), matches); + } +} diff --git a/engine/tests/native_lhs_values.rs b/engine/tests/native_lhs_values.rs new file mode 100644 index 00000000..8e3e5f63 --- /dev/null +++ b/engine/tests/native_lhs_values.rs @@ -0,0 +1,662 @@ +//! Default and downstream contexts expose runtime-typed values through shared field binding. + +use std::cell::Cell; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; +use wirefilter::{ + Array, Compare, CompiledExpr, CompiledValueExpr, Compiler, ContextFamily, DefaultCompiler, + EvaluationContext, ExecutionContext, ExecutionContextFamily, Field, FieldBindingError, + FieldConsumer, FieldDefinitionError, FieldGetter, FieldIndex, FunctionArgs, + FunctionCallArgExpr, GetType, HasScheme, IdentifierExpr, IndexExpr, LhsValue, LhsValueProvider, + ListMatcher, ListRef, Map, Scheme, SchemeBuilder, SimpleFunctionArgKind, + SimpleFunctionDefinition, SimpleFunctionImpl, SimpleFunctionParam, Type, TypeMismatchError, +}; + +#[derive(Default)] +struct Reads { + map: Cell, +} + +struct NativeFields; +struct Getter; + +impl ContextFamily for NativeFields { + type Context<'data> = ExecutionContext<'data, Reads>; + type UserData = Reads; + + fn bind>( + field: Field, + consumer: B, + ) -> Result { + consumer.consume(field, Getter) + } +} + +impl FieldGetter for Getter { + type Provider = LhsValueProvider; + + fn get<'ctx, 'data: 'ctx>( + &self, + context: &'ctx ExecutionContext<'data, Reads>, + field: &Field, + ) -> Option<&'ctx LhsValue<'ctx>> { + if field.name() == "map" { + let count = &context.get_user_data().map; + count.set(count.get() + 1); + } + context.get_field_value(field.as_ref()) + } +} + +fn array<'a>(ty: Type, values: impl IntoIterator>) -> LhsValue<'a> { + Array::try_from_iter(ty, values).unwrap().into() +} + +fn map<'a>(ty: Type, entries: impl IntoIterator)>) -> LhsValue<'a> { + Map::try_from_iter( + ty, + entries + .into_iter() + .map(|(key, value)| Ok::<_, TypeMismatchError>((key.into(), value))), + ) + .unwrap() + .into() +} + +fn identity<'a>(args: FunctionArgs<'_, 'a>) -> Option> { + args.next()?.ok() +} + +fn add_function( + builder: &mut SchemeBuilder, + name: &str, + params: &[Type], + return_type: Type, + implementation: SimpleFunctionImpl, +) { + builder + .add_function( + name, + SimpleFunctionDefinition { + params: params + .iter() + .map(|&val_type| SimpleFunctionParam { + arg_kind: SimpleFunctionArgKind::Field, + val_type, + }) + .collect(), + opt_params: Box::new([]), + return_type, + implementation, + }, + ) + .unwrap(); +} + +fn assert_filter( + scheme: &Scheme, + context: &ExecutionContext<'_, Reads>, + expression: &str, + expected: bool, +) { + let ast = scheme.parse(expression).unwrap(); + let default = ast + .clone() + .compile_with_compiler(&mut DefaultCompiler::::new()); + let native = ast.compile_with_compiler(&mut DefaultCompiler::::new()); + assert_eq!( + default.execute(context).unwrap(), + expected, + "default: {expression}" + ); + assert_eq!( + native.execute(context).unwrap(), + expected, + "native: {expression}" + ); +} + +#[test] +fn one_family_handles_all_scalar_types_and_optional_fields() { + let mut builder = SchemeBuilder::new(); + for (name, ty) in [ + ("number", Type::Int), + ("text", Type::Bytes), + ("flag", Type::Bool), + ("v4", Type::Ip), + ("v6", Type::Ip), + ] { + builder.add_field(name, ty).unwrap(); + } + builder.add_optional_field("missing", Type::Int).unwrap(); + let scheme = builder.build(); + let mut context = ExecutionContext::new_with(&scheme, Reads::default); + context.set_field_value_from_name("number", 42i64).unwrap(); + context.set_field_value_from_name("text", "target").unwrap(); + context.set_field_value_from_name("flag", true).unwrap(); + context + .set_field_value_from_name("v4", IpAddr::V4(Ipv4Addr::new(192, 0, 2, 1))) + .unwrap(); + context + .set_field_value_from_name("v6", IpAddr::V6(Ipv6Addr::LOCALHOST)) + .unwrap(); + for (expression, expected) in [ + ("number ge 42", true), + ("number lt 42", false), + ("number & 2", true), + (r#"text eq "target""#, true), + (r#"text contains "arg""#, true), + ("flag", true), + ("not flag", false), + ("v4 in {192.0.2.0/24}", true), + ("v6 eq ::1", true), + ("missing eq 0", false), + ("missing ne 0", true), + ] { + assert_filter(&scheme, &context, expression, expected); + } + let native = scheme + .parse_value("missing") + .unwrap() + .compile_with_compiler(&mut DefaultCompiler::::new()); + assert_eq!(native.execute(&context).unwrap(), Err(Type::Int)); + assert!( + context + .set_field_value_from_name("number", "wrong type") + .is_err() + ); +} + +// Check actual retained element and map-key addresses. Payload-only checks would +// miss rebuilding containers whose byte elements still borrow the source. +fn assert_borrowed_structure(actual: &LhsValue<'_>, original: &LhsValue<'_>) { + match (actual, original) { + (LhsValue::Array(actual), LhsValue::Array(original)) => { + assert_eq!(actual.len(), original.len()); + for (actual, original) in actual.iter().zip(original.iter()) { + assert!( + std::ptr::eq(actual, original), + "array elements were rebuilt" + ); + assert_borrowed_structure(actual, original); + } + } + (LhsValue::Map(actual), LhsValue::Map(original)) => { + assert_eq!(actual.len(), original.len()); + for ((key, actual), (source_key, original)) in actual.iter().zip(original.iter()) { + assert_eq!(key, source_key); + assert_eq!(key.as_ptr(), source_key.as_ptr(), "map key was copied"); + assert!(std::ptr::eq(actual, original), "map values were rebuilt"); + assert_borrowed_structure(actual, original); + } + } + (LhsValue::Bytes(actual), LhsValue::Bytes(original)) => { + assert_eq!(actual.as_ref(), original.as_ref()); + assert_eq!(actual.as_ref().as_ptr(), original.as_ref().as_ptr()); + assert!(matches!(actual, wirefilter::Bytes::Borrowed(_))); + } + _ => assert_eq!(actual, original), + } +} + +#[test] +fn schemes_can_choose_nested_collection_compositions_at_runtime() { + let payload = String::from("needle"); + // A single ContextFamily/getter handles every composition. No Rust provider + // specialization is declared for any of these runtime-selected types. + for depth in 0..=5 { + for mask in 0..1usize << depth { + let mut value = LhsValue::from(payload.as_str()); + let mut selection = String::new(); + for level in 0..depth { + let ty = value.get_type(); + if mask & (1 << level) == 0 { + value = array(ty, [value]); + selection = format!("[0]{selection}"); + } else { + value = map(ty, [(b"k".as_slice(), value)]); + selection = format!("[\"k\"]{selection}"); + } + } + let ty = value.get_type(); + let mut builder = SchemeBuilder::new(); + builder.add_field("value", ty).unwrap(); + add_function( + &mut builder, + "echo", + &[ty], + ty, + SimpleFunctionImpl::new(identity), + ); + let scheme = builder.build(); + let mut context = ExecutionContext::new_with(&scheme, Reads::default); + context.set_field_value_from_name("value", value).unwrap(); + assert_filter( + &scheme, + &context, + &format!("value{selection} eq \"needle\""), + true, + ); + if depth > 0 { + assert_filter( + &scheme, + &context, + &format!("any(value{} eq \"needle\")", "[*]".repeat(depth)), + true, + ); + } + let ast = scheme.parse_value("echo(value)").unwrap(); + let default = ast + .clone() + .compile_with_compiler(&mut DefaultCompiler::::new()); + let custom = + ast.compile_with_compiler(&mut DefaultCompiler::::new()); + let original = context + .get_field_value(scheme.get_field("value").unwrap()) + .unwrap(); + for actual in [default.execute(&context), custom.execute(&context)] { + assert_borrowed_structure(&actual.unwrap().unwrap(), original); + } + } + } +} + +#[test] +fn indexed_and_mapped_arguments_preserve_missing_values_and_map_compatibility() { + let bytes_array = Type::array(Type::Bytes); + let mut builder = SchemeBuilder::new(); + builder.add_field("map", Type::map(bytes_array)).unwrap(); + builder + .add_optional_field("absent", Type::map(bytes_array)) + .unwrap(); + add_function( + &mut builder, + "echo_bytes", + &[Type::Bytes], + Type::Bytes, + SimpleFunctionImpl::new(identity), + ); + add_function( + &mut builder, + "echo_array", + &[bytes_array], + bytes_array, + SimpleFunctionImpl::new(identity), + ); + let scheme = builder.build(); + let payload = String::from("payload"); + let mut context = ExecutionContext::new_with(&scheme, Reads::default); + context + .set_field_value_from_name( + "map", + map( + bytes_array, + [ + ( + b"a".as_slice(), + array(Type::Bytes, [payload.as_str().into()]), + ), + (b"empty".as_slice(), array(Type::Bytes, [])), + ], + ), + ) + .unwrap(); + for (expression, expected) in [ + ( + r#"echo_array(map["a"])"#, + Ok(array(Type::Bytes, [payload.as_str().into()])), + ), + (r#"echo_array(map["missing"])"#, Err(bytes_array)), + (r#"echo_bytes(map["a"][0])"#, Ok(payload.as_str().into())), + (r#"echo_bytes(map["a"][8])"#, Err(Type::Bytes)), + ( + r#"echo_bytes(map["a"][*])"#, + Ok(array(Type::Bytes, [payload.as_str().into()])), + ), + ( + "echo_bytes(map[*][*])", + Ok(array(Type::Bytes, [payload.as_str().into()])), + ), + ("echo_bytes(absent[*][*])", Err(bytes_array)), + (r#"echo_bytes(map["missing"][*])"#, Err(bytes_array)), + (r#"echo_bytes(map["empty"][*])"#, Ok(array(Type::Bytes, []))), + ] { + let ast = scheme.parse_value(expression).unwrap(); + let default = ast + .clone() + .compile_with_compiler(&mut DefaultCompiler::::new()); + let native = ast.compile_with_compiler(&mut DefaultCompiler::::new()); + assert_eq!( + default.execute(&context).unwrap(), + expected, + "default: {expression}" + ); + assert_eq!( + native.execute(&context).unwrap(), + expected, + "custom: {expression}" + ); + } + + let ast = scheme.parse_value(r#"echo_array(map["a"])"#).unwrap(); + let default = ast + .clone() + .compile_with_compiler(&mut DefaultCompiler::::new()); + let custom = ast.compile_with_compiler(&mut DefaultCompiler::::new()); + let LhsValue::Map(original) = context + .get_field_value(scheme.get_field("map").unwrap()) + .unwrap() + else { + unreachable!() + }; + for selected in [default.execute(&context), custom.execute(&context)] { + assert_borrowed_structure(&selected.unwrap().unwrap(), original.get(b"a").unwrap()); + } + + struct ObserveMapArgument { + arguments: usize, + } + impl Compiler for ObserveMapArgument { + type Family = ExecutionContextFamily; + type U = Reads; + + fn compile_function_call_arg_expr( + &mut self, + node: FunctionCallArgExpr, + ) -> CompiledValueExpr { + self.arguments += 1; + let value = self.compile_value_expr(node); + CompiledValueExpr::new_with_context(move |ctx| { + let result = value.execute(ctx.context()); + assert!( + matches!(&result, Ok(LhsValue::Map(_))), + "trailing map[*] changed the compiler hook's value" + ); + result + }) + } + } + let mut compiler = ObserveMapArgument { arguments: 0 }; + let actual = scheme + .parse_value("echo_array(map[*])") + .unwrap() + .compile_with_compiler(&mut compiler); + assert_eq!(compiler.arguments, 1); + let expected = scheme + .parse_value("echo_array(map[*])") + .unwrap() + .compile_with_compiler(&mut DefaultCompiler::::new()); + assert_eq!(actual.execute(&context), expected.execute(&context)); +} + +fn record_children(value: &LhsValue<'_>, addresses: &mut Vec) { + match value { + LhsValue::Array(array) => { + for value in array.iter() { + addresses.push(std::ptr::from_ref(value) as usize); + record_children(value, addresses); + } + } + LhsValue::Map(map) => { + for (_, value) in map.iter() { + addresses.push(std::ptr::from_ref(value) as usize); + record_children(value, addresses); + } + } + _ => {} + } +} + +struct OriginalCollection { + children: Vec, +} +impl> Compare for OriginalCollection { + fn compare<'ctx, 'data: 'ctx>( + &self, + value: &LhsValue<'ctx>, + _: &'ctx F::Context<'data>, + ) -> bool { + let first = match value { + LhsValue::Array(array) => array.get(0), + LhsValue::Map(map) => map.iter().next().map(|(_, value)| value), + _ => panic!("custom comparison requires a collection"), + }; + if let Some(first) = first { + assert!( + self.children + .contains(&(std::ptr::from_ref(first) as usize)), + "custom comparison received a rebuilt collection" + ); + true + } else { + false + } + } +} + +#[test] +fn compiler_comparisons_can_borrow_selected_collections() { + let bytes = Type::array(Type::Bytes); + let mut builder = SchemeBuilder::new(); + builder + .add_field("rows", Type::array(Type::map(bytes))) + .unwrap(); + let scheme = builder.build(); + let value = array( + Type::map(bytes), + [ + map( + bytes, + [(b"key".as_slice(), array(Type::Bytes, ["value".into()]))], + ), + map(bytes, []), + ], + ); + let mut context = ExecutionContext::new_with(&scheme, Reads::default); + context.set_field_value_from_name("rows", value).unwrap(); + let mut children = Vec::new(); + record_children( + context + .get_field_value(scheme.get_field("rows").unwrap()) + .unwrap(), + &mut children, + ); + for indexes in [ + vec![], + vec![FieldIndex::ArrayIndex(0)], + vec![FieldIndex::ArrayIndex(0), FieldIndex::MapKey("key".into())], + vec![FieldIndex::ArrayIndex(8)], + vec![FieldIndex::MapEach], + vec![FieldIndex::MapEach, FieldIndex::MapEach], + ] { + for missing_default in [false, true] { + let index = IndexExpr { + identifier: IdentifierExpr::Field(scheme.get_field("rows").unwrap().to_owned()), + indexes: indexes.clone().into_boxed_slice(), + }; + let default = index.clone().compile_with( + &mut DefaultCompiler::::new(), + missing_default, + OriginalCollection { + children: children.clone(), + }, + ); + let native = index.compile_with( + &mut DefaultCompiler::::new(), + missing_default, + OriginalCollection { + children: children.clone(), + }, + ); + match (default, native) { + (CompiledExpr::One(default), CompiledExpr::One(native)) => { + assert_eq!(default.execute(&context), native.execute(&context)) + } + (CompiledExpr::Vec(default), CompiledExpr::Vec(native)) => { + assert_eq!(default.execute(&context), native.execute(&context)) + } + _ => panic!("comparison cardinality changed"), + } + } + } +} + +#[test] +fn mapped_extra_stored_arguments_keep_cheap_lazy_access() { + fn consume_both<'a>(args: FunctionArgs<'_, 'a>) -> Option> { + args.next()?.ok()?; + args.next()?.ok()?; + Some(true.into()) + } + fn consume_first<'a>(args: FunctionArgs<'_, 'a>) -> Option> { + args.next()?.ok()?; + Some(true.into()) + } + let mut builder = SchemeBuilder::new(); + builder + .add_optional_field("numbers", Type::array(Type::Int)) + .unwrap(); + builder.add_field("map", Type::map(Type::Int)).unwrap(); + for (name, extra, implementation) in [ + ( + "map_arg", + Type::map(Type::Int), + consume_both as for<'i, 'a> fn(FunctionArgs<'i, 'a>) -> Option>, + ), + ("int_arg", Type::Int, consume_both), + ("ignore_map", Type::map(Type::Int), consume_first), + ] { + add_function( + &mut builder, + name, + &[Type::Int, extra], + Type::Bool, + SimpleFunctionImpl::new(implementation), + ); + } + let scheme = builder.build(); + let mut context = ExecutionContext::new_with(&scheme, Reads::default); + context + .set_field_value_from_name("numbers", array(Type::Int, [1.into(), 2.into(), 3.into()])) + .unwrap(); + context + .set_field_value_from_name("map", map(Type::Int, [(b"key".as_slice(), 7.into())])) + .unwrap(); + for expression in [ + "any(map_arg(numbers[*], map))", + r#"any(int_arg(numbers[*], map["key"]))"#, + ] { + let native = scheme + .parse(expression) + .unwrap() + .compile_with_compiler(&mut DefaultCompiler::::new()); + context.get_user_data().map.set(0); + assert!(native.execute(&context).unwrap()); + // Repeated cheap borrowing is intentional. Treating this provider as a + // materializing native collection would read once and allocate caches. + assert_eq!(context.get_user_data().map.get(), 3, "{expression}"); + } + // Leaving the mandatory map absent proves ignored arguments stay unread. + let mut missing = ExecutionContext::new_with(&scheme, Reads::default); + missing + .set_field_value_from_name("numbers", array(Type::Int, [1.into(), 2.into()])) + .unwrap(); + assert_filter(&scheme, &missing, "any(ignore_map(numbers[*], map))", true); + assert_eq!(missing.get_user_data().map.get(), 0); + missing + .set_field_value_from_name("numbers", array(Type::Int, [])) + .unwrap(); + assert_filter(&scheme, &missing, "any(map_arg(numbers[*], map))", false); + assert_eq!(missing.get_user_data().map.get(), 0); +} + +struct BorrowingContext<'data> { + scheme: &'data Scheme, + value: LhsValue<'data>, + optional: Option>, +} +impl HasScheme for BorrowingContext<'_> { + fn scheme(&self) -> &Scheme { + self.scheme + } +} +impl EvaluationContext for BorrowingContext<'_> { + type UserData = (); + + fn get_user_data(&self) -> &() { + &() + } + + fn get_list_matcher(&self, _: ListRef<'_>) -> &dyn ListMatcher { + unreachable!() + } +} +wirefilter::wirefilter_fields! { + impl<'data> BorrowingFields for BorrowingContext<'data> { + "value" => { + provider LhsValueProvider; + fn(ctx: &BorrowingContext<'data>) -> &LhsValue<'data> { &ctx.value } + }, + "optional" => { + provider LhsValueProvider; + fn(ctx: &BorrowingContext<'data>) -> Option<&LhsValue<'data>> { ctx.optional.as_ref() } + }, + } +} + +#[test] +fn macro_supports_borrowed_runtime_values_and_validates_presence() { + let mut builder = SchemeBuilder::new(); + builder.add_field("value", Type::Bytes).unwrap(); + builder.add_optional_field("optional", Type::Int).unwrap(); + let scheme = builder.build(); + BorrowingFields::validate_scheme(&scheme).unwrap(); + let payload = String::from("borrowed"); + let context = BorrowingContext { + scheme: &scheme, + value: payload.as_str().into(), + optional: None, + }; + let filter = scheme + .parse(r#"value eq "borrowed" and optional ne 1"#) + .unwrap() + .compile_with_compiler(&mut DefaultCompiler::<(), BorrowingFields>::new()); + assert!(filter.execute(&context).unwrap()); + let value = scheme + .parse_value("value") + .unwrap() + .compile_with_compiler(&mut DefaultCompiler::<(), BorrowingFields>::new()) + .execute(&context) + .unwrap() + .unwrap(); + assert_borrowed_structure(&value, &context.value); + + let mut builder = SchemeBuilder::new(); + builder.add_field("value", Type::Bytes).unwrap(); + builder.add_field("optional", Type::Int).unwrap(); + let errors = BorrowingFields::validate_scheme(&builder.build()).unwrap_err(); + assert!(errors.0.iter().any(|error| matches!(error, FieldDefinitionError::OptionalGetterForMandatoryField(name) if name.as_ref() == "optional"))); + + // These same getters can serve another scheme's runtime-selected types. + let mut builder = SchemeBuilder::new(); + builder.add_field("value", Type::Int).unwrap(); + builder.add_optional_field("optional", Type::Bytes).unwrap(); + let scheme = builder.build(); + BorrowingFields::validate_scheme(&scheme).unwrap(); + let context = BorrowingContext { + scheme: &scheme, + value: 42.into(), + optional: Some("present".into()), + }; + let filter = scheme + .parse(r#"value eq 42 and optional eq "present""#) + .unwrap() + .compile_with_compiler(&mut DefaultCompiler::<(), BorrowingFields>::new()); + assert!(filter.execute(&context).unwrap()); + + let mut builder = SchemeBuilder::new(); + builder.add_field("value", Type::Int).unwrap(); + builder.add_optional_field("optional", Type::Bytes).unwrap(); + builder.add_field("undefined", Type::Bool).unwrap(); + let errors = BorrowingFields::validate_scheme(&builder.build()).unwrap_err(); + assert!(errors.0.iter().any(|error| matches!(error, FieldDefinitionError::MissingGetter(name) if name.as_ref() == "undefined"))); +} diff --git a/engine/tests/native_operations.rs b/engine/tests/native_operations.rs new file mode 100644 index 00000000..cc38cb73 --- /dev/null +++ b/engine/tests/native_operations.rs @@ -0,0 +1,270 @@ +#![cfg(feature = "bytes")] + +use bytes::Bytes; +use std::any::TypeId; +use std::cell::Cell; +use wirefilter::values::{self, IntoLhsValue, SliceView}; +use wirefilter::{ + Compiler, ContextFamily, ContextRef, DefaultCompiler, EvaluationContext, Field, + FieldBindingError, FieldConsumer, FieldGetter, FieldIndex, HasScheme, IdentifierExpr, + IndexExpr, LhsValue, ListMatcher, ListRef, ProviderFamily, Scheme, SchemeBuilder, Type, + ValueExpr, +}; + +// Cell makes the context invariant in 'data. The descriptor borrows the request +// storage for 'data, rather than being tied to a temporary getter result. +struct BorrowingContext<'data> { + scheme: &'data Scheme, + rows: Cell<&'data [Vec]>, + getter_calls: Cell, +} + +impl HasScheme for BorrowingContext<'_> { + fn scheme(&self) -> &Scheme { + self.scheme + } +} + +impl EvaluationContext for BorrowingContext<'_> { + type UserData = (); + + fn get_user_data(&self) -> &() { + &() + } + + fn get_list_matcher(&self, _: ListRef<'_>) -> &dyn ListMatcher { + unreachable!("no list matchers in this test") + } +} + +struct RowsProvider; + +impl ProviderFamily for RowsProvider { + type Storage<'data> = &'data [Vec]; + type Value<'ctx, 'data> + = SliceView<'data, Vec> + where + 'data: 'ctx; + type ValueType = values::Array>; + + fn reborrow<'ctx, 'data: 'ctx>(value: &'ctx Self::Storage<'data>) -> Self::Value<'ctx, 'data> { + SliceView(value) + } +} + +wirefilter::wirefilter_fields! { + impl<'data> BorrowingFields for BorrowingContext<'data> { + "rows" => { + provider RowsProvider; + fn(ctx: &BorrowingContext<'data>, field: &Field) -> SliceView<'data, Vec> { + assert_eq!(field.name(), "rows"); + ctx.getter_calls.set(ctx.getter_calls.get() + 1); + SliceView(ctx.rows.get()) + } + }, + "first" => { + provider RowsProvider; + fn(ctx: &BorrowingContext<'data>, field: &Field) -> SliceView<'data, Vec> { + assert_eq!(field.name(), "first"); + assert!(!field.optional()); + ctx.getter_calls.set(ctx.getter_calls.get() + 1); + let rows = ctx.rows.get(); + SliceView(&rows[..rows.len().min(1)]) + } + }, + "optional" => { + provider RowsProvider; + fn(ctx: &BorrowingContext<'data>, field: &Field) -> Option>> { + assert_eq!(field.name(), "optional"); + ctx.getter_calls.set(ctx.getter_calls.get() + 1); + let rows = ctx.rows.get(); + (!rows.is_empty()).then_some(SliceView(rows)) + } + }, + } +} + +#[derive(Default)] +struct RecordingCompiler { + bindings: Vec, +} + +impl Compiler for RecordingCompiler { + type Family = BorrowingFields; + type U = (); + + fn bind_field>( + &mut self, + field: Field, + consumer: B, + ) -> Result { + self.bindings.push(field.name().into()); + BorrowingFields::bind(field, consumer) + } +} + +fn scheme() -> Scheme { + let mut builder = SchemeBuilder::new(); + let rows = Type::array(Type::array(Type::Bytes)); + builder.add_field("rows", rows).unwrap(); + builder.add_field("first", rows).unwrap(); + builder.add_optional_field("optional", rows).unwrap(); + let scheme = builder.build(); + BorrowingFields::validate_scheme(&scheme).unwrap(); + scheme +} + +#[test] +fn test_same_provider_and_presence_getters_share_a_type_and_preserve_field_selection() { + type ReadValue = Box< + dyn for<'ctx, 'data> Fn(ContextRef<'ctx, 'data, BorrowingFields>) -> Option>, + >; + + struct ReadConsumer; + + impl FieldConsumer for ReadConsumer { + type Output = (TypeId, ReadValue); + + fn consume>( + self, + field: Field, + getter: G, + ) -> Result { + let read: ReadValue = Box::new(move |context| { + let value = { + let metadata = field.clone(); + getter.get(context.context(), &metadata) + }; + // Dropping the independently borrowed field metadata must not + // prevent retaining the native value's request references. + value.map(IntoLhsValue::into_lhs) + }); + Ok((TypeId::of::(), read)) + } + } + + let scheme = scheme(); + let (required_type, required) = + BorrowingFields::bind(scheme.get_field("rows").unwrap().to_owned(), ReadConsumer).unwrap(); + let (first_type, first) = + BorrowingFields::bind(scheme.get_field("first").unwrap().to_owned(), ReadConsumer).unwrap(); + let (optional_type, optional) = BorrowingFields::bind( + scheme.get_field("optional").unwrap().to_owned(), + ReadConsumer, + ) + .unwrap(); + assert_eq!(required_type, first_type); + assert_ne!(required_type, optional_type); + + let rows = [ + vec![Bytes::from_static(b"first payload")], + vec![Bytes::from_static(b"second payload")], + ]; + let context = BorrowingContext { + scheme: &scheme, + rows: Cell::new(rows.as_slice()), + getter_calls: Cell::new(0), + }; + let required_value = required(ContextRef::new(&context)).unwrap(); + let optional_value = optional(ContextRef::new(&context)).unwrap(); + assert_eq!(required_value, optional_value); + let first_value = first(ContextRef::new(&context)).unwrap(); + assert_eq!(first_value, SliceView(&rows[..1]).into_lhs()); + assert_ne!(required_value, first_value); + assert_eq!(context.getter_calls.get(), 3); + + context.rows.set(&[]); + assert!(required(ContextRef::new(&context)).is_some()); + assert!(first(ContextRef::new(&context)).is_some()); + assert!(optional(ContextRef::new(&context)).is_none()); + assert_eq!(context.getter_calls.get(), 6); +} + +#[test] +fn descriptor_comparisons_borrow_invariant_context_and_preserve_getter_hooks() { + let scheme = scheme(); + let first = [ + vec![Bytes::from_static(b"first"), Bytes::from_static(b"target")], + vec![], + ]; + let second = [ + vec![Bytes::from_static(b"second")], + vec![Bytes::from_static(b"target")], + ]; + let context = BorrowingContext { + scheme: &scheme, + rows: Cell::new(first.as_slice()), + getter_calls: Cell::new(0), + }; + let mut compiler = RecordingCompiler::default(); + for (expression, field, expected_first, expected_second) in [ + (r#"rows[0][0] eq "first""#, "rows", true, false), + (r#"any(rows[0][*] eq "target")"#, "rows", true, false), + (r#"any(rows[*][*] eq "target")"#, "rows", true, true), + (r#"any(rows[*][0] eq "target")"#, "rows", false, true), + (r#"optional[0][0] eq "first""#, "optional", true, false), + (r#"any(optional[*][*] eq "target")"#, "optional", true, true), + ] { + let binding_count = compiler.bindings.len(); + let filter = scheme + .parse(expression) + .unwrap() + .compile_with_compiler(&mut compiler); + assert_eq!(&compiler.bindings[binding_count..], &[field]); + for (rows, expected) in [ + (first.as_slice(), expected_first), + (second.as_slice(), expected_second), + ([].as_slice(), false), + ] { + context.rows.set(rows); + let getter_count = context.getter_calls.get(); + assert_eq!(filter.execute(&context).unwrap(), expected, "{expression}"); + assert_eq!(context.getter_calls.get(), getter_count + 1); + assert_eq!(compiler.bindings.len(), binding_count + 1); + } + } +} + +#[test] +fn descriptor_materialization_keeps_payloads_borrowed_from_invariant_context() { + let scheme = scheme(); + let rows = [vec![ + Bytes::from_static(b"first"), + Bytes::from_static(b"second"), + ]]; + let context = BorrowingContext { + scheme: &scheme, + rows: Cell::new(rows.as_slice()), + getter_calls: Cell::new(0), + }; + let mut compiler = DefaultCompiler::<(), BorrowingFields>::new(); + // The public IndexExpr API supports general `[*]` selections used by + // mapped function arguments; parse_value rejects this top-level `[*]`. + for indexes in [ + vec![], + vec![FieldIndex::ArrayIndex(0), FieldIndex::ArrayIndex(0)], + vec![FieldIndex::MapEach, FieldIndex::MapEach], + ] { + let expression = IndexExpr { + identifier: IdentifierExpr::Field(scheme.get_field("rows").unwrap().to_owned()), + indexes: indexes.into_boxed_slice(), + }; + let filter = expression.compile_with_compiler(&mut compiler); + let getter_count = context.getter_calls.get(); + let value = filter.execute(&context).unwrap(); + let bytes = match &value { + LhsValue::Bytes(bytes) => bytes, + LhsValue::Array(array) => match array.get(0).unwrap() { + LhsValue::Bytes(bytes) => bytes, + LhsValue::Array(array) => match array.get(0).unwrap() { + LhsValue::Bytes(bytes) => bytes, + _ => panic!("expected nested bytes"), + }, + _ => panic!("expected bytes or array"), + }, + _ => panic!("unexpected materialized value"), + }; + assert_eq!(bytes.as_ref().as_ptr(), rows[0][0].as_ptr()); + assert_eq!(context.getter_calls.get(), getter_count + 1); + } +} diff --git a/engine/tests/reader_traversal.rs b/engine/tests/reader_traversal.rs new file mode 100644 index 00000000..b4d17e61 --- /dev/null +++ b/engine/tests/reader_traversal.rs @@ -0,0 +1,440 @@ +//! Traversal must preserve ownership as well as selection and comparison results. + +use std::cell::RefCell; +use std::collections::BTreeMap; +use wirefilter::{ + Array, Bytes, Compare, CompiledExpr, CompiledValueExpr, ContextFamily, ContextState, + DefaultCompiler, EvaluationContext, ExecutionContext, FieldIndex, FilterValueExpr, + FunctionArgs, HasScheme, IdentifierExpr, IndexExpr, LhsValue, ListMatcher, ListRef, Map, + Scheme, SchemeBuilder, SimpleFunctionArgKind, SimpleFunctionDefinition, SimpleFunctionImpl, + SimpleFunctionParam, Type, TypeMismatchError, ValueExpr, +}; + +fn array<'a>(ty: Type, values: impl IntoIterator>) -> LhsValue<'a> { + Array::try_from_iter(ty, values).unwrap().into() +} + +fn map<'a>(ty: Type, entries: impl IntoIterator)>) -> LhsValue<'a> { + Map::try_from_iter( + ty, + entries + .into_iter() + .map(|(key, value)| Ok::<_, TypeMismatchError>((key.into(), value))), + ) + .unwrap() + .into() +} + +fn add_function( + builder: &mut SchemeBuilder, + name: &str, + parameters: &[Type], + output: Type, + implementation: SimpleFunctionImpl, +) { + builder + .add_function( + name, + SimpleFunctionDefinition { + params: parameters + .iter() + .map(|&val_type| SimpleFunctionParam { + arg_kind: SimpleFunctionArgKind::Field, + val_type, + }) + .collect(), + opt_params: Box::new([]), + return_type: output, + implementation, + }, + ) + .unwrap(); +} + +fn compile_selection(scheme: &Scheme, call: &str, indexes: Vec) -> CompiledValueExpr { + // Bare [*] value expressions are not accepted as a FilterValueAst. Construct + // the same IndexExpr node that occurs within a mapped function argument. + let FilterValueExpr::Index(mut index) = scheme.parse_value(call).unwrap().expression().clone() + else { + panic!("expected function-call index expression") + }; + index.indexes = indexes.into_boxed_slice(); + index.compile() +} + +thread_local! { + // Record the allocations produced inside the function, before its temporary + // outer containers are consumed by index selection. + static GENERATED: RefCell, usize)>> = const { RefCell::new(Vec::new()) }; +} + +fn owned_bytes(text: &str) -> LhsValue<'static> { + let bytes = text.as_bytes().to_vec().into_boxed_slice(); + GENERATED.with_borrow_mut(|records| { + records.push((text.as_bytes().to_vec(), bytes.as_ptr() as usize)); + }); + LhsValue::Bytes(Bytes::Owned(bytes)) +} + +fn owned_rows<'a>(_: FunctionArgs<'_, 'a>) -> Option> { + GENERATED.with_borrow_mut(Vec::clear); + Some(map( + Type::array(Type::Bytes), + [ + ( + b"z".as_slice(), + array(Type::Bytes, [owned_bytes("z0"), owned_bytes("z1")]), + ), + (b"a".as_slice(), array(Type::Bytes, [owned_bytes("a0")])), + (b"empty".as_slice(), array(Type::Bytes, [])), + ], + )) +} + +fn absent<'a>(_: FunctionArgs<'_, 'a>) -> Option> { + None +} + +fn identity<'a>(args: FunctionArgs<'_, 'a>) -> Option> { + args.next()?.ok() +} + +fn assert_original_owned_payload(value: &LhsValue<'_>) { + match value { + LhsValue::Bytes(Bytes::Owned(bytes)) => GENERATED.with_borrow(|records| { + let (_, address) = records + .iter() + .find(|(text, _)| text.as_slice() == bytes.as_ref()) + .unwrap(); + assert_eq!( + bytes.as_ptr() as usize, + *address, + "owned payload was copied" + ); + }), + LhsValue::Array(values) => { + for value in values.iter() { + assert_original_owned_payload(value); + } + } + value => panic!("expected retained owned bytes, got {value:?}"), + } +} + +#[test] +fn selections_move_owned_function_payloads_and_preserve_map_order() { + use FieldIndex::{ArrayIndex, MapEach, MapKey}; + + let mut builder = SchemeBuilder::new(); + let rows = Type::map(Type::array(Type::Bytes)); + add_function( + &mut builder, + "owned_rows", + &[], + rows, + SimpleFunctionImpl::new(owned_rows), + ); + add_function( + &mut builder, + "absent", + &[], + rows, + SimpleFunctionImpl::new(absent), + ); + let scheme = builder.build(); + let context = ExecutionContext::new(&scheme); + for (indexes, expected) in [ + ( + vec![MapEach, MapEach], + array(Type::Bytes, ["a0".into(), "z0".into(), "z1".into()]), + ), + ( + vec![MapEach, ArrayIndex(0)], + array(Type::Bytes, ["a0".into(), "z0".into()]), + ), + ( + vec![MapKey("z".into()), MapEach], + array(Type::Bytes, ["z0".into(), "z1".into()]), + ), + (vec![MapKey("z".into()), ArrayIndex(1)], "z1".into()), + (vec![MapEach, ArrayIndex(9)], array(Type::Bytes, [])), + ] { + // Selection consumes the function's temporary root. The compiled + // expression is also dropped before the selected payloads are inspected. + let result = { + let filter = compile_selection(&scheme, "owned_rows()", indexes.clone()); + filter.execute(&context).unwrap() + }; + assert_eq!(result, expected, "{indexes:?}"); + assert_original_owned_payload(&result); + } + for (call, indexes, expected) in [ + ( + "owned_rows()", + vec![MapKey("missing".into()), MapEach], + Type::array(Type::Bytes), + ), + ("absent()", vec![MapEach, MapEach], Type::array(Type::Bytes)), + ( + "absent()", + vec![MapKey("z".into()), ArrayIndex(0)], + Type::Bytes, + ), + ] { + let filter = compile_selection(&scheme, call, indexes); + assert_eq!(filter.execute(&context), Err(expected), "{call}"); + } + for (expression, expected) in [ + (r#"any(owned_rows()[*][*] eq "z1")"#, true), + (r#"all(owned_rows()[*][*] eq "z1")"#, false), + (r#"owned_rows()["z"][0] eq "z0""#, true), + (r#"owned_rows()["missing"][0] eq "x""#, false), + (r#"owned_rows()["missing"][0] ne "x""#, true), + (r#"any(absent()[*][*] eq "x")"#, false), + (r#"absent()["z"][0] ne "x""#, true), + ] { + assert_eq!( + scheme + .parse(expression) + .unwrap() + .compile() + .execute(&context) + .unwrap(), + expected, + "{expression}" + ); + } +} + +#[test] +fn flattened_borrowed_function_results_keep_source_payloads() { + let payloads = [String::from("first"), String::from("second")]; + let mut builder = SchemeBuilder::new(); + let rows = Type::map(Type::array(Type::Bytes)); + builder.add_field("rows", rows).unwrap(); + add_function( + &mut builder, + "echo", + &[rows], + rows, + SimpleFunctionImpl::new(identity), + ); + let scheme = builder.build(); + let mut context = ExecutionContext::new(&scheme); + context + .set_field_value_from_name( + "rows", + map( + Type::array(Type::Bytes), + [ + ( + b"z".as_slice(), + array(Type::Bytes, [payloads[1].as_str().into()]), + ), + ( + b"a".as_slice(), + array(Type::Bytes, [payloads[0].as_str().into()]), + ), + ], + ), + ) + .unwrap(); + let result = compile_selection(&scheme, "echo(rows)", vec![FieldIndex::MapEach; 2]) + .execute(&context) + .unwrap(); + let LhsValue::Array(values) = result else { + panic!("expected flattened array") + }; + assert_eq!(values.len(), 2); + for (value, original) in values.iter().zip(&payloads) { + let LhsValue::Bytes(Bytes::Borrowed(bytes)) = value else { + panic!("borrowed function payload was copied") + }; + assert_eq!(*bytes, original.as_bytes()); + assert_eq!(bytes.as_ptr(), original.as_ptr()); + } +} + +type Rows = Vec, Vec>>; + +struct NativeContext { + state: ContextState, + rows: Rows, + missing: Option, + flags: Vec, +} + +impl HasScheme for NativeContext { + fn scheme(&self) -> &Scheme { + self.state.scheme() + } +} + +impl EvaluationContext for NativeContext { + type UserData = (); + + fn get_user_data(&self) -> &() { + self.state.get_user_data() + } + + fn get_list_matcher(&self, list: ListRef<'_>) -> &dyn ListMatcher { + self.state.get_list_matcher(list) + } +} + +wirefilter::wirefilter_fields! { + impl NativeFields for NativeContext { + "rows" => { fn(ctx: &NativeContext) -> &Rows { &ctx.rows } }, + "missing" => { fn(ctx: &NativeContext) -> Option<&Rows> { ctx.missing.as_ref() } }, + "flags" => { fn(ctx: &NativeContext) -> &Vec { &ctx.flags } }, + } +} + +fn native_context() -> NativeContext { + let mut builder = SchemeBuilder::new(); + let rows = Type::array(Type::map(Type::array(Type::Int))); + builder.add_field("rows", rows).unwrap(); + builder.add_optional_field("missing", rows).unwrap(); + builder.add_field("flags", Type::array(Type::Bool)).unwrap(); + add_function( + &mut builder, + "echo_flags", + &[Type::array(Type::Bool)], + Type::array(Type::Bool), + SimpleFunctionImpl::new(identity), + ); + let scheme = builder.build(); + NativeFields::validate_scheme(&scheme).unwrap(); + NativeContext { + state: ContextState::new(&scheme), + rows: vec![ + BTreeMap::from([ + (Box::from(b"z".as_slice()), vec![1, 2]), + (Box::from(b"a".as_slice()), vec![]), + ]), + BTreeMap::new(), + BTreeMap::from([(Box::from(b"a".as_slice()), vec![3])]), + ], + missing: None, + flags: vec![], + } +} + +struct Nonempty; + +impl> Compare<(), F> for Nonempty { + fn compare<'ctx, 'data: 'ctx>( + &self, + value: &LhsValue<'ctx>, + _: &'ctx F::Context<'data>, + ) -> bool { + match value { + LhsValue::Array(values) => !values.is_empty(), + LhsValue::Map(values) => !values.is_empty(), + _ => panic!("expected custom comparison of a collection"), + } + } +} + +#[test] +fn native_custom_collection_comparisons_cover_flat_and_nested_selections() { + use FieldIndex::{ArrayIndex, MapEach, MapKey}; + let context = native_context(); + for (field, indexes, expected) in [ + ("rows", vec![], true.into()), + ("rows", vec![ArrayIndex(0)], true.into()), + ( + "rows", + vec![ArrayIndex(0), MapKey("a".into())], + false.into(), + ), + ( + "rows", + vec![MapEach], + array(Type::Bool, [true.into(), false.into(), true.into()]), + ), + ( + "rows", + vec![MapEach, MapEach], + array(Type::Bool, [false.into(), true.into(), true.into()]), + ), + ( + "rows", + vec![MapEach, MapKey("a".into())], + array(Type::Bool, [false.into(), true.into()]), + ), + ( + "rows", + vec![MapEach, MapKey("missing".into())], + array(Type::Bool, []), + ), + ("rows", vec![ArrayIndex(1), MapEach], array(Type::Bool, [])), + ("missing", vec![MapEach], array(Type::Bool, [])), + ("missing", vec![MapEach, MapEach], array(Type::Bool, [])), + ] { + let index = IndexExpr { + identifier: IdentifierExpr::Field( + context.scheme().get_field(field).unwrap().to_owned(), + ), + indexes: indexes.into_boxed_slice(), + }; + let compiled = index.compile_with( + &mut DefaultCompiler::<(), NativeFields>::new(), + false, + Nonempty, + ); + let actual = match compiled { + CompiledExpr::One(value) => value.execute(&context).into(), + CompiledExpr::Vec(value) => LhsValue::Array(value.execute(&context).into()), + }; + assert_eq!(actual, expected, "{field}"); + } + for field in ["rows", "missing"] { + for default in [false, true] { + let index = IndexExpr { + identifier: IdentifierExpr::Field( + context.scheme().get_field(field).unwrap().to_owned(), + ), + indexes: Box::new([ArrayIndex(9)]), + }; + let CompiledExpr::One(compiled) = index.compile_with( + &mut DefaultCompiler::<(), NativeFields>::new(), + default, + Nonempty, + ) else { + panic!("expected scalar result") + }; + assert_eq!(compiled.execute(&context), default); + } + } +} + +#[test] +fn implicit_boolean_array_iteration_covers_fields_and_function_results() { + let mut context = native_context(); + for flags in [ + vec![], + vec![false], + vec![true, false, true], + vec![true, true], + ] { + for (expression, expected) in [ + ("any(flags)", flags.iter().any(|&value| value)), + ("all(flags)", flags.iter().all(|&value| value)), + ("any(echo_flags(flags))", flags.iter().any(|&value| value)), + ("all(echo_flags(flags))", flags.iter().all(|&value| value)), + ] { + context.flags.clone_from(&flags); + let compiled = context + .scheme() + .parse(expression) + .unwrap() + .compile_with_compiler(&mut DefaultCompiler::<(), NativeFields>::new()); + assert_eq!( + compiled.execute(&context).unwrap(), + expected, + "{expression}" + ); + } + } +}