diff --git a/Cargo.lock b/Cargo.lock index df973240..4f95ef27 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -142,6 +142,12 @@ version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + [[package]] name = "cast" version = "0.3.0" @@ -1226,6 +1232,7 @@ version = "0.7.0" dependencies = [ "arcstr", "backtrace", + "bytes", "cfg-if", "cidr", "criterion", diff --git a/Cargo.toml b/Cargo.toml index 1d8b2591..4dfcb6c5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,7 @@ publish = true [workspace.dependencies] arcstr = "1.2.0" backtrace = "0.3.76" +bytes = "1.12.1" cbindgen = "0.29.4" cfg-if = "1.0.4" cidr = { version = "0.2.3", features = ["serde"] } diff --git a/engine/Cargo.toml b/engine/Cargo.toml index 2add20fa..ca4d47c6 100644 --- a/engine/Cargo.toml +++ b/engine/Cargo.toml @@ -11,6 +11,12 @@ keywords = ["engine", "filter", "parser", "runtime", "wireshark"] categories = ["config", "parser-implementations"] publish.workspace = true +[features] +default = ["get-size2", "regex"] +bytes = ["dep:bytes"] +get-size2 = ["dep:get-size2"] +regex = ["dep:regex-automata"] + [lib] bench = false name = "wirefilter" @@ -22,6 +28,7 @@ name = "bench" [dependencies] arcstr.workspace = true backtrace.workspace = true +bytes = { workspace = true, optional = true } cfg-if.workspace = true cidr.workspace = true dyn-clone.workspace = true @@ -46,8 +53,3 @@ serde_json.workspace = true # This optional dependency allows us to build with `--features getrandom/wasm_js`. # For more information see: https://docs.rs/getrandom/#webassembly-support getrandom.workspace = true - -[features] -default = ["get-size2", "regex"] -get-size2 = ["dep:get-size2"] -regex = ["dep:regex-automata"] diff --git a/engine/benches/bench.rs b/engine/benches/bench.rs index e79d04db..16d02c4e 100644 --- a/engine/benches/bench.rs +++ b/engine/benches/bench.rs @@ -1,3 +1,6 @@ +mod lhs_value_reader; +mod reader_traversal; + use std::alloc::System; // Most of our usage will be via FFI as a dynamic library, so we're interested @@ -250,6 +253,8 @@ criterion_group! { bench_string_comparisons, bench_string_matches, bench_string_function_comparison, + lhs_value_reader::bench, + reader_traversal::bench, } criterion_main!(field_benchmarks); diff --git a/engine/benches/lhs_value_reader/mod.rs b/engine/benches/lhs_value_reader/mod.rs new file mode 100644 index 00000000..65e1a5a7 --- /dev/null +++ b/engine/benches/lhs_value_reader/mod.rs @@ -0,0 +1,279 @@ +//! Compare built-in and downstream field getters using the same LhsValue reader. +//! +//! `execution_context` uses the default family and internal field accessor. +//! `public_getter` uses a downstream getter and the public field accessor, which +//! checks the scheme on each read. + +use criterion::{BenchmarkId, Criterion}; +use std::hint::black_box; +use wirefilter::{ + Array, Compare, CompiledExpr, ContextFamily, DefaultCompiler, ExecutionContext, Field, + FieldBindingError, FieldConsumer, FieldGetter, FieldIndex, FunctionArgs, IdentifierExpr, + IndexExpr, LhsValue, LhsValueProvider, Map, Scheme, SchemeBuilder, SimpleFunctionArgKind, + SimpleFunctionDefinition, SimpleFunctionImpl, SimpleFunctionParam, Type, TypeMismatchError, +}; + +struct LhsFields; + +impl ContextFamily for LhsFields { + type Context<'data> = ExecutionContext<'data>; + type UserData = (); + + fn bind>( + field: Field, + consumer: B, + ) -> Result { + consumer.consume(field, LhsGetter) + } +} + +struct LhsGetter; + +impl FieldGetter for LhsGetter { + type Provider = LhsValueProvider; + + fn get<'ctx, 'data: 'ctx>( + &self, + context: &'ctx ExecutionContext<'data>, + field: &Field, + ) -> Option<&'ctx LhsValue<'ctx>> { + context.get_field_value(field.as_ref()) + } +} + +fn count<'a>(args: FunctionArgs<'_, 'a>) -> Option> { + let len = match args.next()?.ok()? { + LhsValue::Array(values) => values.len(), + LhsValue::Map(values) => values.len(), + _ => unreachable!(), + }; + Some(LhsValue::Int(i64::try_from(len).unwrap())) +} + +fn has_header<'a>(args: FunctionArgs<'_, 'a>) -> Option> { + let LhsValue::Bytes(name) = args.next()?.ok()? else { + unreachable!() + }; + let LhsValue::Map(headers) = args.next()?.ok()? else { + unreachable!() + }; + Some(LhsValue::Bool(headers.get(name.as_ref()).is_some())) +} + +fn function( + params: &[Type], + return_type: Type, + implementation: SimpleFunctionImpl, +) -> SimpleFunctionDefinition { + SimpleFunctionDefinition { + params: params + .iter() + .map(|&val_type| SimpleFunctionParam { + arg_kind: SimpleFunctionArgKind::Field, + val_type, + }) + .collect(), + opt_params: Box::new([]), + return_type, + implementation, + } +} + +fn context() -> (Scheme, ExecutionContext<'static>) { + let strings = Type::array(Type::Bytes); + let headers = Type::map(strings); + let mut builder = SchemeBuilder::new(); + for (name, ty) in [ + ("flag", Type::Bool), + ("port", Type::Int), + ("ip", Type::Ip), + ("text", Type::Bytes), + ("numbers", Type::array(Type::Int)), + ("matrix", Type::array(Type::array(Type::Int))), + ("names", strings), + ("headers", headers), + ] { + builder.add_field(name, ty).unwrap(); + } + for (name, params, output, implementation) in [ + ( + "lower", + vec![Type::Bytes], + Type::Bytes, + SimpleFunctionImpl::new(super::lowercase), + ), + ( + "count_values", + vec![strings], + Type::Int, + SimpleFunctionImpl::new(count), + ), + ( + "count_headers", + vec![headers], + Type::Int, + SimpleFunctionImpl::new(count), + ), + ( + "has_header", + vec![Type::Bytes, headers], + Type::Bool, + SimpleFunctionImpl::new(has_header), + ), + ] { + builder + .add_function(name, function(¶ms, output, implementation)) + .unwrap(); + } + let scheme = builder.build(); + let mut context = ExecutionContext::new(&scheme); + for (name, value) in [ + ("flag", LhsValue::Bool(true)), + ("port", LhsValue::Int(443)), + ("ip", LhsValue::Ip("192.0.2.1".parse().unwrap())), + ("text", LhsValue::from("example-value-0123456789")), + ( + "numbers", + Array::try_from_iter(Type::Int, 0..64_i64).unwrap().into(), + ), + ( + "matrix", + Array::try_from_iter( + Type::array(Type::Int), + (0..8).map(|row| Array::try_from_iter(Type::Int, row * 8..row * 8 + 8).unwrap()), + ) + .unwrap() + .into(), + ), + ( + "names", + Array::try_from_iter( + Type::Bytes, + (0..64).map(|index| LhsValue::from(format!("x-test-{index}").into_bytes())), + ) + .unwrap() + .into(), + ), + ( + "headers", + Map::try_from_iter( + strings, + (0..64).map(|index| { + Ok::<_, TypeMismatchError>(( + format!("x-test-{index}").into_bytes().into_boxed_slice(), + Array::try_from_iter(Type::Bytes, ["clean", "also-clean"]).unwrap(), + )) + }), + ) + .unwrap() + .into(), + ), + ] { + context + .set_field_value(scheme.get_field(name).unwrap(), value) + .unwrap(); + } + (scheme, context) +} + +pub(super) fn bench(c: &mut Criterion) { + let (scheme, context) = context(); + let mut group = c.benchmark_group("lhs_value_reader"); + for (name, expression) in [ + ("bool", "flag"), + ("integer_ordering", "port > 80"), + ("ip_membership", "ip in {192.0.2.0/24}"), + ("bytes_equal", r#"text eq "example-value-0123456789""#), + ("bytes_contains", r#"text contains "value""#), + ("array_index", "numbers[63] eq 63"), + ("array_missing", "numbers[100] eq 63"), + ("array_each", "any(numbers[*] eq 63)"), + ("nested_each", "any(matrix[*][*] eq 63)"), + ("map_index", r#"headers["x-test-0"][0] eq "clean""#), + ("map_missing", r#"headers["missing"][0] eq "clean""#), + ("map_each", r#"any(headers[*][*] eq "missing")"#), + ( + "selected_function", + r#"any(lower(headers["x-test-0"][*])[*] eq "clean")"#, + ), + ("array_function", "count_values(names) eq 64"), + ("map_function", "count_headers(headers) eq 64"), + ("map_each_function", "all(count_values(headers[*])[*] eq 2)"), + ("mapped_extra_map", "all(has_header(names[*], headers))"), + ] { + let ast = scheme.parse(expression).unwrap(); + let execution_context = ast.clone().compile(); + let public_getter = ast.compile_with_compiler(&mut DefaultCompiler::<(), LhsFields>::new()); + let expected = execution_context.execute(&context).unwrap(); + assert_eq!(expected, public_getter.execute(&context).unwrap(), "{name}"); + group.bench_function(BenchmarkId::new("execution_context", name), |b| { + b.iter(|| execution_context.execute(black_box(&context)).unwrap()) + }); + group.bench_function(BenchmarkId::new("public_getter", name), |b| { + b.iter(|| public_getter.execute(black_box(&context)).unwrap()) + }); + } + for (name, field, indexes) in [ + ("custom_map", "headers", vec![]), + ( + "custom_selected_array", + "headers", + vec![FieldIndex::MapKey("x-test-0".into())], + ), + ("custom_each_array", "matrix", vec![FieldIndex::MapEach]), + ] { + let index = IndexExpr { + identifier: IdentifierExpr::Field(scheme.get_field(field).unwrap().to_owned()), + indexes: indexes.into_boxed_slice(), + }; + let execution_context = + index + .clone() + .compile_with(&mut DefaultCompiler::<()>::new(), false, Nonempty); + let public_getter = index.compile_with( + &mut DefaultCompiler::<(), LhsFields>::new(), + false, + Nonempty, + ); + match (execution_context, public_getter) { + (CompiledExpr::One(execution_context), CompiledExpr::One(public_getter)) => { + let expected = execution_context.execute(&context); + assert_eq!(expected, public_getter.execute(&context)); + group.bench_function(BenchmarkId::new("execution_context", name), |b| { + b.iter(|| execution_context.execute(black_box(&context))) + }); + group.bench_function(BenchmarkId::new("public_getter", name), |b| { + b.iter(|| public_getter.execute(black_box(&context))) + }); + } + (CompiledExpr::Vec(execution_context), CompiledExpr::Vec(public_getter)) => { + let expected = execution_context.execute(&context); + assert_eq!(expected, public_getter.execute(&context)); + group.bench_function(BenchmarkId::new("execution_context", name), |b| { + b.iter(|| execution_context.execute(black_box(&context))) + }); + group.bench_function(BenchmarkId::new("public_getter", name), |b| { + b.iter(|| public_getter.execute(black_box(&context))) + }); + } + _ => panic!("comparison cardinality changed"), + } + } + group.finish(); +} + +struct Nonempty; + +impl> Compare<(), F> for Nonempty { + fn compare<'ctx, 'data: 'ctx>( + &self, + value: &LhsValue<'ctx>, + _: &'ctx F::Context<'data>, + ) -> bool { + match value { + LhsValue::Array(values) => !values.is_empty(), + LhsValue::Map(values) => !values.is_empty(), + _ => panic!("custom comparator expects a collection"), + } + } +} diff --git a/engine/benches/reader_traversal/mod.rs b/engine/benches/reader_traversal/mod.rs new file mode 100644 index 00000000..01e7c1b4 --- /dev/null +++ b/engine/benches/reader_traversal/mod.rs @@ -0,0 +1,337 @@ +//! Function-result traversal and native custom-collection comparison. + +use criterion::Criterion; +use std::hint::black_box; +use wirefilter::{ + Array, Compare, CompiledExpr, CompiledValueExpr, ContextFamily, DefaultCompiler, + EvaluationContext, ExecutionContext, FieldIndex, FilterValueExpr, FunctionArgs, HasScheme, + IdentifierExpr, IndexExpr, LhsValue, ListMatcher, ListRef, Scheme, SchemeBuilder, + SimpleFunctionArgKind, SimpleFunctionDefinition, SimpleFunctionImpl, SimpleFunctionParam, Type, + ValueExpr, +}; + +fn echo<'a>(args: FunctionArgs<'_, 'a>) -> Option> { + args.next()?.ok() +} + +fn own<'a>(args: FunctionArgs<'_, 'a>) -> Option> { + Some(args.next()?.ok()?.into_owned()) +} + +fn function(ty: Type, implementation: SimpleFunctionImpl) -> SimpleFunctionDefinition { + SimpleFunctionDefinition { + params: vec![SimpleFunctionParam { + arg_kind: SimpleFunctionArgKind::Field, + val_type: ty, + }] + .into_boxed_slice(), + opt_params: Box::new([]), + return_type: ty, + implementation, + } +} + +struct NativeContext { + scheme: Scheme, + number: i64, + optional_number: Option, + text: Box, + optional_text: Option>, + numbers: Vec, + matrix: Vec>, + cube: Vec>>, + flags: Vec, +} + +impl HasScheme for NativeContext { + fn scheme(&self) -> &Scheme { + &self.scheme + } +} + +impl EvaluationContext for NativeContext { + type UserData = (); + + fn get_user_data(&self) -> &() { + &() + } + + fn get_list_matcher(&self, _: ListRef<'_>) -> &dyn ListMatcher { + unreachable!("this benchmark has no lists") + } +} + +wirefilter::wirefilter_fields! { + impl NativeFields for NativeContext { + "number" => { fn(ctx: &NativeContext) -> i64 { ctx.number } }, + "optional_number" => { fn(ctx: &NativeContext) -> Option { ctx.optional_number } }, + "text" => { fn(ctx: &NativeContext) -> &str { &ctx.text } }, + "optional_text" => { fn(ctx: &NativeContext) -> Option<&str> { ctx.optional_text.as_deref() } }, + "numbers" => { fn(ctx: &NativeContext) -> &[i64] { &ctx.numbers } }, + "matrix" => { fn(ctx: &NativeContext) -> &[Vec] { &ctx.matrix } }, + "cube" => { fn(ctx: &NativeContext) -> &[Vec>] { &ctx.cube } }, + "flags" => { fn(ctx: &NativeContext) -> &[bool] { &ctx.flags } }, + } +} + +fn context() -> (Scheme, ExecutionContext<'static>, NativeContext) { + let numbers = Type::array(Type::Int); + let matrix = Type::array(numbers); + let cube = Type::array(matrix); + let flags = Type::array(Type::Bool); + let mut builder = SchemeBuilder::new(); + for (name, ty) in [ + ("number", Type::Int), + ("text", Type::Bytes), + ("numbers", numbers), + ("matrix", matrix), + ("cube", cube), + ("flags", flags), + ] { + builder.add_field(name, ty).unwrap(); + builder + .add_function( + format!("echo_{name}"), + function(ty, SimpleFunctionImpl::new(echo)), + ) + .unwrap(); + builder + .add_function( + format!("own_{name}"), + function(ty, SimpleFunctionImpl::new(own)), + ) + .unwrap(); + } + builder + .add_optional_field("optional_number", Type::Int) + .unwrap(); + builder + .add_optional_field("optional_text", Type::Bytes) + .unwrap(); + let scheme = builder.build(); + let native = NativeContext { + scheme: scheme.clone(), + number: 63, + optional_number: Some(63), + text: "target".into(), + optional_text: Some("target".into()), + numbers: (0..64).collect(), + matrix: (0..8).map(|row| (row * 8..row * 8 + 8).collect()).collect(), + cube: (0..4) + .map(|plane| { + (0..4) + .map(|row| (plane * 16 + row * 4..plane * 16 + row * 4 + 4).collect()) + .collect() + }) + .collect(), + flags: (0..64).map(|i| i == 63).collect(), + }; + NativeFields::validate_scheme(&scheme).unwrap(); + let mut context = ExecutionContext::new(&scheme); + context + .set_field_value_from_name("number", native.number) + .unwrap(); + context + .set_field_value_from_name("numbers", Array::from_iter(native.numbers.iter().copied())) + .unwrap(); + context + .set_field_value_from_name( + "matrix", + Array::try_from_iter( + numbers, + native + .matrix + .iter() + .map(|row| Array::from_iter(row.iter().copied())), + ) + .unwrap(), + ) + .unwrap(); + context + .set_field_value_from_name("flags", Array::from_iter(native.flags.iter().copied())) + .unwrap(); + (scheme, context, native) +} + +fn compile_value_selection(scheme: &Scheme, expression: &str, depth: usize) -> CompiledValueExpr { + // Parse a complete function call, then benchmark its indexing node directly. + // Top-level value parsing rejects [*], while function arguments compile this + // same node and consume its flattened result. + let ast = scheme.parse_value(expression).unwrap(); + let FilterValueExpr::Index(mut index) = ast.expression().clone() else { + unreachable!() + }; + index.indexes = vec![FieldIndex::MapEach; depth].into_boxed_slice(); + index.compile() +} + +pub(super) fn bench(c: &mut Criterion) { + let (scheme, context, mut native) = context(); + let mut group = c.benchmark_group("reader_traversal"); + for (name, expression) in [ + ("function_scalar", "echo_number(number) eq 63"), + ("function_index", "echo_numbers(numbers)[63] eq 63"), + ( + "function_flat_borrowed", + "any(echo_numbers(numbers)[*] eq 63)", + ), + ("function_flat_owned", "any(own_numbers(numbers)[*] eq 63)"), + ( + "function_nested_borrowed", + "any(echo_matrix(matrix)[*][*] eq 63)", + ), + ( + "function_nested_owned", + "any(own_matrix(matrix)[*][*] eq 63)", + ), + ("implicit_bool_array", "any(flags)"), + ("implicit_bool_function_borrowed", "any(echo_flags(flags))"), + ("implicit_bool_function_owned", "any(own_flags(flags))"), + ] { + let filter = scheme.parse(expression).unwrap().compile(); + assert!(filter.execute(&context).unwrap(), "{name}"); + group.bench_function(name, |b| { + b.iter(|| filter.execute(black_box(&context)).unwrap()) + }); + } + for (name, expression, depth, expected_type, expected_len) in [ + ( + "value_root_borrowed", + "echo_matrix(matrix)", + 0, + Type::array(Type::array(Type::Int)), + 8, + ), + ( + "value_root_owned", + "own_matrix(matrix)", + 0, + Type::array(Type::array(Type::Int)), + 8, + ), + ( + "value_flat_borrowed", + "echo_numbers(numbers)", + 1, + Type::array(Type::Int), + 64, + ), + ( + "value_nested_borrowed", + "echo_matrix(matrix)", + 2, + Type::array(Type::Int), + 64, + ), + ( + "value_nested_owned", + "own_matrix(matrix)", + 2, + Type::array(Type::Int), + 64, + ), + ] { + let value = compile_value_selection(&scheme, expression, depth); + let result = value.execute(&context).unwrap(); + assert_eq!(wirefilter::GetType::get_type(&result), expected_type); + let LhsValue::Array(array) = result else { + unreachable!() + }; + assert_eq!(array.len(), expected_len); + group.bench_function(name, |b| { + b.iter(|| value.execute(black_box(&context)).unwrap()) + }); + } + let implicit = scheme + .parse("any(flags)") + .unwrap() + .compile_with_compiler(&mut DefaultCompiler::<(), NativeFields>::new()); + assert!(implicit.execute(&native).unwrap()); + group.bench_function("implicit_bool_native", |b| { + b.iter(|| implicit.execute(black_box(&native)).unwrap()) + }); + + for (name, expression) in [ + ("native_scalar", "number eq 63"), + ("native_optional_scalar", "optional_number eq 63"), + ("native_bytes", r#"text eq "target""#), + ("native_optional_bytes", r#"optional_text eq "target""#), + ("native_array", "any(numbers[*] eq 63)"), + ("native_nested", "any(matrix[*][*] eq 63)"), + ] { + let filter = scheme + .parse(expression) + .unwrap() + .compile_with_compiler(&mut DefaultCompiler::<(), NativeFields>::new()); + assert!(filter.execute(&native).unwrap(), "{name}"); + group.bench_function(name, |b| { + b.iter(|| filter.execute(black_box(&native)).unwrap()) + }); + } + + for (name, field, indexes) in [ + ("custom_native_root", "matrix", vec![]), + ( + "custom_native_index", + "matrix", + vec![FieldIndex::ArrayIndex(3)], + ), + ("custom_native_flat", "matrix", vec![FieldIndex::MapEach]), + ( + "custom_native_nested", + "cube", + vec![FieldIndex::MapEach, FieldIndex::MapEach], + ), + ] { + let expression = IndexExpr { + identifier: IdentifierExpr::Field(scheme.get_field(field).unwrap().to_owned()), + indexes: indexes.into_boxed_slice(), + } + .compile_with( + &mut DefaultCompiler::<(), NativeFields>::new(), + false, + Nonempty, + ); + match expression { + CompiledExpr::One(expression) => { + assert!(expression.execute(&native)); + group.bench_function(name, |b| b.iter(|| expression.execute(black_box(&native)))); + } + CompiledExpr::Vec(expression) => { + assert!(!expression.execute(&native).is_empty()); + group.bench_function(name, |b| b.iter(|| expression.execute(black_box(&native)))); + } + } + } + native.optional_number = None; + native.optional_text = None; + for (name, expression) in [ + ("native_missing_scalar", "optional_number eq 63"), + ("native_missing_bytes", r#"optional_text eq "target""#), + ] { + let filter = scheme + .parse(expression) + .unwrap() + .compile_with_compiler(&mut DefaultCompiler::<(), NativeFields>::new()); + assert!(!filter.execute(&native).unwrap(), "{name}"); + group.bench_function(name, |b| { + b.iter(|| filter.execute(black_box(&native)).unwrap()) + }); + } + group.finish(); +} + +struct Nonempty; + +impl> Compare<(), F> for Nonempty { + fn compare<'ctx, 'data: 'ctx>( + &self, + value: &LhsValue<'ctx>, + _: &'ctx F::Context<'data>, + ) -> bool { + let LhsValue::Array(values) = value else { + panic!("expected array") + }; + !values.is_empty() + } +} diff --git a/engine/src/ast/field_expr.rs b/engine/src/ast/field_expr.rs index 6b45ef65..e6abc240 100644 --- a/engine/src/ast/field_expr.rs +++ b/engine/src/ast/field_expr.rs @@ -6,16 +6,14 @@ use crate::ast::index_expr::{Compare, IndexExpr}; use crate::compiler::Compiler; use crate::filter::{CompiledExpr, CompiledOneExpr, CompiledVecExpr}; use crate::lex::{Lex, LexErrorKind, LexResult, LexWith, expect, skip_space, span}; -use crate::lhs_types::TypedArray; +use crate::lhs_types::{LhsValue, TypedArray}; use crate::range_set::RangeSet; use crate::rhs_types::{BytesExpr, ExplicitIpRange, ListName, Regex, Wildcard}; use crate::scheme::{Field, Identifier, List}; use crate::searcher::{EmptySearcher, MemmemSearcher}; use crate::strict_partial_ord::StrictPartialOrd; -use crate::types::{ - ExpectedTypeList, GetType, LhsValue, LiteralSet, LiteralValue, Type, TypeMismatchError, -}; -use crate::{ExecutionContext, Scheme}; +use crate::types::{ExpectedTypeList, GetType, LiteralSet, LiteralValue, Type, TypeMismatchError}; +use crate::{ContextFamily, EvaluationContext, Scheme}; use serde::{Serialize, Serializer}; use sliceslice::MemchrSearcher; use std::cmp::Ordering; @@ -710,11 +708,18 @@ macro_rules! cast_value { }; } -impl Compare for Wildcard { +impl> Compare + for Wildcard +{ #[inline] - fn compare<'e>(&self, value: &LhsValue<'e>, _: &'e ExecutionContext<'e, U>) -> bool { + fn compare<'e, 'data: 'e>(&self, value: &LhsValue<'e>, _: &'e F::Context<'data>) -> bool { self.is_match(cast_value!(value, Bytes)) } + + #[inline] + fn compare_bytes<'e, 'data: 'e>(&self, value: &'e [u8], _: &'e F::Context<'data>) -> bool { + self.is_match(value) + } } fn compile_dynamic_comparison( @@ -724,7 +729,7 @@ fn compile_dynamic_comparison( missing_one_default: bool, missing_both_default: bool, compiler: &mut C, -) -> CompiledExpr +) -> CompiledExpr where C: Compiler, F: for<'l, 'r> Fn(&LhsValue<'l>, &LhsValue<'r>) -> bool + Copy + Send + Sync + 'static, @@ -782,7 +787,7 @@ impl Expr for ComparisonExpr { } } - fn compile_with_compiler(self, compiler: &mut C) -> CompiledExpr { + fn compile_with_compiler(self, compiler: &mut C) -> CompiledExpr { let lhs = self.lhs; let nil_not_equal_behavior = lhs.identifier.scheme().nil_not_equal_behavior(); @@ -790,15 +795,24 @@ impl Expr for ComparisonExpr { ComparisonOpExpr::IsTrue => { struct IsTrue; - impl Compare for IsTrue { + impl> Compare for IsTrue { #[inline] - fn compare<'e>( + fn compare<'e, 'data: 'e>( &self, value: &LhsValue<'e>, - _: &'e ExecutionContext<'e, U>, + _: &'e F::Context<'data>, ) -> bool { *cast_value!(value, Bool) } + + #[inline] + fn compare_bool<'e, 'data: 'e>( + &self, + value: bool, + _: &'e F::Context<'data>, + ) -> bool { + value + } } if lhs.get_type() == Type::Bool { @@ -866,46 +880,55 @@ impl Expr for ComparisonExpr { }; macro_rules! gen_ordering { - ($op:tt, $def:ident) => { + ($ordering:ident, $op:tt, $def:ident) => { match rhs { LiteralValue::Bytes(bytes) => { - struct BytesOp(BytesExpr); + struct BytesOp(Box<[u8]>); - impl Compare for BytesOp { + impl> Compare for BytesOp { #[inline] - fn compare<'e>(&self, value: &LhsValue<'e>, _: &'e ExecutionContext<'e, U>) -> bool { + fn compare<'e, 'data: 'e>(&self, value: &LhsValue<'e>, _: &'e F::Context<'data>) -> bool { cast_value!(value, Bytes).as_ref() $op self.0.as_ref() } + #[inline] + fn compare_bytes<'e, 'data: 'e>(&self, value: &'e [u8], _: &'e F::Context<'data>) -> bool { + value $op self.0.as_ref() + } } - lhs.compile_with(compiler, $def, BytesOp(bytes)) + lhs.compile_with(compiler, $def, BytesOp(bytes.into())) } LiteralValue::Int(int) => { struct IntOp(i64); - impl Compare for IntOp { + impl> Compare for IntOp { #[inline] - fn compare<'e>(&self, value: &LhsValue<'e>, _: &'e ExecutionContext<'e, U>) -> bool { + fn compare<'e, 'data: 'e>(&self, value: &LhsValue<'e>, _: &'e F::Context<'data>) -> bool { *cast_value!(value, Int) $op self.0 } + #[inline] + fn compare_int<'e, 'data: 'e>(&self, value: i64, _: &'e F::Context<'data>) -> bool { + value $op self.0 + } } lhs.compile_with(compiler, $def, IntOp(int)) } LiteralValue::Ip(ip) => { - struct IpOp { - op: OrderingOp, - ip: IpAddr, - } + struct IpOp(IpAddr); - impl Compare for IpOp { + impl> Compare for IpOp { #[inline] - fn compare<'e>(&self, value: &LhsValue<'e>, _: &'e ExecutionContext<'e, U>) -> bool { - self.op.matches_opt(cast_value!(value, Ip).strict_partial_cmp(&self.ip)) + fn compare<'e, 'data: 'e>(&self, value: &LhsValue<'e>, _: &'e F::Context<'data>) -> bool { + OrderingOp::$ordering.matches_opt(cast_value!(value, Ip).strict_partial_cmp(&self.0)) + } + #[inline] + fn compare_ip<'e, 'data: 'e>(&self, value: IpAddr, _: &'e F::Context<'data>) -> bool { + OrderingOp::$ordering.matches_opt(value.strict_partial_cmp(&self.0)) } } - lhs.compile_with(compiler, $def, IpOp { op, ip }) + lhs.compile_with(compiler, $def, IpOp(ip)) } LiteralValue::Bool(_) | LiteralValue::Array(_) | LiteralValue::Map(_) => unreachable!(), } @@ -913,12 +936,12 @@ impl Expr for ComparisonExpr { } match op { - OrderingOp::NotEqual => gen_ordering!(!=, nil_not_equal_behavior), - OrderingOp::Equal => gen_ordering!(==, false), - OrderingOp::GreaterThanEqual => gen_ordering!(>=, false), - OrderingOp::LessThanEqual => gen_ordering!(<=, false), - OrderingOp::GreaterThan => gen_ordering!(>, false), - OrderingOp::LessThan => gen_ordering!(<, false), + OrderingOp::NotEqual => gen_ordering!(NotEqual, !=, nil_not_equal_behavior), + OrderingOp::Equal => gen_ordering!(Equal, ==, false), + OrderingOp::GreaterThanEqual => gen_ordering!(GreaterThanEqual, >=, false), + OrderingOp::LessThanEqual => gen_ordering!(LessThanEqual, <=, false), + OrderingOp::GreaterThan => gen_ordering!(GreaterThan, >, false), + OrderingOp::LessThan => gen_ordering!(LessThan, <, false), } } ComparisonOpExpr::Int { @@ -941,15 +964,24 @@ impl Expr for ComparisonExpr { struct BitwiseAnd(i64); - impl Compare for BitwiseAnd { + impl> Compare for BitwiseAnd { #[inline] - fn compare<'e>( + fn compare<'e, 'data: 'e>( &self, value: &LhsValue<'e>, - _: &'e ExecutionContext<'e, U>, + _: &'e F::Context<'data>, ) -> bool { cast_value!(value, Int) & self.0 != 0 } + + #[inline] + fn compare_int<'e, 'data: 'e>( + &self, + value: i64, + _: &'e F::Context<'data>, + ) -> bool { + value & self.0 != 0 + } } lhs.compile_with(compiler, false, BitwiseAnd(rhs)) } @@ -975,28 +1007,50 @@ impl Expr for ComparisonExpr { struct ArraySearcher(Avx2Searcher<[u8; N]>); - impl Compare for ArraySearcher { + impl> Compare + for ArraySearcher + { #[inline] - fn compare<'e>( + fn compare<'e, 'data: 'e>( &self, value: &LhsValue<'e>, - _: &'e ExecutionContext<'e, U>, + _: &'e F::Context<'data>, ) -> bool { unsafe { self.0.search_in(cast_value!(value, Bytes)) } } + + #[inline] + fn compare_bytes<'e, 'data: 'e>( + &self, + value: &'e [u8], + _: &'e F::Context<'data>, + ) -> bool { + // SAFETY: This searcher is only constructed after USE_AVX2 succeeds. + unsafe { self.0.search_in(value) } + } } struct BoxSearcher(Avx2Searcher>); - impl Compare for BoxSearcher { + impl> Compare for BoxSearcher { #[inline] - fn compare<'e>( + fn compare<'e, 'data: 'e>( &self, value: &LhsValue<'e>, - _: &'e ExecutionContext<'e, U>, + _: &'e F::Context<'data>, ) -> bool { unsafe { self.0.search_in(cast_value!(value, Bytes)) } } + + #[inline] + fn compare_bytes<'e, 'data: 'e>( + &self, + value: &'e [u8], + _: &'e F::Context<'data>, + ) -> bool { + // SAFETY: This searcher is only constructed after USE_AVX2 succeeds. + unsafe { self.0.search_in(value) } + } } fn slice_to_array(slice: &[u8]) -> [u8; N] { @@ -1079,15 +1133,25 @@ impl Expr for ComparisonExpr { struct WasmSearcher(Wasm32Searcher>); - impl Compare for WasmSearcher { + impl> Compare for WasmSearcher { #[inline] - fn compare<'e>( + fn compare<'e, 'data: 'e>( &self, value: &LhsValue<'e>, - _: &'e ExecutionContext<'e, U>, + _: &'e F::Context<'data>, ) -> bool { unsafe { self.0.search_in(cast_value!(value, Bytes)) } } + + #[inline] + fn compare_bytes<'e, 'data: 'e>( + &self, + value: &'e [u8], + _: &'e F::Context<'data>, + ) -> bool { + // SAFETY: This searcher is only constructed after USE_SIMD128 succeeds. + unsafe { self.0.search_in(value) } + } } let position = rng().random_range(1..bytes.len()); @@ -1099,7 +1163,9 @@ impl Expr for ComparisonExpr { search!(MemmemSearcher::new(bytes)) } - ComparisonOpExpr::Matches(regex) => lhs.compile_with(compiler, false, regex), + ComparisonOpExpr::Matches(regex) => { + lhs.compile_with(compiler, false, regex.into_compiled()) + } ComparisonOpExpr::Wildcard(wildcard) => lhs.compile_with(compiler, false, wildcard), ComparisonOpExpr::StrictWildcard(wildcard) => { lhs.compile_with(compiler, false, wildcard) @@ -1122,18 +1188,30 @@ impl Expr for ComparisonExpr { v6: RangeSet, } - impl Compare for OneOfIp { + impl> Compare for OneOfIp { #[inline] - fn compare<'e>( + fn compare<'e, 'data: 'e>( &self, value: &LhsValue<'e>, - _: &'e ExecutionContext<'e, U>, + _: &'e F::Context<'data>, ) -> bool { match cast_value!(value, Ip) { IpAddr::V4(addr) => self.v4.contains(addr), IpAddr::V6(addr) => self.v6.contains(addr), } } + + #[inline] + fn compare_ip<'e, 'data: 'e>( + &self, + value: IpAddr, + _: &'e F::Context<'data>, + ) -> bool { + match &value { + IpAddr::V4(addr) => self.v4.contains(addr), + IpAddr::V6(addr) => self.v6.contains(addr), + } + } } lhs.compile_with(compiler, false, OneOfIp { v4, v6 }) @@ -1143,15 +1221,24 @@ impl Expr for ComparisonExpr { struct OneOfInt(RangeSet); - impl Compare for OneOfInt { + impl> Compare for OneOfInt { #[inline] - fn compare<'e>( + fn compare<'e, 'data: 'e>( &self, value: &LhsValue<'e>, - _: &'e ExecutionContext<'e, U>, + _: &'e F::Context<'data>, ) -> bool { self.0.contains(cast_value!(value, Int)) } + + #[inline] + fn compare_int<'e, 'data: 'e>( + &self, + value: i64, + _: &'e F::Context<'data>, + ) -> bool { + self.0.contains(&value) + } } lhs.compile_with(compiler, false, OneOfInt(values)) @@ -1161,15 +1248,24 @@ impl Expr for ComparisonExpr { struct Contains(BTreeSet>); - impl Compare for Contains { + impl> Compare for Contains { #[inline] - fn compare<'e>( + fn compare<'e, 'data: 'e>( &self, value: &LhsValue<'e>, - _: &'e ExecutionContext<'e, U>, + _: &'e F::Context<'data>, ) -> bool { self.0.contains(cast_value!(value, Bytes).as_ref()) } + + #[inline] + fn compare_bytes<'e, 'data: 'e>( + &self, + value: &'e [u8], + _: &'e F::Context<'data>, + ) -> bool { + self.0.contains(value) + } } lhs.compile_with(compiler, false, Contains(values)) @@ -1187,12 +1283,12 @@ impl Expr for ComparisonExpr { list: List, } - impl Compare for InList { + impl> Compare for InList { #[inline] - fn compare<'e>( + fn compare<'e, 'data: 'e>( &self, value: &LhsValue<'e>, - ctx: &'e ExecutionContext<'e, U>, + ctx: &'e F::Context<'data>, ) -> bool { ctx.get_list_matcher_unchecked(&self.list) .match_value(self.name.as_str(), value) diff --git a/engine/src/ast/function_expr.rs b/engine/src/ast/function_expr.rs index d061f370..855049c3 100644 --- a/engine/src/ast/function_expr.rs +++ b/engine/src/ast/function_expr.rs @@ -12,9 +12,9 @@ use crate::functions::{ FunctionParamError, }; use crate::lex::{Lex, LexError, LexErrorKind, LexResult, LexWith, expect, skip_space, span}; -use crate::lhs_types::Array; +use crate::lhs_types::{Array, LhsValue}; use crate::scheme::Function; -use crate::types::{GetType, LhsValue, LiteralValue, Type}; +use crate::types::{GetType, LiteralValue, Type}; use serde::Serialize; use std::hash::{Hash, Hasher}; use std::iter::once; @@ -54,9 +54,18 @@ impl ValueExpr for FunctionCallArgExpr { } } - fn compile_with_compiler(self, compiler: &mut C) -> CompiledValueExpr { + fn compile_with_compiler( + self, + compiler: &mut C, + ) -> CompiledValueExpr { match self { FunctionCallArgExpr::IndexExpr(index_expr) => compiler.compile_index_expr(index_expr), + FunctionCallArgExpr::Literal(LiteralValue::Bytes(bytes)) => { + // Formatting is only needed by the AST. Keep the payload and + // clone it on access so the result can outlive this expression. + let bytes: Box<[u8]> = bytes.into(); + CompiledValueExpr::new(move |_| LhsValue::from(bytes.clone()).into()) + } FunctionCallArgExpr::Literal(literal) => { CompiledValueExpr::new(move |_| LhsValue::from(literal.clone()).into()) } @@ -70,7 +79,8 @@ impl ValueExpr for FunctionCallArgExpr { CompiledExpr::One(expr) => { CompiledValueExpr::new(move |ctx| LhsValue::from(expr.execute(ctx)).into()) } - CompiledExpr::Vec(expr) => CompiledValueExpr::new(move |ctx| { + CompiledExpr::Vec(expr) => CompiledValueExpr::new_with_context(move |ctx| { + let ctx = ctx.context(); let result = expr.execute(ctx); LhsValue::Array(result.into()).into() }), @@ -91,8 +101,8 @@ impl FunctionCallArgExpr { /// Returns `true` if re-evaluating this argument for every mapped element /// could be costly, i.e. it is a nested function call or a logical - /// sub-expression. Literals and plain field accesses are cheap to evaluate - /// repeatedly, so memoizing them would only add overhead. + /// sub-expression. Literals and stored field accesses are cheap to evaluate + /// repeatedly. Field conversion is classified separately at binding. fn is_expensive_to_reevaluate(&self) -> bool { match self { FunctionCallArgExpr::Literal(_) => false, @@ -267,7 +277,10 @@ impl ValueExpr for FunctionCallExpr { visitor.visit_function(&self.function) } - fn compile_with_compiler(self, compiler: &mut C) -> CompiledValueExpr { + fn compile_with_compiler( + self, + compiler: &mut C, + ) -> CompiledValueExpr { let return_type = self.return_type(); let Self { @@ -282,7 +295,7 @@ impl ValueExpr for FunctionCallExpr { .compile(&mut args.iter().map(|arg| arg.into()), context); // For `map_each`, only bother evaluating the non-mapped arguments once // (instead of once per element) when at least one of them is expensive - // to re-evaluate. For trivial arguments (literals / plain field + // to re-evaluate. For trivial arguments (literals / stored field // accesses) inline re-evaluation is just as cheap and avoids a // per-call allocation. let memoize_extra_args = map_each_count > 0 @@ -290,6 +303,15 @@ impl ValueExpr for FunctionCallExpr { .iter() .skip(1) .any(FunctionCallArgExpr::is_expensive_to_reevaluate); + // Field values can require whole-container conversion, including + // when an index subsequently selects one scalar. Classify the provider + // during compilation so ordinary stored field accesses stay cheap. + let memoize_converted_extra_args = map_each_count > 0 + && !memoize_extra_args + && args.iter().skip(1).any(|arg| { + matches!(arg, FunctionCallArgExpr::IndexExpr(index) + if index.materializes_collection(compiler)) + }); let mut args = args .into_iter() .map(|arg| compiler.compile_function_call_arg_expr(arg)); @@ -340,7 +362,8 @@ impl ValueExpr for FunctionCallExpr { } if args.is_empty() { - CompiledValueExpr::new(move |ctx| { + CompiledValueExpr::new_with_context(move |ctx| { + let ctx = ctx.context(); compute( first.execute(ctx), &call, @@ -350,7 +373,8 @@ impl ValueExpr for FunctionCallExpr { ) }) } else if memoize_extra_args { - CompiledValueExpr::new(move |ctx| { + CompiledValueExpr::new_with_context(move |ctx| { + let ctx = ctx.context(); // At least one non-mapped argument is expensive to // re-evaluate, so evaluate all of them once per call and // reuse them (cheaply cloned) for every element instead of @@ -367,8 +391,31 @@ impl ValueExpr for FunctionCallExpr { |elem| ExactSizeChain::new(once(Ok(elem)), extra_args.iter().cloned()), ) }) + } else if memoize_converted_extra_args { + CompiledValueExpr::new_with_context(move |ctx| { + let ctx = ctx.context(); + // Preserve lazy argument consumption: empty mapped inputs + // and functions that ignore an argument must not read it. + // Cloning cached LhsValues retains the function interface's + // ownership contract, but conversion runs only once. + let extra_args = std::cell::OnceCell::new(); + compute(first.execute(ctx), &call, return_type, |elem| { + let values = extra_args.get_or_init(|| { + (0..args.len()) + .map(|_| std::cell::OnceCell::new()) + .collect::>() + }); + ExactSizeChain::new( + once(Ok(elem)), + args.iter() + .zip(values.iter()) + .map(|(arg, value)| value.get_or_init(|| arg.execute(ctx)).clone()), + ) + }) + }) } else { - CompiledValueExpr::new(move |ctx| { + CompiledValueExpr::new_with_context(move |ctx| { + let ctx = ctx.context(); compute( first.execute(ctx), &call, @@ -385,7 +432,8 @@ impl ValueExpr for FunctionCallExpr { } } else { let args = args.collect::>(); - CompiledValueExpr::new(move |ctx| { + CompiledValueExpr::new_with_context(move |ctx| { + let ctx = ctx.context(); match call(&mut args.iter().map(|arg| arg.execute(ctx))) { Some(value) => { debug_assert!(value.get_type() == return_type); @@ -1510,4 +1558,868 @@ mod tests { "http.headers[*]" ); } + #[test] + fn test_indexed_native_function_arguments_materialize_only_the_selection() { + use crate::values::{ArrayRead, ArrayValue, IntoLhsValue, MapRead, MapValue, Value}; + use crate::{EvaluationContext, HasScheme, ListMatcher, ListRef}; + use std::cell::{Cell, RefCell}; + use std::collections::BTreeMap; + + struct HeaderValues { + values: Vec>, + gets: Cell, + iterations: Cell, + } + impl Value for HeaderValues { + type Read = ArrayRead; + type ValueType = crate::values::Array; + } + impl ArrayValue for HeaderValues { + type Element = crate::values::Bytes; + type Item<'a> = &'a Box<[u8]>; + type Iter<'a> = std::slice::Iter<'a, Box<[u8]>>; + + fn get(&self, index: usize) -> Option> { + self.gets.set(self.gets.get() + 1); + self.values.get(index) + } + + fn iter(&self) -> Self::Iter<'_> { + self.iterations.set(self.iterations.get() + 1); + self.values.iter() + } + } + struct Headers { + entries: BTreeMap, HeaderValues>, + keys: RefCell>>, + value_iterations: Cell, + entry_iterations: Cell, + } + impl Value for Headers { + type Read = MapRead; + type ValueType = crate::values::Map>; + } + impl MapValue for Headers { + type Element = crate::values::Array; + type Item<'a> = &'a HeaderValues; + type Iter<'a> = std::collections::btree_map::Values<'a, Box<[u8]>, HeaderValues>; + + fn get(&self, key: &[u8]) -> Option> { + self.keys.borrow_mut().push(key.into()); + self.entries.get(key) + } + + fn values(&self) -> Self::Iter<'_> { + self.value_iterations.set(self.value_iterations.get() + 1); + self.entries.values() + } + + fn for_each_entry<'a>(&'a self, mut visitor: impl FnMut(&[u8], Self::Item<'a>)) { + self.entry_iterations.set(self.entry_iterations.get() + 1); + for (key, value) in &self.entries { + visitor(key, value); + } + } + } + struct Request { + scheme: Scheme, + headers: Headers, + } + impl HasScheme for Request { + fn scheme(&self) -> &Scheme { + &self.scheme + } + } + impl EvaluationContext for Request { + type UserData = (); + + fn get_user_data(&self) -> &() { + &() + } + + fn get_list_matcher(&self, _: ListRef<'_>) -> &dyn ListMatcher { + unreachable!() + } + } + crate::wirefilter_fields! { + impl HeaderFields for Request { + "http.request.headers" => { + fn(request: &Request) -> &Headers { &request.headers } + }, + } + } + fn lower_borrowed<'a>(args: FunctionArgs<'_, 'a>) -> Option> { + let LhsValue::Bytes(mut value) = args.next()?.ok()? else { + unreachable!() + }; + assert!(matches!(value, crate::Bytes::Borrowed(_))); + if value.iter().any(u8::is_ascii_uppercase) { + value.to_mut().make_ascii_lowercase(); + } + Some(value.into()) + } + let array_type = Type::array(Type::Bytes); + let map_type = Type::map(array_type); + let mut builder = crate::SchemeBuilder::new(); + builder.add_field("http.request.headers", map_type).unwrap(); + for (name, val_type, implementation) in [ + ( + "lower", + Type::Bytes, + SimpleFunctionImpl::new(lower_borrowed), + ), + ( + "echo_array", + array_type, + SimpleFunctionImpl::new(echo_function), + ), + ("echo_map", map_type, SimpleFunctionImpl::new(echo_function)), + ] { + builder + .add_function( + name, + SimpleFunctionDefinition { + params: Box::new([SimpleFunctionParam { + arg_kind: SimpleFunctionArgKind::Field, + val_type, + }]), + opt_params: Box::new([]), + return_type: val_type, + implementation, + }, + ) + .unwrap(); + } + let scheme = builder.build(); + HeaderFields::validate_scheme(&scheme).unwrap(); + let entries: BTreeMap, Vec>> = (0..128) + .map(|index| { + ( + format!("x-bench-{index}").into_bytes().into_boxed_slice(), + vec![format!("value-{index}").into_bytes().into_boxed_slice()], + ) + }) + .collect(); + let request = || Request { + scheme: scheme.clone(), + headers: Headers { + entries: entries + .iter() + .map(|(key, values)| { + ( + key.clone(), + HeaderValues { + values: values.clone(), + gets: Cell::new(0), + iterations: Cell::new(0), + }, + ) + }) + .collect(), + keys: RefCell::new(Vec::new()), + value_iterations: Cell::new(0), + entry_iterations: Cell::new(0), + }, + }; + let mut stored = crate::ExecutionContext::new(&scheme); + stored + .set_field_value_from_name("http.request.headers", (&entries).into_lhs()) + .unwrap(); + // Expected accesses per evaluation: key, array lookup, array iteration, + // and whether the function needs the complete map. + for (expression, expected, key, array_gets, array_iterations, complete_map) in [ + ( + r#"any(lower(http.request.headers["x-bench-0"][*])[*] eq "value-0")"#, + true, + Some(b"x-bench-0".as_slice()), + 0, + 1, + false, + ), + ( + r#"lower(http.request.headers["x-bench-0"][0]) eq "value-0""#, + true, + Some(b"x-bench-0".as_slice()), + 1, + 0, + false, + ), + ( + r#"lower(http.request.headers["missing"][0]) eq """#, + false, + Some(b"missing".as_slice()), + 0, + 0, + false, + ), + ( + r#"any(lower(http.request.headers["missing"][*])[*] eq "")"#, + false, + Some(b"missing".as_slice()), + 0, + 0, + false, + ), + ( + r#"lower(http.request.headers["x-bench-0"][99]) eq """#, + false, + Some(b"x-bench-0".as_slice()), + 1, + 0, + false, + ), + ( + r#"any(echo_array(http.request.headers["x-bench-0"])[*] eq "value-0")"#, + true, + Some(b"x-bench-0".as_slice()), + 0, + 1, + false, + ), + ( + r#"echo_map(http.request.headers)["x-bench-0"][0] eq "value-0""#, + true, + None, + 0, + 1, + true, + ), + ] { + let native = request(); + let filter = scheme + .parse(expression) + .unwrap() + .compile_with_compiler(&mut crate::DefaultCompiler::<(), HeaderFields>::new()); + let stored_filter = scheme.parse(expression).unwrap().compile(); + for evaluations in 1..=4 { + assert_eq!(filter.execute(&native).unwrap(), expected, "{expression}"); + assert_eq!( + stored_filter.execute(&stored).unwrap(), + expected, + "{expression}" + ); + assert_eq!( + native.headers.keys.borrow().as_slice(), + key.map_or_else(Vec::new, |key| vec![Box::<[u8]>::from(key); evaluations]), + "{expression}", + ); + assert_eq!( + native.headers.value_iterations.get(), + evaluations * usize::from(complete_map), + "{expression}" + ); + assert_eq!( + native.headers.entry_iterations.get(), + evaluations * usize::from(complete_map), + "{expression}" + ); + for (name, values) in &native.headers.entries { + let selected = name.as_ref() == b"x-bench-0"; + assert_eq!( + values.gets.get(), + evaluations * if selected { array_gets } else { 0 }, + "{expression}: {name:?}" + ); + assert_eq!( + values.iterations.get(), + evaluations + * if selected { + array_iterations + } else { + usize::from(complete_map) + }, + "{expression}: {name:?}" + ); + } + } + } + + let native = request(); + let filter = scheme + .parse_value(r#"lower(http.request.headers["x-bench-0"][0])"#) + .unwrap() + .compile_with_compiler(&mut crate::DefaultCompiler::<(), HeaderFields>::new()); + let value = filter.execute(&native).unwrap().unwrap(); + let LhsValue::Bytes(crate::Bytes::Borrowed(value)) = value else { + panic!("selected header payload should remain borrowed"); + }; + assert!(std::ptr::eq( + value, + native.headers.entries[b"x-bench-0".as_slice()].values[0].as_ref() + )); + + for (selection, array_index) in + [("[*][0]", Some(0)), ("[*][*]", None), ("[*][99]", Some(99))] + { + for empty_map in [false, true] { + let mut native = request(); + if empty_map { + native.headers.entries.clear(); + } else { + native + .headers + .entries + .get_mut(b"x-bench-1".as_slice()) + .unwrap() + .values + .clear(); + native + .headers + .entries + .get_mut(b"x-bench-2".as_slice()) + .unwrap() + .values + .push(b"second-value".as_slice().into()); + } + let entries: BTreeMap<_, _> = native + .headers + .entries + .iter() + .map(|(key, values)| (key.clone(), values.values.clone())) + .collect(); + let mut stored = crate::ExecutionContext::new(&scheme); + stored + .set_field_value_from_name("http.request.headers", (&entries).into_lhs()) + .unwrap(); + let expression = format!("lower(http.request.headers{selection})"); + let filter = scheme + .parse_value(&expression) + .unwrap() + .compile_with_compiler(&mut crate::DefaultCompiler::<(), HeaderFields>::new()); + let stored_filter = scheme.parse_value(&expression).unwrap().compile(); + let expected: Vec<_> = native + .headers + .entries + .values() + .flat_map(|values| match array_index { + Some(index) => values.values.get(index).into_iter().collect::>(), + None => values.values.iter().collect(), + }) + .collect(); + + for evaluations in 1..=4 { + let value = filter.execute(&native).unwrap().unwrap(); + assert_eq!( + value, + stored_filter.execute(&stored).unwrap().unwrap(), + "{expression}" + ); + let LhsValue::Array(values) = value else { + panic!("flattened selection should return an array"); + }; + assert_eq!(values.len(), expected.len(), "{expression}"); + for (actual, expected) in values.into_iter().zip(&expected) { + let LhsValue::Bytes(crate::Bytes::Borrowed(actual)) = actual else { + panic!("selected header payload should remain borrowed"); + }; + assert!(std::ptr::eq(actual, expected.as_ref()), "{expression}"); + } + assert!(native.headers.keys.borrow().is_empty(), "{expression}"); + assert_eq!( + native.headers.value_iterations.get(), + evaluations, + "{expression}" + ); + assert_eq!(native.headers.entry_iterations.get(), 0, "{expression}"); + for values in native.headers.entries.values() { + assert_eq!( + values.gets.get(), + evaluations * usize::from(array_index.is_some()), + "{expression}" + ); + assert_eq!( + values.iterations.get(), + evaluations * usize::from(array_index.is_none()), + "{expression}" + ); + } + } + } + } + + struct PreserveMap { + argument_hooks: usize, + index_hooks: usize, + family: std::marker::PhantomData, + } + impl PreserveMap { + fn new() -> Self { + Self { + argument_hooks: 0, + index_hooks: 0, + family: std::marker::PhantomData, + } + } + } + impl> crate::Compiler for PreserveMap { + type Family = F; + type U = (); + + fn compile_function_call_arg_expr( + &mut self, + node: FunctionCallArgExpr, + ) -> CompiledValueExpr { + self.argument_hooks += 1; + let value = self.compile_value_expr(node); + CompiledValueExpr::new_with_context(move |ctx| { + let result = value.execute(ctx.context()); + assert!(matches!(&result, Ok(LhsValue::Map(_)))); + result + }) + } + + fn compile_index_expr( + &mut self, + node: IndexExpr, + ) -> CompiledValueExpr { + let terminal_map = node.indexes.as_ref() == [FieldIndex::MapEach]; + let value = self.compile_value_expr(node); + if terminal_map { + self.index_hooks += 1; + CompiledValueExpr::new_with_context(move |ctx| { + let result = value.execute(ctx.context()); + assert!(matches!(&result, Ok(LhsValue::Map(_)))); + result + }) + } else { + value + } + } + } + + // A single terminal `[*]` preserves the map visible to compiler hooks, + // even though the mapped function consumes only its values. + for (expression, argument_hooks) in [ + ("echo_array(http.request.headers[*])", 1), + ("echo_array(echo_map(http.request.headers)[*])", 2), + ] { + let mut native_compiler = PreserveMap::::new(); + let native_filter = scheme + .parse_value(expression) + .unwrap() + .compile_with_compiler(&mut native_compiler); + let mut stored_compiler = PreserveMap::>::new(); + let stored_filter = scheme + .parse_value(expression) + .unwrap() + .compile_with_compiler(&mut stored_compiler); + assert_eq!(native_compiler.argument_hooks, argument_hooks); + assert_eq!(native_compiler.index_hooks, 1); + assert_eq!(stored_compiler.argument_hooks, argument_hooks); + assert_eq!(stored_compiler.index_hooks, 1); + let native = request(); + assert_eq!( + native_filter.execute(&native).unwrap(), + stored_filter.execute(&stored).unwrap() + ); + assert_eq!(native.headers.entry_iterations.get(), 1); + } + } + + #[test] + fn test_mapped_native_extra_arguments_convert_once_and_remain_lazy() { + use crate::values::{IntoLhsValue, MapRead, MapValue, Value}; + use crate::{EvaluationContext, HasScheme, ListMatcher, ListRef}; + use std::cell::Cell; + use std::collections::BTreeMap; + use std::sync::OnceLock; + + struct CountingMap { + entries: BTreeMap, i64>, + conversions: Cell, + } + impl Value for CountingMap { + type Read = MapRead; + type ValueType = crate::values::Map; + } + impl MapValue for CountingMap { + type Element = crate::values::Int; + type Item<'a> = &'a i64; + type Iter<'a> = std::collections::btree_map::Values<'a, Box<[u8]>, i64>; + + fn get(&self, key: &[u8]) -> Option> { + self.entries.get(key) + } + + fn values(&self) -> Self::Iter<'_> { + self.entries.values() + } + + fn for_each_entry<'a>(&'a self, mut visitor: impl FnMut(&[u8], Self::Item<'a>)) { + self.conversions.set(self.conversions.get() + 1); + for (key, value) in &self.entries { + visitor(key, value); + } + } + } + struct Request { + scheme: Scheme, + numbers: Option>, + number: i64, + boolean: bool, + bytes: Box, + ip: std::net::IpAddr, + scalar_getters: Cell, + map: OnceLock, + provide_map: bool, + map_getters: Cell, + } + impl HasScheme for Request { + fn scheme(&self) -> &Scheme { + &self.scheme + } + } + impl EvaluationContext for Request { + type UserData = (); + + fn get_user_data(&self) -> &() { + &() + } + + fn get_list_matcher(&self, _: ListRef<'_>) -> &dyn ListMatcher { + unreachable!() + } + } + crate::wirefilter_fields! { + impl NativeFields for Request { + "numbers" => { + fn(request: &Request) -> Option<&Vec> { + request.numbers.as_ref() + } + }, + "number" => { + fn(request: &Request) -> i64 { + request.scalar_getters.set(request.scalar_getters.get() + 1); + request.number + } + }, + "boolean" => { + fn(request: &Request) -> bool { + request.scalar_getters.set(request.scalar_getters.get() + 1); + request.boolean + } + }, + "bytes" => { + fn(request: &Request) -> &str { + request.scalar_getters.set(request.scalar_getters.get() + 1); + &request.bytes + } + }, + "ip" => { + fn(request: &Request) -> std::net::IpAddr { + request.scalar_getters.set(request.scalar_getters.get() + 1); + request.ip + } + }, + "map" => { + fn(request: &Request) -> &CountingMap { + request.map_getters.set(request.map_getters.get() + 1); + assert!(request.provide_map, "mandatory map accessed while unavailable"); + request.map.get_or_init(|| CountingMap { + entries: BTreeMap::from([(Box::from(b"a".as_slice()), 1)]), + conversions: Cell::new(0), + }) + } + }, + } + } + fn consume_both<'a>(args: FunctionArgs<'_, 'a>) -> Option> { + args.next()?.ok()?; + args.next()?.ok()?; + Some(true.into()) + } + fn consume_first<'a>(args: FunctionArgs<'_, 'a>) -> Option> { + args.next()?.ok()?; + Some(true.into()) + } + fn identity<'a>(args: FunctionArgs<'_, 'a>) -> Option> { + args.next()?.ok() + } + let map_type = Type::map(Type::Int); + let mut builder = crate::SchemeBuilder::new(); + builder + .add_optional_field("numbers", Type::array(Type::Int)) + .unwrap(); + builder.add_field("map", map_type).unwrap(); + for (name, ty) in [ + ("number", Type::Int), + ("boolean", Type::Bool), + ("bytes", Type::Bytes), + ("ip", Type::Ip), + ] { + builder.add_field(name, ty).unwrap(); + } + for (name, extra, implementation) in [ + ("map_arg", map_type, SimpleFunctionImpl::new(consume_both)), + ("int_arg", Type::Int, SimpleFunctionImpl::new(consume_both)), + ( + "bool_arg", + Type::Bool, + SimpleFunctionImpl::new(consume_both), + ), + ( + "bytes_arg", + Type::Bytes, + SimpleFunctionImpl::new(consume_both), + ), + ("ip_arg", Type::Ip, SimpleFunctionImpl::new(consume_both)), + ( + "ignore_map", + map_type, + SimpleFunctionImpl::new(consume_first), + ), + ] { + builder + .add_function( + name, + SimpleFunctionDefinition { + params: Box::new([ + SimpleFunctionParam { + arg_kind: SimpleFunctionArgKind::Field, + val_type: Type::Int, + }, + SimpleFunctionParam { + arg_kind: SimpleFunctionArgKind::Field, + val_type: extra, + }, + ]), + opt_params: Box::new([]), + return_type: Type::Bool, + implementation, + }, + ) + .unwrap(); + } + builder + .add_function( + "echo_map", + SimpleFunctionDefinition { + params: Box::new([SimpleFunctionParam { + arg_kind: SimpleFunctionArgKind::Field, + val_type: map_type, + }]), + opt_params: Box::new([]), + return_type: map_type, + implementation: SimpleFunctionImpl::new(identity), + }, + ) + .unwrap(); + let scheme = builder.build(); + NativeFields::validate_scheme(&scheme).unwrap(); + let request = |numbers, provide_map| Request { + scheme: scheme.clone(), + numbers, + number: 1, + boolean: true, + bytes: Box::from("value"), + ip: std::net::Ipv4Addr::LOCALHOST.into(), + scalar_getters: Cell::new(0), + map: OnceLock::new(), + provide_map, + map_getters: Cell::new(0), + }; + let compile = |expression| { + scheme + .parse(expression) + .unwrap() + .compile_with_compiler(&mut crate::DefaultCompiler::<(), NativeFields>::new()) + }; + + struct StoredReads; + impl crate::Compiler for StoredReads { + type Family = crate::ExecutionContextFamily; + type U = Cell; + + fn compile_index_expr(&mut self, node: IndexExpr) -> CompiledValueExpr { + let track = matches!(&node.identifier, IdentifierExpr::Field(field) if field.name() == "map"); + let value = self.compile_value_expr(node); + if track { + CompiledValueExpr::::new_with_context(move |ctx| { + let ctx = ctx.context(); + let reads = ctx.get_user_data(); + reads.set(reads.get() + 1); + value.execute(ctx) + }) + } else { + value + } + } + } + let map = BTreeMap::from([(Box::<[u8]>::from(b"a".as_slice()), 1i64)]); + for (expression, conversions_per_call) in [ + ("any(map_arg(numbers[*], map))", 1), + (r#"any(int_arg(numbers[*], map["a"]))"#, 0), + ] { + let native = request(Some(vec![1, 2, 3]), true); + let filter = compile(expression); + assert!(native.map.get().is_none()); + assert_eq!(native.map_getters.get(), 0); + for calls in [1, 2] { + assert!(filter.execute(&native).unwrap()); + assert_eq!(native.map_getters.get(), calls); + assert_eq!( + native.map.get().unwrap().conversions.get(), + calls * conversions_per_call + ); + } + let stored_filter = scheme + .parse(expression) + .unwrap() + .compile_with_compiler(&mut StoredReads); + let mut stored = crate::ExecutionContext::new_with(&scheme, || Cell::new(0usize)); + stored + .set_field_value_from_name( + "numbers", + Array::try_from_iter(Type::Int, [1i64, 2, 3]).unwrap(), + ) + .unwrap(); + stored + .set_field_value_from_name("map", (&map).into_lhs()) + .unwrap(); + assert!(stored_filter.execute(&stored).unwrap()); + assert_eq!( + stored.get_user_data().get(), + 3, + "stored fields keep their existing cheap access path" + ); + + for numbers in [Some(Vec::new()), None] { + for provide_map in [false, true] { + let native = request(numbers.clone(), provide_map); + assert!(!filter.execute(&native).unwrap()); + assert_eq!(native.map_getters.get(), 0); + assert!(native.map.get().is_none()); + let mut stored = + crate::ExecutionContext::new_with(&scheme, || Cell::new(0usize)); + if let Some(numbers) = &numbers { + stored + .set_field_value_from_name( + "numbers", + Array::try_from_iter(Type::Int, numbers.iter().copied()).unwrap(), + ) + .unwrap(); + } + assert!(!stored_filter.execute(&stored).unwrap()); + assert_eq!(stored.get_user_data().get(), 0); + } + } + } + struct CountingBindings { + fields: BTreeMap, usize>, + family: std::marker::PhantomData F>, + } + impl CountingBindings { + fn new() -> Self { + Self { + fields: BTreeMap::new(), + family: std::marker::PhantomData, + } + } + } + impl crate::Compiler for CountingBindings { + type Family = F; + type U = F::UserData; + + fn bind_field>( + &mut self, + field: crate::Field, + consumer: B, + ) -> Result { + *self.fields.entry(field.name().into()).or_default() += 1; + F::bind(field, consumer) + } + } + // Scalar roots only bind for value compilation. Indexed collection roots + // still need classification, even though the argument itself is scalar. + for (expression, extra_field, bindings, scalar_getters) in [ + ("any(int_arg(numbers[*], number))", "number", 1, 3), + ("any(bool_arg(numbers[*], boolean))", "boolean", 1, 3), + ("any(bytes_arg(numbers[*], bytes))", "bytes", 1, 3), + ("any(ip_arg(numbers[*], ip))", "ip", 1, 3), + (r#"any(int_arg(numbers[*], map["a"]))"#, "map", 2, 0), + // The mapped argument contributes the third binding of numbers. + ("any(int_arg(numbers[*], numbers[0]))", "numbers", 3, 0), + ] { + let native = request(Some(vec![1, 2, 3]), true); + let mut native_compiler = CountingBindings::::new(); + let native_filter = scheme + .parse(expression) + .unwrap() + .compile_with_compiler(&mut native_compiler); + assert_eq!( + native_compiler.fields[extra_field], bindings, + "{expression}" + ); + assert_eq!(native.scalar_getters.get(), 0); + let native_bindings = native_compiler.fields.clone(); + + let mut stored_compiler = CountingBindings::>::new(); + let stored_filter = scheme + .parse(expression) + .unwrap() + .compile_with_compiler(&mut stored_compiler); + assert_eq!( + stored_compiler.fields[extra_field], bindings, + "{expression}" + ); + let stored_bindings = stored_compiler.fields.clone(); + let mut stored = crate::ExecutionContext::new(&scheme); + stored + .set_field_value_from_name( + "numbers", + Array::try_from_iter(Type::Int, [1i64, 2, 3]).unwrap(), + ) + .unwrap(); + for (name, value) in [ + ("number", native.number.into()), + ("boolean", native.boolean.into()), + ("bytes", native.bytes.as_bytes().into()), + ("ip", native.ip.into()), + ("map", (&map).into_lhs()), + ] { + stored.set_field_value_from_name(name, value).unwrap(); + } + for calls in [1, 2] { + assert!(native_filter.execute(&native).unwrap(), "{expression}"); + assert!(stored_filter.execute(&stored).unwrap(), "{expression}"); + assert_eq!(native.scalar_getters.get(), calls * scalar_getters); + assert_eq!(native_compiler.fields, native_bindings); + assert_eq!(stored_compiler.fields, stored_bindings); + } + } + // FunctionArgs may leave an extra argument unconsumed even when a mapped + // call runs. Its mandatory field must remain unread in that case. + let ignored = request(Some(vec![1, 2, 3]), false); + assert!( + compile("any(ignore_map(numbers[*], map))") + .execute(&ignored) + .unwrap() + ); + assert_eq!(ignored.map_getters.get(), 0); + assert!(ignored.map.get().is_none()); + + // Preserve the established eager handling for an existing expensive + // function argument, including when the mapped input is empty. + let eager_expression = "any(map_arg(numbers[*], echo_map(map)))"; + let empty = request(Some(Vec::new()), true); + assert!(!compile(eager_expression).execute(&empty).unwrap()); + assert_eq!(empty.map_getters.get(), 1); + assert_eq!(empty.map.get().unwrap().conversions.get(), 1); + let mut stored = crate::ExecutionContext::new_with(&scheme, || Cell::new(0usize)); + stored + .set_field_value_from_name("numbers", Array::new(Type::Int)) + .unwrap(); + stored + .set_field_value_from_name("map", (&map).into_lhs()) + .unwrap(); + assert!( + !scheme + .parse(eager_expression) + .unwrap() + .compile_with_compiler(&mut StoredReads) + .execute(&stored) + .unwrap() + ); + assert_eq!(stored.get_user_data().get(), 1); + } } diff --git a/engine/src/ast/index_expr.rs b/engine/src/ast/index_expr.rs index 7c35e56a..237d4905 100644 --- a/engine/src/ast/index_expr.rs +++ b/engine/src/ast/index_expr.rs @@ -2,17 +2,23 @@ use super::ValueExpr; use super::field_expr::IdentifierExpr; use super::parse::FilterParser; use super::visitor::{Visitor, VisitorMut}; +#[cfg(test)] +use crate::ExecutionContext; +use crate::Field; use crate::compiler::Compiler; -use crate::execution_context::ExecutionContext; +use crate::context::{ + ContextFamily, ExecutionContextFamily, FieldBindingError, FieldConsumer, FieldGetter, + ProviderFamily, check_field_type, +}; use crate::filter::{CompiledExpr, CompiledOneExpr, CompiledValueExpr, CompiledVecExpr}; use crate::lex::{Lex, LexErrorKind, LexResult, LexWith, expect, skip_space, span}; -use crate::lhs_types::{Array, Map, TypedArray}; +use crate::lhs_types::{Array, LhsValue, TypedArray}; use crate::scheme::{FieldIndex, IndexAccessError}; -use crate::types::{GetType, IntoIter, LhsValue, Type}; +use crate::types::{GetType, Type}; +use crate::values::{self, FieldValueType, IntoLhsValue, LhsValueRead, Value, ValuePredicate}; use serde::ser::SerializeSeq; use serde::{Serialize, Serializer}; - -const BOOL_ARRAY: TypedArray<'_, bool> = TypedArray::new(); +use std::net::IpAddr; /// IndexExpr is an expr that destructures an index into an IdentifierExpr. /// @@ -28,15 +34,6 @@ pub struct IndexExpr { pub indexes: Box<[FieldIndex]>, } -#[allow(clippy::manual_ok_err)] -#[inline] -pub fn ok_ref(result: &Result) -> Option<&T> { - match result { - Ok(x) => Some(x), - Err(_) => None, - } -} - impl ValueExpr for IndexExpr { #[inline] fn walk<'a, V: Visitor<'a>>(&'a self, visitor: &mut V) { @@ -56,71 +53,70 @@ impl ValueExpr for IndexExpr { } } - fn compile_with_compiler(self, compiler: &mut C) -> CompiledValueExpr { + fn compile_with_compiler( + self, + compiler: &mut C, + ) -> CompiledValueExpr { let mut ty = self.get_type(); - let map_each_count = self.map_each_count(); + let count = self.map_each_count(); + let flatten = match count { + 0 => false, + 1 if self.indexes.last() == Some(&FieldIndex::MapEach) => { + // Preserve the parent container: a terminal `map[*]` evaluates + // to a Map here, and mapped-function execution consumes its values. + // Native maps could instead produce an array of values directly, + // avoiding map materialization and key copies. That would change + // the value visible to Compiler hooks wrapping this expression + // (e.g. compile_index_expr / compile_function_call_arg_expr), so + // it requires a separate compatibility decision. + ty = Type::Array(ty.into()); + false + } + _ => true, + }; let Self { identifier, indexes, } = self; - - let last = match map_each_count { - 0 => Some(indexes.len()), - 1 if indexes.last() == Some(&FieldIndex::MapEach) => { - ty = Type::Array(ty.into()); - Some(indexes.len() - 1) - } - _ => None, + // Resolve the terminal `[*]` selection during compilation so the + // closure owns only the indices it follows, without a separate end offset. + let indexes = if flatten { + indexes + } else { + simplify_indexes(indexes) }; - if last == Some(0) { - // Fast path - match identifier { - IdentifierExpr::Field(f) => CompiledValueExpr::new(move |ctx| { - ctx.get_field_value_unchecked(&f) - .map(LhsValue::as_ref) - .ok_or(ty) - }), - IdentifierExpr::FunctionCallExpr(call) => compiler.compile_function_call_expr(call), - } - } else if let Some(last) = last { - // Average path - match identifier { - IdentifierExpr::Field(f) => CompiledValueExpr::new(move |ctx| { - ctx.get_field_value_unchecked(&f) - .and_then(|value| value.get_nested(&indexes[..last])) - .map(LhsValue::as_ref) - .ok_or(ty) - }), - IdentifierExpr::FunctionCallExpr(call) => { - let call = compiler.compile_function_call_expr(call); - CompiledValueExpr::new(move |ctx| { + match identifier { + IdentifierExpr::Field(field) => compiler + .bind_field( + field, + ValueConsumer { + indexes, + ty, + flatten, + }, + ) + .expect("field binding or path preparation failed"), + IdentifierExpr::FunctionCallExpr(call) => { + let call = compiler.compile_function_call_expr(call); + match (flatten, indexes.is_empty()) { + (false, true) => call, + (false, false) => CompiledValueExpr::new_with_context(move |ctx| { + let ctx = ctx.context(); call.execute(ctx) .ok() - .and_then(|val| val.extract_nested(&indexes[..last])) + .and_then(|value| value.extract_nested(&indexes)) .ok_or(ty) - }) - } - } - } else { - let return_type = Type::Array(ty.into()); - // Slow path - match identifier { - IdentifierExpr::Field(f) => CompiledValueExpr::new(move |ctx| { - let mut iter = MapEachIterator::from_indexes(&indexes[..]); - iter.reset( - ctx.get_field_value_unchecked(&f) - .map(LhsValue::as_ref) - .ok_or(return_type)?, - ); - Ok(LhsValue::Array(Array::try_from_iter(ty, iter).unwrap())) - }), - IdentifierExpr::FunctionCallExpr(call) => { - let call = compiler.compile_function_call_expr(call); - CompiledValueExpr::new(move |ctx| { - let mut iter = MapEachIterator::from_indexes(&indexes[..]); - iter.reset(call.execute(ctx).map_err(|_| return_type)?); - Ok(LhsValue::Array(Array::try_from_iter(ty, iter).unwrap())) - }) + }), + (true, _) => { + let return_type = Type::Array(ty.into()); + CompiledValueExpr::new_with_context(move |ctx| { + let ctx = ctx.context(); + let value = call.execute(ctx).map_err(|_| return_type)?; + let mut selected = Vec::new(); + value.append_lhs_at(&indexes, &mut selected); + Ok(LhsValue::Array(Array::try_from_vec(ty, selected).unwrap())) + }) + } } } } @@ -137,19 +133,511 @@ fn simplify_indexes(indexes: Box<[FieldIndex]>) -> Box<[FieldIndex]> { } } +fn with_terminal_each(indexes: Box<[FieldIndex]>) -> Vec { + let mut indexes = indexes.into_vec(); + if indexes.last() != Some(&FieldIndex::MapEach) { + // Boolean-array comparisons can request iteration without an explicit + // [*] in the parsed expression. Other field paths already contain it. + indexes.push(FieldIndex::MapEach); + } + indexes +} + /// Interface used to implement comparison against a value. -pub trait Compare: Send + Sync + 'static { +pub trait Compare = ExecutionContextFamily>: + Send + Sync + 'static +{ /// Returns true if the value matches the comparison. - fn compare<'e>(&self, value: &LhsValue<'e>, ctx: &'e ExecutionContext<'e, U>) -> bool; + fn compare<'ctx, 'data: 'ctx>( + &self, + value: &LhsValue<'ctx>, + ctx: &'ctx F::Context<'data>, + ) -> bool; + + /// Compare native bytes. Override to operate directly on the native value. + #[inline] + fn compare_bytes<'ctx, 'data: 'ctx>( + &self, + value: &'ctx [u8], + ctx: &'ctx F::Context<'data>, + ) -> bool { + self.compare(&LhsValue::Bytes(value.into()), ctx) + } + /// Compare a native integer. + #[inline] + fn compare_int<'ctx, 'data: 'ctx>(&self, value: i64, ctx: &'ctx F::Context<'data>) -> bool { + self.compare(&LhsValue::Int(value), ctx) + } + /// Compare a native Boolean. + #[inline] + fn compare_bool<'ctx, 'data: 'ctx>(&self, value: bool, ctx: &'ctx F::Context<'data>) -> bool { + self.compare(&LhsValue::Bool(value), ctx) + } + /// Compare a native IP address. + #[inline] + fn compare_ip<'ctx, 'data: 'ctx>(&self, value: IpAddr, ctx: &'ctx F::Context<'data>) -> bool { + self.compare(&LhsValue::Ip(value), ctx) + } +} + +#[inline] +fn required(value: Option, field: &Field) -> Option { + if value.is_none() && !field.optional() { + panic!( + "Field {} was registered as mandatory but not given a value", + field.name() + ); + } + value +} + +struct MaterializationCost; + +impl FieldConsumer for MaterializationCost { + type Output = bool; + + fn consume>(self, field: Field, _: G) -> Result { + // Stored values also use this interface, but borrowing their containers + // must not introduce the memoization allocation for native conversion. + Ok(<::Value<'static, 'static> as IntoLhsValue< + 'static, + >>::MATERIALIZES_COLLECTIONS + && matches!(field.get_type(), Type::Array(_) | Type::Map(_))) + } +} + +struct ValueConsumer { + indexes: Box<[FieldIndex]>, + ty: Type, + flatten: bool, +} + +impl FieldConsumer for ValueConsumer { + type Output = CompiledValueExpr; + + fn consume>( + self, + field: Field, + getter: G, + ) -> Result { + check_field_type::<::ValueType>(&field)?; + let Self { + indexes, + ty, + flatten, + } = self; + Ok(match (flatten, indexes.is_empty()) { + (false, true) => CompiledValueExpr::new_with_context(move |ctx| { + let ctx = ctx.context(); + read_field_root(getter.get(ctx, &field), &field, ty) + }), + (false, false) => CompiledValueExpr::new_with_context(move |ctx| { + let ctx = ctx.context(); + read_field_selection(getter.get(ctx, &field), &field, &indexes, ty) + }), + (true, _) => { + let return_type = Type::Array(ty.into()); + CompiledValueExpr::new_with_context(move |ctx| { + let ctx = ctx.context(); + read_field_each(getter.get(ctx, &field), &field, &indexes, ty, return_type) + }) + } + }) + } +} + +// Single-result and vector comparisons use separate consumers so binding +// returns the concrete compiled expression without a runtime mode dispatch. +struct OneComparisonConsumer

{ + indexes: Box<[FieldIndex]>, + default: bool, + comp: P, +} + +// Select the collection fallback during compilation; the mode is not retained +// by the executable expression. Items keeps one trailing iteration implicit. +enum CollectionComparisonMode { + Items, + Path, +} + +struct ArrayComparisonConsumer

{ + indexes: Vec, + comp: P, + mode: CollectionComparisonMode, +} + +struct ContextComparison<'p, 'ctx, 'data, F: ContextFamily, P> { + comp: &'p P, + ctx: &'ctx F::Context<'data>, +} + +impl> ValuePredicate + for ContextComparison<'_, '_, '_, F, P> +{ + #[inline] + fn lhs(&self, value: &LhsValue<'_>) -> bool { + self.comp.compare(value, self.ctx) + } + + #[inline] + fn bytes(&self, value: &[u8]) -> bool { + self.comp.compare_bytes(value, self.ctx) + } + + #[inline] + fn int(&self, value: i64) -> bool { + self.comp.compare_int(value, self.ctx) + } + + #[inline] + fn boolean(&self, value: bool) -> bool { + self.comp.compare_bool(value, self.ctx) + } + + #[inline] + fn ip(&self, value: IpAddr) -> bool { + self.comp.compare_ip(value, self.ctx) + } +} + +// Share type validation and path-type calculation across getter types. +fn comparison_input_type(field: &Field, depth: usize) -> Type { + let mut input_type = field.get_type(); + for _ in 0..depth { + input_type = input_type + .next() + .expect("parsed field path must have valid types"); + } + input_type +} + +// Prepare paths independently of getter types. +enum PreparedFieldPath { + Reader(S::Path), + Collection(Box<[FieldIndex]>), +} + +fn prepare_field_path( + field: &Field, + indexes: Vec, +) -> Result, FieldBindingError> { + check_field_type::(field)?; + let input_type = comparison_input_type(field, indexes.len()); + if needs_collection_conversion::() && matches!(input_type, Type::Array(_) | Type::Map(_)) { + return Ok(PreparedFieldPath::Collection(indexes.into_boxed_slice())); + } + let path = S::compile_field_path(field.get_type(), indexes, input_type) + .map_err(|error| FieldBindingError::Unsupported(error.0))?; + Ok(PreparedFieldPath::Reader(path)) +} + +// Preserve the constant guards that prevent scalar providers from instantiating +// collection-conversion helpers. Readers supporting direct collection comparison +// also bypass those helpers. +const fn needs_collection_conversion() -> bool { + !S::DIRECT_COLLECTION_COMPARISON + && matches!(S::STATIC_TYPE, None | Some(Type::Array(_) | Type::Map(_))) +} + +// Keep the execute_* helpers separate from getter-specific closures. +// Execution depends on the context family, produced value type and comparator, +// not the getter type, so fields with the same types can share execution code +// while their closures retain concrete getter calls. Moving these bodies into +// the closures substantially increased release-build compiler memory and code +// size. Leave inlining decisions to the compiler. +fn execute_one( + ctx: &F::Context<'_>, + field: &Field, + value: Option, + path: &::Path, + comp: &P, + default: bool, +) -> bool +where + F: ContextFamily, + V: Value, + P: Compare, +{ + let Some(value) = required(value, field) else { + return default; + }; + let predicate = ContextComparison:: { comp, ctx }; + values::compare_one(&value, path, &predicate).unwrap_or(default) +} + +fn execute_vec( + ctx: &F::Context<'_>, + field: &Field, + value: Option, + path: &::Path, + comp: &P, +) -> TypedArray<'static, bool> +where + F: ContextFamily, + V: Value, + P: Compare, +{ + let mut output = TypedArray::new(); + if let Some(value) = required(value, field) { + let predicate = ContextComparison:: { comp, ctx }; + values::compare_vec(&value, path, &predicate, &mut output); + } + output +} + +fn execute_collection_one<'ctx, 'data: 'ctx, F, V, P>( + ctx: &'ctx F::Context<'data>, + field: &Field, + value: Option, + indexes: &[FieldIndex], + comp: &P, + default: bool, +) -> bool +where + F: ContextFamily, + V: IntoLhsValue<'ctx>, + P: Compare, +{ + let Some(value) = required(value, field) else { + return default; + }; + // Select on native storage before materializing the comparator's collection. + let Some(value) = value.into_lhs_at(indexes) else { + return default; + }; + comp.compare(&value, ctx) +} + +fn execute_collection_items<'ctx, 'data: 'ctx, F, V, P>( + ctx: &'ctx F::Context<'data>, + field: &Field, + value: Option, + prefix: &[FieldIndex], + comp: &P, +) -> TypedArray<'static, bool> +where + F: ContextFamily, + V: IntoLhsValue<'ctx>, + P: Compare, +{ + let mut output = TypedArray::new(); + if let Some(value) = required(value, field) { + let Some(value) = value.into_lhs_at(prefix) else { + return output; + }; + let predicate = ContextComparison:: { comp, ctx }; + LhsValueRead::compare_items(&value, &[], &predicate, &mut output); + } + output +} + +fn execute_collection_path<'ctx, 'data: 'ctx, F, V, P>( + ctx: &'ctx F::Context<'data>, + field: &Field, + value: Option, + prefix: &[FieldIndex], + rest: &[FieldIndex], + comp: &P, +) -> TypedArray<'static, bool> +where + F: ContextFamily, + V: IntoLhsValue<'ctx>, + P: Compare, +{ + let mut output = TypedArray::new(); + if let Some(value) = required(value, field) { + let Some(value) = value.into_lhs_at(prefix) else { + return output; + }; + let predicate = ContextComparison:: { comp, ctx }; + LhsValueRead::compare_vec(&value, rest, &predicate, &mut output); + } + output +} + +fn read_field_root<'ctx, V: IntoLhsValue<'ctx>>( + value: Option, + field: &Field, + ty: Type, +) -> Result, Type> { + required(value, field).map(IntoLhsValue::into_lhs).ok_or(ty) +} + +fn read_field_selection<'ctx, V: IntoLhsValue<'ctx>>( + value: Option, + field: &Field, + indexes: &[FieldIndex], + ty: Type, +) -> Result, Type> { + required(value, field) + .and_then(|value| value.into_lhs_at(indexes)) + .ok_or(ty) +} + +fn read_field_each<'ctx, V: IntoLhsValue<'ctx>>( + value: Option, + field: &Field, + indexes: &[FieldIndex], + ty: Type, + return_type: Type, +) -> Result, Type> { + let value = required(value, field).ok_or(return_type)?; + // Missing children contribute no elements; an absent root remains an error. + // Only selected values are materialized, retaining terminal map[*] behavior. + let mut values = Vec::new(); + value.append_lhs_at(indexes, &mut values); + Ok(LhsValue::Array(Array::try_from_vec(ty, values).unwrap())) +} + +// Field array comparisons share this getter-specific execution closure. +fn compile_field_vec( + field: Field, + getter: G, + path: <::ValueType as FieldValueType>::Path, + comp: P, +) -> CompiledVecExpr +where + F: ContextFamily, + G: FieldGetter, + P: Compare, +{ + CompiledVecExpr::new(move |ctx| { + execute_vec::(ctx, &field, getter.get(ctx, &field), &path, &comp) + }) +} + +impl> FieldConsumer for OneComparisonConsumer

{ + type Output = CompiledOneExpr; + + fn consume>( + self, + field: Field, + getter: G, + ) -> Result { + let Self { + indexes, + default, + comp, + } = self; + let prepared = prepare_field_path::<::ValueType>( + &field, + indexes.into_vec(), + )?; + match prepared { + PreparedFieldPath::Reader(path) => Ok(CompiledOneExpr::new(move |ctx| { + execute_one::(ctx, &field, getter.get(ctx, &field), &path, &comp, default) + })), + PreparedFieldPath::Collection(indexes) => { + // Retain the constant root-type guard so scalar providers do not + // instantiate collection conversion closures behind this boundary. + if needs_collection_conversion::<::ValueType>() { + Ok(CompiledOneExpr::new(move |ctx| { + execute_collection_one::( + ctx, + &field, + getter.get(ctx, &field), + &indexes, + &comp, + default, + ) + })) + } else { + unreachable!("provider does not require collection conversion") + } + } + } + } +} + +impl> FieldConsumer for ArrayComparisonConsumer

{ + type Output = CompiledVecExpr; + + fn consume>( + self, + field: Field, + getter: G, + ) -> Result { + let Self { + indexes, + comp, + mode, + } = self; + let prepared = + prepare_field_path::<::ValueType>(&field, indexes)?; + match prepared { + PreparedFieldPath::Reader(path) => { + Ok(compile_field_vec::(field, getter, path, comp)) + } + PreparedFieldPath::Collection(indexes) => { + // Retain the constant guard to avoid instantiating conversion + // helpers for readers that never need collection conversion. + if needs_collection_conversion::<::ValueType>() { + match mode { + CollectionComparisonMode::Items => { + let prefix = simplify_indexes(indexes); + Ok(CompiledVecExpr::new(move |ctx| { + execute_collection_items::( + ctx, + &field, + getter.get(ctx, &field), + &prefix, + &comp, + ) + })) + } + CollectionComparisonMode::Path => { + // The path is fixed for this expression. Retain one + // split offset rather than searching during execution. + let prefix_len = indexes + .iter() + .position(|index| *index == FieldIndex::MapEach) + .unwrap_or(indexes.len()); + Ok(CompiledVecExpr::new(move |ctx| { + let (prefix, rest) = indexes.split_at(prefix_len); + execute_collection_path::( + ctx, + &field, + getter.get(ctx, &field), + prefix, + rest, + &comp, + ) + })) + } + } + } else { + unreachable!("provider does not require collection conversion") + } + } + } + } } impl IndexExpr { + /// Determine conversion cost while the field's provider type is known. + pub(crate) fn materializes_collection(&self, compiler: &mut C) -> bool { + match &self.identifier { + // Custom by-value descriptors may still convert before indexing. + // Conservatively memoize collection-backed arguments even when + // borrowed native storage can select a scalar without conversion. + IdentifierExpr::Field(field) + if matches!(field.get_type(), Type::Array(_) | Type::Map(_)) => + { + compiler + .bind_field(field.clone(), MaterializationCost) + .expect("field binding or path preparation failed") + } + IdentifierExpr::Field(_) | IdentifierExpr::FunctionCallExpr(_) => false, + } + } + fn compile_one_with( self, compiler: &mut C, default: bool, - comp: impl Compare, - ) -> CompiledOneExpr { + comp: impl Compare, + ) -> CompiledOneExpr { let Self { identifier, indexes, @@ -168,115 +656,105 @@ impl IndexExpr { }) } else { CompiledOneExpr::new(move |ctx| { - ok_ref(&call.execute(ctx)) - .and_then(|val| val.get_nested(&indexes)) - .map_or( - default, - #[inline] - |val| comp.compare(val, ctx), - ) - }) - } - } - IdentifierExpr::Field(f) => { - if indexes.is_empty() { - CompiledOneExpr::new(move |ctx| { - ctx.get_field_value_unchecked(&f) - .map(|value| comp.compare(value, ctx)) - .unwrap_or(default) - }) - } else { - CompiledOneExpr::new(move |ctx| { - ctx.get_field_value_unchecked(&f) - .and_then(|value| value.get_nested(&indexes)) - .map_or( - default, - #[inline] - |val| comp.compare(val, ctx), - ) + let Ok(value) = call.execute(ctx) else { + return default; + }; + LhsValueRead::compare_one( + &value, + &indexes, + &ContextComparison:: { comp: &comp, ctx }, + ) + .unwrap_or(default) }) } } + IdentifierExpr::Field(field) => compiler + .bind_field( + field, + OneComparisonConsumer { + indexes, + default, + comp, + }, + ) + .expect("field binding or path preparation failed"), } } pub(crate) fn compile_vec_with( self, compiler: &mut C, - comp: impl Compare, - ) -> CompiledVecExpr { + comp: impl Compare, + ) -> CompiledVecExpr { let Self { identifier, indexes, } = self; - let indexes = simplify_indexes(indexes); match identifier { IdentifierExpr::FunctionCallExpr(call) => { + // The reader can iterate children without retaining a terminal + // [*] entry in function-result closures. + let indexes = simplify_indexes(indexes); let call = compiler.compile_function_call_expr(call); CompiledVecExpr::new(move |ctx| { - let comp = ∁ - ok_ref(&call.execute(ctx)) - .and_then(|val| val.get_nested(&indexes)) - .map_or( - BOOL_ARRAY, - #[inline] - |val: &LhsValue<'_>| { - TypedArray::from_iter( - val.iter().unwrap().map(|item| comp.compare(item, ctx)), - ) - }, - ) + let mut output = TypedArray::new(); + if let Ok(value) = call.execute(ctx) { + LhsValueRead::compare_items( + &value, + &indexes, + &ContextComparison:: { comp: &comp, ctx }, + &mut output, + ); + } + output }) } - IdentifierExpr::Field(f) => CompiledVecExpr::new(move |ctx| { - let comp = ∁ - ctx.get_field_value_unchecked(&f) - .and_then(|value| value.get_nested(&indexes)) - .map_or( - BOOL_ARRAY, - #[inline] - |val: &LhsValue<'_>| { - TypedArray::from_iter( - val.iter().unwrap().map(|item| comp.compare(item, ctx)), - ) - }, - ) - }), + IdentifierExpr::Field(field) => compiler + .bind_field( + field, + ArrayComparisonConsumer { + indexes: with_terminal_each(indexes), + comp, + mode: CollectionComparisonMode::Items, + }, + ) + .expect("field binding or path preparation failed"), } } pub(crate) fn compile_iter_with( self, compiler: &mut C, - comp: impl Compare, - ) -> CompiledVecExpr { + comp: impl Compare, + ) -> CompiledVecExpr { let Self { identifier, indexes, } = self; match identifier { - IdentifierExpr::Field(f) => CompiledVecExpr::new(move |ctx| { - let mut iter = MapEachIterator::from_indexes(&indexes[..]); - match ctx.get_field_value_unchecked(&f) { - Some(value) => iter.reset(value.as_ref()), - None => return TypedArray::default(), - }; - TypedArray::from_iter(iter.map(|item| comp.compare(&item, ctx))) - }), + IdentifierExpr::Field(field) => compiler + .bind_field( + field, + ArrayComparisonConsumer { + indexes: indexes.into_vec(), + comp, + mode: CollectionComparisonMode::Path, + }, + ) + .expect("field binding or path preparation failed"), IdentifierExpr::FunctionCallExpr(call) => { let call = compiler.compile_function_call_expr(call); CompiledVecExpr::new(move |ctx| { - let mut iter = MapEachIterator::from_indexes(&indexes[..]); - match call.execute(ctx) { - Ok(val) => { - iter.reset(val); - } - _ => { - return TypedArray::default(); - } + let mut output = TypedArray::new(); + if let Ok(value) = call.execute(ctx) { + LhsValueRead::compare_vec( + &value, + &indexes, + &ContextComparison:: { comp: &comp, ctx }, + &mut output, + ); } - - TypedArray::from_iter(iter.map(|item| comp.compare(&item, ctx))) + output }) } } @@ -288,8 +766,8 @@ impl IndexExpr { self, compiler: &mut C, default: bool, - comp: impl Compare, - ) -> CompiledExpr { + comp: impl Compare, + ) -> CompiledExpr { match self.map_each_count() { 0 => CompiledExpr::One(self.compile_one_with(compiler, default, comp)), 1 if self.indexes.last() == Some(&FieldIndex::MapEach) => { @@ -447,104 +925,12 @@ impl Serialize for IndexExpr { } } -enum FieldIndexIterator<'a, 'b> { - ArrayIndex(Option<(Array<'a>, u32)>), - MapKey(Option<(Map<'a>, &'b [u8])>), - MapEach(IntoIter<'a>), -} - -impl<'a, 'b> FieldIndexIterator<'a, 'b> { - fn new(val: LhsValue<'a>, idx: &'b FieldIndex) -> Result { - match idx { - FieldIndex::ArrayIndex(idx) => match val { - LhsValue::Array(arr) => Ok(Self::ArrayIndex(Some((arr, *idx)))), - _ => Err(IndexAccessError { - index: FieldIndex::ArrayIndex(*idx), - actual: val.get_type(), - }), - }, - FieldIndex::MapKey(key) => match val { - LhsValue::Map(map) => Ok(Self::MapKey(Some((map, key.as_bytes())))), - _ => Err(IndexAccessError { - index: FieldIndex::MapKey(key.clone()), - actual: val.get_type(), - }), - }, - FieldIndex::MapEach => match val { - LhsValue::Array(_) | LhsValue::Map(_) => Ok(Self::MapEach(val.into_iter())), - _ => Err(IndexAccessError { - index: FieldIndex::MapEach, - actual: val.get_type(), - }), - }, - } - } -} - -impl<'a> Iterator for FieldIndexIterator<'a, '_> { - type Item = LhsValue<'a>; - - fn next(&mut self) -> Option { - match self { - Self::ArrayIndex(opt) => opt.take().and_then(|(arr, idx)| arr.extract(idx as usize)), - Self::MapKey(opt) => opt.take().and_then(|(map, key)| map.extract(key)), - Self::MapEach(iter) => iter.next(), - } - } -} - -struct MapEachIterator<'a, 'b> { - indexes: &'b [FieldIndex], - stack: Vec>, -} - -impl<'a, 'b> MapEachIterator<'a, 'b> { - fn from_indexes(indexes: &'b [FieldIndex]) -> Self { - Self { - indexes, - stack: Vec::with_capacity(indexes.len()), - } - } - - fn reset(&mut self, val: LhsValue<'a>) { - self.stack.clear(); - let first = self.indexes.first().unwrap(); - self.stack - .push(FieldIndexIterator::new(val, first).unwrap()); - } -} - -impl<'a> Iterator for MapEachIterator<'a, '_> { - type Item = LhsValue<'a>; - - fn next(&mut self) -> Option> { - while !self.stack.is_empty() { - assert!(self.stack.len() <= self.indexes.len()); - if let Some(nxt) = self.stack.last_mut().unwrap().next() { - // Check that current iterator is a leaf iterator - if self.stack.len() == self.indexes.len() { - // Return a value if a leaf iterator returned a value - return Some(nxt); - } else { - self.stack.push( - FieldIndexIterator::new(nxt, &self.indexes[self.stack.len()]).unwrap(), - ); - } - } else { - // Last iterator is finished, remove it - self.stack.pop(); - } - } - None - } -} - #[cfg(test)] mod tests { use super::*; use crate::ast::field_expr::IdentifierExpr; use crate::{ - Array, FieldIndex, FilterParser, FunctionArgs, FunctionCallArgExpr, FunctionCallExpr, + Array, FieldIndex, FilterParser, FunctionArgs, FunctionCallArgExpr, FunctionCallExpr, Map, Scheme, SchemeBuilder, SimpleFunctionArgKind, SimpleFunctionDefinition, SimpleFunctionImpl, SimpleFunctionParam, }; @@ -673,6 +1059,99 @@ mod tests { ); } + #[test] + fn test_function_return_selection() { + let mut ctx = ExecutionContext::new(&SCHEME); + ctx.set_field_value_from_name("test", Array::from_iter(["value"])) + .unwrap(); + let array = LhsValue::Array(Array::from_iter(["value"])); + for (expression, expected) in [ + ("array(test[0])", Ok(array.clone())), + ("array(test[0])[0]", Ok(LhsValue::from("value"))), + ("array(test[0])[*]", Ok(array.clone())), + ("array2(test[0])[0]", Ok(array.clone())), + ("array2(test[0])[0][*]", Ok(array.clone())), + ("array2(test[0])[*][0]", Ok(array.clone())), + ("array2(test[0])[*][*]", Ok(array)), + ("array2(test[0])[9][*]", Err(Type::array(Type::Bytes))), + ("array2(test[0])[*][9]", Ok(Array::new(Type::Bytes).into())), + ] { + let (index, rest) = FilterParser::new(&SCHEME) + .lex_as::(expression) + .unwrap(); + assert!(rest.is_empty()); + assert_eq!(index.compile().execute(&ctx), expected, "{expression}"); + } + } + + #[test] + fn test_terminal_map_each_preserves_function_result_and_error_types() { + fn echo<'a>(args: FunctionArgs<'_, 'a>) -> Option> { + args.next()?.ok() + } + + let map_type = Type::map(Type::Bytes); + let mut builder = SchemeBuilder::new(); + for (field, function, ty) in [ + ("map", "echo_map", map_type), + ("maps", "echo_maps", Type::array(map_type)), + ] { + builder.add_optional_field(field, ty).unwrap(); + builder + .add_function( + function, + SimpleFunctionDefinition { + params: Box::new([SimpleFunctionParam { + arg_kind: SimpleFunctionArgKind::Field, + val_type: ty, + }]), + opt_params: Box::new([]), + return_type: ty, + implementation: SimpleFunctionImpl::new(echo), + }, + ) + .unwrap(); + } + let scheme = builder.build(); + let map = Map::try_from_iter::( + Type::Bytes, + [Ok((Box::from(b"key".as_slice()), "value"))], + ) + .unwrap(); + let missing = ExecutionContext::new(&scheme); + let mut present = ExecutionContext::new(&scheme); + present + .set_field_value_from_name("map", map.clone()) + .unwrap(); + present + .set_field_value_from_name( + "maps", + Array::try_from_iter(map_type, [map.clone()]).unwrap(), + ) + .unwrap(); + for (expression, missing_type) in [ + ("map[*]", Type::array(Type::Bytes)), + ("echo_map(map)[*]", map_type), + ("echo_maps(maps)[0][*]", Type::array(Type::Bytes)), + ] { + let (index, rest) = FilterParser::new(&scheme) + .lex_as::(expression) + .unwrap(); + assert!(rest.is_empty()); + let compiled = index.compile(); + assert_eq!( + compiled.execute(&present), + Ok(LhsValue::Map(map.clone())), + "{expression}" + ); + assert_eq!( + compiled.execute(&missing), + Err(missing_type), + "{expression}" + ); + } + } + #[test] fn test_function_call_with_missing_argument_then_index_access() { let expr = assert_ok!( @@ -948,10 +1427,9 @@ mod tests { for i in 0..10 { let indexes = [FieldIndex::ArrayIndex(i), FieldIndex::MapEach]; - let mut iter = MapEachIterator::from_indexes(&indexes[..]); - - iter.reset(arr.clone()); - for (j, elem) in iter.enumerate() { + let mut selected = Vec::new(); + arr.clone().append_lhs_at(&indexes, &mut selected); + for (j, elem) in selected.into_iter().enumerate() { let bytes = match elem { LhsValue::Bytes(bytes) => bytes, _ => unreachable!(), @@ -960,10 +1438,9 @@ mod tests { } let indexes = [FieldIndex::MapEach, FieldIndex::ArrayIndex(i)]; - let mut iter = MapEachIterator::from_indexes(&indexes[..]); - - iter.reset(arr.clone()); - for (j, elem) in iter.enumerate() { + let mut selected = Vec::new(); + arr.clone().append_lhs_at(&indexes, &mut selected); + for (j, elem) in selected.into_iter().enumerate() { let bytes = match elem { LhsValue::Bytes(bytes) => bytes, _ => unreachable!(), @@ -973,12 +1450,12 @@ mod tests { } let indexes = [FieldIndex::MapEach, FieldIndex::MapEach]; - let mut iter = MapEachIterator::from_indexes(&indexes[..]); + let mut selected = Vec::new(); + arr.append_lhs_at(&indexes, &mut selected); let mut i = 0; let mut j = 0; - iter.reset(arr.clone()); - for elem in iter { + for elem in selected { let bytes = match elem { LhsValue::Bytes(bytes) => bytes, _ => unreachable!(), @@ -988,4 +1465,319 @@ mod tests { i += (j == 0) as u32; } } + #[test] + fn test_native_custom_comparators_can_consume_collections() { + use crate::{DefaultCompiler, EvaluationContext, HasScheme, ListMatcher, ListRef}; + use std::collections::BTreeMap; + + struct Request { + scheme: Scheme, + rows: Vec>>, + maps: Vec, String>>, + } + impl HasScheme for Request { + fn scheme(&self) -> &Scheme { + &self.scheme + } + } + impl EvaluationContext for Request { + type UserData = (); + + fn get_user_data(&self) -> &() { + &() + } + + fn get_list_matcher(&self, _: ListRef<'_>) -> &dyn ListMatcher { + unreachable!() + } + } + crate::wirefilter_fields! { + impl NativeFields for Request { + "rows" => { + fn(request: &Request) -> &Vec>> { + &request.rows + } + }, + "maps" => { + fn(request: &Request) -> &Vec, String>> { + &request.maps + } + }, + } + } + struct Nonempty; + impl> Compare<(), F> for Nonempty { + fn compare<'ctx, 'data: 'ctx>( + &self, + value: &LhsValue<'ctx>, + _: &'ctx F::Context<'data>, + ) -> bool { + match value { + LhsValue::Array(values) => !values.is_empty(), + LhsValue::Map(values) => !values.is_empty(), + _ => panic!("custom comparator requires a complete collection"), + } + } + } + let scheme = crate::Scheme! { + rows: Array(Array(Array(Bytes))), + maps: Array(Map(Bytes)), + } + .build(); + let native = Request { + scheme: scheme.clone(), + rows: vec![vec![vec![], vec!["value".into()]], vec![]], + maps: vec![ + BTreeMap::new(), + BTreeMap::from([(b"key".as_slice().into(), "value".into())]), + ], + }; + NativeFields::validate_scheme(&scheme).unwrap(); + let mut stored = ExecutionContext::new(&scheme); + stored + .set_field_value(scheme.get_field("rows").unwrap(), (&native.rows).into_lhs()) + .unwrap(); + stored + .set_field_value(scheme.get_field("maps").unwrap(), (&native.maps).into_lhs()) + .unwrap(); + for (name, indexes) in [ + ("rows", vec![]), + ("rows", vec![FieldIndex::ArrayIndex(8)]), + ("rows", vec![FieldIndex::MapEach]), + ("rows", vec![FieldIndex::MapEach, FieldIndex::MapEach]), + ("maps", vec![FieldIndex::MapEach]), + ] { + for missing in [false, true] { + let index = IndexExpr { + identifier: IdentifierExpr::Field(scheme.get_field(name).unwrap().to_owned()), + indexes: indexes.clone().into_boxed_slice(), + }; + let expected = index.clone().compile_with( + &mut DefaultCompiler::<()>::new(), + missing, + Nonempty, + ); + let actual = index.compile_with( + &mut DefaultCompiler::<(), NativeFields>::new(), + missing, + Nonempty, + ); + match (expected, actual) { + (CompiledExpr::One(expected), CompiledExpr::One(actual)) => { + assert_eq!(expected.execute(&stored), actual.execute(&native)); + } + (CompiledExpr::Vec(expected), CompiledExpr::Vec(actual)) => { + assert_eq!(expected.execute(&stored), actual.execute(&native)); + } + _ => panic!("comparison cardinality changed"), + } + } + } + } + + #[test] + fn test_native_indexed_value_expressions_match_stored_values() { + use crate::{DefaultCompiler, EvaluationContext, HasScheme, ListMatcher, ListRef}; + use std::collections::BTreeMap; + + type Rows = BTreeMap, Vec>>>; + type Groups = Vec, Vec>>>; + struct Request { + scheme: Scheme, + rows: Option, + groups: Option, + } + impl HasScheme for Request { + fn scheme(&self) -> &Scheme { + &self.scheme + } + } + impl EvaluationContext for Request { + type UserData = (); + + fn get_user_data(&self) -> &() { + &() + } + + fn get_list_matcher(&self, _: ListRef<'_>) -> &dyn ListMatcher { + unreachable!() + } + } + crate::wirefilter_fields! { + impl NativeFields for Request { + "rows" => { + fn(request: &Request) -> Option<&Rows> { + request.rows.as_ref() + } + }, + "groups" => { + fn(request: &Request) -> Option<&Groups> { + request.groups.as_ref() + } + }, + } + } + let mut builder = SchemeBuilder::new(); + builder + .add_optional_field("rows", Type::map(Type::array(Type::array(Type::Bytes)))) + .unwrap(); + builder + .add_optional_field("groups", Type::array(Type::map(Type::array(Type::Bytes)))) + .unwrap(); + let scheme = builder.build(); + NativeFields::validate_scheme(&scheme).unwrap(); + + for present in [true, false] { + let native = Request { + scheme: scheme.clone(), + rows: present.then(|| { + BTreeMap::from([ + (Box::from(b"z".as_slice()), vec![vec!["z0".into()]]), + ( + Box::from(b"selected".as_slice()), + vec![vec!["s0".into(), "s1".into()], vec![], vec!["s2".into()]], + ), + (Box::from(b"empty".as_slice()), vec![]), + (Box::from(b"a".as_slice()), vec![vec!["a0".into()]]), + ]) + }), + groups: present.then(|| { + vec![ + BTreeMap::from([ + (Box::from(b"z".as_slice()), vec!["z1".into()]), + ( + Box::from(b"values".as_slice()), + vec!["v0".into(), "v1".into()], + ), + (Box::from(b"empty".as_slice()), vec![]), + (Box::from(b"a".as_slice()), vec!["a1".into()]), + ]), + BTreeMap::new(), + BTreeMap::from([(Box::from(b"values".as_slice()), vec!["v2".into()])]), + ] + }), + }; + let mut stored = ExecutionContext::new(&scheme); + if let Some(rows) = &native.rows { + stored + .set_field_value_from_name("rows", rows.into_lhs()) + .unwrap(); + } + if let Some(groups) = &native.groups { + stored + .set_field_value_from_name("groups", groups.into_lhs()) + .unwrap(); + } + + for expression in [ + "rows", + r#"rows["selected"]"#, + r#"rows["selected"][0]"#, + r#"rows["selected"][0][1]"#, + r#"rows["selected"][0][*]"#, + r#"rows["selected"][*]"#, + r#"rows["selected"][*][*]"#, + r#"rows["selected"][*][0]"#, + r#"rows["selected"][*][9]"#, + r#"rows["selected"][9]"#, + r#"rows["selected"][9][*]"#, + r#"rows["selected"][9][0]"#, + r#"rows["empty"][*]"#, + r#"rows["empty"][*][*]"#, + r#"rows["empty"][*][0]"#, + r#"rows["missing"]"#, + r#"rows["missing"][*]"#, + r#"rows["missing"][0][*]"#, + r#"rows["missing"][*][*]"#, + r#"rows["missing"][*][0]"#, + "rows[*]", + "rows[*][*]", + "rows[*][*][*]", + "rows[*][0][*]", + "groups", + "groups[0]", + "groups[0][*]", + "groups[0][*][*]", + "groups[0][*][0]", + r#"groups[0]["values"][*]"#, + r#"groups[0]["values"][1]"#, + r#"groups[0]["missing"][*]"#, + "groups[1][*]", + "groups[1][*][*]", + "groups[9][*]", + "groups[9][*][*]", + "groups[9][*][0]", + r#"groups[9]["values"][*]"#, + r#"groups[*]["values"][*]"#, + r#"groups[*]["values"][0]"#, + r#"groups[*]["missing"][*]"#, + "groups[*][*][*]", + ] { + let (index, rest) = FilterParser::new(&scheme) + .lex_as::(expression) + .unwrap(); + assert!(rest.is_empty(), "{expression}"); + let expected = index + .clone() + .compile_with_compiler(&mut DefaultCompiler::<()>::new()); + let actual = + index.compile_with_compiler(&mut DefaultCompiler::<(), NativeFields>::new()); + let expected = expected.execute(&stored); + let actual = actual.execute(&native); + assert_eq!(actual, expected, "{expression}, present={present}"); + if !present { + assert!(actual.is_err(), "{expression}"); + continue; + } + + // Check flattening order and the difference between a missing + // selection before `[*]` and a missing optional root. + let flat: Option<&[&str]> = match expression { + r#"rows["selected"][*][*]"# => Some(&["s0", "s1", "s2"]), + r#"rows["selected"][*][0]"# => Some(&["s0", "s2"]), + "rows[*][*][*]" => Some(&["a0", "s0", "s1", "s2", "z0"]), + "rows[*][0][*]" => Some(&["a0", "s0", "s1", "z0"]), + "groups[0][*][*]" => Some(&["a1", "v0", "v1", "z1"]), + "groups[0][*][0]" => Some(&["a1", "v0", "z1"]), + r#"groups[*]["values"][*]"# => Some(&["v0", "v1", "v2"]), + r#"groups[*]["values"][0]"# => Some(&["v0", "v2"]), + "groups[*][*][*]" => Some(&["a1", "v0", "v1", "z1", "v2"]), + r#"rows["selected"][*][9]"# + | r#"rows["empty"][*][*]"# + | r#"rows["empty"][*][0]"# + | r#"rows["missing"][*][*]"# + | r#"rows["missing"][*][0]"# + | "groups[1][*][*]" + | "groups[9][*][*]" + | "groups[9][*][0]" + | r#"groups[*]["missing"][*]"# => Some(&[]), + _ => None, + }; + if let Some(values) = flat { + assert_eq!( + actual.as_ref().unwrap(), + &LhsValue::Array(Array::from_iter(values.iter().copied())), + "{expression}", + ); + } + if expression == "groups[0][*]" { + // A single terminal `map[*]` retains its map for the + // mapped function, whose iteration follows byte-key order. + let LhsValue::Map(map) = actual.unwrap() else { + panic!("terminal map[*] must retain the map"); + }; + let values: Vec<_> = map.into_values().collect(); + assert_eq!( + values, + vec![ + LhsValue::Array(Array::from_iter(["a1"])), + LhsValue::Array(Array::new(Type::Bytes)), + LhsValue::Array(Array::from_iter(["v0", "v1"])), + LhsValue::Array(Array::from_iter(["z1"])), + ], + ); + } + } + } + } } diff --git a/engine/src/ast/logical_expr.rs b/engine/src/ast/logical_expr.rs index 1f8a0073..e10928ec 100644 --- a/engine/src/ast/logical_expr.rs +++ b/engine/src/ast/logical_expr.rs @@ -7,7 +7,8 @@ use super::visitor::{Visitor, VisitorMut}; use crate::compiler::Compiler; use crate::filter::{CompiledExpr, CompiledOneExpr, CompiledVecExpr}; use crate::lex::{Lex, LexErrorKind, LexResult, LexWith, expect, skip_space, span}; -use crate::types::{GetType, LhsValue, Type, TypeMismatchError}; +use crate::lhs_types::LhsValue; +use crate::types::{GetType, Type, TypeMismatchError}; use serde::Serialize; lex_enum!( @@ -362,7 +363,7 @@ impl Expr for LogicalExpr { } } - fn compile_with_compiler(self, compiler: &mut C) -> CompiledExpr { + fn compile_with_compiler(self, compiler: &mut C) -> CompiledExpr { match self { LogicalExpr::Comparison(op) => compiler.compile_comparison_expr(op), LogicalExpr::Parenthesized(node) => compiler.compile_logical_expr(node.expr), diff --git a/engine/src/ast/mod.rs b/engine/src/ast/mod.rs index 79d7d054..d29445b3 100644 --- a/engine/src/ast/mod.rs +++ b/engine/src/ast/mod.rs @@ -12,8 +12,9 @@ use self::visitor::{UsesListVisitor, UsesVisitor, Visitor, VisitorMut}; use crate::compiler::{Compiler, DefaultCompiler}; use crate::filter::{CompiledExpr, CompiledValueExpr, Filter, FilterValue}; use crate::lex::{LexErrorKind, LexResult, LexWith}; +use crate::lhs_types::LhsValue; use crate::scheme::{Scheme, UnknownFieldError}; -use crate::types::{GetType, LhsValue, Type, TypeMismatchError}; +use crate::types::{GetType, Type, TypeMismatchError}; use serde::Serialize; use std::fmt::{self, Debug}; @@ -26,7 +27,7 @@ pub trait Expr: /// Recursively visit all nodes in the AST using a [`VisitorMut`]. fn walk_mut<'a, V: VisitorMut<'a>>(&'a mut self, visitor: &mut V); /// Compiles current node into a [`CompiledExpr`] using [`Compiler`]. - fn compile_with_compiler(self, compiler: &mut C) -> CompiledExpr; + fn compile_with_compiler(self, compiler: &mut C) -> CompiledExpr; /// Compiles current node into a [`CompiledExpr`] using [`DefaultCompiler`]. fn compile(self) -> CompiledExpr { let mut compiler = DefaultCompiler::new(); @@ -43,7 +44,10 @@ pub trait ValueExpr: /// Recursively visit all nodes in the AST using a [`VisitorMut`]. fn walk_mut<'a, V: VisitorMut<'a>>(&'a mut self, visitor: &mut V); /// Compiles current node into a [`CompiledValueExpr`] using [`Compiler`]. - fn compile_with_compiler(self, compiler: &mut C) -> CompiledValueExpr; + fn compile_with_compiler( + self, + compiler: &mut C, + ) -> CompiledValueExpr; /// Compiles current node into a [`CompiledValueExpr`] using [`DefaultCompiler`]. fn compile(self) -> CompiledValueExpr { let mut compiler = DefaultCompiler::new(); @@ -150,7 +154,11 @@ impl FilterAst { } /// Compiles a [`FilterAst`] into a [`Filter`] using a specific [`Compiler`]. - pub fn compile_with_compiler(self, compiler: &mut C) -> Filter { + /// + /// Field definitions must be validated explicitly during application + /// initialization with [`crate::ContextFamily::validate_scheme`]. Compilation + /// does not repeat that validation. + pub fn compile_with_compiler(self, compiler: &mut C) -> Filter { match compiler.compile_logical_expr(self.op) { CompiledExpr::One(one) => Filter::new(one, self.scheme), CompiledExpr::Vec(_) => unreachable!(), @@ -221,7 +229,10 @@ impl ValueExpr for FilterValueExpr { } } - fn compile_with_compiler(self, compiler: &mut C) -> CompiledValueExpr { + fn compile_with_compiler( + self, + compiler: &mut C, + ) -> CompiledValueExpr { match self { Self::Index(expr) => compiler.compile_index_expr(expr), Self::Quantifier { op, arg } => { @@ -336,7 +347,14 @@ impl FilterValueAst { } /// Compiles a [`FilterValueAst`] into a [`FilterValue`] using a specific [`Compiler`]. - pub fn compile_with_compiler(self, compiler: &mut C) -> FilterValue { + /// + /// Field definitions must be validated explicitly during application + /// initialization with [`crate::ContextFamily::validate_scheme`]. Compilation + /// does not repeat that validation. + pub fn compile_with_compiler( + self, + compiler: &mut C, + ) -> FilterValue { FilterValue::new(compiler.compile_value_expr(self.op), self.scheme) } diff --git a/engine/src/compiler.rs b/engine/src/compiler.rs index 3b01ffad..f018940f 100644 --- a/engine/src/compiler.rs +++ b/engine/src/compiler.rs @@ -1,6 +1,7 @@ use crate::{ - ComparisonExpr, CompiledExpr, CompiledValueExpr, Expr, FunctionCallArgExpr, FunctionCallExpr, - IndexExpr, LogicalExpr, ValueExpr, + ComparisonExpr, CompiledExpr, CompiledValueExpr, ContextFamily, ExecutionContextFamily, Expr, + Field, FieldBindingError, FieldConsumer, FunctionCallArgExpr, FunctionCallExpr, IndexExpr, + LogicalExpr, ValueExpr, }; /// Trait used to drive the compilation of a [`crate::FilterAst`] into a [`crate::Filter`]. @@ -8,33 +9,57 @@ pub trait Compiler: Sized { /// The user data type passed in the [`crate::ExecutionContext`]. type U: 'static; + /// Context and concrete field bindings used by compiled expressions. + type Family: ContextFamily; + + /// Resolve a field while retaining the concrete getter type during compilation. + /// The binding takes ownership so the compiled expression can retain the field + /// without cloning it. + #[inline] + fn bind_field>( + &mut self, + field: Field, + consumer: B, + ) -> Result { + Self::Family::bind(field, consumer) + } + /// Compiles a [`Expr`] node into a [`CompiledExpr`] (boxed closure). #[inline] - fn compile_expr(&mut self, node: impl Expr) -> CompiledExpr { + fn compile_expr(&mut self, node: impl Expr) -> CompiledExpr { node.compile_with_compiler(self) } /// Compiles a [`LogicalExpr`] node into a [`CompiledExpr`] (boxed closure). #[inline] - fn compile_logical_expr(&mut self, node: LogicalExpr) -> CompiledExpr { + fn compile_logical_expr(&mut self, node: LogicalExpr) -> CompiledExpr { self.compile_expr(node) } /// Compiles a [`ComparisonExpr`] node into a [`CompiledExpr`] (boxed closure). #[inline] - fn compile_comparison_expr(&mut self, node: ComparisonExpr) -> CompiledExpr { + fn compile_comparison_expr( + &mut self, + node: ComparisonExpr, + ) -> CompiledExpr { self.compile_expr(node) } /// Compiles a [`ValueExpr`] node into a [`CompiledValueExpr`] (boxed closure). #[inline] - fn compile_value_expr(&mut self, node: impl ValueExpr) -> CompiledValueExpr { + fn compile_value_expr( + &mut self, + node: impl ValueExpr, + ) -> CompiledValueExpr { node.compile_with_compiler(self) } /// Compiles a [`FunctionCallExpr`] node into a [`CompiledValueExpr`] (boxed closure). #[inline] - fn compile_function_call_expr(&mut self, node: FunctionCallExpr) -> CompiledValueExpr { + fn compile_function_call_expr( + &mut self, + node: FunctionCallExpr, + ) -> CompiledValueExpr { self.compile_value_expr(node) } @@ -43,24 +68,38 @@ pub trait Compiler: Sized { fn compile_function_call_arg_expr( &mut self, node: FunctionCallArgExpr, - ) -> CompiledValueExpr { + ) -> CompiledValueExpr { self.compile_value_expr(node) } /// Compiles a [`IndexExpr`] node into a [`CompiledValueExpr`] (boxed closure). #[inline] - fn compile_index_expr(&mut self, node: IndexExpr) -> CompiledValueExpr { + fn compile_index_expr(&mut self, node: IndexExpr) -> CompiledValueExpr { self.compile_value_expr(node) } } /// Default compiler -#[derive(Clone, Copy, Debug)] -pub struct DefaultCompiler { - _marker: std::marker::PhantomData, +pub struct DefaultCompiler> { + _marker: std::marker::PhantomData (U, F)>, +} + +impl Copy for DefaultCompiler {} + +impl Clone for DefaultCompiler { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +impl std::fmt::Debug for DefaultCompiler { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DefaultCompiler").finish() + } } -impl Default for DefaultCompiler { +impl Default for DefaultCompiler { #[inline] fn default() -> Self { Self { @@ -69,7 +108,7 @@ impl Default for DefaultCompiler { } } -impl DefaultCompiler { +impl DefaultCompiler { /// Creates a new [`DefaultCompiler`]. #[inline] pub fn new() -> Self { @@ -77,6 +116,7 @@ impl DefaultCompiler { } } -impl Compiler for DefaultCompiler { +impl> Compiler for DefaultCompiler { + type Family = F; type U = U; } diff --git a/engine/src/context/fields.rs b/engine/src/context/fields.rs new file mode 100644 index 00000000..d88ee839 --- /dev/null +++ b/engine/src/context/fields.rs @@ -0,0 +1,1255 @@ +//! Field definitions for existing runtime contexts. + +use crate::{FieldDefinitionError, GetType, Scheme, SchemeValidationError, Type}; +use std::collections::BTreeSet; + +/// Validate complete scheme coverage and getter types without running getters. +/// +/// Public for calls generated by [`crate::wirefilter_fields!`] in other crates. +/// Applications normally use the generated family's `validate_scheme` method. +/// Each entry is the field name, its static value type (or `None` for a +/// scheme-typed stored value), and whether its getter returns `Option`. A +/// scheme-typed context must validate values on insertion; this function only +/// checks coverage and optionality for such getters. +#[doc(hidden)] +pub fn validate_field_definitions( + scheme: &Scheme, + definitions: &[(&str, Option, bool)], +) -> Result<(), SchemeValidationError> { + let mut errors = Vec::new(); + let mut names = BTreeSet::new(); + for &(name, getter_type, may_be_absent) in definitions { + if !names.insert(name) { + errors.push(FieldDefinitionError::DuplicateGetter(name.into())); + } + match scheme.get_field(name) { + Ok(field) => { + if let Some(getter_type) = getter_type + && field.get_type() != getter_type + { + errors.push(FieldDefinitionError::TypeMismatch { + name: name.into(), + scheme_type: field.get_type(), + getter_type, + }); + } + if may_be_absent && !field.optional() { + errors.push(FieldDefinitionError::OptionalGetterForMandatoryField( + name.into(), + )); + } + } + Err(_) => errors.push(FieldDefinitionError::UnknownField(name.into())), + } + } + for field in scheme.fields() { + if !names.contains(field.name()) { + errors.push(FieldDefinitionError::MissingGetter(field.name().into())); + } + } + if errors.is_empty() { + Ok(()) + } else { + Err(SchemeValidationError(errors.into_boxed_slice())) + } +} + +/// Define field getters on an existing evaluation context. +/// +/// The context implements [`crate::HasScheme`] and [`crate::EvaluationContext`] +/// separately. Each signature declares the getter's argument and return type. +/// Mandatory fields use `&T` for borrowed storage or `T` for an owned value. +/// Optional fields may additionally use `Option<&T>` or `Option` for absence. +/// Startup validation rejects an `Option` getter for a mandatory field, even if +/// its body always returns `Some`. Plain getter bodies retain their declared +/// return type, so returning `None` from one is a Rust type error. +/// Each comma-separated field entry groups its getter in braces so declarations +/// can be parsed independently without expansion depth growing with field count. +/// Plain-return getters use [`crate::RequiredFieldGetterFn`]; `Option` getters use +/// [`crate::FieldGetterFn`]. Getters sharing a provider and presence use the same +/// getter type. Field names are resolved during filter compilation, never on +/// field access. Evaluation calls the selected getter through a function pointer. +/// Attributes placed directly before `fn` apply to the function containing the +/// getter body, for both plain and optional returns. This function has no +/// attributes unless explicitly supplied. The `FieldGetter::get` adapters remain +/// `#[inline]`. Plain-return getters wrap their result in `Some` after calling the +/// function pointer; `Option` getters forward their result unchanged. Getter +/// attributes do not conditionally include or omit the field definition. +/// A getter may also declare a second parameter, `field: &Field`, to access the +/// original field's name, index, type, and optionality. Both getter forms support +/// the same return types and provider overrides. A borrowed return is tied to +/// the context parameter, independently of the field parameter's lifetime. +/// +/// In the declaration below, `RequestState` is the application's existing +/// context and `RequestFields` is a new zero-sized [`crate::ContextFamily`] +/// marker generated by this macro. The marker contains no request values and +/// is not instantiated per request. For a borrowing context, it lets compiled +/// filters name `Context<'data>` independently of any particular request lifetime. +/// An owned context could implement `ContextFamily` directly, but this macro +/// consistently generates a separate marker. +/// +/// ```ignore +/// wirefilter_fields! { +/// pub impl RequestFields for RequestState { +/// "http.request.headers.names" => { +/// fn(state: &RequestState) -> &Vec { +/// state.names.get_or_init(|| compute_names(state)) +/// } +/// }, +/// "tcp.port" => { +/// fn(state: &RequestState) -> i64 { +/// i64::from(state.port) +/// } +/// }, +/// } +/// } +/// RequestFields::validate_scheme(&scheme)?; +/// ``` +/// +/// A context borrowing request data can use `impl<'data> RequestFields for +/// RequestState<'data>`. Return types borrowing that data through a +/// descriptor can select their provider explicitly by placing +/// `provider SelectionProvider;` before the getter signature inside its group. +/// The getter then returns `Selection<'data>` or `Option>`. +/// When combining a provider override with attributes, put the attributes after +/// `provider SelectionProvider;` and immediately before `fn`. For example: +/// +/// ```ignore +/// "request.computed_value" => { +/// #[inline(never)] +/// fn(state: &RequestState) -> &ComputedValue { +/// state.computed_value() +/// } +/// }, +/// ``` +/// +/// Validation reports missing, unknown, duplicate, mistyped, and incorrectly +/// optional definitions, including unused and optional scheme fields. It does not execute getters or +/// initialize lazy values. Applications must call the generated `validate_scheme` +/// method during initialization for every scheme/family pair, before loading +/// rulesets or compiling expressions. Parsing, compilation, and execution do not +/// invoke it automatically. +#[macro_export] +macro_rules! wirefilter_fields { + ( + $(#[$meta:meta])* $visibility:vis impl<$data:lifetime> $family:ident + for $context:ident<$context_data:lifetime> { + $($name:literal => $getter:tt),* $(,)? + } + ) => { + $crate::wirefilter_fields! { + @emit [$(#[$meta])*] [$visibility] [$family] [$data] + [$context<$context_data>] [$context<'static>] + { $($name => $getter),* } + } + }; + ( + $(#[$meta:meta])* $visibility:vis impl $family:ident for $context:ty { + $($name:literal => $getter:tt),* $(,)? + } + ) => { + $crate::wirefilter_fields! { + @emit [$(#[$meta])*] [$visibility] [$family] ['__wirefilter_data] + [$context] [$context] { $($name => $getter),* } + } + }; + ( + @provider { provider $provider:ty; $($getter:tt)* } + ) => { $provider }; + ( + @optional { provider $provider:ty; $($getter:tt)* } + ) => { $crate::wirefilter_fields!(@optional { $($getter)* }) }; + ( + @optional { + $(#[$attribute:meta])* + fn($argument:ident: &$argument_type:ty $(, $field:ident: &$field_type:ty)?) + -> Option<$return_type:ty> $(provider $provider:ty)? $body:block + } + ) => { true }; + ( + @optional { + $(#[$attribute:meta])* + fn($argument:ident: &$argument_type:ty $(, $field:ident: &$field_type:ty)?) + -> $return_type:ty $body:block + } + ) => { false }; + // Keep each getter as a token-tree group so these independent helpers can + // distinguish borrowed and owned return types without parsing the field list. + ( + @provider { + $(#[$attribute:meta])* + fn($argument:ident: &$argument_type:ty $(, $field:ident: &$field_type:ty)?) + -> Option<$return_type:ty> provider $provider:ty $body:block + } + ) => { $provider }; + ( + @provider { + $(#[$attribute:meta])* + fn($argument:ident: &$argument_type:ty $(, $field:ident: &$field_type:ty)?) + -> Option<&$return_type:ty> $body:block + } + ) => { $crate::BorrowedValueProvider<$return_type> }; + ( + @provider { + $(#[$attribute:meta])* + fn($argument:ident: &$argument_type:ty $(, $field:ident: &$field_type:ty)?) + -> Option<$return_type:ty> $body:block + } + ) => { $crate::OwnedValueProvider<$return_type> }; + ( + @provider { + $(#[$attribute:meta])* + fn($argument:ident: &$argument_type:ty $(, $field:ident: &$field_type:ty)?) + -> &$return_type:ty $body:block + } + ) => { $crate::BorrowedValueProvider<$return_type> }; + ( + @provider { + $(#[$attribute:meta])* + fn($argument:ident: &$argument_type:ty $(, $field:ident: &$field_type:ty)?) + -> $return_type:ty $body:block + } + ) => { $crate::OwnedValueProvider<$return_type> }; + ( + @getter [$family:ident] [$data:lifetime] { + provider $provider:ty; + $(#[$attribute:meta])* + fn($argument:ident: &$argument_type:ty $(, $field:ident: &$field_type:ty)?) + -> Option<&$return_type:ty> $body:block + } + ) => { + $crate::wirefilter_fields! { + @impl_getter [$family] [$data] [$(#[$attribute])*] + [$argument: $argument_type] + [$($field: $field_type)?] + [&'__wirefilter_ctx $return_type] [$provider] $body + } + }; + ( + @getter [$family:ident] [$data:lifetime] { + provider $provider:ty; + $(#[$attribute:meta])* + fn($argument:ident: &$argument_type:ty $(, $field:ident: &$field_type:ty)?) + -> Option<$return_type:ty> $body:block + } + ) => { + $crate::wirefilter_fields! { + @impl_getter [$family] [$data] [$(#[$attribute])*] + [$argument: $argument_type] + [$($field: $field_type)?] + [$return_type] [$provider] $body + } + }; + ( + @getter [$family:ident] [$data:lifetime] { + provider $provider:ty; + $(#[$attribute:meta])* + fn($argument:ident: &$argument_type:ty $(, $field:ident: &$field_type:ty)?) + -> &$return_type:ty $body:block + } + ) => { + $crate::wirefilter_fields! { + @impl_required_getter [$family] [$data] [$(#[$attribute])*] + [$argument: $argument_type] + [$($field: $field_type)?] + [&'__wirefilter_ctx $return_type] [$provider] $body + } + }; + ( + @getter [$family:ident] [$data:lifetime] { + provider $provider:ty; + $(#[$attribute:meta])* + fn($argument:ident: &$argument_type:ty $(, $field:ident: &$field_type:ty)?) + -> $return_type:ty $body:block + } + ) => { + $crate::wirefilter_fields! { + @impl_required_getter [$family] [$data] [$(#[$attribute])*] + [$argument: $argument_type] + [$($field: $field_type)?] + [$return_type] [$provider] $body + } + }; + ( + @getter [$family:ident] [$data:lifetime] { + $(#[$attribute:meta])* + fn($argument:ident: &$argument_type:ty $(, $field:ident: &$field_type:ty)?) + -> Option<&$return_type:ty> provider $provider:ty $body:block + } + ) => { + $crate::wirefilter_fields! { + @impl_getter [$family] [$data] [$(#[$attribute])*] + [$argument: $argument_type] + [$($field: $field_type)?] + [&'__wirefilter_ctx $return_type] [$provider] $body + } + }; + ( + @getter [$family:ident] [$data:lifetime] { + $(#[$attribute:meta])* + fn($argument:ident: &$argument_type:ty $(, $field:ident: &$field_type:ty)?) + -> Option<$return_type:ty> provider $provider:ty $body:block + } + ) => { + $crate::wirefilter_fields! { + @impl_getter [$family] [$data] [$(#[$attribute])*] + [$argument: $argument_type] + [$($field: $field_type)?] + [$return_type] [$provider] $body + } + }; + ( + @getter [$family:ident] [$data:lifetime] { + $(#[$attribute:meta])* + fn($argument:ident: &$argument_type:ty $(, $field:ident: &$field_type:ty)?) + -> Option<&$return_type:ty> $body:block + } + ) => { + $crate::wirefilter_fields! { + @impl_getter [$family] [$data] [$(#[$attribute])*] + [$argument: $argument_type] + [$($field: $field_type)?] + [&'__wirefilter_ctx $return_type] [$crate::BorrowedValueProvider<$return_type>] $body + } + }; + ( + @getter [$family:ident] [$data:lifetime] { + $(#[$attribute:meta])* + fn($argument:ident: &$argument_type:ty $(, $field:ident: &$field_type:ty)?) + -> Option<$return_type:ty> $body:block + } + ) => { + $crate::wirefilter_fields! { + @impl_getter [$family] [$data] [$(#[$attribute])*] + [$argument: $argument_type] + [$($field: $field_type)?] + [$return_type] [$crate::OwnedValueProvider<$return_type>] $body + } + }; + ( + @getter [$family:ident] [$data:lifetime] { + $(#[$attribute:meta])* + fn($argument:ident: &$argument_type:ty $(, $field:ident: &$field_type:ty)?) + -> &$return_type:ty $body:block + } + ) => { + $crate::wirefilter_fields! { + @impl_required_getter [$family] [$data] [$(#[$attribute])*] + [$argument: $argument_type] + [$($field: $field_type)?] + [&'__wirefilter_ctx $return_type] [$crate::BorrowedValueProvider<$return_type>] $body + } + }; + ( + @getter [$family:ident] [$data:lifetime] { + $(#[$attribute:meta])* + fn($argument:ident: &$argument_type:ty $(, $field:ident: &$field_type:ty)?) + -> $return_type:ty $body:block + } + ) => { + $crate::wirefilter_fields! { + @impl_required_getter [$family] [$data] [$(#[$attribute])*] + [$argument: $argument_type] + [$($field: $field_type)?] + [$return_type] [$crate::OwnedValueProvider<$return_type>] $body + } + }; + ( + @impl_required_getter [$family:ident] [$data:lifetime] [$(#[$attribute:meta])*] + [$argument:ident: $argument_type:ty] [$($field:ident: $field_type:ty)?] + [$return_type:ty] [$provider:ty] $body:block + ) => {{ + // The parameter shadows this function's name inside the body. Reusing + // that caller-provided identifier avoids introducing a helper name that + // could shadow application items referenced by the body or return type. + $(#[$attribute])* + fn $argument<'__wirefilter_ctx, $data: '__wirefilter_ctx>( + $argument: &'__wirefilter_ctx $argument_type, + __wirefilter_field: &$crate::Field, + ) -> $return_type { + $(let $field: &$field_type = __wirefilter_field;)? + $body + } + + // Keep the callback's name outside the body function's scope. + { + fn __wirefilter_call<'__wirefilter_ctx, $data>( + context: $crate::ContextRef<'__wirefilter_ctx, $data, $family>, + field: &$crate::Field, + ) -> $crate::RequiredFieldGetterResult<'__wirefilter_ctx, $data, $provider> { + $crate::RequiredFieldGetterResult::new($argument(context.context(), field)) + } + $crate::RequiredFieldGetterFn::<$family, $provider>::new(__wirefilter_call) + } + }}; + ( + @impl_getter [$family:ident] [$data:lifetime] [$(#[$attribute:meta])*] + [$argument:ident: $argument_type:ty] [$($field:ident: $field_type:ty)?] + [$return_type:ty] [$provider:ty] $body:block + ) => {{ + // Preserve the same name isolation as the plain-return getter. + $(#[$attribute])* + fn $argument<'__wirefilter_ctx, $data: '__wirefilter_ctx>( + $argument: &'__wirefilter_ctx $argument_type, + __wirefilter_field: &$crate::Field, + ) -> Option<$return_type> { + $(let $field: &$field_type = __wirefilter_field;)? + $body + } + + { + fn __wirefilter_call<'__wirefilter_ctx, $data>( + context: $crate::ContextRef<'__wirefilter_ctx, $data, $family>, + field: &$crate::Field, + ) -> $crate::OptionalFieldGetterResult<'__wirefilter_ctx, $data, $provider> { + $crate::OptionalFieldGetterResult::new($argument(context.context(), field)) + } + $crate::FieldGetterFn::<$family, $provider>::new(__wirefilter_call) + } + }}; + ( + @emit [$($meta:tt)*] [$visibility:vis] [$family:ident] [$data:lifetime] + [$context:ty] [$static_context:ty] { $($name:literal => $getter:tt),* } + ) => { + $($meta)* + $visibility struct $family; + + impl $family { + /// Check all field definitions against the runtime scheme. + /// + /// Call during application initialization before compiling expressions. + /// This check is never invoked automatically by parsing or execution. + $visibility fn validate_scheme( + scheme: &$crate::Scheme, + ) -> Result<(), $crate::SchemeValidationError> { + ::validate_scheme(scheme) + } + } + + impl $crate::ContextFamily for $family { + type UserData = <$static_context as $crate::EvaluationContext>::UserData; + type Context<$data> = $context; + + fn validate_scheme(scheme: &$crate::Scheme) -> Result<(), $crate::SchemeValidationError> { + $crate::validate_field_definitions(scheme, &[ + $(($name, + <<$crate::wirefilter_fields!(@provider $getter) as $crate::ProviderFamily>::ValueType + as $crate::values::FieldValueType>::STATIC_TYPE, + $crate::wirefilter_fields!(@optional $getter) + ),)* + ]) + } + + fn bind>( + field: $crate::Field, + consumer: B, + ) -> Result { + #[allow(unreachable_patterns)] + match field.name() { + $($name => consumer.consume( + field, + $crate::wirefilter_fields!(@getter [$family] [$data] $getter) + ),)* + name => Err($crate::FieldBindingError::UnknownField(name.into())), + } + } + } + }; +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{EvaluationContext, HasScheme, ListMatcher, ListRef, SchemeBuilder}; + use std::sync::OnceLock; + + struct State { + scheme: Scheme, + names: Box<[Box]>, + country: Option>, + lazy: OnceLock]>>, + } + + impl HasScheme for State { + fn scheme(&self) -> &Scheme { + &self.scheme + } + } + + impl EvaluationContext for State { + type UserData = (); + + fn get_user_data(&self) -> &() { + &() + } + + fn get_list_matcher(&self, _: ListRef<'_>) -> &dyn ListMatcher { + panic!("these fields have no lists") + } + } + + wirefilter_fields! { + impl Fields for State { + "names" => { + fn(state: &State) -> &[Box] { + if state.names.is_empty() { + return &[]; + } + &state.names + } + }, + "country" => { + fn(state: &State) -> Option<&str> { state.country.as_deref() } + }, + "lazy" => { + fn(state: &State) -> &[Box] { + state.lazy.get_or_init(|| state.names.clone()) + } + }, + "count" => { + fn(state: &State) -> i64 { + if state.names.is_empty() { + return 0; + } + state.names.len() as i64 + } + }, + } + } + + wirefilter_fields! { + impl ValidationOnly for State { + "names" => { + fn(state: &State) -> &[Box] { + let _ = state; + panic!("validation must not evaluate getters") + } + }, + "country" => { + fn(state: &State) -> Option<&str> { + let _ = state; + panic!("validation must not evaluate getters") + } + }, + "lazy" => { + fn(state: &State) -> &[Box] { + let _ = state; + panic!("validation must not initialize lazy data") + } + }, + "count" => { + fn(state: &State) -> i64 { + let _ = state; + panic!("validation must not evaluate getters") + } + }, + } + } + + fn schema(fields: &[(&str, Type, bool)]) -> Scheme { + let mut builder = SchemeBuilder::new(); + for &(name, ty, optional) in fields { + if optional { + builder.add_optional_field(name, ty).unwrap(); + } else { + builder.add_field(name, ty).unwrap(); + } + } + builder.build() + } + + fn definitions() -> Vec<(&'static str, Type, bool)> { + vec![ + ("names", Type::array(Type::Bytes), false), + ("country", Type::Bytes, true), + ("lazy", Type::array(Type::Bytes), false), + ("count", Type::Int, false), + ] + } + + #[test] + fn test_complete_validation_does_not_run_getters() { + let scheme = schema(&definitions()); + assert_eq!(Fields::validate_scheme(&scheme), Ok(())); + assert_eq!(ValidationOnly::validate_scheme(&scheme), Ok(())); + } + + #[test] + fn test_existing_state_lazy_getters_and_phase_reuse() { + let scheme = schema(&definitions()); + Fields::validate_scheme(&scheme).unwrap(); + let mut compiler = crate::DefaultCompiler::<(), Fields>::new(); + let ordinary = scheme + .parse(r#"any(names[*] == "accept") and count == 1"#) + .unwrap() + .compile_with_compiler(&mut compiler); + let lazy = scheme + .parse(r#"any(lazy[*] == "accept")"#) + .unwrap() + .compile_with_compiler(&mut compiler); + let mut state = State { + scheme, + names: vec![Box::from("other")].into_boxed_slice(), + country: None, + lazy: OnceLock::new(), + }; + assert!(state.lazy.get().is_none()); + assert!(!ordinary.execute(&state).unwrap()); + state.names[0] = "accept".into(); + assert!(ordinary.execute(&state).unwrap()); + assert!(state.lazy.get().is_none()); + assert!(lazy.execute(&state).unwrap()); + let pointer = state.lazy.get().unwrap().as_ptr(); + assert!(lazy.execute(&state).unwrap()); + assert_eq!(pointer, state.lazy.get().unwrap().as_ptr()); + // The runtime owns cache invalidation when source data changes. + state.names[0] = "other".into(); + state.lazy.take(); + assert!(!lazy.execute(&state).unwrap()); + } + + struct BorrowingState<'data> { + scheme: &'data Scheme, + names: &'data [Box], + } + + impl HasScheme for BorrowingState<'_> { + fn scheme(&self) -> &Scheme { + self.scheme + } + } + + impl EvaluationContext for BorrowingState<'_> { + type UserData = (); + + fn get_user_data(&self) -> &() { + &() + } + + fn get_list_matcher(&self, _: ListRef<'_>) -> &dyn ListMatcher { + panic!("these fields have no lists") + } + } + + wirefilter_fields! { + impl<'data> BorrowingFields for BorrowingState<'data> { + "names" => { + fn(state: &BorrowingState<'data>) -> &[Box] { + state.names + } + }, + } + } + + #[test] + fn test_context_can_borrow_request_storage() { + let scheme = schema(&[("names", Type::array(Type::Bytes), false)]); + BorrowingFields::validate_scheme(&scheme).unwrap(); + let mut compiler = crate::DefaultCompiler::<(), BorrowingFields>::new(); + let filter = scheme + .parse(r#"names[0] == "accept""#) + .unwrap() + .compile_with_compiler(&mut compiler); + let names = [Box::::from("accept")]; + let state = BorrowingState { + scheme: &scheme, + names: &names, + }; + assert!(filter.execute(&state).unwrap()); + } + + struct SelectionProvider; + + impl crate::ProviderFamily for SelectionProvider { + type Storage<'data> = &'data [Box]; + type Value<'ctx, 'data> + = crate::values::SliceView<'data, Box> + where + 'data: 'ctx; + type ValueType = crate::values::Array; + + fn reborrow<'ctx, 'data: 'ctx>( + value: &'ctx Self::Storage<'data>, + ) -> Self::Value<'ctx, 'data> { + crate::values::SliceView(value) + } + } + + wirefilter_fields! { + impl<'data> SelectionFields for BorrowingState<'data> { + "names" => { + provider SelectionProvider; + fn(state: &BorrowingState<'data>, field: &crate::Field) + -> crate::values::SliceView<'data, Box> { + assert_eq!(field.as_ref(), state.scheme.get_field("names").unwrap()); + crate::values::SliceView(state.names) + } + }, + } + } + + #[test] + fn test_explicit_descriptor_provider_converts_general_values_safely() { + let scheme = schema(&[("names", Type::array(Type::Bytes), false)]); + SelectionFields::validate_scheme(&scheme).unwrap(); + let mut compiler = crate::DefaultCompiler::<(), SelectionFields>::new(); + let value = scheme + .parse_value("names") + .unwrap() + .compile_with_compiler(&mut compiler); + let names = [Box::::from("accept")]; + let state = BorrowingState { + scheme: &scheme, + names: &names, + }; + let crate::LhsValue::Array(array) = value.execute(&state).unwrap().unwrap() else { + panic!("expected an array") + }; + let crate::LhsValue::Bytes(bytes) = array.get(0).unwrap() else { + panic!("expected a byte string") + }; + assert_eq!(bytes.as_ref().as_ptr(), names[0].as_ptr()); + } + + #[test] + fn test_large_field_family_with_default_recursion_limit() { + // A complete schema larger than Rust's default macro recursion limit. + // Repetition keeps this fixture compact without changing the crate limit. + macro_rules! define_fields { + ($(($borrowed:literal, $owned:literal, $selection:literal)),* $(,)?) => { + wirefilter_fields! { + impl<'data> LargeFields for BorrowingState<'data> { + $( + $borrowed => { + fn(state: &BorrowingState<'data>) -> &[Box] { + state.names + } + }, + $owned => { + fn(state: &BorrowingState<'data>) -> i64 { + state.names.len() as i64 + } + }, + $selection => { + provider SelectionProvider; + fn(state: &BorrowingState<'data>) + -> crate::values::SliceView<'data, Box> { + crate::values::SliceView(state.names) + } + }, + )* + } + } + fn large_schema() -> Scheme { + schema(&[$( + ($borrowed, Type::array(Type::Bytes), false), + ($owned, Type::Int, false), + ($selection, Type::array(Type::Bytes), false), + )*]) + } + }; + } + + define_fields! { + ("names000", "count000", "selection000"), + ("names001", "count001", "selection001"), + ("names002", "count002", "selection002"), + ("names003", "count003", "selection003"), + ("names004", "count004", "selection004"), + ("names005", "count005", "selection005"), + ("names006", "count006", "selection006"), + ("names007", "count007", "selection007"), + ("names008", "count008", "selection008"), + ("names009", "count009", "selection009"), + ("names010", "count010", "selection010"), + ("names011", "count011", "selection011"), + ("names012", "count012", "selection012"), + ("names013", "count013", "selection013"), + ("names014", "count014", "selection014"), + ("names015", "count015", "selection015"), + ("names016", "count016", "selection016"), + ("names017", "count017", "selection017"), + ("names018", "count018", "selection018"), + ("names019", "count019", "selection019"), + ("names020", "count020", "selection020"), + ("names021", "count021", "selection021"), + ("names022", "count022", "selection022"), + ("names023", "count023", "selection023"), + ("names024", "count024", "selection024"), + ("names025", "count025", "selection025"), + ("names026", "count026", "selection026"), + ("names027", "count027", "selection027"), + ("names028", "count028", "selection028"), + ("names029", "count029", "selection029"), + ("names030", "count030", "selection030"), + ("names031", "count031", "selection031"), + ("names032", "count032", "selection032"), + ("names033", "count033", "selection033"), + ("names034", "count034", "selection034"), + ("names035", "count035", "selection035"), + ("names036", "count036", "selection036"), + ("names037", "count037", "selection037"), + ("names038", "count038", "selection038"), + ("names039", "count039", "selection039"), + ("names040", "count040", "selection040"), + ("names041", "count041", "selection041"), + ("names042", "count042", "selection042"), + ("names043", "count043", "selection043"), + ("names044", "count044", "selection044"), + ("names045", "count045", "selection045"), + ("names046", "count046", "selection046"), + ("names047", "count047", "selection047"), + ("names048", "count048", "selection048"), + ("names049", "count049", "selection049"), + ("names050", "count050", "selection050"), + ("names051", "count051", "selection051"), + ("names052", "count052", "selection052"), + ("names053", "count053", "selection053"), + ("names054", "count054", "selection054"), + ("names055", "count055", "selection055"), + ("names056", "count056", "selection056"), + ("names057", "count057", "selection057"), + ("names058", "count058", "selection058"), + ("names059", "count059", "selection059"), + ("names060", "count060", "selection060"), + ("names061", "count061", "selection061"), + ("names062", "count062", "selection062"), + ("names063", "count063", "selection063"), + ("names064", "count064", "selection064"), + ("names065", "count065", "selection065"), + ("names066", "count066", "selection066"), + ("names067", "count067", "selection067"), + ("names068", "count068", "selection068"), + ("names069", "count069", "selection069"), + ("names070", "count070", "selection070"), + ("names071", "count071", "selection071"), + ("names072", "count072", "selection072"), + ("names073", "count073", "selection073"), + ("names074", "count074", "selection074"), + ("names075", "count075", "selection075"), + ("names076", "count076", "selection076"), + ("names077", "count077", "selection077"), + ("names078", "count078", "selection078"), + ("names079", "count079", "selection079"), + ("names080", "count080", "selection080"), + ("names081", "count081", "selection081"), + ("names082", "count082", "selection082"), + ("names083", "count083", "selection083"), + ("names084", "count084", "selection084"), + ("names085", "count085", "selection085"), + ("names086", "count086", "selection086"), + ("names087", "count087", "selection087"), + ("names088", "count088", "selection088"), + ("names089", "count089", "selection089"), + ("names090", "count090", "selection090"), + ("names091", "count091", "selection091"), + ("names092", "count092", "selection092"), + ("names093", "count093", "selection093"), + ("names094", "count094", "selection094"), + ("names095", "count095", "selection095"), + ("names096", "count096", "selection096"), + ("names097", "count097", "selection097"), + ("names098", "count098", "selection098"), + ("names099", "count099", "selection099"), + } + let scheme = large_schema(); + assert_eq!(scheme.field_count(), 300); + LargeFields::validate_scheme(&scheme).unwrap(); + let names = [Box::::from("accept"), Box::::from("other")]; + let state = BorrowingState { + scheme: &scheme, + names: &names, + }; + let mut compiler = crate::DefaultCompiler::<(), LargeFields>::new(); + for index in [0, 49, 99] { + for expression in [ + format!("names{index:03}[0] == \"accept\""), + format!("count{index:03} == 2"), + format!("any(selection{index:03}[*] == \"other\")"), + ] { + let filter = scheme + .parse(&expression) + .unwrap() + .compile_with_compiler(&mut compiler); + assert!(filter.execute(&state).unwrap(), "{expression}"); + } + let value = scheme + .parse_value(&format!("selection{index:03}")) + .unwrap() + .compile_with_compiler(&mut compiler); + let crate::LhsValue::Array(array) = value.execute(&state).unwrap().unwrap() else { + panic!("expected an array"); + }; + let crate::LhsValue::Bytes(bytes) = array.get(0).unwrap() else { + panic!("expected bytes"); + }; + assert_eq!(bytes.as_ref().as_ptr(), names[0].as_ptr()); + } + } + + #[test] + fn test_plain_getter_can_call_application_value_function() { + fn value(state: &State) -> i64 { + state.names.len() as i64 + } + wirefilter_fields! { + impl HelperFields for State { + "count" => { + fn(state: &State) -> i64 { value(state) } + }, + } + } + let scheme = schema(&[("count", Type::Int, false)]); + HelperFields::validate_scheme(&scheme).unwrap(); + let filter = scheme + .parse("count == 1") + .unwrap() + .compile_with_compiler(&mut crate::DefaultCompiler::<(), HelperFields>::new()); + let state = State { + scheme, + names: vec![Box::from("accept")].into_boxed_slice(), + country: None, + lazy: OnceLock::new(), + }; + assert!(filter.execute(&state).unwrap()); + } + + #[test] + fn test_getters_with_field_metadata() { + use crate::Field; + + fn check_field(state: &State, field: &Field, name: &str) { + assert_eq!(field.as_ref(), state.scheme.get_field(name).unwrap()); + } + + wirefilter_fields! { + impl MetadataFields for State { + "names" => { + fn(state: &State, field: &Field) -> &[Box] { + check_field(state, field, "names"); + if state.names.is_empty() { + return &[]; + } + &state.names + } + }, + "count" => { + fn(state: &State, field: &crate::Field) -> i64 { + check_field(state, field, "count"); + if state.names.is_empty() { + return 0; + } + state.names.len() as i64 + } + }, + "country" => { + fn(state: &State, field: &Field) -> Option<&str> { + check_field(state, field, "country"); + state.country.as_deref() + } + }, + "country_length" => { + fn(state: &State, field: &Field) -> Option { + check_field(state, field, "country_length"); + Some(state.country.as_ref()?.len() as i64) + } + }, + } + } + + let scheme = schema(&[ + ("names", Type::array(Type::Bytes), false), + ("count", Type::Int, false), + ("country", Type::Bytes, true), + ("country_length", Type::Int, true), + ]); + MetadataFields::validate_scheme(&scheme).unwrap(); + let mut compiler = crate::DefaultCompiler::<(), MetadataFields>::new(); + let filter = scheme + .parse(r#"any(names[*] == "accept") and count == 1 and country == "GB" and country_length == 2"#) + .unwrap() + .compile_with_compiler(&mut compiler); + let names_value = scheme + .parse_value("names") + .unwrap() + .compile_with_compiler(&mut compiler); + let country_value = scheme + .parse_value("country") + .unwrap() + .compile_with_compiler(&mut compiler); + let length_value = scheme + .parse_value("country_length") + .unwrap() + .compile_with_compiler(&mut compiler); + let mut state = State { + scheme, + names: vec![Box::from("accept")].into_boxed_slice(), + country: Some("GB".into()), + lazy: OnceLock::new(), + }; + assert!(filter.execute(&state).unwrap()); + let crate::LhsValue::Array(names) = names_value.execute(&state).unwrap().unwrap() else { + panic!("expected an array"); + }; + // The result borrows the context, not the field retained by the expression. + drop(names_value); + let crate::LhsValue::Bytes(bytes) = names.get(0).unwrap() else { + panic!("expected bytes"); + }; + assert_eq!(bytes.as_ref().as_ptr(), state.names[0].as_ptr()); + assert_eq!( + country_value.execute(&state).unwrap(), + Ok(crate::LhsValue::from("GB")) + ); + assert_eq!( + length_value.execute(&state).unwrap(), + Ok(crate::LhsValue::Int(2)) + ); + state.country = None; + assert!(!filter.execute(&state).unwrap()); + assert_eq!(country_value.execute(&state).unwrap(), Err(Type::Bytes)); + assert_eq!(length_value.execute(&state).unwrap(), Err(Type::Int)); + state.names = Box::new([]); + assert!(!filter.execute(&state).unwrap()); + } + + #[test] + fn test_plain_returns_and_optional_values() { + for optional_count in [false, true] { + let mut defs = definitions(); + defs[3].2 = optional_count; + let scheme = schema(&defs); + Fields::validate_scheme(&scheme).unwrap(); + let mut compiler = crate::DefaultCompiler::<(), Fields>::new(); + let names = scheme + .parse_value("names") + .unwrap() + .compile_with_compiler(&mut compiler); + let count = scheme + .parse_value("count") + .unwrap() + .compile_with_compiler(&mut compiler); + let country = scheme + .parse_value("country") + .unwrap() + .compile_with_compiler(&mut compiler); + let mut state = State { + scheme, + names: Box::new([]), + country: None, + lazy: OnceLock::new(), + }; + // Both plain bodies take an early-return branch for empty input. + assert_eq!(count.execute(&state).unwrap(), Ok(crate::LhsValue::Int(0))); + assert_eq!(country.execute(&state).unwrap(), Err(Type::Bytes)); + let crate::LhsValue::Array(empty) = names.execute(&state).unwrap().unwrap() else { + panic!("expected a present empty array"); + }; + assert!(empty.is_empty()); + + state.names = vec![Box::from("accept")].into_boxed_slice(); + state.country = Some("GB".into()); + assert_eq!(count.execute(&state).unwrap(), Ok(crate::LhsValue::Int(1))); + assert_eq!( + country.execute(&state).unwrap(), + Ok(crate::LhsValue::from("GB")) + ); + let crate::LhsValue::Array(values) = names.execute(&state).unwrap().unwrap() else { + panic!("expected an array"); + }; + let crate::LhsValue::Bytes(bytes) = values.get(0).unwrap() else { + panic!("expected bytes"); + }; + assert_eq!(bytes.as_ref().as_ptr(), state.names[0].as_ptr()); + } + } + + #[test] + fn test_mandatory_fields_reject_optional_getter_signatures() { + wirefilter_fields! { + impl OptionalFields for State { + "borrowed" => { + fn(_state: &State) -> Option<&str> { + panic!("validation must not execute getters") + } + }, + "owned" => { + fn(_state: &State, _field: &crate::Field) -> Option { Some(42) } + }, + "explicit" => { + fn(_state: &State, _field: &crate::Field) -> Option provider crate::OwnedValueProvider { + None + } + }, + } + } + let mut defs = [ + ("borrowed", Type::Bytes, false), + ("owned", Type::Int, false), + ("explicit", Type::Int, false), + ]; + assert_eq!( + OptionalFields::validate_scheme(&schema(&defs)), + Err(SchemeValidationError( + ["borrowed", "owned", "explicit"] + .map(|name| FieldDefinitionError::OptionalGetterForMandatoryField(name.into())) + .into(), + )) + ); + for (_, _, optional) in &mut defs { + *optional = true; + } + assert_eq!(OptionalFields::validate_scheme(&schema(&defs)), Ok(())); + } + + #[test] + fn test_explicit_providers_support_plain_and_optional_returns() { + wirefilter_fields! { + impl<'data> ProviderFields for BorrowingState<'data> { + "names" => { + provider crate::BorrowedValueProvider<[Box]>; + fn(state: &BorrowingState<'data>, field: &crate::Field) -> &[Box] { + assert_eq!(field.as_ref(), state.scheme.get_field("names").unwrap()); + state.names + } + }, + "first" => { + provider crate::BorrowedValueProvider; + fn(state: &BorrowingState<'data>, field: &crate::Field) -> Option<&str> { + assert_eq!(field.as_ref(), state.scheme.get_field("first").unwrap()); + Some(state.names.first()?.as_ref()) + } + }, + "legacy" => { + fn(state: &BorrowingState<'data>, field: &crate::Field) -> Option<&str> + provider crate::BorrowedValueProvider { + assert_eq!(field.as_ref(), state.scheme.get_field("legacy").unwrap()); + Some(state.names.first()?.as_ref()) + } + }, + "selection" => { + provider SelectionProvider; + fn(state: &BorrowingState<'data>, field: &crate::Field) + -> Option>> { + assert_eq!(field.as_ref(), state.scheme.get_field("selection").unwrap()); + state.names.first()?; + Some(crate::values::SliceView(state.names)) + } + }, + } + } + let scheme = schema(&[ + ("names", Type::array(Type::Bytes), false), + ("first", Type::Bytes, true), + ("legacy", Type::Bytes, true), + ("selection", Type::array(Type::Bytes), true), + ]); + ProviderFields::validate_scheme(&scheme).unwrap(); + let mut compiler = crate::DefaultCompiler::<(), ProviderFields>::new(); + let names = [Box::::from("accept")]; + for field in ["names", "first", "legacy", "selection"] { + let value = scheme + .parse_value(field) + .unwrap() + .compile_with_compiler(&mut compiler); + let present = BorrowingState { + scheme: &scheme, + names: &names, + }; + let result = value.execute(&present).unwrap().unwrap(); + let bytes = match &result { + crate::LhsValue::Bytes(bytes) => bytes, + crate::LhsValue::Array(values) => match values.get(0).unwrap() { + crate::LhsValue::Bytes(bytes) => bytes, + _ => panic!("expected bytes"), + }, + _ => panic!("unexpected field type"), + }; + assert_eq!(bytes.as_ref().as_ptr(), names[0].as_ptr()); + let absent = BorrowingState { + scheme: &scheme, + names: &[], + }; + let result = value.execute(&absent).unwrap(); + if field == "names" { + let crate::LhsValue::Array(values) = result.unwrap() else { + panic!("expected a present empty array"); + }; + assert!(values.is_empty()); + } else { + assert_eq!(result, Err(scheme.get_field(field).unwrap().get_type())); + } + } + } + + #[test] + fn test_missing_unused_and_optional_fields() { + for optional in [false, true] { + let mut fields = definitions(); + fields.push(("unused", Type::Int, optional)); + assert_eq!( + Fields::validate_scheme(&schema(&fields)), + Err(SchemeValidationError( + vec![FieldDefinitionError::MissingGetter("unused".into())].into_boxed_slice() + )), + ); + } + } + + #[test] + fn test_unknown_getter() { + let mut fields = definitions(); + fields.remove(1); + assert_eq!( + Fields::validate_scheme(&schema(&fields)), + Err(SchemeValidationError( + vec![FieldDefinitionError::UnknownField("country".into())].into_boxed_slice() + )), + ); + } + + #[test] + fn test_nested_type_mismatch() { + let mut fields = definitions(); + fields[0].1 = Type::array(Type::Int); + assert_eq!( + Fields::validate_scheme(&schema(&fields)), + Err(SchemeValidationError( + vec![FieldDefinitionError::TypeMismatch { + name: "names".into(), + scheme_type: Type::array(Type::Int), + getter_type: Type::array(Type::Bytes), + }] + .into_boxed_slice() + )), + ); + } + + #[test] + fn test_all_registration_errors_reported_together() { + let scheme = schema(&[("field", Type::Bytes, false), ("missing", Type::Int, true)]); + assert_eq!( + validate_field_definitions( + &scheme, + &[ + ("field", Some(Type::Int), false), + ("unknown", Some(Type::Bool), false), + ("field", Some(Type::Bytes), false), + ] + ), + Err(SchemeValidationError( + vec![ + FieldDefinitionError::TypeMismatch { + name: "field".into(), + scheme_type: Type::Bytes, + getter_type: Type::Int, + }, + FieldDefinitionError::UnknownField("unknown".into()), + FieldDefinitionError::DuplicateGetter("field".into()), + FieldDefinitionError::MissingGetter("missing".into()), + ] + .into_boxed_slice() + )), + ); + } +} diff --git a/engine/src/context/mod.rs b/engine/src/context/mod.rs new file mode 100644 index 00000000..cab2151b --- /dev/null +++ b/engine/src/context/mod.rs @@ -0,0 +1,1646 @@ +//! Runtime contexts, reusable state, and statically bound field access. +//! +//! Filters read fields through typed getters bound during compilation. An +//! application can expose its Rust values directly, or retain fields as +//! [`LhsValue`]. Both representations use the same field-binding +//! and reader interfaces. Contexts may own their data or borrow it from the +//! application. +//! +//! # Context interfaces and state +//! +//! [`EvaluationContext`] exposes user data and list +//! matchers, and its [`HasScheme`] supertrait exposes the scheme. +//! These traits do not prescribe storage or define field getters. +//! [`EvaluationContextMut`] adds mutable access for +//! applications that need to update user data or list matchers. +//! +//! [`ContextState`](crate::ContextState) is an optional owner of a scheme, +//! list matchers, and user data. It contains no field values. A context can +//! borrow it and forward the evaluation-context methods to it, while getters read +//! the application's own fields. Applications that already store this state can +//! implement the interfaces directly without using `ContextState`. +//! +//! [`ContextFamily`] associates compiled expressions with +//! the application's context type, including contexts that borrow data. It also +//! binds fields to typed getters during compilation. The +//! [`wirefilter_fields!`](crate::wirefilter_fields) macro generates a family and +//! its field bindings; it does not create a context or store request values. +//! Getters read values when an expression is evaluated. +//! +//! Call [`ContextFamily::validate_scheme`] +//! during application initialization for each scheme/family pair. Its default +//! implementation binds every scheme field and checks static provider types +//! without evaluating getters. The macro additionally checks extra and duplicate +//! definitions and rejects optional getter signatures for mandatory fields. +//! Handwritten families must uphold those constraints themselves or override +//! validation. Parsing, compilation, and execution do not run validation. +//! +//! The default [`ExecutionContext`] implements the same +//! evaluation interfaces and already owns equivalent state alongside its stored +//! field values. It does not require a separate `ContextState`. +//! Its default [`ExecutionContextFamily`] binds those +//! fields through [`LhsValueProvider`], using the same +//! reader interface as application-owned Rust values. +//! +//! # Borrowing request data and reusable state +//! +//! This example borrows a request's `Vec` without constructing a +//! Wirefilter array. `RequestFields` is the family generated by the macro; +//! `RequestContext` is the value passed to the compiled filter. +//! +//! ``` +//! use wirefilter::{ +//! ContextState, DefaultCompiler, EvaluationContext, HasScheme, List, ListMatcher, ListRef, +//! Scheme, SchemeBuilder, Type, +//! }; +//! +//! struct Request { +//! names: Vec, +//! } +//! +//! struct RequestContext<'a> { +//! request: &'a Request, +//! state: &'a ContextState<()>, +//! } +//! +//! impl HasScheme for RequestContext<'_> { +//! fn scheme(&self) -> &Scheme { +//! self.state.scheme() +//! } +//! } +//! +//! impl EvaluationContext for RequestContext<'_> { +//! type UserData = (); +//! +//! fn get_user_data(&self) -> &() { +//! self.state.get_user_data() +//! } +//! +//! fn get_list_matcher(&self, list: ListRef<'_>) -> &dyn ListMatcher { +//! self.state.get_list_matcher(list) +//! } +//! +//! fn get_list_matcher_unchecked(&self, list: &List) -> &dyn ListMatcher { +//! self.state.get_list_matcher_unchecked(list) +//! } +//! } +//! +//! wirefilter::wirefilter_fields! { +//! impl<'data> RequestFields for RequestContext<'data> { +//! "request.names" => { +//! fn(ctx: &RequestContext<'data>) -> &[String] { +//! &ctx.request.names +//! } +//! }, +//! } +//! } +//! +//! # fn main() -> Result<(), Box> { +//! let mut builder = SchemeBuilder::new(); +//! builder.add_field("request.names", Type::array(Type::Bytes))?; +//! let scheme = builder.build(); +//! +//! // Validate every field definition during initialization, before compilation. +//! RequestFields::validate_scheme(&scheme)?; +//! let mut compiler = DefaultCompiler::<(), RequestFields>::new(); +//! let filter = scheme +//! .parse(r#"any(request.names[*] eq "example")"#)? +//! .compile_with_compiler(&mut compiler); +//! +//! let request = Request { +//! names: vec!["example".into()], +//! }; +//! let state = ContextState::<()>::new(&scheme); +//! +//! // Each evaluation borrows the same request and state. +//! for _ in 0..2 { +//! let context = RequestContext { +//! request: &request, +//! state: &state, +//! }; +//! assert!(filter.execute(&context)?); +//! } +//! # Ok(()) +//! # } +//! ``` +//! +//! Initialization validation checks getter coverage, types, and optionality; it +//! does not call getters. Parsing and compilation do not invoke it automatically. +//! +//! The application controls the lifetime and cleanup of reusable state. +//! [`ContextState::clear`](crate::ContextState::clear) clears list contents +//! but retains user data; any compiler-specific cleanup remains the application's +//! responsibility. See [`crate::values`] for supported storage and custom collection +//! interfaces, and [`wirefilter_fields!`](crate::wirefilter_fields) for getter forms. +//! +//! # Fields backed by `LhsValue` +//! +//! A context may also store [`LhsValue`] fields whose types are +//! chosen when the scheme is built, including arbitrary array/map nesting. +//! [`LhsValueProvider`] reads these fields through the +//! same field-binding interface. Its reader traverses and compares the existing +//! values without rebuilding containers. Function arguments borrow selected +//! values; flattening multiple selections still needs an output container. +//! +//! The default `ExecutionContext` already uses this provider. The explicit +//! binding below shows how a downstream context can implement its own table and +//! evaluation state, then provide a getter with the same return signature. The +//! example reuses `ExecutionContext` as its table to focus on field binding. +//! +//! ``` +//! use wirefilter::{ +//! ContextFamily, DefaultCompiler, ExecutionContext, Field, FieldBindingError, FieldConsumer, +//! FieldGetter, LhsValue, LhsValueProvider, SchemeBuilder, Type, +//! }; +//! +//! struct Fields; +//! struct Getter; +//! +//! impl ContextFamily for Fields { +//! type Context<'data> = ExecutionContext<'data>; +//! type UserData = (); +//! +//! fn bind>( +//! field: Field, +//! consumer: B, +//! ) -> Result { +//! consumer.consume(field, Getter) +//! } +//! } +//! +//! impl FieldGetter for Getter { +//! type Provider = LhsValueProvider; +//! +//! fn get<'ctx, 'data: 'ctx>( +//! &self, +//! context: &'ctx ExecutionContext<'data>, +//! field: &Field, +//! ) -> Option<&'ctx LhsValue<'ctx>> { +//! context.get_field_value(field.as_ref()) +//! } +//! } +//! +//! # fn main() -> Result<(), Box> { +//! let mut builder = SchemeBuilder::new(); +//! builder.add_field("port", Type::Int)?; +//! let scheme = builder.build(); +//! Fields::validate_scheme(&scheme)?; +//! let filter = scheme +//! .parse("port eq 443")? +//! .compile_with_compiler(&mut DefaultCompiler::<(), Fields>::new()); +//! let mut context = ExecutionContext::new(&scheme); +//! context.set_field_value(scheme.get_field("port")?, 443i64)?; +//! assert!(filter.execute(&context)?); +//! # Ok(()) +//! # } +//! ``` +//! +//! The field's scheme type determines comparison and traversal during filter +//! compilation. Because a getter returning `&LhsValue` has no fixed logical type, +//! startup validation cannot check which variant it returns. The context must +//! check values against the scheme when inserting them, as `ExecutionContext` +//! does. The field macro still validates getter coverage and optionality. +//! +//! [`crate::values::FieldValueType`] describes these scheme-typed field roots as +//! well as ordinary static [`crate::values::ValueType`] implementations. The +//! collection interfaces require a static `ValueType` for their elements: for +//! example, `Vec` is supported, while a runtime-typed array uses +//! `LhsValue::Array` rather than `Vec`. + +pub(crate) mod fields; +pub(crate) mod state; + +use crate::values::{FieldValueType, IntoLhsValue, LhsValueType, Value}; +use crate::{ExecutionContext, Field, GetType, LhsValue, List, ListMatcher, ListRef, Scheme, Type}; +use std::marker::PhantomData; + +/// The scheme associated with a runtime context. +pub trait HasScheme { + /// Scheme used to interpret fields and lists. + fn scheme(&self) -> &Scheme; +} + +/// State shared by expression evaluation, independently of field storage. +/// +/// Together with [`HasScheme`], this trait exposes the scheme, list matchers, +/// and user data available during evaluation. Field getters are bound separately +/// through [`ContextFamily`]. +/// +/// An application context can use its existing storage or forward these methods +/// to a borrowed [`crate::ContextState`]. [`ExecutionContext`] already implements +/// this interface alongside its stored field values. See [`crate::context`] for +/// a complete example using application-owned fields and reusable state. +pub trait EvaluationContext: HasScheme { + /// Per-evaluation state used by compiler customizations. + type UserData; + + /// State retained by the embedding runtime. + fn get_user_data(&self) -> &Self::UserData; + + /// Matcher for a list registered in this context's scheme. + fn get_list_matcher(&self, list: ListRef<'_>) -> &dyn ListMatcher; + + /// Matcher for the list registered for a Wirefilter type, if present. + #[inline] + fn get_list_matcher_from_type(&self, ty: &Type) -> Option<&dyn ListMatcher> { + let list = self.scheme().get_list(ty)?; + Some(self.get_list_matcher(list)) + } + + /// Matcher access after the outer filter has checked scheme compatibility. + /// + /// The default delegates to the checked accessor. Contexts forwarding matcher + /// storage to another context, such as [`crate::ContextState`], should + /// also forward this method to avoid repeating its scheme assertion for each + /// list comparison. The supplied list must belong to this context's scheme. + #[inline] + fn get_list_matcher_unchecked(&self, list: &List) -> &dyn ListMatcher { + self.get_list_matcher(list.as_ref()) + } +} + +/// Mutable executor state, independently of field storage. +/// +/// Extends [`EvaluationContext`] for updating list matchers and user data. Filter +/// evaluation only requires the shared interface; an application adds this +/// trait when it also needs mutable access. Both [`ExecutionContext`] and +/// [`crate::ContextState`] implement it. +pub trait EvaluationContextMut: EvaluationContext { + /// State used to initialize and clean up compiler customizations. + fn get_user_data_mut(&mut self) -> &mut Self::UserData; + + /// Mutable matcher for a list registered in this context's scheme. + fn get_list_matcher_mut(&mut self, list: ListRef<'_>) -> &mut dyn ListMatcher; + + /// Mutable matcher for the list registered for a Wirefilter type, if present. + #[inline] + fn get_list_matcher_mut_from_type(&mut self, ty: &Type) -> Option<&mut dyn ListMatcher> { + let list = self.scheme().get_list(ty)?.to_owned(); + Some(self.get_list_matcher_mut(list.as_ref())) + } +} + +/// A field-definition problem detected before filters are evaluated. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum FieldDefinitionError { + /// A scheme field failed binding or static type checking during validation. + #[error("Field {name} failed validation: {error}")] + Binding { + /// Scheme field being validated. + name: Box, + /// The binding or type-checking failure. + #[source] + error: FieldBindingError, + }, + /// A scheme field has no getter. + #[error("Missing field getter for {0}")] + MissingGetter(Box), + /// A getter names a field absent from the scheme. + #[error("Unknown field {0}")] + UnknownField(Box), + /// More than one getter names the same field. + #[error("Duplicate field getter for {0}")] + DuplicateGetter(Box), + /// A getter for a mandatory scheme field can return no value. + #[error("Field {0} is mandatory but its getter returns Option")] + OptionalGetterForMandatoryField(Box), + /// Getter and scheme types differ. + #[error("Field {name} has type {getter_type}, expected {scheme_type}")] + TypeMismatch { + /// Field name. + name: Box, + /// Type declared in the scheme. + scheme_type: Type, + /// Type returned by the getter. + getter_type: Type, + }, +} + +/// All field-definition problems found during explicit scheme validation. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[error("Invalid field definitions: {0:?}")] +pub struct SchemeValidationError(pub Box<[FieldDefinitionError]>); + +/// A failure to bind a field or prepare its access. +/// +/// Binding APIs return this error. Default scheme validation collects it in +/// [`FieldDefinitionError::Binding`]. The infallible expression compilation APIs +/// panic on a binding failure; filter execution does not return this error. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum FieldBindingError { + /// A field cannot be resolved by the context family. + #[error("Unknown field {0}")] + UnknownField(Box), + /// The getter's logical type differs from the scheme. + #[error("Field {field} has type {actual}, expected {expected}")] + TypeMismatch { + /// Field whose type differs. + field: Box, + /// Type declared in the scheme. + expected: Type, + /// Type provided by the getter. + actual: Type, + }, + /// A path is incompatible with the declared value type. + /// + /// Produced during binding or path compilation, not evaluation. Parsed paths + /// for matching scheme and getter types are expected to be compatible. + #[error("Unsupported value operation: {0}")] + Unsupported(&'static str), +} + +/// A lifetime-independent description of contexts and their field bindings. +/// +/// A separate marker can associate `Context<'data>` with a borrowing request +/// context while compiled filters retain no request lifetime. An owned context +/// type satisfying `'static` may instead implement this trait itself and use +/// `type Context<'data> = Self`. The family is used as a type parameter; compiled +/// expressions do not store an instance of it. [`crate::wirefilter_fields!`] +/// generates a separate marker for both forms of context. +/// +/// [`EvaluationContext`] describes the state available during evaluation; this +/// trait selects that context type and binds its field getters for validation +/// and compilation. It does not own the context, request data, or +/// [`crate::ContextState`]. See [`crate::context`] for a complete example +/// showing their relationship. +pub trait ContextFamily: Sized + 'static { + /// State exposed to compiler customizations. + type UserData: 'static; + + /// The runtime context, possibly retaining request references. + type Context<'data>: EvaluationContext; + + /// Resolve a field and pass its concrete getter to the consumer. + /// Called by default scheme validation and field compilation. + /// Ownership of the logical field transfers to the consumer. + fn bind>( + field: Field, + consumer: B, + ) -> Result; + + /// Validate field definitions without evaluating getters. + /// + /// Applications must call this explicitly during initialization for each + /// scheme and context family, before loading rulesets or compiling + /// expressions. A failed check is an initialization error. Parsing, + /// compilation, and evaluation do not invoke this method automatically. + /// + /// The default implementation binds every scheme field and checks its + /// provider's static type, collecting all binding failures. It constructs + /// getters through [`Self::bind`] but never calls [`FieldGetter::get`]. + /// Scheme-typed providers such as [`LhsValueProvider`] have no static type to + /// check; their storage must validate values separately. + /// + /// This cannot detect extra or duplicate getter definitions, or determine + /// whether a mandatory getter can return `None`. Handwritten families remain + /// responsible for those constraints and may override this method for + /// stronger checks. [`crate::wirefilter_fields!`] generates an override that + /// checks all definitions, including their declared optionality. + /// [`ExecutionContextFamily`] overrides this with a no-op: its field table + /// comes from the scheme and its context validates values on insertion. + fn validate_scheme(scheme: &Scheme) -> Result<(), SchemeValidationError> { + let errors = scheme + .fields() + .filter_map(|field| { + Self::bind(field.to_owned(), CheckFieldType) + .err() + .map(|error| FieldDefinitionError::Binding { + name: field.name().into(), + error, + }) + }) + .collect::>(); + if errors.is_empty() { + Ok(()) + } else { + Err(SchemeValidationError(errors)) + } + } +} + +struct CheckFieldType; + +impl FieldConsumer for CheckFieldType { + type Output = (); + + fn consume>( + self, + field: Field, + _: G, + ) -> Result { + check_field_type::<::ValueType>(&field) + } +} + +pub(crate) fn check_field_type(field: &Field) -> Result<(), FieldBindingError> { + let expected = field.get_type(); + if let Some(actual) = S::STATIC_TYPE + && actual != expected + { + return Err(FieldBindingError::TypeMismatch { + field: field.name().into(), + expected, + actual, + }); + } + Ok(()) +} + +/// A context reference whose type records that request data outlives this access. +/// +/// Higher-ranked closures over an arbitrary context family's associated type do +/// not otherwise imply this lifetime relationship. The witness lets compiled +/// value expressions return borrowed payloads, including from invariant contexts. +/// It contains only the original reference and requires no allocation. +#[repr(transparent)] +pub struct ContextRef<'ctx, 'data, F: ContextFamily> { + context: &'ctx F::Context<'data>, + lifetime: PhantomData<&'ctx &'data ()>, +} + +impl Copy for ContextRef<'_, '_, F> {} + +impl Clone for ContextRef<'_, '_, F> { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +impl<'ctx, 'data: 'ctx, F: ContextFamily> ContextRef<'ctx, 'data, F> { + /// Record the relationship between the context borrow and its retained data. + #[inline] + pub fn new(context: &'ctx F::Context<'data>) -> Self { + Self { + context, + lifetime: PhantomData, + } + } + + /// Return the original context borrow, independently of the witness's storage. + #[inline] + pub fn context(self) -> &'ctx F::Context<'data> { + self.context + } +} + +/// An operation that validates a field binding or compiles field access. +pub trait FieldConsumer: Sized { + /// Result produced by this operation, such as an executable expression. + type Output; + + /// Process a getter whose concrete type remains known. + /// `field` is the original logical field supplied to the binding operation. + /// This includes getters borrowing existing [`LhsValue`] fields through + /// [`LhsValueProvider`]. + fn consume>( + self, + field: Field, + getter: G, + ) -> Result; +} + +/// A field's representation across request and access lifetimes. +pub trait ProviderFamily: 'static { + /// Recursive Wirefilter type represented by the provider. + type ValueType: FieldValueType; + + /// Storage usable by generated contexts and custom adapters. + type Storage<'data>; + + /// A reference or selection descriptor returned by one field access. + type Value<'ctx, 'data>: Value + IntoLhsValue<'ctx> + where + 'data: 'ctx; + + /// Obtain a shorter access to retained storage. + fn reborrow<'ctx, 'data: 'ctx>(value: &'ctx Self::Storage<'data>) -> Self::Value<'ctx, 'data>; +} + +/// A provider borrowing storage whose Rust type has no request lifetime. +pub struct BorrowedValueProvider(PhantomData T>); + +impl ProviderFamily for BorrowedValueProvider { + type Storage<'data> = &'data T; + type Value<'ctx, 'data> + = &'ctx T + where + 'data: 'ctx; + type ValueType = T::ValueType; + + #[inline] + fn reborrow<'ctx, 'data: 'ctx>(value: &'ctx Self::Storage<'data>) -> Self::Value<'ctx, 'data> { + value + } +} + +/// A provider returning a value whose Rust type has no request lifetime. +/// +/// Scalar getters can return values directly. Custom by-value descriptors that +/// borrow request data instead implement [`ProviderFamily`] with those lifetimes. +pub struct OwnedValueProvider(PhantomData T>); + +impl ProviderFamily for OwnedValueProvider +where + T: Value + Clone + 'static, + for<'a> T: IntoLhsValue<'a>, +{ + type Storage<'data> = T; + type Value<'ctx, 'data> + = T + where + 'data: 'ctx; + type ValueType = T::ValueType; + + #[inline] + fn reborrow<'ctx, 'data: 'ctx>(value: &'ctx Self::Storage<'data>) -> Self::Value<'ctx, 'data> { + value.clone() + } +} + +/// A provider borrowing a stored value with request-borrowed payloads. +/// +/// The scheme supplies the logical field type. A context must validate inserted +/// values against it; this provider does not convert or copy the retained value. +/// Handwritten getters can use this provider to read an ordinary field table +/// through the field-binding interface. +pub struct LhsValueProvider; + +impl ProviderFamily for LhsValueProvider { + type Storage<'data> = LhsValue<'data>; + type Value<'ctx, 'data> + = &'ctx LhsValue<'ctx> + where + 'data: 'ctx; + type ValueType = LhsValueType; + + #[inline] + fn reborrow<'ctx, 'data: 'ctx>(value: &'ctx Self::Storage<'data>) -> Self::Value<'ctx, 'data> { + value + } +} + +/// Concrete field access captured inside an executable expression. +pub trait FieldGetter: Send + Sync + 'static { + /// The field representation. + type Provider: ProviderFamily; + + /// Read the field for this evaluation without constructing a value enum. + /// + /// `field` is the original logical field supplied to the binding operation, + /// including its scheme index, name, type, and optionality. Its borrow is + /// independent of the context borrow and cannot be retained in the result. + /// + /// A getter bound to a mandatory scheme field must return `Some`. The field + /// macro enforces this through its return signatures and explicit scheme + /// validation; handwritten implementations must uphold the same contract. + fn get<'ctx, 'data: 'ctx>( + &self, + context: &'ctx F::Context<'data>, + field: &Field, + ) -> Option<::Value<'ctx, 'data>>; +} + +/// A getter result preserving the relationship between access and request lifetimes. +/// +/// This transparent wrapper keeps the provider's associated value outside the +/// higher-ranked function pointer's return projection. The value has exactly +/// the representation of `Option>`. +// Returning the projection directly encounters rustc's higher-ranked lifetime +// limitation (rust-lang/rust#100013), despite ContextRef carrying the constraint. +#[repr(transparent)] +pub struct OptionalFieldGetterResult<'ctx, 'data: 'ctx, P: ProviderFamily> { + value: Option>, +} + +impl<'ctx, 'data: 'ctx, P: ProviderFamily> OptionalFieldGetterResult<'ctx, 'data, P> { + /// Retain the result of a getter for the context access lifetime. + pub fn new(value: Option>) -> Self { + Self { value } + } +} + +/// Field access represented by a typed function pointer. +/// +/// Fields in the same context family using the same provider share this getter +/// type, allowing consumers to share their compiled expression implementations. +/// The pointer selects the field's getter on each access; values retain +/// their concrete type and do not require an intermediate value enum or allocation. +/// Handwritten getters retaining additional state can implement [`FieldGetter`] +/// directly instead. +pub struct FieldGetterFn { + callback: for<'ctx, 'data, 'field> fn( + ContextRef<'ctx, 'data, F>, + &'field Field, + ) -> OptionalFieldGetterResult<'ctx, 'data, P>, +} + +impl FieldGetterFn { + /// Wrap a getter without allocating or retaining a request lifetime. + /// + /// The context witness records that request data outlives the access. Field + /// metadata is borrowed independently and cannot be retained in the result. + /// A mandatory field's result must contain `Some`, as for [`FieldGetter`]. + pub fn new( + callback: for<'ctx, 'data, 'field> fn( + ContextRef<'ctx, 'data, F>, + &'field Field, + ) -> OptionalFieldGetterResult<'ctx, 'data, P>, + ) -> Self { + Self { callback } + } +} + +impl FieldGetter for FieldGetterFn { + type Provider = P; + + // Keep this adapter as thin as the macro's concrete FieldGetter adapter. + #[inline] + fn get<'ctx, 'data: 'ctx>( + &self, + context: &'ctx F::Context<'data>, + field: &Field, + ) -> Option> { + (self.callback)(ContextRef::new(context), field).value + } +} + +/// A plain getter result preserving access and request lifetime relationships. +/// +/// Like [`OptionalFieldGetterResult`], this transparent wrapper avoids a higher-ranked +/// associated-type projection in the callback signature. It contains a plain +/// value; presence is added by [`RequiredFieldGetterFn::get`] after the +/// function pointer returns. +#[repr(transparent)] +pub struct RequiredFieldGetterResult<'ctx, 'data: 'ctx, P: ProviderFamily> { + value: P::Value<'ctx, 'data>, +} + +impl<'ctx, 'data: 'ctx, P: ProviderFamily> RequiredFieldGetterResult<'ctx, 'data, P> { + /// Retain an always-present value for the context access lifetime. + pub fn new(value: P::Value<'ctx, 'data>) -> Self { + Self { value } + } +} + +/// Always-present field access represented by a typed function pointer. +/// +/// Plain-return getters in the same context family using the same provider share +/// this getter type. The callback returns a plain value; this wrapper +/// constructs `Some` after calling it, so consumers can establish presence +/// independently of the selected callback. The scheme field may be mandatory or +/// optional. Getters returning `Option` use [`FieldGetterFn`] instead. +pub struct RequiredFieldGetterFn { + callback: for<'ctx, 'data, 'field> fn( + ContextRef<'ctx, 'data, F>, + &'field Field, + ) -> RequiredFieldGetterResult<'ctx, 'data, P>, +} + +impl RequiredFieldGetterFn { + /// Wrap a plain-return getter without allocation or a retained request lifetime. + /// + /// Field metadata is borrowed independently and cannot be retained in the + /// result. The callback must produce a value on every access. + pub fn new( + callback: for<'ctx, 'data, 'field> fn( + ContextRef<'ctx, 'data, F>, + &'field Field, + ) -> RequiredFieldGetterResult<'ctx, 'data, P>, + ) -> Self { + Self { callback } + } +} + +impl FieldGetter for RequiredFieldGetterFn { + type Provider = P; + + // Match the optional wrapper's thin adapter while making presence visible + // to consumers without inspecting the callback implementation. + #[inline] + fn get<'ctx, 'data: 'ctx>( + &self, + context: &'ctx F::Context<'data>, + field: &Field, + ) -> Option> { + Some((self.callback)(ContextRef::new(context), field).value) + } +} + +/// Default family for the ordinary field-table [`ExecutionContext`]. +/// +/// Fields use [`LhsValueProvider`] through the same binding interface as +/// application-owned Rust values. The context owns the field table and validates +/// inserted values against the scheme; readers borrow the retained values and +/// containers. +pub struct ExecutionContextFamily(PhantomData U>); + +impl ContextFamily for ExecutionContextFamily { + type Context<'data> = ExecutionContext<'data, U>; + type UserData = U; + + #[inline] + fn bind>( + field: Field, + consumer: B, + ) -> Result { + consumer.consume(field, ExecutionContextGetter) + } + + fn validate_scheme(_: &Scheme) -> Result<(), SchemeValidationError> { + // Field coverage comes from the scheme; values are checked on insertion. + Ok(()) + } +} + +struct ExecutionContextGetter; + +impl FieldGetter> for ExecutionContextGetter { + type Provider = LhsValueProvider; + + #[inline] + fn get<'ctx, 'data: 'ctx>( + &self, + context: &'ctx ExecutionContext<'data, U>, + field: &Field, + ) -> Option<&'ctx LhsValue<'ctx>> { + context.get_field_value_unchecked(field) + } +} + +#[cfg(test)] +mod tests { + use super::{ + ContextFamily, FieldBindingError, FieldConsumer, FieldGetter, LhsValueProvider, + SchemeValidationError, + }; + use crate::{ExecutionContext, Field, Scheme}; + + #[test] + fn test_schema_validation_is_explicit() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + static VALIDATIONS: AtomicUsize = AtomicUsize::new(0); + struct Family; + struct Getter; + impl ContextFamily for Family { + type Context<'data> = ExecutionContext<'data>; + type UserData = (); + + fn bind>( + field: Field, + consumer: B, + ) -> Result { + consumer.consume(field, Getter) + } + + fn validate_scheme(scheme: &Scheme) -> Result<(), SchemeValidationError> { + VALIDATIONS.fetch_add(1, Ordering::Relaxed); + crate::validate_field_definitions( + scheme, + &[("flag", Some(crate::Type::Bool), false)], + ) + } + } + impl FieldGetter for Getter { + type Provider = LhsValueProvider; + + fn get<'ctx, 'data: 'ctx>( + &self, + context: &'ctx ExecutionContext<'data>, + field: &Field, + ) -> Option<&'ctx crate::LhsValue<'ctx>> { + context.get_field_value_unchecked(field) + } + } + + let scheme = Scheme! { flag: Bool }.build(); + Family::validate_scheme(&scheme).unwrap(); + assert_eq!(VALIDATIONS.load(Ordering::Relaxed), 1); + + let mut compiler = crate::DefaultCompiler::<(), Family>::new(); + let mut context = ExecutionContext::new(&scheme); + context.set_field_value_from_name("flag", true).unwrap(); + for _ in 0..3 { + let filter = scheme + .parse("flag") + .unwrap() + .compile_with_compiler(&mut compiler); + let value = scheme + .parse_value("flag") + .unwrap() + .compile_with_compiler(&mut compiler); + assert!(filter.execute(&context).unwrap()); + assert_eq!( + value.execute(&context).unwrap(), + Ok(crate::LhsValue::Bool(true)) + ); + } + assert_eq!(VALIDATIONS.load(Ordering::Relaxed), 1); + } + + #[test] + fn test_custom_lhs_value_getter_uses_logical_field_metadata() { + struct Family; + struct Getter; + impl ContextFamily for Family { + type Context<'data> = ExecutionContext<'data>; + type UserData = (); + + fn bind>( + field: Field, + consumer: B, + ) -> Result { + consumer.consume(field, Getter) + } + } + impl FieldGetter for Getter { + type Provider = LhsValueProvider; + + fn get<'ctx, 'data: 'ctx>( + &self, + context: &'ctx ExecutionContext<'data>, + _: &Field, + ) -> Option<&'ctx crate::LhsValue<'ctx>> { + // Both logical fields read the same optional storage slot. + let storage = context.scheme().get_field("storage").unwrap(); + context.get_field_value(storage) + } + } + #[derive(Default)] + struct Compiler { + bindings: usize, + } + impl crate::Compiler for Compiler { + type Family = Family; + type U = (); + + fn bind_field>( + &mut self, + field: Field, + consumer: B, + ) -> Result { + self.bindings += 1; + Family::bind(field, consumer) + } + } + + let mut builder = crate::SchemeBuilder::new(); + builder.add_field("required", crate::Type::Bool).unwrap(); + builder + .add_optional_field("optional", crate::Type::Bool) + .unwrap(); + builder + .add_optional_field("storage", crate::Type::Bool) + .unwrap(); + let scheme = builder.build(); + let mut compiler = Compiler::default(); + let required = scheme + .parse("required") + .unwrap() + .compile_with_compiler(&mut compiler); + let optional = scheme + .parse("optional") + .unwrap() + .compile_with_compiler(&mut compiler); + let required_value = scheme + .parse_value("required") + .unwrap() + .compile_with_compiler(&mut compiler); + let optional_value = scheme + .parse_value("optional") + .unwrap() + .compile_with_compiler(&mut compiler); + assert_eq!(compiler.bindings, 4); + + let mut context = ExecutionContext::new(&scheme); + assert!(!optional.execute(&context).unwrap()); + assert_eq!( + optional_value.execute(&context).unwrap(), + Err(crate::Type::Bool) + ); + let operations: [&dyn Fn(); 2] = [ + &|| { + let _ = required.execute(&context); + }, + &|| { + let _ = required_value.execute(&context); + }, + ]; + for operation in operations { + let panic = + std::panic::catch_unwind(std::panic::AssertUnwindSafe(operation)).unwrap_err(); + assert_eq!( + panic.downcast_ref::().map(String::as_str), + Some("Field required was registered as mandatory but not given a value"), + ); + } + for value in [false, true] { + context.set_field_value_from_name("storage", value).unwrap(); + assert_eq!(required.execute(&context).unwrap(), value); + assert_eq!(optional.execute(&context).unwrap(), value); + assert_eq!( + required_value.execute(&context).unwrap(), + Ok(crate::LhsValue::Bool(value)) + ); + assert_eq!( + optional_value.execute(&context).unwrap(), + Ok(crate::LhsValue::Bool(value)) + ); + } + assert_eq!(compiler.bindings, 4); + } + + type Headers = std::collections::BTreeMap, Vec>>; + + struct NativeContext { + scheme: Scheme, + names: Vec>, + flags: Vec, + other_flags: Vec, + text: Option>, + other_text: Option>, + num: Option, + other_num: Option, + ip: std::net::IpAddr, + headers: Headers, + required: Option>, + lists: Box<[Box]>, + list_accesses: std::cell::Cell, + } + + impl super::HasScheme for NativeContext { + fn scheme(&self) -> &Scheme { + &self.scheme + } + } + + impl super::EvaluationContext for NativeContext { + type UserData = (); + + fn get_user_data(&self) -> &() { + &() + } + + fn get_list_matcher(&self, list: crate::ListRef<'_>) -> &dyn crate::ListMatcher { + self.list_accesses.set(self.list_accesses.get() + 1); + &*self.lists[list.index()] + } + } + + crate::wirefilter_fields! { + impl NativeFields for NativeContext { + "names" => { + fn(ctx: &NativeContext) -> &Vec> { &ctx.names } + }, + "flags" => { + fn(ctx: &NativeContext) -> &Vec { &ctx.flags } + }, + "other_flags" => { + fn(ctx: &NativeContext) -> &Vec { &ctx.other_flags } + }, + "text" => { + fn(ctx: &NativeContext) -> Option<&str> { ctx.text.as_deref() } + }, + "other_text" => { + fn(ctx: &NativeContext) -> Option<&str> { ctx.other_text.as_deref() } + }, + "num" => { + fn(ctx: &NativeContext) -> Option { ctx.num } + }, + "other_num" => { + fn(ctx: &NativeContext) -> Option { ctx.other_num } + }, + "ip" => { + fn(ctx: &NativeContext) -> std::net::IpAddr { ctx.ip } + }, + "headers" => { + fn(ctx: &NativeContext) -> &Headers { &ctx.headers } + }, + "required" => { + fn(ctx: &NativeContext) -> &str { + ctx.required.as_deref().expect("test context has its required field") + } + }, + } + } + + fn echo<'a>(args: crate::FunctionArgs<'_, 'a>) -> Option> { + args.next()?.ok() + } + + fn owned_names<'a>(args: crate::FunctionArgs<'_, 'a>) -> Option> { + args.next()?.as_ref().ok()?; + Some(crate::LhsValue::Array( + crate::Array::try_from_iter( + crate::Type::Bytes, + ["Accept", "Other"].map(|s| crate::LhsValue::from(Box::::from(s))), + ) + .unwrap(), + )) + } + + fn native_context(nil_not_equal: bool) -> NativeContext { + use crate::{ + SimpleFunctionArgKind, SimpleFunctionDefinition, SimpleFunctionImpl, + SimpleFunctionParam, Type, + }; + let array_bytes = Type::array(Type::Bytes); + let headers = Type::map(array_bytes); + let mut builder = crate::SchemeBuilder::new(); + for (name, ty, optional) in [ + ("names", array_bytes, false), + ("flags", Type::array(Type::Bool), false), + ("other_flags", Type::array(Type::Bool), false), + ("text", Type::Bytes, true), + ("other_text", Type::Bytes, true), + ("num", Type::Int, true), + ("other_num", Type::Int, true), + ("ip", Type::Ip, false), + ("headers", headers, false), + ("required", Type::Bytes, false), + ] { + if optional { + builder.add_optional_field(name, ty).unwrap(); + } else { + builder.add_field(name, ty).unwrap(); + } + } + for (name, ty) in [ + ("echo", Type::Bytes), + ("echo_names", array_bytes), + ("echo_headers", headers), + ] { + builder + .add_function( + name, + SimpleFunctionDefinition { + params: Box::new([SimpleFunctionParam { + arg_kind: SimpleFunctionArgKind::Field, + val_type: ty, + }]), + opt_params: Box::new([]), + return_type: ty, + implementation: SimpleFunctionImpl::new(echo), + }, + ) + .unwrap(); + } + builder + .add_function( + "owned_names", + SimpleFunctionDefinition { + params: Box::new([SimpleFunctionParam { + arg_kind: SimpleFunctionArgKind::Field, + val_type: Type::Bytes, + }]), + opt_params: Box::new([]), + return_type: array_bytes, + implementation: SimpleFunctionImpl::new(owned_names), + }, + ) + .unwrap(); + builder + .add_list(Type::Int, crate::AlwaysList::default()) + .unwrap(); + builder.set_nil_not_equal_behavior(nil_not_equal); + let scheme = builder.build(); + NativeFields::validate_scheme(&scheme).unwrap(); + let lists = scheme + .lists() + .map(|list| list.definition().new_matcher()) + .collect(); + NativeContext { + scheme, + names: vec!["Accept".into(), "Other".into(), "".into()], + flags: vec![true, false, true], + other_flags: vec![false], + text: Some("Accept".into()), + other_text: Some("Other".into()), + num: Some(7), + other_num: Some(2), + ip: "192.0.2.1".parse().unwrap(), + headers: [ + ( + Box::from(b"accept".as_slice()), + vec!["text/html".into(), "application/json".into()], + ), + (Box::from(b"empty".as_slice()), Vec::new()), + ] + .into_iter() + .collect(), + required: Some("ok".into()), + lists, + list_accesses: std::cell::Cell::new(0), + } + } + + fn byte_array(values: &[Box]) -> crate::LhsValue<'_> { + crate::LhsValue::Array( + crate::Array::try_from_iter( + crate::Type::Bytes, + values.iter().map(|s| crate::LhsValue::from(s.as_ref())), + ) + .unwrap(), + ) + } + + fn stored_context(ctx: &NativeContext) -> ExecutionContext<'_> { + use crate::{LhsValue, Type}; + let mut stored = ExecutionContext::new(&ctx.scheme); + for (name, value) in [ + ("names", byte_array(&ctx.names)), + ( + "flags", + crate::Array::try_from_iter(Type::Bool, ctx.flags.iter().copied()) + .unwrap() + .into(), + ), + ( + "other_flags", + crate::Array::try_from_iter(Type::Bool, ctx.other_flags.iter().copied()) + .unwrap() + .into(), + ), + ("ip", LhsValue::Ip(ctx.ip)), + ( + "headers", + crate::Map::try_from_iter( + Type::array(Type::Bytes), + ctx.headers.iter().map(|(key, values)| { + Ok::<_, crate::TypeMismatchError>((key.clone(), byte_array(values))) + }), + ) + .unwrap() + .into(), + ), + ] { + stored.set_field_value_from_name(name, value).unwrap(); + } + for (name, value) in [ + ("text", &ctx.text), + ("other_text", &ctx.other_text), + ("required", &ctx.required), + ] { + if let Some(value) = value { + stored + .set_field_value_from_name(name, value.as_ref()) + .unwrap(); + } + } + for (name, value) in [("num", ctx.num), ("other_num", ctx.other_num)] { + if let Some(value) = value { + stored.set_field_value_from_name(name, value).unwrap(); + } + } + stored + } + + fn compare_filters(ctx: &NativeContext, expressions: &[&str]) { + let stored = stored_context(ctx); + let mut compiler = crate::DefaultCompiler::<(), NativeFields>::new(); + for expression in expressions { + let ast = ctx + .scheme + .parse(expression) + .unwrap_or_else(|error| panic!("{expression}: {error}")); + let expected = ast.clone().compile().execute(&stored).unwrap(); + let actual = ast + .compile_with_compiler(&mut compiler) + .execute(ctx) + .unwrap(); + assert_eq!(actual, expected, "{expression}"); + } + } + + #[test] + fn test_comparison_bindings_use_compiler_hooks_only_during_compilation() { + struct RecordingCompiler { + fields: Vec>, + family: std::marker::PhantomData, + } + + impl> crate::Compiler for RecordingCompiler { + type Family = F; + type U = (); + + fn bind_field>( + &mut self, + field: Field, + consumer: B, + ) -> Result { + self.fields.push(field.name().into()); + F::bind(field, consumer) + } + } + + let context = native_context(true); + let stored = stored_context(&context); + let mut native_compiler = RecordingCompiler:: { + fields: Vec::new(), + family: std::marker::PhantomData, + }; + let mut stored_compiler = RecordingCompiler::> { + fields: Vec::new(), + family: std::marker::PhantomData, + }; + + for (expression, field) in [ + (r#"names[0] == "Accept""#, "names"), + (r#"any(names[*] == "Accept")"#, "names"), + (r#"any(headers[*][0] == "text/html")"#, "headers"), + (r#"any(headers[*][*] == "application/json")"#, "headers"), + ] { + let ast = context.scheme.parse(expression).unwrap(); + let native_filter = ast.clone().compile_with_compiler(&mut native_compiler); + let stored_filter = ast.compile_with_compiler(&mut stored_compiler); + assert_eq!(native_compiler.fields.len(), 1); + assert_eq!(native_compiler.fields[0].as_ref(), field); + assert_eq!(stored_compiler.fields.len(), 1); + assert_eq!(stored_compiler.fields[0].as_ref(), field); + + for _ in 0..2 { + assert!(native_filter.execute(&context).unwrap(), "{expression}"); + assert!(stored_filter.execute(&stored).unwrap(), "{expression}"); + } + assert_eq!(native_compiler.fields.len(), 1); + assert_eq!(stored_compiler.fields.len(), 1); + native_compiler.fields.clear(); + stored_compiler.fields.clear(); + } + } + + #[test] + fn test_native_complete_operator_and_collection_paths() { + let ctx = native_context(true); + compare_filters( + &ctx, + &[ + "num == 7", + "num != 7", + "num < 8", + "num <= 7", + "num > 7", + "num >= 7", + "num & 2", + "num & other_num", + "num in {1 5..9}", + "num == other_num", + r#"text == "Accept""#, + r#"text != "Accept""#, + r#"text < "Other""#, + r#"text contains "cep""#, + r#"text contains """#, + r#"text contains "a string longer than the entire field""#, + r#"text matches "^A.*t$""#, + r#"text wildcard "a*""#, + r#"text strict wildcard "a*""#, + r#"text in {"Other" "Accept"}"#, + "text == other_text", + "text > other_text", + "ip == 192.0.2.1", + "ip < 192.0.2.2", + "ip < ::1", + "ip in {192.0.2.0/24}", + "any(flags)", + "all(flags)", + "any(not flags)", + "any((flags and other_flags))", + "any((flags or other_flags))", + "all((flags xor other_flags))", + r#"any(names[*] == "Accept")"#, + r#"all(names[*] != "absent")"#, + r#"names[0] == "Accept""#, + r#"names[20] != "absent""#, + r#"any(headers[*][*] == "text/html")"#, + r#"headers["accept"][1] == "application/json""#, + r#"headers["ACCEPT"][0] == "text/html""#, + r#"all(headers["empty"][*] == "x")"#, + r#"all(headers["missing"][*] == "x")"#, + r#"any(echo(names[*])[*] == "Accept")"#, + r#"any(echo(headers[*][*])[*] == "text/html")"#, + r#"echo_names(names)[0] == "Accept""#, + r#"echo_headers(headers)["accept"][0] == "text/html""#, + r#"any(echo_headers(headers)[*][*] == "text/html")"#, + r#"owned_names(text)[1] == "Other""#, + r#"any(owned_names(text)[*] == "Accept")"#, + "num in $named_list", + ], + ); + assert!(ctx.list_accesses.get() > 0); + } + + #[test] + fn test_native_and_stored_byte_literal_formats() { + let ctx = native_context(true); + let stored = stored_context(&ctx); + for literal in [r#""Accept""#, r###"r##"Accept"##"###, "41:63:63:65:70:74"] { + for (operator, expected) in [ + ("==", true), + ("!=", false), + ("<=", true), + (">=", true), + ("<", false), + (">", false), + ("contains", true), + ] { + let expression = format!("text {operator} {literal}"); + let ast = ctx.scheme.parse(&expression).unwrap(); + let native = ast + .clone() + .compile_with_compiler(&mut crate::DefaultCompiler::<(), NativeFields>::new()); + assert_eq!(native.execute(&ctx).unwrap(), expected, "{expression}"); + assert_eq!( + ast.compile().execute(&stored).unwrap(), + expected, + "{expression}" + ); + } + } + } + + #[cfg(feature = "regex")] + #[test] + fn test_native_and_stored_regex_literal_formats() { + let ctx = native_context(true); + let stored = stored_context(&ctx); + for literal in [r#""^A.*t$""#, r###"r##"^A.*t$"##"###] { + for (lhs, expected) in [("text", true), ("other_text", false)] { + let expression = format!("{lhs} matches {literal}"); + let ast = ctx.scheme.parse(&expression).unwrap(); + let native = ast + .clone() + .compile_with_compiler(&mut crate::DefaultCompiler::<(), NativeFields>::new()); + assert_eq!(native.execute(&ctx).unwrap(), expected, "{expression}"); + assert_eq!( + ast.compile().execute(&stored).unwrap(), + expected, + "{expression}" + ); + } + } + } + + #[test] + fn test_native_and_stored_ip_literal_ordering() { + let mut ctx = native_context(true); + for (literal, values) in [ + ( + "192.0.2.2", + ["192.0.2.1", "192.0.2.2", "192.0.2.3", "2001:db8::2"], + ), + ( + "2001:db8::2", + ["2001:db8::1", "2001:db8::2", "2001:db8::3", "192.0.2.2"], + ), + ] { + for (operator, expected) in [ + ("==", [false, true, false, false]), + ("!=", [true, false, true, true]), + ("<", [true, false, false, false]), + ("<=", [true, true, false, false]), + (">", [false, false, true, false]), + (">=", [false, true, true, false]), + ] { + let expression = format!("ip {operator} {literal}"); + let ast = ctx.scheme.parse(&expression).unwrap(); + let native = ast + .clone() + .compile_with_compiler(&mut crate::DefaultCompiler::<(), NativeFields>::new()); + let stored = ast.compile(); + for (value, expected) in values.into_iter().zip(expected) { + ctx.ip = value.parse().unwrap(); + assert_eq!( + native.execute(&ctx).unwrap(), + expected, + "{value}: {expression}" + ); + assert_eq!( + stored.execute(&stored_context(&ctx)).unwrap(), + expected, + "{value}: {expression}" + ); + } + } + } + } + + #[test] + fn test_native_general_values_match_stored_values() { + let ctx = native_context(true); + let stored = stored_context(&ctx); + let mut compiler = crate::DefaultCompiler::<(), NativeFields>::new(); + for expression in [ + "text", + "names", + "headers", + "flags", + "num", + "ip", + "echo_names(names)", + "echo_headers(headers)", + "echo(names[*])", + "owned_names(text)", + "any((flags or other_flags))", + r#"headers["accept"]"#, + ] { + let ast = ctx + .scheme + .parse_value(expression) + .unwrap_or_else(|error| panic!("{expression}: {error}")); + let expected = ast.clone().compile(); + let actual = ast.compile_with_compiler(&mut compiler); + assert_eq!( + actual.execute(&ctx).unwrap(), + expected.execute(&stored).unwrap(), + "{expression}" + ); + } + } + + #[test] + fn test_native_dynamic_missing_values_follow_nil_policy() { + for nil_not_equal in [false, true] { + for (left, right) in [(true, true), (false, true), (true, false), (false, false)] { + let mut ctx = native_context(nil_not_equal); + if !left { + ctx.text = None; + ctx.num = None; + } + if !right { + ctx.other_text = None; + ctx.other_num = None; + } + compare_filters( + &ctx, + &[ + "text == other_text", + "text != other_text", + "text < other_text", + "text >= other_text", + "num == other_num", + "num != other_num", + "num < other_num", + "num >= other_num", + "num & other_num", + r#"text != "Accept""#, + "any(names[*] == other_text)", + "any(names[*] != other_text)", + r#"names[50] != "absent""#, + r#"any(echo(names[*])[*] == other_text)"#, + ], + ); + } + } + } + + #[test] + fn test_native_required_missing_and_scheme_identity() { + // Handwritten getters may violate mandatory-field presence even though + // the field macro rejects optional getters during scheme validation. + // Retain coverage for the execution-time invariant in that case. + struct MissingRequiredFields; + struct RequiredGetter; + + impl ContextFamily for MissingRequiredFields { + type Context<'data> = NativeContext; + type UserData = (); + + fn bind>( + field: Field, + consumer: B, + ) -> Result { + assert_eq!(field.name(), "required"); + consumer.consume(field, RequiredGetter) + } + } + + impl super::FieldGetter for RequiredGetter { + type Provider = super::BorrowedValueProvider; + + fn get<'ctx, 'data: 'ctx>( + &self, + context: &'ctx NativeContext, + field: &Field, + ) -> Option<&'ctx str> { + assert_eq!( + field.as_ref(), + context.scheme.get_field("required").unwrap() + ); + context.required.as_deref() + } + } + + let mut ctx = native_context(true); + let ast = ctx.scheme.parse(r#"required == "ok""#).unwrap(); + let stored_filter = ast.clone().compile(); + let native_filter = ast + .compile_with_compiler(&mut crate::DefaultCompiler::<(), MissingRequiredFields>::new()); + let different = native_context(true); + assert_eq!( + native_filter.execute(&different), + Err(crate::SchemeMismatchError) + ); + ctx.required = None; + let stored = stored_context(&ctx); + assert!( + std::panic::catch_unwind(std::panic::AssertUnwindSafe( + || stored_filter.execute(&stored) + )) + .is_err() + ); + let panic = + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| native_filter.execute(&ctx))) + .unwrap_err(); + assert_eq!( + panic.downcast_ref::().map(String::as_str), + Some("Field required was registered as mandatory but not given a value") + ); + compare_filters(&ctx, &[r#"num == 7 or required == "ok""#]); + } + + #[test] + fn test_native_value_borrows_invariant_context_for_one_access() { + use std::cell::Cell; + + struct InvariantContext<'data> { + scheme: &'data Scheme, + value: Cell<&'data str>, + } + + impl super::HasScheme for InvariantContext<'_> { + fn scheme(&self) -> &Scheme { + self.scheme + } + } + + impl super::EvaluationContext for InvariantContext<'_> { + type UserData = (); + + fn get_user_data(&self) -> &() { + &() + } + + fn get_list_matcher(&self, _: crate::ListRef<'_>) -> &dyn crate::ListMatcher { + unreachable!("the test scheme has no lists") + } + } + + crate::wirefilter_fields! { + impl<'data> InvariantFields for InvariantContext<'data> { + "value" => { + fn(ctx: &InvariantContext<'data>) -> &str { + ctx.value.get() + } + }, + } + } + + let scheme = Scheme! { value: Bytes }.build(); + InvariantFields::validate_scheme(&scheme).unwrap(); + let filter = scheme + .parse_value("value") + .unwrap() + .compile_with_compiler(&mut crate::DefaultCompiler::<(), InvariantFields>::new()); + let first = Box::::from("first"); + let second = Box::::from("second"); + let mut context = InvariantContext { + scheme: &scheme, + value: Cell::new(&first), + }; + let result = filter.execute(&context).unwrap().unwrap(); + let crate::LhsValue::Bytes(bytes) = result else { + panic!("expected bytes") + }; + assert_eq!(bytes.as_ref().as_ptr(), first.as_ptr()); + *context.value.get_mut() = &second; + assert_eq!( + filter.execute(&context).unwrap().unwrap(), + crate::LhsValue::from(second.as_ref()) + ); + assert_eq!( + std::mem::size_of::>(), + std::mem::size_of::<&InvariantContext<'_>>(), + ); + } + + #[test] + fn test_native_map_order_matches_materialized_identity_function() { + let mut context = native_context(true); + context + .headers + .insert(b"z-after".as_slice().into(), vec!["text/html".into()]); + context + .headers + .insert(b"a-before".as_slice().into(), vec!["other".into()]); + let expression = + r#"all((headers[*][*] == "text/html" xor echo_headers(headers)[*][*] == "text/html"))"#; + let stored = stored_context(&context); + let ast = context.scheme.parse(expression).unwrap(); + let stored_filter = ast.clone().compile(); + let native_filter = + ast.compile_with_compiler(&mut crate::DefaultCompiler::<(), NativeFields>::new()); + assert!(!stored_filter.execute(&stored).unwrap()); + assert!(!native_filter.execute(&context).unwrap()); + } +} diff --git a/engine/src/context/state.rs b/engine/src/context/state.rs new file mode 100644 index 00000000..0ff2c499 --- /dev/null +++ b/engine/src/context/state.rs @@ -0,0 +1,457 @@ +//! Persistent executor state independent of field storage. + +use crate::{ + EvaluationContext, EvaluationContextMut, HasScheme, List, ListMatcher, ListRef, Scheme, Type, +}; + +/// Scheme, list matchers, and compiler state retained across evaluations. +/// +/// This type has no field-value slots. The embedding runtime supplies +/// fields through its context family and can borrow this state while evaluating +/// them. Neither the state nor its list matchers must be rebuilt for each phase. +/// +/// This is an optional storage implementation of [`HasScheme`], +/// [`EvaluationContext`], and [`EvaluationContextMut`]. An application context can +/// borrow it and forward those methods while its [`crate::ContextFamily`] supplies +/// getters for application-owned field values. Applications can instead implement +/// the same interfaces using their existing state. See [`crate::context`] for a +/// complete example. +/// +/// Forward [`EvaluationContext::get_list_matcher_unchecked`] as well as the +/// checked accessor so list comparisons do not repeat the scheme check already +/// performed by the filter. +/// +/// [`crate::ExecutionContext`] already owns equivalent state together with its +/// stored field values and does not need a separate `ContextState`. The +/// application is responsible for resetting state when reusing it; [`Self::clear`] +/// clears list contents but retains user data. +#[derive(Debug, PartialEq)] +pub struct ContextState { + scheme: Scheme, + list_matchers: Box<[Box]>, + user_data: U, +} + +impl ContextState { + /// Create persistent state with default user data and the scheme's matchers. + pub fn new(scheme: &Scheme) -> Self + where + U: Default, + { + Self::new_with(scheme, U::default) + } + + /// Create persistent state with the supplied user-data initializer. + pub fn new_with(scheme: &Scheme, user_data: impl FnOnce() -> U) -> Self { + Self { + scheme: scheme.clone(), + list_matchers: scheme + .lists() + .map(|list| list.definition().new_matcher()) + .collect(), + user_data: user_data(), + } + } + + /// Scheme associated with this state. + #[inline] + pub fn scheme(&self) -> &Scheme { + &self.scheme + } + + /// Compiler-specific execution state. + #[inline] + pub fn get_user_data(&self) -> &U { + &self.user_data + } + + /// Mutable compiler-specific execution state. + #[inline] + pub fn get_user_data_mut(&mut self) -> &mut U { + &mut self.user_data + } + + /// Matcher registered for the supplied list. + #[inline] + pub fn get_list_matcher(&self, list: ListRef<'_>) -> &dyn ListMatcher { + assert!(self.scheme() == list.scheme()); + &*self.list_matchers[list.index()] + } + + /// Matcher registered for the supplied type, if any. + #[inline] + pub fn get_list_matcher_from_type(&self, ty: &Type) -> Option<&dyn ListMatcher> { + let list = self.scheme.get_list(ty)?; + Some(&*self.list_matchers[list.index()]) + } + + /// Mutable matcher registered for the supplied list. + #[inline] + pub fn get_list_matcher_mut(&mut self, list: ListRef<'_>) -> &mut dyn ListMatcher { + assert!(self.scheme() == list.scheme()); + &mut *self.list_matchers[list.index()] + } + + /// Mutable matcher registered for the supplied type, if any. + #[inline] + pub fn get_list_matcher_mut_from_type(&mut self, ty: &Type) -> Option<&mut dyn ListMatcher> { + let list = self.scheme.get_list(ty)?; + Some(&mut *self.list_matchers[list.index()]) + } + + /// Clear list contents while retaining their storage and user data. + /// + /// As with [`crate::ExecutionContext::clear`], compiler-specific user-data + /// cleanup remains the embedding runtime's responsibility. + pub fn clear(&mut self) { + self.list_matchers + .iter_mut() + .for_each(|matcher| matcher.clear()); + } +} + +impl HasScheme for ContextState { + #[inline] + fn scheme(&self) -> &Scheme { + self.scheme() + } +} + +impl EvaluationContext for ContextState { + type UserData = U; + + #[inline] + fn get_user_data(&self) -> &U { + self.get_user_data() + } + + #[inline] + fn get_list_matcher(&self, list: ListRef<'_>) -> &dyn ListMatcher { + self.get_list_matcher(list) + } + + #[inline] + fn get_list_matcher_from_type(&self, ty: &Type) -> Option<&dyn ListMatcher> { + self.get_list_matcher_from_type(ty) + } + + #[inline] + fn get_list_matcher_unchecked(&self, list: &List) -> &dyn ListMatcher { + debug_assert!(self.scheme() == list.scheme()); + &*self.list_matchers[list.index()] + } +} + +impl EvaluationContextMut for ContextState { + #[inline] + fn get_user_data_mut(&mut self) -> &mut U { + self.get_user_data_mut() + } + + #[inline] + fn get_list_matcher_mut(&mut self, list: ListRef<'_>) -> &mut dyn ListMatcher { + self.get_list_matcher_mut(list) + } + + #[inline] + fn get_list_matcher_mut_from_type(&mut self, ty: &Type) -> Option<&mut dyn ListMatcher> { + self.get_list_matcher_mut_from_type(ty) + } +} + +#[cfg(test)] +mod tests { + use super::ContextState; + use crate::{ + DefaultCompiler, EvaluationContext, EvaluationContextMut, HasScheme, LhsValue, List, + ListDefinition, ListMatcher, ListRef, Scheme, SchemeBuilder, Type, + }; + use serde::{Deserialize, Serialize}; + use std::collections::BTreeSet; + + #[derive(Debug)] + struct ValuesList; + + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)] + struct ValuesMatcher { + values: BTreeSet, + clears: usize, + } + + impl ListDefinition for ValuesList { + fn deserialize_matcher<'de>( + &self, + _: Type, + deserializer: &mut dyn erased_serde::Deserializer<'de>, + ) -> Result, erased_serde::Error> { + erased_serde::deserialize::(deserializer) + .map(|matcher| Box::new(matcher) as Box) + } + + fn new_matcher(&self) -> Box { + Box::new(ValuesMatcher::default()) + } + } + + impl ListMatcher for ValuesMatcher { + fn match_value(&self, name: &str, value: &LhsValue<'_>) -> bool { + name == "numbers" + && matches!(value, LhsValue::Int(value) if self.values.contains(value)) + } + + fn clear(&mut self) { + self.values.clear(); + self.clears += 1; + } + } + + fn scheme() -> Scheme { + let mut builder = SchemeBuilder::new(); + builder.add_field("name", Type::Bytes).unwrap(); + builder + .add_field("numbers", Type::array(Type::Int)) + .unwrap(); + builder.add_list(Type::Int, ValuesList).unwrap(); + builder.build() + } + + fn insert(context: &mut impl EvaluationContextMut, value: i64) { + context + .get_list_matcher_mut_from_type(&Type::Int) + .unwrap() + .as_any_mut() + .downcast_mut::() + .unwrap() + .values + .insert(value); + *context.get_user_data_mut() += 1; + } + + struct StateAdapter<'a>(&'a mut ContextState); + + impl HasScheme for StateAdapter<'_> { + fn scheme(&self) -> &Scheme { + self.0.scheme() + } + } + + impl EvaluationContext for StateAdapter<'_> { + type UserData = usize; + + fn get_user_data(&self) -> &usize { + self.0.get_user_data() + } + + fn get_list_matcher(&self, list: ListRef<'_>) -> &dyn ListMatcher { + self.0.get_list_matcher(list) + } + + fn get_list_matcher_unchecked(&self, list: &List) -> &dyn ListMatcher { + self.0.get_list_matcher_unchecked(list) + } + } + + impl EvaluationContextMut for StateAdapter<'_> { + fn get_user_data_mut(&mut self) -> &mut usize { + self.0.get_user_data_mut() + } + + fn get_list_matcher_mut(&mut self, list: ListRef<'_>) -> &mut dyn ListMatcher { + self.0.get_list_matcher_mut(list) + } + } + + #[test] + fn test_list_state_reuse_and_generic_mutation() { + let scheme = scheme(); + let mut state = ContextState::new_with(&scheme, || 7usize); + assert!(state.get_list_matcher_from_type(&Type::Bytes).is_none()); + assert!(state.get_list_matcher_mut_from_type(&Type::Bytes).is_none()); + insert(&mut StateAdapter(&mut state), 41); + assert_eq!(*state.get_user_data(), 8); + let matcher = state + .get_list_matcher(scheme.get_list(&Type::Int).unwrap()) + .as_any() + .downcast_ref::() + .unwrap(); + assert!(matcher.match_value("numbers", &LhsValue::Int(41))); + let address = matcher as *const ValuesMatcher; + state.clear(); + let matcher = state + .get_list_matcher_from_type(&Type::Int) + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(matcher as *const ValuesMatcher, address); + assert!(!matcher.match_value("numbers", &LhsValue::Int(41))); + assert_eq!(matcher.clears, 1); + assert_eq!(*state.get_user_data(), 8); + let adapter = StateAdapter(&mut state); + assert!(EvaluationContext::get_list_matcher_from_type(&adapter, &Type::Int).is_some()); + + let mut stored = crate::ExecutionContext::new_with(&scheme, || 7usize); + insert(&mut stored, 41); + assert_eq!(*stored.get_user_data(), 8); + assert!( + stored + .get_list_matcher_from_type(&Type::Int) + .unwrap() + .match_value("numbers", &LhsValue::Int(41)) + ); + } + + #[test] + fn test_explicit_stored_snapshot_preserves_lists_without_populating_fields() { + let scheme = scheme(); + let mut state = ContextState::new_with(&scheme, || 0usize); + insert(&mut state, 41); + let snapshot = crate::ExecutionContext::new_from_context(&state, 9usize); + assert!( + snapshot + .get_field_value(scheme.get_field("name").unwrap()) + .is_none() + ); + assert!( + snapshot + .get_field_value(scheme.get_field("numbers").unwrap()) + .is_none() + ); + state.clear(); + assert!( + !state + .get_list_matcher_from_type(&Type::Int) + .unwrap() + .match_value("numbers", &LhsValue::Int(41)) + ); + assert!( + snapshot + .get_list_matcher_from_type(&Type::Int) + .unwrap() + .match_value("numbers", &LhsValue::Int(41)) + ); + assert_eq!(*snapshot.get_user_data(), 9); + assert_eq!(*state.get_user_data(), 1); + } + + #[test] + fn test_list_access_checks_scheme_identity() { + let scheme = scheme(); + let other = self::scheme(); + let mut state = ContextState::<()>::new(&scheme); + let foreign = other.get_list(&Type::Int).unwrap(); + assert!( + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + state.get_list_matcher(foreign); + })) + .is_err() + ); + assert!( + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + state.get_list_matcher_mut(foreign); + })) + .is_err() + ); + } + + struct Request { + name: Box, + numbers: Vec, + } + + struct NativeContext<'a> { + request: &'a Request, + state: &'a ContextState, + } + + impl HasScheme for NativeContext<'_> { + fn scheme(&self) -> &Scheme { + self.state.scheme() + } + } + + impl EvaluationContext for NativeContext<'_> { + type UserData = usize; + + fn get_user_data(&self) -> &usize { + self.state.get_user_data() + } + + fn get_list_matcher(&self, list: ListRef<'_>) -> &dyn ListMatcher { + self.state.get_list_matcher(list) + } + + fn get_list_matcher_unchecked(&self, list: &List) -> &dyn ListMatcher { + self.state.get_list_matcher_unchecked(list) + } + } + + crate::wirefilter_fields! { + impl<'data> NativeFields for NativeContext<'data> { + "name" => { + fn(context: &NativeContext<'data>) -> &str { + &context.request.name + } + }, + "numbers" => { + fn(context: &NativeContext<'data>) -> &Vec { + &context.request.numbers + } + }, + } + } + + #[test] + fn test_owned_state_can_move_to_worker_and_borrow_native_fields_there() { + let scheme = scheme(); + NativeFields::validate_scheme(&scheme).unwrap(); + let mut compiler = DefaultCompiler::::new(); + let filter = scheme + .parse(r#"name == "first" and any(numbers[*] in $numbers)"#) + .unwrap() + .compile_with_compiler(&mut compiler); + let value = scheme + .parse_value("name") + .unwrap() + .compile_with_compiler(&mut compiler); + let mut state = ContextState::new_with(&scheme, || 0usize); + insert(&mut state, 41); + let request = Request { + name: "first".into(), + numbers: vec![40, 41], + }; + let caller = std::thread::current().id(); + let (mut request, state, filter) = std::thread::spawn(move || { + assert_ne!(std::thread::current().id(), caller); + let context = NativeContext { + request: &request, + state: &state, + }; + assert!(filter.execute(&context).unwrap()); + let LhsValue::Bytes(bytes) = value.execute(&context).unwrap().unwrap() else { + panic!("expected bytes"); + }; + assert_eq!(bytes.as_ref().as_ptr(), request.name.as_ptr()); + assert_eq!(*context.get_user_data(), 1); + (request, state, filter) + }) + .join() + .unwrap(); + request.name = "second".into(); + assert!( + !filter + .execute(&NativeContext { + request: &request, + state: &state, + }) + .unwrap() + ); + assert_eq!(*state.get_user_data(), 1); + assert!( + state + .get_list_matcher_from_type(&Type::Int) + .unwrap() + .match_value("numbers", &LhsValue::Int(41)) + ); + } +} diff --git a/engine/src/execution_context.rs b/engine/src/execution_context.rs index fdb9eb05..c03dcbb5 100644 --- a/engine/src/execution_context.rs +++ b/engine/src/execution_context.rs @@ -1,5 +1,6 @@ +use crate::lhs_types::{LhsValue, LhsValueSeed}; use crate::scheme::{Field, List, Scheme, SchemeMismatchError}; -use crate::types::{GetType, LhsValue, LhsValueSeed, Type, TypeMismatchError}; +use crate::types::{GetType, Type, TypeMismatchError}; use crate::{FieldRef, ListMatcher, ListRef, UnknownFieldError}; use serde::Serialize; use serde::de::{self, DeserializeSeed, Deserializer, MapAccess, SeqAccess, Visitor}; @@ -38,6 +39,17 @@ pub struct InvalidListMatcherError { /// /// It acts as a map in terms of public API, but provides a constant-time /// index-based access to values for a filter during execution. +/// +/// The default [`crate::ExecutionContextFamily`] binds its fields using +/// [`crate::LhsValueProvider`]. It borrows existing values through the same +/// reader interface used for native Rust fields, without converting the stored +/// containers. Setters validate values against the field types in the scheme. +/// +/// This type implements [`crate::EvaluationContext`] and +/// [`crate::EvaluationContextMut`], owning its scheme, list matchers, and user data +/// alongside the field values. Custom contexts can use [`crate::ContextState`] +/// to own equivalent state without field-value slots. See [`crate::context`] for +/// the context interfaces and an example. #[derive(Debug, PartialEq)] pub struct ExecutionContext<'e, U = ()> { scheme: Scheme, @@ -46,6 +58,54 @@ pub struct ExecutionContext<'e, U = ()> { user_data: U, } +impl crate::HasScheme for ExecutionContext<'_, U> { + #[inline] + fn scheme(&self) -> &Scheme { + self.scheme() + } +} + +impl crate::EvaluationContext for ExecutionContext<'_, U> { + type UserData = U; + + #[inline] + fn get_user_data(&self) -> &U { + self.get_user_data() + } + + #[inline] + fn get_list_matcher(&self, list: ListRef<'_>) -> &dyn ListMatcher { + self.get_list_matcher(list) + } + + #[inline] + fn get_list_matcher_from_type(&self, ty: &Type) -> Option<&dyn ListMatcher> { + self.get_list_matcher_from_type(ty) + } + + #[inline] + fn get_list_matcher_unchecked(&self, list: &List) -> &dyn ListMatcher { + self.get_list_matcher_unchecked(list) + } +} + +impl crate::EvaluationContextMut for ExecutionContext<'_, U> { + #[inline] + fn get_user_data_mut(&mut self) -> &mut U { + self.get_user_data_mut() + } + + #[inline] + fn get_list_matcher_mut(&mut self, list: ListRef<'_>) -> &mut dyn ListMatcher { + self.get_list_matcher_mut(list) + } + + #[inline] + fn get_list_matcher_mut_from_type(&mut self, ty: &Type) -> Option<&mut dyn ListMatcher> { + self.get_list_matcher_mut_from_type(ty) + } +} + impl<'e, U> ExecutionContext<'e, U> { /// Creates an execution context associated with a given scheme. /// @@ -72,6 +132,25 @@ impl<'e, U> ExecutionContext<'e, U> { } } + /// Create an empty stored-value snapshot with a context's scheme and lists. + /// + /// Applications can use this explicitly for consumers that require stored + /// values, such as diagnostic serialization. Field getters are not evaluated; + /// the caller chooses which fields to materialize. List matchers are cloned + /// independently so later changes to either context do not affect the other. + pub fn new_from_context(context: &impl crate::EvaluationContext, user_data: U) -> Self { + let scheme = context.scheme(); + Self { + scheme: scheme.clone(), + values: vec![None; scheme.field_count()].into(), + list_matchers: scheme + .lists() + .map(|list| dyn_clone::clone_box(context.get_list_matcher(list))) + .collect(), + user_data, + } + } + /// Returns the associated scheme. pub fn scheme(&self) -> &Scheme { &self.scheme diff --git a/engine/src/filter.rs b/engine/src/filter.rs index 510672cf..849cac75 100644 --- a/engine/src/filter.rs +++ b/engine/src/filter.rs @@ -5,19 +5,27 @@ //! their `execute` methods and aggregating results into a single boolean value //! as recursion unwinds. -use crate::execution_context::ExecutionContext; -use crate::lhs_types::TypedArray; +use crate::context::{ContextFamily, ContextRef, ExecutionContextFamily, HasScheme}; +use crate::lhs_types::{LhsValue, TypedArray}; use crate::scheme::{Scheme, SchemeMismatchError}; -use crate::types::{LhsValue, Type}; +use crate::types::Type; use std::fmt; +use std::marker::PhantomData; -type BoxedClosureToOneBool = - Box Fn(&'e ExecutionContext<'e, U>) -> bool + Sync + Send + 'static>; +type BoxedClosureToOneBool = Box< + dyn for<'ctx, 'data> Fn(&'ctx ::Context<'data>) -> bool + + Sync + + Send + + 'static, +>; /// Boxed closure for [`crate::Expr`] AST node that evaluates to a simple [`bool`]. -pub struct CompiledOneExpr(BoxedClosureToOneBool); +pub struct CompiledOneExpr = ExecutionContextFamily>( + BoxedClosureToOneBool, + PhantomData U>, +); -impl fmt::Debug for CompiledOneExpr { +impl> fmt::Debug for CompiledOneExpr { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { fmt.debug_tuple("CompiledOneExpr") .field(&((&*self.0) as *const _)) @@ -25,35 +33,44 @@ impl fmt::Debug for CompiledOneExpr { } } -impl CompiledOneExpr { +impl> CompiledOneExpr { /// Creates a compiled expression IR from a generic closure. pub fn new( - closure: impl for<'e> Fn(&'e ExecutionContext<'e, U>) -> bool + Sync + Send + 'static, + closure: impl for<'ctx, 'data> Fn(&'ctx ::Context<'data>) -> bool + + Sync + + Send + + 'static, ) -> Self { - CompiledOneExpr(Box::new(closure)) + CompiledOneExpr(Box::new(closure), PhantomData) } /// Executes the closure against a provided context with values. - pub fn execute<'e>(&self, ctx: &'e ExecutionContext<'e, U>) -> bool { + pub fn execute<'ctx, 'data: 'ctx>(&self, ctx: &'ctx F::Context<'data>) -> bool { self.0(ctx) } /// Extracts the underlying boxed closure. - pub fn into_boxed_closure(self) -> BoxedClosureToOneBool { + pub fn into_boxed_closure(self) -> BoxedClosureToOneBool { self.0 } } pub(crate) type CompiledVecExprResult = TypedArray<'static, bool>; -type BoxedClosureToVecBool = Box< - dyn for<'e> Fn(&'e ExecutionContext<'e, U>) -> CompiledVecExprResult + Sync + Send + 'static, +type BoxedClosureToVecBool = Box< + dyn for<'ctx, 'data> Fn(&'ctx ::Context<'data>) -> CompiledVecExprResult + + Sync + + Send + + 'static, >; /// Boxed closure for [`crate::Expr`] AST node that evaluates to a list of [`bool`]. -pub struct CompiledVecExpr(BoxedClosureToVecBool); +pub struct CompiledVecExpr = ExecutionContextFamily>( + BoxedClosureToVecBool, + PhantomData U>, +); -impl fmt::Debug for CompiledVecExpr { +impl> fmt::Debug for CompiledVecExpr { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { fmt.debug_tuple("CompiledVecExpr") .field(&((&*self.0) as *const _)) @@ -61,40 +78,53 @@ impl fmt::Debug for CompiledVecExpr { } } -impl CompiledVecExpr { +impl> CompiledVecExpr { /// Creates a compiled expression IR from a generic closure. pub fn new( - closure: impl for<'e> Fn(&'e ExecutionContext<'e, U>) -> CompiledVecExprResult + closure: impl for<'ctx, 'data> Fn( + &'ctx ::Context<'data>, + ) -> CompiledVecExprResult + Sync + Send + 'static, ) -> Self { - CompiledVecExpr(Box::new(closure)) + CompiledVecExpr(Box::new(closure), PhantomData) } /// Executes the closure against a provided context with values. - pub fn execute<'e>(&self, ctx: &'e ExecutionContext<'e, U>) -> CompiledVecExprResult { + pub fn execute<'ctx, 'data: 'ctx>( + &self, + ctx: &'ctx F::Context<'data>, + ) -> CompiledVecExprResult { self.0(ctx) } /// Extracts the underlying boxed closure. - pub fn into_boxed_closure(self) -> BoxedClosureToVecBool { + pub fn into_boxed_closure(self) -> BoxedClosureToVecBool { self.0 } } /// Enum of boxed closure for [`crate::Expr`] AST nodes. -#[derive(Debug)] -pub enum CompiledExpr { +pub enum CompiledExpr = ExecutionContextFamily> { /// Variant for [`crate::Expr`] AST node that evaluates to a simple [`bool`]. - One(CompiledOneExpr), + One(CompiledOneExpr), /// Variant for [`crate::Expr`] AST node that evaluates to a list of [`bool`]. - Vec(CompiledVecExpr), + Vec(CompiledVecExpr), } -impl CompiledExpr { +impl> fmt::Debug for CompiledExpr { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::One(expr) => fmt.debug_tuple("One").field(expr).finish(), + Self::Vec(expr) => fmt.debug_tuple("Vec").field(expr).finish(), + } + } +} + +impl> CompiledExpr { #[cfg(test)] - pub(crate) fn execute_one<'e>(&self, ctx: &'e ExecutionContext<'e, U>) -> bool { + pub(crate) fn execute_one<'ctx, 'data: 'ctx>(&self, ctx: &'ctx F::Context<'data>) -> bool { match self { CompiledExpr::One(one) => one.execute(ctx), CompiledExpr::Vec(_) => unreachable!(), @@ -102,9 +132,9 @@ impl CompiledExpr { } #[cfg(test)] - pub(crate) fn execute_vec<'e>( + pub(crate) fn execute_vec<'ctx, 'data: 'ctx>( &self, - ctx: &'e ExecutionContext<'e, U>, + ctx: &'ctx F::Context<'data>, ) -> CompiledVecExprResult { match self { CompiledExpr::One(_) => unreachable!(), @@ -127,14 +157,24 @@ impl From for CompiledValueResult<'_> { } } -type BoxedClosureToValue = Box< - dyn for<'e> Fn(&'e ExecutionContext<'e, U>) -> CompiledValueResult<'e> + Sync + Send + 'static, +type BoxedClosureToValue = Box< + dyn for<'ctx, 'data> Fn(ContextRef<'ctx, 'data, F>) -> CompiledValueResult<'ctx> + + Sync + + Send + + 'static, +>; + +type BoxedStoredClosureToValue = Box< + dyn for<'e> Fn(&'e crate::ExecutionContext<'e, U>) -> CompiledValueResult<'e> + Sync + Send, >; /// Boxed closure for [`crate::ValueExpr`] AST node that evaluates to an [`LhsValue`]. -pub struct CompiledValueExpr(BoxedClosureToValue); +pub struct CompiledValueExpr = ExecutionContextFamily>( + BoxedClosureToValue, + PhantomData U>, +); -impl fmt::Debug for CompiledValueExpr { +impl> fmt::Debug for CompiledValueExpr { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { fmt.debug_tuple("CompiledValueExpr") .field(&((&*self.0) as *const _)) @@ -142,28 +182,57 @@ impl fmt::Debug for CompiledValueExpr { } } -impl CompiledValueExpr { +impl> CompiledValueExpr { /// Creates a compiled expression IR from a generic closure. pub fn new( - closure: impl for<'e> Fn(&'e ExecutionContext<'e, U>) -> CompiledValueResult<'e> + closure: impl for<'ctx, 'data> Fn( + &'ctx ::Context<'data>, + ) -> CompiledValueResult<'ctx> + + Sync + + Send + + 'static, + ) -> Self { + Self::new_with_context(move |ctx| closure(ctx.context())) + } + + /// Create a value expression that can borrow context payloads. + /// + /// The callback's witness records that the context's retained data outlives + /// this evaluation. Call [`ContextRef::context`] to obtain the original + /// context reference before invoking field getters or child expressions. + pub fn new_with_context( + closure: impl for<'ctx, 'data> Fn(ContextRef<'ctx, 'data, F>) -> CompiledValueResult<'ctx> + Sync + Send + 'static, ) -> Self { - CompiledValueExpr(Box::new(closure)) + CompiledValueExpr(Box::new(closure), PhantomData) } /// Executes the closure against a provided context with values. - pub fn execute<'e>(&self, ctx: &'e ExecutionContext<'e, U>) -> CompiledValueResult<'e> { - self.0(ctx) + pub fn execute<'ctx, 'data: 'ctx>( + &self, + ctx: &'ctx F::Context<'data>, + ) -> CompiledValueResult<'ctx> { + self.0(ContextRef::new(ctx)) } - /// Extracts the underlying boxed closure. - pub fn into_boxed_closure(self) -> BoxedClosureToValue { + /// Extract the callback, retaining its context-lifetime witness. + pub fn into_boxed_context_closure(self) -> BoxedClosureToValue { self.0 } } +impl CompiledValueExpr { + /// Convert to a callback accepting the ordinary stored execution context. + /// + /// This compatibility adapter adds one boxed closure around the expression. + /// [`Self::into_boxed_context_closure`] extracts the original callback directly. + pub fn into_boxed_closure(self) -> BoxedStoredClosureToValue { + Box::new(move |ctx| self.execute(ctx)) + } +} + /// An IR for a compiled filter expression. /// /// Currently it works by creating and combining boxed untyped closures and @@ -179,12 +248,12 @@ impl CompiledValueExpr { /// In the future the underlying representation might change, but for now it /// provides the best trade-off between safety and performance of compilation /// and execution. -pub struct Filter { - root_expr: CompiledOneExpr, +pub struct Filter = ExecutionContextFamily> { + root_expr: CompiledOneExpr, scheme: Scheme, } -impl std::fmt::Debug for Filter { +impl> std::fmt::Debug for Filter { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("Filter") .field("root", &self.root_expr) @@ -193,16 +262,16 @@ impl std::fmt::Debug for Filter { } } -impl Filter { +impl> Filter { /// Creates a compiled expression IR from a generic closure. - pub(crate) fn new(root_expr: CompiledOneExpr, scheme: Scheme) -> Self { + pub(crate) fn new(root_expr: CompiledOneExpr, scheme: Scheme) -> Self { Filter { root_expr, scheme } } /// Executes a compiled filter expression against a provided context with values. - pub fn execute<'e>( + pub fn execute<'ctx, 'data: 'ctx>( &self, - ctx: &'e ExecutionContext<'e, U>, + ctx: &'ctx F::Context<'data>, ) -> Result { if ctx.scheme() == &self.scheme { Ok(self.root_expr.execute(ctx)) @@ -213,22 +282,22 @@ impl Filter { } /// An IR for a compiled value expression. -pub struct FilterValue { - root_expr: CompiledValueExpr, +pub struct FilterValue = ExecutionContextFamily> { + root_expr: CompiledValueExpr, scheme: Scheme, } -impl FilterValue { +impl> FilterValue { /// Creates a compiled expression IR from a generic closure. - pub(crate) fn new(root_expr: CompiledValueExpr, scheme: Scheme) -> Self { + pub(crate) fn new(root_expr: CompiledValueExpr, scheme: Scheme) -> Self { FilterValue { root_expr, scheme } } /// Executes a compiled value expression against a provided context with values. - pub fn execute<'e>( + pub fn execute<'ctx, 'data: 'ctx>( &self, - ctx: &'e ExecutionContext<'e, U>, - ) -> Result, Type>, SchemeMismatchError> { + ctx: &'ctx F::Context<'data>, + ) -> Result, Type>, SchemeMismatchError> { if ctx.scheme() == &self.scheme { Ok(self.root_expr.execute(ctx)) } else { @@ -260,4 +329,70 @@ mod tests { is_send::>>(); is_sync::>>(); } + + #[test] + fn test_distinct_context_and_data_lifetimes() { + use crate::{ + CompiledOneExpr, CompiledValueExpr, ContextFamily, EvaluationContext, Field, + FieldBindingError, FieldConsumer, HasScheme, LhsValue, ListMatcher, ListRef, Scheme, + }; + use std::cell::Cell; + + struct Context<'data> { + scheme: &'data Scheme, + value: Cell<&'data str>, + } + + impl HasScheme for Context<'_> { + fn scheme(&self) -> &Scheme { + self.scheme + } + } + + impl EvaluationContext for Context<'_> { + type UserData = (); + + fn get_user_data(&self) -> &() { + &() + } + + fn get_list_matcher(&self, _: ListRef<'_>) -> &dyn ListMatcher { + unreachable!("the test scheme has no lists") + } + } + + struct Family; + + impl ContextFamily for Family { + type Context<'data> = Context<'data>; + type UserData = (); + + fn bind>( + field: Field, + _: B, + ) -> Result { + Err(FieldBindingError::UnknownField(field.name().into())) + } + } + + fn assert_send_sync() {} + assert_send_sync::>(); + assert_send_sync::>(); + + let value = + CompiledValueExpr::<(), Family>::new(|ctx| Ok(LhsValue::Bytes(ctx.value.get().into()))); + let matches = CompiledOneExpr::<(), Family>::new(|ctx| ctx.value.get() == "first"); + let first = String::from("first"); + let second = String::from("second"); + let scheme = crate::SchemeBuilder::new().build(); + let mut ctx = Context { + scheme: &scheme, + value: Cell::new(&first), + }; + assert_eq!(value.execute(&ctx).unwrap(), LhsValue::from("first")); + assert!(matches.execute(&ctx)); + *ctx.value.get_mut() = &second; + assert_eq!(value.execute(&ctx).unwrap(), LhsValue::from("second")); + assert!(!matches.execute(&ctx)); + } } diff --git a/engine/src/functions/mod.rs b/engine/src/functions/mod.rs index d180b214..ccf943de 100644 --- a/engine/src/functions/mod.rs +++ b/engine/src/functions/mod.rs @@ -3,8 +3,9 @@ pub(crate) mod concat; pub use self::concat::ConcatFunction; use crate::ParserSettings; use crate::filter::CompiledValueResult; +use crate::lhs_types::LhsValue; use crate::types::{ - ExpectedType, ExpectedTypeList, GetType, LhsValue, LiteralValue, Type, TypeMismatchError, + ExpectedType, ExpectedTypeList, GetType, LiteralValue, Type, TypeMismatchError, }; use std::any::Any; use std::convert::TryFrom; diff --git a/engine/src/lhs_types/array.rs b/engine/src/lhs_types/array.rs index 341961ee..16ee2eeb 100644 --- a/engine/src/lhs_types/array.rs +++ b/engine/src/lhs_types/array.rs @@ -1,9 +1,7 @@ use super::TypedMap; use super::map::InnerMap; -use crate::lhs_types::AsRefIterator; -use crate::types::{ - CompoundType, GetType, IntoValue, LhsValue, LhsValueSeed, Type, TypeMismatchError, -}; +use crate::lhs_types::{AsRefIterator, IntoValue, LhsValue, LhsValueSeed}; +use crate::types::{CompoundType, GetType, Type, TypeMismatchError}; #[cfg(feature = "get-size2")] use get_size2::{GetSize, GetSizeTracker}; use serde::de::{self, DeserializeSeed, Deserializer, SeqAccess, Visitor}; @@ -481,6 +479,15 @@ impl<'a, V: IntoValue<'a>> TypedArray<'a, V> { } } + /// Creates an empty typed array with space for at least `capacity` values. + #[inline] + pub fn with_capacity(capacity: usize) -> Self { + Self { + array: InnerArray::Owned(Vec::with_capacity(capacity)), + _marker: std::marker::PhantomData, + } + } + #[inline] fn as_vec_ref(&self) -> &Vec> { match &self.array { @@ -608,6 +615,8 @@ impl<'a, V: IntoValue<'a>> FromIterator for TypedArray<'a, V> { } } +impl<'a, V: IntoValue<'a>> super::private::SealedIntoValue for TypedArray<'a, V> {} + impl<'a, V: IntoValue<'a>> IntoValue<'a> for TypedArray<'a, V> { const TYPE: Type = Type::Array(CompoundType::from_type(V::TYPE)); diff --git a/engine/src/lhs_types/bytes.rs b/engine/src/lhs_types/bytes.rs index 1e07dd0a..f60fd9d0 100644 --- a/engine/src/lhs_types/bytes.rs +++ b/engine/src/lhs_types/bytes.rs @@ -1,3 +1,5 @@ +use crate::LhsValue; +use crate::values::{Bytes as BytesType, BytesRead, IntoLhsValue, Value}; #[cfg(feature = "get-size2")] use get_size2::{GetSize, GetSizeTracker}; use serde::de::Visitor; @@ -25,6 +27,18 @@ impl GetSize for Bytes<'_> { } } +impl Value for Bytes<'_> { + type Read = BytesRead; + type ValueType = BytesType; +} + +impl<'a, 'data: 'a> IntoLhsValue<'a> for Bytes<'data> { + #[inline] + fn into_lhs(self) -> LhsValue<'a> { + LhsValue::Bytes(self) + } +} + impl<'a> Bytes<'a> { /// Clones self into a fully owned byte string. #[inline] @@ -254,6 +268,20 @@ impl PartialEq for Bytes<'_> { impl Eq for Bytes<'_> {} +impl PartialOrd for Bytes<'_> { + #[inline] + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for Bytes<'_> { + #[inline] + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.deref().cmp(other.deref()) + } +} + impl Hash for Bytes<'_> { #[inline] fn hash(&self, h: &mut H) { @@ -351,6 +379,26 @@ impl<'de> Deserialize<'de> for Bytes<'de> { mod tests { use super::*; + #[test] + fn test_byte_order_is_independent_of_storage() { + use std::collections::BTreeMap; + + let mut values = BTreeMap::new(); + values.insert(Bytes::from(b"b".to_vec()), 1); + values.insert(Bytes::from(&b"a"[..]), 2); + values.insert(Bytes::from(&b"a\0"[..]), 3); + values.insert(Bytes::from(vec![0xff]), 4); + values.insert(Bytes::from(b"a".to_vec()), 5); + + assert_eq!(values.len(), 4); + assert_eq!(values.get(&b"a"[..]), Some(&5)); + assert_eq!(values.get(&b"b"[..]), Some(&1)); + assert_eq!( + values.keys().map(AsRef::as_ref).collect::>(), + [&b"a"[..], &b"a\0"[..], &b"b"[..], &[0xff][..]], + ); + } + #[test] fn test_bytes_deserialize() { let bytes = serde_json::from_str::>("\"a JSON string with unicode ❤\"").unwrap(); diff --git a/engine/src/lhs_types/map.rs b/engine/src/lhs_types/map.rs index 8e9ed581..7e08bd46 100644 --- a/engine/src/lhs_types/map.rs +++ b/engine/src/lhs_types/map.rs @@ -1,8 +1,8 @@ use super::TypedArray; use super::array::InnerArray; use crate::TypeMismatchError; -use crate::lhs_types::{AsRefIterator, Bytes}; -use crate::types::{CompoundType, GetType, IntoValue, LhsValue, LhsValueSeed, Type}; +use crate::lhs_types::{AsRefIterator, Bytes, IntoValue, LhsValue, LhsValueSeed}; +use crate::types::{CompoundType, GetType, Type}; #[cfg(feature = "get-size2")] use get_size2::{GetSize, GetSizeTracker}; use serde::de::{self, DeserializeSeed, Deserializer, MapAccess, SeqAccess, Visitor}; @@ -631,6 +631,8 @@ impl<'a, V: IntoValue<'a>> FromIterator<(Box<[u8]>, V)> for TypedMap<'a, V> { } } +impl<'a, V: IntoValue<'a>> super::private::SealedIntoValue for TypedMap<'a, V> {} + impl<'a, V: IntoValue<'a>> IntoValue<'a> for TypedMap<'a, V> { const TYPE: Type = Type::Map(CompoundType::from_type(V::TYPE)); diff --git a/engine/src/lhs_types/mod.rs b/engine/src/lhs_types/mod.rs index 2f4c6c3b..68a5ecf9 100644 --- a/engine/src/lhs_types/mod.rs +++ b/engine/src/lhs_types/mod.rs @@ -1,3 +1,5 @@ +//! Values stored by execution contexts and returned by value expressions. + mod array; mod bytes; mod map; @@ -5,7 +7,519 @@ mod map; pub use self::array::{Array, ArrayIntoIter, ArrayIter, TypedArray}; pub use self::bytes::Bytes; pub use self::map::{Map, MapIter, MapValuesIntoIter, TypedMap}; -use crate::types::LhsValue; +use crate::scheme::{FieldIndex, IndexAccessError}; +use crate::strict_partial_ord::StrictPartialOrd; +use crate::types::{ + ExpectedType, GetType, LiteralValue, Type, TypeMismatchError, declare_types, + specialized_get_type, specialized_try_from, +}; +use crate::values::{FieldValueType, IntoLhsValue, ReadValue, Value, ValueError, ValuePredicate}; +#[cfg(feature = "get-size2")] +use get_size2::{GetSize, GetSizeTracker}; +use serde::de::{DeserializeSeed, Deserializer}; +use serde::{Deserialize, Serialize, Serializer}; +use std::cmp::Ordering; +use std::fmt::{self, Debug, Formatter}; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; + +macro_rules! declare_lhs_value { + ($($(# $attrs:tt)* $name:ident $([$val_ty:ty])? ( $(# $lhs_attrs:tt)* $lhs_ty:ty | $literal_ty:ty | $literal_set_ty:ty ) , )*) => { + declare_types! { + /// An LHS value provided for filter execution. + /// + /// These are passed to the [`crate::ExecutionContext`] + /// and are used by [`crate::Filter`] for execution and comparisons. + #[derive(PartialEq, Eq, Clone, Deserialize, Hash)] + #[serde(untagged)] + enum LhsValue<'a> { + $($(# $attrs)* $(# $lhs_attrs)* $name($lhs_ty),)* + } + } + + $(impl<'a> TryFrom> for $lhs_ty { + type Error = TypeMismatchError; + + fn try_from(value: LhsValue<'a>) -> Result<$lhs_ty, TypeMismatchError> { + match value { + LhsValue::$name(value) => Ok(value), + _ => Err(TypeMismatchError { + expected: specialized_try_from!($name).into(), + actual: value.get_type(), + }), + } + } + })* + + $(impl<'a> TryFrom<&'a LhsValue<'a>> for &'a $lhs_ty { + type Error = TypeMismatchError; + + fn try_from(value: &'a LhsValue<'a>) -> Result<&'a $lhs_ty, TypeMismatchError> { + match value { + LhsValue::$name(value) => Ok(value), + _ => Err(TypeMismatchError { + expected: specialized_try_from!($name).into(), + actual: value.get_type(), + }), + } + } + })* + + impl<'a> PartialOrd for LhsValue<'a> { + fn partial_cmp(&self, other: &LiteralValue) -> Option { + match (self, other) { + $((LhsValue::$name(lhs), LiteralValue::$name(rhs)) => { + lhs.strict_partial_cmp(rhs) + },)* + _ => None, + } + } + } + + }; +} + +crate::types::with_value_types!(declare_lhs_value); + +impl PartialEq<&LhsValue<'_>> for LhsValue<'_> { + fn eq(&self, other: &&LhsValue<'_>) -> bool { + self.eq(*other) + } +} + +impl StrictPartialOrd for LhsValue<'_> {} + +impl PartialEq for LhsValue<'_> { + fn eq(&self, other: &LiteralValue) -> bool { + self.strict_partial_cmp(other) == Some(Ordering::Equal) + } +} + +mod private { + use crate::Bytes; + use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; + + pub trait SealedIntoValue {} + + impl SealedIntoValue for bool {} + + impl SealedIntoValue for i8 {} + impl SealedIntoValue for u8 {} + impl SealedIntoValue for i16 {} + impl SealedIntoValue for u16 {} + impl SealedIntoValue for i32 {} + impl SealedIntoValue for u32 {} + impl SealedIntoValue for i64 {} + + impl SealedIntoValue for IpAddr {} + impl SealedIntoValue for Ipv4Addr {} + impl SealedIntoValue for Ipv6Addr {} + + impl<'a, T> SealedIntoValue for T where Bytes<'a>: From {} +} + +/// Converts a value into an `LhsValue`. +/// It is a stronger version of `Into>` in that +/// any value of the input type will *always* convert to the +/// same statically known `LhsValue` variant. +pub trait IntoValue<'a>: private::SealedIntoValue { + const TYPE: Type; + + fn into_value(self) -> LhsValue<'a>; +} + +impl<'a> IntoValue<'a> for bool { + const TYPE: Type = Type::Bool; + + #[inline] + fn into_value(self) -> LhsValue<'a> { + LhsValue::Bool(self) + } +} + +impl<'a> IntoValue<'a> for i64 { + const TYPE: Type = Type::Int; + + #[inline] + fn into_value(self) -> LhsValue<'a> { + LhsValue::Int(self) + } +} + +impl<'a> IntoValue<'a> for i32 { + const TYPE: Type = Type::Int; + + #[inline] + fn into_value(self) -> LhsValue<'a> { + LhsValue::Int(i64::from(self)) + } +} + +impl<'a> IntoValue<'a> for i16 { + const TYPE: Type = Type::Int; + + #[inline] + fn into_value(self) -> LhsValue<'a> { + LhsValue::Int(i64::from(self)) + } +} + +impl<'a> IntoValue<'a> for u16 { + const TYPE: Type = Type::Int; + + #[inline] + fn into_value(self) -> LhsValue<'a> { + LhsValue::Int(i64::from(self)) + } +} + +impl<'a> IntoValue<'a> for i8 { + const TYPE: Type = Type::Int; + + #[inline] + fn into_value(self) -> LhsValue<'a> { + LhsValue::Int(i64::from(self)) + } +} + +impl<'a> IntoValue<'a> for u8 { + const TYPE: Type = Type::Int; + + #[inline] + fn into_value(self) -> LhsValue<'a> { + LhsValue::Int(i64::from(self)) + } +} + +impl<'a> IntoValue<'a> for IpAddr { + const TYPE: Type = Type::Ip; + + #[inline] + fn into_value(self) -> LhsValue<'a> { + LhsValue::Ip(self) + } +} + +impl<'a> IntoValue<'a> for Ipv4Addr { + const TYPE: Type = Type::Ip; + + #[inline] + fn into_value(self) -> LhsValue<'a> { + LhsValue::Ip(IpAddr::V4(self)) + } +} + +impl<'a> IntoValue<'a> for Ipv6Addr { + const TYPE: Type = Type::Ip; + + #[inline] + fn into_value(self) -> LhsValue<'a> { + LhsValue::Ip(IpAddr::V6(self)) + } +} + +impl<'a, T> IntoValue<'a> for T +where + Bytes<'a>: From, +{ + const TYPE: Type = Type::Bytes; + + #[inline] + fn into_value(self) -> LhsValue<'a> { + LhsValue::Bytes(Bytes::from(self)) + } +} + +impl<'a, T: IntoValue<'a>> From for LhsValue<'a> { + #[inline] + fn from(value: T) -> Self { + value.into_value() + } +} + +// Array cannot implement `IntoValue` as the +// underlying element type is not statically +// known. +impl<'a> From> for LhsValue<'a> { + #[inline] + fn from(value: Array<'a>) -> LhsValue<'a> { + LhsValue::Array(value) + } +} + +// Map cannot implement `IntoValue` as the +// underlying element type is not statically +// known. +impl<'a> From> for LhsValue<'a> { + #[inline] + fn from(value: Map<'a>) -> LhsValue<'a> { + LhsValue::Map(value) + } +} + +impl<'a> TryFrom<&'a LhsValue<'a>> for &'a [u8] { + type Error = TypeMismatchError; + + fn try_from(value: &'a LhsValue<'_>) -> Result { + match value { + LhsValue::Bytes(value) => Ok(value), + _ => Err(TypeMismatchError { + expected: Type::Bytes.into(), + actual: value.get_type(), + }), + } + } +} + +impl<'a> From<&'a LiteralValue> for LhsValue<'a> { + fn from(literal: &'a LiteralValue) -> Self { + match literal { + LiteralValue::Ip(ip) => LhsValue::Ip(*ip), + LiteralValue::Bytes(bytes) => LhsValue::Bytes(Bytes::Borrowed(bytes)), + LiteralValue::Int(integer) => LhsValue::Int(*integer), + LiteralValue::Bool(b) => match *b {}, + LiteralValue::Array(a) => match *a {}, + LiteralValue::Map(m) => match *m {}, + } + } +} + +impl From for LhsValue<'_> { + fn from(literal: LiteralValue) -> Self { + match literal { + LiteralValue::Ip(ip) => LhsValue::Ip(ip), + LiteralValue::Bytes(bytes) => LhsValue::Bytes(Bytes::Owned(bytes.into())), + LiteralValue::Int(integer) => LhsValue::Int(integer), + LiteralValue::Bool(b) => match b {}, + LiteralValue::Array(a) => match a {}, + LiteralValue::Map(m) => match m {}, + } + } +} + +impl<'a> LhsValue<'a> { + /// Converts a reference to an LhsValue to an LhsValue with an internal + /// references + pub fn as_ref(&'a self) -> Self { + match self { + LhsValue::Ip(ip) => LhsValue::Ip(*ip), + LhsValue::Bytes(bytes) => LhsValue::Bytes(Bytes::Borrowed(bytes)), + LhsValue::Int(integer) => LhsValue::Int(*integer), + LhsValue::Bool(b) => LhsValue::Bool(*b), + LhsValue::Array(a) => LhsValue::Array(a.as_ref()), + LhsValue::Map(m) => LhsValue::Map(m.as_ref()), + } + } + + /// Converts an `LhsValue` with borrowed data to a fully owned `LhsValue`. + pub fn into_owned(self) -> LhsValue<'static> { + match self { + LhsValue::Ip(ip) => LhsValue::Ip(ip), + LhsValue::Bytes(bytes) => LhsValue::Bytes(Bytes::Owned(bytes.into_owned())), + LhsValue::Int(i) => LhsValue::Int(i), + LhsValue::Bool(b) => LhsValue::Bool(b), + LhsValue::Array(arr) => LhsValue::Array(arr.into_owned()), + LhsValue::Map(map) => LhsValue::Map(map.into_owned()), + } + } + + /// Retrieve an element from an LhsValue given a path item and a specified + /// type. + /// Returns a TypeMismatchError error if current type does not support it + /// nested element. + /// + /// Both LhsValue::Array and LhsValue::Map support nested elements. + pub(crate) fn get( + &'a self, + item: &FieldIndex, + ) -> Result>, IndexAccessError> { + match (self, item) { + (LhsValue::Array(arr), FieldIndex::ArrayIndex(idx)) => Ok(arr.get(*idx as usize)), + (_, FieldIndex::ArrayIndex(_)) => Err(IndexAccessError { + index: item.clone(), + actual: self.get_type(), + }), + (LhsValue::Map(map), FieldIndex::MapKey(key)) => Ok(map.get(key.as_bytes())), + (_, FieldIndex::MapKey(_)) => Err(IndexAccessError { + index: item.clone(), + actual: self.get_type(), + }), + (_, FieldIndex::MapEach) => Err(IndexAccessError { + index: item.clone(), + actual: self.get_type(), + }), + } + } + + #[inline] + pub(crate) fn get_nested(&'a self, indexes: &[FieldIndex]) -> Option<&'a LhsValue<'a>> { + indexes + .iter() + .try_fold(self, |value, idx| value.get(idx).unwrap()) + } + + pub(crate) fn extract( + self, + item: &FieldIndex, + ) -> Result>, IndexAccessError> { + match item { + FieldIndex::ArrayIndex(idx) => match self { + LhsValue::Array(arr) => Ok(arr.extract(*idx as usize)), + _ => Err(IndexAccessError { + index: item.clone(), + actual: self.get_type(), + }), + }, + FieldIndex::MapKey(key) => match self { + LhsValue::Map(map) => Ok(map.extract(key.as_bytes())), + _ => Err(IndexAccessError { + index: item.clone(), + actual: self.get_type(), + }), + }, + FieldIndex::MapEach => Err(IndexAccessError { + index: item.clone(), + actual: self.get_type(), + }), + } + } + + #[inline] + pub(crate) fn extract_nested(self, indexes: &[FieldIndex]) -> Option> { + indexes + .iter() + .try_fold(self, |value, idx| value.extract(idx).unwrap()) + } + + /// Returns an iterator over the Map or Array + pub(crate) fn iter(&'a self) -> Option> { + match self { + LhsValue::Array(array) => Some(Iter::IterArray(array.iter())), + LhsValue::Map(map) => Some(Iter::IterMap(map.iter())), + _ => None, + } + } +} + +impl Serialize for LhsValue<'_> { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + match self { + LhsValue::Ip(ip) => ip.serialize(serializer), + LhsValue::Bytes(bytes) => { + if let Ok(s) = std::str::from_utf8(bytes) { + serializer.serialize_str(s) + } else { + serializer.serialize_bytes(bytes) + } + } + LhsValue::Int(num) => num.serialize(serializer), + LhsValue::Bool(b) => b.serialize(serializer), + LhsValue::Array(arr) => arr.serialize(serializer), + LhsValue::Map(map) => map.serialize(serializer), + } + } +} + +pub(crate) struct LhsValueSeed<'a>(pub &'a Type); + +impl<'de> DeserializeSeed<'de> for LhsValueSeed<'_> { + type Value = LhsValue<'de>; + + fn deserialize(self, deserializer: D) -> Result + where + D: Deserializer<'de>, + { + match self.0 { + Type::Ip => Ok(LhsValue::Ip(std::net::IpAddr::deserialize(deserializer)?)), + Type::Int => Ok(LhsValue::Int(i64::deserialize(deserializer)?)), + Type::Bool => Ok(LhsValue::Bool(bool::deserialize(deserializer)?)), + Type::Bytes => Ok(LhsValue::Bytes(Bytes::deserialize(deserializer)?)), + Type::Array(ty) => Ok(LhsValue::Array({ + let mut arr = Array::new(*ty); + arr.deserialize(deserializer)?; + arr + })), + Type::Map(ty) => Ok(LhsValue::Map({ + let mut map = Map::new(*ty); + map.deserialize(deserializer)?; + map + })), + } + } +} + +pub enum IntoIter<'a> { + IntoArray(ArrayIntoIter<'a>), + IntoMap(MapValuesIntoIter<'a>), +} + +impl<'a> Iterator for IntoIter<'a> { + type Item = LhsValue<'a>; + + fn next(&mut self) -> Option> { + match self { + IntoIter::IntoArray(array) => array.next(), + IntoIter::IntoMap(map) => map.next(), + } + } + + fn size_hint(&self) -> (usize, Option) { + (self.len(), Some(self.len())) + } +} + +impl ExactSizeIterator for IntoIter<'_> { + fn len(&self) -> usize { + match self { + IntoIter::IntoArray(array) => array.len(), + IntoIter::IntoMap(map) => map.len(), + } + } +} + +impl<'a> IntoIterator for LhsValue<'a> { + type IntoIter = IntoIter<'a>; + type Item = LhsValue<'a>; + + fn into_iter(self) -> Self::IntoIter { + match self { + LhsValue::Array(array) => IntoIter::IntoArray(array.into_iter()), + LhsValue::Map(map) => IntoIter::IntoMap(map.into_values()), + _ => unreachable!(), + } + } +} + +pub(crate) enum Iter<'a> { + IterArray(ArrayIter<'a, 'a>), + IterMap(MapIter<'a, 'a>), +} + +impl<'a> Iterator for Iter<'a> { + type Item = &'a LhsValue<'a>; + + fn next(&mut self) -> Option<&'a LhsValue<'a>> { + match self { + Iter::IterArray(array) => array.next(), + Iter::IterMap(map) => map.next().map(|(_, v)| v), + } + } + + fn size_hint(&self) -> (usize, Option) { + (self.len(), Some(self.len())) + } +} + +impl ExactSizeIterator for Iter<'_> { + fn len(&self) -> usize { + match self { + Iter::IterArray(array) => array.len(), + Iter::IterMap(map) => map.len(), + } + } +} pub struct AsRefIterator<'a, T: Iterator>>(T); @@ -28,3 +542,376 @@ impl<'a, T: ExactSizeIterator>> ExactSizeIterator for As self.0.len() } } + +pub(crate) fn append_stored_lhs_at<'a>( + mut value: LhsValue<'a>, + mut indexes: &[FieldIndex], + output: &mut Vec>, +) { + while let Some((index, rest)) = indexes.split_first() { + if *index == FieldIndex::MapEach { + let values = value.into_iter(); + if rest.is_empty() { + output.extend(values); + } else { + for value in values { + append_stored_lhs_at(value, rest, output); + } + } + return; + } + let Some(selected) = value.extract(index).expect("invalid value selection path") else { + return; + }; + value = selected; + indexes = rest; + } + output.push(value); +} + +/// A stored field whose logical type is supplied by the scheme. +/// +/// Unlike a static [`super::values::ValueType`], this describes an entire field, +/// including arbitrary collection nesting, rather than a statically typed container element. +/// Contexts must check values against the declared type when inserting them, as +/// [`crate::ExecutionContext::set_field_value`] does. Startup validation cannot +/// infer the variant returned by an arbitrary getter of [`LhsValue`]. +pub struct LhsValueType; + +impl FieldValueType for LhsValueType { + type Path = Box<[FieldIndex]>; + + const DIRECT_COLLECTION_COMPARISON: bool = true; + const STATIC_TYPE: Option = None; + + fn compile_field_path( + mut field_type: Type, + path: Vec, + input_type: Type, + ) -> Result { + for step in &path { + field_type = match (field_type, step) { + (Type::Array(child), FieldIndex::ArrayIndex(_) | FieldIndex::MapEach) + | (Type::Map(child), FieldIndex::MapKey(_) | FieldIndex::MapEach) => child.into(), + _ => return Err(ValueError("index does not match the declared field type")), + }; + } + if field_type != input_type { + return Err(ValueError( + "predicate input type differs from selected value", + )); + } + Ok(path.into_boxed_slice()) + } +} + +/// Engine reader borrowing existing Wirefilter values and their containers. +pub struct LhsValueRead; + +impl LhsValueRead { + /// Compare a selected value while retaining the caller's value ownership. + pub(crate) fn compare_one( + value: &LhsValue<'_>, + path: &[FieldIndex], + predicate: &P, + ) -> Option { + value.get_nested(path).map(|value| predicate.lhs(value)) + } + + /// Compare selections from a possibly nested path. + pub(crate) fn compare_vec( + value: &LhsValue<'_>, + path: &[FieldIndex], + predicate: &P, + output: &mut TypedArray<'static, bool>, + ) { + compare_each(value, path, predicate, output); + } + + /// Compare each child of the collection selected by a concrete prefix. + /// Keeping the final iteration implicit avoids retaining another path entry. + pub(crate) fn compare_items( + value: &LhsValue<'_>, + prefix: &[FieldIndex], + predicate: &P, + output: &mut TypedArray<'static, bool>, + ) { + if let Some(value) = value.get_nested(prefix) { + compare_each(value, &[FieldIndex::MapEach], predicate, output); + } + } +} + +impl Value for LhsValue<'_> { + type Read = LhsValueRead; + type ValueType = LhsValueType; +} + +impl<'a, 'data: 'a> IntoLhsValue<'a> for LhsValue<'data> { + const MATERIALIZES_COLLECTIONS: bool = false; + + #[inline] + fn into_lhs(self) -> LhsValue<'a> { + self + } + + #[inline] + fn append_lhs_at(self, indexes: &[FieldIndex], output: &mut Vec>) { + append_stored_lhs_at(self, indexes, output); + } +} + +impl ReadValue, LhsValueType> for LhsValueRead { + const MATERIALIZES_COLLECTIONS: bool = false; + + #[inline] + fn to_lhs<'a>(value: &'a LhsValue<'_>) -> LhsValue<'a> { + value.as_ref() + } + + #[inline] + fn to_lhs_at<'a>(value: &'a LhsValue<'_>, indexes: &[FieldIndex]) -> Option> { + value.get_nested(indexes).map(LhsValue::as_ref) + } + + #[inline] + fn append_lhs_at<'a>( + value: &'a LhsValue<'_>, + indexes: &[FieldIndex], + output: &mut Vec>, + ) { + append_stored_lhs_at(value.as_ref(), indexes, output); + } + + #[inline] + fn compare_one( + value: &LhsValue<'_>, + path: &Box<[FieldIndex]>, + predicate: &P, + ) -> Option { + Self::compare_one(value, path, predicate) + } + + #[inline] + fn compare_vec( + value: &LhsValue<'_>, + path: &Box<[FieldIndex]>, + predicate: &P, + output: &mut TypedArray<'static, bool>, + ) { + Self::compare_vec(value, path, predicate, output); + } +} + +fn compare_each<'a, P: ValuePredicate + ?Sized>( + mut value: &'a LhsValue<'a>, + mut path: &[FieldIndex], + predicate: &P, + output: &mut TypedArray<'static, bool>, +) { + // Follow concrete indices while reading values, without first searching for + // [*]. Only collection iteration recurses; concrete indices add no calls. + while let Some((index, rest)) = path.split_first() { + if *index == FieldIndex::MapEach { + let children = value.iter().expect("[*] requires a collection"); + if rest.is_empty() { + if output.is_empty() { + *output = TypedArray::with_capacity(children.size_hint().0); + } + for child in children { + output.push(predicate.lhs(child)); + } + } else { + for child in children { + compare_each(child, rest, predicate, output); + } + } + return; + } + let Some(child) = value.get(index).unwrap() else { + return; + }; + value = child; + path = rest; + } + output.push(predicate.lhs(value)); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn validates_paths_against_the_scheme_type() { + let root = Type::map(Type::array(Type::Int)); + let path = vec![FieldIndex::MapKey("values".into()), FieldIndex::MapEach]; + assert_eq!( + LhsValueType::compile_field_path(root, path.clone(), Type::Int).unwrap(), + path.into_boxed_slice(), + ); + assert!(LhsValueType::compile_field_path(root, vec![], root).is_ok()); + assert!(LhsValueType::compile_field_path(root, vec![], Type::Int).is_err()); + assert!( + LhsValueType::compile_field_path( + root, + vec![FieldIndex::ArrayIndex(0)], + Type::array(Type::Int), + ) + .is_err() + ); + assert!( + LhsValueType::compile_field_path(Type::Int, vec![FieldIndex::MapEach], Type::Int,) + .is_err() + ); + assert!( + LhsValueType::compile_field_path( + Type::array(Type::Int), + vec![FieldIndex::MapKey("values".into())], + Type::Int, + ) + .is_err() + ); + } +} + +#[cfg(feature = "get-size2")] +impl GetSize for LhsValue<'_> { + fn get_heap_size_with_tracker(&self, tracker: T) -> (usize, T) { + match self { + Self::Bool(_) | Self::Int(_) | Self::Ip(_) => (0, tracker), + Self::Bytes(value) => value.get_heap_size_with_tracker(tracker), + Self::Array(value) => value.get_heap_size_with_tracker(tracker), + Self::Map(value) => value.get_heap_size_with_tracker(tracker), + } + } +} + +#[test] +fn test_lhs_value_deserialize() { + use std::str::FromStr; + + let ipv4: LhsValue<'_> = serde_json::from_str("\"127.0.0.1\"").unwrap(); + assert_eq!(ipv4, LhsValue::Ip(IpAddr::from_str("127.0.0.1").unwrap())); + + let ipv6: LhsValue<'_> = serde_json::from_str("\"::1\"").unwrap(); + assert_eq!(ipv6, LhsValue::Ip(IpAddr::from_str("::1").unwrap())); + + let bytes: LhsValue<'_> = serde_json::from_str("\"a JSON string with unicode ❤\"").unwrap(); + assert_eq!( + bytes, + LhsValue::from(&b"a JSON string with unicode \xE2\x9D\xA4"[..]) + ); + + let bytes = + serde_json::from_str::>("\"a JSON string with escaped-unicode \\u2764\"") + .unwrap(); + assert_eq!( + bytes, + LhsValue::from(&b"a JSON string with escaped-unicode \xE2\x9D\xA4"[..]) + ); + + let bytes: LhsValue<'_> = serde_json::from_str("\"1337\"").unwrap(); + assert_eq!(bytes, LhsValue::from(&b"1337"[..])); + + let integer: LhsValue<'_> = serde_json::from_str("1337").unwrap(); + assert_eq!(integer, LhsValue::Int(1337)); + + let b: LhsValue<'_> = serde_json::from_str("false").unwrap(); + assert_eq!(b, LhsValue::Bool(false)); +} + +#[test] +fn test_size_of_lhs_value() { + assert_eq!(std::mem::size_of::>(), 48); +} + +#[cfg(feature = "get-size2")] +#[test] +fn test_lhs_value_get_size() { + let stack_size = std::mem::size_of::>(); + + assert_eq!(LhsValue::Bool(false).get_size(), stack_size); + assert_eq!(LhsValue::Int(42).get_size(), stack_size); + assert_eq!( + LhsValue::Ip(IpAddr::V4(Ipv4Addr::LOCALHOST)).get_size(), + stack_size + ); + + let borrowed_bytes = LhsValue::Bytes(Bytes::Borrowed(b"borrowed")); + assert_eq!(borrowed_bytes.get_size(), stack_size); + + let owned_bytes = LhsValue::Bytes(Bytes::Owned(Box::from(&b"owned"[..]))); + assert_eq!(owned_bytes.get_size(), stack_size + b"owned".len()); +} + +#[cfg(feature = "get-size2")] +#[test] +fn test_array_lhs_value_get_size() { + let stack_size = std::mem::size_of::>(); + + let mut borrowed_values = Vec::with_capacity(4); + borrowed_values.push(LhsValue::Bytes(Bytes::Borrowed(b"borrowed"))); + let borrowed_values_capacity = borrowed_values.capacity(); + let owned_array_of_borrowed = Array::try_from_vec(Type::Bytes, borrowed_values).unwrap(); + assert_eq!( + LhsValue::Array(owned_array_of_borrowed.as_ref()).get_size(), + stack_size + ); + assert_eq!( + LhsValue::Array(owned_array_of_borrowed).get_size(), + stack_size + borrowed_values_capacity * stack_size + ); + + let mut owned_values = Vec::with_capacity(4); + owned_values.push(LhsValue::Bytes(Bytes::Owned(Box::from(&b"owned"[..])))); + let owned_values_capacity = owned_values.capacity(); + let owned_array_of_owned = Array::try_from_vec(Type::Bytes, owned_values).unwrap(); + assert_eq!( + LhsValue::Array(owned_array_of_owned.as_ref()).get_size(), + stack_size + ); + assert_eq!( + LhsValue::Array(owned_array_of_owned).get_size(), + stack_size + owned_values_capacity * stack_size + b"owned".len() + ); +} + +#[cfg(feature = "get-size2")] +#[test] +fn test_map_lhs_value_get_size() { + let stack_size = std::mem::size_of::>(); + + let owned_map_of_borrowed = Map::try_from_iter( + Type::Bytes, + [Ok::<_, TypeMismatchError>(( + Box::from(b"key".as_slice()), + Bytes::Borrowed(b"borrowed"), + ))], + ) + .unwrap(); + assert_eq!( + LhsValue::Map(owned_map_of_borrowed.as_ref()).get_size(), + stack_size + ); + assert_eq!( + LhsValue::Map(owned_map_of_borrowed).get_size(), + stack_size + std::mem::size_of::>() + b"key".len() + stack_size + ); + + let owned_map_of_owned = Map::try_from_iter( + Type::Bytes, + [Ok::<_, TypeMismatchError>(( + Box::from(b"key".as_slice()), + Bytes::Owned(Box::from(&b"owned"[..])), + ))], + ) + .unwrap(); + assert_eq!( + LhsValue::Map(owned_map_of_owned.as_ref()).get_size(), + stack_size + ); + assert_eq!( + LhsValue::Map(owned_map_of_owned).get_size(), + stack_size + std::mem::size_of::>() + b"key".len() + stack_size + b"owned".len() + ); +} diff --git a/engine/src/lib.rs b/engine/src/lib.rs index a16c005d..aa00c255 100644 --- a/engine/src/lib.rs +++ b/engine/src/lib.rs @@ -67,6 +67,7 @@ mod scheme; mod ast; mod compiler; +pub mod context; mod execution_context; mod filter; mod functions; @@ -78,6 +79,7 @@ mod rhs_types; mod searcher; mod strict_partial_ord; mod types; +pub mod values; pub use self::ast::field_expr::{ ComparisonExpr, ComparisonOpExpr, ComparisonRhs, IdentifierExpr, IntOp, IntRhs, OrderingOp, @@ -92,6 +94,16 @@ pub use self::ast::parse::{FilterParser, ParseError, ParserSettings}; pub use self::ast::visitor::{Visitor, VisitorMut}; pub use self::ast::{Expr, FilterAst, FilterValueAst, FilterValueExpr, ValueExpr}; pub use self::compiler::{Compiler, DefaultCompiler}; +// The exported field macro calls this helper from downstream crates. +#[doc(hidden)] +pub use self::context::fields::validate_field_definitions; +pub use self::context::state::ContextState; +pub use self::context::{ + BorrowedValueProvider, ContextFamily, ContextRef, EvaluationContext, EvaluationContextMut, + ExecutionContextFamily, FieldBindingError, FieldConsumer, FieldDefinitionError, FieldGetter, + FieldGetterFn, HasScheme, LhsValueProvider, OptionalFieldGetterResult, OwnedValueProvider, + ProviderFamily, RequiredFieldGetterFn, RequiredFieldGetterResult, SchemeValidationError, +}; pub use self::execution_context::{ ExecutionContext, ExecutionContextGuard, InvalidListMatcherError, SetFieldValueError, }; @@ -105,7 +117,7 @@ pub use self::functions::{ SimpleFunctionImpl, SimpleFunctionOptParam, SimpleFunctionParam, }; pub use self::lex::LexErrorKind; -pub use self::lhs_types::{Array, Bytes, Map, MapIter, TypedArray, TypedMap}; +pub use self::lhs_types::{Array, Bytes, LhsValue, Map, MapIter, TypedArray, TypedMap}; pub use self::list_matcher::{ AlwaysList, AlwaysListMatcher, ListDefinition, ListMatcher, NeverList, NeverListMatcher, }; @@ -123,6 +135,6 @@ pub use self::scheme::{ ListRef, Scheme, SchemeBuilder, SchemeMismatchError, UnknownFieldError, }; pub use self::types::{ - CompoundType, ExpectedType, ExpectedTypeList, GetType, LhsValue, LiteralSet, LiteralValue, - Type, TypeMismatchError, + CompoundType, ExpectedType, ExpectedTypeList, GetType, LiteralSet, LiteralValue, Type, + TypeMismatchError, }; diff --git a/engine/src/rhs_types/regex/imp_real.rs b/engine/src/rhs_types/regex/imp_real.rs index ec821ad8..5ab33bef 100644 --- a/engine/src/rhs_types/regex/imp_real.rs +++ b/engine/src/rhs_types/regex/imp_real.rs @@ -12,7 +12,23 @@ pub struct Regex { format: RegexFormat, } +/// Matching state retained by a compiled filter, without the source representation. +pub(crate) struct CompiledRegex(regex_automata::meta::Regex); + +impl CompiledRegex { + #[inline] + pub(super) fn is_match(&self, input: &[u8]) -> bool { + self.0.is_match(input) + } +} + impl Regex { + /// Move the existing matcher and its cache into the filter without retaining + /// the pattern and formatting used by the AST. + pub(crate) fn into_compiled(self) -> CompiledRegex { + CompiledRegex(self.regex) + } + /// Retrieves the syntax configuration that will be used to build the regex. #[inline] pub fn syntax_config() -> regex_automata::util::syntax::Config { diff --git a/engine/src/rhs_types/regex/imp_stub.rs b/engine/src/rhs_types/regex/imp_stub.rs index 117b4036..3cbf371a 100644 --- a/engine/src/rhs_types/regex/imp_stub.rs +++ b/engine/src/rhs_types/regex/imp_stub.rs @@ -10,7 +10,19 @@ pub struct Regex { format: RegexFormat, } +pub(crate) struct CompiledRegex; + +impl CompiledRegex { + pub(super) fn is_match(&self, _input: &[u8]) -> bool { + unimplemented!("Engine was built without regex support") + } +} + impl Regex { + pub(crate) fn into_compiled(self) -> CompiledRegex { + CompiledRegex + } + /// Creates a new dummy regex. pub fn new( pattern: impl Into, diff --git a/engine/src/rhs_types/regex/mod.rs b/engine/src/rhs_types/regex/mod.rs index 67b1918a..a36a6003 100644 --- a/engine/src/rhs_types/regex/mod.rs +++ b/engine/src/rhs_types/regex/mod.rs @@ -1,6 +1,6 @@ use crate::lex::{LexErrorKind, LexResult, LexWith, span}; use crate::rhs_types::bytes::lex_raw_string_as_str; -use crate::{Compare, ExecutionContext, FilterParser, LhsValue}; +use crate::{Compare, ContextFamily, FilterParser, LhsValue}; use cfg_if::cfg_if; use serde::{Serialize, Serializer}; use std::fmt::{self, Debug, Display, Formatter}; @@ -146,14 +146,34 @@ pub enum Error { Other(Box), } -impl Compare for Regex { +impl> Compare for Regex { #[inline] - fn compare<'e>(&self, value: &LhsValue<'e>, _: &'e ExecutionContext<'e, U>) -> bool { + fn compare<'e, 'data: 'e>(&self, value: &LhsValue<'e>, _: &'e F::Context<'data>) -> bool { self.is_match(match value { LhsValue::Bytes(bytes) => bytes, _ => unreachable!(), }) } + + #[inline] + fn compare_bytes<'e, 'data: 'e>(&self, value: &'e [u8], _: &'e F::Context<'data>) -> bool { + self.is_match(value) + } +} + +impl> Compare for CompiledRegex { + #[inline] + fn compare<'e, 'data: 'e>(&self, value: &LhsValue<'e>, _: &'e F::Context<'data>) -> bool { + self.is_match(match value { + LhsValue::Bytes(bytes) => bytes, + _ => unreachable!(), + }) + } + + #[inline] + fn compare_bytes<'e, 'data: 'e>(&self, value: &'e [u8], _: &'e F::Context<'data>) -> bool { + self.is_match(value) + } } #[cfg(test)] @@ -161,6 +181,53 @@ mod test { use super::*; use crate::{ParserSettings, SchemeBuilder}; + #[test] + fn test_compiled_filter_releases_regex_source() { + use crate::{ComparisonOpExpr, LogicalExpr, Type}; + use arcstr::ArcStr; + + let mut builder = SchemeBuilder::new(); + builder.add_field("text", Type::Bytes).unwrap(); + let scheme = builder.build(); + for literal in [r#""a.b""#, r###"r##"a.b"##"###] { + let expression = format!("text matches {literal}"); + let ast = scheme.parse(&expression).unwrap(); + let LogicalExpr::Comparison(comparison) = ast.expression() else { + unreachable!(); + }; + let ComparisonOpExpr::Matches(regex) = &comparison.op else { + unreachable!(); + }; + let pattern = regex.pattern().clone(); + assert_eq!(ArcStr::strong_count(&pattern), Some(2)); + let filter = ast.compile(); + assert_eq!(ArcStr::strong_count(&pattern), Some(1)); + + #[cfg(feature = "regex")] + { + let mut ctx = crate::ExecutionContext::new(&scheme); + for (value, expected) in [(b"a\xffb".as_slice(), true), (b"ac", false)] { + ctx.set_field_value_from_name("text", value).unwrap(); + assert_eq!(filter.execute(&ctx).unwrap(), expected); + } + } + drop(filter); + } + } + + #[cfg(not(feature = "regex"))] + #[test] + #[should_panic(expected = "Engine was built without regex support")] + fn test_compiled_regex_without_feature_panics_on_execution() { + let mut builder = SchemeBuilder::new(); + builder.add_field("text", crate::Type::Bytes).unwrap(); + let scheme = builder.build(); + let filter = scheme.parse(r#"text matches "a.b""#).unwrap().compile(); + let mut ctx = crate::ExecutionContext::new(&scheme); + ctx.set_field_value_from_name("text", "axb").unwrap(); + filter.execute(&ctx).unwrap(); + } + #[test] fn test() { let scheme = SchemeBuilder::new().build(); diff --git a/engine/src/scheme.rs b/engine/src/scheme.rs index 3e6b0343..0dc814a9 100644 --- a/engine/src/scheme.rs +++ b/engine/src/scheme.rs @@ -2149,3 +2149,72 @@ fn test_nil_not_equal_behavior_false() { assert_eq!(filter.execute(&ctx), Ok(false)); } + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + #[test] + fn test_filter_compilation_does_not_duplicate_field_ownership() { + let scheme = Scheme! { + number: Int, + numbers: Array(Int), + headers: Map(Bytes), + } + .build(); + let baseline = Arc::strong_count(&scheme.inner); + + for input in [ + "number == 42", + "numbers[0] == 42", + "any(numbers[*] == 42)", + "number == 42 or number == 43", + r#"headers["host"] == "example.com""#, + ] { + let ast = scheme.parse(input).unwrap(); + let parsed_owners = Arc::strong_count(&scheme.inner); + let filter = ast.compile(); + let compiled_owners = Arc::strong_count(&scheme.inner); + + // Fields own a Scheme reference. Moving the AST's fields into + // compiled getters must not retain additional field copies. + assert!( + compiled_owners <= parsed_owners, + "compiling {input:?} increased Scheme owners from {parsed_owners} to {compiled_owners}" + ); + drop(filter); + assert_eq!(Arc::strong_count(&scheme.inner), baseline, "{input:?}"); + } + } + + #[test] + fn test_value_compilation_does_not_duplicate_field_ownership() { + let scheme = Scheme! { + number: Int, + numbers: Array(Int), + headers: Map(Bytes), + } + .build(); + let baseline = Arc::strong_count(&scheme.inner); + + for input in [ + "number", + "numbers", + "numbers[0]", + r#"headers["host"]"#, + "any(numbers[*] == 42)", + ] { + let ast = scheme.parse_value(input).unwrap(); + let parsed_owners = Arc::strong_count(&scheme.inner); + let value = ast.compile(); + let compiled_owners = Arc::strong_count(&scheme.inner); + + assert!( + compiled_owners <= parsed_owners, + "compiling {input:?} increased Scheme owners from {parsed_owners} to {compiled_owners}" + ); + drop(value); + assert_eq!(Arc::strong_count(&scheme.inner), baseline, "{input:?}"); + } + } +} diff --git a/engine/src/searcher.rs b/engine/src/searcher.rs index 23846c71..24e8e3bb 100644 --- a/engine/src/searcher.rs +++ b/engine/src/searcher.rs @@ -1,12 +1,17 @@ -use crate::{Compare, ExecutionContext, LhsValue}; +use crate::{Compare, ContextFamily, LhsValue}; use memchr::memmem::{Finder, FinderBuilder}; use sliceslice::MemchrSearcher; pub struct EmptySearcher; -impl Compare for EmptySearcher { +impl> Compare for EmptySearcher { #[inline] - fn compare<'e>(&self, _: &LhsValue<'e>, _: &'e ExecutionContext<'e, U>) -> bool { + fn compare<'e, 'data: 'e>(&self, _: &LhsValue<'e>, _: &'e F::Context<'data>) -> bool { + true + } + + #[inline] + fn compare_bytes<'e, 'data: 'e>(&self, _: &'e [u8], _: &'e F::Context<'data>) -> bool { true } } @@ -20,9 +25,9 @@ impl MemmemSearcher { } } -impl Compare for MemmemSearcher { +impl> Compare for MemmemSearcher { #[inline] - fn compare<'e>(&self, value: &LhsValue<'e>, _: &'e ExecutionContext<'e, U>) -> bool { + fn compare<'e, 'data: 'e>(&self, value: &LhsValue<'e>, _: &'e F::Context<'data>) -> bool { self.0 .find(match value { LhsValue::Bytes(bytes) => bytes, @@ -30,14 +35,24 @@ impl Compare for MemmemSearcher { }) .is_some() } + + #[inline] + fn compare_bytes<'e, 'data: 'e>(&self, value: &'e [u8], _: &'e F::Context<'data>) -> bool { + self.0.find(value).is_some() + } } -impl Compare for MemchrSearcher { +impl> Compare for MemchrSearcher { #[inline] - fn compare<'e>(&self, value: &LhsValue<'e>, _: &'e ExecutionContext<'e, U>) -> bool { + fn compare<'e, 'data: 'e>(&self, value: &LhsValue<'e>, _: &'e F::Context<'data>) -> bool { self.search_in(match value { LhsValue::Bytes(bytes) => bytes, _ => unreachable!(), }) } + + #[inline] + fn compare_bytes<'e, 'data: 'e>(&self, value: &'e [u8], _: &'e F::Context<'data>) -> bool { + self.search_in(value) + } } diff --git a/engine/src/types.rs b/engine/src/types.rs index c31d65c3..7e593d7e 100644 --- a/engine/src/types.rs +++ b/engine/src/types.rs @@ -1,20 +1,15 @@ use crate::lex::{Lex, LexResult, LexWith, expect, skip_space}; -use crate::lhs_types::{Array, ArrayIntoIter, ArrayIter, Bytes, Map, MapIter, MapValuesIntoIter}; +use crate::lhs_types::{LhsValue, LhsValueSeed}; use crate::rhs_types::{ BytesExpr, IntRange, IpRange, UninhabitedArray, UninhabitedBool, UninhabitedMap, }; -use crate::scheme::{FieldIndex, IndexAccessError}; -use crate::strict_partial_ord::StrictPartialOrd; -#[cfg(feature = "get-size2")] -use get_size2::{GetSize, GetSizeTracker}; use serde::de::{DeserializeSeed, Deserializer}; use serde::{Deserialize, Serialize, Serializer}; -use std::cmp::Ordering; use std::collections::BTreeSet; use std::convert::TryFrom; use std::fmt::{self, Debug, Formatter}; use std::iter::once; -use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; +use std::net::IpAddr; use thiserror::Error; fn lex_literal_set<'i, T: Lex<'i>>(input: &'i str) -> LexResult<'i, Vec> { @@ -170,7 +165,8 @@ macro_rules! specialized_try_from { }; } -// This macro generates `Type`, `LhsValue`, `LiteralValue`, `LiteralSet`. +// This macro generates `Type`, `LiteralValue`, and `LiteralSet`. +// `lhs_types` generates `LhsValue` from the same `with_value_types` table. // // Before the parenthesis is the variant for the `Type` enum (`Type::Ip`). // First argument is the corresponding `LhsValue` variant (`LhsValue::Ip(IpAddr)`). @@ -216,46 +212,6 @@ macro_rules! declare_types { $($(# $attrs)* $name$(($val_ty))?,)* } - declare_types! { - /// An LHS value provided for filter execution. - /// - /// These are passed to the [`crate::ExecutionContext`] - /// and are used by [`crate::Filter`] for execution and comparisons. - #[derive(PartialEq, Eq, Clone, Deserialize, Hash)] - #[serde(untagged)] - enum LhsValue<'a> { - $($(# $attrs)* $(# $lhs_attrs)* $name($lhs_ty),)* - } - } - - $(impl<'a> TryFrom> for $lhs_ty { - type Error = TypeMismatchError; - - fn try_from(value: LhsValue<'a>) -> Result<$lhs_ty, TypeMismatchError> { - match value { - LhsValue::$name(value) => Ok(value), - _ => Err(TypeMismatchError { - expected: specialized_try_from!($name).into(), - actual: value.get_type(), - }), - } - } - })* - - $(impl<'a> TryFrom<&'a LhsValue<'a>> for &'a $lhs_ty { - type Error = TypeMismatchError; - - fn try_from(value: &'a LhsValue<'a>) -> Result<&'a $lhs_ty, TypeMismatchError> { - match value { - LhsValue::$name(value) => Ok(value), - _ => Err(TypeMismatchError { - expected: specialized_try_from!($name).into(), - actual: value.get_type(), - }), - } - } - })* - declare_types! { /// A literal value parsed from a filter string. #[derive(PartialEq, Eq, Clone, Hash, Serialize)] @@ -276,17 +232,6 @@ macro_rules! declare_types { } } - impl<'a> PartialOrd for LhsValue<'a> { - fn partial_cmp(&self, other: &LiteralValue) -> Option { - match (self, other) { - $((LhsValue::$name(lhs), LiteralValue::$name(rhs)) => { - lhs.strict_partial_cmp(rhs) - },)* - _ => None, - } - } - } - $(impl<'a> TryFrom for $literal_ty { type Error = TypeMismatchError; @@ -455,451 +400,6 @@ impl GetType for CompoundType { } } -impl PartialEq<&LhsValue<'_>> for LhsValue<'_> { - fn eq(&self, other: &&LhsValue<'_>) -> bool { - self.eq(*other) - } -} - -impl StrictPartialOrd for LhsValue<'_> {} - -impl PartialEq for LhsValue<'_> { - fn eq(&self, other: &LiteralValue) -> bool { - self.strict_partial_cmp(other) == Some(Ordering::Equal) - } -} - -mod private { - use super::IntoValue; - use crate::{Bytes, TypedArray, TypedMap}; - use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; - - pub trait SealedIntoValue {} - - impl SealedIntoValue for bool {} - - impl SealedIntoValue for i8 {} - impl SealedIntoValue for u8 {} - impl SealedIntoValue for i16 {} - impl SealedIntoValue for u16 {} - impl SealedIntoValue for i32 {} - impl SealedIntoValue for u32 {} - impl SealedIntoValue for i64 {} - - impl SealedIntoValue for IpAddr {} - impl SealedIntoValue for Ipv4Addr {} - impl SealedIntoValue for Ipv6Addr {} - - impl<'a, T> SealedIntoValue for T where Bytes<'a>: From {} - - impl<'a, V: IntoValue<'a>> SealedIntoValue for TypedArray<'a, V> {} - impl<'a, V: IntoValue<'a>> SealedIntoValue for TypedMap<'a, V> {} -} - -/// Converts a value into an `LhsValue`. -/// It is a stronger version of `Into>` in that -/// any value of the input type will *always* convert to the -/// same statically known `LhsValue` variant. -pub trait IntoValue<'a>: private::SealedIntoValue { - const TYPE: Type; - - fn into_value(self) -> LhsValue<'a>; -} - -impl<'a> IntoValue<'a> for bool { - const TYPE: Type = Type::Bool; - - #[inline] - fn into_value(self) -> LhsValue<'a> { - LhsValue::Bool(self) - } -} - -impl<'a> IntoValue<'a> for i64 { - const TYPE: Type = Type::Int; - - #[inline] - fn into_value(self) -> LhsValue<'a> { - LhsValue::Int(self) - } -} - -impl<'a> IntoValue<'a> for i32 { - const TYPE: Type = Type::Int; - - #[inline] - fn into_value(self) -> LhsValue<'a> { - LhsValue::Int(i64::from(self)) - } -} - -impl<'a> IntoValue<'a> for i16 { - const TYPE: Type = Type::Int; - - #[inline] - fn into_value(self) -> LhsValue<'a> { - LhsValue::Int(i64::from(self)) - } -} - -impl<'a> IntoValue<'a> for u16 { - const TYPE: Type = Type::Int; - - #[inline] - fn into_value(self) -> LhsValue<'a> { - LhsValue::Int(i64::from(self)) - } -} - -impl<'a> IntoValue<'a> for i8 { - const TYPE: Type = Type::Int; - - #[inline] - fn into_value(self) -> LhsValue<'a> { - LhsValue::Int(i64::from(self)) - } -} - -impl<'a> IntoValue<'a> for u8 { - const TYPE: Type = Type::Int; - - #[inline] - fn into_value(self) -> LhsValue<'a> { - LhsValue::Int(i64::from(self)) - } -} - -impl<'a> IntoValue<'a> for IpAddr { - const TYPE: Type = Type::Ip; - - #[inline] - fn into_value(self) -> LhsValue<'a> { - LhsValue::Ip(self) - } -} - -impl<'a> IntoValue<'a> for Ipv4Addr { - const TYPE: Type = Type::Ip; - - #[inline] - fn into_value(self) -> LhsValue<'a> { - LhsValue::Ip(IpAddr::V4(self)) - } -} - -impl<'a> IntoValue<'a> for Ipv6Addr { - const TYPE: Type = Type::Ip; - - #[inline] - fn into_value(self) -> LhsValue<'a> { - LhsValue::Ip(IpAddr::V6(self)) - } -} - -impl<'a, T> IntoValue<'a> for T -where - Bytes<'a>: From, -{ - const TYPE: Type = Type::Bytes; - - #[inline] - fn into_value(self) -> LhsValue<'a> { - LhsValue::Bytes(Bytes::from(self)) - } -} - -impl<'a, T: IntoValue<'a>> From for LhsValue<'a> { - #[inline] - fn from(value: T) -> Self { - value.into_value() - } -} - -// Array cannot implement `IntoValue` as the -// underlying element type is not statically -// known. -impl<'a> From> for LhsValue<'a> { - #[inline] - fn from(value: Array<'a>) -> LhsValue<'a> { - LhsValue::Array(value) - } -} - -// Map cannot implement `IntoValue` as the -// underlying element type is not statically -// known. -impl<'a> From> for LhsValue<'a> { - #[inline] - fn from(value: Map<'a>) -> LhsValue<'a> { - LhsValue::Map(value) - } -} - -impl<'a> TryFrom<&'a LhsValue<'a>> for &'a [u8] { - type Error = TypeMismatchError; - - fn try_from(value: &'a LhsValue<'_>) -> Result { - match value { - LhsValue::Bytes(value) => Ok(value), - _ => Err(TypeMismatchError { - expected: Type::Bytes.into(), - actual: value.get_type(), - }), - } - } -} - -impl<'a> From<&'a LiteralValue> for LhsValue<'a> { - fn from(literal: &'a LiteralValue) -> Self { - match literal { - LiteralValue::Ip(ip) => LhsValue::Ip(*ip), - LiteralValue::Bytes(bytes) => LhsValue::Bytes(Bytes::Borrowed(bytes)), - LiteralValue::Int(integer) => LhsValue::Int(*integer), - LiteralValue::Bool(b) => match *b {}, - LiteralValue::Array(a) => match *a {}, - LiteralValue::Map(m) => match *m {}, - } - } -} - -impl From for LhsValue<'_> { - fn from(literal: LiteralValue) -> Self { - match literal { - LiteralValue::Ip(ip) => LhsValue::Ip(ip), - LiteralValue::Bytes(bytes) => LhsValue::Bytes(Bytes::Owned(bytes.into())), - LiteralValue::Int(integer) => LhsValue::Int(integer), - LiteralValue::Bool(b) => match b {}, - LiteralValue::Array(a) => match a {}, - LiteralValue::Map(m) => match m {}, - } - } -} - -impl<'a> LhsValue<'a> { - /// Converts a reference to an LhsValue to an LhsValue with an internal - /// references - pub fn as_ref(&'a self) -> Self { - match self { - LhsValue::Ip(ip) => LhsValue::Ip(*ip), - LhsValue::Bytes(bytes) => LhsValue::Bytes(Bytes::Borrowed(bytes)), - LhsValue::Int(integer) => LhsValue::Int(*integer), - LhsValue::Bool(b) => LhsValue::Bool(*b), - LhsValue::Array(a) => LhsValue::Array(a.as_ref()), - LhsValue::Map(m) => LhsValue::Map(m.as_ref()), - } - } - - /// Converts an `LhsValue` with borrowed data to a fully owned `LhsValue`. - pub fn into_owned(self) -> LhsValue<'static> { - match self { - LhsValue::Ip(ip) => LhsValue::Ip(ip), - LhsValue::Bytes(bytes) => LhsValue::Bytes(Bytes::Owned(bytes.into_owned())), - LhsValue::Int(i) => LhsValue::Int(i), - LhsValue::Bool(b) => LhsValue::Bool(b), - LhsValue::Array(arr) => LhsValue::Array(arr.into_owned()), - LhsValue::Map(map) => LhsValue::Map(map.into_owned()), - } - } - - /// Retrieve an element from an LhsValue given a path item and a specified - /// type. - /// Returns a TypeMismatchError error if current type does not support it - /// nested element. - /// - /// Both LhsValue::Array and LhsValue::Map support nested elements. - pub(crate) fn get( - &'a self, - item: &FieldIndex, - ) -> Result>, IndexAccessError> { - match (self, item) { - (LhsValue::Array(arr), FieldIndex::ArrayIndex(idx)) => Ok(arr.get(*idx as usize)), - (_, FieldIndex::ArrayIndex(_)) => Err(IndexAccessError { - index: item.clone(), - actual: self.get_type(), - }), - (LhsValue::Map(map), FieldIndex::MapKey(key)) => Ok(map.get(key.as_bytes())), - (_, FieldIndex::MapKey(_)) => Err(IndexAccessError { - index: item.clone(), - actual: self.get_type(), - }), - (_, FieldIndex::MapEach) => Err(IndexAccessError { - index: item.clone(), - actual: self.get_type(), - }), - } - } - - #[inline] - pub(crate) fn get_nested(&'a self, indexes: &[FieldIndex]) -> Option<&'a LhsValue<'a>> { - indexes - .iter() - .try_fold(self, |value, idx| value.get(idx).unwrap()) - } - - pub(crate) fn extract( - self, - item: &FieldIndex, - ) -> Result>, IndexAccessError> { - match item { - FieldIndex::ArrayIndex(idx) => match self { - LhsValue::Array(arr) => Ok(arr.extract(*idx as usize)), - _ => Err(IndexAccessError { - index: item.clone(), - actual: self.get_type(), - }), - }, - FieldIndex::MapKey(key) => match self { - LhsValue::Map(map) => Ok(map.extract(key.as_bytes())), - _ => Err(IndexAccessError { - index: item.clone(), - actual: self.get_type(), - }), - }, - FieldIndex::MapEach => Err(IndexAccessError { - index: item.clone(), - actual: self.get_type(), - }), - } - } - - #[inline] - pub(crate) fn extract_nested(self, indexes: &[FieldIndex]) -> Option> { - indexes - .iter() - .try_fold(self, |value, idx| value.extract(idx).unwrap()) - } - - /// Returns an iterator over the Map or Array - pub(crate) fn iter(&'a self) -> Option> { - match self { - LhsValue::Array(array) => Some(Iter::IterArray(array.iter())), - LhsValue::Map(map) => Some(Iter::IterMap(map.iter())), - _ => None, - } - } -} - -impl Serialize for LhsValue<'_> { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - match self { - LhsValue::Ip(ip) => ip.serialize(serializer), - LhsValue::Bytes(bytes) => { - if let Ok(s) = std::str::from_utf8(bytes) { - serializer.serialize_str(s) - } else { - serializer.serialize_bytes(bytes) - } - } - LhsValue::Int(num) => num.serialize(serializer), - LhsValue::Bool(b) => b.serialize(serializer), - LhsValue::Array(arr) => arr.serialize(serializer), - LhsValue::Map(map) => map.serialize(serializer), - } - } -} - -pub(crate) struct LhsValueSeed<'a>(pub &'a Type); - -impl<'de> DeserializeSeed<'de> for LhsValueSeed<'_> { - type Value = LhsValue<'de>; - - fn deserialize(self, deserializer: D) -> Result - where - D: Deserializer<'de>, - { - match self.0 { - Type::Ip => Ok(LhsValue::Ip(std::net::IpAddr::deserialize(deserializer)?)), - Type::Int => Ok(LhsValue::Int(i64::deserialize(deserializer)?)), - Type::Bool => Ok(LhsValue::Bool(bool::deserialize(deserializer)?)), - Type::Bytes => Ok(LhsValue::Bytes(Bytes::deserialize(deserializer)?)), - Type::Array(ty) => Ok(LhsValue::Array({ - let mut arr = Array::new(*ty); - arr.deserialize(deserializer)?; - arr - })), - Type::Map(ty) => Ok(LhsValue::Map({ - let mut map = Map::new(*ty); - map.deserialize(deserializer)?; - map - })), - } - } -} - -pub enum IntoIter<'a> { - IntoArray(ArrayIntoIter<'a>), - IntoMap(MapValuesIntoIter<'a>), -} - -impl<'a> Iterator for IntoIter<'a> { - type Item = LhsValue<'a>; - - fn next(&mut self) -> Option> { - match self { - IntoIter::IntoArray(array) => array.next(), - IntoIter::IntoMap(map) => map.next(), - } - } - - fn size_hint(&self) -> (usize, Option) { - (self.len(), Some(self.len())) - } -} - -impl ExactSizeIterator for IntoIter<'_> { - fn len(&self) -> usize { - match self { - IntoIter::IntoArray(array) => array.len(), - IntoIter::IntoMap(map) => map.len(), - } - } -} - -impl<'a> IntoIterator for LhsValue<'a> { - type IntoIter = IntoIter<'a>; - type Item = LhsValue<'a>; - - fn into_iter(self) -> Self::IntoIter { - match self { - LhsValue::Array(array) => IntoIter::IntoArray(array.into_iter()), - LhsValue::Map(map) => IntoIter::IntoMap(map.into_values()), - _ => unreachable!(), - } - } -} - -pub(crate) enum Iter<'a> { - IterArray(ArrayIter<'a, 'a>), - IterMap(MapIter<'a, 'a>), -} - -impl<'a> Iterator for Iter<'a> { - type Item = &'a LhsValue<'a>; - - fn next(&mut self) -> Option<&'a LhsValue<'a>> { - match self { - Iter::IterArray(array) => array.next(), - Iter::IterMap(map) => map.next().map(|(_, v)| v), - } - } - - fn size_hint(&self) -> (usize, Option) { - (self.len(), Some(self.len())) - } -} - -impl ExactSizeIterator for Iter<'_> { - fn len(&self) -> usize { - match self { - Iter::IterArray(array) => array.len(), - Iter::IterMap(map) => map.len(), - } - } -} - #[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize, Hash, PartialOrd, Ord)] enum PrimitiveType { Bool, @@ -1039,76 +539,43 @@ impl From for Type { } } -declare_types!( - /// A boolean. - Bool(bool | UninhabitedBool | UninhabitedBool), +// Keep the variants and their storage/literal representations in one place. +// Each definition module expands its own types and implementations. +macro_rules! with_value_types { + ($declare:ident) => { + $declare! { + /// A boolean. + Bool(bool | UninhabitedBool | UninhabitedBool), - /// A 64-bit integer number. - Int(i64 | i64 | IntRange), + /// A 64-bit integer number. + Int(i64 | i64 | IntRange), - /// An IPv4 or IPv6 address. - /// - /// These are represented as a single type to allow interop comparisons. - Ip(IpAddr | IpAddr | IpRange), - - /// A raw bytes or a string field. - /// - /// These are completely interchangeable in runtime and differ only in - /// syntax representation, so we represent them as a single type. - Bytes(#[serde(borrow)] Bytes<'a> | BytesExpr | BytesExpr), + /// An IPv4 or IPv6 address. + /// + /// These are represented as a single type to allow interop comparisons. + Ip(IpAddr | IpAddr | IpRange), - /// An Array of [`Type`]. - Array[CompoundType](#[serde(skip_deserializing)] Array<'a> | UninhabitedArray | UninhabitedArray), + /// A raw bytes or a string field. + /// + /// These are completely interchangeable in runtime and differ only in + /// syntax representation, so we represent them as a single type. + Bytes(#[serde(borrow)] Bytes<'a> | BytesExpr | BytesExpr), - /// A Map of string to [`Type`]. - Map[CompoundType](#[serde(skip_deserializing)] Map<'a> | UninhabitedMap | UninhabitedMap), -); + /// An Array of [`Type`]. + Array[CompoundType](#[serde(skip_deserializing)] Array<'a> | UninhabitedArray | UninhabitedArray), -#[cfg(feature = "get-size2")] -impl GetSize for LhsValue<'_> { - fn get_heap_size_with_tracker(&self, tracker: T) -> (usize, T) { - match self { - Self::Bool(_) | Self::Int(_) | Self::Ip(_) => (0, tracker), - Self::Bytes(value) => value.get_heap_size_with_tracker(tracker), - Self::Array(value) => value.get_heap_size_with_tracker(tracker), - Self::Map(value) => value.get_heap_size_with_tracker(tracker), + /// A Map of string to [`Type`]. + Map[CompoundType](#[serde(skip_deserializing)] Map<'a> | UninhabitedMap | UninhabitedMap), } - } + }; } -#[test] -fn test_lhs_value_deserialize() { - use std::str::FromStr; - - let ipv4: LhsValue<'_> = serde_json::from_str("\"127.0.0.1\"").unwrap(); - assert_eq!(ipv4, LhsValue::Ip(IpAddr::from_str("127.0.0.1").unwrap())); - - let ipv6: LhsValue<'_> = serde_json::from_str("\"::1\"").unwrap(); - assert_eq!(ipv6, LhsValue::Ip(IpAddr::from_str("::1").unwrap())); - - let bytes: LhsValue<'_> = serde_json::from_str("\"a JSON string with unicode ❤\"").unwrap(); - assert_eq!( - bytes, - LhsValue::from(&b"a JSON string with unicode \xE2\x9D\xA4"[..]) - ); - - let bytes = - serde_json::from_str::>("\"a JSON string with escaped-unicode \\u2764\"") - .unwrap(); - assert_eq!( - bytes, - LhsValue::from(&b"a JSON string with escaped-unicode \xE2\x9D\xA4"[..]) - ); +pub(crate) use declare_types; +pub(crate) use specialized_get_type; +pub(crate) use specialized_try_from; +pub(crate) use with_value_types; - let bytes: LhsValue<'_> = serde_json::from_str("\"1337\"").unwrap(); - assert_eq!(bytes, LhsValue::from(&b"1337"[..])); - - let integer: LhsValue<'_> = serde_json::from_str("1337").unwrap(); - assert_eq!(integer, LhsValue::Int(1337)); - - let b: LhsValue<'_> = serde_json::from_str("false").unwrap(); - assert_eq!(b, LhsValue::Bool(false)); -} +with_value_types!(declare_types); #[test] fn test_type_serialize() { @@ -1185,99 +652,3 @@ fn test_type_deserialize() { Type::Array(Type::Map(Type::Bytes.into()).into()), ); } - -#[test] -fn test_size_of_lhs_value() { - assert_eq!(std::mem::size_of::>(), 48); -} - -#[cfg(feature = "get-size2")] -#[test] -fn test_lhs_value_get_size() { - let stack_size = std::mem::size_of::>(); - - assert_eq!(LhsValue::Bool(false).get_size(), stack_size); - assert_eq!(LhsValue::Int(42).get_size(), stack_size); - assert_eq!( - LhsValue::Ip(IpAddr::V4(Ipv4Addr::LOCALHOST)).get_size(), - stack_size - ); - - let borrowed_bytes = LhsValue::Bytes(Bytes::Borrowed(b"borrowed")); - assert_eq!(borrowed_bytes.get_size(), stack_size); - - let owned_bytes = LhsValue::Bytes(Bytes::Owned(Box::from(&b"owned"[..]))); - assert_eq!(owned_bytes.get_size(), stack_size + b"owned".len()); -} - -#[cfg(feature = "get-size2")] -#[test] -fn test_array_lhs_value_get_size() { - let stack_size = std::mem::size_of::>(); - - let mut borrowed_values = Vec::with_capacity(4); - borrowed_values.push(LhsValue::Bytes(Bytes::Borrowed(b"borrowed"))); - let borrowed_values_capacity = borrowed_values.capacity(); - let owned_array_of_borrowed = Array::try_from_vec(Type::Bytes, borrowed_values).unwrap(); - assert_eq!( - LhsValue::Array(owned_array_of_borrowed.as_ref()).get_size(), - stack_size - ); - assert_eq!( - LhsValue::Array(owned_array_of_borrowed).get_size(), - stack_size + borrowed_values_capacity * stack_size - ); - - let mut owned_values = Vec::with_capacity(4); - owned_values.push(LhsValue::Bytes(Bytes::Owned(Box::from(&b"owned"[..])))); - let owned_values_capacity = owned_values.capacity(); - let owned_array_of_owned = Array::try_from_vec(Type::Bytes, owned_values).unwrap(); - assert_eq!( - LhsValue::Array(owned_array_of_owned.as_ref()).get_size(), - stack_size - ); - assert_eq!( - LhsValue::Array(owned_array_of_owned).get_size(), - stack_size + owned_values_capacity * stack_size + b"owned".len() - ); -} - -#[cfg(feature = "get-size2")] -#[test] -fn test_map_lhs_value_get_size() { - let stack_size = std::mem::size_of::>(); - - let owned_map_of_borrowed = Map::try_from_iter( - Type::Bytes, - [Ok::<_, TypeMismatchError>(( - Box::from(b"key".as_slice()), - Bytes::Borrowed(b"borrowed"), - ))], - ) - .unwrap(); - assert_eq!( - LhsValue::Map(owned_map_of_borrowed.as_ref()).get_size(), - stack_size - ); - assert_eq!( - LhsValue::Map(owned_map_of_borrowed).get_size(), - stack_size + std::mem::size_of::>() + b"key".len() + stack_size - ); - - let owned_map_of_owned = Map::try_from_iter( - Type::Bytes, - [Ok::<_, TypeMismatchError>(( - Box::from(b"key".as_slice()), - Bytes::Owned(Box::from(&b"owned"[..])), - ))], - ) - .unwrap(); - assert_eq!( - LhsValue::Map(owned_map_of_owned.as_ref()).get_size(), - stack_size - ); - assert_eq!( - LhsValue::Map(owned_map_of_owned).get_size(), - stack_size + std::mem::size_of::>() + b"key".len() + stack_size + b"owned".len() - ); -} diff --git a/engine/src/values/mod.rs b/engine/src/values/mod.rs new file mode 100644 index 00000000..a6c3be83 --- /dev/null +++ b/engine/src/values/mod.rs @@ -0,0 +1,1802 @@ +//! Value interfaces, concrete collection traversal, and value conversion. +//! +//! Implement [`Value`] to associate storage with a [`FieldValueType`] and an +//! engine-provided reader. Custom containers implement [`ArrayValue`] or +//! [`MapValue`] and select [`ArrayRead`] or [`MapRead`]. Comparison execution is +//! internal: scalar comparison returns an optional boolean, while array +//! comparison appends results to an ordinary [`TypedArray`]. + +use crate::lhs_types::append_stored_lhs_at; +pub use crate::lhs_types::{LhsValueRead, LhsValueType}; +use crate::{CompoundType, FieldIndex, LhsValue, Type, TypedArray}; +use std::borrow::{Borrow, Cow}; +use std::collections::BTreeMap; +use std::marker::PhantomData; +use std::net::IpAddr; + +/// A path incompatible with the declared value type. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[error("unsupported value operation: {0}")] +pub struct ValueError(pub &'static str); + +/// A Wirefilter value type, including recursively composed containers. +/// +/// Types such as [`Bytes`] and [`Array`] describe the value independently +/// of its Rust storage and borrow lifetime. [`Self::TYPE`] is the corresponding +/// [`Type`] used by the scheme and compiler. +/// +/// Paths contain owned keys and concrete child paths. Their lifetimes are +/// independent of runtime storage or borrowed selection descriptors. +pub trait ValueType { + /// Whether this type is a scalar, for one-pass output-capacity hints. + const IS_PRIMITIVE: bool; + + /// Concrete traversal independent of a scalar predicate. + type Path: Send + Sync + 'static; + + /// Wirefilter's corresponding logical type. + const TYPE: Type; + + /// Validate a path ending at the custom predicate's declared scalar type. + fn compile_path(path: &[FieldIndex], input_type: Type) -> Result; +} + +/// Type and traversal of a field, resolved when its filter is compiled. +/// +/// Every static [`ValueType`] implements this trait automatically. [`LhsValueType`] +/// instead obtains its type from the scheme, allowing one provider to expose +/// stored values with arbitrary collection nesting. Collection elements exposed +/// through [`ArrayValue`] or [`MapValue`] still require a static [`ValueType`]; a +/// scheme-typed value is a complete field. +pub trait FieldValueType { + /// A fixed getter type, or `None` when the scheme supplies the field's type. + const STATIC_TYPE: Option; + + /// Whether the reader can compare a selected collection directly. + /// Typed collections normally convert for custom collection comparisons. + const DIRECT_COLLECTION_COMPARISON: bool = false; + + /// Compiled traversal, independent of request data and the field getter. + type Path: Send + Sync + 'static; + + /// Validate traversal from the declared field type to the comparator's input. + fn compile_field_path( + field_type: Type, + path: Vec, + input_type: Type, + ) -> Result; +} + +impl FieldValueType for S { + type Path = S::Path; + + const STATIC_TYPE: Option = Some(S::TYPE); + + fn compile_field_path( + field_type: Type, + path: Vec, + input_type: Type, + ) -> Result { + if field_type != S::TYPE { + return Err(ValueError("getter type differs from declared field type")); + } + S::compile_path(&path, input_type) + } +} + +/// A statically dispatched comparison over values and their scalar representations. +/// +/// The compiler selects the scalar comparison method from the validated path's +/// type. Implementations override that method. A compiler adapts a comparison +/// that needs the evaluation context to this interface by +/// borrowing its evaluation context; traversal does not erase the predicate or +/// construct a value enum for each element. +pub(crate) trait ValuePredicate { + /// Compare an existing stored value, including a selected collection. + fn lhs(&self, _: &LhsValue<'_>) -> bool { + panic!("predicate does not accept stored values") + } + + /// Compare a borrowed byte string. + fn bytes(&self, _: &[u8]) -> bool { + panic!("predicate does not accept byte strings") + } + + /// Compare an integer. + fn int(&self, _: i64) -> bool { + panic!("predicate does not accept integers") + } + + /// Compare an IP address. + fn ip(&self, _: IpAddr) -> bool { + panic!("predicate does not accept IP addresses") + } + + /// Compare a boolean. + fn boolean(&self, _: bool) -> bool { + panic!("predicate does not accept booleans") + } +} + +/// A value selects its logical type and an engine-provided reader. +/// +/// A custom array implements [`ArrayValue`] and selects [`ArrayRead`]; a custom +/// map implements [`MapValue`] and selects [`MapRead`]. Neither implementation +/// needs to implement comparison algorithms or construct erased value enums. +/// +/// Byte storage includes `Vec`, `Cow<[u8]>`, and [`crate::Bytes`]. The optional +/// `bytes` feature also supports `bytes::Bytes`, including inside arrays and maps. +pub trait Value { + /// Logical type, independent of the borrow lifetime of this provider. + type ValueType: FieldValueType; + + /// Engine-provided reader for the selected type and storage interface. + /// + /// Select a supplied reader such as [`BytesRead`], [`ArrayRead`], or + /// [`MapRead`]. Its execution interface is internal to Wirefilter; + /// applications customize their storage through the collection traits. + // The private bound intentionally permits selecting supported readers while + // keeping their comparison implementation inside Wirefilter. + #[allow(private_bounds)] + type Read: ReadValue; +} + +/// Convert a value into the representation used by general value expressions. +/// +/// Direct comparisons through the value reader do not require conversion. +/// Application arrays and maps materialize their container structure when passed +/// to functions or other operations requiring an [`LhsValue`]. Borrowed values +/// retain references to byte payloads. +/// +/// A descriptor returned by value may implement this trait with the default method: +/// the resulting value owns its payloads and cannot borrow the temporary descriptor. +/// Descriptors that retain longer-lived references can override it to keep those +/// payloads borrowed, as [`SliceView`] does. +pub trait IntoLhsValue<'a>: Value + Sized { + /// Whether converting a collection builds container storage. + /// + /// The compiler uses this hint to memoize converted non-mapped function + /// arguments. Borrowing existing Wirefilter containers is cheap and returns + /// `false`; application arrays and maps retain the default. This does not + /// change whether the value can be passed to a function. + const MATERIALIZES_COLLECTIONS: bool = true; + + /// Consume the value, preserving references valid for `'a` where possible. + fn into_lhs(self) -> LhsValue<'a> { + Self::Read::to_lhs(&self).into_owned() + } + + /// Consume the value and convert the selection reached by keys and indexes. + /// + /// Borrowed values select directly from their source storage and convert only + /// the selected value. + /// The default preserves custom conversion and ownership behavior by calling + /// [`Self::into_lhs`] before indexing. Borrowing descriptors can override this + /// method to select first, as [`SliceView`] does. + /// + /// Returns `None` for a missing key or index. An empty path converts the whole + /// value. Paths must contain valid array indexes and map keys, not `[*]`; + /// incompatible paths panic, as they indicate an invalid compiled expression. + #[inline] + fn into_lhs_at(self, indexes: &[FieldIndex]) -> Option> { + self.into_lhs().extract_nested(indexes) + } + + /// Consume the value and append selections reached by keys, indexes, and `[*]`. + /// + /// Missing children contribute no values. The `[*]` operator visits arrays in + /// array order and maps in the byte-key order required by [`MapValue::values`]. + /// An empty path appends the whole value, including its container structure. + /// Existing entries in `output` are preserved. + /// + /// Borrowed values select directly from their source storage and convert only + /// the selected values. + /// The default selects the fixed prefix through [`Self::into_lhs_at`], then + /// traverses the converted value, preserving custom conversion and ownership. + /// Borrowing descriptors can override this method to retain their source + /// lifetime during traversal, as [`SliceView`] does. + /// + /// Paths must be compatible with the declared type; incompatible paths panic. + #[inline] + fn append_lhs_at(self, indexes: &[FieldIndex], output: &mut Vec>) { + let prefix = indexes + .iter() + .position(|index| *index == FieldIndex::MapEach) + .unwrap_or(indexes.len()); + if let Some(value) = self.into_lhs_at(&indexes[..prefix]) { + append_stored_lhs_at(value, &indexes[prefix..], output); + } + } +} + +impl<'a, 'data: 'a, T: Value + ?Sized> IntoLhsValue<'a> for &'data T { + const MATERIALIZES_COLLECTIONS: bool = T::Read::MATERIALIZES_COLLECTIONS; + + #[inline] + fn into_lhs(self) -> LhsValue<'a> { + T::Read::to_lhs(self) + } + + #[inline] + fn into_lhs_at(self, indexes: &[FieldIndex]) -> Option> { + T::Read::to_lhs_at(self, indexes) + } + + #[inline] + fn append_lhs_at(self, indexes: &[FieldIndex], output: &mut Vec>) { + T::Read::append_lhs_at(self, indexes, output); + } +} + +/// Internal conversion and comparison operations for a value. +/// +/// The compiler chooses scalar or array evaluation before constructing the +/// closure. Both operations preserve concrete storage and predicate types. +pub(crate) trait ReadValue { + /// Cost of converting borrowed storage, used by `IntoLhsValue for &T`. + const MATERIALIZES_COLLECTIONS: bool = true; + + /// Borrow storage as an ordinary value of its declared field type. + fn to_lhs(value: &V) -> LhsValue<'_>; + + /// Convert a selection without materializing its ancestor containers. + /// Container readers consume child projections so temporary descriptors can + /// preserve their own payload lifetime through `IntoLhsValue`. + #[inline] + fn to_lhs_at<'a>(value: &'a V, indexes: &[FieldIndex]) -> Option> { + Self::to_lhs(value).extract_nested(indexes) + } + + /// Append selected values without materializing their ancestor containers. + /// Child projections are consumed so borrowed descriptors retain their source + /// lifetime instead of borrowing a temporary on the traversal stack. + #[inline] + fn append_lhs_at<'a>(value: &'a V, indexes: &[FieldIndex], output: &mut Vec>) { + assert!(indexes.is_empty(), "path continues beyond scalar"); + output.push(Self::to_lhs(value)); + } + + /// Compare the value selected by a path containing only indexes and keys. + /// A missing index or key returns `None`; `[*]` is a compiler error. + fn compare_one( + value: &V, + path: &S::Path, + predicate: &P, + ) -> Option; + + /// Append comparisons in traversal order, skipping missing indexes and keys. + /// Scalar readers append one result; container readers also handle `[*]`. + #[inline] + fn compare_vec( + value: &V, + path: &S::Path, + predicate: &P, + output: &mut TypedArray<'static, bool>, + ) { + if let Some(result) = Self::compare_one(value, path, predicate) { + output.push(result); + } + } +} + +/// Compare a scalar selection without constructing a result collector. +#[inline] +pub(crate) fn compare_one( + value: &V, + path: &::Path, + predicate: &P, +) -> Option { + V::Read::compare_one(value, path, predicate) +} + +/// Append comparisons for a possibly nested array or map selection. +#[inline] +pub(crate) fn compare_vec( + value: &V, + path: &::Path, + predicate: &P, + output: &mut TypedArray<'static, bool>, +) { + V::Read::compare_vec(value, path, predicate, output); +} + +/// A concrete array projection. Elements can be references or borrowed descriptors. +pub trait ArrayValue { + /// Stable logical type of each element. + type Element: ValueType; + + /// A projection tied to the borrow of the container. + type Item<'a>: Value + IntoLhsValue<'a> + where + Self: 'a; + + /// A concrete iterator; no trait-object iterator or exact-length scan is needed. + type Iter<'a>: Iterator> + where + Self: 'a; + + /// Retrieve a single element without erasing its type. + fn get(&self, index: usize) -> Option>; + + /// Iterate in logical array order. + fn iter(&self) -> Self::Iter<'_>; +} + +/// A byte-keyed map with concrete value projections. +pub trait MapValue { + /// Stable logical type of each map value. + type Element: ValueType; + + /// A reference or descriptor returned by value. + type Item<'a>: Value + IntoLhsValue<'a> + where + Self: 'a; + + /// A concrete iterator over map values. + type Iter<'a>: Iterator> + where + Self: 'a; + + /// Look up a key with exact byte equality, without case normalization. + fn get(&self, key: &[u8]) -> Option>; + + /// Iterate values in ascending lexicographic order of their exact byte keys. + /// + /// This must match ordinary Wirefilter maps, including after conversion for + /// function arguments. Vector logical operations combine results by position; + /// a different iteration order can change results even through an identity + /// function. Adapters over differently ordered storage must supply this order. + fn values(&self) -> Self::Iter<'_>; + + /// Visit key-value pairs when a general value expression needs a stored map. + /// + /// Keys use exact bytes and must be unique. Keys may be temporary: conversion + /// copies each key, while the returned value may borrow the map for `'a`. + fn for_each_entry<'a>(&'a self, visitor: impl FnMut(&[u8], Self::Item<'a>)); +} + +/// An array type composed with any other type. +pub struct Array(PhantomData); +/// A map type composed with any other type. +pub struct Map(PhantomData); + +/// A concrete array access followed by a concrete child path. +/// +/// This is [`ValueType::Path`] for [`Array`], with `P = S::Path`. +/// [`ValueType::compile_path`] constructs it during expression compilation; +/// [`ArrayRead`] follows it during evaluation. The path owns no request data. +/// Indices retain the parsed [`FieldIndex::ArrayIndex`] width to keep compiled +/// paths compact, and are converted to `usize` only when accessing a collection. +pub enum ArrayPath

{ + /// Select one index, then follow the child path if the element exists. + Index(u32, P), + /// Follow the child path for each element in array order. + Each(P), +} + +/// A concrete map access followed by a concrete child path. +/// +/// This is [`ValueType::Path`] for [`Map`], with `P = S::Path`. +/// [`ValueType::compile_path`] constructs it during expression compilation; +/// [`MapRead`] follows it during evaluation. Keys are owned so the path can +/// outlive both the parsed expression and any request. +pub enum MapPath

{ + /// Select an exact byte key, then follow the child path if the key exists. + Key(Box<[u8]>, P), + /// Follow the child path for each value in ascending byte-key order. + Each(P), +} + +impl ValueType for Array { + type Path = ArrayPath; + + const IS_PRIMITIVE: bool = false; + const TYPE: Type = Type::Array(CompoundType::from_type(S::TYPE)); + + fn compile_path(path: &[FieldIndex], input_type: Type) -> Result { + let (step, rest) = path + .split_first() + .ok_or(ValueError("array requires an index or [*]"))?; + match step { + FieldIndex::ArrayIndex(index) => { + Ok(ArrayPath::Index(*index, S::compile_path(rest, input_type)?)) + } + FieldIndex::MapEach => Ok(ArrayPath::Each(S::compile_path(rest, input_type)?)), + FieldIndex::MapKey(_) => Err(ValueError("map key used on array")), + } + } +} + +impl ValueType for Map { + type Path = MapPath; + + const IS_PRIMITIVE: bool = false; + const TYPE: Type = Type::Map(CompoundType::from_type(S::TYPE)); + + fn compile_path(path: &[FieldIndex], input_type: Type) -> Result { + let (step, rest) = path + .split_first() + .ok_or(ValueError("map requires a key or [*]"))?; + match step { + FieldIndex::MapKey(key) => Ok(MapPath::Key( + key.as_bytes().into(), + S::compile_path(rest, input_type)?, + )), + FieldIndex::MapEach => Ok(MapPath::Each(S::compile_path(rest, input_type)?)), + FieldIndex::ArrayIndex(_) => Err(ValueError("array index used on map")), + } + } +} + +/// Engine reader for any concrete [`ArrayValue`] implementation. +pub struct ArrayRead; + +impl ReadValue> for ArrayRead { + fn to_lhs(array: &A) -> LhsValue<'_> { + crate::Array::try_from_iter(A::Element::TYPE, array.iter().map(IntoLhsValue::into_lhs)) + .expect("array items must match their declared type") + .into() + } + + #[inline] + fn to_lhs_at<'a>(array: &'a A, indexes: &[FieldIndex]) -> Option> { + match indexes.split_first() { + None => Some(Self::to_lhs(array)), + Some((FieldIndex::ArrayIndex(index), rest)) => { + array.get(*index as usize)?.into_lhs_at(rest) + } + Some(_) => panic!("array selection requires an array index"), + } + } + + #[inline] + fn append_lhs_at<'a>(array: &'a A, indexes: &[FieldIndex], output: &mut Vec>) { + match indexes.split_first() { + None => output.push(Self::to_lhs(array)), + Some((FieldIndex::ArrayIndex(index), rest)) => { + if let Some(value) = array.get(*index as usize) { + value.append_lhs_at(rest, output); + } + } + Some((FieldIndex::MapEach, rest)) => { + let values = array.iter(); + if rest.is_empty() { + output.reserve(values.size_hint().0); + } + for value in values { + value.append_lhs_at(rest, output); + } + } + Some(_) => panic!("map key used on array"), + } + } + + #[inline] + fn compare_one( + array: &A, + path: &ArrayPath<::Path>, + predicate: &P, + ) -> Option { + match path { + ArrayPath::Index(index, child) => { + let value = array.get(*index as usize)?; + compare_one(&value, child, predicate) + } + ArrayPath::Each(_) => panic!("[*] in scalar comparison path"), + } + } + + #[inline] + fn compare_vec( + array: &A, + path: &ArrayPath<::Path>, + predicate: &P, + output: &mut TypedArray<'static, bool>, + ) { + match path { + ArrayPath::Index(index, child) => { + if let Some(value) = array.get(*index as usize) { + compare_vec(&value, child, predicate, output); + } + } + ArrayPath::Each(child) => { + let values = array.iter(); + if A::Element::IS_PRIMITIVE && output.is_empty() { + let capacity = values.size_hint().0; + if capacity != 0 { + *output = TypedArray::with_capacity(capacity); + } + } + for value in values { + compare_vec(&value, child, predicate, output); + } + } + } + } +} + +/// Engine reader for any concrete [`MapValue`] implementation. +pub struct MapRead; + +impl ReadValue> for MapRead { + fn to_lhs(map: &M) -> LhsValue<'_> { + let mut entries = Vec::with_capacity(map.values().size_hint().0); + map.for_each_entry(|key, value| { + entries.push(Ok::<_, crate::TypeMismatchError>(( + Box::<[u8]>::from(key), + value.into_lhs(), + ))); + }); + crate::Map::try_from_iter(M::Element::TYPE, entries) + .expect("map values must match their declared type") + .into() + } + + #[inline] + fn to_lhs_at<'a>(map: &'a M, indexes: &[FieldIndex]) -> Option> { + match indexes.split_first() { + None => Some(Self::to_lhs(map)), + Some((FieldIndex::MapKey(key), rest)) => map.get(key.as_bytes())?.into_lhs_at(rest), + Some(_) => panic!("map selection requires a map key"), + } + } + + #[inline] + fn append_lhs_at<'a>(map: &'a M, indexes: &[FieldIndex], output: &mut Vec>) { + match indexes.split_first() { + None => output.push(Self::to_lhs(map)), + Some((FieldIndex::MapKey(key), rest)) => { + if let Some(value) = map.get(key.as_bytes()) { + value.append_lhs_at(rest, output); + } + } + Some((FieldIndex::MapEach, rest)) => { + let values = map.values(); + if rest.is_empty() { + output.reserve(values.size_hint().0); + } + for value in values { + value.append_lhs_at(rest, output); + } + } + Some(_) => panic!("array index used on map"), + } + } + + #[inline] + fn compare_one( + map: &M, + path: &MapPath<::Path>, + predicate: &P, + ) -> Option { + match path { + MapPath::Key(key, child) => { + let value = map.get(key)?; + compare_one(&value, child, predicate) + } + MapPath::Each(_) => panic!("[*] in scalar comparison path"), + } + } + + #[inline] + fn compare_vec( + map: &M, + path: &MapPath<::Path>, + predicate: &P, + output: &mut TypedArray<'static, bool>, + ) { + match path { + MapPath::Key(key, child) => { + if let Some(value) = map.get(key) { + compare_vec(&value, child, predicate, output); + } + } + MapPath::Each(child) => { + let values = map.values(); + if M::Element::IS_PRIMITIVE && output.is_empty() { + let capacity = values.size_hint().0; + if capacity != 0 { + *output = TypedArray::with_capacity(capacity); + } + } + for value in values { + compare_vec(&value, child, predicate, output); + } + } + } + } +} + +/// Forward a borrowed provider to its concrete engine reader. +pub struct Borrowed(PhantomData); + +impl Value for &T { + type Read = Borrowed; + type ValueType = T::ValueType; +} + +impl ReadValue<&T, T::ValueType> for Borrowed { + const MATERIALIZES_COLLECTIONS: bool = T::Read::MATERIALIZES_COLLECTIONS; + + #[inline] + fn to_lhs<'a>(value: &'a &T) -> LhsValue<'a> { + T::Read::to_lhs(value) + } + + #[inline] + fn to_lhs_at<'a>(value: &'a &T, indexes: &[FieldIndex]) -> Option> { + T::Read::to_lhs_at(value, indexes) + } + + #[inline] + fn append_lhs_at<'a>(value: &'a &T, indexes: &[FieldIndex], output: &mut Vec>) { + T::Read::append_lhs_at(value, indexes, output); + } + + #[inline] + fn compare_one( + value: &&T, + path: &::Path, + predicate: &P, + ) -> Option { + compare_one::(value, path, predicate) + } + + #[inline] + fn compare_vec( + value: &&T, + path: &::Path, + predicate: &P, + output: &mut TypedArray<'static, bool>, + ) { + compare_vec::(value, path, predicate, output); + } +} + +/// Forward boxed storage to its concrete reader. +pub struct BoxRead(PhantomData); + +impl Value for Box { + type Read = BoxRead; + type ValueType = T::ValueType; +} + +impl<'a, T: Value + ?Sized> IntoLhsValue<'a> for Box { + #[inline] + fn append_lhs_at(self, indexes: &[FieldIndex], output: &mut Vec>) { + append_stored_lhs_at(self.into_lhs(), indexes, output); + } +} + +impl ReadValue, T::ValueType> for BoxRead { + const MATERIALIZES_COLLECTIONS: bool = T::Read::MATERIALIZES_COLLECTIONS; + + #[inline] + fn to_lhs(value: &Box) -> LhsValue<'_> { + T::Read::to_lhs(value) + } + + #[inline] + fn to_lhs_at<'a>(value: &'a Box, indexes: &[FieldIndex]) -> Option> { + T::Read::to_lhs_at(value, indexes) + } + + #[inline] + fn append_lhs_at<'a>( + value: &'a Box, + indexes: &[FieldIndex], + output: &mut Vec>, + ) { + T::Read::append_lhs_at(value, indexes, output); + } + + #[inline] + fn compare_one( + value: &Box, + path: &::Path, + predicate: &P, + ) -> Option { + compare_one::(value, path, predicate) + } + + #[inline] + fn compare_vec( + value: &Box, + path: &::Path, + predicate: &P, + output: &mut TypedArray<'static, bool>, + ) { + compare_vec::(value, path, predicate, output); + } +} + +impl ArrayValue for Vec +where + T::ValueType: ValueType, +{ + type Element = T::ValueType; + type Item<'a> + = &'a T + where + Self: 'a; + type Iter<'a> + = std::slice::Iter<'a, T> + where + Self: 'a; + + #[inline] + fn get(&self, index: usize) -> Option> { + self.as_slice().get(index) + } + + #[inline] + fn iter(&self) -> Self::Iter<'_> { + self.as_slice().iter() + } +} + +impl Value for Vec +where + T::ValueType: ValueType, +{ + type Read = ArrayRead; + type ValueType = Array; +} + +impl<'a, T: Value> IntoLhsValue<'a> for Vec +where + T::ValueType: ValueType, +{ + #[inline] + fn append_lhs_at(self, indexes: &[FieldIndex], output: &mut Vec>) { + append_stored_lhs_at(self.into_lhs(), indexes, output); + } +} + +impl ArrayValue for [T] +where + T::ValueType: ValueType, +{ + type Element = T::ValueType; + type Item<'a> + = &'a T + where + Self: 'a; + type Iter<'a> + = std::slice::Iter<'a, T> + where + Self: 'a; + + #[inline] + fn get(&self, index: usize) -> Option> { + <[T]>::get(self, index) + } + + #[inline] + fn iter(&self) -> Self::Iter<'_> { + <[T]>::iter(self) + } +} + +impl Value for [T] +where + T::ValueType: ValueType, +{ + type Read = ArrayRead; + type ValueType = Array; +} + +impl + Ord, V: Value> MapValue for BTreeMap +where + V::ValueType: ValueType, +{ + type Element = V::ValueType; + type Item<'a> + = &'a V + where + Self: 'a; + type Iter<'a> + = std::collections::btree_map::Values<'a, K, V> + where + Self: 'a; + + #[inline] + fn get(&self, key: &[u8]) -> Option> { + BTreeMap::get(self, key) + } + + #[inline] + fn values(&self) -> Self::Iter<'_> { + BTreeMap::values(self) + } + + #[inline] + fn for_each_entry<'a>(&'a self, mut visitor: impl FnMut(&[u8], Self::Item<'a>)) { + for (key, value) in self { + visitor(key.borrow(), value); + } + } +} + +impl + Ord, V: Value> Value for BTreeMap +where + V::ValueType: ValueType, +{ + type Read = MapRead; + type ValueType = Map; +} + +impl<'a, K: Borrow<[u8]> + Ord, V: Value> IntoLhsValue<'a> for BTreeMap +where + V::ValueType: ValueType, +{ + #[inline] + fn append_lhs_at(self, indexes: &[FieldIndex], output: &mut Vec>) { + append_stored_lhs_at(self.into_lhs(), indexes, output); + } +} + +/// A small borrowed selection descriptor that can be returned by value. +#[derive(Debug, Clone, Copy)] +pub struct SliceView<'a, T>(pub &'a [T]); + +impl ArrayValue for SliceView<'_, T> +where + T::ValueType: ValueType, +{ + type Element = T::ValueType; + type Item<'a> + = &'a T + where + Self: 'a; + type Iter<'a> + = std::slice::Iter<'a, T> + where + Self: 'a; + + #[inline] + fn get(&self, index: usize) -> Option> { + self.0.get(index) + } + + #[inline] + fn iter(&self) -> Self::Iter<'_> { + self.0.iter() + } +} + +impl Value for SliceView<'_, T> +where + T::ValueType: ValueType, +{ + type Read = ArrayRead; + type ValueType = Array; +} + +impl<'a, 'data: 'a, T: Value> IntoLhsValue<'a> for SliceView<'data, T> +where + T::ValueType: ValueType, +{ + fn into_lhs(self) -> LhsValue<'a> { + ArrayRead::to_lhs(self.0) + } + + #[inline] + fn into_lhs_at(self, indexes: &[FieldIndex]) -> Option> { + self.0.into_lhs_at(indexes) + } + + #[inline] + fn append_lhs_at(self, indexes: &[FieldIndex], output: &mut Vec>) { + self.0.append_lhs_at(indexes, output); + } +} + +/// Byte-string type; storage can be any concrete byte-slice provider. +pub struct Bytes; +/// Engine reader for a provider implementing `AsRef<[u8]>`. +pub struct BytesRead; + +fn validate_scalar_path( + path: &[FieldIndex], + input_type: Type, + scalar_type: Type, +) -> Result<(), ValueError> { + if !path.is_empty() { + Err(ValueError("path continues beyond scalar")) + } else if input_type != scalar_type { + Err(ValueError( + "predicate input type differs from selected scalar", + )) + } else { + Ok(()) + } +} + +impl ValueType for Bytes { + type Path = (); + + const IS_PRIMITIVE: bool = true; + const TYPE: Type = Type::Bytes; + + fn compile_path(path: &[FieldIndex], input_type: Type) -> Result { + validate_scalar_path(path, input_type, Type::Bytes) + } +} + +impl + ?Sized> ReadValue for BytesRead { + #[inline] + fn to_lhs(value: &T) -> LhsValue<'_> { + value.as_ref().into() + } + + #[inline] + fn compare_one(value: &T, _: &(), predicate: &P) -> Option { + Some(predicate.bytes(value.as_ref())) + } +} + +macro_rules! bytes_storage { + ($($ty:ty),* $(,)?) => {$( + impl Value for $ty { + type ValueType = Bytes; + type Read = BytesRead; + } + )*}; +} + +// `u8` deliberately has no Value implementation: Vec and [u8] are +// byte strings, while Vec and [T] compose arrays for element types implementing Value. +bytes_storage!(str, String, [u8], Vec, Cow<'_, [u8]>); + +#[cfg(feature = "bytes")] +bytes_storage!(bytes::Bytes); + +impl IntoLhsValue<'_> for String {} +impl IntoLhsValue<'_> for Vec {} + +#[cfg(feature = "bytes")] +impl IntoLhsValue<'_> for bytes::Bytes {} + +impl<'a, 'data: 'a> IntoLhsValue<'a> for Cow<'data, [u8]> { + #[inline] + fn into_lhs(self) -> LhsValue<'a> { + LhsValue::Bytes(crate::Bytes::from(self)) + } +} + +macro_rules! ordered_scalar { + ($marker:ident, $ty:ty, $variant:ident, $compare:ident, $doc:literal) => { + #[doc = $doc] + pub struct $marker; + impl ValueType for $marker { + type Path = (); + + const IS_PRIMITIVE: bool = true; + const TYPE: Type = Type::$variant; + + fn compile_path( + path: &[FieldIndex], + input_type: Type, + ) -> Result { + validate_scalar_path(path, input_type, Type::$variant) + } + } + impl Value for $ty { + type Read = $marker; + type ValueType = $marker; + } + impl ReadValue<$ty, $marker> for $marker { + #[inline] + fn to_lhs(value: &$ty) -> LhsValue<'_> { + LhsValue::$variant(*value) + } + + #[inline] + fn compare_one( + value: &$ty, + _: &(), + predicate: &P, + ) -> Option { + Some(predicate.$compare(*value)) + } + } + impl IntoLhsValue<'_> for $ty {} + }; +} + +ordered_scalar!(Int, i64, Int, int, "Integer type and its engine reader."); +ordered_scalar!(Ip, IpAddr, Ip, ip, "IP-address type and its engine reader."); + +/// Boolean type and reader for direct boolean evaluation. +pub struct Bool; + +impl ValueType for Bool { + type Path = (); + + const IS_PRIMITIVE: bool = true; + const TYPE: Type = Type::Bool; + + fn compile_path(path: &[FieldIndex], input_type: Type) -> Result { + validate_scalar_path(path, input_type, Type::Bool) + } +} + +impl Value for bool { + type Read = Bool; + type ValueType = Bool; +} + +impl ReadValue for Bool { + #[inline] + fn to_lhs(value: &bool) -> LhsValue<'_> { + LhsValue::Bool(*value) + } + + #[inline] + fn compare_one( + value: &bool, + _: &(), + predicate: &P, + ) -> Option { + Some(predicate.boolean(*value)) + } +} + +impl IntoLhsValue<'_> for bool {} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::Cell; + + fn bytes<'a>(value: &'a LhsValue<'_>) -> &'a [u8] { + match value { + LhsValue::Bytes(value) => value.as_ref(), + _ => panic!("expected bytes"), + } + } + + #[test] + fn test_bytes_materialization_preserves_payload_storage() { + let input = vec![0xff, b'a']; + let borrowed = crate::Bytes::Borrowed(&input).into_lhs(); + assert_eq!(bytes(&borrowed).as_ptr(), input.as_ptr()); + assert!(matches!( + borrowed, + LhsValue::Bytes(crate::Bytes::Borrowed(_)) + )); + + let input = input.into_boxed_slice(); + let ptr = input.as_ptr(); + let owned = crate::Bytes::Owned(input).into_lhs(); + assert_eq!(bytes(&owned).as_ptr(), ptr); + assert!(matches!(owned, LhsValue::Bytes(crate::Bytes::Owned(_)))); + + let source = vec![ + crate::Bytes::Borrowed(b"borrowed"), + crate::Bytes::from("owned".to_owned()), + ]; + let LhsValue::Array(array) = (&source).into_lhs() else { + panic!("expected an array"); + }; + for (index, original) in source.iter().enumerate() { + let value = array.get(index).unwrap(); + assert_eq!(bytes(value), original.as_ref()); + assert_eq!(bytes(value).as_ptr(), original.as_ref().as_ptr()); + assert!(matches!(value, LhsValue::Bytes(crate::Bytes::Borrowed(_)))); + } + } + + #[test] + fn test_cow_materialization_preserves_ownership() { + let input = vec![0xff, b'a']; + let borrowed = Cow::Borrowed(input.as_slice()).into_lhs(); + assert_eq!(bytes(&borrowed).as_ptr(), input.as_ptr()); + + let owned: LhsValue<'static> = Cow::<'_, [u8]>::Owned(input).into_lhs(); + assert_eq!(bytes(&owned), &[0xff, b'a']); + assert!(matches!(owned, LhsValue::Bytes(crate::Bytes::Owned(_)))); + } + + #[cfg(feature = "bytes")] + #[test] + fn test_shared_bytes_materialization_borrows_or_copies_as_needed() { + let input = bytes::Bytes::from(vec![0xff, b'a']); + let borrowed = (&input).into_lhs(); + assert_eq!(bytes(&borrowed).as_ptr(), input.as_ptr()); + assert!(matches!( + borrowed, + LhsValue::Bytes(crate::Bytes::Borrowed(_)) + )); + + // LhsValue has no shared ownership variant, so consuming the last owner + // requires an owned copy. Borrowed field getters avoid this copy. + let owned: LhsValue<'static> = input.into_lhs(); + assert_eq!(bytes(&owned), &[0xff, b'a']); + assert!(matches!(owned, LhsValue::Bytes(crate::Bytes::Owned(_)))); + } + + #[test] + fn test_nested_materialization_preserves_types_and_borrows_payloads() { + let source = BTreeMap::from([( + Box::<[u8]>::from(&b"Mixed\xff"[..]), + vec![Box::::from("first"), Box::::from("second")].into_boxed_slice(), + )]); + let value = (&source).into_lhs(); + let LhsValue::Map(map) = value else { + panic!("expected a map") + }; + assert_eq!(map.value_type(), Type::array(Type::Bytes)); + assert!(map.get(b"mixed\xff").is_none()); + let LhsValue::Array(array) = map.get(b"Mixed\xff").unwrap() else { + panic!("expected an array") + }; + assert_eq!(array.value_type(), Type::Bytes); + let source_values = source.get(&b"Mixed\xff"[..]).unwrap(); + for (index, text) in source_values.iter().enumerate() { + let actual = bytes(array.get(index).unwrap()); + assert_eq!(actual, text.as_bytes()); + assert_eq!(actual.as_ptr(), text.as_ptr()); + } + + let empty: Vec, Vec>> = Vec::new(); + let LhsValue::Array(array) = (&empty).into_lhs() else { + panic!("expected an array") + }; + assert_eq!(array.value_type(), Type::map(Type::array(Type::Int))); + assert!(array.is_empty()); + } + + struct SelectionMap(Vec); + + impl Value for SelectionMap { + type Read = MapRead; + type ValueType = Map>; + } + + impl MapValue for SelectionMap { + type Element = Array; + type Item<'a> = SliceView<'a, String>; + type Iter<'a> = std::iter::Once>; + + fn get(&self, key: &[u8]) -> Option> { + (key == b"accept").then_some(SliceView(&self.0)) + } + + fn values(&self) -> Self::Iter<'_> { + std::iter::once(SliceView(&self.0)) + } + + fn for_each_entry<'a>(&'a self, mut visitor: impl FnMut(&[u8], Self::Item<'a>)) { + visitor(b"accept", SliceView(&self.0)); + } + } + + #[test] + fn test_temporary_array_selection_can_preserve_borrowed_payloads() { + let source = SelectionMap(vec!["text/html".into()]); + // The selection descriptor is consumed here; only its payload outlives it. + let selected = source.get(b"accept").unwrap().into_lhs(); + let LhsValue::Array(array) = selected else { + panic!("expected an array") + }; + assert_eq!(bytes(array.get(0).unwrap()).as_ptr(), source.0[0].as_ptr()); + + let LhsValue::Map(map) = (&source).into_lhs() else { + panic!("expected a map") + }; + let LhsValue::Array(array) = map.get(b"accept").unwrap() else { + panic!("expected an array") + }; + assert_eq!(bytes(array.get(0).unwrap()).as_ptr(), source.0[0].as_ptr()); + } + + #[test] + fn test_indexed_materialization_preserves_borrowed_descriptors() { + let source = Box::new(SelectionMap(vec!["text/html".into()])); + let reference = &source; + let indexes = [ + FieldIndex::MapKey("accept".into()), + FieldIndex::ArrayIndex(0), + ]; + // The map creates a temporary SliceView. Its consuming conversion must + // retain the source lifetime through both indexes and reader wrappers. + for selected in [ + reference.into_lhs_at(&indexes).unwrap(), + (&reference).into_lhs_at(&indexes).unwrap(), + SliceView(source.0.as_slice()) + .into_lhs_at(&indexes[1..]) + .unwrap(), + ] { + assert!(matches!( + selected, + LhsValue::Bytes(crate::Bytes::Borrowed(_)) + )); + assert_eq!(bytes(&selected).as_ptr(), source.0[0].as_ptr()); + } + assert!( + reference + .into_lhs_at(&[FieldIndex::MapKey("missing".into())]) + .is_none() + ); + assert!( + reference + .into_lhs_at(&[ + FieldIndex::MapKey("accept".into()), + FieldIndex::ArrayIndex(1), + ]) + .is_none() + ); + } + + #[test] + fn test_map_each_materialization_preserves_borrowed_descriptors() { + let source = Box::new(SelectionMap(vec!["text/html".into(), "text/plain".into()])); + let reference = &source; + for indexes in [ + vec![FieldIndex::MapEach, FieldIndex::MapEach], + vec![FieldIndex::MapKey("accept".into()), FieldIndex::MapEach], + ] { + // Each map item is a temporary SliceView, consumed during traversal. + // Both wrapper readers must preserve its references to the source. + let mut actual = vec![LhsValue::Bool(true)]; + reference.append_lhs_at(&indexes, &mut actual); + (&reference).append_lhs_at(&indexes, &mut actual); + assert_eq!(actual[0], LhsValue::Bool(true)); + assert_eq!(actual.len(), 5); + for (value, original) in actual[1..].iter().zip(source.0.iter().cycle()) { + assert!(matches!(value, LhsValue::Bytes(crate::Bytes::Borrowed(_)))); + assert_eq!(bytes(value).as_ptr(), original.as_ptr()); + } + } + + let mut selected = Vec::new(); + reference.append_lhs_at( + &[FieldIndex::MapEach, FieldIndex::ArrayIndex(1)], + &mut selected, + ); + assert_eq!(selected.len(), 1); + assert_eq!(bytes(&selected[0]).as_ptr(), source.0[1].as_ptr()); + for indexes in [ + vec![FieldIndex::MapEach, FieldIndex::ArrayIndex(99)], + vec![FieldIndex::MapKey("missing".into()), FieldIndex::MapEach], + ] { + reference.append_lhs_at(&indexes, &mut selected); + assert_eq!(selected.len(), 1); + } + + // Stopping at a container preserves that container, rather than + // flattening beyond the specified `[*]`. + let mut containers = Vec::new(); + reference.append_lhs_at(&[FieldIndex::MapEach], &mut containers); + assert_eq!(containers.len(), 1); + let LhsValue::Array(array) = &containers[0] else { + panic!("expected the selected array"); + }; + assert_eq!(array.len(), 2); + assert_eq!(bytes(array.get(0).unwrap()).as_ptr(), source.0[0].as_ptr()); + } + + #[test] + fn test_indexed_descriptor_fallback_preserves_custom_conversion() { + struct Descriptor<'a> { + values: &'a [String], + conversions: &'a Cell, + } + impl Value for Descriptor<'_> { + type Read = ArrayRead; + type ValueType = Array; + } + impl ArrayValue for Descriptor<'_> { + type Element = Bytes; + type Item<'a> + = &'a String + where + Self: 'a; + type Iter<'a> + = std::slice::Iter<'a, String> + where + Self: 'a; + + fn get(&self, index: usize) -> Option> { + self.values.get(index) + } + + fn iter(&self) -> Self::Iter<'_> { + self.values.iter() + } + } + impl<'a> IntoLhsValue<'a> for Descriptor<'a> { + fn into_lhs(self) -> LhsValue<'a> { + self.conversions.set(self.conversions.get() + 1); + self.values.into_lhs() + } + } + + let values = vec!["borrowed".to_owned()]; + let conversions = Cell::new(0); + let selected = Descriptor { + values: &values, + conversions: &conversions, + } + .into_lhs_at(&[FieldIndex::ArrayIndex(0)]) + .unwrap(); + assert_eq!(conversions.get(), 1); + assert!(matches!( + selected, + LhsValue::Bytes(crate::Bytes::Borrowed(_)) + )); + assert_eq!(bytes(&selected).as_ptr(), values[0].as_ptr()); + + let mut flattened = Vec::new(); + Descriptor { + values: &values, + conversions: &conversions, + } + .append_lhs_at(&[FieldIndex::MapEach], &mut flattened); + assert_eq!(conversions.get(), 2); + assert_eq!(flattened.len(), 1); + assert!(matches!( + flattened[0], + LhsValue::Bytes(crate::Bytes::Borrowed(_)) + )); + assert_eq!(bytes(&flattened[0]).as_ptr(), values[0].as_ptr()); + + // The same fallback must also remain safe for a temporary owning its + // payload: after it is consumed, the selected result owns its bytes. + let selected: LhsValue<'static> = vec!["owned".to_owned()] + .into_lhs_at(&[FieldIndex::ArrayIndex(0)]) + .unwrap(); + assert!(matches!(selected, LhsValue::Bytes(crate::Bytes::Owned(_)))); + assert_eq!(bytes(&selected), b"owned"); + + let mut flattened: Vec> = Vec::new(); + vec![vec!["owned".to_owned()], vec![]].append_lhs_at( + &[FieldIndex::MapEach, FieldIndex::ArrayIndex(0)], + &mut flattened, + ); + assert_eq!(flattened.len(), 1); + assert!(matches!( + flattened[0], + LhsValue::Bytes(crate::Bytes::Owned(_)) + )); + assert_eq!(bytes(&flattened[0]), b"owned"); + } + + #[test] + fn test_map_each_fallback_preserves_custom_indexed_conversion() { + struct IndexedSelection<'a>(&'a [Vec]); + + impl Value for IndexedSelection<'_> { + type Read = ArrayRead; + type ValueType = Array>; + } + + impl ArrayValue for IndexedSelection<'_> { + type Element = Array; + type Item<'a> + = &'a Vec + where + Self: 'a; + type Iter<'a> + = std::slice::Iter<'a, Vec> + where + Self: 'a; + + fn get(&self, index: usize) -> Option> { + self.0.get(index) + } + + fn iter(&self) -> Self::Iter<'_> { + self.0.iter() + } + } + + impl<'a> IntoLhsValue<'a> for IndexedSelection<'a> { + fn into_lhs(self) -> LhsValue<'a> { + panic!("the custom indexed conversion must be used"); + } + + fn into_lhs_at(self, indexes: &[FieldIndex]) -> Option> { + self.0.into_lhs_at(indexes) + } + } + + let source = vec![vec!["first".to_owned(), "second".to_owned()]]; + let mut selected = Vec::new(); + IndexedSelection(&source).append_lhs_at( + &[FieldIndex::ArrayIndex(0), FieldIndex::MapEach], + &mut selected, + ); + assert_eq!(selected.len(), 2); + for (value, original) in selected.iter().zip(&source[0]) { + assert!(matches!(value, LhsValue::Bytes(crate::Bytes::Borrowed(_)))); + assert_eq!(bytes(value).as_ptr(), original.as_ptr()); + } + } + + #[test] + fn test_map_each_preserves_custom_child_append() { + struct AppendingSelection<'a>(&'a [String]); + + impl Value for AppendingSelection<'_> { + type Read = ArrayRead; + type ValueType = Array; + } + + impl ArrayValue for AppendingSelection<'_> { + type Element = Bytes; + type Item<'a> + = &'a String + where + Self: 'a; + type Iter<'a> + = std::slice::Iter<'a, String> + where + Self: 'a; + + fn get(&self, _: usize) -> Option> { + panic!("the custom append method must be used"); + } + + fn iter(&self) -> Self::Iter<'_> { + panic!("the custom append method must be used"); + } + } + + impl<'a> IntoLhsValue<'a> for AppendingSelection<'a> { + fn into_lhs(self) -> LhsValue<'a> { + panic!("the custom append method must be used"); + } + + fn into_lhs_at(self, _: &[FieldIndex]) -> Option> { + panic!("the custom append method must be used"); + } + + fn append_lhs_at(self, indexes: &[FieldIndex], output: &mut Vec>) { + assert_eq!(indexes, &[FieldIndex::ArrayIndex(0)]); + self.0.append_lhs_at(indexes, output); + } + } + + struct MapWithSelection(Vec); + + impl Value for MapWithSelection { + type Read = MapRead; + type ValueType = Map>; + } + + impl MapValue for MapWithSelection { + type Element = Array; + type Item<'a> = AppendingSelection<'a>; + type Iter<'a> = std::iter::Once>; + + fn get(&self, key: &[u8]) -> Option> { + (key == b"values").then_some(AppendingSelection(&self.0)) + } + + fn values(&self) -> Self::Iter<'_> { + std::iter::once(AppendingSelection(&self.0)) + } + + fn for_each_entry<'a>(&'a self, mut visitor: impl FnMut(&[u8], Self::Item<'a>)) { + visitor(b"values", AppendingSelection(&self.0)); + } + } + + let source = MapWithSelection(vec!["selected".to_owned(), "other".to_owned()]); + let mut selected = vec![LhsValue::Bool(true)]; + (&source).append_lhs_at( + &[FieldIndex::MapEach, FieldIndex::ArrayIndex(0)], + &mut selected, + ); + assert_eq!(selected.len(), 2); + assert_eq!(selected[0], LhsValue::Bool(true)); + assert!(matches!( + selected[1], + LhsValue::Bytes(crate::Bytes::Borrowed(_)) + )); + assert_eq!(bytes(&selected[1]).as_ptr(), source.0[0].as_ptr()); + } + + #[test] + fn test_owned_containers_append_selected_values_with_owned_payloads() { + fn check(value: impl IntoLhsValue<'static>, indexes: &[FieldIndex]) { + let mut selected = vec![LhsValue::Bool(true)]; + value.append_lhs_at(indexes, &mut selected); + assert_eq!(selected.len(), 2); + assert_eq!(selected[0], LhsValue::Bool(true)); + assert!(matches!( + selected[1], + LhsValue::Bytes(crate::Bytes::Owned(_)) + )); + assert_eq!(bytes(&selected[1]), b"selected"); + } + + let rows = vec![vec!["selected".to_owned()], vec![]]; + check( + vec![vec![vec!["other".to_owned()]], rows.clone()], + &[ + FieldIndex::ArrayIndex(1), + FieldIndex::MapEach, + FieldIndex::ArrayIndex(0), + ], + ); + check( + Box::new(rows.clone()), + &[FieldIndex::MapEach, FieldIndex::ArrayIndex(0)], + ); + check( + BTreeMap::from([ + (Box::<[u8]>::from(&b"wanted"[..]), rows), + ( + Box::<[u8]>::from(&b"other"[..]), + vec![vec!["other".to_owned()]], + ), + ]), + &[ + FieldIndex::MapKey("wanted".into()), + FieldIndex::MapEach, + FieldIndex::ArrayIndex(0), + ], + ); + } + + #[test] + fn test_default_descriptor_conversion_owns_temporary_payload() { + struct Temporary(Box<[u8]>); + + impl AsRef<[u8]> for Temporary { + fn as_ref(&self) -> &[u8] { + &self.0 + } + } + + impl Value for Temporary { + type Read = BytesRead; + type ValueType = Bytes; + } + + impl IntoLhsValue<'_> for Temporary {} + + let result: LhsValue<'static> = Temporary(Box::from(&b"temporary"[..])).into_lhs(); + assert_eq!(bytes(&result), b"temporary"); + } + + #[test] + fn test_owned_primitive_and_collection_conversion() { + assert_eq!(42i64.into_lhs(), LhsValue::Int(42)); + assert_eq!(true.into_lhs(), LhsValue::Bool(true)); + let ip = IpAddr::V4(std::net::Ipv4Addr::LOCALHOST); + assert_eq!(ip.into_lhs(), LhsValue::Ip(ip)); + let value: LhsValue<'static> = vec!["one".to_owned()].into_lhs(); + let LhsValue::Array(array) = value else { + panic!("expected an array") + }; + assert_eq!(bytes(array.get(0).unwrap()), b"one"); + } + + struct Matches<'a> { + needle: &'a [u8], + reads: &'a Cell, + } + + impl ValuePredicate for Matches<'_> { + fn bytes(&self, value: &[u8]) -> bool { + self.reads.set(self.reads.get() + 1); + value == self.needle + } + } + + #[test] + fn test_scalar_comparison_distinguishes_false_from_missing() { + type Input = Box, Vec>>>; + let input: Input = Box::new(vec![BTreeMap::from([( + Box::from(b"key".as_slice()), + vec!["miss".into(), "hit".into()], + )])]); + let reads = Cell::new(0); + let predicate = Matches { + needle: b"hit", + reads: &reads, + }; + // Exercise both reference and Box forwarding through nested selections. + for (outer, key, inner, expected) in [ + (0, "key", 0, Some(false)), + (0, "key", 1, Some(true)), + (0, "key", 2, None), + (0, "absent", 0, None), + (1, "key", 0, None), + (u32::MAX, "key", 0, None), + (0, "key", u32::MAX, None), + ] { + let path = ::ValueType::compile_path( + &[ + FieldIndex::ArrayIndex(outer), + FieldIndex::MapKey(key.into()), + FieldIndex::ArrayIndex(inner), + ], + Type::Bytes, + ) + .unwrap(); + assert_eq!(compare_one(&&input, &path, &predicate), expected); + } + assert_eq!(reads.get(), 2); + } + + #[test] + fn test_array_comparison_appends_in_map_order() { + type Input = Box, Vec>>; + let input: Input = Box::new(BTreeMap::from([ + (Box::from(b"c".as_slice()), vec!["hit".into()]), + ( + Box::from(b"a".as_slice()), + vec!["miss".into(), "hit".into()], + ), + (Box::from(b"b".as_slice()), vec![]), + ])); + let reads = Cell::new(0); + let predicate = Matches { + needle: b"hit", + reads: &reads, + }; + let path = ::ValueType::compile_path( + &[FieldIndex::MapEach, FieldIndex::MapEach], + Type::Bytes, + ) + .unwrap(); + let mut output = TypedArray::from_iter([true]); + compare_vec(&&input, &path, &predicate, &mut output); + assert_eq!(output, [true, false, true, true]); + assert_eq!(reads.get(), 3); + + // Scalar-valued maps also append without replacing earlier results. + let values = BTreeMap::from([(Box::<[u8]>::from(b"a".as_slice()), "miss".to_owned())]); + let path = Map::::compile_path(&[FieldIndex::MapEach], Type::Bytes).unwrap(); + compare_vec(&values, &path, &predicate, &mut output); + assert_eq!(output, [true, false, true, true, false]); + assert_eq!(reads.get(), 4); + } + + #[test] + #[should_panic(expected = "[*] in scalar comparison path")] + fn test_scalar_array_comparison_rejects_map_each() { + let path = Array::::compile_path(&[FieldIndex::MapEach], Type::Bytes).unwrap(); + compare_one( + &Vec::::new(), + &path, + &Matches { + needle: b"hit", + reads: &Cell::new(0), + }, + ); + } + + #[test] + #[should_panic(expected = "[*] in scalar comparison path")] + fn test_scalar_map_comparison_rejects_map_each() { + let path = Map::::compile_path(&[FieldIndex::MapEach], Type::Bytes).unwrap(); + compare_one( + &BTreeMap::, String>::new(), + &path, + &Matches { + needle: b"hit", + reads: &Cell::new(0), + }, + ); + } + + #[test] + fn test_custom_predicate_traverses_nested_projections_without_missing_values() { + type Input = Vec, Vec>>; + let key = Box::<[u8]>::from(&b"key"[..]); + let input: Input = vec![ + BTreeMap::from([(key.clone(), vec!["one".into(), "two".into()])]), + BTreeMap::new(), + BTreeMap::from([(key, vec!["two".into()])]), + ]; + let reads = Cell::new(0); + let predicate = Matches { + needle: b"two", + reads: &reads, + }; + let path = ::ValueType::compile_path( + &[ + FieldIndex::MapEach, + FieldIndex::MapKey("key".into()), + FieldIndex::MapEach, + ], + Type::Bytes, + ) + .unwrap(); + let mut results = TypedArray::new(); + compare_vec(&input, &path, &predicate, &mut results); + assert_eq!(results, [false, true, true]); + assert_eq!(reads.get(), 3); + + let absent = ::ValueType::compile_path( + &[ + FieldIndex::ArrayIndex(8), + FieldIndex::MapKey("key".into()), + FieldIndex::ArrayIndex(0), + ], + Type::Bytes, + ) + .unwrap(); + assert_eq!(compare_one(&input, &absent, &predicate), None); + assert_eq!(reads.get(), 3); + + for indexes in [ + [ + FieldIndex::ArrayIndex(u32::MAX), + FieldIndex::MapKey("key".into()), + FieldIndex::MapEach, + ], + [ + FieldIndex::MapEach, + FieldIndex::MapKey("key".into()), + FieldIndex::ArrayIndex(u32::MAX), + ], + ] { + let path = ::ValueType::compile_path(&indexes, Type::Bytes).unwrap(); + compare_vec(&input, &path, &predicate, &mut results); + assert_eq!(results, [false, true, true]); + assert_eq!(reads.get(), 3); + } + } + + #[test] + fn test_custom_predicate_path_validates_value_type() { + type Nested = Array>>; + let path = [ + FieldIndex::MapEach, + FieldIndex::MapKey("key".into()), + FieldIndex::MapEach, + ]; + assert!(Nested::compile_path(&path, Type::Bytes).is_ok()); + assert!(Nested::compile_path(&path, Type::Int).is_err()); + assert!(Nested::compile_path(&path[..2], Type::Bytes).is_err()); + assert!( + Array::::compile_path(&[FieldIndex::MapKey("key".into())], Type::Int).is_err() + ); + assert!(Map::::compile_path(&[FieldIndex::ArrayIndex(0)], Type::Int).is_err()); + assert!(Int::compile_path(&[FieldIndex::MapEach], Type::Int).is_err()); + assert!(Int::compile_path(&[], Type::Int).is_ok()); + assert!(Ip::compile_path(&[], Type::Ip).is_ok()); + assert!(Bool::compile_path(&[], Type::Bool).is_ok()); + assert!(Bytes::compile_path(&[], Type::array(Type::Bytes)).is_err()); + } + + #[test] + fn test_custom_predicate_receives_each_scalar_kind() { + struct IntPredicate; + impl ValuePredicate for IntPredicate { + fn int(&self, value: i64) -> bool { + value == 42 + } + } + struct IpPredicate; + impl ValuePredicate for IpPredicate { + fn ip(&self, value: IpAddr) -> bool { + value.is_loopback() + } + } + struct BoolPredicate; + impl ValuePredicate for BoolPredicate { + fn boolean(&self, value: bool) -> bool { + !value + } + } + let mut results = TypedArray::new(); + compare_vec(&42i64, &(), &IntPredicate, &mut results); + compare_vec( + &IpAddr::V4(std::net::Ipv4Addr::LOCALHOST), + &(), + &IpPredicate, + &mut results, + ); + compare_vec(&false, &(), &BoolPredicate, &mut results); + assert_eq!(results, [true, true, true]); + } +} diff --git a/engine/tests/native_collection_comparisons.rs b/engine/tests/native_collection_comparisons.rs new file mode 100644 index 00000000..d795bd91 --- /dev/null +++ b/engine/tests/native_collection_comparisons.rs @@ -0,0 +1,291 @@ +//! Custom collection comparisons select native storage before materialization. + +use std::cell::RefCell; +use std::collections::BTreeMap; +use std::sync::Arc; +use wirefilter::values::{ + self, ArrayRead, ArrayValue, IntoLhsValue, MapRead, MapValue, Value, ValueType, +}; +use wirefilter::{ + Compare, CompiledExpr, ContextFamily, ContextState, DefaultCompiler, EvaluationContext, + ExecutionContext, FieldIndex, HasScheme, IdentifierExpr, IndexExpr, LhsValue, List, + ListMatcher, ListRef, Scheme, SchemeBuilder, Type, +}; + +thread_local! { + static MATERIALIZED: RefCell> = const { RefCell::new(Vec::new()) }; +} + +struct CountedArray { + label: String, + items: Vec, +} + +impl Value for CountedArray +where + T::ValueType: ValueType, +{ + type Read = ArrayRead; + type ValueType = values::Array; +} + +impl ArrayValue for CountedArray +where + T::ValueType: ValueType, +{ + type Element = T::ValueType; + type Item<'a> + = &'a T + where + Self: 'a; + type Iter<'a> + = std::slice::Iter<'a, T> + where + Self: 'a; + + fn get(&self, index: usize) -> Option> { + self.items.get(index) + } + + fn iter(&self) -> Self::Iter<'_> { + MATERIALIZED.with_borrow_mut(|values| values.push(self.label.clone())); + self.items.iter() + } +} + +type Row = CountedArray; +type Section = CountedArray; +type Group = CountedArray

; +type Groups = BTreeMap, Group>; + +// This by-value descriptor intentionally keeps the default into_lhs_at method. +#[derive(Clone)] +struct Descriptor(Arc); + +impl Value for Descriptor { + type Read = MapRead; + type ValueType = values::Map<::ValueType>; +} + +impl IntoLhsValue<'_> for Descriptor {} + +impl MapValue for Descriptor { + type Element = ::ValueType; + type Item<'a> = &'a Group; + type Iter<'a> = std::collections::btree_map::Values<'a, Box<[u8]>, Group>; + + fn get(&self, key: &[u8]) -> Option> { + self.0.get(key) + } + + fn values(&self) -> Self::Iter<'_> { + self.0.values() + } + + fn for_each_entry<'a>(&'a self, mut visitor: impl FnMut(&[u8], Self::Item<'a>)) { + for (key, value) in self.0.iter() { + visitor(key, value); + } + } +} + +struct Context { + state: ContextState<()>, + groups: Option>, +} + +impl HasScheme for Context { + fn scheme(&self) -> &Scheme { + self.state.scheme() + } +} + +impl EvaluationContext for Context { + type UserData = (); + + fn get_user_data(&self) -> &() { + &() + } + + fn get_list_matcher(&self, list: ListRef<'_>) -> &dyn ListMatcher { + self.state.get_list_matcher(list) + } + + fn get_list_matcher_unchecked(&self, list: &List) -> &dyn ListMatcher { + EvaluationContext::get_list_matcher_unchecked(&self.state, list) + } +} + +wirefilter::wirefilter_fields! { + impl Fields for Context { + "groups" => { + fn(context: &Context) -> Option<&Groups> { + context.groups.as_deref() + } + }, + "owned" => { + fn(context: &Context) -> Option { + context.groups.clone().map(Descriptor) + } + }, + } +} + +struct Nonempty; + +impl> Compare<(), F> for Nonempty { + fn compare<'ctx, 'data: 'ctx>( + &self, + value: &LhsValue<'ctx>, + _: &'ctx F::Context<'data>, + ) -> bool { + match value { + LhsValue::Array(array) => !array.is_empty(), + LhsValue::Map(map) => !map.is_empty(), + _ => panic!("custom comparator requires a collection"), + } + } +} + +fn fixture(present: bool) -> Context { + let mut builder = SchemeBuilder::new(); + let ty = Type::map(Type::array(Type::array(Type::array(Type::Bytes)))); + builder.add_optional_field("groups", ty).unwrap(); + builder.add_optional_field("owned", ty).unwrap(); + let scheme = builder.build(); + Fields::validate_scheme(&scheme).unwrap(); + let groups = present.then(|| { + Arc::new( + ["wanted", "other"] + .into_iter() + .map(|name| { + let group = CountedArray { + label: name.into(), + items: (0..2) + .map(|section| CountedArray { + label: format!("{name}/{section}"), + items: (0..2) + .map(|row| CountedArray { + label: format!("{name}/{section}/{row}"), + items: if row == 0 { + vec!["value".into()] + } else { + vec![] + }, + }) + .collect(), + }) + .collect(), + }; + (Box::<[u8]>::from(name.as_bytes()), group) + }) + .collect(), + ) + }); + Context { + state: ContextState::new(&scheme), + groups, + } +} + +fn evaluate(native: &Context, field: &str, indexes: Vec, default: bool) -> Vec { + let scheme = native.scheme(); + let mut stored = ExecutionContext::new(scheme); + if let Some(groups) = &native.groups { + stored + .set_field_value_from_name(field, groups.as_ref().into_lhs()) + .unwrap(); + } + let index = IndexExpr { + identifier: IdentifierExpr::Field(scheme.get_field(field).unwrap().to_owned()), + indexes: indexes.into_boxed_slice(), + }; + let stored_expr = + index + .clone() + .compile_with(&mut DefaultCompiler::<()>::new(), default, Nonempty); + let native_expr = + index.compile_with(&mut DefaultCompiler::<(), Fields>::new(), default, Nonempty); + MATERIALIZED.with_borrow_mut(Vec::clear); + match (stored_expr, native_expr) { + (CompiledExpr::One(stored_expr), CompiledExpr::One(native_expr)) => { + assert_eq!(stored_expr.execute(&stored), native_expr.execute(native)); + } + (CompiledExpr::Vec(stored_expr), CompiledExpr::Vec(native_expr)) => { + assert_eq!(stored_expr.execute(&stored), native_expr.execute(native)); + } + _ => panic!("comparison cardinality changed"), + } + MATERIALIZED.with_borrow_mut(std::mem::take) +} + +#[test] +fn borrowed_comparisons_only_materialize_the_selected_subtree() { + use FieldIndex::{ArrayIndex as At, MapEach as Each, MapKey}; + let native = fixture(true); + for (path, selected) in [ + (vec![MapKey("wanted".into()), At(0), At(0)], "wanted/0/0"), + (vec![MapKey("wanted".into()), At(0), Each], "wanted/0"), + (vec![MapKey("wanted".into()), Each, At(0)], "wanted"), + (vec![MapKey("wanted".into()), Each, Each], "wanted"), + // Missing children after [*] still produce the same empty comparison vector. + (vec![MapKey("wanted".into()), Each, At(99)], "wanted"), + ] { + let materialized = evaluate(&native, "groups", path, false); + assert!(!materialized.is_empty()); + assert!( + materialized.iter().all(|name| name.starts_with(selected)), + "converted unrelated containers: {materialized:?}" + ); + } +} + +#[test] +fn missing_selections_and_optional_roots_keep_stored_comparison_semantics() { + use FieldIndex::{ArrayIndex as At, MapEach as Each, MapKey}; + let present = fixture(true); + let absent = fixture(false); + for default in [false, true] { + for path in [ + vec![MapKey("missing".into()), At(0)], + vec![MapKey("wanted".into()), At(99)], + vec![MapKey("missing".into()), Each], + vec![MapKey("missing".into()), Each, At(0)], + vec![MapKey("wanted".into()), At(99), Each], + ] { + assert!(evaluate(&present, "groups", path, default).is_empty()); + } + for path in [ + vec![MapKey("wanted".into()), At(0)], + vec![MapKey("wanted".into()), Each], + vec![MapKey("wanted".into()), Each, At(0)], + ] { + assert!(evaluate(&absent, "groups", path, default).is_empty()); + } + } +} + +#[test] +fn by_value_descriptors_keep_the_default_conversion_before_selection() { + use FieldIndex::{ArrayIndex as At, MapEach as Each, MapKey}; + let native = fixture(true); + for path in [ + vec![MapKey("wanted".into()), At(0), At(0)], + vec![MapKey("wanted".into()), At(0), Each], + vec![MapKey("wanted".into()), Each, At(0)], + ] { + let materialized = evaluate(&native, "owned", path, false); + assert!(materialized.iter().any(|name| name == "wanted")); + assert!(materialized.iter().any(|name| name == "other")); + } +} + +#[test] +fn comparisons_without_a_concrete_prefix_still_materialize_the_whole_map() { + let native = fixture(true); + for path in [vec![], vec![FieldIndex::MapEach]] { + let materialized = evaluate(&native, "groups", path, false); + assert!(materialized.iter().any(|name| name == "wanted")); + assert!(materialized.iter().any(|name| name == "other")); + } +} diff --git a/engine/tests/native_family_validation.rs b/engine/tests/native_family_validation.rs new file mode 100644 index 00000000..82c9eaf7 --- /dev/null +++ b/engine/tests/native_family_validation.rs @@ -0,0 +1,164 @@ +//! Startup validation for context families implemented without the field macro. + +use std::cell::RefCell; +use std::collections::BTreeMap; +use wirefilter::{ + BorrowedValueProvider, ContextFamily, ExecutionContext, Field, FieldBindingError, + FieldConsumer, FieldDefinitionError, FieldGetter, LhsValue, LhsValueProvider, + OwnedValueProvider, Scheme, SchemeBuilder, SchemeValidationError, Type, +}; + +thread_local! { + static BINDINGS: RefCell> = const { RefCell::new(Vec::new()) }; +} + +struct Fields; +struct IntegerGetter; +struct HeadersGetter; +struct StoredGetter; + +type Headers = BTreeMap, Vec>; + +impl ContextFamily for Fields { + type Context<'data> = ExecutionContext<'data>; + type UserData = (); + + fn bind>( + field: Field, + consumer: B, + ) -> Result { + BINDINGS.with_borrow_mut(|bindings| bindings.push(field.name().into())); + match field.name() { + "count" | "after_errors" => consumer.consume(field, IntegerGetter), + "headers" => consumer.consume(field, HeadersGetter), + "stored_scalar" | "stored_collection" => consumer.consume(field, StoredGetter), + "unsupported" => Err(FieldBindingError::Unsupported( + "application binding rejected", + )), + name => Err(FieldBindingError::UnknownField(name.into())), + } + } +} + +impl FieldGetter for IntegerGetter { + type Provider = OwnedValueProvider; + + fn get<'ctx, 'data: 'ctx>(&self, _: &'ctx ExecutionContext<'data>, _: &Field) -> Option { + panic!("scheme validation must not read integer values") + } +} + +impl FieldGetter for HeadersGetter { + type Provider = BorrowedValueProvider; + + fn get<'ctx, 'data: 'ctx>( + &self, + _: &'ctx ExecutionContext<'data>, + _: &Field, + ) -> Option<&'ctx Headers> { + panic!("scheme validation must not read header values") + } +} + +impl FieldGetter for StoredGetter { + type Provider = LhsValueProvider; + + fn get<'ctx, 'data: 'ctx>( + &self, + _: &'ctx ExecutionContext<'data>, + _: &Field, + ) -> Option<&'ctx LhsValue<'ctx>> { + panic!("scheme validation must not read stored values") + } +} + +fn validate(scheme: &Scheme) -> Result<(), SchemeValidationError> { + BINDINGS.with_borrow_mut(Vec::clear); + let result = Fields::validate_scheme(scheme); + let bindings = BINDINGS.with_borrow_mut(std::mem::take); + let expected: Vec<_> = scheme + .fields() + .map(|field| field.name().to_owned()) + .collect(); + assert_eq!( + bindings, expected, + "every scheme field must bind exactly once" + ); + result +} + +#[test] +fn default_validation_checks_scalar_and_nested_collection_types_without_reading_values() { + let mut builder = SchemeBuilder::new(); + builder.add_field("count", Type::Int).unwrap(); + builder + .add_optional_field("headers", Type::map(Type::array(Type::Bytes))) + .unwrap(); + + // No request context or field values are needed for validation. + assert_eq!(validate(&builder.build()), Ok(())); +} + +#[test] +fn default_validation_aggregates_binding_errors_for_unused_and_optional_fields() { + let mut builder = SchemeBuilder::new(); + builder.add_field("count", Type::Bytes).unwrap(); + builder + .add_field("headers", Type::map(Type::array(Type::Int))) + .unwrap(); + builder.add_field("missing", Type::Int).unwrap(); + builder + .add_optional_field("missing_optional", Type::Bool) + .unwrap(); + builder.add_field("unsupported", Type::Bool).unwrap(); + builder.add_field("after_errors", Type::Int).unwrap(); + + // No filter references these fields: startup checks the complete scheme. + let errors = validate(&builder.build()).unwrap_err(); + let expected = [ + ( + "count", + FieldBindingError::TypeMismatch { + field: "count".into(), + expected: Type::Bytes, + actual: Type::Int, + }, + ), + ( + "headers", + FieldBindingError::TypeMismatch { + field: "headers".into(), + expected: Type::map(Type::array(Type::Int)), + actual: Type::map(Type::array(Type::Bytes)), + }, + ), + ("missing", FieldBindingError::UnknownField("missing".into())), + ( + "missing_optional", + FieldBindingError::UnknownField("missing_optional".into()), + ), + ( + "unsupported", + FieldBindingError::Unsupported("application binding rejected"), + ), + ] + .into_iter() + .map(|(name, error)| FieldDefinitionError::Binding { + name: name.into(), + error, + }) + .collect::>(); + assert_eq!(errors, SchemeValidationError(expected)); +} + +#[test] +fn default_validation_accepts_runtime_typed_providers_without_reading_values() { + let mut builder = SchemeBuilder::new(); + builder.add_field("stored_scalar", Type::Ip).unwrap(); + builder + .add_optional_field("stored_collection", Type::array(Type::map(Type::Bool))) + .unwrap(); + + // LhsValueProvider has no static type; its context must check actual values. + assert_eq!(validate(&builder.build()), Ok(())); +} diff --git a/engine/tests/native_getter_attributes.rs b/engine/tests/native_getter_attributes.rs new file mode 100644 index 00000000..61a2ea42 --- /dev/null +++ b/engine/tests/native_getter_attributes.rs @@ -0,0 +1,224 @@ +#![deny(unused_variables)] + +use std::cell::Cell; +use wirefilter::values::{self, SliceView}; +use wirefilter::{ + DefaultCompiler, EvaluationContext, Field, HasScheme, LhsValue, ListMatcher, ListRef, + ProviderFamily, Scheme, SchemeBuilder, Type, +}; + +struct Request<'data> { + scheme: &'data Scheme, + names: &'data [Box], + number: Option, + accesses: Cell, +} + +#[test] +fn default_compiler_traits_do_not_require_user_data_or_family_traits() { + struct UserData; + + fn assert_compiler_traits() {} + + assert_compiler_traits::>(); + assert_compiler_traits::>(); + assert_compiler_traits::>(); +} + +impl Request<'_> { + fn record_access(&self) { + self.accesses.set(self.accesses.get() + 1); + } +} + +impl HasScheme for Request<'_> { + fn scheme(&self) -> &Scheme { + self.scheme + } +} + +impl EvaluationContext for Request<'_> { + type UserData = (); + + fn get_user_data(&self) -> &() { + &() + } + + fn get_list_matcher(&self, _: ListRef<'_>) -> &dyn ListMatcher { + unreachable!("the test scheme has no lists") + } +} + +struct NamesProvider; + +impl ProviderFamily for NamesProvider { + type Storage<'data> = &'data [Box]; + type Value<'ctx, 'data> + = SliceView<'data, Box> + where + 'data: 'ctx; + type ValueType = values::Array; + + fn reborrow<'ctx, 'data: 'ctx>(value: &'ctx Self::Storage<'data>) -> Self::Value<'ctx, 'data> { + SliceView(value) + } +} + +// The unused locals require attributes to reach the function containing each +// getter body; applying them only to the FieldGetter adapter is insufficient. +wirefilter::wirefilter_fields! { + impl<'data> RequestFields for Request<'data> { + "name" => { + provider wirefilter::BorrowedValueProvider; + #[inline(never)] + #[allow(unused_variables)] + fn(request: &Request<'data>, field: &Field) -> &str { + let intentionally_unused = (); + assert_eq!(field.name(), "name"); + request.record_access(); + if let Some(name) = request.names.first() { + return name.as_ref(); + } + "" + } + }, + "count" => { + #[inline(always)] + #[cfg_attr(test, allow(unused_variables))] + fn(request: &Request<'data>) -> i64 { + let intentionally_unused = (); + request.record_access(); + request.names.len() as i64 + } + }, + "maybe_name" => { + #[inline(never)] + #[cfg_attr(test, allow(unused_variables))] + fn(request: &Request<'data>) -> Option<&str> { + let intentionally_unused = (); + request.record_access(); + let name = request.names.first()?; + if name.is_empty() { + return None; + } + Some(name.as_ref()) + } + }, + "maybe_number" => { + #[inline(always)] + #[allow(unused_variables)] + fn(request: &Request<'data>) -> Option { + let intentionally_unused = (); + request.record_access(); + let number = request.number?; + if number < 0 { + return None; + } + Some(number) + } + }, + "selection" => { + provider NamesProvider; + #[inline(never)] + #[cfg_attr(test, allow(unused_variables))] + fn(request: &Request<'data>, field: &Field) -> Option>> { + let intentionally_unused = (); + assert_eq!(field.name(), "selection"); + request.record_access(); + request.names.first()?; + Some(SliceView(request.names)) + } + }, + } +} + +fn scheme() -> Scheme { + let mut builder = SchemeBuilder::new(); + builder.add_field("name", Type::Bytes).unwrap(); + builder.add_field("count", Type::Int).unwrap(); + builder + .add_optional_field("maybe_name", Type::Bytes) + .unwrap(); + builder + .add_optional_field("maybe_number", Type::Int) + .unwrap(); + builder + .add_optional_field("selection", Type::array(Type::Bytes)) + .unwrap(); + let scheme = builder.build(); + RequestFields::validate_scheme(&scheme).unwrap(); + scheme +} + +#[test] +fn test_attributed_getters_preserve_returns_and_optional_control_flow() { + let scheme = scheme(); + let names = [Box::::from("alpha"), Box::::from("beta")]; + let empty_name = [Box::::from("")]; + let mut compiler = DefaultCompiler::<(), RequestFields>::new(); + let filters = [ + r#"name eq "alpha""#, + "count eq 2", + r#"maybe_name eq "alpha""#, + "maybe_number eq 7", + r#"any(selection[*] eq "alpha")"#, + ] + .map(|expression| { + scheme + .parse(expression) + .unwrap() + .compile_with_compiler(&mut compiler) + }); + + for (names, number, expected) in [ + (names.as_slice(), Some(7), [true, true, true, true, true]), + (names.as_slice(), None, [true, true, true, false, true]), + (names.as_slice(), Some(-1), [true, true, true, false, true]), + ([].as_slice(), Some(7), [false, false, false, true, false]), + ( + empty_name.as_slice(), + Some(7), + [false, false, false, true, false], + ), + ] { + let request = Request { + scheme: &scheme, + names, + number, + accesses: Cell::new(0), + }; + for (index, (filter, expected)) in filters.iter().zip(expected).enumerate() { + assert_eq!(filter.execute(&request).unwrap(), expected); + assert_eq!(request.accesses.get(), index + 1); + } + } +} + +#[test] +fn test_attributed_getters_preserve_borrowed_payloads() { + let scheme = scheme(); + let names = [Box::::from("alpha")]; + let request = Request { + scheme: &scheme, + names: &names, + number: None, + accesses: Cell::new(0), + }; + let mut compiler = DefaultCompiler::<(), RequestFields>::new(); + for field in ["name", "maybe_name", "selection"] { + let filter = scheme + .parse_value(field) + .unwrap() + .compile_with_compiler(&mut compiler); + let value = filter.execute(&request).unwrap().unwrap(); + let bytes = match &value { + LhsValue::Bytes(bytes) => bytes, + LhsValue::Array(values) => match values.get(0).unwrap() { + LhsValue::Bytes(bytes) => bytes, + _ => panic!("expected array of byte strings"), + }, + _ => panic!("expected bytes or array"), + }; + assert_eq!(bytes.as_ref().as_ptr(), names[0].as_ptr(), "{field}"); + } +} diff --git a/engine/tests/native_getter_hygiene.rs b/engine/tests/native_getter_hygiene.rs new file mode 100644 index 00000000..a7cdffd2 --- /dev/null +++ b/engine/tests/native_getter_hygiene.rs @@ -0,0 +1,128 @@ +use wirefilter::{ + DefaultCompiler, EvaluationContext, Field, HasScheme, LhsValue, ListMatcher, ListRef, Scheme, + SchemeBuilder, Type, +}; + +// These names belong to the application. Generated getter helpers must not +// replace them when resolving the context type or expressions in a getter body. +struct Getter<'data> { + scheme: &'data Scheme, + number: i64, + optional: Option, + name: &'data str, +} + +impl Getter<'_> { + fn read(&self) -> i64 { + self.number + } + + fn read_optional(&self) -> Option { + self.optional + } + + fn name(&self) -> &str { + self.name + } +} + +impl HasScheme for Getter<'_> { + fn scheme(&self) -> &Scheme { + self.scheme + } +} + +impl EvaluationContext for Getter<'_> { + type UserData = (); + + fn get_user_data(&self) -> &() { + &() + } + + fn get_list_matcher(&self, _: ListRef<'_>) -> &dyn ListMatcher { + unreachable!("this test has no lists") + } +} + +fn value(context: &Getter<'_>) -> i64 { + context.number + 1 +} + +fn call(context: &Getter<'_>) -> i64 { + context.number + 2 +} + +fn __wirefilter_call(context: &Getter<'_>) -> i64 { + context.number + 3 +} + +wirefilter::wirefilter_fields! { + impl<'data> Fields for Getter<'data> { + "required" => { + fn(field: &Getter<'data>, metadata: &Field) -> i64 { + assert_eq!(metadata.name(), "required"); + Getter::read(field) + value(field) + call(field) + __wirefilter_call(field) + } + }, + "optional" => { + fn(consumer: &Getter<'data>, field: &Field) -> Option { + assert_eq!(field.name(), "optional"); + assert!(field.optional()); + let number = Getter::read_optional(consumer)?; + Some(number + value(consumer) + call(consumer) + __wirefilter_call(consumer)) + } + }, + "name" => { + fn(context: &Getter<'data>) -> &str { + Getter::name(context) + } + }, + } +} + +#[test] +fn test_getter_bodies_resolve_application_items() { + let mut builder = SchemeBuilder::new(); + builder.add_field("required", Type::Int).unwrap(); + builder.add_optional_field("optional", Type::Int).unwrap(); + builder.add_field("name", Type::Bytes).unwrap(); + let scheme = builder.build(); + Fields::validate_scheme(&scheme).unwrap(); + let mut compiler = DefaultCompiler::<(), Fields>::new(); + let values = ["required", "optional", "name"].map(|field| { + scheme + .parse_value(field) + .unwrap() + .compile_with_compiler(&mut compiler) + }); + let comparison = scheme + .parse(r#"required eq 34 and optional eq 38 and name eq "example""#) + .unwrap() + .compile_with_compiler(&mut compiler); + + for (number, optional, expected_required, expected_optional, matches) in [ + (7, Some(11), 34, Some(38), true), + (7, None, 34, None, false), + (-2, Some(3), -2, Some(3), false), + ] { + let context = Getter { + scheme: &scheme, + number, + optional, + name: "example", + }; + assert_eq!( + values[0].execute(&context).unwrap(), + Ok(LhsValue::Int(expected_required)), + ); + assert_eq!( + values[1].execute(&context).unwrap(), + expected_optional.map(LhsValue::Int).ok_or(Type::Int), + ); + assert_eq!( + values[2].execute(&context).unwrap(), + Ok(LhsValue::from("example")), + ); + assert_eq!(comparison.execute(&context).unwrap(), matches); + } +} diff --git a/engine/tests/native_lhs_values.rs b/engine/tests/native_lhs_values.rs new file mode 100644 index 00000000..8e3e5f63 --- /dev/null +++ b/engine/tests/native_lhs_values.rs @@ -0,0 +1,662 @@ +//! Default and downstream contexts expose runtime-typed values through shared field binding. + +use std::cell::Cell; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; +use wirefilter::{ + Array, Compare, CompiledExpr, CompiledValueExpr, Compiler, ContextFamily, DefaultCompiler, + EvaluationContext, ExecutionContext, ExecutionContextFamily, Field, FieldBindingError, + FieldConsumer, FieldDefinitionError, FieldGetter, FieldIndex, FunctionArgs, + FunctionCallArgExpr, GetType, HasScheme, IdentifierExpr, IndexExpr, LhsValue, LhsValueProvider, + ListMatcher, ListRef, Map, Scheme, SchemeBuilder, SimpleFunctionArgKind, + SimpleFunctionDefinition, SimpleFunctionImpl, SimpleFunctionParam, Type, TypeMismatchError, +}; + +#[derive(Default)] +struct Reads { + map: Cell, +} + +struct NativeFields; +struct Getter; + +impl ContextFamily for NativeFields { + type Context<'data> = ExecutionContext<'data, Reads>; + type UserData = Reads; + + fn bind>( + field: Field, + consumer: B, + ) -> Result { + consumer.consume(field, Getter) + } +} + +impl FieldGetter for Getter { + type Provider = LhsValueProvider; + + fn get<'ctx, 'data: 'ctx>( + &self, + context: &'ctx ExecutionContext<'data, Reads>, + field: &Field, + ) -> Option<&'ctx LhsValue<'ctx>> { + if field.name() == "map" { + let count = &context.get_user_data().map; + count.set(count.get() + 1); + } + context.get_field_value(field.as_ref()) + } +} + +fn array<'a>(ty: Type, values: impl IntoIterator>) -> LhsValue<'a> { + Array::try_from_iter(ty, values).unwrap().into() +} + +fn map<'a>(ty: Type, entries: impl IntoIterator)>) -> LhsValue<'a> { + Map::try_from_iter( + ty, + entries + .into_iter() + .map(|(key, value)| Ok::<_, TypeMismatchError>((key.into(), value))), + ) + .unwrap() + .into() +} + +fn identity<'a>(args: FunctionArgs<'_, 'a>) -> Option> { + args.next()?.ok() +} + +fn add_function( + builder: &mut SchemeBuilder, + name: &str, + params: &[Type], + return_type: Type, + implementation: SimpleFunctionImpl, +) { + builder + .add_function( + name, + SimpleFunctionDefinition { + params: params + .iter() + .map(|&val_type| SimpleFunctionParam { + arg_kind: SimpleFunctionArgKind::Field, + val_type, + }) + .collect(), + opt_params: Box::new([]), + return_type, + implementation, + }, + ) + .unwrap(); +} + +fn assert_filter( + scheme: &Scheme, + context: &ExecutionContext<'_, Reads>, + expression: &str, + expected: bool, +) { + let ast = scheme.parse(expression).unwrap(); + let default = ast + .clone() + .compile_with_compiler(&mut DefaultCompiler::::new()); + let native = ast.compile_with_compiler(&mut DefaultCompiler::::new()); + assert_eq!( + default.execute(context).unwrap(), + expected, + "default: {expression}" + ); + assert_eq!( + native.execute(context).unwrap(), + expected, + "native: {expression}" + ); +} + +#[test] +fn one_family_handles_all_scalar_types_and_optional_fields() { + let mut builder = SchemeBuilder::new(); + for (name, ty) in [ + ("number", Type::Int), + ("text", Type::Bytes), + ("flag", Type::Bool), + ("v4", Type::Ip), + ("v6", Type::Ip), + ] { + builder.add_field(name, ty).unwrap(); + } + builder.add_optional_field("missing", Type::Int).unwrap(); + let scheme = builder.build(); + let mut context = ExecutionContext::new_with(&scheme, Reads::default); + context.set_field_value_from_name("number", 42i64).unwrap(); + context.set_field_value_from_name("text", "target").unwrap(); + context.set_field_value_from_name("flag", true).unwrap(); + context + .set_field_value_from_name("v4", IpAddr::V4(Ipv4Addr::new(192, 0, 2, 1))) + .unwrap(); + context + .set_field_value_from_name("v6", IpAddr::V6(Ipv6Addr::LOCALHOST)) + .unwrap(); + for (expression, expected) in [ + ("number ge 42", true), + ("number lt 42", false), + ("number & 2", true), + (r#"text eq "target""#, true), + (r#"text contains "arg""#, true), + ("flag", true), + ("not flag", false), + ("v4 in {192.0.2.0/24}", true), + ("v6 eq ::1", true), + ("missing eq 0", false), + ("missing ne 0", true), + ] { + assert_filter(&scheme, &context, expression, expected); + } + let native = scheme + .parse_value("missing") + .unwrap() + .compile_with_compiler(&mut DefaultCompiler::::new()); + assert_eq!(native.execute(&context).unwrap(), Err(Type::Int)); + assert!( + context + .set_field_value_from_name("number", "wrong type") + .is_err() + ); +} + +// Check actual retained element and map-key addresses. Payload-only checks would +// miss rebuilding containers whose byte elements still borrow the source. +fn assert_borrowed_structure(actual: &LhsValue<'_>, original: &LhsValue<'_>) { + match (actual, original) { + (LhsValue::Array(actual), LhsValue::Array(original)) => { + assert_eq!(actual.len(), original.len()); + for (actual, original) in actual.iter().zip(original.iter()) { + assert!( + std::ptr::eq(actual, original), + "array elements were rebuilt" + ); + assert_borrowed_structure(actual, original); + } + } + (LhsValue::Map(actual), LhsValue::Map(original)) => { + assert_eq!(actual.len(), original.len()); + for ((key, actual), (source_key, original)) in actual.iter().zip(original.iter()) { + assert_eq!(key, source_key); + assert_eq!(key.as_ptr(), source_key.as_ptr(), "map key was copied"); + assert!(std::ptr::eq(actual, original), "map values were rebuilt"); + assert_borrowed_structure(actual, original); + } + } + (LhsValue::Bytes(actual), LhsValue::Bytes(original)) => { + assert_eq!(actual.as_ref(), original.as_ref()); + assert_eq!(actual.as_ref().as_ptr(), original.as_ref().as_ptr()); + assert!(matches!(actual, wirefilter::Bytes::Borrowed(_))); + } + _ => assert_eq!(actual, original), + } +} + +#[test] +fn schemes_can_choose_nested_collection_compositions_at_runtime() { + let payload = String::from("needle"); + // A single ContextFamily/getter handles every composition. No Rust provider + // specialization is declared for any of these runtime-selected types. + for depth in 0..=5 { + for mask in 0..1usize << depth { + let mut value = LhsValue::from(payload.as_str()); + let mut selection = String::new(); + for level in 0..depth { + let ty = value.get_type(); + if mask & (1 << level) == 0 { + value = array(ty, [value]); + selection = format!("[0]{selection}"); + } else { + value = map(ty, [(b"k".as_slice(), value)]); + selection = format!("[\"k\"]{selection}"); + } + } + let ty = value.get_type(); + let mut builder = SchemeBuilder::new(); + builder.add_field("value", ty).unwrap(); + add_function( + &mut builder, + "echo", + &[ty], + ty, + SimpleFunctionImpl::new(identity), + ); + let scheme = builder.build(); + let mut context = ExecutionContext::new_with(&scheme, Reads::default); + context.set_field_value_from_name("value", value).unwrap(); + assert_filter( + &scheme, + &context, + &format!("value{selection} eq \"needle\""), + true, + ); + if depth > 0 { + assert_filter( + &scheme, + &context, + &format!("any(value{} eq \"needle\")", "[*]".repeat(depth)), + true, + ); + } + let ast = scheme.parse_value("echo(value)").unwrap(); + let default = ast + .clone() + .compile_with_compiler(&mut DefaultCompiler::::new()); + let custom = + ast.compile_with_compiler(&mut DefaultCompiler::::new()); + let original = context + .get_field_value(scheme.get_field("value").unwrap()) + .unwrap(); + for actual in [default.execute(&context), custom.execute(&context)] { + assert_borrowed_structure(&actual.unwrap().unwrap(), original); + } + } + } +} + +#[test] +fn indexed_and_mapped_arguments_preserve_missing_values_and_map_compatibility() { + let bytes_array = Type::array(Type::Bytes); + let mut builder = SchemeBuilder::new(); + builder.add_field("map", Type::map(bytes_array)).unwrap(); + builder + .add_optional_field("absent", Type::map(bytes_array)) + .unwrap(); + add_function( + &mut builder, + "echo_bytes", + &[Type::Bytes], + Type::Bytes, + SimpleFunctionImpl::new(identity), + ); + add_function( + &mut builder, + "echo_array", + &[bytes_array], + bytes_array, + SimpleFunctionImpl::new(identity), + ); + let scheme = builder.build(); + let payload = String::from("payload"); + let mut context = ExecutionContext::new_with(&scheme, Reads::default); + context + .set_field_value_from_name( + "map", + map( + bytes_array, + [ + ( + b"a".as_slice(), + array(Type::Bytes, [payload.as_str().into()]), + ), + (b"empty".as_slice(), array(Type::Bytes, [])), + ], + ), + ) + .unwrap(); + for (expression, expected) in [ + ( + r#"echo_array(map["a"])"#, + Ok(array(Type::Bytes, [payload.as_str().into()])), + ), + (r#"echo_array(map["missing"])"#, Err(bytes_array)), + (r#"echo_bytes(map["a"][0])"#, Ok(payload.as_str().into())), + (r#"echo_bytes(map["a"][8])"#, Err(Type::Bytes)), + ( + r#"echo_bytes(map["a"][*])"#, + Ok(array(Type::Bytes, [payload.as_str().into()])), + ), + ( + "echo_bytes(map[*][*])", + Ok(array(Type::Bytes, [payload.as_str().into()])), + ), + ("echo_bytes(absent[*][*])", Err(bytes_array)), + (r#"echo_bytes(map["missing"][*])"#, Err(bytes_array)), + (r#"echo_bytes(map["empty"][*])"#, Ok(array(Type::Bytes, []))), + ] { + let ast = scheme.parse_value(expression).unwrap(); + let default = ast + .clone() + .compile_with_compiler(&mut DefaultCompiler::::new()); + let native = ast.compile_with_compiler(&mut DefaultCompiler::::new()); + assert_eq!( + default.execute(&context).unwrap(), + expected, + "default: {expression}" + ); + assert_eq!( + native.execute(&context).unwrap(), + expected, + "custom: {expression}" + ); + } + + let ast = scheme.parse_value(r#"echo_array(map["a"])"#).unwrap(); + let default = ast + .clone() + .compile_with_compiler(&mut DefaultCompiler::::new()); + let custom = ast.compile_with_compiler(&mut DefaultCompiler::::new()); + let LhsValue::Map(original) = context + .get_field_value(scheme.get_field("map").unwrap()) + .unwrap() + else { + unreachable!() + }; + for selected in [default.execute(&context), custom.execute(&context)] { + assert_borrowed_structure(&selected.unwrap().unwrap(), original.get(b"a").unwrap()); + } + + struct ObserveMapArgument { + arguments: usize, + } + impl Compiler for ObserveMapArgument { + type Family = ExecutionContextFamily; + type U = Reads; + + fn compile_function_call_arg_expr( + &mut self, + node: FunctionCallArgExpr, + ) -> CompiledValueExpr { + self.arguments += 1; + let value = self.compile_value_expr(node); + CompiledValueExpr::new_with_context(move |ctx| { + let result = value.execute(ctx.context()); + assert!( + matches!(&result, Ok(LhsValue::Map(_))), + "trailing map[*] changed the compiler hook's value" + ); + result + }) + } + } + let mut compiler = ObserveMapArgument { arguments: 0 }; + let actual = scheme + .parse_value("echo_array(map[*])") + .unwrap() + .compile_with_compiler(&mut compiler); + assert_eq!(compiler.arguments, 1); + let expected = scheme + .parse_value("echo_array(map[*])") + .unwrap() + .compile_with_compiler(&mut DefaultCompiler::::new()); + assert_eq!(actual.execute(&context), expected.execute(&context)); +} + +fn record_children(value: &LhsValue<'_>, addresses: &mut Vec) { + match value { + LhsValue::Array(array) => { + for value in array.iter() { + addresses.push(std::ptr::from_ref(value) as usize); + record_children(value, addresses); + } + } + LhsValue::Map(map) => { + for (_, value) in map.iter() { + addresses.push(std::ptr::from_ref(value) as usize); + record_children(value, addresses); + } + } + _ => {} + } +} + +struct OriginalCollection { + children: Vec, +} +impl> Compare for OriginalCollection { + fn compare<'ctx, 'data: 'ctx>( + &self, + value: &LhsValue<'ctx>, + _: &'ctx F::Context<'data>, + ) -> bool { + let first = match value { + LhsValue::Array(array) => array.get(0), + LhsValue::Map(map) => map.iter().next().map(|(_, value)| value), + _ => panic!("custom comparison requires a collection"), + }; + if let Some(first) = first { + assert!( + self.children + .contains(&(std::ptr::from_ref(first) as usize)), + "custom comparison received a rebuilt collection" + ); + true + } else { + false + } + } +} + +#[test] +fn compiler_comparisons_can_borrow_selected_collections() { + let bytes = Type::array(Type::Bytes); + let mut builder = SchemeBuilder::new(); + builder + .add_field("rows", Type::array(Type::map(bytes))) + .unwrap(); + let scheme = builder.build(); + let value = array( + Type::map(bytes), + [ + map( + bytes, + [(b"key".as_slice(), array(Type::Bytes, ["value".into()]))], + ), + map(bytes, []), + ], + ); + let mut context = ExecutionContext::new_with(&scheme, Reads::default); + context.set_field_value_from_name("rows", value).unwrap(); + let mut children = Vec::new(); + record_children( + context + .get_field_value(scheme.get_field("rows").unwrap()) + .unwrap(), + &mut children, + ); + for indexes in [ + vec![], + vec![FieldIndex::ArrayIndex(0)], + vec![FieldIndex::ArrayIndex(0), FieldIndex::MapKey("key".into())], + vec![FieldIndex::ArrayIndex(8)], + vec![FieldIndex::MapEach], + vec![FieldIndex::MapEach, FieldIndex::MapEach], + ] { + for missing_default in [false, true] { + let index = IndexExpr { + identifier: IdentifierExpr::Field(scheme.get_field("rows").unwrap().to_owned()), + indexes: indexes.clone().into_boxed_slice(), + }; + let default = index.clone().compile_with( + &mut DefaultCompiler::::new(), + missing_default, + OriginalCollection { + children: children.clone(), + }, + ); + let native = index.compile_with( + &mut DefaultCompiler::::new(), + missing_default, + OriginalCollection { + children: children.clone(), + }, + ); + match (default, native) { + (CompiledExpr::One(default), CompiledExpr::One(native)) => { + assert_eq!(default.execute(&context), native.execute(&context)) + } + (CompiledExpr::Vec(default), CompiledExpr::Vec(native)) => { + assert_eq!(default.execute(&context), native.execute(&context)) + } + _ => panic!("comparison cardinality changed"), + } + } + } +} + +#[test] +fn mapped_extra_stored_arguments_keep_cheap_lazy_access() { + fn consume_both<'a>(args: FunctionArgs<'_, 'a>) -> Option> { + args.next()?.ok()?; + args.next()?.ok()?; + Some(true.into()) + } + fn consume_first<'a>(args: FunctionArgs<'_, 'a>) -> Option> { + args.next()?.ok()?; + Some(true.into()) + } + let mut builder = SchemeBuilder::new(); + builder + .add_optional_field("numbers", Type::array(Type::Int)) + .unwrap(); + builder.add_field("map", Type::map(Type::Int)).unwrap(); + for (name, extra, implementation) in [ + ( + "map_arg", + Type::map(Type::Int), + consume_both as for<'i, 'a> fn(FunctionArgs<'i, 'a>) -> Option>, + ), + ("int_arg", Type::Int, consume_both), + ("ignore_map", Type::map(Type::Int), consume_first), + ] { + add_function( + &mut builder, + name, + &[Type::Int, extra], + Type::Bool, + SimpleFunctionImpl::new(implementation), + ); + } + let scheme = builder.build(); + let mut context = ExecutionContext::new_with(&scheme, Reads::default); + context + .set_field_value_from_name("numbers", array(Type::Int, [1.into(), 2.into(), 3.into()])) + .unwrap(); + context + .set_field_value_from_name("map", map(Type::Int, [(b"key".as_slice(), 7.into())])) + .unwrap(); + for expression in [ + "any(map_arg(numbers[*], map))", + r#"any(int_arg(numbers[*], map["key"]))"#, + ] { + let native = scheme + .parse(expression) + .unwrap() + .compile_with_compiler(&mut DefaultCompiler::::new()); + context.get_user_data().map.set(0); + assert!(native.execute(&context).unwrap()); + // Repeated cheap borrowing is intentional. Treating this provider as a + // materializing native collection would read once and allocate caches. + assert_eq!(context.get_user_data().map.get(), 3, "{expression}"); + } + // Leaving the mandatory map absent proves ignored arguments stay unread. + let mut missing = ExecutionContext::new_with(&scheme, Reads::default); + missing + .set_field_value_from_name("numbers", array(Type::Int, [1.into(), 2.into()])) + .unwrap(); + assert_filter(&scheme, &missing, "any(ignore_map(numbers[*], map))", true); + assert_eq!(missing.get_user_data().map.get(), 0); + missing + .set_field_value_from_name("numbers", array(Type::Int, [])) + .unwrap(); + assert_filter(&scheme, &missing, "any(map_arg(numbers[*], map))", false); + assert_eq!(missing.get_user_data().map.get(), 0); +} + +struct BorrowingContext<'data> { + scheme: &'data Scheme, + value: LhsValue<'data>, + optional: Option>, +} +impl HasScheme for BorrowingContext<'_> { + fn scheme(&self) -> &Scheme { + self.scheme + } +} +impl EvaluationContext for BorrowingContext<'_> { + type UserData = (); + + fn get_user_data(&self) -> &() { + &() + } + + fn get_list_matcher(&self, _: ListRef<'_>) -> &dyn ListMatcher { + unreachable!() + } +} +wirefilter::wirefilter_fields! { + impl<'data> BorrowingFields for BorrowingContext<'data> { + "value" => { + provider LhsValueProvider; + fn(ctx: &BorrowingContext<'data>) -> &LhsValue<'data> { &ctx.value } + }, + "optional" => { + provider LhsValueProvider; + fn(ctx: &BorrowingContext<'data>) -> Option<&LhsValue<'data>> { ctx.optional.as_ref() } + }, + } +} + +#[test] +fn macro_supports_borrowed_runtime_values_and_validates_presence() { + let mut builder = SchemeBuilder::new(); + builder.add_field("value", Type::Bytes).unwrap(); + builder.add_optional_field("optional", Type::Int).unwrap(); + let scheme = builder.build(); + BorrowingFields::validate_scheme(&scheme).unwrap(); + let payload = String::from("borrowed"); + let context = BorrowingContext { + scheme: &scheme, + value: payload.as_str().into(), + optional: None, + }; + let filter = scheme + .parse(r#"value eq "borrowed" and optional ne 1"#) + .unwrap() + .compile_with_compiler(&mut DefaultCompiler::<(), BorrowingFields>::new()); + assert!(filter.execute(&context).unwrap()); + let value = scheme + .parse_value("value") + .unwrap() + .compile_with_compiler(&mut DefaultCompiler::<(), BorrowingFields>::new()) + .execute(&context) + .unwrap() + .unwrap(); + assert_borrowed_structure(&value, &context.value); + + let mut builder = SchemeBuilder::new(); + builder.add_field("value", Type::Bytes).unwrap(); + builder.add_field("optional", Type::Int).unwrap(); + let errors = BorrowingFields::validate_scheme(&builder.build()).unwrap_err(); + assert!(errors.0.iter().any(|error| matches!(error, FieldDefinitionError::OptionalGetterForMandatoryField(name) if name.as_ref() == "optional"))); + + // These same getters can serve another scheme's runtime-selected types. + let mut builder = SchemeBuilder::new(); + builder.add_field("value", Type::Int).unwrap(); + builder.add_optional_field("optional", Type::Bytes).unwrap(); + let scheme = builder.build(); + BorrowingFields::validate_scheme(&scheme).unwrap(); + let context = BorrowingContext { + scheme: &scheme, + value: 42.into(), + optional: Some("present".into()), + }; + let filter = scheme + .parse(r#"value eq 42 and optional eq "present""#) + .unwrap() + .compile_with_compiler(&mut DefaultCompiler::<(), BorrowingFields>::new()); + assert!(filter.execute(&context).unwrap()); + + let mut builder = SchemeBuilder::new(); + builder.add_field("value", Type::Int).unwrap(); + builder.add_optional_field("optional", Type::Bytes).unwrap(); + builder.add_field("undefined", Type::Bool).unwrap(); + let errors = BorrowingFields::validate_scheme(&builder.build()).unwrap_err(); + assert!(errors.0.iter().any(|error| matches!(error, FieldDefinitionError::MissingGetter(name) if name.as_ref() == "undefined"))); +} diff --git a/engine/tests/native_operations.rs b/engine/tests/native_operations.rs new file mode 100644 index 00000000..cc38cb73 --- /dev/null +++ b/engine/tests/native_operations.rs @@ -0,0 +1,270 @@ +#![cfg(feature = "bytes")] + +use bytes::Bytes; +use std::any::TypeId; +use std::cell::Cell; +use wirefilter::values::{self, IntoLhsValue, SliceView}; +use wirefilter::{ + Compiler, ContextFamily, ContextRef, DefaultCompiler, EvaluationContext, Field, + FieldBindingError, FieldConsumer, FieldGetter, FieldIndex, HasScheme, IdentifierExpr, + IndexExpr, LhsValue, ListMatcher, ListRef, ProviderFamily, Scheme, SchemeBuilder, Type, + ValueExpr, +}; + +// Cell makes the context invariant in 'data. The descriptor borrows the request +// storage for 'data, rather than being tied to a temporary getter result. +struct BorrowingContext<'data> { + scheme: &'data Scheme, + rows: Cell<&'data [Vec]>, + getter_calls: Cell, +} + +impl HasScheme for BorrowingContext<'_> { + fn scheme(&self) -> &Scheme { + self.scheme + } +} + +impl EvaluationContext for BorrowingContext<'_> { + type UserData = (); + + fn get_user_data(&self) -> &() { + &() + } + + fn get_list_matcher(&self, _: ListRef<'_>) -> &dyn ListMatcher { + unreachable!("no list matchers in this test") + } +} + +struct RowsProvider; + +impl ProviderFamily for RowsProvider { + type Storage<'data> = &'data [Vec]; + type Value<'ctx, 'data> + = SliceView<'data, Vec> + where + 'data: 'ctx; + type ValueType = values::Array>; + + fn reborrow<'ctx, 'data: 'ctx>(value: &'ctx Self::Storage<'data>) -> Self::Value<'ctx, 'data> { + SliceView(value) + } +} + +wirefilter::wirefilter_fields! { + impl<'data> BorrowingFields for BorrowingContext<'data> { + "rows" => { + provider RowsProvider; + fn(ctx: &BorrowingContext<'data>, field: &Field) -> SliceView<'data, Vec> { + assert_eq!(field.name(), "rows"); + ctx.getter_calls.set(ctx.getter_calls.get() + 1); + SliceView(ctx.rows.get()) + } + }, + "first" => { + provider RowsProvider; + fn(ctx: &BorrowingContext<'data>, field: &Field) -> SliceView<'data, Vec> { + assert_eq!(field.name(), "first"); + assert!(!field.optional()); + ctx.getter_calls.set(ctx.getter_calls.get() + 1); + let rows = ctx.rows.get(); + SliceView(&rows[..rows.len().min(1)]) + } + }, + "optional" => { + provider RowsProvider; + fn(ctx: &BorrowingContext<'data>, field: &Field) -> Option>> { + assert_eq!(field.name(), "optional"); + ctx.getter_calls.set(ctx.getter_calls.get() + 1); + let rows = ctx.rows.get(); + (!rows.is_empty()).then_some(SliceView(rows)) + } + }, + } +} + +#[derive(Default)] +struct RecordingCompiler { + bindings: Vec, +} + +impl Compiler for RecordingCompiler { + type Family = BorrowingFields; + type U = (); + + fn bind_field>( + &mut self, + field: Field, + consumer: B, + ) -> Result { + self.bindings.push(field.name().into()); + BorrowingFields::bind(field, consumer) + } +} + +fn scheme() -> Scheme { + let mut builder = SchemeBuilder::new(); + let rows = Type::array(Type::array(Type::Bytes)); + builder.add_field("rows", rows).unwrap(); + builder.add_field("first", rows).unwrap(); + builder.add_optional_field("optional", rows).unwrap(); + let scheme = builder.build(); + BorrowingFields::validate_scheme(&scheme).unwrap(); + scheme +} + +#[test] +fn test_same_provider_and_presence_getters_share_a_type_and_preserve_field_selection() { + type ReadValue = Box< + dyn for<'ctx, 'data> Fn(ContextRef<'ctx, 'data, BorrowingFields>) -> Option>, + >; + + struct ReadConsumer; + + impl FieldConsumer for ReadConsumer { + type Output = (TypeId, ReadValue); + + fn consume>( + self, + field: Field, + getter: G, + ) -> Result { + let read: ReadValue = Box::new(move |context| { + let value = { + let metadata = field.clone(); + getter.get(context.context(), &metadata) + }; + // Dropping the independently borrowed field metadata must not + // prevent retaining the native value's request references. + value.map(IntoLhsValue::into_lhs) + }); + Ok((TypeId::of::(), read)) + } + } + + let scheme = scheme(); + let (required_type, required) = + BorrowingFields::bind(scheme.get_field("rows").unwrap().to_owned(), ReadConsumer).unwrap(); + let (first_type, first) = + BorrowingFields::bind(scheme.get_field("first").unwrap().to_owned(), ReadConsumer).unwrap(); + let (optional_type, optional) = BorrowingFields::bind( + scheme.get_field("optional").unwrap().to_owned(), + ReadConsumer, + ) + .unwrap(); + assert_eq!(required_type, first_type); + assert_ne!(required_type, optional_type); + + let rows = [ + vec![Bytes::from_static(b"first payload")], + vec![Bytes::from_static(b"second payload")], + ]; + let context = BorrowingContext { + scheme: &scheme, + rows: Cell::new(rows.as_slice()), + getter_calls: Cell::new(0), + }; + let required_value = required(ContextRef::new(&context)).unwrap(); + let optional_value = optional(ContextRef::new(&context)).unwrap(); + assert_eq!(required_value, optional_value); + let first_value = first(ContextRef::new(&context)).unwrap(); + assert_eq!(first_value, SliceView(&rows[..1]).into_lhs()); + assert_ne!(required_value, first_value); + assert_eq!(context.getter_calls.get(), 3); + + context.rows.set(&[]); + assert!(required(ContextRef::new(&context)).is_some()); + assert!(first(ContextRef::new(&context)).is_some()); + assert!(optional(ContextRef::new(&context)).is_none()); + assert_eq!(context.getter_calls.get(), 6); +} + +#[test] +fn descriptor_comparisons_borrow_invariant_context_and_preserve_getter_hooks() { + let scheme = scheme(); + let first = [ + vec![Bytes::from_static(b"first"), Bytes::from_static(b"target")], + vec![], + ]; + let second = [ + vec![Bytes::from_static(b"second")], + vec![Bytes::from_static(b"target")], + ]; + let context = BorrowingContext { + scheme: &scheme, + rows: Cell::new(first.as_slice()), + getter_calls: Cell::new(0), + }; + let mut compiler = RecordingCompiler::default(); + for (expression, field, expected_first, expected_second) in [ + (r#"rows[0][0] eq "first""#, "rows", true, false), + (r#"any(rows[0][*] eq "target")"#, "rows", true, false), + (r#"any(rows[*][*] eq "target")"#, "rows", true, true), + (r#"any(rows[*][0] eq "target")"#, "rows", false, true), + (r#"optional[0][0] eq "first""#, "optional", true, false), + (r#"any(optional[*][*] eq "target")"#, "optional", true, true), + ] { + let binding_count = compiler.bindings.len(); + let filter = scheme + .parse(expression) + .unwrap() + .compile_with_compiler(&mut compiler); + assert_eq!(&compiler.bindings[binding_count..], &[field]); + for (rows, expected) in [ + (first.as_slice(), expected_first), + (second.as_slice(), expected_second), + ([].as_slice(), false), + ] { + context.rows.set(rows); + let getter_count = context.getter_calls.get(); + assert_eq!(filter.execute(&context).unwrap(), expected, "{expression}"); + assert_eq!(context.getter_calls.get(), getter_count + 1); + assert_eq!(compiler.bindings.len(), binding_count + 1); + } + } +} + +#[test] +fn descriptor_materialization_keeps_payloads_borrowed_from_invariant_context() { + let scheme = scheme(); + let rows = [vec![ + Bytes::from_static(b"first"), + Bytes::from_static(b"second"), + ]]; + let context = BorrowingContext { + scheme: &scheme, + rows: Cell::new(rows.as_slice()), + getter_calls: Cell::new(0), + }; + let mut compiler = DefaultCompiler::<(), BorrowingFields>::new(); + // The public IndexExpr API supports general `[*]` selections used by + // mapped function arguments; parse_value rejects this top-level `[*]`. + for indexes in [ + vec![], + vec![FieldIndex::ArrayIndex(0), FieldIndex::ArrayIndex(0)], + vec![FieldIndex::MapEach, FieldIndex::MapEach], + ] { + let expression = IndexExpr { + identifier: IdentifierExpr::Field(scheme.get_field("rows").unwrap().to_owned()), + indexes: indexes.into_boxed_slice(), + }; + let filter = expression.compile_with_compiler(&mut compiler); + let getter_count = context.getter_calls.get(); + let value = filter.execute(&context).unwrap(); + let bytes = match &value { + LhsValue::Bytes(bytes) => bytes, + LhsValue::Array(array) => match array.get(0).unwrap() { + LhsValue::Bytes(bytes) => bytes, + LhsValue::Array(array) => match array.get(0).unwrap() { + LhsValue::Bytes(bytes) => bytes, + _ => panic!("expected nested bytes"), + }, + _ => panic!("expected bytes or array"), + }, + _ => panic!("unexpected materialized value"), + }; + assert_eq!(bytes.as_ref().as_ptr(), rows[0][0].as_ptr()); + assert_eq!(context.getter_calls.get(), getter_count + 1); + } +} diff --git a/engine/tests/reader_traversal.rs b/engine/tests/reader_traversal.rs new file mode 100644 index 00000000..b4d17e61 --- /dev/null +++ b/engine/tests/reader_traversal.rs @@ -0,0 +1,440 @@ +//! Traversal must preserve ownership as well as selection and comparison results. + +use std::cell::RefCell; +use std::collections::BTreeMap; +use wirefilter::{ + Array, Bytes, Compare, CompiledExpr, CompiledValueExpr, ContextFamily, ContextState, + DefaultCompiler, EvaluationContext, ExecutionContext, FieldIndex, FilterValueExpr, + FunctionArgs, HasScheme, IdentifierExpr, IndexExpr, LhsValue, ListMatcher, ListRef, Map, + Scheme, SchemeBuilder, SimpleFunctionArgKind, SimpleFunctionDefinition, SimpleFunctionImpl, + SimpleFunctionParam, Type, TypeMismatchError, ValueExpr, +}; + +fn array<'a>(ty: Type, values: impl IntoIterator>) -> LhsValue<'a> { + Array::try_from_iter(ty, values).unwrap().into() +} + +fn map<'a>(ty: Type, entries: impl IntoIterator)>) -> LhsValue<'a> { + Map::try_from_iter( + ty, + entries + .into_iter() + .map(|(key, value)| Ok::<_, TypeMismatchError>((key.into(), value))), + ) + .unwrap() + .into() +} + +fn add_function( + builder: &mut SchemeBuilder, + name: &str, + parameters: &[Type], + output: Type, + implementation: SimpleFunctionImpl, +) { + builder + .add_function( + name, + SimpleFunctionDefinition { + params: parameters + .iter() + .map(|&val_type| SimpleFunctionParam { + arg_kind: SimpleFunctionArgKind::Field, + val_type, + }) + .collect(), + opt_params: Box::new([]), + return_type: output, + implementation, + }, + ) + .unwrap(); +} + +fn compile_selection(scheme: &Scheme, call: &str, indexes: Vec) -> CompiledValueExpr { + // Bare [*] value expressions are not accepted as a FilterValueAst. Construct + // the same IndexExpr node that occurs within a mapped function argument. + let FilterValueExpr::Index(mut index) = scheme.parse_value(call).unwrap().expression().clone() + else { + panic!("expected function-call index expression") + }; + index.indexes = indexes.into_boxed_slice(); + index.compile() +} + +thread_local! { + // Record the allocations produced inside the function, before its temporary + // outer containers are consumed by index selection. + static GENERATED: RefCell, usize)>> = const { RefCell::new(Vec::new()) }; +} + +fn owned_bytes(text: &str) -> LhsValue<'static> { + let bytes = text.as_bytes().to_vec().into_boxed_slice(); + GENERATED.with_borrow_mut(|records| { + records.push((text.as_bytes().to_vec(), bytes.as_ptr() as usize)); + }); + LhsValue::Bytes(Bytes::Owned(bytes)) +} + +fn owned_rows<'a>(_: FunctionArgs<'_, 'a>) -> Option> { + GENERATED.with_borrow_mut(Vec::clear); + Some(map( + Type::array(Type::Bytes), + [ + ( + b"z".as_slice(), + array(Type::Bytes, [owned_bytes("z0"), owned_bytes("z1")]), + ), + (b"a".as_slice(), array(Type::Bytes, [owned_bytes("a0")])), + (b"empty".as_slice(), array(Type::Bytes, [])), + ], + )) +} + +fn absent<'a>(_: FunctionArgs<'_, 'a>) -> Option> { + None +} + +fn identity<'a>(args: FunctionArgs<'_, 'a>) -> Option> { + args.next()?.ok() +} + +fn assert_original_owned_payload(value: &LhsValue<'_>) { + match value { + LhsValue::Bytes(Bytes::Owned(bytes)) => GENERATED.with_borrow(|records| { + let (_, address) = records + .iter() + .find(|(text, _)| text.as_slice() == bytes.as_ref()) + .unwrap(); + assert_eq!( + bytes.as_ptr() as usize, + *address, + "owned payload was copied" + ); + }), + LhsValue::Array(values) => { + for value in values.iter() { + assert_original_owned_payload(value); + } + } + value => panic!("expected retained owned bytes, got {value:?}"), + } +} + +#[test] +fn selections_move_owned_function_payloads_and_preserve_map_order() { + use FieldIndex::{ArrayIndex, MapEach, MapKey}; + + let mut builder = SchemeBuilder::new(); + let rows = Type::map(Type::array(Type::Bytes)); + add_function( + &mut builder, + "owned_rows", + &[], + rows, + SimpleFunctionImpl::new(owned_rows), + ); + add_function( + &mut builder, + "absent", + &[], + rows, + SimpleFunctionImpl::new(absent), + ); + let scheme = builder.build(); + let context = ExecutionContext::new(&scheme); + for (indexes, expected) in [ + ( + vec![MapEach, MapEach], + array(Type::Bytes, ["a0".into(), "z0".into(), "z1".into()]), + ), + ( + vec![MapEach, ArrayIndex(0)], + array(Type::Bytes, ["a0".into(), "z0".into()]), + ), + ( + vec![MapKey("z".into()), MapEach], + array(Type::Bytes, ["z0".into(), "z1".into()]), + ), + (vec![MapKey("z".into()), ArrayIndex(1)], "z1".into()), + (vec![MapEach, ArrayIndex(9)], array(Type::Bytes, [])), + ] { + // Selection consumes the function's temporary root. The compiled + // expression is also dropped before the selected payloads are inspected. + let result = { + let filter = compile_selection(&scheme, "owned_rows()", indexes.clone()); + filter.execute(&context).unwrap() + }; + assert_eq!(result, expected, "{indexes:?}"); + assert_original_owned_payload(&result); + } + for (call, indexes, expected) in [ + ( + "owned_rows()", + vec![MapKey("missing".into()), MapEach], + Type::array(Type::Bytes), + ), + ("absent()", vec![MapEach, MapEach], Type::array(Type::Bytes)), + ( + "absent()", + vec![MapKey("z".into()), ArrayIndex(0)], + Type::Bytes, + ), + ] { + let filter = compile_selection(&scheme, call, indexes); + assert_eq!(filter.execute(&context), Err(expected), "{call}"); + } + for (expression, expected) in [ + (r#"any(owned_rows()[*][*] eq "z1")"#, true), + (r#"all(owned_rows()[*][*] eq "z1")"#, false), + (r#"owned_rows()["z"][0] eq "z0""#, true), + (r#"owned_rows()["missing"][0] eq "x""#, false), + (r#"owned_rows()["missing"][0] ne "x""#, true), + (r#"any(absent()[*][*] eq "x")"#, false), + (r#"absent()["z"][0] ne "x""#, true), + ] { + assert_eq!( + scheme + .parse(expression) + .unwrap() + .compile() + .execute(&context) + .unwrap(), + expected, + "{expression}" + ); + } +} + +#[test] +fn flattened_borrowed_function_results_keep_source_payloads() { + let payloads = [String::from("first"), String::from("second")]; + let mut builder = SchemeBuilder::new(); + let rows = Type::map(Type::array(Type::Bytes)); + builder.add_field("rows", rows).unwrap(); + add_function( + &mut builder, + "echo", + &[rows], + rows, + SimpleFunctionImpl::new(identity), + ); + let scheme = builder.build(); + let mut context = ExecutionContext::new(&scheme); + context + .set_field_value_from_name( + "rows", + map( + Type::array(Type::Bytes), + [ + ( + b"z".as_slice(), + array(Type::Bytes, [payloads[1].as_str().into()]), + ), + ( + b"a".as_slice(), + array(Type::Bytes, [payloads[0].as_str().into()]), + ), + ], + ), + ) + .unwrap(); + let result = compile_selection(&scheme, "echo(rows)", vec![FieldIndex::MapEach; 2]) + .execute(&context) + .unwrap(); + let LhsValue::Array(values) = result else { + panic!("expected flattened array") + }; + assert_eq!(values.len(), 2); + for (value, original) in values.iter().zip(&payloads) { + let LhsValue::Bytes(Bytes::Borrowed(bytes)) = value else { + panic!("borrowed function payload was copied") + }; + assert_eq!(*bytes, original.as_bytes()); + assert_eq!(bytes.as_ptr(), original.as_ptr()); + } +} + +type Rows = Vec, Vec>>; + +struct NativeContext { + state: ContextState, + rows: Rows, + missing: Option, + flags: Vec, +} + +impl HasScheme for NativeContext { + fn scheme(&self) -> &Scheme { + self.state.scheme() + } +} + +impl EvaluationContext for NativeContext { + type UserData = (); + + fn get_user_data(&self) -> &() { + self.state.get_user_data() + } + + fn get_list_matcher(&self, list: ListRef<'_>) -> &dyn ListMatcher { + self.state.get_list_matcher(list) + } +} + +wirefilter::wirefilter_fields! { + impl NativeFields for NativeContext { + "rows" => { fn(ctx: &NativeContext) -> &Rows { &ctx.rows } }, + "missing" => { fn(ctx: &NativeContext) -> Option<&Rows> { ctx.missing.as_ref() } }, + "flags" => { fn(ctx: &NativeContext) -> &Vec { &ctx.flags } }, + } +} + +fn native_context() -> NativeContext { + let mut builder = SchemeBuilder::new(); + let rows = Type::array(Type::map(Type::array(Type::Int))); + builder.add_field("rows", rows).unwrap(); + builder.add_optional_field("missing", rows).unwrap(); + builder.add_field("flags", Type::array(Type::Bool)).unwrap(); + add_function( + &mut builder, + "echo_flags", + &[Type::array(Type::Bool)], + Type::array(Type::Bool), + SimpleFunctionImpl::new(identity), + ); + let scheme = builder.build(); + NativeFields::validate_scheme(&scheme).unwrap(); + NativeContext { + state: ContextState::new(&scheme), + rows: vec![ + BTreeMap::from([ + (Box::from(b"z".as_slice()), vec![1, 2]), + (Box::from(b"a".as_slice()), vec![]), + ]), + BTreeMap::new(), + BTreeMap::from([(Box::from(b"a".as_slice()), vec![3])]), + ], + missing: None, + flags: vec![], + } +} + +struct Nonempty; + +impl> Compare<(), F> for Nonempty { + fn compare<'ctx, 'data: 'ctx>( + &self, + value: &LhsValue<'ctx>, + _: &'ctx F::Context<'data>, + ) -> bool { + match value { + LhsValue::Array(values) => !values.is_empty(), + LhsValue::Map(values) => !values.is_empty(), + _ => panic!("expected custom comparison of a collection"), + } + } +} + +#[test] +fn native_custom_collection_comparisons_cover_flat_and_nested_selections() { + use FieldIndex::{ArrayIndex, MapEach, MapKey}; + let context = native_context(); + for (field, indexes, expected) in [ + ("rows", vec![], true.into()), + ("rows", vec![ArrayIndex(0)], true.into()), + ( + "rows", + vec![ArrayIndex(0), MapKey("a".into())], + false.into(), + ), + ( + "rows", + vec![MapEach], + array(Type::Bool, [true.into(), false.into(), true.into()]), + ), + ( + "rows", + vec![MapEach, MapEach], + array(Type::Bool, [false.into(), true.into(), true.into()]), + ), + ( + "rows", + vec![MapEach, MapKey("a".into())], + array(Type::Bool, [false.into(), true.into()]), + ), + ( + "rows", + vec![MapEach, MapKey("missing".into())], + array(Type::Bool, []), + ), + ("rows", vec![ArrayIndex(1), MapEach], array(Type::Bool, [])), + ("missing", vec![MapEach], array(Type::Bool, [])), + ("missing", vec![MapEach, MapEach], array(Type::Bool, [])), + ] { + let index = IndexExpr { + identifier: IdentifierExpr::Field( + context.scheme().get_field(field).unwrap().to_owned(), + ), + indexes: indexes.into_boxed_slice(), + }; + let compiled = index.compile_with( + &mut DefaultCompiler::<(), NativeFields>::new(), + false, + Nonempty, + ); + let actual = match compiled { + CompiledExpr::One(value) => value.execute(&context).into(), + CompiledExpr::Vec(value) => LhsValue::Array(value.execute(&context).into()), + }; + assert_eq!(actual, expected, "{field}"); + } + for field in ["rows", "missing"] { + for default in [false, true] { + let index = IndexExpr { + identifier: IdentifierExpr::Field( + context.scheme().get_field(field).unwrap().to_owned(), + ), + indexes: Box::new([ArrayIndex(9)]), + }; + let CompiledExpr::One(compiled) = index.compile_with( + &mut DefaultCompiler::<(), NativeFields>::new(), + default, + Nonempty, + ) else { + panic!("expected scalar result") + }; + assert_eq!(compiled.execute(&context), default); + } + } +} + +#[test] +fn implicit_boolean_array_iteration_covers_fields_and_function_results() { + let mut context = native_context(); + for flags in [ + vec![], + vec![false], + vec![true, false, true], + vec![true, true], + ] { + for (expression, expected) in [ + ("any(flags)", flags.iter().any(|&value| value)), + ("all(flags)", flags.iter().all(|&value| value)), + ("any(echo_flags(flags))", flags.iter().any(|&value| value)), + ("all(echo_flags(flags))", flags.iter().all(|&value| value)), + ] { + context.flags.clone_from(&flags); + let compiled = context + .scheme() + .parse(expression) + .unwrap() + .compile_with_compiler(&mut DefaultCompiler::<(), NativeFields>::new()); + assert_eq!( + compiled.execute(&context).unwrap(), + expected, + "{expression}" + ); + } + } +}