From 813e19e0960991023320e62e3af0e7df339faf00 Mon Sep 17 00:00:00 2001 From: JT Date: Wed, 21 Oct 2026 18:22:41 +0800 Subject: [PATCH 1/2] Add support for aggregate and group-by expressions to continuous aggregates --- .claude/reference/architecture.md | 2 +- .claude/reference/patterns.md | 4 +- docs/03-scaffolding.md | 2 +- .../data-annotations/continuous-aggregates.md | 28 +- docs/fluent-api/continuous-aggregates.md | 69 +++- .../ContinuousAggregateAnnotationRenderer.cs | 108 +++--- .../ContinuousAggregateCSharpGenerator.cs | 37 +- .../TimescaleDbAnnotationCodeGenerator.cs | 1 + .../Scaffolding/ViewDefinitionParser.cs | 68 +++- .../ContinuousAggregateExpression.cs | 32 ++ .../AggregateExpressionAttribute.cs | 21 ++ .../ContinuousAggregateAnnotations.cs | 12 + .../ContinuousAggregateBuilder.cs | 50 +++ .../ContinuousAggregateBuilderCore.cs | 16 + .../ContinuousAggregateConvention.cs | 31 +- .../ContinuousAggregateStringBuilder.cs | 30 ++ .../ContinuousAggregateTypeBuilder.cs | 32 ++ .../GroupByExpressionAttribute.cs | 21 ++ ...AggregateViewColumnValidationConvention.cs | 42 ++- src/Eftdb/Diagnostics/TimescaleDbEventId.cs | 8 + .../TimescaleDbLoggerExtensions.cs | 54 +++ .../TimescaleDbLoggingDefinitions.cs | 5 + .../ContinuousAggregateSqlGenerator.cs | 20 + src/Eftdb/Internals/ColumnTemplate.cs | 84 +++++ .../ContinuousAggregateExpressionJson.cs | 61 +++ .../ContinuousAggregateDiffer.cs | 10 +- .../ContinuousAggregateModelExtractor.cs | 88 +++-- .../Internals/TimescaleModelValidator.cs | 35 ++ .../ContinuousAggregateMigrationExtensions.cs | 6 +- .../CreateContinuousAggregateOperation.cs | 13 +- ...tinuousAggregateAnnotationRendererTests.cs | 44 ++- ...ContinuousAggregateCSharpGeneratorTests.cs | 30 ++ .../ScaffoldedModelRenderingTests.cs | 2 - .../Scaffolding/ViewDefinitionParserTests.cs | 39 ++ ...inuousAggregateEntryIgnoredWarningTests.cs | 106 ++++++ .../TimescaleDbWarningEmissionTests.cs | 1 + ...ontinuousAggregateExpressionDifferTests.cs | 116 ++++++ ...inuousAggregateExpressionExtractorTests.cs | 320 ++++++++++++++++ ...uousAggregateExpressionIntegrationTests.cs | 236 ++++++++++++ .../Integration/ScaffoldRoundTripTests.cs | 22 ++ .../Internals/ColumnTemplateTests.cs | 79 ++++ ...ntinuousAggregateExpressionBuilderTests.cs | 355 ++++++++++++++++++ tests/Eftdb.Tests/Utils/TimescaleImages.cs | 8 + 43 files changed, 2239 insertions(+), 109 deletions(-) create mode 100644 src/Eftdb/Abstractions/ContinuousAggregateExpression.cs create mode 100644 src/Eftdb/Configuration/ContinuousAggregate/AggregateExpressionAttribute.cs create mode 100644 src/Eftdb/Configuration/ContinuousAggregate/GroupByExpressionAttribute.cs create mode 100644 src/Eftdb/Internals/ColumnTemplate.cs create mode 100644 src/Eftdb/Internals/ContinuousAggregateExpressionJson.cs create mode 100644 tests/Eftdb.Tests/Diagnostics/ContinuousAggregateEntryIgnoredWarningTests.cs create mode 100644 tests/Eftdb.Tests/Differs/ContinuousAggregateExpressionDifferTests.cs create mode 100644 tests/Eftdb.Tests/Extractors/ContinuousAggregateExpressionExtractorTests.cs create mode 100644 tests/Eftdb.Tests/Integration/ContinuousAggregateExpressionIntegrationTests.cs create mode 100644 tests/Eftdb.Tests/Internals/ColumnTemplateTests.cs create mode 100644 tests/Eftdb.Tests/TypeBuilders/ContinuousAggregateExpressionBuilderTests.cs diff --git a/.claude/reference/architecture.md b/.claude/reference/architecture.md index a73b811..c59600e 100644 --- a/.claude/reference/architecture.md +++ b/.claude/reference/architecture.md @@ -119,7 +119,7 @@ Drops negative (before EF table drops, reverse dependency order); adds/alters po ## Continuous Aggregate Notes -- Operation properties: `MaterializedViewName`, `ParentName` (entity name, resolved via EF metadata), `TimeBucketWidth`, `TimeBucketSourceColumn`, `AggregateFunctions` (colon-delimited wire format, see patterns.md), `GroupByColumns`, `WhereClause` (raw SQL, emitted verbatim — quoted identifiers must match resolved column names) +- Operation properties: `MaterializedViewName`, `ParentName` (entity name, resolved via EF metadata), `TimeBucketWidth`, `TimeBucketSourceColumn`, `AggregateFunctions` (colon-delimited wire format, see patterns.md), `GroupByColumns`, `WhereClause` (raw SQL, emitted verbatim — quoted identifiers must match resolved column names), `AggregateExpressions` / `GroupByExpressions` (`ContinuousAggregateExpression` lists, see patterns.md) - `first()`/`last()` take the time-bucket column as second argument: `last("price", "timestamp")` - Aggregate column aliases must match property names for EF mapping - **Bucket column naming**: `WithTimeBucketProperty` designates the property whose mapped column becomes the view's bucket alias; without it the alias falls back to `time_bucket` (snapshot compatibility). Renaming the bucket column is a structural change (drop + recreate). diff --git a/.claude/reference/patterns.md b/.claude/reference/patterns.md index af26111..9acd428 100644 --- a/.claude/reference/patterns.md +++ b/.claude/reference/patterns.md @@ -77,7 +77,9 @@ SqlBuilderHelper.QuoteIdentifier("my_column"); // "my_column ## 7. Continuous Aggregate Function Encoding -Typed API: `ContinuousAggregateFunction(Alias, Function, SourceColumn)`. Wire format on the operation: `"alias:Function:sourceColumn"` — always exactly three parts; malformed entries are skipped on parse. `First`/`Last` take no time column in the wire format; the SQL generator supplies the time-bucket column as second argument (`last("price", "timestamp")`). +Typed API: `ContinuousAggregateFunction(Alias, Function, SourceColumn)`. Wire format on the operation: `"alias:Function:sourceColumn"` — always exactly three parts; malformed entries are skipped on parse and raise the `ContinuousAggregateEntryIgnored` warning at model validation. `First`/`Last` take no time column in the wire format; the SQL generator supplies the time-bucket column as second argument (`last("price", "timestamp")`). + +Expressions: `ContinuousAggregateExpression(Alias, Sql, Columns)` for aggregates and grouped expressions the enum cannot express. `Sql` is a template with `{n}` placeholders (`Internals/ColumnTemplate` parses and formats them, skipping quoted text; `{{`/`}}` escape braces). Annotations `AggregateExpressions` / `GroupByExpressions` hold JSON lists with model names (`Internals/ContinuousAggregateExpressionJson`); the extractor resolves aliases against the view and columns against the parent, throwing when a column does not resolve. Group-by expressions are selected with their alias and repeated in GROUP BY. ## 8. Expression-Based Configuration diff --git a/docs/03-scaffolding.md b/docs/03-scaffolding.md index 843425c..9df42f9 100644 --- a/docs/03-scaffolding.md +++ b/docs/03-scaffolding.md @@ -42,4 +42,4 @@ Scaffolding reproduces what the database stores, which is not always the literal - **CLR type fidelity is not preserved**: custom mapped types (e.g. NodaTime `LocalDateTime`) scaffold as BCL types with an explicit store type (`[Column(TypeName = "timestamp without time zone")] DateTime`). The column definition is identical; re-apply the custom type mapping manually if needed. - **`InitialStart` sub-microsecond precision is lost**: PostgreSQL stores microseconds, so .NET's seventh fractional digit (100 ns ticks) does not survive — `…19.3905112Z` scaffolds as `…19.3905110Z`. - **Non-column GROUP BY expressions have no data-annotations representation**: in `--data-annotations` mode they are reported as a warning and must be configured via the Fluent API's `AddGroupByColumn(string)`. -- **Unparseable view definitions degrade gracefully**: when a continuous aggregate's view SQL cannot be parsed into typed configuration, a warning is reported and the configuration is preserved as `.HasAnnotation(...)` calls; migrations still recreate the view correctly from the raw SQL definition. +- **Views the builder cannot express are scaffolded as SQL**: when a continuous aggregate's view has a column that is neither the time bucket, a supported aggregate function nor a plain GROUP BY column (for example a toolkit sketch such as `mcv_agg(...)` or a grouped expression), a warning is reported and the entity is scaffolded with `IsContinuousAggregateFromSql(...)` holding the view SQL verbatim. In `--data-annotations` mode the configuration is preserved as `.HasAnnotation(...)` calls instead. Either way migrations recreate the view from the same SQL. diff --git a/docs/data-annotations/continuous-aggregates.md b/docs/data-annotations/continuous-aggregates.md index b3daab8..211ebe7 100644 --- a/docs/data-annotations/continuous-aggregates.md +++ b/docs/data-annotations/continuous-aggregates.md @@ -367,7 +367,33 @@ public class TradeHourlyAggregate } ``` -> :warning: **Note:** Raw SQL GROUP BY expressions (e.g. `EXTRACT(HOUR FROM time)`) cannot be expressed as an attribute. Use the [Fluent API](../fluent-api/continuous-aggregates#grouping-data)'s `AddGroupByColumn(string)` overload for those. +## Aggregate and Group-By Expressions + +`[AggregateExpression]` defines a property as a SQL aggregate expression, and `[GroupByExpression]` as a grouped expression selected as a column. Column references in the SQL are `{0}`, `{1}`, … placeholders bound to the source columns that follow, which resolve to mapped column names. Literal braces are written as `{{` and `}}`. + +```csharp +[Keyless] +[ContinuousAggregate( + MaterializedViewName = "trade_daily_stats", + ParentName = nameof(Trade))] +[TimeBucket("1 day", nameof(Trade.Timestamp))] +public class TradeDailyAggregate +{ + [GroupByExpression("{0} >= 1000", nameof(Trade.Size))] + public bool IsBlockTrade { get; set; } + + [AggregateExpression("count(*) FILTER (WHERE {0} = 'NASDAQ')", nameof(Trade.Exchange))] + public long NasdaqTrades { get; set; } + + // Requires the timescaledb_toolkit extension. + [AggregateExpression("mcv_agg(10, {0})", nameof(Trade.Ticker))] + public string TopTickers { get; set; } = string.Empty; +} +``` + +See the [Fluent API](../fluent-api/continuous-aggregates#aggregate-expressions) page for validation rules and rollups in hierarchical aggregates. + +> :warning: **Note:** A raw `AddGroupByColumn(string)` expression appears only in `GROUP BY` and has no attribute form; use `[GroupByExpression]` to select it as a column instead. ## Refresh Policies diff --git a/docs/fluent-api/continuous-aggregates.md b/docs/fluent-api/continuous-aggregates.md index 4256d59..6db6ef1 100644 --- a/docs/fluent-api/continuous-aggregates.md +++ b/docs/fluent-api/continuous-aggregates.md @@ -86,6 +86,35 @@ public void Configure(EntityTypeBuilder builder) } ``` +### Aggregate Expressions + +For aggregates outside `EAggregateFunction`, such as filtered counts, `bool_or` or [timescaledb_toolkit](https://docs.tigerdata.com/api/latest/hyperfunctions/) sketches, use `.AddAggregateExpression()`. The SQL is a template whose column references are `{0}`, `{1}`, … placeholders bound to the selectors that follow it, so each reference resolves to its mapped column name (including naming conventions such as snake case). Literal braces are written as `{{` and `}}`; braces inside string literals and quoted identifiers are left alone. + +```csharp +public void Configure(EntityTypeBuilder builder) +{ + builder.HasNoKey(); + + builder.IsContinuousAggregate( + "trade_daily_stats", + "1 day", + x => x.Timestamp) + .AddAggregateExpression(x => x.LargeTrades, "count(*) FILTER (WHERE {0} >= 1000)", x => x.Size) + .AddAggregateExpression(x => x.PriceRange, "max({0}) - min({0})", x => x.Price) + // Requires the timescaledb_toolkit extension: modelBuilder.HasPostgresExtension("timescaledb_toolkit") + .AddAggregateExpression(x => x.TopTickers, "mcv_agg(10, {0})", x => x.Ticker); +} +``` + +The model fails to build when a placeholder has no selector, a selector is never referenced, or a selector does not resolve to a column of the source. A change to an expression's template or columns drops and recreates the aggregate, as with any other structural change. + +Hierarchical aggregates combine sketches from their parent with the matching rollup function, referencing the parent aggregate's property: + +```csharp +builder.IsContinuousAggregate("trade_weekly_stats", "7 days", x => x.Bucket) + .AddAggregateExpression(x => x.TopTickers, "rollup({0})", x => x.TopTickers); +``` + ## Grouping Data Beyond the time bucket, continuous aggregates can group data by additional columns from the source hypertable. In Data Annotations configuration, the same is expressed with the property-level [`[GroupByColumn]` attribute](../data-annotations/continuous-aggregates#grouping-by-additional-columns); raw SQL expressions are Fluent-API-only. @@ -126,6 +155,40 @@ public void Configure(EntityTypeBuilder builder) } ``` +A raw `AddGroupByColumn` expression appears only in `GROUP BY`, so no property can read it. To select a grouped expression as a column, use `.AddGroupByExpression()`, which takes a placeholder template like `.AddAggregateExpression()`: + +```csharp +public void Configure(EntityTypeBuilder builder) +{ + builder.HasNoKey(); + + builder.IsContinuousAggregate( + "trade_hourly_stats_by_size", + "1 hour", + x => x.Timestamp) + .AddAggregateFunction(x => x.AveragePrice, x => x.Price, EAggregateFunction.Avg) + // SELECT "size" >= 1000 AS "is_block_trade" ... GROUP BY 1, ("size" >= 1000) + .AddGroupByExpression(x => x.IsBlockTrade, "{0} >= 1000", x => x.Size); +} +``` + +## Defining the View in SQL + +When a definition cannot be expressed through the builder, `.IsContinuousAggregateFromSql()` takes the SELECT statement verbatim. Identifiers in it are not resolved through the model, so it must use database column names. Options such as `.MaterializedOnly()`, `.WithNoData()`, compression and policies still apply. + +```csharp +builder.IsContinuousAggregateFromSql( + "trade_daily_vwap", + nameof(Trade), + """ + SELECT time_bucket('1 day', "timestamp") AS "bucket", "ticker", + sum("price" * "size") / nullif(sum("size"), 0) AS "vwap" + FROM "trades" + GROUP BY 1, "ticker" + """) + .MaterializedOnly(false); +``` + ## Filtering Source Data Apply filtering conditions to the source hypertable before aggregation using the `.Where()` method. @@ -201,11 +264,13 @@ The string-based builder used by scaffolded code exposes an equivalent `.WithTim Structured aggregates (those configured through the builders rather than a raw view definition) are validated at model finalization: -- Duplicate output column names are rejected with an `InvalidOperationException`. The check compares the bucket column, all GROUP BY columns, and every aggregate alias after resolving them to database column names. A source column that collides with the bucket column name is caught at model build. +- Duplicate output column names are rejected with an `InvalidOperationException`. The check compares the bucket column, all GROUP BY columns, every aggregate alias and every expression alias after resolving them to database column names. A source column that collides with the bucket column name is caught at model build. +- An expression column that does not resolve to a column of the source, or expressions combined with `IsContinuousAggregateFromSql`, raise an `InvalidOperationException`. +- An `.AddAggregateFunction(...)` or `[Aggregate]` entry that is malformed, or whose source column does not resolve, is left out of the view and raises the `TimescaleDbEventId.ContinuousAggregateEntryIgnored` warning, which can be escalated with `ConfigureWarnings(w => w.Throw(...))`. - A property designated via `.WithTimeBucketProperty(...)` that does not exist on the entity raises an `InvalidOperationException`. - An aggregate with no time-bucket designation (`.WithTimeBucketProperty(...)`) and no property mapping to the default bucket column `time_bucket` emits a warning through the configured EF logger. The view still exposes a `time_bucket` column, but it cannot be queried through the entity; previously this surfaced only at query time as an opaque Postgres "column does not exist" error. Remedy by designating a property or mapping one to `time_bucket`. This is a warning rather than an exception because deliberately not exposing the bucket is legal. The warning is raised as `TimescaleDbEventId.TimeBucketColumnUnmapped` and can be suppressed with `ConfigureWarnings(w => w.Ignore(...))` or escalated to an exception with `w.Throw(...)`. -> :warning: **Note:** Entities scaffolded with a raw view definition are exempt from all checks, because the structured projection fields are unused on that path. +> :warning: **Note:** Aggregates defined with `IsContinuousAggregateFromSql` (including scaffolded ones) are exempt from the other checks, because the structured projection fields are unused on that path. ## Hierarchical Continuous Aggregates diff --git a/src/Eftdb.Design/Features/ContinuousAggregate/ContinuousAggregateAnnotationRenderer.cs b/src/Eftdb.Design/Features/ContinuousAggregate/ContinuousAggregateAnnotationRenderer.cs index 8dbf737..ce0d149 100644 --- a/src/Eftdb.Design/Features/ContinuousAggregate/ContinuousAggregateAnnotationRenderer.cs +++ b/src/Eftdb.Design/Features/ContinuousAggregate/ContinuousAggregateAnnotationRenderer.cs @@ -7,6 +7,7 @@ using Microsoft.EntityFrameworkCore.Design.Internal; using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Metadata; +using System.Diagnostics.CodeAnalysis; using System.Reflection; using static CmdScale.EntityFrameworkCore.TimescaleDB.Design.Generators.AnnotationRendererHelper; @@ -17,8 +18,10 @@ namespace CmdScale.EntityFrameworkCore.TimescaleDB.Design.Features.ContinuousAgg /// Renders ContinuousAggregate annotations as IsContinuousAggregate(...) Fluent API chains or /// [ContinuousAggregate] attributes. The ViewDefinition annotation is parsed to reconstruct /// the structured configuration; on success it is consumed so no raw SQL leaks into the generated file. - /// When the view definition cannot be parsed, a warning is reported and the annotations are left in - /// place so the .HasAnnotation(...) fallback preserves the configuration. + /// A view the structured builder cannot express (unparseable, or with columns such as toolkit sketches) + /// is rendered as IsContinuousAggregateFromSql(...) with its SQL verbatim, and a warning is + /// reported. In data-annotations mode such a view keeps its annotations, so the .HasAnnotation(...) + /// fallback preserves the configuration. /// internal sealed class ContinuousAggregateAnnotationRenderer(IOperationReporter reporter) : IFeatureAnnotationRenderer { @@ -28,6 +31,9 @@ internal sealed class ContinuousAggregateAnnotationRenderer(IOperationReporter r .First(m => m.Name == nameof(ContinuousAggregateTypeBuilder.IsContinuousAggregate) && m.GetParameters().Length == 5); + private static readonly MethodInfo IsContinuousAggregateFromSqlMethod = + typeof(ContinuousAggregateTypeBuilder).GetMethod(nameof(ContinuousAggregateTypeBuilder.IsContinuousAggregateFromSql))!; + private static MethodInfo BuilderMethod(string name) => typeof(ContinuousAggregateStringBuilder<>) .GetMethods(BindingFlags.Public | BindingFlags.Instance) @@ -55,23 +61,19 @@ public IReadOnlyList GenerateFluentApiCalls( } string? viewDefinition = GetString(annotations, ContinuousAggregateAnnotations.ViewDefinition); + if (viewDefinition is null) + { + ReportUnparseableViewDefinition(materializedViewName); + return []; + } + string? parentName = GetString(annotations, ContinuousAggregateAnnotations.ParentName); string? chunkInterval = GetString(annotations, ContinuousAggregateAnnotations.ChunkInterval); bool materializedOnly = Find(annotations, ContinuousAggregateAnnotations.MaterializedOnly)?.Value is true; bool withNoData = Find(annotations, ContinuousAggregateAnnotations.WithNoData)?.Value is true; bool? createGroupIndexes = Find(annotations, ContinuousAggregateAnnotations.CreateGroupIndexes)?.Value as bool?; - ViewDefinitionParser.ParsedViewDefinition? parsed = viewDefinition is not null - ? ViewDefinitionParser.Parse(viewDefinition) - : null; - - if (parsed?.TimeBucketWidth is null || parsed.TimeBucketSourceColumn is null) - { - ReportUnparseableViewDefinition(materializedViewName); - return []; - } - - string humanizedWidth = IntervalParsingHelper.NormalizeInterval(parsed.TimeBucketWidth); + ViewDefinitionParser.ParsedViewDefinition parsed = ViewDefinitionParser.Parse(viewDefinition); IEntityType? parentEntityType = ParentEntityTypeResolver.Resolve(entityType.Model, parentName); string parentClrName = parentEntityType?.ShortName() ?? parentName ?? materializedViewName; @@ -80,14 +82,48 @@ public IReadOnlyList GenerateFluentApiCalls( ? new NameOfCodeFragment(parentClrName) : (object)(parentName ?? string.Empty); - object timeBucketArg = ResolveParentColumnArg(parentEntityType, parentClrName, parsed.TimeBucketSourceColumn); - - MethodCallCodeFragment call = new(IsContinuousAggregateMethod, materializedViewName, parentNameArg, humanizedWidth, timeBucketArg); - string caEntityClrName = entityType.ShortName(); - if (TryResolveTimeBucketProperty(entityType, parsed.TimeBucketAlias, out string bucketProperty)) + MethodCallCodeFragment call; + if (!IsStructurallyRepresentable(parsed)) { - call = call.Chain(WithTimeBucketPropertyMethod, new NameOfCodeFragment($"{caEntityClrName}.{bucketProperty}")); + reporter.WriteWarning( + $"The view definition of continuous aggregate '{materializedViewName}' has columns the structured " + + "builder cannot express. It is scaffolded with IsContinuousAggregateFromSql(...) and its SQL is kept verbatim."); + call = new MethodCallCodeFragment(IsContinuousAggregateFromSqlMethod, materializedViewName, parentNameArg, viewDefinition.Trim()); + } + else + { + string humanizedWidth = IntervalParsingHelper.NormalizeInterval(parsed.TimeBucketWidth!); + object timeBucketArg = ResolveParentColumnArg(parentEntityType, parentClrName, parsed.TimeBucketSourceColumn!); + call = new(IsContinuousAggregateMethod, materializedViewName, parentNameArg, humanizedWidth, timeBucketArg); + + if (TryResolveTimeBucketProperty(entityType, parsed.TimeBucketAlias, out string bucketProperty)) + { + call = call.Chain(WithTimeBucketPropertyMethod, new NameOfCodeFragment($"{caEntityClrName}.{bucketProperty}")); + } + + foreach (ViewDefinitionParser.ParsedAggregate agg in parsed.Aggregates) + { + object aliasArg = TryResolvePropertyName(entityType, agg.Alias, out string aliasProperty) + ? new NameOfCodeFragment($"{caEntityClrName}.{aliasProperty}") + : (object)agg.Alias; + + object sourceArg = agg.SourceColumn == "*" + ? (object)"*" + : ResolveParentColumnArg(parentEntityType, parentClrName, agg.SourceColumn); + + call = call.Chain(AddAggregateFunctionMethod, [aliasArg, sourceArg, agg.Function]); + } + + foreach (string col in parsed.GroupByColumns) + { + call = call.Chain(AddGroupByColumnMethod, ResolveParentColumnArg(parentEntityType, parentClrName, col)); + } + + if (!string.IsNullOrWhiteSpace(parsed.WhereClause)) + { + call = call.Chain(WhereMethod, parsed.WhereClause); + } } if (materializedOnly) @@ -105,29 +141,6 @@ public IReadOnlyList GenerateFluentApiCalls( call = call.Chain(CreateGroupIndexesMethod, false); } - foreach (ViewDefinitionParser.ParsedAggregate agg in parsed.Aggregates) - { - object aliasArg = TryResolvePropertyName(entityType, agg.Alias, out string aliasProperty) - ? new NameOfCodeFragment($"{caEntityClrName}.{aliasProperty}") - : (object)agg.Alias; - - object sourceArg = agg.SourceColumn == "*" - ? (object)"*" - : ResolveParentColumnArg(parentEntityType, parentClrName, agg.SourceColumn); - - call = call.Chain(AddAggregateFunctionMethod, [aliasArg, sourceArg, agg.Function]); - } - - foreach (string col in parsed.GroupByColumns) - { - call = call.Chain(AddGroupByColumnMethod, ResolveParentColumnArg(parentEntityType, parentClrName, col)); - } - - if (!string.IsNullOrWhiteSpace(parsed.WhereClause)) - { - call = call.Chain(WhereMethod, parsed.WhereClause); - } - if (!string.IsNullOrWhiteSpace(chunkInterval) && !IsDerivedDefaultChunkInterval(chunkInterval, parentEntityType)) { call = call.Chain(WithChunkIntervalMethod, IntervalParsingHelper.NormalizeInterval(chunkInterval)); @@ -178,7 +191,7 @@ public IReadOnlyList GenerateDataAnnotationAttributes( ? ViewDefinitionParser.Parse(viewDefinition) : null; - if (parsed?.TimeBucketWidth is null || parsed.TimeBucketSourceColumn is null) + if (!IsStructurallyRepresentable(parsed)) { return []; } @@ -189,7 +202,7 @@ public IReadOnlyList GenerateDataAnnotationAttributes( ConsumeAllCaAnnotations(annotations); - string humanizedWidth = IntervalParsingHelper.NormalizeInterval(parsed.TimeBucketWidth); + string humanizedWidth = IntervalParsingHelper.NormalizeInterval(parsed.TimeBucketWidth!); IEntityType? parentEntityType = ParentEntityTypeResolver.Resolve(entityType.Model, parentName); string parentClrName = parentEntityType?.ShortName() ?? parentName ?? materializedViewName; @@ -198,7 +211,7 @@ public IReadOnlyList GenerateDataAnnotationAttributes( ? new NameOfCodeFragment(parentClrName) : (object)(parentName ?? string.Empty); - object timeBucketArg = ResolveParentColumnArg(parentEntityType, parentClrName, parsed.TimeBucketSourceColumn); + object timeBucketArg = ResolveParentColumnArg(parentEntityType, parentClrName, parsed.TimeBucketSourceColumn!); ReportUnrepresentableGroupByEntries(entityType, materializedViewName, parsed.GroupByColumns); @@ -277,7 +290,7 @@ public void ConsumeFeatureAnnotations(IEntityType entityType, IDictionary parsed is { TimeBucketWidth: not null, TimeBucketSourceColumn: not null, IsComplete: true }; + /// /// A continuous aggregate's chunk interval defaults to 10x the parent hypertable's chunk /// interval; a value equal to that derived default is elided, symmetric with the hypertable's diff --git a/src/Eftdb.Design/Features/ContinuousAggregate/ContinuousAggregateCSharpGenerator.cs b/src/Eftdb.Design/Features/ContinuousAggregate/ContinuousAggregateCSharpGenerator.cs index 191656e..b460075 100644 --- a/src/Eftdb.Design/Features/ContinuousAggregate/ContinuousAggregateCSharpGenerator.cs +++ b/src/Eftdb.Design/Features/ContinuousAggregate/ContinuousAggregateCSharpGenerator.cs @@ -67,6 +67,12 @@ public void Generate(CreateContinuousAggregateOperation operation, IndentedStrin if (operation.CompressionOrderBy is { Count: > 0 }) call.Arg("compressionOrderBy", CSharpGeneratorHelper.LiteralStringList(code, operation.CompressionOrderBy)); + + if (operation.AggregateExpressions is { Count: > 0 }) + call.Arg("aggregateExpressions", b => AppendExpressionList(b, operation.AggregateExpressions)); + + if (operation.GroupByExpressions is { Count: > 0 }) + call.Arg("groupByExpressions", b => AppendExpressionList(b, operation.GroupByExpressions)); } public void Generate(AlterContinuousAggregateOperation operation, IndentedStringBuilder builder) @@ -121,6 +127,27 @@ public void Generate(DropContinuousAggregateOperation operation, IndentedStringB call.Arg("schema", code.Literal(operation.Schema)); } + private void AppendExpressionList(IndentedStringBuilder builder, IReadOnlyList expressions) + { + string typeRef = typeof(ContinuousAggregateExpression).FullName!; + AppendCollection(builder, [.. expressions.Select(x => + $"new {typeRef}({code.Literal(x.Alias)}, {code.Literal(x.Sql)}, {CSharpGeneratorHelper.LiteralStringList(code, x.Columns)})")]); + } + + // Writes the entries as a multi-line collection expression + private static void AppendCollection(IndentedStringBuilder builder, IReadOnlyList entries) + { + builder.AppendLine("["); + using (builder.Indent()) + { + for (int i = 0; i < entries.Count; i++) + { + builder.AppendLine(i < entries.Count - 1 ? entries[i] + "," : entries[i]); + } + } + builder.Append("]"); + } + // Writes the aggregate functions as a collection expression private void AppendAggregateFunctionList(IndentedStringBuilder builder, IReadOnlyList aggregateFunctions) { @@ -139,15 +166,7 @@ private void AppendAggregateFunctionList(IndentedStringBuilder builder, IReadOnl entries.Add($"new {typeRef}({code.Literal(parts[0])}, {enumRef}.{parts[1]}, {code.Literal(parts[2])})"); } - builder.AppendLine("["); - using (builder.Indent()) - { - for (int i = 0; i < entries.Count; i++) - { - builder.AppendLine(i < entries.Count - 1 ? entries[i] + "," : entries[i]); - } - } - builder.Append("]"); + AppendCollection(builder, entries); } } } diff --git a/src/Eftdb.Design/Generators/TimescaleDbAnnotationCodeGenerator.cs b/src/Eftdb.Design/Generators/TimescaleDbAnnotationCodeGenerator.cs index 4183840..918f355 100644 --- a/src/Eftdb.Design/Generators/TimescaleDbAnnotationCodeGenerator.cs +++ b/src/Eftdb.Design/Generators/TimescaleDbAnnotationCodeGenerator.cs @@ -173,6 +173,7 @@ private static IReadOnlyList GenerateContinuousAggregateP if (entityType.FindAnnotation(ContinuousAggregateAnnotations.ViewDefinition)?.Value is not string viewDefinition) return []; ViewDefinitionParser.ParsedViewDefinition parsed = ViewDefinitionParser.Parse(viewDefinition); + if (!parsed.IsComplete) return []; string viewName = entityType.GetViewName() ?? entityType.GetTableName() ?? entityType.Name; string? viewSchema = entityType.GetViewSchema() ?? entityType.GetSchema(); diff --git a/src/Eftdb.Design/Scaffolding/ViewDefinitionParser.cs b/src/Eftdb.Design/Scaffolding/ViewDefinitionParser.cs index 6fdd706..7f56767 100644 --- a/src/Eftdb.Design/Scaffolding/ViewDefinitionParser.cs +++ b/src/Eftdb.Design/Scaffolding/ViewDefinitionParser.cs @@ -16,7 +16,10 @@ internal sealed record ParsedAggregate( string SourceColumn); /// - /// Complete parse result of a continuous aggregate view definition. + /// Complete parse result of a continuous aggregate view definition. is + /// false when a SELECT item is neither the time bucket, a recognised aggregate nor a + /// GROUP BY column (for example a toolkit sketch or a grouped expression), so the structured + /// fields alone would drop it. /// internal sealed record ParsedViewDefinition( string? TimeBucketWidth, @@ -24,7 +27,8 @@ internal sealed record ParsedViewDefinition( string? TimeBucketAlias, IReadOnlyList Aggregates, IReadOnlyList GroupByColumns, - string? WhereClause); + string? WhereClause, + bool IsComplete); private static readonly ConcurrentDictionary Cache = new(); @@ -33,13 +37,53 @@ internal sealed record ParsedViewDefinition( /// property against the same SQL; the cache avoids re-parsing the full definition each time. /// public static ParsedViewDefinition Parse(string viewDefinition) - => Cache.GetOrAdd(viewDefinition, static vd => new ParsedViewDefinition( - ParseTimeBucketWidth(vd), - ParseTimeBucketSourceColumn(vd), - ParseTimeBucketAlias(vd), - ParseAggregates(vd), - ParseGroupByColumns(vd), - ParseWhereClause(vd))); + => Cache.GetOrAdd(viewDefinition, static vd => + { + IReadOnlyList aggregates = ParseAggregates(vd); + IReadOnlyList groupByColumns = ParseGroupByColumns(vd); + return new ParsedViewDefinition( + ParseTimeBucketWidth(vd), + ParseTimeBucketSourceColumn(vd), + ParseTimeBucketAlias(vd), + aggregates, + groupByColumns, + ParseWhereClause(vd), + AccountsForEverySelectItem(vd, aggregates, groupByColumns)); + }); + + /// + /// Checks that each top-level SELECT item is the time bucket, one of , + /// or a plain column listed in . + /// + private static bool AccountsForEverySelectItem(string viewDefinition, IReadOnlyList aggregates, IReadOnlyList groupByColumns) + { + string selectList = SelectKeywordRegex().Replace(ExtractSelectClause(viewDefinition), string.Empty, 1); + HashSet aggregateAliases = [.. aggregates.Select(a => a.Alias)]; + HashSet groupColumns = [.. groupByColumns]; + + foreach (string rawItem in SplitTopLevel(selectList)) + { + string item = rawItem.Trim(); + if (item.Length == 0 || item.Contains("time_bucket", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + Match aliased = SelectItemAliasRegex().Match(item); + string expression = aliased.Success ? aliased.Groups[1].Value.Trim() : item; + string alias = aliased.Success ? StripQuotes(aliased.Groups[2].Value) : StripQualifierAndQuotes(item); + + bool isAggregate = aggregateAliases.Contains(alias); + bool isGroupColumn = SimpleColumnReferenceRegex().IsMatch(expression) + && groupColumns.Contains(StripQualifierAndQuotes(expression)); + if (!isAggregate && !isGroupColumn) + { + return false; + } + } + + return true; + } /// /// Extracts the time bucket interval from a time_bucket('interval'::interval, col) call. @@ -272,6 +316,12 @@ private static List SplitTopLevel(string input) [GeneratedRegex(@"\bFROM\b", RegexOptions.IgnoreCase)] private static partial Regex FromRegex(); + [GeneratedRegex(@"^\s*SELECT\s+", RegexOptions.IgnoreCase)] + private static partial Regex SelectKeywordRegex(); + + [GeneratedRegex(@"^(.*)\s+AS\s+(""[^""]+""|\w+)$", RegexOptions.IgnoreCase | RegexOptions.Singleline)] + private static partial Regex SelectItemAliasRegex(); + [GeneratedRegex(@"^(?:(?:""[^""]+""|\w+)\.)*(""[^""]+""|\w+)$")] private static partial Regex SimpleColumnReferenceRegex(); diff --git a/src/Eftdb/Abstractions/ContinuousAggregateExpression.cs b/src/Eftdb/Abstractions/ContinuousAggregateExpression.cs new file mode 100644 index 0000000..b4f5256 --- /dev/null +++ b/src/Eftdb/Abstractions/ContinuousAggregateExpression.cs @@ -0,0 +1,32 @@ +namespace CmdScale.EntityFrameworkCore.TimescaleDB.Abstractions +{ + /// + /// Maps a column on a continuous aggregate to a SQL expression over source columns — e.g. + /// actors = mcv_agg(500, actor_id) or impersonated = impersonated_actor_id IS NOT NULL. + /// Column references in are written as {0}, {1}, … placeholders, + /// each bound to the entry at that index in ; literal braces are written as + /// {{ and }}. + /// + public sealed class ContinuousAggregateExpression(string alias, string sql, IReadOnlyList columns) + : IEquatable + { + /// The name of the resulting column on the continuous aggregate. + public string Alias { get; } = alias; + + /// The SQL expression template, with {n} placeholders for column references. + public string Sql { get; } = sql; + + /// + /// The source columns bound to the placeholders. In model annotations these are property names, + /// complex-type paths or column names; on migration operations they are database column names. + /// + public IReadOnlyList Columns { get; } = columns; + + public bool Equals(ContinuousAggregateExpression? other) + => other is not null && Alias == other.Alias && Sql == other.Sql && Columns.SequenceEqual(other.Columns); + + public override bool Equals(object? obj) => Equals(obj as ContinuousAggregateExpression); + + public override int GetHashCode() => HashCode.Combine(Alias, Sql, Columns.Count); + } +} diff --git a/src/Eftdb/Configuration/ContinuousAggregate/AggregateExpressionAttribute.cs b/src/Eftdb/Configuration/ContinuousAggregate/AggregateExpressionAttribute.cs new file mode 100644 index 0000000..ee569c6 --- /dev/null +++ b/src/Eftdb/Configuration/ContinuousAggregate/AggregateExpressionAttribute.cs @@ -0,0 +1,21 @@ +namespace CmdScale.EntityFrameworkCore.TimescaleDB.Configuration.ContinuousAggregate +{ + /// + /// Defines a property of a continuous aggregate entity as a SQL aggregate expression, such as a + /// toolkit sketch: [AggregateExpression("mcv_agg(500, {0})", nameof(Event.ActorId))]. + /// + /// + /// The aggregate expression template. Column references are {0}, {1}, … placeholders + /// bound to ; literal braces are written as {{ and }}. + /// + /// Source property or column names bound to the placeholders, in order. + [AttributeUsage(AttributeTargets.Property)] + public sealed class AggregateExpressionAttribute(string sql, params string[] columns) : Attribute + { + /// The aggregate expression template. + public string Sql { get; } = sql; + + /// The source columns bound to the placeholders. + public string[] Columns { get; } = columns; + } +} diff --git a/src/Eftdb/Configuration/ContinuousAggregate/ContinuousAggregateAnnotations.cs b/src/Eftdb/Configuration/ContinuousAggregate/ContinuousAggregateAnnotations.cs index 3faeffc..f97e6e7 100644 --- a/src/Eftdb/Configuration/ContinuousAggregate/ContinuousAggregateAnnotations.cs +++ b/src/Eftdb/Configuration/ContinuousAggregate/ContinuousAggregateAnnotations.cs @@ -22,6 +22,18 @@ public static class ContinuousAggregateAnnotations public const string WhereClause = "TimescaleDB:WhereClause"; public const string GroupByColumns = "TimescaleDB:GroupByColumns"; + /// + /// JSON array of entries selected as + /// aggregate columns. Columns hold model names, resolved to database columns at extraction. + /// + public const string AggregateExpressions = "TimescaleDB:AggregateExpressions"; + + /// + /// JSON array of entries selected as + /// aliased columns and repeated in GROUP BY. + /// + public const string GroupByExpressions = "TimescaleDB:GroupByExpressions"; + /// /// Raw SQL view body captured by the design-time scaffolder. When present, the /// runtime extractor and generator use it verbatim as the SELECT clause of diff --git a/src/Eftdb/Configuration/ContinuousAggregate/ContinuousAggregateBuilder.cs b/src/Eftdb/Configuration/ContinuousAggregate/ContinuousAggregateBuilder.cs index c5bbed2..958af37 100644 --- a/src/Eftdb/Configuration/ContinuousAggregate/ContinuousAggregateBuilder.cs +++ b/src/Eftdb/Configuration/ContinuousAggregate/ContinuousAggregateBuilder.cs @@ -101,6 +101,56 @@ public ContinuousAggregateBuilder AddGroupByColumn(strin return this; } + /// + /// Adds an aggregate column computed by a SQL expression, such as a toolkit sketch + /// (mcv_agg(500, {0})) or a filtered count (count(*) FILTER (WHERE {0} = 1)). + /// Column references are {0}, {1}, … placeholders bound to , + /// so naming conventions apply to them; literal braces are written as {{ and }}. + /// + /// The type of the property on the continuous aggregate. + /// Expression selecting the property that receives the result. + /// The aggregate expression template. + /// Expressions selecting the source columns bound to the placeholders, in order. + /// The builder for method chaining. + /// The placeholders and do not match. + public ContinuousAggregateBuilder AddAggregateExpression( + Expression> propertyExpression, + string sql, + params Expression>[] columns) + { + ContinuousAggregateBuilderCore.AddExpression( + EntityTypeBuilder, + ContinuousAggregateAnnotations.AggregateExpressions, + ExpressionHelper.GetPropertyName(propertyExpression), + sql, + [.. columns.Select(ExpressionHelper.GetPropertyName)]); + return this; + } + + /// + /// Adds a GROUP BY expression selected as a column, such as {0} IS NOT NULL. + /// Column references are {0}, {1}, … placeholders bound to . + /// + /// The type of the property on the continuous aggregate. + /// Expression selecting the property that receives the grouped value. + /// The grouping expression template. + /// Expressions selecting the source columns bound to the placeholders, in order. + /// The builder for method chaining. + /// The placeholders and do not match. + public ContinuousAggregateBuilder AddGroupByExpression( + Expression> propertyExpression, + string sql, + params Expression>[] columns) + { + ContinuousAggregateBuilderCore.AddExpression( + EntityTypeBuilder, + ContinuousAggregateAnnotations.GroupByExpressions, + ExpressionHelper.GetPropertyName(propertyExpression), + sql, + [.. columns.Select(ExpressionHelper.GetPropertyName)]); + return this; + } + /// /// Designates the property on the aggregate entity that represents the bucket column. /// The view's bucket alias derives from that property's mapped column name, removing the diff --git a/src/Eftdb/Configuration/ContinuousAggregate/ContinuousAggregateBuilderCore.cs b/src/Eftdb/Configuration/ContinuousAggregate/ContinuousAggregateBuilderCore.cs index 660fb4a..a176e8f 100644 --- a/src/Eftdb/Configuration/ContinuousAggregate/ContinuousAggregateBuilderCore.cs +++ b/src/Eftdb/Configuration/ContinuousAggregate/ContinuousAggregateBuilderCore.cs @@ -1,5 +1,6 @@ using CmdScale.EntityFrameworkCore.TimescaleDB.Abstractions; using CmdScale.EntityFrameworkCore.TimescaleDB.Configuration.Hypertable; +using CmdScale.EntityFrameworkCore.TimescaleDB.Internals; using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Metadata.Builders; @@ -98,5 +99,20 @@ public static void AddGroupByColumn(EntityTypeBuilder builder, string groupByExp groupByColumns.Add(groupByExpression); builder.HasAnnotation(ContinuousAggregateAnnotations.GroupByColumns, groupByColumns); } + + /// + /// Appends an expression to the JSON list (aggregate or group-by + /// expressions). An alias already present is replaced. + /// + /// The template and columns do not match. + public static void AddExpression(EntityTypeBuilder builder, string annotation, string alias, string sql, IReadOnlyList columns) + { + ContinuousAggregateExpressionJson.Validate(alias, sql, columns); + + List expressions = ContinuousAggregateExpressionJson.Read(builder.Metadata, annotation); + expressions.RemoveAll(x => x.Alias == alias); + expressions.Add(new ContinuousAggregateExpression(alias, sql, [.. columns])); + builder.HasAnnotation(annotation, ContinuousAggregateExpressionJson.Write(expressions)); + } } } diff --git a/src/Eftdb/Configuration/ContinuousAggregate/ContinuousAggregateConvention.cs b/src/Eftdb/Configuration/ContinuousAggregate/ContinuousAggregateConvention.cs index 7f6bfdf..874a184 100644 --- a/src/Eftdb/Configuration/ContinuousAggregate/ContinuousAggregateConvention.cs +++ b/src/Eftdb/Configuration/ContinuousAggregate/ContinuousAggregateConvention.cs @@ -1,4 +1,6 @@ +using CmdScale.EntityFrameworkCore.TimescaleDB.Abstractions; using CmdScale.EntityFrameworkCore.TimescaleDB.Configuration.Hypertable; +using CmdScale.EntityFrameworkCore.TimescaleDB.Internals; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.EntityFrameworkCore.Metadata.Builders; @@ -8,7 +10,8 @@ namespace CmdScale.EntityFrameworkCore.TimescaleDB.Configuration.ContinuousAggregate { /// - /// Reads the [ContinuousAggregate], [TimeBucket], and [Aggregate] attributes + /// Reads the [ContinuousAggregate], [TimeBucket], [Aggregate], [AggregateExpression], [GroupByColumn] + /// and [GroupByExpression] attributes /// to configure an entity as a TimescaleDB continuous aggregate. /// internal class ContinuousAggregateConvention : IEntityTypeAddedConvention @@ -65,6 +68,8 @@ public void ProcessEntityTypeAdded(IConventionEntityTypeBuilder entityTypeBuilde // Discover property-level configurations List aggregateFunctions = []; List groupByColumns = []; + List aggregateExpressions = []; + List groupByExpressions = []; foreach (IConventionProperty property in entityType.GetProperties()) { @@ -82,6 +87,20 @@ public void ProcessEntityTypeAdded(IConventionEntityTypeBuilder entityTypeBuilde aggregateFunctions.Add($"{property.Name}:{aggregateAttr.Function}:{sourceColumn}"); } + AggregateExpressionAttribute? aggregateExpressionAttr = propertyInfo.GetCustomAttribute(); + if (aggregateExpressionAttr != null) + { + ContinuousAggregateExpressionJson.Validate(property.Name, aggregateExpressionAttr.Sql, aggregateExpressionAttr.Columns); + aggregateExpressions.Add(new ContinuousAggregateExpression(property.Name, aggregateExpressionAttr.Sql, aggregateExpressionAttr.Columns)); + } + + GroupByExpressionAttribute? groupByExpressionAttr = propertyInfo.GetCustomAttribute(); + if (groupByExpressionAttr != null) + { + ContinuousAggregateExpressionJson.Validate(property.Name, groupByExpressionAttr.Sql, groupByExpressionAttr.Columns); + groupByExpressions.Add(new ContinuousAggregateExpression(property.Name, groupByExpressionAttr.Sql, groupByExpressionAttr.Columns)); + } + GroupByColumnAttribute? groupByAttr = propertyInfo.GetCustomAttribute(); if (groupByAttr != null) { @@ -108,6 +127,16 @@ public void ProcessEntityTypeAdded(IConventionEntityTypeBuilder entityTypeBuilde { entityTypeBuilder.HasAnnotation(ContinuousAggregateAnnotations.GroupByColumns, groupByColumns); } + + if (aggregateExpressions.Count != 0) + { + entityTypeBuilder.HasAnnotation(ContinuousAggregateAnnotations.AggregateExpressions, ContinuousAggregateExpressionJson.Write(aggregateExpressions)); + } + + if (groupByExpressions.Count != 0) + { + entityTypeBuilder.HasAnnotation(ContinuousAggregateAnnotations.GroupByExpressions, ContinuousAggregateExpressionJson.Write(groupByExpressions)); + } } } } diff --git a/src/Eftdb/Configuration/ContinuousAggregate/ContinuousAggregateStringBuilder.cs b/src/Eftdb/Configuration/ContinuousAggregate/ContinuousAggregateStringBuilder.cs index 8cd12d8..d8c3424 100644 --- a/src/Eftdb/Configuration/ContinuousAggregate/ContinuousAggregateStringBuilder.cs +++ b/src/Eftdb/Configuration/ContinuousAggregate/ContinuousAggregateStringBuilder.cs @@ -89,6 +89,36 @@ public ContinuousAggregateStringBuilder AddGroupByColumn(string groupBy return this; } + /// + /// Adds an aggregate column computed by a SQL expression whose {0}, {1}, … + /// placeholders are bound to . + /// + /// The output column or property name on the continuous aggregate view. + /// The aggregate expression template. + /// Source property or column names bound to the placeholders, in order. + /// The builder for method chaining. + /// The placeholders and do not match. + public ContinuousAggregateStringBuilder AddAggregateExpression(string alias, string sql, params string[] columns) + { + ContinuousAggregateBuilderCore.AddExpression(_builder, ContinuousAggregateAnnotations.AggregateExpressions, alias, sql, columns); + return this; + } + + /// + /// Adds a GROUP BY expression selected as a column, whose {0}, {1}, … + /// placeholders are bound to . + /// + /// The output column or property name on the continuous aggregate view. + /// The grouping expression template. + /// Source property or column names bound to the placeholders, in order. + /// The builder for method chaining. + /// The placeholders and do not match. + public ContinuousAggregateStringBuilder AddGroupByExpression(string alias, string sql, params string[] columns) + { + ContinuousAggregateBuilderCore.AddExpression(_builder, ContinuousAggregateAnnotations.GroupByExpressions, alias, sql, columns); + return this; + } + /// /// Designates the property on the aggregate entity that represents the bucket column. /// The view's bucket alias derives from that property's mapped column name, removing the diff --git a/src/Eftdb/Configuration/ContinuousAggregate/ContinuousAggregateTypeBuilder.cs b/src/Eftdb/Configuration/ContinuousAggregate/ContinuousAggregateTypeBuilder.cs index 585a129..bfdd944 100644 --- a/src/Eftdb/Configuration/ContinuousAggregate/ContinuousAggregateTypeBuilder.cs +++ b/src/Eftdb/Configuration/ContinuousAggregate/ContinuousAggregateTypeBuilder.cs @@ -145,6 +145,38 @@ public static ContinuousAggregateStringBuilder IsContinuousAggregate(entityTypeBuilder); } + /// + /// Configures the entity as a TimescaleDB continuous aggregate whose body is the given SELECT + /// statement, used verbatim. Identifiers in it are not resolved through the model, so it must use + /// database column names. Prefer the structured overloads; this one covers definitions they cannot + /// express. + /// + /// The continuous aggregate entity type. + /// The entity type builder. + /// The name of the materialized view. + /// The name of the source hypertable entity or its database table name. + /// The SELECT statement that defines the view. + /// A builder for further continuous aggregate configuration. + /// Thrown when is empty. + public static ContinuousAggregateStringBuilder IsContinuousAggregateFromSql( + this EntityTypeBuilder entityTypeBuilder, + string materializedViewName, + string parentName, + string viewDefinition) + where TEntity : class + { + if (string.IsNullOrWhiteSpace(viewDefinition)) + { + throw new ArgumentException("The view definition must not be empty.", nameof(viewDefinition)); + } + + entityTypeBuilder.ToView(materializedViewName, entityTypeBuilder.Metadata.GetViewSchema()); + entityTypeBuilder.HasAnnotation(ContinuousAggregateAnnotations.MaterializedViewName, materializedViewName); + entityTypeBuilder.HasAnnotation(ContinuousAggregateAnnotations.ParentName, parentName); + entityTypeBuilder.HasAnnotation(ContinuousAggregateAnnotations.ViewDefinition, viewDefinition); + return new ContinuousAggregateStringBuilder(entityTypeBuilder); + } + private static ContinuousAggregateBuilder IsContinuousAggregateCore( EntityTypeBuilder entityTypeBuilder, string materializedViewName, diff --git a/src/Eftdb/Configuration/ContinuousAggregate/GroupByExpressionAttribute.cs b/src/Eftdb/Configuration/ContinuousAggregate/GroupByExpressionAttribute.cs new file mode 100644 index 0000000..64354c0 --- /dev/null +++ b/src/Eftdb/Configuration/ContinuousAggregate/GroupByExpressionAttribute.cs @@ -0,0 +1,21 @@ +namespace CmdScale.EntityFrameworkCore.TimescaleDB.Configuration.ContinuousAggregate +{ + /// + /// Defines a property of a continuous aggregate entity as a GROUP BY expression selected as a column: + /// [GroupByExpression("{0} IS NOT NULL", nameof(Event.ImpersonatedActorId))]. + /// + /// + /// The grouping expression template. Column references are {0}, {1}, … placeholders + /// bound to ; literal braces are written as {{ and }}. + /// + /// Source property or column names bound to the placeholders, in order. + [AttributeUsage(AttributeTargets.Property)] + public sealed class GroupByExpressionAttribute(string sql, params string[] columns) : Attribute + { + /// The grouping expression template. + public string Sql { get; } = sql; + + /// The source columns bound to the placeholders. + public string[] Columns { get; } = columns; + } +} diff --git a/src/Eftdb/Configuration/ContinuousAggregateViewColumnValidationConvention.cs b/src/Eftdb/Configuration/ContinuousAggregateViewColumnValidationConvention.cs index ffdc416..d28b694 100644 --- a/src/Eftdb/Configuration/ContinuousAggregateViewColumnValidationConvention.cs +++ b/src/Eftdb/Configuration/ContinuousAggregateViewColumnValidationConvention.cs @@ -1,3 +1,4 @@ +using CmdScale.EntityFrameworkCore.TimescaleDB.Abstractions; using CmdScale.EntityFrameworkCore.TimescaleDB.Configuration.ContinuousAggregate; using CmdScale.EntityFrameworkCore.TimescaleDB.Internals; using Microsoft.EntityFrameworkCore; @@ -8,7 +9,7 @@ namespace CmdScale.EntityFrameworkCore.TimescaleDB.Configuration { /// /// Guards against colliding output column names in a structured continuous aggregate view. - /// The bucket column, group-by columns and aggregate aliases share the view's projection, so + /// The bucket column, group-by columns, aggregate aliases and expression aliases share the view's projection, so /// two of them resolving to the same database column would produce ambiguous SQL. Validation /// runs at model finalization against resolved store column names. /// @@ -37,10 +38,20 @@ private static void ValidateViewColumns(IModel model, IEntityType entityType) return; } + List aggregateExpressions = ContinuousAggregateExpressionJson.Read(entityType, ContinuousAggregateAnnotations.AggregateExpressions); + List groupByExpressions = ContinuousAggregateExpressionJson.Read(entityType, ContinuousAggregateAnnotations.GroupByExpressions); + // The raw view-definition path does not use the structured projection fields. string? viewDefinition = entityType.FindAnnotation(ContinuousAggregateAnnotations.ViewDefinition)?.Value as string; if (!string.IsNullOrWhiteSpace(viewDefinition)) { + if (aggregateExpressions.Count > 0 || groupByExpressions.Count > 0) + { + throw new InvalidOperationException( + $"The continuous aggregate '{EntityStoreObjectResolver.DisplayName(entityType)}' (materialized view '{materializedViewName}') " + + $"has a raw view definition, which replaces its aggregate and group-by expressions. Remove one or the other."); + } + return; } @@ -58,6 +69,11 @@ private static void ValidateViewColumns(IModel model, IEntityType entityType) List outputColumns = [bucketColumnName]; outputColumns.AddRange(ResolveGroupByColumns(entityType, parentEntityType, parentStoreIdentifier)); outputColumns.AddRange(ResolveAggregateAliasColumns(entityType, aggregateStoreIdentifier.Value)); + foreach (ContinuousAggregateExpression expression in groupByExpressions.Concat(aggregateExpressions)) + { + ValidateExpressionColumns(entityType, materializedViewName, expression, parentEntityType, parentStoreIdentifier); + outputColumns.Add(ColumnNameResolver.Resolve(entityType, expression.Alias, aggregateStoreIdentifier.Value) ?? expression.Alias); + } HashSet seen = []; foreach (string column in outputColumns) @@ -149,6 +165,30 @@ private static IEnumerable ResolveAggregateAliasColumns(IEntityType enti } } + private static void ValidateExpressionColumns( + IEntityType entityType, + string materializedViewName, + ContinuousAggregateExpression expression, + IEntityType? parentEntityType, + StoreObjectIdentifier? parentStoreIdentifier) + { + if (parentEntityType == null || parentStoreIdentifier == null) + { + return; + } + + foreach (string column in expression.Columns) + { + if (ColumnNameResolver.Resolve(parentEntityType, column, parentStoreIdentifier.Value) == null) + { + throw new InvalidOperationException( + $"The continuous aggregate '{EntityStoreObjectResolver.DisplayName(entityType)}' (materialized view '{materializedViewName}') " + + $"references '{column}' in the expression for '{expression.Alias}', which is not a property or column of " + + $"'{EntityStoreObjectResolver.DisplayName(parentEntityType)}'."); + } + } + } + private static IEntityType? ResolveParent(IModel model, IEntityType entityType) { string? parentName = entityType.FindAnnotation(ContinuousAggregateAnnotations.ParentName)?.Value as string; diff --git a/src/Eftdb/Diagnostics/TimescaleDbEventId.cs b/src/Eftdb/Diagnostics/TimescaleDbEventId.cs index c2c53d6..9a83a5f 100644 --- a/src/Eftdb/Diagnostics/TimescaleDbEventId.cs +++ b/src/Eftdb/Diagnostics/TimescaleDbEventId.cs @@ -27,6 +27,7 @@ private enum Id CommunityFeatureSkipped = Base, TimeBucketColumnUnmapped, UnsupportedHypertableChangeSkipped, + ContinuousAggregateEntryIgnored, } /// @@ -52,6 +53,13 @@ private enum Id /// public static readonly EventId UnsupportedHypertableChangeSkipped = MakeMigrationsId(Id.UnsupportedHypertableChangeSkipped); + /// + /// A continuous aggregate entry configured with AddAggregateFunction or [Aggregate] is left out + /// of the view because it is malformed or its source column does not resolve against the parent. Raised at + /// model validation time in the category. + /// + public static readonly EventId ContinuousAggregateEntryIgnored = MakeValidationId(Id.ContinuousAggregateEntryIgnored); + private static readonly string MigrationsPrefix = DbLoggerCategory.Migrations.Name + "."; private static readonly string ValidationPrefix = DbLoggerCategory.Model.Validation.Name + "."; diff --git a/src/Eftdb/Diagnostics/TimescaleDbLoggerExtensions.cs b/src/Eftdb/Diagnostics/TimescaleDbLoggerExtensions.cs index 90c286e..215bf0e 100644 --- a/src/Eftdb/Diagnostics/TimescaleDbLoggerExtensions.cs +++ b/src/Eftdb/Diagnostics/TimescaleDbLoggerExtensions.cs @@ -94,6 +94,60 @@ public static void UnsupportedHypertableChangeSkipped( } } + /// + /// Warns that a continuous aggregate entry is left out of the view. + /// + /// The model-validation diagnostics logger. + /// The continuous aggregate entity display name. + /// The materialized view name. + /// The ignored entry, in alias:Function:sourceColumn form. + /// Why the entry is ignored. + public static void ContinuousAggregateEntryIgnored( + this IDiagnosticsLogger diagnostics, + string aggregate, + string materializedView, + string entry, + string reason) + { + EventDefinition definition = LogContinuousAggregateEntryIgnored(diagnostics); + + if (diagnostics.ShouldLog(definition)) + { + definition.Log(diagnostics, aggregate, materializedView, entry, reason); + } + + if (diagnostics.NeedsEventData(definition, out bool diagnosticSourceEnabled, out bool simpleLogEnabled)) + { + EventData eventData = new( + definition, + (d, _) => ((EventDefinition)d).GenerateMessage(aggregate, materializedView, entry, reason)); + diagnostics.DispatchEventData(definition, eventData, diagnosticSourceEnabled, simpleLogEnabled); + } + } + + private static EventDefinition LogContinuousAggregateEntryIgnored(IDiagnosticsLogger diagnostics) + { + TimescaleDbLoggingDefinitions definitions = GetDefinitions(diagnostics); + EventDefinitionBase? definition = definitions.LogContinuousAggregateEntryIgnored; + if (definition == null) + { + EventDefinition created = new( + diagnostics.Options, + TimescaleDbEventId.ContinuousAggregateEntryIgnored, + LogLevel.Warning, + "TimescaleDbEventId.ContinuousAggregateEntryIgnored", + static level => LoggerMessage.Define( + level, + TimescaleDbEventId.ContinuousAggregateEntryIgnored, + "The continuous aggregate '{Aggregate}' (materialized view '{MaterializedView}') leaves out the aggregate " + + "entry '{Entry}': {Reason} Use AddAggregateExpression for aggregates that AddAggregateFunction cannot express.")); + + definition = Interlocked.CompareExchange(ref definitions.LogContinuousAggregateEntryIgnored, created, null) ?? created; + } + + return (EventDefinition)definition; + } + private static EventDefinition LogCommunityFeatureSkipped(IDiagnosticsLogger diagnostics) { TimescaleDbLoggingDefinitions definitions = GetDefinitions(diagnostics); diff --git a/src/Eftdb/Diagnostics/TimescaleDbLoggingDefinitions.cs b/src/Eftdb/Diagnostics/TimescaleDbLoggingDefinitions.cs index e80e6bf..3b8e6a8 100644 --- a/src/Eftdb/Diagnostics/TimescaleDbLoggingDefinitions.cs +++ b/src/Eftdb/Diagnostics/TimescaleDbLoggingDefinitions.cs @@ -25,6 +25,11 @@ public class TimescaleDbLoggingDefinitions : NpgsqlLoggingDefinitions /// Cached definition for . /// public EventDefinitionBase? LogUnsupportedHypertableChangeSkipped; + + /// + /// Cached definition for . + /// + public EventDefinitionBase? LogContinuousAggregateEntryIgnored; } #pragma warning restore EF1001 } diff --git a/src/Eftdb/Generators/ContinuousAggregateSqlGenerator.cs b/src/Eftdb/Generators/ContinuousAggregateSqlGenerator.cs index b0ce5da..5797b0d 100644 --- a/src/Eftdb/Generators/ContinuousAggregateSqlGenerator.cs +++ b/src/Eftdb/Generators/ContinuousAggregateSqlGenerator.cs @@ -1,3 +1,5 @@ +using CmdScale.EntityFrameworkCore.TimescaleDB.Abstractions; +using CmdScale.EntityFrameworkCore.TimescaleDB.Internals; using CmdScale.EntityFrameworkCore.TimescaleDB.Operations; using System.Text; @@ -107,6 +109,11 @@ private static List GenerateFromStructuredQuery( } } + foreach (ContinuousAggregateExpression groupByExpression in operation.GroupByExpressions) + { + selectList.Add($"{FormatExpression(groupByExpression)} AS {SqlBuilderHelper.QuoteIdentifier(groupByExpression.Alias)}"); + } + // Build aggregate functions foreach (string aggInfo in operation.AggregateFunctions) { @@ -140,6 +147,11 @@ private static List GenerateFromStructuredQuery( selectList.Add($"{aggregateExpression} AS {quotedAlias}"); } + foreach (ContinuousAggregateExpression aggregate in operation.AggregateExpressions) + { + selectList.Add($"{FormatExpression(aggregate)} AS {SqlBuilderHelper.QuoteIdentifier(aggregate.Alias)}"); + } + // Build the GROUP BY list List groupByList = []; if (operation.TimeBucketGroupBy) @@ -159,6 +171,11 @@ private static List GenerateFromStructuredQuery( } } + foreach (ContinuousAggregateExpression groupByExpression in operation.GroupByExpressions) + { + groupByList.Add($"({FormatExpression(groupByExpression)})"); + } + // Build the complete CREATE MATERIALIZED VIEW statement as a single string StringBuilder sqlBuilder = new(); sqlBuilder.Append($"CREATE MATERIALIZED VIEW {qualifiedIdentifier}"); @@ -264,6 +281,9 @@ public static List Generate(DropContinuousAggregateOperation operation) return statements; } + private static string FormatExpression(ContinuousAggregateExpression expression) + => ColumnTemplate.Format(expression.Sql, i => SqlBuilderHelper.QuoteIdentifier(expression.Columns[i])); + /// /// Translates the string representation of EAggregateFunction into a SQL function. /// diff --git a/src/Eftdb/Internals/ColumnTemplate.cs b/src/Eftdb/Internals/ColumnTemplate.cs new file mode 100644 index 0000000..a679917 --- /dev/null +++ b/src/Eftdb/Internals/ColumnTemplate.cs @@ -0,0 +1,84 @@ +using System.Text; + +namespace CmdScale.EntityFrameworkCore.TimescaleDB.Internals +{ + /// + /// Parses and formats SQL templates whose column references are {n} placeholders, as used by + /// . Braces inside string literals and quoted + /// identifiers are left alone; {{ and }} outside them stand for literal braces. + /// + internal static class ColumnTemplate + { + /// Returns the placeholder indices in order of appearance. + /// The template has an unescaped brace that is not a placeholder. + public static List GetPlaceholderIndices(string template) + { + List indices = []; + Walk(template, indices.Add, (_, _) => { }); + return indices; + } + + /// Replaces each placeholder with the text returned for its index. + /// The template has an unescaped brace that is not a placeholder. + public static string Format(string template, Func column) + { + StringBuilder builder = new(template.Length); + Walk(template, index => builder.Append(column(index)), (start, length) => builder.Append(template, start, length)); + return builder.ToString(); + } + + /// + /// Reports each placeholder index and each run of literal text (as start and length). An escaped brace + /// ends the current run after its first character, so the second is skipped. + /// + private static void Walk(string template, Action onPlaceholder, Action onText) + { + int runStart = 0; + int i = 0; + while (i < template.Length) + { + char c = template[i]; + if (c is '\'' or '"') + { + int end = template.IndexOf(c, i + 1); + // A doubled quote is an escaped quote inside the literal or identifier. + while (end >= 0 && end + 1 < template.Length && template[end + 1] == c) + { + end = template.IndexOf(c, end + 2); + } + + i = end < 0 ? template.Length : end + 1; + } + else if (c is '{' or '}' && i + 1 < template.Length && template[i + 1] == c) + { + onText(runStart, i + 1 - runStart); + i += 2; + runStart = i; + } + else if (c == '{') + { + int close = template.IndexOf('}', i + 1); + if (close < 0 || !int.TryParse(template.AsSpan(i + 1, close - i - 1), out int index) || index < 0) + { + throw new FormatException($"Invalid placeholder at position {i} in SQL template '{template}'. Column references are written as {{0}}, {{1}}, …; literal braces as {{{{ and }}}}."); + } + + onText(runStart, i - runStart); + onPlaceholder(index); + i = close + 1; + runStart = i; + } + else if (c == '}') + { + throw new FormatException($"Unmatched '}}' at position {i} in SQL template '{template}'. Literal braces are written as {{{{ and }}}}."); + } + else + { + i++; + } + } + + onText(runStart, template.Length - runStart); + } + } +} diff --git a/src/Eftdb/Internals/ContinuousAggregateExpressionJson.cs b/src/Eftdb/Internals/ContinuousAggregateExpressionJson.cs new file mode 100644 index 0000000..151fe8f --- /dev/null +++ b/src/Eftdb/Internals/ContinuousAggregateExpressionJson.cs @@ -0,0 +1,61 @@ +using CmdScale.EntityFrameworkCore.TimescaleDB.Abstractions; +using Microsoft.EntityFrameworkCore.Infrastructure; +using System.Text.Json; + +namespace CmdScale.EntityFrameworkCore.TimescaleDB.Internals +{ + /// + /// Reads and writes the JSON form of lists stored in the + /// AggregateExpressions and GroupByExpressions annotations. + /// + internal static class ContinuousAggregateExpressionJson + { + public static List Read(IReadOnlyAnnotatable annotatable, string annotation) + => annotatable.FindAnnotation(annotation)?.Value is string { Length: > 0 } json + ? JsonSerializer.Deserialize>(json) ?? [] + : []; + + public static string Write(IReadOnlyList expressions) + => JsonSerializer.Serialize(expressions); + + /// + /// Checks that every placeholder in has a column and every column is + /// referenced. + /// + /// The template and columns do not match. + public static void Validate(string alias, string sql, IReadOnlyList columns) + { + if (string.IsNullOrWhiteSpace(alias)) + { + throw new ArgumentException("The expression alias must not be empty.", nameof(alias)); + } + + if (string.IsNullOrWhiteSpace(sql)) + { + throw new ArgumentException($"The SQL expression for '{alias}' must not be empty.", nameof(sql)); + } + + List indices; + try + { + indices = ColumnTemplate.GetPlaceholderIndices(sql); + } + catch (FormatException ex) + { + throw new ArgumentException(ex.Message, nameof(sql), ex); + } + + int missing = indices.FirstOrDefault(i => i >= columns.Count, -1); + if (missing >= 0) + { + throw new ArgumentException($"The SQL expression for '{alias}' references {{{missing}}}, but only {columns.Count} column(s) were given.", nameof(sql)); + } + + int unused = Enumerable.Range(0, columns.Count).FirstOrDefault(i => !indices.Contains(i), -1); + if (unused >= 0) + { + throw new ArgumentException($"The SQL expression for '{alias}' never references column {{{unused}}} ('{columns[unused]}').", nameof(columns)); + } + } + } +} diff --git a/src/Eftdb/Internals/Features/ContinuousAggregates/ContinuousAggregateDiffer.cs b/src/Eftdb/Internals/Features/ContinuousAggregates/ContinuousAggregateDiffer.cs index 77ad0aa..c77b14e 100644 --- a/src/Eftdb/Internals/Features/ContinuousAggregates/ContinuousAggregateDiffer.cs +++ b/src/Eftdb/Internals/Features/ContinuousAggregates/ContinuousAggregateDiffer.cs @@ -111,8 +111,10 @@ private static void FindStructuralChanges( x.Target.TimeBucketGroupBy != x.Source.TimeBucketGroupBy || x.Target.WithNoData != x.Source.WithNoData || (x.Target.CreateGroupIndexes ?? true) != (x.Source.CreateGroupIndexes ?? true) || - !AreAggregateFunctionsEqual(x.Target.AggregateFunctions, x.Source.AggregateFunctions) || - !AreGroupByColumnsEqual(x.Target.GroupByColumns, x.Source.GroupByColumns) || + !AreSequencesEqual(x.Target.AggregateFunctions, x.Source.AggregateFunctions) || + !AreSetsEqual(x.Target.GroupByColumns, x.Source.GroupByColumns) || + !AreSequencesEqual(x.Target.AggregateExpressions, x.Source.AggregateExpressions) || + !AreSetsEqual(x.Target.GroupByExpressions, x.Source.GroupByExpressions) || x.Target.WhereClause != x.Source.WhereClause || x.Target.ViewDefinition != x.Source.ViewDefinition ); @@ -195,7 +197,7 @@ private static Dictionary TopologicalIndexByViewName(List? list1, IReadOnlyList? list2) + private static bool AreSequencesEqual(IReadOnlyList? list1, IReadOnlyList? list2) { if (list1 == null && list2 == null) return true; if (list1 == null || list2 == null) return false; @@ -204,7 +206,7 @@ private static bool AreAggregateFunctionsEqual(IReadOnlyList? list1, IRe return list1.SequenceEqual(list2); } - private static bool AreGroupByColumnsEqual(IReadOnlyList? list1, IReadOnlyList? list2) + private static bool AreSetsEqual(IReadOnlyList? list1, IReadOnlyList? list2) { if (list1 == null && list2 == null) return true; if (list1 == null || list2 == null) return false; diff --git a/src/Eftdb/Internals/Features/ContinuousAggregates/ContinuousAggregateModelExtractor.cs b/src/Eftdb/Internals/Features/ContinuousAggregates/ContinuousAggregateModelExtractor.cs index 4237242..37076fc 100644 --- a/src/Eftdb/Internals/Features/ContinuousAggregates/ContinuousAggregateModelExtractor.cs +++ b/src/Eftdb/Internals/Features/ContinuousAggregates/ContinuousAggregateModelExtractor.cs @@ -1,3 +1,4 @@ +using CmdScale.EntityFrameworkCore.TimescaleDB.Abstractions; using CmdScale.EntityFrameworkCore.TimescaleDB.Configuration.ContinuousAggregate; using CmdScale.EntityFrameworkCore.TimescaleDB.Configuration.Hypertable; using CmdScale.EntityFrameworkCore.TimescaleDB.Operations; @@ -103,6 +104,10 @@ private static IEnumerable ExtractContinuous List aggregateFunctions = ResolveAggregateFunctions(entityType, parentEntityType, parentStoreIdentifier, aggregateStoreIdentifier); List groupByColumns = ResolveGroupByColumns(entityType, parentEntityType, parentStoreIdentifier); + List aggregateExpressions = ResolveExpressions( + ContinuousAggregateAnnotations.AggregateExpressions, entityType, parentEntityType, parentStoreIdentifier, aggregateStoreIdentifier); + List groupByExpressions = ResolveExpressions( + ContinuousAggregateAnnotations.GroupByExpressions, entityType, parentEntityType, parentStoreIdentifier, aggregateStoreIdentifier); // Schema resolution: prefer the CA's own view schema (set by .ToView(...) // or by the scaffolder), fall back to the parent's schema, finally default. @@ -132,6 +137,8 @@ private static IEnumerable ExtractContinuous AggregateFunctions = aggregateFunctions, GroupByColumns = groupByColumns, WhereClause = whereClause, + AggregateExpressions = aggregateExpressions, + GroupByExpressions = groupByExpressions, ViewDefinition = useRawDefinition ? viewDefinition : null, EnableCompression = enableCompression || compressionSegmentBy?.Count > 0 || compressionOrderBy?.Count > 0, CompressionSegmentBy = compressionSegmentBy, @@ -155,37 +162,72 @@ private static List ResolveAggregateFunctions( foreach (string aggInfo in modelAggregateFunctions) { - string[] parts = aggInfo.Split(':'); - if (parts.Length != 3) + string? resolved = ResolveAggregateFunctionEntry(aggInfo, entityType, parentEntityType, parentStoreIdentifier, aggregateStoreIdentifier, out _); + if (resolved != null) { - continue; + aggregateFunctions.Add(resolved); } + } - string aliasModelName = parts[0]; - string functionEnumString = parts[1]; - string sourceColumnModelName = parts[2]; + return aggregateFunctions; + } - // Resolve source column name from parent entity. "*" is not a column - // but the COUNT(*) wildcard, so it bypasses resolution. - string? sourceColumnDbName = sourceColumnModelName == "*" - ? "*" - : ColumnNameResolver.Resolve(parentEntityType, sourceColumnModelName, parentStoreIdentifier); - if (string.IsNullOrWhiteSpace(sourceColumnDbName)) - { - continue; - } + /// + /// Resolves an alias:Function:sourceColumn entry to database column names, or returns + /// null with the reason the entry is left out of the view. The model validator warns with the + /// same reason. + /// + internal static string? ResolveAggregateFunctionEntry( + string entry, + IEntityType entityType, + IEntityType parentEntityType, + StoreObjectIdentifier parentStoreIdentifier, + StoreObjectIdentifier aggregateStoreIdentifier, + out string? ignoredReason) + { + string[] parts = entry.Split(':'); + if (parts.Length != 3) + { + ignoredReason = "it is not in 'alias:Function:sourceColumn' form."; + return null; + } - // Resolve alias column name from aggregate entity to respect naming conventions - string? aliasDbName = ColumnNameResolver.Resolve(entityType, aliasModelName, aggregateStoreIdentifier); - if (string.IsNullOrWhiteSpace(aliasDbName)) - { - aliasDbName = aliasModelName; - } + // "*" is not a column but the COUNT(*) wildcard, so it bypasses resolution. + string? sourceColumnDbName = parts[2] == "*" + ? "*" + : ColumnNameResolver.Resolve(parentEntityType, parts[2], parentStoreIdentifier); + if (string.IsNullOrWhiteSpace(sourceColumnDbName)) + { + ignoredReason = $"'{parts[2]}' is not a property or column of '{EntityStoreObjectResolver.DisplayName(parentEntityType)}'."; + return null; + } - aggregateFunctions.Add($"{aliasDbName}:{functionEnumString}:{sourceColumnDbName}"); + string aliasDbName = ColumnNameResolver.Resolve(entityType, parts[0], aggregateStoreIdentifier) ?? parts[0]; + ignoredReason = null; + return $"{aliasDbName}:{parts[1]}:{sourceColumnDbName}"; + } + + /// + /// Resolves expression aliases against the aggregate view and placeholder columns against the + /// parent, so naming conventions apply. Columns that do not resolve are rejected at model + /// finalization by . + /// + private static List ResolveExpressions( + string annotation, + IEntityType entityType, + IEntityType parentEntityType, + StoreObjectIdentifier parentStoreIdentifier, + StoreObjectIdentifier aggregateStoreIdentifier) + { + List resolved = []; + foreach (ContinuousAggregateExpression expression in ContinuousAggregateExpressionJson.Read(entityType, annotation)) + { + string alias = ColumnNameResolver.Resolve(entityType, expression.Alias, aggregateStoreIdentifier) ?? expression.Alias; + List columns = [.. expression.Columns.Select(c => ColumnNameResolver.Resolve(parentEntityType, c, parentStoreIdentifier) ?? c)]; + resolved.Add(new ContinuousAggregateExpression(alias, expression.Sql, columns)); } - return aggregateFunctions; + return resolved; } /// diff --git a/src/Eftdb/Internals/TimescaleModelValidator.cs b/src/Eftdb/Internals/TimescaleModelValidator.cs index 4b39ce8..7abbb50 100644 --- a/src/Eftdb/Internals/TimescaleModelValidator.cs +++ b/src/Eftdb/Internals/TimescaleModelValidator.cs @@ -1,5 +1,6 @@ using CmdScale.EntityFrameworkCore.TimescaleDB.Configuration.ContinuousAggregate; using CmdScale.EntityFrameworkCore.TimescaleDB.Diagnostics; +using CmdScale.EntityFrameworkCore.TimescaleDB.Internals.Features.ContinuousAggregates; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Diagnostics; using Microsoft.EntityFrameworkCore.Infrastructure; @@ -26,6 +27,40 @@ public override void Validate(IModel model, IDiagnosticsLogger + /// Warns for each entry the model extractor + /// leaves out of the view, using the extractor's own resolution. A warning rather than an exception keeps + /// models that already migrate working. + /// + private static void WarnWhenAggregateEntryIgnored(IModel model, IEntityType entityType, IDiagnosticsLogger logger) + { + string? materializedViewName = entityType.FindAnnotation(ContinuousAggregateAnnotations.MaterializedViewName)?.Value as string; + if (string.IsNullOrWhiteSpace(materializedViewName) + || !string.IsNullOrWhiteSpace(entityType.FindAnnotation(ContinuousAggregateAnnotations.ViewDefinition)?.Value as string) + || entityType.FindAnnotation(ContinuousAggregateAnnotations.AggregateFunctions)?.Value is not List entries) + { + return; + } + + IEntityType? parent = ParentEntityTypeResolver.Resolve(model, entityType.FindAnnotation(ContinuousAggregateAnnotations.ParentName)?.Value as string); + StoreObjectIdentifier? parentStoreIdentifier = parent == null ? null : EntityStoreObjectResolver.GetStoreObjectIdentifier(parent); + StoreObjectIdentifier? aggregateStoreIdentifier = EntityStoreObjectResolver.GetStoreObjectIdentifier(entityType); + if (parent == null || parentStoreIdentifier == null || aggregateStoreIdentifier == null) + { + return; + } + + foreach (string entry in entries) + { + if (ContinuousAggregateModelExtractor.ResolveAggregateFunctionEntry( + entry, entityType, parent, parentStoreIdentifier.Value, aggregateStoreIdentifier.Value, out string? reason) == null) + { + logger.ContinuousAggregateEntryIgnored(EntityStoreObjectResolver.DisplayName(entityType), materializedViewName, entry, reason!); + } } } diff --git a/src/Eftdb/MigrationExtensions/ContinuousAggregateMigrationExtensions.cs b/src/Eftdb/MigrationExtensions/ContinuousAggregateMigrationExtensions.cs index d97c48c..4dd256b 100644 --- a/src/Eftdb/MigrationExtensions/ContinuousAggregateMigrationExtensions.cs +++ b/src/Eftdb/MigrationExtensions/ContinuousAggregateMigrationExtensions.cs @@ -25,7 +25,9 @@ public static OperationBuilder CreateContinu string? viewDefinition = null, bool enableCompression = false, IReadOnlyList? compressionSegmentBy = null, - IReadOnlyList? compressionOrderBy = null) + IReadOnlyList? compressionOrderBy = null, + IReadOnlyList? aggregateExpressions = null, + IReadOnlyList? groupByExpressions = null) { CreateContinuousAggregateOperation operation = new() { @@ -47,6 +49,8 @@ public static OperationBuilder CreateContinu EnableCompression = enableCompression, CompressionSegmentBy = compressionSegmentBy, CompressionOrderBy = compressionOrderBy, + AggregateExpressions = aggregateExpressions ?? [], + GroupByExpressions = groupByExpressions ?? [], }; migrationBuilder.Operations.Add(operation); diff --git a/src/Eftdb/Operations/CreateContinuousAggregateOperation.cs b/src/Eftdb/Operations/CreateContinuousAggregateOperation.cs index a1a9476..144bcdf 100644 --- a/src/Eftdb/Operations/CreateContinuousAggregateOperation.cs +++ b/src/Eftdb/Operations/CreateContinuousAggregateOperation.cs @@ -1,4 +1,5 @@ -using Microsoft.EntityFrameworkCore.Migrations.Operations; +using CmdScale.EntityFrameworkCore.TimescaleDB.Abstractions; +using Microsoft.EntityFrameworkCore.Migrations.Operations; namespace CmdScale.EntityFrameworkCore.TimescaleDB.Operations { @@ -24,6 +25,16 @@ public class CreateContinuousAggregateOperation : MigrationOperation public IReadOnlyList GroupByColumns { get; set; } = []; public string? WhereClause { get; set; } + /// + /// Aggregate columns computed by SQL expressions, with placeholders bound to database column names. + /// + public IReadOnlyList AggregateExpressions { get; set; } = []; + + /// + /// GROUP BY expressions selected as aliased columns, with placeholders bound to database column names. + /// + public IReadOnlyList GroupByExpressions { get; set; } = []; + /// /// Raw SQL body for the materialized view. When non-null the generator uses this /// verbatim (CREATE MATERIALIZED VIEW ... AS {ViewDefinition}) and ignores the diff --git a/tests/Eftdb.Tests/Design/Features/ContinuousAggregate/ContinuousAggregateAnnotationRendererTests.cs b/tests/Eftdb.Tests/Design/Features/ContinuousAggregate/ContinuousAggregateAnnotationRendererTests.cs index 2128a62..7c4bc74 100644 --- a/tests/Eftdb.Tests/Design/Features/ContinuousAggregate/ContinuousAggregateAnnotationRendererTests.cs +++ b/tests/Eftdb.Tests/Design/Features/ContinuousAggregate/ContinuousAggregateAnnotationRendererTests.cs @@ -330,10 +330,10 @@ public void GenerateFluentApiCalls_ConsumesAllCaAnnotations() #endregion - #region GenerateFluentApiCalls_LeavesAnnotationsUnconsumed_WhenViewDefUnparseable + #region GenerateFluentApiCalls_RendersFromSql_WhenViewDefUnparseable [Fact] - public void GenerateFluentApiCalls_LeavesAnnotationsUnconsumed_WhenViewDefUnparseable() + public void GenerateFluentApiCalls_RendersFromSql_WhenViewDefUnparseable() { // Arrange using CaTestContext context = new(); @@ -348,10 +348,42 @@ public void GenerateFluentApiCalls_LeavesAnnotationsUnconsumed_WhenViewDefUnpars .GenerateFluentApiCalls(entityType, annotations); // Assert - Assert.DoesNotContain(result, f => CollectMethodChain(f).Contains("IsContinuousAggregate")); - Assert.Contains(ContinuousAggregateAnnotations.MaterializedViewName, annotations.Keys); - Assert.Contains(ContinuousAggregateAnnotations.ParentName, annotations.Keys); - Assert.Contains(ContinuousAggregateAnnotations.ViewDefinition, annotations.Keys); + MethodCallCodeFragment call = Assert.Single(result, f => f.Method == nameof(ContinuousAggregateTypeBuilder.IsContinuousAggregateFromSql)); + Assert.Equal("UNPARSEABLE SQL", call.Arguments[2]); + Assert.DoesNotContain(annotations.Keys, k => k.StartsWith("TimescaleDB:", StringComparison.Ordinal)); + } + + #endregion + + #region GenerateFluentApiCalls_RendersFromSql_WhenViewHasToolkitAggregate + + [Fact] + public void GenerateFluentApiCalls_RendersFromSql_WhenViewHasToolkitAggregate() + { + // Arrange + const string viewDefinition = """ + SELECT time_bucket('1 day'::interval, "OccurredAt") AS bucket, + (imp IS NOT NULL) AS impersonated, + count(*) AS n, + mcv_agg(5, actor_id) AS actors + FROM ev + GROUP BY (time_bucket('1 day'::interval, "OccurredAt")), (imp IS NOT NULL); + """; + using CaTestContext context = new(); + IEntityType entityType = GetEntityType(context); + Dictionary annotations = Annotations( + (ContinuousAggregateAnnotations.MaterializedViewName, "ranks"), + (ContinuousAggregateAnnotations.ParentName, "ev"), + (ContinuousAggregateAnnotations.ViewDefinition, viewDefinition)); + + // Act + IReadOnlyList result = CreateAnnotationCodeGenerator() + .GenerateFluentApiCalls(entityType, annotations); + + // Assert + MethodCallCodeFragment call = Assert.Single(result, f => f.Method == nameof(ContinuousAggregateTypeBuilder.IsContinuousAggregateFromSql)); + Assert.Equal(viewDefinition.Trim(), call.Arguments[2]); + Assert.DoesNotContain("AddAggregateFunction", CollectMethodChain(call)); } #endregion diff --git a/tests/Eftdb.Tests/Design/Features/ContinuousAggregate/ContinuousAggregateCSharpGeneratorTests.cs b/tests/Eftdb.Tests/Design/Features/ContinuousAggregate/ContinuousAggregateCSharpGeneratorTests.cs index 2c541ef..c1f41c3 100644 --- a/tests/Eftdb.Tests/Design/Features/ContinuousAggregate/ContinuousAggregateCSharpGeneratorTests.cs +++ b/tests/Eftdb.Tests/Design/Features/ContinuousAggregate/ContinuousAggregateCSharpGeneratorTests.cs @@ -378,5 +378,35 @@ public void CreateContinuousAggregate_NullGroupByColumns_Omits_GroupByColumnsArg } #endregion + + #region CreateContinuousAggregate_Expressions_FullyQualifiedTypedEntries + + [Fact] + public void CreateContinuousAggregate_Expressions_FullyQualifiedTypedEntries() + { + // Arrange + CreateContinuousAggregateOperation op = new() + { + MaterializedViewName = "ranks", + ParentName = "events", + AggregateExpressions = [new("actors", "mcv_agg(500, {0})", ["actor_id"])], + GroupByExpressions = [new("impersonated", "{0} IS NOT NULL", ["impersonated_actor_id"])], + }; + + // Act + string result = Generate(op); + + // Assert + Assert.Contains( + "new CmdScale.EntityFrameworkCore.TimescaleDB.Abstractions.ContinuousAggregateExpression(\"actors\", \"mcv_agg(500, {0})\", [\"actor_id\"])", + result); + Assert.Contains("aggregateExpressions:", result); + Assert.Contains("groupByExpressions:", result); + Assert.Contains( + "new CmdScale.EntityFrameworkCore.TimescaleDB.Abstractions.ContinuousAggregateExpression(\"impersonated\", \"{0} IS NOT NULL\", [\"impersonated_actor_id\"])", + result); + } + + #endregion } } diff --git a/tests/Eftdb.Tests/Design/Generators/ScaffoldedModelRenderingTests.cs b/tests/Eftdb.Tests/Design/Generators/ScaffoldedModelRenderingTests.cs index f360390..16cc280 100644 --- a/tests/Eftdb.Tests/Design/Generators/ScaffoldedModelRenderingTests.cs +++ b/tests/Eftdb.Tests/Design/Generators/ScaffoldedModelRenderingTests.cs @@ -447,7 +447,6 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) e.Property(x => x.AvgValue).HasColumnName("avg_value"); e.HasAnnotation(ContinuousAggregateAnnotations.MaterializedViewName, "render_policy_ca_view_e"); e.HasAnnotation(ContinuousAggregateAnnotations.ParentName, "render_policy_source"); - e.HasAnnotation(ContinuousAggregateAnnotations.ViewDefinition, "UNPARSEABLE SQL"); e.HasAnnotation(ContinuousAggregatePolicyAnnotations.HasRefreshPolicy, true); e.HasAnnotation(ContinuousAggregatePolicyAnnotations.StartOffset, "7 days"); }); @@ -1009,7 +1008,6 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) e.Property(x => x.AvgValue).HasColumnName("avg_value"); e.HasAnnotation(ContinuousAggregateAnnotations.MaterializedViewName, "rp_guard_ca_view"); e.HasAnnotation(ContinuousAggregateAnnotations.ParentName, "rp_guard_ca_source"); - e.HasAnnotation(ContinuousAggregateAnnotations.ViewDefinition, "UNPARSEABLE SQL"); e.HasAnnotation(RetentionPolicyAnnotations.HasRetentionPolicy, true); e.HasAnnotation(RetentionPolicyAnnotations.DropAfter, "7 days"); }); diff --git a/tests/Eftdb.Tests/Design/Scaffolding/ViewDefinitionParserTests.cs b/tests/Eftdb.Tests/Design/Scaffolding/ViewDefinitionParserTests.cs index da2b65e..b859d76 100644 --- a/tests/Eftdb.Tests/Design/Scaffolding/ViewDefinitionParserTests.cs +++ b/tests/Eftdb.Tests/Design/Scaffolding/ViewDefinitionParserTests.cs @@ -1323,4 +1323,43 @@ public void ParseTimeBucketAlias_Parse_Populates_TimeBucketAlias() } #endregion + + #region Parse_IsComplete_For_Structured_View + + [Fact] + public void Parse_IsComplete_For_Structured_View() + { + // Act + ViewDefinitionParser.ParsedViewDefinition parsed = ViewDefinitionParser.Parse(FullViewDef); + + // Assert + Assert.True(parsed.IsComplete); + } + + #endregion + + #region Parse_IsNotComplete_When_View_Has_Expressions + + [Theory] + [InlineData("mcv_agg(5, actor_id) AS actors")] + [InlineData("(imp IS NOT NULL) AS impersonated")] + [InlineData("count(*) FILTER (WHERE action = 1) AS reads")] + public void Parse_IsNotComplete_When_View_Has_Expressions(string item) + { + // Arrange + string viewDefinition = + " SELECT time_bucket('1 day'::interval, \"OccurredAt\") AS bucket,\n" + + $" {item},\n" + + " count(*) AS n\n" + + " FROM ev\n" + + " GROUP BY (time_bucket('1 day'::interval, \"OccurredAt\")), (imp IS NOT NULL);"; + + // Act + ViewDefinitionParser.ParsedViewDefinition parsed = ViewDefinitionParser.Parse(viewDefinition); + + // Assert + Assert.False(parsed.IsComplete); + } + + #endregion } diff --git a/tests/Eftdb.Tests/Diagnostics/ContinuousAggregateEntryIgnoredWarningTests.cs b/tests/Eftdb.Tests/Diagnostics/ContinuousAggregateEntryIgnoredWarningTests.cs new file mode 100644 index 0000000..f89a827 --- /dev/null +++ b/tests/Eftdb.Tests/Diagnostics/ContinuousAggregateEntryIgnoredWarningTests.cs @@ -0,0 +1,106 @@ +using CmdScale.EntityFrameworkCore.TimescaleDB.Configuration.ContinuousAggregate; +using CmdScale.EntityFrameworkCore.TimescaleDB.Configuration.Hypertable; +using CmdScale.EntityFrameworkCore.TimescaleDB.Diagnostics; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Diagnostics; +using Microsoft.EntityFrameworkCore.Infrastructure; + +namespace CmdScale.EntityFrameworkCore.TimescaleDB.Tests.Diagnostics; + +/// +/// Tests the warning raised for continuous aggregate entries the model extractor leaves out of the view. +/// +public class ContinuousAggregateEntryIgnoredWarningTests +{ + private class IgnoredEvent + { + public DateTime OccurredAt { get; set; } + public double Value { get; set; } + } + + private class IgnoredAggregate + { + public DateTime Bucket { get; set; } + public double Total { get; set; } + } + + private class IgnoredContext(string entry, Action? warnings = null) : DbContext + { + public List Log { get; } = []; + + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + => optionsBuilder.UseNpgsql("Host=localhost;Database=dummy;Username=x;Password=x") + .LogTo(Log.Add) + .ConfigureWarnings(w => warnings?.Invoke(w)) + .EnableServiceProviderCaching(false) + .UseTimescaleDb(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity(e => + { + e.HasNoKey(); + e.ToTable("ignored_events"); + e.IsHypertable(x => x.OccurredAt); + }); + modelBuilder.Entity(e => + { + e.HasNoKey(); + e.Property(x => x.Bucket).HasColumnName("time_bucket"); + e.IsContinuousAggregate("ignored_hourly", nameof(IgnoredEvent), "1 hour", nameof(IgnoredEvent.OccurredAt)); + e.HasAnnotation(ContinuousAggregateAnnotations.AggregateFunctions, new List { entry }); + }); + } + } + + #region Should_Warn_For_Ignored_Entry + + [Theory] + [InlineData("Total:Sum:NoSuchColumn", "NoSuchColumn")] + [InlineData("Total:Sum", "alias:Function:sourceColumn")] + public void Should_Warn_For_Ignored_Entry(string entry, string expectedFragment) + { + // Arrange + using IgnoredContext context = new(entry); + + // Act + _ = context.Model; + + // Assert + string warning = Assert.Single(context.Log, l => l.Contains(TimescaleDbEventId.ContinuousAggregateEntryIgnored.Name!)); + Assert.Contains(expectedFragment, warning); + } + + #endregion + + #region Should_Not_Warn_For_Valid_Entry + + [Fact] + public void Should_Not_Warn_For_Valid_Entry() + { + // Arrange + using IgnoredContext context = new("Total:Sum:Value"); + + // Act + _ = context.Model; + + // Assert + Assert.DoesNotContain(context.Log, l => l.Contains(TimescaleDbEventId.ContinuousAggregateEntryIgnored.Name!)); + } + + #endregion + + #region Should_Throw_When_Ignored_Entry_Warning_Is_An_Error + + [Fact] + public void Should_Throw_When_Ignored_Entry_Warning_Is_An_Error() + { + // Arrange + using IgnoredContext context = new("Total:Sum:NoSuchColumn", w => w.Throw(TimescaleDbEventId.ContinuousAggregateEntryIgnored)); + + // Act & Assert + Assert.Throws(() => context.Model); + } + + #endregion +} diff --git a/tests/Eftdb.Tests/Diagnostics/TimescaleDbWarningEmissionTests.cs b/tests/Eftdb.Tests/Diagnostics/TimescaleDbWarningEmissionTests.cs index 1d023d4..a33c51e 100644 --- a/tests/Eftdb.Tests/Diagnostics/TimescaleDbWarningEmissionTests.cs +++ b/tests/Eftdb.Tests/Diagnostics/TimescaleDbWarningEmissionTests.cs @@ -267,6 +267,7 @@ public void Should_Keep_Published_EventId_Values_Stable() Assert.Equal(63000, TimescaleDbEventId.CommunityFeatureSkipped.Id); Assert.Equal(63001, TimescaleDbEventId.TimeBucketColumnUnmapped.Id); Assert.Equal(63002, TimescaleDbEventId.UnsupportedHypertableChangeSkipped.Id); + Assert.Equal(63003, TimescaleDbEventId.ContinuousAggregateEntryIgnored.Id); } #endregion diff --git a/tests/Eftdb.Tests/Differs/ContinuousAggregateExpressionDifferTests.cs b/tests/Eftdb.Tests/Differs/ContinuousAggregateExpressionDifferTests.cs new file mode 100644 index 0000000..65260ac --- /dev/null +++ b/tests/Eftdb.Tests/Differs/ContinuousAggregateExpressionDifferTests.cs @@ -0,0 +1,116 @@ +using CmdScale.EntityFrameworkCore.TimescaleDB.Abstractions; +using CmdScale.EntityFrameworkCore.TimescaleDB.Configuration.ContinuousAggregate; +using CmdScale.EntityFrameworkCore.TimescaleDB.Configuration.Hypertable; +using CmdScale.EntityFrameworkCore.TimescaleDB.Internals.Features.ContinuousAggregates; +using CmdScale.EntityFrameworkCore.TimescaleDB.Operations; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations.Operations; + +namespace CmdScale.EntityFrameworkCore.TimescaleDB.Tests.Differs; + +/// +/// Tests that changes to aggregate and group-by expressions recreate the continuous aggregate. +/// +public class ContinuousAggregateExpressionDifferTests +{ + private static IRelationalModel GetModel(DbContext context) + => context.GetService().Model.GetRelationalModel(); + + private class RankEvent + { + public DateTime OccurredAt { get; set; } + public string ActorId { get; set; } = ""; + public string? ImpersonatedActorId { get; set; } + } + + private class Rank + { + public DateTime Bucket { get; set; } + public bool Impersonated { get; set; } + public string Actors { get; set; } = ""; + } + + private class RankContext(string aggregateSql, string groupBySql) : DbContext + { + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + => optionsBuilder.UseNpgsql("Host=localhost;Database=test;Username=test;Password=test") + .EnableServiceProviderCaching(false) + .UseTimescaleDb(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity(e => + { + e.HasNoKey(); + e.ToTable("expression_diff_events"); + e.IsHypertable(x => x.OccurredAt); + }); + modelBuilder.Entity(e => + { + e.HasNoKey(); + e.IsContinuousAggregate("expression_diff_ranks", "1 day", x => x.OccurredAt) + .AddGroupByExpression(x => x.Impersonated, groupBySql, x => x.ImpersonatedActorId) + .AddAggregateExpression(x => x.Actors, aggregateSql, x => x.ActorId); + }); + } + } + + #region Should_Not_Diff_Unchanged_Expressions + + [Fact] + public void Should_Not_Diff_Unchanged_Expressions() + { + // Arrange + using RankContext source = new("mcv_agg(500, {0})", "{0} IS NOT NULL"); + using RankContext target = new("mcv_agg(500, {0})", "{0} IS NOT NULL"); + + // Act + IReadOnlyList operations = new ContinuousAggregateDiffer().GetDifferences(GetModel(source), GetModel(target)); + + // Assert + Assert.Empty(operations); + } + + #endregion + + #region Should_Recreate_When_Aggregate_Expression_Changes + + [Fact] + public void Should_Recreate_When_Aggregate_Expression_Changes() + { + // Arrange + using RankContext source = new("mcv_agg(500, {0})", "{0} IS NOT NULL"); + using RankContext target = new("mcv_agg(50, 2.0, {0})", "{0} IS NOT NULL"); + + // Act + IReadOnlyList operations = new ContinuousAggregateDiffer().GetDifferences(GetModel(source), GetModel(target)); + + // Assert + Assert.Single(operations.OfType()); + CreateContinuousAggregateOperation create = Assert.Single(operations.OfType()); + Assert.Equal("mcv_agg(50, 2.0, {0})", Assert.Single(create.AggregateExpressions).Sql); + } + + #endregion + + #region Should_Recreate_When_GroupBy_Expression_Changes + + [Fact] + public void Should_Recreate_When_GroupBy_Expression_Changes() + { + // Arrange + using RankContext source = new("mcv_agg(500, {0})", "{0} IS NOT NULL"); + using RankContext target = new("mcv_agg(500, {0})", "{0} IS NULL"); + + // Act + IReadOnlyList operations = new ContinuousAggregateDiffer().GetDifferences(GetModel(source), GetModel(target)); + + // Assert + Assert.Single(operations.OfType()); + Assert.Single(operations.OfType()); + } + + #endregion +} diff --git a/tests/Eftdb.Tests/Extractors/ContinuousAggregateExpressionExtractorTests.cs b/tests/Eftdb.Tests/Extractors/ContinuousAggregateExpressionExtractorTests.cs new file mode 100644 index 0000000..4ae289f --- /dev/null +++ b/tests/Eftdb.Tests/Extractors/ContinuousAggregateExpressionExtractorTests.cs @@ -0,0 +1,320 @@ +using CmdScale.EntityFrameworkCore.TimescaleDB.Abstractions; +using CmdScale.EntityFrameworkCore.TimescaleDB.Configuration.ContinuousAggregate; +using CmdScale.EntityFrameworkCore.TimescaleDB.Configuration.Hypertable; +using CmdScale.EntityFrameworkCore.TimescaleDB.Generators; +using CmdScale.EntityFrameworkCore.TimescaleDB.Internals.Features.ContinuousAggregates; +using CmdScale.EntityFrameworkCore.TimescaleDB.Operations; +using CmdScale.EntityFrameworkCore.TimescaleDB.Tests.Utils; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; + +namespace CmdScale.EntityFrameworkCore.TimescaleDB.Tests.Extractors; + +/// +/// Tests that aggregate and group-by expressions resolve against the model and produce the view SQL. +/// +public class ContinuousAggregateExpressionExtractorTests +{ + private const string ConnectionString = "Host=localhost;Database=test;Username=test;Password=test"; + + private static CreateContinuousAggregateOperation ExtractSingle(DbContext context) + => Assert.Single(ContinuousAggregateModelExtractor.GetContinuousAggregates( + context.GetService().Model.GetRelationalModel())); + + #region Should_Resolve_Expression_Columns_With_Naming_Conventions + + private class SnakeEvent + { + public DateTime OccurredAt { get; set; } + public string TenantId { get; set; } = ""; + public string ActorId { get; set; } = ""; + public string? ImpersonatedActorId { get; set; } + } + + private class SnakeRank + { + public DateTime Day { get; set; } + public string TenantId { get; set; } = ""; + public bool Impersonated { get; set; } + public long EventCount { get; set; } + public string Actors { get; set; } = ""; + } + + private class SnakeContext : DbContext + { + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + => optionsBuilder.UseNpgsql(ConnectionString).UseSnakeCaseNamingConvention().UseTimescaleDb(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity(e => + { + e.HasNoKey(); + e.ToTable("audit_events"); + e.IsHypertable(x => x.OccurredAt); + }); + modelBuilder.Entity(e => + { + e.HasNoKey(); + e.IsContinuousAggregate("audit_actor_ranks_daily", "1 day", x => x.OccurredAt) + .WithTimeBucketProperty(x => x.Day) + .MaterializedOnly(false) + .AddGroupByColumn(x => x.TenantId) + .AddGroupByExpression(x => x.Impersonated, "{0} IS NOT NULL", x => x.ImpersonatedActorId) + .AddAggregateFunction(x => x.EventCount, x => x.ActorId, EAggregateFunction.Count) + .AddAggregateExpression(x => x.Actors, "mcv_agg(500, {0})", x => x.ActorId); + }); + } + } + + [Fact] + public void Should_Resolve_Expression_Columns_With_Naming_Conventions() + { + // Arrange + using SnakeContext context = new(); + + // Act + CreateContinuousAggregateOperation operation = ExtractSingle(context); + + // Assert + Assert.Equal(new ContinuousAggregateExpression("actors", "mcv_agg(500, {0})", ["actor_id"]), Assert.Single(operation.AggregateExpressions)); + Assert.Equal(new ContinuousAggregateExpression("impersonated", "{0} IS NOT NULL", ["impersonated_actor_id"]), Assert.Single(operation.GroupByExpressions)); + } + + [Fact] + public void Should_Select_And_Group_By_Expressions() + { + // Arrange + using SnakeContext context = new(); + CreateContinuousAggregateOperation operation = ExtractSingle(context); + + // Act + string sql = Assert.Single(ContinuousAggregateSqlGenerator.Generate(operation)); + + // Assert + Assert.Equal( + SqlHelper.NormalizeSql(""" + CREATE MATERIALIZED VIEW "public"."audit_actor_ranks_daily" + WITH (timescaledb.continuous, timescaledb.materialized_only = false) AS + SELECT time_bucket('1 day', "occurred_at") AS "day", "tenant_id", "impersonated_actor_id" IS NOT NULL AS "impersonated", COUNT("actor_id") AS "event_count", mcv_agg(500, "actor_id") AS "actors" + FROM "public"."audit_events" + GROUP BY 1, "tenant_id", ("impersonated_actor_id" IS NOT NULL); + """), + SqlHelper.NormalizeSql(sql)); + } + + #endregion + + #region Should_Resolve_Rollup_Against_Parent_Aggregate + + private class RollupEvent + { + public DateTime OccurredAt { get; set; } + public string ActorId { get; set; } = ""; + } + + private class RollupDaily + { + public DateTime Day { get; set; } + public string Actors { get; set; } = ""; + } + + private class RollupWeekly + { + public DateTime Week { get; set; } + public string Actors { get; set; } = ""; + } + + private class RollupContext : DbContext + { + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + => optionsBuilder.UseNpgsql(ConnectionString).UseSnakeCaseNamingConvention().UseTimescaleDb(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity(e => + { + e.HasNoKey(); + e.ToTable("rollup_events"); + e.IsHypertable(x => x.OccurredAt); + }); + modelBuilder.Entity(e => + { + e.HasNoKey(); + e.IsContinuousAggregate("rollup_daily", "1 day", x => x.OccurredAt) + .WithTimeBucketProperty(x => x.Day) + .AddAggregateExpression(x => x.Actors, "mcv_agg(500, {0})", x => x.ActorId); + }); + modelBuilder.Entity(e => + { + e.HasNoKey(); + e.IsContinuousAggregate("rollup_weekly", "7 days", x => x.Day) + .WithTimeBucketProperty(x => x.Week) + .AddAggregateExpression(x => x.Actors, "rollup({0})", x => x.Actors); + }); + } + } + + [Fact] + public void Should_Resolve_Rollup_Against_Parent_Aggregate() + { + // Arrange + using RollupContext context = new(); + + // Act + List operations = [.. ContinuousAggregateModelExtractor.GetContinuousAggregates( + context.GetService().Model.GetRelationalModel())]; + + // Assert + CreateContinuousAggregateOperation weekly = Assert.Single(operations, x => x.MaterializedViewName == "rollup_weekly"); + Assert.Equal("rollup_daily", weekly.ParentName); + Assert.Equal(new ContinuousAggregateExpression("actors", "rollup({0})", ["actors"]), Assert.Single(weekly.AggregateExpressions)); + } + + #endregion + + #region Should_Throw_When_Expression_Column_Does_Not_Resolve + + private class MissingEvent + { + public DateTime OccurredAt { get; set; } + } + + private class MissingRank + { + public DateTime Bucket { get; set; } + public string Actors { get; set; } = ""; + } + + private class MissingContext : DbContext + { + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + => optionsBuilder.UseNpgsql(ConnectionString).UseTimescaleDb(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity(e => + { + e.HasNoKey(); + e.IsHypertable(x => x.OccurredAt); + }); + modelBuilder.Entity(e => + { + e.HasNoKey(); + e.IsContinuousAggregate("missing_ranks", nameof(MissingEvent), "1 day", nameof(MissingEvent.OccurredAt)) + .AddAggregateExpression(nameof(MissingRank.Actors), "mcv_agg(500, {0})", "NoSuchColumn"); + }); + } + } + + [Fact] + public void Should_Throw_When_Expression_Column_Does_Not_Resolve() + { + // Arrange + using MissingContext context = new(); + + // Act & Assert + InvalidOperationException exception = Assert.Throws(() => context.GetService().Model); + Assert.Contains("NoSuchColumn", exception.Message); + } + + #endregion + + #region Should_Throw_When_Expressions_Combine_With_Raw_View_Definition + + private class MixedEvent + { + public DateTime OccurredAt { get; set; } + public string ActorId { get; set; } = ""; + } + + private class MixedRank + { + public DateTime Bucket { get; set; } + public string Actors { get; set; } = ""; + } + + private class MixedContext : DbContext + { + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + => optionsBuilder.UseNpgsql(ConnectionString).UseTimescaleDb(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity(e => + { + e.HasNoKey(); + e.IsHypertable(x => x.OccurredAt); + }); + modelBuilder.Entity(e => + { + e.HasNoKey(); + e.IsContinuousAggregateFromSql("mixed_ranks", nameof(MixedEvent), "SELECT 1") + .AddAggregateExpression(nameof(MixedRank.Actors), "mcv_agg(500, {0})", nameof(MixedEvent.ActorId)); + }); + } + } + + [Fact] + public void Should_Throw_When_Expressions_Combine_With_Raw_View_Definition() + { + // Arrange + using MixedContext context = new(); + + // Act & Assert + InvalidOperationException exception = Assert.Throws(() => context.GetService().Model); + Assert.Contains("raw view definition", exception.Message); + } + + #endregion + + #region Should_Throw_When_Expression_Alias_Collides + + private class CollideEvent + { + public DateTime OccurredAt { get; set; } + public string ActorId { get; set; } = ""; + } + + private class CollideRank + { + public DateTime Bucket { get; set; } + public long Actors { get; set; } + } + + private class CollideContext : DbContext + { + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + => optionsBuilder.UseNpgsql(ConnectionString).UseTimescaleDb(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity(e => + { + e.HasNoKey(); + e.IsHypertable(x => x.OccurredAt); + }); + modelBuilder.Entity(e => + { + e.HasNoKey(); + e.IsContinuousAggregate("collide_ranks", "1 day", x => x.OccurredAt) + .AddAggregateFunction(x => x.Actors, x => x.ActorId, EAggregateFunction.Count) + .AddAggregateExpression(x => x.Actors, "count(DISTINCT {0})", x => x.ActorId); + }); + } + } + + [Fact] + public void Should_Throw_When_Expression_Alias_Collides() + { + // Arrange + using CollideContext context = new(); + + // Act & Assert + InvalidOperationException exception = Assert.Throws(() => context.GetService().Model); + Assert.Contains("more than once", exception.Message); + } + + #endregion +} diff --git a/tests/Eftdb.Tests/Integration/ContinuousAggregateExpressionIntegrationTests.cs b/tests/Eftdb.Tests/Integration/ContinuousAggregateExpressionIntegrationTests.cs new file mode 100644 index 0000000..d718a15 --- /dev/null +++ b/tests/Eftdb.Tests/Integration/ContinuousAggregateExpressionIntegrationTests.cs @@ -0,0 +1,236 @@ +using CmdScale.EntityFrameworkCore.TimescaleDB.Configuration.ContinuousAggregate; +using CmdScale.EntityFrameworkCore.TimescaleDB.Configuration.Hypertable; +using CmdScale.EntityFrameworkCore.TimescaleDB.Tests.Utils; +using Microsoft.EntityFrameworkCore; +using Testcontainers.PostgreSql; + +namespace CmdScale.EntityFrameworkCore.TimescaleDB.Tests.Integration +{ + /// + /// Tests continuous aggregates built from aggregate and group-by expressions against a real database. + /// + public class ContinuousAggregateExpressionIntegrationTests : MigrationTestBase, IAsyncLifetime + { + private PostgreSqlContainer? _container; + private string? _connectionString; + + public async ValueTask InitializeAsync() + { + _container = new PostgreSqlBuilder(TimescaleImages.Community) + .WithDatabase("test_db") + .WithUsername("test_user") + .WithPassword("test_password") + .Build(); + + await _container.StartAsync(); + _connectionString = _container.GetConnectionString(); + } + + public async ValueTask DisposeAsync() + { + if (_container != null) + { + await _container.DisposeAsync(); + } + GC.SuppressFinalize(this); + } + + #region Should_Group_And_Aggregate_By_Expressions + + private class ExpressionEvent + { + public DateTime OccurredAt { get; set; } + public string ActorId { get; set; } = string.Empty; + public string? ImpersonatedActorId { get; set; } + public short Action { get; set; } + } + + private class ExpressionDaily + { + public DateTime Day { get; set; } + public bool Impersonated { get; set; } + public long Reads { get; set; } + public bool AnyDeletes { get; set; } + } + + private class ExpressionContext(string connectionString) : DbContext + { + public DbSet Events => Set(); + public DbSet Daily => Set(); + + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + => optionsBuilder.UseNpgsql(connectionString).UseSnakeCaseNamingConvention().UseTimescaleDb(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity(entity => + { + entity.ToTable("expression_events"); + entity.HasNoKey(); + entity.IsHypertable(x => x.OccurredAt); + }); + + modelBuilder.Entity(entity => + { + entity.HasNoKey(); + entity.IsContinuousAggregate("expression_daily", "1 day", x => x.OccurredAt) + .WithTimeBucketProperty(x => x.Day) + .AddGroupByExpression(x => x.Impersonated, "{0} IS NOT NULL", x => x.ImpersonatedActorId) + .AddAggregateExpression(x => x.Reads, "count(*) FILTER (WHERE {0} = 1)", x => x.Action) + .AddAggregateExpression(x => x.AnyDeletes, "bool_or({0} = 4)", x => x.Action); + }); + } + } + + [Fact] + public async Task Should_Group_And_Aggregate_By_Expressions() + { + // Arrange + await using ExpressionContext context = new(_connectionString!); + await CreateDatabaseViaMigrationAsync(context); + DateTime day = new(2025, 1, 6, 10, 0, 0, DateTimeKind.Utc); + await context.Database.ExecuteSqlInterpolatedAsync($""" + INSERT INTO expression_events (occurred_at, actor_id, impersonated_actor_id, action) + VALUES ({day}, {"a"}, NULL, {(short)1}), ({day}, {"a"}, NULL, {(short)1}), ({day}, {"b"}, {"p"}, {(short)4}) + """, TestContext.Current.CancellationToken); + + // Act + await context.Database.ExecuteSqlRawAsync( + "CALL refresh_continuous_aggregate('public.expression_daily', NULL, NULL);", [], TestContext.Current.CancellationToken); + List rows = await context.Daily.OrderBy(x => x.Impersonated).ToListAsync(TestContext.Current.CancellationToken); + + // Assert + Assert.Collection( + rows, + direct => + { + Assert.False(direct.Impersonated); + Assert.Equal(2, direct.Reads); + Assert.False(direct.AnyDeletes); + }, + impersonated => + { + Assert.True(impersonated.Impersonated); + Assert.Equal(0, impersonated.Reads); + Assert.True(impersonated.AnyDeletes); + }); + } + + #endregion + } + + /// + /// Tests continuous aggregates built from timescaledb_toolkit sketches, which need the toolkit image. + /// + [Trait("Category", TimescaleImages.ToolkitTrait)] + public class ContinuousAggregateToolkitExpressionIntegrationTests : MigrationTestBase, IAsyncLifetime + { + private PostgreSqlContainer? _container; + private string? _connectionString; + + public async ValueTask InitializeAsync() + { + _container = new PostgreSqlBuilder(TimescaleImages.Toolkit) + .WithDatabase("test_db") + .WithUsername("test_user") + .WithPassword("test_password") + .Build(); + + await _container.StartAsync(); + _connectionString = _container.GetConnectionString(); + } + + public async ValueTask DisposeAsync() + { + if (_container != null) + { + await _container.DisposeAsync(); + } + GC.SuppressFinalize(this); + } + + #region Should_Rank_With_Sketches_And_Roll_Them_Up + + private class SketchEvent + { + public DateTime OccurredAt { get; set; } + public string ActorId { get; set; } = string.Empty; + } + + private class SketchDaily + { + public DateTime Day { get; set; } + public string Actors { get; set; } = string.Empty; + } + + private class SketchWeekly + { + public DateTime Week { get; set; } + public string Actors { get; set; } = string.Empty; + } + + private class SketchContext(string connectionString) : DbContext + { + public DbSet Events => Set(); + + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + => optionsBuilder.UseNpgsql(connectionString).UseSnakeCaseNamingConvention().UseTimescaleDb(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.HasPostgresExtension("timescaledb_toolkit"); + + modelBuilder.Entity(entity => + { + entity.ToTable("sketch_events"); + entity.HasNoKey(); + entity.IsHypertable(x => x.OccurredAt); + }); + + modelBuilder.Entity(entity => + { + entity.HasNoKey(); + entity.IsContinuousAggregate("sketch_daily", "1 day", x => x.OccurredAt) + .WithTimeBucketProperty(x => x.Day) + .AddAggregateExpression(x => x.Actors, "mcv_agg(5, {0})", x => x.ActorId); + }); + + modelBuilder.Entity(entity => + { + entity.HasNoKey(); + entity.IsContinuousAggregate("sketch_weekly", "7 days", x => x.Day) + .WithTimeBucketProperty(x => x.Week) + .AddAggregateExpression(x => x.Actors, "rollup({0})", x => x.Actors); + }); + } + } + + [Fact] + public async Task Should_Rank_With_Sketches_And_Roll_Them_Up() + { + // Arrange + await using SketchContext context = new(_connectionString!); + await CreateDatabaseViaMigrationAsync(context); + DateTime monday = new(2025, 1, 6, 10, 0, 0, DateTimeKind.Utc); + DateTime tuesday = monday.AddDays(1); + await context.Database.ExecuteSqlInterpolatedAsync($""" + INSERT INTO sketch_events (occurred_at, actor_id) + VALUES ({monday}, {"busy"}), ({monday}, {"busy"}), ({tuesday}, {"busy"}), ({tuesday}, {"quiet"}) + """, TestContext.Current.CancellationToken); + + // Act + await context.Database.ExecuteSqlRawAsync( + "CALL refresh_continuous_aggregate('public.sketch_daily', NULL, NULL);", [], TestContext.Current.CancellationToken); + await context.Database.ExecuteSqlRawAsync( + "CALL refresh_continuous_aggregate('public.sketch_weekly', NULL, NULL);", [], TestContext.Current.CancellationToken); + List busiest = await context.Database + .SqlQueryRaw("SELECT topn(actors, 1) AS \"Value\" FROM sketch_weekly") + .ToListAsync(TestContext.Current.CancellationToken); + + // Assert + Assert.Equal(["busy"], busiest); + } + + #endregion + } +} diff --git a/tests/Eftdb.Tests/Integration/ScaffoldRoundTripTests.cs b/tests/Eftdb.Tests/Integration/ScaffoldRoundTripTests.cs index ac8b200..9ef5ced 100644 --- a/tests/Eftdb.Tests/Integration/ScaffoldRoundTripTests.cs +++ b/tests/Eftdb.Tests/Integration/ScaffoldRoundTripTests.cs @@ -111,6 +111,13 @@ private class CaHourlyView public long CountRows { get; set; } } + private class CaFlagView + { + public DateTime Bucket { get; set; } + public bool Expensive { get; set; } + public bool AnyLargeVolume { get; set; } + } + #endregion private sealed class RoundTripReferenceContext(string connectionString) : DbContext @@ -120,6 +127,7 @@ private sealed class RoundTripReferenceContext(string connectionString) : DbCont public DbSet DropCreatedBeforeMetrics => Set(); public DbSet CaSourceMetrics => Set(); public DbSet CaHourlyViews => Set(); + public DbSet CaFlagViews => Set(); protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) => optionsBuilder.UseNpgsql(connectionString).UseTimescaleDb(); @@ -197,6 +205,16 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) entity.WithRetentionPolicy(dropAfter: "90 days"); }); + + modelBuilder.Entity(entity => + { + entity.HasNoKey(); + entity.Property(x => x.Bucket).HasColumnName("bucket"); + entity.IsContinuousAggregate("rrt_ca_flags", "1 day", x => x.Timestamp) + .WithTimeBucketProperty(x => x.Bucket) + .AddGroupByExpression(x => x.Expensive, "{0} > 100", x => x.Price) + .AddAggregateExpression(x => x.AnyLargeVolume, "bool_or({0} > 1000)", x => x.Volume); + }); } } @@ -596,6 +614,10 @@ public async Task Should_Produce_Identical_Continuous_Aggregate_When_Migrating_F Assert.Equal(matOnlyDb1, matOnlyDb2); Assert.Equal(chunkDb1, chunkDb2); + (string flagsDb1, _, _) = await QueryContinuousAggregateInfoAsync(db1, "rrt_ca_flags"); + (string flagsDb2, _, _) = await QueryContinuousAggregateInfoAsync(db2, "rrt_ca_flags"); + Assert.Equal(flagsDb1, flagsDb2); + (string? startDb1, string? endDb1, string schedDb1) = await QueryCaRefreshPolicyAsync(db1, "rrt_ca_hourly"); (string? startDb2, string? endDb2, string schedDb2) = await QueryCaRefreshPolicyAsync(db2, "rrt_ca_hourly"); Assert.Equal(startDb1, startDb2); diff --git a/tests/Eftdb.Tests/Internals/ColumnTemplateTests.cs b/tests/Eftdb.Tests/Internals/ColumnTemplateTests.cs new file mode 100644 index 0000000..ee683c6 --- /dev/null +++ b/tests/Eftdb.Tests/Internals/ColumnTemplateTests.cs @@ -0,0 +1,79 @@ +using CmdScale.EntityFrameworkCore.TimescaleDB.Internals; + +namespace CmdScale.EntityFrameworkCore.TimescaleDB.Tests.Internals; + +/// +/// Tests placeholder parsing and formatting for continuous aggregate expression templates. +/// +public class ColumnTemplateTests +{ + #region Should_Replace_Placeholders_In_Order + + [Fact] + public void Should_Replace_Placeholders_In_Order() + { + // Act + string sql = ColumnTemplate.Format("mcv_agg(50, 2.0, {1}) FILTER (WHERE {0} = 1)", i => $"\"c{i}\""); + + // Assert + Assert.Equal("mcv_agg(50, 2.0, \"c1\") FILTER (WHERE \"c0\" = 1)", sql); + } + + #endregion + + #region Should_Ignore_Braces_Inside_Literals_And_Quoted_Identifiers + + [Fact] + public void Should_Ignore_Braces_Inside_Literals_And_Quoted_Identifiers() + { + // Act + string sql = ColumnTemplate.Format("coalesce({0}, '{x}', 'it''s {1}') || \"odd{name}\"", _ => "\"v\""); + + // Assert + Assert.Equal("coalesce(\"v\", '{x}', 'it''s {1}') || \"odd{name}\"", sql); + } + + #endregion + + #region Should_Unescape_Doubled_Braces + + [Fact] + public void Should_Unescape_Doubled_Braces() + { + // Act + string sql = ColumnTemplate.Format("ARRAY[{0}]::text[] = '{{}}' OR {{0}}", _ => "\"v\""); + + // Assert + Assert.Equal("ARRAY[\"v\"]::text[] = '{{}}' OR {0}", sql); + } + + #endregion + + #region Should_List_Placeholder_Indices + + [Fact] + public void Should_List_Placeholder_Indices() + { + // Act + List indices = ColumnTemplate.GetPlaceholderIndices("{1} + {0} + {1} + '{2}'"); + + // Assert + Assert.Equal([1, 0, 1], indices); + } + + #endregion + + #region Should_Throw_For_Invalid_Placeholder + + [Theory] + [InlineData("count({column})")] + [InlineData("count({0)")] + [InlineData("count(0})")] + public void Should_Throw_For_Invalid_Placeholder(string template) + { + // Act & Assert + Assert.Throws(() => ColumnTemplate.GetPlaceholderIndices(template)); + } + + #endregion +} diff --git a/tests/Eftdb.Tests/TypeBuilders/ContinuousAggregateExpressionBuilderTests.cs b/tests/Eftdb.Tests/TypeBuilders/ContinuousAggregateExpressionBuilderTests.cs new file mode 100644 index 0000000..1ff6c65 --- /dev/null +++ b/tests/Eftdb.Tests/TypeBuilders/ContinuousAggregateExpressionBuilderTests.cs @@ -0,0 +1,355 @@ +using CmdScale.EntityFrameworkCore.TimescaleDB.Abstractions; +using CmdScale.EntityFrameworkCore.TimescaleDB.Configuration.ContinuousAggregate; +using CmdScale.EntityFrameworkCore.TimescaleDB.Configuration.Hypertable; +using CmdScale.EntityFrameworkCore.TimescaleDB.Internals; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; + +namespace CmdScale.EntityFrameworkCore.TimescaleDB.Tests.TypeBuilders; + +/// +/// Tests the annotations written by AddAggregateExpression, AddGroupByExpression, their attributes and +/// IsContinuousAggregateFromSql. +/// +public class ContinuousAggregateExpressionBuilderTests +{ + private const string ConnectionString = "Host=localhost;Database=test;Username=test;Password=test"; + + private static IEntityType GetEntityType(DbContext context) + => context.GetService().Model.FindEntityType(typeof(TEntity))!; + + #region Should_Store_Expressions_With_Property_Names + + private class TypedEvent + { + public DateTime OccurredAt { get; set; } + public string ActorId { get; set; } = ""; + public string? ImpersonatedActorId { get; set; } + } + + private class TypedRank + { + public DateTime Day { get; set; } + public bool Impersonated { get; set; } + public string Actors { get; set; } = ""; + } + + private class TypedContext : DbContext + { + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + => optionsBuilder.UseNpgsql(ConnectionString).UseTimescaleDb(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity(e => + { + e.HasNoKey(); + e.IsHypertable(x => x.OccurredAt); + }); + modelBuilder.Entity(e => + { + e.HasNoKey(); + e.IsContinuousAggregate("typed_ranks", "1 day", x => x.OccurredAt) + .WithTimeBucketProperty(x => x.Day) + .AddGroupByExpression(x => x.Impersonated, "{0} IS NOT NULL", x => x.ImpersonatedActorId) + .AddAggregateExpression(x => x.Actors, "mcv_agg(500, {0})", x => x.ActorId); + }); + } + } + + [Fact] + public void Should_Store_Expressions_With_Property_Names() + { + // Arrange + using TypedContext context = new(); + + // Act + IEntityType entityType = GetEntityType(context); + + // Assert + ContinuousAggregateExpression aggregate = Assert.Single(ContinuousAggregateExpressionJson.Read(entityType, ContinuousAggregateAnnotations.AggregateExpressions)); + Assert.Equal(new ContinuousAggregateExpression("Actors", "mcv_agg(500, {0})", ["ActorId"]), aggregate); + ContinuousAggregateExpression groupBy = Assert.Single(ContinuousAggregateExpressionJson.Read(entityType, ContinuousAggregateAnnotations.GroupByExpressions)); + Assert.Equal(new ContinuousAggregateExpression("Impersonated", "{0} IS NOT NULL", ["ImpersonatedActorId"]), groupBy); + } + + #endregion + + #region Should_Replace_Expression_With_Same_Alias + + private class ReplaceEvent + { + public DateTime OccurredAt { get; set; } + public string ActorId { get; set; } = ""; + } + + private class ReplaceRank + { + public DateTime Bucket { get; set; } + public string Actors { get; set; } = ""; + } + + private class ReplaceContext : DbContext + { + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + => optionsBuilder.UseNpgsql(ConnectionString).UseTimescaleDb(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity(e => + { + e.HasNoKey(); + e.IsHypertable(x => x.OccurredAt); + }); + modelBuilder.Entity(e => + { + e.HasNoKey(); + e.IsContinuousAggregate("replace_ranks", "1 day", x => x.OccurredAt) + .AddAggregateExpression(x => x.Actors, "mcv_agg(10, {0})", x => x.ActorId) + .AddAggregateExpression(x => x.Actors, "mcv_agg(20, {0})", x => x.ActorId); + }); + } + } + + [Fact] + public void Should_Replace_Expression_With_Same_Alias() + { + // Arrange + using ReplaceContext context = new(); + + // Act + IEntityType entityType = GetEntityType(context); + + // Assert + ContinuousAggregateExpression aggregate = Assert.Single(ContinuousAggregateExpressionJson.Read(entityType, ContinuousAggregateAnnotations.AggregateExpressions)); + Assert.Equal("mcv_agg(20, {0})", aggregate.Sql); + } + + #endregion + + #region Should_Throw_When_Placeholders_And_Columns_Do_Not_Match + + private class MismatchEvent + { + public DateTime OccurredAt { get; set; } + public string ActorId { get; set; } = ""; + public string EntityId { get; set; } = ""; + } + + private class MismatchRank + { + public DateTime Bucket { get; set; } + public string Actors { get; set; } = ""; + } + + private class MismatchContext(string sql, bool secondColumn) : DbContext + { + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + => optionsBuilder.UseNpgsql(ConnectionString).EnableServiceProviderCaching(false).UseTimescaleDb(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity(e => + { + e.HasNoKey(); + e.IsHypertable(x => x.OccurredAt); + }); + modelBuilder.Entity(e => + { + e.HasNoKey(); + ContinuousAggregateBuilder builder = + e.IsContinuousAggregate("mismatch_ranks", "1 day", x => x.OccurredAt); + if (secondColumn) + { + builder.AddAggregateExpression(x => x.Actors, sql, x => x.ActorId, x => x.EntityId); + } + else + { + builder.AddAggregateExpression(x => x.Actors, sql, x => x.ActorId); + } + }); + } + } + + [Theory] + [InlineData("mcv_agg(500, {1})", false)] + [InlineData("mcv_agg(500, {0})", true)] + [InlineData("mcv_agg(500, {actor})", false)] + public void Should_Throw_When_Placeholders_And_Columns_Do_Not_Match(string sql, bool secondColumn) + { + // Arrange + using MismatchContext context = new(sql, secondColumn); + + // Act & Assert + Assert.Throws(() => GetEntityType(context)); + } + + #endregion + + #region Should_Store_Expressions_From_String_Builder + + private class StringEvent + { + public DateTime OccurredAt { get; set; } + public string ActorId { get; set; } = ""; + } + + private class StringRank + { + public DateTime Bucket { get; set; } + public string Actors { get; set; } = ""; + public bool Late { get; set; } + } + + private class StringContext : DbContext + { + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + => optionsBuilder.UseNpgsql(ConnectionString).UseTimescaleDb(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity(e => + { + e.HasNoKey(); + e.IsHypertable(x => x.OccurredAt); + }); + modelBuilder.Entity(e => + { + e.HasNoKey(); + e.IsContinuousAggregate("string_ranks", nameof(StringEvent), "1 day", nameof(StringEvent.OccurredAt)) + .AddGroupByExpression(nameof(StringRank.Late), "{0} < now() - interval '1 day'", nameof(StringEvent.OccurredAt)) + .AddAggregateExpression(nameof(StringRank.Actors), "mcv_agg(500, {0})", nameof(StringEvent.ActorId)); + }); + } + } + + [Fact] + public void Should_Store_Expressions_From_String_Builder() + { + // Arrange + using StringContext context = new(); + + // Act + IEntityType entityType = GetEntityType(context); + + // Assert + Assert.Equal("Actors", Assert.Single(ContinuousAggregateExpressionJson.Read(entityType, ContinuousAggregateAnnotations.AggregateExpressions)).Alias); + Assert.Equal("Late", Assert.Single(ContinuousAggregateExpressionJson.Read(entityType, ContinuousAggregateAnnotations.GroupByExpressions)).Alias); + } + + #endregion + + #region Should_Store_Expressions_From_Attributes + + private class AttributeEvent + { + public DateTime OccurredAt { get; set; } + public string ActorId { get; set; } = ""; + public string? ImpersonatedActorId { get; set; } + } + + [ContinuousAggregate(MaterializedViewName = "attribute_ranks", ParentName = nameof(AttributeEvent))] + [TimeBucket("1 day", nameof(AttributeEvent.OccurredAt))] + private class AttributeRank + { + public DateTime Bucket { get; set; } + + [GroupByExpression("{0} IS NOT NULL", nameof(AttributeEvent.ImpersonatedActorId))] + public bool Impersonated { get; set; } + + [AggregateExpression("mcv_agg(500, {0})", nameof(AttributeEvent.ActorId))] + public string Actors { get; set; } = ""; + } + + private class AttributeContext : DbContext + { + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + => optionsBuilder.UseNpgsql(ConnectionString).UseTimescaleDb(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity(e => + { + e.HasNoKey(); + e.IsHypertable(x => x.OccurredAt); + }); + modelBuilder.Entity().HasNoKey(); + } + } + + [Fact] + public void Should_Store_Expressions_From_Attributes() + { + // Arrange + using AttributeContext context = new(); + + // Act + IEntityType entityType = GetEntityType(context); + + // Assert + Assert.Equal( + new ContinuousAggregateExpression("Actors", "mcv_agg(500, {0})", ["ActorId"]), + Assert.Single(ContinuousAggregateExpressionJson.Read(entityType, ContinuousAggregateAnnotations.AggregateExpressions))); + Assert.Equal( + new ContinuousAggregateExpression("Impersonated", "{0} IS NOT NULL", ["ImpersonatedActorId"]), + Assert.Single(ContinuousAggregateExpressionJson.Read(entityType, ContinuousAggregateAnnotations.GroupByExpressions))); + } + + #endregion + + #region Should_Store_Raw_View_Definition + + private class RawEvent + { + public DateTime OccurredAt { get; set; } + public string ActorId { get; set; } = ""; + } + + private class RawRank + { + public DateTime Bucket { get; set; } + public string Actors { get; set; } = ""; + } + + private class RawContext : DbContext + { + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + => optionsBuilder.UseNpgsql(ConnectionString).UseTimescaleDb(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity(e => + { + e.HasNoKey(); + e.IsHypertable(x => x.OccurredAt); + }); + modelBuilder.Entity(e => + { + e.HasNoKey(); + e.IsContinuousAggregateFromSql( + "raw_ranks", + nameof(RawEvent), + """SELECT time_bucket('1 day', "OccurredAt") AS "Bucket", mcv_agg(5, "ActorId") AS "Actors" FROM "RawEvent" GROUP BY 1""") + .MaterializedOnly(false); + }); + } + } + + [Fact] + public void Should_Store_Raw_View_Definition() + { + // Arrange + using RawContext context = new(); + + // Act + IEntityType entityType = GetEntityType(context); + + // Assert + Assert.Equal("raw_ranks", entityType.GetViewName()); + Assert.StartsWith("SELECT time_bucket", entityType.FindAnnotation(ContinuousAggregateAnnotations.ViewDefinition)?.Value as string); + Assert.Null(entityType.FindAnnotation(ContinuousAggregateAnnotations.TimeBucketWidth)); + } + + #endregion +} diff --git a/tests/Eftdb.Tests/Utils/TimescaleImages.cs b/tests/Eftdb.Tests/Utils/TimescaleImages.cs index ab7ed24..eaad673 100644 --- a/tests/Eftdb.Tests/Utils/TimescaleImages.cs +++ b/tests/Eftdb.Tests/Utils/TimescaleImages.cs @@ -7,5 +7,13 @@ internal static class TimescaleImages { public const string Community = "timescale/timescaledb:latest-pg17"; public const string Apache = "timescale/timescaledb:latest-pg17-oss"; + + /// + /// Ships the timescaledb_toolkit extension, which the plain images lack. Tests using it carry + /// so they can be filtered. + /// + public const string Toolkit = "timescale/timescaledb-ha:pg17"; + + public const string ToolkitTrait = "Toolkit"; } } From 2b8d3d640a8346793b24a9b5519f01fc7ffe0521 Mon Sep 17 00:00:00 2001 From: JT Date: Tue, 27 Oct 2026 21:44:29 +0800 Subject: [PATCH 2/2] Fix code coverage --- .../ContinuousAggregateCSharpGenerator.cs | 4 +- .../Scaffolding/ViewDefinitionParser.cs | 2 +- .../ContinuousAggregateExpressionJson.cs | 2 +- ...ContinuousAggregateCSharpGeneratorTests.cs | 27 +++++ ...inuousAggregateEntryIgnoredWarningTests.cs | 64 +++++++++++ ...inuousAggregateExpressionExtractorTests.cs | 86 +++++++++++++++ .../Internals/ColumnTemplateTests.cs | 14 +++ ...ntinuousAggregateExpressionBuilderTests.cs | 100 ++++++++++++++++++ 8 files changed, 295 insertions(+), 4 deletions(-) diff --git a/src/Eftdb.Design/Features/ContinuousAggregate/ContinuousAggregateCSharpGenerator.cs b/src/Eftdb.Design/Features/ContinuousAggregate/ContinuousAggregateCSharpGenerator.cs index b460075..24659d2 100644 --- a/src/Eftdb.Design/Features/ContinuousAggregate/ContinuousAggregateCSharpGenerator.cs +++ b/src/Eftdb.Design/Features/ContinuousAggregate/ContinuousAggregateCSharpGenerator.cs @@ -68,10 +68,10 @@ public void Generate(CreateContinuousAggregateOperation operation, IndentedStrin if (operation.CompressionOrderBy is { Count: > 0 }) call.Arg("compressionOrderBy", CSharpGeneratorHelper.LiteralStringList(code, operation.CompressionOrderBy)); - if (operation.AggregateExpressions is { Count: > 0 }) + if (operation.AggregateExpressions.Count > 0) call.Arg("aggregateExpressions", b => AppendExpressionList(b, operation.AggregateExpressions)); - if (operation.GroupByExpressions is { Count: > 0 }) + if (operation.GroupByExpressions.Count > 0) call.Arg("groupByExpressions", b => AppendExpressionList(b, operation.GroupByExpressions)); } diff --git a/src/Eftdb.Design/Scaffolding/ViewDefinitionParser.cs b/src/Eftdb.Design/Scaffolding/ViewDefinitionParser.cs index 7f56767..fb212f0 100644 --- a/src/Eftdb.Design/Scaffolding/ViewDefinitionParser.cs +++ b/src/Eftdb.Design/Scaffolding/ViewDefinitionParser.cs @@ -64,7 +64,7 @@ private static bool AccountsForEverySelectItem(string viewDefinition, IReadOnlyL foreach (string rawItem in SplitTopLevel(selectList)) { string item = rawItem.Trim(); - if (item.Length == 0 || item.Contains("time_bucket", StringComparison.OrdinalIgnoreCase)) + if (item.Contains("time_bucket", StringComparison.OrdinalIgnoreCase)) { continue; } diff --git a/src/Eftdb/Internals/ContinuousAggregateExpressionJson.cs b/src/Eftdb/Internals/ContinuousAggregateExpressionJson.cs index 151fe8f..ae784f2 100644 --- a/src/Eftdb/Internals/ContinuousAggregateExpressionJson.cs +++ b/src/Eftdb/Internals/ContinuousAggregateExpressionJson.cs @@ -12,7 +12,7 @@ internal static class ContinuousAggregateExpressionJson { public static List Read(IReadOnlyAnnotatable annotatable, string annotation) => annotatable.FindAnnotation(annotation)?.Value is string { Length: > 0 } json - ? JsonSerializer.Deserialize>(json) ?? [] + ? JsonSerializer.Deserialize>(json)! : []; public static string Write(IReadOnlyList expressions) diff --git a/tests/Eftdb.Tests/Design/Features/ContinuousAggregate/ContinuousAggregateCSharpGeneratorTests.cs b/tests/Eftdb.Tests/Design/Features/ContinuousAggregate/ContinuousAggregateCSharpGeneratorTests.cs index c1f41c3..53678e6 100644 --- a/tests/Eftdb.Tests/Design/Features/ContinuousAggregate/ContinuousAggregateCSharpGeneratorTests.cs +++ b/tests/Eftdb.Tests/Design/Features/ContinuousAggregate/ContinuousAggregateCSharpGeneratorTests.cs @@ -408,5 +408,32 @@ public void CreateContinuousAggregate_Expressions_FullyQualifiedTypedEntries() } #endregion + + #region CreateContinuousAggregate_MultipleExpressions_SeparatedByCommas + + [Fact] + public void CreateContinuousAggregate_MultipleExpressions_SeparatedByCommas() + { + // Arrange + CreateContinuousAggregateOperation op = new() + { + MaterializedViewName = "ranks", + ParentName = "events", + AggregateExpressions = + [ + new("reads", "count(*) FILTER (WHERE {0} = 1)", ["action"]), + new("actors", "mcv_agg(500, {0})", ["actor_id"]), + ], + }; + + // Act + string result = Generate(op).ReplaceLineEndings("\n"); + + // Assert + Assert.Contains("[\"action\"]),\n", result); + Assert.Contains("[\"actor_id\"])\n", result); + } + + #endregion } } diff --git a/tests/Eftdb.Tests/Diagnostics/ContinuousAggregateEntryIgnoredWarningTests.cs b/tests/Eftdb.Tests/Diagnostics/ContinuousAggregateEntryIgnoredWarningTests.cs index f89a827..52f8cbb 100644 --- a/tests/Eftdb.Tests/Diagnostics/ContinuousAggregateEntryIgnoredWarningTests.cs +++ b/tests/Eftdb.Tests/Diagnostics/ContinuousAggregateEntryIgnoredWarningTests.cs @@ -4,6 +4,7 @@ using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Diagnostics; using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.Extensions.Logging; namespace CmdScale.EntityFrameworkCore.TimescaleDB.Tests.Diagnostics; @@ -103,4 +104,67 @@ public void Should_Throw_When_Ignored_Entry_Warning_Is_An_Error() } #endregion + + #region Should_Reach_ILogger_Sink_For_Ignored_Entry + + private sealed class CapturingLoggerProvider(List<(EventId EventId, string Message)> entries) : ILoggerProvider + { + public ILogger CreateLogger(string categoryName) => new CapturingLogger(entries); + + public void Dispose() { } + + private sealed class CapturingLogger(List<(EventId, string)> sink) : ILogger + { + public IDisposable? BeginScope(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + => sink.Add((eventId, formatter(state, exception))); + } + } + + private class LoggerFactoryContext(ILoggerFactory loggerFactory) : DbContext + { + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + => optionsBuilder.UseNpgsql("Host=localhost;Database=dummy;Username=x;Password=x") + .UseLoggerFactory(loggerFactory) + .EnableServiceProviderCaching(false) + .UseTimescaleDb(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity(e => + { + e.HasNoKey(); + e.ToTable("ignored_logger_events"); + e.IsHypertable(x => x.OccurredAt); + }); + modelBuilder.Entity(e => + { + e.HasNoKey(); + e.Property(x => x.Bucket).HasColumnName("time_bucket"); + e.IsContinuousAggregate("ignored_logger_hourly", nameof(IgnoredEvent), "1 hour", nameof(IgnoredEvent.OccurredAt)); + e.HasAnnotation(ContinuousAggregateAnnotations.AggregateFunctions, new List { "Total:Sum:NoSuchColumn" }); + }); + } + } + + [Fact] + public void Should_Reach_ILogger_Sink_For_Ignored_Entry() + { + // Arrange + List<(EventId EventId, string Message)> entries = []; + using ILoggerFactory loggerFactory = LoggerFactory.Create(b => b.AddProvider(new CapturingLoggerProvider(entries))); + using LoggerFactoryContext context = new(loggerFactory); + + // Act + _ = context.Model; + + // Assert + (EventId _, string message) = Assert.Single(entries, e => e.EventId == TimescaleDbEventId.ContinuousAggregateEntryIgnored); + Assert.Contains("NoSuchColumn", message); + } + + #endregion } diff --git a/tests/Eftdb.Tests/Extractors/ContinuousAggregateExpressionExtractorTests.cs b/tests/Eftdb.Tests/Extractors/ContinuousAggregateExpressionExtractorTests.cs index 4ae289f..ef0d937 100644 --- a/tests/Eftdb.Tests/Extractors/ContinuousAggregateExpressionExtractorTests.cs +++ b/tests/Eftdb.Tests/Extractors/ContinuousAggregateExpressionExtractorTests.cs @@ -317,4 +317,90 @@ public void Should_Throw_When_Expression_Alias_Collides() } #endregion + + #region Should_Keep_Unmapped_Expression_Alias_Verbatim + + private class UnmappedEvent + { + public DateTime OccurredAt { get; set; } + public string ActorId { get; set; } = ""; + } + + private class UnmappedRank + { + public DateTime Bucket { get; set; } + } + + private class UnmappedContext : DbContext + { + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + => optionsBuilder.UseNpgsql(ConnectionString).UseTimescaleDb(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity(e => + { + e.HasNoKey(); + e.IsHypertable(x => x.OccurredAt); + }); + modelBuilder.Entity(e => + { + e.HasNoKey(); + e.IsContinuousAggregate("unmapped_ranks", nameof(UnmappedEvent), "1 day", nameof(UnmappedEvent.OccurredAt)) + .AddAggregateExpression("actor_sketch", "mcv_agg(5, {0})", nameof(UnmappedEvent.ActorId)); + }); + } + } + + [Fact] + public void Should_Keep_Unmapped_Expression_Alias_Verbatim() + { + // Arrange + using UnmappedContext context = new(); + + // Act + CreateContinuousAggregateOperation operation = ExtractSingle(context); + + // Assert + Assert.Equal(new ContinuousAggregateExpression("actor_sketch", "mcv_agg(5, {0})", ["ActorId"]), Assert.Single(operation.AggregateExpressions)); + } + + #endregion + + #region Should_Skip_Expression_Validation_When_Parent_Does_Not_Resolve + + private class OrphanRank + { + public DateTime Bucket { get; set; } + } + + private class OrphanContext : DbContext + { + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + => optionsBuilder.UseNpgsql(ConnectionString).UseTimescaleDb(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + => modelBuilder.Entity(e => + { + e.HasNoKey(); + e.IsContinuousAggregate("orphan_ranks", "missing_parent", "1 day", "OccurredAt") + .AddAggregateExpression("actors", "mcv_agg(5, {0})", "actor_id"); + }); + } + + [Fact] + public void Should_Skip_Expression_Validation_When_Parent_Does_Not_Resolve() + { + // Arrange + using OrphanContext context = new(); + + // Act + List operations = [.. ContinuousAggregateModelExtractor.GetContinuousAggregates( + context.GetService().Model.GetRelationalModel())]; + + // Assert + Assert.Empty(operations); + } + + #endregion } diff --git a/tests/Eftdb.Tests/Internals/ColumnTemplateTests.cs b/tests/Eftdb.Tests/Internals/ColumnTemplateTests.cs index ee683c6..6f1f67b 100644 --- a/tests/Eftdb.Tests/Internals/ColumnTemplateTests.cs +++ b/tests/Eftdb.Tests/Internals/ColumnTemplateTests.cs @@ -76,4 +76,18 @@ public void Should_Throw_For_Invalid_Placeholder(string template) } #endregion + + #region Should_Keep_Unterminated_Literal_Verbatim + + [Fact] + public void Should_Keep_Unterminated_Literal_Verbatim() + { + // Act + string sql = ColumnTemplate.Format("coalesce({0}, 'open {1}", _ => "\"v\""); + + // Assert + Assert.Equal("coalesce(\"v\", 'open {1}", sql); + } + + #endregion } diff --git a/tests/Eftdb.Tests/TypeBuilders/ContinuousAggregateExpressionBuilderTests.cs b/tests/Eftdb.Tests/TypeBuilders/ContinuousAggregateExpressionBuilderTests.cs index 1ff6c65..5d1facc 100644 --- a/tests/Eftdb.Tests/TypeBuilders/ContinuousAggregateExpressionBuilderTests.cs +++ b/tests/Eftdb.Tests/TypeBuilders/ContinuousAggregateExpressionBuilderTests.cs @@ -352,4 +352,104 @@ public void Should_Store_Raw_View_Definition() } #endregion + + #region Should_Throw_For_Empty_View_Definition + + private class EmptyRawRank + { + public DateTime Bucket { get; set; } + } + + private class EmptyRawContext : DbContext + { + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + => optionsBuilder.UseNpgsql(ConnectionString).EnableServiceProviderCaching(false).UseTimescaleDb(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + => modelBuilder.Entity(e => + { + e.HasNoKey(); + e.IsContinuousAggregateFromSql("empty_raw_ranks", "events", " "); + }); + } + + [Fact] + public void Should_Throw_For_Empty_View_Definition() + { + // Arrange + using EmptyRawContext context = new(); + + // Act & Assert + ArgumentException exception = Assert.Throws(() => GetEntityType(context)); + Assert.Equal("viewDefinition", exception.ParamName); + } + + #endregion + + #region Should_Throw_For_Empty_Alias_Or_Sql + + private class EmptyPartEvent + { + public DateTime OccurredAt { get; set; } + } + + private class EmptyPartRank + { + public DateTime Bucket { get; set; } + } + + private class EmptyPartContext(string alias, string sql) : DbContext + { + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + => optionsBuilder.UseNpgsql(ConnectionString).EnableServiceProviderCaching(false).UseTimescaleDb(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity(e => + { + e.HasNoKey(); + e.IsHypertable(x => x.OccurredAt); + }); + modelBuilder.Entity(e => + { + e.HasNoKey(); + e.IsContinuousAggregate("empty_part_ranks", nameof(EmptyPartEvent), "1 day", nameof(EmptyPartEvent.OccurredAt)) + .AddAggregateExpression(alias, sql); + }); + } + } + + [Theory] + [InlineData("", "count(*)", "alias")] + [InlineData("Total", " ", "sql")] + public void Should_Throw_For_Empty_Alias_Or_Sql(string alias, string sql, string parameter) + { + // Arrange + using EmptyPartContext context = new(alias, sql); + + // Act & Assert + ArgumentException exception = Assert.Throws(() => GetEntityType(context)); + Assert.Equal(parameter, exception.ParamName); + } + + #endregion + + #region Should_Compare_Expressions_By_Value + + [Fact] + public void Should_Compare_Expressions_By_Value() + { + // Arrange + ContinuousAggregateExpression expression = new("actors", "mcv_agg(500, {0})", ["actor_id"]); + object same = new ContinuousAggregateExpression("actors", "mcv_agg(500, {0})", ["actor_id"]); + object otherColumn = new ContinuousAggregateExpression("actors", "mcv_agg(500, {0})", ["entity_id"]); + + // Act & Assert + Assert.True(expression.Equals(same)); + Assert.Equal(expression.GetHashCode(), same.GetHashCode()); + Assert.False(expression.Equals(otherColumn)); + Assert.False(expression.Equals("actors")); + } + + #endregion }