From 8e1c0cf871c6d85d221cb4c411ab40776eaaa3e6 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 07:10:05 +0000 Subject: [PATCH 1/3] =?UTF-8?q?fix(objectql,core,driver-sql,driver-memory)?= =?UTF-8?q?:=20a=20per-aggregation=20filter=20and=20having=20read=20a=20te?= =?UTF-8?q?mporal=20comparand=20by=20the=20column's=20storage=20rule=20?= =?UTF-8?q?=E2=80=94=20one=20rule,=20lifted=20into=20core,=20shared=20with?= =?UTF-8?q?=20both=20drivers'=20where?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude-Session: https://claude.ai/code/session_01Bvd69VPa6puiNzzPUroDBx Co-authored-by: Claude --- packages/core/src/index.ts | 7 + packages/core/src/utils/temporal-comparand.ts | 14 +- .../core/src/utils/temporal-storage-form.ts | 152 ++++++++++++++++ .../driver-memory/src/memory-temporal.ts | 123 ++----------- packages/drivers/driver-sql/src/sql-driver.ts | 80 ++------- packages/objectql/src/engine.ts | 29 +-- packages/objectql/src/having-filter.ts | 167 +++++++++++++++--- .../objectql/src/in-memory-aggregation.ts | 25 ++- 8 files changed, 371 insertions(+), 226 deletions(-) create mode 100644 packages/core/src/utils/temporal-storage-form.ts diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index fc1f415cd69..61f02b9828f 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -97,6 +97,13 @@ export * from './utils/analytics-date-range-conformance.js'; // on each other. export * from './utils/temporal-comparand.js'; +// [#20176] …and the storage FORM that column's rule puts a value in — the one +// rule `driver-sql` and `driver-memory` apply to a `where` comparand (and on +// write), and `@objectstack/objectql` applies to a per-aggregation `filter` and +// `having`, which it evaluates itself. Here because the drivers and the engine +// do not depend on each other, and each driver used to carry its own copy. +export * from './utils/temporal-storage-form.js'; + // [#12350 / ADR-0126 §4] THE activation-ledger row contract, parameterized by // `metadata_type`. Same reason as the two entries above: its consumers — // `@objectstack/objectql` (packaged actions) and diff --git a/packages/core/src/utils/temporal-comparand.ts b/packages/core/src/utils/temporal-comparand.ts index 3ecfc133ca6..6c4996d480c 100644 --- a/packages/core/src/utils/temporal-comparand.ts +++ b/packages/core/src/utils/temporal-comparand.ts @@ -41,9 +41,9 @@ * ## Interpretability is defined by the DRIVERS' own totals, deliberately * * Each predicate below mirrors the total function that would receive the value - * if the door let it through — `storageDatetimeValue` / `storageDateValue` / - * `storageTimeValue` in `driver-memory`, and their `SqlDriver` twins. Those - * functions are total on purpose: an input they cannot interpret is returned + * if the door let it through — `temporalStorageForm` (`temporal-storage-form.ts`, + * the one rule both drivers read since #20176; before it, a copy in each). + * That function is total on purpose: an input it cannot interpret is returned * UNCHANGED rather than becoming an invented instant. So "the driver would * return it unchanged" IS the definition of uninterpretable, and defining it * any other way would refuse comparands that work today. @@ -87,7 +87,7 @@ export function temporalComparandKind(fieldType: unknown): TemporalComparandKind } /** - * `storageDatetimeValue`'s reading, as a yes/no. + * `temporalStorageForm`'s `datetime` reading, as a yes/no. * * Mirrors it step for step: a bare integer in either sign is epoch * milliseconds; a bare `YYYY-MM-DD` is midnight UTC; a zone-naive @@ -105,11 +105,11 @@ function readsAsInstant(s: string): boolean { } /** - * `storageDateValue`'s reading of a STRING: its leading `YYYY-MM-DD`, and + * `temporalStorageForm`'s `date` reading of a STRING: its leading `YYYY-MM-DD`, and * nothing else. * * Narrower than {@link readsAsInstant} on purpose, because the rule it mirrors - * is narrower: `storageDateValue` collapses a leading calendar day and returns + * is narrower: the `date` rule collapses a leading calendar day and returns * every other string untouched. `2026/07/15` is therefore uninterpretable for a * `date` column even though `Date.parse` reads it — today it survives to the * driver and compares as text against `YYYY-MM-DD` values, which is the silent @@ -120,7 +120,7 @@ function readsAsCalendarDay(s: string): boolean { } /** - * `storageTimeValue`'s reading: a bare wall clock whose components are in + * `temporalStorageForm`'s `time` reading: a bare wall clock whose components are in * range. Out-of-range (`25:00`) is uninterpretable — the rule it mirrors * returns such a value untouched rather than wrapping it. */ diff --git a/packages/core/src/utils/temporal-storage-form.ts b/packages/core/src/utils/temporal-storage-form.ts new file mode 100644 index 00000000000..b97f195d35b --- /dev/null +++ b/packages/core/src/utils/temporal-storage-form.ts @@ -0,0 +1,152 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#20176] THE storage form of a temporal value — one rule, read by every face + * that compares one. + * + * ADR-0053 fixes one LOGICAL storage form per temporal field type: + * + * | Field type | Stored as | Why | + * |---|---|---| + * | `datetime` | canonical UTC ISO text, `YYYY-MM-DDTHH:MM:SS.sssZ` (`Date#toISOString`) | fixed width + UTC, so text order IS chronological order, and it is the wire form (D-B) | + * | `date` | `YYYY-MM-DD` | a timezone-naive calendar day (Phase 1) | + * | `time` | `HH:MM:SS`, `.fff` only when non-zero | a timezone-naive wall clock (D-C1); `.` sorts below every digit, so the two widths still sort chronologically | + * + * A filter comparand must be put into the column's form before it is compared, + * or the comparison is decided by the two sides' SHAPES disagreeing rather than + * by their values. Three faces apply that pairing: + * + * - `driver-sql` — on write (`formatInput`), on read presentation and on every + * `where` comparand (`coerceFilterValue`, public as `temporalFilterValue`). + * A MySQL `datetime` then takes a PHYSICAL spelling of the same instant + * (`mysqlDatetimeLiteral`), which stays the driver's: it is a dialect's + * literal syntax, not a second rule. + * - `driver-memory` — on write and on every `where` comparand + * (`coerceTemporalValue`). + * - `@objectstack/objectql` — a per-aggregation `filter` and `having`, which + * the engine evaluates itself over the rows a driver returned + * (`having-filter.ts`, with the column's class from the object's declaration + * or the aggregated row's). + * + * Each driver used to carry its own copy of the three functions below, word for + * word; the engine carried none and compared type-blind, so an ISO instant + * against a `date` column counted 1 row where the same condition in a `where` + * counted 3. The copies agreed shape for shape when they were lifted here + * (measured on 71 shapes × 3 kinds: `Date`s valid, invalid and out of range, + * epoch numbers and strings, bare days, zone-naive and zone-explicit + * timestamps, wall clocks in and out of range, junk, `null`, structures) — so + * this is a move, not a change, for either driver. + * + * ## Form only, never bound semantics + * + * The function is operator-blind. The whole-day reading of a bare + * `YYYY-MM-DD` used as an UPPER bound on a `datetime` column (`$lte`, a + * `$between` max → `< nextUtcCalendarDay(day)`, ADR-0053 D-D) is a separate, + * operator-sensitive rule every emitter applies BEFORE this one, from + * `@objectstack/spec/data`'s `nextUtcCalendarDay`. + * + * ## Total, on purpose + * + * `null` / `undefined`, the empty string, an out-of-range wall clock + * (`'25:00'`), an Invalid Date and unparseable junk come back UNCHANGED rather + * than as an invented value — a value the rule cannot interpret is never + * silently rewritten, so junk keeps failing its comparison. Which strings are + * uninterpretable is the question `isUninterpretableTemporalComparand` + * (`temporal-comparand.ts`) answers for the doors that refuse them. + */ + +import type { TemporalComparandKind } from './temporal-comparand.js'; + +/** + * The storage form of `value` for a column of `kind` — see the module note for + * the table. + * + * - `datetime`: a `Date` → its ISO text; a finite number, or a bare integer + * string in either sign → epoch milliseconds; a bare `YYYY-MM-DD` → midnight + * UTC, stated explicitly so it is never re-read in a server's local zone; a + * zone-naive `YYYY-MM-DD[ T]HH:MM[:SS[.fff]]` → its wall clock read AS UTC + * (the rule `CURRENT_TIMESTAMP`-written rows take on read, ADR-0074); any + * other string → whatever `Date.parse` reads, re-spelled in UTC. `Z` + * everywhere: `'…T12:00+08:00'` and `'…T04:00Z'` are one instant but sort + * differently as text. + * - `date`: a `Date` → its UTC calendar day (the UTC clock, never the host's — + * `SqlDriver.toDateOnly` records why); a string → its leading `YYYY-MM-DD`. + * - `time`: a bare `HH:MM[:SS[.f…]]` in range → `HH:MM:SS`, `.fff` kept only + * when non-zero (fractions beyond milliseconds truncated); anything else is + * read as an INSTANT by the `datetime` rule and keeps its UTC time of day. + * + * An array is NOT mapped — a caller comparing a list maps its members, the way + * the drivers' `coerceFilterValue` / `coerceTemporalValue` do. + */ +export function temporalStorageForm(value: unknown, kind: TemporalComparandKind): unknown { + if (kind === 'datetime') return canonicalUtcDatetime(value); + if (kind === 'time') return canonicalTimeOfDay(value); + return canonicalCalendarDay(value); +} + +function canonicalUtcDatetime(value: unknown): unknown { + if (value == null) return value; + if (value instanceof Date) { + return Number.isNaN(value.getTime()) ? value : value.toISOString(); + } + if (typeof value === 'number') { + if (!Number.isFinite(value)) return value; + const d = new Date(value); + return Number.isNaN(d.getTime()) ? value : d.toISOString(); + } + if (typeof value !== 'string') return value; + const s = value.trim(); + if (s === '') return value; + // A bare integer (in either JS or string form) is epoch milliseconds — the + // shape better-sqlite3 wrote for every `Date` bound before the canon (#3912). + if (/^-?\d+$/.test(s)) { + const d = new Date(Number(s)); + return Number.isNaN(d.getTime()) ? value : d.toISOString(); + } + const iso = /^\d{4}-\d{2}-\d{2}$/.test(s) + ? `${s}T00:00:00.000Z` + : /^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}(:\d{2}(\.\d+)?)?$/.test(s) + ? `${s.replace(' ', 'T')}Z` + : s; + const ms = Date.parse(iso); + return Number.isFinite(ms) ? new Date(ms).toISOString() : value; +} + +function canonicalCalendarDay(value: unknown): unknown { + if (value == null) return value; + if (value instanceof Date) { + if (Number.isNaN(value.getTime())) return value; + const y = value.getUTCFullYear(); + const m = String(value.getUTCMonth() + 1).padStart(2, '0'); + const d = String(value.getUTCDate()).padStart(2, '0'); + return `${y}-${m}-${d}`; + } + if (typeof value === 'string') { + const trimmed = value.trim(); + if (/^\d{4}-\d{2}-\d{2}/.test(trimmed)) return trimmed.slice(0, 10); + } + return value; +} + +function canonicalTimeOfDay(value: unknown): unknown { + if (value == null) return value; + if (typeof value === 'string') { + const s = value.trim(); + if (s === '') return value; + const m = /^(\d{2}):(\d{2})(?::(\d{2})(?:\.(\d+))?)?$/.exec(s); + if (m) { + const [, hh, mm, ss = '00', frac] = m; + if (Number(hh) > 23 || Number(mm) > 59 || Number(ss) > 59) return value; + const ms = frac ? `${frac}000`.slice(0, 3) : '000'; + return ms === '000' ? `${hh}:${mm}:${ss}` : `${hh}:${mm}:${ss}.${ms}`; + } + } + // Not a bare wall clock — a `Date`, epoch ms, or a full / zone-naive + // timestamp: an instant. Read it by the ONE instant rule, keep its UTC time. + const instant = canonicalUtcDatetime(value); + if (typeof instant === 'string' && /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/.test(instant)) { + const time = instant.slice(11, 23); + return time.endsWith('.000') ? time.slice(0, 8) : time; + } + return value; +} diff --git a/packages/drivers/driver-memory/src/memory-temporal.ts b/packages/drivers/driver-memory/src/memory-temporal.ts index a7f7cd9e1e0..a1d3ec9dcaa 100644 --- a/packages/drivers/driver-memory/src/memory-temporal.ts +++ b/packages/drivers/driver-memory/src/memory-temporal.ts @@ -25,118 +25,19 @@ * * The rule is applied on write ({@link toStorageForms}) and to filter * comparands, which is the pairing that keeps the two sides from disagreeing. - */ - -/** - * The canonical UTC ISO text form of a `Field.datetime` value. * - * Total by design: an input this cannot interpret is returned unchanged rather - * than becoming `Invalid Date`, so junk keeps failing the comparison instead of - * silently becoming a wrong instant. A bare `YYYY-MM-DD` means **midnight - * UTC**, stated explicitly so it is never re-read in the host's local zone; the - * whole-day reading of a bare day used as an UPPER bound is the separate, - * operator-sensitive concern `nextUtcCalendarDay` handles (#3777/#4042) before - * this runs. - * - * Mirrors `SqlDriver`'s `canonicalUtcDatetime` — same inputs, same outputs — so - * a fixture that moves between the two backends compares identically. - */ -export function storageDatetimeValue(value: unknown): unknown { - if (value == null) return value; - if (value instanceof Date) { - return Number.isNaN(value.getTime()) ? value : value.toISOString(); - } - if (typeof value === 'number') { - if (!Number.isFinite(value)) return value; - const d = new Date(value); - return Number.isNaN(d.getTime()) ? value : d.toISOString(); - } - if (typeof value !== 'string') return value; - const s = value.trim(); - if (s === '') return value; - // A bare integer (in either form) is epoch milliseconds. - if (/^-?\d+$/.test(s)) { - const d = new Date(Number(s)); - return Number.isNaN(d.getTime()) ? value : d.toISOString(); - } - const iso = /^\d{4}-\d{2}-\d{2}$/.test(s) - ? `${s}T00:00:00.000Z` - // Zone-naive `YYYY-MM-DD[ T]HH:MM[:SS[.fff]]` → its wall clock IS UTC. - : /^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}(:\d{2}(\.\d+)?)?$/.test(s) - ? `${s.replace(' ', 'T')}Z` - : s; - const ms = Date.parse(iso); - return Number.isFinite(ms) ? new Date(ms).toISOString() : value; -} - -/** - * Collapse a `Field.date` value to timezone-naive `YYYY-MM-DD` — a `Date` to - * its UTC calendar day, a string to its leading date. Mirrors - * `SqlDriver.toDateOnly`. + * [#20176] The rule itself is `@objectstack/core`'s `temporalStorageForm` — + * the one function `driver-sql` and the engine's per-aggregation `filter` and + * `having` read too. This file carried a word-for-word copy of it + * (`storageDatetimeValue` / `storageDateValue` / `storageTimeValue`, mirroring + * `SqlDriver`'s `canonicalUtcDatetime` / `toDateOnly` / `canonicalTimeOfDay`); + * the copies agreed shape for shape when they were lifted, so every answer + * this driver gives is unchanged. What stays here is this driver's own half: + * which declared fields are temporal, and that a list comparand is mapped + * member by member. */ -export function storageDateValue(value: unknown): unknown { - if (value == null) return value; - if (value instanceof Date) { - if (Number.isNaN(value.getTime())) return value; - const y = value.getUTCFullYear(); - const m = String(value.getUTCMonth() + 1).padStart(2, '0'); - const d = String(value.getUTCDate()).padStart(2, '0'); - return `${y}-${m}-${d}`; - } - if (typeof value === 'string') { - const trimmed = value.trim(); - if (/^\d{4}-\d{2}-\d{2}/.test(trimmed)) return trimmed.slice(0, 10); - } - return value; -} -/** - * Collapse a `Field.time` value to a canonical timezone-naive wall clock — - * `HH:MM:SS`, extended to `HH:MM:SS.fff` exactly when the milliseconds are - * non-zero. Mirrors `SqlDriver`'s `canonicalTimeOfDay` (ADR-0053 D-C1), so a - * fixture that moves between the two backends compares identically. - * - * This driver had no time rule at all, which is the same meta-problem #3994 - * found on SQL and #4047 found here for `datetime`: with nothing normalising - * the column, a `Date` write and a `'09:00:00'` write sat side by side, and - * mingo's cross-type comparison meant a text bound matched no `Date` row in - * either direction. Measured: 8 of the 9 shared time cases returned only the - * text-written half of the fixture. - * - * Why variable width rather than a fixed `.000`: `.` sorts below every digit, - * so lexicographic order — which is exactly what mingo performs on strings — - * stays chronological across the two widths (`'14:30:00.100' < '14:30:01'`), - * and the zero-millisecond spelling stays the `HH:MM:SS` every dialect's - * native TIME emits. - * - * A `Date` / epoch-ms / full-timestamp value folds to its **UTC** time-of-day, - * never the host's, matching the platform's instant semantics everywhere else. - * Total: an out-of-range wall clock (`'25:00'`) or unparseable junk passes - * through untouched rather than being silently rewritten. - */ -export function storageTimeValue(value: unknown): unknown { - if (value == null) return value; - if (typeof value === 'string') { - const s = value.trim(); - if (s === '') return value; - const m = /^(\d{2}):(\d{2})(?::(\d{2})(?:\.(\d+))?)?$/.exec(s); - if (m) { - const [, hh, mm, ss = '00', frac] = m; - if (Number(hh) > 23 || Number(mm) > 59 || Number(ss) > 59) return value; - const ms = frac ? `${frac}000`.slice(0, 3) : '000'; - return ms === '000' ? `${hh}:${mm}:${ss}` : `${hh}:${mm}:${ss}.${ms}`; - } - } - // Not a bare wall clock — a `Date`, epoch ms, or a full/zone-naive timestamp - // string. Delegate to the one function that owns instants, then keep its UTC - // time-of-day. Same delegation `canonicalTimeOfDay` performs. - const instant = storageDatetimeValue(value); - if (typeof instant === 'string' && /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/.test(instant)) { - const time = instant.slice(11, 23); - return time.endsWith('.000') ? time.slice(0, 8) : time; - } - return value; -} +import { temporalStorageForm } from '@objectstack/core'; /** Which temporal rule a declared field takes, if any. */ export type TemporalFieldKind = 'datetime' | 'date' | 'time'; @@ -149,9 +50,7 @@ export type TemporalFieldKind = 'datetime' | 'date' | 'time'; export function coerceTemporalValue(value: unknown, kind: TemporalFieldKind | undefined): unknown { if (kind === undefined) return value; if (Array.isArray(value)) return value.map((v) => coerceTemporalValue(v, kind)); - if (kind === 'datetime') return storageDatetimeValue(value); - if (kind === 'time') return storageTimeValue(value); - return storageDateValue(value); + return temporalStorageForm(value, kind); } /** diff --git a/packages/drivers/driver-sql/src/sql-driver.ts b/packages/drivers/driver-sql/src/sql-driver.ts index 63fa2de13c9..bf1077542da 100644 --- a/packages/drivers/driver-sql/src/sql-driver.ts +++ b/packages/drivers/driver-sql/src/sql-driver.ts @@ -78,7 +78,7 @@ import { declareTargetedTable, } from '@objectstack/types'; import { postureEnforcesWall } from '@objectstack/spec/security'; -import { nextUtcCalendarDay } from '@objectstack/core'; +import { nextUtcCalendarDay, temporalStorageForm } from '@objectstack/core'; import { applyIndexKeyParts, buildIndexName, @@ -512,36 +512,10 @@ function normalizeSqliteDatetimeOutput(value: unknown): unknown { * interpret is never silently rewritten. */ function canonicalUtcDatetime(value: unknown): unknown { - if (value == null) return value; - if (value instanceof Date) { - return Number.isNaN(value.getTime()) ? value : value.toISOString(); - } - if (typeof value === 'number') { - if (!Number.isFinite(value)) return value; - const d = new Date(value); - return Number.isNaN(d.getTime()) ? value : d.toISOString(); - } - if (typeof value !== 'string') return value; - const s = value.trim(); - if (s === '') return value; - // A bare integer (in either JS or string form) is epoch milliseconds — the - // shape better-sqlite3 wrote for every `Date` bound before this convention. - if (/^-?\d+$/.test(s)) { - const d = new Date(Number(s)); - return Number.isNaN(d.getTime()) ? value : d.toISOString(); - } - // A bare calendar day means midnight UTC. Stated explicitly so it cannot be - // re-read as midnight in the server's local zone — the Postgres divergence - // where the same query lands a row on a different calendar day. - const iso = /^\d{4}-\d{2}-\d{2}$/.test(s) - ? `${s}T00:00:00.000Z` - // Zone-naive `YYYY-MM-DD[ T]HH:MM[:SS[.fff]]` → its wall-clock IS UTC, the - // same rule `CURRENT_TIMESTAMP`-written rows take on read (ADR-0074). - : /^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}(:\d{2}(\.\d+)?)?$/.test(s) - ? `${s.replace(' ', 'T')}Z` - : s; - const ms = Date.parse(iso); - return Number.isFinite(ms) ? new Date(ms).toISOString() : value; + // [#20176] The rule is `@objectstack/core`'s — the same function + // `driver-memory` and the engine's per-aggregation `filter` / `having` read. + // This body was a word-for-word copy of it until the rule was lifted there. + return temporalStorageForm(value, 'datetime'); } /** @@ -618,27 +592,10 @@ function mysqlDatetimeLiteral(canonical: unknown): unknown { * interpret is never silently rewritten. */ function canonicalTimeOfDay(value: unknown): unknown { - if (value == null) return value; - if (typeof value === 'string') { - const s = value.trim(); - if (s === '') return value; - const m = /^(\d{2}):(\d{2})(?::(\d{2})(?:\.(\d+))?)?$/.exec(s); - if (m) { - const [, hh, mm, ss = '00', frac] = m; - if (Number(hh) > 23 || Number(mm) > 59 || Number(ss) > 59) return value; - const ms = frac ? `${frac}000`.slice(0, 3) : '000'; - return ms === '000' ? `${hh}:${mm}:${ss}` : `${hh}:${mm}:${ss}.${ms}`; - } - } - // Everything that is not a bare time-of-day — `Date`, epoch ms, full ISO or - // zone-naive timestamp strings — is an instant: delegate its interpretation - // to the ONE function that owns instants, then keep the UTC time-of-day. - const instant = canonicalUtcDatetime(value); - if (typeof instant === 'string' && /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/.test(instant)) { - const time = instant.slice(11, 23); - return time.endsWith('.000') ? time.slice(0, 8) : time; - } - return value; + // [#20176] The rule is `@objectstack/core`'s — see {@link canonicalUtcDatetime}. + // A value that is not a bare time-of-day is read as an instant by the + // `datetime` rule there and keeps its UTC time-of-day, as it did here. + return temporalStorageForm(value, 'time'); } /** @@ -14704,23 +14661,14 @@ export class SqlDriver implements IDataDriver { * {@link withPostgresCalendarDayAsText} — so on every dialect a `date` * column now arrives here as TEXT and no driver-materialised `Date` reaches * this helper at all. ⛔ Do not "repair" a residual date skew by switching - * the getters below to their local twins; that reintroduces #11389 in the + * the rule's UTC getters to their local twins; that reintroduces #11389 in the * mirror direction, and `sql-driver-11389-date-tz-skew.test.ts` pins it. */ protected toDateOnly(value: any): any { - if (value == null) return value; - if (value instanceof Date) { - if (Number.isNaN(value.getTime())) return value; - const y = value.getUTCFullYear(); - const m = String(value.getUTCMonth() + 1).padStart(2, '0'); - const d = String(value.getUTCDate()).padStart(2, '0'); - return `${y}-${m}-${d}`; - } - if (typeof value === 'string') { - const trimmed = value.trim(); - if (/^\d{4}-\d{2}-\d{2}/.test(trimmed)) return trimmed.slice(0, 10); - } - return value; + // [#20176] The rule — UTC getters included — is `@objectstack/core`'s + // `temporalStorageForm`, the function `driver-memory` and the engine's + // per-aggregation `filter` / `having` read too; this body was a copy of it. + return temporalStorageForm(value, 'date'); } /** diff --git a/packages/objectql/src/engine.ts b/packages/objectql/src/engine.ts index 4e1ebbf2a4a..106083bc8a1 100644 --- a/packages/objectql/src/engine.ts +++ b/packages/objectql/src/engine.ts @@ -16242,20 +16242,27 @@ export class ObjectQL implements IObjectQLEngine { // (having-filter.ts `compareWithReference`) on both doors below. assertHavingIsFilterCondition(query.having); assertListComparandShapes(object, 'aggregate', query.having, 'having'); + // [#20176] The object's declaration, read once: the per-aggregation + // `filter` reads a temporal comparand by its column's storage rule + // (`applyInMemoryAggregation` below), and `having` by its aggregated + // column's — the rule the drivers apply to the same comparand in a + // `where`. Before, both positions compared it as written, so an ISO + // instant against a `date` field counted 1 row where the `where` twin + // counted 3, over REST, on driver-memory and driver-sql alike. + const declaredFields = (this._registry.getObject(object) as { fields?: Record } | undefined)?.fields; + // [#20127] …and each aggregated column's class, read off the query and the + // object's declaration, so a `{ $field, addDays }` pair is judged by the + // rule `FieldReferenceSchema.addDays` declares (two temporal columns of + // one class) rather than answered by epoch-ms coercion. [#20176] Kept for + // both `applyHaving` doors below, which read a temporal column's + // comparands by its storage rule. + const havingColumnClasses = aggregatedRowColumnClasses(query.groupBy, query.aggregations, declaredFields); { const having = normalizeFilterComparandTypes(query.having, `aggregate('${object}')`, 'having'); - // [#20127] …and each column's class, read off the query and the - // object's declaration, so a `{ $field, addDays }` pair is judged by - // the rule `FieldReferenceSchema.addDays` declares (two temporal - // columns of one class) rather than answered by epoch-ms coercion. assertHavingIsEvaluable( having, aggregatedRowColumns(query.groupBy, query.aggregations), - aggregatedRowColumnClasses( - query.groupBy, - query.aggregations, - (this._registry.getObject(object) as { fields?: Record } | undefined)?.fields, - ), + havingColumnClasses, ); if (having !== query.having) query = { ...query, having }; } @@ -16359,7 +16366,7 @@ export class ObjectQL implements IObjectQLEngine { // capability flag, at which point this post-filter becomes the // fallback tier — the dateGranularity two-tier pattern. const aggregated = await drv.aggregate(object, ast, this.buildDriverOptions(object, opCtx.context)); - return applyHaving(aggregated, ast.having); + return applyHaving(aggregated, ast.having, havingColumnClasses); } // In-memory fallback path: ask the driver for raw rows, then bucket + // aggregate here. This guarantees `groupBy` (incl. structured items @@ -16403,7 +16410,7 @@ export class ObjectQL implements IObjectQLEngine { delete rowsAst.aggregations; delete rowsAst.having; const raw = await driver.find(object, rowsAst, this.buildDriverOptions(object, opCtx.context)); - return applyHaving(applyInMemoryAggregation(raw, ast, tz), ast.having); + return applyHaving(applyInMemoryAggregation(raw, ast, tz, declaredFields), ast.having, havingColumnClasses); }); return opCtx.result as any[]; diff --git a/packages/objectql/src/having-filter.ts b/packages/objectql/src/having-filter.ts index ab6ba1df005..77400359cc9 100644 --- a/packages/objectql/src/having-filter.ts +++ b/packages/objectql/src/having-filter.ts @@ -100,6 +100,17 @@ // for the same comparison ({@link assertAggregationFilterReferencesAreDeclared}). // And on both positions a `Date` bound is compared as an instant // ({@link instantsOf}), as the same bound in a `where` is. +// +// [#20176] …and, where the column's CLASS is known, every temporal comparand is +// compared by that column's storage rule — the rule the drivers apply to the +// same comparand in a `where`, `@objectstack/core`'s `temporalStorageForm`, +// with ADR-0053 D-D's whole-day reading of a bare-day upper bound on a +// `datetime` column. The class comes from the object's declaration for the +// per-aggregation `filter` (`declaredFieldClasses`) and from the query for +// `having` (`aggregatedRowColumnClasses`, #20127's rule). See +// {@link checkCondition}. Before, both positions compared a temporal comparand +// as written, so an ISO instant against a `date` column counted 1 row where the +// same condition in a `where` counted 3. import type { FilterCondition } from '@objectstack/spec/data'; // [#20099] The reference's own declaration, so a malformed `addDays` is refused @@ -132,6 +143,12 @@ import { asciiCaseInsensitiveContains } from '@objectstack/spec/data'; // evaluator — the lift `@objectstack/formula` applies to the same pairing — so a // `Date` bound here compares the way the same bound in a `where` does. import { utcInstantMs } from '@objectstack/spec/data'; +// [#20176] The storage rule a temporal column puts a value in — ONE function, +// shared with `driver-sql`'s and `driver-memory`'s `where` — and the whole-day +// reading of a bare-day upper bound on a `datetime` column (ADR-0053 D-D), from +// the spec, where that rule is declared. +import { temporalStorageForm, type TemporalComparandKind } from '@objectstack/core'; +import { nextUtcCalendarDay } from '@objectstack/spec/data'; // [#7047] The ADR-0112 envelope this face's refusals used to omit. Shared with // `filter-comparand-shape.ts` rather than re-declared here — see the note on // {@link invalidFilterError} and on {@link unknownOperator} below. @@ -684,6 +701,33 @@ export function aggregatedRowColumnClasses( return classes; } +/** + * [#20176] Each declared field's class — what a per-aggregation `filter` reads + * a comparand against, since that filter narrows the object's RAW rows. The + * same classification {@link aggregatedRowColumnClasses} gives a groupBy + * projection of the field, and the same three temporal types the drivers index + * for their `where` (`driver-memory`'s `indexTemporalFields`, + * `SqlDriver.temporalFieldKind`), so this position and the driver's `where` + * read one field by one rule. No usable field map (a registry-less host) ⇒ an + * empty map, and every comparand is compared as written. + */ +export function declaredFieldClasses(fields: unknown): Map { + const classes = new Map(); + if (!fields || typeof fields !== 'object' || Array.isArray(fields)) return classes; + const map = fields as Record; + for (const name of Object.keys(map)) classes.set(name, declaredFieldClass(map, name)); + return classes; +} + +/** + * [#20176] The storage rule a column of `cls` takes, or `undefined` for a class + * that has none (numeric, text, boolean) and for a column whose class the + * declaration cannot tell. + */ +function temporalKindOf(cls: AggregatedColumnClass | undefined): TemporalComparandKind | undefined { + return cls === 'date' || cls === 'datetime' || cls === 'time' ? cls : undefined; +} + /** * [#20099] Refuse a `having` that is not a filter condition at all: anything but * `null` / `undefined` (no clause) and a plain object. See @@ -1077,10 +1121,18 @@ function compareWithReference( * Filter aggregated rows by the query's `having` condition. An absent or empty * condition returns the rows unchanged (same vacuous-filter convention as * `where`). + * + * [#20176] `classes` is each aggregated column's class + * ({@link aggregatedRowColumnClasses}): a temporal column's comparands are + * compared by its storage rule. Absent ⇒ every comparand as written. */ -export function applyHaving(rows: any[], having: FilterCondition | null | undefined): any[] { +export function applyHaving( + rows: any[], + having: FilterCondition | null | undefined, + classes?: ReadonlyMap, +): any[] { if (!having || typeof having !== 'object' || Object.keys(having).length === 0) return rows; - return rows.filter((row) => matchesHaving(row, having)); + return rows.filter((row) => matchesHaving(row, having, 'having', HAVING_CLAUSE, classes)); } /** @@ -1090,28 +1142,35 @@ export function applyHaving(rows: any[], having: FilterCondition | null | undefi * `having`, `having.$and[0]`, `having.$not` — carried down so a comparand * refusal can NAME where the offending key sits. It defaults, so this stays the * two-argument function every existing caller (and `applyHaving` below) uses. + * + * [#20176] `classes` maps a column to its class — the aggregated row's for + * `having`, the object's declared fields for a per-aggregation `filter` — and a + * column whose class is temporal has its comparands compared by that column's + * storage rule ({@link checkCondition}). Absent, or a column it does not name ⇒ + * compared as written. */ export function matchesHaving( row: Record, cond: any, path = 'having', clause: FilterClause = HAVING_CLAUSE, + classes?: ReadonlyMap, ): boolean { if (!cond || typeof cond !== 'object') return true; for (const [key, value] of Object.entries(cond)) { const here = `${path}.${key}`; if (key === '$and') { const branches = Array.isArray(value) ? value : [value]; - if (!branches.every((c, i) => matchesHaving(row, c, `${here}[${i}]`, clause))) return false; + if (!branches.every((c, i) => matchesHaving(row, c, `${here}[${i}]`, clause, classes))) return false; continue; } if (key === '$or') { const branches = Array.isArray(value) ? value : [value]; - if (!branches.some((c, i) => matchesHaving(row, c, `${here}[${i}]`, clause))) return false; + if (!branches.some((c, i) => matchesHaving(row, c, `${here}[${i}]`, clause, classes))) return false; continue; } if (key === '$not') { - if (matchesHaving(row, value, here, clause)) return false; + if (matchesHaving(row, value, here, clause, classes)) return false; continue; } if (key.startsWith('$')) throw unknownOperator(key, 'logical', [], clause); @@ -1119,8 +1178,8 @@ export function matchesHaving( // no dotted-path resolution. [#10576] The per-aggregation filter walks the // same way on purpose: it reads `driver.find()` rows, which are flat too. // [#20099] The row itself goes down too: a `{ $field }` comparand resolves - // against it. - if (!checkCondition(row?.[key], value, key, here, clause, row)) return false; + // against it. [#20176] …and the column's storage rule, when it has one. + if (!checkCondition(row?.[key], value, key, here, clause, row, temporalKindOf(classes?.get(key)))) return false; } return true; } @@ -1139,14 +1198,19 @@ export function matchesHaving( * unknown operator here is refused naming the aggregation position it sits in, * because ignoring it would silently answer the UNFILTERED aggregate — the * precise #10413 defect this key exists to close. + * + * [#20176] `classes` is the object's declared field classes + * ({@link declaredFieldClasses}), so a temporal comparand here is read by the + * column's storage rule, as the same comparand in a `where` is by the driver. */ export function matchesAggregationFilter( row: Record, filter: FilterCondition, index: number, + classes?: ReadonlyMap, ): boolean { const clause = aggregationFilterClause(index); - return matchesHaving(row, filter, clause.root, clause); + return matchesHaving(row, filter, clause.root, clause, classes); } /** @@ -1162,8 +1226,10 @@ export function matchesAggregationFilter( * per-aggregation `filter` (and kept no group in `having`), `$ne` / `$nin` * counted every row, and a `$between` of two `Date`s kept every row — while the * same bound in a `where` answered 4 of 6 on driver-memory and driver-sql - * alike, each driver reading it by the column's storage rule. This walker is - * TYPE-BLIND (it holds a flat row, no declaration), so it reads the pair the + * alike, each driver reading it by the column's storage rule. Where this walker + * is TYPE-BLIND — [#20176] a column whose class it is not handed, or whose class + * is not temporal; a temporal column's pair is put in its storage form first + * ({@link checkCondition}), and no `Date` is left in it — it reads the pair the * way the spec defines for a type-blind evaluator — {@link utcInstantMs}, the * lift `@objectstack/formula`'s evaluator applies to the same pairing: a * `Date`, epoch milliseconds, a bare `YYYY-MM-DD` (its UTC midnight) or an @@ -1198,7 +1264,41 @@ function listHolds(list: readonly unknown[], value: unknown): boolean { }); } -/** One column's condition — implicit equality or an operator object. */ +/** + * [#20176] ADR-0053 D-D: a bare `YYYY-MM-DD` as the UPPER bound of a + * `datetime` column (`$lte`, a `$between` max) means that WHOLE day — the + * exclusive bound at the next day's midnight, in the column's storage form. + * `undefined` when the rule does not apply: another class, or a bound that is + * not a bare calendar day (a full timestamp and a `Date` keep instant + * semantics). The same decision both drivers' `where` emitters take + * (`SqlDriver.calendarDayUpperBoundRewrite`, `driver-memory`'s `$lte` arm), + * read from the spec's `nextUtcCalendarDay`. + */ +function wholeDayUpperBound(bound: unknown, kind: TemporalComparandKind | undefined): unknown { + if (kind !== 'datetime') return undefined; + const next = nextUtcCalendarDay(bound); + return next === null ? undefined : temporalStorageForm(next, 'datetime'); +} + +/** + * One column's condition — implicit equality or an operator object. + * + * [#20176] `kind` is the column's temporal storage rule, when the caller knows + * its class and the class is `date`, `datetime` or `time`. Then the row's value + * AND every comparand of `$eq` / `$ne` / the four orderings / `$between` / + * `$in` / `$nin` / implicit equality are put in that rule's storage form + * (`temporalStorageForm`) before they are compared, and a bare-day upper bound + * on a `datetime` column reads as the whole day ({@link wholeDayUpperBound}) — + * the reading the drivers give the same comparand in a `where`. So + * `'2026-02-01T00:00:00.000Z'` against a `date` column is the day + * `'2026-02-01'`, `'2026-02-01'` as a `datetime` `$lte` includes that day's + * rows, epoch milliseconds are an instant, and a `Date` against a `date` or + * `time` column is its UTC day or time of day. The value takes the form too + * because that is the pairing the drivers compare (`driver-sql` wraps a legacy + * SQLite column in the same canon); rows a driver returns are in it already. + * Presence (`$exists`, `$null`), the text operators and a `{ $field }` + * reference are not comparands of a value, and are read as before. + */ function checkCondition( value: any, condition: any, @@ -1206,17 +1306,20 @@ function checkCondition( path: string, clause: FilterClause = HAVING_CLAUSE, row: Record = {}, + kind?: TemporalComparandKind, ): boolean { + const form = (operand: unknown): unknown => (kind === undefined ? operand : temporalStorageForm(operand, kind)); // Implicit equality (primitives, null, Date, array exact-match) — loose `==` // to mirror the Filter Protocol's memory evaluation. [#20148] A `Date` bound - // is compared as an instant ({@link instantsOf}). + // is compared as an instant ({@link instantsOf}). [#20176] On a temporal + // column, both sides in the column's storage form first. if ( typeof condition !== 'object' || condition === null || condition instanceof Date || Array.isArray(condition) ) { - return comparandEquals(value, condition); + return comparandEquals(form(value), form(condition)); } const keys = Object.keys(condition); @@ -1255,20 +1358,34 @@ function checkCondition( } // [#20148] The comparison and list arms read a `Date` bound — or a `Date` // value — as an instant ({@link instantsOf}); every other pair compares - // exactly as before. + // exactly as before. [#20176] On a temporal column both sides are in its + // storage form first (`form`), and a bare-day upper bound on a `datetime` + // column is the whole day ({@link wholeDayUpperBound}). + const stored = form(value); switch (op) { - case '$eq': if (!comparandEquals(value, target)) return false; break; - case '$ne': if (comparandEquals(value, target)) return false; break; - case '$gt': if (!ordered(value, target, (a, b) => a > b)) return false; break; - case '$gte': if (!ordered(value, target, (a, b) => a >= b)) return false; break; - case '$lt': if (!ordered(value, target, (a, b) => a < b)) return false; break; - case '$lte': if (!ordered(value, target, (a, b) => a <= b)) return false; break; - case '$between': - if (Array.isArray(target) - && (ordered(value, target[0], (a, b) => a < b) || ordered(value, target[1], (a, b) => a > b))) return false; + case '$eq': if (!comparandEquals(stored, form(target))) return false; break; + case '$ne': if (comparandEquals(stored, form(target))) return false; break; + case '$gt': if (!ordered(stored, form(target), (a, b) => a > b)) return false; break; + case '$gte': if (!ordered(stored, form(target), (a, b) => a >= b)) return false; break; + case '$lt': if (!ordered(stored, form(target), (a, b) => a < b)) return false; break; + case '$lte': { + const dayAfter = wholeDayUpperBound(target, kind); + if (dayAfter !== undefined + ? !ordered(stored, dayAfter, (a, b) => a < b) + : !ordered(stored, form(target), (a, b) => a <= b)) return false; break; - case '$in': if (!Array.isArray(target) || !listHolds(target, value)) return false; break; - case '$nin': if (Array.isArray(target) && listHolds(target, value)) return false; break; + } + case '$between': { + if (!Array.isArray(target)) break; + const dayAfter = wholeDayUpperBound(target[1], kind); + if (ordered(stored, form(target[0]), (a, b) => a < b) + || (dayAfter !== undefined + ? ordered(stored, dayAfter, (a, b) => a >= b) + : ordered(stored, form(target[1]), (a, b) => a > b))) return false; + break; + } + case '$in': if (!Array.isArray(target) || !listHolds(target.map(form), stored)) return false; break; + case '$nin': if (Array.isArray(target) && listHolds(target.map(form), stored)) return false; break; case '$exists': { const exists = value !== undefined && value !== null; if (exists !== !!target) return false; diff --git a/packages/objectql/src/in-memory-aggregation.ts b/packages/objectql/src/in-memory-aggregation.ts index 0d522f08004..b1ebc47def0 100644 --- a/packages/objectql/src/in-memory-aggregation.ts +++ b/packages/objectql/src/in-memory-aggregation.ts @@ -86,7 +86,7 @@ import { bucketDateKey } from '@objectstack/core'; import type { QueryAST, GroupByNode, AggregationNode, DateGranularityValue } from '@objectstack/spec/data'; -import { matchesAggregationFilter } from './having-filter.js'; +import { declaredFieldClasses, matchesAggregationFilter } from './having-filter.js'; /** * Group + aggregate raw rows according to the AST's `groupBy` / @@ -96,19 +96,30 @@ import { matchesAggregationFilter } from './having-filter.js'; * so a row near a tz day-boundary lands in the right day/week/month/quarter. * It is only consulted by `groupBy` items carrying a `dateGranularity`; an * unset or `'UTC'` value keeps the historical UTC bucketing. + * + * [#20176] `fields` is the object's declared field map. A per-aggregation + * `filter` reads a temporal comparand by the column's storage rule when it is + * handed one — the rule the driver applies to the same comparand in a `where` + * (having-filter.ts `checkCondition`). Absent (a registry-less caller) ⇒ every + * comparand is compared as written, as before. */ export function applyInMemoryAggregation( rows: any[], ast: Pick, timezone?: string, + fields?: Record, ): any[] { const groupBy = (ast.groupBy ?? []) as GroupByNode[]; const aggregations = (ast.aggregations ?? []) as AggregationNode[]; if (groupBy.length === 0 && aggregations.length === 0) return rows; + // [#20176] Read once per call, and only when some aggregation carries a filter. + const filterClasses = fields && aggregations.some((a) => a?.filter && Object.keys(a.filter).length > 0) + ? declaredFieldClasses(fields) + : undefined; if (groupBy.length === 0) { // Pure aggregation — single result row. - return [aggregateBucket(rows, aggregations)]; + return [aggregateBucket(rows, aggregations, filterClasses)]; } const buckets = new Map; rows: any[] }>(); @@ -136,7 +147,7 @@ export function applyInMemoryAggregation( const out: any[] = []; for (const { key, rows: bucketRows } of buckets.values()) { - const aggValues = aggregateBucket(bucketRows, aggregations); + const aggValues = aggregateBucket(bucketRows, aggregations, filterClasses); out.push({ ...key, ...aggValues }); } return out; @@ -167,7 +178,11 @@ function projectGroupValue(row: any, g: GroupByNode, timezone?: string): unknown return v ?? null; } -function aggregateBucket(allRows: any[], aggregations: AggregationNode[]): Record { +function aggregateBucket( + allRows: any[], + aggregations: AggregationNode[], + filterClasses?: ReturnType, +): Record { const out: Record = {}; for (const [index, agg] of aggregations.entries()) { const alias = agg.alias; @@ -183,7 +198,7 @@ function aggregateBucket(allRows: any[], aggregations: AggregationNode[]): Recor // still lists, objectui#3136). const aggFilter = agg.filter; const rows = aggFilter && Object.keys(aggFilter).length > 0 - ? allRows.filter((row) => matchesAggregationFilter(row, aggFilter, index)) + ? allRows.filter((row) => matchesAggregationFilter(row, aggFilter, index, filterClasses)) : allRows; if (fn === 'count') { // `*` is the count-all sentinel: the Cube `count` measure and a dataset From 9e5c30909882656e437efdf10239879511cc1663 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 07:30:31 +0000 Subject: [PATCH 2/3] test(objectql,rest,core,driver-sql,driver-memory): pin the card's rows, the family and the temporal conformance kit through a per-aggregation filter and having; changesets Claude-Session: https://claude.ai/code/session_01Bvd69VPa6puiNzzPUroDBx Co-authored-by: Claude --- .../20148-aggregation-filter-where-doors.md | 2 +- ...20176-aggregation-temporal-storage-rule.md | 35 ++ .../src/utils/temporal-storage-form.test.ts | 89 +++++ .../src/memory-temporal-storage-form.test.ts | 108 ++++++ .../sql-driver-temporal-storage-form.test.ts | 86 +++++ ...ne-aggregate-temporal-storage-rule.test.ts | 326 ++++++++++++++++++ ...ation-filter-temporal-storage-rule.test.ts | 162 +++++++++ 7 files changed, 807 insertions(+), 1 deletion(-) create mode 100644 .changeset/20176-aggregation-temporal-storage-rule.md create mode 100644 packages/core/src/utils/temporal-storage-form.test.ts create mode 100644 packages/drivers/driver-memory/src/memory-temporal-storage-form.test.ts create mode 100644 packages/drivers/driver-sql/src/sql-driver-temporal-storage-form.test.ts create mode 100644 packages/objectql/src/engine-aggregate-temporal-storage-rule.test.ts create mode 100644 packages/rest/src/aggregation-filter-temporal-storage-rule.test.ts diff --git a/.changeset/20148-aggregation-filter-where-doors.md b/.changeset/20148-aggregation-filter-where-doors.md index 2e8d050ab96..fac395f4a5a 100644 --- a/.changeset/20148-aggregation-filter-where-doors.md +++ b/.changeset/20148-aggregation-filter-where-doors.md @@ -24,6 +24,6 @@ These refusals run after the walker's own (an unknown operator, a malformed refe Not refused, but answering differently: -- **A `Date` bound is compared as an instant.** `{ opened_at: { $gt: new Date('2026-02-01') } }` against the ISO text a `datetime` column holds counted no row: JS compared the `Date` with the string by coercion. `$ne` and `$nin` counted every row, and a `$between` of two `Date`s counted every row. The same bound in a `where` counted 4 of 6 on both drivers. The comparison now reads the pair through `@objectstack/spec/data`'s `utcInstantMs`, the lift `@objectstack/formula`'s evaluator applies, whenever one side is a `Date` and both sides denote an instant. Every `datetime` row and a UTC-midnight `Date` on a `date` field now count what the same bound counts in a `where`. `having` shares this comparison, so a `Date` bound in `having` keeps the groups its ISO spelling keeps on a `datetime` column; on a `date`-class column (`min` / `max` of a `date` field) a UTC-midnight `Date` follows the calendar-day reading a `where` gives (`{ $gte: new Date('2026-02-01') }` keeps the group whose max is `2026-02-01`, and so does `$eq`), which the ISO-instant text, compared as text, did not. A `Date` is not JSON, so this reaches in-process callers only. Two readings still differ from a `where`, because the per-row comparison holds no declaration: a `Date` carrying a time of day against a `date` field, which a `where` reads as that UTC calendar day, and a `Date` against a `time` field, which is not an instant and is left as before. +- **A `Date` bound is compared as an instant.** `{ opened_at: { $gt: new Date('2026-02-01') } }` against the ISO text a `datetime` column holds counted no row: JS compared the `Date` with the string by coercion. `$ne` and `$nin` counted every row, and a `$between` of two `Date`s counted every row. The same bound in a `where` counted 4 of 6 on both drivers. The comparison now reads the pair through `@objectstack/spec/data`'s `utcInstantMs`, the lift `@objectstack/formula`'s evaluator applies, whenever one side is a `Date` and both sides denote an instant. Every `datetime` row and a UTC-midnight `Date` on a `date` field now count what the same bound counts in a `where`. `having` shares this comparison, so a `Date` bound in `having` keeps the groups its ISO spelling keeps on a `datetime` column; on a `date`-class column (`min` / `max` of a `date` field) a UTC-midnight `Date` follows the calendar-day reading a `where` gives (`{ $gte: new Date('2026-02-01') }` keeps the group whose max is `2026-02-01`, and so does `$eq`), which the ISO-instant text, compared as text, did not. A `Date` is not JSON, so this reaches in-process callers only. Two readings still differed from a `where` in this change alone, because the per-row comparison held no declaration (#20176 closes both in the same release, reading every temporal comparand at this position by the column's storage rule): a `Date` carrying a time of day against a `date` field, which a `where` reads as that UTC calendar day, and a `Date` against a `time` field, which is not an instant and is left as before. Not changed, measured identical before and after on both drivers: every `where`, `groupBy` and `having` answer outside the `Date` shapes above, and a per-aggregation filter that uses implicit equality, ordering bounds, `$in`, `$between`, `$or`, `{}`, a `{ $field }` between two declared numeric fields, `addDays` between two `date` or two `datetime` fields (literal or read from a numeric column), and a reference to `created_at` or `id`. diff --git a/.changeset/20176-aggregation-temporal-storage-rule.md b/.changeset/20176-aggregation-temporal-storage-rule.md new file mode 100644 index 00000000000..fa7304ef6d4 --- /dev/null +++ b/.changeset/20176-aggregation-temporal-storage-rule.md @@ -0,0 +1,35 @@ +--- +"@objectstack/core": minor +"@objectstack/objectql": minor +"@objectstack/driver-sql": patch +"@objectstack/driver-memory": patch +--- + +fix(objectql,core): a per-aggregation `filter` and `having` on `engine.aggregate` read a temporal comparand by the column's storage rule, the rule `where` already applies — one function, `temporalStorageForm`, now exported by `@objectstack/core` and shared by both drivers (#20176) + +A per-aggregation `filter` (`aggregations[i].filter`) and `having` are evaluated by the engine itself, over the rows (or aggregated rows) a driver returns. Both compared a temporal comparand exactly as written, while the same condition as a `where` is put into the column's storage form by the driver first. So they counted differently. Measured through `engine.aggregate` and through `POST /data/:object/query`, on `driver-memory` and `driver-sql`, over six rows: + +| in `aggregations[i].filter` (or `having`) | before | now, and the `where` twin | +|:--|:--|:--| +| an ISO instant on a `date` field, `{ placed_on: { $gte: '2026-02-01T00:00:00.000Z' } }` | 1 | 3 | +| the same instant under `$eq` | 0 | 2 | +| a bare day as the upper bound of a `datetime`, `{ opened_at: { $lte: '2026-02-01' } }`, or as a `$between` max | 2 | 3 | +| an epoch-millisecond bound on a `datetime` | 0 | 3 | +| a `Date` carrying a time of day on a `date` field, `$gte` / `$lt` / `$eq` (in-process only) | 1 / 5 / 0 | 3 / 3 / 2 | +| a `Date` on a `time` field (in-process only) | 0 | 3 | +| `having` on `max` of a `date` field with an ISO-instant bound | kept one group | keeps the two groups whose day is on or after it | + +The same holds for `$ne`, `$in` / `$nin` members, `$between` endpoints, implicit equality, an offset instant (`'…T18:00:00+08:00'`), an epoch-millisecond string, a zone-naive `'2026-02-01T10:00'`, and a short wall clock (`'11:00'`) or an ISO instant on a `time` field. On a `having` column, the class comes from the query, as the `addDays` rule already reads it: `min` / `max` take the class of the field they read, a `groupBy` projection takes its field's, and a `day` date bucket is a `date`. + +What the rule does, now in one place: + +- A comparand, and the row's value, are put into the column's storage form: canonical UTC ISO text for `datetime`, `YYYY-MM-DD` for `date`, and `HH:MM:SS` (`.fff` only when non-zero) for `time`. +- A bare `YYYY-MM-DD` used as the upper bound of a `datetime` (`$lte`, a `$between` max) means that whole day, as it does in a `where` (ADR-0053 D-D). On a `date` or `time` column it is not widened. +- A value the rule cannot read is compared as written, and so is every non-temporal column, presence tests (`$exists`, `$null`), the text operators and a `{ $field }` reference. +- An object whose declared fields the engine cannot see keeps the previous comparison. + +`@objectstack/core` exports the rule as `temporalStorageForm(value, kind)`, `kind` being `'datetime' | 'date' | 'time'`. `driver-sql` (`canonicalUtcDatetime`, `toDateOnly`, `canonicalTimeOfDay`) and `driver-memory` (`coerceTemporalValue`) each carried a copy of it; both now call it. The copies agreed on every shape measured when they were lifted, so no `where`, write or read answer of either driver changes. MySQL still binds a `datetime` in its own literal spelling. + +`@objectstack/objectql`'s `applyInMemoryAggregation(rows, ast, timezone?, fields?)` takes the object's declared field map as an optional fourth argument, and a per-aggregation `filter` reads a temporal comparand by the rule only when it is given. Called without it, the function answers as before. + +Not changed, measured identical before and after on both drivers: every `where` answer, every refusal a per-aggregation `filter` or `having` gives, and every per-aggregation `filter` and `having` cell whose column is not temporal. diff --git a/packages/core/src/utils/temporal-storage-form.test.ts b/packages/core/src/utils/temporal-storage-form.test.ts new file mode 100644 index 00000000000..8311d95c2a1 --- /dev/null +++ b/packages/core/src/utils/temporal-storage-form.test.ts @@ -0,0 +1,89 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. +// +// [#20176] `temporalStorageForm` — the ONE storage rule a temporal comparand is +// put in before it is compared: by `driver-sql` and `driver-memory` on `where` +// (and on write), and by `@objectstack/objectql` on a per-aggregation `filter` +// and `having`. Each driver carried a word-for-word copy until it was lifted +// here; the drivers' own suites pin that their `where` coercion IS this +// function (`memory-temporal-storage-form.test.ts`, +// `sql-driver-temporal-storage-form.test.ts`). This file pins the rule itself. + +import { describe, it, expect } from 'vitest'; +import { temporalStorageForm } from './temporal-storage-form.js'; + +describe('temporalStorageForm — datetime: canonical UTC ISO text', () => { + const cases: ReadonlyArray = [ + ['a Date', new Date('2026-02-01T10:00:00.000Z'), '2026-02-01T10:00:00.000Z'], + ['epoch milliseconds', 1769940000000, '2026-02-01T10:00:00.000Z'], + ['an epoch-millisecond string, padded', ' 1769940000000 ', '2026-02-01T10:00:00.000Z'], + ['a negative epoch string', '-86400000', '1969-12-31T00:00:00.000Z'], + ['a bare day — midnight UTC, never the host zone', '2026-02-01', '2026-02-01T00:00:00.000Z'], + ['a zone-naive wall clock — read AS UTC', '2026-02-01 09:00', '2026-02-01T09:00:00.000Z'], + ['a zone-naive T spelling with a long fraction', '2026-02-01T09:00:00.123456', '2026-02-01T09:00:00.123Z'], + ['an ISO instant without milliseconds', '2026-02-01T10:00:00Z', '2026-02-01T10:00:00.000Z'], + ['an offset instant — re-spelled in Z', '2026-02-01T18:00:00+08:00', '2026-02-01T10:00:00.000Z'], + ]; + for (const [name, input, expected] of cases) { + it(name, () => expect(temporalStorageForm(input, 'datetime')).toBe(expected)); + } +}); + +describe('temporalStorageForm — date: the UTC calendar day', () => { + const cases: ReadonlyArray = [ + ['an ISO instant — its leading day', '2026-02-01T00:00:00.000Z', '2026-02-01'], + ['a Date carrying a time of day — its UTC day', new Date('2026-02-01T23:30:00.000Z'), '2026-02-01'], + ['a bare day, padded', ' 2026-02-01 ', '2026-02-01'], + ['a zone-naive timestamp', '2026-02-01 23:59', '2026-02-01'], + ]; + for (const [name, input, expected] of cases) { + it(name, () => expect(temporalStorageForm(input, 'date')).toBe(expected)); + } + + it('epoch milliseconds are not a calendar day — returned as they came', () => { + expect(temporalStorageForm(1769940000000, 'date')).toBe(1769940000000); + }); +}); + +describe('temporalStorageForm — time: the UTC wall clock, .fff only when non-zero', () => { + const cases: ReadonlyArray = [ + ['a short wall clock', '09:00', '09:00:00'], + ['a zero-millisecond suffix is trimmed', '09:00:00.000', '09:00:00'], + ['a fraction is kept to milliseconds', '09:00:00.5', '09:00:00.500'], + ['a Date — its UTC time of day', new Date('2026-02-01T11:00:00.000Z'), '11:00:00'], + ['an ISO instant — its UTC time of day', '2026-02-01T11:00:00.250Z', '11:00:00.250'], + ['a bare day — its midnight', '2026-02-01', '00:00:00'], + ]; + for (const [name, input, expected] of cases) { + it(name, () => expect(temporalStorageForm(input, 'time')).toBe(expected)); + } +}); + +describe('temporalStorageForm — total: what the rule cannot read comes back unchanged', () => { + const invalid = new Date('nope'); + const untouched: ReadonlyArray = [ + ['null', null], + ['undefined', undefined], + ['the empty string', ''], + ['whitespace', ' '], + ['junk', 'not-a-date'], + ['a preset name', 'last_30_days'], + ['a placeholder', '{today}'], + ['a boolean', true], + ['an Invalid Date', invalid], + ['NaN', Number.NaN], + ]; + for (const kind of ['datetime', 'date', 'time'] as const) { + for (const [name, input] of untouched) { + it(`${kind}: ${name}`, () => expect(temporalStorageForm(input, kind)).toBe(input)); + } + } + + it('time: an out-of-range wall clock is not wrapped', () => { + expect(temporalStorageForm('25:00', 'time')).toBe('25:00'); + }); + + it('a list is NOT mapped — a caller comparing a list maps its members', () => { + const list = ['2026-02-01', '2026-02-02']; + expect(temporalStorageForm(list, 'datetime')).toBe(list); + }); +}); diff --git a/packages/drivers/driver-memory/src/memory-temporal-storage-form.test.ts b/packages/drivers/driver-memory/src/memory-temporal-storage-form.test.ts new file mode 100644 index 00000000000..3a2aeeeb45e --- /dev/null +++ b/packages/drivers/driver-memory/src/memory-temporal-storage-form.test.ts @@ -0,0 +1,108 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#20176] This driver's `where` reads a temporal comparand by the ONE storage + * rule — `@objectstack/core`'s `temporalStorageForm` — that `driver-sql` and the + * engine's per-aggregation `filter` / `having` read too. + * + * Two halves: + * + * 1. `coerceTemporalValue` IS that rule, shape for shape, with a list mapped + * member by member. This file used to carry its own copy + * (`storageDatetimeValue` / `storageDateValue` / `storageTimeValue`); the + * copy and `driver-sql`'s agreed on every shape measured when the rule was + * lifted, so the re-pointing moved no answer. + * 2. The `where` answers the engine's per-aggregation `filter` is now held to — + * the #20176 card's rows on its fixture. `@objectstack/objectql`'s + * `engine-aggregate-temporal-storage-rule.test.ts` asserts the same numbers + * for the per-aggregation position; this is the control half, on this + * driver (`driver-sql`'s twin is the REST suite + * `aggregation-filter-temporal-storage-rule.test.ts`). + */ + +import { describe, it, expect, beforeAll } from 'vitest'; +import { temporalStorageForm } from '@objectstack/core'; +import { coerceTemporalValue } from './memory-temporal.js'; +import { InMemoryDriver } from './memory-driver.js'; + +const SHAPES: ReadonlyArray unknown]> = [ + ['Date', () => new Date('2026-02-01T10:00:00.000Z')], + ['Invalid Date', () => new Date('nope')], + ['epoch ms', () => 1769940000000], + ['epoch ms string', () => '1769940000000'], + ['bare day', () => '2026-02-01'], + ['zone-naive', () => '2026-02-01 09:00'], + ['ISO instant', () => '2026-02-01T10:00:00Z'], + ['offset instant', () => '2026-02-01T18:00:00+08:00'], + ['wall clock', () => '09:00'], + ['wall clock .5', () => '09:00:00.5'], + ['out-of-range wall clock', () => '25:00'], + ['junk', () => 'not-a-date'], + ['empty', () => ''], + ['null', () => null], + ['boolean', () => true], +]; + +describe('[#20176] coerceTemporalValue is core\'s temporalStorageForm', () => { + for (const kind of ['datetime', 'date', 'time'] as const) { + for (const [name, make] of SHAPES) { + it(`${kind}: ${name}`, () => { + expect(coerceTemporalValue(make(), kind)).toStrictEqual(temporalStorageForm(make(), kind)); + }); + } + it(`${kind}: a list comparand is mapped member by member`, () => { + const list = SHAPES.map(([, make]) => make()); + expect(coerceTemporalValue(list, kind)).toStrictEqual(list.map((v) => temporalStorageForm(v, kind))); + }); + } + + it('a non-temporal field passes through untouched', () => { + const value = '2026-02-01T10:00:00Z'; + expect(coerceTemporalValue(value, undefined)).toBe(value); + }); +}); + +const OBJECT = 'ledger_order'; +const ROWS = [ + { id: 'o1', placed_on: '2026-01-10', opened_at: '2026-01-01T10:00:00.000Z', slot: '09:00:00' }, + { id: 'o2', placed_on: '2026-01-02', opened_at: '2026-01-02T10:00:00.000Z', slot: '10:30:00' }, + { id: 'o3', placed_on: '2026-03-01', opened_at: '2026-02-01T10:00:00.000Z', slot: '11:00:00' }, + { id: 'o4', placed_on: '2026-02-01', opened_at: '2026-02-05T10:00:00.000Z', slot: '12:00:00' }, + { id: 'o5', placed_on: '2026-01-15', opened_at: '2026-02-06T10:00:00.000Z', slot: '13:00:00' }, + { id: 'o6', placed_on: '2026-02-01', opened_at: '2026-03-01T10:00:00.000Z', slot: '14:00:00' }, +]; + +/** The #20176 card's rows, as a `where`, with the count this driver answers. */ +const CARD_WHERE_TWINS: ReadonlyArray Record, number]> = [ + ['row 1 — an ISO instant $gte on a date field', () => ({ placed_on: { $gte: '2026-02-01T00:00:00.000Z' } }), 3], + ['row 2 — an ISO instant $eq on a date field', () => ({ placed_on: { $eq: '2026-02-01T00:00:00.000Z' } }), 2], + ['row 3 — a bare day as the $lte of a datetime', () => ({ opened_at: { $lte: '2026-02-01' } }), 3], + ['row 4 — a bare day as the $between max of a datetime', () => ({ opened_at: { $between: ['2026-01-01', '2026-02-01'] } }), 3], + ['row 5 — an epoch-ms bound on a datetime', () => ({ opened_at: { $gt: 1769940000000 } }), 3], + ['row 6 — a Date with a time of day, $gte on a date field', () => ({ placed_on: { $gte: new Date('2026-02-01T10:00:00.000Z') } }), 3], + ['row 6 — the same Date, $lt', () => ({ placed_on: { $lt: new Date('2026-02-01T10:00:00.000Z') } }), 3], + ['row 6 — the same Date, $eq', () => ({ placed_on: { $eq: new Date('2026-02-01T10:00:00.000Z') } }), 2], + ['row 7 — a Date on a time field', () => ({ slot: { $gt: new Date('2026-02-01T11:00:00.000Z') } }), 3], + ['control — a zone-naive datetime string', () => ({ opened_at: { $gt: '2026-02-01 09:00' } }), 4], +]; + +describe('[#20176] the where twins of the card\'s rows, on this driver', () => { + let driver: InMemoryDriver; + + beforeAll(async () => { + driver = new InMemoryDriver({}); + await driver.connect(); + await driver.syncSchema(OBJECT, { + name: OBJECT, + fields: { placed_on: { type: 'date' }, opened_at: { type: 'datetime' }, slot: { type: 'time' } }, + }); + for (const row of ROWS) await driver.create(OBJECT, row); + }); + + for (const [name, where, expected] of CARD_WHERE_TWINS) { + it(`${name}: ${expected} of 6`, async () => { + const rows = await driver.find(OBJECT, { where: where() } as any); + expect(rows).toHaveLength(expected); + }); + } +}); diff --git a/packages/drivers/driver-sql/src/sql-driver-temporal-storage-form.test.ts b/packages/drivers/driver-sql/src/sql-driver-temporal-storage-form.test.ts new file mode 100644 index 00000000000..8c5782da038 --- /dev/null +++ b/packages/drivers/driver-sql/src/sql-driver-temporal-storage-form.test.ts @@ -0,0 +1,86 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#20176] `temporalFilterValue` — this driver's `where` coercion of a temporal + * comparand — IS `@objectstack/core`'s `temporalStorageForm`, the one rule + * `driver-memory` and the engine's per-aggregation `filter` / `having` read + * too, with MySQL's `datetime` then spelled as that dialect's literal. + * + * `canonicalUtcDatetime`, `toDateOnly` and `canonicalTimeOfDay` were a + * word-for-word copy of the rule (and `driver-memory` carried a second); they + * agreed with it on every shape measured when it was lifted, so re-pointing + * them moved no `where`, write or read answer. The where half of the card's + * rows on this driver is pinned through the REST door in + * `packages/rest/src/aggregation-filter-temporal-storage-rule.test.ts`. + * + * No connection is opened: the field-type maps are seeded the way + * `initObjects` would, as in `sql-driver-temporal-dialect.test.ts`. + */ + +import { describe, it, expect } from 'vitest'; +import { temporalStorageForm } from '@objectstack/core'; +import { SqlDriver } from '../src/index.js'; + +class ProbeDriver extends SqlDriver { + seed(kind: 'datetime' | 'date' | 'time', table: string, field: string): void { + const map = kind === 'datetime' ? this.datetimeFields : kind === 'date' ? this.dateFields : this.timeFields; + (map[table] ??= new Set()).add(field); + } +} + +function makeDriver(client: string): ProbeDriver { + const d = new ProbeDriver({ client, connection: { filename: ':memory:' }, useNullAsDefault: true } as any); + d.seed('datetime', 't', 'at'); + d.seed('date', 't', 'on'); + d.seed('time', 't', 'clock'); + return d; +} + +const SHAPES: ReadonlyArray unknown]> = [ + ['Date', () => new Date('2026-02-01T10:00:00.000Z')], + ['Invalid Date', () => new Date('nope')], + ['epoch ms', () => 1769940000000], + ['epoch ms string', () => '1769940000000'], + ['bare day', () => '2026-02-01'], + ['zone-naive', () => '2026-02-01 09:00'], + ['ISO instant', () => '2026-02-01T10:00:00Z'], + ['offset instant', () => '2026-02-01T18:00:00+08:00'], + ['wall clock', () => '09:00'], + ['wall clock .5', () => '09:00:00.5'], + ['out-of-range wall clock', () => '25:00'], + ['junk', () => 'not-a-date'], + ['empty', () => ''], + ['boolean', () => true], +]; + +const FIELD = { datetime: 'at', date: 'on', time: 'clock' } as const; + +/** MySQL's spelling of a canonical instant (#3942) — the dialect half, which stays this driver's. */ +const mysqlLiteral = (v: unknown): unknown => { + const m = typeof v === 'string' ? /^(\d{4}-\d{2}-\d{2})T(\d{2}:\d{2}:\d{2}\.\d{3})Z$/.exec(v) : null; + return m ? `${m[1]} ${m[2]}` : v; +}; + +describe('[#20176] temporalFilterValue is core\'s temporalStorageForm', () => { + for (const client of ['better-sqlite3', 'pg', 'mysql2']) { + for (const kind of ['datetime', 'date', 'time'] as const) { + it(`${client} · ${kind}: every shape, and a list member by member`, () => { + const d = makeDriver(client); + const physical = (v: unknown): unknown => { + const form = temporalStorageForm(v, kind); + return client === 'mysql2' && kind === 'datetime' ? mysqlLiteral(form) : form; + }; + for (const [name, make] of SHAPES) { + expect(d.temporalFilterValue('t', FIELD[kind], make()), name).toStrictEqual(physical(make())); + } + const list = SHAPES.map(([, make]) => make()); + expect(d.temporalFilterValue('t', FIELD[kind], list)).toStrictEqual(list.map(physical)); + }); + } + } + + it('a field that is not temporal passes through untouched', () => { + const d = makeDriver('better-sqlite3'); + expect(d.temporalFilterValue('t', 'note', '2026-02-01T10:00:00Z')).toBe('2026-02-01T10:00:00Z'); + }); +}); diff --git a/packages/objectql/src/engine-aggregate-temporal-storage-rule.test.ts b/packages/objectql/src/engine-aggregate-temporal-storage-rule.test.ts new file mode 100644 index 00000000000..41ac90d7198 --- /dev/null +++ b/packages/objectql/src/engine-aggregate-temporal-storage-rule.test.ts @@ -0,0 +1,326 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. +// +// [#20176] A per-aggregation `filter` and `having` read a temporal comparand by +// the COLUMN's storage rule — `@objectstack/core`'s `temporalStorageForm`, the +// one function both drivers' `where` applies to the same comparand — instead of +// comparing it as written. +// +// Measured on the base (`cfe2387a3b`) through `engine.aggregate` and +// `POST /api/v1/data/:object/query`, on a real `InMemoryDriver` and a real +// `SqlDriver`, populated and empty: every row of the card counted differently +// from its `where` twin on both drivers — +// +// | # | per-aggregation `filter` (or `having`) | base | `where` twin | +// |:--|:--|:--|:--| +// | 1 | an ISO instant `$gte` on a `date` field | 1 | 3 | +// | 2 | an ISO instant `$eq` on a `date` field | 0 | 2 | +// | 3 | a bare day as the `$lte` of a `datetime` | 2 | 3 | +// | 4 | a bare day as the `$between` max of a `datetime` | 2 | 3 | +// | 5 | an epoch-ms bound on a `datetime` | 0 | 3 | +// | 6 | a `Date` with a time of day on a `date` field (`$gte` / `$lt` / `$eq`) | 1 / 5 / 0 | 3 / 3 / 2 | +// | 7 | a `Date` on a `time` field | 0 | 3 | +// | 8 | `having` on `max(date)` with an ISO bound | keeps c2 | keeps c2, c3 | +// +// and a zone-naive datetime string counted 4 on both sides (the control). The +// `where` counts are pinned on each driver in its own package +// (`driver-memory`'s `memory-temporal-storage-form.test.ts`; `driver-sql`'s +// through the REST door, `packages/rest/src/aggregation-filter-temporal-storage-rule.test.ts`). +// Here the per-aggregation position and `having` are held to them, on rows in +// the storage form both drivers present, and to the shared temporal +// conformance kit (`@objectstack/spec/data`'s `TEMPORAL_CASES` / +// `TEMPORAL_TIME_CASES`) every `where` backend is already held to. + +import { describe, it, expect } from 'vitest'; +import { + TEMPORAL_CASES, + TEMPORAL_NOW, + TEMPORAL_ROWS, + TEMPORAL_TIME_CASES, + TEMPORAL_TIME_ROWS, + type EngineAggregateOptions, +} from '@objectstack/spec/data'; +import { resolveFilterTokens } from '@objectstack/core'; +import { ObjectQL } from './engine.js'; +import { applyInMemoryAggregation } from './in-memory-aggregation.js'; +import { declaredFieldClasses, matchesAggregationFilter } from './having-filter.js'; + +const OBJECT = 'ledger_order'; + +const FIELDS = { + customer_id: { type: 'text' }, + amount: { type: 'number' }, + placed_on: { type: 'date' }, + due_on: { type: 'date' }, + opened_at: { type: 'datetime' }, + slot: { type: 'time' }, + note: { type: 'text' }, +}; + +// The card's fixture, in the storage form both drivers present a row in +// (ADR-0053: `YYYY-MM-DD`, canonical UTC ISO, `HH:MM:SS`). +const ROWS = [ + { id: 'o1', customer_id: 'c1', amount: 100, placed_on: '2026-01-10', due_on: '2026-01-05', opened_at: '2026-01-01T10:00:00.000Z', slot: '09:00:00', note: 'a' }, + { id: 'o2', customer_id: 'c1', amount: 400, placed_on: '2026-01-02', due_on: '2026-01-20', opened_at: '2026-01-02T10:00:00.000Z', slot: '10:30:00', note: null }, + { id: 'o3', customer_id: 'c2', amount: 900, placed_on: '2026-03-01', due_on: '2026-01-01', opened_at: '2026-02-01T10:00:00.000Z', slot: '11:00:00', note: 'b' }, + { id: 'o4', customer_id: 'c2', amount: 300, placed_on: '2026-02-01', due_on: '2026-02-01', opened_at: '2026-02-05T10:00:00.000Z', slot: '12:00:00', note: 'c' }, + { id: 'o5', customer_id: 'c2', amount: 50, placed_on: '2026-01-15', due_on: '2026-03-01', opened_at: '2026-02-06T10:00:00.000Z', slot: '13:00:00', note: null }, + { id: 'o6', customer_id: 'c3', amount: 20, placed_on: '2026-02-01', due_on: '2026-01-31', opened_at: '2026-03-01T10:00:00.000Z', slot: '14:00:00', note: 'd' }, +]; + +const DRIVER_BASE = { + version: '0.0.0', + supports: {}, + async connect() {}, async disconnect() {}, async checkHealth() { return true; }, async execute() { return null; }, + async findOne() { return null; }, + async create(_o: string, d: any) { return d; }, + async update(_o: string, _id: string, d: any) { return d; }, + async delete() { return true; }, + async count() { return 0; }, + async bulkCreate(_o: string, r: any[]) { return r; }, + async bulkUpdate() { return []; }, async bulkDelete() {}, + async beginTransaction() { return { commit: async () => {}, rollback: async () => {} }; }, + async commit() {}, async rollback() {}, +}; + +/** + * Two driver shapes: one the engine asks for ROWS (`find()` + the in-memory + * lowering, where the per-aggregation filter is evaluated), and one with a + * native `aggregate()` (the other `having` door). Neither reads the query's + * `where`; none of these tests carries one. + */ +function makeDriver(kind: 'rows' | 'native', rows: any[]) { + const driver: any = { ...DRIVER_BASE, name: `${kind}-mock`, async find() { return rows.slice(); } }; + if (kind === 'native') driver.aggregate = async (_o: string, ast: any) => applyInMemoryAggregation(rows, ast); + return driver; +} + +async function makeEngine(kind: 'rows' | 'native', rows: any[], object = OBJECT, fields: Record = FIELDS) { + const engine = new ObjectQL(); + engine.registerDriver(makeDriver(kind, rows), true); + await engine.init(); + (engine.registry as any).registerObject({ name: object, fields }); + return engine; +} + +type Shape = readonly [name: string, filter: () => Record, whereCount: number]; + +const D = (iso: string) => new Date(iso); + +/** The card's rows, with the count the same condition as a `where` answers on both drivers. */ +const CARD_ROWS: readonly Shape[] = [ + ['row 1 — an ISO instant $gte on a date field', () => ({ placed_on: { $gte: '2026-02-01T00:00:00.000Z' } }), 3], + ['row 2 — an ISO instant $eq on a date field', () => ({ placed_on: { $eq: '2026-02-01T00:00:00.000Z' } }), 2], + ['row 3 — a bare day as the $lte of a datetime', () => ({ opened_at: { $lte: '2026-02-01' } }), 3], + ['row 4 — a bare day as the $between max of a datetime', () => ({ opened_at: { $between: ['2026-01-01', '2026-02-01'] } }), 3], + ['row 5 — an epoch-ms bound on a datetime', () => ({ opened_at: { $gt: 1769940000000 } }), 3], + ['row 6 — a Date with a time of day, $gte on a date field', () => ({ placed_on: { $gte: D('2026-02-01T10:00:00.000Z') } }), 3], + ['row 6 — the same Date, $lt', () => ({ placed_on: { $lt: D('2026-02-01T10:00:00.000Z') } }), 3], + ['row 6 — the same Date, $eq', () => ({ placed_on: { $eq: D('2026-02-01T10:00:00.000Z') } }), 2], + ['row 7 — a Date on a time field', () => ({ slot: { $gt: D('2026-02-01T11:00:00.000Z') } }), 3], + ['control — a zone-naive datetime string', () => ({ opened_at: { $gt: '2026-02-01 09:00' } }), 4], +]; + +/** The family's other members — every comparison operator, both list operators, composition. */ +const FAMILY_ROWS: readonly Shape[] = [ + ['an ISO instant $ne on a date field', () => ({ placed_on: { $ne: '2026-02-01T00:00:00.000Z' } }), 4], + ['an ISO instant as implicit equality on a date field', () => ({ placed_on: '2026-02-01T00:00:00.000Z' }), 2], + ['ISO instants as $in members on a date field', () => ({ placed_on: { $in: ['2026-02-01T00:00:00.000Z', '2026-01-10T05:00:00Z'] } }), 3], + ['ISO instants as $nin members on a date field', () => ({ placed_on: { $nin: ['2026-02-01T00:00:00.000Z', '2026-01-10T05:00:00Z'] } }), 3], + ['ISO instants as $between endpoints on a date field', () => ({ placed_on: { $between: ['2026-01-10T00:00:00Z', '2026-02-01T23:00:00Z'] } }), 4], + ['an ISO instant $lt on a date field', () => ({ placed_on: { $lt: '2026-02-01T10:00:00.000Z' } }), 3], + ['bare days as both $between endpoints on a datetime', () => ({ opened_at: { $between: ['2026-02-01', '2026-02-05'] } }), 2], + ['an epoch-ms string $gt on a datetime', () => ({ opened_at: { $gt: '1769940000000' } }), 3], + ['an epoch-ms $lte on a datetime', () => ({ opened_at: { $lte: 1769940000000 } }), 3], + ['an epoch-ms $eq on a datetime', () => ({ opened_at: { $eq: 1769940000000 } }), 1], + ['an offset instant $gte on a datetime', () => ({ opened_at: { $gte: '2026-02-01T18:00:00+08:00' } }), 4], + ['a zone-naive T spelling $eq on a datetime', () => ({ opened_at: { $eq: '2026-02-01T10:00' } }), 1], + ['an ISO instant without milliseconds $nin on a datetime', () => ({ opened_at: { $nin: ['2026-02-01T10:00:00Z'] } }), 5], + ['an ISO instant $gt on a time field', () => ({ slot: { $gt: '2026-02-01T11:00:00.000Z' } }), 3], + ['a short wall clock $eq on a time field', () => ({ slot: { $eq: '11:00' } }), 1], + ['a short wall clock $lte on a time field', () => ({ slot: { $lte: '11:00' } }), 3], + ['short wall clocks as $in members on a time field', () => ({ slot: { $in: ['09:00', '11:00'] } }), 2], + ['short wall clocks as $between endpoints on a time field', () => ({ slot: { $between: ['09:00', '11:00'] } }), 3], + ['a bare day $lte on a time field — no whole-day widening', () => ({ slot: { $lte: '2026-02-01' } }), 0], + ['a Date as implicit equality on a time field', () => ({ slot: D('2026-02-01T11:00:00.000Z') }), 1], + ['a Date as a $nin member on a date field', () => ({ placed_on: { $nin: [D('2026-02-01T10:00:00.000Z')] } }), 4], + ['Dates as $between endpoints on a date field', () => ({ placed_on: { $between: [D('2026-01-10T10:00:00.000Z'), D('2026-02-01T10:00:00.000Z')] } }), 4], + ['under $or', () => ({ $or: [{ placed_on: { $gte: '2026-02-01T00:00:00.000Z' } }, { amount: { $gt: 800 } }] }), 3], + ['under $not', () => ({ $not: { placed_on: { $gte: '2026-02-01T00:00:00.000Z' } } }), 3], + ['under $and', () => ({ $and: [{ placed_on: { $gte: '2026-02-01T00:00:00.000Z' } }, { amount: { $lt: 800 } }] }), 2], + ['a bare-day $lte beside an author $lt on the same datetime', () => ({ opened_at: { $lte: '2026-02-05', $lt: '2026-02-05T12:00:00.000Z' } }), 4], +]; + +/** Positions no storage rule reaches — compared exactly as before. */ +const UNMOVED_ROWS: readonly Shape[] = [ + ['an ISO-looking string on a TEXT field', () => ({ note: { $gte: '2026-02-01T00:00:00.000Z' } }), 4], + ['an epoch-ms number on a NUMBER field', () => ({ amount: { $gt: 1769940000000 } }), 0], + ['presence on a date field', () => ({ placed_on: { $exists: true } }), 6], + ['a null $eq on a date field', () => ({ placed_on: { $eq: null } }), 0], +]; + +const perAggregation = (filter: Record): EngineAggregateOptions => ({ + aggregations: [{ function: 'count', alias: 'n' }, { function: 'count', alias: 'm', filter: filter as any }], +}); + +describe('[#20176] a per-aggregation filter counts a temporal comparand by the column\'s storage rule', () => { + for (const [group, shapes] of [['the card', CARD_ROWS], ['the family', FAMILY_ROWS], ['unmoved', UNMOVED_ROWS]] as const) { + for (const [name, filter, whereCount] of shapes) { + it(`${group}: ${name} — counts ${whereCount}, as its where twin; 0 on an empty object`, async () => { + for (const kind of ['rows', 'native'] as const) { + const populated = await makeEngine(kind, ROWS); + expect(await populated.aggregate(OBJECT, perAggregation(filter())), `${kind}, populated`) + .toEqual([{ n: 6, m: whereCount }]); + const empty = await makeEngine(kind, []); + expect(await empty.aggregate(OBJECT, perAggregation(filter())), `${kind}, empty`) + .toEqual([{ n: 0, m: 0 }]); + } + }); + } + } + + it('grouped: each bucket counts by the same rule (row 1, per customer)', async () => { + const engine = await makeEngine('rows', ROWS); + const rows = await engine.aggregate(OBJECT, { + groupBy: ['customer_id'], + aggregations: [{ function: 'count', alias: 'm', filter: { placed_on: { $gte: '2026-02-01T00:00:00.000Z' } } }], + }); + expect(Object.fromEntries(rows.map((r: any) => [r.customer_id, r.m]))).toEqual({ c1: 0, c2: 2, c3: 1 }); + }); +}); + +/** `having` over customer groups: the aggregated columns the rows below name. */ +const HAVING_AGGREGATIONS: NonNullable = [ + { function: 'sum', field: 'amount', alias: 'total' }, + { function: 'max', field: 'placed_on', alias: 'last_placed' }, + { function: 'min', field: 'due_on', alias: 'first_due' }, + { function: 'min', field: 'opened_at', alias: 'first_opened' }, + { function: 'max', field: 'slot', alias: 'last_slot' }, +]; +// c1: last_placed 2026-01-10, first_due 2026-01-05, first_opened 2026-01-01T10:00Z, last_slot 10:30:00 +// c2: last_placed 2026-03-01, first_due 2026-01-01, first_opened 2026-02-01T10:00Z, last_slot 13:00:00 +// c3: last_placed 2026-02-01, first_due 2026-01-31, first_opened 2026-03-01T10:00Z, last_slot 14:00:00 + +const HAVING_ROWS: ReadonlyArray Record, string[]]> = [ + ['row 8 — an ISO instant $gte on max(date)', () => ({ last_placed: { $gte: '2026-02-01T00:00:00.000Z' } }), ['c2', 'c3']], + ['an ISO instant $eq on max(date)', () => ({ last_placed: { $eq: '2026-03-01T00:00:00.000Z' } }), ['c2']], + ['an ISO instant $lt on min(date)', () => ({ first_due: { $lt: '2026-01-05T10:00:00.000Z' } }), ['c2']], + ['ISO instants as $in members on max(date)', () => ({ last_placed: { $in: ['2026-03-01T00:00:00Z', '2026-02-01T05:00:00Z'] } }), ['c2', 'c3']], + ['an ISO instant $ne on max(date)', () => ({ last_placed: { $ne: '2026-03-01T00:00:00Z' } }), ['c1', 'c3']], + ['a Date with a time of day $gte on max(date)', () => ({ last_placed: { $gte: D('2026-02-01T10:00:00.000Z') } }), ['c2', 'c3']], + ['a bare day as the $lte of min(datetime)', () => ({ first_opened: { $lte: '2026-02-01' } }), ['c1', 'c2']], + ['a bare day as the $between max of min(datetime)', () => ({ first_opened: { $between: ['2026-01-01', '2026-02-01'] } }), ['c1', 'c2']], + ['an epoch-ms $gte on min(datetime)', () => ({ first_opened: { $gte: 1769940000000 } }), ['c2', 'c3']], + ['an offset instant $gte on min(datetime)', () => ({ first_opened: { $gte: '2026-02-01T18:00:00+08:00' } }), ['c2', 'c3']], + ['a short wall clock $eq on max(time)', () => ({ last_slot: { $eq: '14:00' } }), ['c3']], + ['an ISO instant $gte on max(time)', () => ({ last_slot: { $gte: '2026-01-01T13:00:00.000Z' } }), ['c2', 'c3']], + ['control — a zone-naive $gt on min(datetime)', () => ({ first_opened: { $gt: '2026-02-01 09:00' } }), ['c2', 'c3']], + ['control — a numeric column', () => ({ total: { $gt: 1000 } }), ['c2']], +]; + +describe('[#20176] having reads a temporal comparand by the aggregated column\'s storage rule — both doors', () => { + for (const [name, having, kept] of HAVING_ROWS) { + it(`${name}: keeps ${kept.join(', ')}; nothing on an empty object`, async () => { + // `native`: the driver aggregates, the engine applies `having` after it. + // `rows`: the engine aggregates in memory (a filtered aggregation forces + // that door), then applies `having`. + for (const kind of ['native', 'rows'] as const) { + const aggregations = kind === 'native' + ? HAVING_AGGREGATIONS + : [...HAVING_AGGREGATIONS, { function: 'count' as const, alias: 'fb', filter: { customer_id: { $ne: '' } } }]; + const query: EngineAggregateOptions = { groupBy: ['customer_id'], aggregations, having: having() as any }; + const populated = await makeEngine(kind, ROWS); + const got = (await populated.aggregate(OBJECT, query)).map((r: any) => r.customer_id).sort(); + expect(got, kind).toEqual(kept); + const empty = await makeEngine(kind, []); + expect(await empty.aggregate(OBJECT, query), `${kind}, empty`).toEqual([]); + } + }); + } + + it('a groupBy projection of a date field, and a day bucket, are dates', async () => { + const engine = await makeEngine('native', ROWS); + const byDay = await engine.aggregate(OBJECT, { + groupBy: ['placed_on'], + aggregations: [{ function: 'count', alias: 'n' }], + having: { placed_on: { $gte: '2026-02-01T00:00:00.000Z' } }, + }); + expect(byDay.map((r: any) => r.placed_on).sort()).toEqual(['2026-02-01', '2026-03-01']); + const bucketed = await engine.aggregate(OBJECT, { + groupBy: [{ field: 'opened_at', dateGranularity: 'day', alias: 'd' }], + aggregations: [{ function: 'count', alias: 'n' }], + having: { d: { $gte: '2026-02-01T00:00:00.000Z' } }, + }); + expect(bucketed.map((r: any) => r.d).sort()).toEqual(['2026-02-01', '2026-02-05', '2026-02-06', '2026-03-01']); + }); +}); + +// ── The shared kit: every `where` backend is held to these; now these two positions too ── + +const resolveTokens = (filter: T): T => resolveFilterTokens(filter, { now: new Date(TEMPORAL_NOW) }); + +async function idsKeptByFilter(kind: 'rows' | 'native', object: string, fields: Record, rows: any[], filter: unknown) { + const engine = await makeEngine(kind, rows, object, fields); + const out = await engine.aggregate(object, { + groupBy: ['id'], + aggregations: [{ function: 'count', alias: 'm', filter: filter as any }], + }); + return out.filter((r: any) => r.m === 1).map((r: any) => r.id).sort(); +} + +async function idsKeptByHaving(kind: 'rows' | 'native', object: string, fields: Record, rows: any[], column: string, having: unknown) { + const engine = await makeEngine(kind, rows, object, fields); + const aggregations: NonNullable = [{ function: 'max', field: column, alias: column }]; + if (kind === 'rows') aggregations.push({ function: 'count', alias: 'fb', filter: { id: { $ne: '' } } }); + const out = await engine.aggregate(object, { groupBy: ['id'], aggregations, having: having as any }); + return out.map((r: any) => r.id).sort(); +} + +describe('[#20176] the temporal conformance kit, through a per-aggregation filter and having', () => { + const fields = { at: { type: 'datetime' }, on: { type: 'date' } }; + const rows = TEMPORAL_ROWS.map((r) => ({ id: r.id, at: r.at, on: r.on })); + for (const c of TEMPORAL_CASES) { + it(c.name, async () => { + const expected = [...c.expected].sort(); + const spellings = c.tokenFilter ? [c.filter, resolveTokens(c.tokenFilter)] : [c.filter]; + for (const filter of spellings) { + for (const kind of ['rows', 'native'] as const) { + expect(await idsKeptByFilter(kind, 'kit', fields, rows, filter), `filter, ${kind} — ${c.note ?? ''}`).toEqual(expected); + expect(await idsKeptByHaving(kind, 'kit', fields, rows, c.field, filter), `having, ${kind} — ${c.note ?? ''}`).toEqual(expected); + } + } + }); + } + + const timeFields = { at: { type: 'time' } }; + const timeRows = TEMPORAL_TIME_ROWS.map((r) => ({ id: r.id, at: r.at })); + for (const c of TEMPORAL_TIME_CASES) { + it(c.name, async () => { + const expected = [...c.expected].sort(); + for (const kind of ['rows', 'native'] as const) { + expect(await idsKeptByFilter(kind, 'kit_time', timeFields, timeRows, c.filter), `filter, ${kind}`).toEqual(expected); + expect(await idsKeptByHaving(kind, 'kit_time', timeFields, timeRows, 'at', c.filter), `having, ${kind}`).toEqual(expected); + } + }); + } +}); + +describe('[#20176] where no class is known, a comparand is compared as written', () => { + const row = { placed_on: '2026-02-01', stamp: '2026-02-01T10:00:00.000Z' }; + const iso = { placed_on: { $gte: '2026-02-01T00:00:00.000Z' } }; + + it('the walker with the declaration reads the storage rule; without it, the text', () => { + expect(matchesAggregationFilter(row, iso, 0, declaredFieldClasses(FIELDS))).toBe(true); + expect(matchesAggregationFilter(row, iso, 0)).toBe(false); + }); + + it('applyInMemoryAggregation keeps its published default: no field map, no rule', () => { + const ast = { aggregations: [{ function: 'count' as const, alias: 'm', filter: iso }] }; + expect(applyInMemoryAggregation(ROWS, ast)).toEqual([{ m: 1 }]); + expect(applyInMemoryAggregation(ROWS, ast, undefined, FIELDS)).toEqual([{ m: 3 }]); + }); + + it('an undeclared column keeps the #20148 instant reading of a Date', () => { + const bound = { stamp: { $gte: D('2026-02-01T10:00:00.000Z') } }; + expect(matchesAggregationFilter(row, bound, 0, declaredFieldClasses(FIELDS))).toBe(true); + }); +}); diff --git a/packages/rest/src/aggregation-filter-temporal-storage-rule.test.ts b/packages/rest/src/aggregation-filter-temporal-storage-rule.test.ts new file mode 100644 index 00000000000..8d5047f841e --- /dev/null +++ b/packages/rest/src/aggregation-filter-temporal-storage-rule.test.ts @@ -0,0 +1,162 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#20176] A per-aggregation `filter` and `having` read a temporal comparand by + * the column's storage rule, through the door a caller actually uses: + * `POST /api/v1/data/:object/query` → `RestServer` → + * `ObjectStackProtocolImplementation.findData` → `ObjectQL.aggregate` → a real + * sqlite `SqlDriver` (the harness `aggregation-filter-where-doors.test.ts` + * boots). + * + * Measured on the base through this door, on driver-sql and driver-memory + * alike: each row below counted differently from the same condition as the + * call's `where` — the TWIN every row runs beside it — because the engine + * compared the comparand as written while the driver's `where` put it in the + * column's storage form first: + * + * | row | per-aggregation `filter` | base | `where` twin | + * |:--|:--|:--|:--| + * | 1 | an ISO instant `$gte` on a `date` field | 1 | 3 | + * | 2 | an ISO instant `$eq` on a `date` field | 0 | 2 | + * | 3 | a bare day as the `$lte` of a `datetime` | 2 | 3 | + * | 4 | a bare day as the `$between` max of a `datetime` | 2 | 3 | + * | 5 | an epoch-ms bound on a `datetime` | 0 | 3 | + * | 8 | `having` on `max(date)` with an ISO bound | keeps c2 | keeps c2, c3 | + * + * Rows 6 and 7 (a `Date` bound) reach this door only as their ISO text, which + * the JSON body carries; that spelling is the "on the wire" block below. The + * engine-level `Date` pins are `packages/objectql`'s + * `engine-aggregate-temporal-storage-rule.test.ts`. + */ + +import { describe, it, expect, afterEach } from 'vitest'; +import { ObjectQL } from '@objectstack/objectql'; +import { SqlDriver } from '@objectstack/driver-sql'; +import { ObjectStackProtocolImplementation } from '@objectstack/metadata-protocol'; +import { RestServer } from './rest-server'; + +const OBJECT = 'ledger_order'; + +const LEDGER_ORDER = { + name: OBJECT, + label: 'Ledger Order', + fields: { + customer_id: { name: 'customer_id', type: 'text' as const }, + amount: { name: 'amount', type: 'number' as const }, + placed_on: { name: 'placed_on', type: 'date' as const }, + opened_at: { name: 'opened_at', type: 'datetime' as const }, + slot: { name: 'slot', type: 'time' as const }, + }, +}; + +const ROWS = [ + { id: 'o1', customer_id: 'c1', amount: 100, placed_on: '2026-01-10', opened_at: '2026-01-01T10:00:00.000Z', slot: '09:00:00' }, + { id: 'o2', customer_id: 'c1', amount: 400, placed_on: '2026-01-02', opened_at: '2026-01-02T10:00:00.000Z', slot: '10:30:00' }, + { id: 'o3', customer_id: 'c2', amount: 900, placed_on: '2026-03-01', opened_at: '2026-02-01T10:00:00.000Z', slot: '11:00:00' }, + { id: 'o4', customer_id: 'c2', amount: 300, placed_on: '2026-02-01', opened_at: '2026-02-05T10:00:00.000Z', slot: '12:00:00' }, + { id: 'o5', customer_id: 'c2', amount: 50, placed_on: '2026-01-15', opened_at: '2026-02-06T10:00:00.000Z', slot: '13:00:00' }, + { id: 'o6', customer_id: 'c3', amount: 20, placed_on: '2026-02-01', opened_at: '2026-03-01T10:00:00.000Z', slot: '14:00:00' }, +]; + +const liveEngines: ObjectQL[] = []; +afterEach(async () => { + while (liveEngines.length) { + try { await liveEngines.pop()?.destroy(); } catch { /* noop */ } + } +}); + +function createMockServer() { + const noop = () => {}; + return { get: noop, post: noop, put: noop, delete: noop, patch: noop, use: noop, listen: async () => {}, close: async () => {} }; +} + +function makeRes() { + const res: any = { + write: () => true, end: () => {}, + header: () => res, + status: (code: number) => { res._status = code; return res; }, + json: (body: any) => { res._json = body; return res; }, + }; + return res; +} + +/** A booted door over a real sqlite table, populated or empty. */ +async function boot(populated: boolean) { + const driver: any = new SqlDriver({ client: 'better-sqlite3', connection: { filename: ':memory:' }, useNullAsDefault: true }); + const engine = new ObjectQL(); + liveEngines.push(engine); + engine.registerDriver(driver, true); + await engine.init(); + engine.registry.registerObject(LEDGER_ORDER as any); + await engine.syncSchemas(); + if (populated) await engine.insert(OBJECT, ROWS as any); + + const protocol = new ObjectStackProtocolImplementation(engine as any); + const rest = new RestServer(createMockServer() as any, protocol as any, { api: { requireAuth: false } } as any); + (rest as any).resolveExecCtx = async () => ({ userId: 'test-user' }); + rest.registerRoutes(); + const route = rest.getRoutes().find((r: any) => r.method === 'POST' && r.path === '/api/v1/data/:object/query'); + expect(route).toBeDefined(); + return async (body: Record) => { + const res = makeRes(); + await route!.handler({ params: { object: OBJECT }, body: JSON.parse(JSON.stringify(body)) } as any, res); + expect(res._status ?? 200, JSON.stringify(res._json)).toBe(200); + return res._json.records as any[]; + }; +} + +const perAggregation = (filter: unknown) => ({ + aggregations: [{ function: 'count', alias: 'n' }, { function: 'count', alias: 'm', filter }], +}); +const whereTwin = (filter: unknown) => ({ where: filter, aggregations: [{ function: 'count', alias: 'n' }] }); + +const ROWS_COUNTED: ReadonlyArray, number]> = [ + ['row 1 — an ISO instant $gte on a date field', { placed_on: { $gte: '2026-02-01T00:00:00.000Z' } }, 3], + ['row 2 — an ISO instant $eq on a date field', { placed_on: { $eq: '2026-02-01T00:00:00.000Z' } }, 2], + ['row 3 — a bare day as the $lte of a datetime', { opened_at: { $lte: '2026-02-01' } }, 3], + ['row 4 — a bare day as the $between max of a datetime', { opened_at: { $between: ['2026-01-01', '2026-02-01'] } }, 3], + ['row 5 — an epoch-ms bound on a datetime', { opened_at: { $gt: 1769940000000 } }, 3], + ['row 6 on the wire — the Date as its ISO text, $gte on a date field', { placed_on: { $gte: '2026-02-01T10:00:00.000Z' } }, 3], + ['row 7 on the wire — the Date as its ISO text, on a time field', { slot: { $gt: '2026-02-01T11:00:00.000Z' } }, 3], + ['a short wall clock $eq on a time field', { slot: { $eq: '11:00' } }, 1], + ['ISO instants as $in members on a date field', { placed_on: { $in: ['2026-02-01T00:00:00.000Z', '2026-01-10T05:00:00Z'] } }, 3], + ['control — a zone-naive datetime string', { opened_at: { $gt: '2026-02-01 09:00' } }, 4], +]; + +describe('[#20176] POST /data/:object/query — a per-aggregation filter counts as its where twin', () => { + for (const [name, filter, count] of ROWS_COUNTED) { + it(`${name}: ${count} of 6 on a populated table, 0 on an empty one — the where twin's numbers`, async () => { + for (const populated of [false, true]) { + const post = await boot(populated); + const expected = populated ? count : 0; + expect(await post(perAggregation(filter)), `populated=${populated}`).toEqual([{ n: populated ? 6 : 0, m: expected }]); + expect(await post(whereTwin(filter)), `where twin, populated=${populated}`).toEqual([{ n: expected }]); + } + }); + } +}); + +describe('[#20176] POST /data/:object/query — having reads the aggregated column\'s storage rule', () => { + const grouped = (having: unknown) => ({ + groupBy: ['customer_id'], + aggregations: [ + { function: 'max', field: 'placed_on', alias: 'last_placed' }, + { function: 'min', field: 'opened_at', alias: 'first_opened' }, + ], + having, + }); + const HAVING: ReadonlyArray, string[]]> = [ + ['row 8 — an ISO instant $gte on max(date)', { last_placed: { $gte: '2026-02-01T00:00:00.000Z' } }, ['c2', 'c3']], + ['a bare day as the $lte of min(datetime)', { first_opened: { $lte: '2026-02-01' } }, ['c1', 'c2']], + ['control — a bare day $gte on max(date)', { last_placed: { $gte: '2026-02-01' } }, ['c2', 'c3']], + ]; + for (const [name, having, kept] of HAVING) { + it(`${name}: keeps ${kept.join(', ')}; nothing on an empty table`, async () => { + for (const populated of [false, true]) { + const post = await boot(populated); + const got = (await post(grouped(having))).map((r) => r.customer_id).sort(); + expect(got, `populated=${populated}`).toEqual(populated ? kept : []); + } + }); + } +}); From a76b115eb2552281888a70cd3f27c7077af9e37d Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 07:52:08 +0000 Subject: [PATCH 3/3] test(driver-memory): type the where twin instead of erasing the query to any Claude-Session: https://claude.ai/code/session_01Bvd69VPa6puiNzzPUroDBx Co-authored-by: Claude --- .../driver-memory/src/memory-temporal-storage-form.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/drivers/driver-memory/src/memory-temporal-storage-form.test.ts b/packages/drivers/driver-memory/src/memory-temporal-storage-form.test.ts index 3a2aeeeb45e..249030b082f 100644 --- a/packages/drivers/driver-memory/src/memory-temporal-storage-form.test.ts +++ b/packages/drivers/driver-memory/src/memory-temporal-storage-form.test.ts @@ -22,6 +22,7 @@ import { describe, it, expect, beforeAll } from 'vitest'; import { temporalStorageForm } from '@objectstack/core'; +import type { FilterCondition } from '@objectstack/spec/data'; import { coerceTemporalValue } from './memory-temporal.js'; import { InMemoryDriver } from './memory-driver.js'; @@ -73,7 +74,7 @@ const ROWS = [ ]; /** The #20176 card's rows, as a `where`, with the count this driver answers. */ -const CARD_WHERE_TWINS: ReadonlyArray Record, number]> = [ +const CARD_WHERE_TWINS: ReadonlyArray FilterCondition, number]> = [ ['row 1 — an ISO instant $gte on a date field', () => ({ placed_on: { $gte: '2026-02-01T00:00:00.000Z' } }), 3], ['row 2 — an ISO instant $eq on a date field', () => ({ placed_on: { $eq: '2026-02-01T00:00:00.000Z' } }), 2], ['row 3 — a bare day as the $lte of a datetime', () => ({ opened_at: { $lte: '2026-02-01' } }), 3], @@ -101,7 +102,7 @@ describe('[#20176] the where twins of the card\'s rows, on this driver', () => { for (const [name, where, expected] of CARD_WHERE_TWINS) { it(`${name}: ${expected} of 6`, async () => { - const rows = await driver.find(OBJECT, { where: where() } as any); + const rows = await driver.find(OBJECT, { where: where() }); expect(rows).toHaveLength(expected); }); }