From 99fd7b9efea4280c73181896862e2e4b7d40f07f Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 7 Oct 2026 11:39:46 +0800 Subject: [PATCH 01/51] Optimize generated shared load state and governed dynamic views Refs #204. Qualify generated wide/inherited examples, atomic dynamic persistence, nested graph serialization, and typed JDBC streaming on local source dependencies. Public release and full artifact acceptance remain separate. --- examples/shared-load-state/.gitignore | 1 + examples/shared-load-state/README.md | 111 ++++ .../shared-load-state/benchmark-driver.sh | 17 + examples/shared-load-state/model.xml | 14 + examples/shared-load-state/pom.xml | 34 ++ ...GeneratedInheritedSchoolLoadStateTest.java | 122 +++++ .../DriverQueryBenchmarkTest.java | 151 +++++ .../GeneratedNestedGraphAssertions.java | 128 +++++ .../GeneratedSchoolLoadStateTest.java | 282 ++++++++++ .../GeneratedWideHydrationAllocationTest.java | 124 +++++ .../HydrationAllocationTest.java | 232 ++++++++ .../LoadStateAllocationTest.java | 87 +++ .../TypedStreamAllocationTest.java | 99 ++++ .../shared-load-state/verify-allocations.sh | 44 ++ examples/shared-load-state/verify.sh | 80 +++ examples/verify-runtime-examples.sh | 7 +- pom.xml | 7 + .../main/java/io/teaql/core/BaseEntity.java | 275 ++++++---- .../java/io/teaql/core/EntityChangeSet.java | 22 +- .../io/teaql/core/EntityMutationLedger.java | 9 +- .../main/java/io/teaql/core/FieldLayout.java | 151 +++++ .../main/java/io/teaql/core/LoadState.java | 138 +++++ .../main/java/io/teaql/core/SmartList.java | 36 ++ .../io/teaql/core/TransactionExecutor.java | 2 + .../core/meta/SimpleEntityMetaFactory.java | 11 + .../java/io/teaql/core/BaseEntityTest.java | 27 +- .../io/teaql/core/DynamicMemberStateTest.java | 64 +++ .../io/teaql/core/EntityChangeSetTest.java | 18 + .../java/io/teaql/core/LoadStateTest.java | 286 ++++++++++ .../sql/SqlDataServiceExecutor.java | 53 +- .../dataservice/sql/SqlExecutionAdapter.java | 11 + .../dynamic/DefaultDynamicFieldsFacade.java | 63 ++- .../data/dynamic/DynamicFieldMetadata.java | 62 +++ .../data/dynamic/DynamicFieldMutation.java | 40 ++ .../data/dynamic/DynamicFieldSelection.java | 5 + .../teaql/data/dynamic/DynamicFieldValue.java | 20 +- .../data/dynamic/DynamicFieldValues.java | 58 +- .../data/dynamic/DynamicFieldsFacade.java | 22 + .../data/dynamic/DynamicFieldsProvider.java | 10 + .../data/dynamic/DynamicGraphMutation.java | 10 + .../InMemoryDynamicFieldsProvider.java | 29 +- .../dynamic/DynamicFieldValueStateTest.java | 89 +++ .../InMemoryDynamicFieldsProviderTest.java | 51 +- .../jdbc/JdbcDynamicFieldsProvider.java | 67 ++- .../jdbc/JdbcDynamicFieldsProviderTest.java | 54 ++ .../jackson/BaseEntityJsonSerializer.java | 57 +- .../jackson/BaseEntitySerializationTest.java | 183 +++++++ .../teaql/provider/jdbc/JdbcSqlExecutor.java | 47 +- .../provider/jdbc/JdbcSqlExecutorTest.java | 51 ++ .../java/io/teaql/runtime/TeaQLRuntime.java | 136 ++++- .../core/sql/expression/PropertyParser.java | 2 +- .../core/sql/portable/IdSpaceIdGenerator.java | 12 +- .../sql/portable/PortableSQLDataService.java | 63 ++- .../sql/portable/PortableSQLRepository.java | 134 ++++- .../core/sql/portable/TeaQLDatabase.java | 8 + .../sql/portable/IdSpaceIdGeneratorTest.java | 35 ++ .../InheritedRelationOwnershipTest.java | 43 ++ .../sql/portable/PortableSQLDatabaseTest.java | 8 +- .../PortableSQLTemporalConversionTest.java | 10 + teaql-sqlite/pom.xml | 7 + .../io/teaql/sqlite/GraphTraceSqliteTest.java | 8 +- .../teaql/sqlite/SqliteIntegrationTest.java | 518 +++++++++++++++++- 62 files changed, 4308 insertions(+), 237 deletions(-) create mode 100644 examples/shared-load-state/.gitignore create mode 100644 examples/shared-load-state/README.md create mode 100644 examples/shared-load-state/benchmark-driver.sh create mode 100644 examples/shared-load-state/model.xml create mode 100644 examples/shared-load-state/pom.xml create mode 100644 examples/shared-load-state/src/inheritance/java/com/example/schoolmanagementservice/GeneratedInheritedSchoolLoadStateTest.java create mode 100644 examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/DriverQueryBenchmarkTest.java create mode 100644 examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedNestedGraphAssertions.java create mode 100644 examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java create mode 100644 examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedWideHydrationAllocationTest.java create mode 100644 examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/HydrationAllocationTest.java create mode 100644 examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/LoadStateAllocationTest.java create mode 100644 examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/TypedStreamAllocationTest.java create mode 100644 examples/shared-load-state/verify-allocations.sh create mode 100644 examples/shared-load-state/verify.sh create mode 100644 teaql-core/src/main/java/io/teaql/core/FieldLayout.java create mode 100644 teaql-core/src/main/java/io/teaql/core/LoadState.java create mode 100644 teaql-core/src/test/java/io/teaql/core/DynamicMemberStateTest.java create mode 100644 teaql-core/src/test/java/io/teaql/core/LoadStateTest.java create mode 100644 teaql-dynamic-fields-api/src/main/java/io/teaql/data/dynamic/DynamicFieldMetadata.java create mode 100644 teaql-dynamic-fields-api/src/main/java/io/teaql/data/dynamic/DynamicFieldMutation.java create mode 100644 teaql-dynamic-fields-api/src/main/java/io/teaql/data/dynamic/DynamicGraphMutation.java create mode 100644 teaql-dynamic-fields-api/src/test/java/io/teaql/data/dynamic/DynamicFieldValueStateTest.java create mode 100644 teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/InheritedRelationOwnershipTest.java diff --git a/examples/shared-load-state/.gitignore b/examples/shared-load-state/.gitignore new file mode 100644 index 00000000..b83d2226 --- /dev/null +++ b/examples/shared-load-state/.gitignore @@ -0,0 +1 @@ +/target/ diff --git a/examples/shared-load-state/README.md b/examples/shared-load-state/README.md new file mode 100644 index 00000000..6221a0ab --- /dev/null +++ b/examples/shared-load-state/README.md @@ -0,0 +1,111 @@ +# Generated Java shared load state example + +This small SQLite acceptance example uses a newly generated School library and +the Java runtime source from the same Maven reactor. It does not install, +publish or substitute TeaQL jars from a public repository. + +## Run + +Requires Java 21 or newer, Maven, Bash, `rg`, `sha256sum`, and a local generator +checkout. Third-party dependencies may download on the first run. + +```bash +export TEAQL_CODEGEN_DIR=/path/to/teaql-code-gen +bash examples/shared-load-state/verify.sh --generate +``` + +The producer reads the current runtime version from its parent POM, generates +the library and Query/Create/Expression/Delete Assist into `target/generated`, +and never edits generated Java source. The `shared-load-state-example` Maven +profile builds the generated library and application-owned tests with the +runtime dependencies in one reactor. Omit `--generate` to replay that artifact. + +Use `bash examples/shared-load-state/verify.sh --generate --wide` for the overflow +gate. The producer adds 130 nullable School probes, producing 141 fixed slots, +and retains `model-under-test.xml` plus `fixture-mode.txt`. Both rounds assert +generated slots 0/31/32/63/64/65/129, actual NULL payloads, sparse NotLoaded and +overflow copy-on-write. A requested wide run rejects a narrow artifact; an +existing wide artifact automatically keeps the wide assertions enabled. + +Add `--inheritance` during generation to append one Academy subtype with a +campus code. The normal three-object model remains unchanged. The producer +retains a separate inheritance marker and the verifier selects an additional +Maven test source only for that generated artifact. A requested inherited run +rejects a flat artifact; replay preserves the inherited gate automatically. +The inherited gate checks shared parent indexes and separate type identity, +Q/E, actual snapshot sharing and governed create/update/delete with a fully +loaded object. Its qualification is separate from the plain School gate. + +Each invocation retains a unique temporary directory containing its SQLite file, +round logs and generated-source hashes. Both rounds use that same database +without cleanup. The test checks the actual class-loading locations for core, +runtime, JDBC, SQLite and the generated School: every one must resolve to local +reactor `target/classes`, not an installed jar. + +## Acceptance + +`GeneratedNestedGraphAssertions` also verifies a bounded Platform-to-Schools +view and School-to-Platform-to-Schools graph through generated Q/E and JSON. +Loaded empty lists serialize as arrays; unselected lists stay absent. A target +filter cannot erase the source's known FK or invent loaded target detail. +These cases are required in both retained rounds, not optional smoke output. + +- Repeated schema bootstrap preserves Platform 1 and SchoolType 1001/1002. +- Generated indexes and revision match the installed type layout. +- Compatible full and sparse lists share exact immutable snapshot references + within each shape, without sharing values or merging their load boundaries. +- Filling an omitted field detaches that row's state; changing an existing value + keeps the snapshot reference. Zero/false remain loaded values. +- Generated Q/E traverse forward relations, and the full-object Checker rejects + sparse saves. Audited create, update and soft delete preserve sibling values + and optimistic versions. +- A JDBC Dynamic Field provider uses the same SQL executor as native mutation. + Governed save distinguishes stored Value, explicit NULL and Delete. +- A held dynamic view rejects a changed storage profile before DML, keeps its + pending intent, and retries after restoring the original profile. Rebuilding + the provider over the same executor is valid; it does not change storage identity. +- Readonly `_name` coexists with native `name`; a missing dynamic property reads + null, and ordinary save does not persist derived properties. + +Partial child projections are valid for E reads, but the whole-graph Checker +does not accept them as complete writable objects. The example reloads the full +School before deletion rather than disabling that validation. + +The persistent ID allocator uses its framework database bridge. Application +business data is created through generated mutation APIs, not handwritten SQL. +Dynamic-field definitions are registered explicitly as provider metadata; each +round has its own test definition. + +This is a small generated-consumer gate, not an allocation or throughput benchmark. +The optional wide fixture covers compiled overflow entities. Nested extension +graphs, broader provider provenance, serialization and immutable internal-artifact +consumers remain separate gates. + +For a new object with no queried extension view, obtain definitions with +`context.dynamicFields().metadata(ownerType)` and install an empty +`DynamicFieldValues` using that snapshot before updating extensions. This reads +definitions, not owner values. Unbound hand-built metadata cannot be silently +assigned to the current storage during save. + +## State allocation probe + +```bash +bash examples/shared-load-state/verify-allocations.sh +``` + +The probe lives in the unnamed example module, so the core module gains no JVM +management dependency. It uses `ThreadMXBean` allocated-byte counters after +warmup, with 4/64/130 fixed fields and 1/100/10,000 operations. Fixture setup, +payloads, JDBC and output formatting are outside the counted region. Ordinary +sets and Map-backed code views are both checked. A value-free private raw-code +array is memoized in the shared snapshot; it neither assigns dynamic-field +indexes nor changes the snapshot's availability, equality or hash. Real selection +changes still detach, and caller-owned sets cannot mutate the saved shape. + +The script retains separate state and four-field hydration CSV/logs. After +generating a wide fixture, add `--wide` to measure actual generated School and +optional Academy hydration at 3/64/full selected fields and 1/100/10,000 rows. +This checks shared overflow snapshots, COW isolation and lazy ledgers, requires +unchanged generated-source hashes, and rejects a non-wide artifact. Prepared +inputs, JDBC and logging remain outside the measured hydration region. These +are not whole-query latency, retained heap, CPU saturation or ORM comparisons. diff --git a/examples/shared-load-state/benchmark-driver.sh b/examples/shared-load-state/benchmark-driver.sh new file mode 100644 index 00000000..069ed145 --- /dev/null +++ b/examples/shared-load-state/benchmark-driver.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -euo pipefail +runtime_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +run_dir="$(mktemp -d -t teaql-java-driver-evidence.XXXXXXXX)" +cd "$runtime_dir" +java -version >"$run_dir/toolchain.txt" 2>&1 +git rev-parse HEAD >"$run_dir/base-commit.txt" +sha256sum examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/{DriverQueryBenchmarkTest,HydrationAllocationTest}.java >"$run_dir/source.sha256" +# Requires the example's locally generated fixture, as for verify.sh. This +# CPU receipt includes Maven compilation/JIT and is not query CPU attribution. +/usr/bin/time -v -o "$run_dir/maven-cpu.txt" mvn -q -Pshared-load-state-example \ + -pl examples/shared-load-state -am -Dtest=DriverQueryBenchmarkTest \ + -Dsurefire.failIfNoSpecifiedTests=false -Dteaql.driverBenchmark=true test \ + >"$run_dir/results.log" 2>&1 +rg -F 'PASS matched JDBC typed results and shared snapshots' "$run_dir/results.log" +sha256sum --check --quiet "$run_dir/source.sha256" +printf 'Driver benchmark evidence: %s\n' "$run_dir" diff --git a/examples/shared-load-state/model.xml b/examples/shared-load-state/model.xml new file mode 100644 index 00000000..614f709c --- /dev/null +++ b/examples/shared-load-state/model.xml @@ -0,0 +1,14 @@ + + + + + <_value id="1001" name="Primary" code="PRIMARY" display_order="1"/> + <_value id="1002" name="Secondary" code="SECONDARY" display_order="2"/> + + + diff --git a/examples/shared-load-state/pom.xml b/examples/shared-load-state/pom.xml new file mode 100644 index 00000000..dd4d5cac --- /dev/null +++ b/examples/shared-load-state/pom.xml @@ -0,0 +1,34 @@ + + + 4.0.0 + + io.teaqlteaql-java-parent1.553-RELEASE + ../../pom.xml + + shared-load-state-example + 21 + + com.exampleschool-management-service-core0.0.1-SNAPSHOT + io.teaqlteaql-sqlite${project.version} + io.teaqlteaql-provider-jdbc${project.version} + io.teaqlteaql-dynamic-fields-jdbc${project.version} + io.teaqlteaql-jackson${project.version} + com.fasterxml.jackson.datatypejackson-datatype-jsr310test + org.xerialsqlite-jdbc3.53.2.1 + org.junit.jupiterjunit-jupitertest + + + + inherited-load-state-example + teaql.loadState.inheritancetrue + + org.codehaus.mojobuild-helper-maven-plugin3.6.1 + inherited-test-sourcegenerate-test-sources + add-test-source + src/inheritance/java + + + + + diff --git a/examples/shared-load-state/src/inheritance/java/com/example/schoolmanagementservice/GeneratedInheritedSchoolLoadStateTest.java b/examples/shared-load-state/src/inheritance/java/com/example/schoolmanagementservice/GeneratedInheritedSchoolLoadStateTest.java new file mode 100644 index 00000000..21f57175 --- /dev/null +++ b/examples/shared-load-state/src/inheritance/java/com/example/schoolmanagementservice/GeneratedInheritedSchoolLoadStateTest.java @@ -0,0 +1,122 @@ +package com.example.schoolmanagementservice; + +import com.example.schoolmanagementservice.academy.Academy; +import com.example.schoolmanagementservice.school.School; +import io.teaql.core.FieldLayout; +import io.teaql.core.UserContext; +import io.teaql.core.checker.CheckException; +import io.teaql.core.meta.SimpleEntityMetaFactory; +import io.teaql.core.sql.portable.IdSpaceIdGenerator; +import io.teaql.core.sqlite.SqliteDataServiceExecutor; +import io.teaql.provider.jdbc.JdbcSqlExecutor; +import io.teaql.runtime.TeaQLRuntime; +import java.time.LocalDate; +import java.time.LocalDateTime; +import org.sqlite.SQLiteDataSource; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.*; + +class GeneratedInheritedSchoolLoadStateTest { + @Test + void inheritedLayoutKeepsParentPositionsWithoutSharingTypeIdentity() { + FieldLayout parent = School.__TEAQL_FIELD_LAYOUT; + FieldLayout child = Academy.__TEAQL_FIELD_LAYOUT; + School.__TEAQL_FIXED_FIELD_INDEXES.forEach((name, index) -> + assertEquals(index, child.findIndex(name), "inherited field " + name)); + assertTrue(child.findIndex("campus_code") >= parent.indexes().size()); + assertNotEquals(parent.revision(), child.revision()); + assertNotSame(parent, child); + } + + @Test + void inheritedQEMutationAndSnapshots() { + String round = System.getenv("TEAQL_LOAD_STATE_ROUND"); + var dataSource = new SQLiteDataSource(); + dataSource.setUrl("jdbc:sqlite:" + System.getenv("TEAQL_LOAD_STATE_DATABASE")); + var sql = new JdbcSqlExecutor(dataSource); + var ids = new IdSpaceIdGenerator(new GeneratedSchoolLoadStateTest.DatabaseBridge(sql)); + ids.ensureIdSpaceTable(); + var service = new SqliteDataServiceExecutor("sqlite", sql, dataSource); + var runtime = TeaQLRuntime.builder().metadata(new SimpleEntityMetaFactory()) + .dataService("default", service).dataService("sqlite", service).idGenerationService(ids) + .build().install(GeneratedRuntimeModule.module()); + UserContext context = new GeneratedSchoolLoadStateTest.Context(runtime); + context.ensureSchema(); + assertTrue(runtime.getMetadata().resolveEntityDescriptor("Academy").getAuditMaskFields().contains("address")); + var platform = Q.platforms().withIdIs(1L).limit(1).comment("what: reuse the provisioned root") + .purpose("why: attach the inherited fixture").executeForOne(context); + String[] codes = {round + "-campus-a", round + "-campus-b"}; + for (String code : codes) { + var row = Q.academies().comment("what: create a controlled inherited entity") + .purpose("why: verify all inherited model fields").newEntity(context); + row.updatePlatform(platform); + row.updateSchoolTypeToPrimary(); + row.updateName(code); + row.updateAddress("Private inherited address"); + row.updateEstablishedDate(LocalDate.of(1995, 9, 1)); + row.updateStudentCapacity(0); + row.updateActive(false); + row.updateCreateTime(LocalDateTime.of(2023, 11, 14, 22, 13)); + row.updateUpdateTime(LocalDateTime.of(2023, 11, 14, 22, 13)); + row.updateCampusCode(code); + row.auditAs("create inherited state fixture").save(context); + } + var full = Q.academies().withCampusCodeIn(codes).selectSelfFields().orderByIdAscending().limit(2) + .comment("what: load two full inherited rows").purpose("why: inspect actual shared snapshots") + .executeForList(context); + assertEquals(2, full.size()); + var snapshot = full.get(0).__internalLoadState(); + assertSame(snapshot, full.get(1).__internalLoadState()); + assertEquals(codes[0], E.academy(full.get(0)).getName().eval()); + assertEquals(codes[0], E.academy(full.get(0)).getCampusCode().eval()); + assertEquals(0, E.academy(full.get(0)).getStudentCapacity().eval()); + assertEquals(false, E.academy(full.get(0)).isActive().eval()); + var streamRequest=Q.academiesWithMinimalFields().withCampusCodeIn(codes).selectCampusCode() + .orderByIdAscending().limit(2).comment("what: stream two sparse inherited rows") + .purpose("why: qualify typed cursor overflow and NotLoaded boundaries"); + try(var stream=context.executeForStream(streamRequest)) { + var streamed=stream.toList();assertEquals(2,streamed.size()); + assertSame(streamed.get(0).__internalLoadState(),streamed.get(1).__internalLoadState()); + assertEquals(codes[0],E.academy(streamed.get(0)).getCampusCode().eval()); + assertFalse(streamed.get(0).isPropertyLoaded("name")); + assertFalse(streamed.get(0).__internalHasMutationLedger()); + if("true".equals(System.getenv("TEAQL_LOAD_STATE_WIDE"))) + assertTrue(Academy.__TEAQL_FIXED_FIELD_INDEXES.get("campus_code")>=64); + } + if ("true".equals(System.getenv("TEAQL_LOAD_STATE_WIDE"))) { + for (int slot : new int[]{63, 64, 65, 129}) { + String field = School.__TEAQL_FIXED_FIELD_INDEXES.entrySet().stream() + .filter(entry -> entry.getValue() == slot).map(java.util.Map.Entry::getKey).findFirst().orElseThrow(); + assertTrue(snapshot.isLoaded(field)); + assertNull(full.get(0).__internalGet(School.__TEAQL_FIXED_FIELD_MAPPINGS.get(field).get(0))); + } + } + Long id = full.get(0).getId(); + var sparse = Q.academiesWithMinimalFields().withIdIs(id).selectCampusCode().limit(1) + .comment("what: load an incomplete inherited projection").purpose("why: retain full-object Checker enforcement") + .executeForOne(context); + assertFalse(sparse.__internalLoadState().isLoaded("name")); + assertThrows(CheckException.class, () -> sparse.auditAs("reject sparse inherited save").save(context)); + var changed = full.get(0); + Long siblingVersion = full.get(1).getVersion(); + String renamed = round + " renamed inherited entity"; + changed.updateName(renamed); + assertSame(snapshot, changed.__internalLoadState()); + changed.auditAs("change only one inherited object").save(context); + assertEquals(2L, changed.getVersion(), "inherited version readback must update the saved instance"); + assertEquals(siblingVersion, full.get(1).getVersion()); + assertEquals(codes[1], E.academy(full.get(1)).getName().eval()); + var stored = Q.academies().withIdIs(id).selectSelfFields().limit(1) + .comment("what: inspect the persisted inherited update").purpose("why: verify optimistic version and values") + .executeForOne(context); + assertEquals(2L, stored.getVersion()); + assertEquals(renamed, E.academy(stored).getName().eval()); + stored.markForDeletion().auditAs("delete only the changed inherited object").save(context); + var remaining = Q.academies().withCampusCodeIn(codes).limit(2) + .comment("what: inspect the independent inherited sibling").purpose("why: verify isolation after deletion") + .executeForList(context); + assertEquals(1, remaining.size()); + assertEquals(siblingVersion, remaining.get(0).getVersion()); + System.out.println("PASS generated Java inherited indexes Q/E/save and snapshot isolation " + round); + } +} diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/DriverQueryBenchmarkTest.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/DriverQueryBenchmarkTest.java new file mode 100644 index 00000000..ec0c81a7 --- /dev/null +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/DriverQueryBenchmarkTest.java @@ -0,0 +1,151 @@ +package com.example.schoolmanagementservice; + +import io.teaql.core.*; +import io.teaql.core.meta.SimpleEntityMetaFactory; +import io.teaql.core.sql.GenericSQLProperty; +import io.teaql.core.sql.SQLEntityDescriptor; +import io.teaql.core.sqlite.SqliteDataServiceExecutor; +import io.teaql.provider.jdbc.JdbcSqlExecutor; +import io.teaql.runtime.DefaultUserContext; +import io.teaql.runtime.TeaQLRuntime; +import java.lang.management.ManagementFactory; +import java.nio.file.Files; +import java.sql.*; +import java.util.*; +import java.util.function.Supplier; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; +import org.sqlite.SQLiteDataSource; +import static org.junit.jupiter.api.Assertions.*; + +/** Opt-in matched JDBC track. Both lanes open/close one connection per query. */ +@EnabledIfSystemProperty(named = "teaql.driverBenchmark", matches = "true") +class DriverQueryBenchmarkTest { + record Row(Long id, Long version, String name, String note) {} + record StatementInput(String sql, Object[] params) {} + record Sample(T value, long bytes, long nanos) {} + static volatile Object consumed; + static final class Capture extends JdbcSqlExecutor { + final List inputs = new ArrayList<>(); + Capture(SQLiteDataSource source) { super(source); } + @Override public List query(String sql, Object[] params, CompiledRowMapper mapper) { + inputs.add(new StatementInput(sql, params.clone())); + return super.query(sql, params, mapper); + } + } + static Sample measure(com.sun.management.ThreadMXBean bean, Supplier action) { + long thread = Thread.currentThread().threadId(); + long before = bean.getThreadAllocatedBytes(thread), start = System.nanoTime(); + T value = action.get(); + long nanos = System.nanoTime() - start; + consumed = value; + return new Sample<>(value, bean.getThreadAllocatedBytes(thread)-before, nanos); + } + static List nativeQuery(SQLiteDataSource source, StatementInput input, boolean full) { + try (Connection connection = source.getConnection(); + PreparedStatement statement = connection.prepareStatement(input.sql())) { + for (int i=0; i result = new ArrayList<>(); + while (cursor.next()) result.add(new Row(cursor.getLong(1), cursor.getLong(2), + cursor.getString(3), full ? cursor.getString(4) : null)); + return result; + } + } catch (SQLException error) { throw new IllegalStateException(error); } + } + static void report(String lane, boolean full, int count, List> samples) { + for (int i=0; i> nativeAction = () -> nativeQuery(source,input,full); + Supplier> runtimeAction = () -> executable.executeForList(context); + for (int i=0; i<20; i++) { consumed=nativeAction.get(); consumed=runtimeAction.get(); } + // Keep metrics, not all entity graphs: retaining 31 result lists + // would distort heap/GC pressure compared with a normal consumer. + List> nativeSamples = new ArrayList<>(); + List> runtimeSamples = new ArrayList<>(); + for (int i=0; i<31; i++) { + Sample> raw; + Sample> typed; + if (i%2==0) { raw=measure(bean,nativeAction); typed=measure(bean,runtimeAction); } + else { typed=measure(bean,runtimeAction); raw=measure(bean,nativeAction); } + assertEquals(count,raw.value().size()); assertEquals(count,typed.value().size()); + LoadState state = typed.value().get(0).__internalLoadState(); + for (int row=0; row(null,raw.bytes(),raw.nanos())); + runtimeSamples.add(new Sample<>(null,typed.bytes(),typed.nanos())); + } + report("jdbc",full,count,nativeSamples); report("teaql",full,count,runtimeSamples); + } + System.out.println("PASS matched JDBC typed results and shared snapshots; log-off; per-query connections"); + } +} diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedNestedGraphAssertions.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedNestedGraphAssertions.java new file mode 100644 index 00000000..1cf51cd2 --- /dev/null +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedNestedGraphAssertions.java @@ -0,0 +1,128 @@ +package com.example.schoolmanagementservice; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.teaql.core.UserContext; +import io.teaql.data.dynamic.DynamicFieldSelection; +import io.teaql.data.dynamic.DynamicFieldValue; +import java.time.LocalDate; +import java.time.LocalDateTime; +import static org.junit.jupiter.api.Assertions.*; + +/** Application-owned acceptance using current object/field Assist only. */ +final class GeneratedNestedGraphAssertions { + private GeneratedNestedGraphAssertions() {} + + static void verifyDynamic(UserContext context, Long schoolId, String renamed, + String second, String extension, ObjectMapper mapper) throws Exception { + String nullName = renamed + " Dynamic NULL Probe"; + var platform = Q.platforms().withIdIs(1L).limit(1) + .comment("what: reuse the full seeded root for the NULL probe") + .purpose("why: retain whole-object mutation validation").executeForOne(context); + var probe = Q.schools().comment("what: allocate a separate nullable extension probe") + .purpose("why: preserve ordinary sibling mutation isolation").newEntity(context); + probe.updatePlatform(platform); probe.updateSchoolTypeToPrimary(); + probe.updateName(nullName); probe.updateAddress("Controlled extension probe"); + probe.updateEstablishedDate(LocalDate.of(1995,9,1)); probe.updateStudentCapacity(0); probe.updateActive(false); + probe.updateCreateTime(LocalDateTime.of(2023,11,14,22,13)); probe.updateUpdateTime(LocalDateTime.of(2023,11,14,22,13)); + probe.auditAs("Create isolated nested extension probe").save(context); + var reload = Q.schools().withIdIs(probe.getId()); + reload.selectDynamicFieldsWith(new DynamicFieldSelection().selectString(extension)); + probe = reload.limit(1).comment("what: load storage-bound extension definitions") + .purpose("why: seed explicit NULL through audited mutation").executeForOne(context); + probe.updateDynamicField(extension, null); + probe.auditAs("Seed explicit NULL through the generated mutation API").save(context); + + var child = Q.schoolsWithMinimalFields().withNameIn(renamed, second, nullName).selectName(); + child.selectDynamicFieldsWith(new DynamicFieldSelection().selectString(extension)); + var loaded = Q.platformsWithMinimalFields().withIdIs(1L) + .selectSchoolListWith(child.orderByIdAscending().limit(3)).limit(1) + .comment("what: load Value, absent and NULL extensions on bounded children") + .purpose("why: verify nested row-owned dynamic field carriers").executeForOne(context); + var rows = loaded.getSchoolList(); + assertEquals(3, rows.size()); + assertEquals("Private extension", rows.get(0).dynamicFields().field(extension).value()); + assertEquals(DynamicFieldValue.State.NOT_LOADED, rows.get(1).dynamicFields().field(extension).state()); + assertEquals(DynamicFieldValue.State.NULL, rows.get(2).dynamicFields().field(extension).state()); + assertSame(rows.get(0).__internalLoadState(), rows.get(2).__internalLoadState()); + assertNotSame(rows.get(0).__internalLoadState(), rows.get(1).__internalLoadState()); + var json = mapper.readTree(mapper.writeValueAsString(loaded)); + assertEquals("Private extension", json.get("schoolList").get(0).get("#" + extension).asText()); + assertFalse(json.get("schoolList").get(1).has("#" + extension)); + assertTrue(json.get("schoolList").get(2).get("#" + extension).isNull()); + + var nestedChild = Q.schoolsWithMinimalFields().withNameIn(renamed, nullName).selectName(); + nestedChild.selectDynamicFieldsWith(new DynamicFieldSelection().selectString(extension)); + var nested = Q.schoolsWithMinimalFields().withIdIs(schoolId) + .selectPlatformWith(Q.platformsWithMinimalFields() + .selectSchoolListWith(nestedChild.orderByIdAscending().limit(2))) + .limit(1).comment("what: retain dynamic children through a forward ancestor") + .purpose("why: verify nested graph presentation").executeForOne(context); + json = mapper.readTree(mapper.writeValueAsString(nested)); + assertEquals("Private extension", json.get("platform").get("schoolList").get(0).get("#" + extension).asText()); + assertTrue(json.get("platform").get("schoolList").get(1).get("#" + extension).isNull()); + System.out.println("PASS generated Java nested dynamic Value/NULL/NotLoaded and shared snapshots"); + } + + static void verify(UserContext context, Long schoolId, String renamed, + String second, ObjectMapper mapper) throws Exception { + var platform = Q.platformsWithMinimalFields().withIdIs(1L) + .selectSchoolListWith(Q.schoolsWithMinimalFields().withNameIn(renamed, second) + .selectName().orderByIdAscending().limit(2)) + .limit(1).comment("what: load two selected Schools under the seeded Platform") + .purpose("why: verify generated reverse graph state and JSON").executeForOne(context); + assertTrue(platform.isPropertyLoaded("schoolList")); + var children = platform.getSchoolList(); + assertEquals(2, children.size()); + assertEquals(renamed, E.school(children.get(0)).getName().eval()); + assertEquals(second, E.school(children.get(1)).getName().eval()); + var state = children.get(0).__internalLoadState(); + assertSame(state, children.get(1).__internalLoadState()); + var json = mapper.readTree(mapper.writeValueAsString(platform)); + assertEquals(renamed, json.get("schoolList").get(0).get("name").asText()); + assertEquals(second, json.get("schoolList").get(1).get("name").asText()); + assertFalse(json.get("schoolList").get(0).has("address")); + assertFalse(json.get("schoolList").get(0).has("_original_values")); + assertSame(state, children.get(0).__internalLoadState()); + + var nested = Q.schoolsWithMinimalFields().withIdIs(schoolId) + .selectPlatformWith(Q.platformsWithMinimalFields() + .selectSchoolListWith(Q.schoolsWithMinimalFields().withNameIn(renamed, second) + .selectName().orderByIdAscending().limit(2))) + .limit(1).comment("what: load School to Platform to its bounded School list") + .purpose("why: verify forward ancestor and reverse descendant availability") + .executeForOne(context); + var parent = E.school(nested).getPlatform().eval(); + assertTrue(parent.isPropertyLoaded("schoolList")); + assertEquals(2, parent.getSchoolList().size()); + var nestedJson = mapper.readTree(mapper.writeValueAsString(nested)); + assertEquals(renamed, nestedJson.get("platform").get("schoolList").get(0).get("name").asText()); + assertEquals(second, nestedJson.get("platform").get("schoolList").get(1).get("name").asText()); + + var empty = Q.schoolTypesWithMinimalFields().withIdIs(1002L) + .selectSchoolListWith(Q.schoolsWithMinimalFields().selectName().orderByIdAscending().limit(2)) + .limit(1).comment("what: explicitly load the unused Secondary constant's Schools") + .purpose("why: distinguish loaded empty from omitted reverse detail").executeForOne(context); + assertTrue(empty.isPropertyLoaded("schoolList")); + assertNotNull(empty.getSchoolList()); + assertTrue(empty.getSchoolList().isEmpty()); + var emptyJson = mapper.readTree(mapper.writeValueAsString(empty)); + assertTrue(emptyJson.get("schoolList").isArray()); + assertEquals(0, emptyJson.get("schoolList").size()); + + var unselected = Q.schoolTypesWithMinimalFields().withIdIs(1002L) + .limit(1).comment("what: load the same constant without selecting its Schools") + .purpose("why: NotLoaded must not serialize as an empty list").executeForOne(context); + assertFalse(unselected.isPropertyLoaded("schoolList")); + assertFalse(mapper.readTree(mapper.writeValueAsString(unselected)).has("schoolList")); + + var filtered = Q.schoolsWithMinimalFields().withIdIs(schoolId) + .selectSchoolTypeWith(Q.schoolTypesWithMinimalFields().withIdIs(1002L)) + .limit(1).comment("what: retain Primary FK while excluding it from selected target detail") + .purpose("why: verify current filtered-forward NotLoaded semantics").executeForOne(context); + assertEquals(1001L, filtered.getSchoolType().getId()); + assertFalse(filtered.getSchoolType().isPropertyLoaded("code")); + var filteredJson = mapper.readTree(mapper.writeValueAsString(filtered)); + assertFalse(filteredJson.get("schoolType").has("code")); + System.out.println("PASS generated Java nested/reverse graph Q/E/JSON and Empty/NotLoaded isolation"); + } +} diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java new file mode 100644 index 00000000..8e8bd511 --- /dev/null +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java @@ -0,0 +1,282 @@ +package com.example.schoolmanagementservice; + +import com.example.schoolmanagementservice.school.School; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import io.teaql.jackson.TeaQLModule; +import io.teaql.core.BaseEntity; +import io.teaql.core.FieldLayout; +import io.teaql.core.LoadState; +import io.teaql.core.UserContext; +import io.teaql.core.checker.CheckException; +import io.teaql.core.meta.SimpleEntityMetaFactory; +import io.teaql.core.sql.portable.IdSpaceIdGenerator; +import io.teaql.core.sql.portable.TeaQLDatabase; +import io.teaql.core.sqlite.SqliteDataServiceExecutor; +import io.teaql.dataservice.sql.SqlExecutionAdapter; +import io.teaql.data.dynamic.DefaultDynamicFieldsFacade; +import io.teaql.data.dynamic.DynamicDataType; +import io.teaql.data.dynamic.DynamicFieldContext; +import io.teaql.data.dynamic.DynamicFieldException; +import io.teaql.data.dynamic.DynamicFieldDef; +import io.teaql.data.dynamic.DynamicFieldScope; +import io.teaql.data.dynamic.DynamicFieldSelection; +import io.teaql.data.dynamic.DynamicFieldValue; +import io.teaql.data.dynamic.DynamicFieldsFacade; +import io.teaql.data.dynamic.jdbc.JdbcDynamicFieldsProvider; +import io.teaql.provider.jdbc.JdbcSqlExecutor; +import io.teaql.runtime.DefaultUserContext; +import io.teaql.runtime.TeaQLRuntime; +import java.nio.file.Path; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.sqlite.SQLiteDataSource; +import static org.junit.jupiter.api.Assertions.*; + +/** Runtime-owned acceptance code; generated APIs come from current model-aware Assist. */ +class GeneratedSchoolLoadStateTest { + static final class Context extends DefaultUserContext implements SchoolManagementServiceUserContext { + Context(TeaQLRuntime runtime) { super(runtime); } + } + + @Test + void generatedSchoolUsesLocalRuntimeAndSharedState() throws Exception { + Path root = Path.of(System.getenv("TEAQL_LOCAL_RUNTIME_ROOT")).toRealPath(); + for (Class type : List.of(BaseEntity.class, LoadState.class, TeaQLRuntime.class, + JdbcSqlExecutor.class, SqliteDataServiceExecutor.class, TeaQLModule.class, School.class)) { + Path location = Path.of(type.getProtectionDomain().getCodeSource().getLocation().toURI()).toRealPath(); + assertTrue(location.startsWith(root), "Non-local dependency " + type + " from " + location); + assertTrue(location.endsWith(Path.of("target/classes")), "Expected reactor classes: " + location); + System.out.println("LOCAL_CLASS " + type.getName() + " " + location); + } + String round = System.getenv("TEAQL_LOAD_STATE_ROUND"); + SQLiteDataSource dataSource = new SQLiteDataSource(); + dataSource.setUrl("jdbc:sqlite:" + System.getenv("TEAQL_LOAD_STATE_DATABASE")); + JdbcSqlExecutor sql = new JdbcSqlExecutor(dataSource); + IdSpaceIdGenerator ids = new IdSpaceIdGenerator(new DatabaseBridge(sql)); + ids.ensureIdSpaceTable(); + var service = new SqliteDataServiceExecutor("sqlite", sql, dataSource); + TeaQLRuntime runtime = TeaQLRuntime.builder().metadata(new SimpleEntityMetaFactory()) + .dataService("default", service).dataService("sqlite", service) + .idGenerationService(ids).build().install(GeneratedRuntimeModule.module()); + UserContext context = new Context(runtime); + context.ensureSchema(); + context.ensureSchema(); + var constants = Q.schoolTypes().orderByIdAscending().limit(10) + .comment("what: read model-owned constants").purpose("why: verify repeat bootstrap") + .executeForList(context); + assertEquals(2, constants.size()); + assertEquals(1001L, constants.get(0).getId()); + assertEquals(1002L, constants.get(1).getId()); + var platform = Q.platforms().withIdIs(1L).limit(1) + .comment("what: reuse the model-owned root").purpose("why: attach the test Schools") + .executeForOne(context); + assertNotNull(platform); + String first = round + " First School"; + String second = round + " Second School"; + for (String name : List.of(first, second)) { + var school = Q.schools().comment("what: allocate a controlled School") + .purpose("why: verify the generated mutation contract").newEntity(context); + school.updatePlatform(platform); + school.updateSchoolTypeToPrimary(); + school.updateName(name); + school.updateAddress("Private row address"); + school.updateEstablishedDate(LocalDate.of(1995, 9, 1)); + school.updateStudentCapacity(0); + school.updateActive(false); + school.updateCreateTime(LocalDateTime.of(2023, 11, 14, 22, 13)); + school.updateUpdateTime(LocalDateTime.of(2023, 11, 14, 22, 13)); + school.auditAs("Create controlled shared-state probe").save(context); + } + var full = Q.schools().withNameIn(first, second).selectSelfFields() + .orderByIdAscending().limit(2).comment("what: read compatible full projections") + .purpose("why: inspect exact immutable state references").executeForList(context); + assertEquals(2, full.size()); + LoadState snapshot = full.get(0).__internalLoadState(); + assertSame(snapshot, full.get(1).__internalLoadState()); + FieldLayout layout = FieldLayout.forType(School.class); + assertSame(layout, snapshot.layout()); + assertEquals(School.__TEAQL_FIELD_LAYOUT_REVISION, layout.revision()); + School.__TEAQL_FIXED_FIELD_INDEXES.forEach((name, index) -> assertEquals(index, layout.findIndex(name))); + boolean wide = Boolean.parseBoolean(System.getenv("TEAQL_LOAD_STATE_WIDE")); + if (wide) { + assertTrue(School.__TEAQL_FIXED_FIELD_INDEXES.size() > 130); + for (int index : new int[]{0, 31, 32, 63, 64, 65, 129}) { + String name = School.__TEAQL_FIXED_FIELD_INDEXES.entrySet().stream() + .filter(entry -> entry.getValue() == index).findFirst().orElseThrow().getKey(); + assertTrue(full.get(0).isPropertyLoaded(name), "missing generated slot " + index); + if (index < 64) assertNotEquals(0L, snapshot.bits() & (1L << index)); + else assertTrue(snapshot.overflow().contains(index)); + if (index >= 31) { + assertTrue(name.startsWith("probe_"), "expected nullable boundary probe"); + assertNull(full.get(0).__internalGet(School.__TEAQL_FIXED_FIELD_MAPPINGS.get(name).get(0))); + } + } + String name = School.__TEAQL_FIXED_FIELD_INDEXES.entrySet().stream() + .filter(entry -> entry.getValue() == 64).findFirst().orElseThrow().getKey(); + LoadState detached = snapshot.withLoaded(name, false); + assertNotSame(snapshot, detached); + assertFalse(detached.isLoaded(name)); + assertTrue(full.get(1).isPropertyLoaded(name)); + System.out.println("WIDE generated Java fields=" + School.__TEAQL_FIXED_FIELD_INDEXES.size() + + " bit63/overflow64/65/129 loaded NULL and COW"); + } + assertEquals(0, E.school(full.get(0)).getStudentCapacity().eval()); + assertEquals(false, E.school(full.get(0)).isActive().eval()); + assertEquals(1001L, full.get(0).getSchoolType().getId()); + + var sparse = Q.schoolsWithMinimalFields().withNameIn(first, second).selectName() + .orderByIdAscending().limit(2).comment("what: read matching sparse projections") + .purpose("why: preserve separate field availability boundaries").executeForList(context); + assertEquals(2, sparse.size()); + LoadState sparseState = sparse.get(0).__internalLoadState(); + assertSame(sparseState, sparse.get(1).__internalLoadState()); + assertNotSame(snapshot, sparseState); + assertTrue(sparse.get(0).isPropertyLoaded("id")); + assertTrue(sparse.get(0).isPropertyLoaded("version")); + assertFalse(sparse.get(0).isPropertyLoaded("address")); + var mapper = new ObjectMapper().registerModule(TeaQLModule.INSTANCE) + .registerModule(new JavaTimeModule()).disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); + var sparseJson = mapper.readTree(mapper.writeValueAsString(sparse)); + assertEquals(first, sparseJson.get(0).get("name").asText()); + assertFalse(sparseJson.get(0).has("address"), "NotLoaded must not become serialized NULL"); + assertFalse(sparseJson.get(0).has("establishedDate")); + assertSame(sparseState, sparse.get(0).__internalLoadState()); + var fullJson = mapper.readTree(mapper.writeValueAsString(full)); + assertEquals("1995-09-01", fullJson.get(0).get("establishedDate").asText()); + assertEquals(0, fullJson.get(0).get("studentCapacity").asInt()); + assertFalse(fullJson.get(0).get("active").asBoolean()); + assertSame(snapshot, full.get(0).__internalLoadState()); + if (wide) { + for (int index : new int[]{31, 32, 63, 64, 65, 129}) { + String name = School.__TEAQL_FIXED_FIELD_INDEXES.entrySet().stream() + .filter(entry -> entry.getValue() == index).findFirst().orElseThrow().getKey(); + assertFalse(sparse.get(0).isPropertyLoaded(name), "sparse slot " + index + " must remain NotLoaded"); + } + } + assertEquals(first, E.school(sparse.get(0)).getName().eval()); + sparse.get(0).updateAddress("New private address"); + assertNotSame(sparseState, sparse.get(0).__internalLoadState()); + assertSame(sparseState, sparse.get(1).__internalLoadState()); + assertFalse(sparse.get(1).isPropertyLoaded("address")); + assertThrows(CheckException.class, () -> sparse.get(0) + .auditAs("Reject sparse whole-object update").save(context)); + + var changed = full.get(0); + Long originalVersion = changed.getVersion(); + String renamed = round + " Renamed School"; + changed.updateName(renamed); + assertSame(snapshot, changed.__internalLoadState()); + assertEquals(second, full.get(1).getName()); + changed.auditAs("Rename only one independently loaded row").save(context); + assertEquals(originalVersion + 1, changed.getVersion()); + var related = Q.schools().withIdIs(changed.getId()) + .selectPlatformWith(Q.platformsWithMinimalFields().selectName()) + .selectSchoolTypeWith(Q.schoolTypesWithMinimalFields().selectCode()) + .limit(1).comment("what: read the renamed School graph") + .purpose("why: verify generated Q and typed E relation traversal").executeForOne(context); + assertEquals("Campus Learning Platform", E.school(related).getPlatform().eval().getName()); + assertEquals("PRIMARY", E.school(related).getSchoolType().eval().getCode()); + GeneratedNestedGraphAssertions.verify(context, changed.getId(), renamed, second, mapper); + var provider = new JdbcDynamicFieldsProvider(sql); + provider.ensureSchema(); + String extension = "note_" + round; + DynamicFieldContext definitionContext = new DynamicFieldContext() { + public String scopeType() { return "GLOBAL"; } + public String scopeId() { return "default"; } + public String userId() { return "generated-example"; } + public String purpose() { return "register controlled example definitions"; } + public String comment() { return "what: register one persistent extension"; } + public boolean strictIntent() { return true; } + public long nextId(String type) { return ids.nextId(type); } + }; + DynamicFieldDef definition = new DynamicFieldDef(); + definition.setScope(DynamicFieldScope.global()); + definition.setOwnerType("School"); + definition.setCode(extension); + definition.setName(extension); + definition.setDataType(DynamicDataType.STRING); + provider.registerFieldDef(definitionContext, definition); + context.putAttribute(DynamicFieldsFacade.class.getName(), new DefaultDynamicFieldsFacade(provider)); + var extensionRequest = Q.schools().withIdIs(changed.getId()); + extensionRequest.selectDynamicFieldsWith(new DynamicFieldSelection().selectString(extension)); + var extended = extensionRequest.limit(1).comment("what: load persistent extension definitions") + .purpose("why: qualify the generated carrier and local provider").executeForOne(context); + assertEquals(DynamicFieldValue.State.NOT_LOADED, extended.dynamicFields().field(extension).state()); + assertNull(layout.findIndex("#" + extension)); + extended.addDynamicProperty("name", "readonly-derived-name"); + assertNull(layout.findIndex("_name")); + assertEquals(renamed, E.school(extended).getName().eval()); + assertEquals("readonly-derived-name", extended.getProperty("_name")); + assertNull(extended.getProperty("_missing_property")); + var beforeExtensionSave = mapper.readTree(mapper.writeValueAsString(extended)); + assertEquals(renamed, beforeExtensionSave.get("name").asText()); + assertEquals("readonly-derived-name", beforeExtensionSave.get("_name").asText()); + assertFalse(beforeExtensionSave.has("#" + extension)); + extended.updateDynamicField(extension, "Private extension"); + extended.auditAs("Persist the controlled extension through governed save").save(context); + assertEquals("Private extension", extended.dynamicFields().field(extension).value()); + var afterExtensionSave = mapper.readTree(mapper.writeValueAsString(extended)); + assertEquals("Private extension", afterExtensionSave.get("#" + extension).asText()); + assertEquals(renamed, afterExtensionSave.get("name").asText()); + assertFalse(afterExtensionSave.has("traceChain")); + assertFalse(afterExtensionSave.has("loadState")); + System.out.println("PASS generated Java namespace serialization and NotLoaded boundary"); + GeneratedNestedGraphAssertions.verifyDynamic(context, changed.getId(), renamed, second, extension, mapper); + var heldVersion = extended.getVersion(); + extended.updateDynamicField(extension, "must not cross storage profile"); + context.putAttribute(DynamicFieldsFacade.class.getName(), + new DefaultDynamicFieldsFacade(provider, DynamicFieldScope.of("PROFILE", "other"))); + var rejected = assertThrows(DynamicFieldException.class, + () -> extended.auditAs("Reject held dynamic view in another profile").save(context)); + assertEquals("DYNAMIC_FIELD_STORAGE_PROVENANCE_MISMATCH", rejected.errorCode()); + assertEquals(heldVersion, extended.getVersion()); + assertFalse(extended.__internalDynamicMutations().isEmpty()); + context.putAttribute(DynamicFieldsFacade.class.getName(), + new DefaultDynamicFieldsFacade(new JdbcDynamicFieldsProvider(sql))); + var stored = context.dynamicFields().comment("what: inspect original extension after rejected save") + .purpose("why: prove a changed profile cannot redirect held data") + .owner(extended.typeName(), extended.getId()) + .readAll(new DynamicFieldSelection().selectString(extension)); + assertEquals("Private extension", stored.getString(extension)); + extended.updateDynamicField(extension, null); + extended.auditAs("Persist explicit extension NULL").save(context); + assertEquals(DynamicFieldValue.State.NULL, extended.dynamicFields().field(extension).state()); + extended.deleteDynamicField(extension); + extended.auditAs("Delete the extension without confusing it with NULL").save(context); + assertEquals(DynamicFieldValue.State.NOT_LOADED, extended.dynamicFields().field(extension).state()); + assertEquals("readonly-derived-name", extended.getProperty("_name")); + System.out.println("PASS generated Java dynamic storage provenance and retry"); + // Partial related objects are valid for E reads, not for whole-graph Checker validation. + var forDeletion = Q.schools().withIdIs(changed.getId()).limit(1) + .comment("what: reload the full native School before deletion") + .purpose("why: preserve whole-object Checker validation without partial child projections") + .executeForOne(context); + assertEquals(renamed, E.school(forDeletion).getName().eval()); + assertNull(forDeletion.getProperty("_name"), "readonly dynamic properties must not be persisted by save"); + forDeletion.markForDeletion().auditAs("Soft-delete the selected probe").save(context); + var remaining = Q.schools().withNameIn(renamed, second).orderByIdAscending().limit(2) + .comment("what: inspect the surviving sibling").purpose("why: verify mutation isolation") + .executeForList(context); + assertEquals(1, remaining.size()); + assertEquals(second, remaining.get(0).getName()); + assertEquals(full.get(1).getVersion(), remaining.get(0).getVersion()); + System.out.println("PASS generated Java indexed Q/E/Checker/create/update/delete and snapshot sharing " + round); + } + + /** Framework ID-allocation bridge, not an application-owned persistence alternative. */ + record DatabaseBridge(SqlExecutionAdapter sql) implements TeaQLDatabase { + public List> query(String query, Object[] args) { return sql.queryForList(query, args); } + public int executeUpdate(String query, Object[] args) { return sql.update(query, args); } + public int[] batchUpdate(String query, List args) { return sql.batchUpdate(query, args); } + public void execute(String query) { sql.execute(query); } + public void executeInTransaction(Runnable action) { sql.executeInTransaction(action); } + public List> getTableColumns(String table) { + throw new UnsupportedOperationException("Schema inspection belongs to the dialect"); + } + } +} diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedWideHydrationAllocationTest.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedWideHydrationAllocationTest.java new file mode 100644 index 00000000..30bc9ae4 --- /dev/null +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedWideHydrationAllocationTest.java @@ -0,0 +1,124 @@ +package com.example.schoolmanagementservice; + +import io.teaql.core.*; +import io.teaql.core.meta.*; +import io.teaql.core.sql.portable.*; +import io.teaql.runtime.*; +import java.lang.management.ManagementFactory; +import java.time.*; +import java.util.*; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.*; + +/** Generated metadata and real repository mapping; input preparation is uncounted. */ +class GeneratedWideHydrationAllocationTest { + static volatile Object consumed; + static final class Request extends BaseRequest { + @SuppressWarnings("unchecked") + Request(EntityDescriptor descriptor, SimpleEntityMetaFactory metadata, List names, int size) { + super((Class)descriptor.getTargetType(), () -> (BaseEntity)descriptor.createEntity()); + bindMetadata(metadata); names.forEach(this::selectProperty); setSize(size); + internalComment("hydrate generated wide allocation rows"); + internalPurpose("verify shared overflow state without database or logging costs"); + } + @Override public String getTypeName() { return returnType().getSimpleName(); } + } + record Row(Object[] cells) implements DataRow { + public Object get(int index) { return cells[index-1]; } + @SuppressWarnings("unchecked") + public V get(int index, Class type) { return (V)get(index); } + } + static final class Database implements TeaQLDatabase { + List rows; + List> maps; + int compiledCalls; + int mapCalls; + public boolean supportsCompiledRowMapping() { return true; } + public List query(UserContext context,String sql,Object[] args,CompiledRowMapper mapper) { + compiledCalls++; var result = new ArrayList(rows.size()); + for (Row row : rows) result.add(mapper.map(row)); return result; + } + public List> query(String sql,Object[] args) { mapCalls++; return maps; } + public int executeUpdate(String sql,Object[] args) { throw new AssertionError("read-only fixture"); } + public int[] batchUpdate(String sql,List args) { throw new AssertionError("read-only fixture"); } + public void execute(String sql) { throw new AssertionError("no schema I/O in allocation fixture"); } + public void executeInTransaction(Runnable action) { throw new AssertionError("read-only fixture"); } + public List> getTableColumns(String table) { throw new AssertionError("metadata comes from generated module"); } + void prepare(List names,Object[] defaults,int count) { + rows = new ArrayList<>(count); maps = new ArrayList<>(count); + for (int i=1; i<=count; i++) { + Object[] cells = defaults.clone(); cells[0] = (long)i; + rows.add(new Row(cells)); Map map = new LinkedHashMap<>(); + for (int column=0; column type = property(descriptor,member).getType().javaType(); + if (member.startsWith("probe")) return null; + if (Entity.class.isAssignableFrom(type)) return 1L; + if (type == String.class) return "sample name"; + if (type == Long.class || type == long.class) return 1L; + if (type == Integer.class || type == int.class) return 0; + if (type == Boolean.class || type == boolean.class) return false; + if (type == LocalDate.class) return LocalDate.of(2000,1,1); + if (type == LocalDateTime.class) return LocalDateTime.of(2023,11,14,22,13); + throw new AssertionError("unexpected controlled fixture type " + type); + } + @Test void generatedWideHydrationKeepsOverflowShared() { + var metadata = new SimpleEntityMetaFactory(); + var runtime = TeaQLRuntime.builder().metadata(metadata).queryExecutionLogging(false).build() + .install(GeneratedRuntimeModule.module()); + var context = new DefaultUserContext(runtime); + var bean = (com.sun.management.ThreadMXBean)ManagementFactory.getThreadMXBean(); + assertTrue(bean.isThreadAllocatedMemorySupported()); bean.setThreadAllocatedMemoryEnabled(true); + System.out.println("case,entity,selected_fields,rows,allocated_bytes,elapsed_ns"); + for (EntityDescriptor descriptor : metadata.allEntityDescriptors()) { + if (!Set.of("School","Academy").contains(descriptor.getType())) continue; + var layout = FieldLayout.forType(descriptor.getTargetType()); + assertTrue(layout.indexes().size()>130, "requires generated wide fixture"); + var database = new Database(); + var repository = new PortableSQLRepository(descriptor,database,type -> null,metadata); + List ordered = layout.indexes().entrySet().stream().sorted(Map.Entry.comparingByValue()) + .map(entry -> layout.memberName(entry.getKey())).toList(); + for (int width : new int[]{3,64,layout.indexes().size()}) { + List names = new ArrayList<>(List.of("id","version","name")); + for (String name : ordered) { if (names.size()>=width) break; if (!names.contains(name)) names.add(name); } + Object[] defaults = names.stream().map(name -> cell(descriptor,name)).toArray(); + database.prepare(names,defaults,1); var warm = new Request(descriptor,metadata,names,1); + for (int i=0; i<1000; i++) consumed = repository.loadInternal(context,warm); + for (int count : new int[]{1,100,10_000}) { + database.prepare(names,defaults,count); var request = new Request(descriptor,metadata,names,count); + long thread = Thread.currentThread().getId(); long before = bean.getThreadAllocatedBytes(thread); + long start = System.nanoTime(); var entities = repository.loadInternal(context,request); consumed = entities; + long nanos = System.nanoTime()-start; long bytes = bean.getThreadAllocatedBytes(thread)-before; + System.out.printf("generated_wide_hydration,%s,%d,%d,%d,%d%n",descriptor.getType(),width,count,bytes,nanos); + assertEquals(count,entities.size()); LoadState state = entities.get(0).__internalLoadState(); + for (int i=0; i1) { + String probe = layout.indexes().entrySet().stream().filter(entry -> entry.getValue()==64) + .map(Map.Entry::getKey).findFirst().orElseThrow(); + LoadState changed = state.withLoaded(probe,!state.isLoaded(probe)); + assertNotSame(state,changed); assertSame(state,entities.get(1).__internalLoadState()); + } + } + } + assertEquals(descriptor.hasChildren(),database.compiledCalls==0); + assertEquals(!descriptor.hasChildren(),database.mapCalls==0); + } + System.out.println("PASS generated Java wide hydration shares one overflow snapshot per shape"); + } +} diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/HydrationAllocationTest.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/HydrationAllocationTest.java new file mode 100644 index 00000000..b81e41e9 --- /dev/null +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/HydrationAllocationTest.java @@ -0,0 +1,232 @@ +package com.example.schoolmanagementservice; + +import io.teaql.core.*; +import io.teaql.core.meta.SimpleEntityMetaFactory; +import io.teaql.core.sql.GenericSQLProperty; +import io.teaql.core.sql.SQLEntityDescriptor; +import io.teaql.core.sql.portable.PortableSQLRepository; +import io.teaql.core.sql.portable.TeaQLDatabase; +import io.teaql.runtime.DefaultUserContext; +import io.teaql.runtime.TeaQLRuntime; +import java.lang.management.ManagementFactory; +import java.util.*; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.*; + +/** Real repository row hydration with prepared inputs; no JDBC, logging or seeding. */ +class HydrationAllocationTest { + static volatile Object consumed; + static final class Probe extends BaseEntity { + static final FieldLayout LAYOUT = FieldLayout.installGenerated(FieldLayout.generated( + Probe.class, "hydration-probe-v1", Map.of("id",0,"version",1,"name",2,"note",3), + Map.of("id",List.of("id","id"),"version",List.of("version","version"), + "name",List.of("name","name"),"note",List.of("note","note")),Set.of())); + String name; + String note; + @Override public Object __internalGet(String field) { + return switch (field) { case "name" -> name; case "note" -> note; default -> super.__internalGet(field); }; + } + @Override public void __internalSet(String field, Object value) { + switch (field) { + case "name" -> { markPropertyLoaded(field); name = (String) value; } + case "note" -> { markPropertyLoaded(field); note = (String) value; } + default -> super.__internalSet(field,value); + } + } + } + static final class Request extends BaseRequest { + Request(SimpleEntityMetaFactory metadata, boolean full, int size) { + super(Probe.class, Probe::new); + bindMetadata(metadata); + for (String field : full ? List.of("id","version","name","note") : List.of("id","version","name")) selectProperty(field); + setSize(size); + internalComment("hydrate controlled allocation rows"); + internalPurpose("measure row payload versus shared availability allocation"); + } + @Override public String getTypeName() { return "Probe"; } + } + record Plain(Long id, Long version, String name, String note) {} + record Input(Long id, Long version, String name, String note) implements DataRow { + public Object get(int index) { return switch (index) { case 1 -> id; case 2 -> version; case 3 -> name; case 4 -> note; default -> throw new AssertionError(index); }; } + public V get(int index, Class type) { return type.cast(get(index)); } + } + static final class PreparedDatabase implements TeaQLDatabase { + boolean compiled; + List rows; + List> maps; + int compiledCalls; + int genericCalls; + int typedStreamCalls; + int mapStreamCalls; + boolean allocateStreamMaps; + void prepare(int size, boolean full) { + rows = new ArrayList<>(size); maps = new ArrayList<>(size); + for (int i=1; i<=size; i++) { + Input row = new Input((long)i,1L,"sample name",null); rows.add(row); + Map map = new LinkedHashMap<>(); + map.put("id",row.id()); map.put("version",row.version()); map.put("name",row.name()); + if (full) map.put("note",null); + maps.add(map); + } + } + public boolean supportsCompiledRowMapping() { return compiled; } + public List query(UserContext context, String sql, Object[] args, CompiledRowMapper mapper) { + compiledCalls++; + List result = new ArrayList<>(rows.size()); + for (Input row : rows) result.add(mapper.map(row)); + return result; + } + public List> query(String sql, Object[] args) { genericCalls++; return maps; } + public boolean supportsCompiledStreamMapping() { return compiled; } + public java.util.stream.Stream queryForStream(UserContext context, String sql, Object[] args, + CompiledRowMapper mapper, io.teaql.core.sql.portable.SqlLogBindings bindings) { + typedStreamCalls++; return rows.stream().map(mapper::map); + } + public java.util.stream.Stream> queryForStream(UserContext context, String sql, Object[] args) { + mapStreamCalls++; + // Explicit control for provider-side map creation; fixed input values + // remain prepared outside measurement in both lanes. + return allocateStreamMaps ? maps.stream().map(LinkedHashMap::new) : maps.stream(); + } + public int executeUpdate(String sql, Object[] args) { throw new AssertionError("read-only probe"); } + public int[] batchUpdate(String sql, List args) { throw new AssertionError("read-only probe"); } + public void execute(String sql) { throw new AssertionError("no schema I/O in measured probe"); } + public void executeInTransaction(Runnable action) { throw new AssertionError("read-only probe"); } + public List> getTableColumns(String table) { throw new AssertionError("metadata prepared outside measurement"); } + } + record Sample(Object result, long bytes, long nanos) {} + + @Test void compiledAndFallbackStreamsKeepSharedNullAndNotLoadedWithLessAllocation() { + assertNotNull(Probe.LAYOUT); + var metadata = new SimpleEntityMetaFactory(); + var descriptor = new SQLEntityDescriptor(); descriptor.setType("Probe"); + descriptor.setTargetType(Probe.class); descriptor.setEntitySupplier(Probe::new); + for (String field : List.of("id","version","name","note")) { + var property = (GenericSQLProperty) descriptor.addSimpleProperty(field, + field.equals("id") || field.equals("version") ? Long.class : String.class); + property.setColumnType(field.equals("id") || field.equals("version") ? "BIGINT" : "VARCHAR(255)"); + } + metadata.register(descriptor); + var database = new PreparedDatabase(); + database.allocateStreamMaps = true; + var repository = new PortableSQLRepository(descriptor,database,type -> null,metadata); + var context = new DefaultUserContext(TeaQLRuntime.builder().metadata(metadata).queryExecutionLogging(false).build()); + var bean = (com.sun.management.ThreadMXBean) ManagementFactory.getThreadMXBean(); + assertTrue(bean.isThreadAllocatedMemorySupported()); bean.setThreadAllocatedMemoryEnabled(true); + for (boolean full : List.of(false,true)) { + database.prepare(1,full); + var warm = new Request(metadata,full,1); + for (int i=0; i<2_000; i++) for (boolean compiled : List.of(false,true)) { + database.compiled=compiled; + try (var stream=repository.streamInternal(context,warm)) { consumed=stream.toList(); } + } + for (int count : new int[]{1,100,10_000}) { + database.prepare(count,full); + var request = new Request(metadata,full,count); + long compiledBytes=0, mapBytes=0; + for (boolean compiled : List.of(false,true)) { + database.compiled=compiled; + int typedBefore=database.typedStreamCalls, mapsBefore=database.mapStreamCalls; + Sample sample=measure(bean,() -> { try(var stream=repository.streamInternal(context,request)) {return stream.toList();} }); + @SuppressWarnings("unchecked") var rows=(List)sample.result(); + assertEquals(count,rows.size()); LoadState shape=rows.get(0).__internalLoadState(); + for (Probe row : rows) { + assertSame(shape,row.__internalLoadState()); assertEquals(full,row.isPropertyLoaded("note")); + assertNull(row.note); assertEquals("sample name",row.name); + assertFalse(row.__internalHasMutationLedger()); assertTrue(row.getUpdatedProperties().isEmpty()); + } + assertEquals(typedBefore+(compiled?1:0),database.typedStreamCalls); + assertEquals(mapsBefore+(compiled?0:1),database.mapStreamCalls); + if(compiled) compiledBytes=sample.bytes(); else mapBytes=sample.bytes(); + System.out.printf("STREAM_HYDRATION,%s,%s,%d,%d%n",compiled?"compiled":"map",full?"full":"sparse",count,sample.bytes()); + } + if(count==10_000) assertTrue(compiledBytes action) { + long thread = Thread.currentThread().getId(); + long before = bean.getThreadAllocatedBytes(thread); + long start = System.nanoTime(); + Object result = action.get(); consumed = result; + long nanos = System.nanoTime()-start; + long bytes = bean.getThreadAllocatedBytes(thread)-before; + return new Sample(result,bytes,nanos); + } + @Test void fullRepositoryHydrationAllocationProbe() { + assertNotNull(Probe.LAYOUT); // Generated modules install layouts before query planning. + var bean = (com.sun.management.ThreadMXBean) ManagementFactory.getThreadMXBean(); + assertTrue(bean.isThreadAllocatedMemorySupported()); bean.setThreadAllocatedMemoryEnabled(true); + var metadata = new SimpleEntityMetaFactory(); + var descriptor = new SQLEntityDescriptor(); descriptor.setType("Probe"); + descriptor.setTargetType(Probe.class); descriptor.setEntitySupplier(Probe::new); + for (String field : List.of("id","version","name","note")) { + var property = (GenericSQLProperty) descriptor.addSimpleProperty(field, + field.equals("id") || field.equals("version") ? Long.class : String.class); + property.setColumnType(field.equals("id") || field.equals("version") ? "BIGINT" : "VARCHAR(255)"); + } + metadata.register(descriptor); + var database = new PreparedDatabase(); + var repository = new PortableSQLRepository(descriptor,database,type -> null,metadata); + var context = new DefaultUserContext(TeaQLRuntime.builder().metadata(metadata).queryExecutionLogging(false).build()); + System.out.println("case,projection,rows,allocated_bytes,elapsed_ns"); + for (boolean full : List.of(false,true)) { + String shape = full ? "full" : "sparse"; + for (boolean compiled : List.of(true,false)) { + database.compiled = compiled; database.prepare(1,full); + var warm = new Request(metadata,full,1); + for (int i=0; i<4_000; i++) consumed = repository.loadInternal(context,warm); + for (int count : new int[]{1,100,10_000}) { + database.prepare(count,full); + var request = new Request(metadata,full,count); + int before = compiled ? database.compiledCalls : database.genericCalls; + var sample = measure(bean,() -> repository.loadInternal(context,request)); + System.out.printf("%s,%s,%d,%d,%d%n",compiled ? "compiled_hydration" : "map_hydration",shape,count,sample.bytes(),sample.nanos()); + assertEquals(before+1,compiled ? database.compiledCalls : database.genericCalls); + @SuppressWarnings("unchecked") var result = (SmartList) sample.result(); + assertEquals(count,result.size()); + LoadState state = result.get(0).__internalLoadState(); + for (int i=0; i { + List resultRows = new ArrayList<>(count); + for (Input row : database.rows) resultRows.add(new Plain(row.id(),row.version(),row.name(),row.note())); + return resultRows; + }); + System.out.printf("plain_hydration,%s,%d,%d,%d%n",shape,count,plain.bytes(),plain.nanos()); + assertEquals(count,((List)plain.result()).size()); + } + } + } + // Mixed row shapes must not turn absence into loaded NULL, including + // a parallel stream consumer racing only the value-free shape hint. + database.compiled = false; database.prepare(100, true); + for (int i=0; i codes) { + Set names = new java.util.HashSet<>(state.selectedNames()); + names.removeIf(name -> name.startsWith("#")); + for (String code : codes) names.add("#" + code); + if (!names.equals(state.selectedNames())) throw new AssertionError("reference accepts only unchanged geometry"); + return state; + } + + private static Sample measure(com.sun.management.ThreadMXBean bean, int count, Runnable action) { + long thread = Thread.currentThread().getId(); + long startBytes = bean.getThreadAllocatedBytes(thread); + long startTime = System.nanoTime(); + for (int i = 0; i < count; i++) action.run(); + long nanos = System.nanoTime() - startTime; + long bytes = bean.getThreadAllocatedBytes(thread) - startBytes; + return new Sample(bytes, nanos); + } + + @Test + public void loadedAvailabilityAllocationProbe() { + assertTrue(ManagementFactory.getThreadMXBean() instanceof com.sun.management.ThreadMXBean, "Requires a JVM thread-allocation counter"); + var bean = (com.sun.management.ThreadMXBean) ManagementFactory.getThreadMXBean(); + assertTrue(bean.isThreadAllocatedMemorySupported(), "JVM thread allocation counter must be supported"); + bean.setThreadAllocatedMemoryEnabled(true); + System.out.println("case,width,iterations,allocated_bytes,elapsed_ns"); + for (int width : new int[]{4, 64, 130}) { + Map indexes = new LinkedHashMap<>(); + Map> mappings = new LinkedHashMap<>(); + List names = new ArrayList<>(); + for (int i = 0; i < width; i++) { + String name = i == 0 ? "id" : i == 1 ? "version" : "field_" + i; + indexes.put(name, i); + mappings.put(name, List.of(name, name)); + names.add(name); + } + var layout = FieldLayout.generated(Probe.class, "width-" + width, indexes, mappings, Set.of("child_list")); + var base = LoadState.projection(layout, names); + Set codes = Set.of("note"); + var selected = base.withDynamicSelection(codes); + Runnable fixed = () -> consumed = base.withLoaded("id", true); + Runnable dynamic = () -> consumed = selected.withDynamicSelection(codes); + Set viewCodes = new java.util.HashMap<>(Map.of("note", "private payload")).keySet(); + Runnable dynamicView = () -> consumed = selected.withDynamicSelection(viewCodes); + Runnable reference = () -> consumed = referenceDynamicNoop(selected, codes); + for (int i = 0; i < 30_000; i++) { fixed.run(); dynamic.run(); dynamicView.run(); reference.run(); } + for (int iterations : new int[]{1, 100, 10_000}) { + Sample result = measure(bean, iterations, fixed); + System.out.printf("loaded_fixed_noop,%d,%d,%d,%d%n", width, iterations, result.bytes(), result.nanos()); + assertSame(base, consumed); + assertEquals(0, result.bytes(), "fixed value-only availability must not allocate"); + result = measure(bean, iterations, dynamic); + System.out.printf("loaded_dynamic_noop,%d,%d,%d,%d%n", width, iterations, result.bytes(), result.nanos()); + assertSame(selected, consumed); + assertEquals(0, result.bytes(), "unchanged dynamic selection must not allocate"); + result = measure(bean, iterations, dynamicView); + System.out.printf("loaded_dynamic_map_view_noop,%d,%d,%d,%d%n", width, iterations, result.bytes(), result.nanos()); + assertSame(selected, consumed); + assertEquals(0, result.bytes(), "map-backed selected-code views must not allocate iterators"); + result = measure(bean, iterations, reference); + System.out.printf("reference_dynamic_noop,%d,%d,%d,%d%n", width, iterations, result.bytes(), result.nanos()); + assertSame(selected, consumed); + } + } + } +} diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/TypedStreamAllocationTest.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/TypedStreamAllocationTest.java new file mode 100644 index 00000000..ffdad930 --- /dev/null +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/TypedStreamAllocationTest.java @@ -0,0 +1,99 @@ +package com.example.schoolmanagementservice; + +import io.teaql.core.*; +import io.teaql.core.meta.SimpleEntityMetaFactory; +import io.teaql.core.sql.GenericSQLProperty; +import io.teaql.core.sql.SQLEntityDescriptor; +import io.teaql.core.sqlite.SqliteDataServiceExecutor; +import io.teaql.provider.jdbc.JdbcSqlExecutor; +import io.teaql.runtime.DefaultUserContext; +import io.teaql.runtime.TeaQLRuntime; +import java.lang.management.ManagementFactory; +import java.nio.file.Files; +import java.util.*; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; +import org.sqlite.SQLiteDataSource; +import static org.junit.jupiter.api.Assertions.*; + +/** Real JDBC cursor allocations; no prepared Map inputs hide provider costs. */ +class TypedStreamAllocationTest { + static final class Adapter extends JdbcSqlExecutor { + boolean compiled; + int typedCalls, mapCalls; + Adapter(SQLiteDataSource source) { super(source); } + @Override public boolean supportsCompiledStreamMapping() { return compiled; } + @Override public Stream queryForStream(String sql,Object[] args,CompiledRowMapper mapper) { + typedCalls++; return super.queryForStream(sql,args,mapper); + } + @Override public Stream> queryForStream(String sql,Object[] args) { + mapCalls++; return super.queryForStream(sql,args); + } + } + + @Test void realSQLiteTypedStreamRemovesMapsWithoutChangingLoadedState() throws Exception { + assertNotNull(HydrationAllocationTest.Probe.LAYOUT); + var metadata=new SimpleEntityMetaFactory(); var descriptor=new SQLEntityDescriptor(); + descriptor.setType("Probe"); descriptor.setTargetType(HydrationAllocationTest.Probe.class); + descriptor.setEntitySupplier(HydrationAllocationTest.Probe::new); + for(String field:List.of("id","version","name","note")) { + var property=(GenericSQLProperty)descriptor.addSimpleProperty(field, + field.equals("id")||field.equals("version")?Long.class:String.class); + property.setColumnType(field.equals("id")||field.equals("version")?"BIGINT":"VARCHAR(255)"); + } + metadata.register(descriptor); + var source=new SQLiteDataSource(); + var path=Files.createTempDirectory("teaql-java-typed-stream-").resolve("probe.sqlite"); + source.setUrl("jdbc:sqlite:"+path); System.out.println("STREAM_DATABASE "+path); + var adapter=new Adapter(source); + var service=new SqliteDataServiceExecutor("sqlite",adapter,source); + var context=new DefaultUserContext(TeaQLRuntime.builder().metadata(metadata) + .dataService("default",service).queryExecutionLogging(false).build()); + context.ensureSchema(); + // Synthetic allocation input only; not a business/bootstrap mutation + // alternative. Fixture construction is excluded from measurements. + try(var connection=source.getConnection()) { + connection.setAutoCommit(false); + try(var statement=connection.prepareStatement("INSERT INTO probe_data(id,version,name,note) VALUES(?,?,?,?)")) { + for(int id=1;id<=10_000;id++) { + statement.setLong(1,id);statement.setLong(2,1);statement.setString(3,"row-"+id); + statement.setString(4,id%2==0?null:"nullable note");statement.addBatch(); + } + statement.executeBatch(); + } + connection.commit(); + } + var bean=(com.sun.management.ThreadMXBean)ManagementFactory.getThreadMXBean(); + assertTrue(bean.isThreadAllocatedMemorySupported()); bean.setThreadAllocatedMemoryEnabled(true); + for(boolean full:List.of(false,true)) { + var warm=new HydrationAllocationTest.Request(metadata,full,1); + var executable=warm.purpose("why: warm both cursor hydration paths"); + for(int i=0;i<300;i++) for(boolean compiled:List.of(false,true)) { + adapter.compiled=compiled; try(var stream=executable.executeForStream(context)) {HydrationAllocationTest.consumed=stream.toList();} + } + for(int count:new int[]{1,100,10_000}) { + var request=new HydrationAllocationTest.Request(metadata,full,count);request.addOrderByAscending("id"); + var query=request.purpose("why: isolate real provider Map allocation from typed hydration"); + long typedBytes=0,mapBytes=0; + for(boolean compiled:List.of(false,true)) { + adapter.compiled=compiled; int typedBefore=adapter.typedCalls,mapBefore=adapter.mapCalls; + var sample=HydrationAllocationTest.measure(bean,()->{try(var stream=query.executeForStream(context)){return stream.toList();}}); + @SuppressWarnings("unchecked") var rows=(List)sample.result(); + assertEquals(count,rows.size());var state=rows.get(0).__internalLoadState(); + for(int i=0;i&2; exit 2 ;; +esac +if (( $# > 1 )); then printf 'Unexpected arguments\n' >&2; exit 2; fi +if [[ ! -f "$example_dir/target/generated/lib/pom.xml" ]]; then + printf 'Run the generated example verification with --generate first.\n' >&2 + exit 1 +fi +if [[ "$wide" == true ]]; then + mode="" + IFS= read -r mode <"$example_dir/target/generated/fixture-mode.txt" + [[ "$mode" == wide ]] || { printf 'Generate a wide fixture before wide allocation verification\n' >&2; exit 1; } +fi +run_dir="$(mktemp -d -t teaql-java-state-allocations.XXXXXXXX)" +printf 'Evidence retained at %s\n' "$run_dir" +java -version >"$run_dir/toolchain.txt" 2>&1 +rg --files --hidden --no-ignore "$example_dir/target/generated/lib/src" -0 | + sort -z | xargs -0 sha256sum >"$run_dir/generated-source.sha256" +mvn -q -f "$runtime_dir/pom.xml" -Pshared-load-state-example \ + -pl examples/shared-load-state -am "-Dtest=$test_targets" \ + -Dsurefire.failIfNoSpecifiedTests=false test >"$run_dir/probe.log" 2>&1 || { + tail -80 "$run_dir/probe.log" >&2 + exit 1 + } +printf 'case,width,iterations,allocated_bytes,elapsed_ns\n' >"$run_dir/allocations.csv" +rg '^(loaded_|reference_)' "$run_dir/probe.log" >>"$run_dir/allocations.csv" +printf 'case,projection,rows,allocated_bytes,elapsed_ns\n' >"$run_dir/hydration.csv" +rg '^(compiled_hydration|map_hydration|plain_hydration),' "$run_dir/probe.log" >>"$run_dir/hydration.csv" +rg -F 'PASS mixed map/parallel-stream projection and dynamic-property NULL presence' "$run_dir/probe.log" +if [[ "$wide" == true ]]; then + printf 'case,entity,selected_fields,rows,allocated_bytes,elapsed_ns\n' >"$run_dir/wide-hydration.csv" + rg '^generated_wide_hydration,' "$run_dir/probe.log" >>"$run_dir/wide-hydration.csv" + rg -F 'PASS generated Java wide hydration shares one overflow snapshot per shape' "$run_dir/probe.log" +fi +sha256sum --check --quiet "$run_dir/generated-source.sha256" +printf 'PASS Java warmed load-state allocation probe\n' diff --git a/examples/shared-load-state/verify.sh b/examples/shared-load-state/verify.sh new file mode 100644 index 00000000..d34deb8d --- /dev/null +++ b/examples/shared-load-state/verify.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +set -euo pipefail +example_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +runtime_dir="$(cd "$example_dir/../.." && pwd)" +generate=false +wide=false +inheritance=false +for argument in "$@"; do + case "$argument" in + --generate) generate=true ;; + --wide) wide=true ;; + --inheritance) inheritance=true ;; + *) printf 'Usage: bash verify.sh [--generate] [--wide] [--inheritance]\n' >&2; exit 2 ;; + esac +done +if [[ "$generate" == true ]]; then + : "${TEAQL_CODEGEN_DIR:?Set TEAQL_CODEGEN_DIR to the local generator checkout}" + mvn -q -f "$TEAQL_CODEGEN_DIR/pom.xml" -pl generator -am \ + '-Dtest=JavaGeneratedMysqlSqliteIntegrationTest#generatedJavaSharedLoadStateRuntimeExample' \ + -Dsurefire.failIfNoSpecifiedTests=false -Dteaql.loadState.generateOnly=true "-Dteaql.loadState.wide=$wide" "-Dteaql.loadState.inheritance=$inheritance" \ + "-Dteaql.java.dir=$runtime_dir" test +fi +fixture_mode=narrow +if [[ -f "$example_dir/target/generated/fixture-mode.txt" ]]; then + IFS= read -r fixture_mode <"$example_dir/target/generated/fixture-mode.txt" +fi +if [[ "$wide" == true && "$fixture_mode" != wide ]]; then + printf 'Wide verification requires a newly generated wide fixture.\n' >&2 + exit 1 +fi +case "$fixture_mode" in + wide) export TEAQL_LOAD_STATE_WIDE=true ;; + narrow) export TEAQL_LOAD_STATE_WIDE=false ;; + *) printf 'Invalid generated fixture mode\n' >&2; exit 1 ;; +esac +fixture_inheritance=false +if [[ -f "$example_dir/target/generated/fixture-inheritance.txt" ]]; then + IFS= read -r fixture_inheritance <"$example_dir/target/generated/fixture-inheritance.txt" +fi +if [[ "$inheritance" == true && "$fixture_inheritance" != true ]]; then + printf 'Inheritance verification requires a newly generated inherited fixture.\n' >&2 + exit 1 +fi +test_targets=GeneratedSchoolLoadStateTest,TypedStreamAllocationTest +case "$fixture_inheritance" in + true) test_targets+=,GeneratedInheritedSchoolLoadStateTest ;; + false) ;; + *) printf 'Invalid generated inheritance mode\n' >&2; exit 1 ;; +esac +if [[ ! -f "$example_dir/target/generated/lib/pom.xml" ]]; then + printf 'Generate first with TEAQL_CODEGEN_DIR= bash verify.sh --generate\n' >&2 + exit 1 +fi +run_dir="$(mktemp -d -t teaql-java-generated-state.XXXXXXXX)" +printf 'Evidence retained at %s\n' "$run_dir" +export TEAQL_LOCAL_RUNTIME_ROOT="$runtime_dir" +export TEAQL_LOAD_STATE_DATABASE="$run_dir/school.sqlite" +rg --files --hidden --no-ignore "$example_dir/target/generated/lib/src" -0 | + sort -z | xargs -0 sha256sum >"$run_dir/generated-source.sha256" +for round in first second; do + export TEAQL_LOAD_STATE_ROUND="$round" + mvn -q -f "$runtime_dir/pom.xml" -Pshared-load-state-example \ + -pl examples/shared-load-state -am \ + "-Dtest=$test_targets" "-Dteaql.loadState.inheritance=$fixture_inheritance" -Dsurefire.failIfNoSpecifiedTests=false test \ + >"$run_dir/$round.log" 2>&1 || { + tail -80 "$run_dir/$round.log" >&2 + exit 1 + } + rg -F "PASS generated Java indexed Q/E/Checker/create/update/delete and snapshot sharing $round" "$run_dir/$round.log" + rg -F "PASS generated Java dynamic storage provenance and retry" "$run_dir/$round.log" + rg -F "PASS generated Java namespace serialization and NotLoaded boundary" "$run_dir/$round.log" + rg -F "PASS generated Java nested/reverse graph Q/E/JSON and Empty/NotLoaded isolation" "$run_dir/$round.log" + rg -F "PASS generated Java nested dynamic Value/NULL/NotLoaded and shared snapshots" "$run_dir/$round.log" + rg -F "PASS Java real typed stream loaded state and provider allocation controls" "$run_dir/$round.log" + if [[ "$fixture_inheritance" == true ]]; then + rg -F "PASS generated Java inherited indexes Q/E/save and snapshot isolation $round" "$run_dir/$round.log" + fi +done +sha256sum --check --quiet "$run_dir/generated-source.sha256" +printf 'PASS fresh generated Java shared-load-state twice\n' diff --git a/examples/verify-runtime-examples.sh b/examples/verify-runtime-examples.sh index 883e4e63..2e072eab 100755 --- a/examples/verify-runtime-examples.sh +++ b/examples/verify-runtime-examples.sh @@ -85,4 +85,9 @@ for repetition in 1 2; do done bash "$repo_dir/examples/trace-chain/verify.sh" -printf 'PASS Java runtime examples: 3/3\n' +if [[ -n "${TEAQL_CODEGEN_DIR:-}" ]]; then + bash "$repo_dir/examples/shared-load-state/verify.sh" --generate +else + bash "$repo_dir/examples/shared-load-state/verify.sh" +fi +printf 'PASS Java runtime examples: 4/4\n' diff --git a/pom.xml b/pom.xml index 33fbe6c5..f9bca17e 100644 --- a/pom.xml +++ b/pom.xml @@ -360,6 +360,13 @@ + + shared-load-state-example + + examples/shared-load-state/target/generated/lib + examples/shared-load-state + + runtime-examples diff --git a/teaql-core/src/main/java/io/teaql/core/BaseEntity.java b/teaql-core/src/main/java/io/teaql/core/BaseEntity.java index b31a0139..aae1a1f8 100644 --- a/teaql-core/src/main/java/io/teaql/core/BaseEntity.java +++ b/teaql-core/src/main/java/io/teaql/core/BaseEntity.java @@ -24,23 +24,17 @@ public class BaseEntity implements Entity { private String displayName; - private Map updatedProperties = new ConcurrentHashMap<>(); + private static final Map NO_UPDATES = Map.of(); + private Map updatedProperties = NO_UPDATES; - private static final ClassValue LOADED_PROPERTY_LAYOUTS = - new ClassValue<>() { - @Override - protected LoadedPropertyLayout computeValue(Class type) { - return new LoadedPropertyLayout(); - } - }; - - private long loadedPropertyBits; - private Set overflowLoadedProperties; + private LoadState loadState = FieldLayout.forType(getClass()).emptyState(); private boolean hydratingProperty; - private Map additionalInfo = new ConcurrentHashMap<>(); + private Map additionalInfo = Map.of(); private DynamicFieldValues dynamicFieldValues; + private Map dynamicMutations = Map.of(); + private Map dynamicOriginalValues = Map.of(); // Query-only sidecar. Never a model property or mutation-ledger entry. private transient Map> queryFacets = Map.of(); @@ -56,7 +50,7 @@ public void __internalSetQueryFacets(Map> facets) queryFacets = facets == null || facets.isEmpty() ? Map.of() : Map.copyOf(facets); } - private Map relationCache = new HashMap<>(); + private Map relationCache = Map.of(); private List actionList; @@ -65,7 +59,9 @@ public void __internalSetQueryFacets(Map> facets) /** * Shared change tracking root for the entire entity graph. */ - private EntityMutationLedger entityMutationLedger = new EntityMutationLedger(); + private EntityMutationLedger entityMutationLedger; + // Database baseline is a row-owned scalar, not a reason to allocate a ledger. + private Long hydratedOriginalVersion; @Override public String getComment() { @@ -126,7 +122,7 @@ public void __internalInitializeNewEntityId(Long fixedId) { } id = fixedId; markPropertyLoaded(ID_PROPERTY); - entityMutationLedger.markAsNew(new EntityKey(typeName(), fixedId)); + getEntityMutationLedger().markAsNew(new EntityKey(typeName(), fixedId)); } @Override @@ -158,6 +154,7 @@ public void __internalRestorePersistenceState( Long restoredVersion, EntityStatus restoredStatus, boolean versionWasLoaded) { this.version = restoredVersion; this.$status = restoredStatus; + if (hydratedOriginalVersion != null) hydratedOriginalVersion = restoredVersion; restorePropertyLoaded(VERSION_PROPERTY, versionWasLoaded); } @@ -238,7 +235,13 @@ public List getUpdatedProperties() { EntityKey key = new EntityKey(typeName(), id); Set rootChanges = entityMutationLedger.changedFieldNames(key); if (rootChanges != null && !rootChanges.isEmpty()) { - return new ArrayList<>(rootChanges); + // Native SQL/checker callers consume predefined properties. + // dirtyFields()/the ledger expose # extension intents separately. + var nativeChanges = new ArrayList(); + for (String field : rootChanges) if (!field.startsWith("#")) nativeChanges.add(field); + if (nativeChanges.isEmpty()) return new ArrayList<>(updatedProperties.keySet()); + if (updatedProperties.containsKey(VERSION_PROPERTY) && !nativeChanges.contains(VERSION_PROPERTY)) nativeChanges.add(VERSION_PROPERTY); + return nativeChanges; } } return new ArrayList<>(updatedProperties.keySet()); @@ -303,8 +306,7 @@ private void addRelation( @Override public void addDynamicProperty(String propertyName, Object value) { - if (value == null) return; - additionalInfo.put(dynamicPropertyNameOf(propertyName), value); + mutableAdditionalInfo().put(dynamicPropertyNameOf(propertyName), value); } @Override @@ -313,7 +315,7 @@ public void appendDynamicProperty(String propertyName, Object value) { List existing = (List) additionalInfo.get(key); if (existing == null) { existing = new ArrayList<>(); - additionalInfo.put(key, existing); + mutableAdditionalInfo().put(key, existing); } existing.add(value); } @@ -325,17 +327,20 @@ public T getDynamicProperty(String propertyName) { } private String dynamicPropertyNameOf(String propertyName) { - if (propertyName.startsWith("#")) { + if (propertyName == null || propertyName.isBlank() || propertyName.startsWith("#")) { + throw new IllegalArgumentException("Dynamic properties use '_' names; '#' belongs to persistent dynamic fields"); + } + if (propertyName.startsWith("_")) { return propertyName; } - return "#" + propertyName; + return "_" + propertyName; } @Override public BaseEntity markForDeletion() { gotoNextStatus(EntityAction.DELETE); - if (entityMutationLedger != null && id != null) { - entityMutationLedger.markAsDelete(new EntityKey(typeName(), id)); + if (id != null) { + getEntityMutationLedger().markAsDelete(new EntityKey(typeName(), id)); } return this; } @@ -343,8 +348,8 @@ public BaseEntity markForDeletion() { @Override public void markAsRecover() { gotoNextStatus(EntityAction.RECOVER); - if (entityMutationLedger != null && id != null) { - entityMutationLedger.markAsRecover(new EntityKey(typeName(), id)); + if (id != null) { + getEntityMutationLedger().markAsRecover(new EntityKey(typeName(), id)); } } @@ -354,7 +359,7 @@ public boolean recoverItem() { } public void clearUpdatedProperties() { - this.updatedProperties.clear(); + if (!updatedProperties.isEmpty()) updatedProperties.clear(); } public void addAction(Object action) { @@ -391,9 +396,24 @@ public void setDisplayName(String pDisplayName) { // --- EntityMutationLedger integration --- public EntityMutationLedger getEntityMutationLedger() { + if (entityMutationLedger == null) { + entityMutationLedger = new EntityMutationLedger(); + entityMutationLedger.setComment(_comment); + if (id != null) { + EntityKey key = new EntityKey(typeName(), id); + // New-entity intent is installed by its explicit lifecycle entry + // point, not invented merely by requesting an empty ledger. + if (hydratedOriginalVersion != null) entityMutationLedger.setOriginalVersion(key, hydratedOriginalVersion); + if (!_traceChain.isEmpty()) entityMutationLedger.setTraceChain(key, _traceChain); + } + } return entityMutationLedger; } + @FrameworkInternal("Allocation and hydration qualification only") + @com.fasterxml.jackson.annotation.JsonIgnore + public boolean __internalHasMutationLedger() { return entityMutationLedger != null; } + public void setEntityMutationLedger(EntityMutationLedger entityMutationLedger) { this.entityMutationLedger = entityMutationLedger; if (entityMutationLedger != null && id != null && newItem()) { @@ -430,8 +450,9 @@ public boolean isNew() { } public Long getOriginalVersion() { - if (entityMutationLedger == null || id == null) { - return null; + if (id == null) return null; + if (entityMutationLedger == null) { + return hydratedOriginalVersion; } return entityMutationLedger.getOriginalVersion(new EntityKey(typeName(), id)); } @@ -445,32 +466,42 @@ public void setProperty(String propertyName, Object value) { @FrameworkInternal("Expression and hydration infrastructure only") public void markPropertyLoaded(String propertyName) { if (propertyName == null || hydratingProperty) return; - markPropertyLoaded(loadedPropertyIndex(getClass(), propertyName), propertyName); + loadState = loadState.withLoaded(propertyName, true); } public boolean isPropertyLoaded(String propertyName) { - if (propertyName == null) return false; - Integer index = LOADED_PROPERTY_LAYOUTS.get(getClass()).find(propertyName); - if (index == null) return overflowLoadedProperties != null - && overflowLoadedProperties.contains(propertyName); - if (index < Long.SIZE) return (loadedPropertyBits & (1L << index)) != 0; - return overflowLoadedProperties != null && overflowLoadedProperties.contains(propertyName); + return loadState.isLoaded(propertyName); } @FrameworkInternal("Compiled hydration infrastructure only") public static int loadedPropertyIndex(Class entityType, String propertyName) { - return LOADED_PROPERTY_LAYOUTS.get(entityType).index(propertyName); + return FieldLayout.forType(entityType).indexForLoad(propertyName); + } + + @FrameworkInternal("Immutable loaded-state sharing and hydration only") + @com.fasterxml.jackson.annotation.JsonIgnore + public LoadState __internalLoadState() { return loadState; } + + @FrameworkInternal("Immutable loaded-state sharing and hydration only") + public void __internalUseLoadState(LoadState state) { + if (state == null || state.layout() != FieldLayout.forType(getClass())) { + throw new IllegalArgumentException("Incompatible entity load-state layout: " + typeName()); + } + loadState = state; } @FrameworkInternal("Compiled hydration infrastructure only") public void __internalHydrate(String propertyName, Object value, int loadedPropertyIndex) { + if (loadState.layout().indexForLoad(propertyName) != loadedPropertyIndex) { + throw new IllegalArgumentException("Hydration index disagrees with generated layout: " + propertyName); + } hydratingProperty = true; try { __internalSet(propertyName, value); } finally { hydratingProperty = false; } - markPropertyLoaded(loadedPropertyIndex, propertyName); + loadState = loadState.withLoaded(propertyName, true); captureHydratedOriginalVersion(); } @@ -480,53 +511,13 @@ public void __internalHydrate(String propertyName, Object value, int loadedPrope * entity-owned Mutation Ledger without creating a business mutation. */ private void captureHydratedOriginalVersion() { - if (entityMutationLedger == null || id == null || version == null) return; - entityMutationLedger.setOriginalVersion(new EntityKey(typeName(), id), version); - } - - private void markPropertyLoaded(int index, String propertyName) { - if (index < Long.SIZE) { - loadedPropertyBits |= 1L << index; - return; - } - if (overflowLoadedProperties == null) overflowLoadedProperties = new java.util.HashSet<>(); - overflowLoadedProperties.add(propertyName); + if (id == null || version == null) return; + hydratedOriginalVersion = version; + if (entityMutationLedger != null) entityMutationLedger.setOriginalVersion(new EntityKey(typeName(), id), version); } private void restorePropertyLoaded(String propertyName, boolean loaded) { - Integer index = LOADED_PROPERTY_LAYOUTS.get(getClass()).find(propertyName); - if (index == null) { - if (loaded) { - if (overflowLoadedProperties == null) overflowLoadedProperties = new java.util.HashSet<>(); - overflowLoadedProperties.add(propertyName); - } else if (overflowLoadedProperties != null) { - overflowLoadedProperties.remove(propertyName); - } - return; - } - if (index < Long.SIZE) { - if (loaded) loadedPropertyBits |= 1L << index; - else loadedPropertyBits &= ~(1L << index); - } else if (loaded) { - if (overflowLoadedProperties == null) overflowLoadedProperties = new java.util.HashSet<>(); - overflowLoadedProperties.add(propertyName); - } else if (overflowLoadedProperties != null) { - overflowLoadedProperties.remove(propertyName); - } - } - - private static final class LoadedPropertyLayout { - private final ConcurrentHashMap indexes = new ConcurrentHashMap<>(); - private final java.util.concurrent.atomic.AtomicInteger nextIndex = - new java.util.concurrent.atomic.AtomicInteger(); - - int index(String propertyName) { - return indexes.computeIfAbsent(propertyName, ignored -> nextIndex.getAndIncrement()); - } - - Integer find(String propertyName) { - return indexes.get(propertyName); - } + loadState = loadState.withLoaded(propertyName, loaded); } @Override @@ -544,9 +535,8 @@ public

P getProperty(String propertyName) { if (o != null) { return (P) o; } - Object dynamicProperty = this.additionalInfo.get(dynamicPropertyNameOf(propertyName)); - if (dynamicProperty != null) { - return (P) dynamicProperty; + if (propertyName != null && (propertyName.startsWith("_") || propertyName.startsWith("#"))) { + return (P) additionalInfo.get(propertyName); } return Entity.super.getProperty(propertyName); } @@ -559,12 +549,13 @@ public void handleUpdate(String propertyName, Object oldValue, Object newValue) oldValue = propertyChange.getOldValue(); } if (ObjectUtil.equals(oldValue, newValue)) { - updatedProperties.remove(propertyName); + if (!updatedProperties.isEmpty()) updatedProperties.remove(propertyName); return; } - updatedProperties.put(propertyName, new PropertyChange(propertyName, oldValue, newValue)); + mutableUpdatedProperties().put(propertyName, new PropertyChange(propertyName, oldValue, newValue)); - if (entityMutationLedger != null && id != null) { + if (id != null) { + EntityMutationLedger entityMutationLedger = getEntityMutationLedger(); EntityKey key = new EntityKey(typeName(), id); entityMutationLedger.set(key, propertyName, newValue); if (!_traceChain.isEmpty()) { @@ -573,13 +564,19 @@ public void handleUpdate(String propertyName, Object oldValue, Object newValue) } } + private Map mutableUpdatedProperties() { + if (updatedProperties == NO_UPDATES) updatedProperties = new ConcurrentHashMap<>(); + return updatedProperties; + } + public void gotoNextStatus(EntityAction action) { set$status(get$status().next(action)); } public void cacheRelation(String relationName, Entity relation) { markPropertyLoaded(relationName); - this.relationCache.put(relationName, relation); + if (relationCache.isEmpty()) relationCache = new HashMap<>(); + relationCache.put(relationName, relation); Object initValue = getProperty(relationName); handleUpdate(relationName, initValue, relation); } @@ -615,17 +612,85 @@ public int hashCode() { } public Map getAdditionalInfo() { - return additionalInfo; + return mutableAdditionalInfo(); } public void setAdditionalInfo(Map additionalInfo) { - this.additionalInfo = additionalInfo; + this.additionalInfo = additionalInfo == null || additionalInfo.isEmpty() + ? Map.of() : new HashMap<>(additionalInfo); + } + + private Map mutableAdditionalInfo() { + if (additionalInfo.isEmpty()) additionalInfo = new HashMap<>(); + return additionalInfo; + } + + @Override + public DynamicFieldValues dynamicFields() { + return dynamicFieldValues == null ? collectDynamicFieldValues() : dynamicFieldValues; } public DynamicFieldValues getDynamicFieldValues() { return dynamicFieldValues; } + public BaseEntity updateDynamicField(String code, Object value) { + if (dynamicFieldValues == null) throw new io.teaql.data.dynamic.DynamicFieldException("DYNAMIC_FIELD_DEFINITIONS_MISSING", "Load dynamic field definitions before editing"); + var mutation = io.teaql.data.dynamic.DynamicFieldMutation.set(code, dynamicFieldValues.metadata().requireType(code), value); + stageDynamicMutation(mutation); + return this; + } + + public BaseEntity deleteDynamicField(String code) { + if (dynamicFieldValues == null) throw new io.teaql.data.dynamic.DynamicFieldException("DYNAMIC_FIELD_DEFINITIONS_MISSING", "Load dynamic field definitions before editing"); + stageDynamicMutation(io.teaql.data.dynamic.DynamicFieldMutation.delete(code, dynamicFieldValues.metadata().requireType(code))); + return this; + } + + private void stageDynamicMutation(io.teaql.data.dynamic.DynamicFieldMutation mutation) { + EntityStatus nextStatus = get$status().next(EntityAction.UPDATE); + Map next = new HashMap<>(dynamicFieldValues.toMap()); + if (mutation.kind() == io.teaql.data.dynamic.DynamicFieldMutation.Kind.SET) next.put(mutation.code(), mutation.asLoadedValue()); + else next.remove(mutation.code()); + DynamicFieldValues values = new DynamicFieldValues(dynamicFieldValues.metadata(), next); + LoadState state = loadState.withDynamicSelection(values.selectedCodes()); + if (dynamicOriginalValues.isEmpty()) dynamicOriginalValues = new HashMap<>(); + dynamicOriginalValues.putIfAbsent(mutation.code(), dynamicFieldValues.field(mutation.code())); + __internalHydrateDynamicFields(values, state); + if (dynamicMutations.isEmpty()) dynamicMutations = new HashMap<>(); + dynamicMutations.put(mutation.code(), mutation); + set$status(nextStatus); + if (id != null) getEntityMutationLedger().setDynamic(new EntityKey(typeName(), id), mutation); + } + + @FrameworkInternal("Graph mutation planning only") + public Map __internalDynamicMutations() { + return dynamicMutations.isEmpty() ? Map.of() : java.util.Collections.unmodifiableMap(dynamicMutations); + } + + @FrameworkInternal("Invocation-local audit and log privacy provenance only") + public Map __internalDynamicOriginalValues() { + return dynamicOriginalValues.isEmpty() ? Map.of() : java.util.Collections.unmodifiableMap(dynamicOriginalValues); + } + + @FrameworkInternal("Successful graph commit only") + public void __internalClearDynamicMutations() { dynamicMutations = Map.of(); dynamicOriginalValues = Map.of(); } + + @FrameworkInternal("Runtime optimistic guard for an extension-only update") + public void __internalRequireVersionUpdate() { + if (version == null) throw new TeaQLRuntimeException("Dynamic update requires a loaded optimistic version"); + mutableUpdatedProperties().put(VERSION_PROPERTY, new PropertyChange(VERSION_PROPERTY, version, version)); + } + + @FrameworkInternal("Replay validated ledger intent into a provider-owned mutation entity") + public void __internalApplyRecordedMutation(String property, Object value) { + Object before = __internalGet(property); + __internalSet(property, value); + // The blank provider entity is not the original loaded snapshot. Equal + // defaults here must not discard an explicit NULL/zero/false intent. + mutableUpdatedProperties().put(property, new PropertyChange(property, before, value)); + } + public DynamicFieldValues collectDynamicFieldValues() { List fields = new ArrayList<>(); for (Map.Entry entry : additionalInfo.entrySet()) { @@ -655,21 +720,45 @@ public DynamicFieldValues collectDynamicFieldValues() { } public void setDynamicFieldValues(DynamicFieldValues values) { + // Hydration replaces this view's selection, not stored values or mutation intent. + if (!additionalInfo.isEmpty()) { + for (String key : new ArrayList<>(additionalInfo.keySet())) { + if (key.startsWith("#")) { + additionalInfo.remove(key); + loadState = loadState.withLoaded(key, false); + } + } + } this.dynamicFieldValues = values; if (values != null) { for (Map.Entry entry : values.toMap().entrySet()) { String key = "#" + entry.getKey(); - Object val = entry.getValue().value(); - additionalInfo.put(key, val); + if (!entry.getValue().isLoaded()) continue; + mutableAdditionalInfo().put(key, entry.getValue().value()); + loadState = loadState.withLoaded(key, true); } } } + /** Bulk hydration supplies a shared final selection; values and wrappers stay row-owned. */ + @FrameworkInternal("Dynamic-field query hydration only") + public void __internalHydrateDynamicFields(DynamicFieldValues values, LoadState sharedState) { + __internalUseLoadState(sharedState); + if (!additionalInfo.isEmpty()) additionalInfo.keySet().removeIf(key -> key.startsWith("#")); + dynamicFieldValues = values; + for (Map.Entry entry : values.toMap().entrySet()) { + if (entry.getValue().isLoaded()) mutableAdditionalInfo().put("#" + entry.getKey(), entry.getValue().value()); + } + } + /** * Put additional property directly without prefix. * Used by deserializers to populate entity fields. */ public void putAdditional(String propertyName, Object value) { - additionalInfo.put(propertyName, value); + mutableAdditionalInfo().put(propertyName, value); + if (propertyName != null && propertyName.startsWith("#")) { + loadState = loadState.withLoaded(propertyName, true); + } } } diff --git a/teaql-core/src/main/java/io/teaql/core/EntityChangeSet.java b/teaql-core/src/main/java/io/teaql/core/EntityChangeSet.java index 840f7d97..13d22dbd 100644 --- a/teaql-core/src/main/java/io/teaql/core/EntityChangeSet.java +++ b/teaql-core/src/main/java/io/teaql/core/EntityChangeSet.java @@ -8,9 +8,10 @@ */ public class EntityChangeSet { private final Map> changes = new TreeMap<>(); + private Map> dynamicChanges; public boolean isEmpty() { - return changes.isEmpty(); + return changes.isEmpty() && (dynamicChanges == null || dynamicChanges.isEmpty()); } public void set(EntityKey key, String field, Object value) { @@ -31,12 +32,31 @@ public Map> changes() { return Collections.unmodifiableMap(changes); } + public void setDynamic(EntityKey key, io.teaql.data.dynamic.DynamicFieldMutation mutation) { + if (dynamicChanges == null) dynamicChanges = new TreeMap<>(); + dynamicChanges.computeIfAbsent(key, ignored -> new TreeMap<>()).put(mutation.code(), mutation); + } + + public Map> dynamicChanges() { + if (dynamicChanges == null) return Collections.emptyMap(); + Map> view = new TreeMap<>(); + dynamicChanges.forEach((key, values) -> view.put(key, Collections.unmodifiableMap(values))); + return Collections.unmodifiableMap(view); + } + public void clearEntity(EntityKey key) { changes.remove(key); + if (dynamicChanges != null) dynamicChanges.remove(key); } public Set fieldNames(EntityKey key) { Map record = changes.get(key); + var dynamic = dynamicChanges == null ? null : dynamicChanges.get(key); + if (dynamic != null && !dynamic.isEmpty()) { + Set names = new TreeSet<>(record == null ? Collections.emptySet() : record.keySet()); + dynamic.keySet().forEach(code -> names.add("#" + code)); + return Collections.unmodifiableSet(names); + } return record != null ? Collections.unmodifiableSet(record.keySet()) : Collections.emptySet(); } } diff --git a/teaql-core/src/main/java/io/teaql/core/EntityMutationLedger.java b/teaql-core/src/main/java/io/teaql/core/EntityMutationLedger.java index 335cdaca..d1cdd4cd 100644 --- a/teaql-core/src/main/java/io/teaql/core/EntityMutationLedger.java +++ b/teaql-core/src/main/java/io/teaql/core/EntityMutationLedger.java @@ -45,6 +45,10 @@ public void set(EntityKey key, String field, Object value) { changeSets.set(key, field, value); } + public void setDynamic(EntityKey key, io.teaql.data.dynamic.DynamicFieldMutation mutation) { + changeSets.currentMut().setDynamic(key, mutation); + } + public Object get(EntityKey key, String field) { return changeSets.get(key, field); } @@ -139,10 +143,12 @@ public Long getOriginalVersion(EntityKey key) { public boolean mergeEntityFrom(EntityMutationLedger other, EntityKey key) { if (other == null || other == this) return false; Map fields = other.currentChangeSet().changes().get(key); + var dynamic = other.currentChangeSet().dynamicChanges().get(key); boolean pending = (fields != null && !fields.isEmpty()) || other.newKeys.contains(key) - || other.deletedKeys.contains(key) || other.recoveredKeys.contains(key); + || other.deletedKeys.contains(key) || other.recoveredKeys.contains(key) || (dynamic != null && !dynamic.isEmpty()); if (!pending) return false; if (fields != null) fields.forEach((field, value) -> set(key, field, value)); + if (dynamic != null) dynamic.values().forEach(mutation -> setDynamic(key, mutation)); if (other.deletedKeys.contains(key)) markAsDelete(key); if (other.recoveredKeys.contains(key)) markAsRecover(key); if (other.newKeys.contains(key)) markAsNew(key); @@ -160,6 +166,7 @@ public void mergeFrom(EntityMutationLedger other) { // Merge change sets EntityChangeSet otherChangeSet = other.currentChangeSet(); + otherChangeSet.dynamicChanges().forEach((key, mutations) -> mutations.values().forEach(mutation -> setDynamic(key, mutation))); for (Map.Entry> entry : otherChangeSet.changes().entrySet()) { EntityKey key = entry.getKey(); for (Map.Entry fieldEntry : entry.getValue().entrySet()) { diff --git a/teaql-core/src/main/java/io/teaql/core/FieldLayout.java b/teaql-core/src/main/java/io/teaql/core/FieldLayout.java new file mode 100644 index 00000000..1f30b3b2 --- /dev/null +++ b/teaql-core/src/main/java/io/teaql/core/FieldLayout.java @@ -0,0 +1,151 @@ +package io.teaql.core; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** Immutable, type-local fixed positions supplied by the generated library. */ +public final class FieldLayout { + private static final ClassValue TYPES = new ClassValue<>() { + @Override + protected Installation computeValue(Class type) { + return new Installation(); + } + }; + + private static final class Installation { + private volatile FieldLayout layout; + + FieldLayout resolve(Class type) { + FieldLayout current = layout; + if (current != null) return current; + synchronized (this) { + if (layout == null) layout = new FieldLayout(type, "unindexed", Map.of(), Map.of(), Set.of(), false); + return layout; + } + } + + synchronized FieldLayout install(FieldLayout candidate) { + if (layout == null) { + layout = candidate; + } else if (!layout.generated || !layout.revision.equals(candidate.revision) + || !layout.indexes.equals(candidate.indexes) || !layout.aliases.equals(candidate.aliases) + || !layout.members.equals(candidate.members) + || !layout.relations.equals(candidate.relations)) { + throw new IllegalArgumentException("Incompatible or late generated field-layout installation: " + candidate.entityType.getName()); + } + return layout; + } + } + + private final Class entityType; + private final String revision; + private final Map indexes; + private final Map aliases; + private final Map members; + private final Set relations; + private final boolean generated; + private final LoadState empty; + + private FieldLayout(Class entityType, String revision, Map indexes, + Map> mappings, Set relations, boolean generated) { + this.entityType = java.util.Objects.requireNonNull(entityType, "entityType"); + if (revision == null || revision.isBlank()) throw new IllegalArgumentException("Missing field-layout revision"); + this.revision = revision; + this.generated = generated; + this.indexes = Map.copyOf(indexes); + this.relations = Set.copyOf(relations); + Map aliases = new HashMap<>(); + Map members = new HashMap<>(); + Set positions = new HashSet<>(); + if (!mappings.keySet().equals(indexes.keySet())) { + throw new IllegalArgumentException("Incomplete field mappings for " + entityType.getName()); + } + for (Map.Entry field : indexes.entrySet()) { + String name = field.getKey(); + Integer index = field.getValue(); + if (name == null || name.isBlank() || name.startsWith("#") || name.startsWith("_")) { + throw new IllegalArgumentException("Invalid fixed field: " + name); + } + if (index == null || index < 0 || index >= indexes.size() || !positions.add(index)) { + throw new IllegalArgumentException("Invalid or duplicate fixed index: " + name + "=" + index); + } + List mapping = mappings.get(name); + if (mapping == null || mapping.size() != 2) { + throw new IllegalArgumentException("Expected member/column mapping for " + name); + } + addAlias(aliases, name, index); + addAlias(aliases, mapping.get(0), index); + addAlias(aliases, mapping.get(1), index); + members.put(name, mapping.get(0)); + } + if (generated && (!indexes.containsKey("id") || !indexes.containsKey("version"))) { + throw new IllegalArgumentException("Generated layout must include id and version: " + entityType.getName()); + } + this.aliases = Map.copyOf(aliases); + this.members = Map.copyOf(members); + empty = LoadState.empty(this); + } + + public static FieldLayout generated(Class entityType, String revision, Map indexes, + Map> mappings, Set relations) { + return new FieldLayout(entityType, revision, indexes, mappings, relations, true); + } + + @FrameworkInternal("Generated metadata and hydration infrastructure only") + public static FieldLayout forType(Class entityType) { + return TYPES.get(entityType).resolve(entityType); + } + + /** Generated static initialization supplies metadata directly, without reflection or row construction. */ + @FrameworkInternal("Generated metadata installation only") + public static FieldLayout installGenerated(FieldLayout layout) { + java.util.Objects.requireNonNull(layout, "layout"); + if (!layout.generated) throw new IllegalArgumentException("Only generated field layouts can be installed"); + return TYPES.get(layout.entityType).install(layout); + } + + public Class entityType() { return entityType; } + public String revision() { return revision; } + public boolean isGenerated() { return generated; } + public Map indexes() { return indexes; } + public Integer findIndex(String fieldOrAlias) { return aliases.get(fieldOrAlias); } + /** Generated language-native accessor key, not a SQL column or reflected bean property. */ + public String memberName(String canonicalField) { return members.get(canonicalField); } + /** Shared named relation metadata; these members do not consume fixed-field positions. */ + @FrameworkInternal("Typed graph hydration and presentation only") + public Set relationNames() { return relations; } + public LoadState emptyState() { return empty; } + + int indexForLoad(String name) { + Integer index = findIndex(name); + if (index != null) return index; + if (name == null || name.isBlank() || name.startsWith("_")) { + throw new IllegalArgumentException("Not a loadable field: " + name); + } + if (!generated || name.startsWith("#") || relations.contains(name)) return -1; + throw new IllegalArgumentException("Unknown field in generated layout: " + entityType.getName() + "." + name); + } + + public void validateExpectedMembers(Set members) { + if (!generated) return; + Set expected = new HashSet<>(); + for (String member : members) { + Integer index = findIndex(member); + if (index == null) throw new IllegalArgumentException("Missing generated field index: " + member); + expected.add(index); + } + if (expected.size() != indexes.size()) { + throw new IllegalArgumentException("Generated layout disagrees with entity descriptor: " + entityType.getName()); + } + } + + private static void addAlias(Map aliases, String alias, int index) { + if (alias == null || alias.isBlank()) throw new IllegalArgumentException("Blank field alias"); + Integer previous = aliases.putIfAbsent(alias, index); + if (previous != null && previous != index) throw new IllegalArgumentException("Ambiguous field alias: " + alias); + } + +} diff --git a/teaql-core/src/main/java/io/teaql/core/LoadState.java b/teaql-core/src/main/java/io/teaql/core/LoadState.java new file mode 100644 index 00000000..5d2baf19 --- /dev/null +++ b/teaql-core/src/main/java/io/teaql/core/LoadState.java @@ -0,0 +1,138 @@ +package io.teaql.core; + +import java.util.HashSet; +import java.util.Objects; +import java.util.Set; + +/** Immutable availability snapshot; it never contains entity values or mutation ownership. */ +public final class LoadState { + private static final String[] NO_DYNAMIC_CODES = new String[0]; + private final FieldLayout layout; + private final long bits; + private final Set overflow; + // Dynamic field selection, named relation materialization and unindexed adapter fields. + private final Set selectedNames; + private final int hash; + // Value-free memoization shared by the snapshot, never a per-entity payload. + private volatile String[] dynamicCodes; + + private LoadState(FieldLayout layout, long bits, Set overflow, Set selectedNames) { + this(layout, bits, overflow, selectedNames, selectedNames.isEmpty() ? NO_DYNAMIC_CODES : null); + } + + private LoadState(FieldLayout layout, long bits, Set overflow, Set selectedNames, + String[] dynamicCodes) { + this.layout = Objects.requireNonNull(layout, "layout"); + this.bits = bits; + this.overflow = overflow; + this.selectedNames = selectedNames; + this.dynamicCodes = dynamicCodes; + int result = System.identityHashCode(layout); + result = 31 * result + Long.hashCode(bits); + result = 31 * result + overflow.hashCode(); + hash = 31 * result + selectedNames.hashCode(); + } + + static LoadState empty(FieldLayout layout) { + return new LoadState(layout, 0L, Set.of(), Set.of()); + } + + /** Build once per actual query shape, not once per row. */ + public static LoadState projection(FieldLayout layout, Iterable fields) { + long bits = 0L; + Set overflow = null; + Set names = null; + for (String field : fields) { + int index = layout.indexForLoad(field); + if (index >= 0 && index < Long.SIZE) bits |= 1L << index; + else if (index >= Long.SIZE) { + if (overflow == null) overflow = new HashSet<>(); + overflow.add(index); + } else { + if (names == null) names = new HashSet<>(); + names.add(field); + } + } + if (bits == 0L && overflow == null && names == null) return layout.emptyState(); + return new LoadState(layout, bits, overflow == null ? Set.of() : Set.copyOf(overflow), + names == null ? Set.of() : Set.copyOf(names)); + } + + public FieldLayout layout() { return layout; } + public long bits() { return bits; } + public Set overflow() { return overflow; } + public Set selectedNames() { return selectedNames; } + + /** Build once for a dynamic-field batch shape, preserving fixed slots and relation markers. */ + public LoadState withDynamicSelection(Set codes) { + // A custom sorted comparator may equate different literal field codes. + // Preserve the original String-identity semantics before the fast membership check. + if (codes instanceof java.util.SortedSet sorted && sorted.comparator() != null) { + codes = Set.copyOf(codes); + } + String[] current = dynamicCodes(); + boolean same = current.length == codes.size(); + if (same) { + for (String code : current) { + if (!codes.contains(code)) { same = false; break; } + } + } + if (same) return this; + String[] stableCodes = codes.isEmpty() ? NO_DYNAMIC_CODES : codes.toArray(String[]::new); + Set names = new HashSet<>(selectedNames); + names.removeIf(name -> name.startsWith("#")); + for (String code : stableCodes) names.add("#" + Objects.requireNonNull(code, "dynamic code")); + return new LoadState(layout, bits, overflow, Set.copyOf(names), stableCodes); + } + + private String[] dynamicCodes() { + String[] current = dynamicCodes; + if (current != null) return current; + java.util.List codes = null; + for (String name : selectedNames) { + if (name.startsWith("#")) { + if (codes == null) codes = new java.util.ArrayList<>(); + codes.add(name.substring(1)); + } + } + current = codes == null ? NO_DYNAMIC_CODES : codes.toArray(String[]::new); + dynamicCodes = current; + return current; + } + + public boolean isLoaded(String fieldOrAlias) { + if (fieldOrAlias == null) return false; + Integer index = layout.findIndex(fieldOrAlias); + if (index == null) return selectedNames.contains(fieldOrAlias); + return index < Long.SIZE ? (bits & (1L << index)) != 0 : overflow.contains(index); + } + + /** Value-only updates keep this exact reference; availability changes create a private snapshot. */ + public LoadState withLoaded(String field, boolean loaded) { + int index = layout.indexForLoad(field); + if (isLoaded(field) == loaded) return this; + if (index >= 0 && index < Long.SIZE) { + long next = loaded ? bits | (1L << index) : bits & ~(1L << index); + return new LoadState(layout, next, overflow, selectedNames, dynamicCodes); + } + if (index >= Long.SIZE) { + Set next = new HashSet<>(overflow); + if (loaded) next.add(index); else next.remove(index); + return new LoadState(layout, bits, Set.copyOf(next), selectedNames, dynamicCodes); + } + Set next = new HashSet<>(selectedNames); + if (loaded) next.add(field); else next.remove(field); + return new LoadState(layout, bits, overflow, Set.copyOf(next)); + } + + @Override + public int hashCode() { return hash; } + + @Override + public boolean equals(Object other) { + if (this == other) return true; + if (!(other instanceof LoadState state)) return false; + return layout == state.layout && bits == state.bits + && overflow.equals(state.overflow) && selectedNames.equals(state.selectedNames); + } +} diff --git a/teaql-core/src/main/java/io/teaql/core/SmartList.java b/teaql-core/src/main/java/io/teaql/core/SmartList.java index b847331c..9c0cb2e9 100644 --- a/teaql-core/src/main/java/io/teaql/core/SmartList.java +++ b/teaql-core/src/main/java/io/teaql/core/SmartList.java @@ -41,6 +41,7 @@ public SmartList(List data) { this(data == null ? 0 : data.size()); if (data != null) { this.data.addAll(data); + __internalShareLoadStates(); } } @@ -59,6 +60,7 @@ public static SmartList empty(Class entityTyp public static SmartList takeOwnership(List data) { SmartList result = new SmartList<>(); result.data = data == null ? new ArrayList<>() : data; + result.__internalShareLoadStates(); return result; } @@ -97,11 +99,13 @@ public Map> groupBy(Function key) { public void add(T pValue) { ensureMutable(); + shareExistingState(pValue); data.add(pValue); } public void set(int index, T pValue) { ensureMutable(); + shareExistingState(pValue); data.set(index, pValue); } @@ -112,6 +116,38 @@ public List getData() { public void setData(List pData) { ensureMutable(); data = pData; + __internalShareLoadStates(); + } + + /** Group by exact availability, not by a union of requested fields. The temporary + * index is discarded: a long-lived list does not retain historical COW states. */ + @FrameworkInternal("Hydration result finalization only") + public void __internalShareLoadStates() { + if (data == null || data.size() < 2) return; + Map shapes = new HashMap<>(); + for (T entity : data) { + if (!(entity instanceof BaseEntity base)) continue; + LoadState state = base.__internalLoadState(); + LoadState shared = shapes.putIfAbsent(state, state); + if (shared != null && shared != state) base.__internalUseLoadState(shared); + } + } + + private void shareExistingState(T entity) { + if (!(entity instanceof BaseEntity base) || data == null || data.isEmpty()) return; + LoadState state = base.__internalLoadState(); + // Homogeneous hydration/append is the usual case and only checks the tail. + T tail = data.get(data.size() - 1); + if (tail instanceof BaseEntity previous && state.equals(previous.__internalLoadState())) { + base.__internalUseLoadState(previous.__internalLoadState()); + return; + } + for (T existing : data) { + if (existing instanceof BaseEntity previous && state.equals(previous.__internalLoadState())) { + base.__internalUseLoadState(previous.__internalLoadState()); + return; + } + } } public void addAggregationResult(UserContext userContext, AggregationResult aggregationResult) { diff --git a/teaql-core/src/main/java/io/teaql/core/TransactionExecutor.java b/teaql-core/src/main/java/io/teaql/core/TransactionExecutor.java index ab059f81..6f831c90 100644 --- a/teaql-core/src/main/java/io/teaql/core/TransactionExecutor.java +++ b/teaql-core/src/main/java/io/teaql/core/TransactionExecutor.java @@ -1,5 +1,7 @@ package io.teaql.core; public interface TransactionExecutor extends DataServiceExecutor { + @FrameworkInternal("Atomic graph participant identity; not a workspace API") + default Object transactionResource() { return null; } T executeInTransaction(UserContext context, TransactionCallback action); } diff --git a/teaql-core/src/main/java/io/teaql/core/meta/SimpleEntityMetaFactory.java b/teaql-core/src/main/java/io/teaql/core/meta/SimpleEntityMetaFactory.java index c1a2b4ee..6dc39188 100644 --- a/teaql-core/src/main/java/io/teaql/core/meta/SimpleEntityMetaFactory.java +++ b/teaql-core/src/main/java/io/teaql/core/meta/SimpleEntityMetaFactory.java @@ -23,6 +23,17 @@ public void register(EntityDescriptor entityDescriptor) { if (entityDescriptor == null) { return; } + if (entityDescriptor.getTargetType() != null + && io.teaql.core.BaseEntity.class.isAssignableFrom(entityDescriptor.getTargetType())) { + io.teaql.core.FieldLayout layout = io.teaql.core.FieldLayout.forType(entityDescriptor.getTargetType()); + if (layout.isGenerated()) { + java.util.Set members = new java.util.HashSet<>(); + for (EntityDescriptor current = entityDescriptor; current != null; current = current.getParent()) { + for (PropertyDescriptor property : current.getOwnProperties()) members.add(property.getName()); + } + layout.validateExpectedMembers(members); + } + } registeredEntities.put(entityDescriptor.getType(), entityDescriptor); } diff --git a/teaql-core/src/test/java/io/teaql/core/BaseEntityTest.java b/teaql-core/src/test/java/io/teaql/core/BaseEntityTest.java index 18da6982..44316b4f 100644 --- a/teaql-core/src/test/java/io/teaql/core/BaseEntityTest.java +++ b/teaql-core/src/test/java/io/teaql/core/BaseEntityTest.java @@ -246,12 +246,13 @@ public void testSubType() { } @Test - public void testLoadedPropertyBitsAndOverflowPreserveSemantics() { + public void testUnindexedAdaptersPreserveMembershipWithoutInventingFixedPositions() { TestEntity entity = new TestEntity(); for (int i = 0; i < 70; i++) entity.markPropertyLoaded("field" + i); for (int i = 0; i < 70; i++) assertTrue(entity.isPropertyLoaded("field" + i)); assertFalse(entity.isPropertyLoaded("notLoaded")); + assertEquals(-1, BaseEntity.loadedPropertyIndex(TestEntity.class, "field0")); } @Test @@ -334,8 +335,9 @@ public void testDynamicProperties() { e.addDynamicProperty("prop1", "val1"); assertEquals("val1", e.getDynamicProperty("prop1")); - e.addDynamicProperty("#prop2", "val2"); - assertEquals("val2", e.getDynamicProperty("#prop2")); + e.addDynamicProperty("_prop2", "val2"); + assertEquals("val2", e.getDynamicProperty("_prop2")); + assertThrows(IllegalArgumentException.class, () -> e.addDynamicProperty("#prop2", "value")); e.appendDynamicProperty("listProp", "item1"); e.appendDynamicProperty("listProp", "item2"); @@ -399,11 +401,11 @@ public void testCacheRelation() { @Test public void testDynamicFieldValues() { TestEntity e = new TestEntity(); - e.addDynamicProperty("str", "hello"); - e.addDynamicProperty("num", 100); - e.addDynamicProperty("bool", true); - e.addDynamicProperty("nullval", null); - e.addDynamicProperty("obj", new Object() { + e.putAdditional("#str", "hello"); + e.putAdditional("#num", 100); + e.putAdditional("#bool", true); + e.putAdditional("#nullval", null); + e.putAdditional("#obj", new Object() { @Override public String toString() { return "obj_string"; @@ -415,10 +417,11 @@ public String toString() { TestEntity e2 = new TestEntity(); e2.setDynamicFieldValues(dfv); - assertEquals("hello", e2.getDynamicProperty("str")); - assertEquals(Integer.valueOf(100), e2.getDynamicProperty("num")); - assertEquals(Boolean.TRUE, e2.getDynamicProperty("bool")); - assertEquals("obj_string", e2.getDynamicProperty("obj")); + assertEquals("hello", e2.dynamicFields().getString("str")); + assertEquals(Integer.valueOf(100), e2.dynamicFields().getNumber("num")); + assertEquals(Boolean.TRUE, e2.dynamicFields().getBool("bool")); + assertEquals("obj_string", e2.dynamicFields().getString("obj")); + assertTrue(e2.dynamicFields().isNull("nullval")); assertEquals(dfv, e2.getDynamicFieldValues()); e2.setDynamicFieldValues(null); diff --git a/teaql-core/src/test/java/io/teaql/core/DynamicMemberStateTest.java b/teaql-core/src/test/java/io/teaql/core/DynamicMemberStateTest.java new file mode 100644 index 00000000..31077c87 --- /dev/null +++ b/teaql-core/src/test/java/io/teaql/core/DynamicMemberStateTest.java @@ -0,0 +1,64 @@ +package io.teaql.core; + +import io.teaql.data.dynamic.*; +import java.util.List; +import java.util.Map; +import org.junit.Test; +import static org.junit.Assert.*; + +public class DynamicMemberStateTest { + static class Row extends BaseEntity { + String name; + @Override public Object __internalGet(String field) { + return "name".equals(field) ? name : super.__internalGet(field); + } + @Override public void __internalSet(String field, Object value) { + if ("name".equals(field)) name = (String) value; else super.__internalSet(field, value); + } + } + + @Test + public void fixedNameDerivedNameAndPersistentNameDoNotShadowEachOther() { + Row row = new Row(); row.setProperty("name", "fixed"); + row.addDynamicProperty("name", "derived"); + row.setDynamicFieldValues(DynamicFieldValues.of(List.of(DynamicFieldValue.ofString("name", "extension")))); + assertEquals("fixed", row.getProperty("name")); + assertEquals("derived", row.getProperty("_name")); + assertEquals("extension", row.getProperty("#name")); + assertEquals("derived", row.getDynamicProperty("name")); + assertEquals("extension", row.dynamicFields().getString("name")); + assertEquals(1, row.collectDynamicFieldValues().size()); + assertTrue(row.getUpdatedProperties().isEmpty()); + } + + @Test + public void derivedNullAndMissingReadNullWithoutInventingZeroOrPersistentFields() { + Row row = new Row(); + assertNull(row.getDynamicProperty("missing")); + row.addDynamicProperty("count", null); + assertTrue(row.getAdditionalInfo().containsKey("_count")); + assertNull(row.getDynamicProperty("count")); + row.addDynamicProperty("count", 0); + assertEquals(Integer.valueOf(0), row.getDynamicProperty("count")); + assertFalse(row.isPropertyLoaded("_count")); + assertEquals(0, row.collectDynamicFieldValues().size()); + } + + @Test + public void sharedSelectionKeepsWrapperNullAndValuePrivateAndLeavesFixedSlotsUnchanged() { + DynamicFieldMetadata metadata = new DynamicFieldMetadata(Map.of("note", DynamicDataType.STRING, "extra", DynamicDataType.STRING)); + Row first = new Row(); Row second = new Row(); + first.setDynamicFieldValues(new DynamicFieldValues(metadata, Map.of("note", DynamicFieldValue.ofString("note", "first")))); + second.setDynamicFieldValues(new DynamicFieldValues(metadata, Map.of("note", DynamicFieldValue.ofNull("note", DynamicDataType.STRING)))); + SmartList list = new SmartList<>(List.of(first, second)); + assertSame(first.__internalLoadState(), second.__internalLoadState()); + assertEquals(DynamicFieldValue.State.VALUE, first.dynamicFields().field("note").state()); + assertEquals(DynamicFieldValue.State.NULL, second.dynamicFields().field("note").state()); + assertEquals(DynamicFieldValue.State.NOT_LOADED, second.dynamicFields().field("extra").state()); + assertTrue(first.isPropertyLoaded("#note")); + assertFalse(first.isPropertyLoaded("name")); + assertEquals(0L, first.__internalLoadState().bits()); + assertNotSame(first.getEntityMutationLedger(), second.getEntityMutationLedger()); + assertEquals(2, list.size()); + } +} diff --git a/teaql-core/src/test/java/io/teaql/core/EntityChangeSetTest.java b/teaql-core/src/test/java/io/teaql/core/EntityChangeSetTest.java index d442c281..50731ab0 100644 --- a/teaql-core/src/test/java/io/teaql/core/EntityChangeSetTest.java +++ b/teaql-core/src/test/java/io/teaql/core/EntityChangeSetTest.java @@ -23,6 +23,24 @@ public void newChangeSetIsEmpty() { assertTrue(changeSet.changes().isEmpty()); } + @Test + public void dynamicIntentsHaveDistinctNamesAndStayTypeQualified() { + var changes = new EntityChangeSet(); + var otherType = new EntityKey("Invoice", 1L); + changes.set(ORDER, "status", "PAID"); + changes.setDynamic(ORDER, io.teaql.data.dynamic.DynamicFieldMutation.set("status", io.teaql.data.dynamic.DynamicDataType.STRING, null)); + changes.setDynamic(otherType, io.teaql.data.dynamic.DynamicFieldMutation.delete("status", io.teaql.data.dynamic.DynamicDataType.STRING)); + assertEquals(Set.of("status", "#status"), changes.fieldNames(ORDER)); + assertEquals(Set.of("#status"), changes.fieldNames(otherType)); + assertThrows(UnsupportedOperationException.class, () -> changes.dynamicChanges().get(ORDER).clear()); + assertEquals(io.teaql.data.dynamic.DynamicFieldMutation.Kind.SET, changes.dynamicChanges().get(ORDER).get("status").kind()); + assertNull(changes.dynamicChanges().get(ORDER).get("status").value()); + changes.clearEntity(ORDER); + assertFalse(changes.isEmpty()); + assertTrue(changes.fieldNames(ORDER).isEmpty()); + assertEquals(io.teaql.data.dynamic.DynamicFieldMutation.Kind.DELETE, changes.dynamicChanges().get(otherType).get("status").kind()); + } + @Test public void setMakesFieldValueReadable() { EntityChangeSet changeSet = new EntityChangeSet(); diff --git a/teaql-core/src/test/java/io/teaql/core/LoadStateTest.java b/teaql-core/src/test/java/io/teaql/core/LoadStateTest.java new file mode 100644 index 00000000..5513e4ff --- /dev/null +++ b/teaql-core/src/test/java/io/teaql/core/LoadStateTest.java @@ -0,0 +1,286 @@ +package io.teaql.core; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.junit.Test; +import org.junit.BeforeClass; +import static org.junit.Assert.*; + +public class LoadStateTest { + @BeforeClass + public static void installFixtureMetadata() { + assertTrue(IndexedEntity.__TEAQL_FIELD_LAYOUT.isGenerated()); + } + + @Test + public void readonlyHydrationKeepsLedgersLazyAndFirstMutationCapturesThePrivateBaseline() { + var shape = LoadState.projection(IndexedEntity.__TEAQL_FIELD_LAYOUT, List.of("id", "version", "base_url")); + var first = new IndexedEntity(); var second = new IndexedEntity(); + for (var entity : List.of(first, second)) { + entity.__internalUseLoadState(shape); + entity.__internalHydrate("id", 1001L, 0); + entity.__internalHydrate("version", 7L, 1); + entity.__internalHydrate("baseUrl", null, 2); + entity.set$status(EntityStatus.PERSISTED); + assertFalse(entity.__internalHasMutationLedger()); + assertEquals(Long.valueOf(7), entity.getOriginalVersion()); + assertTrue(entity.getUpdatedProperties().isEmpty()); + entity.clearUpdatedProperties(); + } + first.setComment("private change"); + first.handleUpdate("baseUrl", null, "changed"); + first.__internalSet("baseUrl", "changed"); + assertTrue(first.__internalHasMutationLedger()); + assertFalse(second.__internalHasMutationLedger()); + assertEquals(Long.valueOf(7), first.getOriginalVersion()); + assertEquals("changed", first.getProperty("baseUrl")); + assertNull(second.getProperty("baseUrl")); + assertSame(shape, first.__internalLoadState()); assertSame(shape, second.__internalLoadState()); + assertEquals("private change", first.getEntityMutationLedger().getComment()); + assertNotSame(first.getEntityMutationLedger(), second.getEntityMutationLedger()); + } + + @Test + public void rollbackRestoresTheLazyLoadedVersionWithoutMaterializingALedger() { + var entity = new IndexedEntity(); + entity.__internalHydrate("id", 1001L, 0); + entity.__internalHydrate("version", 7L, 1); + entity.__internalHydrate("version", 8L, 1); + entity.__internalRestorePersistenceState(7L, EntityStatus.PERSISTED, true); + assertEquals(Long.valueOf(7), entity.getOriginalVersion()); + assertFalse(entity.__internalHasMutationLedger()); + assertEquals(Long.valueOf(7), entity.getEntityMutationLedger().getOriginalVersion(new EntityKey(entity.typeName(), 1001L))); + } + + @Test + public void dynamicSelectionMemoizationPreservesRelationsAndCopyOnWrite() { + var layout = FieldLayout.forType(IndexedEntity.class); + var base = LoadState.projection(layout, List.of("id", "field_64", "childList")); + Set mutable = new java.util.HashSet<>(Set.of("note")); + var selected = base.withDynamicSelection(mutable); + mutable.clear(); + assertTrue(selected.isLoaded("#note")); + assertSame(selected, selected.withDynamicSelection(Set.of("note"))); + var caseInsensitive = new java.util.TreeSet(String.CASE_INSENSITIVE_ORDER); + caseInsensitive.add("NOTE"); + var literalCase = selected.withDynamicSelection(caseInsensitive); + assertNotSame(selected, literalCase); + assertTrue(literalCase.isLoaded("#NOTE")); + assertFalse(literalCase.isLoaded("#note")); + var changed = selected.withDynamicSelection(Set.of("other")); + assertNotSame(selected, changed); + assertFalse(changed.isLoaded("#note")); + assertTrue(changed.isLoaded("#other")); + assertTrue(selected.isLoaded("#note")); + assertTrue(changed.isLoaded("childList")); + assertTrue(changed.isLoaded("field_64")); + var fixedChange = changed.withLoaded("field_65", true); + assertSame(fixedChange, fixedChange.withDynamicSelection(Set.of("other"))); + var cleared = changed.withDynamicSelection(Set.of()); + assertEquals(base, cleared); + assertSame(cleared, cleared.withDynamicSelection(Set.of())); + assertTrue(cleared.isLoaded("childList")); + var fromProjection = LoadState.projection(layout, List.of("id", "#note", "childList")); + assertSame(fromProjection, fromProjection.withDynamicSelection(Set.of("note"))); + assertEquals(fromProjection.hashCode(), fromProjection.withDynamicSelection(Set.of("note")).hashCode()); + } + static final class IndexedEntity extends BaseEntity { + public static final String __TEAQL_FIELD_LAYOUT_REVISION = "load-state-fixture-v1"; + public static final Map __TEAQL_FIXED_FIELD_INDEXES = indexes(); + public static final Map> __TEAQL_FIXED_FIELD_MAPPINGS = mappings(); + public static final Set __TEAQL_FIXED_RELATION_NAMES = Set.of("childList"); + public static final FieldLayout __TEAQL_FIELD_LAYOUT = FieldLayout.installGenerated(FieldLayout.generated( + IndexedEntity.class, __TEAQL_FIELD_LAYOUT_REVISION, __TEAQL_FIXED_FIELD_INDEXES, + __TEAQL_FIXED_FIELD_MAPPINGS, __TEAQL_FIXED_RELATION_NAMES)); + private Object baseUrl; + private static Map indexes() { + Map indexes = new HashMap<>(); + indexes.put("id", 0); + indexes.put("version", 1); + indexes.put("base_url", 2); + for (int i = 3; i < 130; i++) indexes.put("field_" + i, i); + return Map.copyOf(indexes); + } + private static Map> mappings() { + Map> mappings = new HashMap<>(); + for (String name : __TEAQL_FIXED_FIELD_INDEXES.keySet()) mappings.put(name, List.of(name, name)); + mappings.put("base_url", List.of("baseUrl", "legacy_url")); + return Map.copyOf(mappings); + } + @Override + public void __internalSet(String property, Object value) { + if ("baseUrl".equals(property)) baseUrl = value; + else super.__internalSet(property, value); + } + @Override + public Object __internalGet(String property) { + if ("baseUrl".equals(property)) return baseUrl; + return super.__internalGet(property); + } + } + + static final class LateLayoutEntity extends BaseEntity { } + + @Test + public void explicitInstallationIsSharedAndCannotReplaceAnAlreadyUsedLayout() { + var installed = IndexedEntity.__TEAQL_FIELD_LAYOUT; + assertSame(installed, FieldLayout.installGenerated(installed)); + var identical = FieldLayout.generated(IndexedEntity.class, installed.revision(), + IndexedEntity.__TEAQL_FIXED_FIELD_INDEXES, IndexedEntity.__TEAQL_FIXED_FIELD_MAPPINGS, + IndexedEntity.__TEAQL_FIXED_RELATION_NAMES); + assertSame(installed, FieldLayout.installGenerated(identical)); + var changed = FieldLayout.generated(IndexedEntity.class, "another-revision", + IndexedEntity.__TEAQL_FIXED_FIELD_INDEXES, IndexedEntity.__TEAQL_FIXED_FIELD_MAPPINGS, + IndexedEntity.__TEAQL_FIXED_RELATION_NAMES); + assertThrows(IllegalArgumentException.class, () -> FieldLayout.installGenerated(changed)); + assertFalse(FieldLayout.forType(LateLayoutEntity.class).isGenerated()); + var late = FieldLayout.generated(LateLayoutEntity.class, "late", Map.of("id", 0, "version", 1), + Map.of("id", List.of("id", "id"), "version", List.of("version", "version")), Set.of()); + assertThrows(IllegalArgumentException.class, () -> FieldLayout.installGenerated(late)); + } + + @Test + public void generatedPositionsAreIndependentOfAccessAndConcurrency() { + int[] slots = {0, 31, 32, 63, 64, 65, 129}; + FieldLayout layout = FieldLayout.forType(IndexedEntity.class); + java.util.stream.IntStream.range(0, 100).parallel().forEach(i -> { + for (int slot : slots) assertEquals(Integer.valueOf(slot), layout.findIndex(name(slot))); + assertSame(layout, FieldLayout.forType(IndexedEntity.class)); + }); + assertEquals(2, BaseEntity.loadedPropertyIndex(IndexedEntity.class, "baseUrl")); + assertEquals(2, BaseEntity.loadedPropertyIndex(IndexedEntity.class, "legacy_url")); + } + + @Test + public void bitmapBoundariesAndOverflowNeverWrap() { + FieldLayout layout = FieldLayout.forType(IndexedEntity.class); + LoadState state = LoadState.projection(layout, List.of("id", "field_31", "field_32", "field_63")); + assertTrue(state.bits() < 0L); + assertTrue(state.overflow().isEmpty()); + assertFalse(state.isLoaded("field_64")); + LoadState wide = state.withLoaded("field_64", true).withLoaded("field_65", true).withLoaded("field_129", true); + assertEquals(state.bits(), wide.bits()); + assertEquals(Set.of(64, 65, 129), wide.overflow()); + LoadState cleared = wide.withLoaded("field_65", false); + assertFalse(cleared.isLoaded("field_65")); + assertTrue(wide.isLoaded("field_65")); + assertThrows(UnsupportedOperationException.class, () -> wide.overflow().add(100)); + } + + @Test + public void smartListSharesWholeSnapshotAndOnlyAvailabilityDetaches() { + List rows = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + IndexedEntity row = new IndexedEntity(); + row.setProperty("baseUrl", i == 0 ? null : "row-" + i); + row.markPropertyLoaded("field_64"); + row.markPropertyLoaded("#extra_note"); + rows.add(row); + } + SmartList list = SmartList.takeOwnership(rows); + LoadState shared = list.first().__internalLoadState(); + for (IndexedEntity row : list) assertSame(shared, row.__internalLoadState()); + assertTrue(list.first().isPropertyLoaded("baseUrl")); + assertNull(list.first().getProperty("baseUrl")); + assertSame(shared, list.get(1).__internalLoadState().withLoaded("baseUrl", true)); + list.get(1).setProperty("baseUrl", "changed"); + assertSame(shared, list.get(1).__internalLoadState()); + list.first().markPropertyLoaded("field_65"); + assertNotSame(shared, list.first().__internalLoadState()); + assertFalse(list.get(1).isPropertyLoaded("field_65")); + assertNotSame(list.first().getEntityMutationLedger(), list.get(1).getEntityMutationLedger()); + assertTrue(shared.selectedNames().contains("#extra_note")); + assertEquals(Set.of(64), shared.overflow()); + } + + @Test + public void appendAndMixedProjectionNeverUnionStatesAndSurviveListLifetime() { + IndexedEntity full = new IndexedEntity(); + full.setProperty("baseUrl", false); + IndexedEntity minimal = new IndexedEntity(); + minimal.setProperty("id", 7L); + SmartList list = new SmartList<>(List.of(full, minimal)); + IndexedEntity anotherMinimal = new IndexedEntity(); + anotherMinimal.setProperty("id", 8L); + list.add(anotherMinimal); + assertSame(minimal.__internalLoadState(), anotherMinimal.__internalLoadState()); + assertFalse(minimal.isPropertyLoaded("baseUrl")); + assertEquals(false, full.getProperty("baseUrl")); + LoadState kept = minimal.__internalLoadState(); + list.setData(new ArrayList<>()); + assertSame(kept, minimal.__internalLoadState()); + assertTrue(minimal.isPropertyLoaded("id")); + } + + @Test + public void hydrateIndexMismatchFailsBeforeWritingAndUnknownNamesCannotCorruptBits() { + IndexedEntity row = new IndexedEntity(); + assertThrows(IllegalArgumentException.class, () -> row.__internalHydrate("baseUrl", "bad", 1)); + assertFalse(row.isPropertyLoaded("baseUrl")); + assertNull(row.getProperty("baseUrl")); + assertThrows(IllegalArgumentException.class, () -> row.markPropertyLoaded("typo")); + assertEquals(0L, row.__internalLoadState().bits()); + } + + @Test + public void compiledProjectionIsSharedWithoutDirtyHydrationAndRollbackIsPrivate() { + FieldLayout layout = FieldLayout.forType(IndexedEntity.class); + LoadState shape = LoadState.projection(layout, List.of("id", "baseUrl")); + IndexedEntity left = new IndexedEntity(); + IndexedEntity right = new IndexedEntity(); + for (IndexedEntity row : List.of(left, right)) { + row.__internalUseLoadState(shape); + row.__internalHydrate("id", row == left ? 1L : 2L, 0); + row.__internalHydrate("baseUrl", "", 2); + assertSame(shape, row.__internalLoadState()); + assertTrue(row.getUpdatedProperties().isEmpty()); + } + left.__internalHydrate("version", 1L, 1); + assertTrue(left.isPropertyLoaded("version")); + left.__internalRestorePersistenceState(null, EntityStatus.PERSISTED, false); + assertFalse(left.isPropertyLoaded("version")); + assertFalse(right.isPropertyLoaded("version")); + assertSame(shape, right.__internalLoadState()); + } + + @Test + public void malformedDescriptorsAndIncompatibleLayoutsAreRejected() { + Map> mappings = Map.of("id", List.of("id", "id"), "version", List.of("version", "version")); + assertThrows(IllegalArgumentException.class, () -> FieldLayout.generated(IndexedEntity.class, "v1", + Map.of("id", 0, "version", 0), mappings, Set.of())); + assertThrows(IllegalArgumentException.class, () -> FieldLayout.generated(IndexedEntity.class, "v1", + Map.of("id", 0, "version", 64), mappings, Set.of())); + FieldLayout otherRevision = FieldLayout.generated(IndexedEntity.class, "v2", + Map.of("id", 0, "version", 1), mappings, Set.of()); + IndexedEntity row = new IndexedEntity(); + assertThrows(IllegalArgumentException.class, () -> row.__internalUseLoadState(otherRevision.emptyState())); + assertThrows(IllegalArgumentException.class, + () -> row.__internalLoadState().layout().validateExpectedMembers(Set.of("id", "version"))); + } + + @Test + public void ordinaryCompiledShapesRetainOneSnapshotAtAllRequiredRowScales() { + FieldLayout layout = FieldLayout.forType(IndexedEntity.class); + for (int count : new int[]{1, 100, 10_000}) { + LoadState shape = LoadState.projection(layout, List.of("id", "baseUrl")); + List rows = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + IndexedEntity row = new IndexedEntity(); + row.__internalUseLoadState(shape); + row.__internalHydrate("id", (long) i + 1, 0); + row.__internalHydrate("baseUrl", i == 0 ? null : "row-" + i, 2); + assertSame(shape, row.__internalLoadState()); + assertTrue(row.__internalLoadState().overflow().isEmpty()); + assertTrue(row.__internalLoadState().selectedNames().isEmpty()); + rows.add(row); + } + SmartList list = SmartList.takeOwnership(rows); + for (IndexedEntity row : list) assertSame(shape, row.__internalLoadState()); + } + } + + private static String name(int index) { return index == 0 ? "id" : "field_" + index; } +} diff --git a/teaql-data-service-sql/src/main/java/io/teaql/dataservice/sql/SqlDataServiceExecutor.java b/teaql-data-service-sql/src/main/java/io/teaql/dataservice/sql/SqlDataServiceExecutor.java index 9e45d598..8ab2d4d5 100644 --- a/teaql-data-service-sql/src/main/java/io/teaql/dataservice/sql/SqlDataServiceExecutor.java +++ b/teaql-data-service-sql/src/main/java/io/teaql/dataservice/sql/SqlDataServiceExecutor.java @@ -76,6 +76,8 @@ public T executeInTransaction(UserContext context, TransactionCallback ac return getPortableService(context).executeInTransaction(context, action); } + @Override public Object transactionResource() { return executionAdapter; } + @Override public void ensureSchema(UserContext context, io.teaql.core.SchemaExecutor.Invocation invocation) { io.teaql.core.SchemaExecutor.Invocation.requireContextOwned(invocation); @@ -144,6 +146,17 @@ public boolean supportsCompiledRowMapping() { return true; } + @Override public boolean supportsCompiledStreamMapping() { + return executionAdapter.supportsCompiledStreamMapping(); + } + + @Override public java.util.stream.Stream queryForStream( + io.teaql.core.UserContext context, String sql, Object[] args, + io.teaql.core.CompiledRowMapper mapper, io.teaql.core.sql.portable.SqlLogBindings bindings) { + return observedQueryStream(context, sql, args, bindings, + () -> executionAdapter.queryForStream(sql, args, mapper)); + } + @Override public java.util.List> query(String sql, Object[] args) { return executionAdapter.queryForList(sql, args); @@ -155,23 +168,8 @@ public java.util.stream.Stream> queryForStream(io. @Override public java.util.stream.Stream> queryForStream(io.teaql.core.UserContext context, String sql, Object[] args, io.teaql.core.sql.portable.SqlLogBindings bindings) { - boolean logging = context.isQueryExecutionLoggingEnabled(); - boolean collecting = logging || bindings.collectsStatements(); - long start = collecting ? System.nanoTime() : 0L; - io.teaql.core.ExecutionMetadata meta = null; - if (collecting) { - meta = statementMetadata(context, sql, args, bindings, io.teaql.core.DataServiceOperation.QUERY); - // Snapshot before request trace scopes are popped; lazy consumption may happen later. - if (bindings.executionTrace() == null) { - var trace = context.getTraceChain(); - meta.setTraceChain(trace == null || trace.isEmpty() - ? java.util.List.of(new io.teaql.core.TraceNode(io.teaql.core.TraceKind.OPERATION, "stream", "query")) - : java.util.List.copyOf(trace)); - } - } - var stream = diagnosed(context, sql, args, bindings, io.teaql.core.DataServiceOperation.QUERY, - logging, start, () -> executionAdapter.queryForStream(sql, args)); - return SqlDiagnosticStream.wrap(context, stream, meta, start, bindings, logging); + return observedQueryStream(context, sql, args, bindings, + () -> executionAdapter.queryForStream(sql, args)); } @Override public int executeUpdate(String sql, Object[] args) { @@ -340,6 +338,27 @@ public void execute(io.teaql.core.UserContext context, String sql) { return portableService; } + private java.util.stream.Stream observedQueryStream(UserContext context, String sql, Object[] args, + io.teaql.core.sql.portable.SqlLogBindings bindings, java.util.function.Supplier> action) { + boolean logging = context.isQueryExecutionLoggingEnabled(); + boolean collecting = logging || bindings.collectsStatements(); + long start = collecting ? System.nanoTime() : 0L; + io.teaql.core.ExecutionMetadata meta = null; + if (collecting) { + meta = statementMetadata(context, sql, args, bindings, io.teaql.core.DataServiceOperation.QUERY); + // Snapshot before request trace scopes are popped; consumption is lazy. + if (bindings.executionTrace() == null) { + var trace = context.getTraceChain(); + meta.setTraceChain(trace == null || trace.isEmpty() + ? java.util.List.of(new io.teaql.core.TraceNode(io.teaql.core.TraceKind.OPERATION, "stream", "query")) + : java.util.List.copyOf(trace)); + } + } + var stream = diagnosed(context, sql, args, bindings, io.teaql.core.DataServiceOperation.QUERY, + logging, start, action); + return SqlDiagnosticStream.wrap(context, stream, meta, start, bindings, logging); + } + private io.teaql.core.ExecutionMetadata statementMetadata(UserContext context, String sql, Object[] args, io.teaql.core.sql.portable.SqlLogBindings bindings, io.teaql.core.DataServiceOperation operation) { var meta = new io.teaql.core.ExecutionMetadata(); diff --git a/teaql-data-service-sql/src/main/java/io/teaql/dataservice/sql/SqlExecutionAdapter.java b/teaql-data-service-sql/src/main/java/io/teaql/dataservice/sql/SqlExecutionAdapter.java index adc7708f..708d9faf 100644 --- a/teaql-data-service-sql/src/main/java/io/teaql/dataservice/sql/SqlExecutionAdapter.java +++ b/teaql-data-service-sql/src/main/java/io/teaql/dataservice/sql/SqlExecutionAdapter.java @@ -17,6 +17,13 @@ default List query( default Stream> queryForStream(String sql, Object[] params) { throw new UnsupportedOperationException("streaming query is not supported"); } + + default boolean supportsCompiledStreamMapping() { return false; } + + default Stream queryForStream( + String sql, Object[] params, io.teaql.core.CompiledRowMapper mapper) { + throw new UnsupportedOperationException("compiled streaming row mapping is not supported"); + } List> queryForList(String sql, Map params); @@ -37,4 +44,8 @@ default Stream> queryForStream(String sql, Object[] params) default void executeInTransaction(Runnable action) { action.run(); } + default boolean hasActiveTransaction() { return false; } + + /** Opaque, non-reused instance token for held storage views; never a URL or transaction handle. */ + default Object storageIdentity() { return null; } } diff --git a/teaql-dynamic-fields-api/src/main/java/io/teaql/data/dynamic/DefaultDynamicFieldsFacade.java b/teaql-dynamic-fields-api/src/main/java/io/teaql/data/dynamic/DefaultDynamicFieldsFacade.java index f99cef97..71cb74cc 100644 --- a/teaql-dynamic-fields-api/src/main/java/io/teaql/data/dynamic/DefaultDynamicFieldsFacade.java +++ b/teaql-dynamic-fields-api/src/main/java/io/teaql/data/dynamic/DefaultDynamicFieldsFacade.java @@ -72,8 +72,68 @@ public OwnerBound owner(String ownerType, long ownerId) { return new DefaultOwnerBound(ownerType, ownerId); } + @Override + public DynamicFieldMetadata metadata(String ownerType) { + Objects.requireNonNull(ownerType, "ownerType"); + DynamicFieldContext context = buildContext(); + for (DynamicFieldDef field : provider.listFieldDefs(context, ownerType)) { + if (field.isActive()) checkReadable(field); + } + return provider.metadata(context, ownerType); + } + + @Override + public java.util.Map readAll( + java.util.List owners, DynamicFieldSelection selection) { + DynamicFieldContext context = buildContext(); + java.util.Set checked = new java.util.HashSet<>(); + for (DynamicOwnerRef owner : owners) { + if (!checked.add(owner.ownerType())) continue; + if (selection.isSelectAll()) { + for (DynamicFieldDef field : provider.listFieldDefs(context, owner.ownerType())) { + if (field.isActive()) checkReadable(field); + } + } else { + for (DynamicFieldSelection.DynamicFieldSelectionEntry entry : selection.getEntries()) { + DynamicFieldDef field = requireFieldDef(context, owner.ownerType(), entry.code()); + checkReadable(field); checkType(field, entry.dataType()); + } + } + } + return provider.loadValues(context, owners, selection); + } + // ─── Context Builder ─────────────────────────────────────────────── + @Override + public void prepareGraphMutations(Object resource, java.util.List mutations) { + if (!provider.participatesInGraphTransaction(resource)) throw new DynamicFieldException( + "DYNAMIC_FIELD_TRANSACTION_BINDING_REQUIRED", "Dynamic field provider must share the exact active graph executor"); + if (purpose == null || purpose.isBlank() || comment == null || comment.isBlank()) throw new DynamicFieldException( + "DYNAMIC_FIELD_INTENT_REQUIRED", "Graph dynamic changes require non-empty comment and purpose"); + DynamicFieldContext context = buildContext(); + for (DynamicGraphMutation request : mutations) { + provider.validateGraphMutation(context, request); + DynamicFieldDef def = requireFieldDef(context, request.owner().ownerType(), request.mutation().code()); + checkWritable(def); checkType(def, request.mutation().dataType()); + } + } + + @Override + public void applyGraphMutations(Object resource, java.util.List mutations) { + prepareGraphMutations(resource, mutations); + DynamicFieldContext context = buildContext(); + for (DynamicGraphMutation request : mutations) { + DynamicFieldMutation mutation = request.mutation(); + if (mutation.kind() == DynamicFieldMutation.Kind.SET) { + provider.saveValue(context, DynamicSetCommand.of(request.owner(), mutation.code(), mutation.dataType(), mutation.value(), purpose, comment)); + } else { + DynamicFieldDef def = requireFieldDef(context, request.owner().ownerType(), mutation.code()); + provider.deleteValue(context, DynamicValueRef.of(request.owner(), def.getId())); + } + } + } + private DynamicFieldContext buildContext() { return new DynamicFieldContext() { @Override public String scopeType() { return defaultScope.scopeType(); } @@ -154,9 +214,8 @@ public BoolFieldBound bool(String fieldCode) { @Override public DynamicFieldValues readAll(DynamicFieldSelection selection) { - DynamicFieldContext context = buildContext(); DynamicOwnerRef ownerRef = DynamicOwnerRef.of(ownerType, ownerId); - return provider.loadValues(context, ownerRef, selection); + return DefaultDynamicFieldsFacade.this.readAll(java.util.List.of(ownerRef), selection).get(ownerRef); } } diff --git a/teaql-dynamic-fields-api/src/main/java/io/teaql/data/dynamic/DynamicFieldMetadata.java b/teaql-dynamic-fields-api/src/main/java/io/teaql/data/dynamic/DynamicFieldMetadata.java new file mode 100644 index 00000000..e7f25ab1 --- /dev/null +++ b/teaql-dynamic-fields-api/src/main/java/io/teaql/data/dynamic/DynamicFieldMetadata.java @@ -0,0 +1,62 @@ +package io.teaql.data.dynamic; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** Immutable runtime-defined names/types, shared by compatible rows. No numeric slots. */ +public final class DynamicFieldMetadata { + private final Map types; + private final Object storageIdentity; + private final String scopeType; + private final String scopeId; + private final String ownerType; + // Value-free immutable absence wrappers, bounded by this definition snapshot's known names. + private final java.util.concurrent.ConcurrentMap absent = new java.util.concurrent.ConcurrentHashMap<>(); + + public DynamicFieldMetadata(Map types) { + this.types = Collections.unmodifiableMap(new LinkedHashMap<>(types)); + storageIdentity = null; scopeType = null; scopeId = null; ownerType = null; + } + + private DynamicFieldMetadata(DynamicFieldMetadata source, Object identity, String scopeType, String scopeId, String ownerType) { + types = source.types; + storageIdentity = java.util.Objects.requireNonNull(identity); + this.scopeType = scopeType; this.scopeId = scopeId; this.ownerType = ownerType; + } + + /** Trusted provider provenance only; no business meaning and no numeric dynamic slots. */ + public DynamicFieldMetadata withStorageBinding(Object identity, String scopeType, String scopeId, String ownerType) { + java.util.Objects.requireNonNull(identity, "storage identity"); + if (matchesStorage(identity, scopeType, scopeId, ownerType)) return this; + return new DynamicFieldMetadata(this, identity, scopeType, scopeId, ownerType); + } + + public boolean matchesStorage(Object identity, String scopeType, String scopeId, String ownerType) { + return storageIdentity != null && storageIdentity == identity && java.util.Objects.equals(this.scopeType, scopeType) + && java.util.Objects.equals(this.scopeId, scopeId) && java.util.Objects.equals(this.ownerType, ownerType); + } + + public static DynamicFieldMetadata fromDefinitions(List definitions) { + Map types = new LinkedHashMap<>(); + for (DynamicFieldDef definition : definitions) { + if (definition.isActive()) types.put(definition.getCode(), definition.getDataType()); + } + return new DynamicFieldMetadata(types); + } + + public DynamicDataType requireType(String code) { + if (!types.containsKey(code)) throw DynamicFieldException.notFound(code); + return types.get(code); + } + + public Map types() { return types; } + + public DynamicFieldValue notLoaded(String code) { + DynamicDataType type = requireType(code); + DynamicFieldValue cached = absent.get(code); + if (cached != null) return cached; + return absent.computeIfAbsent(code, name -> DynamicFieldValue.notLoaded(name, type)); + } +} diff --git a/teaql-dynamic-fields-api/src/main/java/io/teaql/data/dynamic/DynamicFieldMutation.java b/teaql-dynamic-fields-api/src/main/java/io/teaql/data/dynamic/DynamicFieldMutation.java new file mode 100644 index 00000000..615f26a3 --- /dev/null +++ b/teaql-dynamic-fields-api/src/main/java/io/teaql/data/dynamic/DynamicFieldMutation.java @@ -0,0 +1,40 @@ +package io.teaql.data.dynamic; + +/** Explicit extension intent; Set(null) is a stored NULL, never deletion. */ +public record DynamicFieldMutation(String code, DynamicDataType dataType, Kind kind, Object value) { + public enum Kind { SET, DELETE } + public DynamicFieldMutation { + if (code == null || code.isBlank() || code.startsWith("#") || code.startsWith("_")) + throw new DynamicFieldException("DYNAMIC_FIELD_INVALID_CODE", "Expected a bare dynamic field code"); + java.util.Objects.requireNonNull(dataType, "dataType"); + java.util.Objects.requireNonNull(kind, "kind"); + if (kind == Kind.DELETE && value != null) throw new IllegalArgumentException("Delete has no value"); + if (value != null && kind == Kind.SET) { + boolean valid = switch (dataType) { + case STRING, ENUM -> value instanceof String; + case NUMBER -> value instanceof Number; + case BOOL -> value instanceof Boolean; + case DATE_TIME -> value instanceof Number || value instanceof java.util.Date || value instanceof java.time.temporal.TemporalAccessor; + }; + if (!valid) throw new DynamicFieldException("DYNAMIC_FIELD_TYPE_MISMATCH", "Invalid dynamic field type: " + code); + } + } + public static DynamicFieldMutation set(String code, DynamicDataType type, Object value) { + return new DynamicFieldMutation(code, type, Kind.SET, value); + } + public static DynamicFieldMutation delete(String code, DynamicDataType type) { + return new DynamicFieldMutation(code, type, Kind.DELETE, null); + } + public DynamicFieldValue asLoadedValue() { + if (kind != Kind.SET) throw new IllegalStateException("Delete is not a loaded value"); + if (value == null) return DynamicFieldValue.ofNull(code, dataType); + return switch (dataType) { + case STRING -> DynamicFieldValue.ofString(code, (String) value); + case ENUM -> DynamicFieldValue.ofEnum(code, (String) value); + case NUMBER -> DynamicFieldValue.ofNumber(code, (Number) value); + case BOOL -> DynamicFieldValue.ofBool(code, (Boolean) value); + case DATE_TIME -> DynamicFieldValue.ofDateTime(code, value); + }; + } + @Override public String toString() { return "DynamicFieldMutation[" + code + ", " + kind + ", " + dataType + "]"; } +} diff --git a/teaql-dynamic-fields-api/src/main/java/io/teaql/data/dynamic/DynamicFieldSelection.java b/teaql-dynamic-fields-api/src/main/java/io/teaql/data/dynamic/DynamicFieldSelection.java index 0039d4e9..932402d5 100644 --- a/teaql-dynamic-fields-api/src/main/java/io/teaql/data/dynamic/DynamicFieldSelection.java +++ b/teaql-dynamic-fields-api/src/main/java/io/teaql/data/dynamic/DynamicFieldSelection.java @@ -49,6 +49,11 @@ public DynamicFieldSelection selectAll() { return this; } + public DynamicFieldSelection select(String code, DynamicDataType type) { + entries.add(new DynamicFieldSelectionEntry(java.util.Objects.requireNonNull(code), java.util.Objects.requireNonNull(type))); + return this; + } + public boolean isSelectAll() { return selectAll; } diff --git a/teaql-dynamic-fields-api/src/main/java/io/teaql/data/dynamic/DynamicFieldValue.java b/teaql-dynamic-fields-api/src/main/java/io/teaql/data/dynamic/DynamicFieldValue.java index 3186bdb9..766ce9f3 100644 --- a/teaql-dynamic-fields-api/src/main/java/io/teaql/data/dynamic/DynamicFieldValue.java +++ b/teaql-dynamic-fields-api/src/main/java/io/teaql/data/dynamic/DynamicFieldValue.java @@ -4,14 +4,22 @@ public final class DynamicFieldValue { + public enum State { VALUE, NULL, NOT_LOADED } + private final String fieldCode; private final DynamicDataType dataType; private final Object value; + private final State state; private DynamicFieldValue(String fieldCode, DynamicDataType dataType, Object value) { + this(fieldCode, dataType, value, value == null ? State.NULL : State.VALUE); + } + + private DynamicFieldValue(String fieldCode, DynamicDataType dataType, Object value, State state) { this.fieldCode = Objects.requireNonNull(fieldCode, "fieldCode"); this.dataType = dataType; this.value = value; + this.state = state; } public static DynamicFieldValue ofString(String code, String value) { @@ -38,6 +46,13 @@ public static DynamicFieldValue ofNull(String code, DynamicDataType dataType) { return new DynamicFieldValue(code, dataType, null); } + public static DynamicFieldValue notLoaded(String code, DynamicDataType dataType) { + return new DynamicFieldValue(code, dataType, null, State.NOT_LOADED); + } + + public State state() { return state; } + public boolean isLoaded() { return state != State.NOT_LOADED; } + public String fieldCode() { return fieldCode; } @@ -68,16 +83,17 @@ public boolean equals(Object o) { if (!(o instanceof DynamicFieldValue that)) return false; return fieldCode.equals(that.fieldCode) && dataType == that.dataType + && state == that.state && Objects.equals(value, that.value); } @Override public int hashCode() { - return Objects.hash(fieldCode, dataType, value); + return Objects.hash(fieldCode, dataType, value, state); } @Override public String toString() { - return "DynamicFieldValue{" + fieldCode + '=' + value + " (" + dataType + ")}"; + return "DynamicFieldValue{" + fieldCode + '=' + value + " (" + dataType + ", " + state + ")}"; } } diff --git a/teaql-dynamic-fields-api/src/main/java/io/teaql/data/dynamic/DynamicFieldValues.java b/teaql-dynamic-fields-api/src/main/java/io/teaql/data/dynamic/DynamicFieldValues.java index 06ac00e4..abf55626 100644 --- a/teaql-dynamic-fields-api/src/main/java/io/teaql/data/dynamic/DynamicFieldValues.java +++ b/teaql-dynamic-fields-api/src/main/java/io/teaql/data/dynamic/DynamicFieldValues.java @@ -8,9 +8,39 @@ public final class DynamicFieldValues { private final Map values; + private final DynamicFieldMetadata metadata; + private final java.util.Set selectedCodes; public DynamicFieldValues(Map values) { - this.values = new LinkedHashMap<>(values); + this(inferMetadata(values), values); + } + + public DynamicFieldValues(DynamicFieldMetadata metadata, Map values) { + this.metadata = java.util.Objects.requireNonNull(metadata, "metadata"); + this.values = values.isEmpty() ? Collections.emptyMap() : Collections.unmodifiableMap(new LinkedHashMap<>(values)); + for (Map.Entry entry : this.values.entrySet()) { + if (!entry.getKey().equals(entry.getValue().fieldCode())) throw new IllegalArgumentException("Dynamic field key/code mismatch"); + DynamicDataType declared = metadata.requireType(entry.getKey()); + if (declared != entry.getValue().dataType()) throw DynamicFieldException.typeMismatch(entry.getKey(), declared, entry.getValue().dataType()); + } + // Presence is a view of the private payload map, not a copied field-name set per row. + if (this.values.values().stream().allMatch(DynamicFieldValue::isLoaded)) { + selectedCodes = this.values.keySet(); + } else { + selectedCodes = Collections.unmodifiableSet(new java.util.AbstractSet<>() { + @Override public java.util.Iterator iterator() { + return DynamicFieldValues.this.values.entrySet().stream() + .filter(entry -> entry.getValue().isLoaded()).map(Map.Entry::getKey).iterator(); + } + @Override public int size() { + return (int) DynamicFieldValues.this.values.values().stream().filter(DynamicFieldValue::isLoaded).count(); + } + @Override public boolean contains(Object code) { + DynamicFieldValue value = DynamicFieldValues.this.values.get(code); + return value != null && value.isLoaded(); + } + }); + } } public static DynamicFieldValues empty() { @@ -42,16 +72,28 @@ public Object get(String fieldCode) { } public boolean isSelected(String fieldCode) { - return values.containsKey(fieldCode); + DynamicFieldValue value = values.get(fieldCode); + return value != null && value.isLoaded(); } public boolean isNull(String fieldCode) { DynamicFieldValue v = values.get(fieldCode); - return v != null && v.value() == null; + return v != null && v.state() == DynamicFieldValue.State.NULL; + } + + /** Unlike strict typed getters, this wrapper exposes NotLoaded without I/O or an invented NULL. */ + public DynamicFieldValue field(String fieldCode) { + DynamicFieldValue value = values.get(fieldCode); + return value == null ? metadata.notLoaded(fieldCode) : value; } + public DynamicFieldMetadata metadata() { return metadata; } + + /** Immutable loaded-code view; NULL counts as loaded, NotLoaded does not. */ + public java.util.Set selectedCodes() { return selectedCodes; } + public Map toMap() { - return Collections.unmodifiableMap(values); + return values; } public int size() { @@ -60,9 +102,15 @@ public int size() { private DynamicFieldValue requireSelected(String fieldCode) { DynamicFieldValue v = values.get(fieldCode); - if (v == null) { + if (v == null || !v.isLoaded()) { throw DynamicFieldException.notSelected(fieldCode); } return v; } + + private static DynamicFieldMetadata inferMetadata(Map values) { + Map types = new LinkedHashMap<>(); + values.forEach((code, value) -> types.put(code, value.dataType())); + return new DynamicFieldMetadata(types); + } } diff --git a/teaql-dynamic-fields-api/src/main/java/io/teaql/data/dynamic/DynamicFieldsFacade.java b/teaql-dynamic-fields-api/src/main/java/io/teaql/data/dynamic/DynamicFieldsFacade.java index 07360fe5..ae376047 100644 --- a/teaql-dynamic-fields-api/src/main/java/io/teaql/data/dynamic/DynamicFieldsFacade.java +++ b/teaql-dynamic-fields-api/src/main/java/io/teaql/data/dynamic/DynamicFieldsFacade.java @@ -10,6 +10,28 @@ public interface DynamicFieldsFacade { OwnerBound owner(String ownerType, long ownerId); + /** Prepare definitions for new entities, without an owner ID or owner-data query. */ + default DynamicFieldMetadata metadata(String ownerType) { + throw new UnsupportedOperationException("Dynamic field definition preparation is not supported"); + } + + /** Framework graph participant; not an independent save entry point. */ + default void prepareGraphMutations(Object resource, java.util.List mutations) { + throw new DynamicFieldException("DYNAMIC_FIELD_TRANSACTION_BINDING_REQUIRED", "Dynamic field provider is not bound to the graph transaction"); + } + + default void applyGraphMutations(Object resource, java.util.List mutations) { + throw new DynamicFieldException("DYNAMIC_FIELD_TRANSACTION_BINDING_REQUIRED", "Dynamic field provider is not bound to the graph transaction"); + } + + /** Explicit bulk read; custom facades may override to preserve their permission boundary. */ + default java.util.Map readAll( + java.util.List owners, DynamicFieldSelection selection) { + java.util.Map result = new java.util.LinkedHashMap<>(); + for (DynamicOwnerRef owner : owners) result.put(owner, owner(owner.ownerType(), owner.ownerId()).readAll(selection)); + return result; + } + interface OwnerBound { StringFieldBound string(String fieldCode); NumberFieldBound number(String fieldCode); diff --git a/teaql-dynamic-fields-api/src/main/java/io/teaql/data/dynamic/DynamicFieldsProvider.java b/teaql-dynamic-fields-api/src/main/java/io/teaql/data/dynamic/DynamicFieldsProvider.java index 0418ddb9..f160f3c0 100644 --- a/teaql-dynamic-fields-api/src/main/java/io/teaql/data/dynamic/DynamicFieldsProvider.java +++ b/teaql-dynamic-fields-api/src/main/java/io/teaql/data/dynamic/DynamicFieldsProvider.java @@ -21,4 +21,14 @@ Map loadValues(DynamicFieldContext context, void deleteValue(DynamicFieldContext context, DynamicValueRef valueRef); DynamicFieldCapabilities capabilities(); + + /** Prepare definitions without reading owner values. Durable providers attach provenance. */ + default DynamicFieldMetadata metadata(DynamicFieldContext context, String ownerType) { + return DynamicFieldMetadata.fromDefinitions(listFieldDefs(context, ownerType)); + } + + /** A matching URL/DataSource is insufficient: the graph must own this exact active executor. */ + default boolean participatesInGraphTransaction(Object resource) { return false; } + + default void validateGraphMutation(DynamicFieldContext context, DynamicGraphMutation mutation) { } } diff --git a/teaql-dynamic-fields-api/src/main/java/io/teaql/data/dynamic/DynamicGraphMutation.java b/teaql-dynamic-fields-api/src/main/java/io/teaql/data/dynamic/DynamicGraphMutation.java new file mode 100644 index 00000000..5b8fd512 --- /dev/null +++ b/teaql-dynamic-fields-api/src/main/java/io/teaql/data/dynamic/DynamicGraphMutation.java @@ -0,0 +1,10 @@ +package io.teaql.data.dynamic; + +/** Runtime-owned graph write request. Provider scopes/permissions are still resolved from context. */ +public record DynamicGraphMutation(DynamicOwnerRef owner, DynamicFieldMutation mutation, DynamicFieldMetadata sourceMetadata) { + public DynamicGraphMutation { + java.util.Objects.requireNonNull(owner, "owner"); + java.util.Objects.requireNonNull(mutation, "mutation"); + java.util.Objects.requireNonNull(sourceMetadata, "source metadata"); + } +} diff --git a/teaql-dynamic-fields-api/src/main/java/io/teaql/data/dynamic/InMemoryDynamicFieldsProvider.java b/teaql-dynamic-fields-api/src/main/java/io/teaql/data/dynamic/InMemoryDynamicFieldsProvider.java index 80e82b20..611100f2 100644 --- a/teaql-dynamic-fields-api/src/main/java/io/teaql/data/dynamic/InMemoryDynamicFieldsProvider.java +++ b/teaql-dynamic-fields-api/src/main/java/io/teaql/data/dynamic/InMemoryDynamicFieldsProvider.java @@ -24,6 +24,7 @@ public class InMemoryDynamicFieldsProvider implements DynamicFieldsProvider { private static final Logger LOG = Logger.getLogger(InMemoryDynamicFieldsProvider.class.getName()); private final AtomicLong idSequence = new AtomicLong(1); + private static final Object EXPLICIT_NULL = new Object(); // key: scope/ownerType/code -> DynamicFieldDef private final Map fieldDefs = new ConcurrentHashMap<>(); @@ -112,6 +113,7 @@ public Map loadValues( } Map result = new HashMap<>(); + Map metadata = new HashMap<>(); for (DynamicOwnerRef ownerRef : ownerRefs) { List values = new ArrayList<>(); @@ -123,9 +125,11 @@ public Map loadValues( if (!def.isActive()) continue; String vKey = valueKey(context, ownerRef, def.getId()); Object val = fieldValues.get(vKey); - values.add(toFieldValue(def.getCode(), def.getDataType(), val)); + if (val != null) values.add(toFieldValue(def.getCode(), def.getDataType(), val)); } - result.put(ownerRef, DynamicFieldValues.of(values)); + DynamicFieldMetadata fields = metadata.computeIfAbsent(ownerRef.ownerType(), + type -> DynamicFieldMetadata.fromDefinitions(defs)); + result.put(ownerRef, withMetadata(fields, values)); continue; } // Load selected fields @@ -138,13 +142,21 @@ public Map loadValues( if (def == null) continue; String vKey = valueKey(context, ownerRef, def.getId()); Object val = fieldValues.get(vKey); - values.add(toFieldValue(entry.code(), entry.dataType(), val)); + if (val != null) values.add(toFieldValue(entry.code(), entry.dataType(), val)); } - result.put(ownerRef, DynamicFieldValues.of(values)); + DynamicFieldMetadata fields = metadata.computeIfAbsent(ownerRef.ownerType(), + type -> DynamicFieldMetadata.fromDefinitions(listFieldDefs(context, type))); + result.put(ownerRef, withMetadata(fields, values)); } return result; } + private static DynamicFieldValues withMetadata(DynamicFieldMetadata metadata, List values) { + Map selected = new HashMap<>(); + values.forEach(value -> selected.put(value.fieldCode(), value)); + return new DynamicFieldValues(metadata, selected); + } + @Override public void saveValue(DynamicFieldContext context, DynamicSetCommand command) { DynamicFieldRef ref = DynamicFieldRef.of( @@ -164,10 +176,11 @@ public void saveValue(DynamicFieldContext context, DynamicSetCommand command) { } String vKey = valueKey(context, command.ownerRef(), def.getId()); - fieldValues.remove(vKey); - if (command.value() != null) { - fieldValues.put(vKey, command.value()); + if (def.getDataType() != command.dataType()) { + throw DynamicFieldException.typeMismatch(command.fieldCode(), def.getDataType(), command.dataType()); } + // Explicit NULL is persisted; removing the entry is a distinct delete intent. + fieldValues.put(vKey, command.value() == null ? EXPLICIT_NULL : command.value()); } @Override @@ -204,7 +217,7 @@ private static String valueKey(DynamicFieldContext context, DynamicOwnerRef owne } private static DynamicFieldValue toFieldValue(String code, DynamicDataType dataType, Object val) { - if (val == null) { + if (val == EXPLICIT_NULL) { return DynamicFieldValue.ofNull(code, dataType); } return switch (dataType) { diff --git a/teaql-dynamic-fields-api/src/test/java/io/teaql/data/dynamic/DynamicFieldValueStateTest.java b/teaql-dynamic-fields-api/src/test/java/io/teaql/data/dynamic/DynamicFieldValueStateTest.java new file mode 100644 index 00000000..d1346511 --- /dev/null +++ b/teaql-dynamic-fields-api/src/test/java/io/teaql/data/dynamic/DynamicFieldValueStateTest.java @@ -0,0 +1,89 @@ +package io.teaql.data.dynamic; + +import java.util.Map; +import org.junit.Test; +import static org.junit.Assert.*; + +public class DynamicFieldValueStateTest { + @Test + public void boundDefinitionsShareTheDictionaryButKeepStorageAndScopeIdentity() { + var unbound = new DynamicFieldMetadata(Map.of("note", DynamicDataType.STRING)); + Object storage = new Object(); + var bound = unbound.withStorageBinding(storage, "PROFILE", "one", "School"); + assertSame(unbound.types(), bound.types()); + assertSame(bound, bound.withStorageBinding(storage, "PROFILE", "one", "School")); + assertTrue(bound.matchesStorage(storage, "PROFILE", "one", "School")); + assertFalse(unbound.matchesStorage(storage, "PROFILE", "one", "School")); + assertFalse(unbound.matchesStorage(null, null, null, null)); + assertThrows(NullPointerException.class, () -> unbound.withStorageBinding(null, null, null, null)); + assertFalse(bound.matchesStorage(new Object(), "PROFILE", "one", "School")); + assertFalse(bound.matchesStorage(storage, "PROFILE", "two", "School")); + assertFalse(bound.matchesStorage(storage, "PROFILE", "one", "Platform")); + assertThrows(UnsupportedOperationException.class, () -> bound.types().clear()); + assertFalse(bound.toString().contains("PROFILE")); + } + + @Test + public void zeroFalseEmptyNullAndNotLoadedRemainDistinct() { + for (DynamicFieldValue value : new DynamicFieldValue[]{ + DynamicFieldValue.ofNumber("number", 0), DynamicFieldValue.ofBool("flag", false), + DynamicFieldValue.ofString("text", "")}) { + assertEquals(DynamicFieldValue.State.VALUE, value.state()); + assertTrue(value.isLoaded()); + } + DynamicFieldValue nil = DynamicFieldValue.ofNull("text", DynamicDataType.STRING); + DynamicFieldValue missing = DynamicFieldValue.notLoaded("text", DynamicDataType.STRING); + assertEquals(DynamicFieldValue.State.NULL, nil.state()); + assertEquals(DynamicFieldValue.State.NOT_LOADED, missing.state()); + assertNull(nil.value()); assertNull(missing.value()); + assertNotEquals(nil, missing); + } + + @Test + public void sharedMetadataKeepsRowPayloadsPrivateAndSynthesizesOnlyKnownWrappers() { + DynamicFieldMetadata metadata = new DynamicFieldMetadata(Map.of("note", DynamicDataType.STRING, "extra", DynamicDataType.STRING)); + DynamicFieldValues first = new DynamicFieldValues(metadata, Map.of("note", DynamicFieldValue.ofString("note", "private"))); + DynamicFieldValues second = new DynamicFieldValues(metadata, Map.of("note", DynamicFieldValue.ofNull("note", DynamicDataType.STRING))); + assertSame(first.metadata(), second.metadata()); + assertEquals(DynamicFieldValue.State.VALUE, first.field("note").state()); + assertEquals(DynamicFieldValue.State.NULL, second.field("note").state()); + assertEquals(DynamicFieldValue.State.NOT_LOADED, first.field("extra").state()); + assertFalse(first.isSelected("extra")); assertFalse(first.isNull("extra")); + assertEquals("private", first.field("note").value()); + assertThrows(DynamicFieldException.class, () -> first.field("unknown")); + assertEquals(1, first.toMap().size()); + assertEquals(java.util.Set.of("note"), first.selectedCodes()); + assertEquals(first.selectedCodes(), second.selectedCodes()); + assertSame(first.selectedCodes(), first.selectedCodes()); + assertThrows(UnsupportedOperationException.class, () -> first.selectedCodes().clear()); + } + + @Test + public void explicitNotLoadedWrapperDoesNotPretendToBeSelectedNull() { + DynamicFieldValues values = DynamicFieldValues.of(java.util.List.of(DynamicFieldValue.notLoaded("note", DynamicDataType.STRING))); + assertFalse(values.isSelected("note")); assertFalse(values.isNull("note")); + assertThrows(DynamicFieldException.class, () -> values.getString("note")); + assertNull(values.field("note").value()); + assertTrue(values.selectedCodes().isEmpty()); + assertFalse(values.selectedCodes().contains("note")); + } + + @Test + public void missingWrappersShareOnlyValueFreeMetadataAndCannotChangeLoadedRows() { + DynamicFieldMetadata metadata = new DynamicFieldMetadata(Map.of("note", DynamicDataType.STRING)); + DynamicFieldValues first = new DynamicFieldValues(metadata, Map.of()); + DynamicFieldValues second = new DynamicFieldValues(metadata, Map.of()); + DynamicFieldValue missing = first.field("note"); + java.util.stream.IntStream.range(0, 10_000).parallel().forEach(i -> assertSame(missing, second.field("note"))); + assertSame(missing, first.field("note")); + DynamicFieldValues loaded = new DynamicFieldValues(metadata, Map.of("note", DynamicFieldValue.ofString("note", "private"))); + DynamicFieldValues nil = new DynamicFieldValues(metadata, Map.of("note", DynamicFieldValue.ofNull("note", DynamicDataType.STRING))); + assertNotSame(missing, loaded.field("note")); + assertNotSame(missing, nil.field("note")); + assertEquals(DynamicFieldValue.State.NOT_LOADED, missing.state()); + assertNull(missing.value()); + assertFalse(first.isSelected("note")); + assertEquals("private", loaded.field("note").value()); + assertThrows(DynamicFieldException.class, () -> first.field("unknown")); + } +} diff --git a/teaql-dynamic-fields-api/src/test/java/io/teaql/data/dynamic/InMemoryDynamicFieldsProviderTest.java b/teaql-dynamic-fields-api/src/test/java/io/teaql/data/dynamic/InMemoryDynamicFieldsProviderTest.java index 1354f86c..74977b79 100644 --- a/teaql-dynamic-fields-api/src/test/java/io/teaql/data/dynamic/InMemoryDynamicFieldsProviderTest.java +++ b/teaql-dynamic-fields-api/src/test/java/io/teaql/data/dynamic/InMemoryDynamicFieldsProviderTest.java @@ -142,10 +142,10 @@ public void testSelectAllLoadsActiveFields() { DynamicFieldValues values = provider.loadValues(context, owner, new DynamicFieldSelection().selectAll()); - assertEquals(3, values.size()); // all 3 active fields + assertEquals(2, values.size()); // two stored fields; the known absent field is synthesized as NotLoaded assertEquals("A-10086", values.getString("customer_asset_no")); assertEquals(42, values.getNumber("priority_score")); - assertTrue(values.isNull("enabled_for_custom_flow")); + assertEquals(DynamicFieldValue.State.NOT_LOADED, values.field("enabled_for_custom_flow").state()); } @Test @@ -154,7 +154,8 @@ public void testNullValueIsDistinguishedFromUnselected() { DynamicFieldContext context = globalCtx(); DynamicOwnerRef owner = DynamicOwnerRef.of("Platform", 1001L); - // Don't write any value — field exists but has no value + provider.saveValue(context, DynamicSetCommand.of(owner, "customer_asset_no", DynamicDataType.STRING, + null, "test", "explicit null")); DynamicFieldValues values = provider.loadValues(context, owner, new DynamicFieldSelection().selectString("customer_asset_no")); @@ -162,6 +163,26 @@ public void testNullValueIsDistinguishedFromUnselected() { assertTrue("Value should be null", values.isNull("customer_asset_no")); } + @Test + public void batchSharesDefinitionsButNotValuesAndKnownUnselectedFieldIsNotLoaded() { + InMemoryDynamicFieldsProvider provider = createProvider(); + DynamicFieldContext context = globalCtx(); + DynamicOwnerRef first = DynamicOwnerRef.of("Platform", 1L); + DynamicOwnerRef second = DynamicOwnerRef.of("Platform", 2L); + provider.saveValue(context, DynamicSetCommand.of(first, "customer_asset_no", DynamicDataType.STRING, "private", "verify", "set first value")); + provider.saveValue(context, DynamicSetCommand.of(second, "customer_asset_no", DynamicDataType.STRING, null, "verify", "set explicit null")); + java.util.Map batch = provider.loadValues(context, + java.util.List.of(first, second), new DynamicFieldSelection().selectString("customer_asset_no")); + assertSame(batch.get(first).metadata(), batch.get(second).metadata()); + assertEquals(DynamicFieldValue.State.VALUE, batch.get(first).field("customer_asset_no").state()); + assertEquals(DynamicFieldValue.State.NULL, batch.get(second).field("customer_asset_no").state()); + assertEquals(DynamicFieldValue.State.NOT_LOADED, batch.get(first).field("priority_score").state()); + assertEquals("private", batch.get(first).getString("customer_asset_no")); + assertNull(batch.get(second).getString("customer_asset_no")); + assertEquals("DYNAMIC_FIELD_NOT_FOUND", assertThrows(DynamicFieldException.class, + () -> batch.get(first).field("unknown")).errorCode()); + } + @Test public void testFacadeEndToEnd() { InMemoryDynamicFieldsProvider provider = createProvider(); @@ -247,9 +268,29 @@ public void testDeleteValue() { // Delete provider.deleteValue(context, DynamicValueRef.of(owner, def.getId())); - // Verify deleted (should be null) + // Deleted is absent, not a persisted explicit NULL. values = provider.loadValues(context, owner, new DynamicFieldSelection().selectString("customer_asset_no")); - assertTrue(values.isNull("customer_asset_no")); + assertEquals(DynamicFieldValue.State.NOT_LOADED, values.field("customer_asset_no").state()); + assertFalse(values.isNull("customer_asset_no")); + } + + @Test + public void incompatibleWriteCannotEraseAStoredValueOrTurnAbsenceIntoNull() { + var provider = createProvider(); + var context = globalCtx(); + var owner = DynamicOwnerRef.of("Platform", 1001L); + var selection = new DynamicFieldSelection().selectString("customer_asset_no"); + assertEquals(DynamicFieldValue.State.NOT_LOADED, provider.loadValues(context, owner, selection) + .field("customer_asset_no").state()); + provider.saveValue(context, DynamicSetCommand.of(owner, "customer_asset_no", DynamicDataType.STRING, + "retained", "test", "store original")); + assertThrows(DynamicFieldException.class, () -> provider.saveValue(context, DynamicSetCommand.of( + owner, "customer_asset_no", DynamicDataType.NUMBER, 0, "test", "reject wrong type"))); + assertEquals("retained", provider.loadValues(context, owner, selection).getString("customer_asset_no")); + provider.saveValue(context, DynamicSetCommand.of(owner, "customer_asset_no", DynamicDataType.STRING, + null, "test", "store explicit null")); + assertEquals(DynamicFieldValue.State.NULL, provider.loadValues(context, owner, selection) + .field("customer_asset_no").state()); } } diff --git a/teaql-dynamic-fields-jdbc/src/main/java/io/teaql/data/dynamic/jdbc/JdbcDynamicFieldsProvider.java b/teaql-dynamic-fields-jdbc/src/main/java/io/teaql/data/dynamic/jdbc/JdbcDynamicFieldsProvider.java index 32210f8e..5e390dc8 100644 --- a/teaql-dynamic-fields-jdbc/src/main/java/io/teaql/data/dynamic/jdbc/JdbcDynamicFieldsProvider.java +++ b/teaql-dynamic-fields-jdbc/src/main/java/io/teaql/data/dynamic/jdbc/JdbcDynamicFieldsProvider.java @@ -18,7 +18,6 @@ *

Call {@link #ensureSchema()} once at application startup to auto-create the tables.

*/ public class JdbcDynamicFieldsProvider implements DynamicFieldsProvider { - private static final Logger LOG = Logger.getLogger(JdbcDynamicFieldsProvider.class.getName()); private final SqlExecutionAdapter executor; @@ -97,6 +96,39 @@ public void ensureSchema() { // ─── Field Definition Management ─────────────────────────────────── + @Override public boolean participatesInGraphTransaction(Object resource) { + return resource == executor && executor.hasActiveTransaction(); + } + + @Override public void validateGraphMutation(DynamicFieldContext context, DynamicGraphMutation request) { + if (!request.sourceMetadata().matchesStorage(requireStorageIdentity(), context.scopeType(), context.scopeId(), request.owner().ownerType())) { + throw new DynamicFieldException("DYNAMIC_FIELD_STORAGE_PROVENANCE_MISMATCH", "Reload the dynamic view through the current storage provider and scope"); + } + if (request.mutation().kind() == DynamicFieldMutation.Kind.SET && request.mutation().value() != null) { + if (request.mutation().dataType() == DynamicDataType.NUMBER) exactLong((Number) request.mutation().value()); + if (request.mutation().dataType() == DynamicDataType.DATE_TIME) toTimestamp(request.mutation().value()); + } + } + + private Object requireStorageIdentity() { + Object identity = executor.storageIdentity(); + if (identity == null) throw new DynamicFieldException("DYNAMIC_FIELD_STORAGE_IDENTITY_REQUIRED", + "The storage adapter must supply an opaque instance identity for dynamic field views"); + return identity; + } + + @Override public DynamicFieldMetadata metadata(DynamicFieldContext context, String ownerType) { + return DynamicFieldMetadata.fromDefinitions(listFieldDefs(context, ownerType)) + .withStorageBinding(requireStorageIdentity(), context.scopeType(), context.scopeId(), ownerType); + } + + private static long exactLong(Number value) { + try { return new java.math.BigDecimal(value.toString()).longValueExact(); } + catch (ArithmeticException | NumberFormatException invalid) { + throw new DynamicFieldException("DYNAMIC_FIELD_NUMERIC_RANGE", "This JDBC dynamic NUMBER column requires an exact 64-bit integer"); + } + } + /** * Registers a field definition. Assigns an ID via teaql_id_space if not set. */ @@ -158,7 +190,7 @@ public DynamicFieldValues loadValues(DynamicFieldContext context, DynamicOwnerRe context.scopeType(), context.scopeId(), ownerRef.ownerType(), ownerRef.ownerId() }); - return buildFieldValues(rows, selection); + return buildFieldValues(rows, selection, metadata(context, ownerRef.ownerType())); } @Override @@ -170,8 +202,17 @@ public Map loadValues( return Collections.emptyMap(); } + Map> types = new LinkedHashMap<>(); + for (DynamicOwnerRef owner : ownerRefs) types.computeIfAbsent(owner.ownerType(), ignored -> new ArrayList<>()).add(owner); + if (types.size() > 1) { + Map result = new LinkedHashMap<>(); + for (List group : types.values()) result.putAll(loadValues(context, group, selection)); + return result; + } + // Build IN clause dynamically String ownerType = ownerRefs.get(0).ownerType(); + DynamicFieldMetadata metadata = metadata(context, ownerType); StringBuilder sql = new StringBuilder(); sql.append("SELECT v.owner_id, v.field_id, d.code, d.data_type, "); sql.append("v.string_value, v.number_value, v.bool_value, v.datetime_value, v.enum_value "); @@ -204,7 +245,7 @@ public Map loadValues( Map result = new LinkedHashMap<>(); for (DynamicOwnerRef ref : ownerRefs) { List> ownerRows = grouped.getOrDefault(ref.ownerId(), Collections.emptyList()); - result.put(ref, buildFieldValues(ownerRows, selection)); + result.put(ref, buildFieldValues(ownerRows, selection, metadata)); } return result; } @@ -220,6 +261,7 @@ public void saveValue(DynamicFieldContext context, DynamicSetCommand command) { if (def == null) { throw DynamicFieldException.notFound(command.fieldCode()); } + if (def.getDataType() != command.dataType()) throw DynamicFieldException.typeMismatch(command.fieldCode(), def.getDataType(), command.dataType()); if (!def.isActive()) { throw new DynamicFieldException("DYNAMIC_FIELD_NOT_ACTIVE", "Dynamic field '" + command.fieldCode() + "' is not active (status: " + def.getStatus() + ")"); @@ -241,7 +283,7 @@ public void saveValue(DynamicFieldContext context, DynamicSetCommand command) { if (command.value() != null) { switch (command.dataType()) { case STRING -> stringVal = command.value().toString(); - case NUMBER -> numberVal = ((Number) command.value()).longValue(); + case NUMBER -> numberVal = exactLong((Number) command.value()); case BOOL -> boolVal = boolToInt((Boolean) command.value()); case DATE_TIME -> datetimeVal = toTimestamp(command.value()); case ENUM -> enumVal = command.value().toString(); @@ -294,8 +336,8 @@ public DynamicFieldCapabilities capabilities() { // ─── Internal Helpers ────────────────────────────────────────────── private DynamicFieldValues buildFieldValues(List> rows, - DynamicFieldSelection selection) { - List values = new ArrayList<>(); + DynamicFieldSelection selection, DynamicFieldMetadata metadata) { + Map values = new LinkedHashMap<>(); Set selectedCodes = null; if (!selection.isSelectAll()) { @@ -308,6 +350,7 @@ private DynamicFieldValues buildFieldValues(List> rows, for (Map row : rows) { String code = (String) row.get("code"); String dataTypeStr = (String) row.get("data_type"); + if (!metadata.types().containsKey(code)) continue; // Filter by selection if not selectAll if (selectedCodes != null && !selectedCodes.contains(code)) { @@ -315,10 +358,10 @@ private DynamicFieldValues buildFieldValues(List> rows, } DynamicDataType dataType = DynamicDataType.valueOf(dataTypeStr); - values.add(extractFieldValue(code, dataType, row)); + values.put(code, extractFieldValue(code, dataType, row)); } - return DynamicFieldValues.of(values); + return new DynamicFieldValues(metadata, values); } private DynamicFieldValue extractFieldValue(String code, DynamicDataType dataType, @@ -396,8 +439,12 @@ protected String resolveLogicalTypeName(DynamicFieldDef def) { } protected long toTimestamp(Object value) { - if (value instanceof Number n) { return n.longValue(); } - return System.currentTimeMillis(); + if (value instanceof Number n) return exactLong(n); + if (value instanceof java.util.Date date) return date.getTime(); + if (value instanceof java.time.Instant instant) return instant.toEpochMilli(); + if (value instanceof java.time.OffsetDateTime date) return date.toInstant().toEpochMilli(); + if (value instanceof java.time.ZonedDateTime date) return date.toInstant().toEpochMilli(); + throw new DynamicFieldException("DYNAMIC_FIELD_DATETIME_ZONE_REQUIRED", "Use epoch milliseconds or an instant with an explicit timezone"); } protected DynamicFieldValue toStringFieldValue(String code, Object v) { diff --git a/teaql-dynamic-fields-jdbc/src/test/java/io/teaql/data/dynamic/jdbc/JdbcDynamicFieldsProviderTest.java b/teaql-dynamic-fields-jdbc/src/test/java/io/teaql/data/dynamic/jdbc/JdbcDynamicFieldsProviderTest.java index 04efa478..456eb18f 100644 --- a/teaql-dynamic-fields-jdbc/src/test/java/io/teaql/data/dynamic/jdbc/JdbcDynamicFieldsProviderTest.java +++ b/teaql-dynamic-fields-jdbc/src/test/java/io/teaql/data/dynamic/jdbc/JdbcDynamicFieldsProviderTest.java @@ -20,6 +20,60 @@ public class JdbcDynamicFieldsProviderTest { private JdbcDynamicFieldsProvider provider; + @Test + public void newEntityMetadataNeedsNoOwnerDataReadAndBindsToExecutorNotProvider() { + DataSource dataSource = new SimpleDataSource("jdbc:h2:mem:metadata_" + System.nanoTime() + ";DB_CLOSE_DELAY=-1"); + var executor = new JdbcSqlExecutor(dataSource) { + @Override public List> queryForList(String sql, Object[] parameters) { + if (sql.contains("teaql_dynamic_field_value")) fail("metadata preparation read owner data"); + return super.queryForList(sql, parameters); + } + }; + var first = new JdbcDynamicFieldsProvider(executor); + first.ensureSchema(); + first.registerFieldDef(globalCtx(), createStringField("note", "Note")); + var metadata = new DefaultDynamicFieldsFacade(first).metadata("Platform"); + assertEquals(DynamicDataType.STRING, metadata.requireType("note")); + var rebuilt = new JdbcDynamicFieldsProvider(executor).metadata(globalCtx(), "Platform"); + assertTrue(metadata.matchesStorage(executor.storageIdentity(), "GLOBAL", "default", "Platform")); + assertTrue(rebuilt.matchesStorage(executor.storageIdentity(), "GLOBAL", "default", "Platform")); + assertFalse(metadata.matchesStorage(new JdbcSqlExecutor(dataSource).storageIdentity(), "GLOBAL", "default", "Platform")); + } + + @Test + public void adapterWithoutAnOpaqueStorageIdentityCannotCreateWritableViews() { + var executor = new JdbcSqlExecutor(new SimpleDataSource("jdbc:h2:mem:unbound_" + System.nanoTime() + ";DB_CLOSE_DELAY=-1")) { + @Override public Object storageIdentity() { return null; } + }; + var unsupported = new JdbcDynamicFieldsProvider(executor); + unsupported.ensureSchema(); + var error = assertThrows(DynamicFieldException.class, () -> unsupported.metadata(globalCtx(), "Platform")); + assertEquals("DYNAMIC_FIELD_STORAGE_IDENTITY_REQUIRED", error.errorCode()); + } + + @Test + public void metadataIsSharedForBatchButOwnerTypesAndPayloadsRemainIsolated() { + DynamicFieldContext context = globalCtx(); + provider.registerFieldDef(context, createStringField("note", "Note")); + DynamicFieldDef schoolField = createStringField("note", "School note"); + schoolField.setOwnerType("School"); + provider.registerFieldDef(context, schoolField); + DynamicOwnerRef first = DynamicOwnerRef.of("Platform", 1L); + DynamicOwnerRef second = DynamicOwnerRef.of("Platform", 2L); + DynamicOwnerRef school = DynamicOwnerRef.of("School", 1L); + provider.saveValue(context, DynamicSetCommand.of(first, "note", DynamicDataType.STRING, "platform", "verify", "set platform")); + provider.saveValue(context, DynamicSetCommand.of(second, "note", DynamicDataType.STRING, null, "verify", "set known null")); + provider.saveValue(context, DynamicSetCommand.of(school, "note", DynamicDataType.STRING, "school", "verify", "set school")); + Map batch = provider.loadValues(context, List.of(first, second, school), new DynamicFieldSelection().selectString("note")); + assertSame(batch.get(first).metadata(), batch.get(second).metadata()); + assertEquals("platform", batch.get(first).getString("note")); + assertEquals("school", batch.get(school).getString("note")); + assertEquals(DynamicFieldValue.State.NULL, batch.get(second).field("note").state()); + DynamicFieldValues omitted = provider.loadValues(context, first, new DynamicFieldSelection().selectString("other")); + assertEquals(DynamicFieldValue.State.NOT_LOADED, omitted.field("note").state()); + assertThrows(DynamicFieldException.class, () -> omitted.field("undefined")); + } + @Before public void setUp() { // H2 in-memory database diff --git a/teaql-jackson/src/main/java/io/teaql/jackson/BaseEntityJsonSerializer.java b/teaql-jackson/src/main/java/io/teaql/jackson/BaseEntityJsonSerializer.java index 60ea85b3..72900ac4 100644 --- a/teaql-jackson/src/main/java/io/teaql/jackson/BaseEntityJsonSerializer.java +++ b/teaql-jackson/src/main/java/io/teaql/jackson/BaseEntityJsonSerializer.java @@ -9,21 +9,60 @@ import java.util.Map; public class BaseEntityJsonSerializer extends JsonSerializer { + private static final Object ACTIVE_PATH = new Object(); @Override public void serialize(BaseEntity value, JsonGenerator gen, SerializerProvider serializers) throws IOException { - gen.writeStartObject(); - if (value.getId() != null) { - gen.writeObjectField(BaseEntity.ID_PROPERTY, value.getId()); + @SuppressWarnings("unchecked") + var path = (java.util.Set) serializers.getAttribute(ACTIVE_PATH); + if (path == null) { + path = java.util.Collections.newSetFromMap(new java.util.IdentityHashMap<>()); + serializers.setAttribute(ACTIVE_PATH, path); } - if (value.getVersion() != null) { - gen.writeObjectField(BaseEntity.VERSION_PROPERTY, value.getVersion()); + boolean added = path.add(value); + try { + gen.writeStartObject(); + if (value.getId() != null) { + serializers.defaultSerializeField(BaseEntity.ID_PROPERTY, value.getId(), gen); + } + if (value.getVersion() != null) { + serializers.defaultSerializeField(BaseEntity.VERSION_PROPERTY, value.getVersion(), gen); + } + // A cycle retains the existing identity-only reference shape, without + // recursively reserializing the same object's business fields. + if (!added) { + gen.writeEndObject(); + return; + } + var state = value.__internalLoadState(); + var layout = state.layout(); + if (layout.isGenerated()) { + for (String field : layout.indexes().keySet()) { + if (BaseEntity.ID_PROPERTY.equals(field) || BaseEntity.VERSION_PROPERTY.equals(field)) continue; + if (!state.isLoaded(field)) continue; + String member = layout.memberName(field); + // Preserve this provider's graph path, configured serializers + // and per-call attributes during nested serialization. + serializers.defaultSerializeField(member, value.__internalGet(member), gen); + } + for (String relation : layout.relationNames()) { + if (layout.findIndex(relation) != null || !state.isLoaded(relation)) continue; + serializers.defaultSerializeField(relation, value.__internalGet(relation), gen); + } + } + for (Map.Entry entry : value.getAdditionalInfo().entrySet()) { + if (BaseEntity.ID_PROPERTY.equals(entry.getKey()) || BaseEntity.VERSION_PROPERTY.equals(entry.getKey()) + || layout.findIndex(entry.getKey()) != null + || layout.relationNames().contains(entry.getKey())) continue; + serializers.defaultSerializeField(entry.getKey(), entry.getValue(), gen); + } + gen.writeEndObject(); + } finally { + if (added) path.remove(value); + // Retain an empty, value-free guard in this serialization provider, + // so list rows reuse it rather than allocating one set per entity. } - for (Map.Entry entry : value.getAdditionalInfo().entrySet()) { - gen.writeObjectField(entry.getKey(), entry.getValue()); - } - gen.writeEndObject(); } @Override diff --git a/teaql-jackson/src/test/java/io/teaql/jackson/BaseEntitySerializationTest.java b/teaql-jackson/src/test/java/io/teaql/jackson/BaseEntitySerializationTest.java index 3e7a79b6..c9fccc24 100644 --- a/teaql-jackson/src/test/java/io/teaql/jackson/BaseEntitySerializationTest.java +++ b/teaql-jackson/src/test/java/io/teaql/jackson/BaseEntitySerializationTest.java @@ -3,6 +3,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -10,6 +11,188 @@ import org.junit.Test; public class BaseEntitySerializationTest { + static final class NamedRelationsEntity extends BaseEntity { + static final io.teaql.core.FieldLayout LAYOUT = io.teaql.core.FieldLayout.installGenerated( + io.teaql.core.FieldLayout.generated(NamedRelationsEntity.class, "named-relations-v1", + java.util.Map.of("id", 0, "version", 1), + java.util.Map.of("id", java.util.List.of("id", "id"), + "version", java.util.List.of("version", "version")), + java.util.Set.of("childList", "child"))); + io.teaql.core.SmartList childList; + BaseEntity child; + @Override public void __internalSet(String field, Object value) { + switch (field) { + case "childList" -> { markPropertyLoaded(field); childList = (io.teaql.core.SmartList) value; } + case "child" -> { markPropertyLoaded(field); child = (BaseEntity) value; } + default -> super.__internalSet(field, value); + } + } + @Override public Object __internalGet(String field) { + return switch (field) { + case "childList" -> childList; + case "child" -> child; + default -> super.__internalGet(field); + }; + } + } + + @Test public void loadedNamedRelationsSerializeWithoutFixedIndexesOrBeanReflection() throws Exception { + var parent = new NamedRelationsEntity(); parent.__internalSet("id", 1L); parent.__internalSet("version", 1L); + var child = new JsonGraphEntity(); child.__internalSet("id", 1L); child.__internalSet("version", 1L); child.__internalSet("name", "same ID different type"); + var list = new io.teaql.core.SmartList(); list.add(child); + parent.__internalSet("childList", list); + parent.__internalSet("child", null); + parent.putAdditional("childList", "must not replace the declared relation"); + parent.addDynamicProperty("childList", "readonly summary"); + var state = parent.__internalLoadState(); + boolean hadLedger = parent.__internalHasMutationLedger(); + var mapper = new ObjectMapper().registerModule(TeaQLModule.INSTANCE); + var json = mapper.readTree(mapper.writeValueAsString(parent)); + assertEquals("same ID different type", json.get("childList").get(0).get("name").asText()); + assertEquals("readonly summary", json.get("_childList").asText()); + assertTrue(json.get("child").isNull()); + assertTrue(state == parent.__internalLoadState()); + assertEquals(hadLedger, parent.__internalHasMutationLedger()); + assertNull(NamedRelationsEntity.LAYOUT.findIndex("childList")); + var empty = new NamedRelationsEntity(); empty.__internalSet("childList", new io.teaql.core.SmartList()); + json = mapper.readTree(mapper.writeValueAsString(empty)); + assertTrue(json.get("childList").isArray()); assertEquals(0, json.get("childList").size()); + assertFalse(json.has("child")); + json = mapper.readTree(mapper.writeValueAsString(new NamedRelationsEntity())); + assertFalse(json.has("childList")); assertFalse(json.has("child")); + } + + static final class JsonGraphEntity extends BaseEntity { + static final io.teaql.core.FieldLayout LAYOUT = io.teaql.core.FieldLayout.installGenerated( + io.teaql.core.FieldLayout.generated(JsonGraphEntity.class, "nested-json-v1", + java.util.Map.of("id",0,"version",1,"name",2,"child",3), + java.util.Map.of("id",java.util.List.of("id","id"),"version",java.util.List.of("version","version"), + "name",java.util.List.of("name","name"),"child",java.util.List.of("child","child_id")),java.util.Set.of())); + String name; + BaseEntity child; + @Override public void __internalSet(String field,Object value) { + markPropertyLoaded(field); + switch (field) { case "name" -> name=(String)value; case "child" -> child=(BaseEntity)value; default -> super.__internalSet(field,value); } + } + @Override public Object __internalGet(String field) { + return switch (field) { case "name" -> name; case "child" -> child; default -> super.__internalGet(field); }; + } + } + + @Test public void nestedGraphsKeepThreeNamespacesAndLiteralPayloadWithoutInternalMetadata() throws Exception { + var parent = new JsonGraphEntity(); parent.__internalSet("id",1L); parent.__internalSet("version",1L); parent.__internalSet("name","parent"); + var child = new JsonGraphEntity(); child.__internalSet("id",2L); child.__internalSet("version",1L); child.__internalSet("name","native"); + child.addDynamicProperty("name","derived"); child.setComment("private child reason"); + child.setDynamicFieldValues(io.teaql.data.dynamic.DynamicFieldValues.of(java.util.List.of( + io.teaql.data.dynamic.DynamicFieldValue.ofString("name","persistent"), + io.teaql.data.dynamic.DynamicFieldValue.ofNull("nil",io.teaql.data.dynamic.DynamicDataType.STRING), + io.teaql.data.dynamic.DynamicFieldValue.notLoaded("missing",io.teaql.data.dynamic.DynamicDataType.STRING)))); + parent.__internalSet("child",child); parent.addDynamicProperty("children",java.util.List.of(child)); + parent.addDynamicProperty("literal",java.util.Map.of("_comment","literal business data")); + var mapper=new ObjectMapper().registerModule(TeaQLModule.INSTANCE); + var json=mapper.readTree(mapper.writeValueAsString(parent)); + for (var node : java.util.List.of(json.get("child"),json.get("_children").get(0))) { + assertEquals("native",node.get("name").asText()); assertEquals("derived",node.get("_name").asText()); + assertEquals("persistent",node.get("#name").asText()); assertTrue(node.get("#nil").isNull()); + assertFalse(node.has("#missing")); assertFalse(node.has("comment")); assertFalse(node.has("entityMutationLedger")); + assertFalse(node.has("child")); + } + assertEquals("literal business data",json.get("_literal").get("_comment").asText()); + child.__internalSet("child",null); + json=mapper.readTree(mapper.writeValueAsString(parent)); assertTrue(json.get("child").get("child").isNull()); + } + public static class CyclicEntity extends BaseEntity { + static final io.teaql.core.FieldLayout LAYOUT = io.teaql.core.FieldLayout.installGenerated( + io.teaql.core.FieldLayout.generated(CyclicEntity.class, "cycle-v1", + java.util.Map.of("id", 0, "version", 1, "peer", 2), + java.util.Map.of("id", java.util.List.of("id", "id"), "version", java.util.List.of("version", "version"), + "peer", java.util.List.of("peer", "peer")), java.util.Set.of())); + CyclicEntity peer; + @Override public Object __internalGet(String field) { return "peer".equals(field) ? peer : super.__internalGet(field); } + @Override public void __internalSet(String field, Object value) { + if ("peer".equals(field)) { markPropertyLoaded(field); peer = (CyclicEntity) value; } + else super.__internalSet(field, value); + } + } + + @Test + public void cyclicIndexedGraphsUseIdentityReferencesAndDoNotRetainAPathAcrossWrites() throws Exception { + var first = new CyclicEntity(); first.updateId(1L); first.updateVersion(1L); + var second = new CyclicEntity(); second.updateId(2L); second.updateVersion(1L); + first.__internalSet("peer", second); second.__internalSet("peer", first); + var mapper = new ObjectMapper().registerModule(TeaQLModule.INSTANCE); + var json = mapper.readTree(mapper.writeValueAsString(first)); + assertEquals(2L, json.get("peer").get("id").asLong()); + assertEquals(1L, json.get("peer").get("peer").get("id").asLong()); + assertFalse(json.get("peer").get("peer").has("peer")); + assertEquals(json, mapper.readTree(mapper.writeValueAsString(first))); + } + + @Test + public void sharedReferencesOutsideTheActivePathAreNotReducedToIdentityOnly() throws Exception { + var first = new CyclicEntity(); first.updateId(1L); first.updateVersion(1L); + var second = new CyclicEntity(); second.updateId(2L); second.updateVersion(1L); + first.__internalSet("peer", second); + second.putAdditional("_label", "shared child"); + first.putAdditional("_again", second); + var mapper = new ObjectMapper().registerModule(TeaQLModule.INSTANCE); + var json = mapper.readTree(mapper.writeValueAsString(java.util.List.of(first, first))); + for (var row : json) { + assertEquals("shared child", row.get("peer").get("_label").asText()); + assertEquals(row.get("peer"), row.get("_again")); + } + assertEquals(json.get(0), json.get(1)); + } + + public static class IndexedEntity extends BaseEntity { + static final io.teaql.core.FieldLayout LAYOUT = io.teaql.core.FieldLayout.installGenerated( + io.teaql.core.FieldLayout.generated(IndexedEntity.class, "serialization-v1", + java.util.Map.of("id", 0, "version", 1, "name", 2, "base_url", 3), + java.util.Map.of("id", java.util.List.of("id", "id"), "version", java.util.List.of("version", "version"), + "name", java.util.List.of("name", "name"), "base_url", java.util.List.of("baseUrl", "legacy_url")), java.util.Set.of())); + private String name; + private String baseUrl; + @Override public void __internalSet(String field, Object value) { + markPropertyLoaded(field); + switch (field) { + case "name" -> name = (String) value; + case "baseUrl" -> baseUrl = (String) value; + default -> super.__internalSet(field, value); + } + } + @Override public Object __internalGet(String field) { + return switch (field) { case "name" -> name; case "baseUrl" -> baseUrl; default -> super.__internalGet(field); }; + } + } + + @Test + public void indexedFixedDerivedAndPersistentNamespacesSurviveWithoutBeanReflection() throws Exception { + var entity = new IndexedEntity(); + entity.__internalSet("id", 1L); entity.__internalSet("version", 1L); + entity.__internalSet("name", "native"); + entity.addDynamicProperty("name", "derived"); + entity.addDynamicProperty("nil", null); + entity.setDynamicFieldValues(io.teaql.data.dynamic.DynamicFieldValues.of(java.util.List.of( + io.teaql.data.dynamic.DynamicFieldValue.ofString("name", "persistent"), + io.teaql.data.dynamic.DynamicFieldValue.ofNull("nil", io.teaql.data.dynamic.DynamicDataType.STRING), + io.teaql.data.dynamic.DynamicFieldValue.notLoaded("missing", io.teaql.data.dynamic.DynamicDataType.STRING)))); + entity.putAdditional("name", "must-not-shadow-native"); + entity.putAdditional("id", 999L); + var mapper = new ObjectMapper().registerModule(TeaQLModule.INSTANCE); + var json = mapper.readTree(mapper.writeValueAsString(entity)); + assertEquals("native", json.get("name").asText()); + assertEquals("derived", json.get("_name").asText()); + assertEquals("persistent", json.get("#name").asText()); + assertEquals(1L, json.get("id").asLong()); + assertTrue(json.get("_nil").isNull()); assertTrue(json.get("#nil").isNull()); + assertFalse(json.has("#missing")); assertFalse(json.has("baseUrl")); + assertFalse(json.has("loadState")); assertFalse(json.has("entityMutationLedger")); + entity.__internalSet("baseUrl", null); + json = mapper.readTree(mapper.writeValueAsString(entity)); + assertTrue(json.has("baseUrl")); assertTrue(json.get("baseUrl").isNull()); + assertFalse(json.has("base_url")); assertFalse(json.has("legacy_url")); + } + public static class NamedEntity extends BaseEntity { private String name; diff --git a/teaql-provider-jdbc/src/main/java/io/teaql/provider/jdbc/JdbcSqlExecutor.java b/teaql-provider-jdbc/src/main/java/io/teaql/provider/jdbc/JdbcSqlExecutor.java index 97b3ac3b..c7949342 100644 --- a/teaql-provider-jdbc/src/main/java/io/teaql/provider/jdbc/JdbcSqlExecutor.java +++ b/teaql-provider-jdbc/src/main/java/io/teaql/provider/jdbc/JdbcSqlExecutor.java @@ -18,6 +18,8 @@ public class JdbcSqlExecutor implements SqlExecutionAdapter { private final DataSource dataSource; + // A held view retains only this token, not the executor, connection or DataSource. + private final Object storageIdentity = new Object(); private final ThreadLocal transactionConnection = new ThreadLocal<>(); private final List> connectionInitializers = new CopyOnWriteArrayList<>(); @@ -25,6 +27,9 @@ public JdbcSqlExecutor(DataSource dataSource) { this.dataSource = dataSource; } + @Override + public Object storageIdentity() { return storageIdentity; } + public JdbcSqlExecutor addConnectionInitializer(Consumer initializer) { connectionInitializers.add(initializer); return this; @@ -54,6 +59,31 @@ public Stream queryForStream(String sql, Map params, SqlR @Override public Stream> queryForStream(String sql, Object[] params) { + return streamQuery(sql, params, rows -> { + String[] columnLabels = columnLabels(rows); + return () -> { + Map row = new java.util.HashMap<>(); + for (int i = 0; i < columnLabels.length; i++) row.put(columnLabels[i], rows.getObject(i + 1)); + return row; + }; + }); + } + + @Override public boolean supportsCompiledStreamMapping() { return true; } + + @Override + public Stream queryForStream( + String sql, Object[] params, io.teaql.core.CompiledRowMapper mapper) { + return streamQuery(sql, params, rows -> { + io.teaql.core.DataRow row = new JdbcDataRow(rows); + return () -> mapper.map(row); + }); + } + + @FunctionalInterface private interface RowReader { T read() throws SQLException; } + @FunctionalInterface private interface RowReaderFactory { RowReader create(ResultSet rows) throws SQLException; } + + private Stream streamQuery(String sql, Object[] params, RowReaderFactory factory) { StreamResources resources = new StreamResources(); try { Connection connection = resources.connection = openConnection(); @@ -65,8 +95,8 @@ public Stream> queryForStream(String sql, Object[] params) { } } ResultSet rs = resources.resultSet = ps.executeQuery(); - String[] columnLabels = columnLabels(rs); - java.util.Iterator> iterator = new java.util.Iterator<>() { + RowReader reader = factory.create(rs); + java.util.Iterator iterator = new java.util.Iterator<>() { private boolean ready; private boolean hasNext; @@ -86,18 +116,17 @@ public boolean hasNext() { } @Override - public Map next() { + public T next() { if (!hasNext()) throw new java.util.NoSuchElementException(); ready = false; try { - Map row = new java.util.HashMap<>(); - for (int i = 0; i < columnLabels.length; i++) { - row.put(columnLabels[i], rs.getObject(i + 1)); - } - return row; + return reader.read(); } catch (SQLException e) { resources.close(); throw new RuntimeException(e); + } catch (RuntimeException | Error failure) { + resources.close(); + throw failure; } } }; @@ -374,6 +403,8 @@ public void executeInTransaction(Runnable action) { } } + @Override public boolean hasActiveTransaction() { return transactionConnection.get() != null; } + static void bind(PreparedStatement statement, int index, Object value) throws SQLException { if (value == null) { // Let the driver infer the target column type. DB2 rejects the diff --git a/teaql-provider-jdbc/src/test/java/io/teaql/provider/jdbc/JdbcSqlExecutorTest.java b/teaql-provider-jdbc/src/test/java/io/teaql/provider/jdbc/JdbcSqlExecutorTest.java index cab8d8a2..09e09f13 100644 --- a/teaql-provider-jdbc/src/test/java/io/teaql/provider/jdbc/JdbcSqlExecutorTest.java +++ b/teaql-provider-jdbc/src/test/java/io/teaql/provider/jdbc/JdbcSqlExecutorTest.java @@ -211,6 +211,57 @@ private void failedStreamClosesConnection(String sql) throws Exception { } } + private static final class TypedRow extends io.teaql.core.BaseEntity { + String name; + Integer age; + } + + @Test + public void compiledStreamIsLazyTypedAndClosesOnExhaustionEarlyCloseAndMapperFailure() throws Exception { + sqlExecutor.update("INSERT INTO test_user (id,name,age) VALUES (?,?,?)", new Object[]{1,"typed",null}); + sqlExecutor.update("INSERT INTO test_user (id,name,age) VALUES (?,?,?)", new Object[]{2,"second",20}); + for (String outcome : List.of("exhaustion", "early", "runtime", "error")) { + var acquired = new ArrayList(); + var tracking = (DataSource) java.lang.reflect.Proxy.newProxyInstance( + DataSource.class.getClassLoader(), new Class[]{DataSource.class}, (proxy, method, args) -> { + try { + Object value = method.invoke(dataSource,args); + if (value instanceof Connection connection) acquired.add(connection); + return value; + } catch (java.lang.reflect.InvocationTargetException error) { throw error.getCause(); } + }); + var adapter = new JdbcSqlExecutor(tracking); + var calls = new java.util.concurrent.atomic.AtomicInteger(); + io.teaql.core.CompiledRowMapper mapper = row -> { + calls.incrementAndGet(); + if (outcome.equals("runtime")) throw new IllegalStateException("mapper failed"); + if (outcome.equals("error")) throw new AssertionError("mapper error"); + TypedRow value = new TypedRow(); + value.__internalSet("id",row.get(1,Long.class)); + value.name=row.get(2,String.class); value.age=row.get(3,Integer.class); + return value; + }; + var stream = adapter.queryForStream("SELECT id,name,age FROM test_user ORDER BY id",new Object[0],mapper); + assertTrue(adapter.supportsCompiledStreamMapping()); + assertEquals(0,calls.get()); assertEquals(1,acquired.size()); + if (outcome.equals("runtime")) org.junit.Assert.assertThrows(IllegalStateException.class,stream::findFirst); + else if (outcome.equals("error")) org.junit.Assert.assertThrows(AssertionError.class,stream::findFirst); + else if (outcome.equals("early")) { + TypedRow first=stream.findFirst().orElseThrow(); + assertEquals(1L,first.getId().longValue()); assertEquals("typed",first.name); + org.junit.Assert.assertNull(first.age); assertEquals(1,calls.get()); + assertTrue(!acquired.get(0).isClosed()); + } else { + var rows=stream.toList(); assertEquals(2,rows.size()); + assertEquals(Integer.valueOf(20),rows.get(1).age); + assertTrue(acquired.get(0).isClosed()); + } + if (outcome.equals("runtime") || outcome.equals("error")) assertTrue(acquired.get(0).isClosed()); + stream.close(); stream.close(); + assertTrue(acquired.get(0).isClosed()); + } + } + private static class SimpleDataSource implements DataSource { private final String url; private final String user; diff --git a/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java b/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java index e855c33b..72bb3295 100644 --- a/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java +++ b/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java @@ -447,13 +447,44 @@ public void saveGraph(UserContext context, Entity entity) { MutationPlan mutationPlan = buildMutationPlan(entity, entityMutationLedger, realEntities, intent); requireSingleMutationRoute(mutationPlan, route); MutationGovernanceSnapshot governance = reviewMutationPlan(context, mutationPlan); + var dynamicChanges = entityMutationLedger.currentChangeSet().dynamicChanges(); + var dynamicRequests = new ArrayList(); + dynamicChanges.forEach((key, mutations) -> { + if (!entityMutationLedger.deletedKeys().contains(key)) mutations.values().forEach(mutation -> dynamicRequests.add( + new io.teaql.data.dynamic.DynamicGraphMutation(io.teaql.data.dynamic.DynamicOwnerRef.of(key.entity(), key.id()), mutation, + realEntities.get(key).dynamicFields().metadata()))); + }); + Map dynamicReadbacks = dynamicRequests.isEmpty() ? Collections.emptyMap() : new HashMap<>(); List completed; try { if (mutationExecutor instanceof TransactionExecutor transactionExecutor) { - completed = transactionExecutor.executeInTransaction(context, () -> - executeLedgerPlan(context, entityMutationLedger, mutationExecutor, - realEntities, governance, intent, traceScopes, graphScope)); + completed = transactionExecutor.executeInTransaction(context, () -> { + io.teaql.data.dynamic.DynamicFieldsFacade dynamic = null; + if (!dynamicRequests.isEmpty()) { + dynamic = context.dynamicFields().comment(intent.comment()).purpose("runtime: persist audited graph extensions"); + dynamic.prepareGraphMutations(transactionExecutor.transactionResource(), dynamicRequests); + } + List results = executeLedgerPlan(context, entityMutationLedger, mutationExecutor, + realEntities, governance, intent, traceScopes, graphScope); + if (dynamic != null) { + dynamic.applyGraphMutations(transactionExecutor.transactionResource(), dynamicRequests); + for (var entry : dynamicChanges.entrySet()) { + if (entityMutationLedger.deletedKeys().contains(entry.getKey())) continue; + BaseEntity target = realEntities.get(entry.getKey()); + if (target == null) throw new TeaQLRuntimeException("Dynamic mutation has no materialized target"); + var selection = new io.teaql.data.dynamic.DynamicFieldSelection(); + Set codes = new TreeSet<>(target.dynamicFields().selectedCodes()); + codes.addAll(entry.getValue().keySet()); + for (String code : codes) selection.select(code, target.dynamicFields().metadata().requireType(code)); + var value = dynamic.owner(entry.getKey().entity(), entry.getKey().id()).readAll(selection); + if (value == null) throw new TeaQLRuntimeException("Dynamic mutation readback omitted owner"); + dynamicReadbacks.put(entry.getKey(), value); + } + } + return results; + }); } else { + if (!dynamicRequests.isEmpty()) throw new TeaQLRuntimeException("DYNAMIC_FIELD_TRANSACTION_BINDING_REQUIRED: native graph executor has no transaction"); completed = executeLedgerPlan( context, entityMutationLedger, mutationExecutor, realEntities, governance, intent, traceScopes, graphScope); } @@ -465,6 +496,10 @@ public void saveGraph(UserContext context, Entity entity) { Collections.newSetFromMap(new IdentityHashMap<>())); throw failure; } + dynamicReadbacks.forEach((key, values) -> { + BaseEntity target = realEntities.get(key); + target.__internalHydrateDynamicFields(values, target.__internalLoadState().withDynamicSelection(values.selectedCodes())); + }); completeLedgerPlan(context, completed); entityMutationLedger.clearCurrentChangeSet(); telemetryScope.success(); @@ -588,6 +623,7 @@ private void recordGraphChanges( for (String property : baseEntity.getUpdatedProperties()) { targetRoot.set(key, property, baseEntity.__internalGet(property)); } + if (!targetRoot.isMarkedAsDelete(key)) baseEntity.__internalDynamicMutations().values().forEach(mutation -> targetRoot.setDynamic(key, mutation)); } visitRelatedEntities( entity, related -> recordGraphChanges(related, targetRoot, visited)); @@ -599,21 +635,23 @@ private void visitRelatedEntities( EntityDescriptor descriptor = metadata.resolveEntityDescriptor(entity.typeName()); if (descriptor == null) return; - for (PropertyDescriptor prop : descriptor.getProperties()) { - if (!(prop instanceof io.teaql.core.meta.Relation)) continue; - Object value = entity.getProperty(prop.getName()); - if (value instanceof Entity relEntity) { - visitor.accept(relEntity); - } else if (value instanceof Collection collection) { - for (Object item : collection) { - if (item instanceof Entity relEntity) { - visitor.accept(relEntity); + for (EntityDescriptor current = descriptor; current != null; current = current.getParent()) { + for (PropertyDescriptor prop : current.getProperties()) { + if (!(prop instanceof io.teaql.core.meta.Relation)) continue; + Object value = entity.getProperty(prop.getName()); + if (value instanceof Entity relEntity) { + visitor.accept(relEntity); + } else if (value instanceof Collection collection) { + for (Object item : collection) { + if (item instanceof Entity relEntity) { + visitor.accept(relEntity); + } } - } - } else if (value instanceof Iterable iterable) { - for (Object item : iterable) { - if (item instanceof Entity relEntity) { - visitor.accept(relEntity); + } else if (value instanceof Iterable iterable) { + for (Object item : iterable) { + if (item instanceof Entity relEntity) { + visitor.accept(relEntity); + } } } } @@ -671,6 +709,7 @@ private MutationPlan buildMutationPlan( Set created = ledger.newKeys(); Set keys = new TreeSet<>(); keys.addAll(changeSet.changes().keySet()); + keys.addAll(changeSet.dynamicChanges().keySet()); keys.addAll(deleted); keys.addAll(ledger.recoveredKeys()); @@ -683,7 +722,7 @@ private MutationPlan buildMutationPlan( kind = MutationOperationKind.DELETE; changes = Map.of(); } else { - changes = changeSet.changes().getOrDefault(key, Map.of()); + changes = changesWithDynamic(changeSet, key); if (created.contains(key) || key.id() == null) { kind = MutationOperationKind.CREATE; } else if (ledger.recoveredKeys().contains(key)) { @@ -786,11 +825,25 @@ private List executeLedgerPlan( var graphRedactions = new SqlIntentRedactions(); realEntities.values().forEach(value -> graphRedactions.captureEntity( value, metadata.resolveEntityDescriptor(value.typeName()))); + realEntities.values().forEach(entity -> { + var descriptor = metadata.resolveEntityDescriptor(entity.typeName()); + if (entity.getDynamicFieldValues() != null) entity.getDynamicFieldValues().toMap().forEach((field, value) -> + graphRedactions.capture(List.of(SqlFieldLogPolicy.resolve(descriptor, "#" + field)), new Object[]{value.value()})); + entity.__internalDynamicOriginalValues().forEach((field, value) -> graphRedactions.capture( + List.of(SqlFieldLogPolicy.resolve(descriptor, "#" + field)), new Object[]{value.value()})); + }); changeSet.changes().forEach((key, values) -> { var descriptor = metadata.resolveEntityDescriptor(key.entity()); values.forEach((field, value) -> graphRedactions.capture( List.of(SqlFieldLogPolicy.resolve(descriptor, field)), new Object[]{value})); }); + // Extension intents are not reflected in the predefined-field descriptor. + // Capture their actual operands, not the mutation's deliberately safe toString. + changeSet.dynamicChanges().forEach((key, values) -> { + var descriptor = metadata.resolveEntityDescriptor(key.entity()); + values.forEach((field, mutation) -> graphRedactions.capture( + List.of(SqlFieldLogPolicy.resolve(descriptor, "#" + field)), new Object[]{mutation.value()})); + }); // 1. Execute Deletes List sortedDeletedKeys = new ArrayList<>(deletedKeys); @@ -833,6 +886,7 @@ private List executeLedgerPlan( Map> updateBatches = new TreeMap<>(); Set changedKeys = new TreeSet<>(changeSet.changes().keySet()); + changedKeys.addAll(changeSet.dynamicChanges().keySet()); changedKeys.addAll(root.recoveredKeys()); for (EntityKey key : changedKeys) { if (deletedKeys.contains(key)) continue; @@ -858,8 +912,7 @@ private List executeLedgerPlan( List> snapshots = new ArrayList<>(); Collections.sort(keys); for (EntityKey key : keys) { - Map changes = changeSet.changes().get(key); - if (changes == null) continue; + Map changes = changeSet.changes().getOrDefault(key, Map.of()); BaseEntity target = realEntities.get(key); BaseEntity entity = mutationEntity(descriptor, target); entity.__internalSet("id", key.id()); @@ -868,7 +921,7 @@ private List executeLedgerPlan( entity.__internalSet("version", version); } for (Map.Entry change : changes.entrySet()) { - entity.updateProperty(change.getKey(), change.getValue()); + entity.__internalApplyRecordedMutation(change.getKey(), change.getValue()); } if (root.getComment() != null) entity.setComment(root.getComment()); @@ -876,7 +929,7 @@ private List executeLedgerPlan( entity, EntityPersistenceMutation.Action.SAVE, intent, mutationTrace(root, key, traceScopes, graphScope), target, graphRedactions); requests.add(mutationRequest); targets.add(target == null ? entity : target); - snapshots.add(snapshotChanges(changes)); + snapshots.add(snapshotChanges(changesWithDynamic(changeSet, key))); } List results = mutateBatchWithTelemetry( context, mutationExecutor, intent, requests, entityName, "save"); @@ -913,8 +966,9 @@ private List executeLedgerPlan( entity.__internalSet("version", version); } for (Map.Entry change : changes.entrySet()) { - entity.updateProperty(change.getKey(), change.getValue()); + entity.__internalApplyRecordedMutation(change.getKey(), change.getValue()); } + if (changeSet.dynamicChanges().containsKey(key)) entity.__internalRequireVersionUpdate(); entity.set$status(recovering ? io.teaql.core.EntityStatus.UPDATED_RECOVER : io.teaql.core.EntityStatus.UPDATED); if (root.getComment() != null) entity.setComment(root.getComment()); @@ -922,7 +976,7 @@ private List executeLedgerPlan( entity, EntityPersistenceMutation.Action.SAVE, intent, mutationTrace(root, key, traceScopes, graphScope), target, graphRedactions); requests.add(mutationRequest); targets.add(target == null ? entity : target); - snapshots.add(snapshotChanges(changes)); + snapshots.add(snapshotChanges(changesWithDynamic(changeSet, key))); } MutationAuditKind auditKind = recovering ? MutationAuditKind.RECOVERED : MutationAuditKind.UPDATED; List results = mutateBatchWithTelemetry( @@ -943,6 +997,8 @@ private void completeLedgerPlan(UserContext context, List compl context, mutation.target(), mutation.auditKind(), mutation.changedValues(), mutation.governance(), mutation.intent(), mutation.traceChain(), mutation.redactions()); mutation.target().clearUpdatedProperties(); + // A committed delete supersedes any staged extension edits too. + mutation.target().__internalClearDynamicMutations(); } } @@ -950,6 +1006,12 @@ private Map snapshotChanges(Map changes) { return Collections.unmodifiableMap(new LinkedHashMap<>(changes)); } + private Map changesWithDynamic(EntityChangeSet changes, EntityKey key) { + Map fields = new LinkedHashMap<>(changes.changes().getOrDefault(key, Map.of())); + changes.dynamicChanges().getOrDefault(key, Map.of()).forEach((code, mutation) -> fields.put("#" + code, mutation)); + return fields; + } + private BaseEntity mutationEntity(EntityDescriptor descriptor, BaseEntity target) { try { return (BaseEntity) descriptor.createEntity(); @@ -998,9 +1060,11 @@ private void applyPersistedEntity( + descriptor.getType()); } BaseEntity persisted = (BaseEntity) result.persistedEntity(); - for (PropertyDescriptor property : descriptor.getProperties()) { - if (!persisted.isPropertyLoaded(property.getName())) continue; - target.__internalSet(property.getName(), persisted.getProperty(property.getName())); + for (EntityDescriptor current = descriptor; current != null; current = current.getParent()) { + for (PropertyDescriptor property : current.getProperties()) { + if (!persisted.isPropertyLoaded(property.getName())) continue; + target.__internalSet(property.getName(), persisted.getProperty(property.getName())); + } } target.set$status(((BaseEntity) persisted).get$status()); target.clearUpdatedProperties(); @@ -1082,7 +1146,19 @@ private void emitAuditEvent( if (changedValues != null) { for (Map.Entry entry : changedValues.entrySet()) { if (entry.getKey() == null || entry.getKey().startsWith("_")) continue; - changes.add(new AuditFieldChange(entry.getKey(), null, entry.getValue())); + Object value = entry.getValue(); + Object previous = null; + if (value instanceof io.teaql.data.dynamic.DynamicFieldMutation mutation) { + if (entity instanceof BaseEntity base) { + var original = base.__internalDynamicOriginalValues().get(mutation.code()); + if (original != null && original.isLoaded()) previous = original.value(); + } + var details = new LinkedHashMap(); + details.put("operation", mutation.kind().name()); + if (mutation.kind() == io.teaql.data.dynamic.DynamicFieldMutation.Kind.SET) details.put("value", mutation.value()); + value = Collections.unmodifiableMap(details); + } + changes.add(new AuditFieldChange(entry.getKey(), previous, value)); } } changes.sort(Comparator.comparing(AuditFieldChange::field)); @@ -1134,7 +1210,7 @@ private SafeAuditEvent buildSafeAuditEvent(RawAuditEvent event, SqlIntentRedacti boolean allowPlaintext = LogPrivacy.plaintextEnabled(); redactions.appendTo(sensitiveValues, allowPlaintext); for (AuditFieldChange change : event.changes()) { - if ((!allowPlaintext && maskFields.contains(change.field())) || LogPrivacy.credential(change.field()) + if ((!allowPlaintext && (maskFields.contains(change.field()) || change.field().startsWith("#"))) || LogPrivacy.credential(change.field()) || LogPrivacy.hasCredentials(change.oldValue()) || LogPrivacy.hasCredentials(change.newValue())) { sensitiveValues.add(change.oldValue()); sensitiveValues.add(change.newValue()); @@ -1143,7 +1219,7 @@ private SafeAuditEvent buildSafeAuditEvent(RawAuditEvent event, SqlIntentRedacti for (AuditFieldChange change : event.changes()) { Object value = change.newValue() != null ? change.newValue() : change.oldValue(); String raw = value == null ? null : String.valueOf(value); - boolean masked = raw != null && ((!allowPlaintext && maskFields.contains(change.field())) + boolean masked = raw != null && ((!allowPlaintext && (maskFields.contains(change.field()) || change.field().startsWith("#"))) || LogPrivacy.credential(change.field()) || LogPrivacy.hasCredentials(change.oldValue()) || LogPrivacy.hasCredentials(change.newValue())); boolean credential = LogPrivacy.credential(change.field()) diff --git a/teaql-sql-portable/src/main/java/io/teaql/core/sql/expression/PropertyParser.java b/teaql-sql-portable/src/main/java/io/teaql/core/sql/expression/PropertyParser.java index ca12e624..9cdbd183 100644 --- a/teaql-sql-portable/src/main/java/io/teaql/core/sql/expression/PropertyParser.java +++ b/teaql-sql-portable/src/main/java/io/teaql/core/sql/expression/PropertyParser.java @@ -25,7 +25,7 @@ public String toSql( SQLColumnResolver sqlColumnResolver) { String propertyName = property.getPropertyName(); SQLColumn propertyColumn = sqlColumnResolver.getPropertyColumn(idTable, propertyName); - if (userContext.getBool("MULTI_TABLE", false)) { + if (userContext.getBool(io.teaql.core.sql.SqlAstCompiler.MULTI_TABLE, false)) { return StrUtil.format("{}.{}", sqlColumnResolver.escapeIdentifier(propertyColumn.getTableName()), sqlColumnResolver.escapeIdentifier(propertyColumn.getColumnName())); } return StrUtil.format("{}", sqlColumnResolver.escapeIdentifier(propertyColumn.getColumnName())); diff --git a/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/IdSpaceIdGenerator.java b/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/IdSpaceIdGenerator.java index 866d91ba..13945201 100644 --- a/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/IdSpaceIdGenerator.java +++ b/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/IdSpaceIdGenerator.java @@ -153,7 +153,17 @@ private Long readCurrentLevel(String typeName) { @Override public Long generateId(io.teaql.core.UserContext context, io.teaql.core.Entity entity) { - return nextId(entity.typeName()); + // Joined inheritance inserts the same ID into ancestor tables, so its + // family must allocate from the root space rather than a child counter. + var descriptor = io.teaql.core.meta.EntityMetaFactory.requireFrom(context) + .resolveEntityDescriptor(entity.typeName()); + var visited = java.util.Collections.newSetFromMap( + new java.util.IdentityHashMap()); + while (descriptor.getParent() != null) { + if (!visited.add(descriptor)) throw new IllegalArgumentException("Cyclic ID-space inheritance metadata"); + descriptor = descriptor.getParent(); + } + return nextId(descriptor.getType()); } /** diff --git a/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLDataService.java b/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLDataService.java index 2228ea3a..a5d21970 100644 --- a/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLDataService.java +++ b/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLDataService.java @@ -93,10 +93,33 @@ public QueryResult query(UserContext context, QueryRequest request) { enhanceRelations(context, (SmartList) result, searchRequest, intent); } attachDynamicAggregations(context, (SmartList) result, searchRequest, intent); + attachDynamicFields(context, result, searchRequest); + result.__internalShareLoadStates(); return new DefaultQueryResult((SmartList) result, null, statements); } + private void attachDynamicFields(UserContext context, SmartList results, SearchRequest request) { + var selection = request.getDynamicFieldSelection(); + if (selection == null || results.isEmpty()) return; + var owners = new java.util.ArrayList(results.size()); + for (Entity entity : results) owners.add(io.teaql.data.dynamic.DynamicOwnerRef.of(entity.typeName(), entity.getId())); + var batch = context.dynamicFields().purpose(request.purpose()).comment(request.comment()).readAll(owners, selection); + // One named-selection set per actual loaded shape, not one copy per row. + var shapes = new java.util.HashMap, io.teaql.core.LoadState>>(); + int index = 0; + for (Entity entity : results) { + var owner = owners.get(index++); + var values = batch.get(owner); + if (values == null) throw new TeaQLRuntimeException("Dynamic field batch omitted owner " + owner); + if (!(entity instanceof io.teaql.core.BaseEntity base)) throw new TeaQLRuntimeException("Dynamic fields require a runtime-owned entity carrier"); + var codes = values.selectedCodes(); + var state = shapes.computeIfAbsent(base.__internalLoadState(), ignored -> new java.util.HashMap<>()) + .computeIfAbsent(codes, base.__internalLoadState()::withDynamicSelection); + base.__internalHydrateDynamicFields(values, state); + } + } + @Override @SuppressWarnings("unchecked") public java.util.stream.Stream queryForStream(UserContext context, QueryRequest request) { @@ -114,8 +137,9 @@ public QueryCursor queryForCursor(UserContext context, Que if (source == null) source = new SqlIntentRedactions(); var statements = new java.util.concurrent.CopyOnWriteArrayList(); SearchRequest scoped = SqlDiagnosticRequest.collecting(searchRequest, source, request.intent(), statements::add); - if (scoped.hasSimpleAgg() || !scoped.enhanceRelations().isEmpty() || !scoped.enhanceChildren().isEmpty()) { - throw new TeaQLRuntimeException("Streaming aggregation/relation enhancement is not supported; stream root rows only"); + if (scoped.hasSimpleAgg() || !scoped.enhanceRelations().isEmpty() || !scoped.enhanceChildren().isEmpty() + || scoped.getDynamicFieldSelection() != null) { + throw new TeaQLRuntimeException("Streaming aggregation/relation/dynamic-field enhancement is not supported; stream root rows only"); } return new QueryCursor<>(this.getRepository(scoped.getTypeName()).streamInternal(context, scoped), () -> statements); @@ -390,6 +414,17 @@ private void collectChildren( userContext.internalExecuteForList(childTempRequest).forEach(children::add); } + // All successful reverse selections have a view, including empty ones. + // Build each resulting shape once, rather than copying its named set + // independently for every parent as setters mark the relation loaded. + Map loadedViews = new HashMap<>(); + for (Entity parent : dataSet) { + if (parent instanceof BaseEntity base) { + var shared = loadedViews.computeIfAbsent(base.__internalLoadState(), + state -> state.withLoaded(relation.getName(), true)); + base.__internalUseLoadState(shared); + } + } Map longTMap = dataSet.mapById(); for (Entity childEntity : children) { Object parent = childEntity.getProperty(reverseProperty.getName()); @@ -400,6 +435,17 @@ private void collectChildren( } } } + // A successful explicit selection must materialize even an empty list. + // Use the shared typed empty payload; availability stays in the parent's + // private view, and no mutation ledger is created by this hydration. + if (relation.getType() != null && SmartList.class.isAssignableFrom(relation.getType().javaType())) { + Class childType = reverseProperty.getOwner().getTargetType(); + for (Entity parent : dataSet) { + if (parent.getProperty(relation.getName()) == null) { + parent.setProperty(relation.getName(), SmartList.empty(childType)); + } + } + } } @SuppressWarnings({"rawtypes", "unchecked"}) @@ -408,7 +454,7 @@ private void attachRelation(Entity target, PropertyDescriptor relation, Entity v Class relationType = relation.getType().javaType(); if (SmartList.class.isAssignableFrom(relationType)) { SmartList existing = target.getProperty(relation.getName()); - if (existing == null) { + if (existing == null || existing.isSharedEmpty()) { existing = new SmartList<>(); target.setProperty(relation.getName(), existing); } @@ -598,21 +644,16 @@ private String batchOperation(EntityPersistenceMutation item) { @SuppressWarnings("unchecked") public T executeInTransaction(UserContext context, TransactionCallback action) { final Object[] resultHolder = new Object[1]; - final Exception[] exceptionHolder = new Exception[1]; database.executeInTransaction(() -> { try { resultHolder[0] = action.doInTransaction(); + } catch (RuntimeException e) { + // Preserve structured failures while the adapter rolls back. + throw e; } catch (Exception e) { - exceptionHolder[0] = e; throw new TeaQLRuntimeException("Transaction failed", e); } }); - if (exceptionHolder[0] != null) { - if (exceptionHolder[0] instanceof RuntimeException) { - throw (RuntimeException) exceptionHolder[0]; - } - throw new TeaQLRuntimeException("Transaction failed", exceptionHolder[0]); - } return (T) resultHolder[0]; } diff --git a/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLRepository.java b/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLRepository.java index ec2e608d..d202815a 100644 --- a/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLRepository.java +++ b/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLRepository.java @@ -788,8 +788,11 @@ private SmartList loadWithIntent(UserContext userContext, SearchRequest re return SmartList.empty(request.returnType()); } smartList = new SmartList<>(rows.size()); + MapProjection[] projections = new MapProjection[16]; + SimpleNamedExpression[] dynamicProperties = executedRequest.getSimpleDynamicProperties().toArray(SimpleNamedExpression[]::new); for (Map row : rows) { - smartList.add(mapRowToEntity(userContext, executedRequest, row)); + MapProjection projection = resolveMapProjection(executedRequest.returnType(), row, projections); + smartList.add(mapRowToEntity(userContext, executedRequest, row, projection, dynamicProperties)); } } registerContinuousPage(userContext, request, pageExecution, smartList.getData()); @@ -919,11 +922,17 @@ private io.teaql.core.CompiledRowMapper compileRowMapper(SearchRequest req property.getName()))); } + io.teaql.core.LoadState loadedShape = io.teaql.core.LoadState.projection( + io.teaql.core.FieldLayout.forType(resultDescriptor.getTargetType()), + selected.stream().map(PropertyDescriptor::getName).toList()); + ColumnBinding[] columns = bindings.toArray(ColumnBinding[]::new); + return row -> { @SuppressWarnings("unchecked") T entity = (T) resultDescriptor.createEntity(); BaseEntity base = (BaseEntity) entity; - for (ColumnBinding binding : bindings) { + base.__internalUseLoadState(loadedShape); + for (ColumnBinding binding : columns) { PropertyDescriptor property = binding.property(); Object value; if (binding.relationDescriptor() == null) { @@ -1089,8 +1098,31 @@ T loadPersistedById(UserContext userContext, Long id, io.teaql.core.SqlIntentRed ? thisPrimaryTableName : tableName(entityDescriptor.getType()); String sql = "SELECT * FROM " + escapeIdentifier(primaryTable) + " WHERE " + escapeIdentifier("id") + " = ?"; - List> rows = database.query(userContext, sql, new Object[] {id}, - new SqlLogBindings(List.of(parameterLogPolicy("id")), true, null, intent, trace)); + Object[] arguments = new Object[]{id}; + var bindings = new SqlLogBindings(List.of(parameterLogPolicy("id")), true, null, intent, trace); + if (types.size() > 1) { + // An authoritative inherited view needs all participating tables, + // including the root version and tombstones. Reuse the normal AST + // compiler rather than copying the proposed values into readback. + var readback = new io.teaql.core.internal.TempRequest(entityDescriptor.getTargetType(), entityDescriptor.getType()) { + { withDeletedRows(); } + }; + for (PropertyDescriptor property : allProperties) { + if (shouldHandle(property)) readback.selectProperty(property.getName()); + } + readback.appendSearchCriteria(new io.teaql.core.criteria.EQ( + new io.teaql.core.PropertyReference("id"), new io.teaql.core.Parameter("readbackId", id, io.teaql.core.criteria.Operator.EQUAL))); + // Explicit version intent prevents the compiler's normal visible-row + // default from hiding the negative tombstone we just committed. + readback.appendSearchCriteria(readback.createBasicSearchCriteria("version", io.teaql.core.criteria.Operator.IS_NOT_NULL)); + readback.setSize(1); + Map parameters = new io.teaql.core.sql.SqlParameters(); + var statement = toPositional(buildDataSQL(userContext, readback, parameters), parameters); + sql = statement.sql; + arguments = statement.args; + bindings = new SqlLogBindings(statement.logBindings.policies(), true, null, intent, trace); + } + List> rows = database.query(userContext, sql, arguments, bindings); if (rows.size() != 1) { throw new TeaQLRuntimeException( "Persisted " + entityDescriptor.getType() + "(" + id + ") could not be read back"); @@ -1131,18 +1163,76 @@ public Stream streamInternal(UserContext userContext, SearchRequest reques if (ObjectUtil.isEmpty(sql)) return Stream.empty(); PositionalSQL psql = withQueryIntent(toPositional(sql, params), intent, request); + if (database.supportsCompiledStreamMapping()) { + Object extension = request.getExtension(COMPILED_ROW_MAPPER); + @SuppressWarnings("unchecked") + io.teaql.core.CompiledRowMapper mapper = extension instanceof io.teaql.core.CompiledRowMapper supplied + ? (io.teaql.core.CompiledRowMapper) supplied : compileRowMapper(request); + if (mapper != null) { + return database.queryForStream(userContext, psql.sql, psql.args, mapper, psql.logBindings); + } + } + MapProjection[] projections = new MapProjection[16]; + SimpleNamedExpression[] dynamicProperties = request.getSimpleDynamicProperties().toArray(SimpleNamedExpression[]::new); return database.queryForStream(userContext, psql.sql, psql.args, psql.logBindings) - .map(row -> mapRowToEntity(userContext, request, row)); + .map(row -> { + MapProjection shape = resolveMapProjection(request.returnType(), row, projections); + return mapRowToEntity(userContext, request, row, shape, dynamicProperties); + }); + } + + private record MapBinding(PropertyDescriptor property, String column) {} + + private record MapProjection(String[] columns, io.teaql.core.LoadState loadedState, MapBinding[] bindings) { + boolean matches(Map row) { + if (row.size() != columns.length) return false; + for (String column : columns) if (!row.containsKey(column)) return false; + return true; + } + } + + private MapProjection resolveMapProjection(Class type, Map row, + MapProjection[] projections) { + int empty = -1; + for (int i = 0; i < projections.length; i++) { + MapProjection shape = projections[i]; + if (shape != null && shape.matches(row)) return shape; + if (shape == null && empty < 0) empty = i; + } + MapProjection shape = mapProjection(type, row); + // Query-owned, bounded, value-free hints. Concurrent stream writes may + // replace a hint, but each row keeps its own validated local shape. + projections[empty < 0 ? projections.length - 1 : empty] = shape; + return shape; } - private T mapRowToEntity(UserContext userContext, SearchRequest request, Map row) { + private MapProjection mapProjection(Class type, Map row) { + io.teaql.core.LoadState loaded = null; + List selected = new ArrayList<>(); + List bindings = new ArrayList<>(); + for (PropertyDescriptor property : allProperties) { + if (!shouldHandle(property)) continue; + String column = findColumnKey(row, property.getName()); + if (column != null) { + selected.add(property.getName()); + bindings.add(new MapBinding(property, column)); + } + } + if (BaseEntity.class.isAssignableFrom(type)) { + loaded = io.teaql.core.LoadState.projection(io.teaql.core.FieldLayout.forType(type), selected); + } + return new MapProjection(row.keySet().toArray(String[]::new), loaded, bindings.toArray(MapBinding[]::new)); + } + + private T mapRowToEntity(UserContext userContext, SearchRequest request, Map row, + MapProjection projection, SimpleNamedExpression[] dynamicProperties) { Class returnType = request.returnType(); T entity = createEntity(returnType); - for (PropertyDescriptor property : this.allProperties) { - if (!shouldHandle(property)) continue; + if (entity instanceof BaseEntity base && projection.loadedState() != null) base.__internalUseLoadState(projection.loadedState()); + for (MapBinding binding : projection.bindings()) { + PropertyDescriptor property = binding.property(); + String columnKey = binding.column(); if (!(property instanceof Relation)) { - String columnKey = findColumnKey(row, property.getName()); - if (columnKey == null) continue; Object value = row.get(columnKey); Class targetType = property.getType().javaType(); entity.setProperty( @@ -1151,8 +1241,6 @@ private T mapRowToEntity(UserContext userContext, SearchRequest request, Map< ? null : convertColumnValue(targetType, value)); } else if (property instanceof Relation) { - String columnKey = findColumnKey(row, property.getName()); - if (columnKey == null) continue; Object value = row.get(columnKey); if (value == null) { entity.setProperty(property.getName(), null); @@ -1184,10 +1272,9 @@ private T mapRowToEntity(UserContext userContext, SearchRequest request, Map< be.set$status(status); } // Dynamic properties - List simpleDynamicProperties = request.getSimpleDynamicProperties(); - for (SimpleNamedExpression dp : simpleDynamicProperties) { + for (SimpleNamedExpression dp : dynamicProperties) { Object value = row.get(dp.name()); - if (value != null) entity.addDynamicProperty(dp.name(), value); + if (row.containsKey(dp.name())) entity.addDynamicProperty(dp.name(), value); } return entity; @@ -1202,6 +1289,10 @@ private String findColumnKey(Map row, String propertyName) { } static Object convertTemporalColumnValue(Class targetType, Object value) { + if ((targetType == java.time.LocalDateTime.class || targetType == java.time.LocalDate.class + || targetType == java.time.LocalTime.class) && targetType.isInstance(value)) { + return value; + } if (targetType == java.time.LocalDateTime.class && value instanceof java.sql.Timestamp timestamp) { return timestamp.toLocalDateTime(); @@ -1468,7 +1559,10 @@ void updateInternal(UserContext userContext, Collection updateItems, io.teaql if (ObjectUtil.isEmpty(sqlEntities)) return; for (SQLEntity sqlEntity : sqlEntities) { - if (sqlEntity.isEmpty()) continue; + // A version-only graph participant (for example a dynamic-field + // edit) still needs the optimistic guard below, even without native + // column assignments. Null conversions, not empty column maps, + // represent entities with no update intent. Map> tableColumnNames = sqlEntity.getTableColumnNames(); Map tableColumnValues = sqlEntity.getTableColumnValues(); @@ -2033,7 +2127,13 @@ private boolean shouldHandle(PropertyDescriptor pProperty) { } public boolean shouldHandle(Relation relation) { - return relation.getRelationKeeper() == this.entityDescriptor; + // Only the declaring side stores the FK. A subclass also persists its + // ancestor's declaring fields; the reverse side must never become a column. + if (relation.getRelationKeeper() != relation.getOwner()) return false; + for (EntityDescriptor owner = this.entityDescriptor; owner != null; owner = owner.getParent()) { + if (relation.getOwner() == owner) return true; + } + return false; } private void initSQLMeta(EntityDescriptor entityDescriptor) { diff --git a/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/TeaQLDatabase.java b/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/TeaQLDatabase.java index 33f36914..d4f510fa 100644 --- a/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/TeaQLDatabase.java +++ b/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/TeaQLDatabase.java @@ -37,6 +37,14 @@ default boolean supportsCompiledRowMapping() { return false; } + default boolean supportsCompiledStreamMapping() { return false; } + + default Stream queryForStream( + io.teaql.core.UserContext context, String sql, Object[] args, + io.teaql.core.CompiledRowMapper mapper, SqlLogBindings bindings) { + throw new UnsupportedOperationException("compiled streaming row mapping is not supported"); + } + /** * Execute a query and return a list of rows. Each row is a Map (column name -> value). */ diff --git a/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/IdSpaceIdGeneratorTest.java b/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/IdSpaceIdGeneratorTest.java index cda450fa..1b23df49 100644 --- a/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/IdSpaceIdGeneratorTest.java +++ b/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/IdSpaceIdGeneratorTest.java @@ -13,6 +13,41 @@ import static org.junit.Assert.assertTrue; public class IdSpaceIdGeneratorTest { + @Test + public void parentChildrenAndSiblingShareTheContextOwnedRootIdSpace() { + var database = new RecordingDatabase(); + database.levels.put("Parent", 5L); + var generator = new IdSpaceIdGenerator(database); + var factory = new io.teaql.core.meta.SimpleEntityMetaFactory(); + var parent = new io.teaql.core.meta.EntityDescriptor(); parent.setType("Parent"); + var child = new io.teaql.core.meta.EntityDescriptor(); child.setType("Child"); child.setParent(parent); + var sibling = new io.teaql.core.meta.EntityDescriptor(); sibling.setType("Sibling"); sibling.setParent(parent); + factory.register(parent); factory.register(child); factory.register(sibling); + var context = new io.teaql.runtime.DefaultUserContext(io.teaql.runtime.TeaQLRuntime.builder().metadata(factory).build()); + assertEquals(Long.valueOf(6), generator.generateId(context, new NamedEntity("Child"))); + assertEquals(Long.valueOf(7), generator.generateId(context, new NamedEntity("Parent"))); + assertEquals(Long.valueOf(8), generator.generateId(context, new NamedEntity("Sibling"))); + assertEquals(Map.of("Parent", 8L), database.levels); + } + + @Test + public void cyclicAllocationMetadataRejectsBeforeDatabaseAccess() { + var database = new RecordingDatabase(); + var parent = new io.teaql.core.meta.EntityDescriptor(); parent.setType("Parent"); + var child = new io.teaql.core.meta.EntityDescriptor(); child.setType("Child"); + parent.setParent(child); child.setParent(parent); + var factory = new io.teaql.core.meta.SimpleEntityMetaFactory(); factory.register(parent); factory.register(child); + var context = new io.teaql.runtime.DefaultUserContext(io.teaql.runtime.TeaQLRuntime.builder().metadata(factory).build()); + assertThrows(IllegalArgumentException.class, () -> new IdSpaceIdGenerator(database).generateId(context, new NamedEntity("Child"))); + assertTrue(database.queries.isEmpty()); + } + + private static final class NamedEntity extends io.teaql.core.BaseEntity { + private final String type; + NamedEntity(String type) { this.type = type; } + @Override public String typeName() { return type; } + } + @Test public void incrementsExistingRowWithOptimisticCompareAndSet() { diff --git a/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/InheritedRelationOwnershipTest.java b/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/InheritedRelationOwnershipTest.java new file mode 100644 index 00000000..0c756120 --- /dev/null +++ b/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/InheritedRelationOwnershipTest.java @@ -0,0 +1,43 @@ +package io.teaql.core.sql.portable; + +import io.teaql.core.BaseEntity; +import io.teaql.core.meta.EntityDescriptor; +import io.teaql.core.meta.Relation; +import io.teaql.core.sql.GenericSQLProperty; +import io.teaql.core.sql.SQLEntityDescriptor; +import org.junit.Test; +import static org.junit.Assert.*; + +public class InheritedRelationOwnershipTest { + private SQLEntityDescriptor descriptor(String name) { + var descriptor = new SQLEntityDescriptor(); + descriptor.setType(name); + descriptor.with("table_name", name.toLowerCase() + "_data"); + for (String field : new String[]{"id", "version"}) { + var property = (GenericSQLProperty) descriptor.addSimpleProperty(field, Long.class); + property.setColumnType("BIGINT"); + } + return descriptor; + } + + private Relation relation(EntityDescriptor owner, EntityDescriptor keeper) { + var relation = new Relation(); + relation.setOwner(owner); + relation.setRelationKeeper(keeper); + return relation; + } + + @Test + public void inheritedForwardFkIsHandledButAncestorReverseAndUnrelatedRelationsAreNot() { + var parent = descriptor("Parent"); + var child = descriptor("Child"); + child.setParent(parent); + var other = descriptor("Other"); + var repository = new PortableSQLRepository(child, null, null); + assertTrue(repository.shouldHandle(relation(child, child))); + assertTrue(repository.shouldHandle(relation(parent, parent))); + assertFalse(repository.shouldHandle(relation(parent, child))); + assertFalse(repository.shouldHandle(relation(child, parent))); + assertFalse(repository.shouldHandle(relation(other, other))); + } +} diff --git a/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/PortableSQLDatabaseTest.java b/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/PortableSQLDatabaseTest.java index 97300aaf..7bdcc2ef 100644 --- a/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/PortableSQLDatabaseTest.java +++ b/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/PortableSQLDatabaseTest.java @@ -437,9 +437,15 @@ private Map> loadTopNFixture(Integer threshold) { .purpose("verify governed relation plan equivalence") .executeForList(context); Map> result = new LinkedHashMap<>(); + io.teaql.core.LoadState shared = rows.isEmpty() ? null : rows.get(0).__internalLoadState(); for (TopNParent row : rows) { SmartList children = row.getProperty("children"); - result.put(row.getId(), children == null ? List.of() : children.toList(Entity::getId)); + assertNotNull("selected empty children are a real loaded SmartList", children); + assertTrue("selected relation marker must include empty parents", row.isPropertyLoaded("children")); + assertFalse("read hydration must not allocate a mutation ledger", row.__internalHasMutationLedger()); + assertSame("loaded and empty reverse parents share one actual shape", shared, row.__internalLoadState()); + if (children.isEmpty()) assertSame(SmartList.empty(TopNChild.class), children); + result.put(row.getId(), children.toList(Entity::getId)); } return result; } diff --git a/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/PortableSQLTemporalConversionTest.java b/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/PortableSQLTemporalConversionTest.java index 6c0cc5ad..ad5cf7e4 100644 --- a/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/PortableSQLTemporalConversionTest.java +++ b/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/PortableSQLTemporalConversionTest.java @@ -8,6 +8,16 @@ import org.junit.Test; public class PortableSQLTemporalConversionTest { + @org.junit.Test + public void nativeImmutableTemporalValuesAreNotFormattedAndParsedAgain() { + var date = java.time.LocalDate.of(2026, 10, 7); + var time = java.time.LocalTime.of(12, 34, 56, 123456789); + var timestamp = java.time.LocalDateTime.of(date, time); + org.junit.Assert.assertSame(date, PortableSQLRepository.convertTemporalColumnValue(java.time.LocalDate.class, date)); + org.junit.Assert.assertSame(time, PortableSQLRepository.convertTemporalColumnValue(java.time.LocalTime.class, time)); + org.junit.Assert.assertSame(timestamp, PortableSQLRepository.convertTemporalColumnValue(java.time.LocalDateTime.class, timestamp)); + org.junit.Assert.assertEquals(123456789, timestamp.getNano()); + } @Test public void convertsJdbcStyleTemporalStrings() { String timestamp = "2026-08-12 05:25:00.0"; diff --git a/teaql-sqlite/pom.xml b/teaql-sqlite/pom.xml index f826e91d..9ad6749c 100644 --- a/teaql-sqlite/pom.xml +++ b/teaql-sqlite/pom.xml @@ -34,6 +34,12 @@ ${project.version} + + io.teaql + teaql-dynamic-fields-jdbc + ${project.version} + test + io.teaql teaql-query-json @@ -64,6 +70,7 @@ --add-reads io.teaql.sqlite=io.teaql.query.json --add-reads io.teaql.sqlite=com.fasterxml.jackson.databind --add-opens io.teaql.sqlite/io.teaql.sqlite=io.teaql.utils + --add-opens io.teaql.sqlite/io.teaql.sqlite=io.teaql.core --add-opens io.teaql.core/io.teaql.core=io.teaql.utils diff --git a/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java b/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java index f58040ef..74bdefcd 100644 --- a/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java +++ b/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java @@ -224,7 +224,13 @@ private static CheckedSave checkedSave(Fixture fixture, GraphEntity entity, Stri try { return checkedSave(fixture, valid, "save valid overlapping graph"); } finally { validFinished.countDown(); } }); - var second = workers.submit(() -> checkedSave(fixture, invalid, "reject invalid overlapping graph")); + var second = workers.submit(() -> { + // The runtime captures its fix clock before entering the Checker. + // Order the operation starts, not just the Checker bodies, so + // the date assignment below is deterministic while saves overlap. + await(validEntered); + return checkedSave(fixture, invalid, "reject invalid overlapping graph"); + }); var accepted = first.get(20, TimeUnit.SECONDS); var rejected = second.get(20, TimeUnit.SECONDS); assertNull("another graph's required-field failure must not reject this valid graph: " + accepted.failure(), accepted.failure()); diff --git a/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteIntegrationTest.java b/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteIntegrationTest.java index 88805dd5..6517edd5 100644 --- a/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteIntegrationTest.java +++ b/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteIntegrationTest.java @@ -10,8 +10,14 @@ import io.teaql.provider.jdbc.JdbcSqlExecutor; import io.teaql.dataservice.sql.SqlDataServiceExecutor; import io.teaql.runtime.DefaultUserContext; +import io.teaql.runtime.DefaultQueryRequest; import io.teaql.runtime.RuntimeLogSink; import io.teaql.runtime.TeaQLRuntime; +import io.teaql.runtime.RawAuditEvent; +import io.teaql.runtime.SafeAuditEvent; +import io.teaql.runtime.AppAuditEventSink; +import io.teaql.data.dynamic.*; +import io.teaql.data.dynamic.jdbc.JdbcDynamicFieldsProvider; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -36,6 +42,493 @@ public class SqliteIntegrationTest { + private static final AtomicLong dynamicDefinitionIds = new AtomicLong(100_000); + private static final DynamicFieldContext definitionContext = new DynamicFieldContext() { + public String scopeType() { return "GLOBAL"; } + public String scopeId() { return "default"; } + public String userId() { return "sqlite-regression"; } + public String purpose() { return "register isolated regression definitions"; } + public String comment() { return "what: prepare dynamic field metadata"; } + public boolean strictIntent() { return true; } + public long nextId(String type) { return dynamicDefinitionIds.getAndIncrement(); } + }; + + private record DynamicFixture(DefaultUserContext context, List audits, + List appAudits, List logs) {} + + private DynamicFixture dynamicFixture(JdbcDynamicFieldsProvider provider) { + provider.ensureSchema(); + var audits = new ArrayList(); + var appAudits = new ArrayList(); + var logs = new ArrayList(); + var local = new DefaultUserContext(TeaQLRuntime.builder().metadata(runtime.getMetadata()) + .dataService("sqlite", runtime.getRegistry().resolve("sqlite")) + .idGenerationService(runtime.getIdGenerationService()).logSink(new RuntimeLogSink() { + @Override public void writeExecutionLog(UserContext caller, ExecutionMetadata value) { logs.add(value); } + @Override public void writeAuditEvent(UserContext caller, RawAuditEvent value) { audits.add(value); } + }).build()); + local.putAttribute(DynamicFieldsFacade.class.getName(), new DefaultDynamicFieldsFacade(provider)); + local.putAttribute(AppAuditEventSink.class.getName(), (AppAuditEventSink) (caller, value) -> appAudits.add(value)); + return new DynamicFixture(local, audits, appAudits, logs); + } + + private DynamicFieldDef define(JdbcDynamicFieldsProvider provider, String code, DynamicDataType type) { + return define(provider, code, type, definitionContext); + } + + private DynamicFieldDef define(JdbcDynamicFieldsProvider provider, String code, DynamicDataType type, DynamicFieldContext context) { + var field = new DynamicFieldDef(); + field.setScope(DynamicFieldScope.of(context.scopeType(), context.scopeId())); + field.setOwnerType("Task"); + field.setCode(code); + field.setName(code); + field.setDataType(type); + return provider.registerFieldDef(context, field); + } + + private DynamicFieldContext alternateScopeContext() { + return new DynamicFieldContext() { + public String scopeType() { return "PROFILE"; } + public String scopeId() { return "other"; } + public String userId() { return definitionContext.userId(); } + public String purpose() { return definitionContext.purpose(); } + public String comment() { return definitionContext.comment(); } + public boolean strictIntent() { return true; } + public long nextId(String type) { return definitionContext.nextId(type); } + }; + } + + @Test + public void loadedDynamicViewCannotFollowAChangedStorageScope() { + var provider = new JdbcDynamicFieldsProvider(jdbcSqlExecutor); + var fixture = dynamicFixture(provider); + define(provider, "profile_note", DynamicDataType.STRING); + Task task = dynamicTask(fixture, "PROFILE-ORIGIN", new DynamicFieldSelection().selectString("profile_note")); + var alternate = alternateScopeContext(); + define(provider, "profile_note", DynamicDataType.STRING, alternate); + provider.saveValue(alternate, DynamicSetCommand.of(DynamicOwnerRef.of("Task", task.getId()), + "profile_note", DynamicDataType.STRING, "other profile value", alternate.purpose(), alternate.comment())); + task.updateTitle("must-not-cross-provider"); + task.updateDynamicField("profile_note", "private held value"); + fixture.context().putAttribute(DynamicFieldsFacade.class.getName(), + new DefaultDynamicFieldsFacade(provider, DynamicFieldScope.of("PROFILE", "other"))); + try { + task.auditAs("reject changed provider provenance").save(fixture.context()); + fail("held metadata was accepted by a different storage scope"); + } catch (DynamicFieldException expected) { + assertEquals("DYNAMIC_FIELD_STORAGE_PROVENANCE_MISMATCH", expected.errorCode()); + } + assertEquals("initial", nativeColumn(task, "title")); + assertEquals(Long.valueOf(1), task.getVersion()); + assertEquals(1, dynamicRowCount(task, "profile_note")); + assertEquals("other profile value", provider.loadValues(alternate, DynamicOwnerRef.of("Task", task.getId()), + new DynamicFieldSelection().selectString("profile_note")).getString("profile_note")); + assertFalse(task.__internalDynamicMutations().isEmpty()); + assertEquals(1, fixture.audits().size()); + fixture.context().putAttribute(DynamicFieldsFacade.class.getName(), new DefaultDynamicFieldsFacade(provider)); + task.auditAs("retry with original provider").save(fixture.context()); + assertEquals("must-not-cross-provider", nativeColumn(task, "title")); + assertEquals("private held value", task.dynamicFields().getString("profile_note")); + assertEquals("other profile value", provider.loadValues(alternate, DynamicOwnerRef.of("Task", task.getId()), + new DynamicFieldSelection().selectString("profile_note")).getString("profile_note")); + assertTrue(task.__internalDynamicMutations().isEmpty()); + } + + @Test + public void replacingProviderOverTheSameExecutorPreservesStorageProvenance() { + var provider = new JdbcDynamicFieldsProvider(jdbcSqlExecutor); + var fixture = dynamicFixture(provider); + define(provider, "same_source_note", DynamicDataType.STRING); + Task task = dynamicTask(fixture, "PROFILE-SAME", new DynamicFieldSelection().selectString("same_source_note")); + task.updateDynamicField("same_source_note", "same storage"); + fixture.context().putAttribute(DynamicFieldsFacade.class.getName(), + new DefaultDynamicFieldsFacade(new JdbcDynamicFieldsProvider(jdbcSqlExecutor))); + task.auditAs("keep same executor provenance").save(fixture.context()); + assertEquals("same storage", task.dynamicFields().getString("same_source_note")); + assertEquals(Long.valueOf(2), task.getVersion()); + assertEquals(2, fixture.audits().size()); + } + + private Task dynamicTask(DynamicFixture fixture, String status, DynamicFieldSelection selection) { + var task = new Task().updateTitle("initial").updateStatus(status); + task.auditAs("seed atomic dynamic regression").save(fixture.context()); + return loadDynamicTask(fixture, status, selection); + } + + private Task loadDynamicTask(DynamicFixture fixture, String status, DynamicFieldSelection selection) { + var request = new TaskRequest().filterByStatus(status); + request.setSize(1); + request.selectDynamicFieldsWith(selection); + return request.comment("load fixed and dynamic regression fields").purpose("verify graph mutation isolation") + .executeForList(fixture.context()).get(0); + } + + private Object nativeColumn(Task task, String column) { + return jdbcSqlExecutor.queryForList("SELECT " + column + " FROM task_data WHERE id=?", new Object[]{task.getId()}) + .get(0).get(column); + } + + private long dynamicRowCount(Task task, String code) { + return ((Number) jdbcSqlExecutor.queryForList("SELECT COUNT(*) AS n FROM teaql_dynamic_field_value v " + + "JOIN teaql_dynamic_field_def d ON d.id=v.field_id WHERE v.owner_type='Task' AND v.owner_id=? AND d.code=?", + new Object[]{task.getId(), code}).get(0).get("n")).longValue(); + } + + @Test + public void dynamicGraphSavePersistsNullAndDeleteDistinctlyAndAuditsAfterCommit() { + var provider = new JdbcDynamicFieldsProvider(jdbcSqlExecutor); + var fixture = dynamicFixture(provider); + define(provider, "atomic_note", DynamicDataType.STRING); + define(provider, "atomic_untouched", DynamicDataType.STRING); + var selection = new DynamicFieldSelection().selectString("atomic_note"); + Task task = dynamicTask(fixture, "DYNAMIC-ATOMIC", selection); + assertEquals(DynamicFieldValue.State.NOT_LOADED, task.dynamicFields().field("atomic_note").state()); + fixture.context().dynamicFields().comment("seed unselected extension").purpose("verify save does not clear it") + .owner("Task", task.getId()).string("atomic_untouched").set("preserved"); + LoadState before = task.__internalLoadState(); + task.updateTitle("updated"); + task.updateDynamicField("atomic_note", "PRIVATE-DYNAMIC-CANARY"); + assertNotSame(before, task.__internalLoadState()); + task.addDynamicProperty("atomic_note", "readonly-derived"); + task.auditAs("save extension PRIVATE-DYNAMIC-CANARY").save(fixture.context()); + assertEquals("updated", nativeColumn(task, "title")); + assertEquals(Long.valueOf(2), task.getVersion()); + assertEquals("PRIVATE-DYNAMIC-CANARY", task.dynamicFields().getString("atomic_note")); + assertTrue(task.__internalDynamicMutations().isEmpty()); + assertTrue(task.getEntityMutationLedger().currentChangeSet().isEmpty()); + assertEquals("readonly-derived", task.getProperty("_atomic_note")); + assertEquals("preserved", fixture.context().dynamicFields().comment("read untouched extension").purpose("verify isolation") + .owner("Task", task.getId()).string("atomic_untouched").get()); + assertEquals(2, fixture.audits().size()); + var audit = fixture.audits().get(1); + assertTrue(audit.changes().stream().anyMatch(change -> change.field().equals("#atomic_note"))); + assertFalse(audit.toString().contains("PRIVATE-DYNAMIC-CANARY")); + assertFalse(fixture.appAudits().toString().contains("PRIVATE-DYNAMIC-CANARY")); + assertTrue(fixture.appAudits().get(1).fields().stream().anyMatch(field -> field.name().equals("#atomic_note") && field.masked())); + assertTrue(fixture.logs().stream().noneMatch(log -> String.valueOf(log.getDebugQuery()).contains("PRIVATE-DYNAMIC-CANARY"))); + + LoadState loaded = task.__internalLoadState(); + task.updateDynamicField("atomic_note", null); + assertSame("value to NULL preserves loaded geometry", loaded, task.__internalLoadState()); + task.auditAs("persist explicit dynamic null after PRIVATE-DYNAMIC-CANARY").save(fixture.context()); + assertEquals(Long.valueOf(3), task.getVersion()); + assertEquals(1, dynamicRowCount(task, "atomic_note")); + assertEquals(DynamicFieldValue.State.NULL, task.dynamicFields().field("atomic_note").state()); + assertEquals(DynamicFieldValue.State.NULL, loadDynamicTask(fixture, "DYNAMIC-ATOMIC", selection).dynamicFields().field("atomic_note").state()); + assertFalse("overwritten extension values must also be scrubbed from reasons", fixture.audits().toString().contains("PRIVATE-DYNAMIC-CANARY")); + assertFalse(fixture.appAudits().toString().contains("PRIVATE-DYNAMIC-CANARY")); + assertTrue(task.__internalDynamicOriginalValues().isEmpty()); + task.deleteDynamicField("atomic_note"); + task.auditAs("delete dynamic field without deleting entity").save(fixture.context()); + assertEquals(Long.valueOf(4), task.getVersion()); + assertEquals(0, dynamicRowCount(task, "atomic_note")); + assertEquals(DynamicFieldValue.State.NOT_LOADED, task.dynamicFields().field("atomic_note").state()); + assertFalse(task.isPropertyLoaded("#atomic_note")); + } + + @Test + public void dynamicFailureAfterWritingRollsBackNativeAndExtensionAndRetainsRetryIntent() { + class FailingProvider extends JdbcDynamicFieldsProvider { + boolean fail; + FailingProvider() { super(jdbcSqlExecutor); } + @Override public void saveValue(DynamicFieldContext caller, DynamicSetCommand command) { + super.saveValue(caller, command); + if (fail) throw new IllegalStateException("injected failure after extension DML"); + } + } + var provider = new FailingProvider(); + var fixture = dynamicFixture(provider); + define(provider, "rollback_note", DynamicDataType.STRING); + Task task = dynamicTask(fixture, "DYNAMIC-ROLLBACK", new DynamicFieldSelection().selectString("rollback_note")); + task.updateTitle("retry-title"); + task.updateDynamicField("rollback_note", "retry-value"); + provider.fail = true; + assertThrows(IllegalStateException.class, () -> task.auditAs("rollback both writes").save(fixture.context())); + assertEquals("initial", nativeColumn(task, "title")); + assertEquals(1L, ((Number) nativeColumn(task, "version")).longValue()); + assertEquals(0, dynamicRowCount(task, "rollback_note")); + assertEquals(Long.valueOf(1), task.getVersion()); + assertEquals(EntityStatus.UPDATED, task.get$status()); + assertEquals("retry-value", task.dynamicFields().getString("rollback_note")); + assertFalse(task.__internalDynamicMutations().isEmpty()); + assertFalse(task.getEntityMutationLedger().currentChangeSet().isEmpty()); + assertEquals("failed transaction must not emit committed audit", 1, fixture.audits().size()); + provider.fail = false; + task.auditAs("retry original graph intent").save(fixture.context()); + assertEquals("retry-title", nativeColumn(task, "title")); + assertEquals(Long.valueOf(2), task.getVersion()); + assertEquals(1, dynamicRowCount(task, "rollback_note")); + assertEquals(2, fixture.audits().size()); + } + + @Test + public void dynamicProviderMustShareExecutorNotMerelyDataSource() { + var provider = new JdbcDynamicFieldsProvider(nativeDataSource); + var fixture = dynamicFixture(provider); + define(provider, "wrong_executor", DynamicDataType.STRING); + Task task = dynamicTask(fixture, "DYNAMIC-WRONG-EXECUTOR", new DynamicFieldSelection().selectString("wrong_executor")); + task.updateTitle("must-not-commit"); + task.updateDynamicField("wrong_executor", "must-not-commit"); + var error = assertThrows(DynamicFieldException.class, () -> task.auditAs("reject split transaction").save(fixture.context())); + assertTrue(error.getMessage().contains("exact active graph executor")); + assertEquals("initial", nativeColumn(task, "title")); + assertEquals(1L, ((Number) nativeColumn(task, "version")).longValue()); + assertEquals(0, dynamicRowCount(task, "wrong_executor")); + assertEquals(1, fixture.audits().size()); + } + + @Test + public void staleNativeVersionRejectsExtensionOnlyUpdate() { + var provider = new JdbcDynamicFieldsProvider(jdbcSqlExecutor); + var fixture = dynamicFixture(provider); + define(provider, "stale_note", DynamicDataType.STRING); + var selection = new DynamicFieldSelection().selectString("stale_note"); + Task current = dynamicTask(fixture, "DYNAMIC-STALE", selection); + Task stale = loadDynamicTask(fixture, "DYNAMIC-STALE", selection); + current.updateDynamicField("stale_note", "current"); + current.auditAs("advance optimistic version").save(fixture.context()); + stale.updateDynamicField("stale_note", "stale"); + assertThrows(RuntimeException.class, () -> stale.auditAs("reject stale extension-only write").save(fixture.context())); + assertEquals("current", loadDynamicTask(fixture, "DYNAMIC-STALE", selection).dynamicFields().getString("stale_note")); + assertEquals(Long.valueOf(1), stale.getVersion()); + assertFalse(stale.__internalDynamicMutations().isEmpty()); + assertEquals(2, fixture.audits().size()); + } + + @Test + public void dynamicReadbackUsesDatabaseAuthoritativeValueInSameTransaction() { + var provider = new JdbcDynamicFieldsProvider(jdbcSqlExecutor); + var fixture = dynamicFixture(provider); + DynamicFieldDef field = define(provider, "trigger_note", DynamicDataType.STRING); + jdbcSqlExecutor.execute("CREATE TRIGGER normalize_dynamic_note AFTER INSERT ON teaql_dynamic_field_value " + + "WHEN NEW.field_id=" + field.getId() + " BEGIN UPDATE teaql_dynamic_field_value SET string_value=UPPER(NEW.string_value) " + + "WHERE field_id=NEW.field_id AND owner_id=NEW.owner_id AND owner_type=NEW.owner_type; END"); + Task task = dynamicTask(fixture, "DYNAMIC-READBACK", new DynamicFieldSelection().selectString("trigger_note")); + task.updateDynamicField("trigger_note", "canonicalize me"); + task.auditAs("read authoritative extension payload").save(fixture.context()); + assertEquals("CANONICALIZE ME", task.dynamicFields().getString("trigger_note")); + assertTrue(task.__internalDynamicMutations().isEmpty()); + } + + @Test + public void unboundNewEntityDefinitionsCannotBeSilentlyAssignedToCurrentStorage() { + var provider = new JdbcDynamicFieldsProvider(jdbcSqlExecutor); + var fixture = dynamicFixture(provider); + define(provider, "unbound_note", DynamicDataType.STRING); + var task = new Task().updateTitle("must-not-insert").updateStatus("DYNAMIC-UNBOUND"); + task.setDynamicFieldValues(new DynamicFieldValues( + DynamicFieldMetadata.fromDefinitions(provider.listFieldDefs(definitionContext, "Task")), java.util.Map.of())); + task.updateDynamicField("unbound_note", "unbound value"); + var rejected = assertThrows(DynamicFieldException.class, + () -> task.auditAs("reject unbound definition source").save(fixture.context())); + assertEquals("DYNAMIC_FIELD_STORAGE_PROVENANCE_MISMATCH", rejected.errorCode()); + assertTrue(jdbcSqlExecutor.queryForList("SELECT id FROM task_data WHERE status=?", + new Object[]{"DYNAMIC-UNBOUND"}).isEmpty()); + assertFalse(task.__internalDynamicMutations().isEmpty()); + assertTrue(fixture.audits().isEmpty()); + } + + @Test + public void newEntityDynamicIntentSurvivesIdAssignment() { + var provider = new JdbcDynamicFieldsProvider(jdbcSqlExecutor); + var fixture = dynamicFixture(provider); + define(provider, "created_note", DynamicDataType.STRING); + var task = new Task().updateTitle("new graph").updateStatus("DYNAMIC-CREATE"); + task.setDynamicFieldValues(new DynamicFieldValues(fixture.context().dynamicFields().metadata("Task"), java.util.Map.of())); + task.updateDynamicField("created_note", "before-id"); + assertNull(task.getId()); + task.auditAs("create entity and extension together").save(fixture.context()); + assertNotNull(task.getId()); + assertEquals(Long.valueOf(1), task.getVersion()); + assertEquals(1, dynamicRowCount(task, "created_note")); + assertEquals("before-id", task.dynamicFields().getString("created_note")); + assertEquals(1, fixture.audits().size()); + } + + @Test + public void unsupportedDynamicNumberPrecisionIsRejectedBeforeAnyGraphDml() { + var provider = new JdbcDynamicFieldsProvider(jdbcSqlExecutor); + var fixture = dynamicFixture(provider); + define(provider, "integer_storage", DynamicDataType.NUMBER); + Task task = dynamicTask(fixture, "DYNAMIC-PRECISION", new DynamicFieldSelection().selectNumber("integer_storage")); + task.updateTitle("must-not-commit"); + task.updateDynamicField("integer_storage", new java.math.BigDecimal("1.25")); + assertThrows(DynamicFieldException.class, () -> task.auditAs("reject lossy numeric persistence").save(fixture.context())); + assertEquals("initial", nativeColumn(task, "title")); + assertEquals(1L, ((Number) nativeColumn(task, "version")).longValue()); + assertEquals(0, dynamicRowCount(task, "integer_storage")); + assertEquals(1, fixture.audits().size()); + } + + @Test + public void dynamicPermissionChangeIsRevalidatedBeforeNativeWrite() { + var provider = new JdbcDynamicFieldsProvider(jdbcSqlExecutor); + var fixture = dynamicFixture(provider); + var field = define(provider, "revoked_edit", DynamicDataType.STRING); + Task task = dynamicTask(fixture, "DYNAMIC-PERMISSION", new DynamicFieldSelection().selectString("revoked_edit")); + task.updateTitle("must-not-commit"); + task.updateDynamicField("revoked_edit", "pending"); + jdbcSqlExecutor.update("UPDATE teaql_dynamic_field_def SET editable=0 WHERE id=?", new Object[]{field.getId()}); + assertThrows(DynamicFieldException.class, () -> task.auditAs("respect updated write permission").save(fixture.context())); + assertEquals("initial", nativeColumn(task, "title")); + assertEquals(0, dynamicRowCount(task, "revoked_edit")); + assertEquals(1, fixture.audits().size()); + } + + @Test + public void explicitFixedNullAndDynamicValuePersistTogether() { + var provider = new JdbcDynamicFieldsProvider(jdbcSqlExecutor); + var fixture = dynamicFixture(provider); + define(provider, "fixed_null_note", DynamicDataType.STRING); + Task task = dynamicTask(fixture, "DYNAMIC-FIXED-NULL", new DynamicFieldSelection().selectString("fixed_null_note")); + task.updateTitle(null); + task.updateDynamicField("fixed_null_note", "stored"); + task.auditAs("persist explicit fixed null alongside dynamic edit").save(fixture.context()); + assertNull(nativeColumn(task, "title")); + assertNull(task.getTitle()); + assertEquals("stored", task.dynamicFields().getString("fixed_null_note")); + } + + @Test + public void multiEntityDynamicGraphRollsBackAsOneUnitAndDoesNotAdoptUnrelatedContextEntity() { + class FailingProvider extends JdbcDynamicFieldsProvider { + int calls; + boolean fail; + FailingProvider() { super(jdbcSqlExecutor); } + @Override public void saveValue(DynamicFieldContext caller, DynamicSetCommand command) { + super.saveValue(caller, command); + if (++calls == 2 && fail) throw new IllegalStateException("second extension write failed"); + } + } + var provider = new FailingProvider(); + var fixture = dynamicFixture(provider); + define(provider, "graph_note", DynamicDataType.STRING); + var selection = new DynamicFieldSelection().selectString("graph_note"); + Task parent = dynamicTask(fixture, "DYNAMIC-PARENT", selection); + Task child = dynamicTask(fixture, "DYNAMIC-CHILD", selection); + Task unrelated = dynamicTask(fixture, "DYNAMIC-UNRELATED", selection); + LoadState shared = LoadState.projection(parent.__internalLoadState().layout(), List.of("id", "version", "title", "status")); + parent.__internalUseLoadState(shared); + child.__internalUseLoadState(shared); + assertSame(parent.__internalLoadState(), child.__internalLoadState()); + parent.children = new SmartList<>(); + parent.children.add(child); + parent.updateTitle("parent-pending"); + child.updateTitle("child-pending"); + parent.updateDynamicField("graph_note", "parent-extension"); + child.updateDynamicField("graph_note", "child-extension"); + unrelated.updateDynamicField("graph_note", "unrelated-pending"); + provider.calls = 0; + provider.fail = true; + assertThrows(IllegalStateException.class, () -> parent.auditAs("rollback complete related graph").save(fixture.context())); + for (Task task : List.of(parent, child)) { + assertEquals("initial", nativeColumn(task, "title")); + assertEquals(1L, ((Number) nativeColumn(task, "version")).longValue()); + assertEquals(0, dynamicRowCount(task, "graph_note")); + assertEquals(Long.valueOf(1), task.getVersion()); + assertFalse(task.__internalDynamicMutations().isEmpty()); + } + assertEquals(3, fixture.audits().size()); + provider.fail = false; + provider.calls = 0; + parent.auditAs("retry related graph only").save(fixture.context()); + assertEquals(Long.valueOf(2), parent.getVersion()); + assertEquals(Long.valueOf(2), child.getVersion()); + assertEquals("parent-extension", parent.dynamicFields().getString("graph_note")); + assertEquals("child-extension", child.dynamicFields().getString("graph_note")); + assertEquals(5, fixture.audits().size()); + assertEquals(0, dynamicRowCount(unrelated, "graph_note")); + assertFalse(unrelated.getEntityMutationLedger().currentChangeSet().isEmpty()); + } + + @Test + public void dynamicFieldQueryBatchesThroughContextAndSharesOnlyActualLoadedShapes() { + class ObservedProvider extends io.teaql.data.dynamic.InMemoryDynamicFieldsProvider { + int batches; + io.teaql.data.dynamic.DynamicFieldContext lastContext; + List lastOwners; + @Override public java.util.Map loadValues( + io.teaql.data.dynamic.DynamicFieldContext caller, List owners, + io.teaql.data.dynamic.DynamicFieldSelection selection) { + batches++; + lastContext = caller; + lastOwners = List.copyOf(owners); + return super.loadValues(caller, owners, selection); + } + } + var provider = new ObservedProvider(); + for (String code : List.of("title", "extra")) { + var def = new io.teaql.data.dynamic.DynamicFieldDef(); + def.setScope(io.teaql.data.dynamic.DynamicFieldScope.global()); + def.setOwnerType("Task"); + def.setCode(code); + def.setDataType(io.teaql.data.dynamic.DynamicDataType.STRING); + provider.registerFieldDef(def); + } + var local = new DefaultUserContext(runtime); + local.putAttribute(io.teaql.data.dynamic.DynamicFieldsFacade.class.getName(), + new io.teaql.data.dynamic.DefaultDynamicFieldsFacade(provider)); + var tasks = new ArrayList(); + for (int i = 0; i < 3; i++) { + var task = new Task().updateTitle("fixed-" + i).updateStatus("DYNAMIC-LOAD-SHAPE"); + task.auditAs("seed dynamic query shape counterexamples").save(local); + tasks.add(task); + } + var writer = local.dynamicFields().purpose("prepare stored extensions").comment("write value and explicit null"); + writer.owner("Task", tasks.get(0).getId()).string("title").set("extension"); + writer.owner("Task", tasks.get(1).getId()).string("title").set(null); + + TaskRequest request = new TaskRequest().filterByStatus("DYNAMIC-LOAD-SHAPE"); + request.setSize(3); + request.addOrderBy("id", true); + request.selectDynamicFieldsWith(new io.teaql.data.dynamic.DynamicFieldSelection().selectString("title")); + var rows = request.comment("load dynamic availability counterexamples") + .purpose("verify batch state isolation").executeForList(local); + assertEquals(3, rows.size()); + assertEquals(1, provider.batches); + assertEquals(3, provider.lastOwners.size()); + assertEquals("load dynamic availability counterexamples", provider.lastContext.comment()); + assertEquals("verify batch state isolation", provider.lastContext.purpose()); + Task value = rows.get(0), nil = rows.get(1), missing = rows.get(2); + assertSame(value.__internalLoadState(), nil.__internalLoadState()); + assertNotSame(nil.__internalLoadState(), missing.__internalLoadState()); + assertSame(value.dynamicFields().metadata(), missing.dynamicFields().metadata()); + assertEquals("fixed-0", value.getTitle()); + assertEquals("extension", value.getProperty("#title")); + assertTrue(nil.isPropertyLoaded("#title")); + assertNull(nil.getProperty("#title")); + assertEquals(io.teaql.data.dynamic.DynamicFieldValue.State.NULL, nil.dynamicFields().field("title").state()); + assertEquals(io.teaql.data.dynamic.DynamicFieldValue.State.NOT_LOADED, missing.dynamicFields().field("title").state()); + assertFalse(missing.isPropertyLoaded("#title")); + assertEquals(io.teaql.data.dynamic.DynamicFieldValue.State.NOT_LOADED, value.dynamicFields().field("extra").state()); + assertTrue(rows.stream().allMatch(row -> row.getUpdatedProperties().isEmpty())); + assertNotSame(value.getEntityMutationLedger(), nil.getEntityMutationLedger()); + LoadState shared = nil.__internalLoadState(); + value.putAdditional("#extra", "private"); + assertSame(shared, nil.__internalLoadState()); + assertNotSame(shared, value.__internalLoadState()); + assertFalse(nil.isPropertyLoaded("#extra")); + value.addDynamicProperty("title", "derived"); + assertEquals("derived", value.getProperty("_title")); + assertEquals("extension", value.getProperty("#title")); + assertEquals("fixed-0", value.getProperty("title")); + + TaskRequest wrongType = new TaskRequest().filterByStatus("DYNAMIC-LOAD-SHAPE"); + wrongType.setSize(3); + wrongType.selectDynamicFieldsWith(new io.teaql.data.dynamic.DynamicFieldSelection().selectNumber("title")); + assertThrows(io.teaql.data.dynamic.DynamicFieldException.class, () -> wrongType + .comment("reject incompatible extension projection").purpose("verify type boundary").executeForList(local)); + assertEquals("provider must not receive a rejected projection", 1, provider.batches); + + TaskRequest streamed = new TaskRequest().filterByStatus("DYNAMIC-LOAD-SHAPE"); + streamed.setSize(3); + streamed.selectDynamicFieldsWith(new io.teaql.data.dynamic.DynamicFieldSelection().selectString("title")); + streamed.comment("reject unsupported streaming extension load").purpose("do not silently drop projections"); + var streaming = (StreamingQueryExecutor) runtime.getRegistry().resolve("sqlite"); + assertThrows(TeaQLRuntimeException.class, () -> streaming.queryForCursor(local, new DefaultQueryRequest(streamed))); + assertEquals(1, provider.batches); + } + @Test public void realSqliteFailureRetainsSafeIntentAndRecoversAfterSchemaInitialization() throws Exception { var logs = new ArrayList(); @@ -201,10 +694,20 @@ public void localDynamicSearchPreservesTrustedScopeInSqlite() { private static UserContext context; private static TeaQLRuntime runtime; private static JdbcSqlExecutor jdbcSqlExecutor; + private static DataSource nativeDataSource; public static class Task extends BaseEntity { + public static final String __TEAQL_FIELD_LAYOUT_REVISION = "sqlite-task-fixture-v1"; + public static final java.util.Map __TEAQL_FIXED_FIELD_INDEXES = + java.util.Map.of("id", 0, "version", 1, "title", 2, "status", 3); + public static final java.util.Map> __TEAQL_FIXED_FIELD_MAPPINGS = + java.util.Map.of("id", List.of("id", "id"), "version", List.of("version", "version"), + "title", List.of("title", "title"), "status", List.of("status", "status")); + public static final java.util.Set __TEAQL_FIXED_RELATION_NAMES = java.util.Set.of("children"); public String title; public String status; + public SmartList children; + public SmartList getChildren() { return children; } public String getTitle() { return title; } public Task updateTitle(String title) { @@ -228,6 +731,7 @@ public void __internalSet(String property, Object value) { switch (property) { case "title": this.title = (String) value; break; case "status": this.status = (String) value; break; + case "children": this.children = (SmartList) value; break; default: super.__internalSet(property, value); } } @@ -237,13 +741,17 @@ public Object __internalGet(String property) { switch (property) { case "title": return this.title; case "status": return this.status; + case "children": return this.children; default: return super.__internalGet(property); } } } public static class TaskRequest extends BaseRequest { - public TaskRequest() { super(Task.class); } + public TaskRequest() { + super(Task.class); + for (String field : List.of("id", "version", "title", "status")) selectProperty(field); + } @Override public String getTypeName() { return "Task"; } @@ -320,10 +828,18 @@ public static void setup() throws Exception { statusProp.setColumnType("VARCHAR(50)"); taskDescriptor.with("table_name", "task_data"); + var children = new io.teaql.core.meta.Relation(); + children.setName("children"); + children.setOwner(taskDescriptor); + children.setType(new io.teaql.core.meta.SimplePropertyType(SmartList.class)); + var properties = new ArrayList<>(taskDescriptor.getProperties()); + properties.add(children); + taskDescriptor.setProperties(properties); metaFactory.register(taskDescriptor); EntityMetaFactory.registerGlobal(metaFactory); DataSource ds = new SimpleDataSource(url, user, password); + nativeDataSource = ds; jdbcSqlExecutor = new JdbcSqlExecutor(ds); io.teaql.core.sqlite.SqliteDataServiceExecutor sqliteExecutor = new io.teaql.core.sqlite.SqliteDataServiceExecutor("sqlite", jdbcSqlExecutor, ds); From 22f4d26e33bc7ace3294dcd2c9564db7946b2444 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 7 Oct 2026 11:50:10 +0800 Subject: [PATCH 02/51] Verify shared-state examples against isolated runtime artifacts Refs #204. Keep workspace checks and add a strict registry mode requiring candidate JARs under an isolated cache; no generated sources or runtime behavior changed. --- .../GeneratedSchoolLoadStateTest.java | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java index 8e8bd511..aa0b9285 100644 --- a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java @@ -45,13 +45,25 @@ static final class Context extends DefaultUserContext implements SchoolManagemen @Test void generatedSchoolUsesLocalRuntimeAndSharedState() throws Exception { - Path root = Path.of(System.getenv("TEAQL_LOCAL_RUNTIME_ROOT")).toRealPath(); + String dependencyMode = System.getenv().getOrDefault("TEAQL_RUNTIME_DEPENDENCY_MODE", "workspace"); + assertTrue(List.of("workspace", "registry").contains(dependencyMode), "Unknown dependency verification mode"); + Path root = Path.of(System.getenv(dependencyMode.equals("registry") + ? "TEAQL_ARTIFACT_CACHE" : "TEAQL_LOCAL_RUNTIME_ROOT")).toRealPath(); for (Class type : List.of(BaseEntity.class, LoadState.class, TeaQLRuntime.class, JdbcSqlExecutor.class, SqliteDataServiceExecutor.class, TeaQLModule.class, School.class)) { Path location = Path.of(type.getProtectionDomain().getCodeSource().getLocation().toURI()).toRealPath(); assertTrue(location.startsWith(root), "Non-local dependency " + type + " from " + location); - assertTrue(location.endsWith(Path.of("target/classes")), "Expected reactor classes: " + location); - System.out.println("LOCAL_CLASS " + type.getName() + " " + location); + if (dependencyMode.equals("registry")) { + assertTrue(location.toString().endsWith(".jar"), "Expected isolated downloaded JAR: " + location); + if (type.getName().startsWith("io.teaql.")) { + String version = System.getenv("TEAQL_CANDIDATE_VERSION"); + assertNotNull(version); assertTrue(location.toString().contains("/" + version + "/")); + } + System.out.println("ARTIFACT_CLASS " + type.getName() + " " + location); + } else { + assertTrue(location.endsWith(Path.of("target/classes")), "Expected reactor classes: " + location); + System.out.println("LOCAL_CLASS " + type.getName() + " " + location); + } } String round = System.getenv("TEAQL_LOAD_STATE_ROUND"); SQLiteDataSource dataSource = new SQLiteDataSource(); From 1f37406f22a42e7627d5445bf284823a7f1bcd51 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 7 Oct 2026 12:33:52 +0800 Subject: [PATCH 03/51] Hydrate typed JSON through context model metadata Refs #204. Preserve indexed load state and private mutation ownership; reject unknown types, internal state and aliases. Add generated native and relation graph roundtrip examples. --- README.md | 19 ++- examples/shared-load-state/README.md | 9 ++ .../GeneratedNestedGraphAssertions.java | 19 +++ .../GeneratedSchoolLoadStateTest.java | 19 ++- examples/shared-load-state/verify.sh | 2 + teaql-jackson/pom.xml | 16 ++ .../java/io/teaql/jackson/TeaQLModule.java | 49 ++++++ .../jackson/TypedEntityJsonDeserializer.java | 128 ++++++++++++++++ .../TypedEntityJsonDeserializerTest.java | 140 ++++++++++++++++++ 9 files changed, 399 insertions(+), 2 deletions(-) create mode 100644 teaql-jackson/src/main/java/io/teaql/jackson/TypedEntityJsonDeserializer.java create mode 100644 teaql-jackson/src/test/java/io/teaql/jackson/TypedEntityJsonDeserializerTest.java diff --git a/README.md b/README.md index a55147cb..42ea3fc3 100644 --- a/README.md +++ b/README.md @@ -370,7 +370,9 @@ The main entity construction, JSON, and SQL row-mapping paths are designed to avoid reflection-heavy bean mutation: - `teaql-jackson` registers explicit entity serializers and deserializers - through `TeaQLModule`. + through `TeaQLModule`. For indexed domain-entity reads, bind a mapper to the + installed model with `new TeaQLModule(context)`; the legacy singleton retains + generic BaseEntity reading and entity serialization compatibility. - `teaql-sql-portable` creates entities through `EntityDescriptor.createEntity()`. - Generated or hand-written metadata registers an `entitySupplier`, such as @@ -380,6 +382,21 @@ Dynamic and additional values should remain JSON-friendly: scalars, maps, lists, or other explicitly serializable values. Arbitrary application objects may still trigger Jackson's default bean introspection. +The context-bound reader uses registered entity suppliers and typed internal +hydration, preserving loaded NULL versus omitted NotLoaded without dirtying a +Mutation Ledger. Compatible JSON shapes share immutable state; the reader's +value-free shape cache is bounded to 128 entries. Unknown model types, fields, +internal state and duplicate aliases reject instead of invoking bean setters. +Keep one configured mapper/model binding per context metadata snapshot; do not +switch models on a mapper that has already cached readers. + +JSON hydration does not make client data authoritative or authorize a save. +Load the governed database object before applying business mutations and retain +whole-object Checker and audited-save requirements. Readonly `_` properties may +roundtrip as data; persistent `#` fields require a separately trusted definition +and storage-provenance contract and currently reject in the typed JSON reader. +See the [generated shared-state example](examples/shared-load-state/README.md). + See the [Native Image Reflection Guide](NATIVE_IMAGE_REFLECTION_GUIDE.md) for the baseline and coding rules. diff --git a/examples/shared-load-state/README.md b/examples/shared-load-state/README.md index 6221a0ab..416bbb37 100644 --- a/examples/shared-load-state/README.md +++ b/examples/shared-load-state/README.md @@ -50,6 +50,15 @@ Loaded empty lists serialize as arrays; unselected lists stay absent. A target filter cannot erase the source's known FK or invent loaded target detail. These cases are required in both retained rounds, not optional smoke output. +The mapper uses `new TeaQLModule(context)` for indexed, strongly typed JSON +reads through installed suppliers rather than bean reflection. Both rounds +roundtrip full and minimal School projections, nested forward/reverse graphs, +and loaded empty versus omitted reverse lists. Missing target details stay +NotLoaded, actual NULL stays loaded, and compatible decoded rows share state +without creating mutation intent. This does not certify arbitrary polymorphic +or persistent `#` JSON input: those require their own contract. Decoded data is +not a substitute for loading authoritative state before mutation. + - Repeated schema bootstrap preserves Platform 1 and SchoolType 1001/1002. - Generated indexes and revision match the installed type layout. - Compatible full and sparse lists share exact immutable snapshot references diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedNestedGraphAssertions.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedNestedGraphAssertions.java index 1cf51cd2..00c77824 100644 --- a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedNestedGraphAssertions.java +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedNestedGraphAssertions.java @@ -83,6 +83,12 @@ static void verify(UserContext context, Long schoolId, String renamed, assertFalse(json.get("schoolList").get(0).has("address")); assertFalse(json.get("schoolList").get(0).has("_original_values")); assertSame(state, children.get(0).__internalLoadState()); + var restored = mapper.treeToValue(json, platform.getClass()); + assertEquals(2, restored.getSchoolList().size()); + assertEquals(renamed, E.school(restored.getSchoolList().get(0)).getName().eval()); + assertSame(restored.getSchoolList().get(0).__internalLoadState(), restored.getSchoolList().get(1).__internalLoadState()); + assertFalse(restored.getSchoolList().get(0).isPropertyLoaded("address")); + assertFalse(restored.__internalHasMutationLedger()); var nested = Q.schoolsWithMinimalFields().withIdIs(schoolId) .selectPlatformWith(Q.platformsWithMinimalFields() @@ -97,6 +103,9 @@ static void verify(UserContext context, Long schoolId, String renamed, var nestedJson = mapper.readTree(mapper.writeValueAsString(nested)); assertEquals(renamed, nestedJson.get("platform").get("schoolList").get(0).get("name").asText()); assertEquals(second, nestedJson.get("platform").get("schoolList").get(1).get("name").asText()); + var restoredNested = mapper.treeToValue(nestedJson, nested.getClass()); + assertEquals(2, E.school(restoredNested).getPlatform().eval().getSchoolList().size()); + assertEquals(nestedJson, mapper.readTree(mapper.writeValueAsString(restoredNested))); var empty = Q.schoolTypesWithMinimalFields().withIdIs(1002L) .selectSchoolListWith(Q.schoolsWithMinimalFields().selectName().orderByIdAscending().limit(2)) @@ -108,12 +117,17 @@ static void verify(UserContext context, Long schoolId, String renamed, var emptyJson = mapper.readTree(mapper.writeValueAsString(empty)); assertTrue(emptyJson.get("schoolList").isArray()); assertEquals(0, emptyJson.get("schoolList").size()); + var restoredEmpty = mapper.treeToValue(emptyJson, empty.getClass()); + assertTrue(restoredEmpty.isPropertyLoaded("schoolList")); + assertTrue(restoredEmpty.getSchoolList().isEmpty()); var unselected = Q.schoolTypesWithMinimalFields().withIdIs(1002L) .limit(1).comment("what: load the same constant without selecting its Schools") .purpose("why: NotLoaded must not serialize as an empty list").executeForOne(context); assertFalse(unselected.isPropertyLoaded("schoolList")); assertFalse(mapper.readTree(mapper.writeValueAsString(unselected)).has("schoolList")); + var restoredUnselected = mapper.readValue(mapper.writeValueAsString(unselected), unselected.getClass()); + assertFalse(restoredUnselected.isPropertyLoaded("schoolList")); var filtered = Q.schoolsWithMinimalFields().withIdIs(schoolId) .selectSchoolTypeWith(Q.schoolTypesWithMinimalFields().withIdIs(1002L)) @@ -123,6 +137,11 @@ static void verify(UserContext context, Long schoolId, String renamed, assertFalse(filtered.getSchoolType().isPropertyLoaded("code")); var filteredJson = mapper.readTree(mapper.writeValueAsString(filtered)); assertFalse(filteredJson.get("schoolType").has("code")); + var restoredFiltered = mapper.treeToValue(filteredJson, filtered.getClass()); + assertEquals(1001L, restoredFiltered.getSchoolType().getId()); + assertFalse(restoredFiltered.getSchoolType().isPropertyLoaded("code")); + assertFalse(restoredFiltered.__internalHasMutationLedger()); + System.out.println("PASS generated Java typed JSON graph roundtrip and Empty/NotLoaded isolation"); System.out.println("PASS generated Java nested/reverse graph Q/E/JSON and Empty/NotLoaded isolation"); } } diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java index aa0b9285..81f167dd 100644 --- a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java @@ -151,18 +151,35 @@ void generatedSchoolUsesLocalRuntimeAndSharedState() throws Exception { assertTrue(sparse.get(0).isPropertyLoaded("id")); assertTrue(sparse.get(0).isPropertyLoaded("version")); assertFalse(sparse.get(0).isPropertyLoaded("address")); - var mapper = new ObjectMapper().registerModule(TeaQLModule.INSTANCE) + var mapper = new ObjectMapper().registerModule(new TeaQLModule(context)) .registerModule(new JavaTimeModule()).disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); var sparseJson = mapper.readTree(mapper.writeValueAsString(sparse)); assertEquals(first, sparseJson.get(0).get("name").asText()); assertFalse(sparseJson.get(0).has("address"), "NotLoaded must not become serialized NULL"); assertFalse(sparseJson.get(0).has("establishedDate")); assertSame(sparseState, sparse.get(0).__internalLoadState()); + var sparseRestored = mapper.treeToValue(sparseJson.get(0), School.class); + assertEquals(first, E.school(sparseRestored).getName().eval()); + assertFalse(sparseRestored.isPropertyLoaded("address")); + assertFalse(sparseRestored.__internalHasMutationLedger()); + assertTrue(sparseRestored.getUpdatedProperties().isEmpty()); + var sparseSecondRestored = mapper.treeToValue(sparseJson.get(1), School.class); + assertSame(sparseRestored.__internalLoadState(), sparseSecondRestored.__internalLoadState()); + assertEquals(sparseJson.get(0), mapper.readTree(mapper.writeValueAsString(sparseRestored))); var fullJson = mapper.readTree(mapper.writeValueAsString(full)); assertEquals("1995-09-01", fullJson.get(0).get("establishedDate").asText()); assertEquals(0, fullJson.get(0).get("studentCapacity").asInt()); assertFalse(fullJson.get(0).get("active").asBoolean()); assertSame(snapshot, full.get(0).__internalLoadState()); + var fullRestored = mapper.treeToValue(fullJson.get(0), School.class); + assertEquals(0, E.school(fullRestored).getStudentCapacity().eval()); + assertEquals(false, E.school(fullRestored).isActive().eval()); + assertEquals(LocalDate.of(1995, 9, 1), fullRestored.getEstablishedDate()); + assertEquals(1001L, fullRestored.getSchoolType().getId()); + assertFalse(fullRestored.__internalHasMutationLedger()); + assertTrue(fullRestored.getUpdatedProperties().isEmpty()); + assertEquals(fullJson.get(0), mapper.readTree(mapper.writeValueAsString(fullRestored))); + System.out.println("PASS generated Java typed native JSON roundtrip and snapshot sharing " + round); if (wide) { for (int index : new int[]{31, 32, 63, 64, 65, 129}) { String name = School.__TEAQL_FIXED_FIELD_INDEXES.entrySet().stream() diff --git a/examples/shared-load-state/verify.sh b/examples/shared-load-state/verify.sh index d34deb8d..a6f9d0f7 100644 --- a/examples/shared-load-state/verify.sh +++ b/examples/shared-load-state/verify.sh @@ -67,6 +67,8 @@ for round in first second; do exit 1 } rg -F "PASS generated Java indexed Q/E/Checker/create/update/delete and snapshot sharing $round" "$run_dir/$round.log" + rg -F "PASS generated Java typed native JSON roundtrip and snapshot sharing $round" "$run_dir/$round.log" + rg -F "PASS generated Java typed JSON graph roundtrip and Empty/NotLoaded isolation" "$run_dir/$round.log" rg -F "PASS generated Java dynamic storage provenance and retry" "$run_dir/$round.log" rg -F "PASS generated Java namespace serialization and NotLoaded boundary" "$run_dir/$round.log" rg -F "PASS generated Java nested/reverse graph Q/E/JSON and Empty/NotLoaded isolation" "$run_dir/$round.log" diff --git a/teaql-jackson/pom.xml b/teaql-jackson/pom.xml index bc15874d..0ab6824d 100644 --- a/teaql-jackson/pom.xml +++ b/teaql-jackson/pom.xml @@ -12,8 +12,12 @@ teaql-jackson teaql-jackson Jackson integration for TeaQL serialization + + + io.teaqlteaql-runtimetest + io.teaql teaql-core @@ -29,4 +33,16 @@ test + + + + org.apache.maven.plugins + maven-surefire-plugin + + + @{argLine} --add-reads io.teaql.jackson=io.teaql.runtime + + + + diff --git a/teaql-jackson/src/main/java/io/teaql/jackson/TeaQLModule.java b/teaql-jackson/src/main/java/io/teaql/jackson/TeaQLModule.java index 950d60df..e3c12c33 100644 --- a/teaql-jackson/src/main/java/io/teaql/jackson/TeaQLModule.java +++ b/teaql-jackson/src/main/java/io/teaql/jackson/TeaQLModule.java @@ -1,12 +1,19 @@ package io.teaql.jackson; import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.databind.deser.Deserializers; +import java.util.HashSet; +import java.util.Set; import io.teaql.core.BaseEntity; import io.teaql.core.SmartList; +import io.teaql.core.UserContext; +import io.teaql.core.meta.EntityMetaFactory; public class TeaQLModule extends SimpleModule { public static final TeaQLModule INSTANCE = new TeaQLModule(); + private final Set> typedEntities = new HashSet<>(); + private boolean contextBound; public TeaQLModule() { super("TeaQL"); @@ -14,4 +21,46 @@ public TeaQLModule() { addDeserializer(BaseEntity.class, new BaseEntityJsonDeserializer()); addSerializer(SmartList.class, new SmartListAsListSerializer(SmartList.class)); } + + /** Bind typed readers to this context's installed model, never process-global metadata. */ + public TeaQLModule(UserContext context) { + this(); + contextBound = true; + EntityMetaFactory metadata = EntityMetaFactory.requireFrom(context); + for (var descriptor : metadata.allEntityDescriptors()) { + Class type = descriptor.getTargetType(); + if (type != null && BaseEntity.class.isAssignableFrom(type) && type != BaseEntity.class) { + if (!typedEntities.add(type)) throw new IllegalArgumentException("Duplicate model entity class: " + type.getName()); + registerTyped(type, new TypedEntityJsonDeserializer(descriptor)); + } + } + } + + @SuppressWarnings("unchecked") + private void registerTyped(Class type, TypedEntityJsonDeserializer reader) { + addDeserializer((Class) type, reader); + } + + // A context-bound module must not be ignored after the serialization-only singleton. + @Override public Object getTypeId() { return contextBound ? this : super.getTypeId(); } + + @Override public void setupModule(SetupContext setup) { + super.setupModule(setup); + if (!contextBound) return; + setup.addDeserializers(new Deserializers.Base() { + @Override public com.fasterxml.jackson.databind.JsonDeserializer findBeanDeserializer( + com.fasterxml.jackson.databind.JavaType type, + com.fasterxml.jackson.databind.DeserializationConfig config, + com.fasterxml.jackson.databind.BeanDescription description) { + Class target = type.getRawClass(); + if (target == BaseEntity.class || !BaseEntity.class.isAssignableFrom(target) || typedEntities.contains(target)) return null; + return new com.fasterxml.jackson.databind.JsonDeserializer() { + @Override public BaseEntity deserialize(com.fasterxml.jackson.core.JsonParser parser, + com.fasterxml.jackson.databind.DeserializationContext context) throws java.io.IOException { + return context.reportInputMismatch(target, "Entity type is not installed in this context's model"); + } + }; + } + }); + } } diff --git a/teaql-jackson/src/main/java/io/teaql/jackson/TypedEntityJsonDeserializer.java b/teaql-jackson/src/main/java/io/teaql/jackson/TypedEntityJsonDeserializer.java new file mode 100644 index 00000000..61043845 --- /dev/null +++ b/teaql-jackson/src/main/java/io/teaql/jackson/TypedEntityJsonDeserializer.java @@ -0,0 +1,128 @@ +package io.teaql.jackson; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonNode; +import io.teaql.core.BaseEntity; +import io.teaql.core.Entity; +import io.teaql.core.FieldLayout; +import io.teaql.core.LoadState; +import io.teaql.core.SmartList; +import io.teaql.core.meta.EntityDescriptor; +import io.teaql.core.meta.Relation; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** Model-bound JSON hydration; no bean setters, reflection, database calls or mutation APIs. */ +final class TypedEntityJsonDeserializer extends JsonDeserializer { + private record Binding(String member, Class type, int index, Class element) {} + private final Class target; + private final java.util.function.Supplier supplier; + private final FieldLayout layout; + private final Map fixed; + private final Map relations; + // Bounded, value-free geometry, owned by one mapper/model reader, not a process-global cache. + private final Map, LoadState> shapes = new LinkedHashMap<>(16, .75f, true); + + TypedEntityJsonDeserializer(EntityDescriptor descriptor) { + target = descriptor.getTargetType(); + supplier = Objects.requireNonNull(descriptor.getEntitySupplier(), "Typed JSON requires an installed entity supplier"); + layout = FieldLayout.forType(target); + if (!layout.isGenerated()) throw new IllegalArgumentException("Typed JSON requires generated field indexes: " + target.getName()); + Map fields = new HashMap<>(); + Map named = new HashMap<>(); + for (EntityDescriptor current = descriptor; current != null; current = current.getParent()) { + for (var property : current.getProperties()) { + Integer index = layout.findIndex(property.getName()); + Class type = property.getType().javaType(); + Class element = null; + if (SmartList.class.isAssignableFrom(type)) { + if (!(property instanceof Relation relation) || relation.getReverseProperty() == null) { + throw new IllegalArgumentException("Typed JSON list requires relation metadata: " + property.getName()); + } + element = relation.getReverseProperty().getOwner().getTargetType(); + Objects.requireNonNull(element, "Typed JSON relation element type"); + } + Binding binding = new Binding(property.getName(), type, index == null ? -1 : index, element); + if (index != null) fields.putIfAbsent(index, binding); + else if (layout.relationNames().contains(property.getName())) named.putIfAbsent(property.getName(), binding); + else throw new IllegalArgumentException("Metadata is outside the installed field layout: " + property.getName()); + } + } + if (fields.size() != layout.indexes().size()) throw new IllegalArgumentException("Incomplete typed JSON field metadata: " + target.getName()); + fixed = Map.copyOf(fields); relations = Map.copyOf(named); + } + + private Binding binding(String name) { + Integer index = layout.findIndex(name); + return index == null ? relations.get(name) : fixed.get(index); + } + + @Override public BaseEntity deserialize(JsonParser parser, DeserializationContext context) throws IOException { + JsonNode node = parser.getCodec().readTree(parser); + if (!node.isObject()) return context.reportInputMismatch(target, "Expected a model entity object"); + List members = new ArrayList<>(); + var names = node.fieldNames(); + while (names.hasNext()) { + String name = names.next(); + if (name.startsWith("_") && name.length() > 1) continue; // readonly derived data, not load-state metadata + Binding field = binding(name); + if (field == null) return context.reportInputMismatch(target, "Unknown or unsupported model field: %s", name); + members.add(field.member()); + } + members.sort(String::compareTo); + for (int index = 1; index < members.size(); index++) { + if (members.get(index).equals(members.get(index - 1))) { + return context.reportInputMismatch(target, "Duplicate aliases for model field: %s", members.get(index)); + } + } + LoadState shape; + synchronized (shapes) { + shape = shapes.get(members); + if (shape == null) { + shape = LoadState.projection(layout, members); + shapes.put(List.copyOf(members), shape); + if (shapes.size() > 128) shapes.remove(shapes.keySet().iterator().next()); + } + } + Entity created = supplier.get(); + if (created == null || created.getClass() != target || !(created instanceof BaseEntity)) { + return context.reportInputMismatch(target, "Installed model supplier returned an incompatible entity"); + } + BaseEntity entity = (BaseEntity) created; + LoadState initial = entity.__internalLoadState(); + if (initial.bits() != 0L || !initial.overflow().isEmpty() || !initial.selectedNames().isEmpty() + || entity.__internalHasMutationLedger()) { + return context.reportInputMismatch(target, "Installed model supplier must return a fresh entity"); + } + entity.__internalUseLoadState(shape); + var fields = node.fields(); + while (fields.hasNext()) { + var field = fields.next(); String name = field.getKey(); JsonNode value = field.getValue(); + if (name.startsWith("_") && name.length() > 1) { + entity.addDynamicProperty(name.substring(1), parser.getCodec().treeToValue(value, Object.class)); + continue; + } + Binding binding = binding(name); + Object converted; + if (binding.element() != null && !value.isNull()) { + if (!value.isArray()) return context.reportInputMismatch(target, "Expected relation array: %s", name); + List children = new ArrayList<>(value.size()); + for (JsonNode child : value) { + if (child.isNull()) return context.reportInputMismatch(target, "Null relation-list element: %s", name); + children.add(parser.getCodec().treeToValue(child, binding.element())); + } + converted = SmartList.takeOwnership(children); + } else converted = parser.getCodec().treeToValue(value, binding.type()); + if (binding.index() >= 0) entity.__internalHydrate(binding.member(), converted, binding.index()); + else entity.__internalSet(binding.member(), converted); + } + return entity; + } +} diff --git a/teaql-jackson/src/test/java/io/teaql/jackson/TypedEntityJsonDeserializerTest.java b/teaql-jackson/src/test/java/io/teaql/jackson/TypedEntityJsonDeserializerTest.java new file mode 100644 index 00000000..d6afbcab --- /dev/null +++ b/teaql-jackson/src/test/java/io/teaql/jackson/TypedEntityJsonDeserializerTest.java @@ -0,0 +1,140 @@ +package io.teaql.jackson; + +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.teaql.core.BaseEntity; +import io.teaql.core.FieldLayout; +import io.teaql.core.SmartList; +import io.teaql.core.meta.EntityDescriptor; +import io.teaql.core.meta.SimpleEntityMetaFactory; +import io.teaql.runtime.DefaultUserContext; +import io.teaql.runtime.TeaQLRuntime; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.Test; +import static org.junit.Assert.*; + +public class TypedEntityJsonDeserializerTest { + public static class UnregisteredRow extends BaseEntity {} + public static class Row extends BaseEntity { + static final FieldLayout LAYOUT = FieldLayout.installGenerated(FieldLayout.generated(Row.class, "json-v1", + Map.of("id", 0, "version", 1, "name", 2, "address", 3, "count", 4, "active", 5, "base_url", 6), + Map.of("id", List.of("id", "id"), "version", List.of("version", "version"), + "name", List.of("name", "name"), "address", List.of("address", "address"), + "count", List.of("count", "count"), "active", List.of("active", "active"), + "base_url", List.of("baseUrl", "legacy_url")), Set.of("child", "childList"))); + String name, address, baseUrl, owner; + Integer count; + Boolean active; + Row child; + SmartList childList; + @Override public void __internalSet(String field, Object value) { + markPropertyLoaded(field); + switch (field) { + case "name" -> name = (String) value; + case "address" -> address = (String) value; + case "baseUrl" -> baseUrl = (String) value; + case "count" -> count = (Integer) value; + case "active" -> active = (Boolean) value; + case "child" -> child = (Row) value; + case "childList" -> childList = (SmartList) value; + default -> super.__internalSet(field, value); + } + } + @Override public Object __internalGet(String field) { + return switch (field) { + case "name" -> name; case "address" -> address; case "baseUrl" -> baseUrl; + case "count" -> count; case "active" -> active; case "child" -> child; case "childList" -> childList; + default -> super.__internalGet(field); + }; + } + } + + private static DefaultUserContext context(String owner, AtomicInteger constructed) { + assertTrue(Row.LAYOUT.isGenerated()); + var metadata = new SimpleEntityMetaFactory(); + var descriptor = new EntityDescriptor(); descriptor.setType("Row"); descriptor.setTargetType(Row.class); + descriptor.withEntitySupplier(() -> { constructed.incrementAndGet(); var row = new Row(); row.owner = owner; return row; }); + descriptor.addSimpleProperty("id", Long.class); descriptor.addSimpleProperty("version", Long.class); + descriptor.addSimpleProperty("name", String.class); descriptor.addSimpleProperty("address", String.class); + descriptor.addSimpleProperty("count", Integer.class); descriptor.addSimpleProperty("active", Boolean.class); + descriptor.addSimpleProperty("baseUrl", String.class); + metadata.register(descriptor); + return new DefaultUserContext(TeaQLRuntime.builder().metadata(metadata).build()); + } + + @Test public void typedRoundTripPreservesLoadedNullOmissionAndFalsyValuesWithoutMutation() throws Exception { + var mapper = new ObjectMapper().registerModule(new TeaQLModule(context("A", new AtomicInteger()))); + var row = mapper.readValue("{\"id\":1,\"version\":7,\"name\":null,\"count\":0,\"active\":false,\"baseUrl\":\"\",\"_total\":null}", Row.class); + assertEquals(Long.valueOf(1), row.getId()); assertEquals(Long.valueOf(7), row.getVersion()); + assertTrue(row.isPropertyLoaded("name")); assertNull(row.name); + assertFalse(row.isPropertyLoaded("address")); assertNull(row.address); + assertEquals(Integer.valueOf(0), row.count); assertEquals(Boolean.FALSE, row.active); assertEquals("", row.baseUrl); + assertTrue(row.isPropertyLoaded("base_url")); assertTrue(row.getAdditionalInfo().containsKey("_total")); + assertNull(row.getDynamicProperty("total")); assertNull(row.getDynamicProperty("missing")); + assertFalse(row.__internalHasMutationLedger()); assertTrue(row.getUpdatedProperties().isEmpty()); + var json = mapper.writeValueAsString(row); var restored = mapper.readValue(json, Row.class); + assertEquals(mapper.readTree(json), mapper.readTree(mapper.writeValueAsString(restored))); + assertSame(row.__internalLoadState(), restored.__internalLoadState()); + assertFalse(restored.__internalHasMutationLedger()); assertFalse(restored.isPropertyLoaded("address")); + } + + @Test public void sameShapesShareStateDespiteFieldOrderAliasesAndPrivateValues() throws Exception { + var mapper = new ObjectMapper().registerModule(new TeaQLModule(context("A", new AtomicInteger()))); + Row first = mapper.readValue("{\"id\":1,\"baseUrl\":null}", Row.class); + Row second = mapper.readValue("{\"legacy_url\":\"private\",\"id\":2}", Row.class); + assertSame(first.__internalLoadState(), second.__internalLoadState()); assertNull(first.baseUrl); assertEquals("private", second.baseUrl); + Row minimal = mapper.readValue("{\"id\":3}", Row.class); + assertNotSame(first.__internalLoadState(), minimal.__internalLoadState()); assertFalse(minimal.isPropertyLoaded("base_url")); + first.__internalHydrate("address", "only first", 3); + assertFalse(second.isPropertyLoaded("address")); assertFalse(minimal.isPropertyLoaded("address")); + } + + @Test public void unknownFieldsInternalStateAndDuplicateAliasesRejectBeforeConstruction() throws Exception { + var count = new AtomicInteger(); var mapper = new ObjectMapper().registerModule(new TeaQLModule(context("A", count))); + for (String json : List.of("{\"unknown\":1}", "{\"entityMutationLedger\":{}}", "{\"loadState\":{}}", + "{\"$status\":\"PERSISTED\"}", "{\"comment\":\"untrusted\"}", "{\"#name\":\"needs definitions\"}", + "{\"baseUrl\":\"A\",\"legacy_url\":\"B\"}")) { + assertThrows(JsonMappingException.class, () -> mapper.readValue(json, Row.class)); + } + assertEquals(0, count.get()); + } + + @Test public void contextReadersUseIndependentSuppliersAndCanFollowTheSingletonModule() throws Exception { + var first = new ObjectMapper().registerModule(TeaQLModule.INSTANCE).registerModule(new TeaQLModule(context("A", new AtomicInteger()))); + var second = new ObjectMapper().registerModule(new TeaQLModule(context("B", new AtomicInteger()))); + assertEquals("A", first.readValue("{\"id\":1}", Row.class).owner); + assertEquals("B", second.readValue("{\"id\":1}", Row.class).owner); + assertThrows(RuntimeException.class, () -> new TeaQLModule((io.teaql.core.UserContext) null)); + } + + @Test public void nonFreshOrWrongTypeSuppliersCannotHideValuesBehindANewProjection() throws Exception { + var context = context("A", new AtomicInteger()); + var descriptor = context.capability(io.teaql.core.meta.EntityMetaFactory.class).resolveEntityDescriptor("Row"); + descriptor.setEntitySupplier(() -> { + var row = new Row(); row.__internalSet("name", "preset must not become NotLoaded"); return row; + }); + var mapper = new ObjectMapper().registerModule(new TeaQLModule(context)); + assertThrows(JsonMappingException.class, () -> mapper.readValue("{\"id\":1}", Row.class)); + descriptor.setEntitySupplier(BaseEntity::new); + var wrongType = new ObjectMapper().registerModule(new TeaQLModule(context)); + assertThrows(JsonMappingException.class, () -> wrongType.readValue("{}", Row.class)); + } + + @Test public void derivedNamespaceCannotChangeNativeAvailabilityOrCreateMutationIntent() throws Exception { + var mapper = new ObjectMapper().registerModule(new TeaQLModule(context("A", new AtomicInteger()))); + var row = mapper.readValue("{\"name\":\"fixed\",\"_name\":\"derived\",\"_address\":\"not native\",\"_loadState\":{\"bits\":127}}", Row.class); + assertEquals("fixed", row.name); assertEquals("derived", row.getDynamicProperty("name")); + assertFalse(row.isPropertyLoaded("address")); assertNull(row.address); + assertFalse(row.__internalHasMutationLedger()); assertTrue(row.getUpdatedProperties().isEmpty()); + } + + @Test public void unregisteredEntityCannotFallBackToBeanMutation() throws Exception { + var mapper = new ObjectMapper().registerModule(new TeaQLModule(context("A", new AtomicInteger()))); + var error = assertThrows(JsonMappingException.class, + () -> mapper.readValue("{\"comment\":\"untrusted\"}", UnregisteredRow.class)); + assertTrue(error.getMessage().contains("not installed in this context")); + } +} From 7119f6c4aa29181b73fe04f027bd66cf591f01bb Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 7 Oct 2026 13:31:55 +0800 Subject: [PATCH 04/51] Test cold concurrent generated field layout installation --- .../java/io/teaql/core/LoadStateTest.java | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/teaql-core/src/test/java/io/teaql/core/LoadStateTest.java b/teaql-core/src/test/java/io/teaql/core/LoadStateTest.java index 5513e4ff..805dd624 100644 --- a/teaql-core/src/test/java/io/teaql/core/LoadStateTest.java +++ b/teaql-core/src/test/java/io/teaql/core/LoadStateTest.java @@ -124,6 +124,53 @@ public Object __internalGet(String property) { static final class LateLayoutEntity extends BaseEntity { } + private static final java.util.concurrent.atomic.AtomicInteger COLD_INSTALLATIONS = + new java.util.concurrent.atomic.AtomicInteger(); + + // Only the cold-start control touches this class; the other fixture is warmed in BeforeClass. + static final class ColdLayoutEntity extends BaseEntity { + static final FieldLayout LAYOUT = install(); + private static FieldLayout install() { + COLD_INSTALLATIONS.incrementAndGet(); + return FieldLayout.installGenerated(FieldLayout.generated(ColdLayoutEntity.class, "cold-v1", + Map.of("id", 0, "version", 1, "base_url", 2), + Map.of("id", List.of("id", "id"), "version", List.of("version", "version"), + "base_url", List.of("baseUrl", "legacy_url")), Set.of())); + } + } + + @Test + public void coldConcurrentTypeInitializationInstallsOneGeneratedLayout() throws Exception { + assertEquals(0, COLD_INSTALLATIONS.get()); + var executor = java.util.concurrent.Executors.newFixedThreadPool(16); + var start = new java.util.concurrent.CyclicBarrier(16); + try { + List> futures = new ArrayList<>(); + for (int i = 0; i < 16; i++) { + futures.add(executor.submit(() -> { + start.await(10, java.util.concurrent.TimeUnit.SECONDS); + var installed = ColdLayoutEntity.LAYOUT; + assertSame(installed, FieldLayout.forType(ColdLayoutEntity.class)); + assertTrue(installed.isGenerated()); + assertEquals(Integer.valueOf(2), installed.findIndex("base_url")); + assertEquals(Integer.valueOf(2), installed.findIndex("baseUrl")); + assertEquals(Integer.valueOf(2), installed.findIndex("legacy_url")); + return installed; + })); + } + FieldLayout first = futures.get(0).get(15, java.util.concurrent.TimeUnit.SECONDS); + for (var future : futures) { + var current = future.get(15, java.util.concurrent.TimeUnit.SECONDS); + assertSame(first, current); + assertSame(first.emptyState(), current.emptyState()); + } + assertEquals(1, COLD_INSTALLATIONS.get()); + } finally { + executor.shutdownNow(); + assertTrue(executor.awaitTermination(10, java.util.concurrent.TimeUnit.SECONDS)); + } + } + @Test public void explicitInstallationIsSharedAndCannotReplaceAnAlreadyUsedLayout() { var installed = IndexedEntity.__TEAQL_FIELD_LAYOUT; From 96739f3c87b485abfe35df2cbc18b78161f6926c Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 7 Oct 2026 14:01:11 +0800 Subject: [PATCH 05/51] Count JDBC provider entry around generated sparse Checker rejection --- .../GeneratedSchoolLoadStateTest.java | 10 +++- .../ObservedJdbcExecutor.java | 51 +++++++++++++++++++ examples/shared-load-state/verify.sh | 1 + 3 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/ObservedJdbcExecutor.java diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java index 81f167dd..7421a409 100644 --- a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java @@ -68,7 +68,7 @@ void generatedSchoolUsesLocalRuntimeAndSharedState() throws Exception { String round = System.getenv("TEAQL_LOAD_STATE_ROUND"); SQLiteDataSource dataSource = new SQLiteDataSource(); dataSource.setUrl("jdbc:sqlite:" + System.getenv("TEAQL_LOAD_STATE_DATABASE")); - JdbcSqlExecutor sql = new JdbcSqlExecutor(dataSource); + ObservedJdbcExecutor sql = new ObservedJdbcExecutor(dataSource); IdSpaceIdGenerator ids = new IdSpaceIdGenerator(new DatabaseBridge(sql)); ids.ensureIdSpaceTable(); var service = new SqliteDataServiceExecutor("sqlite", sql, dataSource); @@ -192,8 +192,11 @@ void generatedSchoolUsesLocalRuntimeAndSharedState() throws Exception { assertNotSame(sparseState, sparse.get(0).__internalLoadState()); assertSame(sparseState, sparse.get(1).__internalLoadState()); assertFalse(sparse.get(1).isPropertyLoaded("address")); + long[] beforeRejectedSave = sql.counts(); assertThrows(CheckException.class, () -> sparse.get(0) .auditAs("Reject sparse whole-object update").save(context)); + assertArrayEquals(beforeRejectedSave, sql.counts(), + "Checker must reject before read, write or transaction provider entry"); var changed = full.get(0); Long originalVersion = changed.getVersion(); @@ -202,6 +205,11 @@ void generatedSchoolUsesLocalRuntimeAndSharedState() throws Exception { assertSame(snapshot, changed.__internalLoadState()); assertEquals(second, full.get(1).getName()); changed.auditAs("Rename only one independently loaded row").save(context); + long[] afterAcceptedSave = sql.counts(); + assertTrue(afterAcceptedSave[0] > beforeRejectedSave[0], "accepted save must exercise observed reads"); + assertTrue(afterAcceptedSave[1] > beforeRejectedSave[1], "accepted save must exercise observed writes"); + assertTrue(afterAcceptedSave[2] > beforeRejectedSave[2], "accepted save must exercise observed transactions"); + System.out.println("PASS generated Java sparse Checker rejects before provider entry; positive save observed"); assertEquals(originalVersion + 1, changed.getVersion()); var related = Q.schools().withIdIs(changed.getId()) .selectPlatformWith(Q.platformsWithMinimalFields().selectName()) diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/ObservedJdbcExecutor.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/ObservedJdbcExecutor.java new file mode 100644 index 00000000..afa5e5eb --- /dev/null +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/ObservedJdbcExecutor.java @@ -0,0 +1,51 @@ +package com.example.schoolmanagementservice; + +import io.teaql.core.CompiledRowMapper; +import io.teaql.core.Entity; +import io.teaql.dataservice.sql.SqlRowMapper; +import io.teaql.provider.jdbc.JdbcSqlExecutor; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLongArray; +import java.util.stream.Stream; +import javax.sql.DataSource; + +/** Test-only provider-entry counter. SQL, parameters and row values are never retained. */ +final class ObservedJdbcExecutor extends JdbcSqlExecutor { + private final AtomicLongArray calls = new AtomicLongArray(3); + ObservedJdbcExecutor(DataSource source) { super(source); } + /** Read, write and transaction entry; nested delegations may count more than once. */ + long[] counts() { return new long[]{calls.get(0), calls.get(1), calls.get(2)}; } + @Override public List query(String sql, Map params, SqlRowMapper mapper) { + calls.incrementAndGet(0); return super.query(sql, params, mapper); + } + @Override public List query(String sql, Object[] params, CompiledRowMapper mapper) { + calls.incrementAndGet(0); return super.query(sql, params, mapper); + } + @Override public Stream queryForStream(String sql, Map params, SqlRowMapper mapper) { + calls.incrementAndGet(0); return super.queryForStream(sql, params, mapper); + } + @Override public Stream> queryForStream(String sql, Object[] params) { + calls.incrementAndGet(0); return super.queryForStream(sql, params); + } + @Override public Stream queryForStream(String sql, Object[] params, CompiledRowMapper mapper) { + calls.incrementAndGet(0); return super.queryForStream(sql, params, mapper); + } + @Override public List> queryForList(String sql, Map params) { + calls.incrementAndGet(0); return super.queryForList(sql, params); + } + @Override public List> queryForList(String sql, Object[] params) { + calls.incrementAndGet(0); return super.queryForList(sql, params); + } + @Override public Map queryForMap(String sql, Map params) { + calls.incrementAndGet(0); return super.queryForMap(sql, params); + } + @Override public T queryForObject(String sql, Map params, Class type) { + calls.incrementAndGet(0); return super.queryForObject(sql, params, type); + } + @Override public void execute(String sql) { calls.incrementAndGet(1); super.execute(sql); } + @Override public int update(String sql, Map params) { calls.incrementAndGet(1); return super.update(sql, params); } + @Override public int update(String sql, Object[] params) { calls.incrementAndGet(1); return super.update(sql, params); } + @Override public int[] batchUpdate(String sql, List params) { calls.incrementAndGet(1); return super.batchUpdate(sql, params); } + @Override public void executeInTransaction(Runnable action) { calls.incrementAndGet(2); super.executeInTransaction(action); } +} diff --git a/examples/shared-load-state/verify.sh b/examples/shared-load-state/verify.sh index a6f9d0f7..71d7964d 100644 --- a/examples/shared-load-state/verify.sh +++ b/examples/shared-load-state/verify.sh @@ -69,6 +69,7 @@ for round in first second; do rg -F "PASS generated Java indexed Q/E/Checker/create/update/delete and snapshot sharing $round" "$run_dir/$round.log" rg -F "PASS generated Java typed native JSON roundtrip and snapshot sharing $round" "$run_dir/$round.log" rg -F "PASS generated Java typed JSON graph roundtrip and Empty/NotLoaded isolation" "$run_dir/$round.log" + rg -F "PASS generated Java sparse Checker rejects before provider entry" "$run_dir/$round.log" rg -F "PASS generated Java dynamic storage provenance and retry" "$run_dir/$round.log" rg -F "PASS generated Java namespace serialization and NotLoaded boundary" "$run_dir/$round.log" rg -F "PASS generated Java nested/reverse graph Q/E/JSON and Empty/NotLoaded isolation" "$run_dir/$round.log" From f51db62bf9835a5f99da9a65207b461e8cc89bce Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 7 Oct 2026 14:36:05 +0800 Subject: [PATCH 06/51] Verify generated Q selection and JDBC field-index order independence --- .../GeneratedFieldOrderAcceptance.java | 71 +++++++++++++++++++ .../GeneratedSchoolLoadStateTest.java | 1 + examples/shared-load-state/verify.sh | 1 + 3 files changed, 73 insertions(+) create mode 100644 examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedFieldOrderAcceptance.java diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedFieldOrderAcceptance.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedFieldOrderAcceptance.java new file mode 100644 index 00000000..8623c592 --- /dev/null +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedFieldOrderAcceptance.java @@ -0,0 +1,71 @@ +package com.example.schoolmanagementservice; + +import com.example.schoolmanagementservice.school.School; +import io.teaql.core.LoadState; +import io.teaql.core.SmartList; +import io.teaql.core.UserContext; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import javax.sql.DataSource; +import static org.junit.jupiter.api.Assertions.*; + +/** Q/E behavior plus trusted test-only driver ordinals, not an application SQL recipe. */ +final class GeneratedFieldOrderAcceptance { + static void verify(UserContext context, DataSource source, long id) throws Exception { + var left = Q.schoolsWithMinimalFields().withIdIs(id).selectName().selectAddress().limit(1) + .comment("what: select name then address").purpose("why: verify Q selection order cannot change fixed slots") + .executeForOne(context); + var right = Q.schoolsWithMinimalFields().withIdIs(id).selectAddress().selectName().limit(1) + .comment("what: select address then name").purpose("why: compare reversed Q selection order") + .executeForOne(context); + assertEquals(E.school(left).getName().eval(), E.school(right).getName().eval()); + assertEquals(E.school(left).getAddress().eval(), E.school(right).getAddress().eval()); + assertEquals(left.__internalLoadState(), right.__internalLoadState()); + var selected = new SmartList(List.of(left,right)); + assertSame(selected.get(0).__internalLoadState(), selected.get(1).__internalLoadState()); + assertFalse(left.isPropertyLoaded("active")); + assertFalse(left.__internalHasMutationLedger());assertFalse(right.__internalHasMutationLedger()); + + List permuted = new ArrayList<>(); + // Read only rows seeded through the generated audited mutation API. + try (var connection = source.getConnection()) { + for (String columns : List.of("name,address,id,version", "address,version,name,id")) { + try (var statement = connection.prepareStatement("SELECT " + columns + " FROM school_data WHERE id = ? LIMIT 1")) { + statement.setLong(1,id); + try (var rows = statement.executeQuery()) { + assertTrue(rows.next()); + var labels = new ArrayList(); + for (int ordinal = 1; ordinal <= rows.getMetaData().getColumnCount(); ordinal++) + labels.add(rows.getMetaData().getColumnLabel(ordinal)); + assertEquals(Arrays.asList(columns.split(",")), labels); + School entity = new School(); + entity.__internalUseLoadState(LoadState.projection(School.__TEAQL_FIELD_LAYOUT, labels)); + for (int ordinal = 1; ordinal <= labels.size(); ordinal++) { + String canonical = labels.get(ordinal-1); + Object value; + if (canonical.equals("id") || canonical.equals("version")) { + value = rows.getLong(ordinal);assertFalse(rows.wasNull()); + } else value = rows.getString(ordinal); + String member = School.__TEAQL_FIXED_FIELD_MAPPINGS.get(canonical).get(0); + entity.__internalHydrate(member, value, School.__TEAQL_FIXED_FIELD_INDEXES.get(canonical)); + } + assertFalse(rows.next()); + assertSame(School.__TEAQL_FIELD_LAYOUT, entity.__internalLoadState().layout()); + for (var field : School.__TEAQL_FIXED_FIELD_INDEXES.entrySet()) + assertEquals(field.getValue(), entity.__internalLoadState().layout().findIndex(field.getKey())); + assertEquals(left.getId(), entity.getId());assertEquals(left.getVersion(), entity.getVersion()); + assertEquals(E.school(left).getName().eval(), E.school(entity).getName().eval()); + assertEquals(E.school(left).getAddress().eval(), E.school(entity).getAddress().eval()); + assertFalse(entity.isPropertyLoaded("active")); + assertFalse(entity.__internalHasMutationLedger());assertTrue(entity.getUpdatedProperties().isEmpty()); + permuted.add(entity); + } + } + } + } + var joined = new SmartList(permuted); + assertSame(joined.get(0).__internalLoadState(), joined.get(1).__internalLoadState()); + System.out.println("PASS generated Java Q selection and real JDBC column-order invariance"); + } +} diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java index 7421a409..e63d94db 100644 --- a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java @@ -140,6 +140,7 @@ void generatedSchoolUsesLocalRuntimeAndSharedState() throws Exception { assertEquals(0, E.school(full.get(0)).getStudentCapacity().eval()); assertEquals(false, E.school(full.get(0)).isActive().eval()); assertEquals(1001L, full.get(0).getSchoolType().getId()); + GeneratedFieldOrderAcceptance.verify(context, dataSource, full.get(0).getId()); var sparse = Q.schoolsWithMinimalFields().withNameIn(first, second).selectName() .orderByIdAscending().limit(2).comment("what: read matching sparse projections") diff --git a/examples/shared-load-state/verify.sh b/examples/shared-load-state/verify.sh index 71d7964d..a3913da2 100644 --- a/examples/shared-load-state/verify.sh +++ b/examples/shared-load-state/verify.sh @@ -70,6 +70,7 @@ for round in first second; do rg -F "PASS generated Java typed native JSON roundtrip and snapshot sharing $round" "$run_dir/$round.log" rg -F "PASS generated Java typed JSON graph roundtrip and Empty/NotLoaded isolation" "$run_dir/$round.log" rg -F "PASS generated Java sparse Checker rejects before provider entry" "$run_dir/$round.log" + rg -F "PASS generated Java Q selection and real JDBC column-order invariance" "$run_dir/$round.log" rg -F "PASS generated Java dynamic storage provenance and retry" "$run_dir/$round.log" rg -F "PASS generated Java namespace serialization and NotLoaded boundary" "$run_dir/$round.log" rg -F "PASS generated Java nested/reverse graph Q/E/JSON and Empty/NotLoaded isolation" "$run_dir/$round.log" From 0026d752a7b2ed7395670434faa38c51dc95001b Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 7 Oct 2026 14:47:21 +0800 Subject: [PATCH 07/51] Verify generated page and cursor loaded-state sharing and null boundaries --- .../GeneratedPageStreamAcceptance.java | 55 +++++++++++++++++++ .../GeneratedSchoolLoadStateTest.java | 1 + examples/shared-load-state/verify.sh | 1 + 3 files changed, 57 insertions(+) create mode 100644 examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedPageStreamAcceptance.java diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedPageStreamAcceptance.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedPageStreamAcceptance.java new file mode 100644 index 00000000..2bdcb55f --- /dev/null +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedPageStreamAcceptance.java @@ -0,0 +1,55 @@ +package com.example.schoolmanagementservice; + +import com.example.schoolmanagementservice.school.School; +import io.teaql.core.UserContext; +import java.util.List; +import static org.junit.jupiter.api.Assertions.*; + +/** Generated APIs; retained rows must remain readable after the cursor closes. */ +final class GeneratedPageStreamAcceptance { + static void verify(UserContext context, String first, String second) { + var left = Q.schoolsWithMinimalFields().withNameIn(first,second).selectName().orderByIdAscending() + .comment("what: load the first matching School page").purpose("why: qualify sparse page hydration") + .executeForPage(context,0,1); + var right = Q.schoolsWithMinimalFields().withNameIn(first,second).selectName().orderByIdAscending() + .comment("what: load the second matching School page").purpose("why: qualify adjacent pages and filtered total") + .executeForPage(context,1,1); + assertEquals(2,left.getTotalCount());assertEquals(2,right.getTotalCount()); + assertEquals(1,left.size());assertEquals(1,right.size()); + assertNotEquals(left.first().getId(),right.first().getId()); + assertEquals(first,E.school(left.first()).getName().eval()); + assertEquals(second,E.school(right.first()).getName().eval()); + assertFalse(left.first().isPropertyLoaded("address")); + assertFalse(left.first().__internalHasMutationLedger()); + assertEquals(left.first().__internalLoadState(),right.first().__internalLoadState()); + for (boolean full : List.of(false,true)) { + List rows; + if (full) { + try (var stream = Q.schools().withNameIn(first,second).selectSelfFields().orderByIdAscending().limit(2) + .comment("what: stream two complete Schools").purpose("why: retain loaded NULL and overflow after cursor cleanup") + .executeForStream(context)) { rows = stream.toList(); } + } else { + try (var stream = Q.schoolsWithMinimalFields().withNameIn(first,second).selectName().orderByIdAscending().limit(2) + .comment("what: stream two sparse Schools").purpose("why: retain NotLoaded boundaries after cursor cleanup") + .executeForStream(context)) { rows = stream.toList(); } + } + assertEquals(2,rows.size()); + var state=rows.get(0).__internalLoadState(); + for (int i=0;ie.getValue()==slot).findFirst().orElseThrow().getKey(); + assertTrue(row.isPropertyLoaded(name)); + assertNull(row.__internalGet(School.__TEAQL_FIXED_FIELD_MAPPINGS.get(name).get(0))); + } + } + } + } + System.out.println("PASS generated Java page and stream shared load state"); + } +} diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java index e63d94db..874db74e 100644 --- a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java @@ -141,6 +141,7 @@ void generatedSchoolUsesLocalRuntimeAndSharedState() throws Exception { assertEquals(false, E.school(full.get(0)).isActive().eval()); assertEquals(1001L, full.get(0).getSchoolType().getId()); GeneratedFieldOrderAcceptance.verify(context, dataSource, full.get(0).getId()); + GeneratedPageStreamAcceptance.verify(context, first, second); var sparse = Q.schoolsWithMinimalFields().withNameIn(first, second).selectName() .orderByIdAscending().limit(2).comment("what: read matching sparse projections") diff --git a/examples/shared-load-state/verify.sh b/examples/shared-load-state/verify.sh index a3913da2..818f598b 100644 --- a/examples/shared-load-state/verify.sh +++ b/examples/shared-load-state/verify.sh @@ -71,6 +71,7 @@ for round in first second; do rg -F "PASS generated Java typed JSON graph roundtrip and Empty/NotLoaded isolation" "$run_dir/$round.log" rg -F "PASS generated Java sparse Checker rejects before provider entry" "$run_dir/$round.log" rg -F "PASS generated Java Q selection and real JDBC column-order invariance" "$run_dir/$round.log" + rg -F "PASS generated Java page and stream shared load state" "$run_dir/$round.log" rg -F "PASS generated Java dynamic storage provenance and retry" "$run_dir/$round.log" rg -F "PASS generated Java namespace serialization and NotLoaded boundary" "$run_dir/$round.log" rg -F "PASS generated Java nested/reverse graph Q/E/JSON and Empty/NotLoaded isolation" "$run_dir/$round.log" From 33b6fb350dc7ffed8b3cf8e3e57488eaea283a58 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 7 Oct 2026 14:56:40 +0800 Subject: [PATCH 08/51] Assert Checker rejection preserves absent fields and sibling projections --- .../GeneratedSchoolLoadStateTest.java | 6 ++++++ examples/shared-load-state/verify.sh | 1 + 2 files changed, 7 insertions(+) diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java index 874db74e..5b6d52cf 100644 --- a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java @@ -199,6 +199,12 @@ void generatedSchoolUsesLocalRuntimeAndSharedState() throws Exception { .auditAs("Reject sparse whole-object update").save(context)); assertArrayEquals(beforeRejectedSave, sql.counts(), "Checker must reject before read, write or transaction provider entry"); + assertFalse(sparse.get(0).isPropertyLoaded("active"), + "checker cannot promote an absent false default into loaded state"); + assertFalse(sparse.get(0).isPropertyLoaded("establishedDate")); + assertTrue(sparse.get(0).isPropertyLoaded("address"), "the explicit caller mutation must remain loaded"); + assertSame(sparseState, sparse.get(1).__internalLoadState(), "validation cannot widen a sibling projection"); + System.out.println("PASS generated Java Checker preserves NotLoaded and sibling load boundaries"); var changed = full.get(0); Long originalVersion = changed.getVersion(); diff --git a/examples/shared-load-state/verify.sh b/examples/shared-load-state/verify.sh index 818f598b..6ec54382 100644 --- a/examples/shared-load-state/verify.sh +++ b/examples/shared-load-state/verify.sh @@ -72,6 +72,7 @@ for round in first second; do rg -F "PASS generated Java sparse Checker rejects before provider entry" "$run_dir/$round.log" rg -F "PASS generated Java Q selection and real JDBC column-order invariance" "$run_dir/$round.log" rg -F "PASS generated Java page and stream shared load state" "$run_dir/$round.log" + rg -F "PASS generated Java Checker preserves NotLoaded and sibling load boundaries" "$run_dir/$round.log" rg -F "PASS generated Java dynamic storage provenance and retry" "$run_dir/$round.log" rg -F "PASS generated Java namespace serialization and NotLoaded boundary" "$run_dir/$round.log" rg -F "PASS generated Java nested/reverse graph Q/E/JSON and Empty/NotLoaded isolation" "$run_dir/$round.log" From 99c06e09692a8e388f054d42864f436cc5d42d5d Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 7 Oct 2026 15:36:02 +0800 Subject: [PATCH 09/51] fix(sql): bound concurrent compiled plan admission and verify load-state lifetime --- .../sql/portable/PortableSQLRepository.java | 18 +- .../CompiledPlanLoadStateLifecycleTest.java | 167 ++++++++++++++++++ 2 files changed, 180 insertions(+), 5 deletions(-) create mode 100644 teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/CompiledPlanLoadStateLifecycleTest.java diff --git a/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLRepository.java b/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLRepository.java index d202815a..03fe06cd 100644 --- a/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLRepository.java +++ b/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLRepository.java @@ -749,16 +749,12 @@ private SmartList loadWithIntent(UserContext userContext, SearchRequest re } psql = toPositional(sql, params); if (shape != null && psql.args.length == shape.arguments().length) { - if (compiledQueryPlans.size() >= MAX_COMPILED_QUERY_PLANS) { - compiledQueryPlans.clear(); - } CompiledQueryPlan candidate = new CompiledQueryPlan( psql.sql, psql.args.length, psql.logBindings, compileRowMapper(executedRequest)); - CompiledQueryPlan existing = compiledQueryPlans.putIfAbsent(shape.key(), candidate); - plan = existing == null ? candidate : existing; + plan = cacheCompiledQueryPlan(shape.key(), candidate); } } // Attach only after inserting the reusable plan: no original values enter the plan cache. @@ -879,6 +875,18 @@ private SmartList loadWithIntent(UserContext userContext, SearchRequest re return smartList; } + private CompiledQueryPlan cacheCompiledQueryPlan(String key, CompiledQueryPlan candidate) { + // Misses compile outside the lock; only admission/eviction is atomic. + // ConcurrentHashMap keeps the much more common hit path lock-free. + synchronized (compiledQueryPlans) { + CompiledQueryPlan existing = compiledQueryPlans.get(key); + if (existing != null) return existing; + if (compiledQueryPlans.size() >= MAX_COMPILED_QUERY_PLANS) compiledQueryPlans.clear(); + compiledQueryPlans.put(key, candidate); + return candidate; + } + } + private io.teaql.core.CompiledRowMapper compileRowMapper(SearchRequest request) { // Subtype discriminators and dynamic projections carry values that are not entity properties. // Keep those uncommon shapes on the generic mapper until their binding model is explicit. diff --git a/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/CompiledPlanLoadStateLifecycleTest.java b/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/CompiledPlanLoadStateLifecycleTest.java new file mode 100644 index 00000000..ce12ab01 --- /dev/null +++ b/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/CompiledPlanLoadStateLifecycleTest.java @@ -0,0 +1,167 @@ +package io.teaql.core.sql.portable; + +import io.teaql.core.*; +import io.teaql.core.meta.SimpleEntityMetaFactory; +import io.teaql.core.sql.GenericSQLProperty; +import io.teaql.core.sql.SQLEntityDescriptor; +import io.teaql.runtime.DefaultUserContext; +import io.teaql.runtime.TeaQLRuntime; +import java.lang.ref.WeakReference; +import java.lang.reflect.Proxy; +import java.util.*; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.Test; +import static org.junit.Assert.*; + +public class CompiledPlanLoadStateLifecycleTest { + public static class Row extends BaseEntity { + static { + Map slots = new LinkedHashMap<>(); + Map> mappings = new LinkedHashMap<>(); + slots.put("id", 0); slots.put("version", 1); + for (int i = 0; i < 10; i++) slots.put("p" + i, i + 2); + slots.keySet().forEach(name -> mappings.put(name, List.of(name, name))); + FieldLayout.installGenerated(FieldLayout.generated(Row.class, "cache-lifetime-v1", slots, mappings, Set.of())); + } + private final String[] payload = new String[10]; + @Override public String typeName() { return "CacheRow"; } + @Override public Object __internalGet(String name) { + if (name.startsWith("p")) return payload[Integer.parseInt(name.substring(1))]; + return super.__internalGet(name); + } + @Override public void __internalSet(String name, Object value) { + if (name.startsWith("p")) { payload[Integer.parseInt(name.substring(1))] = (String) value; return; } + super.__internalSet(name, value); + } + } + private static class Request extends BaseRequest { + Request(int mask) { + super(Row.class); + selectProperty("id"); selectProperty("version"); + for (int i = 0; i < 10; i++) if ((mask & (1 << i)) != 0) selectProperty("p" + i); + offset(0, 1); + internalComment("check compiled projection cache lifecycle"); + internalPurpose("retain immutable row states without retaining historic plans"); + } + @Override public String getTypeName() { return "CacheRow"; } + } + private static class Fixture { + volatile CyclicBarrier mapperBarrier; + final AtomicInteger compilations = new AtomicInteger(); + final PortableSQLRepository repository; + final TeaQLRuntime runtime; + Fixture() { + new Row(); // Install fixed metadata before the repository compiles its first shape. + SQLEntityDescriptor descriptor = new SQLEntityDescriptor() { + @Override public boolean hasChildren() { + // Test-only rendezvous at actual mapper construction, after the old + // implementation's capacity check. No production hook is added. + var barrier = mapperBarrier; + if (barrier != null && StackWalker.getInstance().walk(frames -> + frames.anyMatch(frame -> frame.getMethodName().equals("compileRowMapper")))) { + try { barrier.await(10, TimeUnit.SECONDS); } + catch (Exception error) { throw new AssertionError(error); } + } + return super.hasChildren(); + } + }; + descriptor.setType("CacheRow"); descriptor.setTargetType(Row.class); + descriptor.setEntitySupplier(Row::new); descriptor.setDataService("cache-test"); + descriptor.with("table_name", "cache_row_data"); descriptor.setAuditMaskFields(List.of()); + for (String name : List.of("id", "version", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9")) { + var property = (GenericSQLProperty) descriptor.addSimpleProperty(name, + name.equals("id") || name.equals("version") ? Long.class : String.class); + property.setColumnType(name.equals("id") || name.equals("version") ? "BIGINT" : "VARCHAR(40)"); + } + var metadata = new SimpleEntityMetaFactory(); metadata.register(descriptor); + runtime = TeaQLRuntime.builder().metadata(metadata).build(); + TeaQLDatabase database = (TeaQLDatabase) Proxy.newProxyInstance(TeaQLDatabase.class.getClassLoader(), + new Class[]{TeaQLDatabase.class}, (proxy, method, args) -> { + if (method.getName().equals("supportsCompiledRowMapping")) return true; + if (method.getName().equals("query") && args.length == 5 && args[3] instanceof CompiledRowMapper mapper) { + return List.of(mapper.map(new DataRow() { + @Override public Object get(int index) { return index < 3 ? 1L : "value"; } + @Override public V get(int index, Class type) { return type.cast(get(index)); } + })); + } + throw new AssertionError("unexpected provider call " + method); + }); + repository = new PortableSQLRepository<>(descriptor, database, null, metadata) { + @Override public String buildDataSQL(UserContext context, SearchRequest request, Map params) { + compilations.incrementAndGet(); return super.buildDataSQL(context, request, params); + } + }; + } + SmartList load(int mask) { return repository.loadInternal(new DefaultUserContext(runtime), new Request(mask)); } + Map plans() throws Exception { + var field = PortableSQLRepository.class.getDeclaredField("compiledQueryPlans"); + field.setAccessible(true); return (Map) field.get(repository); + } + } + + @Test public void concurrentMissesCannotExceedThePlanCapacity() throws Exception { + Fixture fixture = new Fixture(); + for (int mask = 1; mask <= 508; mask++) fixture.load(mask); + assertEquals(508, fixture.plans().size()); + fixture.mapperBarrier = new CyclicBarrier(8); + ExecutorService workers = Executors.newFixedThreadPool(8); + try { + List>> futures = new ArrayList<>(); + for (int mask = 509; mask <= 516; mask++) { + int selected = mask; futures.add(workers.submit(() -> fixture.load(selected))); + } + for (var future : futures) assertEquals(1, future.get(15, TimeUnit.SECONDS).size()); + } finally { + fixture.mapperBarrier = null; workers.shutdownNow(); + } + assertTrue("cache capacity must hold after simultaneous misses: " + fixture.plans().size(), fixture.plans().size() <= 512); + } + + @Test public void evictionKeepsLiveResultsButDoesNotRetainReleasedSnapshots() throws Exception { + Fixture fixture = new Fixture(); + WeakReference state = evictWhileResultAlive(fixture); + for (int attempt = 0; state.get() != null && attempt < 50; attempt++) { + System.gc(); Thread.sleep(20); + } + assertNull("evicted plans must not retain a released result's load geometry", state.get()); + } + + @Test public void concurrentSameShapeMissesReuseOneWinningImmutableState() throws Exception { + Fixture fixture = new Fixture(); + fixture.mapperBarrier = new CyclicBarrier(8); + ExecutorService workers = Executors.newFixedThreadPool(8); + try { + List>> futures = new ArrayList<>(); + for (int i = 0; i < 8; i++) futures.add(workers.submit(() -> fixture.load(3))); + LoadState shared = null; + for (var future : futures) { + var row = future.get(15, TimeUnit.SECONDS).get(0); + if (shared == null) shared = row.__internalLoadState(); + else assertSame(shared, row.__internalLoadState()); + assertTrue(row.isPropertyLoaded("p0")); assertTrue(row.isPropertyLoaded("p1")); + assertFalse(row.isPropertyLoaded("p2")); + } + } finally { + fixture.mapperBarrier = null; workers.shutdownNow(); + } + assertEquals(1, fixture.plans().size()); + } + + private static WeakReference evictWhileResultAlive(Fixture fixture) throws Exception { + SmartList live = fixture.load(1); + LoadState state = live.get(0).__internalLoadState(); + var oldPlan = fixture.plans().values().iterator().next(); + assertTrue(state.isLoaded("p0")); assertFalse(state.isLoaded("p1")); + assertTrue(state.overflow().isEmpty()); assertTrue(state.selectedNames().isEmpty()); + assertSame(state, fixture.load(1).get(0).__internalLoadState()); + assertEquals("identical request shape must hit the plan cache", 1, fixture.compilations.get()); + for (int mask = 2; mask <= 513; mask++) fixture.load(mask); + assertTrue(fixture.plans().size() <= 512); + assertFalse("the oldest plan must have been evicted", fixture.plans().containsValue(oldPlan)); + assertEquals("value", live.get(0).__internalGet("p0")); + assertSame(state, live.get(0).__internalLoadState()); + assertFalse(live.get(0).isPropertyLoaded("p1")); + return new WeakReference<>(state); + } +} From 584b7edca12d9318cb6783076bbfb3c60835fa45 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 7 Oct 2026 15:57:12 +0800 Subject: [PATCH 10/51] test(example): verify dynamic mixed-list readback rollback and retry --- .../GeneratedDynamicRollbackAcceptance.java | 93 +++++++++++++++++++ .../GeneratedSchoolLoadStateTest.java | 9 +- .../ObservedJdbcExecutor.java | 26 +++++- examples/shared-load-state/verify.sh | 1 + 4 files changed, 122 insertions(+), 7 deletions(-) create mode 100644 examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedDynamicRollbackAcceptance.java diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedDynamicRollbackAcceptance.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedDynamicRollbackAcceptance.java new file mode 100644 index 00000000..78a5dd64 --- /dev/null +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedDynamicRollbackAcceptance.java @@ -0,0 +1,93 @@ +package com.example.schoolmanagementservice; + +import com.example.schoolmanagementservice.school.School; +import io.teaql.core.SmartList; +import io.teaql.core.UserContext; +import io.teaql.data.dynamic.*; +import io.teaql.data.dynamic.jdbc.JdbcDynamicFieldsProvider; +import static org.junit.jupiter.api.Assertions.*; + +/** Real transaction/readback failure plus mixed-projection list lifetime. */ +final class GeneratedDynamicRollbackAcceptance { + static School verify(UserContext context, ObservedJdbcExecutor sql, School original, + String firstName, String secondName, String code, JdbcDynamicFieldsProvider provider, + DynamicFieldContext definitionContext) { + String unused = code + "_unused"; + DynamicFieldDef def = new DynamicFieldDef(); + def.setScope(DynamicFieldScope.global()); def.setOwnerType("School"); + def.setCode(unused); def.setName(unused); def.setDataType(DynamicDataType.STRING); + provider.registerFieldDef(definitionContext, def); + var selection = new DynamicFieldSelection().selectString(code).selectString(unused); + original.updateDynamicField(code, "matrix baseline"); + original.auditAs("seed the combined-state value control").save(context); + var companionRequest = Q.schools().withNameIn(secondName); + companionRequest.selectDynamicFieldsWith(selection); + var companion = companionRequest.limit(1).comment("what: select companion extension metadata") + .purpose("why: seed an explicit NULL through audited mutation").executeForOne(context); + companion.updateDynamicField(code, null); + companion.auditAs("seed the combined-state null control").save(context); + var all = Q.schools().withNameIn(firstName, secondName); + all.selectDynamicFieldsWith(selection); + var rows = all.orderByIdAscending().limit(2).comment("what: load value and null extensions together") + .purpose("why: combine shared geometry with private row payloads").executeForList(context); + assertEquals(2, rows.size()); + var state = rows.get(0).__internalLoadState(); + assertSame(state, rows.get(1).__internalLoadState()); + if (Boolean.parseBoolean(System.getenv("TEAQL_LOAD_STATE_WIDE"))) assertFalse(state.overflow().isEmpty()); + assertEquals(DynamicFieldValue.State.VALUE, rows.get(0).dynamicFields().field(code).state()); + assertEquals(DynamicFieldValue.State.NULL, rows.get(1).dynamicFields().field(code).state()); + for (var row : rows.getData()) assertEquals(DynamicFieldValue.State.NOT_LOADED, row.dynamicFields().field(unused).state()); + var sparseRequest = Q.schoolsWithMinimalFields().withNameIn(firstName).selectName(); + sparseRequest.selectDynamicFieldsWith(selection); + var sparse = sparseRequest.limit(1).comment("what: append the same identity with a sparse projection") + .purpose("why: mixed lists must not union native loaded fields").executeForOne(context); + var sparseState = sparse.__internalLoadState(); + SmartList held = new SmartList<>(); + held.add(rows.get(0)); held.add(rows.get(1)); held.add(sparse); + rows = null; + assertFalse(held.get(2).isPropertyLoaded("address")); + assertSame(sparseState, held.get(2).__internalLoadState()); + assertSame(state, held.get(0).__internalLoadState()); + assertSame(state, held.get(1).__internalLoadState()); + School pending = held.get(0); + Long version = pending.getVersion(), companionVersion = held.get(1).getVersion(); + String nextName = firstName + " retry"; + pending.updateName(nextName); pending.updateDynamicField(code, "matrix retry"); + assertSame(state, pending.__internalLoadState()); + long[] before = sql.counts(); + sql.failNextDynamicReadbackAfterWrite(); + try { assertThrows(RuntimeException.class, () -> pending.auditAs("rollback a failed extension readback").save(context)); } + finally { sql.clearReadbackFailure(); } + assertTrue(sql.readbackFailureObserved(), "the fault must happen after actual extension DML"); + assertTrue(sql.counts()[1] > before[1]); assertTrue(sql.counts()[2] > before[2]); + assertEquals(version, pending.getVersion()); + assertFalse(pending.__internalDynamicMutations().isEmpty()); + assertSame(state, pending.__internalLoadState()); + assertEquals(companionVersion, held.get(1).getVersion()); + assertTrue(held.get(1).getUpdatedProperties().isEmpty()); + assertEquals(DynamicFieldValue.State.NULL, held.get(1).dynamicFields().field(code).state()); + assertEquals("matrix baseline", held.get(2).dynamicFields().field(code).value()); + assertFalse(held.get(2).isPropertyLoaded("address")); + var inspect = Q.schools().withNameIn(firstName, secondName); + inspect.selectDynamicFieldsWith(selection); + var stored = inspect.orderByIdAscending().limit(2).comment("what: inspect both stores after readback failure") + .purpose("why: failure must roll back native and extension DML atomically").executeForList(context); + assertEquals(2, stored.size()); assertEquals(version, stored.get(0).getVersion()); + assertEquals(firstName, E.school(stored.get(0)).getName().eval()); + assertEquals("matrix baseline", stored.get(0).dynamicFields().field(code).value()); + assertEquals(companionVersion, stored.get(1).getVersion()); + assertEquals(DynamicFieldValue.State.NULL, stored.get(1).dynamicFields().field(code).state()); + pending.auditAs("retry the original native and extension intent").save(context); + assertEquals(version + 1, pending.getVersion()); + assertEquals(nextName, E.school(pending).getName().eval()); + assertEquals("matrix retry", pending.dynamicFields().field(code).value()); + assertEquals(DynamicFieldValue.State.NOT_LOADED, pending.dynamicFields().field(unused).state()); + assertSame(state, pending.__internalLoadState()); + assertTrue(pending.__internalDynamicMutations().isEmpty()); + assertTrue(pending.getUpdatedProperties().isEmpty()); + assertEquals(companionVersion, held.get(1).getVersion()); + assertSame(state, held.get(1).__internalLoadState()); + System.out.println("PASS generated Java mixed dynamic Value/Null/NotLoaded list lifetime readback rollback and retry"); + return pending; + } +} diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java index 5b6d52cf..62f350b7 100644 --- a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java @@ -296,20 +296,23 @@ void generatedSchoolUsesLocalRuntimeAndSharedState() throws Exception { assertEquals(DynamicFieldValue.State.NOT_LOADED, extended.dynamicFields().field(extension).state()); assertEquals("readonly-derived-name", extended.getProperty("_name")); System.out.println("PASS generated Java dynamic storage provenance and retry"); + var combined = GeneratedDynamicRollbackAcceptance.verify(context, sql, extended, renamed, second, extension, provider, definitionContext); + String deletionName = combined.getName(); // Partial related objects are valid for E reads, not for whole-graph Checker validation. var forDeletion = Q.schools().withIdIs(changed.getId()).limit(1) .comment("what: reload the full native School before deletion") .purpose("why: preserve whole-object Checker validation without partial child projections") .executeForOne(context); - assertEquals(renamed, E.school(forDeletion).getName().eval()); + assertEquals(deletionName, E.school(forDeletion).getName().eval()); assertNull(forDeletion.getProperty("_name"), "readonly dynamic properties must not be persisted by save"); forDeletion.markForDeletion().auditAs("Soft-delete the selected probe").save(context); - var remaining = Q.schools().withNameIn(renamed, second).orderByIdAscending().limit(2) + var remaining = Q.schools().withNameIn(deletionName, second).orderByIdAscending().limit(2) .comment("what: inspect the surviving sibling").purpose("why: verify mutation isolation") .executeForList(context); assertEquals(1, remaining.size()); assertEquals(second, remaining.get(0).getName()); - assertEquals(full.get(1).getVersion(), remaining.get(0).getVersion()); + // The combined control explicitly seeds the companion's NULL extension once. + assertEquals(full.get(1).getVersion() + 1, remaining.get(0).getVersion()); System.out.println("PASS generated Java indexed Q/E/Checker/create/update/delete and snapshot sharing " + round); } diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/ObservedJdbcExecutor.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/ObservedJdbcExecutor.java index afa5e5eb..0afc8b53 100644 --- a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/ObservedJdbcExecutor.java +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/ObservedJdbcExecutor.java @@ -7,15 +7,33 @@ import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicLongArray; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Stream; import javax.sql.DataSource; /** Test-only provider-entry counter. SQL, parameters and row values are never retained. */ final class ObservedJdbcExecutor extends JdbcSqlExecutor { private final AtomicLongArray calls = new AtomicLongArray(3); + private final AtomicBoolean failDynamicReadback = new AtomicBoolean(); + private final AtomicBoolean dynamicWriteSeen = new AtomicBoolean(); + private final AtomicBoolean readbackFailureSeen = new AtomicBoolean(); ObservedJdbcExecutor(DataSource source) { super(source); } /** Read, write and transaction entry; nested delegations may count more than once. */ long[] counts() { return new long[]{calls.get(0), calls.get(1), calls.get(2)}; } + void failNextDynamicReadbackAfterWrite() { + dynamicWriteSeen.set(false); readbackFailureSeen.set(false); failDynamicReadback.set(true); + } + boolean readbackFailureObserved() { return readbackFailureSeen.get(); } + void clearReadbackFailure() { failDynamicReadback.set(false); } + private void observeDynamicWrite(String sql, int changed) { + if (changed > 0 && failDynamicReadback.get() && sql.contains("teaql_dynamic_field_value")) dynamicWriteSeen.set(true); + } + private void rejectDynamicReadback(String sql) { + if (dynamicWriteSeen.get() && sql.contains("teaql_dynamic_field_value") && failDynamicReadback.compareAndSet(true, false)) { + readbackFailureSeen.set(true); + throw new IllegalStateException("CONTROLLED_DYNAMIC_READBACK_FAILURE"); + } + } @Override public List query(String sql, Map params, SqlRowMapper mapper) { calls.incrementAndGet(0); return super.query(sql, params, mapper); } @@ -32,10 +50,10 @@ final class ObservedJdbcExecutor extends JdbcSqlExecutor { calls.incrementAndGet(0); return super.queryForStream(sql, params, mapper); } @Override public List> queryForList(String sql, Map params) { - calls.incrementAndGet(0); return super.queryForList(sql, params); + calls.incrementAndGet(0); rejectDynamicReadback(sql); return super.queryForList(sql, params); } @Override public List> queryForList(String sql, Object[] params) { - calls.incrementAndGet(0); return super.queryForList(sql, params); + calls.incrementAndGet(0); rejectDynamicReadback(sql); return super.queryForList(sql, params); } @Override public Map queryForMap(String sql, Map params) { calls.incrementAndGet(0); return super.queryForMap(sql, params); @@ -44,8 +62,8 @@ final class ObservedJdbcExecutor extends JdbcSqlExecutor { calls.incrementAndGet(0); return super.queryForObject(sql, params, type); } @Override public void execute(String sql) { calls.incrementAndGet(1); super.execute(sql); } - @Override public int update(String sql, Map params) { calls.incrementAndGet(1); return super.update(sql, params); } - @Override public int update(String sql, Object[] params) { calls.incrementAndGet(1); return super.update(sql, params); } + @Override public int update(String sql, Map params) { calls.incrementAndGet(1); int changed = super.update(sql, params); observeDynamicWrite(sql, changed); return changed; } + @Override public int update(String sql, Object[] params) { calls.incrementAndGet(1); int changed = super.update(sql, params); observeDynamicWrite(sql, changed); return changed; } @Override public int[] batchUpdate(String sql, List params) { calls.incrementAndGet(1); return super.batchUpdate(sql, params); } @Override public void executeInTransaction(Runnable action) { calls.incrementAndGet(2); super.executeInTransaction(action); } } diff --git a/examples/shared-load-state/verify.sh b/examples/shared-load-state/verify.sh index 6ec54382..20589c4e 100644 --- a/examples/shared-load-state/verify.sh +++ b/examples/shared-load-state/verify.sh @@ -74,6 +74,7 @@ for round in first second; do rg -F "PASS generated Java page and stream shared load state" "$run_dir/$round.log" rg -F "PASS generated Java Checker preserves NotLoaded and sibling load boundaries" "$run_dir/$round.log" rg -F "PASS generated Java dynamic storage provenance and retry" "$run_dir/$round.log" + rg -F "PASS generated Java mixed dynamic Value/Null/NotLoaded list lifetime readback rollback and retry" "$run_dir/$round.log" rg -F "PASS generated Java namespace serialization and NotLoaded boundary" "$run_dir/$round.log" rg -F "PASS generated Java nested/reverse graph Q/E/JSON and Empty/NotLoaded isolation" "$run_dir/$round.log" rg -F "PASS generated Java nested dynamic Value/NULL/NotLoaded and shared snapshots" "$run_dir/$round.log" From 43e6f6deba211316b55aa495940892e4e24d18c0 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 7 Oct 2026 16:10:02 +0800 Subject: [PATCH 11/51] test(example): keep stored unselected extensions and derived properties readonly --- .../GeneratedDynamicRollbackAcceptance.java | 28 ++++++++++++++++--- examples/shared-load-state/verify.sh | 1 + 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedDynamicRollbackAcceptance.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedDynamicRollbackAcceptance.java index 78a5dd64..252d87b2 100644 --- a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedDynamicRollbackAcceptance.java +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedDynamicRollbackAcceptance.java @@ -17,9 +17,15 @@ static School verify(UserContext context, ObservedJdbcExecutor sql, School origi def.setScope(DynamicFieldScope.global()); def.setOwnerType("School"); def.setCode(unused); def.setName(unused); def.setDataType(DynamicDataType.STRING); provider.registerFieldDef(definitionContext, def); - var selection = new DynamicFieldSelection().selectString(code).selectString(unused); - original.updateDynamicField(code, "matrix baseline"); - original.auditAs("seed the combined-state value control").save(context); + var allFields = new DynamicFieldSelection().selectString(code).selectString(unused); + var seedRequest = Q.schools().withIdIs(original.getId()); + seedRequest.selectDynamicFieldsWith(allFields); + var seed = seedRequest.limit(1).comment("what: load metadata for the hidden stored extension") + .purpose("why: seed both fields through governed mutation").executeForOne(context); + seed.updateDynamicField(code, "matrix baseline"); + seed.updateDynamicField(unused, "keep unselected"); + seed.auditAs("seed the combined-state value and hidden controls").save(context); + var selection = new DynamicFieldSelection().selectString(code); var companionRequest = Q.schools().withNameIn(secondName); companionRequest.selectDynamicFieldsWith(selection); var companion = companionRequest.limit(1).comment("what: select companion extension metadata") @@ -50,9 +56,13 @@ static School verify(UserContext context, ObservedJdbcExecutor sql, School origi assertSame(state, held.get(0).__internalLoadState()); assertSame(state, held.get(1).__internalLoadState()); School pending = held.get(0); + pending.addDynamicProperty("matrix_total", 17); + assertEquals(17, (int) pending.getProperty("_matrix_total")); Long version = pending.getVersion(), companionVersion = held.get(1).getVersion(); String nextName = firstName + " retry"; pending.updateName(nextName); pending.updateDynamicField(code, "matrix retry"); + assertFalse(pending.getUpdatedProperties().contains("_matrix_total")); + assertFalse(pending.__internalDynamicMutations().containsKey(unused)); assertSame(state, pending.__internalLoadState()); long[] before = sql.counts(); sql.failNextDynamicReadbackAfterWrite(); @@ -61,6 +71,7 @@ static School verify(UserContext context, ObservedJdbcExecutor sql, School origi assertTrue(sql.readbackFailureObserved(), "the fault must happen after actual extension DML"); assertTrue(sql.counts()[1] > before[1]); assertTrue(sql.counts()[2] > before[2]); assertEquals(version, pending.getVersion()); + assertEquals(17, (int) pending.getProperty("_matrix_total")); assertFalse(pending.__internalDynamicMutations().isEmpty()); assertSame(state, pending.__internalLoadState()); assertEquals(companionVersion, held.get(1).getVersion()); @@ -69,13 +80,15 @@ static School verify(UserContext context, ObservedJdbcExecutor sql, School origi assertEquals("matrix baseline", held.get(2).dynamicFields().field(code).value()); assertFalse(held.get(2).isPropertyLoaded("address")); var inspect = Q.schools().withNameIn(firstName, secondName); - inspect.selectDynamicFieldsWith(selection); + inspect.selectDynamicFieldsWith(allFields); var stored = inspect.orderByIdAscending().limit(2).comment("what: inspect both stores after readback failure") .purpose("why: failure must roll back native and extension DML atomically").executeForList(context); assertEquals(2, stored.size()); assertEquals(version, stored.get(0).getVersion()); assertEquals(firstName, E.school(stored.get(0)).getName().eval()); assertEquals("matrix baseline", stored.get(0).dynamicFields().field(code).value()); assertEquals(companionVersion, stored.get(1).getVersion()); + assertEquals("keep unselected", stored.get(0).dynamicFields().field(unused).value()); + assertNull(stored.get(0).getProperty("_matrix_total")); assertEquals(DynamicFieldValue.State.NULL, stored.get(1).dynamicFields().field(code).state()); pending.auditAs("retry the original native and extension intent").save(context); assertEquals(version + 1, pending.getVersion()); @@ -85,9 +98,16 @@ static School verify(UserContext context, ObservedJdbcExecutor sql, School origi assertSame(state, pending.__internalLoadState()); assertTrue(pending.__internalDynamicMutations().isEmpty()); assertTrue(pending.getUpdatedProperties().isEmpty()); + var verifyHidden = Q.schools().withIdIs(pending.getId()); + verifyHidden.selectDynamicFieldsWith(allFields); + var inspected = verifyHidden.limit(1).comment("what: inspect the hidden extension after retry") + .purpose("why: NotLoaded must not become erase intent").executeForOne(context); + assertEquals("keep unselected", inspected.dynamicFields().field(unused).value()); + assertNull(inspected.getProperty("_matrix_total")); assertEquals(companionVersion, held.get(1).getVersion()); assertSame(state, held.get(1).__internalLoadState()); System.out.println("PASS generated Java mixed dynamic Value/Null/NotLoaded list lifetime readback rollback and retry"); + System.out.println("PASS generated Java stored unselected extension survives rollback retry and readonly property is not persisted"); return pending; } } diff --git a/examples/shared-load-state/verify.sh b/examples/shared-load-state/verify.sh index 20589c4e..70b171b6 100644 --- a/examples/shared-load-state/verify.sh +++ b/examples/shared-load-state/verify.sh @@ -75,6 +75,7 @@ for round in first second; do rg -F "PASS generated Java Checker preserves NotLoaded and sibling load boundaries" "$run_dir/$round.log" rg -F "PASS generated Java dynamic storage provenance and retry" "$run_dir/$round.log" rg -F "PASS generated Java mixed dynamic Value/Null/NotLoaded list lifetime readback rollback and retry" "$run_dir/$round.log" + rg -F "PASS generated Java stored unselected extension survives rollback retry and readonly property is not persisted" "$run_dir/$round.log" rg -F "PASS generated Java namespace serialization and NotLoaded boundary" "$run_dir/$round.log" rg -F "PASS generated Java nested/reverse graph Q/E/JSON and Empty/NotLoaded isolation" "$run_dir/$round.log" rg -F "PASS generated Java nested dynamic Value/NULL/NotLoaded and shared snapshots" "$run_dir/$round.log" From 8127a0d9486a4eb9e046e917b494be54dce947d2 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 7 Oct 2026 16:14:01 +0800 Subject: [PATCH 12/51] test(benchmark): record actual JDBC driver and SQLite engine versions --- .../schoolmanagementservice/DriverQueryBenchmarkTest.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/DriverQueryBenchmarkTest.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/DriverQueryBenchmarkTest.java index ec0c81a7..a45ce2ff 100644 --- a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/DriverQueryBenchmarkTest.java +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/DriverQueryBenchmarkTest.java @@ -89,6 +89,10 @@ static void report(String lane, boolean full, int count, List Date: Wed, 7 Oct 2026 17:01:05 +0800 Subject: [PATCH 13/51] feat(portable): hydrate selected dynamic fields in bounded root streams --- .../GeneratedDynamicRollbackAcceptance.java | 18 ++++++ examples/shared-load-state/verify.sh | 1 + .../core/sql/portable/BatchedRootStream.java | 51 +++++++++++++++++ .../sql/portable/PortableSQLDataService.java | 27 +++++++-- .../sql/portable/BatchedRootStreamTest.java | 57 +++++++++++++++++++ 5 files changed, 148 insertions(+), 6 deletions(-) create mode 100644 teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/BatchedRootStream.java create mode 100644 teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/BatchedRootStreamTest.java diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedDynamicRollbackAcceptance.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedDynamicRollbackAcceptance.java index 252d87b2..edd4cae6 100644 --- a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedDynamicRollbackAcceptance.java +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedDynamicRollbackAcceptance.java @@ -32,6 +32,24 @@ static School verify(UserContext context, ObservedJdbcExecutor sql, School origi .purpose("why: seed an explicit NULL through audited mutation").executeForOne(context); companion.updateDynamicField(code, null); companion.auditAs("seed the combined-state null control").save(context); + var streamRequest = Q.schoolsWithMinimalFields().withNameIn(firstName, secondName).selectName(); + streamRequest.selectDynamicFieldsWith(selection); + java.util.List streamed; + try (var stream = streamRequest.orderByIdAscending().limit(2) + .comment("what: stream sparse Schools with selected persistent extensions") + .purpose("why: retain Value NULL and NotLoaded after cursor cleanup").executeForStream(context)) { + streamed = stream.toList(); + } + assertEquals(2, streamed.size()); + assertSame(streamed.get(0).__internalLoadState(), streamed.get(1).__internalLoadState()); + assertEquals("matrix baseline", streamed.get(0).dynamicFields().field(code).value()); + assertEquals(DynamicFieldValue.State.NULL, streamed.get(1).dynamicFields().field(code).state()); + for (var row : streamed) { + assertFalse(row.isPropertyLoaded("address")); + assertEquals(DynamicFieldValue.State.NOT_LOADED, row.dynamicFields().field(unused).state()); + assertFalse(row.__internalHasMutationLedger()); + } + System.out.println("PASS generated Java dynamic stream Value/Null/NotLoaded and shared snapshots"); var all = Q.schools().withNameIn(firstName, secondName); all.selectDynamicFieldsWith(selection); var rows = all.orderByIdAscending().limit(2).comment("what: load value and null extensions together") diff --git a/examples/shared-load-state/verify.sh b/examples/shared-load-state/verify.sh index 70b171b6..2fc7a9ba 100644 --- a/examples/shared-load-state/verify.sh +++ b/examples/shared-load-state/verify.sh @@ -72,6 +72,7 @@ for round in first second; do rg -F "PASS generated Java sparse Checker rejects before provider entry" "$run_dir/$round.log" rg -F "PASS generated Java Q selection and real JDBC column-order invariance" "$run_dir/$round.log" rg -F "PASS generated Java page and stream shared load state" "$run_dir/$round.log" + rg -F "PASS generated Java dynamic stream Value/Null/NotLoaded and shared snapshots" "$run_dir/$round.log" rg -F "PASS generated Java Checker preserves NotLoaded and sibling load boundaries" "$run_dir/$round.log" rg -F "PASS generated Java dynamic storage provenance and retry" "$run_dir/$round.log" rg -F "PASS generated Java mixed dynamic Value/Null/NotLoaded list lifetime readback rollback and retry" "$run_dir/$round.log" diff --git a/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/BatchedRootStream.java b/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/BatchedRootStream.java new file mode 100644 index 00000000..3e84e2c3 --- /dev/null +++ b/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/BatchedRootStream.java @@ -0,0 +1,51 @@ +package io.teaql.core.sql.portable; + +import io.teaql.core.Entity; +import io.teaql.core.SmartList; +import java.util.Iterator; +import java.util.Spliterator; +import java.util.Spliterators; +import java.util.function.Consumer; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +/** Sequential, bounded enhancement. No batch is delivered before its validation succeeds. */ +final class BatchedRootStream { + private BatchedRootStream() {} + + static Stream enhance( + Stream source, int batchSize, Consumer> hydrate) { + if (batchSize < 1) throw new IllegalArgumentException("batch size must be positive"); + var cursor = new Spliterators.AbstractSpliterator(Long.MAX_VALUE, Spliterator.ORDERED) { + private Iterator input; + private Iterator ready = java.util.Collections.emptyIterator(); + private boolean closed; + + private void close() { + if (!closed) { closed = true; ready = java.util.Collections.emptyIterator(); source.close(); } + } + + @Override public Spliterator trySplit() { return null; } + + @Override public boolean tryAdvance(Consumer action) { + if (closed) return false; + try { + if (!ready.hasNext()) { + if (input == null) input = source.iterator(); + var batch = new SmartList(batchSize); + while (batch.size() < batchSize && input.hasNext()) batch.add(input.next()); + if (batch.isEmpty()) { close(); return false; } + hydrate.accept(batch); + ready = batch.iterator(); + } + action.accept(ready.next()); + return true; + } catch (RuntimeException | Error error) { + try { close(); } catch (RuntimeException | Error cleanup) { error.addSuppressed(cleanup); } + throw error; + } + } + }; + return StreamSupport.stream(cursor, false).onClose(cursor::close); + } +} diff --git a/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLDataService.java b/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLDataService.java index a5d21970..58305685 100644 --- a/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLDataService.java +++ b/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLDataService.java @@ -100,13 +100,24 @@ public QueryResult query(UserContext context, QueryRequest request) { } private void attachDynamicFields(UserContext context, SmartList results, SearchRequest request) { + attachDynamicFields(context, results, request, new java.util.HashMap<>()); + } + + private void attachDynamicFields(UserContext context, SmartList results, SearchRequest request, + Map, LoadState>> shapes) { var selection = request.getDynamicFieldSelection(); if (selection == null || results.isEmpty()) return; var owners = new java.util.ArrayList(results.size()); for (Entity entity : results) owners.add(io.teaql.data.dynamic.DynamicOwnerRef.of(entity.typeName(), entity.getId())); var batch = context.dynamicFields().purpose(request.purpose()).comment(request.comment()).readAll(owners, selection); + // Validate every owner/carrier before changing any entity in the batch. + for (var owner : owners) { + if (batch.get(owner) == null) throw new TeaQLRuntimeException("Dynamic field batch omitted owner " + owner); + } + for (Entity entity : results) { + if (!(entity instanceof BaseEntity)) throw new TeaQLRuntimeException("Dynamic fields require a runtime-owned entity carrier"); + } // One named-selection set per actual loaded shape, not one copy per row. - var shapes = new java.util.HashMap, io.teaql.core.LoadState>>(); int index = 0; for (Entity entity : results) { var owner = owners.get(index++); @@ -137,12 +148,16 @@ public QueryCursor queryForCursor(UserContext context, Que if (source == null) source = new SqlIntentRedactions(); var statements = new java.util.concurrent.CopyOnWriteArrayList(); SearchRequest scoped = SqlDiagnosticRequest.collecting(searchRequest, source, request.intent(), statements::add); - if (scoped.hasSimpleAgg() || !scoped.enhanceRelations().isEmpty() || !scoped.enhanceChildren().isEmpty() - || scoped.getDynamicFieldSelection() != null) { - throw new TeaQLRuntimeException("Streaming aggregation/relation/dynamic-field enhancement is not supported; stream root rows only"); + if (scoped.hasSimpleAgg() || !scoped.enhanceRelations().isEmpty() || !scoped.enhanceChildren().isEmpty()) { + throw new TeaQLRuntimeException("Streaming aggregation/relation enhancement is not supported; stream root rows only"); + } + var stream = this.getRepository(scoped.getTypeName()).streamInternal(context, scoped); + if (scoped.getDynamicFieldSelection() != null) { + var shapes = new HashMap, LoadState>>(); + stream = BatchedRootStream.enhance(stream, 128, + batch -> attachDynamicFields(context, batch, scoped, shapes)); } - return new QueryCursor<>(this.getRepository(scoped.getTypeName()).streamInternal(context, scoped), - () -> statements); + return new QueryCursor<>(stream, () -> statements); } private void attachDynamicAggregations( diff --git a/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/BatchedRootStreamTest.java b/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/BatchedRootStreamTest.java new file mode 100644 index 00000000..f77dba70 --- /dev/null +++ b/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/BatchedRootStreamTest.java @@ -0,0 +1,57 @@ +package io.teaql.core.sql.portable; + +import io.teaql.core.BaseEntity; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.IntStream; +import java.util.stream.Stream; +import org.junit.Test; +import static org.junit.Assert.*; + +public class BatchedRootStreamTest { + @Test public void boundsReadAheadAndClosesEarlyWithoutLoadingTheNextBatch() { + var reads = new AtomicInteger(); var batches = new AtomicInteger(); var closes = new AtomicInteger(); + var source = IntStream.range(0, 10).mapToObj(i -> { reads.incrementAndGet(); return new BaseEntity(); }) + .onClose(closes::incrementAndGet); + try (var stream = BatchedRootStream.enhance(source, 3, rows -> { + assertEquals(3, rows.size()); batches.incrementAndGet(); + })) { + assertEquals(0, reads.get()); + assertEquals(1, stream.limit(1).count()); + assertEquals(3, reads.get()); assertEquals(1, batches.get()); + } + assertEquals(1, closes.get()); + } + + @Test public void emptyStreamDoesNotCallHydrationAndCloses() { + var closes = new AtomicInteger(); + try (var stream = BatchedRootStream.enhance(Stream.empty().onClose(closes::incrementAndGet), + 3, rows -> fail("empty streams must not access dynamic provider"))) { + assertEquals(0, stream.count()); + } + assertEquals(1, closes.get()); + } + + @Test public void failedBatchDeliversNothingClosesAndCannotResume() { + var delivered = new AtomicInteger(); var closes = new AtomicInteger(); + var failure = new IllegalStateException("controlled invalid batch"); + try (var stream = BatchedRootStream.enhance(Stream.of(new BaseEntity(), new BaseEntity()) + .onClose(closes::incrementAndGet), 2, rows -> { throw failure; })) { + var cursor = stream.spliterator(); + try { cursor.tryAdvance(row -> delivered.incrementAndGet()); fail("must reject"); } + catch (IllegalStateException error) { assertSame(failure, error); } + assertEquals(0, delivered.get()); assertEquals(1, closes.get()); + assertFalse(cursor.tryAdvance(row -> delivered.incrementAndGet())); + } + assertEquals(1, closes.get()); + } + + @Test public void preservesOrderingAndLoadsTheTailOnce() { + var batches = new java.util.ArrayList(); + var entities = List.of(new BaseEntity(), new BaseEntity(), new BaseEntity()); + try (var stream = BatchedRootStream.enhance(entities.stream(), 2, rows -> batches.add(rows.size()))) { + assertEquals(entities, stream.toList()); + } + assertEquals(List.of(2, 1), batches); + } +} From a50d87c33f6fccb97f9f419b5264835c00ecf934 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 7 Oct 2026 17:08:59 +0800 Subject: [PATCH 14/51] test(sqlite): qualify dynamic root streams after enhancement support --- .../teaql/sqlite/SqliteIntegrationTest.java | 29 +++++++++++++++++-- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteIntegrationTest.java b/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteIntegrationTest.java index 6517edd5..124172e5 100644 --- a/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteIntegrationTest.java +++ b/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteIntegrationTest.java @@ -522,11 +522,34 @@ class ObservedProvider extends io.teaql.data.dynamic.InMemoryDynamicFieldsProvid TaskRequest streamed = new TaskRequest().filterByStatus("DYNAMIC-LOAD-SHAPE"); streamed.setSize(3); + streamed.addOrderBy("id", true); streamed.selectDynamicFieldsWith(new io.teaql.data.dynamic.DynamicFieldSelection().selectString("title")); - streamed.comment("reject unsupported streaming extension load").purpose("do not silently drop projections"); + streamed.comment("stream persistent extension availability").purpose("preserve sparse wrappers after cursor cleanup"); var streaming = (StreamingQueryExecutor) runtime.getRegistry().resolve("sqlite"); - assertThrows(TeaQLRuntimeException.class, () -> streaming.queryForCursor(local, new DefaultQueryRequest(streamed))); - assertEquals(1, provider.batches); + List streamedRows; + try (var cursor = streaming.queryForCursor(local, new DefaultQueryRequest(streamed))) { + assertEquals("cursor creation must not hydrate extensions", 1, provider.batches); + streamedRows = cursor.stream().toList(); + } + assertEquals(3, streamedRows.size()); + assertEquals(2, provider.batches); + assertEquals("stream persistent extension availability", provider.lastContext.comment()); + assertEquals("preserve sparse wrappers after cursor cleanup", provider.lastContext.purpose()); + assertSame(streamedRows.get(0).__internalLoadState(), streamedRows.get(1).__internalLoadState()); + assertNotSame(streamedRows.get(1).__internalLoadState(), streamedRows.get(2).__internalLoadState()); + assertEquals("extension", streamedRows.get(0).dynamicFields().field("title").value()); + assertEquals(DynamicFieldValue.State.NULL, streamedRows.get(1).dynamicFields().field("title").state()); + assertEquals(DynamicFieldValue.State.NOT_LOADED, streamedRows.get(2).dynamicFields().field("title").state()); + for (Task row : streamedRows) { + assertFalse(row.__internalHasMutationLedger()); + assertEquals(DynamicFieldValue.State.NOT_LOADED, row.dynamicFields().field("extra").state()); + assertTrue(row.getUpdatedProperties().isEmpty()); + } + // Invalid type projections still fail before invoking the provider, even on streams. + try (var cursor = streaming.queryForCursor(local, new DefaultQueryRequest(wrongType))) { + assertThrows(DynamicFieldException.class, () -> cursor.stream().toList()); + } + assertEquals("rejected stream projection must not reach provider", 2, provider.batches); } @Test From dc314f8d96184f1f5278885f6bb09a2520923186 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 7 Oct 2026 17:20:35 +0800 Subject: [PATCH 15/51] test(example): qualify forward reverse and dynamic metadata state boundaries --- .../GeneratedNestedGraphAssertions.java | 10 ++++++++++ examples/shared-load-state/verify.sh | 3 +++ 2 files changed, 13 insertions(+) diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedNestedGraphAssertions.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedNestedGraphAssertions.java index 00c77824..0d3f439e 100644 --- a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedNestedGraphAssertions.java +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedNestedGraphAssertions.java @@ -40,6 +40,12 @@ static void verifyDynamic(UserContext context, Long schoolId, String renamed, .purpose("why: verify nested row-owned dynamic field carriers").executeForOne(context); var rows = loaded.getSchoolList(); assertEquals(3, rows.size()); + var metadata = rows.get(0).dynamicFields().metadata(); + for (var row : rows.getData()) { + assertSame(metadata, row.dynamicFields().metadata(), "compatible children share immutable name/type metadata"); + assertFalse(com.example.schoolmanagementservice.school.School.__TEAQL_FIXED_FIELD_INDEXES.containsKey("#" + extension)); + assertFalse(com.example.schoolmanagementservice.school.School.__TEAQL_FIXED_FIELD_INDEXES.containsKey("_" + extension)); + } assertEquals("Private extension", rows.get(0).dynamicFields().field(extension).value()); assertEquals(DynamicFieldValue.State.NOT_LOADED, rows.get(1).dynamicFields().field(extension).state()); assertEquals(DynamicFieldValue.State.NULL, rows.get(2).dynamicFields().field(extension).state()); @@ -61,6 +67,7 @@ static void verifyDynamic(UserContext context, Long schoolId, String renamed, assertEquals("Private extension", json.get("platform").get("schoolList").get(0).get("#" + extension).asText()); assertTrue(json.get("platform").get("schoolList").get(1).get("#" + extension).isNull()); System.out.println("PASS generated Java nested dynamic Value/NULL/NotLoaded and shared snapshots"); + System.out.println("PASS generated Java LF17 dynamic metadata shared without fixed slots"); } static void verify(UserContext context, Long schoolId, String renamed, @@ -128,12 +135,14 @@ static void verify(UserContext context, Long schoolId, String renamed, assertFalse(mapper.readTree(mapper.writeValueAsString(unselected)).has("schoolList")); var restoredUnselected = mapper.readValue(mapper.writeValueAsString(unselected), unselected.getClass()); assertFalse(restoredUnselected.isPropertyLoaded("schoolList")); + System.out.println("PASS generated Java LF09 reverse Loaded/Empty/NotLoaded through Q/E/JSON"); var filtered = Q.schoolsWithMinimalFields().withIdIs(schoolId) .selectSchoolTypeWith(Q.schoolTypesWithMinimalFields().withIdIs(1002L)) .limit(1).comment("what: retain Primary FK while excluding it from selected target detail") .purpose("why: verify current filtered-forward NotLoaded semantics").executeForOne(context); assertEquals(1001L, filtered.getSchoolType().getId()); + assertTrue(filtered.getSchoolType().isPropertyLoaded("id"), "FK identity cannot become NotLoaded with excluded details"); assertFalse(filtered.getSchoolType().isPropertyLoaded("code")); var filteredJson = mapper.readTree(mapper.writeValueAsString(filtered)); assertFalse(filteredJson.get("schoolType").has("code")); @@ -141,6 +150,7 @@ static void verify(UserContext context, Long schoolId, String renamed, assertEquals(1001L, restoredFiltered.getSchoolType().getId()); assertFalse(restoredFiltered.getSchoolType().isPropertyLoaded("code")); assertFalse(restoredFiltered.__internalHasMutationLedger()); + System.out.println("PASS generated Java LF08 loaded FK and excluded forward details stay distinct"); System.out.println("PASS generated Java typed JSON graph roundtrip and Empty/NotLoaded isolation"); System.out.println("PASS generated Java nested/reverse graph Q/E/JSON and Empty/NotLoaded isolation"); } diff --git a/examples/shared-load-state/verify.sh b/examples/shared-load-state/verify.sh index 2fc7a9ba..5b2c9e92 100644 --- a/examples/shared-load-state/verify.sh +++ b/examples/shared-load-state/verify.sh @@ -80,6 +80,9 @@ for round in first second; do rg -F "PASS generated Java namespace serialization and NotLoaded boundary" "$run_dir/$round.log" rg -F "PASS generated Java nested/reverse graph Q/E/JSON and Empty/NotLoaded isolation" "$run_dir/$round.log" rg -F "PASS generated Java nested dynamic Value/NULL/NotLoaded and shared snapshots" "$run_dir/$round.log" + rg -F "PASS generated Java LF08 loaded FK and excluded forward details stay distinct" "$run_dir/$round.log" + rg -F "PASS generated Java LF09 reverse Loaded/Empty/NotLoaded through Q/E/JSON" "$run_dir/$round.log" + rg -F "PASS generated Java LF17 dynamic metadata shared without fixed slots" "$run_dir/$round.log" rg -F "PASS Java real typed stream loaded state and provider allocation controls" "$run_dir/$round.log" if [[ "$fixture_inheritance" == true ]]; then rg -F "PASS generated Java inherited indexes Q/E/save and snapshot isolation $round" "$run_dir/$round.log" From b886ffdd9cca2fe2d244235924e1116c19d7efbf Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 7 Oct 2026 18:25:40 +0800 Subject: [PATCH 16/51] perf(core): defer SmartList grouping for already-shared load states --- .../GeneratedSchoolLoadStateTest.java | 1 + .../LoadStateAllocationTest.java | 61 +++++++++++++++++++ examples/shared-load-state/verify.sh | 1 + .../main/java/io/teaql/core/SmartList.java | 18 +++++- .../java/io/teaql/core/LoadStateTest.java | 26 ++++++++ 5 files changed, 105 insertions(+), 2 deletions(-) diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java index 62f350b7..81e6ab2a 100644 --- a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java @@ -110,6 +110,7 @@ void generatedSchoolUsesLocalRuntimeAndSharedState() throws Exception { assertEquals(2, full.size()); LoadState snapshot = full.get(0).__internalLoadState(); assertSame(snapshot, full.get(1).__internalLoadState()); + LoadStateAllocationTest.verifyGeneratedSharing(full); FieldLayout layout = FieldLayout.forType(School.class); assertSame(layout, snapshot.layout()); assertEquals(School.__TEAQL_FIELD_LAYOUT_REVISION, layout.revision()); diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/LoadStateAllocationTest.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/LoadStateAllocationTest.java index 546fe4f9..d001ff23 100644 --- a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/LoadStateAllocationTest.java +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/LoadStateAllocationTest.java @@ -18,6 +18,67 @@ public class LoadStateAllocationTest { private static volatile Object consumed; private static final class Probe extends BaseEntity {} private record Sample(long bytes, long nanos) {} + static void verifyGeneratedSharing(io.teaql.core.SmartList list) { + var bean=(com.sun.management.ThreadMXBean) ManagementFactory.getThreadMXBean(); + assertTrue(bean.isThreadAllocatedMemorySupported());bean.setThreadAllocatedMemoryEnabled(true); + var control=measure(bean,1,() -> consumed=new byte[8]); + assertTrue(control.bytes()>0,"allocation counter must observe its positive control"); + var state=((BaseEntity)list.first()).__internalLoadState(); + for(var row:list)assertSame(state,((BaseEntity)row).__internalLoadState()); + Runnable action=list::__internalShareLoadStates; + for(int i=0;i<10_000;i++)action.run(); + // Warm the exact counter/action invocation path, not only the list loop. + // Counter/JIT initialization is excluded from the steady-state claim. + for(int i=0;i<100;i++)measure(bean,100,action); + for(int iterations:new int[]{1,100,10_000}) { + int consecutiveZero=0; + for(int attempt=0;attempt<20 && consecutiveZero<3;attempt++) { + long bytes=measure(bean,iterations,action).bytes(); + // A bounded stable window excludes intermittent JVM counter/JIT + // setup, but cannot accept an action that allocates on every call. + consecutiveZero=bytes==0?consecutiveZero+1:0; + System.out.printf("GENERATED_LIST_FINALIZE,%d,%d,%d%n",iterations,attempt,bytes); + } + assertEquals(3,consecutiveZero,"shared generated lists require three consecutive allocation-free samples"); + } + for(var row:list)assertSame(state,((BaseEntity)row).__internalLoadState()); + System.out.println("PASS generated Java homogeneous list finalization allocates zero and preserves shared state"); + } + private static final class SharingProbe extends BaseEntity { + static { + Map indexes = new LinkedHashMap<>(); + Map> mappings = new LinkedHashMap<>(); + for(int i=0;i<130;i++) { + String name=i==0?"id":i==1?"version":"field_"+i; + indexes.put(name,i);mappings.put(name,List.of(name,name)); + } + FieldLayout.installGenerated(FieldLayout.generated(SharingProbe.class,"sharing-v1",indexes,mappings,Set.of())); + } + } + + @Test + public void homogeneousListFinalizationAllocationProbe() { + var bean=(com.sun.management.ThreadMXBean) ManagementFactory.getThreadMXBean(); + assertTrue(bean.isThreadAllocatedMemorySupported());bean.setThreadAllocatedMemoryEnabled(true); + new SharingProbe(); // Install type metadata outside measurement. + var layout=FieldLayout.forType(SharingProbe.class); + for(boolean wide:new boolean[]{false,true}) { + List fields=new ArrayList<>(); + for(int i=0;i<(wide?130:3);i++)fields.add(i==0?"id":i==1?"version":"field_"+i); + var shape=LoadState.projection(layout,fields).withDynamicSelection(Set.of("note")); + for(int count:new int[]{1,100,10_000}) { + List rows=new ArrayList<>(count); + for(int i=0;i codes) { diff --git a/examples/shared-load-state/verify.sh b/examples/shared-load-state/verify.sh index 5b2c9e92..212bb2ea 100644 --- a/examples/shared-load-state/verify.sh +++ b/examples/shared-load-state/verify.sh @@ -83,6 +83,7 @@ for round in first second; do rg -F "PASS generated Java LF08 loaded FK and excluded forward details stay distinct" "$run_dir/$round.log" rg -F "PASS generated Java LF09 reverse Loaded/Empty/NotLoaded through Q/E/JSON" "$run_dir/$round.log" rg -F "PASS generated Java LF17 dynamic metadata shared without fixed slots" "$run_dir/$round.log" + rg -F "PASS generated Java homogeneous list finalization allocates zero and preserves shared state" "$run_dir/$round.log" rg -F "PASS Java real typed stream loaded state and provider allocation controls" "$run_dir/$round.log" if [[ "$fixture_inheritance" == true ]]; then rg -F "PASS generated Java inherited indexes Q/E/save and snapshot isolation $round" "$run_dir/$round.log" diff --git a/teaql-core/src/main/java/io/teaql/core/SmartList.java b/teaql-core/src/main/java/io/teaql/core/SmartList.java index 9c0cb2e9..ae7c6073 100644 --- a/teaql-core/src/main/java/io/teaql/core/SmartList.java +++ b/teaql-core/src/main/java/io/teaql/core/SmartList.java @@ -124,10 +124,24 @@ public void setData(List pData) { @FrameworkInternal("Hydration result finalization only") public void __internalShareLoadStates() { if (data == null || data.size() < 2) return; - Map shapes = new HashMap<>(); - for (T entity : data) { + LoadState first = null; + Map shapes = null; + // Indexed traversal avoids an iterator allocation on the normal ArrayList + // result, while non-random-access lists retain linear traversal. + Iterator iterator = data instanceof java.util.RandomAccess ? null : data.iterator(); + int index = 0; + while (iterator == null ? index < data.size() : iterator.hasNext()) { + T entity = iterator == null ? data.get(index++) : iterator.next(); if (!(entity instanceof BaseEntity base)) continue; LoadState state = base.__internalLoadState(); + if (first == null) { first = state; continue; } + // Compiled hydration already supplies the same immutable snapshot. + // No grouping index is needed unless another reference appears. + if (state == first) continue; + if (shapes == null) { + shapes = new HashMap<>(); + shapes.put(first, first); + } LoadState shared = shapes.putIfAbsent(state, state); if (shared != null && shared != state) base.__internalUseLoadState(shared); } diff --git a/teaql-core/src/test/java/io/teaql/core/LoadStateTest.java b/teaql-core/src/test/java/io/teaql/core/LoadStateTest.java index 805dd624..f91a03cc 100644 --- a/teaql-core/src/test/java/io/teaql/core/LoadStateTest.java +++ b/teaql-core/src/test/java/io/teaql/core/LoadStateTest.java @@ -262,6 +262,32 @@ public void appendAndMixedProjectionNeverUnionStatesAndSurviveListLifetime() { assertTrue(minimal.isPropertyLoaded("id")); } + @Test + public void deferredGroupingHandlesRepeatedFirstShapeAndLinkedLists() { + var layout = FieldLayout.forType(IndexedEntity.class); + for (boolean linked : new boolean[]{false,true}) { + var firstShape = LoadState.projection(layout, List.of("id")); + var fullShape = LoadState.projection(layout, List.of("id", "baseUrl")); + var equalFirst = LoadState.projection(layout, List.of("id")); + var equalFull = LoadState.projection(layout, List.of("baseUrl", "id")); + assertNotSame(firstShape,equalFirst);assertNotSame(fullShape,equalFull); + List rows = linked ? new java.util.LinkedList<>() : new ArrayList<>(); + for(var state : List.of(firstShape,firstShape,fullShape,equalFirst,equalFull,firstShape)) { + var row=new IndexedEntity();row.__internalUseLoadState(state);rows.add(row); + } + var list=SmartList.takeOwnership(rows); + assertSame(firstShape,list.get(0).__internalLoadState()); + assertSame(firstShape,list.get(1).__internalLoadState()); + assertSame(firstShape,list.get(3).__internalLoadState()); + assertSame(firstShape,list.get(5).__internalLoadState()); + assertSame(fullShape,list.get(2).__internalLoadState()); + assertSame(fullShape,list.get(4).__internalLoadState()); + assertFalse(list.get(3).isPropertyLoaded("baseUrl")); + list.__internalShareLoadStates(); + assertSame(firstShape,list.get(3).__internalLoadState()); + } + } + @Test public void hydrateIndexMismatchFailsBeforeWritingAndUnknownNamesCannotCorruptBits() { IndexedEntity row = new IndexedEntity(); From a4483d8c2acdfebce0cbc6dfcd457b2ed4fab4d3 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 7 Oct 2026 18:45:35 +0800 Subject: [PATCH 17/51] test(example): qualify same-name namespaces and view-local availability --- .../GeneratedNamespaceCowAcceptance.java | 53 +++++++++++++++++++ .../GeneratedSchoolLoadStateTest.java | 1 + examples/shared-load-state/verify.sh | 2 + 3 files changed, 56 insertions(+) create mode 100644 examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedNamespaceCowAcceptance.java diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedNamespaceCowAcceptance.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedNamespaceCowAcceptance.java new file mode 100644 index 00000000..23369ad2 --- /dev/null +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedNamespaceCowAcceptance.java @@ -0,0 +1,53 @@ +package com.example.schoolmanagementservice; + +import com.example.schoolmanagementservice.school.School; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.teaql.core.UserContext; +import io.teaql.data.dynamic.*; +import io.teaql.data.dynamic.jdbc.JdbcDynamicFieldsProvider; +import static org.junit.jupiter.api.Assertions.*; + +final class GeneratedNamespaceCowAcceptance { + static School verify(UserContext context, Long id, String name, String sibling, String note, + JdbcDynamicFieldsProvider provider, DynamicFieldContext definitionContext, ObjectMapper mapper) throws Exception { + var def=new DynamicFieldDef();def.setScope(DynamicFieldScope.global()); + def.setOwnerType("School");def.setCode("name");def.setName("name");def.setDataType(DynamicDataType.STRING); + var existing=provider.listFieldDefs(definitionContext,"School").stream() + .filter(field -> "name".equals(field.getCode())).findFirst(); + if(existing.isEmpty()) provider.registerFieldDef(definitionContext,def); + else assertEquals(DynamicDataType.STRING,existing.get().getDataType()); + var request=Q.schools().withNameIn(name,sibling).selectSelfFields(); + request.selectDynamicFieldsWith(new DynamicFieldSelection().selectString(note)); + var rows=request.orderByIdAscending().limit(2).comment("what: load same-shape namespace controls") + .purpose("why: separate fixed derived and persistent same-name members").executeForList(context); + assertEquals(2,rows.size()); + var shared=rows.get(0).__internalLoadState();assertSame(shared,rows.get(1).__internalLoadState()); + var first=rows.get(0);assertEquals(id,first.getId()); + assertEquals(DynamicFieldValue.State.NOT_LOADED,first.dynamicFields().field("name").state()); + first.addDynamicProperty("name","readonly same name"); + first.updateDynamicField("name","persistent same name"); + assertNotSame(shared,first.__internalLoadState());assertSame(shared,rows.get(1).__internalLoadState()); + assertFalse(shared.isLoaded("#name")); + assertEquals(DynamicFieldValue.State.NOT_LOADED,rows.get(1).dynamicFields().field("name").state()); + assertTrue(rows.get(1).getUpdatedProperties().isEmpty()); + assertFalse(first.getUpdatedProperties().contains("name")); + var json=mapper.readTree(mapper.writeValueAsString(first)); + assertEquals(name,json.get("name").asText()); + assertEquals("readonly same name",json.get("_name").asText()); + assertEquals("persistent same name",json.get("#name").asText()); + first.auditAs("persist only the namespaced extension").save(context); + var reread=Q.schools().withIdIs(id).selectSelfFields(); + reread.selectDynamicFieldsWith(new DynamicFieldSelection().selectString("name")); + var read=reread.limit(1).comment("what: read back the namespace controls") + .purpose("why: prove readonly properties never persist").executeForOne(context); + assertEquals(name,read.getName());assertNull(read.getProperty("_name")); + assertEquals("persistent same name",read.dynamicFields().field("name").value()); + var before=read.__internalLoadState();read.deleteDynamicField("name"); + assertNotSame(before,read.__internalLoadState());assertTrue(before.isLoaded("#name")); + assertEquals(DynamicFieldValue.State.NOT_LOADED,read.dynamicFields().field("name").state()); + read.auditAs("delete only the namespaced extension control").save(context); + System.out.println("PASS generated Java LF19 fixed derived and persistent same-name namespace isolation"); + System.out.println("PASS generated Java LF23 dynamic availability detaches only one view"); + return read; + } +} diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java index 81e6ab2a..eb8e4392 100644 --- a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java @@ -299,6 +299,7 @@ void generatedSchoolUsesLocalRuntimeAndSharedState() throws Exception { System.out.println("PASS generated Java dynamic storage provenance and retry"); var combined = GeneratedDynamicRollbackAcceptance.verify(context, sql, extended, renamed, second, extension, provider, definitionContext); String deletionName = combined.getName(); + GeneratedNamespaceCowAcceptance.verify(context,combined.getId(),deletionName,second,extension,provider,definitionContext,mapper); // Partial related objects are valid for E reads, not for whole-graph Checker validation. var forDeletion = Q.schools().withIdIs(changed.getId()).limit(1) .comment("what: reload the full native School before deletion") diff --git a/examples/shared-load-state/verify.sh b/examples/shared-load-state/verify.sh index 212bb2ea..b568f7e2 100644 --- a/examples/shared-load-state/verify.sh +++ b/examples/shared-load-state/verify.sh @@ -83,6 +83,8 @@ for round in first second; do rg -F "PASS generated Java LF08 loaded FK and excluded forward details stay distinct" "$run_dir/$round.log" rg -F "PASS generated Java LF09 reverse Loaded/Empty/NotLoaded through Q/E/JSON" "$run_dir/$round.log" rg -F "PASS generated Java LF17 dynamic metadata shared without fixed slots" "$run_dir/$round.log" + rg -F "PASS generated Java LF19 fixed derived and persistent same-name namespace isolation" "$run_dir/$round.log" + rg -F "PASS generated Java LF23 dynamic availability detaches only one view" "$run_dir/$round.log" rg -F "PASS generated Java homogeneous list finalization allocates zero and preserves shared state" "$run_dir/$round.log" rg -F "PASS Java real typed stream loaded state and provider allocation controls" "$run_dir/$round.log" if [[ "$fixture_inheritance" == true ]]; then From 8c7dab11df1fc896c6020b360a6220c450445218 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 7 Oct 2026 19:04:40 +0800 Subject: [PATCH 18/51] test: persist readonly totals through a modeled reporting entity --- examples/shared-load-state/README.md | 12 +++- examples/shared-load-state/model.xml | 3 + .../GeneratedMaterializationAcceptance.java | 62 +++++++++++++++++++ .../GeneratedSchoolLoadStateTest.java | 1 + examples/shared-load-state/verify.sh | 1 + 5 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedMaterializationAcceptance.java diff --git a/examples/shared-load-state/README.md b/examples/shared-load-state/README.md index 416bbb37..2adb9d58 100644 --- a/examples/shared-load-state/README.md +++ b/examples/shared-load-state/README.md @@ -28,7 +28,8 @@ overflow copy-on-write. A requested wide run rejects a narrow artifact; an existing wide artifact automatically keeps the wide assertions enabled. Add `--inheritance` during generation to append one Academy subtype with a -campus code. The normal three-object model remains unchanged. The producer +campus code. The base model has Platform, SchoolType, School and a small +SchoolCapacitySummary reporting target. The producer retains a separate inheritance marker and the verifier selects an additional Maven test source only for that generated artifact. A requested inherited run rejects a flat artifact; replay preserves the inherited gate automatically. @@ -44,6 +45,15 @@ reactor `target/classes`, not an installed jar. ## Acceptance +`GeneratedMaterializationAcceptance` calculates a capacity total over two +bounded School Q results using E. The readonly `_total_capacity` result gets +no fixed slot and must not persist when the source is saved. Persistence is +explicit: create a modeled SchoolCapacitySummary with its generated Mutation +API, attach an audit reason, save, and query it back through Q/E. The expected +total is 37 and contributor count is 2. The source sibling retains its original +shared snapshot and clean mutation state. This is application-side calculation +and modeled storage, not a database aggregate-query performance claim. + `GeneratedNestedGraphAssertions` also verifies a bounded Platform-to-Schools view and School-to-Platform-to-Schools graph through generated Q/E and JSON. Loaded empty lists serialize as arrays; unselected lists stay absent. A target diff --git a/examples/shared-load-state/model.xml b/examples/shared-load-state/model.xml index 614f709c..9b05e52a 100644 --- a/examples/shared-load-state/model.xml +++ b/examples/shared-load-state/model.xml @@ -11,4 +11,7 @@ name="Riverside Primary School" address="12 River Road, Springfield" established_date="1995-09-01" student_capacity="800" active="true" create_time="createTime()" update_time="updateTime()"/> + diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedMaterializationAcceptance.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedMaterializationAcceptance.java new file mode 100644 index 00000000..5511d3d8 --- /dev/null +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedMaterializationAcceptance.java @@ -0,0 +1,62 @@ +package com.example.schoolmanagementservice; + +import com.example.schoolmanagementservice.school.School; +import com.example.schoolmanagementservice.schoolcapacitysummary.SchoolCapacitySummary; +import io.teaql.core.FieldLayout; +import io.teaql.core.UserContext; +import static org.junit.jupiter.api.Assertions.*; + +/** Bounded Q/E calculation followed by an explicitly modeled, audited write. */ +final class GeneratedMaterializationAcceptance { + static void verify(UserContext context, Long id, String name, String sibling, String round) { + var source = Q.schools().withIdIs(id).selectSelfFields().limit(1) + .comment("what: load the capacity source before its audited change") + .purpose("why: retain complete Checker input").executeForOne(context); + source.updateStudentCapacity(37); + source.auditAs("Set the nonzero materialization control").save(context); + var rows = Q.schools().withNameIn(name, sibling).selectSelfFields() + .orderByIdAscending().limit(2).comment("what: read the two capacity contributors") + .purpose("why: calculate the bounded reporting snapshot with E").executeForList(context); + assertEquals(2, rows.size()); + int total = rows.stream().mapToInt(row -> E.school(row).getStudentCapacity().eval()).sum(); + assertEquals(37, total); + var shared = rows.get(0).__internalLoadState(); + assertSame(shared, rows.get(1).__internalLoadState()); + var view = rows.get(0); + view.addDynamicProperty("total_capacity", total); + assertEquals(37, ((Number) view.getProperty("_total_capacity")).intValue()); + assertNull(view.getProperty("_missing_total")); + assertTrue(view.getUpdatedProperties().isEmpty()); + assertNull(FieldLayout.forType(School.class).findIndex("_total_capacity")); + view.updateActive(true); + assertFalse(view.getUpdatedProperties().contains("_total_capacity")); + view.auditAs("Save a native change without persisting its readonly total").save(context); + + var platform = Q.platforms().withIdIs(1L).limit(1) + .comment("what: reuse the model-owned reporting root") + .purpose("why: attach the explicit summary target").executeForOne(context); + var target = Q.schoolCapacitySummaries() + .comment("what: allocate a modeled capacity summary") + .purpose("why: explicitly persist the reporting result").newEntity(context); + target.updatePlatform(platform); + target.updateName(round + " capacity snapshot"); + target.updateTotalCapacity(total); + target.updateSchoolCount(rows.size()); + target.auditAs("Materialize the calculated total in its own entity").save(context); + var read = Q.schoolCapacitySummaries().withIdIs(target.getId()).selectSelfFields().limit(1) + .comment("what: reload the persisted summary") + .purpose("why: verify modeled values through E").executeForOne(context); + assertEquals(37, E.schoolCapacitySummary(read).getTotalCapacity().eval()); + assertEquals(2, E.schoolCapacitySummary(read).getSchoolCount().eval()); + assertNotNull(FieldLayout.forType(SchoolCapacitySummary.class).findIndex("total_capacity")); + assertSame(shared, rows.get(1).__internalLoadState()); + assertTrue(rows.get(1).getUpdatedProperties().isEmpty()); + var fresh = Q.schools().withIdIs(id).selectSelfFields().limit(1) + .comment("what: reload the source after materialization") + .purpose("why: prove the readonly total never became stored source data").executeForOne(context); + assertNull(fresh.getProperty("_total_capacity")); + assertEquals(37, E.school(fresh).getStudentCapacity().eval()); + assertEquals(true, E.school(fresh).isActive().eval()); + System.out.println("PASS generated Java LF20 readonly total persists only through modeled materialization"); + } +} diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java index eb8e4392..f9aa171b 100644 --- a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java @@ -300,6 +300,7 @@ void generatedSchoolUsesLocalRuntimeAndSharedState() throws Exception { var combined = GeneratedDynamicRollbackAcceptance.verify(context, sql, extended, renamed, second, extension, provider, definitionContext); String deletionName = combined.getName(); GeneratedNamespaceCowAcceptance.verify(context,combined.getId(),deletionName,second,extension,provider,definitionContext,mapper); + GeneratedMaterializationAcceptance.verify(context, combined.getId(), deletionName, second, round); // Partial related objects are valid for E reads, not for whole-graph Checker validation. var forDeletion = Q.schools().withIdIs(changed.getId()).limit(1) .comment("what: reload the full native School before deletion") diff --git a/examples/shared-load-state/verify.sh b/examples/shared-load-state/verify.sh index b568f7e2..26ce87b0 100644 --- a/examples/shared-load-state/verify.sh +++ b/examples/shared-load-state/verify.sh @@ -85,6 +85,7 @@ for round in first second; do rg -F "PASS generated Java LF17 dynamic metadata shared without fixed slots" "$run_dir/$round.log" rg -F "PASS generated Java LF19 fixed derived and persistent same-name namespace isolation" "$run_dir/$round.log" rg -F "PASS generated Java LF23 dynamic availability detaches only one view" "$run_dir/$round.log" + rg -F "PASS generated Java LF20 readonly total persists only through modeled materialization" "$run_dir/$round.log" rg -F "PASS generated Java homogeneous list finalization allocates zero and preserves shared state" "$run_dir/$round.log" rg -F "PASS Java real typed stream loaded state and provider allocation controls" "$run_dir/$round.log" if [[ "$fixture_inheritance" == true ]]; then From 8247890a4920cc9215b31b23199a770a26fbbd24 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 7 Oct 2026 19:21:56 +0800 Subject: [PATCH 19/51] test: separate default masked logging from log-off driver measurements --- .../shared-load-state/benchmark-driver.sh | 9 ++++++++ .../DriverQueryBenchmarkTest.java | 15 ++++++++++--- .../LoadStateAllocationTest.java | 22 ++++++++++++++++--- 3 files changed, 40 insertions(+), 6 deletions(-) diff --git a/examples/shared-load-state/benchmark-driver.sh b/examples/shared-load-state/benchmark-driver.sh index 069ed145..d9ba469e 100644 --- a/examples/shared-load-state/benchmark-driver.sh +++ b/examples/shared-load-state/benchmark-driver.sh @@ -1,6 +1,15 @@ #!/usr/bin/env bash set -euo pipefail runtime_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +case "${1:-}" in + "") export TEAQL_DRIVER_BENCHMARK_LOG_MODE=off ;; + --default-log) export TEAQL_DRIVER_BENCHMARK_LOG_MODE=default ;; + *) printf 'Usage: bash benchmark-driver.sh [--default-log]\n' >&2; exit 2 ;; +esac +(( $# <= 1 )) || { printf 'Unexpected arguments\n' >&2; exit 2; } +if [[ -n "${TEAQL_ALLOW_SENSITIVE_PLAINTEXT_LOGS:-}" ]]; then + printf 'Unset TEAQL_ALLOW_SENSITIVE_PLAINTEXT_LOGS for the masked logging benchmark\n' >&2; exit 2 +fi run_dir="$(mktemp -d -t teaql-java-driver-evidence.XXXXXXXX)" cd "$runtime_dir" java -version >"$run_dir/toolchain.txt" 2>&1 diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/DriverQueryBenchmarkTest.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/DriverQueryBenchmarkTest.java index a45ce2ff..0634cb4a 100644 --- a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/DriverQueryBenchmarkTest.java +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/DriverQueryBenchmarkTest.java @@ -66,6 +66,10 @@ static void report(String lane, boolean full, int count, List consumed = new byte[4096]).bytes() >= 4096, + "positive allocation calibration must be visible"); System.out.println("case,width,iterations,allocated_bytes,elapsed_ns"); for (int width : new int[]{4, 64, 130}) { Map indexes = new LinkedHashMap<>(); @@ -127,15 +143,15 @@ public void loadedAvailabilityAllocationProbe() { Runnable reference = () -> consumed = referenceDynamicNoop(selected, codes); for (int i = 0; i < 30_000; i++) { fixed.run(); dynamic.run(); dynamicView.run(); reference.run(); } for (int iterations : new int[]{1, 100, 10_000}) { - Sample result = measure(bean, iterations, fixed); + Sample result = steadyNoop(bean, "fixed", width, iterations, fixed, base); System.out.printf("loaded_fixed_noop,%d,%d,%d,%d%n", width, iterations, result.bytes(), result.nanos()); assertSame(base, consumed); assertEquals(0, result.bytes(), "fixed value-only availability must not allocate"); - result = measure(bean, iterations, dynamic); + result = steadyNoop(bean, "dynamic", width, iterations, dynamic, selected); System.out.printf("loaded_dynamic_noop,%d,%d,%d,%d%n", width, iterations, result.bytes(), result.nanos()); assertSame(selected, consumed); assertEquals(0, result.bytes(), "unchanged dynamic selection must not allocate"); - result = measure(bean, iterations, dynamicView); + result = steadyNoop(bean, "dynamic-map-view", width, iterations, dynamicView, selected); System.out.printf("loaded_dynamic_map_view_noop,%d,%d,%d,%d%n", width, iterations, result.bytes(), result.nanos()); assertSame(selected, consumed); assertEquals(0, result.bytes(), "map-backed selected-code views must not allocate iterators"); From 17c4252f8c6450407fc752470e985ef6b2af6b77 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 7 Oct 2026 19:39:10 +0800 Subject: [PATCH 20/51] docs: explain matched driver logging and bounded allocation probes --- examples/shared-load-state/README.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/examples/shared-load-state/README.md b/examples/shared-load-state/README.md index 2adb9d58..6a078c36 100644 --- a/examples/shared-load-state/README.md +++ b/examples/shared-load-state/README.md @@ -128,3 +128,24 @@ This checks shared overflow snapshots, COW isolation and lazy ledgers, requires unchanged generated-source hashes, and rejects a non-wide artifact. Prepared inputs, JDBC and logging remain outside the measured hydration region. These are not whole-query latency, retained heap, CPU saturation or ORM comparisons. + +The no-op probe preserves every bounded attempt, including JVM transient bytes, +and requires three consecutive zero-allocation samples within twenty attempts. +An allocating positive control must remain visible. This is a warmed-window +claim, not a guarantee of zero allocation for every JVM observation. + +## Matched driver logging modes + +```bash +bash examples/shared-load-state/benchmark-driver.sh +bash examples/shared-load-state/benchmark-driver.sh --default-log +``` + +Run these sequentially. Both modes retain six sparse/full cases, identical +native SQL/binds and typed-result/load-state checks over 31 alternating samples. +Default mode uses the unchanged stderr sink and privacy policy; off mode +disables query logging. The scripts reject plaintext opt-in. Query samples +exclude setup and correctness checks; the CPU receipt includes Maven/JIT. +Native-driver drift between processes prevents treating an absolute latency +difference as pure logging cost. Different Rust connection and output strategies +prohibit a language ranking. No publication occurs. From 22ae72c69b8f900ec79dab87f3984204ca540df0 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 7 Oct 2026 20:01:17 +0800 Subject: [PATCH 21/51] fix: hydrate authoritative native readback through physical column metadata --- .../GeneratedNativeRollbackAcceptance.java | 58 ++++++++++++ .../GeneratedSchoolLoadStateTest.java | 1 + examples/shared-load-state/verify.sh | 1 + .../sql/portable/PortableSQLRepository.java | 21 +++-- .../NativeReadbackColumnMappingTest.java | 90 +++++++++++++++++++ 5 files changed, 166 insertions(+), 5 deletions(-) create mode 100644 examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedNativeRollbackAcceptance.java create mode 100644 teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/NativeReadbackColumnMappingTest.java diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedNativeRollbackAcceptance.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedNativeRollbackAcceptance.java new file mode 100644 index 00000000..1f678c5f --- /dev/null +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedNativeRollbackAcceptance.java @@ -0,0 +1,58 @@ +package com.example.schoolmanagementservice; + +import io.teaql.core.UserContext; +import javax.sql.DataSource; +import java.time.LocalDate; +import static org.junit.jupiter.api.Assertions.*; + +/** Native-only invalid authoritative readback, not an extension-provider failure. */ +final class GeneratedNativeRollbackAcceptance { + static void verify(UserContext context, DataSource source, ObservedJdbcExecutor sql, + String name, String sibling) throws Exception { + var rows = Q.schools().withNameIn(name, sibling).selectSelfFields().orderByIdAscending().limit(2) + .comment("what: load native-only rollback controls") + .purpose("why: preserve siblings through authoritative readback failure").executeForList(context); + assertEquals(2, rows.size()); + var snapshot = rows.get(0).__internalLoadState(); + assertSame(snapshot, rows.get(1).__internalLoadState()); + var pending = rows.get(0); + Long version = pending.getVersion(), siblingVersion = rows.get(1).getVersion(); + String nextName = name + " native-retry"; + pending.updateName(nextName); + assertSame(snapshot, pending.__internalLoadState()); + long[] before = sql.counts(); + try (var connection = source.getConnection(); var statement = connection.createStatement()) { + statement.execute("CREATE TRIGGER controlled_native_readback AFTER UPDATE OF name ON school_data " + + "WHEN NEW.name LIKE '% native-retry' " + + "BEGIN UPDATE school_data SET established_date='not-a-date' WHERE id=NEW.id; END"); + try { + assertThrows(RuntimeException.class, () -> pending + .auditAs("Rollback an invalid native authoritative readback").save(context)); + } finally { statement.execute("DROP TRIGGER controlled_native_readback"); } + } + assertTrue(sql.counts()[1] > before[1], "native DML must occur before readback fails"); + assertTrue(sql.counts()[2] > before[2]); + assertEquals(version, pending.getVersion()); + assertTrue(pending.getUpdatedProperties().contains("name")); + assertSame(snapshot, pending.__internalLoadState()); + assertEquals(siblingVersion, rows.get(1).getVersion()); + assertTrue(rows.get(1).getUpdatedProperties().isEmpty()); + assertSame(snapshot, rows.get(1).__internalLoadState()); + var stored = Q.schools().withNameIn(name, sibling).selectSelfFields().orderByIdAscending().limit(2) + .comment("what: inspect native rows after readback failure") + .purpose("why: prove the native transaction was rolled back").executeForList(context); + assertEquals(2, stored.size()); + assertEquals(name, E.school(stored.get(0)).getName().eval()); + assertEquals(version, stored.get(0).getVersion()); + assertEquals(LocalDate.of(1995, 9, 1), E.school(stored.get(0)).getEstablishedDate().eval()); + assertEquals(siblingVersion, stored.get(1).getVersion()); + pending.auditAs("Retry the unchanged native mutation intent").save(context); + assertEquals(version + 1, pending.getVersion()); + assertEquals(nextName, E.school(pending).getName().eval()); + assertTrue(pending.getUpdatedProperties().isEmpty()); + assertSame(snapshot, pending.__internalLoadState()); + pending.updateName(name); + pending.auditAs("Restore the native rollback fixture name").save(context); + System.out.println("PASS generated Java LF11 native readback rollback retains loaded state and retry intent"); + } +} diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java index f9aa171b..38b809b3 100644 --- a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java @@ -300,6 +300,7 @@ void generatedSchoolUsesLocalRuntimeAndSharedState() throws Exception { var combined = GeneratedDynamicRollbackAcceptance.verify(context, sql, extended, renamed, second, extension, provider, definitionContext); String deletionName = combined.getName(); GeneratedNamespaceCowAcceptance.verify(context,combined.getId(),deletionName,second,extension,provider,definitionContext,mapper); + GeneratedNativeRollbackAcceptance.verify(context, dataSource, sql, deletionName, second); GeneratedMaterializationAcceptance.verify(context, combined.getId(), deletionName, second, round); // Partial related objects are valid for E reads, not for whole-graph Checker validation. var forDeletion = Q.schools().withIdIs(changed.getId()).limit(1) diff --git a/examples/shared-load-state/verify.sh b/examples/shared-load-state/verify.sh index 26ce87b0..0f881389 100644 --- a/examples/shared-load-state/verify.sh +++ b/examples/shared-load-state/verify.sh @@ -86,6 +86,7 @@ for round in first second; do rg -F "PASS generated Java LF19 fixed derived and persistent same-name namespace isolation" "$run_dir/$round.log" rg -F "PASS generated Java LF23 dynamic availability detaches only one view" "$run_dir/$round.log" rg -F "PASS generated Java LF20 readonly total persists only through modeled materialization" "$run_dir/$round.log" + rg -F "PASS generated Java LF11 native readback rollback retains loaded state and retry intent" "$run_dir/$round.log" rg -F "PASS generated Java homogeneous list finalization allocates zero and preserves shared state" "$run_dir/$round.log" rg -F "PASS Java real typed stream loaded state and provider allocation controls" "$run_dir/$round.log" if [[ "$fixture_inheritance" == true ]]; then diff --git a/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLRepository.java b/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLRepository.java index 03fe06cd..fabf3d5c 100644 --- a/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLRepository.java +++ b/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLRepository.java @@ -1137,9 +1137,15 @@ T loadPersistedById(UserContext userContext, Long id, io.teaql.core.SqlIntentRed } T entity = (T) entityDescriptor.createEntity(); Map row = rows.get(0); + // SELECT * returns physical column names, unlike the aliased normal Q path. + // Install the actual geometry once before setters hydrate native values. + MapProjection projection = mapProjection(entityDescriptor.getTargetType(), row); + if (entity instanceof BaseEntity base && projection.loadedState() != null) { + base.__internalUseLoadState(projection.loadedState()); + } for (PropertyDescriptor property : this.allProperties) { if (!shouldHandle(property)) continue; - String columnKey = findColumnKey(row, property.getName()); + String columnKey = findColumnKey(row, property); if (columnKey == null) continue; Object value = row.get(columnKey); if (!(property instanceof Relation)) { @@ -1220,7 +1226,7 @@ private MapProjection mapProjection(Class type, Map bindings = new ArrayList<>(); for (PropertyDescriptor property : allProperties) { if (!shouldHandle(property)) continue; - String column = findColumnKey(row, property.getName()); + String column = findColumnKey(row, property); if (column != null) { selected.add(property.getName()); bindings.add(new MapBinding(property, column)); @@ -1288,6 +1294,13 @@ private T mapRowToEntity(UserContext userContext, SearchRequest request, Map< return entity; } + private String findColumnKey(Map row, PropertyDescriptor property) { + String member = findColumnKey(row, property.getName()); + if (member != null) return member; + SQLColumn column = getSqlColumn(property); + return column == null ? null : findColumnKey(row, column.getColumnName()); + } + private String findColumnKey(Map row, String propertyName) { if (row.containsKey(propertyName)) return propertyName; for (String key : row.keySet()) { @@ -1317,9 +1330,7 @@ static Object convertTemporalColumnValue(Class targetType, Object value) { return timestamp.toLocalDateTime().toLocalDate(); } String text = String.valueOf(value); - if (text.length() >= 10) { - return java.time.LocalDate.parse(text.substring(0, 10)); - } + return java.time.LocalDate.parse(text.substring(0, Math.min(10, text.length()))); } if (targetType == java.time.LocalTime.class) { if (value instanceof java.sql.Time time) return time.toLocalTime(); diff --git a/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/NativeReadbackColumnMappingTest.java b/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/NativeReadbackColumnMappingTest.java new file mode 100644 index 00000000..d84fa877 --- /dev/null +++ b/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/NativeReadbackColumnMappingTest.java @@ -0,0 +1,90 @@ +package io.teaql.core.sql.portable; + +import io.teaql.core.*; +import io.teaql.core.meta.*; +import io.teaql.core.sql.GenericSQLProperty; +import io.teaql.runtime.DefaultUserContext; +import io.teaql.runtime.TeaQLRuntime; +import java.time.LocalDate; +import java.time.format.DateTimeParseException; +import java.util.*; +import org.junit.Test; +import static org.junit.Assert.*; + +/** Actual SELECT * maps physical metadata, not inferred member spelling. */ +public class NativeReadbackColumnMappingTest { + public static final class Row extends BaseEntity { + static final FieldLayout LAYOUT = FieldLayout.installGenerated(FieldLayout.generated( + Row.class, "native-readback-v1", Map.of("id", 0, "version", 1, + "customer_reference", 2, "established_date", 3, "counter_value", 4), + Map.of("id", List.of("id", "id"), "version", List.of("version", "version"), + "customer_reference", List.of("customerReference", "external_ref"), + "established_date", List.of("establishedDate", "STARTED_ON"), + "counter_value", List.of("counterValue", "counter_value")), Set.of())); + String customerReference; + LocalDate establishedDate; + Integer counterValue; + @Override public Object __internalGet(String name) { + return switch (name) { + case "customerReference" -> customerReference; + case "establishedDate" -> establishedDate; + case "counterValue" -> counterValue; + default -> super.__internalGet(name); + }; + } + @Override public void __internalSet(String name, Object value) { + markPropertyLoaded(name); + switch (name) { + case "customerReference" -> customerReference = (String) value; + case "establishedDate" -> establishedDate = (LocalDate) value; + case "counterValue" -> counterValue = (Integer) value; + default -> super.__internalSet(name, value); + } + } + } + record Fixture(PortableSQLDatabaseTest.SQLiteTeaQLDatabase database, + PortableSQLRepository repository, DefaultUserContext context) {} + private Fixture fixture() throws Exception { + assertNotNull(Row.LAYOUT); + var descriptor = new EntityDescriptor(); + descriptor.setType("Row"); descriptor.setTargetType(Row.class); descriptor.setEntitySupplier(Row::new); + var definitions = List.of( + new Object[]{"id", "id", Long.class, "INTEGER"}, + new Object[]{"version", "version", Long.class, "INTEGER"}, + new Object[]{"customerReference", "external_ref", String.class, "VARCHAR(100)"}, + new Object[]{"establishedDate", "STARTED_ON", LocalDate.class, "DATE"}, + new Object[]{"counterValue", "counter_value", Integer.class, "INTEGER"}); + List properties = new ArrayList<>(); + for (Object[] definition : definitions) { + var property = new GenericSQLProperty("native_readback_data", (String) definition[1], (String) definition[3]); + property.setName((String) definition[0]); property.setOwner(descriptor); + property.setType(new SimplePropertyType((Class) definition[2])); properties.add(property); + } + descriptor.setProperties(properties); + var metadata = new SimpleEntityMetaFactory(); metadata.register(descriptor); + var context = new DefaultUserContext(TeaQLRuntime.builder().metadata(metadata).queryExecutionLogging(false).build()); + var database = new PortableSQLDatabaseTest.SQLiteTeaQLDatabase(); + var repository = new PortableSQLRepository(descriptor, database, type -> null, metadata); + repository.ensurePhysicalSchema(context); + return new Fixture(database, repository, context); + } + @Test public void physicalCustomColumnsHydrateValuesAndLoadedNull() throws Exception { + var fixture = fixture(); + fixture.database.executeUpdate("INSERT INTO native_readback_data(id,version,external_ref,STARTED_ON,counter_value) VALUES(?,?,?,?,?)", + new Object[]{1L, 1L, "CR-A", "2026-10-07", null}); + var row = fixture.repository.loadPersistedById(fixture.context, 1L); + assertEquals("CR-A", row.customerReference); assertEquals(LocalDate.of(2026, 10, 7), row.establishedDate); + assertNull(row.counterValue); + for (String name : List.of("id", "version", "customer_reference", "established_date", "counter_value")) assertTrue(row.isPropertyLoaded(name)); + assertSame(Row.LAYOUT, row.__internalLoadState().layout()); + assertFalse(row.__internalHasMutationLedger()); assertTrue(row.getUpdatedProperties().isEmpty()); + } + @Test public void invalidPhysicalDateCannotBeSkippedOrConvertedToLoadedNull() throws Exception { + for (String invalid : List.of("not-a-date", "bad", "")) { + var fixture = fixture(); + fixture.database.executeUpdate("INSERT INTO native_readback_data(id,version,external_ref,STARTED_ON,counter_value) VALUES(?,?,?,?,?)", + new Object[]{1L, 1L, "CR-A", invalid, null}); + assertThrows(DateTimeParseException.class, () -> fixture.repository.loadPersistedById(fixture.context, 1L)); + } + } +} From 97e61d98c082e20939abd514783c574abf833831 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 7 Oct 2026 20:27:02 +0800 Subject: [PATCH 22/51] perf: reuse bounded native readback projection geometry --- .../GeneratedReadbackAllocationTest.java | 111 ++++++++++++++++++ .../shared-load-state/verify-allocations.sh | 5 +- .../sql/portable/PortableSQLRepository.java | 8 +- .../NativeReadbackColumnMappingTest.java | 53 ++++++++- 4 files changed, 174 insertions(+), 3 deletions(-) create mode 100644 examples/shared-load-state/src/test/java/io/teaql/core/sql/portable/GeneratedReadbackAllocationTest.java diff --git a/examples/shared-load-state/src/test/java/io/teaql/core/sql/portable/GeneratedReadbackAllocationTest.java b/examples/shared-load-state/src/test/java/io/teaql/core/sql/portable/GeneratedReadbackAllocationTest.java new file mode 100644 index 00000000..f2848554 --- /dev/null +++ b/examples/shared-load-state/src/test/java/io/teaql/core/sql/portable/GeneratedReadbackAllocationTest.java @@ -0,0 +1,111 @@ +package io.teaql.core.sql.portable; + +import com.example.schoolmanagementservice.GeneratedRuntimeModule; +import io.teaql.core.*; +import io.teaql.core.meta.*; +import io.teaql.runtime.*; +import java.lang.management.ManagementFactory; +import java.time.*; +import java.util.*; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.*; + +/** Generated School, prepared driver rows, no SQL I/O, intent logging or setup in counters. */ +class GeneratedReadbackAllocationTest { + static volatile Object consumed; + record Binding(PropertyDescriptor property, String column, EntityDescriptor target) {} + record Sample(List rows, long bytes) {} + static class Database implements TeaQLDatabase { + List> rows; + public List> query(String sql,Object[] args) { + return List.of(rows.get(((Number)args[0]).intValue()-1)); + } + public int executeUpdate(String sql,Object[] args) { throw new AssertionError("read only"); } + public int[] batchUpdate(String sql,List args) { throw new AssertionError("read only"); } + public void execute(String sql) { throw new AssertionError("no schema I/O"); } + public void executeInTransaction(Runnable action) { throw new AssertionError("no transaction I/O"); } + public List> getTableColumns(String table) { throw new AssertionError("generated metadata"); } + } + static Object value(PropertyDescriptor property) { + Class type=property.getType().javaType(); + if(property.getName().startsWith("probe"))return null; + if(Entity.class.isAssignableFrom(type))return property.getName().equals("schoolType")?1001L:1L; + if(type==String.class)return "readback name"; + if(type==Long.class||type==long.class)return 1L; + if(type==Integer.class||type==int.class)return 0; + if(type==Boolean.class||type==boolean.class)return false; + if(type==LocalDate.class)return LocalDate.of(1995,9,1); + if(type==LocalDateTime.class)return LocalDateTime.of(2023,11,14,22,13); + throw new AssertionError(type); + } + static Sample sample(com.sun.management.ThreadMXBean bean,int count, + java.util.function.IntFunction loader) { + long thread=Thread.currentThread().threadId(),before=bean.getThreadAllocatedBytes(thread); + List result=new ArrayList<>(count); + for(int id=1;id<=count;id++)result.add(loader.apply(id)); + consumed=result; + return new Sample(result,bean.getThreadAllocatedBytes(thread)-before); + } + @Test void generatedReadbackSharesGeometryWithoutPerFieldOverflowCopies() { + var metadata=new SimpleEntityMetaFactory(); + var context=new DefaultUserContext(TeaQLRuntime.builder().metadata(metadata).queryExecutionLogging(false) + .build().install(GeneratedRuntimeModule.module())); + var descriptor=metadata.allEntityDescriptors().stream().filter(d->d.getType().equals("School")).findFirst().orElseThrow(); + var layout=FieldLayout.forType(descriptor.getTargetType());assertTrue(layout.indexes().size()>130); + List bindings=new ArrayList<>(); + for(var property:descriptor.getProperties()) { + EntityDescriptor target=metadata.allEntityDescriptors().stream() + .filter(d->d.getTargetType()==property.getType().javaType()).findFirst().orElse(null); + bindings.add(new Binding(property,SQLPropertyUtil.getColumns(property).get(0).getColumnName(),target)); + } + var database=new Database();database.rows=new ArrayList<>(10000); + for(int id=1;id<=10000;id++) { + Map row=new LinkedHashMap<>(); + for(var binding:bindings)row.put(binding.column(),binding.property().isId()?(long)id:value(binding.property())); + database.rows.add(row); + } + var repository=new PortableSQLRepository(descriptor,database,type->null,metadata); + java.util.function.IntFunction cached=id->repository.loadPersistedById(context,(long)id); + // Correct typed values, but reproduce the previous incremental availability algorithm. + java.util.function.IntFunction incremental=id->{ + BaseEntity entity=(BaseEntity)descriptor.createEntity();Map row=database.rows.get(id-1); + for(var binding:bindings) { + Object cell=row.get(binding.column()); + if(binding.target()!=null&&cell!=null) { + BaseEntity reference=(BaseEntity)binding.target().createEntity(); + reference.__internalSet("id",cell);reference.set$status(EntityStatus.REFER);cell=reference; + } + entity.setProperty(binding.property().getName(),cell); + } + entity.set$status(EntityStatus.PERSISTED);entity.clearUpdatedProperties();return entity; + }; + var bean=(com.sun.management.ThreadMXBean)ManagementFactory.getThreadMXBean(); + assertTrue(bean.isThreadAllocatedMemorySupported());bean.setThreadAllocatedMemoryEnabled(true); + for(int i=0;i<2000;i++){consumed=cached.apply(1);consumed=incremental.apply(1);} + for(int count:new int[]{1,100,10000}) { + var old=sample(bean,count,incremental);var current=sample(bean,count,cached); + System.out.printf("NATIVE_READBACK_ALLOC,%d,%d,%d%n",count,old.bytes(),current.bytes()); + assertTrue(current.bytes()getProperty("name"),row.getProperty("name")); + assertEquals(LocalDate.of(1995,9,1),row.getProperty("establishedDate")); + for(var binding:bindings) { + Object expected=reference.__internalGet(binding.property().getName()); + Object actual=row.__internalGet(binding.property().getName()); + if(expected instanceof Entity entity) { + assertTrue(actual instanceof Entity); + assertEquals(entity.getId(),((Entity)actual).getId()); + assertEquals(entity.getClass(),actual.getClass()); + } else assertEquals(expected,actual,binding.property().getName()); + } + assertTrue(row.getUpdatedProperties().isEmpty());assertFalse(row.__internalHasMutationLedger()); + } + } + System.out.println("PASS generated Java authoritative readback reuses one actual shape and avoids per-field overflow copies"); + } +} diff --git a/examples/shared-load-state/verify-allocations.sh b/examples/shared-load-state/verify-allocations.sh index 2a10eccb..23b2cda1 100644 --- a/examples/shared-load-state/verify-allocations.sh +++ b/examples/shared-load-state/verify-allocations.sh @@ -6,7 +6,7 @@ test_targets=LoadStateAllocationTest,HydrationAllocationTest wide=false case "${1:-}" in "") ;; - --wide) wide=true; test_targets+=,GeneratedWideHydrationAllocationTest ;; + --wide) wide=true; test_targets+=,GeneratedWideHydrationAllocationTest,GeneratedReadbackAllocationTest ;; *) printf 'Usage: bash verify-allocations.sh [--wide]\n' >&2; exit 2 ;; esac if (( $# > 1 )); then printf 'Unexpected arguments\n' >&2; exit 2; fi @@ -39,6 +39,9 @@ if [[ "$wide" == true ]]; then printf 'case,entity,selected_fields,rows,allocated_bytes,elapsed_ns\n' >"$run_dir/wide-hydration.csv" rg '^generated_wide_hydration,' "$run_dir/probe.log" >>"$run_dir/wide-hydration.csv" rg -F 'PASS generated Java wide hydration shares one overflow snapshot per shape' "$run_dir/probe.log" + printf 'case,rows,incremental_reference_bytes,current_bytes\n' >"$run_dir/native-readback.csv" + rg '^NATIVE_READBACK_ALLOC,' "$run_dir/probe.log" >>"$run_dir/native-readback.csv" + rg -F 'PASS generated Java authoritative readback reuses one actual shape and avoids per-field overflow copies' "$run_dir/probe.log" fi sha256sum --check --quiet "$run_dir/generated-source.sha256" printf 'PASS Java warmed load-state allocation probe\n' diff --git a/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLRepository.java b/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLRepository.java index fabf3d5c..625137df 100644 --- a/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLRepository.java +++ b/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLRepository.java @@ -91,6 +91,9 @@ * Reuses SQLRepository's SQL building logic (buildDataSQL, etc.). */ public class PortableSQLRepository implements SqlCompilerDelegate { + // Value-free, bounded hints for repeated authoritative reads of this fixed type. + // Neither row values, actor policy nor mutation ownership enters this cache. + private final MapProjection[] nativeReadbackProjections = new MapProjection[16]; private static final Pattern NAMED_PARAM = Pattern.compile(":(\\w+)"); public static final String CONTINUOUS_PAGE_PLAN = "teaql.continuousPage.plan"; @@ -1139,7 +1142,10 @@ T loadPersistedById(UserContext userContext, Long id, io.teaql.core.SqlIntentRed Map row = rows.get(0); // SELECT * returns physical column names, unlike the aliased normal Q path. // Install the actual geometry once before setters hydrate native values. - MapProjection projection = mapProjection(entityDescriptor.getTargetType(), row); + MapProjection projection; + synchronized (nativeReadbackProjections) { + projection = resolveMapProjection(entityDescriptor.getTargetType(), row, nativeReadbackProjections); + } if (entity instanceof BaseEntity base && projection.loadedState() != null) { base.__internalUseLoadState(projection.loadedState()); } diff --git a/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/NativeReadbackColumnMappingTest.java b/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/NativeReadbackColumnMappingTest.java index d84fa877..32f23ff6 100644 --- a/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/NativeReadbackColumnMappingTest.java +++ b/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/NativeReadbackColumnMappingTest.java @@ -24,6 +24,9 @@ public static final class Row extends BaseEntity { String customerReference; LocalDate establishedDate; Integer counterValue; + void updateReference(String value) { + handleUpdate("customerReference",customerReference,value);customerReference=value; + } @Override public Object __internalGet(String name) { return switch (name) { case "customerReference" -> customerReference; @@ -45,6 +48,9 @@ public static final class Row extends BaseEntity { record Fixture(PortableSQLDatabaseTest.SQLiteTeaQLDatabase database, PortableSQLRepository repository, DefaultUserContext context) {} private Fixture fixture() throws Exception { + return fixture(false); + } + private Fixture fixture(boolean preparedQueries) throws Exception { assertNotNull(Row.LAYOUT); var descriptor = new EntityDescriptor(); descriptor.setType("Row"); descriptor.setTargetType(Row.class); descriptor.setEntitySupplier(Row::new); @@ -63,7 +69,17 @@ private Fixture fixture() throws Exception { descriptor.setProperties(properties); var metadata = new SimpleEntityMetaFactory(); metadata.register(descriptor); var context = new DefaultUserContext(TeaQLRuntime.builder().metadata(metadata).queryExecutionLogging(false).build()); - var database = new PortableSQLDatabaseTest.SQLiteTeaQLDatabase(); + var database = new PortableSQLDatabaseTest.SQLiteTeaQLDatabase() { + @Override public List> query(String sql,Object[] args) { + if(preparedQueries && sql.startsWith("SELECT *") && args.length==1) { + Map row=new LinkedHashMap<>(); + row.put("id",args[0]);row.put("version",1L);row.put("external_ref","CR-"+args[0]); + row.put("started_on","2026-10-07");row.put("counter_value",null); + return List.of(row); + } + return super.query(sql,args); + } + }; var repository = new PortableSQLRepository(descriptor, database, type -> null, metadata); repository.ensurePhysicalSchema(context); return new Fixture(database, repository, context); @@ -87,4 +103,39 @@ private Fixture fixture() throws Exception { assertThrows(DateTimeParseException.class, () -> fixture.repository.loadPersistedById(fixture.context, 1L)); } } + @Test public void repeatedNativeReadbackSharesOnlyGeometryNotValuesOrMutationOwnership() throws Exception { + var fixture = fixture(); + fixture.database.executeUpdate("INSERT INTO native_readback_data(id,version,external_ref,STARTED_ON,counter_value) VALUES(?,?,?,?,?)", + new Object[]{1L,1L,"CR-A","2026-10-07",null}); + fixture.database.executeUpdate("INSERT INTO native_readback_data(id,version,external_ref,STARTED_ON,counter_value) VALUES(?,?,?,?,?)", + new Object[]{2L,1L,"CR-B","2026-10-08",0}); + var first=fixture.repository.loadPersistedById(fixture.context,1L); + var second=fixture.repository.loadPersistedById(fixture.context,2L); + assertSame(first.__internalLoadState(),second.__internalLoadState()); + assertEquals("CR-A",first.customerReference);assertEquals("CR-B",second.customerReference); + assertNull(first.counterValue);assertEquals(Integer.valueOf(0),second.counterValue); + first.updateReference("CR-C"); + assertSame(first.__internalLoadState(),second.__internalLoadState()); + assertEquals("CR-B",second.customerReference); + assertTrue(second.getUpdatedProperties().isEmpty());assertFalse(second.__internalHasMutationLedger()); + } + @Test public void concurrentColdReadbacksUseOneWinningImmutableShape() throws Exception { + var fixture=fixture(true); + var start=new java.util.concurrent.CountDownLatch(1); + var pool=java.util.concurrent.Executors.newFixedThreadPool(16); + try { + List> tasks=new ArrayList<>(); + for(long id=1;id<=16;id++) { + long requested=id; + tasks.add(pool.submit(()->{start.await();return fixture.repository.loadPersistedById(fixture.context,requested);})); + } + start.countDown();LoadState first=null; + for(int index=0;index Date: Wed, 7 Oct 2026 20:33:22 +0800 Subject: [PATCH 23/51] docs: define native readback allocation probe scope --- examples/shared-load-state/README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/examples/shared-load-state/README.md b/examples/shared-load-state/README.md index 6a078c36..2d280da2 100644 --- a/examples/shared-load-state/README.md +++ b/examples/shared-load-state/README.md @@ -124,6 +124,11 @@ changes still detach, and caller-owned sets cannot mutate the saved shape. The script retains separate state and four-field hydration CSV/logs. After generating a wide fixture, add `--wide` to measure actual generated School and optional Academy hydration at 3/64/full selected fields and 1/100/10,000 rows. +Wide mode also requires `GeneratedReadbackAllocationTest`: generated School +authoritative reads reuse value-free projection geometry. The prepared-map +probe compares gross thread allocation with a typed incremental-state reference +at 1/100/10,000 readbacks. It checks every value, shared state and private ledgers; +it does not measure complete saves, SQL I/O, live heap or old-version latency. This checks shared overflow snapshots, COW isolation and lazy ledgers, requires unchanged generated-source hashes, and rejects a non-wide artifact. Prepared inputs, JDBC and logging remain outside the measured hydration region. These From 44377968f1be86862ab579e99cf2644e46ae764d Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 7 Oct 2026 20:53:28 +0800 Subject: [PATCH 24/51] fix: reject incomplete authoritative native readback before commit --- .../GeneratedNativeRollbackAcceptance.java | 28 +++++++++++++++++++ .../ObservedJdbcExecutor.java | 21 +++++++++++++- examples/shared-load-state/verify.sh | 1 + .../sql/portable/PortableSQLRepository.java | 5 +++- .../NativeReadbackColumnMappingTest.java | 12 ++++++++ 5 files changed, 65 insertions(+), 2 deletions(-) diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedNativeRollbackAcceptance.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedNativeRollbackAcceptance.java index 1f678c5f..408fbf17 100644 --- a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedNativeRollbackAcceptance.java +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedNativeRollbackAcceptance.java @@ -53,6 +53,34 @@ static void verify(UserContext context, DataSource source, ObservedJdbcExecutor assertSame(snapshot, pending.__internalLoadState()); pending.updateName(name); pending.auditAs("Restore the native rollback fixture name").save(context); + verifyMissingColumn(context,sql,name,sibling); System.out.println("PASS generated Java LF11 native readback rollback retains loaded state and retry intent"); } + private static void verifyMissingColumn(UserContext context,ObservedJdbcExecutor sql,String name,String sibling) { + var rows=Q.schools().withNameIn(name,sibling).selectSelfFields().orderByIdAscending().limit(2) + .comment("what: load complete native-column controls") + .purpose("why: test authoritative omission without changing Q projection semantics").executeForList(context); + assertEquals(2,rows.size());var pending=rows.get(0);var held=rows.get(1); + var state=pending.__internalLoadState();Long version=pending.getVersion(),siblingVersion=held.getVersion(); + String next=name+" missing-column retry";pending.updateName(next); + sql.omitNativeDateAfterWrite(); + try { + var error=assertThrows(RuntimeException.class,()->pending.auditAs("Reject incomplete native readback").save(context)); + assertTrue(error.getMessage().contains("missing mapped field School.establishedDate")); + } finally {sql.clearNativeOmission();} + assertTrue(sql.nativeOmissionObserved(),"omission must happen after successful native DML"); + assertEquals(version,pending.getVersion());assertSame(state,pending.__internalLoadState()); + assertTrue(pending.getUpdatedProperties().contains("name")); + assertEquals(siblingVersion,held.getVersion());assertTrue(held.getUpdatedProperties().isEmpty()); + var stored=Q.schools().withIdIs(pending.getId()).selectSelfFields().limit(1) + .comment("what: inspect native state after omitted readback") + .purpose("why: verify rollback preserved name version and legitimate date").executeForOne(context); + assertEquals(name,E.school(stored).getName().eval());assertEquals(version,stored.getVersion()); + assertEquals(LocalDate.of(1995,9,1),E.school(stored).getEstablishedDate().eval()); + pending.auditAs("Retry the original intent after readback omission").save(context); + assertEquals(version+1,pending.getVersion());assertEquals(next,E.school(pending).getName().eval()); + assertSame(state,pending.__internalLoadState());assertTrue(pending.getUpdatedProperties().isEmpty()); + pending.updateName(name);pending.auditAs("Restore the missing-column fixture name").save(context); + System.out.println("PASS generated Java authoritative missing column rolls back without turning absence into null"); + } } diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/ObservedJdbcExecutor.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/ObservedJdbcExecutor.java index 0afc8b53..8bd87abc 100644 --- a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/ObservedJdbcExecutor.java +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/ObservedJdbcExecutor.java @@ -17,6 +17,9 @@ final class ObservedJdbcExecutor extends JdbcSqlExecutor { private final AtomicBoolean failDynamicReadback = new AtomicBoolean(); private final AtomicBoolean dynamicWriteSeen = new AtomicBoolean(); private final AtomicBoolean readbackFailureSeen = new AtomicBoolean(); + private final AtomicBoolean omitNativeColumn = new AtomicBoolean(); + private final AtomicBoolean nativeWriteSeen = new AtomicBoolean(); + private final AtomicBoolean nativeColumnOmissionSeen = new AtomicBoolean(); ObservedJdbcExecutor(DataSource source) { super(source); } /** Read, write and transaction entry; nested delegations may count more than once. */ long[] counts() { return new long[]{calls.get(0), calls.get(1), calls.get(2)}; } @@ -25,7 +28,23 @@ void failNextDynamicReadbackAfterWrite() { } boolean readbackFailureObserved() { return readbackFailureSeen.get(); } void clearReadbackFailure() { failDynamicReadback.set(false); } + void omitNativeDateAfterWrite() { + nativeWriteSeen.set(false);nativeColumnOmissionSeen.set(false);omitNativeColumn.set(true); + } + void clearNativeOmission() { omitNativeColumn.set(false); } + boolean nativeOmissionObserved() { return nativeColumnOmissionSeen.get(); } + private List> nativeReadbackView(String sql,List> rows) { + if(nativeWriteSeen.get() && sql.startsWith("SELECT *") && sql.contains("school_data") + && omitNativeColumn.compareAndSet(true,false)) { + nativeColumnOmissionSeen.set(true); + return rows.stream().map(row->{Map copy=new java.util.LinkedHashMap<>(row); + copy.remove("established_date");return copy;}).toList(); + } + return rows; + } private void observeDynamicWrite(String sql, int changed) { + if(changed>0 && omitNativeColumn.get() && sql.toLowerCase(java.util.Locale.ROOT).startsWith("update") + && sql.contains("school_data"))nativeWriteSeen.set(true); if (changed > 0 && failDynamicReadback.get() && sql.contains("teaql_dynamic_field_value")) dynamicWriteSeen.set(true); } private void rejectDynamicReadback(String sql) { @@ -53,7 +72,7 @@ private void rejectDynamicReadback(String sql) { calls.incrementAndGet(0); rejectDynamicReadback(sql); return super.queryForList(sql, params); } @Override public List> queryForList(String sql, Object[] params) { - calls.incrementAndGet(0); rejectDynamicReadback(sql); return super.queryForList(sql, params); + calls.incrementAndGet(0); rejectDynamicReadback(sql); return nativeReadbackView(sql,super.queryForList(sql, params)); } @Override public Map queryForMap(String sql, Map params) { calls.incrementAndGet(0); return super.queryForMap(sql, params); diff --git a/examples/shared-load-state/verify.sh b/examples/shared-load-state/verify.sh index 0f881389..32bcf2fc 100644 --- a/examples/shared-load-state/verify.sh +++ b/examples/shared-load-state/verify.sh @@ -87,6 +87,7 @@ for round in first second; do rg -F "PASS generated Java LF23 dynamic availability detaches only one view" "$run_dir/$round.log" rg -F "PASS generated Java LF20 readonly total persists only through modeled materialization" "$run_dir/$round.log" rg -F "PASS generated Java LF11 native readback rollback retains loaded state and retry intent" "$run_dir/$round.log" + rg -F "PASS generated Java authoritative missing column rolls back without turning absence into null" "$run_dir/$round.log" rg -F "PASS generated Java homogeneous list finalization allocates zero and preserves shared state" "$run_dir/$round.log" rg -F "PASS Java real typed stream loaded state and provider allocation controls" "$run_dir/$round.log" if [[ "$fixture_inheritance" == true ]]; then diff --git a/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLRepository.java b/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLRepository.java index 625137df..3da3e65b 100644 --- a/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLRepository.java +++ b/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLRepository.java @@ -1152,7 +1152,10 @@ T loadPersistedById(UserContext userContext, Long id, io.teaql.core.SqlIntentRed for (PropertyDescriptor property : this.allProperties) { if (!shouldHandle(property)) continue; String columnKey = findColumnKey(row, property); - if (columnKey == null) continue; + if (columnKey == null) { + throw new TeaQLRuntimeException("Authoritative readback missing mapped field " + + entityDescriptor.getType() + "." + property.getName()); + } Object value = row.get(columnKey); if (!(property instanceof Relation)) { Class targetType = property.getType().javaType(); diff --git a/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/NativeReadbackColumnMappingTest.java b/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/NativeReadbackColumnMappingTest.java index 32f23ff6..0c707b94 100644 --- a/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/NativeReadbackColumnMappingTest.java +++ b/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/NativeReadbackColumnMappingTest.java @@ -51,6 +51,9 @@ private Fixture fixture() throws Exception { return fixture(false); } private Fixture fixture(boolean preparedQueries) throws Exception { + return fixture(preparedQueries,Set.of()); + } + private Fixture fixture(boolean preparedQueries,Set omittedColumns) throws Exception { assertNotNull(Row.LAYOUT); var descriptor = new EntityDescriptor(); descriptor.setType("Row"); descriptor.setTargetType(Row.class); descriptor.setEntitySupplier(Row::new); @@ -75,6 +78,7 @@ private Fixture fixture(boolean preparedQueries) throws Exception { Map row=new LinkedHashMap<>(); row.put("id",args[0]);row.put("version",1L);row.put("external_ref","CR-"+args[0]); row.put("started_on","2026-10-07");row.put("counter_value",null); + omittedColumns.forEach(row::remove); return List.of(row); } return super.query(sql,args); @@ -138,4 +142,12 @@ private Fixture fixture(boolean preparedQueries) throws Exception { } } finally {pool.shutdownNow();} } + @Test public void missingAuthoritativeColumnCannotMasqueradeAsLoadedNullOrOldValue() throws Exception { + for(String missing:List.of("id","version","started_on","counter_value","external_ref")) { + var fixture=fixture(true,Set.of(missing)); + var failure=assertThrows(TeaQLRuntimeException.class, + ()->fixture.repository.loadPersistedById(fixture.context,1L)); + assertTrue(failure.getMessage().contains("missing")); + } + } } From d9dc8977e01d250cffcfbcdc62ef3540efa007af Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 7 Oct 2026 22:31:14 +0800 Subject: [PATCH 25/51] feat: share readonly property schema through Java result snapshots --- .../GeneratedPropertyMetadataAcceptance.java | 41 +++++++++++++ .../GeneratedSchoolLoadStateTest.java | 1 + examples/shared-load-state/verify.sh | 1 + .../main/java/io/teaql/core/BaseEntity.java | 17 +++++- .../main/java/io/teaql/core/BaseRequest.java | 6 ++ .../teaql/core/DynamicPropertyMetadata.java | 41 +++++++++++++ .../src/main/java/io/teaql/core/Entity.java | 3 + .../main/java/io/teaql/core/LoadState.java | 29 ++++++++-- .../java/io/teaql/core/SearchRequest.java | 1 + .../core/DynamicPropertyMetadataTest.java | 57 +++++++++++++++++++ .../io/teaql/core/internal/TempRequest.java | 1 + .../sql/portable/PortableSQLRepository.java | 25 +++++++- .../sql/portable/PortableSQLDatabaseTest.java | 19 +++++++ 13 files changed, 232 insertions(+), 10 deletions(-) create mode 100644 examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedPropertyMetadataAcceptance.java create mode 100644 teaql-core/src/main/java/io/teaql/core/DynamicPropertyMetadata.java create mode 100644 teaql-core/src/test/java/io/teaql/core/DynamicPropertyMetadataTest.java diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedPropertyMetadataAcceptance.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedPropertyMetadataAcceptance.java new file mode 100644 index 00000000..130b51a4 --- /dev/null +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedPropertyMetadataAcceptance.java @@ -0,0 +1,41 @@ +package com.example.schoolmanagementservice; + +import io.teaql.core.DynamicPropertyMetadata; +import io.teaql.core.UserContext; +import java.util.Map; +import static org.junit.jupiter.api.Assertions.*; + +final class GeneratedPropertyMetadataAcceptance { + static void verify(UserContext context, String first, String second) { + var request = Q.schoolsWithMinimalFields().withNameIn(first,second).selectName().orderByIdAscending().limit(2); + var metadata = new DynamicPropertyMetadata(Map.of("_missing_count",Long.class,"_count",Number.class,"_null",String.class)); + request.addSimpleDynamicProperty("_count",new io.teaql.core.PropertyReference("studentCapacity")); + String nullableMember=io.teaql.core.FieldLayout.forType(com.example.schoolmanagementservice.school.School.class).memberName("probe_0"); + if(nullableMember!=null) request.addSimpleDynamicProperty("_null",new io.teaql.core.PropertyReference(nullableMember)); + request.setDynamicPropertyMetadata(metadata); + var rows = request.comment("what: load generated School with a readonly schema") + .purpose("why: keep schema sharing distinct from value presence").executeForList(context); + assertEquals(2,rows.size()); + assertEquals(first,E.school(rows.first()).getName().eval()); + assertSame(rows.get(0).__internalLoadState(),rows.get(1).__internalLoadState()); + for(var row:rows.getData()) { + assertSame(metadata,row.__internalLoadState().dynamicPropertyMetadata()); + assertSame(Long.class,row.getDynamicPropertyType("_missing_count")); + assertNull(row.getDynamicProperty("_missing_count")); + assertFalse(row.hasDynamicProperty("_missing_count")); + assertFalse(row.isPropertyLoaded("_missing_count")); + assertTrue(row.getUpdatedProperties().isEmpty()); + assertEquals(0,((Number)row.getDynamicProperty("_count")).intValue()); + assertTrue(row.hasDynamicProperty("_count")); + assertNull(row.getDynamicProperty("_null")); + assertEquals(nullableMember!=null,row.hasDynamicProperty("_null")); + } + try(var stream=request.comment("what: stream generated School readonly schema") + .purpose("why: retain schema after cursor cleanup").executeForStream(context)) { + var streamed=stream.toList();assertEquals(2,streamed.size()); + assertSame(streamed.get(0).__internalLoadState(),streamed.get(1).__internalLoadState()); + assertSame(Long.class,streamed.get(0).getDynamicPropertyType("_missing_count")); + } + System.out.println("PASS generated Java readonly property metadata shares schema without installing values or fixed slots"); + } +} diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java index 38b809b3..3af363b6 100644 --- a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java @@ -143,6 +143,7 @@ void generatedSchoolUsesLocalRuntimeAndSharedState() throws Exception { assertEquals(1001L, full.get(0).getSchoolType().getId()); GeneratedFieldOrderAcceptance.verify(context, dataSource, full.get(0).getId()); GeneratedPageStreamAcceptance.verify(context, first, second); + GeneratedPropertyMetadataAcceptance.verify(context, first, second); var sparse = Q.schoolsWithMinimalFields().withNameIn(first, second).selectName() .orderByIdAscending().limit(2).comment("what: read matching sparse projections") diff --git a/examples/shared-load-state/verify.sh b/examples/shared-load-state/verify.sh index 32bcf2fc..148058ce 100644 --- a/examples/shared-load-state/verify.sh +++ b/examples/shared-load-state/verify.sh @@ -72,6 +72,7 @@ for round in first second; do rg -F "PASS generated Java sparse Checker rejects before provider entry" "$run_dir/$round.log" rg -F "PASS generated Java Q selection and real JDBC column-order invariance" "$run_dir/$round.log" rg -F "PASS generated Java page and stream shared load state" "$run_dir/$round.log" + rg -F "PASS generated Java readonly property metadata shares schema without installing values or fixed slots" "$run_dir/$round.log" rg -F "PASS generated Java dynamic stream Value/Null/NotLoaded and shared snapshots" "$run_dir/$round.log" rg -F "PASS generated Java Checker preserves NotLoaded and sibling load boundaries" "$run_dir/$round.log" rg -F "PASS generated Java dynamic storage provenance and retry" "$run_dir/$round.log" diff --git a/teaql-core/src/main/java/io/teaql/core/BaseEntity.java b/teaql-core/src/main/java/io/teaql/core/BaseEntity.java index aae1a1f8..d2fa2f4f 100644 --- a/teaql-core/src/main/java/io/teaql/core/BaseEntity.java +++ b/teaql-core/src/main/java/io/teaql/core/BaseEntity.java @@ -306,7 +306,9 @@ private void addRelation( @Override public void addDynamicProperty(String propertyName, Object value) { - mutableAdditionalInfo().put(dynamicPropertyNameOf(propertyName), value); + String key = dynamicPropertyNameOf(propertyName); + if (loadState.dynamicPropertyMetadata() != null) loadState.dynamicPropertyMetadata().validate(key, value); + mutableAdditionalInfo().put(key, value); } @Override @@ -326,6 +328,16 @@ public T getDynamicProperty(String propertyName) { return (T) additionalInfo.get(dynamicPropertyNameOf(propertyName)); } + public boolean hasDynamicProperty(String propertyName) { + return additionalInfo.containsKey(dynamicPropertyNameOf(propertyName)); + } + + @Override + public Class getDynamicPropertyType(String propertyName) { + DynamicPropertyMetadata metadata = loadState.dynamicPropertyMetadata(); + return metadata == null ? null : metadata.type(dynamicPropertyNameOf(propertyName)); + } + private String dynamicPropertyNameOf(String propertyName) { if (propertyName == null || propertyName.isBlank() || propertyName.startsWith("#")) { throw new IllegalArgumentException("Dynamic properties use '_' names; '#' belongs to persistent dynamic fields"); @@ -487,6 +499,9 @@ public void __internalUseLoadState(LoadState state) { if (state == null || state.layout() != FieldLayout.forType(getClass())) { throw new IllegalArgumentException("Incompatible entity load-state layout: " + typeName()); } + if (state.dynamicPropertyMetadata() != null) { + additionalInfo.forEach((name, value) -> state.dynamicPropertyMetadata().validate(name, value)); + } loadState = state; } diff --git a/teaql-core/src/main/java/io/teaql/core/BaseRequest.java b/teaql-core/src/main/java/io/teaql/core/BaseRequest.java index aebc5d16..98abe986 100644 --- a/teaql-core/src/main/java/io/teaql/core/BaseRequest.java +++ b/teaql-core/src/main/java/io/teaql/core/BaseRequest.java @@ -31,6 +31,10 @@ public abstract class BaseRequest implements SearchRequest protected String comment; protected String purpose; + protected DynamicPropertyMetadata dynamicPropertyMetadata; + + @Override public DynamicPropertyMetadata getDynamicPropertyMetadata() { return dynamicPropertyMetadata; } + public void setDynamicPropertyMetadata(DynamicPropertyMetadata metadata) { dynamicPropertyMetadata = metadata; } // select properties protected List projections = new ArrayList<>(); @@ -925,6 +929,7 @@ public boolean equals(Object pO) { if (!(pO instanceof BaseRequest that)) return false; return Objects.equals(getProjections(), that.getProjections()) && Objects.equals(getSimpleDynamicProperties(), that.getSimpleDynamicProperties()) + && Objects.equals(getDynamicPropertyMetadata(), that.getDynamicPropertyMetadata()) && Objects.equals(getSearchCriteria(), that.getSearchCriteria()) && Objects.equals(orderBys, that.orderBys) && Objects.equals(getSlice(), that.getSlice()) @@ -946,6 +951,7 @@ public int hashCode() { return Objects.hash( getProjections(), getSimpleDynamicProperties(), + getDynamicPropertyMetadata(), getSearchCriteria(), orderBys, getSlice(), diff --git a/teaql-core/src/main/java/io/teaql/core/DynamicPropertyMetadata.java b/teaql-core/src/main/java/io/teaql/core/DynamicPropertyMetadata.java new file mode 100644 index 00000000..73273896 --- /dev/null +++ b/teaql-core/src/main/java/io/teaql/core/DynamicPropertyMetadata.java @@ -0,0 +1,41 @@ +package io.teaql.core; + +import java.util.Map; +import java.util.Objects; + +/** Immutable readonly property names/types; no values, presence markers or fixed slots. */ +public final class DynamicPropertyMetadata { + private final Map> types; + private final int hash; + + public DynamicPropertyMetadata(Map> types) { + for (Map.Entry> entry : types.entrySet()) { + String name = entry.getKey(); + if (name == null || !name.startsWith("_") || name.substring(1).isBlank()) { + throw new IllegalArgumentException("Dynamic property metadata requires '_' names"); + } + Class type = Objects.requireNonNull(entry.getValue(), "dynamic property type"); + if (type.isPrimitive()) { + throw new IllegalArgumentException("Dynamic property metadata requires boxed types"); + } + } + this.types = Map.copyOf(types); + this.hash = this.types.hashCode(); + } + + public Map> types() { return types; } + public Class type(String name) { return types.get(name); } + + /** Validation never prints the value; absence is not validated as a NULL assignment. */ + public void validate(String name, Object value) { + Class type = type(name); + if (type != null && value != null && !type.isInstance(value)) { + throw new IllegalArgumentException("Invalid dynamic property type: " + name); + } + } + + @Override public boolean equals(Object other) { + return this == other || other instanceof DynamicPropertyMetadata metadata && types.equals(metadata.types); + } + @Override public int hashCode() { return hash; } +} diff --git a/teaql-core/src/main/java/io/teaql/core/Entity.java b/teaql-core/src/main/java/io/teaql/core/Entity.java index 9aca8060..47659760 100644 --- a/teaql-core/src/main/java/io/teaql/core/Entity.java +++ b/teaql-core/src/main/java/io/teaql/core/Entity.java @@ -5,6 +5,9 @@ // the super interface in TEAQL repository public interface Entity { + /** Optional readonly result-schema information, not an availability or persistence contract. */ + default Class getDynamicPropertyType(String name) { return null; } + Long getId(); Long getVersion(); diff --git a/teaql-core/src/main/java/io/teaql/core/LoadState.java b/teaql-core/src/main/java/io/teaql/core/LoadState.java index 5d2baf19..ba72722c 100644 --- a/teaql-core/src/main/java/io/teaql/core/LoadState.java +++ b/teaql-core/src/main/java/io/teaql/core/LoadState.java @@ -12,6 +12,7 @@ public final class LoadState { private final Set overflow; // Dynamic field selection, named relation materialization and unindexed adapter fields. private final Set selectedNames; + private final DynamicPropertyMetadata dynamicPropertyMetadata; private final int hash; // Value-free memoization shared by the snapshot, never a per-entity payload. private volatile String[] dynamicCodes; @@ -22,15 +23,22 @@ private LoadState(FieldLayout layout, long bits, Set overflow, Set overflow, Set selectedNames, String[] dynamicCodes) { + this(layout, bits, overflow, selectedNames, dynamicCodes, null); + } + + private LoadState(FieldLayout layout, long bits, Set overflow, Set selectedNames, + String[] dynamicCodes, DynamicPropertyMetadata metadata) { this.layout = Objects.requireNonNull(layout, "layout"); this.bits = bits; this.overflow = overflow; this.selectedNames = selectedNames; this.dynamicCodes = dynamicCodes; + this.dynamicPropertyMetadata = metadata; int result = System.identityHashCode(layout); result = 31 * result + Long.hashCode(bits); result = 31 * result + overflow.hashCode(); - hash = 31 * result + selectedNames.hashCode(); + result = 31 * result + selectedNames.hashCode(); + hash = 31 * result + Objects.hashCode(metadata); } static LoadState empty(FieldLayout layout) { @@ -63,6 +71,14 @@ else if (index >= Long.SIZE) { public Set overflow() { return overflow; } public Set selectedNames() { return selectedNames; } + public DynamicPropertyMetadata dynamicPropertyMetadata() { return dynamicPropertyMetadata; } + + /** Shared schema only: never marks a readonly property as loaded or present. */ + public LoadState withDynamicPropertyMetadata(DynamicPropertyMetadata metadata) { + if (Objects.equals(dynamicPropertyMetadata, metadata)) return this; + return new LoadState(layout, bits, overflow, selectedNames, dynamicCodes, metadata); + } + /** Build once for a dynamic-field batch shape, preserving fixed slots and relation markers. */ public LoadState withDynamicSelection(Set codes) { // A custom sorted comparator may equate different literal field codes. @@ -82,7 +98,7 @@ public LoadState withDynamicSelection(Set codes) { Set names = new HashSet<>(selectedNames); names.removeIf(name -> name.startsWith("#")); for (String code : stableCodes) names.add("#" + Objects.requireNonNull(code, "dynamic code")); - return new LoadState(layout, bits, overflow, Set.copyOf(names), stableCodes); + return new LoadState(layout, bits, overflow, Set.copyOf(names), stableCodes, dynamicPropertyMetadata); } private String[] dynamicCodes() { @@ -113,16 +129,16 @@ public LoadState withLoaded(String field, boolean loaded) { if (isLoaded(field) == loaded) return this; if (index >= 0 && index < Long.SIZE) { long next = loaded ? bits | (1L << index) : bits & ~(1L << index); - return new LoadState(layout, next, overflow, selectedNames, dynamicCodes); + return new LoadState(layout, next, overflow, selectedNames, dynamicCodes, dynamicPropertyMetadata); } if (index >= Long.SIZE) { Set next = new HashSet<>(overflow); if (loaded) next.add(index); else next.remove(index); - return new LoadState(layout, bits, Set.copyOf(next), selectedNames, dynamicCodes); + return new LoadState(layout, bits, Set.copyOf(next), selectedNames, dynamicCodes, dynamicPropertyMetadata); } Set next = new HashSet<>(selectedNames); if (loaded) next.add(field); else next.remove(field); - return new LoadState(layout, bits, overflow, Set.copyOf(next)); + return new LoadState(layout, bits, overflow, Set.copyOf(next), null, dynamicPropertyMetadata); } @Override @@ -133,6 +149,7 @@ public boolean equals(Object other) { if (this == other) return true; if (!(other instanceof LoadState state)) return false; return layout == state.layout && bits == state.bits - && overflow.equals(state.overflow) && selectedNames.equals(state.selectedNames); + && overflow.equals(state.overflow) && selectedNames.equals(state.selectedNames) + && Objects.equals(dynamicPropertyMetadata, state.dynamicPropertyMetadata); } } diff --git a/teaql-core/src/main/java/io/teaql/core/SearchRequest.java b/teaql-core/src/main/java/io/teaql/core/SearchRequest.java index 3b8c5988..707da0a6 100644 --- a/teaql-core/src/main/java/io/teaql/core/SearchRequest.java +++ b/teaql-core/src/main/java/io/teaql/core/SearchRequest.java @@ -11,6 +11,7 @@ import io.teaql.data.dynamic.DynamicFieldSelection; public interface SearchRequest { + default DynamicPropertyMetadata getDynamicPropertyMetadata() { return null; } int DEFAULT_HARD_LIMIT = 10_000; default String getTypeName() { String simpleName = this.getClass().getSimpleName(); diff --git a/teaql-core/src/test/java/io/teaql/core/DynamicPropertyMetadataTest.java b/teaql-core/src/test/java/io/teaql/core/DynamicPropertyMetadataTest.java new file mode 100644 index 00000000..efd9ac5a --- /dev/null +++ b/teaql-core/src/test/java/io/teaql/core/DynamicPropertyMetadataTest.java @@ -0,0 +1,57 @@ +package io.teaql.core; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.junit.Test; +import static org.junit.Assert.*; + +public class DynamicPropertyMetadataTest { + static class Row extends BaseEntity { + static final FieldLayout LAYOUT = FieldLayout.installGenerated(FieldLayout.generated( + Row.class, "metadata-v1", Map.of("id",0,"version",1,"name",2), + Map.of("id",List.of("id","id"),"version",List.of("version","version"),"name",List.of("name","name")), Set.of())); + String name; + @Override public Object __internalGet(String field) { return "name".equals(field) ? name : super.__internalGet(field); } + @Override public void __internalSet(String field,Object value) { + if("name".equals(field)) { name=(String)value;markPropertyLoaded(field); } else super.__internalSet(field,value); + } + } + @Test public void sharedSchemaDoesNotInstallValuesOrNumberReadonlyProperties() { + DynamicPropertyMetadata metadata = new DynamicPropertyMetadata(Map.of("_count",Long.class,"_missing",Long.class)); + LoadState state = LoadState.projection(Row.LAYOUT,List.of("id","version","name")).withDynamicPropertyMetadata(metadata); + Row first=new Row();Row second=new Row(); + first.__internalUseLoadState(state);second.__internalUseLoadState(state); + first.addDynamicProperty("count",0L);second.addDynamicProperty("count",null); + SmartList rows=new SmartList<>(List.of(first,second)); + assertSame(first.__internalLoadState(),second.__internalLoadState()); + assertSame(metadata,first.__internalLoadState().dynamicPropertyMetadata()); + assertSame(Long.class,first.getDynamicPropertyType("missing")); + assertNull(first.getDynamicProperty("missing"));assertFalse(first.getAdditionalInfo().containsKey("_missing")); + assertEquals(Long.valueOf(0),first.getDynamicProperty("count"));assertNull(second.getDynamicProperty("count")); + assertTrue(second.getAdditionalInfo().containsKey("_count")); + assertFalse(state.isLoaded("_count"));assertNull(Row.LAYOUT.findIndex("_count")); + assertTrue(first.getUpdatedProperties().isEmpty());assertEquals(0,first.collectDynamicFieldValues().size()); + assertEquals(2,rows.size()); + } + @Test public void incompatibleSchemasNeverShareAndAvailabilityChangesKeepMetadata() { + LoadState original=LoadState.projection(Row.LAYOUT,List.of("id")) + .withDynamicPropertyMetadata(new DynamicPropertyMetadata(Map.of("_count",Long.class))); + LoadState other=LoadState.projection(Row.LAYOUT,List.of("id")) + .withDynamicPropertyMetadata(new DynamicPropertyMetadata(Map.of("_count",String.class))); + assertNotEquals(original,other); + LoadState changed=original.withLoaded("name",true).withDynamicSelection(Set.of("note")); + assertSame(original.dynamicPropertyMetadata(),changed.dynamicPropertyMetadata()); + assertFalse(original.isLoaded("name"));assertTrue(changed.isLoaded("name")); + assertFalse(changed.isLoaded("_count")); + } + @Test public void invalidTypesAndNamespacesRejectWithoutExposingValues() { + assertThrows(IllegalArgumentException.class,()->new DynamicPropertyMetadata(Map.of("name",String.class))); + assertThrows(IllegalArgumentException.class,()->new DynamicPropertyMetadata(Map.of("#count",Long.class))); + assertThrows(IllegalArgumentException.class,()->new DynamicPropertyMetadata(Map.of("_count",long.class))); + DynamicPropertyMetadata metadata=new DynamicPropertyMetadata(Map.of("_count",Long.class)); + Exception error=assertThrows(IllegalArgumentException.class,()->metadata.validate("_count","secret-value")); + assertFalse(error.getMessage().contains("secret-value")); + assertThrows(UnsupportedOperationException.class,()->metadata.types().put("_other",String.class)); + } +} diff --git a/teaql-sql-portable/src/main/java/io/teaql/core/internal/TempRequest.java b/teaql-sql-portable/src/main/java/io/teaql/core/internal/TempRequest.java index 51ba8ba3..ca98ad7e 100644 --- a/teaql-sql-portable/src/main/java/io/teaql/core/internal/TempRequest.java +++ b/teaql-sql-portable/src/main/java/io/teaql/core/internal/TempRequest.java @@ -41,6 +41,7 @@ public TempRequest(Class returnType, String typeName) { private void copy(SearchRequest pRequest) { projections.addAll(pRequest.getProjections()); simpleDynamicProperties.addAll(pRequest.getSimpleDynamicProperties()); + dynamicPropertyMetadata = pRequest.getDynamicPropertyMetadata(); searchCriteria = pRequest.getSearchCriteria(); orderBys = pRequest.getOrderBy(); slice = pRequest.getSlice(); diff --git a/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLRepository.java b/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLRepository.java index 3da3e65b..b38c95c6 100644 --- a/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLRepository.java +++ b/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLRepository.java @@ -794,6 +794,10 @@ private SmartList loadWithIntent(UserContext userContext, SearchRequest re smartList.add(mapRowToEntity(userContext, executedRequest, row, projection, dynamicProperties)); } } + if (request.getDynamicPropertyMetadata() != null) { + Map states = new HashMap<>(); + for (T entity : smartList.getData()) attachPropertyMetadata(entity, request.getDynamicPropertyMetadata(), states); + } registerContinuousPage(userContext, request, pageExecution, smartList.getData()); if (idSetExecution.optimized()) { Map positions = new HashMap<>(); @@ -1192,16 +1196,31 @@ public Stream streamInternal(UserContext userContext, SearchRequest reques io.teaql.core.CompiledRowMapper mapper = extension instanceof io.teaql.core.CompiledRowMapper supplied ? (io.teaql.core.CompiledRowMapper) supplied : compileRowMapper(request); if (mapper != null) { - return database.queryForStream(userContext, psql.sql, psql.args, mapper, psql.logBindings); + return withPropertyMetadata(database.queryForStream(userContext, psql.sql, psql.args, mapper, psql.logBindings), request.getDynamicPropertyMetadata()); } } MapProjection[] projections = new MapProjection[16]; SimpleNamedExpression[] dynamicProperties = request.getSimpleDynamicProperties().toArray(SimpleNamedExpression[]::new); - return database.queryForStream(userContext, psql.sql, psql.args, psql.logBindings) + return withPropertyMetadata(database.queryForStream(userContext, psql.sql, psql.args, psql.logBindings) .map(row -> { MapProjection shape = resolveMapProjection(request.returnType(), row, projections); return mapRowToEntity(userContext, request, row, shape, dynamicProperties); - }); + }), request.getDynamicPropertyMetadata()); + } + + private void attachPropertyMetadata(T entity, io.teaql.core.DynamicPropertyMetadata metadata, + Map states) { + if (!(entity instanceof BaseEntity base)) { + throw new TeaQLRuntimeException("Readonly property metadata requires a runtime-owned entity carrier"); + } + base.__internalUseLoadState(states.computeIfAbsent(base.__internalLoadState(), + shape -> shape.withDynamicPropertyMetadata(metadata))); + } + + private Stream withPropertyMetadata(Stream source, io.teaql.core.DynamicPropertyMetadata metadata) { + if (metadata == null) return source; + Map states = new HashMap<>(); + return source.map(entity -> { attachPropertyMetadata(entity, metadata, states); return entity; }); } private record MapBinding(PropertyDescriptor property, String column) {} diff --git a/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/PortableSQLDatabaseTest.java b/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/PortableSQLDatabaseTest.java index 7bdcc2ef..6cc8408e 100644 --- a/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/PortableSQLDatabaseTest.java +++ b/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/PortableSQLDatabaseTest.java @@ -157,6 +157,25 @@ public void testSelectedSqlNullIsMappedAsLoadedNull() { assertTrue("selected SQL NULL must still invoke the entity mapper", loaded.isLoaded("title")); } + @Test public void testReadonlyPropertySchemaSurvivesQueryWithoutInstallingMissingValues() { + sqliteDb.executeUpdate("INSERT INTO task_data (id, version, title, status) VALUES (?, ?, ?, ?)", + new Object[]{901L,1L,"schema control","PROPERTY-SCHEMA"}); + TaskRequest request = new TaskRequest().filterByStatus("PROPERTY-SCHEMA"); + request.setDynamicPropertyMetadata(new DynamicPropertyMetadata(Map.of("_missing",Long.class))); + SmartList rows = request.comment("load a readonly result-schema control") + .purpose("keep missing property distinct from a computed zero").executeForList(context); + assertEquals(1, rows.size()); + Task row = rows.get(0); + assertSame(Long.class,row.getDynamicPropertyType("missing")); + assertNull(row.getDynamicProperty("missing"));assertFalse(row.hasDynamicProperty("missing")); + assertFalse(row.isPropertyLoaded("_missing"));assertTrue(row.getUpdatedProperties().isEmpty()); + try (var stream = sqlDataService.getRepository("Task").streamInternal(context,request)) { + Task streamed = stream.findFirst().orElseThrow(); + assertSame(Long.class,streamed.getDynamicPropertyType("missing")); + assertFalse(streamed.hasDynamicProperty("missing")); + } + } + @Test public void TOPN_001_TO_009_windowAndProbePlansAreEquivalentOnSQLite() { registerTopNFixture(); From 84eab542065014272d51cf456da52c493895d98c Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 7 Oct 2026 23:24:43 +0800 Subject: [PATCH 26/51] test: verify each bitmap overflow slot and exact dynamic wrapper values --- .../src/test/java/io/teaql/core/LoadStateTest.java | 11 +++++++++++ .../data/dynamic/DynamicFieldValueStateTest.java | 3 +++ 2 files changed, 14 insertions(+) diff --git a/teaql-core/src/test/java/io/teaql/core/LoadStateTest.java b/teaql-core/src/test/java/io/teaql/core/LoadStateTest.java index f91a03cc..a09e4526 100644 --- a/teaql-core/src/test/java/io/teaql/core/LoadStateTest.java +++ b/teaql-core/src/test/java/io/teaql/core/LoadStateTest.java @@ -204,7 +204,18 @@ public void generatedPositionsAreIndependentOfAccessAndConcurrency() { @Test public void bitmapBoundariesAndOverflowNeverWrap() { FieldLayout layout = FieldLayout.forType(IndexedEntity.class); + for (int slot : new int[]{0, 31, 32, 63, 64, 65, 129}) { + String selected = slot == 0 ? "id" : "field_" + slot; + LoadState isolated = LoadState.projection(layout, List.of(selected)); + assertEquals("exact bit for slot " + slot, slot < 64 ? 1L << slot : 0L, isolated.bits()); + assertEquals(slot < 64 ? Set.of() : Set.of(slot), isolated.overflow()); + for (int other : new int[]{0, 31, 32, 63, 64, 65, 129}) { + assertEquals("independent slot " + other, slot == other, + isolated.isLoaded(other == 0 ? "id" : "field_" + other)); + } + } LoadState state = LoadState.projection(layout, List.of("id", "field_31", "field_32", "field_63")); + assertEquals(1L | (1L << 31) | (1L << 32) | (1L << 63), state.bits()); assertTrue(state.bits() < 0L); assertTrue(state.overflow().isEmpty()); assertFalse(state.isLoaded("field_64")); diff --git a/teaql-dynamic-fields-api/src/test/java/io/teaql/data/dynamic/DynamicFieldValueStateTest.java b/teaql-dynamic-fields-api/src/test/java/io/teaql/data/dynamic/DynamicFieldValueStateTest.java index d1346511..4a5c6849 100644 --- a/teaql-dynamic-fields-api/src/test/java/io/teaql/data/dynamic/DynamicFieldValueStateTest.java +++ b/teaql-dynamic-fields-api/src/test/java/io/teaql/data/dynamic/DynamicFieldValueStateTest.java @@ -25,6 +25,9 @@ public void boundDefinitionsShareTheDictionaryButKeepStorageAndScopeIdentity() { @Test public void zeroFalseEmptyNullAndNotLoadedRemainDistinct() { + assertEquals(0, DynamicFieldValue.ofNumber("number", 0).value()); + assertEquals(Boolean.FALSE, DynamicFieldValue.ofBool("flag", false).value()); + assertEquals("", DynamicFieldValue.ofString("text", "").value()); for (DynamicFieldValue value : new DynamicFieldValue[]{ DynamicFieldValue.ofNumber("number", 0), DynamicFieldValue.ofBool("flag", false), DynamicFieldValue.ofString("text", "")}) { From 45ad7f6ab78dd6bc8854d4ef56aa4641e6c695ee Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 7 Oct 2026 23:40:32 +0800 Subject: [PATCH 27/51] test: reject cross-type masks and ambiguous fixed-field aliases --- .../java/io/teaql/core/LoadStateTest.java | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/teaql-core/src/test/java/io/teaql/core/LoadStateTest.java b/teaql-core/src/test/java/io/teaql/core/LoadStateTest.java index a09e4526..df2a5af2 100644 --- a/teaql-core/src/test/java/io/teaql/core/LoadStateTest.java +++ b/teaql-core/src/test/java/io/teaql/core/LoadStateTest.java @@ -124,6 +124,12 @@ public Object __internalGet(String property) { static final class LateLayoutEntity extends BaseEntity { } + static final class ForeignLayoutEntity extends BaseEntity { + static final FieldLayout LAYOUT = FieldLayout.installGenerated(FieldLayout.generated( + ForeignLayoutEntity.class, "load-state-fixture-v1", IndexedEntity.__TEAQL_FIXED_FIELD_INDEXES, + IndexedEntity.__TEAQL_FIXED_FIELD_MAPPINGS, IndexedEntity.__TEAQL_FIXED_RELATION_NAMES)); + } + private static final java.util.concurrent.atomic.AtomicInteger COLD_INSTALLATIONS = new java.util.concurrent.atomic.AtomicInteger(); @@ -309,6 +315,53 @@ public void hydrateIndexMismatchFailsBeforeWritingAndUnknownNamesCannotCorruptBi assertEquals(0L, row.__internalLoadState().bits()); } + @Test + public void canonicalMemberAndPhysicalAliasesShareOneBitWithoutCorruptingOtherFields() { + var layout = IndexedEntity.__TEAQL_FIELD_LAYOUT; + for (String alias : List.of("base_url", "baseUrl", "legacy_url")) { + var state = LoadState.projection(layout, List.of(alias)); + assertEquals(1L << 2, state.bits()); + for (String spelling : List.of("base_url", "baseUrl", "legacy_url")) { + assertTrue(state.isLoaded(spelling)); + assertSame(state, state.withLoaded(spelling, true)); + } + assertFalse(state.isLoaded("id")); + assertFalse(state.isLoaded("version")); + assertThrows(IllegalArgumentException.class, () -> state.withLoaded("base_ur1", true)); + assertEquals(1L << 2, state.bits()); + } + assertThrows(IllegalArgumentException.class, () -> FieldLayout.generated(IndexedEntity.class, "ambiguous", + Map.of("id", 0, "version", 1, "base_url", 2), + Map.of("id", List.of("id", "id"), "version", List.of("version", "version"), + "base_url", List.of("baseUrl", "id")), Set.of())); + } + + @Test + public void identicalBitPatternsCannotCrossEntityTypesOrModelRevisions() { + var first = new IndexedEntity(); + var second = new ForeignLayoutEntity(); + var firstState = LoadState.projection(IndexedEntity.__TEAQL_FIELD_LAYOUT, List.of("id", "field_64")); + var secondState = LoadState.projection(ForeignLayoutEntity.LAYOUT, List.of("id", "field_64")); + first.__internalUseLoadState(firstState); + second.__internalUseLoadState(secondState); + assertEquals(firstState.bits(), secondState.bits()); + assertEquals(firstState.overflow(), secondState.overflow()); + assertNotEquals(firstState, secondState); + assertThrows(IllegalArgumentException.class, () -> first.__internalUseLoadState(secondState)); + assertThrows(IllegalArgumentException.class, () -> second.__internalUseLoadState(firstState)); + var revision = FieldLayout.generated(IndexedEntity.class, "revision-v2", + IndexedEntity.__TEAQL_FIXED_FIELD_INDEXES, IndexedEntity.__TEAQL_FIXED_FIELD_MAPPINGS, + IndexedEntity.__TEAQL_FIXED_RELATION_NAMES); + assertThrows(IllegalArgumentException.class, () -> first.__internalUseLoadState( + LoadState.projection(revision, List.of("id", "field_64")))); + List rows = new ArrayList<>(List.of(first, second)); + SmartList.takeOwnership(rows); + assertSame(firstState, first.__internalLoadState()); + assertSame(secondState, second.__internalLoadState()); + assertFalse(first.__internalHasMutationLedger()); + assertFalse(second.__internalHasMutationLedger()); + } + @Test public void compiledProjectionIsSharedWithoutDirtyHydrationAndRollbackIsPrivate() { FieldLayout layout = FieldLayout.forType(IndexedEntity.class); From 0cadf9d73fded67866b96b6c55654aa31a071750 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 7 Oct 2026 23:56:27 +0800 Subject: [PATCH 28/51] test: retain shared state on value-only dynamic mutation --- .../GeneratedNamespaceCowAcceptance.java | 10 ++++++++++ examples/shared-load-state/verify.sh | 1 + 2 files changed, 11 insertions(+) diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedNamespaceCowAcceptance.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedNamespaceCowAcceptance.java index 23369ad2..5509c606 100644 --- a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedNamespaceCowAcceptance.java +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedNamespaceCowAcceptance.java @@ -24,6 +24,16 @@ static School verify(UserContext context, Long id, String name, String sibling, var shared=rows.get(0).__internalLoadState();assertSame(shared,rows.get(1).__internalLoadState()); var first=rows.get(0);assertEquals(id,first.getId()); assertEquals(DynamicFieldValue.State.NOT_LOADED,first.dynamicFields().field("name").state()); + assertEquals(DynamicFieldValue.State.VALUE,first.dynamicFields().field(note).state()); + Object siblingNote=rows.get(1).dynamicFields().field(note).value(); + first.updateDynamicField(note,"value-only private control"); + assertSame(shared,first.__internalLoadState()); + assertSame(shared,rows.get(1).__internalLoadState()); + assertEquals("value-only private control",first.dynamicFields().field(note).value()); + assertEquals(siblingNote,rows.get(1).dynamicFields().field(note).value()); + assertTrue(rows.get(1).getUpdatedProperties().isEmpty()); + assertTrue(rows.get(1).__internalDynamicMutations().isEmpty()); + System.out.println("PASS generated Java value-only dynamic mutation retains shared snapshot and sibling payload"); first.addDynamicProperty("name","readonly same name"); first.updateDynamicField("name","persistent same name"); assertNotSame(shared,first.__internalLoadState());assertSame(shared,rows.get(1).__internalLoadState()); diff --git a/examples/shared-load-state/verify.sh b/examples/shared-load-state/verify.sh index 148058ce..ef875ce0 100644 --- a/examples/shared-load-state/verify.sh +++ b/examples/shared-load-state/verify.sh @@ -86,6 +86,7 @@ for round in first second; do rg -F "PASS generated Java LF17 dynamic metadata shared without fixed slots" "$run_dir/$round.log" rg -F "PASS generated Java LF19 fixed derived and persistent same-name namespace isolation" "$run_dir/$round.log" rg -F "PASS generated Java LF23 dynamic availability detaches only one view" "$run_dir/$round.log" + rg -F "PASS generated Java value-only dynamic mutation retains shared snapshot and sibling payload" "$run_dir/$round.log" rg -F "PASS generated Java LF20 readonly total persists only through modeled materialization" "$run_dir/$round.log" rg -F "PASS generated Java LF11 native readback rollback retains loaded state and retry intent" "$run_dir/$round.log" rg -F "PASS generated Java authoritative missing column rolls back without turning absence into null" "$run_dir/$round.log" From e2e025db0b1f8f75669504d56a7c78870773be29 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Thu, 8 Oct 2026 00:10:09 +0800 Subject: [PATCH 29/51] test: keep independent dirty roots outside sibling saves --- ...eneratedIndependentMutationAcceptance.java | 43 +++++++++++++++++++ .../GeneratedSchoolLoadStateTest.java | 1 + examples/shared-load-state/verify.sh | 1 + 3 files changed, 45 insertions(+) create mode 100644 examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedIndependentMutationAcceptance.java diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedIndependentMutationAcceptance.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedIndependentMutationAcceptance.java new file mode 100644 index 00000000..eafb7373 --- /dev/null +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedIndependentMutationAcceptance.java @@ -0,0 +1,43 @@ +package com.example.schoolmanagementservice; + +import io.teaql.core.UserContext; +import static org.junit.jupiter.api.Assertions.*; + +/** Two independently mutable roots may share availability, never save authority. */ +final class GeneratedIndependentMutationAcceptance { + static void verify(UserContext context, Long firstId, String firstName, String secondName) { + var rows = Q.schools().withNameIn(firstName, secondName).orderByIdAscending().limit(2) + .comment("what: load two roots with shared field availability") + .purpose("why: saving one root must not consume another root's pending intent").executeForList(context); + assertEquals(2, rows.size()); + var first = rows.get(0); var second = rows.get(1); + assertEquals(firstId, first.getId()); + var state = first.__internalLoadState(); + assertSame(state, second.__internalLoadState()); + Long firstVersion = first.getVersion(), secondVersion = second.getVersion(); + String firstPending = firstName + " independent ledger probe"; + String secondPending = secondName + " abandoned ledger probe"; + first.updateName(firstPending); second.updateName(secondPending); + assertSame(state, first.__internalLoadState()); + assertSame(state, second.__internalLoadState()); + assertNotSame(first.getEntityMutationLedger(), second.getEntityMutationLedger()); + first.auditAs("save only the first root while another root remains dirty").save(context); + var stored = Q.schools().withNameIn(firstPending, secondName).orderByIdAscending().limit(2) + .comment("what: inspect both roots after saving only the first") + .purpose("why: detect accidental persistence of an independent pending mutation").executeForList(context); + assertEquals(2, stored.size()); + assertEquals(firstPending, E.school(stored.get(0)).getName().eval()); + assertEquals(firstVersion + 1, stored.get(0).getVersion()); + assertEquals(secondName, E.school(stored.get(1)).getName().eval()); + assertEquals(secondVersion, stored.get(1).getVersion()); + assertEquals(secondPending, E.school(second).getName().eval()); + assertEquals(secondVersion, second.getVersion()); + assertTrue(second.getUpdatedProperties().contains("name")); + assertTrue(first.getUpdatedProperties().isEmpty()); + assertSame(state, second.__internalLoadState()); + // Abandon the second root and restore the first through the same audited API. + first.updateName(firstName); + first.auditAs("restore the independently saved test root").save(context); + System.out.println("PASS generated Java saving one dirty root leaves another dirty root unpersisted"); + } +} diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java index 3af363b6..9fcfc0fa 100644 --- a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java @@ -221,6 +221,7 @@ void generatedSchoolUsesLocalRuntimeAndSharedState() throws Exception { assertTrue(afterAcceptedSave[2] > beforeRejectedSave[2], "accepted save must exercise observed transactions"); System.out.println("PASS generated Java sparse Checker rejects before provider entry; positive save observed"); assertEquals(originalVersion + 1, changed.getVersion()); + GeneratedIndependentMutationAcceptance.verify(context, changed.getId(), renamed, second); var related = Q.schools().withIdIs(changed.getId()) .selectPlatformWith(Q.platformsWithMinimalFields().selectName()) .selectSchoolTypeWith(Q.schoolTypesWithMinimalFields().selectCode()) diff --git a/examples/shared-load-state/verify.sh b/examples/shared-load-state/verify.sh index ef875ce0..2d6efdab 100644 --- a/examples/shared-load-state/verify.sh +++ b/examples/shared-load-state/verify.sh @@ -87,6 +87,7 @@ for round in first second; do rg -F "PASS generated Java LF19 fixed derived and persistent same-name namespace isolation" "$run_dir/$round.log" rg -F "PASS generated Java LF23 dynamic availability detaches only one view" "$run_dir/$round.log" rg -F "PASS generated Java value-only dynamic mutation retains shared snapshot and sibling payload" "$run_dir/$round.log" + rg -F "PASS generated Java saving one dirty root leaves another dirty root unpersisted" "$run_dir/$round.log" rg -F "PASS generated Java LF20 readonly total persists only through modeled materialization" "$run_dir/$round.log" rg -F "PASS generated Java LF11 native readback rollback retains loaded state and retry intent" "$run_dir/$round.log" rg -F "PASS generated Java authoritative missing column rolls back without turning absence into null" "$run_dir/$round.log" From 275a5acd1373acef21f938b5393d391709c0f549 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Thu, 8 Oct 2026 00:55:19 +0800 Subject: [PATCH 30/51] fix: validate modified graph objects without requiring untouched partial references --- ...GeneratedInheritedSchoolLoadStateTest.java | 1 + .../GeneratedNestedGraphAssertions.java | 2 + ...eneratedPartialGraphCheckerAcceptance.java | 50 +++++++++++++++++++ .../GeneratedSchoolLoadStateTest.java | 9 ++++ examples/shared-load-state/verify.sh | 3 ++ .../java/io/teaql/core/checker/Checker.java | 6 +++ .../checker/internal/CheckerInvocation.java | 15 +++++- .../io/teaql/core/checker/CheckerTest.java | 25 ++++++++++ .../java/io/teaql/runtime/TeaQLRuntime.java | 47 +++++++++++++---- .../io/teaql/runtime/TeaQLRuntimeTest.java | 42 +++++++++++++++- 10 files changed, 188 insertions(+), 12 deletions(-) create mode 100644 examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedPartialGraphCheckerAcceptance.java diff --git a/examples/shared-load-state/src/inheritance/java/com/example/schoolmanagementservice/GeneratedInheritedSchoolLoadStateTest.java b/examples/shared-load-state/src/inheritance/java/com/example/schoolmanagementservice/GeneratedInheritedSchoolLoadStateTest.java index 21f57175..87109fbf 100644 --- a/examples/shared-load-state/src/inheritance/java/com/example/schoolmanagementservice/GeneratedInheritedSchoolLoadStateTest.java +++ b/examples/shared-load-state/src/inheritance/java/com/example/schoolmanagementservice/GeneratedInheritedSchoolLoadStateTest.java @@ -96,6 +96,7 @@ void inheritedQEMutationAndSnapshots() { .comment("what: load an incomplete inherited projection").purpose("why: retain full-object Checker enforcement") .executeForOne(context); assertFalse(sparse.__internalLoadState().isLoaded("name")); + sparse.updateCampusCode("incomplete inherited mutation"); assertThrows(CheckException.class, () -> sparse.auditAs("reject sparse inherited save").save(context)); var changed = full.get(0); Long siblingVersion = full.get(1).getVersion(); diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedNestedGraphAssertions.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedNestedGraphAssertions.java index 0d3f439e..526d07c8 100644 --- a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedNestedGraphAssertions.java +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedNestedGraphAssertions.java @@ -144,11 +144,13 @@ static void verify(UserContext context, Long schoolId, String renamed, assertEquals(1001L, filtered.getSchoolType().getId()); assertTrue(filtered.getSchoolType().isPropertyLoaded("id"), "FK identity cannot become NotLoaded with excluded details"); assertFalse(filtered.getSchoolType().isPropertyLoaded("code")); + assertThrows(RuntimeException.class, () -> E.school(filtered).getSchoolType().getCode().eval()); var filteredJson = mapper.readTree(mapper.writeValueAsString(filtered)); assertFalse(filteredJson.get("schoolType").has("code")); var restoredFiltered = mapper.treeToValue(filteredJson, filtered.getClass()); assertEquals(1001L, restoredFiltered.getSchoolType().getId()); assertFalse(restoredFiltered.getSchoolType().isPropertyLoaded("code")); + assertThrows(RuntimeException.class, () -> E.school(restoredFiltered).getSchoolType().getCode().eval()); assertFalse(restoredFiltered.__internalHasMutationLedger()); System.out.println("PASS generated Java LF08 loaded FK and excluded forward details stay distinct"); System.out.println("PASS generated Java typed JSON graph roundtrip and Empty/NotLoaded isolation"); diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedPartialGraphCheckerAcceptance.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedPartialGraphCheckerAcceptance.java new file mode 100644 index 00000000..eeba4039 --- /dev/null +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedPartialGraphCheckerAcceptance.java @@ -0,0 +1,50 @@ +package com.example.schoolmanagementservice; + +import io.teaql.core.UserContext; +import io.teaql.core.checker.CheckException; +import static org.junit.jupiter.api.Assertions.*; + +/** A referenced object that is actually modified must itself be complete. */ +final class GeneratedPartialGraphCheckerAcceptance { + static void verify(UserContext context, ObservedJdbcExecutor sql, String name) { + var row = Q.schools().withNameIn(name) + .selectPlatformWith(Q.platformsWithMinimalFields().selectName()).limit(1) + .comment("what: load a full School with partial Platform detail") + .purpose("why: Checker must reject mutation of a partial referenced object").executeForOne(context); + assertTrue(row.isPropertyLoaded("address")); + assertTrue(row.getPlatform().isPropertyLoaded("name")); + assertFalse(row.getPlatform().isPropertyLoaded("baseUrl")); + var untouched = row.getPlatform(); + var untouchedState = untouched.__internalLoadState(); + row.updateName("Complete parent control"); + row.auditAs("save complete parent without modifying partial Platform").save(context); + assertSame(untouchedState, untouched.__internalLoadState()); + assertFalse(untouched.isPropertyLoaded("baseUrl")); + assertFalse(untouched.isPropertyLoaded("updateTime")); + var restored = Q.schools().withNameIn("Complete parent control").limit(1) + .comment("what: read saved complete parent") + .purpose("why: verify untouched partial reference does not block persistence").executeForOne(context); + assertNotNull(restored); + restored.updateName(name); + restored.auditAs("restore parent after modified-only Checker control").save(context); + System.out.println("PASS generated Java complete parent saves with untouched partial reference"); + row = Q.schools().withNameIn(name) + .selectPlatformWith(Q.platformsWithMinimalFields().selectName()).limit(1) + .comment("what: reload partial Platform detail after the positive save") + .purpose("why: mutate the materialized child rather than a readback identity reference").executeForOne(context); + var child = row.getPlatform(); child.updateName("Partial child control"); + var state = row.__internalLoadState(); + long[] before = sql.counts(); + assertThrows(CheckException.class, () -> child.auditAs("reject mutation of the incomplete referenced object").save(context)); + assertArrayEquals(before, sql.counts(), "partial graph must reject before any provider entry"); + assertSame(state, row.__internalLoadState()); + assertFalse(row.getPlatform().isPropertyLoaded("baseUrl")); + assertFalse(row.getPlatform().isPropertyLoaded("createTime")); + // The declared updateTime Fix is a real assignment and may detach the + // child's snapshot. It must not mark missing business fields loaded. + assertTrue(row.getPlatform().isPropertyLoaded("updateTime")); + assertFalse(untouchedState.isLoaded("baseUrl")); + assertFalse(untouchedState.isLoaded("updateTime")); + System.out.println("PASS generated Java modified partial referenced object rejects before provider entry without loading missing business fields"); + } +} diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java index 9fcfc0fa..bb8f3b58 100644 --- a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java @@ -104,6 +104,7 @@ void generatedSchoolUsesLocalRuntimeAndSharedState() throws Exception { school.updateUpdateTime(LocalDateTime.of(2023, 11, 14, 22, 13)); school.auditAs("Create controlled shared-state probe").save(context); } + GeneratedPartialGraphCheckerAcceptance.verify(context, sql, first); var full = Q.schools().withNameIn(first, second).selectSelfFields() .orderByIdAscending().limit(2).comment("what: read compatible full projections") .purpose("why: inspect exact immutable state references").executeForList(context); @@ -167,6 +168,14 @@ void generatedSchoolUsesLocalRuntimeAndSharedState() throws Exception { assertFalse(sparseRestored.isPropertyLoaded("address")); assertFalse(sparseRestored.__internalHasMutationLedger()); assertTrue(sparseRestored.getUpdatedProperties().isEmpty()); + var emptyNameJson = (com.fasterxml.jackson.databind.node.ObjectNode) sparseJson.get(0).deepCopy(); + emptyNameJson.put("name", ""); + var emptyName = mapper.treeToValue(emptyNameJson, School.class); + assertEquals("", E.school(emptyName).getName().eval()); + assertTrue(emptyName.isPropertyLoaded("name")); + assertFalse(emptyName.isPropertyLoaded("address")); + assertFalse(emptyName.__internalHasMutationLedger()); + System.out.println("PASS generated Java empty native value stays loaded without widening absent fields"); var sparseSecondRestored = mapper.treeToValue(sparseJson.get(1), School.class); assertSame(sparseRestored.__internalLoadState(), sparseSecondRestored.__internalLoadState()); assertEquals(sparseJson.get(0), mapper.readTree(mapper.writeValueAsString(sparseRestored))); diff --git a/examples/shared-load-state/verify.sh b/examples/shared-load-state/verify.sh index 2d6efdab..eacb00fe 100644 --- a/examples/shared-load-state/verify.sh +++ b/examples/shared-load-state/verify.sh @@ -70,6 +70,9 @@ for round in first second; do rg -F "PASS generated Java typed native JSON roundtrip and snapshot sharing $round" "$run_dir/$round.log" rg -F "PASS generated Java typed JSON graph roundtrip and Empty/NotLoaded isolation" "$run_dir/$round.log" rg -F "PASS generated Java sparse Checker rejects before provider entry" "$run_dir/$round.log" + rg -F "PASS generated Java complete parent saves with untouched partial reference" "$run_dir/$round.log" + rg -F "PASS generated Java modified partial referenced object rejects before provider entry without loading missing business fields" "$run_dir/$round.log" + rg -F "PASS generated Java empty native value stays loaded without widening absent fields" "$run_dir/$round.log" rg -F "PASS generated Java Q selection and real JDBC column-order invariance" "$run_dir/$round.log" rg -F "PASS generated Java page and stream shared load state" "$run_dir/$round.log" rg -F "PASS generated Java readonly property metadata shares schema without installing values or fixed slots" "$run_dir/$round.log" diff --git a/teaql-core/src/main/java/io/teaql/core/checker/Checker.java b/teaql-core/src/main/java/io/teaql/core/checker/Checker.java index 7088de74..e92d23a1 100644 --- a/teaql-core/src/main/java/io/teaql/core/checker/Checker.java +++ b/teaql-core/src/main/java/io/teaql/core/checker/Checker.java @@ -48,6 +48,12 @@ default boolean needCheck(UserContext context, T entity) { if (entity.get$status() == EntityStatus.REFER) { return false; } + CheckerInvocation invocation = CheckerInvocation.current(context); + if (invocation != null && invocation.mutationOnly() + && (entity.get$status() == EntityStatus.PERSISTED + || entity.get$status() == EntityStatus.PERSISTED_DELETED)) { + return false; + } return true; } diff --git a/teaql-core/src/main/java/io/teaql/core/checker/internal/CheckerInvocation.java b/teaql-core/src/main/java/io/teaql/core/checker/internal/CheckerInvocation.java index 4169e200..cf37f394 100644 --- a/teaql-core/src/main/java/io/teaql/core/checker/internal/CheckerInvocation.java +++ b/teaql-core/src/main/java/io/teaql/core/checker/internal/CheckerInvocation.java @@ -28,12 +28,14 @@ private record Completed(WeakReference context, List e private final UserContext context; private final CheckerInvocation parent; + private final boolean mutationOnly; private final Map attributes = new HashMap<>(); private boolean closed; - private CheckerInvocation(UserContext context) { + private CheckerInvocation(UserContext context, boolean mutationOnly) { this.context = java.util.Objects.requireNonNull(context, "context"); this.parent = ACTIVE.get(); + this.mutationOnly = mutationOnly; attributes.put(Checker.TEAQL_DATA_CHECK_RESULT, new ArrayList()); attributes.put(Checker.TEAQL_DATA_CHECKED_ITEMS, new ArrayList<>()); attributes.put(UserContext.TEAQL_FIX_EVIDENCE_CURRENT, new ArrayList()); @@ -44,7 +46,16 @@ private CheckerInvocation(UserContext context) { /** Internal synchronous runtime boundary; do not open scopes in business code. */ public static CheckerInvocation open(UserContext context) { - return new CheckerInvocation(context); + return new CheckerInvocation(context, false); + } + + /** Save validates changed objects, not untouched partial reference details. */ + public static CheckerInvocation openMutation(UserContext context) { + return new CheckerInvocation(context, true); + } + + public boolean mutationOnly() { + return mutationOnly; } public static CheckerInvocation current(UserContext context) { diff --git a/teaql-core/src/test/java/io/teaql/core/checker/CheckerTest.java b/teaql-core/src/test/java/io/teaql/core/checker/CheckerTest.java index 2bc8ee63..6487c596 100644 --- a/teaql-core/src/test/java/io/teaql/core/checker/CheckerTest.java +++ b/teaql-core/src/test/java/io/teaql/core/checker/CheckerTest.java @@ -83,6 +83,31 @@ public void testMarkAsCheckedAndNeedCheck() { assertFalse(checker.needCheck(context, referEntity)); } + @Test + public void mutationScopeChecksOnlyPendingObjectsAndRestoresExplicitValidation() { + var checker = new DummyChecker(); + var context = createDummyContext(); + var entity = new DummyEntity(); + entity.set$status(EntityStatus.PERSISTED); + assertTrue("explicit validation still checks unchanged objects", checker.needCheck(context, entity)); + try (var scope = io.teaql.core.checker.internal.CheckerInvocation.openMutation(context)) { + assertFalse(checker.needCheck(context, entity)); + entity.set$status(EntityStatus.PERSISTED_DELETED); + assertFalse(checker.needCheck(context, entity)); + for (var status : List.of(EntityStatus.NEW, EntityStatus.UPDATED, + EntityStatus.UPDATED_DELETED, EntityStatus.UPDATED_RECOVER)) { + entity.set$status(status); + assertTrue(status.toString(), checker.needCheck(context, entity)); + } + entity.set$status(EntityStatus.PERSISTED); + try (var explicit = io.teaql.core.checker.internal.CheckerInvocation.open(context)) { + assertTrue(checker.needCheck(context, entity)); + } + assertFalse(checker.needCheck(context, entity)); + } + assertTrue(checker.needCheck(context, entity)); + } + @Test public void testNewLocation() { DummyChecker checker = new DummyChecker(); diff --git a/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java b/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java index 72bb3295..2cdbd6e3 100644 --- a/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java +++ b/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java @@ -547,17 +547,12 @@ private List mutationTrace(EntityMutationLedger ledger, EntityKey key @SuppressWarnings({"rawtypes", "unchecked"}) private void checkAndFix(UserContext context, Entity entity) { - Checker checker = checkers.get(entity.runtimeType()); - if (checker == null) { - checker = checkers.get(entity.typeName()); - } - if (checker == null) { - return; - } - try (var invocation = io.teaql.core.checker.internal.CheckerInvocation.open(context)) { + if (checkers.isEmpty()) return; + try (var invocation = io.teaql.core.checker.internal.CheckerInvocation.openMutation(context)) { context.beginFixEvidence(); try { - checker.checkAndFix(context, (BaseEntity) entity); + checkGraphMutations(context, entity, null, + Collections.newSetFromMap(new IdentityHashMap<>())); List violations = (List) invocation.attribute(Checker.TEAQL_DATA_CHECK_RESULT); if (violations != null && !violations.isEmpty()) { throw new CheckException(new ArrayList<>(violations)); @@ -568,6 +563,40 @@ private void checkAndFix(UserContext context, Entity entity) { } } + @SuppressWarnings({"rawtypes", "unchecked"}) + private void checkGraphMutations(UserContext context, Entity entity, + io.teaql.core.checker.ObjectLocation location, Set visited) { + if (!(entity instanceof BaseEntity base) || !visited.add(entity)) return; + Checker checker = checkers.get(entity.runtimeType()); + if (checker == null) checker = checkers.get(entity.typeName()); + if (checker != null && checker.needCheck(context, base)) { + checker.checkAndFix(context, base, location); + } + // An untouched partial parent must not hide a dirty descendant. Generated + // Checkers may stop at that parent, so persistence walks the graph itself. + EntityDescriptor descriptor = metadata.resolveEntityDescriptor(entity.typeName()); + for (EntityDescriptor current = descriptor; current != null; current = current.getParent()) { + for (PropertyDescriptor property : current.getProperties()) { + if (!(property instanceof io.teaql.core.meta.Relation)) continue; + String member = io.teaql.core.utils.NamingCase.toUnderlineCase(property.getName()); + var childLocation = location == null + ? io.teaql.core.checker.ObjectLocation.hashRoot(member) : location.member(member); + Object value = entity.getProperty(property.getName()); + if (value instanceof Entity child) { + checkGraphMutations(context, child, childLocation, visited); + } else if (value instanceof Iterable children) { + int index = 0; + for (Object child : children) { + if (child instanceof Entity related) { + checkGraphMutations(context, related, childLocation.element(index), visited); + } + index++; + } + } + } + } + } + /** * Merge related entities' EntityMutationLedgers into the main entity's EntityMutationLedger. * This ensures that when saving an Order, its OrderItems' changes are also saved. diff --git a/teaql-runtime/src/test/java/io/teaql/runtime/TeaQLRuntimeTest.java b/teaql-runtime/src/test/java/io/teaql/runtime/TeaQLRuntimeTest.java index 6edee167..3e729441 100644 --- a/teaql-runtime/src/test/java/io/teaql/runtime/TeaQLRuntimeTest.java +++ b/teaql-runtime/src/test/java/io/teaql/runtime/TeaQLRuntimeTest.java @@ -81,6 +81,7 @@ public void checkAndFix(UserContext context, DummyEntity entity, ObjectLocation public static class DummyEntity extends BaseEntity { private String name; + private DummyEntity child; @Override public String typeName() { @@ -98,7 +99,9 @@ public void __internalSet(String property, Object value) { @Override public Object __internalGet(String property) { - if ("name".equals(property)) { + if ("child".equals(property)) { + return child; + } else if ("name".equals(property)) { return this.name; } else { return super.__internalGet(property); @@ -449,6 +452,43 @@ public void testBuilder() { Assert.assertNotNull(runtime); } + @Test + public void modifiedDescendantIsCheckedThroughUntouchedPartialParent() { + RecordingMutationExecutor executor = new RecordingMutationExecutor(); + DummyChecker checker = new DummyChecker() { + @Override public void checkAndFix(UserContext context, DummyEntity entity, ObjectLocation location) { + if (!needCheck(context, entity)) return; + markAsChecked(context, entity); + requiredCheck(context, newLocation(location, "name"), entity.name); + checkAndFix(context, entity.child, newLocation(location, "child")); + } + }; + DummyMetaFactory metadata = new DummyMetaFactory() { + @Override public EntityDescriptor resolveEntityDescriptor(String type) { + var descriptor = super.resolveEntityDescriptor(type); + var relation = new io.teaql.core.meta.Relation(); + relation.setName("child"); + descriptor.getProperties().add(relation); + return descriptor; + } + }; + TeaQLRuntime runtime = TeaQLRuntime.builder().metadata(metadata) + .dataService("dummy", executor).build() + .install(RuntimeModule.of().withCheckers(checker)); + var context = new DefaultUserContext(runtime); + var root = new DummyEntity(); root.name = "complete parent"; + root.set$status(EntityStatus.UPDATED); root.setComment("check modified descendants"); + root.child = new DummyEntity(); root.child.set$status(EntityStatus.PERSISTED); + root.child.child = new DummyEntity(); root.child.child.set$status(EntityStatus.UPDATED); + // Cycle exercises the independent persistence walk's identity boundary. + root.child.child.child = root; + var error = Assert.assertThrows(CheckException.class, () -> runtime.saveGraph(context, root)); + Assert.assertEquals(1, error.getViolates().size()); + Assert.assertEquals("child.child.name", error.getViolates().get(0).getLocation().modelPath()); + Assert.assertTrue(executor.requests.isEmpty()); + Assert.assertEquals(EntityStatus.PERSISTED, root.child.get$status()); + } + @Test public void checkerFailurePreventsMutationAndTraversalStateIsSaveScoped() { RecordingMutationExecutor executor = new RecordingMutationExecutor(); From 90dd850df7a023d17dfd851436ab33b2d97176a5 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Thu, 8 Oct 2026 01:35:54 +0800 Subject: [PATCH 31/51] test: require exact NotLoaded diagnostics for filtered relation views --- .../GeneratedNestedGraphAssertions.java | 13 +++++++++++-- examples/shared-load-state/verify.sh | 1 + 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedNestedGraphAssertions.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedNestedGraphAssertions.java index 526d07c8..b2898906 100644 --- a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedNestedGraphAssertions.java +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedNestedGraphAssertions.java @@ -144,14 +144,23 @@ static void verify(UserContext context, Long schoolId, String renamed, assertEquals(1001L, filtered.getSchoolType().getId()); assertTrue(filtered.getSchoolType().isPropertyLoaded("id"), "FK identity cannot become NotLoaded with excluded details"); assertFalse(filtered.getSchoolType().isPropertyLoaded("code")); - assertThrows(RuntimeException.class, () -> E.school(filtered).getSchoolType().getCode().eval()); + var missing = assertThrows(io.teaql.core.value.TeaQLNotLoadedException.class, + () -> E.school(filtered).getSchoolType().getCode().eval()); + assertEquals("schoolType.code", missing.getAccessPath()); + assertEquals("code", missing.getBreakPoint()); + assertEquals("School(id=" + schoolId + ")", missing.getRoot()); var filteredJson = mapper.readTree(mapper.writeValueAsString(filtered)); assertFalse(filteredJson.get("schoolType").has("code")); var restoredFiltered = mapper.treeToValue(filteredJson, filtered.getClass()); assertEquals(1001L, restoredFiltered.getSchoolType().getId()); assertFalse(restoredFiltered.getSchoolType().isPropertyLoaded("code")); - assertThrows(RuntimeException.class, () -> E.school(restoredFiltered).getSchoolType().getCode().eval()); + var restoredMissing = assertThrows(io.teaql.core.value.TeaQLNotLoadedException.class, + () -> E.school(restoredFiltered).getSchoolType().getCode().eval()); + assertEquals("schoolType.code", restoredMissing.getAccessPath()); + assertEquals("code", restoredMissing.getBreakPoint()); + assertEquals("School(id=" + schoolId + ")", restoredMissing.getRoot()); assertFalse(restoredFiltered.__internalHasMutationLedger()); + System.out.println("PASS generated Java LF08 exact NotLoaded type and path in Q and typed JSON"); System.out.println("PASS generated Java LF08 loaded FK and excluded forward details stay distinct"); System.out.println("PASS generated Java typed JSON graph roundtrip and Empty/NotLoaded isolation"); System.out.println("PASS generated Java nested/reverse graph Q/E/JSON and Empty/NotLoaded isolation"); diff --git a/examples/shared-load-state/verify.sh b/examples/shared-load-state/verify.sh index eacb00fe..ca06a3bf 100644 --- a/examples/shared-load-state/verify.sh +++ b/examples/shared-load-state/verify.sh @@ -85,6 +85,7 @@ for round in first second; do rg -F "PASS generated Java nested/reverse graph Q/E/JSON and Empty/NotLoaded isolation" "$run_dir/$round.log" rg -F "PASS generated Java nested dynamic Value/NULL/NotLoaded and shared snapshots" "$run_dir/$round.log" rg -F "PASS generated Java LF08 loaded FK and excluded forward details stay distinct" "$run_dir/$round.log" + rg -F "PASS generated Java LF08 exact NotLoaded type and path in Q and typed JSON" "$run_dir/$round.log" rg -F "PASS generated Java LF09 reverse Loaded/Empty/NotLoaded through Q/E/JSON" "$run_dir/$round.log" rg -F "PASS generated Java LF17 dynamic metadata shared without fixed slots" "$run_dir/$round.log" rg -F "PASS generated Java LF19 fixed derived and persistent same-name namespace isolation" "$run_dir/$round.log" From 21e0bb1546e518fbc39f9ebac206998dd1b4d056 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Thu, 8 Oct 2026 02:06:23 +0800 Subject: [PATCH 32/51] perf: avoid allocating enhancement batches for empty cursors --- .../core/sql/portable/BatchedRootStream.java | 5 +-- .../BatchedRootStreamAllocationTest.java | 36 +++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/BatchedRootStreamAllocationTest.java diff --git a/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/BatchedRootStream.java b/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/BatchedRootStream.java index 3e84e2c3..eb06428a 100644 --- a/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/BatchedRootStream.java +++ b/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/BatchedRootStream.java @@ -32,9 +32,10 @@ private void close() { try { if (!ready.hasNext()) { if (input == null) input = source.iterator(); + if (!input.hasNext()) { close(); return false; } var batch = new SmartList(batchSize); - while (batch.size() < batchSize && input.hasNext()) batch.add(input.next()); - if (batch.isEmpty()) { close(); return false; } + do { batch.add(input.next()); } + while (batch.size() < batchSize && input.hasNext()); hydrate.accept(batch); ready = batch.iterator(); } diff --git a/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/BatchedRootStreamAllocationTest.java b/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/BatchedRootStreamAllocationTest.java new file mode 100644 index 00000000..2fc23892 --- /dev/null +++ b/teaql-sql-portable/src/test/java/io/teaql/core/sql/portable/BatchedRootStreamAllocationTest.java @@ -0,0 +1,36 @@ +package io.teaql.core.sql.portable; + +import io.teaql.core.BaseEntity; +import java.util.stream.Stream; +import org.junit.Test; +import static org.junit.Assert.*; + +/** Empty cursor allocation must not scale with the configured batch limit. */ +public class BatchedRootStreamAllocationTest { + private static final int REPEATS = 1_000; + private static long measure(Object bean, java.lang.reflect.Method allocated, int limit) throws Exception { + long thread = Thread.currentThread().getId(); + long before = (Long)allocated.invoke(bean, thread); + for (int i=0; iempty(), limit, + rows -> fail("empty cursor must not reach enhancement"))) { + assertEquals(0, stream.count()); + } + } + return (Long)allocated.invoke(bean, thread) - before; + } + @Test public void emptyAllocationDoesNotGrowWithBatchLimit() throws Exception { + // JVM-only observation: do not add a management-module dependency to the runtime SPI. + var bean = Class.forName("java.lang.management.ManagementFactory").getMethod("getThreadMXBean").invoke(null); + var api = Class.forName("com.sun.management.ThreadMXBean"); + assertEquals(true, api.getMethod("isThreadAllocatedMemorySupported").invoke(bean)); + api.getMethod("setThreadAllocatedMemoryEnabled",boolean.class).invoke(bean,true); + var allocated = api.getMethod("getThreadAllocatedBytes",long.class); + for(int i=0; i<8; i++) measure(bean, allocated, 4); + for(int round=0; round<3; round++) { + long small = measure(bean, allocated, 4), large = measure(bean, allocated, 8_192); + System.out.printf("EMPTY_STREAM,%d,%d,%d,%d%n", round, REPEATS, small, large); + assertTrue("empty stream must not allocate a batch-sized backing array",large <= small + 256L * REPEATS); + } + } +} From 250fbf04fd1ad3eaaf9e5f1cf891a56127195f29 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Thu, 8 Oct 2026 02:24:08 +0800 Subject: [PATCH 33/51] test: prove overflow storage remains lazy and ordinary shapes share the empty sentinel --- .../test/java/io/teaql/core/LoadStateTest.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/teaql-core/src/test/java/io/teaql/core/LoadStateTest.java b/teaql-core/src/test/java/io/teaql/core/LoadStateTest.java index df2a5af2..b05ff443 100644 --- a/teaql-core/src/test/java/io/teaql/core/LoadStateTest.java +++ b/teaql-core/src/test/java/io/teaql/core/LoadStateTest.java @@ -398,6 +398,22 @@ public void malformedDescriptorsAndIncompatibleLayoutsAreRejected() { () -> row.__internalLoadState().layout().validateExpectedMembers(Set.of("id", "version"))); } + @Test + public void overflowUsesOneEmptySentinelUntilAHighSlotIsActuallyLoaded() { + var layout = FieldLayout.forType(IndexedEntity.class); + var noOverflow = layout.emptyState().overflow(); + var ordinary = LoadState.projection(layout, List.of("id", "field_63")); + assertSame(noOverflow, ordinary.overflow()); + var wide = ordinary.withLoaded("field_64", true); + assertNotSame(noOverflow, wide.overflow()); + assertEquals(Set.of(64), wide.overflow()); + assertSame(noOverflow, ordinary.overflow()); + var cleared = wide.withLoaded("field_64", false); + assertSame(noOverflow, cleared.overflow()); + assertEquals(ordinary.bits(), cleared.bits()); + assertTrue(wide.isLoaded("field_64")); + } + @Test public void ordinaryCompiledShapesRetainOneSnapshotAtAllRequiredRowScales() { FieldLayout layout = FieldLayout.forType(IndexedEntity.class); From 0af667eed9886225bed23c497eb42d4fad8e1d0c Mon Sep 17 00:00:00 2001 From: Philip Z Date: Thu, 8 Oct 2026 02:40:49 +0800 Subject: [PATCH 34/51] test: distinguish readonly false empty null and absence without mutation intent --- .../io/teaql/core/DynamicMemberStateTest.java | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/teaql-core/src/test/java/io/teaql/core/DynamicMemberStateTest.java b/teaql-core/src/test/java/io/teaql/core/DynamicMemberStateTest.java index 31077c87..3ce48035 100644 --- a/teaql-core/src/test/java/io/teaql/core/DynamicMemberStateTest.java +++ b/teaql-core/src/test/java/io/teaql/core/DynamicMemberStateTest.java @@ -44,6 +44,31 @@ public void derivedNullAndMissingReadNullWithoutInventingZeroOrPersistentFields( assertEquals(0, row.collectDynamicFieldValues().size()); } + @Test + public void readonlyFalseEmptyNullAndAbsenceHavePrivatePresenceWithoutMutationOrSlots() { + Row row = new Row(); + var state = row.__internalLoadState(); + row.addDynamicProperty("flag", false); + row.addDynamicProperty("empty", ""); + row.addDynamicProperty("null", null); + assertEquals(Boolean.FALSE, row.getDynamicProperty("flag")); + assertEquals("", row.getDynamicProperty("empty")); + assertNull(row.getDynamicProperty("null")); + assertNull(row.getDynamicProperty("absent")); + assertTrue(row.hasDynamicProperty("flag")); + assertTrue(row.hasDynamicProperty("empty")); + assertTrue(row.hasDynamicProperty("null")); + assertFalse(row.hasDynamicProperty("absent")); + assertSame(state, row.__internalLoadState()); + for (String name : List.of("_flag", "_empty", "_null", "_absent")) { + assertFalse(row.isPropertyLoaded(name)); + assertNull(state.layout().findIndex(name)); + } + assertTrue(row.getUpdatedProperties().isEmpty()); + assertFalse(row.__internalHasMutationLedger()); + assertEquals(0, row.collectDynamicFieldValues().size()); + } + @Test public void sharedSelectionKeepsWrapperNullAndValuePrivateAndLeavesFixedSlotsUnchanged() { DynamicFieldMetadata metadata = new DynamicFieldMetadata(Map.of("note", DynamicDataType.STRING, "extra", DynamicDataType.STRING)); From d607988b435b75f4f7b62b4ee064ef8667d98da6 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Thu, 8 Oct 2026 04:02:54 +0800 Subject: [PATCH 35/51] test: preserve streamed overflow and reject incomplete extension updates --- .../GeneratedDynamicRollbackAcceptance.java | 43 +++++++++++++++++++ examples/shared-load-state/verify.sh | 1 + 2 files changed, 44 insertions(+) diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedDynamicRollbackAcceptance.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedDynamicRollbackAcceptance.java index edd4cae6..6647ed4e 100644 --- a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedDynamicRollbackAcceptance.java +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedDynamicRollbackAcceptance.java @@ -3,6 +3,7 @@ import com.example.schoolmanagementservice.school.School; import io.teaql.core.SmartList; import io.teaql.core.UserContext; +import io.teaql.core.checker.CheckException; import io.teaql.data.dynamic.*; import io.teaql.data.dynamic.jdbc.JdbcDynamicFieldsProvider; import static org.junit.jupiter.api.Assertions.*; @@ -50,6 +51,48 @@ static School verify(UserContext context, ObservedJdbcExecutor sql, School origi assertFalse(row.__internalHasMutationLedger()); } System.out.println("PASS generated Java dynamic stream Value/Null/NotLoaded and shared snapshots"); + var streamedState = streamed.get(0).__internalLoadState(); + long[] beforeStreamRejection = sql.counts(); + streamed.get(0).updateDynamicField(code, "must-not-persist"); + assertThrows(CheckException.class, () -> streamed.get(0) + .auditAs("reject extension mutation on an incomplete streamed object").save(context)); + assertArrayEquals(beforeStreamRejection, sql.counts(), "incomplete stream mutation must reject before provider entry"); + // The declared timestamp Fix may detach only this modified row's state. + assertNotSame(streamedState, streamed.get(0).__internalLoadState()); + assertTrue(streamed.get(0).isPropertyLoaded("updateTime")); + assertSame(streamedState, streamed.get(1).__internalLoadState()); + assertFalse(streamedState.isLoaded("updateTime")); + assertFalse(streamed.get(0).isPropertyLoaded("address")); + assertFalse(streamed.get(0).__internalDynamicMutations().isEmpty()); + assertEquals(DynamicFieldValue.State.NULL, streamed.get(1).dynamicFields().field(code).state()); + assertFalse(streamed.get(1).__internalHasMutationLedger()); + var completeStreamRequest = Q.schools().withNameIn(firstName, secondName).selectSelfFields(); + completeStreamRequest.selectDynamicFieldsWith(selection); + java.util.List completeStreamed; + try (var stream = completeStreamRequest.orderByIdAscending().limit(2) + .comment("what: stream complete Schools with durable extensions") + .purpose("why: preserve fixed slots and overflow during enhancement").executeForStream(context)) { + completeStreamed = stream.toList(); + } + assertEquals(2, completeStreamed.size()); + var completeStreamState = completeStreamed.get(0).__internalLoadState(); + assertSame(completeStreamState, completeStreamed.get(1).__internalLoadState()); + for (var row : completeStreamed) { + for (var field : School.__TEAQL_FIXED_FIELD_INDEXES.entrySet()) { + assertTrue(row.isPropertyLoaded(field.getKey()), "complete stream omitted " + field.getKey()); + if (field.getValue() >= 64) assertTrue(row.__internalLoadState().overflow().contains(field.getValue())); + } + if (Boolean.parseBoolean(System.getenv("TEAQL_LOAD_STATE_WIDE"))) { + for (int slot : new int[]{63,64,65,129}) { + String field = School.__TEAQL_FIXED_FIELD_INDEXES.entrySet().stream() + .filter(e -> e.getValue() == slot).findFirst().orElseThrow().getKey(); + assertNull(row.__internalGet(School.__TEAQL_FIXED_FIELD_MAPPINGS.get(field).get(0))); + } + } + assertFalse(row.__internalHasMutationLedger()); + assertEquals(DynamicFieldValue.State.NOT_LOADED, row.dynamicFields().field(unused).state()); + } + System.out.println("PASS generated Java dynamic stream full/overflow/sparse state and Checker-before-provider"); var all = Q.schools().withNameIn(firstName, secondName); all.selectDynamicFieldsWith(selection); var rows = all.orderByIdAscending().limit(2).comment("what: load value and null extensions together") diff --git a/examples/shared-load-state/verify.sh b/examples/shared-load-state/verify.sh index ca06a3bf..f34bd44c 100644 --- a/examples/shared-load-state/verify.sh +++ b/examples/shared-load-state/verify.sh @@ -77,6 +77,7 @@ for round in first second; do rg -F "PASS generated Java page and stream shared load state" "$run_dir/$round.log" rg -F "PASS generated Java readonly property metadata shares schema without installing values or fixed slots" "$run_dir/$round.log" rg -F "PASS generated Java dynamic stream Value/Null/NotLoaded and shared snapshots" "$run_dir/$round.log" + rg -F "PASS generated Java dynamic stream full/overflow/sparse state and Checker-before-provider" "$run_dir/$round.log" rg -F "PASS generated Java Checker preserves NotLoaded and sibling load boundaries" "$run_dir/$round.log" rg -F "PASS generated Java dynamic storage provenance and retry" "$run_dir/$round.log" rg -F "PASS generated Java mixed dynamic Value/Null/NotLoaded list lifetime readback rollback and retry" "$run_dir/$round.log" From 5aa91b1eda85300508a3a2602dc58007e160bc64 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Thu, 8 Oct 2026 04:30:11 +0800 Subject: [PATCH 36/51] fix: reject incoming runtime state in typed JSON before entity construction --- .../GeneratedSchoolLoadStateTest.java | 13 ++++++++ examples/shared-load-state/verify.sh | 1 + .../jackson/TypedEntityJsonDeserializer.java | 12 ++++++- .../TypedEntityJsonDeserializerTest.java | 33 +++++++++++++++++++ 4 files changed, 58 insertions(+), 1 deletion(-) diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java index bb8f3b58..da54fe8a 100644 --- a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java @@ -158,6 +158,19 @@ void generatedSchoolUsesLocalRuntimeAndSharedState() throws Exception { assertFalse(sparse.get(0).isPropertyLoaded("address")); var mapper = new ObjectMapper().registerModule(new TeaQLModule(context)) .registerModule(new JavaTimeModule()).disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); + long[] beforeJsonGuard = sql.counts(); + for (String key : java.util.List.of("_comment", "_dirty_fields", "_original_values", "_is_new", + "_is_deleted", "__load_state", "__teaql_runtime_state")) { + var input = mapper.createObjectNode().put("id", sparse.get(0).getId()); + input.putObject(key).put("payload", "PRIVATE-STATE-CANARY"); + var error = assertThrows(com.fasterxml.jackson.databind.JsonMappingException.class, + () -> mapper.treeToValue(input, School.class)); + assertTrue(error.getMessage().contains("Incoming runtime state is forbidden")); + assertFalse(error.getMessage().contains("PRIVATE-STATE-CANARY")); + } + assertArrayEquals(beforeJsonGuard, sql.counts(), "JSON guard must not enter any provider"); + assertFalse(sparse.get(0).__internalHasMutationLedger()); + System.out.println("PASS generated Java reserved runtime JSON keys reject without provider entry or value leakage"); var sparseJson = mapper.readTree(mapper.writeValueAsString(sparse)); assertEquals(first, sparseJson.get(0).get("name").asText()); assertFalse(sparseJson.get(0).has("address"), "NotLoaded must not become serialized NULL"); diff --git a/examples/shared-load-state/verify.sh b/examples/shared-load-state/verify.sh index f34bd44c..9382d9b3 100644 --- a/examples/shared-load-state/verify.sh +++ b/examples/shared-load-state/verify.sh @@ -68,6 +68,7 @@ for round in first second; do } rg -F "PASS generated Java indexed Q/E/Checker/create/update/delete and snapshot sharing $round" "$run_dir/$round.log" rg -F "PASS generated Java typed native JSON roundtrip and snapshot sharing $round" "$run_dir/$round.log" + rg -F "PASS generated Java reserved runtime JSON keys reject without provider entry or value leakage" "$run_dir/$round.log" rg -F "PASS generated Java typed JSON graph roundtrip and Empty/NotLoaded isolation" "$run_dir/$round.log" rg -F "PASS generated Java sparse Checker rejects before provider entry" "$run_dir/$round.log" rg -F "PASS generated Java complete parent saves with untouched partial reference" "$run_dir/$round.log" diff --git a/teaql-jackson/src/main/java/io/teaql/jackson/TypedEntityJsonDeserializer.java b/teaql-jackson/src/main/java/io/teaql/jackson/TypedEntityJsonDeserializer.java index 61043845..e770f1fa 100644 --- a/teaql-jackson/src/main/java/io/teaql/jackson/TypedEntityJsonDeserializer.java +++ b/teaql-jackson/src/main/java/io/teaql/jackson/TypedEntityJsonDeserializer.java @@ -21,6 +21,11 @@ /** Model-bound JSON hydration; no bean setters, reflection, database calls or mutation APIs. */ final class TypedEntityJsonDeserializer extends JsonDeserializer { + // Same wire-state boundary as the Rust context-owned reader. These are not + // readonly business properties and must reject before the supplier is used. + private static final java.util.Set FORBIDDEN_RUNTIME_KEYS = java.util.Set.of( + "_comment", "_dirty_fields", "_original_values", "_is_new", "_is_deleted", + "__load_state", "__teaql_runtime_state"); private record Binding(String member, Class type, int index, Class element) {} private final Class target; private final java.util.function.Supplier supplier; @@ -71,7 +76,12 @@ private Binding binding(String name) { var names = node.fieldNames(); while (names.hasNext()) { String name = names.next(); - if (name.startsWith("_") && name.length() > 1) continue; // readonly derived data, not load-state metadata + if (name.startsWith("_") && name.length() > 1) { + if (FORBIDDEN_RUNTIME_KEYS.contains(name)) { + return context.reportInputMismatch(target, "Incoming runtime state is forbidden: %s", name); + } + continue; // readonly derived data, not load-state metadata + } Binding field = binding(name); if (field == null) return context.reportInputMismatch(target, "Unknown or unsupported model field: %s", name); members.add(field.member()); diff --git a/teaql-jackson/src/test/java/io/teaql/jackson/TypedEntityJsonDeserializerTest.java b/teaql-jackson/src/test/java/io/teaql/jackson/TypedEntityJsonDeserializerTest.java index d6afbcab..d9682ed9 100644 --- a/teaql-jackson/src/test/java/io/teaql/jackson/TypedEntityJsonDeserializerTest.java +++ b/teaql-jackson/src/test/java/io/teaql/jackson/TypedEntityJsonDeserializerTest.java @@ -131,6 +131,39 @@ private static DefaultUserContext context(String owner, AtomicInteger constructe assertFalse(row.__internalHasMutationLedger()); assertTrue(row.getUpdatedProperties().isEmpty()); } + @Test public void reservedRuntimeKeysCannotMasqueradeAsReadonlyProperties() throws Exception { + var constructed = new AtomicInteger(); + var mapper = new ObjectMapper().registerModule(new TeaQLModule(context("A", constructed))); + for (String key : List.of("_comment", "_dirty_fields", "_original_values", "_is_new", + "_is_deleted", "__load_state", "__teaql_runtime_state")) { + var input = mapper.createObjectNode().put("id", 1).put("name", "legitimate name"); + input.putObject(key).put("payload", "PRIVATE-STATE-CANARY"); + var error = assertThrows(JsonMappingException.class, + () -> mapper.treeToValue(input, Row.class)); + assertTrue(error.getMessage().contains("Incoming runtime state is forbidden")); + assertFalse(error.getMessage().contains("PRIVATE-STATE-CANARY")); + } + assertEquals("reject before constructing any model object", 0, constructed.get()); + } + + @Test public void reservedRuntimeKeysRejectTheWholeArrayWithoutConstructingTheInvalidRow() throws Exception { + var constructed = new AtomicInteger(); + var mapper = new ObjectMapper().registerModule(new TeaQLModule(context("A", constructed))); + for (String key : List.of("_comment", "_dirty_fields", "_original_values", "_is_new", + "_is_deleted", "__load_state", "__teaql_runtime_state")) { + var input = mapper.createObjectNode().put("id", 1); + input.putObject(key).put("payload", "PRIVATE-STATE-CANARY"); + var batch = mapper.createArrayNode(); + batch.addObject().put("id", 2).put("name", "valid first row"); + batch.add(input); + var error = assertThrows(JsonMappingException.class, + () -> mapper.readerForListOf(Row.class).readValue(batch)); + assertTrue(error.getMessage().contains("Incoming runtime state is forbidden")); + assertFalse(error.getMessage().contains("PRIVATE-STATE-CANARY")); + } + assertEquals("only each valid first row may reach the supplier", 7, constructed.get()); + } + @Test public void unregisteredEntityCannotFallBackToBeanMutation() throws Exception { var mapper = new ObjectMapper().registerModule(new TeaQLModule(context("A", new AtomicInteger()))); var error = assertThrows(JsonMappingException.class, From 59cc4075eaa0bc677def523d0aa72d6941b4eb00 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Thu, 8 Oct 2026 04:58:50 +0800 Subject: [PATCH 37/51] test: reject held dynamic storage provenance before provider access --- .../GeneratedSchoolLoadStateTest.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java index da54fe8a..4b490e1c 100644 --- a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java @@ -301,9 +301,12 @@ void generatedSchoolUsesLocalRuntimeAndSharedState() throws Exception { extended.updateDynamicField(extension, "must not cross storage profile"); context.putAttribute(DynamicFieldsFacade.class.getName(), new DefaultDynamicFieldsFacade(provider, DynamicFieldScope.of("PROFILE", "other"))); + var beforeProvenanceRejection = sql.counts(); var rejected = assertThrows(DynamicFieldException.class, () -> extended.auditAs("Reject held dynamic view in another profile").save(context)); assertEquals("DYNAMIC_FIELD_STORAGE_PROVENANCE_MISMATCH", rejected.errorCode()); + assertArrayEquals(beforeProvenanceRejection, sql.counts(), + "Held storage provenance must reject before provider entry"); assertEquals(heldVersion, extended.getVersion()); assertFalse(extended.__internalDynamicMutations().isEmpty()); context.putAttribute(DynamicFieldsFacade.class.getName(), @@ -314,13 +317,17 @@ void generatedSchoolUsesLocalRuntimeAndSharedState() throws Exception { .readAll(new DynamicFieldSelection().selectString(extension)); assertEquals("Private extension", stored.getString(extension)); extended.updateDynamicField(extension, null); + var beforeValidExtensionSave = sql.counts(); extended.auditAs("Persist explicit extension NULL").save(context); + assertFalse(java.util.Arrays.equals(beforeValidExtensionSave, sql.counts()), + "Positive control: a valid extension save must enter the provider"); assertEquals(DynamicFieldValue.State.NULL, extended.dynamicFields().field(extension).state()); extended.deleteDynamicField(extension); extended.auditAs("Delete the extension without confusing it with NULL").save(context); assertEquals(DynamicFieldValue.State.NOT_LOADED, extended.dynamicFields().field(extension).state()); assertEquals("readonly-derived-name", extended.getProperty("_name")); System.out.println("PASS generated Java dynamic storage provenance and retry"); + System.out.println("PASS generated Java LF20 held provenance rejects before provider with valid-save positive control"); var combined = GeneratedDynamicRollbackAcceptance.verify(context, sql, extended, renamed, second, extension, provider, definitionContext); String deletionName = combined.getName(); GeneratedNamespaceCowAcceptance.verify(context,combined.getId(),deletionName,second,extension,provider,definitionContext,mapper); From 44e80fffa543c40c43e06c9baf880a516bfc8aa9 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Thu, 8 Oct 2026 05:02:04 +0800 Subject: [PATCH 38/51] test: allow transaction boundary while proving zero provenance rejection DML --- .../GeneratedSchoolLoadStateTest.java | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java index 4b490e1c..7ba8086b 100644 --- a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedSchoolLoadStateTest.java @@ -305,8 +305,11 @@ void generatedSchoolUsesLocalRuntimeAndSharedState() throws Exception { var rejected = assertThrows(DynamicFieldException.class, () -> extended.auditAs("Reject held dynamic view in another profile").save(context)); assertEquals("DYNAMIC_FIELD_STORAGE_PROVENANCE_MISMATCH", rejected.errorCode()); - assertArrayEquals(beforeProvenanceRejection, sql.counts(), - "Held storage provenance must reject before provider entry"); + var afterProvenanceRejection = sql.counts(); + assertEquals(beforeProvenanceRejection[0], afterProvenanceRejection[0], + "Held storage provenance must reject before owner-data reads"); + assertEquals(beforeProvenanceRejection[1], afterProvenanceRejection[1], + "Held storage provenance must reject before native or extension DML"); assertEquals(heldVersion, extended.getVersion()); assertFalse(extended.__internalDynamicMutations().isEmpty()); context.putAttribute(DynamicFieldsFacade.class.getName(), @@ -319,15 +322,15 @@ void generatedSchoolUsesLocalRuntimeAndSharedState() throws Exception { extended.updateDynamicField(extension, null); var beforeValidExtensionSave = sql.counts(); extended.auditAs("Persist explicit extension NULL").save(context); - assertFalse(java.util.Arrays.equals(beforeValidExtensionSave, sql.counts()), - "Positive control: a valid extension save must enter the provider"); + assertTrue(sql.counts()[1] > beforeValidExtensionSave[1], + "Positive control: a valid extension save must execute DML"); assertEquals(DynamicFieldValue.State.NULL, extended.dynamicFields().field(extension).state()); extended.deleteDynamicField(extension); extended.auditAs("Delete the extension without confusing it with NULL").save(context); assertEquals(DynamicFieldValue.State.NOT_LOADED, extended.dynamicFields().field(extension).state()); assertEquals("readonly-derived-name", extended.getProperty("_name")); System.out.println("PASS generated Java dynamic storage provenance and retry"); - System.out.println("PASS generated Java LF20 held provenance rejects before provider with valid-save positive control"); + System.out.println("PASS generated Java LF20 held provenance rejects before DML with valid-save positive control"); var combined = GeneratedDynamicRollbackAcceptance.verify(context, sql, extended, renamed, second, extension, provider, definitionContext); String deletionName = combined.getName(); GeneratedNamespaceCowAcceptance.verify(context,combined.getId(),deletionName,second,extension,provider,definitionContext,mapper); From 2448d3de56d3c99864f3f260072d961c48f2d469 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Thu, 8 Oct 2026 05:02:25 +0800 Subject: [PATCH 39/51] test: retain stream allocation samples and bind source hashes --- examples/shared-load-state/verify-allocations.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/examples/shared-load-state/verify-allocations.sh b/examples/shared-load-state/verify-allocations.sh index 23b2cda1..7e770fc2 100644 --- a/examples/shared-load-state/verify-allocations.sh +++ b/examples/shared-load-state/verify-allocations.sh @@ -22,6 +22,13 @@ fi run_dir="$(mktemp -d -t teaql-java-state-allocations.XXXXXXXX)" printf 'Evidence retained at %s\n' "$run_dir" java -version >"$run_dir/toolchain.txt" 2>&1 +git -C "$runtime_dir" rev-parse HEAD >"$run_dir/base-commit.txt" +git -C "$runtime_dir" diff --exit-code -- examples/shared-load-state teaql-core teaql-runtime teaql-sql-portable teaql-provider-jdbc teaql-sqlite +( + cd "$runtime_dir" + git ls-files -z teaql-core/src teaql-runtime/src teaql-sql-portable/src teaql-provider-jdbc/src teaql-sqlite/src examples/shared-load-state/src examples/shared-load-state/verify-allocations.sh | + xargs -0 sha256sum +) >"$run_dir/source.sha256" rg --files --hidden --no-ignore "$example_dir/target/generated/lib/src" -0 | sort -z | xargs -0 sha256sum >"$run_dir/generated-source.sha256" mvn -q -f "$runtime_dir/pom.xml" -Pshared-load-state-example \ @@ -34,6 +41,10 @@ printf 'case,width,iterations,allocated_bytes,elapsed_ns\n' >"$run_dir/allocatio rg '^(loaded_|reference_)' "$run_dir/probe.log" >>"$run_dir/allocations.csv" printf 'case,projection,rows,allocated_bytes,elapsed_ns\n' >"$run_dir/hydration.csv" rg '^(compiled_hydration|map_hydration|plain_hydration),' "$run_dir/probe.log" >>"$run_dir/hydration.csv" +printf 'case,lane,projection,rows,allocated_bytes\n' >"$run_dir/stream-hydration.csv" +rg '^STREAM_HYDRATION,' "$run_dir/probe.log" >>"$run_dir/stream-hydration.csv" +printf 'case,wide,rows,allocated_bytes\n' >"$run_dir/list-finalization.csv" +rg '^LIST_FINALIZE,' "$run_dir/probe.log" >>"$run_dir/list-finalization.csv" rg -F 'PASS mixed map/parallel-stream projection and dynamic-property NULL presence' "$run_dir/probe.log" if [[ "$wide" == true ]]; then printf 'case,entity,selected_fields,rows,allocated_bytes,elapsed_ns\n' >"$run_dir/wide-hydration.csv" @@ -44,4 +55,5 @@ if [[ "$wide" == true ]]; then rg -F 'PASS generated Java authoritative readback reuses one actual shape and avoids per-field overflow copies' "$run_dir/probe.log" fi sha256sum --check --quiet "$run_dir/generated-source.sha256" +(cd "$runtime_dir" && sha256sum --check --quiet "$run_dir/source.sha256") printf 'PASS Java warmed load-state allocation probe\n' From 6036d55e62020c3cbf6baa526d06249b2bc8a88e Mon Sep 17 00:00:00 2001 From: Philip Z Date: Thu, 8 Oct 2026 05:24:44 +0800 Subject: [PATCH 40/51] test: bound fresh projection allocation independently of row count --- .../LoadStateAllocationTest.java | 41 +++++++++++++++++++ .../shared-load-state/verify-allocations.sh | 2 +- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/LoadStateAllocationTest.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/LoadStateAllocationTest.java index 0ed2441e..fe4cfa5d 100644 --- a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/LoadStateAllocationTest.java +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/LoadStateAllocationTest.java @@ -56,6 +56,47 @@ private static final class SharingProbe extends BaseEntity { } } + @Test + public void freshProjectionSnapshotCostDoesNotGrowWithRows() { + var bean=(com.sun.management.ThreadMXBean) ManagementFactory.getThreadMXBean(); + assertTrue(bean.isThreadAllocatedMemorySupported());bean.setThreadAllocatedMemoryEnabled(true); + assertTrue(measure(bean,1,() -> consumed=new byte[128]).bytes()>=128); + new SharingProbe(); + var layout=FieldLayout.forType(SharingProbe.class); + for(int width:new int[]{4,64,130}) { + List fields=new ArrayList<>(); + for(int i=0;i rows=new ArrayList<>(count); + for(int i=0;i{ + var snapshot=LoadState.projection(layout,fields); + for(var row:rows)row.__internalUseLoadState(snapshot); + consumed=snapshot; + }; + // Warm the JVM code path, never reuse the measured snapshot. + for(int i=0;i<1000;i++)bind.run(); + var previous=rows.get(0).__internalLoadState(); + var sample=measure(bean,1,bind); + var snapshot=rows.get(0).__internalLoadState(); + assertNotSame(previous,snapshot,"new projection must not be a cached no-op"); + assertEquals(width>64,!snapshot.overflow().isEmpty()); + for(var row:rows) { + assertSame(snapshot,row.__internalLoadState()); + for(var field:fields)assertTrue(row.isPropertyLoaded(field)); + assertFalse(row.__internalHasMutationLedger()); + } + assertTrue(sample.bytes()>0,"a fresh snapshot must allocate"); + System.out.printf("fresh_projection,%d,%d,%d,%d%n",width,count,sample.bytes(),sample.nanos()); + // JIT may eliminate the one list iterator as the loop gets hot. + // More rows may therefore allocate less, never more state. + if(baseline!=null)assertTrue(sample.bytes()<=baseline,"state allocation grew with row count"); + else baseline=sample.bytes(); + } + } + } + @Test public void homogeneousListFinalizationAllocationProbe() { var bean=(com.sun.management.ThreadMXBean) ManagementFactory.getThreadMXBean(); diff --git a/examples/shared-load-state/verify-allocations.sh b/examples/shared-load-state/verify-allocations.sh index 7e770fc2..7e6e0794 100644 --- a/examples/shared-load-state/verify-allocations.sh +++ b/examples/shared-load-state/verify-allocations.sh @@ -38,7 +38,7 @@ mvn -q -f "$runtime_dir/pom.xml" -Pshared-load-state-example \ exit 1 } printf 'case,width,iterations,allocated_bytes,elapsed_ns\n' >"$run_dir/allocations.csv" -rg '^(loaded_|reference_)' "$run_dir/probe.log" >>"$run_dir/allocations.csv" +rg '^(fresh_projection|loaded_|reference_)' "$run_dir/probe.log" >>"$run_dir/allocations.csv" printf 'case,projection,rows,allocated_bytes,elapsed_ns\n' >"$run_dir/hydration.csv" rg '^(compiled_hydration|map_hydration|plain_hydration),' "$run_dir/probe.log" >>"$run_dir/hydration.csv" printf 'case,lane,projection,rows,allocated_bytes\n' >"$run_dir/stream-hydration.csv" From 44d5526353c737c192b36a65bd9be358b679aa73 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Thu, 8 Oct 2026 05:44:15 +0800 Subject: [PATCH 41/51] test: compare mainstream MyBatis mapped queries with governed typed runtime lists --- .../shared-load-state/benchmark-mainstream.sh | 18 +++ examples/shared-load-state/pom.xml | 1 + .../MainstreamMapperBenchmarkTest.java | 124 ++++++++++++++++++ 3 files changed, 143 insertions(+) create mode 100644 examples/shared-load-state/benchmark-mainstream.sh create mode 100644 examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/MainstreamMapperBenchmarkTest.java diff --git a/examples/shared-load-state/benchmark-mainstream.sh b/examples/shared-load-state/benchmark-mainstream.sh new file mode 100644 index 00000000..38cae64e --- /dev/null +++ b/examples/shared-load-state/benchmark-mainstream.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail +(( $# == 0 )) || { printf 'Usage: bash benchmark-mainstream.sh\n' >&2; exit 2; } +runtime_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +run_dir="$(mktemp -d -t teaql-java-mainstream.XXXXXXXX)" +printf 'Mainstream benchmark evidence: %s\n' "$run_dir" +cd "$runtime_dir" +git diff HEAD --exit-code -- examples/shared-load-state teaql-core teaql-runtime teaql-sql-portable teaql-provider-jdbc teaql-sqlite +git rev-parse HEAD >"$run_dir/base-commit.txt" +java -version >"$run_dir/toolchain.txt" 2>&1 +sha256sum examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/{MainstreamMapperBenchmarkTest,DriverQueryBenchmarkTest,HydrationAllocationTest}.java examples/shared-load-state/pom.xml >"$run_dir/source.sha256" +/usr/bin/time -v -o "$run_dir/cpu.txt" mvn -q -Pshared-load-state-example \ + -pl examples/shared-load-state -am -Dtest=MainstreamMapperBenchmarkTest \ + -Dsurefire.failIfNoSpecifiedTests=false -Dteaql.ormBenchmark=true -Dteaql.ormSamples=31 test \ + >"$run_dir/results.log" 2>&1 +rg -F 'PASS mainstream MyBatis and TeaQL bounded typed results with independent version and prefix filters' "$run_dir/results.log" +cp examples/shared-load-state/target/surefire-reports/TEST-com.example.schoolmanagementservice.MainstreamMapperBenchmarkTest.xml "$run_dir/test-result.xml" +sha256sum --check --quiet "$run_dir/source.sha256" diff --git a/examples/shared-load-state/pom.xml b/examples/shared-load-state/pom.xml index dd4d5cac..d2ceb383 100644 --- a/examples/shared-load-state/pom.xml +++ b/examples/shared-load-state/pom.xml @@ -17,6 +17,7 @@ com.fasterxml.jackson.datatypejackson-datatype-jsr310test org.xerialsqlite-jdbc3.53.2.1 org.junit.jupiterjunit-jupitertest + org.mybatismybatis3.5.19test diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/MainstreamMapperBenchmarkTest.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/MainstreamMapperBenchmarkTest.java new file mode 100644 index 00000000..339656ff --- /dev/null +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/MainstreamMapperBenchmarkTest.java @@ -0,0 +1,124 @@ +package com.example.schoolmanagementservice; + +import io.teaql.core.*; +import io.teaql.core.criteria.Operator; +import io.teaql.core.meta.SimpleEntityMetaFactory; +import io.teaql.core.sql.GenericSQLProperty; +import io.teaql.core.sql.SQLEntityDescriptor; +import io.teaql.core.sqlite.SqliteDataServiceExecutor; +import io.teaql.provider.jdbc.JdbcSqlExecutor; +import io.teaql.runtime.DefaultUserContext; +import io.teaql.runtime.TeaQLRuntime; +import org.apache.ibatis.annotations.*; +import org.apache.ibatis.mapping.Environment; +import org.apache.ibatis.session.*; +import org.apache.ibatis.transaction.jdbc.JdbcTransactionFactory; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; +import org.sqlite.SQLiteDataSource; +import java.lang.management.ManagementFactory; +import java.nio.file.Files; +import java.sql.*; +import java.util.*; +import java.util.function.Supplier; +import static org.junit.jupiter.api.Assertions.*; + +/** Mainstream mapper track, not matched-SQL or generated-library performance. */ +@EnabledIfSystemProperty(named="teaql.ormBenchmark", matches="true") +public class MainstreamMapperBenchmarkTest { + public record Sparse(Long id, Long version, String name) {} + public record Full(Long id, Long version, String name, String note) {} + public interface Mapper { + @Select("SELECT id,version,name FROM probe_data WHERE version>0 AND name LIKE #{prefix} ORDER BY id ASC LIMIT #{limit}") + @ConstructorArgs({@Arg(column="id",javaType=Long.class),@Arg(column="version",javaType=Long.class),@Arg(column="name",javaType=String.class)}) + List sparse(@Param("prefix") String prefix, @Param("limit") int limit); + @Select("SELECT id,version,name,note FROM probe_data WHERE version>0 AND name LIKE #{prefix} ORDER BY id ASC LIMIT #{limit}") + @ConstructorArgs({@Arg(column="id",javaType=Long.class),@Arg(column="version",javaType=Long.class),@Arg(column="name",javaType=String.class),@Arg(column="note",javaType=String.class)}) + List full(@Param("prefix") String prefix, @Param("limit") int limit); + } + static List load(SqlSessionFactory factory, boolean full, int limit) { + // A fresh method-scoped session prevents repeated-query result caching. + try (SqlSession session=factory.openSession()) { + var mapper=session.getMapper(Mapper.class); + return full?mapper.full("row-%",limit):mapper.sparse("row-%",limit); + } + } + @Test void mainstreamMapperAndGovernedQueriesAgree() throws Exception { + int samples=Integer.getInteger("teaql.ormSamples",31); + assertTrue(samples>=3 && samples<=31); + assertNotNull(HydrationAllocationTest.Probe.LAYOUT); + var metadata=new SimpleEntityMetaFactory(); + var descriptor=new SQLEntityDescriptor();descriptor.setType("Probe"); + descriptor.setTargetType(HydrationAllocationTest.Probe.class); + descriptor.setEntitySupplier(HydrationAllocationTest.Probe::new); + for(String field:List.of("id","version","name","note")) { + var property=(GenericSQLProperty)descriptor.addSimpleProperty(field, + field.equals("id")||field.equals("version")?Long.class:String.class); + property.setColumnType(field.equals("id")||field.equals("version")?"BIGINT":"VARCHAR(255)"); + } + metadata.register(descriptor); + var source=new SQLiteDataSource(); + source.setUrl("jdbc:sqlite:"+Files.createTempDirectory("teaql-mapper-").resolve("probe.sqlite")); + var executor=new JdbcSqlExecutor(source); + var context=new DefaultUserContext(TeaQLRuntime.builder().metadata(metadata) + .dataService("default",new SqliteDataServiceExecutor("sqlite",executor,source)) + .queryExecutionLogging(false).build()); + context.ensureSchema(); + // Deterministic benchmark setup only; never a runtime seeding API. + try(Connection connection=source.getConnection()) { + System.out.println("ENGINE_VERSION,sqlite,"+connection.getMetaData().getDatabaseProductVersion()); + System.out.println("DRIVER_VERSION,"+connection.getMetaData().getDriverVersion()); + connection.setAutoCommit(false); + try(PreparedStatement insert=connection.prepareStatement("INSERT INTO probe_data(id,version,name,note) VALUES(?,?,?,?)")) { + for(long id=1;id<=10_002;id++) { + insert.setLong(1,id);insert.setLong(2,id==10_001?-1:1); + insert.setString(3,id==10_002?"not-selected":"row-"+id); + insert.setString(4,id%2==0?null:"nullable note");insert.addBatch(); + } + insert.executeBatch(); + } + connection.commit(); + } + var configuration=new Configuration(new Environment("bench",new JdbcTransactionFactory(),source)); + configuration.addMapper(Mapper.class); + var factory=new SqlSessionFactoryBuilder().build(configuration); + System.out.println("FRAMEWORK,mybatis,"+SqlSession.class.getPackage().getImplementationVersion()); + System.out.println("ORM_LOG_MODE,off"); + System.out.println("ORM_CONNECTION_POLICY,file-backed-method-scoped-both-lanes"); + var bean=(com.sun.management.ThreadMXBean)ManagementFactory.getThreadMXBean(); + assertTrue(bean.isThreadAllocatedMemorySupported());bean.setThreadAllocatedMemoryEnabled(true); + for(boolean full:List.of(false,true))for(int count:new int[]{1,100,10_000}) { + var request=new HydrationAllocationTest.Request(metadata,full,count); + request.appendSearchCriteria(request.createBasicSearchCriteria("name",Operator.BEGIN_WITH,"row-")); + request.addOrderByAscending("id"); + var query=request.purpose("why: compare mainstream bounded typed retrieval"); + Supplier> mapperAction=()->load(factory,full,count); + Supplier> teaqlAction=()->query.executeForList(context); + for(int i=0;i<20;i++){DriverQueryBenchmarkTest.consumed=mapperAction.get();DriverQueryBenchmarkTest.consumed=teaqlAction.get();} + List> mapperSamples=new ArrayList<>(),teaqlSamples=new ArrayList<>(); + for(int sample=0;sample> mapped; + DriverQueryBenchmarkTest.Sample> typed; + if(sample%2==0){mapped=DriverQueryBenchmarkTest.measure(bean,mapperAction);typed=DriverQueryBenchmarkTest.measure(bean,teaqlAction);} + else{typed=DriverQueryBenchmarkTest.measure(bean,teaqlAction);mapped=DriverQueryBenchmarkTest.measure(bean,mapperAction);} + assertEquals(count,mapped.value().size());assertEquals(count,typed.value().size()); + var state=typed.value().get(0).__internalLoadState(); + for(int index=0;index(null,mapped.bytes(),mapped.nanos())); + teaqlSamples.add(new DriverQueryBenchmarkTest.Sample<>(null,typed.bytes(),typed.nanos())); + } + DriverQueryBenchmarkTest.report("mybatis",full,count,mapperSamples); + DriverQueryBenchmarkTest.report("teaql",full,count,teaqlSamples); + } + System.out.println("PASS mainstream MyBatis and TeaQL bounded typed results with independent version and prefix filters"); + } +} From 81cb30286159ec20d1f014687c129ff661ecc648 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Thu, 8 Oct 2026 06:01:07 +0800 Subject: [PATCH 42/51] test: put excluded rows before bounded result windows --- .../MainstreamMapperBenchmarkTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/MainstreamMapperBenchmarkTest.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/MainstreamMapperBenchmarkTest.java index 339656ff..d92fed36 100644 --- a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/MainstreamMapperBenchmarkTest.java +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/MainstreamMapperBenchmarkTest.java @@ -71,8 +71,8 @@ static List load(SqlSessionFactory factory, boolean full, int limit) { connection.setAutoCommit(false); try(PreparedStatement insert=connection.prepareStatement("INSERT INTO probe_data(id,version,name,note) VALUES(?,?,?,?)")) { for(long id=1;id<=10_002;id++) { - insert.setLong(1,id);insert.setLong(2,id==10_001?-1:1); - insert.setString(3,id==10_002?"not-selected":"row-"+id); + insert.setLong(1,id);insert.setLong(2,id==1?-1:1); + insert.setString(3,id==2?"not-selected":"row-"+id); insert.setString(4,id%2==0?null:"nullable note");insert.addBatch(); } insert.executeBatch(); @@ -105,7 +105,7 @@ static List load(SqlSessionFactory factory, boolean full, int limit) { var state=typed.value().get(0).__internalLoadState(); for(int index=0;index Date: Thu, 8 Oct 2026 06:28:58 +0800 Subject: [PATCH 43/51] test: measure generated wide dynamic and relation queries with Q E controls --- .../GeneratedQueryBenchmarkTest.java | 167 ++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedQueryBenchmarkTest.java diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedQueryBenchmarkTest.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedQueryBenchmarkTest.java new file mode 100644 index 00000000..214b851c --- /dev/null +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedQueryBenchmarkTest.java @@ -0,0 +1,167 @@ +package com.example.schoolmanagementservice; + +import com.example.schoolmanagementservice.school.School; +import io.teaql.core.*; +import io.teaql.core.meta.SimpleEntityMetaFactory; +import io.teaql.core.sql.portable.IdSpaceIdGenerator; +import io.teaql.core.sqlite.SqliteDataServiceExecutor; +import io.teaql.provider.jdbc.JdbcSqlExecutor; +import io.teaql.runtime.DefaultUserContext; +import io.teaql.runtime.TeaQLRuntime; +import io.teaql.data.dynamic.*; +import io.teaql.data.dynamic.jdbc.JdbcDynamicFieldsProvider; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; +import org.sqlite.SQLiteDataSource; +import java.lang.management.ManagementFactory; +import java.nio.file.Files; +import java.sql.*; +import java.time.*; +import java.util.*; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import static org.junit.jupiter.api.Assertions.*; + +/** Real generated Q/E; fixture expansion alone uses SQL outside measurements. */ +@EnabledIfSystemProperty(named="teaql.generatedBenchmark",matches="true") +class GeneratedQueryBenchmarkTest { + static final class Context extends DefaultUserContext implements SchoolManagementServiceUserContext { + Context(TeaQLRuntime runtime){super(runtime);} + } + static String quoted(String name){return "\""+name.replace("\"","\"\"")+"\"";} + static void expandFixture(SQLiteDataSource source) throws Exception { + try(Connection connection=source.getConnection()) { + List columns=new ArrayList<>(); + try(Statement statement=connection.createStatement();ResultSet rows=statement.executeQuery("PRAGMA table_info(school_data)")) { + while(rows.next())columns.add(rows.getString("name")); + } + assertTrue(columns.containsAll(List.of("id","name","version"))); + assertTrue(columns.size()>130,"use the wide generated fixture"); + String target=columns.stream().map(GeneratedQueryBenchmarkTest::quoted).collect(Collectors.joining(",")); + String expressions=columns.stream().map(c->c.equals("id")||c.equals("name")?"?":quoted(c)).collect(Collectors.joining(",")); + String sql="INSERT INTO school_data("+target+") SELECT "+expressions+" FROM school_data WHERE id=1"; + int idParameter=columns.indexOf("id") query(UserContext context,String scenario,int count) { + if(scenario.equals("reverse")) { + var parent=Q.platformsWithMinimalFields().withIdIs(1L) + .selectSchoolListWith(Q.schoolsWithMinimalFields().selectName().orderByIdAscending().limit(count)) + .limit(1).comment("what: load bounded generated reverse graph") + .purpose("why: measure relation hydration and shared state").executeForOne(context); + assertTrue(parent.isPropertyLoaded("schoolList")); + return parent.getSchoolList(); + } + var request=scenario.equals("wide")||scenario.equals("dynamic") + ?Q.schools().selectSelfFields():Q.schoolsWithMinimalFields().selectName(); + if(scenario.equals("dynamic"))request.selectDynamicFieldsWith(new DynamicFieldSelection().selectString("note")); + if(scenario.equals("forward")) { + request.selectPlatformWith(Q.platformsWithMinimalFields().selectName()); + request.selectSchoolTypeWith(Q.schoolTypesWithMinimalFields().selectCode()); + } + return request.orderByIdAscending().limit(count).comment("what: load generated "+scenario+" Schools") + .purpose("why: measure typed generated runtime query cost").executeForList(context); + } + static void validate(SmartList rows,String scenario,int count) { + assertEquals(count,rows.size()); + boolean complete=scenario.equals("wide")||scenario.equals("dynamic"); + LoadState first=rows.get(0).__internalLoadState();LoadState missing=null; + for(int index=0;index1) { + rows.get(0).updateAddress("private divergent value"); + assertTrue(rows.get(0).isPropertyLoaded("address")); + assertFalse(rows.get(1).isPropertyLoaded("address")); + assertSame(first,rows.get(1).__internalLoadState()); + assertNotSame(first,rows.get(0).__internalLoadState()); + } + } + @Test void generatedWideDynamicAndRelationQueries() throws Exception { + String mode=System.getenv().getOrDefault("TEAQL_GENERATED_BENCHMARK_LOG_MODE","off"); + assertTrue(Set.of("off","default").contains(mode)); + int samples=Integer.getInteger("teaql.generatedSamples",31);assertTrue(samples>=3&&samples<=31); + var source=new SQLiteDataSource();source.setUrl("jdbc:sqlite:"+Files.createTempDirectory("teaql-generated-bench-").resolve("school.sqlite")); + var sql=new JdbcSqlExecutor(source); + var ids=new IdSpaceIdGenerator(new GeneratedSchoolLoadStateTest.DatabaseBridge(sql));ids.ensureIdSpaceTable(); + var service=new SqliteDataServiceExecutor("sqlite",sql,source); + var context=new Context(TeaQLRuntime.builder().metadata(new SimpleEntityMetaFactory()) + .dataService("default",service).dataService("sqlite",service).idGenerationService(ids) + .queryExecutionLogging(mode.equals("default")).build().install(GeneratedRuntimeModule.module())); + context.ensureSchema(); + var platform=Q.platforms().withIdIs(1L).limit(1).comment("what: reuse the generated root") + .purpose("why: construct a compliant benchmark prototype").executeForOne(context); + var prototype=Q.schools().comment("what: allocate a benchmark prototype") + .purpose("why: validate the generated mutation contract before SQL fixture expansion").newEntity(context); + prototype.updatePlatform(platform);prototype.updateSchoolTypeToPrimary(); + prototype.updateName("bench-school-1");prototype.updateAddress("Benchmark address"); + prototype.updateEstablishedDate(LocalDate.of(1995,9,1));prototype.updateStudentCapacity(17);prototype.updateActive(false); + prototype.updateCreateTime(LocalDateTime.of(2023,11,14,22,13));prototype.updateUpdateTime(LocalDateTime.of(2023,11,14,22,13)); + prototype.auditAs("Create the single generated benchmark prototype").save(context);assertEquals(1L,prototype.getId()); + expandFixture(source); + var provider=new JdbcDynamicFieldsProvider(sql);provider.ensureSchema(); + DynamicFieldContext definitionContext=new DynamicFieldContext() { + public String scopeType(){return "GLOBAL";}public String scopeId(){return "default";} + public String userId(){return "generated-benchmark";}public String purpose(){return "prepare controlled extensions";} + public String comment(){return "what: register benchmark note";}public boolean strictIntent(){return true;} + public long nextId(String type){return ids.nextId(type);} + }; + var definition=new DynamicFieldDef();definition.setScope(DynamicFieldScope.global()); + definition.setOwnerType("School");definition.setCode("note");definition.setName("note");definition.setDataType(DynamicDataType.STRING); + provider.registerFieldDef(definitionContext,definition); + context.putAttribute(DynamicFieldsFacade.class.getName(),new DefaultDynamicFieldsFacade(provider)); + for(long id:new long[]{1,2}) { + var request=Q.schools().withIdIs(id).selectSelfFields();request.selectDynamicFieldsWith(new DynamicFieldSelection().selectString("note")); + var row=request.limit(1).comment("what: load trusted extension input").purpose("why: seed Value and NULL through audited save").executeForOne(context); + row.updateDynamicField("note",id==1?"benchmark note":null);row.auditAs("Seed generated benchmark extension").save(context); + } + System.out.println("GENERATED_LOG_MODE,"+mode);System.out.println("GENERATED_FIXED_FIELDS,"+School.__TEAQL_FIXED_FIELD_INDEXES.size()); + var bean=(com.sun.management.ThreadMXBean)ManagementFactory.getThreadMXBean();assertTrue(bean.isThreadAllocatedMemorySupported());bean.setThreadAllocatedMemoryEnabled(true); + for(String scenario:List.of("sparse","wide","dynamic","forward","reverse"))for(int count:new int[]{1,100,10000}) { + Supplier> action=()->query(context,scenario,count); + for(int i=0;i<5;i++)DriverQueryBenchmarkTest.consumed=action.get(); + long[] times=new long[samples],bytes=new long[samples]; + for(int index=0;index Date: Thu, 8 Oct 2026 06:32:38 +0800 Subject: [PATCH 44/51] perf: share JDBC map column layout and keep row values private --- .../shared-load-state/benchmark-generated.sh | 29 ++++++ .../io/teaql/provider/jdbc/JdbcColumnRow.java | 91 +++++++++++++++++++ .../teaql/provider/jdbc/JdbcSqlExecutor.java | 21 +++-- .../provider/jdbc/JdbcColumnRowTest.java | 51 +++++++++++ .../provider/jdbc/JdbcSqlExecutorTest.java | 26 ++++++ 5 files changed, 208 insertions(+), 10 deletions(-) create mode 100644 examples/shared-load-state/benchmark-generated.sh create mode 100644 teaql-provider-jdbc/src/main/java/io/teaql/provider/jdbc/JdbcColumnRow.java create mode 100644 teaql-provider-jdbc/src/test/java/io/teaql/provider/jdbc/JdbcColumnRowTest.java diff --git a/examples/shared-load-state/benchmark-generated.sh b/examples/shared-load-state/benchmark-generated.sh new file mode 100644 index 00000000..9cdba2d2 --- /dev/null +++ b/examples/shared-load-state/benchmark-generated.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -euo pipefail +mode=off +samples=31 +for option in "$@"; do + case "$option" in + --default-log) mode=default ;; + --smoke) samples=3 ;; + *) printf 'Usage: bash benchmark-generated.sh [--default-log] [--smoke]\n' >&2; exit 2 ;; + esac +done +runtime_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +run_dir="$(mktemp -d -t teaql-java-generated-query.XXXXXXXX)" +printf 'Generated query evidence: %s\n' "$run_dir" +cd "$runtime_dir" +git diff HEAD --exit-code -- examples/shared-load-state teaql-core teaql-runtime teaql-provider-jdbc teaql-sql-portable +git rev-parse HEAD >"$run_dir/base-commit.txt" +java -version >"$run_dir/toolchain.txt" 2>&1 +sha256sum examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedQueryBenchmarkTest.java >"$run_dir/source.sha256" +rg --files --hidden --no-ignore examples/shared-load-state/target/generated/lib/src -0 | + sort -z | xargs -0 sha256sum >"$run_dir/generated-source.sha256" +TEAQL_GENERATED_BENCHMARK_LOG_MODE="$mode" /usr/bin/time -v -o "$run_dir/cpu.txt" \ + mvn -q -Pshared-load-state-example -pl examples/shared-load-state -am \ + -Dtest=GeneratedQueryBenchmarkTest -Dsurefire.failIfNoSpecifiedTests=false \ + -Dteaql.generatedBenchmark=true "-Dteaql.generatedSamples=$samples" test >"$run_dir/results.log" 2>&1 +rg -F 'PASS generated Java wide dynamic forward reverse Q/E sharing and private divergence benchmark' "$run_dir/results.log" +cp examples/shared-load-state/target/surefire-reports/TEST-com.example.schoolmanagementservice.GeneratedQueryBenchmarkTest.xml "$run_dir/results.xml" +sha256sum --check --quiet "$run_dir/source.sha256" +sha256sum --check --quiet "$run_dir/generated-source.sha256" diff --git a/teaql-provider-jdbc/src/main/java/io/teaql/provider/jdbc/JdbcColumnRow.java b/teaql-provider-jdbc/src/main/java/io/teaql/provider/jdbc/JdbcColumnRow.java new file mode 100644 index 00000000..8017107a --- /dev/null +++ b/teaql-provider-jdbc/src/main/java/io/teaql/provider/jdbc/JdbcColumnRow.java @@ -0,0 +1,91 @@ +package io.teaql.provider.jdbc; + +import java.util.*; + +/** Query-local immutable column geometry and row-private values, never entity load state. */ +final class JdbcColumnRow extends AbstractMap { + static final class Layout { + private final Map indexes; + private final String[] keys; + private final int width; + Layout(String[] labels) { + var positions=new LinkedHashMap(); + for(int index=0;index expanded; + private int modifications; + JdbcColumnRow(Layout layout,Object[] ownedValues) { + this.layout=Objects.requireNonNull(layout); + if(ownedValues.length!=layout.width)throw new IllegalArgumentException("JDBC row width differs from column layout"); + values=ownedValues; + } + @Override public int size(){return expanded==null?layout.keys.length:expanded.size();} + @Override public boolean containsKey(Object key){return expanded==null?layout.indexes.containsKey(key):expanded.containsKey(key);} + @Override public Object get(Object key) { + if(expanded!=null)return expanded.get(key); + Integer index=layout.indexes.get(key);return index==null?null:values[index]; + } + private Map expand() { + if(expanded==null) { + expanded=new HashMap<>(); + for(String key:layout.keys)expanded.put(key,getOriginal(key)); + } + return expanded; + } + private Object getOriginal(String key){return values[layout.indexes.get(key)];} + @Override public Object put(String key,Object value) { + if(expanded==null) { + Integer index=layout.indexes.get(key); + if(index!=null){Object old=values[index];values[index]=value;return old;} + } + boolean present=containsKey(key);Object old=expand().put(key,value); + if(!present)modifications++;return old; + } + @Override public Object remove(Object key) { + if(!containsKey(key))return null; + Object old=expand().remove(key);modifications++;return old; + } + @Override public void clear() { + if(isEmpty())return; + if(expanded==null)expanded=new HashMap<>();else expanded.clear(); + modifications++; + } + @Override public Set> entrySet() { + return new AbstractSet<>() { + @Override public int size(){return JdbcColumnRow.this.size();} + @Override public void clear(){JdbcColumnRow.this.clear();} + @Override public Iterator> iterator() { + // Use a stable key snapshot only after this row's geometry has + // changed. The normal readonly path shares layout.keys directly. + String[] keys=expanded==null?layout.keys:expanded.keySet().toArray(String[]::new); + return new Iterator<>() { + int position,expected=modifications;String last;boolean removable; + private void check(){if(expected!=modifications)throw new ConcurrentModificationException();} + @Override public boolean hasNext(){check();return position next() { + if(!hasNext())throw new NoSuchElementException(); + String key=keys[position++];last=key;removable=true; + return new Entry<>() { + public String getKey(){return key;} + public Object getValue(){return JdbcColumnRow.this.get(key);} + public Object setValue(Object value){return JdbcColumnRow.this.put(key,value);} + public boolean equals(Object other){return other instanceof Entry e&&Objects.equals(key,e.getKey())&&Objects.equals(getValue(),e.getValue());} + public int hashCode(){return Objects.hashCode(key)^Objects.hashCode(getValue());} + public String toString(){return key+"="+getValue();} + }; + } + @Override public void remove() { + check();if(!removable)throw new IllegalStateException(); + JdbcColumnRow.this.remove(last);expected=modifications;removable=false; + } + }; + } + }; + } +} diff --git a/teaql-provider-jdbc/src/main/java/io/teaql/provider/jdbc/JdbcSqlExecutor.java b/teaql-provider-jdbc/src/main/java/io/teaql/provider/jdbc/JdbcSqlExecutor.java index c7949342..c433a51e 100644 --- a/teaql-provider-jdbc/src/main/java/io/teaql/provider/jdbc/JdbcSqlExecutor.java +++ b/teaql-provider-jdbc/src/main/java/io/teaql/provider/jdbc/JdbcSqlExecutor.java @@ -61,11 +61,8 @@ public Stream queryForStream(String sql, Map params, SqlR public Stream> queryForStream(String sql, Object[] params) { return streamQuery(sql, params, rows -> { String[] columnLabels = columnLabels(rows); - return () -> { - Map row = new java.util.HashMap<>(); - for (int i = 0; i < columnLabels.length; i++) row.put(columnLabels[i], rows.getObject(i + 1)); - return row; - }; + JdbcColumnRow.Layout layout = new JdbcColumnRow.Layout(columnLabels); + return () -> readColumnRow(rows, layout, columnLabels.length); }); } @@ -178,12 +175,9 @@ public List> queryForList(String sql, Object[] params) { } try (ResultSet rs = ps.executeQuery()) { String[] columnLabels = columnLabels(rs); + JdbcColumnRow.Layout layout = new JdbcColumnRow.Layout(columnLabels); while (rs.next()) { - java.util.Map row = new java.util.HashMap<>(); - for (int i = 0; i < columnLabels.length; i++) { - row.put(columnLabels[i], rs.getObject(i + 1)); - } - result.add(row); + result.add(readColumnRow(rs, layout, columnLabels.length)); } } return result; @@ -292,6 +286,13 @@ private static V convertColumn(Class type, Object value) { } } + private static JdbcColumnRow readColumnRow(ResultSet resultSet, JdbcColumnRow.Layout layout, int width) + throws SQLException { + Object[] values = new Object[width]; + for (int i = 0; i < width; i++) values[i] = resultSet.getObject(i + 1); + return new JdbcColumnRow(layout, values); + } + private static String[] columnLabels(ResultSet resultSet) throws SQLException { java.sql.ResultSetMetaData metadata = resultSet.getMetaData(); int columnCount = metadata.getColumnCount(); diff --git a/teaql-provider-jdbc/src/test/java/io/teaql/provider/jdbc/JdbcColumnRowTest.java b/teaql-provider-jdbc/src/test/java/io/teaql/provider/jdbc/JdbcColumnRowTest.java new file mode 100644 index 00000000..69b6c37c --- /dev/null +++ b/teaql-provider-jdbc/src/test/java/io/teaql/provider/jdbc/JdbcColumnRowTest.java @@ -0,0 +1,51 @@ +package io.teaql.provider.jdbc; + +import java.util.*; +import org.junit.Test; +import static org.junit.Assert.*; + +public class JdbcColumnRowTest { + @Test public void nullAbsenceAndRealValuesRemainDistinct() { + var row=new JdbcColumnRow(new JdbcColumnRow.Layout(new String[]{"null","zero","false","empty"}),new Object[]{null,0,false,""}); + assertTrue(row.containsKey("null"));assertNull(row.get("null")); + assertFalse(row.containsKey("missing"));assertNull(row.get("missing")); + assertEquals(0,row.get("zero"));assertEquals(false,row.get("false"));assertEquals("",row.get("empty")); + var reference=new HashMap();reference.put("null",null);reference.put("zero",0);reference.put("false",false);reference.put("empty",""); + assertEquals(reference,row);assertEquals(row,reference);assertEquals(reference.hashCode(),row.hashCode()); + } + @Test public void duplicateAndNullLabelsKeepLastColumnLikeHashMap() { + String[] labels={"id","id",null};var layout=new JdbcColumnRow.Layout(labels);labels[0]="changed"; + var row=new JdbcColumnRow(layout,new Object[]{1,2,3}); + assertEquals(2,row.size());assertEquals(2,row.get("id"));assertEquals(3,row.get(null));assertFalse(row.containsKey("changed")); + assertThrows(IllegalArgumentException.class,()->new JdbcColumnRow(layout,new Object[2])); + } + @Test public void valueAndGeometryChangesArePrivateToTheRow() { + var layout=new JdbcColumnRow.Layout(new String[]{"id","note"}); + var first=new JdbcColumnRow(layout,new Object[]{1,null});var second=new JdbcColumnRow(layout,new Object[]{2,"private"}); + assertNull(first.put("note","changed"));assertEquals("private",second.get("note")); + first.put("extra",null);assertTrue(first.containsKey("extra"));assertFalse(second.containsKey("extra")); + assertEquals("changed",first.remove("note"));assertFalse(first.containsKey("note"));assertTrue(second.containsKey("note")); + first.clear();assertTrue(first.isEmpty());assertEquals(2,second.size()); + } + @Test public void entryValuesAndIteratorRemovalUseOnlyPrivatePayloads() { + var row=new JdbcColumnRow(new JdbcColumnRow.Layout(new String[]{"id","note"}),new Object[]{1,"before"}); + var iterator=row.entrySet().iterator();var id=iterator.next();assertEquals(1,id.setValue(3));assertEquals(3,row.get("id")); + var note=iterator.next();assertEquals("before",note.setValue("after"));iterator.remove(); + assertFalse(row.containsKey("note"));assertFalse(iterator.hasNext());assertThrows(IllegalStateException.class,iterator::remove); + row.put("extra",5);assertEquals(Set.of("id","extra"),row.keySet()); + assertTrue(row.values().remove(5));assertFalse(row.containsKey("extra")); + } + @Test public void structuralChangesFailFastButValueUpdatesDoNot() { + var row=new JdbcColumnRow(new JdbcColumnRow.Layout(new String[]{"id"}),new Object[]{1}); + var iterator=row.entrySet().iterator();row.put("id",2);assertEquals(2,iterator.next().getValue()); + iterator=row.entrySet().iterator();row.put("new",null);assertThrows(ConcurrentModificationException.class,iterator::hasNext); + } + @Test public void defaultMapOperationsPreserveNullMembershipAndViews() { + var row=new JdbcColumnRow(new JdbcColumnRow.Layout(new String[]{"id","note"}),new Object[]{1,null}); + row.computeIfAbsent("note",key->"loaded");row.replaceAll((key,value)->value.toString()); + assertEquals(Map.of("id","1","note","loaded"),row); + assertTrue(row.keySet().remove("id"));row.putAll(Map.of("x",3));assertEquals(2,row.size()); + assertTrue(row.entrySet().remove(new AbstractMap.SimpleEntry<>("x",3))); + row.entrySet().clear();assertTrue(row.isEmpty()); + } +} diff --git a/teaql-provider-jdbc/src/test/java/io/teaql/provider/jdbc/JdbcSqlExecutorTest.java b/teaql-provider-jdbc/src/test/java/io/teaql/provider/jdbc/JdbcSqlExecutorTest.java index 09e09f13..7e311267 100644 --- a/teaql-provider-jdbc/src/test/java/io/teaql/provider/jdbc/JdbcSqlExecutorTest.java +++ b/teaql-provider-jdbc/src/test/java/io/teaql/provider/jdbc/JdbcSqlExecutorTest.java @@ -85,6 +85,32 @@ public void testTypedJdbcParametersRemainPortable() { assertEquals(1, ((Number) rows.get(0).get("id")).intValue()); } + @Test + public void testListRowMutationCannotChangeSiblingColumnsOrValues() { + var rows = sqlExecutor.queryForList("SELECT 1 AS id, NULL AS note UNION ALL SELECT 2, 'private'", new Object[0]); + assertTrue(rows.get(0) instanceof JdbcColumnRow); + assertTrue(rows.get(0).containsKey("note")); + rows.get(0).put("note", "changed"); + rows.get(0).remove("id"); + rows.get(0).put("extra", null); + assertEquals(2, ((Number) rows.get(1).get("id")).intValue()); + assertEquals("private", rows.get(1).get("note")); + assertTrue(!rows.get(1).containsKey("extra")); + } + + @Test + public void testMapStreamRowsSurviveCloseAndDuplicateAliasesKeepLastValue() { + List> rows; + try (var stream = sqlExecutor.queryForStream("SELECT 1 AS \"Name\", 2 AS \"NAME\" UNION ALL SELECT 3, 4", new Object[0])) { + rows = stream.toList(); + } + assertTrue(rows.get(0) instanceof JdbcColumnRow); + assertEquals(1, rows.get(0).size()); + assertEquals(2, ((Number) rows.get(0).get("name")).intValue()); + rows.get(0).put("name", 5); + assertEquals(4, ((Number) rows.get(1).get("name")).intValue()); + } + @Test public void testNullParameterUsesSqlNull() { sqlExecutor.update( From 2c3e9537f718dcec25766c3b5b66ae338e44abf0 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Thu, 8 Oct 2026 06:40:47 +0800 Subject: [PATCH 45/51] test: release expanded row arrays and alternate matched JDBC map reference --- .../GeneratedQueryBenchmarkTest.java | 47 +++++++++++++++++-- .../io/teaql/provider/jdbc/JdbcColumnRow.java | 6 ++- .../provider/jdbc/JdbcColumnRowTest.java | 10 ++++ 3 files changed, 59 insertions(+), 4 deletions(-) diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedQueryBenchmarkTest.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedQueryBenchmarkTest.java index 214b851c..5bb0af94 100644 --- a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedQueryBenchmarkTest.java +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedQueryBenchmarkTest.java @@ -28,6 +28,29 @@ class GeneratedQueryBenchmarkTest { static final class Context extends DefaultUserContext implements SchoolManagementServiceUserContext { Context(TeaQLRuntime runtime){super(runtime);} } + /** Controlled reconstruction of the previous generic JDBC query path. */ + static final class SwitchingSql extends JdbcSqlExecutor { + private final SQLiteDataSource source; + boolean reference; + SwitchingSql(SQLiteDataSource source){super(source);this.source=source;} + @Override public List> queryForList(String sql,Object[] params) { + if(!reference)return super.queryForList(sql,params); + try(Connection connection=source.getConnection();PreparedStatement statement=connection.prepareStatement(sql)) { + if(params!=null)for(int i=0;i> result=new ArrayList<>(); + while(rows.next()) { + Map row=new HashMap<>(); + for(int i=0;i rows,String scenario,int count) { assertTrue(Set.of("off","default").contains(mode)); int samples=Integer.getInteger("teaql.generatedSamples",31);assertTrue(samples>=3&&samples<=31); var source=new SQLiteDataSource();source.setUrl("jdbc:sqlite:"+Files.createTempDirectory("teaql-generated-bench-").resolve("school.sqlite")); - var sql=new JdbcSqlExecutor(source); + var sql=new SwitchingSql(source); var ids=new IdSpaceIdGenerator(new GeneratedSchoolLoadStateTest.DatabaseBridge(sql));ids.ensureIdSpaceTable(); var service=new SqliteDataServiceExecutor("sqlite",sql,source); var context=new Context(TeaQLRuntime.builder().metadata(new SimpleEntityMetaFactory()) @@ -149,18 +172,36 @@ static void validate(SmartList rows,String scenario,int count) { row.updateDynamicField("note",id==1?"benchmark note":null);row.auditAs("Seed generated benchmark extension").save(context); } System.out.println("GENERATED_LOG_MODE,"+mode);System.out.println("GENERATED_FIXED_FIELDS,"+School.__TEAQL_FIXED_FIELD_INDEXES.size()); + boolean compare=Boolean.getBoolean("teaql.generatedCompare"); var bean=(com.sun.management.ThreadMXBean)ManagementFactory.getThreadMXBean();assertTrue(bean.isThreadAllocatedMemorySupported());bean.setThreadAllocatedMemoryEnabled(true); for(String scenario:List.of("sparse","wide","dynamic","forward","reverse"))for(int count:new int[]{1,100,10000}) { Supplier> action=()->query(context,scenario,count); - for(int i=0;i<5;i++)DriverQueryBenchmarkTest.consumed=action.get(); + for(int i=0;i<5;i++) { + if(compare){sql.reference=true;DriverQueryBenchmarkTest.consumed=action.get();} + sql.reference=false;DriverQueryBenchmarkTest.consumed=action.get(); + } long[] times=new long[samples],bytes=new long[samples]; + long[] referenceTimes=new long[samples],referenceBytes=new long[samples]; for(int index=0;index> reference=null,result; + if(compare&&index%2==0){sql.reference=true;reference=DriverQueryBenchmarkTest.measure(bean,action);} + sql.reference=false;result=DriverQueryBenchmarkTest.measure(bean,action); + if(compare&&index%2!=0){sql.reference=true;reference=DriverQueryBenchmarkTest.measure(bean,action);} + sql.reference=false; + validate(result.value(),scenario,count); + if(reference!=null) { + validate(reference.value(),scenario,count);referenceTimes[index]=reference.nanos();referenceBytes[index]=reference.bytes(); + System.out.printf("REFERENCE_SAMPLE,%s,%d,%d,%d,%d%n",scenario,count,index,referenceTimes[index],referenceBytes[index]); + } times[index]=result.nanos();bytes[index]=result.bytes(); System.out.printf("GENERATED_SAMPLE,%s,%d,%d,%d,%d%n",scenario,count,index,times[index],bytes[index]); } long total=Arrays.stream(times).sum();Arrays.sort(times);Arrays.sort(bytes); System.out.printf(Locale.ROOT,"GENERATED_SUMMARY,%s,%d,%d,%d,%d,%.2f,%d%n",scenario,count,samples,times[samples/2],times[(int)Math.ceil(samples*.95)-1],samples*1e9/total,bytes[samples/2]); + if(compare) { + long referenceTotal=Arrays.stream(referenceTimes).sum();Arrays.sort(referenceTimes);Arrays.sort(referenceBytes); + System.out.printf(Locale.ROOT,"REFERENCE_SUMMARY,%s,%d,%d,%d,%d,%.2f,%d%n",scenario,count,samples,referenceTimes[samples/2],referenceTimes[(int)Math.ceil(samples*.95)-1],samples*1e9/referenceTotal,referenceBytes[samples/2]); + } } System.out.println("PASS generated Java wide dynamic forward reverse Q/E sharing and private divergence benchmark"); } diff --git a/teaql-provider-jdbc/src/main/java/io/teaql/provider/jdbc/JdbcColumnRow.java b/teaql-provider-jdbc/src/main/java/io/teaql/provider/jdbc/JdbcColumnRow.java index 8017107a..f4e17d50 100644 --- a/teaql-provider-jdbc/src/main/java/io/teaql/provider/jdbc/JdbcColumnRow.java +++ b/teaql-provider-jdbc/src/main/java/io/teaql/provider/jdbc/JdbcColumnRow.java @@ -17,7 +17,7 @@ static final class Layout { } } private final Layout layout; - private final Object[] values; + private Object[] values; private Map expanded; private int modifications; JdbcColumnRow(Layout layout,Object[] ownedValues) { @@ -35,6 +35,9 @@ private Map expand() { if(expanded==null) { expanded=new HashMap<>(); for(String key:layout.keys)expanded.put(key,getOriginal(key)); + // After geometry changes the private map owns the surviving + // payloads. Do not retain removed/replaced values through the array. + values=null; } return expanded; } @@ -54,6 +57,7 @@ private Map expand() { @Override public void clear() { if(isEmpty())return; if(expanded==null)expanded=new HashMap<>();else expanded.clear(); + values=null; modifications++; } @Override public Set> entrySet() { diff --git a/teaql-provider-jdbc/src/test/java/io/teaql/provider/jdbc/JdbcColumnRowTest.java b/teaql-provider-jdbc/src/test/java/io/teaql/provider/jdbc/JdbcColumnRowTest.java index 69b6c37c..b49666f1 100644 --- a/teaql-provider-jdbc/src/test/java/io/teaql/provider/jdbc/JdbcColumnRowTest.java +++ b/teaql-provider-jdbc/src/test/java/io/teaql/provider/jdbc/JdbcColumnRowTest.java @@ -48,4 +48,14 @@ public class JdbcColumnRowTest { assertTrue(row.entrySet().remove(new AbstractMap.SimpleEntry<>("x",3))); row.entrySet().clear();assertTrue(row.isEmpty()); } + @Test public void geometryChangesReleaseTheOldPayloadArray() throws Exception { + var values=JdbcColumnRow.class.getDeclaredField("values");values.setAccessible(true); + for(String operation:List.of("add","remove","clear")) { + var row=new JdbcColumnRow(new JdbcColumnRow.Layout(new String[]{"id","payload"}),new Object[]{1,new byte[4096]}); + assertNotNull(values.get(row)); + switch(operation){case "add"->row.put("extra",2);case "remove"->row.remove("payload");case "clear"->row.clear();} + assertNull(values.get(row)); + if(!operation.equals("clear"))assertEquals(1,row.get("id")); + } + } } From 88426fb7a88b571e9ad9b9ef098e7f9a7b1a3a2e Mon Sep 17 00:00:00 2001 From: Philip Z Date: Thu, 8 Oct 2026 06:45:26 +0800 Subject: [PATCH 46/51] test: preserve connection initialization in matched row-map reference --- examples/shared-load-state/benchmark-generated.sh | 6 ++++-- .../GeneratedQueryBenchmarkTest.java | 11 ++++++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/examples/shared-load-state/benchmark-generated.sh b/examples/shared-load-state/benchmark-generated.sh index 9cdba2d2..7418648e 100644 --- a/examples/shared-load-state/benchmark-generated.sh +++ b/examples/shared-load-state/benchmark-generated.sh @@ -2,11 +2,13 @@ set -euo pipefail mode=off samples=31 +compare=false for option in "$@"; do case "$option" in --default-log) mode=default ;; --smoke) samples=3 ;; - *) printf 'Usage: bash benchmark-generated.sh [--default-log] [--smoke]\n' >&2; exit 2 ;; + --compare) compare=true ;; + *) printf 'Usage: bash benchmark-generated.sh [--default-log] [--smoke] [--compare]\n' >&2; exit 2 ;; esac done runtime_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" @@ -22,7 +24,7 @@ rg --files --hidden --no-ignore examples/shared-load-state/target/generated/lib/ TEAQL_GENERATED_BENCHMARK_LOG_MODE="$mode" /usr/bin/time -v -o "$run_dir/cpu.txt" \ mvn -q -Pshared-load-state-example -pl examples/shared-load-state -am \ -Dtest=GeneratedQueryBenchmarkTest -Dsurefire.failIfNoSpecifiedTests=false \ - -Dteaql.generatedBenchmark=true "-Dteaql.generatedSamples=$samples" test >"$run_dir/results.log" 2>&1 + -Dteaql.generatedBenchmark=true "-Dteaql.generatedSamples=$samples" "-Dteaql.generatedCompare=$compare" test >"$run_dir/results.log" 2>&1 rg -F 'PASS generated Java wide dynamic forward reverse Q/E sharing and private divergence benchmark' "$run_dir/results.log" cp examples/shared-load-state/target/surefire-reports/TEST-com.example.schoolmanagementservice.GeneratedQueryBenchmarkTest.xml "$run_dir/results.xml" sha256sum --check --quiet "$run_dir/source.sha256" diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedQueryBenchmarkTest.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedQueryBenchmarkTest.java index 5bb0af94..035bd730 100644 --- a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedQueryBenchmarkTest.java +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedQueryBenchmarkTest.java @@ -31,11 +31,20 @@ static final class Context extends DefaultUserContext implements SchoolManagemen /** Controlled reconstruction of the previous generic JDBC query path. */ static final class SwitchingSql extends JdbcSqlExecutor { private final SQLiteDataSource source; + private final List> initializers=new ArrayList<>(); boolean reference; SwitchingSql(SQLiteDataSource source){super(source);this.source=source;} + @Override public SwitchingSql addConnectionInitializer(java.util.function.Consumer initializer) { + initializers.add(initializer);super.addConnectionInitializer(initializer);return this; + } + private Connection openReferenceConnection() throws SQLException { + Connection connection=source.getConnection(); + try {for(var initializer:initializers)initializer.accept(connection);return connection;} + catch(RuntimeException failure){try{connection.close();}catch(SQLException suppressed){failure.addSuppressed(suppressed);}throw failure;} + } @Override public List> queryForList(String sql,Object[] params) { if(!reference)return super.queryForList(sql,params); - try(Connection connection=source.getConnection();PreparedStatement statement=connection.prepareStatement(sql)) { + try(Connection connection=openReferenceConnection();PreparedStatement statement=connection.prepareStatement(sql)) { if(params!=null)for(int i=0;i Date: Thu, 8 Oct 2026 07:03:31 +0800 Subject: [PATCH 47/51] test: separate nested Checker identity isolation from unchanged object status --- .../java/io/teaql/sqlite/GraphTraceSqliteTest.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java b/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java index 74bdefcd..c0159efb 100644 --- a/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java +++ b/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java @@ -277,7 +277,15 @@ private static CheckedSave checkedSave(Fixture fixture, GraphEntity entity, Stri inner.auditAs("save independent nested graph").save(caller); assertEquals("nested execution must restore the outer captured clock", now, caller.evaluate("now")); assertFalse("outer entity must remain checked", needCheck(caller, outer)); - assertTrue("inner visited identities must not leak into the outer graph", needCheck(caller, inner)); + assertEquals(EntityStatus.PERSISTED, inner.get$status()); + assertFalse("an unchanged object saved by the nested invocation must not be checked again", + needCheck(caller, inner)); + // needCheck also considers mutation status. Test the visited + // identities directly so the isolation assertion does not + // mistake the now-persisted inner graph for a dirty object. + var visited = (List) caller.getAttribute(Checker.TEAQL_DATA_CHECKED_ITEMS); + assertEquals("inner visited identities must not leak into the outer graph", 1, visited.size()); + assertSame(outer, visited.get(0)); } entity.updateProperty("memo", now.toString()); } From efcddc7d714cd9a595972afa262c8b1e4c49d16b Mon Sep 17 00:00:00 2001 From: Philip Z Date: Thu, 8 Oct 2026 07:54:18 +0800 Subject: [PATCH 48/51] test: measure fresh JVM Context initialization and first generated query --- examples/shared-load-state/benchmark-cold.sh | 37 +++++++++ .../GeneratedColdQueryProbe.java | 78 +++++++++++++++++++ 2 files changed, 115 insertions(+) create mode 100644 examples/shared-load-state/benchmark-cold.sh create mode 100644 examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedColdQueryProbe.java diff --git a/examples/shared-load-state/benchmark-cold.sh b/examples/shared-load-state/benchmark-cold.sh new file mode 100644 index 00000000..b89de4f8 --- /dev/null +++ b/examples/shared-load-state/benchmark-cold.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +set -euo pipefail +mode=off +case "${1:-}" in + '') ;; + --default-log) mode=default ;; + *) printf 'Usage: bash benchmark-cold.sh [--default-log]\n' >&2; exit 2 ;; +esac +(( $# <= 1 )) || exit 2 +runtime_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +example_dir="$runtime_dir/examples/shared-load-state" +run_dir="$(mktemp -d -t teaql-java-cold-query.XXXXXXXX)" +printf 'Cold query evidence: %s\n' "$run_dir" +cd "$runtime_dir" +git diff HEAD --exit-code -- examples/shared-load-state teaql-core teaql-runtime teaql-provider-jdbc +git rev-parse HEAD >"$run_dir/base-commit.txt" +java -version >"$run_dir/toolchain.txt" 2>&1 +sha256sum examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedColdQueryProbe.java >"$run_dir/source.sha256" +rg --files --hidden --no-ignore "$example_dir/target/generated/lib/src" -0 | + sort -z | xargs -0 sha256sum >"$run_dir/generated-source.sha256" +# Run the local verifier first to provide a reactor-resolved test classpath. +report="$example_dir/target/surefire-reports/TEST-com.example.schoolmanagementservice.GeneratedSchoolLoadStateTest.xml" +[[ -f "$report" ]] || { printf 'Run examples/shared-load-state/verify.sh first\n' >&2; exit 1; } +mvn -q -o -Pshared-load-state-example -pl examples/shared-load-state -am -DskipTests test-compile >"$run_dir/build.log" 2>&1 +classpath="$(xmllint --xpath 'string(/testsuite/properties/property[@name="java.class.path"]/@value)' "$report")" +[[ -n "$classpath" ]] +export TEAQL_LOCAL_RUNTIME_ROOT="$runtime_dir" +TEAQL_COLD_LOG_MODE=off java -cp "$classpath" com.example.schoolmanagementservice.GeneratedColdQueryProbe prepare "$run_dir/school.sqlite" >"$run_dir/prepare.log" 2>&1 +sha256sum "$run_dir/school.sqlite" >"$run_dir/database.sha256" +for round in 1 2 3 4 5 6 7; do + TEAQL_COLD_LOG_MODE="$mode" /usr/bin/time -v -o "$run_dir/cpu-$round.txt" \ + java -cp "$classpath" com.example.schoolmanagementservice.GeneratedColdQueryProbe query "$run_dir/school.sqlite" >"$run_dir/round-$round.log" 2>&1 + rg -F 'PASS generated Java process-cold Q/E and shared snapshot' "$run_dir/round-$round.log" +done +sha256sum --check --quiet "$run_dir/database.sha256" +sha256sum --check --quiet "$run_dir/source.sha256" +sha256sum --check --quiet "$run_dir/generated-source.sha256" diff --git a/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedColdQueryProbe.java b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedColdQueryProbe.java new file mode 100644 index 00000000..d273b02e --- /dev/null +++ b/examples/shared-load-state/src/test/java/com/example/schoolmanagementservice/GeneratedColdQueryProbe.java @@ -0,0 +1,78 @@ +package com.example.schoolmanagementservice; + +import com.example.schoolmanagementservice.school.School; +import io.teaql.core.*; +import io.teaql.core.meta.SimpleEntityMetaFactory; +import io.teaql.core.sql.portable.IdSpaceIdGenerator; +import io.teaql.core.sqlite.SqliteDataServiceExecutor; +import io.teaql.provider.jdbc.JdbcSqlExecutor; +import io.teaql.runtime.TeaQLRuntime; +import java.lang.management.ManagementFactory; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.Set; +import org.sqlite.SQLiteDataSource; +import static org.junit.jupiter.api.Assertions.*; + +/** Independent JVM, pre-created fixture, first generated Q only; no warmup. */ +public final class GeneratedColdQueryProbe { + public static void main(String[] arguments) throws Exception { + if (arguments.length != 2 || !Set.of("prepare", "query").contains(arguments[0])) + throw new IllegalArgumentException("Usage: GeneratedColdQueryProbe prepare|query "); + boolean prepare=arguments[0].equals("prepare"); + Path database=Path.of(arguments[1]); + assertEquals(!prepare,Files.exists(database)); + String mode=System.getenv().getOrDefault("TEAQL_COLD_LOG_MODE","off"); + assertTrue(Set.of("off","default").contains(mode)); + var bean=(com.sun.management.ThreadMXBean)ManagementFactory.getThreadMXBean(); + assertTrue(bean.isThreadAllocatedMemorySupported());bean.setThreadAllocatedMemoryEnabled(true); + long thread=Thread.currentThread().threadId(),before=bean.getThreadAllocatedBytes(thread),start=System.nanoTime(); + var source=new SQLiteDataSource();source.setUrl("jdbc:sqlite:"+database); + var sql=new JdbcSqlExecutor(source); + var ids=new IdSpaceIdGenerator(new GeneratedSchoolLoadStateTest.DatabaseBridge(sql)); + var service=new SqliteDataServiceExecutor("sqlite",sql,source); + var context=new GeneratedQueryBenchmarkTest.Context(TeaQLRuntime.builder() + .metadata(new SimpleEntityMetaFactory()).dataService("default",service).dataService("sqlite",service) + .idGenerationService(ids).queryExecutionLogging(mode.equals("default")).build() + .install(GeneratedRuntimeModule.module())); + long initNs=System.nanoTime()-start,initBytes=bean.getThreadAllocatedBytes(thread)-before; + if (prepare) { + ids.ensureIdSpaceTable();context.ensureSchema(); + var platform=Q.platforms().withIdIs(1L).limit(1) + .comment("what: reuse cold fixture root").purpose("why: seed through generated audited mutation").executeForOne(context); + for(String name:new String[]{"cold-school-1","cold-school-2"}) { + var school=Q.schools().comment("what: allocate a cold fixture School") + .purpose("why: prepare outside the cold measurement process").newEntity(context); + school.updatePlatform(platform);school.updateSchoolTypeToPrimary(); + school.updateName(name);school.updateAddress("Cold fixture address"); + school.updateEstablishedDate(LocalDate.of(1995,9,1));school.updateStudentCapacity(17);school.updateActive(false); + school.updateCreateTime(LocalDateTime.of(2023,11,14,22,13));school.updateUpdateTime(LocalDateTime.of(2023,11,14,22,13)); + school.auditAs("prepare the generated cold-query fixture").save(context); + } + System.out.println("PASS prepared generated Java cold-query fixture");return; + } + before=bean.getThreadAllocatedBytes(thread);start=System.nanoTime(); + var rows=Q.schoolsWithMinimalFields().selectName().orderByIdAscending().limit(2) + .comment("what: load the first bounded generated cold-query result") + .purpose("why: separate process-cold startup from warmed query costs").executeForList(context); + long queryNs=System.nanoTime()-start,queryBytes=bean.getThreadAllocatedBytes(thread)-before; + assertEquals(2,rows.size());var state=rows.get(0).__internalLoadState(); + assertSame(state,rows.get(1).__internalLoadState()); + for(int index=0;index type:new Class[]{LoadState.class,JdbcSqlExecutor.class,TeaQLRuntime.class,School.class}) { + Path location=Path.of(type.getProtectionDomain().getCodeSource().getLocation().toURI()).toRealPath(); + assertTrue(location.startsWith(root),"not local runtime/generated code: "+location); + System.out.println("COLD_CLASS "+type.getName()+" "+location); + } + System.out.printf("JAVA_COLD_QUERY,%s,%d,%d,%d,%d,%d%n",mode,ProcessHandle.current().pid(),initNs,queryNs,initBytes,queryBytes); + System.out.println("PASS generated Java process-cold Q/E and shared snapshot"); + } +} From 2e317370c3ed1610c7c33ae182a824eb7e35bddb Mon Sep 17 00:00:00 2001 From: Philip Z Date: Thu, 8 Oct 2026 08:03:47 +0800 Subject: [PATCH 49/51] docs: document cold JVM measurements and modified-only Checker contract --- examples/shared-load-state/README.md | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/examples/shared-load-state/README.md b/examples/shared-load-state/README.md index 2d280da2..65984b94 100644 --- a/examples/shared-load-state/README.md +++ b/examples/shared-load-state/README.md @@ -86,9 +86,11 @@ not a substitute for loading authoritative state before mutation. - Readonly `_name` coexists with native `name`; a missing dynamic property reads null, and ordinary save does not persist derived properties. -Partial child projections are valid for E reads, but the whole-graph Checker -does not accept them as complete writable objects. The example reloads the full -School before deletion rather than disabling that validation. +Partial child projections are valid for E reads and do not block saving a +complete parent when those children are untouched. Every actually modified +object must itself be fully loaded; modified descendants cannot hide behind an +untouched parent. The example reloads the full School before deletion rather +than disabling that validation. The persistent ID allocator uses its framework database bridge. Application business data is created through generated mutation APIs, not handwritten SQL. @@ -139,6 +141,24 @@ and requires three consecutive zero-allocation samples within twenty attempts. An allocating positive control must remain visible. This is a warmed-window claim, not a guarantee of zero allocation for every JVM observation. +## Process cold startup probe + +```bash +bash examples/shared-load-state/benchmark-cold.sh +bash examples/shared-load-state/benchmark-cold.sh --default-log +``` + +Run the local verifier once before this probe to provide a reactor-resolved +test classpath. The probe requires `xmllint`. It prepares two Schools using +generated audited mutations in a separate JVM, then launches seven fresh JVMs. +Each records Context initialization and its first sparse two-row Q query +separately, with E/name/version, shared snapshot and local class-location checks. +No schema writes or query warmup occur in measured JVMs. Database and generated +source hashes must stay unchanged. The external CPU receipt includes JVM startup, +class loading, validation and output, not Maven compilation or fixture setup. +OS page caches are not cleared. Byte counters report gross requested thread +allocations, not retained heap or allocation-call counts. + ## Matched driver logging modes ```bash From da105309b3907505161ece121c482db8f436b95d Mon Sep 17 00:00:00 2001 From: Philip Z Date: Thu, 8 Oct 2026 17:27:43 +0800 Subject: [PATCH 50/51] release: prepare Java 1.554 shared loaded-state runtime --- examples/business-id-runtime/pom.xml | 2 +- examples/conformance/lib/pom.xml | 2 +- examples/conformance/pom.xml | 2 +- examples/order-management/java-app-console/pom.xml | 2 +- examples/order-management/java-lib-core/lib/pom.xml | 2 +- examples/round-trip-reference-runtime/pom.xml | 2 +- examples/school-management/lib/pom.xml | 2 +- examples/school-management/pom.xml | 2 +- examples/security-foundations/pom.xml | 2 +- examples/shared-load-state/pom.xml | 2 +- examples/trace-chain/lib/pom.xml | 6 +++--- examples/trace-chain/pom.xml | 2 +- pom.xml | 2 +- scripts/verify-examples.sh | 4 ++-- teaql-android/pom.xml | 2 +- teaql-business-id-jdbc/pom.xml | 2 +- teaql-cloud/pom.xml | 2 +- teaql-context-runtime-tools/pom.xml | 2 +- teaql-core/pom.xml | 2 +- teaql-data-service-sql/pom.xml | 2 +- teaql-db2/pom.xml | 2 +- teaql-dm8/pom.xml | 2 +- teaql-duckdb/pom.xml | 2 +- teaql-dynamic-fields-api/pom.xml | 2 +- teaql-dynamic-fields-jdbc/pom.xml | 2 +- teaql-hana/pom.xml | 2 +- teaql-jackson/pom.xml | 2 +- teaql-mssql/pom.xml | 2 +- teaql-mysql/pom.xml | 2 +- teaql-opentelemetry/pom.xml | 2 +- teaql-oracle/pom.xml | 2 +- teaql-postgres/pom.xml | 2 +- teaql-provider-jdbc/pom.xml | 2 +- teaql-provider-spring-jdbc/pom.xml | 2 +- teaql-query-json/pom.xml | 2 +- teaql-redis/pom.xml | 2 +- teaql-runtime-log/pom.xml | 2 +- teaql-runtime/pom.xml | 2 +- teaql-snowflake/pom.xml | 2 +- teaql-sql-portable/pom.xml | 2 +- teaql-sqlite/pom.xml | 2 +- teaql-tfp-endpoint/pom.xml | 2 +- teaql-tool-http/pom.xml | 2 +- teaql-utils-json/pom.xml | 2 +- teaql-utils-reflection/pom.xml | 2 +- teaql-utils-spring/pom.xml | 2 +- teaql-utils/pom.xml | 2 +- 47 files changed, 50 insertions(+), 50 deletions(-) diff --git a/examples/business-id-runtime/pom.xml b/examples/business-id-runtime/pom.xml index d3cd21b6..59b8df27 100644 --- a/examples/business-id-runtime/pom.xml +++ b/examples/business-id-runtime/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.553-RELEASE + 1.554-RELEASE ../../pom.xml teaql-business-id-example diff --git a/examples/conformance/lib/pom.xml b/examples/conformance/lib/pom.xml index 8c70f07e..134ccf0b 100644 --- a/examples/conformance/lib/pom.xml +++ b/examples/conformance/lib/pom.xml @@ -12,7 +12,7 @@ 21 4.1.1 - 1.553-RELEASE + 1.554-RELEASE UTF-8 true diff --git a/examples/conformance/pom.xml b/examples/conformance/pom.xml index 8b2fe544..71ab3830 100644 --- a/examples/conformance/pom.xml +++ b/examples/conformance/pom.xml @@ -16,7 +16,7 @@ 1.0.0 UTF-8 true - 1.553-RELEASE + 1.554-RELEASE diff --git a/examples/order-management/java-app-console/pom.xml b/examples/order-management/java-app-console/pom.xml index a3f75918..d78ee861 100644 --- a/examples/order-management/java-app-console/pom.xml +++ b/examples/order-management/java-app-console/pom.xml @@ -11,7 +11,7 @@ 21 UTF-8 - 1.553-RELEASE + 1.554-RELEASE diff --git a/examples/order-management/java-lib-core/lib/pom.xml b/examples/order-management/java-lib-core/lib/pom.xml index 45054540..7d83f2d3 100644 --- a/examples/order-management/java-lib-core/lib/pom.xml +++ b/examples/order-management/java-lib-core/lib/pom.xml @@ -12,7 +12,7 @@ 21 4.1.1 - 1.553-RELEASE + 1.554-RELEASE UTF-8 true diff --git a/examples/round-trip-reference-runtime/pom.xml b/examples/round-trip-reference-runtime/pom.xml index 6a1cb2a9..f8746316 100644 --- a/examples/round-trip-reference-runtime/pom.xml +++ b/examples/round-trip-reference-runtime/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.553-RELEASE + 1.554-RELEASE ../../pom.xml teaql-round-trip-reference-example diff --git a/examples/school-management/lib/pom.xml b/examples/school-management/lib/pom.xml index 84eb36d9..721bdeda 100644 --- a/examples/school-management/lib/pom.xml +++ b/examples/school-management/lib/pom.xml @@ -12,7 +12,7 @@ 21 4.1.1 - 1.553-RELEASE + 1.554-RELEASE UTF-8 true diff --git a/examples/school-management/pom.xml b/examples/school-management/pom.xml index 4e89b07e..4af77b4f 100644 --- a/examples/school-management/pom.xml +++ b/examples/school-management/pom.xml @@ -16,7 +16,7 @@ 0.0.1-SNAPSHOT UTF-8 true - 1.553-RELEASE + 1.554-RELEASE diff --git a/examples/security-foundations/pom.xml b/examples/security-foundations/pom.xml index ba9149e4..d9c1d45d 100644 --- a/examples/security-foundations/pom.xml +++ b/examples/security-foundations/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.553-RELEASE + 1.554-RELEASE ../../pom.xml teaql-security-foundations-example diff --git a/examples/shared-load-state/pom.xml b/examples/shared-load-state/pom.xml index d2ceb383..dfafaa26 100644 --- a/examples/shared-load-state/pom.xml +++ b/examples/shared-load-state/pom.xml @@ -3,7 +3,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - io.teaqlteaql-java-parent1.553-RELEASE + io.teaqlteaql-java-parent1.554-RELEASE ../../pom.xml shared-load-state-example diff --git a/examples/trace-chain/lib/pom.xml b/examples/trace-chain/lib/pom.xml index 35c0a42f..9ef081b1 100644 --- a/examples/trace-chain/lib/pom.xml +++ b/examples/trace-chain/lib/pom.xml @@ -12,7 +12,7 @@ 21 4.1.1 - 1.553-RELEASE + 1.554-RELEASE UTF-8 true @@ -67,7 +67,7 @@ io.teaql teaql-sqlite - 1.553-RELEASE + 1.554-RELEASE io.teaql @@ -139,4 +139,4 @@ https://maven.teaql.io/repository/maven-releases/ - \ No newline at end of file + diff --git a/examples/trace-chain/pom.xml b/examples/trace-chain/pom.xml index c2134168..b7958ec9 100644 --- a/examples/trace-chain/pom.xml +++ b/examples/trace-chain/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.553-RELEASE + 1.554-RELEASE ../../pom.xml teaql-trace-chain-example diff --git a/pom.xml b/pom.xml index f9bca17e..940e8791 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.553-RELEASE + 1.554-RELEASE pom teaql-java-parent diff --git a/scripts/verify-examples.sh b/scripts/verify-examples.sh index 7922135a..11b1d852 100755 --- a/scripts/verify-examples.sh +++ b/scripts/verify-examples.sh @@ -4,7 +4,7 @@ set -euo pipefail repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" verification_dir="$(mktemp -d)" trap 'rm -rf -- "$verification_dir"' EXIT -expected=(business-id-runtime conformance order-management round-trip-reference-runtime school-management security-foundations trace-chain) +expected=(business-id-runtime conformance order-management round-trip-reference-runtime school-management security-foundations shared-load-state trace-chain) mapfile -t actual < <(find "$repo/examples" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' | sort) if [[ "${actual[*]}" != "${expected[*]}" ]]; then echo "example inventory changed; update scripts/verify-examples.sh: ${actual[*]}" >&2 @@ -32,4 +32,4 @@ mvn -q -f examples/order-management/pom.xml install -DskipTests # the developer's retained database while verifying the runtime checkout. (cd "$verification_dir" && mvn -q -f "$repo/examples/order-management/pom.xml" exec:java -pl java-app-console) bash examples/trace-chain/verify.sh -echo "PASS: all Java examples" +echo "PASS: standard Java examples; shared-load-state has its own generator-backed verify.sh gate" diff --git a/teaql-android/pom.xml b/teaql-android/pom.xml index 72b68ea6..2f4e456f 100644 --- a/teaql-android/pom.xml +++ b/teaql-android/pom.xml @@ -7,7 +7,7 @@ io.teaql teaql-java-parent - 1.553-RELEASE + 1.554-RELEASE teaql-android diff --git a/teaql-business-id-jdbc/pom.xml b/teaql-business-id-jdbc/pom.xml index 844a268a..134e21fd 100644 --- a/teaql-business-id-jdbc/pom.xml +++ b/teaql-business-id-jdbc/pom.xml @@ -7,7 +7,7 @@ io.teaql teaql-java-parent - 1.553-RELEASE + 1.554-RELEASE ../pom.xml diff --git a/teaql-cloud/pom.xml b/teaql-cloud/pom.xml index 655f742e..5d4a9f9e 100644 --- a/teaql-cloud/pom.xml +++ b/teaql-cloud/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.553-RELEASE + 1.554-RELEASE teaql-cloud teaql-cloud diff --git a/teaql-context-runtime-tools/pom.xml b/teaql-context-runtime-tools/pom.xml index 3ba21919..fcd36ea7 100644 --- a/teaql-context-runtime-tools/pom.xml +++ b/teaql-context-runtime-tools/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.553-RELEASE + 1.554-RELEASE teaql-context-runtime-tools diff --git a/teaql-core/pom.xml b/teaql-core/pom.xml index 56329d77..5a118b3b 100644 --- a/teaql-core/pom.xml +++ b/teaql-core/pom.xml @@ -7,7 +7,7 @@ io.teaql teaql-java-parent - 1.553-RELEASE + 1.554-RELEASE ../pom.xml diff --git a/teaql-data-service-sql/pom.xml b/teaql-data-service-sql/pom.xml index aaa0fda1..88126ffe 100644 --- a/teaql-data-service-sql/pom.xml +++ b/teaql-data-service-sql/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.553-RELEASE + 1.554-RELEASE ../pom.xml diff --git a/teaql-db2/pom.xml b/teaql-db2/pom.xml index 73d48e0d..b6f95893 100644 --- a/teaql-db2/pom.xml +++ b/teaql-db2/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.553-RELEASE + 1.554-RELEASE teaql-db2 teaql-db2 diff --git a/teaql-dm8/pom.xml b/teaql-dm8/pom.xml index cc320c98..db7d1c65 100644 --- a/teaql-dm8/pom.xml +++ b/teaql-dm8/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.553-RELEASE + 1.554-RELEASE teaql-dm8 teaql-dm8 diff --git a/teaql-duckdb/pom.xml b/teaql-duckdb/pom.xml index 96bef910..ec5de021 100644 --- a/teaql-duckdb/pom.xml +++ b/teaql-duckdb/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.553-RELEASE + 1.554-RELEASE teaql-duckdb teaql-duckdb diff --git a/teaql-dynamic-fields-api/pom.xml b/teaql-dynamic-fields-api/pom.xml index 87511c69..193fd567 100644 --- a/teaql-dynamic-fields-api/pom.xml +++ b/teaql-dynamic-fields-api/pom.xml @@ -7,7 +7,7 @@ io.teaql teaql-java-parent - 1.553-RELEASE + 1.554-RELEASE ../pom.xml diff --git a/teaql-dynamic-fields-jdbc/pom.xml b/teaql-dynamic-fields-jdbc/pom.xml index 63b8a8b7..d648741a 100644 --- a/teaql-dynamic-fields-jdbc/pom.xml +++ b/teaql-dynamic-fields-jdbc/pom.xml @@ -7,7 +7,7 @@ io.teaql teaql-java-parent - 1.553-RELEASE + 1.554-RELEASE ../pom.xml diff --git a/teaql-hana/pom.xml b/teaql-hana/pom.xml index 588c94a4..de24c117 100644 --- a/teaql-hana/pom.xml +++ b/teaql-hana/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.553-RELEASE + 1.554-RELEASE teaql-hana teaql-hana diff --git a/teaql-jackson/pom.xml b/teaql-jackson/pom.xml index 0ab6824d..22f6da68 100644 --- a/teaql-jackson/pom.xml +++ b/teaql-jackson/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.553-RELEASE + 1.554-RELEASE teaql-jackson diff --git a/teaql-mssql/pom.xml b/teaql-mssql/pom.xml index e357326a..6191e367 100644 --- a/teaql-mssql/pom.xml +++ b/teaql-mssql/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.553-RELEASE + 1.554-RELEASE teaql-mssql teaql-mssql diff --git a/teaql-mysql/pom.xml b/teaql-mysql/pom.xml index 193f082c..ba415521 100644 --- a/teaql-mysql/pom.xml +++ b/teaql-mysql/pom.xml @@ -7,7 +7,7 @@ io.teaql teaql-java-parent - 1.553-RELEASE + 1.554-RELEASE teaql-mysql diff --git a/teaql-opentelemetry/pom.xml b/teaql-opentelemetry/pom.xml index d0701152..9ba310db 100644 --- a/teaql-opentelemetry/pom.xml +++ b/teaql-opentelemetry/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.553-RELEASE + 1.554-RELEASE teaql-opentelemetry diff --git a/teaql-oracle/pom.xml b/teaql-oracle/pom.xml index 56b94bd9..bc01a766 100644 --- a/teaql-oracle/pom.xml +++ b/teaql-oracle/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.553-RELEASE + 1.554-RELEASE teaql-oracle teaql-oracle diff --git a/teaql-postgres/pom.xml b/teaql-postgres/pom.xml index d65b9d01..144c3cf8 100644 --- a/teaql-postgres/pom.xml +++ b/teaql-postgres/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.553-RELEASE + 1.554-RELEASE teaql-postgres teaql-postgres diff --git a/teaql-provider-jdbc/pom.xml b/teaql-provider-jdbc/pom.xml index 93a6b274..1de23b21 100644 --- a/teaql-provider-jdbc/pom.xml +++ b/teaql-provider-jdbc/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.553-RELEASE + 1.554-RELEASE ../pom.xml diff --git a/teaql-provider-spring-jdbc/pom.xml b/teaql-provider-spring-jdbc/pom.xml index ce53a687..ff392d65 100644 --- a/teaql-provider-spring-jdbc/pom.xml +++ b/teaql-provider-spring-jdbc/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.553-RELEASE + 1.554-RELEASE ../pom.xml diff --git a/teaql-query-json/pom.xml b/teaql-query-json/pom.xml index 1e21637b..94c61415 100644 --- a/teaql-query-json/pom.xml +++ b/teaql-query-json/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.553-RELEASE + 1.554-RELEASE teaql-query-json diff --git a/teaql-redis/pom.xml b/teaql-redis/pom.xml index 184ce2ba..bf724fe7 100644 --- a/teaql-redis/pom.xml +++ b/teaql-redis/pom.xml @@ -7,7 +7,7 @@ io.teaql teaql-java-parent - 1.553-RELEASE + 1.554-RELEASE ../pom.xml diff --git a/teaql-runtime-log/pom.xml b/teaql-runtime-log/pom.xml index 1c46995b..1cc4b2a6 100644 --- a/teaql-runtime-log/pom.xml +++ b/teaql-runtime-log/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.553-RELEASE + 1.554-RELEASE teaql-runtime-log diff --git a/teaql-runtime/pom.xml b/teaql-runtime/pom.xml index 1dcaf9e4..2d212f7c 100644 --- a/teaql-runtime/pom.xml +++ b/teaql-runtime/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.553-RELEASE + 1.554-RELEASE teaql-runtime diff --git a/teaql-snowflake/pom.xml b/teaql-snowflake/pom.xml index a444bb54..936881d1 100644 --- a/teaql-snowflake/pom.xml +++ b/teaql-snowflake/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.553-RELEASE + 1.554-RELEASE teaql-snowflake teaql-snowflake diff --git a/teaql-sql-portable/pom.xml b/teaql-sql-portable/pom.xml index 7b720235..a05b7f93 100644 --- a/teaql-sql-portable/pom.xml +++ b/teaql-sql-portable/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.553-RELEASE + 1.554-RELEASE teaql-sql-portable diff --git a/teaql-sqlite/pom.xml b/teaql-sqlite/pom.xml index 9ad6749c..3670630b 100644 --- a/teaql-sqlite/pom.xml +++ b/teaql-sqlite/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.553-RELEASE + 1.554-RELEASE teaql-sqlite teaql-sqlite diff --git a/teaql-tfp-endpoint/pom.xml b/teaql-tfp-endpoint/pom.xml index ca9b7ac0..4a5070ac 100644 --- a/teaql-tfp-endpoint/pom.xml +++ b/teaql-tfp-endpoint/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.553-RELEASE + 1.554-RELEASE ../pom.xml diff --git a/teaql-tool-http/pom.xml b/teaql-tool-http/pom.xml index 38401ab4..b4e90309 100644 --- a/teaql-tool-http/pom.xml +++ b/teaql-tool-http/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.553-RELEASE + 1.554-RELEASE teaql-tool-http diff --git a/teaql-utils-json/pom.xml b/teaql-utils-json/pom.xml index dac3abb2..69cbc5cf 100644 --- a/teaql-utils-json/pom.xml +++ b/teaql-utils-json/pom.xml @@ -6,7 +6,7 @@ io.teaql teaql-java-parent - 1.553-RELEASE + 1.554-RELEASE teaql-utils-json diff --git a/teaql-utils-reflection/pom.xml b/teaql-utils-reflection/pom.xml index fade8a8c..359bebbb 100644 --- a/teaql-utils-reflection/pom.xml +++ b/teaql-utils-reflection/pom.xml @@ -7,7 +7,7 @@ io.teaql teaql-java-parent - 1.553-RELEASE + 1.554-RELEASE ../pom.xml diff --git a/teaql-utils-spring/pom.xml b/teaql-utils-spring/pom.xml index e2b412bf..d455784a 100644 --- a/teaql-utils-spring/pom.xml +++ b/teaql-utils-spring/pom.xml @@ -7,7 +7,7 @@ io.teaql teaql-java-parent - 1.553-RELEASE + 1.554-RELEASE ../pom.xml diff --git a/teaql-utils/pom.xml b/teaql-utils/pom.xml index 2a8fe617..146e88b2 100644 --- a/teaql-utils/pom.xml +++ b/teaql-utils/pom.xml @@ -7,7 +7,7 @@ io.teaql teaql-java-parent - 1.553-RELEASE + 1.554-RELEASE ../pom.xml From b0760afb9f009b3d7ae8cb75d7fd714f861e09cb Mon Sep 17 00:00:00 2001 From: Philip Z Date: Thu, 8 Oct 2026 17:32:41 +0800 Subject: [PATCH 51/51] test: read relation aggregate through dynamic property namespace --- .../io/teaql/examples/security/SqlRelationMaskingTest.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/examples/security-foundations/src/test/java/io/teaql/examples/security/SqlRelationMaskingTest.java b/examples/security-foundations/src/test/java/io/teaql/examples/security/SqlRelationMaskingTest.java index d240fdde..1fd0bac5 100644 --- a/examples/security-foundations/src/test/java/io/teaql/examples/security/SqlRelationMaskingTest.java +++ b/examples/security-foundations/src/test/java/io/teaql/examples/security/SqlRelationMaskingTest.java @@ -131,7 +131,11 @@ private static SQLEntityDescriptor descriptor(String type, Class .purpose("inspect Riverside RELATION-PASSWORD-CANARY relations").executeForList(context); assertEquals(1, rows.size()); assertEquals("Riverside", rows.get(0).getProperty("name")); - if (shape.equals("aggregate")) assertEquals(1, ((Number) rows.get(0).getProperty("childCount")).intValue()); + if (shape.equals("aggregate")) { + assertEquals(1, ((Number) rows.get(0).getDynamicProperty("childCount")).intValue()); + assertEquals(1, ((Number) rows.get(0).getProperty("_childCount")).intValue()); + assertNull("Aggregate results must not overwrite the native namespace", rows.get(0).getProperty("childCount")); + } else if (shape.equals("facet")) { SmartList parents = rows.getFacet("parentFacet"); assertEquals(1, parents.size());