diff --git a/.github/workflows/example-gate.yml b/.github/workflows/example-gate.yml index b501732..998a831 100644 --- a/.github/workflows/example-gate.yml +++ b/.github/workflows/example-gate.yml @@ -15,5 +15,5 @@ jobs: with: python-version: '3.10' cache: pip - - run: pip install -e . aiosqlite pytest pytest-asyncio + - run: pip install -e ".[test,telemetry]" - run: ./scripts/verify-examples.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4a16c3b..d90b552 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,7 +14,7 @@ jobs: with: python-version: "3.12" - run: python -m pip install --upgrade build - - run: python -m pip install --editable ".[test]" + - run: python -m pip install --editable ".[test,telemetry]" - name: Verify tag matches package version run: test "${GITHUB_REF_NAME#python-v}" = "$(python -c 'import importlib.metadata; print(importlib.metadata.version("teaql"))')" - run: python -m pytest diff --git a/README.md b/README.md index 3997cc2..d148560 100644 --- a/README.md +++ b/README.md @@ -265,6 +265,86 @@ repository's local runtime. No external database server or generator is needed to run the retained generated example. Use only the example's test database; reset/repeated-run behavior is documented in its README. +## Optional standard OpenTelemetry module + +The local standard-module changes are not yet released. Install the optional +SDK dependencies from this checkout when testing telemetry: + +```bash +python -m pip install -e '.[telemetry]' +export TEAQL_TELEMETRY_ENABLED=true +export TEAQL_TELEMETRY_SERVICE_NAME=my-application +export TEAQL_TELEMETRY_OTLP_ENDPOINT=http://localhost:4318 +``` + +`UserContext` initializes the standard module automatically. Without enablement, +startup prints support and the three variable names without importing the SDK. +Enabled telemetry records supported operations without percentage sampling, +using private providers that do not replace customer globals. Expanded SQL uses +the existing masking rules. Disabling ordinary SQL logging does not disable +telemetry or committed audit. Queues and export lifetimes are bounded; failed +export does not fail business operations. + +The [standard Order example](examples/trace-chain/standard-order/main.py) +uses generated Q and E to load a bounded list, forward detail, reverse previews, +full child counts and status facets. Explicit count is optional; both variants +use `execute_for_list(context)` and one root through entity construction. +Its gate runs twice on one SQLite file and verifies audited, idempotent bootstrap: + +```bash +PYTHONDONTWRITEBYTECODE=1 bash scripts/verify-examples.sh +``` + +The [standard graph example](examples/trace-chain/standard_graph.py) saves six +related entities, performs five updates and one deletion, then verifies rollback +and retry. One automatic graph root covers the transaction, committed audit and +cleanup; each SQL completion carries its actual execution span. Child comments +remain local or inherit the root. No application SDK setup or manual spans are +needed. The example keeps ordinary SQL logging off and preserves the original +business error without exporting its message. + +```bash +bash examples/trace-chain/verify-standard-graph.sh +``` + +Current local qualification passes 844 tests with eight configured skips, all +eleven example groups and fresh-generation checks. The migrated TraceChain and +Order libraries use ordinary list execution, isolated row ledgers and reached-only +graph adoption. Runtime-owned original scalar snapshots protect log/audit intent +after changes or deletion. + +Ten actual OTLP response scenarios qualify failed export, partial acceptance, +warnings and invalid acknowledgements across traces, logs and metrics. Official +protobuf encoding/decoding uses private HTTP connections, with no change to +customer log settings. Requests have a three-second attempt budget, no retry or +redirect, and four-MiB response limits. Remote status/header/body text does not +leak even under DEBUG. Failed MetricData objects and receiver-rejected metric +points are counted separately. Cumulative health metrics follow record settlement +and include a final shutdown snapshot; export failure preserves business and audit. +Default span/log admission counts queued and exporting records in one 2,048-record +budget, with visible overflow. Successful handoffs define the flush prefix; +later completions cannot replace older accepted work. Real generated Q/E pressure +qualifies recovery and backlog shutdown in about six seconds, separating failed +attempts from queued retirement. Post-shutdown reads stay functional and late +completions are counted without fake SQL correlation. Generated streams explicitly +close their owned cursor, rather than relying on garbage collection. +The [standard scalar-stream example](examples/trace-chain/standard_stream.py) +verifies complete delivery of three rows, early close after one row with chunk +size two, an independent query between suspended cursors and an unpolled stream. +Runtime owns the logical root, actual SQL correlation and delivered cardinality; +activation exists only during poll/close. Neither idle nor unpolled streams +capture unrelated work. Generated requests retain creation-time query semantics. + +```bash +bash examples/trace-chain/verify-standard-stream.sh +``` + +Actual failure/cancellation backend variants, advanced concurrency/lifecycle and +internal-artifact qualification remain separate gates. Relation/facet-enhanced +streaming is explicitly unsupported. + +Current qualification uses local source, not an installed public artifact. + ## Use the published runtime Current documented runtime: **0.3.0**. Keep domain-library and runtime diff --git a/docs/releases/0.3.1.md b/docs/releases/0.3.1.md new file mode 100644 index 0000000..ec42065 --- /dev/null +++ b/docs/releases/0.3.1.md @@ -0,0 +1,18 @@ +# Python 0.3.1 + +Tracking: [Python #50](https://github.com/teaql/teaql-python/issues/50) and +[conformance #109](https://github.com/teaql/teaql-conformance/issues/109). + +The optional `teaql[telemetry]` extra supplies the standard SDK module. Context +startup reports support and the three activation variables; enabling them records +full safe traces, metrics and logs. Disabled startup does not load SDK resources. + +Expanded masked SQL carries the original physical identity. Owned pipelines keep +customer globals unchanged and bound admission, flush and shutdown. Export failure +does not change business outcomes or independent audit. Scalar stream diagnostics +count delivered entities, not merely prefetched rows. + +Regenerate libraries for current request-tree/search metadata and ordinary list +pagination. Source/internal evidence is retained in the conformance +`2026-10-13-standard-telemetry-onboarding` report; final internal/public package and +signed-tag verification remain separate gates. diff --git a/examples/trace-chain/lib/generated_search_models.py b/examples/trace-chain/lib/generated_search_models.py new file mode 100644 index 0000000..56c89e4 --- /dev/null +++ b/examples/trace-chain/lib/generated_search_models.py @@ -0,0 +1,128 @@ +from types import MappingProxyType + +# Generated trusted field/type metadata. Runtime owns parsing, binding and warnings. +GENERATED_SEARCH_MODELS = MappingProxyType({ + "Platform": MappingProxyType({ + "fields": MappingProxyType({ + "id": "integer", + "name": "string", + "version": "integer", + }), + "query_fields": MappingProxyType({ + "id": "id", + "name": "name", + "version": "version", + }), + "relations": MappingProxyType({ + "customerOrderList": "CustomerOrder", + }), + }), + "CustomerOrder": MappingProxyType({ + "fields": MappingProxyType({ + "id": "integer", + "platform": "integer", + "order_number": "string", + "orderNumber": "string", + "description": "string", + "version": "integer", + }), + "query_fields": MappingProxyType({ + "id": "id", + "platform": "platform", + "order_number": "order_number", + "orderNumber": "order_number", + "description": "description", + "version": "version", + }), + "relations": MappingProxyType({ + "platform": "Platform", + "orderItemList": "OrderItem", + "paymentList": "Payment", + "shipmentList": "Shipment", + }), + }), + "OrderItem": MappingProxyType({ + "fields": MappingProxyType({ + "id": "integer", + "customer_order": "integer", + "customerOrder": "integer", + "name": "string", + "version": "integer", + }), + "query_fields": MappingProxyType({ + "id": "id", + "customer_order": "customer_order", + "customerOrder": "customer_order", + "name": "name", + "version": "version", + }), + "relations": MappingProxyType({ + "customer_order": "CustomerOrder", + "customerOrder": "CustomerOrder", + }), + }), + "Payment": MappingProxyType({ + "fields": MappingProxyType({ + "id": "integer", + "customer_order": "integer", + "customerOrder": "integer", + "reference_code": "string", + "referenceCode": "string", + "version": "integer", + }), + "query_fields": MappingProxyType({ + "id": "id", + "customer_order": "customer_order", + "customerOrder": "customer_order", + "reference_code": "reference_code", + "referenceCode": "reference_code", + "version": "version", + }), + "relations": MappingProxyType({ + "customer_order": "CustomerOrder", + "customerOrder": "CustomerOrder", + "paymentAttemptList": "PaymentAttempt", + }), + }), + "PaymentAttempt": MappingProxyType({ + "fields": MappingProxyType({ + "id": "integer", + "payment": "integer", + "reference_code": "string", + "referenceCode": "string", + "version": "integer", + }), + "query_fields": MappingProxyType({ + "id": "id", + "payment": "payment", + "reference_code": "reference_code", + "referenceCode": "reference_code", + "version": "version", + }), + "relations": MappingProxyType({ + "payment": "Payment", + }), + }), + "Shipment": MappingProxyType({ + "fields": MappingProxyType({ + "id": "integer", + "customer_order": "integer", + "customerOrder": "integer", + "reference_code": "string", + "referenceCode": "string", + "version": "integer", + }), + "query_fields": MappingProxyType({ + "id": "id", + "customer_order": "customer_order", + "customerOrder": "customer_order", + "reference_code": "reference_code", + "referenceCode": "reference_code", + "version": "version", + }), + "relations": MappingProxyType({ + "customer_order": "CustomerOrder", + "customerOrder": "CustomerOrder", + }), + }), +}) \ No newline at end of file diff --git a/examples/trace-chain/lib/models/customer_order.py b/examples/trace-chain/lib/models/customer_order.py index 34459e4..c756e23 100644 --- a/examples/trace-chain/lib/models/customer_order.py +++ b/examples/trace-chain/lib/models/customer_order.py @@ -1,7 +1,8 @@ from teaql.core.mutation import InsertCommand, UpdateCommand, DeleteCommand, MutationRequest -from teaql.core.entity import _LoadedScalarSnapshot, _QueryProjectionSnapshot -from teaql.core import MutationIntent from teaql.core.value import Value +from teaql.core.request_intent import MutationIntent +from teaql.core.projection_properties import ProjectionProperties +from teaql.runtime.loaded_scalar_snapshot import LoadedScalarSnapshots from teaql.runtime import CheckException, CheckResult, EntityKey, EntityRoot, ObjectLocation import itertools from models.platform import Platform @@ -35,16 +36,8 @@ def __init__(self, **kwargs): kwargs["description"] = kwargs.pop("description") if "version" in kwargs and "version" not in kwargs: kwargs["version"] = kwargs.pop("version") - self._teaql_query_projections = _QueryProjectionSnapshot(kwargs, ( - "id", "id", - "platform", "platform", - "orderNumber", "order_number", - "description", "description", - "version", "version", - "order_item_list", - "payment_list", - "shipment_list", - )) + self._projection_properties = ProjectionProperties(kwargs, + ["id", "platform", "orderNumber", "description", "version", "order_item_list", "payment_list", "shipment_list"]) self._action = "Update" if kwargs.get("id") else "Create" self._comment = None self._loaded_fields = set(kwargs.keys()) @@ -72,19 +65,19 @@ def __init__(self, **kwargs): if self._order_item_list: from models.order_item import OrderItem self._order_item_list = [ - item if isinstance(item, OrderItem) else OrderItem(_entity_root=self._entity_root, **item) + item if isinstance(item, OrderItem) else OrderItem(**item) for item in self._order_item_list ] if self._payment_list: from models.payment import Payment self._payment_list = [ - item if isinstance(item, Payment) else Payment(_entity_root=self._entity_root, **item) + item if isinstance(item, Payment) else Payment(**item) for item in self._payment_list ] if self._shipment_list: from models.shipment import Shipment self._shipment_list = [ - item if isinstance(item, Shipment) else Shipment(_entity_root=self._entity_root, **item) + item if isinstance(item, Shipment) else Shipment(**item) for item in self._shipment_list ] self._ledger_id = getattr(self, "id", None) @@ -95,23 +88,36 @@ def __init__(self, **kwargs): self._entity_root.mark_as_new(key) elif getattr(self, "version", None) is not None: self._entity_root.set_original_version(key, int(self.version)) - self._teaql_loaded_snapshot = _LoadedScalarSnapshot( - self._teaql_scalar_payload() if self._action == "Update" else {}) + LoadedScalarSnapshots.capture(self, self._teaql_snapshot_values()) - def query_projection(self, alias): - """Read a query-only alias; raises KeyError when it was not returned.""" - return self._teaql_query_projections.get(alias) + def _teaql_snapshot_values(self): + payload = {} + if "id" in self._loaded_fields: + payload["id"] = Value.I64(self.id) - def has_query_projection(self, alias): - return self._teaql_query_projections.contains(alias) + if "platform" in self._loaded_fields: + payload["platform"] = Value.I64(getattr(self.platform, "id", self.platform)) + + if "orderNumber" in self._loaded_fields: + payload["order_number"] = Value.Text(self.orderNumber) + + if "description" in self._loaded_fields: + payload["description"] = Value.Text(self.description) + + if "version" in self._loaded_fields: + payload["version"] = Value.I64(self.version) + + return payload def _teaql_entity_key(self): return EntityKey("CustomerOrder", self._ledger_id) + def get_dynamic_property(self, name): + return self._projection_properties.get(name) + def _teaql_attach_root(self, root): - key = self._teaql_entity_key() - if self._entity_root is not root and self._entity_root.has_pending(key): - root.merge_entity_from(self._entity_root, key) + if self._entity_root is not root: + root.merge_entity_from(self._entity_root, self._teaql_entity_key()) self._entity_root = root for child in self._order_item_list: child._teaql_attach_root(root) @@ -131,15 +137,21 @@ def audit_as(self, comment: str): return self async def save(self, context): - intent = MutationIntent(self._comment) - return await context.execute_graph_save(self._teaql_preflight_and_save, comment=intent.comment) + # Validate the root reason before ID allocation or provider access. + MutationIntent(self._comment) + # Reserve stable internal IDs before the transaction and before policy + # review. A relation to a new parent must have the same identity in the + # reviewed graph plan and in the commands eventually sent to SQL. + await self._teaql_reserve_graph_ids(context, set()) + await self._teaql_ensure_business_ids(context) + return await context.execute_graph_save( + lambda session: self._teaql_preflight_and_save(session.context, session), comment=self._comment) async def _teaql_ensure_business_ids(self, context): if self._action != "Create": return - async def _teaql_reserve_graph_ids(self, graph, visited): - context = graph.context + async def _teaql_reserve_graph_ids(self, context, visited): object_identity = id(self) if object_identity in visited: return @@ -152,37 +164,27 @@ async def _teaql_reserve_graph_ids(self, graph, visited): "Configured dataService does not support stable ID reservation" ) old_key = self._teaql_entity_key() - old_loaded = set(self._loaded_fields) self.id = int(await allocator("CustomerOrder")) self._loaded_fields.add("id") self._ledger_id = self.id new_key = self._teaql_entity_key() self._entity_root.rekey(old_key, new_key) self._entity_root.set(new_key, "id", Value.I64(self.id)) - def rollback_reservation(): - self.id = None - self._ledger_id = old_key.id - self._loaded_fields = old_loaded - self._entity_root.rekey(new_key, old_key) - self._entity_root.set(old_key, "id", Value.from_any(None)) - graph.after_rollback(rollback_reservation) - await self._teaql_ensure_business_ids(context) for child in self._order_item_list: child._teaql_attach_root(self._entity_root) - await child._teaql_reserve_graph_ids(graph, visited) + await child._teaql_reserve_graph_ids(context, visited) for child in self._payment_list: child._teaql_attach_root(self._entity_root) - await child._teaql_reserve_graph_ids(graph, visited) + await child._teaql_reserve_graph_ids(context, visited) for child in self._shipment_list: child._teaql_attach_root(self._entity_root) - await child._teaql_reserve_graph_ids(graph, visited) + await child._teaql_reserve_graph_ids(context, visited) - async def _teaql_preflight_and_save(self, graph): - await self._teaql_reserve_graph_ids(graph, set()) - self._teaql_preflight_graph(graph) - return await self._teaql_save_within_graph(graph) + async def _teaql_preflight_and_save(self, context, session): + self._teaql_preflight_graph(context, session) + return await self._teaql_save_within_graph(context, session) - def _teaql_scalar_payload(self): + def _teaql_build_command(self): payload = {} if "id" in self._loaded_fields: payload["id"] = Value.I64(self.id) @@ -203,10 +205,6 @@ def _teaql_scalar_payload(self): if "version" in self._loaded_fields: payload["version"] = Value.I64(self.version) - return payload - - def _teaql_build_command(self): - payload = self._teaql_scalar_payload() action = self._action if action == "Update": ledger = dict(self._entity_root.current_change_set().changes()).get(self._teaql_entity_key(), {}) @@ -214,22 +212,21 @@ def _teaql_build_command(self): if action == "Create": cmd = InsertCommand("CustomerOrder", payload) elif action == "Update": - original_version = self._entity_root.original_version(self._teaql_entity_key()) - cmd = UpdateCommand("CustomerOrder", Value.from_any(getattr(self, "id", None)), - original_version if original_version is not None else getattr(self, "version", None)) + cmd = UpdateCommand("CustomerOrder", Value.from_any(getattr(self, "id", None)), getattr(self, "version", None)) for key, value in payload.items(): if key not in ("id", "version"): cmd.value(key, value) else: - original_version = self._entity_root.original_version(self._teaql_entity_key()) - cmd = DeleteCommand("CustomerOrder", Value.from_any(getattr(self, "id", None)), - original_version if original_version is not None else getattr(self, "version", None)) + cmd = DeleteCommand("CustomerOrder", Value.from_any(getattr(self, "id", None)), getattr(self, "version", None)) if action != "Create": - cmd.old_values = self._teaql_loaded_snapshot.business_values() + cmd.old_values = LoadedScalarSnapshots.values(self) return action, cmd - def _teaql_preflight_graph(self, graph): - context = graph.context - if self._action != "Update" or self._entity_root.has_pending(self._teaql_entity_key()): + def _teaql_preflight_graph(self, context, session, parent=None): + if parent is None: + MutationIntent(self._comment) + scope = session.scope(self._teaql_entity_key(), parent, self._comment) + session.capture_loaded_values("CustomerOrder", LoadedScalarSnapshots.values(self)) + if self._entity_root.has_pending(self._teaql_entity_key()): if self._action == "Update": if "id" not in self._loaded_fields: raise CheckException([CheckResult("invalid_type", ObjectLocation().property("id"), message="Mutation requires a fully loaded entity")]) @@ -250,11 +247,9 @@ def _teaql_preflight_graph(self, graph): self._entity_root.set(self._teaql_entity_key(), field, value) for index, child in enumerate(self._order_item_list): child._teaql_attach_root(self._entity_root) - current = getattr(child, "customerOrder", None) - if getattr(current, "id", current) != self.id: - child.update_customer_order(self) + child.update_customer_order(self) try: - child._teaql_preflight_graph(graph) + child._teaql_preflight_graph(context, session, scope) except CheckException as error: prefix = ObjectLocation().property("order_item_list").index(index) raise CheckException([ @@ -263,11 +258,9 @@ def _teaql_preflight_graph(self, graph): ]) from error for index, child in enumerate(self._payment_list): child._teaql_attach_root(self._entity_root) - current = getattr(child, "customerOrder", None) - if getattr(current, "id", current) != self.id: - child.update_customer_order(self) + child.update_customer_order(self) try: - child._teaql_preflight_graph(graph) + child._teaql_preflight_graph(context, session, scope) except CheckException as error: prefix = ObjectLocation().property("payment_list").index(index) raise CheckException([ @@ -276,11 +269,9 @@ def _teaql_preflight_graph(self, graph): ]) from error for index, child in enumerate(self._shipment_list): child._teaql_attach_root(self._entity_root) - current = getattr(child, "customerOrder", None) - if getattr(current, "id", current) != self.id: - child.update_customer_order(self) + child.update_customer_order(self) try: - child._teaql_preflight_graph(graph) + child._teaql_preflight_graph(context, session, scope) except CheckException as error: prefix = ObjectLocation().property("shipment_list").index(index) raise CheckException([ @@ -288,19 +279,19 @@ def _teaql_preflight_graph(self, graph): for v in error.violations ]) from error - async def _teaql_save_within_graph(self, graph, parent_scope=None): - context = graph.context - scope = graph.scope(self._teaql_entity_key(), parent_scope, self._comment) - - if self._action == "Update" and not self._entity_root.has_pending(self._teaql_entity_key()): - await self._teaql_save_children(graph, scope) - return self + async def _teaql_save_within_graph(self, context, session, parent=None): + if parent is None: + MutationIntent(self._comment) self._teaql_attach_root(self._entity_root) + scope = session.scope(self._teaql_entity_key(), parent, self._comment) + if not self._entity_root.has_pending(self._teaql_entity_key()): + await self._teaql_cascade_children(context, session, scope) + return self action, cmd = self._teaql_build_command() - req = graph.request(cmd, scope, self._entity_root, self._teaql_entity_key()) + req = session.request(cmd, scope, self._entity_root, self._teaql_entity_key()) try: context.check_and_fix_mutation(cmd) @@ -317,6 +308,7 @@ async def _teaql_save_within_graph(self, graph, parent_scope=None): "Mutation provider did not return authoritative persisted state for CustomerOrder" ) rollback_payload = {field: getattr(self, field, None) for field in self._loaded_fields | {"id", "version"}} + rollback_snapshot = LoadedScalarSnapshots.values(self) rollback_ledger_id = self._ledger_id rollback_action = self._action rollback_loaded_fields = set(self._loaded_fields) @@ -351,7 +343,6 @@ async def _teaql_save_within_graph(self, graph, parent_scope=None): elif "version" in persisted: self.version = persisted["version"] self._loaded_fields.add("version") - committed_snapshot = _LoadedScalarSnapshot(self._teaql_scalar_payload()) self._ledger_id = getattr(self, "id", self._ledger_id) new_key = self._teaql_entity_key() if old_key != new_key: @@ -362,72 +353,91 @@ def rollback_entity(): self._ledger_id = rollback_ledger_id self._action = rollback_action self._loaded_fields = rollback_loaded_fields + LoadedScalarSnapshots.restore(self, rollback_snapshot) if old_key != new_key: self._entity_root.rekey(new_key, old_key) - graph.after_rollback(rollback_entity) + session.after_rollback(rollback_entity) if action != "Delete": self._action = "Update" if action != "Delete": - await self._teaql_save_children(graph, scope) + await self._teaql_cascade_children(context, session, scope) def commit_entity(): - self._teaql_loaded_snapshot = committed_snapshot + LoadedScalarSnapshots.capture(self, self._teaql_snapshot_values()) self._entity_root.clear_entity(new_key) if getattr(self, "version", None) is not None: self._entity_root.accept_committed_version(new_key, int(self.version)) - graph.after_commit(commit_entity) + session.after_commit(commit_entity) return self - async def _teaql_save_children(self, graph, scope): + async def _teaql_cascade_children(self, context, session, scope): cascade_relations = [] - cascade_relations.append(("order_item_list", self._order_item_list, "update_customer_order", "customerOrder")) - cascade_relations.append(("payment_list", self._payment_list, "update_customer_order", "customerOrder")) - cascade_relations.append(("shipment_list", self._shipment_list, "update_customer_order", "customerOrder")) - for relation_name, children, updater, member in cascade_relations: + cascade_relations.append(("order_item_list", self._order_item_list, "update_customer_order")) + cascade_relations.append(("payment_list", self._payment_list, "update_customer_order")) + cascade_relations.append(("shipment_list", self._shipment_list, "update_customer_order")) + for relation_name, children, updater in cascade_relations: for index, child in enumerate(children): child._teaql_attach_root(self._entity_root) - current = getattr(child, member, None) - if getattr(current, "id", current) != self.id: - getattr(child, updater)(self) + getattr(child, updater)(self) try: - await child._teaql_save_within_graph(graph, scope) + await child._teaql_save_within_graph(context, session, scope) except CheckException as error: prefix = ObjectLocation().property(relation_name).index(index) raise CheckException([ - CheckResult(v.rule_id, v.location.prefixed_by(prefix), v.input_value, v.system_value, v.message) - for v in error.violations + CheckResult( + violation.rule_id, + violation.location.prefixed_by(prefix), + violation.input_value, + violation.system_value, + violation.message, + ) + for violation in error.violations ]) from error def update_id(self, value): - if not isinstance(value, int) or isinstance(value, bool) or value <= 0: - raise ValueError("id must be a positive integer") + if self._action != "Create": + raise ValueError("ENTITY_ID_IMMUTABLE: an existing entity ID cannot be changed") + value = int(value) if isinstance(value, str) and value.isdecimal() else value + if type(value) is not int or not 0 < value <= 18446744073709551615: + raise ValueError("ENTITY_ID_INVALID: an assigned unsigned-64 identity is required") old_key = self._teaql_entity_key() - new_key = EntityKey(old_key.entity, value) - self._entity_root.rekey(old_key, new_key) self._ledger_id = value + self._entity_root.rekey(old_key, self._teaql_entity_key()) + if "id" in self._loaded_fields and self.id == value: + return self self.id = value self._loaded_fields.add("id") self._entity_root.set(self._teaql_entity_key(), "id", Value.from_any(value)) return self def update_order_number(self, value): + if "orderNumber" in self._loaded_fields and self.orderNumber == value: + return self self.orderNumber = value self._loaded_fields.add("orderNumber") self._entity_root.set(self._teaql_entity_key(), "order_number", Value.from_any(value)) return self def update_description(self, value): + if "description" in self._loaded_fields and self.description == value: + return self self.description = value self._loaded_fields.add("description") self._entity_root.set(self._teaql_entity_key(), "description", Value.from_any(value)) return self def update_version(self, value): + if "version" in self._loaded_fields and self.version == value: + return self self.version = value self._loaded_fields.add("version") self._entity_root.set(self._teaql_entity_key(), "version", Value.from_any(value)) return self def update_platform(self, value): + assigned_id = getattr(value, "id", value) if value else None + current_id = getattr(self.platform, "id", self.platform) + if "platform" in self._loaded_fields and current_id == assigned_id: + return self self.platform = getattr(value, "id", value) if value else None self._loaded_fields.add("platform") self._entity_root.set(self._teaql_entity_key(), "platform", Value.from_any(self.platform)) diff --git a/examples/trace-chain/lib/models/order_item.py b/examples/trace-chain/lib/models/order_item.py index b6fd2a5..0f2c79d 100644 --- a/examples/trace-chain/lib/models/order_item.py +++ b/examples/trace-chain/lib/models/order_item.py @@ -1,7 +1,8 @@ from teaql.core.mutation import InsertCommand, UpdateCommand, DeleteCommand, MutationRequest -from teaql.core.entity import _LoadedScalarSnapshot, _QueryProjectionSnapshot -from teaql.core import MutationIntent from teaql.core.value import Value +from teaql.core.request_intent import MutationIntent +from teaql.core.projection_properties import ProjectionProperties +from teaql.runtime.loaded_scalar_snapshot import LoadedScalarSnapshots from teaql.runtime import CheckException, CheckResult, EntityKey, EntityRoot, ObjectLocation import itertools from models.customer_order import CustomerOrder @@ -33,12 +34,8 @@ def __init__(self, **kwargs): kwargs["name"] = kwargs.pop("name") if "version" in kwargs and "version" not in kwargs: kwargs["version"] = kwargs.pop("version") - self._teaql_query_projections = _QueryProjectionSnapshot(kwargs, ( - "id", "id", - "customerOrder", "customer_order", - "name", "name", - "version", "version", - )) + self._projection_properties = ProjectionProperties(kwargs, + ["id", "customerOrder", "name", "version"]) self._action = "Update" if kwargs.get("id") else "Create" self._comment = None self._loaded_fields = set(kwargs.keys()) @@ -60,23 +57,33 @@ def __init__(self, **kwargs): self._entity_root.mark_as_new(key) elif getattr(self, "version", None) is not None: self._entity_root.set_original_version(key, int(self.version)) - self._teaql_loaded_snapshot = _LoadedScalarSnapshot( - self._teaql_scalar_payload() if self._action == "Update" else {}) + LoadedScalarSnapshots.capture(self, self._teaql_snapshot_values()) - def query_projection(self, alias): - """Read a query-only alias; raises KeyError when it was not returned.""" - return self._teaql_query_projections.get(alias) + def _teaql_snapshot_values(self): + payload = {} + if "id" in self._loaded_fields: + payload["id"] = Value.I64(self.id) + + if "customerOrder" in self._loaded_fields: + payload["customer_order"] = Value.I64(getattr(self.customerOrder, "id", self.customerOrder)) - def has_query_projection(self, alias): - return self._teaql_query_projections.contains(alias) + if "name" in self._loaded_fields: + payload["name"] = Value.Text(self.name) + + if "version" in self._loaded_fields: + payload["version"] = Value.I64(self.version) + + return payload def _teaql_entity_key(self): return EntityKey("OrderItem", self._ledger_id) + def get_dynamic_property(self, name): + return self._projection_properties.get(name) + def _teaql_attach_root(self, root): - key = self._teaql_entity_key() - if self._entity_root is not root and self._entity_root.has_pending(key): - root.merge_entity_from(self._entity_root, key) + if self._entity_root is not root: + root.merge_entity_from(self._entity_root, self._teaql_entity_key()) self._entity_root = root return self def mark_for_deletion(self): @@ -90,15 +97,21 @@ def audit_as(self, comment: str): return self async def save(self, context): - intent = MutationIntent(self._comment) - return await context.execute_graph_save(self._teaql_preflight_and_save, comment=intent.comment) + # Validate the root reason before ID allocation or provider access. + MutationIntent(self._comment) + # Reserve stable internal IDs before the transaction and before policy + # review. A relation to a new parent must have the same identity in the + # reviewed graph plan and in the commands eventually sent to SQL. + await self._teaql_reserve_graph_ids(context, set()) + await self._teaql_ensure_business_ids(context) + return await context.execute_graph_save( + lambda session: self._teaql_preflight_and_save(session.context, session), comment=self._comment) async def _teaql_ensure_business_ids(self, context): if self._action != "Create": return - async def _teaql_reserve_graph_ids(self, graph, visited): - context = graph.context + async def _teaql_reserve_graph_ids(self, context, visited): object_identity = id(self) if object_identity in visited: return @@ -111,28 +124,18 @@ async def _teaql_reserve_graph_ids(self, graph, visited): "Configured dataService does not support stable ID reservation" ) old_key = self._teaql_entity_key() - old_loaded = set(self._loaded_fields) self.id = int(await allocator("OrderItem")) self._loaded_fields.add("id") self._ledger_id = self.id new_key = self._teaql_entity_key() self._entity_root.rekey(old_key, new_key) self._entity_root.set(new_key, "id", Value.I64(self.id)) - def rollback_reservation(): - self.id = None - self._ledger_id = old_key.id - self._loaded_fields = old_loaded - self._entity_root.rekey(new_key, old_key) - self._entity_root.set(old_key, "id", Value.from_any(None)) - graph.after_rollback(rollback_reservation) - await self._teaql_ensure_business_ids(context) - async def _teaql_preflight_and_save(self, graph): - await self._teaql_reserve_graph_ids(graph, set()) - self._teaql_preflight_graph(graph) - return await self._teaql_save_within_graph(graph) + async def _teaql_preflight_and_save(self, context, session): + self._teaql_preflight_graph(context, session) + return await self._teaql_save_within_graph(context, session) - def _teaql_scalar_payload(self): + def _teaql_build_command(self): payload = {} if "id" in self._loaded_fields: payload["id"] = Value.I64(self.id) @@ -150,10 +153,6 @@ def _teaql_scalar_payload(self): if "version" in self._loaded_fields: payload["version"] = Value.I64(self.version) - return payload - - def _teaql_build_command(self): - payload = self._teaql_scalar_payload() action = self._action if action == "Update": ledger = dict(self._entity_root.current_change_set().changes()).get(self._teaql_entity_key(), {}) @@ -161,22 +160,21 @@ def _teaql_build_command(self): if action == "Create": cmd = InsertCommand("OrderItem", payload) elif action == "Update": - original_version = self._entity_root.original_version(self._teaql_entity_key()) - cmd = UpdateCommand("OrderItem", Value.from_any(getattr(self, "id", None)), - original_version if original_version is not None else getattr(self, "version", None)) + cmd = UpdateCommand("OrderItem", Value.from_any(getattr(self, "id", None)), getattr(self, "version", None)) for key, value in payload.items(): if key not in ("id", "version"): cmd.value(key, value) else: - original_version = self._entity_root.original_version(self._teaql_entity_key()) - cmd = DeleteCommand("OrderItem", Value.from_any(getattr(self, "id", None)), - original_version if original_version is not None else getattr(self, "version", None)) + cmd = DeleteCommand("OrderItem", Value.from_any(getattr(self, "id", None)), getattr(self, "version", None)) if action != "Create": - cmd.old_values = self._teaql_loaded_snapshot.business_values() + cmd.old_values = LoadedScalarSnapshots.values(self) return action, cmd - def _teaql_preflight_graph(self, graph): - context = graph.context - if self._action != "Update" or self._entity_root.has_pending(self._teaql_entity_key()): + def _teaql_preflight_graph(self, context, session, parent=None): + if parent is None: + MutationIntent(self._comment) + scope = session.scope(self._teaql_entity_key(), parent, self._comment) + session.capture_loaded_values("OrderItem", LoadedScalarSnapshots.values(self)) + if self._entity_root.has_pending(self._teaql_entity_key()): if self._action == "Update": if "id" not in self._loaded_fields: raise CheckException([CheckResult("invalid_type", ObjectLocation().property("id"), message="Mutation requires a fully loaded entity")]) @@ -194,19 +192,19 @@ def _teaql_preflight_graph(self, graph): if field not in ("id", "version"): self._entity_root.set(self._teaql_entity_key(), field, value) - async def _teaql_save_within_graph(self, graph, parent_scope=None): - context = graph.context - scope = graph.scope(self._teaql_entity_key(), parent_scope, self._comment) - - if self._action == "Update" and not self._entity_root.has_pending(self._teaql_entity_key()): - await self._teaql_save_children(graph, scope) - return self + async def _teaql_save_within_graph(self, context, session, parent=None): + if parent is None: + MutationIntent(self._comment) self._teaql_attach_root(self._entity_root) + scope = session.scope(self._teaql_entity_key(), parent, self._comment) + if not self._entity_root.has_pending(self._teaql_entity_key()): + await self._teaql_cascade_children(context, session, scope) + return self action, cmd = self._teaql_build_command() - req = graph.request(cmd, scope, self._entity_root, self._teaql_entity_key()) + req = session.request(cmd, scope, self._entity_root, self._teaql_entity_key()) try: context.check_and_fix_mutation(cmd) @@ -223,6 +221,7 @@ async def _teaql_save_within_graph(self, graph, parent_scope=None): "Mutation provider did not return authoritative persisted state for OrderItem" ) rollback_payload = {field: getattr(self, field, None) for field in self._loaded_fields | {"id", "version"}} + rollback_snapshot = LoadedScalarSnapshots.values(self) rollback_ledger_id = self._ledger_id rollback_action = self._action rollback_loaded_fields = set(self._loaded_fields) @@ -251,7 +250,6 @@ async def _teaql_save_within_graph(self, graph, parent_scope=None): elif "version" in persisted: self.version = persisted["version"] self._loaded_fields.add("version") - committed_snapshot = _LoadedScalarSnapshot(self._teaql_scalar_payload()) self._ledger_id = getattr(self, "id", self._ledger_id) new_key = self._teaql_entity_key() if old_key != new_key: @@ -262,63 +260,80 @@ def rollback_entity(): self._ledger_id = rollback_ledger_id self._action = rollback_action self._loaded_fields = rollback_loaded_fields + LoadedScalarSnapshots.restore(self, rollback_snapshot) if old_key != new_key: self._entity_root.rekey(new_key, old_key) - graph.after_rollback(rollback_entity) + session.after_rollback(rollback_entity) if action != "Delete": self._action = "Update" if action != "Delete": - await self._teaql_save_children(graph, scope) + await self._teaql_cascade_children(context, session, scope) def commit_entity(): - self._teaql_loaded_snapshot = committed_snapshot + LoadedScalarSnapshots.capture(self, self._teaql_snapshot_values()) self._entity_root.clear_entity(new_key) if getattr(self, "version", None) is not None: self._entity_root.accept_committed_version(new_key, int(self.version)) - graph.after_commit(commit_entity) + session.after_commit(commit_entity) return self - async def _teaql_save_children(self, graph, scope): + async def _teaql_cascade_children(self, context, session, scope): cascade_relations = [] - for relation_name, children, updater, member in cascade_relations: + for relation_name, children, updater in cascade_relations: for index, child in enumerate(children): child._teaql_attach_root(self._entity_root) - current = getattr(child, member, None) - if getattr(current, "id", current) != self.id: - getattr(child, updater)(self) + getattr(child, updater)(self) try: - await child._teaql_save_within_graph(graph, scope) + await child._teaql_save_within_graph(context, session, scope) except CheckException as error: prefix = ObjectLocation().property(relation_name).index(index) raise CheckException([ - CheckResult(v.rule_id, v.location.prefixed_by(prefix), v.input_value, v.system_value, v.message) - for v in error.violations + CheckResult( + violation.rule_id, + violation.location.prefixed_by(prefix), + violation.input_value, + violation.system_value, + violation.message, + ) + for violation in error.violations ]) from error def update_id(self, value): - if not isinstance(value, int) or isinstance(value, bool) or value <= 0: - raise ValueError("id must be a positive integer") + if self._action != "Create": + raise ValueError("ENTITY_ID_IMMUTABLE: an existing entity ID cannot be changed") + value = int(value) if isinstance(value, str) and value.isdecimal() else value + if type(value) is not int or not 0 < value <= 18446744073709551615: + raise ValueError("ENTITY_ID_INVALID: an assigned unsigned-64 identity is required") old_key = self._teaql_entity_key() - new_key = EntityKey(old_key.entity, value) - self._entity_root.rekey(old_key, new_key) self._ledger_id = value + self._entity_root.rekey(old_key, self._teaql_entity_key()) + if "id" in self._loaded_fields and self.id == value: + return self self.id = value self._loaded_fields.add("id") self._entity_root.set(self._teaql_entity_key(), "id", Value.from_any(value)) return self def update_name(self, value): + if "name" in self._loaded_fields and self.name == value: + return self self.name = value self._loaded_fields.add("name") self._entity_root.set(self._teaql_entity_key(), "name", Value.from_any(value)) return self def update_version(self, value): + if "version" in self._loaded_fields and self.version == value: + return self self.version = value self._loaded_fields.add("version") self._entity_root.set(self._teaql_entity_key(), "version", Value.from_any(value)) return self def update_customer_order(self, value): + assigned_id = getattr(value, "id", value) if value else None + current_id = getattr(self.customerOrder, "id", self.customerOrder) + if "customerOrder" in self._loaded_fields and current_id == assigned_id: + return self self.customerOrder = getattr(value, "id", value) if value else None self._loaded_fields.add("customerOrder") self._entity_root.set(self._teaql_entity_key(), "customer_order", Value.from_any(self.customerOrder)) diff --git a/examples/trace-chain/lib/models/payment.py b/examples/trace-chain/lib/models/payment.py index 36256e0..8cad9c2 100644 --- a/examples/trace-chain/lib/models/payment.py +++ b/examples/trace-chain/lib/models/payment.py @@ -1,7 +1,8 @@ from teaql.core.mutation import InsertCommand, UpdateCommand, DeleteCommand, MutationRequest -from teaql.core.entity import _LoadedScalarSnapshot, _QueryProjectionSnapshot -from teaql.core import MutationIntent from teaql.core.value import Value +from teaql.core.request_intent import MutationIntent +from teaql.core.projection_properties import ProjectionProperties +from teaql.runtime.loaded_scalar_snapshot import LoadedScalarSnapshots from teaql.runtime import CheckException, CheckResult, EntityKey, EntityRoot, ObjectLocation import itertools from models.customer_order import CustomerOrder @@ -33,13 +34,8 @@ def __init__(self, **kwargs): kwargs["referenceCode"] = kwargs.pop("reference_code") if "version" in kwargs and "version" not in kwargs: kwargs["version"] = kwargs.pop("version") - self._teaql_query_projections = _QueryProjectionSnapshot(kwargs, ( - "id", "id", - "customerOrder", "customer_order", - "referenceCode", "reference_code", - "version", "version", - "payment_attempt_list", - )) + self._projection_properties = ProjectionProperties(kwargs, + ["id", "customerOrder", "referenceCode", "version", "payment_attempt_list"]) self._action = "Update" if kwargs.get("id") else "Create" self._comment = None self._loaded_fields = set(kwargs.keys()) @@ -59,7 +55,7 @@ def __init__(self, **kwargs): if self._payment_attempt_list: from models.payment_attempt import PaymentAttempt self._payment_attempt_list = [ - item if isinstance(item, PaymentAttempt) else PaymentAttempt(_entity_root=self._entity_root, **item) + item if isinstance(item, PaymentAttempt) else PaymentAttempt(**item) for item in self._payment_attempt_list ] self._ledger_id = getattr(self, "id", None) @@ -70,23 +66,33 @@ def __init__(self, **kwargs): self._entity_root.mark_as_new(key) elif getattr(self, "version", None) is not None: self._entity_root.set_original_version(key, int(self.version)) - self._teaql_loaded_snapshot = _LoadedScalarSnapshot( - self._teaql_scalar_payload() if self._action == "Update" else {}) + LoadedScalarSnapshots.capture(self, self._teaql_snapshot_values()) - def query_projection(self, alias): - """Read a query-only alias; raises KeyError when it was not returned.""" - return self._teaql_query_projections.get(alias) + def _teaql_snapshot_values(self): + payload = {} + if "id" in self._loaded_fields: + payload["id"] = Value.I64(self.id) + + if "customerOrder" in self._loaded_fields: + payload["customer_order"] = Value.I64(getattr(self.customerOrder, "id", self.customerOrder)) - def has_query_projection(self, alias): - return self._teaql_query_projections.contains(alias) + if "referenceCode" in self._loaded_fields: + payload["reference_code"] = Value.Text(self.referenceCode) + + if "version" in self._loaded_fields: + payload["version"] = Value.I64(self.version) + + return payload def _teaql_entity_key(self): return EntityKey("Payment", self._ledger_id) + def get_dynamic_property(self, name): + return self._projection_properties.get(name) + def _teaql_attach_root(self, root): - key = self._teaql_entity_key() - if self._entity_root is not root and self._entity_root.has_pending(key): - root.merge_entity_from(self._entity_root, key) + if self._entity_root is not root: + root.merge_entity_from(self._entity_root, self._teaql_entity_key()) self._entity_root = root for child in self._payment_attempt_list: child._teaql_attach_root(root) @@ -102,15 +108,21 @@ def audit_as(self, comment: str): return self async def save(self, context): - intent = MutationIntent(self._comment) - return await context.execute_graph_save(self._teaql_preflight_and_save, comment=intent.comment) + # Validate the root reason before ID allocation or provider access. + MutationIntent(self._comment) + # Reserve stable internal IDs before the transaction and before policy + # review. A relation to a new parent must have the same identity in the + # reviewed graph plan and in the commands eventually sent to SQL. + await self._teaql_reserve_graph_ids(context, set()) + await self._teaql_ensure_business_ids(context) + return await context.execute_graph_save( + lambda session: self._teaql_preflight_and_save(session.context, session), comment=self._comment) async def _teaql_ensure_business_ids(self, context): if self._action != "Create": return - async def _teaql_reserve_graph_ids(self, graph, visited): - context = graph.context + async def _teaql_reserve_graph_ids(self, context, visited): object_identity = id(self) if object_identity in visited: return @@ -123,31 +135,21 @@ async def _teaql_reserve_graph_ids(self, graph, visited): "Configured dataService does not support stable ID reservation" ) old_key = self._teaql_entity_key() - old_loaded = set(self._loaded_fields) self.id = int(await allocator("Payment")) self._loaded_fields.add("id") self._ledger_id = self.id new_key = self._teaql_entity_key() self._entity_root.rekey(old_key, new_key) self._entity_root.set(new_key, "id", Value.I64(self.id)) - def rollback_reservation(): - self.id = None - self._ledger_id = old_key.id - self._loaded_fields = old_loaded - self._entity_root.rekey(new_key, old_key) - self._entity_root.set(old_key, "id", Value.from_any(None)) - graph.after_rollback(rollback_reservation) - await self._teaql_ensure_business_ids(context) for child in self._payment_attempt_list: child._teaql_attach_root(self._entity_root) - await child._teaql_reserve_graph_ids(graph, visited) + await child._teaql_reserve_graph_ids(context, visited) - async def _teaql_preflight_and_save(self, graph): - await self._teaql_reserve_graph_ids(graph, set()) - self._teaql_preflight_graph(graph) - return await self._teaql_save_within_graph(graph) + async def _teaql_preflight_and_save(self, context, session): + self._teaql_preflight_graph(context, session) + return await self._teaql_save_within_graph(context, session) - def _teaql_scalar_payload(self): + def _teaql_build_command(self): payload = {} if "id" in self._loaded_fields: payload["id"] = Value.I64(self.id) @@ -165,10 +167,6 @@ def _teaql_scalar_payload(self): if "version" in self._loaded_fields: payload["version"] = Value.I64(self.version) - return payload - - def _teaql_build_command(self): - payload = self._teaql_scalar_payload() action = self._action if action == "Update": ledger = dict(self._entity_root.current_change_set().changes()).get(self._teaql_entity_key(), {}) @@ -176,22 +174,21 @@ def _teaql_build_command(self): if action == "Create": cmd = InsertCommand("Payment", payload) elif action == "Update": - original_version = self._entity_root.original_version(self._teaql_entity_key()) - cmd = UpdateCommand("Payment", Value.from_any(getattr(self, "id", None)), - original_version if original_version is not None else getattr(self, "version", None)) + cmd = UpdateCommand("Payment", Value.from_any(getattr(self, "id", None)), getattr(self, "version", None)) for key, value in payload.items(): if key not in ("id", "version"): cmd.value(key, value) else: - original_version = self._entity_root.original_version(self._teaql_entity_key()) - cmd = DeleteCommand("Payment", Value.from_any(getattr(self, "id", None)), - original_version if original_version is not None else getattr(self, "version", None)) + cmd = DeleteCommand("Payment", Value.from_any(getattr(self, "id", None)), getattr(self, "version", None)) if action != "Create": - cmd.old_values = self._teaql_loaded_snapshot.business_values() + cmd.old_values = LoadedScalarSnapshots.values(self) return action, cmd - def _teaql_preflight_graph(self, graph): - context = graph.context - if self._action != "Update" or self._entity_root.has_pending(self._teaql_entity_key()): + def _teaql_preflight_graph(self, context, session, parent=None): + if parent is None: + MutationIntent(self._comment) + scope = session.scope(self._teaql_entity_key(), parent, self._comment) + session.capture_loaded_values("Payment", LoadedScalarSnapshots.values(self)) + if self._entity_root.has_pending(self._teaql_entity_key()): if self._action == "Update": if "id" not in self._loaded_fields: raise CheckException([CheckResult("invalid_type", ObjectLocation().property("id"), message="Mutation requires a fully loaded entity")]) @@ -210,11 +207,9 @@ def _teaql_preflight_graph(self, graph): self._entity_root.set(self._teaql_entity_key(), field, value) for index, child in enumerate(self._payment_attempt_list): child._teaql_attach_root(self._entity_root) - current = getattr(child, "payment", None) - if getattr(current, "id", current) != self.id: - child.update_payment(self) + child.update_payment(self) try: - child._teaql_preflight_graph(graph) + child._teaql_preflight_graph(context, session, scope) except CheckException as error: prefix = ObjectLocation().property("payment_attempt_list").index(index) raise CheckException([ @@ -222,19 +217,19 @@ def _teaql_preflight_graph(self, graph): for v in error.violations ]) from error - async def _teaql_save_within_graph(self, graph, parent_scope=None): - context = graph.context - scope = graph.scope(self._teaql_entity_key(), parent_scope, self._comment) - - if self._action == "Update" and not self._entity_root.has_pending(self._teaql_entity_key()): - await self._teaql_save_children(graph, scope) - return self + async def _teaql_save_within_graph(self, context, session, parent=None): + if parent is None: + MutationIntent(self._comment) self._teaql_attach_root(self._entity_root) + scope = session.scope(self._teaql_entity_key(), parent, self._comment) + if not self._entity_root.has_pending(self._teaql_entity_key()): + await self._teaql_cascade_children(context, session, scope) + return self action, cmd = self._teaql_build_command() - req = graph.request(cmd, scope, self._entity_root, self._teaql_entity_key()) + req = session.request(cmd, scope, self._entity_root, self._teaql_entity_key()) try: context.check_and_fix_mutation(cmd) @@ -251,6 +246,7 @@ async def _teaql_save_within_graph(self, graph, parent_scope=None): "Mutation provider did not return authoritative persisted state for Payment" ) rollback_payload = {field: getattr(self, field, None) for field in self._loaded_fields | {"id", "version"}} + rollback_snapshot = LoadedScalarSnapshots.values(self) rollback_ledger_id = self._ledger_id rollback_action = self._action rollback_loaded_fields = set(self._loaded_fields) @@ -279,7 +275,6 @@ async def _teaql_save_within_graph(self, graph, parent_scope=None): elif "version" in persisted: self.version = persisted["version"] self._loaded_fields.add("version") - committed_snapshot = _LoadedScalarSnapshot(self._teaql_scalar_payload()) self._ledger_id = getattr(self, "id", self._ledger_id) new_key = self._teaql_entity_key() if old_key != new_key: @@ -290,64 +285,81 @@ def rollback_entity(): self._ledger_id = rollback_ledger_id self._action = rollback_action self._loaded_fields = rollback_loaded_fields + LoadedScalarSnapshots.restore(self, rollback_snapshot) if old_key != new_key: self._entity_root.rekey(new_key, old_key) - graph.after_rollback(rollback_entity) + session.after_rollback(rollback_entity) if action != "Delete": self._action = "Update" if action != "Delete": - await self._teaql_save_children(graph, scope) + await self._teaql_cascade_children(context, session, scope) def commit_entity(): - self._teaql_loaded_snapshot = committed_snapshot + LoadedScalarSnapshots.capture(self, self._teaql_snapshot_values()) self._entity_root.clear_entity(new_key) if getattr(self, "version", None) is not None: self._entity_root.accept_committed_version(new_key, int(self.version)) - graph.after_commit(commit_entity) + session.after_commit(commit_entity) return self - async def _teaql_save_children(self, graph, scope): + async def _teaql_cascade_children(self, context, session, scope): cascade_relations = [] - cascade_relations.append(("payment_attempt_list", self._payment_attempt_list, "update_payment", "payment")) - for relation_name, children, updater, member in cascade_relations: + cascade_relations.append(("payment_attempt_list", self._payment_attempt_list, "update_payment")) + for relation_name, children, updater in cascade_relations: for index, child in enumerate(children): child._teaql_attach_root(self._entity_root) - current = getattr(child, member, None) - if getattr(current, "id", current) != self.id: - getattr(child, updater)(self) + getattr(child, updater)(self) try: - await child._teaql_save_within_graph(graph, scope) + await child._teaql_save_within_graph(context, session, scope) except CheckException as error: prefix = ObjectLocation().property(relation_name).index(index) raise CheckException([ - CheckResult(v.rule_id, v.location.prefixed_by(prefix), v.input_value, v.system_value, v.message) - for v in error.violations + CheckResult( + violation.rule_id, + violation.location.prefixed_by(prefix), + violation.input_value, + violation.system_value, + violation.message, + ) + for violation in error.violations ]) from error def update_id(self, value): - if not isinstance(value, int) or isinstance(value, bool) or value <= 0: - raise ValueError("id must be a positive integer") + if self._action != "Create": + raise ValueError("ENTITY_ID_IMMUTABLE: an existing entity ID cannot be changed") + value = int(value) if isinstance(value, str) and value.isdecimal() else value + if type(value) is not int or not 0 < value <= 18446744073709551615: + raise ValueError("ENTITY_ID_INVALID: an assigned unsigned-64 identity is required") old_key = self._teaql_entity_key() - new_key = EntityKey(old_key.entity, value) - self._entity_root.rekey(old_key, new_key) self._ledger_id = value + self._entity_root.rekey(old_key, self._teaql_entity_key()) + if "id" in self._loaded_fields and self.id == value: + return self self.id = value self._loaded_fields.add("id") self._entity_root.set(self._teaql_entity_key(), "id", Value.from_any(value)) return self def update_reference_code(self, value): + if "referenceCode" in self._loaded_fields and self.referenceCode == value: + return self self.referenceCode = value self._loaded_fields.add("referenceCode") self._entity_root.set(self._teaql_entity_key(), "reference_code", Value.from_any(value)) return self def update_version(self, value): + if "version" in self._loaded_fields and self.version == value: + return self self.version = value self._loaded_fields.add("version") self._entity_root.set(self._teaql_entity_key(), "version", Value.from_any(value)) return self def update_customer_order(self, value): + assigned_id = getattr(value, "id", value) if value else None + current_id = getattr(self.customerOrder, "id", self.customerOrder) + if "customerOrder" in self._loaded_fields and current_id == assigned_id: + return self self.customerOrder = getattr(value, "id", value) if value else None self._loaded_fields.add("customerOrder") self._entity_root.set(self._teaql_entity_key(), "customer_order", Value.from_any(self.customerOrder)) diff --git a/examples/trace-chain/lib/models/payment_attempt.py b/examples/trace-chain/lib/models/payment_attempt.py index 8aeeac0..875e32e 100644 --- a/examples/trace-chain/lib/models/payment_attempt.py +++ b/examples/trace-chain/lib/models/payment_attempt.py @@ -1,7 +1,8 @@ from teaql.core.mutation import InsertCommand, UpdateCommand, DeleteCommand, MutationRequest -from teaql.core.entity import _LoadedScalarSnapshot, _QueryProjectionSnapshot -from teaql.core import MutationIntent from teaql.core.value import Value +from teaql.core.request_intent import MutationIntent +from teaql.core.projection_properties import ProjectionProperties +from teaql.runtime.loaded_scalar_snapshot import LoadedScalarSnapshots from teaql.runtime import CheckException, CheckResult, EntityKey, EntityRoot, ObjectLocation import itertools from models.payment import Payment @@ -33,12 +34,8 @@ def __init__(self, **kwargs): kwargs["referenceCode"] = kwargs.pop("reference_code") if "version" in kwargs and "version" not in kwargs: kwargs["version"] = kwargs.pop("version") - self._teaql_query_projections = _QueryProjectionSnapshot(kwargs, ( - "id", "id", - "payment", "payment", - "referenceCode", "reference_code", - "version", "version", - )) + self._projection_properties = ProjectionProperties(kwargs, + ["id", "payment", "referenceCode", "version"]) self._action = "Update" if kwargs.get("id") else "Create" self._comment = None self._loaded_fields = set(kwargs.keys()) @@ -60,23 +57,33 @@ def __init__(self, **kwargs): self._entity_root.mark_as_new(key) elif getattr(self, "version", None) is not None: self._entity_root.set_original_version(key, int(self.version)) - self._teaql_loaded_snapshot = _LoadedScalarSnapshot( - self._teaql_scalar_payload() if self._action == "Update" else {}) + LoadedScalarSnapshots.capture(self, self._teaql_snapshot_values()) - def query_projection(self, alias): - """Read a query-only alias; raises KeyError when it was not returned.""" - return self._teaql_query_projections.get(alias) + def _teaql_snapshot_values(self): + payload = {} + if "id" in self._loaded_fields: + payload["id"] = Value.I64(self.id) + + if "payment" in self._loaded_fields: + payload["payment"] = Value.I64(getattr(self.payment, "id", self.payment)) - def has_query_projection(self, alias): - return self._teaql_query_projections.contains(alias) + if "referenceCode" in self._loaded_fields: + payload["reference_code"] = Value.Text(self.referenceCode) + + if "version" in self._loaded_fields: + payload["version"] = Value.I64(self.version) + + return payload def _teaql_entity_key(self): return EntityKey("PaymentAttempt", self._ledger_id) + def get_dynamic_property(self, name): + return self._projection_properties.get(name) + def _teaql_attach_root(self, root): - key = self._teaql_entity_key() - if self._entity_root is not root and self._entity_root.has_pending(key): - root.merge_entity_from(self._entity_root, key) + if self._entity_root is not root: + root.merge_entity_from(self._entity_root, self._teaql_entity_key()) self._entity_root = root return self def mark_for_deletion(self): @@ -90,15 +97,21 @@ def audit_as(self, comment: str): return self async def save(self, context): - intent = MutationIntent(self._comment) - return await context.execute_graph_save(self._teaql_preflight_and_save, comment=intent.comment) + # Validate the root reason before ID allocation or provider access. + MutationIntent(self._comment) + # Reserve stable internal IDs before the transaction and before policy + # review. A relation to a new parent must have the same identity in the + # reviewed graph plan and in the commands eventually sent to SQL. + await self._teaql_reserve_graph_ids(context, set()) + await self._teaql_ensure_business_ids(context) + return await context.execute_graph_save( + lambda session: self._teaql_preflight_and_save(session.context, session), comment=self._comment) async def _teaql_ensure_business_ids(self, context): if self._action != "Create": return - async def _teaql_reserve_graph_ids(self, graph, visited): - context = graph.context + async def _teaql_reserve_graph_ids(self, context, visited): object_identity = id(self) if object_identity in visited: return @@ -111,28 +124,18 @@ async def _teaql_reserve_graph_ids(self, graph, visited): "Configured dataService does not support stable ID reservation" ) old_key = self._teaql_entity_key() - old_loaded = set(self._loaded_fields) self.id = int(await allocator("PaymentAttempt")) self._loaded_fields.add("id") self._ledger_id = self.id new_key = self._teaql_entity_key() self._entity_root.rekey(old_key, new_key) self._entity_root.set(new_key, "id", Value.I64(self.id)) - def rollback_reservation(): - self.id = None - self._ledger_id = old_key.id - self._loaded_fields = old_loaded - self._entity_root.rekey(new_key, old_key) - self._entity_root.set(old_key, "id", Value.from_any(None)) - graph.after_rollback(rollback_reservation) - await self._teaql_ensure_business_ids(context) - async def _teaql_preflight_and_save(self, graph): - await self._teaql_reserve_graph_ids(graph, set()) - self._teaql_preflight_graph(graph) - return await self._teaql_save_within_graph(graph) + async def _teaql_preflight_and_save(self, context, session): + self._teaql_preflight_graph(context, session) + return await self._teaql_save_within_graph(context, session) - def _teaql_scalar_payload(self): + def _teaql_build_command(self): payload = {} if "id" in self._loaded_fields: payload["id"] = Value.I64(self.id) @@ -150,10 +153,6 @@ def _teaql_scalar_payload(self): if "version" in self._loaded_fields: payload["version"] = Value.I64(self.version) - return payload - - def _teaql_build_command(self): - payload = self._teaql_scalar_payload() action = self._action if action == "Update": ledger = dict(self._entity_root.current_change_set().changes()).get(self._teaql_entity_key(), {}) @@ -161,22 +160,21 @@ def _teaql_build_command(self): if action == "Create": cmd = InsertCommand("PaymentAttempt", payload) elif action == "Update": - original_version = self._entity_root.original_version(self._teaql_entity_key()) - cmd = UpdateCommand("PaymentAttempt", Value.from_any(getattr(self, "id", None)), - original_version if original_version is not None else getattr(self, "version", None)) + cmd = UpdateCommand("PaymentAttempt", Value.from_any(getattr(self, "id", None)), getattr(self, "version", None)) for key, value in payload.items(): if key not in ("id", "version"): cmd.value(key, value) else: - original_version = self._entity_root.original_version(self._teaql_entity_key()) - cmd = DeleteCommand("PaymentAttempt", Value.from_any(getattr(self, "id", None)), - original_version if original_version is not None else getattr(self, "version", None)) + cmd = DeleteCommand("PaymentAttempt", Value.from_any(getattr(self, "id", None)), getattr(self, "version", None)) if action != "Create": - cmd.old_values = self._teaql_loaded_snapshot.business_values() + cmd.old_values = LoadedScalarSnapshots.values(self) return action, cmd - def _teaql_preflight_graph(self, graph): - context = graph.context - if self._action != "Update" or self._entity_root.has_pending(self._teaql_entity_key()): + def _teaql_preflight_graph(self, context, session, parent=None): + if parent is None: + MutationIntent(self._comment) + scope = session.scope(self._teaql_entity_key(), parent, self._comment) + session.capture_loaded_values("PaymentAttempt", LoadedScalarSnapshots.values(self)) + if self._entity_root.has_pending(self._teaql_entity_key()): if self._action == "Update": if "id" not in self._loaded_fields: raise CheckException([CheckResult("invalid_type", ObjectLocation().property("id"), message="Mutation requires a fully loaded entity")]) @@ -194,19 +192,19 @@ def _teaql_preflight_graph(self, graph): if field not in ("id", "version"): self._entity_root.set(self._teaql_entity_key(), field, value) - async def _teaql_save_within_graph(self, graph, parent_scope=None): - context = graph.context - scope = graph.scope(self._teaql_entity_key(), parent_scope, self._comment) - - if self._action == "Update" and not self._entity_root.has_pending(self._teaql_entity_key()): - await self._teaql_save_children(graph, scope) - return self + async def _teaql_save_within_graph(self, context, session, parent=None): + if parent is None: + MutationIntent(self._comment) self._teaql_attach_root(self._entity_root) + scope = session.scope(self._teaql_entity_key(), parent, self._comment) + if not self._entity_root.has_pending(self._teaql_entity_key()): + await self._teaql_cascade_children(context, session, scope) + return self action, cmd = self._teaql_build_command() - req = graph.request(cmd, scope, self._entity_root, self._teaql_entity_key()) + req = session.request(cmd, scope, self._entity_root, self._teaql_entity_key()) try: context.check_and_fix_mutation(cmd) @@ -223,6 +221,7 @@ async def _teaql_save_within_graph(self, graph, parent_scope=None): "Mutation provider did not return authoritative persisted state for PaymentAttempt" ) rollback_payload = {field: getattr(self, field, None) for field in self._loaded_fields | {"id", "version"}} + rollback_snapshot = LoadedScalarSnapshots.values(self) rollback_ledger_id = self._ledger_id rollback_action = self._action rollback_loaded_fields = set(self._loaded_fields) @@ -251,7 +250,6 @@ async def _teaql_save_within_graph(self, graph, parent_scope=None): elif "version" in persisted: self.version = persisted["version"] self._loaded_fields.add("version") - committed_snapshot = _LoadedScalarSnapshot(self._teaql_scalar_payload()) self._ledger_id = getattr(self, "id", self._ledger_id) new_key = self._teaql_entity_key() if old_key != new_key: @@ -262,63 +260,80 @@ def rollback_entity(): self._ledger_id = rollback_ledger_id self._action = rollback_action self._loaded_fields = rollback_loaded_fields + LoadedScalarSnapshots.restore(self, rollback_snapshot) if old_key != new_key: self._entity_root.rekey(new_key, old_key) - graph.after_rollback(rollback_entity) + session.after_rollback(rollback_entity) if action != "Delete": self._action = "Update" if action != "Delete": - await self._teaql_save_children(graph, scope) + await self._teaql_cascade_children(context, session, scope) def commit_entity(): - self._teaql_loaded_snapshot = committed_snapshot + LoadedScalarSnapshots.capture(self, self._teaql_snapshot_values()) self._entity_root.clear_entity(new_key) if getattr(self, "version", None) is not None: self._entity_root.accept_committed_version(new_key, int(self.version)) - graph.after_commit(commit_entity) + session.after_commit(commit_entity) return self - async def _teaql_save_children(self, graph, scope): + async def _teaql_cascade_children(self, context, session, scope): cascade_relations = [] - for relation_name, children, updater, member in cascade_relations: + for relation_name, children, updater in cascade_relations: for index, child in enumerate(children): child._teaql_attach_root(self._entity_root) - current = getattr(child, member, None) - if getattr(current, "id", current) != self.id: - getattr(child, updater)(self) + getattr(child, updater)(self) try: - await child._teaql_save_within_graph(graph, scope) + await child._teaql_save_within_graph(context, session, scope) except CheckException as error: prefix = ObjectLocation().property(relation_name).index(index) raise CheckException([ - CheckResult(v.rule_id, v.location.prefixed_by(prefix), v.input_value, v.system_value, v.message) - for v in error.violations + CheckResult( + violation.rule_id, + violation.location.prefixed_by(prefix), + violation.input_value, + violation.system_value, + violation.message, + ) + for violation in error.violations ]) from error def update_id(self, value): - if not isinstance(value, int) or isinstance(value, bool) or value <= 0: - raise ValueError("id must be a positive integer") + if self._action != "Create": + raise ValueError("ENTITY_ID_IMMUTABLE: an existing entity ID cannot be changed") + value = int(value) if isinstance(value, str) and value.isdecimal() else value + if type(value) is not int or not 0 < value <= 18446744073709551615: + raise ValueError("ENTITY_ID_INVALID: an assigned unsigned-64 identity is required") old_key = self._teaql_entity_key() - new_key = EntityKey(old_key.entity, value) - self._entity_root.rekey(old_key, new_key) self._ledger_id = value + self._entity_root.rekey(old_key, self._teaql_entity_key()) + if "id" in self._loaded_fields and self.id == value: + return self self.id = value self._loaded_fields.add("id") self._entity_root.set(self._teaql_entity_key(), "id", Value.from_any(value)) return self def update_reference_code(self, value): + if "referenceCode" in self._loaded_fields and self.referenceCode == value: + return self self.referenceCode = value self._loaded_fields.add("referenceCode") self._entity_root.set(self._teaql_entity_key(), "reference_code", Value.from_any(value)) return self def update_version(self, value): + if "version" in self._loaded_fields and self.version == value: + return self self.version = value self._loaded_fields.add("version") self._entity_root.set(self._teaql_entity_key(), "version", Value.from_any(value)) return self def update_payment(self, value): + assigned_id = getattr(value, "id", value) if value else None + current_id = getattr(self.payment, "id", self.payment) + if "payment" in self._loaded_fields and current_id == assigned_id: + return self self.payment = getattr(value, "id", value) if value else None self._loaded_fields.add("payment") self._entity_root.set(self._teaql_entity_key(), "payment", Value.from_any(self.payment)) diff --git a/examples/trace-chain/lib/models/platform.py b/examples/trace-chain/lib/models/platform.py index b1cf88e..e6bc089 100644 --- a/examples/trace-chain/lib/models/platform.py +++ b/examples/trace-chain/lib/models/platform.py @@ -1,7 +1,8 @@ from teaql.core.mutation import InsertCommand, UpdateCommand, DeleteCommand, MutationRequest -from teaql.core.entity import _LoadedScalarSnapshot, _QueryProjectionSnapshot -from teaql.core import MutationIntent from teaql.core.value import Value +from teaql.core.request_intent import MutationIntent +from teaql.core.projection_properties import ProjectionProperties +from teaql.runtime.loaded_scalar_snapshot import LoadedScalarSnapshots from teaql.runtime import CheckException, CheckResult, EntityKey, EntityRoot, ObjectLocation import itertools @@ -30,12 +31,8 @@ def __init__(self, **kwargs): kwargs["name"] = kwargs.pop("name") if "version" in kwargs and "version" not in kwargs: kwargs["version"] = kwargs.pop("version") - self._teaql_query_projections = _QueryProjectionSnapshot(kwargs, ( - "id", "id", - "name", "name", - "version", "version", - "customer_order_list", - )) + self._projection_properties = ProjectionProperties(kwargs, + ["id", "name", "version", "customer_order_list"]) self._action = "Update" if kwargs.get("id") else "Create" self._comment = None self._loaded_fields = set(kwargs.keys()) @@ -51,7 +48,7 @@ def __init__(self, **kwargs): if self._customer_order_list: from models.customer_order import CustomerOrder self._customer_order_list = [ - item if isinstance(item, CustomerOrder) else CustomerOrder(_entity_root=self._entity_root, **item) + item if isinstance(item, CustomerOrder) else CustomerOrder(**item) for item in self._customer_order_list ] self._ledger_id = getattr(self, "id", None) @@ -62,23 +59,30 @@ def __init__(self, **kwargs): self._entity_root.mark_as_new(key) elif getattr(self, "version", None) is not None: self._entity_root.set_original_version(key, int(self.version)) - self._teaql_loaded_snapshot = _LoadedScalarSnapshot( - self._teaql_scalar_payload() if self._action == "Update" else {}) + LoadedScalarSnapshots.capture(self, self._teaql_snapshot_values()) - def query_projection(self, alias): - """Read a query-only alias; raises KeyError when it was not returned.""" - return self._teaql_query_projections.get(alias) + def _teaql_snapshot_values(self): + payload = {} + if "id" in self._loaded_fields: + payload["id"] = Value.I64(self.id) - def has_query_projection(self, alias): - return self._teaql_query_projections.contains(alias) + if "name" in self._loaded_fields: + payload["name"] = Value.Text(self.name) + + if "version" in self._loaded_fields: + payload["version"] = Value.I64(self.version) + + return payload def _teaql_entity_key(self): return EntityKey("Platform", self._ledger_id) + def get_dynamic_property(self, name): + return self._projection_properties.get(name) + def _teaql_attach_root(self, root): - key = self._teaql_entity_key() - if self._entity_root is not root and self._entity_root.has_pending(key): - root.merge_entity_from(self._entity_root, key) + if self._entity_root is not root: + root.merge_entity_from(self._entity_root, self._teaql_entity_key()) self._entity_root = root for child in self._customer_order_list: child._teaql_attach_root(root) @@ -94,15 +98,21 @@ def audit_as(self, comment: str): return self async def save(self, context): - intent = MutationIntent(self._comment) - return await context.execute_graph_save(self._teaql_preflight_and_save, comment=intent.comment) + # Validate the root reason before ID allocation or provider access. + MutationIntent(self._comment) + # Reserve stable internal IDs before the transaction and before policy + # review. A relation to a new parent must have the same identity in the + # reviewed graph plan and in the commands eventually sent to SQL. + await self._teaql_reserve_graph_ids(context, set()) + await self._teaql_ensure_business_ids(context) + return await context.execute_graph_save( + lambda session: self._teaql_preflight_and_save(session.context, session), comment=self._comment) async def _teaql_ensure_business_ids(self, context): if self._action != "Create": return - async def _teaql_reserve_graph_ids(self, graph, visited): - context = graph.context + async def _teaql_reserve_graph_ids(self, context, visited): object_identity = id(self) if object_identity in visited: return @@ -115,31 +125,21 @@ async def _teaql_reserve_graph_ids(self, graph, visited): "Configured dataService does not support stable ID reservation" ) old_key = self._teaql_entity_key() - old_loaded = set(self._loaded_fields) self.id = int(await allocator("Platform")) self._loaded_fields.add("id") self._ledger_id = self.id new_key = self._teaql_entity_key() self._entity_root.rekey(old_key, new_key) self._entity_root.set(new_key, "id", Value.I64(self.id)) - def rollback_reservation(): - self.id = None - self._ledger_id = old_key.id - self._loaded_fields = old_loaded - self._entity_root.rekey(new_key, old_key) - self._entity_root.set(old_key, "id", Value.from_any(None)) - graph.after_rollback(rollback_reservation) - await self._teaql_ensure_business_ids(context) for child in self._customer_order_list: child._teaql_attach_root(self._entity_root) - await child._teaql_reserve_graph_ids(graph, visited) + await child._teaql_reserve_graph_ids(context, visited) - async def _teaql_preflight_and_save(self, graph): - await self._teaql_reserve_graph_ids(graph, set()) - self._teaql_preflight_graph(graph) - return await self._teaql_save_within_graph(graph) + async def _teaql_preflight_and_save(self, context, session): + self._teaql_preflight_graph(context, session) + return await self._teaql_save_within_graph(context, session) - def _teaql_scalar_payload(self): + def _teaql_build_command(self): payload = {} if "id" in self._loaded_fields: payload["id"] = Value.I64(self.id) @@ -150,10 +150,6 @@ def _teaql_scalar_payload(self): if "version" in self._loaded_fields: payload["version"] = Value.I64(self.version) - return payload - - def _teaql_build_command(self): - payload = self._teaql_scalar_payload() action = self._action if action == "Update": ledger = dict(self._entity_root.current_change_set().changes()).get(self._teaql_entity_key(), {}) @@ -161,22 +157,21 @@ def _teaql_build_command(self): if action == "Create": cmd = InsertCommand("Platform", payload) elif action == "Update": - original_version = self._entity_root.original_version(self._teaql_entity_key()) - cmd = UpdateCommand("Platform", Value.from_any(getattr(self, "id", None)), - original_version if original_version is not None else getattr(self, "version", None)) + cmd = UpdateCommand("Platform", Value.from_any(getattr(self, "id", None)), getattr(self, "version", None)) for key, value in payload.items(): if key not in ("id", "version"): cmd.value(key, value) else: - original_version = self._entity_root.original_version(self._teaql_entity_key()) - cmd = DeleteCommand("Platform", Value.from_any(getattr(self, "id", None)), - original_version if original_version is not None else getattr(self, "version", None)) + cmd = DeleteCommand("Platform", Value.from_any(getattr(self, "id", None)), getattr(self, "version", None)) if action != "Create": - cmd.old_values = self._teaql_loaded_snapshot.business_values() + cmd.old_values = LoadedScalarSnapshots.values(self) return action, cmd - def _teaql_preflight_graph(self, graph): - context = graph.context - if self._action != "Update" or self._entity_root.has_pending(self._teaql_entity_key()): + def _teaql_preflight_graph(self, context, session, parent=None): + if parent is None: + MutationIntent(self._comment) + scope = session.scope(self._teaql_entity_key(), parent, self._comment) + session.capture_loaded_values("Platform", LoadedScalarSnapshots.values(self)) + if self._entity_root.has_pending(self._teaql_entity_key()): if self._action == "Update": if "id" not in self._loaded_fields: raise CheckException([CheckResult("invalid_type", ObjectLocation().property("id"), message="Mutation requires a fully loaded entity")]) @@ -193,11 +188,9 @@ def _teaql_preflight_graph(self, graph): self._entity_root.set(self._teaql_entity_key(), field, value) for index, child in enumerate(self._customer_order_list): child._teaql_attach_root(self._entity_root) - current = getattr(child, "platform", None) - if getattr(current, "id", current) != self.id: - child.update_platform(self) + child.update_platform(self) try: - child._teaql_preflight_graph(graph) + child._teaql_preflight_graph(context, session, scope) except CheckException as error: prefix = ObjectLocation().property("customer_order_list").index(index) raise CheckException([ @@ -205,19 +198,19 @@ def _teaql_preflight_graph(self, graph): for v in error.violations ]) from error - async def _teaql_save_within_graph(self, graph, parent_scope=None): - context = graph.context - scope = graph.scope(self._teaql_entity_key(), parent_scope, self._comment) - - if self._action == "Update" and not self._entity_root.has_pending(self._teaql_entity_key()): - await self._teaql_save_children(graph, scope) - return self + async def _teaql_save_within_graph(self, context, session, parent=None): + if parent is None: + MutationIntent(self._comment) self._teaql_attach_root(self._entity_root) + scope = session.scope(self._teaql_entity_key(), parent, self._comment) + if not self._entity_root.has_pending(self._teaql_entity_key()): + await self._teaql_cascade_children(context, session, scope) + return self action, cmd = self._teaql_build_command() - req = graph.request(cmd, scope, self._entity_root, self._teaql_entity_key()) + req = session.request(cmd, scope, self._entity_root, self._teaql_entity_key()) try: context.check_and_fix_mutation(cmd) @@ -234,6 +227,7 @@ async def _teaql_save_within_graph(self, graph, parent_scope=None): "Mutation provider did not return authoritative persisted state for Platform" ) rollback_payload = {field: getattr(self, field, None) for field in self._loaded_fields | {"id", "version"}} + rollback_snapshot = LoadedScalarSnapshots.values(self) rollback_ledger_id = self._ledger_id rollback_action = self._action rollback_loaded_fields = set(self._loaded_fields) @@ -256,7 +250,6 @@ async def _teaql_save_within_graph(self, graph, parent_scope=None): elif "version" in persisted: self.version = persisted["version"] self._loaded_fields.add("version") - committed_snapshot = _LoadedScalarSnapshot(self._teaql_scalar_payload()) self._ledger_id = getattr(self, "id", self._ledger_id) new_key = self._teaql_entity_key() if old_key != new_key: @@ -267,59 +260,72 @@ def rollback_entity(): self._ledger_id = rollback_ledger_id self._action = rollback_action self._loaded_fields = rollback_loaded_fields + LoadedScalarSnapshots.restore(self, rollback_snapshot) if old_key != new_key: self._entity_root.rekey(new_key, old_key) - graph.after_rollback(rollback_entity) + session.after_rollback(rollback_entity) if action != "Delete": self._action = "Update" if action != "Delete": - await self._teaql_save_children(graph, scope) + await self._teaql_cascade_children(context, session, scope) def commit_entity(): - self._teaql_loaded_snapshot = committed_snapshot + LoadedScalarSnapshots.capture(self, self._teaql_snapshot_values()) self._entity_root.clear_entity(new_key) if getattr(self, "version", None) is not None: self._entity_root.accept_committed_version(new_key, int(self.version)) - graph.after_commit(commit_entity) + session.after_commit(commit_entity) return self - async def _teaql_save_children(self, graph, scope): + async def _teaql_cascade_children(self, context, session, scope): cascade_relations = [] - cascade_relations.append(("customer_order_list", self._customer_order_list, "update_platform", "platform")) - for relation_name, children, updater, member in cascade_relations: + cascade_relations.append(("customer_order_list", self._customer_order_list, "update_platform")) + for relation_name, children, updater in cascade_relations: for index, child in enumerate(children): child._teaql_attach_root(self._entity_root) - current = getattr(child, member, None) - if getattr(current, "id", current) != self.id: - getattr(child, updater)(self) + getattr(child, updater)(self) try: - await child._teaql_save_within_graph(graph, scope) + await child._teaql_save_within_graph(context, session, scope) except CheckException as error: prefix = ObjectLocation().property(relation_name).index(index) raise CheckException([ - CheckResult(v.rule_id, v.location.prefixed_by(prefix), v.input_value, v.system_value, v.message) - for v in error.violations + CheckResult( + violation.rule_id, + violation.location.prefixed_by(prefix), + violation.input_value, + violation.system_value, + violation.message, + ) + for violation in error.violations ]) from error def update_id(self, value): - if not isinstance(value, int) or isinstance(value, bool) or value <= 0: - raise ValueError("id must be a positive integer") + if self._action != "Create": + raise ValueError("ENTITY_ID_IMMUTABLE: an existing entity ID cannot be changed") + value = int(value) if isinstance(value, str) and value.isdecimal() else value + if type(value) is not int or not 0 < value <= 18446744073709551615: + raise ValueError("ENTITY_ID_INVALID: an assigned unsigned-64 identity is required") old_key = self._teaql_entity_key() - new_key = EntityKey(old_key.entity, value) - self._entity_root.rekey(old_key, new_key) self._ledger_id = value + self._entity_root.rekey(old_key, self._teaql_entity_key()) + if "id" in self._loaded_fields and self.id == value: + return self self.id = value self._loaded_fields.add("id") self._entity_root.set(self._teaql_entity_key(), "id", Value.from_any(value)) return self def update_name(self, value): + if "name" in self._loaded_fields and self.name == value: + return self self.name = value self._loaded_fields.add("name") self._entity_root.set(self._teaql_entity_key(), "name", Value.from_any(value)) return self def update_version(self, value): + if "version" in self._loaded_fields and self.version == value: + return self self.version = value self._loaded_fields.add("version") self._entity_root.set(self._teaql_entity_key(), "version", Value.from_any(value)) diff --git a/examples/trace-chain/lib/models/shipment.py b/examples/trace-chain/lib/models/shipment.py index 406d1c6..feaa9df 100644 --- a/examples/trace-chain/lib/models/shipment.py +++ b/examples/trace-chain/lib/models/shipment.py @@ -1,7 +1,8 @@ from teaql.core.mutation import InsertCommand, UpdateCommand, DeleteCommand, MutationRequest -from teaql.core.entity import _LoadedScalarSnapshot, _QueryProjectionSnapshot -from teaql.core import MutationIntent from teaql.core.value import Value +from teaql.core.request_intent import MutationIntent +from teaql.core.projection_properties import ProjectionProperties +from teaql.runtime.loaded_scalar_snapshot import LoadedScalarSnapshots from teaql.runtime import CheckException, CheckResult, EntityKey, EntityRoot, ObjectLocation import itertools from models.customer_order import CustomerOrder @@ -33,12 +34,8 @@ def __init__(self, **kwargs): kwargs["referenceCode"] = kwargs.pop("reference_code") if "version" in kwargs and "version" not in kwargs: kwargs["version"] = kwargs.pop("version") - self._teaql_query_projections = _QueryProjectionSnapshot(kwargs, ( - "id", "id", - "customerOrder", "customer_order", - "referenceCode", "reference_code", - "version", "version", - )) + self._projection_properties = ProjectionProperties(kwargs, + ["id", "customerOrder", "referenceCode", "version"]) self._action = "Update" if kwargs.get("id") else "Create" self._comment = None self._loaded_fields = set(kwargs.keys()) @@ -60,23 +57,33 @@ def __init__(self, **kwargs): self._entity_root.mark_as_new(key) elif getattr(self, "version", None) is not None: self._entity_root.set_original_version(key, int(self.version)) - self._teaql_loaded_snapshot = _LoadedScalarSnapshot( - self._teaql_scalar_payload() if self._action == "Update" else {}) + LoadedScalarSnapshots.capture(self, self._teaql_snapshot_values()) - def query_projection(self, alias): - """Read a query-only alias; raises KeyError when it was not returned.""" - return self._teaql_query_projections.get(alias) + def _teaql_snapshot_values(self): + payload = {} + if "id" in self._loaded_fields: + payload["id"] = Value.I64(self.id) + + if "customerOrder" in self._loaded_fields: + payload["customer_order"] = Value.I64(getattr(self.customerOrder, "id", self.customerOrder)) - def has_query_projection(self, alias): - return self._teaql_query_projections.contains(alias) + if "referenceCode" in self._loaded_fields: + payload["reference_code"] = Value.Text(self.referenceCode) + + if "version" in self._loaded_fields: + payload["version"] = Value.I64(self.version) + + return payload def _teaql_entity_key(self): return EntityKey("Shipment", self._ledger_id) + def get_dynamic_property(self, name): + return self._projection_properties.get(name) + def _teaql_attach_root(self, root): - key = self._teaql_entity_key() - if self._entity_root is not root and self._entity_root.has_pending(key): - root.merge_entity_from(self._entity_root, key) + if self._entity_root is not root: + root.merge_entity_from(self._entity_root, self._teaql_entity_key()) self._entity_root = root return self def mark_for_deletion(self): @@ -90,15 +97,21 @@ def audit_as(self, comment: str): return self async def save(self, context): - intent = MutationIntent(self._comment) - return await context.execute_graph_save(self._teaql_preflight_and_save, comment=intent.comment) + # Validate the root reason before ID allocation or provider access. + MutationIntent(self._comment) + # Reserve stable internal IDs before the transaction and before policy + # review. A relation to a new parent must have the same identity in the + # reviewed graph plan and in the commands eventually sent to SQL. + await self._teaql_reserve_graph_ids(context, set()) + await self._teaql_ensure_business_ids(context) + return await context.execute_graph_save( + lambda session: self._teaql_preflight_and_save(session.context, session), comment=self._comment) async def _teaql_ensure_business_ids(self, context): if self._action != "Create": return - async def _teaql_reserve_graph_ids(self, graph, visited): - context = graph.context + async def _teaql_reserve_graph_ids(self, context, visited): object_identity = id(self) if object_identity in visited: return @@ -111,28 +124,18 @@ async def _teaql_reserve_graph_ids(self, graph, visited): "Configured dataService does not support stable ID reservation" ) old_key = self._teaql_entity_key() - old_loaded = set(self._loaded_fields) self.id = int(await allocator("Shipment")) self._loaded_fields.add("id") self._ledger_id = self.id new_key = self._teaql_entity_key() self._entity_root.rekey(old_key, new_key) self._entity_root.set(new_key, "id", Value.I64(self.id)) - def rollback_reservation(): - self.id = None - self._ledger_id = old_key.id - self._loaded_fields = old_loaded - self._entity_root.rekey(new_key, old_key) - self._entity_root.set(old_key, "id", Value.from_any(None)) - graph.after_rollback(rollback_reservation) - await self._teaql_ensure_business_ids(context) - async def _teaql_preflight_and_save(self, graph): - await self._teaql_reserve_graph_ids(graph, set()) - self._teaql_preflight_graph(graph) - return await self._teaql_save_within_graph(graph) + async def _teaql_preflight_and_save(self, context, session): + self._teaql_preflight_graph(context, session) + return await self._teaql_save_within_graph(context, session) - def _teaql_scalar_payload(self): + def _teaql_build_command(self): payload = {} if "id" in self._loaded_fields: payload["id"] = Value.I64(self.id) @@ -150,10 +153,6 @@ def _teaql_scalar_payload(self): if "version" in self._loaded_fields: payload["version"] = Value.I64(self.version) - return payload - - def _teaql_build_command(self): - payload = self._teaql_scalar_payload() action = self._action if action == "Update": ledger = dict(self._entity_root.current_change_set().changes()).get(self._teaql_entity_key(), {}) @@ -161,22 +160,21 @@ def _teaql_build_command(self): if action == "Create": cmd = InsertCommand("Shipment", payload) elif action == "Update": - original_version = self._entity_root.original_version(self._teaql_entity_key()) - cmd = UpdateCommand("Shipment", Value.from_any(getattr(self, "id", None)), - original_version if original_version is not None else getattr(self, "version", None)) + cmd = UpdateCommand("Shipment", Value.from_any(getattr(self, "id", None)), getattr(self, "version", None)) for key, value in payload.items(): if key not in ("id", "version"): cmd.value(key, value) else: - original_version = self._entity_root.original_version(self._teaql_entity_key()) - cmd = DeleteCommand("Shipment", Value.from_any(getattr(self, "id", None)), - original_version if original_version is not None else getattr(self, "version", None)) + cmd = DeleteCommand("Shipment", Value.from_any(getattr(self, "id", None)), getattr(self, "version", None)) if action != "Create": - cmd.old_values = self._teaql_loaded_snapshot.business_values() + cmd.old_values = LoadedScalarSnapshots.values(self) return action, cmd - def _teaql_preflight_graph(self, graph): - context = graph.context - if self._action != "Update" or self._entity_root.has_pending(self._teaql_entity_key()): + def _teaql_preflight_graph(self, context, session, parent=None): + if parent is None: + MutationIntent(self._comment) + scope = session.scope(self._teaql_entity_key(), parent, self._comment) + session.capture_loaded_values("Shipment", LoadedScalarSnapshots.values(self)) + if self._entity_root.has_pending(self._teaql_entity_key()): if self._action == "Update": if "id" not in self._loaded_fields: raise CheckException([CheckResult("invalid_type", ObjectLocation().property("id"), message="Mutation requires a fully loaded entity")]) @@ -194,19 +192,19 @@ def _teaql_preflight_graph(self, graph): if field not in ("id", "version"): self._entity_root.set(self._teaql_entity_key(), field, value) - async def _teaql_save_within_graph(self, graph, parent_scope=None): - context = graph.context - scope = graph.scope(self._teaql_entity_key(), parent_scope, self._comment) - - if self._action == "Update" and not self._entity_root.has_pending(self._teaql_entity_key()): - await self._teaql_save_children(graph, scope) - return self + async def _teaql_save_within_graph(self, context, session, parent=None): + if parent is None: + MutationIntent(self._comment) self._teaql_attach_root(self._entity_root) + scope = session.scope(self._teaql_entity_key(), parent, self._comment) + if not self._entity_root.has_pending(self._teaql_entity_key()): + await self._teaql_cascade_children(context, session, scope) + return self action, cmd = self._teaql_build_command() - req = graph.request(cmd, scope, self._entity_root, self._teaql_entity_key()) + req = session.request(cmd, scope, self._entity_root, self._teaql_entity_key()) try: context.check_and_fix_mutation(cmd) @@ -223,6 +221,7 @@ async def _teaql_save_within_graph(self, graph, parent_scope=None): "Mutation provider did not return authoritative persisted state for Shipment" ) rollback_payload = {field: getattr(self, field, None) for field in self._loaded_fields | {"id", "version"}} + rollback_snapshot = LoadedScalarSnapshots.values(self) rollback_ledger_id = self._ledger_id rollback_action = self._action rollback_loaded_fields = set(self._loaded_fields) @@ -251,7 +250,6 @@ async def _teaql_save_within_graph(self, graph, parent_scope=None): elif "version" in persisted: self.version = persisted["version"] self._loaded_fields.add("version") - committed_snapshot = _LoadedScalarSnapshot(self._teaql_scalar_payload()) self._ledger_id = getattr(self, "id", self._ledger_id) new_key = self._teaql_entity_key() if old_key != new_key: @@ -262,63 +260,80 @@ def rollback_entity(): self._ledger_id = rollback_ledger_id self._action = rollback_action self._loaded_fields = rollback_loaded_fields + LoadedScalarSnapshots.restore(self, rollback_snapshot) if old_key != new_key: self._entity_root.rekey(new_key, old_key) - graph.after_rollback(rollback_entity) + session.after_rollback(rollback_entity) if action != "Delete": self._action = "Update" if action != "Delete": - await self._teaql_save_children(graph, scope) + await self._teaql_cascade_children(context, session, scope) def commit_entity(): - self._teaql_loaded_snapshot = committed_snapshot + LoadedScalarSnapshots.capture(self, self._teaql_snapshot_values()) self._entity_root.clear_entity(new_key) if getattr(self, "version", None) is not None: self._entity_root.accept_committed_version(new_key, int(self.version)) - graph.after_commit(commit_entity) + session.after_commit(commit_entity) return self - async def _teaql_save_children(self, graph, scope): + async def _teaql_cascade_children(self, context, session, scope): cascade_relations = [] - for relation_name, children, updater, member in cascade_relations: + for relation_name, children, updater in cascade_relations: for index, child in enumerate(children): child._teaql_attach_root(self._entity_root) - current = getattr(child, member, None) - if getattr(current, "id", current) != self.id: - getattr(child, updater)(self) + getattr(child, updater)(self) try: - await child._teaql_save_within_graph(graph, scope) + await child._teaql_save_within_graph(context, session, scope) except CheckException as error: prefix = ObjectLocation().property(relation_name).index(index) raise CheckException([ - CheckResult(v.rule_id, v.location.prefixed_by(prefix), v.input_value, v.system_value, v.message) - for v in error.violations + CheckResult( + violation.rule_id, + violation.location.prefixed_by(prefix), + violation.input_value, + violation.system_value, + violation.message, + ) + for violation in error.violations ]) from error def update_id(self, value): - if not isinstance(value, int) or isinstance(value, bool) or value <= 0: - raise ValueError("id must be a positive integer") + if self._action != "Create": + raise ValueError("ENTITY_ID_IMMUTABLE: an existing entity ID cannot be changed") + value = int(value) if isinstance(value, str) and value.isdecimal() else value + if type(value) is not int or not 0 < value <= 18446744073709551615: + raise ValueError("ENTITY_ID_INVALID: an assigned unsigned-64 identity is required") old_key = self._teaql_entity_key() - new_key = EntityKey(old_key.entity, value) - self._entity_root.rekey(old_key, new_key) self._ledger_id = value + self._entity_root.rekey(old_key, self._teaql_entity_key()) + if "id" in self._loaded_fields and self.id == value: + return self self.id = value self._loaded_fields.add("id") self._entity_root.set(self._teaql_entity_key(), "id", Value.from_any(value)) return self def update_reference_code(self, value): + if "referenceCode" in self._loaded_fields and self.referenceCode == value: + return self self.referenceCode = value self._loaded_fields.add("referenceCode") self._entity_root.set(self._teaql_entity_key(), "reference_code", Value.from_any(value)) return self def update_version(self, value): + if "version" in self._loaded_fields and self.version == value: + return self self.version = value self._loaded_fields.add("version") self._entity_root.set(self._teaql_entity_key(), "version", Value.from_any(value)) return self def update_customer_order(self, value): + assigned_id = getattr(value, "id", value) if value else None + current_id = getattr(self.customerOrder, "id", self.customerOrder) + if "customerOrder" in self._loaded_fields and current_id == assigned_id: + return self self.customerOrder = getattr(value, "id", value) if value else None self._loaded_fields.add("customerOrder") self._entity_root.set(self._teaql_entity_key(), "customer_order", Value.from_any(self.customerOrder)) diff --git a/examples/trace-chain/lib/requests/customer_order_request.py b/examples/trace-chain/lib/requests/customer_order_request.py index 98aa857..35612aa 100644 --- a/examples/trace-chain/lib/requests/customer_order_request.py +++ b/examples/trace-chain/lib/requests/customer_order_request.py @@ -1,7 +1,11 @@ -from teaql.core.query import RelationAggregate, SelectQuery -from teaql.core.list import SmartList, TeaQLPage +from teaql.core.query import AggregateFunction, RelationAggregate, SelectQuery +from teaql.core.request_tree import QueryOperation +from teaql.core.dynamic_search import merge_generated_scalar_search +from generated_search_models import GENERATED_SEARCH_MODELS +from teaql.core.list import SmartList +from teaql.core.request_intent import QueryIntent +from teaql.runtime import EntityRoot from teaql.data_service import QueryRequest -from teaql.core import QueryIntent from copy import deepcopy from teaql.core.expr import ( begin_with, between, column, contain, end_with, eq, gt, gte, @@ -20,6 +24,9 @@ def __init__(self, minimal=False): self.query = SelectQuery("CustomerOrder") self._purpose = None self._comment = None + self._page_count_alias = None + self._page_count_method = None + self._dynamic_search_warnings = [] self.query.and_filter(gte("version", 1)) if minimal: self.select_id() @@ -32,6 +39,17 @@ def comment(self, c: str): self._comment = c return self + def filter_by_json(self, source, warn=None): + kwargs = {} if warn is None else {"warn": warn} + query, warnings = merge_generated_scalar_search(self.query, source, GENERATED_SEARCH_MODELS, **kwargs) + self.query = query + self._dynamic_search_warnings.extend(warnings) + return self + + @property + def dynamic_search_warnings(self): + return tuple(dict(warning) for warning in self._dynamic_search_warnings) + def purpose(self, p: str): self.query.purpose(p) self._purpose = p @@ -96,7 +114,8 @@ def select_version(self): def select_platform_with(self, child_request): self.query.project("platform") - self.query.relation_query("platform", child_request.query) + self.query.relation_query("platform", child_request.query, + QueryOperation("relation-select", "select_platform_with", relation="platform")) return self def with_platform_matching(self, child_request): child_request.query.projection = ["id"] @@ -386,10 +405,14 @@ def order_by_version_descending(self): def count(self): self.query.count_field("id", "count") + self._page_count_alias = "count" + self._page_count_method = "count" return self def count_as(self, ret_name: str): self.query.count_field("id", ret_name) + self._page_count_alias = ret_name + self._page_count_method = "count_as" return self def group_by_id(self): @@ -432,21 +455,24 @@ def select_order_item_list(self): return self.select_order_item_list_with(OrderItemRequest()) def select_order_item_list_with(self, child_request): - self.query.relation_query("order_item_list", child_request.query) + self.query.relation_query("order_item_list", child_request.query, + QueryOperation("relation-select", "select_order_item_list_with", relation="order_item_list")) return self def select_payment_list(self): from requests.payment_request import PaymentRequest return self.select_payment_list_with(PaymentRequest()) def select_payment_list_with(self, child_request): - self.query.relation_query("payment_list", child_request.query) + self.query.relation_query("payment_list", child_request.query, + QueryOperation("relation-select", "select_payment_list_with", relation="payment_list")) return self def select_shipment_list(self): from requests.shipment_request import ShipmentRequest return self.select_shipment_list_with(ShipmentRequest()) def select_shipment_list_with(self, child_request): - self.query.relation_query("shipment_list", child_request.query) + self.query.relation_query("shipment_list", child_request.query, + QueryOperation("relation-select", "select_shipment_list_with", relation="shipment_list")) return self def have_order_items(self): from requests.order_item_request import OrderItemRequest @@ -504,12 +530,18 @@ def count_order_items(self): def count_order_items_as(self, alias: str): from requests.order_item_request import OrderItemRequest - return self.count_order_items_with(alias, OrderItemRequest()) + self.count_order_items_with(alias, OrderItemRequest()) + self.query.relation_aggregates[-1].operation = QueryOperation( + "relation-count", "count_order_items_as", alias, "order_item_list") + return self def count_order_items_with(self, alias: str, child_request): - child_request.query.count_field("id", alias) + query = deepcopy(child_request.query) + query.aggregates = [] + query.count_field("id", alias) self.query.relation_aggregates.append( - RelationAggregate("order_item_list", alias, child_request.query, True) + RelationAggregate("order_item_list", alias, query, True, + QueryOperation("relation-count", "count_order_items_with", alias, "order_item_list")) ) return self @@ -519,12 +551,18 @@ def count_payments(self): def count_payments_as(self, alias: str): from requests.payment_request import PaymentRequest - return self.count_payments_with(alias, PaymentRequest()) + self.count_payments_with(alias, PaymentRequest()) + self.query.relation_aggregates[-1].operation = QueryOperation( + "relation-count", "count_payments_as", alias, "payment_list") + return self def count_payments_with(self, alias: str, child_request): - child_request.query.count_field("id", alias) + query = deepcopy(child_request.query) + query.aggregates = [] + query.count_field("id", alias) self.query.relation_aggregates.append( - RelationAggregate("payment_list", alias, child_request.query, True) + RelationAggregate("payment_list", alias, query, True, + QueryOperation("relation-count", "count_payments_with", alias, "payment_list")) ) return self @@ -534,19 +572,26 @@ def count_shipments(self): def count_shipments_as(self, alias: str): from requests.shipment_request import ShipmentRequest - return self.count_shipments_with(alias, ShipmentRequest()) + self.count_shipments_with(alias, ShipmentRequest()) + self.query.relation_aggregates[-1].operation = QueryOperation( + "relation-count", "count_shipments_as", alias, "shipment_list") + return self def count_shipments_with(self, alias: str, child_request): - child_request.query.count_field("id", alias) + query = deepcopy(child_request.query) + query.aggregates = [] + query.count_field("id", alias) self.query.relation_aggregates.append( - RelationAggregate("shipment_list", alias, child_request.query, True) + RelationAggregate("shipment_list", alias, query, True, + QueryOperation("relation-count", "count_shipments_with", alias, "shipment_list")) ) return self def facet_by_platform_as(self, name: str, request: QuerySelection, include_all_facets: bool = True): - self.query.facet_by(name, "platform", request.query, include_all_facets) + self.query.facet_by(name, "platform", request.query, include_all_facets, + QueryOperation("facet", "facet_by_platform_as", name, "platform")) return self @@ -560,7 +605,8 @@ def comment(self, c: str): def new_entity(self, context) -> CustomerOrder: request = self._request - QueryIntent(request._comment, request._purpose) + if not request._comment or not request._comment.strip() or not request._purpose or not request._purpose.strip(): + raise ValueError("Security audit failure: non-empty comment() and purpose() are required before new_entity()") entity = context.initialize_entity("CustomerOrder", CustomerOrder()) if not isinstance(entity, CustomerOrder): raise TypeError("entity initializer returned an incompatible CustomerOrder") @@ -568,46 +614,28 @@ def new_entity(self, context) -> CustomerOrder: async def execute_for_result(self, context): self = self._request - req = QueryRequest(self.query, _comment=self._comment, _purpose=self._purpose) - req = context.prepare_query_request(req) + if not self._purpose or not self._purpose.strip() or not self._comment or not self._comment.strip(): + raise Exception("Security audit failure: comment() and purpose() must be called before execute_for_rows()") service = context.require_resource("dataService") + req = QueryRequest(context.prepare_query(self.query), _comment=self._comment, _purpose=self._purpose) return await service.query(context, req) async def execute_for_rows(self, context): return (await self.execute_for_result(context)).rows async def execute_for_list(self, context) -> SmartList[CustomerOrder]: - result = await self.execute_for_result(context) - return SmartList( - (CustomerOrder(**row) for row in result.rows), - facets=result.facets) - - async def execute_for_page(self, context, offset: int, limit: int) -> TeaQLPage[CustomerOrder]: + from teaql.runtime.query_list import execute_typed_list request = self._request - intent = QueryIntent(request._comment, request._purpose) + QueryIntent(request._comment, request._purpose) query = deepcopy(request.query) - query.offset(offset).limit(limit) - authorized = context.prepare_query_request(QueryRequest(query, _comment=intent.comment, _purpose=intent.purpose)) - service = context.require_resource("dataService") - alias = "__teaql_total" - if authorized.query.id_set_pagination is not None: - row_result = await service.query(context, authorized) - retained_count, accuracy = context.id_set_count() - if accuracy == "EXACT": - total_count = retained_count - else: - count_result = await service.query(context, authorized.with_query(authorized.query.for_exact_count(alias))) - if not count_result.rows or not isinstance(count_result.rows[0].get(alias), (int, float)): - raise RuntimeError("dataService did not return an exact page count") - total_count = int(count_result.rows[0][alias]) - else: - count_result = await service.query(context, authorized.with_query(authorized.query.for_exact_count(alias))) - if not count_result.rows or not isinstance(count_result.rows[0].get(alias), (int, float)): - raise RuntimeError("dataService did not return an exact page count") - total_count = int(count_result.rows[0][alias]) - row_result = await service.query(context, authorized) - data = SmartList(CustomerOrder(**row) for row in row_result.rows) - return TeaQLPage(data=data, total_count=total_count, offset=offset, limit=limit) + authorized = context.prepare_query(query) + root = QueryRequest(authorized, _comment=request._comment, _purpose=request._purpose) + operation = (QueryOperation("named-count", request._page_count_method or "count", + request._page_count_alias if request._page_count_method == "count_as" else None) + if request._page_count_alias is not None else None) + return await execute_typed_list(context, root, + lambda row: CustomerOrder(_entity_root=EntityRoot(), **row), + count_operation=operation, count_alias=request._page_count_alias) async def execute_for_one(self, context): request = deepcopy(self._request) @@ -617,20 +645,12 @@ async def execute_for_one(self, context): def execute_for_stream(self, context, chunk_size: int = 1000): """Yield entity chunks lazily from the provider cursor.""" - request = self._request - req = QueryRequest(request.query, _comment=request._comment, _purpose=request._purpose) - req = context.prepare_query_request(req) + request = deepcopy(self._request) + QueryIntent(request._comment, request._purpose) service = context.require_resource("dataService") if not hasattr(service, "query_stream"): raise RuntimeError("dataService does not implement query_stream") - stream = service.query_stream(context, req, chunk_size) - async def entities(): - try: - async for chunk in stream: - for row in chunk.rows: - yield CustomerOrder(**row) - finally: - close = getattr(stream, "aclose", None) - if close is not None: - await close() - return entities() + root = QueryRequest(context.prepare_query(request.query), _comment=request._comment, _purpose=request._purpose) + from teaql.runtime.query_stream import execute_typed_stream + return execute_typed_stream(context, root, + lambda row: CustomerOrder(_entity_root=EntityRoot(), **row), chunk_size) diff --git a/examples/trace-chain/lib/requests/order_item_request.py b/examples/trace-chain/lib/requests/order_item_request.py index a1130e4..4c33775 100644 --- a/examples/trace-chain/lib/requests/order_item_request.py +++ b/examples/trace-chain/lib/requests/order_item_request.py @@ -1,7 +1,11 @@ -from teaql.core.query import RelationAggregate, SelectQuery -from teaql.core.list import SmartList, TeaQLPage +from teaql.core.query import AggregateFunction, RelationAggregate, SelectQuery +from teaql.core.request_tree import QueryOperation +from teaql.core.dynamic_search import merge_generated_scalar_search +from generated_search_models import GENERATED_SEARCH_MODELS +from teaql.core.list import SmartList +from teaql.core.request_intent import QueryIntent +from teaql.runtime import EntityRoot from teaql.data_service import QueryRequest -from teaql.core import QueryIntent from copy import deepcopy from teaql.core.expr import ( begin_with, between, column, contain, end_with, eq, gt, gte, @@ -20,6 +24,9 @@ def __init__(self, minimal=False): self.query = SelectQuery("OrderItem") self._purpose = None self._comment = None + self._page_count_alias = None + self._page_count_method = None + self._dynamic_search_warnings = [] self.query.and_filter(gte("version", 1)) if minimal: self.select_id() @@ -32,6 +39,17 @@ def comment(self, c: str): self._comment = c return self + def filter_by_json(self, source, warn=None): + kwargs = {} if warn is None else {"warn": warn} + query, warnings = merge_generated_scalar_search(self.query, source, GENERATED_SEARCH_MODELS, **kwargs) + self.query = query + self._dynamic_search_warnings.extend(warnings) + return self + + @property + def dynamic_search_warnings(self): + return tuple(dict(warning) for warning in self._dynamic_search_warnings) + def purpose(self, p: str): self.query.purpose(p) self._purpose = p @@ -92,7 +110,8 @@ def select_version(self): def select_customer_order_with(self, child_request): self.query.project("customer_order") - self.query.relation_query("customer_order", child_request.query) + self.query.relation_query("customer_order", child_request.query, + QueryOperation("relation-select", "select_customer_order_with", relation="customer_order")) return self def with_customer_order_matching(self, child_request): child_request.query.projection = ["id"] @@ -303,10 +322,14 @@ def order_by_version_descending(self): def count(self): self.query.count_field("id", "count") + self._page_count_alias = "count" + self._page_count_method = "count" return self def count_as(self, ret_name: str): self.query.count_field("id", ret_name) + self._page_count_alias = ret_name + self._page_count_method = "count_as" return self def group_by_id(self): @@ -339,7 +362,8 @@ def group_by_version_as(self, ret_name: str): return self def facet_by_customer_order_as(self, name: str, request: QuerySelection, include_all_facets: bool = True): - self.query.facet_by(name, "customer_order", request.query, include_all_facets) + self.query.facet_by(name, "customer_order", request.query, include_all_facets, + QueryOperation("facet", "facet_by_customer_order_as", name, "customer_order")) return self @@ -353,7 +377,8 @@ def comment(self, c: str): def new_entity(self, context) -> OrderItem: request = self._request - QueryIntent(request._comment, request._purpose) + if not request._comment or not request._comment.strip() or not request._purpose or not request._purpose.strip(): + raise ValueError("Security audit failure: non-empty comment() and purpose() are required before new_entity()") entity = context.initialize_entity("OrderItem", OrderItem()) if not isinstance(entity, OrderItem): raise TypeError("entity initializer returned an incompatible OrderItem") @@ -361,46 +386,28 @@ def new_entity(self, context) -> OrderItem: async def execute_for_result(self, context): self = self._request - req = QueryRequest(self.query, _comment=self._comment, _purpose=self._purpose) - req = context.prepare_query_request(req) + if not self._purpose or not self._purpose.strip() or not self._comment or not self._comment.strip(): + raise Exception("Security audit failure: comment() and purpose() must be called before execute_for_rows()") service = context.require_resource("dataService") + req = QueryRequest(context.prepare_query(self.query), _comment=self._comment, _purpose=self._purpose) return await service.query(context, req) async def execute_for_rows(self, context): return (await self.execute_for_result(context)).rows async def execute_for_list(self, context) -> SmartList[OrderItem]: - result = await self.execute_for_result(context) - return SmartList( - (OrderItem(**row) for row in result.rows), - facets=result.facets) - - async def execute_for_page(self, context, offset: int, limit: int) -> TeaQLPage[OrderItem]: + from teaql.runtime.query_list import execute_typed_list request = self._request - intent = QueryIntent(request._comment, request._purpose) + QueryIntent(request._comment, request._purpose) query = deepcopy(request.query) - query.offset(offset).limit(limit) - authorized = context.prepare_query_request(QueryRequest(query, _comment=intent.comment, _purpose=intent.purpose)) - service = context.require_resource("dataService") - alias = "__teaql_total" - if authorized.query.id_set_pagination is not None: - row_result = await service.query(context, authorized) - retained_count, accuracy = context.id_set_count() - if accuracy == "EXACT": - total_count = retained_count - else: - count_result = await service.query(context, authorized.with_query(authorized.query.for_exact_count(alias))) - if not count_result.rows or not isinstance(count_result.rows[0].get(alias), (int, float)): - raise RuntimeError("dataService did not return an exact page count") - total_count = int(count_result.rows[0][alias]) - else: - count_result = await service.query(context, authorized.with_query(authorized.query.for_exact_count(alias))) - if not count_result.rows or not isinstance(count_result.rows[0].get(alias), (int, float)): - raise RuntimeError("dataService did not return an exact page count") - total_count = int(count_result.rows[0][alias]) - row_result = await service.query(context, authorized) - data = SmartList(OrderItem(**row) for row in row_result.rows) - return TeaQLPage(data=data, total_count=total_count, offset=offset, limit=limit) + authorized = context.prepare_query(query) + root = QueryRequest(authorized, _comment=request._comment, _purpose=request._purpose) + operation = (QueryOperation("named-count", request._page_count_method or "count", + request._page_count_alias if request._page_count_method == "count_as" else None) + if request._page_count_alias is not None else None) + return await execute_typed_list(context, root, + lambda row: OrderItem(_entity_root=EntityRoot(), **row), + count_operation=operation, count_alias=request._page_count_alias) async def execute_for_one(self, context): request = deepcopy(self._request) @@ -410,20 +417,12 @@ async def execute_for_one(self, context): def execute_for_stream(self, context, chunk_size: int = 1000): """Yield entity chunks lazily from the provider cursor.""" - request = self._request - req = QueryRequest(request.query, _comment=request._comment, _purpose=request._purpose) - req = context.prepare_query_request(req) + request = deepcopy(self._request) + QueryIntent(request._comment, request._purpose) service = context.require_resource("dataService") if not hasattr(service, "query_stream"): raise RuntimeError("dataService does not implement query_stream") - stream = service.query_stream(context, req, chunk_size) - async def entities(): - try: - async for chunk in stream: - for row in chunk.rows: - yield OrderItem(**row) - finally: - close = getattr(stream, "aclose", None) - if close is not None: - await close() - return entities() + root = QueryRequest(context.prepare_query(request.query), _comment=request._comment, _purpose=request._purpose) + from teaql.runtime.query_stream import execute_typed_stream + return execute_typed_stream(context, root, + lambda row: OrderItem(_entity_root=EntityRoot(), **row), chunk_size) diff --git a/examples/trace-chain/lib/requests/payment_attempt_request.py b/examples/trace-chain/lib/requests/payment_attempt_request.py index d841ed0..3b23ca3 100644 --- a/examples/trace-chain/lib/requests/payment_attempt_request.py +++ b/examples/trace-chain/lib/requests/payment_attempt_request.py @@ -1,7 +1,11 @@ -from teaql.core.query import RelationAggregate, SelectQuery -from teaql.core.list import SmartList, TeaQLPage +from teaql.core.query import AggregateFunction, RelationAggregate, SelectQuery +from teaql.core.request_tree import QueryOperation +from teaql.core.dynamic_search import merge_generated_scalar_search +from generated_search_models import GENERATED_SEARCH_MODELS +from teaql.core.list import SmartList +from teaql.core.request_intent import QueryIntent +from teaql.runtime import EntityRoot from teaql.data_service import QueryRequest -from teaql.core import QueryIntent from copy import deepcopy from teaql.core.expr import ( begin_with, between, column, contain, end_with, eq, gt, gte, @@ -20,6 +24,9 @@ def __init__(self, minimal=False): self.query = SelectQuery("PaymentAttempt") self._purpose = None self._comment = None + self._page_count_alias = None + self._page_count_method = None + self._dynamic_search_warnings = [] self.query.and_filter(gte("version", 1)) if minimal: self.select_id() @@ -32,6 +39,17 @@ def comment(self, c: str): self._comment = c return self + def filter_by_json(self, source, warn=None): + kwargs = {} if warn is None else {"warn": warn} + query, warnings = merge_generated_scalar_search(self.query, source, GENERATED_SEARCH_MODELS, **kwargs) + self.query = query + self._dynamic_search_warnings.extend(warnings) + return self + + @property + def dynamic_search_warnings(self): + return tuple(dict(warning) for warning in self._dynamic_search_warnings) + def purpose(self, p: str): self.query.purpose(p) self._purpose = p @@ -92,7 +110,8 @@ def select_version(self): def select_payment_with(self, child_request): self.query.project("payment") - self.query.relation_query("payment", child_request.query) + self.query.relation_query("payment", child_request.query, + QueryOperation("relation-select", "select_payment_with", relation="payment")) return self def with_payment_matching(self, child_request): child_request.query.projection = ["id"] @@ -303,10 +322,14 @@ def order_by_version_descending(self): def count(self): self.query.count_field("id", "count") + self._page_count_alias = "count" + self._page_count_method = "count" return self def count_as(self, ret_name: str): self.query.count_field("id", ret_name) + self._page_count_alias = ret_name + self._page_count_method = "count_as" return self def group_by_id(self): @@ -339,7 +362,8 @@ def group_by_version_as(self, ret_name: str): return self def facet_by_payment_as(self, name: str, request: QuerySelection, include_all_facets: bool = True): - self.query.facet_by(name, "payment", request.query, include_all_facets) + self.query.facet_by(name, "payment", request.query, include_all_facets, + QueryOperation("facet", "facet_by_payment_as", name, "payment")) return self @@ -353,7 +377,8 @@ def comment(self, c: str): def new_entity(self, context) -> PaymentAttempt: request = self._request - QueryIntent(request._comment, request._purpose) + if not request._comment or not request._comment.strip() or not request._purpose or not request._purpose.strip(): + raise ValueError("Security audit failure: non-empty comment() and purpose() are required before new_entity()") entity = context.initialize_entity("PaymentAttempt", PaymentAttempt()) if not isinstance(entity, PaymentAttempt): raise TypeError("entity initializer returned an incompatible PaymentAttempt") @@ -361,46 +386,28 @@ def new_entity(self, context) -> PaymentAttempt: async def execute_for_result(self, context): self = self._request - req = QueryRequest(self.query, _comment=self._comment, _purpose=self._purpose) - req = context.prepare_query_request(req) + if not self._purpose or not self._purpose.strip() or not self._comment or not self._comment.strip(): + raise Exception("Security audit failure: comment() and purpose() must be called before execute_for_rows()") service = context.require_resource("dataService") + req = QueryRequest(context.prepare_query(self.query), _comment=self._comment, _purpose=self._purpose) return await service.query(context, req) async def execute_for_rows(self, context): return (await self.execute_for_result(context)).rows async def execute_for_list(self, context) -> SmartList[PaymentAttempt]: - result = await self.execute_for_result(context) - return SmartList( - (PaymentAttempt(**row) for row in result.rows), - facets=result.facets) - - async def execute_for_page(self, context, offset: int, limit: int) -> TeaQLPage[PaymentAttempt]: + from teaql.runtime.query_list import execute_typed_list request = self._request - intent = QueryIntent(request._comment, request._purpose) + QueryIntent(request._comment, request._purpose) query = deepcopy(request.query) - query.offset(offset).limit(limit) - authorized = context.prepare_query_request(QueryRequest(query, _comment=intent.comment, _purpose=intent.purpose)) - service = context.require_resource("dataService") - alias = "__teaql_total" - if authorized.query.id_set_pagination is not None: - row_result = await service.query(context, authorized) - retained_count, accuracy = context.id_set_count() - if accuracy == "EXACT": - total_count = retained_count - else: - count_result = await service.query(context, authorized.with_query(authorized.query.for_exact_count(alias))) - if not count_result.rows or not isinstance(count_result.rows[0].get(alias), (int, float)): - raise RuntimeError("dataService did not return an exact page count") - total_count = int(count_result.rows[0][alias]) - else: - count_result = await service.query(context, authorized.with_query(authorized.query.for_exact_count(alias))) - if not count_result.rows or not isinstance(count_result.rows[0].get(alias), (int, float)): - raise RuntimeError("dataService did not return an exact page count") - total_count = int(count_result.rows[0][alias]) - row_result = await service.query(context, authorized) - data = SmartList(PaymentAttempt(**row) for row in row_result.rows) - return TeaQLPage(data=data, total_count=total_count, offset=offset, limit=limit) + authorized = context.prepare_query(query) + root = QueryRequest(authorized, _comment=request._comment, _purpose=request._purpose) + operation = (QueryOperation("named-count", request._page_count_method or "count", + request._page_count_alias if request._page_count_method == "count_as" else None) + if request._page_count_alias is not None else None) + return await execute_typed_list(context, root, + lambda row: PaymentAttempt(_entity_root=EntityRoot(), **row), + count_operation=operation, count_alias=request._page_count_alias) async def execute_for_one(self, context): request = deepcopy(self._request) @@ -410,20 +417,12 @@ async def execute_for_one(self, context): def execute_for_stream(self, context, chunk_size: int = 1000): """Yield entity chunks lazily from the provider cursor.""" - request = self._request - req = QueryRequest(request.query, _comment=request._comment, _purpose=request._purpose) - req = context.prepare_query_request(req) + request = deepcopy(self._request) + QueryIntent(request._comment, request._purpose) service = context.require_resource("dataService") if not hasattr(service, "query_stream"): raise RuntimeError("dataService does not implement query_stream") - stream = service.query_stream(context, req, chunk_size) - async def entities(): - try: - async for chunk in stream: - for row in chunk.rows: - yield PaymentAttempt(**row) - finally: - close = getattr(stream, "aclose", None) - if close is not None: - await close() - return entities() + root = QueryRequest(context.prepare_query(request.query), _comment=request._comment, _purpose=request._purpose) + from teaql.runtime.query_stream import execute_typed_stream + return execute_typed_stream(context, root, + lambda row: PaymentAttempt(_entity_root=EntityRoot(), **row), chunk_size) diff --git a/examples/trace-chain/lib/requests/payment_request.py b/examples/trace-chain/lib/requests/payment_request.py index 9c63b9b..4573888 100644 --- a/examples/trace-chain/lib/requests/payment_request.py +++ b/examples/trace-chain/lib/requests/payment_request.py @@ -1,7 +1,11 @@ -from teaql.core.query import RelationAggregate, SelectQuery -from teaql.core.list import SmartList, TeaQLPage +from teaql.core.query import AggregateFunction, RelationAggregate, SelectQuery +from teaql.core.request_tree import QueryOperation +from teaql.core.dynamic_search import merge_generated_scalar_search +from generated_search_models import GENERATED_SEARCH_MODELS +from teaql.core.list import SmartList +from teaql.core.request_intent import QueryIntent +from teaql.runtime import EntityRoot from teaql.data_service import QueryRequest -from teaql.core import QueryIntent from copy import deepcopy from teaql.core.expr import ( begin_with, between, column, contain, end_with, eq, gt, gte, @@ -20,6 +24,9 @@ def __init__(self, minimal=False): self.query = SelectQuery("Payment") self._purpose = None self._comment = None + self._page_count_alias = None + self._page_count_method = None + self._dynamic_search_warnings = [] self.query.and_filter(gte("version", 1)) if minimal: self.select_id() @@ -32,6 +39,17 @@ def comment(self, c: str): self._comment = c return self + def filter_by_json(self, source, warn=None): + kwargs = {} if warn is None else {"warn": warn} + query, warnings = merge_generated_scalar_search(self.query, source, GENERATED_SEARCH_MODELS, **kwargs) + self.query = query + self._dynamic_search_warnings.extend(warnings) + return self + + @property + def dynamic_search_warnings(self): + return tuple(dict(warning) for warning in self._dynamic_search_warnings) + def purpose(self, p: str): self.query.purpose(p) self._purpose = p @@ -92,7 +110,8 @@ def select_version(self): def select_customer_order_with(self, child_request): self.query.project("customer_order") - self.query.relation_query("customer_order", child_request.query) + self.query.relation_query("customer_order", child_request.query, + QueryOperation("relation-select", "select_customer_order_with", relation="customer_order")) return self def with_customer_order_matching(self, child_request): child_request.query.projection = ["id"] @@ -303,10 +322,14 @@ def order_by_version_descending(self): def count(self): self.query.count_field("id", "count") + self._page_count_alias = "count" + self._page_count_method = "count" return self def count_as(self, ret_name: str): self.query.count_field("id", ret_name) + self._page_count_alias = ret_name + self._page_count_method = "count_as" return self def group_by_id(self): @@ -342,7 +365,8 @@ def select_payment_attempt_list(self): return self.select_payment_attempt_list_with(PaymentAttemptRequest()) def select_payment_attempt_list_with(self, child_request): - self.query.relation_query("payment_attempt_list", child_request.query) + self.query.relation_query("payment_attempt_list", child_request.query, + QueryOperation("relation-select", "select_payment_attempt_list_with", relation="payment_attempt_list")) return self def have_payment_attempts(self): from requests.payment_attempt_request import PaymentAttemptRequest @@ -366,19 +390,26 @@ def count_payment_attempts(self): def count_payment_attempts_as(self, alias: str): from requests.payment_attempt_request import PaymentAttemptRequest - return self.count_payment_attempts_with(alias, PaymentAttemptRequest()) + self.count_payment_attempts_with(alias, PaymentAttemptRequest()) + self.query.relation_aggregates[-1].operation = QueryOperation( + "relation-count", "count_payment_attempts_as", alias, "payment_attempt_list") + return self def count_payment_attempts_with(self, alias: str, child_request): - child_request.query.count_field("id", alias) + query = deepcopy(child_request.query) + query.aggregates = [] + query.count_field("id", alias) self.query.relation_aggregates.append( - RelationAggregate("payment_attempt_list", alias, child_request.query, True) + RelationAggregate("payment_attempt_list", alias, query, True, + QueryOperation("relation-count", "count_payment_attempts_with", alias, "payment_attempt_list")) ) return self def facet_by_customer_order_as(self, name: str, request: QuerySelection, include_all_facets: bool = True): - self.query.facet_by(name, "customer_order", request.query, include_all_facets) + self.query.facet_by(name, "customer_order", request.query, include_all_facets, + QueryOperation("facet", "facet_by_customer_order_as", name, "customer_order")) return self @@ -392,7 +423,8 @@ def comment(self, c: str): def new_entity(self, context) -> Payment: request = self._request - QueryIntent(request._comment, request._purpose) + if not request._comment or not request._comment.strip() or not request._purpose or not request._purpose.strip(): + raise ValueError("Security audit failure: non-empty comment() and purpose() are required before new_entity()") entity = context.initialize_entity("Payment", Payment()) if not isinstance(entity, Payment): raise TypeError("entity initializer returned an incompatible Payment") @@ -400,46 +432,28 @@ def new_entity(self, context) -> Payment: async def execute_for_result(self, context): self = self._request - req = QueryRequest(self.query, _comment=self._comment, _purpose=self._purpose) - req = context.prepare_query_request(req) + if not self._purpose or not self._purpose.strip() or not self._comment or not self._comment.strip(): + raise Exception("Security audit failure: comment() and purpose() must be called before execute_for_rows()") service = context.require_resource("dataService") + req = QueryRequest(context.prepare_query(self.query), _comment=self._comment, _purpose=self._purpose) return await service.query(context, req) async def execute_for_rows(self, context): return (await self.execute_for_result(context)).rows async def execute_for_list(self, context) -> SmartList[Payment]: - result = await self.execute_for_result(context) - return SmartList( - (Payment(**row) for row in result.rows), - facets=result.facets) - - async def execute_for_page(self, context, offset: int, limit: int) -> TeaQLPage[Payment]: + from teaql.runtime.query_list import execute_typed_list request = self._request - intent = QueryIntent(request._comment, request._purpose) + QueryIntent(request._comment, request._purpose) query = deepcopy(request.query) - query.offset(offset).limit(limit) - authorized = context.prepare_query_request(QueryRequest(query, _comment=intent.comment, _purpose=intent.purpose)) - service = context.require_resource("dataService") - alias = "__teaql_total" - if authorized.query.id_set_pagination is not None: - row_result = await service.query(context, authorized) - retained_count, accuracy = context.id_set_count() - if accuracy == "EXACT": - total_count = retained_count - else: - count_result = await service.query(context, authorized.with_query(authorized.query.for_exact_count(alias))) - if not count_result.rows or not isinstance(count_result.rows[0].get(alias), (int, float)): - raise RuntimeError("dataService did not return an exact page count") - total_count = int(count_result.rows[0][alias]) - else: - count_result = await service.query(context, authorized.with_query(authorized.query.for_exact_count(alias))) - if not count_result.rows or not isinstance(count_result.rows[0].get(alias), (int, float)): - raise RuntimeError("dataService did not return an exact page count") - total_count = int(count_result.rows[0][alias]) - row_result = await service.query(context, authorized) - data = SmartList(Payment(**row) for row in row_result.rows) - return TeaQLPage(data=data, total_count=total_count, offset=offset, limit=limit) + authorized = context.prepare_query(query) + root = QueryRequest(authorized, _comment=request._comment, _purpose=request._purpose) + operation = (QueryOperation("named-count", request._page_count_method or "count", + request._page_count_alias if request._page_count_method == "count_as" else None) + if request._page_count_alias is not None else None) + return await execute_typed_list(context, root, + lambda row: Payment(_entity_root=EntityRoot(), **row), + count_operation=operation, count_alias=request._page_count_alias) async def execute_for_one(self, context): request = deepcopy(self._request) @@ -449,20 +463,12 @@ async def execute_for_one(self, context): def execute_for_stream(self, context, chunk_size: int = 1000): """Yield entity chunks lazily from the provider cursor.""" - request = self._request - req = QueryRequest(request.query, _comment=request._comment, _purpose=request._purpose) - req = context.prepare_query_request(req) + request = deepcopy(self._request) + QueryIntent(request._comment, request._purpose) service = context.require_resource("dataService") if not hasattr(service, "query_stream"): raise RuntimeError("dataService does not implement query_stream") - stream = service.query_stream(context, req, chunk_size) - async def entities(): - try: - async for chunk in stream: - for row in chunk.rows: - yield Payment(**row) - finally: - close = getattr(stream, "aclose", None) - if close is not None: - await close() - return entities() + root = QueryRequest(context.prepare_query(request.query), _comment=request._comment, _purpose=request._purpose) + from teaql.runtime.query_stream import execute_typed_stream + return execute_typed_stream(context, root, + lambda row: Payment(_entity_root=EntityRoot(), **row), chunk_size) diff --git a/examples/trace-chain/lib/requests/platform_request.py b/examples/trace-chain/lib/requests/platform_request.py index 7839df1..4ec0d8f 100644 --- a/examples/trace-chain/lib/requests/platform_request.py +++ b/examples/trace-chain/lib/requests/platform_request.py @@ -1,7 +1,11 @@ -from teaql.core.query import RelationAggregate, SelectQuery -from teaql.core.list import SmartList, TeaQLPage +from teaql.core.query import AggregateFunction, RelationAggregate, SelectQuery +from teaql.core.request_tree import QueryOperation +from teaql.core.dynamic_search import merge_generated_scalar_search +from generated_search_models import GENERATED_SEARCH_MODELS +from teaql.core.list import SmartList +from teaql.core.request_intent import QueryIntent +from teaql.runtime import EntityRoot from teaql.data_service import QueryRequest -from teaql.core import QueryIntent from copy import deepcopy from teaql.core.expr import ( begin_with, between, column, contain, end_with, eq, gt, gte, @@ -20,6 +24,9 @@ def __init__(self, minimal=False): self.query = SelectQuery("Platform") self._purpose = None self._comment = None + self._page_count_alias = None + self._page_count_method = None + self._dynamic_search_warnings = [] self.query.and_filter(gte("version", 1)) if minimal: self.select_id() @@ -32,6 +39,17 @@ def comment(self, c: str): self._comment = c return self + def filter_by_json(self, source, warn=None): + kwargs = {} if warn is None else {"warn": warn} + query, warnings = merge_generated_scalar_search(self.query, source, GENERATED_SEARCH_MODELS, **kwargs) + self.query = query + self._dynamic_search_warnings.extend(warnings) + return self + + @property + def dynamic_search_warnings(self): + return tuple(dict(warning) for warning in self._dynamic_search_warnings) + def purpose(self, p: str): self.query.purpose(p) self._purpose = p @@ -276,10 +294,14 @@ def order_by_version_descending(self): def count(self): self.query.count_field("id", "count") + self._page_count_alias = "count" + self._page_count_method = "count" return self def count_as(self, ret_name: str): self.query.count_field("id", ret_name) + self._page_count_alias = ret_name + self._page_count_method = "count_as" return self def group_by_id(self): @@ -308,7 +330,8 @@ def select_customer_order_list(self): return self.select_customer_order_list_with(CustomerOrderRequest()) def select_customer_order_list_with(self, child_request): - self.query.relation_query("customer_order_list", child_request.query) + self.query.relation_query("customer_order_list", child_request.query, + QueryOperation("relation-select", "select_customer_order_list_with", relation="customer_order_list")) return self def have_customer_orders(self): from requests.customer_order_request import CustomerOrderRequest @@ -332,12 +355,18 @@ def count_customer_orders(self): def count_customer_orders_as(self, alias: str): from requests.customer_order_request import CustomerOrderRequest - return self.count_customer_orders_with(alias, CustomerOrderRequest()) + self.count_customer_orders_with(alias, CustomerOrderRequest()) + self.query.relation_aggregates[-1].operation = QueryOperation( + "relation-count", "count_customer_orders_as", alias, "customer_order_list") + return self def count_customer_orders_with(self, alias: str, child_request): - child_request.query.count_field("id", alias) + query = deepcopy(child_request.query) + query.aggregates = [] + query.count_field("id", alias) self.query.relation_aggregates.append( - RelationAggregate("customer_order_list", alias, child_request.query, True) + RelationAggregate("customer_order_list", alias, query, True, + QueryOperation("relation-count", "count_customer_orders_with", alias, "customer_order_list")) ) return self @@ -353,7 +382,8 @@ def comment(self, c: str): def new_entity(self, context) -> Platform: request = self._request - QueryIntent(request._comment, request._purpose) + if not request._comment or not request._comment.strip() or not request._purpose or not request._purpose.strip(): + raise ValueError("Security audit failure: non-empty comment() and purpose() are required before new_entity()") entity = context.initialize_entity("Platform", Platform()) if not isinstance(entity, Platform): raise TypeError("entity initializer returned an incompatible Platform") @@ -361,46 +391,28 @@ def new_entity(self, context) -> Platform: async def execute_for_result(self, context): self = self._request - req = QueryRequest(self.query, _comment=self._comment, _purpose=self._purpose) - req = context.prepare_query_request(req) + if not self._purpose or not self._purpose.strip() or not self._comment or not self._comment.strip(): + raise Exception("Security audit failure: comment() and purpose() must be called before execute_for_rows()") service = context.require_resource("dataService") + req = QueryRequest(context.prepare_query(self.query), _comment=self._comment, _purpose=self._purpose) return await service.query(context, req) async def execute_for_rows(self, context): return (await self.execute_for_result(context)).rows async def execute_for_list(self, context) -> SmartList[Platform]: - result = await self.execute_for_result(context) - return SmartList( - (Platform(**row) for row in result.rows), - facets=result.facets) - - async def execute_for_page(self, context, offset: int, limit: int) -> TeaQLPage[Platform]: + from teaql.runtime.query_list import execute_typed_list request = self._request - intent = QueryIntent(request._comment, request._purpose) + QueryIntent(request._comment, request._purpose) query = deepcopy(request.query) - query.offset(offset).limit(limit) - authorized = context.prepare_query_request(QueryRequest(query, _comment=intent.comment, _purpose=intent.purpose)) - service = context.require_resource("dataService") - alias = "__teaql_total" - if authorized.query.id_set_pagination is not None: - row_result = await service.query(context, authorized) - retained_count, accuracy = context.id_set_count() - if accuracy == "EXACT": - total_count = retained_count - else: - count_result = await service.query(context, authorized.with_query(authorized.query.for_exact_count(alias))) - if not count_result.rows or not isinstance(count_result.rows[0].get(alias), (int, float)): - raise RuntimeError("dataService did not return an exact page count") - total_count = int(count_result.rows[0][alias]) - else: - count_result = await service.query(context, authorized.with_query(authorized.query.for_exact_count(alias))) - if not count_result.rows or not isinstance(count_result.rows[0].get(alias), (int, float)): - raise RuntimeError("dataService did not return an exact page count") - total_count = int(count_result.rows[0][alias]) - row_result = await service.query(context, authorized) - data = SmartList(Platform(**row) for row in row_result.rows) - return TeaQLPage(data=data, total_count=total_count, offset=offset, limit=limit) + authorized = context.prepare_query(query) + root = QueryRequest(authorized, _comment=request._comment, _purpose=request._purpose) + operation = (QueryOperation("named-count", request._page_count_method or "count", + request._page_count_alias if request._page_count_method == "count_as" else None) + if request._page_count_alias is not None else None) + return await execute_typed_list(context, root, + lambda row: Platform(_entity_root=EntityRoot(), **row), + count_operation=operation, count_alias=request._page_count_alias) async def execute_for_one(self, context): request = deepcopy(self._request) @@ -410,20 +422,12 @@ async def execute_for_one(self, context): def execute_for_stream(self, context, chunk_size: int = 1000): """Yield entity chunks lazily from the provider cursor.""" - request = self._request - req = QueryRequest(request.query, _comment=request._comment, _purpose=request._purpose) - req = context.prepare_query_request(req) + request = deepcopy(self._request) + QueryIntent(request._comment, request._purpose) service = context.require_resource("dataService") if not hasattr(service, "query_stream"): raise RuntimeError("dataService does not implement query_stream") - stream = service.query_stream(context, req, chunk_size) - async def entities(): - try: - async for chunk in stream: - for row in chunk.rows: - yield Platform(**row) - finally: - close = getattr(stream, "aclose", None) - if close is not None: - await close() - return entities() + root = QueryRequest(context.prepare_query(request.query), _comment=request._comment, _purpose=request._purpose) + from teaql.runtime.query_stream import execute_typed_stream + return execute_typed_stream(context, root, + lambda row: Platform(_entity_root=EntityRoot(), **row), chunk_size) diff --git a/examples/trace-chain/lib/requests/shipment_request.py b/examples/trace-chain/lib/requests/shipment_request.py index ceffe65..5207322 100644 --- a/examples/trace-chain/lib/requests/shipment_request.py +++ b/examples/trace-chain/lib/requests/shipment_request.py @@ -1,7 +1,11 @@ -from teaql.core.query import RelationAggregate, SelectQuery -from teaql.core.list import SmartList, TeaQLPage +from teaql.core.query import AggregateFunction, RelationAggregate, SelectQuery +from teaql.core.request_tree import QueryOperation +from teaql.core.dynamic_search import merge_generated_scalar_search +from generated_search_models import GENERATED_SEARCH_MODELS +from teaql.core.list import SmartList +from teaql.core.request_intent import QueryIntent +from teaql.runtime import EntityRoot from teaql.data_service import QueryRequest -from teaql.core import QueryIntent from copy import deepcopy from teaql.core.expr import ( begin_with, between, column, contain, end_with, eq, gt, gte, @@ -20,6 +24,9 @@ def __init__(self, minimal=False): self.query = SelectQuery("Shipment") self._purpose = None self._comment = None + self._page_count_alias = None + self._page_count_method = None + self._dynamic_search_warnings = [] self.query.and_filter(gte("version", 1)) if minimal: self.select_id() @@ -32,6 +39,17 @@ def comment(self, c: str): self._comment = c return self + def filter_by_json(self, source, warn=None): + kwargs = {} if warn is None else {"warn": warn} + query, warnings = merge_generated_scalar_search(self.query, source, GENERATED_SEARCH_MODELS, **kwargs) + self.query = query + self._dynamic_search_warnings.extend(warnings) + return self + + @property + def dynamic_search_warnings(self): + return tuple(dict(warning) for warning in self._dynamic_search_warnings) + def purpose(self, p: str): self.query.purpose(p) self._purpose = p @@ -92,7 +110,8 @@ def select_version(self): def select_customer_order_with(self, child_request): self.query.project("customer_order") - self.query.relation_query("customer_order", child_request.query) + self.query.relation_query("customer_order", child_request.query, + QueryOperation("relation-select", "select_customer_order_with", relation="customer_order")) return self def with_customer_order_matching(self, child_request): child_request.query.projection = ["id"] @@ -303,10 +322,14 @@ def order_by_version_descending(self): def count(self): self.query.count_field("id", "count") + self._page_count_alias = "count" + self._page_count_method = "count" return self def count_as(self, ret_name: str): self.query.count_field("id", ret_name) + self._page_count_alias = ret_name + self._page_count_method = "count_as" return self def group_by_id(self): @@ -339,7 +362,8 @@ def group_by_version_as(self, ret_name: str): return self def facet_by_customer_order_as(self, name: str, request: QuerySelection, include_all_facets: bool = True): - self.query.facet_by(name, "customer_order", request.query, include_all_facets) + self.query.facet_by(name, "customer_order", request.query, include_all_facets, + QueryOperation("facet", "facet_by_customer_order_as", name, "customer_order")) return self @@ -353,7 +377,8 @@ def comment(self, c: str): def new_entity(self, context) -> Shipment: request = self._request - QueryIntent(request._comment, request._purpose) + if not request._comment or not request._comment.strip() or not request._purpose or not request._purpose.strip(): + raise ValueError("Security audit failure: non-empty comment() and purpose() are required before new_entity()") entity = context.initialize_entity("Shipment", Shipment()) if not isinstance(entity, Shipment): raise TypeError("entity initializer returned an incompatible Shipment") @@ -361,46 +386,28 @@ def new_entity(self, context) -> Shipment: async def execute_for_result(self, context): self = self._request - req = QueryRequest(self.query, _comment=self._comment, _purpose=self._purpose) - req = context.prepare_query_request(req) + if not self._purpose or not self._purpose.strip() or not self._comment or not self._comment.strip(): + raise Exception("Security audit failure: comment() and purpose() must be called before execute_for_rows()") service = context.require_resource("dataService") + req = QueryRequest(context.prepare_query(self.query), _comment=self._comment, _purpose=self._purpose) return await service.query(context, req) async def execute_for_rows(self, context): return (await self.execute_for_result(context)).rows async def execute_for_list(self, context) -> SmartList[Shipment]: - result = await self.execute_for_result(context) - return SmartList( - (Shipment(**row) for row in result.rows), - facets=result.facets) - - async def execute_for_page(self, context, offset: int, limit: int) -> TeaQLPage[Shipment]: + from teaql.runtime.query_list import execute_typed_list request = self._request - intent = QueryIntent(request._comment, request._purpose) + QueryIntent(request._comment, request._purpose) query = deepcopy(request.query) - query.offset(offset).limit(limit) - authorized = context.prepare_query_request(QueryRequest(query, _comment=intent.comment, _purpose=intent.purpose)) - service = context.require_resource("dataService") - alias = "__teaql_total" - if authorized.query.id_set_pagination is not None: - row_result = await service.query(context, authorized) - retained_count, accuracy = context.id_set_count() - if accuracy == "EXACT": - total_count = retained_count - else: - count_result = await service.query(context, authorized.with_query(authorized.query.for_exact_count(alias))) - if not count_result.rows or not isinstance(count_result.rows[0].get(alias), (int, float)): - raise RuntimeError("dataService did not return an exact page count") - total_count = int(count_result.rows[0][alias]) - else: - count_result = await service.query(context, authorized.with_query(authorized.query.for_exact_count(alias))) - if not count_result.rows or not isinstance(count_result.rows[0].get(alias), (int, float)): - raise RuntimeError("dataService did not return an exact page count") - total_count = int(count_result.rows[0][alias]) - row_result = await service.query(context, authorized) - data = SmartList(Shipment(**row) for row in row_result.rows) - return TeaQLPage(data=data, total_count=total_count, offset=offset, limit=limit) + authorized = context.prepare_query(query) + root = QueryRequest(authorized, _comment=request._comment, _purpose=request._purpose) + operation = (QueryOperation("named-count", request._page_count_method or "count", + request._page_count_alias if request._page_count_method == "count_as" else None) + if request._page_count_alias is not None else None) + return await execute_typed_list(context, root, + lambda row: Shipment(_entity_root=EntityRoot(), **row), + count_operation=operation, count_alias=request._page_count_alias) async def execute_for_one(self, context): request = deepcopy(self._request) @@ -410,20 +417,12 @@ async def execute_for_one(self, context): def execute_for_stream(self, context, chunk_size: int = 1000): """Yield entity chunks lazily from the provider cursor.""" - request = self._request - req = QueryRequest(request.query, _comment=request._comment, _purpose=request._purpose) - req = context.prepare_query_request(req) + request = deepcopy(self._request) + QueryIntent(request._comment, request._purpose) service = context.require_resource("dataService") if not hasattr(service, "query_stream"): raise RuntimeError("dataService does not implement query_stream") - stream = service.query_stream(context, req, chunk_size) - async def entities(): - try: - async for chunk in stream: - for row in chunk.rows: - yield Shipment(**row) - finally: - close = getattr(stream, "aclose", None) - if close is not None: - await close() - return entities() + root = QueryRequest(context.prepare_query(request.query), _comment=request._comment, _purpose=request._purpose) + from teaql.runtime.query_stream import execute_typed_stream + return execute_typed_stream(context, root, + lambda row: Shipment(_entity_root=EntityRoot(), **row), chunk_size) diff --git a/examples/trace-chain/page_stream.py b/examples/trace-chain/page_stream.py index 3ce877f..f9fc7dd 100644 --- a/examples/trace-chain/page_stream.py +++ b/examples/trace-chain/page_stream.py @@ -119,23 +119,23 @@ async def pages(context, platform, service, sink, policy): await revised.update_description('prior page revision').audit_as('advance paged root').save(context) builder = (Q.customer_orders().with_id_in(*ids).order_by_id_ascending() .select_platform_with(Q.platforms().limit(1)) - .select_order_item_list_with(Q.order_items().limit(10)).limit(3)) + .select_order_item_list_with(Q.order_items().limit(10)).offset(1).limit(2).count_as('totalOrders')) executable = builder.comment('load independent page graphs').purpose('verify page mutation boundaries') entered, release = asyncio.Event(), asyncio.Event() service.count_pause = (entered, release) service.reuse_platform = True clear(service, sink, context, policy) - task = asyncio.create_task(executable.execute_for_page(context, 1, 2)) + task = asyncio.create_task(executable.execute_for_list(context)) await asyncio.wait_for(entered.wait(), 10) builder.with_id_is(9223372036854775807).comment('LATE-PAGE-COMMENT').purpose('LATE-PAGE-PURPOSE') release.set() page = await asyncio.wait_for(task, 10) service.reuse_platform = False - first, second = page.data - assert page.total_count == 3 and [E.customer_order(row).id().eval() for row in page.data] == ids[1:] - assert [E.customer_order(row).version().eval() for row in page.data] == [2, 1] + first, second = page + assert page.total_count == 3 and [E.customer_order(row).id().eval() for row in page] == ids[1:] + assert [E.customer_order(row).version().eval() for row in page] == [2, 1] assert first._entity_root is not second._entity_root and service.snapshot_uses == 2 - assert all(not row._entity_root.has_pending(row._teaql_entity_key()) for row in page.data) + assert all(not row._entity_root.has_pending(row._teaql_entity_key()) for row in page) assert service.requests == [] and policy.plans == [] and sink.events == [] assert all(entry.comment == 'load independent page graphs' and entry.purpose == 'verify page mutation boundaries' for entry in context.sql_logs()) @@ -177,9 +177,9 @@ async def count_privacy(context, service, sink, policy, ids): context.configure_audit_policy('OrderItem', ['name']) clear(service, sink, context, policy) page = await (Q.customer_orders().with_id_in(*ids) - .select_order_item_list_with(Q.order_items().with_name_is(private).limit(1)).limit(3) + .select_order_item_list_with(Q.order_items().with_name_is(private).limit(1)).offset(0).limit(2).count_as('totalOrders') .comment('inspect page referencing ' + private).purpose('count page referencing ' + private) - .execute_for_page(context, 0, 2)) + .execute_for_list(context)) assert page.total_count == 3 logs = context.sql_logs() assert any('COUNT(' in entry.sql.upper() for entry in logs) diff --git a/examples/trace-chain/relation_aggregate.py b/examples/trace-chain/relation_aggregate.py index a02f168..a8f8851 100644 --- a/examples/trace-chain/relation_aggregate.py +++ b/examples/trace-chain/relation_aggregate.py @@ -148,16 +148,12 @@ def assert_unfetched(): assert type(error).__name__ == 'TeaQLNotLoadedError', error else: raise AssertionError('aggregate child detail became loaded-null') - assert loaded.has_query_projection('filtered_count') - assert loaded.query_projection('filtered_count') == 1 - assert loaded.query_projection('save') == 2 and callable(loaded.save) - assert not loaded.has_query_projection('id') - try: - loaded.query_projection('absent') - except KeyError: - pass - else: - raise AssertionError('missing aggregate silently became a value') + assert loaded.get_dynamic_property('filtered_count') == 1 + assert loaded.get_dynamic_property('save') == 2 and callable(loaded.save) + assert loaded.get_dynamic_property('id') is None + # Computed dynamic properties are nullable when absent. They do not + # use fixed-field NotLoaded or the retired projection exception API. + assert loaded.get_dynamic_property('absent') is None assert len(reads) == (7 if nested else 6), reads prefix = [('customer_order', 'Payment.customer_order')] if nested else [] item_route = prefix + [('order_item_list', 'CustomerOrder.order_item_list')] @@ -184,7 +180,7 @@ def assert_unfetched(): else: assert context.sql_logs() == diagnostic == [] print('AGGREGATE_OBSERVED ' + json.dumps({'nested': nested, 'logging': logging, - 'count': loaded.query_projection('filtered_count'), 'provider_calls': len(reads), + 'count': loaded.get_dynamic_property('filtered_count'), 'provider_calls': len(reads), 'sql': [log_row(entry) for entry in context.sql_logs()], 'raw': raw_rows(), 'membership': {'items': 2, 'payments': 1, 'foreign_id': order_id, @@ -195,14 +191,14 @@ def assert_unfetched(): assert len(service.requests) == len(sink.events) == len(policy.plans) == 1 operation = policy.plans[0].operations[0] assert set(operation.changed_values) == {'description'}, operation.changed_values - assert loaded.query_projection('save') == 2 + assert loaded.get_dynamic_property('save') == 2 print('AGGREGATE_SAVE ' + json.dumps({'nested': nested, 'logging': logging, 'commands': len(service.requests), 'audits': len(sink.events), - 'fields': sorted(operation.changed_values), 'projection': loaded.query_projection('save')})) + 'fields': sorted(operation.changed_values), 'projection': loaded.get_dynamic_property('save')})) fresh = (await query(nested, 'NO-MATCH-' + uuid.uuid4().hex).execute_for_list(context))[0] fresh = E.payment(fresh).customer_order().eval() if nested else fresh assert E.customer_order(fresh).description().eval() == E.customer_order(loaded).description().eval() - assert fresh.has_query_projection('filtered_count') and fresh.query_projection('filtered_count') == 0 + assert fresh.get_dynamic_property('filtered_count') == 0 clear() fault[0] = True diff --git a/examples/trace-chain/standard-order/lib/E.py b/examples/trace-chain/standard-order/lib/E.py new file mode 100644 index 0000000..b565286 --- /dev/null +++ b/examples/trace-chain/standard-order/lib/E.py @@ -0,0 +1,244 @@ +class TeaQLNotLoadedError(RuntimeError): + def __init__(self, root, access_path, break_point): + self.root = root + self.access_path = access_path + self.break_point = break_point + super().__init__( + f"TeaQLNotLoadedError: root={root} access_path={access_path} " + f"break_point={break_point} suggested_fix=select_{break_point}(...)" + ) + + +class ValueExpression: + def __init__(self, value=None, error=None): + self._value = value + self._error = error + + def eval(self): + if self._error is not None: + raise self._error + return self._value + + def or_if_null(self, fallback): + value = self.eval() + return fallback if value is None else value + + +class EntityExpression: + def __init__(self, value, root=None, path="", error=None): + self._value = value + self._root = root or f"{type(value).__name__ if value is not None else 'Entity'}(null)" + self._path = path + self._error = error + + def eval(self): + if self._error is not None: + raise self._error + return self._value + + def _path_for(self, field): + return f"{self._path}.{field}" if self._path else field + + def _not_loaded(self, field): + path = self._path_for(field) + return TeaQLNotLoadedError(self._root, path, field) + + def _scalar(self, field, relation_id=False): + if self._error is not None: + return ValueExpression(error=self._error) + if self._value is None: + return ValueExpression(None) + if field not in getattr(self._value, "_loaded_fields", set()): + return ValueExpression(error=self._not_loaded(field)) + value = getattr(self._value, field) + if relation_id and value is not None and not isinstance(value, (int, str)): + value = getattr(value, "id", None) + return ValueExpression(value) + + def _relation(self, field, expression_type): + path = self._path_for(field) + if self._error is not None: + return expression_type(None, self._root, path, self._error) + if self._value is None: + return expression_type(None, self._root, path) + if field not in getattr(self._value, "_loaded_fields", set()): + return expression_type(None, self._root, path, self._not_loaded(field)) + value = getattr(self._value, field) + if value is not None and isinstance(value, (int, str)): + return expression_type(None, self._root, path, self._not_loaded(field)) + return expression_type(value, self._root, path) + + +class ListExpression: + def __init__(self, values, root, path, item_expression, error=None): + self._values = values + self._root = root + self._path = path + self._item_expression = item_expression + self._error = error + + def size(self): + return ValueExpression(error=self._error) if self._error else ValueExpression(len(self._values)) + + def first(self): + return self.get(0) + + def get(self, index): + path = f"{self._path}.get({index})" + if self._error is not None: + return self._item_expression(None, self._root, path, self._error) + value = self._values[index] if 0 <= index < len(self._values) else None + return self._item_expression(value, self._root, path) + + +class PlatformExpression(EntityExpression): + def id(self): + return self._scalar("id") + def name(self): + return self._scalar("name") + def version(self): + return self._scalar("version") + def order_status_list(self): + path = self._path_for("order_status_list") + if self._error is not None: + return ListExpression([], self._root, path, OrderStatusExpression, self._error) + if self._value is None: + return ListExpression([], self._root, path, OrderStatusExpression) + if "order_status_list" not in getattr(self._value, "_loaded_fields", set()): + return ListExpression([], self._root, path, OrderStatusExpression, self._not_loaded("order_status_list")) + return ListExpression(getattr(self._value, "_order_status_list"), self._root, path, OrderStatusExpression) + def customer_list(self): + path = self._path_for("customer_list") + if self._error is not None: + return ListExpression([], self._root, path, CustomerExpression, self._error) + if self._value is None: + return ListExpression([], self._root, path, CustomerExpression) + if "customer_list" not in getattr(self._value, "_loaded_fields", set()): + return ListExpression([], self._root, path, CustomerExpression, self._not_loaded("customer_list")) + return ListExpression(getattr(self._value, "_customer_list"), self._root, path, CustomerExpression) + pass + +class OrderStatusExpression(EntityExpression): + def id(self): + return self._scalar("id") + def name(self): + return self._scalar("name") + def code(self): + return self._scalar("code") + def color(self): + return self._scalar("color") + def display_order(self): + return self._scalar("displayOrder") + def version(self): + return self._scalar("version") + def platform_id(self): + return self._scalar("platform", relation_id=True) + + def platform(self): + return self._relation("platform", PlatformExpression) + def customer_order_list(self): + path = self._path_for("customer_order_list") + if self._error is not None: + return ListExpression([], self._root, path, CustomerOrderExpression, self._error) + if self._value is None: + return ListExpression([], self._root, path, CustomerOrderExpression) + if "customer_order_list" not in getattr(self._value, "_loaded_fields", set()): + return ListExpression([], self._root, path, CustomerOrderExpression, self._not_loaded("customer_order_list")) + return ListExpression(getattr(self._value, "_customer_order_list"), self._root, path, CustomerOrderExpression) + pass + +class CustomerExpression(EntityExpression): + def id(self): + return self._scalar("id") + def name(self): + return self._scalar("name") + def version(self): + return self._scalar("version") + def platform_id(self): + return self._scalar("platform", relation_id=True) + + def platform(self): + return self._relation("platform", PlatformExpression) + def customer_order_list(self): + path = self._path_for("customer_order_list") + if self._error is not None: + return ListExpression([], self._root, path, CustomerOrderExpression, self._error) + if self._value is None: + return ListExpression([], self._root, path, CustomerOrderExpression) + if "customer_order_list" not in getattr(self._value, "_loaded_fields", set()): + return ListExpression([], self._root, path, CustomerOrderExpression, self._not_loaded("customer_order_list")) + return ListExpression(getattr(self._value, "_customer_order_list"), self._root, path, CustomerOrderExpression) + pass + +class CustomerOrderExpression(EntityExpression): + def id(self): + return self._scalar("id") + def order_number(self): + return self._scalar("orderNumber") + def order_date(self): + return self._scalar("orderDate") + def total_amount(self): + return self._scalar("totalAmount") + def version(self): + return self._scalar("version") + def customer_id(self): + return self._scalar("customer", relation_id=True) + + def customer(self): + return self._relation("customer", CustomerExpression) + def status_id(self): + return self._scalar("status", relation_id=True) + + def status(self): + return self._relation("status", OrderStatusExpression) + def order_item_list(self): + path = self._path_for("order_item_list") + if self._error is not None: + return ListExpression([], self._root, path, OrderItemExpression, self._error) + if self._value is None: + return ListExpression([], self._root, path, OrderItemExpression) + if "order_item_list" not in getattr(self._value, "_loaded_fields", set()): + return ListExpression([], self._root, path, OrderItemExpression, self._not_loaded("order_item_list")) + return ListExpression(getattr(self._value, "_order_item_list"), self._root, path, OrderItemExpression) + pass + +class OrderItemExpression(EntityExpression): + def id(self): + return self._scalar("id") + def product_name(self): + return self._scalar("productName") + def quantity(self): + return self._scalar("quantity") + def unit_price(self): + return self._scalar("unitPrice") + def version(self): + return self._scalar("version") + def customer_order_id(self): + return self._scalar("customerOrder", relation_id=True) + + def customer_order(self): + return self._relation("customerOrder", CustomerOrderExpression) + pass + +class E: + @staticmethod + def platform(value): + entity_id = getattr(value, "id", None) + return PlatformExpression(value, "Platform(id={})".format(entity_id)) + @staticmethod + def order_status(value): + entity_id = getattr(value, "id", None) + return OrderStatusExpression(value, "OrderStatus(id={})".format(entity_id)) + @staticmethod + def customer(value): + entity_id = getattr(value, "id", None) + return CustomerExpression(value, "Customer(id={})".format(entity_id)) + @staticmethod + def customer_order(value): + entity_id = getattr(value, "id", None) + return CustomerOrderExpression(value, "CustomerOrder(id={})".format(entity_id)) + @staticmethod + def order_item(value): + entity_id = getattr(value, "id", None) + return OrderItemExpression(value, "OrderItem(id={})".format(entity_id)) + pass \ No newline at end of file diff --git a/examples/trace-chain/standard-order/lib/Q.py b/examples/trace-chain/standard-order/lib/Q.py new file mode 100644 index 0000000..5c3abba --- /dev/null +++ b/examples/trace-chain/standard-order/lib/Q.py @@ -0,0 +1,49 @@ +# Generated by teaql-code-gen +from requests.platform_request import PlatformRequest +from requests.order_status_request import OrderStatusRequest +from requests.customer_request import CustomerRequest +from requests.customer_order_request import CustomerOrderRequest +from requests.order_item_request import OrderItemRequest + +class Q: + @staticmethod + def platforms() -> PlatformRequest: + return PlatformRequest(minimal=False) + + @staticmethod + def platforms_minimal() -> PlatformRequest: + return PlatformRequest(minimal=True) + + @staticmethod + def order_statuses() -> OrderStatusRequest: + return OrderStatusRequest(minimal=False) + + @staticmethod + def order_statuses_minimal() -> OrderStatusRequest: + return OrderStatusRequest(minimal=True) + + @staticmethod + def customers() -> CustomerRequest: + return CustomerRequest(minimal=False) + + @staticmethod + def customers_minimal() -> CustomerRequest: + return CustomerRequest(minimal=True) + + @staticmethod + def customer_orders() -> CustomerOrderRequest: + return CustomerOrderRequest(minimal=False) + + @staticmethod + def customer_orders_minimal() -> CustomerOrderRequest: + return CustomerOrderRequest(minimal=True) + + @staticmethod + def order_items() -> OrderItemRequest: + return OrderItemRequest(minimal=False) + + @staticmethod + def order_items_minimal() -> OrderItemRequest: + return OrderItemRequest(minimal=True) + + diff --git a/examples/trace-chain/standard-order/lib/generated_search_models.py b/examples/trace-chain/standard-order/lib/generated_search_models.py new file mode 100644 index 0000000..e928e59 --- /dev/null +++ b/examples/trace-chain/standard-order/lib/generated_search_models.py @@ -0,0 +1,124 @@ +from types import MappingProxyType + +# Generated trusted field/type metadata. Runtime owns parsing, binding and warnings. +GENERATED_SEARCH_MODELS = MappingProxyType({ + "Platform": MappingProxyType({ + "fields": MappingProxyType({ + "id": "integer", + "name": "string", + "version": "integer", + }), + "query_fields": MappingProxyType({ + "id": "id", + "name": "name", + "version": "version", + }), + "relations": MappingProxyType({ + "orderStatusList": "OrderStatus", + "customerList": "Customer", + }), + }), + "OrderStatus": MappingProxyType({ + "fields": MappingProxyType({ + "platform": "integer", + "id": "integer", + "name": "string", + "code": "string", + "color": "string", + "display_order": "integer", + "displayOrder": "integer", + "version": "integer", + }), + "query_fields": MappingProxyType({ + "platform": "platform", + "id": "id", + "name": "name", + "code": "code", + "color": "color", + "display_order": "display_order", + "displayOrder": "display_order", + "version": "version", + }), + "relations": MappingProxyType({ + "platform": "Platform", + "customerOrderList": "CustomerOrder", + }), + }), + "Customer": MappingProxyType({ + "fields": MappingProxyType({ + "id": "integer", + "platform": "integer", + "name": "string", + "version": "integer", + }), + "query_fields": MappingProxyType({ + "id": "id", + "platform": "platform", + "name": "name", + "version": "version", + }), + "relations": MappingProxyType({ + "platform": "Platform", + "customerOrderList": "CustomerOrder", + }), + }), + "CustomerOrder": MappingProxyType({ + "fields": MappingProxyType({ + "id": "integer", + "customer": "integer", + "status": "integer", + "order_number": "string", + "orderNumber": "string", + "order_date": "date", + "orderDate": "date", + "total_amount": "decimal", + "totalAmount": "decimal", + "version": "integer", + }), + "query_fields": MappingProxyType({ + "id": "id", + "customer": "customer", + "status": "status", + "order_number": "order_number", + "orderNumber": "order_number", + "order_date": "order_date", + "orderDate": "order_date", + "total_amount": "total_amount", + "totalAmount": "total_amount", + "version": "version", + }), + "relations": MappingProxyType({ + "customer": "Customer", + "status": "OrderStatus", + "orderItemList": "OrderItem", + }), + }), + "OrderItem": MappingProxyType({ + "fields": MappingProxyType({ + "id": "integer", + "customer_order": "integer", + "customerOrder": "integer", + "product_name": "string", + "productName": "string", + "quantity": "integer", + "unit_price": "decimal", + "unitPrice": "decimal", + "version": "integer", + }), + "query_fields": MappingProxyType({ + "id": "id", + "customer_order": "customer_order", + "customerOrder": "customer_order", + "product_name": "product_name", + "productName": "product_name", + "quantity": "quantity", + "unit_price": "unit_price", + "unitPrice": "unit_price", + "version": "version", + }), + "relations": MappingProxyType({ + "customer_order": "CustomerOrder", + "customerOrder": "CustomerOrder", + }), + }), +}) \ No newline at end of file diff --git a/examples/trace-chain/standard-order/lib/models/__init__.py b/examples/trace-chain/standard-order/lib/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/examples/trace-chain/standard-order/lib/models/customer.py b/examples/trace-chain/standard-order/lib/models/customer.py new file mode 100644 index 0000000..2c3f9a6 --- /dev/null +++ b/examples/trace-chain/standard-order/lib/models/customer.py @@ -0,0 +1,370 @@ +from teaql.core.mutation import InsertCommand, UpdateCommand, DeleteCommand, MutationRequest +from teaql.core.value import Value +from teaql.core.request_intent import MutationIntent +from teaql.core.projection_properties import ProjectionProperties +from teaql.runtime.loaded_scalar_snapshot import LoadedScalarSnapshots +from teaql.runtime import CheckException, CheckResult, EntityKey, EntityRoot, ObjectLocation +import itertools +from models.platform import Platform + + +class Customer: + _teaql_temporary_ids = itertools.count(1) + @classmethod + def refer(cls, entity_id): + return cls(id=entity_id) + + @classmethod + def _teaql_new_with_fixed_id(cls, entity_id): + """Generated bootstrap capability; application code must not call it.""" + return cls(id=entity_id)._teaql_force_create() + + def _teaql_force_create(self): + self._action = "Create" + self._entity_root.mark_as_new(self._teaql_entity_key()) + return self + + def __init__(self, **kwargs): + self._entity_root = kwargs.pop("_entity_root", None) or EntityRoot() + if "id" in kwargs and "id" not in kwargs: + kwargs["id"] = kwargs.pop("id") + if "platform" in kwargs and "platform" not in kwargs: + kwargs["platform"] = kwargs.pop("platform") + if "name" in kwargs and "name" not in kwargs: + kwargs["name"] = kwargs.pop("name") + if "version" in kwargs and "version" not in kwargs: + kwargs["version"] = kwargs.pop("version") + self._projection_properties = ProjectionProperties(kwargs, + ["id", "platform", "name", "version", "customer_order_list"]) + self._action = "Update" if kwargs.get("id") else "Create" + self._comment = None + self._loaded_fields = set(kwargs.keys()) + self.id = kwargs.get("id") + + self.platform = kwargs.get("platform") + + self.name = kwargs.get("name") + + self.version = kwargs.get("version") + + if isinstance(self.platform, dict): + self.platform = Platform(**self.platform) + self._customer_order_list = kwargs.get("customer_order_list", []) + if "customer_order_list" in kwargs or kwargs.get("id") is None: + self._loaded_fields.add("customer_order_list") + if self._customer_order_list: + from models.customer_order import CustomerOrder + self._customer_order_list = [ + item if isinstance(item, CustomerOrder) else CustomerOrder(**item) + for item in self._customer_order_list + ] + self._ledger_id = getattr(self, "id", None) + if self._ledger_id is None: + self._ledger_id = -next(self._teaql_temporary_ids) + key = self._teaql_entity_key() + if self._action == "Create": + self._entity_root.mark_as_new(key) + elif getattr(self, "version", None) is not None: + self._entity_root.set_original_version(key, int(self.version)) + LoadedScalarSnapshots.capture(self, self._teaql_snapshot_values()) + + def _teaql_snapshot_values(self): + payload = {} + if "id" in self._loaded_fields: + payload["id"] = Value.I64(self.id) + + if "platform" in self._loaded_fields: + payload["platform"] = Value.I64(getattr(self.platform, "id", self.platform)) + + if "name" in self._loaded_fields: + payload["name"] = Value.Text(self.name) + + if "version" in self._loaded_fields: + payload["version"] = Value.I64(self.version) + + return payload + + def _teaql_entity_key(self): + return EntityKey("Customer", self._ledger_id) + + def get_dynamic_property(self, name): + return self._projection_properties.get(name) + + def _teaql_attach_root(self, root): + if self._entity_root is not root: + root.merge_entity_from(self._entity_root, self._teaql_entity_key()) + self._entity_root = root + for child in self._customer_order_list: + child._teaql_attach_root(root) + return self + def mark_for_deletion(self): + self._action = "Delete" + self._entity_root.mark_as_deleted(self._teaql_entity_key()) + return self + + def audit_as(self, comment: str): + MutationIntent(comment) + self._comment = comment + return self + + async def save(self, context): + # Validate the root reason before ID allocation or provider access. + MutationIntent(self._comment) + # Reserve stable internal IDs before the transaction and before policy + # review. A relation to a new parent must have the same identity in the + # reviewed graph plan and in the commands eventually sent to SQL. + await self._teaql_reserve_graph_ids(context, set()) + await self._teaql_ensure_business_ids(context) + return await context.execute_graph_save( + lambda session: self._teaql_preflight_and_save(session.context, session), comment=self._comment) + + async def _teaql_ensure_business_ids(self, context): + if self._action != "Create": + return + + async def _teaql_reserve_graph_ids(self, context, visited): + object_identity = id(self) + if object_identity in visited: + return + visited.add(object_identity) + if self._action == "Create" and getattr(self, "id", None) is None: + service = context.require_resource("dataService") + allocator = getattr(service, "next_id", None) + if not callable(allocator): + raise RuntimeError( + "Configured dataService does not support stable ID reservation" + ) + old_key = self._teaql_entity_key() + self.id = int(await allocator("Customer")) + self._loaded_fields.add("id") + self._ledger_id = self.id + new_key = self._teaql_entity_key() + self._entity_root.rekey(old_key, new_key) + self._entity_root.set(new_key, "id", Value.I64(self.id)) + for child in self._customer_order_list: + child._teaql_attach_root(self._entity_root) + await child._teaql_reserve_graph_ids(context, visited) + + async def _teaql_preflight_and_save(self, context, session): + self._teaql_preflight_graph(context, session) + return await self._teaql_save_within_graph(context, session) + + def _teaql_build_command(self): + payload = {} + if "id" in self._loaded_fields: + payload["id"] = Value.I64(self.id) + + if "platform" in self._loaded_fields: + reference = self.platform + reference_id = getattr(reference, "id", reference) + payload["platform"] = (Value.Object(reference) + if reference is not None and hasattr(reference, "id") and reference_id is None + else Value.I64(reference_id)) + + if "name" in self._loaded_fields: + payload["name"] = Value.Text(self.name) + + if "version" in self._loaded_fields: + payload["version"] = Value.I64(self.version) + + action = self._action + if action == "Update": + ledger = dict(self._entity_root.current_change_set().changes()).get(self._teaql_entity_key(), {}) + payload = {field: value for field, value in ledger.items() if field not in ("id", "version")} + if action == "Create": + cmd = InsertCommand("Customer", payload) + elif action == "Update": + cmd = UpdateCommand("Customer", Value.from_any(getattr(self, "id", None)), getattr(self, "version", None)) + for key, value in payload.items(): + if key not in ("id", "version"): cmd.value(key, value) + else: + cmd = DeleteCommand("Customer", Value.from_any(getattr(self, "id", None)), getattr(self, "version", None)) + if action != "Create": + cmd.old_values = LoadedScalarSnapshots.values(self) + return action, cmd + + def _teaql_preflight_graph(self, context, session, parent=None): + if parent is None: + MutationIntent(self._comment) + scope = session.scope(self._teaql_entity_key(), parent, self._comment) + session.capture_loaded_values("Customer", LoadedScalarSnapshots.values(self)) + if self._entity_root.has_pending(self._teaql_entity_key()): + if self._action == "Update": + if "id" not in self._loaded_fields: + raise CheckException([CheckResult("invalid_type", ObjectLocation().property("id"), message="Mutation requires a fully loaded entity")]) + if "platform" not in self._loaded_fields: + raise CheckException([CheckResult("invalid_type", ObjectLocation().property("platform"), message="Mutation requires a fully loaded entity")]) + if "name" not in self._loaded_fields: + raise CheckException([CheckResult("invalid_type", ObjectLocation().property("name"), message="Mutation requires a fully loaded entity")]) + if "version" not in self._loaded_fields: + raise CheckException([CheckResult("invalid_type", ObjectLocation().property("version"), message="Mutation requires a fully loaded entity")]) + _action, cmd = self._teaql_build_command() + try: + context.preflight_mutation(cmd) + finally: + for field, value in getattr(cmd, "values", {}).items(): + if field not in ("id", "version"): + self._entity_root.set(self._teaql_entity_key(), field, value) + for index, child in enumerate(self._customer_order_list): + child._teaql_attach_root(self._entity_root) + child.update_customer(self) + try: + child._teaql_preflight_graph(context, session, scope) + except CheckException as error: + prefix = ObjectLocation().property("customer_order_list").index(index) + raise CheckException([ + CheckResult(v.rule_id, v.location.prefixed_by(prefix), v.input_value, v.system_value, v.message) + for v in error.violations + ]) from error + + async def _teaql_save_within_graph(self, context, session, parent=None): + if parent is None: + MutationIntent(self._comment) + + self._teaql_attach_root(self._entity_root) + scope = session.scope(self._teaql_entity_key(), parent, self._comment) + if not self._entity_root.has_pending(self._teaql_entity_key()): + await self._teaql_cascade_children(context, session, scope) + return self + action, cmd = self._teaql_build_command() + + + req = session.request(cmd, scope, self._entity_root, self._teaql_entity_key()) + + try: + context.check_and_fix_mutation(cmd) + finally: + for field, value in getattr(cmd, "values", {}).items(): + if field not in ("id", "version"): + self._entity_root.set(self._teaql_entity_key(), field, value) + context.mark_mutation_checked(cmd) + service = context.require_resource("dataService") + result = await service.mutate(context, req) + persisted = result.persisted_record + if persisted is None: + raise RuntimeError( + "Mutation provider did not return authoritative persisted state for Customer" + ) + rollback_payload = {field: getattr(self, field, None) for field in self._loaded_fields | {"id", "version"}} + rollback_snapshot = LoadedScalarSnapshots.values(self) + rollback_ledger_id = self._ledger_id + rollback_action = self._action + rollback_loaded_fields = set(self._loaded_fields) + old_key = self._teaql_entity_key() + if "id" in persisted: + self.id = persisted["id"] + self._loaded_fields.add("id") + elif "id" in persisted: + self.id = persisted["id"] + self._loaded_fields.add("id") + if "platform" in persisted: + self.platform = persisted["platform"] + self._loaded_fields.add("platform") + elif "platform" in persisted: + self.platform = persisted["platform"] + self._loaded_fields.add("platform") + if "name" in persisted: + self.name = persisted["name"] + self._loaded_fields.add("name") + elif "name" in persisted: + self.name = persisted["name"] + self._loaded_fields.add("name") + if "version" in persisted: + self.version = persisted["version"] + self._loaded_fields.add("version") + elif "version" in persisted: + self.version = persisted["version"] + self._loaded_fields.add("version") + self._ledger_id = getattr(self, "id", self._ledger_id) + new_key = self._teaql_entity_key() + if old_key != new_key: + self._entity_root.rekey(old_key, new_key) + def rollback_entity(): + for field, value in rollback_payload.items(): + setattr(self, field, value) + self._ledger_id = rollback_ledger_id + self._action = rollback_action + self._loaded_fields = rollback_loaded_fields + LoadedScalarSnapshots.restore(self, rollback_snapshot) + if old_key != new_key: + self._entity_root.rekey(new_key, old_key) + session.after_rollback(rollback_entity) + if action != "Delete": + self._action = "Update" + + if action != "Delete": + await self._teaql_cascade_children(context, session, scope) + def commit_entity(): + LoadedScalarSnapshots.capture(self, self._teaql_snapshot_values()) + self._entity_root.clear_entity(new_key) + if getattr(self, "version", None) is not None: + self._entity_root.accept_committed_version(new_key, int(self.version)) + session.after_commit(commit_entity) + return self + + async def _teaql_cascade_children(self, context, session, scope): + cascade_relations = [] + cascade_relations.append(("customer_order_list", self._customer_order_list, "update_customer")) + for relation_name, children, updater in cascade_relations: + for index, child in enumerate(children): + child._teaql_attach_root(self._entity_root) + getattr(child, updater)(self) + try: + await child._teaql_save_within_graph(context, session, scope) + except CheckException as error: + prefix = ObjectLocation().property(relation_name).index(index) + raise CheckException([ + CheckResult( + violation.rule_id, + violation.location.prefixed_by(prefix), + violation.input_value, + violation.system_value, + violation.message, + ) + for violation in error.violations + ]) from error + + def update_id(self, value): + if self._action != "Create": + raise ValueError("ENTITY_ID_IMMUTABLE: an existing entity ID cannot be changed") + value = int(value) if isinstance(value, str) and value.isdecimal() else value + if type(value) is not int or not 0 < value <= 18446744073709551615: + raise ValueError("ENTITY_ID_INVALID: an assigned unsigned-64 identity is required") + old_key = self._teaql_entity_key() + self._ledger_id = value + self._entity_root.rekey(old_key, self._teaql_entity_key()) + if "id" in self._loaded_fields and self.id == value: + return self + self.id = value + self._loaded_fields.add("id") + self._entity_root.set(self._teaql_entity_key(), "id", Value.from_any(value)) + return self + + def update_name(self, value): + if "name" in self._loaded_fields and self.name == value: + return self + self.name = value + self._loaded_fields.add("name") + self._entity_root.set(self._teaql_entity_key(), "name", Value.from_any(value)) + return self + + def update_version(self, value): + if "version" in self._loaded_fields and self.version == value: + return self + self.version = value + self._loaded_fields.add("version") + self._entity_root.set(self._teaql_entity_key(), "version", Value.from_any(value)) + return self + def update_platform(self, value): + assigned_id = getattr(value, "id", value) if value else None + current_id = getattr(self.platform, "id", self.platform) + if "platform" in self._loaded_fields and current_id == assigned_id: + return self + self.platform = getattr(value, "id", value) if value else None + self._loaded_fields.add("platform") + self._entity_root.set(self._teaql_entity_key(), "platform", Value.from_any(self.platform)) + return self + + def customer_order_list(self) -> list: + self._loaded_fields.add("customer_order_list") + return self._customer_order_list diff --git a/examples/trace-chain/standard-order/lib/models/customer_order.py b/examples/trace-chain/standard-order/lib/models/customer_order.py new file mode 100644 index 0000000..7420cad --- /dev/null +++ b/examples/trace-chain/standard-order/lib/models/customer_order.py @@ -0,0 +1,473 @@ +from teaql.core.mutation import InsertCommand, UpdateCommand, DeleteCommand, MutationRequest +from teaql.core.value import Value +from teaql.core.request_intent import MutationIntent +from teaql.core.projection_properties import ProjectionProperties +from teaql.runtime.loaded_scalar_snapshot import LoadedScalarSnapshots +from teaql.runtime import CheckException, CheckResult, EntityKey, EntityRoot, ObjectLocation +import itertools +from models.customer import Customer +from models.order_status import OrderStatus + + +class CustomerOrder: + _teaql_temporary_ids = itertools.count(1) + @classmethod + def refer(cls, entity_id): + return cls(id=entity_id) + + @classmethod + def _teaql_new_with_fixed_id(cls, entity_id): + """Generated bootstrap capability; application code must not call it.""" + return cls(id=entity_id)._teaql_force_create() + + def _teaql_force_create(self): + self._action = "Create" + self._entity_root.mark_as_new(self._teaql_entity_key()) + return self + + def __init__(self, **kwargs): + self._entity_root = kwargs.pop("_entity_root", None) or EntityRoot() + if "id" in kwargs and "id" not in kwargs: + kwargs["id"] = kwargs.pop("id") + if "customer" in kwargs and "customer" not in kwargs: + kwargs["customer"] = kwargs.pop("customer") + if "status" in kwargs and "status" not in kwargs: + kwargs["status"] = kwargs.pop("status") + if "order_number" in kwargs and "orderNumber" not in kwargs: + kwargs["orderNumber"] = kwargs.pop("order_number") + if "order_date" in kwargs and "orderDate" not in kwargs: + kwargs["orderDate"] = kwargs.pop("order_date") + if "total_amount" in kwargs and "totalAmount" not in kwargs: + kwargs["totalAmount"] = kwargs.pop("total_amount") + if "version" in kwargs and "version" not in kwargs: + kwargs["version"] = kwargs.pop("version") + self._projection_properties = ProjectionProperties(kwargs, + ["id", "customer", "status", "orderNumber", "orderDate", "totalAmount", "version", "order_item_list"]) + self._action = "Update" if kwargs.get("id") else "Create" + self._comment = None + self._loaded_fields = set(kwargs.keys()) + self.id = kwargs.get("id") + + self.customer = kwargs.get("customer") + + self.status = kwargs.get("status") + + self.orderNumber = kwargs.get("orderNumber") + + self.orderDate = kwargs.get("orderDate") + + self.totalAmount = kwargs.get("totalAmount") + + self.version = kwargs.get("version") + + if isinstance(self.customer, dict): + self.customer = Customer(**self.customer) + if isinstance(self.status, dict): + self.status = OrderStatus(**self.status) + self._order_item_list = kwargs.get("order_item_list", []) + if "order_item_list" in kwargs or kwargs.get("id") is None: + self._loaded_fields.add("order_item_list") + if self._order_item_list: + from models.order_item import OrderItem + self._order_item_list = [ + item if isinstance(item, OrderItem) else OrderItem(**item) + for item in self._order_item_list + ] + self._ledger_id = getattr(self, "id", None) + if self._ledger_id is None: + self._ledger_id = -next(self._teaql_temporary_ids) + key = self._teaql_entity_key() + if self._action == "Create": + self._entity_root.mark_as_new(key) + elif getattr(self, "version", None) is not None: + self._entity_root.set_original_version(key, int(self.version)) + LoadedScalarSnapshots.capture(self, self._teaql_snapshot_values()) + + def _teaql_snapshot_values(self): + payload = {} + if "id" in self._loaded_fields: + payload["id"] = Value.I64(self.id) + + if "customer" in self._loaded_fields: + payload["customer"] = Value.I64(getattr(self.customer, "id", self.customer)) + + if "status" in self._loaded_fields: + payload["status"] = Value.I64(getattr(self.status, "id", self.status)) + + if "orderNumber" in self._loaded_fields: + payload["order_number"] = Value.Text(self.orderNumber) + + if "orderDate" in self._loaded_fields: + payload["order_date"] = Value.Date(self.orderDate) + + if "totalAmount" in self._loaded_fields: + payload["total_amount"] = Value.Decimal(self.totalAmount) + + if "version" in self._loaded_fields: + payload["version"] = Value.I64(self.version) + + return payload + + def _teaql_entity_key(self): + return EntityKey("CustomerOrder", self._ledger_id) + + def get_dynamic_property(self, name): + return self._projection_properties.get(name) + + def _teaql_attach_root(self, root): + if self._entity_root is not root: + root.merge_entity_from(self._entity_root, self._teaql_entity_key()) + self._entity_root = root + for child in self._order_item_list: + child._teaql_attach_root(root) + return self + def mark_for_deletion(self): + self._action = "Delete" + self._entity_root.mark_as_deleted(self._teaql_entity_key()) + return self + + def audit_as(self, comment: str): + MutationIntent(comment) + self._comment = comment + return self + + async def save(self, context): + # Validate the root reason before ID allocation or provider access. + MutationIntent(self._comment) + # Reserve stable internal IDs before the transaction and before policy + # review. A relation to a new parent must have the same identity in the + # reviewed graph plan and in the commands eventually sent to SQL. + await self._teaql_reserve_graph_ids(context, set()) + await self._teaql_ensure_business_ids(context) + return await context.execute_graph_save( + lambda session: self._teaql_preflight_and_save(session.context, session), comment=self._comment) + + async def _teaql_ensure_business_ids(self, context): + if self._action != "Create": + return + + async def _teaql_reserve_graph_ids(self, context, visited): + object_identity = id(self) + if object_identity in visited: + return + visited.add(object_identity) + if self._action == "Create" and getattr(self, "id", None) is None: + service = context.require_resource("dataService") + allocator = getattr(service, "next_id", None) + if not callable(allocator): + raise RuntimeError( + "Configured dataService does not support stable ID reservation" + ) + old_key = self._teaql_entity_key() + self.id = int(await allocator("CustomerOrder")) + self._loaded_fields.add("id") + self._ledger_id = self.id + new_key = self._teaql_entity_key() + self._entity_root.rekey(old_key, new_key) + self._entity_root.set(new_key, "id", Value.I64(self.id)) + for child in self._order_item_list: + child._teaql_attach_root(self._entity_root) + await child._teaql_reserve_graph_ids(context, visited) + + async def _teaql_preflight_and_save(self, context, session): + self._teaql_preflight_graph(context, session) + return await self._teaql_save_within_graph(context, session) + + def _teaql_build_command(self): + payload = {} + if "id" in self._loaded_fields: + payload["id"] = Value.I64(self.id) + + if "customer" in self._loaded_fields: + reference = self.customer + reference_id = getattr(reference, "id", reference) + payload["customer"] = (Value.Object(reference) + if reference is not None and hasattr(reference, "id") and reference_id is None + else Value.I64(reference_id)) + + if "status" in self._loaded_fields: + reference = self.status + reference_id = getattr(reference, "id", reference) + payload["status"] = (Value.Object(reference) + if reference is not None and hasattr(reference, "id") and reference_id is None + else Value.I64(reference_id)) + + if "orderNumber" in self._loaded_fields: + payload["order_number"] = Value.Text(self.orderNumber) + + if "orderDate" in self._loaded_fields: + payload["order_date"] = Value.Date(self.orderDate) + + if "totalAmount" in self._loaded_fields: + payload["total_amount"] = Value.Decimal(self.totalAmount) + + if "version" in self._loaded_fields: + payload["version"] = Value.I64(self.version) + + action = self._action + if action == "Update": + ledger = dict(self._entity_root.current_change_set().changes()).get(self._teaql_entity_key(), {}) + payload = {field: value for field, value in ledger.items() if field not in ("id", "version")} + if action == "Create": + cmd = InsertCommand("CustomerOrder", payload) + elif action == "Update": + cmd = UpdateCommand("CustomerOrder", Value.from_any(getattr(self, "id", None)), getattr(self, "version", None)) + for key, value in payload.items(): + if key not in ("id", "version"): cmd.value(key, value) + else: + cmd = DeleteCommand("CustomerOrder", Value.from_any(getattr(self, "id", None)), getattr(self, "version", None)) + if action != "Create": + cmd.old_values = LoadedScalarSnapshots.values(self) + return action, cmd + + def _teaql_preflight_graph(self, context, session, parent=None): + if parent is None: + MutationIntent(self._comment) + scope = session.scope(self._teaql_entity_key(), parent, self._comment) + session.capture_loaded_values("CustomerOrder", LoadedScalarSnapshots.values(self)) + if self._entity_root.has_pending(self._teaql_entity_key()): + if self._action == "Update": + if "id" not in self._loaded_fields: + raise CheckException([CheckResult("invalid_type", ObjectLocation().property("id"), message="Mutation requires a fully loaded entity")]) + if "customer" not in self._loaded_fields: + raise CheckException([CheckResult("invalid_type", ObjectLocation().property("customer"), message="Mutation requires a fully loaded entity")]) + if "status" not in self._loaded_fields: + raise CheckException([CheckResult("invalid_type", ObjectLocation().property("status"), message="Mutation requires a fully loaded entity")]) + if "orderNumber" not in self._loaded_fields: + raise CheckException([CheckResult("invalid_type", ObjectLocation().property("order_number"), message="Mutation requires a fully loaded entity")]) + if "orderDate" not in self._loaded_fields: + raise CheckException([CheckResult("invalid_type", ObjectLocation().property("order_date"), message="Mutation requires a fully loaded entity")]) + if "totalAmount" not in self._loaded_fields: + raise CheckException([CheckResult("invalid_type", ObjectLocation().property("total_amount"), message="Mutation requires a fully loaded entity")]) + if "version" not in self._loaded_fields: + raise CheckException([CheckResult("invalid_type", ObjectLocation().property("version"), message="Mutation requires a fully loaded entity")]) + _action, cmd = self._teaql_build_command() + try: + context.preflight_mutation(cmd) + finally: + for field, value in getattr(cmd, "values", {}).items(): + if field not in ("id", "version"): + self._entity_root.set(self._teaql_entity_key(), field, value) + for index, child in enumerate(self._order_item_list): + child._teaql_attach_root(self._entity_root) + child.update_customer_order(self) + try: + child._teaql_preflight_graph(context, session, scope) + except CheckException as error: + prefix = ObjectLocation().property("order_item_list").index(index) + raise CheckException([ + CheckResult(v.rule_id, v.location.prefixed_by(prefix), v.input_value, v.system_value, v.message) + for v in error.violations + ]) from error + + async def _teaql_save_within_graph(self, context, session, parent=None): + if parent is None: + MutationIntent(self._comment) + + self._teaql_attach_root(self._entity_root) + scope = session.scope(self._teaql_entity_key(), parent, self._comment) + if not self._entity_root.has_pending(self._teaql_entity_key()): + await self._teaql_cascade_children(context, session, scope) + return self + action, cmd = self._teaql_build_command() + + + req = session.request(cmd, scope, self._entity_root, self._teaql_entity_key()) + + try: + context.check_and_fix_mutation(cmd) + finally: + for field, value in getattr(cmd, "values", {}).items(): + if field not in ("id", "version"): + self._entity_root.set(self._teaql_entity_key(), field, value) + context.mark_mutation_checked(cmd) + service = context.require_resource("dataService") + result = await service.mutate(context, req) + persisted = result.persisted_record + if persisted is None: + raise RuntimeError( + "Mutation provider did not return authoritative persisted state for CustomerOrder" + ) + rollback_payload = {field: getattr(self, field, None) for field in self._loaded_fields | {"id", "version"}} + rollback_snapshot = LoadedScalarSnapshots.values(self) + rollback_ledger_id = self._ledger_id + rollback_action = self._action + rollback_loaded_fields = set(self._loaded_fields) + old_key = self._teaql_entity_key() + if "id" in persisted: + self.id = persisted["id"] + self._loaded_fields.add("id") + elif "id" in persisted: + self.id = persisted["id"] + self._loaded_fields.add("id") + if "customer" in persisted: + self.customer = persisted["customer"] + self._loaded_fields.add("customer") + elif "customer" in persisted: + self.customer = persisted["customer"] + self._loaded_fields.add("customer") + if "status" in persisted: + self.status = persisted["status"] + self._loaded_fields.add("status") + elif "status" in persisted: + self.status = persisted["status"] + self._loaded_fields.add("status") + if "order_number" in persisted: + self.orderNumber = persisted["order_number"] + self._loaded_fields.add("orderNumber") + elif "orderNumber" in persisted: + self.orderNumber = persisted["orderNumber"] + self._loaded_fields.add("orderNumber") + if "order_date" in persisted: + self.orderDate = persisted["order_date"] + self._loaded_fields.add("orderDate") + elif "orderDate" in persisted: + self.orderDate = persisted["orderDate"] + self._loaded_fields.add("orderDate") + if "total_amount" in persisted: + self.totalAmount = persisted["total_amount"] + self._loaded_fields.add("totalAmount") + elif "totalAmount" in persisted: + self.totalAmount = persisted["totalAmount"] + self._loaded_fields.add("totalAmount") + if "version" in persisted: + self.version = persisted["version"] + self._loaded_fields.add("version") + elif "version" in persisted: + self.version = persisted["version"] + self._loaded_fields.add("version") + self._ledger_id = getattr(self, "id", self._ledger_id) + new_key = self._teaql_entity_key() + if old_key != new_key: + self._entity_root.rekey(old_key, new_key) + def rollback_entity(): + for field, value in rollback_payload.items(): + setattr(self, field, value) + self._ledger_id = rollback_ledger_id + self._action = rollback_action + self._loaded_fields = rollback_loaded_fields + LoadedScalarSnapshots.restore(self, rollback_snapshot) + if old_key != new_key: + self._entity_root.rekey(new_key, old_key) + session.after_rollback(rollback_entity) + if action != "Delete": + self._action = "Update" + + if action != "Delete": + await self._teaql_cascade_children(context, session, scope) + def commit_entity(): + LoadedScalarSnapshots.capture(self, self._teaql_snapshot_values()) + self._entity_root.clear_entity(new_key) + if getattr(self, "version", None) is not None: + self._entity_root.accept_committed_version(new_key, int(self.version)) + session.after_commit(commit_entity) + return self + + async def _teaql_cascade_children(self, context, session, scope): + cascade_relations = [] + cascade_relations.append(("order_item_list", self._order_item_list, "update_customer_order")) + for relation_name, children, updater in cascade_relations: + for index, child in enumerate(children): + child._teaql_attach_root(self._entity_root) + getattr(child, updater)(self) + try: + await child._teaql_save_within_graph(context, session, scope) + except CheckException as error: + prefix = ObjectLocation().property(relation_name).index(index) + raise CheckException([ + CheckResult( + violation.rule_id, + violation.location.prefixed_by(prefix), + violation.input_value, + violation.system_value, + violation.message, + ) + for violation in error.violations + ]) from error + + def update_id(self, value): + if self._action != "Create": + raise ValueError("ENTITY_ID_IMMUTABLE: an existing entity ID cannot be changed") + value = int(value) if isinstance(value, str) and value.isdecimal() else value + if type(value) is not int or not 0 < value <= 18446744073709551615: + raise ValueError("ENTITY_ID_INVALID: an assigned unsigned-64 identity is required") + old_key = self._teaql_entity_key() + self._ledger_id = value + self._entity_root.rekey(old_key, self._teaql_entity_key()) + if "id" in self._loaded_fields and self.id == value: + return self + self.id = value + self._loaded_fields.add("id") + self._entity_root.set(self._teaql_entity_key(), "id", Value.from_any(value)) + return self + + def update_order_number(self, value): + if "orderNumber" in self._loaded_fields and self.orderNumber == value: + return self + self.orderNumber = value + self._loaded_fields.add("orderNumber") + self._entity_root.set(self._teaql_entity_key(), "order_number", Value.from_any(value)) + return self + + def update_order_date(self, value): + if "orderDate" in self._loaded_fields and self.orderDate == value: + return self + self.orderDate = value + self._loaded_fields.add("orderDate") + self._entity_root.set(self._teaql_entity_key(), "order_date", Value.from_any(value)) + return self + + def update_total_amount(self, value): + if "totalAmount" in self._loaded_fields and self.totalAmount == value: + return self + self.totalAmount = value + self._loaded_fields.add("totalAmount") + self._entity_root.set(self._teaql_entity_key(), "total_amount", Value.from_any(value)) + return self + + def update_version(self, value): + if "version" in self._loaded_fields and self.version == value: + return self + self.version = value + self._loaded_fields.add("version") + self._entity_root.set(self._teaql_entity_key(), "version", Value.from_any(value)) + return self + def update_customer(self, value): + assigned_id = getattr(value, "id", value) if value else None + current_id = getattr(self.customer, "id", self.customer) + if "customer" in self._loaded_fields and current_id == assigned_id: + return self + self.customer = getattr(value, "id", value) if value else None + self._loaded_fields.add("customer") + self._entity_root.set(self._teaql_entity_key(), "customer", Value.from_any(self.customer)) + return self + + + def update_status(self, value): + assigned_id = getattr(value, "id", value) if value else None + current_id = getattr(self.status, "id", self.status) + if "status" in self._loaded_fields and current_id == assigned_id: + return self + self.status = getattr(value, "id", value) if value else None + self._loaded_fields.add("status") + self._entity_root.set(self._teaql_entity_key(), "status", Value.from_any(self.status)) + return self + def update_status_to_new(self): + self.status = 1001 + self._loaded_fields.add("status") + self._entity_root.set(self._teaql_entity_key(), "status", Value.from_any(self.status)) + return self + def update_status_to_paid(self): + self.status = 1002 + self._loaded_fields.add("status") + self._entity_root.set(self._teaql_entity_key(), "status", Value.from_any(self.status)) + return self + def update_status_to_cancelled(self): + self.status = 1003 + self._loaded_fields.add("status") + self._entity_root.set(self._teaql_entity_key(), "status", Value.from_any(self.status)) + return self + + def order_item_list(self) -> list: + self._loaded_fields.add("order_item_list") + return self._order_item_list diff --git a/examples/trace-chain/standard-order/lib/models/order_item.py b/examples/trace-chain/standard-order/lib/models/order_item.py new file mode 100644 index 0000000..58e8b72 --- /dev/null +++ b/examples/trace-chain/standard-order/lib/models/order_item.py @@ -0,0 +1,392 @@ +from teaql.core.mutation import InsertCommand, UpdateCommand, DeleteCommand, MutationRequest +from teaql.core.value import Value +from teaql.core.request_intent import MutationIntent +from teaql.core.projection_properties import ProjectionProperties +from teaql.runtime.loaded_scalar_snapshot import LoadedScalarSnapshots +from teaql.runtime import CheckException, CheckResult, EntityKey, EntityRoot, ObjectLocation +import itertools +from models.customer_order import CustomerOrder + + +class OrderItem: + _teaql_temporary_ids = itertools.count(1) + @classmethod + def refer(cls, entity_id): + return cls(id=entity_id) + + @classmethod + def _teaql_new_with_fixed_id(cls, entity_id): + """Generated bootstrap capability; application code must not call it.""" + return cls(id=entity_id)._teaql_force_create() + + def _teaql_force_create(self): + self._action = "Create" + self._entity_root.mark_as_new(self._teaql_entity_key()) + return self + + def __init__(self, **kwargs): + self._entity_root = kwargs.pop("_entity_root", None) or EntityRoot() + if "id" in kwargs and "id" not in kwargs: + kwargs["id"] = kwargs.pop("id") + if "customer_order" in kwargs and "customerOrder" not in kwargs: + kwargs["customerOrder"] = kwargs.pop("customer_order") + if "product_name" in kwargs and "productName" not in kwargs: + kwargs["productName"] = kwargs.pop("product_name") + if "quantity" in kwargs and "quantity" not in kwargs: + kwargs["quantity"] = kwargs.pop("quantity") + if "unit_price" in kwargs and "unitPrice" not in kwargs: + kwargs["unitPrice"] = kwargs.pop("unit_price") + if "version" in kwargs and "version" not in kwargs: + kwargs["version"] = kwargs.pop("version") + self._projection_properties = ProjectionProperties(kwargs, + ["id", "customerOrder", "productName", "quantity", "unitPrice", "version"]) + self._action = "Update" if kwargs.get("id") else "Create" + self._comment = None + self._loaded_fields = set(kwargs.keys()) + self.id = kwargs.get("id") + + self.customerOrder = kwargs.get("customerOrder") + + self.productName = kwargs.get("productName") + + self.quantity = kwargs.get("quantity") + + self.unitPrice = kwargs.get("unitPrice") + + self.version = kwargs.get("version") + + if isinstance(self.customerOrder, dict): + self.customerOrder = CustomerOrder(**self.customerOrder) + self._ledger_id = getattr(self, "id", None) + if self._ledger_id is None: + self._ledger_id = -next(self._teaql_temporary_ids) + key = self._teaql_entity_key() + if self._action == "Create": + self._entity_root.mark_as_new(key) + elif getattr(self, "version", None) is not None: + self._entity_root.set_original_version(key, int(self.version)) + LoadedScalarSnapshots.capture(self, self._teaql_snapshot_values()) + + def _teaql_snapshot_values(self): + payload = {} + if "id" in self._loaded_fields: + payload["id"] = Value.I64(self.id) + + if "customerOrder" in self._loaded_fields: + payload["customer_order"] = Value.I64(getattr(self.customerOrder, "id", self.customerOrder)) + + if "productName" in self._loaded_fields: + payload["product_name"] = Value.Text(self.productName) + + if "quantity" in self._loaded_fields: + payload["quantity"] = Value.I64(self.quantity) + + if "unitPrice" in self._loaded_fields: + payload["unit_price"] = Value.Decimal(self.unitPrice) + + if "version" in self._loaded_fields: + payload["version"] = Value.I64(self.version) + + return payload + + def _teaql_entity_key(self): + return EntityKey("OrderItem", self._ledger_id) + + def get_dynamic_property(self, name): + return self._projection_properties.get(name) + + def _teaql_attach_root(self, root): + if self._entity_root is not root: + root.merge_entity_from(self._entity_root, self._teaql_entity_key()) + self._entity_root = root + return self + def mark_for_deletion(self): + self._action = "Delete" + self._entity_root.mark_as_deleted(self._teaql_entity_key()) + return self + + def audit_as(self, comment: str): + MutationIntent(comment) + self._comment = comment + return self + + async def save(self, context): + # Validate the root reason before ID allocation or provider access. + MutationIntent(self._comment) + # Reserve stable internal IDs before the transaction and before policy + # review. A relation to a new parent must have the same identity in the + # reviewed graph plan and in the commands eventually sent to SQL. + await self._teaql_reserve_graph_ids(context, set()) + await self._teaql_ensure_business_ids(context) + return await context.execute_graph_save( + lambda session: self._teaql_preflight_and_save(session.context, session), comment=self._comment) + + async def _teaql_ensure_business_ids(self, context): + if self._action != "Create": + return + + async def _teaql_reserve_graph_ids(self, context, visited): + object_identity = id(self) + if object_identity in visited: + return + visited.add(object_identity) + if self._action == "Create" and getattr(self, "id", None) is None: + service = context.require_resource("dataService") + allocator = getattr(service, "next_id", None) + if not callable(allocator): + raise RuntimeError( + "Configured dataService does not support stable ID reservation" + ) + old_key = self._teaql_entity_key() + self.id = int(await allocator("OrderItem")) + self._loaded_fields.add("id") + self._ledger_id = self.id + new_key = self._teaql_entity_key() + self._entity_root.rekey(old_key, new_key) + self._entity_root.set(new_key, "id", Value.I64(self.id)) + + async def _teaql_preflight_and_save(self, context, session): + self._teaql_preflight_graph(context, session) + return await self._teaql_save_within_graph(context, session) + + def _teaql_build_command(self): + payload = {} + if "id" in self._loaded_fields: + payload["id"] = Value.I64(self.id) + + if "customerOrder" in self._loaded_fields: + reference = self.customerOrder + reference_id = getattr(reference, "id", reference) + payload["customer_order"] = (Value.Object(reference) + if reference is not None and hasattr(reference, "id") and reference_id is None + else Value.I64(reference_id)) + + if "productName" in self._loaded_fields: + payload["product_name"] = Value.Text(self.productName) + + if "quantity" in self._loaded_fields: + payload["quantity"] = Value.I64(self.quantity) + + if "unitPrice" in self._loaded_fields: + payload["unit_price"] = Value.Decimal(self.unitPrice) + + if "version" in self._loaded_fields: + payload["version"] = Value.I64(self.version) + + action = self._action + if action == "Update": + ledger = dict(self._entity_root.current_change_set().changes()).get(self._teaql_entity_key(), {}) + payload = {field: value for field, value in ledger.items() if field not in ("id", "version")} + if action == "Create": + cmd = InsertCommand("OrderItem", payload) + elif action == "Update": + cmd = UpdateCommand("OrderItem", Value.from_any(getattr(self, "id", None)), getattr(self, "version", None)) + for key, value in payload.items(): + if key not in ("id", "version"): cmd.value(key, value) + else: + cmd = DeleteCommand("OrderItem", Value.from_any(getattr(self, "id", None)), getattr(self, "version", None)) + if action != "Create": + cmd.old_values = LoadedScalarSnapshots.values(self) + return action, cmd + + def _teaql_preflight_graph(self, context, session, parent=None): + if parent is None: + MutationIntent(self._comment) + scope = session.scope(self._teaql_entity_key(), parent, self._comment) + session.capture_loaded_values("OrderItem", LoadedScalarSnapshots.values(self)) + if self._entity_root.has_pending(self._teaql_entity_key()): + if self._action == "Update": + if "id" not in self._loaded_fields: + raise CheckException([CheckResult("invalid_type", ObjectLocation().property("id"), message="Mutation requires a fully loaded entity")]) + if "customerOrder" not in self._loaded_fields: + raise CheckException([CheckResult("invalid_type", ObjectLocation().property("customer_order"), message="Mutation requires a fully loaded entity")]) + if "productName" not in self._loaded_fields: + raise CheckException([CheckResult("invalid_type", ObjectLocation().property("product_name"), message="Mutation requires a fully loaded entity")]) + if "quantity" not in self._loaded_fields: + raise CheckException([CheckResult("invalid_type", ObjectLocation().property("quantity"), message="Mutation requires a fully loaded entity")]) + if "unitPrice" not in self._loaded_fields: + raise CheckException([CheckResult("invalid_type", ObjectLocation().property("unit_price"), message="Mutation requires a fully loaded entity")]) + if "version" not in self._loaded_fields: + raise CheckException([CheckResult("invalid_type", ObjectLocation().property("version"), message="Mutation requires a fully loaded entity")]) + _action, cmd = self._teaql_build_command() + try: + context.preflight_mutation(cmd) + finally: + for field, value in getattr(cmd, "values", {}).items(): + if field not in ("id", "version"): + self._entity_root.set(self._teaql_entity_key(), field, value) + + async def _teaql_save_within_graph(self, context, session, parent=None): + if parent is None: + MutationIntent(self._comment) + + self._teaql_attach_root(self._entity_root) + scope = session.scope(self._teaql_entity_key(), parent, self._comment) + if not self._entity_root.has_pending(self._teaql_entity_key()): + await self._teaql_cascade_children(context, session, scope) + return self + action, cmd = self._teaql_build_command() + + + req = session.request(cmd, scope, self._entity_root, self._teaql_entity_key()) + + try: + context.check_and_fix_mutation(cmd) + finally: + for field, value in getattr(cmd, "values", {}).items(): + if field not in ("id", "version"): + self._entity_root.set(self._teaql_entity_key(), field, value) + context.mark_mutation_checked(cmd) + service = context.require_resource("dataService") + result = await service.mutate(context, req) + persisted = result.persisted_record + if persisted is None: + raise RuntimeError( + "Mutation provider did not return authoritative persisted state for OrderItem" + ) + rollback_payload = {field: getattr(self, field, None) for field in self._loaded_fields | {"id", "version"}} + rollback_snapshot = LoadedScalarSnapshots.values(self) + rollback_ledger_id = self._ledger_id + rollback_action = self._action + rollback_loaded_fields = set(self._loaded_fields) + old_key = self._teaql_entity_key() + if "id" in persisted: + self.id = persisted["id"] + self._loaded_fields.add("id") + elif "id" in persisted: + self.id = persisted["id"] + self._loaded_fields.add("id") + if "customer_order" in persisted: + self.customerOrder = persisted["customer_order"] + self._loaded_fields.add("customerOrder") + elif "customerOrder" in persisted: + self.customerOrder = persisted["customerOrder"] + self._loaded_fields.add("customerOrder") + if "product_name" in persisted: + self.productName = persisted["product_name"] + self._loaded_fields.add("productName") + elif "productName" in persisted: + self.productName = persisted["productName"] + self._loaded_fields.add("productName") + if "quantity" in persisted: + self.quantity = persisted["quantity"] + self._loaded_fields.add("quantity") + elif "quantity" in persisted: + self.quantity = persisted["quantity"] + self._loaded_fields.add("quantity") + if "unit_price" in persisted: + self.unitPrice = persisted["unit_price"] + self._loaded_fields.add("unitPrice") + elif "unitPrice" in persisted: + self.unitPrice = persisted["unitPrice"] + self._loaded_fields.add("unitPrice") + if "version" in persisted: + self.version = persisted["version"] + self._loaded_fields.add("version") + elif "version" in persisted: + self.version = persisted["version"] + self._loaded_fields.add("version") + self._ledger_id = getattr(self, "id", self._ledger_id) + new_key = self._teaql_entity_key() + if old_key != new_key: + self._entity_root.rekey(old_key, new_key) + def rollback_entity(): + for field, value in rollback_payload.items(): + setattr(self, field, value) + self._ledger_id = rollback_ledger_id + self._action = rollback_action + self._loaded_fields = rollback_loaded_fields + LoadedScalarSnapshots.restore(self, rollback_snapshot) + if old_key != new_key: + self._entity_root.rekey(new_key, old_key) + session.after_rollback(rollback_entity) + if action != "Delete": + self._action = "Update" + + if action != "Delete": + await self._teaql_cascade_children(context, session, scope) + def commit_entity(): + LoadedScalarSnapshots.capture(self, self._teaql_snapshot_values()) + self._entity_root.clear_entity(new_key) + if getattr(self, "version", None) is not None: + self._entity_root.accept_committed_version(new_key, int(self.version)) + session.after_commit(commit_entity) + return self + + async def _teaql_cascade_children(self, context, session, scope): + cascade_relations = [] + for relation_name, children, updater in cascade_relations: + for index, child in enumerate(children): + child._teaql_attach_root(self._entity_root) + getattr(child, updater)(self) + try: + await child._teaql_save_within_graph(context, session, scope) + except CheckException as error: + prefix = ObjectLocation().property(relation_name).index(index) + raise CheckException([ + CheckResult( + violation.rule_id, + violation.location.prefixed_by(prefix), + violation.input_value, + violation.system_value, + violation.message, + ) + for violation in error.violations + ]) from error + + def update_id(self, value): + if self._action != "Create": + raise ValueError("ENTITY_ID_IMMUTABLE: an existing entity ID cannot be changed") + value = int(value) if isinstance(value, str) and value.isdecimal() else value + if type(value) is not int or not 0 < value <= 18446744073709551615: + raise ValueError("ENTITY_ID_INVALID: an assigned unsigned-64 identity is required") + old_key = self._teaql_entity_key() + self._ledger_id = value + self._entity_root.rekey(old_key, self._teaql_entity_key()) + if "id" in self._loaded_fields and self.id == value: + return self + self.id = value + self._loaded_fields.add("id") + self._entity_root.set(self._teaql_entity_key(), "id", Value.from_any(value)) + return self + + def update_product_name(self, value): + if "productName" in self._loaded_fields and self.productName == value: + return self + self.productName = value + self._loaded_fields.add("productName") + self._entity_root.set(self._teaql_entity_key(), "product_name", Value.from_any(value)) + return self + + def update_quantity(self, value): + if "quantity" in self._loaded_fields and self.quantity == value: + return self + self.quantity = value + self._loaded_fields.add("quantity") + self._entity_root.set(self._teaql_entity_key(), "quantity", Value.from_any(value)) + return self + + def update_unit_price(self, value): + if "unitPrice" in self._loaded_fields and self.unitPrice == value: + return self + self.unitPrice = value + self._loaded_fields.add("unitPrice") + self._entity_root.set(self._teaql_entity_key(), "unit_price", Value.from_any(value)) + return self + + def update_version(self, value): + if "version" in self._loaded_fields and self.version == value: + return self + self.version = value + self._loaded_fields.add("version") + self._entity_root.set(self._teaql_entity_key(), "version", Value.from_any(value)) + return self + def update_customer_order(self, value): + assigned_id = getattr(value, "id", value) if value else None + current_id = getattr(self.customerOrder, "id", self.customerOrder) + if "customerOrder" in self._loaded_fields and current_id == assigned_id: + return self + self.customerOrder = getattr(value, "id", value) if value else None + self._loaded_fields.add("customerOrder") + self._entity_root.set(self._teaql_entity_key(), "customer_order", Value.from_any(self.customerOrder)) + return self diff --git a/examples/trace-chain/standard-order/lib/models/order_status.py b/examples/trace-chain/standard-order/lib/models/order_status.py new file mode 100644 index 0000000..922303d --- /dev/null +++ b/examples/trace-chain/standard-order/lib/models/order_status.py @@ -0,0 +1,448 @@ +from teaql.core.mutation import InsertCommand, UpdateCommand, DeleteCommand, MutationRequest +from teaql.core.value import Value +from teaql.core.request_intent import MutationIntent +from teaql.core.projection_properties import ProjectionProperties +from teaql.runtime.loaded_scalar_snapshot import LoadedScalarSnapshots +from teaql.runtime import CheckException, CheckResult, EntityKey, EntityRoot, ObjectLocation +import itertools +from models.platform import Platform + + +class OrderStatus: + _teaql_temporary_ids = itertools.count(1) + @classmethod + def refer(cls, entity_id): + return cls(id=entity_id) + + @classmethod + def _teaql_new_with_fixed_id(cls, entity_id): + """Generated bootstrap capability; application code must not call it.""" + return cls(id=entity_id)._teaql_force_create() + + def _teaql_force_create(self): + self._action = "Create" + self._entity_root.mark_as_new(self._teaql_entity_key()) + return self + + def __init__(self, **kwargs): + self._entity_root = kwargs.pop("_entity_root", None) or EntityRoot() + if "platform" in kwargs and "platform" not in kwargs: + kwargs["platform"] = kwargs.pop("platform") + if "id" in kwargs and "id" not in kwargs: + kwargs["id"] = kwargs.pop("id") + if "name" in kwargs and "name" not in kwargs: + kwargs["name"] = kwargs.pop("name") + if "code" in kwargs and "code" not in kwargs: + kwargs["code"] = kwargs.pop("code") + if "color" in kwargs and "color" not in kwargs: + kwargs["color"] = kwargs.pop("color") + if "display_order" in kwargs and "displayOrder" not in kwargs: + kwargs["displayOrder"] = kwargs.pop("display_order") + if "version" in kwargs and "version" not in kwargs: + kwargs["version"] = kwargs.pop("version") + self._projection_properties = ProjectionProperties(kwargs, + ["platform", "id", "name", "code", "color", "displayOrder", "version", "customer_order_list"]) + self._action = "Update" if kwargs.get("id") else "Create" + self._comment = None + self._loaded_fields = set(kwargs.keys()) + self.platform = kwargs.get("platform") + + self.id = kwargs.get("id") + + self.name = kwargs.get("name") + + self.code = kwargs.get("code") + + self.color = kwargs.get("color") + + self.displayOrder = kwargs.get("displayOrder") + + self.version = kwargs.get("version") + + if isinstance(self.platform, dict): + self.platform = Platform(**self.platform) + self._customer_order_list = kwargs.get("customer_order_list", []) + if "customer_order_list" in kwargs or kwargs.get("id") is None: + self._loaded_fields.add("customer_order_list") + if self._customer_order_list: + from models.customer_order import CustomerOrder + self._customer_order_list = [ + item if isinstance(item, CustomerOrder) else CustomerOrder(**item) + for item in self._customer_order_list + ] + self._ledger_id = getattr(self, "id", None) + if self._ledger_id is None: + self._ledger_id = -next(self._teaql_temporary_ids) + key = self._teaql_entity_key() + if self._action == "Create": + self._entity_root.mark_as_new(key) + elif getattr(self, "version", None) is not None: + self._entity_root.set_original_version(key, int(self.version)) + LoadedScalarSnapshots.capture(self, self._teaql_snapshot_values()) + + def _teaql_snapshot_values(self): + payload = {} + if "platform" in self._loaded_fields: + payload["platform"] = Value.I64(getattr(self.platform, "id", self.platform)) + + if "id" in self._loaded_fields: + payload["id"] = Value.I64(self.id) + + if "name" in self._loaded_fields: + payload["name"] = Value.Text(self.name) + + if "code" in self._loaded_fields: + payload["code"] = Value.Text(self.code) + + if "color" in self._loaded_fields: + payload["color"] = Value.Text(self.color) + + if "displayOrder" in self._loaded_fields: + payload["display_order"] = Value.I64(self.displayOrder) + + if "version" in self._loaded_fields: + payload["version"] = Value.I64(self.version) + + return payload + + def _teaql_entity_key(self): + return EntityKey("OrderStatus", self._ledger_id) + + def get_dynamic_property(self, name): + return self._projection_properties.get(name) + + def _teaql_attach_root(self, root): + if self._entity_root is not root: + root.merge_entity_from(self._entity_root, self._teaql_entity_key()) + self._entity_root = root + for child in self._customer_order_list: + child._teaql_attach_root(root) + return self + def mark_for_deletion(self): + self._action = "Delete" + self._entity_root.mark_as_deleted(self._teaql_entity_key()) + return self + + def audit_as(self, comment: str): + MutationIntent(comment) + self._comment = comment + return self + + async def save(self, context): + # Validate the root reason before ID allocation or provider access. + MutationIntent(self._comment) + # Reserve stable internal IDs before the transaction and before policy + # review. A relation to a new parent must have the same identity in the + # reviewed graph plan and in the commands eventually sent to SQL. + await self._teaql_reserve_graph_ids(context, set()) + await self._teaql_ensure_business_ids(context) + return await context.execute_graph_save( + lambda session: self._teaql_preflight_and_save(session.context, session), comment=self._comment) + + async def _teaql_ensure_business_ids(self, context): + if self._action != "Create": + return + + async def _teaql_reserve_graph_ids(self, context, visited): + object_identity = id(self) + if object_identity in visited: + return + visited.add(object_identity) + if self._action == "Create" and getattr(self, "id", None) is None: + service = context.require_resource("dataService") + allocator = getattr(service, "next_id", None) + if not callable(allocator): + raise RuntimeError( + "Configured dataService does not support stable ID reservation" + ) + old_key = self._teaql_entity_key() + self.id = int(await allocator("OrderStatus")) + self._loaded_fields.add("id") + self._ledger_id = self.id + new_key = self._teaql_entity_key() + self._entity_root.rekey(old_key, new_key) + self._entity_root.set(new_key, "id", Value.I64(self.id)) + for child in self._customer_order_list: + child._teaql_attach_root(self._entity_root) + await child._teaql_reserve_graph_ids(context, visited) + + async def _teaql_preflight_and_save(self, context, session): + self._teaql_preflight_graph(context, session) + return await self._teaql_save_within_graph(context, session) + + def _teaql_build_command(self): + payload = {} + if "platform" in self._loaded_fields: + reference = self.platform + reference_id = getattr(reference, "id", reference) + payload["platform"] = (Value.Object(reference) + if reference is not None and hasattr(reference, "id") and reference_id is None + else Value.I64(reference_id)) + + if "id" in self._loaded_fields: + payload["id"] = Value.I64(self.id) + + if "name" in self._loaded_fields: + payload["name"] = Value.Text(self.name) + + if "code" in self._loaded_fields: + payload["code"] = Value.Text(self.code) + + if "color" in self._loaded_fields: + payload["color"] = Value.Text(self.color) + + if "displayOrder" in self._loaded_fields: + payload["display_order"] = Value.I64(self.displayOrder) + + if "version" in self._loaded_fields: + payload["version"] = Value.I64(self.version) + + action = self._action + if action == "Update": + ledger = dict(self._entity_root.current_change_set().changes()).get(self._teaql_entity_key(), {}) + payload = {field: value for field, value in ledger.items() if field not in ("id", "version")} + if action == "Create": + cmd = InsertCommand("OrderStatus", payload) + elif action == "Update": + cmd = UpdateCommand("OrderStatus", Value.from_any(getattr(self, "id", None)), getattr(self, "version", None)) + for key, value in payload.items(): + if key not in ("id", "version"): cmd.value(key, value) + else: + cmd = DeleteCommand("OrderStatus", Value.from_any(getattr(self, "id", None)), getattr(self, "version", None)) + if action != "Create": + cmd.old_values = LoadedScalarSnapshots.values(self) + return action, cmd + + def _teaql_preflight_graph(self, context, session, parent=None): + if parent is None: + MutationIntent(self._comment) + scope = session.scope(self._teaql_entity_key(), parent, self._comment) + session.capture_loaded_values("OrderStatus", LoadedScalarSnapshots.values(self)) + if self._entity_root.has_pending(self._teaql_entity_key()): + if self._action == "Update": + if "platform" not in self._loaded_fields: + raise CheckException([CheckResult("invalid_type", ObjectLocation().property("platform"), message="Mutation requires a fully loaded entity")]) + if "id" not in self._loaded_fields: + raise CheckException([CheckResult("invalid_type", ObjectLocation().property("id"), message="Mutation requires a fully loaded entity")]) + if "name" not in self._loaded_fields: + raise CheckException([CheckResult("invalid_type", ObjectLocation().property("name"), message="Mutation requires a fully loaded entity")]) + if "code" not in self._loaded_fields: + raise CheckException([CheckResult("invalid_type", ObjectLocation().property("code"), message="Mutation requires a fully loaded entity")]) + if "color" not in self._loaded_fields: + raise CheckException([CheckResult("invalid_type", ObjectLocation().property("color"), message="Mutation requires a fully loaded entity")]) + if "displayOrder" not in self._loaded_fields: + raise CheckException([CheckResult("invalid_type", ObjectLocation().property("display_order"), message="Mutation requires a fully loaded entity")]) + if "version" not in self._loaded_fields: + raise CheckException([CheckResult("invalid_type", ObjectLocation().property("version"), message="Mutation requires a fully loaded entity")]) + _action, cmd = self._teaql_build_command() + try: + context.preflight_mutation(cmd) + finally: + for field, value in getattr(cmd, "values", {}).items(): + if field not in ("id", "version"): + self._entity_root.set(self._teaql_entity_key(), field, value) + for index, child in enumerate(self._customer_order_list): + child._teaql_attach_root(self._entity_root) + child.update_status(self) + try: + child._teaql_preflight_graph(context, session, scope) + except CheckException as error: + prefix = ObjectLocation().property("customer_order_list").index(index) + raise CheckException([ + CheckResult(v.rule_id, v.location.prefixed_by(prefix), v.input_value, v.system_value, v.message) + for v in error.violations + ]) from error + + async def _teaql_save_within_graph(self, context, session, parent=None): + if parent is None: + MutationIntent(self._comment) + + self._teaql_attach_root(self._entity_root) + scope = session.scope(self._teaql_entity_key(), parent, self._comment) + if not self._entity_root.has_pending(self._teaql_entity_key()): + await self._teaql_cascade_children(context, session, scope) + return self + action, cmd = self._teaql_build_command() + + + req = session.request(cmd, scope, self._entity_root, self._teaql_entity_key()) + + try: + context.check_and_fix_mutation(cmd) + finally: + for field, value in getattr(cmd, "values", {}).items(): + if field not in ("id", "version"): + self._entity_root.set(self._teaql_entity_key(), field, value) + context.mark_mutation_checked(cmd) + service = context.require_resource("dataService") + result = await service.mutate(context, req) + persisted = result.persisted_record + if persisted is None: + raise RuntimeError( + "Mutation provider did not return authoritative persisted state for OrderStatus" + ) + rollback_payload = {field: getattr(self, field, None) for field in self._loaded_fields | {"id", "version"}} + rollback_snapshot = LoadedScalarSnapshots.values(self) + rollback_ledger_id = self._ledger_id + rollback_action = self._action + rollback_loaded_fields = set(self._loaded_fields) + old_key = self._teaql_entity_key() + if "platform" in persisted: + self.platform = persisted["platform"] + self._loaded_fields.add("platform") + elif "platform" in persisted: + self.platform = persisted["platform"] + self._loaded_fields.add("platform") + if "id" in persisted: + self.id = persisted["id"] + self._loaded_fields.add("id") + elif "id" in persisted: + self.id = persisted["id"] + self._loaded_fields.add("id") + if "name" in persisted: + self.name = persisted["name"] + self._loaded_fields.add("name") + elif "name" in persisted: + self.name = persisted["name"] + self._loaded_fields.add("name") + if "code" in persisted: + self.code = persisted["code"] + self._loaded_fields.add("code") + elif "code" in persisted: + self.code = persisted["code"] + self._loaded_fields.add("code") + if "color" in persisted: + self.color = persisted["color"] + self._loaded_fields.add("color") + elif "color" in persisted: + self.color = persisted["color"] + self._loaded_fields.add("color") + if "display_order" in persisted: + self.displayOrder = persisted["display_order"] + self._loaded_fields.add("displayOrder") + elif "displayOrder" in persisted: + self.displayOrder = persisted["displayOrder"] + self._loaded_fields.add("displayOrder") + if "version" in persisted: + self.version = persisted["version"] + self._loaded_fields.add("version") + elif "version" in persisted: + self.version = persisted["version"] + self._loaded_fields.add("version") + self._ledger_id = getattr(self, "id", self._ledger_id) + new_key = self._teaql_entity_key() + if old_key != new_key: + self._entity_root.rekey(old_key, new_key) + def rollback_entity(): + for field, value in rollback_payload.items(): + setattr(self, field, value) + self._ledger_id = rollback_ledger_id + self._action = rollback_action + self._loaded_fields = rollback_loaded_fields + LoadedScalarSnapshots.restore(self, rollback_snapshot) + if old_key != new_key: + self._entity_root.rekey(new_key, old_key) + session.after_rollback(rollback_entity) + if action != "Delete": + self._action = "Update" + + if action != "Delete": + await self._teaql_cascade_children(context, session, scope) + def commit_entity(): + LoadedScalarSnapshots.capture(self, self._teaql_snapshot_values()) + self._entity_root.clear_entity(new_key) + if getattr(self, "version", None) is not None: + self._entity_root.accept_committed_version(new_key, int(self.version)) + session.after_commit(commit_entity) + return self + + async def _teaql_cascade_children(self, context, session, scope): + cascade_relations = [] + cascade_relations.append(("customer_order_list", self._customer_order_list, "update_status")) + for relation_name, children, updater in cascade_relations: + for index, child in enumerate(children): + child._teaql_attach_root(self._entity_root) + getattr(child, updater)(self) + try: + await child._teaql_save_within_graph(context, session, scope) + except CheckException as error: + prefix = ObjectLocation().property(relation_name).index(index) + raise CheckException([ + CheckResult( + violation.rule_id, + violation.location.prefixed_by(prefix), + violation.input_value, + violation.system_value, + violation.message, + ) + for violation in error.violations + ]) from error + + def update_id(self, value): + if self._action != "Create": + raise ValueError("ENTITY_ID_IMMUTABLE: an existing entity ID cannot be changed") + value = int(value) if isinstance(value, str) and value.isdecimal() else value + if type(value) is not int or not 0 < value <= 18446744073709551615: + raise ValueError("ENTITY_ID_INVALID: an assigned unsigned-64 identity is required") + old_key = self._teaql_entity_key() + self._ledger_id = value + self._entity_root.rekey(old_key, self._teaql_entity_key()) + if "id" in self._loaded_fields and self.id == value: + return self + self.id = value + self._loaded_fields.add("id") + self._entity_root.set(self._teaql_entity_key(), "id", Value.from_any(value)) + return self + + def update_name(self, value): + if "name" in self._loaded_fields and self.name == value: + return self + self.name = value + self._loaded_fields.add("name") + self._entity_root.set(self._teaql_entity_key(), "name", Value.from_any(value)) + return self + + def update_code(self, value): + if "code" in self._loaded_fields and self.code == value: + return self + self.code = value + self._loaded_fields.add("code") + self._entity_root.set(self._teaql_entity_key(), "code", Value.from_any(value)) + return self + + def update_color(self, value): + if "color" in self._loaded_fields and self.color == value: + return self + self.color = value + self._loaded_fields.add("color") + self._entity_root.set(self._teaql_entity_key(), "color", Value.from_any(value)) + return self + + def update_display_order(self, value): + if "displayOrder" in self._loaded_fields and self.displayOrder == value: + return self + self.displayOrder = value + self._loaded_fields.add("displayOrder") + self._entity_root.set(self._teaql_entity_key(), "display_order", Value.from_any(value)) + return self + + def update_version(self, value): + if "version" in self._loaded_fields and self.version == value: + return self + self.version = value + self._loaded_fields.add("version") + self._entity_root.set(self._teaql_entity_key(), "version", Value.from_any(value)) + return self + def update_platform(self, value): + assigned_id = getattr(value, "id", value) if value else None + current_id = getattr(self.platform, "id", self.platform) + if "platform" in self._loaded_fields and current_id == assigned_id: + return self + self.platform = getattr(value, "id", value) if value else None + self._loaded_fields.add("platform") + self._entity_root.set(self._teaql_entity_key(), "platform", Value.from_any(self.platform)) + return self + + def customer_order_list(self) -> list: + self._loaded_fields.add("customer_order_list") + return self._customer_order_list diff --git a/examples/trace-chain/standard-order/lib/models/platform.py b/examples/trace-chain/standard-order/lib/models/platform.py new file mode 100644 index 0000000..70bda84 --- /dev/null +++ b/examples/trace-chain/standard-order/lib/models/platform.py @@ -0,0 +1,365 @@ +from teaql.core.mutation import InsertCommand, UpdateCommand, DeleteCommand, MutationRequest +from teaql.core.value import Value +from teaql.core.request_intent import MutationIntent +from teaql.core.projection_properties import ProjectionProperties +from teaql.runtime.loaded_scalar_snapshot import LoadedScalarSnapshots +from teaql.runtime import CheckException, CheckResult, EntityKey, EntityRoot, ObjectLocation +import itertools + + +class Platform: + _teaql_temporary_ids = itertools.count(1) + @classmethod + def refer(cls, entity_id): + return cls(id=entity_id) + + @classmethod + def _teaql_new_with_fixed_id(cls, entity_id): + """Generated bootstrap capability; application code must not call it.""" + return cls(id=entity_id)._teaql_force_create() + + def _teaql_force_create(self): + self._action = "Create" + self._entity_root.mark_as_new(self._teaql_entity_key()) + return self + + def __init__(self, **kwargs): + self._entity_root = kwargs.pop("_entity_root", None) or EntityRoot() + if "id" in kwargs and "id" not in kwargs: + kwargs["id"] = kwargs.pop("id") + if "name" in kwargs and "name" not in kwargs: + kwargs["name"] = kwargs.pop("name") + if "version" in kwargs and "version" not in kwargs: + kwargs["version"] = kwargs.pop("version") + self._projection_properties = ProjectionProperties(kwargs, + ["id", "name", "version", "order_status_list", "customer_list"]) + self._action = "Update" if kwargs.get("id") else "Create" + self._comment = None + self._loaded_fields = set(kwargs.keys()) + self.id = kwargs.get("id") + + self.name = kwargs.get("name") + + self.version = kwargs.get("version") + + self._order_status_list = kwargs.get("order_status_list", []) + if "order_status_list" in kwargs or kwargs.get("id") is None: + self._loaded_fields.add("order_status_list") + self._customer_list = kwargs.get("customer_list", []) + if "customer_list" in kwargs or kwargs.get("id") is None: + self._loaded_fields.add("customer_list") + if self._order_status_list: + from models.order_status import OrderStatus + self._order_status_list = [ + item if isinstance(item, OrderStatus) else OrderStatus(**item) + for item in self._order_status_list + ] + if self._customer_list: + from models.customer import Customer + self._customer_list = [ + item if isinstance(item, Customer) else Customer(**item) + for item in self._customer_list + ] + self._ledger_id = getattr(self, "id", None) + if self._ledger_id is None: + self._ledger_id = -next(self._teaql_temporary_ids) + key = self._teaql_entity_key() + if self._action == "Create": + self._entity_root.mark_as_new(key) + elif getattr(self, "version", None) is not None: + self._entity_root.set_original_version(key, int(self.version)) + LoadedScalarSnapshots.capture(self, self._teaql_snapshot_values()) + + def _teaql_snapshot_values(self): + payload = {} + if "id" in self._loaded_fields: + payload["id"] = Value.I64(self.id) + + if "name" in self._loaded_fields: + payload["name"] = Value.Text(self.name) + + if "version" in self._loaded_fields: + payload["version"] = Value.I64(self.version) + + return payload + + def _teaql_entity_key(self): + return EntityKey("Platform", self._ledger_id) + + def get_dynamic_property(self, name): + return self._projection_properties.get(name) + + def _teaql_attach_root(self, root): + if self._entity_root is not root: + root.merge_entity_from(self._entity_root, self._teaql_entity_key()) + self._entity_root = root + for child in self._order_status_list: + child._teaql_attach_root(root) + for child in self._customer_list: + child._teaql_attach_root(root) + return self + def mark_for_deletion(self): + self._action = "Delete" + self._entity_root.mark_as_deleted(self._teaql_entity_key()) + return self + + def audit_as(self, comment: str): + MutationIntent(comment) + self._comment = comment + return self + + async def save(self, context): + # Validate the root reason before ID allocation or provider access. + MutationIntent(self._comment) + # Reserve stable internal IDs before the transaction and before policy + # review. A relation to a new parent must have the same identity in the + # reviewed graph plan and in the commands eventually sent to SQL. + await self._teaql_reserve_graph_ids(context, set()) + await self._teaql_ensure_business_ids(context) + return await context.execute_graph_save( + lambda session: self._teaql_preflight_and_save(session.context, session), comment=self._comment) + + async def _teaql_ensure_business_ids(self, context): + if self._action != "Create": + return + + async def _teaql_reserve_graph_ids(self, context, visited): + object_identity = id(self) + if object_identity in visited: + return + visited.add(object_identity) + if self._action == "Create" and getattr(self, "id", None) is None: + service = context.require_resource("dataService") + allocator = getattr(service, "next_id", None) + if not callable(allocator): + raise RuntimeError( + "Configured dataService does not support stable ID reservation" + ) + old_key = self._teaql_entity_key() + self.id = int(await allocator("Platform")) + self._loaded_fields.add("id") + self._ledger_id = self.id + new_key = self._teaql_entity_key() + self._entity_root.rekey(old_key, new_key) + self._entity_root.set(new_key, "id", Value.I64(self.id)) + for child in self._order_status_list: + child._teaql_attach_root(self._entity_root) + await child._teaql_reserve_graph_ids(context, visited) + for child in self._customer_list: + child._teaql_attach_root(self._entity_root) + await child._teaql_reserve_graph_ids(context, visited) + + async def _teaql_preflight_and_save(self, context, session): + self._teaql_preflight_graph(context, session) + return await self._teaql_save_within_graph(context, session) + + def _teaql_build_command(self): + payload = {} + if "id" in self._loaded_fields: + payload["id"] = Value.I64(self.id) + + if "name" in self._loaded_fields: + payload["name"] = Value.Text(self.name) + + if "version" in self._loaded_fields: + payload["version"] = Value.I64(self.version) + + action = self._action + if action == "Update": + ledger = dict(self._entity_root.current_change_set().changes()).get(self._teaql_entity_key(), {}) + payload = {field: value for field, value in ledger.items() if field not in ("id", "version")} + if action == "Create": + cmd = InsertCommand("Platform", payload) + elif action == "Update": + cmd = UpdateCommand("Platform", Value.from_any(getattr(self, "id", None)), getattr(self, "version", None)) + for key, value in payload.items(): + if key not in ("id", "version"): cmd.value(key, value) + else: + cmd = DeleteCommand("Platform", Value.from_any(getattr(self, "id", None)), getattr(self, "version", None)) + if action != "Create": + cmd.old_values = LoadedScalarSnapshots.values(self) + return action, cmd + + def _teaql_preflight_graph(self, context, session, parent=None): + if parent is None: + MutationIntent(self._comment) + scope = session.scope(self._teaql_entity_key(), parent, self._comment) + session.capture_loaded_values("Platform", LoadedScalarSnapshots.values(self)) + if self._entity_root.has_pending(self._teaql_entity_key()): + if self._action == "Update": + if "id" not in self._loaded_fields: + raise CheckException([CheckResult("invalid_type", ObjectLocation().property("id"), message="Mutation requires a fully loaded entity")]) + if "name" not in self._loaded_fields: + raise CheckException([CheckResult("invalid_type", ObjectLocation().property("name"), message="Mutation requires a fully loaded entity")]) + if "version" not in self._loaded_fields: + raise CheckException([CheckResult("invalid_type", ObjectLocation().property("version"), message="Mutation requires a fully loaded entity")]) + _action, cmd = self._teaql_build_command() + try: + context.preflight_mutation(cmd) + finally: + for field, value in getattr(cmd, "values", {}).items(): + if field not in ("id", "version"): + self._entity_root.set(self._teaql_entity_key(), field, value) + for index, child in enumerate(self._order_status_list): + child._teaql_attach_root(self._entity_root) + child.update_platform(self) + try: + child._teaql_preflight_graph(context, session, scope) + except CheckException as error: + prefix = ObjectLocation().property("order_status_list").index(index) + raise CheckException([ + CheckResult(v.rule_id, v.location.prefixed_by(prefix), v.input_value, v.system_value, v.message) + for v in error.violations + ]) from error + for index, child in enumerate(self._customer_list): + child._teaql_attach_root(self._entity_root) + child.update_platform(self) + try: + child._teaql_preflight_graph(context, session, scope) + except CheckException as error: + prefix = ObjectLocation().property("customer_list").index(index) + raise CheckException([ + CheckResult(v.rule_id, v.location.prefixed_by(prefix), v.input_value, v.system_value, v.message) + for v in error.violations + ]) from error + + async def _teaql_save_within_graph(self, context, session, parent=None): + if parent is None: + MutationIntent(self._comment) + + self._teaql_attach_root(self._entity_root) + scope = session.scope(self._teaql_entity_key(), parent, self._comment) + if not self._entity_root.has_pending(self._teaql_entity_key()): + await self._teaql_cascade_children(context, session, scope) + return self + action, cmd = self._teaql_build_command() + + + req = session.request(cmd, scope, self._entity_root, self._teaql_entity_key()) + + try: + context.check_and_fix_mutation(cmd) + finally: + for field, value in getattr(cmd, "values", {}).items(): + if field not in ("id", "version"): + self._entity_root.set(self._teaql_entity_key(), field, value) + context.mark_mutation_checked(cmd) + service = context.require_resource("dataService") + result = await service.mutate(context, req) + persisted = result.persisted_record + if persisted is None: + raise RuntimeError( + "Mutation provider did not return authoritative persisted state for Platform" + ) + rollback_payload = {field: getattr(self, field, None) for field in self._loaded_fields | {"id", "version"}} + rollback_snapshot = LoadedScalarSnapshots.values(self) + rollback_ledger_id = self._ledger_id + rollback_action = self._action + rollback_loaded_fields = set(self._loaded_fields) + old_key = self._teaql_entity_key() + if "id" in persisted: + self.id = persisted["id"] + self._loaded_fields.add("id") + elif "id" in persisted: + self.id = persisted["id"] + self._loaded_fields.add("id") + if "name" in persisted: + self.name = persisted["name"] + self._loaded_fields.add("name") + elif "name" in persisted: + self.name = persisted["name"] + self._loaded_fields.add("name") + if "version" in persisted: + self.version = persisted["version"] + self._loaded_fields.add("version") + elif "version" in persisted: + self.version = persisted["version"] + self._loaded_fields.add("version") + self._ledger_id = getattr(self, "id", self._ledger_id) + new_key = self._teaql_entity_key() + if old_key != new_key: + self._entity_root.rekey(old_key, new_key) + def rollback_entity(): + for field, value in rollback_payload.items(): + setattr(self, field, value) + self._ledger_id = rollback_ledger_id + self._action = rollback_action + self._loaded_fields = rollback_loaded_fields + LoadedScalarSnapshots.restore(self, rollback_snapshot) + if old_key != new_key: + self._entity_root.rekey(new_key, old_key) + session.after_rollback(rollback_entity) + if action != "Delete": + self._action = "Update" + + if action != "Delete": + await self._teaql_cascade_children(context, session, scope) + def commit_entity(): + LoadedScalarSnapshots.capture(self, self._teaql_snapshot_values()) + self._entity_root.clear_entity(new_key) + if getattr(self, "version", None) is not None: + self._entity_root.accept_committed_version(new_key, int(self.version)) + session.after_commit(commit_entity) + return self + + async def _teaql_cascade_children(self, context, session, scope): + cascade_relations = [] + cascade_relations.append(("order_status_list", self._order_status_list, "update_platform")) + cascade_relations.append(("customer_list", self._customer_list, "update_platform")) + for relation_name, children, updater in cascade_relations: + for index, child in enumerate(children): + child._teaql_attach_root(self._entity_root) + getattr(child, updater)(self) + try: + await child._teaql_save_within_graph(context, session, scope) + except CheckException as error: + prefix = ObjectLocation().property(relation_name).index(index) + raise CheckException([ + CheckResult( + violation.rule_id, + violation.location.prefixed_by(prefix), + violation.input_value, + violation.system_value, + violation.message, + ) + for violation in error.violations + ]) from error + + def update_id(self, value): + if self._action != "Create": + raise ValueError("ENTITY_ID_IMMUTABLE: an existing entity ID cannot be changed") + value = int(value) if isinstance(value, str) and value.isdecimal() else value + if type(value) is not int or not 0 < value <= 18446744073709551615: + raise ValueError("ENTITY_ID_INVALID: an assigned unsigned-64 identity is required") + old_key = self._teaql_entity_key() + self._ledger_id = value + self._entity_root.rekey(old_key, self._teaql_entity_key()) + if "id" in self._loaded_fields and self.id == value: + return self + self.id = value + self._loaded_fields.add("id") + self._entity_root.set(self._teaql_entity_key(), "id", Value.from_any(value)) + return self + + def update_name(self, value): + if "name" in self._loaded_fields and self.name == value: + return self + self.name = value + self._loaded_fields.add("name") + self._entity_root.set(self._teaql_entity_key(), "name", Value.from_any(value)) + return self + + def update_version(self, value): + if "version" in self._loaded_fields and self.version == value: + return self + self.version = value + self._loaded_fields.add("version") + self._entity_root.set(self._teaql_entity_key(), "version", Value.from_any(value)) + return self + def order_status_list(self) -> list: + self._loaded_fields.add("order_status_list") + return self._order_status_list + + def customer_list(self) -> list: + self._loaded_fields.add("customer_list") + return self._customer_list diff --git a/examples/trace-chain/standard-order/lib/requests/__init__.py b/examples/trace-chain/standard-order/lib/requests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/examples/trace-chain/standard-order/lib/requests/customer_order_request.py b/examples/trace-chain/standard-order/lib/requests/customer_order_request.py new file mode 100644 index 0000000..dfcc617 --- /dev/null +++ b/examples/trace-chain/standard-order/lib/requests/customer_order_request.py @@ -0,0 +1,921 @@ +from teaql.core.query import AggregateFunction, RelationAggregate, SelectQuery +from teaql.core.request_tree import QueryOperation +from teaql.core.dynamic_search import merge_generated_scalar_search +from generated_search_models import GENERATED_SEARCH_MODELS +from teaql.core.list import SmartList +from teaql.core.request_intent import QueryIntent +from teaql.runtime import EntityRoot +from teaql.data_service import QueryRequest +from copy import deepcopy +from teaql.core.expr import ( + begin_with, between, column, contain, end_with, eq, gt, gte, + in_list, in_subquery, is_not_null, is_null, lt, lte, ne, not_begin_with, + not_contain, not_end_with, not_in_list, not_in_subquery, value, + sound_like, +) +from models.customer_order import CustomerOrder +from typing import Protocol + +class QuerySelection(Protocol): + query: SelectQuery + +class CustomerOrderRequest: + def __init__(self, minimal=False): + self.query = SelectQuery("CustomerOrder") + self._purpose = None + self._comment = None + self._page_count_alias = None + self._page_count_method = None + self._dynamic_search_warnings = [] + self.query.and_filter(gte("version", 1)) + if minimal: + self.select_id() + self.select_version() + else: + self.select_self_fields() + + def comment(self, c: str): + self.query.comment(c) + self._comment = c + return self + + def filter_by_json(self, source, warn=None): + kwargs = {} if warn is None else {"warn": warn} + query, warnings = merge_generated_scalar_search(self.query, source, GENERATED_SEARCH_MODELS, **kwargs) + self.query = query + self._dynamic_search_warnings.extend(warnings) + return self + + @property + def dynamic_search_warnings(self): + return tuple(dict(warning) for warning in self._dynamic_search_warnings) + + def purpose(self, p: str): + self.query.purpose(p) + self._purpose = p + return ExecutableCustomerOrderRequest(self) + + def optimize_for_continuous_page_fetch(self): + self.query.optimize_for_continuous_page_fetch() + return self + + def optimize_for_continuous_page_fetch_with(self, namespace: str, ttl_seconds: int): + self.query.optimize_for_continuous_page_fetch_with(namespace, ttl_seconds) + return self + + def optimize_pagination_with_id_set(self): + self.query.optimize_pagination_with_id_set() + return self + + def optimize_pagination_with_id_set_config(self, namespace: str, ttl_seconds: int, max_ids: int): + self.query.optimize_pagination_with_id_set_config(namespace, ttl_seconds, max_ids) + return self + + def top_n_probe_parent_threshold(self, threshold: int): + self.query.top_n_probe_parent_threshold(threshold) + return self + + def limit(self, n: int): + self.query.limit(n) + return self + + def offset(self, n: int): + self.query.offset(n) + return self + + def with_deleted_rows(self): + self.query.with_deleted_rows() + return self + + def deleted_rows_only(self): + self.query.deleted_rows_only() + return self + + def select_self_fields(self): + self.query.project("id", "customer", "status", "order_number", "order_date", "total_amount", "version") + return self + + def select_id(self): + self.query.project("id") + return self + + + + def select_order_number(self): + self.query.project("order_number") + return self + + def select_order_date(self): + self.query.project("order_date") + return self + + def select_total_amount(self): + self.query.project("total_amount") + return self + + def select_version(self): + self.query.project("version") + return self + + def select_customer_with(self, child_request): + self.query.project("customer") + self.query.relation_query("customer", child_request.query, + QueryOperation("relation-select", "select_customer_with", relation="customer")) + return self + def select_status_with(self, child_request): + self.query.project("status") + self.query.relation_query("status", child_request.query, + QueryOperation("relation-select", "select_status_with", relation="status")) + return self + def with_customer_matching(self, child_request): + child_request.query.projection = ["id"] + self.query.and_filter(in_subquery(column("customer"), "Customer", child_request.query)) + return self + + def without_customer_matching(self, child_request): + child_request.query.projection = ["id"] + self.query.and_filter(not_in_subquery(column("customer"), "Customer", child_request.query)) + return self + + def have_customer(self): + self.query.and_filter(is_not_null(column("customer"))) + return self + + def have_no_customer(self): + self.query.and_filter(is_null(column("customer"))) + return self + def with_status_matching(self, child_request): + child_request.query.projection = ["id"] + self.query.and_filter(in_subquery(column("status"), "OrderStatus", child_request.query)) + return self + + def without_status_matching(self, child_request): + child_request.query.projection = ["id"] + self.query.and_filter(not_in_subquery(column("status"), "OrderStatus", child_request.query)) + return self + + def have_status(self): + self.query.and_filter(is_not_null(column("status"))) + return self + + def have_no_status(self): + self.query.and_filter(is_null(column("status"))) + return self + + def with_id_is(self, val): + self.query.and_filter(eq("id", val)) + return self + + def with_id_is_not(self, val): + self.query.and_filter(ne("id", val)) + return self + + def with_id_in(self, *vals): + self.query.and_filter(in_list("id", list(vals))) + return self + + def with_id_not_in(self, *vals): + self.query.and_filter(not_in_list("id", list(vals))) + return self + + def with_id_greater_than(self, val): + self.query.and_filter(gt("id", val)) + return self + + def with_id_greater_than_or_equal_to(self, val): + self.query.and_filter(gte("id", val)) + return self + + def with_id_less_than(self, val): + self.query.and_filter(lt("id", val)) + return self + + def with_id_less_than_or_equal_to(self, val): + self.query.and_filter(lte("id", val)) + return self + + def with_id_between(self, lower, upper): + self.query.and_filter(between(column("id"), value(lower), value(upper))) + return self + + def with_id_is_known(self): + self.query.and_filter(is_not_null(column("id"))) + return self + + def with_id_is_unknown(self): + self.query.and_filter(is_null(column("id"))) + return self + + def filter_by_customer(self, val): + self.query.and_filter(eq("customer", val)) + return self + + def filter_by_status(self, val): + self.query.and_filter(eq("status", val)) + return self + def with_status_is_new(self): + self.query.and_filter(eq("status", 1001)) + return self + def with_status_is_paid(self): + self.query.and_filter(eq("status", 1002)) + return self + def with_status_is_cancelled(self): + self.query.and_filter(eq("status", 1003)) + return self + + def with_order_number_containing(self, val: str): + self.query.and_filter(contain("order_number", val)) + return self + + def with_order_number_not_containing(self, val: str): + self.query.and_filter(not_contain("order_number", val)) + return self + + def with_order_number_starting_with(self, val: str): + self.query.and_filter(begin_with("order_number", val)) + return self + + def with_order_number_not_starting_with(self, val: str): + self.query.and_filter(not_begin_with("order_number", val)) + return self + + def with_order_number_ending_with(self, val: str): + self.query.and_filter(end_with("order_number", val)) + return self + + def with_order_number_not_ending_with(self, val: str): + self.query.and_filter(not_end_with("order_number", val)) + return self + + def with_order_number_sounding_like(self, val: str): + self.query.and_filter(sound_like("order_number", val)) + return self + + def with_order_number_is(self, val: str): + self.query.and_filter(eq("order_number", val)) + return self + def with_order_number_is_not(self, val): + self.query.and_filter(ne("order_number", val)) + return self + + def with_order_number_in(self, *vals): + self.query.and_filter(in_list("order_number", list(vals))) + return self + + def with_order_number_not_in(self, *vals): + self.query.and_filter(not_in_list("order_number", list(vals))) + return self + + def with_order_number_greater_than(self, val): + self.query.and_filter(gt("order_number", val)) + return self + + def with_order_number_greater_than_or_equal_to(self, val): + self.query.and_filter(gte("order_number", val)) + return self + + def with_order_number_less_than(self, val): + self.query.and_filter(lt("order_number", val)) + return self + + def with_order_number_less_than_or_equal_to(self, val): + self.query.and_filter(lte("order_number", val)) + return self + + def with_order_number_between(self, lower, upper): + self.query.and_filter(between(column("order_number"), value(lower), value(upper))) + return self + + def with_order_number_is_known(self): + self.query.and_filter(is_not_null(column("order_number"))) + return self + + def with_order_number_is_unknown(self): + self.query.and_filter(is_null(column("order_number"))) + return self + + def with_order_date_is(self, val): + self.query.and_filter(eq("order_date", val)) + return self + + def with_order_date_is_not(self, val): + self.query.and_filter(ne("order_date", val)) + return self + + def with_order_date_in(self, *vals): + self.query.and_filter(in_list("order_date", list(vals))) + return self + + def with_order_date_not_in(self, *vals): + self.query.and_filter(not_in_list("order_date", list(vals))) + return self + + def with_order_date_greater_than(self, val): + self.query.and_filter(gt("order_date", val)) + return self + + def with_order_date_greater_than_or_equal_to(self, val): + self.query.and_filter(gte("order_date", val)) + return self + + def with_order_date_less_than(self, val): + self.query.and_filter(lt("order_date", val)) + return self + + def with_order_date_less_than_or_equal_to(self, val): + self.query.and_filter(lte("order_date", val)) + return self + + def with_order_date_between(self, lower, upper): + self.query.and_filter(between(column("order_date"), value(lower), value(upper))) + return self + + def with_order_date_is_known(self): + self.query.and_filter(is_not_null(column("order_date"))) + return self + + def with_order_date_is_unknown(self): + self.query.and_filter(is_null(column("order_date"))) + return self + + def with_total_amount_is(self, val): + self.query.and_filter(eq("total_amount", val)) + return self + + def with_total_amount_is_not(self, val): + self.query.and_filter(ne("total_amount", val)) + return self + + def with_total_amount_in(self, *vals): + self.query.and_filter(in_list("total_amount", list(vals))) + return self + + def with_total_amount_not_in(self, *vals): + self.query.and_filter(not_in_list("total_amount", list(vals))) + return self + + def with_total_amount_greater_than(self, val): + self.query.and_filter(gt("total_amount", val)) + return self + + def with_total_amount_greater_than_or_equal_to(self, val): + self.query.and_filter(gte("total_amount", val)) + return self + + def with_total_amount_less_than(self, val): + self.query.and_filter(lt("total_amount", val)) + return self + + def with_total_amount_less_than_or_equal_to(self, val): + self.query.and_filter(lte("total_amount", val)) + return self + + def with_total_amount_between(self, lower, upper): + self.query.and_filter(between(column("total_amount"), value(lower), value(upper))) + return self + + def with_total_amount_is_known(self): + self.query.and_filter(is_not_null(column("total_amount"))) + return self + + def with_total_amount_is_unknown(self): + self.query.and_filter(is_null(column("total_amount"))) + return self + + def with_version_is(self, val): + self.query.and_filter(eq("version", val)) + return self + + def with_version_is_not(self, val): + self.query.and_filter(ne("version", val)) + return self + + def with_version_in(self, *vals): + self.query.and_filter(in_list("version", list(vals))) + return self + + def with_version_not_in(self, *vals): + self.query.and_filter(not_in_list("version", list(vals))) + return self + + def with_version_greater_than(self, val): + self.query.and_filter(gt("version", val)) + return self + + def with_version_greater_than_or_equal_to(self, val): + self.query.and_filter(gte("version", val)) + return self + + def with_version_less_than(self, val): + self.query.and_filter(lt("version", val)) + return self + + def with_version_less_than_or_equal_to(self, val): + self.query.and_filter(lte("version", val)) + return self + + def with_version_between(self, lower, upper): + self.query.and_filter(between(column("version"), value(lower), value(upper))) + return self + + def with_version_is_known(self): + self.query.and_filter(is_not_null(column("version"))) + return self + + def with_version_is_unknown(self): + self.query.and_filter(is_null(column("version"))) + return self + + def order_by_id_ascending(self): + self.query.order_by("id", "asc") + return self + + def order_by_id_descending(self): + self.query.order_by("id", "desc") + return self + + + + def order_by_order_number_ascending(self): + self.query.order_by("order_number", "asc") + return self + + def order_by_order_number_descending(self): + self.query.order_by("order_number", "desc") + return self + + def order_by_order_date_ascending(self): + self.query.order_by("order_date", "asc") + return self + + def order_by_order_date_descending(self): + self.query.order_by("order_date", "desc") + return self + + def order_by_total_amount_ascending(self): + self.query.order_by("total_amount", "asc") + return self + + def order_by_total_amount_descending(self): + self.query.order_by("total_amount", "desc") + return self + + def order_by_version_ascending(self): + self.query.order_by("version", "asc") + return self + + def order_by_version_descending(self): + self.query.order_by("version", "desc") + return self + + + def count(self): + self.query.count_field("id", "count") + self._page_count_alias = "count" + self._page_count_method = "count" + return self + + def count_as(self, ret_name: str): + self.query.count_field("id", ret_name) + self._page_count_alias = ret_name + self._page_count_method = "count_as" + return self + + def min_total_amount(self): + return self.min_total_amount_as("minOfTotalAmount") + + def min_total_amount_as(self, ret_name: str): + self.query.min("total_amount", ret_name) + return self + def max_total_amount(self): + return self.max_total_amount_as("maxOfTotalAmount") + + def max_total_amount_as(self, ret_name: str): + self.query.max("total_amount", ret_name) + return self + def sum_total_amount(self): + return self.sum_total_amount_as("sumOfTotalAmount") + + def sum_total_amount_as(self, ret_name: str): + self.query.sum("total_amount", ret_name) + return self + def avg_total_amount(self): + return self.avg_total_amount_as("avgOfTotalAmount") + + def avg_total_amount_as(self, ret_name: str): + self.query.avg("total_amount", ret_name) + return self + def standardDeviation_total_amount(self): + return self.standardDeviation_total_amount_as("standardDeviationOfTotalAmount") + + def standardDeviation_total_amount_as(self, ret_name: str): + self.query.standardDeviation("total_amount", ret_name) + return self + def squareRootOfPopulationStandardDeviation_total_amount(self): + return self.squareRootOfPopulationStandardDeviation_total_amount_as("squareRootOfPopulationStandardDeviationOfTotalAmount") + + def squareRootOfPopulationStandardDeviation_total_amount_as(self, ret_name: str): + self.query.squareRootOfPopulationStandardDeviation("total_amount", ret_name) + return self + def sampleVariance_total_amount(self): + return self.sampleVariance_total_amount_as("sampleVarianceOfTotalAmount") + + def sampleVariance_total_amount_as(self, ret_name: str): + self.query.sampleVariance("total_amount", ret_name) + return self + def samplePopulationVariance_total_amount(self): + return self.samplePopulationVariance_total_amount_as("samplePopulationVarianceOfTotalAmount") + + def samplePopulationVariance_total_amount_as(self, ret_name: str): + self.query.samplePopulationVariance("total_amount", ret_name) + return self + def group_by_id(self): + self.query.group_by("id") + return self + + def group_by_id_as(self, ret_name: str): + self.query.group_by("id") + return self + def group_by_customer(self): + self.query.group_by("customer") + return self + + def group_by_customer_as(self, ret_name: str): + self.query.group_by("customer") + return self + def group_by_status(self): + self.query.group_by("status") + return self + + def group_by_status_as(self, ret_name: str): + self.query.group_by("status") + return self + def group_by_order_number(self): + self.query.group_by("order_number") + return self + + def group_by_order_number_as(self, ret_name: str): + self.query.group_by("order_number") + return self + def group_by_order_date(self): + self.query.group_by("order_date") + return self + + def group_by_order_date_as(self, ret_name: str): + self.query.group_by("order_date") + return self + def group_by_total_amount(self): + self.query.group_by("total_amount") + return self + + def group_by_total_amount_as(self, ret_name: str): + self.query.group_by("total_amount") + return self + def group_by_version(self): + self.query.group_by("version") + return self + + def group_by_version_as(self, ret_name: str): + self.query.group_by("version") + return self + def select_order_item_list(self): + from requests.order_item_request import OrderItemRequest + return self.select_order_item_list_with(OrderItemRequest()) + + def select_order_item_list_with(self, child_request): + self.query.relation_query("order_item_list", child_request.query, + QueryOperation("relation-select", "select_order_item_list_with", relation="order_item_list")) + return self + def have_order_items(self): + from requests.order_item_request import OrderItemRequest + return self.with_order_item_list_matching(OrderItemRequest()) + + def have_no_order_items(self): + from requests.order_item_request import OrderItemRequest + return self.without_order_item_list_matching(OrderItemRequest()) + + def with_order_item_list_matching(self, child_request): + child_request.query.projection = ["customer_order"] + self.query.and_filter(in_subquery(column("id"), "OrderItem", child_request.query)) + return self + + def without_order_item_list_matching(self, child_request): + child_request.query.projection = ["customer_order"] + self.query.and_filter(not_in_subquery(column("id"), "OrderItem", child_request.query)) + return self + def count_order_items(self): + return self.count_order_items_as("count_order_items") + + def count_order_items_as(self, alias: str): + from requests.order_item_request import OrderItemRequest + self.count_order_items_with(alias, OrderItemRequest()) + self.query.relation_aggregates[-1].operation = QueryOperation( + "relation-count", "count_order_items_as", alias, "order_item_list") + return self + + def count_order_items_with(self, alias: str, child_request): + query = deepcopy(child_request.query) + query.aggregates = [] + query.count_field("id", alias) + self.query.relation_aggregates.append( + RelationAggregate("order_item_list", alias, query, True, + QueryOperation("relation-count", "count_order_items_with", alias, "order_item_list")) + ) + return self + + def min_quantity_of_order_items(self): + from requests.order_item_request import OrderItemRequest + return self.min_quantity_of_order_items_as( + "min_quantity_of_order_items", OrderItemRequest()) + + def min_quantity_of_order_items_as(self, alias: str, child_request): + query = deepcopy(child_request.query) + query.aggregates = [] + query.min("quantity", alias) + self.query.relation_aggregates.append( + RelationAggregate("order_item_list", alias, query, True, + QueryOperation("aggregate", "min_quantity_of_order_items_as", alias, "order_item_list")) + ) + return self + def max_quantity_of_order_items(self): + from requests.order_item_request import OrderItemRequest + return self.max_quantity_of_order_items_as( + "max_quantity_of_order_items", OrderItemRequest()) + + def max_quantity_of_order_items_as(self, alias: str, child_request): + query = deepcopy(child_request.query) + query.aggregates = [] + query.max("quantity", alias) + self.query.relation_aggregates.append( + RelationAggregate("order_item_list", alias, query, True, + QueryOperation("aggregate", "max_quantity_of_order_items_as", alias, "order_item_list")) + ) + return self + def sum_quantity_of_order_items(self): + from requests.order_item_request import OrderItemRequest + return self.sum_quantity_of_order_items_as( + "sum_quantity_of_order_items", OrderItemRequest()) + + def sum_quantity_of_order_items_as(self, alias: str, child_request): + query = deepcopy(child_request.query) + query.aggregates = [] + query.sum("quantity", alias) + self.query.relation_aggregates.append( + RelationAggregate("order_item_list", alias, query, True, + QueryOperation("aggregate", "sum_quantity_of_order_items_as", alias, "order_item_list")) + ) + return self + def avg_quantity_of_order_items(self): + from requests.order_item_request import OrderItemRequest + return self.avg_quantity_of_order_items_as( + "avg_quantity_of_order_items", OrderItemRequest()) + + def avg_quantity_of_order_items_as(self, alias: str, child_request): + query = deepcopy(child_request.query) + query.aggregates = [] + query.avg("quantity", alias) + self.query.relation_aggregates.append( + RelationAggregate("order_item_list", alias, query, True, + QueryOperation("aggregate", "avg_quantity_of_order_items_as", alias, "order_item_list")) + ) + return self + def standardDeviation_quantity_of_order_items(self): + from requests.order_item_request import OrderItemRequest + return self.standardDeviation_quantity_of_order_items_as( + "standardDeviation_quantity_of_order_items", OrderItemRequest()) + + def standardDeviation_quantity_of_order_items_as(self, alias: str, child_request): + query = deepcopy(child_request.query) + query.aggregates = [] + query.standardDeviation("quantity", alias) + self.query.relation_aggregates.append( + RelationAggregate("order_item_list", alias, query, True, + QueryOperation("aggregate", "standardDeviation_quantity_of_order_items_as", alias, "order_item_list")) + ) + return self + def squareRootOfPopulationStandardDeviation_quantity_of_order_items(self): + from requests.order_item_request import OrderItemRequest + return self.squareRootOfPopulationStandardDeviation_quantity_of_order_items_as( + "squareRootOfPopulationStandardDeviation_quantity_of_order_items", OrderItemRequest()) + + def squareRootOfPopulationStandardDeviation_quantity_of_order_items_as(self, alias: str, child_request): + query = deepcopy(child_request.query) + query.aggregates = [] + query.squareRootOfPopulationStandardDeviation("quantity", alias) + self.query.relation_aggregates.append( + RelationAggregate("order_item_list", alias, query, True, + QueryOperation("aggregate", "squareRootOfPopulationStandardDeviation_quantity_of_order_items_as", alias, "order_item_list")) + ) + return self + def sampleVariance_quantity_of_order_items(self): + from requests.order_item_request import OrderItemRequest + return self.sampleVariance_quantity_of_order_items_as( + "sampleVariance_quantity_of_order_items", OrderItemRequest()) + + def sampleVariance_quantity_of_order_items_as(self, alias: str, child_request): + query = deepcopy(child_request.query) + query.aggregates = [] + query.sampleVariance("quantity", alias) + self.query.relation_aggregates.append( + RelationAggregate("order_item_list", alias, query, True, + QueryOperation("aggregate", "sampleVariance_quantity_of_order_items_as", alias, "order_item_list")) + ) + return self + def samplePopulationVariance_quantity_of_order_items(self): + from requests.order_item_request import OrderItemRequest + return self.samplePopulationVariance_quantity_of_order_items_as( + "samplePopulationVariance_quantity_of_order_items", OrderItemRequest()) + + def samplePopulationVariance_quantity_of_order_items_as(self, alias: str, child_request): + query = deepcopy(child_request.query) + query.aggregates = [] + query.samplePopulationVariance("quantity", alias) + self.query.relation_aggregates.append( + RelationAggregate("order_item_list", alias, query, True, + QueryOperation("aggregate", "samplePopulationVariance_quantity_of_order_items_as", alias, "order_item_list")) + ) + return self + def min_unit_price_of_order_items(self): + from requests.order_item_request import OrderItemRequest + return self.min_unit_price_of_order_items_as( + "min_unit_price_of_order_items", OrderItemRequest()) + + def min_unit_price_of_order_items_as(self, alias: str, child_request): + query = deepcopy(child_request.query) + query.aggregates = [] + query.min("unit_price", alias) + self.query.relation_aggregates.append( + RelationAggregate("order_item_list", alias, query, True, + QueryOperation("aggregate", "min_unit_price_of_order_items_as", alias, "order_item_list")) + ) + return self + def max_unit_price_of_order_items(self): + from requests.order_item_request import OrderItemRequest + return self.max_unit_price_of_order_items_as( + "max_unit_price_of_order_items", OrderItemRequest()) + + def max_unit_price_of_order_items_as(self, alias: str, child_request): + query = deepcopy(child_request.query) + query.aggregates = [] + query.max("unit_price", alias) + self.query.relation_aggregates.append( + RelationAggregate("order_item_list", alias, query, True, + QueryOperation("aggregate", "max_unit_price_of_order_items_as", alias, "order_item_list")) + ) + return self + def sum_unit_price_of_order_items(self): + from requests.order_item_request import OrderItemRequest + return self.sum_unit_price_of_order_items_as( + "sum_unit_price_of_order_items", OrderItemRequest()) + + def sum_unit_price_of_order_items_as(self, alias: str, child_request): + query = deepcopy(child_request.query) + query.aggregates = [] + query.sum("unit_price", alias) + self.query.relation_aggregates.append( + RelationAggregate("order_item_list", alias, query, True, + QueryOperation("aggregate", "sum_unit_price_of_order_items_as", alias, "order_item_list")) + ) + return self + def avg_unit_price_of_order_items(self): + from requests.order_item_request import OrderItemRequest + return self.avg_unit_price_of_order_items_as( + "avg_unit_price_of_order_items", OrderItemRequest()) + + def avg_unit_price_of_order_items_as(self, alias: str, child_request): + query = deepcopy(child_request.query) + query.aggregates = [] + query.avg("unit_price", alias) + self.query.relation_aggregates.append( + RelationAggregate("order_item_list", alias, query, True, + QueryOperation("aggregate", "avg_unit_price_of_order_items_as", alias, "order_item_list")) + ) + return self + def standardDeviation_unit_price_of_order_items(self): + from requests.order_item_request import OrderItemRequest + return self.standardDeviation_unit_price_of_order_items_as( + "standardDeviation_unit_price_of_order_items", OrderItemRequest()) + + def standardDeviation_unit_price_of_order_items_as(self, alias: str, child_request): + query = deepcopy(child_request.query) + query.aggregates = [] + query.standardDeviation("unit_price", alias) + self.query.relation_aggregates.append( + RelationAggregate("order_item_list", alias, query, True, + QueryOperation("aggregate", "standardDeviation_unit_price_of_order_items_as", alias, "order_item_list")) + ) + return self + def squareRootOfPopulationStandardDeviation_unit_price_of_order_items(self): + from requests.order_item_request import OrderItemRequest + return self.squareRootOfPopulationStandardDeviation_unit_price_of_order_items_as( + "squareRootOfPopulationStandardDeviation_unit_price_of_order_items", OrderItemRequest()) + + def squareRootOfPopulationStandardDeviation_unit_price_of_order_items_as(self, alias: str, child_request): + query = deepcopy(child_request.query) + query.aggregates = [] + query.squareRootOfPopulationStandardDeviation("unit_price", alias) + self.query.relation_aggregates.append( + RelationAggregate("order_item_list", alias, query, True, + QueryOperation("aggregate", "squareRootOfPopulationStandardDeviation_unit_price_of_order_items_as", alias, "order_item_list")) + ) + return self + def sampleVariance_unit_price_of_order_items(self): + from requests.order_item_request import OrderItemRequest + return self.sampleVariance_unit_price_of_order_items_as( + "sampleVariance_unit_price_of_order_items", OrderItemRequest()) + + def sampleVariance_unit_price_of_order_items_as(self, alias: str, child_request): + query = deepcopy(child_request.query) + query.aggregates = [] + query.sampleVariance("unit_price", alias) + self.query.relation_aggregates.append( + RelationAggregate("order_item_list", alias, query, True, + QueryOperation("aggregate", "sampleVariance_unit_price_of_order_items_as", alias, "order_item_list")) + ) + return self + def samplePopulationVariance_unit_price_of_order_items(self): + from requests.order_item_request import OrderItemRequest + return self.samplePopulationVariance_unit_price_of_order_items_as( + "samplePopulationVariance_unit_price_of_order_items", OrderItemRequest()) + + def samplePopulationVariance_unit_price_of_order_items_as(self, alias: str, child_request): + query = deepcopy(child_request.query) + query.aggregates = [] + query.samplePopulationVariance("unit_price", alias) + self.query.relation_aggregates.append( + RelationAggregate("order_item_list", alias, query, True, + QueryOperation("aggregate", "samplePopulationVariance_unit_price_of_order_items_as", alias, "order_item_list")) + ) + return self + def facet_by_customer_as(self, name: str, request: QuerySelection, + include_all_facets: bool = True): + self.query.facet_by(name, "customer", request.query, include_all_facets, + QueryOperation("facet", "facet_by_customer_as", name, "customer")) + return self + + def facet_by_status_as(self, name: str, request: QuerySelection, + include_all_facets: bool = True): + self.query.facet_by(name, "status", request.query, include_all_facets, + QueryOperation("facet", "facet_by_status_as", name, "status")) + return self + + +class ExecutableCustomerOrderRequest: + def __init__(self, request): + self._request = request + + def comment(self, c: str): + self._request.comment(c) + return self + + def new_entity(self, context) -> CustomerOrder: + request = self._request + if not request._comment or not request._comment.strip() or not request._purpose or not request._purpose.strip(): + raise ValueError("Security audit failure: non-empty comment() and purpose() are required before new_entity()") + entity = context.initialize_entity("CustomerOrder", CustomerOrder()) + if not isinstance(entity, CustomerOrder): + raise TypeError("entity initializer returned an incompatible CustomerOrder") + return entity + + async def execute_for_result(self, context): + self = self._request + if not self._purpose or not self._purpose.strip() or not self._comment or not self._comment.strip(): + raise Exception("Security audit failure: comment() and purpose() must be called before execute_for_rows()") + service = context.require_resource("dataService") + req = QueryRequest(context.prepare_query(self.query), _comment=self._comment, _purpose=self._purpose) + return await service.query(context, req) + + async def execute_for_rows(self, context): + return (await self.execute_for_result(context)).rows + + async def execute_for_list(self, context) -> SmartList[CustomerOrder]: + from teaql.runtime.query_list import execute_typed_list + request = self._request + QueryIntent(request._comment, request._purpose) + query = deepcopy(request.query) + authorized = context.prepare_query(query) + root = QueryRequest(authorized, _comment=request._comment, _purpose=request._purpose) + operation = (QueryOperation("named-count", request._page_count_method or "count", + request._page_count_alias if request._page_count_method == "count_as" else None) + if request._page_count_alias is not None else None) + return await execute_typed_list(context, root, + lambda row: CustomerOrder(_entity_root=EntityRoot(), **row), + count_operation=operation, count_alias=request._page_count_alias) + + async def execute_for_one(self, context): + request = deepcopy(self._request) + request.limit(1) + entities = await ExecutableCustomerOrderRequest(request).execute_for_list(context) + return entities[0] if entities else None + + def execute_for_stream(self, context, chunk_size: int = 1000): + """Yield entity chunks lazily from the provider cursor.""" + request = deepcopy(self._request) + QueryIntent(request._comment, request._purpose) + service = context.require_resource("dataService") + if not hasattr(service, "query_stream"): + raise RuntimeError("dataService does not implement query_stream") + root = QueryRequest(context.prepare_query(request.query), _comment=request._comment, _purpose=request._purpose) + from teaql.runtime.query_stream import execute_typed_stream + return execute_typed_stream(context, root, + lambda row: CustomerOrder(_entity_root=EntityRoot(), **row), chunk_size) diff --git a/examples/trace-chain/standard-order/lib/requests/customer_request.py b/examples/trace-chain/standard-order/lib/requests/customer_request.py new file mode 100644 index 0000000..0a29996 --- /dev/null +++ b/examples/trace-chain/standard-order/lib/requests/customer_request.py @@ -0,0 +1,585 @@ +from teaql.core.query import AggregateFunction, RelationAggregate, SelectQuery +from teaql.core.request_tree import QueryOperation +from teaql.core.dynamic_search import merge_generated_scalar_search +from generated_search_models import GENERATED_SEARCH_MODELS +from teaql.core.list import SmartList +from teaql.core.request_intent import QueryIntent +from teaql.runtime import EntityRoot +from teaql.data_service import QueryRequest +from copy import deepcopy +from teaql.core.expr import ( + begin_with, between, column, contain, end_with, eq, gt, gte, + in_list, in_subquery, is_not_null, is_null, lt, lte, ne, not_begin_with, + not_contain, not_end_with, not_in_list, not_in_subquery, value, + sound_like, +) +from models.customer import Customer +from typing import Protocol + +class QuerySelection(Protocol): + query: SelectQuery + +class CustomerRequest: + def __init__(self, minimal=False): + self.query = SelectQuery("Customer") + self._purpose = None + self._comment = None + self._page_count_alias = None + self._page_count_method = None + self._dynamic_search_warnings = [] + self.query.and_filter(gte("version", 1)) + if minimal: + self.select_id() + self.select_version() + else: + self.select_self_fields() + + def comment(self, c: str): + self.query.comment(c) + self._comment = c + return self + + def filter_by_json(self, source, warn=None): + kwargs = {} if warn is None else {"warn": warn} + query, warnings = merge_generated_scalar_search(self.query, source, GENERATED_SEARCH_MODELS, **kwargs) + self.query = query + self._dynamic_search_warnings.extend(warnings) + return self + + @property + def dynamic_search_warnings(self): + return tuple(dict(warning) for warning in self._dynamic_search_warnings) + + def purpose(self, p: str): + self.query.purpose(p) + self._purpose = p + return ExecutableCustomerRequest(self) + + def optimize_for_continuous_page_fetch(self): + self.query.optimize_for_continuous_page_fetch() + return self + + def optimize_for_continuous_page_fetch_with(self, namespace: str, ttl_seconds: int): + self.query.optimize_for_continuous_page_fetch_with(namespace, ttl_seconds) + return self + + def optimize_pagination_with_id_set(self): + self.query.optimize_pagination_with_id_set() + return self + + def optimize_pagination_with_id_set_config(self, namespace: str, ttl_seconds: int, max_ids: int): + self.query.optimize_pagination_with_id_set_config(namespace, ttl_seconds, max_ids) + return self + + def top_n_probe_parent_threshold(self, threshold: int): + self.query.top_n_probe_parent_threshold(threshold) + return self + + def limit(self, n: int): + self.query.limit(n) + return self + + def offset(self, n: int): + self.query.offset(n) + return self + + def with_deleted_rows(self): + self.query.with_deleted_rows() + return self + + def deleted_rows_only(self): + self.query.deleted_rows_only() + return self + + def select_self_fields(self): + self.query.project("id", "platform", "name", "version") + return self + + def select_id(self): + self.query.project("id") + return self + + + def select_name(self): + self.query.project("name") + return self + + def select_version(self): + self.query.project("version") + return self + + def select_platform_with(self, child_request): + self.query.project("platform") + self.query.relation_query("platform", child_request.query, + QueryOperation("relation-select", "select_platform_with", relation="platform")) + return self + def with_platform_matching(self, child_request): + child_request.query.projection = ["id"] + self.query.and_filter(in_subquery(column("platform"), "Platform", child_request.query)) + return self + + def without_platform_matching(self, child_request): + child_request.query.projection = ["id"] + self.query.and_filter(not_in_subquery(column("platform"), "Platform", child_request.query)) + return self + + def have_platform(self): + self.query.and_filter(is_not_null(column("platform"))) + return self + + def have_no_platform(self): + self.query.and_filter(is_null(column("platform"))) + return self + + def with_id_is(self, val): + self.query.and_filter(eq("id", val)) + return self + + def with_id_is_not(self, val): + self.query.and_filter(ne("id", val)) + return self + + def with_id_in(self, *vals): + self.query.and_filter(in_list("id", list(vals))) + return self + + def with_id_not_in(self, *vals): + self.query.and_filter(not_in_list("id", list(vals))) + return self + + def with_id_greater_than(self, val): + self.query.and_filter(gt("id", val)) + return self + + def with_id_greater_than_or_equal_to(self, val): + self.query.and_filter(gte("id", val)) + return self + + def with_id_less_than(self, val): + self.query.and_filter(lt("id", val)) + return self + + def with_id_less_than_or_equal_to(self, val): + self.query.and_filter(lte("id", val)) + return self + + def with_id_between(self, lower, upper): + self.query.and_filter(between(column("id"), value(lower), value(upper))) + return self + + def with_id_is_known(self): + self.query.and_filter(is_not_null(column("id"))) + return self + + def with_id_is_unknown(self): + self.query.and_filter(is_null(column("id"))) + return self + + def filter_by_platform(self, val): + self.query.and_filter(eq("platform", val)) + return self + + def with_name_containing(self, val: str): + self.query.and_filter(contain("name", val)) + return self + + def with_name_not_containing(self, val: str): + self.query.and_filter(not_contain("name", val)) + return self + + def with_name_starting_with(self, val: str): + self.query.and_filter(begin_with("name", val)) + return self + + def with_name_not_starting_with(self, val: str): + self.query.and_filter(not_begin_with("name", val)) + return self + + def with_name_ending_with(self, val: str): + self.query.and_filter(end_with("name", val)) + return self + + def with_name_not_ending_with(self, val: str): + self.query.and_filter(not_end_with("name", val)) + return self + + def with_name_sounding_like(self, val: str): + self.query.and_filter(sound_like("name", val)) + return self + + def with_name_is(self, val: str): + self.query.and_filter(eq("name", val)) + return self + def with_name_is_not(self, val): + self.query.and_filter(ne("name", val)) + return self + + def with_name_in(self, *vals): + self.query.and_filter(in_list("name", list(vals))) + return self + + def with_name_not_in(self, *vals): + self.query.and_filter(not_in_list("name", list(vals))) + return self + + def with_name_greater_than(self, val): + self.query.and_filter(gt("name", val)) + return self + + def with_name_greater_than_or_equal_to(self, val): + self.query.and_filter(gte("name", val)) + return self + + def with_name_less_than(self, val): + self.query.and_filter(lt("name", val)) + return self + + def with_name_less_than_or_equal_to(self, val): + self.query.and_filter(lte("name", val)) + return self + + def with_name_between(self, lower, upper): + self.query.and_filter(between(column("name"), value(lower), value(upper))) + return self + + def with_name_is_known(self): + self.query.and_filter(is_not_null(column("name"))) + return self + + def with_name_is_unknown(self): + self.query.and_filter(is_null(column("name"))) + return self + + def with_version_is(self, val): + self.query.and_filter(eq("version", val)) + return self + + def with_version_is_not(self, val): + self.query.and_filter(ne("version", val)) + return self + + def with_version_in(self, *vals): + self.query.and_filter(in_list("version", list(vals))) + return self + + def with_version_not_in(self, *vals): + self.query.and_filter(not_in_list("version", list(vals))) + return self + + def with_version_greater_than(self, val): + self.query.and_filter(gt("version", val)) + return self + + def with_version_greater_than_or_equal_to(self, val): + self.query.and_filter(gte("version", val)) + return self + + def with_version_less_than(self, val): + self.query.and_filter(lt("version", val)) + return self + + def with_version_less_than_or_equal_to(self, val): + self.query.and_filter(lte("version", val)) + return self + + def with_version_between(self, lower, upper): + self.query.and_filter(between(column("version"), value(lower), value(upper))) + return self + + def with_version_is_known(self): + self.query.and_filter(is_not_null(column("version"))) + return self + + def with_version_is_unknown(self): + self.query.and_filter(is_null(column("version"))) + return self + + def order_by_id_ascending(self): + self.query.order_by("id", "asc") + return self + + def order_by_id_descending(self): + self.query.order_by("id", "desc") + return self + + + def order_by_name_ascending(self): + self.query.order_by("name", "asc") + return self + + def order_by_name_descending(self): + self.query.order_by("name", "desc") + return self + + def order_by_version_ascending(self): + self.query.order_by("version", "asc") + return self + + def order_by_version_descending(self): + self.query.order_by("version", "desc") + return self + + + def count(self): + self.query.count_field("id", "count") + self._page_count_alias = "count" + self._page_count_method = "count" + return self + + def count_as(self, ret_name: str): + self.query.count_field("id", ret_name) + self._page_count_alias = ret_name + self._page_count_method = "count_as" + return self + + def group_by_id(self): + self.query.group_by("id") + return self + + def group_by_id_as(self, ret_name: str): + self.query.group_by("id") + return self + def group_by_platform(self): + self.query.group_by("platform") + return self + + def group_by_platform_as(self, ret_name: str): + self.query.group_by("platform") + return self + def group_by_name(self): + self.query.group_by("name") + return self + + def group_by_name_as(self, ret_name: str): + self.query.group_by("name") + return self + def group_by_version(self): + self.query.group_by("version") + return self + + def group_by_version_as(self, ret_name: str): + self.query.group_by("version") + return self + def select_customer_order_list(self): + from requests.customer_order_request import CustomerOrderRequest + return self.select_customer_order_list_with(CustomerOrderRequest()) + + def select_customer_order_list_with(self, child_request): + self.query.relation_query("customer_order_list", child_request.query, + QueryOperation("relation-select", "select_customer_order_list_with", relation="customer_order_list")) + return self + def have_customer_orders(self): + from requests.customer_order_request import CustomerOrderRequest + return self.with_customer_order_list_matching(CustomerOrderRequest()) + + def have_no_customer_orders(self): + from requests.customer_order_request import CustomerOrderRequest + return self.without_customer_order_list_matching(CustomerOrderRequest()) + + def with_customer_order_list_matching(self, child_request): + child_request.query.projection = ["customer"] + self.query.and_filter(in_subquery(column("id"), "CustomerOrder", child_request.query)) + return self + + def without_customer_order_list_matching(self, child_request): + child_request.query.projection = ["customer"] + self.query.and_filter(not_in_subquery(column("id"), "CustomerOrder", child_request.query)) + return self + def count_customer_orders(self): + return self.count_customer_orders_as("count_customer_orders") + + def count_customer_orders_as(self, alias: str): + from requests.customer_order_request import CustomerOrderRequest + self.count_customer_orders_with(alias, CustomerOrderRequest()) + self.query.relation_aggregates[-1].operation = QueryOperation( + "relation-count", "count_customer_orders_as", alias, "customer_order_list") + return self + + def count_customer_orders_with(self, alias: str, child_request): + query = deepcopy(child_request.query) + query.aggregates = [] + query.count_field("id", alias) + self.query.relation_aggregates.append( + RelationAggregate("customer_order_list", alias, query, True, + QueryOperation("relation-count", "count_customer_orders_with", alias, "customer_order_list")) + ) + return self + + def min_total_amount_of_customer_orders(self): + from requests.customer_order_request import CustomerOrderRequest + return self.min_total_amount_of_customer_orders_as( + "min_total_amount_of_customer_orders", CustomerOrderRequest()) + + def min_total_amount_of_customer_orders_as(self, alias: str, child_request): + query = deepcopy(child_request.query) + query.aggregates = [] + query.min("total_amount", alias) + self.query.relation_aggregates.append( + RelationAggregate("customer_order_list", alias, query, True, + QueryOperation("aggregate", "min_total_amount_of_customer_orders_as", alias, "customer_order_list")) + ) + return self + def max_total_amount_of_customer_orders(self): + from requests.customer_order_request import CustomerOrderRequest + return self.max_total_amount_of_customer_orders_as( + "max_total_amount_of_customer_orders", CustomerOrderRequest()) + + def max_total_amount_of_customer_orders_as(self, alias: str, child_request): + query = deepcopy(child_request.query) + query.aggregates = [] + query.max("total_amount", alias) + self.query.relation_aggregates.append( + RelationAggregate("customer_order_list", alias, query, True, + QueryOperation("aggregate", "max_total_amount_of_customer_orders_as", alias, "customer_order_list")) + ) + return self + def sum_total_amount_of_customer_orders(self): + from requests.customer_order_request import CustomerOrderRequest + return self.sum_total_amount_of_customer_orders_as( + "sum_total_amount_of_customer_orders", CustomerOrderRequest()) + + def sum_total_amount_of_customer_orders_as(self, alias: str, child_request): + query = deepcopy(child_request.query) + query.aggregates = [] + query.sum("total_amount", alias) + self.query.relation_aggregates.append( + RelationAggregate("customer_order_list", alias, query, True, + QueryOperation("aggregate", "sum_total_amount_of_customer_orders_as", alias, "customer_order_list")) + ) + return self + def avg_total_amount_of_customer_orders(self): + from requests.customer_order_request import CustomerOrderRequest + return self.avg_total_amount_of_customer_orders_as( + "avg_total_amount_of_customer_orders", CustomerOrderRequest()) + + def avg_total_amount_of_customer_orders_as(self, alias: str, child_request): + query = deepcopy(child_request.query) + query.aggregates = [] + query.avg("total_amount", alias) + self.query.relation_aggregates.append( + RelationAggregate("customer_order_list", alias, query, True, + QueryOperation("aggregate", "avg_total_amount_of_customer_orders_as", alias, "customer_order_list")) + ) + return self + def standardDeviation_total_amount_of_customer_orders(self): + from requests.customer_order_request import CustomerOrderRequest + return self.standardDeviation_total_amount_of_customer_orders_as( + "standardDeviation_total_amount_of_customer_orders", CustomerOrderRequest()) + + def standardDeviation_total_amount_of_customer_orders_as(self, alias: str, child_request): + query = deepcopy(child_request.query) + query.aggregates = [] + query.standardDeviation("total_amount", alias) + self.query.relation_aggregates.append( + RelationAggregate("customer_order_list", alias, query, True, + QueryOperation("aggregate", "standardDeviation_total_amount_of_customer_orders_as", alias, "customer_order_list")) + ) + return self + def squareRootOfPopulationStandardDeviation_total_amount_of_customer_orders(self): + from requests.customer_order_request import CustomerOrderRequest + return self.squareRootOfPopulationStandardDeviation_total_amount_of_customer_orders_as( + "squareRootOfPopulationStandardDeviation_total_amount_of_customer_orders", CustomerOrderRequest()) + + def squareRootOfPopulationStandardDeviation_total_amount_of_customer_orders_as(self, alias: str, child_request): + query = deepcopy(child_request.query) + query.aggregates = [] + query.squareRootOfPopulationStandardDeviation("total_amount", alias) + self.query.relation_aggregates.append( + RelationAggregate("customer_order_list", alias, query, True, + QueryOperation("aggregate", "squareRootOfPopulationStandardDeviation_total_amount_of_customer_orders_as", alias, "customer_order_list")) + ) + return self + def sampleVariance_total_amount_of_customer_orders(self): + from requests.customer_order_request import CustomerOrderRequest + return self.sampleVariance_total_amount_of_customer_orders_as( + "sampleVariance_total_amount_of_customer_orders", CustomerOrderRequest()) + + def sampleVariance_total_amount_of_customer_orders_as(self, alias: str, child_request): + query = deepcopy(child_request.query) + query.aggregates = [] + query.sampleVariance("total_amount", alias) + self.query.relation_aggregates.append( + RelationAggregate("customer_order_list", alias, query, True, + QueryOperation("aggregate", "sampleVariance_total_amount_of_customer_orders_as", alias, "customer_order_list")) + ) + return self + def samplePopulationVariance_total_amount_of_customer_orders(self): + from requests.customer_order_request import CustomerOrderRequest + return self.samplePopulationVariance_total_amount_of_customer_orders_as( + "samplePopulationVariance_total_amount_of_customer_orders", CustomerOrderRequest()) + + def samplePopulationVariance_total_amount_of_customer_orders_as(self, alias: str, child_request): + query = deepcopy(child_request.query) + query.aggregates = [] + query.samplePopulationVariance("total_amount", alias) + self.query.relation_aggregates.append( + RelationAggregate("customer_order_list", alias, query, True, + QueryOperation("aggregate", "samplePopulationVariance_total_amount_of_customer_orders_as", alias, "customer_order_list")) + ) + return self + def facet_by_platform_as(self, name: str, request: QuerySelection, + include_all_facets: bool = True): + self.query.facet_by(name, "platform", request.query, include_all_facets, + QueryOperation("facet", "facet_by_platform_as", name, "platform")) + return self + + +class ExecutableCustomerRequest: + def __init__(self, request): + self._request = request + + def comment(self, c: str): + self._request.comment(c) + return self + + def new_entity(self, context) -> Customer: + request = self._request + if not request._comment or not request._comment.strip() or not request._purpose or not request._purpose.strip(): + raise ValueError("Security audit failure: non-empty comment() and purpose() are required before new_entity()") + entity = context.initialize_entity("Customer", Customer()) + if not isinstance(entity, Customer): + raise TypeError("entity initializer returned an incompatible Customer") + return entity + + async def execute_for_result(self, context): + self = self._request + if not self._purpose or not self._purpose.strip() or not self._comment or not self._comment.strip(): + raise Exception("Security audit failure: comment() and purpose() must be called before execute_for_rows()") + service = context.require_resource("dataService") + req = QueryRequest(context.prepare_query(self.query), _comment=self._comment, _purpose=self._purpose) + return await service.query(context, req) + + async def execute_for_rows(self, context): + return (await self.execute_for_result(context)).rows + + async def execute_for_list(self, context) -> SmartList[Customer]: + from teaql.runtime.query_list import execute_typed_list + request = self._request + QueryIntent(request._comment, request._purpose) + query = deepcopy(request.query) + authorized = context.prepare_query(query) + root = QueryRequest(authorized, _comment=request._comment, _purpose=request._purpose) + operation = (QueryOperation("named-count", request._page_count_method or "count", + request._page_count_alias if request._page_count_method == "count_as" else None) + if request._page_count_alias is not None else None) + return await execute_typed_list(context, root, + lambda row: Customer(_entity_root=EntityRoot(), **row), + count_operation=operation, count_alias=request._page_count_alias) + + async def execute_for_one(self, context): + request = deepcopy(self._request) + request.limit(1) + entities = await ExecutableCustomerRequest(request).execute_for_list(context) + return entities[0] if entities else None + + def execute_for_stream(self, context, chunk_size: int = 1000): + """Yield entity chunks lazily from the provider cursor.""" + request = deepcopy(self._request) + QueryIntent(request._comment, request._purpose) + service = context.require_resource("dataService") + if not hasattr(service, "query_stream"): + raise RuntimeError("dataService does not implement query_stream") + root = QueryRequest(context.prepare_query(request.query), _comment=request._comment, _purpose=request._purpose) + from teaql.runtime.query_stream import execute_typed_stream + return execute_typed_stream(context, root, + lambda row: Customer(_entity_root=EntityRoot(), **row), chunk_size) diff --git a/examples/trace-chain/standard-order/lib/requests/order_item_request.py b/examples/trace-chain/standard-order/lib/requests/order_item_request.py new file mode 100644 index 0000000..4b85814 --- /dev/null +++ b/examples/trace-chain/standard-order/lib/requests/order_item_request.py @@ -0,0 +1,650 @@ +from teaql.core.query import AggregateFunction, RelationAggregate, SelectQuery +from teaql.core.request_tree import QueryOperation +from teaql.core.dynamic_search import merge_generated_scalar_search +from generated_search_models import GENERATED_SEARCH_MODELS +from teaql.core.list import SmartList +from teaql.core.request_intent import QueryIntent +from teaql.runtime import EntityRoot +from teaql.data_service import QueryRequest +from copy import deepcopy +from teaql.core.expr import ( + begin_with, between, column, contain, end_with, eq, gt, gte, + in_list, in_subquery, is_not_null, is_null, lt, lte, ne, not_begin_with, + not_contain, not_end_with, not_in_list, not_in_subquery, value, + sound_like, +) +from models.order_item import OrderItem +from typing import Protocol + +class QuerySelection(Protocol): + query: SelectQuery + +class OrderItemRequest: + def __init__(self, minimal=False): + self.query = SelectQuery("OrderItem") + self._purpose = None + self._comment = None + self._page_count_alias = None + self._page_count_method = None + self._dynamic_search_warnings = [] + self.query.and_filter(gte("version", 1)) + if minimal: + self.select_id() + self.select_version() + else: + self.select_self_fields() + + def comment(self, c: str): + self.query.comment(c) + self._comment = c + return self + + def filter_by_json(self, source, warn=None): + kwargs = {} if warn is None else {"warn": warn} + query, warnings = merge_generated_scalar_search(self.query, source, GENERATED_SEARCH_MODELS, **kwargs) + self.query = query + self._dynamic_search_warnings.extend(warnings) + return self + + @property + def dynamic_search_warnings(self): + return tuple(dict(warning) for warning in self._dynamic_search_warnings) + + def purpose(self, p: str): + self.query.purpose(p) + self._purpose = p + return ExecutableOrderItemRequest(self) + + def optimize_for_continuous_page_fetch(self): + self.query.optimize_for_continuous_page_fetch() + return self + + def optimize_for_continuous_page_fetch_with(self, namespace: str, ttl_seconds: int): + self.query.optimize_for_continuous_page_fetch_with(namespace, ttl_seconds) + return self + + def optimize_pagination_with_id_set(self): + self.query.optimize_pagination_with_id_set() + return self + + def optimize_pagination_with_id_set_config(self, namespace: str, ttl_seconds: int, max_ids: int): + self.query.optimize_pagination_with_id_set_config(namespace, ttl_seconds, max_ids) + return self + + def top_n_probe_parent_threshold(self, threshold: int): + self.query.top_n_probe_parent_threshold(threshold) + return self + + def limit(self, n: int): + self.query.limit(n) + return self + + def offset(self, n: int): + self.query.offset(n) + return self + + def with_deleted_rows(self): + self.query.with_deleted_rows() + return self + + def deleted_rows_only(self): + self.query.deleted_rows_only() + return self + + def select_self_fields(self): + self.query.project("id", "customer_order", "product_name", "quantity", "unit_price", "version") + return self + + def select_id(self): + self.query.project("id") + return self + + + def select_product_name(self): + self.query.project("product_name") + return self + + def select_quantity(self): + self.query.project("quantity") + return self + + def select_unit_price(self): + self.query.project("unit_price") + return self + + def select_version(self): + self.query.project("version") + return self + + def select_customer_order_with(self, child_request): + self.query.project("customer_order") + self.query.relation_query("customer_order", child_request.query, + QueryOperation("relation-select", "select_customer_order_with", relation="customer_order")) + return self + def with_customer_order_matching(self, child_request): + child_request.query.projection = ["id"] + self.query.and_filter(in_subquery(column("customer_order"), "CustomerOrder", child_request.query)) + return self + + def without_customer_order_matching(self, child_request): + child_request.query.projection = ["id"] + self.query.and_filter(not_in_subquery(column("customer_order"), "CustomerOrder", child_request.query)) + return self + + def have_customer_order(self): + self.query.and_filter(is_not_null(column("customer_order"))) + return self + + def have_no_customer_order(self): + self.query.and_filter(is_null(column("customer_order"))) + return self + + def with_id_is(self, val): + self.query.and_filter(eq("id", val)) + return self + + def with_id_is_not(self, val): + self.query.and_filter(ne("id", val)) + return self + + def with_id_in(self, *vals): + self.query.and_filter(in_list("id", list(vals))) + return self + + def with_id_not_in(self, *vals): + self.query.and_filter(not_in_list("id", list(vals))) + return self + + def with_id_greater_than(self, val): + self.query.and_filter(gt("id", val)) + return self + + def with_id_greater_than_or_equal_to(self, val): + self.query.and_filter(gte("id", val)) + return self + + def with_id_less_than(self, val): + self.query.and_filter(lt("id", val)) + return self + + def with_id_less_than_or_equal_to(self, val): + self.query.and_filter(lte("id", val)) + return self + + def with_id_between(self, lower, upper): + self.query.and_filter(between(column("id"), value(lower), value(upper))) + return self + + def with_id_is_known(self): + self.query.and_filter(is_not_null(column("id"))) + return self + + def with_id_is_unknown(self): + self.query.and_filter(is_null(column("id"))) + return self + + def filter_by_customer_order(self, val): + self.query.and_filter(eq("customer_order", val)) + return self + + def with_product_name_containing(self, val: str): + self.query.and_filter(contain("product_name", val)) + return self + + def with_product_name_not_containing(self, val: str): + self.query.and_filter(not_contain("product_name", val)) + return self + + def with_product_name_starting_with(self, val: str): + self.query.and_filter(begin_with("product_name", val)) + return self + + def with_product_name_not_starting_with(self, val: str): + self.query.and_filter(not_begin_with("product_name", val)) + return self + + def with_product_name_ending_with(self, val: str): + self.query.and_filter(end_with("product_name", val)) + return self + + def with_product_name_not_ending_with(self, val: str): + self.query.and_filter(not_end_with("product_name", val)) + return self + + def with_product_name_sounding_like(self, val: str): + self.query.and_filter(sound_like("product_name", val)) + return self + + def with_product_name_is(self, val: str): + self.query.and_filter(eq("product_name", val)) + return self + def with_product_name_is_not(self, val): + self.query.and_filter(ne("product_name", val)) + return self + + def with_product_name_in(self, *vals): + self.query.and_filter(in_list("product_name", list(vals))) + return self + + def with_product_name_not_in(self, *vals): + self.query.and_filter(not_in_list("product_name", list(vals))) + return self + + def with_product_name_greater_than(self, val): + self.query.and_filter(gt("product_name", val)) + return self + + def with_product_name_greater_than_or_equal_to(self, val): + self.query.and_filter(gte("product_name", val)) + return self + + def with_product_name_less_than(self, val): + self.query.and_filter(lt("product_name", val)) + return self + + def with_product_name_less_than_or_equal_to(self, val): + self.query.and_filter(lte("product_name", val)) + return self + + def with_product_name_between(self, lower, upper): + self.query.and_filter(between(column("product_name"), value(lower), value(upper))) + return self + + def with_product_name_is_known(self): + self.query.and_filter(is_not_null(column("product_name"))) + return self + + def with_product_name_is_unknown(self): + self.query.and_filter(is_null(column("product_name"))) + return self + + def with_quantity_is(self, val): + self.query.and_filter(eq("quantity", val)) + return self + + def with_quantity_is_not(self, val): + self.query.and_filter(ne("quantity", val)) + return self + + def with_quantity_in(self, *vals): + self.query.and_filter(in_list("quantity", list(vals))) + return self + + def with_quantity_not_in(self, *vals): + self.query.and_filter(not_in_list("quantity", list(vals))) + return self + + def with_quantity_greater_than(self, val): + self.query.and_filter(gt("quantity", val)) + return self + + def with_quantity_greater_than_or_equal_to(self, val): + self.query.and_filter(gte("quantity", val)) + return self + + def with_quantity_less_than(self, val): + self.query.and_filter(lt("quantity", val)) + return self + + def with_quantity_less_than_or_equal_to(self, val): + self.query.and_filter(lte("quantity", val)) + return self + + def with_quantity_between(self, lower, upper): + self.query.and_filter(between(column("quantity"), value(lower), value(upper))) + return self + + def with_quantity_is_known(self): + self.query.and_filter(is_not_null(column("quantity"))) + return self + + def with_quantity_is_unknown(self): + self.query.and_filter(is_null(column("quantity"))) + return self + + def with_unit_price_is(self, val): + self.query.and_filter(eq("unit_price", val)) + return self + + def with_unit_price_is_not(self, val): + self.query.and_filter(ne("unit_price", val)) + return self + + def with_unit_price_in(self, *vals): + self.query.and_filter(in_list("unit_price", list(vals))) + return self + + def with_unit_price_not_in(self, *vals): + self.query.and_filter(not_in_list("unit_price", list(vals))) + return self + + def with_unit_price_greater_than(self, val): + self.query.and_filter(gt("unit_price", val)) + return self + + def with_unit_price_greater_than_or_equal_to(self, val): + self.query.and_filter(gte("unit_price", val)) + return self + + def with_unit_price_less_than(self, val): + self.query.and_filter(lt("unit_price", val)) + return self + + def with_unit_price_less_than_or_equal_to(self, val): + self.query.and_filter(lte("unit_price", val)) + return self + + def with_unit_price_between(self, lower, upper): + self.query.and_filter(between(column("unit_price"), value(lower), value(upper))) + return self + + def with_unit_price_is_known(self): + self.query.and_filter(is_not_null(column("unit_price"))) + return self + + def with_unit_price_is_unknown(self): + self.query.and_filter(is_null(column("unit_price"))) + return self + + def with_version_is(self, val): + self.query.and_filter(eq("version", val)) + return self + + def with_version_is_not(self, val): + self.query.and_filter(ne("version", val)) + return self + + def with_version_in(self, *vals): + self.query.and_filter(in_list("version", list(vals))) + return self + + def with_version_not_in(self, *vals): + self.query.and_filter(not_in_list("version", list(vals))) + return self + + def with_version_greater_than(self, val): + self.query.and_filter(gt("version", val)) + return self + + def with_version_greater_than_or_equal_to(self, val): + self.query.and_filter(gte("version", val)) + return self + + def with_version_less_than(self, val): + self.query.and_filter(lt("version", val)) + return self + + def with_version_less_than_or_equal_to(self, val): + self.query.and_filter(lte("version", val)) + return self + + def with_version_between(self, lower, upper): + self.query.and_filter(between(column("version"), value(lower), value(upper))) + return self + + def with_version_is_known(self): + self.query.and_filter(is_not_null(column("version"))) + return self + + def with_version_is_unknown(self): + self.query.and_filter(is_null(column("version"))) + return self + + def order_by_id_ascending(self): + self.query.order_by("id", "asc") + return self + + def order_by_id_descending(self): + self.query.order_by("id", "desc") + return self + + + def order_by_product_name_ascending(self): + self.query.order_by("product_name", "asc") + return self + + def order_by_product_name_descending(self): + self.query.order_by("product_name", "desc") + return self + + def order_by_quantity_ascending(self): + self.query.order_by("quantity", "asc") + return self + + def order_by_quantity_descending(self): + self.query.order_by("quantity", "desc") + return self + + def order_by_unit_price_ascending(self): + self.query.order_by("unit_price", "asc") + return self + + def order_by_unit_price_descending(self): + self.query.order_by("unit_price", "desc") + return self + + def order_by_version_ascending(self): + self.query.order_by("version", "asc") + return self + + def order_by_version_descending(self): + self.query.order_by("version", "desc") + return self + + + def count(self): + self.query.count_field("id", "count") + self._page_count_alias = "count" + self._page_count_method = "count" + return self + + def count_as(self, ret_name: str): + self.query.count_field("id", ret_name) + self._page_count_alias = ret_name + self._page_count_method = "count_as" + return self + + def min_quantity(self): + return self.min_quantity_as("minOfQuantity") + + def min_quantity_as(self, ret_name: str): + self.query.min("quantity", ret_name) + return self + def max_quantity(self): + return self.max_quantity_as("maxOfQuantity") + + def max_quantity_as(self, ret_name: str): + self.query.max("quantity", ret_name) + return self + def sum_quantity(self): + return self.sum_quantity_as("sumOfQuantity") + + def sum_quantity_as(self, ret_name: str): + self.query.sum("quantity", ret_name) + return self + def avg_quantity(self): + return self.avg_quantity_as("avgOfQuantity") + + def avg_quantity_as(self, ret_name: str): + self.query.avg("quantity", ret_name) + return self + def standardDeviation_quantity(self): + return self.standardDeviation_quantity_as("standardDeviationOfQuantity") + + def standardDeviation_quantity_as(self, ret_name: str): + self.query.standardDeviation("quantity", ret_name) + return self + def squareRootOfPopulationStandardDeviation_quantity(self): + return self.squareRootOfPopulationStandardDeviation_quantity_as("squareRootOfPopulationStandardDeviationOfQuantity") + + def squareRootOfPopulationStandardDeviation_quantity_as(self, ret_name: str): + self.query.squareRootOfPopulationStandardDeviation("quantity", ret_name) + return self + def sampleVariance_quantity(self): + return self.sampleVariance_quantity_as("sampleVarianceOfQuantity") + + def sampleVariance_quantity_as(self, ret_name: str): + self.query.sampleVariance("quantity", ret_name) + return self + def samplePopulationVariance_quantity(self): + return self.samplePopulationVariance_quantity_as("samplePopulationVarianceOfQuantity") + + def samplePopulationVariance_quantity_as(self, ret_name: str): + self.query.samplePopulationVariance("quantity", ret_name) + return self + def min_unit_price(self): + return self.min_unit_price_as("minOfUnitPrice") + + def min_unit_price_as(self, ret_name: str): + self.query.min("unit_price", ret_name) + return self + def max_unit_price(self): + return self.max_unit_price_as("maxOfUnitPrice") + + def max_unit_price_as(self, ret_name: str): + self.query.max("unit_price", ret_name) + return self + def sum_unit_price(self): + return self.sum_unit_price_as("sumOfUnitPrice") + + def sum_unit_price_as(self, ret_name: str): + self.query.sum("unit_price", ret_name) + return self + def avg_unit_price(self): + return self.avg_unit_price_as("avgOfUnitPrice") + + def avg_unit_price_as(self, ret_name: str): + self.query.avg("unit_price", ret_name) + return self + def standardDeviation_unit_price(self): + return self.standardDeviation_unit_price_as("standardDeviationOfUnitPrice") + + def standardDeviation_unit_price_as(self, ret_name: str): + self.query.standardDeviation("unit_price", ret_name) + return self + def squareRootOfPopulationStandardDeviation_unit_price(self): + return self.squareRootOfPopulationStandardDeviation_unit_price_as("squareRootOfPopulationStandardDeviationOfUnitPrice") + + def squareRootOfPopulationStandardDeviation_unit_price_as(self, ret_name: str): + self.query.squareRootOfPopulationStandardDeviation("unit_price", ret_name) + return self + def sampleVariance_unit_price(self): + return self.sampleVariance_unit_price_as("sampleVarianceOfUnitPrice") + + def sampleVariance_unit_price_as(self, ret_name: str): + self.query.sampleVariance("unit_price", ret_name) + return self + def samplePopulationVariance_unit_price(self): + return self.samplePopulationVariance_unit_price_as("samplePopulationVarianceOfUnitPrice") + + def samplePopulationVariance_unit_price_as(self, ret_name: str): + self.query.samplePopulationVariance("unit_price", ret_name) + return self + def group_by_id(self): + self.query.group_by("id") + return self + + def group_by_id_as(self, ret_name: str): + self.query.group_by("id") + return self + def group_by_customer_order(self): + self.query.group_by("customer_order") + return self + + def group_by_customer_order_as(self, ret_name: str): + self.query.group_by("customer_order") + return self + def group_by_product_name(self): + self.query.group_by("product_name") + return self + + def group_by_product_name_as(self, ret_name: str): + self.query.group_by("product_name") + return self + def group_by_quantity(self): + self.query.group_by("quantity") + return self + + def group_by_quantity_as(self, ret_name: str): + self.query.group_by("quantity") + return self + def group_by_unit_price(self): + self.query.group_by("unit_price") + return self + + def group_by_unit_price_as(self, ret_name: str): + self.query.group_by("unit_price") + return self + def group_by_version(self): + self.query.group_by("version") + return self + + def group_by_version_as(self, ret_name: str): + self.query.group_by("version") + return self + def facet_by_customer_order_as(self, name: str, request: QuerySelection, + include_all_facets: bool = True): + self.query.facet_by(name, "customer_order", request.query, include_all_facets, + QueryOperation("facet", "facet_by_customer_order_as", name, "customer_order")) + return self + + +class ExecutableOrderItemRequest: + def __init__(self, request): + self._request = request + + def comment(self, c: str): + self._request.comment(c) + return self + + def new_entity(self, context) -> OrderItem: + request = self._request + if not request._comment or not request._comment.strip() or not request._purpose or not request._purpose.strip(): + raise ValueError("Security audit failure: non-empty comment() and purpose() are required before new_entity()") + entity = context.initialize_entity("OrderItem", OrderItem()) + if not isinstance(entity, OrderItem): + raise TypeError("entity initializer returned an incompatible OrderItem") + return entity + + async def execute_for_result(self, context): + self = self._request + if not self._purpose or not self._purpose.strip() or not self._comment or not self._comment.strip(): + raise Exception("Security audit failure: comment() and purpose() must be called before execute_for_rows()") + service = context.require_resource("dataService") + req = QueryRequest(context.prepare_query(self.query), _comment=self._comment, _purpose=self._purpose) + return await service.query(context, req) + + async def execute_for_rows(self, context): + return (await self.execute_for_result(context)).rows + + async def execute_for_list(self, context) -> SmartList[OrderItem]: + from teaql.runtime.query_list import execute_typed_list + request = self._request + QueryIntent(request._comment, request._purpose) + query = deepcopy(request.query) + authorized = context.prepare_query(query) + root = QueryRequest(authorized, _comment=request._comment, _purpose=request._purpose) + operation = (QueryOperation("named-count", request._page_count_method or "count", + request._page_count_alias if request._page_count_method == "count_as" else None) + if request._page_count_alias is not None else None) + return await execute_typed_list(context, root, + lambda row: OrderItem(_entity_root=EntityRoot(), **row), + count_operation=operation, count_alias=request._page_count_alias) + + async def execute_for_one(self, context): + request = deepcopy(self._request) + request.limit(1) + entities = await ExecutableOrderItemRequest(request).execute_for_list(context) + return entities[0] if entities else None + + def execute_for_stream(self, context, chunk_size: int = 1000): + """Yield entity chunks lazily from the provider cursor.""" + request = deepcopy(self._request) + QueryIntent(request._comment, request._purpose) + service = context.require_resource("dataService") + if not hasattr(service, "query_stream"): + raise RuntimeError("dataService does not implement query_stream") + root = QueryRequest(context.prepare_query(request.query), _comment=request._comment, _purpose=request._purpose) + from teaql.runtime.query_stream import execute_typed_stream + return execute_typed_stream(context, root, + lambda row: OrderItem(_entity_root=EntityRoot(), **row), chunk_size) diff --git a/examples/trace-chain/standard-order/lib/requests/order_status_request.py b/examples/trace-chain/standard-order/lib/requests/order_status_request.py new file mode 100644 index 0000000..ddff7d2 --- /dev/null +++ b/examples/trace-chain/standard-order/lib/requests/order_status_request.py @@ -0,0 +1,874 @@ +from teaql.core.query import AggregateFunction, RelationAggregate, SelectQuery +from teaql.core.request_tree import QueryOperation +from teaql.core.dynamic_search import merge_generated_scalar_search +from generated_search_models import GENERATED_SEARCH_MODELS +from teaql.core.list import SmartList +from teaql.core.request_intent import QueryIntent +from teaql.runtime import EntityRoot +from teaql.data_service import QueryRequest +from copy import deepcopy +from teaql.core.expr import ( + begin_with, between, column, contain, end_with, eq, gt, gte, + in_list, in_subquery, is_not_null, is_null, lt, lte, ne, not_begin_with, + not_contain, not_end_with, not_in_list, not_in_subquery, value, + sound_like, +) +from models.order_status import OrderStatus +from typing import Protocol + +class QuerySelection(Protocol): + query: SelectQuery + +class OrderStatusRequest: + def __init__(self, minimal=False): + self.query = SelectQuery("OrderStatus") + self._purpose = None + self._comment = None + self._page_count_alias = None + self._page_count_method = None + self._dynamic_search_warnings = [] + self.query.and_filter(gte("version", 1)) + if minimal: + self.select_id() + self.select_version() + else: + self.select_self_fields() + + def comment(self, c: str): + self.query.comment(c) + self._comment = c + return self + + def filter_by_json(self, source, warn=None): + kwargs = {} if warn is None else {"warn": warn} + query, warnings = merge_generated_scalar_search(self.query, source, GENERATED_SEARCH_MODELS, **kwargs) + self.query = query + self._dynamic_search_warnings.extend(warnings) + return self + + @property + def dynamic_search_warnings(self): + return tuple(dict(warning) for warning in self._dynamic_search_warnings) + + def purpose(self, p: str): + self.query.purpose(p) + self._purpose = p + return ExecutableOrderStatusRequest(self) + + def optimize_for_continuous_page_fetch(self): + self.query.optimize_for_continuous_page_fetch() + return self + + def optimize_for_continuous_page_fetch_with(self, namespace: str, ttl_seconds: int): + self.query.optimize_for_continuous_page_fetch_with(namespace, ttl_seconds) + return self + + def optimize_pagination_with_id_set(self): + self.query.optimize_pagination_with_id_set() + return self + + def optimize_pagination_with_id_set_config(self, namespace: str, ttl_seconds: int, max_ids: int): + self.query.optimize_pagination_with_id_set_config(namespace, ttl_seconds, max_ids) + return self + + def top_n_probe_parent_threshold(self, threshold: int): + self.query.top_n_probe_parent_threshold(threshold) + return self + + def limit(self, n: int): + self.query.limit(n) + return self + + def offset(self, n: int): + self.query.offset(n) + return self + + def with_deleted_rows(self): + self.query.with_deleted_rows() + return self + + def deleted_rows_only(self): + self.query.deleted_rows_only() + return self + + def select_self_fields(self): + self.query.project("platform", "id", "name", "code", "color", "display_order", "version") + return self + + def select_id(self): + self.query.project("id") + return self + + def select_name(self): + self.query.project("name") + return self + + def select_code(self): + self.query.project("code") + return self + + def select_color(self): + self.query.project("color") + return self + + def select_display_order(self): + self.query.project("display_order") + return self + + def select_version(self): + self.query.project("version") + return self + + def select_platform_with(self, child_request): + self.query.project("platform") + self.query.relation_query("platform", child_request.query, + QueryOperation("relation-select", "select_platform_with", relation="platform")) + return self + def with_platform_matching(self, child_request): + child_request.query.projection = ["id"] + self.query.and_filter(in_subquery(column("platform"), "Platform", child_request.query)) + return self + + def without_platform_matching(self, child_request): + child_request.query.projection = ["id"] + self.query.and_filter(not_in_subquery(column("platform"), "Platform", child_request.query)) + return self + + def have_platform(self): + self.query.and_filter(is_not_null(column("platform"))) + return self + + def have_no_platform(self): + self.query.and_filter(is_null(column("platform"))) + return self + + def filter_by_platform(self, val): + self.query.and_filter(eq("platform", val)) + return self + + def with_id_is(self, val): + self.query.and_filter(eq("id", val)) + return self + + def with_id_is_not(self, val): + self.query.and_filter(ne("id", val)) + return self + + def with_id_in(self, *vals): + self.query.and_filter(in_list("id", list(vals))) + return self + + def with_id_not_in(self, *vals): + self.query.and_filter(not_in_list("id", list(vals))) + return self + + def with_id_greater_than(self, val): + self.query.and_filter(gt("id", val)) + return self + + def with_id_greater_than_or_equal_to(self, val): + self.query.and_filter(gte("id", val)) + return self + + def with_id_less_than(self, val): + self.query.and_filter(lt("id", val)) + return self + + def with_id_less_than_or_equal_to(self, val): + self.query.and_filter(lte("id", val)) + return self + + def with_id_between(self, lower, upper): + self.query.and_filter(between(column("id"), value(lower), value(upper))) + return self + + def with_id_is_known(self): + self.query.and_filter(is_not_null(column("id"))) + return self + + def with_id_is_unknown(self): + self.query.and_filter(is_null(column("id"))) + return self + + def with_name_containing(self, val: str): + self.query.and_filter(contain("name", val)) + return self + + def with_name_not_containing(self, val: str): + self.query.and_filter(not_contain("name", val)) + return self + + def with_name_starting_with(self, val: str): + self.query.and_filter(begin_with("name", val)) + return self + + def with_name_not_starting_with(self, val: str): + self.query.and_filter(not_begin_with("name", val)) + return self + + def with_name_ending_with(self, val: str): + self.query.and_filter(end_with("name", val)) + return self + + def with_name_not_ending_with(self, val: str): + self.query.and_filter(not_end_with("name", val)) + return self + + def with_name_sounding_like(self, val: str): + self.query.and_filter(sound_like("name", val)) + return self + + def with_name_is(self, val: str): + self.query.and_filter(eq("name", val)) + return self + def with_name_is_not(self, val): + self.query.and_filter(ne("name", val)) + return self + + def with_name_in(self, *vals): + self.query.and_filter(in_list("name", list(vals))) + return self + + def with_name_not_in(self, *vals): + self.query.and_filter(not_in_list("name", list(vals))) + return self + + def with_name_greater_than(self, val): + self.query.and_filter(gt("name", val)) + return self + + def with_name_greater_than_or_equal_to(self, val): + self.query.and_filter(gte("name", val)) + return self + + def with_name_less_than(self, val): + self.query.and_filter(lt("name", val)) + return self + + def with_name_less_than_or_equal_to(self, val): + self.query.and_filter(lte("name", val)) + return self + + def with_name_between(self, lower, upper): + self.query.and_filter(between(column("name"), value(lower), value(upper))) + return self + + def with_name_is_known(self): + self.query.and_filter(is_not_null(column("name"))) + return self + + def with_name_is_unknown(self): + self.query.and_filter(is_null(column("name"))) + return self + + def with_code_containing(self, val: str): + self.query.and_filter(contain("code", val)) + return self + + def with_code_not_containing(self, val: str): + self.query.and_filter(not_contain("code", val)) + return self + + def with_code_starting_with(self, val: str): + self.query.and_filter(begin_with("code", val)) + return self + + def with_code_not_starting_with(self, val: str): + self.query.and_filter(not_begin_with("code", val)) + return self + + def with_code_ending_with(self, val: str): + self.query.and_filter(end_with("code", val)) + return self + + def with_code_not_ending_with(self, val: str): + self.query.and_filter(not_end_with("code", val)) + return self + + def with_code_sounding_like(self, val: str): + self.query.and_filter(sound_like("code", val)) + return self + + def with_code_is(self, val: str): + self.query.and_filter(eq("code", val)) + return self + def with_code_is_not(self, val): + self.query.and_filter(ne("code", val)) + return self + + def with_code_in(self, *vals): + self.query.and_filter(in_list("code", list(vals))) + return self + + def with_code_not_in(self, *vals): + self.query.and_filter(not_in_list("code", list(vals))) + return self + + def with_code_greater_than(self, val): + self.query.and_filter(gt("code", val)) + return self + + def with_code_greater_than_or_equal_to(self, val): + self.query.and_filter(gte("code", val)) + return self + + def with_code_less_than(self, val): + self.query.and_filter(lt("code", val)) + return self + + def with_code_less_than_or_equal_to(self, val): + self.query.and_filter(lte("code", val)) + return self + + def with_code_between(self, lower, upper): + self.query.and_filter(between(column("code"), value(lower), value(upper))) + return self + + def with_code_is_known(self): + self.query.and_filter(is_not_null(column("code"))) + return self + + def with_code_is_unknown(self): + self.query.and_filter(is_null(column("code"))) + return self + + def with_color_containing(self, val: str): + self.query.and_filter(contain("color", val)) + return self + + def with_color_not_containing(self, val: str): + self.query.and_filter(not_contain("color", val)) + return self + + def with_color_starting_with(self, val: str): + self.query.and_filter(begin_with("color", val)) + return self + + def with_color_not_starting_with(self, val: str): + self.query.and_filter(not_begin_with("color", val)) + return self + + def with_color_ending_with(self, val: str): + self.query.and_filter(end_with("color", val)) + return self + + def with_color_not_ending_with(self, val: str): + self.query.and_filter(not_end_with("color", val)) + return self + + def with_color_sounding_like(self, val: str): + self.query.and_filter(sound_like("color", val)) + return self + + def with_color_is(self, val: str): + self.query.and_filter(eq("color", val)) + return self + def with_color_is_not(self, val): + self.query.and_filter(ne("color", val)) + return self + + def with_color_in(self, *vals): + self.query.and_filter(in_list("color", list(vals))) + return self + + def with_color_not_in(self, *vals): + self.query.and_filter(not_in_list("color", list(vals))) + return self + + def with_color_greater_than(self, val): + self.query.and_filter(gt("color", val)) + return self + + def with_color_greater_than_or_equal_to(self, val): + self.query.and_filter(gte("color", val)) + return self + + def with_color_less_than(self, val): + self.query.and_filter(lt("color", val)) + return self + + def with_color_less_than_or_equal_to(self, val): + self.query.and_filter(lte("color", val)) + return self + + def with_color_between(self, lower, upper): + self.query.and_filter(between(column("color"), value(lower), value(upper))) + return self + + def with_color_is_known(self): + self.query.and_filter(is_not_null(column("color"))) + return self + + def with_color_is_unknown(self): + self.query.and_filter(is_null(column("color"))) + return self + + def with_display_order_is(self, val): + self.query.and_filter(eq("display_order", val)) + return self + + def with_display_order_is_not(self, val): + self.query.and_filter(ne("display_order", val)) + return self + + def with_display_order_in(self, *vals): + self.query.and_filter(in_list("display_order", list(vals))) + return self + + def with_display_order_not_in(self, *vals): + self.query.and_filter(not_in_list("display_order", list(vals))) + return self + + def with_display_order_greater_than(self, val): + self.query.and_filter(gt("display_order", val)) + return self + + def with_display_order_greater_than_or_equal_to(self, val): + self.query.and_filter(gte("display_order", val)) + return self + + def with_display_order_less_than(self, val): + self.query.and_filter(lt("display_order", val)) + return self + + def with_display_order_less_than_or_equal_to(self, val): + self.query.and_filter(lte("display_order", val)) + return self + + def with_display_order_between(self, lower, upper): + self.query.and_filter(between(column("display_order"), value(lower), value(upper))) + return self + + def with_display_order_is_known(self): + self.query.and_filter(is_not_null(column("display_order"))) + return self + + def with_display_order_is_unknown(self): + self.query.and_filter(is_null(column("display_order"))) + return self + + def with_version_is(self, val): + self.query.and_filter(eq("version", val)) + return self + + def with_version_is_not(self, val): + self.query.and_filter(ne("version", val)) + return self + + def with_version_in(self, *vals): + self.query.and_filter(in_list("version", list(vals))) + return self + + def with_version_not_in(self, *vals): + self.query.and_filter(not_in_list("version", list(vals))) + return self + + def with_version_greater_than(self, val): + self.query.and_filter(gt("version", val)) + return self + + def with_version_greater_than_or_equal_to(self, val): + self.query.and_filter(gte("version", val)) + return self + + def with_version_less_than(self, val): + self.query.and_filter(lt("version", val)) + return self + + def with_version_less_than_or_equal_to(self, val): + self.query.and_filter(lte("version", val)) + return self + + def with_version_between(self, lower, upper): + self.query.and_filter(between(column("version"), value(lower), value(upper))) + return self + + def with_version_is_known(self): + self.query.and_filter(is_not_null(column("version"))) + return self + + def with_version_is_unknown(self): + self.query.and_filter(is_null(column("version"))) + return self + + def order_by_id_ascending(self): + self.query.order_by("id", "asc") + return self + + def order_by_id_descending(self): + self.query.order_by("id", "desc") + return self + + def order_by_name_ascending(self): + self.query.order_by("name", "asc") + return self + + def order_by_name_descending(self): + self.query.order_by("name", "desc") + return self + + def order_by_code_ascending(self): + self.query.order_by("code", "asc") + return self + + def order_by_code_descending(self): + self.query.order_by("code", "desc") + return self + + def order_by_color_ascending(self): + self.query.order_by("color", "asc") + return self + + def order_by_color_descending(self): + self.query.order_by("color", "desc") + return self + + def order_by_display_order_ascending(self): + self.query.order_by("display_order", "asc") + return self + + def order_by_display_order_descending(self): + self.query.order_by("display_order", "desc") + return self + + def order_by_version_ascending(self): + self.query.order_by("version", "asc") + return self + + def order_by_version_descending(self): + self.query.order_by("version", "desc") + return self + + + def count(self): + self.query.count_field("id", "count") + self._page_count_alias = "count" + self._page_count_method = "count" + return self + + def count_as(self, ret_name: str): + self.query.count_field("id", ret_name) + self._page_count_alias = ret_name + self._page_count_method = "count_as" + return self + + def min_display_order(self): + return self.min_display_order_as("minOfDisplayOrder") + + def min_display_order_as(self, ret_name: str): + self.query.min("display_order", ret_name) + return self + def max_display_order(self): + return self.max_display_order_as("maxOfDisplayOrder") + + def max_display_order_as(self, ret_name: str): + self.query.max("display_order", ret_name) + return self + def sum_display_order(self): + return self.sum_display_order_as("sumOfDisplayOrder") + + def sum_display_order_as(self, ret_name: str): + self.query.sum("display_order", ret_name) + return self + def avg_display_order(self): + return self.avg_display_order_as("avgOfDisplayOrder") + + def avg_display_order_as(self, ret_name: str): + self.query.avg("display_order", ret_name) + return self + def standardDeviation_display_order(self): + return self.standardDeviation_display_order_as("standardDeviationOfDisplayOrder") + + def standardDeviation_display_order_as(self, ret_name: str): + self.query.standardDeviation("display_order", ret_name) + return self + def squareRootOfPopulationStandardDeviation_display_order(self): + return self.squareRootOfPopulationStandardDeviation_display_order_as("squareRootOfPopulationStandardDeviationOfDisplayOrder") + + def squareRootOfPopulationStandardDeviation_display_order_as(self, ret_name: str): + self.query.squareRootOfPopulationStandardDeviation("display_order", ret_name) + return self + def sampleVariance_display_order(self): + return self.sampleVariance_display_order_as("sampleVarianceOfDisplayOrder") + + def sampleVariance_display_order_as(self, ret_name: str): + self.query.sampleVariance("display_order", ret_name) + return self + def samplePopulationVariance_display_order(self): + return self.samplePopulationVariance_display_order_as("samplePopulationVarianceOfDisplayOrder") + + def samplePopulationVariance_display_order_as(self, ret_name: str): + self.query.samplePopulationVariance("display_order", ret_name) + return self + def group_by_platform(self): + self.query.group_by("platform") + return self + + def group_by_platform_as(self, ret_name: str): + self.query.group_by("platform") + return self + def group_by_id(self): + self.query.group_by("id") + return self + + def group_by_id_as(self, ret_name: str): + self.query.group_by("id") + return self + def group_by_name(self): + self.query.group_by("name") + return self + + def group_by_name_as(self, ret_name: str): + self.query.group_by("name") + return self + def group_by_code(self): + self.query.group_by("code") + return self + + def group_by_code_as(self, ret_name: str): + self.query.group_by("code") + return self + def group_by_color(self): + self.query.group_by("color") + return self + + def group_by_color_as(self, ret_name: str): + self.query.group_by("color") + return self + def group_by_display_order(self): + self.query.group_by("display_order") + return self + + def group_by_display_order_as(self, ret_name: str): + self.query.group_by("display_order") + return self + def group_by_version(self): + self.query.group_by("version") + return self + + def group_by_version_as(self, ret_name: str): + self.query.group_by("version") + return self + def select_customer_order_list(self): + from requests.customer_order_request import CustomerOrderRequest + return self.select_customer_order_list_with(CustomerOrderRequest()) + + def select_customer_order_list_with(self, child_request): + self.query.relation_query("customer_order_list", child_request.query, + QueryOperation("relation-select", "select_customer_order_list_with", relation="customer_order_list")) + return self + def have_customer_orders(self): + from requests.customer_order_request import CustomerOrderRequest + return self.with_customer_order_list_matching(CustomerOrderRequest()) + + def have_no_customer_orders(self): + from requests.customer_order_request import CustomerOrderRequest + return self.without_customer_order_list_matching(CustomerOrderRequest()) + + def with_customer_order_list_matching(self, child_request): + child_request.query.projection = ["status"] + self.query.and_filter(in_subquery(column("id"), "CustomerOrder", child_request.query)) + return self + + def without_customer_order_list_matching(self, child_request): + child_request.query.projection = ["status"] + self.query.and_filter(not_in_subquery(column("id"), "CustomerOrder", child_request.query)) + return self + def count_customer_orders(self): + return self.count_customer_orders_as("count_customer_orders") + + def count_customer_orders_as(self, alias: str): + from requests.customer_order_request import CustomerOrderRequest + self.count_customer_orders_with(alias, CustomerOrderRequest()) + self.query.relation_aggregates[-1].operation = QueryOperation( + "relation-count", "count_customer_orders_as", alias, "customer_order_list") + return self + + def count_customer_orders_with(self, alias: str, child_request): + query = deepcopy(child_request.query) + query.aggregates = [] + query.count_field("id", alias) + self.query.relation_aggregates.append( + RelationAggregate("customer_order_list", alias, query, True, + QueryOperation("relation-count", "count_customer_orders_with", alias, "customer_order_list")) + ) + return self + + def min_total_amount_of_customer_orders(self): + from requests.customer_order_request import CustomerOrderRequest + return self.min_total_amount_of_customer_orders_as( + "min_total_amount_of_customer_orders", CustomerOrderRequest()) + + def min_total_amount_of_customer_orders_as(self, alias: str, child_request): + query = deepcopy(child_request.query) + query.aggregates = [] + query.min("total_amount", alias) + self.query.relation_aggregates.append( + RelationAggregate("customer_order_list", alias, query, True, + QueryOperation("aggregate", "min_total_amount_of_customer_orders_as", alias, "customer_order_list")) + ) + return self + def max_total_amount_of_customer_orders(self): + from requests.customer_order_request import CustomerOrderRequest + return self.max_total_amount_of_customer_orders_as( + "max_total_amount_of_customer_orders", CustomerOrderRequest()) + + def max_total_amount_of_customer_orders_as(self, alias: str, child_request): + query = deepcopy(child_request.query) + query.aggregates = [] + query.max("total_amount", alias) + self.query.relation_aggregates.append( + RelationAggregate("customer_order_list", alias, query, True, + QueryOperation("aggregate", "max_total_amount_of_customer_orders_as", alias, "customer_order_list")) + ) + return self + def sum_total_amount_of_customer_orders(self): + from requests.customer_order_request import CustomerOrderRequest + return self.sum_total_amount_of_customer_orders_as( + "sum_total_amount_of_customer_orders", CustomerOrderRequest()) + + def sum_total_amount_of_customer_orders_as(self, alias: str, child_request): + query = deepcopy(child_request.query) + query.aggregates = [] + query.sum("total_amount", alias) + self.query.relation_aggregates.append( + RelationAggregate("customer_order_list", alias, query, True, + QueryOperation("aggregate", "sum_total_amount_of_customer_orders_as", alias, "customer_order_list")) + ) + return self + def avg_total_amount_of_customer_orders(self): + from requests.customer_order_request import CustomerOrderRequest + return self.avg_total_amount_of_customer_orders_as( + "avg_total_amount_of_customer_orders", CustomerOrderRequest()) + + def avg_total_amount_of_customer_orders_as(self, alias: str, child_request): + query = deepcopy(child_request.query) + query.aggregates = [] + query.avg("total_amount", alias) + self.query.relation_aggregates.append( + RelationAggregate("customer_order_list", alias, query, True, + QueryOperation("aggregate", "avg_total_amount_of_customer_orders_as", alias, "customer_order_list")) + ) + return self + def standardDeviation_total_amount_of_customer_orders(self): + from requests.customer_order_request import CustomerOrderRequest + return self.standardDeviation_total_amount_of_customer_orders_as( + "standardDeviation_total_amount_of_customer_orders", CustomerOrderRequest()) + + def standardDeviation_total_amount_of_customer_orders_as(self, alias: str, child_request): + query = deepcopy(child_request.query) + query.aggregates = [] + query.standardDeviation("total_amount", alias) + self.query.relation_aggregates.append( + RelationAggregate("customer_order_list", alias, query, True, + QueryOperation("aggregate", "standardDeviation_total_amount_of_customer_orders_as", alias, "customer_order_list")) + ) + return self + def squareRootOfPopulationStandardDeviation_total_amount_of_customer_orders(self): + from requests.customer_order_request import CustomerOrderRequest + return self.squareRootOfPopulationStandardDeviation_total_amount_of_customer_orders_as( + "squareRootOfPopulationStandardDeviation_total_amount_of_customer_orders", CustomerOrderRequest()) + + def squareRootOfPopulationStandardDeviation_total_amount_of_customer_orders_as(self, alias: str, child_request): + query = deepcopy(child_request.query) + query.aggregates = [] + query.squareRootOfPopulationStandardDeviation("total_amount", alias) + self.query.relation_aggregates.append( + RelationAggregate("customer_order_list", alias, query, True, + QueryOperation("aggregate", "squareRootOfPopulationStandardDeviation_total_amount_of_customer_orders_as", alias, "customer_order_list")) + ) + return self + def sampleVariance_total_amount_of_customer_orders(self): + from requests.customer_order_request import CustomerOrderRequest + return self.sampleVariance_total_amount_of_customer_orders_as( + "sampleVariance_total_amount_of_customer_orders", CustomerOrderRequest()) + + def sampleVariance_total_amount_of_customer_orders_as(self, alias: str, child_request): + query = deepcopy(child_request.query) + query.aggregates = [] + query.sampleVariance("total_amount", alias) + self.query.relation_aggregates.append( + RelationAggregate("customer_order_list", alias, query, True, + QueryOperation("aggregate", "sampleVariance_total_amount_of_customer_orders_as", alias, "customer_order_list")) + ) + return self + def samplePopulationVariance_total_amount_of_customer_orders(self): + from requests.customer_order_request import CustomerOrderRequest + return self.samplePopulationVariance_total_amount_of_customer_orders_as( + "samplePopulationVariance_total_amount_of_customer_orders", CustomerOrderRequest()) + + def samplePopulationVariance_total_amount_of_customer_orders_as(self, alias: str, child_request): + query = deepcopy(child_request.query) + query.aggregates = [] + query.samplePopulationVariance("total_amount", alias) + self.query.relation_aggregates.append( + RelationAggregate("customer_order_list", alias, query, True, + QueryOperation("aggregate", "samplePopulationVariance_total_amount_of_customer_orders_as", alias, "customer_order_list")) + ) + return self + def facet_by_platform_as(self, name: str, request: QuerySelection, + include_all_facets: bool = True): + self.query.facet_by(name, "platform", request.query, include_all_facets, + QueryOperation("facet", "facet_by_platform_as", name, "platform")) + return self + + +class ExecutableOrderStatusRequest: + def __init__(self, request): + self._request = request + + def comment(self, c: str): + self._request.comment(c) + return self + + def new_entity(self, context) -> OrderStatus: + request = self._request + if not request._comment or not request._comment.strip() or not request._purpose or not request._purpose.strip(): + raise ValueError("Security audit failure: non-empty comment() and purpose() are required before new_entity()") + entity = context.initialize_entity("OrderStatus", OrderStatus()) + if not isinstance(entity, OrderStatus): + raise TypeError("entity initializer returned an incompatible OrderStatus") + return entity + + async def execute_for_result(self, context): + self = self._request + if not self._purpose or not self._purpose.strip() or not self._comment or not self._comment.strip(): + raise Exception("Security audit failure: comment() and purpose() must be called before execute_for_rows()") + service = context.require_resource("dataService") + req = QueryRequest(context.prepare_query(self.query), _comment=self._comment, _purpose=self._purpose) + return await service.query(context, req) + + async def execute_for_rows(self, context): + return (await self.execute_for_result(context)).rows + + async def execute_for_list(self, context) -> SmartList[OrderStatus]: + from teaql.runtime.query_list import execute_typed_list + request = self._request + QueryIntent(request._comment, request._purpose) + query = deepcopy(request.query) + authorized = context.prepare_query(query) + root = QueryRequest(authorized, _comment=request._comment, _purpose=request._purpose) + operation = (QueryOperation("named-count", request._page_count_method or "count", + request._page_count_alias if request._page_count_method == "count_as" else None) + if request._page_count_alias is not None else None) + return await execute_typed_list(context, root, + lambda row: OrderStatus(_entity_root=EntityRoot(), **row), + count_operation=operation, count_alias=request._page_count_alias) + + async def execute_for_one(self, context): + request = deepcopy(self._request) + request.limit(1) + entities = await ExecutableOrderStatusRequest(request).execute_for_list(context) + return entities[0] if entities else None + + def execute_for_stream(self, context, chunk_size: int = 1000): + """Yield entity chunks lazily from the provider cursor.""" + request = deepcopy(self._request) + QueryIntent(request._comment, request._purpose) + service = context.require_resource("dataService") + if not hasattr(service, "query_stream"): + raise RuntimeError("dataService does not implement query_stream") + root = QueryRequest(context.prepare_query(request.query), _comment=request._comment, _purpose=request._purpose) + from teaql.runtime.query_stream import execute_typed_stream + return execute_typed_stream(context, root, + lambda row: OrderStatus(_entity_root=EntityRoot(), **row), chunk_size) diff --git a/examples/trace-chain/standard-order/lib/requests/platform_request.py b/examples/trace-chain/standard-order/lib/requests/platform_request.py new file mode 100644 index 0000000..f8d3656 --- /dev/null +++ b/examples/trace-chain/standard-order/lib/requests/platform_request.py @@ -0,0 +1,590 @@ +from teaql.core.query import AggregateFunction, RelationAggregate, SelectQuery +from teaql.core.request_tree import QueryOperation +from teaql.core.dynamic_search import merge_generated_scalar_search +from generated_search_models import GENERATED_SEARCH_MODELS +from teaql.core.list import SmartList +from teaql.core.request_intent import QueryIntent +from teaql.runtime import EntityRoot +from teaql.data_service import QueryRequest +from copy import deepcopy +from teaql.core.expr import ( + begin_with, between, column, contain, end_with, eq, gt, gte, + in_list, in_subquery, is_not_null, is_null, lt, lte, ne, not_begin_with, + not_contain, not_end_with, not_in_list, not_in_subquery, value, + sound_like, +) +from models.platform import Platform +from typing import Protocol + +class QuerySelection(Protocol): + query: SelectQuery + +class PlatformRequest: + def __init__(self, minimal=False): + self.query = SelectQuery("Platform") + self._purpose = None + self._comment = None + self._page_count_alias = None + self._page_count_method = None + self._dynamic_search_warnings = [] + self.query.and_filter(gte("version", 1)) + if minimal: + self.select_id() + self.select_version() + else: + self.select_self_fields() + + def comment(self, c: str): + self.query.comment(c) + self._comment = c + return self + + def filter_by_json(self, source, warn=None): + kwargs = {} if warn is None else {"warn": warn} + query, warnings = merge_generated_scalar_search(self.query, source, GENERATED_SEARCH_MODELS, **kwargs) + self.query = query + self._dynamic_search_warnings.extend(warnings) + return self + + @property + def dynamic_search_warnings(self): + return tuple(dict(warning) for warning in self._dynamic_search_warnings) + + def purpose(self, p: str): + self.query.purpose(p) + self._purpose = p + return ExecutablePlatformRequest(self) + + def optimize_for_continuous_page_fetch(self): + self.query.optimize_for_continuous_page_fetch() + return self + + def optimize_for_continuous_page_fetch_with(self, namespace: str, ttl_seconds: int): + self.query.optimize_for_continuous_page_fetch_with(namespace, ttl_seconds) + return self + + def optimize_pagination_with_id_set(self): + self.query.optimize_pagination_with_id_set() + return self + + def optimize_pagination_with_id_set_config(self, namespace: str, ttl_seconds: int, max_ids: int): + self.query.optimize_pagination_with_id_set_config(namespace, ttl_seconds, max_ids) + return self + + def top_n_probe_parent_threshold(self, threshold: int): + self.query.top_n_probe_parent_threshold(threshold) + return self + + def limit(self, n: int): + self.query.limit(n) + return self + + def offset(self, n: int): + self.query.offset(n) + return self + + def with_deleted_rows(self): + self.query.with_deleted_rows() + return self + + def deleted_rows_only(self): + self.query.deleted_rows_only() + return self + + def select_self_fields(self): + self.query.project("id", "name", "version") + return self + + def select_id(self): + self.query.project("id") + return self + + def select_name(self): + self.query.project("name") + return self + + def select_version(self): + self.query.project("version") + return self + + + def with_id_is(self, val): + self.query.and_filter(eq("id", val)) + return self + + def with_id_is_not(self, val): + self.query.and_filter(ne("id", val)) + return self + + def with_id_in(self, *vals): + self.query.and_filter(in_list("id", list(vals))) + return self + + def with_id_not_in(self, *vals): + self.query.and_filter(not_in_list("id", list(vals))) + return self + + def with_id_greater_than(self, val): + self.query.and_filter(gt("id", val)) + return self + + def with_id_greater_than_or_equal_to(self, val): + self.query.and_filter(gte("id", val)) + return self + + def with_id_less_than(self, val): + self.query.and_filter(lt("id", val)) + return self + + def with_id_less_than_or_equal_to(self, val): + self.query.and_filter(lte("id", val)) + return self + + def with_id_between(self, lower, upper): + self.query.and_filter(between(column("id"), value(lower), value(upper))) + return self + + def with_id_is_known(self): + self.query.and_filter(is_not_null(column("id"))) + return self + + def with_id_is_unknown(self): + self.query.and_filter(is_null(column("id"))) + return self + + def with_name_containing(self, val: str): + self.query.and_filter(contain("name", val)) + return self + + def with_name_not_containing(self, val: str): + self.query.and_filter(not_contain("name", val)) + return self + + def with_name_starting_with(self, val: str): + self.query.and_filter(begin_with("name", val)) + return self + + def with_name_not_starting_with(self, val: str): + self.query.and_filter(not_begin_with("name", val)) + return self + + def with_name_ending_with(self, val: str): + self.query.and_filter(end_with("name", val)) + return self + + def with_name_not_ending_with(self, val: str): + self.query.and_filter(not_end_with("name", val)) + return self + + def with_name_sounding_like(self, val: str): + self.query.and_filter(sound_like("name", val)) + return self + + def with_name_is(self, val: str): + self.query.and_filter(eq("name", val)) + return self + def with_name_is_not(self, val): + self.query.and_filter(ne("name", val)) + return self + + def with_name_in(self, *vals): + self.query.and_filter(in_list("name", list(vals))) + return self + + def with_name_not_in(self, *vals): + self.query.and_filter(not_in_list("name", list(vals))) + return self + + def with_name_greater_than(self, val): + self.query.and_filter(gt("name", val)) + return self + + def with_name_greater_than_or_equal_to(self, val): + self.query.and_filter(gte("name", val)) + return self + + def with_name_less_than(self, val): + self.query.and_filter(lt("name", val)) + return self + + def with_name_less_than_or_equal_to(self, val): + self.query.and_filter(lte("name", val)) + return self + + def with_name_between(self, lower, upper): + self.query.and_filter(between(column("name"), value(lower), value(upper))) + return self + + def with_name_is_known(self): + self.query.and_filter(is_not_null(column("name"))) + return self + + def with_name_is_unknown(self): + self.query.and_filter(is_null(column("name"))) + return self + + def with_version_is(self, val): + self.query.and_filter(eq("version", val)) + return self + + def with_version_is_not(self, val): + self.query.and_filter(ne("version", val)) + return self + + def with_version_in(self, *vals): + self.query.and_filter(in_list("version", list(vals))) + return self + + def with_version_not_in(self, *vals): + self.query.and_filter(not_in_list("version", list(vals))) + return self + + def with_version_greater_than(self, val): + self.query.and_filter(gt("version", val)) + return self + + def with_version_greater_than_or_equal_to(self, val): + self.query.and_filter(gte("version", val)) + return self + + def with_version_less_than(self, val): + self.query.and_filter(lt("version", val)) + return self + + def with_version_less_than_or_equal_to(self, val): + self.query.and_filter(lte("version", val)) + return self + + def with_version_between(self, lower, upper): + self.query.and_filter(between(column("version"), value(lower), value(upper))) + return self + + def with_version_is_known(self): + self.query.and_filter(is_not_null(column("version"))) + return self + + def with_version_is_unknown(self): + self.query.and_filter(is_null(column("version"))) + return self + + def order_by_id_ascending(self): + self.query.order_by("id", "asc") + return self + + def order_by_id_descending(self): + self.query.order_by("id", "desc") + return self + + def order_by_name_ascending(self): + self.query.order_by("name", "asc") + return self + + def order_by_name_descending(self): + self.query.order_by("name", "desc") + return self + + def order_by_version_ascending(self): + self.query.order_by("version", "asc") + return self + + def order_by_version_descending(self): + self.query.order_by("version", "desc") + return self + + + def count(self): + self.query.count_field("id", "count") + self._page_count_alias = "count" + self._page_count_method = "count" + return self + + def count_as(self, ret_name: str): + self.query.count_field("id", ret_name) + self._page_count_alias = ret_name + self._page_count_method = "count_as" + return self + + def group_by_id(self): + self.query.group_by("id") + return self + + def group_by_id_as(self, ret_name: str): + self.query.group_by("id") + return self + def group_by_name(self): + self.query.group_by("name") + return self + + def group_by_name_as(self, ret_name: str): + self.query.group_by("name") + return self + def group_by_version(self): + self.query.group_by("version") + return self + + def group_by_version_as(self, ret_name: str): + self.query.group_by("version") + return self + def select_order_status_list(self): + from requests.order_status_request import OrderStatusRequest + return self.select_order_status_list_with(OrderStatusRequest()) + + def select_order_status_list_with(self, child_request): + self.query.relation_query("order_status_list", child_request.query, + QueryOperation("relation-select", "select_order_status_list_with", relation="order_status_list")) + return self + def select_customer_list(self): + from requests.customer_request import CustomerRequest + return self.select_customer_list_with(CustomerRequest()) + + def select_customer_list_with(self, child_request): + self.query.relation_query("customer_list", child_request.query, + QueryOperation("relation-select", "select_customer_list_with", relation="customer_list")) + return self + def have_order_statuses(self): + from requests.order_status_request import OrderStatusRequest + return self.with_order_status_list_matching(OrderStatusRequest()) + + def have_no_order_statuses(self): + from requests.order_status_request import OrderStatusRequest + return self.without_order_status_list_matching(OrderStatusRequest()) + + def with_order_status_list_matching(self, child_request): + child_request.query.projection = ["platform"] + self.query.and_filter(in_subquery(column("id"), "OrderStatus", child_request.query)) + return self + + def without_order_status_list_matching(self, child_request): + child_request.query.projection = ["platform"] + self.query.and_filter(not_in_subquery(column("id"), "OrderStatus", child_request.query)) + return self + def have_customers(self): + from requests.customer_request import CustomerRequest + return self.with_customer_list_matching(CustomerRequest()) + + def have_no_customers(self): + from requests.customer_request import CustomerRequest + return self.without_customer_list_matching(CustomerRequest()) + + def with_customer_list_matching(self, child_request): + child_request.query.projection = ["platform"] + self.query.and_filter(in_subquery(column("id"), "Customer", child_request.query)) + return self + + def without_customer_list_matching(self, child_request): + child_request.query.projection = ["platform"] + self.query.and_filter(not_in_subquery(column("id"), "Customer", child_request.query)) + return self + def count_order_statuses(self): + return self.count_order_statuses_as("count_order_statuses") + + def count_order_statuses_as(self, alias: str): + from requests.order_status_request import OrderStatusRequest + self.count_order_statuses_with(alias, OrderStatusRequest()) + self.query.relation_aggregates[-1].operation = QueryOperation( + "relation-count", "count_order_statuses_as", alias, "order_status_list") + return self + + def count_order_statuses_with(self, alias: str, child_request): + query = deepcopy(child_request.query) + query.aggregates = [] + query.count_field("id", alias) + self.query.relation_aggregates.append( + RelationAggregate("order_status_list", alias, query, True, + QueryOperation("relation-count", "count_order_statuses_with", alias, "order_status_list")) + ) + return self + + def min_display_order_of_order_statuses(self): + from requests.order_status_request import OrderStatusRequest + return self.min_display_order_of_order_statuses_as( + "min_display_order_of_order_statuses", OrderStatusRequest()) + + def min_display_order_of_order_statuses_as(self, alias: str, child_request): + query = deepcopy(child_request.query) + query.aggregates = [] + query.min("display_order", alias) + self.query.relation_aggregates.append( + RelationAggregate("order_status_list", alias, query, True, + QueryOperation("aggregate", "min_display_order_of_order_statuses_as", alias, "order_status_list")) + ) + return self + def max_display_order_of_order_statuses(self): + from requests.order_status_request import OrderStatusRequest + return self.max_display_order_of_order_statuses_as( + "max_display_order_of_order_statuses", OrderStatusRequest()) + + def max_display_order_of_order_statuses_as(self, alias: str, child_request): + query = deepcopy(child_request.query) + query.aggregates = [] + query.max("display_order", alias) + self.query.relation_aggregates.append( + RelationAggregate("order_status_list", alias, query, True, + QueryOperation("aggregate", "max_display_order_of_order_statuses_as", alias, "order_status_list")) + ) + return self + def sum_display_order_of_order_statuses(self): + from requests.order_status_request import OrderStatusRequest + return self.sum_display_order_of_order_statuses_as( + "sum_display_order_of_order_statuses", OrderStatusRequest()) + + def sum_display_order_of_order_statuses_as(self, alias: str, child_request): + query = deepcopy(child_request.query) + query.aggregates = [] + query.sum("display_order", alias) + self.query.relation_aggregates.append( + RelationAggregate("order_status_list", alias, query, True, + QueryOperation("aggregate", "sum_display_order_of_order_statuses_as", alias, "order_status_list")) + ) + return self + def avg_display_order_of_order_statuses(self): + from requests.order_status_request import OrderStatusRequest + return self.avg_display_order_of_order_statuses_as( + "avg_display_order_of_order_statuses", OrderStatusRequest()) + + def avg_display_order_of_order_statuses_as(self, alias: str, child_request): + query = deepcopy(child_request.query) + query.aggregates = [] + query.avg("display_order", alias) + self.query.relation_aggregates.append( + RelationAggregate("order_status_list", alias, query, True, + QueryOperation("aggregate", "avg_display_order_of_order_statuses_as", alias, "order_status_list")) + ) + return self + def standardDeviation_display_order_of_order_statuses(self): + from requests.order_status_request import OrderStatusRequest + return self.standardDeviation_display_order_of_order_statuses_as( + "standardDeviation_display_order_of_order_statuses", OrderStatusRequest()) + + def standardDeviation_display_order_of_order_statuses_as(self, alias: str, child_request): + query = deepcopy(child_request.query) + query.aggregates = [] + query.standardDeviation("display_order", alias) + self.query.relation_aggregates.append( + RelationAggregate("order_status_list", alias, query, True, + QueryOperation("aggregate", "standardDeviation_display_order_of_order_statuses_as", alias, "order_status_list")) + ) + return self + def squareRootOfPopulationStandardDeviation_display_order_of_order_statuses(self): + from requests.order_status_request import OrderStatusRequest + return self.squareRootOfPopulationStandardDeviation_display_order_of_order_statuses_as( + "squareRootOfPopulationStandardDeviation_display_order_of_order_statuses", OrderStatusRequest()) + + def squareRootOfPopulationStandardDeviation_display_order_of_order_statuses_as(self, alias: str, child_request): + query = deepcopy(child_request.query) + query.aggregates = [] + query.squareRootOfPopulationStandardDeviation("display_order", alias) + self.query.relation_aggregates.append( + RelationAggregate("order_status_list", alias, query, True, + QueryOperation("aggregate", "squareRootOfPopulationStandardDeviation_display_order_of_order_statuses_as", alias, "order_status_list")) + ) + return self + def sampleVariance_display_order_of_order_statuses(self): + from requests.order_status_request import OrderStatusRequest + return self.sampleVariance_display_order_of_order_statuses_as( + "sampleVariance_display_order_of_order_statuses", OrderStatusRequest()) + + def sampleVariance_display_order_of_order_statuses_as(self, alias: str, child_request): + query = deepcopy(child_request.query) + query.aggregates = [] + query.sampleVariance("display_order", alias) + self.query.relation_aggregates.append( + RelationAggregate("order_status_list", alias, query, True, + QueryOperation("aggregate", "sampleVariance_display_order_of_order_statuses_as", alias, "order_status_list")) + ) + return self + def samplePopulationVariance_display_order_of_order_statuses(self): + from requests.order_status_request import OrderStatusRequest + return self.samplePopulationVariance_display_order_of_order_statuses_as( + "samplePopulationVariance_display_order_of_order_statuses", OrderStatusRequest()) + + def samplePopulationVariance_display_order_of_order_statuses_as(self, alias: str, child_request): + query = deepcopy(child_request.query) + query.aggregates = [] + query.samplePopulationVariance("display_order", alias) + self.query.relation_aggregates.append( + RelationAggregate("order_status_list", alias, query, True, + QueryOperation("aggregate", "samplePopulationVariance_display_order_of_order_statuses_as", alias, "order_status_list")) + ) + return self + def count_customers(self): + return self.count_customers_as("count_customers") + + def count_customers_as(self, alias: str): + from requests.customer_request import CustomerRequest + self.count_customers_with(alias, CustomerRequest()) + self.query.relation_aggregates[-1].operation = QueryOperation( + "relation-count", "count_customers_as", alias, "customer_list") + return self + + def count_customers_with(self, alias: str, child_request): + query = deepcopy(child_request.query) + query.aggregates = [] + query.count_field("id", alias) + self.query.relation_aggregates.append( + RelationAggregate("customer_list", alias, query, True, + QueryOperation("relation-count", "count_customers_with", alias, "customer_list")) + ) + return self + + + +class ExecutablePlatformRequest: + def __init__(self, request): + self._request = request + + def comment(self, c: str): + self._request.comment(c) + return self + + def new_entity(self, context) -> Platform: + request = self._request + if not request._comment or not request._comment.strip() or not request._purpose or not request._purpose.strip(): + raise ValueError("Security audit failure: non-empty comment() and purpose() are required before new_entity()") + entity = context.initialize_entity("Platform", Platform()) + if not isinstance(entity, Platform): + raise TypeError("entity initializer returned an incompatible Platform") + return entity + + async def execute_for_result(self, context): + self = self._request + if not self._purpose or not self._purpose.strip() or not self._comment or not self._comment.strip(): + raise Exception("Security audit failure: comment() and purpose() must be called before execute_for_rows()") + service = context.require_resource("dataService") + req = QueryRequest(context.prepare_query(self.query), _comment=self._comment, _purpose=self._purpose) + return await service.query(context, req) + + async def execute_for_rows(self, context): + return (await self.execute_for_result(context)).rows + + async def execute_for_list(self, context) -> SmartList[Platform]: + from teaql.runtime.query_list import execute_typed_list + request = self._request + QueryIntent(request._comment, request._purpose) + query = deepcopy(request.query) + authorized = context.prepare_query(query) + root = QueryRequest(authorized, _comment=request._comment, _purpose=request._purpose) + operation = (QueryOperation("named-count", request._page_count_method or "count", + request._page_count_alias if request._page_count_method == "count_as" else None) + if request._page_count_alias is not None else None) + return await execute_typed_list(context, root, + lambda row: Platform(_entity_root=EntityRoot(), **row), + count_operation=operation, count_alias=request._page_count_alias) + + async def execute_for_one(self, context): + request = deepcopy(self._request) + request.limit(1) + entities = await ExecutablePlatformRequest(request).execute_for_list(context) + return entities[0] if entities else None + + def execute_for_stream(self, context, chunk_size: int = 1000): + """Yield entity chunks lazily from the provider cursor.""" + request = deepcopy(self._request) + QueryIntent(request._comment, request._purpose) + service = context.require_resource("dataService") + if not hasattr(service, "query_stream"): + raise RuntimeError("dataService does not implement query_stream") + root = QueryRequest(context.prepare_query(request.query), _comment=request._comment, _purpose=request._purpose) + from teaql.runtime.query_stream import execute_typed_stream + return execute_typed_stream(context, root, + lambda row: Platform(_entity_root=EntityRoot(), **row), chunk_size) diff --git a/examples/trace-chain/standard-order/lib/runtime_module.py b/examples/trace-chain/standard-order/lib/runtime_module.py new file mode 100644 index 0000000..58caa92 --- /dev/null +++ b/examples/trace-chain/standard-order/lib/runtime_module.py @@ -0,0 +1,326 @@ +import asyncio +from datetime import datetime, timezone +from teaql.runtime import CheckResult, ContextEntityRef, JsonFieldNamingProfile, ObjectLocation, RuntimeModule, create_wire_entity_metadata +from teaql.core.meta import EntityDescriptor, PropertyDescriptor, RelationDescriptor +from teaql.core.value import DataType +from Q import Q +from teaql.core.value import Value +try: + from teaql.core.graph import GraphNode +except ImportError: + class GraphNode: + def __init__(self, entity): + self.entity, self.fields = entity, {} + def set(self, field, value): + self.fields[field] = value + return self +from models.platform import Platform +from models.order_status import OrderStatus +from models.customer import Customer +from models.customer_order import CustomerOrder +from models.order_item import OrderItem + +def _teaql_is_null(value): + return value.is_null() if hasattr(value, "is_null") else value is None + +def _teaql_raw(value): + return value.val if hasattr(value, "val") else value + +def _teaql_entity_id(value): + value = _teaql_raw(value) + if hasattr(value, "id"): + return value.id + if isinstance(value, dict): + return value.get("id") + return value + +class _PlatformChecker: + def check_and_fix(self, context, record, location, results): + operation = context.get_resource("fix_operation") + now = context.get_resource("fix_time") + if (operation == "insert" and "name" not in record) or ("name" in record and _teaql_is_null(record["name"])): + results.append(CheckResult("required", ObjectLocation().property("name"))) + if "name" in record and _teaql_raw(record["name"]) is not None and len(_teaql_raw(record["name"])) > 100: + results.append(CheckResult("max_length", ObjectLocation().property("name"), _teaql_raw(record["name"]), 100)) + + + +class _OrderStatusChecker: + def check_and_fix(self, context, record, location, results): + operation = context.get_resource("fix_operation") + now = context.get_resource("fix_time") + if (operation == "insert" and "platform" not in record) or ("platform" in record and _teaql_is_null(record["platform"])): + results.append(CheckResult("required", ObjectLocation().property("platform"))) + + + if (operation == "insert" and "name" not in record) or ("name" in record and _teaql_is_null(record["name"])): + results.append(CheckResult("required", ObjectLocation().property("name"))) + if "name" in record and _teaql_raw(record["name"]) is not None and len(_teaql_raw(record["name"])) > 100: + results.append(CheckResult("max_length", ObjectLocation().property("name"), _teaql_raw(record["name"]), 100)) + + if (operation == "insert" and "code" not in record) or ("code" in record and _teaql_is_null(record["code"])): + results.append(CheckResult("required", ObjectLocation().property("code"))) + if "code" in record and _teaql_raw(record["code"]) is not None and len(_teaql_raw(record["code"])) > 100: + results.append(CheckResult("max_length", ObjectLocation().property("code"), _teaql_raw(record["code"]), 100)) + + if (operation == "insert" and "color" not in record) or ("color" in record and _teaql_is_null(record["color"])): + results.append(CheckResult("required", ObjectLocation().property("color"))) + if "color" in record and _teaql_raw(record["color"]) is not None and len(_teaql_raw(record["color"])) > 100: + results.append(CheckResult("max_length", ObjectLocation().property("color"), _teaql_raw(record["color"]), 100)) + + if (operation == "insert" and "display_order" not in record) or ("display_order" in record and _teaql_is_null(record["display_order"])): + results.append(CheckResult("required", ObjectLocation().property("display_order"))) + + + +class _CustomerChecker: + def check_and_fix(self, context, record, location, results): + operation = context.get_resource("fix_operation") + now = context.get_resource("fix_time") + if (operation == "insert" and "platform" not in record) or ("platform" in record and _teaql_is_null(record["platform"])): + results.append(CheckResult("required", ObjectLocation().property("platform"))) + + if (operation == "insert" and "name" not in record) or ("name" in record and _teaql_is_null(record["name"])): + results.append(CheckResult("required", ObjectLocation().property("name"))) + if "name" in record and _teaql_raw(record["name"]) is not None and len(_teaql_raw(record["name"])) > 100: + results.append(CheckResult("max_length", ObjectLocation().property("name"), _teaql_raw(record["name"]), 100)) + + + +class _CustomerOrderChecker: + def check_and_fix(self, context, record, location, results): + operation = context.get_resource("fix_operation") + now = context.get_resource("fix_time") + if (operation == "insert" and "customer" not in record) or ("customer" in record and _teaql_is_null(record["customer"])): + results.append(CheckResult("required", ObjectLocation().property("customer"))) + + if (operation == "insert" and "status" not in record) or ("status" in record and _teaql_is_null(record["status"])): + results.append(CheckResult("required", ObjectLocation().property("status"))) + + if (operation == "insert" and "order_number" not in record) or ("order_number" in record and _teaql_is_null(record["order_number"])): + results.append(CheckResult("required", ObjectLocation().property("order_number"))) + if "order_number" in record and _teaql_raw(record["order_number"]) is not None and len(_teaql_raw(record["order_number"])) > 100: + results.append(CheckResult("max_length", ObjectLocation().property("order_number"), _teaql_raw(record["order_number"]), 100)) + + if (operation == "insert" and "order_date" not in record) or ("order_date" in record and _teaql_is_null(record["order_date"])): + results.append(CheckResult("required", ObjectLocation().property("order_date"))) + + if (operation == "insert" and "total_amount" not in record) or ("total_amount" in record and _teaql_is_null(record["total_amount"])): + results.append(CheckResult("required", ObjectLocation().property("total_amount"))) + + + +class _OrderItemChecker: + def check_and_fix(self, context, record, location, results): + operation = context.get_resource("fix_operation") + now = context.get_resource("fix_time") + if (operation == "insert" and "customer_order" not in record) or ("customer_order" in record and _teaql_is_null(record["customer_order"])): + results.append(CheckResult("required", ObjectLocation().property("customer_order"))) + + if (operation == "insert" and "product_name" not in record) or ("product_name" in record and _teaql_is_null(record["product_name"])): + results.append(CheckResult("required", ObjectLocation().property("product_name"))) + if "product_name" in record and _teaql_raw(record["product_name"]) is not None and len(_teaql_raw(record["product_name"])) > 100: + results.append(CheckResult("max_length", ObjectLocation().property("product_name"), _teaql_raw(record["product_name"]), 100)) + + if (operation == "insert" and "quantity" not in record) or ("quantity" in record and _teaql_is_null(record["quantity"])): + results.append(CheckResult("required", ObjectLocation().property("quantity"))) + + if (operation == "insert" and "unit_price" not in record) or ("unit_price" in record and _teaql_is_null(record["unit_price"])): + results.append(CheckResult("required", ObjectLocation().property("unit_price"))) + + + +_Platform_DESCRIPTOR = (EntityDescriptor("Platform") + .audit_mask_fields([]) + .table_name("platform_data").property(PropertyDescriptor("id", DataType.I64).column_name("id").log_policy("plain").is_id().required()).property(PropertyDescriptor("name", DataType.Text).column_name("name").log_policy("plain").required()).property(PropertyDescriptor("version", DataType.I64).column_name("version").log_policy("plain").is_version().required()).relation(RelationDescriptor("order_status_list", "OrderStatus").local("id").foreign("platform").many()).relation(RelationDescriptor("customer_list", "Customer").local("id").foreign("platform").many()) +) + +_OrderStatus_DESCRIPTOR = (EntityDescriptor("OrderStatus") + .audit_mask_fields([]) + .table_name("order_status_data").property(PropertyDescriptor("platform", DataType.I64).column_name("platform").log_policy("plain").required()).property(PropertyDescriptor("id", DataType.I64).column_name("id").log_policy("plain").is_id().required()).property(PropertyDescriptor("name", DataType.Text).column_name("name").log_policy("plain").required()).property(PropertyDescriptor("code", DataType.Text).column_name("code").log_policy("plain").required()).property(PropertyDescriptor("color", DataType.Text).column_name("color").log_policy("plain").required()).property(PropertyDescriptor("display_order", DataType.I64).column_name("display_order").log_policy("plain").required()).property(PropertyDescriptor("version", DataType.I64).column_name("version").log_policy("plain").is_version().required()).relation(RelationDescriptor("platform", "Platform").local("platform").foreign("id")).relation(RelationDescriptor("customer_order_list", "CustomerOrder").local("id").foreign("status").many()) +) + +_Customer_DESCRIPTOR = (EntityDescriptor("Customer") + .audit_mask_fields([]) + .table_name("customer_data").property(PropertyDescriptor("id", DataType.I64).column_name("id").log_policy("plain").is_id().required()).property(PropertyDescriptor("platform", DataType.I64).column_name("platform").log_policy("plain").required()).property(PropertyDescriptor("name", DataType.Text).column_name("name").log_policy("plain").required()).property(PropertyDescriptor("version", DataType.I64).column_name("version").log_policy("plain").is_version().required()).relation(RelationDescriptor("platform", "Platform").local("platform").foreign("id")).relation(RelationDescriptor("customer_order_list", "CustomerOrder").local("id").foreign("customer").many()) +) + +_CustomerOrder_DESCRIPTOR = (EntityDescriptor("CustomerOrder") + .audit_mask_fields([]) + .table_name("customer_order_data").property(PropertyDescriptor("id", DataType.I64).column_name("id").log_policy("plain").is_id().required()).property(PropertyDescriptor("customer", DataType.I64).column_name("customer").log_policy("plain").required()).property(PropertyDescriptor("status", DataType.I64).column_name("status").log_policy("plain").required()).property(PropertyDescriptor("order_number", DataType.Text).column_name("order_number").log_policy("plain").required()).property(PropertyDescriptor("order_date", DataType.Date).column_name("order_date").log_policy("plain").required()).property(PropertyDescriptor("total_amount", DataType.Decimal).column_name("total_amount").log_policy("plain").required()).property(PropertyDescriptor("version", DataType.I64).column_name("version").log_policy("plain").is_version().required()).relation(RelationDescriptor("customer", "Customer").local("customer").foreign("id")).relation(RelationDescriptor("status", "OrderStatus").local("status").foreign("id")).relation(RelationDescriptor("order_item_list", "OrderItem").local("id").foreign("customer_order").many()) +) + +_OrderItem_DESCRIPTOR = (EntityDescriptor("OrderItem") + .audit_mask_fields([]) + .table_name("order_item_data").property(PropertyDescriptor("id", DataType.I64).column_name("id").log_policy("plain").is_id().required()).property(PropertyDescriptor("customer_order", DataType.I64).column_name("customer_order").log_policy("plain").required()).property(PropertyDescriptor("product_name", DataType.Text).column_name("product_name").log_policy("plain").required()).property(PropertyDescriptor("quantity", DataType.I64).column_name("quantity").log_policy("plain").required()).property(PropertyDescriptor("unit_price", DataType.Decimal).column_name("unit_price").log_policy("plain").required()).property(PropertyDescriptor("version", DataType.I64).column_name("version").log_policy("plain").is_version().required()).relation(RelationDescriptor("customer_order", "CustomerOrder").local("customer_order").foreign("id")) +) + +async def _ensure_generated_bootstrap_once(context): + previous_actor = context.user_identifier() if hasattr(context, 'user_identifier') else None + previous_category = context.get_resource('bootstrapCategory') + if hasattr(context, 'set_user_identifier'): + context.set_user_identifier('teaql-generated-bootstrap') + context.insert_resource('bootstrapCategory', 'runtime-bootstrap') + try: + platform_1 = await (Q.platforms().with_id_is(1).comment('what: locate generated bootstrap entity').purpose('why: idempotent runtime bootstrap').execute_for_one(context)) + if platform_1 is None: + platform_1 = Platform._teaql_new_with_fixed_id(1) + platform_1.update_name("Commerce Test Platform") + try: + await platform_1.audit_as('create model root Platform(1)').save(context) + except Exception as _teaql_create_error: + for _teaql_attempt in range(5): + platform_1 = await (Q.platforms().with_id_is(1).comment('what: recover concurrent bootstrap').purpose('why: make generated bootstrap idempotent').execute_for_one(context)) + if platform_1 is not None: + break + if _teaql_attempt < 4: + await asyncio.sleep((_teaql_attempt + 1) * 0.01) + if platform_1 is None: + raise _teaql_create_error + context.with_active_root(ContextEntityRef("Platform", 1)) + order_status_1001 = await (Q.order_statuses().with_id_is(1001).comment('what: locate generated bootstrap entity').purpose('why: idempotent runtime bootstrap').execute_for_one(context)) + if order_status_1001 is None: + order_status_1001 = OrderStatus._teaql_new_with_fixed_id(1001) + order_status_1001.update_platform(Platform.refer(1)) + order_status_1001.update_name("New") + order_status_1001.update_code("NEW") + order_status_1001.update_color("#94A3B8") + order_status_1001.update_display_order(1) + try: + await order_status_1001.audit_as('create model constant OrderStatus(1001)').save(context) + except Exception as _teaql_create_error: + for _teaql_attempt in range(5): + order_status_1001 = await (Q.order_statuses().with_id_is(1001).comment('what: recover concurrent bootstrap').purpose('why: make generated bootstrap idempotent').execute_for_one(context)) + if order_status_1001 is not None: + break + if _teaql_attempt < 4: + await asyncio.sleep((_teaql_attempt + 1) * 0.01) + if order_status_1001 is None: + raise _teaql_create_error + _teaql_changed = False + if order_status_1001.platform != 1: + order_status_1001.update_platform(Platform.refer(1)) + _teaql_changed = True + if order_status_1001.name != "New": + order_status_1001.update_name("New") + _teaql_changed = True + if order_status_1001.code != "NEW": + order_status_1001.update_code("NEW") + _teaql_changed = True + if order_status_1001.color != "#94A3B8": + order_status_1001.update_color("#94A3B8") + _teaql_changed = True + if order_status_1001.displayOrder != 1: + order_status_1001.update_display_order(1) + _teaql_changed = True + if _teaql_changed: + await order_status_1001.audit_as('reconcile model constant OrderStatus(1001)').save(context) + order_status_1002 = await (Q.order_statuses().with_id_is(1002).comment('what: locate generated bootstrap entity').purpose('why: idempotent runtime bootstrap').execute_for_one(context)) + if order_status_1002 is None: + order_status_1002 = OrderStatus._teaql_new_with_fixed_id(1002) + order_status_1002.update_platform(Platform.refer(1)) + order_status_1002.update_name("Paid") + order_status_1002.update_code("PAID") + order_status_1002.update_color("#22C55E") + order_status_1002.update_display_order(2) + try: + await order_status_1002.audit_as('create model constant OrderStatus(1002)').save(context) + except Exception as _teaql_create_error: + for _teaql_attempt in range(5): + order_status_1002 = await (Q.order_statuses().with_id_is(1002).comment('what: recover concurrent bootstrap').purpose('why: make generated bootstrap idempotent').execute_for_one(context)) + if order_status_1002 is not None: + break + if _teaql_attempt < 4: + await asyncio.sleep((_teaql_attempt + 1) * 0.01) + if order_status_1002 is None: + raise _teaql_create_error + _teaql_changed = False + if order_status_1002.platform != 1: + order_status_1002.update_platform(Platform.refer(1)) + _teaql_changed = True + if order_status_1002.name != "Paid": + order_status_1002.update_name("Paid") + _teaql_changed = True + if order_status_1002.code != "PAID": + order_status_1002.update_code("PAID") + _teaql_changed = True + if order_status_1002.color != "#22C55E": + order_status_1002.update_color("#22C55E") + _teaql_changed = True + if order_status_1002.displayOrder != 2: + order_status_1002.update_display_order(2) + _teaql_changed = True + if _teaql_changed: + await order_status_1002.audit_as('reconcile model constant OrderStatus(1002)').save(context) + order_status_1003 = await (Q.order_statuses().with_id_is(1003).comment('what: locate generated bootstrap entity').purpose('why: idempotent runtime bootstrap').execute_for_one(context)) + if order_status_1003 is None: + order_status_1003 = OrderStatus._teaql_new_with_fixed_id(1003) + order_status_1003.update_platform(Platform.refer(1)) + order_status_1003.update_name("Cancelled") + order_status_1003.update_code("CANCELLED") + order_status_1003.update_color("#EF4444") + order_status_1003.update_display_order(3) + try: + await order_status_1003.audit_as('create model constant OrderStatus(1003)').save(context) + except Exception as _teaql_create_error: + for _teaql_attempt in range(5): + order_status_1003 = await (Q.order_statuses().with_id_is(1003).comment('what: recover concurrent bootstrap').purpose('why: make generated bootstrap idempotent').execute_for_one(context)) + if order_status_1003 is not None: + break + if _teaql_attempt < 4: + await asyncio.sleep((_teaql_attempt + 1) * 0.01) + if order_status_1003 is None: + raise _teaql_create_error + _teaql_changed = False + if order_status_1003.platform != 1: + order_status_1003.update_platform(Platform.refer(1)) + _teaql_changed = True + if order_status_1003.name != "Cancelled": + order_status_1003.update_name("Cancelled") + _teaql_changed = True + if order_status_1003.code != "CANCELLED": + order_status_1003.update_code("CANCELLED") + _teaql_changed = True + if order_status_1003.color != "#EF4444": + order_status_1003.update_color("#EF4444") + _teaql_changed = True + if order_status_1003.displayOrder != 3: + order_status_1003.update_display_order(3) + _teaql_changed = True + if _teaql_changed: + await order_status_1003.audit_as('reconcile model constant OrderStatus(1003)').save(context) + finally: + if hasattr(context, 'set_user_identifier'): + context.set_user_identifier(previous_actor) + context.insert_resource('bootstrapCategory', previous_category) + +async def _ensure_generated_bootstrap(context): + for _teaql_attempt in range(5): + try: + await _ensure_generated_bootstrap_once(context) + return + except Exception: + if _teaql_attempt == 4: + raise + await asyncio.sleep((_teaql_attempt + 1) * 0.01) + + +# Passive generated manifest. Call ensure_schema() separately and explicitly. +GENERATED_RUNTIME_MODULE = (RuntimeModule().entity(Platform) + .schema_entity(_Platform_DESCRIPTOR) + .checker("Platform", _PlatformChecker()) + .wire_metadata("Platform", create_wire_entity_metadata("Platform", ["id", "name", "version"], JsonFieldNamingProfile.CAMEL_CASE, {"id": ["id"], "name": ["name"], "version": ["version"]})).entity(OrderStatus) + .schema_entity(_OrderStatus_DESCRIPTOR) + .checker("OrderStatus", _OrderStatusChecker()) + .wire_metadata("OrderStatus", create_wire_entity_metadata("OrderStatus", ["platform", "id", "name", "code", "color", "display_order", "version"], JsonFieldNamingProfile.CAMEL_CASE, {"platform": ["platform"], "id": ["id"], "name": ["name"], "code": ["code"], "color": ["color"], "display_order": ["display_order"], "version": ["version"]})).entity(Customer) + .schema_entity(_Customer_DESCRIPTOR) + .checker("Customer", _CustomerChecker()) + .wire_metadata("Customer", create_wire_entity_metadata("Customer", ["id", "platform", "name", "version"], JsonFieldNamingProfile.CAMEL_CASE, {"id": ["id"], "platform": ["platform"], "name": ["name"], "version": ["version"]})).entity(CustomerOrder) + .schema_entity(_CustomerOrder_DESCRIPTOR) + .checker("CustomerOrder", _CustomerOrderChecker()) + .wire_metadata("CustomerOrder", create_wire_entity_metadata("CustomerOrder", ["id", "customer", "status", "order_number", "order_date", "total_amount", "version"], JsonFieldNamingProfile.CAMEL_CASE, {"id": ["id"], "customer": ["customer"], "status": ["status"], "order_number": ["order_number"], "order_date": ["order_date"], "total_amount": ["total_amount"], "version": ["version"]})).entity(OrderItem) + .schema_entity(_OrderItem_DESCRIPTOR) + .checker("OrderItem", _OrderItemChecker()) + .wire_metadata("OrderItem", create_wire_entity_metadata("OrderItem", ["id", "customer_order", "product_name", "quantity", "unit_price", "version"], JsonFieldNamingProfile.CAMEL_CASE, {"id": ["id"], "customer_order": ["customer_order"], "product_name": ["product_name"], "quantity": ["quantity"], "unit_price": ["unit_price"], "version": ["version"]})) + .generated_bootstrap(_ensure_generated_bootstrap) +) \ No newline at end of file diff --git a/examples/trace-chain/standard-order/main.py b/examples/trace-chain/standard-order/main.py new file mode 100644 index 0000000..6d21d76 --- /dev/null +++ b/examples/trace-chain/standard-order/main.py @@ -0,0 +1,106 @@ +"""Generated Order Q/E and audited seed replay; no application SDK setup.""" +import asyncio +from datetime import date +from decimal import Decimal +import os +from pathlib import Path +import sys +import tempfile +from uuid import uuid4 + +from Q import Q +from E import E +from runtime_module import GENERATED_RUNTIME_MODULE +from teaql.data_service import SQLiteTeaQLClient +from teaql.runtime import UserContext + + +class Audits: + def __init__(self): + self.count = 0 + + async def on_safe_event(self, context, event): + self.count += 1 + + +async def main(): + database = os.environ.get('TEAQL_STANDARD_ORDER_DB') + if not database: + database = str(Path(tempfile.mkdtemp(prefix='teaql-python-order-')) / 'orders.sqlite') + service = SQLiteTeaQLClient(database) + context = UserContext.new().install(GENERATED_RUNTIME_MODULE).insert_resource('dataService', service) + assert context.entity('CustomerOrder') is not None, 'standard Order metadata missing' + context.configure_audit_policy('CustomerOrder', ['order_number']) + audit = Audits() + context.with_app_audit_event_sink(audit).disable_sql_log() + try: + await context.ensure_schema() + first = audit.count + assert first in (0, 4), 'generated platform and three constants' + await context.ensure_schema() + assert audit.count == first, 'repeated schema must not write more seeds' + platforms = await (Q.platforms().with_id_is(1).limit(1).comment('load seeded platform') + .purpose('prepare standard Order graph').execute_for_list(context)) + assert len(platforms) == 1 and E.platform(platforms[0]).id().eval() == 1 + secret = 'ORDER-PRIVATE-' + uuid4().hex + customer = Q.customers().comment('create standard customer').purpose('prepare Order graph').new_entity(context) + customer.update_name('North Star Retail').update_platform(platforms[0]) + await customer.audit_as('seed standard customer').save(context) + for index in range(1, 5): + order = Q.customer_orders().comment('create standard order').purpose('prepare Order graph').new_entity(context) + order.update_customer(customer).update_order_number(f'{secret}-{index:03}' if index < 4 else 'OTHER-004') + order.update_order_date(date(2026, 10, index)).update_total_amount(Decimal(index * 100 if index < 4 else 900)) + if index == 1: + order.update_status_to_new() + else: + order.update_status_to_paid() + await order.audit_as('seed standard order').save(context) + for item in range(1, 6): + line = Q.order_items().comment('create standard item').purpose('prepare bounded previews').new_entity(context) + line.update_customer_order(order).update_product_name(f'Box {item}').update_quantity(item) + line.update_unit_price(Decimal(20)) + await line.audit_as('seed standard item').save(context) + for repetition in range(4): + counted = repetition < 2 + request = (Q.customer_orders_minimal().with_order_number_containing(secret) + .facet_by_status_as('statusCounts', Q.order_statuses_minimal() + .select_name().select_color().count_as('orderCount') + .sum_total_amount_of_customer_orders_as('orderTotalAmount', Q.customer_orders_minimal()) + .order_by_display_order_ascending().comment('load status facets')) + .select_order_number().select_order_date() + .select_status_with(Q.order_statuses_minimal().select_name().select_color().comment('load status')) + .select_order_item_list_with(Q.order_items_minimal().select_product_name().select_quantity() + .order_by_id_ascending().limit(3).comment('load item previews')) + .count_order_items_as('itemCount').order_by_order_date_descending().order_by_id_descending() + .offset(1).limit(1).comment('load order list')) + if counted: + request.count_as('totalOrders') + rows = await request.purpose('render order list page').execute_for_list(context) + assert len(rows) == 1 + if counted: + assert rows.total_count == rows.aggregations['totalOrders'] == 3 + else: + assert rows.aggregations == {}, 'ordinary list does not request an exact total' + row = rows[0] + assert E.customer_order(row).order_number().eval() == secret + '-002' + assert E.customer_order(row).order_date().eval() == date(2026, 10, 2) + assert E.customer_order(row).status().name().eval() == 'Paid' + assert E.customer_order(row).order_item_list().size().eval() == 3 + assert E.customer_order(row).order_item_list().first().product_name().eval() == 'Box 1' + assert row.get_dynamic_property('itemCount') == 5 + facets = rows.facets['statusCounts'] + assert [(f['id'], f['orderCount'], f['orderTotalAmount']) for f in facets] == [ + (1001, 1, Decimal(100)), (1002, 2, Decimal(500)), (1003, 0, None)] + assert audit.count == first + 25 and not context.sql_logs() + if os.environ.get('TEAQL_TELEMETRY_ENABLED', '').lower() != 'true': + assert not any(name.startswith('opentelemetry') for name in sys.modules) + await asyncio.to_thread(context.runtime_telemetry().flush) + print(f'PYTHON_STANDARD_ORDER PASSED queries=4 counted=2 uncounted=2 total=3 preview=3 itemCount=5 facets=3 ' + f'first_seed_audits={first} repeat_seed_audits=0 audits={audit.count} ordinary_sql_logs=0 database={database}') + finally: + await service.close() + await asyncio.to_thread(context.runtime_telemetry().shutdown) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/examples/trace-chain/standard_graph.py b/examples/trace-chain/standard_graph.py new file mode 100644 index 0000000..2297338 --- /dev/null +++ b/examples/trace-chain/standard_graph.py @@ -0,0 +1,198 @@ +"""Real generated graph saves; standard telemetry owns all instrumentation.""" +import asyncio +import json +import logging +import os +from pathlib import Path +import sys +import tempfile +from time import monotonic +from uuid import uuid4 + +from E import E +from Q import Q +from runtime_module import GENERATED_RUNTIME_MODULE +from teaql.provider.sqlite import create_sqlite_service +from teaql.runtime import UserContext +from teaql.sql.executor import TransportError +from main import AuditSink, ObservedService, chain, expected_graph, make_graph, reset, scalar + + +class FailingService(ObservedService): + """Fault injection at the physical transport, never synthetic telemetry.""" + inject = False + + def __init__(self, service): + super().__init__(service) + self.failure = RuntimeError('GRAPH-PRIVATE-TRANSPORT-' + uuid4().hex) + + async def begin(self, context): + transaction = await super().begin(context) + original = transaction.transport.execute_sql + + async def execute(compiled): + if self.inject and 'payment_attempt_data' in compiled.sql: + raise self.failure + return await original(compiled) + + transaction.transport.execute_sql = execute + return transaction + + +def facts(service, sink, phase, reason): + statements = [statement for metadata in service.results for statement in metadata.statements] + # The failed operation has no successful result object. Its physical span + # and SQL completion are checked independently in the collector readback. + return {'phase': phase, 'reason': reason, 'requests': len(service.requests), + 'successful_results': len(service.results), 'audits': len(sink.events), + 'actions': list(service.actions), 'finishes': list(service.finishes), + 'successful_statements': len(statements)} + + +def committed_lineage(graph, service, sink, reason, explicit): + expected = expected_graph(graph, reason) if explicit else { + (entity, identity): [('CustomerOrder', E.customer_order(graph[0]).id().eval(), reason)] + for entity, identity, _ in service.requests} + commands = {(entity, identity): nodes for entity, identity, nodes in service.requests} + audits = {(event.entity, scalar(event.entity_id)): chain(event.trace_chain) for event in sink.events} + assert len(commands) == len(audits) == len(service.requests) == len(sink.events) == 6 + assert commands == audits == expected, (commands, audits, expected) + assert service.finishes == ['commit'] + assert len(service.results) == 6 + for (entity, identity, _), result in zip(service.requests, service.results): + assert chain(result.mutation_lineage) == expected[(entity, identity)] + assert len(result.statements) == 2 + for statement in result.statements: + assert chain(statement.mutation_lineage) == expected[(entity, identity)] + assert statement.execution_outcome == 'success' + + +async def main(): + if os.environ.get('TEAQL_STANDARD_EXPORT_DEBUG') == 'true': + logging.basicConfig(level=logging.DEBUG) + database = os.environ.get('TEAQL_STANDARD_GRAPH_DB') or str( + Path(tempfile.mkdtemp(prefix='teaql-python-graph-')) / 'graph.sqlite') + context = UserContext.new().install(GENERATED_RUNTIME_MODULE) + service = FailingService(create_sqlite_service(database)) + sink = AuditSink(service) + context.insert_resource('dataService', service).with_app_audit_event_sink(sink).disable_sql_log() + # Numeric identities are private audit inputs. Do not accidentally embed + # those ID spellings in the public business-reason correlation label. + nonce = uuid4().hex.translate(str.maketrans('0123456789', 'klmnopqrst')) + observations = [] + try: + await context.ensure_schema() + first = len(sink.events) + assert first in (0, 1) + await context.ensure_schema() + assert len(sink.events) == first, 'repeated seed must not write' + platform = await (Q.platforms().with_id_is(1).limit(1).comment('load seeded graph platform') + .purpose('prepare standard graph example').execute_for_one(context)) + graph = make_graph(context, platform, 'GRAPH-' + nonce) + order, item, payment, attempt, shipment, removed = graph + reason = 'GRAPH-create-' + nonce + reset(service, sink, context) + await order.audit_as(reason).save(context) + committed_lineage(graph, service, sink, reason, False) + observations.append(facts(service, sink, 'create', reason)) + assert E.payment(payment).id().eval() == E.customer_order(order).id().eval(), 'same ID, distinct entity types' + + order.update_description('Submitted standard graph') + item.update_name('Updated standard item') + payment.update_reference_code('Authorized payment').audit_as('authorize payment') + attempt.update_reference_code('Completed payment attempt') + shipment.update_reference_code('Dispatched shipment').audit_as('dispatch shipment') + removed.mark_for_deletion().audit_as('remove unavailable item') + reason = 'GRAPH-mixed-' + nonce + reset(service, sink, context) + await order.audit_as(reason).save(context) + committed_lineage(graph, service, sink, reason, True) + assert service.actions.count('update') == 5 and service.actions.count('delete') == 1 + observations.append(facts(service, sink, 'mixed', reason)) + loaded = await (Q.customer_orders().with_id_is(E.customer_order(order).id().eval()) + .select_order_item_list_with(Q.order_items().limit(10)) + .select_payment_list_with(Q.payments().limit(10) + .select_payment_attempt_list_with(Q.payment_attempts().limit(10))) + .select_shipment_list_with(Q.shipments().limit(10)).limit(1) + .comment('reload committed standard graph').purpose('verify generated loaded E').execute_for_one(context)) + assert E.customer_order(loaded).order_item_list().size().eval() == 1 + assert E.customer_order(loaded).payment_list().size().eval() == 1 + deleted = await (Q.order_items().with_id_is(E.order_item(removed).id().eval()).deleted_rows_only() + .limit(1).comment('verify standard graph deletion').purpose('verify persisted negative version') + .execute_for_one(context)) + assert E.order_item(deleted).version().eval() < 0 + + retry = make_graph(context, platform, 'RETRY-' + nonce) + reason = 'GRAPH-rollback-' + nonce + reset(service, sink, context) + service.inject = True + try: + await retry[0].audit_as(reason).save(context) + except (TransportError, RuntimeError) as error: + actual = error.error if isinstance(error, TransportError) else error + assert actual is service.failure, 'original business exception must survive' + else: + raise AssertionError('physical failure was not raised') + finally: + service.inject = False + assert service.finishes == ['rollback'] and not sink.events + observations.append(facts(service, sink, 'rollback', reason)) + absent = await (Q.customer_orders().with_order_number_is('RETRY-' + nonce).limit(1) + .comment('verify rolled back graph').purpose('verify no committed order').execute_for_list(context)) + assert len(absent) == 0 + reason = 'GRAPH-retry-' + nonce + reset(service, sink, context) + await retry[0].audit_as(reason).save(context) + committed_lineage(retry, service, sink, reason, False) + observations.append(facts(service, sink, 'retry', reason)) + assert not context.sql_logs(), 'ordinary SQL logging stays disabled' + if os.environ.get('TEAQL_TELEMETRY_ENABLED', '').lower() != 'true': + assert not any(name.startswith('opentelemetry') for name in sys.modules) + await asyncio.to_thread(context.runtime_telemetry().flush) + mode = os.environ.get('TEAQL_STANDARD_QUEUE_MODE') + if mode: + assert mode in ('recovery', 'backlog') + telemetry = context.runtime_telemetry() + baseline = telemetry.health() + audit_count = len(sink.events) + async def read_platform(): + rows = await (Q.platforms().with_id_is(1).limit(1).comment('read platform under queue pressure') + .purpose('verify default SDK admission and business continuity').execute_for_list(context)) + assert len(rows) == 1 and E.platform(rows[0]).id().eval() == 1 + for _ in range(1500): + await read_platform() + saturated = telemetry.health() + assert len(sink.events) == audit_count == 6 + print('PYTHON_PRESSURE_READY ' + json.dumps({'mode': mode, 'reads': 1500, + 'baseline': baseline, 'saturated': saturated}, sort_keys=True), flush=True) + started = monotonic() + if mode == 'recovery': + await asyncio.to_thread(telemetry.flush) + drained = telemetry.health() + await read_platform() + await asyncio.to_thread(telemetry.flush) + recovered = telemetry.health() + print('PYTHON_PRESSURE_RESULT ' + json.dumps({'mode': mode, 'elapsed_ms': (monotonic() - started) * 1000, + 'drained': drained, 'recovered': recovered, 'audits': len(sink.events)}, sort_keys=True), flush=True) + else: + await asyncio.to_thread(telemetry.shutdown) + closed = telemetry.health() + elapsed = (monotonic() - started) * 1000 + await read_platform() + late = telemetry.health() + assert len(sink.events) == audit_count and not context.sql_logs() + print('PYTHON_PRESSURE_RESULT ' + json.dumps({'mode': mode, 'elapsed_ms': elapsed, + 'closed': closed, 'late': late, 'audits': len(sink.events)}, sort_keys=True), flush=True) + print('PYTHON_STANDARD_GRAPH_FACTS ' + json.dumps({'first_seed_audits': first, + 'repeat_seed_audits': 0, 'phases': observations}, sort_keys=True)) + print('PYTHON_STANDARD_GRAPH PASSED create=6 mixed=6 rollback=0 retry=6 ordinary_sql_logs=0') + finally: + await service.close() + await asyncio.to_thread(context.runtime_telemetry().shutdown) + health = getattr(context.runtime_telemetry(), 'health', None) + if health is not None: + print('PYTHON_STANDARD_EXPORT_HEALTH ' + json.dumps(health(), sort_keys=True)) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/examples/trace-chain/standard_stream.py b/examples/trace-chain/standard_stream.py new file mode 100644 index 0000000..63675df --- /dev/null +++ b/examples/trace-chain/standard_stream.py @@ -0,0 +1,77 @@ +"""Generated Q/E scalar streams; no application SDK setup or manual spans.""" +import asyncio +import json +import os +from pathlib import Path +import sys +import tempfile +from uuid import uuid4 + +from Q import Q +from E import E +from runtime_module import GENERATED_RUNTIME_MODULE +from teaql.provider.sqlite import create_sqlite_service +from teaql.runtime import UserContext + + +class Audits: + count = 0 + async def on_safe_event(self, context, event): + self.count += 1 + + +async def main(): + database = os.environ.get('TEAQL_STANDARD_STREAM_DB') or str( + Path(tempfile.mkdtemp(prefix='teaql-python-stream-')) / 'streams.sqlite') + context = UserContext.new().install(GENERATED_RUNTIME_MODULE) + service = create_sqlite_service(database) + audits = Audits() + context.insert_resource('dataService', service).with_app_audit_event_sink(audits).disable_sql_log() + nonce = uuid4().hex.translate(str.maketrans('0123456789', 'klmnopqrst')) + try: + await context.ensure_schema() + first = audits.count + await context.ensure_schema() + assert audits.count == first and first in (0, 1) + platform = await (Q.platforms().with_id_is(1).limit(1).comment('load stream platform') + .purpose('prepare standard scalar streams').execute_for_one(context)) + ids = [] + for index in range(3): + row = Q.customer_orders().comment('create scalar stream row').purpose('prepare bounded stream input').new_entity(context) + row.update_platform(platform).update_order_number('STREAM-' + nonce + str(index)).update_description('Scalar stream row') + await row.audit_as('seed scalar stream row').save(context) + ids.append(E.customer_order(row).id().eval()) + + builder = Q.customer_orders().with_id_in(*ids).order_by_id_ascending().limit(3) + first_stream = builder.comment('complete standard stream').purpose('verify actual scalar stream diagnostics').execute_for_stream(context, chunk_size=2) + second_stream = (Q.customer_orders().with_id_in(*ids).order_by_id_ascending().limit(3) + .comment('close standard stream early').purpose('verify actual scalar stream diagnostics').execute_for_stream(context, chunk_size=2)) + untouched = (Q.customer_orders().with_id_in(*ids).limit(3).comment('unpolled standard stream') + .purpose('verify actual scalar stream diagnostics').execute_for_stream(context, chunk_size=2)) + await untouched.aclose() + builder.with_id_is(9223372036854775807).comment('LATE-STREAM-COMMENT').purpose('LATE-STREAM-PURPOSE') + first_row, second_row = await anext(first_stream), await anext(second_stream) + assert E.customer_order(first_row).id().eval() == E.customer_order(second_row).id().eval() == ids[0] + independent = await (Q.customer_orders().with_id_is(ids[0]).limit(1).comment('independent between standard streams') + .purpose('verify idle stream isolation').execute_for_list(context)) + assert len(independent) == 1 + await second_stream.aclose() + remaining = [row async for row in first_stream] + await first_stream.aclose() + rows = [first_row, *remaining] + assert [E.customer_order(row).id().eval() for row in rows] == ids + assert len({id(row._entity_root) for row in rows}) == 3 + assert audits.count == first + 3 and not context.sql_logs() + if os.environ.get('TEAQL_TELEMETRY_ENABLED', '').lower() != 'true': + assert not any(name.startswith('opentelemetry') for name in sys.modules) + await asyncio.to_thread(context.runtime_telemetry().flush) + print('PYTHON_STANDARD_STREAM_FACTS ' + json.dumps({'first_seed_audits': first, 'repeat_seed_audits': 0, + 'complete': 3, 'early_close': 1, 'independent': 1, 'unpolled': 0, 'chunk_size': 2}, sort_keys=True)) + print('PYTHON_STANDARD_STREAM PASSED complete=3 early_close=1 independent=1 ordinary_sql_logs=0') + finally: + await service.close() + await asyncio.to_thread(context.runtime_telemetry().shutdown) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/examples/trace-chain/standard_telemetry.py b/examples/trace-chain/standard_telemetry.py new file mode 100644 index 0000000..fa3f0aa --- /dev/null +++ b/examples/trace-chain/standard_telemetry.py @@ -0,0 +1,50 @@ +"""Repository-local generated seed/Q/E example; no SDK setup or manual spans.""" +import asyncio +import os +import sys +import tempfile +from pathlib import Path +from Q import Q +from E import E +from runtime_module import GENERATED_RUNTIME_MODULE +from teaql.runtime import UserContext +from teaql.provider.sqlite import create_sqlite_service + + +class AuditSink: + def __init__(self): + self.count = 0 + + async def on_safe_event(self, context, event): + self.count += 1 + + +async def main(): + directory = Path(tempfile.mkdtemp(prefix='teaql-python-standard-')) + database = directory / 'example.sqlite' + context = UserContext.new().install(GENERATED_RUNTIME_MODULE) + service = create_sqlite_service(str(database)) + audit = AuditSink() + context.insert_resource('dataService', service).with_app_audit_event_sink(audit) + context.disable_sql_log() + try: + await context.ensure_schema() + first = audit.count + await context.ensure_schema() + assert first == audit.count == 1, 'one committed seed; no repeated seed write' + for _ in range(2): + rows = await Q.platforms().with_id_is(1).limit(1).comment('load seeded platform').purpose( + 'verify standard telemetry through generated Q and E').execute_for_list(context) + assert len(rows) == 1 and E.platform(rows[0]).id().eval() == 1 and E.platform(rows[0]).name().eval() + assert not context.sql_logs(), 'ordinary SQL logging stays off' + if os.environ.get('TEAQL_TELEMETRY_ENABLED', '').lower() != 'true': + assert not any(name.startswith('opentelemetry') for name in sys.modules), 'disabled SDK import' + await asyncio.to_thread(context.runtime_telemetry().flush) + print(f'PYTHON_STANDARD_TELEMETRY_EXAMPLE PASSED first_seed_audits=1 repeat_seed_audits=0 queries=2 ordinary_sql_logs=0 database={database}') + finally: + await service.close() + await asyncio.to_thread(context.runtime_telemetry().shutdown) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/examples/trace-chain/verify-standard-graph.sh b/examples/trace-chain/verify-standard-graph.sh new file mode 100644 index 0000000..53c5fa6 --- /dev/null +++ b/examples/trace-chain/verify-standard-graph.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail +repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +evidence="$(mktemp -d -t teaql-python-standard-graph.XXXXXXXX)" +cd "$repo" +export PYTHONPATH="$repo/examples/trace-chain/lib:$repo/src" +export PYTHONDONTWRITEBYTECODE=1 +export TEAQL_STANDARD_GRAPH_DB="$evidence/graph.sqlite" +fingerprint() { find examples/trace-chain/lib -type f -not -path '*/__pycache__/*' -print0 | sort -z | xargs -0 sha256sum; } +fingerprint > "$evidence/library-before.sha256" +for round in 1 2; do + python examples/trace-chain/standard_graph.py > "$evidence/run-$round.log" 2>&1 + rg -q '^PYTHON_STANDARD_GRAPH PASSED create=6 mixed=6 rollback=0 retry=6 ordinary_sql_logs=0$' "$evidence/run-$round.log" + cat "$evidence/run-$round.log" +done +fingerprint > "$evidence/library-after.sha256" +cmp "$evidence/library-before.sha256" "$evidence/library-after.sha256" +echo "PASS: Python standard graph twice on retained SQLite; generated library unchanged; evidence=$evidence" diff --git a/examples/trace-chain/verify-standard-order.sh b/examples/trace-chain/verify-standard-order.sh new file mode 100644 index 0000000..768df1b --- /dev/null +++ b/examples/trace-chain/verify-standard-order.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -euo pipefail +repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +example="$repo/examples/trace-chain/standard-order" +evidence="$(mktemp -d -t teaql-python-standard-order.XXXXXXXX)" +echo "Python standard Order evidence: $evidence" +fingerprint() { (cd "$example/lib" && rg --files -g '*.py' | LC_ALL=C sort | xargs -d '\n' sha256sum); } +fingerprint > "$evidence/library-before.sha256" +for round in 1 2; do + PYTHONDONTWRITEBYTECODE=1 PYTHONPATH="$example/lib:$repo/src" \ + TEAQL_STANDARD_ORDER_DB="$evidence/orders.sqlite" python3 "$example/main.py" > "$evidence/run-$round.log" 2>&1 + rg -q 'PYTHON_STANDARD_ORDER PASSED queries=4 counted=2 uncounted=2 total=3 preview=3 itemCount=5 facets=3' "$evidence/run-$round.log" + if [[ "$round" == 1 ]]; then expected=4; else expected=0; fi + rg -q "first_seed_audits=$expected repeat_seed_audits=0" "$evidence/run-$round.log" + for name in TEAQL_TELEMETRY_ENABLED TEAQL_TELEMETRY_SERVICE_NAME TEAQL_TELEMETRY_OTLP_ENDPOINT; do rg -q "$name" "$evidence/run-$round.log"; done + cat "$evidence/run-$round.log" +done +fingerprint > "$evidence/library-after.sha256" +cmp "$evidence/library-before.sha256" "$evidence/library-after.sha256" +echo 'PASS: Python standard complete Order twice without database cleanup; library unchanged' diff --git a/examples/trace-chain/verify-standard-stream.sh b/examples/trace-chain/verify-standard-stream.sh new file mode 100644 index 0000000..4486a19 --- /dev/null +++ b/examples/trace-chain/verify-standard-stream.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail +repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +evidence="$(mktemp -d -t teaql-python-standard-stream.XXXXXXXX)" +cd "$repo" +export PYTHONPATH="$repo/examples/trace-chain/lib:$repo/src" +export PYTHONDONTWRITEBYTECODE=1 +export TEAQL_STANDARD_STREAM_DB="$evidence/stream.sqlite" +fingerprint() { find examples/trace-chain/lib -type f -not -path '*/__pycache__/*' -print0 | sort -z | xargs -0 sha256sum; } +fingerprint > "$evidence/library-before.sha256" +for round in 1 2; do + python examples/trace-chain/standard_stream.py > "$evidence/run-$round.log" 2>&1 + rg -q '^PYTHON_STANDARD_STREAM PASSED complete=3 early_close=1 independent=1 ordinary_sql_logs=0$' "$evidence/run-$round.log" + cat "$evidence/run-$round.log" +done +fingerprint > "$evidence/library-after.sha256" +cmp "$evidence/library-before.sha256" "$evidence/library-after.sha256" +echo "PASS: Python standard stream twice on retained SQLite; generated library unchanged; evidence=$evidence" diff --git a/examples/trace-chain/verify-standard-telemetry.sh b/examples/trace-chain/verify-standard-telemetry.sh new file mode 100644 index 0000000..3478615 --- /dev/null +++ b/examples/trace-chain/verify-standard-telemetry.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail +repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +evidence="$(mktemp -d -t teaql-python-standard-example.XXXXXXXX)" +cd "$repo" +export PYTHONPATH="$repo/examples/trace-chain/lib:$repo/src" +export PYTHONDONTWRITEBYTECODE=1 +fingerprint() { find examples/trace-chain/lib -type f -not -path '*/__pycache__/*' -print0 | sort -z | xargs -0 sha256sum; } +fingerprint > "$evidence/library-before.sha256" +for round in 1 2; do + python examples/trace-chain/standard_telemetry.py > "$evidence/run-$round.log" 2>&1 + rg -q 'PYTHON_STANDARD_TELEMETRY_EXAMPLE PASSED first_seed_audits=1 repeat_seed_audits=0 queries=2 ordinary_sql_logs=0' "$evidence/run-$round.log" + for name in TEAQL_TELEMETRY_ENABLED TEAQL_TELEMETRY_SERVICE_NAME TEAQL_TELEMETRY_OTLP_ENDPOINT; do rg -q "$name" "$evidence/run-$round.log"; done + cat "$evidence/run-$round.log" +done +fingerprint > "$evidence/library-after.sha256" +cmp "$evidence/library-before.sha256" "$evidence/library-after.sha256" +echo "PASS: Python standard telemetry example twice; generated library unchanged; evidence=$evidence" diff --git a/pyproject.toml b/pyproject.toml index 4591dd4..886e8e5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "teaql" -version = "0.3.0" +version = "0.3.1" description = "TeaQL Runtime and Data Service for Python" readme = "README.md" requires-python = ">=3.10" diff --git a/scripts/verify-examples.sh b/scripts/verify-examples.sh index 5199bd9..8f55316 100755 --- a/scripts/verify-examples.sh +++ b/scripts/verify-examples.sh @@ -41,4 +41,8 @@ TEAQL_TASK_BOARD_DB="$task_board_tmp/task_board.db" PYTHONPATH="$repo/examples/t PYTHONPATH="$repo/src" python -m unittest discover -s "$repo/examples/task_board" -p 'test_task_board.py' -v bash "$repo/examples/trace-chain/verify.sh" bash "$repo/examples/facet-trace/verify.sh" +bash "$repo/examples/trace-chain/verify-standard-telemetry.sh" +bash "$repo/examples/trace-chain/verify-standard-order.sh" +bash "$repo/examples/trace-chain/verify-standard-graph.sh" +bash "$repo/examples/trace-chain/verify-standard-stream.sh" echo "PASS: all Python examples" diff --git a/src/teaql.egg-info/PKG-INFO b/src/teaql.egg-info/PKG-INFO index 81d683a..bd087b7 100644 --- a/src/teaql.egg-info/PKG-INFO +++ b/src/teaql.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 2.4 Name: teaql -Version: 0.2.0 +Version: 0.3.0 Summary: TeaQL Runtime and Data Service for Python Author: TeaQL License-Expression: Apache-2.0 @@ -24,6 +24,7 @@ Requires-Dist: aiosqlite>=0.19.0 Requires-Dist: aiomysql>=0.2.0 Requires-Dist: asyncpg>=0.29.0 Requires-Dist: httpx>=0.24.0 +Requires-Dist: cryptography>=42.0.0 Provides-Extra: telemetry Requires-Dist: opentelemetry-api<2,>=1.44.0; extra == "telemetry" Requires-Dist: opentelemetry-sdk<2,>=1.44.0; extra == "telemetry" @@ -39,7 +40,29 @@ Dynamic: license-file # TeaQL Python SDK -TeaQL Python SDK is a runtime engine and toolkit for building data-driven business applications. It provides seamless integration with the TeaQL ecosystem, fully aligned with the `teaql-rs` baseline. +## Sensitive log data + +Runtime diagnostic logs redact payload values by default, before delivery to +file, console, buffers, or custom logging sinks. Selecting a diagnostic sink +alone does not authorize plaintext. For controlled troubleshooting only: + +```bash +export TEAQL_ALLOW_SENSITIVE_PLAINTEXT_LOGS=I_UNDERSTAND_SENSITIVE_DATA_MAY_BE_WRITTEN_TO_DISK +``` + +Only this exact value enables plaintext permission; empty values, `true`, and +whitespace variants do not. Enabling it emits a warning. Credential-classified +fields remain redacted. The flag does not force every sink to expose values. +SQL without reliable field/literal provenance may be suppressed and marked +`NOT REPLAYABLE`. Execution parameters and persisted business data are unchanged. + +Do not put sensitive data in free-text comments or purpose declarations. +TeaQL cannot govern arbitrary application prints or independent driver loggers; +configure those separately. This setting does not erase older plaintext files. +Restrict access and retention when using plaintext diagnostics, then unset the +variable and restart processes when troubleshooting is complete. + +TeaQL Python SDK is a runtime engine and toolkit for building data-driven business applications. It uses `teaql-rs` as the cross-language design baseline; verified coverage and remaining gaps are tracked in [conformance](https://github.com/teaql/teaql-conformance). ## Recommended Agent Harness @@ -54,12 +77,12 @@ and evidence-based verification as the generator and runtimes evolve. * **Python**: 3.10+ (Recommended 3.12+) * **Testing**: `pytest` 7.4+ -* **Dependencies**: `pydantic` >= 2.0, `aiosqlite`, `aiomysql`, `asyncpg` +* **Dependencies**: `pydantic` >= 2.0, `aiosqlite`, `aiomysql`, `asyncpg`, `cryptography` >= 42 ## 2. Tests Performed -After rigorous AST semantic analysis and manual verification, this SDK has successfully passed the following tests: -* ✅ **100% API Signature and Logic Parity**: Scanned with Tree-Sitter and implemented all internal methods and logic to match the Rust baseline. +The SDK includes tests in the following areas. These suites do not establish complete Rust parity or acceptance of every external provider: +* **Cross-language contracts**: Shared fixtures and integration tests verify individual portable behaviors; unresolved coverage stays explicit in conformance. * ✅ **`teaql.core` Core Tests**: Extensively tested attribute extraction, safe nullability checks, and relationship building for `Value`, `GraphNode`, `Entity`, `Mutation`, `Query`, `Expr`, and `SafeExpression`. * ✅ **`teaql.runtime` Runtime Tests**: Verified the context propagation of `UserContext` and the complete lifecycle hooking for `record_sql_log` and `record_metadata_log`. * ✅ **`teaql.sql` / `teaql.data_service` Tests**: Tested the SQL AST compilation engine and the underlying command dispatch mechanism. @@ -82,6 +105,15 @@ The SDK's organizational architecture strictly mirrors the Rust version: * **Facet Aggregation & Grouping**: Out-of-the-box support for multi-dimensional facet aggregations, group-bys, and hierarchical data processing. * **Provider Support**: Highly extensible asynchronous database connectivity (integrating third-party async drivers like `aiosqlite` through a unified Transport layer). * **Context & Logging Management**: Built-in support for lifecycle context passing, end-to-end tracing, and SQL execution log interception and dispatch. +* **Governed Mutation Policy**: An application-owned policy can review an + immutable whole-graph plan after Checker/Fix and before the first provider + mutation. Exact policy identity and approval state are retained with audit + evidence. Missing customer policy or approval emits stable warnings without + changing persistence semantics; an explicit denial fails closed. +* **Governed Business ID Lifecycle**: Core model contracts, a context-owned + profile/key/service boundary, retry-safe assignment, an in-memory allocator, + and explicit-schema durable SQLite allocation extend the portable + `daily-permuted-v1` encoder without exposing its internal sequence. * **TeaQL Federal Protocol Client**: `TeaQLFederalClient` and `TfpHttpProvider` execute governed canonical TFP v1 queries and audited mutations against a remote TeaQL endpoint such as Rust. Direct query execution returns @@ -100,5 +132,165 @@ orders = await client.execute_query(FederalQuery( await client.aclose() ``` +## Trace Chain Local Source Status + +Query and Mutation Requests own their required non-blank intent independently +of logging. Physical SQL paths use the Rust canonical algorithm, with twelve +byte-identical frozen vectors, owned query origins and qualified relation frames. +Native SQLite tests observe three real relation levels without inserting expected +frames, including a deepest-query failure. A successful write and a failed +authoritative readback keep separate canonical paths and statement outcomes; +the failed transaction rolls back. + +Parent query diagnostics include declared descendant binding provenance before +safe projection. The existing mask algorithm, expanded SQL and execution values +are unchanged; provenance is not stored on Context or exposed in log entries. +This additional compilation has not been performance benchmarked. + +Graph saves now use an explicit invocation-owned `GraphMutationSession` and an +immutable persistent parent scope. The original Context's data service is never +swapped; child saves receive the session and their parent scope explicitly. +Unannotated children inherit, local reasons append once, and deletion creates its +scope before execution. The typed ledger can store an owned complete replacement +chain. Five shared graph fixtures verify fifteen per-entity expectations; these +helper tests are separate from generated traversal evidence. + +[The generated Trace Chain example](examples/trace-chain/) drives six mutations +through actual Q/E/save APIs and SQLite, observing request lineage, SQL metadata +and safe audits after commit. It also verifies three query relation levels, +concurrent independent saves in one Context, write/readback failure rollback, +and missing root intent before transaction access. Its verifier executes twice +on one SQLite file without cleanup and checks all generated-library hashes. +All ten example groups and the source suite pass twice locally (424 tests and +eight explicit external-provider skips per source run). + +SQL transactions queue owned audit snapshots until commit, including explicit +transactions and automatic batches. Rollback discards them. A completion failure +is reported with `GraphCommittedError.committed == True`; all remaining audit +deliveries and graph cleanup callbacks are still attempted, with no postcommit +rollback. Reentrant independent root saves fail instead of silently joining. +The private generated save implementation has changed; old generated libraries +must be regenerated, while public `audit_as(...).save(context)` stays unchanged. + +The aggregate follow-up adds runtime-owned query projection snapshots and generated +`query_projection(alias)` / `has_query_projection(alias)` accessors. Missing aliases +raise `KeyError`; zero and null are present values. Projections are owned snapshots, +not writable model fields, live calculations or part of a mutation payload. The +generated example covers root/nested counts, original Trace Chain ancestry, +masked descendant intent, logging on/off, provider failure recovery and saving +only a modeled field. An alias named `save` cannot replace the save method. +Regenerate libraries to acquire this projection API; related aggregate streaming +remains unsupported and rejects before provider I/O. + +Relation attachment retains original scalar keys in invocation-local runtime +state. It does not infer membership from hydrated objects: a forward reference +filtered to `None` must not remove its child from a separately loaded parent +list, and a later sibling can still load using the same FK. Keys are not added +to returned records or mutation payloads. Native SQLite cases exercise these +boundaries with related counts, nested ancestry and logging both on and off. + +The reusable [generated Facet example](examples/facet-trace/) exercises 20 +root/nested/loaded-relation scenarios twice on one retained SQLite database. +It checks actual counts and returned metadata, original root/ancestor routes, +future-binding masking, logging on/off, and independent next-query state. The +all-examples gate now includes it and fingerprints the generated library. + +This is local-source/generated-consumer evidence, not complete Trace Chain. +Prepared-batch grouping, generated ledger overrides, complete entry-point and +inherited mutation privacy coverage, successful readback diagnostic alignment, +live PostgreSQL/MySQL graph acceptance, and immutable internal Registry replay +remain open. No released-package parity is claimed. + +```bash +PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=src python -m pytest -q +PYTHONDONTWRITEBYTECODE=1 bash scripts/verify-examples.sh +# Repeat both commands without source changes. +``` + +## Security Foundation Status + +Python currently provides governed local SQL execution and a **TFP client**; +it does not claim a public TFP server endpoint. Application queries retain +non-empty comment/purpose and audited mutations retain their audit reason. +Runtime logging should keep parameterized SQL and intent separate from any +restricted value-bearing diagnostic output. + +Python implements the portable `tqr1` tuple codec shared with Go and .NET. It +encrypts and authenticates entity type, internal ID, optimistic version, +issued/expiry time, and purpose with AES-256-GCM. Key rings permit rotation; +encoding always uses the active key while decoding can accept retained keys. + +```python +import os +from datetime import timedelta +from teaql.runtime import AeadEntityReferenceCodec, UserContext + +codec = AeadEntityReferenceCodec(2, { + 1: bytes.fromhex(os.environ["TEAQL_ENTITY_REFERENCE_KEY_V1_HEX"]), + 2: bytes.fromhex(os.environ["TEAQL_ENTITY_REFERENCE_KEY_V2_HEX"]), +}) +context = UserContext().with_entity_reference_codec(codec) +token = context.encode_entity_reference( + "OrderItem", 42, 7, "edit-order", timedelta(minutes=30) +) +claims = context.decode_entity_reference(token, "OrderItem", "edit-order") +``` + +Without a configured codec the runtime fails closed. Local debugging can use +the canonical long `TEAQL_UNSAFE_RAW_ENTITY_REFERENCES` acknowledgement, which +emits visibly distinct `tqr0` references. It must not be enabled in production. + +The wire format, fail-closed behavior, shared golden vector, and exact +development-only acknowledgement are maintained in the canonical +[opaque entity reference contract](https://github.com/teaql/teaql-conformance/blob/main/design/opaque-entity-references.md). +Opaque tokens never replace the backend's authorization, tenant, ownership, +role, or optimistic-version checks. + +The repeatable [`examples/opaque-entity-reference`](examples/opaque-entity-reference) +example proves the exact cross-language golden vector and purpose-substitution +rejection. + +## Business ID V1 Foundation + +The pure runtime encoder maps a durable internal sequence into the canonical +six-character, scope-specific Base36 code: + +```python +from teaql.core import BusinessIdEncodingKey, BusinessIdScope +from teaql.runtime import encode_business_id_permutation_v1 + +scope = BusinessIdScope( + "tenant-a", "commerce_order", "order_number", "20260925" +) +key = BusinessIdEncodingKey(1, key_from_secret_manager) +code = encode_business_id_permutation_v1(0, scope, key) +``` + +The retained [`examples/business-id`](examples/business-id) flow proves durable +concurrent allocation infrastructure and aggregate retry reuse. Generated +strongly typed fields and external lookup remain a separate generator +capability. Secret key material is application-owned and must not be placed in +KSML or generated source. + +### Mutation Policy installation + +Policy implementations are installed from trusted application startup through +`UserContext`; request JSON cannot select or replace them. Built-in SQL and TFP +providers enter the same governed boundary. + +```python +context = ( + UserContext.new() + .with_mutation_policy_registry(policy_registry) + .with_mutation_policy_approval_provider(approval_provider) + .with_mutation_governance_sink(warning_sink) +) +``` + +Generated graph saves call `preflight_mutation(...)` for every operation before +the first provider write. See the repeatable +[`examples/mutation-policy`](examples/mutation-policy) example for allow, +approval, audit propagation, and zero-write denial evidence. + --- To run test validations and business logic simulations locally, simply run `pytest` in the project root. diff --git a/src/teaql.egg-info/SOURCES.txt b/src/teaql.egg-info/SOURCES.txt index d5bf3b7..564ab0a 100644 --- a/src/teaql.egg-info/SOURCES.txt +++ b/src/teaql.egg-info/SOURCES.txt @@ -9,6 +9,8 @@ src/teaql.egg-info/dependency_links.txt src/teaql.egg-info/requires.txt src/teaql.egg-info/top_level.txt src/teaql/core/__init__.py +src/teaql/core/business_id.py +src/teaql/core/dynamic_search.py src/teaql/core/entity.py src/teaql/core/eval.py src/teaql/core/expr.py @@ -17,7 +19,10 @@ src/teaql/core/list.py src/teaql/core/meta.py src/teaql/core/mutation.py src/teaql/core/query.py +src/teaql/core/request_intent.py src/teaql/core/safe_expression.py +src/teaql/core/trace.py +src/teaql/core/trace_scope.py src/teaql/core/value.py src/teaql/core/xls.py src/teaql/data_service/__init__.py @@ -39,15 +44,23 @@ src/teaql/runtime/__init__.py src/teaql/runtime/_schema_capability.py src/teaql/runtime/audit.py src/teaql/runtime/builtin-messages-v1.json +src/teaql/runtime/business_clock.py +src/teaql/runtime/business_id.py src/teaql/runtime/context.py +src/teaql/runtime/entity_reference.py src/teaql/runtime/env.py +src/teaql/runtime/graph_session.py src/teaql/runtime/i18n.py +src/teaql/runtime/log_privacy.py src/teaql/runtime/module.py +src/teaql/runtime/mutation_policy.py src/teaql/runtime/opentelemetry.py src/teaql/runtime/store.py src/teaql/runtime/telemetry.py src/teaql/runtime/tools.py +src/teaql/runtime/wire_fields.py src/teaql/sql/__init__.py +src/teaql/sql/business_id.py src/teaql/sql/dialect.py src/teaql/sql/executor.py src/teaql/sql/types.py diff --git a/src/teaql.egg-info/requires.txt b/src/teaql.egg-info/requires.txt index 99183e3..88016b4 100644 --- a/src/teaql.egg-info/requires.txt +++ b/src/teaql.egg-info/requires.txt @@ -3,6 +3,7 @@ aiosqlite>=0.19.0 aiomysql>=0.2.0 asyncpg>=0.29.0 httpx>=0.24.0 +cryptography>=42.0.0 [telemetry] opentelemetry-api<2,>=1.44.0 diff --git a/src/teaql/core/__init__.py b/src/teaql/core/__init__.py index 44ca5b3..688c9db 100644 --- a/src/teaql/core/__init__.py +++ b/src/teaql/core/__init__.py @@ -19,6 +19,7 @@ ) from .graph import GraphNode from .request_intent import QueryIntent, MutationIntent, RequestIntentError +from .request_tree import QueryOperation from .meta import EntityDescriptor, PropertyDescriptor from .list import SmartList, TeaQLPage @@ -44,7 +45,7 @@ ) __all__ = [ - "QueryIntent", "MutationIntent", "RequestIntentError", + "QueryIntent", "MutationIntent", "RequestIntentError", "QueryOperation", "Value", "DataType", "Timestamp", "BaseEntityData", "EntityKey", "EntityChangeSet", "EntityRoot", "Expr", "ExprBuilder", "BinaryOp", "ExprFunction", diff --git a/src/teaql/core/dynamic_search.py b/src/teaql/core/dynamic_search.py index 0d80ed8..712a720 100644 --- a/src/teaql/core/dynamic_search.py +++ b/src/teaql/core/dynamic_search.py @@ -5,7 +5,7 @@ import re from copy import deepcopy from datetime import date -from .expr import Expr +from .expr import Expr, column, eq, ne, gt, gte, lt, lte, contain, in_list, not_in_list, is_null, is_not_null from .query import OrderBy _LOG = logging.getLogger(__name__) @@ -96,6 +96,8 @@ def scalar(item, kind): if kind is None: missing(path, 'FILTER') continue + if kind == 'unsupported': + raise ValueError('Known search field has no supported scalar binding') if operator == '$contains' and kind != 'string': raise ValueError('String operator requires a string field') if isinstance(item, list): @@ -110,9 +112,12 @@ def scalar(item, kind): if (not isinstance(order, dict) or set(order) != {'field', 'direction'} or order['direction'] not in ('asc', 'desc')): raise ValueError('Invalid dynamic search ordering') - if field_type(order['field']) is None: + kind = field_type(order['field']) + if kind is None: missing(order['field'], 'ORDER_BY') else: + if kind == 'unsupported': + raise ValueError('Known search field has no supported scalar binding') result['orderBy'].append(dict(order)) for warning in warnings: warn(dict(warning)) @@ -133,3 +138,32 @@ def merge_dynamic_search(base, source, models, filter_binding, order_binding, wa for warning in warnings: warn(dict(warning)) return query, warnings + + +def merge_generated_scalar_search(base, source, models, warn=_warning): + """Generated scalar keys/types are trusted metadata, never source JSON. + + Known relation paths require application bindings through merge_dynamic_search; + those bindings own related authorization and are never guessed here. + """ + def field(path): + if '.' in path: + raise ValueError('Dynamic search requires a trusted relation binding') + fields = models[base.entity].get('query_fields', {}) + if path not in fields: + raise ValueError('Missing generated scalar search binding') + return fields[path] + + def predicate(path, condition): + name = field(path) + operator, item = next(iter(condition.items())) + if item is None and operator in ('$eq', '$ne'): + return is_null(column(name)) if operator == '$eq' else is_not_null(column(name)) + if item is None: + raise ValueError('Null supports only equality predicates') + builders = {'$eq': eq, '$ne': ne, '$gt': gt, '$gte': gte, '$lt': lt, '$lte': lte, + '$in': in_list, '$notIn': not_in_list, '$contains': contain} + return builders[operator](name, item) + + return merge_dynamic_search(base, source, models, predicate, + lambda path, direction: OrderBy.asc(field(path)) if direction == 'asc' else OrderBy.desc(field(path)), warn) diff --git a/src/teaql/core/list.py b/src/teaql/core/list.py index 84917f5..0959a95 100644 --- a/src/teaql/core/list.py +++ b/src/teaql/core/list.py @@ -12,10 +12,11 @@ class TeaQLPage(Generic[T]): class SmartList(List[T], Generic[T]): def __init__(self, data: Iterable[T] = (), facets: Optional[Dict[str, Any]] = None, - total_count: Optional[int] = None): + total_count: Optional[int] = None, aggregations: Optional[Dict[str, Any]] = None): super().__init__(data) self.facets = facets or {} self.total_count = total_count if total_count is not None else len(self) + self.aggregations = dict(aggregations or {}) @property def data(self) -> 'SmartList[T]': @@ -25,16 +26,16 @@ def facet(self, name: str) -> Any: return self.facets.get(name) def map(self, f: Callable[[T], Any]) -> 'SmartList[Any]': - return SmartList((f(x) for x in self), self.facets, self.total_count) + return SmartList((f(x) for x in self), self.facets, self.total_count, self.aggregations) def filter(self, f) -> 'SmartList': - return SmartList((x for x in self if f(x)), self.facets, self.total_count) + return SmartList((x for x in self if f(x)), self.facets, self.total_count, self.aggregations) def flat_map(self, f) -> 'SmartList': result = [] for x in self: result.extend(f(x)) - return SmartList(result, self.facets, self.total_count) + return SmartList(result, self.facets, self.total_count, self.aggregations) def first(self) -> Any: return self[0] if self else None diff --git a/src/teaql/core/projection_properties.py b/src/teaql/core/projection_properties.py new file mode 100644 index 0000000..f974240 --- /dev/null +++ b/src/teaql/core/projection_properties.py @@ -0,0 +1,23 @@ +"""Read-only computed query properties, never a mutation payload or loaded field.""" +from copy import deepcopy +from types import MappingProxyType + + +class ProjectionProperties: + __slots__ = ('__values',) + + def __init__(self, record, model_fields): + known = frozenset(model_fields) + self.__values = MappingProxyType({key: deepcopy(value) for key, value in record.items() + if key not in known}) + + def get(self, name): + # An absent dynamic property is null, not a NotLoaded exception. + return deepcopy(self.__values.get(name)) + + def as_dict(self): + return deepcopy(dict(self.__values)) + + def __deepcopy__(self, memo): + # No mutable values escape this read-only snapshot. + return self diff --git a/src/teaql/core/query.py b/src/teaql/core/query.py index 66d7f8c..5d5a74e 100644 --- a/src/teaql/core/query.py +++ b/src/teaql/core/query.py @@ -5,6 +5,7 @@ from .expr import (AndExpr, BinaryExpr, BinaryOp, ColumnExpr, Expr, ExprBuilder, ValueExpr) from .mutation import TraceNode +from .request_tree import QueryOperation, _copy_owner class SortDirection(Enum): Asc = auto() @@ -82,6 +83,7 @@ class Slice: class RelationLoad: name: str query: Optional['SelectQuery'] = None + operation: Optional[QueryOperation] = None @dataclass class RelationAggregate: @@ -89,6 +91,7 @@ class RelationAggregate: alias: str query: 'SelectQuery' single_result: bool = True + operation: Optional[QueryOperation] = None @dataclass @@ -134,6 +137,7 @@ class FacetRequest: relation_name: str query: 'SelectQuery' include_all_facets: bool = True + operation: Optional[QueryOperation] = None @dataclass class AggregationCacheOptions: @@ -185,6 +189,13 @@ def __post_init__(self) -> None: # Local runtime policy: intentionally excluded from federation serialization. self.hard_limit_value = 10_000 + def __deepcopy__(self, memo): + copied = object.__new__(type(self)) + memo[id(self)] = copied + copied.__dict__.update(deepcopy(self.__dict__, memo)) + _copy_owner(self, copied) + return copied + @classmethod def new(cls, entity: str) -> 'SelectQuery': return cls(entity=entity) @@ -231,6 +242,8 @@ def for_exact_count(self, alias: str = "__teaql_total") -> 'SelectQuery': count_query.order_by_items = [] count_query.slice = None count_query.relations = [] + count_query.relation_aggregates = [] + count_query.partition_by = None count_query.child_enhancements = [] count_query.object_group_bys = [] count_query.facets = [] @@ -497,16 +510,16 @@ def object_group_by(self, property_name: str, storage_field: str, query: 'Select return self def facet_by(self, name: str, relation_name: str, query: 'SelectQuery', - include_all_facets: bool = True) -> 'SelectQuery': - self.facets.append(FacetRequest(name, relation_name, query, include_all_facets)) + include_all_facets: bool = True, operation: Optional[QueryOperation] = None) -> 'SelectQuery': + self.facets.append(FacetRequest(name, relation_name, query, include_all_facets, operation)) return self def relation(self, name: str) -> 'SelectQuery': self.relations.append(RelationLoad(name, None)) return self - def relation_query(self, name: str, query: 'SelectQuery') -> 'SelectQuery': - self.relations.append(RelationLoad(name, query)) + def relation_query(self, name: str, query: 'SelectQuery', operation: Optional[QueryOperation] = None) -> 'SelectQuery': + self.relations.append(RelationLoad(name, query, operation)) return self def enable_aggregation_cache(self, expire_millis: int) -> 'SelectQuery': diff --git a/src/teaql/core/request_tree.py b/src/teaql/core/request_tree.py new file mode 100644 index 0000000..21e55e5 --- /dev/null +++ b/src/teaql/core/request_tree.py @@ -0,0 +1,78 @@ +"""Immutable request ownership; never part of query JSON or UserContext state.""" +from dataclasses import dataclass, field +import re +import weakref +from typing import Any, Optional + + +@dataclass(frozen=True, slots=True) +class QueryOperation: + kind: str + method: Optional[str] = None + alias: Optional[str] = field(default=None, repr=False) + relation: Optional[str] = None + + def __post_init__(self): + if (self.kind not in ('named-count', 'facet', 'relation-count', 'relation-select', + 'aggregate', 'pagination') + or self.method is not None and ( + not isinstance(self.method, str) + or re.fullmatch(r'[A-Za-z_][A-Za-z0-9_]*', self.method) is None) + or self.alias is not None and not isinstance(self.alias, str) + or self.relation is not None and not isinstance(self.relation, str)): + raise ValueError('QUERY_OPERATION_INVALID: invalid generated operation descriptor') + + def comment(self) -> str: + def literal(text): + return "'" + (text.replace('\\', '\\\\').replace("'", "\\'") + .replace('\n', '\\n').replace('\r', '\\r').replace('\t', '\\t') + .replace('\u2028', '\\u2028').replace('\u2029', '\\u2029')) + "'" + if self.method is None: + return ('framework: ' + self.kind + (' ' + self.relation if self.relation else '') + + (' ' + literal(self.alias) if self.alias is not None else '')) + argument = literal(self.alias) if self.alias is not None else ( + '...' if self.kind == 'relation-select' else '') + return self.method + '(' + argument + ')' + + +@dataclass(frozen=True, slots=True) +class _RequestOwner: + parent: Any = field(repr=False) + name: str + comment: str = field(repr=False) + intent: Any = field(repr=False) + operation: Optional[QueryOperation] = None + source: tuple = field(default=(), repr=False) + invocation: object = field(default=None, repr=False, compare=False) + + def __post_init__(self): + if self.parent is not None: + object.__setattr__(self, 'invocation', self.parent._invocation_identity) + elif self.invocation is None: + object.__setattr__(self, 'invocation', object()) + + +# SelectQuery is mutable and unhashable, so WeakKeyDictionary is unsuitable. +# Identity is checked on both lookup and finalization to avoid id reuse. Values +# hold immutable ownership, not the owning query/request itself. +_owners = {} + + +def _owner_for(query): + item = _owners.get(id(query)) + return item[1] if item is not None and item[0]() is query else None + + +def _retain_owner(query, owner): + key = id(query) + def remove(reference): + current = _owners.get(key) + if current is not None and current[0] is reference: + _owners.pop(key, None) + _owners[key] = (weakref.ref(query, remove), owner) + + +def _copy_owner(source, target): + owner = _owner_for(source) + if owner is not None: + _retain_owner(target, owner) diff --git a/src/teaql/data_service/__init__.py b/src/teaql/data_service/__init__.py index cc0d3ef..f2edc1b 100644 --- a/src/teaql/data_service/__init__.py +++ b/src/teaql/data_service/__init__.py @@ -1,11 +1,12 @@ -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from datetime import datetime from enum import Enum, auto from typing import List, Dict, Any, Optional, Protocol, Union, AsyncIterator from copy import deepcopy from teaql.core.query import SelectQuery -from teaql.core.request_intent import QueryIntent +from teaql.core.request_intent import QueryIntent, require_intent +from teaql.core.request_tree import QueryOperation, _RequestOwner, _owner_for, _retain_owner from teaql.core.mutation import ( MutationRequest, InsertCommand as CoreInsertCommand, @@ -27,21 +28,74 @@ class DataServiceCapabilities: returning: bool = False +_BIND_REQUEST = object() + + class QueryRequest: __slots__ = ('query', 'trace_chain', '__intent', '__origin_entity', - '_log_intent_source', '_log_intent_queries') + '_log_intent_source', '_log_intent_queries', '_relation_assembly', '__owner') _UNSET = object() def __init__(self, query: SelectQuery, trace_chain=None, _comment=_UNSET, _purpose=_UNSET, - *, _origin_entity=None): + *, _origin_entity=None, _binding=None): comment = getattr(query, 'comment_text', None) if _comment is self._UNSET else _comment purpose = getattr(query, 'purpose_text', None) if _purpose is self._UNSET else _purpose self.__intent = QueryIntent(comment, purpose) - self.__origin_entity = query.entity if _origin_entity is None else _origin_entity + previous = _owner_for(query) + if _binding is not None: + if (not isinstance(_binding, tuple) or len(_binding) != 2 + or _binding[0] is not _BIND_REQUEST or not isinstance(_binding[1], _RequestOwner)): + raise ValueError('QUERY_REQUEST_OWNERSHIP_INVALID: request binding is runtime-owned') + self.__owner = _binding[1] + elif previous is not None and previous.intent == self.__intent: + self.__owner = previous + else: + self.__owner = _RequestOwner(None, query.entity, self.__intent.comment, self.__intent) + self.__origin_entity = (_origin_entity if _origin_entity is not None else + self.parent_request.origin_entity if self.parent_request is not None else self.request_name) self.query = deepcopy(query) self.query.comment_text = self.intent.comment self.query.purpose_text = self.intent.purpose - self.trace_chain = deepcopy(trace_chain) if trace_chain is not None else [] + self.trace_chain = deepcopy(trace_chain) if trace_chain is not None else list(deepcopy(self.__owner.source)) + self.__owner = replace(self.__owner, source=tuple(deepcopy(self.trace_chain))) + _retain_owner(self.query, self.__owner) + + @property + def parent_request(self): + return self.__owner.parent + + @property + def _invocation_identity(self): + return self.__owner.invocation + + @property + def root_request(self): + root = self + while root.parent_request is not None: + root = root.parent_request + return root + + @property + def request_name(self): + return self.__owner.name + + @property + def local_comment(self): + return self.__owner.comment + + @property + def operation(self): + return self.__owner.operation + + @property + def business_trace_chain(self): + comments, current = [], self + while current is not None: + comments.append(TraceNode(kind='comment', name=current.request_name, + comment=current.local_comment)) + current = current.parent_request + return (TraceNode(kind='purpose', name=self.root_request.request_name, + comment=self._purpose), *reversed(comments)) @property def intent(self) -> QueryIntent: @@ -62,12 +116,16 @@ def _purpose(self) -> str: def validate(self) -> None: intent = getattr(self, '_QueryRequest__intent', None) QueryIntent(getattr(intent, 'comment', None), getattr(intent, 'purpose', None)) + require_intent(self.local_comment, 'comment', 'query') if any(not isinstance(node, TraceNode) for node in self.trace_chain): raise TypeError('QueryRequest trace must contain typed TraceNode values') def with_query(self, query: SelectQuery) -> 'QueryRequest': + previous = _owner_for(query) + if previous is not None and previous.invocation is not self.__owner.invocation: + raise ValueError('QUERY_SCOPE_OWNER_MISMATCH: replacement belongs to another query invocation') result = QueryRequest(query, self.trace_chain, self._comment, self._purpose, - _origin_entity=self.origin_entity) + _origin_entity=self.origin_entity, _binding=(_BIND_REQUEST, self.__owner)) if hasattr(self, '_log_intent_source'): result._log_intent_source = deepcopy(self._log_intent_source) # Derived work may drop a relation/projection whose private values are @@ -75,17 +133,89 @@ def with_query(self, query: SelectQuery) -> 'QueryRequest': # solely for compiler classification, never on Context or log payloads. result._log_intent_queries = (*deepcopy(getattr(self, '_log_intent_queries', ())), deepcopy(self.query)) + if hasattr(self, '_relation_assembly'): + result._relation_assembly = self._relation_assembly return result + def _child(self, query, comment, operation=None, relation=None): + owner = _RequestOwner(self, query.entity, require_intent(comment, 'comment', 'query'), + self.intent, operation) + traces = deepcopy(self.trace_chain) + if relation is not None: + traces.append(TraceNode(kind='relation', name=relation, + comment=f'{self.query.entity}.{relation}')) + result = QueryRequest(query, traces, self._comment, self._purpose, + _origin_entity=self.origin_entity, _binding=(_BIND_REQUEST, owner)) + if hasattr(self, '_log_intent_source'): + result._log_intent_source = self._log_intent_source + result._log_intent_queries = (*getattr(self, '_log_intent_queries', ()), self.query) + return result + + def derive_operation(self, query, operation: QueryOperation, relation=None): + if not isinstance(operation, QueryOperation): + raise ValueError('QUERY_OPERATION_INVALID: a typed generated descriptor is required') + return self._child(query, operation.comment(), operation, relation) + + def derive_helper(self, query, comment): + return self._child(query, comment) + + def for_count(self, alias=None, operation=None): + """Derive exact-count execution without losing the caller's named result.""" + if alias is not None and not isinstance(alias, str): + raise ValueError('QUERY_COUNT_ALIAS_INVALID: alias must be a string') + if operation is not None and operation.alias is not None and operation.alias != alias: + raise ValueError('QUERY_OPERATION_ALIAS_MISMATCH: count descriptor and result alias differ') + return self.derive_operation(self.query.for_exact_count('__teaql_total' if alias is None else alias), + operation or QueryOperation('named-count', alias=alias)) + + def derive_child(self, query, relation=None): + comment = getattr(query, 'comment_text', None) + return self._child(query, 'load ' + query.entity if comment is None else comment, relation=relation) + + def derive(self, query, relation, operation=None): + parent = self.derive_operation(self.query, operation or QueryOperation('relation-select', relation=relation)) + return parent.derive_child(query, relation) + + def derive_aggregate(self, query, relation, operation): + """Anonymous aggregate work executes its operation, not a value load.""" + facet = self.parent_request + if facet is not None and facet.operation is not None and facet.operation.kind == 'facet': + branch = facet.derive_helper(query, 'aggregate ' + facet.operation.relation) + else: + branch = self.derive_operation(query, operation) + branch.trace_chain = [*deepcopy(self.trace_chain), TraceNode( + kind='relation', name=relation, comment=f'{self.query.entity}.{relation}')] + branch.__owner = replace(branch.__owner, source=tuple(deepcopy(branch.trace_chain))) + _retain_owner(branch.query, branch.__owner) + branch._log_intent_queries = (*getattr(self, '_log_intent_queries', ()), self.query) + if hasattr(self, '_log_intent_source'): + branch._log_intent_source = self._log_intent_source + return branch if query.comment_text is None else branch.derive_child(query) + + def with_log_intent(self, source, assembly=None): + self._log_intent_source = source + self._relation_assembly = assembly + return self + def comment(self, text: str) -> 'QueryRequest': result = self.with_query(self.query) - result.__intent = QueryIntent(text, self._purpose) + require_intent(text, 'comment', 'query') + if result.parent_request is None: + result.__intent = QueryIntent(text, self._purpose) + result.__owner = _RequestOwner(result.parent_request, result.request_name, text, + result.intent, result.operation) + _retain_owner(result.query, result.__owner) result.query.comment_text = result.intent.comment return result def purpose(self, text: str) -> 'QueryRequest': + if self.parent_request is not None and text != self._purpose: + raise ValueError('QUERY_ROOT_PURPOSE_IMMUTABLE: derived requests retain the root purpose') result = self.with_query(self.query) result.__intent = QueryIntent(self._comment, text) + result.__owner = _RequestOwner(result.parent_request, result.request_name, result.local_comment, + result.intent, result.operation) + _retain_owner(result.query, result.__owner) result.query.purpose_text = result.intent.purpose return result @@ -111,6 +241,9 @@ class ExecutionMetadata: affected_rows: Optional[int] = None result_count: Optional[int] = None trace_chain: List[TraceNode] = field(default_factory=list) + # Legacy trace_chain is the physical route. This separate business chain is + # projected as SqlLogEntry.trace_chain; SqlLogEntry.trace_path stays physical. + request_trace_chain: tuple[TraceNode, ...] = field(default_factory=tuple) mutation_lineage: tuple[TraceNode, ...] = field(default_factory=tuple) comment: Optional[str] = None purpose: Optional[str] = None diff --git a/src/teaql/runtime/context.py b/src/teaql/runtime/context.py index 8e5d421..764b1e1 100644 --- a/src/teaql/runtime/context.py +++ b/src/teaql/runtime/context.py @@ -104,8 +104,8 @@ def __init__(self): self._app_audit_sink: Optional[Any] = None self._entity_initializers: Dict[str, List[EntityInitializer]] = {} self._managed_entities: List[Any] = [] - from .telemetry import NOOP_RUNTIME_TELEMETRY - self._runtime_telemetry = NOOP_RUNTIME_TELEMETRY + from .telemetry_startup import default_telemetry + self._runtime_telemetry = default_telemetry() self._id_set_store = _default_id_set_store self._id_set_plan = "ID_SET_DISABLED" self._id_set_count = 0 @@ -218,36 +218,45 @@ async def execute_graph_save(self, work, *, comment=None): if self._graph_session is not None or (guard is not None and guard['active']): raise RuntimeError('independent graph save cannot re-enter an active graph; compose children explicitly') async with self._graph_save_lock: - provider = self.require_resource("dataService") - begin = getattr(provider, "begin", None) - if not callable(begin): - raise RuntimeError("Configured dataService does not support graph transactions") - transaction = await self._call_transaction_method(begin, self) - guard = {'active': True} - guard_token = self._graph_save_guard.set(guard) - session = None + from .telemetry import RuntimeOperation, observe_runtime_operation + # One owned operation covers physical writes, commit, audit and + # cleanup. It starts only after admission to the existing save gate; + # generated entities do not initialize SDKs or create manual spans. + return await observe_runtime_operation( + self._runtime_telemetry, RuntimeOperation('mutation', 'graph.save'), + lambda: self._execute_graph_transaction(work, intent, GraphMutationSession)) + + async def _execute_graph_transaction(self, work, intent, session_type): + provider = self.require_resource("dataService") + begin = getattr(provider, "begin", None) + if not callable(begin): + raise RuntimeError("Configured dataService does not support graph transactions") + transaction = await self._call_transaction_method(begin, self) + guard = {'active': True} + guard_token = self._graph_save_guard.set(guard) + session = None + try: + session = session_type(self, transaction, intent) + result = await work(session) + session.context._mutation_policy.ensure_graph_complete() + await self._finish_graph_transaction(transaction, "commit", session.context) + except BaseException: try: - session = GraphMutationSession(self, transaction, intent) - result = await work(session) - session.context._mutation_policy.ensure_graph_complete() - await self._finish_graph_transaction(transaction, "commit", session.context) + await self._finish_graph_transaction(transaction, "rollback", session.context if session else self) except BaseException: - try: - await self._finish_graph_transaction(transaction, "rollback", session.context if session else self) - except BaseException: - pass # Preserve the original operation/commit failure. - if session is not None: - await session.rolled_back() - raise - else: - await session.committed() - return result - finally: - if session is not None: - session.close() - self._fix_evidence_last = list(session.context.last_fix_evidence()) - guard['active'] = False - self._graph_save_guard.reset(guard_token) + pass # Preserve the original operation/commit failure. + if session is not None: + await session.rolled_back() + raise + else: + await session.committed() + return result + finally: + if session is not None: + session.close() + self._fix_evidence_last = list(session.context.last_fix_evidence()) + guard['active'] = False + self._graph_save_guard.reset(guard_token) @staticmethod async def _call_transaction_method(method, context): @@ -910,7 +919,10 @@ def _record_metadata_log(self, metadata: Any, *, intent_source=None, intent_valu elif 'update' in op_str: op = SqlLogOperation.Update elif 'delete' in op_str: op = SqlLogOperation.Delete elif 'recover' in op_str: op = SqlLogOperation.Recover - if not self.sql_log_options().enabled_for(op): + logging_enabled = self.sql_log_options().enabled_for(op) + writer = getattr(self._runtime_telemetry, 'record_diagnostic', None) + correlation = getattr(metadata, '_sql_correlation', None) + if not logging_enabled and not (writer and correlation): return started_at = getattr(metadata, 'started_at', datetime.now()) @@ -921,6 +933,7 @@ def _record_metadata_log(self, metadata: Any, *, intent_source=None, intent_valu purpose=getattr(metadata, 'purpose', None), audit_reason=getattr(metadata, 'audit_reason', None), trace_path=list(getattr(metadata, 'trace_chain', [])), + trace_chain=tuple(getattr(metadata, 'request_trace_chain', ())), mutation_lineage=tuple(getattr(metadata, 'mutation_lineage', ())), sql=getattr(metadata, 'parameterized_sql', ''), params=list(getattr(metadata, 'parameters', [])), @@ -944,6 +957,30 @@ def _record_metadata_log(self, metadata: Any, *, intent_source=None, intent_valu entry.result_summary = f"{entry.affected_rows} rows affected" from .log_privacy import sql_log_projection + if writer and correlation: + try: + import json + safe = sql_log_projection(entry, _intent_source=intent_source, _intent_values=intent_values, + _intent_privacy=intent_privacy, _allow_plaintext=False) + def nodes(source): + return json.dumps([{'kind': getattr(node, 'kind', ''), 'name': getattr(node, 'name', ''), + 'detail': getattr(node, 'comment', '')} for node in source]) + attributes = { + 'teaql.sql.debug': safe.debug_sql, 'teaql.sql.log_mode': 'masked' if not getattr(safe, 'omission_reason', None) else 'omitted', + 'teaql.sql.operation': 'select' if op == SqlLogOperation.Select else op.name.lower(), + 'teaql.sql.outcome': safe.execution_outcome or 'unknown', 'teaql.sql.elapsed_us': int(safe.elapsed.total_seconds() * 1000000), + 'teaql.intent.comment': safe.comment or safe.audit_reason or '', 'teaql.intent.purpose': safe.purpose or '', + 'teaql.trace_chain': nodes(safe.trace_chain), 'teaql.trace_path': nodes(safe.trace_path), 'teaql.mutation_lineage': nodes(safe.mutation_lineage) + } + if safe.result_count is not None: + attributes['teaql.sql.result_count'] = safe.result_count + if safe.affected_rows is not None: + attributes['teaql.sql.affected_rows'] = safe.affected_rows + writer(attributes, correlation) + except Exception: + pass + if not logging_enabled: + return entry = sql_log_projection(entry, _intent_source=intent_source, _intent_values=intent_values, _intent_privacy=intent_privacy) logs = self.sql_logs() @@ -1259,6 +1296,7 @@ class SqlLogEntry: omission_reason: Optional[str] = None execution_outcome: Optional[str] = None mutation_lineage: tuple[Any, ...] = () + trace_chain: tuple[Any, ...] = () class DiagnosticSqlLogSink: """Policy-projected SQL destination; the text sink is installed by default.""" @@ -1276,7 +1314,7 @@ def write(self, entry: SqlLogEntry) -> None: self._writer( f"[TeaQL SQL][{entry.operation.name.lower()}][{elapsed_us}us] " f"{entry.result_summary} outcome={entry.execution_outcome or 'unknown'} comment={entry.comment!r} purpose={entry.purpose!r} " - f"auditReason={entry.audit_reason!r} tracePath={entry.trace_path!r}\n" + f"auditReason={entry.audit_reason!r} tracePath={entry.trace_path!r} traceChain={entry.trace_chain!r}\n" f"SQL omission reason: {entry.omission_reason or 'none'}\n" f"Debug SQL: {entry.debug_sql}" ) diff --git a/src/teaql/runtime/graph_session.py b/src/teaql/runtime/graph_session.py index 15b36ce..af050bd 100644 --- a/src/teaql/runtime/graph_session.py +++ b/src/teaql/runtime/graph_session.py @@ -65,6 +65,14 @@ def request(self, command, scope, ledger=None, key=None): return MutationRequest(command, comment=self.intent.comment).with_mutation_lineage( specific or scope.recover()) + def capture_loaded_values(self, entity_type, values): + """Unchanged/removed graph values still protect invocation annotations.""" + from types import SimpleNamespace + from .log_privacy import _MutationIntentPrivacy + self._require_active() + self._intent_privacy = self._intent_privacy.merge(_MutationIntentPrivacy.capture( + SimpleNamespace(entity=entity_type, values=values), self.context.entity)) + def after_commit(self, action): self._require_active() self._commit_actions.append(action) diff --git a/src/teaql/runtime/loaded_scalar_snapshot.py b/src/teaql/runtime/loaded_scalar_snapshot.py new file mode 100644 index 0000000..9497d7c --- /dev/null +++ b/src/teaql/runtime/loaded_scalar_snapshot.py @@ -0,0 +1,19 @@ +"""Original loaded scalars owned by runtime, not serialized entity state.""" +from copy import deepcopy +from weakref import WeakKeyDictionary + + +class LoadedScalarSnapshots: + _values = WeakKeyDictionary() + + @classmethod + def capture(cls, entity, values): + cls._values[entity] = deepcopy(values) + + @classmethod + def values(cls, entity): + return deepcopy(cls._values.get(entity, {})) + + @classmethod + def restore(cls, entity, values): + cls.capture(entity, values) diff --git a/src/teaql/runtime/log_privacy.py b/src/teaql/runtime/log_privacy.py index 56026b6..12b0e89 100644 --- a/src/teaql/runtime/log_privacy.py +++ b/src/teaql/runtime/log_privacy.py @@ -162,9 +162,9 @@ def capture(cls, data, resolve_entity): return cls(tuple(bindings)) -def sql_log_projection(entry, *, _intent_source=None, _intent_values=(), _intent_privacy=None): +def sql_log_projection(entry, *, _intent_source=None, _intent_values=(), _intent_privacy=None, _allow_plaintext=None): """Source bindings are call-local runtime plumbing, never stored on a log entry.""" - allow = plaintext_enabled() and entry.log_mode != 'masked' + allow = (plaintext_enabled() if _allow_plaintext is None else _allow_plaintext) and entry.log_mode != 'masked' prior = _projections.get(id(entry)) # Log entries are mutable: reuse only an unchanged projection. Weak references # keep this idempotence bookkeeping from retaining a long-running log history. @@ -239,5 +239,6 @@ def literal(index): else f'{entry.affected_rows} rows affected' if entry.affected_rows is not None else scrub(entry.result_summary, secrets, hide_all=unknown_debug_intent)), trace_path=intent(entry.trace_path), + trace_chain=intent(entry.trace_chain), mutation_lineage=intent(entry.mutation_lineage)) return projected diff --git a/src/teaql/runtime/opentelemetry.py b/src/teaql/runtime/opentelemetry.py index 9cfc8ce..abf637b 100644 --- a/src/teaql/runtime/opentelemetry.py +++ b/src/teaql/runtime/opentelemetry.py @@ -73,6 +73,26 @@ def __init__(self, span, activation, family, name, started_at, duration, operati self._operations = operations self._ended = False + def activate(self): + if self._activation is None and not self._ended: + self._activation = use_span(self._span, end_on_exit=False, record_exception=False, set_status_on_exception=False) + self._activation.__enter__() + + def deactivate(self): + if self._activation is not None: + activation, self._activation = self._activation, None + activation.__exit__(None, None, None) + + def cancelled(self, attributes=None): + if self._ended: + return + for key, value in (attributes or {}).items(): + if key == 'teaql.result.cardinality': + self._span.set_attribute(key, value) + self._span.set_attribute('teaql.error.category', 'cancelled') + self._span.set_status(Status(StatusCode.ERROR)) + self._finish('cancelled', 'cancelled') + def success(self, attributes: Optional[Dict[str, AttributeValue]] = None) -> None: if self._ended: return @@ -112,5 +132,5 @@ def _finish(self, outcome: str, error_category: Optional[str] = None) -> None: "TeaQL runtime operation completed", extra=log_attributes, ) - self._activation.__exit__(None, None, None) + self.deactivate() self._span.end() diff --git a/src/teaql/runtime/query_list.py b/src/teaql/runtime/query_list.py new file mode 100644 index 0000000..688aac4 --- /dev/null +++ b/src/teaql/runtime/query_list.py @@ -0,0 +1,82 @@ +"""Invocation-local list execution shared by generated libraries and providers.""" +from __future__ import annotations + +import asyncio +import math +from contextvars import ContextVar +from copy import deepcopy +from typing import Any, Awaitable, Callable, TypeVar + +from teaql.core.list import SmartList +from teaql.core.query import AggregateFunction +from teaql.data_service import QueryRequest +from .telemetry import RuntimeOperation, observe_runtime_operation + +T = TypeVar('T') + +# This is a task-local execution capability, not a Context-owned query frame or +# mutation ledger. A different service, UserContext, invocation or task does not +# inherit ownership merely because an OTEL span is currently active. +_query_owner: ContextVar[tuple | None] = ContextVar('teaql_query_execution_owner', default=None) + + +async def observe_query_operation(context, service, request: QueryRequest, + work: Callable[[], Awaitable[T]], + completion: Callable[[T], dict]) -> T: + request.validate() + owner = (service, context, request._invocation_identity, asyncio.current_task()) + active = _query_owner.get() + if active is not None and all(left is right for left, right in zip(active, owner)): + return await work() + token = _query_owner.set(owner) + try: + telemetry = context.runtime_telemetry() if context is not None else None + return await observe_runtime_operation( + telemetry, RuntimeOperation('query', f'{request.root_request.query.entity}.list', { + 'teaql.entity.type': request.root_request.query.entity, + }), work, completion) + finally: + _query_owner.reset(token) + + +async def execute_typed_list(context, request: QueryRequest, hydrate: Callable[[dict], T], + *, count_operation=None, count_alias: str | None = None) -> SmartList[T]: + """Generated bridge: trusted policy and request snapshot precede this call. + + Do not install SDK spans in generated entities. Count is explicit metadata; + the provider handles relations/facets and hydration stays inside this root. + """ + root = request.with_query(request.query) + service = context.require_resource('dataService') + + async def execute(): + total = None + if count_operation is None: + row_result = await service.query(context, root) + else: + alias = count_alias if count_alias is not None else '__teaql_total' + query = deepcopy(root.query) + query.aggregates = [item for item in query.aggregates + if not (item.function == AggregateFunction.Count and item.alias == count_alias)] + rows_request = root.with_query(query) + + async def count(): + result = await service.query(context, root.for_count(count_alias, count_operation)) + value = result.rows[0].get(alias) if len(result.rows) == 1 else None + if (isinstance(value, bool) or not isinstance(value, (int, float)) or value < 0 + or isinstance(value, float) and not math.isfinite(value) or int(value) != value): + raise RuntimeError('dataService did not return an exact list count') + return int(value) + + if root.query.id_set_pagination is not None: + row_result = await service.query(context, rows_request) + retained, accuracy = context.id_set_count() + total = retained if accuracy == 'EXACT' else await count() + else: + total = await count() + row_result = await service.query(context, rows_request) + return SmartList((hydrate(row) for row in row_result.rows), facets=row_result.facets, + total_count=total, aggregations={count_alias: total} if count_alias is not None else {}) + + return await observe_query_operation(context, service, root, execute, + lambda result: {'teaql.result.cardinality': len(result)}) diff --git a/src/teaql/runtime/query_stream.py b/src/teaql/runtime/query_stream.py new file mode 100644 index 0000000..dc812fa --- /dev/null +++ b/src/teaql/runtime/query_stream.py @@ -0,0 +1,130 @@ +"""Lazy typed stream ownership and delivered-row accounting for generated Q.""" +import asyncio +from contextvars import ContextVar + +from .telemetry import RuntimeOperation, start_runtime_operation + + +_active_stream = ContextVar('teaql_typed_stream_execution', default=None) + + +def stream_observation(context, service, request): + active = _active_stream.get() + if active is None: + return None + owner, task = active + if (owner.context is context and owner.service is service + and owner.request._invocation_identity is request._invocation_identity + and task is asyncio.current_task()): + return owner + return None + + +def execute_typed_stream(context, request, hydrate, chunk_size): + request.validate() + if type(chunk_size) is not int or chunk_size <= 0: + raise ValueError('chunk_size must be positive') + service = context.require_resource('dataService') + return TypedStream(context, service, request.with_query(request.query), hydrate, chunk_size) + + +class TypedStream: + def __init__(self, context, service, request, hydrate, chunk_size): + self.context, self.service, self.request = context, service, request + self.hydrate, self.chunk_size = hydrate, chunk_size + self.cursor = self.scope = None + self.scopes = [] + self.rows = iter(()) + self.delivered = 0 + self.closed = False + self.busy = False + + def __aiter__(self): + return self + + def bind_scope(self, scope): + self.scopes.append(scope) + + def _activate(self): + if self.scope is None: + self.scope = start_runtime_operation(self.context.runtime_telemetry(), RuntimeOperation( + 'query', self.request.query.entity + '.stream', {'teaql.entity.type': self.request.query.entity})) + self.scopes.append(self.scope) + else: + for scope in self.scopes: + scope.activate() + return _active_stream.set((self, asyncio.current_task())) + + def _deactivate(self, token): + for scope in reversed(self.scopes): + scope.deactivate() + _active_stream.reset(token) + + async def _close_cursor(self): + if self.cursor is not None: + cursor, self.cursor = self.cursor, None + close = getattr(cursor, 'aclose', None) + if close is not None: + await close() + + async def __anext__(self): + if self.closed: + raise StopAsyncIteration + if self.busy: + raise RuntimeError('stream already has an active poll or close') + self.busy = True + token = self._activate() + try: + if self.cursor is None: + self.cursor = self.service.query_stream(self.context, self.request, self.chunk_size) + while True: + raw = next(self.rows, None) + if raw is not None: + value = self.hydrate(raw) + self.delivered += 1 + return value + chunk = await anext(self.cursor) + self.rows = iter(chunk.rows) + except StopAsyncIteration: + self.closed = True + try: + await self._close_cursor() + except BaseException as error: + self.scope.failure(error) + raise + self.scope.success({'teaql.result.cardinality': self.delivered}) + raise + except BaseException as error: + self.closed = True + try: + await self._close_cursor() + except BaseException: + pass # Preserve the original source/hydration/cancellation error. + if isinstance(error, asyncio.CancelledError): + self.scope.cancelled({'teaql.result.cardinality': self.delivered}) + else: + self.scope.failure(error) + raise + finally: + self._deactivate(token) + self.busy = False + + async def aclose(self): + if self.closed: + return + if self.busy: + raise RuntimeError('stream already has an active poll or close') + self.closed = True + if self.scope is None: + return # Unpolled streams create no SDK span, SQL or driver cursor. + self.busy = True + token = self._activate() + try: + await self._close_cursor() + self.scope.cancelled({'teaql.result.cardinality': self.delivered}) + except BaseException as error: + self.scope.failure(error) + raise + finally: + self._deactivate(token) + self.busy = False diff --git a/src/teaql/runtime/standard_telemetry.py b/src/teaql/runtime/standard_telemetry.py new file mode 100644 index 0000000..2a0b60d --- /dev/null +++ b/src/teaql/runtime/standard_telemetry.py @@ -0,0 +1,314 @@ +"""Optional isolated SDK. This module is loaded only after explicit enablement.""" +import threading +from time import monotonic, time_ns +from opentelemetry import context as otel_context, trace +from opentelemetry._logs import LogRecord, SeverityNumber +from opentelemetry.metrics import NoOpMeterProvider +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.sampling import ALWAYS_ON +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from .telemetry_http import HttpExporter, record_count, export_result +from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator +from .telemetry import RuntimeOperation, runtime_error_category +from .telemetry_startup import TelemetryConfiguration, ENABLED, SERVICE, ENDPOINT, notice +from .telemetry_queue import ExportQueue, AdmissionBatchProcessor + + +class _SafeExporter: + def __init__(self, delegate, pipeline, signal, queue=None): + self.delegate, self.pipeline, self.signal, self.queue = delegate, pipeline, signal, queue + + def __getattr__(self, name): + return getattr(self.delegate, name) + + def export(self, batch, *args, **kwargs): + result_type = export_result(self.signal) + ticket = self.queue.begin_export(batch) if self.queue is not None else None + if self.queue is not None and ticket is None: + return result_type.SUCCESS # Retired SDK deque entries cannot re-export. + try: + count = record_count(batch, self.signal) + except Exception: + count = 0 + try: + result = self.delegate.export(batch, *args, **kwargs) + except Exception: + result = result_type.FAILURE + failed = getattr(result, 'name', '') == 'FAILURE' + if ticket is not None: + ticket(failed) + elif failed: + try: + self.pipeline._lost(self.signal, count) + except Exception: + notice('TeaQL telemetry: export loss accounting incomplete; business outcome unchanged.') + return result + + +class StandardTelemetry: + def __init__(self, config): + valid = TelemetryConfiguration.read({ENABLED: 'true', SERVICE: config.service, ENDPOINT: config.endpoint}) + if not valid.enabled: + raise ValueError('invalid-telemetry-configuration') + self.closed = False + self._lock = threading.Lock() + self._lifecycle_lock = threading.Lock() + self._flush_jobs = {} + self.losses = {'traces': 0, 'logs': 0, 'metrics': 0} + self.loss_reasons = {signal: {reason: 0 for reason in ( + 'queue_overflow', 'export_failure', 'shutdown', 'handoff_failure')} for signal in self.losses} + self.queues = {signal: ExportQueue(signal, 2048, self._queue_loss) for signal in ('traces', 'logs')} + self.responses = {signal: {'rejected': 0, 'partial_success': 0, 'warning': 0, 'invalid_response': 0} + for signal in self.losses} + resource = Resource({'service.name': valid.service, 'teaql.runtime.language': 'python'}) + sdk_metrics = NoOpMeterProvider() + self.traces = TracerProvider(resource=resource, sampler=ALWAYS_ON, shutdown_on_exit=False, meter_provider=sdk_metrics) + self.http = {signal: HttpExporter(valid.endpoint + '/v1/' + signal, signal, self._response) + for signal in ('traces', 'logs', 'metrics')} + trace_processor = BatchSpanProcessor(_SafeExporter( + self.http['traces'], self, 'traces', self.queues['traces']), + max_queue_size=2048, max_export_batch_size=128, schedule_delay_millis=1000, export_timeout_millis=5000, meter_provider=sdk_metrics) + trace_processor._batch_processor = AdmissionBatchProcessor(trace_processor._batch_processor, self.queues['traces']) + self.traces.add_span_processor(trace_processor) + reader = PeriodicExportingMetricReader(_SafeExporter( + self.http['metrics'], self, 'metrics'), + export_interval_millis=10000, export_timeout_millis=5000) + self.metrics = MeterProvider(resource=resource, metric_readers=[reader], shutdown_on_exit=False) + self.logs = LoggerProvider(resource=resource, shutdown_on_exit=False, meter_provider=sdk_metrics) + log_processor = BatchLogRecordProcessor(_SafeExporter( + self.http['logs'], self, 'logs', self.queues['logs']), + max_queue_size=2048, max_export_batch_size=128, schedule_delay_millis=1000, export_timeout_millis=5000, meter_provider=sdk_metrics) + log_processor._batch_processor = AdmissionBatchProcessor(log_processor._batch_processor, self.queues['logs']) + self.logs.add_log_record_processor(log_processor) + self.tracer = self.traces.get_tracer('io.teaql.runtime') + self.logger = self.logs.get_logger('io.teaql.runtime') + meter = self.metrics.get_meter('io.teaql.runtime') + self.duration = meter.create_histogram('teaql.runtime.operation.duration', unit='ms') + self.operations = meter.create_counter('teaql.runtime.operation.count', unit='{operation}') + self.lost_counter = meter.create_counter('teaql.telemetry.export.lost', unit='{record}') + self.rejected_counter = meter.create_counter('teaql.telemetry.export.rejected', unit='{record_or_point}') + self.response_counter = meter.create_counter('teaql.telemetry.export.response', unit='{response}') + for signal in self.losses: + for reason in self.loss_reasons[signal]: + self.lost_counter.add(0, {'teaql.telemetry.signal': signal, 'teaql.telemetry.reason': reason}) + self.rejected_counter.add(0, {'teaql.telemetry.signal': signal}) + for reason in ('partial_success', 'warning', 'invalid_response'): + self.response_counter.add(0, {'teaql.telemetry.signal': signal, 'teaql.telemetry.reason': reason}) + self.propagator = TraceContextTextMapPropagator() + + def _lost(self, signal, count): + self._queue_loss(signal, count, 'export_failure') + + def _queue_loss(self, signal, count, reason): + with self._lock: + first = self.loss_reasons[signal][reason] == 0 + self.loss_reasons[signal][reason] += count + if reason == 'export_failure': + self.losses[signal] += count + self.lost_counter.add(count, {'teaql.telemetry.signal': signal, 'teaql.telemetry.reason': reason}) + if first: + notice(f'TeaQL telemetry: records lost; signal={signal} reason={reason}; business outcome unchanged.') + + def _response(self, signal, rejected, reason): + with self._lock: + first = self.responses[signal][reason] == 0 + self.responses[signal][reason] += 1 + self.responses[signal]['rejected'] += rejected + self.rejected_counter.add(rejected, {'teaql.telemetry.signal': signal}) + self.response_counter.add(1, {'teaql.telemetry.signal': signal, 'teaql.telemetry.reason': reason}) + if first: + notice(f'TeaQL telemetry: {signal} {reason}; fixed diagnostic counted; business outcome unchanged.') + + def health(self): + with self._lock: + snapshot = {'failed_records': dict(self.losses), 'responses': {signal: dict(values) for signal, values in self.responses.items()}, + 'loss_reasons': {signal: dict(values) for signal, values in self.loss_reasons.items()}} + snapshot['queues'] = {signal: queue.snapshot() for signal, queue in self.queues.items()} + return snapshot + + def start(self, operation): + if self.closed: + return _ClosedScope(self) + return _Scope(self, operation.safe()) + + def inject(self, carrier): + self.propagator.inject(carrier) + + def emit(self, body, attributes, correlation): + if not correlation or self.closed: + return + span_context = trace.SpanContext(correlation[0], correlation[1], False, trace.TraceFlags.SAMPLED) + if not span_context.is_valid: + notice('TeaQL telemetry: SQL correlation unavailable; no replacement span created.') + return + context = trace.set_span_in_context(trace.NonRecordingSpan(span_context), otel_context.Context()) + self.logger.emit(LogRecord(timestamp=time_ns(), context=context, severity_number=SeverityNumber.INFO, + severity_text='INFO', body=body, attributes=attributes)) + + def record_diagnostic(self, attributes, correlation): + self.emit('TeaQL SQL execution', attributes, correlation) + + def _lifecycle(self, close): + started = monotonic() + if not self._lifecycle_lock.acquire(timeout=6): + notice('TeaQL telemetry: lifecycle timed out; business outcome unchanged.') + return + try: + if self.closed: + return + if close: + self.closed = True + for queue in self.queues.values(): + queue.close() + self._settle_lifecycle(close, started + 6) + finally: + self._lifecycle_lock.release() + + def _settle_lifecycle(self, close, deadline): + prefixes = {signal: queue.prefix() for signal, queue in self.queues.items()} + completed = [] + for signal, provider in (('traces', self.traces), ('logs', self.logs)): + event = self._flush_jobs.get(signal) if not close else None + if event is not None and not event.is_set(): + completed.append(event) + continue + event = threading.Event() + if not close: + self._flush_jobs[signal] = event + completed.append(event) + def run(provider=provider, event=event): + try: + if close: + provider.shutdown() + else: + provider.force_flush(timeout_millis=5000) + except Exception: + notice('TeaQL telemetry: lifecycle incomplete; business outcome unchanged.') + finally: + event.set() + threading.Thread(target=run, daemon=True).start() + drained = all(queue.wait_prefix(prefixes[signal], max(0, deadline - 1 - monotonic())) + for signal, queue in self.queues.items()) + if not drained: + notice('TeaQL telemetry: lifecycle timed out; business outcome unchanged.') + if close: + for event in completed: + event.wait(max(0, deadline - 1 - monotonic())) + if close and not drained: + for queue in self.queues.values(): + queue.abort_queued() + for signal in ('traces', 'logs'): + self.http[signal].abort() + for signal in ('traces', 'logs'): + self.queues[signal].wait_idle(max(0, deadline - .3 - monotonic())) + self.http[signal].wait_idle(max(0, deadline - .3 - monotonic())) + for event in completed: + event.wait(max(0, deadline - .3 - monotonic())) + # The cumulative metric snapshot follows trace/log acknowledgements. + # It can therefore report rejection/loss from those exact attempts. + metric_done = threading.Event() + def metrics(): + try: + remaining = max(1, int((deadline - monotonic()) * 1000)) + if close: + self.metrics.shutdown(timeout_millis=remaining) + else: + self.metrics.force_flush(timeout_millis=remaining) + except Exception: + notice('TeaQL telemetry: lifecycle incomplete; business outcome unchanged.') + finally: + metric_done.set() + threading.Thread(target=metrics, daemon=True).start() + if not metric_done.wait(max(0, deadline - monotonic())): + notice('TeaQL telemetry: lifecycle timed out; business outcome unchanged.') + if close: + for queue in self.queues.values(): + queue.abort_queued() + for exporter in self.http.values(): + exporter.shutdown() + exporter.wait_idle(max(0, deadline - monotonic())) + + def flush(self): + self._lifecycle(False) + + def shutdown(self): + self._lifecycle(True) + + +class _Scope: + def __init__(self, pipeline, operation): + self.pipeline, self.operation, self.started = pipeline, operation, monotonic() + self.span = pipeline.tracer.start_span('teaql.' + operation.family, attributes=operation.attributes) + state = self.span.get_span_context() + self.correlation = (state.trace_id, state.span_id) + self.token = otel_context.attach(trace.set_span_in_context(self.span)) + self.ended = False + + def activate(self): + if self.token is None and not self.ended: + self.token = otel_context.attach(trace.set_span_in_context(self.span)) + + def deactivate(self): + if self.token is not None: + token, self.token = self.token, None + otel_context.detach(token) + + def cancelled(self, attributes=None): + for key, value in (attributes or {}).items(): + if key == 'teaql.result.cardinality': + self.span.set_attribute(key, value) + self._finish('cancelled', 'cancelled') + + def success(self, attributes=None): + for key, value in (attributes or {}).items(): + if key in ('teaql.result.cardinality', 'teaql.cache.result'): + self.span.set_attribute(key, value) + self._finish('success') + + def failure(self, error): + self.span.set_attribute('teaql.error.type', type(error).__name__) + self.span.set_attribute('teaql.error.category', runtime_error_category(error)) + self._finish('failure', runtime_error_category(error)) + + def _finish(self, outcome, category=None): + if self.ended: + return + self.ended = True + try: + if self.pipeline.closed: + self.pipeline.queues['logs'].reject_late() + return + self.span.set_attribute('teaql.operation.outcome', outcome) + self.span.set_status(trace.Status(trace.StatusCode.OK if outcome == 'success' else trace.StatusCode.ERROR)) + elapsed = max(0, monotonic() - self.started) * 1000 + tags = {'teaql.operation.family': self.operation.family, 'teaql.operation.outcome': outcome} + self.pipeline.duration.record(elapsed, tags) + self.pipeline.operations.add(1, tags) + values = {**tags, 'teaql.operation.name': self.operation.name, 'teaql.operation.duration_ms': elapsed} + if category: + values['teaql.error.category'] = category + self.pipeline.emit('TeaQL runtime operation completed', values, self.correlation) + finally: + self.deactivate() + self.span.end() + + +class _ClosedScope: + correlation = None + + def __init__(self, pipeline): + self.pipeline, self.ended = pipeline, False + + def success(self, attributes=None): + if not self.ended: + self.ended = True + for queue in self.pipeline.queues.values(): + queue.reject_late() + + def failure(self, error): + self.success() diff --git a/src/teaql/runtime/telemetry.py b/src/teaql/runtime/telemetry.py index 88320bc..b7cc703 100644 --- a/src/teaql/runtime/telemetry.py +++ b/src/teaql/runtime/telemetry.py @@ -68,12 +68,22 @@ def shutdown(self) -> None: ... class _NoopScope: + correlation = None def success(self, attributes: Optional[Dict[str, AttributeValue]] = None) -> None: pass def failure(self, error: BaseException) -> None: pass + def activate(self): + pass + + def deactivate(self): + pass + + def cancelled(self, attributes=None): + pass + class NoopRuntimeTelemetry: def start(self, operation: RuntimeOperation) -> RuntimeTelemetryScope: @@ -112,6 +122,43 @@ def __init__(self, delegate: RuntimeTelemetryScope): self._delegate = delegate self._ended = False + @property + def correlation(self): + try: + return getattr(self._delegate, 'correlation', None) + except Exception: + return None + + def activate(self): + try: + method = getattr(self._delegate, 'activate', None) + if method is not None: + method() + except BaseException: + pass + + def deactivate(self): + try: + method = getattr(self._delegate, 'deactivate', None) + if method is not None: + method() + except BaseException: + pass + + def cancelled(self, attributes=None): + if self._ended: + return + self._ended = True + try: + method = getattr(self._delegate, 'cancelled', None) + if method is not None: + method(attributes) + else: + import asyncio + self._delegate.failure(asyncio.CancelledError()) + except BaseException: + pass + def success(self, attributes: Optional[Dict[str, AttributeValue]] = None) -> None: if self._ended: return diff --git a/src/teaql/runtime/telemetry_http.py b/src/teaql/runtime/telemetry_http.py new file mode 100644 index 0000000..bd52049 --- /dev/null +++ b/src/teaql/runtime/telemetry_http.py @@ -0,0 +1,265 @@ +"""Owned finite OTLP HTTP transport using the official protobuf encoders. + +The generated flat library owns a `requests` package. The private stdlib HTTP +connections need neither that third-party package nor customer HTTP log settings. +""" +import http.client +import socket +import threading +from time import monotonic +from urllib.parse import urlsplit +from opentelemetry.sdk.metrics.export import MetricExporter, MetricExportResult +from opentelemetry.sdk.trace.export import SpanExportResult +from opentelemetry.sdk._logs.export import LogRecordExportResult +from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans +from opentelemetry.exporter.otlp.proto.common.metrics_encoder import encode_metrics +from opentelemetry.exporter.otlp.proto.common._log_encoder import encode_logs +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceResponse +from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import ExportLogsServiceResponse +from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import ExportMetricsServiceResponse + + +MAX_RESPONSE_BYTES = 4 * 1024 * 1024 + + +def export_result(signal): + return {'traces': SpanExportResult, 'metrics': MetricExportResult, 'logs': LogRecordExportResult}[signal] + + +def record_count(batch, signal): + if signal != 'metrics': + return len(batch) + return sum(len(scope.metrics) for resource in batch.resource_metrics for scope in resource.scope_metrics) + + +def _varint(data, offset): + value = 0 + for index in range(10): + if offset >= len(data): + raise ValueError('invalid-response') + byte = data[offset] + offset += 1 + if index == 9 and byte > 1: + raise ValueError('invalid-response') + value |= (byte & 127) << (index * 7) + if byte < 128: + return value, offset + raise ValueError('invalid-response') + + +def _validate_wire(data, inner=False): + """Bound the two-level proto3 schema before official decoding. + + Unknown proto2 groups are not part of an OTLP acknowledgement. Reject them + rather than relying on a decoder's recursive unknown-field skipping. + """ + offset = 0 + while offset < len(data): + tag, offset = _varint(data, offset) + field, wire = tag >> 3, tag & 7 + if not 0 < field < 2**29 or (field == 1 and wire != (0 if inner else 2)) or (inner and field == 2 and wire != 2): + raise ValueError('invalid-response') + if wire == 0: + _, offset = _varint(data, offset) + elif wire in (1, 5): + offset += 8 if wire == 1 else 4 + elif wire == 2: + size, offset = _varint(data, offset) + end = offset + size + if end > len(data): + raise ValueError('invalid-response') + if field == 1 and not inner: + _validate_wire(data[offset:end], True) + elif field == 2 and inner: + data[offset:end].decode('utf-8', 'strict') + offset = end + else: + raise ValueError('invalid-response') + if offset > len(data): + raise ValueError('invalid-response') + + +def _acknowledgement(data, signal, submitted): + _validate_wire(data) + response_type, field = { + 'traces': (ExportTraceServiceResponse, 'rejected_spans'), + 'logs': (ExportLogsServiceResponse, 'rejected_log_records'), + 'metrics': (ExportMetricsServiceResponse, 'rejected_data_points'), + }[signal] + response = response_type.FromString(data) + rejected = getattr(response.partial_success, field) + if rejected < 0 or rejected > submitted: + raise ValueError('invalid-response') + return rejected, ('partial_success' if rejected else 'warning' if response.partial_success.error_message else None) + + +class _Deadline: + """Cancel only sockets belonging to this owned HTTP attempt.""" + def __init__(self, seconds): + self.ends = monotonic() + seconds + self.lock = threading.Lock() + self.sockets = [] + self.connection = None + self.expired = False + self.timer = threading.Timer(seconds, self.abort) + self.timer.daemon = True + self.timer.start() + + def abort(self): + with self.lock: + self.expired = True + if self.connection is not None and self.connection.sock is not None: + self.sockets.append(self.connection.sock) + for sock in self.sockets: + try: + sock.shutdown(socket.SHUT_RDWR) + except (OSError, ValueError): + pass + + def attach(self, connection): + with self.lock: + self.connection = connection + self.check() + + def check(self): + if self.expired or monotonic() >= self.ends: + raise TimeoutError('export-deadline') + + def close(self): + self.timer.cancel() + + +class HttpExporter(MetricExporter): + def __init__(self, endpoint, signal, on_response=None): + super().__init__() + self.endpoint, self.signal = endpoint, signal + self._lock = threading.Lock() + self._active = None + self.closed = False + self.on_response = on_response + + def _report(self, reason, rejected=0): + if self.on_response and reason: + try: + self.on_response(self.signal, rejected, reason) + except Exception: + pass # Optional diagnostics must not change export/business work. + + def export(self, batch, timeout_millis=None, **kwargs): + result = export_result(self.signal) + if self.closed: + return result.FAILURE + encoder = {'traces': encode_spans, 'metrics': encode_metrics, 'logs': encode_logs}[self.signal] + try: + payload = encoder(batch) + submitted = (sum(len(getattr(metric, metric.WhichOneof('data')).data_points) + for resource in payload.resource_metrics for scope in resource.scope_metrics + for metric in scope.metrics if metric.WhichOneof('data')) + if self.signal == 'metrics' else len(batch)) + seconds = min(3, timeout_millis / 1000) if timeout_millis is not None else 3 + if seconds <= 0: + return result.FAILURE + with self._lock: + if self.closed or self._active is not None: + return result.FAILURE + attempt = _Attempt(self.endpoint, self.signal, payload.SerializeToString(), submitted, seconds) + self._active = attempt + def work(): + try: + attempt.run() + finally: + with self._lock: + if self._active is attempt: + self._active = None + attempt.done.set() + threading.Thread(target=work, daemon=True).start() + if not attempt.done.wait(seconds): + attempt.deadline.abort() + return result.FAILURE + if attempt.deadline.expired: + return result.FAILURE + if attempt.outcome == 'invalid_response': + self._report('invalid_response') + return result.FAILURE + if attempt.outcome == 'failure': + return result.FAILURE + self._report(attempt.outcome, attempt.rejected) + return result.SUCCESS + except Exception: + # No driver/endpoint/response text can escape into SDK log handlers. + return result.FAILURE + + def force_flush(self, timeout_millis=5000): + return True + + def shutdown(self, timeout_millis=5000, **kwargs): + with self._lock: + self.closed = True + attempt = self._active + if attempt: + attempt.deadline.abort() + + def abort(self): + with self._lock: + attempt = self._active + if attempt: + attempt.deadline.abort() + + def wait_idle(self, timeout): + with self._lock: + attempt = self._active + return attempt is None or attempt.done.wait(timeout) + + +class _Attempt: + """At most one private attempt/worker per exporter, no customer HTTP logs.""" + def __init__(self, endpoint, signal, payload, submitted, seconds): + self.endpoint, self.signal, self.payload = endpoint, signal, payload + self.submitted, self.seconds = submitted, seconds + self.done = threading.Event() + self.deadline = _Deadline(seconds) + self.outcome, self.rejected = 'failure', 0 + + def run(self): + connection = None + try: + uri = urlsplit(self.endpoint) + connection_type = http.client.HTTPSConnection if uri.scheme == 'https' else http.client.HTTPConnection + connection = connection_type(uri.hostname, uri.port, timeout=self.seconds) + connection.set_debuglevel(0) + self.deadline.attach(connection) + connection.connect() + self.deadline.check() + connection.request('POST', uri.path or '/', self.payload, + headers={'Content-Type': 'application/x-protobuf', 'Accept-Encoding': 'identity', 'Connection': 'close'}) + response = connection.getresponse() + self.deadline.check() + if response.status != 200: + return + if (response.getheader('Content-Type', '').split(';')[0].strip().lower() != 'application/x-protobuf' + or response.getheader('Content-Encoding', 'identity').lower() != 'identity'): + self.outcome = 'invalid_response' + return + try: + length = response.getheader('Content-Length') + if length is not None and (not length.isdecimal() or int(length) > MAX_RESPONSE_BYTES): + raise ValueError('invalid-response') + data = bytearray() + while True: + self.deadline.check() + chunk = response.read(16384) + if not chunk: + break + if len(data) + len(chunk) > MAX_RESPONSE_BYTES: + raise ValueError('invalid-response') + data.extend(chunk) + self.deadline.check() + self.rejected, self.outcome = _acknowledgement(bytes(data), self.signal, self.submitted) + except ValueError: + self.outcome = 'invalid_response' + except Exception: + self.outcome = 'failure' # Never return/log raw transport exceptions. + finally: + if connection: + connection.close() + self.deadline.close() diff --git a/src/teaql/runtime/telemetry_queue.py b/src/teaql/runtime/telemetry_queue.py new file mode 100644 index 0000000..cfe692c --- /dev/null +++ b/src/teaql/runtime/telemetry_queue.py @@ -0,0 +1,155 @@ +"""Exact, bounded SDK-record admission; no replacement SDK batching queue.""" +import threading +from time import monotonic + + +class _Admission: + def __init__(self, record): + self.record = record + self.sequence = None + self.state = 'queued' + + +class ExportQueue: + def __init__(self, signal, capacity, on_loss): + if type(capacity) is not int or capacity < 1: + raise ValueError('invalid-queue-capacity') + self.signal, self.capacity, self.on_loss = signal, capacity, on_loss + self.condition = threading.Condition(threading.RLock()) + self.pending = {} + self.offered = self.admitted = self.completed = self.overflow = 0 + self.post_shutdown = self.handoff_failed = self.retired = 0 + self.closed = self.aborted = False + + def _lost(self, amount, reason): + if amount: + try: + self.on_loss(self.signal, amount, reason) + except Exception: + pass + + def handoff(self, record, submit): + reason = None + with self.condition: + self.offered += 1 + if self.closed: + self.post_shutdown += 1 + reason = 'shutdown' + elif len(self.pending) >= self.capacity: + self.overflow += 1 + reason = 'queue_overflow' + elif id(record) in self.pending: + self.handoff_failed += 1 + reason = 'handoff_failure' + else: + key = id(record) + admission = _Admission(record) + self.pending[key] = admission + try: + submit() + self.admitted += 1 + admission.sequence = self.admitted + if admission.state == 'complete': + self.completed += 1 + except Exception: + self.pending.pop(key, None) + self.handoff_failed += 1 + reason = 'handoff_failure' + self.condition.notify_all() + if reason: + self._lost(1, reason) + + def begin_export(self, records): + with self.condition: + if self.aborted: + return None + admissions = [self.pending.get(id(record)) for record in records] + if any(item is None or item.record is not record or item.state != 'queued' + for item, record in zip(admissions, records)): + return None + for item in admissions: + item.state = 'exporting' + completed = False + def complete(failed): + nonlocal completed + settled = 0 + with self.condition: + if completed: + return + completed = True + for item in admissions: + if self.pending.pop(id(item.record), None) is not item: + continue + item.state = 'complete' + settled += 1 + if item.sequence is not None: + self.completed += 1 + if failed: + self._lost(settled, 'export_failure') + self.condition.notify_all() + return complete + + def prefix(self): + with self.condition: + return self.admitted + + def wait_prefix(self, target, timeout): + deadline = monotonic() + timeout + with self.condition: + while any(item.sequence is not None and item.sequence <= target for item in self.pending.values()): + remaining = deadline - monotonic() + if remaining <= 0: + return False + self.condition.wait(remaining) + return True + + def wait_idle(self, timeout): + deadline = monotonic() + timeout + with self.condition: + while self.pending: + remaining = deadline - monotonic() + if remaining <= 0: + return False + self.condition.wait(remaining) + return True + + def close(self): + with self.condition: + self.closed = True + + def abort_queued(self): + with self.condition: + if self.aborted: + return + self.aborted = True + retired = [key for key, item in self.pending.items() if item.state == 'queued'] + for key in retired: + self.pending.pop(key).state = 'retired' + self.retired += len(retired) + self.condition.notify_all() + self._lost(len(retired), 'shutdown') + + def reject_late(self): + with self.condition: + self.offered += 1 + self.post_shutdown += 1 + self._lost(1, 'shutdown') + + def snapshot(self): + with self.condition: + return {'offered': self.offered, 'admitted': self.admitted, 'completed': self.completed, + 'outstanding': len(self.pending), 'exporting': sum(item.state == 'exporting' for item in self.pending.values()), + 'overflow': self.overflow, 'post_shutdown': self.post_shutdown, + 'handoff_failed': self.handoff_failed, 'retired': self.retired} + + +class AdmissionBatchProcessor: + """Wrap the SDK's already-snapshotted record handoff, not mutable log input.""" + def __init__(self, delegate, queue): + self.delegate, self.queue = delegate, queue + + def __getattr__(self, name): + return getattr(self.delegate, name) + + def emit(self, record): + self.queue.handoff(record, lambda: self.delegate.emit(record)) diff --git a/src/teaql/runtime/telemetry_startup.py b/src/teaql/runtime/telemetry_startup.py new file mode 100644 index 0000000..e8db480 --- /dev/null +++ b/src/teaql/runtime/telemetry_startup.py @@ -0,0 +1,109 @@ +"""SDK-neutral process configuration; importing this file imports no SDK.""" +from dataclasses import dataclass +import os +import re +import sys +import threading +from urllib.parse import urlsplit + +ENABLED = 'TEAQL_TELEMETRY_ENABLED' +SERVICE = 'TEAQL_TELEMETRY_SERVICE_NAME' +ENDPOINT = 'TEAQL_TELEMETRY_OTLP_ENDPOINT' +HINTS = f'{ENABLED}=true {SERVICE}= {ENDPOINT}=' + + +def notice(message): + try: + print(message, file=sys.stderr) + except Exception: + pass + + +@dataclass(frozen=True) +class TelemetryConfiguration: + enabled: bool = False + service: str = '' + endpoint: str = '' + error: str = '' + + @classmethod + def read(cls, environment): + flag = environment.get(ENABLED, '').strip().lower() + if flag in ('', 'false'): + return cls() + if flag != 'true': + return cls(error='invalid-enable') + service = environment.get(SERVICE, '').strip() + if not re.fullmatch(r'[a-zA-Z0-9_.-]{1,120}', service): + return cls(error='invalid-service') + endpoint = environment.get(ENDPOINT, '') + try: + uri = urlsplit(endpoint) + _ = uri.port + if uri.scheme not in ('http', 'https') or not uri.hostname or uri.username is not None or uri.password is not None or '?' in endpoint or '#' in endpoint or any(c.isspace() for c in endpoint): + return cls(error='invalid-endpoint') + except ValueError: + return cls(error='invalid-endpoint') + return cls(True, service, endpoint.rstrip('/')) + + @property + def message(self): + if self.error: + return f'TeaQL telemetry: disabled; configuration={self.error}. {HINTS}' + if not self.enabled: + return f'TeaQL telemetry: supported, disabled. Enable full masked diagnostics with {HINTS}' + return f'TeaQL telemetry: starting standard module; full recording, masked SQL; {ENABLED}=true {SERVICE}={self.service} {ENDPOINT}={self.endpoint}' + + +_lock = threading.Lock() +_pipelines = {} +_announced = set() + + +def default_telemetry(environment=None): + from .telemetry import NOOP_RUNTIME_TELEMETRY + config = TelemetryConfiguration.read(os.environ if environment is None else environment) + with _lock: + if config.message not in _announced: + _announced.add(config.message) + notice(config.message) + if not config.enabled: + return NOOP_RUNTIME_TELEMETRY + key = (config.service, config.endpoint) + pipeline = _pipelines.get(key) + if pipeline is not None and not pipeline.closed: + return pipeline + try: + from .standard_telemetry import StandardTelemetry + except ImportError: + notice('TeaQL telemetry: standard module unavailable; install teaql[telemetry]. ' + HINTS) + return NOOP_RUNTIME_TELEMETRY + completed = threading.Event() + result = [] + abandoned = threading.Event() + handshake = threading.Lock() + def build(): + try: + candidate = StandardTelemetry(config) + with handshake: + cleanup = abandoned.is_set() + if not cleanup: + result.append(candidate) + if cleanup: + candidate.shutdown() + except Exception: + pass + finally: + completed.set() + threading.Thread(target=build, daemon=True).start() + completed.wait(5) + with handshake: + failed = not result + if failed: + abandoned.set() + if failed: + notice('TeaQL telemetry: initialization failed; business continues without export.') + return NOOP_RUNTIME_TELEMETRY + _pipelines[key] = result[0] + notice('TeaQL telemetry: enabled; full recording, traces, metrics, logs and masked SQL.') + return result[0] diff --git a/src/teaql/sql/executor.py b/src/teaql/sql/executor.py index 7b45aa9..b5bbf21 100644 --- a/src/teaql/sql/executor.py +++ b/src/teaql/sql/executor.py @@ -1,7 +1,7 @@ from abc import ABC, abstractmethod from typing import List, Dict, Any, Optional, AsyncIterator from copy import deepcopy -from datetime import datetime +from datetime import date, datetime import asyncio import hashlib import logging @@ -10,6 +10,7 @@ from array import array from dataclasses import fields, is_dataclass, replace from enum import Enum +from uuid import uuid4 from teaql.data_service import ( DataServiceExecutor, QueryExecutor, MutationExecutor, DataServiceCapabilities, QueryRequest, QueryResult, @@ -26,23 +27,17 @@ IsNullExpr, NotExpr, OrExpr, SubQueryExpr, ColumnExpr, ValueExpr, ) from teaql.core.query import Aggregate, AggregateFunction, SelectQuery -from teaql.core.value import Value +from teaql.core.request_tree import QueryOperation +from teaql.core.value import DataType, Value from teaql.core.trace import canonical_sql_trace_path, physical_readback_path, trace_name from teaql.runtime.telemetry import RuntimeOperation, observe_runtime_operation, start_runtime_operation +from teaql.runtime.query_list import observe_query_operation from teaql.runtime.context import RetainedIdSet _id_set_build_locks = {} _id_set_build_locks_guard = threading.RLock() -class _QueryWithLogIntent(QueryRequest): - """Invocation-local compiler plumbing, excluded from dataclass/wire fields.""" - def __init__(self, query, trace_chain, comment, purpose, source, origin_entity=None, assembly=None): - super().__init__(query, trace_chain, comment, purpose, _origin_entity=origin_entity) - self._log_intent_source = source - self._relation_assembly = assembly - - class _RelationAssembly: """One derived load owns scalar keys; never attach them to result records.""" def __init__(self, field): @@ -161,12 +156,41 @@ def __init__(self, dialect: SqlDialect, transport: SqlTransport, schema_provider self.dialect = dialect self.transport = transport self.schema_provider = schema_provider + # Context may legitimately have no ambient "db" resource. The actual + # executor still needs a stable, non-recycled cache isolation identity. + self._id_set_source_identity = uuid4().hex def _backend_name(self) -> str: kind = self.dialect.kind() name = getattr(kind, 'name', str(kind)).lower() return 'postgres' if name == 'postgresql' else name + @staticmethod + def _normalize_loaded_dates(rows, descriptor, query=None): + """Declared Date fields stay native through list, cursor and save readback. + + Do not guess from text content or convert computed aggregate aliases. + Missing columns and loaded SQL NULL retain their distinct states. + """ + aliases = {aggregate.alias for aggregate in query.aggregates} if query is not None else set() + properties = [prop for prop in descriptor.properties + if prop.property_type == DataType.Date or str(prop.property_type).lower() == 'date'] + for row in rows: + for prop in properties: + for key in {prop.name, prop.column_name_val} - aliases: + if key not in row or row[key] is None: + continue + current = row[key] + if isinstance(current, datetime): + row[key] = current.date() + elif isinstance(current, str): + try: + row[key] = date.fromisoformat(current) + except ValueError: + raise ValueError(f'invalid loaded Date field {descriptor._name}.{prop.name}') from None + elif not isinstance(current, date): + raise ValueError(f'invalid loaded Date field {descriptor._name}.{prop.name}') + def _query_intent_bindings(self, compiled, request): """Capture declared descendant bindings before any physical parent log. @@ -274,7 +298,7 @@ def capabilities(self) -> DataServiceCapabilities: ) def _record_statement(self, context, request, compiled, started_at, operation, - outcome, result_count=None, affected_rows=None, intent_privacy=None): + outcome, result_count=None, affected_rows=None, intent_privacy=None, correlation=None): """Project through context; never attach driver exceptions to diagnostics.""" query = operation == DataServiceOperation.Query entity = request.query.entity if query else request._data.entity @@ -306,8 +330,11 @@ def _record_statement(self, context, request, compiled, started_at, operation, comment=comment, purpose=request._purpose if query else None, audit_reason=None if query else comment, trace_chain=canonical_sql_trace_path(trace_source, provider, sql_operation), + request_trace_chain=request.business_trace_chain if query else (), mutation_lineage=() if query else request.mutation_lineage, ) + # Not a dataclass/wire field. This identity belongs to the physical call. + metadata._sql_correlation = correlation if context is not None: try: source = getattr(request, '_log_intent_source', None) if query else None @@ -359,46 +386,71 @@ async def _query_stream(self, context, request: QueryRequest, chunk_size: int): delivered = 0 started_at = datetime.now() outcome = 'cancelled' - stream = self.transport.stream_sql(compiled, chunk_size) + from teaql.runtime.query_stream import stream_observation + observation = stream_observation(context, self, request) + telemetry = context.runtime_telemetry() if context is not None else None + statement = start_runtime_operation(telemetry, RuntimeOperation('provider', f'{self.dialect.kind()}.stream', { + 'teaql.provider.kind': str(self.dialect.kind()), 'teaql.provider.operation': 'query', + 'teaql.provider.execution': 'statement'})) + if observation is not None: + observation.bind_scope(statement) + failure = None + stream = None try: + stream = self.transport.stream_sql(compiled, chunk_size) async for rows in stream: + self._normalize_loaded_dates(rows, entity_desc, request.query) if pending is not None: delivered += len(pending) - yield StreamChunk(pending, index, False) + statement.deactivate() + try: + yield StreamChunk(pending, index, False) + finally: + statement.activate() index += 1 pending = rows if pending is not None: delivered += len(pending) - yield StreamChunk(pending, index, True) + statement.deactivate() + try: + yield StreamChunk(pending, index, True) + finally: + statement.activate() outcome = 'success' except (asyncio.CancelledError, GeneratorExit): outcome = 'cancelled' raise - except BaseException: + except BaseException as error: outcome = 'failure' + failure = error raise finally: try: close = getattr(stream, 'aclose', None) if close is not None: await close() - except BaseException: + except BaseException as error: if outcome == 'success': outcome = 'failure' + failure = error raise # Preserve the error/close already being propagated. finally: + count = observation.delivered if observation is not None else delivered + if outcome == 'success': + statement.success({'teaql.result.cardinality': count}) + elif outcome == 'cancelled': + statement.cancelled({'teaql.result.cardinality': count}) + else: + statement.failure(failure) self._record_statement(context, request, compiled, started_at, - DataServiceOperation.Query, outcome, result_count=delivered) + DataServiceOperation.Query, outcome, result_count=count, + correlation=statement.correlation) async def query(self, context: 'UserContext', request: QueryRequest) -> QueryResult: request.validate() - telemetry = context.runtime_telemetry() if context is not None else None - return await observe_runtime_operation( - telemetry, - RuntimeOperation("query", f"{request.query.entity}.list", { - "teaql.entity.type": request.query.entity, - }), + return await observe_query_operation( + context, self, request, lambda: self._query(context, request), lambda result: {"teaql.result.cardinality": len(result.rows)}, ) @@ -409,25 +461,22 @@ async def _query(self, context: 'UserContext', request: QueryRequest) -> QueryRe request = request.with_query(request.query) self._sync_generated_schema(context) request.query.prepare_for_list() - execution_query, retained_order, retained_empty = await self._prepare_id_set_page(context, request.query) + execution_query, retained_order, retained_empty = await self._prepare_id_set_page(context, request) if retained_empty: now = datetime.now() provider = self._backend_name() metadata = ExecutionMetadata( backend=provider, operation=DataServiceOperation.Query, started_at=now, ended_at=now, result_count=0, - trace_chain=canonical_sql_trace_path( - [TraceNode(kind='comment', name=request.origin_entity, comment=request._comment), - *request.trace_chain], provider, 'select'), + trace_chain=[], # Logical empty result; no provider statement executed. comment=request._comment, purpose=request._purpose, + request_trace_chain=request.business_trace_chain, ) - if context is not None: - context.record_metadata_log(metadata) return QueryResult(rows=[], metadata=metadata) source = getattr(request, '_log_intent_source', None) - request = (_QueryWithLogIntent(execution_query, request.trace_chain, request._comment, - request._purpose, source, request.origin_entity) if source is not None else - request.with_query(execution_query)) + request = request.with_query(execution_query) + if source is not None: + request._log_intent_source = source entity_desc = self.schema_provider.get_entity(request.query.entity) if not entity_desc and context: entities = context.get_resource("entities") @@ -451,22 +500,25 @@ async def _query(self, context: 'UserContext', request: QueryRequest) -> QueryRe start = datetime.now() try: telemetry = context.runtime_telemetry() if context is not None else None - rows = await observe_runtime_operation( - telemetry, - RuntimeOperation("provider", f"{self.dialect.kind()}.query", { + statement = start_runtime_operation(telemetry, RuntimeOperation("provider", f"{self.dialect.kind()}.query", { "teaql.provider.kind": str(self.dialect.kind()), "teaql.provider.operation": "query", - }), - lambda: self.transport.fetch_all_sql(compiled), - ) + "teaql.provider.execution": "statement", + })) + rows = await self.transport.fetch_all_sql(compiled) + statement.success() except BaseException as e: + if 'statement' in locals(): statement.failure(e) self._record_statement(context, request, compiled, start, DataServiceOperation.Query, - 'cancelled' if isinstance(e, asyncio.CancelledError) else 'failure') + 'cancelled' if isinstance(e, asyncio.CancelledError) else 'failure', + correlation=getattr(locals().get('statement'), 'correlation', None)) if isinstance(e, Exception): raise TransportError(e) from e raise metadata = self._record_statement(context, request, compiled, start, - DataServiceOperation.Query, 'success', result_count=len(rows)) + DataServiceOperation.Query, 'success', result_count=len(rows), correlation=statement.correlation) + + self._normalize_loaded_dates(rows, entity_desc, request.query) if assembly is not None: assembly.capture(rows) @@ -491,6 +543,8 @@ async def _query_facets(self, context, request, entity_desc): from teaql.core.list import SmartList facets = {} for facet in getattr(request.query, 'facets', []): + operation = request.derive_operation(request.query, facet.operation or QueryOperation( + 'facet', alias=facet.name, relation=facet.relation_name)) relation = entity_desc.relation_by_name(facet.relation_name) # Legacy numeric partitions still work, but only model metadata can # identify a relationship or add a logical traversal to the trace. @@ -499,6 +553,9 @@ async def _query_facets(self, context, request, entity_desc): if relation and relation.target_entity != facet.query.entity: raise CompileError(SqlCompileError( f'facet target differs from relation: {request.query.entity}.{facet.relation_name}')) + count_aliases = [aggregate.alias for aggregate in facet.query.aggregates + if aggregate.function == AggregateFunction.Count] + sql_count_aliases = count_aliases or ['__teaql_facet_count'] membership_query = deepcopy(request.query) membership_query.facets = [] membership_query.relations = [] @@ -512,25 +569,44 @@ async def _query_facets(self, context, request, entity_desc): membership_query.partition_by = None membership_query.projection = [] membership_query.expr_projection = [] - membership_query.aggregates = [Aggregate( - AggregateFunction.Count, "id", "__teaql_facet_count")] + membership_query.aggregates = [Aggregate(AggregateFunction.Count, 'id', alias) + for alias in sql_count_aliases] membership_query.group_by_items = [member_field] - membership_result = await self._query(context, request.with_query(membership_query)) + membership_result = await self._query(context, operation.derive_helper( + membership_query, 'aggregate ' + facet.relation_name)) member_property = entity_desc.property_by_name(member_field) member_column = member_property.column_name_val if member_property else member_field counts = { - str(row[member_column]): int(row["__teaql_facet_count"]) + str(row[member_column]): int(row[sql_count_aliases[0]]) for row in membership_result.rows if row.get(member_column) is not None } nested_query = deepcopy(facet.query) - count_aliases = [ - aggregate.alias for aggregate in nested_query.aggregates - if aggregate.function == AggregateFunction.Count - ] nested_query.aggregates = [] nested_query.group_by_items = [] + for aggregate in nested_query.relation_aggregates: + if aggregate.query.entity != request.query.entity: + continue + # Reverse sums inherit complete source membership, not the + # limited page or every row referencing this facet value. + membership = deepcopy(request.query) + membership.projection = ['id'] + membership.expr_projection = [] + membership.raw_projections = [] + membership.aggregates = [] + membership.group_by_items = [] + membership.relations = [] + membership.relation_aggregates = [] + membership.facets = [] + membership.child_enhancements = [] + membership.object_group_bys = [] + membership.dynamic_properties = [] + membership.order_by_items = [] + membership.slice = None + membership.partition_by = None + aggregate.query.and_filter(SubQueryExpr(ColumnExpr('id'), BinaryOp.In, + entity_desc, membership)) if not facet.include_all_facets: # The target selection (and any Facets nested inside it) must # see only matching candidates before pagination/recursion. @@ -542,11 +618,7 @@ async def _query_facets(self, context, request, entity_desc): ValueExpr(Value.List(member_values))) if member_values else ValueExpr(Value.Bool(False))) nested_query.and_filter(membership) - nested_request = request.with_query(nested_query) - if relation: - nested_request.trace_chain.append(TraceNode( - kind='relation', name=facet.relation_name, - comment=f'{request.query.entity}.{facet.relation_name}')) + nested_request = operation.derive_child(nested_query, facet.relation_name if relation else None) nested_result = await self._query(context, nested_request) facet_rows = [] for row in nested_result.rows: @@ -560,7 +632,8 @@ async def _query_facets(self, context, request, entity_desc): facets[facet.name] = SmartList(facet_rows, facets=nested_result.facets) return facets - async def _prepare_id_set_page(self, context, query: SelectQuery): + async def _prepare_id_set_page(self, context, request: QueryRequest): + query = request.query options = getattr(query, "id_set_pagination", None) if context is None or options is None: if context is not None: @@ -607,8 +680,10 @@ async def _prepare_id_set_page(self, context, query: SelectQuery): id_query.facets = [] id_query.slice = None id_query.limit(options.max_ids + 1) + id_query.hard_limit_value = options.max_ids + 1 id_query.id_set_pagination = None - id_result = await self._query(context, QueryRequest(id_query)) + id_result = await self._query(context, request.derive_helper( + id_query, 'collect ' + query.entity + ' IDs for pagination')) try: ids = array("Q", (int(row["id"]) for row in id_result.rows)) except (KeyError, TypeError, ValueError, OverflowError): @@ -642,8 +717,7 @@ async def _prepare_id_set_page(self, context, query: SelectQuery): ValueExpr(Value.List([Value.from_any(entity_id) for entity_id in page_ids])))) return page, page_ids, False - @staticmethod - def _id_set_query_key(context, query: SelectQuery, namespace: str) -> str: + def _id_set_query_key(self, context, query: SelectQuery, namespace: str) -> str: normalized = deepcopy(query) normalized.slice = None normalized.projection = [] @@ -653,7 +727,7 @@ def _id_set_query_key(context, query: SelectQuery, namespace: str) -> str: normalized.comment_text = None normalized.trace_chain = [] normalized.id_set_pagination = None - scope = (namespace, context.user_identifier(), id(context.get_resource("db")), + scope = (namespace, self._id_set_source_identity, context.user_identifier(), id(context.get_resource("db")), id(context.get_resource("request_policy")), _canonical_id_set_value(context.get_resource("active_root")), _canonical_id_set_value(normalized)) @@ -706,9 +780,6 @@ async def _enhance_relations(self, context, parents: List[Dict[str, Any]], reque provider_policy == "always_probe" and threshold is None or threshold is not None and threshold > 0 and len(parent_ids) <= threshold ) - child_trace = [*request.trace_chain, TraceNode( - kind="relation", name=load.name, - comment=f"{query.entity}.{load.name}")] if not parent_ids: # Missing/NULL local keys have no relation membership. # Do not compile IN [] or accidentally load orphan rows. @@ -724,9 +795,8 @@ async def _enhance_relations(self, context, parents: List[Dict[str, Any]], reque ColumnExpr(relation.foreign_key), BinaryOp.Eq, ValueExpr(Value.from_any(parent_id)))) assembly = _RelationAssembly(relation.foreign_key) - loaded = (await self.query(context, _QueryWithLogIntent( - probe, child_trace, request._comment, request._purpose, intent_source, - request.origin_entity, assembly))).rows + loaded = (await self.query(context, request.derive( + probe, load.name, load.operation).with_log_intent(intent_source, assembly))).rows children.extend(loaded) child_keys.extend(assembly.key(row) for row in loaded) selected_plan = "bounded_probes" @@ -738,9 +808,8 @@ async def _enhance_relations(self, context, parents: List[Dict[str, Any]], reque if limited: child_query.partition_by_field(relation.foreign_key) assembly = _RelationAssembly(relation.foreign_key) - children = (await self.query(context, _QueryWithLogIntent( - child_query, child_trace, request._comment, request._purpose, intent_source, - request.origin_entity, assembly))).rows + children = (await self.query(context, request.derive( + child_query, load.name, load.operation).with_log_intent(intent_source, assembly))).rows child_keys = [assembly.key(row) for row in children] selected_plan = "window" if limited else "batch" probe_count = 0 @@ -758,9 +827,7 @@ async def _enhance_relations(self, context, parents: List[Dict[str, Any]], reque per_parent.and_filter(BinaryExpr( ColumnExpr(relation.foreign_key), BinaryOp.Eq, ValueExpr(Value.from_any(key)))) - facet_request = _QueryWithLogIntent( - per_parent, child_trace, request._comment, request._purpose, - intent_source, request.origin_entity) + facet_request = request.derive(per_parent, load.name, load.operation).with_log_intent(intent_source) child_desc = self.schema_provider.get_entity(relation.target_entity) related = SmartList(related, facets=await self._query_facets( context, facet_request, child_desc)) @@ -816,12 +883,11 @@ async def _enhance_relation_aggregates(self, context, parents: List[Dict[str, An values = Value.List([Value.from_any(value) for value in parent_ids]) child_query.and_filter(BinaryExpr( ColumnExpr(relation.foreign_key), BinaryOp.In, ValueExpr(values))) - child_trace = [*request.trace_chain, TraceNode( - kind="relation", name=aggregate.relation_name, - comment=f"{query.entity}.{aggregate.relation_name}")] - rows = (await self.query(context, _QueryWithLogIntent( - child_query, child_trace, request._comment, request._purpose, intent_source, - request.origin_entity))).rows + operation = aggregate.operation or QueryOperation( + 'relation-count' if child_query.aggregates[0].function == AggregateFunction.Count else 'aggregate', + alias=aggregate.alias, relation=aggregate.relation_name) + child_request = request.derive_aggregate(child_query, aggregate.relation_name, operation) + rows = (await self.query(context, child_request.with_log_intent(intent_source))).rows child_desc = self.schema_provider.get_entity(relation.target_entity) foreign_property = child_desc.property_by_name(relation.foreign_key) if child_desc else None if foreign_property and foreign_property.column_name_val != relation.foreign_key: @@ -1010,23 +1076,23 @@ async def _mutate(self, context: 'UserContext', request: MutationRequest, privac 'delete': DataServiceOperation.Delete, 'recover': DataServiceOperation.Recover}[op] try: telemetry = context.runtime_telemetry() if context is not None else None - affected_rows, last_insert_id = await observe_runtime_operation( - telemetry, - RuntimeOperation("provider", f"{self.dialect.kind()}.mutation", { + statement = start_runtime_operation(telemetry, RuntimeOperation("provider", f"{self.dialect.kind()}.mutation", { "teaql.provider.kind": str(self.dialect.kind()), "teaql.provider.operation": op, - }), - lambda: self.transport.execute_sql(compiled), - ) + "teaql.provider.execution": "statement", + })) + affected_rows, last_insert_id = await self.transport.execute_sql(compiled) + statement.success() except BaseException as e: + if 'statement' in locals(): statement.failure(e) self._record_statement(context, request, compiled, start, operation, 'cancelled' if isinstance(e, asyncio.CancelledError) else 'failure', - intent_privacy=privacy) + intent_privacy=privacy, correlation=getattr(locals().get('statement'), 'correlation', None)) if isinstance(e, Exception): raise TransportError(e) from e raise metadata = self._record_statement(context, request, compiled, start, operation, - 'success', affected_rows=affected_rows, intent_privacy=privacy) + 'success', affected_rows=affected_rows, intent_privacy=privacy, correlation=statement.correlation) generated_values = {} if op == "insert" and last_insert_id: @@ -1072,18 +1138,23 @@ async def _mutate(self, context: 'UserContext', request: MutationRequest, privac ) read_start = datetime.now() persisted_rows = None + read_scope = start_runtime_operation(telemetry, RuntimeOperation('provider', f'{self.dialect.kind()}.readback', { + 'teaql.provider.kind': str(self.dialect.kind()), 'teaql.provider.operation': 'query', 'teaql.provider.execution': 'statement'})) try: persisted_rows = await self.transport.fetch_all_sql(readback) + read_scope.success() if len(persisted_rows) != 1: raise TransportError(RuntimeError( f"expected one authoritative persisted row for {req_data.entity}, got {len(persisted_rows)}")) + self._normalize_loaded_dates(persisted_rows, entity_desc) except BaseException as error: + read_scope.failure(error) self._record_readback(context, readback, compiled, metadata, read_start, - persisted_rows, error, privacy) + persisted_rows, error, privacy, read_scope.correlation) raise persisted_record = persisted_rows[0] read_metadata = self._record_readback(context, readback, compiled, metadata, - read_start, persisted_rows, None, privacy) + read_start, persisted_rows, None, privacy, read_scope.correlation) metadata = replace(metadata, statements=(metadata, read_metadata)) if affected_rows > 0 and context is not None: @@ -1123,7 +1194,7 @@ async def _mutate(self, context: 'UserContext', request: MutationRequest, privac persisted_record=persisted_record, ) - def _record_readback(self, context, readback, source, write_metadata, started_at, rows, error, privacy=None): + def _record_readback(self, context, readback, source, write_metadata, started_at, rows, error, privacy=None, correlation=None): # A driver returning zero/multiple rows succeeded as SQL; validation of # the authoritative snapshot is a separate business failure. outcome = ('success' if rows is not None else 'cancelled' @@ -1135,6 +1206,7 @@ def _record_readback(self, context, readback, source, write_metadata, started_at parameter_log_policies=readback.parameter_log_policies, sql_origin=readback.sql_origin, affected_rows=None, result_count=len(rows) if rows is not None else None, trace_chain=physical_readback_path(write_metadata.trace_chain)) + metadata._sql_correlation = correlation if context is None: return metadata try: diff --git a/tests/core/test_generated_scalar_search.py b/tests/core/test_generated_scalar_search.py new file mode 100644 index 0000000..7884ba8 --- /dev/null +++ b/tests/core/test_generated_scalar_search.py @@ -0,0 +1,67 @@ +from copy import deepcopy +import pytest +from teaql.core.dynamic_search import merge_generated_scalar_search +from teaql.core.expr import Expr, column, contain, eq, gte, is_null, is_not_null +from teaql.core.query import OrderBy, SelectQuery + +MODELS = { + 'Invoice': {'fields': {'referenceCode': 'string', 'reference_code': 'string', + 'totalAmount': 'decimal', 'active': 'boolean', 'payload': 'unsupported'}, + 'query_fields': {'referenceCode': 'reference_code', 'reference_code': 'reference_code', + 'totalAmount': 'total_amount', 'active': 'active', 'payload': 'payload'}, + 'relations': {'owner': 'Owner'}}, + 'Owner': {'fields': {'name': 'string'}, 'query_fields': {'name': 'name'}, 'relations': {}}, +} + + +def test_generated_search_merges_all_siblings_without_mutating_scope_or_input(): + base = SelectQuery('Invoice').filter(Expr.eq('tenant', 1)).limit(4) + base.comment('load scoped invoices').purpose('verify generated search') + base.order_by_items.append(OrderBy.asc('id')) + before = deepcopy(base) + source = {'filter': {'referenceCode': {'$contains': 'KEEP'}, 'totalAmount': {'$gte': '50.00'}, + 'active': True, 'removed': 'PRIVATE'}, + 'orderBy': [{'field': 'referenceCode', 'direction': 'desc'}]} + recorded = [] + query, warnings = merge_generated_scalar_search(base, source, MODELS, recorded.append) + source['filter']['referenceCode']['$contains'] = 'changed' + assert base == before + assert query.slice == base.slice + assert query.comment_text == base.comment_text and query.purpose_text == base.purpose_text + assert query.order_by_items == [OrderBy.asc('id'), OrderBy.desc('reference_code')] + assert 'reference_code' in repr(query.filter_expr) and 'total_amount' in repr(query.filter_expr) + expected = Expr.new_and(Expr.new_and(Expr.new_and(base.filter_expr, + contain('reference_code', 'KEEP')), gte('total_amount', '50.00')), eq('active', True)) + assert query.filter_expr == expected + assert len(recorded) == len(warnings) == 1 and 'PRIVATE' not in repr(recorded) + + +def test_known_relation_path_fails_atomically_without_dropping_it_or_emitting_partial_warnings(): + base = SelectQuery('Invoice').filter(eq('tenant', 1)) + before = deepcopy(base) + warnings = [] + with pytest.raises(ValueError, match='trusted relation binding'): + merge_generated_scalar_search(base, {'filter': {'removed': 'secret', 'owner.name': 'Ada'}}, MODELS, warnings.append) + assert base == before and warnings == [] + + +@pytest.mark.parametrize('operator, expected', [('$eq', is_null(column('reference_code'))), + ('$ne', is_not_null(column('reference_code')))]) +def test_null_uses_native_sql_null_predicates(operator, expected): + query, _ = merge_generated_scalar_search(SelectQuery('Invoice'), + {'filter': {'referenceCode': {operator: None}}}, MODELS) + assert query.filter_expr == expected + + +@pytest.mark.parametrize('source', [ + {'filter': {'active': 'true'}}, {'filter': {'totalAmount': {'$contains': '1'}}}, + {'filter': {'referenceCode': {'$rawSQL': 'true'}}}, {'context': {'actor': 'admin'}}, + {'filter': {'payload': None}}, {'orderBy': [{'field': 'payload', 'direction': 'asc'}]}, + {'filter': {'referenceCode': {'$contains': None}}}, {'filter': {'totalAmount': {'$gt': None}}}, +]) +def test_invalid_known_values_and_trusted_controls_stay_fatal(source): + base = SelectQuery('Invoice').filter(eq('tenant', 1)) + before = deepcopy(base) + with pytest.raises(ValueError): + merge_generated_scalar_search(base, source, MODELS) + assert base == before diff --git a/tests/core/test_projection_properties.py b/tests/core/test_projection_properties.py new file mode 100644 index 0000000..0174535 --- /dev/null +++ b/tests/core/test_projection_properties.py @@ -0,0 +1,13 @@ +from teaql.core.projection_properties import ProjectionProperties + + +def test_computed_properties_do_not_shadow_model_or_methods_and_missing_returns_null(): + values = {'id': 1, 'itemCount': 7, 'save': {'unsafe': ['original']}} + properties = ProjectionProperties(values, ['id']) + assert properties.get('id') is None + assert properties.get('missing') is None + assert properties.get('itemCount') == 7 + values['save']['unsafe'].append('changed') + extracted = properties.get('save') + extracted['unsafe'].append('another change') + assert properties.get('save') == {'unsafe': ['original']} diff --git a/tests/core/test_query_request_tree.py b/tests/core/test_query_request_tree.py new file mode 100644 index 0000000..bb350b6 --- /dev/null +++ b/tests/core/test_query_request_tree.py @@ -0,0 +1,162 @@ +from copy import deepcopy +from dataclasses import asdict, FrozenInstanceError +import gc +import weakref + +import pytest + +from teaql.core import QueryOperation, RequestIntentError, SelectQuery +from teaql.data_service import QueryRequest + + +def root(comment='load schools', purpose='render school list'): + return QueryRequest(SelectQuery('School').comment(comment).purpose(purpose)) + + +def details(request): + return [node.comment for node in request.business_trace_chain] + + +def test_explicit_child_and_native_operation_have_distinct_parents(): + parent = root() + child = parent.derive(SelectQuery('SchoolType').comment('load visible types'), 'school_type', + QueryOperation('relation-select', 'select_school_type_with', relation='school_type')) + assert child.root_request is parent + assert child.parent_request.parent_request is parent + assert child.local_comment == 'load visible types' + assert details(child) == ['render school list', 'load schools', 'select_school_type_with(...)', + 'load visible types'] + assert child._comment == parent._comment + with pytest.raises(AttributeError): + child.parent_request = child + with pytest.raises(AttributeError): + child.operation = QueryOperation('facet') # read-only property, not a caller-owned field + with pytest.raises(FrozenInstanceError): + child.parent_request.operation.method = 'forged' + + +def test_missing_child_gets_local_default_but_missing_root_is_rejected(): + child = root().derive(SelectQuery('SchoolType'), 'school_type') + assert child.local_comment == 'load SchoolType' + with pytest.raises(RequestIntentError, match='REQUEST_COMMENT_REQUIRED'): + QueryRequest(SelectQuery('School').purpose('render schools')) + + +@pytest.mark.parametrize('comment', ['', ' \n ', '\u0085']) +def test_explicit_blank_child_is_not_replaced_with_a_default(comment): + with pytest.raises(RequestIntentError, match='REQUEST_COMMENT_REQUIRED'): + root().derive(SelectQuery('SchoolType').comment(comment), 'school_type') + + +def test_clones_and_replacements_keep_private_ownership_and_the_physical_relation(): + parent = root() + child = parent.derive(SelectQuery('SchoolType').comment('load labels'), 'school_type') + updated = child.with_query(deepcopy(child.query).limit(3)) + captured = QueryRequest(deepcopy(updated.query)) + assert captured.root_request is parent + assert captured.origin_entity == 'School' + assert captured.local_comment == 'load labels' + assert [(node.kind, node.name, node.comment) for node in captured.trace_chain] == [ + ('relation', 'school_type', 'School.school_type')] + assert '_owner' not in asdict(captured.query) + assert 'parent_request' not in asdict(captured.query) + + +def test_unrelated_invocation_cannot_replace_owned_query_even_with_identical_intent(): + first = QueryRequest(SelectQuery('School').comment('same comment').purpose('same purpose')) + second = QueryRequest(SelectQuery('School').comment('same comment').purpose('same purpose')) + with pytest.raises(ValueError, match='QUERY_SCOPE_OWNER_MISMATCH'): + first.with_query(deepcopy(second.query)) + own_copy = first.with_query(deepcopy(first.query)) + own_copy.with_query(deepcopy(first.query)) + with pytest.raises(ValueError, match='QUERY_SCOPE_OWNER_MISMATCH'): + own_copy.with_query(deepcopy(second.query)) + + +def test_reusing_a_template_keeps_each_parent_and_does_not_mutate_the_template(): + template = SelectQuery('SchoolType').comment('load labels') + first, second = root('first page'), root('second page') + a, b = first.derive(template, 'school_type'), second.derive(template, 'school_type') + assert a.root_request is first and b.root_request is second + assert a.query is not b.query + assert template.comment_text == 'load labels' + assert not hasattr(template, 'purpose_text') + assert 'second page' not in details(a) + + +def test_named_count_uses_centralized_native_escaping_and_no_physical_relation(): + parent = root() + alias = "total'\\schools\n" + operation = QueryOperation('named-count', 'count_as', alias) + count = parent.for_count(alias, operation) + assert count.local_comment == "count_as('total\\'\\\\schools\\n')" + assert count.parent_request is parent + assert count.query.aggregates[0].alias == alias + assert count.trace_chain == [] + assert parent.query.aggregates == [] + + +def test_facet_branches_share_one_operation_but_not_sibling_comments(): + parent = root() + operation = parent.derive_operation(parent.query, + QueryOperation('facet', 'facet_by_school_type_as', 'types', 'school_type')) + aggregate = operation.derive_helper(SelectQuery('School'), 'aggregate school_type') + values = operation.derive_child(SelectQuery('SchoolType').comment('load labels'), 'school_type') + assert aggregate.parent_request is values.parent_request is operation + assert details(aggregate)[-1] == 'aggregate school_type' + assert details(values)[-1] == 'load labels' + assert details(aggregate)[:3] == details(values)[:3] + assert not aggregate.trace_chain + + +def test_prefixes_are_text_and_public_query_fields_cannot_forge_business_ancestry(): + query = SelectQuery('School').comment('what: load schools').purpose('why: render schools') + query.parent_request = root('forged parent') + query.business_trace_chain = ('forged chain',) + captured = QueryRequest(query) + assert details(captured) == ['why: render schools', 'what: load schools'] + assert captured.parent_request is None + with pytest.raises(ValueError, match='QUERY_REQUEST_OWNERSHIP_INVALID'): + QueryRequest(query, _binding=(object(), {})) + + +def test_identity_registry_does_not_keep_a_discarded_root_query_alive(): + request = root() + reference = weakref.ref(request.query) + del request + gc.collect() + assert reference() is None + + +def test_invalid_operations_and_alias_mismatch_use_value_free_diagnostics(): + with pytest.raises(ValueError, match='QUERY_OPERATION_INVALID') as error: + QueryOperation('facet', "ILLEGAL-PRIVATE-CANARY\n") + assert 'ILLEGAL-PRIVATE-CANARY' not in str(error.value) + with pytest.raises(ValueError, match='QUERY_OPERATION_ALIAS_MISMATCH'): + root().for_count('a', QueryOperation('named-count', 'count_as', 'b')) + + +def test_facet_reverse_aggregate_shares_operation_parent_and_keeps_physical_relations(): + parent = root() + facet = parent.derive_operation(SelectQuery('School'), + QueryOperation('facet', 'facet_by_school_type_as', 'types', 'school_type')) + values = facet.derive_child(SelectQuery('SchoolType').comment('load visible types'), 'school_type') + operation = QueryOperation('aggregate', 'sum_capacity_of_schools_as', 'capacity', 'school_list') + aggregate = values.derive_aggregate(SelectQuery('School'), 'school_list', operation) + assert aggregate.parent_request is facet + assert aggregate.root_request is parent + assert details(aggregate) == ['render school list', 'load schools', "facet_by_school_type_as('types')", 'aggregate school_type'] + assert [node.name for node in aggregate.trace_chain if node.kind == 'relation'] == ['school_type', 'school_list'] + explicit = values.derive_aggregate(SelectQuery('School').comment('sum visible capacity'), 'school_list', operation) + assert details(explicit) == [*details(aggregate), 'sum visible capacity'] + with pytest.raises(RequestIntentError): + values.derive_aggregate(SelectQuery('School').comment(' '), 'school_list', operation) + assert values.local_comment == 'load visible types' + + +def test_relation_count_has_one_operation_request_without_inventing_a_value_load(): + parent = root() + child = parent.derive_aggregate(SelectQuery('SchoolItem'), 'school_item_list', + QueryOperation('relation-count', 'count_school_items_as', 'itemCount', 'school_item_list')) + assert child.parent_request is parent + assert details(child) == ['render school list', 'load schools', "count_school_items_as('itemCount')"] diff --git a/tests/provider/sqlite/test_loaded_dates.py b/tests/provider/sqlite/test_loaded_dates.py new file mode 100644 index 0000000..60dc4b2 --- /dev/null +++ b/tests/provider/sqlite/test_loaded_dates.py @@ -0,0 +1,45 @@ +from datetime import date + +import pytest + +from teaql.core.meta import EntityDescriptor, PropertyDescriptor +from teaql.core.mutation import InsertCommand, MutationRequest, UpdateCommand +from teaql.core.query import SelectQuery +from teaql.core.value import DataType +from teaql.data_service import QueryRequest +from teaql.provider.sqlite import SimpleSchemaProvider, create_sqlite_service +from teaql.runtime import RuntimeModule + + +@pytest.mark.asyncio +async def test_declared_date_stays_native_across_save_query_and_stream(tmp_path): + descriptor = EntityDescriptor('Visit').table_name('visit_data') + descriptor.property(PropertyDescriptor('id', DataType.I64).is_id()) + descriptor.property(PropertyDescriptor('version', DataType.I64).is_version()) + descriptor.property(PropertyDescriptor('visit_date', DataType.Date)) + descriptor.property(PropertyDescriptor('date_label', DataType.Text)) + schema = SimpleSchemaProvider() + schema.register_entity(descriptor) + service = create_sqlite_service(str(tmp_path / 'dates.sqlite'), schema) + context = RuntimeModule.new().entity(descriptor).into_context().with_schema_provider(service) + await context.ensure_schema() + inserted = await service.mutate(context, MutationRequest( + InsertCommand.new('Visit').value('visit_date', date(2024, 2, 29)).value('date_label', '2024-02-29'), + comment='create dated visit')) + assert inserted.persisted_record['visit_date'] == date(2024, 2, 29) + query = SelectQuery('Visit').comment('load dated visit').purpose('prove declared types') + rows = (await service.query(context, QueryRequest(query))).rows + assert rows[0]['visit_date'] == date(2024, 2, 29) + assert rows[0]['date_label'] == '2024-02-29' + chunks = [chunk async for chunk in service.query_stream(context, QueryRequest(query), 1)] + assert chunks[0].rows[0]['visit_date'] == date(2024, 2, 29) + sparse = (await service.query(context, QueryRequest( + SelectQuery('Visit').project('id').comment('load visit identity').purpose('preserve sparse projection')))).rows + assert 'visit_date' not in sparse[0] + aggregate = (await service.query(context, QueryRequest( + SelectQuery('Visit').count_field('id', 'visit_date').comment('count visits').purpose('retain aggregate type')))).rows + assert aggregate[0]['visit_date'] == 1 + updated = await service.mutate(context, MutationRequest( + UpdateCommand.new('Visit', rows[0]['id']).expected_version(rows[0]['version']).value('visit_date', None), comment='clear optional date')) + assert updated.persisted_record['visit_date'] is None + await service.close() diff --git a/tests/provider/sqlite/test_trace_chain.py b/tests/provider/sqlite/test_trace_chain.py index 8190b61..6ea5278 100644 --- a/tests/provider/sqlite/test_trace_chain.py +++ b/tests/provider/sqlite/test_trace_chain.py @@ -113,6 +113,417 @@ def three_levels(): .relation_query('children', SelectQuery('Shipment').project('id').limit(1))))) +@pytest.mark.asyncio +@pytest.mark.parametrize('logging', [False, True]) +@pytest.mark.parametrize('field', ['comment', 'purpose']) +async def test_annotated_children_cannot_authorize_missing_root_intent(tmp_path, logging, field): + from teaql.core import RequestIntentError + from teaql.runtime.context import SqlLogOptions + context, service, transport, entries = await fixture(tmp_path) + if not logging: + context.with_sql_log_options(SqlLogOptions.disabled()) + policies = [] + context.with_request_policy(lambda query: policies.append(query.entity) or query) + child = SelectQuery('Payment').project('id').comment('load payment details').purpose('render payment details') + query = SelectQuery('CustomerOrder').project('id').comment('load orders').purpose('render orders') + query.relation_query('children', child) + setattr(query, field + '_text', None) + assert child.comment_text == 'load payment details' + assert child.purpose_text == 'render payment details' + with pytest.raises(RequestIntentError) as caught: + await service.query(context, context.prepare_query_request(QueryRequest(query))) + assert caught.value.field == field + assert policies == [] + assert transport.reads == transport.streams == transport.writes == [] + assert transport.begins == 0 + assert entries == context.sql_logs() == [] + query.comment('load validation control').purpose('verify live SQLite') + result = await service.query(context, context.prepare_query_request(QueryRequest(query))) + assert len(result.rows) == 1 and len(result.rows[0]['children']) == 1 + assert policies and len(transport.reads) == 2 + assert bool(entries) == logging + print(f'RTC_COM_01_PASS logging={str(logging).lower()} field={field} invalid_provider=0 invalid_policy=0 annotated_child=true live_control=true') + + +@pytest.mark.asyncio +@pytest.mark.parametrize('logging', [False, True]) +async def test_request_tree_retains_operation_explicit_and_default_comments_in_native_sqlite(tmp_path, logging): + from teaql.core import QueryOperation + from teaql.runtime.context import SqlLogOptions + context, service, transport, entries = await fixture(tmp_path) + if not logging: + context.with_sql_log_options(SqlLogOptions.disabled()) + query = three_levels().comment('load order graph').purpose('render graph') + query.relations[0].operation = QueryOperation('relation-select', 'select_children_with', relation='children') + query.relations[0].query.comment('load payment details') + result = await service.query(context, QueryRequest(query)) + assert result.rows[0]['children'][0]['children'][0]['children'][0]['id'] == 1 + assert len(transport.reads) == 4 + assert len(entries) == (4 if logging else 0) + if logging: + assert [node.comment for node in entries[1].trace_chain] == [ + 'render graph', 'load order graph', 'select_children_with(...)', 'load payment details'] + assert [node.comment for node in entries[2].trace_chain] == [ + 'render graph', 'load order graph', 'select_children_with(...)', 'load payment details', + 'framework: relation-select children', 'load PaymentAttempt'] + assert all(entry.comment == 'load order graph' and entry.purpose == 'render graph' for entry in entries) + assert all(node.kind not in ('purpose', 'comment') for entry in entries for node in entry.trace_path) + assert query.relations[0].query.relations[0].query.comment_text is None + print(f'RTC_COM_02_PASS logging={str(logging).lower()} explicit=both') + + +@pytest.mark.asyncio +@pytest.mark.parametrize('logging', [False, True]) +@pytest.mark.parametrize('blank', ['', ' \t', '\u0085']) +async def test_explicitly_blank_relation_comment_rejects_before_child_sqlite(tmp_path, logging, blank): + from teaql.core import RequestIntentError + from teaql.runtime.context import SqlLogOptions + context, service, transport, entries = await fixture(tmp_path) + if not logging: + context.with_sql_log_options(SqlLogOptions.disabled()) + query = three_levels().comment('load order graph').purpose('validate nested input') + query.relations[0].query.comment(blank) + with pytest.raises(RequestIntentError) as caught: + await service.query(context, QueryRequest(query)) + assert caught.value.field == 'comment' + assert len(transport.reads) <= 1 + assert all('payment_data' not in compiled.sql for compiled in transport.reads) + assert query.relations[0].query.comment_text == blank + transport.reads.clear() + valid = three_levels().comment('load validation control').purpose('verify real child provider') + result = await service.query(context, QueryRequest(valid)) + assert result.rows[0]['children'][0]['children'][0]['children'][0]['id'] == 1 + assert len(transport.reads) == 4 + print(f'RTC_COM_04_RELATION_PASS logging={str(logging).lower()} invalid_child_provider=0') + + +@pytest.mark.asyncio +@pytest.mark.parametrize('logging', [False, True]) +@pytest.mark.parametrize('blank', ['', ' \t', '\u0085']) +async def test_explicitly_blank_facet_value_comment_rejects_before_value_sqlite(tmp_path, logging, blank): + from teaql.core import RequestIntentError + from teaql.runtime.context import SqlLogOptions + context, service, transport, entries = await fixture(tmp_path) + if not logging: + context.with_sql_log_options(SqlLogOptions.disabled()) + child = SelectQuery('Payment').count('orderCount').comment(blank) + outer = SelectQuery('CustomerOrder').comment('load orders').purpose('validate facet input') + outer.facet_by('paymentCounts', 'children', child, True) + with pytest.raises(RequestIntentError) as caught: + await service.query(context, QueryRequest(outer)) + assert caught.value.field == 'comment' + assert all('payment_data' not in compiled.sql for compiled in transport.reads) + assert child.comment_text == blank + print(f'RTC_COM_04_FACET_PASS logging={str(logging).lower()} invalid_value_provider=0') + + +@pytest.mark.asyncio +@pytest.mark.parametrize('logging', [False, True]) +async def test_compound_aggregates_emit_one_physical_statement(tmp_path, logging): + context, service, transport, entries = await fixture(tmp_path) + if not logging: + context.disable_select_sql_log() + query = SelectQuery('CustomerOrder').count('orderCount').sum('id', 'idTotal').comment('calculate order totals').purpose('render totals') + result = await service.query(context, QueryRequest(query)) + assert result.rows == [{'orderCount': 1, 'idTotal': 1}] + assert len(transport.reads) == 1 + assert 'COUNT(' in transport.reads[0].sql.upper() and 'SUM(' in transport.reads[0].sql.upper() + assert len(entries) == int(logging) + if logging: + assert [node.comment for node in entries[0].trace_chain] == ['render totals', 'calculate order totals'] + print(f'RTC_COM_07_AGGREGATE_PASS logging={str(logging).lower()} expressions=2 statements=1') + + +@pytest.mark.asyncio +@pytest.mark.parametrize('logging', [False, True]) +async def test_cached_exact_total_emits_only_page_sql_on_hit(tmp_path, logging): + context, service, transport, entries = await fixture(tmp_path) + if not logging: + context.disable_select_sql_log() + query = (SelectQuery('CustomerOrder').project('id').order_desc('id').limit(1) + .optimize_pagination_with_id_set().comment('load order page').purpose('render order page')) + first = await service.query(context, QueryRequest(query)) + assert [row['id'] for row in first.rows] == [1] + assert context.id_set_plan() == 'ID_SET_BUILD' and context.id_set_count() == (1, 'EXACT') + assert len(transport.reads) == 2 + transport.reads.clear(); entries.clear() + second = await service.query(context, QueryRequest(query)) + assert [row['id'] for row in second.rows] == [1] + assert context.id_set_plan() == 'ID_SET_HIT' and context.id_set_count() == (1, 'EXACT') + assert len(transport.reads) == 1 and 'COUNT(' not in transport.reads[0].sql.upper() + assert len(entries) == int(logging) + if logging: + assert [node.comment for node in entries[0].trace_chain] == ['render order page', 'load order page'] + print(f'RTC_COM_07_CACHE_PASS logging={str(logging).lower()} accuracy=EXACT hit_statements=1') + + +@pytest.mark.asyncio +@pytest.mark.parametrize('logging', [False, True]) +async def test_named_count_preserves_result_alias_sql_and_business_parent_without_fake_relation(tmp_path, logging): + from teaql.core import QueryOperation + from teaql.runtime.context import SqlLogOptions + context, service, transport, entries = await fixture(tmp_path) + if not logging: + context.with_sql_log_options(SqlLogOptions.disabled()) + original = QueryRequest(SelectQuery('CustomerOrder').limit(1).comment('load orders').purpose('render page')) + result = await service.query(context, original.for_count('totalResults', + QueryOperation('named-count', 'count_as', 'totalResults'))) + assert result.rows == [{'totalResults': 1}] + assert len(transport.reads) == 1 and len(entries) == int(logging) + assert 'totalResults' in transport.reads[0].sql + if logging: + assert [node.comment for node in entries[0].trace_chain] == [ + 'render page', 'load orders', "count_as('totalResults')"] + assert not any(node.kind == 'relation' for node in entries[0].trace_path) + transport.reads.clear(); entries.clear() + default = original.for_count() + assert default.local_comment == 'framework: named-count' + result = await service.query(context, default) + assert result.rows == [{'__teaql_total': 1}] + assert len(transport.reads) == 1 and len(entries) == int(logging) + assert '__teaql_total' in transport.reads[0].sql + if logging: + assert [node.comment for node in entries[0].trace_chain] == [ + 'render page', 'load orders', 'framework: named-count'] + assert not any(node.kind == 'relation' for node in entries[0].trace_path) + print(f'RTC_COM_05_PASS logging={str(logging).lower()} named=true unnamed=true') + + +@pytest.mark.asyncio +async def test_generated_relation_count_is_one_request_with_real_relation_provenance(tmp_path): + from teaql.core import QueryOperation + from teaql.core.query import RelationAggregate + context, service, transport, entries = await fixture(tmp_path) + query = SelectQuery('CustomerOrder').project('id').comment('load orders').purpose('render page') + query.relation_aggregates.append(RelationAggregate('children', 'itemCount', SelectQuery('Payment').count('itemCount'), + True, QueryOperation('relation-count', 'count_children_as', 'itemCount', 'children'))) + result = await service.query(context, QueryRequest(query)) + assert result.rows[0]['itemCount'] == 1 + assert len(entries) == len(transport.reads) == 2 + assert [node.comment for node in entries[1].trace_chain] == [ + 'render page', 'load orders', "count_children_as('itemCount')"] + assert [node.comment for node in entries[1].trace_path if node.kind == 'relation'] == [ + 'CustomerOrder.children'] + + +@pytest.mark.asyncio +async def test_facet_sql_branches_share_native_operation_and_keep_explicit_child_comment(tmp_path): + from teaql.core import QueryOperation + context, service, transport, entries = await fixture(tmp_path) + nested = SelectQuery('Payment').count('orderCount').comment('load payment facets') + outer = SelectQuery('CustomerOrder').comment('load orders').purpose('render page') + outer.facet_by('paymentCounts', 'children', nested, True, + QueryOperation('facet', 'facet_by_children_as', 'paymentCounts', 'children')) + result = await service.query(context, QueryRequest(outer)) + assert result.facets['paymentCounts'][0]['orderCount'] == 1 + assert len(entries) == len(transport.reads) == 3 + prefix = ['render page', 'load orders', "facet_by_children_as('paymentCounts')"] + assert [node.comment for node in entries[1].trace_chain] == [*prefix, 'aggregate children'] + assert [node.comment for node in entries[2].trace_chain] == [*prefix, 'load payment facets'] + + +@pytest.mark.asyncio +async def test_new_business_comments_follow_existing_mask_projection(tmp_path): + from teaql.core import QueryOperation + context, service, transport, entries = await fixture(tmp_path) + query = three_levels().filter(Expr.eq('name', 'CustomerOrder')) + query.comment('load CustomerOrder graph').purpose('inspect CustomerOrder') + query.relations[0].query.comment('load CustomerOrder payment') + query.relations[0].operation = QueryOperation('relation-select', 'select_children_with', relation='children') + await service.query(context, QueryRequest(query)) + assert len(entries) == 4 + for entry in entries: + assert entry.trace_chain + assert 'CustomerOrder' not in '/'.join(node.comment for node in entry.trace_chain) + + +@pytest.mark.asyncio +@pytest.mark.parametrize('logging', [False, True]) +async def test_prefixes_and_escaped_facet_aliases_preserve_bindings_under_privacy_projection(tmp_path, logging): + from copy import deepcopy + from teaql.core import QueryOperation + from teaql.core.mutation import UpdateCommand + context, service, transport, entries = await fixture(tmp_path) + secret = 'TRACE-PRIVATE-CANARY' + service.schema_provider.get_entity('CustomerOrder').audit_mask_fields(['name']) + service.schema_provider.get_entity('Payment').relation(RelationDescriptor('owner', 'CustomerOrder').local('parent_id').foreign('id')) + await service.mutate(context, MutationRequest(UpdateCommand.new('CustomerOrder', 1).expected_version(1).value('name', secret), comment='seed privacy canary')) + if not logging: + context.disable_select_sql_log() + for private_input in (False, True): + entries.clear(); transport.reads.clear() + alias = 'choices"\\\n' + (secret if private_input else '') + native_comment = "facet_by_owner_as('choices\"\\\\\\n" + (secret if private_input else '') + "')" + comment = f'what: load {secret}' if private_input else 'what: load orders' + purpose = f'why: inspect {secret}' if private_input else 'why: inspect orders' + child = SelectQuery('CustomerOrder').project('id', 'name').limit(10).count('paymentCount').comment( + f'load {secret} facets' if private_input else 'load order facets') + query = SelectQuery('Payment').project('id', 'parent_id').limit(10).comment(comment).purpose(purpose) + if private_input: + child.filter(Expr.eq('name', secret)) + operation = QueryOperation('facet', 'facet_by_owner_as', alias, 'owner') + assert operation.comment() == native_comment + query.facet_by(alias, 'owner', child, False, operation) + before = deepcopy(query) + request = QueryRequest(query) + result = await service.query(context, request) + assert result.rows[0]['id'] == 1 + assert result.facets[alias][0]['name'] == secret + assert result.facets[alias][0]['paymentCount'] == 1 + assert len(transport.reads) == 3 and len(entries) == (3 if logging else 0) + assert request.intent.comment == comment and request.intent.purpose == purpose + assert query == before + if private_input: + assert any(secret in [value.val for value in read.params] for read in transport.reads) + if logging: + safe_comment = 'what: load [REDACTED]' if private_input else comment + safe_purpose = 'why: inspect [REDACTED]' if private_input else purpose + chains = [[node.comment for node in entry.trace_chain] for entry in entries] + assert [safe_purpose, safe_comment] in chains + for local in ('aggregate owner', 'load [REDACTED] facets' if private_input else 'load order facets'): + assert [safe_purpose, safe_comment, native_comment.replace(secret, '[REDACTED]'), local] in chains + if private_input: + assert secret not in repr(entries) + print(f'RTC_COM_09_12_PASS logging={str(logging).lower()} plain=true private=true escaped=true bindings=true') + + +@pytest.mark.asyncio +@pytest.mark.parametrize('logging', [True, False]) +@pytest.mark.parametrize('comment', ['sum visible attempt IDs', '', ' \t', '\u0085']) +async def test_facet_reverse_sum_keeps_explicit_comment_and_rejects_blank(tmp_path, monkeypatch, logging, comment): + from teaql.core import QueryOperation, RequestIntentError + from teaql.core.query import RelationAggregate + context, service, transport, entries = await fixture(tmp_path) + metadata = [] + record = context._record_metadata_log + + def observe(actual, **kwargs): + metadata.append(actual) + return record(actual, **kwargs) + + monkeypatch.setattr(context, '_record_metadata_log', observe) + if not logging: + context.disable_select_sql_log() + service.schema_provider.get_entity('PaymentAttempt').relation( + RelationDescriptor('owner', 'Payment').local('parent_id').foreign('id')) + sum_query = SelectQuery('PaymentAttempt').sum('id', 'attemptTotal').comment(comment) + values = SelectQuery('Payment').project('id').limit(10).count('attemptCount').comment('load payment facets') + values.relation_aggregates.append(RelationAggregate('children', 'attemptTotal', sum_query, True, + QueryOperation('aggregate', 'sum_id_of_children_as', 'attemptTotal', 'children'))) + outer = (SelectQuery('PaymentAttempt').project('id', 'parent_id').limit(10) + .comment('load attempts').purpose('render attempts') + .facet_by('payments', 'owner', values, True, + QueryOperation('facet', 'facet_by_owner_as', 'payments', 'owner'))) + if comment != 'sum visible attempt IDs': + with pytest.raises(RequestIntentError): + await service.query(context, QueryRequest(outer)) + assert all(len(entry.trace_chain) <= 4 for entry in entries) + assert all('SUM(' not in statement.sql.upper() for statement in transport.reads) + return + result = await service.query(context, QueryRequest(outer)) + assert result.facets['payments'][0]['attemptTotal'] == 1 + aggregate = [entry for entry in metadata if entry.request_trace_chain[-1].comment == comment] + assert len(aggregate) == 1 + assert [node.comment for node in aggregate[0].request_trace_chain] == ['render attempts', 'load attempts', + "facet_by_owner_as('payments')", 'aggregate owner', comment] + assert len(entries) == (len(transport.reads) if logging else 0) + assert len(metadata) == len(transport.reads) + assert sum_query.comment_text == comment + + +@pytest.mark.asyncio +@pytest.mark.parametrize('logging', [True, False]) +async def test_id_set_empty_cache_keeps_helper_ancestry_and_emits_no_fictitious_sql(tmp_path, monkeypatch, logging): + context, service, transport, entries = await fixture(tmp_path) + metadata = [] + record = context._record_metadata_log + + def observe(actual, **kwargs): + metadata.append(actual) + return record(actual, **kwargs) + + monkeypatch.setattr(context, '_record_metadata_log', observe) + requests = [] + execute = service._query + + async def observe_request(caller, request): + requests.append(request) + return await execute(caller, request) + + monkeypatch.setattr(service, '_query', observe_request) + if not logging: + context.disable_select_sql_log() + query = (SelectQuery('CustomerOrder').filter(Expr.eq('id', 999)).limit(2) + .optimize_pagination_with_id_set() + .comment('load empty page').purpose('render filtered page')) + for _ in range(2): + result = await service.query(context, QueryRequest(query)) + assert result.rows == [] + assert result.metadata.parameterized_sql == '' + assert context.id_set_count() == (0, 'EXACT') + assert len(transport.reads) == 1 + assert len(metadata) == 1 + assert len(entries) == (1 if logging else 0) + helpers = [request for request in requests if request.local_comment == 'collect CustomerOrder IDs for pagination'] + assert len(helpers) == 1 + assert [node.comment for node in helpers[0].business_trace_chain] == [ + 'render filtered page', 'load empty page', 'collect CustomerOrder IDs for pagination'] + assert metadata[0].execution_outcome == 'success' + + +@pytest.mark.asyncio +@pytest.mark.parametrize('logging', [True, False]) +async def test_actual_sqlite_stream_retains_a_derived_count_request_parent(tmp_path, monkeypatch, logging): + from teaql.core import QueryOperation + context, service, transport, entries = await fixture(tmp_path) + metadata = [] + record = context._record_metadata_log + + def observe(actual, **kwargs): + metadata.append(actual) + return record(actual, **kwargs) + + monkeypatch.setattr(context, '_record_metadata_log', observe) + if not logging: + context.disable_select_sql_log() + root = QueryRequest(SelectQuery('CustomerOrder').comment('load orders').purpose('render totals')) + counted = root.for_count('totalResults', QueryOperation('named-count', 'count_as', 'totalResults')) + chunks = [chunk async for chunk in service.query_stream(context, counted, 1)] + assert chunks[0].rows == [{'totalResults': 1}] + assert len(metadata) == len(transport.streams) == 1 + assert len(entries) == (1 if logging else 0) + assert [node.comment for node in metadata[0].request_trace_chain] == [ + 'render totals', 'load orders', "count_as('totalResults')"] + assert not any(node.kind == 'relation' for node in metadata[0].trace_chain) + + +@pytest.mark.asyncio +async def test_id_set_cache_isolates_actual_sqlite_sources_without_ambient_db_registration(tmp_path): + from teaql.runtime.context import InMemoryIdSetStore + store = InMemoryIdSetStore() + first_path = tmp_path / 'first'; first_path.mkdir() + second_path = tmp_path / 'second'; second_path.mkdir() + first, first_service, first_transport, _ = await fixture(first_path) + second, second_service, second_transport, _ = await fixture(second_path) + first.set_id_set_store(store); second.set_id_set_store(store) + assert first.get_resource('db') is second.get_resource('db') is None + await second_service.mutate(second, MutationRequest(InsertCommand.new('CustomerOrder') + .value('id', 999).value('version', 1).value('name', 'second database only'), comment='seed source isolation')) + second_transport.reads.clear() + + def page(): + return QueryRequest(SelectQuery('CustomerOrder').project('id').filter(Expr.eq('id', 999)).limit(1) + .optimize_pagination_with_id_set().comment('load isolated page').purpose('verify source isolation')) + + assert (await first_service.query(first, page())).rows == [] + assert len(first_transport.reads) == 1 + result = await second_service.query(second, page()) + assert result.rows == [{'id': 999}] + assert len(second_transport.reads) == 2 + result = await second_service.query(second, page()) + assert result.rows == [{'id': 999}] and len(second_transport.reads) == 3 + + INTENT_FAILURES = [case for case in json.loads( (Path(__file__).parents[2] / 'fixtures/request-intent-v1.json').read_text())['cases'] if 'error' in case] @@ -330,6 +741,43 @@ async def test_nested_facets_keep_original_route_and_nested_materialization(tmp_ assert query.facets[0].query.facets[0].name == 'orders', 'execution changed the caller Facet tree' +@pytest.mark.asyncio +@pytest.mark.parametrize('logging', [False, True]) +async def test_nested_facet_siblings_preserve_native_parent_and_local_comments(tmp_path, logging): + from teaql.core import QueryOperation + from teaql.runtime.context import SqlLogOptions + context, service, transport, entries = await fixture(tmp_path) + if not logging: + context.with_sql_log_options(SqlLogOptions.disabled()) + for owner, target in [('PaymentAttempt', 'Payment'), ('Payment', 'CustomerOrder')]: + service.schema_provider.get_entity(owner).relation(RelationDescriptor('owner', target).local('parent_id').foreign('id')) + query = SelectQuery('PaymentAttempt').project('id').limit(10).comment('load branches').purpose('render branches') + for alias in ('left', 'right'): + orders = SelectQuery('CustomerOrder').project('id').limit(10).count('paymentCount').comment(f'load {alias} orders') + payments = SelectQuery('Payment').project('id', 'parent_id').limit(10).count('attemptCount').comment(f'load {alias} payments') + payments.facet_by('orders', 'owner', orders, True, QueryOperation('facet', 'facet_by_owner_as', 'orders', 'owner')) + query.facet_by(alias, 'owner', payments, True, QueryOperation('facet', 'facet_by_owner_as', alias, 'owner')) + result = await service.query(context, QueryRequest(query)) + assert len(transport.reads) == 9 + assert len(entries) == (9 if logging else 0) + for alias in ('left', 'right'): + assert [(row['id'], row['attemptCount']) for row in result.facets[alias]] == [(1, 1)] + assert [(row['id'], row['paymentCount']) for row in result.facets[alias].facet('orders')] == [(1, 1)] + if logging: + prefix = ['render branches', 'load branches', f"facet_by_owner_as('{alias}')"] + chains = [[node.comment for node in entry.trace_chain] for entry in entries] + branches = [chain for chain in chains if len(chain) > 2 and chain[2] == prefix[2]] + assert len(branches) == 4 + assert [*prefix, 'aggregate owner'] in branches + assert [*prefix, f'load {alias} payments'] in branches + for local in ('aggregate owner', f'load {alias} orders'): + assert [*prefix, f'load {alias} payments', "facet_by_owner_as('orders')", local] in branches + assert query.comment_text == 'load branches' + assert query.facets[0].query.comment_text == 'load left payments' + assert query.facets[1].query.comment_text == 'load right payments' + print(f'RTC_COM_06_PASS logging={str(logging).lower()} nested=true siblings=true') + + @pytest.mark.asyncio @pytest.mark.parametrize('include_all', [False, True]) @pytest.mark.parametrize('empty', [False, True]) @@ -1004,7 +1452,8 @@ async def test_inherited_debug_opt_in_keeps_credentials_hidden_and_can_reproject @pytest.mark.asyncio @pytest.mark.parametrize('logging', [False, True]) -async def test_two_live_query_graphs_keep_independent_intent(tmp_path, monkeypatch, logging): +@pytest.mark.parametrize('shared_template', [False, True]) +async def test_two_live_query_graphs_keep_independent_intent(tmp_path, monkeypatch, logging, shared_template): context, service, transport, entries = await fixture(tmp_path) if not logging: context.disable_select_sql_log() @@ -1032,7 +1481,23 @@ def observe(actual, **kwargs): monkeypatch.setattr(transport, 'fetch_all_sql', pause_real_roots) monkeypatch.setattr(context, '_record_metadata_log', observe) - requests = [QueryRequest(three_levels(), _comment=f'load {label} graph', + from copy import deepcopy + from teaql.core.request_tree import _owner_for + template = three_levels().relations[0].query + template_before = deepcopy(template) + def parent_query(): + query = three_levels() + if shared_template: + query.relations[0].query = template + return query + def assert_template_unchanged(): + assert template == template_before + query = template + while query is not None: + assert _owner_for(query) is None + assert query.comment_text is None and getattr(query, 'purpose_text', None) is None + query = query.relations[0].query if query.relations else None + requests = [QueryRequest(parent_query(), _comment=f'load {label} graph', _purpose=f'render {label} graph') for label in ('alpha', 'beta')] # Native test-only inspection: query frames must not become ambient Context # fields/resources. Keep legitimate SQL-log appends observable separately. @@ -1057,6 +1522,7 @@ def assert_context_unchanged(): assert metadata == entries == [] assert [request.intent.comment for request in requests] == ['load alpha graph', 'load beta graph'] assert_context_unchanged() + assert_template_unchanged() release.set() results = await asyncio.wait_for(asyncio.gather(*pending), timeout=5) assert all(result.rows[0]['children'][0]['children'][0]['children'][0]['id'] == 1 @@ -1064,6 +1530,7 @@ def assert_context_unchanged(): assert len(transport.reads) == len(metadata) == 8 assert len(entries) == (8 if logging else 0) assert_context_unchanged() + assert_template_unchanged() for observed, path_attr in [(metadata, 'trace_chain'), (entries, 'trace_path')]: for label in ('alpha', 'beta'): own = [entry for entry in observed if entry.comment == f'load {label} graph'] diff --git a/tests/runtime/test_graph_telemetry.py b/tests/runtime/test_graph_telemetry.py new file mode 100644 index 0000000..6a5763b --- /dev/null +++ b/tests/runtime/test_graph_telemetry.py @@ -0,0 +1,101 @@ +"""The runtime graph parent covers all transaction and completion phases.""" +import pytest + +from teaql.core.entity import EntityKey +from teaql.runtime import UserContext +from teaql.runtime.audit import MutationAuditKind, RawAuditEvent + + +class Recorder: + def __init__(self, events, broken=False): + self.events, self.broken = events, broken + + def start(self, operation): + if self.broken: + raise RuntimeError('optional adapter unavailable') + self.events.append(('start', operation.name)) + events = self.events + class Scope: + def success(self, attributes=None): + events.append(('success', operation.name)) + + def failure(self, error): + events.append(('failure', operation.name, error)) + return Scope() + + +@pytest.mark.asyncio +@pytest.mark.parametrize('failure', ['none', 'begin', 'work', 'commit', 'audit', 'cleanup', 'adapter']) +async def test_graph_parent_covers_begin_commit_audit_and_cleanup(failure): + events, original = [], RuntimeError('private provider failure') + class Provider: + async def begin(self, context): + events.append('begin') + if failure == 'begin': + raise original + return self + + async def commit(self, context): + events.append('commit') + if failure == 'commit': + raise original + + async def rollback(self, context): + events.append('rollback') + + context = UserContext().insert_resource('dataService', Provider()) + context.with_runtime_telemetry(Recorder(events, failure == 'adapter')) + class Sink: + def on_safe_event(self, context, event): + events.append('audit') + if failure == 'audit': + raise original + context.with_app_audit_event_sink(Sink()) + retained = [] + async def work(graph): + retained.append(graph) + events.append('work') + if failure == 'work': + raise original + await graph.context.send_audit_event(RawAuditEvent(MutationAuditKind.CREATED, 'CustomerOrder', 1, ())) + def cleanup(): + events.append('cleanup') + if failure == 'cleanup': + raise original + graph.after_commit(cleanup) + return 'saved' + + if failure in ('none', 'adapter'): + assert await context.execute_graph_save(work, comment='save graph') == 'saved' + else: + with pytest.raises(RuntimeError) as caught: + await context.execute_graph_save(work, comment='save graph') + if failure in ('audit', 'cleanup'): + assert caught.value.committed is True + assert original in caught.value.failures + else: + assert caught.value is original + assert events[-1] == ('failure', 'graph.save', caught.value) + if failure != 'adapter': + assert events[0] == ('start', 'graph.save') + assert sum(event == ('start', 'graph.save') for event in events) == 1 + assert sum(isinstance(event, tuple) and event[:2] in ( + ('success', 'graph.save'), ('failure', 'graph.save')) for event in events) == 1 + if retained: + with pytest.raises(RuntimeError, match='no longer writable'): + retained[0].scope(EntityKey('CustomerOrder', 1)) + assert not context._graph_save_lock.locked() + assert context._graph_save_guard.get() is None + if failure in ('none', 'audit', 'cleanup', 'adapter'): + assert 'commit' in events and 'rollback' not in events + assert events.index('commit') < events.index('audit') < events.index('cleanup') + + +@pytest.mark.asyncio +async def test_rejected_root_starts_no_span_and_no_transaction(): + events = [] + context = UserContext().with_runtime_telemetry(Recorder(events)) + for comment in (None, '', ' \t'): + with pytest.raises(ValueError): + await context.execute_graph_save(lambda graph: None, comment=comment) + assert events == [] diff --git a/tests/runtime/test_loaded_scalar_snapshot.py b/tests/runtime/test_loaded_scalar_snapshot.py new file mode 100644 index 0000000..d8c4c7b --- /dev/null +++ b/tests/runtime/test_loaded_scalar_snapshot.py @@ -0,0 +1,36 @@ +import gc +import weakref +from teaql.core.value import Value +from teaql.runtime.loaded_scalar_snapshot import LoadedScalarSnapshots + + +class Entity: + pass + + +def test_snapshot_copies_input_and_output_and_is_not_entity_state(): + entity = Entity() + source = {'name': Value.Text('original'), 'json': Value.JSON({'nested': ['original']})} + LoadedScalarSnapshots.capture(entity, source) + source['json'].val['nested'].append('outside mutation') + source['name'] = Value.Text('changed') + first = LoadedScalarSnapshots.values(entity) + assert first['name'].val == 'original' and first['json'].val == {'nested': ['original']} + first['json'].val['nested'].append('output mutation') + assert LoadedScalarSnapshots.values(entity)['json'].val == {'nested': ['original']} + assert vars(entity) == {} + + +def test_snapshot_rollback_is_independent_and_gc_releases_owner(): + entity, another = Entity(), Entity() + LoadedScalarSnapshots.capture(entity, {'secret': Value.Text('first')}) + LoadedScalarSnapshots.capture(another, {'secret': Value.Text('other')}) + old = LoadedScalarSnapshots.values(entity) + LoadedScalarSnapshots.capture(entity, {'secret': Value.Text('persisted')}) + LoadedScalarSnapshots.restore(entity, old) + assert LoadedScalarSnapshots.values(entity)['secret'].val == 'first' + assert LoadedScalarSnapshots.values(another)['secret'].val == 'other' + reference = weakref.ref(entity) + del entity + gc.collect() + assert reference() is None diff --git a/tests/runtime/test_query_list.py b/tests/runtime/test_query_list.py new file mode 100644 index 0000000..f3edcea --- /dev/null +++ b/tests/runtime/test_query_list.py @@ -0,0 +1,194 @@ +import asyncio +from types import SimpleNamespace + +import pytest + +from teaql.core.query import SelectQuery +from teaql.core.request_tree import QueryOperation +from teaql.data_service import QueryRequest +from teaql.runtime import UserContext +from teaql.runtime.query_list import execute_typed_list, observe_query_operation, _query_owner + + +class Probe: + def __init__(self): + self.events = [] + + def start(self, operation): + self.events.append(('start', operation.name)) + events = self.events + + class Scope: + def success(self, values): + events.append(('success', values['teaql.result.cardinality'])) + + def failure(self, error): + events.append(('failure', error)) + + return Scope() + + +class Service: + def __init__(self): + self.queries = [] + self.callback = None + self.failure = None + self.count = 3 + + async def query(self, context, request): + async def execute(): + self.queries.append(request) + if self.callback: + await self.callback(context, request) + if self.failure: + raise self.failure + if request.query.aggregates: + return SimpleNamespace(rows=[{'totalOrders': self.count}], facets={}) + return SimpleNamespace(rows=[{'id': 2, 'version': 1}], facets={'statusCounts': []}) + + return await observe_query_operation(context, self, request, execute, + lambda result: {'teaql.result.cardinality': len(result.rows)}) + + +def setup(counted=True): + probe, service = Probe(), Service() + context = UserContext().with_runtime_telemetry(probe).insert_resource('dataService', service) + query = SelectQuery('Order').limit(1).comment('load orders').purpose('render order list') + if counted: + query.count_field('id', 'totalOrders') + return context, service, probe, QueryRequest(query) + + +async def typed(context, request, counted=True, hydrate=lambda row: row['id']): + return await execute_typed_list(context, request, hydrate, + count_operation=QueryOperation('named-count', 'count_as', 'totalOrders') if counted else None, + count_alias='totalOrders' if counted else None) + + +@pytest.mark.asyncio +@pytest.mark.parametrize('counted', [False, True]) +async def test_one_root_covers_optional_count_facets_and_hydration(counted): + context, service, probe, root = setup(counted) + + def hydrate(row): + assert probe.events == [('start', 'Order.list')], 'root ended before typed hydration' + return row['id'] + + result = await typed(context, root, counted, hydrate) + assert list(result) == [2] and result.facets == {'statusCounts': []} + assert len(service.queries) == (2 if counted else 1) + assert result.aggregations == ({'totalOrders': 3} if counted else {}) + assert probe.events == [('start', 'Order.list'), ('success', 1)] + assert _query_owner.get() is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize('phase', ['count', 'rows', 'hydrate']) +async def test_failure_preserves_original_error_closes_root_and_restores_ownership(phase): + context, service, probe, root = setup(phase != 'rows') + original = ValueError('PRIVATE-QUERY-FAILURE') + if phase != 'hydrate': + service.failure = original + + def hydrate(row): + raise original + + with pytest.raises(ValueError) as caught: + await typed(context, root, phase != 'rows', hydrate) + assert caught.value is original + assert probe.events == [('start', 'Order.list'), ('failure', original)] + assert _query_owner.get() is None + assert len(service.queries) == (2 if phase == 'hydrate' else 1) + + +@pytest.mark.asyncio +async def test_independent_nested_request_and_different_service_keep_their_own_root(): + context, service, probe, root = setup() + other = Service() + + async def nested(ctx, request): + if not request.query.aggregates: + return + independent = QueryRequest(SelectQuery('Status').limit(1).comment('load status').purpose('inspect status')) + await service.query(ctx, independent) + await other.query(ctx, request.with_query(request.query)) + + service.callback = nested + await typed(context, root) + assert [value for phase, value in probe.events if phase == 'start'] == ['Order.list', 'Status.list', 'Order.list'] + assert _query_owner.get() is None + + +@pytest.mark.asyncio +async def test_same_context_concurrent_lists_have_separate_task_ownership(): + context, service, probe, root = setup() + arrivals, release = 0, asyncio.Event() + + async def barrier(ctx, request): + nonlocal arrivals + if request.query.aggregates: + arrivals += 1 + if arrivals == 2: + release.set() + await asyncio.wait_for(release.wait(), 2) + + service.callback = barrier + result = await asyncio.wait_for(asyncio.gather(typed(context, root), typed(context, root)), 3) + assert [list(rows) for rows in result] == [[2], [2]] + assert [value for phase, value in probe.events if phase == 'start'] == ['Order.list', 'Order.list'] + assert len([event for event in probe.events if event == ('success', 1)]) == 2 + assert len(service.queries) == 4 and _query_owner.get() is None + + +@pytest.mark.asyncio +async def test_child_task_does_not_silently_inherit_parent_invocation_ownership(): + context, service, probe, root = setup(False) + + async def execute(): + return await asyncio.create_task(service.query(context, root.with_query(root.query))) + + await observe_query_operation(context, service, root, execute, + lambda result: {'teaql.result.cardinality': len(result.rows)}) + assert len([event for event in probe.events if event[0] == 'start']) == 2 + assert _query_owner.get() is None + + +@pytest.mark.asyncio +async def test_cancellation_is_preserved_and_next_query_is_independent(): + context, service, probe, root = setup() + original = asyncio.CancelledError() + service.failure = original + with pytest.raises(asyncio.CancelledError) as caught: + await typed(context, root) + assert caught.value is original and _query_owner.get() is None + service.failure = None + await typed(context, root) + assert probe.events == [('start', 'Order.list'), ('failure', original), ('start', 'Order.list'), ('success', 1)] + + +@pytest.mark.asyncio +@pytest.mark.parametrize('value', [-1, 1.5, float('nan'), float('inf'), True, '3']) +async def test_invalid_count_does_not_invent_row_result(value): + context, service, probe, root = setup() + service.count = value + with pytest.raises(RuntimeError, match='exact list count'): + await typed(context, root) + assert len(service.queries) == 1 and probe.events[-1][0] == 'failure' + assert _query_owner.get() is None + + +@pytest.mark.asyncio +async def test_adapter_faults_do_not_replace_results_or_failures(): + context, service, _, root = setup() + + class Broken: + def start(self, operation): + raise RuntimeError('PRIVATE-ADAPTER-FAILURE') + + context.with_runtime_telemetry(Broken()) + assert list(await typed(context, root)) == [2] + original = ValueError('PRIVATE-QUERY-FAILURE') + service.failure = original + with pytest.raises(ValueError) as caught: + await typed(context, root) + assert caught.value is original and _query_owner.get() is None diff --git a/tests/runtime/test_query_stream.py b/tests/runtime/test_query_stream.py new file mode 100644 index 0000000..ff6f8f8 --- /dev/null +++ b/tests/runtime/test_query_stream.py @@ -0,0 +1,135 @@ +import asyncio +from contextvars import ContextVar +from types import SimpleNamespace +import pytest + +from teaql.core.query import SelectQuery +from teaql.data_service import QueryRequest +from teaql.runtime import UserContext +from teaql.runtime.query_stream import execute_typed_stream + + +active = ContextVar('test_stream_active', default=None) + + +class Recorder: + def __init__(self): + self.events = [] + def start(self, operation): + events = self.events + events.append(('start', operation.name)) + class Scope: + token = active.set(operation.name) + def activate(self): + if self.token is None: + self.token = active.set(operation.name) + def deactivate(self): + if self.token is not None: + token, self.token = self.token, None + active.reset(token) + def success(self, attributes=None): + events.append(('success', operation.name, attributes)) + self.deactivate() + def cancelled(self, attributes=None): + events.append(('cancelled', operation.name, attributes)) + self.deactivate() + def failure(self, error): + events.append(('failure', operation.name, error)) + self.deactivate() + return Scope() + + +class Cursor: + def __init__(self): + self.closed = self.polled = 0 + def __aiter__(self): + return self + async def __anext__(self): + assert active.get() == 'School.stream' + self.polled += 1 + if self.polled > 1: + raise StopAsyncIteration + return SimpleNamespace(rows=[{'id': 1}, {'id': 2}]) + async def aclose(self): + assert active.get() == 'School.stream' + self.closed += 1 + + +def fixture(cursor=None, hydrate=lambda row: row): + telemetry = Recorder() + cursor = cursor or Cursor() + class Service: + calls = 0 + def query_stream(self, context, request, chunk_size): + self.calls += 1 + return cursor + service = Service() + context = UserContext().with_runtime_telemetry(telemetry).insert_resource('dataService', service) + request = QueryRequest(SelectQuery('School'), _comment='bounded stream', _purpose='test ownership') + return execute_typed_stream(context, request, hydrate, 2), telemetry, cursor, service + + +@pytest.mark.asyncio +async def test_lazy_close_and_poll_scoped_activation(): + stream, telemetry, cursor, service = fixture() + assert telemetry.events == [] and service.calls == 0 + await stream.aclose() + assert telemetry.events == [] and service.calls == 0 and cursor.closed == 0 + stream, telemetry, cursor, service = fixture() + assert (await anext(stream))['id'] == 1 + assert active.get() is None + assert [row async for row in stream] == [{'id': 2}] + assert active.get() is None and cursor.closed == 1 + assert telemetry.events[-1] == ('success', 'School.stream', {'teaql.result.cardinality': 2}) + + +@pytest.mark.asyncio +async def test_early_close_counts_delivered_not_prefetched_and_is_once_only(): + stream, telemetry, cursor, service = fixture() + await anext(stream) + await stream.aclose() + await stream.aclose() + assert cursor.closed == 1 and active.get() is None + assert telemetry.events[-1] == ('cancelled', 'School.stream', {'teaql.result.cardinality': 1}) + + +@pytest.mark.asyncio +async def test_hydration_failure_is_original_and_closes_cursor(): + original = RuntimeError('private hydration error') + def hydrate(row): + raise original + stream, telemetry, cursor, service = fixture(hydrate=hydrate) + with pytest.raises(RuntimeError) as caught: + await anext(stream) + assert caught.value is original and cursor.closed == 1 and active.get() is None + assert telemetry.events[-1] == ('failure', 'School.stream', original) + + +@pytest.mark.asyncio +async def test_cleanup_failure_after_exhaustion_fails_root_and_restores_owner(): + original = RuntimeError('private cleanup error') + class Broken(Cursor): + async def aclose(self): + raise original + stream, telemetry, cursor, service = fixture(cursor=Broken()) + with pytest.raises(RuntimeError) as caught: + [row async for row in stream] + assert caught.value is original and active.get() is None + assert telemetry.events[-1] == ('failure', 'School.stream', original) + + +@pytest.mark.asyncio +async def test_cancelled_poll_closes_cursor_and_preserves_cancellation(): + entered = asyncio.Event() + class Waiting(Cursor): + async def __anext__(self): + entered.set() + await asyncio.Event().wait() + stream, telemetry, cursor, service = fixture(cursor=Waiting()) + task = asyncio.create_task(anext(stream)) + await entered.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert cursor.closed == 1 and active.get() is None + assert telemetry.events[-1] == ('cancelled', 'School.stream', {'teaql.result.cardinality': 0}) diff --git a/tests/runtime/test_standard_queue.py b/tests/runtime/test_standard_queue.py new file mode 100644 index 0000000..7034b3a --- /dev/null +++ b/tests/runtime/test_standard_queue.py @@ -0,0 +1,92 @@ +"""Actual SDK queues and owned HTTP, not simulated queue lengths.""" +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +from teaql.runtime.standard_telemetry import StandardTelemetry +from teaql.runtime.telemetry import RuntimeOperation +from teaql.runtime.telemetry_startup import TelemetryConfiguration +from teaql.runtime.telemetry_http import HttpExporter, export_result + + +def burst(pipeline, count): + for _ in range(count): + pipeline.start(RuntimeOperation('query', 'native queue fixture')).success() + + +def test_inflight_and_queued_sdk_records_share_one_budget(monkeypatch): + release = threading.Event() + entered = {signal: threading.Event() for signal in ('traces', 'logs')} + def export(self, batch, *args, **kwargs): + if self.signal in entered: + entered[self.signal].set() + release.wait(3) + return export_result(self.signal).SUCCESS + monkeypatch.setattr(HttpExporter, 'export', export) + pipeline = StandardTelemetry(TelemetryConfiguration(True, 'native-quota', 'http://127.0.0.1:1')) + try: + burst(pipeline, 128) + assert all(event.wait(1) for event in entered.values()) + burst(pipeline, 2172) + health = pipeline.health() + for signal in ('traces', 'logs'): + assert health.get('loss_reasons', {}).get(signal, {}).get('queue_overflow', 0) == 252 + finally: + release.set() + pipeline.shutdown() + + +def test_shutdown_settles_actual_never_acknowledged_backlog_before_return(): + class Handler(BaseHTTPRequestHandler): + def do_POST(self): + self.rfile.read(int(self.headers['Content-Length'])) + self.connection.settimeout(8) + try: + while self.connection.recv(1): + pass + except OSError: + pass + def log_message(self, *args): + pass + server = ThreadingHTTPServer(('127.0.0.1', 0), Handler) + server.daemon_threads = True + threading.Thread(target=server.serve_forever, daemon=True).start() + pipeline = StandardTelemetry(TelemetryConfiguration(True, 'native-backlog', f'http://127.0.0.1:{server.server_port}')) + try: + burst(pipeline, 2300) + started = time.monotonic() + pipeline.shutdown() + assert time.monotonic() - started < 6.5 + # The SDK's actual owned workers must terminate, not continue exporting + # the remaining deque after the public lifecycle method returned. + trace_processor = pipeline.traces._active_span_processor._span_processors[0] + log_processor = pipeline.logs._multi_log_record_processor._log_record_processors[0] + assert not trace_processor._batch_processor._worker_thread.is_alive() + assert not log_processor._batch_processor._worker_thread.is_alive() + for signal in ('traces', 'logs'): + queue = pipeline.health()['queues'][signal] + assert queue['outstanding'] == queue['exporting'] == 0 + assert queue['admitted'] == queue['completed'] + queue['retired'] == 2048 + finally: + pipeline.shutdown() + server.shutdown() + server.server_close() + + +def test_closed_and_active_scopes_report_late_once_without_new_spans(monkeypatch): + monkeypatch.setattr(HttpExporter, 'export', lambda self, batch, *args, **kwargs: export_result(self.signal).SUCCESS) + pipeline = StandardTelemetry(TelemetryConfiguration(True, 'native-late', 'http://127.0.0.1:1')) + active = pipeline.start(RuntimeOperation('query', 'active before shutdown')) + pipeline.shutdown() + before = pipeline.health() + active.success() + active.success() + late = pipeline.start(RuntimeOperation('query', 'new after shutdown')) + assert late.correlation is None + late.success() + late.failure(RuntimeError('late private error')) + after = pipeline.health() + for signal in ('traces', 'logs'): + assert after['queues'][signal]['post_shutdown'] - before['queues'][signal]['post_shutdown'] == 2 + assert after['queues'][signal]['admitted'] == before['queues'][signal]['admitted'] + assert after['queues'][signal]['outstanding'] == 0 diff --git a/tests/runtime/test_standard_telemetry.py b/tests/runtime/test_standard_telemetry.py new file mode 100644 index 0000000..9011c14 --- /dev/null +++ b/tests/runtime/test_standard_telemetry.py @@ -0,0 +1,96 @@ +import json +import subprocess +import sys +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import threading +import pytest +from teaql.runtime.telemetry_startup import TelemetryConfiguration, ENABLED, SERVICE, ENDPOINT + + +def test_disabled_has_hints_without_importing_sdk(): + script = "from teaql.runtime.context import UserContext; import sys; UserContext(); assert not any(name.startswith('opentelemetry') for name in sys.modules)" + result = subprocess.run([sys.executable, '-c', script], capture_output=True, text=True) + assert result.returncode == 0, result.stderr + for name in (ENABLED, SERVICE, ENDPOINT): + assert name in result.stderr + + +@pytest.mark.parametrize('endpoint', ['http://secret:password@localhost', 'http://localhost?token=secret', 'http://localhost?', 'file:///secret', 'http://localhost/#secret']) +def test_invalid_urls_never_echo(endpoint): + config = TelemetryConfiguration.read({ENABLED: 'true', SERVICE: 'example', ENDPOINT: endpoint}) + assert not config.enabled and config.error == 'invalid-endpoint' + assert 'secret' not in config.message + + +def test_official_sdk_exports_three_signals_with_carried_identity_and_no_global_replacement(monkeypatch): + from teaql.runtime.standard_telemetry import StandardTelemetry + from teaql.runtime.telemetry import RuntimeOperation + from opentelemetry import trace + from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import ExportLogsServiceRequest + from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest + messages = [] + class Handler(BaseHTTPRequestHandler): + def do_POST(self): + messages.append((self.path, dict(self.headers), self.rfile.read(int(self.headers['Content-Length'])))) + self.send_response(200); self.send_header('Content-Type', 'application/x-protobuf'); self.end_headers() + def log_message(self, *args): + pass + server = ThreadingHTTPServer(('127.0.0.1', 0), Handler) + threading.Thread(target=server.serve_forever, daemon=True).start() + monkeypatch.setenv('OTEL_EXPORTER_OTLP_HEADERS', 'Authorization=PRIVATE-CUSTOMER-TOKEN') + previous = trace.get_tracer_provider() + pipeline = StandardTelemetry(TelemetryConfiguration(True, 'sdk-test', f'http://127.0.0.1:{server.server_port}')) + try: + first = pipeline.start(RuntimeOperation('provider', 'first')) + first.success() + second = pipeline.start(RuntimeOperation('provider', 'second')) + second.success() + unrelated = pipeline.start(RuntimeOperation('query', 'unrelated')) + pipeline.record_diagnostic({'teaql.sql.debug': 'SELECT second'}, second.correlation) + pipeline.record_diagnostic({'teaql.sql.debug': 'SELECT first'}, first.correlation) + unrelated.success() + pipeline.flush() + assert {path for path, _, _ in messages} == {'/v1/traces', '/v1/logs', '/v1/metrics'} + assert all('Authorization' not in headers for _, headers, _ in messages) + spans, logs = [], [] + for path, _, data in messages: + if path == '/v1/traces': + payload = ExportTraceServiceRequest.FromString(data) + spans.extend(span for resource in payload.resource_spans for scope in resource.scope_spans for span in scope.spans) + elif path == '/v1/logs': + payload = ExportLogsServiceRequest.FromString(data) + logs.extend(log for resource in payload.resource_logs for scope in resource.scope_logs for log in scope.log_records) + assert len(spans) == 3 + sql = [log for log in logs if log.body.string_value == 'TeaQL SQL execution'] + assert len(sql) == 2 + for record, identity in zip(sql, [second.correlation, first.correlation]): + assert int.from_bytes(record.trace_id, 'big') == identity[0] + assert int.from_bytes(record.span_id, 'big') == identity[1] + assert trace.get_tracer_provider() is previous + finally: + pipeline.shutdown(); server.shutdown(); server.server_close() + + +def test_runtime_projection_is_masked_when_ordinary_logging_is_disabled(monkeypatch): + from datetime import datetime + from teaql.runtime.context import UserContext + from teaql.data_service import ExecutionMetadata, DataServiceOperation + from teaql.sql.types import DatabaseKind + from teaql.core.value import Value + class Probe: + def record_diagnostic(self, attributes, correlation): + self.event = attributes, correlation + probe = Probe() + context = UserContext().with_runtime_telemetry(probe) + context.disable_select_sql_log() + context.disable_mutation_sql_log() + monkeypatch.setenv('TEAQL_ALLOW_SENSITIVE_PLAINTEXT_LOGS', 'I_UNDERSTAND_SENSITIVE_DATA_MAY_BE_WRITTEN_TO_DISK') + metadata = ExecutionMetadata('sqlite', DataServiceOperation.Query, datetime.now(), datetime.now(), + parameterized_sql='SELECT name FROM sample WHERE name = ?', parameters=[Value.ValText('PRIVATE-VALUE')], + parameter_log_policies=['masked'], sql_origin='generated', database_kind=DatabaseKind.Sqlite, + comment='load PRIVATE-VALUE', execution_outcome='success') + metadata._sql_correlation = (123, 456) + context.record_metadata_log(metadata) + assert probe.event[1] == (123, 456) + assert 'PRIVATE-VALUE' not in json.dumps(probe.event[0]) + assert context.sql_logs() == [] diff --git a/tests/runtime/test_telemetry_queue.py b/tests/runtime/test_telemetry_queue.py new file mode 100644 index 0000000..e81d1e5 --- /dev/null +++ b/tests/runtime/test_telemetry_queue.py @@ -0,0 +1,101 @@ +import threading +from teaql.runtime.telemetry_queue import ExportQueue + + +def fixture(capacity=4): + lost = [] + return ExportQueue('traces', capacity, lambda signal, amount, reason: lost.append((signal, amount, reason))), lost + + +def test_failed_handoff_releases_capacity_and_is_not_accepted_prefix(): + queue, lost = fixture(1) + def broken(): + raise RuntimeError('private enqueue error') + queue.handoff(object(), broken) + assert queue.snapshot()['admitted'] == queue.snapshot()['outstanding'] == 0 + assert queue.snapshot()['handoff_failed'] == 1 + assert queue.wait_prefix(queue.prefix(), .01) + assert lost == [('traces', 1, 'handoff_failure')] + + +def test_later_completion_cannot_satisfy_older_prefix(): + queue, lost = fixture() + first, second = object(), object() + queue.handoff(first, lambda: None) + target = queue.prefix() + queue.handoff(second, lambda: None) + queue.begin_export([second])(False) + assert not queue.wait_prefix(target, .01) + queue.begin_export([first])(False) + assert queue.wait_prefix(target, .01) and not lost + + +def test_retirement_is_separate_from_active_failed_attempt_and_once_only(): + queue, lost = fixture() + first, second = object(), object() + for record in (first, second): + queue.handoff(record, lambda: None) + ticket = queue.begin_export([first]) + queue.abort_queued() + queue.abort_queued() + assert queue.snapshot()['retired'] == 1 and queue.snapshot()['exporting'] == 1 + assert queue.begin_export([second]) is None + ticket(True) + ticket(True) + snapshot = queue.snapshot() + assert snapshot['admitted'] == snapshot['completed'] + snapshot['retired'] == 2 + assert snapshot['outstanding'] == snapshot['exporting'] == 0 + assert lost == [('traces', 1, 'shutdown'), ('traces', 1, 'export_failure')] + + +def test_identity_is_not_inferred_from_equal_record_data(): + class Equal: + def __eq__(self, other): + return True + queue, lost = fixture() + admitted, other = Equal(), Equal() + queue.handoff(admitted, lambda: None) + assert queue.begin_export([other]) is None + assert not queue.wait_prefix(queue.prefix(), .01) + queue.begin_export([admitted])(False) + assert queue.wait_idle(.01) and not lost + + +def test_concurrent_admission_is_exact_and_drop_does_not_call_sdk(): + queue, lost = fixture(128) + submitted = [] + def offer(): + for _ in range(100): + record = object() + queue.handoff(record, lambda: submitted.append(record)) + threads = [threading.Thread(target=offer) for _ in range(8)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + snapshot = queue.snapshot() + assert snapshot['offered'] == 800 + assert snapshot['admitted'] == snapshot['outstanding'] == len(submitted) == 128 + assert snapshot['overflow'] == sum(amount for _, amount, _ in lost) == 672 + + +def test_close_rejects_new_handoff_and_explicit_late_completion(): + queue, lost = fixture() + queue.close() + queue.handoff(object(), lambda: (_ for _ in ()).throw(AssertionError('must not call SDK'))) + queue.reject_late() + assert queue.snapshot()['post_shutdown'] == 2 + assert queue.snapshot()['admitted'] == queue.snapshot()['outstanding'] == 0 + assert lost == [('traces', 1, 'shutdown'), ('traces', 1, 'shutdown')] + + +def test_duplicate_record_cannot_replace_an_accepted_ticket(): + queue, lost = fixture() + record = object() + queue.handoff(record, lambda: None) + queue.handoff(record, lambda: (_ for _ in ()).throw(AssertionError('duplicate SDK enqueue'))) + assert queue.snapshot()['admitted'] == queue.snapshot()['outstanding'] == 1 + assert queue.snapshot()['handoff_failed'] == 1 + queue.begin_export([record])(False) + assert queue.snapshot()['completed'] == 1 + assert lost == [('traces', 1, 'handoff_failure')] diff --git a/tests/runtime/test_telemetry_response.py b/tests/runtime/test_telemetry_response.py new file mode 100644 index 0000000..568d9f3 --- /dev/null +++ b/tests/runtime/test_telemetry_response.py @@ -0,0 +1,167 @@ +"""Real exporter response and SDK metric-object accounting counterexamples.""" +from contextlib import contextmanager +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import threading +import logging +import time +import pytest +from opentelemetry.sdk.trace.export import SpanExportResult +from opentelemetry.sdk.metrics.export import MetricExportResult +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from teaql.runtime.telemetry_http import HttpExporter +from teaql.runtime.standard_telemetry import _SafeExporter + + +def one_span(): + memory = InMemorySpanExporter() + provider = TracerProvider(shutdown_on_exit=False) + provider.add_span_processor(SimpleSpanProcessor(memory)) + span = provider.get_tracer('response-test').start_span('native response fixture') + span.end() + result = memory.get_finished_spans() + provider.shutdown() + return result + + +def test_invalid_protobuf_is_not_full_acceptance(): + with receiver(b'\xff' * 8) as endpoint: + exporter = HttpExporter(endpoint + '/v1/traces', 'traces') + try: + assert exporter.export(one_span()) == SpanExportResult.FAILURE + finally: + exporter.shutdown() + + +@contextmanager +def receiver(body, status=200, content_type='application/x-protobuf', behavior=None): + class Handler(BaseHTTPRequestHandler): + def do_POST(self): + self.rfile.read(int(self.headers['Content-Length'])) + if behavior: + behavior(self) + return + self.send_response(status, 'PRIVATE-REMOTE-REASON') + self.send_header('Content-Type', content_type) + self.send_header('X-Private', 'PRIVATE-REMOTE-HEADER') + self.end_headers() + self.wfile.write(body) + def log_message(self, *args): + pass + server = ThreadingHTTPServer(('127.0.0.1', 0), Handler) + server.daemon_threads = True + server.handle_error = lambda *args: None + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + yield f'http://127.0.0.1:{server.server_port}' + finally: + server.shutdown() + server.server_close() + + +def test_failed_metric_batch_counts_metric_data_objects_not_one_batch(): + from types import SimpleNamespace + batch = SimpleNamespace(resource_metrics=[SimpleNamespace(scope_metrics=[ + SimpleNamespace(metrics=[object(), object(), object()]), SimpleNamespace(metrics=[object(), object()])])]) + class Exporter: + def export(self, batch): + return MetricExportResult.FAILURE + class Pipeline: + def __init__(self): + self.lost = [] + def _lost(self, signal, count): + self.lost.append((signal, count)) + pipeline = Pipeline() + assert _SafeExporter(Exporter(), pipeline, 'metrics').export(batch) == MetricExportResult.FAILURE + assert pipeline.lost == [('metrics', 5)] + + +def acknowledgement(rejected, message='PRIVATE-REMOTE-BODY'): + from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceResponse + response = ExportTraceServiceResponse() + response.partial_success.rejected_spans = rejected + response.partial_success.error_message = message + return response.SerializeToString() + + +@pytest.mark.parametrize('body,reason,rejected', [ + (acknowledgement(1), 'partial_success', 1), + (acknowledgement(0), 'warning', 0), + (b'', None, 0), (b'\x0a\x00', None, 0), +]) +def test_official_ack_reports_fixed_semantics_without_raw_text(body, reason, rejected, caplog, capsys, monkeypatch): + import http.client + monkeypatch.setattr(http.client.HTTPConnection, 'debuglevel', 1) + reports = [] + with caplog.at_level(logging.DEBUG), receiver(body) as endpoint: + exporter = HttpExporter(endpoint + '/v1/traces', 'traces', lambda *args: reports.append(args)) + try: + assert exporter.export(one_span()) == SpanExportResult.SUCCESS + assert reports == ([('traces', rejected, reason)] if reason else []) + finally: + exporter.shutdown() + captured = capsys.readouterr() + text = caplog.text + captured.out + captured.err + assert 'PRIVATE-REMOTE-' not in text + assert http.client.HTTPConnection.debuglevel == 1, 'customer debug settings unchanged' + + +@pytest.mark.parametrize('body,content_type', [ + (acknowledgement(-1), 'application/x-protobuf'), + (acknowledgement(2), 'application/x-protobuf'), + (b'\x08\x01', 'application/x-protobuf'), + (b'\x0a\x03\x0a\x01\x01', 'application/x-protobuf'), + (b'\x0a\x03\x12\x01\xff', 'application/x-protobuf'), + (b'\x95\x03\x00', 'application/x-protobuf'), + (b'\x93\x03' * 150 + b'\x94\x03' * 150, 'application/x-protobuf'), + (b'X' * (4 * 1024 * 1024 + 1), 'application/x-protobuf'), + (b'\x0a\x00', 'text/plain'), +]) +def test_invalid_acknowledgement_is_failure_with_fixed_counter(body, content_type): + reports = [] + with receiver(body, content_type=content_type) as endpoint: + exporter = HttpExporter(endpoint + '/v1/traces', 'traces', lambda *args: reports.append(args)) + try: + assert exporter.export(one_span()) == SpanExportResult.FAILURE + assert reports == [('traces', 0, 'invalid_response')] + finally: + exporter.shutdown() + + +@pytest.mark.parametrize('phase', ['headers', 'body']) +def test_whole_attempt_deadline_bounds_slow_headers_or_trickle_body(phase): + def slow(handler): + if phase == 'body': + handler.send_response(200) + handler.send_header('Content-Type', 'application/x-protobuf') + handler.end_headers() + for _ in range(8): + time.sleep(0.15) + if phase == 'body': + handler.wfile.write(b'\x00') + handler.wfile.flush() + with receiver(b'', behavior=slow) as endpoint: + exporter = HttpExporter(endpoint + '/v1/traces', 'traces') + started = time.monotonic() + try: + assert exporter.export(one_span(), timeout_millis=300) == SpanExportResult.FAILURE + assert time.monotonic() - started < 0.9 + finally: + exporter.shutdown() + + +def test_exporter_and_loss_callback_exceptions_are_not_retried_or_leaked(capsys): + class Broken: + def export(self, batch): + raise RuntimeError('PRIVATE-EXPORTER-ERROR') + class Pipeline: + calls = 0 + def _lost(self, signal, count): + self.calls += 1 + raise RuntimeError('PRIVATE-COUNTER-ERROR') + pipeline = Pipeline() + assert _SafeExporter(Broken(), pipeline, 'traces').export(one_span()) == SpanExportResult.FAILURE + assert pipeline.calls == 1 + captured = capsys.readouterr() + assert 'PRIVATE-' not in captured.out + captured.err