From 09795eaa62b5dd5d87f1b9f92efe04340be9dcd0 Mon Sep 17 00:00:00 2001 From: skishore23 Date: Fri, 30 Oct 2026 02:52:15 -0700 Subject: [PATCH 1/2] feat: add experimental program discovery and controlled benchmarks --- .github/workflows/ci.yml | 3 + README.md | 5 +- docs/advanced/program-discovery.md | 187 +++++ examples/discovery/experiments.test.cjs | 90 +++ examples/discovery/results/README.md | 53 ++ examples/discovery/results/manifest.json | 37 + examples/discovery/results/scheduling.json | 621 +++++++++++++++++ examples/discovery/results/transfer.json | 642 ++++++++++++++++++ examples/discovery/scheduling-discovery.cjs | 131 ++++ examples/discovery/scheduling.cjs | 193 ++++++ examples/discovery/transfer.cjs | 177 +++++ package.json | 5 +- packages/fx-core/package.json | 5 + .../src/__tests__/discovery-learning.test.ts | 249 +++++++ .../src/__tests__/discovery-runtime.test.ts | 147 ++++ .../fx-core/src/discovery/abstractions.ts | 211 ++++++ packages/fx-core/src/discovery/index.ts | 6 + packages/fx-core/src/discovery/model.ts | 82 +++ .../fx-core/src/discovery/optimization.ts | 86 +++ packages/fx-core/src/discovery/runtime.ts | 152 +++++ packages/fx-core/src/discovery/search.ts | 123 ++++ packages/fx-core/src/discovery/state.ts | 51 ++ packages/fx-core/src/index.ts | 5 +- 23 files changed, 3258 insertions(+), 3 deletions(-) create mode 100644 docs/advanced/program-discovery.md create mode 100644 examples/discovery/experiments.test.cjs create mode 100644 examples/discovery/results/README.md create mode 100644 examples/discovery/results/manifest.json create mode 100644 examples/discovery/results/scheduling.json create mode 100644 examples/discovery/results/transfer.json create mode 100644 examples/discovery/scheduling-discovery.cjs create mode 100644 examples/discovery/scheduling.cjs create mode 100644 examples/discovery/transfer.cjs create mode 100644 packages/fx-core/src/__tests__/discovery-learning.test.ts create mode 100644 packages/fx-core/src/__tests__/discovery-runtime.test.ts create mode 100644 packages/fx-core/src/discovery/abstractions.ts create mode 100644 packages/fx-core/src/discovery/index.ts create mode 100644 packages/fx-core/src/discovery/model.ts create mode 100644 packages/fx-core/src/discovery/optimization.ts create mode 100644 packages/fx-core/src/discovery/runtime.ts create mode 100644 packages/fx-core/src/discovery/search.ts create mode 100644 packages/fx-core/src/discovery/state.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f5ea9e8..e1588bc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,6 +34,9 @@ jobs: - name: Run tests run: npm run test + + - name: Run discovery experiments and controls + run: npm run test:experiments - name: Build packages run: npm run build diff --git a/README.md b/README.md index 93c6650..f6c3bbe 100644 --- a/README.md +++ b/README.md @@ -281,6 +281,9 @@ console.log(result.issues.length); // 3 ## Examples +- **[Experimental program discovery](./docs/advanced/program-discovery.md)** — inspectable programs, + parameterized abstraction learning, bounded search, and reproducible transfer and GPU scheduling experiments. + - **[Coding Agent](./examples/coding-agent/)** - Full-featured coding assistant - **[Research Agent](./examples/research-agent/)** - Advanced research and analysis @@ -323,4 +326,4 @@ npm test ## Support -- 📖 [Documentation](./docs/) \ No newline at end of file +- 📖 [Documentation](./docs/) diff --git a/docs/advanced/program-discovery.md b/docs/advanced/program-discovery.md new file mode 100644 index 0000000..ebbf39a --- /dev/null +++ b/docs/advanced/program-discovery.md @@ -0,0 +1,187 @@ +# Experimental program discovery + +FX can now represent a bounded program as JSON, execute it as an ordinary FX +`Step`, search for programs against an external evaluator, and extract +parameterized abstractions from a successful program corpus. This is an +experimental research kernel, not a general self-improving agent. + +No model, API key, coding agent, new dependency or production integration is +required for the included experiments. Node 18+ is required for the example +test runner (CI uses Node 20); the library remains compatible with the existing +package's target. + +## Run it + +From the repository root: + +```sh +npm ci +npm run test:experiments +npm run experiment:transfer +npm run experiment:scheduling +``` + +The experiment commands build the package and print JSON. To save JSON without +build logs, build first and run the example directly: + +```sh +npm run build +node examples/discovery/transfer.cjs > transfer.json +node examples/discovery/scheduling-discovery.cjs > scheduling.json +``` + +## Programs are data + +```ts +import { createAgent, createPlan } from '@fx/core'; +import { ProgramRuntime, Program, Abstraction } from '@fx/core/discovery'; + +type Counter = { readonly value: number }; +const affine: Abstraction = { + name: 'affine', arity: 2, + body: [ + { op: 'add', args: [{ parameter: 0 }] }, + { op: 'scale', args: [{ parameter: 1 }] } + ] +}; +const runtime = new ProgramRuntime([ + { name: 'add', arity: 1, + execute: (state, args) => ({ value: state.value + args[0]! }) }, + { name: 'scale', arity: 1, + execute: (state, args) => ({ value: state.value * args[0]! }) } +], [affine]); +const program: Program = { + version: 1, body: [{ op: 'affine', args: [3, 2] }] +}; + +runtime.expand(program); // add(3), scale(2) +runtime.run(program, { value: 4 }); // { state: { value: 14 }, primitiveCalls: 2 } +const agent = createAgent('discovered', createPlan('plan', [runtime.asStep(program)])); +await agent.start({ value: 4 }); // { value: 14 } +``` + +Version 1 has sequential calls, finite numeric arguments, and parameterized +abstractions. It deliberately has no arbitrary source evaluation, branches, +loops, general expression language or dynamic creation of native primitives. +It is possible to build other domains by registering trusted primitives. + +The runtime validates the whole expanded program before executing a primitive. +Unknown names, incorrect arity, unbound parameters, recursive definitions, +excessive call depth and expanded operation overruns fail before execution. +Defaults: 1,024 expanded primitive calls, 32 abstraction levels, 128 instructions +per serialized body, 16 arguments per call. + +States are isolated, deeply frozen JSON snapshots. Non-finite results, +non-JSON values and cycles are rejected. Primitives must be trusted synchronous +functions: the runtime cannot stop their JavaScript from accessing global state, +performing I/O or failing to return. Resource counts bound interpreted operations, +not CPU time or native memory. They are not a security sandbox or proof of purity. + +## Abstraction learning + +`learnAbstractions(successfulPrograms)`: + +1. Deduplicates closed, expanded programs by canonical content hash. +2. Finds repeated contiguous operation structures. +3. Replaces varying literal columns with parameters. Columns that vary together + share a parameter; fixed constants remain constants. +4. Counts non-overlapping structural replacements. +5. Keeps candidates with positive description-length savings, charging both + call sites and the new definition (including its parameters). + +Each result includes the definition, source program hashes, occurrence count, +compression gain and witnessed argument tuples. Results are independently scored +candidates, not an optimally selected joint library. The current miner is a +bounded contiguous-pattern learner, not Stitch or an e-graph implementation. + +`abstractProgram` factors a program without changing its expansion. +`abstractionActions` constructs calls for witnessed argument tuples only. +A caller can supply new tuples after validating them. Structural compression +does not establish utility, semantic novelty, behavioral generalization or +authority to activate a policy. The caller must supply genuinely successful +programs and independently assess learned candidates. + +## Search, evaluation and feedback + +`enumeratePrograms(actions, maxLength)` is a deterministic proposer. +`searchProgram(runtime, examples, proposals, budgets)` accepts any iterable of +candidate programs, returns the first exact solution, and retains every attempted +candidate with its identity, outcome and expanded operation cost. Both candidate +and primitive-call budgets apply; failing primitive invocations are charged. +It refuses to start an example whose full expansion cannot fit the remaining +budget. Verification cases are kept outside this search API. + +`optimizePrograms(proposals, evaluate, maxEvaluations)` selects the highest-scoring +valid candidate from a bounded proposal stream. `proposals` can also be a callback +`(archive, iteration) => program`, enabling outcome-guided mutations or an external +model adapter. A negative finite score is valid; invalid candidates cannot win. +Programs and scored feedback are frozen. Duplicates consume proposal budget but +are not evaluated twice. The budget counts proposals, not arbitrary computation +inside a trusted proposer or evaluator; domain experiments report that work +separately. The library has no built-in model training or automatic policy activation. + +## Experiment 1: abstraction transfer + +The initial eight arithmetic tasks are actually solved by primitive enumeration. +The resulting corpus supports an inferred two-parameter `add; scale` abstraction. +Eight distinct four-operation functions are withheld from that corpus. Search +receives three I/O examples per task and selected programs are checked against +five additional inputs, including a fractional and negative input. + +Three arms receive the same per-task budgets (256 proposals, 1,024 primitive +calls, at most four symbolic instructions): + +- Primitive operations only. +- The same operations plus eight saved, opaque training programs. +- The same operations plus calls to the learned parameterized abstraction. + +Every macro expansion is checked to use the original primitive action vocabulary. +Learning costs are reported separately. The designed experiment currently gives +0/8, 8/8 and 8/8 solved tasks respectively. This demonstrates bounded search reuse +and correct factoring; it **does not demonstrate an advantage over saved snippets**. +Vocabulary order and the symbolic search horizon affect the result. A stronger +next experiment must show useful new bindings or structures beyond that baseline. + +## Experiment 2: GPU scheduling policy discovery + +The discrete-event simulator models arrivals, heterogeneous worker speeds, +memory eligibility, per-tick prices, deadlines, and persistent model caches +with a four-tick reload cost. One job occupies one worker slot. Every run checks +for lost or duplicated jobs. Tests independently check overlapping assignments, +memory constraints, arrival ordering and a manually calculated timing/cost witness. + +A policy scores eligible job/worker pairs using a program over duration, estimated +cost, waiting time, remaining deadline slack and model-load cost. Policies are +linear weighted feature combinations. They cannot change simulator physics or +the external evaluator. The authored objective, minimized, is: + +```text +mean flow time + 0.25 * mean tardiness + 0.08 * total monetary cost +``` + +The adaptive proposer mutates high-scoring programs from its archive. A separate +random-search control uses the same proposal budget, initial baseline policies +and vocabulary. Both use six training scenarios, then select from a fixed +shortlist on four different validation scenarios. Final evaluation uses twelve +untouched workloads across mixed, burst, long-job and many-model regimes. + +Reports retain all six authored heuristics (FIFO, shortest duration, earliest +deadline, locality, cheapest placement and a cost/locality hybrid), the random +control, discovered programs, per-regime results and primitive-call costs. +Do not interpret the best aggregate as winning every metric or workload. +There are no production traces, native conformance tests, network failures, +retries, cancellations or confidence intervals. This experiment demonstrates +evaluator-guided policy search, not abstraction transfer or a production scheduler. + +## Next research gates + +- Parameterized libraries must outperform saved snippets and a strong program + search baseline under matched discovery costs on unseen task structures. +- Adding library candidates must improve future search more than their discovery, + validation, storage and vocabulary-expansion costs. +- Scheduling gains must survive multiple search seeds, stronger optimization + baselines, new workload distributions and production-trace simulation. +- Effect contracts and semantics-preserving rewrites require explicit semantics; + the existing `Context -> Context` type does not establish either property. + +These gates are experiments to run, not capabilities claimed by this version. diff --git a/examples/discovery/experiments.test.cjs b/examples/discovery/experiments.test.cjs new file mode 100644 index 0000000..50b7ff9 --- /dev/null +++ b/examples/discovery/experiments.test.cjs @@ -0,0 +1,90 @@ +const assert = require('node:assert/strict'); +const { test } = require('node:test'); +const { runTransfer } = require('./transfer.cjs'); +const { runScheduling } = require('./scheduling-discovery.cjs'); +const { simulate, workload, baselines } = require('./scheduling.cjs'); + +test('abstraction transfer is reproducible and includes the saved-program control', () => { + const result = runTransfer(); + assert.deepEqual(runTransfer(), result); + assert.equal(result.result.learnedSolved, 8); + assert.equal(result.result.savedProgramsSolved, 8); + assert.equal(result.result.baselineSolved, 0); + assert.ok(result.learning.primitiveCalls > 0); + for (const row of result.rows) { + for (const method of [row.baseline, row.savedPrograms, row.learned]) { + assert.ok(method.candidates <= result.budgets.maxCandidates); + assert.ok(method.primitiveCalls <= result.budgets.maxPrimitiveCalls); + } + // Prevent claiming an advantage caused solely by hiding expanded work. + assert.equal(row.learned.primitiveCalls, row.savedPrograms.primitiveCalls); + } +}); + +test('scheduler never double-books workers, starts early, violates memory, or loses jobs', () => { + for (const regime of ['mixed', 'burst', 'long', 'many-models']) { + const scenario = workload(911, 60, regime); + const result = simulate(baselines.hybrid, scenario); + assert.equal(result.jobs, scenario.jobs.length); + assert.deepEqual(simulate(baselines.hybrid, scenario), result); + for (const execution of result.assignments) { + const worker = scenario.workers.find((item) => item.id === execution.worker); + assert.ok(execution.start >= execution.arrival); + assert.ok(execution.memory <= worker.memory); + assert.ok(execution.finish > execution.start); + } + for (const worker of scenario.workers) { + const assigned = result.assignments + .filter((item) => item.worker === worker.id) + .sort((a, b) => a.start - b.start); + assigned.slice(1).forEach((item, i) => assert.ok(item.start >= assigned[i].finish)); + } + } +}); + +test('simulator has an independently calculated witness for duration, setup, flow and price', () => { + const scenario = { + workers: [{ id: 0, memory: 8, speed: 2, price: 3, model: -1 }], + jobs: [ + { id: 0, arrival: 0, work: 4, memory: 4, model: 0, deadline: 4 }, + { id: 1, arrival: 1, work: 6, memory: 4, model: 0, deadline: 10 }, + ], + }; + const result = simulate(baselines.fifo, scenario); + assert.deepEqual( + result.assignments.map((item) => [item.start, item.finish, item.cost]), + [ + [0, 6, 18], + [6, 9, 9], + ], + ); + assert.equal(result.meanFlow, 7); + assert.equal(result.meanTardiness, 1); + assert.equal(result.cost, 27); + assert.equal(result.score, -(7 + 0.25 + 0.08 * 27)); + assert.throws(() => simulate(baselines.fifo, { ...scenario, workers: [] }), /infeasible/); + assert.equal(simulate(baselines.fifo, { ...scenario, jobs: [] }).jobs, 0); +}); + +test('policy search uses separate training, selection and final workloads, and retains all regimes', () => { + const result = runScheduling({ budget: 24, seed: 7 }); + assert.deepEqual(runScheduling({ budget: 24, seed: 7 }), result); + const seen = new Set([...result.trainingSeeds, ...result.validationSeeds]); + assert.equal(seen.size, result.trainingSeeds.length + result.validationSeeds.length); + assert.ok(result.testScenarios.every((item) => !seen.has(item.seed))); + assert.ok(result.evaluatedCandidates <= result.proposalBudget); + assert.ok(result.learningCost.trainingPrimitiveCalls > 0); + assert.deepEqual(Object.keys(result.byRegime), ['mixed', 'burst', 'long', 'many-models']); + for (const method of [ + 'fifo', + 'shortest', + 'deadline', + 'locality', + 'cheapest', + 'hybrid', + 'randomSearch', + 'discovered', + ]) { + assert.ok(Number.isFinite(result.heldOut[method].score)); + } +}); diff --git a/examples/discovery/results/README.md b/examples/discovery/results/README.md new file mode 100644 index 0000000..593e312 --- /dev/null +++ b/examples/discovery/results/README.md @@ -0,0 +1,53 @@ +# Initial reproducible results + +These are synthetic experiments, not production performance claims. The manifest +records the base revision, exact source/output hashes, Node version and local +check results. Both JSON files can be regenerated from the documented commands +without a model or API key. Runtime and discovery costs are not interchangeable. + +## Abstraction transfer + +| Method | Verified held-out tasks solved | +| --- | ---: | +| Primitive-only enumeration | 0 / 8 | +| Enumeration with saved training programs | 8 / 8 | +| Enumeration with learned parameterized abstraction | 8 / 8 | + +All methods have the same per-task candidate and primitive-call caps. The initial +corpus itself costs 172 candidate trials and 341 primitive calls to discover. +The abstraction has two parameters, eight source programs, and one node of net +description-length savings after its definition is charged. + +The positive result is exact factoring and reuse on unseen compositions. The +important negative result is that parameterized learning does not beat saved +snippets in this experiment. It establishes no advantage over a strong coding +agent, no new primitive semantics, and no general intelligence claim. + +## Scheduling + +One search seed (42), 100 proposals per search method, six training workloads, +four validation workloads, and twelve final workloads across four regimes. +Lower objective is better. Time and price are synthetic units. + +| Method | Objective | Mean flow time | Mean tardiness | Mean workload cost | +| --- | ---: | ---: | ---: | ---: | +| FIFO | 100.5154 | 46.6910 | 31.4097 | 574.6500 | +| Shortest duration | 76.6566 | 31.8576 | 19.0625 | 500.4167 | +| Earliest deadline | 112.7492 | 53.9653 | 37.9410 | 616.2333 | +| Model locality | 82.0757 | 37.2118 | 23.2743 | 488.0667 | +| Cheapest placement | 78.1788 | 32.8194 | 19.8160 | 505.0667 | +| Authored cost/locality hybrid | 77.0728 | 32.5278 | 19.3854 | 496.2333 | +| Random search + validation | 77.0728 | 32.5278 | 19.3854 | 496.2333 | +| Adaptive search + validation | 76.3691 | 32.5208 | 19.4306 | 487.3833 | + +The selected adaptive policy scores placements as `-4 * setup - cost`. Its +aggregate weighted objective is slightly better than the included baselines. +It has worse flow time and tardiness than shortest-duration scheduling, loses +in some regimes, and executes two scoring primitives per placement comparison. +The random control selects the authored hybrid. Training and validation costs, +as well as per-regime final results, remain in scheduling.json. + +The policy grammar expresses linear feature combinations. This is evidence of +working evaluator feedback and policy selection, not discovery of an arbitrary +new scheduling algorithm. Multiple search seeds, stronger optimization baselines +and trace-driven workloads are required before claiming a useful scheduling gain. diff --git a/examples/discovery/results/manifest.json b/examples/discovery/results/manifest.json new file mode 100644 index 0000000..073456f --- /dev/null +++ b/examples/discovery/results/manifest.json @@ -0,0 +1,37 @@ +{ + "baseRevision": "61e11f428660cc215d4467153b47204280b81ff6", + "node": "v24.22.1", + "sourceSha256": { + "examples/discovery/experiments.test.cjs": "d8ea5c4077192b010e879d4ebec5d9f611ad0d962b464584bbc69b31eb557515", + "examples/discovery/scheduling-discovery.cjs": "816dadc6a6f5d7f371d4121ae215bdae4d2f4e9c872966e0f58d1e69509595f5", + "examples/discovery/scheduling.cjs": "3d8894c30f56d2b324b1ebd73a8f77361a4c5e9c6d9af7ed7d74d3c0e95f4401", + "examples/discovery/transfer.cjs": "d594c1e37ffe2d20310efcaadf702338cfd7e67b1625ae0f1ecdb005ddac219c", + "package.json": "36e372c4d01bb3b439c70ed4512c8e525d1092a935ed8485290782df3ebc1390", + "packages/fx-core/package.json": "f1c969ad9c921f4721a7320fa2f6e7f86fc2032861a3778ebc5928e286a59391", + "packages/fx-core/src/discovery/abstractions.ts": "2816285cba38041bfb87fd8c93bf707672106596d9cf41d616dbbeacd7fb77ee", + "packages/fx-core/src/discovery/index.ts": "d677da1ffed8fa46636d0aa8e2a2312e61932534fed45f2af82271ea9d992a92", + "packages/fx-core/src/discovery/model.ts": "14be2a8b12799012ae2de3073285cf715c666d3d5d3f004d4ccfc26ccdc03a39", + "packages/fx-core/src/discovery/optimization.ts": "6bd7c36a67b081fca0ca70be56c5419a87005b6463b90d72aaa2225e120fd086", + "packages/fx-core/src/discovery/runtime.ts": "60d478996cc47f915f5b87cf9a87f5e7f28235aafc66a5f32d8e8f407b930068", + "packages/fx-core/src/discovery/search.ts": "385d53ef14505733f5aeda3b55056be1f7c5642700fcecb5fbd7be973ab6dc03", + "packages/fx-core/src/discovery/state.ts": "fb107bbbded284f32a09cdb50909cd7a999343f27431fc77435771ee4a0d9942", + "packages/fx-core/src/index.ts": "8e5f25252124a7493fda966f31a30d2c42593d21ba9c8d1c6f41d6892199cfc8", + "packages/fx-core/src/__tests__/discovery-learning.test.ts": "c329454c55569e56c2d91793f0b57cb6ca50ec3b620d952e07eb774c3ed155b3", + "packages/fx-core/src/__tests__/discovery-runtime.test.ts": "1d0cb5e3af4e45717368911f7e082506788600482a4c30c05749b823851307bf", + ".github/workflows/ci.yml": "36ba15f487863752952456e800631c39aafce3adec3ffdc8f314806aff858b24" + }, + "checks": { + "libraryTests": 161, + "experimentTests": 4, + "lint": "passed", + "typeCheck": "passed", + "build": "passed", + "esmExport": "passed", + "discoveryLineCoverage": 98.35, + "discoveryBranchCoverage": 82.73 + }, + "outputSha256": { + "transfer.json": "2c4cbc6cbddc16cb306e07c85378a2fc9e461d996e7a9fdd045948ca1bcdb29f", + "scheduling.json": "527aefeef11e5872f91f852a8089e682e3bddbb3ee61a03881bcd5ec0d58f7c3" + } +} diff --git a/examples/discovery/results/scheduling.json b/examples/discovery/results/scheduling.json new file mode 100644 index 0000000..026dce7 --- /dev/null +++ b/examples/discovery/results/scheduling.json @@ -0,0 +1,621 @@ +{ + "experiment": "gpu-scheduling-policy-discovery", + "version": 1, + "seed": 42, + "proposalBudget": 100, + "evaluatedCandidates": 70, + "trainingSeeds": [ + 1, + 2, + 3, + 4, + 5, + 6 + ], + "validationSeeds": [ + 101, + 102, + 103, + 104 + ], + "testScenarios": [ + { + "seed": 201, + "regime": "mixed" + }, + { + "seed": 202, + "regime": "mixed" + }, + { + "seed": 203, + "regime": "mixed" + }, + { + "seed": 211, + "regime": "burst" + }, + { + "seed": 212, + "regime": "burst" + }, + { + "seed": 213, + "regime": "burst" + }, + { + "seed": 221, + "regime": "long" + }, + { + "seed": 222, + "regime": "long" + }, + { + "seed": 223, + "regime": "long" + }, + { + "seed": 231, + "regime": "many-models" + }, + { + "seed": 232, + "regime": "many-models" + }, + { + "seed": 233, + "regime": "many-models" + } + ], + "selected": { + "program": { + "version": 1, + "body": [ + { + "op": "setup", + "args": [ + -4 + ] + }, + { + "op": "cost", + "args": [ + -1 + ] + } + ] + }, + "training": { + "valid": true, + "score": -54.43705555555556, + "metrics": { + "meanFlow": 22.36111111111111, + "meanTardiness": 9.86111111111111, + "cost": 370.1333333333334, + "primitiveCalls": 2106, + "policyCalls": 1053 + } + }, + "validation": { + "valid": true, + "score": -48.44591666666666, + "metrics": { + "meanFlow": 18.78125, + "meanTardiness": 7.666666666666666, + "cost": 346.84999999999997, + "primitiveCalls": 1212, + "policyCalls": 606 + } + } + }, + "randomSearch": { + "program": { + "version": 1, + "body": [ + { + "op": "setup", + "args": [ + -1 + ] + }, + { + "op": "cost", + "args": [ + -1 + ] + } + ] + }, + "evaluatedCandidates": 92, + "training": { + "valid": true, + "score": -55.43618055555556, + "metrics": { + "meanFlow": 22.333333333333332, + "meanTardiness": 10.118055555555555, + "cost": 382.1666666666667, + "primitiveCalls": 2108, + "policyCalls": 1054 + } + }, + "validation": { + "valid": true, + "score": -48.92158333333334, + "metrics": { + "meanFlow": 18.5625, + "meanTardiness": 7.708333333333334, + "cost": 355.3999999999999, + "primitiveCalls": 1214, + "policyCalls": 607 + } + } + }, + "learningCost": { + "trainingPrimitiveCalls": 165074, + "validationPrimitiveCalls": 13573, + "randomSearchTrainingPrimitiveCalls": 330895, + "randomSearchValidationPrimitiveCalls": 12373 + }, + "heldOut": { + "fifo": { + "valid": true, + "score": -100.5154027777778, + "metrics": { + "meanFlow": 46.69097222222223, + "meanTardiness": 31.409722222222225, + "cost": 574.65, + "primitiveCalls": 2458, + "policyCalls": 2458 + } + }, + "shortest": { + "valid": true, + "score": -76.65659722222222, + "metrics": { + "meanFlow": 31.85763888888889, + "meanTardiness": 19.062500000000004, + "cost": 500.4166666666667, + "primitiveCalls": 2343, + "policyCalls": 2343 + } + }, + "deadline": { + "valid": true, + "score": -112.74918750000002, + "metrics": { + "meanFlow": 53.96527777777778, + "meanTardiness": 37.94097222222222, + "cost": 616.2333333333332, + "primitiveCalls": 2651, + "policyCalls": 2651 + } + }, + "locality": { + "valid": true, + "score": -82.07571527777777, + "metrics": { + "meanFlow": 37.21180555555556, + "meanTardiness": 23.274305555555557, + "cost": 488.06666666666666, + "primitiveCalls": 2440, + "policyCalls": 2440 + } + }, + "cheapest": { + "valid": true, + "score": -78.17877083333335, + "metrics": { + "meanFlow": 32.81944444444444, + "meanTardiness": 19.815972222222225, + "cost": 505.06666666666666, + "primitiveCalls": 2406, + "policyCalls": 2406 + } + }, + "hybrid": { + "valid": true, + "score": -77.07279861111111, + "metrics": { + "meanFlow": 32.52777777777778, + "meanTardiness": 19.385416666666668, + "cost": 496.2333333333333, + "primitiveCalls": 4838, + "policyCalls": 2419 + } + }, + "randomSearch": { + "valid": true, + "score": -77.07279861111111, + "metrics": { + "meanFlow": 32.52777777777778, + "meanTardiness": 19.385416666666668, + "cost": 496.2333333333333, + "primitiveCalls": 4838, + "policyCalls": 2419 + } + }, + "discovered": { + "valid": true, + "score": -76.36913888888888, + "metrics": { + "meanFlow": 32.520833333333336, + "meanTardiness": 19.430555555555557, + "cost": 487.3833333333334, + "primitiveCalls": 4728, + "policyCalls": 2364 + } + } + }, + "byRegime": { + "mixed": { + "fifo": { + "valid": true, + "score": -78.45688888888888, + "metrics": { + "meanFlow": 35.236111111111114, + "meanTardiness": 19.944444444444443, + "cost": 477.9333333333334, + "primitiveCalls": 565, + "policyCalls": 565 + } + }, + "shortest": { + "valid": true, + "score": -55.41683333333333, + "metrics": { + "meanFlow": 22.083333333333332, + "meanTardiness": 9.75, + "cost": 386.2, + "primitiveCalls": 505, + "policyCalls": 505 + } + }, + "deadline": { + "valid": true, + "score": -91.62600000000002, + "metrics": { + "meanFlow": 41.75, + "meanTardiness": 26, + "cost": 542.1999999999999, + "primitiveCalls": 603, + "policyCalls": 603 + } + }, + "locality": { + "valid": true, + "score": -62.45591666666667, + "metrics": { + "meanFlow": 26.694444444444446, + "meanTardiness": 13.680555555555557, + "cost": 404.2666666666667, + "primitiveCalls": 511, + "policyCalls": 511 + } + }, + "cheapest": { + "valid": true, + "score": -57.193694444444446, + "metrics": { + "meanFlow": 23.13888888888889, + "meanTardiness": 11.013888888888888, + "cost": 391.26666666666665, + "primitiveCalls": 473, + "policyCalls": 473 + } + }, + "hybrid": { + "valid": true, + "score": -58.04122222222222, + "metrics": { + "meanFlow": 23.569444444444443, + "meanTardiness": 11.444444444444445, + "cost": 395.1333333333334, + "primitiveCalls": 960, + "policyCalls": 480 + } + }, + "randomSearch": { + "valid": true, + "score": -58.04122222222222, + "metrics": { + "meanFlow": 23.569444444444443, + "meanTardiness": 11.444444444444445, + "cost": 395.1333333333334, + "primitiveCalls": 960, + "policyCalls": 480 + } + }, + "discovered": { + "valid": true, + "score": -58.04122222222222, + "metrics": { + "meanFlow": 23.569444444444443, + "meanTardiness": 11.444444444444445, + "cost": 395.1333333333334, + "primitiveCalls": 960, + "policyCalls": 480 + } + } + }, + "burst": { + "fifo": { + "valid": true, + "score": -74.21294444444445, + "metrics": { + "meanFlow": 33.86111111111111, + "meanTardiness": 19.583333333333332, + "cost": 443.2, + "primitiveCalls": 677, + "policyCalls": 677 + } + }, + "shortest": { + "valid": true, + "score": -53.181305555555554, + "metrics": { + "meanFlow": 21.930555555555557, + "meanTardiness": 10.208333333333334, + "cost": 358.7333333333333, + "primitiveCalls": 676, + "policyCalls": 676 + } + }, + "deadline": { + "valid": true, + "score": -84.60377777777778, + "metrics": { + "meanFlow": 40.02777777777778, + "meanTardiness": 24, + "cost": 482.2, + "primitiveCalls": 792, + "policyCalls": 792 + } + }, + "locality": { + "valid": true, + "score": -54.97652777777777, + "metrics": { + "meanFlow": 24.38888888888889, + "meanTardiness": 11.097222222222223, + "cost": 347.6666666666667, + "primitiveCalls": 679, + "policyCalls": 679 + } + }, + "cheapest": { + "valid": true, + "score": -53.65594444444444, + "metrics": { + "meanFlow": 22.055555555555557, + "meanTardiness": 10.305555555555555, + "cost": 362.79999999999995, + "primitiveCalls": 675, + "policyCalls": 675 + } + }, + "hybrid": { + "valid": true, + "score": -53.71422222222222, + "metrics": { + "meanFlow": 22.166666666666668, + "meanTardiness": 10.222222222222221, + "cost": 362.40000000000003, + "primitiveCalls": 1364, + "policyCalls": 682 + } + }, + "randomSearch": { + "valid": true, + "score": -53.71422222222222, + "metrics": { + "meanFlow": 22.166666666666668, + "meanTardiness": 10.222222222222221, + "cost": 362.40000000000003, + "primitiveCalls": 1364, + "policyCalls": 682 + } + }, + "discovered": { + "valid": true, + "score": -52.32152777777778, + "metrics": { + "meanFlow": 21.88888888888889, + "meanTardiness": 10.26388888888889, + "cost": 348.3333333333333, + "primitiveCalls": 1326, + "policyCalls": 663 + } + } + }, + "long": { + "fifo": { + "valid": true, + "score": -168.1066388888889, + "metrics": { + "meanFlow": 81.15277777777777, + "meanTardiness": 65.31944444444444, + "cost": 882.8000000000001, + "primitiveCalls": 701, + "policyCalls": 701 + } + }, + "shortest": { + "valid": true, + "score": -133.86247222222224, + "metrics": { + "meanFlow": 57.18055555555555, + "meanTardiness": 42.791666666666664, + "cost": 824.8000000000001, + "primitiveCalls": 674, + "policyCalls": 674 + } + }, + "deadline": { + "valid": true, + "score": -185.44752777777776, + "metrics": { + "meanFlow": 92.41666666666667, + "meanTardiness": 76.4861111111111, + "cost": 923.8666666666667, + "primitiveCalls": 705, + "policyCalls": 705 + } + }, + "locality": { + "valid": true, + "score": -144.57163888888888, + "metrics": { + "meanFlow": 68.79166666666667, + "meanTardiness": 53.263888888888886, + "cost": 780.7999999999998, + "primitiveCalls": 709, + "policyCalls": 709 + } + }, + "cheapest": { + "valid": true, + "score": -135.92966666666666, + "metrics": { + "meanFlow": 59.055555555555564, + "meanTardiness": 43.94444444444445, + "cost": 823.5999999999999, + "primitiveCalls": 720, + "policyCalls": 720 + } + }, + "hybrid": { + "valid": true, + "score": -132.7355, + "metrics": { + "meanFlow": 57.98611111111111, + "meanTardiness": 42.80555555555555, + "cost": 800.5999999999999, + "primitiveCalls": 1466, + "policyCalls": 733 + } + }, + "randomSearch": { + "valid": true, + "score": -132.7355, + "metrics": { + "meanFlow": 57.98611111111111, + "meanTardiness": 42.80555555555555, + "cost": 800.5999999999999, + "primitiveCalls": 1466, + "policyCalls": 733 + } + }, + "discovered": { + "valid": true, + "score": -132.75233333333333, + "metrics": { + "meanFlow": 58.68055555555555, + "meanTardiness": 43.44444444444445, + "cost": 790.1333333333333, + "primitiveCalls": 1432, + "policyCalls": 716 + } + } + }, + "many-models": { + "fifo": { + "valid": true, + "score": -81.2851388888889, + "metrics": { + "meanFlow": 36.51388888888889, + "meanTardiness": 20.791666666666668, + "cost": 494.6666666666667, + "primitiveCalls": 515, + "policyCalls": 515 + } + }, + "shortest": { + "valid": true, + "score": -64.16577777777778, + "metrics": { + "meanFlow": 26.236111111111114, + "meanTardiness": 13.5, + "cost": 431.9333333333332, + "primitiveCalls": 488, + "policyCalls": 488 + } + }, + "deadline": { + "valid": true, + "score": -89.31944444444444, + "metrics": { + "meanFlow": 41.666666666666664, + "meanTardiness": 25.277777777777775, + "cost": 516.6666666666666, + "primitiveCalls": 551, + "policyCalls": 551 + } + }, + "locality": { + "valid": true, + "score": -66.29877777777777, + "metrics": { + "meanFlow": 28.972222222222225, + "meanTardiness": 15.055555555555555, + "cost": 419.5333333333333, + "primitiveCalls": 541, + "policyCalls": 541 + } + }, + "cheapest": { + "valid": true, + "score": -65.93577777777777, + "metrics": { + "meanFlow": 27.027777777777775, + "meanTardiness": 14, + "cost": 442.59999999999997, + "primitiveCalls": 538, + "policyCalls": 538 + } + }, + "hybrid": { + "valid": true, + "score": -63.80025, + "metrics": { + "meanFlow": 26.38888888888889, + "meanTardiness": 13.069444444444445, + "cost": 426.79999999999995, + "primitiveCalls": 1048, + "policyCalls": 524 + } + }, + "randomSearch": { + "valid": true, + "score": -63.80025, + "metrics": { + "meanFlow": 26.38888888888889, + "meanTardiness": 13.069444444444445, + "cost": 426.79999999999995, + "primitiveCalls": 1048, + "policyCalls": 524 + } + }, + "discovered": { + "valid": true, + "score": -62.36147222222221, + "metrics": { + "meanFlow": 25.944444444444443, + "meanTardiness": 12.569444444444445, + "cost": 415.9333333333334, + "primitiveCalls": 1010, + "policyCalls": 505 + } + } + } + }, + "limitations": [ + "Synthetic, single-slot workers with deterministic durations and a four-tick model-load cost.", + "No production traces, cancellations, retries, network effects, or simulator/native conformance.", + "Objective is meanFlow + 0.25*meanTardiness + 0.08*totalCost; weights are authored.", + "This experiment evolves policies using evaluator feedback; abstraction transfer is tested separately.", + "The policy language is linear feature scoring, not arbitrary algorithm synthesis.", + "Evaluation count is bounded; actual primitive calls and validation cost are reported separately.", + "A negative result on a baseline or regime is retained, not hidden." + ] +} diff --git a/examples/discovery/results/transfer.json b/examples/discovery/results/transfer.json new file mode 100644 index 0000000..3ebffbd --- /dev/null +++ b/examples/discovery/results/transfer.json @@ -0,0 +1,642 @@ +{ + "experiment": "bounded-abstraction-transfer", + "version": 1, + "budgets": { + "maxCandidates": 256, + "maxPrimitiveCalls": 1024, + "maxLength": 4 + }, + "searchInputs": [ + -2, + 0, + 3 + ], + "verificationInputs": [ + -7, + -1, + 0.5, + 9, + 20 + ], + "learning": { + "tasks": 8, + "candidates": 172, + "primitiveCalls": 341, + "selected": [ + { + "abstraction": { + "name": "learned_899753c3410b7be8", + "arity": 2, + "body": [ + { + "op": "add", + "args": [ + { + "parameter": 0 + } + ] + }, + { + "op": "scale", + "args": [ + { + "parameter": 1 + } + ] + } + ] + }, + "compressionGain": 1, + "occurrences": 8, + "sources": [ + "08d2046794aad0acbb7a0bb0e9c909f6915390d6e248c0d62fc6cad76f9a1f4d", + "2f28219f8ad3381d4292d290a95ad00a21e78d9b68d03995a538f5f7405c25a4", + "4811134d0dddd67be8425217e75ed72388158ecc04beca5983f4f0820c4bda14", + "72b7bd73a773b619190e1c5f106fdc146bf03928edaa01a388e1e22aa68a31ea", + "7c58bc1afe1dda835d57d1c216e0ddfd7cc38f86a849e0057a0855c2299c4c30", + "90fe552dc1aec18a0e9d724e04ce7b1d47eb4926c1a701c23eae60ab4119dc10", + "9fdb65cd7d6e75abff553da737efb3dd6cc0cec783f286136a2423dade0be601", + "d5bca351c9aa348abf9ad066fb27ed73d628a5c4de81dd7550f778389935c82f" + ], + "bindings": [ + [ + 1, + 2 + ], + [ + 1, + 3 + ], + [ + 2, + 2 + ], + [ + 2, + 3 + ], + [ + 3, + 2 + ], + [ + 3, + 3 + ], + [ + 4, + 2 + ], + [ + 4, + 3 + ] + ] + } + ] + }, + "result": { + "tasks": 8, + "baselineSolved": 0, + "savedProgramsSolved": 8, + "learnedSolved": 8 + }, + "rows": [ + { + "task": 1, + "baseline": { + "solved": false, + "candidates": 256, + "primitiveCalls": 717 + }, + "savedPrograms": { + "solved": true, + "candidates": 182, + "primitiveCalls": 542 + }, + "learned": { + "solved": true, + "candidates": 182, + "primitiveCalls": 542 + }, + "program": { + "version": 1, + "body": [ + { + "op": "learned_899753c3410b7be8", + "args": [ + 3, + 3 + ] + }, + { + "op": "learned_899753c3410b7be8", + "args": [ + 4, + 2 + ] + } + ] + }, + "expanded": { + "version": 1, + "body": [ + { + "op": "add", + "args": [ + 3 + ] + }, + { + "op": "scale", + "args": [ + 3 + ] + }, + { + "op": "add", + "args": [ + 4 + ] + }, + { + "op": "scale", + "args": [ + 2 + ] + } + ] + } + }, + { + "task": 2, + "baseline": { + "solved": false, + "candidates": 256, + "primitiveCalls": 717 + }, + "savedPrograms": { + "solved": true, + "candidates": 206, + "primitiveCalls": 630 + }, + "learned": { + "solved": true, + "candidates": 206, + "primitiveCalls": 630 + }, + "program": { + "version": 1, + "body": [ + { + "op": "learned_899753c3410b7be8", + "args": [ + 4, + 3 + ] + }, + { + "op": "learned_899753c3410b7be8", + "args": [ + 2, + 2 + ] + } + ] + }, + "expanded": { + "version": 1, + "body": [ + { + "op": "add", + "args": [ + 4 + ] + }, + { + "op": "scale", + "args": [ + 3 + ] + }, + { + "op": "add", + "args": [ + 2 + ] + }, + { + "op": "scale", + "args": [ + 2 + ] + } + ] + } + }, + { + "task": 3, + "baseline": { + "solved": false, + "candidates": 256, + "primitiveCalls": 723 + }, + "savedPrograms": { + "solved": true, + "candidates": 169, + "primitiveCalls": 499 + }, + "learned": { + "solved": true, + "candidates": 169, + "primitiveCalls": 499 + }, + "program": { + "version": 1, + "body": [ + { + "op": "learned_899753c3410b7be8", + "args": [ + 3, + 2 + ] + }, + { + "op": "learned_899753c3410b7be8", + "args": [ + 4, + 3 + ] + } + ] + }, + "expanded": { + "version": 1, + "body": [ + { + "op": "add", + "args": [ + 3 + ] + }, + { + "op": "scale", + "args": [ + 2 + ] + }, + { + "op": "add", + "args": [ + 4 + ] + }, + { + "op": "scale", + "args": [ + 3 + ] + } + ] + } + }, + { + "task": 4, + "baseline": { + "solved": false, + "candidates": 256, + "primitiveCalls": 717 + }, + "savedPrograms": { + "solved": true, + "candidates": 210, + "primitiveCalls": 642 + }, + "learned": { + "solved": true, + "candidates": 210, + "primitiveCalls": 642 + }, + "program": { + "version": 1, + "body": [ + { + "op": "learned_899753c3410b7be8", + "args": [ + 4, + 3 + ] + }, + { + "op": "learned_899753c3410b7be8", + "args": [ + 4, + 2 + ] + } + ] + }, + "expanded": { + "version": 1, + "body": [ + { + "op": "add", + "args": [ + 4 + ] + }, + { + "op": "scale", + "args": [ + 3 + ] + }, + { + "op": "add", + "args": [ + 4 + ] + }, + { + "op": "scale", + "args": [ + 2 + ] + } + ] + } + }, + { + "task": 5, + "baseline": { + "solved": false, + "candidates": 256, + "primitiveCalls": 723 + }, + "savedPrograms": { + "solved": true, + "candidates": 167, + "primitiveCalls": 494 + }, + "learned": { + "solved": true, + "candidates": 167, + "primitiveCalls": 494 + }, + "program": { + "version": 1, + "body": [ + { + "op": "learned_899753c3410b7be8", + "args": [ + 3, + 2 + ] + }, + { + "op": "learned_899753c3410b7be8", + "args": [ + 3, + 3 + ] + } + ] + }, + "expanded": { + "version": 1, + "body": [ + { + "op": "add", + "args": [ + 3 + ] + }, + { + "op": "scale", + "args": [ + 2 + ] + }, + { + "op": "add", + "args": [ + 3 + ] + }, + { + "op": "scale", + "args": [ + 3 + ] + } + ] + } + }, + { + "task": 6, + "baseline": { + "solved": false, + "candidates": 256, + "primitiveCalls": 717 + }, + "savedPrograms": { + "solved": true, + "candidates": 207, + "primitiveCalls": 634 + }, + "learned": { + "solved": true, + "candidates": 207, + "primitiveCalls": 634 + }, + "program": { + "version": 1, + "body": [ + { + "op": "learned_899753c3410b7be8", + "args": [ + 4, + 3 + ] + }, + { + "op": "learned_899753c3410b7be8", + "args": [ + 2, + 3 + ] + } + ] + }, + "expanded": { + "version": 1, + "body": [ + { + "op": "add", + "args": [ + 4 + ] + }, + { + "op": "scale", + "args": [ + 3 + ] + }, + { + "op": "add", + "args": [ + 2 + ] + }, + { + "op": "scale", + "args": [ + 3 + ] + } + ] + } + }, + { + "task": 7, + "baseline": { + "solved": false, + "candidates": 256, + "primitiveCalls": 717 + }, + "savedPrograms": { + "solved": true, + "candidates": 195, + "primitiveCalls": 592 + }, + "learned": { + "solved": true, + "candidates": 195, + "primitiveCalls": 592 + }, + "program": { + "version": 1, + "body": [ + { + "op": "learned_899753c3410b7be8", + "args": [ + 4, + 2 + ] + }, + { + "op": "learned_899753c3410b7be8", + "args": [ + 3, + 3 + ] + } + ] + }, + "expanded": { + "version": 1, + "body": [ + { + "op": "add", + "args": [ + 4 + ] + }, + { + "op": "scale", + "args": [ + 2 + ] + }, + { + "op": "add", + "args": [ + 3 + ] + }, + { + "op": "scale", + "args": [ + 3 + ] + } + ] + } + }, + { + "task": 8, + "baseline": { + "solved": false, + "candidates": 256, + "primitiveCalls": 717 + }, + "savedPrograms": { + "solved": true, + "candidates": 197, + "primitiveCalls": 596 + }, + "learned": { + "solved": true, + "candidates": 197, + "primitiveCalls": 596 + }, + "program": { + "version": 1, + "body": [ + { + "op": "learned_899753c3410b7be8", + "args": [ + 4, + 2 + ] + }, + { + "op": "learned_899753c3410b7be8", + "args": [ + 4, + 3 + ] + } + ] + }, + "expanded": { + "version": 1, + "body": [ + { + "op": "add", + "args": [ + 4 + ] + }, + { + "op": "scale", + "args": [ + 2 + ] + }, + { + "op": "add", + "args": [ + 4 + ] + }, + { + "op": "scale", + "args": [ + 3 + ] + } + ] + } + } + ], + "limitations": [ + "Designed arithmetic benchmark; not evidence of general intelligence or GPU scheduling improvement.", + "Same candidate/primitive-call caps; vocabulary order and description length affect bounded enumeration.", + "Learning cost is reported separately and is not amortized away.", + "Macros preserve the original primitive language; the measured claim is search efficiency.", + "The saved-program baseline tests whether parameterized learning adds anything beyond storing snippets.", + "Verification inputs are excluded from search; test task specifications are input/output examples." + ] +} diff --git a/examples/discovery/scheduling-discovery.cjs b/examples/discovery/scheduling-discovery.cjs new file mode 100644 index 0000000..339b79e --- /dev/null +++ b/examples/discovery/scheduling-discovery.cjs @@ -0,0 +1,131 @@ +const { optimizePrograms } = require('../../packages/fx-core/dist'); +const { assess, baselines, features, random, workload } = require('./scheduling.cjs'); + +function runScheduling(options = {}) { + const budget = options.budget ?? 100; + const training = [1, 2, 3, 4, 5, 6].map((seed) => workload(seed)); + const validation = [101, 102, 103, 104].map((seed) => workload(seed)); + const test = ['mixed', 'burst', 'long', 'many-models'].flatMap((regime, i) => + [201, 202, 203].map((seed) => workload(seed + i * 10, 24, regime)), + ); + const seeds = Object.values(baselines); + const proposer = (feedback) => { + const rng = random(options.seed ?? 42); + const pick = (values) => values[Math.floor(rng() * values.length)]; + const atom = () => ({ op: pick(features), args: [pick([-4, -1, 1, 4])] }); + return (archive, iteration) => { + if (iteration < seeds.length) return seeds[iteration]; + if (!feedback) + return { version: 1, body: Array.from({ length: 1 + Math.floor(rng() * 5) }, atom) }; + const elite = archive + .filter((trial) => trial.evaluation?.valid) + .sort((a, b) => b.evaluation.score - a.evaluation.score) + .slice(0, 8); + const parent = + elite.length && rng() < 0.8 ? pick(elite).program : { version: 1, body: [atom()] }; + const body = parent.body.map((item) => ({ op: item.op, args: [...item.args] })); + const mutation = rng(); + if (mutation < 0.35 && body.length < 5) + body.splice(Math.floor(rng() * (body.length + 1)), 0, atom()); + else if (mutation < 0.5 && body.length > 1) body.splice(Math.floor(rng() * body.length), 1); + else body[Math.floor(rng() * body.length)] = atom(); + return { version: 1, body }; + }; + }; + const search = (feedback) => { + let trainingCalls = 0; + const optimization = optimizePrograms( + proposer(feedback), + (program) => { + const result = assess(program, training); + trainingCalls += result.metrics.primitiveCalls; + return result; + }, + budget, + ); + // Validation selects from a fixed shortlist. Test workloads are not consulted. + const shortlist = optimization.archive + .filter((trial) => trial.evaluation?.valid) + .sort((a, b) => b.evaluation.score - a.evaluation.score) + .slice(0, 8); + const validated = shortlist + .map((trial) => ({ ...trial, validation: assess(trial.program, validation) })) + .sort((a, b) => b.validation.score - a.validation.score); + const selected = validated[0]; + if (!selected) throw new Error('No valid policy'); + return { + optimization, + selected, + trainingCalls, + validationCalls: validated.reduce( + (sum, item) => sum + item.validation.metrics.primitiveCalls, + 0, + ), + }; + }; + const adaptive = search(true); + const randomSearch = search(false); + const { optimization, selected } = adaptive; + const policies = { + ...baselines, + randomSearch: randomSearch.selected.program, + discovered: selected.program, + }; + const heldOut = Object.fromEntries( + Object.entries(policies).map(([name, program]) => [name, assess(program, test)]), + ); + return { + experiment: 'gpu-scheduling-policy-discovery', + version: 1, + seed: options.seed ?? 42, + proposalBudget: budget, + evaluatedCandidates: optimization.evaluations, + trainingSeeds: training.map((row) => row.seed), + validationSeeds: validation.map((row) => row.seed), + testScenarios: test.map((row) => ({ seed: row.seed, regime: row.regime })), + selected: { + program: selected.program, + training: selected.evaluation, + validation: selected.validation, + }, + randomSearch: { + program: randomSearch.selected.program, + evaluatedCandidates: randomSearch.optimization.evaluations, + training: randomSearch.selected.evaluation, + validation: randomSearch.selected.validation, + }, + learningCost: { + trainingPrimitiveCalls: adaptive.trainingCalls, + validationPrimitiveCalls: adaptive.validationCalls, + randomSearchTrainingPrimitiveCalls: randomSearch.trainingCalls, + randomSearchValidationPrimitiveCalls: randomSearch.validationCalls, + }, + heldOut, + byRegime: Object.fromEntries( + ['mixed', 'burst', 'long', 'many-models'].map((regime) => [ + regime, + Object.fromEntries( + Object.entries(policies).map(([name, program]) => [ + name, + assess( + program, + test.filter((row) => row.regime === regime), + ), + ]), + ), + ]), + ), + limitations: [ + 'Synthetic, single-slot workers with deterministic durations and a four-tick model-load cost.', + 'No production traces, cancellations, retries, network effects, or simulator/native conformance.', + 'Objective is meanFlow + 0.25*meanTardiness + 0.08*totalCost; weights are authored.', + 'This experiment evolves policies using evaluator feedback; abstraction transfer is tested separately.', + 'The policy language is linear feature scoring, not arbitrary algorithm synthesis.', + 'Evaluation count is bounded; actual primitive calls and validation cost are reported separately.', + 'A negative result on a baseline or regime is retained, not hidden.', + ], + }; +} + +if (require.main === module) process.stdout.write(`${JSON.stringify(runScheduling(), null, 2)}\n`); +module.exports = { runScheduling }; diff --git a/examples/discovery/scheduling.cjs b/examples/discovery/scheduling.cjs new file mode 100644 index 0000000..a93cb07 --- /dev/null +++ b/examples/discovery/scheduling.cjs @@ -0,0 +1,193 @@ +const { ProgramRuntime } = require('../../packages/fx-core/dist'); + +const features = ['duration', 'cost', 'wait', 'slack', 'setup']; +const primitives = features.map((name) => ({ + name, + arity: 1, + execute: (state, [weight]) => ({ ...state, score: state.score + weight * state[name] }), +})); +const runtime = new ProgramRuntime(primitives); +const baselines = { + fifo: { version: 1, body: [{ op: 'wait', args: [1] }] }, + shortest: { version: 1, body: [{ op: 'duration', args: [-1] }] }, + deadline: { version: 1, body: [{ op: 'slack', args: [-1] }] }, + locality: { version: 1, body: [{ op: 'setup', args: [-1] }] }, + cheapest: { version: 1, body: [{ op: 'cost', args: [-1] }] }, + hybrid: { + version: 1, + body: [ + { op: 'setup', args: [-1] }, + { op: 'cost', args: [-1] }, + ], + }, +}; + +function random(seed) { + let value = seed >>> 0; + return () => { + value += 0x6d2b79f5; + let n = value; + n = Math.imul(n ^ (n >>> 15), n | 1); + n ^= n + Math.imul(n ^ (n >>> 7), n | 61); + return ((n ^ (n >>> 14)) >>> 0) / 4294967296; + }; +} + +function workload(seed, jobs = 24, regime = 'mixed') { + const rng = random(seed); + const workers = [ + { id: 0, memory: 8, speed: 1, price: 1, model: -1 }, + { id: 1, memory: 24, speed: 2, price: 1.8, model: -1 }, + { id: 2, memory: 24, speed: 3, price: 2.8, model: -1 }, + ]; + const tasks = Array.from({ length: jobs }, (_, id) => { + const arrival = regime === 'burst' ? (id < jobs / 2 ? 0 : 20) : Math.floor(rng() * 30); + const work = 2 + Math.floor(rng() * (regime === 'long' ? 60 : 24)); + return { + id, + arrival, + work, + memory: rng() < 0.3 ? 16 : 4, + model: Math.floor(rng() * (regime === 'many-models' ? 8 : 3)), + deadline: arrival + 5 + Math.floor(rng() * 24), + }; + }); + return { seed, regime, workers, jobs: tasks }; +} + +/** Discrete-event simulation; each worker has one slot and persistent model cache. */ +function simulate(program, scenario) { + const policyLength = runtime.expand(program).length; + const workers = scenario.workers.map((worker) => ({ ...worker, busy: false })); + const pending = scenario.jobs.map((job) => ({ ...job })).sort((a, b) => a.id - b.id); + if ( + new Set(workers.map((worker) => worker.id)).size !== workers.length || + new Set(pending.map((job) => job.id)).size !== pending.length || + pending.some( + (job) => + !Number.isFinite(job.arrival) || + job.arrival < 0 || + !Number.isFinite(job.work) || + job.work <= 0 || + !Number.isFinite(job.deadline) || + !Number.isFinite(job.memory) || + job.memory <= 0 || + !workers.some((worker) => worker.memory >= job.memory), + ) + ) + throw new Error('Invalid or infeasible workload'); + if ( + workers.some( + (worker) => + worker.speed <= 0 || + worker.price < 0 || + !Number.isFinite(worker.speed) || + !Number.isFinite(worker.price), + ) + ) { + throw new Error('Invalid worker'); + } + const active = []; + const completed = []; + const assignments = []; + let now = 0; + let primitiveCalls = 0; + let policyCalls = 0; + while (pending.length || active.length) { + for (let i = active.length - 1; i >= 0; i--) { + const execution = active[i]; + if (execution.finish <= now) { + workers.find((worker) => worker.id === execution.worker).busy = false; + completed.push(execution); + active.splice(i, 1); + } + } + while (true) { + let selected; + for (const job of pending) { + if (job.arrival > now) continue; + for (const worker of workers) { + if (worker.busy || job.memory > worker.memory) continue; + const setup = worker.model === job.model ? 0 : 4; + const duration = Math.ceil(job.work / worker.speed) + setup; + const result = runtime.run(program, { + score: 0, + duration, + cost: duration * worker.price, + wait: now - job.arrival, + slack: job.deadline - now - duration, + setup, + }); + primitiveCalls += result.primitiveCalls; + policyCalls++; + const score = result.state.score; + if (!selected || score > selected.score) selected = { job, worker, duration, score }; + } + } + if (!selected) break; + const { job, worker, duration } = selected; + pending.splice(pending.indexOf(job), 1); + worker.busy = true; + worker.model = job.model; + const execution = { + ...job, + worker: worker.id, + start: now, + finish: now + duration, + cost: duration * worker.price, + }; + active.push(execution); + assignments.push(execution); + } + const events = [ + ...pending.filter((job) => job.arrival > now).map((job) => job.arrival), + ...active.map((job) => job.finish), + ]; + if (events.length === 0) { + if (pending.length) throw new Error('Simulation deadlock'); + break; + } + now = Math.min(...events); + } + if ( + completed.length !== scenario.jobs.length || + new Set(completed.map((job) => job.id)).size !== completed.length + ) { + throw new Error('Lost or duplicated jobs'); + } + const count = Math.max(1, completed.length); + const meanFlow = completed.reduce((sum, job) => sum + job.finish - job.arrival, 0) / count; + const meanTardiness = + completed.reduce((sum, job) => sum + Math.max(0, job.finish - job.deadline), 0) / count; + const cost = completed.reduce((sum, job) => sum + job.cost, 0); + return { + score: -(meanFlow + 0.25 * meanTardiness + 0.08 * cost), + meanFlow, + meanTardiness, + cost, + jobs: completed.length, + primitiveCalls, + policyCalls, + policyLength, + assignments, + }; +} + +function assess(program, scenarios) { + const results = scenarios.map((scenario) => simulate(program, scenario)); + const mean = (key) => + results.reduce((sum, row) => sum + row[key], 0) / Math.max(1, results.length); + return { + valid: true, + score: mean('score'), + metrics: { + meanFlow: mean('meanFlow'), + meanTardiness: mean('meanTardiness'), + cost: mean('cost'), + primitiveCalls: results.reduce((sum, row) => sum + row.primitiveCalls, 0), + policyCalls: results.reduce((sum, row) => sum + row.policyCalls, 0), + }, + }; +} + +module.exports = { random, workload, simulate, assess, features, baselines, runtime }; diff --git a/examples/discovery/transfer.cjs b/examples/discovery/transfer.cjs new file mode 100644 index 0000000..4b76932 --- /dev/null +++ b/examples/discovery/transfer.cjs @@ -0,0 +1,177 @@ +const assert = require('node:assert/strict'); +const { + ProgramRuntime, + abstractionActions, + abstractProgram, + enumeratePrograms, + learnAbstractions, + programId, + searchProgram, +} = require('../../packages/fx-core/dist'); + +const primitives = [ + { name: 'add', arity: 1, execute: (state, [n]) => ({ value: state.value + n }) }, + { name: 'scale', arity: 1, execute: (state, [n]) => ({ value: state.value * n }) }, +]; +const actions = [1, 2, 3, 4] + .map((n) => ({ op: 'add', args: [n] })) + .concat([2, 3].map((n) => ({ op: 'scale', args: [n] }))); +const searchInputs = [-2, 0, 3]; +const verificationInputs = [-7, -1, 0.5, 9, 20]; +const budgets = { maxCandidates: 256, maxPrimitiveCalls: 1024, maxLength: 4 }; +const examples = (fn, inputs) => + inputs.map((value) => ({ input: { value }, expected: { value: fn(value) } })); + +function runTransfer() { + const runtime = new ProgramRuntime(primitives); + const corpus = []; + let learningCandidates = 0; + let learningCalls = 0; + for (const offset of [1, 2, 3, 4]) { + for (const factor of [2, 3]) { + const found = searchProgram( + runtime, + examples((x) => (x + offset) * factor, searchInputs), + enumeratePrograms(actions, 2), + { ...budgets, maxLength: 2 }, + ); + assert.ok( + found.program, + 'Initial tasks must actually be solved, not supplied as an oracle corpus', + ); + corpus.push({ version: 1, body: runtime.expand(found.program) }); + learningCandidates += found.candidates; + learningCalls += found.primitiveCalls; + } + } + const learned = learnAbstractions(corpus); + assert.ok(learned.length > 0, 'The corpus should support a compressive abstraction'); + const selected = [learned[0]]; + const withLibrary = new ProgramRuntime( + primitives, + selected.map((item) => item.abstraction), + ); + for (const program of corpus) { + assert.deepEqual( + withLibrary.expand(abstractProgram(program, selected[0].abstraction).program), + runtime.expand(program), + ); + } + const macroActions = abstractionActions(selected); + const saved = corpus.map((program, index) => ({ + name: `saved_${index}`, + arity: 0, + body: program.body, + })); + const withSavedPrograms = new ProgramRuntime(primitives, saved); + const savedActions = saved.map((item) => ({ op: item.name, args: [] })); + const baseActions = new Set(actions.map((action) => JSON.stringify(action))); + for (const action of macroActions) { + for (const expanded of withLibrary.expand({ version: 1, body: [action] })) { + assert.ok( + baseActions.has(JSON.stringify(expanded)), + 'Library must not introduce new primitive actions/constants', + ); + } + } + // Entire task compositions withheld from learning; search still gets I/O specs. + const targets = [ + [3, 3, 4, 2], + [4, 3, 2, 2], + [4, 3, 3, 2], + [4, 3, 4, 2], + [3, 2, 3, 3], + [4, 3, 2, 3], + [4, 2, 3, 3], + [4, 2, 4, 3], + ]; + assert.equal( + new Set(targets.map(([a, b, c, d]) => JSON.stringify([b * d, (a * b + c) * d]))).size, + targets.length, + 'Held-out tasks must represent distinct functions', + ); + const rows = targets.map(([a, b, c, d], index) => { + const fn = (x) => ((x + a) * b + c) * d; + const spec = examples(fn, searchInputs); + const baseline = searchProgram( + runtime, + spec, + enumeratePrograms(actions, budgets.maxLength), + budgets, + ); + const learnedSearch = searchProgram( + withLibrary, + spec, + enumeratePrograms([...actions, ...macroActions], budgets.maxLength), + budgets, + ); + const savedSearch = searchProgram( + withSavedPrograms, + spec, + enumeratePrograms([...actions, ...savedActions], budgets.maxLength), + budgets, + ); + const verify = (machine, result) => + Boolean(result.program) && + examples(fn, verificationInputs).every( + (example) => + machine.run(result.program, example.input).state.value === example.expected.value, + ); + const expanded = learnedSearch.program && { + version: 1, + body: withLibrary.expand(learnedSearch.program), + }; + if (expanded) assert.ok(!corpus.some((program) => programId(program) === programId(expanded))); + return { + task: index + 1, + baseline: { + solved: verify(runtime, baseline), + candidates: baseline.candidates, + primitiveCalls: baseline.primitiveCalls, + }, + savedPrograms: { + solved: verify(withSavedPrograms, savedSearch), + candidates: savedSearch.candidates, + primitiveCalls: savedSearch.primitiveCalls, + }, + learned: { + solved: verify(withLibrary, learnedSearch), + candidates: learnedSearch.candidates, + primitiveCalls: learnedSearch.primitiveCalls, + }, + program: learnedSearch.program, + expanded, + }; + }); + return { + experiment: 'bounded-abstraction-transfer', + version: 1, + budgets, + searchInputs, + verificationInputs, + learning: { + tasks: corpus.length, + candidates: learningCandidates, + primitiveCalls: learningCalls, + selected, + }, + result: { + tasks: rows.length, + baselineSolved: rows.filter((row) => row.baseline.solved).length, + savedProgramsSolved: rows.filter((row) => row.savedPrograms.solved).length, + learnedSolved: rows.filter((row) => row.learned.solved).length, + }, + rows, + limitations: [ + 'Designed arithmetic benchmark; not evidence of general intelligence or GPU scheduling improvement.', + 'Same candidate/primitive-call caps; vocabulary order and description length affect bounded enumeration.', + 'Learning cost is reported separately and is not amortized away.', + 'Macros preserve the original primitive language; the measured claim is search efficiency.', + 'The saved-program baseline tests whether parameterized learning adds anything beyond storing snippets.', + 'Verification inputs are excluded from search; test task specifications are input/output examples.', + ], + }; +} + +if (require.main === module) process.stdout.write(`${JSON.stringify(runTransfer(), null, 2)}\n`); +module.exports = { runTransfer }; diff --git a/package.json b/package.json index 47e079b..d7186f4 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,10 @@ "test": "npm run test --workspaces", "clean": "npm run clean --workspaces && rm -rf dist", "lint": "npm run lint --workspaces", - "type-check": "npm run type-check --workspaces" + "type-check": "npm run type-check --workspaces", + "test:experiments": "npm run build && node --test examples/discovery/experiments.test.cjs", + "experiment:transfer": "npm run build --silent && node examples/discovery/transfer.cjs", + "experiment:scheduling": "npm run build --silent && node examples/discovery/scheduling-discovery.cjs" }, "keywords": [ "functional-programming", diff --git a/packages/fx-core/package.json b/packages/fx-core/package.json index 132503a..9b0c2d2 100644 --- a/packages/fx-core/package.json +++ b/packages/fx-core/package.json @@ -5,6 +5,11 @@ "main": "dist/index.js", "types": "dist/index.d.ts", "exports": { + "./discovery": { + "types": "./dist/discovery/index.d.ts", + "import": "./dist/discovery/index.js", + "require": "./dist/discovery/index.js" + }, ".": { "types": "./dist/index.d.ts", "import": "./dist/index.js", diff --git a/packages/fx-core/src/__tests__/discovery-learning.test.ts b/packages/fx-core/src/__tests__/discovery-learning.test.ts new file mode 100644 index 0000000..1e6225e --- /dev/null +++ b/packages/fx-core/src/__tests__/discovery-learning.test.ts @@ -0,0 +1,249 @@ +import { + Abstraction, + Program, + ProgramRuntime, + abstractProgram, + abstractionActions, + enumeratePrograms, + learnAbstractions, + optimizePrograms, + searchProgram, +} from '../discovery'; + +const corpus: Program[] = Array.from({ length: 4 }, (_, i) => + [2, 3].map((scale) => ({ + version: 1 as const, + body: [ + { op: 'add', args: [i + 1] }, + { op: 'scale', args: [scale] }, + ], + })), +).flat(); +const runtime = new ProgramRuntime([ + { + name: 'add', + arity: 1, + execute: (state: { value: number }, args) => ({ value: state.value + args[0]! }), + }, + { + name: 'scale', + arity: 1, + execute: (state: { value: number }, args) => ({ value: state.value * args[0]! }), + }, +]); +const actions = [1, 2, 3, 4] + .map((value) => ({ op: 'add', args: [value] })) + .concat([2, 3].map((value) => ({ op: 'scale', args: [value] }))); + +describe('Learning and bounded discovery', () => { + it('infers shared parameters and fixed constants, including an unseen binding', () => { + const programs: Program[] = Array.from({ length: 8 }, (_, i) => ({ + version: 1, + body: [ + { op: 'add', args: [i + 1] }, + { op: 'scale', args: [i + 1] }, + { op: 'add', args: [10] }, + ], + })); + const learned = learnAbstractions(programs)[0]!; + expect(learned.abstraction.body).toEqual([ + { op: 'add', args: [{ parameter: 0 }] }, + { op: 'scale', args: [{ parameter: 0 }] }, + { op: 'add', args: [10] }, + ]); + const expanded = new ProgramRuntime( + [ + { + name: 'add', + arity: 1, + execute: (state: { value: number }, args) => ({ value: state.value + args[0]! }), + }, + { + name: 'scale', + arity: 1, + execute: (state: { value: number }, args) => ({ value: state.value * args[0]! }), + }, + ], + [learned.abstraction], + ); + expect( + expanded.run( + { version: 1, body: [{ op: learned.abstraction.name, args: [9] }] }, + { value: 2 }, + ).state.value, + ).toBe(109); + expect(learnAbstractions(corpus, { maxParameters: 1 })).toEqual([]); + }); + it('learns parameters from varied programs and preserves expansion exactly', () => { + const learned = learnAbstractions(corpus); + expect(learned).toHaveLength(1); + const candidate = learned[0]!; + expect(candidate.abstraction.arity).toBe(2); + expect(candidate.compressionGain).toBeGreaterThan(0); + expect(candidate.sources).toHaveLength(8); + const withLibrary = new ProgramRuntime( + [ + { + name: 'add', + arity: 1, + execute: (state: { value: number }, args) => ({ value: state.value + args[0]! }), + }, + { + name: 'scale', + arity: 1, + execute: (state: { value: number }, args) => ({ value: state.value * args[0]! }), + }, + ], + [candidate.abstraction], + ); + for (const original of corpus) { + const rewritten = abstractProgram(original, candidate.abstraction).program; + expect(rewritten.body).toHaveLength(1); + expect(withLibrary.expand(rewritten)).toEqual(runtime.expand(original)); + for (const value of [-9, 0, 0.25, 13]) { + expect(withLibrary.run(rewritten, { value })).toEqual(runtime.run(original, { value })); + } + } + expect(abstractionActions(learned)).toHaveLength(8); + expect(learnAbstractions([...corpus, ...corpus])).toEqual(learned); + }); + + it('preserves tied parameters/constants and never matches inconsistent bindings', () => { + const template: Abstraction = { + name: 'paired', + arity: 1, + body: [ + { op: 'add', args: [{ parameter: 0 }] }, + { op: 'scale', args: [{ parameter: 0 }] }, + ], + }; + const consistent: Program = { + version: 1, + body: [ + { op: 'add', args: [2] }, + { op: 'scale', args: [2] }, + ], + }; + expect(abstractProgram(consistent, template).bindings).toEqual([[2]]); + expect(abstractProgram(corpus[0]!, template).bindings).toEqual([]); + expect(learnAbstractions([consistent, consistent, consistent])).toEqual([]); + expect(() => + learnAbstractions([{ version: 1, body: [{ op: 'add', args: [{ parameter: 0 }] }] }]), + ).toThrow('closed'); + expect(() => abstractProgram(consistent, { ...template, arity: 2 })).toThrow('dense'); + }); + + it('counts non-overlapping occurrences and charges library definition cost', () => { + const repeat: Program = { + version: 1, + body: Array.from({ length: 4 }, () => ({ op: 'add', args: [1] })), + }; + const other: Program = { version: 1, body: [...repeat.body, { op: 'scale', args: [2] }] }; + const learned = learnAbstractions([repeat, other], { maxLength: 2 }); + expect(learned[0]!.occurrences).toBe(4); + expect(learned[0]!.compressionGain).toBe(7); + expect(learnAbstractions(corpus.slice(0, 2))).toEqual([]); + }); + + it('searches under both budgets and returns replayable evidence including errors', () => { + const cases = [ + { input: { value: 2 }, expected: { value: 6 } }, + { input: { value: 5 }, expected: { value: 12 } }, + ]; + const result = searchProgram(runtime, cases, enumeratePrograms(actions, 3), { + maxCandidates: 100, + maxPrimitiveCalls: 200, + maxLength: 3, + }); + expect(result.program).toBeDefined(); + expect(runtime.run(result.program!, { value: 7 }).state).toEqual({ value: 16 }); + expect(result.trials.at(-1)?.status).toBe('passed'); + expect(result.primitiveCalls).toBe( + result.trials.reduce((sum, trial) => sum + trial.primitiveCalls, 0), + ); + const bounded = searchProgram(runtime, cases, enumeratePrograms(actions, 4), { + maxCandidates: 2, + maxPrimitiveCalls: 2, + maxLength: 4, + }); + expect(bounded.exhausted).toBe(true); + expect(bounded.candidates).toBeLessThanOrEqual(2); + expect(bounded.primitiveCalls).toBeLessThanOrEqual(2); + const errors = searchProgram( + runtime, + cases, + [{ version: 1, body: [{ op: 'missing', args: [] }] }], + { + maxCandidates: 1, + maxPrimitiveCalls: 10, + maxLength: 4, + }, + ); + expect(errors.trials[0]?.status).toBe('error'); + }); + + it('charges failing primitive invocations and refuses a run that cannot fit', () => { + const failing = new ProgramRuntime([ + { + name: 'fail', + arity: 0, + execute: (_state: { value: number }) => { + throw new Error('failure'); + }, + }, + ]); + const result = searchProgram( + failing, + [{ input: { value: 0 }, expected: { value: 1 } }], + [{ version: 1, body: [{ op: 'fail', args: [] }] }], + { maxCandidates: 1, maxPrimitiveCalls: 1, maxLength: 2 }, + ); + expect(result.primitiveCalls).toBe(1); + const capped = searchProgram( + runtime, + [{ input: { value: 0 }, expected: { value: 9 } }], + [{ version: 1, body: actions.slice(0, 2) }], + { maxCandidates: 1, maxPrimitiveCalls: 1, maxLength: 2 }, + ); + expect(capped.trials[0]?.status).toBe('budget_exhausted'); + expect(capped.primitiveCalls).toBe(0); + }); + + it('selects only valid finite scores, retains failures and bounds duplicate proposals', () => { + const proposals: Program[] = [{ version: 1, body: [] }, ...corpus]; + const result = optimizePrograms( + proposals, + (program) => ({ valid: program.body.length > 0, score: 100 - program.body.length }), + 3, + ); + expect(result.best?.program.body).toHaveLength(2); + expect(result.evaluations).toBe(3); + expect(optimizePrograms(corpus, () => ({ valid: true, score: NaN }), 2).best).toBeUndefined(); + const duplicates = function* (): Generator { + while (true) yield corpus[0]!; + }; + expect(optimizePrograms(duplicates(), () => ({ valid: true, score: 1 }), 3).evaluations).toBe( + 1, + ); + }); + + it('feeds immutable, independently scored outcomes back to the next proposer', () => { + const observations: number[] = []; + const result = optimizePrograms( + (archive, iteration) => { + if (archive[0]) { + observations.push(archive[0].evaluation!.score); + expect(Object.isFrozen(archive[0].evaluation)).toBe(true); + expect(() => Object.assign(archive[0]!.evaluation!, { score: 999 })).toThrow(); + expect(Object.isFrozen(archive[0].program.body)).toBe(true); + expect(Object.isFrozen(archive[0].program.body[0]?.args)).toBe(true); + } + return corpus[iteration]!; + }, + (program) => ({ valid: true, score: program.body[0]!.args[0] as number }), + 3, + ); + expect(observations).toEqual([1, 1]); + expect(result.best?.evaluation?.score).toBe(2); + }); +}); diff --git a/packages/fx-core/src/__tests__/discovery-runtime.test.ts b/packages/fx-core/src/__tests__/discovery-runtime.test.ts new file mode 100644 index 0000000..cf2b3b5 --- /dev/null +++ b/packages/fx-core/src/__tests__/discovery-runtime.test.ts @@ -0,0 +1,147 @@ +import { createAgent, createPlan } from '../index'; +import { Abstraction, Primitive, Program, ProgramRuntime } from '../discovery'; + +interface State { + readonly value: number; + readonly [key: string]: number; +} +const primitives: readonly Primitive[] = [ + { + name: 'add', + arity: 1, + execute: (state, args) => ({ ...state, value: state.value + args[0]! }), + }, + { + name: 'scale', + arity: 1, + execute: (state, args) => ({ ...state, value: state.value * args[0]! }), + }, +]; +const affine: Abstraction = { + name: 'affine', + arity: 2, + body: [ + { op: 'add', args: [{ parameter: 0 }] }, + { op: 'scale', args: [{ parameter: 1 }] }, + ], +}; +const program: Program = { version: 1, body: [{ op: 'affine', args: [3, 2] }] }; + +describe('Inspectable program execution', () => { + it('round-trips data and executes parameterized composition through an existing FX agent', async () => { + const runtime = new ProgramRuntime(primitives, [affine]); + const restored: Program = JSON.parse(JSON.stringify(program)); + expect(runtime.expand(restored)).toEqual([ + { op: 'add', args: [3] }, + { op: 'scale', args: [2] }, + ]); + expect(runtime.run(restored, { value: 4 })).toEqual({ + state: { value: 14 }, + primitiveCalls: 2, + }); + const agent = createAgent('discovered', createPlan('plan', [runtime.asStep(restored)])); + expect(await agent.start({ value: 4 })).toEqual({ value: 14 }); + }); + + it('validates the whole program before effects and rejects invalid serialized syntax', () => { + const execute = jest.fn((state: State) => state); + const runtime = new ProgramRuntime([{ name: 'touch', arity: 0, execute }]); + expect(() => + runtime.run( + { + version: 1, + body: [ + { op: 'touch', args: [] }, + { op: 'missing', args: [] }, + ], + }, + { value: 1 }, + ), + ).toThrow('Unknown'); + expect(execute).not.toHaveBeenCalled(); + expect(() => + runtime.expand({ version: 1, body: [{ op: 'touch', args: [{ parameter: 0 }] }] }), + ).toThrow('Unbound'); + expect(() => + runtime.expand({ version: 1, body: [{ op: 'touch', args: [Infinity] }] }), + ).toThrow(); + expect(() => runtime.expand(JSON.parse('{"version":1,"body":[],"source":"eval()"}'))).toThrow(); + expect(() => new ProgramRuntime(primitives).expand(program)).toThrow('Unknown'); + }); + + it('rejects cycles, name collisions, wrong arities and expanded resource overruns', () => { + expect( + () => + new ProgramRuntime(primitives, [ + { name: 'a', arity: 0, body: [{ op: 'b', args: [] }] }, + { name: 'b', arity: 0, body: [{ op: 'a', args: [] }] }, + ]), + ).toThrow('Recursive'); + expect(() => new ProgramRuntime(primitives, [{ ...affine, name: 'add' }])).toThrow('Duplicate'); + expect(() => new ProgramRuntime(primitives, [affine], { maxPrimitiveCalls: 1 })).toThrow( + 'primitive call limit', + ); + expect(() => + new ProgramRuntime(primitives, [affine]).expand({ + version: 1, + body: [{ op: 'affine', args: [2] }], + }), + ).toThrow('arity'); + expect(() => + new ProgramRuntime(primitives).expand({ version: 1, body: [{ op: 'add', args: [] }] }), + ).toThrow('arity'); + expect(() => new ProgramRuntime(primitives, [], { maxCallDepth: 0 })).toThrow(); + }); + + it('isolates and freezes nested state and rejects non-JSON outputs', () => { + const input = { value: 1, nested: { values: [2] } }; + const runtime = new ProgramRuntime([ + { + name: 'mutate', + arity: 0, + execute: (state) => { + state.nested.values.push(3); + return state; + }, + }, + ]); + expect(() => runtime.run({ version: 1, body: [{ op: 'mutate', args: [] }] }, input)).toThrow(); + expect(input.nested.values).toEqual([2]); + const overflow = new ProgramRuntime([ + { name: 'overflow', arity: 0, execute: () => ({ value: Infinity }) }, + ]); + expect(() => + overflow.run({ version: 1, body: [{ op: 'overflow', args: [] }] }, { value: 0 }), + ).toThrow('JSON'); + }); + + it('snapshots definitions and compiled programs against later caller mutations', async () => { + const mutable = { name: 'plus', arity: 0, body: [{ op: 'add', args: [2] }] }; + const runtime = new ProgramRuntime(primitives, [mutable]); + mutable.body[0]!.args[0] = 99; + const callerProgram = { version: 1 as const, body: [{ op: 'plus', args: [] as number[] }] }; + const step = runtime.asStep(callerProgram); + callerProgram.body[0]!.op = 'missing'; + expect(await step({ value: 1 })).toEqual({ value: 3 }); + }); + + it('rejects excessive nesting, cycles and non-JSON state before execution', () => { + const runtime = new ProgramRuntime(primitives); + const cycle: { nested?: unknown } = {}; + cycle.nested = cycle; + for (const state of [cycle, new Date(), [], { value: undefined }]) { + expect(() => runtime.run({ version: 1, body: [] }, state as unknown as State)).toThrow(); + } + expect( + () => + new ProgramRuntime( + primitives, + [ + { name: 'outer', arity: 0, body: [{ op: 'inner', args: [] }] }, + { name: 'inner', arity: 0, body: [{ op: 'add', args: [1] }] }, + ], + { maxCallDepth: 1 }, + ), + ).toThrow('depth limit'); + }); +}); diff --git a/packages/fx-core/src/discovery/abstractions.ts b/packages/fx-core/src/discovery/abstractions.ts new file mode 100644 index 0000000..b176d14 --- /dev/null +++ b/packages/fx-core/src/discovery/abstractions.ts @@ -0,0 +1,211 @@ +import { createHash } from 'crypto'; +import { + Abstraction, + Argument, + Instruction, + Program, + abstractionSchema, + positiveInteger, + programId, + programSchema, + programSize, +} from './model'; + +export interface LearnedAbstraction { + readonly abstraction: Abstraction; + readonly compressionGain: number; + readonly occurrences: number; + readonly sources: readonly string[]; + /** Observed argument tuples, useful as a conservative search vocabulary. */ + readonly bindings: readonly (readonly number[])[]; +} +export interface LearningOptions { + readonly maxLength?: number; + readonly minOccurrences?: number; + readonly maxParameters?: number; +} +interface Occurrence { + readonly body: readonly Instruction[]; +} + +function numeric(arg: Argument): number { + if (typeof arg !== 'number') throw new Error('Learning requires expanded, closed programs'); + return arg; +} + +/** Match a template with consistent parameter bindings; constants stay constants. */ +function match( + template: Abstraction, + body: readonly Instruction[], + offset: number, +): readonly number[] | undefined { + const args: number[] = []; + for (let i = 0; i < template.body.length; i++) { + const expected = template.body[i]; + const actual = body[offset + i]; + if ( + !expected || + !actual || + expected.op !== actual.op || + expected.args.length !== actual.args.length + ) + return undefined; + for (let j = 0; j < expected.args.length; j++) { + const pattern = expected.args[j]; + const value = actual.args[j]; + if (pattern === undefined || typeof value !== 'number') return undefined; + if (typeof pattern === 'number') { + if (pattern !== value) return undefined; + } else { + const prior = args[pattern.parameter]; + if (prior !== undefined && prior !== value) return undefined; + args[pattern.parameter] = value; + } + } + } + if (Array.from({ length: template.arity }, (_, i) => args[i]).some((arg) => arg === undefined)) + return undefined; + return args; +} + +/** Exact structural factoring, with non-overlapping replacements. */ +export function abstractProgram( + value: Program, + valueTemplate: Abstraction, +): { + readonly program: Program; + readonly bindings: readonly (readonly number[])[]; +} { + const program = programSchema.parse(value); + const template = abstractionSchema.parse(valueTemplate); + // Inferred templates always have dense parameters, but public callers may not. + const used = new Set( + template.body.flatMap((item) => + item.args.flatMap((arg) => (typeof arg === 'number' ? [] : [arg.parameter])), + ), + ); + if (used.size !== template.arity || [...used].some((index) => index >= template.arity)) { + throw new Error('Abstraction parameters must be dense and all used'); + } + const body: Instruction[] = []; + const bindings: (readonly number[])[] = []; + for (let offset = 0; offset < program.body.length; ) { + const args = match(template, program.body, offset); + if (args) { + body.push({ op: template.name, args }); + bindings.push(args); + offset += template.body.length; + } else { + const item = program.body[offset++]; + if (item) body.push(item); + } + } + return { program: { version: 1, body }, bindings }; +} + +function inferTemplate(occurrences: readonly Occurrence[]): Abstraction { + const exemplar = occurrences[0]; + if (!exemplar) throw new Error('No occurrences'); + const parameters = new Map(); + const body = exemplar.body.map( + (item, i): Instruction => ({ + op: item.op, + args: item.args.map((_, j): Argument => { + const values = occurrences.map((occurrence) => { + const value = occurrence.body[i]?.args[j]; + if (value === undefined) throw new Error('Incompatible occurrences'); + return numeric(value); + }); + const first = values[0]; + if (first === undefined) throw new Error('No argument values'); + if (values.every((value) => value === first)) return first; + // Equal value vectors preserve parameter equality across positions. + const key = JSON.stringify(values); + let index = parameters.get(key); + if (index === undefined) { + index = parameters.size; + parameters.set(key, index); + } + return { parameter: index }; + }), + }), + ); + const digest = createHash('sha256').update(JSON.stringify(body)).digest('hex').slice(0, 16); + return { name: `learned_${digest}`, arity: parameters.size, body }; +} + +/** + * Anti-unify repeated contiguous structures from successful programs supplied by + * the caller. No evaluator or task outcome is inferred from the program itself. + * Each returned candidate is scored independently, not as a joint library. + */ +export function learnAbstractions( + successfulPrograms: readonly Program[], + options: LearningOptions = {}, +): readonly LearnedAbstraction[] { + const maxLength = positiveInteger(options.maxLength ?? 6, 'maxLength', 32); + const minOccurrences = positiveInteger(options.minOccurrences ?? 3, 'minOccurrences', 100_000); + const maxParameters = positiveInteger(options.maxParameters ?? 8, 'maxParameters', 16); + if (successfulPrograms.length > 1000) throw new Error('Learning corpus exceeds 1000 programs'); + const unique = new Map( + successfulPrograms.map((value) => { + const program = programSchema.parse(value); + program.body.forEach((item) => item.args.forEach(numeric)); + return [programId(program), program] as const; + }), + ); + const corpus = [...unique].sort(([a], [b]) => a.localeCompare(b)); + const groups = new Map(); + for (const [, program] of corpus) { + for (let length = 2; length <= Math.min(maxLength, program.body.length); length++) { + for (let start = 0; start + length <= program.body.length; start++) { + const body = program.body.slice(start, start + length); + const signature = JSON.stringify(body.map((item) => [item.op, item.args.length])); + const group = groups.get(signature) ?? []; + group.push({ body }); + groups.set(signature, group); + } + } + } + const learned: LearnedAbstraction[] = []; + for (const occurrences of groups.values()) { + if (occurrences.length < minOccurrences) continue; + const abstraction = inferTemplate(occurrences); + if (abstraction.arity > maxParameters) continue; + const sources: string[] = []; + const bindings = new Map(); + let count = 0; + let savings = 0; + for (const [id, original] of corpus) { + const rewritten = abstractProgram(original, abstraction); + if (rewritten.bindings.length === 0) continue; + sources.push(id); + count += rewritten.bindings.length; + savings += programSize(original) - programSize(rewritten.program); + rewritten.bindings.forEach((args) => bindings.set(JSON.stringify(args), args)); + } + const definitionCost = + programSize({ version: 1, body: abstraction.body }) + abstraction.arity + 1; + const compressionGain = savings - definitionCost; + if (count >= minOccurrences && sources.length >= 2 && compressionGain > 0) { + learned.push({ + abstraction, + compressionGain, + occurrences: count, + sources, + bindings: [...bindings].sort(([a], [b]) => a.localeCompare(b)).map(([, args]) => args), + }); + } + } + return learned.sort( + (a, b) => + b.compressionGain - a.compressionGain || a.abstraction.name.localeCompare(b.abstraction.name), + ); +} + +/** Calls only bindings witnessed in the training corpus; no new constants. */ +export function abstractionActions(learned: readonly LearnedAbstraction[]): readonly Instruction[] { + return learned.flatMap((item) => + item.bindings.map((args) => ({ op: item.abstraction.name, args })), + ); +} diff --git a/packages/fx-core/src/discovery/index.ts b/packages/fx-core/src/discovery/index.ts new file mode 100644 index 0000000..a7e819e --- /dev/null +++ b/packages/fx-core/src/discovery/index.ts @@ -0,0 +1,6 @@ +export * from './model'; +export * from './state'; +export * from './runtime'; +export * from './abstractions'; +export * from './search'; +export * from './optimization'; diff --git a/packages/fx-core/src/discovery/model.ts b/packages/fx-core/src/discovery/model.ts new file mode 100644 index 0000000..b21fe7d --- /dev/null +++ b/packages/fx-core/src/discovery/model.ts @@ -0,0 +1,82 @@ +import { createHash } from 'crypto'; +import { z } from 'zod'; + +/** A deliberately small, serializable language. No source code or eval. */ +export type Argument = number | { readonly parameter: number }; +export interface Instruction { + readonly op: string; + readonly args: readonly Argument[]; +} +export interface Program { + readonly version: 1; + readonly body: readonly Instruction[]; +} +export interface Abstraction { + readonly name: string; + readonly arity: number; + readonly body: readonly Instruction[]; +} +export interface ExpandedInstruction { + readonly op: string; + readonly args: readonly number[]; +} + +const name = z + .string() + .regex(/^[a-zA-Z][a-zA-Z0-9_.-]*$/) + .max(100); +const argument = z.union([ + z.number().finite(), + z.object({ parameter: z.number().int().min(0).max(15) }).strict(), +]); +const instruction = z.object({ op: name, args: z.array(argument).max(16) }).strict(); +export const programSchema: z.ZodType = z + .object({ + version: z.literal(1), + body: z.array(instruction).max(128), + }) + .strict(); +export const abstractionSchema: z.ZodType = z + .object({ + name, + arity: z.number().int().min(0).max(16), + body: z.array(instruction).min(1).max(128), + }) + .strict(); + +/** Own and freeze the program so a proposer cannot rewrite recorded evidence. */ +export function parseProgram(value: unknown): Program { + const parsed = programSchema.parse(value); + return Object.freeze({ + version: 1 as const, + body: Object.freeze( + parsed.body.map((item) => + Object.freeze({ + op: item.op, + args: Object.freeze( + item.args.map((arg) => (typeof arg === 'number' ? arg : Object.freeze({ ...arg }))), + ), + }), + ), + ), + }); +} + +/** Canonical for this schema: parsing fixes property order and rejects extra fields. */ +export function programId(program: Program): string { + return createHash('sha256') + .update(JSON.stringify(parseProgram(program))) + .digest('hex'); +} + +/** Description length in instruction/argument nodes, not runtime cost. */ +export function programSize(program: Program): number { + return program.body.reduce((sum, item) => sum + 1 + item.args.length, 0); +} + +export function positiveInteger(value: number, label: string, maximum: number): number { + if (!Number.isSafeInteger(value) || value < 1 || value > maximum) { + throw new Error(`${label} must be an integer between 1 and ${maximum}`); + } + return value; +} diff --git a/packages/fx-core/src/discovery/optimization.ts b/packages/fx-core/src/discovery/optimization.ts new file mode 100644 index 0000000..a6b123e --- /dev/null +++ b/packages/fx-core/src/discovery/optimization.ts @@ -0,0 +1,86 @@ +import { Program, parseProgram, positiveInteger, programId } from './model'; + +export interface Evaluation { + /** Higher is better; invalid candidates are never eligible for selection. */ + readonly score: number; + readonly valid: boolean; + readonly metrics?: Readonly>; +} +export interface OptimizationTrial { + readonly id: string; + readonly program: Program; + readonly evaluation?: Evaluation; + readonly error?: string; +} +export interface OptimizationResult { + readonly best?: OptimizationTrial; + readonly archive: readonly OptimizationTrial[]; + readonly evaluations: number; +} +export type CandidateProposer = ( + archive: readonly OptimizationTrial[], + iteration: number, +) => Program; + +/** + * Evaluator-driven selection, separate from the proposer and execution runtime. + * This bounds proposal attempts (duplicates count), not arbitrary work inside a + * trusted evaluator. Programs and scored evidence are frozen before feedback. + * An evaluator may implement simulation, tests or an independent verification tool. + */ +export function optimizePrograms( + proposals: Iterable | CandidateProposer, + evaluate: (program: Program) => Evaluation, + maxEvaluations: number, +): OptimizationResult { + positiveInteger(maxEvaluations, 'maxEvaluations', 1_000_000); + const archive: OptimizationTrial[] = []; + const seen = new Set(); + let best: OptimizationTrial | undefined; + let attempted = 0; + const candidates = + typeof proposals === 'function' + ? (function* () { + for (let iteration = 0; iteration < maxEvaluations; iteration++) { + yield proposals(archive.slice(), iteration); + } + })() + : proposals; + // Count proposals too: an infinite stream of duplicates cannot avoid the limit. + for (const value of candidates) { + if (attempted++ >= maxEvaluations) break; + let program = value; + let id = ''; + let trial: OptimizationTrial; + try { + program = parseProgram(value); + id = programId(program); + if (seen.has(id)) continue; + seen.add(id); + const result = evaluate(program); + if ( + typeof result.valid !== 'boolean' || + !Number.isFinite(result.score) || + Object.values(result.metrics ?? {}).some((metric) => !Number.isFinite(metric)) + ) { + throw new Error('Evaluator must return a validity flag and finite scores/metrics'); + } + const evaluation: Evaluation = Object.freeze({ + valid: result.valid, + score: result.score, + ...(result.metrics ? { metrics: Object.freeze({ ...result.metrics }) } : {}), + }); + trial = Object.freeze({ id, program, evaluation }); + if (evaluation.valid && (!best || evaluation.score > (best.evaluation?.score ?? -Infinity))) + best = trial; + } catch (error) { + trial = Object.freeze({ + id, + program, + error: error instanceof Error ? error.message : String(error), + }); + } + archive.push(trial); + } + return { ...(best ? { best } : {}), archive, evaluations: archive.length }; +} diff --git a/packages/fx-core/src/discovery/runtime.ts b/packages/fx-core/src/discovery/runtime.ts new file mode 100644 index 0000000..9ab908c --- /dev/null +++ b/packages/fx-core/src/discovery/runtime.ts @@ -0,0 +1,152 @@ +import type { Step } from '../types'; +import { + Abstraction, + Argument, + ExpandedInstruction, + Instruction, + Program, + abstractionSchema, + positiveInteger, + programSchema, +} from './model'; +import { ProgramState, snapshot } from './state'; + +/** Trusted synchronous primitives. The interpreter does not sandbox JavaScript. */ +export interface Primitive { + readonly name: string; + readonly arity: number; + readonly execute: (state: S, args: readonly number[]) => S; +} +export interface RuntimeLimits { + readonly maxPrimitiveCalls?: number; + readonly maxCallDepth?: number; +} +export interface Execution { + readonly state: S; + readonly primitiveCalls: number; +} +export class ProgramExecutionError extends Error { + constructor( + message: string, + public readonly primitiveCalls: number, + ) { + super(message); + this.name = 'ProgramExecutionError'; + } +} + +export class ProgramRuntime { + private readonly primitives = new Map>(); + private readonly library = new Map(); + private readonly maxPrimitiveCalls: number; + private readonly maxCallDepth: number; + + constructor( + primitives: readonly Primitive[], + abstractions: readonly Abstraction[] = [], + limits: RuntimeLimits = {}, + ) { + this.maxPrimitiveCalls = positiveInteger( + limits.maxPrimitiveCalls ?? 1024, + 'maxPrimitiveCalls', + 100_000, + ); + this.maxCallDepth = positiveInteger(limits.maxCallDepth ?? 32, 'maxCallDepth', 128); + for (const primitive of primitives) { + // Reuse the language's name/arity validation without executing a primitive. + abstractionSchema.parse({ + name: primitive.name, + arity: primitive.arity, + body: [{ op: primitive.name, args: [] }], + }); + if (this.primitives.has(primitive.name)) + throw new Error(`Duplicate operation: ${primitive.name}`); + if (typeof primitive.execute !== 'function') + throw new Error(`Missing implementation: ${primitive.name}`); + this.primitives.set(primitive.name, Object.freeze({ ...primitive })); + } + for (const value of abstractions) { + const abstraction = abstractionSchema.parse(value); + if (this.primitives.has(abstraction.name) || this.library.has(abstraction.name)) { + throw new Error(`Duplicate operation: ${abstraction.name}`); + } + this.library.set(abstraction.name, abstraction); + } + // Check even unused definitions, including forward references and cycles. + for (const abstraction of this.library.values()) { + this.expand({ + version: 1, + body: [{ op: abstraction.name, args: Array(abstraction.arity).fill(0) as number[] }], + }); + } + } + + /** Validate the entire program before any primitive executes. */ + expand(value: Program): readonly ExpandedInstruction[] { + const program = programSchema.parse(value); + const result: ExpandedInstruction[] = []; + let visits = 0; + const resolve = (arg: Argument, parameters: readonly number[]): number => { + if (typeof arg === 'number') return arg; + const bound = parameters[arg.parameter]; + if (bound === undefined) throw new Error(`Unbound parameter: ${arg.parameter}`); + return bound; + }; + const visit = ( + body: readonly Instruction[], + parameters: readonly number[], + stack: readonly string[], + ): void => { + for (const item of body) { + if (++visits > 100_000) throw new Error('Program expansion exceeds 100000 visits'); + const args = Object.freeze(item.args.map((arg) => resolve(arg, parameters))); + const primitive = this.primitives.get(item.op); + if (primitive) { + if (args.length !== primitive.arity) throw new Error(`Wrong arity for ${item.op}`); + if (result.length >= this.maxPrimitiveCalls) + throw new Error('Program exceeds primitive call limit'); + result.push(Object.freeze({ op: item.op, args })); + continue; + } + const abstraction = this.library.get(item.op); + if (!abstraction) throw new Error(`Unknown operation: ${item.op}`); + if (args.length !== abstraction.arity) throw new Error(`Wrong arity for ${item.op}`); + if (stack.includes(item.op)) throw new Error(`Recursive abstraction: ${item.op}`); + if (stack.length >= this.maxCallDepth) + throw new Error('Program exceeds abstraction depth limit'); + visit(abstraction.body, args, [...stack, item.op]); + } + }; + visit(program.body, [], []); + return Object.freeze(result); + } + + private execute(body: readonly ExpandedInstruction[], initialState: S): Execution { + let state = snapshot(initialState); + let primitiveCalls = 0; + for (const item of body) { + try { + primitiveCalls++; + const primitive = this.primitives.get(item.op); + if (!primitive) throw new Error(`Unknown primitive: ${item.op}`); + state = snapshot(primitive.execute(state, item.args)); + } catch (error) { + throw new ProgramExecutionError( + error instanceof Error ? error.message : String(error), + primitiveCalls, + ); + } + } + return { state, primitiveCalls }; + } + + run(program: Program, initialState: S): Execution { + return this.execute(this.expand(program), initialState); + } + + /** Existing FX agents can execute an inspected program as an ordinary Step. */ + asStep(program: Program): Step { + const body = this.expand(program); + return async (state) => this.execute(body, state).state; + } +} diff --git a/packages/fx-core/src/discovery/search.ts b/packages/fx-core/src/discovery/search.ts new file mode 100644 index 0000000..be3ae71 --- /dev/null +++ b/packages/fx-core/src/discovery/search.ts @@ -0,0 +1,123 @@ +import { Instruction, Program, parseProgram, positiveInteger, programId } from './model'; +import { ProgramExecutionError, ProgramRuntime } from './runtime'; +import { ProgramState, sameState } from './state'; + +export interface Example { + readonly input: S; + readonly expected: S; +} +export interface SearchOptions { + readonly maxCandidates: number; + readonly maxPrimitiveCalls: number; + readonly maxLength: number; +} +export interface Trial { + readonly id: string; + readonly program: Program; + readonly status: 'passed' | 'rejected' | 'error' | 'budget_exhausted'; + readonly passedExamples: number; + readonly primitiveCalls: number; + readonly error?: string; +} +export interface SearchResult { + readonly program?: Program; + readonly trials: readonly Trial[]; + readonly candidates: number; + readonly primitiveCalls: number; + readonly exhausted: boolean; +} + +/** Deterministic proposer; callers may replace it with model-generated programs. */ +export function* enumeratePrograms( + actions: readonly Instruction[], + maxLength: number, +): Generator { + positiveInteger(maxLength, 'maxLength', 16); + if (actions.length > 128) throw new Error('Search vocabulary exceeds 128 actions'); + yield { version: 1, body: [] }; + function* bodies(prefix: readonly Instruction[], remaining: number): Generator { + if (remaining === 0) { + yield { version: 1, body: prefix }; + return; + } + for (const action of actions) yield* bodies([...prefix, action], remaining - 1); + } + for (let length = 1; length <= maxLength; length++) yield* bodies([], length); +} + +/** + * Bounded synthesis from input/output examples. The proposer sees no verification + * split: verification belongs to the caller after selection. Costs include failed + * candidates, repeated examples and failing primitive invocations. + */ +export function searchProgram( + runtime: ProgramRuntime, + examples: readonly Example[], + proposals: Iterable, + options: SearchOptions, +): SearchResult { + positiveInteger(options.maxCandidates, 'maxCandidates', 1_000_000); + positiveInteger(options.maxPrimitiveCalls, 'maxPrimitiveCalls', 100_000_000); + positiveInteger(options.maxLength, 'maxLength', 128); + if (examples.length === 0) throw new Error('Search requires at least one example'); + const trials: Trial[] = []; + let primitiveCalls = 0; + const finish = (exhausted: boolean, program?: Program): SearchResult => ({ + ...(program ? { program } : {}), + trials, + candidates: trials.length, + primitiveCalls, + exhausted, + }); + for (const value of proposals) { + if (trials.length >= options.maxCandidates) return finish(true); + let passedExamples = 0; + let calls = 0; + let status: Trial['status'] = 'rejected'; + let error: string | undefined; + let id = ''; + let program = value; + try { + program = parseProgram(value); + id = programId(program); + if (program.body.length > options.maxLength) + throw new Error('Candidate exceeds symbolic length limit'); + const expanded = runtime.expand(program); + for (const example of examples) { + if (primitiveCalls + expanded.length > options.maxPrimitiveCalls) { + status = 'budget_exhausted'; + break; + } + let actual: S; + try { + const result = runtime.run(program, example.input); + calls += result.primitiveCalls; + primitiveCalls += result.primitiveCalls; + actual = result.state; + } catch (failure) { + const spent = failure instanceof ProgramExecutionError ? failure.primitiveCalls : 0; + calls += spent; + primitiveCalls += spent; + throw failure; + } + if (!sameState(actual, example.expected)) break; + passedExamples++; + } + if (passedExamples === examples.length) status = 'passed'; + } catch (failure) { + status = 'error'; + error = failure instanceof Error ? failure.message : String(failure); + } + trials.push({ + id, + program, + status, + passedExamples, + primitiveCalls: calls, + ...(error ? { error } : {}), + }); + if (status === 'passed') return finish(false, program); + if (status === 'budget_exhausted') return finish(true); + } + return finish(false); +} diff --git a/packages/fx-core/src/discovery/state.ts b/packages/fx-core/src/discovery/state.ts new file mode 100644 index 0000000..4e0665e --- /dev/null +++ b/packages/fx-core/src/discovery/state.ts @@ -0,0 +1,51 @@ +export type JsonValue = + | null + | boolean + | number + | string + | readonly JsonValue[] + | { readonly [key: string]: JsonValue }; +export type ProgramState = Readonly>; + +/** Isolated, deeply frozen JSON snapshots at every primitive boundary. */ +export function snapshot(state: S): S { + const active = new Set(); + const copy = (value: unknown, depth: number): JsonValue => { + if (depth > 64) throw new Error('State nesting exceeds 64 levels'); + if (value === null || typeof value === 'string' || typeof value === 'boolean') return value; + if (typeof value === 'number' && Number.isFinite(value)) return value; + if (typeof value !== 'object' || value === null) + throw new Error('State must contain only JSON values'); + if (active.has(value)) throw new Error('Cyclic state is not supported'); + active.add(value); + let result: JsonValue; + if (Array.isArray(value)) { + // Array.from also visits holes rather than silently changing them to null. + result = Object.freeze(Array.from(value, (item) => copy(item, depth + 1))); + } else { + if ( + Object.getPrototypeOf(value) !== Object.prototype && + Object.getPrototypeOf(value) !== null + ) { + throw new Error('State objects must be plain JSON objects'); + } + result = Object.freeze( + Object.fromEntries( + Object.keys(value) + .sort() + .map((key) => [key, copy((value as Record)[key], depth + 1)]), + ), + ); + } + active.delete(value); + return result; + }; + if (state === null || typeof state !== 'object' || Array.isArray(state)) { + throw new Error('State must be a JSON object'); + } + return copy(state, 0) as S; +} + +export function sameState(left: S, right: S): boolean { + return JSON.stringify(snapshot(left)) === JSON.stringify(snapshot(right)); +} diff --git a/packages/fx-core/src/index.ts b/packages/fx-core/src/index.ts index 8f211cb..affe4ed 100644 --- a/packages/fx-core/src/index.ts +++ b/packages/fx-core/src/index.ts @@ -59,6 +59,9 @@ export { enableLogging, disableLogging, logEvent, getEvents, clearEvents, getEve // ---------- Observability ---------- export { ObservabilityManager, appendDecision, getDecisionHistory } from './observability'; +// ---------- Experimental Program Discovery ---------- +export * from './discovery'; + // ---------- High-Level API ---------- export function createPlan( name: string, @@ -108,4 +111,4 @@ export function createAgent( plan: Plan ): Agent { return new Agent(plan); -} \ No newline at end of file +} From 69b0c134b7ab4d78d45147b307641628bf97a4d9 Mon Sep 17 00:00:00 2001 From: skishore23 Date: Fri, 30 Oct 2026 03:20:49 -0700 Subject: [PATCH 2/2] feat: search bounded algorithms for FX and Beetle targets --- README.md | 3 + docs/advanced/algorithm-search.md | 169 ++ docs/advanced/program-discovery.md | 4 + .../algorithm-search.test.cjs | 75 + examples/algorithm-search/builders.cjs | 17 + examples/algorithm-search/memory.cjs | 156 ++ examples/algorithm-search/results/README.md | 45 + .../results/beetle-routes.algorithm.json | 283 +++ .../results/beetle-routes.json | 838 +++++++ .../results/fx-memory.algorithm.json | 986 ++++++++ .../algorithm-search/results/fx-memory.json | 2157 +++++++++++++++++ .../algorithm-search/results/manifest.json | 54 + examples/algorithm-search/routes.cjs | 158 ++ examples/algorithm-search/run.cjs | 116 + examples/discovery/results/manifest.json | 39 +- package.json | 5 +- .../src/__tests__/algorithm-proposer.test.ts | 75 + .../src/__tests__/algorithm-runtime.test.ts | 212 ++ .../src/__tests__/algorithm-search.test.ts | 137 ++ .../fx-core/src/discovery/algorithm-model.ts | 201 ++ .../src/discovery/algorithm-proposer.ts | 106 + .../src/discovery/algorithm-runtime.ts | 211 ++ .../fx-core/src/discovery/algorithm-search.ts | 232 ++ .../fx-core/src/discovery/algorithm-values.ts | 122 + packages/fx-core/src/discovery/index.ts | 4 + 25 files changed, 6391 insertions(+), 14 deletions(-) create mode 100644 docs/advanced/algorithm-search.md create mode 100644 examples/algorithm-search/algorithm-search.test.cjs create mode 100644 examples/algorithm-search/builders.cjs create mode 100644 examples/algorithm-search/memory.cjs create mode 100644 examples/algorithm-search/results/README.md create mode 100644 examples/algorithm-search/results/beetle-routes.algorithm.json create mode 100644 examples/algorithm-search/results/beetle-routes.json create mode 100644 examples/algorithm-search/results/fx-memory.algorithm.json create mode 100644 examples/algorithm-search/results/fx-memory.json create mode 100644 examples/algorithm-search/results/manifest.json create mode 100644 examples/algorithm-search/routes.cjs create mode 100644 examples/algorithm-search/run.cjs create mode 100644 packages/fx-core/src/__tests__/algorithm-proposer.test.ts create mode 100644 packages/fx-core/src/__tests__/algorithm-runtime.test.ts create mode 100644 packages/fx-core/src/__tests__/algorithm-search.test.ts create mode 100644 packages/fx-core/src/discovery/algorithm-model.ts create mode 100644 packages/fx-core/src/discovery/algorithm-proposer.ts create mode 100644 packages/fx-core/src/discovery/algorithm-runtime.ts create mode 100644 packages/fx-core/src/discovery/algorithm-search.ts create mode 100644 packages/fx-core/src/discovery/algorithm-values.ts diff --git a/README.md b/README.md index f6c3bbe..19f3374 100644 --- a/README.md +++ b/README.md @@ -281,6 +281,9 @@ console.log(result.issues.length); // 3 ## Examples +- **[Algorithm search for FX and Beetle](./docs/advanced/algorithm-search.md)** — search scan, + early-exit and indexing algorithms against independent correctness checks and withheld cases. + - **[Experimental program discovery](./docs/advanced/program-discovery.md)** — inspectable programs, parameterized abstraction learning, bounded search, and reproducible transfer and GPU scheduling experiments. diff --git a/docs/advanced/algorithm-search.md b/docs/advanced/algorithm-search.md new file mode 100644 index 0000000..feff727 --- /dev/null +++ b/docs/advanced/algorithm-search.md @@ -0,0 +1,169 @@ +# Algorithm search for FX and Beetle + +This extension searches executable **control flow**: loops, scan direction, +conditions, early exits and indexes. It complements the numeric program and +abstraction experiments; it does not require an LLM or coding agent to run. + +```sh +npm ci +npm run algorithm:search -- --target fx-memory +npm run algorithm:search -- --target beetle-routes +npm run algorithm:search -- --target all --out ./algorithm-results +``` + +The output directory receives a report and a runnable `.algorithm.json` winner +for each accepted target. A rejected final result sets a nonzero CLI exit code +and writes `null` to the algorithm file, replacing any stale export. The report +retains the failed candidate. No production file, route, authorization, deployment or active +policy is changed. Search results must be inspected before integration. + +## Included targets + +| Target | Correctness source | Structures searched | +| --- | --- | --- | +| `fx-memory` | The actual `getMemory` implementation in `src/memory.ts` | Predicate ordering, full forward scan, guarded reverse scan with early exit, plus intentionally incorrect variants | +| `beetle-routes` | A synthetic reference model of nearest-address / first-converse selection over a fixed batch | Repeated scans, bucket indexes, precomputed reader indexes, preference and address-order variants | + +### FX memory filtering + +Each input contains JSON-normalized memory `entries` and a `filter` with explicit +nullable `type`, `since`, `search` and `limit` fields. Entry timestamps are finite +millisecond numbers. The oracle reconstructs Dates, calls the real FX +`getMemory`, and normalizes the complete resulting entries back to JSON. + +The candidate must preserve case-insensitive search, type and date filtering, +array order and every entry field. Cases cover empty results, missing/zero limits, +negative and fractional limits, large limits and nonchronological entry arrays. +For a positive integer limit, scanning from the end can stop after enough matches, +then reverse the matches to preserve output order. Other limits need a compatible +fallback. Incorrect early exits and incorrect output order are rejected. + +This is a real-function correctness target. Its cost baseline is an authored +forward-scan **IR implementation**, not timing of the native function. Interpreter +speedup must not be represented as a TypeScript performance gain. + +### Beetle batch route readers + +For each ordered address query, choose the nearest address with any routes. Within +that level return its first converse reader, otherwise its first route, otherwise +null. Keep all route fields and query order. The batch reads one immutable route +snapshot, allowing index construction to be amortized over queries. + +The target tests algorithmic selection only. It does not model author authority, +audience visibility, journal recovery, subscription mutation or the rest of the +live router. Its oracle is a small reference port of the selection contract; +**native Rust conformance is still required** before a Beetle implementation. +All fixtures are synthetic and contain no private Slack or organizational data. + +## A bounded language instead of arbitrary source execution + +An `Algorithm` is serializable JSON with: + +- JSON literals, variables, own-property reads and fixed intrinsics; +- assignment, conditionals, forward/reverse `for` loops, `break` and `return`; +- local arrays, append operations, and local string-keyed indexes; +- scalar comparisons, arithmetic, string filtering and array operations. + +There is no `eval`, source import, native callback, clock, randomness or ambient +I/O in the candidate language. Loop bounds are snapshotted before iteration. +Inputs are isolated immutable JSON; local indexes cannot escape as output. +Syntax depth/size, interpreted steps and allocated cells are bounded. Costs for +collection/string intrinsics scale with their inputs; index build cost is charged. + +The cell/step model is not an exact CPU or byte-memory limit. Parsing, JSON +serialization, oracle setup and host proposer work are outside the interpreted +step count. Proposers, references and integration adapters are trusted host code. +Do not execute externally generated JavaScript just because the IR interpreter +is bounded. A future source-code runner needs its own actual isolation boundary. + +## Acceptance protocol + +`searchAlgorithms(target, proposer, options)`: + +1. Copies fixed training, validation and held-out cases; rejects duplicate inputs. +2. Obtains expected results from the independent reference, outside candidate code. +3. Tests the baseline and proposed candidates against training cases. +4. Gives the proposer only the contract and immutable training feedback. +5. Selects a fixed shortlist plus the baseline fallback and checks validation cases. +6. Chooses by validation work count, then tests that winner once on held-out cases. + +A held-out failure sets `accepted: false`; it never triggers reselection. Incorrect +or over-budget candidates cannot win. A correct baseline remains an explicit +fallback. The archive records failed candidates and their work as well as winners. +The total step cap includes baseline, candidate, validation and final executions. +Exhausting that cap can abort the experiment rather than silently dropping checks. + +Results are correctness evidence on finite cases, not a proof for all inputs. +Native timing, production distributions and effect-boundary conformance remain +separate verification steps. + +## Proposers and additional targets + +The built-in targets use authored structural grammars. Search discovers the best +candidate **within those grammars**; it does not establish invention of a globally +new algorithm. A model can propose a wider set of JSON algorithms through the +same interface, without gaining access to held-out cases or changing the evaluator. + +```ts +const report = await searchAlgorithms(target, async context => { + // Supply an enumerator, a structural mutator, or a model adapter here. + // context contains { iteration, target: { id, contract }, archive }. + // Return { name, algorithm }, or undefined to finish. + return myProposer(context); +}, { maxProposals: 64, maxSteps: 1_000_000, maxTotalSteps: 30_000_000 }); +``` + +To evaluate JSON generated by a coding agent or another process: + +```sh +npm run algorithm:search -- --target fx-memory --proposals ./candidates.json +``` + +The file is an array of `{ "name": "candidate", "algorithm": { "version": 1, +"body": [...] } }` records. It is parsed as data, never imported as executable +JavaScript. Register another target by supplying its contract, baseline IR, +reference function and three fixed case splits. A natural-language wish alone +does not establish correctness or a useful optimization objective. + +### Optional model-generated search + +`createModelAlgorithmProposer(provider, options)` works with FX's existing +`LLMProvider.chat` interface. It supplies the language grammar, input contract, +baseline and bounded training feedback. Invalid responses are retained as +proposer errors; a call cap stops repeated generation. Reported usage includes +provider calls and token counts when supplied by that provider. There is no +monetary cap, and provider-internal retries are outside this call counter. + +With an already configured `OPENAI_API_KEY`, the CLI can use the existing FX +chat-completions provider; `OPENAI_BASE_URL` optionally selects a compatible host: + +```sh +npm run algorithm:search -- --target fx-memory --model YOUR_CHAT_MODEL --budget 16 +``` + +The model must support the existing provider's chat options. Other providers can +be injected programmatically through `runTarget` or `searchAlgorithms`. This +implementation's provider tests use mocks; the checked-in results use the +deterministic grammar, and no live model run is claimed. + +## Use a selected algorithm inside FX + +```ts +import { createAgent, createPlan, algorithmStep, JsonValue } from '@fx/core'; + +type SearchState = { algorithmInput: JsonValue; algorithmResult?: JsonValue }; + +const agent = createAgent('selected-algorithm', createPlan('plan', [ + algorithmStep( + report.selected.algorithm, + state => state.algorithmInput, + (state, result) => ({ ...state, algorithmResult: result }) + ) +])); +``` + +Supply a typed JSON input adapter appropriate to the target. Memory timestamps +must be normalized and restored by the adapter if the surrounding API uses Dates. +Running through the interpreter may be slower than native code even when the +selected algorithm performs fewer modeled operations. The next deployment gate +is a reviewed native implementation, conformance tests and representative timing. diff --git a/docs/advanced/program-discovery.md b/docs/advanced/program-discovery.md index ebbf39a..6a7ffad 100644 --- a/docs/advanced/program-discovery.md +++ b/docs/advanced/program-discovery.md @@ -1,5 +1,9 @@ # Experimental program discovery +For control-flow and indexing algorithm search targeting FX and Beetle, see +[Algorithm search](algorithm-search.md). That extension has a separate bounded +language with loops and branches; the numeric `Program` language below remains unchanged. + FX can now represent a bounded program as JSON, execute it as an ordinary FX `Step`, search for programs against an external evaluator, and extract parameterized abstractions from a successful program corpus. This is an diff --git a/examples/algorithm-search/algorithm-search.test.cjs b/examples/algorithm-search/algorithm-search.test.cjs new file mode 100644 index 0000000..1da1dc5 --- /dev/null +++ b/examples/algorithm-search/algorithm-search.test.cjs @@ -0,0 +1,75 @@ +const assert = require('node:assert/strict'); +const { test } = require('node:test'); +const { executeAlgorithm } = require('../../packages/fx-core/dist/discovery'); +const { runTarget } = require('./run.cjs'); +const { memoryTarget } = require('./memory.cjs'); +const { routeTarget, routeReference } = require('./routes.cjs'); + +test('a model adapter can propose runnable FX algorithms without live credentials', async () => { + const baseline = memoryTarget().baseline; + let calls = 0; + const provider = { + chat: async () => { + calls++; + return { + content: JSON.stringify({ name: 'model-candidate', algorithm: baseline }), + usage: { promptTokens: 10, completionTokens: 2, totalTokens: 12 }, + }; + }, + }; + const result = await runTarget('fx-memory', { provider, model: 'mock', budget: 2 }); + assert.equal(result.accepted, true); + assert.equal(result.proposalSource, 'model-generated-IR'); + assert.equal(calls, 2); + assert.deepEqual(result.modelUsage, { calls: 2, promptTokens: 20, completionTokens: 4 }); +}); + +test('algorithm search finds a correct FX scan variant and rejects broken early exits', async () => { + const result = await runTarget('fx-memory'); + assert.equal(result.accepted, true); + assert.ok(result.selected.heldOut.steps < result.baseline.heldOut.steps); + assert.ok(result.interpretation.rejectedCandidates > 0); + const reference = memoryTarget().reference; + for (let seed = 0; seed < 60; seed++) { + const input = { + entries: Array.from({ length: seed % 23 }, (_, i) => ({ + id: `${seed}-${i}`, + type: i % 2 ? 'action' : 'error', + content: i % 3 ? 'normal' : 'ÅNGSTRÖM build', + timestamp: (i % 5) * 1000, + })), + filter: { + type: seed % 3 ? 'error' : null, + since: seed % 2 ? 1000 : null, + search: seed % 4 ? 'build' : null, + limit: [null, 0, 1, -2, 2.5, 0.5, 9999][seed % 7], + }, + }; + assert.deepEqual( + executeAlgorithm(result.selected.algorithm, input).output, + await reference(input), + ); + } +}); + +test('indexed batch routing preserves precedence, first-converse order and missing routes', async () => { + const result = await runTarget('beetle-routes'); + assert.equal(result.accepted, true); + assert.ok(result.selected.heldOut.steps < result.baseline.heldOut.steps); + assert.ok(result.selected.heldOut.loopVisits < result.baseline.heldOut.loopVisits); + for (let seed = 0; seed < 60; seed++) { + const input = { + routes: Array.from({ length: seed }, (_, i) => ({ + id: i, + address: `a${i % 7}`, + policy: i % 4 ? 'observe' : 'converse', + })), + queries: [['a1', 'a2'], ['missing'], ['a2', 'a1'], [], ['a1', 'a1']], + }; + assert.deepEqual( + executeAlgorithm(result.selected.algorithm, input).output, + routeReference(input), + ); + } + assert.equal(routeTarget().id, 'beetle.route.batch-most-specific'); +}); diff --git a/examples/algorithm-search/builders.cjs b/examples/algorithm-search/builders.cjs new file mode 100644 index 0000000..36b28fe --- /dev/null +++ b/examples/algorithm-search/builders.cjs @@ -0,0 +1,17 @@ +const { expressionBuilder } = require('../../packages/fx-core/dist/discovery'); +const { literal: lit, variable: variable, get, call } = expressionBuilder; +const set = (name, value) => ({ kind: 'set', name, value }); +const append = (name, value) => ({ kind: 'append', name, value }); +const put = (name, key, value) => ({ kind: 'put', name, key, value }); +const branch = (condition, then, otherwise = []) => ({ kind: 'if', condition, then, otherwise }); +const loop = (name, values, body, direction = 'forward') => ({ + kind: 'for', + name, + values, + body, + direction, +}); +const ret = (value) => ({ kind: 'return', value }); +const stop = { kind: 'break' }; +const input = (key) => get(variable('input'), key); +module.exports = { lit, variable, get, call, set, append, put, branch, loop, ret, stop, input }; diff --git a/examples/algorithm-search/memory.cjs b/examples/algorithm-search/memory.cjs new file mode 100644 index 0000000..1ce186b --- /dev/null +++ b/examples/algorithm-search/memory.cjs @@ -0,0 +1,156 @@ +const { getMemory } = require('../../packages/fx-core/dist/memory'); +const { + lit, + variable: v, + get, + call, + set, + append, + branch, + loop, + ret, + stop, + input, +} = require('./builders.cjs'); +const filter = (key) => get(input('filter'), key); +const entry = (key) => get(v('entry'), key); +const eq = (a, b) => call('eq', a, b); +const notNull = (value) => call('not', eq(value, lit(null))); +const hasLimit = call('and', notNull(filter('limit')), call('not', eq(filter('limit'), lit(0)))); +const fastLimit = call( + 'and', + notNull(filter('limit')), + call( + 'and', + call('gt', filter('limit'), lit(0)), + eq(filter('limit'), call('floor', filter('limit'))), + ), +); +const predicates = { + type: call('or', eq(filter('type'), lit(null)), eq(entry('type'), filter('type'))), + since: call( + 'or', + eq(filter('since'), lit(null)), + call('gte', entry('timestamp'), filter('since')), + ), + search: call( + 'or', + eq(v('needle'), lit('')), + call('contains', call('lower', entry('content')), v('needle')), + ), +}; +const orders = [ + ['type', 'since', 'search'], + ['type', 'search', 'since'], + ['since', 'type', 'search'], + ['since', 'search', 'type'], + ['search', 'type', 'since'], + ['search', 'since', 'type'], +]; + +function memoryAlgorithm(strategy, order) { + const matches = order + .map((name) => predicates[name]) + .reduce((left, right) => call('and', left, right)); + const scan = (reverse, early) => + loop( + 'entry', + input('entries'), + [ + branch(matches, [ + append('result', v('entry')), + ...(early + ? [ + branch( + call('and', fastLimit, call('gte', call('length', v('result')), filter('limit'))), + [stop], + ), + ] + : []), + ]), + ], + reverse ? 'reverse' : 'forward', + ); + const full = [ + scan(false, false), + branch(hasLimit, [set('result', call('slice', v('result'), call('neg', filter('limit'))))]), + ret(v('result')), + ]; + const body = [ + set('result', call('array')), + set('needle', lit('')), + branch(notNull(filter('search')), [set('needle', call('lower', filter('search')))]), + ]; + if (strategy === 'forward') body.push(...full); + else if (strategy === 'reverse-guarded') + body.push(branch(fastLimit, [scan(true, true), ret(call('reverse', v('result')))], full)); + else if (strategy === 'forward-early') body.push(scan(false, true), ret(v('result'))); + else body.push(scan(true, true), ret(v('result'))); // intentionally fails output order + return { version: 1, body }; +} + +function fixture(seed, size, filterOverride = {}) { + const types = ['observation', 'action', 'result', 'error']; + return { + entries: Array.from({ length: size }, (_, i) => ({ + id: `entry-${seed}-${i}`, + type: types[(i + seed) % 4], + content: i % 7 === 0 ? 'URGENT deploy' : i % 3 === 0 ? 'Build succeeded' : 'routine status', + timestamp: seed * 1000 + (size - i) * 10, + })), + filter: { type: null, since: null, search: null, limit: 3, ...filterOverride }, + }; +} +function cases(prefix, seed, sizes) { + const filters = [ + {}, + { limit: null }, + { limit: 0 }, + { limit: -2 }, + { limit: 2.5 }, + { type: 'error', limit: 2 }, + { search: 'bUiLd', limit: 4 }, + { since: 0, search: '', limit: 1 }, + { search: 'missing', limit: 1 }, + { type: 'observation', search: 'urgent', limit: 2 }, + ]; + return filters.map((filter, i) => ({ + name: `${prefix}-${i}`, + input: fixture(seed + i, sizes[i % sizes.length], filter), + })); +} +function memoryTarget() { + return { + id: 'fx.memory.getMemory', + contract: + 'Input is {entries:[{id,type,content,timestamp}],filter:{type,since,search,limit}}. Timestamps/since are finite milliseconds; every filter field is nullable. Return complete matching entries in input array order. Preserve actual FX getMemory semantics: type and since filtering, case-insensitive substring search, and finite numeric limit behavior including negative, fractional, zero or null limits.', + baseline: memoryAlgorithm('forward', orders[0]), + reference: async (input) => { + const filter = Object.fromEntries( + Object.entries(input.filter).filter(([, value]) => value !== null), + ); + if (filter.since !== undefined) filter.since = new Date(filter.since); + const memory = input.entries.map((entry) => ({ + ...entry, + timestamp: new Date(entry.timestamp), + })); + const result = await getMemory(filter)({ memory }); + return result.memoryFiltered.map((entry) => ({ + ...entry, + timestamp: entry.timestamp.getTime(), + })); + }, + training: cases('train', 10, [16, 48, 128]), + validation: cases('validation', 100, [0, 80, 180]), + heldOut: cases('test', 200, [512, 256, 31]), + }; +} +function memoryCandidates() { + return ['forward-early', 'reverse-unordered', 'forward', 'reverse-guarded'].flatMap((strategy) => + orders.map((order) => ({ + name: `${strategy}:${order.join(',')}`, + algorithm: memoryAlgorithm(strategy, order), + })), + ); +} +module.exports = { memoryTarget, memoryCandidates, memoryAlgorithm }; diff --git a/examples/algorithm-search/results/README.md b/examples/algorithm-search/results/README.md new file mode 100644 index 0000000..cde412e --- /dev/null +++ b/examples/algorithm-search/results/README.md @@ -0,0 +1,45 @@ +# Initial algorithm-search results + +These results use the deterministic, authored structural grammars, not a live +model. The model proposer has mock tests but no paid/provider run is claimed. +Each selected algorithm passes validation and then one fixed held-out split. +The full reports retain rejected candidates, source contracts and split hashes. + +| Target | Selected algorithm | Final cases passing | Baseline steps | Selected steps | Baseline loop visits | Selected loop visits | +| --- | --- | ---: | ---: | ---: | ---: | ---: | +| FX `getMemory` contract | Guarded reverse scan; search/type/since predicate order | 10 / 10 | 113,056 | 33,578 | 2,909 | 1,138 | +| Beetle-compatible batch route readers | Index each address's first preferred reader, then probe per query | 3 / 3 | 409,613 | 45,615 | 19,971 | 904 | + +The step reductions are approximately 70.3% and 88.9% in the explicit cost model. +They are **not native runtime speedups**. Baselines are authored IR implementations +of the contracts, not timings of the original TypeScript or Rust code. Input +copying/parsing and native VM dispatch costs can dominate execution time. + +For FX, expected results come from the actual `getMemory` function. Twelve +incorrect candidates are rejected, including variants with broken early exits +or result order. Separate regression tests compare the winner to the real API +on 60 additional generated cases with Unicode search and unusual finite limits. + +For Beetle, expected results come from a synthetic reference model of immutable +batch selection. Nine incorrect candidates are rejected. Separate tests exercise +60 additional batches, nearest-address precedence, converse preference, missing +addresses and repeated queries. This is not native Rust conformance and does not +qualify the complete production router or authorization behavior. + +Both searches include preprocessing/index work in their cost. An indexed batch +strategy may lose on small batches, and a reverse scan may lose when no early +exit is possible. The aggregate result is not an all-input performance guarantee. +New algorithms are limited by the proposer grammar and language; no global +algorithmic novelty is claimed. + +Reproduce: + +```sh +npm ci +npm run algorithm:search -- --target all --out ./algorithm-results +npm run test:experiments +``` + +Use the `.algorithm.json` files with `executeAlgorithm` or `algorithmStep`. Review, +native implementation, native timing and domain conformance are separate gates +before production integration. No active system was modified by these searches. diff --git a/examples/algorithm-search/results/beetle-routes.algorithm.json b/examples/algorithm-search/results/beetle-routes.algorithm.json new file mode 100644 index 0000000..d509671 --- /dev/null +++ b/examples/algorithm-search/results/beetle-routes.algorithm.json @@ -0,0 +1,283 @@ +{ + "version": 1, + "body": [ + { + "kind": "set", + "name": "output", + "value": { + "kind": "call", + "op": "array", + "args": [] + } + }, + { + "kind": "set", + "name": "index", + "value": { + "kind": "call", + "op": "map", + "args": [] + } + }, + { + "kind": "for", + "name": "route", + "values": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "routes" + } + }, + "direction": "forward", + "body": [ + { + "kind": "if", + "condition": { + "kind": "call", + "op": "not", + "args": [ + { + "kind": "call", + "op": "has", + "args": [ + { + "kind": "variable", + "name": "index" + }, + { + "kind": "get", + "object": { + "kind": "variable", + "name": "route" + }, + "key": { + "kind": "literal", + "value": "address" + } + } + ] + } + ] + }, + "then": [ + { + "kind": "put", + "name": "index", + "key": { + "kind": "get", + "object": { + "kind": "variable", + "name": "route" + }, + "key": { + "kind": "literal", + "value": "address" + } + }, + "value": { + "kind": "variable", + "name": "route" + } + } + ], + "otherwise": [ + { + "kind": "if", + "condition": { + "kind": "call", + "op": "and", + "args": [ + { + "kind": "call", + "op": "not", + "args": [ + { + "kind": "call", + "op": "eq", + "args": [ + { + "kind": "get", + "object": { + "kind": "call", + "op": "lookup", + "args": [ + { + "kind": "variable", + "name": "index" + }, + { + "kind": "get", + "object": { + "kind": "variable", + "name": "route" + }, + "key": { + "kind": "literal", + "value": "address" + } + } + ] + }, + "key": { + "kind": "literal", + "value": "policy" + } + }, + { + "kind": "literal", + "value": "converse" + } + ] + } + ] + }, + { + "kind": "call", + "op": "eq", + "args": [ + { + "kind": "get", + "object": { + "kind": "variable", + "name": "route" + }, + "key": { + "kind": "literal", + "value": "policy" + } + }, + { + "kind": "literal", + "value": "converse" + } + ] + } + ] + }, + "then": [ + { + "kind": "put", + "name": "index", + "key": { + "kind": "get", + "object": { + "kind": "variable", + "name": "route" + }, + "key": { + "kind": "literal", + "value": "address" + } + }, + "value": { + "kind": "variable", + "name": "route" + } + } + ], + "otherwise": [] + } + ] + } + ] + }, + { + "kind": "for", + "name": "query", + "values": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "queries" + } + }, + "direction": "forward", + "body": [ + { + "kind": "set", + "name": "chosen", + "value": { + "kind": "literal", + "value": null + } + }, + { + "kind": "for", + "name": "address", + "values": { + "kind": "variable", + "name": "query" + }, + "direction": "forward", + "body": [ + { + "kind": "if", + "condition": { + "kind": "call", + "op": "has", + "args": [ + { + "kind": "variable", + "name": "index" + }, + { + "kind": "variable", + "name": "address" + } + ] + }, + "then": [ + { + "kind": "set", + "name": "chosen", + "value": { + "kind": "call", + "op": "lookup", + "args": [ + { + "kind": "variable", + "name": "index" + }, + { + "kind": "variable", + "name": "address" + } + ] + } + }, + { + "kind": "break" + } + ], + "otherwise": [] + } + ] + }, + { + "kind": "append", + "name": "output", + "value": { + "kind": "variable", + "name": "chosen" + } + } + ] + }, + { + "kind": "return", + "value": { + "kind": "variable", + "name": "output" + } + } + ] +} diff --git a/examples/algorithm-search/results/beetle-routes.json b/examples/algorithm-search/results/beetle-routes.json new file mode 100644 index 0000000..5f62057 --- /dev/null +++ b/examples/algorithm-search/results/beetle-routes.json @@ -0,0 +1,838 @@ +{ + "version": 1, + "target": { + "id": "beetle.route.batch-most-specific", + "contract": "Input is {routes:[{id,address,policy,...otherFields}],queries:[orderedAddressStrings[]]}. For every query return the first converse route at the nearest nonempty address level, otherwise its first route, otherwise null. Return an array preserving full route records and query order. Routes are an immutable snapshot for the batch. This is a selection reference model, not the live Rust authorization router." + }, + "accepted": true, + "selected": { + "id": "f27588c19750c2b54afc261630314ec550a5e73f082d3dc031c280140e703c5f", + "name": "reader-index:converse=true:reverse=false", + "algorithm": { + "version": 1, + "body": [ + { + "kind": "set", + "name": "output", + "value": { + "kind": "call", + "op": "array", + "args": [] + } + }, + { + "kind": "set", + "name": "index", + "value": { + "kind": "call", + "op": "map", + "args": [] + } + }, + { + "kind": "for", + "name": "route", + "values": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "routes" + } + }, + "direction": "forward", + "body": [ + { + "kind": "if", + "condition": { + "kind": "call", + "op": "not", + "args": [ + { + "kind": "call", + "op": "has", + "args": [ + { + "kind": "variable", + "name": "index" + }, + { + "kind": "get", + "object": { + "kind": "variable", + "name": "route" + }, + "key": { + "kind": "literal", + "value": "address" + } + } + ] + } + ] + }, + "then": [ + { + "kind": "put", + "name": "index", + "key": { + "kind": "get", + "object": { + "kind": "variable", + "name": "route" + }, + "key": { + "kind": "literal", + "value": "address" + } + }, + "value": { + "kind": "variable", + "name": "route" + } + } + ], + "otherwise": [ + { + "kind": "if", + "condition": { + "kind": "call", + "op": "and", + "args": [ + { + "kind": "call", + "op": "not", + "args": [ + { + "kind": "call", + "op": "eq", + "args": [ + { + "kind": "get", + "object": { + "kind": "call", + "op": "lookup", + "args": [ + { + "kind": "variable", + "name": "index" + }, + { + "kind": "get", + "object": { + "kind": "variable", + "name": "route" + }, + "key": { + "kind": "literal", + "value": "address" + } + } + ] + }, + "key": { + "kind": "literal", + "value": "policy" + } + }, + { + "kind": "literal", + "value": "converse" + } + ] + } + ] + }, + { + "kind": "call", + "op": "eq", + "args": [ + { + "kind": "get", + "object": { + "kind": "variable", + "name": "route" + }, + "key": { + "kind": "literal", + "value": "policy" + } + }, + { + "kind": "literal", + "value": "converse" + } + ] + } + ] + }, + "then": [ + { + "kind": "put", + "name": "index", + "key": { + "kind": "get", + "object": { + "kind": "variable", + "name": "route" + }, + "key": { + "kind": "literal", + "value": "address" + } + }, + "value": { + "kind": "variable", + "name": "route" + } + } + ], + "otherwise": [] + } + ] + } + ] + }, + { + "kind": "for", + "name": "query", + "values": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "queries" + } + }, + "direction": "forward", + "body": [ + { + "kind": "set", + "name": "chosen", + "value": { + "kind": "literal", + "value": null + } + }, + { + "kind": "for", + "name": "address", + "values": { + "kind": "variable", + "name": "query" + }, + "direction": "forward", + "body": [ + { + "kind": "if", + "condition": { + "kind": "call", + "op": "has", + "args": [ + { + "kind": "variable", + "name": "index" + }, + { + "kind": "variable", + "name": "address" + } + ] + }, + "then": [ + { + "kind": "set", + "name": "chosen", + "value": { + "kind": "call", + "op": "lookup", + "args": [ + { + "kind": "variable", + "name": "index" + }, + { + "kind": "variable", + "name": "address" + } + ] + } + }, + { + "kind": "break" + } + ], + "otherwise": [] + } + ] + }, + { + "kind": "append", + "name": "output", + "value": { + "kind": "variable", + "name": "chosen" + } + } + ] + }, + { + "kind": "return", + "value": { + "kind": "variable", + "name": "output" + } + } + ] + }, + "training": { + "passed": 5, + "total": 5, + "steps": 14293, + "loopVisits": 417, + "maxCells": 11694, + "failures": [] + }, + "validation": { + "passed": 3, + "total": 3, + "steps": 18902, + "loopVisits": 417, + "maxCells": 16589, + "failures": [] + }, + "heldOut": { + "passed": 3, + "total": 3, + "steps": 45615, + "loopVisits": 904, + "maxCells": 43356, + "failures": [] + } + }, + "baseline": { + "algorithm": { + "version": 1, + "body": [ + { + "kind": "set", + "name": "output", + "value": { + "kind": "call", + "op": "array", + "args": [] + } + }, + { + "kind": "for", + "name": "query", + "values": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "queries" + } + }, + "direction": "forward", + "body": [ + { + "kind": "set", + "name": "chosen", + "value": { + "kind": "literal", + "value": null + } + }, + { + "kind": "for", + "name": "address", + "values": { + "kind": "variable", + "name": "query" + }, + "direction": "forward", + "body": [ + { + "kind": "set", + "name": "first", + "value": { + "kind": "literal", + "value": null + } + }, + { + "kind": "for", + "name": "route", + "values": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "routes" + } + }, + "direction": "forward", + "body": [ + { + "kind": "if", + "condition": { + "kind": "call", + "op": "eq", + "args": [ + { + "kind": "get", + "object": { + "kind": "variable", + "name": "route" + }, + "key": { + "kind": "literal", + "value": "address" + } + }, + { + "kind": "variable", + "name": "address" + } + ] + }, + "then": [ + { + "kind": "if", + "condition": { + "kind": "call", + "op": "eq", + "args": [ + { + "kind": "variable", + "name": "first" + }, + { + "kind": "literal", + "value": null + } + ] + }, + "then": [ + { + "kind": "set", + "name": "first", + "value": { + "kind": "variable", + "name": "route" + } + } + ], + "otherwise": [] + }, + { + "kind": "if", + "condition": { + "kind": "call", + "op": "eq", + "args": [ + { + "kind": "get", + "object": { + "kind": "variable", + "name": "route" + }, + "key": { + "kind": "literal", + "value": "policy" + } + }, + { + "kind": "literal", + "value": "converse" + } + ] + }, + "then": [ + { + "kind": "set", + "name": "first", + "value": { + "kind": "variable", + "name": "route" + } + }, + { + "kind": "break" + } + ], + "otherwise": [] + } + ], + "otherwise": [] + } + ] + }, + { + "kind": "if", + "condition": { + "kind": "call", + "op": "not", + "args": [ + { + "kind": "call", + "op": "eq", + "args": [ + { + "kind": "variable", + "name": "first" + }, + { + "kind": "literal", + "value": null + } + ] + } + ] + }, + "then": [ + { + "kind": "set", + "name": "chosen", + "value": { + "kind": "variable", + "name": "first" + } + }, + { + "kind": "break" + } + ], + "otherwise": [] + } + ] + }, + { + "kind": "append", + "name": "output", + "value": { + "kind": "variable", + "name": "chosen" + } + } + ] + }, + { + "kind": "return", + "value": { + "kind": "variable", + "name": "output" + } + } + ] + }, + "training": { + "passed": 5, + "total": 5, + "steps": 118767, + "loopVisits": 6407, + "maxCells": 38641, + "failures": [] + }, + "heldOut": { + "passed": 3, + "total": 3, + "steps": 409613, + "loopVisits": 19971, + "maxCells": 131743, + "failures": [] + } + }, + "archive": [ + { + "id": "b968fca49df8f9c3656493e8096336e4b6c353f01ff0660417bb1bab10a791a7", + "name": "scan:converse=false:reverse=true", + "training": { + "passed": 2, + "total": 5, + "steps": 156828, + "loopVisits": 8513, + "maxCells": 50754, + "failures": [ + { + "name": "train-2", + "error": "Output differs from independent reference" + }, + { + "name": "train-3", + "error": "Output differs from independent reference" + }, + { + "name": "train-precedence", + "error": "Output differs from independent reference" + } + ] + } + }, + { + "id": "cf5d9ef42ae1852d4e38d8a4b80e7d525cc6093def857ca5631b82c6b7e9d05f", + "name": "scan:converse=false:reverse=false", + "training": { + "passed": 2, + "total": 5, + "steps": 156828, + "loopVisits": 8513, + "maxCells": 50754, + "failures": [ + { + "name": "train-2", + "error": "Output differs from independent reference" + }, + { + "name": "train-3", + "error": "Output differs from independent reference" + }, + { + "name": "train-precedence", + "error": "Output differs from independent reference" + } + ] + } + }, + { + "id": "427cc1d1b50683d5e39f23dacb2a0de5c1137ffb8164437d40136601ca3bb93e", + "name": "scan:converse=true:reverse=true", + "training": { + "passed": 4, + "total": 5, + "steps": 118767, + "loopVisits": 6407, + "maxCells": 38641, + "failures": [ + { + "name": "train-precedence", + "error": "Output differs from independent reference" + } + ] + } + }, + { + "id": "9bcba76f1e1338289a3da90c8588e7034b5453abda6789a8d9b6c327de90c7b5", + "name": "scan:converse=true:reverse=false", + "training": { + "passed": 5, + "total": 5, + "steps": 118767, + "loopVisits": 6407, + "maxCells": 38641, + "failures": [] + } + }, + { + "id": "c75d12d6a4bfe29a58c6c8ed0906a1f42af794e1ca7fe60b862f3c75563f7157", + "name": "bucket-index:converse=false:reverse=true", + "training": { + "passed": 2, + "total": 5, + "steps": 18571, + "loopVisits": 630, + "maxCells": 10956, + "failures": [ + { + "name": "train-2", + "error": "Output differs from independent reference" + }, + { + "name": "train-3", + "error": "Output differs from independent reference" + }, + { + "name": "train-precedence", + "error": "Output differs from independent reference" + } + ] + } + }, + { + "id": "6a118d708cffadc84078bfe19c2337ea671de73426af84065f01dbf5b2861c94", + "name": "bucket-index:converse=false:reverse=false", + "training": { + "passed": 2, + "total": 5, + "steps": 18547, + "loopVisits": 630, + "maxCells": 10956, + "failures": [ + { + "name": "train-2", + "error": "Output differs from independent reference" + }, + { + "name": "train-3", + "error": "Output differs from independent reference" + }, + { + "name": "train-precedence", + "error": "Output differs from independent reference" + } + ] + } + }, + { + "id": "10511624fda83257bf768b5aa2b0e99481604eca791163d32fcbc2e60ff9232c", + "name": "bucket-index:converse=true:reverse=true", + "training": { + "passed": 4, + "total": 5, + "steps": 17705, + "loopVisits": 521, + "maxCells": 11003, + "failures": [ + { + "name": "train-precedence", + "error": "Output differs from independent reference" + } + ] + } + }, + { + "id": "449bbf4ce5b813bc81f9312c143c3704921c75f8e56614e93482167a19038f55", + "name": "bucket-index:converse=true:reverse=false", + "training": { + "passed": 5, + "total": 5, + "steps": 17681, + "loopVisits": 521, + "maxCells": 11003, + "failures": [] + } + }, + { + "id": "46f2dbee9ac85bf5f0836c728811f84712b083445be20d1293dfe1bb3f12c052", + "name": "reader-index:converse=false:reverse=true", + "training": { + "passed": 2, + "total": 5, + "steps": 8144, + "loopVisits": 417, + "maxCells": 8393, + "failures": [ + { + "name": "train-2", + "error": "Output differs from independent reference" + }, + { + "name": "train-3", + "error": "Output differs from independent reference" + }, + { + "name": "train-precedence", + "error": "Output differs from independent reference" + } + ] + } + }, + { + "id": "65547bb2e5a67d7739674108eecb76a401e87a1504a20c87414baa89f3977ac4", + "name": "reader-index:converse=false:reverse=false", + "training": { + "passed": 2, + "total": 5, + "steps": 8120, + "loopVisits": 417, + "maxCells": 8393, + "failures": [ + { + "name": "train-2", + "error": "Output differs from independent reference" + }, + { + "name": "train-3", + "error": "Output differs from independent reference" + }, + { + "name": "train-precedence", + "error": "Output differs from independent reference" + } + ] + } + }, + { + "id": "63320bd73d81a7b1309cbffd5f580583dda177b73304997503c7c78c8946451f", + "name": "reader-index:converse=true:reverse=true", + "training": { + "passed": 4, + "total": 5, + "steps": 14317, + "loopVisits": 417, + "maxCells": 11694, + "failures": [ + { + "name": "train-precedence", + "error": "Output differs from independent reference" + } + ] + } + }, + { + "id": "f27588c19750c2b54afc261630314ec550a5e73f082d3dc031c280140e703c5f", + "name": "reader-index:converse=true:reverse=false", + "training": { + "passed": 5, + "total": 5, + "steps": 14293, + "loopVisits": 417, + "maxCells": 11694, + "failures": [] + } + } + ], + "splitDigests": { + "training": [ + "9c4212aa05573c75a2f853353a465c15889891dea836ab8360a77cb51ca54382", + "dc3ea2af0f9b6100bd729dcc8c78cef0922fed84599aa9dc24bbca10aea05519", + "ad936483f949cf710857f66ad0cd3b9ac266f3c30213fa6bb04872c6ab42710a", + "58e0b85048ff473e9339cea714f61ad11eb0f22acc9351b614c72d6f41531dfd", + "2c305539d89b28052469b5319313e3805b65709c9148efeff1150a222c9e1c29" + ], + "validation": [ + "567fa95c77e3c8382269e24bfa79ca48d5db79ed522b22090810357aa99e3596", + "d293078e4b93cbded0f144867fd52fd29dc07e8999f140c2b1a1749c7635986f", + "5f92fcbe1b3f45ac8802ecee4c7ea5d99cba6102606c6e16269e70e81d5ae5b6" + ], + "heldOut": [ + "b68e85dc8fbf24df639733ae52c043e8a453ac10a0284bfef29394dd12c58215", + "4b1f41bf7473bfa9882de506fbfbf76c0c14f5b90a362c52c4a05623a4354d01", + "c5282e3f717ca835a43573bb7bf8eaa12b9792eac480cfa63ea0d49d69391657" + ] + }, + "cost": { + "totalSteps": 1450811, + "maxTotalSteps": 30000000, + "maxProposals": 64, + "maxStepsPerCase": 1000000, + "maxCellsPerCase": 200000 + }, + "note": "Steps are deterministic interpreter work units, not native time. No production activation.", + "proposalSource": "authored-structural-grammar", + "interpretation": { + "rejectedCandidates": 9, + "heldOutStepReduction": 0.8886387883197067, + "limitations": [ + "Search is bounded by its proposal grammar; it does not claim globally novel algorithms.", + "Cost is interpreter steps, loop visits and allocated cells, not measured native speed.", + "Cost baselines are explicit IR implementations, not timings of the original FX or Rust code.", + "Memory correctness uses the actual FX getMemory function as its oracle.", + "The Beetle target is a synthetic batch reference model; native Rust conformance is still required.", + "The winner is exported as data and is never automatically deployed." + ] + } +} diff --git a/examples/algorithm-search/results/fx-memory.algorithm.json b/examples/algorithm-search/results/fx-memory.algorithm.json new file mode 100644 index 0000000..6ed6501 --- /dev/null +++ b/examples/algorithm-search/results/fx-memory.algorithm.json @@ -0,0 +1,986 @@ +{ + "version": 1, + "body": [ + { + "kind": "set", + "name": "result", + "value": { + "kind": "call", + "op": "array", + "args": [] + } + }, + { + "kind": "set", + "name": "needle", + "value": { + "kind": "literal", + "value": "" + } + }, + { + "kind": "if", + "condition": { + "kind": "call", + "op": "not", + "args": [ + { + "kind": "call", + "op": "eq", + "args": [ + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "search" + } + }, + { + "kind": "literal", + "value": null + } + ] + } + ] + }, + "then": [ + { + "kind": "set", + "name": "needle", + "value": { + "kind": "call", + "op": "lower", + "args": [ + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "search" + } + } + ] + } + } + ], + "otherwise": [] + }, + { + "kind": "if", + "condition": { + "kind": "call", + "op": "and", + "args": [ + { + "kind": "call", + "op": "not", + "args": [ + { + "kind": "call", + "op": "eq", + "args": [ + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "limit" + } + }, + { + "kind": "literal", + "value": null + } + ] + } + ] + }, + { + "kind": "call", + "op": "and", + "args": [ + { + "kind": "call", + "op": "gt", + "args": [ + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "limit" + } + }, + { + "kind": "literal", + "value": 0 + } + ] + }, + { + "kind": "call", + "op": "eq", + "args": [ + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "limit" + } + }, + { + "kind": "call", + "op": "floor", + "args": [ + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "limit" + } + } + ] + } + ] + } + ] + } + ] + }, + "then": [ + { + "kind": "for", + "name": "entry", + "values": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "entries" + } + }, + "direction": "reverse", + "body": [ + { + "kind": "if", + "condition": { + "kind": "call", + "op": "and", + "args": [ + { + "kind": "call", + "op": "and", + "args": [ + { + "kind": "call", + "op": "or", + "args": [ + { + "kind": "call", + "op": "eq", + "args": [ + { + "kind": "variable", + "name": "needle" + }, + { + "kind": "literal", + "value": "" + } + ] + }, + { + "kind": "call", + "op": "contains", + "args": [ + { + "kind": "call", + "op": "lower", + "args": [ + { + "kind": "get", + "object": { + "kind": "variable", + "name": "entry" + }, + "key": { + "kind": "literal", + "value": "content" + } + } + ] + }, + { + "kind": "variable", + "name": "needle" + } + ] + } + ] + }, + { + "kind": "call", + "op": "or", + "args": [ + { + "kind": "call", + "op": "eq", + "args": [ + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "type" + } + }, + { + "kind": "literal", + "value": null + } + ] + }, + { + "kind": "call", + "op": "eq", + "args": [ + { + "kind": "get", + "object": { + "kind": "variable", + "name": "entry" + }, + "key": { + "kind": "literal", + "value": "type" + } + }, + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "type" + } + } + ] + } + ] + } + ] + }, + { + "kind": "call", + "op": "or", + "args": [ + { + "kind": "call", + "op": "eq", + "args": [ + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "since" + } + }, + { + "kind": "literal", + "value": null + } + ] + }, + { + "kind": "call", + "op": "gte", + "args": [ + { + "kind": "get", + "object": { + "kind": "variable", + "name": "entry" + }, + "key": { + "kind": "literal", + "value": "timestamp" + } + }, + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "since" + } + } + ] + } + ] + } + ] + }, + "then": [ + { + "kind": "append", + "name": "result", + "value": { + "kind": "variable", + "name": "entry" + } + }, + { + "kind": "if", + "condition": { + "kind": "call", + "op": "and", + "args": [ + { + "kind": "call", + "op": "and", + "args": [ + { + "kind": "call", + "op": "not", + "args": [ + { + "kind": "call", + "op": "eq", + "args": [ + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "limit" + } + }, + { + "kind": "literal", + "value": null + } + ] + } + ] + }, + { + "kind": "call", + "op": "and", + "args": [ + { + "kind": "call", + "op": "gt", + "args": [ + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "limit" + } + }, + { + "kind": "literal", + "value": 0 + } + ] + }, + { + "kind": "call", + "op": "eq", + "args": [ + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "limit" + } + }, + { + "kind": "call", + "op": "floor", + "args": [ + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "limit" + } + } + ] + } + ] + } + ] + } + ] + }, + { + "kind": "call", + "op": "gte", + "args": [ + { + "kind": "call", + "op": "length", + "args": [ + { + "kind": "variable", + "name": "result" + } + ] + }, + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "limit" + } + } + ] + } + ] + }, + "then": [ + { + "kind": "break" + } + ], + "otherwise": [] + } + ], + "otherwise": [] + } + ] + }, + { + "kind": "return", + "value": { + "kind": "call", + "op": "reverse", + "args": [ + { + "kind": "variable", + "name": "result" + } + ] + } + } + ], + "otherwise": [ + { + "kind": "for", + "name": "entry", + "values": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "entries" + } + }, + "direction": "forward", + "body": [ + { + "kind": "if", + "condition": { + "kind": "call", + "op": "and", + "args": [ + { + "kind": "call", + "op": "and", + "args": [ + { + "kind": "call", + "op": "or", + "args": [ + { + "kind": "call", + "op": "eq", + "args": [ + { + "kind": "variable", + "name": "needle" + }, + { + "kind": "literal", + "value": "" + } + ] + }, + { + "kind": "call", + "op": "contains", + "args": [ + { + "kind": "call", + "op": "lower", + "args": [ + { + "kind": "get", + "object": { + "kind": "variable", + "name": "entry" + }, + "key": { + "kind": "literal", + "value": "content" + } + } + ] + }, + { + "kind": "variable", + "name": "needle" + } + ] + } + ] + }, + { + "kind": "call", + "op": "or", + "args": [ + { + "kind": "call", + "op": "eq", + "args": [ + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "type" + } + }, + { + "kind": "literal", + "value": null + } + ] + }, + { + "kind": "call", + "op": "eq", + "args": [ + { + "kind": "get", + "object": { + "kind": "variable", + "name": "entry" + }, + "key": { + "kind": "literal", + "value": "type" + } + }, + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "type" + } + } + ] + } + ] + } + ] + }, + { + "kind": "call", + "op": "or", + "args": [ + { + "kind": "call", + "op": "eq", + "args": [ + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "since" + } + }, + { + "kind": "literal", + "value": null + } + ] + }, + { + "kind": "call", + "op": "gte", + "args": [ + { + "kind": "get", + "object": { + "kind": "variable", + "name": "entry" + }, + "key": { + "kind": "literal", + "value": "timestamp" + } + }, + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "since" + } + } + ] + } + ] + } + ] + }, + "then": [ + { + "kind": "append", + "name": "result", + "value": { + "kind": "variable", + "name": "entry" + } + } + ], + "otherwise": [] + } + ] + }, + { + "kind": "if", + "condition": { + "kind": "call", + "op": "and", + "args": [ + { + "kind": "call", + "op": "not", + "args": [ + { + "kind": "call", + "op": "eq", + "args": [ + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "limit" + } + }, + { + "kind": "literal", + "value": null + } + ] + } + ] + }, + { + "kind": "call", + "op": "not", + "args": [ + { + "kind": "call", + "op": "eq", + "args": [ + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "limit" + } + }, + { + "kind": "literal", + "value": 0 + } + ] + } + ] + } + ] + }, + "then": [ + { + "kind": "set", + "name": "result", + "value": { + "kind": "call", + "op": "slice", + "args": [ + { + "kind": "variable", + "name": "result" + }, + { + "kind": "call", + "op": "neg", + "args": [ + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "limit" + } + } + ] + } + ] + } + } + ], + "otherwise": [] + }, + { + "kind": "return", + "value": { + "kind": "variable", + "name": "result" + } + } + ] + } + ] +} diff --git a/examples/algorithm-search/results/fx-memory.json b/examples/algorithm-search/results/fx-memory.json new file mode 100644 index 0000000..04d1705 --- /dev/null +++ b/examples/algorithm-search/results/fx-memory.json @@ -0,0 +1,2157 @@ +{ + "version": 1, + "target": { + "id": "fx.memory.getMemory", + "contract": "Input is {entries:[{id,type,content,timestamp}],filter:{type,since,search,limit}}. Timestamps/since are finite milliseconds; every filter field is nullable. Return complete matching entries in input array order. Preserve actual FX getMemory semantics: type and since filtering, case-insensitive substring search, and finite numeric limit behavior including negative, fractional, zero or null limits." + }, + "accepted": true, + "selected": { + "id": "363bd969920073b374862797dfa6b0a70ce2568aa18a211fffb9d6801ce65e34", + "name": "reverse-guarded:search,type,since", + "algorithm": { + "version": 1, + "body": [ + { + "kind": "set", + "name": "result", + "value": { + "kind": "call", + "op": "array", + "args": [] + } + }, + { + "kind": "set", + "name": "needle", + "value": { + "kind": "literal", + "value": "" + } + }, + { + "kind": "if", + "condition": { + "kind": "call", + "op": "not", + "args": [ + { + "kind": "call", + "op": "eq", + "args": [ + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "search" + } + }, + { + "kind": "literal", + "value": null + } + ] + } + ] + }, + "then": [ + { + "kind": "set", + "name": "needle", + "value": { + "kind": "call", + "op": "lower", + "args": [ + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "search" + } + } + ] + } + } + ], + "otherwise": [] + }, + { + "kind": "if", + "condition": { + "kind": "call", + "op": "and", + "args": [ + { + "kind": "call", + "op": "not", + "args": [ + { + "kind": "call", + "op": "eq", + "args": [ + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "limit" + } + }, + { + "kind": "literal", + "value": null + } + ] + } + ] + }, + { + "kind": "call", + "op": "and", + "args": [ + { + "kind": "call", + "op": "gt", + "args": [ + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "limit" + } + }, + { + "kind": "literal", + "value": 0 + } + ] + }, + { + "kind": "call", + "op": "eq", + "args": [ + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "limit" + } + }, + { + "kind": "call", + "op": "floor", + "args": [ + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "limit" + } + } + ] + } + ] + } + ] + } + ] + }, + "then": [ + { + "kind": "for", + "name": "entry", + "values": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "entries" + } + }, + "direction": "reverse", + "body": [ + { + "kind": "if", + "condition": { + "kind": "call", + "op": "and", + "args": [ + { + "kind": "call", + "op": "and", + "args": [ + { + "kind": "call", + "op": "or", + "args": [ + { + "kind": "call", + "op": "eq", + "args": [ + { + "kind": "variable", + "name": "needle" + }, + { + "kind": "literal", + "value": "" + } + ] + }, + { + "kind": "call", + "op": "contains", + "args": [ + { + "kind": "call", + "op": "lower", + "args": [ + { + "kind": "get", + "object": { + "kind": "variable", + "name": "entry" + }, + "key": { + "kind": "literal", + "value": "content" + } + } + ] + }, + { + "kind": "variable", + "name": "needle" + } + ] + } + ] + }, + { + "kind": "call", + "op": "or", + "args": [ + { + "kind": "call", + "op": "eq", + "args": [ + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "type" + } + }, + { + "kind": "literal", + "value": null + } + ] + }, + { + "kind": "call", + "op": "eq", + "args": [ + { + "kind": "get", + "object": { + "kind": "variable", + "name": "entry" + }, + "key": { + "kind": "literal", + "value": "type" + } + }, + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "type" + } + } + ] + } + ] + } + ] + }, + { + "kind": "call", + "op": "or", + "args": [ + { + "kind": "call", + "op": "eq", + "args": [ + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "since" + } + }, + { + "kind": "literal", + "value": null + } + ] + }, + { + "kind": "call", + "op": "gte", + "args": [ + { + "kind": "get", + "object": { + "kind": "variable", + "name": "entry" + }, + "key": { + "kind": "literal", + "value": "timestamp" + } + }, + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "since" + } + } + ] + } + ] + } + ] + }, + "then": [ + { + "kind": "append", + "name": "result", + "value": { + "kind": "variable", + "name": "entry" + } + }, + { + "kind": "if", + "condition": { + "kind": "call", + "op": "and", + "args": [ + { + "kind": "call", + "op": "and", + "args": [ + { + "kind": "call", + "op": "not", + "args": [ + { + "kind": "call", + "op": "eq", + "args": [ + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "limit" + } + }, + { + "kind": "literal", + "value": null + } + ] + } + ] + }, + { + "kind": "call", + "op": "and", + "args": [ + { + "kind": "call", + "op": "gt", + "args": [ + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "limit" + } + }, + { + "kind": "literal", + "value": 0 + } + ] + }, + { + "kind": "call", + "op": "eq", + "args": [ + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "limit" + } + }, + { + "kind": "call", + "op": "floor", + "args": [ + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "limit" + } + } + ] + } + ] + } + ] + } + ] + }, + { + "kind": "call", + "op": "gte", + "args": [ + { + "kind": "call", + "op": "length", + "args": [ + { + "kind": "variable", + "name": "result" + } + ] + }, + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "limit" + } + } + ] + } + ] + }, + "then": [ + { + "kind": "break" + } + ], + "otherwise": [] + } + ], + "otherwise": [] + } + ] + }, + { + "kind": "return", + "value": { + "kind": "call", + "op": "reverse", + "args": [ + { + "kind": "variable", + "name": "result" + } + ] + } + } + ], + "otherwise": [ + { + "kind": "for", + "name": "entry", + "values": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "entries" + } + }, + "direction": "forward", + "body": [ + { + "kind": "if", + "condition": { + "kind": "call", + "op": "and", + "args": [ + { + "kind": "call", + "op": "and", + "args": [ + { + "kind": "call", + "op": "or", + "args": [ + { + "kind": "call", + "op": "eq", + "args": [ + { + "kind": "variable", + "name": "needle" + }, + { + "kind": "literal", + "value": "" + } + ] + }, + { + "kind": "call", + "op": "contains", + "args": [ + { + "kind": "call", + "op": "lower", + "args": [ + { + "kind": "get", + "object": { + "kind": "variable", + "name": "entry" + }, + "key": { + "kind": "literal", + "value": "content" + } + } + ] + }, + { + "kind": "variable", + "name": "needle" + } + ] + } + ] + }, + { + "kind": "call", + "op": "or", + "args": [ + { + "kind": "call", + "op": "eq", + "args": [ + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "type" + } + }, + { + "kind": "literal", + "value": null + } + ] + }, + { + "kind": "call", + "op": "eq", + "args": [ + { + "kind": "get", + "object": { + "kind": "variable", + "name": "entry" + }, + "key": { + "kind": "literal", + "value": "type" + } + }, + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "type" + } + } + ] + } + ] + } + ] + }, + { + "kind": "call", + "op": "or", + "args": [ + { + "kind": "call", + "op": "eq", + "args": [ + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "since" + } + }, + { + "kind": "literal", + "value": null + } + ] + }, + { + "kind": "call", + "op": "gte", + "args": [ + { + "kind": "get", + "object": { + "kind": "variable", + "name": "entry" + }, + "key": { + "kind": "literal", + "value": "timestamp" + } + }, + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "since" + } + } + ] + } + ] + } + ] + }, + "then": [ + { + "kind": "append", + "name": "result", + "value": { + "kind": "variable", + "name": "entry" + } + } + ], + "otherwise": [] + } + ] + }, + { + "kind": "if", + "condition": { + "kind": "call", + "op": "and", + "args": [ + { + "kind": "call", + "op": "not", + "args": [ + { + "kind": "call", + "op": "eq", + "args": [ + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "limit" + } + }, + { + "kind": "literal", + "value": null + } + ] + } + ] + }, + { + "kind": "call", + "op": "not", + "args": [ + { + "kind": "call", + "op": "eq", + "args": [ + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "limit" + } + }, + { + "kind": "literal", + "value": 0 + } + ] + } + ] + } + ] + }, + "then": [ + { + "kind": "set", + "name": "result", + "value": { + "kind": "call", + "op": "slice", + "args": [ + { + "kind": "variable", + "name": "result" + }, + { + "kind": "call", + "op": "neg", + "args": [ + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "limit" + } + } + ] + } + ] + } + } + ], + "otherwise": [] + }, + { + "kind": "return", + "value": { + "kind": "variable", + "name": "result" + } + } + ] + } + ] + }, + "training": { + "passed": 10, + "total": 10, + "steps": 16379, + "loopVisits": 406, + "maxCells": 13346, + "failures": [] + }, + "validation": { + "passed": 10, + "total": 10, + "steps": 19923, + "loopVisits": 527, + "maxCells": 19182, + "failures": [] + }, + "heldOut": { + "passed": 10, + "total": 10, + "steps": 33578, + "loopVisits": 1138, + "maxCells": 55248, + "failures": [] + } + }, + "baseline": { + "algorithm": { + "version": 1, + "body": [ + { + "kind": "set", + "name": "result", + "value": { + "kind": "call", + "op": "array", + "args": [] + } + }, + { + "kind": "set", + "name": "needle", + "value": { + "kind": "literal", + "value": "" + } + }, + { + "kind": "if", + "condition": { + "kind": "call", + "op": "not", + "args": [ + { + "kind": "call", + "op": "eq", + "args": [ + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "search" + } + }, + { + "kind": "literal", + "value": null + } + ] + } + ] + }, + "then": [ + { + "kind": "set", + "name": "needle", + "value": { + "kind": "call", + "op": "lower", + "args": [ + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "search" + } + } + ] + } + } + ], + "otherwise": [] + }, + { + "kind": "for", + "name": "entry", + "values": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "entries" + } + }, + "direction": "forward", + "body": [ + { + "kind": "if", + "condition": { + "kind": "call", + "op": "and", + "args": [ + { + "kind": "call", + "op": "and", + "args": [ + { + "kind": "call", + "op": "or", + "args": [ + { + "kind": "call", + "op": "eq", + "args": [ + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "type" + } + }, + { + "kind": "literal", + "value": null + } + ] + }, + { + "kind": "call", + "op": "eq", + "args": [ + { + "kind": "get", + "object": { + "kind": "variable", + "name": "entry" + }, + "key": { + "kind": "literal", + "value": "type" + } + }, + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "type" + } + } + ] + } + ] + }, + { + "kind": "call", + "op": "or", + "args": [ + { + "kind": "call", + "op": "eq", + "args": [ + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "since" + } + }, + { + "kind": "literal", + "value": null + } + ] + }, + { + "kind": "call", + "op": "gte", + "args": [ + { + "kind": "get", + "object": { + "kind": "variable", + "name": "entry" + }, + "key": { + "kind": "literal", + "value": "timestamp" + } + }, + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "since" + } + } + ] + } + ] + } + ] + }, + { + "kind": "call", + "op": "or", + "args": [ + { + "kind": "call", + "op": "eq", + "args": [ + { + "kind": "variable", + "name": "needle" + }, + { + "kind": "literal", + "value": "" + } + ] + }, + { + "kind": "call", + "op": "contains", + "args": [ + { + "kind": "call", + "op": "lower", + "args": [ + { + "kind": "get", + "object": { + "kind": "variable", + "name": "entry" + }, + "key": { + "kind": "literal", + "value": "content" + } + } + ] + }, + { + "kind": "variable", + "name": "needle" + } + ] + } + ] + } + ] + }, + "then": [ + { + "kind": "append", + "name": "result", + "value": { + "kind": "variable", + "name": "entry" + } + } + ], + "otherwise": [] + } + ] + }, + { + "kind": "if", + "condition": { + "kind": "call", + "op": "and", + "args": [ + { + "kind": "call", + "op": "not", + "args": [ + { + "kind": "call", + "op": "eq", + "args": [ + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "limit" + } + }, + { + "kind": "literal", + "value": null + } + ] + } + ] + }, + { + "kind": "call", + "op": "not", + "args": [ + { + "kind": "call", + "op": "eq", + "args": [ + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "limit" + } + }, + { + "kind": "literal", + "value": 0 + } + ] + } + ] + } + ] + }, + "then": [ + { + "kind": "set", + "name": "result", + "value": { + "kind": "call", + "op": "slice", + "args": [ + { + "kind": "variable", + "name": "result" + }, + { + "kind": "call", + "op": "neg", + "args": [ + { + "kind": "get", + "object": { + "kind": "get", + "object": { + "kind": "variable", + "name": "input" + }, + "key": { + "kind": "literal", + "value": "filter" + } + }, + "key": { + "kind": "literal", + "value": "limit" + } + } + ] + } + ] + } + } + ], + "otherwise": [] + }, + { + "kind": "return", + "value": { + "kind": "variable", + "name": "result" + } + } + ] + }, + "training": { + "passed": 10, + "total": 10, + "steps": 23972, + "loopVisits": 592, + "maxCells": 13318, + "failures": [] + }, + "heldOut": { + "passed": 10, + "total": 10, + "steps": 113056, + "loopVisits": 2909, + "maxCells": 55220, + "failures": [] + } + }, + "archive": [ + { + "id": "526975f09319eea1074438bec1f2dbaad91a37ba064258fc8dd576c7900c05a2", + "name": "forward-early:type,since,search", + "training": { + "passed": 4, + "total": 10, + "steps": 22793, + "loopVisits": 406, + "maxCells": 16874, + "failures": [ + { + "name": "train-0", + "error": "Output differs from independent reference" + }, + { + "name": "train-3", + "error": "Output differs from independent reference" + }, + { + "name": "train-4", + "error": "Output differs from independent reference" + }, + { + "name": "train-5", + "error": "Output differs from independent reference" + }, + { + "name": "train-6", + "error": "Output differs from independent reference" + }, + { + "name": "train-7", + "error": "Output differs from independent reference" + } + ] + } + }, + { + "id": "839af090740a9c6eb48887c97cf1306842142d715a5cb6ccb564eec9ded70a6e", + "name": "forward-early:type,search,since", + "training": { + "passed": 4, + "total": 10, + "steps": 21665, + "loopVisits": 406, + "maxCells": 16874, + "failures": [ + { + "name": "train-0", + "error": "Output differs from independent reference" + }, + { + "name": "train-3", + "error": "Output differs from independent reference" + }, + { + "name": "train-4", + "error": "Output differs from independent reference" + }, + { + "name": "train-5", + "error": "Output differs from independent reference" + }, + { + "name": "train-6", + "error": "Output differs from independent reference" + }, + { + "name": "train-7", + "error": "Output differs from independent reference" + } + ] + } + }, + { + "id": "db2e57362560f60fff2c878b7c5edd922081c5da40392c2da3ea71767405362c", + "name": "forward-early:since,type,search", + "training": { + "passed": 4, + "total": 10, + "steps": 22913, + "loopVisits": 406, + "maxCells": 16874, + "failures": [ + { + "name": "train-0", + "error": "Output differs from independent reference" + }, + { + "name": "train-3", + "error": "Output differs from independent reference" + }, + { + "name": "train-4", + "error": "Output differs from independent reference" + }, + { + "name": "train-5", + "error": "Output differs from independent reference" + }, + { + "name": "train-6", + "error": "Output differs from independent reference" + }, + { + "name": "train-7", + "error": "Output differs from independent reference" + } + ] + } + }, + { + "id": "465a424cbdb2534b25e6000d7044a30ac4458e7eccd849fa750e7e7caebf9f03", + "name": "forward-early:since,search,type", + "training": { + "passed": 4, + "total": 10, + "steps": 22067, + "loopVisits": 406, + "maxCells": 16874, + "failures": [ + { + "name": "train-0", + "error": "Output differs from independent reference" + }, + { + "name": "train-3", + "error": "Output differs from independent reference" + }, + { + "name": "train-4", + "error": "Output differs from independent reference" + }, + { + "name": "train-5", + "error": "Output differs from independent reference" + }, + { + "name": "train-6", + "error": "Output differs from independent reference" + }, + { + "name": "train-7", + "error": "Output differs from independent reference" + } + ] + } + }, + { + "id": "22501a5aa4bfc7a8ecda89a21fe93622055ba63d7cae51452e723c671926328f", + "name": "forward-early:search,type,since", + "training": { + "passed": 4, + "total": 10, + "steps": 20819, + "loopVisits": 406, + "maxCells": 16874, + "failures": [ + { + "name": "train-0", + "error": "Output differs from independent reference" + }, + { + "name": "train-3", + "error": "Output differs from independent reference" + }, + { + "name": "train-4", + "error": "Output differs from independent reference" + }, + { + "name": "train-5", + "error": "Output differs from independent reference" + }, + { + "name": "train-6", + "error": "Output differs from independent reference" + }, + { + "name": "train-7", + "error": "Output differs from independent reference" + } + ] + } + }, + { + "id": "940d0d4e6b4b0bcb5b3058ae2d1b4764874655fa2020a360ed56e94b9f158795", + "name": "forward-early:search,since,type", + "training": { + "passed": 4, + "total": 10, + "steps": 20867, + "loopVisits": 406, + "maxCells": 16874, + "failures": [ + { + "name": "train-0", + "error": "Output differs from independent reference" + }, + { + "name": "train-3", + "error": "Output differs from independent reference" + }, + { + "name": "train-4", + "error": "Output differs from independent reference" + }, + { + "name": "train-5", + "error": "Output differs from independent reference" + }, + { + "name": "train-6", + "error": "Output differs from independent reference" + }, + { + "name": "train-7", + "error": "Output differs from independent reference" + } + ] + } + }, + { + "id": "5d7c27014ad6f767326a340bf67dbbbe4b176b0f8a8e347b133df14ee955a210", + "name": "reverse-unordered:type,since,search", + "training": { + "passed": 3, + "total": 10, + "steps": 22675, + "loopVisits": 406, + "maxCells": 16874, + "failures": [ + { + "name": "train-0", + "error": "Output differs from independent reference" + }, + { + "name": "train-1", + "error": "Output differs from independent reference" + }, + { + "name": "train-2", + "error": "Output differs from independent reference" + }, + { + "name": "train-3", + "error": "Output differs from independent reference" + }, + { + "name": "train-4", + "error": "Output differs from independent reference" + }, + { + "name": "train-5", + "error": "Output differs from independent reference" + }, + { + "name": "train-6", + "error": "Output differs from independent reference" + } + ] + } + }, + { + "id": "4004e87b0bc1b97f603adccf3f6093c4ee02169238eb0be0f74a6c47bcf04c52", + "name": "reverse-unordered:type,search,since", + "training": { + "passed": 3, + "total": 10, + "steps": 21571, + "loopVisits": 406, + "maxCells": 16874, + "failures": [ + { + "name": "train-0", + "error": "Output differs from independent reference" + }, + { + "name": "train-1", + "error": "Output differs from independent reference" + }, + { + "name": "train-2", + "error": "Output differs from independent reference" + }, + { + "name": "train-3", + "error": "Output differs from independent reference" + }, + { + "name": "train-4", + "error": "Output differs from independent reference" + }, + { + "name": "train-5", + "error": "Output differs from independent reference" + }, + { + "name": "train-6", + "error": "Output differs from independent reference" + } + ] + } + }, + { + "id": "42a702d1f703032ea38112db3919842406f93c77f23c51b36caa58c6693cb0e1", + "name": "reverse-unordered:since,type,search", + "training": { + "passed": 3, + "total": 10, + "steps": 22819, + "loopVisits": 406, + "maxCells": 16874, + "failures": [ + { + "name": "train-0", + "error": "Output differs from independent reference" + }, + { + "name": "train-1", + "error": "Output differs from independent reference" + }, + { + "name": "train-2", + "error": "Output differs from independent reference" + }, + { + "name": "train-3", + "error": "Output differs from independent reference" + }, + { + "name": "train-4", + "error": "Output differs from independent reference" + }, + { + "name": "train-5", + "error": "Output differs from independent reference" + }, + { + "name": "train-6", + "error": "Output differs from independent reference" + } + ] + } + }, + { + "id": "c3e2e6eaefb5de422d245e12cc7b1bf0146c069c7fa750f5583d54bdda700da3", + "name": "reverse-unordered:since,search,type", + "training": { + "passed": 3, + "total": 10, + "steps": 22009, + "loopVisits": 406, + "maxCells": 16874, + "failures": [ + { + "name": "train-0", + "error": "Output differs from independent reference" + }, + { + "name": "train-1", + "error": "Output differs from independent reference" + }, + { + "name": "train-2", + "error": "Output differs from independent reference" + }, + { + "name": "train-3", + "error": "Output differs from independent reference" + }, + { + "name": "train-4", + "error": "Output differs from independent reference" + }, + { + "name": "train-5", + "error": "Output differs from independent reference" + }, + { + "name": "train-6", + "error": "Output differs from independent reference" + } + ] + } + }, + { + "id": "8e306413eab6325223c8353f94b7bd8dd50999bf9a4ea2d2a6e34eea951e2505", + "name": "reverse-unordered:search,type,since", + "training": { + "passed": 3, + "total": 10, + "steps": 20761, + "loopVisits": 406, + "maxCells": 16874, + "failures": [ + { + "name": "train-0", + "error": "Output differs from independent reference" + }, + { + "name": "train-1", + "error": "Output differs from independent reference" + }, + { + "name": "train-2", + "error": "Output differs from independent reference" + }, + { + "name": "train-3", + "error": "Output differs from independent reference" + }, + { + "name": "train-4", + "error": "Output differs from independent reference" + }, + { + "name": "train-5", + "error": "Output differs from independent reference" + }, + { + "name": "train-6", + "error": "Output differs from independent reference" + } + ] + } + }, + { + "id": "6cb31665913dc61578b4b79e2d2b12b1e8df36a4a17a6462ada5628e725fa8fd", + "name": "reverse-unordered:search,since,type", + "training": { + "passed": 3, + "total": 10, + "steps": 20833, + "loopVisits": 406, + "maxCells": 16874, + "failures": [ + { + "name": "train-0", + "error": "Output differs from independent reference" + }, + { + "name": "train-1", + "error": "Output differs from independent reference" + }, + { + "name": "train-2", + "error": "Output differs from independent reference" + }, + { + "name": "train-3", + "error": "Output differs from independent reference" + }, + { + "name": "train-4", + "error": "Output differs from independent reference" + }, + { + "name": "train-5", + "error": "Output differs from independent reference" + }, + { + "name": "train-6", + "error": "Output differs from independent reference" + } + ] + } + }, + { + "id": "eda440c8a1776537aa5781e64a8a302e17bc29f7903829766e9745363f151d72", + "name": "forward:type,since,search", + "training": { + "passed": 10, + "total": 10, + "steps": 23972, + "loopVisits": 592, + "maxCells": 13318, + "failures": [] + } + }, + { + "id": "255fd5de87ab30b652786853c61742f19ce994371e81035b6cbed0e2e9c972bf", + "name": "forward:type,search,since", + "training": { + "passed": 10, + "total": 10, + "steps": 22828, + "loopVisits": 592, + "maxCells": 13318, + "failures": [] + } + }, + { + "id": "e9d05297a1a8cd6339f41d750ee844e365e1b06fab3448cf0e22db57bd30b0fa", + "name": "forward:since,type,search", + "training": { + "passed": 10, + "total": 10, + "steps": 24836, + "loopVisits": 592, + "maxCells": 13318, + "failures": [] + } + }, + { + "id": "f6ddf4a8381e413a9df1eb94b6d6771d90e65d6bfe1f001e3e0077da85ba1063", + "name": "forward:since,search,type", + "training": { + "passed": 10, + "total": 10, + "steps": 24346, + "loopVisits": 592, + "maxCells": 13318, + "failures": [] + } + }, + { + "id": "c7135a7a906b9bedd66daf05fe547f764efc626469c0797675bd7f8a56cfddee", + "name": "forward:search,type,since", + "training": { + "passed": 10, + "total": 10, + "steps": 22338, + "loopVisits": 592, + "maxCells": 13318, + "failures": [] + } + }, + { + "id": "20a017ef6c858665a499fb1cfc2c6b5d67ab528426a38d9122b2b805656c5eba", + "name": "forward:search,since,type", + "training": { + "passed": 10, + "total": 10, + "steps": 23130, + "loopVisits": 592, + "maxCells": 13318, + "failures": [] + } + }, + { + "id": "2bcbafd92e9b7593d1e2349a5e17ef084475c299933e96aeb6d1552513971e0e", + "name": "reverse-guarded:type,since,search", + "training": { + "passed": 10, + "total": 10, + "steps": 18293, + "loopVisits": 406, + "maxCells": 13346, + "failures": [] + } + }, + { + "id": "653c8035482cd803235ea1a4207ba273707ef107806f7441aec922dc2c4f1169", + "name": "reverse-guarded:type,search,since", + "training": { + "passed": 10, + "total": 10, + "steps": 17189, + "loopVisits": 406, + "maxCells": 13346, + "failures": [] + } + }, + { + "id": "4ba5712f27d3073936147cfa11ca4eaabb22c16f2520e783fa1f0c0d58e1f9c6", + "name": "reverse-guarded:since,type,search", + "training": { + "passed": 10, + "total": 10, + "steps": 18437, + "loopVisits": 406, + "maxCells": 13346, + "failures": [] + } + }, + { + "id": "5f79e179ab91dbad40e9fe0e37524828323ebbb1597df62dbd779783420a329f", + "name": "reverse-guarded:since,search,type", + "training": { + "passed": 10, + "total": 10, + "steps": 17627, + "loopVisits": 406, + "maxCells": 13346, + "failures": [] + } + }, + { + "id": "363bd969920073b374862797dfa6b0a70ce2568aa18a211fffb9d6801ce65e34", + "name": "reverse-guarded:search,type,since", + "training": { + "passed": 10, + "total": 10, + "steps": 16379, + "loopVisits": 406, + "maxCells": 13346, + "failures": [] + } + }, + { + "id": "8e6a0ca555c32f99fc62ae36def78ade731f519a925048c30812614e588d19fa", + "name": "reverse-guarded:search,since,type", + "training": { + "passed": 10, + "total": 10, + "steps": 16451, + "loopVisits": 406, + "maxCells": 13346, + "failures": [] + } + } + ], + "splitDigests": { + "training": [ + "dc05fb2f78aab3c8eab1d999dda515bd86c633154aa7d8e317d03ff1ab9e2144", + "bb6d80ee1029394d588049aa3acafb5d955c15567bd457c7ea443644cf263ee4", + "40ebcc1d4d3db94774c489de9560c28f86900cde0f823b4229f53940c0539df3", + "f26ef56b4cb2ca7725cf65f82faa74ceaf8c2d8ac427cbccc5e8fd17193d2ab0", + "38846887b15263c8c92263edd4d5911a60e02aad57a6e19fdd2fda688a85ed78", + "127714431a74e917b8c013d8cffee3f24bbaf2073692cfb960c13bf673678848", + "f774265d5fb2e30ddfa06fc72ebe012147a8327382baa0f612ea6d19b41445fe", + "08aa364a9da25d96a0ee70640f54f3008268c0a35d672bf7d299c3f5abad3cc7", + "d7e1b171b0b1b7c5353743dd9f3efe691c442f2599a5bf77c946e4e02bf09ab2", + "ce20f500d238b8ee933d3ae04d2b2b893dc0a257041ed7be6b18fb885ee2c77f" + ], + "validation": [ + "c113c16155e79a552e953ab4545c8c4f47768f9d294baf02626647242a511862", + "c76987cc222e9c5c48d612db0aaf0d5778e5660edb758e02d92f8fa1f339fb86", + "221cb08a4d68138961c8dbe9e9013e3bdfe7893baeb83d6445b03d8d4a280442", + "6ba40a4086dfcc8bf8ece302b53390468f509ffc0fbb15d9cd92f5b598eac86f", + "5017aa5cc6f83ac0267ba4832d0a4e43af387fd2ddae05eb137426356c8ebd3d", + "305bb835663e0cb55dffebdbbe767bff46fbe1f186e021afe41ff2c3e6a161cd", + "66a32b99d1e6828e2f094fa2226eb6ef6e2ec670301423ab32f8596ced950a45", + "1b68b52d63ab75d6e11855c0aeac58b685882f02e3df25d75c3b885bf3c1ec79", + "3e6737c4c1c52249d3145fbdf97200a117c7680a69857ed9c1412d88a242eb11", + "45674e67beb17efe27f371a33de6aa696fd79269d6785e319171868ac5bd8b99" + ], + "heldOut": [ + "07eaec1f052c12b9a44ff982b04c605f023ba1131d5eb890f1351a4b5976f420", + "2a72b03c7741b4436bd7fa5ead2a121a575bcc7284bbebad4a2f3d31771f6057", + "d5408302554d8f2a86561e262a53954f15505319e1300093b2187968a2b8792d", + "51cba8e9078e2edd25931234883b467056ea7ff560351fb673345d21cf3839a5", + "18fb4fb6d7b312fe5539071c3b0524181a2bb77bffa47ad9932b32b85d0855a9", + "317a7134ea856e264806e42bcd2cc571a8e5cbf0bdc7041c8780d6e33cb72b62", + "362aa256c5e7fff98caa83c06565ed33968099ad61462e04a6b3b10c23613d6f", + "317a4a1ba38849315f0a25012f28c465ba241287e9ddf72be0405c31ad61ab67", + "b7dd63ba89eee29f75062d684927268a4a7c3a59ef29f4a616857cbbe12307fa", + "c3cf0517eb27e390cdfe6950d8c6cb3ad9715a0a6aef1d8092bb41985061ce38" + ] + }, + "cost": { + "totalSteps": 814436, + "maxTotalSteps": 30000000, + "maxProposals": 64, + "maxStepsPerCase": 1000000, + "maxCellsPerCase": 200000 + }, + "note": "Steps are deterministic interpreter work units, not native time. No production activation.", + "proposalSource": "authored-structural-grammar", + "interpretation": { + "rejectedCandidates": 12, + "heldOutStepReduction": 0.7029967449759411, + "limitations": [ + "Search is bounded by its proposal grammar; it does not claim globally novel algorithms.", + "Cost is interpreter steps, loop visits and allocated cells, not measured native speed.", + "Cost baselines are explicit IR implementations, not timings of the original FX or Rust code.", + "Memory correctness uses the actual FX getMemory function as its oracle.", + "The Beetle target is a synthetic batch reference model; native Rust conformance is still required.", + "The winner is exported as data and is never automatically deployed." + ] + } +} diff --git a/examples/algorithm-search/results/manifest.json b/examples/algorithm-search/results/manifest.json new file mode 100644 index 0000000..6cd9791 --- /dev/null +++ b/examples/algorithm-search/results/manifest.json @@ -0,0 +1,54 @@ +{ + "baseRevision": "09795eaa62b5dd5d87f1b9f92efe04340be9dcd0", + "node": "v24.22.1", + "sourceSha256": { + ".github/workflows/ci.yml": "36ba15f487863752952456e800631c39aafce3adec3ffdc8f314806aff858b24", + "examples/algorithm-search/algorithm-search.test.cjs": "4ae01daed8395e5ff333874a0f5724e6a1a9d85c59ec1e82140561888cfac77c", + "examples/algorithm-search/builders.cjs": "6a4a98ba0d6d53761fec501928c034985637abc98e5c08c9e3d9955296bda5af", + "examples/algorithm-search/memory.cjs": "6a62f8dbaada8196a808785984592d51cb27a8f9aa69f8d750050fc515a93d71", + "examples/algorithm-search/routes.cjs": "8c1773c2108bc54ed2260ecb89c4ae1cde16f2a9782b08d2b1d33dfa6bfbcecf", + "examples/algorithm-search/run.cjs": "5c39395630e6cd793c20ec3db2b32c27a4a8e9dfa8454fff0fedd0b98764581a", + "examples/discovery/experiments.test.cjs": "d8ea5c4077192b010e879d4ebec5d9f611ad0d962b464584bbc69b31eb557515", + "examples/discovery/scheduling-discovery.cjs": "816dadc6a6f5d7f371d4121ae215bdae4d2f4e9c872966e0f58d1e69509595f5", + "examples/discovery/scheduling.cjs": "3d8894c30f56d2b324b1ebd73a8f77361a4c5e9c6d9af7ed7d74d3c0e95f4401", + "examples/discovery/transfer.cjs": "d594c1e37ffe2d20310efcaadf702338cfd7e67b1625ae0f1ecdb005ddac219c", + "package.json": "6f6a6ed6458a65f30a0f8f2cad0feebbc8915883f9d0b9a080783dd940eb4b12", + "packages/fx-core/package.json": "f1c969ad9c921f4721a7320fa2f6e7f86fc2032861a3778ebc5928e286a59391", + "packages/fx-core/src/__tests__/algorithm-proposer.test.ts": "58d77f35d5590ce76fc303028eae48f56bc1ec9cb96bb345d2f041d67d51a091", + "packages/fx-core/src/__tests__/algorithm-runtime.test.ts": "a966ff5c51692d7038a022b68d3fd36a78ab94fba76763e93a29784e847a6dbe", + "packages/fx-core/src/__tests__/algorithm-search.test.ts": "4ee75694a33dc5c39277b133c4d190e22fc9ba14ccbc580e7bf670eac01cb6d6", + "packages/fx-core/src/__tests__/discovery-learning.test.ts": "c329454c55569e56c2d91793f0b57cb6ca50ec3b620d952e07eb774c3ed155b3", + "packages/fx-core/src/__tests__/discovery-runtime.test.ts": "1d0cb5e3af4e45717368911f7e082506788600482a4c30c05749b823851307bf", + "packages/fx-core/src/discovery/abstractions.ts": "2816285cba38041bfb87fd8c93bf707672106596d9cf41d616dbbeacd7fb77ee", + "packages/fx-core/src/discovery/algorithm-model.ts": "dc2adfbdebe6e1a926891fa0a7f6330aed27ee6c8ecf5b9d6db83ae90e2861a0", + "packages/fx-core/src/discovery/algorithm-proposer.ts": "32956a7e50b759278f9da40bb3a5ea101b1b83219ddcf5db83c57349cd48a945", + "packages/fx-core/src/discovery/algorithm-runtime.ts": "18f8c35a70e529c87bdcfce4f6da5f67ac825939d44b2fb5cdfe54e5b628765e", + "packages/fx-core/src/discovery/algorithm-search.ts": "0e46ea637a0aa9c33a2c07adcb85fb5bc54688720c52a5ad192938092f988031", + "packages/fx-core/src/discovery/algorithm-values.ts": "671769e0fb6c11f3f8b840ed4611060b0e0095f122937ca5c821b55ccfdb580e", + "packages/fx-core/src/discovery/index.ts": "6585c3cdfdc7c32c734b71030fe5b833d7e6df2ebed27d9384dccb90f60e641f", + "packages/fx-core/src/discovery/model.ts": "14be2a8b12799012ae2de3073285cf715c666d3d5d3f004d4ccfc26ccdc03a39", + "packages/fx-core/src/discovery/optimization.ts": "6bd7c36a67b081fca0ca70be56c5419a87005b6463b90d72aaa2225e120fd086", + "packages/fx-core/src/discovery/runtime.ts": "60d478996cc47f915f5b87cf9a87f5e7f28235aafc66a5f32d8e8f407b930068", + "packages/fx-core/src/discovery/search.ts": "385d53ef14505733f5aeda3b55056be1f7c5642700fcecb5fbd7be973ab6dc03", + "packages/fx-core/src/discovery/state.ts": "fb107bbbded284f32a09cdb50909cd7a999343f27431fc77435771ee4a0d9942", + "packages/fx-core/src/index.ts": "8e5f25252124a7493fda966f31a30d2c42593d21ba9c8d1c6f41d6892199cfc8", + "packages/fx-core/src/memory.ts": "5e795790cff6d096b7906e7a2130553672ab44e0b34fb59c57fe8d8bc9b704ba" + }, + "checks": { + "libraryTests": 193, + "experimentTests": 7, + "lint": "passed", + "typeCheck": "passed", + "build": "passed", + "esmExport": "passed", + "discoveryLineCoverage": 97.03, + "discoveryBranchCoverage": 82.17, + "liveModelRun": false + }, + "outputSha256": { + "beetle-routes.algorithm.json": "74778035d5736fba1e89d0e857a8dfe518952a846caf3e24d5fe1af39afcea43", + "beetle-routes.json": "8036366192880d4ca62c8001f0029097652a9b3a104bec686e713027c288b6bc", + "fx-memory.algorithm.json": "0e690971be59d1a8df946c530231f776066162df2611ad5eb733dad4512a0ae0", + "fx-memory.json": "0f7bde410ad8c83ac3593b4d9c2235550f027dc2dd69ca95726e49b3236613d3" + } +} diff --git a/examples/algorithm-search/routes.cjs b/examples/algorithm-search/routes.cjs new file mode 100644 index 0000000..be201bf --- /dev/null +++ b/examples/algorithm-search/routes.cjs @@ -0,0 +1,158 @@ +const { + lit, + variable: v, + get, + call, + set, + append, + put, + branch, + loop, + ret, + stop, + input, +} = require('./builders.cjs'); +const routeAddress = get(v('route'), 'address'); +const converse = (route) => call('eq', get(route, 'policy'), lit('converse')); +const notNull = (value) => call('not', call('eq', value, lit(null))); + +function routeAlgorithm(strategy, preferConverse = true, reverse = false) { + const indexed = strategy !== 'scan'; + const body = [set('output', call('array'))]; + if (indexed) { + body.push(set('index', call('map'))); + if (strategy === 'bucket-index') + body.push( + loop('route', input('routes'), [ + branch(call('not', call('has', v('index'), routeAddress)), [ + put('index', routeAddress, call('array')), + ]), + set('bucket', call('lookup', v('index'), routeAddress)), + append('bucket', v('route')), + ]), + ); + else + body.push( + loop('route', input('routes'), [ + branch( + call('not', call('has', v('index'), routeAddress)), + [put('index', routeAddress, v('route'))], + preferConverse + ? [ + branch( + call( + 'and', + call('not', converse(call('lookup', v('index'), routeAddress))), + converse(v('route')), + ), + [put('index', routeAddress, v('route'))], + ), + ] + : [], + ), + ]), + ); + } + const addressBody = []; + if (strategy === 'reader-index') + addressBody.push( + branch(call('has', v('index'), v('address')), [ + set('chosen', call('lookup', v('index'), v('address'))), + stop, + ]), + ); + else { + if (indexed) + addressBody.push( + set('bucket', call('array')), + branch(call('has', v('index'), v('address')), [ + set('bucket', call('lookup', v('index'), v('address'))), + ]), + ); + addressBody.push( + set('first', lit(null)), + loop('route', indexed ? v('bucket') : input('routes'), [ + branch(call('eq', routeAddress, v('address')), [ + branch(call('eq', v('first'), lit(null)), [set('first', v('route'))]), + ...(preferConverse + ? [branch(converse(v('route')), [set('first', v('route')), stop])] + : []), + ]), + ]), + branch(notNull(v('first')), [set('chosen', v('first')), stop]), + ); + } + body.push( + loop('query', input('queries'), [ + set('chosen', lit(null)), + loop('address', v('query'), addressBody, reverse ? 'reverse' : 'forward'), + append('output', v('chosen')), + ]), + ret(v('output')), + ); + return { version: 1, body }; +} +function routeReference(input) { + return input.queries.map((addresses) => { + for (const address of addresses) { + const level = input.routes.filter((route) => route.address === address); + if (level.length) return level.find((route) => route.policy === 'converse') ?? level[0]; + } + return null; + }); +} +function fixture(seed, size, queries) { + const routes = Array.from({ length: size }, (_, i) => ({ + id: `route-${seed}-${i}`, + address: `channel:${seed}:${i % 17}`, + policy: i % 5 === 0 ? 'converse' : 'observe', + metadata: { source: 'synthetic' }, + })); + return { + routes, + queries: Array.from({ length: queries }, (_, i) => [ + `thread:${seed}:${i}`, + `channel:${seed}:${i % 19}`, + `workspace:${seed}`, + ]), + }; +} +function routeTarget() { + const make = (prefix, seed, sizes) => + sizes.map((n, i) => ({ name: `${prefix}-${i}`, input: fixture(seed + i, n, i ? 20 : 3) })); + return { + id: 'beetle.route.batch-most-specific', + contract: + 'Input is {routes:[{id,address,policy,...otherFields}],queries:[orderedAddressStrings[]]}. For every query return the first converse route at the nearest nonempty address level, otherwise its first route, otherwise null. Return an array preserving full route records and query order. Routes are an immutable snapshot for the batch. This is a selection reference model, not the live Rust authorization router.', + baseline: routeAlgorithm('scan'), + reference: routeReference, + training: [ + ...make('train', 1, [0, 4, 64, 128]), + { + name: 'train-precedence', + input: { + routes: [ + { id: 'observer', address: 'thread', policy: 'observe' }, + { id: 'primary', address: 'thread', policy: 'converse' }, + { id: 'later', address: 'thread', policy: 'converse' }, + { id: 'channel', address: 'channel', policy: 'converse' }, + ], + queries: [['thread', 'channel'], ['channel', 'thread'], ['absent']], + }, + }, + ], + validation: make('validation', 101, [8, 96, 180]), + heldOut: make('test', 201, [1, 256, 512]), + }; +} +function routeCandidates() { + return ['scan', 'bucket-index', 'reader-index'].flatMap((strategy) => + [false, true].flatMap((prefer) => + [true, false].map((reverse) => ({ + name: `${strategy}:converse=${prefer}:reverse=${reverse}`, + algorithm: routeAlgorithm(strategy, prefer, reverse), + })), + ), + ); +} +module.exports = { routeTarget, routeCandidates, routeAlgorithm, routeReference }; diff --git a/examples/algorithm-search/run.cjs b/examples/algorithm-search/run.cjs new file mode 100644 index 0000000..04168a4 --- /dev/null +++ b/examples/algorithm-search/run.cjs @@ -0,0 +1,116 @@ +const fs = require('node:fs'); +const path = require('node:path'); +const { + searchAlgorithms, + createModelAlgorithmProposer, +} = require('../../packages/fx-core/dist/discovery'); +const { createOpenAIProvider } = require('../../packages/fx-core/dist'); +const { memoryTarget, memoryCandidates } = require('./memory.cjs'); +const { routeTarget, routeCandidates } = require('./routes.cjs'); + +const targets = { + 'fx-memory': [memoryTarget, memoryCandidates], + 'beetle-routes': [routeTarget, routeCandidates], +}; + +async function runTarget(name, options = {}) { + if (!targets[name]) throw new Error(`Unknown target: ${name}`); + const [makeTarget, makeCandidates] = targets[name]; + const proposals = options.proposals ?? makeCandidates(); + let next = 0; + const proposer = options.provider + ? createModelAlgorithmProposer(options.provider, { + model: options.model, + maxCalls: options.budget ?? 16, + }) + : () => proposals[next++]; + const result = await searchAlgorithms(makeTarget(), proposer, { + maxProposals: options.budget ?? 64, + maxSteps: 1_000_000, + maxTotalSteps: 30_000_000, + }); + return { + ...result, + archive: result.archive.map(({ algorithm, ...trial }) => trial), + proposalSource: options.provider + ? 'model-generated-IR' + : options.proposals + ? 'external-JSON' + : 'authored-structural-grammar', + ...(proposer.usage ? { modelUsage: proposer.usage() } : {}), + interpretation: { + rejectedCandidates: result.archive.filter( + (trial) => trial.error || trial.training?.passed !== trial.training?.total, + ).length, + heldOutStepReduction: 1 - result.selected.heldOut.steps / result.baseline.heldOut.steps, + limitations: [ + 'Search is bounded by its proposal grammar; it does not claim globally novel algorithms.', + 'Cost is interpreter steps, loop visits and allocated cells, not measured native speed.', + 'Cost baselines are explicit IR implementations, not timings of the original FX or Rust code.', + 'Memory correctness uses the actual FX getMemory function as its oracle.', + 'The Beetle target is a synthetic batch reference model; native Rust conformance is still required.', + 'The winner is exported as data and is never automatically deployed.', + ], + }, + }; +} + +async function main(args) { + const options = {}; + for (let i = 0; i < args.length; i += 2) { + if ( + !['--target', '--out', '--budget', '--proposals', '--model'].includes(args[i]) || + !args[i + 1] + ) + throw new Error( + 'Expected --target, --out, --budget, --proposals or --model followed by a value', + ); + options[args[i].slice(2)] = args[i + 1]; + } + const names = + !options.target || options.target === 'all' ? Object.keys(targets) : [options.target]; + if (options.proposals && names.length !== 1) + throw new Error('External proposals require one explicit target'); + if (options.proposals && options.model) + throw new Error('Choose a model or an external proposals file'); + const provider = options.model + ? createOpenAIProvider({ + apiKey: process.env.OPENAI_API_KEY, + ...(process.env.OPENAI_BASE_URL ? { baseURL: process.env.OPENAI_BASE_URL } : {}), + }) + : undefined; + const proposals = options.proposals + ? JSON.parse(fs.readFileSync(options.proposals, 'utf8')) + : undefined; + if (proposals && !Array.isArray(proposals)) + throw new Error('Proposals must be an array of {name,algorithm}'); + const reports = []; + for (const name of names) { + const report = await runTarget(name, { + budget: options.budget ? Number(options.budget) : options.model ? 16 : 64, + proposals, + provider, + model: options.model, + }); + reports.push(report); + if (options.out) { + fs.mkdirSync(options.out, { recursive: true }); + fs.writeFileSync( + path.join(options.out, `${name}.json`), + JSON.stringify(report, null, 2) + '\n', + ); + fs.writeFileSync( + path.join(options.out, `${name}.algorithm.json`), + JSON.stringify(report.accepted ? report.selected.algorithm : null, null, 2) + '\n', + ); + } + if (!report.accepted) process.exitCode = 1; + } + process.stdout.write(JSON.stringify(reports, null, 2) + '\n'); +} +if (require.main === module) + main(process.argv.slice(2)).catch((error) => { + console.error(error.message); + process.exitCode = 1; + }); +module.exports = { runTarget }; diff --git a/examples/discovery/results/manifest.json b/examples/discovery/results/manifest.json index 073456f..9b22203 100644 --- a/examples/discovery/results/manifest.json +++ b/examples/discovery/results/manifest.json @@ -1,37 +1,52 @@ { - "baseRevision": "61e11f428660cc215d4467153b47204280b81ff6", + "baseRevision": "09795eaa62b5dd5d87f1b9f92efe04340be9dcd0", "node": "v24.22.1", "sourceSha256": { + ".github/workflows/ci.yml": "36ba15f487863752952456e800631c39aafce3adec3ffdc8f314806aff858b24", + "examples/algorithm-search/algorithm-search.test.cjs": "4ae01daed8395e5ff333874a0f5724e6a1a9d85c59ec1e82140561888cfac77c", + "examples/algorithm-search/builders.cjs": "6a4a98ba0d6d53761fec501928c034985637abc98e5c08c9e3d9955296bda5af", + "examples/algorithm-search/memory.cjs": "6a62f8dbaada8196a808785984592d51cb27a8f9aa69f8d750050fc515a93d71", + "examples/algorithm-search/routes.cjs": "8c1773c2108bc54ed2260ecb89c4ae1cde16f2a9782b08d2b1d33dfa6bfbcecf", + "examples/algorithm-search/run.cjs": "5c39395630e6cd793c20ec3db2b32c27a4a8e9dfa8454fff0fedd0b98764581a", "examples/discovery/experiments.test.cjs": "d8ea5c4077192b010e879d4ebec5d9f611ad0d962b464584bbc69b31eb557515", "examples/discovery/scheduling-discovery.cjs": "816dadc6a6f5d7f371d4121ae215bdae4d2f4e9c872966e0f58d1e69509595f5", "examples/discovery/scheduling.cjs": "3d8894c30f56d2b324b1ebd73a8f77361a4c5e9c6d9af7ed7d74d3c0e95f4401", "examples/discovery/transfer.cjs": "d594c1e37ffe2d20310efcaadf702338cfd7e67b1625ae0f1ecdb005ddac219c", - "package.json": "36e372c4d01bb3b439c70ed4512c8e525d1092a935ed8485290782df3ebc1390", + "package.json": "6f6a6ed6458a65f30a0f8f2cad0feebbc8915883f9d0b9a080783dd940eb4b12", "packages/fx-core/package.json": "f1c969ad9c921f4721a7320fa2f6e7f86fc2032861a3778ebc5928e286a59391", + "packages/fx-core/src/__tests__/algorithm-proposer.test.ts": "58d77f35d5590ce76fc303028eae48f56bc1ec9cb96bb345d2f041d67d51a091", + "packages/fx-core/src/__tests__/algorithm-runtime.test.ts": "a966ff5c51692d7038a022b68d3fd36a78ab94fba76763e93a29784e847a6dbe", + "packages/fx-core/src/__tests__/algorithm-search.test.ts": "4ee75694a33dc5c39277b133c4d190e22fc9ba14ccbc580e7bf670eac01cb6d6", + "packages/fx-core/src/__tests__/discovery-learning.test.ts": "c329454c55569e56c2d91793f0b57cb6ca50ec3b620d952e07eb774c3ed155b3", + "packages/fx-core/src/__tests__/discovery-runtime.test.ts": "1d0cb5e3af4e45717368911f7e082506788600482a4c30c05749b823851307bf", "packages/fx-core/src/discovery/abstractions.ts": "2816285cba38041bfb87fd8c93bf707672106596d9cf41d616dbbeacd7fb77ee", - "packages/fx-core/src/discovery/index.ts": "d677da1ffed8fa46636d0aa8e2a2312e61932534fed45f2af82271ea9d992a92", + "packages/fx-core/src/discovery/algorithm-model.ts": "dc2adfbdebe6e1a926891fa0a7f6330aed27ee6c8ecf5b9d6db83ae90e2861a0", + "packages/fx-core/src/discovery/algorithm-proposer.ts": "32956a7e50b759278f9da40bb3a5ea101b1b83219ddcf5db83c57349cd48a945", + "packages/fx-core/src/discovery/algorithm-runtime.ts": "18f8c35a70e529c87bdcfce4f6da5f67ac825939d44b2fb5cdfe54e5b628765e", + "packages/fx-core/src/discovery/algorithm-search.ts": "0e46ea637a0aa9c33a2c07adcb85fb5bc54688720c52a5ad192938092f988031", + "packages/fx-core/src/discovery/algorithm-values.ts": "671769e0fb6c11f3f8b840ed4611060b0e0095f122937ca5c821b55ccfdb580e", + "packages/fx-core/src/discovery/index.ts": "6585c3cdfdc7c32c734b71030fe5b833d7e6df2ebed27d9384dccb90f60e641f", "packages/fx-core/src/discovery/model.ts": "14be2a8b12799012ae2de3073285cf715c666d3d5d3f004d4ccfc26ccdc03a39", "packages/fx-core/src/discovery/optimization.ts": "6bd7c36a67b081fca0ca70be56c5419a87005b6463b90d72aaa2225e120fd086", "packages/fx-core/src/discovery/runtime.ts": "60d478996cc47f915f5b87cf9a87f5e7f28235aafc66a5f32d8e8f407b930068", "packages/fx-core/src/discovery/search.ts": "385d53ef14505733f5aeda3b55056be1f7c5642700fcecb5fbd7be973ab6dc03", "packages/fx-core/src/discovery/state.ts": "fb107bbbded284f32a09cdb50909cd7a999343f27431fc77435771ee4a0d9942", "packages/fx-core/src/index.ts": "8e5f25252124a7493fda966f31a30d2c42593d21ba9c8d1c6f41d6892199cfc8", - "packages/fx-core/src/__tests__/discovery-learning.test.ts": "c329454c55569e56c2d91793f0b57cb6ca50ec3b620d952e07eb774c3ed155b3", - "packages/fx-core/src/__tests__/discovery-runtime.test.ts": "1d0cb5e3af4e45717368911f7e082506788600482a4c30c05749b823851307bf", - ".github/workflows/ci.yml": "36ba15f487863752952456e800631c39aafce3adec3ffdc8f314806aff858b24" + "packages/fx-core/src/memory.ts": "5e795790cff6d096b7906e7a2130553672ab44e0b34fb59c57fe8d8bc9b704ba" }, "checks": { - "libraryTests": 161, - "experimentTests": 4, + "libraryTests": 193, + "experimentTests": 7, "lint": "passed", "typeCheck": "passed", "build": "passed", "esmExport": "passed", - "discoveryLineCoverage": 98.35, - "discoveryBranchCoverage": 82.73 + "discoveryLineCoverage": 97.03, + "discoveryBranchCoverage": 82.17, + "liveModelRun": false }, "outputSha256": { - "transfer.json": "2c4cbc6cbddc16cb306e07c85378a2fc9e461d996e7a9fdd045948ca1bcdb29f", - "scheduling.json": "527aefeef11e5872f91f852a8089e682e3bddbb3ee61a03881bcd5ec0d58f7c3" + "scheduling.json": "527aefeef11e5872f91f852a8089e682e3bddbb3ee61a03881bcd5ec0d58f7c3", + "transfer.json": "2c4cbc6cbddc16cb306e07c85378a2fc9e461d996e7a9fdd045948ca1bcdb29f" } } diff --git a/package.json b/package.json index d7186f4..25f7ab1 100644 --- a/package.json +++ b/package.json @@ -12,9 +12,10 @@ "clean": "npm run clean --workspaces && rm -rf dist", "lint": "npm run lint --workspaces", "type-check": "npm run type-check --workspaces", - "test:experiments": "npm run build && node --test examples/discovery/experiments.test.cjs", + "test:experiments": "npm run build && node --test examples/discovery/experiments.test.cjs examples/algorithm-search/algorithm-search.test.cjs", "experiment:transfer": "npm run build --silent && node examples/discovery/transfer.cjs", - "experiment:scheduling": "npm run build --silent && node examples/discovery/scheduling-discovery.cjs" + "experiment:scheduling": "npm run build --silent && node examples/discovery/scheduling-discovery.cjs", + "algorithm:search": "npm run build --silent && node examples/algorithm-search/run.cjs" }, "keywords": [ "functional-programming", diff --git a/packages/fx-core/src/__tests__/algorithm-proposer.test.ts b/packages/fx-core/src/__tests__/algorithm-proposer.test.ts new file mode 100644 index 0000000..dc87348 --- /dev/null +++ b/packages/fx-core/src/__tests__/algorithm-proposer.test.ts @@ -0,0 +1,75 @@ +import { + Algorithm, + AlgorithmTarget, + createModelAlgorithmProposer, + expressionBuilder as e, + searchAlgorithms, +} from '../discovery'; +const identity: Algorithm = { version: 1, body: [{ kind: 'return', value: e.variable('input') }] }; +const target: AlgorithmTarget = { + id: 'identity', + contract: 'Return JSON input unchanged', + baseline: identity, + reference: (x) => x, + training: [{ name: 'train', input: 0 }], + validation: [{ name: 'validation', input: 1 }], + heldOut: [{ name: 'final', input: 2 }], +}; + +describe('Model-proposed bounded algorithms', () => { + it('uses the existing FX provider, records failures, and caps model calls', async () => { + const chat = jest + .fn() + .mockResolvedValueOnce({ + content: 'not json', + usage: { promptTokens: 10, completionTokens: 5, totalTokens: 15 }, + }) + .mockResolvedValue({ + content: JSON.stringify({ name: 'identity', algorithm: identity }), + usage: { promptTokens: 10, completionTokens: 5, totalTokens: 15 }, + }); + const propose = createModelAlgorithmProposer({ chat }, { model: 'test-model', maxCalls: 2 }); + const result = await searchAlgorithms(target, propose, { maxProposals: 5 }); + expect(result.accepted).toBe(true); + expect(result.archive[0]?.name).toBe('proposer-error-0'); + expect(chat).toHaveBeenCalledTimes(2); + expect(propose.usage()).toEqual({ calls: 2, promptTokens: 20, completionTokens: 10 }); + expect(chat.mock.calls[0]![1].model).toBe('test-model'); + const prompt = JSON.parse(chat.mock.calls[0]![0][1].content); + expect(Object.keys(prompt)).toEqual(['target', 'iteration', 'feedback']); + expect(prompt.feedback[0].name).toBe('baseline'); + expect(prompt.feedback[0].heldOut).toBeUndefined(); + expect(prompt.feedback[0].validation).toBeUndefined(); + }); + + it('parses model output only as data and rejects native-source proposals', async () => { + const chat = jest + .fn() + .mockResolvedValue({ + content: JSON.stringify({ + name: 'native', + algorithm: { version: 1, body: [{ kind: 'native', source: 'process.exit()' }] }, + }), + }); + const result = await searchAlgorithms( + target, + createModelAlgorithmProposer({ chat }, { maxCalls: 1 }), + ); + expect(result.accepted).toBe(true); + expect(result.archive[0]?.error).toBeDefined(); + expect(result.selected.name).toBe('baseline'); + }); + + it('bounds response size and feedback, and stops after exhausted calls', async () => { + const chat = jest.fn().mockResolvedValue({ content: 'x'.repeat(100) }); + const propose = createModelAlgorithmProposer( + { chat }, + { maxCalls: 1, maxResponseChars: 20, maxFeedbackChars: 10 }, + ); + const context = { iteration: 0, target: { id: 'test', contract: 'test' }, archive: [] }; + await expect(propose(context)).rejects.toThrow('character'); + expect(await propose(context)).toBeUndefined(); + expect(chat).toHaveBeenCalledTimes(1); + expect(() => createModelAlgorithmProposer({ chat }, { maxCalls: 0 })).toThrow(); + }); +}); diff --git a/packages/fx-core/src/__tests__/algorithm-runtime.test.ts b/packages/fx-core/src/__tests__/algorithm-runtime.test.ts new file mode 100644 index 0000000..42e0bd2 --- /dev/null +++ b/packages/fx-core/src/__tests__/algorithm-runtime.test.ts @@ -0,0 +1,212 @@ +import { + Algorithm, + AlgorithmFailure, + algorithmStep, + Statement, + algorithmId, + executeAlgorithm, + expressionBuilder as e, + parseAlgorithm, +} from '../discovery'; +import { createAgent, createPlan } from '../index'; + +const variable = e.variable; +const literal = e.literal; +const program = (...body: Statement[]): Algorithm => ({ version: 1, body }); + +describe('Bounded algorithm language', () => { + it.each([ + ['sub', [5, 2], 3], + ['neg', [2], -2], + ['floor', [2.9], 2], + ['gte', [2, 2], true], + ['lt', [1, 2], true], + ['or', [false, true], true], + ['not', [true], false], + ['length', [[1, 2]], 2], + ['lower', ['ÅNGSTRÖM'], 'ångström'], + ['contains', ['build finished', 'build'], true], + ['slice', [[1, 2, 3], -2], [2, 3]], + ['reverse', [[1, 2, 3]], [3, 2, 1]], + ['indexOf', [[3, 4], 4], 1], + ] as const)('implements %s with JSON values', (op, args, expected) => { + const source = program({ + kind: 'return', + value: e.call(op, ...args.map((value) => literal(value))), + }); + expect(executeAlgorithm(source, null).output).toEqual(expected); + }); + it('runs a selected algorithm as a step in an existing FX agent', async () => { + type State = { value: number; result?: number }; + const algorithm = program({ + kind: 'return', + value: e.call('add', variable('input'), literal(2)), + }); + const agent = createAgent( + 'algorithm', + createPlan('plan', [ + algorithmStep( + algorithm, + (state) => state.value, + (state, output) => ({ ...state, result: output as number }), + ), + ]), + ); + expect(await agent.start({ value: 3 })).toEqual({ value: 3, result: 5 }); + }); + it('executes loops, branches, early exits and stable output order', () => { + const algorithm = program( + { kind: 'set', name: 'result', value: e.call('array') }, + { + kind: 'for', + name: 'item', + direction: 'reverse', + values: variable('input'), + body: [ + { + kind: 'if', + condition: e.call('gt', variable('item'), literal(1)), + then: [{ kind: 'append', name: 'result', value: variable('item') }, { kind: 'break' }], + otherwise: [], + }, + ], + }, + { kind: 'return', value: variable('result') }, + ); + expect(executeAlgorithm(algorithm, [0, 2, 3]).output).toEqual([3]); + expect(executeAlgorithm(algorithm, [0, 1]).output).toEqual([]); + expect(algorithmId(JSON.parse(JSON.stringify(algorithm)))).toBe(algorithmId(algorithm)); + const parsed = parseAlgorithm(algorithm); + expect(Object.isFrozen(parsed.body)).toBe(true); + expect(parseAlgorithm(parsed)).toBe(parsed); + }); + + it('builds local indexes and never exposes a native Map as output', () => { + const algorithm = program( + { kind: 'set', name: 'index', value: e.call('map') }, + { kind: 'put', name: 'index', key: literal('__proto__'), value: literal(7) }, + { kind: 'return', value: e.call('lookup', variable('index'), literal('__proto__')) }, + ); + const result = executeAlgorithm(algorithm, null); + expect(result.output).toBe(7); + expect(result.usage.mapWrites).toBe(1); + expect(result.usage.mapReads).toBe(1); + expect(() => + executeAlgorithm( + program( + { kind: 'set', name: 'index', value: e.call('map') }, + { kind: 'return', value: variable('index') }, + ), + null, + ), + ).toThrow('cannot escape'); + }); + + it('restores loop-variable bindings and snapshots iteration bounds', () => { + const algorithm = program( + { kind: 'set', name: 'item', value: literal(99) }, + { kind: 'set', name: 'items', value: literal([1]) }, + { + kind: 'for', + name: 'item', + values: variable('items'), + direction: 'forward', + body: [{ kind: 'append', name: 'items', value: literal(2) }], + }, + { kind: 'return', value: variable('item') }, + ); + const result = executeAlgorithm(algorithm, null); + expect(result.output).toBe(99); + expect(result.usage.loopVisits).toBe(1); + }); + + it('enforces instruction and allocation budgets and reports failed work', () => { + const algorithm = program( + { + kind: 'for', + name: 'x', + values: variable('input'), + direction: 'forward', + body: [{ kind: 'set', name: 'last', value: variable('x') }], + }, + { kind: 'return', value: literal(null) }, + ); + try { + executeAlgorithm(algorithm, Array(100).fill(1), { maxSteps: 10 }); + throw new Error('Expected bounded failure'); + } catch (error) { + expect(error).toBeInstanceOf(AlgorithmFailure); + expect((error as AlgorithmFailure).usage.steps).toBeLessThanOrEqual(10); + } + expect(() => + executeAlgorithm(program({ kind: 'return', value: literal('x'.repeat(100)) }), null, { + maxCells: 5, + }), + ).toThrow('allocation'); + }); + + it('short-circuits conditions and rejects invalid values, arities and missing variables', () => { + expect( + executeAlgorithm( + program({ kind: 'return', value: e.call('and', literal(false), variable('missing')) }), + null, + ).output, + ).toBe(false); + expect( + executeAlgorithm( + program({ kind: 'return', value: e.call('eq', literal({ a: 1 }), literal(null)) }), + null, + ).output, + ).toBe(false); + for (const value of [ + variable('missing'), + e.call('add', literal(1)), + e.call('lower', literal(1)), + e.call('add', literal(Number.MAX_VALUE), literal(Number.MAX_VALUE)), + e.call('eq', literal({}), literal({})), + ]) { + expect(() => executeAlgorithm(program({ kind: 'return', value }), null)).toThrow(); + } + expect(() => executeAlgorithm(program({ kind: 'break' }), null)).toThrow('return'); + }); + + it('does not mutate input or resolve prototype properties', () => { + const input = { items: [1] }; + expect(() => + executeAlgorithm( + program( + { kind: 'set', name: 'items', value: e.get(variable('input'), 'items') }, + { kind: 'append', name: 'items', value: literal(2) }, + { kind: 'return', value: variable('items') }, + ), + input, + ), + ).toThrow(); + expect(input.items).toEqual([1]); + expect( + executeAlgorithm( + program({ kind: 'return', value: e.get(variable('input'), 'constructor') }), + {}, + ).output, + ).toBe(null); + }); + + it('rejects native code, recursive syntax, oversized syntax and input reassignment', () => { + const cycle: Record = {}; + cycle.self = cycle; + for (const source of [ + cycle, + { version: 1, body: [{ kind: 'native', source: 'process.exit()' }] }, + { version: 1, body: [{ kind: 'set', name: 'input', value: { kind: 'literal', value: 0 } }] }, + { version: 1, body: [{ kind: 'return', value: { kind: 'literal', value: Infinity } }] }, + ]) { + expect(() => parseAlgorithm(source)).toThrow(); + } + expect(() => + parseAlgorithm({ version: 1, body: Array(130).fill({ kind: 'break' }) }), + ).toThrow(); + expect(() => + parseAlgorithm(program({ kind: 'return', value: literal('x'.repeat(1_000_001)) })), + ).toThrow('text'); + }); +}); diff --git a/packages/fx-core/src/__tests__/algorithm-search.test.ts b/packages/fx-core/src/__tests__/algorithm-search.test.ts new file mode 100644 index 0000000..e106678 --- /dev/null +++ b/packages/fx-core/src/__tests__/algorithm-search.test.ts @@ -0,0 +1,137 @@ +import { Algorithm, AlgorithmTarget, expressionBuilder as e, searchAlgorithms } from '../discovery'; + +const identity: Algorithm = { version: 1, body: [{ kind: 'return', value: e.variable('input') }] }; +const target = (): AlgorithmTarget => ({ + id: 'identity', + contract: 'Return input unchanged', + baseline: identity, + reference: (input) => input, + training: [{ name: 'train', input: 0 }], + validation: [{ name: 'validation', input: 1 }], + heldOut: [{ name: 'test', input: 2 }], +}); + +describe('Algorithm search and independent acceptance', () => { + it('keeps the baseline fallback even when the ranked shortlist overfits', async () => { + const slow: Algorithm = { + version: 1, + body: [ + { + kind: 'for', + name: 'x', + direction: 'forward', + values: e.literal([1, 2, 3, 4]), + body: [{ kind: 'set', name: 'last', value: e.variable('x') }], + }, + ...identity.body, + ], + }; + const result = await searchAlgorithms( + { ...target(), baseline: slow }, + (context) => + context.iteration === 0 + ? { + name: 'overfit', + algorithm: { version: 1, body: [{ kind: 'return', value: e.literal(0) }] }, + } + : undefined, + { shortlist: 1 }, + ); + expect(result.accepted).toBe(true); + expect(result.selected.name).toBe('baseline'); + }); + it('rejects training overfits, retains failures and prevents validation feedback leakage', async () => { + const proposals = [ + { + name: 'overfit', + algorithm: { + version: 1 as const, + body: [{ kind: 'return' as const, value: e.literal(0) }], + }, + }, + { name: 'identity', algorithm: identity }, + ]; + const result = await searchAlgorithms(target(), (context) => { + expect(Object.keys(context.target)).toEqual(['id', 'contract']); + expect(Object.isFrozen(context.archive)).toBe(true); + if (context.archive[0]) expect(Object.isFrozen(context.archive[0].training)).toBe(true); + return proposals[context.iteration]; + }); + expect(result.accepted).toBe(true); + expect(result.selected.heldOut.passed).toBe(1); + expect(result.archive).toHaveLength(2); + expect(result.cost.totalSteps).toBeGreaterThan(0); + expect(new Set(Object.values(result.splitDigests).flat()).size).toBe(3); + }); + + it('never reselects a different winner after observing final-case failure', async () => { + const slow: Algorithm = { + version: 1, + body: [ + { + kind: 'for', + name: 'x', + direction: 'forward', + values: e.literal([1, 2, 3, 4, 5]), + body: [{ kind: 'set', name: 'last', value: e.variable('x') }], + }, + ...identity.body, + ], + }; + const cheat: Algorithm = { + version: 1, + body: [ + { + kind: 'if', + condition: e.call('eq', e.variable('input'), e.literal(2)), + then: [{ kind: 'return', value: e.literal(0) }], + otherwise: identity.body, + }, + ], + }; + const t = { ...target(), baseline: slow }; + const result = await searchAlgorithms(t, (context) => + context.iteration === 0 ? { name: 'cheat', algorithm: cheat } : undefined, + ); + expect(result.selected.name).toBe('cheat'); + expect(result.accepted).toBe(false); + expect(result.selected.heldOut.failures[0]?.name).toBe('test'); + }); + + it('rejects split overlap and an incorrect baseline', async () => { + const t = target(); + await expect(searchAlgorithms({ ...t, heldOut: t.training }, () => undefined)).rejects.toThrow( + 'Duplicate', + ); + await expect(searchAlgorithms({ ...t, reference: () => 99 }, () => undefined)).rejects.toThrow( + 'Baseline', + ); + }); + + it('bounds duplicate proposals and records malformed candidates without running source', async () => { + let calls = 0; + const result = await searchAlgorithms( + target(), + () => { + calls++; + return { name: 'duplicate', algorithm: identity }; + }, + { maxProposals: 3 }, + ); + expect(calls).toBe(3); + expect(result.archive).toHaveLength(1); + const malformed = await searchAlgorithms(target(), (context) => + context.iteration === 0 + ? { + name: 'invalid', + algorithm: { version: 1, body: [{ kind: 'native' }] } as unknown as Algorithm, + } + : undefined, + ); + expect(malformed.archive[0]?.error).toBeDefined(); + expect(malformed.accepted).toBe(true); + await expect( + searchAlgorithms(target(), () => undefined, { maxTotalSteps: 1 }), + ).rejects.toThrow(); + }); +}); diff --git a/packages/fx-core/src/discovery/algorithm-model.ts b/packages/fx-core/src/discovery/algorithm-model.ts new file mode 100644 index 0000000..e2dfb4b --- /dev/null +++ b/packages/fx-core/src/discovery/algorithm-model.ts @@ -0,0 +1,201 @@ +import { createHash } from 'crypto'; +import { z } from 'zod'; +import { JsonValue } from './state'; + +export type Intrinsic = + | 'eq' + | 'gt' + | 'gte' + | 'lt' + | 'add' + | 'sub' + | 'neg' + | 'floor' + | 'and' + | 'or' + | 'not' + | 'length' + | 'lower' + | 'contains' + | 'slice' + | 'reverse' + | 'array' + | 'map' + | 'has' + | 'lookup' + | 'indexOf'; +export type Expression = + | { readonly kind: 'literal'; readonly value: JsonValue } + | { readonly kind: 'variable'; readonly name: string } + | { readonly kind: 'get'; readonly object: Expression; readonly key: Expression } + | { readonly kind: 'call'; readonly op: Intrinsic; readonly args: readonly Expression[] }; +export type Statement = + | { readonly kind: 'set'; readonly name: string; readonly value: Expression } + | { readonly kind: 'append'; readonly name: string; readonly value: Expression } + | { + readonly kind: 'put'; + readonly name: string; + readonly key: Expression; + readonly value: Expression; + } + | { + readonly kind: 'if'; + readonly condition: Expression; + readonly then: readonly Statement[]; + readonly otherwise: readonly Statement[]; + } + | { + readonly kind: 'for'; + readonly name: string; + readonly values: Expression; + readonly direction: 'forward' | 'reverse'; + readonly body: readonly Statement[]; + } + | { readonly kind: 'return'; readonly value: Expression } + | { readonly kind: 'break' }; +export interface Algorithm { + readonly version: 1; + readonly body: readonly Statement[]; +} + +const identifier = z + .string() + .regex(/^[a-zA-Z][a-zA-Z0-9_]*$/) + .max(64); +const writable = identifier.refine((name) => name !== 'input', 'input is read-only'); +const json: z.ZodType = z.lazy(() => + z.union([z.null(), z.boolean(), z.number().finite(), z.string(), z.array(json), z.record(json)]), +); +const expression: z.ZodType = z.lazy(() => + z.union([ + z.object({ kind: z.literal('literal'), value: json }).strict(), + z.object({ kind: z.literal('variable'), name: identifier }).strict(), + z.object({ kind: z.literal('get'), object: expression, key: expression }).strict(), + z + .object({ + kind: z.literal('call'), + op: z.enum([ + 'eq', + 'gt', + 'gte', + 'lt', + 'add', + 'sub', + 'neg', + 'floor', + 'and', + 'or', + 'not', + 'length', + 'lower', + 'contains', + 'slice', + 'reverse', + 'array', + 'map', + 'has', + 'lookup', + 'indexOf', + ]), + args: z.array(expression).max(3), + }) + .strict(), + ]), +); +const statement: z.ZodType = z.lazy(() => + z.union([ + z.object({ kind: z.literal('set'), name: writable, value: expression }).strict(), + z.object({ kind: z.literal('append'), name: writable, value: expression }).strict(), + z + .object({ kind: z.literal('put'), name: writable, key: expression, value: expression }) + .strict(), + z + .object({ + kind: z.literal('if'), + condition: expression, + then: z.array(statement).max(128), + otherwise: z.array(statement).max(128), + }) + .strict(), + z + .object({ + kind: z.literal('for'), + name: writable, + values: expression, + direction: z.enum(['forward', 'reverse']), + body: z.array(statement).max(128), + }) + .strict(), + z.object({ kind: z.literal('return'), value: expression }).strict(), + z.object({ kind: z.literal('break') }).strict(), + ]), +); +const algorithmSchema: z.ZodType = z + .object({ version: z.literal(1), body: z.array(statement).max(128) }) + .strict(); +const validated = new WeakSet(); + +/** Bound and copy syntax before recursive schema parsing; callbacks never enter this language. */ +export function parseAlgorithm(value: unknown): Algorithm { + if (value !== null && typeof value === 'object' && validated.has(value)) + return value as Algorithm; + const active = new Set(); + let nodes = 0; + let textUnits = 0; + const visit = (item: unknown, depth: number): unknown => { + if (++nodes > 12_000 || depth > 48) + throw new Error('Algorithm syntax exceeds size/depth limit'); + if (typeof item === 'string') { + textUnits += item.length; + if (textUnits > 1_000_000) throw new Error('Algorithm text exceeds size limit'); + return item; + } + if (item === null || typeof item === 'boolean') return item; + if (typeof item === 'number' && Number.isFinite(item)) return item; + if (typeof item !== 'object' || item === null) throw new Error('Algorithm must be JSON data'); + if (active.has(item)) throw new Error('Cyclic algorithm syntax'); + active.add(item); + if ( + !Array.isArray(item) && + Object.getPrototypeOf(item) !== Object.prototype && + Object.getPrototypeOf(item) !== null + ) { + throw new Error('Algorithm must use plain JSON objects'); + } + const copied = Array.isArray(item) + ? Array.from(item, (child) => visit(child, depth + 1)) + : Object.fromEntries( + Object.entries(item).map(([key, child]) => [key, visit(child, depth + 1)]), + ); + active.delete(item); + return copied; + }; + const parsed = algorithmSchema.parse(visit(value, 0)); + const freeze = (item: unknown): void => { + if (item !== null && typeof item === 'object') { + Object.values(item).forEach(freeze); + Object.freeze(item); + } + }; + freeze(parsed); + validated.add(parsed); + return parsed; +} + +export function algorithmId(value: Algorithm): string { + return createHash('sha256') + .update(JSON.stringify(parseAlgorithm(value))) + .digest('hex'); +} + +/** Small construction helpers; generated candidates remain ordinary JSON. */ +export const expressionBuilder = { + literal: (value: JsonValue): Expression => ({ kind: 'literal', value }), + variable: (name: string): Expression => ({ kind: 'variable', name }), + get: (object: Expression, key: string | number): Expression => ({ + kind: 'get', + object, + key: { kind: 'literal', value: key }, + }), + call: (op: Intrinsic, ...args: Expression[]): Expression => ({ kind: 'call', op, args }), +}; diff --git a/packages/fx-core/src/discovery/algorithm-proposer.ts b/packages/fx-core/src/discovery/algorithm-proposer.ts new file mode 100644 index 0000000..ca4d593 --- /dev/null +++ b/packages/fx-core/src/discovery/algorithm-proposer.ts @@ -0,0 +1,106 @@ +import { z } from 'zod'; +import type { LLMOptions, LLMProvider } from '../llm'; +import { AlgorithmProposer, AlgorithmTrial } from './algorithm-search'; +import { positiveInteger } from './model'; + +export interface ModelAlgorithmOptions { + readonly model?: string; + readonly maxCalls?: number; + readonly maxResponseChars?: number; + readonly maxFeedbackChars?: number; + readonly maxTokens?: number; +} +export interface ModelAlgorithmUsage { + readonly calls: number; + readonly promptTokens: number; + readonly completionTokens: number; +} + +const grammar = `Return only JSON: {"name":"short-name","algorithm":{"version":1,"body":[statements]}}. +Expressions: +{"kind":"literal","value":JSON}, {"kind":"variable","name":"x"}, +{"kind":"get","object":expression,"key":expression}, +{"kind":"call","op":intrinsic,"args":[expressions]}. +Intrinsics: eq,gt,gte,lt,add,sub,neg,floor,and,or,not,length,lower,contains,slice,reverse,array,map,has,lookup,indexOf. +array/map take 0 args; neg/floor/not/length/lower/reverse take 1; all others take 2. +and/or short-circuit. eq supports scalars and null checks, not object equality. +slice(array,start) uses JavaScript slice semantics. map creates a local string-keyed index. +Statements: +{"kind":"set","name":"x","value":expression}, +{"kind":"append","name":"array-variable","value":expression}, +{"kind":"put","name":"map-variable","key":expression,"value":expression}, +{"kind":"if","condition":expression,"then":[statements],"otherwise":[statements]}, +{"kind":"for","name":"item","values":expression,"direction":"forward","body":[statements]} (direction may also be "reverse"), +{"kind":"break"}, {"kind":"return","value":expression}. +input is a read-only variable holding the target input. No native source, functions, imports or I/O. +Preserve the contract on all inputs; optimize work without hardcoding examples. +Loops have snapshotted bounds; arrays from input are immutable; create local arrays/indexes as needed. +Feedback contains training results only. Validation and final cases are unavailable.`; +const response = z.object({ name: z.string().min(1).max(120), algorithm: z.unknown() }).strict(); + +/** Uses FX's existing provider interface; tests can supply a mock without credentials. */ +export function createModelAlgorithmProposer( + provider: Pick, + options: ModelAlgorithmOptions = {}, +): AlgorithmProposer & { readonly usage: () => ModelAlgorithmUsage } { + const maxCalls = positiveInteger(options.maxCalls ?? 16, 'maxCalls', 10_000); + const maxResponseChars = positiveInteger( + options.maxResponseChars ?? 100_000, + 'maxResponseChars', + 1_000_000, + ); + const maxFeedbackChars = positiveInteger( + options.maxFeedbackChars ?? 24_000, + 'maxFeedbackChars', + 1_000_000, + ); + const maxTokens = positiveInteger(options.maxTokens ?? 4096, 'maxTokens', 100_000); + let calls = 0; + let promptTokens = 0; + let completionTokens = 0; + const propose: AlgorithmProposer = async (context) => { + if (calls >= maxCalls) return undefined; + const best = context.archive + .filter((trial) => trial.training?.passed === trial.training?.total && trial.training) + .sort((a, b) => a.training!.steps - b.training!.steps)[0]; + const feedback: AlgorithmTrial[] = []; + const ordered = [best, ...context.archive.slice(-3)].filter((trial): trial is AlgorithmTrial => + Boolean(trial), + ); + for (const trial of ordered) { + if (feedback.includes(trial)) continue; + if (JSON.stringify([...feedback, trial]).length <= maxFeedbackChars) feedback.push(trial); + } + const request: LLMOptions = { + ...(options.model ? { model: options.model } : {}), + temperature: 0.4, + maxTokens, + }; + calls++; + const result = await provider.chat( + [ + { role: 'system', content: grammar }, + { + role: 'user', + content: JSON.stringify({ + target: context.target, + iteration: context.iteration, + feedback, + }), + }, + ], + request, + ); + promptTokens += result.usage?.promptTokens ?? 0; + completionTokens += result.usage?.completionTokens ?? 0; + if (result.content.length > maxResponseChars) + throw new Error('Model response exceeds character limit'); + const parsed = response.parse(JSON.parse(result.content)); + if (!Object.prototype.hasOwnProperty.call(parsed, 'algorithm')) + throw new Error('Model response is missing algorithm'); + return { name: parsed.name, algorithm: parsed.algorithm }; + }; + return Object.assign(propose, { + usage: () => Object.freeze({ calls, promptTokens, completionTokens }), + }); +} diff --git a/packages/fx-core/src/discovery/algorithm-runtime.ts b/packages/fx-core/src/discovery/algorithm-runtime.ts new file mode 100644 index 0000000..491ce1b --- /dev/null +++ b/packages/fx-core/src/discovery/algorithm-runtime.ts @@ -0,0 +1,211 @@ +import { Algorithm, Expression, Statement, parseAlgorithm } from './algorithm-model'; +import { + AlgorithmValue, + arrayValue, + booleanValue, + invokeIntrinsic, + mapValue, + stringValue, +} from './algorithm-values'; +import { positiveInteger } from './model'; +import { JsonValue, snapshot } from './state'; +import type { BaseContext, Step } from '../types'; + +export interface AlgorithmLimits { + readonly maxSteps?: number; + readonly maxCells?: number; +} +export interface AlgorithmUsage { + readonly steps: number; + readonly cells: number; + readonly loopVisits: number; + readonly mapReads: number; + readonly mapWrites: number; +} +export interface AlgorithmExecution { + readonly output: JsonValue; + readonly usage: AlgorithmUsage; +} +export class AlgorithmFailure extends Error { + constructor( + message: string, + public readonly usage: AlgorithmUsage, + ) { + super(message); + this.name = 'AlgorithmFailure'; + } +} +type Signal = + | { readonly kind: 'return'; readonly value: AlgorithmValue } + | { readonly kind: 'break' }; + +/** No eval, Function, native callback, module access, clock or ambient I/O. */ +export function executeAlgorithm( + source: Algorithm, + input: JsonValue, + limits: AlgorithmLimits = {}, +): AlgorithmExecution { + const program = parseAlgorithm(source); + const maxSteps = positiveInteger(limits.maxSteps ?? 100_000, 'maxSteps', 100_000_000); + const maxCells = positiveInteger(limits.maxCells ?? 100_000, 'maxCells', 10_000_000); + let steps = 0; + let cells = 0; + let loopVisits = 0; + const usage = (): AlgorithmUsage => ({ + steps, + cells, + loopVisits, + mapReads: meter.mapReads, + mapWrites: meter.mapWrites, + }); + const meter = { + mapReads: 0, + mapWrites: 0, + step: (amount = 1): void => { + if (steps + amount > maxSteps) throw new Error('Algorithm step budget exceeded'); + steps += amount; + }, + allocate: (amount = 1): void => { + if (cells + amount > maxCells) throw new Error('Algorithm allocation budget exceeded'); + cells += amount; + }, + }; + const count = (value: unknown, depth = 0): void => { + meter.allocate(); + if (depth > 64) throw new Error('Value nesting limit exceeded'); + if (typeof value === 'string') meter.allocate(value.length); + if (value !== null && typeof value === 'object') + Object.values(value).forEach((child) => count(child, depth + 1)); + }; + const variables = new Map(); + const read = (name: string): AlgorithmValue => { + if (!variables.has(name)) throw new Error(`Unbound variable: ${name}`); + return variables.get(name) ?? null; + }; + const expr = (value: Expression): AlgorithmValue => { + meter.step(); + switch (value.kind) { + case 'literal': + count(value.value); + return JSON.parse(JSON.stringify(value.value)) as JsonValue; + case 'variable': + return read(value.name); + case 'get': { + const object = expr(value.object); + const key = expr(value.key); + if (object === null || typeof object !== 'object' || object instanceof Map) + throw new Error('Property read requires JSON collection'); + if (Array.isArray(object)) { + if (typeof key !== 'number' || !Number.isInteger(key)) + throw new Error('Array key must be an integer'); + return object[key] ?? null; + } + const name = stringValue(key); + return Object.prototype.hasOwnProperty.call(object, name) + ? ((object as Record)[name] ?? null) + : null; + } + case 'call': { + if (value.op === 'and' || value.op === 'or') { + if (value.args.length !== 2) throw new Error(`Wrong arity for ${value.op}`); + const first = booleanValue(expr(value.args[0]!)); + if ((value.op === 'and' && !first) || (value.op === 'or' && first)) return first; + return booleanValue(expr(value.args[1]!)); + } + const result = invokeIntrinsic(value.op, value.args.map(expr), meter); + if (typeof result === 'number' && !Number.isFinite(result)) + throw new Error('Non-finite arithmetic result'); + return result; + } + } + }; + const block = (body: readonly Statement[]): Signal | undefined => { + for (const statement of body) { + meter.step(); + switch (statement.kind) { + case 'set': + if (!variables.has(statement.name)) meter.allocate(); + variables.set(statement.name, expr(statement.value)); + break; + case 'append': { + const list = arrayValue(read(statement.name)); + const value = expr(statement.value); + if (value instanceof Map) throw new Error('Indexes cannot be appended to JSON arrays'); + meter.allocate(); + (list as JsonValue[]).push(value); + break; + } + case 'put': { + const map = mapValue(read(statement.name)); + const key = stringValue(expr(statement.key)); + const value = expr(statement.value); + meter.step(key.length); + if (!map.has(key)) meter.allocate(2); + map.set(key, value); + meter.mapWrites++; + break; + } + case 'if': { + const signal = block( + booleanValue(expr(statement.condition)) ? statement.then : statement.otherwise, + ); + if (signal) return signal; + break; + } + case 'for': { + const values = arrayValue(expr(statement.values)); + // Snapshot iteration bounds, so appending cannot create an unbounded loop. + const length = values.length; + const had = variables.has(statement.name); + const previous = variables.get(statement.name); + if (!had) meter.allocate(); + try { + for (let i = 0; i < length; i++) { + meter.step(); + loopVisits++; + variables.set( + statement.name, + values[statement.direction === 'forward' ? i : length - i - 1] ?? null, + ); + const signal = block(statement.body); + if (signal?.kind === 'return') return signal; + if (signal?.kind === 'break') break; + } + } finally { + if (had) variables.set(statement.name, previous ?? null); + else variables.delete(statement.name); + } + break; + } + case 'return': + return { kind: 'return', value: expr(statement.value) }; + case 'break': + return { kind: 'break' }; + } + } + return undefined; + }; + try { + count(input); + variables.set('input', snapshot({ value: input }).value!); + const result = block(program.body); + if (!result || result.kind !== 'return') throw new Error('Algorithm must return a JSON value'); + if (result.value instanceof Map) throw new Error('Local indexes cannot escape as output'); + count(result.value); + return { output: snapshot({ value: result.value }).value!, usage: usage() }; + } catch (error) { + throw new AlgorithmFailure(error instanceof Error ? error.message : String(error), usage()); + } +} + +/** Explicit integration into an existing FX agent; adapters stay trusted host code. */ +export function algorithmStep( + source: Algorithm, + selectInput: (state: S) => JsonValue, + mergeOutput: (state: S, output: JsonValue) => S, + limits: AlgorithmLimits = {}, +): Step { + const algorithm = parseAlgorithm(source); + return async (state) => + mergeOutput(state, executeAlgorithm(algorithm, selectInput(state), limits).output); +} diff --git a/packages/fx-core/src/discovery/algorithm-search.ts b/packages/fx-core/src/discovery/algorithm-search.ts new file mode 100644 index 0000000..1a4cc31 --- /dev/null +++ b/packages/fx-core/src/discovery/algorithm-search.ts @@ -0,0 +1,232 @@ +import { Algorithm, algorithmId, parseAlgorithm } from './algorithm-model'; +import { AlgorithmFailure, AlgorithmLimits, executeAlgorithm } from './algorithm-runtime'; +import { positiveInteger } from './model'; +import { JsonValue, snapshot } from './state'; +import { createHash } from 'crypto'; + +export interface AlgorithmCase { + readonly name: string; + readonly input: JsonValue; +} +export interface AlgorithmTarget { + readonly id: string; + readonly contract: string; + readonly baseline: Algorithm; + readonly reference: (input: JsonValue) => JsonValue | Promise; + readonly training: readonly AlgorithmCase[]; + readonly validation: readonly AlgorithmCase[]; + readonly heldOut: readonly AlgorithmCase[]; +} +export interface CaseAssessment { + readonly passed: number; + readonly total: number; + readonly steps: number; + readonly loopVisits: number; + readonly maxCells: number; + readonly failures: readonly { readonly name: string; readonly error: string }[]; +} +export interface AlgorithmTrial { + readonly id: string; + readonly name: string; + readonly algorithm?: Algorithm; + readonly training?: CaseAssessment; + readonly error?: string; +} +export interface AlgorithmProposal { + readonly name: string; + readonly algorithm: unknown; +} +export type AlgorithmProposer = (context: { + readonly iteration: number; + readonly target: { readonly id: string; readonly contract: string }; + readonly archive: readonly AlgorithmTrial[]; +}) => AlgorithmProposal | undefined | Promise; +export interface AlgorithmSearchOptions extends AlgorithmLimits { + readonly maxProposals?: number; + readonly maxTotalSteps?: number; + readonly shortlist?: number; +} +interface PreparedCase extends AlgorithmCase { + readonly expected: JsonValue; +} +const canonical = (value: JsonValue): string => JSON.stringify(snapshot({ value }).value); +const digest = (value: JsonValue): string => + createHash('sha256').update(canonical(value)).digest('hex'); + +/** Offline algorithm search: oracle, search feedback, selection and final test are separate. */ +export async function searchAlgorithms( + target: AlgorithmTarget, + propose: AlgorithmProposer, + options: AlgorithmSearchOptions = {}, +) { + const maxProposals = positiveInteger(options.maxProposals ?? 64, 'maxProposals', 10_000); + const maxTotalSteps = positiveInteger( + options.maxTotalSteps ?? 20_000_000, + 'maxTotalSteps', + 1_000_000_000, + ); + const maxSteps = positiveInteger(options.maxSteps ?? 200_000, 'maxSteps', 100_000_000); + const maxCells = positiveInteger(options.maxCells ?? 200_000, 'maxCells', 10_000_000); + const shortlist = positiveInteger(options.shortlist ?? 5, 'shortlist', 100); + const seenInputs = new Set(); + const splitDigests: Record = {}; + const prepare = async ( + cases: readonly AlgorithmCase[], + split: string, + ): Promise => { + if (!cases.length || cases.length > 128) throw new Error('Each split requires 1..128 cases'); + splitDigests[split] = []; + const prepared: PreparedCase[] = []; + for (const item of cases) { + const input = snapshot({ value: item.input }).value!; + const hash = digest(input); + if (seenInputs.has(hash)) + throw new Error('Duplicate inputs across algorithm experiment cases'); + seenInputs.add(hash); + splitDigests[split]!.push(hash); + const expected = snapshot({ value: await target.reference(input) }).value!; + prepared.push({ name: item.name, input, expected }); + } + return prepared; + }; + const training = await prepare(target.training, 'training'); + const validation = await prepare(target.validation, 'validation'); + const heldOut = await prepare(target.heldOut, 'heldOut'); + let totalSteps = 0; + const assess = (algorithm: Algorithm, cases: readonly PreparedCase[]): CaseAssessment => { + let passed = 0; + let steps = 0; + let loopVisits = 0; + let cells = 0; + const failures: { name: string; error: string }[] = []; + for (const item of cases) { + const remaining = maxTotalSteps - totalSteps; + if (remaining <= 0) throw new Error('Algorithm search total step budget exhausted'); + try { + const result = executeAlgorithm(algorithm, item.input, { + maxSteps: Math.min(maxSteps, remaining), + maxCells, + }); + steps += result.usage.steps; + totalSteps += result.usage.steps; + loopVisits += result.usage.loopVisits; + cells = Math.max(cells, result.usage.cells); + if (canonical(result.output) === canonical(item.expected)) passed++; + else failures.push({ name: item.name, error: 'Output differs from independent reference' }); + } catch (error) { + if (error instanceof AlgorithmFailure) { + steps += error.usage.steps; + totalSteps += error.usage.steps; + loopVisits += error.usage.loopVisits; + cells = Math.max(cells, error.usage.cells); + } + failures.push({ + name: item.name, + error: error instanceof Error ? error.message : String(error), + }); + } + } + return Object.freeze({ + passed, + total: cases.length, + steps, + loopVisits, + maxCells: cells, + failures: Object.freeze(failures.map((item) => Object.freeze(item))), + }); + }; + const baseline = parseAlgorithm(target.baseline); + const baselineTraining = assess(baseline, training); + if (baselineTraining.passed !== baselineTraining.total) + throw new Error('Baseline violates the target reference'); + const baselineTrial: AlgorithmTrial = Object.freeze({ + id: algorithmId(baseline), + name: 'baseline', + algorithm: baseline, + training: baselineTraining, + }); + const archive: AlgorithmTrial[] = []; + const seen = new Set(); + for (let iteration = 0; iteration < maxProposals; iteration++) { + let proposal: AlgorithmProposal | undefined; + try { + proposal = await propose( + Object.freeze({ + iteration, + target: Object.freeze({ id: target.id, contract: target.contract }), + archive: Object.freeze([baselineTrial, ...archive]), + }), + ); + } catch (error) { + archive.push( + Object.freeze({ + id: '', + name: `proposer-error-${iteration}`, + error: error instanceof Error ? error.message : String(error), + }), + ); + continue; + } + if (!proposal) break; + let trial: AlgorithmTrial; + try { + const algorithm = parseAlgorithm(proposal.algorithm); + const id = algorithmId(algorithm); + if (seen.has(id)) continue; + seen.add(id); + trial = { id, name: proposal.name, algorithm, training: assess(algorithm, training) }; + } catch (error) { + trial = { + id: '', + name: proposal.name, + error: error instanceof Error ? error.message : String(error), + }; + } + archive.push(Object.freeze(trial)); + } + // The baseline is always a selectable fallback, never an implicit passing candidate. + const candidates = [baselineTrial, ...archive] + .filter((trial) => trial.algorithm && trial.training?.passed === training.length) + .sort((a, b) => a.training!.steps - b.training!.steps || a.id.localeCompare(b.id)); + const distinct = [...new Map(candidates.map((trial) => [trial.id, trial])).values()].slice( + 0, + shortlist, + ); + if (!distinct.some((trial) => trial.id === baselineTrial.id)) distinct.push(baselineTrial); + const validated = distinct + .map((trial) => ({ trial, validation: assess(trial.algorithm!, validation) })) + .filter((item) => item.validation.passed === validation.length) + .sort( + (a, b) => a.validation.steps - b.validation.steps || a.trial.id.localeCompare(b.trial.id), + ); + const selected = validated[0]; + if (!selected) throw new Error('No candidate passes independent validation'); + const baselineTest = assess(baseline, heldOut); + if (baselineTest.passed !== heldOut.length) + throw new Error('Baseline violates held-out reference'); + // Final cases are consulted exactly once after selection; failure never triggers reselection. + const finalTest = assess(selected.trial.algorithm!, heldOut); + return { + version: 1 as const, + target: { id: target.id, contract: target.contract }, + accepted: finalTest.passed === finalTest.total, + selected: { + ...selected.trial, + algorithm: selected.trial.algorithm!, + training: selected.trial.training!, + validation: selected.validation, + heldOut: finalTest, + }, + baseline: { algorithm: baseline, training: baselineTraining, heldOut: baselineTest }, + archive: Object.freeze(archive), + splitDigests, + cost: { + totalSteps, + maxTotalSteps, + maxProposals, + maxStepsPerCase: maxSteps, + maxCellsPerCase: maxCells, + }, + note: 'Steps are deterministic interpreter work units, not native time. No production activation.', + }; +} diff --git a/packages/fx-core/src/discovery/algorithm-values.ts b/packages/fx-core/src/discovery/algorithm-values.ts new file mode 100644 index 0000000..195b245 --- /dev/null +++ b/packages/fx-core/src/discovery/algorithm-values.ts @@ -0,0 +1,122 @@ +import { JsonValue } from './state'; + +export type AlgorithmValue = JsonValue | Map; +export interface AlgorithmMeter { + step(amount?: number): void; + allocate(amount?: number): void; + mapReads: number; + mapWrites: number; +} +export function numberValue(value: AlgorithmValue): number { + if (typeof value !== 'number' || !Number.isFinite(value)) + throw new Error('Expected finite number'); + return value; +} +export function stringValue(value: AlgorithmValue): string { + if (typeof value !== 'string') throw new Error('Expected string'); + return value; +} +export function booleanValue(value: AlgorithmValue): boolean { + if (typeof value !== 'boolean') throw new Error('Expected boolean'); + return value; +} +export function arrayValue(value: AlgorithmValue): readonly JsonValue[] { + if (!Array.isArray(value)) throw new Error('Expected array'); + return value; +} +export function mapValue(value: AlgorithmValue): Map { + if (!(value instanceof Map)) throw new Error('Expected local index'); + return value; +} + +export function invokeIntrinsic( + op: string, + values: readonly AlgorithmValue[], + meter: AlgorithmMeter, +): AlgorithmValue { + const arity = ['array', 'map'].includes(op) + ? 0 + : ['neg', 'floor', 'not', 'length', 'lower', 'reverse'].includes(op) + ? 1 + : 2; + if (values.length !== arity) throw new Error(`Wrong arity for ${op}`); + const a = values[0] ?? null; + const b = values[1] ?? null; + switch (op) { + case 'array': + meter.allocate(); + return []; + case 'map': + meter.allocate(); + return new Map(); + case 'eq': + if (a === null || b === null) return a === b; + if ((a !== null && typeof a === 'object') || (b !== null && typeof b === 'object')) + throw new Error('Equality requires scalar values'); + if (typeof a === 'string' && typeof b === 'string') meter.step(Math.max(a.length, b.length)); + return a === b; + case 'gt': + return numberValue(a) > numberValue(b); + case 'gte': + return numberValue(a) >= numberValue(b); + case 'lt': + return numberValue(a) < numberValue(b); + case 'add': + return numberValue(a) + numberValue(b); + case 'sub': + return numberValue(a) - numberValue(b); + case 'neg': + return -numberValue(a); + case 'floor': + return Math.floor(numberValue(a)); + case 'not': + return !booleanValue(a); + case 'length': + if (typeof a !== 'string' && !Array.isArray(a)) + throw new Error('Length requires string or array'); + return a.length; + case 'lower': { + const text = stringValue(a); + meter.step(text.length); + meter.allocate(text.length); + return text.toLowerCase(); + } + case 'contains': { + const text = stringValue(a); + const needle = stringValue(b); + meter.step(text.length + needle.length); + return text.includes(needle); + } + case 'slice': { + const array = arrayValue(a); + const start = numberValue(b); + const offset = Math.trunc(start); + const begin = + offset < 0 ? Math.max(array.length + offset, 0) : Math.min(offset, array.length); + meter.step(array.length - begin); + meter.allocate(array.length - begin); + return array.slice(start); + } + case 'reverse': { + const array = arrayValue(a); + meter.step(array.length); + meter.allocate(array.length); + return [...array].reverse(); + } + case 'has': + meter.mapReads++; + meter.step(stringValue(b).length); + return mapValue(a).has(stringValue(b)); + case 'lookup': + meter.mapReads++; + meter.step(stringValue(b).length); + return mapValue(a).get(stringValue(b)) ?? null; + case 'indexOf': { + const array = arrayValue(a); + meter.step(array.length); + return array.indexOf(b as JsonValue); + } + default: + throw new Error(`Unsupported intrinsic: ${op}`); + } +} diff --git a/packages/fx-core/src/discovery/index.ts b/packages/fx-core/src/discovery/index.ts index a7e819e..157c4f2 100644 --- a/packages/fx-core/src/discovery/index.ts +++ b/packages/fx-core/src/discovery/index.ts @@ -4,3 +4,7 @@ export * from './runtime'; export * from './abstractions'; export * from './search'; export * from './optimization'; +export * from './algorithm-model'; +export * from './algorithm-runtime'; +export * from './algorithm-search'; +export * from './algorithm-proposer';