diff --git a/.github/workflows/adr-number-check.yml b/.github/workflows/adr-number-check.yml new file mode 100644 index 000000000..bc3380096 --- /dev/null +++ b/.github/workflows/adr-number-check.yml @@ -0,0 +1,116 @@ +name: ADR number check + +# Enforces ADR-014's amendment: an ADR number is claimed at PR-open time and must be +# unique against `next` AND against every other open PR. Without this, each author picks +# max+1 in isolation and collides with branches they cannot see — which is how the +# register ended up with three different decisions all titled ADR-027. + +on: + pull_request: + # `labeled`/`unlabeled` so adding or removing `adr-reconciliation` re-evaluates + # without needing an empty commit. + types: [opened, synchronize, reopened, labeled, unlabeled] + paths: + - 'docs/architecture-decisions/**' + # Re-check open PRs when the base moves, so the loser of a simultaneous claim goes red + # before it can merge rather than after. + push: + branches: [next] + paths: + - 'docs/architecture-decisions/**' + +permissions: + contents: read + pull-requests: read + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Check ADR numbers are unique + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + REPO: ${{ github.repository }} + shell: bash + run: | + set -euo pipefail + ADR_DIR="docs/architecture-decisions" + + nums_from() { grep -oE '^[0-9]{3}' <<<"$1" || true; } + + # Numbers already on next (permanent — never recycled). + git fetch -q origin next + MERGED=$(git ls-tree --name-only "origin/next" -- "$ADR_DIR/" \ + | xargs -r -n1 basename | grep -oE '^[0-9]{3}' | sort -u || true) + + # Numbers this PR introduces (present here, absent on next). + MINE=$(comm -23 \ + <(ls "$ADR_DIR" | grep -oE '^[0-9]{3}' | sort -u) \ + <(printf '%s\n' "$MERGED" | sort -u)) + + if [ -z "${MINE//[[:space:]]/}" ]; then + echo "No new ADR numbers introduced."; exit 0 + fi + echo "This PR claims: $(echo $MINE | tr '\n' ' ')" + + fail=0 + + # 1. Collision with an already-merged number. + for n in $MINE; do + if grep -qx "$n" <<<"$MERGED"; then + echo "::error::ADR-$n already exists on next. Numbers on next are permanent and are never recycled." + fail=1 + fi + done + + # 2. Collision with another open PR's claim. + # + # Skipped for a reconciliation PR: one that exists to re-assign contested + # numbers necessarily claims numbers the PRs it is reconciling still hold, + # so this check would always fail it. The merged-number and heading checks + # still apply. Opt out with the `adr-reconciliation` label. + RECONCILE="" + if [ -n "${PR_NUMBER:-}" ]; then + RECONCILE=$(gh pr view "$PR_NUMBER" --repo "$REPO" --json labels \ + --jq '.labels[].name' 2>/dev/null | grep -Fx 'adr-reconciliation' || true) + [ -n "$RECONCILE" ] && echo "Label 'adr-reconciliation' present - skipping the cross-PR check." + fi + if [ -n "${PR_NUMBER:-}" ] && [ -z "$RECONCILE" ]; then + for other in $(gh pr list --repo "$REPO" --state open --limit 200 \ + --json number --jq '.[].number'); do + [ "$other" = "$PR_NUMBER" ] && continue + theirs=$(gh pr view "$other" --repo "$REPO" --json files \ + --jq '.files[].path' 2>/dev/null \ + | grep "^$ADR_DIR/[0-9][0-9][0-9]-" | xargs -r -n1 basename \ + | grep -oE '^[0-9]{3}' | sort -u || true) + [ -z "$theirs" ] && continue + for n in $MINE; do + if grep -qx "$n" <<<"$theirs"; then + echo "::error::ADR-$n is also claimed by open PR #$other." + fail=1 + fi + done + done + fi + + # 3. Filename number must match the heading (ADR-014 rule 3). + for f in "$ADR_DIR"/[0-9][0-9][0-9]-*.md; do + n=$(basename "$f" | grep -oE '^[0-9]{3}') + head -1 "$f" | grep -qE "^#[[:space:]]*ADR-$n:[[:space:]]*\S" || { + echo "::error file=$f::First line must be '# ADR-$n: Title' (ADR-014 rule 3)." + fail=1 + } + done + + if [ "$fail" = "1" ]; then + taken=$(printf '%s\n%s\n' "$MERGED" "$MINE" | sort -u) + next=$(seq -f '%03g' 1 999 | grep -vxF -f <(echo "$taken") | head -1) + echo "::notice::Next free ADR number is $next." + exit 1 + fi + echo "ADR numbers OK." diff --git a/docs/architecture-decisions/004-migration-tool-design.md b/docs/architecture-decisions/004-migration-tool-design.md index 3a461d0d8..954def981 100644 --- a/docs/architecture-decisions/004-migration-tool-design.md +++ b/docs/architecture-decisions/004-migration-tool-design.md @@ -1,7 +1,8 @@ # ADR-004: Project Structure Migration Tool -## Status -Proposed +**Status**: Proposed +**Date**: 2025-01-25 +**Deciders**: TBD ## Context diff --git a/docs/architecture-decisions/005-admin-script-runner.md b/docs/architecture-decisions/005-admin-script-runner.md index 3abc916c8..f8d040913 100644 --- a/docs/architecture-decisions/005-admin-script-runner.md +++ b/docs/architecture-decisions/005-admin-script-runner.md @@ -1,7 +1,8 @@ -# Architecture Decision Record: Admin Script Runner Service +# ADR-005: Admin Script Runner Service -## Status -Accepted (Implemented) +**Status**: Accepted (Implemented) +**Date**: 2025-12-10 +**Deciders**: TBD ## Context diff --git a/docs/architecture-decisions/014-consolidate-adr-register.md b/docs/architecture-decisions/014-consolidate-adr-register.md index 18550d798..9e0a5f9a8 100644 --- a/docs/architecture-decisions/014-consolidate-adr-register.md +++ b/docs/architecture-decisions/014-consolidate-adr-register.md @@ -83,6 +83,45 @@ Execution steps: - Numbers get assigned in a logical block rather than strictly by original authoring date; original `Date` fields are preserved in each file. +## Amendment (2026-08-31): claiming a number + +The original decision created one register but no way to **claim** a number. Every author picks +`max + 1` at authoring time, which is only correct if no one else is doing the same — and in practice +several were. At the time of this amendment the register had eight collisions across six open PRs, four +of them against numbers already merged to `next`: three different decisions were all titled `ADR-027`, +and a fourth reused `ADR-010`. Nothing catches this, because each PR is internally consistent and only +collides with branches its author cannot see. + +**Numbers are claimed at PR-open time, and uniqueness is enforced by CI — not assigned at merge.** + +Authoring does not change. Pick the next free number, name the file `NNN-kebab-title.md`, write the +`# ADR-NNN:` heading and cross-link by number as before. A workflow on any PR touching +`docs/architecture-decisions/**` computes the taken set as *numbers on `next`* ∪ *numbers claimed by +every other open PR*, and fails with the conflict and the next free number when they overlap. It re-runs +on every push and when `next` moves, so if two PRs open simultaneously and both pass, the second to +rebase goes red before it can merge. + +### Why not assign the number at merge time + +Deferring assignment sounds tidier and is worse in practice: + +- The file is named wrong for the entire review. Reviewers read `draft-foo.md` / `ADR-XXX` and cannot + cite it in review comments or link it from other PRs. +- Cross-ADR links cannot be written until the number exists, so a cluster of related ADRs (this register + has three such clusters) has to be link-patched after the fact. +- Someone has to perform the rename plus link rewrite at every merge — which is exactly the manual + reconciliation this amendment exists to stop, just relocated and made recurring. + +Enforcing at PR-open keeps the number stable from first commit and moves detection from *after merge* +to *before review*. The cost is a rename when CI reports a conflict, which is cheap because it is a +document, and rarer because the check names the next free number. + +### Numbers are not recycled + +A number that has appeared on `next` is permanent, even if that ADR is later superseded or deprecated. +A number that has only ever existed in an unmerged branch is not yet claimed and may be reassigned — that +is what let this reconciliation close the 028-030 gap rather than leave holes in the sequence. + ## Alternatives Considered - **Keep both, add an index that spans them.** Rejected: still two structures/locations to learn; the drift problem remains. diff --git a/docs/architecture-decisions/028-multi-provider-support.md b/docs/architecture-decisions/028-multi-provider-support.md new file mode 100644 index 000000000..a242b85f8 --- /dev/null +++ b/docs/architecture-decisions/028-multi-provider-support.md @@ -0,0 +1,264 @@ +# ADR-028: Multi-Provider Support + +**Status**: Accepted +**Date**: 2026-03-02 +**Deciders**: Sean Matthews, Frigg Team + +## Context + +Frigg was originally built as an AWS-first framework: Lambda for compute, SQS for queues, EventBridge Scheduler for one-time jobs, KMS for encryption, and Serverless Framework for deployment. Every CLI command (`deploy`, `build`, `start`, `doctor`, `repair`, `generate-iam`) assumed AWS. + +Customers and community members want to deploy Frigg integrations to platforms beyond AWS — starting with Netlify, with potential for Vercel, GCP Cloud Run, and others. Adding a second provider forces the right abstraction boundaries; supporting N providers is then incremental. + +### Requirements + +1. Existing AWS deployments must work with minimal migration (see ADR-029 for breaking changes and migration guide). +2. A single `provider` field in the app definition switches the entire toolchain. +3. Provider-specific code lives in separate, installable packages. +4. Core framework code must not import provider-specific modules directly. +5. The CLI, runtime handlers, queues, scheduling, and encryption must all dispatch through the same provider abstraction. + +## Decision + +Introduce a **provider plugin system** with a standard interface. Each provider is an npm package (`@friggframework/provider-{name}`) that exports adapters for every platform-dependent concern: deployment, configuration generation, queue dispatch, job scheduling, secret loading, and handler creation. + +### Provider Plugin Interface + +A provider package exports an object conforming to this shape: + +```javascript +module.exports = { + // ─── Identity ──────────────────────────────────── + name: 'netlify', // Provider identifier + + // ─── Runtime adapters ──────────────────────────── + createHandler, // Express handler factory for the platform + createAppHandler, // App-level handler factory + QueueProvider, // Class: send(), batchSend(), parseEvent() + SchedulerAdapter, // Class: scheduleOneTime(), deleteSchedule(), getScheduleStatus() + ScheduledJobRepository, // Class: save(), delete(), findByName(), findDue() + loadSecrets, // async fn: load secrets from platform vault + invokeFunctionAdapter, // { invoke(name, payload) } + + // ─── Encryption & WebSockets (optional) ────────── + CryptorAdapter: null, // null = reuse core Cryptor + WebSocketAdapter: null, // null = not supported + + // ─── Build-time ────────────────────────────────── + generateConfig, // fn(appDef) → config file content (e.g. netlify.toml) + generateEnvTemplate, // fn(appDef) → { VAR_NAME: 'description' } + getFunctionEntryPoints, // fn(appDef) → { 'api.js': '...code...' } + + // ─── Deployment ────────────────────────────────── + deploy, // async fn(appDef, options) + preflightCheck, // async fn(appDef) → { ready, missing } + validate, // fn(appDef) → { valid, errors[], warnings[] } + teardown, // async fn() — cleanup + + // ─── Detection & metadata ──────────────────────── + detect, // fn() → boolean (is this env the provider?) + recommendedDatabases: ['postgresql'], + providedEnvVars: ['NETLIFY', 'URL'], +}; +``` + +### Resolution Chain + +The `resolveProvider(appDefinition, options)` function in `@friggframework/core` resolves a provider name to its package: + +``` +1. Explicit providerName argument (programmatic override) +2. appDefinition.provider (app definition file) +3. FRIGG_PROVIDER env var (CI/CD override) +4. Default: 'aws' +``` + +Name → package: `'netlify'` → `require('@friggframework/provider-netlify')` + +AWS is the default, so existing apps that don't set `provider` continue to work unchanged. + +### CLI Command Dispatch + +Each CLI command follows the same pattern: + +``` +1. loadProviderForCli() — reads appDefinition, resolves provider +2. if provider is non-AWS — delegate to provider.{method}() +3. if provider is AWS/null — fall through to existing serverless behavior +``` + +| Command | AWS path | Non-AWS path | +|---------|----------|-------------| +| `deploy` | `osls deploy` | `provider.validate()` → `provider.deploy()` | +| `build` | `osls package` | `provider.validate()` → `provider.generateConfig()` → `provider.getFunctionEntryPoints()` | +| `start` | `serverless-offline` | Platform CLI (e.g. `netlify dev`) | +| `doctor` | CloudFormation health check | Rejected with message (AWS-only) | +| `repair` | CloudFormation import | Rejected with message (AWS-only) | +| `generate-iam` | IAM policy generation | Rejected with message (AWS-only) | + +Commands that are inherently AWS-specific (`doctor`, `repair`, `generate-iam`) guard with an early exit for non-AWS providers rather than implementing no-op stubs. + +### Runtime Adapter Dispatch + +At runtime, integrations use platform-agnostic interfaces. Factories create the right adapter based on the provider: + +**Scheduler**: `SchedulerServiceFactory` creates either: +- `EventBridgeSchedulerAdapter` — AWS push model (precise timing, cloud-native) +- `NetlifySchedulerAdapter` — poll-and-dispatch model (cron queries database for due jobs) +- `MockSchedulerAdapter` — in-memory for dev/test + +**Queue**: `QueueProvider` base class with platform-specific subclasses: +- AWS SQS provider — SQS send/batch/parse +- `NetlifyBackgroundProvider` — HTTP POST to background functions + +**Encryption**: Core `Cryptor` shared across providers. Providers can optionally supply a `CryptorAdapter` override, but `null` reuses the default (KMS or AES based on env vars). + +### Scheduling: Push vs Poll-and-Dispatch + +This is the most significant architectural divergence between providers: + +``` +AWS (Push Model) Netlify (Poll-and-Dispatch) +───────────────── ─────────────────────────── +scheduleOneTime() scheduleOneTime() + │ │ + ▼ ▼ +EventBridge Scheduler Database (state: PENDING) +creates a one-time rule │ + │ │ ← cron fires every N min + │ at(2025-06-01T12:00) │ + │ ▼ + ▼ processDueSchedules() +SQS receives message │ mark PROCESSING +at exact time │ send to queue + │ delete on success + │ mark FAILED on error +``` + +**Trade-offs**: +- Push is more precise (sub-second), poll has up to cron-interval delay +- Push requires cloud-specific APIs, poll works on any platform with a database +- Poll needs the PROCESSING state guard to prevent duplicate dispatch (race condition between concurrent cron invocations) + +### State Machine for Netlify Schedules + +``` +PENDING ──→ PROCESSING ──→ (deleted) + │ + └──→ FAILED +``` + +`findDue()` only returns `PENDING` records, so `PROCESSING` acts as a distributed lock — if a second cron invocation fires while the first is dispatching, it won't re-pick the same schedule. + +## Package Structure + +``` +packages/ +├── core/ +│ ├── providers/ +│ │ └── resolve-provider.js # Resolution chain +│ ├── infrastructure/scheduler/ +│ │ ├── scheduler-service-interface.js +│ │ ├── scheduler-service-factory.js +│ │ ├── eventbridge-scheduler-adapter.js +│ │ ├── netlify-scheduler-adapter.js +│ │ └── mock-scheduler-adapter.js +│ └── queues/ +│ ├── queue-provider.js # Base class +│ └── providers/ +│ └── netlify-background-provider.js +├── providers/ +│ ├── aws/ # @friggframework/provider-aws +│ │ ├── queues/sqs-queue-client.js +│ │ ├── encryption/kms-encryption-key-provider.js +│ │ ├── websocket/api-gateway-message-sender.js +│ │ ├── lambda/lambda-invoker.js +│ │ └── storage/migration-status-repository-s3.js +│ └── netlify/ # @friggframework/provider-netlify +│ ├── index.js # Plugin interface export +│ └── lib/ +│ ├── generate-netlify-config.js +│ ├── get-function-entry-points.js +│ ├── netlify-background-provider.js +│ ├── scheduled-job-repository.js +│ └── ... +├── devtools/ +│ └── frigg-cli/ +│ ├── utils/provider-helper.js # CLI provider loading +│ ├── deploy-command/ # Provider dispatch +│ ├── build-command/ # Provider dispatch +│ └── start-command/ # Provider dispatch +``` + +## How to Add a New Provider + +1. Create `packages/providers/{name}/` implementing the plugin interface. +2. Add the name to `KNOWN_PROVIDERS` in `resolve-provider.js`. +3. Run the existing provider dispatch tests — they should pass without changes. +4. Add provider-specific tests in the new package. +5. Publish as `@friggframework/provider-{name}`. + +The provider dispatch tests (`provider-dispatch.test.js`) verify that: +- AWS falls through to existing serverless behavior +- Non-AWS providers delegate to the plugin methods +- AWS-only commands reject non-AWS providers cleanly + +These tests are provider-agnostic, so they validate the wiring for any new provider. + +## Alternatives Considered + +### Alternative 1: Conditional imports in core + +Scatter `if (provider === 'netlify')` checks throughout framework code. + +**Rejected**: Violates open-closed principle. Every new provider requires modifying core code. Leads to import-time side effects and difficult-to-test conditionals. + +### Alternative 2: Abstract factory for everything + +Create factories for every concern (handler, queue, scheduler, encryption, deployment) and compose them in a single configuration object. + +**Partially adopted**: We use factories for scheduler and queue, but the provider plugin itself acts as the top-level factory. This avoids factory-of-factories complexity while keeping each concern independently testable. + +### Alternative 3: Docker-based universal deployment + +Containerize the app and deploy the same Docker image everywhere. + +**Rejected for now**: Loses platform-native advantages (Netlify Edge Functions, Lambda cold-start optimizations, platform-specific DX). May revisit as a "universal provider" in the future. + +### Alternative 4: Provider stubs for AWS-only commands + +Instead of rejecting `doctor`/`repair`/`generate-iam` for non-AWS providers, implement no-op or generic versions. + +**Rejected**: These commands are deeply tied to CloudFormation concepts. No-op stubs would be misleading. Clear rejection messages are more honest and push providers to implement equivalent commands when platform support exists. + +## Risks and Mitigations + +| Risk | Mitigation | +|------|-----------| +| Provider interface grows unwieldy | Keep interface minimal; optional fields return `null` to skip features | +| Breaking changes to plugin interface | Semantic versioning; interface is tested via `provider-plugin-interface.test.js` | +| Poll-and-dispatch causes duplicate jobs | PROCESSING state guard prevents re-dispatch; tested in `netlify-scheduler-adapter.test.js` | +| Provider packages not installed | `resolveProvider` throws a clear error with `npm install` instructions | +| AWS assumptions leak into core | Provider dispatch tests catch AWS-specific calls that should be guarded | + +## Test Coverage + +| Test Suite | Location | Tests | +|-----------|----------|-------| +| Core resolver | `packages/core/providers/resolve-provider.test.js` | 11 | +| CLI provider helper | `frigg-cli/utils/__tests__/provider-helper.test.js` | 4 | +| Provider dispatch (CLI commands) | `frigg-cli/__tests__/unit/commands/provider-dispatch.test.js` | 9 | +| Netlify scheduler adapter | `core/infrastructure/scheduler/netlify-scheduler-adapter.test.js` | 23 | +| Scheduled job repository | `packages/providers/netlify/__tests__/scheduled-job-repository.test.js` | 9 | +| Netlify plugin interface | `packages/providers/netlify/__tests__/provider-plugin-interface.test.js` | — | +| Netlify config generation | `packages/providers/netlify/__tests__/generate-netlify-config.test.js` | — | +| Netlify deploy | `packages/providers/netlify/__tests__/deploy.test.js` | — | +| Netlify validate | `packages/providers/netlify/__tests__/validate.test.js` | — | + +## References + +- Plugin interface: `packages/providers/netlify/index.js` +- Resolution chain: `packages/core/providers/resolve-provider.js` +- Scheduler interface: `packages/core/infrastructure/scheduler/scheduler-service-interface.js` +- Queue provider base: `packages/core/queues/queue-provider.js` +- CLI dispatch: `packages/devtools/frigg-cli/deploy-command/index.js` diff --git a/docs/architecture-decisions/029-decouple-aws-from-core.md b/docs/architecture-decisions/029-decouple-aws-from-core.md new file mode 100644 index 000000000..cc4b532d2 --- /dev/null +++ b/docs/architecture-decisions/029-decouple-aws-from-core.md @@ -0,0 +1,247 @@ +# ADR-029: Decouple AWS SDK Dependencies from @friggframework/core + +**Status**: Accepted +**Date**: 2026-03-03 +**Deciders**: Sean Matthews, Frigg Team +## Context + +`@friggframework/core` directly imported multiple AWS SDK v3 packages: + +- `@aws-sdk/client-sqs` — in `Worker.js`, `queuer-util.js` +- `@aws-sdk/client-kms` — in `Cryptor.js` +- `@aws-sdk/client-apigatewaymanagementapi` — in WebSocket connection repositories and the Mongoose `WebsocketConnection` model +- AWS-specific health check logic (VPC detection, KMS capability) — in `health.js` + +This tight coupling meant that **any** consumer of `@friggframework/core` pulled in all AWS SDKs at install time, even if deploying to a non-AWS platform (e.g., Netlify, Vercel, or Docker on GCP/Azure). Bundle size on Netlify was unnecessarily large, and esbuild would fail or produce warnings when tree-shaking unused AWS SDK code. + +The recent DDD/hexagonal architecture work established clean layer boundaries (handlers → use cases → repositories), but the infrastructure layer itself was still AWS-native throughout. + +## Decision + +**Extract all AWS SDK dependencies from `@friggframework/core` into `@friggframework/provider-aws`.** + +This is a **breaking change**. Instead of using lazy-loading with backward-compatible auto-discovery of AWS adapters, we require consumers to explicitly install `@friggframework/provider-aws` and inject the appropriate adapters via constructor options or setter methods. + +### Why breaking change instead of backward compatibility? + +1. **Explicit is better than implicit** — Lazy-loading hid a hard dependency behind a `require()` call that would fail at runtime with a confusing error if `@friggframework/provider-aws` wasn't installed. An explicit constructor error at initialization time is clearer. + +2. **No phantom dependencies** — With lazy-loading, `@friggframework/core` still had a runtime dependency on `@friggframework/provider-aws` for AWS users, but this wasn't declared in `package.json`. Forgetting to install it would cause cryptic failures deep in the call stack. + +3. **Clean architecture** — Hexagonal architecture requires that adapters are explicitly wired at the composition root (app startup), not auto-discovered at call time. Lazy-loading violated this principle. + +4. **Bundle safety** — Even with lazy `require()`, some bundlers (esbuild, webpack) will follow the require path and include the AWS SDK in the bundle. Removing the `require()` entirely guarantees zero AWS SDK code in non-AWS bundles. + +### Architecture + +``` +@friggframework/core (ports/interfaces) +├── QueueClientInterface — port for queue messaging +├── EncryptionKeyProviderInterface — port for envelope encryption keys +├── WebSocketMessageSenderInterface — port for WebSocket message sending +├── AesEncryptionKeyProvider — built-in AES adapter (no AWS) +└── StaleConnectionError — shared error type + +@friggframework/provider-aws (adapters) +├── SqsQueueClient — implements QueueClientInterface +├── KmsEncryptionKeyProvider — implements EncryptionKeyProviderInterface +├── ApiGatewayMessageSender — implements WebSocketMessageSenderInterface +├── EventBridgeSchedulerAdapter — scheduler adapter +└── health/kms-health-check — KMS + VPC health checks + +@friggframework/provider-netlify (adapters) +├── NetlifyBackgroundProvider — implements QueueProvider +└── QStashQueueProvider — implements QueueProvider +``` + +## Migration Guide + +### 1. Install the AWS provider package + +```bash +npm install @friggframework/provider-aws +``` + +### 2. Worker — inject queueClient + +**Before (v2):** +```javascript +const { Worker } = require('@friggframework/core'); + +class MyWorker extends Worker { + // queueClient was auto-loaded from SQS +} +const worker = new MyWorker(); +``` + +**After (v3):** +```javascript +const { Worker } = require('@friggframework/core'); +const { SqsQueueClient } = require('@friggframework/provider-aws'); + +class MyWorker extends Worker { + // Same _run() implementation — no changes needed +} +const worker = new MyWorker({ queueClient: new SqsQueueClient() }); +``` + +### 3. Cryptor — inject keyProvider for KMS + +**Before (v2):** +```javascript +const { Cryptor } = require('@friggframework/core'); +const cryptor = new Cryptor({ shouldUseAws: true }); +``` + +**After (v3):** +```javascript +const { Cryptor } = require('@friggframework/core'); +const { KmsEncryptionKeyProvider } = require('@friggframework/provider-aws'); + +const cryptor = new Cryptor({ + shouldUseAws: true, + keyProvider: new KmsEncryptionKeyProvider(), +}); +``` + +**AES mode (unchanged):** +```javascript +// AES mode still works without provider-aws — no changes needed +const cryptor = new Cryptor({ shouldUseAws: false }); +``` + +### 4. QueuerUtil — call setQueueClient() at startup + +**Before (v2):** +```javascript +const { QueuerUtil } = require('@friggframework/core'); +await QueuerUtil.send(message, queueUrl); // auto-loaded SQS +``` + +**After (v3):** +```javascript +const { QueuerUtil } = require('@friggframework/core'); +const { SqsQueueClient } = require('@friggframework/provider-aws'); + +// At application startup: +QueuerUtil.setQueueClient(new SqsQueueClient()); + +// Then use as before: +await QueuerUtil.send(message, queueUrl); +``` + +### 5. WebSocket Connection Repositories — inject messageSender + +**Before (v2):** +```javascript +const repo = new WebsocketConnectionRepository(prisma); +// messageSender was auto-loaded from API Gateway +``` + +**After (v3):** +```javascript +const { ApiGatewayMessageSender } = require('@friggframework/provider-aws'); + +const repo = new WebsocketConnectionRepository( + prisma, + new ApiGatewayMessageSender() +); +``` + +### 6. WebsocketConnection Mongoose model — call setMessageSender() + +**Before (v2):** +```javascript +const { WebsocketConnection } = require('@friggframework/core'); +const connections = await WebsocketConnection.getActiveConnections(); +// messageSender was auto-loaded from API Gateway +``` + +**After (v3):** +```javascript +const { WebsocketConnection } = require('@friggframework/core'); +const { ApiGatewayMessageSender } = require('@friggframework/provider-aws'); + +// At application startup: +WebsocketConnection.setMessageSender(new ApiGatewayMessageSender()); + +// Then use as before: +const connections = await WebsocketConnection.getActiveConnections(); +``` + +### 7. Health checks — no changes needed + +The health router (`health.js`) uses a try/catch around `require('@friggframework/provider-aws')`. If the package is installed, AWS health checks (KMS, VPC) run as before. If not, they return `{ status: 'skipped' }`. No migration needed. + +### 8. Recommended: wire adapters at the composition root + +For clean architecture, wire all adapters at your application's entry point: + +```javascript +// app.js or handler.js — composition root +const { SqsQueueClient, KmsEncryptionKeyProvider, ApiGatewayMessageSender } = + require('@friggframework/provider-aws'); +const { QueuerUtil } = require('@friggframework/core'); + +// Wire queue adapter +QueuerUtil.setQueueClient(new SqsQueueClient()); + +// Wire encryption adapter (in prisma.js or wherever Cryptor is instantiated) +const cryptor = new Cryptor({ + shouldUseAws: true, + keyProvider: new KmsEncryptionKeyProvider(), +}); + +// Wire WebSocket adapter (in WebSocket handler setup) +const wsRepo = new WebsocketConnectionRepository( + prisma, + new ApiGatewayMessageSender() +); +``` + +## Consequences + +### Positive + +- **Zero AWS SDK in non-AWS bundles** — `@friggframework/core` has no AWS SDK imports at all +- **Explicit dependencies** — consumers declare which provider they use in `package.json` +- **Clean hexagonal architecture** — ports in core, adapters in provider packages, wired at composition root +- **Platform flexibility** — same core works on AWS, Netlify, Vercel, Docker, or any other platform +- **Better error messages** — clear errors at initialization time instead of cryptic failures deep in the call stack +- **Testability** — easy to inject mock adapters in tests without mocking AWS SDK internals + +### Negative + +- **Breaking change** — all existing AWS consumers must update their wiring code +- **More boilerplate at startup** — a few extra lines to instantiate and inject adapters +- **Two packages to install** — AWS users need both `@friggframework/core` and `@friggframework/provider-aws` + +### Risks + +- **Missed injection** — if a consumer forgets to inject an adapter, they get a clear error message pointing to this ADR and showing exactly what code to add +- **Version drift** — core and provider-aws must be compatible; managed via monorepo versioning + +## Affected Files + +### Core (ports/interfaces created) +- `packages/core/queues/queue-client-interface.js` +- `packages/core/encrypt/encryption-key-provider-interface.js` +- `packages/core/websocket/websocket-message-sender-interface.js` +- `packages/core/encrypt/aes-encryption-key-provider.js` + +### Core (lazy-loading removed, explicit injection required) +- `packages/core/core/Worker.js` +- `packages/core/encrypt/Cryptor.js` +- `packages/core/queues/queuer-util.js` +- `packages/core/websocket/repositories/websocket-connection-repository.js` +- `packages/core/websocket/repositories/websocket-connection-repository-mongo.js` +- `packages/core/websocket/repositories/websocket-connection-repository-postgres.js` +- `packages/core/websocket/repositories/websocket-connection-repository-documentdb.js` +- `packages/core/database/models/WebsocketConnection.js` +- `packages/core/handlers/routers/health.js` (graceful try/catch, not strict) + +### Provider-AWS (adapters created) +- `packages/providers/aws/queues/sqs-queue-client.js` +- `packages/providers/aws/encryption/kms-encryption-key-provider.js` +- `packages/providers/aws/websocket/api-gateway-message-sender.js` +- `packages/providers/aws/health/kms-health-check.js` diff --git a/docs/architecture-decisions/030-integration-versioning.md b/docs/architecture-decisions/030-integration-versioning.md new file mode 100644 index 000000000..7f001dedb --- /dev/null +++ b/docs/architecture-decisions/030-integration-versioning.md @@ -0,0 +1,260 @@ +# ADR-030: Integration Versioning + +**Status**: Proposed (exploratory: decision deliberately deferred pending research) +**Date**: 2026-09-27 +**Deciders**: Sean Matthews, Daniel Klotz + +## Context + +"Migration" and "versioning" are related but distinct. [ADR-013](./013-integration-version-migrations.md) proposes how to *transform* persisted integration records between versions, but explicitly defers the underlying **version contract**: how versions are declared, compared, gated, and made backward or forward compatible. This ADR opens that contract. + +### Current state (grep-verified in `next`) + +- `IntegrationBase.Definition.version` defaults to `'0.0.0'`. Comment reads "used for migration and storage purposes, as well as display." +- `IntegrationBase.Definition.supportedVersions = []`. Comment reads "Eventually usable for deprecation and future test version purposes." No code reads it today. +- `static getCurrentVersion()` returns `Definition.version`. +- On `createIntegration`, the string is stored in the `Integration.version` column (nullable `String?` in the Postgres and Mongo schemas). +- On subsequent reads, `record.version` is passed through `map-integration-dto.js` for display and preserved in every Mongo repository write. **No code path in `next` gates behavior on the value.** No routing keys on it. No handler differs by it. It is a label. + +### What versioning management would mean + +For Frigg to support integration versions as more than a label, the framework needs answers to four coupled questions: + +| Dimension | Question | +|---|---| +| **Declaration** | How does an author create v2 of an integration that has records at v1 in the wild? | +| **Routing** | When a runtime event (webhook, cron, user action) arrives for record R at version V, which code runs it? | +| **Storage** | How is the version stamped on records, config, entities, credentials, and mappings? What survives a version change? | +| **Compatibility** | What guarantees does a version make about the versions before and after it? What is a "breaking" change? | + +These are not independent. A declaration model constrains the routing model; a storage model constrains what a migration ([ADR-013](./013-integration-version-migrations.md)) can transform. + +## Decision + +This ADR **commits to reasoning about** the four dimensions above as one coupled design space, and lays out the paths that have been explored so far with their tradeoffs, serverless-specific concerns, and edge cases. It **explicitly does not** land a decision on which path to take. Landing that decision is a follow-up step gated on the exploration below plus (per Sean) more research on global routing behavior in serverless environments. + +Concretely, this ADR: + +1. Enumerates the design space as four dimensions. +2. Presents four explored paths that cover the space. +3. Catalogs the serverless-specific concerns that any chosen path must handle. +4. Lists the edge cases that a naive path would break on. +5. Leaves the final path selection to a follow-up ADR (or a revision of this one) once the open questions below are resolved. + +## Design space + +### Path A: Bump-in-place with migration on read + +Author bumps `Definition.version` on the same class. Existing records at the prior version are transformed by an [ADR-013](./013-integration-version-migrations.md) migration on read (or on next event), then stamped with the new version. There is only one class per integration type at any time. + +| Aspect | Details | +|---|---| +| Declaration | Increment `Definition.version` string in the same file. Optionally add the old version to `supportedVersions` for a compatibility window. | +| Routing | No routing decision. One class runs everything. | +| Storage | `record.version` stamps the record; a migration writes the new value after transforming. | +| Compatibility | Semver interpreted at the migration layer. `patch` = no migration needed. `minor` = additive, backward-compatible fields. `major` = mandatory migration. | + +**Tradeoffs** + +- Simple. No dispatch machinery. One code path per integration type at any time. +- Requires reliable migration for every version bump. If migration is slow or expensive, the first event after deploy pays that cost. +- Rollback is only possible if the migration is bidirectional. Prisma-style up-only migrations trap adopters at the newest version. +- The class file is a single mutating surface. Blaming ("what did v1 do?") requires git archaeology. + +**Serverless concerns unique to this path** + +- Migration on read must be idempotent under concurrent reads (two Lambdas may attempt to transform the same record simultaneously). +- Cold-start Lambdas pay the migration cost. Provisioned concurrency does not help because the migration runs on the record, not on the module. +- In-flight jobs enqueued before the deploy may run against a stale record shape. + +### Path B: Side-by-side classes with per-version routing + +Author ships `MyIntegrationV2` as a separately-registered class. Both `MyIntegrationV1` and `MyIntegrationV2` coexist in the app. New records go to V2; existing records stay on V1 until explicitly migrated. A record's version determines which class dispatches events for it. + +| Aspect | Details | +|---|---| +| Declaration | New class file. Same module dependencies (or different, if the modules changed). Registered separately in `appDefinition.integrations`. | +| Routing | Dispatcher reads `record.version` and instantiates the class registered against that version tuple. | +| Storage | `record.version` and optionally `record.classRef` (or a `(type, version)` composite key). | +| Compatibility | Explicit: each class version is its own compatibility island. Migration ([ADR-013](./013-integration-version-migrations.md)) moves records between islands. | + +**Tradeoffs** + +- Clean blame. V1 code stays in the V1 file until deprecated. +- Rollback of V2 is trivial. Records at V2 either migrate back or continue on V2 while newly-registered records go to V1 again. +- Class registry gets busier over time. An integration type at V4 has four classes registered. +- Adopters must decide when to sunset old versions and force migration. + +**Serverless concerns unique to this path** + +- Lambda handler must include both classes' code. Bundle size grows with version count. +- Alternatively, per-version Lambda functions (one Lambda per class version). Solves bundle size, adds deploy-time coordination and cold-start-per-version. +- API Gateway routing: single stage with per-version dispatch inside the handler, or per-version stages? See [Serverless concerns](#serverless-specific-concerns) below. + +### Path C: Version selector on `appDefinition` + +Author registers the same class multiple times with different versions and configuration overrides. The `appDefinition` declares which versions are available and which is the default for new integrations. + +```javascript +integrations: [ + { + Definition: MyIntegration, + versions: { + '1.0.0': { configSchema: v1ConfigSchema }, + '2.0.0': { configSchema: v2ConfigSchema, default: true }, + }, + }, +] +``` + +| Aspect | Details | +|---|---| +| Declaration | Version metadata declared in the app definition, not the class. Class code is version-aware (branches on `this.version`) or a set of version-scoped mixins. | +| Routing | Same as Path B (dispatcher reads `record.version`). | +| Storage | Same as Path B. | +| Compatibility | Declared explicitly in the app definition; the framework can validate. | + +**Tradeoffs** + +- Version policy is centralized in one file. Easy to audit which versions are supported. +- Class code becomes version-aware. Every method may need `if (this.version === '1.0.0')` branches. Readability suffers as versions accumulate. +- Adopters can offer beta versions to a subset of users by declaring them without `default: true`. +- Migration ([ADR-013](./013-integration-version-migrations.md)) still needed for records moving between versions. + +### Path D: Version as a routing dimension outside the class + +The class does not carry a version. Version is a header or path segment resolved by the framework before dispatch. Migration transforms the record's *shape* separately from any class change. Similar to API Gateway stage-based versioning. + +| Aspect | Details | +|---|---| +| Declaration | The framework registers version-aware routes. Class authors do not think about versioning until an incompatibility forces the class to branch (Path A) or split (Path B). | +| Routing | HTTP: `/api/{integration}/v2/*` or `Accept-Version: 2.0.0` header. SQS / EventBridge: message envelope carries version. | +| Storage | `record.version` stamps the record but is derived from the last successful dispatch. | +| Compatibility | Framework-enforced compatibility ranges (min/max supported version per handler). | + +**Tradeoffs** + +- Cleanest separation of routing from code. Version is infrastructure, not integration logic. +- Requires framework work to enforce version-aware routing across HTTP, SQS, EventBridge, and adopter-app calls. +- Webhook receivers with signed URLs (see [Serverless concerns](#serverless-specific-concerns)) do not carry an `Accept-Version` header; version must be encoded in the URL path or in the payload. +- Third-party integration providers control the webhook URL they call. Once registered, changing that URL requires a re-registration flow. + +## Serverless-specific concerns + +Any chosen path must handle the following. These are the reason this ADR is explicit that global routing needs more research. + +### Lambda module-scope caching + +Lambda caches module-level state across invocations on a warm instance. A `Definition.version` read at module-scope is snapshot at deploy time. Path A cannot hot-swap versions without a full deploy. Path B and C avoid this because dispatch is per-invocation. + +### Global routing + +If Frigg apps are fronted by CloudFront or an equivalent CDN, version-aware routing can happen at the edge (Path D) or at the origin. Each has cost: + +- **Edge routing** (CloudFront Functions, Lambda@Edge). Fast, but per-request compute cost multiplies across the fleet. Cannot read the DB, so version must be encoded in the URL or a header. +- **Origin routing** (API Gateway stage or per-version Lambda). Simpler, but the DB lookup to resolve `record.version` happens on every request. Cold-start pays the lookup cost. + +### Signed webhook URLs + +Third-party providers commonly sign the full URL as part of the payload signature (HubSpot v3, Stripe, Slack Events API). Changing the URL when versioning breaks signature verification until the provider is re-registered. + +- Path B with per-version URL paths (`/api/hubspot-v2-integration/webhooks`) requires re-registration of every existing HubSpot webhook when v2 ships. For adopters with hundreds of installed instances, this is a fleet operation. +- Path D with `Accept-Version` headers avoids this if the version is derived at the origin from the payload, not the URL. But not all providers send an `Accept-Version` header. + +### In-flight jobs + +SQS messages enqueued at time T reference a record that may be at v1. At time T+ε the record is migrated to v2. The worker consuming the message must: + +- Detect the version drift. +- Either migrate the message payload on read, reject to DLQ, or dispatch to the v1 handler. + +Path A must handle this at every worker. Paths B, C, D can dispatch by the payload's declared version. + +### EventBridge Scheduler entries + +[Scheduled jobs](../guides/scheduled-jobs.md) target integration IDs (see `packages/core/scheduler-commands.js`). A scheduled job set at v1-time fires at v2-time. Same drift issue as SQS; same handling options. + +### Rollback semantics + +- Path A: rollback requires a down-migration. Not all transformations are reversible. +- Path B: rollback re-points new records at the older class. Records already at V2 either migrate back or stay. +- Path C: rollback is a config change (flip `default: true` back to the prior version). +- Path D: rollback re-routes traffic. Records at V2 stay stamped V2 until re-migrated. + +### Encryption schema + +`packages/core/database/encryption/encryption-schema-registry.js` declares which fields are encrypted per model. If v2 introduces new sensitive fields (or removes them), the registry must handle both versions during the compatibility window. + +### Deploy-time coordination + +- ADR-012 covers DB schema migrations. If a v2 introduces a new required column on `Integration`, the schema migration must land before v2 code deploys. +- ADR-013 covers record migrations. If v2 requires a data transformation, the migration must be executable at deploy time (blocking) or after (best-effort). +- This ADR must define whether integration version bumps are gated on either. + +## Edge cases catalog + +Enumerated to make the failure modes concrete. Any chosen path must handle each or explicitly deprioritize it. + +1. **Mixed-version fleets.** Adopter's app has 10,000 integrations. 60% are still at v1 during a rolling migration. Any query that iterates integrations sees a mixed set. +2. **Third-party API changes force the version bump.** Adopter cannot delay migration because the third-party API deprecated the endpoint their v1 uses. +3. **Adopter apps calling old endpoints.** External systems calling `/api/hubspot-integration/*` don't know about v2. Compatibility window is required. +4. **Webhook subscriptions registered against v1 URLs.** As above; requires re-registration when the URL changes. +5. **Credential shape change.** OAuth flow changes between v1 and v2. Credentials from v1 do not decode against v2's schema. +6. **Mapping schema change.** `IntegrationMapping` records at v1 use one shape; v2 uses another. Migration must transform. +7. **Config shape change.** `Integration.config` is `Json?`. Adopters may be reading fields directly. Version-aware code must know which shape applies. +8. **Beta versions.** Adopter wants to run v2 for 5% of users while v1 continues. Requires per-user version selection. +9. **Multi-tenant version pinning.** Enterprise adopter A wants to stay on v1 for compliance. Adopter B wants v2. Same Frigg app. +10. **Framework version ships breaking change.** `@friggframework/core` v3 changes `IntegrationBase`. Integration versions across all adopters need re-baselining. +11. **Rollback after partial migration.** Migration ran on 40% of records, then failed. Half the fleet is at v2, half at v1, and the code is now v1. +12. **Cross-integration references.** Integration A's config references Integration B by ID. B migrates to v2 in a way A doesn't understand. + +## Consequences + +### Positive + +- Enumerates the design space so subsequent decisions are made against a stated set of tradeoffs rather than intuition. +- Surfaces the coupling between routing, storage, and migration that a narrower ADR would miss. +- Documents serverless-specific constraints so any implementation ADR has to address them explicitly. + +### Negative + +- No decision landed. Every implementation ADR that references this must either accept the deferral or force a decision. +- Length. This ADR is longer than the register's norm because it is exploratory. + +### Neutral + +- Introduces vocabulary (Path A / B / C / D) that subsequent ADRs and PR discussions can reference. + +## Alternatives considered + +- **Land a decision now.** Rejected. Sean's call: this needs more research on global routing behavior in serverless environments before any of Paths A–D can be selected with confidence. Landing prematurely will produce the same "shipped on intuition" failure ADR-EVALS was designed to prevent. +- **Punt to per-adopter.** Rejected. Each adopter would build a slightly different version story, and ADR-013's migration runner needs a shared version contract to key off. +- **Fold this into ADR-013.** Rejected in ADR-013 itself. Migration and versioning are distinct concerns with distinct lifecycles. + +## Open questions + +1. **Which path?** The path selection is the primary open question. Suggested criteria: which path minimizes the fleet-operation cost of a version bump? Which path handles signed webhook URLs without re-registration? Which path is compatible with the shipped `Integration Router v2` (ADR-006)? +2. **Where does the version live?** Class file, appDefinition, framework registry, or split? +3. **Semver semantics.** What counts as `patch`, `minor`, `major` for an integration? Does the framework enforce, or is it convention? +4. **`supportedVersions` behavior.** Currently declared but unread. Should this be authoritative at dispatch time, or advisory? +5. **Cross-integration version references.** How does an integration declare that it depends on another integration being at a compatible version? +6. **Beta / canary versions.** Should the framework support running two versions concurrently for a subset of users, or is that an adopter concern? +7. **Framework-side version.** Does `@friggframework/core` version interact with `Definition.version`? A `core@3.0.0` upgrade that changes `IntegrationBase` semantics could force a rebaseline across every integration. +8. **Fleet coordination.** For multi-instance Frigg deployments (per-tenant Lambdas), how are version bumps sequenced? + +## Research follow-ups + +Explicit follow-ups Sean called out for further investigation before landing a decision: + +- Global routing behavior in serverless environments. CloudFront Functions vs Lambda@Edge vs API Gateway stages vs origin dispatch. Cost, latency, cold-start impact per option. +- Signed-webhook-URL survival across URL-changing versioning schemes. Concrete provider survey (HubSpot v3, Stripe, Slack, Salesforce Streaming, Frontify) of which sign URLs vs headers vs payloads. +- Lambda alias + weighted traffic shifting as a native rollout mechanism. Does it map cleanly onto integration versioning, or is it orthogonal? +- Industry examples: how do Zapier, Merge.dev, Paragon, and similar platforms handle integration versioning behind the scenes? What did they choose, and what did they regret? + +## Related + +- [ADR-006: Integration Router v2](./006-integration-router-v2.md): current routing surface any versioning scheme must integrate with. +- [ADR-012: Database Schema Migrations](./012-database-schema-migrations.md): schema evolution, distinct from record versioning. +- [ADR-013: Integration Version Migrations](./013-integration-version-migrations.md): record transformation between versions. This ADR provides the version contract ADR-013 keys off. +- [ADR-004: Project Structure Migration Tool](./004-migration-tool-design.md): project-scaffold migration; distinct from all of the above. +- Implementation surface: `packages/core/integrations/integration-base.js` (`Definition.version`, `supportedVersions`, `getCurrentVersion`), `packages/core/prisma-postgresql/schema.prisma` (`Integration.version`), `packages/core/integrations/repositories/integration-repository-*.js`. diff --git a/docs/architecture-decisions/032-integration-deletion-cleanup.md b/docs/architecture-decisions/032-integration-deletion-cleanup.md new file mode 100644 index 000000000..6fb09cd73 --- /dev/null +++ b/docs/architecture-decisions/032-integration-deletion-cleanup.md @@ -0,0 +1,197 @@ +# ADR-032: Integration Deletion Data Cleanup + +**Status**: Proposed +**Date**: 2026-08-20 +**Deciders**: TBD + +## Context + +Deleting an integration removes one row: the `Integration` record. Everything that integration +needed in order to work stays in the database — its `Entity` records, and the `Credential` +records holding the OAuth access and refresh tokens. Those tokens are still valid. Nobody owns +them any more, and nothing will ever clean them up. + +How much of the integration's *own* data goes away depends on which database you run: + +| Backend | What happens | Result | +|---|---|---| +| PostgreSQL | Real foreign keys with `ON DELETE CASCADE` | Mappings, associations and processes are removed | +| MongoDB | Same `onDelete: Cascade` in the schema, but no real foreign keys — Prisma emulates it | The same set, usually. The repo already warns against relying on it (`user-repository-mongo.js`) | +| DocumentDB | The adapter deletes through `$runCommandRaw`, which bypasses Prisma's query engine | Nothing cascades. Every child row is orphaned too | + +Syncs are the exception on every backend. The cascade is declared, but `SyncManager` never writes +`Sync.integrationId`, so for every sync the framework has created that foreign key is null and +matches nothing — see the fourth bug below. + +So the schema says one thing and the behaviour is three different things. DocumentDB is the +worst affected and the least obvious, because the schema *does* say `Cascade`. + +### Why we can't just delete everything the integration points at + +Three records are shared on purpose, and deleting them blindly would destroy live data: + +- **An `Entity` can belong to more than one integration.** Entities are found-or-created per + user, module and external id (`process-authorization-callback.js`), so two integrations that + use the same HubSpot account share one entity row. +- **A `Credential` can back more than one entity.** The schema even says what should happen + when a credential goes away: `Entity.credential` is `onDelete: SetNull`, meaning "orphan the + entity", not "delete it". +- **A `User` is one app user, not one integration.** Users are looked up by `appUserId` on every + authenticated request, and one user owns many integrations. Deleting the user would sign them + out of everything. + +So the rule cannot be "delete what it touches". It has to be "delete what it owns, plus what +nothing else needs". + +### Four existing bugs on this path + +Found while working this out. Each is a real bug on its own, and this change depends on all +four being fixed: + +1. **A retried delete returns 500, not 404.** `findIntegrationById` throws on all three + backends, so the `Boom.notFound` check in the delete use case is unreachable. That matters + because retrying is meant to be the recovery path. +2. **Deletion error messages erase each other.** `updateIntegrationMessages` reads a `messages` + column that does not exist — there are four separate arrays (`errors`, `warnings`, `info`, + `logs`) — so it always reads empty and writes a single-element array over the top. +3. **Queue workers will start dead-lettering.** Workers discard messages for a deleted or + `IN_DELETION` integration, but only after hydrating it, and hydration throws if an entity is + missing. The `processId` path throws outright. Once we delete processes and entities, messages + that used to be discarded cleanly will fail instead. +4. **Sync rows are never linked to their integration.** `SyncManager` never writes + `Sync.integrationId`, so the cascade the schema declares has never matched a row. + +## Decision + +Add a `PurgeIntegrationData` use case that runs inside the existing delete flow: after +`ON_DELETE` (so provider-side teardown still happens first), and before the `Integration` row is +removed (so the `IN_DELETION` status keeps queue work away while we work). + +The rules: + +| Record | What we do | When | +|---|---|---| +| `IntegrationMapping`, `Process`, `Sync`, `DataIdentifier`, `Association`, `AssociationObject` | Always delete | They belong to this integration and nothing else | +| `Entity` | Delete only if orphaned | No other integration references it, and its `userId` matches the integration's | +| `Credential` | Delete only if orphaned | No entity is left pointing at it | +| `User`, `Token` | Never | One user owns many integrations | +| `UsageCounter` | Never | Deliberately kept so usage history survives deletion | +| `WebsocketConnection`, `State`, `ScriptSchedule`, `AdminScriptExecution` | Never | Not tied to an integration at all | +| EventBridge jobs, provider webhooks | The integration's `onDelete` | The framework cannot list them | +| In-flight SQS messages | Nothing | Cannot be purged selectively; workers discard them instead | + +Order matters, so the steps run children first and the integration row last: + +1. Delete the integration's own children, deepest first, so nothing is left pointing at a parent + that is already gone. DocumentDB cascades nothing, so each level is deleted explicitly rather + than assumed: + - `DataIdentifier`, then `Sync`. Match syncs on integration id **or** on the integration's + entity ids — the second arm matters, because matching on `Sync.integrationId` alone would + delete nothing today. Scope it to *those* entity ids, never to "where `integrationId` is + null", which would take every parentless sync in the database. On DocumentDB the + identifiers are an embedded array inside the sync document, so deleting the sync removes + them and there is no separate collection to clear. + - `AssociationObject`, then `Association`. + - `IntegrationMapping` and `Process`, by integration id. +2. For each entity: count the *other* integrations using it. Skip it if any remain, if it is + marked global, or if its `userId` does not match. Otherwise delete it, and remember its + credential id. +3. For each remembered credential: delete it only if no entities are left pointing at it. +4. Delete the `Integration` row. + +Every step is explicit, in the use case, and identical on all three backends. We do not rely on +the ORM's cascade behaviour, because it differs per backend and one backend has none. Extra +deletes on PostgreSQL are harmless — the rows are already gone. + +Three details worth writing down: + +- **Ownership is strict.** `Entity.userId` is nullable, and a null owner is the shape a global + entity takes (ADR-024), so a null `userId` skips rather than matches. Skipping leaves an + orphan; a false match destroys live data. +- **Prefer `deleteMany` for encrypted rows.** The encryption extension's `delete` hook decrypts + the deleted record *after* the delete, with no error handling down to KMS, so a bad key throws + with the row already gone. `deleteMany` skips that path. +- **On MongoDB, count from the integration side.** `Entity.integrationIds` is never written, so + reference counting must query `Integration.entityIds`. + +Credential deletion is behind `deletion.purgeOrphanedCredentials` in the app definition, +defaulting to on. Entities and the integration's own children are not configurable. + +Existing deployments already hold orphans from every integration ever deleted. A +`purge-orphaned-records` admin script reports counts by default and deletes with `--apply`. It +needs two new command-layer reads first, because admin scripts talk to the database only through +commands, and neither "which integrations use this entity" nor "which entities use this +credential" is exposed today. + +No schema migration is required. This is application code, so it deploys to existing +installations with no coordination. + +## Consequences + +### Positive + +- Deleting an integration stops leaving valid OAuth refresh tokens at rest. +- DocumentDB stops orphaning the integration's own child rows. +- Deletion behaviour becomes something you can read in one use case, instead of inferring it + from a schema annotation plus the ORM's emulation rules plus which adapter you happen to run. +- Shared entities (ADR-024) get a deletion story before the feature ships. +- Four latent bugs get fixed on the way. + +### Negative + +- More repository surface: bulk deletes across three backends, plus a repository for a model + that has none today. +- Deletion is a multi-step destructive operation with no transaction on two of three backends. + A partial failure leaves extra rows until a retry or the sweeper runs. +- Deleting a credential does not revoke it upstream. The token stops existing locally and keeps + existing at the provider until it expires. And once the row is gone we can never revoke it, so + if revocation is added later it has to run before the purge. +- Checking then deleting is not atomic. A concurrent create could link an entity in between. + The window is small and there is no lock to fall back on. +- The purge is only tested end to end against PostgreSQL and MongoDB. DocumentDB has no local + emulator, so it gets unit tests and a manual check — and it is the backend this helps most. + +### Neutral + +- `User`, `Token` and `UsageCounter` behaviour is unchanged. +- Existing orphans stay until someone runs the sweeper. +- Org-linked users are skipped rather than resolved. The purge compares ids, while the rest of + the codebase resolves ownership through `User.ownsUserId`, so some org setups will accumulate + orphans that only the sweeper clears. + +## Alternatives Considered + +- **Fix the schema and let the database cascade.** Rejected: it cannot express "delete only if + no one else needs this", and it does nothing for DocumentDB, which never reaches the query + engine. +- **Delete the user too.** Rejected: users are per app user and own many integrations. This is a + separate account-erasure operation, not part of deleting one integration. +- **Soft delete (a `deletedAt` column).** Rejected: it does not solve the problem. The point is + to stop storing live credentials, and a soft-deleted credential is still a stored credential. +- **Leave it to each integration's `onDelete`.** Rejected: every integration would reimplement + reference counting, and the default handler is a no-op, so most would simply not do it. +- **Sweep on a schedule instead of on delete.** Rejected as the primary mechanism — credentials + would linger for a whole sweep interval. Kept as the backstop for existing orphans and failed + purges. + +## Open Questions + +1. **Provider-side revocation.** Should deleting a credential try to revoke it upstream where + the module supports it? Best-effort, or fail the delete? +2. **Account erasure.** Should there be a separate "delete this user and everything they own" + use case? The four-step order is already documented in `user-repository-mongo.js` and + implemented nowhere. +3. **`UsageCounter` and lawful erasure.** The policy here is "never delete", which is right for + integration deletion. But no prune path exists at all, so an adopter served an erasure + request has no lawful way to remove usage rows. +4. **Audit trail.** Worth recording what each purge deleted, somewhere durable? + +## Related + +- [ADR-024: Global Entities](./024-global-entities.md) — the sharing model this must not break +- [ADR-005: Admin Script Runner](./005-admin-script-runner.md) — how the orphan sweeper ships +- [ADR-010: Reporting as an Admin Operation](./010-reporting-as-admin-operation.md) — owns the S3 + report artifacts, which are execution-scoped and out of scope here +- Implementation: `packages/core/integrations/use-cases/delete-integration-for-user.js`, + `packages/core/integrations/repositories/`, `packages/core/modules/repositories/`, + `packages/core/credential/repositories/` diff --git a/docs/architecture-decisions/033-aurora-serverless-v2-scale-to-zero-and-nat-free-connectivity.md b/docs/architecture-decisions/033-aurora-serverless-v2-scale-to-zero-and-nat-free-connectivity.md new file mode 100644 index 000000000..926e3b2e2 --- /dev/null +++ b/docs/architecture-decisions/033-aurora-serverless-v2-scale-to-zero-and-nat-free-connectivity.md @@ -0,0 +1,65 @@ +# ADR-033: Aurora Serverless v2 scale-to-zero + NAT-free Lambda connectivity + +**Status**: Proposed +**Date**: 2026-08-20 +**Deciders**: Sean Matthews + +## Context + +When Frigg provisions its own database (`database.postgres.enable: true`, `ownership: 'stack'`), it creates an Aurora Serverless v2 cluster and attaches the app's Lambdas to a VPC so they can reach it privately. Two costs make this untenable for demos and small production apps that should idle at ~$0: + +1. **Aurora never scales to zero.** `aurora-builder.js` sets `ServerlessV2ScalingConfiguration.MinCapacity` from `dbConfig.minCapacity || 0.5` and the validator rejects `minCapacity < 0.5`. So the cluster idles at **0.5 ACU (~$43/mo)** even with zero traffic. AWS added true Aurora Serverless v2 scale-to-zero (`MinCapacity: 0`, with auto-pause after inactivity) in **November 2024**; Frigg has not wired it. + +2. **The VPC forces a NAT Gateway.** Aurora must live in a VPC (an AWS constraint — a VPC itself is free). But Frigg attaches the Lambda to that VPC to reach Aurora, and a VPC-attached Lambda loses default internet egress. A Frigg app is an *integration* app: it calls external SaaS APIs (Gong, Fireflies, Salesforce, …). Reaching them from an in-VPC Lambda requires a **NAT Gateway (~$32/mo + data)**, always on. + +Together that is **~$75/mo just to idle** a Frigg-owned database — which pushes every cost-sensitive deploy to an external DB (Neon/Atlas) instead of using Frigg's own Aurora support. + +There is a well-known topology that removes both costs: put Aurora in **public** subnets with a public endpoint, keep the **Lambda outside the VPC** (so it retains normal internet egress and needs no NAT), and let the Lambda connect to Aurora over its public endpoint with TLS and a security-group allowlist. Frigg already has a `database.postgres.publiclyAccessible` flag, but it still force-attaches the Lambda to the VPC and points the Aurora ingress rule at the Lambda's VPC security group — so the NAT cost remains and the public endpoint is unreachable from the (now VPC-less) intent. This ADR closes that gap. + +## Decision + +Introduce two independent, opt-in capabilities on `database.postgres`. Each is off by default; existing app definitions are byte-for-byte unaffected. + +### 1. Scale-to-zero (`minCapacity: 0`) + +- The validator accepts `minCapacity` of **`0`** (scale-to-zero) **or** a value in **`[0.5, 128]`**. Values in `(0, 0.5)` remain invalid. +- The scaling config reads `MinCapacity: dbConfig.minCapacity ?? 0.5` (nullish coalescing — the current `|| 0.5` silently turns a requested `0` back into `0.5`, the bug this fixes). +- When `minCapacity === 0`, emit `ServerlessV2ScalingConfiguration.SecondsUntilAutoPause` from a new optional `dbConfig.secondsUntilAutoPause` (default **300**, AWS-valid range **300–86400**). Below that idle window the cluster pauses to 0 ACU. +- Scale-to-zero requires a supported engine version (Aurora PostgreSQL 13.15+/14.12+/15.7+/16.3+). Frigg's default `engineVersion` (15.13) qualifies; document the constraint and warn if a user pins an older version with `minCapacity: 0`. + +### 2. Connectivity mode (`connectivity: 'vpc' | 'public'`) + +A new `database.postgres.connectivity` selector (default **`'vpc'`** = today's behavior): + +- **`'vpc'`** (default, unchanged): Aurora in private subnets, Lambda attached to the VPC, ingress from the Lambda security group. Requires a NAT (or VPC endpoints) for the Lambda's external egress. +- **`'public'`** (NAT-free): + - Aurora is placed in **public** subnets with `PubliclyAccessible: true` (implies the existing `publiclyAccessible` behavior). + - The app's **Lambdas are NOT attached to the VPC** — the composer does not set `provider.vpc`, so they keep default internet egress. **No NAT Gateway and no VPC endpoints are provisioned.** + - The Aurora ingress rule opens **5432 to `database.postgres.allowedCidrs`** (a new option; default **`['0.0.0.0/0']`**) via `CidrIp`, instead of `SourceSecurityGroupId` pointing at the Lambda SG (which no longer exists on the Lambda side). A VPC-less Lambda has dynamic egress IPs, so a demo typically needs `0.0.0.0/0`; production should narrow it. + - **TLS is required.** The generated `DATABASE_URL` / connection params must carry `sslmode=require` (or stricter). Public Postgres without TLS is not an allowed configuration. + +The two combine: `connectivity: 'public'` + `minCapacity: 0` yields a **Frigg-owned Aurora that idles at $0 with no NAT** — the goal. + +## Security posture + +`connectivity: 'public'` exposes the database endpoint to the internet. This is an explicit, opt-in trade and the builder must make it loud: + +- Emit a validation **warning** whenever `connectivity: 'public'` is set, and a stronger one when `allowedCidrs` includes `0.0.0.0/0`. +- Require TLS (above). Credentials stay in Secrets Manager with rotation, never in the definition. +- Recommend narrowing `allowedCidrs` to known egress ranges where the deployment can (e.g. a fixed NAT/proxy, office IPs, or a CI runner range). Document that a VPC-less Lambda cannot be pinned to a stable IP without extra infra, which is why the demo default is open. +- The default stays `'vpc'`: nobody gets a public database unless they ask for one. + +## Consequences + +- **Cold-resume latency.** After auto-pause, the first query pays a resume penalty (~seconds to low tens of seconds). Acceptable for demos and low-traffic apps; document it so it is not mistaken for a hang. +- **No behavior change by default.** `'vpc'` connectivity and `minCapacity` defaulting to 0.5 mean every existing definition composes an identical template. New behavior is strictly additive and opt-in. +- **Two supported "$0 idle" paths, clearly separated.** External serverless DB (Neon/Atlas via `DATABASE_URL`) remains the zero-framework path and is cheapest for tiny apps (Atlas M0 is free even while active). This ADR makes *Frigg-owned* Aurora a viable $0-idle option for teams that want Frigg to own the whole stack. + +## Scope of the implementing change + +- `packages/devtools/infrastructure/domains/database/aurora-builder.js` — validator, scaling config (`?? 0.5`, `SecondsUntilAutoPause`), public-mode subnet/PubliclyAccessible selection, and SG ingress via `CidrIp`. +- `packages/devtools/infrastructure/domains/networking/vpc-builder.js` + `infrastructure-composer.js` — in `public` connectivity, provision the public subnets/subnet-group Aurora needs but **do not** emit the Lambda `vpcConfig` and **do not** create a NAT Gateway. +- `packages/devtools/infrastructure/domains/shared/types/app-definition.js` — document `minCapacity: 0`, `secondsUntilAutoPause`, `connectivity`, `allowedCidrs`. +- Tests asserting the generated template: `MinCapacity: 0` + `SecondsUntilAutoPause` present; public mode → no `provider.vpc`/function VPC config, no `AWS::EC2::NatGateway` resource, Aurora ingress `CidrIp`, `PubliclyAccessible: true` in public subnets; and default (`vpc`, no `minCapacity`) composes unchanged. + +**Note:** end-to-end AWS deployment validation is out of scope for the implementing PR's automated tests (it requires a live account); the PR validates template *shape* via unit tests and documents the manual deploy check. diff --git a/docs/architecture-decisions/034-api-key-login-auth-mode.md b/docs/architecture-decisions/034-api-key-login-auth-mode.md new file mode 100644 index 000000000..f0daf45f9 --- /dev/null +++ b/docs/architecture-decisions/034-api-key-login-auth-mode.md @@ -0,0 +1,83 @@ +# ADR-034: API-Key Login Auth Mode + +**Status**: Proposed +**Date**: 2026-08-20 +**Deciders**: Sean Matthews + +## Context + +Frigg apps that ship a browser SPA today have no first-class way to let an end user **log in with their own product API key** and land in an authenticated, tenant-scoped session. The existing `user.authModes` are: + +- `friggToken` — native username/password → bearer. Requires Frigg to own credentials; no notion of "your product's key is your login." +- `sharedSecret` — `x-frigg-api-key` + `x-frigg-appUserId`/`appOrgId` headers. Correct for backend-to-backend, but the master `x-frigg-api-key` can never live in a browser, so it forces every adopter to stand up a **BFF/token-broker** in front of Frigg (this is exactly what `lefthookhq/aes--frigg`'s `auth-proxy/` does: validate the product key against the product's API, derive an org id, then proxy to Frigg with the shared secret + `x-frigg-apporgid`). +- `adopterJwt` — designed for adopter-verified JWTs, but currently a `501` stub. + +The BFF pattern works and is secure, but it makes every "log in with your product key" app carry a second deployable service, duplicate an allowlist, and hand-roll session/refresh logic. For products where **the API key is already the unit of API authority** (the common case), Frigg can offer this natively. + +Critically, the pieces already exist in core: + +- `modules/use-cases/process-authorization-callback.js` already instantiates a `Module` and its Requester and creates/refreshes the **Credential + Entity** from `{ api_key }` — it is what `POST /api/authorize` runs. +- `user/use-cases/get-user-from-x-frigg-headers.js` already **find-or-creates** an individual/org user from identifiers. +- `login-user` / token minting already issues a Frigg session token. + +What's missing is the glue: validate a pasted key **through the api-module itself**, derive the tenant identity **from the provider** (not the client), find-or-create the user, create the credential/entity, and issue a session — behind a declared auth mode. + +## Decision + +Add a first-class, opt-in `apiKey` auth mode. An app declares which module is its **identity provider**: + +```js +user: { + authModes: { apiKey: { module: 'reevo' } }, + organizationUserRequired: true, // so the org user is created from the provider org id +} +``` + +### Route (reuse `POST /user/login`, polymorphic on credential shape) + +There is **no mode-specific path**. The existing `POST /user/login` becomes polymorphic: it dispatches on the credential shape in the body against the app's enabled `authModes`. + +- `{ username, password }` → `friggToken` (existing behavior, unchanged). +- `{ apiKey }` → the `apiKey` mode (this ADR). The identity module is fixed by config (`authModes.apiKey.module`); a multi-identity app MAY send `{ module, apiKey }` restricted to a configured allowlist. + +Both modes may be enabled simultaneously and coexist on the one route (the bodies are disjoint, so dispatch is unambiguous). The endpoint stays unauthenticated and rate-limited. Rationale: "log in" is the resource; which credential counts is a server-config detail, not something the URL should encode — and adding future modes never adds routes. + +### The use case (`LoginWithApiKey`) + +1. **Validate + identify via the module's Requester.** Instantiate the configured identity module with the supplied key and call its `requiredAuthMethods`: `testAuthRequest` (validity) and `getEntityDetails`/`getCredentialDetails` (identity + the properties to persist). The api-module — not a bespoke validator — is the source of truth for "is this key valid, and whose is it." A `401/403` from the provider → invalid key (generic error, no enumeration); a `5xx`/timeout → provider-unavailable (`503`), distinct from a bad key. + + **`testAuthRequest` login contract (normative).** On this path `testAuthRequest` MUST either **throw** (a provider error the classifier splits into 401 vs 503) or **return the strict boolean `true`**. The validity gate requires `=== true`; any other value — including a truthy error object, a non-empty string, or a response body — is treated as a failed validation (generic 401), NOT a pass. A module that signals a bad key by returning a truthy object instead of a falsy value therefore cannot clear the gate. `getEntityDetails` MUST return a stable **scalar** `identifiers.externalId` (string or number); a non-scalar (object/array/boolean) is rejected as "no stable identifier" rather than coerced. +2. **Derive a provider-authoritative identity.** `appOrgId` (and/or `appUserId`) MUST come from the provider response (e.g. the account/org id `getEntityDetails` returns), NEVER from client input, and MUST be a stable, tenant-unique identifier. Hashing the key (`sha256(apiKey)`) is explicitly disallowed as the identity — it changes on key rotation and orphans connections (the AES R1 flaw). The find-or-create identity is **namespaced by the resolved module name** (`${moduleName}:${externalId}`): in a multi-module allowlist two different providers can legitimately return the same `externalId`, and the namespace keeps those distinct tenants from collapsing onto one Frigg user. +3. **Find-or-create the Frigg user** from that identity, reusing the existing find-or-create path. The issued principal is an ordinary **app user**, never an admin. +4. **Create the Credential + Entity** by running `ProcessAuthorizationCallback(userId, module, { api_key })` — the same path `/api/authorize` uses — so the module's source is connected as part of login. The callback's return is asserted to carry a persisted `credential_id` **before** a session is minted; a callback that returns without one fails the login `500`-class rather than handing out a session over a half-provisioned tenant. (The user is found-or-created before the credential is provisioned; an orphaned user on partial callback failure is accepted cleanup debt, tolerated over reordering.) +5. **Issue a Frigg session token** and return it. Default: an httpOnly, `secure`, `sameSite` cookie plus a short-lived access token; the raw key is **not** returned to or re-sent by the browser after login (it lives only as the encrypted Credential). + +### Trust model (documented, accepted) + +Possession of a valid provider API key confers authority over that tenant's integrations in the Frigg app. This **mirrors** the authority the key already grants at the provider — it is not an escalation. There is no second factor; this is bearer-key trust, appropriate for products whose API key is already the unit of API authority. Adopters whose keys are broad, long-lived, and unrotatable should prefer `friggToken` or an external IdP via a BFF instead. + +## Security requirements (normative — the implementation MUST honor these) + +1. **Provider-authoritative identity.** `appOrgId`/`appUserId` derive only from the module's provider response; a client-supplied org/user id is ignored. Reject a login whose module returns no stable identifier. +2. **Session ≠ admin.** The minted session is a normal app-user token scoped to that tenant; it must not authorize `/user/*` management routes or cross-tenant access. +3. **Rate limiting.** `POST /user/login` (the polymorphic route) is rate-limited (per-IP and global) to prevent using it as a key-validation oracle against the provider. Cap key length before any work. Errors are generic (no user/key enumeration). The per-IP bucket key is derived from a **trusted** X-Forwarded-For position (rightmost by default, or `authModes.apiKey.rateLimit.trustedProxyDepth` hops from the right), NOT the client-controlled leftmost hop, so an attacker cannot rotate a spoofed leftmost XFF to mint a fresh bucket per request. The in-process limiter is a floor: `maxGlobal` is the only hard in-process ceiling (and only per container in a multi-instance deployment). The real per-IP control belongs at the edge (WAF / API Gateway throttling). +4. **Key at rest _and in logs_.** The key is persisted only as the Credential, through Frigg's field-level encryption (KMS/AES). It is never returned after login and never placed in a JWT claim. It is **never logged**: the framework logger (`initDebugLog`) redacts a denylist of credential-bearing request-body/header fields (`apiKey`, `api_key`, `password`, `token`, `authorization`, `refresh_token`, `access_token`) to `[REDACTED]` before the Lambda event is buffered — so the raw key cannot leak via the buffered debug dump on a 5xx nor via `DEBUG_VERBOSE=1`. This also protects the `friggToken` `password` on the shared route. +5. **Revocation latency is bounded by TTL.** Access tokens are short-lived; refresh (if implemented) MUST re-validate the stored key via the module's `testAuthRequest` before rotating, so a revoked provider key stops working within one TTL rather than for the life of a long session. +6. **Outage ≠ invalid.** Provider `5xx`/timeout returns `503` and does not revoke the session or clear cookies; only a definitive `401/403` invalidates. +7. **Cookie hygiene.** `httpOnly`, `secure` (in non-local stages), `sameSite: 'strict'`, and a cookie `Max-Age`/`Expires` aligned to the session-token TTL (cookie and token expire together). The Origin/Referer allowlist (CSRF) is **opt-in** via `authModes.apiKey.allowedOrigins` so it does not hard-break unconfigured local dev; when `apiKey` mode is enabled without it, the framework emits a one-time wiring-time `console.warn` noting that Origin enforcement is off and the `sameSite: 'strict'` cookie is the residual protection. `allowedOrigins`, when present, MUST be an array (validated at wiring time). Adopters serving a browser SPA should configure it. + +## Consequences + +- **Removes the mandatory BFF** for "log in with your product key" apps: the browser talks to Frigg directly (login → cookie → normal calls). The BFF remains the right tool when identity must come from a third-party IdP, or when a proxy is wanted for other reasons. +- **Reusable across every api-key module.** Any module exposing the standard `requiredAuthMethods` gets product-key login for free by naming it in `authModes.apiKey.module`. +- **Default-off, additive.** Apps that don't declare `authModes.apiKey` are unchanged. It composes with the existing modes (an app may keep `friggToken`/`sharedSecret` on). +- **Supersedes** the earlier sketches (`adopterJwt` completion, a bespoke `apiKeyResolver` hook): validating through the api-module is strictly better than a hand-supplied resolver because the module already encodes how to auth-test and identify a key. + +## Scope of the implementing change + +- `packages/core/user/use-cases/login-with-api-key.js` — the new use case (validate via module → derive identity → find-or-create user → ProcessAuthorizationCallback → mint token). Plus wiring in `authenticate-user.js`/the user router for the new mode and route. +- `packages/core/handlers/routers/*` — make the existing `POST /user/login` polymorphic (dispatch `{ apiKey }` → apiKey mode, `{ username, password }` → friggToken unchanged); rate-limited; cookie issuance. +- App-definition `user.authModes.apiKey` config validation + docs. +- Tests: valid key → user+credential+entity created and a session returned; invalid key → 401 generic; provider outage → 503 with no session; **client-supplied org id is ignored** (impersonation guard); rate-limit trips; refresh re-validates; the minted token cannot reach `/user/*`. Mutation-test the impersonation guard and the 401-vs-503 split. + +**Note:** end-to-end validation against a live provider is out of scope for the PR's automated tests (uses a mocked module Requester); the security requirements above are enforced by unit tests on the use case and route. diff --git a/docs/architecture-decisions/035-app-init.md b/docs/architecture-decisions/035-app-init.md new file mode 100644 index 000000000..11db1a7df --- /dev/null +++ b/docs/architecture-decisions/035-app-init.md @@ -0,0 +1,129 @@ +# ADR-035: App Init + +**Status**: Proposed +**Date**: 2026-09-27 +**Deciders**: Sean Matthews + +## Context + +The other ADRs in this set describe what you build *inside* a Frigg app: plugins, extensions, integrations, capabilities, templates, artifacts. None of them describe how a Frigg app comes into existence. + +`frigg init` today produces a minimal scaffold. Adopters routinely then hand-configure the shape they actually want (which database plugin, which encryption method, which core extensions, which starter integrations, which L4 ontology conventions). Every adopter does the same setup by hand. Issue [#595](https://github.com/friggframework/frigg/issues/595) tracks the gap and flags the vestigial `--template` flag on the current implementation. + +**Project Templates** are distinct from [Integration Templates](./023-integration-templates.md). Integration Templates are per-category base classes an adopter copies into an existing app. Project Templates initialize the app itself. + +## Decision + +`frigg init` produces a Frigg app from a named **Project Template** plus an interactive interview that scaffolds overrides. The output is a working project with the adopter's plugin choices, seeded core extensions, seeded L4 ontology, and (optionally) a first reference integration. + +### Project Template registry (initial set) + +| Template | Posture | Use case | +|---|---|---| +| `minimal` | Bare skeleton. AWS provider, in-memory queue, no extensions, no reference integration. | Learning, experimentation | +| `production` | Postgres + KMS + private VPC + EventBridge scheduler + audit-log Core Extension + agent-frigg-claude Core Extension. Reference integration scaffolded. | Adopter shipping to prod | +| `agent-enabled` | Production posture plus MCP-server Core Extension, harness pre-configured, adopter skill and agent scaffolding under `.claude/`. | Adopter running agents against the codebase from day one | + +Adopters can publish additional templates under `@friggframework/project-template-*` or `@/project-template-*`. The registry is the same convention as [PLUGINS](./016-plugins.md) (`@friggframework/provider-*` etc). + +### Interview + +After template selection, `frigg init` walks the adopter through the choices that a template cannot pre-decide. Each answer writes into the generated `appDefinition`: + +| Question | Writes to | +|---|---| +| Database (Postgres / Mongo / DocumentDB / SQLite) | `plugins.database.kind` | +| Encryption (KMS / AES) | `plugins.encryption.kind` | +| Deploy target (AWS / Netlify / Vercel / GCP) | `plugins.provider.kind` | +| Auth modes (friggToken / sharedSecret / password / SSO) | `user.authModes` | +| Reference integration (skip / one from a menu / a partner-you-name) | Adds an [Integration Template](./023-integration-templates.md) copy under `backend/src/integrations/` | +| CI provider (GitHub Actions / none) | `.github/workflows/*` files | +| Changesets on/off | `.changeset/config.json` | + +Non-interactive mode: `frigg init --template production --answers answers.yaml` runs the same flow from a file. Both modes produce an identical result. + +### L4 ontology seed + +The init flow writes a starter `ontology/l4.yaml` derived from the interview answers. Example: + +```yaml +version: 1 +layer: L4 +domain: my-app +conventions: + - id: my-app.db + rule: "This app uses Aurora Postgres" + rationale: "Selected at init time" + - id: my-app.auth-modes + rule: "friggToken and sharedSecret are enabled; adopter apps must send x-frigg-api-key or a bearer JWT" +``` + +The seed is a small starting point. Adopters add to it over time as their conventions crystallize. See [ADR-021: Ontology](./021-ontology.md). + +## Shape (worked example) + +```bash +$ frigg init my-frigg-app --template production +✓ Copying production template into ./my-frigg-app/ +? Database › Aurora Postgres +? Encryption › AWS KMS (recommended for production) +? Deploy target › AWS +? Auth modes › friggToken, sharedSecret +? Add a reference integration? › Yes, HubSpot ↔ my adopter API (crm-sync-bidir) +? CI provider › GitHub Actions +? Changesets › Enable +✓ Wrote appDefinition to backend/index.js +✓ Wrote L4 ontology seed to ontology/l4.yaml +✓ Scaffolded reference integration at backend/src/integrations/HubspotSync/ +✓ Wrote CI workflows to .github/workflows/ +✓ Ran npm install +✓ Ran `frigg auth test .` on scaffolded modules. All pass. +ℹ Next: map your adopter API in backend/src/integrations/HubspotSync/mapping.js +``` + +## Architecture + +```mermaid +flowchart LR + subgraph Registry["Project Template registry"] + T1["minimal"] + T2["production"] + T3["agent-enabled"] + T4["@org/project-template-*"] + end + subgraph Cli["frigg init"] + Sel["template select"] + Int["interview
(db, encryption, provider,
auth, reference integration,
CI, changesets)"] + end + subgraph Output["Generated app"] + Def["appDefinition.plugins
appDefinition.extensions"] + Ont["ontology/l4.yaml (seed)"] + Int1["backend/src/integrations/*
(from Integration Template)"] + Ci[".github/workflows/*"] + end + Registry --> Sel + Sel --> Int + Int --> Def & Ont & Int1 & Ci +``` + +## Cross-references + +- [PLUGINS](./016-plugins.md): interview answers write into `plugins.*` selections +- [CORE-EXTENSIONS](./017-core-extensions.md): Project Templates preload zero or more Core Extensions +- [INTEGRATION-TEMPLATES](./023-integration-templates.md): reference integration option delegates to Integration Templates +- [ONTOLOGY](./021-ontology.md): the L4 seed is a starting point for adopter-owned conventions +- [SKILLS](./036-skills.md): agent-enabled template scaffolds `.claude/` with a starter skill and agent set + +## Open questions + +1. **Template distribution.** npm package (`@friggframework/project-template-production`) vs git URL (`--template github:org/repo`) vs both? Lean: npm primary, git URL supported for org-private templates. +2. **Template inheritance.** Can `agent-enabled` be defined as `production` + a delta, or must every template be self-contained? Lean: self-contained; delta introduces a compatibility matrix. +3. **Interview UX.** JSON Schema form (like `frigg auth test`) vs inquirer prompts? Lean: JSON Schema for consistency with the rest of the CLI. +4. **Re-run behavior.** `frigg init` in an existing directory: error, refuse, or `--reconfigure` to update overrides without touching custom code? Lean: refuse unless `--reconfigure` is passed. +5. **The vestigial `--template` flag.** The current implementation has a `-t, --template` option that falls through to a "Legacy template system is no longer supported" error path (issue #595). Remove it, or repurpose it to select from the new registry? Lean: repurpose. + +## References + +- Issue [#595](https://github.com/friggframework/frigg/issues/595): the proposal that motivated this ADR +- Issue [#594](https://github.com/friggframework/frigg/issues/594): related docs drift on the CLI command list +- Existing production Frigg projects at adopters have converged on a common posture (Postgres + KMS + VPC + EventBridge + Changesets + GitHub Actions). That converged shape is the prior art for the `production` Project Template proposed here. diff --git a/docs/architecture-decisions/036-skills.md b/docs/architecture-decisions/036-skills.md new file mode 100644 index 000000000..31f01d5cb --- /dev/null +++ b/docs/architecture-decisions/036-skills.md @@ -0,0 +1,131 @@ +# ADR-036: Skills + +**Status**: Proposed +**Date**: 2026-09-27 +**Deciders**: Sean Matthews + +## Context + +Adopter Frigg apps that use agents (Claude Code, Cursor, custom orchestrators) ship a `.claude/skills/` directory alongside `.claude/agents/`. Skills are the read-on-demand knowledge, contracts, and runbooks that agents load into context. The [ADR-025: Agent Harness](./025-agent-harness.md) proposes a runtime hook that injects compiled ontology + capability data at session start. Skills are the complementary declarative surface: knowledge and procedures the agent can load explicitly when relevant. + +An adopter repo that uses Frigg for repeated CRM integration authoring has an in-production skill layer with three sub-agents and five skills. Reading its actual files surfaces a distinction the current ADR set does not name: skills have distinct subtypes with different lifecycles and different authoring roles. Bundling them under one word obscures which pattern applies when. + +## Decision + +A **Skill** is a Markdown-fronted directory under `.claude/skills//` containing a `SKILL.md` with YAML frontmatter (`name`, `description`, optional `user-invocable`, optional `argument-hint`) plus supporting files. Skills are one of four subtypes: + +| Subtype | What it contains | Example | Loaded by | +|---|---|---|---| +| **Knowledge skill** | Reference material (an L3 or L4 [Ontology](./021-ontology.md) fragment). Ref files per stage. Hard rules. Gotchas. Gold-standard exemplars. | `{adopter}-integration-ontology` | Any agent, preloaded via `skills:` frontmatter | +| **Contract skill** | JSON Schema + template + validator. The machine-checkable shape of a handoff artifact. | `{adopter}-spec-contract` | Producer and consumer agents in a pipeline | +| **Runbook skill** | Human-executable procedure. Scripts, templates, step-by-step. | `{adopter}-test-integration`, `{adopter}-changeset` | User or agent per-invocation | +| **Recipe skill** | Multi-phase orchestration with embedded knowledge and procedure. | `frigg-create-integration` | User or top-level agent | + +### The `user-invocable` flag + +Skills default to user-invocable. Set `user-invocable: false` in frontmatter to hide the skill from user-facing lists. Sub-agent-only knowledge and contract skills should set this so they do not appear as CLI options; they exist as agent preload material only. + +### Skill-to-agent preloading + +A sub-agent declares which skills to preload via the `skills:` field in its frontmatter: + +```yaml +--- +name: spec-author +description: Authors an integration spec from a third-party API's docs. +tools: Read, Grep, Glob, WebFetch, Write, Bash +model: opus +skills: + - {adopter}-integration-ontology + - {adopter}-spec-contract + - frigg +--- +``` + +The framework loads the referenced skills' `SKILL.md` and any explicitly-referenced reference files into the agent's context at spawn time. Reference files under the skill's directory (`reference/*.md`) are read on demand by the agent, not eagerly loaded. + +This declarative form is a valid alternative to the [Agent Harness](./025-agent-harness.md) hook for grounding an agent. The harness compiles a merged ontology + capability graph at session start; `skills:` frontmatter loads specific named skills at agent spawn. Both mechanisms coexist. The harness handles cross-cutting context; `skills:` handles agent-specific bundles. + +## Shape (worked example) + +Directory layout for the four subtypes: + +``` +.claude/skills/ +├── {adopter}-integration-ontology/ # knowledge skill +│ ├── SKILL.md # user-invocable: false +│ └── reference/ +│ ├── base-crm-integration.md +│ ├── api-module.md +│ ├── gotchas-checklist.md +│ ├── hcp-gold-standard.md +│ ├── house-style.md +│ └── {vendor}-module.md +├── {adopter}-spec-contract/ # contract skill +│ ├── SKILL.md # user-invocable: false +│ ├── schema/spec.schema.json # the JSON Schema +│ └── templates/spec-template.md # a human-readable render template +├── {adopter}-test-integration/ # runbook skill +│ ├── SKILL.md # argument-hint: "[spec-path] [local|dev]" +│ ├── scripts/run-lifecycle.sh +│ └── templates/test-report.template.json +└── frigg-create-integration/ # recipe skill + ├── SKILL.md + ├── assets/ + └── references/ +``` + +## Architecture + +```mermaid +flowchart TB + subgraph Skills[".claude/skills/"] + K["Knowledge skill
(L3/L4 ontology fragment)"] + C["Contract skill
(JSON Schema + template)"] + R["Runbook skill
(scripts + templates)"] + Rc["Recipe skill
(multi-phase orchestration)"] + end + subgraph Agents[".claude/agents/"] + A1["spec-author
skills: [ontology, contract]"] + A2["build-engineer
skills: [ontology, contract, recipe]"] + A3["adversarial-reviewer
skills: [ontology]"] + end + subgraph Adopter["Adopter developer"] + U["frigg CLI or agent invocation"] + end + K -- "preloaded via `skills:`" --> A1 & A2 & A3 + C --> A1 & A2 + Rc --> A2 + R -- "invoked directly" --> U + U -- "spawns" --> A1 & A2 & A3 +``` + +## Authoring guidance for adopters + +When authoring a skill layer for a new adopter app: + +1. **Start with the knowledge skill** (`-integration-ontology`). Capture framework contract facts, house style, gotchas, and a gold-standard exemplar. This is the L3+L4 [Ontology](./021-ontology.md) fragment. +2. **Add the contract skill** (`-spec-contract`) if you have a repeated authoring flow that hands off between phases (scoping → building → testing). Ship the JSON Schema, a template, and a validator. +3. **Add runbook skills** for procedures the human runs directly (`-test-integration`, `-changeset`, `-deploy`). +4. **Compose sub-agents** in `.claude/agents/` that reference the skills via `skills:` frontmatter. See [ADR-037: Agent Pipeline](./037-agent-pipeline.md). + +## Cross-references + +- [ONTOLOGY](./021-ontology.md): knowledge skills are the concrete surface for L3 and L4 ontology fragments +- [CAPABILITIES](./020-capabilities.md): a contract skill can be the JSON Schema that a capability's `spec.ref` points at +- [AGENT-HARNESS](./025-agent-harness.md): harness compiles cross-cutting context; `skills:` frontmatter loads agent-specific bundles. Complementary. +- [AGENT-PIPELINE](./037-agent-pipeline.md): pipelines are composed from sub-agents that preload skills +- [APP-INIT](./035-app-init.md): the `agent-enabled` Project Template scaffolds a starter skill layer + +## Open questions + +1. **Skill discovery convention.** `@friggframework/skill-*` packages? Both npm and in-repo `.claude/skills/`? Lean: both, with in-repo taking precedence when both are present. +2. **Skill versioning.** Do skills carry their own version, or inherit from the package that ships them? Lean: inherit; skills change with the framework or adopter code they document. +3. **Reference-file eager vs lazy loading.** `SKILL.md` loads eagerly at agent spawn; `reference/*.md` reads on demand. Is that the right split, or should some reference files also load eagerly? Lean: current split. +4. **Contract-skill validators.** Do contract skills ship a Node validator, a JSON Schema alone, or both? Lean: schema is authoritative; validator is a convenience. +5. **Cross-adopter skill reuse.** If ten adopter apps end up shipping similar `-changeset` runbook skills, should the common shape move into a shared `@friggframework/skill-changeset`? Lean: yes, once three adopters converge on the same shape. + +## References + +- An adopter repo running the pattern in production has five skills (one knowledge, one contract, three runbook / recipe) and three sub-agents. The five-skill, three-agent shape is the reference implementation for this pattern. +- Anthropic's Claude Code documentation on skills and agents is the framework this ADR builds on. diff --git a/docs/architecture-decisions/037-agent-pipeline.md b/docs/architecture-decisions/037-agent-pipeline.md new file mode 100644 index 000000000..73fc0fd4f --- /dev/null +++ b/docs/architecture-decisions/037-agent-pipeline.md @@ -0,0 +1,137 @@ +# ADR-037: Agent Pipeline + +**Status**: Proposed +**Date**: 2026-09-27 +**Deciders**: Sean Matthews + +## Context + +The [Agent Harness](./025-agent-harness.md) grounds a single agent's session with compiled ontology and capability context. The [Skills](./036-skills.md) layer lets agents preload adopter-specific knowledge and contracts. Neither addresses how multiple agents compose into a workflow with typed handoffs. + +An adopter demonstrates a three-agent pipeline in production: a spec-author produces a spec, a build-engineer consumes that spec and implements the integration, an adversarial-reviewer refutes-by-default against a gold-standard exemplar. Each phase has its own model, tools, and role. The handoff between phases is a machine-checkable artifact validated against a JSON Schema. + +The pattern is general to any repeated integration authoring flow that benefits from splitting scoping from building from reviewing. This ADR names the shape and cross-references the pieces that compose it. + +## Decision + +An **Agent Pipeline** is a sequence of sub-agents with typed handoffs. Each agent has a defined role, model tier, tool restriction, and preloaded skill set. The output of one phase is the input to the next, validated against a [Contract Skill](./036-skills.md) schema. + +### Canonical three-phase pipeline + +``` +scope-author → build-engineer → adversarial-reviewer +``` + +| Phase | Role | Model | Tools | Preloaded skills | +|---|---|---|---|---| +| Scope | Author the spec from third-party API docs + adopter ontology | opus (creative discovery) | Read, Grep, Glob, WebFetch, Write, Bash | ontology, contract, recipe | +| Build | Implement the spec | sonnet (mechanical) | Read, Write, Edit, Bash, Grep, Glob | ontology, contract, recipe | +| Review | Refute the artifact meets the quality bar | opus (skeptical judgment) | Read, Grep, Glob, Bash (`disallowedTools: Write, Edit`) | ontology | + +Model tiering is intentional. Scoping and adversarial review benefit from a stronger model because both require judgment. Building is mechanical once the spec exists and works well at a smaller model. + +### The read-only adversary + +The reviewer is structurally read-only via `disallowedTools: Write, Edit`. The reviewer cannot patch the artifact it critiques. The framework enforces this at spawn time. + +Review calibration: + +- Default to skeptical. A finding is "PASS" only if the reviewer genuinely could not refute it. +- Findings cite file:line or the spec field. Free-form judgment without an anchor is not a valid finding. +- Explicit `openQuestions` in the spec are NOT defects. They are the human-decision surface. The reviewer only raises a question if the spec should have answered it from the available API docs. +- Over-engineering and verbosity are penalized as hard as gaps. Speculative abstractions and unrequested config fail review. + +### Typed handoffs + +The output of each phase is validated against a [Contract Skill](./036-skills.md) schema: + +- Scope → Build: `specs/.spec.json` conforms to `spec.schema.json` +- Build → Review: the codebase diff plus the spec ID +- Review → Build (on refutation): a findings JSON conforming to `review.schema.json` with `{finding_id, file, line, category, severity, rationale}` + +Contract skills ship the schemas alongside human-readable templates. Both are versioned with the adopter's skill package. + +### EARS acceptance criteria + +The spec declares acceptance criteria in [EARS](https://alistairmavin.com/ears/) form: + +```json +{ + "acceptanceCriteria": [ + { "id": "AC-1", "statement": "WHEN a new contact is created in HubSpot THE SYSTEM SHALL upsert the corresponding destination CRM contact within 30 seconds" }, + { "id": "AC-2", "statement": "WHEN the third-party OAuth refresh fails THE SYSTEM SHALL mark the integration as ERROR and stop further sync attempts" } + ] +} +``` + +The build agent tags each test with the matching `AC-n` id. The trace from requirement to test survives the pipeline and is grepped by the reviewer. + +## Architecture + +```mermaid +flowchart LR + subgraph Scope["Scope phase (opus)"] + S["spec-author"] + end + subgraph Build["Build phase (sonnet)"] + B["build-engineer"] + end + subgraph Review["Review phase (opus, read-only)"] + R["adversarial-reviewer"] + end + Docs["Third-party API docs
+ adopter ontology"] --> S + S -- "specs/*.spec.json
(validates against
spec.schema.json)" --> B + B -- "diff + tests tagged AC-n" --> R + R -- "refutation
(review.schema.json)" -.-> B + R -- "PASS" --> Ship["Merge → deploy"] +``` + +Adversarial-review refutations loop back to the build agent, which patches and re-submits. The pipeline terminates when the reviewer returns PASS. + +## Shape (worked example) + +Invoking the pipeline from the top-level agent or CLI: + +```bash +$ frigg pipeline run integration --partner hubspot --name HubspotSync +Phase 1/3: Scoping + → spec-author (opus) authoring specs/hubspot-sync.spec.json + ✓ Spec validated against spec.schema.json (12 AC, 8 gotchas addressed) +Phase 2/3: Building + → build-engineer (sonnet) implementing from spec + ✓ Generated backend/src/api-modules/hubspot/ (api.js, definition.js) + ✓ Generated backend/src/integrations/HubspotSyncIntegration.js + ✓ Generated tests (12 tagged AC-1 through AC-12) +Phase 3/3: Reviewing (adversarial) + → adversarial-reviewer (opus, read-only) via 3 lenses: correctness, gotchas, simplicity + ✗ FAIL: AC-7 test missing rate-limit assertion; over-engineered retry helper (lens: simplicity) +Loop back to Phase 2: + → build-engineer patching per review findings + ✓ AC-7 test now asserts rate-limit backoff; retry helper removed +Phase 3/3: Reviewing (retry) + ✓ PASS: all three lenses cleared +Pipeline complete. Ready to merge. +``` + +## Cross-references + +- [AGENT-HARNESS](./025-agent-harness.md): each agent in the pipeline is grounded by the harness at spawn (compiled ontology + capabilities). The pipeline composes agents; the harness grounds them. +- [SKILLS](./036-skills.md): each agent preloads knowledge, contract, and recipe skills via `skills:` frontmatter +- [CAPABILITIES](./020-capabilities.md): the spec-contract schema is what a capability's `spec.ref` can point at when the capability is implemented by an agent-produced integration +- [ONTOLOGY](./021-ontology.md): the reviewer refutes against ontology-encoded conventions and locked constraints +- [APP-INIT](./035-app-init.md): the `agent-enabled` Project Template scaffolds a starter pipeline + +## Open questions + +1. **Pipeline runner.** Where does the pipeline orchestrator live? A `frigg pipeline` CLI command, a Claude Code agent workflow, an SDK API, or all three? Lean: CLI command that spawns agents in sequence. +2. **Retry ceiling.** How many review → build → review loops before human escalation? Lean: 3, configurable. +3. **Parallel review lenses.** The `adversarial-reviewer` reviews through one lens at a time. Should the framework run multiple lens instances in parallel and merge findings? Lean: yes, parallel, merged deterministically. +4. **Cross-adopter pipeline reuse.** Are pipelines adopter-specific or general (`frigg-integration-pipeline`)? Lean: start adopter-specific, generalize once three adopters converge on the same shape. +5. **Failure attribution.** If a review fails, does the framework attribute the failure to the build agent, the spec, or the ontology? Lean: attribute to the earliest phase whose artifact would need to change to resolve the finding. +6. **Human-in-the-loop points.** Where can (or must) a human intervene? Lean: after scoping (approve spec), after successful review (approve merge), never during a phase. + +## References + +- An adopter running the pattern in production ships `spec-author.md`, `build-engineer.md`, and `adversarial-reviewer.md` under `.claude/agents/` as the reference implementation. +- [EARS](https://alistairmavin.com/ears/): Easy Approach to Requirements Syntax; the acceptance criteria form. +- Anthropic's Claude Code sub-agent documentation. diff --git a/docs/architecture-decisions/038-configuration-and-secrets-model.md b/docs/architecture-decisions/038-configuration-and-secrets-model.md new file mode 100644 index 000000000..c1c1987d8 --- /dev/null +++ b/docs/architecture-decisions/038-configuration-and-secrets-model.md @@ -0,0 +1,211 @@ +# ADR-038: Configuration & Secrets — Model & Tiers + +**Status**: Proposed +**Date**: 2026-07-10 +**Deciders**: Sean Matthews + +> Proposed. Companion ADRs: **ADR-039** (provider plugin), **ADR-040** (management interface), and +> **ADR-041** (docs & adopter maturation). Open questions at the end are implementation detail and +> non-blocking. + +## Context + +Today Frigg can really only set environment variables **manually at build time**: the declarative +`appDefinition.environment` list pulls values from the deploy pipeline and injects them into the +functions via serverless-oss. Beyond that, an adopter must hand-roll a Lambda layer or already know +about the `secrets-to-env.js` (Secrets Manager) path; the SSM runtime loader is drafted and unmerged. + +That single flat env list **blends separable concerns with no distinction**: +- **Infra/runtime** config — `DATABASE_URL`, `KMS_KEY_ARN`, `STAGE` (framework-owned). +- **Framework-behavior** settings — how Frigg itself should behave. +- **Per-environment / dev flags** — feature flags, code-path toggles a dev wants per stage. +- **API-module app credentials** — the client id / secret / scopes each API module needs. + +The last category is the one that breaks. **Every adopter eventually hits "env overload,"** usually +at **10+ API modules**, each with OAuth-style credentials (client id + secret + scopes ≈ 3+ values) += **30+ "env variables"** in one undifferentiated list. + +**Correction to an earlier assumption:** the existing DB models are **instance-scoped** and are *not* +a home for app/definition-level env or module credentials: +- `Integration.config` — per-**instance** user/instance configuration. +- `Credential` — auth for a given `Entity` (rolls up to an `Integration` via relationship), + injected into the API-class instance for the `Requester` base-class fetch. Per-instance + authentication only. + +So there is currently **no first-class home** for app-level, per-API-module credentials/config — +they get dumped into the flat platform env, which is exactly what overloads. + +## Decision — a tiered model + +Configuration/secrets sort into **three scopes**, each crossed with **{config, secret}**: + +| Scope | Config | Secret | Where today | +|---|---|---|---| +| **1. Platform / app-wide** (per env) | infra, framework behavior, feature/dev flags | platform secrets | build-time `environment` → env; Secrets Manager → env (needs manual `SECRET_ARN` + Parameters&Secrets extension-layer wiring); SSM (draft) | +| **2. App-level per-API-module** (per env, shared across all connections) | module settings/scopes | **module client id/secret** | ❌ **no home — dumped into flat env (the overload)** | +| **3. Per-connection** | `Integration.config` | `Credential.data` (field-encrypted) | ✅ exists; injected on demand | + +> **Terminology:** tier 3 is **per-connection** — a `Credential` per `Entity`, i.e. the tokens a +> connection holds against an external service. The common case is a **specific end user** who +> authorizes *their* account (`User → Integration → Entity → Credential`) — what Jane gets when she +> clicks "Connect HubSpot." A second case is an **app-owner–owned global connection** (ADR-024 Global +> Entities): an admin authorizes once at deploy, the `Credential`/`Entity` carries `userId: null`, and +> every end user shares that single connection to the external service. Both are tier 3 — connection +> credentials in the DB, field-encrypted — and differ only in ownership (per-end-user vs global). +> Not to be confused with **deployment tenancy** (one Frigg deployment per adopter customer), a +> separate axis. Tier 2 is the **app-level** OAuth *application* credential (one per module per +> environment, shared by all connections) — e.g. the HubSpot `client_id`/`secret` you register once. + +**Invariant (keep):** tier-3 (per-connection) secrets — end-user *and* global — are **never** promoted +to `process.env`; they're decrypted and injected into the module instance on demand. This isolation +already holds and must stay. + +**Two planes.** Every tier has a **management plane** (where values are authored — the admin API / +CLI / UI, ADR-040) and a **runtime plane** (where the app reads them). Any provider (ADR-039) can +serve one or both: e.g. an adopter can make **1Password the system of record** for tiers 1–2 while +the runtime reads from a cloud store (materialized) or from 1Password directly. Tier 3 is authored at +**connect time** — by the end user (per-connection) or by an admin once (global entities, ADR-024) — +not through the platform-env pipeline, so it lives in the DB by default. + +``` + SCOPE MANAGEMENT PLANE STORE (routing map, ADR-040/039) RUNTIME PLANE + admin API / CLI / UI + Tier 1 platform ──write──► SSM / Secrets / 1Password / local ──► process.env (every function) + Tier 2 app-module ──write──► ModuleCredential/Config | provider ──► env of functions running + that module (scoped) + Tier 3 per-conn. (written by running app) DB Credential / Integration.config ──► injected on demand, + never in process.env +``` + +**The new concept is tier 2** — an app-level, per-module credential/config store, distinct from the +instance `Credential`/`config`. It is realized by **mirroring the existing models**: new +`ModuleCredential` / `ModuleConfig` tables/collections, scoped by **app + environment + module** (not +by `Entity`), secrets field-encrypted via the existing registry. (The rejected alternative — repurpose +`Credential`/config with a scope/reference discriminator — is in *Alternatives Considered*.) + +Either way: field-encrypt the secrets, and **inject per module at build/instantiation** rather than +flattening them into the global env — which both removes the overload and scopes each module to only +its own credentials. + +### Maturation path (free & fast → mature) — a hard requirement + +No adopter should pay for a tier they don't use; each level is opt-in and backward compatible: + +- **L0 — day one, free/fast:** build-time env via `appDefinition.environment` from the pipeline. Zero + infra. (What exists today.) +- **L1:** platform secrets via a provider store (Secrets Manager / SSM / provider), opt-in. +- **L2:** structured **app-level module credentials** (tier 2) — solves env overload — DB-backed or + provider-namespaced, managed via the admin API / CLI (ADR-040). +- **L3:** multi-provider (GCP / Azure / 1Password / Vault, ADR-039), **per-function scoping** + (least privilege), and a hosted management GUI. + +The adopter-facing narrative of this journey — plus a cross-pattern worked example — is **ADR-041**. + +### Variable scoping — deliver only what a function needs + +**Principle: a variable reaches only the bundled functions that need it.** Three scoping levels, +which Frigg can **derive from the app definition's integration→module graph** (it already knows which +integration uses which modules — no hand-authored manifest): + +- **Global** — `DATABASE_URL`, `KMS_KEY_ARN`, `STAGE`, framework-behavior settings → every function. +- **Module-scoped** — tier-2 creds like `HUBSPOT_CLIENT_ID`/`SECRET`/scopes → **only functions that + run the HubSpot module.** The Asana function never sees them *unless* it also runs HubSpot — the + rule is **usage-based, not name-based**, so the graph handles the "adopter runs HubSpot through + their Asana app" case correctly. +- **Per-connection** — tier-3 → never in env; fetched on demand for the specific connection. + +This maps onto the two runtime modes (ADR-039): +- **`materialized`** → each function's env = `global ∪ creds(modules it serves)` (a per-function env + manifest). +- **`direct`** → each function's role/identity is scoped to `global ∪ module-cred paths for modules + it serves`; modules pull creds at instantiation — which also **eliminates env-overload** (nothing + sits in env). + +**Boundary rule — derive by default, annotate the edge cases.** Tier-2 module creds are derived from +the module's auth definition + the graph (no annotation); per-connection is inherently distinct; +everything else defaults to platform-global. The app definition can override scope for +adopter-specific values the graph can't infer: + +```js +environment: { + DATABASE_URL: true, // platform (global) — inferred + MY_HUBSPOT_ONLY_FLAG: { scope: 'hubspot' }, // override: only functions running the hubspot module +} +// HUBSPOT_CLIENT_ID/SECRET/scopes (tier-2) need no annotation — derived from the module + graph. +``` + +**Confirmed against the code — the granularity is already there.** The infra builder emits +**per-integration functions**: `integration-builder.js` creates, per integration, an HTTP handler +(`functions[integrationName]`), a webhook handler (`{name}Webhook`), a queue worker, and one function +per extension binding (`{name}__{binding}`) — and packaging is **`package: { individually: true }`** +(`base-definition-factory.js`), so each is a discrete, separately-packaged Lambda. Per-function +env/IAM scoping therefore needs **no bundling restructure**. + +Env is global today only by **wiring**: the composer does +`Object.assign(provider.environment, merged.environment)` (`infrastructure-composer.js`), dumping all +vars onto the shared provider block. The fix (`materialized` mode) is to emit +`functions[name].environment` = `global ∪ creds(that integration's modules)` from the +integration→module graph instead of onto the global block; shared/non-integration functions (auth, +health, reporting, db-migrate) get `global` only. `direct` mode remains the alternative that keeps +creds out of env entirely. + +## Consequences + +### Positive +- Names and separates the four concerns currently mashed into one env list. +- Gives app-level module credentials a real home → directly kills env overload at scale. +- Preserves the (already-true) instance-secret isolation invariant. +- Free/fast default with an opt-in maturation path — no day-one tax. + +### Negative +- Introduces a new storage tier (tier 2) to design, build, and migrate onto. +- Requires reconciling three overlapping platform-env mechanisms (build-time, Secrets Manager, SSM). + +### Neutral +- Establishes scope × sensitivity as the vocabulary the CLI/API and docs are organized around. + +### Resolved so far +- **Tier-2 storage: mirror models.** New `ModuleCredential` / `ModuleConfig` + tables/collections scoped by **app + environment + module** (not by `Entity`), secrets + field-encrypted via the existing registry. Chosen for clarity/cleanliness over overloading + `Credential`/`config`. +- **Runtime modes:** support **both** `materialized` and `direct`; **`materialized` is the default** + for external managers on serverless (avoids a bootstrap token in functions). (ADR-039.) +- **Per-connection creds:** **DB by default.** A pluggable **non-Frigg credential source** (resolve + tier-3 from an adopter's own store) is a considered, overridable **extension point — deferred, not + built now.** +- **Function scoping is feasible now** (confirmed in code): functions are already per-integration and + `individually` packaged, so `materialized` per-function env/IAM is a wiring change (emit per-function + env from the graph), not a bundling restructure. +- **Terminology:** the instance tier is **per-connection** (not "tenant"). +- **Scope boundary:** **derive by default, annotate the edge cases** — tier-2 from module + graph, + per-connection inherently distinct, everything else platform-global; app definition can override a + variable's scope. +- **Precedence:** tiers are separate namespaces (most-specific wins: per-connection > app-module > + platform); the routing map fixes **one backend per (tier, env)** so there's no in-tier collision by + construction; genuine overlaps → provider/routing-map source wins, local overrides for local env, + deploy validation **warns** (opt-in strict mode fails). + +## Open questions (implementation detail, non-blocking) +- Exact **annotation syntax** for scope overrides in the app definition. +- Whether collision detection defaults to **warn** or **strict**. + +## Alternatives Considered +- **Keep the single flat env list.** Rejected: it is exactly what produces env overload at 10+ + modules and blends four separable concerns with no distinction. +- **Tier-2 by repurposing existing models.** App/definition-level rows in `Credential`/`config` + distinguished from instance rows by a scope/reference discriminator. Rejected in favor of mirrored + `ModuleCredential`/`ModuleConfig` models — a dedicated table reads more clearly and avoids + overloading instance semantics. +- **Only `direct` runtime mode.** Rejected: `direct` needs a bootstrap token in platform env for + external managers; `materialized` avoids it, so both are supported with `materialized` as default + (ADR-039). + +## Related +- [ADR-039: Secrets & Config Provider Plugin](./039-secrets-config-provider-plugin.md) +- [ADR-040: Variable & Secret Management (Admin API / CLI / GUI)](./040-variable-secret-management.md) +- [ADR-041: Configuration & Secrets — Docs & Adopter Maturation](./041-configuration-secrets-docs-and-maturation.md) +- [ADR-016: Plugins](./016-plugins.md) (provider/database/encryption/queue/scheduler plugin taxonomy), + [ADR-024: Global Entities](./024-global-entities.md) (admin-authored global tier-3 connections), + ADR-005 / ADR-010 (admin surface), field-level encryption registry + (`packages/core/database/encryption/`). diff --git a/docs/architecture-decisions/039-secrets-config-provider-plugin.md b/docs/architecture-decisions/039-secrets-config-provider-plugin.md new file mode 100644 index 000000000..89d16d8bb --- /dev/null +++ b/docs/architecture-decisions/039-secrets-config-provider-plugin.md @@ -0,0 +1,143 @@ +# ADR-039: Secrets & Config Provider Plugin + +**Status**: Proposed +**Date**: 2026-07-10 +**Deciders**: Sean Matthews + +> Proposed. The "where do values physically live and how are they read" half of the Configuration & +> Secrets work (ADR-038 is the model; ADR-040 is management; ADR-041 is docs & maturation). + +## Context + +The configuration/secrets tiers in ADR-038 need concrete backends, and the reality is multi-cloud: +a live **GCP** deployment, **Azure** on the radar, plus **1Password / Vault** as secret managers +adopters already use. Today only **AWS** is implemented (Secrets Manager → env is live; SSM runtime +loading is a draft; `SsmBuilder` grants IAM but nothing reads it), and there is **no functional +implementation** for GCP Secret Manager, Azure Key Vault, 1Password, or Vault — only commented-out +provider stubs. + +There is already a hexagonal seam — `CloudProviderAdapter` (a port with an AWS impl and GCP/Azure +stubs), and **ADR-016 (Plugins)** proposes required-with-defaults plugins with typed core interfaces +(`provider | database | encryption | queue | scheduler`) selected via `appDefinition.plugins`. The +problem is **cloud-agnostic; only the transport is provider-specific.** + +## Decision + +**Extend the ADR-016 plugin taxonomy with a new `secrets` / `config` plugin type** — this is a +taxonomy extension (a new core interface added to the five existing types), after which adapters +follow ADR-016's no-core-change rule: one typed core interface, many adapters. The framework never +imports a vendor SDK directly; adopters select an adapter in the app definition. + +```js +// appDefinition (illustrative) +plugins: { + secrets: { provider: 'aws' }, // 'aws' | 'gcp' | 'azure' | 'onepassword' | 'vault' | 'database' +} +``` + +**Port (illustrative) — read *and* write, so any provider can be system-of-record and/or runtime source:** +```js +class SecretsConfigProvider { // Port — ADR-016 plugin interface + async resolve(keysOrPrefix, { scope, env }) {} // runtime read → { KEY: value } + async write(key, value, { scope, env, secret }) {} // management-plane write (ADR-040) + runtimeMode() {} // 'materialized' | 'direct' + refreshPolicy() {} // e.g. { ttlSeconds: 300 } +} +``` + +**Runtime mode (decided): support both, `materialized` is the default for external managers.** +- **`materialized`** — values are synced into the function's runtime store (cloud env/SSM/Secrets or + the DB tier) at write/deploy; the function never calls the external manager at runtime. Default for + 1Password/Vault on serverless, because `direct` needs a **bootstrap secret** (a Service-Account + token) living in platform env — chicken-and-egg — whereas cloud-native stores authorize via the + function's IAM role with no stored token. +- **`direct`** — the function reads from the provider at cold start (Connect / Service Account / SDK). + Available for adopters who want the store to *literally be* 1Password/Vault at runtime. + +Mode also selects how **variable scoping** (ADR-038) is realized: `materialized` → per-function env +manifest (`global ∪ creds(modules the function serves)`); `direct` → per-function IAM scoped to those +paths, creds pulled at module instantiation. + +``` + write (ADR-040) runtime read + │ ▲ + ▼ │ + ┌──────────────────── SecretsConfigProvider (port) ──────────────────┐ + │ resolve() write() runtimeMode() refreshPolicy() │ + └───┬──────────┬──────────┬───────────┬──────────────┬───────────────┘ + aws gcp azure 1password database + (SSM+SM) (Secret Mgr)(Key Vault)(Connect/SA) (ModuleCredential / Credential) + + materialized (default, external mgrs): write ─► sync into runtime store ─► process.env + direct: function ─► provider at cold start + (needs bootstrap token; cloud-native uses IAM) +``` + +- **AWS adapter** = the existing SSM Parameter Store + Secrets Manager work, consolidated. The draft + runtime loader collapses into **one** core loader behind the port (`parametersToEnv` / + `secretsToEnv` via the Parameters & Secrets Lambda extension); `SsmBuilder` becomes its IAM half. + This folds in the SSM runtime-loading draft (branch `feature/finish-ssm-based-env-management`) + rather than shipping it as a separate ADR. +- **GCP** (Secret Manager), **Azure** (Key Vault) adapters — turn the current stubs into real impls. +- **`database` adapter** — backs ADR-038's DB tiers (app-level module creds, instance + `Credential`/`config`); the same port, a DB transport. +- **`onepassword` / `vault`** — see below. +- **Transport is adapter-internal** (Lambda extension vs SDK vs platform reference vs DB query); the + port stays cloud-neutral. Refresh/caching is a **port policy** (per-invocation with TTL — on AWS the + TTL cache is provided by the Parameters & Secrets Lambda extension that `secrets-to-env` reads + through — so freshness/rotation works without redeploy). + +### 1Password / Vault +Two modes, both behind the same interface: +- **Source-of-truth sync (default):** secrets live in 1Password/Vault; at **deploy** the CLI/API + resolves `op://` / `vault kv` references and syncs them into the runtime provider store — runtime + stays cloud-native and fast. Answers "is it just their toolchain?" → largely **yes** for authoring. +- **Runtime adapter (optional):** resolve at runtime via 1Password Connect / Service Accounts or + Vault agent. More moving parts; only if the store should *be* 1Password/Vault. + +## Consequences + +### Positive +- One interface unlocks AWS/GCP/Azure/1Password/Vault + DB without further core changes once the type + is added (the ADR-016 promise) — adapters need no core edits. +- Collapses the three competing AWS env mechanisms into a single adapter. +- Runtime stays cloud-native even when the source of truth is 1Password/Vault. + +### Negative +- Real adapter work (GCP/Azure stubs → impls; 1Password/Vault new). +- The Lambda-extension transport has no clean cross-cloud analog — the port must not leak it. + +### Neutral +- Establishes provider selection in `appDefinition.plugins` alongside database/encryption/etc. + +### Resolved +- **Runtime mode:** support both; **`materialized` default** for external managers (see above). +- **1Password/Vault:** both modes (source-of-truth-sync **and** runtime adapter); sync is the default. +- **Per-connection (tier-3):** **DB by default.** A **pluggable non-Frigg credential source** — a + `credentialSource` adapter so an adopter can resolve tier-3 from their own store — is a considered, + overridable extension point, **deferred, not built now.** + +- **Precedence (resolved):** the routing map (ADR-040) fixes **one backend per (tier, env)** → no + in-tier collision by construction. For genuine overlaps (e.g. legacy build-time env vs the + routing-map backend), the **provider/routing-map source wins**; the **local** provider overrides + for the local env only; deploy validation **warns** (opt-in strict mode fails). + +## Open questions +- Layer/extension **default-on vs opt-in** (minor; with per-function scoping confirmed, the extension + attaches only to functions that need `direct` reads). + +## Alternatives Considered +- **Keep AWS-only and hand-roll other clouds per adopter.** Rejected: GCP is already live and Azure is + on the radar; a per-adopter fork multiplies maintenance and contradicts the cloud-agnostic reality. +- **A separate top-level plugin category outside the ADR-016 taxonomy.** Rejected: secrets/config fit + the existing typed-interface-plus-adapters model; a parallel mechanism would fragment plugin + selection (`appDefinition.plugins`). +- **`direct` runtime reads as the only mode.** Rejected: needs a bootstrap token in platform env for + external managers; `materialized` is the default and `direct` remains available. + +## Related +- [ADR-038: Configuration & Secrets — Model & Tiers](./038-configuration-and-secrets-model.md) +- [ADR-040: Variable & Secret Management](./040-variable-secret-management.md) +- [ADR-041: Configuration & Secrets — Docs & Adopter Maturation](./041-configuration-secrets-docs-and-maturation.md) +- [ADR-016: Plugins](./016-plugins.md) (plugin taxonomy this extends); `CloudProviderAdapter` port + + AWS adapter; `secrets-to-env.js`; the SSM draft on `feature/finish-ssm-based-env-management`. diff --git a/docs/architecture-decisions/040-variable-secret-management.md b/docs/architecture-decisions/040-variable-secret-management.md new file mode 100644 index 000000000..05079372a --- /dev/null +++ b/docs/architecture-decisions/040-variable-secret-management.md @@ -0,0 +1,111 @@ +# ADR-040: Variable & Secret Management — Admin API, CLI, GUI + +**Status**: Proposed +**Date**: 2026-07-10 +**Deciders**: Sean Matthews + +> Proposed. The "how do humans/tools manage values across all tiers" half of the Configuration & +> Secrets work (ADR-038 is the model; ADR-039 is the storage/provider port; ADR-041 is docs & +> maturation). + +## Context + +Even with the tier model (ADR-038) and provider port (ADR-039), adopters need a way to *manage* +variables and secrets. Today there is **no `frigg secrets` / `frigg env` command**, no unified +surface, and management is scattered: hand-edited pipeline env, `setup-gh-env-secrets.sh` scripts, a +local-only `.env` editor in the management UI. We must not solve this by putting routing logic in the +CLI (it would diverge from any future GUI). + +## Decision + +**The management interface is a set of admin API routes; every front-end is a thin client of them.** + +- **Admin API is the one brain.** Routes (admin-authed, in the same admin-operation surface family as + ADR-005 / ADR-010; this ADR introduces its own `/api/v2/admin/variables` namespace) + accept a value plus its **category / scope / sensitivity**, and the **API decides where to store + it** based on the Frigg base app config/definition (which providers/tiers are configured per + ADR-038/039) and the target environment. Storage routing lives here, once. +- **The CLI is a client of those routes** — and, because they're plain admin API, the same calls are + **curl-able** and a **hosted GUI/UI** can drive all layers of variables and secrets through the + identical API. One API, many front-ends, no duplicated logic. +- **Smart routing** = a function of (sensitivity: config vs secret) × (scope: platform / + app-level-module / per-connection) × (environment) × (configured provider). The caller states + intent; the API places it: platform → provider store; app-level module cred → tier-2 store + (DB/provider); per-connection → `Integration.config` / `Credential`. +- **Routing map (per tier × environment → backend).** The placement rules live in a declared map so + the same intent routes differently per env, and **"unified 1Password" = set every tier's backend to + `1password`** (persona: "manage all of it in one place"). Mixed setups are just a different map. + ```js + // app definition (illustrative) + secretsRouting: { + prod: { platform: '1password', appModule: '1password', perConnection: 'database' }, + local: { platform: 'local', appModule: 'local', perConnection: 'database' }, + } + ``` +- **Local parity.** A first-class **`local` provider** (gitignored file / docker DB) is the default + backend for local envs, so `frigg start` + the CLI/UI work with **zero cloud**. Because backend is + per-environment, a dev who lives in 1Password can point `local` at it too (via `op`), ideally + `materialized` so there's no runtime dependency. +- **Scoping is emitted, not manual.** On deploy/build the API + infra derive each function's variable + set / IAM from the integration→module graph (ADR-038) — the management layer never hand-maintains + per-function manifests. + +``` + CLI curl GUI / UI (thin clients — no routing logic) + └─────────┼──────────┘ + ▼ + ┌──────────────── Admin API ─────────────────┐ + │ reads routing map (tier × env → backend); │ + │ places / resolves each value once │ + └───┬──────────────┬─────────────────┬────────┘ + platform app-module per-connection + provider store ModuleCredential/ DB Credential / + (SSM/Secrets/ provider Integration.config + 1Password/local) +``` + +**Illustrative CLI (thin wrappers over admin routes):** +``` +frigg config set FEATURE_X=on --stage prod # platform config → SSM / App Config +frigg secret set DATABASE_URL --stage prod # platform secret → Secrets Manager / KV / 1Password +frigg module cred set hubspot --client-id … --secret … --scopes … --stage prod # app-level module → tier-2 store +frigg integration config set key=val # instance config → Integration.config (DB) +``` +``` +GET/PUT /api/v2/admin/variables # curl or GUI hit the same routes the CLI does +``` + +## Consequences + +### Positive +- One place decides storage routing → CLI, curl, and a future GUI stay consistent by construction. +- Adopters manage all tiers through a uniform surface; a hosted UI becomes "just another client." +- Reuses the established admin-auth/admin-operation surface rather than a new mechanism. + +### Negative +- Requires admin write routes that mutate real secret stores — needs careful auth, redaction, audit. +- Routing rules must be well-specified so `set` is predictable. + +### Neutral +- Positions variable/secret management as part of Frigg's admin API, alongside reporting/scripts. + +## Open questions +- **Routing:** inferred (from sensitivity/scope) vs explicit subcommands/flags? +- **Auth:** reuse the admin API key (per ADR-010, which folded the earlier separate reporting key into + one admin key), or mint a dedicated management credential given these routes mutate secret stores? +- **Read-back & redaction:** can values be read back (never secrets in plaintext?), and what's audited? +- Where the **routing policy** itself is declared — app definition vs API config. + +## Alternatives Considered +- **Put routing logic in the CLI.** Rejected explicitly: it would diverge from any future GUI; the + routing decision must live once, in the admin API, so every front-end stays consistent. +- **A dedicated non-admin secrets service/endpoint family.** Rejected: reuses none of the established + admin-auth/admin-operation surface and adds a second security perimeter to harden. +- **Per-tier bespoke commands with no routing map.** Rejected: the routing map is what makes "unified + 1Password" and mixed per-env setups a config change rather than new code. + +## Related +- [ADR-038: Configuration & Secrets — Model & Tiers](./038-configuration-and-secrets-model.md) +- [ADR-039: Secrets & Config Provider Plugin](./039-secrets-config-provider-plugin.md) +- [ADR-041: Configuration & Secrets — Docs & Adopter Maturation](./041-configuration-secrets-docs-and-maturation.md) +- ADR-005 (Admin Script Runner — admin auth/surface), ADR-010 (admin operations / admin API key). diff --git a/docs/architecture-decisions/041-configuration-secrets-docs-and-maturation.md b/docs/architecture-decisions/041-configuration-secrets-docs-and-maturation.md new file mode 100644 index 000000000..d18e7a4be --- /dev/null +++ b/docs/architecture-decisions/041-configuration-secrets-docs-and-maturation.md @@ -0,0 +1,100 @@ +# ADR-041: Configuration & Secrets — Docs & Adopter Maturation + +**Status**: Proposed +**Date**: 2026-07-10 +**Deciders**: Sean Matthews + +> Proposed. The enablement half of the Configuration & Secrets work (model = ADR-038, provider = +> ADR-039, management = ADR-040). Covers how we document it for adopters and how a Frigg app is +> expected to grow over time. The full cross-pattern worked example is a **post-implementation doc +> deliverable**; an illustrative target-state version is included below. + +## Context + +The model (ADR-038–029) is only useful if adopters can (a) find a **free, fast on-ramp**, (b) +understand **when to reach for which tier/provider**, and (c) **grow** without rework. Today the docs +describe a single flat env list; there is no maturation story and no guidance separating the four +concerns that get mashed together (infra, framework behavior, dev/feature flags, API-module app +credentials). Adopters therefore hit "env overload" with no documented path out. + +## Decision + +Ship a dedicated **"Configuration & Secrets" guide** in the Frigg docs, organized around a +maturation journey and a cross-pattern worked example. + +### 1. The guide (pages / questions it must answer) +- **Mental model** — the tiers (platform / app-level module / per-connection) × {config, secret}, the + two planes (management vs runtime), and the isolation invariant. *(from ADR-038)* +- **When to reach for which** — the decision rule (scope first, then sensitivity). +- **Per-environment setup** — how to set values per stage; the routing map. *(ADR-040)* +- **"Only what a function needs"** — scoping derived from the integration→module graph. *(ADR-038)* +- **On-demand vs env** — why per-connection creds are always on-demand. *(ADR-038)* +- **Choosing a provider** — AWS / GCP / Azure / 1Password / Vault / local, and materialized vs + direct. *(ADR-039)* +- **Managing values** — the `frigg` CLI / admin API / GUI, and the routing map. *(ADR-040)* +- **Local development** — the local provider; zero-cloud `frigg start`. *(ADR-040)* +- **Migrating** — moving from a flat pipeline env to structured tiers without downtime. + +### 2. The adopter maturation journey +Document the growth path explicitly (tied to ADR-038's L0–L3) — the headline is **"start with all +globals set manually in your pipeline; graduate to managed config as you scale."** + +| Level | Trigger | What the adopter does | Where values live | +|---|---|---|---| +| **L0 — Manual (free/fast)** | Day one | Set global envs in the deploy pipeline (`appDefinition.environment`) | Pipeline → `process.env` (global) | +| **L1 — Platform secrets** | First real secret / rotation need | Move secrets to a provider store | Secrets Manager / SSM / provider | +| **L2 — Structured module creds** | **Env overload** (10+ modules × OAuth) | Move app-level module creds to the tier-2 store; scope per function | `ModuleCredential`/`ModuleConfig` (or provider) | +| **L3 — Managed / multi-provider** | Fleet / compliance / DX | Multi-provider (GCP/Azure/1Password/Vault), per-function least-privilege, GUI | Chosen provider(s) via the routing map | + +Each level is **opt-in and backward-compatible** — no adopter pays for a tier they don't use, and L0 +keeps working forever. + +### 3. Cross-pattern worked example +Provide **the same env set shown across adopter patterns** so an adopter can locate themselves. This +is a **post-implementation deliverable** (the real doc lands once ADR-038–029 ship); the illustrative +target-state below captures intent. + +**The shared env set:** `DATABASE_URL` (platform secret), `FEATURE_X` (platform/dev flag), +`HUBSPOT_CLIENT_ID` + `HUBSPOT_CLIENT_SECRET` (tier-2 app-level module cred), a HubSpot **access +token** (tier-3 per-connection). + +| Value | A: Day-one (L0) | B: Growth (L2, AWS) | C: 1Password-unified | D: Multi-cloud (GCP) | +|---|---|---|---|---| +| `DATABASE_URL` | pipeline env → global | Secrets Manager → global env | 1Password → materialized → global env | GCP Secret Manager → global env | +| `FEATURE_X` | pipeline env → global | SSM → global env | 1Password → materialized | GCP Runtime Config → global | +| `HUBSPOT_CLIENT_ID/SECRET` | pipeline env → **all** functions | tier-2 store → **only** HubSpot functions | 1Password (tier-2) → HubSpot functions | GCP SM (tier-2) → HubSpot functions | +| HubSpot **access token** (per-conn.) | DB `Credential`, on demand | DB `Credential`, on demand | DB `Credential`, on demand | DB `Credential`, on demand | +| Routing map | — (all pipeline) | `{platform: aws, appModule: aws, perConnection: database}` | `{platform: 1password, appModule: 1password, perConnection: database}` | `{platform: gcp, appModule: gcp, perConnection: database}` | + +Two things the example makes obvious: **per-connection creds stay in the DB across every provider +pattern above** (the isolation invariant holds regardless of which cloud/secret store backs tiers 1–2) +— note this is the provider axis; whether a connection is per-end-user or an admin-authored **global +entity** (ADR-024) is a separate adoption choice, and both still live in the DB — and **moving from +A→B→C is a routing-map + storage change, not an app-code change.** + +## Consequences + +### Positive +- Gives adopters a legible on-ramp and a growth path; directly addresses the env-overload wall. +- The worked example lets adopters self-locate and see that migration is config, not code. + +### Negative +- Docs must track ADR-038–029 as they're implemented (drift risk until they ship). + +### Neutral +- Establishes the maturation levels as the shared vocabulary for docs, CLI help, and onboarding. + +## Alternatives Considered +- **Fold this guidance into ADR-038 as a section rather than its own ADR.** Rejected: the enablement/ + maturation story and the cross-pattern worked example are substantial and adopter-facing; a dedicated + ADR keeps ADR-038 focused on the model and gives docs a single anchor to track. +- **Write the full cross-pattern worked example now.** Deferred: the faithful version depends on the + shipped shapes of ADR-038–029, so an illustrative target-state table is included and the complete + example is called out as a post-implementation deliverable. + +## Related +- [ADR-038](./038-configuration-and-secrets-model.md) (model & maturation levels), + [ADR-039](./039-secrets-config-provider-plugin.md) (providers), + [ADR-040](./040-variable-secret-management.md) (management). +- [ADR-024: Global Entities](./024-global-entities.md) (per-connection vs global adoption choice). +- Frigg docs site (`docs/`) — target home for the guide. diff --git a/docs/architecture-decisions/README.md b/docs/architecture-decisions/README.md index 8e9590398..d70af5412 100644 --- a/docs/architecture-decisions/README.md +++ b/docs/architecture-decisions/README.md @@ -22,10 +22,10 @@ An ADR documents a significant architectural decision made in the project, inclu | [003](./003-runtime-state-only.md) | Runtime State Only for Management GUI | Accepted | 2025-01-25 | | [004](./004-migration-tool-design.md) | Project Structure Migration Tool | Proposed | 2025-01-25 | | [005](./005-admin-script-runner.md) | Admin Script Runner Service | Accepted | 2025-12-10 | -| [006](./006-integration-router-v2.md) | Integration Router v2 | Accepted | 2025-12-14 | +| [006](./006-integration-router-v2.md) | Integration Router v2 Restructuring | Accepted | 2025-12-14 | | [007](./007-management-ui-architecture.md) | Management UI Architecture | Accepted | 2025-12-14 | -| [008](./008-frigg-cli-start-command.md) | Frigg CLI Start Command | Accepted | 2025-12-14 | -| [009](./009-e2e-test-package.md) | E2E Test Package | Accepted | 2025-12-15 | +| [008](./008-frigg-cli-start-command.md) | Frigg CLI Start Command Architecture | Accepted | 2025-12-14 | +| [009](./009-e2e-test-package.md) | E2E Test Package Architecture | Accepted | 2025-12-15 | | [010](./010-reporting-as-admin-operation.md) | Reporting as an Admin Operation | Accepted | 2026-07-03 | | [011](./011-integration-telemetry-and-usage-tracking.md) | Integration Telemetry, Eventing & Feature-Usage Tracking | Accepted | 2026-07-03 | | [012](./012-database-schema-migrations.md) | Database Schema Migrations | Proposed | 2026-07-04 | @@ -34,7 +34,7 @@ An ADR documents a significant architectural decision made in the project, inclu | [015](./015-extensions-taxonomy.md) | Extensions Taxonomy | Proposed | 2026-06-09 | | [016](./016-plugins.md) | Plugins | Proposed | 2026-06-09 | | [017](./017-core-extensions.md) | Core Extensions | Proposed | 2026-06-09 | -| [018](./018-integration-extensions.md) | Integration Extensions | Implemented | 2026-06-09 | +| [018](./018-integration-extensions.md) | Integration Extensions | Implemented ([PR #590](https://github.com/friggframework/frigg/pull/590) and [PR #596](https://github.com/friggframework/frigg/pull/596)). Authoritative quick-start: [`packages/core/integrations/EXTENSIONS.md`](../../packages/core/integrations/EXTENSIONS.md). | 2026-06-09 (decision ratified retroactively) | | [019](./019-api-module-extensions.md) | API Module Extensions | Proposed | 2026-06-09 | | [020](./020-capabilities.md) | Capabilities | Proposed | 2026-06-09 | | [021](./021-ontology.md) | Ontology | Proposed | 2026-06-09 | @@ -44,7 +44,20 @@ An ADR documents a significant architectural decision made in the project, inclu | [025](./025-agent-harness.md) | Agent Harness | Proposed | 2026-06-09 | | [026](./026-evals.md) | Evals | Proposed | 2026-06-09 | | [027](./027-ssm-parameter-offload-and-env-scoping.md) | SSM Parameter Offload and Per-Function Environment Scoping | Accepted | 2026-07-10 | +| [028](./028-multi-provider-support.md) | Multi-Provider Support | Accepted | 2026-03-02 | +| [029](./029-decouple-aws-from-core.md) | Decouple AWS SDK Dependencies from @friggframework/core | Accepted | 2026-03-03 | +| [030](./030-integration-versioning.md) | Integration Versioning | Proposed | 2026-09-27 | | [031](./031-concurrent-oauth-credential-refresh.md) | Concurrent OAuth Credential Refresh Across Lambda Invocations | Proposed | 2026-08-11 | +| [032](./032-integration-deletion-cleanup.md) | Integration Deletion Data Cleanup | Proposed | 2026-08-20 | +| [033](./033-aurora-serverless-v2-scale-to-zero-and-nat-free-connectivity.md) | Aurora Serverless v2 scale-to-zero + NAT-free Lambda connectivity | Proposed | 2026-08-20 | +| [034](./034-api-key-login-auth-mode.md) | API-Key Login Auth Mode | Proposed | 2026-08-20 | +| [035](./035-app-init.md) | App Init | Proposed | 2026-09-27 | +| [036](./036-skills.md) | Skills | Proposed | 2026-09-27 | +| [037](./037-agent-pipeline.md) | Agent Pipeline | Proposed | 2026-09-27 | +| [038](./038-configuration-and-secrets-model.md) | Configuration & Secrets — Model & Tiers | Proposed | 2026-07-10 | +| [039](./039-secrets-config-provider-plugin.md) | Secrets & Config Provider Plugin | Proposed | 2026-07-10 | +| [040](./040-variable-secret-management.md) | Variable & Secret Management — Admin API, CLI, GUI | Proposed | 2026-07-10 | +| [041](./041-configuration-secrets-docs-and-maturation.md) | Configuration & Secrets — Docs & Adopter Maturation | Proposed | 2026-07-10 | ## Conventions diff --git a/website/roadmap/data/adrs.json b/website/roadmap/data/adrs.json index 4f9acacdf..6cf5a8a77 100644 --- a/website/roadmap/data/adrs.json +++ b/website/roadmap/data/adrs.json @@ -1,5 +1,5 @@ { - "count": 28, + "count": 41, "adrs": [ { "num": 1, @@ -54,7 +54,7 @@ { "num": 6, "id": "006", - "title": "Integration Router v2", + "title": "Integration Router v2 Restructuring", "status": "Accepted", "date": "2025-12-14", "theme": "Developer experience", @@ -74,7 +74,7 @@ { "num": 8, "id": "008", - "title": "Frigg CLI Start Command", + "title": "Frigg CLI Start Command Architecture", "status": "Accepted", "date": "2025-12-14", "theme": "Infra & deploy", @@ -84,7 +84,7 @@ { "num": 9, "id": "009", - "title": "E2E Test Package", + "title": "E2E Test Package Architecture", "status": "Accepted", "date": "2025-12-15", "theme": "Developer experience", @@ -175,8 +175,8 @@ "num": 18, "id": "018", "title": "Integration Extensions", - "status": "Implemented", - "date": "2026-06-09", + "status": "Implemented ([PR #590](https://github.com/friggframework/frigg/pull/590) and [PR #596](https://github.com/friggframework/frigg/pull/596)). Authoritative quick-start: [`packages/core/integrations/EXTENSIONS.md`](../../packages/core/integrations/EXTENSIONS.md).", + "date": "2026-06-09 (decision ratified retroactively)", "theme": "Extensions & plugins", "summary": "Integration Extensions let an API module or shared library ship a bundle of routes, events, queues, and workers that an integration binds declaratively.", "url": "https://github.com/friggframework/frigg/blob/next/docs/architecture-decisions/018-integration-extensions.md" @@ -271,6 +271,36 @@ "summary": "Offloads large secrets to SSM and scopes env vars per function so Lambdas stay under the 4KB environment limit as the integration count grows.", "url": "https://github.com/friggframework/frigg/blob/next/docs/architecture-decisions/027-ssm-parameter-offload-and-env-scoping.md" }, + { + "num": 28, + "id": "028", + "title": "Multi-Provider Support", + "status": "Accepted", + "date": "2026-03-02", + "theme": "Infra & deploy", + "summary": "Introduces a provider plugin system so a single `provider` field switches the whole toolchain. Each provider ships as @friggframework/provider-{name} with adapters for deploy, config generation, queues, scheduling, secrets and handlers; Netlify is the first non-AWS target.", + "url": "https://github.com/friggframework/frigg/blob/next/docs/architecture-decisions/028-multi-provider-support.md" + }, + { + "num": 29, + "id": "029", + "title": "Decouple AWS SDK Dependencies from @friggframework/core", + "status": "Accepted", + "date": "2026-03-03", + "theme": "Infra & deploy", + "summary": "Extracts every AWS SDK dependency out of @friggframework/core into @friggframework/provider-aws, wiring adapters explicitly at the composition root. Breaking change; it is what lets a non-AWS bundle contain no AWS SDK code.", + "url": "https://github.com/friggframework/frigg/blob/next/docs/architecture-decisions/029-decouple-aws-from-core.md" + }, + { + "num": 30, + "id": "030", + "title": "Integration Versioning", + "status": "Proposed", + "date": "2026-09-27", + "theme": "Data & migrations", + "summary": "Opens the integration version contract — how versions are declared, compared, gated and made compatible — which ADR-013's record migrations key off. Exploratory: the decision is deliberately deferred pending research.", + "url": "https://github.com/friggframework/frigg/blob/next/docs/architecture-decisions/030-integration-versioning.md" + }, { "num": 31, "id": "031", @@ -280,6 +310,106 @@ "theme": "Data & migrations", "summary": "Deconflicts concurrent OAuth refreshes across Lambda invocations: a reactive database check before refresh and on 401 (adopt the winner's tokens, escalate only on proof of death), plus proactive scheduled refresh via admin scripts, and a fix for the silent SQS ack that hid the losses.", "url": "https://github.com/friggframework/frigg/blob/next/docs/architecture-decisions/031-concurrent-oauth-credential-refresh.md" + }, + { + "num": 32, + "id": "032", + "title": "Integration Deletion Data Cleanup", + "status": "Proposed", + "date": "2026-08-20", + "theme": "Data & migrations", + "summary": "Defines what must be cleaned up when an integration is deleted — which records cascade, which are retained for audit, and which admin operation performs the sweep.", + "url": "https://github.com/friggframework/frigg/blob/next/docs/architecture-decisions/032-integration-deletion-cleanup.md" + }, + { + "num": 33, + "id": "033", + "title": "Aurora Serverless v2 scale-to-zero + NAT-free Lambda connectivity", + "status": "Proposed", + "date": "2026-08-20", + "theme": "Infra & deploy", + "summary": "Runs Aurora Serverless v2 with scale-to-zero and reaches it from Lambda without a NAT gateway, cutting idle database and egress cost for low-traffic Frigg apps.", + "url": "https://github.com/friggframework/frigg/blob/next/docs/architecture-decisions/033-aurora-serverless-v2-scale-to-zero-and-nat-free-connectivity.md" + }, + { + "num": 34, + "id": "034", + "title": "API-Key Login Auth Mode", + "status": "Proposed", + "date": "2026-08-20", + "theme": "Developer experience", + "summary": "Adds an API-key login auth mode validated by the API module, so an integration can authenticate with a key rather than an OAuth round-trip.", + "url": "https://github.com/friggframework/frigg/blob/next/docs/architecture-decisions/034-api-key-login-auth-mode.md" + }, + { + "num": 35, + "id": "035", + "title": "App Init", + "status": "Proposed", + "date": "2026-09-27", + "theme": "Agent tooling", + "summary": "Defines what app init seeds into an adopter app — agent directories, ontology seed and conventions — as a starting point adopters grow over time.", + "url": "https://github.com/friggframework/frigg/blob/next/docs/architecture-decisions/035-app-init.md" + }, + { + "num": 36, + "id": "036", + "title": "Skills", + "status": "Proposed", + "date": "2026-09-27", + "theme": "Agent tooling", + "summary": "Ships .claude/skills/ alongside .claude/agents/ as the declarative, read-on-demand surface of knowledge, contracts and runbooks an agent loads explicitly when relevant.", + "url": "https://github.com/friggframework/frigg/blob/next/docs/architecture-decisions/036-skills.md" + }, + { + "num": 37, + "id": "037", + "title": "Agent Pipeline", + "status": "Proposed", + "date": "2026-09-27", + "theme": "Agent tooling", + "summary": "Describes the agent pipeline that composes app init, skills, capabilities and ontology into a repeatable run.", + "url": "https://github.com/friggframework/frigg/blob/next/docs/architecture-decisions/037-agent-pipeline.md" + }, + { + "num": 38, + "id": "038", + "title": "Configuration & Secrets — Model & Tiers", + "status": "Proposed", + "date": "2026-07-10", + "theme": "Extensions & plugins", + "summary": "Establishes the configuration and secrets model and its tiers, separating framework config, app config and per-connection credentials, and where each physically lives.", + "url": "https://github.com/friggframework/frigg/blob/next/docs/architecture-decisions/038-configuration-and-secrets-model.md" + }, + { + "num": 39, + "id": "039", + "title": "Secrets & Config Provider Plugin", + "status": "Proposed", + "date": "2026-07-10", + "theme": "Extensions & plugins", + "summary": "Extends the ADR-016 plugin taxonomy with a secrets/config provider plugin: one typed core port, many adapters, so a new backing store needs no core change.", + "url": "https://github.com/friggframework/frigg/blob/next/docs/architecture-decisions/039-secrets-config-provider-plugin.md" + }, + { + "num": 40, + "id": "040", + "title": "Variable & Secret Management — Admin API, CLI, GUI", + "status": "Proposed", + "date": "2026-07-10", + "theme": "Developer experience", + "summary": "Adds the admin API, CLI and GUI surface for managing variables and secrets across all tiers, reusing the admin API key from ADR-010.", + "url": "https://github.com/friggframework/frigg/blob/next/docs/architecture-decisions/040-variable-secret-management.md" + }, + { + "num": 41, + "id": "041", + "title": "Configuration & Secrets — Docs & Adopter Maturation", + "status": "Proposed", + "date": "2026-07-10", + "theme": "Developer experience", + "summary": "Covers how the configuration and secrets work is documented for adopters, and the maturation path a Frigg app follows from local env files to a managed store.", + "url": "https://github.com/friggframework/frigg/blob/next/docs/architecture-decisions/041-configuration-secrets-docs-and-maturation.md" } ] }