Self-bootstrapping startup: zero ordered setup commands

.env.example

@@ -1,12 +1,17 @@
-# openoms configuration. Copy to `.env`, edit, then run `make db-setup`.
+# openoms configuration. Copy to `.env`, edit, then just run the app — it
+# self-provisions on boot (roles, schema, ref-data) and syncs instruments in the
+# background. `make db-setup` still works for manual/ordered setup if you prefer.
 # =====================================================================
 
 # --- Postgres host ---
 DB_HOST=localhost
 DB_PORT=5432
 
-# --- Install-time admin (a Postgres superuser) — used by `make db-provision`/
-#     db-migrate/db-access/db-seed to create roles+db and apply schema. Not runtime. ---
+# --- Install-time admin (a Postgres superuser) — creates roles+db and applies
+#     schema/ref-data. Used by `make db-*` AND by the app's boot-time self-provision
+#     (see OMS_BOOTSTRAP). Not the runtime connection. If these + the role passwords
+#     below are incomplete, the app assumes the DB is already provisioned and skips
+#     bootstrap rather than failing. ---
 ADMIN_USER=postgres
 ADMIN_PASSWORD=postgres
 
@@ -28,15 +33,33 @@ OMS_BIND_ADDR=localhost:3001
 OMS_ADMIN_TOKEN=change-me-to-a-secure-random-token
 OMS_ADMIN_AUTH_ENABLED=false
 
+# --- Boot-time self-bootstrap (so a fresh checkout is just `run the app`) ---
+#   OMS_BOOTSTRAP        auto (default) runs provision/migrate/access/seed at boot as
+#                        the admin role; `off` skips it (infra provisioned elsewhere).
+#   OMS_SYNC_ON_BOOT     if-empty (default) syncs instruments from brokers with creds
+#                        when the catalog is empty; `never` disables auto-sync.
+#   OMS_SYNC_UNDERLYINGS comma-separated option underlyings to seed at boot (Alpaca),
+#                        e.g. SPY,QQQ. Empty = equities/spot only.
+#   OMS_DEV_IDENTITY     true creates a dev principal/account with a KNOWN api secret
+#                        (test-trader-key : test-secret) so a fresh install can place
+#                        an order immediately. PAPER/dev only — never in production.
+# OMS_BOOTSTRAP=auto
+# OMS_SYNC_ON_BOOT=if-empty
+# OMS_SYNC_UNDERLYINGS=
+# OMS_DEV_IDENTITY=false
+
+# A broker_connection is auto-created at boot for every broker whose creds are set
+# below — no manual step needed to make a configured broker routable.
+
 # --- Kafka ---
 KAFKA_BROKER=localhost:9092
 KAFKA_TOPIC=orders
 KAFKA_CLIENT_ID=rust-oms
 KAFKA_PROJECTOR_GROUP_ID=position-projector-v1
 
-# --- Optional: instrument seeding (`make sync-broker` = instruments + broker
-#     mapping). On-demand, NOT scheduled. Runs in-process, writes as DB_USER
-#     (oms_user). Data feeds need no seeding — each derives what it can price. ---
+# --- Broker creds. Present here, the app auto-syncs that broker's catalog at boot
+#     (see OMS_SYNC_ON_BOOT); also used by `make sync-broker`. Writes as oms_user.
+#     Data feeds need no seeding — each derives what it can price. ---
 # DATABENTO_API_KEY=
 ALPACA_ENV=PAPER
 ALPACA_PAPER_API_KEY=

cockpit/src/pages/Architecture.tsx

@@ -318,8 +318,11 @@ export function ArchitecturePage() {
           <Text c="dimmed" fz="sm" mb="sm" maw={680}>
             Two steps, and the order is a hard dependency rather than a convention. Each reads what the
             one before it wrote, and nothing ever points backwards — a data feed never creates an
-            instrument, and an instrument never creates a venue. <Code>make seed-live</Code> runs the
-            chain idempotently. Feeds are not part of it: they derive their mapping at startup.
+            instrument, and an instrument never creates a venue. Feeds are not part of it: they derive
+            their mapping at startup. You rarely run these by hand: the app self-provisions on boot
+            (roles, schema, ref-data via the admin role) and background-syncs instruments when the
+            catalog is empty — the make targets are the manual equivalent, kept as an escape hatch
+            (<Code>OMS_BOOTSTRAP=off</Code>).
           </Text>
           <Diagram chart={SEED} />
 

db/scripts/fixtures.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+#
+# Load the minimal no-creds fixture (SPY) so a fresh install has one tradeable
+# instrument without any broker/data-provider credentials. Idempotent — the fixture
+# upserts on natural keys (ON CONFLICT DO NOTHING).
+#
+# Extracted from the `db-fixtures` make recipe so boot-time bootstrap can orchestrate
+# it the same way as provision/migrate/access/seed. Needs an ADMIN role (the fixture
+# writes master tables owned by mdm_master).
+set -euo pipefail
+
+# Repo root is two levels up (db/scripts → db → repo); the fixture lives under the
+# top-level scripts/ dir, not db/scripts/.
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+cd "$ROOT"
+[[ -f .env ]] && { set -a; . ./.env; set +a; }
+
+: "${DB_HOST:?DB_HOST must be set}"
+: "${DB_PORT:?DB_PORT must be set}"
+: "${ADMIN_USER:?ADMIN_USER must be set}"
+: "${ADMIN_PASSWORD:?ADMIN_PASSWORD must be set}"
+ODS_DB="${ODS_DB:-ods}"
+
+echo "→ loading minimal fixture (SPY)"
+PGPASSWORD="$ADMIN_PASSWORD" psql -v ON_ERROR_STOP=1 \
+  -h "$DB_HOST" -p "$DB_PORT" -U "$ADMIN_USER" -d "$ODS_DB" \
+  -f scripts/fixtures/minimal_seed.sql
+
+echo "fixture loaded."

readme.md

@@ -24,25 +24,32 @@ git clone git@github.com:maxkuttner/openoms.git && cd openoms
 cp .env.example .env        # then edit passwords / bind addr
 ```
 
-## Setup
+## Run
 
-One-shot DB bootstrap — roles, schema, reference data, and a no-creds SPY fixture (needs the `ADMIN_*` superuser creds in `.env`):
+With the `ADMIN_*` superuser + role passwords set in `.env`, just start the app — it
+self-provisions on boot (roles, schema, reference data) and, if broker creds are
+present, syncs that broker's instrument catalog in the background. No ordered setup
+commands.
 
 ```sh
-make db-setup
+cargo run                  # OMS on OMS_BIND_ADDR (default localhost:3001)
 ```
 
-Optional live data (on-demand, needs vendor creds in `.env`):
+First boot on an empty database runs provision → migrate → seed before listening (a
+few seconds), then the server binds immediately while instruments populate in the
+background. Subsequent boots are near-instant no-ops. Set `OMS_SYNC_ON_BOOT=never` to
+skip auto-sync, or `OMS_BOOTSTRAP=off` when infrastructure is provisioned elsewhere.
 
-```sh
-make sync-broker BROKER=alpaca               # seed instruments + broker mapping from Alpaca (ALPACA_PAPER_*)
-make sync-broker BROKER=alpaca UNDERLYINGS=SPY,QQQ  # also seed those option chains
-```
+The SPY fixture seeds `alpaca-paper` + a test user (`test-trader-key` : `test-secret`),
+so you can place a paper order immediately. Admin webapp: `cd cockpit && npm install && npm run dev`.
 
-## Run
+### Manual setup (optional)
+
+The bootstrap just orchestrates the same idempotent make targets, if you'd rather run
+them yourself (or set `OMS_BOOTSTRAP=off`):
 
 ```sh
-cargo run                  # OMS on OMS_BIND_ADDR (default localhost:3001)
+make db-setup                                 # roles, schema, ref-data, SPY fixture
+make sync-broker BROKER=alpaca               # instruments + broker mapping from Alpaca
+make sync-broker BROKER=alpaca UNDERLYINGS=SPY,QQQ  # also seed those option chains
 ```
-
-The SPY fixture seeds `alpaca-paper` + a test user (`test-trader-key` : `test-secret`), so you can place a paper order immediately. Admin webapp: `cd cockpit && npm install && npm run dev`.

scripts/fixtures/dev_identity.sql

@@ -0,0 +1,48 @@
+-- Dev identity chain: a ready-to-trade principal so a fresh install can place a
+-- paper order over the API with no admin setup. Gated by OMS_DEV_IDENTITY — this
+-- creates a known API secret and is for PAPER/dev only, never a real deployment.
+-- Idempotent via natural keys. All in `oms`, so the app runs it as oms_user.
+--
+--   HTTP Basic auth:  test-trader-key : test-secret   (PAPER/dev only — not for prod)
+
+-- A broker connection the test account can route through. Real credentialed brokers
+-- also get their own connection auto-created (ensure_broker_connections); this one
+-- is self-contained so the dev chain works even with no broker creds set.
+INSERT INTO oms.broker_connection (code, broker_code, environment, status)
+VALUES ('alpaca-paper', 'ALPACA', 'PAPER', 'ACTIVE')
+ON CONFLICT (code) DO NOTHING;
+
+-- UUID ids are generated, then resolved by code for the FK references
+-- (principal/account/portfolio all have UNIQUE code).
+INSERT INTO oms.principal (id, code, principal_type, display_name, status)
+VALUES (gen_random_uuid(), 'test-trader', 'HUMAN', 'Test Trader', 'ACTIVE')
+ON CONFLICT (code) DO NOTHING;
+
+-- Custodial account on the alpaca-paper connection. external_account_ref is the
+-- broker's own account handle; Alpaca routes on the API key, so it's informational.
+INSERT INTO oms.account (id, code, broker_connection_code, external_account_ref, status)
+VALUES (gen_random_uuid(), 'test-account', 'alpaca-paper', 'PAPER', 'ACTIVE')
+ON CONFLICT (code) DO NOTHING;
+
+-- Portfolio whose default route is the test account, so submits need no account_id.
+INSERT INTO oms.portfolio (id, code, name, status, default_account_id)
+SELECT gen_random_uuid(), 'test-portfolio', 'Test Portfolio', 'ACTIVE', a.id
+FROM   oms.account a WHERE a.code = 'test-account'
+ON CONFLICT (code) DO NOTHING;
+
+-- Entitle the trader to trade the portfolio.
+INSERT INTO oms.principal_portfolio_grant
+    (id, principal_id, portfolio_id, can_trade, can_view, can_allocate)
+SELECT gen_random_uuid(), p.id, pf.id, true, true, false
+FROM   oms.principal p, oms.portfolio pf
+WHERE  p.code = 'test-trader' AND pf.code = 'test-portfolio'
+ON CONFLICT (principal_id, portfolio_id) DO NOTHING;
+
+-- API key for the trader. key_id 'test-trader-key', secret 'test-secret'.
+-- secret_hash is a precomputed bcrypt ($2y$, cost 12) of 'test-secret' — pgcrypto
+-- isn't available on the server, and the app's bcrypt::verify accepts $2y$.
+INSERT INTO oms.api_key (principal_id, key_id, secret_hash, name)
+SELECT p.id, 'test-trader-key',
+       '$2y$12$haJ5Dxm/IzRmKITzcrZ.sOvTyt2KCvS7KAEC2OvDaycL1TmojvuSm', 'test-key'
+FROM   oms.principal p WHERE p.code = 'test-trader'
+ON CONFLICT (key_id) DO NOTHING;

scripts/fixtures/minimal_seed.sql

@@ -1,9 +1,15 @@
--- openoms minimal seed: one tradeable instrument so a fresh install can place a
--- paper order without any data-provider creds. Idempotent (re-runnable).
+-- openoms minimal seed: one tradeable instrument so a fresh no-creds install has
+-- something in the catalog. Idempotent (re-runnable). Writes master tables in
+-- `public` (owned by mdm_master), so this runs as the admin role via fixtures.sh.
+--
+-- Routing config (broker_connection) and the test identity chain are NOT here — they
+-- live in `oms` and are created by the app at boot (see src/setup/bootstrap.rs:
+-- ensure_broker_connections + ensure_dev_identity). Keeping master-data seeding
+-- separate from oms-owned config is what lets each run under the right role.
 --
 -- Depends on `make db-seed` having loaded the FK targets (currency USD, venue ARCX).
 -- instrument.id is GENERATED ALWAYS AS IDENTITY, so we never hardcode it: insert by
--- the (symbol, venue) natural key, then resolve the id for the mappings.
+-- the (symbol, venue) natural key, then resolve the id for the mapping.
 
 INSERT INTO instrument
     (symbol, venue, currency, asset_class, instrument_class, name,
@@ -26,52 +32,4 @@ ON CONFLICT (instrument_id, broker_code) DO NOTHING;
 -- No feed mapping row: each feed derives what it can price from its own symbology
 -- at startup. Note that nothing prices SPY *equity* today — the Databento feed
 -- carries OPRA options only — so a position in it will be reported as unmarkable
--- by preflight. That was equally true before, when a feed_instrument row here
--- claimed otherwise and the subscription silently matched nothing.
-
--- A test broker connection so an account can be created and orders can route.
--- Creds are resolved from env by (broker_code, environment); this is just the
--- routing target an account references. Schema-qualified — it lives in `oms`.
-INSERT INTO oms.broker_connection (code, broker_code, environment, status)
-VALUES ('alpaca-paper', 'ALPACA', 'PAPER', 'ACTIVE')
-ON CONFLICT (code) DO NOTHING;
-
--- --- Test identity chain: a ready-to-trade principal so a fresh install can place
--- a paper order over the API with no admin setup. Idempotent via natural keys.
---
---   HTTP Basic auth:  test-trader-key : test-secret   (PAPER/dev only — not for prod)
---
--- UUID ids are generated, then resolved by code for the FK references
--- (principal/account/portfolio all have UNIQUE code).
-INSERT INTO oms.principal (id, code, principal_type, display_name, status)
-VALUES (gen_random_uuid(), 'test-trader', 'HUMAN', 'Test Trader', 'ACTIVE')
-ON CONFLICT (code) DO NOTHING;
-
--- Custodial account on the alpaca-paper connection. external_account_ref is the
--- broker's own account handle; Alpaca routes on the API key, so it's informational.
-INSERT INTO oms.account (id, code, broker_connection_code, external_account_ref, status)
-VALUES (gen_random_uuid(), 'test-account', 'alpaca-paper', 'PAPER', 'ACTIVE')
-ON CONFLICT (code) DO NOTHING;
-
--- Portfolio whose default route is the test account, so submits need no account_id.
-INSERT INTO oms.portfolio (id, code, name, status, default_account_id)
-SELECT gen_random_uuid(), 'test-portfolio', 'Test Portfolio', 'ACTIVE', a.id
-FROM   oms.account a WHERE a.code = 'test-account'
-ON CONFLICT (code) DO NOTHING;
-
--- Entitle the trader to trade the portfolio.
-INSERT INTO oms.principal_portfolio_grant
-    (id, principal_id, portfolio_id, can_trade, can_view, can_allocate)
-SELECT gen_random_uuid(), p.id, pf.id, true, true, false
-FROM   oms.principal p, oms.portfolio pf
-WHERE  p.code = 'test-trader' AND pf.code = 'test-portfolio'
-ON CONFLICT (principal_id, portfolio_id) DO NOTHING;
-
--- API key for the trader. key_id 'test-trader-key', secret 'test-secret'.
--- secret_hash is a precomputed bcrypt ($2y$, cost 12) of 'test-secret' — pgcrypto
--- isn't available on the server, and the app's bcrypt::verify accepts $2y$.
-INSERT INTO oms.api_key (principal_id, key_id, secret_hash, name)
-SELECT p.id, 'test-trader-key',
-       '$2y$12$haJ5Dxm/IzRmKITzcrZ.sOvTyt2KCvS7KAEC2OvDaycL1TmojvuSm', 'test-key'
-FROM   oms.principal p WHERE p.code = 'test-trader'
-ON CONFLICT (key_id) DO NOTHING;
+-- by preflight.

src/main.rs

@@ -196,6 +196,15 @@ async fn main() {
 // Server entry point (default when no subcommand is given).
 async fn serve() {
 
+    // Self-provision before the runtime pool connects: create roles/db, apply
+    // migrations, seed reference data — all idempotent, all as the admin role. Skips
+    // itself when OMS_BOOTSTRAP=off or no admin creds are present. This is what makes
+    // a fresh checkout `run the app` with no ordered setup commands.
+    if let Err(e) = setup::bootstrap::ensure_ready().await {
+        error!("{e}");
+        return;
+    }
+
     // parse db config or panic
     let db_user = env::var("DB_USER").expect("DB_USER must be set");
     let db_host = env::var("DB_HOST").expect("DB_HOST must be set");
@@ -216,10 +225,22 @@ async fn serve() {
         }
     };
 
+    // On a no-creds fresh install, drop in the SPY fixture so there is one tradeable
+    // instrument even when no broker will populate the catalog. Best-effort.
+    setup::bootstrap::ensure_fixture_if_no_brokers(&pool).await;
+
+    // Routing config for every credentialed broker — without a broker_connection a
+    // configured broker still cannot take an order. Then the optional dev identity
+    // chain (OMS_DEV_IDENTITY), so a fresh install can trade immediately.
+    setup::bootstrap::ensure_broker_connections(&pool).await;
+    setup::bootstrap::ensure_dev_identity(&pool).await;
+
     // Refuse to start on a catalog that cannot work, and name what is merely
     // degraded. Before any feed spawns, so a broken catalog surfaces here rather
-    // than as a feed that quietly subscribes to nothing.
-    if let Err(e) = preflight::run(&pool).await {
+    // than as a feed that quietly subscribes to nothing. An empty catalog is not
+    // fatal when a background sync is about to fill it.
+    let auto_sync_pending = setup::bootstrap::will_sync_on_boot(&pool).await;
+    if let Err(e) = preflight::run(&pool, auto_sync_pending).await {
         error!("preflight failed: {e}");
         return;
     }
@@ -457,6 +478,13 @@ async fn serve() {
         }
     });
 
+    // Populate an empty catalog from brokers in the background, so the minutes-long
+    // option-chain fetch never delays the server binding below. `auto_sync_pending`
+    // was computed above (catalog empty + a broker has creds).
+    if auto_sync_pending {
+        setup::bootstrap::spawn_sync();
+    }
+
     // Register routes
 
     // 1) Register order routes

src/preflight.rs

@@ -32,22 +32,27 @@ impl std::fmt::Display for Fatal {
 }
 
 /// Run every check. `Err` means do not start.
-pub async fn run(pool: &PgPool) -> Result<(), Fatal> {
-    check_catalog(pool).await?;
+///
+/// `auto_sync_pending` is true when a background broker sync is about to populate an
+/// empty catalog (see `setup::bootstrap`); it downgrades an empty `instrument` table
+/// from fatal to expected.
+pub async fn run(pool: &PgPool, auto_sync_pending: bool) -> Result<(), Fatal> {
+    check_catalog(pool, auto_sync_pending).await?;
     report_held(pool).await;
     Ok(())
 }
 
 /// Fatal checks: the master catalog and the FK targets it depends on.
 ///
-/// An empty `venue` or `currency` table is the signature of a half-run `make
-/// db-seed`; every subsequent `sync-broker` would skip every instrument on an FK
-/// miss and report success over an empty catalog.
-async fn check_catalog(pool: &PgPool) -> Result<(), Fatal> {
+/// An empty `venue` or `currency` table is the signature of a DB that never got
+/// seeded; with bootstrap on these are seeded before we ever get here, so a failure
+/// now means `OMS_BOOTSTRAP=off` over an unprepared DB. An empty `instrument` is only
+/// fatal when nothing is about to fill it — a pending background sync makes it
+/// expected, not broken.
+async fn check_catalog(pool: &PgPool, auto_sync_pending: bool) -> Result<(), Fatal> {
     for (table, hint) in [
-        ("venue", "run `make db-seed`"),
-        ("currency", "run `make db-seed`"),
-        ("instrument", "run `make sync-broker BROKER=alpaca`"),
+        ("venue", "run `make db-seed` (or enable bootstrap)"),
+        ("currency", "run `make db-seed` (or enable bootstrap)"),
     ] {
         let n: i64 = sqlx::query_scalar(&format!("SELECT count(*) FROM {table}"))
             .fetch_one(pool)
@@ -57,6 +62,22 @@ async fn check_catalog(pool: &PgPool) -> Result<(), Fatal> {
             return Err(Fatal(format!("{table} is empty — {hint}")));
         }
     }
+
+    let instruments: i64 = sqlx::query_scalar("SELECT count(*) FROM instrument")
+        .fetch_one(pool)
+        .await
+        .map_err(|e| Fatal(format!("preflight: reading instrument failed: {e}")))?;
+    if instruments == 0 {
+        if auto_sync_pending {
+            info!("preflight: catalog empty — a background broker sync will populate it");
+        } else {
+            return Err(Fatal(
+                "instrument is empty — set broker creds (auto-sync), run \
+                 `make sync-broker BROKER=alpaca`, or `make db-fixtures` for the SPY-only set"
+                    .to_string(),
+            ));
+        }
+    }
     Ok(())
 }