diff --git a/.env.example b/.env.example index 94c8922..2d1071a 100644 --- a/.env.example +++ b/.env.example @@ -1,12 +1,17 @@ -# openoms configuration. Copy to `.env`, edit, then run `make db-setup`. +# openoms configuration. Copy to `.env`, edit, then just run the app — it +# self-provisions on boot (roles, schema, ref-data) and syncs instruments in the +# background. `make db-setup` still works for manual/ordered setup if you prefer. # ===================================================================== # --- Postgres host --- DB_HOST=localhost DB_PORT=5432 -# --- Install-time admin (a Postgres superuser) — used by `make db-provision`/ -# db-migrate/db-access/db-seed to create roles+db and apply schema. Not runtime. --- +# --- Install-time admin (a Postgres superuser) — creates roles+db and applies +# schema/ref-data. Used by `make db-*` AND by the app's boot-time self-provision +# (see OMS_BOOTSTRAP). Not the runtime connection. If these + the role passwords +# below are incomplete, the app assumes the DB is already provisioned and skips +# bootstrap rather than failing. --- ADMIN_USER=postgres ADMIN_PASSWORD=postgres @@ -28,15 +33,33 @@ OMS_BIND_ADDR=localhost:3001 OMS_ADMIN_TOKEN=change-me-to-a-secure-random-token OMS_ADMIN_AUTH_ENABLED=false +# --- Boot-time self-bootstrap (so a fresh checkout is just `run the app`) --- +# OMS_BOOTSTRAP auto (default) runs provision/migrate/access/seed at boot as +# the admin role; `off` skips it (infra provisioned elsewhere). +# OMS_SYNC_ON_BOOT if-empty (default) syncs instruments from brokers with creds +# when the catalog is empty; `never` disables auto-sync. +# OMS_SYNC_UNDERLYINGS comma-separated option underlyings to seed at boot (Alpaca), +# e.g. SPY,QQQ. Empty = equities/spot only. +# OMS_DEV_IDENTITY true creates a dev principal/account with a KNOWN api secret +# (test-trader-key : test-secret) so a fresh install can place +# an order immediately. PAPER/dev only — never in production. +# OMS_BOOTSTRAP=auto +# OMS_SYNC_ON_BOOT=if-empty +# OMS_SYNC_UNDERLYINGS= +# OMS_DEV_IDENTITY=false + +# A broker_connection is auto-created at boot for every broker whose creds are set +# below — no manual step needed to make a configured broker routable. + # --- Kafka --- KAFKA_BROKER=localhost:9092 KAFKA_TOPIC=orders KAFKA_CLIENT_ID=rust-oms KAFKA_PROJECTOR_GROUP_ID=position-projector-v1 -# --- Optional: instrument seeding (`make sync-broker` = instruments + broker -# mapping). On-demand, NOT scheduled. Runs in-process, writes as DB_USER -# (oms_user). Data feeds need no seeding — each derives what it can price. --- +# --- Broker creds. Present here, the app auto-syncs that broker's catalog at boot +# (see OMS_SYNC_ON_BOOT); also used by `make sync-broker`. Writes as oms_user. +# Data feeds need no seeding — each derives what it can price. --- # DATABENTO_API_KEY= ALPACA_ENV=PAPER ALPACA_PAPER_API_KEY= diff --git a/cockpit/src/pages/Architecture.tsx b/cockpit/src/pages/Architecture.tsx index a5c1562..5da940e 100644 --- a/cockpit/src/pages/Architecture.tsx +++ b/cockpit/src/pages/Architecture.tsx @@ -318,8 +318,11 @@ export function ArchitecturePage() { Two steps, and the order is a hard dependency rather than a convention. Each reads what the one before it wrote, and nothing ever points backwards — a data feed never creates an - instrument, and an instrument never creates a venue. make seed-live runs the - chain idempotently. Feeds are not part of it: they derive their mapping at startup. + instrument, and an instrument never creates a venue. Feeds are not part of it: they derive + their mapping at startup. You rarely run these by hand: the app self-provisions on boot + (roles, schema, ref-data via the admin role) and background-syncs instruments when the + catalog is empty — the make targets are the manual equivalent, kept as an escape hatch + (OMS_BOOTSTRAP=off). diff --git a/db/scripts/fixtures.sh b/db/scripts/fixtures.sh new file mode 100755 index 0000000..965a22a --- /dev/null +++ b/db/scripts/fixtures.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# +# Load the minimal no-creds fixture (SPY) so a fresh install has one tradeable +# instrument without any broker/data-provider credentials. Idempotent — the fixture +# upserts on natural keys (ON CONFLICT DO NOTHING). +# +# Extracted from the `db-fixtures` make recipe so boot-time bootstrap can orchestrate +# it the same way as provision/migrate/access/seed. Needs an ADMIN role (the fixture +# writes master tables owned by mdm_master). +set -euo pipefail + +# Repo root is two levels up (db/scripts → db → repo); the fixture lives under the +# top-level scripts/ dir, not db/scripts/. +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$ROOT" +[[ -f .env ]] && { set -a; . ./.env; set +a; } + +: "${DB_HOST:?DB_HOST must be set}" +: "${DB_PORT:?DB_PORT must be set}" +: "${ADMIN_USER:?ADMIN_USER must be set}" +: "${ADMIN_PASSWORD:?ADMIN_PASSWORD must be set}" +ODS_DB="${ODS_DB:-ods}" + +echo "→ loading minimal fixture (SPY)" +PGPASSWORD="$ADMIN_PASSWORD" psql -v ON_ERROR_STOP=1 \ + -h "$DB_HOST" -p "$DB_PORT" -U "$ADMIN_USER" -d "$ODS_DB" \ + -f scripts/fixtures/minimal_seed.sql + +echo "fixture loaded." diff --git a/readme.md b/readme.md index 7d859e2..e1cd4a3 100644 --- a/readme.md +++ b/readme.md @@ -24,25 +24,32 @@ git clone git@github.com:maxkuttner/openoms.git && cd openoms cp .env.example .env # then edit passwords / bind addr ``` -## Setup +## Run -One-shot DB bootstrap — roles, schema, reference data, and a no-creds SPY fixture (needs the `ADMIN_*` superuser creds in `.env`): +With the `ADMIN_*` superuser + role passwords set in `.env`, just start the app — it +self-provisions on boot (roles, schema, reference data) and, if broker creds are +present, syncs that broker's instrument catalog in the background. No ordered setup +commands. ```sh -make db-setup +cargo run # OMS on OMS_BIND_ADDR (default localhost:3001) ``` -Optional live data (on-demand, needs vendor creds in `.env`): +First boot on an empty database runs provision → migrate → seed before listening (a +few seconds), then the server binds immediately while instruments populate in the +background. Subsequent boots are near-instant no-ops. Set `OMS_SYNC_ON_BOOT=never` to +skip auto-sync, or `OMS_BOOTSTRAP=off` when infrastructure is provisioned elsewhere. -```sh -make sync-broker BROKER=alpaca # seed instruments + broker mapping from Alpaca (ALPACA_PAPER_*) -make sync-broker BROKER=alpaca UNDERLYINGS=SPY,QQQ # also seed those option chains -``` +The SPY fixture seeds `alpaca-paper` + a test user (`test-trader-key` : `test-secret`), +so you can place a paper order immediately. Admin webapp: `cd cockpit && npm install && npm run dev`. -## Run +### Manual setup (optional) + +The bootstrap just orchestrates the same idempotent make targets, if you'd rather run +them yourself (or set `OMS_BOOTSTRAP=off`): ```sh -cargo run # OMS on OMS_BIND_ADDR (default localhost:3001) +make db-setup # roles, schema, ref-data, SPY fixture +make sync-broker BROKER=alpaca # instruments + broker mapping from Alpaca +make sync-broker BROKER=alpaca UNDERLYINGS=SPY,QQQ # also seed those option chains ``` - -The SPY fixture seeds `alpaca-paper` + a test user (`test-trader-key` : `test-secret`), so you can place a paper order immediately. Admin webapp: `cd cockpit && npm install && npm run dev`. diff --git a/scripts/fixtures/dev_identity.sql b/scripts/fixtures/dev_identity.sql new file mode 100644 index 0000000..218adc2 --- /dev/null +++ b/scripts/fixtures/dev_identity.sql @@ -0,0 +1,48 @@ +-- Dev identity chain: a ready-to-trade principal so a fresh install can place a +-- paper order over the API with no admin setup. Gated by OMS_DEV_IDENTITY — this +-- creates a known API secret and is for PAPER/dev only, never a real deployment. +-- Idempotent via natural keys. All in `oms`, so the app runs it as oms_user. +-- +-- HTTP Basic auth: test-trader-key : test-secret (PAPER/dev only — not for prod) + +-- A broker connection the test account can route through. Real credentialed brokers +-- also get their own connection auto-created (ensure_broker_connections); this one +-- is self-contained so the dev chain works even with no broker creds set. +INSERT INTO oms.broker_connection (code, broker_code, environment, status) +VALUES ('alpaca-paper', 'ALPACA', 'PAPER', 'ACTIVE') +ON CONFLICT (code) DO NOTHING; + +-- UUID ids are generated, then resolved by code for the FK references +-- (principal/account/portfolio all have UNIQUE code). +INSERT INTO oms.principal (id, code, principal_type, display_name, status) +VALUES (gen_random_uuid(), 'test-trader', 'HUMAN', 'Test Trader', 'ACTIVE') +ON CONFLICT (code) DO NOTHING; + +-- Custodial account on the alpaca-paper connection. external_account_ref is the +-- broker's own account handle; Alpaca routes on the API key, so it's informational. +INSERT INTO oms.account (id, code, broker_connection_code, external_account_ref, status) +VALUES (gen_random_uuid(), 'test-account', 'alpaca-paper', 'PAPER', 'ACTIVE') +ON CONFLICT (code) DO NOTHING; + +-- Portfolio whose default route is the test account, so submits need no account_id. +INSERT INTO oms.portfolio (id, code, name, status, default_account_id) +SELECT gen_random_uuid(), 'test-portfolio', 'Test Portfolio', 'ACTIVE', a.id +FROM oms.account a WHERE a.code = 'test-account' +ON CONFLICT (code) DO NOTHING; + +-- Entitle the trader to trade the portfolio. +INSERT INTO oms.principal_portfolio_grant + (id, principal_id, portfolio_id, can_trade, can_view, can_allocate) +SELECT gen_random_uuid(), p.id, pf.id, true, true, false +FROM oms.principal p, oms.portfolio pf +WHERE p.code = 'test-trader' AND pf.code = 'test-portfolio' +ON CONFLICT (principal_id, portfolio_id) DO NOTHING; + +-- API key for the trader. key_id 'test-trader-key', secret 'test-secret'. +-- secret_hash is a precomputed bcrypt ($2y$, cost 12) of 'test-secret' — pgcrypto +-- isn't available on the server, and the app's bcrypt::verify accepts $2y$. +INSERT INTO oms.api_key (principal_id, key_id, secret_hash, name) +SELECT p.id, 'test-trader-key', + '$2y$12$haJ5Dxm/IzRmKITzcrZ.sOvTyt2KCvS7KAEC2OvDaycL1TmojvuSm', 'test-key' +FROM oms.principal p WHERE p.code = 'test-trader' +ON CONFLICT (key_id) DO NOTHING; diff --git a/scripts/fixtures/minimal_seed.sql b/scripts/fixtures/minimal_seed.sql index 8d91eea..9907835 100644 --- a/scripts/fixtures/minimal_seed.sql +++ b/scripts/fixtures/minimal_seed.sql @@ -1,9 +1,15 @@ --- openoms minimal seed: one tradeable instrument so a fresh install can place a --- paper order without any data-provider creds. Idempotent (re-runnable). +-- openoms minimal seed: one tradeable instrument so a fresh no-creds install has +-- something in the catalog. Idempotent (re-runnable). Writes master tables in +-- `public` (owned by mdm_master), so this runs as the admin role via fixtures.sh. +-- +-- Routing config (broker_connection) and the test identity chain are NOT here — they +-- live in `oms` and are created by the app at boot (see src/setup/bootstrap.rs: +-- ensure_broker_connections + ensure_dev_identity). Keeping master-data seeding +-- separate from oms-owned config is what lets each run under the right role. -- -- Depends on `make db-seed` having loaded the FK targets (currency USD, venue ARCX). -- instrument.id is GENERATED ALWAYS AS IDENTITY, so we never hardcode it: insert by --- the (symbol, venue) natural key, then resolve the id for the mappings. +-- the (symbol, venue) natural key, then resolve the id for the mapping. INSERT INTO instrument (symbol, venue, currency, asset_class, instrument_class, name, @@ -26,52 +32,4 @@ ON CONFLICT (instrument_id, broker_code) DO NOTHING; -- No feed mapping row: each feed derives what it can price from its own symbology -- at startup. Note that nothing prices SPY *equity* today — the Databento feed -- carries OPRA options only — so a position in it will be reported as unmarkable --- by preflight. That was equally true before, when a feed_instrument row here --- claimed otherwise and the subscription silently matched nothing. - --- A test broker connection so an account can be created and orders can route. --- Creds are resolved from env by (broker_code, environment); this is just the --- routing target an account references. Schema-qualified — it lives in `oms`. -INSERT INTO oms.broker_connection (code, broker_code, environment, status) -VALUES ('alpaca-paper', 'ALPACA', 'PAPER', 'ACTIVE') -ON CONFLICT (code) DO NOTHING; - --- --- Test identity chain: a ready-to-trade principal so a fresh install can place --- a paper order over the API with no admin setup. Idempotent via natural keys. --- --- HTTP Basic auth: test-trader-key : test-secret (PAPER/dev only — not for prod) --- --- UUID ids are generated, then resolved by code for the FK references --- (principal/account/portfolio all have UNIQUE code). -INSERT INTO oms.principal (id, code, principal_type, display_name, status) -VALUES (gen_random_uuid(), 'test-trader', 'HUMAN', 'Test Trader', 'ACTIVE') -ON CONFLICT (code) DO NOTHING; - --- Custodial account on the alpaca-paper connection. external_account_ref is the --- broker's own account handle; Alpaca routes on the API key, so it's informational. -INSERT INTO oms.account (id, code, broker_connection_code, external_account_ref, status) -VALUES (gen_random_uuid(), 'test-account', 'alpaca-paper', 'PAPER', 'ACTIVE') -ON CONFLICT (code) DO NOTHING; - --- Portfolio whose default route is the test account, so submits need no account_id. -INSERT INTO oms.portfolio (id, code, name, status, default_account_id) -SELECT gen_random_uuid(), 'test-portfolio', 'Test Portfolio', 'ACTIVE', a.id -FROM oms.account a WHERE a.code = 'test-account' -ON CONFLICT (code) DO NOTHING; - --- Entitle the trader to trade the portfolio. -INSERT INTO oms.principal_portfolio_grant - (id, principal_id, portfolio_id, can_trade, can_view, can_allocate) -SELECT gen_random_uuid(), p.id, pf.id, true, true, false -FROM oms.principal p, oms.portfolio pf -WHERE p.code = 'test-trader' AND pf.code = 'test-portfolio' -ON CONFLICT (principal_id, portfolio_id) DO NOTHING; - --- API key for the trader. key_id 'test-trader-key', secret 'test-secret'. --- secret_hash is a precomputed bcrypt ($2y$, cost 12) of 'test-secret' — pgcrypto --- isn't available on the server, and the app's bcrypt::verify accepts $2y$. -INSERT INTO oms.api_key (principal_id, key_id, secret_hash, name) -SELECT p.id, 'test-trader-key', - '$2y$12$haJ5Dxm/IzRmKITzcrZ.sOvTyt2KCvS7KAEC2OvDaycL1TmojvuSm', 'test-key' -FROM oms.principal p WHERE p.code = 'test-trader' -ON CONFLICT (key_id) DO NOTHING; +-- by preflight. diff --git a/src/main.rs b/src/main.rs index d312955..d7ddf8e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -196,6 +196,15 @@ async fn main() { // Server entry point (default when no subcommand is given). async fn serve() { + // Self-provision before the runtime pool connects: create roles/db, apply + // migrations, seed reference data — all idempotent, all as the admin role. Skips + // itself when OMS_BOOTSTRAP=off or no admin creds are present. This is what makes + // a fresh checkout `run the app` with no ordered setup commands. + if let Err(e) = setup::bootstrap::ensure_ready().await { + error!("{e}"); + return; + } + // parse db config or panic let db_user = env::var("DB_USER").expect("DB_USER must be set"); let db_host = env::var("DB_HOST").expect("DB_HOST must be set"); @@ -216,10 +225,22 @@ async fn serve() { } }; + // On a no-creds fresh install, drop in the SPY fixture so there is one tradeable + // instrument even when no broker will populate the catalog. Best-effort. + setup::bootstrap::ensure_fixture_if_no_brokers(&pool).await; + + // Routing config for every credentialed broker — without a broker_connection a + // configured broker still cannot take an order. Then the optional dev identity + // chain (OMS_DEV_IDENTITY), so a fresh install can trade immediately. + setup::bootstrap::ensure_broker_connections(&pool).await; + setup::bootstrap::ensure_dev_identity(&pool).await; + // Refuse to start on a catalog that cannot work, and name what is merely // degraded. Before any feed spawns, so a broken catalog surfaces here rather - // than as a feed that quietly subscribes to nothing. - if let Err(e) = preflight::run(&pool).await { + // than as a feed that quietly subscribes to nothing. An empty catalog is not + // fatal when a background sync is about to fill it. + let auto_sync_pending = setup::bootstrap::will_sync_on_boot(&pool).await; + if let Err(e) = preflight::run(&pool, auto_sync_pending).await { error!("preflight failed: {e}"); return; } @@ -457,6 +478,13 @@ async fn serve() { } }); + // Populate an empty catalog from brokers in the background, so the minutes-long + // option-chain fetch never delays the server binding below. `auto_sync_pending` + // was computed above (catalog empty + a broker has creds). + if auto_sync_pending { + setup::bootstrap::spawn_sync(); + } + // Register routes // 1) Register order routes diff --git a/src/preflight.rs b/src/preflight.rs index 9f75ab2..aa430a3 100644 --- a/src/preflight.rs +++ b/src/preflight.rs @@ -32,22 +32,27 @@ impl std::fmt::Display for Fatal { } /// Run every check. `Err` means do not start. -pub async fn run(pool: &PgPool) -> Result<(), Fatal> { - check_catalog(pool).await?; +/// +/// `auto_sync_pending` is true when a background broker sync is about to populate an +/// empty catalog (see `setup::bootstrap`); it downgrades an empty `instrument` table +/// from fatal to expected. +pub async fn run(pool: &PgPool, auto_sync_pending: bool) -> Result<(), Fatal> { + check_catalog(pool, auto_sync_pending).await?; report_held(pool).await; Ok(()) } /// Fatal checks: the master catalog and the FK targets it depends on. /// -/// An empty `venue` or `currency` table is the signature of a half-run `make -/// db-seed`; every subsequent `sync-broker` would skip every instrument on an FK -/// miss and report success over an empty catalog. -async fn check_catalog(pool: &PgPool) -> Result<(), Fatal> { +/// An empty `venue` or `currency` table is the signature of a DB that never got +/// seeded; with bootstrap on these are seeded before we ever get here, so a failure +/// now means `OMS_BOOTSTRAP=off` over an unprepared DB. An empty `instrument` is only +/// fatal when nothing is about to fill it — a pending background sync makes it +/// expected, not broken. +async fn check_catalog(pool: &PgPool, auto_sync_pending: bool) -> Result<(), Fatal> { for (table, hint) in [ - ("venue", "run `make db-seed`"), - ("currency", "run `make db-seed`"), - ("instrument", "run `make sync-broker BROKER=alpaca`"), + ("venue", "run `make db-seed` (or enable bootstrap)"), + ("currency", "run `make db-seed` (or enable bootstrap)"), ] { let n: i64 = sqlx::query_scalar(&format!("SELECT count(*) FROM {table}")) .fetch_one(pool) @@ -57,6 +62,22 @@ async fn check_catalog(pool: &PgPool) -> Result<(), Fatal> { return Err(Fatal(format!("{table} is empty — {hint}"))); } } + + let instruments: i64 = sqlx::query_scalar("SELECT count(*) FROM instrument") + .fetch_one(pool) + .await + .map_err(|e| Fatal(format!("preflight: reading instrument failed: {e}")))?; + if instruments == 0 { + if auto_sync_pending { + info!("preflight: catalog empty — a background broker sync will populate it"); + } else { + return Err(Fatal( + "instrument is empty — set broker creds (auto-sync), run \ + `make sync-broker BROKER=alpaca`, or `make db-fixtures` for the SPY-only set" + .to_string(), + )); + } + } Ok(()) } diff --git a/src/setup/bootstrap.rs b/src/setup/bootstrap.rs new file mode 100644 index 0000000..0b7e8c1 --- /dev/null +++ b/src/setup/bootstrap.rs @@ -0,0 +1,314 @@ +//! Boot-time self-provisioning, so a fresh checkout is `run the app` with no ordered +//! setup commands. +//! +//! Two phases, deliberately split by *when* they run and *which role* they use: +//! +//! * [`ensure_ready`] runs **before** the runtime pool connects, orchestrating the +//! existing idempotent scripts (`provision` → `migrate` → `access` → `seed`, plus +//! the SPY fixture on a no-creds install) as the **admin** role. The scripts use +//! psql meta-commands (`\gexec`, `\i`, `SET ROLE`) that sqlx can't run, so we +//! shell out rather than reimplement tested SQL. +//! * [`spawn_sync`] runs **after** the server is listening, as the ordinary +//! `oms_user`, populating the instrument catalog from each broker whose creds are +//! present. It is slow (Alpaca's option chain is minutes) so it never blocks boot. +//! +//! The admin phase is gated by `OMS_BOOTSTRAP` (`auto` default | `off`) and skips +//! itself when the provisioning creds are incomplete — that is the +//! externally-provisioned path, and it means bootstrap never turns a DB that already +//! boots into one that doesn't. The least-privilege posture is unchanged: the +//! *runtime* pool is still `oms_user`; only this bounded boot step touches admin +//! creds, which already live in `.env`. + +use std::env; +use std::path::PathBuf; +use std::process::Stdio; + +use sqlx::PgPool; +use tokio::io::{AsyncBufReadExt, BufReader}; +use tokio::process::Command; +use tracing::{error, info, warn}; + +use crate::setup::brokers::{self, Broker}; + +/// Where the `db/scripts/*.sh` orchestration scripts live. Default is relative to +/// the working directory (repo root, as `cargo run` uses); override for an unusual +/// layout. +fn scripts_dir() -> PathBuf { + env::var("OMS_DB_SCRIPTS_DIR") + .unwrap_or_else(|_| "db/scripts".to_string()) + .into() +} + +/// Is admin-phase bootstrap enabled? `off` opts out entirely (infra managed +/// elsewhere); anything else — including unset — is `auto`. +fn bootstrap_enabled() -> bool { + !env::var("OMS_BOOTSTRAP").is_ok_and(|v| v.eq_ignore_ascii_case("off")) +} + +/// The env vars the bootstrap scripts require (provision needs the role passwords; +/// migrate/access/seed need the admin login). If any is absent we skip bootstrap +/// entirely rather than fail — a DB provisioned earlier from a fuller `.env` must +/// still boot from a leaner one. Bootstrap only *adds* the "fresh install just +/// works" path; it must never make an already-working setup worse. +fn have_provision_creds() -> bool { + let set = |k: &str| env::var(k).is_ok_and(|v| !v.is_empty()); + ["ADMIN_USER", "ADMIN_PASSWORD", "MDM_MASTER_PASSWORD", "OMS_USER_PASSWORD"] + .iter() + .all(|k| set(k)) +} + +/// Provision/migrate/seed the database if needed, before the runtime pool connects. +/// +/// Returns `Err` only when a script actually fails — a disabled or credential-less +/// bootstrap is `Ok(())` (the caller then connects and lets preflight judge the DB). +pub async fn ensure_ready() -> Result<(), Box> { + if !bootstrap_enabled() { + info!("bootstrap: OMS_BOOTSTRAP=off — skipping (assuming externally provisioned)"); + return Ok(()); + } + if !have_provision_creds() { + info!("bootstrap: provisioning creds incomplete — skipping (assuming externally provisioned)"); + return Ok(()); + } + + // Idempotent, ordered: each reads what the one before it wrote. Re-running a + // fully-provisioned DB is a cheap sequence of no-ops. + run_script("provision.sh", &[]).await?; + run_script("migrate.sh", &["up"]).await?; + run_script("access.sh", &[]).await?; + run_script("seed.sh", &[]).await?; + + Ok(()) +} + +/// Load the SPY fixture when the catalog is empty and no broker can populate it, so +/// a pure no-creds install still has one tradeable instrument. Best-effort: a +/// failure here logs and is swallowed — it is a convenience, not a prerequisite. +pub async fn ensure_fixture_if_no_brokers(pool: &PgPool) { + if !bootstrap_enabled() || !have_provision_creds() { + return; + } + if Broker::ALL.iter().any(|b| b.has_creds()) { + return; // a broker will populate the catalog via sync_instruments + } + if catalog_nonempty(pool).await.unwrap_or(true) { + return; // already have instruments; nothing to seed + } + if let Err(e) = run_script("fixtures.sh", &[]).await { + warn!("bootstrap: fixture load failed (non-fatal): {e}"); + } +} + +/// Run one `db/scripts/` script, streaming its output into the log with a +/// `bootstrap:` prefix. The child inherits our environment, so the admin/DB creds +/// loaded from `.env` reach the script without re-parsing it. +async fn run_script(name: &str, args: &[&str]) -> Result<(), Box> { + let path = scripts_dir().join(name); + info!("bootstrap: running {}", path.display()); + + let mut child = Command::new("bash") + .arg(&path) + .args(args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|e| format!("bootstrap: could not start {}: {e}", path.display()))?; + + // Drain both pipes concurrently so a chatty script can't deadlock on a full one. + if let Some(s) = child.stdout.take() { + tokio::spawn(log_lines(name.to_string(), s, false)); + } + if let Some(s) = child.stderr.take() { + tokio::spawn(log_lines(name.to_string(), s, true)); + } + + let status = child.wait().await?; + if !status.success() { + return Err(format!( + "bootstrap: {name} exited with {status}. Fix the DB/creds, or set \ + OMS_BOOTSTRAP=off if infrastructure is provisioned elsewhere." + ) + .into()); + } + Ok(()) +} + +/// Forward a child pipe to the log, one line at a time, tagged with the script name. +async fn log_lines(script: String, reader: R, is_err: bool) +where + R: tokio::io::AsyncRead + Unpin, +{ + let mut lines = BufReader::new(reader).lines(); + while let Ok(Some(line)) = lines.next_line().await { + if is_err { + warn!("bootstrap[{script}]: {line}"); + } else { + info!("bootstrap[{script}]: {line}"); + } + } +} + +async fn catalog_nonempty(pool: &PgPool) -> Result { + let n: i64 = sqlx::query_scalar("SELECT count(*) FROM instrument").fetch_one(pool).await?; + Ok(n > 0) +} + +/// Ensure a `broker_connection` exists for every broker whose creds are present. +/// +/// This is routing config, not a fixture: a credentialed broker with no connection +/// row cannot take an order at all. Runs as `oms_user` (the `oms` schema is its own), +/// idempotent on `code`. Independent of `OMS_DEV_IDENTITY` — real routing must work +/// without the dev chain. +pub async fn ensure_broker_connections(pool: &PgPool) { + for broker in Broker::ALL.iter().copied().filter(|b| b.has_creds()) { + let code = broker.connection_code(); + let res = sqlx::query( + "INSERT INTO oms.broker_connection (code, broker_code, environment, status) \ + VALUES ($1, $2, $3, 'ACTIVE') ON CONFLICT (code) DO NOTHING", + ) + .bind(&code) + .bind(broker.code()) + .bind(broker.environment()) + .execute(pool) + .await; + match res { + Ok(r) if r.rows_affected() > 0 => info!("bootstrap: created broker_connection {code}"), + Ok(_) => {} // already existed + Err(e) => error!("bootstrap: could not ensure broker_connection {code}: {e}"), + } + } +} + +/// Is the dev identity chain enabled? Off unless `OMS_DEV_IDENTITY` is truthy — it +/// creates a known API secret and must never fire in a real deployment by default. +fn dev_identity_enabled() -> bool { + env::var("OMS_DEV_IDENTITY") + .is_ok_and(|v| matches!(v.to_lowercase().as_str(), "1" | "true" | "yes")) +} + +/// Create the dev principal/account/portfolio/api-key so a fresh install can place a +/// paper order with no admin setup. Gated by `OMS_DEV_IDENTITY`; best-effort. +/// +/// The SQL is compiled in (`include_str!`), so this needs no scripts on disk, and +/// runs as `oms_user` — every row it writes lives in the `oms` schema. +pub async fn ensure_dev_identity(pool: &PgPool) { + if !dev_identity_enabled() { + return; + } + const SQL: &str = include_str!("../../scripts/fixtures/dev_identity.sql"); + match sqlx::raw_sql(SQL).execute(pool).await { + Ok(_) => info!("bootstrap: dev identity ready (test-trader-key : test-secret)"), + Err(e) => warn!("bootstrap: dev identity setup failed (non-fatal): {e}"), + } +} + +/// True when the app should populate an empty catalog from brokers at boot. +/// +/// `OMS_SYNC_ON_BOOT=never` opts out; anything else — including unset — is +/// `if-empty`. +pub fn sync_on_boot_enabled() -> bool { + !env::var("OMS_SYNC_ON_BOOT").is_ok_and(|v| v.eq_ignore_ascii_case("never")) +} + +/// Whether a boot-time sync will run: enabled, catalog empty, and some broker has +/// creds. The caller uses this to tell preflight an empty catalog is expected. +pub async fn will_sync_on_boot(pool: &PgPool) -> bool { + sync_on_boot_enabled() + && Broker::ALL.iter().any(|b| b.has_creds()) + && !catalog_nonempty(pool).await.unwrap_or(true) +} + +/// Populate the instrument catalog from every broker whose creds are present, on a +/// dedicated background thread. +/// +/// A separate thread with its own current-thread runtime, not `tokio::spawn`: the +/// sync path threads a non-`Send` boxed error across its awaits, and this keeps that +/// contained instead of forcing `Send + Sync` through every adapter's error type. +/// The job is genuinely independent — `setup::brokers::run` opens its own pool as +/// `oms_user` — so nothing is shared with the server runtime. Caller checks +/// [`will_sync_on_boot`] first. +pub fn spawn_sync() { + std::thread::spawn(|| { + let rt = match tokio::runtime::Builder::new_current_thread().enable_all().build() { + Ok(rt) => rt, + Err(e) => { + error!("bootstrap: could not start sync runtime: {e}"); + return; + } + }; + rt.block_on(sync_all_brokers()); + }); +} + +/// Run `sync-broker` for each broker whose creds are present. Each is independent: +/// one being unreachable logs and does not stop the others, nor the server. +async fn sync_all_brokers() { + let underlyings = env::var("OMS_SYNC_UNDERLYINGS").unwrap_or_default(); + let brokers: Vec = Broker::ALL.iter().copied().filter(|b| b.has_creds()).collect(); + info!("bootstrap: catalog empty — syncing {} broker(s) in background", brokers.len()); + + for broker in brokers { + let args = brokers::Args { + broker, + underlyings: underlyings.clone(), + no_enrich: false, + dry_run: false, + allow_skips: true, // a partial catalog beats none; preflight still warns + }; + match brokers::run(args).await { + Ok(()) => info!("bootstrap: {} sync complete", broker.code()), + Err(e) => error!("bootstrap: {} sync failed (server stays up): {e}", broker.code()), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Cred detection reads the `{BROKER}_{ENV}_*` vars for the active env. These + /// tests mutate process env, so they must not run in parallel with anything else + /// reading the same keys — serialized here by living in one test. + #[test] + fn alpaca_creds_detected_only_when_both_present() { + env::set_var("ALPACA_ENV", "PAPER"); + env::remove_var("ALPACA_PAPER_API_KEY"); + env::remove_var("ALPACA_PAPER_API_SECRET"); + assert!(!Broker::Alpaca.has_creds()); + + env::set_var("ALPACA_PAPER_API_KEY", "k"); + assert!(!Broker::Alpaca.has_creds(), "key alone is not enough"); + + env::set_var("ALPACA_PAPER_API_SECRET", "s"); + assert!(Broker::Alpaca.has_creds()); + + // An empty value is not a credential. + env::set_var("ALPACA_PAPER_API_SECRET", ""); + assert!(!Broker::Alpaca.has_creds()); + + env::remove_var("ALPACA_PAPER_API_KEY"); + env::remove_var("ALPACA_PAPER_API_SECRET"); + } + + #[test] + fn bootstrap_off_is_the_only_disabling_value() { + env::set_var("OMS_BOOTSTRAP", "off"); + assert!(!bootstrap_enabled()); + env::set_var("OMS_BOOTSTRAP", "OFF"); + assert!(!bootstrap_enabled(), "case-insensitive"); + env::set_var("OMS_BOOTSTRAP", "auto"); + assert!(bootstrap_enabled()); + env::remove_var("OMS_BOOTSTRAP"); + assert!(bootstrap_enabled(), "unset defaults to enabled"); + } + + #[test] + fn sync_on_boot_defaults_on_and_only_never_disables() { + env::set_var("OMS_SYNC_ON_BOOT", "never"); + assert!(!sync_on_boot_enabled()); + env::set_var("OMS_SYNC_ON_BOOT", "if-empty"); + assert!(sync_on_boot_enabled()); + env::remove_var("OMS_SYNC_ON_BOOT"); + assert!(sync_on_boot_enabled()); + } +} diff --git a/src/setup/brokers.rs b/src/setup/brokers.rs index ef6ecf3..a3336c3 100644 --- a/src/setup/brokers.rs +++ b/src/setup/brokers.rs @@ -30,12 +30,58 @@ pub enum Broker { } impl Broker { - fn code(self) -> &'static str { + /// Every broker the app can sync — the set boot-time auto-sync iterates. + pub const ALL: &'static [Broker] = &[Broker::Alpaca, Broker::Binance]; + + pub fn code(self) -> &'static str { match self { Broker::Alpaca => "ALPACA", Broker::Binance => "BINANCE", } } + + /// The configured environment for this broker (`PAPER` | `LIVE`), from + /// `{BROKER}_ENV`, defaulting to `PAPER`. Together with [`code`](Self::code) this + /// is the `(broker_code, environment)` pair a `broker_connection` routes on. + pub fn environment(self) -> String { + match self { + Broker::Alpaca => alpaca_env(), + Broker::Binance => binance_env(), + } + } + + /// The conventional `broker_connection.code` for this broker+env, e.g. + /// `alpaca-paper`. Matches the code the dev-identity fixture references. + pub fn connection_code(self) -> String { + format!("{}-{}", self.code().to_lowercase(), self.environment().to_lowercase()) + } + + /// Whether this broker's credentials are present in the environment. + /// + /// The single source of truth for "can we sync this broker": the same env vars + /// `build_alpaca`/`build_binance` require, so cred *detection* (boot-time + /// auto-sync) and cred *use* (constructing the adapter) can never disagree. + pub fn has_creds(self) -> bool { + let set = |k: &str| env::var(k).is_ok_and(|v| !v.is_empty()); + match self { + Broker::Alpaca => { + let e = alpaca_env(); + set(&format!("ALPACA_{e}_API_KEY")) && set(&format!("ALPACA_{e}_API_SECRET")) + } + Broker::Binance => { + let e = binance_env(); + set(&format!("BINANCE_{e}_API_KEY")) && set(&format!("BINANCE_{e}_PRIVATE_KEY_PATH")) + } + } + } +} + +fn alpaca_env() -> String { + env::var("ALPACA_ENV").unwrap_or_else(|_| "PAPER".into()).to_uppercase() +} + +fn binance_env() -> String { + env::var("BINANCE_ENV").unwrap_or_else(|_| "PAPER".into()).to_uppercase() } #[derive(ClapArgs, Debug, Clone)] @@ -141,7 +187,7 @@ pub async fn run(args: Args) -> Result<(), Box> { /// Build an `AlpacaAdapter` standalone from env (`ALPACA_ENV` + `ALPACA_{ENV}_API_KEY/SECRET`). fn build_alpaca() -> Result> { - let env_name = env::var("ALPACA_ENV").unwrap_or_else(|_| "PAPER".into()).to_uppercase(); + let env_name = alpaca_env(); let key = env::var(format!("ALPACA_{env_name}_API_KEY")) .map_err(|_| format!("ALPACA_{env_name}_API_KEY must be set"))?; let secret = env::var(format!("ALPACA_{env_name}_API_SECRET")) @@ -153,7 +199,7 @@ fn build_alpaca() -> Result> { /// public, but the adapter constructor needs a valid key pair; reuse the server /// wiring (`BINANCE_{ENV}_API_KEY` + `BINANCE_{ENV}_PRIVATE_KEY_PATH`). fn build_binance() -> Result> { - let env_name = env::var("BINANCE_ENV").unwrap_or_else(|_| "PAPER".into()).to_uppercase(); + let env_name = binance_env(); let key = env::var(format!("BINANCE_{env_name}_API_KEY")) .map_err(|_| format!("BINANCE_{env_name}_API_KEY must be set"))?; let pem_path = env::var(format!("BINANCE_{env_name}_PRIVATE_KEY_PATH")) diff --git a/src/setup/mod.rs b/src/setup/mod.rs index b8cdcf0..2837990 100644 --- a/src/setup/mod.rs +++ b/src/setup/mod.rs @@ -1,5 +1,6 @@ //! Setup / seeding subcommands invoked via `oms setup …`. +pub mod bootstrap; pub mod brokers; pub mod catalog;