diff --git a/.env.example b/.env.example index 36768a3..d1b92dd 100644 --- a/.env.example +++ b/.env.example @@ -34,8 +34,9 @@ KAFKA_TOPIC=orders KAFKA_CLIENT_ID=rust-oms KAFKA_PROJECTOR_GROUP_ID=position-projector-v1 -# --- Optional: live universe seeding (`make seed-instruments` / `make sync-brokers`). -# On-demand, NOT scheduled. Both run in-process and write as DB_USER (oms_user). --- +# --- Optional: instrument seeding (`make sync-broker` = instruments + broker mapping; +# `make map-feed` = data-feed pricing). On-demand, NOT scheduled. Run in-process, +# write as DB_USER (oms_user). --- # DATABENTO_API_KEY= ALPACA_ENV=PAPER ALPACA_PAPER_API_KEY= diff --git a/Cargo.lock b/Cargo.lock index 45e0436..0502d17 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -729,22 +729,15 @@ version = "0.54.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69ee0c66839f827bfd25ee8455f49414cdde18863753451b0dff69268878a041" dependencies = [ - "async-compression", "bon", "chrono", "dbn", - "futures", "hex", - "reqwest", - "serde", - "serde_json", "sha2", "thiserror 2.0.18", "time", "tokio", - "tokio-util", "tracing", - "zstd", ] [[package]] @@ -753,12 +746,9 @@ version = "0.1.0" dependencies = [ "async-trait", "chrono", - "databento", - "dotenvy", "serde", "symbology", "thiserror 1.0.69", - "time", "tokio", ] @@ -1034,21 +1024,6 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" -[[package]] -name = "futures" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" -dependencies = [ - "futures-channel", - "futures-core", - "futures-executor", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", -] - [[package]] name = "futures-channel" version = "0.3.31" @@ -1122,7 +1097,6 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ - "futures-channel", "futures-core", "futures-io", "futures-macro", @@ -2339,7 +2313,6 @@ dependencies = [ "bytes", "encoding_rs", "futures-core", - "futures-util", "h2", "http", "http-body", @@ -2358,18 +2331,15 @@ dependencies = [ "rustls-platform-verifier", "serde", "serde_json", - "serde_urlencoded", "sync_wrapper", "tokio", "tokio-rustls", - "tokio-util", "tower 0.5.3", "tower-http", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams", "web-sys", ] @@ -3785,19 +3755,6 @@ dependencies = [ "wasmparser", ] -[[package]] -name = "wasm-streams" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" -dependencies = [ - "futures-util", - "js-sys", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - [[package]] name = "wasmparser" version = "0.244.0" diff --git a/Makefile b/Makefile index 1ffd488..bd19670 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ .DEFAULT_GOAL := help -.PHONY: help db-provision db-migrate db-access db-seed db-fixtures db-setup db-reset seed-instruments sync-brokers sync-brokers-options +.PHONY: help db-provision db-migrate db-access db-seed db-fixtures db-setup db-reset sync-broker map-feed seed-live # Load .env into the recipe shell (one shell per recipe line, so chain with &&). ENV := set -a && . ./.env && set +a @@ -36,15 +36,17 @@ db-fixtures: ## Load the minimal no-creds fixture (SPY) -h "$$DB_HOST" -p "$$DB_PORT" -U "$$ADMIN_USER" -d "$${ODS_DB:-ods}" \ -f scripts/fixtures/minimal_seed.sql -# --- live universe (on-demand, NOT scheduled) --- +# --- instrument seeding (on-demand, NOT scheduled) --- # Seeding runs in-process as DB_USER (oms_user) — it holds the INSERT/UPDATE grants -# on the master catalog (db/access/ods.sql), so no admin or separate write role. -seed-instruments: ## Seed instrument universes from Databento (interactive; UNIVERSE=CODE for one; DRY_RUN=1 for a cost estimate; ARGS=... e.g. --no-enrich --max-cost 5) - @$(ENV) && cargo run --quiet -- setup universe \ - $${UNIVERSE:+--universe $$UNIVERSE} $${DRY_RUN:+--dry-run} $$ARGS - -sync-brokers: ## Sync broker symbology into instrument_xref (needs ALPACA_PAPER_*; ARGS=... e.g. --asset-class equity; DRY_RUN=1) - @$(ENV) && cargo run --quiet -- setup sync-brokers $${DRY_RUN:+--dry-run} $$ARGS - -sync-brokers-options: ## Sync Alpaca option-contract symbology into instrument_xref (UNDERLYINGS=SPY,QQQ; DRY_RUN=1) - @$(ENV) && cargo run --quiet -- setup sync-brokers --asset-class option --underlyings "$${UNDERLYINGS:-SPY,QQQ}" $${DRY_RUN:+--dry-run} $$ARGS +# on the master catalog + mapping tables (db/access/ods.sql), so no admin role. +# The broker is the instrument source: sync-broker creates the master instrument + +# broker_instrument rows; option chains come per-underlying (UNDERLYINGS=SPY,QQQ). +sync-broker: ## Seed instruments + broker mapping from a broker (BROKER=alpaca|binance; UNDERLYINGS=SPY,QQQ for options; DRY_RUN=1) + @$(ENV) && cargo run --quiet -- setup sync-broker \ + --broker "$${BROKER:-alpaca}" $${UNDERLYINGS:+--underlyings $$UNDERLYINGS} $${DRY_RUN:+--dry-run} $$ARGS + +map-feed: ## Map a data feed's symbols onto seeded instruments (FEED=databento|binance|bybit; DRY_RUN=1) + @$(ENV) && cargo run --quiet -- setup map-feed --feed "$${FEED:?set FEED=databento|binance|bybit}" $${DRY_RUN:+--dry-run} $$ARGS + +seed-live: ## Sync every configured broker + map every feed in one idempotent pass (OPTION_UNDERLYINGS=SPY,QQQ) + @./db/scripts/seed_live.sh diff --git a/cockpit/package-lock.json b/cockpit/package-lock.json index 0aa72fe..58cd5a0 100644 --- a/cockpit/package-lock.json +++ b/cockpit/package-lock.json @@ -14,6 +14,7 @@ "@mantine/notifications": "^7.13.0", "@scalar/api-reference-react": "^0.9.49", "@tanstack/react-query": "^5.59.0", + "mermaid": "^11.16.0", "react": "^18.3.1", "react-dom": "^18.3.1", "react-router-dom": "^6.26.0" @@ -22,6 +23,7 @@ "@types/react": "^18.3.3", "@types/react-dom": "^18.3.0", "@vitejs/plugin-react": "^4.3.1", + "@vitejs/plugin-react-oxc": "^0.4.3", "typescript": "^5.5.3", "vite": "^8.1.3" } @@ -89,6 +91,19 @@ "vue": "^3.3.4" } }, + "node_modules/@antfu/install-pkg": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@antfu/install-pkg/-/install-pkg-1.1.0.tgz", + "integrity": "sha512-MGQsmw10ZyI+EJo45CdSER4zEb+p31LpDAFp2Z3gkSd1yqVZGi0Ebx++YTEMonJy4oChEMLsxZ64j8FH6sSqtQ==", + "license": "MIT", + "dependencies": { + "package-manager-detector": "^1.3.0", + "tinyexec": "^1.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, "node_modules/@babel/code-frame": { "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", @@ -376,6 +391,18 @@ "node": ">=6.9.0" } }, + "node_modules/@braintree/sanitize-url": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/@braintree/sanitize-url/-/sanitize-url-7.1.2.tgz", + "integrity": "sha512-jigsZK+sMF/cuiB7sERuo9V7N9jx+dhmHHnQyDSVdpZwVutaBu7WvNYqMDLSgFgfB30n452TP3vjDAvFC973mA==", + "license": "MIT" + }, + "node_modules/@chevrotain/types": { + "version": "11.1.2", + "resolved": "https://registry.npmjs.org/@chevrotain/types/-/types-11.1.2.tgz", + "integrity": "sha512-U+HFai5+zmJCkK86QsaJtoITlboZHBqrVketcO2ROv865xfCMSFpELQoz1GkX5GzME8pTa+3kbKrZHQtI0gdbw==", + "license": "Apache-2.0" + }, "node_modules/@codemirror/autocomplete": { "version": "6.20.3", "resolved": "https://registry.npmjs.org/@codemirror/autocomplete/-/autocomplete-6.20.3.tgz", @@ -681,6 +708,23 @@ "vue": "^3.2.0" } }, + "node_modules/@iconify/types": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@iconify/types/-/types-2.0.0.tgz", + "integrity": "sha512-+wluvCrRhXrhyOmRDJ3q8mux9JkKy5SJ/v8ol2tu4FVjyYvtEzkc/3pK15ET6RKg4b4w4BmTk1+gsCUhf21Ykg==", + "license": "MIT" + }, + "node_modules/@iconify/utils": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/@iconify/utils/-/utils-3.1.4.tgz", + "integrity": "sha512-b1S7B1k9ohZ+iNTi2ATxbRYG9fTrJmUT0rc46bvVnNxqNRGW7dyo/vRREwyniI5IRN2RSJHDcm+s3BjWrSAjHw==", + "license": "MIT", + "dependencies": { + "@antfu/install-pkg": "^1.1.0", + "@iconify/types": "^2.0.0", + "import-meta-resolve": "^4.2.0" + } + }, "node_modules/@internationalized/date": { "version": "3.12.2", "resolved": "https://registry.npmjs.org/@internationalized/date/-/date-3.12.2.tgz", @@ -910,6 +954,15 @@ "integrity": "sha512-FY+MKLBoTsLNJF/eLWaOsXGdz6uh3Iu1axjPf6TUq92IYumcTcXWHoS747JARLkcdlJ/Waiaxc5wQfFO8jC6NA==", "license": "MIT" }, + "node_modules/@mermaid-js/parser": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@mermaid-js/parser/-/parser-1.2.0.tgz", + "integrity": "sha512-oYPyv8A4As1yH5Bx+04iQEQxXuIQDe0GKCNSRgao6z8AM9jixXIfP0vsppRLvGf+nKIOb9/LdpWA4YuJiVvESA==", + "license": "MIT", + "dependencies": { + "@chevrotain/types": "~11.1.2" + } + }, "node_modules/@napi-rs/wasm-runtime": { "version": "1.1.6", "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.6.tgz", @@ -1926,6 +1979,259 @@ "@babel/types": "^7.28.2" } }, + "node_modules/@types/d3": { + "version": "7.4.3", + "resolved": "https://registry.npmjs.org/@types/d3/-/d3-7.4.3.tgz", + "integrity": "sha512-lZXZ9ckh5R8uiFVt8ogUNf+pIrK4EsWrx2Np75WvF/eTpJ0FMHNhjXk8CKEx/+gpHbNQyJWehbFaTvqmHWB3ww==", + "license": "MIT", + "dependencies": { + "@types/d3-array": "*", + "@types/d3-axis": "*", + "@types/d3-brush": "*", + "@types/d3-chord": "*", + "@types/d3-color": "*", + "@types/d3-contour": "*", + "@types/d3-delaunay": "*", + "@types/d3-dispatch": "*", + "@types/d3-drag": "*", + "@types/d3-dsv": "*", + "@types/d3-ease": "*", + "@types/d3-fetch": "*", + "@types/d3-force": "*", + "@types/d3-format": "*", + "@types/d3-geo": "*", + "@types/d3-hierarchy": "*", + "@types/d3-interpolate": "*", + "@types/d3-path": "*", + "@types/d3-polygon": "*", + "@types/d3-quadtree": "*", + "@types/d3-random": "*", + "@types/d3-scale": "*", + "@types/d3-scale-chromatic": "*", + "@types/d3-selection": "*", + "@types/d3-shape": "*", + "@types/d3-time": "*", + "@types/d3-time-format": "*", + "@types/d3-timer": "*", + "@types/d3-transition": "*", + "@types/d3-zoom": "*" + } + }, + "node_modules/@types/d3-array": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/@types/d3-array/-/d3-array-3.2.2.tgz", + "integrity": "sha512-hOLWVbm7uRza0BYXpIIW5pxfrKe0W+D5lrFiAEYR+pb6w3N2SwSMaJbXdUfSEv+dT4MfHBLtn5js0LAWaO6otw==", + "license": "MIT" + }, + "node_modules/@types/d3-axis": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/d3-axis/-/d3-axis-3.0.6.tgz", + "integrity": "sha512-pYeijfZuBd87T0hGn0FO1vQ/cgLk6E1ALJjfkC0oJ8cbwkZl3TpgS8bVBLZN+2jjGgg38epgxb2zmoGtSfvgMw==", + "license": "MIT", + "dependencies": { + "@types/d3-selection": "*" + } + }, + "node_modules/@types/d3-brush": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/d3-brush/-/d3-brush-3.0.6.tgz", + "integrity": "sha512-nH60IZNNxEcrh6L1ZSMNA28rj27ut/2ZmI3r96Zd+1jrZD++zD3LsMIjWlvg4AYrHn/Pqz4CF3veCxGjtbqt7A==", + "license": "MIT", + "dependencies": { + "@types/d3-selection": "*" + } + }, + "node_modules/@types/d3-chord": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/d3-chord/-/d3-chord-3.0.6.tgz", + "integrity": "sha512-LFYWWd8nwfwEmTZG9PfQxd17HbNPksHBiJHaKuY1XeqscXacsS2tyoo6OdRsjf+NQYeB6XrNL3a25E3gH69lcg==", + "license": "MIT" + }, + "node_modules/@types/d3-color": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/@types/d3-color/-/d3-color-3.1.3.tgz", + "integrity": "sha512-iO90scth9WAbmgv7ogoq57O9YpKmFBbmoEoCHDB2xMBY0+/KVrqAaCDyCE16dUspeOvIxFFRI+0sEtqDqy2b4A==", + "license": "MIT" + }, + "node_modules/@types/d3-contour": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/d3-contour/-/d3-contour-3.0.6.tgz", + "integrity": "sha512-BjzLgXGnCWjUSYGfH1cpdo41/hgdWETu4YxpezoztawmqsvCeep+8QGfiY6YbDvfgHz/DkjeIkkZVJavB4a3rg==", + "license": "MIT", + "dependencies": { + "@types/d3-array": "*", + "@types/geojson": "*" + } + }, + "node_modules/@types/d3-delaunay": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/@types/d3-delaunay/-/d3-delaunay-6.0.4.tgz", + "integrity": "sha512-ZMaSKu4THYCU6sV64Lhg6qjf1orxBthaC161plr5KuPHo3CNm8DTHiLw/5Eq2b6TsNP0W0iJrUOFscY6Q450Hw==", + "license": "MIT" + }, + "node_modules/@types/d3-dispatch": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/@types/d3-dispatch/-/d3-dispatch-3.0.7.tgz", + "integrity": "sha512-5o9OIAdKkhN1QItV2oqaE5KMIiXAvDWBDPrD85e58Qlz1c1kI/J0NcqbEG88CoTwJrYe7ntUCVfeUl2UJKbWgA==", + "license": "MIT" + }, + "node_modules/@types/d3-drag": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/@types/d3-drag/-/d3-drag-3.0.7.tgz", + "integrity": "sha512-HE3jVKlzU9AaMazNufooRJ5ZpWmLIoc90A37WU2JMmeq28w1FQqCZswHZ3xR+SuxYftzHq6WU6KJHvqxKzTxxQ==", + "license": "MIT", + "dependencies": { + "@types/d3-selection": "*" + } + }, + "node_modules/@types/d3-dsv": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/@types/d3-dsv/-/d3-dsv-3.0.7.tgz", + "integrity": "sha512-n6QBF9/+XASqcKK6waudgL0pf/S5XHPPI8APyMLLUHd8NqouBGLsU8MgtO7NINGtPBtk9Kko/W4ea0oAspwh9g==", + "license": "MIT" + }, + "node_modules/@types/d3-ease": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/d3-ease/-/d3-ease-3.0.2.tgz", + "integrity": "sha512-NcV1JjO5oDzoK26oMzbILE6HW7uVXOHLQvHshBUW4UMdZGfiY6v5BeQwh9a9tCzv+CeefZQHJt5SRgK154RtiA==", + "license": "MIT" + }, + "node_modules/@types/d3-fetch": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/@types/d3-fetch/-/d3-fetch-3.0.7.tgz", + "integrity": "sha512-fTAfNmxSb9SOWNB9IoG5c8Hg6R+AzUHDRlsXsDZsNp6sxAEOP0tkP3gKkNSO/qmHPoBFTxNrjDprVHDQDvo5aA==", + "license": "MIT", + "dependencies": { + "@types/d3-dsv": "*" + } + }, + "node_modules/@types/d3-force": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@types/d3-force/-/d3-force-3.0.10.tgz", + "integrity": "sha512-ZYeSaCF3p73RdOKcjj+swRlZfnYpK1EbaDiYICEEp5Q6sUiqFaFQ9qgoshp5CzIyyb/yD09kD9o2zEltCexlgw==", + "license": "MIT" + }, + "node_modules/@types/d3-format": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/d3-format/-/d3-format-3.0.4.tgz", + "integrity": "sha512-fALi2aI6shfg7vM5KiR1wNJnZ7r6UuggVqtDA+xiEdPZQwy/trcQaHnwShLuLdta2rTymCNpxYTiMZX/e09F4g==", + "license": "MIT" + }, + "node_modules/@types/d3-geo": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@types/d3-geo/-/d3-geo-3.1.0.tgz", + "integrity": "sha512-856sckF0oP/diXtS4jNsiQw/UuK5fQG8l/a9VVLeSouf1/PPbBE1i1W852zVwKwYCBkFJJB7nCFTbk6UMEXBOQ==", + "license": "MIT", + "dependencies": { + "@types/geojson": "*" + } + }, + "node_modules/@types/d3-hierarchy": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/@types/d3-hierarchy/-/d3-hierarchy-3.1.7.tgz", + "integrity": "sha512-tJFtNoYBtRtkNysX1Xq4sxtjK8YgoWUNpIiUee0/jHGRwqvzYxkq0hGVbbOGSz+JgFxxRu4K8nb3YpG3CMARtg==", + "license": "MIT" + }, + "node_modules/@types/d3-interpolate": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/d3-interpolate/-/d3-interpolate-3.0.4.tgz", + "integrity": "sha512-mgLPETlrpVV1YRJIglr4Ez47g7Yxjl1lj7YKsiMCb27VJH9W8NVM6Bb9d8kkpG/uAQS5AmbA48q2IAolKKo1MA==", + "license": "MIT", + "dependencies": { + "@types/d3-color": "*" + } + }, + "node_modules/@types/d3-path": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/@types/d3-path/-/d3-path-3.1.1.tgz", + "integrity": "sha512-VMZBYyQvbGmWyWVea0EHs/BwLgxc+MKi1zLDCONksozI4YJMcTt8ZEuIR4Sb1MMTE8MMW49v0IwI5+b7RmfWlg==", + "license": "MIT" + }, + "node_modules/@types/d3-polygon": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/d3-polygon/-/d3-polygon-3.0.2.tgz", + "integrity": "sha512-ZuWOtMaHCkN9xoeEMr1ubW2nGWsp4nIql+OPQRstu4ypeZ+zk3YKqQT0CXVe/PYqrKpZAi+J9mTs05TKwjXSRA==", + "license": "MIT" + }, + "node_modules/@types/d3-quadtree": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/d3-quadtree/-/d3-quadtree-3.0.6.tgz", + "integrity": "sha512-oUzyO1/Zm6rsxKRHA1vH0NEDG58HrT5icx/azi9MF1TWdtttWl0UIUsjEQBBh+SIkrpd21ZjEv7ptxWys1ncsg==", + "license": "MIT" + }, + "node_modules/@types/d3-random": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/d3-random/-/d3-random-3.0.4.tgz", + "integrity": "sha512-UHYId5WTCx4L4YNel7NU00XUXXgvgpgZOvp10PuvsQENjMDXhh2RyFc0KBjO7B45ne4Ha1yVH7ii0vnzKkuzWA==", + "license": "MIT" + }, + "node_modules/@types/d3-scale": { + "version": "4.0.9", + "resolved": "https://registry.npmjs.org/@types/d3-scale/-/d3-scale-4.0.9.tgz", + "integrity": "sha512-dLmtwB8zkAeO/juAMfnV+sItKjlsw2lKdZVVy6LRr0cBmegxSABiLEpGVmSJJ8O08i4+sGR6qQtb6WtuwJdvVw==", + "license": "MIT", + "dependencies": { + "@types/d3-time": "*" + } + }, + "node_modules/@types/d3-scale-chromatic": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@types/d3-scale-chromatic/-/d3-scale-chromatic-3.1.0.tgz", + "integrity": "sha512-iWMJgwkK7yTRmWqRB5plb1kadXyQ5Sj8V/zYlFGMUBbIPKQScw+Dku9cAAMgJG+z5GYDoMjWGLVOvjghDEFnKQ==", + "license": "MIT" + }, + "node_modules/@types/d3-selection": { + "version": "3.0.11", + "resolved": "https://registry.npmjs.org/@types/d3-selection/-/d3-selection-3.0.11.tgz", + "integrity": "sha512-bhAXu23DJWsrI45xafYpkQ4NtcKMwWnAC/vKrd2l+nxMFuvOT3XMYTIj2opv8vq8AO5Yh7Qac/nSeP/3zjTK0w==", + "license": "MIT" + }, + "node_modules/@types/d3-shape": { + "version": "3.1.8", + "resolved": "https://registry.npmjs.org/@types/d3-shape/-/d3-shape-3.1.8.tgz", + "integrity": "sha512-lae0iWfcDeR7qt7rA88BNiqdvPS5pFVPpo5OfjElwNaT2yyekbM0C9vK+yqBqEmHr6lDkRnYNoTBYlAgJa7a4w==", + "license": "MIT", + "dependencies": { + "@types/d3-path": "*" + } + }, + "node_modules/@types/d3-time": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/d3-time/-/d3-time-3.0.4.tgz", + "integrity": "sha512-yuzZug1nkAAaBlBBikKZTgzCeA+k1uy4ZFwWANOfKw5z5LRhV0gNA7gNkKm7HoK+HRN0wX3EkxGk0fpbWhmB7g==", + "license": "MIT" + }, + "node_modules/@types/d3-time-format": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/@types/d3-time-format/-/d3-time-format-4.0.3.tgz", + "integrity": "sha512-5xg9rC+wWL8kdDj153qZcsJ0FWiFt0J5RB6LYUNZjwSnesfblqrI/bJ1wBdJ8OQfncgbJG5+2F+qfqnqyzYxyg==", + "license": "MIT" + }, + "node_modules/@types/d3-timer": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/d3-timer/-/d3-timer-3.0.2.tgz", + "integrity": "sha512-Ps3T8E8dZDam6fUyNiMkekK3XUsaUEik+idO9/YjPtfj2qruF8tFBXS7XhtE4iIXBLxhmLjP3SXpLhVf21I9Lw==", + "license": "MIT" + }, + "node_modules/@types/d3-transition": { + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/@types/d3-transition/-/d3-transition-3.0.9.tgz", + "integrity": "sha512-uZS5shfxzO3rGlu0cC3bjmMFKsXv+SmZZcgp0KD22ts4uGXp5EVYGzu/0YdwZeKmddhcAccYtREJKkPfXkZuCg==", + "license": "MIT", + "dependencies": { + "@types/d3-selection": "*" + } + }, + "node_modules/@types/d3-zoom": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/@types/d3-zoom/-/d3-zoom-3.0.8.tgz", + "integrity": "sha512-iqMC4/YlFCSlO8+2Ii1GGGliCAY4XdeG748w5vQUbevlbDu0zSjH/+jojorQVBK/se0j6DUFNPBGSqD3YWYnDw==", + "license": "MIT", + "dependencies": { + "@types/d3-interpolate": "*", + "@types/d3-selection": "*" + } + }, "node_modules/@types/debug": { "version": "4.1.13", "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.13.tgz", @@ -1935,6 +2241,12 @@ "@types/ms": "*" } }, + "node_modules/@types/geojson": { + "version": "7946.0.16", + "resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-7946.0.16.tgz", + "integrity": "sha512-6C8nqWur3j98U6+lXDfTUWIfgvZU+EumvpHKcYjujKH7woYyLj2sUmff0tRhrqM7BohUw7Pz3ZB1jj2gW9Fvmg==", + "license": "MIT" + }, "node_modules/@types/har-format": { "version": "1.2.16", "resolved": "https://registry.npmjs.org/@types/har-format/-/har-format-1.2.16.tgz", @@ -1978,14 +2290,14 @@ "version": "15.7.15", "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.15.tgz", "integrity": "sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw==", - "devOptional": true, + "dev": true, "license": "MIT" }, "node_modules/@types/react": { "version": "18.3.31", "resolved": "https://registry.npmjs.org/@types/react/-/react-18.3.31.tgz", "integrity": "sha512-vfEqpXTvwT91yhmwdfouStN2hSKwTvyRs8qpLfADyrq/kxDw0hZM7Wk9Ug1FELj8hIby+S/+kQCSRFF32nv2Qw==", - "devOptional": true, + "dev": true, "license": "MIT", "dependencies": { "@types/prop-types": "*", @@ -2002,6 +2314,13 @@ "@types/react": "^18.0.0" } }, + "node_modules/@types/trusted-types": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz", + "integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==", + "license": "MIT", + "optional": true + }, "node_modules/@types/unist": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", @@ -2036,6 +2355,16 @@ "vue": ">=3.5.18" } }, + "node_modules/@upsetjs/venn.js": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@upsetjs/venn.js/-/venn.js-2.0.0.tgz", + "integrity": "sha512-WbBhLrooyePuQ1VZxrJjtLvTc4NVfpOyKx0sKqioq9bX1C1m7Jgykkn8gLrtwumBioXIqam8DLxp88Adbue6Hw==", + "license": "MIT", + "optionalDependencies": { + "d3-selection": "^3.0.0", + "d3-transition": "^3.0.1" + } + }, "node_modules/@vercel/oidc": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/@vercel/oidc/-/oidc-3.1.0.tgz", @@ -2066,6 +2395,29 @@ "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0" } }, + "node_modules/@vitejs/plugin-react-oxc": { + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-react-oxc/-/plugin-react-oxc-0.4.3.tgz", + "integrity": "sha512-eJv6hHOIOVXzA4b2lZwccu/7VNmk9372fGOqsx5tNxiJHLtFBokyCTQUhlgjjXxl7guLPauHp0TqGTVyn1HvQA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@rolldown/pluginutils": "1.0.0-beta.47" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "peerDependencies": { + "vite": "^6.3.0 || ^7.0.0" + } + }, + "node_modules/@vitejs/plugin-react-oxc/node_modules/@rolldown/pluginutils": { + "version": "1.0.0-beta.47", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-beta.47.tgz", + "integrity": "sha512-8QagwMH3kNCuzD8EWL8R2YPW5e4OrHNSAHRFDdmFqEwEaD/KcNKjVoumo+gP2vW5eKB2UPbM6vTYiGZX0ixLnw==", + "dev": true, + "license": "MIT" + }, "node_modules/@vue/compiler-core": { "version": "3.5.39", "resolved": "https://registry.npmjs.org/@vue/compiler-core/-/compiler-core-3.5.39.tgz", @@ -2400,118 +2752,641 @@ "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/chalk": { - "version": "5.6.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.6.2.tgz", - "integrity": "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA==", - "license": "MIT", + "node_modules/chalk": { + "version": "5.6.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.6.2.tgz", + "integrity": "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA==", + "license": "MIT", + "engines": { + "node": "^12.17.0 || ^14.13 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/character-entities": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-2.0.2.tgz", + "integrity": "sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-entities-html4": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/character-entities-html4/-/character-entities-html4-2.1.0.tgz", + "integrity": "sha512-1v7fgQRj6hnSwFpq1Eu0ynr/CDEw0rXo2B61qXrLNdHZmPKgb7fqS1a2JwF0rISo9q77jDI8VMEHoApn8qDoZA==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-entities-legacy": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-3.0.0.tgz", + "integrity": "sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/clsx": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz", + "integrity": "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/comma-separated-tokens": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-2.0.3.tgz", + "integrity": "sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/commander": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz", + "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==", + "license": "MIT", + "engines": { + "node": ">= 10" + } + }, + "node_modules/convert-hrtime": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/convert-hrtime/-/convert-hrtime-5.0.0.tgz", + "integrity": "sha512-lOETlkIeYSJWcbbcvjRKGxVMXJR+8+OQb/mTPbA4ObPMytYIsUbuOE0Jzy60hjARYszq1id0j8KgVhC+WGZVTg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true, + "license": "MIT" + }, + "node_modules/cose-base": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/cose-base/-/cose-base-1.0.3.tgz", + "integrity": "sha512-s9whTXInMSgAp/NVXVNuVxVKzGH2qck3aQlVHxDCdAEPgtMKwc4Wq6/QKhgdEdgbLSi9rBTAcPoRa6JpiG4ksg==", + "license": "MIT", + "dependencies": { + "layout-base": "^1.0.0" + } + }, + "node_modules/crelt": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/crelt/-/crelt-1.0.7.tgz", + "integrity": "sha512-aK6BbWfhf4U/wCcLHKPJl/xa6VkVstRaPywWtMKGwuOLc/wZTyQYuoxgvZnNsBvv7Kg3YTBQYYBCggcviQczuA==", + "license": "MIT" + }, + "node_modules/csstype": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", + "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", + "license": "MIT" + }, + "node_modules/cva": { + "version": "1.0.0-beta.4", + "resolved": "https://registry.npmjs.org/cva/-/cva-1.0.0-beta.4.tgz", + "integrity": "sha512-F/JS9hScapq4DBVQXcK85l9U91M6ePeXoBMSp7vypzShoefUBxjQTo3g3935PUHgQd+IW77DjbPRIxugy4/GCQ==", + "license": "Apache-2.0", + "dependencies": { + "clsx": "^2.1.1" + }, + "funding": { + "url": "https://polar.sh/cva" + }, + "peerDependencies": { + "typescript": ">= 4.5.5" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/cytoscape": { + "version": "3.34.0", + "resolved": "https://registry.npmjs.org/cytoscape/-/cytoscape-3.34.0.tgz", + "integrity": "sha512-62rNSrioXw93uliKFBwjukeQyeWwH2PqDrTac31r2P6464u3AUvTk0xS4LVvT251g7IgkFunrI48ZEZGjywSOg==", + "license": "MIT", + "engines": { + "node": ">=0.10" + } + }, + "node_modules/cytoscape-cose-bilkent": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/cytoscape-cose-bilkent/-/cytoscape-cose-bilkent-4.1.0.tgz", + "integrity": "sha512-wgQlVIUJF13Quxiv5e1gstZ08rnZj2XaLHGoFMYXz7SkNfCDOOteKBE6SYRfA9WxxI/iBc3ajfDoc6hb/MRAHQ==", + "license": "MIT", + "dependencies": { + "cose-base": "^1.0.0" + }, + "peerDependencies": { + "cytoscape": "^3.2.0" + } + }, + "node_modules/cytoscape-fcose": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/cytoscape-fcose/-/cytoscape-fcose-2.2.0.tgz", + "integrity": "sha512-ki1/VuRIHFCzxWNrsshHYPs6L7TvLu3DL+TyIGEsRcvVERmxokbf5Gdk7mFxZnTdiGtnA4cfSmjZJMviqSuZrQ==", + "license": "MIT", + "dependencies": { + "cose-base": "^2.2.0" + }, + "peerDependencies": { + "cytoscape": "^3.2.0" + } + }, + "node_modules/cytoscape-fcose/node_modules/cose-base": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/cose-base/-/cose-base-2.2.0.tgz", + "integrity": "sha512-AzlgcsCbUMymkADOJtQm3wO9S3ltPfYOFD5033keQn9NJzIbtnZj+UdBJe7DYml/8TdbtHJW3j58SOnKhWY/5g==", + "license": "MIT", + "dependencies": { + "layout-base": "^2.0.0" + } + }, + "node_modules/cytoscape-fcose/node_modules/layout-base": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/layout-base/-/layout-base-2.0.1.tgz", + "integrity": "sha512-dp3s92+uNI1hWIpPGH3jK2kxE2lMjdXdr+DH8ynZHpd6PUlH6x6cbuXnoMmiNumznqaNO31xu9e79F0uuZ0JFg==", + "license": "MIT" + }, + "node_modules/d3": { + "version": "7.9.0", + "resolved": "https://registry.npmjs.org/d3/-/d3-7.9.0.tgz", + "integrity": "sha512-e1U46jVP+w7Iut8Jt8ri1YsPOvFpg46k+K8TpCb0P+zjCkjkPnV7WzfDJzMHy1LnA+wj5pLT1wjO901gLXeEhA==", + "license": "ISC", + "dependencies": { + "d3-array": "3", + "d3-axis": "3", + "d3-brush": "3", + "d3-chord": "3", + "d3-color": "3", + "d3-contour": "4", + "d3-delaunay": "6", + "d3-dispatch": "3", + "d3-drag": "3", + "d3-dsv": "3", + "d3-ease": "3", + "d3-fetch": "3", + "d3-force": "3", + "d3-format": "3", + "d3-geo": "3", + "d3-hierarchy": "3", + "d3-interpolate": "3", + "d3-path": "3", + "d3-polygon": "3", + "d3-quadtree": "3", + "d3-random": "3", + "d3-scale": "4", + "d3-scale-chromatic": "3", + "d3-selection": "3", + "d3-shape": "3", + "d3-time": "3", + "d3-time-format": "4", + "d3-timer": "3", + "d3-transition": "3", + "d3-zoom": "3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-array": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-3.2.4.tgz", + "integrity": "sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg==", + "license": "ISC", + "dependencies": { + "internmap": "1 - 2" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-axis": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-axis/-/d3-axis-3.0.0.tgz", + "integrity": "sha512-IH5tgjV4jE/GhHkRV0HiVYPDtvfjHQlQfJHs0usq7M30XcSBvOotpmH1IgkcXsO/5gEQZD43B//fc7SRT5S+xw==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-brush": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-brush/-/d3-brush-3.0.0.tgz", + "integrity": "sha512-ALnjWlVYkXsVIGlOsuWH1+3udkYFI48Ljihfnh8FZPF2QS9o+PzGLBslO0PjzVoHLZ2KCVgAM8NVkXPJB2aNnQ==", + "license": "ISC", + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-drag": "2 - 3", + "d3-interpolate": "1 - 3", + "d3-selection": "3", + "d3-transition": "3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-chord": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-chord/-/d3-chord-3.0.1.tgz", + "integrity": "sha512-VE5S6TNa+j8msksl7HwjxMHDM2yNK3XCkusIlpX5kwauBfXuyLAtNg9jCp/iHH61tgI4sb6R/EIMWCqEIdjT/g==", + "license": "ISC", + "dependencies": { + "d3-path": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-color": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-color/-/d3-color-3.1.0.tgz", + "integrity": "sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-contour": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/d3-contour/-/d3-contour-4.0.2.tgz", + "integrity": "sha512-4EzFTRIikzs47RGmdxbeUvLWtGedDUNkTcmzoeyg4sP/dvCexO47AaQL7VKy/gul85TOxw+IBgA8US2xwbToNA==", + "license": "ISC", + "dependencies": { + "d3-array": "^3.2.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-delaunay": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/d3-delaunay/-/d3-delaunay-6.0.4.tgz", + "integrity": "sha512-mdjtIZ1XLAM8bm/hx3WwjfHt6Sggek7qH043O8KEjDXN40xi3vx/6pYSVTwLjEgiXQTbvaouWKynLBiUZ6SK6A==", + "license": "ISC", + "dependencies": { + "delaunator": "5" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-dispatch": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-dispatch/-/d3-dispatch-3.0.1.tgz", + "integrity": "sha512-rzUyPU/S7rwUflMyLc1ETDeBj0NRuHKKAcvukozwhshr6g6c5d8zh4c2gQjY2bZ0dXeGLWc1PF174P2tVvKhfg==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-drag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-drag/-/d3-drag-3.0.0.tgz", + "integrity": "sha512-pWbUJLdETVA8lQNJecMxoXfH6x+mO2UQo8rSmZ+QqxcbyA3hfeprFgIT//HW2nlHChWeIIMwS2Fq+gEARkhTkg==", + "license": "ISC", + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-selection": "3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-dsv": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-dsv/-/d3-dsv-3.0.1.tgz", + "integrity": "sha512-UG6OvdI5afDIFP9w4G0mNq50dSOsXHJaRE8arAS5o9ApWnIElp8GZw1Dun8vP8OyHOZ/QJUKUJwxiiCCnUwm+Q==", + "license": "ISC", + "dependencies": { + "commander": "7", + "iconv-lite": "0.6", + "rw": "1" + }, + "bin": { + "csv2json": "bin/dsv2json.js", + "csv2tsv": "bin/dsv2dsv.js", + "dsv2dsv": "bin/dsv2dsv.js", + "dsv2json": "bin/dsv2json.js", + "json2csv": "bin/json2dsv.js", + "json2dsv": "bin/json2dsv.js", + "json2tsv": "bin/json2dsv.js", + "tsv2csv": "bin/dsv2dsv.js", + "tsv2json": "bin/dsv2json.js" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-ease": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-ease/-/d3-ease-3.0.1.tgz", + "integrity": "sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-fetch": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-fetch/-/d3-fetch-3.0.1.tgz", + "integrity": "sha512-kpkQIM20n3oLVBKGg6oHrUchHM3xODkTzjMoj7aWQFq5QEM+R6E4WkzT5+tojDY7yjez8KgCBRoj4aEr99Fdqw==", + "license": "ISC", + "dependencies": { + "d3-dsv": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-force": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-force/-/d3-force-3.0.0.tgz", + "integrity": "sha512-zxV/SsA+U4yte8051P4ECydjD/S+qeYtnaIyAs9tgHCqfguma/aAQDjo85A9Z6EKhBirHRJHXIgJUlffT4wdLg==", + "license": "ISC", + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-quadtree": "1 - 3", + "d3-timer": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-format": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/d3-format/-/d3-format-3.1.2.tgz", + "integrity": "sha512-AJDdYOdnyRDV5b6ArilzCPPwc1ejkHcoyFarqlPqT7zRYjhavcT3uSrqcMvsgh2CgoPbK3RCwyHaVyxYcP2Arg==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-geo": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/d3-geo/-/d3-geo-3.1.1.tgz", + "integrity": "sha512-637ln3gXKXOwhalDzinUgY83KzNWZRKbYubaG+fGVuc/dxO64RRljtCTnf5ecMyE1RIdtqpkVcq0IbtU2S8j2Q==", + "license": "ISC", + "dependencies": { + "d3-array": "2.5.0 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-hierarchy": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/d3-hierarchy/-/d3-hierarchy-3.1.2.tgz", + "integrity": "sha512-FX/9frcub54beBdugHjDCdikxThEqjnR93Qt7PvQTOHxyiNCAlvMrHhclk3cD5VeAaq9fxmfRp+CnWw9rEMBuA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-interpolate": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-interpolate/-/d3-interpolate-3.0.1.tgz", + "integrity": "sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==", + "license": "ISC", + "dependencies": { + "d3-color": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-path": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-path/-/d3-path-3.1.0.tgz", + "integrity": "sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-polygon": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-polygon/-/d3-polygon-3.0.1.tgz", + "integrity": "sha512-3vbA7vXYwfe1SYhED++fPUQlWSYTTGmFmQiany/gdbiWgU/iEyQzyymwL9SkJjFFuCS4902BSzewVGsHHmHtXg==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-quadtree": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-quadtree/-/d3-quadtree-3.0.1.tgz", + "integrity": "sha512-04xDrxQTDTCFwP5H6hRhsRcb9xxv2RzkcsygFzmkSIOJy3PeRJP7sNk3VRIbKXcog561P9oU0/rVH6vDROAgUw==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-random": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-random/-/d3-random-3.0.1.tgz", + "integrity": "sha512-FXMe9GfxTxqd5D6jFsQ+DJ8BJS4E/fT5mqqdjovykEB2oFbTMDVdg1MGFxfQW+FBOGoB++k8swBrgwSHT1cUXQ==", + "license": "ISC", "engines": { - "node": "^12.17.0 || ^14.13 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" + "node": ">=12" } }, - "node_modules/character-entities": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-2.0.2.tgz", - "integrity": "sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" + "node_modules/d3-sankey": { + "version": "0.12.3", + "resolved": "https://registry.npmjs.org/d3-sankey/-/d3-sankey-0.12.3.tgz", + "integrity": "sha512-nQhsBRmM19Ax5xEIPLMY9ZmJ/cDvd1BG3UVvt5h3WRxKg5zGRbvnteTyWAbzeSvlh3tW7ZEmq4VwR5mB3tutmQ==", + "license": "BSD-3-Clause", + "dependencies": { + "d3-array": "1 - 2", + "d3-shape": "^1.2.0" } }, - "node_modules/character-entities-html4": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/character-entities-html4/-/character-entities-html4-2.1.0.tgz", - "integrity": "sha512-1v7fgQRj6hnSwFpq1Eu0ynr/CDEw0rXo2B61qXrLNdHZmPKgb7fqS1a2JwF0rISo9q77jDI8VMEHoApn8qDoZA==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" + "node_modules/d3-sankey/node_modules/d3-array": { + "version": "2.12.1", + "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-2.12.1.tgz", + "integrity": "sha512-B0ErZK/66mHtEsR1TkPEEkwdy+WDesimkM5gpZr5Dsg54BiTA5RXtYW5qTLIAcekaS9xfZrzBLF/OAkB3Qn1YQ==", + "license": "BSD-3-Clause", + "dependencies": { + "internmap": "^1.0.0" } }, - "node_modules/character-entities-legacy": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-3.0.0.tgz", - "integrity": "sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" + "node_modules/d3-sankey/node_modules/d3-path": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/d3-path/-/d3-path-1.0.9.tgz", + "integrity": "sha512-VLaYcn81dtHVTjEHd8B+pbe9yHWpXKZUC87PzoFmsFrJqgFwDe/qxfp5MlfsfM1V5E/iVt0MmEbWQ7FVIXh/bg==", + "license": "BSD-3-Clause" + }, + "node_modules/d3-sankey/node_modules/d3-shape": { + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-1.3.7.tgz", + "integrity": "sha512-EUkvKjqPFUAZyOlhY5gzCxCeI0Aep04LwIRpsZ/mLFelJiUfnK56jo5JMDSE7yyP2kLSb6LtF+S5chMk7uqPqw==", + "license": "BSD-3-Clause", + "dependencies": { + "d3-path": "1" } }, - "node_modules/clsx": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz", - "integrity": "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==", - "license": "MIT", + "node_modules/d3-sankey/node_modules/internmap": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/internmap/-/internmap-1.0.1.tgz", + "integrity": "sha512-lDB5YccMydFBtasVtxnZ3MRBHuaoE8GKsppq+EchKL2U4nK/DmEpPHNH8MZe5HkMtpSiTSOZwfN0tzYjO/lJEw==", + "license": "ISC" + }, + "node_modules/d3-scale": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/d3-scale/-/d3-scale-4.0.2.tgz", + "integrity": "sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ==", + "license": "ISC", + "dependencies": { + "d3-array": "2.10.0 - 3", + "d3-format": "1 - 3", + "d3-interpolate": "1.2.0 - 3", + "d3-time": "2.1.1 - 3", + "d3-time-format": "2 - 4" + }, "engines": { - "node": ">=6" + "node": ">=12" } }, - "node_modules/comma-separated-tokens": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-2.0.3.tgz", - "integrity": "sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" + "node_modules/d3-scale-chromatic": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-scale-chromatic/-/d3-scale-chromatic-3.1.0.tgz", + "integrity": "sha512-A3s5PWiZ9YCXFye1o246KoscMWqf8BsD9eRiJ3He7C9OBaxKhAd5TFCdEx/7VbKtxxTsu//1mMJFrEt572cEyQ==", + "license": "ISC", + "dependencies": { + "d3-color": "1 - 3", + "d3-interpolate": "1 - 3" + }, + "engines": { + "node": ">=12" } }, - "node_modules/convert-hrtime": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/convert-hrtime/-/convert-hrtime-5.0.0.tgz", - "integrity": "sha512-lOETlkIeYSJWcbbcvjRKGxVMXJR+8+OQb/mTPbA4ObPMytYIsUbuOE0Jzy60hjARYszq1id0j8KgVhC+WGZVTg==", - "license": "MIT", + "node_modules/d3-selection": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz", + "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==", + "license": "ISC", "engines": { "node": ">=12" + } + }, + "node_modules/d3-shape": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-3.2.0.tgz", + "integrity": "sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA==", + "license": "ISC", + "dependencies": { + "d3-path": "^3.1.0" }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" + "engines": { + "node": ">=12" } }, - "node_modules/convert-source-map": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", - "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", - "dev": true, - "license": "MIT" + "node_modules/d3-time": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-time/-/d3-time-3.1.0.tgz", + "integrity": "sha512-VqKjzBLejbSMT4IgbmVgDjpkYrNWUYJnbCGo874u7MMKIWsILRX+OpX/gTk8MqjpT1A/c6HY2dCA77ZN0lkQ2Q==", + "license": "ISC", + "dependencies": { + "d3-array": "2 - 3" + }, + "engines": { + "node": ">=12" + } }, - "node_modules/crelt": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/crelt/-/crelt-1.0.7.tgz", - "integrity": "sha512-aK6BbWfhf4U/wCcLHKPJl/xa6VkVstRaPywWtMKGwuOLc/wZTyQYuoxgvZnNsBvv7Kg3YTBQYYBCggcviQczuA==", - "license": "MIT" + "node_modules/d3-time-format": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/d3-time-format/-/d3-time-format-4.1.0.tgz", + "integrity": "sha512-dJxPBlzC7NugB2PDLwo9Q8JiTR3M3e4/XANkreKSUxF8vvXKqm1Yfq4Q5dl8budlunRVlUUaDUgFt7eA8D6NLg==", + "license": "ISC", + "dependencies": { + "d3-time": "1 - 3" + }, + "engines": { + "node": ">=12" + } }, - "node_modules/csstype": { - "version": "3.2.3", - "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", - "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", - "license": "MIT" + "node_modules/d3-timer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-timer/-/d3-timer-3.0.1.tgz", + "integrity": "sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA==", + "license": "ISC", + "engines": { + "node": ">=12" + } }, - "node_modules/cva": { - "version": "1.0.0-beta.4", - "resolved": "https://registry.npmjs.org/cva/-/cva-1.0.0-beta.4.tgz", - "integrity": "sha512-F/JS9hScapq4DBVQXcK85l9U91M6ePeXoBMSp7vypzShoefUBxjQTo3g3935PUHgQd+IW77DjbPRIxugy4/GCQ==", - "license": "Apache-2.0", + "node_modules/d3-transition": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-transition/-/d3-transition-3.0.1.tgz", + "integrity": "sha512-ApKvfjsSR6tg06xrL434C0WydLr7JewBB3V+/39RMHsaXTOG0zmt/OAXeng5M5LBm0ojmxJrpomQVZ1aPvBL4w==", + "license": "ISC", "dependencies": { - "clsx": "^2.1.1" + "d3-color": "1 - 3", + "d3-dispatch": "1 - 3", + "d3-ease": "1 - 3", + "d3-interpolate": "1 - 3", + "d3-timer": "1 - 3" }, - "funding": { - "url": "https://polar.sh/cva" + "engines": { + "node": ">=12" }, "peerDependencies": { - "typescript": ">= 4.5.5" + "d3-selection": "2 - 3" + } + }, + "node_modules/d3-zoom": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-zoom/-/d3-zoom-3.0.0.tgz", + "integrity": "sha512-b8AmV3kfQaqWAuacbPuNbL6vahnOJflOhexLzMMNLga62+/nh0JzvJ0aO/5a5MVgUFGS7Hu1P9P03o3fJkDCyw==", + "license": "ISC", + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-drag": "2 - 3", + "d3-interpolate": "1 - 3", + "d3-selection": "2 - 3", + "d3-transition": "2 - 3" }, - "peerDependenciesMeta": { - "typescript": { - "optional": true - } + "engines": { + "node": ">=12" + } + }, + "node_modules/dagre-d3-es": { + "version": "7.0.14", + "resolved": "https://registry.npmjs.org/dagre-d3-es/-/dagre-d3-es-7.0.14.tgz", + "integrity": "sha512-P4rFMVq9ESWqmOgK+dlXvOtLwYg0i7u0HBGJER0LZDJT2VHIPAMZ/riPxqJceWMStH5+E61QxFra9kIS3AqdMg==", + "license": "MIT", + "dependencies": { + "d3": "^7.9.0", + "lodash-es": "^4.17.21" } }, + "node_modules/dayjs": { + "version": "1.11.21", + "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.21.tgz", + "integrity": "sha512-98IT+HOahAisibz/yjKbzuOBwYcjJ7BCLPzARyHiyEBmRz4fatF+KPJszEHXsGYjUG234aH/cOjW1wwTbKUZlA==", + "license": "MIT" + }, "node_modules/debug": { "version": "4.4.3", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", @@ -2548,6 +3423,15 @@ "integrity": "sha512-7z22QmUWiQ/2d0KkdYmANbRUVABpZ9SNYyH5vx6PZ+nE5bcC0l7uFvEfHlyld/HcGBFTL536ClDt3DEcSlEJAQ==", "license": "MIT" }, + "node_modules/delaunator": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/delaunator/-/delaunator-5.1.0.tgz", + "integrity": "sha512-AGrQ4QSgssa1NGmWmLPqN5NY2KajF5MqxetNEO+o0n3ZwZZeTmt7bBnvzHWrmkZFxGgr4HdyFgelzgi06otLuQ==", + "license": "ISC", + "dependencies": { + "robust-predicates": "^3.0.2" + } + }, "node_modules/dequal": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", @@ -2596,6 +3480,15 @@ "csstype": "^3.0.2" } }, + "node_modules/dompurify": { + "version": "3.4.12", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.12.tgz", + "integrity": "sha512-zQvGet8Z2sWbQhCmfFz/T5QWH2oBmjnqK3qvOjaqaNLrLEF912WamU+ohnTp0TCep/MFVHpdJuCZEdFOdTnEFg==", + "license": "(MPL-2.0 OR Apache-2.0)", + "optionalDependencies": { + "@types/trusted-types": "^2.0.7" + } + }, "node_modules/electron-to-chromium": { "version": "1.5.380", "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.380.tgz", @@ -2615,6 +3508,16 @@ "url": "https://github.com/fb55/entities?sponsor=1" } }, + "node_modules/es-toolkit": { + "version": "1.49.0", + "resolved": "https://registry.npmjs.org/es-toolkit/-/es-toolkit-1.49.0.tgz", + "integrity": "sha512-G5iZ6Pc/FNRY/soKZHC+TxGDD83rHUDXxzaWhGCX44vAv/tMs56WMusnm/KMNK+luUPsgA9U28cGr4RDlSzL2g==", + "license": "MIT", + "workspaces": [ + "docs", + "benchmarks" + ] + }, "node_modules/escalade": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", @@ -2777,6 +3680,12 @@ "node": ">=18.18.0" } }, + "node_modules/hachure-fill": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/hachure-fill/-/hachure-fill-0.5.2.tgz", + "integrity": "sha512-3GKBOn+m2LX9iq+JC1064cSFprJY4jL1jCXTcpnfER5HYE2l/4EfWSGzkPa/ZDBmYI0ZOEj5VHV/eKnPGkHuOg==", + "license": "MIT" + }, "node_modules/hast-util-embedded": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/hast-util-embedded/-/hast-util-embedded-3.0.0.tgz", @@ -3097,6 +4006,18 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/identifier-regex": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/identifier-regex/-/identifier-regex-1.0.1.tgz", @@ -3112,6 +4033,25 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/import-meta-resolve": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/import-meta-resolve/-/import-meta-resolve-4.2.0.tgz", + "integrity": "sha512-Iqv2fzaTQN28s/FwZAoFq0ZSs/7hMAHJVX+w8PZl3cY19Pxk6jFFalxQoIfW2826i/fDLXv8IiEZRIT0lDuWcg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/internmap": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/internmap/-/internmap-2.0.3.tgz", + "integrity": "sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, "node_modules/is-absolute-url": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/is-absolute-url/-/is-absolute-url-4.0.1.tgz", @@ -3226,6 +4166,36 @@ "integrity": "sha512-HUgH65KyejrUFPvHFPbqOY0rsFip3Bo5wb4ngvdi1EpCYWUQDC5V+Y7mZws+DLkr4M//zQJoanu1SP+87Dv1oQ==", "license": "MIT" }, + "node_modules/katex": { + "version": "0.16.47", + "resolved": "https://registry.npmjs.org/katex/-/katex-0.16.47.tgz", + "integrity": "sha512-Eeo8Ys1doU1z+x8AZsPpQu+p/QcZBI5PeOo7QGQdy2x2m0MU/hYagBbGOmXwr5KVbEfVuWv9LpnQWeehogurjg==", + "funding": [ + "https://opencollective.com/katex", + "https://github.com/sponsors/katex" + ], + "license": "MIT", + "dependencies": { + "commander": "^8.3.0" + }, + "bin": { + "katex": "cli.js" + } + }, + "node_modules/katex/node_modules/commander": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-8.3.0.tgz", + "integrity": "sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/khroma": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/khroma/-/khroma-2.1.0.tgz", + "integrity": "sha512-Ls993zuzfayK269Svk9hzpeGUKob/sIgZzyHYdjQoAdQetRKpOLj+k/QQQ/6Qi0Yz65mlROrfd+Ev+1+7dz9Kw==" + }, "node_modules/klona": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/klona/-/klona-2.0.6.tgz", @@ -3235,6 +4205,12 @@ "node": ">= 8" } }, + "node_modules/layout-base": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/layout-base/-/layout-base-1.0.2.tgz", + "integrity": "sha512-8h2oVEZNktL4BH2JCOI90iD1yXwL6iNW7KcCKT2QZgQJR2vbqDsldCTPRU9NifTCqHZci57XvQQ15YTu+sTYPg==", + "license": "MIT" + }, "node_modules/lightningcss": { "version": "1.32.0", "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.32.0.tgz", @@ -3508,6 +4484,12 @@ "url": "https://opencollective.com/parcel" } }, + "node_modules/lodash-es": { + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.18.1.tgz", + "integrity": "sha512-J8xewKD/Gk22OZbhpOVSwcs60zhd95ESDwezOFuA3/099925PdHJ7OFHNTGtajL3AlZkykD32HykiMo+BIBI8A==", + "license": "MIT" + }, "node_modules/longest-streak": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/longest-streak/-/longest-streak-3.1.0.tgz", @@ -3591,6 +4573,18 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/marked": { + "version": "16.4.2", + "resolved": "https://registry.npmjs.org/marked/-/marked-16.4.2.tgz", + "integrity": "sha512-TI3V8YYWvkVf3KJe1dRkpnjs68JUPyEa5vjKrp1XEEJUAOaQc+Qj+L1qWbPd0SJuAdQkFU0h73sXXqwDYxsiDA==", + "license": "MIT", + "bin": { + "marked": "bin/marked.js" + }, + "engines": { + "node": ">= 20" + } + }, "node_modules/mdast-util-find-and-replace": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/mdast-util-find-and-replace/-/mdast-util-find-and-replace-3.0.2.tgz", @@ -3801,6 +4795,35 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/mermaid": { + "version": "11.16.0", + "resolved": "https://registry.npmjs.org/mermaid/-/mermaid-11.16.0.tgz", + "integrity": "sha512-Zvm3kbstgdpvIJPPItlL7fppIZ3kibvc1oZIGxdvk9t6UFz6flv+Jw7FtRGKwfcI8OckmH04LqG6LlS6X4B1pA==", + "license": "MIT", + "dependencies": { + "@braintree/sanitize-url": "^7.1.2", + "@iconify/utils": "^3.0.2", + "@mermaid-js/parser": "^1.2.0", + "@types/d3": "^7.4.3", + "@upsetjs/venn.js": "^2.0.0", + "cytoscape": "^3.33.3", + "cytoscape-cose-bilkent": "^4.1.0", + "cytoscape-fcose": "^2.2.0", + "d3": "^7.9.0", + "d3-sankey": "^0.12.3", + "dagre-d3-es": "7.0.14", + "dayjs": "^1.11.20", + "dompurify": "^3.3.3", + "es-toolkit": "^1.45.1", + "katex": "^0.16.45", + "khroma": "^2.1.0", + "marked": "^16.3.0", + "roughjs": "^4.6.6", + "stylis": "^4.3.6", + "ts-dedent": "^2.2.0", + "uuid": "^11.1.0 || ^12 || ^13 || ^14.0.0" + } + }, "node_modules/microdiff": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/microdiff/-/microdiff-1.5.0.tgz", @@ -4446,6 +5469,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/package-manager-detector": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/package-manager-detector/-/package-manager-detector-1.7.0.tgz", + "integrity": "sha512-xg1eHpwYL/D/HEdWw2goFZP6vV0FH7W+PZ5rFkGjdIDLtxq7EkzBUeT3m+lndYCt8wKbmofUu1MUdMCXkCk9ZQ==", + "license": "MIT" + }, "node_modules/parse-ms": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/parse-ms/-/parse-ms-4.0.0.tgz", @@ -4470,6 +5499,12 @@ "url": "https://github.com/inikulin/parse5?sponsor=1" } }, + "node_modules/path-data-parser": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/path-data-parser/-/path-data-parser-0.1.0.tgz", + "integrity": "sha512-NOnmBpt5Y2RWbuv0LMzsayp3lVylAHLPUTut412ZA3l+C4uw4ZVkQbjShYCQ8TCpUMdPapr4YjUqLYD6v68j+w==", + "license": "MIT" + }, "node_modules/pathe": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", @@ -4495,6 +5530,22 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/points-on-curve": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/points-on-curve/-/points-on-curve-0.2.0.tgz", + "integrity": "sha512-0mYKnYYe9ZcqMCWhUjItv/oHjvgEsfKvnUTg8sAtnHr3GVy7rGkXCb6d5cSyqrWqL4k81b9CPg3urd+T7aop3A==", + "license": "MIT" + }, + "node_modules/points-on-path": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/points-on-path/-/points-on-path-0.2.1.tgz", + "integrity": "sha512-25ClnWWuw7JbWZcgqY/gJ4FQWadKxGWk+3kR/7kD0tCaDtPPMj7oHu2ToLaVhfpnHrZzYby2w6tUA0eOIuUg8g==", + "license": "MIT", + "dependencies": { + "path-data-parser": "0.1.0", + "points-on-curve": "0.2.0" + } + }, "node_modules/postcss": { "version": "8.5.16", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.16.tgz", @@ -5047,6 +6098,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/robust-predicates": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/robust-predicates/-/robust-predicates-3.0.3.tgz", + "integrity": "sha512-NS3levdsRIUOmiJ8FZWCP7LG3QpJyrs/TE0Zpf1yvZu8cAJJ6QMW92H1c7kWpdIHo8RvmLxN/o2JXTKHp74lUA==", + "license": "Unlicense" + }, "node_modules/rolldown": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.1.4.tgz", @@ -5088,6 +6145,30 @@ "dev": true, "license": "MIT" }, + "node_modules/roughjs": { + "version": "4.6.6", + "resolved": "https://registry.npmjs.org/roughjs/-/roughjs-4.6.6.tgz", + "integrity": "sha512-ZUz/69+SYpFN/g/lUlo2FXcIjRkSu3nDarreVdGGndHEBJ6cXPdKguS8JGxwj5HA5xIbVKSmLgr5b3AWxtRfvQ==", + "license": "MIT", + "dependencies": { + "hachure-fill": "^0.5.2", + "path-data-parser": "^0.1.0", + "points-on-curve": "^0.2.0", + "points-on-path": "^0.2.1" + } + }, + "node_modules/rw": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/rw/-/rw-1.3.3.tgz", + "integrity": "sha512-PdhdWy89SiZogBLaw42zdeqtRJ//zFd2PgQavcICDUgJT5oW10QCRKbJ6bg4r0/UY2M6BWd5tkxuGFRvCkgfHQ==", + "license": "BSD-3-Clause" + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, "node_modules/scheduler": { "version": "0.23.2", "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.2.tgz", @@ -5188,6 +6269,12 @@ "integrity": "sha512-i/n8VsZydrugj3Iuzll8+x/00GH2vnYsk1eomD8QiRrSAeW6ItbCQDtfXCeJHd0iwiNagqjQkvpvREEPtW3IoQ==", "license": "MIT" }, + "node_modules/stylis": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/stylis/-/stylis-4.4.0.tgz", + "integrity": "sha512-5Z9ZpRzfuH6l/UAvCPAPUo3665Nk2wLaZU3x+TLHKVzIz33+sbJqbtrYoC3KD4/uVOr2Zp+L0LySezP9OHV9yA==", + "license": "MIT" + }, "node_modules/super-regex": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/super-regex/-/super-regex-1.1.0.tgz", @@ -5242,13 +6329,6 @@ "url": "https://github.com/sponsors/dcastil" } }, - "node_modules/tailwindcss": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-4.3.1.tgz", - "integrity": "sha512-hk+TB1m+K8CYNrP6rjQaq/Y+4Zylwpa87mLYBKCunwnnQ9p+fHb7kmSfGqyEJoxF/O6CDyABWVFEafNSYKll+Q==", - "license": "MIT", - "peer": true - }, "node_modules/time-span": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/time-span/-/time-span-5.1.0.tgz", @@ -5264,6 +6344,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/tinyexec": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.2.4.tgz", + "integrity": "sha512-SHf/r48b7vOrjve9PxJo3MN5v5yuyjHvdUcrQffT3WXMUfnGmHDVbC4k3sHJaJTgZCwpUplIaAo5ANtMyp3YHg==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/tinyglobby": { "version": "0.2.17", "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", @@ -5315,6 +6404,15 @@ "node": ">=18.18.0" } }, + "node_modules/ts-dedent": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/ts-dedent/-/ts-dedent-2.3.0.tgz", + "integrity": "sha512-JfJeIHke7y2egdGGgRAvpCwYFUsHlM2gPcrVOxFkznt/4uzQ7HFmvE63iFHVLBJNDuyDOQgijDK/tXH/f6Msjg==", + "license": "MIT", + "engines": { + "node": ">=6.10" + } + }, "node_modules/tslib": { "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", @@ -5337,7 +6435,7 @@ "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", - "devOptional": true, + "dev": true, "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", @@ -5585,6 +6683,19 @@ } } }, + "node_modules/uuid": { + "version": "14.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-14.0.1.tgz", + "integrity": "sha512-6ZxzVpzDXDa3bJWaHilVayA+BH/1zmxCJoVgvmqJnid/gPoKHxUrS/aC/T6LGQtNHT+XHG9fXPJB4d+IrU30Ew==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist-node/bin/uuid" + } + }, "node_modules/vfile": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/vfile/-/vfile-6.0.3.tgz", diff --git a/cockpit/package.json b/cockpit/package.json index 92fad8c..bf9439c 100644 --- a/cockpit/package.json +++ b/cockpit/package.json @@ -15,6 +15,7 @@ "@mantine/notifications": "^7.13.0", "@scalar/api-reference-react": "^0.9.49", "@tanstack/react-query": "^5.59.0", + "mermaid": "^11.16.0", "react": "^18.3.1", "react-dom": "^18.3.1", "react-router-dom": "^6.26.0" @@ -23,6 +24,7 @@ "@types/react": "^18.3.3", "@types/react-dom": "^18.3.0", "@vitejs/plugin-react": "^4.3.1", + "@vitejs/plugin-react-oxc": "^0.4.3", "typescript": "^5.5.3", "vite": "^8.1.3" } diff --git a/cockpit/src/App.tsx b/cockpit/src/App.tsx index 2d4f2f7..a5e3572 100644 --- a/cockpit/src/App.tsx +++ b/cockpit/src/App.tsx @@ -11,20 +11,30 @@ import { BrokerConnectionsPage } from "./pages/BrokerConnections"; import { RiskLimitsPage } from "./pages/RiskLimits"; import { BlotterPage } from "./pages/Blotter"; import { ReconciliationPage } from "./pages/Reconciliation"; -import { UniversesPage } from "./pages/Universes"; +import { InstrumentsPage } from "./pages/Instruments"; +import { DataFeedsPage } from "./pages/DataFeeds"; -// Scalar's bundle is heavy — only load it when the API docs page is opened. +// Heavy bundles (Scalar, Mermaid) — only load when their doc page is opened. const ApiDocsPage = lazy(() => import("./pages/ApiDocs").then((m) => ({ default: m.ApiDocsPage }))); +const ArchitecturePage = lazy(() => + import("./pages/Architecture").then((m) => ({ default: m.ArchitecturePage })), +); const NAV = [ { to: "/principals", label: "Principals" }, { to: "/portfolios", label: "Portfolios" }, { to: "/accounts", label: "Accounts" }, { to: "/broker-connections", label: "Broker connections" }, + { to: "/data-feeds", label: "Data feeds" }, { to: "/risk-limits", label: "Risk limits" }, { to: "/blotter", label: "Blotter" }, - { to: "/universes", label: "Universes" }, + { to: "/instruments", label: "Instruments" }, { to: "/reconciliation", label: "Reconciliation" }, +]; + +// Grouped under a "Docs" section in the navbar. +const DOCS_NAV = [ + { to: "/docs/architecture", label: "Architecture" }, { to: "/api-docs", label: "API docs" }, ]; @@ -92,6 +102,17 @@ export function App() { active={pathname.startsWith(n.to)} /> ))} + + {DOCS_NAV.map((n) => ( + + ))} + @@ -100,10 +121,19 @@ export function App() { } /> } /> } /> + } /> } /> } /> - } /> + } /> } /> + }> + + + } + /> ({ queryKey: [PATH], queryFn: () => api.get(PATH), @@ -32,10 +41,11 @@ export function StreamHealthStrip() { if (isLoading) return ; if (isError) return null; - if (!data || data.length === 0) { + const streams = (data ?? []).filter((s) => s.kind === kind); + if (streams.length === 0) { return ( - No live streams — no broker connections configured in this process. + {emptyText} ); } @@ -43,10 +53,10 @@ export function StreamHealthStrip() { return ( - Live connections + {title} - {data.map((s) => ( + {streams.map((s) => ( OPRA options"]:::feed + BNF["Binance
spot book"]:::feed + BYF["Bybit
spot book"]:::feed + + FS["FeedSymbology
to_feed_symbol()"]:::feed + FI[("feed_instrument")]:::feed + + INST[("instrument
symbol @ venue")]:::core + + BI[("broker_instrument")]:::exec + IP["InstrumentProvider
list_instruments()"]:::exec + + ALP["Alpaca
equities · options"]:::exec + BNB["Binance
crypto spot"]:::exec + + DBN --> FS + BNF --> FS + BYF --> FS + FS -- "map-feed" --> FI + FI -- "n:1" --> INST + INST -- "1:n" --> BI + BI -- "sync-broker" --> IP + IP --> ALP + IP --> BNB + + classDef core fill:#e7edf3,stroke:#3a4a5a,color:#16202b; + classDef exec fill:#f7ecd6,stroke:#b4700e,color:#3a2a06; + classDef feed fill:#dcf0f6,stroke:#0e7490,color:#05323d;`; + +const ER = `erDiagram + VENUE ||--o{ INSTRUMENT : "lists (MIC)" + CURRENCY ||--o{ INSTRUMENT : "quoted in" + INSTRUMENT ||--o| INSTRUMENT_DERIVATIVE : "option legs" + INSTRUMENT ||--o{ BROKER_INSTRUMENT : "tradable via" + INSTRUMENT ||--o{ FEED_INSTRUMENT : "priced by" + BROKER_CONNECTION ||--o{ ACCOUNT : "routing target" + INSTRUMENT { + bigint id PK + text symbol "UNIQUE(symbol,venue)" + text venue FK + text currency FK + text asset_class + text instrument_class + text figi + } + INSTRUMENT_DERIVATIVE { + bigint instrument_id PK,FK + text underlying_symbol + text option_kind + numeric strike_price + date expiry_date + } + BROKER_INSTRUMENT { + bigint instrument_id FK + text broker_code "ALPACA|BINANCE" + text broker_symbol + text native_id + bool is_tradeable + numeric min_quantity + } + FEED_INSTRUMENT { + text feed_code "DATABENTO|BINANCE|BYBIT" + text feed_symbol + bigint instrument_id FK + bool is_active + } + VENUE { text code PK "XNAS, OPRA, BINANCE" } + CURRENCY { text code PK "USD, USDT" } + BROKER_CONNECTION { + text code PK + text broker_code + text environment "PAPER|LIVE" + } + ACCOUNT { text code PK }`; + +const SEED = `flowchart LR + ALP["Alpaca adapter"]:::exec + BIN["Binance adapter"]:::exec + INST[("instrument
+ derivative")]:::core + BI[("broker_instrument")]:::exec + FI[("feed_instrument")]:::feed + ALP -- "sync-broker" --> INST + BIN -- "sync-broker" --> INST + ALP -- "sync-broker" --> BI + BIN -- "sync-broker" --> BI + INST -- "map-feed" --> FI + classDef core fill:#e7edf3,stroke:#3a4a5a,color:#16202b; + classDef exec fill:#f7ecd6,stroke:#b4700e,color:#3a2a06; + classDef feed fill:#dcf0f6,stroke:#0e7490,color:#05323d;`; + +const RUNTIME = `flowchart TB + subgraph PRICE["Pricing path (market data)"] + direction LR + LF["Live feeds
Databento · Binance · Bybit"]:::feed + QF["subscribe held
via feed_instrument"]:::feed + MR["mark_router
ranked by provider_feed_policy"]:::feed + MS[("MarkStore")]:::feed + PL["positions · M2M P/L"]:::core + LF --> QF --> MR --> MS --> PL + end + subgraph EXEC["Execution path (orders)"] + direction LR + ORD["Order"]:::core + ACC["account -> broker_connection
(broker_code, env)"]:::exec + BIx["broker_instrument
broker_symbol / native_id"]:::exec + REG["BrokerRegistry adapter"]:::exec + API["Broker API"]:::exec + ORD --> ACC --> REG + ORD -. "resolve handle" .-> BIx --> REG --> API + end + classDef core fill:#e7edf3,stroke:#3a4a5a,color:#16202b; + classDef exec fill:#f7ecd6,stroke:#b4700e,color:#3a2a06; + classDef feed fill:#dcf0f6,stroke:#0e7490,color:#05323d;`; + +function Diagram({ chart }: { chart: string }) { + const ref = useRef(null); + useEffect(() => { + let cancelled = false; + (async () => { + const mermaid = (await import("mermaid")).default; + mermaid.initialize({ + startOnLoad: false, + // Don't inject an error graphic into document.body on a parse failure — + // we render any error inline in this component's own container instead. + suppressErrorRendering: true, + theme: "base", + themeVariables: { + fontFamily: 'ui-monospace, "SF Mono", Menlo, monospace', + fontSize: "13px", + primaryColor: "#eef2f6", + primaryBorderColor: "#3a4a5a", + primaryTextColor: "#16202b", + lineColor: "#5a6b7a", + }, + }); + if (cancelled || !ref.current) return; + try { + const { svg } = await mermaid.render(`m-${Math.random().toString(36).slice(2)}`, chart); + if (!cancelled && ref.current) ref.current.innerHTML = svg; + } catch (err) { + if (!cancelled && ref.current) { + ref.current.textContent = `Diagram failed to render: ${ + err instanceof Error ? err.message : String(err) + }`; + } + } + })(); + return () => { + cancelled = true; + }; + }, [chart]); + return ( +
+ ); +} + +const CARDS = [ + { + tag: "Master", + color: "#3a4a5a", + title: "instrument", + body: ( + <> + Canonical identity, keyed (symbol, venue). Asset/instrument class, microstructure, + FIGI. Option legs in instrument_derivative. + + ), + }, + { + tag: "Execution", + color: "#b4700e", + title: "broker_instrument", + body: ( + <> + Tradable mapping — one row per instrument per broker: broker_symbol,{" "} + native_id, limits. Written by sync-broker from the adapter's{" "} + InstrumentProvider. Its existence means tradable. + + ), + }, + { + tag: "Market data", + color: "#0e7490", + title: "feed_instrument", + body: ( + <> + Pricing mapping, 1:n. A feed's feed_symbol → instrument(s); one symbol can price + the pair on several venues. Built by map-feed, using each feed's own{" "} + FeedSymbology. + + ), + }, +]; + +export function ArchitecturePage() { + return ( + + +
+ Instrument model + + One canonical instrument catalog in the middle, two independent mappings off it: a + broker's tradable handle (broker_instrument) and a data feed's pricing symbol ( + feed_instrument). Priceable and tradable are independent — each is just the + existence of a row. + +
+ + + {CARDS.map((c) => ( + + + {c.tag} + + + {c.title} + + + {c.body} + + + ))} + + +
+ + End to end + + + One catalog in the middle, an adapter at each end. A feed translates its own symbols + through FeedSymbology; a broker publishes its tradable catalog through{" "} + InstrumentProvider. Each adapter owns its own symbology, so neither end + knows the other exists. + + +
+ +
+ + Tables & relationships + + + Foreign keys shown. broker_instrument and feed_instrument each + reference the master instrument; neither references the other. Two soft links by code (no + FK): broker_codebroker_connection, and feed_code{" "} + is ranked for failover in provider_feed_policy. + + +
+ +
+ + Seeding — where rows come from + + + Broker-first: sync-broker creates the master catalog + broker mapping;{" "} + map-feed then maps feeds onto it. make seed-live runs the whole + chain idempotently. + + +
+ +
+ + Runtime — the two paths + + + Pricing reads feed_instrument; order routing reads broker_instrument. + They never cross — a feed can price an instrument no broker trades, and vice versa. + + +
+ + + Reflects migrations 0016–0019 and the sync-broker / map-feed flow. + See also{" "} + + API docs + + . + +
+
+ ); +} diff --git a/cockpit/src/pages/BrokerConnections.tsx b/cockpit/src/pages/BrokerConnections.tsx index 31deeed..18207b5 100644 --- a/cockpit/src/pages/BrokerConnections.tsx +++ b/cockpit/src/pages/BrokerConnections.tsx @@ -10,7 +10,11 @@ const STATUS = [ export function BrokerConnectionsPage() { return ( - + ({ + queryKey: ["/admin/feeds"], + queryFn: () => api.get("/admin/feeds"), + }); + + return ( + + + + + + Feed policy + {isLoading && } + + + Ranked market-data sources per instrument class; lower rank wins, higher ranks are + failover. Mapped via make map-feed FEED=…. + + {error && Failed to load feeds.} + + + + Feed + Instrument class + Rank + Enabled + Mapped instruments + + + + {(data ?? []).map((f) => ( + + {f.feed_code} + {f.instrument_class} + {f.rank} + + + {f.enabled ? "enabled" : "disabled"} + + + {f.mapped_instruments.toLocaleString()} + + ))} + {!isLoading && (data ?? []).length === 0 && ( + + + + No feed policy configured. + + + + )} + +
+
+
+ ); +} diff --git a/cockpit/src/pages/Instruments.tsx b/cockpit/src/pages/Instruments.tsx new file mode 100644 index 0000000..ed05816 --- /dev/null +++ b/cockpit/src/pages/Instruments.tsx @@ -0,0 +1,78 @@ +import { useState } from "react"; +import { Group, Table, Text, TextInput, Title, Badge, Loader } from "@mantine/core"; +import { useDebouncedValue } from "@mantine/hooks"; +import { useQuery } from "@tanstack/react-query"; +import { api } from "../api/client"; +import type { InstrumentSummary } from "../api/types"; + +// Read-only browser over the seeded master catalog. Instruments are seeded +// broker-first (`oms setup sync-broker`) and priced via feed mapping +// (`oms setup map-feed`) — there is no in-UI seeding flow. Server-side search +// (debounced) against /admin/instruments. +export function InstrumentsPage() { + const [search, setSearch] = useState(""); + const [debounced] = useDebouncedValue(search, 250); + const { data, isLoading, error } = useQuery({ + queryKey: ["/admin/instruments", debounced], + queryFn: () => + api.get( + `/admin/instruments?limit=200${debounced ? `&search=${encodeURIComponent(debounced)}` : ""}`, + ), + }); + + return ( + <> + + Instruments + {isLoading && } + + + Seeded from a broker's catalog (make sync-broker BROKER=alpaca) and priced via + a data feed (make map-feed FEED=databento). + + setSearch(e.currentTarget.value)} + mb="md" + maw={360} + /> + {error && Failed to load instruments.} + + + + Symbol + Name + Venue + Asset class + Status + + + + {(data ?? []).map((i) => ( + + {i.symbol} + {i.name} + {i.venue} + {i.asset_class} + + + {i.status} + + + + ))} + {!isLoading && (data ?? []).length === 0 && ( + + + + No instruments seeded. Run make sync-broker to seed a broker's catalog. + + + + )} + +
+ + ); +} diff --git a/cockpit/src/pages/Universes.tsx b/cockpit/src/pages/Universes.tsx deleted file mode 100644 index c6fe8e1..0000000 --- a/cockpit/src/pages/Universes.tsx +++ /dev/null @@ -1,358 +0,0 @@ -import { useState, useEffect } from "react"; -import { - Stack, Title, Text, Table, Badge, Loader, Group, Button, Modal, - NumberInput, Switch, Alert, Code, TextInput, Checkbox, Pill, ScrollArea, -} from "@mantine/core"; -import { useQuery } from "@tanstack/react-query"; -import { api } from "../api/client"; -import { useApiMutation, notifyError } from "../api/hooks"; -import type { UniverseSummary, EstimateResponse, UnderlyingCandidate } from "../api/types"; - -const STATUS_COLOR: Record = { - SEEDED: "green", - SEEDING: "blue", - PARTIAL: "orange", - PENDING: "gray", - ERROR: "red", -}; - -function fmtDate(iso: string | null): string { - if (!iso) return "—"; - return new Date(iso).toLocaleString(); -} - -function SeedModal({ - universe, - onClose, -}: { - universe: UniverseSummary | null; - onClose: () => void; -}) { - const [estimate, setEstimate] = useState(null); - const [estimating, setEstimating] = useState(false); - const [maxCost, setMaxCost] = useState(""); - const [enrich, setEnrich] = useState(true); - - const opened = universe !== null; - const code = universe?.code; - - const [estimateFailed, setEstimateFailed] = useState(false); - - // Fetch the free estimate whenever the modal opens. Best-effort: Databento's - // get_cost is flaky for definition schema, and the estimate is informational - // (definitions are ~free; the seed doesn't need it unless a max_cost is set). - // So on failure we degrade to "unavailable" rather than a blocking error toast. - useEffect(() => { - if (!code) return; - let live = true; - setEstimate(null); - setEstimateFailed(false); - setEstimating(true); - api - .get(`/admin/universes/${code}/estimate`) - .then((e) => { if (live) setEstimate(e); }) - .catch(() => { if (live) setEstimateFailed(true); }) - .finally(() => { if (live) setEstimating(false); }); - return () => { live = false; }; - }, [code]); - - const seed = useApiMutation( - () => - api.post(`/admin/universes/${code}/seed`, { - max_cost: maxCost === "" ? null : maxCost, - enrich, - }), - { - invalidate: ["/admin/universes"], - success: "Seeding started", - onDone: () => close(), - }, - ); - - function close() { - setEstimate(null); - setMaxCost(""); - setEnrich(true); - onClose(); - } - - return ( - - - {universe?.description} - - - {estimating && } - {estimate && ( - - ${estimate.usd.toFixed(4)} - {estimate.symbol_count != null && ` · ${estimate.symbol_count} symbol(s)`} - - )} - {estimateFailed && ( - - Estimate unavailable (Databento cost API). Definition data is ~free; seeding is unaffected. - - )} - - - setMaxCost(typeof v === "number" ? v : "")} - min={0} - decimalScale={4} - step={0.01} - /> - - setEnrich(e.currentTarget.checked)} - /> - - - Seeding runs in the background. Watch the Status column flip - to SEEDED (or ERROR). - - - - - - - - - ); -} - -function EditUnderlyingsModal({ - universe, - onClose, -}: { - universe: UniverseSummary | null; - onClose: () => void; -}) { - const code = universe?.code; - const opened = universe !== null; - const [selected, setSelected] = useState>(new Set()); - const [search, setSearch] = useState(""); - const [candidates, setCandidates] = useState([]); - const [loadingSel, setLoadingSel] = useState(false); - const [searching, setSearching] = useState(false); - - // Load the current selection when the modal opens. - useEffect(() => { - if (!code) return; - let live = true; - setSearch(""); - setCandidates([]); - setLoadingSel(true); - api - .get(`/admin/universes/${code}/symbols`) - .then((s) => { if (live) setSelected(new Set(s)); }) - .catch((e) => { if (live) notifyError(e); }) - .finally(() => { if (live) setLoadingSel(false); }); - return () => { live = false; }; - }, [code]); - - // Debounced candidate search. - useEffect(() => { - if (!opened) return; - let live = true; - const q = search.trim(); - const t = setTimeout(() => { - setSearching(true); - api - .get(`/admin/underlyings?limit=50${q ? `&search=${encodeURIComponent(q)}` : ""}`) - .then((c) => { if (live) setCandidates(c); }) - .catch((e) => { if (live) notifyError(e); }) - .finally(() => { if (live) setSearching(false); }); - }, 250); - return () => { live = false; clearTimeout(t); }; - }, [search, opened]); - - const save = useApiMutation( - () => api.put(`/admin/universes/${code}/symbols`, { symbols: [...selected] }), - { invalidate: ["/admin/universes"], success: "Underlyings saved", onDone: onClose }, - ); - - function toggle(sym: string, on: boolean) { - setSelected((prev) => { - const next = new Set(prev); - if (on) next.add(sym); else next.delete(sym); - return next; - }); - } - - return ( - - - - Pick the underlyings whose option chains to seed. Candidates are equities - already in the master. Only the chosen chains are loaded — never the whole tape. - - -
- Selected ({selected.size}) - {loadingSel ? : selected.size === 0 ? ( - None selected — this OPTION universe can't be seeded until you pick at least one. - ) : ( - - {[...selected].sort().map((s) => ( - toggle(s, false)}>{s} - ))} - - )} -
- - setSearch(e.currentTarget.value)} - rightSection={searching ? : null} - /> - - - - {candidates.map((c) => ( - toggle(c.symbol, e.currentTarget.checked)} - label={{c.symbol} · {c.venue} · {c.name}} - /> - ))} - {candidates.length === 0 && !searching && ( - No matches. - )} - - - - - - - -
-
- ); -} - -export function UniversesPage() { - const [target, setTarget] = useState(null); - const [editing, setEditing] = useState(null); - const { data, isLoading, error } = useQuery({ - queryKey: ["/admin/universes"], - queryFn: () => api.get("/admin/universes"), - // Poll every 3s while anything is mid-seed so the status column self-updates. - refetchInterval: (q) => - (q.state.data ?? []).some((u) => u.status === "SEEDING") ? 3000 : false, - }); - - const anySeeding = (data ?? []).some((u) => u.status === "SEEDING"); - - // Seeded/seeding/partial universes first, then everything else; alphabetical within each. - const loaded = (s: string) => - s === "SEEDED" || s === "SEEDING" || s === "PARTIAL" ? 0 : 1; - const sorted = [...(data ?? [])].sort( - (a, b) => loaded(a.status) - loaded(b.status) || a.code.localeCompare(b.code), - ); - - return ( - - Instrument universes - - The catalog of provider datasets available for seeding — their seed status - and when each was last loaded. Seed one with the button on its row. - - {anySeeding && ( - A universe is seeding — this list refreshes automatically. - )} - - {isLoading && } - {error && Failed to load universes.} - - {data && ( - - - - Code - Category - Dataset - Status - Instruments - Last loaded - - - - - {sorted.map((u) => ( - - - {u.code} - {u.description && {u.description}} - - {u.category} - - {u.dataset} - {u.option_dataset && + {u.option_dataset}} - - - - {u.status} - {(u.status === "ERROR" || u.status === "PARTIAL") && u.last_error && ( - - {u.last_error} - - )} - - - - {u.instrument_count != null ? u.instrument_count.toLocaleString() : "—"} - - {fmtDate(u.last_seeded_at)} - - - {u.category === "OPTION" && ( - - )} - - - - - ))} - {sorted.length === 0 && ( - - - No universes in the catalog. - - - )} - -
- )} - - setTarget(null)} /> - setEditing(null)} /> -
- ); -} diff --git a/crates/dataprovider/Cargo.toml b/crates/dataprovider/Cargo.toml index 8f333a4..d13bfce 100644 --- a/crates/dataprovider/Cargo.toml +++ b/crates/dataprovider/Cargo.toml @@ -2,7 +2,7 @@ name = "dataprovider" version = "0.1.0" edition = "2021" -description = "Extensible data-provider abstraction for instrument universe seeding + enrichment." +description = "Extensible data-provider abstraction for live quote feeds + instrument enrichment." license = "MIT" [dependencies] @@ -11,12 +11,9 @@ serde = { version = "1", features = ["derive"] } thiserror = "1" chrono = { version = "0.4", features = ["serde"] } tokio = { version = "1", features = ["sync", "rt"] } -time = "0.3" -databento = "0.54" # In-repo FIGI identification engine (re-exported for the seeder). symbology = { path = "../symbology" } [dev-dependencies] tokio = { version = "1", features = ["macros", "rt-multi-thread"] } -dotenvy = "0.15" diff --git a/crates/dataprovider/examples/probe.rs b/crates/dataprovider/examples/probe.rs deleted file mode 100644 index fa043e9..0000000 --- a/crates/dataprovider/examples/probe.rs +++ /dev/null @@ -1,103 +0,0 @@ -//! Poke the Databento `UniverseSource` without a DB or the seeder. -//! -//! Needs `DATABENTO_API_KEY` in the env. -//! -//! Usage: -//! cargo run -p dataprovider --example probe -- [CATEGORY] [DATASET] [SYMBOLS...] [--fetch] -//! -//! Defaults: equity XNAS.ITCH AAPL MSFT (estimate only). -//! CATEGORY : equity | option | future -//! SYMBOLS : space-separated, or `ALL` for the whole dataset -//! --fetch : also pull definitions (COSTS MONEY). Without it, estimate only (free). -//! -//! Examples: -//! cargo run -p dataprovider --example probe -- equity XNAS.ITCH AAPL MSFT -//! cargo run -p dataprovider --example probe -- equity EQUS.SUMMARY ALL -//! cargo run -p dataprovider --example probe -- option OPRA.PILLAR SPY --fetch - -use dataprovider::{Category, DataProvider, DatabentoClient, SType, UniverseSource, UniverseSpec}; - -#[tokio::main] -async fn main() -> Result<(), Box> { - // Load DATABENTO_API_KEY from the repo-root .env (walks up from cwd). - dotenvy::dotenv().ok(); - - let mut args: Vec = std::env::args().skip(1).collect(); - let fetch = args.iter().any(|a| a == "--fetch"); - args.retain(|a| a != "--fetch"); - - let category = match args.first().map(String::as_str) { - Some("option") => Category::Option, - Some("future") => Category::Future, - _ => Category::Equity, - }; - let dataset = args.get(1).cloned().unwrap_or_else(|| "XNAS.ITCH".into()); - let symbols: Vec = match args.get(2).map(String::as_str) { - None => vec!["AAPL".into(), "MSFT".into()], - Some("ALL") => Vec::new(), - Some(_) => args[2..].to_vec(), - }; - - // Option universes read parent symbols off `dataset`; equities use raw_symbol. - let stype_in = match category { - Category::Option => SType::Parent, - _ => SType::RawSymbol, - }; - - let spec = UniverseSpec { - code: "PROBE".into(), - description: Some("ad-hoc probe".into()), - category, - dataset: dataset.clone(), - option_dataset: None, - symbols: symbols.clone(), - stype_in, - include_options: false, - }; - - let dbc = DatabentoClient::from_env()?; - dbc.set_catalog(vec![spec.clone()]).await; - - println!("provider : {}", dbc.code()); - println!("category : {:?}", category); - println!("dataset : {dataset}"); - println!( - "symbols : {}", - if symbols.is_empty() { "ALL".into() } else { symbols.join(",") } - ); - - println!("\n-- discover --"); - for u in dbc.discover().await? { - println!(" {} [{:?}] {}", u.code, u.category, u.dataset); - } - - println!("\n-- estimate_cost (free) --"); - let est = dbc.estimate_cost(&spec).await?; - println!(" ${:.4} (symbols: {:?})", est.usd, est.symbol_count); - - if !fetch { - println!("\n(estimate only. pass --fetch to pull definitions — COSTS MONEY.)"); - return Ok(()); - } - - println!("\n-- fetch_definitions (billed) --"); - let defs = dbc.fetch_definitions(&spec).await?; - println!(" {} instrument(s)\n", defs.len()); - for d in defs.iter().take(20) { - print!( - " {:<24} {:<8} {:<7} {:<4} tick={} contract={}", - d.symbol, d.venue, d.instrument_class, d.currency, d.price_increment, d.contract_size - ); - if let Some(dv) = &d.derivative { - print!( - " under={} {:?} strike={:?} exp={:?}", - dv.underlying_symbol, dv.option_kind, dv.strike_price, dv.expiry_date - ); - } - println!(); - } - if defs.len() > 20 { - println!(" … {} more", defs.len() - 20); - } - Ok(()) -} diff --git a/crates/dataprovider/src/instrument.rs b/crates/dataprovider/src/instrument.rs index 989af84..ad26cef 100644 --- a/crates/dataprovider/src/instrument.rs +++ b/crates/dataprovider/src/instrument.rs @@ -1,5 +1,6 @@ -//! Provider-neutral instrument records produced by a [`crate::UniverseSource`] -//! and mutated in place by [`crate::Enricher`]s. +//! Provider-neutral instrument records, produced by a broker adapter's +//! `InstrumentProvider` during broker-first seeding and mutated in place by +//! [`crate::Enricher`]s before the catalog upsert persists them. use serde::{Deserialize, Serialize}; diff --git a/crates/dataprovider/src/lib.rs b/crates/dataprovider/src/lib.rs index 813102e..b0a7629 100644 --- a/crates/dataprovider/src/lib.rs +++ b/crates/dataprovider/src/lib.rs @@ -1,37 +1,38 @@ //! `dataprovider` — the extensible data-provider abstraction for instrument seeding. //! //! Composable capability traits, so a provider opts into exactly what it supports: -//! - [`DataProvider`] — base identity (`code()`). -//! - [`UniverseSource`] — discover + price + fetch instrument definitions. -//! - [`Enricher`] — fill the [`Identifiers`] bag from a metadata endpoint. -//! - [`LiveQuoteFeed`] — stream normalized top-of-book [`Quote`]s. +//! - [`DataProvider`] — base identity (`code()`). +//! - [`Enricher`] — fill the [`Identifiers`] bag from a metadata endpoint. +//! - [`LiveQuoteFeed`] — stream normalized top-of-book [`Quote`]s. +//! - [`FeedSymbology`] — translate a feed's own symbols to/from the master catalog. //! -//! The seeder (in the `rustoms` app) drives a set of providers and a -//! `Vec>`. Adding a vendor = new `impl UniverseSource`; adding -//! an identifier/metadata source = new `impl Enricher`. The framework is untouched. +//! Instruments are seeded broker-first (a `BrokerAdapter`'s `InstrumentProvider` in +//! the `rustoms` app), not from a vendor definition feed, so this crate no longer +//! discovers instruments — it supplies the shapes ([`InstrumentDef`]) that seeding +//! produces, enrichment for identifiers, and the live quote/symbology traits the +//! feeds implement. //! -//! First provider: [`DatabentoClient`] (definition schema for equities + -//! OPRA.PILLAR listed options). First enricher: [`OpenFigiEnricher`] (FIGI). +//! Feeds live in the app (`opra_stream.rs`, `binance_feed.rs`, `bybit_feed.rs`) and +//! implement [`LiveQuoteFeed`] + [`FeedSymbology`] there, keeping each vendor's wire +//! format and naming rules together. Enricher: [`OpenFigiEnricher`] (FIGI). mod enrich; mod error; mod instrument; mod provider; mod quote; -mod universe; pub mod enrichers; -pub mod providers; pub use enrich::{EnrichReport, Enricher}; pub use error::ProviderError; pub use instrument::{DerivativeDef, Identifiers, InstrumentDef, OptionKind}; pub use provider::DataProvider; -pub use quote::{FeedHealth, LiveQuoteFeed, NoFeedHealth, Quote, SymbolAdds}; -pub use universe::{Category, CostEstimate, SType, UniverseSource, UniverseSpec}; +pub use quote::{ + FeedHealth, FeedSymbology, InstrumentFilter, LiveQuoteFeed, NoFeedHealth, Quote, SymbolAdds, +}; pub use enrichers::openfigi::OpenFigiEnricher; -pub use providers::databento::{DatabentoClient, DatabentoError}; // Re-export the `symbology` surface the seeder needs so the app depends only on // `dataprovider`. diff --git a/crates/dataprovider/src/provider.rs b/crates/dataprovider/src/provider.rs index 6071300..26c7af2 100644 --- a/crates/dataprovider/src/provider.rs +++ b/crates/dataprovider/src/provider.rs @@ -1,12 +1,13 @@ //! The base provider identity trait. //! -//! Every data source (universe source, quote source, …) is first a -//! [`DataProvider`] with a stable `code()`. Capability traits such as -//! [`crate::UniverseSource`] extend it, so a provider opts into exactly the +//! Every data source (quote feed, enricher, …) is first a [`DataProvider`] with a +//! stable `code()`. Capability traits such as [`crate::LiveQuoteFeed`] and +//! [`crate::FeedSymbology`] extend it, so a provider opts into exactly the //! capabilities it supports without a single god-trait. -/// A named data provider. The `code()` is stamped into -/// `oms.instrument_xref.source_code` and used in cost tables / logs. +/// A named data provider. The `code()` is the feed's identity everywhere it is +/// referenced by name: `public.feed_instrument.feed_code`, +/// `oms.provider_feed_policy` ranking, `Quote::source_code`, and logs. pub trait DataProvider: Send + Sync { /// Stable provider code, e.g. `"DATABENTO"`. fn code(&self) -> &'static str; diff --git a/crates/dataprovider/src/providers/databento.rs b/crates/dataprovider/src/providers/databento.rs deleted file mode 100644 index 92f9054..0000000 --- a/crates/dataprovider/src/providers/databento.rs +++ /dev/null @@ -1,410 +0,0 @@ -//! Databento implementation of [`UniverseSource`]. -//! -//! Uses the official `databento` crate: fetches the `definition` schema for -//! equity, option (OPRA.PILLAR), and future universes, estimates cost via -//! `metadata.get_cost` (free), and returns neutral [`InstrumentDef`] rows for the -//! seeder to upsert. - -use std::sync::Arc; - -use async_trait::async_trait; -use chrono::Datelike; -use databento::{ - dbn::{self, InstrumentClass, InstrumentDefMsg, Schema, UNDEF_TIMESTAMP}, - historical::{ - metadata::GetCostParams, timeseries::GetRangeParams, Client as HistoricalClient, DateRange, - DateTimeRange, - }, - Symbols, -}; -use thiserror::Error; -use tokio::sync::Mutex; - -use crate::error::ProviderError; -use crate::instrument::{DerivativeDef, Identifiers, InstrumentDef, OptionKind}; -use crate::provider::DataProvider; -use crate::universe::{Category, CostEstimate, SType, UniverseSource, UniverseSpec}; - -#[derive(Debug, Error)] -pub enum DatabentoError { - #[error("DATABENTO_API_KEY not set")] - MissingKey, - #[error("databento sdk error: {0}")] - Sdk(String), -} - -impl From for ProviderError { - fn from(e: DatabentoError) -> Self { - ProviderError::Request(e.to_string()) - } -} - -impl From for DatabentoError { - fn from(e: databento::Error) -> Self { - DatabentoError::Sdk(e.to_string()) - } -} - -/// Window (in days) of the definition schema we snapshot before the dataset's -/// end date. Definitions are near-static so a small window is enough to catch -/// the latest listing state. -const DEFAULT_WINDOW_DAYS: i64 = 3; - -pub struct DatabentoClient { - api_key: String, - /// Universes this client can seed. Loaded from `public.instrument_universe` - /// by the caller and injected before `discover()` is called. - catalog: Arc>>, -} - -impl DatabentoClient { - pub fn from_env() -> Result { - let api_key = std::env::var("DATABENTO_API_KEY").map_err(|_| DatabentoError::MissingKey)?; - Ok(Self { - api_key, - catalog: Arc::new(Mutex::new(Vec::new())), - }) - } - - pub fn new(api_key: impl Into) -> Self { - Self { - api_key: api_key.into(), - catalog: Arc::new(Mutex::new(Vec::new())), - } - } - - /// Seed the provider's catalog (typically from the DB). - pub async fn set_catalog(&self, catalog: Vec) { - *self.catalog.lock().await = catalog; - } - - fn build_client(&self) -> Result { - Ok(HistoricalClient::builder().key(&self.api_key)?.build()?) - } - - /// Compute the (start, end) date window for a dataset — a small window - /// ending at its last available date. - async fn window_for(&self, dataset: &str) -> Result { - let mut client = self.build_client()?; - let range = client.metadata().get_dataset_range(dataset).await?; - let end = range.end.date(); - let start = end.saturating_sub(time::Duration::days(DEFAULT_WINDOW_DAYS)); - Ok(DateRange::from((start, end))) - } - - async fn dt_window_for(&self, dataset: &str) -> Result { - let dr = self.window_for(dataset).await?; - Ok(dr.into()) - } - - /// Free cost for a single leg (dataset + symbols + stype). - async fn cost_leg( - &self, - dataset: &str, - symbols: Symbols, - stype: dbn::SType, - ) -> Result { - let mut client = self.build_client().map_err(ProviderError::from)?; - let dt_range = self.dt_window_for(dataset).await.map_err(ProviderError::from)?; - let params = GetCostParams::builder() - .dataset(dataset) - .symbols(symbols) - .schema(Schema::Definition) - .date_time_range(dt_range) - .stype_in(stype) - .build(); - match client.metadata().get_cost(¶ms).await { - Ok(v) => Ok(v), - Err(e) if is_no_data_err(&e) => Ok(0.0), - Err(e) => Err(ProviderError::Request(e.to_string())), - } - } - - /// Fetch + map one definition leg into neutral rows. - async fn fetch_leg( - &self, - dataset: &str, - symbols: Symbols, - stype: dbn::SType, - out: &mut Vec, - ) -> Result<(), ProviderError> { - let mut client = self.build_client().map_err(ProviderError::from)?; - let dt_range = self.dt_window_for(dataset).await.map_err(ProviderError::from)?; - let params = GetRangeParams::builder() - .dataset(dataset) - .symbols(symbols) - .schema(Schema::Definition) - .date_time_range(dt_range) - .stype_in(stype) - .build(); - match client.timeseries().get_range(¶ms).await { - Ok(mut dec) => { - while let Some(rec) = dec - .decode_record::() - .await - .map_err(|e| ProviderError::Request(e.to_string()))? - { - if let Some(mapped) = map_def(rec) { - out.push(mapped); - } - } - Ok(()) - } - Err(e) if is_no_data_err(&e) => Ok(()), - Err(e) => Err(ProviderError::Request(e.to_string())), - } - } -} - -fn dbn_stype(s: SType) -> dbn::SType { - match s { - SType::RawSymbol => dbn::SType::RawSymbol, - SType::Parent => dbn::SType::Parent, - } -} - -fn dbn_symbols(spec: &UniverseSpec) -> Symbols { - if spec.symbols.is_empty() { - Symbols::All - } else if matches!(spec.stype_in, SType::Parent) { - // when the stype is parent, we need to add the .OPT suffix to the symbols - Symbols::Symbols(spec.symbols.iter().map(|s| format!("{s}.OPT")).collect()) - } else { - // else just raw symbols - Symbols::Symbols(spec.symbols.clone()) - } -} - -/// Symbols for the parent/options leg (`.OPT` suffix). -fn opt_symbols(spec: &UniverseSpec) -> Symbols { - // Return the symbols for the parent/underlying - // in databento the corresponding symbols are suffixed with `.OPT` - // Note: Most of the time it does not make sense to fetch options data for all available - // symbols -> so use with care. - if spec.symbols.is_empty() { - Symbols::All - } else { - Symbols::Symbols(spec.symbols.iter().map(|s| format!("{s}.OPT")).collect()) - } -} - -fn is_no_data_err(e: &databento::Error) -> bool { - let msg = e.to_string(); - msg.contains("data_no_data_found_for_request") || msg.contains("no_data") -} - -impl DataProvider for DatabentoClient { - fn code(&self) -> &'static str { - "DATABENTO" - } -} - -#[async_trait] -impl UniverseSource for DatabentoClient { - async fn discover(&self) -> Result, ProviderError> { - // Return the catalog of universes as a vector of UniverseSpec - Ok(self.catalog.lock().await.clone()) - } - - async fn estimate_cost(&self, spec: &UniverseSpec) -> Result { - // Calculate the cost estimate for fetching the universe from Databento - let mut total = match spec.category { - // Options - Category::Option => { - self.cost_leg(&spec.dataset, opt_symbols(spec), dbn::SType::Parent) - .await? - } - // Equity - _ => { - self.cost_leg(&spec.dataset, dbn_symbols(spec), dbn_stype(spec.stype_in)) - .await? - } - }; - - // Optional: options leg attached to an equity universe. - // This needed if you have a custom universe like "SPY and its option chain" - if !matches!(spec.category, Category::Option) - && spec.include_options - && !spec.symbols.is_empty() - { - if let Some(od) = &spec.option_dataset { - total += self - .cost_leg(od, opt_symbols(spec), dbn::SType::Parent) - .await?; - } - } - - Ok(CostEstimate { - universe_code: spec.code.clone(), - usd: total, - symbol_count: if spec.symbols.is_empty() { - None - } else { - Some(spec.symbols.len()) - }, - }) - } - - async fn fetch_definitions( - &self, - spec: &UniverseSpec, - ) -> Result, ProviderError> { - let mut out: Vec = Vec::new(); - - match spec.category { - // OPRA.PILLAR listed-option universe: one parent leg on `spec.dataset`. - Category::Option => { - self.fetch_leg(&spec.dataset, opt_symbols(spec), dbn::SType::Parent, &mut out) - .await?; - } - // Equity / future: the primary definition leg. - _ => { - self.fetch_leg( - &spec.dataset, - dbn_symbols(spec), - dbn_stype(spec.stype_in), - &mut out, - ) - .await?; - - // Optional options leg (definition schema on the OPRA/parent dataset). - if spec.include_options && !spec.symbols.is_empty() { - if let Some(od) = &spec.option_dataset { - self.fetch_leg(od, opt_symbols(spec), dbn::SType::Parent, &mut out) - .await?; - } - } - } - } - - // Latest record per (symbol, venue) wins — definitions may repeat. - Ok(dedupe_latest(out)) - } -} - -fn dedupe_latest(defs: Vec) -> Vec { - use std::collections::HashMap; - let mut latest: HashMap<(String, String), InstrumentDef> = HashMap::new(); - for d in defs { - latest.insert((d.symbol.clone(), d.venue.clone()), d); - } - latest.into_values().collect() -} - -/// Databento fixed-point (1e-9) → f64 with heuristic pretty-vs-raw detection. -fn from_fixed(v: i64) -> Option { - if v == i64::MIN || v == i64::MAX { - return None; - } - let f = v as f64; - if f.abs() >= 1e6 { - Some(f / 1e9) - } else { - Some(f) - } -} - -fn decimals_of(tick: f64) -> i32 { - if tick <= 0.0 { - return 2; - } - let (mut d, mut x) = (0i32, tick); - while (x - x.round()).abs() > 1e-12 && d < 12 { - x *= 10.0; - d += 1; - } - d -} - -fn ns_to_date(ns: u64) -> Option { - if ns == UNDEF_TIMESTAMP || ns == 0 { - return None; - } - let secs = (ns / 1_000_000_000) as i64; - chrono::DateTime::::from_timestamp(secs, 0) - .map(|dt| dt.date_naive()) - .filter(|d| d.year() <= 2100) -} - -fn map_def(rec: &InstrumentDefMsg) -> Option { - let iclass = rec.instrument_class().ok()?; - let (instrument_class, opt_kind): (&str, Option) = match iclass { - InstrumentClass::Stock => ("SPOT", None), - InstrumentClass::Call => ("OPTION", Some(OptionKind::Call)), - InstrumentClass::Put => ("OPTION", Some(OptionKind::Put)), - InstrumentClass::Future => ("FUTURE", None), - _ => return None, - }; - - let symbol = rec.raw_symbol().ok()?.to_string(); - let exchange = rec.exchange().ok()?.to_string(); - if symbol.is_empty() || exchange.is_empty() { - return None; - } - let venue = exchange.clone(); - - let currency = rec.currency().ok().map(|s| s.to_string()).unwrap_or_else(|| "USD".into()); - let currency = currency.trim().to_uppercase(); - - let tick = from_fixed(rec.min_price_increment).filter(|v| *v > 0.0).unwrap_or(0.01); - // `contract_multiplier` is UNDEF (`i32::MAX`) for OPRA options and 0 when absent. - let contract = if rec.contract_multiplier != 0 && rec.contract_multiplier != i32::MAX { - rec.contract_multiplier as f64 - } else { - from_fixed(rec.unit_of_measure_qty) - .filter(|v| *v > 0.0) - // Listed US options standardize on a 100-share multiplier. - .unwrap_or(if instrument_class == "OPTION" { 100.0 } else { 1.0 }) - }; - let lot = if rec.min_lot_size_round_lot > 0 { - Some(rec.min_lot_size_round_lot as f64) - } else { - None - }; - - let derivative = if instrument_class == "OPTION" { - let underlying = rec.underlying().ok().unwrap_or("").trim().to_string(); - if underlying.is_empty() { - None - } else { - Some(DerivativeDef { - underlying_symbol: underlying, - option_kind: opt_kind, - strike_price: from_fixed(rec.strike_price), - expiry_date: ns_to_date(rec.expiration), - activation_date: ns_to_date(rec.activation), - }) - } - } else { - None - }; - - // Options require a derivative row to be seeded. - if instrument_class == "OPTION" && derivative.is_none() { - return None; - } - - let native_id = if rec.hd.instrument_id != 0 { - Some(rec.hd.instrument_id.to_string()) - } else { - None - }; - - Some(InstrumentDef { - symbol: symbol.clone(), - venue, - currency, - asset_class: "EQUITY".into(), - instrument_class: instrument_class.into(), - name: Some(symbol), - price_precision: decimals_of(tick), - price_increment: tick, - size_increment: lot.unwrap_or(1.0), - lot_size: lot, - contract_size: if contract > 0.0 { contract } else { 1.0 }, - native_id, - provider_exchange: Some(exchange), - derivative, - identifiers: Identifiers::default(), - }) -} diff --git a/crates/dataprovider/src/providers/mod.rs b/crates/dataprovider/src/providers/mod.rs deleted file mode 100644 index 1005d94..0000000 --- a/crates/dataprovider/src/providers/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -//! Concrete [`crate::UniverseSource`] implementations, one module per vendor. - -pub mod databento; diff --git a/crates/dataprovider/src/quote.rs b/crates/dataprovider/src/quote.rs index 0e89b96..0a038eb 100644 --- a/crates/dataprovider/src/quote.rs +++ b/crates/dataprovider/src/quote.rs @@ -66,19 +66,44 @@ impl FeedHealth for NoFeedHealth { fn on_event(&self) {} } +/// Which instruments a feed is capable of pricing. +/// +/// Declarative rather than a predicate so the seeding side can push it down into a +/// `WHERE` clause instead of scanning the whole catalog. Every `None` means "don't +/// constrain on this" — a feed that leaves `venue` open prices its symbol on every +/// venue that lists it, which is the 1:n case (one crypto pair, several exchanges). +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct InstrumentFilter { + pub instrument_class: Option<&'static str>, + pub asset_class: Option<&'static str>, + pub venue: Option<&'static str>, +} + +/// How a feed names the instruments it prices. +/// +/// The counterpart to [`LiveQuoteFeed`]: that trait moves a vendor's *data*, this one +/// translates its *symbology*. Both live on the same struct so a vendor's naming rules +/// sit next to the code that speaks its protocol, rather than in a central table of +/// every feed's quirks. +/// +/// [`to_feed_symbol`](FeedSymbology::to_feed_symbol) is deliberately pure — no I/O, no +/// database — so the fiddly cases (OSI padding, suffixes, case) are unit-testable. +pub trait FeedSymbology: DataProvider { + /// The subset of the master catalog this feed can price. + fn candidates(&self) -> InstrumentFilter; + + /// Translate a master `instrument.symbol` into what this feed calls it. + /// + /// `None` means the feed cannot express that instrument; the caller skips it. + /// Returning `None` is not an error — it is how a feed declines a symbol that + /// passed [`candidates`](FeedSymbology::candidates) but is malformed for its + /// symbology. + fn to_feed_symbol(&self, symbol: &str) -> Option; +} + /// A source of live quotes. #[async_trait::async_trait] pub trait LiveQuoteFeed: DataProvider { - /// The `instrument_class` values this feed can quote. - /// - /// `code()` alone is not coverage: a vendor spans many datasets, and a feed is - /// one of them. Databento cross-references both equities and options, but the - /// OPRA.PILLAR session can only quote options — handing it an equity symbol - /// would at best return nothing and at worst have the gateway reject the whole - /// subscription. This is the minimum scope a driver needs to pick the right - /// held instruments; a per-feed dataset/schema config is the fuller answer when - /// one vendor runs several feeds. - fn covers(&self) -> &'static [&'static str]; /// Connect, subscribe `symbols`, and emit quotes until the session ends. /// /// `Ok(())` means the venue closed the stream cleanly; the supervisor diff --git a/crates/dataprovider/src/universe.rs b/crates/dataprovider/src/universe.rs deleted file mode 100644 index 73f3dfa..0000000 --- a/crates/dataprovider/src/universe.rs +++ /dev/null @@ -1,74 +0,0 @@ -//! The universe-discovery capability. -//! -//! A [`UniverseSource`] is a [`DataProvider`] that can enumerate seedable -//! universes, price them (free metadata call), and fetch their instrument -//! definitions as neutral [`InstrumentDef`]s. The seeder drives it; adding a -//! new vendor is a new `impl UniverseSource` — the seeding framework is untouched. - -use async_trait::async_trait; -use serde::{Deserialize, Serialize}; - -use crate::error::ProviderError; -use crate::instrument::InstrumentDef; -use crate::provider::DataProvider; - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -pub enum Category { - Equity, - Option, - Future, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -pub enum SType { - RawSymbol, - Parent, -} - -impl SType { - pub fn as_databento_str(self) -> &'static str { - match self { - SType::RawSymbol => "raw_symbol", - SType::Parent => "parent", - } - } -} - -/// A seedable slice of a provider's coverage. -/// -/// `symbols` empty ⇒ whole dataset (the provider translates this to its native -/// "all symbols" sentinel — e.g. `ALL_SYMBOLS` on Databento). -#[derive(Debug, Clone)] -pub struct UniverseSpec { - pub code: String, - pub description: Option, - pub category: Category, - pub dataset: String, - pub option_dataset: Option, - pub symbols: Vec, - pub stype_in: SType, - pub include_options: bool, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CostEstimate { - pub universe_code: String, - pub usd: f64, - pub symbol_count: Option, -} - -#[async_trait] -pub trait UniverseSource: DataProvider { - /// Universes this provider can seed. Impls may consult the DB catalog, a - /// static list, or the vendor's own metadata endpoint. - async fn discover(&self) -> Result, ProviderError>; - - /// Free dry-run price for a spec (metadata call on Databento). - async fn estimate_cost(&self, spec: &UniverseSpec) -> Result; - - /// The actual instrument records for a spec. - async fn fetch_definitions( - &self, - spec: &UniverseSpec, - ) -> Result, ProviderError>; -} diff --git a/db/access/ods.sql b/db/access/ods.sql index e405259..c1807e6 100644 --- a/db/access/ods.sql +++ b/db/access/ods.sql @@ -9,19 +9,15 @@ GRANT USAGE ON SCHEMA public TO oms_user; GRANT SELECT ON ALL TABLES IN SCHEMA public TO oms_user; --- Instrument seeding + symbology: the OMS app now seeds the master instrument --- universe itself (`oms setup universe` CLI and the POST /admin/universes/{code}/seed --- endpoint, src/setup/universe.rs), and its resolver stamps FIGI/CUSIP anchors from --- OpenFIGI. It fetches definitions from the provider, upserts instrument + --- instrument_derivative, and writes the universe seed-state (status, last_seeded_at, --- instrument_count) back on the catalog. So oms_user needs write on those tables. --- (SELECT on the FK targets venue/currency and the catalog is covered by the --- blanket public SELECT above; oms.instrument_xref lives in oms_user's own schema.) +-- Instrument seeding + symbology: the OMS app seeds the master instrument catalog +-- itself, broker-first. Broker sync (`oms setup sync-broker`) creates the master +-- public.instrument (+ instrument_derivative) rows and the broker_instrument mapping +-- in one pass; feed mapping (`oms setup map-feed`) writes feed_instrument; the +-- resolver stamps FIGI/CUSIP anchors from OpenFIGI. So oms_user needs write on the +-- master catalog and both mapping tables. (SELECT on the FK targets venue/currency +-- is covered by the blanket public SELECT above.) GRANT INSERT, UPDATE ON public.instrument, public.instrument_derivative TO oms_user; -GRANT UPDATE ON public.instrument_universe TO oms_user; --- Editing a universe's underlying/child symbol set (the cockpit checkbox picker) --- rewrites instrument_universe_symbol. -GRANT INSERT, DELETE ON public.instrument_universe_symbol TO oms_user; +GRANT INSERT, UPDATE, DELETE ON public.broker_instrument, public.feed_instrument TO oms_user; -- Every future master table mdm_master creates is readable by oms_user. ALTER DEFAULT PRIVILEGES FOR ROLE mdm_master IN SCHEMA public @@ -41,7 +37,6 @@ DO $$ BEGIN IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'market_user') THEN REVOKE ALL ON public.instrument, public.instrument_derivative FROM market_user; - REVOKE ALL ON public.instrument_universe, public.instrument_universe_symbol FROM market_user; REVOKE ALL ON public.venue, public.currency FROM market_user; REVOKE USAGE ON SCHEMA public FROM market_user; EXECUTE format('REVOKE CONNECT ON DATABASE %I FROM market_user', current_database()); diff --git a/db/migrations/ods/oms/0019_DROP_INSTRUMENT_XREF.sql b/db/migrations/ods/oms/0019_DROP_INSTRUMENT_XREF.sql new file mode 100644 index 0000000..41119a2 --- /dev/null +++ b/db/migrations/ods/oms/0019_DROP_INSTRUMENT_XREF.sql @@ -0,0 +1,9 @@ +-- Retire the unified instrument_xref. Its two responsibilities are now split into +-- purpose-built master tables: broker routing → public.broker_instrument (0016), +-- feed market-data mapping → public.feed_instrument (0017). Order routing, recon, +-- the quote feed, and the resolver all read the new tables; nothing reads the xref. +-- +-- provider_feed_policy stays (ranked failover across feeds); its source_code column +-- already holds feed codes (DATABENTO/BINANCE/BYBIT). + +DROP TABLE IF EXISTS instrument_xref; diff --git a/db/migrations/ods/public/0006_CREATE_PROVIDER_INSTRUMENT_TABLE.sql b/db/migrations/ods/public/0006_CREATE_PROVIDER_INSTRUMENT_TABLE.sql index 4027180..3099c2a 100644 --- a/db/migrations/ods/public/0006_CREATE_PROVIDER_INSTRUMENT_TABLE.sql +++ b/db/migrations/ods/public/0006_CREATE_PROVIDER_INSTRUMENT_TABLE.sql @@ -26,7 +26,7 @@ CREATE TABLE provider_instrument ( -- The stable resolver key. A provider symbol resolves to one master instrument -- per exchange — a consolidated provider lists the same symbol across venues -- (SPY on ARCX, XNAS, …), each a distinct (symbol, venue) master instrument, - -- so the exchange is part of the key. Mirrors Nautilus's Symbol@Venue identity. + -- so the exchange is part of the key, mirroring Symbol@Venue identity. CONSTRAINT provider_instrument_provider_symbol_uq UNIQUE (provider_code, provider_symbol, provider_exchange) ); diff --git a/db/migrations/ods/public/0015_CRYPTO_SYMBOL_IS_THE_PAIR.sql b/db/migrations/ods/public/0015_CRYPTO_SYMBOL_IS_THE_PAIR.sql index 3c44bb7..b512b16 100644 --- a/db/migrations/ods/public/0015_CRYPTO_SYMBOL_IS_THE_PAIR.sql +++ b/db/migrations/ods/public/0015_CRYPTO_SYMBOL_IS_THE_PAIR.sql @@ -10,9 +10,9 @@ -- Name the pair, as the venues do. Binance and Bybit both call it SOLUSDT, and our -- own xref already stores SOLUSDT as external_symbol on the BROKER and PROVIDER -- rows -- instrument.symbol was the only place in the system calling it SOL. --- Matches NautilusTrader's raw_symbol convention (SOLUSDT.BINANCE) and CCXT's --- rationale for putting base/quote in the symbol: contracts that differ must have --- distinct symbols. +-- Matches the raw-symbol convention used across the ecosystem (SOLUSDT.BINANCE) +-- and CCXT's rationale for putting base/quote in the symbol: contracts that differ +-- must have distinct symbols. -- -- currency stays USDT, so the pair remains decomposable and the quote asset is -- still first-class for settlement/valuation. The base asset is unchanged on the diff --git a/db/migrations/ods/public/0016_CREATE_BROKER_INSTRUMENT_TABLE.sql b/db/migrations/ods/public/0016_CREATE_BROKER_INSTRUMENT_TABLE.sql new file mode 100644 index 0000000..e6197be --- /dev/null +++ b/db/migrations/ods/public/0016_CREATE_BROKER_INSTRUMENT_TABLE.sql @@ -0,0 +1,52 @@ +-- Broker symbology + execution mapping: master instrument_id <-> a broker's own +-- symbol/handle (Alpaca, IBKR, Binance). Re-introduces broker_instrument as the +-- authoritative home for the broker half of the mapping, replacing the source_type +-- discriminated oms.instrument_xref (dropped in a later migration once routing and +-- the resolver read this table instead). +-- +-- This is also the *seeding* source of truth: broker sync (`oms setup sync-broker`) +-- upserts one row per tradeable instrument the broker offers, having already created +-- the master public.instrument row in the same pass. So a row here means "this +-- broker can route this instrument", and its existence is what makes an instrument +-- tradeable — there is no priceable-but-not-tradeable path. + +CREATE TABLE broker_instrument ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + instrument_id BIGINT NOT NULL REFERENCES instrument(id) ON DELETE CASCADE, + broker_code TEXT NOT NULL, -- ALPACA | IBKR | BINANCE + broker_symbol TEXT NOT NULL, -- the broker's own handle (Alpaca ticker, compact OSI, pair) + broker_exchange TEXT, -- the broker's exchange label, when it exposes one + -- Broker-native instrument id used for order routing. broker_symbol/exchange are + -- the human-facing mapping; the value we actually route on is the broker's own + -- instrument handle (IBKR conId, Alpaca asset UUID, …): instrument_id -> + -- broker_instrument[broker] -> native_id -> broker API. Nullable — options route + -- by symbol and expose no stable native id. + native_id TEXT, + is_tradeable BOOLEAN NOT NULL DEFAULT true, + -- Broker-enforced order limits (vary per broker; the binding constraint at routing time). + min_quantity NUMERIC, + max_quantity NUMERIC, + min_notional NUMERIC, + max_notional NUMERIC, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (instrument_id, broker_code), + -- The routing lookup key: broker_symbol is how order entry addresses the + -- instrument. Unique per (broker, symbol, exchange) — a consolidated broker can + -- list the same symbol on several venues. + -- + -- native_id is deliberately NOT unique: for Alpaca it is a per-instrument asset + -- UUID, but for Binance it is the base asset, which repeats across quote pairs + -- (BTCUSDT and BTCUSDC both base BTC). It is a routing/recon attribute, not a key. + CONSTRAINT broker_instrument_broker_symbol_uq UNIQUE (broker_code, broker_symbol, broker_exchange), + CHECK (max_quantity IS NULL OR min_quantity IS NULL OR max_quantity >= min_quantity), + CHECK (max_notional IS NULL OR min_notional IS NULL OR max_notional >= min_notional) +); + +CREATE INDEX idx_broker_instrument_broker ON broker_instrument(broker_code); +CREATE INDEX idx_broker_instrument_instrument ON broker_instrument(instrument_id); + +COMMENT ON TABLE broker_instrument IS + 'Broker symbology + execution mapping: master instrument_id <-> a broker''s handle. Seeded by broker sync; its existence makes an instrument tradeable.'; +COMMENT ON COLUMN broker_instrument.native_id IS + 'Broker-native instrument id used for order routing (e.g. IBKR conId, Alpaca asset UUID).'; diff --git a/db/migrations/ods/public/0017_CREATE_FEED_INSTRUMENT_TABLE.sql b/db/migrations/ods/public/0017_CREATE_FEED_INSTRUMENT_TABLE.sql new file mode 100644 index 0000000..6fc5779 --- /dev/null +++ b/db/migrations/ods/public/0017_CREATE_FEED_INSTRUMENT_TABLE.sql @@ -0,0 +1,33 @@ +-- Market-data mapping: a data feed's own symbol -> master instrument(s). The feed +-- half of what oms.instrument_xref conflated, given its own home and an explicit, +-- deliberately 1:n shape. +-- +-- 1:n by design. One feed symbol may price several instruments: a BTC/USDT feed can +-- mark BTCUSDT on every venue that trades it, so the identity is the full +-- (feed_code, feed_symbol, instrument_id) triple, not (feed_code, feed_symbol). +-- Independent of brokers — a feed maps onto whatever instruments exist, regardless +-- of which broker seeded them. +-- +-- Drives two things: subscription (instrument_id -> feed_symbol for held +-- instruments, in quote_feed) and inbound resolution (feed_symbol -> instrument_id, +-- when a quote arrives). Ranked failover across feeds stays in provider_feed_policy. + +CREATE TABLE feed_instrument ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + feed_code TEXT NOT NULL, -- DATABENTO | BINANCE | BYBIT + feed_symbol TEXT NOT NULL, -- the feed's own symbol (SOLUSDT, OSI, …) + instrument_id BIGINT NOT NULL REFERENCES instrument(id) ON DELETE CASCADE, + is_active BOOLEAN NOT NULL DEFAULT true, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + -- Full triple: one feed symbol -> many instruments is allowed; the same + -- (feed, symbol, instrument) mapping is not duplicated. + UNIQUE (feed_code, feed_symbol, instrument_id) +); + +CREATE INDEX idx_feed_instrument_feed ON feed_instrument(feed_code); +CREATE INDEX idx_feed_instrument_instrument ON feed_instrument(instrument_id); +CREATE INDEX idx_feed_instrument_lookup ON feed_instrument(feed_code, feed_symbol); + +COMMENT ON TABLE feed_instrument IS + 'Market-data mapping: a feed''s own symbol -> master instrument(s). 1:n; independent of brokers.'; diff --git a/db/migrations/ods/public/0018_DROP_INSTRUMENT_UNIVERSE.sql b/db/migrations/ods/public/0018_DROP_INSTRUMENT_UNIVERSE.sql new file mode 100644 index 0000000..01913d8 --- /dev/null +++ b/db/migrations/ods/public/0018_DROP_INSTRUMENT_UNIVERSE.sql @@ -0,0 +1,8 @@ +-- Retire the Databento-dataset universe catalog. Instrument seeding is now +-- broker-first (`oms setup sync-broker`): a broker adapter's InstrumentProvider is +-- the source of the master catalog, and feed mapping (`oms setup map-feed`) is the +-- source of feed_instrument. Nothing reads instrument_universe any more (the CLI +-- seeder and the /admin/universes endpoints were removed). + +DROP TABLE IF EXISTS instrument_universe_symbol; +DROP TABLE IF EXISTS instrument_universe; diff --git a/db/migrations/ods/public/0019_DROP_BROKER_INSTRUMENT_NATIVE_UQ.sql b/db/migrations/ods/public/0019_DROP_BROKER_INSTRUMENT_NATIVE_UQ.sql new file mode 100644 index 0000000..e8e1e26 --- /dev/null +++ b/db/migrations/ods/public/0019_DROP_BROKER_INSTRUMENT_NATIVE_UQ.sql @@ -0,0 +1,7 @@ +-- Drop the (broker_code, native_id) uniqueness on broker_instrument. It held for +-- Alpaca (native_id = per-instrument asset UUID) but is wrong for Binance, where +-- native_id is the base asset and repeats across quote pairs (BTCUSDT and BTCUSDC +-- both base BTC). native_id is a routing/recon attribute, not a key; the real keys +-- are (instrument_id, broker_code) and (broker_code, broker_symbol, broker_exchange). + +ALTER TABLE broker_instrument DROP CONSTRAINT IF EXISTS broker_instrument_broker_native_uq; diff --git a/db/scripts/seed.sh b/db/scripts/seed.sh index d389e3e..702f971 100755 --- a/db/scripts/seed.sh +++ b/db/scripts/seed.sh @@ -38,12 +38,12 @@ psql_db "$ODS_DB" -f scripts/seed_currencies.sql echo "→ seeding venues (ISO 10383 MIC registry)" python3 scripts/seed_venues.py --source data/ISO10383_MIC.csv -echo "→ seeding instrument universes (seeder catalog)" -psql_db "$ODS_DB" -f scripts/seed_universes.sql +echo "→ seeding crypto exchange venues (synthetic, non-MIC)" +psql_db "$ODS_DB" -f scripts/seed_crypto_venues.sql -# Instruments + broker/provider mappings are seeded on demand (not here, not -# scheduled): `make seed-instruments` (Databento) and `make sync-brokers` (Alpaca), -# or `make db-fixtures` for the no-creds minimal set. They depend on currency + -# venue above. +# Instruments are seeded on demand, broker-first (not here, not scheduled): +# `make sync-broker BROKER=alpaca` creates the master instrument + broker_instrument +# rows, then `make map-feed FEED=databento` maps a data feed onto them. Or +# `make db-fixtures` for the no-creds minimal set. All depend on currency + venue. echo "post-deployment seeding complete." diff --git a/db/scripts/seed_binance_crypto.sql b/db/scripts/seed_binance_crypto.sql deleted file mode 100644 index 6fb7d47..0000000 --- a/db/scripts/seed_binance_crypto.sql +++ /dev/null @@ -1,86 +0,0 @@ --- Seed a minimal Binance Spot (testnet) crypto trading setup: a venue, the USDT --- quote currency, a few spot pairs as instruments, their BROKER xref rows (order --- routing + recon), and the broker_connection. Idempotent. --- --- Crypto instrument model: symbol = the PAIR (BTCUSDT), currency = quote (USDT), --- venue = BINANCE. The symbol names the pair because that is what the instrument --- is -- symbol = base alone cannot represent BTC/USDT and BTC/BTC-quote variants --- at once under UNIQUE (symbol, venue), and every venue calls it BTCUSDT anyway. --- The xref carries that pair as external_symbol (order routing + market data) and --- the base asset as external_native_id (BTC, what account balances report and --- reconciliation matches on). --- --- Account creation (principal/portfolio/account routing to binance-paper) is left --- to the admin/cockpit — this seeds only the shared reference data. - --- Venue + quote currency. -INSERT INTO venue (code, name, country, status) -VALUES ('BINANCE', 'Binance', NULL, 'ACTIVE') -ON CONFLICT (code) DO NOTHING; - -INSERT INTO currency (code, name, minor_units, is_active) -VALUES ('USDT', 'Tether USD', 8, true) -ON CONFLICT (code) DO NOTHING; - --- Spot pairs (base asset vs USDT). Microstructure roughly matches Binance spot. -INSERT INTO instrument - (symbol, venue, name, asset_class, instrument_class, currency, status, - price_precision, size_precision, price_increment, size_increment) -VALUES - ('BTCUSDT', 'BINANCE', 'Bitcoin', 'CRYPTO', 'SPOT', 'USDT', 'ACTIVE', 2, 5, 0.01, 0.00001), - ('ETHUSDT', 'BINANCE', 'Ethereum', 'CRYPTO', 'SPOT', 'USDT', 'ACTIVE', 2, 4, 0.01, 0.0001), - ('SOLUSDT', 'BINANCE', 'Solana', 'CRYPTO', 'SPOT', 'USDT', 'ACTIVE', 2, 3, 0.01, 0.001) -ON CONFLICT (symbol, venue) DO NOTHING; - --- BROKER xref: external_symbol = Binance pair (order routing); --- external_native_id = base asset (recon match); is_tradeable + limits for routing. -INSERT INTO oms.instrument_xref - (instrument_id, source_type, source_code, external_symbol, external_native_id, - method, confidence, is_tradeable, min_quantity, min_notional) -SELECT i.id, 'BROKER', 'BINANCE', x.pair, x.base, 'manual', 'resolved', true, x.min_qty, 5 -FROM (VALUES - ('BTC', 'BTCUSDT', 0.00001), - ('ETH', 'ETHUSDT', 0.0001), - ('SOL', 'SOLUSDT', 0.001) -) AS x(base, pair, min_qty) -JOIN instrument i ON i.symbol = x.pair AND i.venue = 'BINANCE' -ON CONFLICT DO NOTHING; - --- PROVIDER xref: the same pair, but as a market-data identity rather than a --- routing one. Without these rows the crypto is tradable but unmarkable — the --- quote feed has no way to know Binance can price it, so /positions reports a --- null mark for a position we can freely trade. --- --- external_symbol is the pair as Binance's market-data streams report it (the `s` --- field of bookTicker, uppercase); external_exchange is the venue, matching --- instrument.venue. No native_id: the feed resolves by symbol, and Binance's --- market data has no separate id worth storing. -INSERT INTO oms.instrument_xref - (instrument_id, source_type, source_code, external_symbol, external_exchange, - method, confidence) -SELECT i.id, 'PROVIDER', 'BINANCE', x.pair, 'BINANCE', 'manual', 'resolved' -FROM (VALUES - ('BTC', 'BTCUSDT'), - ('ETH', 'ETHUSDT'), - ('SOL', 'SOLUSDT') -) AS x(base, pair) -JOIN instrument i ON i.symbol = x.pair AND i.venue = 'BINANCE' -ON CONFLICT DO NOTHING; - --- A SECOND market-data source for the same pairs. Bybit quotes the identical --- symbol; ranked below Binance in provider_feed_policy, it is the failover feed --- when Binance goes quiet. external_symbol is the same pair string (Bybit's v5 --- orderbook.1 reports it identically); external_exchange = BYBIT so the two --- PROVIDER rows are distinct identities for one instrument. -INSERT INTO oms.instrument_xref - (instrument_id, source_type, source_code, external_symbol, external_exchange, - method, confidence) -SELECT i.id, 'PROVIDER', 'BYBIT', i.symbol, 'BYBIT', 'manual', 'resolved' -FROM instrument i -WHERE i.asset_class = 'CRYPTO' AND i.instrument_class = 'SPOT' -ON CONFLICT DO NOTHING; - --- Routing target. Credentials resolved from env (BINANCE_PAPER_API_KEY/SECRET). -INSERT INTO oms.broker_connection (code, broker_code, environment, status) -VALUES ('binance-paper', 'BINANCE', 'PAPER', 'ACTIVE') -ON CONFLICT (code) DO NOTHING; diff --git a/db/scripts/seed_crypto_venues.sql b/db/scripts/seed_crypto_venues.sql new file mode 100644 index 0000000..7e64895 --- /dev/null +++ b/db/scripts/seed_crypto_venues.sql @@ -0,0 +1,18 @@ +-- Crypto exchange venues. These are NOT in the ISO 10383 MIC registry (seed_venues.py +-- only loads real MICs), so they must be seeded explicitly. Synthetic codes, mic NULL. +-- +-- Broker-first seeding sets a crypto instrument's venue to the exchange code +-- (e.g. Binance pairs -> venue 'BINANCE'), so this venue must exist before +-- `oms setup sync-broker --broker binance` runs, or every pair fails the venue FK. +-- BYBIT is seeded too so a future Bybit broker (or venue-attributed feed) has a home. +-- Run as mdm_master (owner of public), matching seed_currencies.sql. + +SET ROLE mdm_master; +SET search_path TO public; + +INSERT INTO venue (code, name, mic, status) VALUES + ('BINANCE', 'Binance', NULL, 'ACTIVE'), + ('BYBIT', 'Bybit', NULL, 'ACTIVE') +ON CONFLICT (code) DO NOTHING; + +RESET ROLE; diff --git a/db/scripts/seed_currencies.sql b/db/scripts/seed_currencies.sql index 867ecd4..a69c5aa 100644 --- a/db/scripts/seed_currencies.sql +++ b/db/scripts/seed_currencies.sql @@ -22,4 +22,20 @@ INSERT INTO currency (code, name, numeric_code, minor_units) VALUES ('DKK', 'Danish Krone', '208', 2) ON CONFLICT (code) DO NOTHING; +-- Crypto quote assets (not ISO 4217). Binance/Bybit pairs quote in these, so they +-- must exist for a crypto pair's currency FK. numeric_code is NULL (no ISO code). +INSERT INTO currency (code, name, numeric_code, minor_units) VALUES + ('USDT', 'Tether USD', NULL, 8), + ('USDC', 'USD Coin', NULL, 8), + ('BUSD', 'Binance USD', NULL, 8), + ('FDUSD','First Digital USD', NULL, 8), + ('TUSD', 'TrueUSD', NULL, 8), + ('DAI', 'Dai', NULL, 8), + ('BTC', 'Bitcoin', NULL, 8), + ('ETH', 'Ether', NULL, 8), + ('BNB', 'BNB', NULL, 8), + ('TRY', 'Turkish Lira', '949', 2), + ('BRL', 'Brazilian Real', '986', 2) +ON CONFLICT (code) DO NOTHING; + RESET ROLE; diff --git a/db/scripts/seed_live.sh b/db/scripts/seed_live.sh new file mode 100755 index 0000000..3a740ed --- /dev/null +++ b/db/scripts/seed_live.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# +# One-shot live instrument seeding: sync every broker whose creds are configured, +# then map every data feed onto the seeded instruments. Idempotent (every step +# upserts), so it is safe to re-run — on a schedule (cron) or by hand whenever the +# catalog should refresh (new listings, new option expiries). NOT run at server +# startup: it hits external broker APIs and can be slow. +# +# Steps whose creds are absent are skipped with a note, so a partial setup (e.g. +# Alpaca only) still works. Feed mapping needs no external creds — it maps whatever +# instruments already exist — but a feed with nothing seeded simply maps zero rows. +# +# Env knobs: +# OPTION_UNDERLYINGS comma-separated option underlyings for Alpaca (default: none) +# ENRICH=1 run the OpenFIGI enrichment pass (FIGI/CUSIP on the master). +# Off by default — it is the slow phase and not needed to trade. +# +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$ROOT" + +if [[ -f .env ]]; then + set -a + # shellcheck disable=SC1091 + . ./.env + set +a +fi + +run() { echo "→ oms setup $*"; cargo run --quiet -- setup "$@"; } +have() { [[ -n "${!1:-}" ]]; } + +# Enrichment is opt-in (ENRICH=1); off by default it is the slow phase. +enrich_flag="--no-enrich" +[[ -n "${ENRICH:-}" ]] && enrich_flag="" + +env_name_upper() { echo "${1:-PAPER}" | tr '[:lower:]' '[:upper:]'; } + +# --- Brokers: the instrument source. --- +alpaca_env="$(env_name_upper "${ALPACA_ENV:-PAPER}")" +if have "ALPACA_${alpaca_env}_API_KEY" && have "ALPACA_${alpaca_env}_API_SECRET"; then + run sync-broker --broker alpaca $enrich_flag ${OPTION_UNDERLYINGS:+--underlyings "$OPTION_UNDERLYINGS"} +else + echo "· skip alpaca sync-broker (ALPACA_${alpaca_env}_API_KEY/SECRET not set)" +fi + +binance_env="$(env_name_upper "${BINANCE_ENV:-PAPER}")" +if have "BINANCE_${binance_env}_API_KEY" && have "BINANCE_${binance_env}_PRIVATE_KEY_PATH"; then + run sync-broker --broker binance $enrich_flag +else + echo "· skip binance sync-broker (BINANCE_${binance_env}_API_KEY/PRIVATE_KEY_PATH not set)" +fi + +# --- Feeds: map onto the seeded instruments (no external creds needed). --- +# Databento only prices options that were seeded above; skip if no key was ever +# configured (the feed itself won't run without it either). +if have DATABENTO_API_KEY; then + run map-feed --feed databento +else + echo "· skip databento map-feed (DATABENTO_API_KEY not set)" +fi +run map-feed --feed binance +run map-feed --feed bybit + +echo "✅ live seeding complete." diff --git a/db/scripts/seed_universes.sql b/db/scripts/seed_universes.sql deleted file mode 100644 index 80757b3..0000000 --- a/db/scripts/seed_universes.sql +++ /dev/null @@ -1,39 +0,0 @@ --- Catalog of the main Databento datasets, exposed as selectable instrument --- universes. Everything is disabled — the user opts in by running the seeder's --- interactive picker (`make seed-instruments`) and ticking the wanted universes. --- --- Idempotent: ON CONFLICT DO NOTHING so re-running db-seed never clobbers the --- user's `enabled` toggles or seed-state. A whole-dataset universe (no rows in --- instrument_universe_symbol) seeds every symbol (ALL_SYMBOLS). --- --- NOTE: v1 seeding handles category=EQUITY. OPTION/FUTURE datasets are catalogued --- here for discoverability but the loader warns + skips them until per-category --- mapping lands. - -INSERT INTO instrument_universe - (code, description, category, dataset, option_dataset, stype_in, include_options) -VALUES - -- US equities — exchange feeds - ('XNAS_ITCH', 'Nasdaq TotalView-ITCH', 'EQUITY', 'XNAS.ITCH', NULL, 'raw_symbol', false), - ('XNAS_BASIC', 'Nasdaq Basic', 'EQUITY', 'XNAS.BASIC', NULL, 'raw_symbol', false), - ('XBOS_ITCH', 'Nasdaq BX (TotalView-ITCH)', 'EQUITY', 'XBOS.ITCH', NULL, 'raw_symbol', false), - ('XPSX_ITCH', 'Nasdaq PSX (TotalView-ITCH)', 'EQUITY', 'XPSX.ITCH', NULL, 'raw_symbol', false), - ('XNYS_PILLAR', 'NYSE (Pillar)', 'EQUITY', 'XNYS.PILLAR', NULL, 'raw_symbol', false), - ('XASE_PILLAR', 'NYSE American (Pillar)', 'EQUITY', 'XASE.PILLAR', NULL, 'raw_symbol', false), - ('XCHI_PILLAR', 'NYSE Chicago (Pillar)', 'EQUITY', 'XCHI.PILLAR', NULL, 'raw_symbol', false), - ('XCIS_TRADESBBO', 'NYSE National (Trades & BBO)', 'EQUITY', 'XCIS.TRADESBBO', NULL, 'raw_symbol', false), - ('ARCX_PILLAR', 'NYSE Arca (Pillar)', 'EQUITY', 'ARCX.PILLAR', NULL, 'raw_symbol', false), - ('IEXG_TOPS', 'IEX (TOPS)', 'EQUITY', 'IEXG.TOPS', NULL, 'raw_symbol', false), - ('EPRL_DOM', 'MIAX Pearl Equities (Depth)', 'EQUITY', 'EPRL.DOM', NULL, 'raw_symbol', false), - -- US equities — consolidated - ('EQUS_SUMMARY', 'Databento US Equities Summary', 'EQUITY', 'EQUS.SUMMARY', NULL, 'raw_symbol', false), - -- Options - ('OPRA_PILLAR', 'OPRA — all US options', 'OPTION', 'OPRA.PILLAR', NULL, 'raw_symbol', false), - -- Futures - ('GLBX_MDP3', 'CME Globex (MDP 3.0)', 'FUTURE', 'GLBX.MDP3', NULL, 'raw_symbol', false), - ('IFEU_IMPACT', 'ICE Futures Europe (iMpact)', 'FUTURE', 'IFEU.IMPACT', NULL, 'raw_symbol', false), - ('NDEX_IMPACT', 'ICE Endex (iMpact)', 'FUTURE', 'NDEX.IMPACT', NULL, 'raw_symbol', false) -ON CONFLICT (code) DO NOTHING; - --- Option universes (e.g. OPRA_PILLAR) get their underlyings from the cockpit --- picker at seed time, written to instrument_universe_symbol; none are seeded here. diff --git a/docs/tutorials/opra-marks-pnl.md b/docs/tutorials/opra-marks-pnl.md new file mode 100644 index 0000000..a586d24 --- /dev/null +++ b/docs/tutorials/opra-marks-pnl.md @@ -0,0 +1,468 @@ +# Tutorial: OPRA live marks → unrealized P&L + +Implement mark-to-market valuation: feed Databento OPRA live quotes into a marks +store, then value open positions (unrealized P&L). The heavy accounting already +exists — `position` carries `net_qty` + `avg_cost` and `src/positions.rs` +(`PositionMath`) does average-cost accounting + realized P&L on every fill. You +are adding **one missing input (the mark)** plus a feed and a valuation join. + +Options-first (matches the basic OPRA subscription). Equities/crypto marks can +feed the same store later. + +## Mental model + +``` +Databento OPRA live (bbo, held symbols) + → opra_stream task ──writes──► marks store (Arc in AppState) + ├─► /positions valuation (net_qty + avg_cost + mark) + └─► market-order pre-trade risk (est_price) +``` + +Two new things (marks store, feed) + one join (valuation). + +Patterns to mirror that already exist in the repo: +- `src/stream_health.rs` — `Arc>` behind a cheap-clone handle. +- `src/binance_stream.rs` — reconnect/backoff stream task + `stream_health` badge. +- `crates/dataprovider/src/providers/databento.rs` — fixed-point price decoding. + +--- + +## Step 1 — The marks store + +New file `src/marks.rs` (shape copied from `stream_health.rs`): + +```rust +use std::collections::HashMap; +use std::sync::{Arc, RwLock}; +use chrono::{DateTime, Utc}; + +#[derive(Clone, Copy)] +pub struct Mark { + pub bid: f64, + pub ask: f64, + pub ts: DateTime, +} +impl Mark { + pub fn mid(&self) -> f64 { (self.bid + self.ask) / 2.0 } +} + +#[derive(Clone, Default)] +pub struct MarksStore { + inner: Arc>>, // key = our instrument.id +} + +impl MarksStore { + pub fn new() -> Self { Self::default() } + + pub fn set(&self, instrument_id: i64, bid: f64, ask: f64) { + if let Ok(mut m) = self.inner.write() { + m.insert(instrument_id, Mark { bid, ask, ts: Utc::now() }); + } + } + + pub fn get(&self, instrument_id: i64) -> Option { + self.inner.read().ok().and_then(|m| m.get(&instrument_id).copied()) + } + + /// Snapshot for a bulk valuation query. + pub fn all(&self) -> HashMap { + self.inner.read().map(|m| m.clone()).unwrap_or_default() + } +} +``` + +Start in-memory. Add an `oms.instrument_mark` table later only if you need +persistence/EOD — the store interface won't change. + +--- + +## Step 2 — Put it in `AppState` + +In `src/app_state.rs`, mirror the `stream_health` wiring: field + init + accessor. + +```rust +// field +marks: MarksStore, +// in new(): +marks: MarksStore::new(), +// accessor: +pub fn marks(&self) -> &MarksStore { &self.marks } +``` + +Add `mod marks;` in `src/main.rs`. + +--- + +## Step 3 — Enable the Databento `live` dependency + +The `databento` crate (0.54) has a `live` module; the OMS binary doesn't depend +on it directly yet (only `crates/dataprovider` uses `historical`). Add to the +**root** `Cargo.toml`: + +```toml +databento = { version = "0.54", default-features = false, features = ["live"] } +``` + +Verify exact item names for your version: `cargo doc -p databento --no-deps --open`. + +--- + +## Step 4 — The OPRA stream task + +New file `src/opra_stream.rs`, structured like `binance_stream.rs`: a `run()` +with a reconnect loop, using a `stream_health::StreamHandle` for the cockpit badge. + +### 4a. Load the symbol → instrument_id map + +Databento OPRA `raw_symbol` is the space-padded OSI, stored verbatim in +`instrument.symbol` — no transform needed. + +```rust +let rows = sqlx::query( + "SELECT i.symbol, i.id \ + FROM position p \ + JOIN instrument i ON i.id = p.instrument_id \ + WHERE p.net_qty <> 0 AND i.instrument_class = 'OPTION'", +).fetch_all(&pool).await?; + +let sym_to_id: HashMap = rows.iter() + .map(|r| (r.get::("symbol"), r.get::("id"))) + .collect(); +let symbols: Vec = sym_to_id.keys().cloned().collect(); +``` + +### 4b. Connect + subscribe + +Use `bbo-1s` (1-second top-of-book — tiny volume, ideal for marks): + +```rust +use databento::{ + dbn::{Schema, SType}, + live::Subscription, + LiveClient, +}; + +let mut client = LiveClient::builder() + .key_from_env()? // DATABENTO_API_KEY + .dataset("OPRA.PILLAR") + .build().await?; + +client.subscribe( + Subscription::builder() + .symbols(symbols) // the held OSI strings + .schema(Schema::Bbo1S) // or Schema::Mbp1 for every top-of-book tick + .stype_in(SType::RawSymbol) + .build(), +).await?; + +client.start().await?; +health.set_live(); +``` + +### 4c. Decode loop + +Databento sends a `SymbolMappingMsg` (numeric `instrument_id` ↔ `raw_symbol`), +then quote records keyed by that numeric id. Keep a `dbn_id → our_id` map. +Prices are fixed-point `i64` scaled by `1e-9`; `i64::MAX` = undefined. + +```rust +const UNDEF: i64 = i64::MAX; +fn px(v: i64) -> Option { (v != UNDEF).then(|| v as f64 / 1e9) } + +let mut dbn_to_id: HashMap = HashMap::new(); + +while let Some(rec) = client.next_record().await? { + health.record_event(); + + if let Some(sm) = rec.get::() { + if let Ok(osi) = sm.stype_out_symbol() { + if let Some(&our_id) = sym_to_id.get(osi) { + dbn_to_id.insert(sm.hd.instrument_id, our_id); + } + } + continue; + } + + // bbo-1s decodes to a BBO record; verify the exact struct name in your dbn version. + if let Some(q) = rec.get::() { + let level = &q.levels[0]; // top of book + if let (Some(bid), Some(ask)) = (px(level.bid_px), px(level.ask_px)) { + if let Some(&our_id) = dbn_to_id.get(&q.hd.instrument_id) { + marks.set(our_id, bid, ask); + } + } + } +} +``` + +Wrap 4b–4c in `connect_and_run()`; loop with backoff + `health.set_down()` on +error — copy `binance_stream.rs::run` almost verbatim. + +> The one API detail to confirm in dbn 0.54: the record struct for `bbo-1s` +> (may be `BboMsg`/`Bbo1SMsg`) and that `.levels[0].bid_px/ask_px` are the fields. +> `Mbp1Msg` (`Schema::Mbp1`) is the certain fallback — same `levels[0]` shape, +> higher volume. + +--- + +## Step 5 — Spawn it + +In `src/main.rs`, next to the Binance spawn, guarded by the key: + +```rust +if env::var("DATABENTO_API_KEY").is_ok() { + let health = state.stream_health().handle("DATABENTO", "OPRA"); + tokio::spawn(opra_stream::run(state.pool().clone(), state.marks().clone(), health)); +} +``` + +The cockpit stream-health strip then shows `DATABENTO/OPRA live` for free. + +--- + +## Step 6 — Pick up newly-opened positions + +**The problem.** The subscription is built once, at connect time, from what you +hold *right then* (step 4a). Buy an option an hour later and the stream has never +heard of it — no quotes, no mark, no unrealized P&L for that position, until you +restart the OMS. + +**The fix.** Have the fill path ring a doorbell when it applies a fill; the stream +wakes, re-reads the held set, and subscribes the difference. No timer: the task +sleeps until an actual fill happens. + +An **mpsc** is a queue between tasks: many `Sender`s (cloneable), one `Receiver`. +`rx.recv().await` parks the task until someone sends — that's what removes the +timer. The payload is `()`: a doorbell, not a letter. We don't send *which* +instrument, because the stream re-reads the held set from the DB anyway — sending +data would mean trusting the message; sending a nudge means trusting Postgres. + +Known limitation: this only fires for fills **this process** handled. A position +opened by another node or by hand-written SQL won't nudge, and stays unmarked +until the next reconnect. (Fix later, if it matters, with a Postgres +`LISTEN/NOTIFY` trigger on `position` feeding the same channel — 6c doesn't change.) + +### 6a. Ring the doorbell on a fill + +`src/execution.rs` — one extra param, so no caller can forget it: + +```rust +use tokio::sync::mpsc; + +pub async fn process_execution_report( + pool: &PgPool, + kafka: &Option, + order_id: Uuid, + report: ExecutionReport, + actor: &str, + marks_nudge: Option<&mpsc::Sender<()>>, +) -> Result<(), Box> { + const MAX_ATTEMPTS: u32 = 5; + for attempt in 1..=MAX_ATTEMPTS { + match apply_once(pool, kafka, order_id, report.clone(), actor).await { + Ok(()) => { + // A fill moved the position — tell the marks feed to re-read the + // held set. After apply_once, so the row is committed before the + // stream can query it. try_send never blocks the fill path. + if matches!(report, ExecutionReport::Fill { .. }) { + if let Some(tx) = marks_nudge { + let _ = tx.try_send(()); + } + } + return Ok(()); + } + /* … unchanged … */ + } + } + unreachable!() +} +``` + +Ordering matters: nudge **after** `apply_once` returns `Ok`, never before. It +commits the transaction, so a nudge sent earlier could wake the stream to a query +that can't see the new position yet. + +### 6b. Thread the sender to the fill streams + +`Sender` is `Clone`, so each stream gets its own. Four call sites, two per broker +(the live stream + the startup `reconcile_routed_orders` catch-up): + +- `src/alpaca_stream.rs` — `run` → `connect_and_run` → `handle_trade_update` + (`:288`), and `reconcile_routed_orders` (`:127`). +- `src/binance_stream.rs` — `run` → `connect_and_run` → `handle_execution_report` + (`:194`), and `reconcile_routed_orders` (`:264`). + +Each `run()` and the helpers take `marks_nudge: Option<&mpsc::Sender<()>>` (or an +owned `Option>` on `run`) and pass it straight through. +`Option` so a broker that runs without the marks feed configured just passes `None`. + +`src/main.rs` — create the channel and wire both ends. `AppState` doesn't need it: +handlers submit orders, they don't apply fills. + +```rust +// bounded(1): a queued nudge already means "reload", so extras are redundant. +let (marks_tx, marks_rx) = tokio::sync::mpsc::channel::<()>(1); + +// … each fill-stream spawn gets `Some(marks_tx.clone())` … + +if env::var("DATABENTO_API_KEY").map(|k| !k.is_empty()).unwrap_or(false) { + let health = state.stream_health().handle("DATABENTO", "OPRA"); + tokio::spawn(opra_stream::run(state.pool().clone(), state.marks().clone(), health, marks_rx)); +} +``` + +### 6c. Race the doorbell against the quote stream + +The decode loop becomes a flat `loop` + `select!`. Two rules drive the shape: + +- **`subscribe` is NOT cancel-safe** — the crate warns a dropped partial send makes + the gateway reject it and close the connection. So it must never be a `select!` + branch. Queue the symbols, subscribe at the top of the loop where nothing else + borrows `client`. (This also settles the borrow checker: only `next_record` + takes `&mut client`.) +- **`dbn_to_id` must live in `connect_and_run`**, not a helper fn. If it lived + inside a future that `select!` can drop, losing that branch would take the + session's id map with it. + +```rust +// `rx` is owned by run() and borrowed into each session: the doorbell outlives +// reconnects, since it has nothing to do with the Databento session. +pub async fn run(pool: PgPool, marks: MarkStore, health: StreamHandle, mut rx: mpsc::Receiver<()>) { + let mut backoff_secs: u64 = 1; + loop { + health.set_connecting(); + match connect_and_run(&pool, &marks, &health, &mut rx).await { + /* … unchanged … */ + } + } +} +``` + +```rust + let mut sym_to_id = load_held_option_symbols(pool).await?; // now `mut` + // … build client, subscribe the initial set, start(), set_live() … + + let mut dbn_to_id: HashMap = HashMap::new(); + let mut pending: Vec = Vec::new(); + + loop { + // Outside the select: `subscribe` is not cancel-safe. + if !pending.is_empty() { + info!(count = pending.len(), "OPRA stream: subscribing newly-held options"); + client + .subscribe( + Subscription::builder() + .symbols(std::mem::take(&mut pending)) + .schema(Schema::Cmbp1) + .stype_in(SType::RawSymbol) + .build(), + ) + .await?; + } + + tokio::select! { + // Doorbell rang: re-read the held set and diff it. Note we query the + // DB rather than trust a payload — that's why the notify carries none. + Some(()) = rx.recv() => { + for (sym, id) in load_held_option_symbols(pool).await? { + if !sym_to_id.contains_key(&sym) { + pending.push(sym.clone()); + sym_to_id.insert(sym, id); + } + } + } + // Cancel-safe, so losing this branch just drops it mid-await. + rec = client.next_record() => { + let Some(rec) = rec? else { break }; + health.record_event(); + // … your existing SymbolMappingMsg + Cmbp1Msg branches, unchanged … + } + } + } + Ok(()) +``` + +The gateway sends a fresh `SymbolMappingMsg` for each added symbol, so the +existing mapping branch picks them up with no changes — which is why step 4c +handles mappings *in* the loop rather than once at startup. + +### Notes + +- **No unsubscribe** in this API. A closed position keeps streaming until the next + reconnect rebuilds the session. Harmless (a few wasted quotes), but it means + `sym_to_id` only ever grows — don't read it as "currently held". +- `pending` survives loop iterations but not reconnects. Correct: a reconnect + reloads the whole set from scratch anyway. +- Verify end-to-end: with the stream live, buy an option you don't already hold — + the log should show `subscribing newly-held options`, then a mark appears for it + in `/positions` without a restart. That restart-free part is the whole point of + step 6. + +--- + +## Step 7 — Valuation (the payoff) + +In `get_portfolio_positions` (`src/handlers.rs`) — already returns `net_qty` + +`avg_cost`. Add `contract_size` to the query, join the mark in Rust: + +```rust +let marks = state.marks().all(); +for p in &mut rows { + if let Some(m) = marks.get(&p.instrument_id) { + let mid = m.mid(); + p.mark = Some(mid); + p.market_value = Some(p.net_qty * mid * p.contract_size); + p.unrealized_pnl = Some((mid - p.avg_cost) * p.net_qty * p.contract_size); + } +} +``` + +Signed-correct for shorts (`net_qty < 0`). `realized_pnl` already comes from +`positions.rs`. Add the three optional fields to the response struct + the TS +type, and render a cockpit **Positions** page (qty, avg cost, mark, MV, uPnL, +realized) using the same `useList` pattern as the blotter. + +--- + +## Step 8 — Bonus: market-order risk + +Market orders skip notional checks today (`est_price=None`, +`src/risk_engine.rs:101`). Now there's a price: in `orders_submit`, when +`limit_price` is None, look up `marks.get(instrument_id).map(|m| m.mid())` and +pass it as `est_price` into `check_submit`. Enforces notional caps on market +orders too. + +--- + +## Step 9 — Verify + +1. Open a small option position (limit order, RTH or resting). +2. Start the OMS with `DATABENTO_API_KEY` set → logs show `OPRA live` + + `SymbolMappingMsg` then quotes. +3. Poke the store: log `marks.all().len()` or add a tiny `GET /admin/marks` + debug endpoint. +4. Hit `/portfolios/:id/positions` → confirm `mark`, `market_value`, + `unrealized_pnl` populate and move with the market. +5. Hand-check one: `(mid − avg_cost) × qty × 100`. + +--- + +## Build order (smallest shippable steps) + +1. `marks.rs` + `AppState` wiring (compiles, does nothing). +2. `opra_stream.rs` + spawn → marks populate (verify via debug log/endpoint). +3. Valuation join in `/positions` → uPnL shows. +4. Fill-path doorbell → newly-bought options get marked without a restart. +5. Cockpit Positions page. +6. Market-order est_price (bonus). + +Steps 1–5 (marks store, AppState, `live` dep, stream task, spawn) are done; +step 6 above is the next one to build. + +## Three things to watch + +- Exact **dbn 0.54 record struct** for `bbo-1s` (verify with `cargo doc`). +- **Subscription scoping** — only held symbols; never subscribe all of OPRA. +- **Fixed-point price scale** — `/1e9`, guard `i64::MAX`. + +Everything else reuses patterns already in the repo. diff --git a/examples/opra_playground.rs b/examples/opra_playground.rs new file mode 100644 index 0000000..5847521 --- /dev/null +++ b/examples/opra_playground.rs @@ -0,0 +1,154 @@ +//! Databento OPRA live playground — connect, subscribe to a few hard-coded +//! option symbols, and print the raw stream. No DB, no OMS wiring. Meant for +//! poking at the live feed to get a feel for it. +//! +//! Run: +//! export DATABENTO_API_KEY=db-... # your key +//! cargo run --example opra_playground +//! +//! Edit `SYMBOLS` below. These are OSI strings in Databento's space-padded +//! form: 6-char root left-justified + space-padded, then YYMMDD, C/P, strike*1000 +//! zero-padded to 8. The expiries below WILL go stale — pick a live contract +//! (e.g. from https://databento.com or the Alpaca option chain) or you'll just +//! see the symbol mapping and no quotes. + +use databento::{ + dbn::{self, Record, Schema, SType, UNDEF_PRICE}, + live::Subscription, + LiveClient, +}; + +// Space-padded OSI. Root is 6 chars, so "SPY" → "SPY " (3 trailing spaces). +const SYMBOLS: &[&str] = &[ + "SPY 260717C00750000", + "SPY 260717P00750000", +]; + +const DATASET: &str = "OPRA.PILLAR"; + +/// Fixed-point Databento price → f64, or None when undefined (i64::MAX). +fn px(v: i64) -> Option { + (v != UNDEF_PRICE).then_some(v as f64 / 1e9) +} + +/// Readable symbol for a dbn numeric instrument_id (falls back to the raw id). +fn sym_of(map: &std::collections::HashMap, id: u32) -> String { + map.get(&id).cloned().unwrap_or_else(|| format!("id={id}")) +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + // Simple logging so the databento crate's traces show up. + tracing_subscriber::fmt().with_max_level(tracing::Level::INFO).init(); + + println!("connecting to {DATASET} …"); + let mut client = LiveClient::builder() + .key_from_env()? // reads DATABENTO_API_KEY + .dataset(DATASET) + .build() + .await?; + + let symbols: Vec = SYMBOLS.iter().map(|s| s.to_string()).collect(); + // OPRA is multi-venue; use the consolidated top-of-book (cmbp-1) rather than + // per-publisher mbp-1. (tcbbo — quote-on-trade — decodes to the same Cmbp1Msg.) + println!("subscribing {} symbols (cmbp-1):", symbols.len()); + for s in &symbols { + println!(" [{s}]"); + } + + // OPRA is silent outside 09:30–16:00 ET. Set OPRA_REPLAY_MINS=N to replay the + // last N minutes of real quotes instead, so the decode path can be exercised + // off-hours. Unset = live only. + // + // Note: cmbp-1 does NOT support live replay ("Live replay not supported for + // cmbp-1 schema"). tcbbo does, and decodes to the same Cmbp1Msg — so + // OPRA_SCHEMA=tcbbo + OPRA_REPLAY_MINS exercises the identical decode path. + let replay_mins: Option = std::env::var("OPRA_REPLAY_MINS") + .ok() + .and_then(|s| s.parse().ok()); + + let schema = match std::env::var("OPRA_SCHEMA").as_deref() { + Ok("tcbbo") => Schema::Tcbbo, + Ok("mbp-1") => Schema::Mbp1, + _ => Schema::Cmbp1, + }; + println!("schema: {schema:?}"); + + let sub = match replay_mins { + Some(mins) => { + let from = chrono::Utc::now() - chrono::Duration::minutes(mins); + println!("replaying from {from} ({mins} min back)"); + Subscription::builder() + .symbols(symbols) + .schema(schema) + .stype_in(SType::RawSymbol) + .start(from) + .build() + } + None => Subscription::builder() + .symbols(symbols) + .schema(schema) + .stype_in(SType::RawSymbol) + .build(), + }; + + client.subscribe(sub).await?; + + client.start().await?; + println!("live — waiting for records (Ctrl-C to quit)\n"); + + // dbn numeric instrument_id → the raw OSI it maps to, for readable output. + let mut id_to_sym = std::collections::HashMap::::new(); + + while let Some(rec) = client.next_record().await? { + // Symbol mapping: numeric id ↔ raw symbol. Arrives before the quotes. + if let Some(sm) = rec.get::() { + let sym = sm.stype_out_symbol().unwrap_or("?").to_string(); + println!("MAP dbn_id={:<8} → [{sym}]", sm.hd.instrument_id); + id_to_sym.insert(sm.hd.instrument_id, sym); + continue; + } + + // cmbp-1 / tcbbo → Cmbp1Msg. Consolidated top-of-book at levels[0]. + if let Some(q) = rec.get::() { + let level = &q.levels[0]; + let sym = sym_of(&id_to_sym, q.hd.instrument_id); + match (px(level.bid_px), px(level.ask_px)) { + (Some(bid), Some(ask)) => { + let mid = (bid + ask) / 2.0; + println!( + "BBO [{sym}] bid {bid:>8.2} x {:<4} ask {ask:>8.2} x {:<4} mid {mid:>8.2}", + level.bid_sz, level.ask_sz, + ); + } + _ => println!("BBO [{sym}] (one-sided / empty book)"), + } + continue; + } + + // ohlcv-1s → OhlcvMsg. Only emitted when there's trading. + if let Some(b) = rec.get::() { + let sym = sym_of(&id_to_sym, b.hd.instrument_id); + println!( + "BAR [{sym}] o {:.2} h {:.2} l {:.2} c {:.2} vol {}", + b.open as f64 / 1e9, + b.high as f64 / 1e9, + b.low as f64 / 1e9, + b.close as f64 / 1e9, + b.volume, + ); + continue; + } + + // System messages: gateway heartbeats + acks. Print the text, not just rtype. + if let Some(sys) = rec.get::() { + println!("SYS {}", sys.msg().unwrap_or("?")); + continue; + } + + println!("REC rtype={}", rec.header().rtype); + } + + println!("stream closed"); + Ok(()) +} diff --git a/readme.md b/readme.md index fa887ee..c140903 100644 --- a/readme.md +++ b/readme.md @@ -35,8 +35,9 @@ make db-setup Optional live data (on-demand, needs vendor creds in `.env`): ```sh -make seed-instruments # instrument universe from Databento, FIGI-enriched (DATABENTO_API_KEY) -make sync-brokers # broker symbology from Alpaca (ALPACA_PAPER_*) +make sync-broker BROKER=alpaca # seed instruments + broker mapping from Alpaca (ALPACA_PAPER_*) +make sync-broker BROKER=alpaca UNDERLYINGS=SPY,QQQ # also seed those option chains +make map-feed FEED=databento # price the seeded options via Databento OPRA (DATABENTO_API_KEY) ``` ## Run diff --git a/scripts/fixtures/minimal_seed.sql b/scripts/fixtures/minimal_seed.sql index 8707dad..c66a7ca 100644 --- a/scripts/fixtures/minimal_seed.sql +++ b/scripts/fixtures/minimal_seed.sql @@ -13,23 +13,23 @@ VALUES 2, 0, 0.01, 1, 1) ON CONFLICT (symbol, venue) DO NOTHING; --- Symbology cross-reference (oms.instrument_xref): Databento's + Alpaca's view of SPY. --- native_id is SPY's Alpaca asset id (a global, account-independent UUID), so the --- order path can route on it. -INSERT INTO oms.instrument_xref - (instrument_id, source_type, source_code, external_symbol, external_exchange, - external_native_id, is_tradeable, method, confidence) -SELECT i.id, v.source_type, v.source_code, 'SPY', v.external_exchange, v.native_id, - v.is_tradeable, 'fixture', 'resolved' +-- Broker mapping (public.broker_instrument): Alpaca's view of SPY. native_id is +-- SPY's Alpaca asset id (a global, account-independent UUID), so the order path can +-- route on it. +INSERT INTO broker_instrument + (instrument_id, broker_code, broker_symbol, broker_exchange, native_id, is_tradeable) +SELECT i.id, 'ALPACA', 'SPY', 'ARCA', 'b28f4066-5c6d-479b-a2af-85dc1a8f16fb', true FROM instrument i -CROSS JOIN (VALUES - ('PROVIDER', 'DATABENTO', 'ARCX', NULL, NULL::boolean), - ('BROKER', 'ALPACA', 'ARCA', 'b28f4066-5c6d-479b-a2af-85dc1a8f16fb', true) -) AS v(source_type, source_code, external_exchange, native_id, is_tradeable) WHERE i.symbol = 'SPY' AND i.venue = 'ARCX' -ON CONFLICT (source_type, source_code, - COALESCE(external_symbol, ''), COALESCE(external_exchange, '')) -DO NOTHING; +ON CONFLICT (instrument_id, broker_code) DO NOTHING; + +-- Feed mapping (public.feed_instrument): Databento prices SPY on ARCX. Databento's +-- raw symbol equals ours here, so feed_symbol = 'SPY'. +INSERT INTO feed_instrument (feed_code, feed_symbol, instrument_id) +SELECT 'DATABENTO', 'SPY', i.id +FROM instrument i +WHERE i.symbol = 'SPY' AND i.venue = 'ARCX' +ON CONFLICT (feed_code, feed_symbol, instrument_id) DO NOTHING; -- A test broker connection so an account can be created and orders can route. -- Creds are resolved from env by (broker_code, environment); this is just the diff --git a/src/adapters/alpaca.rs b/src/adapters/alpaca.rs index af7a44a..94f0312 100644 --- a/src/adapters/alpaca.rs +++ b/src/adapters/alpaca.rs @@ -1,8 +1,32 @@ +use std::collections::BTreeSet; + +use dataprovider::{DerivativeDef, Identifiers, InstrumentDef, OptionKind}; use reqwest::Client; use serde::Serialize; -use tracing::info; +use tracing::{info, warn}; + +use super::{ + BrokerAdapter, BrokerError, BrokerHolding, BrokerInstrument, BrokerOrderRequest, + BrokerOrderResponse, InstrumentProvider, +}; -use super::{BrokerAdapter, BrokerError, BrokerHolding, BrokerInstrument, BrokerOrderRequest, BrokerOrderResponse}; +/// Map Alpaca's `exchange` label to the ISO 10383 MIC used by `public.venue`. +/// +/// `None` means we have no MIC for that label. The caller drops the instrument and +/// names the label, rather than passing the raw string through as a venue code — a +/// passthrough only fails later, anonymously, inside the catalog's FK filter. +fn alpaca_exchange_to_mic(exchange: &str) -> Option<&'static str> { + Some(match exchange { + "NASDAQ" => "XNAS", + "NYSE" => "XNYS", + "ARCA" | "NYSEARCA" => "ARCX", + "AMEX" => "XASE", + "BATS" => "BATS", + "IEX" => "IEXG", + "OTC" => "OTCM", + _ => return None, + }) +} #[derive(Serialize)] struct AlpacaOrderRequest { @@ -64,8 +88,10 @@ impl AlpacaAdapter { .header("APCA-API-SECRET-KEY", &self.api_secret) } - /// Active, tradeable US-equity assets from Alpaca's catalog - /// (`GET /v2/assets`). Broker symbology, not market data. + /// Active US-equity assets from Alpaca's catalog (`GET /v2/assets`) as canonical + /// instrument records + routing handles. Alpaca is the source of the master + /// instrument here: the ticker is the Symbol@Venue symbol, the exchange maps to + /// the venue MIC, and the asset UUID is the routing native id. pub async fn list_equity_instruments(&self) -> Result, BrokerError> { let url = format!("{}/v2/assets?status=active&asset_class=us_equity", self.base_url); let resp = self.get_json(&url).send().await.map_err(|e| BrokerError::Network(e.to_string()))?; @@ -74,23 +100,66 @@ impl AlpacaAdapter { } let assets: Vec = resp.json().await.map_err(|e| BrokerError::Network(e.to_string()))?; - Ok(assets + + // Exchange labels we have no MIC for. Collected as distinct values so an + // unmapped venue is reported by name once, not buried in a skip counter. + let mut unresolved: BTreeSet = BTreeSet::new(); + let instruments: Vec = assets .iter() .filter_map(|a| { - Some(BrokerInstrument { - symbol: a["symbol"].as_str()?.to_string(), - exchange: a["exchange"].as_str().map(|s| s.to_string()), + let symbol = a["symbol"].as_str()?.to_string(); + let raw_exchange = a["exchange"].as_str(); + let venue = match raw_exchange.and_then(alpaca_exchange_to_mic) { + Some(mic) => mic.to_string(), + None => { + unresolved.insert(raw_exchange.unwrap_or("").to_string()); + return None; + } + }; + let definition = InstrumentDef { + symbol: symbol.clone(), + venue, + currency: "USD".to_string(), + asset_class: "EQUITY".to_string(), + instrument_class: "SPOT".to_string(), + name: a["name"].as_str().map(|s| s.to_string()), + price_precision: 2, + price_increment: 0.01, + size_increment: if a["fractionable"].as_bool().unwrap_or(false) { 0.001 } else { 1.0 }, + lot_size: None, + contract_size: 1.0, native_id: a["id"].as_str().map(|s| s.to_string()), // Alpaca asset UUID + provider_exchange: raw_exchange.map(|s| s.to_string()), + derivative: None, + identifiers: Identifiers::default(), + }; + Some(BrokerInstrument { + broker_symbol: symbol, + broker_exchange: raw_exchange.map(|s| s.to_string()), is_tradeable: a["tradable"].as_bool().unwrap_or(false), min_quantity: a["min_order_size"].as_str().and_then(|s| s.parse().ok()), + max_quantity: None, + min_notional: None, + max_notional: None, + definition, }) }) - .collect()) + .collect(); + + if !unresolved.is_empty() { + warn!( + "alpaca: no venue MIC for exchange label(s) {unresolved:?} — {} asset(s) skipped; \ + add the mapping in `alpaca_exchange_to_mic` or seed the venue", + assets.len() - instruments.len() + ); + } + Ok(instruments) } /// Active option contracts for the given underlyings, following pagination - /// (`GET /v2/options/contracts`). `symbol` is the compact OSI Alpaca's order - /// API expects; options route by symbol so `native_id` is left None. + /// (`GET /v2/options/contracts`). The compact OSI is both the master symbol and + /// the order-entry handle; options route by symbol, so the routing native id is + /// left None. Strike/expiry/kind come straight from Alpaca's contract fields. pub async fn list_option_contracts( &self, underlyings: &[String], @@ -115,12 +184,47 @@ impl AlpacaAdapter { resp.json().await.map_err(|e| BrokerError::Network(e.to_string()))?; for c in body["option_contracts"].as_array().unwrap_or(&Vec::new()) { let Some(symbol) = c["symbol"].as_str() else { continue }; - out.push(BrokerInstrument { + let Some(underlying) = c["underlying_symbol"].as_str() else { continue }; + let option_kind = match c["type"].as_str() { + Some("call") => Some(OptionKind::Call), + Some("put") => Some(OptionKind::Put), + _ => None, + }; + let derivative = DerivativeDef { + underlying_symbol: underlying.to_string(), + option_kind, + strike_price: c["strike_price"].as_str().and_then(|s| s.parse().ok()), + expiry_date: c["expiration_date"] + .as_str() + .and_then(|s| chrono::NaiveDate::parse_from_str(s, "%Y-%m-%d").ok()), + activation_date: None, + }; + let definition = InstrumentDef { symbol: symbol.to_string(), - exchange: Some("OPRA".to_string()), + venue: "OPRA".to_string(), + currency: "USD".to_string(), + asset_class: "EQUITY".to_string(), + instrument_class: "OPTION".to_string(), + name: c["name"].as_str().map(|s| s.to_string()), + price_precision: 2, + price_increment: 0.01, + size_increment: 1.0, + lot_size: None, + contract_size: c["size"].as_str().and_then(|s| s.parse().ok()).unwrap_or(100.0), native_id: None, + provider_exchange: Some("OPRA".to_string()), + derivative: Some(derivative), + identifiers: Identifiers::default(), + }; + out.push(BrokerInstrument { + broker_symbol: symbol.to_string(), + broker_exchange: Some("OPRA".to_string()), is_tradeable: c["tradable"].as_bool().unwrap_or(false), min_quantity: Some(1.0), // options trade in whole contracts + max_quantity: None, + min_notional: None, + max_notional: None, + definition, }); } page_token = body["next_page_token"].as_str().map(|s| s.to_string()); @@ -132,6 +236,20 @@ impl AlpacaAdapter { } } +#[async_trait::async_trait] +impl InstrumentProvider for AlpacaAdapter { + async fn list_instruments( + &self, + option_underlyings: &[String], + ) -> Result, BrokerError> { + let mut out = self.list_equity_instruments().await?; + if !option_underlyings.is_empty() { + out.extend(self.list_option_contracts(option_underlyings).await?); + } + Ok(out) + } +} + #[async_trait::async_trait] impl BrokerAdapter for AlpacaAdapter { async fn submit_order(&self, req: &BrokerOrderRequest) -> Result { @@ -234,3 +352,42 @@ impl BrokerAdapter for AlpacaAdapter { Ok(holdings) } } + +#[cfg(test)] +mod tests { + use super::*; + + /// Every exchange label Alpaca actually returns must map. If this fails, the + /// catalog silently loses that venue's whole listing. + #[test] + fn maps_every_live_alpaca_exchange() { + for label in ["NASDAQ", "NYSE", "ARCA", "AMEX", "BATS", "IEX", "OTC"] { + assert!( + alpaca_exchange_to_mic(label).is_some(), + "no MIC for live Alpaca exchange {label:?}" + ); + } + } + + #[test] + fn maps_to_iso_mics() { + assert_eq!(alpaca_exchange_to_mic("NASDAQ"), Some("XNAS")); + assert_eq!(alpaca_exchange_to_mic("NYSEARCA"), Some("ARCX")); + assert_eq!(alpaca_exchange_to_mic("ARCA"), Some("ARCX")); + } + + /// An unknown label is declined, not passed through. The old behaviour returned + /// it verbatim, which then failed the venue FK anonymously inside the catalog. + #[test] + fn declines_unknown_exchange() { + assert_eq!(alpaca_exchange_to_mic("MOONBASE"), None); + assert_eq!(alpaca_exchange_to_mic(""), None); + } + + /// A MIC is not an Alpaca label — it is declined too. Guards against assuming + /// the passthrough was load-bearing for already-MIC inputs. + #[test] + fn declines_a_bare_mic() { + assert_eq!(alpaca_exchange_to_mic("XNAS"), None); + } +} diff --git a/src/adapters/binance.rs b/src/adapters/binance.rs index 63a6e74..a7df92a 100644 --- a/src/adapters/binance.rs +++ b/src/adapters/binance.rs @@ -11,12 +11,25 @@ //! balances report and reconciliation matches on. use base64::Engine; +use dataprovider::{Identifiers, InstrumentDef}; use ed25519_dalek::pkcs8::DecodePrivateKey; use ed25519_dalek::{Signer, SigningKey}; use reqwest::{Client, Method}; use tracing::info; -use super::{BrokerAdapter, BrokerError, BrokerHolding, BrokerOrderRequest, BrokerOrderResponse}; +use super::{ + BrokerAdapter, BrokerError, BrokerHolding, BrokerInstrument, BrokerOrderRequest, + BrokerOrderResponse, InstrumentProvider, +}; + +/// Read a Binance symbol filter's numeric field (e.g. LOT_SIZE.stepSize). +fn filter_val(filters: &[serde_json::Value], filter_type: &str, field: &str) -> Option { + filters + .iter() + .find(|f| f["filterType"].as_str() == Some(filter_type)) + .and_then(|f| f[field].as_str()) + .and_then(|s| s.parse().ok()) +} /// Percent-encode the non-unreserved characters of a base64 string (`+`, `/`, `=`) /// so an Ed25519 signature is safe inside a URL query string. @@ -105,6 +118,69 @@ impl BinanceAdapter { .map_err(|e| BrokerError::Network(e.to_string())) } + /// GET /api/v3/exchangeInfo — the full spot catalog as canonical instrument + /// records + routing handles. Public (unsigned). Binance is the source of the + /// master crypto instrument: the pair is the Symbol@Venue symbol, venue = + /// BINANCE, currency = the quote asset, and the base asset is the routing native + /// id (what account balances / recon match on). Only `TRADING` spot pairs. + pub async fn list_spot_instruments(&self) -> Result, BrokerError> { + let url = format!("{}/api/v3/exchangeInfo", self.base_url); + let resp = self + .client + .get(&url) + .send() + .await + .map_err(|e| BrokerError::Network(e.to_string()))?; + if !resp.status().is_success() { + return Err(BrokerError::BrokerRejected(resp.text().await.unwrap_or_default())); + } + let body: serde_json::Value = + resp.json().await.map_err(|e| BrokerError::Network(e.to_string()))?; + let symbols = body["symbols"].as_array().cloned().unwrap_or_default(); + Ok(symbols + .iter() + .filter_map(|s| { + let pair = s["symbol"].as_str()?.to_string(); + let base = s["baseAsset"].as_str()?.to_string(); + let quote = s["quoteAsset"].as_str()?.to_string(); + let is_spot = s["isSpotTradingAllowed"].as_bool().unwrap_or(true); + if !is_spot { + return None; + } + let filters: Vec = + s["filters"].as_array().cloned().unwrap_or_default(); + let definition = InstrumentDef { + symbol: pair.clone(), + venue: "BINANCE".to_string(), + currency: quote, + asset_class: "CRYPTO".to_string(), + instrument_class: "SPOT".to_string(), + name: None, + price_precision: s["quoteAssetPrecision"].as_i64().unwrap_or(8) as i32, + price_increment: filter_val(&filters, "PRICE_FILTER", "tickSize").unwrap_or(0.0), + size_increment: filter_val(&filters, "LOT_SIZE", "stepSize").unwrap_or(0.0), + lot_size: None, + contract_size: 1.0, + native_id: Some(base), // base asset — what recon matches balances on + provider_exchange: Some("BINANCE".to_string()), + derivative: None, + identifiers: Identifiers::default(), + }; + Some(BrokerInstrument { + broker_symbol: pair, + broker_exchange: Some("BINANCE".to_string()), + is_tradeable: s["status"].as_str() == Some("TRADING"), + min_quantity: filter_val(&filters, "LOT_SIZE", "minQty"), + max_quantity: filter_val(&filters, "LOT_SIZE", "maxQty"), + min_notional: filter_val(&filters, "NOTIONAL", "minNotional") + .or_else(|| filter_val(&filters, "MIN_NOTIONAL", "minNotional")), + max_notional: None, + definition, + }) + }) + .collect()) + } + /// GET /api/v3/order — fetch one order's state (for startup reconciliation of /// routed orders). Needs the symbol alongside the order id. pub async fn get_order( @@ -216,3 +292,15 @@ impl BrokerAdapter for BinanceAdapter { Ok(holdings) } } + +#[async_trait::async_trait] +impl InstrumentProvider for BinanceAdapter { + /// Binance has no separate option catalog on the spot venue; `option_underlyings` + /// is ignored — it always returns the spot pairs. + async fn list_instruments( + &self, + _option_underlyings: &[String], + ) -> Result, BrokerError> { + self.list_spot_instruments().await + } +} diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index fec3afc..dbf1edb 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -5,6 +5,8 @@ pub mod ibkr; use std::collections::HashMap; use std::sync::Arc; +use dataprovider::InstrumentDef; + use crate::adapters::alpaca::AlpacaAdapter; /// Broker-agnostic order request passed to any adapter. @@ -41,18 +43,42 @@ pub struct BrokerHolding { pub qty: f64, } -/// One row of a broker's tradeable-instrument catalog, as returned by the -/// broker's symbology endpoints. Provider-neutral; consumed by the broker-sync -/// setup task to populate `oms.instrument_xref` (source_type='BROKER'). The -/// `symbol` is the broker's own handle (Alpaca ticker for equities, compact OSI -/// for options); `native_id` is the immutable broker id when the broker exposes -/// one (Alpaca asset UUID for equities; None for options, which route by symbol). +/// One entry of a broker's tradeable catalog: the canonical Symbol@Venue +/// instrument record (→ `public.instrument` / `instrument_derivative`) plus the +/// broker's own routing handle (→ `public.broker_instrument`). Returned by an +/// adapter's [`InstrumentProvider`] impl; broker sync creates both rows from it in +/// one pass, so the broker is the authoritative source of the instrument. +/// +/// `definition.native_id` carries the broker's immutable id when it exposes one +/// (Alpaca asset UUID for equities; None for options, which route by symbol). pub struct BrokerInstrument { - pub symbol: String, - pub exchange: Option, - pub native_id: Option, + /// Canonical instrument definition — master catalog fields. + pub definition: InstrumentDef, + /// The broker's order-entry handle. Usually equals `definition.symbol`; kept + /// separate for brokers whose routing symbol differs from the canonical symbol. + pub broker_symbol: String, + /// The broker's own exchange label for the routing handle, when it exposes one. + pub broker_exchange: Option, pub is_tradeable: bool, pub min_quantity: Option, + pub max_quantity: Option, + pub min_notional: Option, + pub max_notional: Option, +} + +/// A broker/exchange adapter that can enumerate its own tradeable catalog. Broker +/// sync (`oms setup sync-broker`) drives this to seed the master instrument catalog +/// broker-first. Adapters without a catalog endpoint (e.g. the IBKR stub) simply +/// don't implement it. +#[async_trait::async_trait] +pub trait InstrumentProvider: Send + Sync { + /// The broker's tradeable catalog as canonical instrument records + routing + /// handles. `option_underlyings` scopes the (very large) option chain; empty = + /// list equities / spot pairs only. + async fn list_instruments( + &self, + option_underlyings: &[String], + ) -> Result, BrokerError>; } #[derive(Debug)] diff --git a/src/admin.rs b/src/admin.rs index f358094..36b2d1e 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -568,6 +568,43 @@ pub async fn list_stream_health(State(state): State) -> Json, +) -> Result>, AdminError> { + let rows = sqlx::query_as::<_, FeedSummary>( + "SELECT p.source_code AS feed_code, p.instrument_class, p.rank, p.enabled, \ + count(i.id) AS mapped_instruments \ + FROM oms.provider_feed_policy p \ + LEFT JOIN feed_instrument fi ON fi.feed_code = p.source_code AND fi.is_active \ + LEFT JOIN instrument i ON i.id = fi.instrument_id \ + AND i.instrument_class = p.instrument_class \ + GROUP BY p.source_code, p.instrument_class, p.rank, p.enabled \ + ORDER BY p.source_code, p.rank", + ) + .fetch_all(state.pool()) + .await + .map_err(map_db_error)?; + Ok(Json(rows)) +} + #[utoipa::path( get, path = "/admin/broker-connections", tag = "admin", responses( @@ -1229,346 +1266,6 @@ pub async fn list_instruments( Ok(Json(records)) } -// ── Instrument universes ────────────────────────────────────────────────────── - -#[derive(Debug, Serialize, sqlx::FromRow, utoipa::ToSchema)] -pub struct UniverseSummary { - pub code: String, - pub description: Option, - pub provider_code: String, - pub category: String, - pub dataset: String, - pub option_dataset: Option, - pub include_options: bool, - pub status: String, - pub last_seeded_at: Option>, - pub last_error: Option, - pub instrument_count: Option, -} - -/// List the instrument-universe catalog and its seed state (what is available -/// and when each was last loaded). -#[utoipa::path( - get, path = "/admin/universes", tag = "admin", - responses((status = 200, description = "OK", body = [UniverseSummary])), - security(("bearer_token" = [])) -)] -pub async fn list_universes( - State(state): State, -) -> Result>, AdminError> { - let records = sqlx::query_as::<_, UniverseSummary>( - "SELECT code, description, provider_code, category, dataset, option_dataset, \ - include_options, status, last_seeded_at, last_error, instrument_count \ - FROM instrument_universe \ - ORDER BY category, code", - ) - .fetch_all(state.pool()) - .await - .map_err(map_db_error)?; - Ok(Json(records)) -} - -#[derive(Debug, Serialize, utoipa::ToSchema)] -pub struct EstimateResponse { - pub universe_code: String, - pub usd: f64, - pub symbol_count: Option, -} - -/// Reject seeding/estimating an OPTION universe with no underlyings before any -/// provider call — 400, not a 502 from the downstream estimate. `Ok(None)` if the -/// universe is unknown (caller maps to 404). -async fn ensure_seedable(state: &AppState, code: &str) -> Result, AdminError> { - let row: Option<(String, i64)> = sqlx::query_as( - "SELECT u.category, count(s.symbol) \ - FROM instrument_universe u \ - LEFT JOIN instrument_universe_symbol s ON s.universe_code = u.code \ - WHERE u.code = $1 \ - GROUP BY u.category", - ) - .bind(code) - .fetch_optional(state.pool()) - .await - .map_err(map_db_error)?; - let Some((category, symbol_count)) = row else { - return Ok(None); - }; - if category == "OPTION" && symbol_count == 0 { - return Err(AdminError { - status: StatusCode::BAD_REQUEST, - message: "OPTION universe has no underlyings — pick underlyings before seeding (ALL is not allowed for options)".to_string(), - }); - } - Ok(Some(())) -} - -/// Free Databento cost estimate for seeding a single universe. -#[utoipa::path( - get, path = "/admin/universes/{code}/estimate", tag = "admin", - params(("code" = String, Path, description = "Universe code")), - responses((status = 200, description = "OK", body = EstimateResponse)), - security(("bearer_token" = [])) -)] -pub async fn estimate_universe( - State(state): State, - Path(code): Path, -) -> Result, AdminError> { - if ensure_seedable(&state, &code).await?.is_none() { - return Err(AdminError::not_found("universe")); - } - let est = crate::setup::universe::estimate(state.pool(), &code) - .await - .map_err(|e| AdminError { - status: StatusCode::BAD_GATEWAY, - message: format!("estimate failed: {e}"), - })? - .ok_or_else(|| AdminError::not_found("universe"))?; - Ok(Json(EstimateResponse { - universe_code: est.universe_code, - usd: est.usd, - symbol_count: est.symbol_count.map(|n| n as i64), - })) -} - -#[derive(Debug, Deserialize, utoipa::ToSchema)] -pub struct SeedRequest { - /// Abort if the estimated cost (USD) exceeds this. Omit to skip the gate. - pub max_cost: Option, - /// Run the enrichment pipeline (OpenFIGI). Defaults to true. - #[serde(default = "default_true")] - pub enrich: bool, -} - -fn default_true() -> bool { - true -} - -#[derive(Debug, Serialize, utoipa::ToSchema)] -pub struct SeedAccepted { - pub universe_code: String, - pub status: String, -} - -/// Seed a universe. Estimates + gates on `max_cost`, marks the universe -/// `SEEDING`, and runs the fetch/upsert/enrich in the background. Poll -/// `GET /admin/universes` for the terminal `SEEDED`/`ERROR` state. -#[utoipa::path( - post, path = "/admin/universes/{code}/seed", tag = "admin", - params(("code" = String, Path, description = "Universe code")), - request_body = SeedRequest, - responses( - (status = 202, description = "Seeding started (cost gate + errors surface async as ERROR)", body = SeedAccepted), - (status = 400, description = "OPTION universe has no underlyings"), - (status = 404, description = "Unknown universe"), - (status = 409, description = "Already seeding"), - ), - security(("bearer_token" = [])) -)] -pub async fn seed_universe( - State(state): State, - Path(code): Path, - Json(req): Json, -) -> Result<(StatusCode, Json), AdminError> { - // Guard: universe exists and is not already mid-seed. - let current: Option<(String,)> = - sqlx::query_as("SELECT status FROM instrument_universe WHERE code = $1") - .bind(&code) - .fetch_optional(state.pool()) - .await - .map_err(map_db_error)?; - let Some((status,)) = current else { - return Err(AdminError::not_found("universe")); - }; - if status == "SEEDING" { - return Err(AdminError { - status: StatusCode::CONFLICT, - message: "universe is already seeding".to_string(), - }); - } - - // OPTION universes must have underlyings — reject with 400 before spawning. - ensure_seedable(&state, &code).await?; - - // Mark SEEDING now so a poll sees it immediately, then run everything — - // including the cost estimate/gate — in the background so this request - // returns at once (the estimate can be slow, and Databento's get_cost is - // flaky). Over-budget or fetch failure lands as ERROR + last_error. - sqlx::query( - "UPDATE instrument_universe \ - SET status = 'SEEDING', last_error = NULL, updated_at = now() \ - WHERE code = $1", - ) - .bind(&code) - .execute(state.pool()) - .await - .map_err(map_db_error)?; - - let pool = state.pool().clone(); - let bg_code = code.clone(); - let enrich = req.enrich; - let max_cost = req.max_cost; - tokio::spawn(async move { - if let Err(e) = crate::setup::universe::seed(&pool, &bg_code, enrich, max_cost).await { - tracing::error!("background seed {bg_code} failed: {e}"); - } - }); - - Ok(( - StatusCode::ACCEPTED, - Json(SeedAccepted { - universe_code: code, - status: "SEEDING".to_string(), - }), - )) -} - -#[derive(Debug, Serialize, sqlx::FromRow, utoipa::ToSchema)] -pub struct UnderlyingCandidate { - pub symbol: String, - pub name: String, - pub venue: String, -} - -#[derive(Debug, Deserialize, utoipa::IntoParams)] -pub struct UnderlyingSearch { - pub search: Option, - pub limit: Option, -} - -/// Candidate option underlyings: the equities (SPOT) already seeded in the -/// master. Pick from these to build an OPTION universe's underlying list. -#[utoipa::path( - get, path = "/admin/underlyings", tag = "admin", - params(UnderlyingSearch), - responses((status = 200, description = "OK", body = [UnderlyingCandidate])), - security(("bearer_token" = [])) -)] -pub async fn list_underlyings( - State(state): State, - Query(params): Query, -) -> Result>, AdminError> { - let limit = params.limit.unwrap_or(100).clamp(1, 500); - let pattern = params.search.as_deref().map(|s| format!("%{s}%")); - let rows = sqlx::query_as::<_, UnderlyingCandidate>( - "SELECT symbol, name, venue \ - FROM instrument \ - WHERE instrument_class = 'SPOT' AND status = 'ACTIVE' \ - AND ($1::text IS NULL OR symbol ILIKE $1 OR name ILIKE $1) \ - ORDER BY symbol \ - LIMIT $2", - ) - .bind(pattern) - .bind(limit) - .fetch_all(state.pool()) - .await - .map_err(map_db_error)?; - Ok(Json(rows)) -} - -/// The current underlying/child symbols of a universe. -#[utoipa::path( - get, path = "/admin/universes/{code}/symbols", tag = "admin", - params(("code" = String, Path, description = "Universe code")), - responses((status = 200, description = "OK", body = [String])), - security(("bearer_token" = [])) -)] -pub async fn list_universe_symbols( - State(state): State, - Path(code): Path, -) -> Result>, AdminError> { - // 404 if the universe itself is unknown (empty set is otherwise ambiguous). - let exists: Option<(i32,)> = - sqlx::query_as("SELECT 1 FROM instrument_universe WHERE code = $1") - .bind(&code) - .fetch_optional(state.pool()) - .await - .map_err(map_db_error)?; - if exists.is_none() { - return Err(AdminError::not_found("universe")); - } - let symbols: Vec = sqlx::query_scalar( - "SELECT symbol FROM instrument_universe_symbol \ - WHERE universe_code = $1 ORDER BY symbol", - ) - .bind(&code) - .fetch_all(state.pool()) - .await - .map_err(map_db_error)?; - Ok(Json(symbols)) -} - -#[derive(Debug, Deserialize, utoipa::ToSchema)] -pub struct SetSymbolsRequest { - pub symbols: Vec, -} - -#[derive(Debug, Serialize, utoipa::ToSchema)] -pub struct SetSymbolsResponse { - pub universe_code: String, - pub count: usize, -} - -/// Replace a universe's underlying/child symbol set (the checkbox selection). -#[utoipa::path( - put, path = "/admin/universes/{code}/symbols", tag = "admin", - params(("code" = String, Path, description = "Universe code")), - request_body = SetSymbolsRequest, - responses( - (status = 200, description = "OK", body = SetSymbolsResponse), - (status = 404, description = "Unknown universe"), - ), - security(("bearer_token" = [])) -)] -pub async fn set_universe_symbols( - State(state): State, - Path(code): Path, - Json(req): Json, -) -> Result, AdminError> { - let exists: Option<(i32,)> = - sqlx::query_as("SELECT 1 FROM instrument_universe WHERE code = $1") - .bind(&code) - .fetch_optional(state.pool()) - .await - .map_err(map_db_error)?; - if exists.is_none() { - return Err(AdminError::not_found("universe")); - } - - // Normalize: trim, uppercase, dedup, drop blanks. - let mut symbols: Vec = req - .symbols - .iter() - .map(|s| s.trim().to_uppercase()) - .filter(|s| !s.is_empty()) - .collect(); - symbols.sort(); - symbols.dedup(); - - let mut tx = state.pool().begin().await.map_err(map_db_error)?; - sqlx::query("DELETE FROM instrument_universe_symbol WHERE universe_code = $1") - .bind(&code) - .execute(&mut *tx) - .await - .map_err(map_db_error)?; - if !symbols.is_empty() { - sqlx::query( - "INSERT INTO instrument_universe_symbol (universe_code, symbol) \ - SELECT $1, s FROM UNNEST($2::text[]) AS s", - ) - .bind(&code) - .bind(&symbols) - .execute(&mut *tx) - .await - .map_err(map_db_error)?; - } - tx.commit().await.map_err(map_db_error)?; - - Ok(Json(SetSymbolsResponse { - universe_code: code, - count: symbols.len(), - })) -} - // ── Custodian reconciliation ────────────────────────────────────────────────── #[derive(Debug, Deserialize, utoipa::ToSchema)] @@ -1698,8 +1395,6 @@ pub struct ResolveRequest { pub figi: Option, pub mic: Option, pub exch_code: Option, - pub source_type: Option, - pub source_code: Option, } #[derive(Debug, Deserialize, utoipa::ToSchema)] @@ -1748,17 +1443,9 @@ pub async fn resolve_symbology( currency: None, market_sec_des: None, }; - let source_type = req.source_type.unwrap_or_else(|| "MANUAL".to_string()); - let source_code = req.source_code.unwrap_or_else(|| "MANUAL".to_string()); - let outcome = symbology_resolver::resolve( - state.pool(), - state.symbology().as_ref(), - &query, - &source_type, - &source_code, - ) - .await - .map_err(map_resolve_error)?; + let outcome = symbology_resolver::resolve(state.pool(), state.symbology().as_ref(), &query) + .await + .map_err(map_resolve_error)?; Ok(Json(outcome)) } @@ -1795,7 +1482,7 @@ pub async fn backfill_symbology( mic: Some(venue), ..Default::default() }; - match symbology_resolver::resolve(state.pool(), state.symbology().as_ref(), &query, "OPENFIGI", "OPENFIGI").await { + match symbology_resolver::resolve(state.pool(), state.symbology().as_ref(), &query).await { Ok(ResolveOutcome::Resolved { instrument_id: Some(_), .. }) => result.stamped += 1, Ok(ResolveOutcome::Ambiguous { .. }) => result.ambiguous += 1, _ => result.unresolved += 1, diff --git a/src/binance_feed.rs b/src/binance_feed.rs index dd59773..d042250 100644 --- a/src/binance_feed.rs +++ b/src/binance_feed.rs @@ -13,7 +13,10 @@ use std::collections::HashMap; use chrono::Utc; -use dataprovider::{DataProvider, FeedHealth, LiveQuoteFeed, ProviderError, Quote, SymbolAdds}; +use dataprovider::{ + DataProvider, FeedHealth, FeedSymbology, InstrumentFilter, LiveQuoteFeed, ProviderError, Quote, + SymbolAdds, +}; use futures_util::{SinkExt, StreamExt}; use tokio::sync::mpsc; use tokio_tungstenite::{connect_async, tungstenite::Message}; @@ -40,12 +43,28 @@ impl DataProvider for BinanceFeed { } } -#[async_trait::async_trait] -impl LiveQuoteFeed for BinanceFeed { - fn covers(&self) -> &'static [&'static str] { - &["SPOT"] +impl FeedSymbology for BinanceFeed { + /// Only the pairs Binance itself lists. Scoped to the venue because this feed + /// is the exchange's own book — unlike Bybit, it has no claim on a pair listed + /// somewhere else. + fn candidates(&self) -> InstrumentFilter { + InstrumentFilter { + instrument_class: Some("SPOT"), + asset_class: Some("CRYPTO"), + venue: Some("BINANCE"), + } + } + + /// Identity: the master symbol *is* the pair, and matches the uppercase `s` + /// field Binance sends. (The lowercasing for the subscribe frame happens at + /// subscribe time in `run_session` — it is wire framing, not identity.) + fn to_feed_symbol(&self, symbol: &str) -> Option { + Some(symbol.to_string()) } +} +#[async_trait::async_trait] +impl LiveQuoteFeed for BinanceFeed { async fn run_session( &self, symbols: HashMap, diff --git a/src/binance_stream.rs b/src/binance_stream.rs index dcf1899..6a1b473 100644 --- a/src/binance_stream.rs +++ b/src/binance_stream.rs @@ -247,8 +247,8 @@ async fn reconcile_routed_orders( let instrument_id_num: i64 = row.get::("instrument_id").parse().unwrap_or_default(); let symbol: Option = sqlx::query_scalar( - "SELECT external_symbol FROM oms.instrument_xref \ - WHERE instrument_id = $1 AND source_type = 'BROKER' AND source_code = 'BINANCE' LIMIT 1", + "SELECT broker_symbol FROM broker_instrument \ + WHERE instrument_id = $1 AND broker_code = 'BINANCE' LIMIT 1", ) .bind(instrument_id_num) .fetch_optional(pool) diff --git a/src/bybit_feed.rs b/src/bybit_feed.rs index 4015648..2055a81 100644 --- a/src/bybit_feed.rs +++ b/src/bybit_feed.rs @@ -13,7 +13,10 @@ use std::collections::HashMap; use chrono::Utc; -use dataprovider::{DataProvider, FeedHealth, LiveQuoteFeed, ProviderError, Quote, SymbolAdds}; +use dataprovider::{ + DataProvider, FeedHealth, FeedSymbology, InstrumentFilter, LiveQuoteFeed, ProviderError, Quote, + SymbolAdds, +}; use futures_util::{SinkExt, StreamExt}; use tokio::sync::mpsc; use tokio::time::{interval, Duration}; @@ -48,12 +51,27 @@ impl DataProvider for BybitFeed { } } -#[async_trait::async_trait] -impl LiveQuoteFeed for BybitFeed { - fn covers(&self) -> &'static [&'static str] { - &["SPOT"] +impl FeedSymbology for BybitFeed { + /// Every crypto pair we know, on any venue — the 1:n case. Bybit is a data-only + /// source here: under broker-first seeding it lists no instruments of its own, + /// so its price for `BTCUSDT` marks the Binance-seeded `BTCUSDT`. Leaving + /// `venue` open is what lets one feed symbol map to the pair on several venues. + fn candidates(&self) -> InstrumentFilter { + InstrumentFilter { + instrument_class: Some("SPOT"), + asset_class: Some("CRYPTO"), + venue: None, + } + } + + /// Identity: the master symbol is the pair, matching the `s` field verbatim. + fn to_feed_symbol(&self, symbol: &str) -> Option { + Some(symbol.to_string()) } +} +#[async_trait::async_trait] +impl LiveQuoteFeed for BybitFeed { async fn run_session( &self, symbols: HashMap, diff --git a/src/handlers.rs b/src/handlers.rs index e46cc4d..fc44ecf 100644 --- a/src/handlers.rs +++ b/src/handlers.rs @@ -241,12 +241,11 @@ pub async fn orders_submit( } // Validate broker mapping exists and is tradeable; retrieve broker-specific symbol. - // Resolves through the unified instrument_xref (BROKER source). + // Resolves through broker_instrument (the broker routing mapping). let broker_instrument_row = sqlx::query( - "SELECT external_symbol AS broker_symbol, external_native_id AS native_id, \ - min_quantity::float8 AS min_quantity \ - FROM instrument_xref \ - WHERE instrument_id = $1 AND source_type = 'BROKER' AND source_code = $2 AND is_tradeable = true" + "SELECT broker_symbol, native_id, min_quantity::float8 AS min_quantity \ + FROM broker_instrument \ + WHERE instrument_id = $1 AND broker_code = $2 AND is_tradeable = true" ) .bind(instrument_id_bigint) .bind(&broker_code) @@ -727,12 +726,12 @@ pub async fn orders_cancel( })?; // The broker-native symbol — required by venues that scope cancels by - // symbol (Binance). Resolve from the same BROKER xref the submit used. - // order_state.instrument_id is TEXT (stringified bigint); the xref key is BIGINT. + // symbol (Binance). Resolve from the same broker_instrument the submit used. + // order_state.instrument_id is TEXT (stringified bigint); the mapping key is BIGINT. let instrument_id_num: i64 = row.get::("instrument_id").parse().unwrap_or_default(); let broker_symbol: String = sqlx::query_scalar( - "SELECT external_symbol FROM oms.instrument_xref \ - WHERE instrument_id = $1 AND source_type = 'BROKER' AND source_code = $2 \ + "SELECT broker_symbol FROM broker_instrument \ + WHERE instrument_id = $1 AND broker_code = $2 \ LIMIT 1", ) .bind(instrument_id_num) diff --git a/src/main.rs b/src/main.rs index 05b6e4f..a64dced 100644 --- a/src/main.rs +++ b/src/main.rs @@ -100,12 +100,7 @@ mod bybit_feed; admin::update_risk_limit, admin::delete_risk_limit, admin::list_instruments, - admin::list_universes, - admin::list_underlyings, - admin::list_universe_symbols, - admin::set_universe_symbols, - admin::estimate_universe, - admin::seed_universe, + admin::list_feeds, admin::run_recon, admin::list_recon_runs, admin::list_recon_breaks, @@ -124,9 +119,7 @@ mod bybit_feed; CreateKey, ApiKeyRecord, Grant, CreateGrant, UpdateGrant, admin::RiskLimit, admin::CreateRiskLimit, admin::UpdateRiskLimit, - admin::InstrumentSummary, - admin::UniverseSummary, admin::EstimateResponse, admin::SeedRequest, admin::SeedAccepted, - admin::UnderlyingCandidate, admin::SetSymbolsRequest, admin::SetSymbolsResponse, + admin::InstrumentSummary, admin::FeedSummary, admin::RunReconRequest, admin::ReconRunRow, admin::ReconBreakRow, crate::recon::ReconSummary, crate::recon::ReconBreak, crate::recon::BreakKind, admin::ResolveRequest, admin::BackfillRequest, admin::BackfillResult, @@ -177,10 +170,10 @@ enum Command { #[derive(clap::Subcommand)] enum SetupCmd { - /// Seed the master instrument universe from the configured providers. - Universe(setup::universe::Args), - /// Sync broker symbology (instrument_xref BROKER rows) from a broker catalog. - SyncBrokers(setup::brokers::Args), + /// Seed the master instrument catalog + broker_instrument mapping from a broker. + SyncBroker(setup::brokers::Args), + /// Map a data feed's symbols onto seeded instruments (feed_instrument rows). + MapFeed(setup::feeds::Args), } #[tokio::main] @@ -190,15 +183,15 @@ async fn main() { let cli = ::parse(); match cli.command { - Some(Command::Setup(SetupCmd::Universe(args))) => { - if let Err(e) = setup::universe::run(args).await { - error!("setup universe failed: {e}"); + Some(Command::Setup(SetupCmd::SyncBroker(args))) => { + if let Err(e) = setup::brokers::run(args).await { + error!("setup sync-broker failed: {e}"); std::process::exit(1); } } - Some(Command::Setup(SetupCmd::SyncBrokers(args))) => { - if let Err(e) = setup::brokers::run(args).await { - error!("setup sync-brokers failed: {e}"); + Some(Command::Setup(SetupCmd::MapFeed(args))) => { + if let Err(e) = setup::feeds::run(args).await { + error!("setup map-feed failed: {e}"); std::process::exit(1); } } @@ -380,7 +373,7 @@ async fn serve() { if let (Ok(key), Ok(secret)) = (env::var("ALPACA_PAPER_API_KEY"), env::var("ALPACA_PAPER_API_SECRET")) { if !key.is_empty() && !secret.is_empty() { if let Some(adapter) = state.registry().get_alpaca("PAPER") { - let health = state.stream_health().handle("ALPACA", "PAPER"); + let health = state.stream_health().handle("ALPACA", "PAPER", stream_health::StreamKind::Execution); tokio::spawn(alpaca_stream::run("PAPER", key, secret, state.pool().clone(), state.kafka().cloned(), adapter, health, Some(position_changed_tx.clone()))); } } @@ -388,7 +381,7 @@ async fn serve() { if let (Ok(key), Ok(secret)) = (env::var("ALPACA_LIVE_API_KEY"), env::var("ALPACA_LIVE_API_SECRET")) { if !key.is_empty() && !secret.is_empty() { if let Some(adapter) = state.registry().get_alpaca("LIVE") { - let health = state.stream_health().handle("ALPACA", "LIVE"); + let health = state.stream_health().handle("ALPACA", "LIVE", stream_health::StreamKind::Execution); tokio::spawn(alpaca_stream::run("LIVE", key, secret, state.pool().clone(), state.kafka().cloned(), adapter, health, Some(position_changed_tx.clone()))); } } @@ -396,7 +389,7 @@ async fn serve() { // Spawn the Binance user-data stream when configured. if let Some(adapter) = binance_paper { - let health = state.stream_health().handle("BINANCE", "PAPER"); + let health = state.stream_health().handle("BINANCE", "PAPER", stream_health::StreamKind::Execution); tokio::spawn(binance_stream::run("PAPER", state.pool().clone(), state.kafka().cloned(), adapter, health, Some(position_changed_tx.clone()))); } @@ -409,7 +402,7 @@ async fn serve() { if env::var("DATABENTO_API_KEY").map(|k| !k.is_empty()).unwrap_or(false) { let (opra_pos_tx, opra_pos_rx) = tokio::sync::mpsc::channel::<()>(1); marks_doorbells.push(opra_pos_tx); - let health = state.stream_health().handle("DATABENTO", "OPRA"); + let health = state.stream_health().handle("DATABENTO", "OPRA", stream_health::StreamKind::Feed); let session = quote_feed::QuoteFeedSession::new( opra_stream::DatabentoOpraFeed, state.pool().clone(), @@ -426,7 +419,7 @@ async fn serve() { { let (binance_pos_tx, binance_pos_rx) = tokio::sync::mpsc::channel::<()>(1); marks_doorbells.push(binance_pos_tx); - let health = state.stream_health().handle("BINANCE", "SPOT"); + let health = state.stream_health().handle("BINANCE", "SPOT", stream_health::StreamKind::Feed); let session = quote_feed::QuoteFeedSession::new( binance_feed::BinanceFeed, state.pool().clone(), @@ -443,7 +436,7 @@ async fn serve() { { let (bybit_pos_tx, bybit_pos_rx) = tokio::sync::mpsc::channel::<()>(1); marks_doorbells.push(bybit_pos_tx); - let health = state.stream_health().handle("BYBIT", "SPOT"); + let health = state.stream_health().handle("BYBIT", "SPOT", stream_health::StreamKind::Feed); let session = quote_feed::QuoteFeedSession::new( bybit_feed::BybitFeed, state.pool().clone(), @@ -541,14 +534,7 @@ async fn serve() { .delete(admin::delete_risk_limit), ) .route("/admin/instruments", get(admin::list_instruments)) - .route("/admin/universes", get(admin::list_universes)) - .route("/admin/underlyings", get(admin::list_underlyings)) - .route( - "/admin/universes/:code/symbols", - get(admin::list_universe_symbols).put(admin::set_universe_symbols), - ) - .route("/admin/universes/:code/estimate", get(admin::estimate_universe)) - .route("/admin/universes/:code/seed", post(admin::seed_universe)) + .route("/admin/feeds", get(admin::list_feeds)) .route("/admin/recon/run", post(admin::run_recon)) .route("/admin/recon/runs", get(admin::list_recon_runs)) .route("/admin/recon/runs/:id/breaks", get(admin::list_recon_breaks)) diff --git a/src/opra_stream.rs b/src/opra_stream.rs index 247da89..cf0d68a 100644 --- a/src/opra_stream.rs +++ b/src/opra_stream.rs @@ -17,13 +17,19 @@ use databento::{ live::Subscription, LiveClient, }; -use dataprovider::{DataProvider, FeedHealth, LiveQuoteFeed, ProviderError, Quote, SymbolAdds}; +use dataprovider::{ + DataProvider, FeedHealth, FeedSymbology, InstrumentFilter, LiveQuoteFeed, ProviderError, Quote, + SymbolAdds, +}; use tokio::sync::mpsc; use tracing::{info, warn}; const OPRA_DATASET: &str = "OPRA.PILLAR"; const SOURCE_CODE: &str = "DATABENTO"; +/// Fixed tail of an OSI symbol: 6 date + 1 kind + 8 strike. +const OSI_TAIL: usize = 15; + pub struct DatabentoOpraFeed; impl DataProvider for DatabentoOpraFeed { @@ -32,14 +38,30 @@ impl DataProvider for DatabentoOpraFeed { } } -#[async_trait::async_trait] -impl LiveQuoteFeed for DatabentoOpraFeed { - /// OPRA.PILLAR is an options dataset. Databento cross-references our equities - /// too, but this session cannot quote them. - fn covers(&self) -> &'static [&'static str] { - &["OPTION"] +impl FeedSymbology for DatabentoOpraFeed { + fn candidates(&self) -> InstrumentFilter { + InstrumentFilter { + instrument_class: Some("OPTION"), + venue: Some("OPRA"), + ..Default::default() + } } + /// Compact OSI (how the master stores it, and how Alpaca reports it) → the + /// space-padded form Databento puts on the wire: root left-justified in 6. + /// + /// `SPY260724P00739000` → `SPY 260724P00739000` + fn to_feed_symbol(&self, symbol: &str) -> Option { + // A symbol with no room for a root is not an OSI symbol. Guards the + // underflow the equivalent SQL (`left(symbol, length(symbol) - 15)`) had. + let split = symbol.len().checked_sub(OSI_TAIL).filter(|&n| n > 0)?; + let (root, tail) = symbol.split_at(split); + Some(format!("{root:<6}{tail}")) + } +} + +#[async_trait::async_trait] +impl LiveQuoteFeed for DatabentoOpraFeed { async fn run_session( &self, symbols: HashMap, @@ -157,3 +179,54 @@ async fn subscribe(client: &mut LiveClient, symbols: Vec) -> Result<(), fn px(v: i64) -> Option { (v != UNDEF_PRICE).then_some(v as f64 / 1e9) } + +#[cfg(test)] +mod tests { + use super::*; + + /// The case that motivates the whole transform: a 3-char root gets padded to 6 + /// so the wire symbol matches what Databento publishes. + #[test] + fn pads_short_root_to_six() { + assert_eq!( + DatabentoOpraFeed.to_feed_symbol("SPY260724P00739000").as_deref(), + Some("SPY 260724P00739000") + ); + } + + /// A root already 6 wide is unchanged — no padding, no truncation. + #[test] + fn leaves_full_width_root_alone() { + assert_eq!( + DatabentoOpraFeed.to_feed_symbol("BRKB 260116C00500000").as_deref(), + Some("BRKB 260116C00500000") + ); + } + + /// Output is always root(6) + tail(15). Anything else would silently fail to + /// match on the wire rather than erroring. + #[test] + fn output_is_always_21_chars() { + for s in ["A260724P00739000", "SPY260724P00739000", "SPXW 260724C05000000"] { + let out = DatabentoOpraFeed.to_feed_symbol(s).expect("valid OSI"); + assert_eq!(out.len(), 6 + OSI_TAIL, "{s} → {out:?}"); + } + } + + /// Too short to carry a root: declined, not padded into nonsense. The SQL this + /// replaced would have underflowed on `length(symbol) - 15`. + #[test] + fn declines_symbol_with_no_room_for_root() { + assert_eq!(DatabentoOpraFeed.to_feed_symbol("260724P00739000"), None); + assert_eq!(DatabentoOpraFeed.to_feed_symbol("SPY"), None); + assert_eq!(DatabentoOpraFeed.to_feed_symbol(""), None); + } + + #[test] + fn candidates_scope_to_opra_options() { + let f = DatabentoOpraFeed.candidates(); + assert_eq!(f.instrument_class, Some("OPTION")); + assert_eq!(f.venue, Some("OPRA")); + assert_eq!(f.asset_class, None); + } +} diff --git a/src/quote_feed.rs b/src/quote_feed.rs index ff29d89..e6fdfa0 100644 --- a/src/quote_feed.rs +++ b/src/quote_feed.rs @@ -51,7 +51,7 @@ impl QuoteFeedSession { /// with a timer as the backstop for positions this process didn't see. async fn wait_for_subscribable(&mut self) -> Result, sqlx::Error> { loop { - let held = load_subscribable(&self.pool, self.feed.code(), self.feed.covers()).await?; + let held = load_subscribable(&self.pool, self.feed.code()).await?; if !held.is_empty() { return Ok(held); } @@ -75,7 +75,7 @@ impl Session for QuoteFeedSession { // them lets a fill be picked up without waiting for a reconnect. tokio::select! { r = self.feed.run_session(known.clone(), &self.out, &mut add_rx, &self.health) => r.map_err(Into::into), - r = watch_held(&self.pool, self.feed.code(), self.feed.covers(), &mut self.position_changed_rx, &add_tx, known) => r, + r = watch_held(&self.pool, self.feed.code(), &mut self.position_changed_rx, &add_tx, known) => r, } } } @@ -87,7 +87,6 @@ impl Session for QuoteFeedSession { async fn watch_held( pool: &PgPool, source_code: &'static str, - covers: &'static [&'static str], doorbell: &mut mpsc::Receiver<()>, add_tx: &mpsc::Sender, mut known: HashMap, @@ -95,7 +94,7 @@ async fn watch_held( loop { match doorbell.recv().await { Some(()) => { - let latest = load_subscribable(pool, source_code, covers).await?; + let latest = load_subscribable(pool, source_code).await?; let adds: SymbolAdds = latest .into_iter() .filter(|(sym, _)| !known.contains_key(sym)) @@ -123,45 +122,36 @@ async fn watch_held( } } -/// The held instruments this feed can cover, as `external_symbol -> instrument_id`. +/// The held instruments this feed can price, as `feed_symbol -> instrument_id`. /// -/// Driven by `instrument_xref`, which is the point: the *provider's own* symbol -/// drives the subscription, so a feed is no longer required to use strings that -/// happen to match ours. The previous query read `instrument.symbol` directly and -/// worked only by the coincidence that Databento's raw symbol is byte-identical to -/// our master symbol — a coincidence that holds for OSI options and for nothing -/// else. Binance calls it `SOLUSDT` where we call it `SOL`. +/// Driven by `feed_instrument`: the feed's *own* symbol drives the subscription, +/// and the mapping is the single source of what this feed covers. An instrument +/// with no `feed_instrument` row for this feed is silently absent — that is the +/// "held but this feed can't price it" state, not an error; another feed may. /// -/// Coverage is `(source_code, instrument_class)`. Both halves are load-bearing: -/// source_code alone would hand this feed the held SPY *equity*, which is -/// cross-referenced to DATABENTO but not quotable on OPRA.PILLAR. -/// -/// An instrument with no xref row for this source is silently absent — that is the -/// "tradable but unmarkable" state, not an error. +/// `feed_instrument` is 1:n (one feed symbol may price instruments on several +/// venues), so a `feed_symbol` collision keeps the last instrument id. In practice +/// held sets don't collide today (one venue per crypto pair); true fan-out to +/// multiple held instruments from one quote is a later change in the mark path. async fn load_subscribable( pool: &PgPool, - source_code: &str, - covers: &[&str], + feed_code: &str, ) -> Result, sqlx::Error> { - let classes: Vec = covers.iter().map(|c| c.to_string()).collect(); let rows = sqlx::query( // position.instrument_id is text holding the numeric instrument.id. - "SELECT DISTINCT x.external_symbol, i.id \ + "SELECT DISTINCT fi.feed_symbol, i.id \ FROM position p \ JOIN instrument i ON i.id::text = p.instrument_id \ - JOIN oms.instrument_xref x ON x.instrument_id = i.id \ - AND x.source_type = 'PROVIDER' AND x.source_code = $1 \ - WHERE p.net_qty <> 0 \ - AND i.instrument_class = ANY($2) \ - AND x.external_symbol IS NOT NULL", + JOIN feed_instrument fi ON fi.instrument_id = i.id \ + AND fi.feed_code = $1 AND fi.is_active \ + WHERE p.net_qty <> 0", ) - .bind(source_code) - .bind(&classes) + .bind(feed_code) .fetch_all(pool) .await?; Ok(rows .iter() - .map(|r| (r.get::("external_symbol"), r.get::("id"))) + .map(|r| (r.get::("feed_symbol"), r.get::("id"))) .collect()) } diff --git a/src/recon.rs b/src/recon.rs index 4b46705..3e728b8 100644 --- a/src/recon.rs +++ b/src/recon.rs @@ -177,25 +177,24 @@ pub async fn run_reconciliation( })?; // Custodian side: resolve each holding to a master instrument via broker_instrument. - // Load every BROKER xref for this source once, into a lookup keyed by both the - // native id and the external symbol — avoids a round-trip per holding (a crypto - // testnet account can report hundreds of balances). - let xref_rows = sqlx::query( - "SELECT instrument_id, external_native_id, external_symbol FROM instrument_xref \ - WHERE source_type = 'BROKER' AND source_code = $1 AND instrument_id IS NOT NULL", + // Load every mapping for this broker once, into a lookup keyed by both the native + // id and the broker symbol — avoids a round-trip per holding (a crypto testnet + // account can report hundreds of balances). + let mapping_rows = sqlx::query( + "SELECT instrument_id, native_id, broker_symbol FROM broker_instrument \ + WHERE broker_code = $1", ) .bind(&broker_code) .fetch_all(pool) .await?; let mut xref: HashMap = HashMap::new(); - for r in &xref_rows { + for r in &mapping_rows { let id: i64 = r.get("instrument_id"); - if let Some(n) = r.get::, _>("external_native_id") { + if let Some(n) = r.get::, _>("native_id") { xref.entry(n).or_insert(id); } - if let Some(s) = r.get::, _>("external_symbol") { - xref.entry(s).or_insert(id); - } + let s: String = r.get("broker_symbol"); + xref.entry(s).or_insert(id); } let mut resolved = Vec::with_capacity(holdings.len()); @@ -208,7 +207,7 @@ pub async fn run_reconciliation( // Fallback: identify via the symbology engine (OpenFIGI). A US-equity // affordance (FIGI + exch_code "US") — only meaningful for Alpaca. Other - // venues resolve via their seeded BROKER xref only. + // venues resolve via their seeded broker_instrument mapping only. if instrument_id.is_none() && broker_code == "ALPACA" { let query = InstrumentQuery { ticker: Some(h.symbol.clone()), @@ -216,15 +215,15 @@ pub async fn run_reconciliation( ..Default::default() }; if let Ok(ResolveOutcome::Resolved { instrument_id: Some(id), .. }) = - symbology_resolver::resolve(pool, engine, &query, "CUSTODIAN", &broker_code).await + symbology_resolver::resolve(pool, engine, &query).await { instrument_id = Some(id); } } // For non-equity venues, only reconcile assets we actually track (have a - // BROKER xref). A crypto exchange account carries many untracked balances - // (testnet pre-funds hundreds) that aren't real breaks — skip them. + // broker_instrument mapping). A crypto exchange account carries many + // untracked balances (testnet pre-funds hundreds) that aren't real breaks. if instrument_id.is_none() && broker_code != "ALPACA" { continue; } diff --git a/src/setup/brokers.rs b/src/setup/brokers.rs index 9fdd76a..2120680 100644 --- a/src/setup/brokers.rs +++ b/src/setup/brokers.rs @@ -1,41 +1,55 @@ -//! `oms setup sync-brokers` — sync a broker's instrument catalog into -//! `oms.instrument_xref` (source_type='BROKER'), the map the order path resolves -//! `instrument_id -> broker handle` through at routing time. +//! `oms setup sync-broker` — the instrument seeding path. //! -//! Ports the former `scripts/broker_sync.py` into Rust: it reuses the Alpaca -//! adapter's HTTP/auth (`AlpacaAdapter::list_equity_instruments` / -//! `list_option_contracts`) and the universe seeder's UNNEST bulk-upsert idiom. -//! Runs as the ordinary `DB_USER` (oms_user owns the `oms` schema), so unlike the -//! Python script it needs no superuser. +//! Broker-first: an adapter's [`InstrumentProvider`] enumerates the +//! broker's tradeable catalog; we create the master `public.instrument` (+ +//! `instrument_derivative`) rows and the `public.broker_instrument` routing mapping +//! in one pass. The broker is the authoritative source of the instrument — there is +//! no separate dataset catalog and no priceable-but-not-tradeable path. +//! +//! Runs as the ordinary `DB_USER` (oms_user), which holds write on the master +//! catalog and both mapping tables (see `db/access/ods.sql`). -use std::collections::HashMap; use std::env; use clap::{Args as ClapArgs, ValueEnum}; -use sqlx::{PgPool, Postgres, Row, Transaction}; +use dataprovider::{Enricher, InstrumentDef, OpenFigiEnricher}; +use sqlx::{PgPool, Postgres, Transaction}; use tracing::{info, warn}; use crate::adapters::alpaca::AlpacaAdapter; -use crate::adapters::BrokerInstrument; +use crate::adapters::binance::BinanceAdapter; +use crate::adapters::{BrokerInstrument, InstrumentProvider}; +use crate::setup::catalog; -const BROKER_CODE: &str = "ALPACA"; const BATCH: usize = 4000; #[derive(Copy, Clone, Debug, PartialEq, Eq, ValueEnum)] -pub enum AssetClass { - Equity, - Option, - All, +pub enum Broker { + Alpaca, + Binance, +} + +impl Broker { + fn code(self) -> &'static str { + match self { + Broker::Alpaca => "ALPACA", + Broker::Binance => "BINANCE", + } + } } #[derive(ClapArgs, Debug, Clone)] pub struct Args { - /// Which broker catalog to sync. - #[arg(long, value_enum, default_value_t = AssetClass::All)] - pub asset_class: AssetClass, - /// Comma-separated option underlyings (only used for the option leg). - #[arg(long, default_value = "SPY,QQQ")] + /// Which broker's catalog to sync (also the instrument source). + #[arg(long, value_enum, default_value_t = Broker::Alpaca)] + pub broker: Broker, + /// Comma-separated option underlyings to seed the option chain for (Alpaca only). + /// Empty seeds equities/spot only — the full option tape is not seeded wholesale. + #[arg(long, default_value = "")] pub underlyings: String, + /// Skip the enricher pipeline (FIGI via OpenFIGI). + #[arg(long)] + pub no_enrich: bool, /// Fetch + match + print counts, but write nothing. #[arg(long)] pub dry_run: bool, @@ -43,7 +57,6 @@ pub struct Args { pub async fn run(args: Args) -> Result<(), Box> { let pool = PgPool::connect(&super::database_url()?).await?; - let adapter = build_alpaca()?; let underlyings: Vec = args .underlyings .split(',') @@ -51,17 +64,65 @@ pub async fn run(args: Args) -> Result<(), Box> { .filter(|s| !s.is_empty()) .collect(); - if matches!(args.asset_class, AssetClass::Equity | AssetClass::All) { - sync_equities(&pool, &adapter, args.dry_run).await?; + let provider: Box = match args.broker { + Broker::Alpaca => Box::new(build_alpaca()?), + Broker::Binance => Box::new(build_binance()?), + }; + + info!("fetching {} catalog …", args.broker.code()); + let catalog = provider.list_instruments(&underlyings).await?; + info!("fetched {} tradeable instrument(s) from {}", catalog.len(), args.broker.code()); + + if args.dry_run { + for bi in catalog.iter().take(20) { + info!( + " {}@{} -> broker_symbol={} tradeable={}", + bi.definition.symbol, bi.definition.venue, bi.broker_symbol, bi.is_tradeable + ); + } + info!("dry run complete: {} instrument(s), no write", catalog.len()); + return Ok(()); + } + if catalog.is_empty() { + warn!("empty catalog; nothing to seed"); + return Ok(()); } - if matches!(args.asset_class, AssetClass::Option | AssetClass::All) { - sync_options(&pool, &adapter, &underlyings, args.dry_run).await?; + + // 1) Create/refresh the master instrument catalog from the broker's definitions. + let defs: Vec = catalog.iter().map(|bi| bi.definition.clone()).collect(); + let enrichers: Vec> = if args.no_enrich { + Vec::new() + } else { + vec![Box::new(OpenFigiEnricher::new(env::var("OPENFIGI_API_KEY").ok()))] + }; + let (summary, ids) = catalog::upsert_catalog(&pool, &defs, &enrichers).await?; + info!( + "master: upserted={} skipped_fk={} skipped_expired={} derivatives={} enriched={}", + summary.upserted, summary.skipped_fk(), summary.skipped_expired, summary.derivatives, summary.enriched + ); + + // 2) Attach the broker routing mapping for every instrument that landed. + let broker_code = args.broker.code(); + let rows: Vec = catalog + .iter() + .filter_map(|bi| { + let key = (bi.definition.symbol.clone(), bi.definition.venue.clone()); + ids.get(&key).map(|&id| BrokerRow::from(id, bi)) + }) + .collect(); + info!("mapping {} instrument(s) to {broker_code}", rows.len()); + + let mut tx = pool.begin().await?; + let mut n = 0usize; + for chunk in rows.chunks(BATCH) { + n += bulk_upsert_broker_instrument(&mut tx, broker_code, chunk).await?; } + tx.commit().await?; + info!("sync-broker done: upserted {n} {broker_code} broker_instrument row(s)"); Ok(()) } -/// Build an `AlpacaAdapter` standalone from env, matching the server's wiring -/// (`ALPACA_ENV` + `ALPACA_{ENV}_API_KEY/SECRET`). +/// Build an `AlpacaAdapter` standalone from env (`ALPACA_ENV` + `ALPACA_{ENV}_API_KEY/SECRET`). fn build_alpaca() -> Result> { let env_name = env::var("ALPACA_ENV").unwrap_or_else(|_| "PAPER".into()).to_uppercase(); let key = env::var(format!("ALPACA_{env_name}_API_KEY")) @@ -71,178 +132,97 @@ fn build_alpaca() -> Result> { Ok(AlpacaAdapter::new(key, secret, &env_name)) } -/// A catalog row matched to a master instrument, ready to upsert. -struct BrokerXrefRow { +/// Build a `BinanceAdapter` from env. The catalog endpoint (exchangeInfo) is +/// public, but the adapter constructor needs a valid key pair; reuse the server +/// wiring (`BINANCE_{ENV}_API_KEY` + `BINANCE_{ENV}_PRIVATE_KEY_PATH`). +fn build_binance() -> Result> { + let env_name = env::var("BINANCE_ENV").unwrap_or_else(|_| "PAPER".into()).to_uppercase(); + let key = env::var(format!("BINANCE_{env_name}_API_KEY")) + .map_err(|_| format!("BINANCE_{env_name}_API_KEY must be set"))?; + let pem_path = env::var(format!("BINANCE_{env_name}_PRIVATE_KEY_PATH")) + .map_err(|_| format!("BINANCE_{env_name}_PRIVATE_KEY_PATH must be set"))?; + let pem = std::fs::read_to_string(&pem_path) + .map_err(|e| format!("reading {pem_path}: {e}"))?; + BinanceAdapter::new(key, &pem, &env_name).map_err(Into::into) +} + +/// A broker mapping row ready to upsert into `broker_instrument`. +struct BrokerRow { instrument_id: i64, - symbol: String, - exchange: Option, + broker_symbol: String, + broker_exchange: Option, native_id: Option, is_tradeable: bool, min_quantity: Option, + max_quantity: Option, + min_notional: Option, + max_notional: Option, } -fn to_row(instrument_id: i64, bi: &BrokerInstrument) -> BrokerXrefRow { - BrokerXrefRow { - instrument_id, - symbol: bi.symbol.clone(), - exchange: bi.exchange.clone(), - native_id: bi.native_id.clone(), - is_tradeable: bi.is_tradeable, - min_quantity: bi.min_quantity, - } -} - -/// Databento OPRA `raw_symbol` (21-char OCC OSI, root right-padded with spaces) -/// -> Alpaca's compact OSI. The only spaces are the root padding, so stripping -/// them yields Alpaca's `symbol` (`SPY 260713P00775000` -> `SPY260713P00775000`). -fn compact_osi(sym: &str) -> String { - sym.replace(' ', "") -} - -async fn sync_equities( - pool: &PgPool, - adapter: &AlpacaAdapter, - dry_run: bool, -) -> Result<(), Box> { - // symbol -> master id for the equity (SPOT) universe; Alpaca tickers match 1:1. - let id_map: HashMap = sqlx::query( - "SELECT symbol, id FROM instrument WHERE instrument_class = 'SPOT'", - ) - .fetch_all(pool) - .await? - .into_iter() - .map(|r| (r.get::("symbol"), r.get::("id"))) - .collect(); - info!("loaded {} SPOT instrument(s) from master", id_map.len()); - - let catalog = adapter.list_equity_instruments().await?; - info!("fetched {} active {BROKER_CODE} equity asset(s)", catalog.len()); - - let rows: Vec = catalog - .iter() - .filter_map(|bi| id_map.get(&bi.symbol).map(|&id| to_row(id, bi))) - .collect(); - info!("matched {} equity asset(s) to master ({} unmatched)", rows.len(), catalog.len() - rows.len()); - - write_rows(pool, &rows, dry_run, "equity").await -} - -async fn sync_options( - pool: &PgPool, - adapter: &AlpacaAdapter, - underlyings: &[String], - dry_run: bool, -) -> Result<(), Box> { - // compact OSI -> master id, scoped to the requested underlyings. - let id_map: HashMap = sqlx::query( - "SELECT i.id, i.symbol \ - FROM instrument i \ - JOIN instrument_derivative d ON d.instrument_id = i.id \ - WHERE i.instrument_class = 'OPTION' AND d.underlying_symbol = ANY($1)", - ) - .bind(underlyings) - .fetch_all(pool) - .await? - .into_iter() - .map(|r| (compact_osi(&r.get::("symbol")), r.get::("id"))) - .collect(); - info!("loaded {} OPTION instrument(s) for {underlyings:?}", id_map.len()); - - let catalog = adapter.list_option_contracts(underlyings).await?; - info!("fetched {} active {BROKER_CODE} option contract(s)", catalog.len()); - - let rows: Vec = catalog - .iter() - .filter_map(|bi| id_map.get(&bi.symbol).map(|&id| to_row(id, bi))) - .collect(); - info!("matched {} contract(s) to master ({} unmatched)", rows.len(), catalog.len() - rows.len()); - - write_rows(pool, &rows, dry_run, "option").await -} - -async fn write_rows( - pool: &PgPool, - rows: &[BrokerXrefRow], - dry_run: bool, - label: &str, -) -> Result<(), Box> { - if dry_run { - for r in rows.iter().take(20) { - info!(" {} -> instrument_id={} tradeable={}", r.symbol, r.instrument_id, r.is_tradeable); +impl BrokerRow { + fn from(instrument_id: i64, bi: &BrokerInstrument) -> Self { + Self { + instrument_id, + broker_symbol: bi.broker_symbol.clone(), + broker_exchange: bi.broker_exchange.clone(), + native_id: bi.definition.native_id.clone(), + is_tradeable: bi.is_tradeable, + min_quantity: bi.min_quantity, + max_quantity: bi.max_quantity, + min_notional: bi.min_notional, + max_notional: bi.max_notional, } - info!("dry run complete ({label}): {} row(s), no write", rows.len()); - return Ok(()); - } - if rows.is_empty() { - warn!("no {label} rows matched; nothing to upsert"); - return Ok(()); } - - let mut tx = pool.begin().await?; - let mut n = 0usize; - for chunk in rows.chunks(BATCH) { - n += bulk_upsert_broker_xref(&mut tx, chunk).await?; - } - tx.commit().await?; - info!("broker sync done: upserted {n} {BROKER_CODE} {label} xref row(s)"); - Ok(()) } -/// Bulk-upsert BROKER xref rows via UNNEST — mirrors `universe::bulk_upsert_xref` -/// but writes source_type='BROKER' plus the broker-routing columns -/// (`is_tradeable`, `min_quantity`). Same ON CONFLICT key. -async fn bulk_upsert_broker_xref( +/// Bulk-upsert `broker_instrument` rows via UNNEST, one row per instrument per +/// broker. Conflict key is (instrument_id, broker_code). +async fn bulk_upsert_broker_instrument( tx: &mut Transaction<'_, Postgres>, - chunk: &[BrokerXrefRow], + broker_code: &str, + chunk: &[BrokerRow], ) -> Result> { let instrument_id: Vec = chunk.iter().map(|r| r.instrument_id).collect(); - let symbol: Vec = chunk.iter().map(|r| r.symbol.clone()).collect(); - let exchange: Vec> = chunk.iter().map(|r| r.exchange.clone()).collect(); + let broker_symbol: Vec = chunk.iter().map(|r| r.broker_symbol.clone()).collect(); + let broker_exchange: Vec> = chunk.iter().map(|r| r.broker_exchange.clone()).collect(); let native_id: Vec> = chunk.iter().map(|r| r.native_id.clone()).collect(); let is_tradeable: Vec = chunk.iter().map(|r| r.is_tradeable).collect(); let min_quantity: Vec> = chunk.iter().map(|r| r.min_quantity).collect(); + let max_quantity: Vec> = chunk.iter().map(|r| r.max_quantity).collect(); + let min_notional: Vec> = chunk.iter().map(|r| r.min_notional).collect(); + let max_notional: Vec> = chunk.iter().map(|r| r.max_notional).collect(); sqlx::query( - "INSERT INTO oms.instrument_xref \ - (instrument_id, source_type, source_code, external_symbol, external_exchange, \ - external_native_id, is_tradeable, min_quantity, method, confidence) \ - SELECT t.iid, 'BROKER', $1, t.sym, t.exch, t.nid, t.trad, t.minq, 'broker_sync', 'resolved' \ - FROM UNNEST($2::bigint[], $3::text[], $4::text[], $5::text[], $6::bool[], $7::float8[]) \ - AS t(iid, sym, exch, nid, trad, minq) \ - ON CONFLICT (source_type, source_code, \ - COALESCE(external_symbol, ''), \ - COALESCE(external_exchange, '')) \ - DO UPDATE SET instrument_id = EXCLUDED.instrument_id, \ - external_native_id = EXCLUDED.external_native_id, \ - is_tradeable = EXCLUDED.is_tradeable, \ - min_quantity = EXCLUDED.min_quantity, \ - updated_at = now()", + "INSERT INTO broker_instrument \ + (instrument_id, broker_code, broker_symbol, broker_exchange, native_id, \ + is_tradeable, min_quantity, max_quantity, min_notional, max_notional) \ + SELECT t.iid, $1, t.sym, t.exch, t.nid, t.trad, t.minq, t.maxq, t.minn, t.maxn \ + FROM UNNEST($2::bigint[], $3::text[], $4::text[], $5::text[], $6::bool[], \ + $7::float8[], $8::float8[], $9::float8[], $10::float8[]) \ + AS t(iid, sym, exch, nid, trad, minq, maxq, minn, maxn) \ + ON CONFLICT (instrument_id, broker_code) \ + DO UPDATE SET broker_symbol = EXCLUDED.broker_symbol, \ + broker_exchange = EXCLUDED.broker_exchange, \ + native_id = EXCLUDED.native_id, \ + is_tradeable = EXCLUDED.is_tradeable, \ + min_quantity = EXCLUDED.min_quantity, \ + max_quantity = EXCLUDED.max_quantity, \ + min_notional = EXCLUDED.min_notional, \ + max_notional = EXCLUDED.max_notional, \ + updated_at = now()", ) - .bind(BROKER_CODE) + .bind(broker_code) .bind(&instrument_id) - .bind(&symbol) - .bind(&exchange) + .bind(&broker_symbol) + .bind(&broker_exchange) .bind(&native_id) .bind(&is_tradeable) .bind(&min_quantity) + .bind(&max_quantity) + .bind(&min_notional) + .bind(&max_notional) .execute(&mut **tx) .await?; Ok(chunk.len()) } - -#[cfg(test)] -mod tests { - use super::compact_osi; - - #[test] - fn compact_osi_strips_root_padding() { - // Databento space-padded OSI -> Alpaca compact OSI. - assert_eq!(compact_osi("SPY 260713P00775000"), "SPY260713P00775000"); - assert_eq!(compact_osi("QQQ 260713C00495000"), "QQQ260713C00495000"); - // 6-char root has no padding — unchanged. - assert_eq!(compact_osi("SPXW 260713C05000000"), "SPXW260713C05000000"); - // Already compact is idempotent. - assert_eq!(compact_osi("SPY260713P00775000"), "SPY260713P00775000"); - } -} diff --git a/src/setup/catalog.rs b/src/setup/catalog.rs new file mode 100644 index 0000000..0d63582 --- /dev/null +++ b/src/setup/catalog.rs @@ -0,0 +1,321 @@ +//! Shared master-catalog upsert: turn provider-neutral [`InstrumentDef`]s into +//! `public.instrument` (+ `instrument_derivative`) rows, and optionally enrich them +//! with external identifiers (FIGI/CUSIP/ISIN via OpenFIGI). +//! +//! Broker-first seeding drives this from `setup::brokers`: an adapter's +//! `InstrumentProvider` yields the definitions, this module writes the master rows +//! and hands back the `(symbol, venue) -> id` map so the caller can attach the +//! `broker_instrument` mapping. It does not touch any symbology bridge table — +//! broker mapping lives in `broker_instrument`, feed mapping in `feed_instrument`. + +use std::collections::{BTreeSet, HashMap, HashSet}; + +use dataprovider::{Enricher, InstrumentDef, OptionKind}; +use sqlx::{PgPool, Postgres, Transaction}; +use tracing::{info, warn}; + +/// Max rows per bulk statement. Arrays are passed as single binds, so this only +/// bounds statement/memory size, not the Postgres parameter limit. +const BATCH: usize = 4000; + +#[derive(Default, Debug)] +pub struct CatalogSummary { + pub upserted: u64, + pub skipped_venue: u64, + pub skipped_currency: u64, + pub skipped_expired: u64, + pub derivatives: u64, + pub enriched: u64, + /// The distinct venue/currency codes that failed the FK check. Distinct rather + /// than per-row: a skip count alone says something is wrong but not what, and + /// the answer is always a handful of codes even when thousands of rows drop. + pub unknown_venues: BTreeSet, + pub unknown_currencies: BTreeSet, +} + +impl CatalogSummary { + pub fn skipped_fk(&self) -> u64 { + self.skipped_venue + self.skipped_currency + } +} + +/// Upsert a batch of instrument definitions into the master catalog. FK-filters on +/// the known venue/currency sets, drops already-expired options, dedups by +/// (symbol, venue), upserts instruments then derivatives in one transaction, then +/// runs the enricher pipeline. Returns the summary plus the `(symbol, venue) -> id` +/// map for every upserted row so the caller can write dependent mappings. +pub async fn upsert_catalog( + pool: &PgPool, + defs: &[InstrumentDef], + enrichers: &[Box], +) -> Result<(CatalogSummary, HashMap<(String, String), i64>), Box> { + let mut summary = CatalogSummary::default(); + let mut ids: HashMap<(String, String), i64> = HashMap::new(); + if defs.is_empty() { + return Ok((summary, ids)); + } + + // FK filter in-process: load the valid venue/currency sets once instead of an + // EXISTS round-trip per row. + let venues: HashSet = sqlx::query_scalar("SELECT code FROM venue") + .fetch_all(pool) + .await? + .into_iter() + .collect(); + let currencies: HashSet = sqlx::query_scalar("SELECT code FROM currency") + .fetch_all(pool) + .await? + .into_iter() + .collect(); + + // FK-filter, drop expired options, then dedup by the conflict key (symbol, + // venue): a source can list the same instrument twice, which would make a bulk + // `ON CONFLICT` touch a row twice ("cannot affect row a second time"). Last + // write wins. + let today = chrono::Utc::now().date_naive(); + let mut dedup: HashMap<(&str, &str), &InstrumentDef> = HashMap::with_capacity(defs.len()); + for d in defs { + // Checked separately so the summary names which FK failed — "883 skipped" + // cost a manual DB session to diagnose once already. + if !venues.contains(&d.venue) { + summary.skipped_venue += 1; + summary.unknown_venues.insert(d.venue.clone()); + continue; + } + if !currencies.contains(&d.currency) { + summary.skipped_currency += 1; + summary.unknown_currencies.insert(d.currency.clone()); + continue; + } + if let Some(exp) = d.derivative.as_ref().and_then(|dv| dv.expiry_date) { + if exp < today { + summary.skipped_expired += 1; + continue; + } + } + dedup.insert((d.symbol.as_str(), d.venue.as_str()), d); + } + // Upsert SPOT before options so the underlying is visible to the derivative join + // (same transaction sees its own writes). + let mut valid: Vec<&InstrumentDef> = dedup.into_values().collect(); + valid.sort_by_key(|d| d.derivative.is_some()); + + info!( + "upserting {} instruments ({} skipped_fk, {} expired) …", + valid.len(), + summary.skipped_fk(), + summary.skipped_expired + ); + if !summary.unknown_venues.is_empty() { + warn!( + "{} row(s) skipped — no such venue: {:?} (seed the venue, or map the code in the adapter)", + summary.skipped_venue, summary.unknown_venues + ); + } + if !summary.unknown_currencies.is_empty() { + warn!( + "{} row(s) skipped — no such currency: {:?} (add to db/scripts/seed_currencies.sql)", + summary.skipped_currency, summary.unknown_currencies + ); + } + let mut tx = pool.begin().await?; + for chunk in valid.chunks(BATCH) { + let rows = bulk_upsert_instruments(&mut tx, chunk).await?; + summary.upserted += rows.len() as u64; + ids.extend(rows); + } + let derivs: Vec<&InstrumentDef> = + valid.iter().copied().filter(|d| d.derivative.is_some()).collect(); + for chunk in derivs.chunks(BATCH) { + summary.derivatives += bulk_upsert_derivatives(&mut tx, chunk, &ids).await? as u64; + } + tx.commit().await?; + + if !enrichers.is_empty() { + summary.enriched = run_enrichers(pool, enrichers, defs).await?; + } + Ok((summary, ids)) +} + +/// Bulk-upsert a chunk of instruments via UNNEST, returning `(symbol, venue) -> id` +/// for every row (inserted or updated). +async fn bulk_upsert_instruments( + tx: &mut Transaction<'_, Postgres>, + chunk: &[&InstrumentDef], +) -> Result, Box> { + let symbol: Vec = chunk.iter().map(|d| d.symbol.clone()).collect(); + let venue: Vec = chunk.iter().map(|d| d.venue.clone()).collect(); + let currency: Vec = chunk.iter().map(|d| d.currency.clone()).collect(); + let asset_class: Vec = chunk.iter().map(|d| d.asset_class.clone()).collect(); + let instrument_class: Vec = chunk.iter().map(|d| d.instrument_class.clone()).collect(); + let name: Vec = chunk + .iter() + .map(|d| d.name.clone().unwrap_or_else(|| d.symbol.clone())) + .collect(); + let price_precision: Vec = chunk.iter().map(|d| d.price_precision).collect(); + let price_increment: Vec = chunk.iter().map(|d| d.price_increment).collect(); + let size_increment: Vec = chunk.iter().map(|d| d.size_increment).collect(); + let lot_size: Vec> = chunk.iter().map(|d| d.lot_size).collect(); + let contract_size: Vec = chunk.iter().map(|d| d.contract_size).collect(); + + let rows: Vec<(String, String, i64)> = sqlx::query_as( + "INSERT INTO instrument \ + (symbol, venue, currency, asset_class, instrument_class, name, \ + price_precision, size_precision, price_increment, size_increment, \ + lot_size, contract_size) \ + SELECT * FROM UNNEST( \ + $1::text[], $2::text[], $3::text[], $4::text[], $5::text[], $6::text[], \ + $7::int4[], array_fill(0::int4, ARRAY[array_length($1::text[], 1)]), \ + $8::float8[], $9::float8[], $10::float8[], $11::float8[]) \ + ON CONFLICT (symbol, venue) DO UPDATE SET \ + currency = EXCLUDED.currency, \ + asset_class = EXCLUDED.asset_class, \ + instrument_class = EXCLUDED.instrument_class, \ + name = EXCLUDED.name, \ + price_precision = EXCLUDED.price_precision, \ + price_increment = EXCLUDED.price_increment, \ + size_increment = EXCLUDED.size_increment, \ + lot_size = EXCLUDED.lot_size, \ + contract_size = EXCLUDED.contract_size, \ + updated_at = now() \ + RETURNING symbol, venue, id", + ) + .bind(&symbol) + .bind(&venue) + .bind(¤cy) + .bind(&asset_class) + .bind(&instrument_class) + .bind(&name) + .bind(&price_precision) + .bind(&price_increment) + .bind(&size_increment) + .bind(&lot_size) + .bind(&contract_size) + .fetch_all(&mut **tx) + .await?; + + Ok(rows.into_iter().map(|(s, v, id)| ((s, v), id)).collect()) +} + +/// Bulk-upsert derivative rows for a chunk of option legs. The underlying is +/// resolved by an inner join to the SPOT row, so an option whose underlying is +/// absent is skipped. +async fn bulk_upsert_derivatives( + tx: &mut Transaction<'_, Postgres>, + chunk: &[&InstrumentDef], + ids: &HashMap<(String, String), i64>, +) -> Result> { + let mut instrument_id: Vec = Vec::with_capacity(chunk.len()); + let mut underlying_symbol: Vec = Vec::with_capacity(chunk.len()); + let mut option_kind: Vec> = Vec::with_capacity(chunk.len()); + let mut strike_price: Vec> = Vec::with_capacity(chunk.len()); + let mut expiry_date: Vec> = Vec::with_capacity(chunk.len()); + let mut activation_date: Vec> = Vec::with_capacity(chunk.len()); + for d in chunk { + let (Some(&id), Some(dv)) = + (ids.get(&(d.symbol.clone(), d.venue.clone())), d.derivative.as_ref()) + else { + continue; + }; + instrument_id.push(id); + underlying_symbol.push(dv.underlying_symbol.clone()); + option_kind.push(dv.option_kind.map(|k| match k { + OptionKind::Call => "CALL".to_string(), + OptionKind::Put => "PUT".to_string(), + })); + strike_price.push(dv.strike_price); + expiry_date.push(dv.expiry_date); + activation_date.push(dv.activation_date); + } + if instrument_id.is_empty() { + return Ok(0); + } + + let affected = sqlx::query( + "INSERT INTO instrument_derivative \ + (instrument_id, underlying_id, underlying_symbol, \ + option_kind, strike_price, expiry_date, activation_date) \ + SELECT t.iid, u.id, t.us, t.ok, t.strike, t.exp, t.act \ + FROM UNNEST($1::bigint[], $2::text[], $3::text[], $4::float8[], \ + $5::date[], $6::date[]) AS t(iid, us, ok, strike, exp, act) \ + JOIN instrument u ON u.symbol = t.us AND u.instrument_class = 'SPOT' \ + ON CONFLICT (instrument_id) DO UPDATE SET \ + underlying_id = EXCLUDED.underlying_id, \ + underlying_symbol = EXCLUDED.underlying_symbol, \ + option_kind = EXCLUDED.option_kind, \ + strike_price = EXCLUDED.strike_price, \ + expiry_date = EXCLUDED.expiry_date, \ + activation_date = EXCLUDED.activation_date", + ) + .bind(&instrument_id) + .bind(&underlying_symbol) + .bind(&option_kind) + .bind(&strike_price) + .bind(&expiry_date) + .bind(&activation_date) + .execute(&mut **tx) + .await? + .rows_affected(); + + Ok(affected as usize) +} + +/// Run each enricher over the FIGI-less SPOT rows and persist the identifiers it +/// resolves onto the master. Returns the number of instruments stamped. +async fn run_enrichers( + pool: &PgPool, + enrichers: &[Box], + defs: &[InstrumentDef], +) -> Result> { + let mut working: Vec = Vec::new(); + for d in defs.iter().filter(|d| d.instrument_class == "SPOT") { + let figi: Option> = + sqlx::query_scalar("SELECT figi FROM instrument WHERE symbol = $1 AND venue = $2") + .bind(&d.symbol) + .bind(&d.venue) + .fetch_optional(pool) + .await?; + if matches!(figi, Some(None)) { + working.push(d.clone()); + } + } + if working.is_empty() { + return Ok(0); + } + info!( + "enriching {} SPOT symbol(s) via {} enricher(s)", + working.len(), + enrichers.len() + ); + + let mut stamped = 0u64; + for enricher in enrichers { + let report = enricher.enrich(&mut working).await?; + for (symbol, venue) in &report.resolved { + if let Some(d) = working.iter().find(|d| &d.symbol == symbol && &d.venue == venue) { + persist_identifiers(pool, d).await?; + stamped += 1; + } + } + } + Ok(stamped) +} + +/// Stamp an enriched def's identifiers onto the master (only where still empty). +async fn persist_identifiers(pool: &PgPool, d: &InstrumentDef) -> Result<(), Box> { + sqlx::query( + "UPDATE instrument SET \ + figi = COALESCE(figi, $3), \ + cusip = COALESCE(cusip, $4), \ + isin = COALESCE(isin, $5) \ + WHERE symbol = $1 AND venue = $2", + ) + .bind(&d.symbol) + .bind(&d.venue) + .bind(d.identifiers.figi.as_deref()) + .bind(d.identifiers.cusip.as_deref()) + .bind(d.identifiers.isin.as_deref()) + .execute(pool) + .await?; + Ok(()) +} diff --git a/src/setup/feeds.rs b/src/setup/feeds.rs new file mode 100644 index 0000000..71a2204 --- /dev/null +++ b/src/setup/feeds.rs @@ -0,0 +1,137 @@ +//! `oms setup map-feed` — map a data feed's own symbols onto seeded instruments. +//! +//! Populates `public.feed_instrument`, the market-data mapping. Feeds are +//! independent of brokers: this maps a feed's symbology onto whatever instruments +//! exist, and it is deliberately 1:n — a crypto feed maps its pair onto the pair on +//! every venue that trades it (e.g. the Bybit feed prices the Binance-seeded +//! BTCUSDT). Idempotent; safe to re-run after seeding more instruments. +//! +//! This module owns the *orchestration* only. The symbology itself — which +//! instruments a feed can price and what it calls them — lives on each feed struct +//! as a [`FeedSymbology`] impl, next to the code that speaks that vendor's protocol +//! (`opra_stream.rs`, `binance_feed.rs`, `bybit_feed.rs`). That keeps a vendor's +//! quirks in one place and makes them unit-testable without a database. + +use clap::{Args as ClapArgs, ValueEnum}; +use dataprovider::FeedSymbology; +use sqlx::PgPool; +use tracing::{info, warn}; + +/// Max rows per bulk statement, matching `setup::catalog`. +const BATCH: usize = 4000; + +#[derive(Copy, Clone, Debug, PartialEq, Eq, ValueEnum)] +pub enum Feed { + /// Databento OPRA consolidated options NBBO. Maps the space-padded OSI. + Databento, + /// Binance spot book ticker. Maps the pair on the Binance venue. + Binance, + /// Bybit spot orderbook. Maps the pair onto every crypto instrument for it. + Bybit, +} + +impl Feed { + /// The feed struct that owns this feed's symbology. Unit structs, so these are + /// `'static` references to promoted constants — no allocation, no state. + fn symbology(self) -> &'static dyn FeedSymbology { + match self { + Feed::Databento => &crate::opra_stream::DatabentoOpraFeed, + Feed::Binance => &crate::binance_feed::BinanceFeed, + Feed::Bybit => &crate::bybit_feed::BybitFeed, + } + } +} + +#[derive(ClapArgs, Debug, Clone)] +pub struct Args { + /// Which feed's symbol mapping to (re)build. + #[arg(long, value_enum)] + pub feed: Feed, + /// Count what would be mapped, but write nothing. + #[arg(long)] + pub dry_run: bool, +} + +pub async fn run(args: Args) -> Result<(), Box> { + let pool = PgPool::connect(&super::database_url()?).await?; + let feed = args.feed.symbology(); + let feed_code = feed.code(); + + // The feed declares which slice of the catalog it can price; push that down into + // the query rather than scanning every instrument. + let filter = feed.candidates(); + let mut sql = String::from("SELECT id, symbol FROM instrument WHERE status = 'ACTIVE'"); + let mut binds: Vec<&str> = Vec::new(); + for (column, value) in [ + ("instrument_class", filter.instrument_class), + ("asset_class", filter.asset_class), + ("venue", filter.venue), + ] { + if let Some(v) = value { + binds.push(v); + sql.push_str(&format!(" AND {column} = ${}", binds.len())); + } + } + + let mut query = sqlx::query_as::<_, (i64, String)>(&sql); + for b in &binds { + query = query.bind(*b); + } + let candidates = query.fetch_all(&pool).await?; + + // Translate each master symbol into the feed's own. `None` means the feed + // declines it — counted, not silently dropped. + let mut feed_symbols: Vec = Vec::with_capacity(candidates.len()); + let mut instrument_ids: Vec = Vec::with_capacity(candidates.len()); + let mut declined: Vec = Vec::new(); + for (id, symbol) in candidates { + match feed.to_feed_symbol(&symbol) { + Some(feed_symbol) => { + feed_symbols.push(feed_symbol); + instrument_ids.push(id); + } + None => declined.push(symbol), + } + } + + if !declined.is_empty() { + // Bounded sample: the point is to notice a symbology mismatch, not to dump + // the catalog. + let sample: Vec<&str> = declined.iter().take(5).map(String::as_str).collect(); + warn!( + "{feed_code}: {} candidate(s) declined by symbology, e.g. {sample:?}", + declined.len() + ); + } + + if args.dry_run { + info!( + "dry run: {feed_code} would map {} instrument(s) ({} declined), no write", + feed_symbols.len(), + declined.len() + ); + return Ok(()); + } + + let mut affected = 0u64; + for (symbol_chunk, id_chunk) in feed_symbols.chunks(BATCH).zip(instrument_ids.chunks(BATCH)) { + affected += sqlx::query( + "INSERT INTO feed_instrument (feed_code, feed_symbol, instrument_id) \ + SELECT $1, t.feed_symbol, t.instrument_id \ + FROM UNNEST($2::text[], $3::bigint[]) AS t(feed_symbol, instrument_id) \ + ON CONFLICT (feed_code, feed_symbol, instrument_id) DO NOTHING", + ) + .bind(feed_code) + .bind(symbol_chunk) + .bind(id_chunk) + .execute(&pool) + .await? + .rows_affected(); + } + + info!( + "map-feed done: {feed_code} mapped {} instrument(s), {affected} new feed_instrument row(s)", + feed_symbols.len() + ); + Ok(()) +} diff --git a/src/setup/mod.rs b/src/setup/mod.rs index a7a97db..f9c2874 100644 --- a/src/setup/mod.rs +++ b/src/setup/mod.rs @@ -1,7 +1,8 @@ //! Setup / seeding subcommands invoked via `oms setup …`. pub mod brokers; -pub mod universe; +pub mod catalog; +pub mod feeds; use std::env; diff --git a/src/setup/universe.rs b/src/setup/universe.rs deleted file mode 100644 index 3e2f42f..0000000 --- a/src/setup/universe.rs +++ /dev/null @@ -1,809 +0,0 @@ -//! `oms setup universe` — seed the master instrument universe. -//! -//! The seeding framework: it is provider- and enricher-agnostic. It drives a -//! [`UniverseSource`] (Databento) and a `Vec>` (OpenFIGI today): -//! 1. Load universes from `public.instrument_universe` (+ child symbols). -//! 2. Ask the provider for a free cost estimate per universe. -//! 3. Print a cost table + interactive y/N confirm. -//! 4. Fetch definitions, upsert `instrument` + `instrument_derivative` + -//! `oms.instrument_xref` (PROVIDER rows). -//! 5. Run the enricher pipeline, persist `Identifiers` + one xref row per enricher. -//! 6. Write universe status back (`SEEDED` / `ERROR`). -//! -//! Adding a provider or a metadata source does not touch this file beyond -//! registering it in `run()`. - -use std::collections::{HashMap, HashSet}; -use std::env; -use std::io::{self, Write as _}; - -use clap::Args as ClapArgs; -use dataprovider::{ - Category, CostEstimate, DataProvider, DatabentoClient, Enricher, InstrumentDef, - OpenFigiEnricher, OptionKind, SType, UniverseSource, UniverseSpec, -}; -use sqlx::{PgPool, Row}; -use tracing::{info, warn}; - -#[derive(ClapArgs, Debug, Clone)] -pub struct Args { - /// Seed exactly this universe code. Omit to operate on every universe. - #[arg(long)] - pub universe: Option, - /// Estimate + print cost table only; no fetch, no writes. - #[arg(long)] - pub dry_run: bool, - /// Skip the interactive y/N confirm. - #[arg(long, short = 'y')] - pub yes: bool, - /// Skip the enricher pipeline (FIGI via OpenFIGI). - #[arg(long)] - pub no_enrich: bool, - /// Abort if the total estimated cost exceeds this value (USD). - #[arg(long)] - pub max_cost: Option, -} - -pub async fn run(args: Args) -> Result<(), Box> { - let pool = PgPool::connect(&super::database_url()?).await?; - let universes = load_universes(&pool, args.universe.as_deref()).await?; - if universes.is_empty() { - info!("no universes to seed."); - return Ok(()); - } - - let db = DatabentoClient::from_env()?; - db.set_catalog(universes.clone()).await; - - // Cost estimation (free metadata call). - let mut estimates: Vec = Vec::with_capacity(universes.len()); - for u in &universes { - estimates.push(db.estimate_cost(u).await?); - } - let total: f64 = estimates.iter().map(|c| c.usd).sum(); - print_cost_table(&universes, &estimates, total); - - if let Some(max) = args.max_cost { - if total > max { - return Err(format!("estimated cost ${total:.4} exceeds --max-cost ${max:.4}").into()); - } - } - if args.dry_run { - info!("dry run: {} universe(s), estimated ${total:.4}. no fetch or write.", universes.len()); - return Ok(()); - } - if !args.yes && !confirm_prompt(universes.len(), total)? { - info!("aborted."); - return Ok(()); - } - - // The enricher pipeline. Push a new `Box` here to add a - // metadata/identifier source — the rest of the framework is untouched. - let enrichers: Vec> = if args.no_enrich { - Vec::new() - } else { - vec![Box::new(OpenFigiEnricher::new(env::var("OPENFIGI_API_KEY").ok()))] - }; - - let mut failures: Vec = Vec::new(); - for u in &universes { - if let Err(e) = require_underlyings(u) { - warn!("skipping {}: {e}", u.code); - set_status(&pool, &u.code, "ERROR", None, Some(&e.to_string())).await?; - failures.push(u.code.clone()); - continue; - } - info!("seeding universe {} …", u.code); - set_status(&pool, &u.code, "SEEDING", None, None).await?; - - match seed_one(&pool, &db, u, &enrichers).await { - Ok(summary) => { - info!( - "{}: upserted={} skipped_fk={} skipped_expired={} derivatives={} provider_xref={} enriched={}", - u.code, - summary.upserted, - summary.skipped_fk, - summary.skipped_expired, - summary.derivatives, - summary.provider_xref, - summary.enriched - ); - let (status, note) = terminal_status(&pool, u).await; - set_status(&pool, &u.code, status, Some(summary.upserted as i32), note.as_deref()) - .await?; - } - Err(e) => { - warn!("{} failed: {e}", u.code); - set_status(&pool, &u.code, "ERROR", None, Some(&e.to_string())).await?; - failures.push(u.code.clone()); - } - } - } - - if !failures.is_empty() { - return Err(format!("seeding failed for: {}", failures.join(", ")).into()); - } - Ok(()) -} - -// ------------------------------------------------------------------------ -// Reusable core (shared by the CLI above and the admin API) -// ------------------------------------------------------------------------ - -/// Free cost estimate for a single universe. `Ok(None)` if the code is unknown. -pub async fn estimate( - pool: &PgPool, - code: &str, -) -> Result, Box> { - let mut universes = load_universes(pool, Some(code)).await?; - let Some(spec) = universes.pop() else { - return Ok(None); - }; - require_underlyings(&spec)?; - // OPTION universes: Databento's get_cost over OPRA parent symbols is - // pathologically slow (~12s for one underlying, 504 gateway timeout for 2+), - // and definition-schema cost is effectively $0 anyway. Skip the call and - // report a symbolic zero rather than hang/504. - if matches!(spec.category, Category::Option) { - return Ok(Some(CostEstimate { - universe_code: spec.code.clone(), - usd: 0.0, - symbol_count: Some(spec.symbols.len()), - })); - } - let db = DatabentoClient::from_env()?; - db.set_catalog(vec![spec.clone()]).await; - Ok(Some(db.estimate_cost(&spec).await?)) -} - -/// OPTION universes must name their underlyings — seeding the whole OPRA tape -/// (`ALL`) is ~1.5M contracts and not allowed. Equity/future universes may be -/// whole-dataset. -fn require_underlyings(spec: &UniverseSpec) -> Result<(), Box> { - if matches!(spec.category, Category::Option) && spec.symbols.is_empty() { - return Err(format!( - "OPTION universe {} has no underlyings — pick underlyings before seeding (ALL is not allowed for options)", - spec.code - ) - .into()); - } - Ok(()) -} - -/// Seed a single universe end to end: (optionally) gate on cost → fetch → upsert -/// → enrich, writing the universe's seed-state (`SEEDING` → `SEEDED`/`ERROR`) as -/// it goes. Intended for the admin API's background task — everything, including -/// the cost estimate/gate, runs here so the HTTP request returns immediately. -/// Runs in a spawned task, so the returned error must be `Send` — use `String` -/// rather than a `Box` that would poison the future's `Send` bound. -pub async fn seed( - pool: &PgPool, - code: &str, - enrich: bool, - max_cost: Option, -) -> Result<(), String> { - let mut universes = load_universes(pool, Some(code)) - .await - .map_err(|e| e.to_string())?; - let Some(spec) = universes.pop() else { - return Err(format!("unknown universe: {code}")); - }; - require_underlyings(&spec).map_err(|e| e.to_string())?; - - let db = DatabentoClient::from_env().map_err(|e| e.to_string())?; - db.set_catalog(vec![spec.clone()]).await; - - // Cost gate — only estimate when there is a budget to enforce, and never for - // OPTION universes (get_cost 504s on OPRA parents; definitions are ≈$0). - // Skipping otherwise avoids a dependency on Databento's flaky metadata.get_cost. - if let (Some(max), false) = (max_cost, matches!(spec.category, Category::Option)) { - let est = db.estimate_cost(&spec).await.map_err(|e| e.to_string())?; - if est.usd > max { - let msg = format!("estimated ${:.4} exceeds max_cost ${max:.4}", est.usd); - set_status(pool, &spec.code, "ERROR", None, Some(&msg)) - .await - .map_err(|e| e.to_string())?; - return Err(msg); - } - } - - let enrichers: Vec> = if enrich { - vec![Box::new(OpenFigiEnricher::new(env::var("OPENFIGI_API_KEY").ok()))] - } else { - Vec::new() - }; - - info!( - "seeding universe {} (enrich={}, {} underlying(s)) …", - spec.code, - enrich, - spec.symbols.len() - ); - set_status(pool, &spec.code, "SEEDING", None, None) - .await - .map_err(|e| e.to_string())?; - match seed_one(pool, &db, &spec, &enrichers).await.map_err(|e| e.to_string()) { - Ok(summary) => { - info!( - "{}: upserted={} skipped_fk={} skipped_expired={} derivatives={} provider_xref={} enriched={}", - spec.code, - summary.upserted, - summary.skipped_fk, - summary.skipped_expired, - summary.derivatives, - summary.provider_xref, - summary.enriched - ); - let (status, note) = terminal_status(pool, &spec).await; - set_status(pool, &spec.code, status, Some(summary.upserted as i32), note.as_deref()) - .await - .map_err(|e| e.to_string())?; - Ok(()) - } - Err(msg) => { - warn!("{} failed: {msg}", spec.code); - set_status(pool, &spec.code, "ERROR", None, Some(&msg)) - .await - .map_err(|e| e.to_string())?; - Err(msg) - } - } -} - -/// Terminal status for a just-seeded universe. For OPTION universes, `SEEDED` -/// only if every chosen underlying actually produced option rows; otherwise -/// `PARTIAL` with a note listing the underlyings whose chains didn't land (their -/// SPOT equity isn't seeded, or the provider returned no chain). Equity/future -/// universes are always `SEEDED` on success. -async fn terminal_status(pool: &PgPool, spec: &UniverseSpec) -> (&'static str, Option) { - if !matches!(spec.category, Category::Option) || spec.symbols.is_empty() { - return ("SEEDED", None); - } - // Which chosen underlyings ended up with at least one derivative row. - let seeded: Vec = sqlx::query_scalar( - "SELECT DISTINCT ius.symbol \ - FROM instrument_universe_symbol ius \ - JOIN instrument_derivative d ON d.underlying_symbol = ius.symbol \ - WHERE ius.universe_code = $1", - ) - .bind(&spec.code) - .fetch_all(pool) - .await - .unwrap_or_default(); - - let missing: Vec<&str> = spec - .symbols - .iter() - .filter(|s| !seeded.iter().any(|x| x == *s)) - .map(|s| s.as_str()) - .collect(); - if missing.is_empty() { - ("SEEDED", None) - } else { - let note = format!( - "{}/{} underlyings seeded; no chain for: {}", - spec.symbols.len() - missing.len(), - spec.symbols.len(), - missing.join(", ") - ); - ("PARTIAL", Some(note)) - } -} - -// ------------------------------------------------------------------------ -// Per-universe work -// ------------------------------------------------------------------------ - -#[derive(Default)] -struct SeedSummary { - upserted: u64, - skipped_fk: u64, - skipped_expired: u64, - derivatives: u64, - provider_xref: u64, - enriched: u64, -} - -async fn seed_one( - pool: &PgPool, - db: &DatabentoClient, - spec: &UniverseSpec, - enrichers: &[Box], -) -> Result> { - let t_fetch = std::time::Instant::now(); - let defs = db.fetch_definitions(spec).await?; - info!( - "{}: fetched {} definitions in {:.1}s", - spec.code, - defs.len(), - t_fetch.elapsed().as_secs_f64() - ); - if defs.is_empty() { - info!("{}: no mappable instruments.", spec.code); - return Ok(SeedSummary::default()); - } - - let mut summary = SeedSummary::default(); - - // FK filter in-process: load the valid venue/currency sets once instead of - // an EXISTS round-trip per row. - let venues: HashSet = sqlx::query_scalar("SELECT code FROM venue") - .fetch_all(pool) - .await? - .into_iter() - .collect(); - let currencies: HashSet = sqlx::query_scalar("SELECT code FROM currency") - .fetch_all(pool) - .await? - .into_iter() - .collect(); - - // FK-filter, then dedup by the instrument conflict key (symbol, venue): a - // multi-day definition range can return the same instrument more than once, - // which would make a bulk `ON CONFLICT` statement touch a row twice - // ("cannot affect row a second time"). Last write wins. This key also covers - // the xref and derivative statements, whose conflict keys derive from it. - let today = chrono::Utc::now().date_naive(); - let mut dedup: HashMap<(&str, &str), &InstrumentDef> = HashMap::with_capacity(defs.len()); - for d in &defs { - if !(venues.contains(&d.venue) && currencies.contains(&d.currency)) { - summary.skipped_fk += 1; - continue; - } - // Skip already-expired options — dead contracts, not worth seeding. Only - // drops when an expiry is present and in the past; missing expiry is kept. - if let Some(exp) = d.derivative.as_ref().and_then(|dv| dv.expiry_date) { - if exp < today { - summary.skipped_expired += 1; - continue; - } - } - dedup.insert((d.symbol.as_str(), d.venue.as_str()), d); - } - // Upsert equities before options so the underlying SPOT row is visible to the - // derivative join (same transaction sees its own writes). - let mut valid: Vec<&InstrumentDef> = dedup.into_values().collect(); - valid.sort_by_key(|d| d.derivative.is_some()); - - info!( - "{}: upserting {} instruments ({} skipped_fk, {} expired) …", - spec.code, - valid.len(), - summary.skipped_fk, - summary.skipped_expired - ); - let t_upsert = std::time::Instant::now(); - let mut tx = pool.begin().await?; - - // 1. Bulk-upsert instruments, chunked; collect the (symbol, venue) -> id map. - let mut ids: HashMap<(String, String), i64> = HashMap::with_capacity(valid.len()); - for chunk in valid.chunks(BATCH) { - let rows = bulk_upsert_instruments(&mut tx, chunk).await?; - summary.upserted += rows.len() as u64; - ids.extend(rows); - } - - // 2. Bulk-upsert PROVIDER xref rows. - for chunk in valid.chunks(BATCH) { - summary.provider_xref += - bulk_upsert_xref(&mut tx, db.code(), chunk, &ids).await? as u64; - } - - // 3. Bulk-upsert derivative rows for the option legs. - let derivs: Vec<&InstrumentDef> = valid.iter().copied().filter(|d| d.derivative.is_some()).collect(); - for chunk in derivs.chunks(BATCH) { - summary.derivatives += - bulk_upsert_derivatives(&mut tx, chunk, &ids).await? as u64; - } - - tx.commit().await?; - info!( - "{}: upserted {} instruments ({} derivatives, {} xref) in {:.1}s", - spec.code, - summary.upserted, - summary.derivatives, - summary.provider_xref, - t_upsert.elapsed().as_secs_f64() - ); - - // Enricher pipeline (fills Identifiers, persists figi/cusip/isin + xref). - if !enrichers.is_empty() { - let t_enrich = std::time::Instant::now(); - summary.enriched = run_enrichers(pool, enrichers, &defs).await?; - info!( - "{}: enriched {} instruments in {:.1}s", - spec.code, - summary.enriched, - t_enrich.elapsed().as_secs_f64() - ); - } - - Ok(summary) -} - -/// Max rows per bulk statement. Arrays are passed as single binds, so this only -/// bounds statement/memory size, not the Postgres parameter limit. -const BATCH: usize = 4000; - -/// Bulk-upsert a chunk of instruments via UNNEST, returning `(symbol, venue) -> id` -/// for every row (inserted or updated). -async fn bulk_upsert_instruments( - tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, - chunk: &[&InstrumentDef], -) -> Result, Box> { - let symbol: Vec = chunk.iter().map(|d| d.symbol.clone()).collect(); - let venue: Vec = chunk.iter().map(|d| d.venue.clone()).collect(); - let currency: Vec = chunk.iter().map(|d| d.currency.clone()).collect(); - let asset_class: Vec = chunk.iter().map(|d| d.asset_class.clone()).collect(); - let instrument_class: Vec = chunk.iter().map(|d| d.instrument_class.clone()).collect(); - let name: Vec = chunk - .iter() - .map(|d| d.name.clone().unwrap_or_else(|| d.symbol.clone())) - .collect(); - let price_precision: Vec = chunk.iter().map(|d| d.price_precision).collect(); - let price_increment: Vec = chunk.iter().map(|d| d.price_increment).collect(); - let size_increment: Vec = chunk.iter().map(|d| d.size_increment).collect(); - let lot_size: Vec> = chunk.iter().map(|d| d.lot_size).collect(); - let contract_size: Vec = chunk.iter().map(|d| d.contract_size).collect(); - - let rows: Vec<(String, String, i64)> = sqlx::query_as( - "INSERT INTO instrument \ - (symbol, venue, currency, asset_class, instrument_class, name, \ - price_precision, size_precision, price_increment, size_increment, \ - lot_size, contract_size) \ - SELECT * FROM UNNEST( \ - $1::text[], $2::text[], $3::text[], $4::text[], $5::text[], $6::text[], \ - $7::int4[], array_fill(0::int4, ARRAY[array_length($1::text[], 1)]), \ - $8::float8[], $9::float8[], $10::float8[], $11::float8[]) \ - ON CONFLICT (symbol, venue) DO UPDATE SET \ - currency = EXCLUDED.currency, \ - asset_class = EXCLUDED.asset_class, \ - instrument_class = EXCLUDED.instrument_class, \ - name = EXCLUDED.name, \ - price_precision = EXCLUDED.price_precision, \ - price_increment = EXCLUDED.price_increment, \ - size_increment = EXCLUDED.size_increment, \ - lot_size = EXCLUDED.lot_size, \ - contract_size = EXCLUDED.contract_size, \ - updated_at = now() \ - RETURNING symbol, venue, id", - ) - .bind(&symbol) - .bind(&venue) - .bind(¤cy) - .bind(&asset_class) - .bind(&instrument_class) - .bind(&name) - .bind(&price_precision) - .bind(&price_increment) - .bind(&size_increment) - .bind(&lot_size) - .bind(&contract_size) - .fetch_all(&mut **tx) - .await?; - - Ok(rows.into_iter().map(|(s, v, id)| ((s, v), id)).collect()) -} - -/// Bulk-upsert PROVIDER xref rows for a chunk. Returns the number written. -async fn bulk_upsert_xref( - tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, - source_code: &str, - chunk: &[&InstrumentDef], - ids: &HashMap<(String, String), i64>, -) -> Result> { - let mut instrument_id: Vec = Vec::with_capacity(chunk.len()); - let mut external_symbol: Vec = Vec::with_capacity(chunk.len()); - let mut external_exchange: Vec> = Vec::with_capacity(chunk.len()); - let mut external_native_id: Vec> = Vec::with_capacity(chunk.len()); - for d in chunk { - let Some(&id) = ids.get(&(d.symbol.clone(), d.venue.clone())) else { - continue; - }; - instrument_id.push(id); - external_symbol.push(d.symbol.clone()); - external_exchange.push(d.provider_exchange.clone()); - external_native_id.push(d.native_id.clone()); - } - if instrument_id.is_empty() { - return Ok(0); - } - - sqlx::query( - "INSERT INTO oms.instrument_xref \ - (instrument_id, source_type, source_code, external_symbol, \ - external_exchange, external_native_id, method, confidence) \ - SELECT t.iid, 'PROVIDER', $1, t.sym, t.exch, t.nid, 'seed_instruments', 'resolved' \ - FROM UNNEST($2::bigint[], $3::text[], $4::text[], $5::text[]) \ - AS t(iid, sym, exch, nid) \ - ON CONFLICT (source_type, source_code, \ - COALESCE(external_symbol, ''), \ - COALESCE(external_exchange, '')) \ - DO UPDATE SET instrument_id = EXCLUDED.instrument_id, \ - external_native_id = EXCLUDED.external_native_id, \ - updated_at = now()", - ) - .bind(source_code) - .bind(&instrument_id) - .bind(&external_symbol) - .bind(&external_exchange) - .bind(&external_native_id) - .execute(&mut **tx) - .await?; - - Ok(instrument_id.len()) -} - -/// Bulk-upsert derivative rows for a chunk of option legs. The underlying is -/// resolved by an inner join to the SPOT row, so an option whose underlying is -/// absent is skipped (matches the prior per-row behavior). -async fn bulk_upsert_derivatives( - tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, - chunk: &[&InstrumentDef], - ids: &HashMap<(String, String), i64>, -) -> Result> { - let mut instrument_id: Vec = Vec::with_capacity(chunk.len()); - let mut underlying_symbol: Vec = Vec::with_capacity(chunk.len()); - let mut option_kind: Vec> = Vec::with_capacity(chunk.len()); - let mut strike_price: Vec> = Vec::with_capacity(chunk.len()); - let mut expiry_date: Vec> = Vec::with_capacity(chunk.len()); - let mut activation_date: Vec> = Vec::with_capacity(chunk.len()); - for d in chunk { - let (Some(&id), Some(dv)) = - (ids.get(&(d.symbol.clone(), d.venue.clone())), d.derivative.as_ref()) - else { - continue; - }; - instrument_id.push(id); - underlying_symbol.push(dv.underlying_symbol.clone()); - option_kind.push(dv.option_kind.map(|k| match k { - OptionKind::Call => "CALL".to_string(), - OptionKind::Put => "PUT".to_string(), - })); - strike_price.push(dv.strike_price); - expiry_date.push(dv.expiry_date); - activation_date.push(dv.activation_date); - } - if instrument_id.is_empty() { - return Ok(0); - } - - let affected = sqlx::query( - "INSERT INTO instrument_derivative \ - (instrument_id, underlying_id, underlying_symbol, \ - option_kind, strike_price, expiry_date, activation_date) \ - SELECT t.iid, u.id, t.us, t.ok, t.strike, t.exp, t.act \ - FROM UNNEST($1::bigint[], $2::text[], $3::text[], $4::float8[], \ - $5::date[], $6::date[]) AS t(iid, us, ok, strike, exp, act) \ - JOIN instrument u ON u.symbol = t.us AND u.instrument_class = 'SPOT' \ - ON CONFLICT (instrument_id) DO UPDATE SET \ - underlying_id = EXCLUDED.underlying_id, \ - underlying_symbol = EXCLUDED.underlying_symbol, \ - option_kind = EXCLUDED.option_kind, \ - strike_price = EXCLUDED.strike_price, \ - expiry_date = EXCLUDED.expiry_date, \ - activation_date = EXCLUDED.activation_date", - ) - .bind(&instrument_id) - .bind(&underlying_symbol) - .bind(&option_kind) - .bind(&strike_price) - .bind(&expiry_date) - .bind(&activation_date) - .execute(&mut **tx) - .await? - .rows_affected(); - - Ok(affected as usize) -} - -/// Run each enricher over the FIGI-less SPOT rows and persist what they resolve. -/// Returns the number of `(instrument, enricher)` stamps written. -async fn run_enrichers( - pool: &PgPool, - enrichers: &[Box], - defs: &[InstrumentDef], -) -> Result> { - // Working set: SPOT rows still missing a FIGI in the master (idempotent). - let mut working: Vec = Vec::new(); - for d in defs.iter().filter(|d| d.instrument_class == "SPOT") { - let figi: Option> = - sqlx::query_scalar("SELECT figi FROM instrument WHERE symbol = $1 AND venue = $2") - .bind(&d.symbol) - .bind(&d.venue) - .fetch_optional(pool) - .await?; - if matches!(figi, Some(None)) { - working.push(d.clone()); - } - } - if working.is_empty() { - return Ok(0); - } - info!( - "enriching {} SPOT symbol(s) via {} enricher(s) — this is the slow phase (external lookups + per-row writes)", - working.len(), - enrichers.len() - ); - - let mut stamped = 0u64; - for enricher in enrichers { - let report = enricher.enrich(&mut working).await?; - let total = report.resolved.len(); - for (i, (symbol, venue)) in report.resolved.iter().enumerate() { - if let Some(d) = working.iter().find(|d| &d.symbol == symbol && &d.venue == venue) { - persist_identifiers(pool, d, enricher.code()).await?; - stamped += 1; - } - if (i + 1) % 500 == 0 { - info!(" {} enrich: persisted {}/{}", enricher.code(), i + 1, total); - } - } - } - Ok(stamped) -} - -/// Persist an enriched def's identifiers onto the master and stamp one xref row -/// attributed to the enricher (`source_code`). -async fn persist_identifiers( - pool: &PgPool, - d: &InstrumentDef, - source_code: &str, -) -> Result<(), Box> { - sqlx::query( - "UPDATE instrument SET \ - figi = COALESCE(figi, $3), \ - cusip = COALESCE(cusip, $4), \ - isin = COALESCE(isin, $5) \ - WHERE symbol = $1 AND venue = $2", - ) - .bind(&d.symbol) - .bind(&d.venue) - .bind(d.identifiers.figi.as_deref()) - .bind(d.identifiers.cusip.as_deref()) - .bind(d.identifiers.isin.as_deref()) - .execute(pool) - .await?; - - sqlx::query( - "INSERT INTO oms.instrument_xref \ - (instrument_id, source_type, source_code, external_symbol, \ - external_exchange, figi, method, confidence) \ - SELECT i.id, $3, $3, i.symbol, i.venue, $4, 'enrich', 'resolved' \ - FROM instrument i WHERE i.symbol = $1 AND i.venue = $2 \ - ON CONFLICT (source_type, source_code, \ - COALESCE(external_symbol, ''), \ - COALESCE(external_exchange, '')) \ - DO UPDATE SET instrument_id = EXCLUDED.instrument_id, \ - figi = EXCLUDED.figi, updated_at = now()", - ) - .bind(&d.symbol) - .bind(&d.venue) - .bind(source_code) - .bind(d.identifiers.figi.as_deref()) - .execute(pool) - .await?; - Ok(()) -} - -// ------------------------------------------------------------------------ -// Catalog IO -// ------------------------------------------------------------------------ - -async fn load_universes( - pool: &PgPool, - code: Option<&str>, -) -> Result, Box> { - let rows = if let Some(code) = code { - sqlx::query( - "SELECT code, description, category, dataset, option_dataset, stype_in, \ - include_options \ - FROM instrument_universe WHERE code = $1", - ) - .bind(code) - .fetch_all(pool) - .await? - } else { - sqlx::query( - "SELECT code, description, category, dataset, option_dataset, stype_in, \ - include_options \ - FROM instrument_universe ORDER BY category, code", - ) - .fetch_all(pool) - .await? - }; - - let mut out: Vec = Vec::with_capacity(rows.len()); - for r in rows { - let code: String = r.get("code"); - let symbols: Vec = sqlx::query_scalar( - "SELECT symbol FROM instrument_universe_symbol \ - WHERE universe_code = $1 ORDER BY symbol", - ) - .bind(&code) - .fetch_all(pool) - .await?; - let stype_in_s: String = r.get("stype_in"); - let cat_s: String = r.get("category"); - out.push(UniverseSpec { - code, - description: r.get("description"), - category: parse_category(&cat_s)?, - dataset: r.get("dataset"), - option_dataset: r.get("option_dataset"), - symbols, - stype_in: match stype_in_s.as_str() { - "parent" => SType::Parent, - _ => SType::RawSymbol, - }, - include_options: r.get("include_options"), - }); - } - Ok(out) -} - -fn parse_category(s: &str) -> Result> { - Ok(match s { - "EQUITY" => Category::Equity, - "OPTION" => Category::Option, - "FUTURE" => Category::Future, - other => return Err(format!("unknown universe category: {other}").into()), - }) -} - -async fn set_status( - pool: &PgPool, - code: &str, - status: &str, - count: Option, - error: Option<&str>, -) -> Result<(), sqlx::Error> { - sqlx::query( - "UPDATE instrument_universe SET \ - status = $2, \ - last_seeded_at = CASE WHEN $2 = 'SEEDED' THEN now() ELSE last_seeded_at END, \ - instrument_count = COALESCE($3, instrument_count), \ - last_error = $4, \ - updated_at = now() \ - WHERE code = $1", - ) - .bind(code) - .bind(status) - .bind(count) - .bind(error) - .execute(pool) - .await?; - Ok(()) -} - -// ------------------------------------------------------------------------ -// UI helpers -// ------------------------------------------------------------------------ - -fn print_cost_table(universes: &[UniverseSpec], estimates: &[CostEstimate], total: f64) { - println!("\nEstimated Databento cost (definition schema, free to estimate):\n"); - for (u, c) in universes.iter().zip(estimates) { - let nsym = if u.symbols.is_empty() { - "ALL".to_string() - } else { - u.symbols.len().to_string() - }; - println!( - " {:<16} {:<7} sym={:<4} ${:>10.4}", - u.code, - format!("{:?}", u.category).to_uppercase(), - nsym, - c.usd, - ); - } - println!(" {:<30} ${:>10.4}\n", "TOTAL", total); -} - -fn confirm_prompt(n: usize, total: f64) -> io::Result { - print!("Seed {n} universe(s) for ~${total:.4}? [y/N]: "); - io::stdout().flush()?; - let mut buf = String::new(); - io::stdin().read_line(&mut buf)?; - Ok(buf.trim().eq_ignore_ascii_case("y")) -} - diff --git a/src/stream_health.rs b/src/stream_health.rs index 9f9b663..e003f8a 100644 --- a/src/stream_health.rs +++ b/src/stream_health.rs @@ -23,11 +23,22 @@ pub enum StreamState { Down, } +/// What a stream is for — the axis the cockpit splits on. A data `Feed` delivers +/// market data (quotes); an `Execution` stream carries a broker's order/fill +/// updates. The same vendor can run both (Binance feed + Binance execution). +#[derive(Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum StreamKind { + Feed, + Execution, +} + /// A point-in-time snapshot of one stream's health, returned by the API. #[derive(Clone, Serialize)] pub struct StreamHealth { pub broker_code: String, pub environment: String, + pub kind: StreamKind, pub state: StreamState, /// When the current live session was established (cleared on disconnect). pub connected_since: Option>, @@ -51,13 +62,14 @@ impl StreamHealthRegistry { } /// Hand a stream task its own updater. Seeds a `Connecting` entry so the - /// broker shows up in the API from the moment the task starts. - pub fn handle(&self, broker_code: &str, environment: &str) -> StreamHandle { + /// stream shows up in the API from the moment the task starts. + pub fn handle(&self, broker_code: &str, environment: &str, kind: StreamKind) -> StreamHandle { let key = (broker_code.to_string(), environment.to_string()); if let Ok(mut map) = self.inner.write() { map.entry(key.clone()).or_insert_with(|| StreamHealth { broker_code: broker_code.to_string(), environment: environment.to_string(), + kind, state: StreamState::Connecting, connected_since: None, last_event_at: None, diff --git a/src/symbology_resolver.rs b/src/symbology_resolver.rs index 88e2382..48fbf0d 100644 --- a/src/symbology_resolver.rs +++ b/src/symbology_resolver.rs @@ -1,12 +1,15 @@ -//! OMS-side instrument resolution: the `symbology` (OpenFIGI) engine + the DB. +//! OMS-side symbology resolution: identify an external instrument via the +//! `symbology` (OpenFIGI) engine and stamp the FIGI/CUSIP anchor onto the master +//! `public.instrument`. //! -//! `resolve` is xref-first (a local `oms.instrument_xref` hit avoids OpenFIGI), then -//! falls back to the engine; on a hit it matches the FIGI identity to a master -//! `public.instrument`, stamps `figi`/`cusip`, and upserts the xref. Additive: it does -//! not touch the legacy `broker_instrument`/`provider_instrument` bridges or routing. +//! This is enrichment, not mapping. The two mapping tables are owned elsewhere — +//! `broker_instrument` by broker sync, `feed_instrument` by feed mapping. The +//! resolver only answers "which master instrument is this, and what is its FIGI", +//! and records the FIGI anchor so later lookups are cheap. Off the order/quote hot +//! path. use serde::Serialize; -use sqlx::{PgPool, Row}; +use sqlx::PgPool; use symbology::{InstrumentIdentity, InstrumentQuery, Resolution, SymbologyError}; use crate::app_state::SymbologyEngine; @@ -70,41 +73,13 @@ impl From for ResolveError { } } -/// Resolve one query to a master instrument + FIGI, persisting the result. +/// Identify one query via OpenFIGI, match it to a master instrument, and stamp the +/// FIGI/CUSIP anchor onto that master (only where still empty). pub async fn resolve( pool: &PgPool, engine: &SymbologyEngine, query: &InstrumentQuery, - source_type: &str, - source_code: &str, ) -> Result { - let ext_symbol = query.ticker.as_deref(); - let ext_exchange = query.exch_code.as_deref().or(query.mic.as_deref()); - - // 1) xref-first: a prior resolution for this (source, symbol, exchange). - if let Some(row) = sqlx::query( - "SELECT instrument_id, figi FROM instrument_xref \ - WHERE source_code = $1 \ - AND external_symbol IS NOT DISTINCT FROM $2 \ - AND external_exchange IS NOT DISTINCT FROM $3 \ - ORDER BY updated_at DESC LIMIT 1", - ) - .bind(source_code) - .bind(ext_symbol) - .bind(ext_exchange) - .fetch_optional(pool) - .await? - { - if let Some(figi) = row.get::, _>("figi") { - return Ok(ResolveOutcome::Resolved { - instrument_id: row.get("instrument_id"), - figi, - identity: None, // cache hit — minimal - }); - } - } - - // 2) engine (OpenFIGI, cached in-process). let identity = match engine.identify(query).await? { Resolution::Resolved(i) => i, Resolution::Ambiguous(cands) => { @@ -118,7 +93,7 @@ pub async fn resolve( let instrument_id = find_master(pool, &identity, query).await?; if let Some(id) = instrument_id { - // stamp the FIGI/CUSIP anchor on the master (narrow grant; only if empty). + // Stamp the FIGI/CUSIP anchor on the master (narrow grant; only if empty). // NB: only figi/cusip are granted to the app role — do NOT touch updated_at. sqlx::query( "UPDATE instrument SET figi = COALESCE(figi, $2), cusip = COALESCE(cusip, $3) \ @@ -131,8 +106,6 @@ pub async fn resolve( .await?; } - upsert_xref(pool, source_type, source_code, ext_symbol, ext_exchange, &identity, instrument_id).await?; - Ok(ResolveOutcome::Resolved { instrument_id, figi: identity.figi.clone(), @@ -179,31 +152,3 @@ async fn find_master( Ok(None) } - -async fn upsert_xref( - pool: &PgPool, - source_type: &str, - source_code: &str, - symbol: Option<&str>, - exchange: Option<&str>, - identity: &InstrumentIdentity, - instrument_id: Option, -) -> Result<(), sqlx::Error> { - sqlx::query( - "INSERT INTO instrument_xref \ - (instrument_id, source_type, source_code, external_symbol, external_exchange, figi, method, confidence) \ - VALUES ($1, $2, $3, $4, $5, $6, 'openfigi', 'resolved') \ - ON CONFLICT (source_type, source_code, \ - COALESCE(external_symbol, ''), COALESCE(external_exchange, '')) \ - DO UPDATE SET instrument_id = EXCLUDED.instrument_id, figi = EXCLUDED.figi, updated_at = now()", - ) - .bind(instrument_id) - .bind(source_type) - .bind(source_code) - .bind(symbol) - .bind(exchange) - .bind(&identity.figi) - .execute(pool) - .await?; - Ok(()) -}