diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 584bedd..dfe2350 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -4,7 +4,7 @@ name: Pages on: push: branches: [main] - paths: ["site/**", "bindings/**", "src/**", "Cargo.*", ".github/workflows/pages.yml"] + paths: ["site/**", "bindings/**", "src/**", "data/**", "Cargo.*", ".github/workflows/pages.yml"] workflow_dispatch: permissions: @@ -27,6 +27,10 @@ jobs: version=$(cargo metadata --format-version 1 --locked | jq -r '.packages[] | select(.name == "wasm-bindgen") | .version') cargo install wasm-bindgen-cli --version "$version" --locked wasm-bindgen --target web --out-dir site/pkg target/wasm32-unknown-unknown/release/setoff_engine_wasm.wasm + - name: Add the mainnet-day sample + run: | + mkdir -p site/samples + cp data/mainnet-2026-10-21.csv site/samples/mainnet-day.csv - uses: actions/configure-pages@v6 - uses: actions/upload-pages-artifact@v5 with: diff --git a/.gitignore b/.gitignore index 1834630..9ecab35 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ target/ site/pkg/ bindings/wasm/pkg-node/ +site/samples/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 64e2b95..b3eece2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,13 @@ All notable changes to this project are documented here. The format follows ### Added +- A real mainnet day: `tools/mainnet_day.py` builds a netting window from one + UTC day of USDC payments between institutions on Stellar, and + `examples/mainnet_day.rs` nets it. On 2026-10-21, 336 payments between 10 + institutions settle in 9 transfers ([method and limits](docs/mainnet-day.md)). +- Simulator: **A real mainnet day** loads that day, showing results in USDC, + and **Import CSV** reads `id,debtor,creditor,asset,amount` files in the page, + reporting malformed lines by number while loading the rest. - `setoff-engine-wasm` (`bindings/wasm`): the netting engine for JavaScript, with per-row validation errors and both plan strategies. - A [netting simulator](https://setoff-org.github.io/setoff-engine/) on diff --git a/Cargo.toml b/Cargo.toml index ba6e40c..2e7b5cf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,6 +35,10 @@ criterion = { version = "0.8", default-features = false } name = "net" harness = false +[[example]] +name = "mainnet_day" +required-features = ["csv"] + [lints.rust] unsafe_code = "forbid" missing_docs = "warn" diff --git a/README.md b/README.md index 4b4a2f1..53483c3 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,13 @@ What the reference cases in [`tests/vectors`](tests/vectors) show: | 200 pseudo-random, USDC | 145 | 715.48 → 141.83 | 80.2% | 11 | | Fan-in to one merchant | 5 | 150 → 150 | 0% | 5 | +And on real data: every USDC payment between 10 institutions (exchanges, +Circle and others) on Stellar mainnet on 2026-10-21, 336 payments and +55.5 million USDC, would have settled in **9 transfers instead of 336**. Value +moved fell only 3.9% that day, because the flows were mostly one-way; the +gain is 37 times fewer transactions to sign, fee and reconcile. +[How it was measured, and its limits](docs/mainnet-day.md). + ## Install ```sh diff --git a/data/mainnet-2026-10-21.csv b/data/mainnet-2026-10-21.csv new file mode 100644 index 0000000..ed0209a --- /dev/null +++ b/data/mainnet-2026-10-21.csv @@ -0,0 +1,337 @@ +id,debtor,creditor,asset,amount +279398591951519745,coinbase.com,binance.com,USDC,75440000000 +279405296395776001,kraken.com,centre.io,USDC,915370000000 +279409711621713921,coinbase.com,binance.com,USDC,80000000 +279416111123501057,binance.com,centre.io,USDC,190000000 +279417738915856385,kraken.com,centre.io,USDC,670250000000 +279424572208930817,bitstamp.net,coinbase.com,USDC,3359875000 +279426161346531329,changelly.com,hitbtc.com,USDC,600000000000 +279426174231572481,hitbtc.com,binance.com,USDC,700050006000 +279427063289565185,bitstamp.net,binance.com,USDC,4183079800 +279429030384541697,binance.com,centre.io,USDC,500000000 +279429786299125761,binance.com,centre.io,USDC,99999980000000 +279429863608750081,binance.com,centre.io,USDC,99999980000000 +279429940917989377,binance.com,centre.io,USDC,99999980000000 +279430576572813313,binance.com,centre.io,USDC,99999980000000 +279430688241954817,kraken.com,centre.io,USDC,1141320000000 +279430739781808129,binance.com,centre.io,USDC,99999980000000 +279435773483511809,binance.com,coinbase.com,USDC,260000000 +279436331828948993,kraken.com,centre.io,USDC,1586720000000 +279436400548888577,coinbase.com,binance.com,USDC,1101670970 +279439235227254785,binance.com,coinbase.com,USDC,119880000000 +279439239522037761,coinbase.com,bitrue.com,USDC,4000000000 +279439527284785153,binance.com,centre.io,USDC,110399982 +279440704105848833,coinbase.com,binance.com,USDC,12300000000 +279441107832762369,binance.com,kraken.com,USDC,60000000 +279441120717447169,centre.io,binance.com,USDC,41900000 +279441120717488129,centre.io,binance.com,USDC,40600000 +279441120717500417,centre.io,binance.com,USDC,40000000 +279441120717508609,centre.io,binance.com,USDC,41300000 +279441120717520897,centre.io,binance.com,USDC,40400000 +279441120717602817,centre.io,binance.com,USDC,66000000 +279441120717672449,centre.io,binance.com,USDC,56600000 +279441120717733889,centre.io,binance.com,USDC,55100000 +279441120717791233,centre.io,binance.com,USDC,43800000 +279441120717832193,centre.io,binance.com,USDC,96600000 +279441120717860865,centre.io,binance.com,USDC,100000000 +279441120717926401,centre.io,binance.com,USDC,41000000 +279441120717942785,centre.io,binance.com,USDC,43400000 +279441125012426753,centre.io,binance.com,USDC,92200000 +279441125012430849,centre.io,binance.com,USDC,47400000 +279441125012488193,centre.io,binance.com,USDC,42600000 +279441125012578305,centre.io,binance.com,USDC,91300000 +279441125012598785,centre.io,binance.com,USDC,74300000 +279441125012627457,centre.io,binance.com,USDC,40000000 +279441125012799489,centre.io,binance.com,USDC,40100000 +279441125012873217,centre.io,binance.com,USDC,44000000 +279441125012934657,centre.io,binance.com,USDC,53500000 +279441125013102593,centre.io,binance.com,USDC,40000000 +279441125013131265,centre.io,binance.com,USDC,40200000 +279441129307443201,centre.io,binance.com,USDC,40000000 +279441129307504641,centre.io,binance.com,USDC,80100000 +279441129307541505,centre.io,binance.com,USDC,81000000 +279441129307553793,centre.io,binance.com,USDC,49000000 +279441129307598849,centre.io,binance.com,USDC,51000000 +279441129307615233,centre.io,binance.com,USDC,40000000 +279441129307668481,centre.io,binance.com,USDC,50200000 +279441129307738113,centre.io,binance.com,USDC,80000000 +279441129307824129,centre.io,binance.com,USDC,56200000 +279441129307959297,centre.io,binance.com,USDC,60000000 +279441129307967489,centre.io,whitebit.com,USDC,40000000 +279441129307975681,centre.io,binance.com,USDC,41000000 +279441129308004353,centre.io,binance.com,USDC,40000000 +279441129308049409,centre.io,binance.com,USDC,40000000 +279441129308073985,centre.io,binance.com,USDC,41800000 +279441129308094465,centre.io,binance.com,USDC,40200000 +279441129308155905,centre.io,binance.com,USDC,83200000 +279441129308200961,centre.io,binance.com,USDC,51800000 +279441129308295169,centre.io,binance.com,USDC,100000000 +279441129308430337,centre.io,binance.com,USDC,80200000 +279441133602357249,centre.io,binance.com,USDC,47000000 +279441133602373633,centre.io,binance.com,USDC,41800000 +279441133602390017,centre.io,binance.com,USDC,49900000 +279441133602398209,centre.io,binance.com,USDC,41200000 +279441133602443265,centre.io,binance.com,USDC,40500000 +279441133602529281,centre.io,binance.com,USDC,50200000 +279441133602652161,centre.io,binance.com,USDC,97900000 +279441133602803713,centre.io,cex.io,USDC,122000000 +279441133602807809,centre.io,binance.com,USDC,40300000 +279441133602811905,centre.io,binance.com,USDC,40400000 +279441133602918401,centre.io,binance.com,USDC,42600000 +279441133602938881,centre.io,binance.com,USDC,40600000 +279441133603000321,centre.io,binance.com,USDC,41500000 +279441133603049473,centre.io,binance.com,USDC,188100000 +279441137897328641,centre.io,binance.com,USDC,83300000 +279441137897349121,centre.io,binance.com,USDC,72000000 +279441137897361409,centre.io,binance.com,USDC,42400000 +279441137897365505,centre.io,binance.com,USDC,66000000 +279441137897398273,centre.io,binance.com,USDC,41400000 +279441137897402369,centre.io,binance.com,USDC,56100000 +279441137897410561,centre.io,binance.com,USDC,85400000 +279441137897582593,centre.io,binance.com,USDC,50100000 +279441137897590785,centre.io,binance.com,USDC,50200000 +279441137897635841,centre.io,binance.com,USDC,40000000 +279441137897734145,centre.io,binance.com,USDC,40000000 +279441137897738241,centre.io,binance.com,USDC,48000000 +279441137897754625,centre.io,binance.com,USDC,48000000 +279441137897762817,centre.io,binance.com,USDC,40000000 +279441137897787393,centre.io,binance.com,USDC,51800000 +279441137897832449,centre.io,binance.com,USDC,40500000 +279441137897836545,centre.io,binance.com,USDC,40000000 +279441137897852929,centre.io,binance.com,USDC,48600000 +279441137897897985,centre.io,binance.com,USDC,41600000 +279441137897906177,centre.io,binance.com,USDC,53800000 +279441137897975809,centre.io,binance.com,USDC,56000000 +279441142192287745,centre.io,binance.com,USDC,80000000 +279441142192365569,centre.io,binance.com,USDC,40900000 +279441142192410625,centre.io,binance.com,USDC,88000000 +279441142192541697,centre.io,binance.com,USDC,49800000 +279441142192713729,centre.io,binance.com,USDC,40000000 +279441142192799745,centre.io,binance.com,USDC,40200000 +279441142192939009,centre.io,binance.com,USDC,101500000 +279441142192943105,centre.io,binance.com,USDC,40000000 +279441142192967681,centre.io,binance.com,USDC,40200000 +279441142192971777,centre.io,binance.com,USDC,46600000 +279441146487316481,centre.io,binance.com,USDC,74100000 +279441146487353345,centre.io,binance.com,USDC,76000000 +279441146487414785,centre.io,binance.com,USDC,48100000 +279441146487451649,centre.io,binance.com,USDC,40000000 +279441146487480321,centre.io,binance.com,USDC,43000000 +279441146487500801,centre.io,binance.com,USDC,40000000 +279441146487566337,centre.io,binance.com,USDC,40200000 +279441146487578625,centre.io,binance.com,USDC,40000000 +279441146487595009,centre.io,binance.com,USDC,41400000 +279441146487656449,centre.io,binance.com,USDC,68600000 +279441146487697409,centre.io,binance.com,USDC,42500000 +279441146487717889,centre.io,binance.com,USDC,141900000 +279441146487754753,centre.io,binance.com,USDC,44500000 +279441146487816193,centre.io,binance.com,USDC,53400000 +279441146487844865,centre.io,binance.com,USDC,48000000 +279441146487984129,centre.io,binance.com,USDC,91400000 +279441146487992321,centre.io,binance.com,USDC,171000000 +279441150782251009,centre.io,binance.com,USDC,48000000 +279441150782255105,centre.io,binance.com,USDC,50000000 +279441150782291969,centre.io,binance.com,USDC,40000000 +279441150782312449,centre.io,binance.com,USDC,41100000 +279441150782382081,centre.io,binance.com,USDC,46600000 +279441150782402561,centre.io,binance.com,USDC,42300000 +279441150782644225,centre.io,binance.com,USDC,40300000 +279441150782758913,centre.io,binance.com,USDC,40200000 +279441150782799873,centre.io,binance.com,USDC,41500000 +279441150782812161,centre.io,binance.com,USDC,64000000 +279441150782861313,centre.io,binance.com,USDC,111000000 +279441150782922753,centre.io,binance.com,USDC,97300000 +279441150782947329,centre.io,binance.com,USDC,66800000 +279441150782955521,centre.io,binance.com,USDC,40000000 +279441150782959617,centre.io,binance.com,USDC,60700000 +279441150782976001,centre.io,binance.com,USDC,80900000 +279441155077304321,centre.io,binance.com,USDC,96000000 +279441155077316609,centre.io,binance.com,USDC,45900000 +279441155077353473,centre.io,binance.com,USDC,40000000 +279441155077410817,centre.io,binance.com,USDC,40500000 +279441155077455873,centre.io,binance.com,USDC,51600000 +279441155077492737,centre.io,binance.com,USDC,57700000 +279441155077525505,centre.io,binance.com,USDC,87200000 +279441155077632001,centre.io,binance.com,USDC,50600000 +279441155077726209,centre.io,binance.com,USDC,50000000 +279441155077746689,centre.io,binance.com,USDC,40400000 +279441155077828609,centre.io,binance.com,USDC,46800000 +279441155077898241,centre.io,binance.com,USDC,50500000 +279441155077963777,centre.io,binance.com,USDC,80000000 +279441159372152833,centre.io,binance.com,USDC,194200000 +279441159372218369,centre.io,binance.com,USDC,54000000 +279441159372275713,centre.io,binance.com,USDC,54400000 +279441159372361729,centre.io,binance.com,USDC,49300000 +279441159372443649,centre.io,binance.com,USDC,49600000 +279441159372492801,centre.io,binance.com,USDC,40000000 +279441159372509185,centre.io,binance.com,USDC,56200000 +279441159372562433,centre.io,binance.com,USDC,48600000 +279441159372587009,centre.io,binance.com,USDC,62000000 +279441159372627969,centre.io,binance.com,USDC,53100000 +279441159372636161,centre.io,binance.com,USDC,58800000 +279441159372660737,centre.io,binance.com,USDC,56600000 +279441159372726273,centre.io,binance.com,USDC,40300000 +279441159372730369,centre.io,binance.com,USDC,54200000 +279441159372759041,centre.io,binance.com,USDC,43400000 +279441159372881921,centre.io,binance.com,USDC,52500000 +279441159372918785,centre.io,binance.com,USDC,67000000 +279441159372922881,centre.io,binance.com,USDC,50000000 +279441163667124225,centre.io,binance.com,USDC,59400000 +279441163667140609,centre.io,binance.com,USDC,48200000 +279441163667189761,centre.io,binance.com,USDC,43200000 +279441163667247105,centre.io,binance.com,USDC,40000000 +279441163667304449,centre.io,binance.com,USDC,49800000 +279441163667345409,centre.io,binance.com,USDC,46900000 +279441163667402753,centre.io,binance.com,USDC,56800000 +279441163667435521,centre.io,binance.com,USDC,68300000 +279441163667447809,centre.io,binance.com,USDC,66100000 +279441163667542017,centre.io,binance.com,USDC,52500000 +279441163667599361,centre.io,binance.com,USDC,81700000 +279441163667640321,centre.io,binance.com,USDC,49100000 +279441163667673089,centre.io,coinbase.com,USDC,40000000 +279441163667738625,centre.io,binance.com,USDC,48600000 +279441163667841025,centre.io,binance.com,USDC,41500000 +279441167962083329,centre.io,binance.com,USDC,50500000 +279441167962116097,centre.io,binance.com,USDC,82600000 +279441167962161153,centre.io,binance.com,USDC,62600000 +279441167962185729,centre.io,binance.com,USDC,40000000 +279441167962202113,centre.io,binance.com,USDC,40500000 +279441167962226689,centre.io,binance.com,USDC,50000000 +279441167962230785,centre.io,binance.com,USDC,76400000 +279441167962312705,centre.io,binance.com,USDC,46400000 +279441167962611713,centre.io,binance.com,USDC,46500000 +279441167962624001,centre.io,binance.com,USDC,52300000 +279441167962628097,centre.io,binance.com,USDC,58000000 +279441167962640385,centre.io,binance.com,USDC,50000000 +279441167962664961,centre.io,binance.com,USDC,80600000 +279441167962669057,centre.io,binance.com,USDC,76000000 +279441167962681345,centre.io,binance.com,USDC,40000000 +279441167962697729,centre.io,binance.com,USDC,41800000 +279441167962722305,centre.io,binance.com,USDC,40000000 +279441167962750977,centre.io,binance.com,USDC,79400000 +279441167962906625,centre.io,binance.com,USDC,48000000 +279441167962939393,centre.io,binance.com,USDC,56800000 +279441172257054721,centre.io,binance.com,USDC,403800000 +279441172257062913,centre.io,binance.com,USDC,50000000 +279441172257091585,centre.io,binance.com,USDC,52000000 +279441172257124353,centre.io,binance.com,USDC,72700000 +279441172257296385,centre.io,binance.com,USDC,40200000 +279441172257316865,centre.io,binance.com,USDC,90900000 +279441172257345537,centre.io,binance.com,USDC,48400000 +279441172257398785,centre.io,binance.com,USDC,120200000 +279441172257427457,centre.io,binance.com,USDC,40700000 +279441172257431553,centre.io,binance.com,USDC,58700000 +279441172257439745,centre.io,binance.com,USDC,67900000 +279441172257447937,centre.io,binance.com,USDC,46900000 +279441172257775617,centre.io,binance.com,USDC,70400000 +279441172257849345,centre.io,binance.com,USDC,46600000 +279441172258021377,centre.io,binance.com,USDC,101800000 +279441172258074625,centre.io,binance.com,USDC,66600000 +279441176552050689,centre.io,binance.com,USDC,100100000 +279441176552071169,centre.io,binance.com,USDC,45100000 +279441176552095745,centre.io,binance.com,USDC,107800000 +279441176552136705,centre.io,binance.com,USDC,61700000 +279441176552144897,centre.io,binance.com,USDC,47400000 +279441176552153089,centre.io,binance.com,USDC,60600000 +279441176552230913,centre.io,binance.com,USDC,70300000 +279441176552329217,centre.io,binance.com,USDC,57800000 +279441176552394753,centre.io,bitrue.com,USDC,95100000 +279441176552464385,centre.io,binance.com,USDC,51500000 +279441176552476673,centre.io,binance.com,USDC,79400000 +279441176552517633,centre.io,binance.com,USDC,99400000 +279441176552521729,centre.io,binance.com,USDC,100000000 +279441176552542209,centre.io,binance.com,USDC,42500000 +279441176552611841,centre.io,binance.com,USDC,41600000 +279441180846989313,centre.io,binance.com,USDC,44400000 +279441180847038465,centre.io,binance.com,USDC,40000000 +279441180847095809,centre.io,binance.com,USDC,65400000 +279441180847108097,centre.io,binance.com,USDC,44200000 +279441180847120385,centre.io,binance.com,USDC,43000000 +279441180847161345,centre.io,binance.com,USDC,65800000 +279441180847169537,centre.io,binance.com,USDC,48000000 +279441180847185921,centre.io,binance.com,USDC,49300000 +279441180847230977,centre.io,binance.com,USDC,43800000 +279441180847243265,centre.io,binance.com,USDC,44500000 +279441180847296513,centre.io,binance.com,USDC,56600000 +279441180847362049,centre.io,binance.com,USDC,61000000 +279441180847370241,centre.io,binance.com,USDC,48000000 +279441180847394817,centre.io,binance.com,USDC,48800000 +279441180847415297,centre.io,binance.com,USDC,56100000 +279441180847427585,centre.io,binance.com,USDC,45600000 +279441180847575041,centre.io,binance.com,USDC,120200000 +279441180847583233,centre.io,binance.com,USDC,121900000 +279441180847599617,centre.io,binance.com,USDC,53000000 +279441185141981185,centre.io,binance.com,USDC,48200000 +279441185142034433,centre.io,binance.com,USDC,85000000 +279441185142038529,centre.io,binance.com,USDC,40800000 +279441185142194177,centre.io,binance.com,USDC,41400000 +279441185142218753,centre.io,binance.com,USDC,70600000 +279441185142386689,centre.io,binance.com,USDC,48300000 +279441185142480897,centre.io,binance.com,USDC,48500000 +279441185142521857,centre.io,binance.com,USDC,127200000 +279441185142607873,centre.io,binance.com,USDC,60000000 +279441185142775809,centre.io,binance.com,USDC,66800000 +279441185142824961,centre.io,binance.com,USDC,83700000 +279441185142853633,centre.io,binance.com,USDC,51500000 +279441189436928001,centre.io,binance.com,USDC,45000000 +279441189436956673,centre.io,binance.com,USDC,44200000 +279441189436973057,centre.io,binance.com,USDC,64000000 +279441189437030401,centre.io,binance.com,USDC,63100000 +279441189437259777,centre.io,binance.com,USDC,46200000 +279441189437370369,centre.io,binance.com,USDC,40000000 +279441189437390849,centre.io,binance.com,USDC,55000000 +279441189437415425,centre.io,binance.com,USDC,56200000 +279441189437448193,centre.io,binance.com,USDC,55900000 +279441189437468673,centre.io,binance.com,USDC,60600000 +279441189437530113,centre.io,binance.com,USDC,89600000 +279441189437657089,centre.io,binance.com,USDC,40300000 +279441189437677569,centre.io,binance.com,USDC,90000000 +279441189437739009,centre.io,binance.com,USDC,49000000 +279441189437784065,centre.io,binance.com,USDC,40100000 +279441189437800449,centre.io,binance.com,USDC,45900000 +279441193731969025,centre.io,binance.com,USDC,97000000 +279441193732075521,centre.io,binance.com,USDC,53300000 +279441193732169729,centre.io,binance.com,USDC,58000000 +279441193732177921,centre.io,binance.com,USDC,90000000 +279441193732239361,centre.io,binance.com,USDC,43600000 +279441193732272129,centre.io,binance.com,USDC,40200000 +279441193732345857,centre.io,binance.com,USDC,45100000 +279441193732378625,centre.io,binance.com,USDC,120000000 +279441193732444161,centre.io,binance.com,USDC,50000000 +279441193732460545,centre.io,binance.com,USDC,71200000 +279441193732489217,centre.io,binance.com,USDC,46200000 +279441193732526081,centre.io,binance.com,USDC,47800000 +279441193732530177,centre.io,binance.com,USDC,71000000 +279441193732546561,centre.io,binance.com,USDC,56500000 +279441198026895361,centre.io,binance.com,USDC,56400000 +279441198026899457,centre.io,binance.com,USDC,40100000 +279441198026907649,centre.io,binance.com,USDC,56000000 +279441198026915841,centre.io,binance.com,USDC,65700000 +279441198026924033,centre.io,binance.com,USDC,56500000 +279441198026964993,centre.io,binance.com,USDC,40300000 +279441198027005953,centre.io,binance.com,USDC,74800000 +279441198027083777,centre.io,binance.com,USDC,65600000 +279441198027223041,centre.io,binance.com,USDC,40500000 +279441198027268097,centre.io,binance.com,USDC,52200000 +279441198027292673,centre.io,binance.com,USDC,100900000 +279441198027452417,centre.io,binance.com,USDC,62000000 +279441198027460609,centre.io,binance.com,USDC,60000000 +279441198027534337,centre.io,binance.com,USDC,40000000 +279441202321825793,centre.io,binance.com,USDC,113800000 +279441202321854465,centre.io,binance.com,USDC,77800000 +279441202322317313,centre.io,binance.com,USDC,40800000 +279441202322522113,centre.io,binance.com,USDC,63700000 +279441356940967937,coinbase.com,binance.com,USDC,120000000000 +279444681245687809,coinbase.com,bitrue.com,USDC,10150000000 +279445918196559873,binance.com,centre.io,USDC,100000000 +279448641205579777,centre.io,binance.com,USDC,10000000000000 +279448855954157569,kraken.com,centre.io,USDC,1669550000000 +279453924015099905,binance.com,centre.io,USDC,23000000 +279455156671574017,kraken.com,centre.io,USDC,20968260000000 +279461070841352193,kraken.com,centre.io,USDC,16455400000000 +279462960626393089,bitstamp.net,coinbase.com,USDC,194100000 +279463926994231297,bitstamp.net,coinbase.com,USDC,50492300000 +279464042957926401,bitstamp.net,coinbase.com,USDC,50430800000 +279464970671337473,bitstamp.net,coinbase.com,USDC,51316300000 +279464974966583297,bitstamp.net,coinbase.com,USDC,51078300000 +279465279909228545,bitstamp.net,coinbase.com,USDC,49229700000 diff --git a/docs/mainnet-day.md b/docs/mainnet-day.md new file mode 100644 index 0000000..0bc8548 --- /dev/null +++ b/docs/mainnet-day.md @@ -0,0 +1,82 @@ +# One real mainnet day, netted + +What would SetOff have changed for institutions settling USDC with each other +on Stellar? This runs the engine on every USDC payment made between known +institutions on one real day, 2026-10-21 (UTC). + +## Result + +336 payments between 10 institutions, 55,532,924.24 USDC in total. + +| Settlement | USDC moved | Saved | Transfers | +|---|---:|---:|---:| +| Gross (every payment on its own, as happened) | 55,532,924.24 | | 336 | +| Bilateral (each pair settles its difference) | 53,505,367.08 | 3.7% | 13 | +| Multilateral (SetOff: each settles its net position) | 53,355,736.73 | 3.9% | **9** | + +**336 payments become 9 transfers, 37 times fewer.** The value saved is +small on this day because the flows were largely one-way: most of them +were exchanges and Circle (`centre.io`) moving USDC in a single direction, +which netting cannot offset. The saving is in operations: nine transactions +to sign, fee, submit and reconcile instead of 336, and nine places a +settlement can fail instead of 336. + +The largest net debtors, which is what each would actually have paid: + +| Institution | Net USDC owed | +|---|---:| +| binance.com | 48,919,022.28 | +| kraken.com | 4,340,681.00 | +| changelly.com | 60,000.00 | +| bitstamp.net | 26,028.45 | +| hitbtc.com | 10,005.00 | + +The more two-way a corridor is, the more value netting saves too. The +[simulator](https://setoff-org.github.io/setoff-engine/) loads this day with +**A real mainnet day**, and takes your own CSV with **Import CSV**. + +## How it was built + +[`tools/mainnet_day.py`](../tools/mainnet_day.py) (Python standard library only): + +1. **Institutions.** Every classic account that stellar.expert's public + [directory](https://stellar.expert/directory) tags as an exchange, anchor, + custodian or wallet: 480 accounts, grouped by organization (home domain, + else name) into 229 institutions. An institution's accounts are one + participant, so moving funds between its own accounts is not an + obligation. +2. **Payments.** For each account, Horizon's payments for that UTC day. + Kept: successful `payment` operations in USDC issued by Circle + (`GA5Z…KZVN`), and path payments with USDC on both legs, whose two ends + belong to *different* institutions. Each operation is counted once, + though it is seen from both ends. +3. **Obligations.** One row per payment in the engine's CSV format, with the + operation id as the row id, so every row can be looked up on a block + explorer (`https://stellar.expert/explorer/public/op/`). Amounts are + stroops (10⁻⁷ USDC), exact. + +The file is [`data/mainnet-2026-10-21.csv`](../data/mainnet-2026-10-21.csv). + +## Limits + +- **A lower bound.** Three very busy accounts (binance.com, ultracapital.xyz + and aps.money) have more payments that day than the script reads per + account (12,000 operations). Their payments with the other institutions are + still captured from the other end, but payments between two of those three + can be missing. +- **The directory is not complete.** Institutions it does not list, and + their payments, are not included. +- **Observed payments, not obligations.** On chain, these payments settled + one by one. Treating a day's payments as one netting window shows what + settling the net would have changed, not what the institutions owed each + other at any moment. + +## Reproduce + +```sh +python3 tools/mainnet_day.py 2026-10-21 > data/mainnet-2026-10-21.csv +cargo run --example mainnet_day --features csv -- data/mainnet-2026-10-21.csv +``` + +Any other day works the same way. Fetching takes a while within Horizon's +rate limits. diff --git a/examples/mainnet_day.rs b/examples/mainnet_day.rs new file mode 100644 index 0000000..b0cbe63 --- /dev/null +++ b/examples/mainnet_day.rs @@ -0,0 +1,59 @@ +//! Nets one real day of mainnet payments between institutions and reports +//! what settling the net instead of every payment would have saved. +//! +//! python3 tools/mainnet_day.py 2026-10-21 > data/mainnet-2026-10-21.csv +//! cargo run --example mainnet_day --features csv -- data/mainnet-2026-10-21.csv + +#![allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::arithmetic_side_effects, + clippy::cast_precision_loss, + missing_docs +)] + +use std::collections::BTreeSet; + +use setoff_engine::{compare, net, participants, read_csv}; + +fn main() { + let path = std::env::args().nth(1).expect("usage: mainnet_day "); + let obligations = read_csv(std::fs::File::open(&path).expect("open the CSV")).expect("read the CSV"); + let netting = net(&obligations).expect("net the obligations"); + let units = |v: i128| v as f64 / 1e7; + + let institutions: BTreeSet<&str> = + obligations.iter().flat_map(|o| [o.debtor.as_str(), o.creditor.as_str()]).collect(); + println!("{path}"); + println!("{} payments between {} institutions\n", obligations.len(), institutions.len()); + + for c in compare(&obligations).unwrap() { + let payments = obligations.iter().filter(|o| o.asset == c.asset).count(); + let saved = |v: i128| 100.0 * (1.0 - v as f64 / c.gross as f64); + println!("{} moved {:>18} transfers", c.asset, "saved"); + println!(" gross {:>16.2} {:>8} {payments:>9}", units(c.gross), ""); + println!( + " bilateral {:>16.2} {:>7.1}% {:>9}", + units(c.bilateral), + saved(c.bilateral), + c.bilateral_transfers + ); + println!( + " multilateral {:>16.2} {:>7.1}% {:>9}", + units(c.multilateral), + saved(c.multilateral), + c.multilateral_transfers + ); + } + debug_assert_eq!( + netting.transfers.len(), + compare(&obligations).unwrap().iter().map(|c| c.multilateral_transfers).sum::() + ); + + let mut debtors: Vec<_> = participants(&obligations).unwrap().into_iter().filter(|p| p.collateral > 0).collect(); + debtors.sort_by_key(|p| std::cmp::Reverse(p.collateral)); + println!("\nlargest net debtors (what each would actually have to pay)"); + for p in debtors.iter().take(10) { + println!(" {:<28} {:>16.2} {}", p.participant, units(p.collateral), p.asset); + } +} diff --git a/site/app.js b/site/app.js index 3a94bfd..6bf3ec4 100644 --- a/site/app.js +++ b/site/app.js @@ -19,7 +19,17 @@ const strategyEl = document.getElementById("strategy"); let rows = []; let ready = false; -const fmt = (s) => BigInt(s).toLocaleString("en-US"); +// Amounts are integers in each asset's smallest unit. By default they are +// shown as typed; a preset that knows an asset's decimals (the mainnet day: +// USDC in stroops, 7) sets them here so the page shows real USDC. +let decimals = {}; +const fmt = (s, asset) => { + const v = BigInt(s), d = decimals[asset] || 0; + if (!d) return v.toLocaleString("en-US"); + const neg = v < 0n, a = neg ? -v : v, base = 10n ** BigInt(d); + const frac = (a % base).toString().padStart(d, "0").slice(0, 2); + return `${neg ? "-" : ""}${(a / base).toLocaleString("en-US")}.${frac}`; +}; const clean = (s) => String(s).trim().replace(/[,\s_]/g, ""); function randomRows(n) { @@ -34,7 +44,8 @@ function randomRows(n) { return out; } -function load(list) { +function load(list, units = {}) { + decimals = units; rows = list.map(([debtor, creditor, asset, amount]) => ({ debtor, creditor, asset, amount })); render(); } @@ -142,20 +153,20 @@ function renderResult(n, count) { const w = g === 0n ? 0 : Number((s * 1000n) / g) / 10; html += `

${esc(a.asset)}

-
Owed ${fmt(g)}
-
Moves ${fmt(s)}
+
Owed ${fmt(g, a.asset)}
+
Moves ${fmt(s, a.asset)}
`; } html += `

Settlement plan

    `; html += transfers === 0 ? `
  1. Nothing moves: every position nets to zero.
  2. ` - : n.transfers.map((t) => `
  3. ${esc(t.from)} pays ${esc(t.to)} ${fmt(t.amount)} ${esc(t.asset)}
  4. `).join(""); + : n.transfers.map((t) => `
  5. ${esc(t.from)} pays ${esc(t.to)} ${fmt(t.amount, t.asset)} ${esc(t.asset)}
  6. `).join(""); html += `
`; if (n.positions.length) { html += `

Net positions

`; html += n.positions.map((p) => { const v = BigInt(p.net); - return ``; + return ``; }).join(""); html += `
ParticipantAssetNet
${esc(p.participant)}${esc(p.asset)}${v > 0n ? "+" : ""}${fmt(p.net)}
${esc(p.participant)}${esc(p.asset)}${v > 0n ? "+" : ""}${fmt(p.net, p.asset)}
`; } @@ -170,9 +181,103 @@ function esc(s) { function encode() { return btoa(unescape(encodeURIComponent(JSON.stringify(rows.map((r) => [r.debtor, r.creditor, r.asset, r.amount]))))); } function decode(h) { try { return JSON.parse(decodeURIComponent(escape(atob(h)))); } catch { return null; } } +const mainnetNote = document.getElementById("mainnet-note"); document.querySelectorAll("[data-example]").forEach((b) => b.addEventListener("click", () => { + mainnetNote.hidden = true; + csvErrors.hidden = true; load(b.dataset.example === "random" ? randomRows(40) : EXAMPLES[b.dataset.example]); })); + +// CSV in the format `setoff net` and the engine's read_csv take: +// id,debtor,creditor,asset,amount. Quoted fields are allowed. Returns the +// rows and one message per line that could not be read; good lines load. +const CSV_HEADER = ["id", "debtor", "creditor", "asset", "amount"]; +function parseCSV(text) { + const records = []; + let field = "", record = [], quoted = false, line = 1, start = 1; + const endField = () => { record.push(field); field = ""; }; + const endRecord = () => { endField(); records.push({ line: start, fields: record }); record = []; start = line; }; + for (let i = 0; i < text.length; i++) { + const ch = text[i]; + if (quoted) { + if (ch === '"' && text[i + 1] === '"') { field += '"'; i++; } + else if (ch === '"') quoted = false; + else { if (ch === "\n") line++; field += ch; } + } else if (ch === '"') quoted = true; + else if (ch === ",") endField(); + else if (ch === "\n") { line++; endRecord(); } + else if (ch !== "\r") field += ch; + } + if (field !== "" || record.length) endRecord(); + const rows = [], errors = []; + const [head, ...body] = records.filter((r) => r.fields.some((f) => f.trim() !== "")); + if (!head || head.fields.map((f) => f.trim()).join(",") !== CSV_HEADER.join(",")) { + errors.push(`line 1: the header must be ${CSV_HEADER.join(",")}`); + return { rows, errors }; + } + for (const { line: n, fields } of body) { + if (fields.length !== CSV_HEADER.length) { errors.push(`line ${n}: expected ${CSV_HEADER.length} fields, found ${fields.length}`); continue; } + const [, debtor, creditor, asset, amount] = fields.map((f) => f.trim()); + if (!/^(0|[1-9][0-9]*)$/.test(amount)) { errors.push(`line ${n}: amount ${JSON.stringify(amount)} is not a whole number`); continue; } + rows.push([debtor, creditor, asset, amount]); + } + return { rows, errors }; +} + +const csvErrors = document.getElementById("csv-errors"); +function showCSVErrors(errors, name) { + csvErrors.hidden = errors.length === 0; + if (!errors.length) return; + const shown = errors.slice(0, 20).map((e) => `
  • ${esc(e)}
  • `).join(""); + const more = errors.length > 20 ? `
  • …and ${errors.length - 20} more
  • ` : ""; + csvErrors.innerHTML = `${errors.length} line${errors.length === 1 ? "" : "s"} of ${esc(name)} could not be read and ${errors.length === 1 ? "was" : "were"} skipped:
      ${shown}${more}
    `; +} + +document.getElementById("csv-file").addEventListener("change", async (e) => { + const file = e.target.files[0]; + e.target.value = ""; // the same file can be chosen again after a fix + if (!file) return; + mainnetNote.hidden = true; + try { + const { rows: list, errors } = parseCSV(await file.text()); + showCSVErrors(errors, file.name); + if (list.length) { + load(list); + history.replaceState(null, "", location.pathname); + } else if (!errors.length) { + showCSVErrors(["the file has a header but no obligations"], file.name); + } + } catch (err) { + showCSVErrors([`could not read the file: ${err.message}`], file.name); + } +}); + +// One UTC day of real USDC payments between institutions on Stellar mainnet, +// built by tools/mainnet_day.py. Amounts stay exact, in stroops (10^-7 USDC). +const MAINNET = { file: "samples/mainnet-day.csv", day: "2026-10-21" }; +const mainnetBtn = document.getElementById("mainnet"); +mainnetBtn.addEventListener("click", async () => { + mainnetBtn.disabled = true; + mainnetNote.hidden = false; + csvErrors.hidden = true; + mainnetNote.textContent = "Loading the day's payments…"; + try { + const res = await fetch(MAINNET.file); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + const { rows: list, errors } = parseCSV(await res.text()); + if (errors.length) throw new Error(errors[0]); + load(list, { USDC: 7 }); + mainnetNote.innerHTML = `${list.length.toLocaleString("en-US")} real USDC payments made on ${MAINNET.day} (UTC) between institutions + that stellar.expert's directory lists as exchanges, anchors, custodians or wallets, + grouped by institution. The table holds amounts in stroops (10−7 USDC), exactly as recorded on chain; results are shown in USDC. + How this was built.`; + history.replaceState(null, "", location.pathname); + } catch (e) { + mainnetNote.textContent = `The mainnet sample couldn't be loaded (${e.message}). Check your connection and try again.`; + } finally { + mainnetBtn.disabled = false; + } +}); document.getElementById("add").addEventListener("click", () => { rows.push({ debtor: "", creditor: "", asset: rows.at(-1)?.asset || "USDC", amount: "" }); render(); diff --git a/site/index.html b/site/index.html index c69f141..0e19ed8 100644 --- a/site/index.html +++ b/site/index.html @@ -42,8 +42,12 @@

    Obligations

    + + + +

    Amounts are whole units of the asset. Each row is one obligation: the debtor owes the creditor.

    diff --git a/site/style.css b/site/style.css index af64b8d..616269b 100644 --- a/site/style.css +++ b/site/style.css @@ -31,6 +31,12 @@ button, select, input { font: inherit; color: inherit; } button { border: 1px solid var(--line); background: var(--panel); border-radius: 8px; padding: 6px 12px; cursor: pointer; } button:hover { border-color: var(--accent); } button.primary { background: var(--accent); color: var(--accent-ink); border-color: var(--accent); font-weight: 600; } +.file-btn { border: 1px solid var(--line); background: var(--panel); border-radius: 8px; padding: 6px 12px; cursor: pointer; color: var(--ink); } +.file-btn:hover { border-color: var(--accent); } +.file-btn:focus-within { outline: 3px solid var(--accent); outline-offset: 2px; } +.file-btn input { position: absolute; width: 1px; height: 1px; opacity: 0; } +#csv-errors { margin: 8px 0; } +#csv-errors ul { margin: 6px 0 0; padding-left: 20px; } :focus-visible { outline: 3px solid var(--accent); outline-offset: 2px; } table { width: 100%; border-collapse: collapse; } .obligations th { text-align: left; font-size: 13px; color: var(--muted); font-weight: 600; padding: 6px 4px; } diff --git a/tests/csv.rs b/tests/csv.rs index 411e91b..8328c9e 100644 --- a/tests/csv.rs +++ b/tests/csv.rs @@ -43,6 +43,19 @@ fn writes_the_plan_for_payments_systems() { } } +/// The simulator's CSV import is checked against this file, so it must stay +/// the remittance-corridor vector exactly. +#[test] +fn the_corridor_csv_is_the_reference_vector() { + let dir = concat!(env!("CARGO_MANIFEST_DIR"), "/tests"); + let from_csv = read_csv(std::fs::File::open(format!("{dir}/fixtures/remittance-corridor.csv")).unwrap()).unwrap(); + let vector: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(format!("{dir}/vectors/remittance-corridor.json")).unwrap()) + .unwrap(); + let from_json: Vec = serde_json::from_value(vector["obligations"].clone()).unwrap(); + assert_eq!(from_csv, from_json); +} + #[test] fn awkward_names_are_quoted() { let n = net(&[setoff_engine::Obligation::new("1", "Acme, Inc.", "B \"Co\"", "USDC", 9)]).unwrap(); diff --git a/tests/fixtures/remittance-corridor.csv b/tests/fixtures/remittance-corridor.csv new file mode 100644 index 0000000..8c9f8c5 --- /dev/null +++ b/tests/fixtures/remittance-corridor.csv @@ -0,0 +1,8 @@ +id,debtor,creditor,asset,amount +1,anchor-ng,anchor-us,USDC,5000000000 +2,anchor-us,anchor-ng,USDC,3200000000 +3,anchor-ng,anchor-ke,USDC,1100000000 +4,anchor-ke,anchor-us,USDC,900000000 +5,anchor-us,anchor-ke,USDC,1250000000 +6,anchor-eu,anchor-ng,EURC,700000000 +7,anchor-ng,anchor-eu,EURC,650000000 diff --git a/tools/mainnet_day.py b/tools/mainnet_day.py new file mode 100644 index 0000000..c91bd8c --- /dev/null +++ b/tools/mainnet_day.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +"""Build a netting input from one UTC day of real Stellar mainnet payments. + +Participants are institutions, not accounts: every account stellar.expert's +public directory tags as an exchange, anchor, custodian or wallet is grouped +under its organization (its home domain, else its name). The day's successful +USDC payments (Circle's issuer only) between accounts of two *different* +institutions become obligations, one per payment operation, read from +Horizon. Each obligation keeps its operation id, so every row can be checked +on a block explorer. + + python3 tools/mainnet_day.py 2026-10-21 > data/mainnet-2026-10-21.csv + +Output is the engine's CSV (`id,debtor,creditor,asset,amount`; amounts in +stroops, USDC's 7-decimal unit). A summary goes to stderr. Stdlib only. +""" + +import csv +import json +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +from datetime import datetime, timedelta, timezone + +HORIZON = "https://horizon.stellar.org" +DIRECTORY = "https://api.stellar.expert/explorer/directory" +USDC_ISSUER = "GA5ZSEJYB37JRC5AVCIA5MOP4RHTM335X2KGX3IHOJAPP5RE34K4KZVN" +TAGS = ["exchange", "anchor", "custodian", "wallet"] +MAX_PAGES = 60 # per account, 200 operations each; busy accounts are capped and reported +UA = {"User-Agent": "setoff-engine mainnet_day (github.com/SetOff-Org/setoff-engine)"} + + +def get(url, tries=6): + for attempt in range(tries): + try: + with urllib.request.urlopen(urllib.request.Request(url, headers=UA), timeout=60) as r: + return json.load(r) + except urllib.error.HTTPError as e: + if e.code == 404: + return None + if e.code == 429 or e.code >= 500: + time.sleep(min(60, 2 ** attempt * 3)) + continue + raise + except (urllib.error.URLError, TimeoutError): + time.sleep(2 ** attempt) + raise RuntimeError(f"giving up on {url}") + + +def directory(): + """address -> institution, for classic (G…) accounts in the tagged directory.""" + out = {} + for tag in TAGS: + url = f"{DIRECTORY}?tag[]={tag}&limit=200" + while url: + page = get(url) + records = page["_embedded"]["records"] + for r in records: + if r["address"].startswith("G"): + out[r["address"]] = (r.get("domain") or r.get("name") or r["address"]).lower() + nxt = page["_links"]["next"]["href"] + url = f"https://api.stellar.expert{nxt}" if records else None + return out + + +def is_usdc_payment(op): + if not op.get("transaction_successful"): + return False + if op["type"] == "payment": + return op.get("asset_code") == "USDC" and op.get("asset_issuer") == USDC_ISSUER + if op["type"] in ("path_payment_strict_send", "path_payment_strict_receive"): + # Both legs USDC: a plain transfer routed as a path payment. + return (op.get("asset_code") == "USDC" and op.get("asset_issuer") == USDC_ISSUER + and op.get("source_asset_code") == "USDC" and op.get("source_asset_issuer") == USDC_ISSUER) + return False + + +def stroops(amount): + whole, _, frac = amount.partition(".") + return int(whole) * 10_000_000 + int((frac + "0000000")[:7]) + + +def day_payments(account, start, end): + """The account's operations in [start, end), newest first, and whether the page cap was hit.""" + url = f"{HORIZON}/accounts/{account}/payments?order=desc&limit=200" + ops, pages = [], 0 + while url and pages < MAX_PAGES: + page = get(url) + if page is None: + return ops, False + records = page["_embedded"]["records"] + pages += 1 + for op in records: + t = datetime.fromisoformat(op["created_at"].replace("Z", "+00:00")) + if t < start: + return ops, False + if t < end: + ops.append(op) + if len(records) < 200: + return ops, False + url = page["_links"]["next"]["href"] + return ops, url is not None and pages >= MAX_PAGES + + +def main(): + day = datetime.strptime(sys.argv[1], "%Y-%m-%d").replace(tzinfo=timezone.utc) + start, end = day, day + timedelta(days=1) + who = directory() + print(f"directory: {len(who)} accounts, {len(set(who.values()))} institutions", file=sys.stderr) + + seen, rows, capped = set(), [], [] + for i, account in enumerate(sorted(who)): + ops, hit_cap = day_payments(account, start, end) + if hit_cap: + capped.append(f"{who[account]} ({account[:6]}…)") + for op in ops: + frm, to = op.get("from"), op.get("to") + if op["id"] in seen or not is_usdc_payment(op) or frm not in who or to not in who: + continue + if who[frm] == who[to]: + continue # an institution moving funds between its own accounts + seen.add(op["id"]) + rows.append((op["id"], who[frm], who[to], "USDC", stroops(op["amount"]), op["transaction_hash"])) + if (i + 1) % 25 == 0: + print(f" {i + 1}/{len(who)} accounts, {len(rows)} inter-institution payments", file=sys.stderr) + + rows.sort(key=lambda r: int(r[0])) + w = csv.writer(sys.stdout, lineterminator="\n") + w.writerow(["id", "debtor", "creditor", "asset", "amount"]) + for r in rows: + w.writerow(r[:5]) + print(f"{sys.argv[1]}: {len(rows)} USDC payments between {len({r[1] for r in rows} | {r[2] for r in rows})} institutions", + file=sys.stderr) + if capped: + print(f"capped at {MAX_PAGES * 200} operations (more payments exist): {', '.join(capped)}", file=sys.stderr) + + +if __name__ == "__main__": + main()
    DebtorCreditorAssetAmountRemove