diff --git a/benchmarks/README.md b/benchmarks/README.md index 0444d88..0a2a901 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -108,3 +108,13 @@ This is a correctness diagnostic, not a routing latency benchmark. byte-identical replay; it does not compare dispatch strategies. See [simulation documentation](../docs/simulation.md) and the [measured artifact](results/2026-10-04-phase-15-simulation.json). + +## Phase 16 dispatch strategy benchmarking + +`roadrunner benchmark dispatch [--idle-penalty-weight ] [--json]` +replays identical inputs through four assignment policies and computes delivery durations, +waiting/idle time, distance, lateness, utilization, and explicit unfinished populations. +`scripts/collect_dispatch_strategy_benchmark.py` verifies paired/standalone replay and +records per-policy process timings with complete reproducibility metadata. See +[methodology](../docs/dispatch-strategy-benchmarks.md) and +[completion evidence](../docs/phase-16-completion.md). diff --git a/benchmarks/results/2026-10-06-phase-16-dispatch-strategies.json b/benchmarks/results/2026-10-06-phase-16-dispatch-strategies.json new file mode 100644 index 0000000..2468b3c --- /dev/null +++ b/benchmarks/results/2026-10-06-phase-16-dispatch-strategies.json @@ -0,0 +1,1163 @@ +{ + "schema_version": 1, + "date": "2026-10-06", + "purpose": "Paired synthetic dispatch strategy comparison; no production scalability claim", + "hardware": { + "operating_system": "macOS-27.0-arm64-arm-64bit-Mach-O", + "architecture": "arm64", + "cpu": "Apple M3", + "memory_bytes": 17179869184 + }, + "dataset": { + "path": "data/fixtures/phase-16/paired-strategies.json", + "sha256": "6dabddeb01290fe8aeb5dfc2da121cc0b861602bad42bc72c1e093b706f007f8", + "kind": "synthetic", + "graph_artifact": null, + "seed": 16, + "randomness": "splitmix64-upper53/v1", + "graph_digest": "5017e4b91bc6031d70113ac851341e77a954a628ba2234cbf0a94e946d56e24b", + "graph_nodes": 5, + "graph_directed_edges": 6, + "riders": 2, + "scheduled_orders": 14 + }, + "algorithm": { + "routing": "Roadrunner Dijkstra with static traffic overlays", + "events": "BinaryHeap ordered by logical time and insertion sequence", + "coverage": "exhaustive eligible road-feasible riders", + "tie_break": "lowest RiderId" + }, + "configuration": { + "build": "cargo +1.99.0 build -p roadrunner-cli --release --locked", + "rustc": "rustc 1.99.0 (b940084d7 2026-09-28)", + "binary_sha256": "b02585359b10089e53dbc5481e42c93efc1f2ba9f90b6304c665a571ff248f74", + "profile": "release", + "comparison_command": [ + "/Users/admin/Developer/roadrunner/target/release/roadrunner", + "benchmark", + "dispatch", + "/Users/admin/Developer/roadrunner/data/fixtures/phase-16/paired-strategies.json", + "--idle-penalty-weight", + "1.0", + "--json" + ], + "idle_penalty_weight": 1.0, + "warmup_runs_per_strategy": 1, + "measured_runs_per_strategy": 11, + "start_seconds": 0.0, + "end_seconds": 900.0, + "includes": "process startup, graph/scenario loading and validation, simulation, JSON serialization and capture", + "timing_percentiles": "linear interpolation of ordered elapsed process samples", + "outcome_percentiles": "nearest rank; even median midpoint", + "memory": "not measured", + "concurrent_workspace_validation": false + }, + "comparison_replay_sha256": "7e3c80564dc2437987da59e9f9ee4d7fcabf47b6016ab8af36418e8cf497d5ba", + "strategies": [ + { + "dispatch": { + "kind": "nearest_rider" + }, + "summary": { + "scheduled_orders": 14, + "created_orders": 13, + "uncreated_orders": 1, + "assigned_orders": 12, + "delivered_orders": 12, + "unassigned_orders": 1, + "assigned_unfinished_orders": 0, + "outstanding_orders": 1, + "late_deliveries": 12, + "outstanding_past_deadline": 1, + "predicted_eta": { + "samples": 12, + "mean_seconds": 113.04833157075844, + "median_seconds": 66.71704814011974, + "p95_seconds": 200.1511444203593, + "p99_seconds": 200.1511444203593 + }, + "delivered_duration": { + "samples": 12, + "mean_seconds": 242.48163620285425, + "median_seconds": 223.18912007187333, + "p95_seconds": 452.7680215179639, + "p99_seconds": 452.7680215179639 + }, + "pickup_waiting": { + "samples": 12, + "mean_seconds": 1.1278872013002659, + "median_seconds": 0.0, + "p95_seconds": 10.617402178956127, + "p99_seconds": 10.617402178956127 + }, + "completed_distance_meters": 8562.021177982035, + "rider_utilization": 0.7611747918137247, + "mean_rider_idle_seconds": 214.9426873676477 + }, + "orders": [ + { + "order": 1, + "realized_ready_at": 77.3344503190759, + "created_at": 0.0, + "observed_ready_at": 77.3344503190759, + "assigned_at": 0.0, + "rider": 1, + "predicted_eta": 100.07557221017964, + "pickup_arrival": 66.71704814011977, + "picked_up_at": 77.3344503190759, + "delivered_at": 110.69297438913577, + "deadline": 100.0, + "partial_wait_seconds": null + }, + { + "order": 2, + "realized_ready_at": 41.58998703019317, + "created_at": 30.0, + "observed_ready_at": 41.58998703019317, + "assigned_at": 30.0, + "rider": 2, + "predicted_eta": 189.031636397006, + "pickup_arrival": 118.95606418682635, + "picked_up_at": 118.95606418682635, + "delivered_at": 219.031636397006, + "deadline": 130.0, + "partial_wait_seconds": null + }, + { + "order": 3, + "realized_ready_at": 146.9687426958427, + "created_at": 60.0, + "observed_ready_at": 146.9687426958427, + "assigned_at": 110.69297438913577, + "rider": 1, + "predicted_eta": 66.71704814011974, + "pickup_arrival": 144.05149845919564, + "picked_up_at": 146.9687426958427, + "delivered_at": 180.32726676590258, + "deadline": 160.0, + "partial_wait_seconds": null + }, + { + "order": 4, + "realized_ready_at": 171.81549217035592, + "created_at": 90.0, + "observed_ready_at": 171.81549217035592, + "assigned_at": 180.32726676590258, + "rider": 1, + "predicted_eta": 66.71704814011974, + "pickup_arrival": 213.68579083596245, + "picked_up_at": 213.68579083596245, + "delivered_at": 247.04431490602232, + "deadline": 190.0, + "partial_wait_seconds": null + }, + { + "order": 5, + "realized_ready_at": 142.8773117938039, + "created_at": 120.0, + "observed_ready_at": 142.8773117938039, + "assigned_at": 219.031636397006, + "rider": 2, + "predicted_eta": 200.15114442035926, + "pickup_arrival": 319.1072086071856, + "picked_up_at": 319.1072086071856, + "delivered_at": 419.18278081736526, + "deadline": 220.0, + "partial_wait_seconds": null + }, + { + "order": 6, + "realized_ready_at": 224.61472535201995, + "created_at": 150.0, + "observed_ready_at": 224.61472535201995, + "assigned_at": 247.04431490602232, + "rider": 1, + "predicted_eta": 66.71704814011972, + "pickup_arrival": 280.4028389760822, + "picked_up_at": 280.4028389760822, + "delivered_at": 313.76136304614204, + "deadline": 250.0, + "partial_wait_seconds": null + }, + { + "order": 7, + "realized_ready_at": 257.47411499057955, + "created_at": 180.0, + "observed_ready_at": 257.47411499057955, + "assigned_at": 313.76136304614204, + "rider": 1, + "predicted_eta": 66.71704814011969, + "pickup_arrival": 347.1198871162019, + "picked_up_at": 347.1198871162019, + "delivered_at": 380.4784111862617, + "deadline": 280.0, + "partial_wait_seconds": null + }, + { + "order": 8, + "realized_ready_at": 220.00060669842472, + "created_at": 210.0, + "observed_ready_at": 220.00060669842472, + "assigned_at": 380.4784111862617, + "rider": 1, + "predicted_eta": 200.1511444203593, + "pickup_arrival": 480.55398339644137, + "picked_up_at": 480.55398339644137, + "delivered_at": 580.629555606621, + "deadline": 310.0, + "partial_wait_seconds": null + }, + { + "order": 9, + "realized_ready_at": 321.11353742827896, + "created_at": 240.0, + "observed_ready_at": 321.11353742827896, + "assigned_at": 419.18278081736526, + "rider": 2, + "predicted_eta": 66.71704814011969, + "pickup_arrival": 452.5413048874251, + "picked_up_at": 452.5413048874251, + "delivered_at": 485.89982895748494, + "deadline": 340.0, + "partial_wait_seconds": null + }, + { + "order": 10, + "realized_ready_at": 357.5207719241733, + "created_at": 270.0, + "observed_ready_at": 357.5207719241733, + "assigned_at": 485.89982895748494, + "rider": 2, + "predicted_eta": 66.71704814011969, + "pickup_arrival": 519.2583530275448, + "picked_up_at": 519.2583530275448, + "delivered_at": 552.6168770976046, + "deadline": 370.0, + "partial_wait_seconds": null + }, + { + "order": 11, + "realized_ready_at": 323.498135204727, + "created_at": 300.0, + "observed_ready_at": 323.498135204727, + "assigned_at": 552.6168770976046, + "rider": 2, + "predicted_eta": 200.1511444203593, + "pickup_arrival": 652.6924493077843, + "picked_up_at": 652.6924493077843, + "delivered_at": 752.7680215179639, + "deadline": 400.0, + "partial_wait_seconds": null + }, + { + "order": 12, + "realized_ready_at": 412.7209090542287, + "created_at": 330.0, + "observed_ready_at": 412.7209090542287, + "assigned_at": 580.629555606621, + "rider": 1, + "predicted_eta": 66.71704814011969, + "pickup_arrival": 613.9880796766809, + "picked_up_at": 613.9880796766809, + "delivered_at": 647.3466037467407, + "deadline": 430.0, + "partial_wait_seconds": null + }, + { + "order": 13, + "realized_ready_at": 100.0, + "created_at": 100.0, + "observed_ready_at": 100.0, + "assigned_at": null, + "rider": null, + "predicted_eta": null, + "pickup_arrival": null, + "picked_up_at": null, + "delivered_at": null, + "deadline": 200.0, + "partial_wait_seconds": null + }, + { + "order": 14, + "realized_ready_at": 950.0, + "created_at": null, + "observed_ready_at": null, + "assigned_at": null, + "rider": null, + "predicted_eta": null, + "pickup_arrival": null, + "picked_up_at": null, + "delivered_at": null, + "deadline": null, + "partial_wait_seconds": null + } + ], + "riders": [ + { + "rider": 1, + "available": true, + "busy_seconds": 647.3466037467407, + "idle_seconds": 252.6533962532593, + "completed_distance_meters": 4670.193369808382 + }, + { + "rider": 2, + "available": true, + "busy_seconds": 722.7680215179639, + "idle_seconds": 177.23197848203608, + "completed_distance_meters": 3891.8278081736516 + } + ], + "processed_events": 223, + "decision_count": 137, + "replay_sha256": "9afdfde66eb5f55faec063a3bbc8c6a6273421bea2b44d0071503657929909c5", + "timing": { + "samples_ns": [ + 5538625, + 5123041, + 5371584, + 5118750, + 4962916, + 5051000, + 5026208, + 5166167, + 5339167, + 5060792, + 5225667 + ], + "median_ns": 5123041, + "p95_ns": 5455104.5, + "p99_ns": 5521920.9 + } + }, + { + "dispatch": { + "kind": "lowest_pickup_eta" + }, + "summary": { + "scheduled_orders": 14, + "created_orders": 13, + "uncreated_orders": 1, + "assigned_orders": 12, + "delivered_orders": 12, + "unassigned_orders": 1, + "assigned_unfinished_orders": 0, + "outstanding_orders": 1, + "late_deliveries": 12, + "outstanding_past_deadline": 1, + "predicted_eta": { + "samples": 12, + "mean_seconds": 101.92882354740516, + "median_seconds": 66.71704814011974, + "p95_seconds": 200.1511444203593, + "p99_seconds": 200.1511444203593 + }, + "delivered_duration": { + "samples": 12, + "mean_seconds": 207.02396176046295, + "median_seconds": 184.5060814935568, + "p95_seconds": 414.0636518868605, + "p99_seconds": 414.0636518868605 + }, + "pickup_waiting": { + "samples": 12, + "mean_seconds": 5.800222128267463, + "median_seconds": 0.0, + "p95_seconds": 55.09543427236932, + "p99_seconds": 55.09543427236932 + }, + "completed_distance_meters": 8339.631017514968, + "rider_utilization": 0.7181936378378175, + "mean_rider_idle_seconds": 253.62572594596423 + }, + "orders": [ + { + "order": 1, + "realized_ready_at": 77.3344503190759, + "created_at": 0.0, + "observed_ready_at": 77.3344503190759, + "assigned_at": 0.0, + "rider": 2, + "predicted_eta": 55.59754011676645, + "pickup_arrival": 22.239016046706585, + "picked_up_at": 77.3344503190759, + "delivered_at": 110.69297438913577, + "deadline": 100.0, + "partial_wait_seconds": null + }, + { + "order": 2, + "realized_ready_at": 41.58998703019317, + "created_at": 30.0, + "observed_ready_at": 41.58998703019317, + "assigned_at": 30.0, + "rider": 1, + "predicted_eta": 100.07557221017964, + "pickup_arrival": 30.0, + "picked_up_at": 41.58998703019317, + "delivered_at": 141.66555924037283, + "deadline": 130.0, + "partial_wait_seconds": null + }, + { + "order": 3, + "realized_ready_at": 146.9687426958427, + "created_at": 60.0, + "observed_ready_at": 146.9687426958427, + "assigned_at": 110.69297438913577, + "rider": 2, + "predicted_eta": 66.71704814011974, + "pickup_arrival": 144.05149845919564, + "picked_up_at": 146.9687426958427, + "delivered_at": 180.32726676590258, + "deadline": 160.0, + "partial_wait_seconds": null + }, + { + "order": 4, + "realized_ready_at": 171.81549217035592, + "created_at": 90.0, + "observed_ready_at": 171.81549217035592, + "assigned_at": 141.66555924037283, + "rider": 1, + "predicted_eta": 66.71704814011974, + "pickup_arrival": 175.0240833104327, + "picked_up_at": 175.0240833104327, + "delivered_at": 208.38260738049257, + "deadline": 190.0, + "partial_wait_seconds": null + }, + { + "order": 5, + "realized_ready_at": 142.8773117938039, + "created_at": 120.0, + "observed_ready_at": 142.8773117938039, + "assigned_at": 180.32726676590258, + "rider": 2, + "predicted_eta": 200.15114442035926, + "pickup_arrival": 280.4028389760822, + "picked_up_at": 280.4028389760822, + "delivered_at": 380.47841118626184, + "deadline": 220.0, + "partial_wait_seconds": null + }, + { + "order": 6, + "realized_ready_at": 224.61472535201995, + "created_at": 150.0, + "observed_ready_at": 224.61472535201995, + "assigned_at": 208.38260738049257, + "rider": 1, + "predicted_eta": 66.71704814011974, + "pickup_arrival": 241.74113145055244, + "picked_up_at": 241.74113145055244, + "delivered_at": 275.0996555206123, + "deadline": 250.0, + "partial_wait_seconds": null + }, + { + "order": 7, + "realized_ready_at": 257.47411499057955, + "created_at": 180.0, + "observed_ready_at": 257.47411499057955, + "assigned_at": 275.0996555206123, + "rider": 1, + "predicted_eta": 66.71704814011974, + "pickup_arrival": 308.4581795906722, + "picked_up_at": 308.4581795906722, + "delivered_at": 341.81670366073206, + "deadline": 280.0, + "partial_wait_seconds": null + }, + { + "order": 8, + "realized_ready_at": 220.00060669842472, + "created_at": 210.0, + "observed_ready_at": 220.00060669842472, + "assigned_at": 341.81670366073206, + "rider": 1, + "predicted_eta": 200.1511444203593, + "pickup_arrival": 441.8922758709117, + "picked_up_at": 441.8922758709117, + "delivered_at": 541.9678480810913, + "deadline": 310.0, + "partial_wait_seconds": null + }, + { + "order": 9, + "realized_ready_at": 321.11353742827896, + "created_at": 240.0, + "observed_ready_at": 321.11353742827896, + "assigned_at": 380.47841118626184, + "rider": 2, + "predicted_eta": 66.71704814011969, + "pickup_arrival": 413.8369352563217, + "picked_up_at": 413.8369352563217, + "delivered_at": 447.1954593263815, + "deadline": 340.0, + "partial_wait_seconds": null + }, + { + "order": 10, + "realized_ready_at": 357.5207719241733, + "created_at": 270.0, + "observed_ready_at": 357.5207719241733, + "assigned_at": 447.1954593263815, + "rider": 2, + "predicted_eta": 66.71704814011969, + "pickup_arrival": 480.55398339644137, + "picked_up_at": 480.55398339644137, + "delivered_at": 513.9125074665012, + "deadline": 370.0, + "partial_wait_seconds": null + }, + { + "order": 11, + "realized_ready_at": 323.498135204727, + "created_at": 300.0, + "observed_ready_at": 323.498135204727, + "assigned_at": 513.9125074665012, + "rider": 2, + "predicted_eta": 200.1511444203593, + "pickup_arrival": 613.9880796766809, + "picked_up_at": 613.9880796766809, + "delivered_at": 714.0636518868605, + "deadline": 400.0, + "partial_wait_seconds": null + }, + { + "order": 12, + "realized_ready_at": 412.7209090542287, + "created_at": 330.0, + "observed_ready_at": 412.7209090542287, + "assigned_at": 541.9678480810913, + "rider": 1, + "predicted_eta": 66.71704814011969, + "pickup_arrival": 575.3263721511512, + "picked_up_at": 575.3263721511512, + "delivered_at": 608.684896221211, + "deadline": 430.0, + "partial_wait_seconds": null + }, + { + "order": 13, + "realized_ready_at": 100.0, + "created_at": 100.0, + "observed_ready_at": 100.0, + "assigned_at": null, + "rider": null, + "predicted_eta": null, + "pickup_arrival": null, + "picked_up_at": null, + "delivered_at": null, + "deadline": 200.0, + "partial_wait_seconds": null + }, + { + "order": 14, + "realized_ready_at": 950.0, + "created_at": null, + "observed_ready_at": null, + "assigned_at": null, + "rider": null, + "predicted_eta": null, + "pickup_arrival": null, + "picked_up_at": null, + "delivered_at": null, + "deadline": null, + "partial_wait_seconds": null + } + ], + "riders": [ + { + "rider": 1, + "available": true, + "busy_seconds": 578.684896221211, + "idle_seconds": 321.31510377878897, + "completed_distance_meters": 4003.022888407185 + }, + { + "rider": 2, + "available": true, + "busy_seconds": 714.0636518868605, + "idle_seconds": 185.9363481131395, + "completed_distance_meters": 4336.6081291077835 + } + ], + "processed_events": 204, + "decision_count": 118, + "replay_sha256": "009eb84cefe08fb9924a1ab541409462ccf2dba72f5e2cdacb62591cff73986f", + "timing": { + "samples_ns": [ + 5205417, + 5073792, + 5085291, + 5676791, + 5647167, + 5038125, + 5063333, + 4996125, + 5085542, + 5123542, + 5229333 + ], + "median_ns": 5085542, + "p95_ns": 5661979.0, + "p99_ns": 5673828.6 + } + }, + { + "dispatch": { + "kind": "lowest_completion_time" + }, + "summary": { + "scheduled_orders": 14, + "created_orders": 13, + "uncreated_orders": 1, + "assigned_orders": 12, + "delivered_orders": 12, + "unassigned_orders": 1, + "assigned_unfinished_orders": 0, + "outstanding_orders": 1, + "late_deliveries": 12, + "outstanding_past_deadline": 1, + "predicted_eta": { + "samples": 12, + "mean_seconds": 114.15524422574846, + "median_seconds": 66.71704814011974, + "p95_seconds": 200.1511444203593, + "p99_seconds": 200.1511444203593 + }, + "delivered_duration": { + "samples": 12, + "mean_seconds": 242.48163620285425, + "median_seconds": 223.18912007187333, + "p95_seconds": 452.7680215179639, + "p99_seconds": 452.7680215179639 + }, + "pickup_waiting": { + "samples": 12, + "mean_seconds": 1.1278872013002659, + "median_seconds": 0.0, + "p95_seconds": 10.617402178956127, + "p99_seconds": 10.617402178956127 + }, + "completed_distance_meters": 8562.021177982035, + "rider_utilization": 0.7611747918137247, + "mean_rider_idle_seconds": 214.9426873676477 + }, + "orders": [ + { + "order": 1, + "realized_ready_at": 77.3344503190759, + "created_at": 0.0, + "observed_ready_at": 77.3344503190759, + "assigned_at": 0.0, + "rider": 1, + "predicted_eta": 113.35852407005987, + "pickup_arrival": 66.71704814011977, + "picked_up_at": 77.3344503190759, + "delivered_at": 110.69297438913577, + "deadline": 100.0, + "partial_wait_seconds": null + }, + { + "order": 2, + "realized_ready_at": 41.58998703019317, + "created_at": 30.0, + "observed_ready_at": 41.58998703019317, + "assigned_at": 30.0, + "rider": 2, + "predicted_eta": 189.031636397006, + "pickup_arrival": 118.95606418682635, + "picked_up_at": 118.95606418682635, + "delivered_at": 219.031636397006, + "deadline": 130.0, + "partial_wait_seconds": null + }, + { + "order": 3, + "realized_ready_at": 146.9687426958427, + "created_at": 60.0, + "observed_ready_at": 146.9687426958427, + "assigned_at": 110.69297438913577, + "rider": 1, + "predicted_eta": 66.71704814011974, + "pickup_arrival": 144.05149845919564, + "picked_up_at": 146.9687426958427, + "delivered_at": 180.32726676590258, + "deadline": 160.0, + "partial_wait_seconds": null + }, + { + "order": 4, + "realized_ready_at": 171.81549217035592, + "created_at": 90.0, + "observed_ready_at": 171.81549217035592, + "assigned_at": 180.32726676590258, + "rider": 1, + "predicted_eta": 66.71704814011974, + "pickup_arrival": 213.68579083596245, + "picked_up_at": 213.68579083596245, + "delivered_at": 247.04431490602232, + "deadline": 190.0, + "partial_wait_seconds": null + }, + { + "order": 5, + "realized_ready_at": 142.8773117938039, + "created_at": 120.0, + "observed_ready_at": 142.8773117938039, + "assigned_at": 219.031636397006, + "rider": 2, + "predicted_eta": 200.15114442035926, + "pickup_arrival": 319.1072086071856, + "picked_up_at": 319.1072086071856, + "delivered_at": 419.18278081736526, + "deadline": 220.0, + "partial_wait_seconds": null + }, + { + "order": 6, + "realized_ready_at": 224.61472535201995, + "created_at": 150.0, + "observed_ready_at": 224.61472535201995, + "assigned_at": 247.04431490602232, + "rider": 1, + "predicted_eta": 66.71704814011972, + "pickup_arrival": 280.4028389760822, + "picked_up_at": 280.4028389760822, + "delivered_at": 313.76136304614204, + "deadline": 250.0, + "partial_wait_seconds": null + }, + { + "order": 7, + "realized_ready_at": 257.47411499057955, + "created_at": 180.0, + "observed_ready_at": 257.47411499057955, + "assigned_at": 313.76136304614204, + "rider": 1, + "predicted_eta": 66.71704814011969, + "pickup_arrival": 347.1198871162019, + "picked_up_at": 347.1198871162019, + "delivered_at": 380.4784111862617, + "deadline": 280.0, + "partial_wait_seconds": null + }, + { + "order": 8, + "realized_ready_at": 220.00060669842472, + "created_at": 210.0, + "observed_ready_at": 220.00060669842472, + "assigned_at": 380.4784111862617, + "rider": 1, + "predicted_eta": 200.1511444203593, + "pickup_arrival": 480.55398339644137, + "picked_up_at": 480.55398339644137, + "delivered_at": 580.629555606621, + "deadline": 310.0, + "partial_wait_seconds": null + }, + { + "order": 9, + "realized_ready_at": 321.11353742827896, + "created_at": 240.0, + "observed_ready_at": 321.11353742827896, + "assigned_at": 419.18278081736526, + "rider": 2, + "predicted_eta": 66.71704814011969, + "pickup_arrival": 452.5413048874251, + "picked_up_at": 452.5413048874251, + "delivered_at": 485.89982895748494, + "deadline": 340.0, + "partial_wait_seconds": null + }, + { + "order": 10, + "realized_ready_at": 357.5207719241733, + "created_at": 270.0, + "observed_ready_at": 357.5207719241733, + "assigned_at": 485.89982895748494, + "rider": 2, + "predicted_eta": 66.71704814011969, + "pickup_arrival": 519.2583530275448, + "picked_up_at": 519.2583530275448, + "delivered_at": 552.6168770976046, + "deadline": 370.0, + "partial_wait_seconds": null + }, + { + "order": 11, + "realized_ready_at": 323.498135204727, + "created_at": 300.0, + "observed_ready_at": 323.498135204727, + "assigned_at": 552.6168770976046, + "rider": 2, + "predicted_eta": 200.1511444203593, + "pickup_arrival": 652.6924493077843, + "picked_up_at": 652.6924493077843, + "delivered_at": 752.7680215179639, + "deadline": 400.0, + "partial_wait_seconds": null + }, + { + "order": 12, + "realized_ready_at": 412.7209090542287, + "created_at": 330.0, + "observed_ready_at": 412.7209090542287, + "assigned_at": 580.629555606621, + "rider": 1, + "predicted_eta": 66.71704814011969, + "pickup_arrival": 613.9880796766809, + "picked_up_at": 613.9880796766809, + "delivered_at": 647.3466037467407, + "deadline": 430.0, + "partial_wait_seconds": null + }, + { + "order": 13, + "realized_ready_at": 100.0, + "created_at": 100.0, + "observed_ready_at": 100.0, + "assigned_at": null, + "rider": null, + "predicted_eta": null, + "pickup_arrival": null, + "picked_up_at": null, + "delivered_at": null, + "deadline": 200.0, + "partial_wait_seconds": null + }, + { + "order": 14, + "realized_ready_at": 950.0, + "created_at": null, + "observed_ready_at": null, + "assigned_at": null, + "rider": null, + "predicted_eta": null, + "pickup_arrival": null, + "picked_up_at": null, + "delivered_at": null, + "deadline": null, + "partial_wait_seconds": null + } + ], + "riders": [ + { + "rider": 1, + "available": true, + "busy_seconds": 647.3466037467407, + "idle_seconds": 252.6533962532593, + "completed_distance_meters": 4670.193369808382 + }, + { + "rider": 2, + "available": true, + "busy_seconds": 722.7680215179639, + "idle_seconds": 177.23197848203608, + "completed_distance_meters": 3891.8278081736516 + } + ], + "processed_events": 223, + "decision_count": 137, + "replay_sha256": "c3f5b4aa24a5c3c2fcb760d21c108709166d6eb20e254f9d42cf78bb3af5f635", + "timing": { + "samples_ns": [ + 5593792, + 5192459, + 5221666, + 5183584, + 5207000, + 5106583, + 5122542, + 5334250, + 5049792, + 5130959, + 4975125 + ], + "median_ns": 5183584, + "p95_ns": 5464021.0, + "p99_ns": 5567837.8 + } + }, + { + "dispatch": { + "kind": "preparation_aware", + "idle_penalty_weight": 1.0 + }, + "summary": { + "scheduled_orders": 14, + "created_orders": 13, + "uncreated_orders": 1, + "assigned_orders": 12, + "delivered_orders": 12, + "unassigned_orders": 1, + "assigned_unfinished_orders": 0, + "outstanding_orders": 1, + "late_deliveries": 12, + "outstanding_past_deadline": 1, + "predicted_eta": { + "samples": 12, + "mean_seconds": 114.15524422574846, + "median_seconds": 66.71704814011974, + "p95_seconds": 200.1511444203593, + "p99_seconds": 200.1511444203593 + }, + "delivered_duration": { + "samples": 12, + "mean_seconds": 242.48163620285425, + "median_seconds": 223.18912007187333, + "p95_seconds": 452.7680215179639, + "p99_seconds": 452.7680215179639 + }, + "pickup_waiting": { + "samples": 12, + "mean_seconds": 1.1278872013002659, + "median_seconds": 0.0, + "p95_seconds": 10.617402178956127, + "p99_seconds": 10.617402178956127 + }, + "completed_distance_meters": 8562.021177982035, + "rider_utilization": 0.7611747918137247, + "mean_rider_idle_seconds": 214.9426873676477 + }, + "orders": [ + { + "order": 1, + "realized_ready_at": 77.3344503190759, + "created_at": 0.0, + "observed_ready_at": 77.3344503190759, + "assigned_at": 0.0, + "rider": 1, + "predicted_eta": 113.35852407005987, + "pickup_arrival": 66.71704814011977, + "picked_up_at": 77.3344503190759, + "delivered_at": 110.69297438913577, + "deadline": 100.0, + "partial_wait_seconds": null + }, + { + "order": 2, + "realized_ready_at": 41.58998703019317, + "created_at": 30.0, + "observed_ready_at": 41.58998703019317, + "assigned_at": 30.0, + "rider": 2, + "predicted_eta": 189.031636397006, + "pickup_arrival": 118.95606418682635, + "picked_up_at": 118.95606418682635, + "delivered_at": 219.031636397006, + "deadline": 130.0, + "partial_wait_seconds": null + }, + { + "order": 3, + "realized_ready_at": 146.9687426958427, + "created_at": 60.0, + "observed_ready_at": 146.9687426958427, + "assigned_at": 110.69297438913577, + "rider": 1, + "predicted_eta": 66.71704814011974, + "pickup_arrival": 144.05149845919564, + "picked_up_at": 146.9687426958427, + "delivered_at": 180.32726676590258, + "deadline": 160.0, + "partial_wait_seconds": null + }, + { + "order": 4, + "realized_ready_at": 171.81549217035592, + "created_at": 90.0, + "observed_ready_at": 171.81549217035592, + "assigned_at": 180.32726676590258, + "rider": 1, + "predicted_eta": 66.71704814011974, + "pickup_arrival": 213.68579083596245, + "picked_up_at": 213.68579083596245, + "delivered_at": 247.04431490602232, + "deadline": 190.0, + "partial_wait_seconds": null + }, + { + "order": 5, + "realized_ready_at": 142.8773117938039, + "created_at": 120.0, + "observed_ready_at": 142.8773117938039, + "assigned_at": 219.031636397006, + "rider": 2, + "predicted_eta": 200.15114442035926, + "pickup_arrival": 319.1072086071856, + "picked_up_at": 319.1072086071856, + "delivered_at": 419.18278081736526, + "deadline": 220.0, + "partial_wait_seconds": null + }, + { + "order": 6, + "realized_ready_at": 224.61472535201995, + "created_at": 150.0, + "observed_ready_at": 224.61472535201995, + "assigned_at": 247.04431490602232, + "rider": 1, + "predicted_eta": 66.71704814011972, + "pickup_arrival": 280.4028389760822, + "picked_up_at": 280.4028389760822, + "delivered_at": 313.76136304614204, + "deadline": 250.0, + "partial_wait_seconds": null + }, + { + "order": 7, + "realized_ready_at": 257.47411499057955, + "created_at": 180.0, + "observed_ready_at": 257.47411499057955, + "assigned_at": 313.76136304614204, + "rider": 1, + "predicted_eta": 66.71704814011969, + "pickup_arrival": 347.1198871162019, + "picked_up_at": 347.1198871162019, + "delivered_at": 380.4784111862617, + "deadline": 280.0, + "partial_wait_seconds": null + }, + { + "order": 8, + "realized_ready_at": 220.00060669842472, + "created_at": 210.0, + "observed_ready_at": 220.00060669842472, + "assigned_at": 380.4784111862617, + "rider": 1, + "predicted_eta": 200.1511444203593, + "pickup_arrival": 480.55398339644137, + "picked_up_at": 480.55398339644137, + "delivered_at": 580.629555606621, + "deadline": 310.0, + "partial_wait_seconds": null + }, + { + "order": 9, + "realized_ready_at": 321.11353742827896, + "created_at": 240.0, + "observed_ready_at": 321.11353742827896, + "assigned_at": 419.18278081736526, + "rider": 2, + "predicted_eta": 66.71704814011969, + "pickup_arrival": 452.5413048874251, + "picked_up_at": 452.5413048874251, + "delivered_at": 485.89982895748494, + "deadline": 340.0, + "partial_wait_seconds": null + }, + { + "order": 10, + "realized_ready_at": 357.5207719241733, + "created_at": 270.0, + "observed_ready_at": 357.5207719241733, + "assigned_at": 485.89982895748494, + "rider": 2, + "predicted_eta": 66.71704814011969, + "pickup_arrival": 519.2583530275448, + "picked_up_at": 519.2583530275448, + "delivered_at": 552.6168770976046, + "deadline": 370.0, + "partial_wait_seconds": null + }, + { + "order": 11, + "realized_ready_at": 323.498135204727, + "created_at": 300.0, + "observed_ready_at": 323.498135204727, + "assigned_at": 552.6168770976046, + "rider": 2, + "predicted_eta": 200.1511444203593, + "pickup_arrival": 652.6924493077843, + "picked_up_at": 652.6924493077843, + "delivered_at": 752.7680215179639, + "deadline": 400.0, + "partial_wait_seconds": null + }, + { + "order": 12, + "realized_ready_at": 412.7209090542287, + "created_at": 330.0, + "observed_ready_at": 412.7209090542287, + "assigned_at": 580.629555606621, + "rider": 1, + "predicted_eta": 66.71704814011969, + "pickup_arrival": 613.9880796766809, + "picked_up_at": 613.9880796766809, + "delivered_at": 647.3466037467407, + "deadline": 430.0, + "partial_wait_seconds": null + }, + { + "order": 13, + "realized_ready_at": 100.0, + "created_at": 100.0, + "observed_ready_at": 100.0, + "assigned_at": null, + "rider": null, + "predicted_eta": null, + "pickup_arrival": null, + "picked_up_at": null, + "delivered_at": null, + "deadline": 200.0, + "partial_wait_seconds": null + }, + { + "order": 14, + "realized_ready_at": 950.0, + "created_at": null, + "observed_ready_at": null, + "assigned_at": null, + "rider": null, + "predicted_eta": null, + "pickup_arrival": null, + "picked_up_at": null, + "delivered_at": null, + "deadline": null, + "partial_wait_seconds": null + } + ], + "riders": [ + { + "rider": 1, + "available": true, + "busy_seconds": 647.3466037467407, + "idle_seconds": 252.6533962532593, + "completed_distance_meters": 4670.193369808382 + }, + { + "rider": 2, + "available": true, + "busy_seconds": 722.7680215179639, + "idle_seconds": 177.23197848203608, + "completed_distance_meters": 3891.8278081736516 + } + ], + "processed_events": 223, + "decision_count": 137, + "replay_sha256": "c6decf80cbc6e193a53ed831fe026bb11351e14a1975b2ded78d9e9a8ed99b0b", + "timing": { + "samples_ns": [ + 5218542, + 5174667, + 5129458, + 5174166, + 5063500, + 5201958, + 5204209, + 5069209, + 5013083, + 5225750, + 5242250 + ], + "median_ns": 5174667, + "p95_ns": 5234000.0, + "p99_ns": 5240600.0 + } + } + ] +} diff --git a/benchmarks/results/2026-10-06-phase-16-preparation-strategies.json b/benchmarks/results/2026-10-06-phase-16-preparation-strategies.json new file mode 100644 index 0000000..318e061 --- /dev/null +++ b/benchmarks/results/2026-10-06-phase-16-preparation-strategies.json @@ -0,0 +1,715 @@ +{ + "schema_version": 1, + "date": "2026-10-06", + "purpose": "Paired synthetic dispatch strategy comparison; no production scalability claim", + "hardware": { + "operating_system": "macOS-27.0-arm64-arm-64bit-Mach-O", + "architecture": "arm64", + "cpu": "Apple M3", + "memory_bytes": 17179869184 + }, + "dataset": { + "path": "data/fixtures/phase-15/seeded-deliveries.json", + "sha256": "678be268e5c0e6544f4a80f3305d1693a23f2c44036bf37066c65e2bac26b76e", + "kind": "synthetic", + "graph_artifact": null, + "seed": 15, + "randomness": "splitmix64-upper53/v1", + "graph_digest": "5017e4b91bc6031d70113ac851341e77a954a628ba2234cbf0a94e946d56e24b", + "graph_nodes": 5, + "graph_directed_edges": 6, + "riders": 2, + "scheduled_orders": 6 + }, + "algorithm": { + "routing": "Roadrunner Dijkstra with static traffic overlays", + "events": "BinaryHeap ordered by logical time and insertion sequence", + "coverage": "exhaustive eligible road-feasible riders", + "tie_break": "lowest RiderId" + }, + "configuration": { + "build": "cargo +1.99.0 build -p roadrunner-cli --release --locked", + "rustc": "rustc 1.99.0 (b940084d7 2026-09-28)", + "binary_sha256": "b02585359b10089e53dbc5481e42c93efc1f2ba9f90b6304c665a571ff248f74", + "profile": "release", + "comparison_command": [ + "/Users/admin/Developer/roadrunner/target/release/roadrunner", + "benchmark", + "dispatch", + "/Users/admin/Developer/roadrunner/data/fixtures/phase-15/seeded-deliveries.json", + "--idle-penalty-weight", + "1.0", + "--json" + ], + "idle_penalty_weight": 1.0, + "warmup_runs_per_strategy": 1, + "measured_runs_per_strategy": 11, + "start_seconds": 0.0, + "end_seconds": 600.0, + "includes": "process startup, graph/scenario loading and validation, simulation, JSON serialization and capture", + "timing_percentiles": "linear interpolation of ordered elapsed process samples", + "outcome_percentiles": "nearest rank; even median midpoint", + "memory": "not measured", + "concurrent_workspace_validation": false + }, + "comparison_replay_sha256": "34b21683855b5b1ffd152effb180f9dac2fe200bf885924348c27a171ed37b36", + "strategies": [ + { + "dispatch": { + "kind": "nearest_rider" + }, + "summary": { + "scheduled_orders": 6, + "created_orders": 5, + "uncreated_orders": 1, + "assigned_orders": 4, + "delivered_orders": 4, + "unassigned_orders": 1, + "assigned_unfinished_orders": 0, + "outstanding_orders": 1, + "late_deliveries": 1, + "outstanding_past_deadline": 1, + "predicted_eta": { + "samples": 4, + "mean_seconds": 68.38497434362273, + "median_seconds": 71.16485134946106, + "p95_seconds": 86.73216258215567, + "p99_seconds": 86.73216258215567 + }, + "delivered_duration": { + "samples": 4, + "mean_seconds": 90.91893465396457, + "median_seconds": 92.87424790008487, + "p95_seconds": 122.32970269892212, + "p99_seconds": 122.32970269892212 + }, + "pickup_waiting": { + "samples": 4, + "mean_seconds": 10.007533628898468, + "median_seconds": 0.0, + "p95_seconds": 40.03013451559387, + "p99_seconds": 40.03013451559387 + }, + "completed_distance_meters": 2335.0966849041906, + "rider_utilization": 0.261308359908404, + "mean_rider_idle_seconds": 443.2149840549576 + }, + "orders": [ + { + "order": 1, + "realized_ready_at": 51.14964253894716, + "created_at": 0.0, + "observed_ready_at": 51.14964253894716, + "assigned_at": 0.0, + "rider": 1, + "predicted_eta": 44.47803209341316, + "pickup_arrival": 11.119508023353294, + "picked_up_at": 51.14964253894716, + "delivered_at": 84.50816660900703, + "deadline": 200.0, + "partial_wait_seconds": null + }, + { + "order": 2, + "realized_ready_at": 40.0, + "created_at": 20.0, + "observed_ready_at": 40.0, + "assigned_at": 20.0, + "rider": 2, + "predicted_eta": 55.59754011676645, + "pickup_arrival": 42.23901604670658, + "picked_up_at": 42.23901604670658, + "delivered_at": 75.59754011676645, + "deadline": 60.0, + "partial_wait_seconds": null + }, + { + "order": 3, + "realized_ready_at": 101.20611063887397, + "created_at": 40.0, + "observed_ready_at": 101.20611063887397, + "assigned_at": 75.59754011676645, + "rider": 2, + "predicted_eta": 86.73216258215567, + "pickup_arrival": 108.95606418682632, + "picked_up_at": 108.95606418682632, + "delivered_at": 162.32970269892212, + "deadline": 200.0, + "partial_wait_seconds": null + }, + { + "order": 4, + "realized_ready_at": 100.0, + "created_at": 70.0, + "observed_ready_at": 100.0, + "assigned_at": 84.50816660900703, + "rider": 1, + "predicted_eta": 86.73216258215565, + "pickup_arrival": 117.8666906790669, + "picked_up_at": 117.8666906790669, + "delivered_at": 171.2403291911627, + "deadline": 200.0, + "partial_wait_seconds": null + }, + { + "order": 5, + "realized_ready_at": 80.0, + "created_at": 80.0, + "observed_ready_at": 80.0, + "assigned_at": null, + "rider": null, + "predicted_eta": null, + "pickup_arrival": null, + "picked_up_at": null, + "delivered_at": null, + "deadline": 200.0, + "partial_wait_seconds": null + }, + { + "order": 6, + "realized_ready_at": 700.0, + "created_at": null, + "observed_ready_at": null, + "assigned_at": null, + "rider": null, + "predicted_eta": null, + "pickup_arrival": null, + "picked_up_at": null, + "delivered_at": null, + "deadline": 200.0, + "partial_wait_seconds": null + } + ], + "riders": [ + { + "rider": 1, + "available": true, + "busy_seconds": 171.2403291911627, + "idle_seconds": 428.7596708088373, + "completed_distance_meters": 1111.950802335329 + }, + { + "rider": 2, + "available": true, + "busy_seconds": 142.32970269892212, + "idle_seconds": 457.6702973010779, + "completed_distance_meters": 1223.145882568862 + } + ], + "processed_events": 50, + "decision_count": 18, + "replay_sha256": "f64fb4926a8f3e65cfab85a603684dc2fc0d12fe286d32ce26bfb42c7dac1651", + "timing": { + "samples_ns": [ + 4341500, + 4357292, + 4287458, + 4744833, + 4886958, + 4791042, + 4922833, + 4397417, + 4337875, + 4470125, + 4494792 + ], + "median_ns": 4470125, + "p95_ns": 4904895.5, + "p99_ns": 4919245.5 + } + }, + { + "dispatch": { + "kind": "lowest_pickup_eta" + }, + "summary": { + "scheduled_orders": 6, + "created_orders": 5, + "uncreated_orders": 1, + "assigned_orders": 4, + "delivered_orders": 4, + "unassigned_orders": 1, + "assigned_unfinished_orders": 0, + "outstanding_orders": 1, + "late_deliveries": 1, + "outstanding_past_deadline": 1, + "predicted_eta": { + "samples": 4, + "mean_seconds": 68.38497434362273, + "median_seconds": 71.16485134946106, + "p95_seconds": 86.73216258215567, + "p99_seconds": 86.73216258215567 + }, + "delivered_duration": { + "samples": 4, + "mean_seconds": 90.91893465396457, + "median_seconds": 92.87424790008487, + "p95_seconds": 122.32970269892212, + "p99_seconds": 122.32970269892212 + }, + "pickup_waiting": { + "samples": 4, + "mean_seconds": 10.007533628898468, + "median_seconds": 0.0, + "p95_seconds": 40.03013451559387, + "p99_seconds": 40.03013451559387 + }, + "completed_distance_meters": 2335.0966849041906, + "rider_utilization": 0.261308359908404, + "mean_rider_idle_seconds": 443.2149840549576 + }, + "orders": [ + { + "order": 1, + "realized_ready_at": 51.14964253894716, + "created_at": 0.0, + "observed_ready_at": 51.14964253894716, + "assigned_at": 0.0, + "rider": 1, + "predicted_eta": 44.47803209341316, + "pickup_arrival": 11.119508023353294, + "picked_up_at": 51.14964253894716, + "delivered_at": 84.50816660900703, + "deadline": 200.0, + "partial_wait_seconds": null + }, + { + "order": 2, + "realized_ready_at": 40.0, + "created_at": 20.0, + "observed_ready_at": 40.0, + "assigned_at": 20.0, + "rider": 2, + "predicted_eta": 55.59754011676645, + "pickup_arrival": 42.23901604670658, + "picked_up_at": 42.23901604670658, + "delivered_at": 75.59754011676645, + "deadline": 60.0, + "partial_wait_seconds": null + }, + { + "order": 3, + "realized_ready_at": 101.20611063887397, + "created_at": 40.0, + "observed_ready_at": 101.20611063887397, + "assigned_at": 75.59754011676645, + "rider": 2, + "predicted_eta": 86.73216258215567, + "pickup_arrival": 108.95606418682632, + "picked_up_at": 108.95606418682632, + "delivered_at": 162.32970269892212, + "deadline": 200.0, + "partial_wait_seconds": null + }, + { + "order": 4, + "realized_ready_at": 100.0, + "created_at": 70.0, + "observed_ready_at": 100.0, + "assigned_at": 84.50816660900703, + "rider": 1, + "predicted_eta": 86.73216258215565, + "pickup_arrival": 117.8666906790669, + "picked_up_at": 117.8666906790669, + "delivered_at": 171.2403291911627, + "deadline": 200.0, + "partial_wait_seconds": null + }, + { + "order": 5, + "realized_ready_at": 80.0, + "created_at": 80.0, + "observed_ready_at": 80.0, + "assigned_at": null, + "rider": null, + "predicted_eta": null, + "pickup_arrival": null, + "picked_up_at": null, + "delivered_at": null, + "deadline": 200.0, + "partial_wait_seconds": null + }, + { + "order": 6, + "realized_ready_at": 700.0, + "created_at": null, + "observed_ready_at": null, + "assigned_at": null, + "rider": null, + "predicted_eta": null, + "pickup_arrival": null, + "picked_up_at": null, + "delivered_at": null, + "deadline": 200.0, + "partial_wait_seconds": null + } + ], + "riders": [ + { + "rider": 1, + "available": true, + "busy_seconds": 171.2403291911627, + "idle_seconds": 428.7596708088373, + "completed_distance_meters": 1111.950802335329 + }, + { + "rider": 2, + "available": true, + "busy_seconds": 142.32970269892212, + "idle_seconds": 457.6702973010779, + "completed_distance_meters": 1223.145882568862 + } + ], + "processed_events": 50, + "decision_count": 18, + "replay_sha256": "c023a56bf6a60f726e10970a391a3cc4ddf2a2fee91dbce4e70eed3bfdddfe94", + "timing": { + "samples_ns": [ + 4384042, + 4117666, + 4128792, + 4450750, + 4527959, + 4434000, + 4240167, + 4172000, + 4135541, + 4454875, + 4291500 + ], + "median_ns": 4291500, + "p95_ns": 4491417.0, + "p99_ns": 4520650.6 + } + }, + { + "dispatch": { + "kind": "lowest_completion_time" + }, + "summary": { + "scheduled_orders": 6, + "created_orders": 5, + "uncreated_orders": 1, + "assigned_orders": 4, + "delivered_orders": 4, + "unassigned_orders": 1, + "assigned_unfinished_orders": 0, + "outstanding_orders": 1, + "late_deliveries": 1, + "outstanding_past_deadline": 1, + "predicted_eta": { + "samples": 4, + "mean_seconds": 80.60509733778441, + "median_seconds": 86.73216258215567, + "p95_seconds": 93.35852407005987, + "p99_seconds": 93.35852407005987 + }, + "delivered_duration": { + "samples": 4, + "mean_seconds": 90.91893465396457, + "median_seconds": 92.87424790008487, + "p95_seconds": 122.32970269892212, + "p99_seconds": 122.32970269892212 + }, + "pickup_waiting": { + "samples": 4, + "mean_seconds": 10.007533628898468, + "median_seconds": 0.0, + "p95_seconds": 40.03013451559387, + "p99_seconds": 40.03013451559387 + }, + "completed_distance_meters": 2335.0966849041906, + "rider_utilization": 0.261308359908404, + "mean_rider_idle_seconds": 443.2149840549576 + }, + "orders": [ + { + "order": 1, + "realized_ready_at": 51.14964253894716, + "created_at": 0.0, + "observed_ready_at": 51.14964253894716, + "assigned_at": 0.0, + "rider": 1, + "predicted_eta": 93.35852407005987, + "pickup_arrival": 11.119508023353294, + "picked_up_at": 51.14964253894716, + "delivered_at": 84.50816660900703, + "deadline": 200.0, + "partial_wait_seconds": null + }, + { + "order": 2, + "realized_ready_at": 40.0, + "created_at": 20.0, + "observed_ready_at": 40.0, + "assigned_at": 20.0, + "rider": 2, + "predicted_eta": 55.59754011676645, + "pickup_arrival": 42.23901604670658, + "picked_up_at": 42.23901604670658, + "delivered_at": 75.59754011676645, + "deadline": 60.0, + "partial_wait_seconds": null + }, + { + "order": 3, + "realized_ready_at": 101.20611063887397, + "created_at": 40.0, + "observed_ready_at": 101.20611063887397, + "assigned_at": 75.59754011676645, + "rider": 2, + "predicted_eta": 86.73216258215567, + "pickup_arrival": 108.95606418682632, + "picked_up_at": 108.95606418682632, + "delivered_at": 162.32970269892212, + "deadline": 200.0, + "partial_wait_seconds": null + }, + { + "order": 4, + "realized_ready_at": 100.0, + "created_at": 70.0, + "observed_ready_at": 100.0, + "assigned_at": 84.50816660900703, + "rider": 1, + "predicted_eta": 86.73216258215565, + "pickup_arrival": 117.8666906790669, + "picked_up_at": 117.8666906790669, + "delivered_at": 171.2403291911627, + "deadline": 200.0, + "partial_wait_seconds": null + }, + { + "order": 5, + "realized_ready_at": 80.0, + "created_at": 80.0, + "observed_ready_at": 80.0, + "assigned_at": null, + "rider": null, + "predicted_eta": null, + "pickup_arrival": null, + "picked_up_at": null, + "delivered_at": null, + "deadline": 200.0, + "partial_wait_seconds": null + }, + { + "order": 6, + "realized_ready_at": 700.0, + "created_at": null, + "observed_ready_at": null, + "assigned_at": null, + "rider": null, + "predicted_eta": null, + "pickup_arrival": null, + "picked_up_at": null, + "delivered_at": null, + "deadline": 200.0, + "partial_wait_seconds": null + } + ], + "riders": [ + { + "rider": 1, + "available": true, + "busy_seconds": 171.2403291911627, + "idle_seconds": 428.7596708088373, + "completed_distance_meters": 1111.950802335329 + }, + { + "rider": 2, + "available": true, + "busy_seconds": 142.32970269892212, + "idle_seconds": 457.6702973010779, + "completed_distance_meters": 1223.145882568862 + } + ], + "processed_events": 50, + "decision_count": 18, + "replay_sha256": "81ccde8ed1111dd4a7a0d167429b350039db8c7e3f452de03fb4e2f1ef68dea7", + "timing": { + "samples_ns": [ + 4291625, + 4424333, + 4164500, + 4108792, + 4032625, + 4212834, + 4356042, + 4470250, + 4107292, + 4562542, + 4876375 + ], + "median_ns": 4291625, + "p95_ns": 4719458.5, + "p99_ns": 4844991.7 + } + }, + { + "dispatch": { + "kind": "preparation_aware", + "idle_penalty_weight": 1.0 + }, + "summary": { + "scheduled_orders": 6, + "created_orders": 5, + "uncreated_orders": 1, + "assigned_orders": 4, + "delivered_orders": 4, + "unassigned_orders": 1, + "assigned_unfinished_orders": 0, + "outstanding_orders": 1, + "late_deliveries": 1, + "outstanding_past_deadline": 1, + "predicted_eta": { + "samples": 4, + "mean_seconds": 80.04534332610777, + "median_seconds": 86.73216258215567, + "p95_seconds": 93.35852407005987, + "p99_seconds": 93.35852407005987 + }, + "delivered_duration": { + "samples": 4, + "mean_seconds": 89.79942663061128, + "median_seconds": 92.87424790008487, + "p95_seconds": 120.09068665221554, + "p99_seconds": 120.09068665221554 + }, + "pickup_waiting": { + "samples": 4, + "mean_seconds": 9.447779617221821, + "median_seconds": 4.440245988323353, + "p95_seconds": 28.910626492240578, + "p99_seconds": 28.910626492240578 + }, + "completed_distance_meters": 2335.0966849041906, + "rider_utilization": 0.2594425132028152, + "mean_rider_idle_seconds": 444.33449207831086 + }, + "orders": [ + { + "order": 1, + "realized_ready_at": 51.14964253894716, + "created_at": 0.0, + "observed_ready_at": 51.14964253894716, + "assigned_at": 0.0, + "rider": 2, + "predicted_eta": 93.35852407005987, + "pickup_arrival": 22.239016046706585, + "picked_up_at": 51.14964253894716, + "delivered_at": 84.50816660900703, + "deadline": 200.0, + "partial_wait_seconds": null + }, + { + "order": 2, + "realized_ready_at": 40.0, + "created_at": 20.0, + "observed_ready_at": 40.0, + "assigned_at": 20.0, + "rider": 1, + "predicted_eta": 53.35852407005987, + "pickup_arrival": 31.119508023353294, + "picked_up_at": 40.0, + "delivered_at": 73.35852407005987, + "deadline": 60.0, + "partial_wait_seconds": null + }, + { + "order": 3, + "realized_ready_at": 101.20611063887397, + "created_at": 40.0, + "observed_ready_at": 101.20611063887397, + "assigned_at": 73.35852407005987, + "rider": 1, + "predicted_eta": 86.73216258215567, + "pickup_arrival": 106.71704814011974, + "picked_up_at": 106.71704814011974, + "delivered_at": 160.09068665221554, + "deadline": 200.0, + "partial_wait_seconds": null + }, + { + "order": 4, + "realized_ready_at": 100.0, + "created_at": 70.0, + "observed_ready_at": 100.0, + "assigned_at": 84.50816660900703, + "rider": 2, + "predicted_eta": 86.73216258215565, + "pickup_arrival": 117.8666906790669, + "picked_up_at": 117.8666906790669, + "delivered_at": 171.2403291911627, + "deadline": 200.0, + "partial_wait_seconds": null + }, + { + "order": 5, + "realized_ready_at": 80.0, + "created_at": 80.0, + "observed_ready_at": 80.0, + "assigned_at": null, + "rider": null, + "predicted_eta": null, + "pickup_arrival": null, + "picked_up_at": null, + "delivered_at": null, + "deadline": 200.0, + "partial_wait_seconds": null + }, + { + "order": 6, + "realized_ready_at": 700.0, + "created_at": null, + "observed_ready_at": null, + "assigned_at": null, + "rider": null, + "predicted_eta": null, + "pickup_arrival": null, + "picked_up_at": null, + "delivered_at": null, + "deadline": 200.0, + "partial_wait_seconds": null + } + ], + "riders": [ + { + "rider": 1, + "available": true, + "busy_seconds": 140.09068665221554, + "idle_seconds": 459.90931334778446, + "completed_distance_meters": 1111.950802335329 + }, + { + "rider": 2, + "available": true, + "busy_seconds": 171.2403291911627, + "idle_seconds": 428.7596708088373, + "completed_distance_meters": 1223.145882568862 + } + ], + "processed_events": 50, + "decision_count": 18, + "replay_sha256": "3f91000391d46280b94c493a5de23abb59a60f2ab7451de82007423e41eb9b7a", + "timing": { + "samples_ns": [ + 4260917, + 4175375, + 4169292, + 4016459, + 4075250, + 4161541, + 4087000, + 4184250, + 4083417, + 4092000, + 4057500 + ], + "median_ns": 4092000, + "p95_ns": 4222583.5, + "p99_ns": 4253250.3 + } + } + ] +} diff --git a/crates/roadrunner-cli/src/main.rs b/crates/roadrunner-cli/src/main.rs index a6d6737..0cc8990 100644 --- a/crates/roadrunner-cli/src/main.rs +++ b/crates/roadrunner-cli/src/main.rs @@ -106,6 +106,11 @@ fn run(arguments: Vec) -> Result<(), String> { { time_dependent_command::run(snapshot, source, destination, scenario, depart) } + [benchmark, dispatch, scenario, rest @ ..] + if benchmark == "benchmark" && dispatch == "dispatch" => + { + simulation_command::compare_arguments(scenario, rest) + } [simulate, scenario] if simulate == "simulate" => simulation_command::run(scenario, false), [simulate, scenario, json] if simulate == "simulate" && json == "--json" => { simulation_command::run(scenario, true) @@ -145,6 +150,7 @@ fn print_help() { roadrunner route alternatives --count \n \ roadrunner route traffic --scenario \n \ roadrunner route schedule --scenario --depart \n \ - roadrunner simulate [--json]" + roadrunner simulate [--json]\n\ + roadrunner benchmark dispatch [--idle-penalty-weight ] [--json]" ); } diff --git a/crates/roadrunner-cli/src/simulation_command.rs b/crates/roadrunner-cli/src/simulation_command.rs index 32370f7..c236dbe 100644 --- a/crates/roadrunner-cli/src/simulation_command.rs +++ b/crates/roadrunner-cli/src/simulation_command.rs @@ -8,7 +8,7 @@ use roadrunner_core::graph::{ AccessClass, BuilderNodeId, BuilderSegmentId, EdgeProperties, FrozenGraph, GraphBuildIdentity, GraphBuilder, GraphMetadata, GraphSnapshotId, decode_graph_artifact, }; -use roadrunner_simulation::{SimulationScenario, simulate}; +use roadrunner_simulation::{SimulationScenario, compare_strategies, simulate}; use serde::Deserialize; #[derive(Deserialize)] @@ -121,7 +121,7 @@ fn inline_graph( builder.finalize().map_err(|e| e.to_string()) } -pub(super) fn run(path: &str, json: bool) -> Result<(), String> { +fn load(path: &str) -> Result<(FrozenGraph, SimulationScenario), String> { let bytes = std::fs::read(path).map_err(|e| format!("could not read scenario {path}: {e}"))?; let document: Document = serde_json::from_slice(&bytes).map_err(|e| e.to_string())?; let graph = match document.graph { @@ -144,44 +144,91 @@ pub(super) fn run(path: &str, json: bool) -> Result<(), String> { .map_err(|e| e.to_string())? } }; - let result = simulate(&graph, &document.scenario).map_err(|e| e.to_string())?; + Ok((graph, document.scenario)) +} + +pub(super) fn compare_arguments(path: &str, arguments: &[String]) -> Result<(), String> { + let mut json = false; + let mut weight = 1.0; + let mut args = arguments.iter(); + while let Some(arg) = args.next() { + match arg.as_str() { + "--json" if !json => json = true, + "--idle-penalty-weight" => { + weight = args + .next() + .ok_or("missing idle penalty weight")? + .parse::() + .map_err(|e| e.to_string())?; + } + _ => return Err(format!("unknown comparison argument: {arg}")), + } + } + compare(path, json, weight) +} + +pub(super) fn compare(path: &str, json: bool, weight: f64) -> Result<(), String> { + let (graph, scenario) = load(path)?; + let result = compare_strategies(&graph, &scenario, weight).map_err(|e| e.to_string())?; if json { println!( "{}", serde_json::to_string_pretty(&result).map_err(|e| e.to_string())? ); } else { - let s = &result.summary; - println!( - "Orders: {} (scheduled: {}, uncreated: {})\nAssigned: {}\nDelivered: {}\nUnassigned: {}\nAssigned unfinished: {}\nLate deliveries: {}\nOutstanding past deadline: {}", - s.created_orders, - s.scheduled_orders, - s.uncreated_orders, - s.assigned_orders, - s.delivered_orders, - s.unassigned_orders, - s.assigned_unfinished_orders, - s.late_deliveries, - s.outstanding_past_deadline - ); - let duration = |v: Option| { - v.map_or_else(|| "unavailable".into(), |v| format!("{:.3} s", v.value())) - }; + for run in &result.runs { + println!("\nStrategy: {:?}", run.dispatch); + print_summary(run); + } + } + Ok(()) +} + +pub(super) fn run(path: &str, json: bool) -> Result<(), String> { + let (graph, scenario) = load(path)?; + let result = simulate(&graph, &scenario).map_err(|e| e.to_string())?; + if json { println!( - "Median predicted ETA: {}\np95 predicted ETA: {}\nMedian delivered duration: {}\np95 delivered duration: {}\nMean pickup wait: {}\nMean rider idle time: {}\nRider utilization: {}\nCompleted distance: {:.3} m\nWindow: {}..{} s\nSeed: {}", - duration(s.predicted_eta.median_seconds), - duration(s.predicted_eta.p95_seconds), - duration(s.delivered_duration.median_seconds), - duration(s.delivered_duration.p95_seconds), - duration(s.pickup_waiting.mean_seconds), - duration(s.mean_rider_idle_seconds), - s.rider_utilization - .map_or_else(|| "unavailable".into(), |v| format!("{:.3}%", v * 100.0)), - s.completed_distance_meters.value(), - result.started_at.value(), - result.ended_at.value(), - result.seed + "{}", + serde_json::to_string_pretty(&result).map_err(|e| e.to_string())? ); + } else { + print_summary(&result); } Ok(()) } + +fn print_summary(result: &roadrunner_simulation::SimulationResult) { + let s = &result.summary; + println!( + "Orders: {} (scheduled: {}, uncreated: {})\nAssigned: {}\nDelivered: {}\nUnassigned: {}\nAssigned unfinished: {}\nLate deliveries: {}\nOutstanding past deadline: {}", + s.created_orders, + s.scheduled_orders, + s.uncreated_orders, + s.assigned_orders, + s.delivered_orders, + s.unassigned_orders, + s.assigned_unfinished_orders, + s.late_deliveries, + s.outstanding_past_deadline + ); + let duration = |v: Option| { + v.map_or_else(|| "unavailable".into(), |v| format!("{:.3} s", v.value())) + }; + println!( + "Median predicted ETA: {}\np95 predicted ETA: {}\nMean delivered duration: {}\nMedian delivered duration: {}\np95 delivered duration: {}\nMean pickup wait: {}\nMean rider idle time: {}\nRider utilization: {}\nCompleted distance: {:.3} m\nWindow: {}..{} s\nSeed: {}", + duration(s.predicted_eta.median_seconds), + duration(s.predicted_eta.p95_seconds), + duration(s.delivered_duration.mean_seconds), + duration(s.delivered_duration.median_seconds), + duration(s.delivered_duration.p95_seconds), + duration(s.pickup_waiting.mean_seconds), + duration(s.mean_rider_idle_seconds), + s.rider_utilization + .map_or_else(|| "unavailable".into(), |v| format!("{:.3}%", v * 100.0)), + s.completed_distance_meters.value(), + result.started_at.value(), + result.ended_at.value(), + result.seed + ); +} diff --git a/crates/roadrunner-cli/tests/simulation_cli.rs b/crates/roadrunner-cli/tests/simulation_cli.rs index cb9e7eb..0ca2726 100644 --- a/crates/roadrunner-cli/tests/simulation_cli.rs +++ b/crates/roadrunner-cli/tests/simulation_cli.rs @@ -153,3 +153,106 @@ fn inline_coordinates_and_geometry_are_validated_at_the_json_boundary() { } ok(std::fs::remove_dir_all(directory)); } + +#[test] +fn comparison_cli_replays_four_policies_and_exposes_metrics_and_configuration() { + let path = fixture(); + let args = [ + "benchmark", + "dispatch", + ok(path.to_str().ok_or("path")), + "--idle-penalty-weight", + "2", + "--json", + ]; + let first = ok(Command::new(env!("CARGO_BIN_EXE_roadrunner")) + .args(args) + .output()); + let second = ok(Command::new(env!("CARGO_BIN_EXE_roadrunner")) + .args(args) + .output()); + assert!( + first.status.success(), + "{}", + String::from_utf8_lossy(&first.stderr) + ); + assert!(second.status.success()); + assert_eq!(first.stdout, second.stdout); + let result: serde_json::Value = ok(serde_json::from_slice(&first.stdout)); + assert_eq!(result["schema_version"], 1); + let runs = ok(result["runs"].as_array().ok_or("runs")); + assert_eq!(runs.len(), 4); + assert_eq!(runs[0]["dispatch"]["kind"], "nearest_rider"); + assert_eq!(runs[1]["dispatch"]["kind"], "lowest_pickup_eta"); + assert_eq!(runs[2]["dispatch"]["kind"], "lowest_completion_time"); + assert_eq!(runs[3]["dispatch"]["idle_penalty_weight"], 2.0); + for run in runs { + assert_eq!(run["seed"], 15); + assert_eq!(run["summary"]["created_orders"], 5); + assert_eq!(run["summary"]["uncreated_orders"], 1); + assert_eq!( + run["orders"][0]["realized_ready_at"], + runs[0]["orders"][0]["realized_ready_at"] + ); + assert!(run["summary"]["delivered_duration"]["mean_seconds"].is_number()); + } + let text = ok(Command::new(env!("CARGO_BIN_EXE_roadrunner")) + .args(["benchmark", "dispatch", ok(path.to_str().ok_or("path"))]) + .output()); + assert!(text.status.success()); + let text = ok(String::from_utf8(text.stdout)); + assert_eq!(text.matches("Strategy:").count(), 4); + assert!(text.contains("Mean delivered duration:")); + assert!(text.contains("Mean rider idle time:")); + assert!(text.contains("Unassigned:")); +} + +#[test] +fn invalid_comparison_arguments_do_not_emit_success_output() { + let path = fixture(); + for rest in [ + vec!["--idle-penalty-weight", "-1"], + vec!["--idle-penalty-weight", "NaN"], + vec!["--idle-penalty-weight"], + vec!["--unknown"], + ] { + let mut args = vec!["benchmark", "dispatch", ok(path.to_str().ok_or("path"))]; + args.extend(rest); + let output = ok(Command::new(env!("CARGO_BIN_EXE_roadrunner")) + .args(args) + .output()); + assert!(!output.status.success()); + assert_eq!(output.stdout, [] as [u8; 0]); + } +} + +#[test] +fn phase16_fixture_preserves_disconnected_and_future_orders_across_policies() { + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../data/fixtures/phase-16/paired-strategies.json"); + let output = ok(Command::new(env!("CARGO_BIN_EXE_roadrunner")) + .args([ + "benchmark", + "dispatch", + ok(path.to_str().ok_or("path")), + "--json", + ]) + .output()); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let result: serde_json::Value = ok(serde_json::from_slice(&output.stdout)); + let runs = ok(result["runs"].as_array().ok_or("runs")); + for run in runs { + assert_eq!(run["summary"]["scheduled_orders"], 14); + assert_eq!(run["summary"]["delivered_orders"], 12); + assert_eq!(run["summary"]["unassigned_orders"], 1); + assert_eq!(run["summary"]["uncreated_orders"], 1); + assert!(run["orders"][12]["rider"].is_null()); + assert!(run["orders"][13]["created_at"].is_null()); + } + assert_eq!(runs[0]["orders"][0]["rider"], 1); + assert_eq!(runs[1]["orders"][0]["rider"], 2); +} diff --git a/crates/roadrunner-dispatch/src/decision.rs b/crates/roadrunner-dispatch/src/decision.rs index 5bf7262..3efc945 100644 --- a/crates/roadrunner-dispatch/src/decision.rs +++ b/crates/roadrunner-dispatch/src/decision.rs @@ -83,7 +83,10 @@ pub struct ScoreContributions { /// Completion and idle penalty for preparation-aware scoring; absent for baseline. #[serde(skip_serializing_if = "Option::is_none")] pub preparation: Option, - /// Exact checked objective used for ranking, expressed in seconds. + /// Straight-line pickup distance used by nearest-rider ranking; absent otherwise. + #[serde(skip_serializing_if = "Option::is_none")] + pub nearest_distance: Option, + /// Checked time objective in seconds; informational road travel for nearest ranking. pub total: Seconds, } @@ -102,7 +105,7 @@ pub enum CandidateResult { /// Feasible, fully evaluated remaining plan. Feasible { /// Timing and physical metrics, independent of strategy preference. - evaluation: PlanEvaluation, + evaluation: Box, /// Structured contributions for the recorded objective. score: ScoreContributions, }, @@ -222,7 +225,13 @@ impl AssignmentDecision { pub fn explanation(&self) -> String { match &self.outcome { DispatchDecisionOutcome::Assigned(p) => { - let objective = if self.evidence.preparation.is_some() { + let objective = if self.evidence.strategy == "nearest-straight-line/v1" { + "straight-line pickup distance" + } else if self.evidence.strategy == "lowest-pickup-eta/v1" { + "pickup road travel" + } else if self.evidence.strategy == "lowest-completion-time/v1" { + "readiness-aware completion time" + } else if self.evidence.preparation.is_some() { "completion time + rider waiting penalty" } else { "pickup + delivery road travel" diff --git a/crates/roadrunner-dispatch/src/evaluation.rs b/crates/roadrunner-dispatch/src/evaluation.rs index 992cd69..7429754 100644 --- a/crates/roadrunner-dispatch/src/evaluation.rs +++ b/crates/roadrunner-dispatch/src/evaluation.rs @@ -1,5 +1,5 @@ use roadrunner_core::cost::RouteCost; -use roadrunner_core::geo::{KilometersPerHour, Meters, Seconds}; +use roadrunner_core::geo::{KilometersPerHour, Meters, Seconds, haversine_distance}; use roadrunner_core::routing::RoutingError; use serde::Serialize; use thiserror::Error; @@ -224,6 +224,7 @@ impl BaselineStrategy { scores.push(( c.rider, ScoreContributions { + nearest_distance: None, pickup_travel: c.evaluation.pickup_travel, delivery_travel: c.evaluation.delivery_travel, preparation: None, @@ -247,18 +248,14 @@ pub(crate) fn rank_scores( if scores.windows(2).any(|p| p[0].0 == p[1].0) { return Err(DispatchEvaluationError::InvalidMetric); } + let value = |s: &ScoreContributions| s.nearest_distance.map_or(s.total.value(), Meters::value); let mut ranked = scores.clone(); - ranked.sort_by(|(a, sa), (b, sb)| { - sa.total - .value() - .total_cmp(&sb.total.value()) - .then_with(|| a.cmp(b)) - }); + ranked.sort_by(|(a, sa), (b, sb)| value(sa).total_cmp(&value(sb)).then_with(|| a.cmp(b))); let selected = ranked.first().map(|(rider, _)| *rider); let reason = ranked.first().map(|(_, score)| { let mut tied_riders: Vec<_> = ranked .iter() - .filter(|(_, s)| s.total == score.total) + .filter(|(_, s)| value(s).total_cmp(&value(score)).is_eq()) .map(|(r, _)| *r) .collect(); tied_riders.sort(); @@ -474,7 +471,7 @@ pub fn basic_dispatch( id: DecisionId, policy: CandidatePolicy, ) -> Result { - dispatch_idle_order(snapshot, order, id, policy, None) + dispatch_idle_order(snapshot, order, id, policy, IdleStrategy::Basic) } /// Evaluates Phase 14 readiness-aware timing and completion plus waiting-penalty scoring. @@ -489,7 +486,79 @@ pub fn preparation_aware_dispatch( policy: CandidatePolicy, strategy: PreparationAwareStrategy, ) -> Result { - dispatch_idle_order(snapshot, order, id, policy, Some(strategy)) + dispatch_idle_order( + snapshot, + order, + id, + policy, + IdleStrategy::Preparation(strategy), + ) +} + +/// Phase 16 deterministic single-order objectives over the same feasible fleet. +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum DispatchStrategy { + /// Minimum Haversine distance to pickup among road-feasible candidates. + NearestRider, + /// Minimum traffic-aware road travel to pickup, ignoring preparation forecasts. + LowestPickupEta, + /// Readiness-aware evaluation-to-delivery duration, with no waiting penalty. + LowestCompletionTime, + /// Readiness-aware completion plus the configured waiting penalty. + PreparationAware(PreparationAwareStrategy), +} + +#[derive(Clone, Copy)] +enum IdleStrategy { + Basic, + Benchmark(DispatchStrategy), + Preparation(PreparationAwareStrategy), +} + +impl IdleStrategy { + fn preparation(self) -> Option { + match self { + Self::Preparation(s) | Self::Benchmark(DispatchStrategy::PreparationAware(s)) => { + Some(s) + } + Self::Benchmark(DispatchStrategy::LowestCompletionTime) => { + Some(PreparationAwareStrategy::default_zero()) + } + _ => None, + } + } + fn identity(self) -> &'static str { + match self { + Self::Basic => "basic-road-travel/v1", + Self::Benchmark(DispatchStrategy::NearestRider) => "nearest-straight-line/v1", + Self::Benchmark(DispatchStrategy::LowestPickupEta) => "lowest-pickup-eta/v1", + Self::Benchmark(DispatchStrategy::LowestCompletionTime) => "lowest-completion-time/v1", + Self::Preparation(_) | Self::Benchmark(DispatchStrategy::PreparationAware(_)) => { + "preparation-completion-wait/v1" + } + } + } +} + +/// Evaluates a comparison strategy with shared feasibility, routing, and commit evidence. +/// Exact objective ties choose the lowest rider identity. +/// +/// # Errors +/// Shares Basic Dispatch errors; completion policies additionally require readiness. +pub fn strategy_dispatch( + snapshot: &DispatchSnapshot<'_>, + order: OrderId, + id: DecisionId, + policy: CandidatePolicy, + strategy: DispatchStrategy, +) -> Result { + dispatch_idle_order( + snapshot, + order, + id, + policy, + IdleStrategy::Benchmark(strategy), + ) } fn dispatch_idle_order( @@ -497,7 +566,7 @@ fn dispatch_idle_order( order: OrderId, id: DecisionId, policy: CandidatePolicy, - strategy: Option, + strategy: IdleStrategy, ) -> Result { let data = snapshot.world.data(); validate_world(data)?; @@ -536,7 +605,8 @@ fn dispatch_idle_order( let plan = RiderPlan { stops: vec![Stop::Pickup(order), Stop::Dropoff(order)], }; - let preparation = strategy + let preparation_strategy = strategy.preparation(); + let preparation = preparation_strategy .map(|s| PreparationPolicyEvidence::new(data.readiness[&order], s)) .transpose()?; let (evidence, feasible) = evaluate_candidates( @@ -548,10 +618,7 @@ fn dispatch_idle_order( riders, preparation.as_ref(), )?; - let ranking = match strategy { - None => BaselineStrategy.rank(&feasible)?, - Some(s) => s.rank(&feasible)?, - }; + let ranking = rank_idle_candidates(strategy, &feasible, data, request)?; let evidence = candidate_evidence(evidence, feasible, &ranking)?; let outcome = if let Some(rider) = ranking.selected { DispatchDecisionOutcome::Assigned(AssignmentProposal { @@ -578,12 +645,7 @@ fn dispatch_idle_order( evaluated_at: snapshot.at, routing_epoch: snapshot.epoch, routing: snapshot.provenance.clone(), - strategy: if strategy.is_some() { - "preparation-completion-wait/v1" - } else { - "basic-road-travel/v1" - } - .into(), + strategy: strategy.identity().into(), preparation, feasibility_policy: "scalar-capacity-plan-custody-profile/v1".into(), eligibility_policy: "basic-idle/v1".into(), @@ -597,6 +659,39 @@ fn dispatch_idle_order( )) } +fn rank_idle_candidates( + strategy: IdleStrategy, + feasible: &[FeasibleCandidate], + data: &crate::WorldData, + request: &crate::Order, +) -> Result { + let mut ranking = match strategy.preparation() { + None => BaselineStrategy.rank(feasible)?, + Some(s) => s.rank(feasible)?, + }; + if let IdleStrategy::Benchmark(objective) = strategy { + if matches!( + objective, + DispatchStrategy::NearestRider | DispatchStrategy::LowestPickupEta + ) { + for (rider, score) in &mut ranking.scores { + match objective { + DispatchStrategy::NearestRider => { + score.nearest_distance = Some(haversine_distance( + data.riders[rider].coordinate, + request.pickup, + )); + } + DispatchStrategy::LowestPickupEta => score.total = score.pickup_travel, + _ => unreachable!(), + } + } + ranking = rank_scores(ranking.scores)?; + } + } + Ok(ranking) +} + fn candidate_evidence( mut evidence: Vec, feasible: Vec, @@ -612,7 +707,7 @@ fn candidate_evidence( evidence.push(CandidateEvidence { rider: candidate.rider, result: CandidateResult::Feasible { - evaluation: candidate.evaluation, + evaluation: Box::new(candidate.evaluation), score, }, }); diff --git a/crates/roadrunner-dispatch/src/preparation.rs b/crates/roadrunner-dispatch/src/preparation.rs index e5f0290..c8bcb07 100644 --- a/crates/roadrunner-dispatch/src/preparation.rs +++ b/crates/roadrunner-dispatch/src/preparation.rs @@ -79,6 +79,12 @@ impl Default for PreparationAwareStrategy { } impl PreparationAwareStrategy { + pub(crate) const fn default_zero() -> Self { + Self { + idle_penalty_weight: 0.0, + } + } + /// Configures penalty seconds per second waiting at pickup; zero is allowed. /// /// # Errors @@ -118,6 +124,7 @@ impl PreparationAwareStrategy { scores.push(( candidate.rider, ScoreContributions { + nearest_distance: None, pickup_travel: evaluation.pickup_travel, delivery_travel: evaluation.delivery_travel, preparation: Some(PreparationScoreContributions { diff --git a/crates/roadrunner-simulation/src/comparison.rs b/crates/roadrunner-simulation/src/comparison.rs new file mode 100644 index 0000000..118afae --- /dev/null +++ b/crates/roadrunner-simulation/src/comparison.rs @@ -0,0 +1,47 @@ +use roadrunner_core::graph::FrozenGraph; +use roadrunner_dispatch::PreparationAwareStrategy; +use serde::Serialize; + +use crate::{DispatchPolicy, SimulationError, SimulationResult, SimulationScenario, simulate}; + +/// Paired strategy results over one graph and identical exogenous scenario facts. +/// Each result includes summaries, unfinished populations, decisions, and execution evidence. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct StrategyComparison { + /// Comparison schema version, independent from simulation output schema. + pub schema_version: u32, + /// Independent runs in canonical A/B/C/D order. + pub runs: Vec, +} + +/// Runs all four Phase 16 strategies from fresh worlds with identical seed and inputs. +/// The scenario's dispatch field is replaced; all other facts remain fixed. No actual +/// readiness is supplied to ranking before its observation event. +/// +/// # Errors +/// Rejects invalid weights or any scenario/evaluation/execution failure. No partial +/// comparison is returned, and no invalid run becomes an Unassigned result. +pub fn compare_strategies( + graph: &FrozenGraph, + scenario: &SimulationScenario, + idle_penalty_weight: f64, +) -> Result { + PreparationAwareStrategy::new(idle_penalty_weight)?; + let mut runs = Vec::with_capacity(4); + for dispatch in [ + DispatchPolicy::NearestRider, + DispatchPolicy::LowestPickupEta, + DispatchPolicy::LowestCompletionTime, + DispatchPolicy::PreparationAware { + idle_penalty_weight, + }, + ] { + let mut paired = scenario.clone(); + paired.dispatch = dispatch; + runs.push(simulate(graph, &paired)?); + } + Ok(StrategyComparison { + schema_version: 1, + runs, + }) +} diff --git a/crates/roadrunner-simulation/src/engine.rs b/crates/roadrunner-simulation/src/engine.rs index f8b5d1d..c2039ae 100644 --- a/crates/roadrunner-simulation/src/engine.rs +++ b/crates/roadrunner-simulation/src/engine.rs @@ -7,9 +7,10 @@ use roadrunner_core::graph::{EdgeId, FrozenGraph, NodeId}; use roadrunner_dispatch::{ AssignmentDecision, Availability, CandidatePolicy, CandidateResult, CapacityUnits, CoreRouteProvider, DecisionId, DispatchDecisionOutcome, DispatchInstant, DispatchSnapshot, - Order, OrderId, OrderReadiness, PreparationAwareStrategy, RiderId, RiderPlan, RiderProfile, - RiderState, RouteOutcome, RouteProvider, RoutingAnchor, RoutingAnchors, RoutingEpoch, - TrafficContext, TrafficIdentity, World, WorldData, basic_dispatch, preparation_aware_dispatch, + DispatchStrategy, Order, OrderId, OrderReadiness, PreparationAwareStrategy, RiderId, RiderPlan, + RiderProfile, RiderState, RouteOutcome, RouteProvider, RoutingAnchor, RoutingAnchors, + RoutingEpoch, TrafficContext, TrafficIdentity, World, WorldData, basic_dispatch, + preparation_aware_dispatch, strategy_dispatch, }; use crate::metrics::summarize; @@ -279,7 +280,7 @@ impl<'a> Engine<'a> { }; if matches!( self.scenario.dispatch, - DispatchPolicy::PreparationAware { .. } + DispatchPolicy::PreparationAware { .. } | DispatchPolicy::LowestCompletionTime ) && readiness.expected_at.is_none() && actual_ready_at > created_at { @@ -465,6 +466,19 @@ impl<'a> Engine<'a> { DispatchPolicy::Basic => { basic_dispatch(&snapshot, order, id, CandidatePolicy::Exhaustive)? } + DispatchPolicy::NearestRider + | DispatchPolicy::LowestPickupEta + | DispatchPolicy::LowestCompletionTime => strategy_dispatch( + &snapshot, + order, + id, + CandidatePolicy::Exhaustive, + match self.scenario.dispatch { + DispatchPolicy::NearestRider => DispatchStrategy::NearestRider, + DispatchPolicy::LowestPickupEta => DispatchStrategy::LowestPickupEta, + _ => DispatchStrategy::LowestCompletionTime, + }, + )?, DispatchPolicy::PreparationAware { idle_penalty_weight, } => preparation_aware_dispatch( diff --git a/crates/roadrunner-simulation/src/lib.rs b/crates/roadrunner-simulation/src/lib.rs index b63dc32..2847451 100644 --- a/crates/roadrunner-simulation/src/lib.rs +++ b/crates/roadrunner-simulation/src/lib.rs @@ -3,11 +3,13 @@ //! No wall clock, sleeps, or live data are consulted. Graph and exogenous scenario //! inputs are fixed; static traffic changes affect only subsequently departing legs. +mod comparison; mod engine; mod event; mod metrics; mod scenario; +pub use comparison::{StrategyComparison, compare_strategies}; pub use engine::simulate; pub use event::{ExecutedLeg, RecordedEvent, SimulationEvent}; pub use metrics::{Distribution, OrderOutcome, RiderMetrics, SimulationResult, SimulationSummary}; diff --git a/crates/roadrunner-simulation/src/scenario.rs b/crates/roadrunner-simulation/src/scenario.rs index 4824c65..1f85eb7 100644 --- a/crates/roadrunner-simulation/src/scenario.rs +++ b/crates/roadrunner-simulation/src/scenario.rs @@ -19,7 +19,7 @@ pub struct SimulationScenario { /// Optional exact graph binding, checked before any event is executed. #[serde(default)] pub graph_snapshot_digest: Option, - /// Independently callable Phase 13 or Phase 14 single-order policy. + /// Independently callable single-order policy, including Phase 16 comparison objectives. pub dispatch: DispatchPolicy, /// Stable initial rider profiles and positions. pub riders: Vec, @@ -33,12 +33,18 @@ pub struct SimulationScenario { pub traffic_changes: Vec, } -/// Dispatch choice for one run; fleet strategy benchmarking is a later phase. +/// Dispatch choice for an independent deterministic run. #[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] pub enum DispatchPolicy { /// Pickup plus delivery road travel, ignoring preparation in predictions. Basic, + /// Nearest straight-line pickup distance among feasible riders. + NearestRider, + /// Lowest traffic-aware pickup road travel. + LowestPickupEta, + /// Lowest readiness-aware completion time, without a waiting penalty. + LowestCompletionTime, /// Readiness-aware completion plus weighted pickup waiting. PreparationAware { /// Finite non-negative score seconds per second of pickup waiting. diff --git a/crates/roadrunner-simulation/tests/comparison.rs b/crates/roadrunner-simulation/tests/comparison.rs new file mode 100644 index 0000000..3bb2379 --- /dev/null +++ b/crates/roadrunner-simulation/tests/comparison.rs @@ -0,0 +1,191 @@ +//! Paired objectives, shared feasibility, reproducibility, and metric populations. +#![allow(clippy::float_cmp)] +mod support; + +use roadrunner_core::cost::TrafficMultiplier; +use roadrunner_dispatch::{CandidateResult, RiderId}; +use roadrunner_simulation::{ + DispatchPolicy, RiderInput, TrafficOverride, compare_strategies, simulate, +}; +use support::{close, graph, ok, scenario, seconds}; + +fn two_riders() -> roadrunner_simulation::SimulationScenario { + let mut input = scenario(); + input.riders.push(RiderInput { + id: 2, + node: 2, + capacity: 2, + available: true, + }); + input +} + +#[test] +fn completion_ties_and_wait_penalties_select_different_riders_with_explainable_scores() { + let input = two_riders(); + let original = input.clone(); + let result = ok(compare_strategies(&graph(), &input, 1.0)); + assert_eq!(input, original); + assert_eq!(result.runs.len(), 4); + for (index, run) in result.runs.iter().enumerate() { + assert_eq!( + run.orders[0].rider, + Some(RiderId::new(if index == 3 { 2 } else { 1 })) + ); + close( + ok(run.orders[0].delivered_at.ok_or("delivered")).value(), + 55.0, + ); + close( + ok(run.summary.delivered_duration.mean_seconds.ok_or("mean")).value(), + 55.0, + ); + close( + ok(run.summary.delivered_duration.p95_seconds.ok_or("p95")).value(), + 55.0, + ); + close( + ok(run.summary.pickup_waiting.mean_seconds.ok_or("waiting")).value(), + if index == 3 { 5.0 } else { 15.0 }, + ); + assert_eq!(run.summary.delivered_orders, 1); + assert_eq!(run.summary.late_deliveries, 0); + assert_eq!(run.summary.outstanding_orders, 0); + let mut standalone = input.clone(); + standalone.dispatch = run.dispatch; + assert_eq!(*run, ok(simulate(&graph(), &standalone))); + } + let nearest = &result.runs[0].decisions[0]; + assert!( + nearest + .explanation() + .contains("straight-line pickup distance") + ); + let CandidateResult::Feasible { score, .. } = &nearest.evidence().candidates[0].result else { + panic!("feasible") + }; + close( + ok(score.nearest_distance.ok_or("meters")).value(), + 111.195_080_233_532_92, + ); + let completion = &result.runs[2].decisions[0]; + assert_eq!(completion.evidence().strategy, "lowest-completion-time/v1"); + assert!( + completion + .explanation() + .contains("readiness-aware completion time") + ); + assert_eq!( + ok(completion.evidence().preparation.ok_or("readiness")).idle_penalty_weight, + 0.0 + ); + let CandidateResult::Feasible { score, .. } = + &result.runs[3].decisions[0].evidence().candidates[1].result + else { + panic!("feasible") + }; + close(score.total.value(), 60.0); + close( + ok(score.preparation.ok_or("breakdown")) + .idle_penalty + .value(), + 5.0, + ); +} + +#[test] +fn nearest_distance_and_pickup_eta_diverge_under_traffic() { + let mut input = two_riders(); + // Obtain the directed traversal from nearest rider node 1 to pickup node 0. + let roads = graph(); + input.dispatch = DispatchPolicy::NearestRider; + let baseline = ok(simulate(&roads, &input)); + let edge = baseline.events.iter().find_map(|event| match &event.event { + roadrunner_simulation::SimulationEvent::RiderMoved { leg, .. } if leg.from.value() == 1 => { + leg.edges.first().copied() + } + _ => None, + }); + input.initial_traffic = vec![TrafficOverride { + edge_id: ok(edge.ok_or("pickup traversal")).value(), + multiplier: ok(TrafficMultiplier::new(4.0)), + }]; + let result = ok(compare_strategies(&roads, &input, 1.0)); + assert_eq!(result.runs[0].orders[0].rider, Some(RiderId::new(1))); + assert_eq!(result.runs[1].orders[0].rider, Some(RiderId::new(2))); + assert!( + result.runs[1].decisions[0] + .explanation() + .contains("pickup road travel") + ); +} + +#[test] +fn pairing_preserves_readiness_replays_and_canonical_ordering() { + let mut input = two_riders(); + input.orders[0].actual_readiness = roadrunner_simulation::ActualReadiness::SeededDelay { + min_seconds: seconds(20.0), + max_seconds: seconds(40.0), + }; + let first = ok(compare_strategies(&graph(), &input, 2.0)); + input.riders.reverse(); + let second = ok(compare_strategies(&graph(), &input, 2.0)); + assert_eq!( + ok(serde_json::to_vec(&first)), + ok(serde_json::to_vec(&second)) + ); + for run in &first.runs { + assert_eq!( + run.orders[0].realized_ready_at, + first.runs[0].orders[0].realized_ready_at + ); + assert_eq!(run.seed, input.seed); + } + let zero = ok(compare_strategies(&graph(), &input, 0.0)); + assert_eq!(zero.runs[2].orders, zero.runs[3].orders); + assert_eq!(zero.runs[2].summary, zero.runs[3].summary); +} + +#[test] +fn horizon_empty_populations_and_shared_rejections_are_retained() { + let mut input = two_riders(); + input.end_seconds = seconds(5.0); + input.orders[0].deadline_seconds = Some(seconds(1.0)); + for run in ok(compare_strategies(&graph(), &input, 1.0)).runs { + assert_eq!(run.summary.assigned_unfinished_orders, 1); + assert_eq!(run.summary.delivered_duration.samples, 0); + assert_eq!(run.summary.delivered_duration.mean_seconds, None); + assert_eq!(run.summary.outstanding_past_deadline, 1); + } + input.orders[0].demand = 3; + for run in ok(compare_strategies(&graph(), &input, 1.0)).runs { + assert_eq!(run.summary.unassigned_orders, 1); + assert_eq!(run.orders[0].rider, None); + } + input.riders.clear(); + for run in ok(compare_strategies(&graph(), &input, 1.0)).runs { + assert_eq!(run.summary.rider_utilization, None); + assert_eq!(run.summary.mean_rider_idle_seconds, None); + } + input.orders.clear(); + for run in ok(compare_strategies(&graph(), &input, 1.0)).runs { + assert_eq!(run.summary.created_orders, 0); + assert_eq!(run.summary.pickup_waiting.samples, 0); + } +} + +#[test] +fn invalid_comparisons_fail_without_partial_results_or_input_mutation() { + let mut input = two_riders(); + for weight in [-1.0, f64::NAN, f64::INFINITY] { + assert!(compare_strategies(&graph(), &input, weight).is_err()); + } + input.orders[0].expected_ready_at_seconds = None; + let original = input.clone(); + // Non-readiness policies can run, but a four-way comparison requires a forecast. + input.dispatch = DispatchPolicy::NearestRider; + assert!(simulate(&graph(), &input).is_ok()); + input.dispatch = original.dispatch; + assert!(compare_strategies(&graph(), &input, 1.0).is_err()); + assert_eq!(input, original); +} diff --git a/data/fixtures/phase-16/README.md b/data/fixtures/phase-16/README.md new file mode 100644 index 0000000..0904f52 --- /dev/null +++ b/data/fixtures/phase-16/README.md @@ -0,0 +1,18 @@ +# Phase 16 paired dispatch fixture + +`paired-strategies.json` is a synthetic five-node graph with six directed edges, +two riders, fourteen scheduled orders, and a 900-second window. The nearest rider +has a slow pickup road; the farther rider has a faster one. Mixed pickup locations, +seeded readiness delays, soft deadlines, one disconnected pickup, and an order after +the horizon exercise fleet interactions and explicit comparison populations. + +The fixed seed is 16. No live routing or delivery dataset is involved. Readiness is +sampled before dispatch, independently of the policy, and only observed at its event. +The document uses `basic` so it can also be run with `simulate`; the benchmark command +replaces this field with the four comparison policies. + +```bash +cargo run -p roadrunner-cli -- benchmark dispatch data/fixtures/phase-16/paired-strategies.json --json +``` + +See [benchmark methodology](../../../docs/dispatch-strategy-benchmarks.md). diff --git a/data/fixtures/phase-16/paired-strategies.json b/data/fixtures/phase-16/paired-strategies.json new file mode 100644 index 0000000..4ed5407 --- /dev/null +++ b/data/fixtures/phase-16/paired-strategies.json @@ -0,0 +1,287 @@ +{ + "graph": { + "kind": "inline", + "profile": "simulation-motorcycle/v1", + "nodes": [ + { + "id": 0, + "coordinate": { + "latitude_e7": 0, + "longitude_e7": 0 + } + }, + { + "id": 1, + "coordinate": { + "latitude_e7": 0, + "longitude_e7": 10000 + } + }, + { + "id": 2, + "coordinate": { + "latitude_e7": 0, + "longitude_e7": 20000 + } + }, + { + "id": 3, + "coordinate": { + "latitude_e7": 0, + "longitude_e7": 30000 + } + }, + { + "id": 4, + "coordinate": { + "latitude_e7": 0, + "longitude_e7": 40000 + } + } + ], + "roads": [ + { + "id": 0, + "from": 0, + "to": 1, + "speed_kph": 6, + "bidirectional": true + }, + { + "id": 1, + "from": 0, + "to": 2, + "speed_kph": 36, + "bidirectional": true + }, + { + "id": 2, + "from": 0, + "to": 3, + "speed_kph": 36, + "bidirectional": true + } + ] + }, + "scenario": { + "schema_version": 1, + "seed": 16, + "start_seconds": 0, + "end_seconds": 900, + "routing_epoch_seconds": 0, + "dispatch": { + "kind": "basic" + }, + "riders": [ + { + "id": 1, + "node": 1, + "capacity": 2, + "available": true + }, + { + "id": 2, + "node": 2, + "capacity": 2, + "available": true + } + ], + "orders": [ + { + "id": 1, + "pickup_node": 0, + "dropoff_node": 3, + "created_at_seconds": 0, + "expected_ready_at_seconds": 80, + "actual_readiness": { + "kind": "seeded_delay", + "min_seconds": 70, + "max_seconds": 90 + }, + "deadline_seconds": 100, + "demand": 1 + }, + { + "id": 2, + "pickup_node": 1, + "dropoff_node": 3, + "created_at_seconds": 30, + "expected_ready_at_seconds": 50, + "actual_readiness": { + "kind": "seeded_delay", + "min_seconds": 10, + "max_seconds": 30 + }, + "deadline_seconds": 130, + "demand": 1 + }, + { + "id": 3, + "pickup_node": 0, + "dropoff_node": 3, + "created_at_seconds": 60, + "expected_ready_at_seconds": 140, + "actual_readiness": { + "kind": "seeded_delay", + "min_seconds": 70, + "max_seconds": 90 + }, + "deadline_seconds": 160, + "demand": 1 + }, + { + "id": 4, + "pickup_node": 0, + "dropoff_node": 3, + "created_at_seconds": 90, + "expected_ready_at_seconds": 170, + "actual_readiness": { + "kind": "seeded_delay", + "min_seconds": 70, + "max_seconds": 90 + }, + "deadline_seconds": 190, + "demand": 1 + }, + { + "id": 5, + "pickup_node": 1, + "dropoff_node": 3, + "created_at_seconds": 120, + "expected_ready_at_seconds": 140, + "actual_readiness": { + "kind": "seeded_delay", + "min_seconds": 10, + "max_seconds": 30 + }, + "deadline_seconds": 220, + "demand": 1 + }, + { + "id": 6, + "pickup_node": 0, + "dropoff_node": 3, + "created_at_seconds": 150, + "expected_ready_at_seconds": 230, + "actual_readiness": { + "kind": "seeded_delay", + "min_seconds": 70, + "max_seconds": 90 + }, + "deadline_seconds": 250, + "demand": 1 + }, + { + "id": 7, + "pickup_node": 0, + "dropoff_node": 3, + "created_at_seconds": 180, + "expected_ready_at_seconds": 260, + "actual_readiness": { + "kind": "seeded_delay", + "min_seconds": 70, + "max_seconds": 90 + }, + "deadline_seconds": 280, + "demand": 1 + }, + { + "id": 8, + "pickup_node": 1, + "dropoff_node": 3, + "created_at_seconds": 210, + "expected_ready_at_seconds": 230, + "actual_readiness": { + "kind": "seeded_delay", + "min_seconds": 10, + "max_seconds": 30 + }, + "deadline_seconds": 310, + "demand": 1 + }, + { + "id": 9, + "pickup_node": 0, + "dropoff_node": 3, + "created_at_seconds": 240, + "expected_ready_at_seconds": 320, + "actual_readiness": { + "kind": "seeded_delay", + "min_seconds": 70, + "max_seconds": 90 + }, + "deadline_seconds": 340, + "demand": 1 + }, + { + "id": 10, + "pickup_node": 0, + "dropoff_node": 3, + "created_at_seconds": 270, + "expected_ready_at_seconds": 350, + "actual_readiness": { + "kind": "seeded_delay", + "min_seconds": 70, + "max_seconds": 90 + }, + "deadline_seconds": 370, + "demand": 1 + }, + { + "id": 11, + "pickup_node": 1, + "dropoff_node": 3, + "created_at_seconds": 300, + "expected_ready_at_seconds": 320, + "actual_readiness": { + "kind": "seeded_delay", + "min_seconds": 10, + "max_seconds": 30 + }, + "deadline_seconds": 400, + "demand": 1 + }, + { + "id": 12, + "pickup_node": 0, + "dropoff_node": 3, + "created_at_seconds": 330, + "expected_ready_at_seconds": 410, + "actual_readiness": { + "kind": "seeded_delay", + "min_seconds": 70, + "max_seconds": 90 + }, + "deadline_seconds": 430, + "demand": 1 + }, + { + "id": 13, + "pickup_node": 4, + "dropoff_node": 0, + "created_at_seconds": 100, + "expected_ready_at_seconds": 100, + "actual_readiness": { + "kind": "fixed", + "at_seconds": 100 + }, + "deadline_seconds": 200, + "demand": 1 + }, + { + "id": 14, + "pickup_node": 0, + "dropoff_node": 3, + "created_at_seconds": 950, + "expected_ready_at_seconds": 950, + "actual_readiness": { + "kind": "fixed", + "at_seconds": 950 + }, + "demand": 1 + } + ], + "initial_traffic": [], + "traffic_changes": [] + } +} diff --git a/docs/architecture.md b/docs/architecture.md index 0da201d..7cc492a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,6 +1,6 @@ # Roadrunner Architecture -Status: Accepted through Phase 15 deterministic simulation +Status: Accepted through Phase 16 dispatch strategy benchmarking Last updated: 2026-10-04 ## 1. Architectural intent @@ -426,3 +426,13 @@ core or dispatch. Graph, traffic, time, readiness, and scalar-profile inputs are before execution. See [simulation usage and contracts](simulation.md), [ADR 0015](adr/0015-deterministic-simulation-runtime.md), and [completion evidence](phase-15-completion.md). + +## 17. Phase 16 paired strategy comparison + +The dispatch module exposes deterministic nearest-rider, pickup-ETA, readiness-aware +completion, and preparation-aware objectives over its shared idle-order evaluator. +Simulation owns the paired orchestration: four fresh worlds reuse identical graph and +exogenous facts. The CLI loads a scenario once for `benchmark dispatch`; external Python +collection records wall-clock timings and reproducibility metadata. There is no host clock +in the Rust simulation results. See [strategy contracts](dispatch-strategy-benchmarks.md) +and [completion evidence](phase-16-completion.md). diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 87c3657..8b48958 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -139,3 +139,13 @@ python3 scripts/collect_simulation_benchmark.py --output /tmp/simulation-replay. The measured median process time is below the configured logical window, demonstrating execution without wall-clock pacing for this fixture. No fleet strategy or scalability conclusion follows. See [simulation contracts and metric populations](simulation.md). + +## Phase 16 dispatch strategy benchmarking + +`roadrunner benchmark dispatch [--idle-penalty-weight ] [--json]` +replays identical inputs through four assignment policies and computes delivery durations, +waiting/idle time, distance, lateness, utilization, and explicit unfinished populations. +`scripts/collect_dispatch_strategy_benchmark.py` verifies paired/standalone replay and +records per-policy process timings with complete reproducibility metadata. See +[methodology](dispatch-strategy-benchmarks.md) and +[completion evidence](phase-16-completion.md). diff --git a/docs/dispatch-strategy-benchmarks.md b/docs/dispatch-strategy-benchmarks.md new file mode 100644 index 0000000..d593949 --- /dev/null +++ b/docs/dispatch-strategy-benchmarks.md @@ -0,0 +1,96 @@ +# Dispatch strategy benchmarking + +Phase 16 compares fleet outcomes by replaying identical exogenous inputs through four +independent simulation worlds. Run: + +```bash +cargo run -p roadrunner-cli -- benchmark dispatch data/fixtures/phase-16/paired-strategies.json +cargo run -p roadrunner-cli -- benchmark dispatch data/fixtures/phase-16/paired-strategies.json --json +cargo run -p roadrunner-cli -- benchmark dispatch data/fixtures/phase-16/paired-strategies.json --idle-penalty-weight 2 --json +``` + +The comparison accepts the same inline/artifact scenario envelope as `simulate`. +`compare_strategies(graph, scenario, idle_penalty_weight)` is the library entry point. +The source `dispatch` field is replaced for each run; graph, rider profiles/positions, +orders, readiness forecasts, realized seeded readiness, traffic changes, start, horizon, +and routing epoch remain identical. Each run starts with a fresh world. A failure in any +run fails the whole comparison, without printing partial success output. + +## Objectives and shared constraints + +| Strategy | JSON policy | Minimized objective | +| --- | --- | --- | +| A: nearest rider | `nearest_rider` | Haversine rider-to-pickup distance in meters | +| B: lowest pickup ETA | `lowest_pickup_eta` | Traffic-aware rider-to-pickup road travel in seconds | +| C: lowest total completion time | `lowest_completion_time` | Readiness-aware evaluation-to-delivery duration in seconds | +| D: preparation-aware | `preparation_aware` | C plus `idle_penalty_weight × predicted pickup waiting` | + +All policies consider the complete eligible idle fleet and reject capacity, profile, +custody, invalid plan, and NoRoute candidates through the shared dispatch evaluator. +Nearest rider means nearest **feasible** rider, so an unreachable rider does not block +reachable delivery. Both road legs use Roadrunner Dijkstra. Exact objective ties choose +the lowest RiderId. Deadlines remain soft observations, not ranking constraints. + +A and B retain the baseline's zero-wait forecast, while execution still waits for +actual readiness. C uses the preparation evaluator with weight zero; it is distinct from +Phase 13 `basic`, which minimizes pickup plus delivery road travel. D defaults to weight +one, configured independently of the source scenario. C and D require a forecast when +readiness will only be observed later. No policy sees realized readiness before the +actual event. Weight zero makes C and D produce identical physical outcomes; their +strategy identities remain distinct. + +For a single order with a static overlay, B and the Phase 13 basic objective commonly +select the same rider because pickup-to-dropoff travel is independent of the rider. +C can tie multiple riders arriving before readiness; D can then prefer a later arrival +to reduce pickup waiting. Equal routes or fleet outcomes are valid measured results. +There is no requirement that each policy beat another policy. + +Every decision records its versioned strategy, candidate coverage, rejection/evaluation, +objective contributions, selected proposal, and exact tie reason. Nearest decisions +record `score.nearest_distance` in meters as their ranking objective; `score.total` +retains informational road-travel seconds for that policy. Other policies rank by +`score.total` seconds. Completion/preparation decisions retain readiness source and +waiting breakdown. Candidate evaluation is boxed in memory; its JSON shape is unchanged. + +## Comparison populations + +JSON schema 1 has `runs` in A/B/C/D order, each a complete simulation artifact. Readable +output includes each policy's mean/median/p95 observed delivery duration, mean pickup +waiting, mean rider idle seconds, completed distance, late deliveries, utilization, +and created/unassigned/unfinished populations. JSON also records p99 and sample counts. +These are computed from execution, rather than strategy predictions. + +Use [simulation metric definitions](simulation.md#results-and-metric-populations) when +interpreting comparisons. Delivery durations run from creation to delivered dropoff; +unassigned or unfinished orders have no fabricated duration and remain explicit. +Pickup waiting covers completed pickups; partial horizon waits remain per-order facts. +Distance includes only arrived whole legs. Rider utilization is responsibility occupancy +(including pickup waiting), divided by the full initially available rider-seconds window. +Higher occupancy alone is not evidence of a better strategy. Compare delivered counts +and outstanding lateness alongside completed-duration percentiles to avoid censoring bias. + +## Reproducible measurement + +```bash +cargo +1.99.0 build -p roadrunner-cli --release --locked +python3 scripts/collect_dispatch_strategy_benchmark.py --runs 11 --output /tmp/phase16.json +``` + +The collector first verifies byte-identical paired comparisons. It then runs each policy +independently, verifies equivalence to its comparison result, and checks byte-identical +replay for one excluded warmup and every measured run. Relative graph artifact paths +are bound to the source directory before temporary variant documents are written. + +The generated artifact records hardware/OS, compiler and binary hash, source dataset/hash, +graph digest/size, generator/seed, routing/coverage/tie configuration, penalty weight, +observation window, number of runs, replay hashes, raw timings and median/p95/p99, +computed summaries, and canonical order/rider outcomes. Timing includes process startup, +loading/validation, simulation, JSON serialization, and capture. Timing percentiles use +linear interpolation; simulated outcome percentiles use nearest rank. Memory is unmeasured. +Measurements should run without simultaneous workspace checks. The recorded build command +assumes the release binary was built using the command above. + +The [fixture](../data/fixtures/phase-16/paired-strategies.json) is deliberately small and +synthetic. See [Phase 16 completion](phase-16-completion.md) for measured evidence. It +establishes reproducibility and different objective behavior, not production scalability +or a universal strategy ranking. Multi-order routing, VRP and re-dispatch remain later phases. diff --git a/docs/dispatch.md b/docs/dispatch.md index 3c3e598..3d08972 100644 --- a/docs/dispatch.md +++ b/docs/dispatch.md @@ -190,3 +190,8 @@ The [general simulation engine](simulation.md) is implemented in Phase 15. Fleet strategy benchmarking, multi-order insertion, HTTP dispatch endpoints, persistence, and UI remain later phases. The rider lookup benchmark is preserved; these phases make no new dispatch latency or scalability claim. + +## Phase 16 strategy comparison + +The shared evaluator now supports nearest feasible rider, lowest pickup ETA, readiness-aware +completion time, and preparation-aware ranking. See [objectives and comparison contracts](dispatch-strategy-benchmarks.md). diff --git a/docs/phase-16-completion.md b/docs/phase-16-completion.md new file mode 100644 index 0000000..0dc5178 --- /dev/null +++ b/docs/phase-16-completion.md @@ -0,0 +1,100 @@ +# Phase 16 completion + +Date: 2026-10-06 +Status: COMPLETE — Dispatch Strategy Benchmarking + +Phase 16 adds deterministic nearest-feasible-rider, lowest-pickup-ETA, +readiness-aware lowest-completion-time, and preparation-aware assignment strategies. +They share candidate coverage, road evaluation, hard feasibility, atomic assignment, +and execution. `compare_strategies` runs four fresh worlds over identical exogenous +facts. `roadrunner benchmark dispatch [--idle-penalty-weight ] +[--json]` exposes readable summaries and complete deterministic JSON evidence. +See [methodology and contracts](dispatch-strategy-benchmarks.md). + +## Requirement review + +| Requirement | Evidence | +| --- | --- | +| Nearest rider baseline | Haversine distance among shared road-feasible candidates, meter-valued score evidence | +| Lowest pickup ETA | Traffic-aware pickup travel ranking with objective-specific explanation | +| Lowest total completion time | Readiness-aware elapsed delivery completion, waiting penalty zero | +| Preparation-aware assignment | Completion plus configurable non-negative predicted waiting penalty | +| Average / median / p95 delivery time | Observed creation-to-dropoff distribution with sample count and p99 | +| Rider idle time | Available-unassigned rider seconds plus separate observed pickup waiting distribution | +| Distance traveled | Sum of completed road-leg distances; partial travel not fabricated | +| Late orders | Completed deadline misses and outstanding past-deadline populations | +| Rider utilization | Clipped responsibility intervals / full available rider-seconds | +| Automatically generated results | Paired CLI/library artifacts and external replay-checking benchmark collector | +| Reproducible measurements | Hardware, dataset/hash, graph size/digest, algorithms/configuration, compiler/binary hash, warmup/runs, raw timing samples, median/p95/p99 | + +## Measured synthetic comparisons + +The values below are derived from the committed collector artifacts, rounded for display. +Each policy has one excluded warmup and eleven measured byte-identical standalone runs. +The collector also verifies paired replay and standalone equivalence. Measurements ran +without simultaneous workspace validation. Timing includes process/loading/JSON overhead. + +| Dataset | Policy | Delivered | Mean delivery s | Median s | p95 s | Mean pickup wait s | Distance m | Late | Utilization | +| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| Slow pickup road | nearest_rider | 12 | 242.482 | 223.189 | 452.768 | 1.128 | 8562.021 | 12 | 76.117% | +| Slow pickup road | lowest_pickup_eta | 12 | 207.024 | 184.506 | 414.064 | 5.800 | 8339.631 | 12 | 71.819% | +| Slow pickup road | lowest_completion_time | 12 | 242.482 | 223.189 | 452.768 | 1.128 | 8562.021 | 12 | 76.117% | +| Slow pickup road | preparation_aware | 12 | 242.482 | 223.189 | 452.768 | 1.128 | 8562.021 | 12 | 76.117% | +| Preparation delay + traffic | nearest_rider | 4 | 90.919 | 92.874 | 122.330 | 10.008 | 2335.097 | 1 | 26.131% | +| Preparation delay + traffic | lowest_pickup_eta | 4 | 90.919 | 92.874 | 122.330 | 10.008 | 2335.097 | 1 | 26.131% | +| Preparation delay + traffic | lowest_completion_time | 4 | 90.919 | 92.874 | 122.330 | 10.008 | 2335.097 | 1 | 26.131% | +| Preparation delay + traffic | preparation_aware | 4 | 89.799 | 92.874 | 120.091 | 9.448 | 2335.097 | 1 | 25.944% | + +The slow-road scenario contains five nodes, six directed edges, two riders, fourteen +scheduled orders and a 900-second window. All policies deliver twelve orders; one +created order has a disconnected pickup and one order is created after the horizon. +Lowest pickup ETA has lower completed delivery durations in this scenario. A/C/D have +identical observed fleet outcomes, including first-order rider selection. A distance +objective and a waiting penalty can both favor the geographically closer rider when +that rider arrives later over a slower road. + +The preparation/traffic scenario reuses the Phase 15 fixture: five nodes, six edges, +two riders, six scheduled orders, and a 600-second horizon. All policies deliver four, +retain one capacity-infeasible order, and retain one uncreated order. Preparation-aware +assignment chooses a different first rider and reduces mean pickup waiting and delivery +duration in this specific replay. All policies have one late completed delivery. + +Neither scenario establishes a universally best strategy or production scalability. +Responsibility utilization includes waiting; completed-duration percentiles exclude +unfinished work. These populations are explicit in the artifacts. + +Artifacts: + +- [Slow-road comparison](../benchmarks/results/2026-10-06-phase-16-dispatch-strategies.json) +- [Preparation/traffic comparison](../benchmarks/results/2026-10-06-phase-16-preparation-strategies.json) + +Reproduce: + +```bash +cargo +1.99.0 build -p roadrunner-cli --release --locked +python3 scripts/collect_dispatch_strategy_benchmark.py --output /tmp/phase16-slow-road.json +python3 scripts/collect_dispatch_strategy_benchmark.py --scenario data/fixtures/phase-15/seeded-deliveries.json --output /tmp/phase16-preparation.json +``` + +## Correctness and validation + +The new comparison tests verify traffic-induced proximity/ETA disagreement, readiness +completion ties, waiting-penalty selection, meter/time score evidence, objective-specific +explanations, shared standalone equivalence, canonical exact replay, equal realized +readiness across policies, zero-penalty equivalence, empty fleets/populations, capacity +rejections, unfinished horizons, outstanding lateness, invalid weights/missing forecasts, +and immutable caller inputs. CLI tests verify A/B/C/D ordering, configured weight, +byte-identical JSON, readable metrics, invalid-argument errors without success output, +and disconnected/future populations in the Phase 16 fixture. + +| Validation | Result | +| --- | --- | +| `cargo fmt --all -- --check` | PASS | +| `cargo +1.99.0 clippy --workspace --all-targets --all-features --locked -- -D warnings` | PASS | +| `cargo +1.99.0 test --workspace --all-features --locked` | PASS — 138 tests and doc tests | +| Release CLI build | PASS | +| Both benchmark collectors | PASS — eleven replay-checked runs per strategy per dataset | +| Documentation links / `git diff --check` | PASS | + +No unsafe Rust or new dependencies are introduced. Multi-order insertion, VRP, +dynamic re-dispatch, live inputs, UI, HTTP, persistence and ML remain later work. diff --git a/docs/simulation.md b/docs/simulation.md index de1b0d3..2accf04 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -41,7 +41,7 @@ is a complete runnable example. | `start_seconds`, `end_seconds` | Inclusive observation window; end is no earlier than start | | `routing_epoch_seconds` | Dispatch instant corresponding to routing second zero; no later than start | | `graph_snapshot_digest` | Optional for inline graphs, mandatory for artifact loading | -| `dispatch` | `{ "kind": "basic" }` or `{ "kind": "preparation_aware", "idle_penalty_weight": 1.0 }` | +| `dispatch` | `basic`, `nearest_rider`, `lowest_pickup_eta`, `lowest_completion_time`, or `preparation_aware` (with `idle_penalty_weight`), as tagged `kind` objects | | `riders` | Initial `id`, graph `node`, scalar `capacity`, and operational `available` boolean | | `orders` | Scheduled request facts and independently generated actual readiness | | `initial_traffic` | Complete list of directed `edge_id`/`multiplier` overrides; defaults to normal | @@ -68,7 +68,7 @@ Delay bounds must be ordered. The versioned `splitmix64-upper53/v1` generator pr seeded orders in canonical identity order before dispatch. Its discrete uniform source is in `[0, 1)`; rounding can place the resulting delay on an endpoint. Fixed actual readiness may precede creation for stock already ready. The observation is received at -creation and retains its original ready timestamp. Preparation-aware orders need a +creation and retains its original ready timestamp. Readiness-aware completion and preparation-aware orders need a forecast when the actual ready fact will only become known later. Basic Dispatch retains its zero-wait predicted baseline, but execution still waits for actual readiness. @@ -174,7 +174,7 @@ hash are recorded. It is not a fleet strategy comparison or a scalability benchm ## Phase boundary -Phase 16 owns strategy benchmarking. Multi-order plans, initially busy riders, dynamic +Phase 16 implements [paired strategy benchmarking](dispatch-strategy-benchmarks.md). Multi-order plans, initially busy riders, dynamic availability/cancellation/re-dispatch, continuous location interpolation, per-edge traffic changes during movement, live data, UI replay controls, persistence, and production-scale claims remain later work. This engine deliberately shares current scalar idle-rider diff --git a/docs/spec-v1.md b/docs/spec-v1.md index efc9376..b0622ac 100644 --- a/docs/spec-v1.md +++ b/docs/spec-v1.md @@ -1,6 +1,6 @@ # Roadrunner v0 Specification -Status: Accepted; deterministic simulation implemented through Phase 15 +Status: Accepted; dispatch strategy benchmarking implemented through Phase 16 Last updated: 2026-10-04 ## 1. Purpose @@ -257,6 +257,13 @@ Future/uncreated and assigned-but-unfinished work remain explicit at the inclusi Empty populations and zero denominators are unavailable. See [simulation.md](simulation.md) and [ADR 0015](adr/0015-deterministic-simulation-runtime.md). +Phase 16 replays the same scenario through nearest-feasible-rider, lowest-pickup-ETA, +readiness-aware lowest-completion-time, and preparation-aware assignment policies. +All use exhaustive eligible candidate evaluation, shared feasibility and atomic commit. +Comparison artifacts preserve full execution evidence and horizon populations, while +external timing artifacts record hardware, dataset/configuration, runs and percentiles. +See [dispatch strategy benchmarking](dispatch-strategy-benchmarks.md). + ### 5.8 HTTP route API The v0 server exposes: diff --git a/scripts/collect_dispatch_strategy_benchmark.py b/scripts/collect_dispatch_strategy_benchmark.py new file mode 100644 index 0000000..3fdf5c2 --- /dev/null +++ b/scripts/collect_dispatch_strategy_benchmark.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +"""Collect paired Phase 16 outcomes and per-strategy process timings, checking exact replay.""" +import argparse +import copy +import hashlib +import json +import statistics +import subprocess +import tempfile +import time +from datetime import date +from pathlib import Path + +from collect_simulation_benchmark import hardware, percentile + + +def digest(data): + return hashlib.sha256(data).hexdigest() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--binary", type=Path, default=Path("target/release/roadrunner")) + parser.add_argument("--scenario", type=Path, default=Path("data/fixtures/phase-16/paired-strategies.json")) + parser.add_argument("--runs", type=int, default=11) + parser.add_argument("--idle-penalty-weight", type=float, default=1.0) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + if args.runs < 2: + parser.error("at least two measured runs are required") + binary = str(args.binary.resolve()) + scenario_path = args.scenario.resolve() + source = scenario_path.read_bytes() + document = json.loads(source) + command = [binary, "benchmark", "dispatch", str(scenario_path), "--idle-penalty-weight", str(args.idle_penalty_weight), "--json"] + comparison_bytes = subprocess.check_output(command) + if subprocess.check_output(command) != comparison_bytes: + raise ValueError("comparison did not replay byte-for-byte") + comparison = json.loads(comparison_bytes) + # Bind artifacts before moving variant documents into a temporary directory. + graph_artifact = None + if document["graph"]["kind"] == "artifact": + artifact_path = scenario_path.parent / document["graph"]["path"] + graph_artifact = {"path": str(artifact_path.resolve()), "sha256": digest(artifact_path.read_bytes())} + document["graph"]["path"] = str(artifact_path.resolve()) + records = [] + with tempfile.TemporaryDirectory(prefix="roadrunner-phase16-") as directory: + for expected in comparison["runs"]: + variant = copy.deepcopy(document) + variant["scenario"]["dispatch"] = expected["dispatch"] + path = Path(directory) / "scenario.json" + path.write_text(json.dumps(variant)) + run_command = [binary, "simulate", str(path), "--json"] + reference = subprocess.check_output(run_command) # One excluded warmup per policy. + if json.loads(reference) != expected: + raise ValueError("standalone policy result differs from paired comparison") + samples = [] + for _ in range(args.runs): + began = time.perf_counter_ns() + output = subprocess.check_output(run_command) + samples.append(time.perf_counter_ns() - began) + if output != reference: + raise ValueError("standalone simulation did not replay byte-for-byte") + ordered = sorted(samples) + records.append({ + "dispatch": expected["dispatch"], + "summary": expected["summary"], + "orders": expected["orders"], + "riders": expected["riders"], + "processed_events": len(expected["events"]), + "decision_count": len(expected["decisions"]), + "replay_sha256": digest(reference), + "timing": {"samples_ns": samples, "median_ns": statistics.median(ordered), + "p95_ns": percentile(ordered, .95), "p99_ns": percentile(ordered, .99)}, + }) + first = comparison["runs"][0] + record = { + "schema_version": 1, + "date": date.today().isoformat(), + "purpose": "Paired synthetic dispatch strategy comparison; no production scalability claim", + "hardware": hardware(), + "dataset": {"path": str(args.scenario), "sha256": digest(source), + "kind": "synthetic" if graph_artifact is None else "scenario on graph artifact; inspect source provenance", + "graph_artifact": graph_artifact, "seed": first["seed"], "randomness": first["randomness"], + "graph_digest": first["graph_snapshot_digest"], "graph_nodes": first["graph_nodes"], + "graph_directed_edges": first["graph_edges"], "riders": len(first["riders"]), + "scheduled_orders": first["summary"]["scheduled_orders"]}, + "algorithm": {"routing": "Roadrunner Dijkstra with static traffic overlays", + "events": "BinaryHeap ordered by logical time and insertion sequence", + "coverage": "exhaustive eligible road-feasible riders", "tie_break": "lowest RiderId"}, + "configuration": {"build": "cargo +1.99.0 build -p roadrunner-cli --release --locked", + "rustc": subprocess.check_output(["rustc", "+1.99.0", "--version"], text=True).strip(), + "binary_sha256": digest(Path(binary).read_bytes()), "profile": "release", + "comparison_command": command, "idle_penalty_weight": args.idle_penalty_weight, + "warmup_runs_per_strategy": 1, "measured_runs_per_strategy": args.runs, + "start_seconds": first["started_at"], "end_seconds": first["ended_at"], + "includes": "process startup, graph/scenario loading and validation, simulation, JSON serialization and capture", + "timing_percentiles": "linear interpolation of ordered elapsed process samples", + "outcome_percentiles": "nearest rank; even median midpoint", + "memory": "not measured", "concurrent_workspace_validation": False}, + "comparison_replay_sha256": digest(comparison_bytes), + "strategies": records, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(record, indent=2) + "\n") + + +if __name__ == "__main__": + main()