feat(ingest): add HTTP gateway and recoverable Rust producers

.github/workflows/rust-test.yml

@@ -166,6 +166,11 @@ jobs:
         run: |
           cargo clippy -p lance-context-ingest --features materializer --all-targets -- -D warnings
           cargo test -p lance-context-ingest --features materializer --lib
+      - name: HTTP ingestion and recoverable producer journal
+        run: |
+          cargo clippy -p lance-context-ingest --features http --all-targets -- -D warnings
+          cargo clippy -p lance-context-ingest --features http-client,materializer --all-targets -- -D warnings
+          cargo test -p lance-context-ingest --features http-client,materializer --lib
       - name: Start etcd for HA scheduler test
         run: |
           ETCD_VERSION=3.7.0

crates/lance-context-ingest/Cargo.toml

@@ -7,9 +7,14 @@ publish = false
 description = "Durable rollout ingress over the independent lance-context WAL"
 
 [features]
+http = ["dep:axum", "dep:http-body-util"]
+http-client = ["http", "dep:reqwest", "lance-context-wal/client-journal"]
 materializer = ["lance-context-wal/lance-integration", "dep:lance", "dep:lance-table", "dep:object_store", "dep:futures"]
 
 [dependencies]
+axum = { version = "0.8", optional = true, default-features = false, features = ["json", "http1", "tokio"] }
+http-body-util = { version = "0.1", optional = true }
+reqwest = { version = "0.12", optional = true, default-features = false, features = ["json", "rustls-tls"] }
 lance = { version = "13.0.0", optional = true }
 lance-table = { version = "13.0.0", optional = true }
 object_store = { version = "0.14.2", optional = true }
@@ -28,6 +33,7 @@ tokio = { version = "1", features = ["rt", "sync", "time"] }
 uuid = { version = "1.20.0", features = ["v4", "serde"] }
 
 [dev-dependencies]
+reqwest = { version = "0.12", default-features = false, features = ["json", "stream"] }
 arrow-ipc = "58"
 lance = "13.0.0"
 lance-table = "13.0.0"