diff --git a/Tests/Gossipsub/Cache.lean b/Tests/Gossipsub/Cache.lean new file mode 100644 index 00000000..0423a18e --- /dev/null +++ b/Tests/Gossipsub/Cache.lean @@ -0,0 +1,181 @@ +import Tests.Support +import Tests.Gossipsub.Identity +import Libp2p.Protocols.Gossipsub.Cache + +/-! +# Gossipsub suite: Cache + +The checks of `Libp2p.Protocols.Gossipsub.Cache`. Each check is one step +of a fixed transcript: the inputs are literal times and ids, and the +expected result stands beside the call. The suite names the spec +boundaries: exactly `t + ttl`, exactly `until`, exactly the end of a +window, and exactly the capacity of the seen cache. + +The windows of the message cache checks are the spec defaults: five +history windows and three gossip windows. The seen cache uses capacity +three and a TTL of one hundred milliseconds, so each boundary is a round +number. +-/ + +set_option autoImplicit false + +namespace Tests.Gossipsub.Cache + +open Libp2p.Protocols.Gossipsub +open Libp2p.Protocols.Gossipsub.Cache + +/-- The window parameters of the transcript: the spec defaults of five +history windows and three gossip windows. -/ +def window5 : WindowParams := + { historyLength := 5, historyGossip := 3, valid := by decide } + +/-- The seen cache parameters of the transcript: three entries and a TTL +of one hundred milliseconds. -/ +def seen3 : SeenParams := + { capacity := 3, ttl := 100, valid := by decide } + +/-- A message id from one number. -/ +def mid (n : Nat) : MessageId := { bytes := [n] } + +/-- One topic of the transcript. -/ +def topicA : TopicId := { name := "a" } + +/-- A second topic of the transcript. -/ +def topicB : TopicId := { name := "b" } + +/-- A message with a one-byte payload. -/ +def msg (t : TopicId) (n : Nat) : Message := + { «from» := none, data := some [n], seqno := none, topic := t, signature := none, key := none } + +/-- A peer from one number. -/ +def peer (n : Nat) : Peer := Identity.peer n + +/-- A promise with explicit fields, for the expected lists. -/ +def promiseOf (p : Peer) (i : MessageId) (d : Millis) : IwantPromise := + { peer := p, id := i, deadline := d } + +/-- A backoff entry with explicit fields, for the expected lists. -/ +def backoffOf (p : Peer) (t : TopicId) (u : Millis) : BackoffEntry := + { peer := p, topic := t, «until» := u } + +/-- Run the checks of this module. -/ +def run (r : Runner) : IO Unit := do + r.section "gossipsub Cache" "vendor/specs/pubsub" + + -- Message cache: one message, then shifts. + let c0 : MessageCache := MessageCache.empty window5 + let c1 : MessageCache := c0.put (mid 1) (msg topicA 1) + r.check "mcache put then get returns the message" + (decide (MessageCache.get c1 (mid 1) = some (msg topicA 1))) + r.check "mcache get returns none for an absent id" + (decide (MessageCache.get c1 (mid 2) = none)) + r.check "mcache get returns the message after historyLength minus one shifts" + (decide (MessageCache.get (MessageCache.shiftN c1 4) (mid 1) = some (msg topicA 1))) + r.check "mcache get returns none after historyLength shifts" + (decide (MessageCache.get (MessageCache.shiftN c1 5) (mid 1) = none)) + + -- Message cache: four messages entered one heartbeat apart, so their + -- ages are zero to three. + let g1 : MessageCache := c0.put (mid 1) (msg topicA 1) + let g2 : MessageCache := (MessageCache.shift g1).put (mid 2) (msg topicA 2) + let g3 : MessageCache := (MessageCache.shift g2).put (mid 3) (msg topicA 3) + let g4 : MessageCache := (MessageCache.shift g3).put (mid 4) (msg topicA 4) + r.check "mcache gossipIds lists only ids newer than historyGossip windows" + (decide (MessageCache.gossipIds g4 topicA = [mid 4, mid 3, mid 2])) + let g5 : MessageCache := g4.put (mid 9) (msg topicB 9) + r.check "mcache gossipIds filters by topic" + (decide (MessageCache.gossipIds g5 topicA = [mid 4, mid 3, mid 2] ∧ + MessageCache.gossipIds g5 topicB = [mid 9])) + r.check "mcache gossipIds is empty for a message old enough" + (decide (MessageCache.gossipIds (MessageCache.shiftN c1 3) topicA = [])) + + -- Window parameters: the validity predicate of the spec. + r.check "window params reject gossip zero" (decide (¬ (0 < 0 ∧ 0 ≤ 5))) + r.check "window params reject gossip above length" (decide (¬ (0 < 6 ∧ 6 ≤ 5))) + r.check "window params accept gossip within length" (decide (0 < 3 ∧ 3 ≤ 5)) + + -- Seen cache: one id at time ten, and the two time boundaries. + let s0 : SeenCache := SeenCache.empty seen3 + r.check "seen contains after add at the same time" + (SeenCache.contains (s0.add (mid 1) 10) (mid 1) 10) + r.check "seen contains holds below t plus ttl" + (SeenCache.contains (s0.add (mid 1) 10) (mid 1) 109) + r.check "seen contains fails at exactly t plus ttl" + (!SeenCache.contains (s0.add (mid 1) 10) (mid 1) 110) + + -- Seen cache: three adds fill the cache, one more evicts the oldest. + let f1 : SeenCache := s0.add (mid 1) 10 + let f2 : SeenCache := f1.add (mid 2) 20 + let f3 : SeenCache := f2.add (mid 3) 30 + let f4 : SeenCache := f3.add (mid 4) 40 + r.check "seen add at capacity evicts the oldest entry" + (!SeenCache.contains f4 (mid 1) 40) + r.check "seen add at capacity keeps the second oldest entry" + (SeenCache.contains f4 (mid 2) 40) + r.check "seen size stays at capacity after an add at capacity" + (decide (f4.entries.length = seen3.capacity)) + let f5 : SeenCache := f3.add (mid 1) 35 + r.check "seen add refreshes an existing id" + (decide (f5.entries = [ + ({ id := mid 1, «at» := 35 } : SeenEntry), + { id := mid 3, «at» := 30 }, + { id := mid 2, «at» := 20 }])) + let f6 : SeenCache := f2.prune 115 + r.check "seen prune drops only expired entries" + (decide (f6.entries = [{ id := mid 2, «at» := 20 }])) + + -- Backoff table: a period from ten to sixty. + let b1 : BackoffTable := Backoff.add Backoff.empty (peer 1) topicA 50 10 + r.check "backoff active below until" + (Backoff.active b1 (peer 1) topicA 59) + r.check "backoff inactive at exactly until" + (!Backoff.active b1 (peer 1) topicA 60) + r.check "backoff inactive for an unknown pair" + (!Backoff.active b1 (peer 2) topicA 30 && !Backoff.active b1 (peer 1) topicB 30) + let b2 : BackoffTable := Backoff.add b1 (peer 1) topicA 50 40 + r.check "backoff a second add extends the period" + (Backoff.active b2 (peer 1) topicA 60 && Backoff.active b2 (peer 1) topicA 89 && + !Backoff.active b2 (peer 1) topicA 90) + let b3 : BackoffTable := + Backoff.add (Backoff.add Backoff.empty (peer 1) topicA 50 10) (peer 2) topicA 100 10 + let b4 : BackoffTable := Backoff.clear b3 70 + r.check "backoff clear keeps active entries" + (Backoff.active b4 (peer 2) topicA 70) + r.check "backoff clear drops expired entries" + (decide (b4 = [backoffOf (peer 2) topicA 110])) + + -- Promises: one promise, then the same pair again. + let p0 : PromiseTable := PromiseTable.promise [] (peer 1) (mid 7) 300 + r.check "promise records the pair and the deadline" + (decide (p0 = [promiseOf (peer 1) (mid 7) 300])) + let p1 : PromiseTable := PromiseTable.promise p0 (peer 1) (mid 7) 400 + r.check "promise a second time for the same pair replaces the deadline" + (decide (p1 = [promiseOf (peer 1) (mid 7) 400])) + let p2 : PromiseTable := + PromiseTable.promise + (PromiseTable.promise (PromiseTable.promise [] (peer 1) (mid 7) 300) (peer 2) (mid 7) 300) + (peer 1) (mid 8) 300 + r.check "deliver removes every promise for the id" + (decide (PromiseTable.deliver p2 (mid 7) = [promiseOf (peer 1) (mid 8) 300])) + let p3 : PromiseTable := + PromiseTable.promise (PromiseTable.promise [] (peer 1) (mid 7) 300) (peer 2) (mid 8) 400 + r.check "deliver keeps promises of other ids" + (decide (PromiseTable.deliver p3 (mid 7) = [promiseOf (peer 2) (mid 8) 400])) + + -- Broken promises at time five hundred: one expired promise of peer + -- one, and one promise that is not due. + let q0 : PromiseTable := + PromiseTable.promise (PromiseTable.promise [] (peer 1) (mid 7) 400) (peer 1) (mid 8) 900 + r.check "broken lists the peer with the expired count" + (decide (PromiseTable.broken q0 500 = [(peer 1, 1)])) + let q1 : PromiseTable := + PromiseTable.promise (PromiseTable.promise [] (peer 1) (mid 7) 400) (peer 1) (mid 8) 450 + r.check "broken counts two expired promises for one peer" + (decide (PromiseTable.broken q1 500 = [(peer 1, 2)])) + let q2 : PromiseTable := PromiseTable.promise [] (peer 1) (mid 7) 900 + r.check "broken omits peers whose deadlines have not passed" + (decide (PromiseTable.broken q2 500 = [])) + r.check "broken after deliver shows no count for the delivered id" + (decide (PromiseTable.broken (PromiseTable.deliver q1 (mid 7)) 500 = [(peer 1, 1)])) + +end Tests.Gossipsub.Cache diff --git a/Tests/Gossipsub/Core.lean b/Tests/Gossipsub/Core.lean new file mode 100644 index 00000000..93f4bf49 --- /dev/null +++ b/Tests/Gossipsub/Core.lean @@ -0,0 +1,412 @@ +import Tests.Support +import Tests.Gossipsub.Identity +import Libp2p.Protocols.Gossipsub.Core + +/-! +# Gossipsub suite: the router core + +Transcript checks for the gossipsub v1.1 router: the mesh fill and +prune, the backoff, the score gates, the deduplication, the IWANT +answers, the fanout, and the rejection cases. The events, the times, and +the peer orders are fixed, so every run replays. + +The spec sections are `vendor/specs/pubsub/gossipsub/gossipsub-v1.0.md` +(the router, the heartbeat, the mesh, the fanout) and +`gossipsub-v1.1.md` (the score gates, the backoff). +-/ + +set_option autoImplicit false + +namespace Tests.Gossipsub.Core + +open Libp2p.Protocols +open Libp2p.Protocols.Gossipsub +open Libp2p.Protocols.Gossipsub.Core + +/-! ## Fixtures -/ + +private def ta : TopicId := ⟨"t"⟩ +private def tb : TopicId := ⟨"u"⟩ + +private def pa : Peer := Identity.peer 1 +private def pb : Peer := Identity.peer 2 +private def pc : Peer := Identity.peer 3 +private def pd : Peer := Identity.peer 4 +private def px : Peer := Identity.peer 9 + +private def prims : Sign.Prims := + { unmarshalKey := fun _ => none + inlineKey := fun _ => none + idOfKey := fun _ => [] + verify := fun _ _ _ => false } + +/-- The test parameters: a small mesh, and the message id is the data +bytes, so two messages of different payloads have different ids. -/ +private def baseParams : Params := + { defaults with + mesh := { d := 2, dLow := 1, dHigh := 3, dScore := 1, dLazy := 2 } + prims := prims + msgId := fun m => ⟨m.data.getD []⟩ } + +/-- Every peer is below this graylist threshold. -/ +private def grayParams : Params := + { baseParams with + thresholds := { baseParams.thresholds with graylistThreshold := 1 } } + +/-- No peer reaches this gossip threshold. -/ +private def gossipParams : Params := + { baseParams with + thresholds := { baseParams.thresholds with gossipThreshold := 1 } } + +/-- The topic parameters of the scoring checks: one invalid delivery +costs a full score unit, and the decay keeps the counters. -/ +private def scoreTopic : Score.TopicParams := + { topicWeight := 1 + timeInMeshWeight := 0 + timeInMeshQuantum := 1 + timeInMeshCap := 0 + firstMessageDeliveriesWeight := 0 + firstMessageDeliveriesDecay := scoreScale + firstMessageDeliveriesCap := 0 + meshMessageDeliveriesWeight := 0 + meshMessageDeliveriesDecay := scoreScale + meshMessageDeliveriesThreshold := 0 + meshMessageDeliveriesCap := 0 + meshMessageDeliveriesActivation := 0 + meshMessageDeliveriesWindow := 0 + meshFailurePenaltyWeight := 0 + meshFailurePenaltyDecay := scoreScale + invalidMessageDeliveriesWeight := -1000 + invalidMessageDeliveriesDecay := scoreScale } + +/-- The parameters with the topic `t` configured. -/ +private def cfgParams : Params := + { baseParams with + score := { baseParams.score with topics := fun t => if t = ta then some scoreTopic else none } } + +private def peerRec (subs : List TopicId) : PeerData := + { score := Score.PeerState.zero, proto := .meshsub, subs := subs } + +private def emptyState : State := + { peers := [] + mesh := [] + fanout := [] + mcache := Cache.MessageCache.empty baseParams.window + seen := Cache.SeenCache.empty baseParams.seen + backoff := [] + iwant := [] + topics := [] + now := 0 + lastHeartbeat := 0 } + +/-- A state with the four peers known and the topic `t` subscribed. -/ +private def known : State := + { emptyState with + peers := [(pa, peerRec [ta]), (pb, peerRec [ta]), (pc, peerRec [ta]), (pd, peerRec [ta])] + topics := [ta] } + +/-- A message that passes `strictNoSign`: the four signed fields are +absent. The payload is the message id. -/ +private def msgOf (topic : TopicId) (payload : List Nat) : Message := + { «from» := none, data := some payload, seqno := none, topic := topic, + signature := none, key := none } + +/-- A message with a `from` field, which fails `strictNoSign`. -/ +private def badMsg : Message := + { «from» := some [9], data := some [1], seqno := none, topic := ta, + signature := none, key := none } + +private def rpcOfPublish (ms : List Message) : Rpc.RPC := + { subscriptions := [], publish := ms, control := none } + +private def rpcOfControl (c : Rpc.ControlMessage) : Rpc.RPC := + { subscriptions := [], publish := [], control := some c } + +private def controlOf (grafts : List Rpc.ControlGraft) (prunes : List Rpc.ControlPrune) : Rpc.ControlMessage := + { ihave := [], iwant := [], graft := grafts, prune := prunes } + +private def pruneOf (topic : TopicId) : Rpc.ControlPrune := + { topicId := topic.name, peers := [], backoff := none } + +private def graftOf (topic : TopicId) : Rpc.ControlGraft := { topicId := topic.name } + +/-- Malformed bytes: field 1 of length 5 with one byte behind it. -/ +private def malformed : List Nat := [10, 5, 1] + +/-! ## Result accessors -/ + +private def stateOf : Except ProtocolError (State × List Action) → State + | .ok (s, _) => s + | .error _ => emptyState + +private def actionsOf : Except ProtocolError (State × List Action) → List Action + | .ok (_, a) => a + | .error _ => [] + +private def isDecodeError : Except ProtocolError (State × List Action) → Bool + | .error (.decode _) => true + | _ => false + +private def isOk : Except ProtocolError (State × List Action) → Bool + | .ok _ => true + | .error _ => false + +private def delivers (m : Message) (l : List Action) : Nat := + (l.filter (fun a => match a with + | .deliver m' => decide (m' = m) + | .sendRpc _ _ => false)).length + +private def sendsTo (q : Peer) (l : List Action) : Nat := + (l.filter (fun a => match a with + | .sendRpc q' _ => decide (q' = q) + | .deliver _ => false)).length + +private def sends (l : List Action) (test : Rpc.RPC → Bool) : Nat := + (l.filter (fun a => match a with + | .sendRpc _ r => test r + | .deliver _ => false)).length + +private def isPrune (t : TopicId) (r : Rpc.RPC) : Bool := + match r.control with + | some c => c.prune.any (fun p => decide (p.topicId = t.name)) + | none => false + +private def isGraft (t : TopicId) (r : Rpc.RPC) : Bool := + match r.control with + | some c => c.graft.any (fun g => decide (g.topicId = t.name)) + | none => false + +private def hasIhave (r : Rpc.RPC) : Bool := + match r.control with + | some c => !c.ihave.isEmpty + | none => false +private def hasPublish (r : Rpc.RPC) : Bool := !r.publish.isEmpty + +private def pruneCount (t : TopicId) (l : List Action) : Nat := sends l (isPrune t) +private def graftCount (t : TopicId) (l : List Action) : Nat := sends l (isGraft t) +private def publishCount (l : List Action) : Nat := sends l hasPublish +private def ihaveCount (l : List Action) : Nat := sends l hasIhave + +private def invalidCount (s : State) (q : Peer) (t : TopicId) : Nat := + match peerOf s q with + | some pd => (Score.countersOf pd.score t).invalidMessageDeliveries + | none => 0 + +/-- The counters of a peer with one invalid delivery. -/ +private def negCounters : Score.TopicCounters := + { Score.TopicCounters.zero with + inMesh := true, graftTime := 0, invalidMessageDeliveries := 1 } + +/-- The score state of a peer with one invalid delivery in topic `t`. -/ +private def negPeerState : Score.PeerState := + { Score.PeerState.zero with topics := [(ta, negCounters)] } + +/-- A peer with one invalid delivery in a topic whose weight is -1000. +Its score is -1. -/ +private def negPeerData : PeerData := + { score := negPeerState, proto := .meshsub, subs := [ta] } + +/-- The state with the negative peer in the mesh of `t`. -/ +private def negState : State := + { emptyState with + peers := [(pa, negPeerData)] + topics := [ta] + mesh := [(ta, [pa])] + now := 0 + lastHeartbeat := 0 } + +private def meshIs (s : State) (t : TopicId) (ps : List Peer) : Bool := decide (meshOf s t = ps) + +/-! ## Checks -/ + +/-- Run the checks of this module. -/ +def run (r : Runner) : IO Unit := do + r.section "gossipsub Core" + "vendor/specs/pubsub/gossipsub/gossipsub-v1.0.md, gossipsub-v1.1.md" + + -- A malformed RPC is the one error constructor. + r.check "core.reject.malformed-rpc" + (isDecodeError (step baseParams known (.rpc pa malformed))) + + -- The graylist gate runs before the decode: malformed bytes from a + -- graylisted peer are ignored, not rejected. + r.check "core.reject.graylist-malformed-rpc-ignored" + (match step grayParams known (.rpc pa malformed) with + | .ok (s', acts) => acts = [] && meshIs s' ta (meshOf known ta) + | _ => false) + + -- The graylist gate ignores a valid RPC too. + r.check "core.reject.graylist-valid-rpc-ignored" + (match step grayParams known (.rpc pa (Pb.encodeMessage (rpcOfControl (controlOf [graftOf ta] [])))) with + | .ok (s', acts) => acts = [] && meshIs s' ta (meshOf known ta) + | _ => false) + + -- An unknown peer is ignored. + r.check "core.reject.unknown-peer-ignored" + (match step baseParams known (.rpc px (Pb.encodeMessage (rpcOfControl (controlOf [graftOf ta] [])))) with + | .ok (s', acts) => acts = [] && meshIs s' ta (meshOf known ta) + | _ => false) + + -- A graft for a topic we do not serve draws no prune answer. + r.check "core.reject.graft-unknown-topic-ignored" + (match step baseParams known (.rpc pa (Pb.encodeMessage (rpcOfControl (controlOf [graftOf tb] [])))) with + | .ok (s', acts) => acts = [] && meshIs s' tb [] + | _ => false) + + -- A graft inside an active backoff draws a prune and adds no peer. + r.check "core.reject.graft-backoff-pruned" + (match step baseParams + { known with backoff := Cache.Backoff.add [] pa ta 60000 0, now := 1000 } + (.rpc pa (Pb.encodeMessage (rpcOfControl (controlOf [graftOf ta] [])))) with + | .ok (s', acts) => pruneCount ta acts = 1 && meshIs s' ta [] + | _ => false) + + -- A graft when the mesh holds `dHigh` peers draws a prune. + r.check "core.reject.graft-at-dHigh-pruned" + (match step baseParams { known with mesh := [(ta, [pb, pc, pd])] } + (.rpc pa (Pb.encodeMessage (rpcOfControl (controlOf [graftOf ta] [])))) with + | .ok (s', acts) => pruneCount ta acts = 1 && meshIs s' ta [pb, pc, pd] + | _ => false) + + -- An IWANT for an id the message cache does not hold draws no answer. + r.check "core.reject.iwant-uncached-no-answer" + (match step baseParams known + (.rpc pa (Pb.encodeMessage (iwantRpc [⟨[7]⟩]))) with + | .ok (_, acts) => publishCount acts = 0 + | _ => false) + + -- A message that fails `strictNoSign` draws no deliver and no + -- forward, and increments the invalid delivery counter. + r.check "core.reject.invalid-signature-no-forward" + (match step cfgParams { emptyState with peers := [(pa, peerRec [ta])], topics := [ta] } + (.rpc pa (Pb.encodeMessage (rpcOfPublish [badMsg]))) with + | .ok (s', acts) => delivers badMsg acts = 0 && publishCount acts = 0 && + invalidCount s' pa ta = 1 + | _ => false) + + -- A repeated id inside one event is delivered once. + r.check "core.reject.duplicate-delivered-once" + (match step baseParams known + (.rpc pa (Pb.encodeMessage (rpcOfPublish [msgOf ta [1], msgOf ta [1]]))) with + | .ok (_, acts) => delivers (msgOf ta [1]) acts = 1 + | _ => false) + + -- A tick before the interval ends does no maintenance. + r.check "core.reject.early-tick-no-maintenance" + (match step baseParams { known with lastHeartbeat := 0, now := 500 } + (.tick 500 [pa, pb, pc, pd]) with + | .ok (s', acts) => acts = [] && meshIs s' ta (meshOf known ta) + | _ => false) + + -- A prune from a peer outside the mesh still records the backoff pair. + r.check "core.reject.prune-from-nonmesh-records-backoff" + (match step baseParams { known with now := 1000 } + (.rpc pa (Pb.encodeMessage (rpcOfControl (controlOf [] [pruneOf ta])))) with + | .ok (s', _) => Cache.Backoff.active s'.backoff pa ta s'.now + | _ => false) + + -- The mesh fill of a subscribe: `d` peers, one graft each. + r.check "core.mesh.fill" + (let e := step baseParams known (.subscribe ta) + let s := stateOf e + let a := actionsOf e + decide ((meshOf s ta).length = 2) && graftCount ta a = 2) + + -- An overfull mesh is pruned to `d` at the heartbeat. + r.check "core.mesh.prune" + (match step baseParams + { known with mesh := [(ta, [pa, pb, pc, pd])], lastHeartbeat := 0, now := 0 } + (.tick 1000 []) with + | .ok (s', acts) => meshIs s' ta [pc, pd] && pruneCount ta acts = 2 + | _ => false) + + -- Gossip stays away from a peer below the gossip threshold. + r.check "core.gossip.threshold" + (match step gossipParams + { known with mcache := (Cache.MessageCache.empty baseParams.window).put ⟨[1]⟩ (msgOf ta [1]) } + (.tick 1000 [pa, pb, pc, pd]) with + | .ok (_, acts) => ihaveCount acts = 0 + | _ => false) + + -- Gossip goes to peers outside the mesh, and the ids are the ones the + -- cache held before the shift. `pa` and `pb` sit in the mesh, so a + -- router that advertised to mesh members would reach them. + r.check "core.gossip.outside-mesh" + (match step baseParams + { known with + mesh := [(ta, [pa, pb])] + mcache := (Cache.MessageCache.empty baseParams.window).put ⟨[1]⟩ (msgOf ta [1]) } + (.tick 1000 [pa, pb, pc, pd]) with + | .ok (_, acts) => sendsTo pa acts = 0 && sendsTo pb acts = 0 && + sendsTo pc acts = 1 && sendsTo pd acts = 1 && ihaveCount acts = 2 + | _ => false) + + -- A mesh peer below score 0 leaves the mesh at the heartbeat. + r.check "core.score.negative-pruned" + (match step cfgParams negState (.tick 1000 []) with + | .ok (s', acts) => meshIs s' ta [] && pruneCount ta acts = 1 + | _ => false) + + -- A join sends the hello RPC; a leave drops the peer from the mesh. + r.check "core.join.hello" + (match step baseParams { known with peers := [] } (.peerJoined pa .meshsub) with + | .ok (s', acts) => sendsTo pa acts = 1 && (peerOf s' pa).isSome + | _ => false) + r.check "core.leave.mesh" + (match step baseParams { known with mesh := [(ta, [pa, pb])] } (.peerLeft pa) with + | .ok (s', acts) => acts = [] && meshIs s' ta [pb] + | _ => false) + + -- An inbound message reaches every other mesh peer, and not the sender. + r.check "core.forward.mesh" + (match step baseParams { known with mesh := [(ta, [pa, pb, pc])] } + (.rpc pa (Pb.encodeMessage (rpcOfPublish [msgOf ta [1]]))) with + | .ok (_, acts) => delivers (msgOf ta [1]) acts = 1 && + sendsTo pa acts = 0 && sendsTo pb acts = 1 && sendsTo pc acts = 1 + | _ => false) + + -- A local publish records the id. A later echo of that id is not + -- delivered and is not forwarded again. The mesh is nonempty, so a + -- missing seen record would send the echo to `pa` and `pc`. + r.check "core.forward.own-echo-dropped" + (match Libp2p.Protocols.Gossipsub.Core.run baseParams + { known with mesh := [(ta, [pa, pb, pc])] } + [.publish (msgOf ta [1]), + .rpc pb (Pb.encodeMessage (rpcOfPublish [msgOf ta [1]]))] with + | .ok (_, acts) => delivers (msgOf ta [1]) acts = 0 && + sendsTo pa acts = 1 && sendsTo pb acts = 1 && sendsTo pc acts = 1 + | _ => false) + + -- Two payloads under `defaults` stay distinct. `strictNoSign` leaves + -- `from` and `seqno` empty, so the spec's default id would collapse them. + r.check "core.defaults.distinct-payloads" + (match step defaults { known with mesh := [(ta, [pa, pb])] } + (.rpc pa (Pb.encodeMessage (rpcOfPublish [msgOf ta [1], msgOf ta [2]]))) with + | .ok (_, acts) => delivers (msgOf ta [1]) acts = 1 && + delivers (msgOf ta [2]) acts = 1 && sendsTo pb acts = 2 + | _ => false) + + -- A publish without a subscription fills the fanout and forwards. + r.check "core.publish.fanout" + (match step baseParams { known with topics := [] } (.publish (msgOf ta [1])) with + | .ok (s', acts) => (fanoutOf s' ta).length = 2 && publishCount acts = 2 + | _ => false) + + -- A cached id answers an IWANT with one publish. + r.check "core.iwant.cached-answer" + (match step baseParams + { known with mcache := (Cache.MessageCache.empty baseParams.window).put ⟨[7]⟩ (msgOf ta [7]) } + (.rpc pa (Pb.encodeMessage (iwantRpc [⟨[7]⟩]))) with + | .ok (_, acts) => publishCount acts = 1 + | _ => false) + + -- The transcript driver runs two events in order and appends the + -- actions. + r.check "core.run.transcript" + (match Libp2p.Protocols.Gossipsub.Core.run baseParams known + [.subscribe ta, .publish (msgOf ta [1])] with + | .ok (s', acts) => graftCount ta acts = 2 && publishCount acts = 2 && + meshIs s' ta [pa, pb] + | _ => false) + +end Tests.Gossipsub.Core diff --git a/Tests/Gossipsub/Identity.lean b/Tests/Gossipsub/Identity.lean new file mode 100644 index 00000000..c36b6a4b --- /dev/null +++ b/Tests/Gossipsub/Identity.lean @@ -0,0 +1,25 @@ +import Libp2p.Identity + +/-! +# Identity fixtures for protocol transcripts + +The transcripts need distinct validated peer ids without a native crypto +call. Each id is an identity multihash of one byte. Seed-derived keys +belong to the identity and crypto suites. +-/ + +set_option autoImplicit false + +namespace Tests.Gossipsub.Identity + +/-- A validated identity multihash for one transcript peer. -/ +def peer (n : Nat) : Libp2p.Identity.PeerId := + Libp2p.Identity.PeerId.inline (ByteArray.mk #[n.toUInt8]) (by + simp [ByteArray.size, Libp2p.Identity.maxInlineKeyBytes]) + +/-- A sized public key for policy tests that control verification. -/ +def publicKey (n : Nat) : Libp2p.Identity.PubKey := + Libp2p.Identity.PubKey.ofEd25519 + ⟨ByteArray.mk (Array.replicate 32 n.toUInt8), by simp [ByteArray.size]⟩ + +end Tests.Gossipsub.Identity diff --git a/Tests/Gossipsub/Interop.lean b/Tests/Gossipsub/Interop.lean new file mode 100644 index 00000000..e98e6ada --- /dev/null +++ b/Tests/Gossipsub/Interop.lean @@ -0,0 +1,489 @@ +import Tests.Support +import Libp2p.Protocols.Gossipsub.Interop + +/-! +# GossipSub interop pure-core suite + +This suite checks the adapter against the captured 32-node script and the +pure edge cases required by specification sections 2, 3, 5, 6, and 7. +The fixture count is top-level 100. Recursive instruction counts include the +80 conditional wrappers and their nested instructions, so they total 180. +-/ + +set_option autoImplicit false + +namespace Tests.Gossipsub.Interop + +open Lean (Json) +open Tests (Runner loadFixture) +open Libp2p.Protocols.Gossipsub +open Libp2p.Protocols.Gossipsub.Interop + +private def fixture : String := "gossipsub-interop-params.json" + +structure Counts where + total : Nat := 0 + ifNode : Nat := 0 + init : Nat := 0 + connect : Nat := 0 + subscribe : Nat := 0 + publish : Nat := 0 + validationDelay : Nat := 0 + wait : Nat := 0 + +private def add (left right : Counts) : Counts := + { total := left.total + right.total + ifNode := left.ifNode + right.ifNode + init := left.init + right.init + connect := left.connect + right.connect + subscribe := left.subscribe + right.subscribe + publish := left.publish + right.publish + validationDelay := left.validationDelay + right.validationDelay + wait := left.wait + right.wait } + +private def countInstruction : Instruction → Counts + | .initGossipSub _ => { total := 1, init := 1 } + | .ifNodeIDEquals _ nested => add { total := 1, ifNode := 1 } (countInstruction nested) + | .connect _ => { total := 1, connect := 1 } + | .subscribeToTopic _ _ => { total := 1, subscribe := 1 } + | .publish _ _ _ => { total := 1, publish := 1 } + | .setTopicValidationDelay _ _ => { total := 1, validationDelay := 1 } + | .waitUntil _ => { total := 1, wait := 1 } + +private def countInstructions (instructions : List Instruction) : Counts := + instructions.foldl (fun counts instruction => add counts (countInstruction instruction)) {} + +private def firstInit : List Instruction → Option GossipSubParams + | [] => none + | .initGossipSub params :: _ => some params + | .ifNodeIDEquals _ nested :: rest => firstInit (nested :: rest) + | _ :: rest => firstInit rest + +private def firstNested : List Instruction → Option Instruction + | [] => none + | .ifNodeIDEquals nodeId nested :: _ => some (.ifNodeIDEquals nodeId nested) + | _ :: rest => firstNested rest + +private def hasConnect : List Instruction → Bool + | [] => false + | .connect peers :: _ => !peers.isEmpty + | .ifNodeIDEquals _ nested :: rest => hasConnect (nested :: rest) + | _ :: rest => hasConnect rest + +private def hasSubscribe : List Instruction → Bool + | [] => false + | .subscribeToTopic topic isPartial :: _ => topic == "a-subnet" && !isPartial + | .ifNodeIDEquals _ nested :: rest => hasSubscribe (nested :: rest) + | _ :: rest => hasSubscribe rest + +private def hasPublish : List Instruction → Bool + | [] => false + | .publish messageId size topic :: _ => messageId == 0 && size == 98304 && topic == "a-subnet" + | .ifNodeIDEquals _ nested :: rest => hasPublish (nested :: rest) + | _ :: rest => hasPublish rest + +private def hasDelay : List Instruction → Bool + | [] => false + | .setTopicValidationDelay topic delay :: _ => topic == "a-subnet" && delay == 0.005 + | .ifNodeIDEquals _ nested :: rest => hasDelay (nested :: rest) + | _ :: rest => hasDelay rest + +private def hasWait : List Instruction → Bool + | [] => false + | .waitUntil seconds :: rest => seconds == 342 || hasWait rest + | .ifNodeIDEquals _ nested :: rest => hasWait (nested :: rest) + | _ :: rest => hasWait rest + +private def parsedJson (source : String) : Json := + match Json.parse source with + | .ok value => value + | .error _ => Json.null + +private def isError {α : Type} : Except String α → Bool + | .error _ => true + | .ok _ => false + +private def errorIs {α : Type} (message : String) : Except String α → Bool + | .error error => error == message + | .ok _ => false + +private def stringField (value : Json) (name : String) : Option String := + match value.getObjVal? name with + | .ok field => field.getStr?.toOption + | .error _ => none + +private def intField (value : Json) (name : String) : Option Int := + match value.getObjVal? name with + | .ok field => field.getInt?.toOption + | .error _ => none + +/-- The fixture, instruction fields, and parameter conversion. -/ +private def checkScript (r : Runner) : IO Unit := do + r.section "gossipsub interop script" "vendor/test-plans/gossipsub-interop/script_instruction.py" + let fixtureJson ← loadFixture fixture + let parsed := parseScriptJson fixtureJson + match parsed with + | .error _ => + r.check "captured script parses" false + | .ok script => + let counts := countInstructions script.instructions + r.check "top-level instruction count" (script.instructions.length == 100) + r.check "recursive instruction count" (counts.total == 180) + r.check "ifNodeIDEquals count" (counts.ifNode == 80) + r.check "initGossipSub count" (counts.init == 32) + r.check "connect count" (counts.connect == 32) + r.check "publish count" (counts.publish == 16) + r.check "waitUntil count" (counts.wait == 18) + r.check "setTopicValidationDelay count" (counts.validationDelay == 1) + r.check "subscribeToTopic count" (counts.subscribe == 1) + r.check "connect fields decode" (hasConnect script.instructions) + r.check "subscribe fields decode" (hasSubscribe script.instructions) + r.check "publish fields decode" (hasPublish script.instructions) + r.check "validation delay fields decode" (hasDelay script.instructions) + r.check "wait fields decode" (hasWait script.instructions) + match firstNested script.instructions with + | some (.ifNodeIDEquals nodeId (.initGossipSub _)) => + r.check "nested node id decodes" (nodeId == 0) + | _ => r.check "nested instruction decodes" false + match firstInit script.instructions with + | none => r.check "captured parameters present" false + | some params => + r.check "captured parameter pair retained" (params.raw.length == 1) + r.check "HeartbeatInitialDelay retained" (params.heartbeatInitialDelay.map nanosecondsToMillis == some 100) + r.check "HeartbeatInitialDelay is not mapped" (match toCoreParams params with + | .ok core => core.heartbeatInterval == Core.defaults.heartbeatInterval + | .error _ => false) + r.check "HeartbeatInitialDelay is unmapped" (params.unmapped.length == 1 && + params.unmapped.any (fun pair => pair.1 == "HeartbeatInitialDelay")) + +/-- Each unsupported and malformed input returns a typed parse error. -/ +private def checkErrors (r : Runner) : IO Unit := do + r.check "script scalar rejected" (errorIs "root.script: expected array" (parseScript "{\"script\":5}")) + r.check "unknown instruction rejected" (errorIs "script[0].type: unknown instruction type nope" (parseScript "{\"script\":[{\"type\":\"nope\"}]}")) + r.check "publish missing topic rejected" (isError (parseScriptJson (parsedJson "{\"script\":[{\"type\":\"publish\",\"messageID\":1,\"messageSizeBytes\":8}]}"))) + r.check "connect non-integer rejected" (isError (parseScriptJson (parsedJson "{\"script\":[{\"type\":\"connect\",\"connectTo\":[\"x\"]}]}"))) + r.check "addPartialMessage rejected explicitly" + (errorIs "script[0]: addPartialMessage is out of scope for v1.1" + (parseScriptJson (parsedJson "{\"script\":[{\"type\":\"addPartialMessage\"}]}"))) + r.check "publishPartial rejected explicitly" + (errorIs "script[0]: publishPartial is out of scope for v1.1" + (parseScriptJson (parsedJson "{\"script\":[{\"type\":\"publishPartial\"}]}"))) + +/-- The record encoder emits compact, escaped JSON objects. -/ +private def checkRecords (r : Runner) : IO Unit := do + let peer := peerIdLine "2026-11-06T00:00:00Z" "peer" 3 + let received := receivedMessageLine "2026-11-06T00:00:00Z" 31 (some "peer\"\\") (some "topic\"\\") + r.check "PeerID line exact shape" + (peer == "{\"id\":\"peer\",\"msg\":\"PeerID\",\"node_id\":3,\"time\":\"2026-11-06T00:00:00Z\"}") + r.check "PeerID line has required fields" + (stringField (parsedJson peer) "msg" == some "PeerID" && intField (parsedJson peer) "node_id" == some 3) + r.check "received line exact shape" + (received == "{\"from\":\"peer\\\"\\\\\",\"id\":\"31\",\"msg\":\"Received Message\",\"time\":\"2026-11-06T00:00:00Z\",\"topic\":\"topic\\\"\\\\\"}") + r.check "received line parses back" + (stringField (parsedJson received) "topic" == some "topic\"\\" && stringField (parsedJson received) "id" == some "31") + r.check "records stay single line" (!peer.contains '\n' && !received.contains '\n') +/-- The message id and node address edge cases. -/ +private def checkIds (r : Runner) : IO Unit := do + r.check "message id one" (messageIdString [0, 0, 0, 0, 0, 0, 0, 1] == "1") + r.check "message id maximum" (messageIdString [0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff] == "18446744073709551615") + r.check "message id big endian" (messageIdString [1, 0, 0, 0, 0, 0, 0, 0] == "72057594037927936") + r.check "short message id is right padded" (messageIdString [1] == "72057594037927936") + r.check "node zero" (nodeId? "node0" == some 0) + r.check "node thirty one" (nodeId? "node31" == some 31) + r.check "node without number rejected" (nodeId? "node" == none) + r.check "leading zero rejected" (nodeId? "node01" == none) + r.check "negative node rejected" (nodeId? "node-1" == none) + r.check "uppercase node rejected" (nodeId? "Node1" == none) + r.check "suffix text rejected" (nodeId? "node1x" == none) + r.check "empty hostname rejected" (nodeId? "" == none) + r.check "overflow node rejected" (nodeId? "node18446744073709551616" == none) + r.check "peer seed is 32 bytes" ((peerSeed 31).length == 32) + r.check "peer seed uses little endian" ((peerSeed 31).take 8 == [31, 0, 0, 0, 0, 0, 0, 0]) + + +/-- All mapped fields, retained fields, and pure scheduling contracts. -/ +private def checkParameters (r : Runner) : IO Unit := do + let json := parsedJson "{\"D\":8,\"Dlo\":6,\"Dhi\":14,\"Dscore\":5,\"Dlazy\":7,\"HistoryLength\":9,\"HistoryGossip\":4,\"GossipFactor\":0.255,\"HeartbeatInitialDelay\":100100000,\"HeartbeatInterval\":100100000,\"FanoutTTL\":1201000000,\"PruneBackoff\":2001999999,\"Dout\":2}" + match parseGossipSubParams "params" json with + | .error _ => r.check "all parameter fields parse" false + | .ok params => + r.check "all source parameter pairs retained" (params.raw.length == 13) + r.check "unsupported fields retained" (params.unmapped.map Prod.fst == + ["Dout", "HeartbeatInitialDelay"]) + match toCoreParams params with + | .error _ => r.check "valid parameters map" false + | .ok core => + r.check "mesh D maps" (core.mesh.d == 8) + r.check "mesh Dlo maps" (core.mesh.dLow == 6) + r.check "mesh Dhi maps" (core.mesh.dHigh == 14) + r.check "mesh Dscore maps" (core.mesh.dScore == 5) + r.check "mesh Dlazy maps" (core.mesh.dLazy == 7) + r.check "heartbeat truncates nanoseconds" (core.heartbeatInterval == 100) + r.check "fanout truncates nanoseconds" (core.fanoutTtl == 1201) + r.check "prune truncates nanoseconds" (core.pruneBackoff == 2001) + r.check "gossip fraction truncates percent" (core.gossipFactor == 25) + r.check "history length maps" (core.window.historyLength == 9) + r.check "history gossip maps" (core.window.historyGossip == 4) + let message : Message := { + «from» := none + data := some [0,0,0,0,0,0,0,1,9] + seqno := none + topic := ⟨"topic"⟩ + signature := none + key := none } + r.check "core message id uses eight bytes" ((core.msgId message).bytes == [0,0,0,0,0,0,0,1]) + match parseGossipSubParams "params" (parsedJson "{}") with + | .error _ => r.check "empty parameters parse" false + | .ok params => + match toCoreParams params with + | .error _ => r.check "defaults map" false + | .ok core => + let defaults := Core.defaults + r.check "absent mesh fields use defaults" (core.mesh == defaults.mesh) + r.check "absent times use defaults" (core.heartbeatInterval == defaults.heartbeatInterval && + core.fanoutTtl == defaults.fanoutTtl && core.pruneBackoff == defaults.pruneBackoff) + r.check "absent cache uses defaults" (core.window.historyLength == defaults.window.historyLength && + core.window.historyGossip == defaults.window.historyGossip) + r.check "unknown parameter rejected" (isError + (parseGossipSubParams "params" (parsedJson "{\"FutureField\":1}"))) + r.check "negative parameter rejected" (isError + (parseGossipSubParams "params" (parsedJson "{\"HeartbeatInterval\":-1}"))) + match parseGossipSubParams "params" (parsedJson "{\"HistoryLength\":1,\"HistoryGossip\":2}") with + | .error _ => r.check "invalid cache bound rejected" false + | .ok params => r.check "invalid cache bound rejected" (isError (toCoreParams params)) + r.check "exact conversion near millisecond boundary" + (nanosecondsToMillis ⟨999999999999999999, 3⟩ == 999999999) + r.check "fractional nanosecond truncates" (nanosecondsToMillis ⟨1001000005, 1⟩ == 100) + r.check "wait blocks before deadline" (!waitComplete 119 120) + r.check "wait releases at deadline" (waitComplete 120 120) + r.check "wait releases after deadline" (waitComplete 121 120) + let captured ← loadFixture fixture + match parseScriptJson captured with + | .error _ => r.check "startup script parses" false + | .ok script => + for nodeId in List.range 32 do + r.check s!"node {nodeId} initializes first" (match prepareScript nodeId script with + | .ok (_, rest) => !rest.isEmpty + | .error _ => false) + r.check "missing startup rejected" (isError (prepareScript 32 script)) + let late := parseScript "{\"script\":[{\"type\":\"waitUntil\",\"elapsedSeconds\":0},{\"type\":\"initGossipSub\",\"gossipSubParams\":{}}]}" + r.check "late initialization rejected" (match late with + | .ok script => isError (prepareScript 0 script) + | .error _ => false) + r.check "partial default false" (match parseInstruction "i" + (parsedJson "{\"type\":\"subscribeToTopic\",\"topicID\":\"t\"}") with + | .ok (.subscribeToTopic topic flag) => topic == "t" && !flag + | _ => false) + r.check "partial explicit true retained" (match parseInstruction "i" + (parsedJson "{\"type\":\"subscribeToTopic\",\"topicID\":\"t\",\"partial\":true}") with + | .ok (.subscribeToTopic _ flag) => flag + | _ => false) + let topic := "quote\"slash\\line\nreturn\rtab\t" ++ String.singleton (Char.ofNat 1) + let line := receivedMessageLine "2026-11-06T00:00:00Z" 1 none (some topic) + r.check "all control characters escape" (!line.contains '\n' && !line.contains '\r' && + !line.contains '\t' && stringField (parsedJson line) "topic" == some topic) + r.check "empty message id zero" (messageIdString [] == "0") + r.check "message tail does not change id" (messageIdString [0,0,0,0,0,0,0,1,255] == "1") + r.check "maximum node accepted" (nodeId? "node18446744073709551615" == some 18446744073709551615) + r.check "seed spans all eight bytes" ((peerSeed 72623859790382856).take 8 == [8,7,6,5,4,3,2,1]) + r.check "seed suffix zero" ((peerSeed 72623859790382856).drop 8 == List.replicate 24 0) + +/-- The decoder preserves recursion and reports the exact failing field. -/ +private def checkDecoderDetails (r : Runner) : IO Unit := do + let source ← IO.FS.readFile "Tests/fixtures/gossipsub-interop-params.json" + r.check "captured source parses through Json.parse" (match parseScript source with + | .ok script => script.instructions.length == 100 + | .error _ => false) + r.check "missing publish topic names field" + (errorIs "script[0]: missing field topicID" + (parseScript "{\"script\":[{\"type\":\"publish\",\"messageID\":1,\"messageSizeBytes\":8}]}")) + r.check "bad connect element names index" + (errorIs "script[0].connectTo[1]: expected integer" + (parseScript "{\"script\":[{\"type\":\"connect\",\"connectTo\":[1,\"x\"]}]}")) + r.check "partial publish alias rejected explicitly" + (errorIs "script[0]: publishToNodeIDs is out of scope for v1.1" + (parseScript "{\"script\":[{\"type\":\"publishToNodeIDs\"}]}")) + r.check "nested condition recurses twice" (match parseInstruction "i" + (parsedJson "{\"type\":\"ifNodeIDEquals\",\"nodeID\":7,\"instruction\":{\"type\":\"ifNodeIDEquals\",\"nodeID\":7,\"instruction\":{\"type\":\"waitUntil\",\"elapsedSeconds\":42}}}") with + | .ok instruction => match instructionForNode 7 instruction with + | some (.waitUntil seconds) => seconds == 42 + | _ => false + | _ => false) + r.check "nested bad field includes full path" (errorIs "i.instruction: missing field elapsedSeconds" + (parseInstruction "i" (parsedJson "{\"type\":\"ifNodeIDEquals\",\"nodeID\":0,\"instruction\":{\"type\":\"waitUntil\"}}"))) + r.check "bounded decoder rejects excess nesting" (isError + (parseInstructionFuel 1 "i" (parsedJson "{\"type\":\"ifNodeIDEquals\",\"nodeID\":0,\"instruction\":{\"type\":\"waitUntil\",\"elapsedSeconds\":1}}"))) + let minimal := receivedMessageLine "2026-11-06T00:00:00Z" 18446744073709551615 none none + r.check "received optional fields omitted" (minimal == + "{\"id\":\"18446744073709551615\",\"msg\":\"Received Message\",\"time\":\"2026-11-06T00:00:00Z\"}") + +/-- Hosts parsing and exact name resolution. -/ +private def checkHosts (r : Runner) : IO Unit := do + let shadow := "127.0.0.1 localhost\n11.0.0.1 node0\n11.0.0.2 node1 node-one n1 # aliases\n11.0.0.3 node10\n" + let hosts := parseHosts shadow + r.check "Shadow hosts IPv4 rows parse" (hosts.length == 4) + r.check "primary name lookup" (lookup? hosts "node0" == some [11,0,0,1]) + r.check "alias lookup" (lookup? hosts "node-one" == some [11,0,0,2]) + r.check "missing name lookup" (lookup? hosts "node2" == none) + r.check "prefix node1 does not match node10" (lookup? hosts "node1" == some [11,0,0,2]) + r.check "suffix node1x does not match" (lookup? hosts "node1x" == none) + r.check "CRLF and whitespace parse" (lookup? (parseHosts " 11.0.0.4\tnode4\r\n") "node4" == some [11,0,0,4]) + r.check "IPv6 row ignored" (lookup? (parseHosts "::1 localhost\n") "localhost" == none) + r.check "malformed row ignored" (lookup? (parseHosts "11.0.0 node-bad\n") "node-bad" == none) + r.check "comment and blank rows ignored" (parseHosts "# only\n\n # tail\n" == []) + r.check "multiaddr prefix" (multiaddrPrefix [11,0,0,2] == some "/ip4/11.0.0.2/tcp/9000") + +private def nondecreasing : List ScheduledAction → Bool + | [] | [_] => true + | left :: right :: rest => left.time ≤ right.time && nondecreasing (right :: rest) + +private structure FixturePublish where + messageId : Nat + nodeId : Nat + time : Nat + size : Nat + topic : String + deriving BEq, DecidableEq + +private def publishRowsFor (script : Script) (nodeId : Nat) : List FixturePublish := + match plan script (Int.ofNat nodeId) with + | .error _ => [] + | .ok actions => actions.filterMap fun item => match item.action with + | .publish id size topic => some ⟨id.toNat, nodeId, item.time, size.toNat, topic⟩ + | _ => none + +private def allPublishRows (script : Script) : List FixturePublish := + (List.range 32).flatMap (publishRowsFor script) + +private def publishRowsForPlan (script : Script) (nodeId : Nat) : List (Nat × Nat × Nat × String) := + match plan script (Int.ofNat nodeId) with + | .error _ => [] + | .ok actions => actions.filterMap fun item => match item.action with + | .publish id size topic => some (item.time, id.toNat, size.toNat, topic) + | _ => none + +private def waitPlan (source : String) : Option (List ScheduledAction) := + match parseScript source with + | .error _ => none + | .ok script => (plan script 0).toOption + +private def checkPlan (r : Runner) : IO Unit := do + let source ← loadFixture fixture + match parseScriptJson source with + | .error _ => r.check "virtual plan parses fixture" false + | .ok script => + match plan script 0, plan script 0, plan script 1, plan script 2, plan script 31 with + | .ok node0, .ok again, .ok node1, .ok node2, .ok node31 => + r.check "virtual plan is deterministic" (node0 == again) + r.check "node pruning differs" (node0 != node2 && node2 != node31) + r.check "node0 first action is init at t=0" (match node0.head? with + | some action => + action.time == 0 && (match action.action with + | .initGossipSub _ => true + | _ => false) + | none => false) + let node0Publishes := node0.filterMap fun item => match item.action with + | .publish _ _ _ => some item.time + | _ => none + r.check "node0 has zero publishes in this fixture" (node0Publishes == []) + let publishes := node2.filterMap fun item => match item.action with + | .publish id size topic => some (item.time, id.toNat, size.toNat, topic) + | _ => none + r.check "node2 has three publishes" (publishes.length == 3) + r.check "node2 publish times and fields" (publishes == + [(168000, 4, 98304, "a-subnet"), (180000, 5, 98304, "a-subnet"), + (252000, 11, 98304, "a-subnet")]) + r.check "node 1 publish subset empty" (publishRowsForPlan script 1 == []) + r.check "node 2 publish subset exact" (publishRowsForPlan script 2 == + [(168000, 4, 98304, "a-subnet"), (180000, 5, 98304, "a-subnet"), + (252000, 11, 98304, "a-subnet")]) + r.check "node 3 publish subset exact" (publishRowsForPlan script 3 == + [(132000, 1, 98304, "a-subnet"), (264000, 12, 98304, "a-subnet")]) + r.check "node 16 publish subset exact" (publishRowsForPlan script 16 == + [(228000, 9, 98304, "a-subnet"), (288000, 14, 98304, "a-subnet")]) + r.check "node 31 publish subset empty" (publishRowsForPlan script 31 == []) + r.check "publish count per node" (((List.range 32).filterMap fun nodeId => + let count := (publishRowsFor script nodeId).length + if count == 0 then none else some (nodeId, count)) == + [(2,3), (3,2), (5,1), (6,1), (7,1), (8,1), (16,2), (17,1), (20,1), (27,1), (28,1), (30,1)]) + r.check "all 32 node plans succeed" ((List.range 32).all fun nodeId => + (plan script (Int.ofNat nodeId)).isOk) + let publishRows := allPublishRows script + let publishIds := publishRows.map (·.messageId) + r.check "fixture has exactly 16 publishes" (publishRows.length == 16) + r.check "publish IDs cover 0 through 15 exactly once" (publishIds.length == 16 && + publishIds.eraseDups.length == 16 && publishIds.all (fun id => id ≤ 15)) + r.check "all publish sizes and topics match fixture" (publishRows.all (fun row => row.size == 98304 && row.topic == "a-subnet")) + r.check "all 16 fixture publishes stay attached to their nodes" (allPublishRows script == + [⟨4, 2, 168000, 98304, "a-subnet"⟩, + ⟨5, 2, 180000, 98304, "a-subnet"⟩, + ⟨11, 2, 252000, 98304, "a-subnet"⟩, + ⟨1, 3, 132000, 98304, "a-subnet"⟩, + ⟨12, 3, 264000, 98304, "a-subnet"⟩, + ⟨10, 5, 240000, 98304, "a-subnet"⟩, + ⟨3, 6, 156000, 98304, "a-subnet"⟩, + ⟨15, 7, 300000, 98304, "a-subnet"⟩, + ⟨6, 8, 192000, 98304, "a-subnet"⟩, + ⟨9, 16, 228000, 98304, "a-subnet"⟩, + ⟨14, 16, 288000, 98304, "a-subnet"⟩, + ⟨8, 17, 216000, 98304, "a-subnet"⟩, + ⟨13, 20, 276000, 98304, "a-subnet"⟩, + ⟨0, 27, 120000, 98304, "a-subnet"⟩, + ⟨7, 28, 204000, 98304, "a-subnet"⟩, + ⟨2, 30, 144000, 98304, "a-subnet"⟩]) + r.check "node0 subscribe is full at t=0" (node0.any fun item => item.time == 0 && match item.action with + | .subscribeToTopic topic isPartial => topic == "a-subnet" && !isPartial + | _ => false) + r.check "validation delay is topic scoped at 5ms" (node0.any fun item => match item.action with + | .setTopicValidationDelay topic delay => topic == "a-subnet" && delay == 5 && item.time == 0 + | _ => false) + r.check "node0 connects at t=0" (node0.any fun item => item.time == 0 && match item.action with + | .connect _ => true + | _ => false) + r.check "node0 wait reaches 342s" (node0.any fun item => match item.action with + | .waitUntil deadline => deadline == 342 && item.time == 342000 + | _ => false) + r.check "node0 has no action after 342s" (node0.all (fun item => item.time ≤ 342000)) + r.check "node1 has full startup and wait sequence" (node1.length == 22 && + node1.head?.map (fun item => item.time == 0) == some true && + node1.getLast?.map (fun item => item.time == 342000) == some true) + r.check "node31 has full startup and wait sequence" (node31.length == 22 && + node31.head?.map (fun item => item.time == 0) == some true && + node31.getLast?.map (fun item => item.time == 342000) == some true) + let reverseWait := waitPlan "{\"script\":[{\"type\":\"initGossipSub\",\"gossipSubParams\":{}},{\"type\":\"waitUntil\",\"elapsedSeconds\":300},{\"type\":\"waitUntil\",\"elapsedSeconds\":100},{\"type\":\"connect\",\"connectTo\":[]}] }" + r.check "earlier wait barrier remains at t=300" (match reverseWait with + | some actions => nondecreasing actions && actions.length == 4 && + actions[2]?.map (·.time) == some 300000 && actions.getLast?.map (·.time) == some 300000 + | none => false) + r.check "action after earlier wait stays at t=300" (match reverseWait with + | some actions => actions.getLast?.map (fun item => match item.action with + | .connect _ => item.time == 300000 + | _ => false) == some true + | none => false) + let sameWait := waitPlan "{\"script\":[{\"type\":\"initGossipSub\",\"gossipSubParams\":{}},{\"type\":\"waitUntil\",\"elapsedSeconds\":300},{\"type\":\"waitUntil\",\"elapsedSeconds\":300}] }" + r.check "equal wait barrier is idempotent" (match sameWait with + | some actions => actions.length == 3 && actions[1]?.map (·.time) == some 300000 && + actions[2]?.map (·.time) == some 300000 + | none => false) + let zeroWait := waitPlan "{\"script\":[{\"type\":\"initGossipSub\",\"gossipSubParams\":{}},{\"type\":\"waitUntil\",\"elapsedSeconds\":0},{\"type\":\"connect\",\"connectTo\":[]}] }" + r.check "zero wait as first action adds no delay" (match zeroWait with + | some actions => actions.length == 3 && actions.all (fun item => item.time == 0) + | none => false) + r.check "positive waits never move virtual time backwards" (match plan script 0 with + | .ok actions => nondecreasing actions + | .error _ => false) + | _, _, _, _, _ => + r.check "virtual plan builds for nodes 0, 1, 2, and 31" false + +/-- The suite. -/ +def run (r : Runner) : IO Unit := do + checkScript r + checkErrors r + checkRecords r + checkIds r + checkParameters r + checkDecoderDetails r + checkHosts r + checkPlan r + +end Tests.Gossipsub.Interop diff --git a/Tests/Gossipsub/Node.lean b/Tests/Gossipsub/Node.lean new file mode 100644 index 00000000..73a63a73 --- /dev/null +++ b/Tests/Gossipsub/Node.lean @@ -0,0 +1,756 @@ +import Tests.Support +import Tests.Gossipsub.Identity +import Tests.Gossipsub.Switch +import Libp2p.Protocols.Gossipsub.Node + +/-! +# Gossipsub node loop checks + +The suite observes RPCs, deliveries, writer streams and close signals. A +receipt marker provides a queue barrier. TestClock timers belong to the +alarm and track deadlines. The suite never counts timer arms as core events. +Every receive has a real deadline. Stop checks reset their test stream on +expiry, so a failed check can still release its owned tasks. +-/ + +set_option autoImplicit false + +namespace Tests.Gossipsub.Node + +open Conc +open Libp2p.Protocols +open Libp2p.Protocols.Gossipsub +open Libp2p.Protocols.Gossipsub.Node +open Tests.Gossipsub (Switch) + +private def pa : Peer := Identity.peer 1 +private def pb : Peer := Identity.peer 2 +private def pc : Peer := Identity.peer 3 +private def pd : Peer := Identity.peer 4 +private def px : Peer := Identity.peer 9 +private def topic : TopicId := ⟨"t"⟩ +private def patience : Nat := 5000 + +private def message (data : List Nat) : Message := + { «from» := none, data := some data, seqno := none, topic, + signature := none, key := none } + +private def frame (rpc : Rpc.RPC) : ByteArray := + Bytes.ofNatList ((Rpc.encodeFrame? rpc).getD []) + +/-- A receive that fails at a real deadline instead of hanging the suite. -/ +private def receive {α : Type} (c : Chan α) : Async (Option α) := do + let timer ← (Park.sleep patience none : BaseIO (Park (Option α))) + select #[Park.recv c id, timer] + +/-- Wait for one matching value, and drop the values it reads before +it. The loop is `partial` because termination depends on the channel +contents, as with the IO read loops; the deadline of `receive` bounds +every wait. -/ +private partial def receiveWhere {α : Type} (c : Chan α) (wanted : α → Bool) : + Async (Option α) := do + match ← receive c with + | some value => if wanted value then pure (some value) else receiveWhere c wanted + | none => pure none + +private def marker : Rpc.RPC := subOptRpc ⟨"node-suite-barrier"⟩ false + +/-- A clock with a signal for the absolute time of each armed deadline. -/ +private structure ClockProbe where + test : TestClock + armed : Chan Nat + +private def ClockProbe.new : BaseIO ClockProbe := + return ⟨← TestClock.new, ← Chan.new 512⟩ + +private def ClockProbe.clock (c : ClockProbe) : Clock where + nowMs := c.test.clock.nowMs + after ms := do + let timer ← c.test.clock.after ms + discard <| c.armed.trySend ((← c.test.clock.nowMs) + ms) + return timer + +/-- Wait for one exact deadline. Earlier recorded arms may still be queued. -/ +private def armedAt (c : ClockProbe) (due : Nat) : Async Bool := do + for _ in [0:512] do + match ← receive c.armed with + | some t => if t = due then return true + | none => return false + return false + +private structure Seen where + receiver : Peer + peer : Peer + rpc : Rpc.RPC + +/-- A probe reads many frames on one persistent inbound stream. It also +records the end of each handler, so tests can distinguish close from open. -/ +private def probe (sw : Switch) (me : Peer) (seen : Chan Seen) + (ended : Chan Unit) : BaseIO HostSeam := do + let (host, _) ← sw.host me + host.setHandler { + id := Constants.meshsubId + handle := fun ctx stream => ExceptT.lift do + let sender := ctx.remotePeer + let mut buf : List Nat := [] + let mut done := false + repeat + match Rpc.decodeFrame? buf with + | .ok (some (body, rest)) => + buf := rest + if let .ok rpc := Core.decodeRpc body then + discard (seen.trySend { receiver := me, peer := sender, rpc } : BaseIO Bool) + | .error _ => break + | .ok none => + match ← stream.read with + | .ok (some bytes) => buf := buf ++ Bytes.toNatList bytes + | _ => done := true + if done then break + stream.close + discard (ended.trySend () : BaseIO Bool) } + return host + +private def openStream (host : HostSeam) (peer : Peer) : Async Streams.Stream := do + match ← host.newStream peer [Constants.meshsubId] with + | .ok (_, stream) => return stream + | .error e => throw <| IO.userError s!"open: {repr e}" + +/-- Each fixture has a raw control stream and a receipt marker. Sending the +marker after seam posts puts its receipt behind those posts in the queue. -/ +private structure Fixture where + node : Node + clock : ClockProbe + control : Streams.Stream + barriers : Chan Unit + receipts : Chan (Peer × List Nat) + deliveries : Chan Message + hostTasks : TaskSet + opens : OnceRef Nat + +private def fixture (sw : Switch) (me : Peer) (config : Config := {}) : Async Fixture := do + let clock ← (ClockProbe.new : BaseIO ClockProbe) + let (host, hostTasks) ← (sw.host me : BaseIO (HostSeam × TaskSet)) + let opens ← (OnceRef.new 0 : BaseIO (OnceRef Nat)) + let barriers ← (Chan.new 64 : BaseIO (Chan Unit)) + let receipts ← (Chan.new 2048 : BaseIO (Chan (Peer × List Nat))) + let deliveries ← (Chan.new 2048 : BaseIO (Chan Message)) + let controlPeer := Identity.peer (100 + me.hash.digest.headD 0) + let cfg := { config with + clock := clock.clock + order := fun tick peers => config.order tick (peers.filter (· != controlPeer)) + onRpc := fun p bytes => do + config.onRpc p bytes + if bytes == Pb.encodeMessage marker then discard <| barriers.trySend () + else discard <| receipts.trySend (p, bytes) } + let counted : HostSeam := { host with newStream := fun p ids => do + if p != controlPeer then (opens.update (· + 1) : BaseIO Unit) + host.newStream p ids } + let node ← (start counted cfg : BaseIO Node) + -- A raw control peer consumes its hello and has no router of its own. + let seen ← (Chan.new 2048 : BaseIO (Chan Seen)) + let ended ← (Chan.new 64 : BaseIO (Chan Unit)) + let controlHost ← (probe sw controlPeer seen ended : BaseIO HostSeam) + let control ← openStream controlHost me + return { node, clock, control, barriers, receipts, deliveries, hostTasks, opens } + +private def barrier (f : Fixture) : Async Bool := do + discard <| f.control.write (frame marker) + return (← receive f.barriers).isSome + +private def subscribe (f : Fixture) : BaseIO Bool := + f.node.subscribe topic (fun m => discard <| f.deliveries.trySend m) + +private def finish (sw : Switch) (fs : List Fixture) : Async Unit := do + for f in fs do + f.control.close + f.node.stop + (f.clock.test.close : BaseIO Unit) + sw.close + +private def delivered (f : Fixture) (data : List Nat) : Async Bool := do + return (← receive f.deliveries) == some (message data) + +/-- The switch serves handlers by peer and protocol. Half-close preserves the +read direction, and reset wakes a parked read. -/ +private def switchChecks (r : Tests.Runner) : Async Unit := do + let sw ← (Switch.new : BaseIO Switch) + let (a, _) ← (sw.host pa : BaseIO (HostSeam × TaskSet)) + let (b, _) ← (sw.host pb : BaseIO (HostSeam × TaskSet)) + let sender ← (Chan.new 8 : BaseIO (Chan Peer)) + let id : Libp2p.Core.ProtocolId := ⟨"/node-test-echo"⟩ + (b.setHandler { id, handle := fun ctx s => ExceptT.lift do + discard (sender.trySend ctx.remotePeer : BaseIO Bool) + repeat + match ← s.read with + | .ok (some bytes) => discard <| s.write bytes + | _ => break + s.close } : BaseIO Unit) + r.check "switch.unknown-peer" + (match ← a.newStream pc [id] with | .error (.protocol _) => true | _ => false) + r.check "switch.no-common-protocol" + (match ← a.newStream pb [⟨"/missing"⟩] with | .error (.protocol _) => true | _ => false) + r.check "switch.handler-belongs-to-its-peer" + (match ← b.newStream pa [id] with | .error (.protocol _) => true | _ => false) + match ← a.newStream pb [⟨"/missing"⟩, id] with + | .error _ => r.check "switch.negotiates-first-served-protocol" false + | .ok (chosen, s) => + r.check "switch.negotiates-first-served-protocol" (chosen == id) + discard <| s.write (Bytes.ofNatList [1,2,3]) + s.close + r.check "switch.handler-gets-sender" ((← receive sender) == some pa) + r.check "switch.half-close-keeps-read" + (match ← s.read with | .ok (some bytes) => Bytes.toNatList bytes == [1,2,3] | _ => false) + r.check "switch.write-after-close-fails" + (match ← s.write (Bytes.ofNatList [4]) with | .error _ => true | _ => false) + sw.close + sw.close + r.check "switch.repeated-close-returns" true + r.check "switch.no-open-after-close" + (match ← a.newStream pb [id] with | .error (.closed _) => true | _ => false) + +/-- A local publish has no deliver action in the core. The loop serves queued +inputs before stop, without a separate drain loop. -/ +private def localChecks (r : Tests.Runner) : Async Unit := do + let sw ← (Switch.new : BaseIO Switch) + let f ← fixture sw pa + discard (subscribe f : BaseIO Bool) + r.check "node.local.publish-accepted" (← (f.node.publish topic (Bytes.ofNatList [1]) : BaseIO Bool)) + r.check "node.local.publish-unsubscribed-accepted" + (← (f.node.publish ⟨"other"⟩ (Bytes.ofNatList [2]) : BaseIO Bool)) + r.check "node.local.barrier" (← barrier f) + r.check "node.local.no-delivery" ((← (f.deliveries.tryRecv : BaseIO (Option Message))).isNone) + r.check "node.local.no-stream" ((← (f.opens.get : BaseIO Nat)) == 0) + finish sw [f] + +/-- The system-clock alarm ends when the node stops, without an advance of a +test clock. A real heartbeat grafts its known peer. -/ +private def systemClockChecks (r : Tests.Runner) : Async Unit := do + let sw ← (Switch.new : BaseIO Switch) + let seen ← (Chan.new 64 : BaseIO (Chan Seen)) + let ended ← (Chan.new 16 : BaseIO (Chan Unit)) + discard (probe sw pb seen ended : BaseIO HostSeam) + let (host, _) ← (sw.host pa : BaseIO (HostSeam × TaskSet)) + let n ← (start host { params := { Core.defaults with heartbeatInterval := 20 } } : BaseIO Node) + discard (n.subscribe topic (fun _ => pure ()) : BaseIO Bool) + discard (n.peerJoined pb .meshsub : BaseIO Bool) + discard <| receive seen + let graft ← receive seen + r.check "node.system.alarm-sends-graft" + (graft.any (fun s => s.rpc.control.any (fun c => !c.graft.isEmpty))) + let result ← withinDeadline Clock.system patience (pure ()) n.stop + r.check "node.system.stop-closes-alarm" (match result with | .ok () => true | _ => false) + sw.close + +/-- A sender's first frame registers its peer before the core sees the RPC. +The same bytes twice reach the observer twice and deliver once. -/ +private def inboundChecks (r : Tests.Runner) : Async Unit := do + let sw ← (Switch.new : BaseIO Switch) + let f ← fixture sw pb + r.check "node.inbound.subscribe" (← (subscribe f : BaseIO Bool)) + r.check "node.inbound.setup-barrier" (← barrier f) + let seen ← (Chan.new 32 : BaseIO (Chan Seen)) + let ended ← (Chan.new 32 : BaseIO (Chan Unit)) + let source ← (probe sw pa seen ended : BaseIO HostSeam) + let stream ← openStream source pb + let bytes := frame (Core.publishRpc (message [7])) + discard <| stream.write (bytes ++ bytes ++ frame marker) + r.check "node.inbound.two-frames-with-one-id" (← delivered f [7]) + r.check "node.inbound.frames-before-marker" ((← receive f.barriers).isSome) + let first ← receive f.receipts + let second ← receive f.receipts + r.check "node.receipt.sees-both-identical-rpcs" + (first == some (pa, Pb.encodeMessage (Core.publishRpc (message [7]))) && first == second) + r.check "node.receipt.delivery-deduplicates" ((← (f.deliveries.tryRecv : BaseIO (Option Message))).isNone) + r.check "node.inbound.first-rpc-needs-no-notifiee" first.isSome + -- A split frame stays one RPC. The stream remains open for more frames. + for byte in Bytes.toNatList (frame (Core.publishRpc (message [8]))) do + discard <| stream.write (Bytes.ofNatList [byte]) + r.check "node.inbound.single-byte-chunks" (← delivered f [8]) + discard <| stream.write (Bytes.ofNatList [1, 255] ++ frame (Core.publishRpc (message [9]))) + r.check "node.inbound.bad-protobuf-keeps-the-loop" (← delivered f [9]) + -- Distinct messages in one chunk both deliver, in order. + discard <| stream.write (frame (Core.publishRpc (message [10])) ++ + frame (Core.publishRpc (message [11]))) + r.check "node.inbound.distinct-frames-first" (← delivered f [10]) + r.check "node.inbound.distinct-frames-second" (← delivered f [11]) + -- The empty frame is legal and does not end the stream. + discard <| stream.write (Bytes.ofNatList [0] ++ frame (Core.publishRpc (message [12]))) + r.check "node.inbound.zero-length-frame-keeps-stream-open" (← delivered f [12]) + stream.close + -- Drain the successful handler before testing a framing fault. + r.check "node.inbound.barrier-before-fault" (← barrier f) + let bad ← openStream source pb + discard <| bad.write (Bytes.ofNatList (Streams.Framing.encodeLength .uvarint (Constants.rpcCap + 1))) + -- Closing the successful streams lets the host join report this fault. + f.control.close + let failed ← try + f.hostTasks.joinAll + pure false + catch _ => pure true + r.check "node.inbound.over-cap-fails-handler" failed + r.check "node.inbound.over-cap-reset" (match ← bad.write (Bytes.ofNatList [1]) with + | .error _ => true | .ok _ => false) + f.control.close + f.node.stop + -- A bad handler error is reported by the host owner. + try sw.close catch _ => pure () + (f.clock.test.close : BaseIO Unit) + +/-- The core emits no local delivery for publish. These 20 trials use a peer +RPC immediately after subscribe, so they exercise callback registration +against a real delivery. -/ +private def subscribeRace (r : Tests.Runner) : Async Unit := do + let sw ← (Switch.new : BaseIO Switch) + let f ← fixture sw pb + let seen ← (Chan.new 64 : BaseIO (Chan Seen)) + let ended ← (Chan.new 16 : BaseIO (Chan Unit)) + let source ← (probe sw pa seen ended : BaseIO HostSeam) + let stream ← openStream source pb + let deliveries ← (Chan.new 32 : BaseIO (Chan Message)) + let mut successes := 0 + for i in [0:20] do + let t : TopicId := ⟨s!"subscribe-race-{i}"⟩ + let accepted ← (f.node.subscribe t (fun m => discard <| deliveries.trySend m) : BaseIO Bool) + let m := { message [i] with topic := t } + discard <| stream.write (frame (Core.publishRpc m)) + if accepted && (← receive deliveries) == some m then + successes := successes + 1 + r.check "node.surface.subscribe-then-publish-reaches-callback" (successes == 20) + stream.close + finish sw [f] + +/-- Direct handler boundary cases preserve the framing error channel. The +fake stream returns finite chunks and records reset without a parked task. -/ +private def frameBoundaryChecks (r : Tests.Runner) : Async Unit := do + let cases : List (String × List ByteArray) := + [ ("bad-prefix", [Bytes.ofNatList (List.replicate 12 128)]) + , ("eof-inside-prefix", [Bytes.ofNatList [128]]) + , ("eof-inside-body", [Bytes.ofNatList [5,1]]) + , ("large-malformed-chunk", [Bytes.ofNatList (List.replicate (Constants.rpcCap + 6) 128)]) ] + for (name, chunks) in cases do + let remaining ← (OnceRef.new chunks : BaseIO (OnceRef (List ByteArray))) + let reset ← (OnceRef.new false : BaseIO (OnceRef Bool)) + let events ← (Chan.new 16 : BaseIO (Chan Input)) + let stream : Streams.Stream := + { read := do + let next ← (remaining.updateGet (fun xs => (xs.head?, xs.drop 1)) : BaseIO (Option ByteArray)) + return .ok next + write := fun _ => pure (.ok ()) + close := pure () + reset := (reset.set true : BaseIO Unit) } + let result ← ((handler events).handle { remotePeer := pa, remoteAddr := Multiformats.Multiaddr.empty } stream).run + r.check s!"node.framing.{name}" + ((match result with | .error (.protocol _) => true | _ => false) && + (← (reset.get : BaseIO Bool))) + +/-- A failed outbound write retires only its writer. The existing inbound +stream can still publish, and a later send opens a new outbound writer. -/ +private def outboundFailureChecks (r : Tests.Runner) : Async Unit := do + let deliveries ← (Chan.new 8 : BaseIO (Chan Message)) + let receipts ← (Chan.new 8 : BaseIO (Chan Unit)) + let closed ← (Chan.new 8 : BaseIO (Chan Unit)) + let inbound ← (Pipe.new 16 : BaseIO Pipe) + let opens ← (OnceRef.new 0 : BaseIO (OnceRef Nat)) + let saved ← (OnceRef.new (none : Option Libp2p.Core.StreamHandler) : BaseIO (OnceRef (Option Libp2p.Core.StreamHandler))) + let host : HostSeam := { + setHandler := fun h => saved.set (some h) + newStream := fun _ _ => do + (opens.update (· + 1) : BaseIO Unit) + return .ok (Constants.meshsubId, { + read := pure (.ok none) + write := fun _ => pure (.error .closed) + close := discard (closed.trySend () : BaseIO Bool) + reset := pure () }) } + let tc ← (TestClock.new : BaseIO TestClock) + let n ← (start host { clock := tc.clock, onRpc := fun _ _ => discard (receipts.trySend ()) } : BaseIO Node) + discard (n.subscribe topic (fun m => discard <| deliveries.trySend m) : BaseIO Bool) + let some h ← (saved.get : BaseIO (Option Libp2p.Core.StreamHandler)) + | throw <| IO.userError "missing node handler" + let owner ← (TaskSet.new : BaseIO TaskSet) + let stream : Streams.Stream := { + read := return .ok (← inbound.read) + write := fun _ => pure (.ok ()) + close := pure () + reset := (inbound.close : BaseIO Unit) } + owner.spawn do + discard <| (h.handle { remotePeer := pa, remoteAddr := Multiformats.Multiaddr.empty } stream).run + discard <| inbound.write (frame (Core.publishRpc (message [31]))) + r.check "node.failure.first-inbound-publish-delivers" + ((← receive deliveries) == some (message [31])) + r.check "node.failure.outbound-write-failed" ((← receive closed).isSome) + -- The close signal follows retirement, so the second publish sees the + -- post-failure peer state while using the same inbound stream. + discard <| inbound.write (frame (Core.publishRpc (message [32]))) + r.check "node.failure.second-inbound-publish-survives-outbound-failure" + ((← receive deliveries) == some (message [32])) + discard (n.subscribe ⟨"reopen"⟩ (fun _ => pure ()) : BaseIO Bool) + r.check "node.failure.next-send-opens-fresh-writer" ((← receive closed).isSome) + r.check "node.failure.two-outbound-opens" ((← (opens.get : BaseIO Nat)) == 2) + (inbound.close : BaseIO Unit) + owner.joinAll + n.stop + (tc.close : BaseIO Unit) + +/-- A legal body arrives one byte per read. The direct handler queue records +all frame bodies in order, without a router changing or deduplicating them. -/ +private def readerCopyChecks (r : Tests.Runner) : Async Unit := do + let first := Core.publishRpc (message (List.replicate 8192 17)) + let second := Core.publishRpc (message [18]) + let third := Core.publishRpc (message [19]) + let all := frame first ++ frame second ++ frame third + let cursor ← (OnceRef.new 0 : BaseIO (OnceRef Nat)) + let events ← (Chan.new 16 : BaseIO (Chan Input)) + let dribble : Streams.Stream := { + read := do + let i ← (cursor.updateGet (fun i => (i, i + 1)) : BaseIO Nat) + return .ok (if i < all.size then some (all.extract i (i + 1)) else none) + write := fun _ => pure (.ok ()) + close := pure () + reset := pure () } + let result ← ((handler events).handle { remotePeer := pa, remoteAddr := Multiformats.Multiaddr.empty } dribble).run + let inputs ← (events.drain : BaseIO (List Input)) + let bodies := inputs.filterMap fun + | .event (.rpc _ bytes) => some bytes + | _ => none + r.check "node.reader.many-one-byte-chunks-preserve-all-frames" + ((match result with | .ok () => true | _ => false) && + bodies == [Pb.encodeMessage first, Pb.encodeMessage second, Pb.encodeMessage third]) + let split := frame third + let chunks := [frame first ++ frame second ++ split.extract 0 1, + split.extract 1 3, split.extract 3 split.size] + let pending ← (OnceRef.new chunks : BaseIO (OnceRef (List ByteArray))) + let mixedEvents ← (Chan.new 16 : BaseIO (Chan Input)) + let mixed := { dribble with read := do + let bytes ← (pending.updateGet (fun xs => (xs.head?, xs.drop 1)) : BaseIO (Option ByteArray)) + return .ok bytes } + let mixedResult ← ((handler mixedEvents).handle { remotePeer := pa, remoteAddr := Multiformats.Multiaddr.empty } mixed).run + let mixedInputs ← (mixedEvents.drain : BaseIO (List Input)) + let mixedBodies := mixedInputs.filterMap fun + | .event (.rpc _ bytes) => some bytes + | _ => none + r.check "node.reader.two-frames-then-split-frame-preserve-order" + ((match mixedResult with | .ok () => true | _ => false) && mixedBodies == bodies) + +/-- Two real nodes learn subscriptions and deliver a publish without a +heartbeat or an explicit peerJoined call at the receiver. -/ +private def pairChecks (r : Tests.Runner) : Async Unit := do + let sw ← (Switch.new : BaseIO Switch) + let a ← fixture sw pa + let b ← fixture sw pb + discard (subscribe a : BaseIO Bool) + r.check "node.pair.subscribe-barrier" (← barrier a) + discard (b.node.peerJoined pa .meshsub : BaseIO Bool) + -- Wait for A's hello at B through its receipt observer. + r.check "node.pair.hello-reaches-publisher" ((← receive b.receipts).isSome) + discard (b.node.publish topic (Bytes.ofNatList [1]) : BaseIO Bool) + r.check "node.pair.publish-delivers" (← delivered a [1]) + discard (a.node.unsubscribe topic : BaseIO Bool) + r.check "node.pair.unsubscribe-barrier" (← barrier a) + discard (b.node.publish topic (Bytes.ofNatList [2]) : BaseIO Bool) + r.check "node.pair.publisher-barrier" (← barrier b) + r.check "node.pair.no-local-delivery" ((← (b.deliveries.tryRecv : BaseIO (Option Message))).isNone) + finish sw [a, b] + +/-- A subscription made before a peer joins is sent once to that peer. -/ +private def postJoinSubscriptionChecks (r : Tests.Runner) : Async Unit := do + let sw ← (Switch.new : BaseIO Switch) + let f ← fixture sw pa + discard (subscribe f : BaseIO Bool) + r.check "node.post-join-subscription.local-barrier" (← barrier f) + let seen ← (Chan.new 32 : BaseIO (Chan Seen)) + let ended ← (Chan.new 16 : BaseIO (Chan Unit)) + let source ← (probe sw pb seen ended : BaseIO HostSeam) + let stream ← openStream source pa + let first ← receive seen + r.check "node.post-join-subscription.receives-topic" + (first.map (fun s => s.rpc) == some (subOptRpc topic true)) + let markerTopic : TopicId := ⟨"post-join-marker"⟩ + discard (f.node.subscribe markerTopic (fun _ => pure ()) : BaseIO Bool) + r.check "node.post-join-subscription.marker-barrier" (← barrier f) + let second ← receive seen + r.check "node.post-join-subscription.exactly-one" + (second.map (fun s => s.rpc) == some (subOptRpc markerTopic true) && + (← (seen.tryRecv : BaseIO (Option Seen))).isNone) + stream.close + discard (f.node.peerLeft pb : BaseIO Bool) + finish sw [f] + +/-- The loop announces the topic before grafting its already announced peer. +Repeat joins emit no hello. Multiple callbacks preserve registration order. -/ +private def surfaceChecks (r : Tests.Runner) : Async Unit := do + let sw ← (Switch.new : BaseIO Switch) + let f ← fixture sw pa + let seen ← (Chan.new 32 : BaseIO (Chan Seen)) + let ended ← (Chan.new 16 : BaseIO (Chan Unit)) + let source ← (probe sw pb seen ended : BaseIO HostSeam) + let stream ← openStream source pa + discard <| stream.write (frame (Core.helloRpc [topic]) ++ frame marker) + r.check "node.surface.remote-topic-ready" ((← receive f.barriers).isSome) + let hello ← receive seen + r.check "node.surface.initial-hello" (hello.any (fun s => s.rpc.subscriptions.isEmpty)) + discard (f.node.peerJoined pb .meshsub : BaseIO Bool) + r.check "node.surface.repeat-join-barrier" (← barrier f) + r.check "node.surface.repeat-join-emits-no-hello" ((← (seen.tryRecv : BaseIO (Option Seen))).isNone) + let tags ← (Chan.new 16 : BaseIO (Chan Nat)) + discard (f.node.subscribe topic (fun _ => discard <| tags.trySend 1) : BaseIO Bool) + let record ← receive seen + let graft ← receive seen + r.check "node.surface.announcement-precedes-graft" + (record.any (fun s => s.rpc == subOptRpc topic true) && + graft.any (fun s => s.rpc.control.any (fun c => !c.graft.isEmpty))) + discard (f.node.subscribe topic (fun _ => discard <| tags.trySend 2) : BaseIO Bool) + discard (f.node.subscribe ⟨"other"⟩ (fun _ => discard <| tags.trySend 9) : BaseIO Bool) + r.check "node.surface.callback-registration-barrier" (← barrier f) + discard <| stream.write (frame (Core.publishRpc (message [20])) ++ frame marker) + r.check "node.surface.callback-rpc-barrier" ((← receive f.barriers).isSome) + let first ← receive tags + let second ← receive tags + r.check "node.surface.callbacks-in-registration-order" (first == some 1 && second == some 2) + r.check "node.surface.other-topic-callback-silent" ((← (tags.tryRecv : BaseIO (Option Nat))).isNone) + discard (seen.drain : BaseIO (List Seen)) + discard (f.node.unsubscribe topic : BaseIO Bool) + r.check "node.surface.unsubscribe-barrier" (← barrier f) + let off ← receiveWhere seen (fun s => s.rpc == subOptRpc topic false) + r.check "node.surface.negative-subscription-record" off.isSome + discard <| stream.write (frame (Core.publishRpc (message [21])) ++ frame marker) + r.check "node.surface.post-unsubscribe-barrier" ((← receive f.barriers).isSome) + r.check "node.surface.unsubscribe-removes-callbacks" ((← (tags.tryRecv : BaseIO (Option Nat))).isNone) + stream.close + discard (f.node.peerLeft pb : BaseIO Bool) + r.check "node.surface.leave-barrier" (← barrier f) + r.check "node.surface.leave-ends-peer-handler" ((← receive ended).isSome) + finish sw [f] + +private def gossipParams : Core.Params := + { Core.defaults with + heartbeatInterval := 100 + mesh := { d := 0, dLow := 0, dHigh := 0, dScore := 0, dLazy := 2 } } + +private def ihave (s : Seen) : Bool := hasIhave s.rpc + +/-- Explicit delays pin both sides of a deadline and cross a heartbeat. +The run-wide index makes the second heartbeat's first delay 30 ms. -/ +private def trickleChecks (r : Tests.Runner) : Async Unit := do + let sw ← (Switch.new : BaseIO Switch) + let seen ← (Chan.new 64 : BaseIO (Chan Seen)) + let ended ← (Chan.new 16 : BaseIO (Chan Unit)) + discard (probe sw pc seen ended : BaseIO HostSeam) + discard (probe sw pd seen ended : BaseIO HostSeam) + let f ← fixture sw pa { params := gossipParams, trickle := fun i => (i + 1) * 10 } + discard (subscribe f : BaseIO Bool) + discard (f.node.peerJoined pc .meshsub : BaseIO Bool) + discard (f.node.peerJoined pd .meshsub : BaseIO Bool) + discard (f.node.publish topic (Bytes.ofNatList [7]) : BaseIO Bool) + r.check "node.trickle.setup" (← barrier f) + discard <| receive seen + discard <| receive seen + r.check "node.trickle.initial-deadline" (← armedAt f.clock 100) + (f.clock.test.advance 100 : BaseIO Unit) + r.check "node.trickle.first-deadline" (← armedAt f.clock 110) + r.check "node.trickle.no-send-at-tick" ((← (seen.tryRecv : BaseIO (Option Seen))).isNone) + (f.clock.test.advance 9 : BaseIO Unit) + r.check "node.trickle.no-send-at-9" ((← (seen.tryRecv : BaseIO (Option Seen))).isNone) + (f.clock.test.advance 1 : BaseIO Unit) + let one ← receive seen + r.check "node.trickle.first-at-10" (one.any ihave) + r.check "node.trickle.second-deadline" (← armedAt f.clock 120) + (f.clock.test.advance 11 : BaseIO Unit) + let two ← receive seen + r.check "node.trickle.advance-11-serves-second" (two.any ihave) + r.check "node.trickle.two-distinct-receivers" + (match one, two with | some x, some y => x.receiver != y.receiver | _, _ => false) + r.check "node.trickle.next-heartbeat" (← armedAt f.clock 200) + (f.clock.test.advance 79 : BaseIO Unit) + r.check "node.trickle.run-wide-first-index" (← armedAt f.clock 230) + r.check "node.trickle.second-heartbeat-waits" ((← (seen.tryRecv : BaseIO (Option Seen))).isNone) + (f.clock.test.advance 29 : BaseIO Unit) + r.check "node.trickle.no-send-at-29" ((← (seen.tryRecv : BaseIO (Option Seen))).isNone) + (f.clock.test.advance 1 : BaseIO Unit) + r.check "node.trickle.second-heartbeat-first-at-30" ((← receive seen).any ihave) + r.check "node.trickle.second-heartbeat-second-deadline" (← armedAt f.clock 240) + (f.clock.test.advance 10 : BaseIO Unit) + r.check "node.trickle.cross-heartbeat-order" ((← receive seen).any ihave) + finish sw [f] + +private def defaultAndOrderChecks (r : Tests.Runner) : Async Unit := do + r.check "node.trickle.default-zero" ((List.range 16).all (fun i => ({} : Config).trickle i == 0)) + r.check "node.trickle.spread-below-one-second" ((List.range 1000).all (fun i => spread i < 1000)) + r.check "node.tick.order-changes" (defaultOrder 0 [pa,pb,pc,pd] != defaultOrder 1 [pa,pb,pc,pd]) + r.check "node.tick.order-replays" (defaultOrder 7 [pa,pb,pc,pd] == defaultOrder 7 [pa,pb,pc,pd]) + let sw ← (Switch.new : BaseIO Switch) + let seen ← (Chan.new 64 : BaseIO (Chan Seen)) + let ended ← (Chan.new 16 : BaseIO (Chan Unit)) + discard (probe sw pc seen ended : BaseIO HostSeam) + let f ← fixture sw pa { params := gossipParams } + discard (subscribe f : BaseIO Bool) + discard (f.node.peerJoined pc .meshsub : BaseIO Bool) + discard (f.node.publish topic (Bytes.ofNatList [4]) : BaseIO Bool) + r.check "node.default.setup" (← barrier f) + discard <| receive seen + r.check "node.default.timer" (← armedAt f.clock 100) + (f.clock.test.advance 100 : BaseIO Unit) + r.check "node.default.gossip-at-tick" ((← receive seen).any ihave) + finish sw [f] + +/-- Long delays reach the pending bound across heartbeats. Excess entries +increase the same drop counter as full writer queues. -/ +private def pendingBound (r : Tests.Runner) : Async Unit := do + let sw ← (Switch.new : BaseIO Switch) + let seen ← (Chan.new 64 : BaseIO (Chan Seen)) + let ended ← (Chan.new 16 : BaseIO (Chan Unit)) + discard (probe sw pc seen ended : BaseIO HostSeam) + discard (probe sw pd seen ended : BaseIO HostSeam) + let params := { gossipParams with window := { historyLength := 300, historyGossip := 300, valid := by decide } } + let f ← fixture sw pa { params, trickle := fun _ => 1000000 } + discard (subscribe f : BaseIO Bool) + discard (f.node.peerJoined pc .meshsub : BaseIO Bool) + discard (f.node.peerJoined pd .meshsub : BaseIO Bool) + discard (f.node.publish topic (Bytes.ofNatList [5]) : BaseIO Bool) + r.check "node.pending.setup" (← barrier f) + r.check "node.pending.initial-alarm" (← armedAt f.clock 100) + let mut armed := true + for i in [0:130] do + (f.clock.test.advance 100 : BaseIO Unit) + if !(← armedAt f.clock ((i + 2) * 100)) then armed := false + r.check "node.pending.every-heartbeat-ran" armed + r.check "node.pending.bound-256-drops-four" ((← (f.node.dropped : BaseIO Nat)) == 4) + finish sw [f] + +/-- A parked writer leaves the loop free. Sixty offers fill the remaining +slots of its 32-frame queue. Stop resets the stream and returns on deadline. -/ +private def stalledAndDrops (r : Tests.Runner) : Async Unit := do + let sw ← (Switch.new : BaseIO Switch) + let reached ← (Chan.new 1 : BaseIO (Chan Unit)) + let parked ← (Chan.new 1 : BaseIO (Chan Unit)) + let opened ← (OnceRef.new 0 : BaseIO (OnceRef Nat)) + let resets ← (OnceRef.new 0 : BaseIO (OnceRef Nat)) + let receipts ← (Chan.new 128 : BaseIO (Chan Unit)) + let (_, controlTasks) ← (sw.host pa : BaseIO (HostSeam × TaskSet)) + let (base, _) ← (sw.host pb : BaseIO (HostSeam × TaskSet)) + let host : HostSeam := { base with newStream := fun peer ids => do + if peer = pc then + (opened.update (· + 1) : BaseIO Unit) + return .ok (Constants.meshsubId, { + read := pure (.ok none) + write := fun _ => do + discard (reached.trySend () : BaseIO Bool) + discard parked.recv + return .error .closed + close := pure () + reset := do + (resets.update (· + 1) : BaseIO Unit) + (parked.close : BaseIO Unit) }) + base.newStream peer ids } + let tc ← (TestClock.new : BaseIO TestClock) + let n ← (start host { clock := tc.clock, onRpc := fun _ _ => discard (receipts.trySend ()) } : BaseIO Node) + discard (n.peerJoined pc .meshsub : BaseIO Bool) + r.check "node.stall.writer-reached-write" ((← receive reached).isSome) + -- The first hello is parked in write. Each new subscription adds one frame. + for i in [0:60] do + discard (n.subscribe ⟨s!"topic-{i}"⟩ (fun _ => pure ()) : BaseIO Bool) + -- A raw marker is ordered after the 60 offers. + let seen ← (Chan.new 16 : BaseIO (Chan Seen)) + let ended ← (Chan.new 16 : BaseIO (Chan Unit)) + let control ← (probe sw pa seen ended : BaseIO HostSeam) + let stream ← openStream control pb + discard <| stream.write (frame marker) + r.check "node.stall.loop-serves-marker" ((← receive receipts).isSome) + let drops ← (n.dropped : BaseIO Nat) + r.check "node.queue.capacity-32-drops-28" (drops == 28) + IO.println s!"node.measure.full-queue drops={drops}" + let stopped ← withinDeadline Clock.system patience parked.close n.stop + r.check "node.stall.stop-returns-before-deadline" (match stopped with | .ok () => true | _ => false) + r.check "node.stall.stop-reset-the-stream" (0 < (← (resets.get : BaseIO Nat))) + n.stop + n.stop + n.stop + r.check "node.stop.repeated-calls-return" true + r.check "node.stop.publish-refused" (!(← (n.publish topic (Bytes.ofNatList [1]) : BaseIO Bool))) + r.check "node.stop.subscribe-refused" (!(← (n.subscribe topic (fun _ => pure ()) : BaseIO Bool))) + r.check "node.stop.unsubscribe-refused" (!(← (n.unsubscribe topic : BaseIO Bool))) + r.check "node.stop.join-refused" (!(← (n.peerJoined pc .meshsub : BaseIO Bool))) + r.check "node.stop.leave-refused" (!(← (n.peerLeft pc : BaseIO Bool))) + stream.close + sw.close + controlTasks.joinAll + (tc.close : BaseIO Unit) + +/-- Count adjacent sequence inversions in one publisher's received messages. -/ +private def inversions : List Nat → Nat + | a :: b :: rest => (if b < a then 1 else 0) + inversions (b :: rest) + | _ => 0 + +/-- Two publishers run at once in bursts of 16 messages. Each waits for the +burst's callbacks before its next burst, so the 32-slot peer queue has room. +Both writers keep one stream and preserve their peer order. -/ +private def publisherOrder (r : Tests.Runner) : Async Unit := do + let sw ← (Switch.new : BaseIO Switch) + let a ← fixture sw pa + let b ← fixture sw pb + let sink ← fixture sw pc + let ackA ← (Chan.new 32 : BaseIO (Chan Unit)) + let ackB ← (Chan.new 32 : BaseIO (Chan Unit)) + let rows ← (OnceRef.new ([] : List (Nat × Nat)) : BaseIO (OnceRef (List (Nat × Nat)))) + discard (sink.node.subscribe topic (fun m => do + let data := m.data.getD [] + let from_ := data[0]?.getD 0 + let seq := data[1]?.getD 0 + 256 * data[2]?.getD 0 + rows.update (· ++ [(from_, seq)]) + discard <| (if from_ = 1 then ackA else ackB).trySend ()) : BaseIO Bool) + r.check "node.order.sink-ready" (← barrier sink) + discard (a.node.peerJoined pc .meshsub : BaseIO Bool) + discard (b.node.peerJoined pc .meshsub : BaseIO Bool) + -- Each publisher must see the sink's hello before its first publish. + r.check "node.order.a-has-subscription" ((← receive a.receipts).isSome) + r.check "node.order.b-has-subscription" ((← receive b.receipts).isSome) + let owner ← (TaskSet.new : BaseIO TaskSet) + let produce (f : Fixture) (id : Nat) (ack : Chan Unit) : Async Unit := do + for batch in [0:32] do + let count := Nat.min 16 (500 - batch * 16) + for j in [0:count] do + let i := batch * 16 + j + if !(← (f.node.publish topic (Bytes.ofNatList [id, i % 256, i / 256]) : BaseIO Bool)) then + throw <| IO.userError "publisher event refused" + for _ in [0:count] do + if (← receive ack).isNone then throw <| IO.userError "publisher delivery timeout" + owner.spawn (produce a 1 ackA) + owner.spawn (produce b 2 ackB) + owner.joinAll + let data ← (rows.get : BaseIO (List (Nat × Nat))) + let seqA := (data.filter (·.1 == 1)).map (·.2) + let seqB := (data.filter (·.1 == 2)).map (·.2) + let ia := inversions seqA + let ib := inversions seqB + let oa ← (a.opens.get : BaseIO Nat) + let ob ← (b.opens.get : BaseIO Nat) + r.check "node.order.both-publishers-deliver-500" (seqA.length == 500 && seqB.length == 500) + r.check "node.order.zero-adjacent-inversions-per-peer" (ia == 0 && ib == 0) + r.check "node.order.one-stream-per-publisher" (oa == 1 && ob == 1) + r.check "node.order.bursts-drop-no-frames" + ((← (a.node.dropped : BaseIO Nat)) == 0 && (← (b.node.dropped : BaseIO Nat)) == 0) + IO.println s!"node.measure.publishers countA={seqA.length} countB={seqB.length} streamsA={oa} streamsB={ob} inversionsA={ia} inversionsB={ib}" + finish sw [a,b,sink] + +/-- Run the node regressions and print the measured queue and ordering counts. -/ +def run (r : Tests.Runner) : IO Unit := do + r.section "gossipsub Node" "packages/libp2p/protocols/docs/PLAN.md, unit 3" + Async.run do + -- Inbound reset checks use a separate host task owner below. + switchChecks r + localChecks r + systemClockChecks r + inboundChecks r + frameBoundaryChecks r + outboundFailureChecks r + readerCopyChecks r + pairChecks r + postJoinSubscriptionChecks r + surfaceChecks r + subscribeRace r + trickleChecks r + defaultAndOrderChecks r + pendingBound r + stalledAndDrops r + publisherOrder r + +end Tests.Gossipsub.Node diff --git a/Tests/Gossipsub/Rpc.lean b/Tests/Gossipsub/Rpc.lean new file mode 100644 index 00000000..1173afe8 --- /dev/null +++ b/Tests/Gossipsub/Rpc.lean @@ -0,0 +1,327 @@ +import Tests.Support +import Libp2p.Protocols.Gossipsub.Rpc + +/-! +# Gossipsub suite: the RPC codec + +The wire messages of gossipsub v1.1, checked in both directions from the +field tables of the vendored spec, plus the strictness and rejection +cases and the length-delimited frame step. + +The repository carries no gossipsub vector file (`PLAN`, "Vector +stance"). Fixed message bytes close the codec family. Each expected byte +string below carries a comment that derives it from the field numbers. + +A field tag is `field number * 8 + wire type`. The wire types here are 0 +for a varint and 2 for a byte string. A byte string tag is followed by +its length and then its bytes. +-/ + +set_option autoImplicit false + +namespace Tests.Gossipsub.Rpc + +open Libp2p.Protocols +open Libp2p.Protocols.Gossipsub +open Libp2p.Protocols.Gossipsub.Rpc + +/-! ## Fixed message samples -/ + +/-- A subscription of `foo`. -/ +private def subFoo : SubOpts := { subscribe := true, topicid := "foo" } + +/-- A message on topic `t` with no optional field. -/ +private def msgT : Message := + { «from» := none, data := none, seqno := none, topic := ⟨"t"⟩, signature := none, key := none } + +/-- A message with every field of the spec. -/ +private def fullMsg : Message := + { «from» := some [1], data := some [2], seqno := some [3], topic := ⟨"t"⟩, + signature := some [4], key := some [5] } + +/-- A peer with the id bytes `[1, 2]` and no signed record. -/ +private def peer12 : PeerInfo := { peerId := some [1, 2], signedPeerRecord := none } + +/-- A prune of `g` with one peer and a backoff of 60 seconds. -/ +private def pruneG : ControlPrune := { topicId := "g", peers := [peer12], backoff := some 60 } + +/-- A control envelope with one prune. -/ +private def controlPrune : ControlMessage := + { ihave := [], iwant := [], graft := [], prune := [pruneG] } + +/-- An RPC with one subscription and one published message. -/ +private def rpcSubPublish : RPC := + { subscriptions := [subFoo], publish := [msgT], control := none } + +/-- An RPC with one control envelope. -/ +private def rpcControl : RPC := + { subscriptions := [], publish := [], control := some controlPrune } + +/-- An IHAVE of topic `t` with two message ids. -/ +private def ihaveT : ControlIHave := + { topicId := "t", messageIds := [⟨[1]⟩, ⟨[2, 3]⟩] } + +/-! ## Check helpers -/ + +/-- The model decode returns the expected value and consumes all bytes. -/ +private def decodesAs {α : Type} [Pb.ProtoMessage α] [BEq α] (bs : List Nat) (want : α) : Bool := + match Pb.decodeMessage? (α := α) bs with + | .ok (v, []) => v == want + | _ => false + +/-- The value encodes to the expected bytes. -/ +private def encodesAs {α : Type} [Pb.ProtoMessage α] [BEq α] (v : α) (bs : List Nat) : Bool := + Pb.encodeMessage v == bs + +/-- The model decode fails with the expected error. -/ +private def rejectsWith {α : Type} [Pb.ProtoMessage α] (bs : List Nat) (e : Pb.Error) : Bool := + match Pb.decodeMessage? (α := α) bs with + | .error e' => e' == e + | .ok _ => false + +/-- The model decode succeeds, whatever the value. -/ +private def accepts {α : Type} [Pb.ProtoMessage α] (bs : List Nat) : Bool := + match Pb.decodeMessage? (α := α) bs with + | .ok _ => true + | .error _ => false + +/-- The production decoder agrees with the model decoder on value, rest, +and consumed count. -/ +private def productionAgrees {α : Type} [Pb.ProtoMessage α] [BEq α] (bs : List Nat) : Bool := + match Pb.decodeMessage? (α := α) bs, Pb.decodeMessageB? (α := α) (Bytes.ofNatList bs) with + | .ok (v, rest), .ok (w, used) => v == w && rest == [] && used == bs.length + | .error e, .error e' => e == e' + | _, _ => false + +/-- The frame step returns one payload and the rest after it. -/ +private def frameIs (bytes payload rest : List Nat) : Bool := + match decodeFrame? bytes with + | .ok (some (p, r)) => p == payload && r == rest + | _ => false + +/-- The frame step needs more bytes. -/ +private def frameNeedsMore (bytes : List Nat) : Bool := + match decodeFrame? bytes with + | .ok none => true + | _ => false + +/-- The frame step fails with the expected error. -/ +private def frameFails (bytes : List Nat) (e : Pb.Error) : Bool := + match decodeFrame? bytes with + | .error e' => e' == e + | _ => false + +/-! ## Order samples -/ + +private def subA : SubOpts := { subscribe := false, topicid := "a" } +private def subB : SubOpts := { subscribe := true, topicid := "b" } + +private def msgA : Message := + { «from» := none, data := some [7], seqno := none, topic := ⟨"x"⟩, signature := none, key := none } + +private def msgB : Message := + { «from» := some [8], data := none, seqno := none, topic := ⟨"y"⟩, signature := none, key := none } + +private def peerA : PeerInfo := { peerId := some [1], signedPeerRecord := none } + +private def peerB : PeerInfo := { peerId := some [2], signedPeerRecord := some [9] } + +/-- The subscriptions of a decoded RPC, when the decode succeeds. -/ +private def rpcSubscriptions (bs : List Nat) : Option (List SubOpts) := + match Pb.decodeMessage? (α := RPC) bs with + | .ok (r, []) => some r.subscriptions + | _ => none + +/-- The published messages of a decoded RPC, when the decode succeeds. -/ +private def rpcPublish (bs : List Nat) : Option (List Message) := + match Pb.decodeMessage? (α := RPC) bs with + | .ok (r, []) => some r.publish + | _ => none + +/-- The message ids of a decoded IHAVE, when the decode succeeds. -/ +private def ihaveIds (bs : List Nat) : Option (List MessageId) := + match Pb.decodeMessage? (α := ControlIHave) bs with + | .ok (i, []) => some i.messageIds + | _ => none + +/-- The message ids of a decoded IWANT, when the decode succeeds. -/ +private def iwantIds (bs : List Nat) : Option (List MessageId) := + match Pb.decodeMessage? (α := ControlIWant) bs with + | .ok (i, []) => some i.messageIds + | _ => none + +/-- The peers of a decoded PRUNE, when the decode succeeds. -/ +private def prunePeers (bs : List Nat) : Option (List PeerInfo) := + match Pb.decodeMessage? (α := ControlPrune) bs with + | .ok (p, []) => some p.peers + | _ => none + +/-- Run the checks of this module. -/ +def run (r : Runner) : IO Unit := do + r.section "gossipsub Rpc" + "vendor/specs/pubsub (README, gossipsub-v1.0.md, gossipsub-v1.1.md)" + + -- SubOpts: field 1 varint tag is 8, value 1 is true; field 2 bytes tag + -- is 18, length 3, and "foo" is [102, 111, 111]. + r.check "spec SubOpts subscribe foo decodes" + (decodesAs (α := SubOpts) [8, 1, 18, 3, 102, 111, 111] subFoo) + r.check "spec SubOpts subscribe foo encodes byte-exact" + (encodesAs subFoo [8, 1, 18, 3, 102, 111, 111]) + + -- Message: field 4 bytes tag is 34, length 1, and "t" is 116. + r.check "spec Message topic t decodes" + (decodesAs (α := Message) [34, 1, 116] msgT) + r.check "spec Message topic t encodes byte-exact" + (encodesAs msgT [34, 1, 116]) + + -- Full Message: field 1 tag 10, field 2 tag 18, field 3 tag 26, field 4 + -- tag 34, field 5 tag 42, field 6 tag 50. Each payload has length 1. + r.check "spec full Message decodes" + (decodesAs (α := Message) + [10, 1, 1, 18, 1, 2, 26, 1, 3, 34, 1, 116, 42, 1, 4, 50, 1, 5] fullMsg) + r.check "spec full Message encodes byte-exact" + (encodesAs fullMsg [10, 1, 1, 18, 1, 2, 26, 1, 3, 34, 1, 116, 42, 1, 4, 50, 1, 5]) + + -- RPC: field 1 bytes tag is 10, the SubOpts payload has length 7; + -- field 2 bytes tag is 18, the Message payload has length 3. + r.check "spec RPC subscription and publish decodes" + (decodesAs (α := RPC) + [10, 7, 8, 1, 18, 3, 102, 111, 111, 18, 3, 34, 1, 116] rpcSubPublish) + r.check "spec RPC subscription and publish encodes byte-exact" + (encodesAs rpcSubPublish [10, 7, 8, 1, 18, 3, 102, 111, 111, 18, 3, 34, 1, 116]) + + -- ControlPrune: field 1 tag 10, field 2 bytes tag 18 with the PeerInfo + -- payload [10, 2, 1, 2], field 3 varint tag is 24, value 60. + r.check "spec ControlPrune topic peers backoff decodes" + (decodesAs (α := ControlPrune) [10, 1, 103, 18, 4, 10, 2, 1, 2, 24, 60] pruneG) + r.check "spec ControlPrune topic peers backoff encodes byte-exact" + (encodesAs pruneG [10, 1, 103, 18, 4, 10, 2, 1, 2, 24, 60]) + + -- ControlMessage: field 4 bytes tag is 34, the ControlPrune payload has + -- length 11. + r.check "spec ControlMessage with prune decodes" + (decodesAs (α := ControlMessage) + [34, 11, 10, 1, 103, 18, 4, 10, 2, 1, 2, 24, 60] controlPrune) + r.check "spec ControlMessage with prune encodes byte-exact" + (encodesAs controlPrune [34, 11, 10, 1, 103, 18, 4, 10, 2, 1, 2, 24, 60]) + + -- RPC: field 3 bytes tag is 26, the ControlMessage payload has length 13. + r.check "spec RPC with control decodes" + (decodesAs (α := RPC) + [26, 13, 34, 11, 10, 1, 103, 18, 4, 10, 2, 1, 2, 24, 60] rpcControl) + r.check "spec RPC with control encodes byte-exact" + (encodesAs rpcControl [26, 13, 34, 11, 10, 1, 103, 18, 4, 10, 2, 1, 2, 24, 60]) + + -- ControlIHave: field 1 tag 10 with "t"; field 2 bytes tag 18 twice, the + -- payloads [1] and [2, 3]. + r.check "spec ControlIHave topic and ids decodes" + (decodesAs (α := ControlIHave) [10, 1, 116, 18, 1, 1, 18, 2, 2, 3] ihaveT) + r.check "spec ControlIHave topic and ids encodes byte-exact" + (encodesAs ihaveT [10, 1, 116, 18, 1, 1, 18, 2, 2, 3]) + + -- The production decoder and the model decoder agree on the fixed bytes. + r.check "spec RPC production path agrees with the model" + (productionAgrees (α := RPC) + [26, 13, 34, 11, 10, 1, 103, 18, 4, 10, 2, 1, 2, 24, 60]) + + -- Strictness: a bool above 1, a wrong wire type on a singular field, + -- and invalid UTF-8 are badField. + r.check "SubOpts subscribe varint 2 rejected as badField" + (rejectsWith (α := SubOpts) [8, 2] .badField) + -- Field 1 with the bytes wire type: tag is 10, length 1, payload 5. + r.check "SubOpts subscribe bytes wire type rejected as badField" + (rejectsWith (α := SubOpts) [10, 1, 5] .badField) + -- Field 2 bytes with the payload [255], which has no UTF-8 reading. + r.check "SubOpts topicid invalid UTF-8 rejected as badField" + (rejectsWith (α := SubOpts) [18, 1, 255] .badField) + -- Field 2 with the varint wire type: tag is 16. + r.check "SubOpts topicid varint wire type rejected as badField" + (rejectsWith (α := SubOpts) [16, 1] .badField) + + -- The required topic. An empty buffer has no field 4. + r.check "Message without topic rejected as badField" + (rejectsWith (α := Message) [] .badField) + r.check "Message topic invalid UTF-8 rejected as badField" + (rejectsWith (α := Message) [34, 1, 255] .badField) + -- A valid topic, then field 2 with the varint wire type (tag 16). + r.check "Message data varint wire type rejected as badField" + (rejectsWith (α := Message) [34, 1, 116, 16, 1] .badField) + + -- Field 3 with the bytes wire type: tag is 26, length 1, payload 5. + r.check "ControlPrune backoff bytes wire type rejected as badField" + (rejectsWith (α := ControlPrune) [10, 1, 103, 26, 1, 5] .badField) + -- The varint of 2^63 is ten groups, one more than the nine-byte budget. + -- The varint decoder rejects it as malformed before the table runs. + r.check "ControlPrune backoff value 2^63 rejected as malformed" + (rejectsWith (α := ControlPrune) + [10, 1, 103, 24, 128, 128, 128, 128, 128, 128, 128, 128, 128, 1] + (.framing .malformed)) + + -- A control payload whose nested ControlPrune payload is the byte 5, + -- which is an invalid tag: field 0 is not a legal number. + r.check "RPC control payload that is not a ControlMessage rejected" + (rejectsWith (α := RPC) [26, 3, 34, 1, 5] .badField) + -- A publish payload with no topic fails the Message table. + r.check "RPC publish payload that is not a Message rejected" + (rejectsWith (α := RPC) [18, 0] .badField) + + -- A repeated field of another wire type is skipped. Field 1 varint tag + -- is 8, value 1. + r.check "RPC subscriptions entry with varint wire type skipped, decode succeeds" + (decodesAs (α := RPC) [8, 1] { subscriptions := [], publish := [], control := none }) + + -- Field 5 of ControlMessage (`idontwant` in v1.2) is out of scope. Bytes + -- tag is 42, length 1, payload 1. The value drops the field, and the + -- re-encode omits it. + r.check "ControlMessage idontwant field 5 ignored and dropped on re-encode" + (decodesAs (α := ControlMessage) [42, 1, 1] + { ihave := [], iwant := [], graft := [], prune := [] } && + Pb.encodeMessage ({ ihave := [], iwant := [], graft := [], prune := [] } : ControlMessage) == []) + + -- Field 99 bytes: tag 794 is [154, 6], length 1, payload 2. The value + -- drops the field, and the re-encode omits it. + r.check "RPC unknown field 99 ignored, structure drops it" + (decodesAs (α := RPC) [154, 6, 1, 2] + { subscriptions := [], publish := [], control := none } && + Pb.encodeMessage ({ subscriptions := [], publish := [], control := none } : RPC) == []) + + -- The frame step. The length prefix of 1048577 is over the cap, so the + -- decode stops before the body. + r.check "frame length over rpcCap malformed before the body" + (frameFails (Streams.Framing.encodeLength .uvarint 1048577 ++ [1, 2, 3]) + (.framing .malformed)) + -- The byte 128 is a varint continuation with no next byte. + r.check "frame prefix cut short returns none" (frameNeedsMore [128]) + -- A zero-length frame is a legal empty RPC here. Identify rejects it + -- (`Identify.Err.emptyFrame`, the spike's read rule); gossipsub has no + -- such rule, because the empty RPC is a legal value. + r.check "zero-length frame decodes to the empty RPC" + (frameIs (Streams.Framing.encodeLength .uvarint 0) [] []) + -- One frame of a fixed RPC, then the frame step reads its payload back. + r.check "frame round trip: encodeFrame? then decodeFrame? returns the payload" + (match encodeFrame? rpcControl with + | some frame => frameIs frame (Pb.encodeMessage rpcControl) [] + | none => false) + r.check "rpcCap constant is 1048576" (Constants.rpcCap == 1048576) + + -- Repeated fields keep their order. + r.check "subscriptions keep their order through the round trip" + (rpcSubscriptions (Pb.encodeMessage + ({ subscriptions := [subA, subB], publish := [], control := none } : RPC)) == + some [subA, subB]) + r.check "publish keeps their order through the round trip" + (rpcPublish (Pb.encodeMessage + ({ subscriptions := [], publish := [msgA, msgB], control := none } : RPC)) == + some [msgA, msgB]) + r.check "message ids keep their order through the round trip" + (ihaveIds (Pb.encodeMessage + ({ topicId := "o", messageIds := [⟨[1]⟩, ⟨[2]⟩] } : ControlIHave)) == + some [⟨[1]⟩, ⟨[2]⟩] && + iwantIds (Pb.encodeMessage + ({ messageIds := [⟨[3]⟩, ⟨[4]⟩] } : ControlIWant)) == some [⟨[3]⟩, ⟨[4]⟩]) + r.check "prune peers keep their order through the round trip" + (prunePeers (Pb.encodeMessage + ({ topicId := "p", peers := [peerA, peerB], backoff := none } : ControlPrune)) == + some [peerA, peerB]) + +end Tests.Gossipsub.Rpc diff --git a/Tests/Gossipsub/Score.lean b/Tests/Gossipsub/Score.lean new file mode 100644 index 00000000..6e7b2cc0 --- /dev/null +++ b/Tests/Gossipsub/Score.lean @@ -0,0 +1,365 @@ +import Tests.Support +import Libp2p.Protocols.Gossipsub.Score + +/-! +# Gossipsub suite: Score + +The checks of `Libp2p.Protocols.Gossipsub.Score`. One fixed parameter set drives a regression +walk of the score function, its strict boundaries, and the validity rejections. Each expected +number carries its hand computation. + +The fixed-point scale is 1000, so a weight of 1000 is 1.0 and a weight of -100 is -0.1. The +walk's topic `"eth"` carries these values: `topicWeight` 1000, `timeInMeshWeight` 10, +`timeInMeshQuantum` 1000 ms, `timeInMeshCap` 100, `firstMessageDeliveriesWeight` 10, +`firstMessageDeliveriesDecay` 500, `firstMessageDeliveriesCap` 10, `meshMessageDeliveriesWeight` +-100, `meshMessageDeliveriesDecay` 500, `meshMessageDeliveriesThreshold` 4, +`meshMessageDeliveriesCap` 10, `meshMessageDeliveriesActivation` 20000 ms, +`meshMessageDeliveriesWindow` 10 ms, `meshFailurePenaltyWeight` -100, +`meshFailurePenaltyDecay` 500, `invalidMessageDeliveriesWeight` -1000, and +`invalidMessageDeliveriesDecay` 500. The global table carries `appSpecificWeight` 1, +`ipColocationFactorWeight` -100, `ipColocationFactorThreshold` 2, `behaviourPenaltyWeight` +-100, `behaviourPenaltyThreshold` 0, `behaviourPenaltyDecay` 500, `decayInterval` 1000 ms, +`decayToZero` 1, and `topicScoreCap` 10000. +-/ + +set_option autoImplicit false + +namespace Tests.Gossipsub.Score + +open Libp2p.Protocols.Gossipsub +open Libp2p.Protocols.Gossipsub.Score + +/-- The topic of the walk. -/ +private def eth : TopicId := ⟨"eth"⟩ + +/-- A topic with no configured parameters. -/ +private def other : TopicId := ⟨"other"⟩ + +/-- The per-topic parameters of the walk. -/ +private def ethParams : TopicParams where + topicWeight := 1000 + timeInMeshWeight := 10 + timeInMeshQuantum := 1000 + timeInMeshCap := 100 + firstMessageDeliveriesWeight := 10 + firstMessageDeliveriesDecay := 500 + firstMessageDeliveriesCap := 10 + meshMessageDeliveriesWeight := -100 + meshMessageDeliveriesDecay := 500 + meshMessageDeliveriesThreshold := 4 + meshMessageDeliveriesCap := 10 + meshMessageDeliveriesActivation := 20000 + meshMessageDeliveriesWindow := 10 + meshFailurePenaltyWeight := -100 + meshFailurePenaltyDecay := 500 + invalidMessageDeliveriesWeight := -1000 + invalidMessageDeliveriesDecay := 500 + +/-- The global parameters of the walk. The behavior threshold is 0, so the first penalty point +already contributes one squared point. -/ +private def walkGlobal : GlobalParams where + appSpecificWeight := 1 + ipColocationFactorWeight := -100 + ipColocationFactorThreshold := 2 + behaviourPenaltyWeight := -100 + behaviourPenaltyThreshold := 0 + behaviourPenaltyDecay := 500 + decayInterval := 1000 + decayToZero := 1 + topicScoreCap := 10000 + +/-- The parameters of the walk. Only `"eth"` is configured. -/ +private def walkParams : Params where + topics := fun id => if id = eth then some ethParams else none + global := walkGlobal + +/-! ## The regression walk + +Each step applies one update to the state of the step before. +-/ + +/-- Step 2: the peer is grafted at time 0. -/ +private def s1 : PeerState := graft PeerState.zero 0 eth + +/-- Step 3: one mesh delivery and one first delivery. -/ +private def s2 : PeerState := firstDelivery walkParams (meshDelivery walkParams s1 eth) eth + +/-- Step 4: two more mesh deliveries, so the count is 3, one below the threshold 4. -/ +private def s3 : PeerState := + meshDelivery walkParams (meshDelivery walkParams s2 eth) eth + +/-- Step 5: one invalid delivery. -/ +private def s4 : PeerState := invalidDelivery walkParams s3 eth + +/-- Step 6: one behavior penalty. -/ +private def s5 : PeerState := behaviourPenalty s4 + +/-- Step 7: a prune at time 30000, with the delivery deficit of 1 square. -/ +private def s6 : PeerState := prune walkParams s5 30000 eth + +/-- Step 8: one decay interval. -/ +private def s7 : PeerState := decay walkParams s6 + +/-! ## Strictness fixtures -/ + +/-- Counters with the peer in the mesh since time 0 and no deliveries. -/ +private def meshCounters : TopicCounters := + { TopicCounters.zero with inMesh := true, graftTime := 0 } + +/-- The per-topic parameters of the topic cap checks. Only `P2` contributes, at weight 1. -/ +private def capTopic : TopicParams := + { ethParams with + timeInMeshWeight := 0 + firstMessageDeliveriesWeight := 1 + meshMessageDeliveriesWeight := 0 + meshFailurePenaltyWeight := 0 + invalidMessageDeliveriesWeight := 0 } + +/-- The parameters of the topic cap checks: a cap of 5 and the `capTopic` table. -/ +private def capParams : Params where + topics := fun id => if id = eth then some capTopic else none + global := { walkGlobal with topicScoreCap := 5 } + +/-- A state with `n` first message deliveries in `"eth"` and no other input. -/ +private def capState (n : Nat) : PeerState where + topics := [(eth, { TopicCounters.zero with firstMessageDeliveries := n })] + appScore := 0 + ipColocation := 0 + behaviourPenalty := 0 + +/-! ## Rejection fixtures + +Each fixture changes one field of the valid walk parameter set. +-/ + +private def badP2Weight : TopicParams := + { ethParams with firstMessageDeliveriesWeight := -1 } + +private def badP3Weight : TopicParams := + { ethParams with meshMessageDeliveriesWeight := 1 } + +private def badP4Weight : TopicParams := + { ethParams with invalidMessageDeliveriesWeight := 1 } + +private def badTopicWeight : TopicParams := + { ethParams with topicWeight := -1 } + +private def badQuantum : TopicParams := + { ethParams with timeInMeshQuantum := 0 } + +private def badDecay : TopicParams := + { ethParams with firstMessageDeliveriesDecay := scoreScale + 1 } + +private def badP6Weight : GlobalParams := + { walkGlobal with ipColocationFactorWeight := 1 } + +private def badP7Weight : GlobalParams := + { walkGlobal with behaviourPenaltyWeight := 1 } + +private def badIpThreshold : GlobalParams := + { walkGlobal with ipColocationFactorThreshold := 0 } + +/-! ## The compiled rejection proofs + +Each `example` proves the rejection at compile time. The runtime check beside it names the same +case in the suite output. +-/ + +example : TopicParams.Valid ethParams := + (TopicParams.validB_iff ethParams).mp (by decide) + +example : GlobalParams.Valid walkGlobal := + (GlobalParams.validB_iff walkGlobal).mp (by decide) + +/-- The walk parameter set passes `Params.Valid`. -/ +example : Params.Valid walkParams := by + refine ⟨?_, ?_⟩ + · intro id tp h + by_cases hid : id = eth + · subst id + simp [walkParams] at h + rw [← h] + exact (TopicParams.validB_iff ethParams).mp (by decide) + · simp [walkParams, hid] at h + · exact (GlobalParams.validB_iff walkGlobal).mp (by decide) + +/-- A topic parameter set that the Boolean mirror rejects makes every parameters value that +configures it invalid. -/ +private theorem paramsValid_of_topic_validB_false {tp : TopicParams} + (hbad : tp.validB = false) : + ¬ Params.Valid { topics := fun _ => some tp, global := walkGlobal } := by + intro h + have hb := (TopicParams.validB_iff tp).mpr (h.topic eth tp rfl) + rw [hbad] at hb + cases hb + +/-- A global parameter set that the Boolean mirror rejects makes every parameters value that +carries it invalid. -/ +private theorem paramsValid_of_global_validB_false {g : GlobalParams} + (hbad : g.validB = false) : + ¬ Params.Valid { topics := walkParams.topics, global := g } := by + intro h + have hb := (GlobalParams.validB_iff g).mpr h.global + rw [hbad] at hb + cases hb + +example : ¬ Params.Valid + { topics := fun _ => some badP2Weight, global := walkGlobal } := + paramsValid_of_topic_validB_false (by decide) + +example : ¬ Params.Valid + { topics := fun _ => some badP3Weight, global := walkGlobal } := + paramsValid_of_topic_validB_false (by decide) + +example : ¬ Params.Valid + { topics := fun _ => some badP4Weight, global := walkGlobal } := + paramsValid_of_topic_validB_false (by decide) + +example : ¬ Params.Valid + { topics := fun _ => some badTopicWeight, global := walkGlobal } := + paramsValid_of_topic_validB_false (by decide) + +example : ¬ Params.Valid + { topics := fun _ => some badQuantum, global := walkGlobal } := + paramsValid_of_topic_validB_false (by decide) + +example : ¬ Params.Valid + { topics := fun _ => some badDecay, global := walkGlobal } := + paramsValid_of_topic_validB_false (by decide) + +example : ¬ Params.Valid { topics := walkParams.topics, global := badP6Weight } := + paramsValid_of_global_validB_false (by decide) + +example : ¬ Params.Valid { topics := walkParams.topics, global := badP7Weight } := + paramsValid_of_global_validB_false (by decide) + +example : ¬ Params.Valid { topics := walkParams.topics, global := badIpThreshold } := + paramsValid_of_global_validB_false (by decide) + +/-- Run the checks of this module. -/ +def run (r : Runner) : IO Unit := do + r.section "gossipsub Score" "vendor/specs/pubsub" + + -- Step 1: the fresh peer. Every term is 0 and the topic list is empty. + r.check "score.walk.fresh-peer" (score walkParams PeerState.zero 0 == 0) + + -- Step 2: graft at time 0, scored at 10000. Elapsed mesh time 10000 ms gives P1 = 10000 / + -- 1000 = 10, capped at 100, so 10. P3 is 0 because the activation is 20000 ms. Topic score + -- = 10 * 10 + 10 * 0 + (-100) * 0 + (-100) * 0 + (-1000) * 0 = 100. Topic contribution = + -- 1000 * 100 tdiv 1000 = 100. Total = 100. + r.check "score.walk.graft" (score walkParams s1 10000 == 100) + + -- Step 3: one mesh delivery and one first delivery, scored at 10000. P1 = 10, P2 = 1, so the + -- topic score = 100 + 10 * 1 = 110. Contribution = 110. Total = 110. + r.check "score.walk.deliveries" (score walkParams s2 10000 == 110) + + -- Step 4: two more mesh deliveries, so the count is 3, scored at 30000. Elapsed = 30000, + -- P1 = 30, so 300. P3 applies because 20000 < 30000 and 3 < 4: deficit 1, square 1, term + -- = -100 * 1 = -100. Topic score = 300 + 10 - 100 = 210. Contribution = 210. Total = 210. + r.check "score.walk.p3" (score walkParams s3 30000 == 210) + + -- Step 5: one invalid delivery, scored at 30000. P4 = 1 * 1 = 1, term = -1000. Topic score + -- = 210 - 1000 = -790. Contribution = -790. Total = -790. + r.check "score.walk.p4" (score walkParams s4 30000 == -790) + + -- Step 6: one behavior penalty, scored at 30000. The threshold is 0, so P7 = (1 - 0)^2 = 1, + -- global term = -100 * 1 = -100. Total = -790 - 100 = -890. + r.check "score.walk.p7" (score walkParams s5 30000 == -890) + + -- Step 7: prune at 30000. The elapsed time 30000 passes the activation, and the count 3 is + -- below the threshold 4, so the failure penalty grows by (4 - 3)^2 = 1. P1 = 300, P2 = 10, + -- P3 = -100, P3b = -100 * 1 = -100, P4 = -1000: topic score = -890. Global P7 = -100. + -- Total = -990. + r.check "score.walk.prune" (score walkParams s6 30000 == -990) + + -- Step 8: one decay. Each factor is 500, so a counter c becomes c * 500 / 1000, and a result + -- below 1 becomes 0. First deliveries 1 -> 0, mesh deliveries 3 -> 1, failure penalty 1 -> 0, + -- invalid deliveries 1 -> 0, behavior penalty 1 -> 0. The mesh time and the membership stay. + r.check "score.walk.decay.first-message-deliveries" + ((countersOf s7 eth).firstMessageDeliveries == 0) + r.check "score.walk.decay.mesh-message-deliveries" + ((countersOf s7 eth).meshMessageDeliveries == 1) + r.check "score.walk.decay.mesh-failure-penalty" + ((countersOf s7 eth).meshFailurePenalty == 0) + r.check "score.walk.decay.invalid-message-deliveries" + ((countersOf s7 eth).invalidMessageDeliveries == 0) + r.check "score.walk.decay.behavior-penalty" (s7.behaviourPenalty == 0) + r.check "score.walk.decay.keeps-mesh-time" + ((countersOf s7 eth).timeInMesh == 30000 && (countersOf s7 eth).inMesh == false) + + -- Step 9: score after the decay, at 30000. P1 = 300, P2 = 0, P3 = (4 - 1)^2 = 9 with term + -- -900, P3b = 0, P4 = 0. Topic score = 300 - 900 = -600. Contribution = -600. Global P7 = 0. + -- Total = -600. + r.check "score.walk.final" (score walkParams s7 30000 == -600) + + -- The P3 activation boundary. At 20000 ms the activation is equal, so the strict comparison + -- fails and P3 = 0. At 20001 ms the deficit is (4 - 0)^2 = 16. + r.check "score.p3.activation-boundary" + (p3 ethParams 20000 meshCounters == 0 && p3 ethParams 20001 meshCounters == 16) + + -- The P3 threshold boundary. At the count 4 the strict comparison fails and P3 = 0. At 3 the + -- deficit is 1 and P3 = 1. + r.check "score.p3.threshold-boundary" + (p3 ethParams 30000 { meshCounters with meshMessageDeliveries := 4 } == 0 && + p3 ethParams 30000 { meshCounters with meshMessageDeliveries := 3 } == 1) + + -- The decay zero boundary. A counter equal to `decayToZero` with a full decay stays at 1. A + -- counter one below becomes 0. + r.check "score.decay.zero-boundary" + (decayCounter 1000 1 1 == 1 && decayCounter 1000 1 0 == 0) + + -- The P1 cap boundary. Elapsed 100000 ms gives 100 points, equal to the cap, so P1 = 100. One + -- quantum more gives 101 points, and the cap holds P1 at 100. + r.check "score.p1.cap-boundary" + (p1 ethParams 100000 meshCounters == 100 && p1 ethParams 101000 meshCounters == 100) + + -- The P6 threshold boundary. Colocation 2 equals the threshold, so P6 = 0. Colocation 3 gives + -- the surplus 1 squared. + r.check "score.p6.threshold-boundary" + (p6 walkGlobal { PeerState.zero with ipColocation := 2 } == 0 && + p6 walkGlobal { PeerState.zero with ipColocation := 3 } == 1) + + -- The P7 threshold boundary. Penalty 0 equals the threshold, so P7 = 0. Penalty 1 gives the + -- excess 1 squared. + r.check "score.p7.threshold-boundary" + (p7 walkGlobal { PeerState.zero with behaviourPenalty := 0 } == 0 && + p7 walkGlobal { PeerState.zero with behaviourPenalty := 1 } == 1) + + -- The topic cap boundary. With `P2` at weight 1 and no other term, the raw topic part equals + -- the delivery count. At 5 the raw part equals the cap 5 and stays. At 6 the raw part 6 is + -- above the cap, so the topic part is 5. + r.check "score.topic-cap-boundary" + (topicPart capParams 0 (capState 5) == 5 && topicPart capParams 0 (capState 6) == 5) + + -- The cap applies at increment time. A first delivery from 9 reaches the cap 10, and one more + -- from 10 leaves the counter at 10. + r.check "score.cap-at-increment" + ((countersOf (firstDelivery walkParams (capState 9) eth) eth).firstMessageDeliveries == 10 && + (countersOf (firstDelivery walkParams (capState 10) eth) eth).firstMessageDeliveries == 10) + + -- A delivery to a topic with no parameters leaves the score alone. The state s3 scores 210 at + -- 30000, and the first delivery to `"other"` is a no-op. + r.check "score.unconfigured-topic-ignored" + (score walkParams (firstDelivery walkParams s3 other) 30000 == + score walkParams s3 30000) + + -- The rejections. Each Boolean mirror is false, and the `example` beside the definition + -- proves the predicate rejects the same value. + r.check "score.valid.reject-negative-p2-weight" (!badP2Weight.validB) + r.check "score.valid.reject-positive-p3-weight" (!badP3Weight.validB) + r.check "score.valid.reject-positive-p4-weight" (!badP4Weight.validB) + r.check "score.valid.reject-negative-topic-weight" (!badTopicWeight.validB) + r.check "score.valid.reject-zero-quantum" (!badQuantum.validB) + r.check "score.valid.reject-decay-above-scale" (!badDecay.validB) + r.check "score.valid.reject-positive-p6-weight" (!badP6Weight.validB) + r.check "score.valid.reject-positive-p7-weight" (!badP7Weight.validB) + r.check "score.valid.reject-zero-ip-threshold" (!badIpThreshold.validB) + + -- A zero P6 weight and a zero P7 weight are valid: the predicate requires `<= 0`. The + -- walk's global table already carries a zero behavior threshold, and a zero weight is also + -- accepted. + r.check "score.valid.accept-zero-p6-p7-weight" + ({ walkGlobal with ipColocationFactorWeight := 0, behaviourPenaltyWeight := 0 } : + GlobalParams).validB + +end Tests.Gossipsub.Score diff --git a/Tests/Gossipsub/Sign.lean b/Tests/Gossipsub/Sign.lean new file mode 100644 index 00000000..e1b44b5d --- /dev/null +++ b/Tests/Gossipsub/Sign.lean @@ -0,0 +1,188 @@ +import Tests.Support +import Tests.Gossipsub.Identity +import Libp2p.Protocols.Gossipsub.Sign + +/-! +# Gossipsub suite: Sign + +The checks of `Libp2p.Protocols.Gossipsub.Sign` against +`vendor/specs/pubsub/README.md`, sections Message Signing, Signature +Policy Options, and Message Identification. + +The verify and encode functions are executable stand-ins. The +signature math belongs to the crypto package, so the stand-in verifier +answers `true` or `false` and the checks pin the policy logic. The +kernel examples at the end show that each rejection arm is reachable. +Each example closes by `rfl`. +-/ + +set_option autoImplicit false + +namespace Tests.Gossipsub.Sign + +open Libp2p.Protocols.Gossipsub +open Libp2p.Protocols.Gossipsub.Sign +open Libp2p.Identity + +/-- An encoder that reads the `from`, `data`, `seqno`, and topic fields. +It reads neither the `signature` nor the `key` field. -/ +private def testEncode (m : Message) : List Nat := + m.from.getD [] ++ m.data.getD [] ++ m.seqno.getD [] ++ + Bytes.toNatList (Bytes.ofString m.topic.name) + +/-- An encoder that reads every field and marks presence. The signature +input check needs this encoder. An input that fails to clear the +`signature` or `key` field still changes under this encoder, so the +check detects that defect. -/ +private def tagEncode (m : Message) : List Nat := + (match m.from with | none => [0] | some b => 1 :: b) ++ + (match m.data with | none => [0] | some b => 1 :: b) ++ + (match m.seqno with | none => [0] | some b => 1 :: b) ++ + (match m.signature with | none => [0] | some b => 1 :: b) ++ + (match m.key with | none => [0] | some b => 1 :: b) ++ + Bytes.toNatList (Bytes.ofString m.topic.name) + +/-- The accepting stand-ins: a nonempty buffer parses as an ed25519 +key, the derived peer id carries the key bytes, and the verifier +accepts. -/ +private def testPrimsOk : Prims := + { unmarshalKey := fun bs => if bs.isEmpty then none else some (Identity.publicKey (bs.headD 0)) + inlineKey := fun bs => if bs.isEmpty then none else some (Identity.publicKey (bs.headD 0)) + idOfKey := fun k => [k.data[0]!.toNat] + verify := fun _ _ _ => true } + +/-- The rejecting stand-ins: the verifier rejects. -/ +private def testPrimsBad : Prims := + { testPrimsOk with verify := fun _ _ _ => false } + +/-- A message with the four signed fields set as given. -/ +private def testMsg (fr sq sig key : Option (List Nat)) : Message := + { «from» := fr, data := some [9], seqno := sq, topic := ⟨"t"⟩, + signature := sig, key := key } + +/-- The UTF-8 bytes of `libp2p-pubsub:`, from +`vendor/specs/pubsub/README.md` Message Signing and the go reference +implementation. -/ +private def prefixBytes : List Nat := + [108, 105, 98, 112, 50, 112, 45, 112, 117, 98, 115, 117, 98, 58] + +/-! ## Reachability of the rejection arms + +Each example names one arm and supplies a message that reaches it. -/ + +example : validate testPrimsOk testEncode .strictSign + (testMsg none (some [2]) (some [3]) (some [4])) = .error .missingFrom := rfl + +example : validate testPrimsOk testEncode .strictSign + (testMsg (some [1]) none (some [3]) (some [4])) = .error .missingSeqno := rfl + +example : validate testPrimsOk testEncode .strictSign + (testMsg (some [1]) (some [2]) none (some [4])) = .error .missingSignature := rfl + +example : validate testPrimsOk testEncode .strictSign + (testMsg (some [1]) (some [2]) (some [3]) (some [])) = .error .keyUnresolved := rfl + +example : validate testPrimsOk testEncode .strictSign + (testMsg (some []) (some [2]) (some [3]) none) = .error .keyUnresolved := rfl + +example : validate testPrimsOk testEncode .strictSign + (testMsg (some [1]) (some [2]) (some [3]) (some [5])) = .error .keyMismatch := rfl + +example : validate testPrimsBad testEncode .strictSign + (testMsg (some [1]) (some [2]) (some [3]) (some [1])) = .error .badSignature := rfl + +example : validate testPrimsOk testEncode .strictNoSign + (testMsg (some [1]) none none none) = .error .hasFrom := rfl + +example : validate testPrimsOk testEncode .strictNoSign + (testMsg none (some [2]) none none) = .error .hasSeqno := rfl + +example : validate testPrimsOk testEncode .strictNoSign + (testMsg none none (some [3]) none) = .error .hasSignature := rfl + +example : validate testPrimsOk testEncode .strictNoSign + (testMsg none none none (some [4])) = .error .hasKey := rfl + +/-! ## Reachability of the acceptance paths -/ + +example : validate testPrimsOk testEncode .strictSign + (testMsg (some [1]) (some [2]) (some [3]) (some [1])) = .ok () := rfl + +example : validate testPrimsOk testEncode .strictSign + (testMsg (some [1]) (some [2]) (some [3]) none) = .ok () := rfl + +example : validate testPrimsOk testEncode .strictNoSign + (testMsg none none none none) = .ok () := rfl + +/-- Run the checks of this module. -/ +def run (r : Runner) : IO Unit := do + r.section "gossipsub Sign" "vendor/specs/pubsub" + + -- strictNoSign, through the accepting stand-ins + r.check "strictNoSign accepts a message with all four fields absent" + (succeedsWith () (validate testPrimsOk testEncode .strictNoSign + (testMsg none none none none))) + r.check "strictNoSign rejects a present from" + (failsWith SignError.hasFrom (validate testPrimsOk testEncode .strictNoSign + (testMsg (some [1]) none none none))) + r.check "strictNoSign rejects a present seqno" + (failsWith SignError.hasSeqno (validate testPrimsOk testEncode .strictNoSign + (testMsg none (some [2]) none none))) + r.check "strictNoSign rejects a present signature" + (failsWith SignError.hasSignature (validate testPrimsOk testEncode .strictNoSign + (testMsg none none (some [3]) none))) + r.check "strictNoSign rejects a present key" + (failsWith SignError.hasKey (validate testPrimsOk testEncode .strictNoSign + (testMsg none none none (some [4])))) + + -- strictSign, through the accepting stand-ins + r.check "strictSign accepts a fully signed message" + (succeedsWith () (validate testPrimsOk testEncode .strictSign + (testMsg (some [1]) (some [2]) (some [3]) (some [1])))) + r.check "strictSign accepts a message with the key inlined in from" + (succeedsWith () (validate testPrimsOk testEncode .strictSign + (testMsg (some [1]) (some [2]) (some [3]) none))) + r.check "strictSign rejects a missing from" + (failsWith SignError.missingFrom (validate testPrimsOk testEncode .strictSign + (testMsg none (some [2]) (some [3]) none))) + r.check "strictSign rejects a missing seqno" + (failsWith SignError.missingSeqno (validate testPrimsOk testEncode .strictSign + (testMsg (some [1]) none (some [3]) none))) + r.check "strictSign rejects a missing signature" + (failsWith SignError.missingSignature (validate testPrimsOk testEncode .strictSign + (testMsg (some [1]) (some [2]) none none))) + r.check "strictSign rejects a key field that does not parse" + (failsWith SignError.keyUnresolved (validate testPrimsOk testEncode .strictSign + (testMsg (some [1]) (some [2]) (some [3]) (some [])))) + r.check "strictSign rejects a peer id that inlines no key" + (failsWith SignError.keyUnresolved (validate testPrimsOk testEncode .strictSign + (testMsg (some []) (some [2]) (some [3]) none))) + r.check "strictSign rejects a key that derives another peer id" + (failsWith SignError.keyMismatch (validate testPrimsOk testEncode .strictSign + (testMsg (some [1]) (some [2]) (some [3]) (some [5])))) + r.check "strictSign rejects a bad signature" + (failsWith SignError.badSignature (validate testPrimsBad testEncode .strictSign + (testMsg (some [1]) (some [2]) (some [3]) (some [1])))) + + -- message id + r.check "default msg id is from followed by seqno" + (defaultMsgId (testMsg (some [1, 2]) (some [3, 4, 5]) none none) == + ⟨[1, 2, 3, 4, 5]⟩) + r.check "default msg id is empty when the fields are absent" + (defaultMsgId (testMsg none none none none) == ⟨[]⟩) + + -- signature input + r.check "sign input starts with the libp2p-pubsub prefix" + ((signInput testEncode (testMsg (some [1]) (some [2]) (some [3]) none)).take 14 == + prefixBytes) + r.check "sign input drops the signature and key fields" + (signInput tagEncode (testMsg (some [1]) (some [2]) (some [3]) (some [4])) == + signInput tagEncode (testMsg (some [1]) (some [2]) (some [7]) (some [8])) && + signInput tagEncode (testMsg (some [1]) (some [2]) (some [3]) none) == + signInput tagEncode (testMsg (some [1]) (some [2]) (some [7]) (some [8]))) + r.check "sign input covers the data field" + (signInput tagEncode (testMsg (some [1]) (some [2]) (some [3]) none) != + signInput tagEncode + ({ testMsg (some [1]) (some [2]) (some [3]) none with data := some [10] })) + +end Tests.Gossipsub.Sign diff --git a/Tests/Gossipsub/Switch.lean b/Tests/Gossipsub/Switch.lean new file mode 100644 index 00000000..a5a36cc0 --- /dev/null +++ b/Tests/Gossipsub/Switch.lean @@ -0,0 +1,138 @@ +import Libp2p.Protocols.Gossipsub.HostSeam + +set_option autoImplicit false + +namespace Tests.Gossipsub + +open Libp2p.Identity Libp2p.Core Libp2p.Protocols.Gossipsub + +/-! ## The in-memory switch + +The switch stands in for the session table of the host and its serve +task. Two peers meet over a pair of pipes, with no transport. -/ + +/-- One peer of the switch. -/ +private structure Switch.Entry where + /-- The peer. -/ + peer : PeerId + /-- The handlers the peer registered, the oldest first. -/ + handlers : List Libp2p.Core.StreamHandler + /-- The owner of the handler tasks of the inbound streams of the peer. -/ + tasks : Conc.TaskSet + +/-- The state of a switch. -/ +private structure Switch.State where + /-- The peers. -/ + entries : List Switch.Entry + /-- Every pipe the switch opened. The list lives as long as the switch, + which a test double allows. -/ + pipes : List Conc.Pipe + /-- True after `Switch.close`. -/ + closed : Bool + +/-- The in-memory double of the session table: the peers, their handlers, +and the pipes of their streams. -/ +structure Switch where + private state : Conc.OnceRef Switch.State + +namespace Switch + +/-- A switch with no peers. -/ +def new : BaseIO Switch := + return ⟨← Conc.OnceRef.new { entries := [], pipes := [], closed := false }⟩ + +/-- A stream over a pair of pipes. `read` reads `incoming`, and `write` +writes `outgoing`. `close` closes `outgoing`, the half-close. `reset` +closes both, so the peer of a reset reads the end of the stream and its +writes fail. -/ +private def streamOf (incoming outgoing : Conc.Pipe) : Streams.Stream where + read := do + let chunk ← incoming.read + return .ok chunk + write bytes := do + match ← outgoing.write bytes with + | .ok () => return .ok () + | .error _ => return .error .closed + close := (outgoing.close : BaseIO Unit) + reset := do + (incoming.close : BaseIO Unit) + (outgoing.close : BaseIO Unit) + +/-- The first protocol of the list that the handlers serve, with its +handler. -/ +private def choose (handlers : List Libp2p.Core.StreamHandler) : + List Libp2p.Core.ProtocolId → Option (Libp2p.Core.ProtocolId × Libp2p.Core.StreamHandler) + | [] => none + | p :: ps => + match handlers.find? (fun h => decide (h.id = p)) with + | some h => some (p, h) + | none => choose handlers ps + +/-- Run one handler on an inbound stream. A handler error fails the task, +so the `joinAll` of the owner reports it. -/ +private def serve (handler : Libp2p.Core.StreamHandler) (peer : PeerId) + (stream : Streams.Stream) : Conc.Async Unit := do + match ← (handler.handle { remotePeer := peer, remoteAddr := Multiformats.Multiaddr.empty } stream).run with + | .ok () => return () + | .error e => throw <| IO.userError s!"handler {handler.id.toString}: {repr e}" + +/-- Open a stream from `me` to `peer`: pick the protocol, make the pipes, +spawn the handler on the task set of `peer`, and answer the initiator +stream. The call never parks. Eight chunks is far more than one RPC frame +needs, and the bound stays explicit. -/ +private def open_ (sw : Switch) (me peer : PeerId) (protocols : List Libp2p.Core.ProtocolId) : + BaseIO (Except Libp2p.Core.Error (Libp2p.Core.ProtocolId × Streams.Stream)) := do + let st ← sw.state.get + let some entry := st.entries.find? (fun e => decide (e.peer = peer)) + | return .error (.protocol s!"no peer {peer}") + let some (id, handler) := choose entry.handlers protocols + | return .error (.protocol s!"peer {peer} serves none of the protocols") + let toPeer ← Conc.Pipe.new 8 + let fromPeer ← Conc.Pipe.new 8 + -- One atomic step checks the flag and records the pipes, so `close` + -- sees every pipe of a stream that opens before it. + let opened ← sw.state.updateGet fun s => + if s.closed then (false, s) + else (true, { s with pipes := toPeer :: fromPeer :: s.pipes }) + if !opened then return .error (.closed "the switch closed") + entry.tasks.spawn (serve handler me (streamOf toPeer fromPeer)) + return .ok (id, streamOf fromPeer toPeer) + +/-- The seam of the peer `me`, and the task set that owns the handler +tasks of its inbound streams. Each peer starts with an empty handler +table, and a handler registered through one seam serves only that peer. -/ +def host (sw : Switch) (me : PeerId) : BaseIO (HostSeam × Conc.TaskSet) := do + let tasks ← Conc.TaskSet.new + sw.state.update fun s => + { s with + entries := { peer := me, handlers := [], tasks } :: + s.entries.filter (fun e => decide (e.peer ≠ me)) } + let seam : HostSeam := + { newStream := fun peer protocols => (sw.open_ me peer protocols : BaseIO _) + setHandler := fun handler => + sw.state.update fun s => + { s with + entries := s.entries.map fun e => + if e.peer = me then { e with handlers := e.handlers ++ [handler] } else e } } + return (seam, tasks) + +/-- End every stream the switch opened, then join the handler tasks +(ARCHITECTURE §4.1, rule 2). A close wakes every parked read and write. +A failed handler task does not stop the joins of the other peers. The call +throws the first error after the last join. A second call does nothing. -/ +def close (sw : Switch) : Conc.Async Unit := do + let st ← (sw.state.updateGet (fun s => (s, { s with closed := true, pipes := [] })) : + BaseIO Switch.State) + for p in st.pipes do + (p.close : BaseIO Unit) + let mut failure : Option IO.Error := none + for e in st.entries do + try + e.tasks.joinAll + catch err => + failure := failure <|> some err + if let some err := failure then throw err + +end Switch + +end Tests.Gossipsub diff --git a/Tests/Gossipsub/Wire.lean b/Tests/Gossipsub/Wire.lean new file mode 100644 index 00000000..7d4da56e --- /dev/null +++ b/Tests/Gossipsub/Wire.lean @@ -0,0 +1,170 @@ +import Tests.Support +import Libp2p.Protocols.Gossipsub.Rpc + +/-! +# Gossipsub suite: the wire bytes of the reference implementation + +`Tests/fixtures/gossipsub-rpc-go.json` records nine gossipsub RPC values that the pinned go +implementation marshals with its own protobuf encoder. Each case carries the marshaled payload as +hex, the semantic values the generator set, and a `roundtrip` flag that the generator confirmed +with `proto.Unmarshal` and `proto.Equal`. The generator is +`packages/libp2p/protocols/tools/go-rpc-fixtures/main.go`, and the module version comes from the +interop harness pin (`vendor/test-plans/gossipsub-interop/go-libp2p/go.mod:8`). + +This suite decodes each case with this package's codec, checks the decoded value against the +recorded fields, and re-encodes it. The re-encoding decodes to the same value, and for every case +whose payload carries no field that this codec leaves out, the re-encoding is the recorded bytes. +The suite reads one fixture and needs no network, no clock and no task. + +The CI provenance step covers Tests/vectors only. It regenerates that corpus +with scripts/extract-vectors.py. This fixture lives in Tests/fixtures and +checks its own generator, module and version provenance fields. A maintainer +regenerates it by hand with go run from packages/libp2p/protocols/tools/go-rpc-fixtures. +The command is go run . with the generator's default output path. +-/ + +set_option autoImplicit false + +namespace Tests.Gossipsub.Wire + +open Libp2p.Protocols +open Libp2p.Protocols.Gossipsub +open Libp2p.Protocols.Gossipsub.Rpc +open Tests (Runner loadFixture jArr jObj jStr unhex) + +open Lean (Json) + +private def file : String := "gossipsub-rpc-go.json" + +/-- The recorded provenance names this repository's generator, the go module, and the pinned +version, so a fixture that arrives from somewhere else fails its own check. -/ +private def checkProvenance (r : Runner) (j : Json) : IO Unit := do + let p ← jObj file j "_provenance" + let expect (field got : String) (want : String) : IO Unit := + r.check s!"wire provenance {field}" (got == want) + expect "generatedBy" (← jStr file p "generatedBy") + "packages/libp2p/protocols/tools/go-rpc-fixtures/main.go" + expect "module" (← jStr file p "module") "github.com/libp2p/go-libp2p-pubsub, pb package" + expect "version" (← jStr file p "version") "v0.15.1-0.20260305223556-a01777894f4c" + +/-- The bytes of a hex string in the fixture. -/ +private def bytesOf (what hex : String) : IO (List Nat) := do + return Bytes.toNatList (← unhex what hex) + +/-- Decode one payload. The answer is `none` when the codec rejects bytes that the go encoder +produced, which is the failure this suite exists to find. -/ +private def decodePayload (r : Runner) (what : String) (payload : ByteArray) : + IO (Option Rpc.RPC) := + match Pb.decodeMessage? (α := Rpc.RPC) (Bytes.toNatList payload) with + | .ok (rpc, rest) => do + r.check s!"{what} consumes every byte" rest.isEmpty + return some rpc + | .error _ => do + r.check s!"{what} decodes" false + return none + +/-- The value survives one re-encoding: this codec's encoder output decodes to the same value. -/ +private def isStable (rpc : Rpc.RPC) : Bool := + match Pb.decodeMessage? (α := Rpc.RPC) (Pb.encodeMessage rpc) with + | .ok (again, rest) => (again == rpc) && rest.isEmpty + | .error _ => false + +/-- The `fields` object of one case, when the case carries one. -/ +private def fieldsOf (c : Json) : IO (Option Json) := + match c.getObjVal? "fields" with + | .ok v => return some v + | .error _ => return none + +/-- The subscription records the generator set, checked against the decoded value. -/ +private def checkSubs (r : Runner) (name : String) (f : Json) (rpc : Rpc.RPC) : IO Unit := do + let arr := match (f.getObjVal? "subscriptions") with + | .ok v => (v.getArr?.toOption.getD #[]).toList + | .error _ => [] + r.check s!"{name} subscription count" (arr.length == rpc.subscriptions.length) + for (rec, got) in arr.zip rpc.subscriptions do + let topicid := (rec.getObjVal? "topicid").toOption.bind (·.getStr?.toOption) + let sub := (rec.getObjVal? "subscribe").toOption.bind (·.getBool?.toOption) + r.check s!"{name} subscription topic" (topicid == some got.topicid) + r.check s!"{name} subscription flag" (sub == some got.subscribe) + +/-- The published messages the generator set, checked against the decoded value. -/ +private def checkPubs (r : Runner) (name : String) (f : Json) (rpc : Rpc.RPC) : IO Unit := do + let arr := match (f.getObjVal? "publish") with + | .ok v => (v.getArr?.toOption.getD #[]).toList + | .error _ => [] + r.check s!"{name} publish count" (arr.length == rpc.publish.length) + for (rec, got) in arr.zip rpc.publish do + let topic := (rec.getObjVal? "topic_str").toOption.bind (·.getStr?.toOption) + r.check s!"{name} publish topic" (topic == some got.topic.name) + let dataHex : Option String := (rec.getObjVal? "data_hex").toOption.bind (·.getStr?.toOption) + match dataHex with + | some h => r.check s!"{name} publish data" (some (Bytes.toNatList (Bytes.ofHex? h |>.getD .empty)) == got.data) + | none => r.check s!"{name} publish data absent" got.data.isNone + +/-- The control counts and the prune record of the topic, which is the only control message that +carries a peer list and a backoff here. -/ +private def checkControl (r : Runner) (name : String) (f : Json) (rpc : Rpc.RPC) : IO Unit := do + match f.getObjVal? "control" with + | .error _ => r.check s!"{name} no control when none was set" rpc.control.isNone + | .ok c => + match rpc.control with + | none => r.check s!"{name} control decodes" false + | some msg => + let count (k : String) : IO Nat := do + match c.getObjVal? k with + | .ok v => return (v.getArr?.toOption.getD #[]).size + | .error _ => return 0 + let ngraft ← count "graft" + let nprune ← count "prune" + let nihave ← count "ihave" + let niwant ← count "iwant" + r.check s!"{name} graft count" (ngraft == msg.graft.length) + r.check s!"{name} prune count" (nprune == msg.prune.length) + r.check s!"{name} ihave count" (nihave == msg.ihave.length) + r.check s!"{name} iwant count" (niwant == msg.iwant.length) + if let some p := msg.prune.head? then + r.check s!"{name} prune topic" (p.topicId == "test-topic") + r.check s!"{name} prune backoff" (p.backoff == some 60) + r.check s!"{name} prune peers" (p.peers.map (·.peerId) == [some [1, 2, 3]]) + if let some i := msg.ihave.head? then + r.check s!"{name} ihave topic" (i.topicId == "test-topic") + r.check s!"{name} ihave ids" + (i.messageIds.map (·.bytes) == [[1, 0, 0, 0, 0, 0, 0, 0], [2, 0, 0, 0, 0, 0, 0, 0]]) + if let some i := msg.iwant.head? then + r.check s!"{name} iwant ids" + (i.messageIds.map (·.bytes) == [[1, 0, 0, 0, 0, 0, 0, 0], [2, 0, 0, 0, 0, 0, 0, 0]]) + +/-- One case: decode, check the recorded fields, re-encode, and compare the bytes. The +`unknown-fields` case carries a field this codec leaves out on the way back, so its re-encoding is +a prefix of the recorded bytes, and that is the recorded behaviour rather than a surprise. -/ +private def checkCase (r : Runner) (c : Json) : IO Unit := do + let name ← jStr file c "name" + let payload ← bytesOf s!"{file} {name}" (← jStr file c "hex") + let rtRecorded := (c.getObjVal? "roundtrip").toOption.bind (·.getBool?.toOption) + r.check s!"{name} generator roundtrip" (rtRecorded == some true) + match ← decodePayload r name (Bytes.ofNatList payload) with + | none => pure () + | some rpc => + r.check s!"{name} re-encodes stably" (isStable rpc) + if let some f ← fieldsOf c then + checkSubs r name f rpc + checkPubs r name f rpc + checkControl r name f rpc + let encoded := Pb.encodeMessage rpc + if name == "unknown-fields" then + r.check s!"{name} re-encoding is the recorded prefix" + (encoded.length < payload.length && payload.take encoded.length == encoded) + else + r.check s!"{name} re-encoding is the recorded bytes" (encoded == payload) + +/-- The suite. -/ +def run (r : Runner) : IO Unit := do + r.section "gossipsub wire" "tests/fixtures/gossipsub-rpc-go.json" + let j ← loadFixture file + checkProvenance r j + let cases ← jArr file j "cases" + r.check "wire case count" (cases.length == 9) + for c in cases do + checkCase r c + +end Tests.Gossipsub.Wire \ No newline at end of file diff --git a/Tests/Host.lean b/Tests/Host.lean index afc1051b..7d444cc3 100644 --- a/Tests/Host.lean +++ b/Tests/Host.lean @@ -302,4 +302,4 @@ def Host.run (r : Runner) : IO Unit := do for leg in swarmLegs do r.skip s!"host: {leg}" "needs the negotiate swap of the host, issue #70" -end Tests +end Tests \ No newline at end of file diff --git a/Tests/Interop/Noise.lean b/Tests/Interop/Noise.lean index 1039b9f5..a7955e8e 100644 --- a/Tests/Interop/Noise.lean +++ b/Tests/Interop/Noise.lean @@ -138,4 +138,4 @@ def main (args : List String) : IO UInt32 := do return 0 catch e => IO.eprintln s!"{e}" - return 1 + return 1 \ No newline at end of file diff --git a/Tests/Main.lean b/Tests/Main.lean index f51ec59b..b842cd0a 100644 --- a/Tests/Main.lean +++ b/Tests/Main.lean @@ -1,4 +1,12 @@ import Tests.Support +import Tests.Gossipsub.Core +import Tests.Gossipsub.Sign +import Tests.Gossipsub.Cache +import Tests.Gossipsub.Score +import Tests.Gossipsub.Rpc +import Tests.Gossipsub.Node +import Tests.Gossipsub.Wire +import Tests.Gossipsub.Interop import Tests.Vectors import Tests.Concurrency import Tests.Protobuf @@ -44,6 +52,14 @@ pass under a single capability: `lake test -- +RTS -N1 -RTS` def main : IO UInt32 := do let r ← Tests.Runner.new Tests.Vectors.run r + Tests.Gossipsub.Core.run r + Tests.Gossipsub.Sign.run r + Tests.Gossipsub.Cache.run r + Tests.Gossipsub.Score.run r + Tests.Gossipsub.Rpc.run r + Tests.Gossipsub.Node.run r + Tests.Gossipsub.Wire.run r + Tests.Gossipsub.Interop.run r Tests.Concurrency.run r Tests.Protobuf.run r Bytes.runTests r diff --git a/Tests/Negotiate.lean b/Tests/Negotiate.lean index 85a731dc..ce48446f 100644 --- a/Tests/Negotiate.lean +++ b/Tests/Negotiate.lean @@ -256,4 +256,4 @@ def run (r : Runner) : IO Unit := do r.skip "mss generic model pin" "needs the ByteArray adapter, which is not built; the suite runs the model decoder, which is the production path" -end Tests.Negotiate +end Tests.Negotiate \ No newline at end of file diff --git a/Tests/Noise.lean b/Tests/Noise.lean index 367cbbdc..8c2c99c3 100644 --- a/Tests/Noise.lean +++ b/Tests/Noise.lean @@ -809,4 +809,4 @@ def Noise.run (r : Runner) : IO Unit := do hostile r sim r -end Tests +end Tests \ No newline at end of file diff --git a/Tests/Yamux.lean b/Tests/Yamux.lean index ad9e55c2..5dde6226 100644 --- a/Tests/Yamux.lean +++ b/Tests/Yamux.lean @@ -953,29 +953,35 @@ private def leftoverChecks (r : Runner) : Async Unit := do /-- A write whose frame goes out after the loop has ended answers `.ok`. The client's connection reads park until the test closes `fin`, then -end, as after a peer FIN, while its writes still reach the server. Once -`armed` is set, the writes wait at `gate`, so the write's frame is still -queued when the teardown runs. -/ +end, as after a peer FIN, while its writes still reach the server. The +DATA frame signals entry, then waits at `gate` until teardown ends. The +stream-opening frame passes the gate so it cannot block the DATA frame. -/ private def writeAcrossEndChecks (r : Runner) : Async Unit := do let (c, s) ← pair let fin : Chan Unit ← Chan.new 1 let gate : Chan Unit ← Chan.new 1 - let armed : OnceRef Bool ← OnceRef.new false + let entered : Chan Unit ← Chan.new 1 let c' : Connection := { c with read := do discard fin.recv; return none write := fun bytes => do - if ← (armed.get : BaseIO Bool) then discard gate.recv + match Header.decodeB? bytes with + | .ok (h, _) => + if h.type == .data && h.streamId == 1 then + discard <| entered.send () + discard gate.recv + | .error _ => pure () c.write bytes } let client ← Session.new .client c' let server ← Session.new .server s let .ok cs ← client.openStream | throw <| IO.userError "open failed" - armed.set true let tasks ← TaskSet.new let wrote : OnceRef Bool ← OnceRef.new false tasks.spawn do if let .ok () ← cs.write (b "late") then wrote.set true + -- Wait until the target DATA frame has entered the wrapped write. -- The end of the reads ends the loop; the teardown closes the accept -- queue, so this accept answers `none` only after the teardown. + discard entered.recv fin.close r.check "session: the reader's end ends the session" ((← client.acceptStream).isNone) gate.close diff --git a/Tests/fixtures/gossipsub-interop-params.json b/Tests/fixtures/gossipsub-interop-params.json new file mode 100644 index 00000000..f8d6551f --- /dev/null +++ b/Tests/fixtures/gossipsub-interop-params.json @@ -0,0 +1 @@ +{"script": [{"type": "ifNodeIDEquals", "nodeID": 0, "instruction": {"type": "initGossipSub", "gossipSubParams": {"HeartbeatInitialDelay": 100100000}}}, {"type": "ifNodeIDEquals", "nodeID": 1, "instruction": {"type": "initGossipSub", "gossipSubParams": {"HeartbeatInitialDelay": 100200000}}}, {"type": "ifNodeIDEquals", "nodeID": 2, "instruction": {"type": "initGossipSub", "gossipSubParams": {"HeartbeatInitialDelay": 100300000}}}, {"type": "ifNodeIDEquals", "nodeID": 3, "instruction": {"type": "initGossipSub", "gossipSubParams": {"HeartbeatInitialDelay": 100400000}}}, {"type": "ifNodeIDEquals", "nodeID": 4, "instruction": {"type": "initGossipSub", "gossipSubParams": {"HeartbeatInitialDelay": 100500000}}}, {"type": "ifNodeIDEquals", "nodeID": 5, "instruction": {"type": "initGossipSub", "gossipSubParams": {"HeartbeatInitialDelay": 100600000}}}, {"type": "ifNodeIDEquals", "nodeID": 6, "instruction": {"type": "initGossipSub", "gossipSubParams": {"HeartbeatInitialDelay": 100700000}}}, {"type": "ifNodeIDEquals", "nodeID": 7, "instruction": {"type": "initGossipSub", "gossipSubParams": {"HeartbeatInitialDelay": 100800000}}}, {"type": "ifNodeIDEquals", "nodeID": 8, "instruction": {"type": "initGossipSub", "gossipSubParams": {"HeartbeatInitialDelay": 100900000}}}, {"type": "ifNodeIDEquals", "nodeID": 9, "instruction": {"type": "initGossipSub", "gossipSubParams": {"HeartbeatInitialDelay": 101000000}}}, {"type": "ifNodeIDEquals", "nodeID": 10, "instruction": {"type": "initGossipSub", "gossipSubParams": {"HeartbeatInitialDelay": 101100000}}}, {"type": "ifNodeIDEquals", "nodeID": 11, "instruction": {"type": "initGossipSub", "gossipSubParams": {"HeartbeatInitialDelay": 101200000}}}, {"type": "ifNodeIDEquals", "nodeID": 12, "instruction": {"type": "initGossipSub", "gossipSubParams": {"HeartbeatInitialDelay": 101300000}}}, {"type": "ifNodeIDEquals", "nodeID": 13, "instruction": {"type": "initGossipSub", "gossipSubParams": {"HeartbeatInitialDelay": 101400000}}}, {"type": "ifNodeIDEquals", "nodeID": 14, "instruction": {"type": "initGossipSub", "gossipSubParams": {"HeartbeatInitialDelay": 101500000}}}, {"type": "ifNodeIDEquals", "nodeID": 15, "instruction": {"type": "initGossipSub", "gossipSubParams": {"HeartbeatInitialDelay": 101600000}}}, {"type": "ifNodeIDEquals", "nodeID": 16, "instruction": {"type": "initGossipSub", "gossipSubParams": {"HeartbeatInitialDelay": 101700000}}}, {"type": "ifNodeIDEquals", "nodeID": 17, "instruction": {"type": "initGossipSub", "gossipSubParams": {"HeartbeatInitialDelay": 101800000}}}, {"type": "ifNodeIDEquals", "nodeID": 18, "instruction": {"type": "initGossipSub", "gossipSubParams": {"HeartbeatInitialDelay": 101900000}}}, {"type": "ifNodeIDEquals", "nodeID": 19, "instruction": {"type": "initGossipSub", "gossipSubParams": {"HeartbeatInitialDelay": 102000000}}}, {"type": "ifNodeIDEquals", "nodeID": 20, "instruction": {"type": "initGossipSub", "gossipSubParams": {"HeartbeatInitialDelay": 102100000}}}, {"type": "ifNodeIDEquals", "nodeID": 21, "instruction": {"type": "initGossipSub", "gossipSubParams": {"HeartbeatInitialDelay": 102200000}}}, {"type": "ifNodeIDEquals", "nodeID": 22, "instruction": {"type": "initGossipSub", "gossipSubParams": {"HeartbeatInitialDelay": 102300000}}}, {"type": "ifNodeIDEquals", "nodeID": 23, "instruction": {"type": "initGossipSub", "gossipSubParams": {"HeartbeatInitialDelay": 102400000}}}, {"type": "ifNodeIDEquals", "nodeID": 24, "instruction": {"type": "initGossipSub", "gossipSubParams": {"HeartbeatInitialDelay": 102500000}}}, {"type": "ifNodeIDEquals", "nodeID": 25, "instruction": {"type": "initGossipSub", "gossipSubParams": {"HeartbeatInitialDelay": 102600000}}}, {"type": "ifNodeIDEquals", "nodeID": 26, "instruction": {"type": "initGossipSub", "gossipSubParams": {"HeartbeatInitialDelay": 102700000}}}, {"type": "ifNodeIDEquals", "nodeID": 27, "instruction": {"type": "initGossipSub", "gossipSubParams": {"HeartbeatInitialDelay": 102800000}}}, {"type": "ifNodeIDEquals", "nodeID": 28, "instruction": {"type": "initGossipSub", "gossipSubParams": {"HeartbeatInitialDelay": 102900000}}}, {"type": "ifNodeIDEquals", "nodeID": 29, "instruction": {"type": "initGossipSub", "gossipSubParams": {"HeartbeatInitialDelay": 103000000}}}, {"type": "ifNodeIDEquals", "nodeID": 30, "instruction": {"type": "initGossipSub", "gossipSubParams": {"HeartbeatInitialDelay": 103100000}}}, {"type": "ifNodeIDEquals", "nodeID": 31, "instruction": {"type": "initGossipSub", "gossipSubParams": {"HeartbeatInitialDelay": 103200000}}}, {"type": "setTopicValidationDelay", "topicID": "a-subnet", "delaySeconds": 0.005}, {"type": "ifNodeIDEquals", "nodeID": 0, "instruction": {"type": "connect", "connectTo": [8, 4, 16, 7, 31, 28, 30, 24, 13, 6, 1, 27, 17, 14, 20, 22, 29, 18, 26, 11]}}, {"type": "ifNodeIDEquals", "nodeID": 1, "instruction": {"type": "connect", "connectTo": [18, 7, 21, 27, 12, 19, 31, 25, 2, 30, 15, 26, 11, 23, 5, 28, 6, 10, 14]}}, {"type": "ifNodeIDEquals", "nodeID": 2, "instruction": {"type": "connect", "connectTo": [0, 12, 14, 25, 22, 29, 17, 24, 8, 13, 27, 3, 30, 23, 26, 31, 21, 11, 5]}}, {"type": "ifNodeIDEquals", "nodeID": 3, "instruction": {"type": "connect", "connectTo": [16, 4, 5, 1, 28, 0, 17, 15, 7, 11, 22, 18, 10, 29, 20, 31, 30, 19, 24]}}, {"type": "ifNodeIDEquals", "nodeID": 4, "instruction": {"type": "connect", "connectTo": [21, 26, 12, 16, 6, 13, 27, 1, 14, 25, 9, 2, 10, 28, 20, 19, 8, 29]}}, {"type": "ifNodeIDEquals", "nodeID": 5, "instruction": {"type": "connect", "connectTo": [10, 24, 12, 22, 6, 13, 27, 31, 18, 20, 25, 8, 21, 17, 15, 4, 16]}}, {"type": "ifNodeIDEquals", "nodeID": 6, "instruction": {"type": "connect", "connectTo": [23, 21, 7, 18, 15, 31, 8, 20, 2, 26, 24, 9, 14, 17, 29, 12]}}, {"type": "ifNodeIDEquals", "nodeID": 7, "instruction": {"type": "connect", "connectTo": [15, 26, 10, 28, 27, 24, 18, 16, 30, 20, 13, 2, 25, 4, 29, 22]}}, {"type": "ifNodeIDEquals", "nodeID": 8, "instruction": {"type": "connect", "connectTo": [16, 11, 13, 19, 12, 15, 23, 17, 28, 21, 14, 24, 20, 1, 25]}}, {"type": "ifNodeIDEquals", "nodeID": 9, "instruction": {"type": "connect", "connectTo": [17, 1, 0, 18, 22, 31, 30, 20, 11, 19, 8, 13, 2, 27, 10, 3, 15, 16]}}, {"type": "ifNodeIDEquals", "nodeID": 10, "instruction": {"type": "connect", "connectTo": [28, 27, 16, 29, 0, 25, 21, 31, 26, 22, 8, 17, 11, 14]}}, {"type": "ifNodeIDEquals", "nodeID": 11, "instruction": {"type": "connect", "connectTo": [31, 20, 28, 14, 15, 30, 26, 21, 17, 4, 23, 13, 19]}}, {"type": "ifNodeIDEquals", "nodeID": 12, "instruction": {"type": "connect", "connectTo": [19, 23, 10, 29, 7, 24, 11, 9, 16, 27, 13, 3, 31, 25]}}, {"type": "ifNodeIDEquals", "nodeID": 13, "instruction": {"type": "connect", "connectTo": [22, 24, 10, 16, 6, 17, 28, 15, 27, 25, 20]}}, {"type": "ifNodeIDEquals", "nodeID": 14, "instruction": {"type": "connect", "connectTo": [28, 31, 13, 7, 27, 26, 18, 17, 15, 24, 12, 16]}}, {"type": "ifNodeIDEquals", "nodeID": 15, "instruction": {"type": "connect", "connectTo": [18, 12, 17, 19, 16, 28, 10, 22, 31, 26]}}, {"type": "ifNodeIDEquals", "nodeID": 16, "instruction": {"type": "connect", "connectTo": [24, 18, 6, 1, 23, 19, 27, 22]}}, {"type": "ifNodeIDEquals", "nodeID": 17, "instruction": {"type": "connect", "connectTo": [20, 7, 28, 22, 19, 25, 21, 31]}}, {"type": "ifNodeIDEquals", "nodeID": 18, "instruction": {"type": "connect", "connectTo": [24, 13, 17, 12, 29, 26, 19, 10, 28, 23]}}, {"type": "ifNodeIDEquals", "nodeID": 19, "instruction": {"type": "connect", "connectTo": [0, 24, 27, 25, 21, 31, 26, 22, 29]}}, {"type": "ifNodeIDEquals", "nodeID": 20, "instruction": {"type": "connect", "connectTo": [16, 31, 10, 29, 2, 27, 22, 28, 1]}}, {"type": "ifNodeIDEquals", "nodeID": 21, "instruction": {"type": "connect", "connectTo": [25, 13, 15, 23, 29, 3, 22, 14, 30, 16]}}, {"type": "ifNodeIDEquals", "nodeID": 22, "instruction": {"type": "connect", "connectTo": [14, 1, 25, 27, 12, 4]}}, {"type": "ifNodeIDEquals", "nodeID": 23, "instruction": {"type": "connect", "connectTo": [10, 28, 9, 29, 19, 25, 15, 7, 13, 4, 14]}}, {"type": "ifNodeIDEquals", "nodeID": 24, "instruction": {"type": "connect", "connectTo": [25, 20, 31, 11, 1, 28, 21]}}, {"type": "ifNodeIDEquals", "nodeID": 25, "instruction": {"type": "connect", "connectTo": [11, 6, 14, 31, 28]}}, {"type": "ifNodeIDEquals", "nodeID": 26, "instruction": {"type": "connect", "connectTo": [24, 29, 13, 28, 16, 21, 31, 5]}}, {"type": "ifNodeIDEquals", "nodeID": 27, "instruction": {"type": "connect", "connectTo": [30, 24, 18, 25, 3]}}, {"type": "ifNodeIDEquals", "nodeID": 28, "instruction": {"type": "connect", "connectTo": [16, 5]}}, {"type": "ifNodeIDEquals", "nodeID": 29, "instruction": {"type": "connect", "connectTo": [8, 17, 27, 5, 1, 31]}}, {"type": "ifNodeIDEquals", "nodeID": 30, "instruction": {"type": "connect", "connectTo": [8, 17, 12, 28, 24, 16, 15, 22, 26, 4, 6]}}, {"type": "ifNodeIDEquals", "nodeID": 31, "instruction": {"type": "connect", "connectTo": [13]}}, {"type": "subscribeToTopic", "topicID": "a-subnet", "partial": false}, {"type": "waitUntil", "elapsedSeconds": 120}, {"type": "ifNodeIDEquals", "nodeID": 27, "instruction": {"type": "publish", "messageID": 0, "messageSizeBytes": 98304, "topicID": "a-subnet"}}, {"type": "waitUntil", "elapsedSeconds": 132}, {"type": "ifNodeIDEquals", "nodeID": 3, "instruction": {"type": "publish", "messageID": 1, "messageSizeBytes": 98304, "topicID": "a-subnet"}}, {"type": "waitUntil", "elapsedSeconds": 144}, {"type": "ifNodeIDEquals", "nodeID": 30, "instruction": {"type": "publish", "messageID": 2, "messageSizeBytes": 98304, "topicID": "a-subnet"}}, {"type": "waitUntil", "elapsedSeconds": 156}, {"type": "ifNodeIDEquals", "nodeID": 6, "instruction": {"type": "publish", "messageID": 3, "messageSizeBytes": 98304, "topicID": "a-subnet"}}, {"type": "waitUntil", "elapsedSeconds": 168}, {"type": "ifNodeIDEquals", "nodeID": 2, "instruction": {"type": "publish", "messageID": 4, "messageSizeBytes": 98304, "topicID": "a-subnet"}}, {"type": "waitUntil", "elapsedSeconds": 180}, {"type": "ifNodeIDEquals", "nodeID": 2, "instruction": {"type": "publish", "messageID": 5, "messageSizeBytes": 98304, "topicID": "a-subnet"}}, {"type": "waitUntil", "elapsedSeconds": 192}, {"type": "ifNodeIDEquals", "nodeID": 8, "instruction": {"type": "publish", "messageID": 6, "messageSizeBytes": 98304, "topicID": "a-subnet"}}, {"type": "waitUntil", "elapsedSeconds": 204}, {"type": "ifNodeIDEquals", "nodeID": 28, "instruction": {"type": "publish", "messageID": 7, "messageSizeBytes": 98304, "topicID": "a-subnet"}}, {"type": "waitUntil", "elapsedSeconds": 216}, {"type": "ifNodeIDEquals", "nodeID": 17, "instruction": {"type": "publish", "messageID": 8, "messageSizeBytes": 98304, "topicID": "a-subnet"}}, {"type": "waitUntil", "elapsedSeconds": 228}, {"type": "ifNodeIDEquals", "nodeID": 16, "instruction": {"type": "publish", "messageID": 9, "messageSizeBytes": 98304, "topicID": "a-subnet"}}, {"type": "waitUntil", "elapsedSeconds": 240}, {"type": "ifNodeIDEquals", "nodeID": 5, "instruction": {"type": "publish", "messageID": 10, "messageSizeBytes": 98304, "topicID": "a-subnet"}}, {"type": "waitUntil", "elapsedSeconds": 252}, {"type": "ifNodeIDEquals", "nodeID": 2, "instruction": {"type": "publish", "messageID": 11, "messageSizeBytes": 98304, "topicID": "a-subnet"}}, {"type": "waitUntil", "elapsedSeconds": 264}, {"type": "ifNodeIDEquals", "nodeID": 3, "instruction": {"type": "publish", "messageID": 12, "messageSizeBytes": 98304, "topicID": "a-subnet"}}, {"type": "waitUntil", "elapsedSeconds": 276}, {"type": "ifNodeIDEquals", "nodeID": 20, "instruction": {"type": "publish", "messageID": 13, "messageSizeBytes": 98304, "topicID": "a-subnet"}}, {"type": "waitUntil", "elapsedSeconds": 288}, {"type": "ifNodeIDEquals", "nodeID": 16, "instruction": {"type": "publish", "messageID": 14, "messageSizeBytes": 98304, "topicID": "a-subnet"}}, {"type": "waitUntil", "elapsedSeconds": 300}, {"type": "ifNodeIDEquals", "nodeID": 7, "instruction": {"type": "publish", "messageID": 15, "messageSizeBytes": 98304, "topicID": "a-subnet"}}, {"type": "waitUntil", "elapsedSeconds": 312}, {"type": "waitUntil", "elapsedSeconds": 342}]} \ No newline at end of file diff --git a/Tests/fixtures/gossipsub-rpc-go.json b/Tests/fixtures/gossipsub-rpc-go.json new file mode 100644 index 00000000..9eb12660 --- /dev/null +++ b/Tests/fixtures/gossipsub-rpc-go.json @@ -0,0 +1,168 @@ +{ + "_provenance": { + "api": "proto.Marshal of pb.RPC values, then proto.Unmarshal and proto.Equal for the roundtrip field", + "format": "the RPC payload of one gossipsub frame, as lowercase hex; the uvarint prefix is Streams.Framing", + "generatedBy": "packages/libp2p/protocols/tools/go-rpc-fixtures/main.go", + "module": "github.com/libp2p/go-libp2p-pubsub, pb package", + "source": "vendor/test-plans/gossipsub-interop/go-libp2p/go.mod:8 pins the module version", + "version": "v0.15.1-0.20260305223556-a01777894f4c" + }, + "cases": [ + { + "name": "sub-opt-subscribe", + "hex": "0a0e0801120a746573742d746f7069630a050800120174", + "fields": { + "subscriptions": [ + { + "topicid": "test-topic", + "subscribe": true + }, + { + "topicid": "t", + "subscribe": false + } + ] + }, + "roundtrip": true + }, + { + "name": "publish-basic", + "hex": "121612080001020304050607220a746573742d746f706963", + "fields": { + "publish": [ + { + "topic_str": "test-topic", + "topic_bytes_hex": "746573742d746f706963", + "data_hex": "0001020304050607" + } + ] + }, + "roundtrip": true + }, + { + "name": "publish-empty-data", + "hex": "120c220a746573742d746f706963", + "fields": { + "publish": [ + { + "topic_str": "test-topic", + "topic_bytes_hex": "746573742d746f706963" + } + ] + }, + "roundtrip": true + }, + { + "name": "control-graft", + "hex": "1a0e1a0c0a0a746573742d746f706963", + "fields": { + "control": { + "graft": [ + { + "topic_bytes_hex": "746573742d746f706963" + } + ] + } + }, + "roundtrip": true + }, + { + "name": "control-prune", + "hex": "1a1722150a0a746573742d746f70696312050a03010203183c", + "fields": { + "control": { + "prune": [ + { + "topic_bytes_hex": "746573742d746f706963", + "peers_hex_list": [ + "010203" + ], + "backoff": 60 + } + ] + } + }, + "roundtrip": true + }, + { + "name": "control-ihave", + "hex": "1a220a200a0a746573742d746f7069631208010000000000000012080200000000000000", + "fields": { + "control": { + "ihave": [ + { + "topic_bytes_hex": "746573742d746f706963", + "message_ids_hex": [ + "0100000000000000", + "0200000000000000" + ] + } + ] + } + }, + "roundtrip": true + }, + { + "name": "control-iwant", + "hex": "1a1612140a0801000000000000000a080200000000000000", + "fields": { + "control": { + "iwant": [ + { + "message_ids_hex": [ + "0100000000000000", + "0200000000000000" + ] + } + ] + } + }, + "roundtrip": true + }, + { + "name": "mixed", + "hex": "0a0e0801120a746573742d746f7069630a050800120174121612080001020304050607220a746573742d746f7069631a0e1a0c0a0a746573742d746f706963", + "fields": { + "subscriptions": [ + { + "topicid": "test-topic", + "subscribe": true + }, + { + "topicid": "t", + "subscribe": false + } + ], + "publish": [ + { + "topic_str": "test-topic", + "topic_bytes_hex": "746573742d746f706963", + "data_hex": "0001020304050607" + } + ], + "control": { + "graft": [ + { + "topic_bytes_hex": "746573742d746f706963" + } + ] + } + }, + "roundtrip": true + }, + { + "name": "unknown-fields", + "hex": "121612080001020304050607220a746573742d746f7069634a04deadbeef", + "fields": { + "publish": [ + { + "topic_str": "test-topic", + "topic_bytes_hex": "746573742d746f706963", + "data_hex": "0001020304050607" + } + ] + }, + "roundtrip": true + } + ] +} diff --git a/lake-manifest.json b/lake-manifest.json index bbfa9702..f754f817 100644 --- a/lake-manifest.json +++ b/lake-manifest.json @@ -51,6 +51,13 @@ "manifestFile": "lake-manifest.json", "inherited": false, "dir": "packages/crypto/backend", + "configFile": "lakefile.lean"}, + {"type": "path", + "scope": "", + "name": "InteropHostname", + "manifestFile": "lake-manifest.json", + "inherited": false, + "dir": "packages/cli/hostname", "configFile": "lakefile.lean"}], "name": "leanp2p", "lakeDir": ".lake", diff --git a/lakefile.toml b/lakefile.toml index 26f8ac6f..382b7b94 100644 --- a/lakefile.toml +++ b/lakefile.toml @@ -35,6 +35,10 @@ testDriver = "tests" # The executables need no link argument for the shim. The toolchain # links its own static OpenSSL 3 `libcrypto.a` into every executable, # and the shim archive resolves against that copy. +[[require]] +name = "InteropHostname" +path = "packages/cli/hostname" + [[require]] name = "Backend" path = "packages/crypto/backend" @@ -186,7 +190,7 @@ globs = ["Libp2p.Protocols.*"] [[lean_lib]] name = "Cli" srcDir = "packages/cli" -roots = ["Cli", "Interop"] +roots = ["Cli", "Interop", "GossipsubInterop", "GossipsubSession"] globs = ["Cli", "Interop.*"] # The transport-interop harness of root PLAN W5.9. @@ -221,3 +225,8 @@ root = "Tests.Interop.Yamux" [[lean_exe]] name = "noise-interop" root = "Tests.Interop.Noise" + +[[lean_exe]] +name = "gossipsub-interop" +srcDir = "packages/cli" +root = "Main" diff --git a/packages/cli/GossipsubInterop.lean b/packages/cli/GossipsubInterop.lean new file mode 100644 index 00000000..d8a59d32 --- /dev/null +++ b/packages/cli/GossipsubInterop.lean @@ -0,0 +1,257 @@ +import Conc +import InteropHostname +import Libp2p.Identity +import Libp2p.Protocols.Gossipsub.Interop + +/-! The host-free script driver of the interop participant. `Cli.Main` +supplies the driver callbacks over the real host that `Cli.GossipsubSession` +assembles; the structure's defaults name no host. -/ +set_option autoImplicit false + +namespace Libp2p.Protocols.Gossipsub.Interop + +open Libp2p.Identity +open Libp2p.Protocols.Gossipsub + +/-- Parse the harness command line. -/ +def paramsPath (args : List String) : Except String System.FilePath := + match args with + | ["--params", path] => .ok path + | _ => .error "expected --params " + +/-- Prefer the hostname syscall. Environment variables are a fallback. -/ +def processHostname : IO String := do + let hostname ← try + InteropHostname.get + catch e => + IO.eprintln s!"gethostname failed: {e}; trying environment" + pure "" + if (nodeId? hostname).isSome then return hostname + if !hostname.isEmpty then + IO.eprintln s!"gethostname returned '{hostname}', not nodeN; trying environment" + let value ← IO.getEnv "HOSTNAME" + let fallback ← IO.getEnv "COMPUTERNAME" + if let some name := value then + if (nodeId? name).isSome then return name + if let some name := fallback then + if (nodeId? name).isSome then return name + throw (IO.userError "hostname is not nodeN in syscall or environment") + +/-- The 1 MiB RPC body leaves room for protobuf envelope overhead. -/ +def maxPayloadBytes : Int := 1048576 - 1024 + +/-- Encode an id as eight big-endian bytes, then zero-pad to the requested size. -/ +def publishPayload (messageId size : Int) : Except String ByteArray := do + if messageId < 0 || messageId > 18446744073709551615 then + throw "publish.messageID must fit UInt64" + if size < 8 then throw "publish.messageSizeBytes must be at least eight" + if size > maxPayloadBytes then throw "publish.messageSizeBytes exceeds the 1 MiB RPC budget" + let id := messageId.toNat + let head := (List.range 8).map fun i => UInt8.ofNat ((id / (2 ^ (8 * (7 - i)))) % 256) + return ByteArray.mk (head.toArray ++ Array.replicate (size.toNat - 8) 0) + +/-- Derive the deterministic identity and peer id for nodeN. -/ +def identityForNode (nodeId : Nat) : Except String (PrivKey × PeerId) := do + let seed := ByteArray.mk ((peerSeed nodeId).map UInt8.ofNat).toArray + let key ← (PrivKey.ed25519OfData? seed).mapError (fun e => reprStr e) + let pub ← key.toPublic.mapError (fun e => reprStr e) + let peer ← (PeerId.ofPubKey pub).mapError (fun e => reprStr e) + return (key, peer) + +/-- A bounded stdout owner. Observers only offer complete lines. -/ +structure RecordWriter where + records : Conc.Chan String + tasks : Conc.TaskSet + failed : Conc.OnceRef (Option String) + +namespace RecordWriter + +def start : BaseIO RecordWriter := do + let records ← Conc.Chan.new 64 + let tasks ← Conc.TaskSet.new + let failed ← Conc.OnceRef.new (none : Option String) + tasks.spawn <| do + repeat + match ← records.recv with + | none => break + | some line => + try + IO.println line + (← IO.getStdout).flush + catch e => + failed.update (fun old => old.orElse (fun _ => some (toString e))) + records.close + break + return { records, tasks, failed } + +def offer (writer : RecordWriter) (line : String) : BaseIO Bool := do + let ok ← writer.records.trySend line + if !ok then writer.failed.update (fun old => old.orElse (fun _ => some "record queue full or closed")) + return ok + +def close (writer : RecordWriter) : Conc.Async Unit := do + writer.records.close + writer.tasks.joinAll + if let some error ← writer.failed.get then throw (IO.userError error) + +end RecordWriter + +/-- Build the PeerID NDJSON record before any receipt. -/ +def peerRecord (time peer : String) (nodeId : Nat) : String := + peerIdLine time peer (Int.ofNat nodeId) + +/-- Build a pre-deduplication receipt record. -/ +def receivedRecord (time : String) (payload : List Nat) (sender topic : Option String) : String := + receivedMessageLine time (messageIdNumber payload) sender topic + +/-- Format the RPC fields that matter for the mixed-cell wire trace. -/ +def rpcTraceSummary (rpc : Rpc.RPC) : String := + let controlCount := match rpc.control with + | none => 0 + | some c => c.ihave.length + c.iwant.length + c.graft.length + c.prune.length + let subscriptionTopics := rpc.subscriptions.map (fun s => s.topicid) + let publishTopics := rpc.publish.map (fun m => m.topic.name) + let controlTopics := match rpc.control with + | none => [] + | some c => c.ihave.map (fun x => x.topicId) ++ + c.graft.map (fun x => x.topicId) ++ c.prune.map (fun x => x.topicId) + let topics := subscriptionTopics ++ publishTopics ++ controlTopics + s!"subscriptions={rpc.subscriptions.length} publish={rpc.publish.length} control={controlCount} topic_ids=[{String.intercalate "," topics}]" + +/-- Emit one gated inbound RPC trace to stderr. -/ +def traceInbound (enabled : Bool) (sender : String) (bytes : List Nat) : BaseIO Unit := do + if enabled then + match Core.decodeRpc bytes with + | .ok rpc => discard (IO.eprintln s!"INTEROP_TRACE inbound peer={sender} {rpcTraceSummary rpc}").toBaseIO + | .error _ => discard (IO.eprintln s!"INTEROP_TRACE inbound peer={sender} decode_error").toBaseIO + +/-- Emit one gated outbound RPC trace to stderr. -/ +def traceOutbound (enabled : Bool) (peer : Peer) (rpc : Rpc.RPC) : BaseIO Unit := do + if enabled then discard (IO.eprintln s!"INTEROP_TRACE outbound peer={peer} {rpcTraceSummary rpc}").toBaseIO + +/-- Topic validation delays are updated in one atomic step. -/ +structure DelayTable where + entries : Conc.OnceRef (List (String × Nat)) + stopping : Conc.Chan Unit + +/-- Create the delay table and its shutdown signal. -/ +def DelayTable.new : BaseIO DelayTable := do + return ⟨← Conc.OnceRef.new [], ← Conc.Chan.new 1⟩ + +/-- Replace one topic's delay. -/ +def DelayTable.set (table : DelayTable) (topic : String) (delay : Nat) : BaseIO Unit := + table.entries.update fun entries => (topic, delay) :: entries.filter (fun entry => entry.1 != topic) + +/-- The delay for a batch is the largest delay of its published topics. -/ +def DelayTable.batchDelay (entries : List (String × Nat)) (topics : List String) : Nat := + topics.foldl (fun longest topic => max longest ((entries.find? (fun entry => entry.1 == topic)).map Prod.snd |>.getD 0)) 0 + +/-- Park a stream reader until its delay ends or shutdown starts. -/ +def DelayTable.wait (table : DelayTable) (clock : Conc.Clock) (topics : List String) : Conc.Async Unit := do + let delay := batchDelay (← table.entries.get) topics + if delay > 0 then + let timer ← (clock.after delay : BaseIO (Conc.Park Unit)) + Conc.select #[timer, Conc.Park.recv table.stopping (fun _ => ())] + +/-- Counted pre-deduplication receipt observer. It never parks. -/ +structure ReceiptObserver where + writer : RecordWriter + offers : Conc.OnceRef Nat + failures : Conc.OnceRef Nat + +/-- Create receipt accounting state. -/ +def ReceiptObserver.new (writer : RecordWriter) : BaseIO ReceiptObserver := do + return { writer, offers := ← Conc.OnceRef.new 0, failures := ← Conc.OnceRef.new 0 } + +/-- Format a natural number with at least two decimal digits. -/ +def pad2 (value : Nat) : String := + let text := String.ofList (Nat.toDigits 10 value) + if text.length = 1 then "0" ++ text else text + +/-- Format a natural number with at least three decimal digits. -/ +def pad3 (value : Nat) : String := + let text := String.ofList (Nat.toDigits 10 value) + if text.length = 1 then "00" ++ text + else if text.length = 2 then "0" ++ text + else text + +/-- Format a monotonic millisecond value as fixed-width RFC3339 UTC text. -/ +def formatObservationTime (nowMs : Nat) : String := + let secs := (nowMs / 1000) % 86400 + let ms := nowMs % 1000 + let hours := secs / 3600 + let minutes := (secs / 60) % 60 + let seconds := secs % 60 + "1970-01-01T" ++ pad2 hours ++ ":" ++ pad2 minutes ++ ":" ++ + pad2 seconds ++ "." ++ pad3 ms ++ "Z" + +/-- RFC3339 UTC time from the monotonic millisecond clock. -/ +def observationTime : BaseIO String := do + return formatObservationTime (← IO.monoMsNow) + +/-- Decode one RPC and offer one receipt for every publish entry. -/ +def ReceiptObserver.onRpc (observer : ReceiptObserver) (sender : String) (bytes : List Nat) : BaseIO Unit := do + match Core.decodeRpc bytes with + | .error _ => observer.failures.update (· + 1) + | .ok rpc => + for message in rpc.publish do + let time ← observationTime + let payload := message.data.getD [] + let line := receivedRecord time payload (some sender) (some message.topic.name) + observer.offers.update (· + 1) + if !(← observer.writer.offer line) then observer.failures.update (· + 1) + +/-- Count receipt failures. The supervisor uses this before exit 0. -/ +def ReceiptObserver.failureCount (observer : ReceiptObserver) : BaseIO Nat := observer.failures.get + +/-- Provisional future Node.Config.beforeRpc hook. The current Node has no field. -/ +def provisionalBeforeRpc (table : DelayTable) (clock : Conc.Clock) (_sender : Peer) (bytes : List Nat) : Conc.Async Unit := do + match Core.decodeRpc bytes with + | .error _ => pure () + | .ok rpc => table.wait clock (rpc.publish.map (·.topic.name)) + +/-- Provisional BaseIO callback shape for Node.Config.beforeRpc. -/ +def provisionalOnRpc (observer : ReceiptObserver) (sender : String) (bytes : List Nat) : BaseIO Unit := + observer.onRpc sender bytes + +/-- Host-free callbacks used by the script driver. -/ +structure Driver where + connect : Int → Conc.Async (Except String Unit) := fun _ => pure (.error "host wiring is provisional") + subscribe : String → Bool → BaseIO Bool := fun _ _ => pure false + publish : String → ByteArray → BaseIO Bool := fun _ _ => pure false + validationDelay : String → Nat → BaseIO Unit := fun _ _ => pure () + +/-- Execute selected instructions. waitUntil parks only this script computation. -/ +def runScript (driver : Driver) (clock : Conc.Clock) (start : Nat) (actions : List Instruction) : Conc.Async (Except String Unit) := do + for action in actions do + let result ← match action with + | .connect peers => + let mut outcome : Except String Unit := .ok () + for peer in peers do + if outcome.isOk then outcome ← driver.connect peer + pure outcome + | .subscribeToTopic topic isPartial => + if ← driver.subscribe topic isPartial then pure (.ok ()) else pure (.error "subscribe queue refused") + | .publish id size topic => do + match publishPayload id size with + | .error e => pure (.error e) + | .ok bytes => + if ← driver.publish topic bytes then pure (.ok ()) else pure (.error "publish queue refused") + | .setTopicValidationDelay topic seconds => + driver.validationDelay topic (Float.floor (seconds * 1000.0)).toUInt64.toNat + pure (.ok ()) + | .waitUntil deadline => + repeat + let now ← (clock.nowMs : BaseIO Nat) + let due := start + deadline.toNat * 1000 + if now >= due then break + let left := due - now + let timer ← (clock.after left : BaseIO (Conc.Park Unit)) + discard <| Conc.select #[timer] + pure (.ok ()) + | .initGossipSub _ => pure (.error "unexpected initGossipSub after startup") + | .ifNodeIDEquals _ _ => pure (.error "unresolved node condition") + if let .error error := result then return .error error + return .ok () + +end Libp2p.Protocols.Gossipsub.Interop diff --git a/packages/cli/GossipsubSession.lean b/packages/cli/GossipsubSession.lean new file mode 100644 index 00000000..5300c618 --- /dev/null +++ b/packages/cli/GossipsubSession.lean @@ -0,0 +1,92 @@ +import Conc +import Streams +import Libp2p.Core +import Libp2p.Host +import Libp2p.Mux +import Libp2p.Security.Noise +import Libp2p.Protocols.Gossipsub.HostSeam +import Libp2p.Protocols.Gossipsub.Node +import Libp2p.Protocols.Identify + +/-! Real socket participant wiring. This module owns host assembly only. -/ +set_option autoImplicit false + +namespace Cli.GossipsubSession + +open Conc Streams Multiformats Libp2p.Core Libp2p.Identity Libp2p.Host +open Libp2p.Protocols Libp2p.Protocols.Constants +open Conc Streams Multiformats Libp2p.Core Libp2p.Identity Libp2p.Host +open Libp2p.Protocols Libp2p.Protocols.Constants +open Libp2p.Protocols.Gossipsub Libp2p.Protocols.Gossipsub.Node + +/-- Map the protocols package's mock-seam error onto the core error. The identify package still +types over the mock seam (issue #55), so the participant converts at the host boundary. -/ +def mapMockError : Libp2p.Protocols.Mock.Core.Error → Libp2p.Core.Error + | .protocol s => .protocol s + | .authentication s => .authentication s + | .timeout s => .timeout s + | .closed s => .closed s + +/-- Adapt a handler of the protocols package's mock seam onto the real core handler seam. -/ +def toCoreHandler (h : Libp2p.Protocols.Mock.Core.StreamHandler) : Libp2p.Core.StreamHandler where + id := ⟨h.id.toString⟩ + handle := fun ctx stream => ExceptT.mk do + match ← (h.handle ⟨ctx.remotePeer, ctx.remoteAddr⟩ stream).run with + | .ok () => return .ok () + | .error e => return .error (mapMockError e) + +/-- Use the production yamux session through the core muxer seam. -/ +def yamuxMuxer : Libp2p.Core.Muxer := Libp2p.Mux.yamux + +/-- The real host surface needed by the gossipsub node. -/ +structure Running where + host : Libp2p.Host.Host + node : Node + +/-- The protocol ids this host serves, as the identify responder +advertises them. `Host.new` installs identify; `Node.start` installs +meshsub before `listen`, so every peer that asks after the listen sees +both ids. + +A peer that learns no served id here never opens its own gossipsub +stream to this node: go-libp2p-pubsub adds a peer only after identify +reports a protocol that the router speaks, then opens the router stream. +An empty list leaves the peer with no router stream, so it never enters +the mesh. -/ +def servedProtocols : List String := + [identifyId.toString, meshsubId.toString] + +/-- Build Noise and the real TCP host. Node.start installs meshsub. -/ +def start (identity : PrivKey) (nodeConfig : Node.Config := {}) : IO (Except String Running) := do + let generated ← (Libp2p.Security.Noise.Config.generate identity).run + let noise ← match generated with + | .error e => return .error (reprStr e) + | .ok cfg => pure cfg + let pubkey ← match identity.toPublic with + | .error e => return .error (reprStr e) + | .ok key => pure key + let config : Libp2p.Host.Config := { + identity := identity + transports := [Streams.Tcp.transport] + security := [Libp2p.Security.Noise.securityProtocol noise] + muxers := [yamuxMuxer] + handlers := [toCoreHandler (Identify.respondHandler pubkey [] servedProtocols none "leanp2p")] } + let hostResult ← BaseIO.toIO (Libp2p.Host.Host.new config) + let host ← match hostResult with + | .error e => return .error (reprStr e) + | .ok value => pure value + let node ← BaseIO.toIO (Node.start (HostSeam.ofHost host) nodeConfig) + return .ok { host, node } + +/-- Listen only after Node.start has registered meshsub. -/ +def listen (running : Running) (addr : Multiaddr) : Conc.Async (Except String Multiaddr) := do + return (← running.host.startListening addr).mapError (fun e => reprStr e) + +/-- Stop node and host concurrently, then let the caller drain records. -/ +def stop (running : Running) : Conc.Async Unit := do + let tasks ← TaskSet.new + tasks.spawn running.node.stop + tasks.spawn (Libp2p.Host.Host.close running.host) + tasks.joinAll + +end Cli.GossipsubSession diff --git a/packages/cli/Main.lean b/packages/cli/Main.lean new file mode 100644 index 00000000..0e0b20f2 --- /dev/null +++ b/packages/cli/Main.lean @@ -0,0 +1,132 @@ +import Conc +import Libp2p.Protocols.Gossipsub.Interop +import GossipsubInterop +import GossipsubSession + +/-! The socket participant executable. -/ +set_option autoImplicit false + +namespace Cli + +open Conc Streams Multiformats Libp2p.Core Libp2p.Identity Libp2p.Host +open Libp2p.Protocols Libp2p.Protocols.Constants +open Libp2p.Protocols.Gossipsub Libp2p.Protocols.Gossipsub.Node +open Libp2p.Protocols.Gossipsub.Interop + +private def fail (message : String) : IO UInt32 := do + IO.eprintln message + return 1 + +def load (path : System.FilePath) (nodeId : Nat) : IO (Except String (GossipSubParams × List Instruction)) := do + let source ← IO.FS.readFile path + return (parseScript source).bind (prepareScript (Int.ofNat nodeId)) + +def run (args : List String) : IO UInt32 := do + let start ← Conc.Clock.system.nowMs + let path ← match paramsPath args with + | .ok value => pure value + | .error e => return ← fail e + let hostName ← try + processHostname + catch e => + return ← fail (toString e) + let some nodeId := nodeId? hostName | return ← fail s!"invalid node hostname: {hostName}" + let script ← match ← load path nodeId with + | .ok value => pure value + | .error e => return ← fail e + let (params, actions) := script + let (identity, peer) ← match identityForNode nodeId with + | .ok value => pure value + | .error e => return ← fail e + let writer ← BaseIO.toIO RecordWriter.start + let observer ← BaseIO.toIO (ReceiptObserver.new writer) + let delayTable ← BaseIO.toIO DelayTable.new + let traceEnabled := (← IO.getEnv "INTEROP_TRACE").getD "" == "1" + let coreParams ← match toCoreParams params with + | .ok value => pure value + | .error e => + (RecordWriter.close writer).run + return ← fail e + let receiptCallback : Peer → List Nat → BaseIO Unit := fun sender bytes => do + traceInbound traceEnabled (toString sender) bytes + observer.onRpc (toString sender) bytes + let sendCallback : Peer → Rpc.RPC → BaseIO Unit := fun peer rpc => + traceOutbound traceEnabled peer rpc + let config : Node.Config := { params := coreParams, onRpc := receiptCallback, onSend := sendCallback } + let mut running : Option GossipsubSession.Running := none + let mut resultCode : UInt32 := 1 + try + let started ← GossipsubSession.start identity config + let active ← match started with + | .ok value => pure value + | .error e => throw (IO.userError e) + running := some active + let listenAddr ← match Multiaddr.parse? "/ip4/0.0.0.0/tcp/9000" with + | .ok value => pure value + | .error e => throw (IO.userError (reprStr e)) + match ← (active.host.startListening listenAddr).run with + | .error e => throw (IO.userError (reprStr e)) + | .ok _ => pure () + let peerTime ← BaseIO.toIO observationTime + let peerLine ← BaseIO.toIO (RecordWriter.offer writer (peerRecord peerTime peer.toString nodeId)) + unless peerLine do throw (IO.userError "PeerID record queue full or closed") + let driver : Driver := { + connect := fun target => do + if target < 0 then return .error "connectTo must be nonnegative" + let (_, targetPeer) ← match identityForNode target.toNat with + | .ok value => pure value + | .error e => return .error e + let nodeName := s!"node{target.toNat}" + let ip? ← lookupHostsFile nodeName + let addressText ← match ip? with + | none => return .error s!"no /etc/hosts entry for {nodeName}" + | some ip => + match multiaddrPrefix ip with + | some value => pure value + | none => return .error s!"invalid IPv4 entry for {nodeName}" + let address ← match Multiaddr.parse? addressText with + | .ok value => pure value + | .error e => return .error (reprStr e) + active.host.addAddrs targetPeer [address] + let mut dialResult := ← active.host.dialPeer targetPeer + for attempt in [1, 2, 3, 4] do + match dialResult with + | .error (.unreachable _) => + IO.eprintln s!"dial {nodeName} unreachable, retry {attempt}/5 after 200 ms" + let timer ← (Conc.Clock.system.after 200 : BaseIO (Conc.Park Unit)) + discard <| Conc.select #[timer] + dialResult ← active.host.dialPeer targetPeer + | _ => break + match dialResult with + | .ok joined => + if ← active.node.peerJoined joined .meshsub then pure (.ok ()) + else pure (.error "peerJoined queue refused") + | .error e => pure (.error (reprStr e)) + subscribe := fun topic _ => active.node.subscribe ⟨topic⟩ (fun _ => pure ()) + publish := fun topic payload => active.node.publish ⟨topic⟩ payload + validationDelay := fun topic delay => delayTable.set topic delay } + match ← (runScript driver Conc.Clock.system start actions).run with + | .ok () => resultCode := 0 + | .error e => IO.eprintln e + catch e => + IO.eprintln (toString e) + let stopCode ← match running with + | none => pure 0 + | some active => try + (GossipsubSession.stop active).run + pure 0 + catch e => + IO.eprintln (toString e) + pure 1 + let closeCode ← try + (RecordWriter.close writer).run + pure 0 + catch e => + IO.eprintln (toString e) + pure 1 + let failures ← BaseIO.toIO observer.failureCount + return if resultCode = 0 && stopCode = 0 && closeCode = 0 && failures = 0 then 0 else 1 + +end Cli + +def main (args : List String) : IO UInt32 := Cli.run args diff --git a/packages/cli/hostname/HostnameCheck.lean b/packages/cli/hostname/HostnameCheck.lean new file mode 100644 index 00000000..4df9445f --- /dev/null +++ b/packages/cli/hostname/HostnameCheck.lean @@ -0,0 +1,5 @@ +import InteropHostname + +/-- Print the hostname that the native syscall reports. -/ +def main : IO Unit := do + IO.println (← InteropHostname.get) diff --git a/packages/cli/hostname/InteropHostname.lean b/packages/cli/hostname/InteropHostname.lean new file mode 100644 index 00000000..e64c5245 --- /dev/null +++ b/packages/cli/hostname/InteropHostname.lean @@ -0,0 +1,3 @@ +/-! POSIX hostname binding for the interop participant. -/ +@[extern "interop_participant_hostname"] +opaque InteropHostname.get : IO String diff --git a/packages/cli/hostname/c/hostname.c b/packages/cli/hostname/c/hostname.c new file mode 100644 index 00000000..e35d6973 --- /dev/null +++ b/packages/cli/hostname/c/hostname.c @@ -0,0 +1,12 @@ +#define _POSIX_C_SOURCE 200809L +#include +#include + +LEAN_EXPORT lean_obj_res interop_participant_hostname(lean_obj_arg world) { + (void)world; + char name[256] = {0}; + if (gethostname(name, sizeof(name) - 1) != 0) { + return lean_io_result_mk_error(lean_mk_io_user_error(lean_mk_string("gethostname failed"))); + } + return lean_io_result_mk_ok(lean_mk_string(name)); +} diff --git a/packages/cli/hostname/lake-manifest.json b/packages/cli/hostname/lake-manifest.json new file mode 100644 index 00000000..5a1b3bd9 --- /dev/null +++ b/packages/cli/hostname/lake-manifest.json @@ -0,0 +1,6 @@ +{"version": "1.2.0", + "packagesDir": ".lake/packages", + "packages": [], + "name": "InteropHostname", + "lakeDir": ".lake", + "fixedToolchain": false} diff --git a/packages/cli/hostname/lakefile.lean b/packages/cli/hostname/lakefile.lean new file mode 100644 index 00000000..45002b3b --- /dev/null +++ b/packages/cli/hostname/lakefile.lean @@ -0,0 +1,21 @@ +import Lake +open Lake DSL System + +package InteropHostname + +target interop_hostname.o pkg : FilePath := do + let src := pkg.dir / "c" / "hostname.c" + let obj := pkg.buildDir / "c" / "hostname.o" + let weak := #["-I", (← getLeanIncludeDir).toString] + buildO obj (← inputTextFile src) weak #["-fPIC", "-O2", "-Wall", "-Werror"] "cc" getLeanTrace + +extern_lib libleanp2p_interop_hostname pkg := do + let lib := pkg.staticLibDir / nameToStaticLib "leanp2p_interop_hostname" + buildStaticLib lib #[← interop_hostname.o.fetch] + +@[default_target] +lean_lib InteropHostname where + roots := #[`InteropHostname] + +lean_exe hostname_check where + root := `HostnameCheck diff --git a/packages/libp2p/host/Libp2p/Host/Error.lean b/packages/libp2p/host/Libp2p/Host/Error.lean index 8eb88b5d..fc2b87f1 100644 --- a/packages/libp2p/host/Libp2p/Host/Error.lean +++ b/packages/libp2p/host/Libp2p/Host/Error.lean @@ -1,6 +1,6 @@ import Multiformats import Libp2p.Identity -import Libp2p.Core.Error +import Libp2p.Core /-! # `Host.Error` @@ -20,8 +20,6 @@ set_option autoImplicit false namespace Libp2p.Host --- `Core.Error` resolves to `Libp2p.Core.Error` through the parent --- namespace `Libp2p`. open Multiformats Libp2p.Identity /-- The policy checkpoint that refused a connection (§3.4: a closed @@ -95,4 +93,4 @@ instance : ToString Error := ⟨render⟩ end Error -end Libp2p.Host +end Libp2p.Host \ No newline at end of file diff --git a/packages/libp2p/host/Libp2p/Host/Mock/Negotiate.lean b/packages/libp2p/host/Libp2p/Host/Mock/Negotiate.lean deleted file mode 100644 index b5d1bf2b..00000000 --- a/packages/libp2p/host/Libp2p/Host/Mock/Negotiate.lean +++ /dev/null @@ -1,90 +0,0 @@ -import Conc -import Streams -import Libp2p.Identity -import Libp2p.Core - -/-! -# Mock: the surface of `libp2p/negotiate` that the host uses - -The merged `libp2p/negotiate` takes the seam types of `Libp2p.Core` -(issue #53), so this mock carries those types too. The mock stays -until the host swap of issue #70 lands (RULES.md, "Working ahead of -the tree"). It imports nothing from the negotiate package, and it -states no axiom. Both entries fail with the name of the package: no -body negotiates, runs a handshake, or opens a session. - -The host needs two calls beside `Negotiate.upgrade`. The merged -package has neither, so the swap retypes the host's use of them -(issue #70): - -* `upgradeWith`, the upgrade with a checkpoint after the handshake and - before the muxer stage. The host's gater and resource manager decide - there, by the proved peer (the core ARCHITECTURE, "The policy - shapes"); go's - upgrader calls `InterceptSecured` at the same place - (`vendor/go-libp2p/p2p/net/upgrader/upgrader.go`, line 176). A refusal is a - value, `UpgradeResult.refused`, and no `Core.Error` arm (§3.5). -* `negotiateStream`, multistream-select over one muxed stream. The host - serves each inbound stream through it, and `newStream` negotiates - each outbound one. The bytes it reads past the selecting message go - to the stream's front (§4.7, `Stream.prependBytes`). --/ - -set_option autoImplicit false - -namespace Libp2p.Host.Mock.Negotiate - -open Conc Streams Libp2p.Identity Libp2p.Core - -/-- How a stream negotiation failed. The host reads two arms: the end of -the stream before a selection, an orderly end, and the empty proposal -list. -/ -inductive Error where - /-- The stream ended before the negotiation selected a protocol. -/ - | closed - /-- The initiator has no id to propose. -/ - | noProtocols - /-- Any other failure, rendered. -/ - | failed (detail : String) - deriving DecidableEq, Repr - -/-- The failure as the coarse seam error (§3.5). -/ -def Error.toCore : Error → Core.Error - | .closed => .closed "multistream: the stream ended before a selection" - | .noProtocols => .protocol "multistream: no protocol to propose" - | .failed d => .protocol s!"multistream: {d}" - -/-- A connection that the upgrade finished. -/ -structure Upgraded where - /-- The muxer session over the secure connection. -/ - session : MuxSession - /-- The peer id that the handshake proved, and the upgrader checked. -/ - remotePeer : PeerId - /-- Wait until the session and the secure connection under it end. -/ - join : ExceptT Core.Error Async Unit - -/-- How an upgrade with a checkpoint ended, when no stage failed. -/ -inductive UpgradeResult (ρ : Type) where - /-- Every stage ran, and the checkpoint admitted the peer. -/ - | upgraded (u : Upgraded) - /-- The checkpoint refused the proved peer, for `reason`. -/ - | refused (reason : ρ) - -/-- The upgrade with a checkpoint: negotiate the security protocol, run -the handshake, check the expected peer, call `admit` with the proved -peer, negotiate the muxer, and open the session, all under the 30 s -deadline on `clock` (§5.4). -/ -opaque upgradeWith {ρ : Type} (_clock : Clock) (_role : Role) (_conn : Connection) - (_pending : ByteArray) (_expected : Option PeerId) (_admit : PeerId → BaseIO (Option ρ)) - (_secs : List SecurityProtocol) (_muxers : List Muxer) : - ExceptT Core.Error Async (UpgradeResult ρ) := - throw (.closed "libp2p/negotiate (W5.5) is not integrated") - -/-- Negotiate one of `ids` over a stream, as `role`. The answer is the -selected id and the stream, with the bytes read past the selecting -message at its front. -/ -opaque negotiateStream (_role : Role) (_s : Stream) (_ids : List ProtocolId) : - ExceptT Error Async (ProtocolId × Stream) := - throw (.failed "libp2p/negotiate (W5.5) is not integrated") - -end Libp2p.Host.Mock.Negotiate diff --git a/packages/libp2p/host/Libp2p/Host/Policy.lean b/packages/libp2p/host/Libp2p/Host/Policy.lean index b2a0e513..d5ad12d1 100644 --- a/packages/libp2p/host/Libp2p/Host/Policy.lean +++ b/packages/libp2p/host/Libp2p/Host/Policy.lean @@ -1,4 +1,4 @@ -import Libp2p.Core.Policy +import Libp2p.Core /-! # The policy values @@ -52,4 +52,4 @@ no delay, the form of js-libp2p's `AddressSorter` (the core pair, def keepOrder : AddressRanker where rank addrs := addrs.map fun addr => { addr, delayMs := 0 } -end Libp2p.Host.Policy +end Libp2p.Host.Policy \ No newline at end of file diff --git a/packages/libp2p/host/Libp2p/Host/Swarm.lean b/packages/libp2p/host/Libp2p/Host/Swarm.lean index 54ffc2ea..9711349e 100644 --- a/packages/libp2p/host/Libp2p/Host/Swarm.lean +++ b/packages/libp2p/host/Libp2p/Host/Swarm.lean @@ -3,7 +3,7 @@ import Conc import Streams import Libp2p.Identity import Libp2p.Core -import Libp2p.Host.Mock.Negotiate +import Libp2p.Negotiate.Legacy import Libp2p.Host.Error import Libp2p.Host.Registry import Libp2p.Host.Dedup @@ -69,10 +69,7 @@ set_option autoImplicit false namespace Libp2p.Host --- The negotiate mock stays until the host swap of issue #70; --- `Negotiate.Error` resolves through `Libp2p.Host.Mock`. -open Conc Streams Multiformats Libp2p.Host.Mock Libp2p.Identity - Libp2p.Core Libp2p.Host.Mock.Negotiate +open Conc Streams Multiformats Libp2p.Identity Libp2p.Core Libp2p.Negotiate.Legacy /-- A failure of a task that no caller awaits: an inbound upgrade, a session that ended badly, an inbound stream. -/ @@ -260,7 +257,7 @@ private def upgradeConn (h : Host) (role : Role) (conn : Connection) (expected : conn.closeQuietly let unregister : BaseIO Unit := h.pending.update (·.map (·.erase key)) let r ← try - (upgradeWith h.config.clock role conn .empty expected (h.admit role scope conn) + (upgradeAdmit h.config.clock role conn .empty expected (h.admit role scope conn) h.config.security h.config.muxers).run catch e => unregister @@ -295,8 +292,8 @@ private def serveStream (h : Host) (ctx : StreamContext) (scope : ConnectionScop share.release -- The peer ended the stream before it chose a protocol: an orderly -- end, not a failure. - unless e matches .closed do - h.report (some peer) none (.stream e.toCore) + unless e matches .closed _ do + h.report (some peer) none (.stream e) | .ok (id, s) => match (← (h.handlers.get : BaseIO Handlers)).find? id with | none => @@ -508,7 +505,7 @@ owns the stream. -/ def newStream (h : Host) (peer : PeerId) (protocols : List ProtocolId) : Async (Except Error (ProtocolId × Stream)) := do let some first := protocols.head? - | return .error (.stream (Negotiate.Error.noProtocols.toCore)) + | return .error (.stream (.protocol "multistream: no protocol to propose")) if !(← (h.connected peer : BaseIO Bool)) then if let .error e ← h.dialPeer peer then return .error e let some e := SessionTable.live? (← (h.sessions.get : BaseIO (SessionTable Served))) peer @@ -524,7 +521,7 @@ def newStream (h : Host) (peer : PeerId) (protocols : List ProtocolId) : | .error err => s.reset share.release - return .error (.stream err.toCore) + return .error (.stream err) | .ok (id, s) => -- The share returns when the stream is gone: at a reset, or once -- our close and the peer's end have both happened. @@ -549,4 +546,4 @@ def newStream (h : Host) (peer : PeerId) (protocols : List ProtocolId) : end Host -end Libp2p.Host +end Libp2p.Host \ No newline at end of file diff --git a/packages/libp2p/negotiate/Libp2p/Negotiate/Legacy.lean b/packages/libp2p/negotiate/Libp2p/Negotiate/Legacy.lean new file mode 100644 index 00000000..bc5f4610 --- /dev/null +++ b/packages/libp2p/negotiate/Libp2p/Negotiate/Legacy.lean @@ -0,0 +1,14 @@ +import Libp2p.Negotiate.Legacy.Constants +import Libp2p.Negotiate.Legacy.Message +import Libp2p.Negotiate.Legacy.Step +import Libp2p.Negotiate.Legacy.Upgrade + +/-! +# The legacy negotiate surface + +The host of this branch still runs the pre-seam negotiate API: `upgradeWith` with the +checkpoint hook and `negotiateStream` per muxed stream. The reviewed tip carries the seam +versions of the negotiate package (issue #70 tracks the host swap), so this module keeps the +legacy entry points beside them, under their own namespace. Host files import this module +instead of `Libp2p.Negotiate`. +-/ diff --git a/packages/libp2p/negotiate/Libp2p/Negotiate/Legacy/Constants.lean b/packages/libp2p/negotiate/Libp2p/Negotiate/Legacy/Constants.lean new file mode 100644 index 00000000..13e04526 --- /dev/null +++ b/packages/libp2p/negotiate/Libp2p/Negotiate/Legacy/Constants.lean @@ -0,0 +1,22 @@ +import Libp2p.Core + +/-! +# The constants of the negotiation + +One home per fact (root §3.8). The multistream protocol id and the +1 KiB message cap live only here. +-/ + +set_option autoImplicit false + +namespace Libp2p.Negotiate.Legacy + +/-- The multistream 1.0.0 protocol id. The only protocol id literal of +the package (root §3.4). -/ +def protocolId : Libp2p.Core.ProtocolId := ⟨"/multistream/1.0.0"⟩ + +/-- The 1 KiB multistream cap (root §5.1, contract 1 of the package +docs). -/ +def maxMsg : Nat := 1024 + +end Libp2p.Negotiate.Legacy \ No newline at end of file diff --git a/packages/libp2p/negotiate/Libp2p/Negotiate/Legacy/Message.lean b/packages/libp2p/negotiate/Libp2p/Negotiate/Legacy/Message.lean new file mode 100644 index 00000000..bb2d68c7 --- /dev/null +++ b/packages/libp2p/negotiate/Libp2p/Negotiate/Legacy/Message.lean @@ -0,0 +1,461 @@ +import Libp2p.Negotiate.Legacy.Constants +import Varint +import Bytes + +/-! +# The multistream message + +`Msg` is one multistream message: a uvarint length prefix, then the +UTF-8 bytes of the text plus a trailing newline. The length counts the +newline (ARCHITECTURE §3.2, package docs "Public surface"). + +The decoder splits one frame at the index the prefix computes, then +runs the computed inverse of the payload map: it reads the body as +UTF-8, re-encodes it, and compares. `Bytes.toString?` rejects bad +UTF-8. The comparison rejects a missing trailing newline and a model +value at or above 256. So the `encodeDecode` proof needs no UTF-8 +reasoning. + +The 1 KiB cap is checked in `splitFrame`, before any payload +inspection (contract 1). The `ls` reply framing walks its entries as +ordinary `Msg` frames on a fuel counter, so the entry reader reuses +the proved frame decoder. +-/ + +set_option autoImplicit false + +namespace Libp2p.Negotiate.Legacy + +open Bytes (DecodeError) + +/-! ## The payload map and its inverse -/ + +/-- The model bytes of a text, newline included. The one total map +between texts and payloads. -/ +def payloadOf (s : String) : List Nat := + Bytes.toNatList (Bytes.ofString (s ++ "\n")) + +/-- A total `UVarint` constructor for a length. `initiatorStart` and +`responderStart` keep the upgrade's lengths at or under `maxMsg`; the +modulo keeps the constructor total for any input. -/ +def boundedVarint (n : Nat) : Varint.UVarint := + ⟨n % Varint.limit, Nat.mod_lt _ (by decide)⟩ + +/-- The total text encoder: a uvarint length prefix, then the payload. -/ +def encodeText (s : String) : List Nat := + Varint.encode (boundedVarint (payloadOf s).length) ++ payloadOf s + +/-- A multistream message. `text` excludes the trailing newline. +`fits` is the proof-carrying cap fact of root §3.7: the message fits +the 1 KiB cap, so the encoder writes the exact length. -/ +structure Msg where + text : String + fits : (Bytes.ofString (text ++ "\n")).size ≤ maxMsg + deriving Repr, DecidableEq + +/-- The model payload length equals the byte size of the text with its +newline. -/ +theorem payloadOfLength (s : String) : + (payloadOf s).length = (Bytes.ofString (s ++ "\n")).size := by + simp only [payloadOf, Bytes.toNatList, List.length_map, Array.length_toList] + rfl + +/-- The payload of a text is its UTF-8 bytes plus the newline byte. -/ +theorem payloadOfEq (s : String) : + payloadOf s = Bytes.toNatList (Bytes.ofString s) ++ [10] := by + simp [payloadOf, Bytes.toNatList, Bytes.ofString, String.toByteArray_append] + exact ⟨10, by decide, rfl⟩ + +/-- Every payload ends in a newline, so it is never empty. -/ +theorem payloadOfNe (s : String) : payloadOf s ≠ [] := by + rw [payloadOfEq]; simp + +/-- The checked constructor. The tests build messages through it. -/ +def Msg.ofText? (s : String) : Option Msg := + if h : (Bytes.ofString (s ++ "\n")).size ≤ maxMsg then some ⟨s, h⟩ else none + +/-- A text whose frame fits the 1 KiB cap. `initiatorStart` and +`responderStart` call this before any write, so a configured id that +would produce an over-cap frame is a protocol error rather than a frame +the decoder refuses (root §5.1). -/ +def fitsText (s : String) : Bool := + decide ((Bytes.ofString (s ++ "\n")).size ≤ maxMsg) + +/-- The frame the codec writes for a message. -/ +def encodeMsg (m : Msg) : List Nat := encodeText m.text + +/-- The multistream header message. The literal lives in `Constants`. -/ +def header : Msg := ⟨protocolId.toString, by decide⟩ + +/-! ## The frame split -/ + +/-- Split one frame at the index the length prefix computes. The cap +is checked first, before any payload inspection (contract 1). A zero +declared length is malformed. A payload cut short is truncated, so +the caller keeps waiting. -/ +def splitFrame (bs : List Nat) : Except DecodeError (List Nat × List Nat) := + match Varint.decode? bs with + | .error e => .error e + | .ok (v, rest) => + if maxMsg < v.val then .error .malformed + else if v.val = 0 then .error .malformed + else if rest.length < v.val then .error .truncated + else .ok (rest.take v.val, rest.drop v.val) + +/-- The computed inverse of `payloadOf`, checked by re-encoding. -/ +def textOf? (payload : List Nat) : Option String := + match Bytes.toString? (Bytes.ofNatList payload.dropLast) with + | none => none + | some t => if payloadOf t = payload then some t else none + +/-- A text read back from a payload re-encodes to that payload. -/ +theorem textOf?Spec {payload : List Nat} {t : String} (h : textOf? payload = some t) : + payloadOf t = payload := by + unfold textOf? at h + split at h + · simp at h + · split at h + · simp_all + · simp at h + +/-- The inverse reads its own image back. -/ +theorem textOf?PayloadOf (s : String) : textOf? (payloadOf s) = some s := by + unfold textOf? + have hd : (payloadOf s).dropLast = Bytes.toNatList (Bytes.ofString s) := by + rw [payloadOfEq]; simp + rw [hd, Bytes.ofNatListToNatList, Bytes.toString?OfString] + simp + +/-- A payload whose last byte is not the newline reads back as no +text. -/ +theorem textOf?None {payload : List Nat} (hlast : payload.getLast? ≠ some 10) : + textOf? payload = none := by + unfold textOf? + cases hto : Bytes.toString? (Bytes.ofNatList payload.dropLast) with + | none => rfl + | some t => + by_cases heq : payloadOf t = payload + · have hpl : (payloadOf t).getLast? = some 10 := by rw [payloadOfEq]; simp + rw [heq] at hpl + exact (hlast hpl).elim + · simp [heq] + +/-- Decode one message and the bytes left after it. -/ +def decodeMsg? (bs : List Nat) : Except DecodeError (Msg × List Nat) := + match splitFrame bs with + | .error e => .error e + | .ok (payload, rest) => + match textOf? payload with + | none => .error .malformed + | some t => + if hf : (Bytes.ofString (t ++ "\n")).size ≤ maxMsg then .ok (⟨t, hf⟩, rest) + else .error .malformed + +/-! ## The frame lemmas -/ + +/-- One shared lemma: a framed payload splits back into that payload +and the remainder. -/ +theorem splitFrameEncode (s : String) (h : (payloadOf s).length ≤ maxMsg) (rest : List Nat) : + splitFrame (encodeText s ++ rest) = .ok (payloadOf s, rest) := by + have hlim : (payloadOf s).length < Varint.limit := by + unfold Varint.limit Varint.maxBytes maxMsg at *; omega + have hb : (boundedVarint (payloadOf s).length).val = (payloadOf s).length := by + simp [boundedVarint, Nat.mod_eq_of_lt hlim] + have hpos : 0 < (payloadOf s).length := List.length_pos_iff.mpr (payloadOfNe s) + unfold splitFrame encodeText + rw [List.append_assoc, Varint.decodeEncodeSuffix] + simp only [hb] + have h1 : ¬ maxMsg < (payloadOf s).length := by omega + have h2 : ¬ (payloadOf s).length = 0 := by omega + simp [h1, h2] + +/-- The suffix round trip: a message decodes back from its frame plus +any remainder. The lazy dialer's coalesced read needs this form. The +frame is `encodeText m.text`, definitionally `encodeMsg m`. -/ +theorem decodeMsgSuffix (m : Msg) (rest : List Nat) : + decodeMsg? (encodeText m.text ++ rest) = .ok (m, rest) := by + have h : (payloadOf m.text).length ≤ maxMsg := by rw [payloadOfLength]; exact m.fits + unfold decodeMsg? + rw [splitFrameEncode m.text h] + simp [textOf?PayloadOf, m.fits] + +/-- The inverse direction: every accepted input is the encoding of +the decoded value. No UTF-8 reasoning; `textOf?` already re-encoded. -/ +theorem decodeMsgEncodeDecode (bs : List Nat) (m : Msg) (rest : List Nat) + (h : decodeMsg? bs = .ok (m, rest)) : encodeMsg m ++ rest = bs := by + show encodeText m.text ++ rest = bs + unfold decodeMsg? splitFrame at h + split at h + · simp at h + · rename_i payload rest0 hsplit + split at hsplit + · simp at hsplit + · rename_i v r0 hdec + have hbs := Varint.encodeDecode hdec + split at hsplit + · simp at hsplit + split at hsplit + · simp at hsplit + split at hsplit + · simp at hsplit + rename_i hcap _ hlen + simp only [Except.ok.injEq, Prod.mk.injEq] at hsplit + obtain ⟨rfl, rfl⟩ := hsplit + split at h + · simp at h + · rename_i t ht + have hp := textOf?Spec ht + split at h + · simp only [Except.ok.injEq, Prod.mk.injEq] at h + obtain ⟨rfl, rfl⟩ := h + have hl : (payloadOf t).length = v.val := by + rw [hp]; simp; omega + have hv : boundedVarint (payloadOf t).length = v := by + cases v; simp_all [boundedVarint, Nat.mod_eq_of_lt] + rw [encodeText, hv, hp, List.append_assoc, List.take_append_drop] + exact hbs.symm + · simp at h + +/-- The codec writes the frame and reads one message back. -/ +instance : Bytes.Codec Msg where + encode := encodeMsg + decode? := decodeMsg? + +theorem decodeMsgEncode (m : Msg) : decodeMsg? (encodeMsg m) = .ok (m, []) := by + simpa [encodeMsg] using decodeMsgSuffix m [] + +/-- The round-trip law that §3.2 requires. -/ +instance : Bytes.LawfulCodec Msg where + decodeEncode := decodeMsgEncode + encodeDecode := by + intro bs a rest h + exact decodeMsgEncodeDecode bs a rest h + +/-- The cap fires before the payload is read, for every payload that +follows the prefix (contract 1). -/ +theorem decodeMsgCap {v : Varint.UVarint} {payload : List Nat} + (h : maxMsg < v.val) : + decodeMsg? (Varint.encode v ++ payload) = .error .malformed := by + simp [decodeMsg?, splitFrame, Varint.decodeEncodeSuffix, h] + +/-- A payload cut short is incomplete: more bytes may complete it. -/ +theorem decodeMsgTruncated {v : Varint.UVarint} {payload : List Nat} + (hv0 : 0 < v.val) (hcap : v.val ≤ maxMsg) (hshort : payload.length < v.val) : + decodeMsg? (Varint.encode v ++ payload) = .error .truncated := by + unfold decodeMsg? splitFrame + rw [Varint.decodeEncodeSuffix] + have h1 : ¬ maxMsg < v.val := by omega + have h2 : ¬ v.val = 0 := by omega + simp only [h1, h2, hshort, ite_true, ite_false] + +/-- A zero declared length is malformed. -/ +theorem decodeMsgZeroLength {v : Varint.UVarint} {payload : List Nat} + (h : v.val = 0) : + decodeMsg? (Varint.encode v ++ payload) = .error .malformed := by + simp [decodeMsg?, splitFrame, Varint.decodeEncodeSuffix, h, maxMsg] + +/-- A body whose last byte is not the newline is malformed. -/ +theorem decodeMsgNoNewline {v : Varint.UVarint} {payload : List Nat} + (hv0 : 0 < v.val) (hcap : v.val ≤ maxMsg) (hlen : payload.length = v.val) + (hlast : payload.getLast? ≠ some 10) : + decodeMsg? (Varint.encode v ++ payload) = .error .malformed := by + have hnone : textOf? (List.take v.val payload) = none := by + rw [← hlen, List.take_length] + exact textOf?None hlast + unfold decodeMsg? splitFrame + rw [Varint.decodeEncodeSuffix] + have h1 : ¬ maxMsg < v.val := by omega + have h2 : ¬ v.val = 0 := by omega + have h3 : ¬ payload.length < v.val := by omega + simp only [h1, h2, h3, ite_false] + rw [hnone] + +/-- The framing budget: a message frame spans its payload plus at +most nine prefix bytes. -/ +theorem encodeMsgLength (m : Msg) : + (encodeMsg m).length ≤ maxMsg + Varint.maxBytes := by + have h1 := Varint.lengthEncodeLe (boundedVarint (payloadOf m.text).length) + have h2 : (payloadOf m.text).length ≤ maxMsg := by rw [payloadOfLength]; exact m.fits + simp only [encodeMsg, encodeText, List.length_append] + omega + +/-! ## The `ls` reply framing -/ + +/-- One `ls` entry is an ordinary message frame: the entry's text is +the protocol id, and the frame carries the id plus its newline. -/ +def lsEntries (ids : List Libp2p.Core.ProtocolId) : List Nat := + (ids.map (fun p => encodeText p.toString)).flatten + +/-- The body of an `ls` reply: one entry frame per id, then the final +newline byte that ends the list. -/ +def lsBody (ids : List Libp2p.Core.ProtocolId) : List Nat := lsEntries ids ++ [10] + +/-- The full `ls` reply: the body wrapped in one uvarint length +prefix, exactly like a message. -/ +def encodeLs (ids : List Libp2p.Core.ProtocolId) : List Nat := + Varint.encode (boundedVarint (lsBody ids).length) ++ lsBody ids + +/-- Walk the entries of one complete `ls` body. The fuel counts +entries; the caller passes the body's byte length, which is at least +the entry count because every entry carries its newline. `[10]` alone +is the terminator, not a leading byte: a leading `0x0A` can be a +legitimate one-byte prefix for a nine-byte id. -/ +def decodeEntries : Nat → List Nat → Except DecodeError (List Libp2p.Core.ProtocolId) + | 0, _ => .error .malformed + | fuel + 1, bs => + if bs = [10] then .ok [] + else match decodeMsg? bs with + | .error _ => .error .malformed + | .ok (m, rest) => (decodeEntries fuel rest).map (⟨m.text⟩ :: ·) + +/-- Decode one `ls` reply frame. The outer prefix goes through the +same split and cap as a message. An entry that ends mid-body is +malformed, because only the outer prefix and length can be completed +by more bytes. -/ +def decodeLs? (bs : List Nat) : + Except DecodeError (List Libp2p.Core.ProtocolId × List Nat) := + match splitFrame bs with + | .error e => .error e + | .ok (body, rest) => (decodeEntries body.length body).map (·, rest) + +/-- An entry frame is at least as long as its payload. -/ +theorem encodeTextLength (s : String) : (payloadOf s).length ≤ (encodeText s).length := by + simp [encodeText] + +/-- The entries of an encoded list read back in order, under fuel and +cap bounds. -/ +theorem decodeEntriesLs (ids : List Libp2p.Core.ProtocolId) (fuel : Nat) + (hf : ids.length < fuel) (hb : (lsEntries ids).length ≤ maxMsg) : + decodeEntries fuel (lsEntries ids ++ [10]) = .ok ids := by + induction ids generalizing fuel with + | nil => + cases fuel with + | zero => simp at hf + | succ f => simp [decodeEntries, lsEntries] + | cons p ps ih => + cases fuel with + | zero => simp at hf + | succ f => + have hlen : (lsEntries (p :: ps)).length = + (encodeText p.toString).length + (lsEntries ps).length := by + simp [lsEntries] + have hfit : (Bytes.ofString (p.toString ++ "\n")).size ≤ maxMsg := by + rw [← payloadOfLength]; have := encodeTextLength p.toString; omega + have hne : lsEntries (p :: ps) ++ [10] ≠ [10] := by + intro h + have hl := congrArg List.length h + rw [List.length_append, hlen] at hl + have h0 := List.length_pos_iff.mpr (payloadOfNe p.toString) + have h1 := encodeTextLength p.toString + simp only [List.length_singleton] at hl + omega + have hsplit : lsEntries (p :: ps) ++ [10] = + encodeText (Msg.text ⟨p.toString, hfit⟩) ++ (lsEntries ps ++ [10]) := by + simp [lsEntries] + simp only [decodeEntries, hne, ite_false] + rw [hsplit, decodeMsgSuffix ⟨p.toString, hfit⟩ (lsEntries ps ++ [10])] + change (Except.map (fun x => (⟨p.toString⟩ : Libp2p.Core.ProtocolId) :: x) + (decodeEntries f (lsEntries ps ++ [10])) = .ok (p :: ps)) + rw [ih f (by simp at hf; omega) (by omega)] + rfl + +/-- The byte length of the entries is at least the id count. -/ +theorem lsEntriesLength (ids : List Libp2p.Core.ProtocolId) : + ids.length ≤ (lsEntries ids).length := by + induction ids with + | nil => simp [lsEntries] + | cons p ps ih => + have h0 := List.length_pos_iff.mpr (payloadOfNe p.toString) + have := encodeTextLength p.toString + simp [lsEntries] at ih ⊢; omega + +/-- The `ls` reply round trip. The length prefix alone delimits the +entries, so no cleanliness hypothesis is needed. -/ +theorem decodeLsEncodeLs (ids : List Libp2p.Core.ProtocolId) + (hbody : (lsBody ids).length ≤ maxMsg) : + decodeLs? (encodeLs ids) = .ok (ids, []) := by + have hlim : (lsBody ids).length < Varint.limit := by + unfold Varint.limit Varint.maxBytes maxMsg at *; omega + have hb : (boundedVarint (lsBody ids).length).val = (lsBody ids).length := by + simp [boundedVarint, Nat.mod_eq_of_lt hlim] + have hl := lsEntriesLength ids + unfold decodeLs? encodeLs splitFrame + rw [← List.append_nil (lsBody ids), Varint.decodeEncodeSuffix] + simp only [hb, List.append_nil] + have h1 : ¬ maxMsg < (lsBody ids).length := by omega + have h2 : ¬ (lsBody ids).length = 0 := by simp [lsBody] + simp only [h1, h2, ite_false, Nat.lt_irrefl, List.take_length, List.drop_length] + have hb2 : (lsEntries ids).length ≤ maxMsg := by simp [lsBody] at hbody; omega + rw [lsBody, decodeEntriesLs ids _ (by simp; omega) hb2] + rfl + +/-- A frame that splits back is exactly the prefix plus body plus +remainder. -/ +theorem splitFrameInv {bs body rest : List Nat} (h : splitFrame bs = .ok (body, rest)) : + Varint.encode (boundedVarint body.length) ++ body ++ rest = bs := by + unfold splitFrame at h + split at h + · simp at h + · rename_i v r0 hdec + have hbs := Varint.encodeDecode hdec + split at h + · simp at h + split at h + · simp at h + split at h + · simp at h + rename_i hcap _ hlen + simp only [Except.ok.injEq, Prod.mk.injEq] at h + obtain ⟨rfl, rfl⟩ := h + have hl : (r0.take v.val).length = v.val := by simp; omega + have hv : boundedVarint (r0.take v.val).length = v := by + cases v; simp_all [boundedVarint, Nat.mod_eq_of_lt] + rw [hv, List.append_assoc, List.take_append_drop] + exact hbs.symm + +/-- Entries that read back are exactly the encoding of the list. -/ +theorem decodeEntriesInv (f : Nat) (bs : List Nat) (ids : List Libp2p.Core.ProtocolId) + (h : decodeEntries f bs = .ok ids) : lsEntries ids ++ [10] = bs := by + induction f generalizing bs ids with + | zero => simp [decodeEntries] at h + | succ f ih => + unfold decodeEntries at h + split at h + · rename_i hb + simp only [Except.ok.injEq] at h + subst h + simp [lsEntries, hb] + · split at h + · simp at h + · rename_i m rest hm + cases hr : decodeEntries f rest with + | error e => simp [hr, Except.map] at h + | ok ids' => + simp [hr, Except.map] at h + subst h + have h1 := decodeMsgEncodeDecode bs m rest hm + have h2 := ih rest ids' hr + simp only [lsEntries, List.map_cons, List.flatten_cons, List.append_assoc] at h2 ⊢ + rw [h2] + exact h1 + +/-- The all-input inverse of the `ls` reply: every accepted input is +the encoding of the decoded list. -/ +theorem decodeLsInv {bs : List Nat} {ids : List Libp2p.Core.ProtocolId} {rest : List Nat} + (h : decodeLs? bs = .ok (ids, rest)) : encodeLs ids ++ rest = bs := by + unfold decodeLs? at h + split at h + · simp at h + · rename_i body r hs + cases he : decodeEntries body.length body with + | error e => simp [he, Except.map] at h + | ok ids' => + simp [he, Except.map] at h + obtain ⟨rfl, rfl⟩ := h + have hb := decodeEntriesInv _ _ _ he + have hs' := splitFrameInv hs + unfold encodeLs lsBody + rw [hb] + exact hs' + +end Libp2p.Negotiate.Legacy \ No newline at end of file diff --git a/packages/libp2p/negotiate/Libp2p/Negotiate/Legacy/Step.lean b/packages/libp2p/negotiate/Libp2p/Negotiate/Legacy/Step.lean new file mode 100644 index 00000000..8fedc1b1 --- /dev/null +++ b/packages/libp2p/negotiate/Libp2p/Negotiate/Legacy/Step.lean @@ -0,0 +1,509 @@ +import Libp2p.Negotiate.Legacy.Message +import Libp2p.Core +import Bytes + +/-! +# The pure negotiation core + +`State`, `Event`, `Action`, and `step` are the pure state machine of +multistream-select (root §3.3). One `Event.bytes` feeds raw bytes, and +the core decodes exactly one frame per call, so the +malformed-versus-incomplete call stays in the core (§3.1). A decoded +frame goes through a private `classify` step that names the answer, +then a private `act` step that performs it. The theorems about the +chosen id reason over `classify`; the theorems about writes and phase +reason over `act`. + +The step returns `Except Error (State × List Action)`, the root §3.3 +shape. `Action` has no error constructor (package docs, "Decisions"): +the `Except` channel carries the failure (root §3.5). +-/ + +set_option autoImplicit false + +namespace Libp2p.Negotiate.Legacy + +open Bytes (DecodeError) + +/-- The phase of one negotiation. -/ +inductive Phase where + | headerPending + | negotiating + | awaitingAnswer + | selected + deriving Repr, DecidableEq + +/-- The core state. `role` is the side. `supported` is the responder's +ordered list. `proposals` is the initiator's remaining list, in order. +`buffer` holds the bytes no frame has consumed yet. -/ +structure State where + role : Libp2p.Core.Role + supported : List Libp2p.Core.ProtocolId + proposals : List Libp2p.Core.ProtocolId + buffer : List Nat + phase : Phase + deriving Repr, DecidableEq + +/-- One input to the core. Raw bytes arrive as one event; a proposal +is the local dialer's input. -/ +inductive Event where + | bytes (bs : List Nat) + | propose (p : Libp2p.Core.ProtocolId) + deriving Repr, DecidableEq + +/-- One output of the core. A write carries framed bytes. A select +ends the negotiation with the chosen id and the leftover bytes. -/ +inductive Action where + | write (bs : List Nat) + | select (proto : Libp2p.Core.ProtocolId) (leftover : List Nat) + deriving Repr, DecidableEq + +/-- The replies the responder sends, the closed set of §3.4. -/ +inductive Reply where + | na + | ls + deriving DecidableEq, Repr, Hashable + +/-- The wire text of a reply. -/ +def Reply.text : Reply → String + | .na => "na" + | .ls => "ls" + +/-- The table the inverse searches. -/ +def Reply.all : List Reply := [.na, .ls] + +/-- The computed inverse of `Reply.text`: a text names a reply when it +equals the reply's text. -/ +def Reply.ofText? (s : String) : Option Reply := Reply.all.find? (fun r => r.text = s) + +/-- The table conformance: the inverse reads exactly the reply whose +text it is given. -/ +theorem Reply.ofText?EqSome {s : String} {r : Reply} : + Reply.ofText? s = some r ↔ r.text = s := by + constructor + · intro h + have := List.find?_some h + simpa using this + · intro h + subst h + have hmem : r ∈ Reply.all := by cases r <;> simp [Reply.all] + unfold Reply.ofText? + cases r <;> simp [Reply.all, Reply.text] + +/-- The reply as a message. The text is the message text. -/ +def Reply.toMsg : Reply → Msg + | .na => ⟨"na", by decide⟩ + | .ls => ⟨"ls", by decide⟩ + +/-- The message form of the reply recognition. -/ +def Reply.ofMsg? (m : Msg) : Option Reply := Reply.ofText? m.text + +/-- Why a negotiation failed. `decode` carries a frame the decoder +rejects. `negotiationFailed` covers a bad exchange, an exhausted +proposal list, and an invalid local list at start. -/ +inductive Error where + | decode (e : DecodeError) + | negotiationFailed + deriving Repr, DecidableEq, Hashable + +/-- The pure negotiation core of the spike: the first of `ours` that +`theirs` also supports. -/ +def choose (ours theirs : List Libp2p.Core.ProtocolId) : Option Libp2p.Core.ProtocolId := + List.find? theirs.contains ours + +/-- A chosen id is in our list and supported by theirs. -/ +theorem chooseSpec {ours theirs : List Libp2p.Core.ProtocolId} {p : Libp2p.Core.ProtocolId} + (h : choose ours theirs = some p) : p ∈ ours ∧ theirs.contains p = true := by + unfold choose at h + exact ⟨List.mem_of_find?_eq_some h, List.find?_some h⟩ + +/-! ## The answers -/ + +/-- What a decoded text means for this state. Byte-free, so the +theorems about the chosen id reason over this classification. -/ +private inductive Answer where + | echo + | listIds + | pick (p : Libp2p.Core.ProtocolId) + | refuse + | advance + | retry (q : Libp2p.Core.ProtocolId) (ps : List Libp2p.Core.ProtocolId) + | accept (p : Libp2p.Core.ProtocolId) + | fail + +/-- Classify a decoded text for a state. -/ +private def classify (s : State) (text : String) : Answer := + match s.role, s.phase with + | .responder, .headerPending => if text = protocolId.toString then .echo else .fail + | .responder, .negotiating => + match Reply.ofText? text with + | some .ls => .listIds + | _ => + if text = protocolId.toString then .advance + else if (⟨text⟩ : Libp2p.Core.ProtocolId) ∈ s.supported then .pick ⟨text⟩ + else .refuse + | .initiator, .negotiating => if text = protocolId.toString then .advance else .fail + | .initiator, .awaitingAnswer => + match s.proposals with + | [] => .fail + | p :: ps => + if Reply.ofText? text = some .na then + match ps with + | [] => .fail + | q :: qs => .retry q (q :: qs) + else if text = p.toString then .accept p else .fail + | _, _ => .fail + +/-- Perform an answer. It decodes nothing, so its theorems need no +decoder facts. -/ +private def act (s : State) (rest : List Nat) : Answer → Except Error (State × List Action) + | .echo => .ok ({ s with buffer := rest, phase := .negotiating }, [.write (encodeText protocolId.toString)]) + | .listIds => .ok ({ s with buffer := rest }, [.write (encodeLs s.supported)]) + | .pick p => .ok ({ s with buffer := [], phase := .selected }, + [.write (encodeText p.toString), .select p rest]) + | .refuse => .ok ({ s with buffer := rest }, [.write (encodeMsg (Reply.toMsg .na))]) + | .advance => + match s.role with + | .initiator => .ok ({ s with buffer := rest, phase := .awaitingAnswer }, []) + | .responder => .ok ({ s with buffer := rest }, []) + | .retry q ps => .ok ({ s with buffer := rest, proposals := ps }, [.write (encodeText q.toString)]) + | .accept p => .ok ({ s with buffer := [], phase := .selected }, [.select p rest]) + | .fail => .error .negotiationFailed + +/-- A proposal event. An initiator in the header phase with an empty list +records the id and writes the coalesced header plus the first proposal frame. +Any other proposal event leaves the state unchanged: `initiatorStart` seeds +the ordered list before the first frame moves, and the `na` retry walks that +list from `classify`. -/ +private def propose (s : State) (p : Libp2p.Core.ProtocolId) : Except Error (State × List Action) := + if s.role = .initiator then + if s.phase = .headerPending then + if s.proposals = [] then + .ok ({ s with proposals := [p], phase := .negotiating }, + [.write (encodeMsg header ++ encodeText p.toString)]) + else .ok (s, []) + else .ok (s, []) + else .ok (s, []) + +/-- One transition of the core. A `bytes` event appends to the buffer +and decodes one frame. `truncated` keeps the buffer and waits; +`malformed` is a protocol error; a decoded frame goes through +`classify` and `act`. -/ +def step (s : State) : Event → Except Error (State × List Action) + | .propose p => propose s p + | .bytes bs => + match decodeMsg? (s.buffer ++ bs) with + | .error .truncated => .ok ({ s with buffer := s.buffer ++ bs }, []) + | .error .malformed => .error (.decode .malformed) + | .ok (m, rest) => act s rest (classify s m.text) + +/-! ## The answer lemmas -/ + +/-- A step that selects came from the pick or accept answer. -/ +private theorem actSelect {s : State} {rest : List Nat} {v : Answer} {s' : State} + {acts : List Action} {p : Libp2p.Core.ProtocolId} {r : List Nat} + (h : act s rest v = .ok (s', acts)) (hm : Action.select p r ∈ acts) : + v = .pick p ∨ v = .accept p := by + cases v <;> simp only [act] at h + <;> ((try split at h) + <;> (try simp only [Except.ok.injEq, Prod.mk.injEq] at h) + <;> (try obtain ⟨-, rfl⟩ := h) + <;> (try simp_all)) + +/-- A pick comes from the responder's supported list. -/ +private theorem classifyPick {s : State} {t : String} {p : Libp2p.Core.ProtocolId} + (h : classify s t = .pick p) : p ∈ s.supported := by + unfold classify at h + repeat' split at h + all_goals simp_all + +/-- An accept comes from the head of the initiator's proposals. -/ +private theorem classifyAcceptInitiator {s : State} {t : String} {p : Libp2p.Core.ProtocolId} + (h : classify s t = .accept p) : p ∈ s.proposals := by + unfold classify at h + repeat' split at h + all_goals simp_all + +/-- An accept carries the accepted text. -/ +private theorem classifyAcceptText {s : State} {t : String} {p : Libp2p.Core.ProtocolId} + (h : classify s t = .accept p) : p.toString = t := by + unfold classify at h + repeat' split at h + all_goals simp_all + +/-- A responder never accepts. -/ +private theorem classifyAcceptResponder {s : State} {t : String} {p : Libp2p.Core.ProtocolId} + (hr : s.role = .responder) (h : classify s t = .accept p) : False := by + unfold classify at h + repeat' split at h + all_goals simp_all + +/-- An initiator never picks. -/ +private theorem classifyPickInitiator {s : State} {t : String} {p : Libp2p.Core.ProtocolId} + (hr : s.role = .initiator) (h : classify s t = .pick p) : False := by + unfold classify at h + repeat' split at h + all_goals simp_all + +/-- An initiator never sees the echo answer. -/ +private theorem classifyEchoInitiator {s : State} {t : String} + (hr : s.role = .initiator) (h : classify s t = .echo) : False := by + unfold classify at h + repeat' split at h + all_goals simp_all + +/-- An initiator never sees the list answer. -/ +private theorem classifyListIdsInitiator {s : State} {t : String} + (hr : s.role = .initiator) (h : classify s t = .listIds) : False := by + unfold classify at h + repeat' split at h + all_goals simp_all + +/-- An initiator never sees the refuse answer. -/ +private theorem classifyRefuseInitiator {s : State} {t : String} + (hr : s.role = .initiator) (h : classify s t = .refuse) : False := by + unfold classify at h + repeat' split at h + all_goals simp_all + +/-- The `ls` text in the negotiation draws the list answer. -/ +private theorem classifyListIds {s : State} {t : String} + (hrole : s.role = .responder) (hphase : s.phase = .negotiating) + (hls : Reply.ofText? t = some .ls) : classify s t = .listIds := by + unfold classify + rw [hrole, hphase] + rw [hls] + +/-- An unknown id in the negotiation draws the refuse answer. -/ +private theorem classifyRefuse {s : State} {t : String} + (hrole : s.role = .responder) (hphase : s.phase = .negotiating) + (hnot : (⟨t⟩ : Libp2p.Core.ProtocolId) ∉ s.supported) + (hheader : t ≠ protocolId.toString) (hreply : Reply.ofText? t = none) : + classify s t = .refuse := by + unfold classify + rw [hrole, hphase] + simp only [hreply] + rw [ite_eq_right hheader, ite_eq_right hnot] + +/-- A retry pops one proposal: the new list is the tail of the old. -/ +private theorem classifyRetryTail {s : State} {t : String} {q : Libp2p.Core.ProtocolId} + {ps : List Libp2p.Core.ProtocolId} (h : classify s t = .retry q ps) : + s.proposals.tail = ps := by + unfold classify at h + repeat' split at h + all_goals (cases h <;> simp_all [List.tail_cons]) + +/-- A write came from one of the five writing answers. -/ +private theorem actWriteAnswer {s : State} {rest : List Nat} {v : Answer} {s' : State} + {actions : List Action} {w : List Nat} + (h : act s rest v = .ok (s', actions)) (hw : Action.write w ∈ actions) : + v = .echo ∨ v = .listIds ∨ (∃ p, v = .pick p) ∨ v = .refuse ∨ + (∃ q ps, v = .retry q ps) := by + cases v with + | echo => exact Or.inl rfl + | listIds => exact Or.inr (Or.inl rfl) + | pick p => exact Or.inr (Or.inr (Or.inl ⟨p, rfl⟩)) + | refuse => exact Or.inr (Or.inr (Or.inr (Or.inl rfl))) + | retry q ps => exact Or.inr (Or.inr (Or.inr (Or.inr ⟨q, ps, rfl⟩))) + | advance => + simp only [act] at h + split at h <;> simp only [Except.ok.injEq, Prod.mk.injEq] at h + all_goals (obtain ⟨-, rfl⟩ := h; simp at hw) + | accept p => + simp only [act] at h + simp only [Except.ok.injEq, Prod.mk.injEq] at h + obtain ⟨-, rfl⟩ := h + simp at hw + | fail => + simp only [act] at h + cases h + +/-- The list answer writes the `ls` reply and stays in the phase. -/ +private theorem actListIds {s : State} {rest : List Nat} {s' : State} {actions : List Action} + (h : act s rest .listIds = .ok (s', actions)) : + actions = [.write (encodeLs s.supported)] ∧ s'.phase = s.phase ∧ s'.buffer = rest := by + unfold act at h + simp only [Except.ok.injEq, Prod.mk.injEq] at h + obtain ⟨hstate, hact⟩ := h + subst hact + rw [← hstate] + simp + +/-- The refuse answer writes the `na` frame and stays in the phase. -/ +private theorem actRefuse {s : State} {rest : List Nat} {s' : State} {actions : List Action} + (h : act s rest .refuse = .ok (s', actions)) : + actions = [.write (encodeMsg (Reply.toMsg .na))] ∧ s'.phase = s.phase ∧ s'.buffer = rest := by + unfold act at h + simp only [Except.ok.injEq, Prod.mk.injEq] at h + obtain ⟨hstate, hact⟩ := h + subst hact + rw [← hstate] + simp + +/-- The retry answer replaces the proposals with the remaining list. -/ +private theorem actRetryWrites {s : State} {rest : List Nat} {q : Libp2p.Core.ProtocolId} + {ps : List Libp2p.Core.ProtocolId} {s' : State} {actions : List Action} + (h : act s rest (.retry q ps) = .ok (s', actions)) : s'.proposals = ps := by + unfold act at h + simp only [Except.ok.injEq, Prod.mk.injEq] at h + obtain ⟨hstate, _⟩ := h + rw [← hstate] + +/-! ## The step theorems -/ + +/-- A responder selects only from its supported list (contract 2). -/ +theorem stepSelectResponderMem {s : State} {bs : List Nat} {s' : State} + {actions : List Action} {p : Libp2p.Core.ProtocolId} {rest : List Nat} + (hrole : s.role = .responder) + (hstep : step s (.bytes bs) = .ok (s', actions)) + (hmem : Action.select p rest ∈ actions) : p ∈ s.supported := by + simp only [step] at hstep + split at hstep + · simp only [Except.ok.injEq, Prod.mk.injEq] at hstep + obtain ⟨-, rfl⟩ := hstep + simp at hmem + · simp at hstep + · rcases actSelect hstep hmem with hv | hv + · exact classifyPick hv + · exact (classifyAcceptResponder hrole hv).elim + +/-- An initiator selects only from its own proposal list. -/ +theorem stepSelectInitiatorMem {s : State} {bs : List Nat} {s' : State} + {actions : List Action} {p : Libp2p.Core.ProtocolId} {rest : List Nat} + (hrole : s.role = .initiator) + (hstep : step s (.bytes bs) = .ok (s', actions)) + (hmem : Action.select p rest ∈ actions) : p ∈ s.proposals := by + simp only [step] at hstep + split at hstep + · simp only [Except.ok.injEq, Prod.mk.injEq] at hstep + obtain ⟨-, rfl⟩ := hstep + simp at hmem + · simp at hstep + · rcases actSelect hstep hmem with hv | hv + · exact (classifyPickInitiator hrole hv).elim + · exact classifyAcceptInitiator hv + +/-- A step that writes advances the initiator's proposal list by one +head: the new list is the old tail, so it is strictly shorter +(contract 2, the `na` retry). -/ +theorem stepInitiatorNaProgress {s : State} {bs : List Nat} {s' : State} + {actions : List Action} {p : Libp2p.Core.ProtocolId} {ps : List Libp2p.Core.ProtocolId} + {w : List Nat} + (hrole : s.role = .initiator) (hprop : s.proposals = p :: ps) + (hstep : step s (.bytes bs) = .ok (s', actions)) + (hwrite : Action.write w ∈ actions) : + s'.proposals = ps ∧ s'.proposals.length < s.proposals.length := by + simp only [step] at hstep + split at hstep + · simp only [Except.ok.injEq, Prod.mk.injEq] at hstep + obtain ⟨-, rfl⟩ := hstep + simp at hwrite + · simp at hstep + · rcases actWriteAnswer hstep hwrite with hc | hc | hc | hc | hc + · exact (classifyEchoInitiator hrole hc).elim + · exact (classifyListIdsInitiator hrole hc).elim + · rcases hc with ⟨p', hc⟩ + exact (classifyPickInitiator hrole hc).elim + · exact (classifyRefuseInitiator hrole hc).elim + · rcases hc with ⟨q, ps', hc⟩ + rw [hc] at hstep + have hnew : s'.proposals = ps' := actRetryWrites hstep + have htail : s.proposals.tail = ps' := classifyRetryTail hc + have hps : ps' = ps := by + rw [← htail, hprop] + rfl + constructor + · rw [hnew, hps] + · rw [hnew, hps, hprop] + simp only [List.length_cons] + omega + +/-- The `ls` answer writes the reply and stays in the negotiation +(contract 3). -/ +theorem stepResponderLsStays {s : State} {bs : List Nat} {s' : State} + {actions : List Action} {m : Msg} {rest : List Nat} + (hrole : s.role = .responder) (hphase : s.phase = .negotiating) + (hdec : decodeMsg? (s.buffer ++ bs) = .ok (m, rest)) + (hls : Reply.ofMsg? m = some .ls) + (hstep : step s (.bytes bs) = .ok (s', actions)) : + actions = [.write (encodeLs s.supported)] ∧ s'.phase = .negotiating ∧ s'.buffer = rest := by + simp only [step, hdec] at hstep + have hc := classifyListIds hrole hphase hls + rw [hc] at hstep + have ha := actListIds hstep + rcases ha with ⟨hacts, hphase', hbuf⟩ + constructor + · exact hacts + constructor + · rw [hphase'] + exact hphase + · exact hbuf + +/-- An unknown id draws the `na` frame and stays in the negotiation +(contract 2). -/ +theorem stepResponderUnknownStays {s : State} {bs : List Nat} {s' : State} + {actions : List Action} {m : Msg} {rest : List Nat} {id : Libp2p.Core.ProtocolId} + (hrole : s.role = .responder) (hphase : s.phase = .negotiating) + (hdec : decodeMsg? (s.buffer ++ bs) = .ok (m, rest)) + (htext : m.text = id.toString) (hnot : id ∉ s.supported) + (hheader : m.text ≠ protocolId.toString) + (hreply : Reply.ofMsg? m = none) + (hstep : step s (.bytes bs) = .ok (s', actions)) : + actions = [.write (encodeMsg (Reply.toMsg .na))] ∧ s'.phase = .negotiating ∧ s'.buffer = rest := by + have hnot' : (⟨m.text⟩ : Libp2p.Core.ProtocolId) ∉ s.supported := by + intro hm + apply hnot + simpa [htext] using hm + simp only [step, hdec] at hstep + have hc := classifyRefuse hrole hphase hnot' hheader hreply + rw [hc] at hstep + have ha := actRefuse hstep + rcases ha with ⟨hacts, hphase', hbuf⟩ + constructor + · exact hacts + constructor + · rw [hphase'] + exact hphase + · exact hbuf + +/-- Conservation: the bytes the accepting initiator consumed, plus the +leftover, equal the bytes fed in (contract 5). -/ +theorem stepSelectConservation {s : State} {bs : List Nat} {s' : State} + {p : Libp2p.Core.ProtocolId} {rest r : List Nat} {m : Msg} + (hrole : s.role = .initiator) + (hdec : decodeMsg? (s.buffer ++ bs) = .ok (m, r)) + (hstep : step s (.bytes bs) = .ok (s', [Action.select p rest])) : + s.buffer ++ bs = encodeText p.toString ++ rest := by + simp only [step, hdec] at hstep + have hmem : Action.select p rest ∈ [Action.select p rest] := by simp + have hv := actSelect hstep hmem + rcases hv with hc | hc + · exact (classifyPickInitiator hrole hc).elim + · have hpt : p.toString = m.text := classifyAcceptText hc + rw [hc] at hstep + have hr : r = rest := by + unfold act at hstep + simp only [Except.ok.injEq, Prod.mk.injEq] at hstep + obtain ⟨-, hact⟩ := hstep + injection hact with hsel hnil + injection hsel with hp hrest + have hlaw : encodeText m.text ++ r = s.buffer ++ bs := + decodeMsgEncodeDecode (s.buffer ++ bs) m r hdec + rw [← hlaw] + rw [← hpt] + rw [hr] + +/-- A truncated buffer waits: no actions, no error, the phase holds, +and the buffer keeps the bytes (§3.1). -/ +theorem stepTruncatedWaits {s : State} {bs : List Nat} {s' : State} {actions : List Action} + (hdec : decodeMsg? (s.buffer ++ bs) = .error .truncated) + (hstep : step s (.bytes bs) = .ok (s', actions)) : + actions = [] ∧ s'.phase = s.phase ∧ s'.buffer = s.buffer ++ bs := by + simp only [step, hdec] at hstep + simp only [Except.ok.injEq, Prod.mk.injEq] at hstep + obtain ⟨hstate, hact⟩ := hstep + subst hact + rw [← hstate] + simp + +end Libp2p.Negotiate.Legacy \ No newline at end of file diff --git a/packages/libp2p/negotiate/Libp2p/Negotiate/Legacy/Upgrade.lean b/packages/libp2p/negotiate/Libp2p/Negotiate/Legacy/Upgrade.lean new file mode 100644 index 00000000..3587d597 --- /dev/null +++ b/packages/libp2p/negotiate/Libp2p/Negotiate/Legacy/Upgrade.lean @@ -0,0 +1,639 @@ +import Libp2p.Negotiate.Legacy.Step +import Libp2p.Negotiate.Legacy.Message +import Libp2p.Core +import Libp2p.Identity +import Streams.Conn +import Streams.BufReader +import Streams.Constants +import Conc +import Bytes + +/-! +# The upgrader + +`Upgrade` walks a raw `Connection` through security negotiation, the +security handshake, the identity check, and muxer negotiation into a +`MuxSession` (root §4.9, negotiate ARCHITECTURE, "Public surface"). The +pure part is `drive`: one fuel-bounded run over a byte cursor that +feeds the 1a core one `Event.bytes` call per complete frame. The IO +part is `negotiate`, one negotiation stage, and `upgradeWith`, the +three-stage pipeline under one `Conc.withinDeadline`. + +The driver owns the byte cursor. It never reads `s.buffer`; it resets +the buffer before every `step` call, and every unconsumed byte stays +in the `input` argument. The `leftover` field of the result is +authoritative. The shell passes the driver's own states and leftovers +back unchanged. + +The negotiation loop parks on the connection read until a selection, +an error, a close, or the deadline (root §3.3). No measure bounds that +wait, so the loop keeps the package's `repeat` form, the form of +`Conc.TaskSet.joinAll`, and stays outside the proved core. The pure +`drive` is total, and its theorems carry no IO obligation. + +The three stages thread the leftover bytes by contract 5 of the +package docs: the negotiation leftover seeds the handshake, the +secure connection delivers the handshake leftover through its reader, +and the muxer negotiation leftover reaches `Muxer.open`. +-/ + +set_option autoImplicit false + +namespace Libp2p.Negotiate.Legacy + +open Bytes (DecodeError) + +/-! ## The driver -/ + +/-- The answer of one driver run. + +* `selected p consumed leftover writes`: a `step` selected `p`. + `consumed` is the concatenation of the encoded frames the run + consumed, the selected frame included. `leftover` is the byte suffix + after it. `writes` are the `Action.write` payloads, in order. +* `waiting s consumed leftover writes`: `decodeMsg?` reported + `truncated` on `leftover`. The caller reads more bytes and appends + them; it must not call `drive` again without new bytes. +* `running s consumed leftover writes`: the fuel ran out after a + complete frame. The caller calls `drive` again with `leftover`; it + must not read. -/ +inductive DriveResult where + | selected (p : Libp2p.Core.ProtocolId) (consumed leftover : List Nat) + (writes : List (List Nat)) + | waiting (s : State) (consumed leftover : List Nat) (writes : List (List Nat)) + | running (s : State) (consumed leftover : List Nat) (writes : List (List Nat)) + deriving Repr, DecidableEq + +namespace DriveResult + +/-- The frames the run consumed, as their encoded bytes. -/ +@[simp] def consumed : DriveResult → List Nat + | .selected _ c _ _ => c + | .waiting _ c _ _ => c + | .running _ c _ _ => c + +/-- The bytes the run did not consume. -/ +@[simp] def leftover : DriveResult → List Nat + | .selected _ _ l _ => l + | .waiting _ _ l _ => l + | .running _ _ l _ => l + +end DriveResult + +/-- The `Action.write` payloads of an action list, in order. -/ +def writesOf (actions : List Action) : List (List Nat) := + actions.filterMap fun + | .write bs => some bs + | .select .. => none + +/-- The protocol an action list selects, if it selects one. -/ +def selectOf : List Action → Option Libp2p.Core.ProtocolId + | [] => none + | .write _ :: rest => selectOf rest + | .select p _ :: _ => some p + +/-- One fuel unit consumes at most one complete frame. The +accumulators keep the consumed bytes and the writes of the run. -/ +private def driveAux : Nat → State → List Nat → List Nat → List (List Nat) → + Except Error DriveResult + | 0, s, input, consumed, writes => .ok (.running s consumed input writes) + | fuel + 1, s, input, consumed, writes => + match decodeMsg? input with + | .error .truncated => .ok (.waiting s consumed input writes) + | .error .malformed => .error (.decode .malformed) + | .ok (m, rest) => + match step { s with buffer := [] } (.bytes input) with + | .error e => .error e + | .ok (s', actions) => + match selectOf actions with + | some p => + .ok (.selected p (consumed ++ encodeMsg m) rest + (writes ++ writesOf actions)) + | none => + driveAux fuel s' rest (consumed ++ encodeMsg m) + (writes ++ writesOf actions) + +/-- Drive the pure core over a byte cursor. `s.buffer` is +unused: pass every unconsumed byte in `input`, and read the +`leftover` field of the result as authoritative. One fuel unit +consumes at most one frame. -/ +def drive (fuel : Nat) (s : State) (input : List Nat) : Except Error DriveResult := + driveAux fuel s input [] [] + +/-- The initiator's post-propose state and its one coalesced write. +The write holds the header and the first proposal back-to-back, the +lazy dialer's wire shape (contract 4 of the package docs). The state +carries the whole ordered list, so the core's `na` retry walks it. +An empty ordered list is a protocol error before any byte moves. -/ +def initiatorStart (ids : List Libp2p.Core.ProtocolId) : + Except Error (State × List (List Nat)) := + match ids with + | [] => .error .negotiationFailed + | p :: _ => + if ids.any (fun q => !fitsText q.toString) then .error .negotiationFailed + else + .ok ({ role := .initiator, supported := [], proposals := ids, buffer := [], + phase := .negotiating }, + [encodeMsg header ++ encodeText p.toString]) + +/-- The responder's start: validate the offered list, then send the header +at once. The spec lets either side send first, and the responder that waits +for the dialer's header stalls a dialer that also waits. The state starts in +`negotiating` because the header is already on the wire. -/ +def responderStart (ids : List Libp2p.Core.ProtocolId) : + Except Error (State × List (List Nat)) := + if ids.any (fun q => !fitsText q.toString) then .error .negotiationFailed + else if maxMsg < (lsBody ids).length then .error .negotiationFailed + else + .ok ({ role := .responder, supported := ids, proposals := [], buffer := [], + phase := .negotiating }, + [encodeMsg header]) + +/-! ## The error mappings -/ + +/-- The rendered form of a decode error. `Bytes.DecodeError` carries no +`ToString` instance, so the mapping renders it here. -/ +def renderDecodeError : Bytes.DecodeError → String + | .truncated => "truncated" + | .malformed => "malformed" + +/-- Contract 8's one mapping point: every negotiation error becomes +the `protocol` arm, with the detail rendered. -/ +def mapNegotiateError : Error → Libp2p.Core.Error + | .decode e => .protocol ("multistream decode: " ++ renderDecodeError e) + | .negotiationFailed => .protocol "multistream negotiation failed" + +/-- The identity gate. `none` is the listen path, which accepts any +peer. The dial path compares the derived id against the expectation +and answers `authentication` on a difference, before any muxer byte +moves (negotiate ARCHITECTURE, "Decisions"). -/ +def checkIdentity (expected : Option Libp2p.Identity.PeerId) + (remote : Libp2p.Identity.PeerId) : Except Libp2p.Core.Error Unit := + match expected with + | none => .ok () + | some e => + if e = remote then .ok () + else .error (.authentication "remote peer id differs from the expected peer id") + +/-- The race answer flattened. `Conc.withinDeadline` constructs +`.error .deadlineExpired`; the `.closed` arm is required by the match +and is unreachable from `withinDeadline`. -/ +def settleUpgrade {α : Type} : + Except Conc.Error (Except Libp2p.Core.Error α) → Except Libp2p.Core.Error α + | .ok (.ok a) => .ok a + | .ok (.error e) => .error e + | .error .deadlineExpired => .error (.timeout "upgrade deadline expired") + | .error .closed => .error (.closed "upgrade deadline select closed") + +/-- The answer of one negotiation stage: the selected id and the bytes +the stage over-read. `ByteArray` carries no `Repr` in Lean 4.34, so +the structure derives none; the suite compares the fields it needs. -/ +structure StageResult where + proto : Libp2p.Core.ProtocolId + leftover : ByteArray + +/-! ## The driver theorems -/ + +/-- The accumulator invariant behind the conservation law: the bytes +fed in equal the bytes consumed plus the final leftover, for every +arm of the result. The proof inducts on the fuel; +`decodeMsgEncodeDecode` reassociates each consumed frame. -/ +private theorem driveAux_conservation (fuel : Nat) : + ∀ (s : State) (input consumed : List Nat) (writes : List (List Nat)) + (r : DriveResult), + driveAux fuel s input consumed writes = .ok r → + consumed ++ input = r.consumed ++ r.leftover := by + induction fuel with + | zero => + intro s input consumed writes r h + simp only [driveAux, Except.ok.injEq] at h + subst h + simp + | succ fuel ih => + intro s input consumed writes r h + simp only [driveAux] at h + split at h + · simp only [Except.ok.injEq] at h + subst h + simp + · simp at h + · rename_i m rest hdec + split at h + · rename_i e + simp at h + · rename_i s' actions hstep + split at h + · rename_i p + simp only [Except.ok.injEq] at h + subst h + have hbs := decodeMsgEncodeDecode input m rest hdec + rw [← hbs] + simp [List.append_assoc] + · have hrec := ih s' rest (consumed ++ encodeMsg m) + (writes ++ writesOf actions) r h + have hbs := decodeMsgEncodeDecode input m rest hdec + rw [← hbs, ← List.append_assoc] + exact hrec + +/-- Every byte fed to the driver is consumed or left over. -/ +theorem driveConservation {fuel : Nat} {s : State} {input : List Nat} {r : DriveResult} + (h : drive fuel s input = .ok r) : input = r.consumed ++ r.leftover := by + have h' := driveAux_conservation fuel s input [] [] r h + simpa using h' + +/-- The selected arm: the bytes in equal the frames consumed plus +the leftover handed to the next stage (contract 5). -/ +theorem driveSelectedConservation {fuel : Nat} {s : State} {input : List Nat} + {p : Libp2p.Core.ProtocolId} {consumed leftover : List Nat} + {writes : List (List Nat)} + (h : drive fuel s input = .ok (.selected p consumed leftover writes)) : + input = consumed ++ leftover := by + simpa using driveConservation h + +/-- The truncated arm: every byte the run saw sits in the +consumed frames or in the waiting buffer, so the shell loses no byte +when it reads the next chunk. -/ +theorem driveWaitingConservation {fuel : Nat} {s s' : State} {input : List Nat} + {consumed leftover : List Nat} {writes : List (List Nat)} + (h : drive fuel s input = .ok (.waiting s' consumed leftover writes)) : + input = consumed ++ leftover := by + simpa using driveConservation h + +/-- The accumulator prefix invariant: a run appends the frames it +consumes to the accumulator it received. -/ +private theorem driveAux_consumed_prefix (fuel : Nat) : + ∀ (s : State) (input consumed : List Nat) (writes : List (List Nat)) + (r : DriveResult), + driveAux fuel s input consumed writes = .ok r → + ∃ tail : List Nat, r.consumed = consumed ++ tail := by + induction fuel with + | zero => + intro s input consumed writes r h + simp only [driveAux, Except.ok.injEq] at h + subst h + exact ⟨[], by simp⟩ + | succ fuel ih => + intro s input consumed writes r h + simp only [driveAux] at h + split at h + · simp only [Except.ok.injEq] at h + subst h + exact ⟨[], by simp⟩ + · simp at h + · rename_i m rest hdec + split at h + · rename_i e + simp at h + · rename_i s' actions hstep + split at h + · rename_i p + simp only [Except.ok.injEq] at h + subst h + exact ⟨encodeMsg m, rfl⟩ + · obtain ⟨tail, htail⟩ := + ih s' rest (consumed ++ encodeMsg m) (writes ++ writesOf actions) r h + exact ⟨encodeMsg m ++ tail, by rw [htail, List.append_assoc]⟩ + +/-- A frame the shell writes through `encodeMsg` decodes and appears +at the head of the consumed bytes. -/ +theorem driveConsumedHead {fuel : Nat} {s : State} {m : Msg} {rest : List Nat} + {r : DriveResult} + (h : drive (fuel + 1) s (encodeMsg m ++ rest) = .ok r) : + ∃ tail : List Nat, r.consumed = encodeMsg m ++ tail := by + have h' : driveAux (fuel + 1) s (encodeMsg m ++ rest) [] [] = .ok r := h + simp only [driveAux, encodeMsg, decodeMsgSuffix] at h' + split at h' + · rename_i e + simp at h' + · rename_i s' actions hstep + split at h' + · rename_i p + simp only [Except.ok.injEq] at h' + subst h' + exact ⟨[], by simp [encodeMsg]⟩ + · obtain ⟨tail, htail⟩ := + driveAux_consumed_prefix fuel s' rest ([] ++ encodeMsg m) (writesOf actions) r h' + exact ⟨tail, by simpa [encodeMsg, List.append_assoc] using htail⟩ + +/-- Two stages compose: bytes in equal the consumed bytes of both +stages plus the final leftover. The reusable form of the leftover +hand-over (contract 5). -/ +theorem handoverCompose {ε α β : Type} + {f : List Nat → Except ε (α × List Nat)} + {g : List Nat → Except ε (β × List Nat)} + {c₁ : α → List Nat} {c₂ : β → List Nat} + {bs₁ bs₂ l₁ l₂ : List Nat} {a₁ : α} {a₂ : β} + (law₁ : ∀ x a r, f x = .ok (a, r) → x = c₁ a ++ r) + (law₂ : ∀ x a r, g x = .ok (a, r) → x = c₂ a ++ r) + (h₁ : f bs₁ = .ok (a₁, l₁)) + (h₂ : g (l₁ ++ bs₂) = .ok (a₂, l₂)) : + bs₁ ++ bs₂ = c₁ a₁ ++ c₂ a₂ ++ l₂ := by + rw [law₁ bs₁ a₁ l₁ h₁, List.append_assoc, law₂ (l₁ ++ bs₂) a₂ l₂ h₂] + simp only [List.append_assoc] + +/-- The pure form of one hand-over: when a second drive starts from +the first drive's leftover, the bytes fed in equal the bytes both +drives consumed plus the final leftover. -/ +theorem driveTwostage {fuel₁ fuel₂ : Nat} {s₁ s₂ : State} + {input₁ input₂ : List Nat} {p₁ p₂ : Libp2p.Core.ProtocolId} + {c₁ c₂ l₁ l₂ : List Nat} {w₁ w₂ : List (List Nat)} + (h₁ : drive fuel₁ s₁ input₁ = .ok (.selected p₁ c₁ l₁ w₁)) + (h₂ : drive fuel₂ s₂ (l₁ ++ input₂) = .ok (.selected p₂ c₂ l₂ w₂)) : + input₁ ++ input₂ = c₁ ++ c₂ ++ l₂ := by + have h1 := driveSelectedConservation h₁ + have h2 := driveSelectedConservation h₂ + rw [h1, List.append_assoc, h2] + simp only [List.append_assoc] + +/-- The shell's manual first write equals the core's propose step for +a singleton list. The theorem does not cover a longer list. -/ +theorem initiatorStartPropose {p : Libp2p.Core.ProtocolId} + (hfits : fitsText p.toString = true) : + initiatorStart [p] = + (step { role := .initiator, supported := [], proposals := [], buffer := [], + phase := .headerPending } (.propose p)).map + (fun sa => (sa.1, writesOf sa.2)) := by + have hany : ([p].any (fun q => !fitsText q.toString)) = false := by + simp [List.any, hfits] + simp only [initiatorStart, hany] + rfl + +/-- An empty ordered list is a protocol error before any byte is +written. -/ +theorem initiatorStartNil : initiatorStart [] = .error .negotiationFailed := rfl + +/-- The one mapping point of contract 8: every `Negotiate.Error` +becomes the `protocol` arm, with the detail rendered. -/ +theorem mapNegotiateErrorSpec {e : Bytes.DecodeError} : + mapNegotiateError (.decode e) = + .protocol ("multistream decode: " ++ renderDecodeError e) ∧ + mapNegotiateError .negotiationFailed = + .protocol "multistream negotiation failed" := by + simp [mapNegotiateError] + +/-- A differing derived id is authentication, before any muxer +byte moves. -/ +theorem checkIdentityReject {expected remote : Libp2p.Identity.PeerId} + (h : expected ≠ remote) : + checkIdentity (some expected) remote = + .error (.authentication "remote peer id differs from the expected peer id") := by + simp [checkIdentity, h] + +/-- The dial to the expected peer passes. -/ +theorem checkIdentityAccept {e : Libp2p.Identity.PeerId} : + checkIdentity (some e) e = .ok () := by + simp [checkIdentity] + +/-- The listen path accepts any peer. -/ +theorem checkIdentityNone {r : Libp2p.Identity.PeerId} : + checkIdentity none r = .ok () := rfl + +/-- The deadline wiring of contract 6: the race's expiry is the +`timeout` arm, and the seam errors pass through untouched. The +`.closed` arm is required by the match over `Conc.Error`. -/ +theorem settleUpgradeSpec {α : Type} {e : Libp2p.Core.Error} {a : α} : + settleUpgrade (α := α) (.error .deadlineExpired) = + .error (.timeout "upgrade deadline expired") ∧ + settleUpgrade (α := α) (.error .closed) = + .error (.closed "upgrade deadline select closed") ∧ + settleUpgrade (α := α) (.ok (.error e)) = .error e ∧ + settleUpgrade (.ok (.ok a)) = .ok a := by + simp [settleUpgrade] + +/-! ## The IO shell -/ + +/-- Write one frame, and turn a fault of the connection into the +`closed` arm. `Connection.write` throws `IO.Error`, which is outside +this shell's `Core.Error` channel. -/ +private def writeFrame (conn : Streams.Connection) (b : ByteArray) : + ExceptT Libp2p.Core.Error Conc.Async Unit := do + let ok ← (try + conn.write b + pure true + catch _ => pure false : Conc.Async Bool) + if ok then pure () else throw (.closed "connection write failed during the upgrade") + +/-- Write each frame payload through the connection, in order. A +failed write is the `closed` arm: the connection is gone. -/ +private def sendWrites (conn : Streams.Connection) (ws : List (List Nat)) : + ExceptT Libp2p.Core.Error Conc.Async Unit := do + for w in ws do + writeFrame conn (Bytes.ofNatList w) + +/-- The negotiation loop. It parks on `reader.read` only when `drive` +reports `waiting`; a `running` answer costs no read, because the driver +has not yet tried to decode `leftover`. `reader.read` returns the next +reader, and the loop keeps it for the next read. The read parks for as +long as the peer stays silent, so no measure bounds the loop (root +§3.3). The `repeat` form keeps the definition free of a `partial` +marker, like `Conc.TaskSet.joinAll`. -/ +private def negotiateLoop (reader : Streams.BufReader) + (conn : Streams.Connection) (s : State) (input : List Nat) : + ExceptT Libp2p.Core.Error Conc.Async StageResult := do + let mut s := s + let mut input := input + let mut reader := reader + repeat + match drive 1 s input with + | .error e => throw (mapNegotiateError e) + | .ok (.selected p _ leftover writes) => do + sendWrites conn writes + return { proto := p, leftover := Bytes.ofNatList leftover } + | .ok (.waiting s' _ leftover writes) => do + sendWrites conn writes + match ← reader.read with + | none => throw (.closed "multistream negotiation ended before a selection") + | some (chunk, reader') => + s := s' + input := leftover ++ Bytes.toNatList chunk + reader := reader' + | .ok (.running s' _ leftover writes) => do + sendWrites conn writes + s := s' + input := leftover + -- Every arm above returns or loops, so the `repeat` never falls + -- through. The tail gives the `do` block its result type without an + -- `Inhabited`. + throw (.closed "the negotiation loop ended without an answer") + +/-- One negotiation stage. Both roles write before the loop: the +initiator writes the coalesced header plus first proposal, and the +responder writes its header. The seed is the leftover of the previous +stage. The end of the stream mid-negotiation is `Core.Error.closed`; a +`Negotiate.Error` maps once through `mapNegotiateError`. -/ +def negotiate (role : Libp2p.Core.Role) (reader : Streams.BufReader) + (conn : Streams.Connection) (seed : ByteArray) + (ids : List Libp2p.Core.ProtocolId) : + ExceptT Libp2p.Core.Error Conc.Async StageResult := do + match role with + | .initiator => + match initiatorStart ids with + | .error e => throw (mapNegotiateError e) + | .ok (s, first) => do + sendWrites conn first + negotiateLoop reader conn s (Bytes.toNatList seed) + | .responder => + match responderStart ids with + | .error e => throw (mapNegotiateError e) + | .ok (s, first) => do + sendWrites conn first + negotiateLoop reader conn s (Bytes.toNatList seed) + +/-- An upgraded session and its secure connection completion. -/ +structure Upgraded where + session : Libp2p.Core.MuxSession + remotePeer : Libp2p.Identity.PeerId + join : ExceptT Libp2p.Core.Error Conc.Async Unit + +/-- The result of the post-handshake admission checkpoint. -/ +inductive UpgradeResult (ρ : Type) where + | upgraded (value : Upgraded) + | refused (reason : ρ) + +private def upgradeCore {ρ : Type} (role : Libp2p.Core.Role) (conn : Streams.Connection) + (leftover : ByteArray) (expected : Option Libp2p.Identity.PeerId) + (admit : Libp2p.Identity.PeerId → BaseIO (Option ρ)) + (secs : List Libp2p.Core.SecurityProtocol) (muxers : List Libp2p.Core.Muxer) : + Conc.Async (Except Libp2p.Core.Error (UpgradeResult ρ)) := do + let reader := Streams.BufReader.new conn + match ← negotiate role reader conn leftover (secs.map (·.id)) with + | .error e => return .error e + | .ok stage => + match secs.find? (fun s => decide (s.id = stage.proto)) with + | none => return .error (.protocol "selected security protocol is not configured") + | some sec => + match ← sec.handshake role conn stage.leftover with + | .error e => return .error e + | .ok sc => + match checkIdentity expected sc.remotePeer with + | .error e => + sc.toConnection.closeQuietly + discard sc.join.run + return .error e + | .ok () => + if let some reason ← (admit sc.remotePeer : BaseIO (Option ρ)) then + sc.toConnection.closeQuietly + discard sc.join.run + return .ok (.refused reason) + let muxReader := Streams.BufReader.new sc.toConnection + match ← negotiate role muxReader sc.toConnection ByteArray.empty (muxers.map (·.id)) with + | .error e => + sc.toConnection.closeQuietly + discard sc.join.run + return .error e + | .ok stage2 => + match muxers.find? (fun m => decide (m.id = stage2.proto)) with + | none => + sc.toConnection.closeQuietly + discard sc.join.run + return .error (.protocol "selected muxer is not configured") + | some mux => + match ← mux.«open» role sc.toConnection stage2.leftover with + | .error e => + sc.toConnection.closeQuietly + discard sc.join.run + return .error e + | .ok session => + let join : ExceptT Libp2p.Core.Error Conc.Async Unit := ExceptT.mk do + let muxResult ← session.join.run + let secureResult ← sc.join.run + return match muxResult, secureResult with + | .error e, _ => .error e + | _, .error e => .error e + | .ok (), .ok () => .ok () + return .ok (.upgraded { session, remotePeer := sc.remotePeer, join }) + +/-- Whether the deadline boundary already closed the connection. +`withinDeadline` returns an error value only on expiry, after the hook +ran. -/ +private def boundaryClosed {α : Type} : Except Conc.Error α → Bool + | .error _ => true + | .ok _ => false + +/-- Close a session that finished after the timer won. No caller +receives that session. -/ +private def releaseLate {ρ : Type} (r : Except Libp2p.Core.Error (UpgradeResult ρ)) : BaseIO Unit := + match r with + | .ok (.upgraded value) => discard value.session.close.toBaseIO + | _ => pure () + +/-- The upgrade under one `Conc.withinDeadline`. The expiry closes +the connection once at the boundary (root §3.5), and the race's +`deadlineExpired` maps to the `timeout` arm through `settleUpgrade`. +The clock is a seam, so the deadline test can advance it. -/ + +def upgradeAdmit {ρ : Type} (clock : Conc.Clock) (role : Libp2p.Core.Role) + (conn : Streams.Connection) (leftover : ByteArray) + (expected : Option Libp2p.Identity.PeerId) + (admit : Libp2p.Identity.PeerId → BaseIO (Option ρ)) + (secs : List Libp2p.Core.SecurityProtocol) (muxers : List Libp2p.Core.Muxer) : + ExceptT Libp2p.Core.Error Conc.Async (UpgradeResult ρ) := do + let answer ← Conc.withinDeadline clock Streams.unauthDeadlineMs + (Streams.Connection.closeQuietly conn) + (upgradeCore role conn leftover expected admit secs muxers) + releaseLate + match settleUpgrade answer with + | .ok pair => return pair + | .error e => do + -- An error from the boundary means the expiry hook closed the + -- connection. A failure inside the work leaves it open. + if boundaryClosed answer then throw e + else do + Streams.Connection.closeQuietly conn + throw e + +/-- Upgrade with a clock and admit every authenticated peer. -/ +def upgradeWith (clock : Conc.Clock) (role : Libp2p.Core.Role) + (conn : Streams.Connection) (leftover : ByteArray) + (expected : Option Libp2p.Identity.PeerId) + (secs : List Libp2p.Core.SecurityProtocol) (muxers : List Libp2p.Core.Muxer) : + ExceptT Libp2p.Core.Error Conc.Async (Libp2p.Core.MuxSession × Libp2p.Identity.PeerId) := do + match ← upgradeAdmit clock role conn leftover expected (fun _ => pure (none : Option Empty)) secs muxers with + | .upgraded value => return (value.session, value.remotePeer) + | .refused reason => nomatch reason + +/-- The frozen entry of the package docs. The clock is fixed to the +system clock here and stays a seam in `upgradeWith` for the deadline +test. -/ +def upgrade (role : Libp2p.Core.Role) (conn : Streams.Connection) + (leftover : ByteArray) (expected : Option Libp2p.Identity.PeerId) + (secs : List Libp2p.Core.SecurityProtocol) (muxers : List Libp2p.Core.Muxer) : + ExceptT Libp2p.Core.Error Conc.Async (Libp2p.Core.MuxSession × Libp2p.Identity.PeerId) := + upgradeWith Conc.Clock.system role conn leftover expected secs muxers + +/-- Negotiate a protocol on a muxed stream and retain any extra bytes. -/ +def negotiateStream (role : Libp2p.Core.Role) (stream : Streams.Stream) + (ids : List Libp2p.Core.ProtocolId) : + ExceptT Libp2p.Core.Error Conc.Async (Libp2p.Core.ProtocolId × Streams.Stream) := do + let (initial, writes) ← match role with + | .initiator => (initiatorStart ids).mapError mapNegotiateError + | .responder => (responderStart ids).mapError mapNegotiateError + let mut state := initial + let mut input : List Nat := [] + for bytes in writes do + if let .error e ← stream.write (Bytes.ofNatList bytes) then + throw (.closed s!"stream negotiation write failed: {repr e}") + repeat + match drive 1 state input with + | .error e => throw (mapNegotiateError e) + | .ok result => + let (next, rest, writes, selected) := match result with + | .selected p _ rest writes => (state, rest, writes, some p) + | .waiting s _ rest writes => (s, rest, writes, none) + | .running s _ rest writes => (s, rest, writes, none) + for bytes in writes do + if let .error e ← stream.write (Bytes.ofNatList bytes) then + throw (.closed s!"stream negotiation write failed: {repr e}") + if let some p := selected then + let s ← Streams.Stream.prependBytes stream (Bytes.ofNatList rest) + return (p, s) + state := next + input := rest + if result matches .waiting .. then + match ← stream.read with + | .error e => throw (.closed s!"stream negotiation read failed: {repr e}") + | .ok none => throw (.closed "stream ended during negotiation") + | .ok (some bytes) => input := input ++ Bytes.toNatList bytes + throw (.closed "stream negotiation ended without a selection") + +end Libp2p.Negotiate.Legacy \ No newline at end of file diff --git a/packages/libp2p/protocols/Libp2p/Protocols.lean b/packages/libp2p/protocols/Libp2p/Protocols.lean index 54d4dc2e..b2290462 100644 --- a/packages/libp2p/protocols/Libp2p/Protocols.lean +++ b/packages/libp2p/protocols/Libp2p/Protocols.lean @@ -1,4 +1,12 @@ import Libp2p.Protocols.Constants +import Libp2p.Protocols.Gossipsub.Types +import Libp2p.Protocols.Gossipsub.Rpc +import Libp2p.Protocols.Gossipsub.Score +import Libp2p.Protocols.Gossipsub.Cache +import Libp2p.Protocols.Gossipsub.Sign +import Libp2p.Protocols.Gossipsub.Core +import Libp2p.Protocols.Gossipsub.Node +import Libp2p.Protocols.Gossipsub.Interop import Libp2p.Protocols.StreamError import Libp2p.Protocols.Identity import Libp2p.Protocols.Identify @@ -13,7 +21,6 @@ import Libp2p.Protocols.ReqResp.Exchange /-! # Libp2p.Protocols -The protocols subpackage: identify and ping first, then req/resp, and -gossipsub last (ARCHITECTURE §4.9). The package docs live under `docs/` -beside this file. +The protocols subpackage: identify and ping first, then req/resp, and gossipsub +last (ARCHITECTURE §4.9). The package docs live under `docs/` beside this file. -/ diff --git a/packages/libp2p/protocols/Libp2p/Protocols/Constants.lean b/packages/libp2p/protocols/Libp2p/Protocols/Constants.lean index 6e8de797..877b11f8 100644 --- a/packages/libp2p/protocols/Libp2p/Protocols/Constants.lean +++ b/packages/libp2p/protocols/Libp2p/Protocols/Constants.lean @@ -27,6 +27,19 @@ set_option autoImplicit false namespace Libp2p.Protocols.Constants +/-- The gossipsub v1.1 protocol id, `/meshsub/1.1.0`. -/ +def meshsubId : Libp2p.Core.ProtocolId := ⟨"/meshsub/1.1.0"⟩ + +/-- The gossipsub v1.0 protocol id, `/meshsub/1.0.0`. A v1.1 router speaks v1.0 to a peer that +offers only this id. -/ +def meshsubV10Id : Libp2p.Core.ProtocolId := ⟨"/meshsub/1.0.0"⟩ + +/-- The floodsub protocol id, `/floodsub/1.0.0`. -/ +def floodsubId : Libp2p.Core.ProtocolId := ⟨"/floodsub/1.0.0"⟩ + +/-- The gossipsub RPC cap: 1 MiB, the go-libp2p-pubsub default. -/ +def rpcCap : Nat := 1048576 + /-- The identify protocol id, `/ipfs/id/1.0.0`. -/ def identifyId : Mock.Core.ProtocolId := ⟨"/ipfs/id/1.0.0"⟩ diff --git a/packages/libp2p/protocols/Libp2p/Protocols/Gossipsub/Cache.lean b/packages/libp2p/protocols/Libp2p/Protocols/Gossipsub/Cache.lean new file mode 100644 index 00000000..672afd7c --- /dev/null +++ b/packages/libp2p/protocols/Libp2p/Protocols/Gossipsub/Cache.lean @@ -0,0 +1,700 @@ +import Libp2p.Protocols.Gossipsub.Types + +/-! +# Gossipsub caches + +The four tables of the gossipsub router: the message cache, the seen +cache, the backoff table, and the in-flight IWANT promises. + +The tables are pure data. Time is an input (`Millis`), never a clock, so +the router core drives them from a `step` and the suite drives them with +fixed times. + +Every table is a plain `List`, newest first. An add prepends. The spec +describes the message cache as a ring of history windows; this module +stores one flat list instead and carries each entry's age, which states +the same bound (`age < historyLength`) and makes each theorem one list +induction. + +The spec sections are `vendor/specs/pubsub/gossipsub/gossipsub-v1.0.md` +("Message Cache", and the `seen` cache paragraph) and +`vendor/specs/pubsub/gossipsub/gossipsub-v1.1.md` ("Prune Backoff and +Peer Exchange", and "Spam Protection Measures"). + +The cache takes its parameters as arguments. It reads no default from +`Constants.lean`, so the router core owns the spec defaults. +-/ + +set_option autoImplicit false + +namespace Libp2p.Protocols.Gossipsub.Cache + +/-! ## Message cache + +The message cache keeps the newest `historyLength` heartbeats. A `put` +enters the current window with age zero. A `shift` ages every entry by +one heartbeat and drops the entries that reach `historyLength`. Gossip +lists the ids from the newest `historyGossip` windows of one topic. +-/ + +/-- The window counts of the message cache. `historyLength` is the spec +`mcache_len` (default 5), the number of heartbeats the cache holds. +`historyGossip` is the spec `mcache_gossip` (default 3), the number of +newest heartbeats that gossip lists. The `valid` field carries the spec +bound, so no cache can gossip zero windows or gossip more windows than it +stores. -/ +structure WindowParams where + /-- The number of heartbeats the cache keeps. -/ + historyLength : Nat + /-- The number of newest heartbeats that gossip lists. -/ + historyGossip : Nat + /-- The spec bound `0 < historyGossip` and + `historyGossip ≤ historyLength`. -/ + valid : 0 < historyGossip ∧ historyGossip ≤ historyLength + deriving Repr + +/-- One cached message. `age` counts the heartbeats since the `put`, so +age zero is the current window. The `topic` copies the required topic +field of the message, so `gossipIds` filters a topic without unpacking +the message. -/ +structure Cached where + /-- The message id, computed by the caller. -/ + id : MessageId + /-- The topic of the message. -/ + topic : TopicId + /-- The message itself. -/ + message : Message + /-- The number of heartbeats since the `put`. -/ + age : Nat + deriving DecidableEq, Repr + +/-- The message cache: the window parameters and the entries, newest +first. -/ +structure MessageCache where + /-- The window counts. -/ + params : WindowParams + /-- The cached messages, newest first. -/ + entries : List Cached + deriving Repr + +/-- An empty cache with the given parameters. -/ +def MessageCache.empty (p : WindowParams) : MessageCache := + { params := p, entries := [] } + +/-- Add a message to the current window. The id is an argument, because +the message id function lives in the signing policy and this module must +not import it. The entry takes age zero, the current window. -/ +def MessageCache.put (c : MessageCache) (id : MessageId) (m : Message) : MessageCache := + { c with + entries := { id := id, topic := m.topic, message := m, age := 0 } :: c.entries } + +/-- The message with the given id, if the cache holds it. The search runs +newest first, so a re-`put` id finds its newest message. -/ +def MessageCache.get (c : MessageCache) (id : MessageId) : Option Message := + (c.entries.find? (fun e => decide (e.id = id))).map (fun e => e.message) + +/-- Shift one heartbeat. Every entry ages by one, and an entry whose new +age reaches `historyLength` leaves the cache. -/ +def MessageCache.shift (c : MessageCache) : MessageCache := + { c with + entries := + c.entries.filterMap (fun e => + if e.age + 1 < c.params.historyLength then some { e with age := e.age + 1 } else none) } + +/-- Shift `k` heartbeats. This is the iteration form that the lifetime +theorems state, because the `^[ ]` notation is not part of the core +library. -/ +def MessageCache.shiftN (c : MessageCache) : Nat → MessageCache + | 0 => c + | k + 1 => (MessageCache.shiftN c k).shift + +/-- The ids that gossip lists for a topic: the entries of that topic whose +age is below `historyGossip`, newest first. -/ +def MessageCache.gossipIds (c : MessageCache) (topic : TopicId) : List MessageId := + (c.entries.filter (fun e => decide (e.topic = topic ∧ e.age < c.params.historyGossip))).map + (fun e => e.id) + +/-- The window bound: every entry is younger than `historyLength` +heartbeats. This is the spec bound "at most `history_length` windows" in +the flat-list form. -/ +def MessageCache.windowInvariant (c : MessageCache) : Prop := + ∀ e ∈ c.entries, e.age < c.params.historyLength + +/-- The window bound implies a positive length. -/ +theorem historyLength_pos (p : WindowParams) : 0 < p.historyLength := + Nat.lt_of_lt_of_le p.valid.1 p.valid.2 + +/-- A `put` is found by `get` at its own id. -/ +theorem put_get (c : MessageCache) (id : MessageId) (m : Message) : + MessageCache.get (MessageCache.put c id m) id = some m := by + simp [MessageCache.get, MessageCache.put] + +private theorem shiftN_zero (c : MessageCache) : MessageCache.shiftN c 0 = c := rfl + +private theorem shiftN_succ (c : MessageCache) (k : Nat) : + MessageCache.shiftN c (k + 1) = (MessageCache.shiftN c k).shift := rfl + +private theorem shift_params (c : MessageCache) : (MessageCache.shift c).params = c.params := + rfl + +private theorem shiftN_params (c : MessageCache) (k : Nat) : + (MessageCache.shiftN c k).params = c.params := by + induction k with + | zero => rw [shiftN_zero c] + | succ k ih => rw [shiftN_succ, shift_params, ih] + +/-- Every entry of a shifted cache is an input entry with age one higher, +and that higher age is still below `historyLength`. -/ +private theorem mem_shift_entries {c : MessageCache} {e' : Cached} + (h : e' ∈ (MessageCache.shift c).entries) : + ∃ e ∈ c.entries, + e' = { e with age := e.age + 1 } ∧ e.age + 1 < c.params.historyLength := by + simp only [MessageCache.shift, List.mem_filterMap] at h + obtain ⟨e, he, hf⟩ := h + split at hf + · rename_i hlt + rw [Option.some.injEq] at hf + exact ⟨e, he, hf.symm, hlt⟩ + · simp at hf + +private theorem shift_age_lt {c : MessageCache} {e : Cached} + (h : e ∈ (MessageCache.shift c).entries) : e.age < c.params.historyLength := by + obtain ⟨e0, _, hEq, hlt⟩ := mem_shift_entries h + rw [hEq] + exact hlt + +/-- After `k` shifts every entry is at least `k` heartbeats old. -/ +private theorem shiftN_age_ge (c : MessageCache) (k : Nat) : + ∀ e ∈ (MessageCache.shiftN c k).entries, k ≤ e.age := by + induction k with + | zero => + intro e _ + omega + | succ k ih => + intro e he + rw [shiftN_succ] at he + obtain ⟨e0, he0, hEq, _⟩ := mem_shift_entries he + rw [hEq] + have hge := ih e0 he0 + show k + 1 ≤ e0.age + 1 + omega + +/-- The entries of a shifted cache whose head survives: the head moves to +age one, and the tail shifts as usual. -/ +private theorem shift_entries_cons {c : MessageCache} {e : Cached} {es : List Cached} + (h : c.entries = e :: es) (hlt : e.age + 1 < c.params.historyLength) : + (MessageCache.shift c).entries + = { e with age := e.age + 1 } :: es.filterMap (fun e => + if e.age + 1 < c.params.historyLength then some { e with age := e.age + 1 } else none) := by + simp only [MessageCache.shift] + rw [h] + exact List.filterMap_cons_some (f := fun e => if e.age + 1 < c.params.historyLength + then some { e with age := e.age + 1 } else none) (a := e) (l := es) + (b := { e with age := e.age + 1 }) (by simp [hlt]) + +/-- A `put` entry stays at the head of the cache for `k` shifts, as long +as `k` is below `historyLength`. -/ +private theorem shiftN_put_cons (c : MessageCache) (id : MessageId) (m : Message) + (k : Nat) (hk : k < c.params.historyLength) : + ∃ es, (MessageCache.shiftN (MessageCache.put c id m) k).entries + = { id := id, topic := m.topic, message := m, age := k } :: es := by + induction k with + | zero => exact ⟨c.entries, rfl⟩ + | succ k ih => + have hk' : k < c.params.historyLength := by omega + obtain ⟨es, hes⟩ := ih hk' + have hparams : (MessageCache.shiftN (MessageCache.put c id m) k).params + = c.params := by + rw [shiftN_params] + rfl + have hlt : (⟨id, m.topic, m, k⟩ : Cached).age + 1 + < (MessageCache.shiftN (MessageCache.put c id m) k).params.historyLength := by + rw [hparams] + simpa using hk + refine ⟨es.filterMap (fun e => + if e.age + 1 < (MessageCache.shiftN (MessageCache.put c id m) k).params.historyLength + then some { e with age := e.age + 1 } else none), ?_⟩ + rw [shiftN_succ] + exact shift_entries_cons hes hlt + +/-- After `historyLength` shifts any cache is empty. Entries that start old +enough leave on the first shift, and the rest reach the bound by the last +one. -/ +private theorem shiftN_entries_empty (c : MessageCache) : + (MessageCache.shiftN c c.params.historyLength).entries = [] := by + rw [List.eq_nil_iff_forall_not_mem] + intro e he + have hpos : 0 < c.params.historyLength := historyLength_pos c.params + have hL : c.params.historyLength = (c.params.historyLength - 1) + 1 := by omega + have hge : c.params.historyLength ≤ e.age := by + have := shiftN_age_ge c c.params.historyLength e he + omega + have he' : e ∈ (MessageCache.shift (MessageCache.shiftN c (c.params.historyLength - 1))).entries := by + rw [← shiftN_succ, ← hL] + exact he + have hlt := shift_age_lt he' + rw [shiftN_params] at hlt + omega + +/-- A `put` entry is still found after any number of shifts below +`historyLength`. -/ +theorem put_get_below_length_shifts (c : MessageCache) (id : MessageId) (m : Message) + {k : Nat} (hk : k < c.params.historyLength) : + MessageCache.get (MessageCache.shiftN (MessageCache.put c id m) k) id = some m := by + obtain ⟨es, hes⟩ := shiftN_put_cons c id m k hk + simp [MessageCache.get, hes] + +/-- A `put` entry is gone after `historyLength` shifts. -/ +theorem put_gone_after_length_shifts (c : MessageCache) (id : MessageId) (m : Message) : + MessageCache.get (MessageCache.shiftN (MessageCache.put c id m) c.params.historyLength) id + = none := by + have h : (MessageCache.shiftN (MessageCache.put c id m) c.params.historyLength).entries = [] := by + have hp := shiftN_entries_empty (MessageCache.put c id m) + rwa [show (MessageCache.put c id m).params = c.params from rfl] at hp + simp [MessageCache.get, h] + +/-- Every id that gossip lists is an id that `get` finds. -/ +theorem gossipIds_get_finds (c : MessageCache) (topic : TopicId) : + ∀ id, id ∈ MessageCache.gossipIds c topic → ∃ m, MessageCache.get c id = some m := by + intro id hid + simp only [MessageCache.gossipIds, List.mem_map] at hid + obtain ⟨e, he, rfl⟩ := hid + rw [List.mem_filter] at he + have hsome : (MessageCache.get c e.id).isSome := by + simp only [MessageCache.get, Option.isSome_map, List.find?_isSome] + exact ⟨e, he.1, by simp⟩ + exact Option.isSome_iff_exists.mp hsome + +/-- A `put` keeps every entry inside the window bound. -/ +theorem put_preserves_windowInvariant (c : MessageCache) (id : MessageId) (m : Message) : + MessageCache.windowInvariant c → MessageCache.windowInvariant (MessageCache.put c id m) := by + intro h e he + rw [MessageCache.windowInvariant] at h + simp only [MessageCache.put, List.mem_cons] at he + rcases he with rfl | he + · exact historyLength_pos c.params + · exact h e he + +/-- A `shift` keeps every entry inside the window bound. The input +invariant is not needed, because `shift` drops the entries that would +leave the window; the theorem keeps the hypothesis of the stated target. -/ +theorem shift_preserves_windowInvariant (c : MessageCache) : + MessageCache.windowInvariant c → MessageCache.windowInvariant (MessageCache.shift c) := by + intro _ e he + exact shift_age_lt (c := c) he + +/-! ## Seen cache + +The seen cache holds the message ids observed recently. An id is seen +while `at + ttl > now`, the spec `seen_ttl` of two minutes by default. +The cache is bounded: an add at capacity drops the oldest entry. A +re-add refreshes the entry to the newest position, the "timed least +recently used" rule of the spec. +-/ + +/-- The parameters of the seen cache. `capacity` bounds the entry count. +`ttl` is the spec `seen_ttl`. `valid` carries both positivity facts: a +zero `ttl` would make `add` write an entry that is never seen, and a zero +`capacity` would make the trim drop the entry just added. -/ +structure SeenParams where + /-- The largest number of entries the cache holds. -/ + capacity : Nat + /-- The time an id counts as seen. -/ + ttl : Millis + /-- The bound `0 < capacity` and `0 < ttl`. -/ + valid : 0 < capacity ∧ 0 < ttl + deriving Repr + +/-- One seen id and its insertion time. -/ +structure SeenEntry where + /-- The message id. -/ + id : MessageId + /-- The time of the `add`. -/ + «at» : Millis + deriving DecidableEq, Repr + +/-- The seen cache: the parameters and the entries, newest first. -/ +structure SeenCache where + /-- The bounds. -/ + params : SeenParams + /-- The seen ids, newest first. -/ + entries : List SeenEntry + deriving Repr + +/-- An empty cache with the given parameters. -/ +def SeenCache.empty (p : SeenParams) : SeenCache := + { params := p, entries := [] } + +/-- Whether the cache holds the id as seen at `now`. The check reads the +current time, so an entry expires between prunes. -/ +def SeenCache.contains (c : SeenCache) (id : MessageId) (now : Millis) : Bool := + c.entries.any (fun e => decide (e.id = id ∧ e.at + c.params.ttl > now)) + +/-- Record the id as seen at `now`. The add removes the old entry of the +id, so a re-add moves the id to the newest position, and it trims the +list to `capacity`. The new entry is the head, so the trim drops the +oldest entries and never the new one. -/ +def SeenCache.add (c : SeenCache) (id : MessageId) (now : Millis) : SeenCache := + { c with + entries := + { id := id, «at» := now } :: + (c.entries.filter (fun e => decide (e.id ≠ id))).take (c.params.capacity - 1) } + +/-- Drop the entries whose `at + ttl` has passed. -/ +def SeenCache.prune (c : SeenCache) (now : Millis) : SeenCache := + { c with entries := c.entries.filter (fun e => decide (e.at + c.params.ttl > now)) } + +/-- The tail of an `add` never holds the added id, so `contains` reads the +new head alone. -/ +private theorem contains_add_self (c : SeenCache) (id : MessageId) (t now : Millis) : + SeenCache.contains (SeenCache.add c id t) id now = decide (t + c.params.ttl > now) := by + have htail : ((c.entries.filter (fun e => decide (e.id ≠ id))).take + (c.params.capacity - 1)).any + (fun e => decide (e.id = id ∧ e.at + c.params.ttl > now)) = false := by + rw [List.any_eq_false] + intro e he + have hsub : e ∈ c.entries.filter (fun e => decide (e.id ≠ id)) := + List.mem_of_mem_take he + have hne : e.id ≠ id := of_decide_eq_true (List.mem_filter.mp hsub).2 + simp [hne] + simp only [SeenCache.contains] + rw [show (SeenCache.add c id t).params = c.params from rfl] + simp only [SeenCache.add] + rw [List.any_cons, htail, Bool.or_false] + simp + +/-- An add is seen at its own time. -/ +theorem add_contains (c : SeenCache) (id : MessageId) (now : Millis) : + SeenCache.contains (SeenCache.add c id now) id now = true := by + rw [contains_add_self] + have h : now < now + c.params.ttl := Nat.lt_add_of_pos_right c.params.valid.2 + simp [h] + +/-- An id added at `t` is seen at every time below `t + ttl`. -/ +theorem add_seen_below_ttl (c : SeenCache) (id : MessageId) (t t' : Millis) + (_hle : t ≤ t') (hlt : t' < t + c.params.ttl) : + SeenCache.contains (SeenCache.add c id t) id t' = true := by + rw [contains_add_self] + have h : t + c.params.ttl > t' := hlt + simp [h] + +/-- An id added at `t` is not seen at exactly `t + ttl`. -/ +theorem add_unseen_at_ttl (c : SeenCache) (id : MessageId) (t : Millis) : + SeenCache.contains (SeenCache.add c id t) id (t + c.params.ttl) = false := by + rw [contains_add_self] + simp + +/-- An add never leaves more than `capacity` entries. -/ +theorem add_bounded (c : SeenCache) (id : MessageId) (now : Millis) : + (SeenCache.add c id now).entries.length ≤ c.params.capacity := by + have hlen : (SeenCache.add c id now).entries.length + = min (c.params.capacity - 1) + (c.entries.filter (fun e => decide (e.id ≠ id))).length + 1 := by + simp [SeenCache.add, List.length_take] + rw [hlen] + have hmin : min (c.params.capacity - 1) + (c.entries.filter (fun e => decide (e.id ≠ id))).length ≤ c.params.capacity - 1 := + Nat.min_le_left _ _ + have hcap : 0 < c.params.capacity := c.params.valid.1 + omega + +/-- A prune leaves only entries that are still live at `now`. -/ +theorem prune_removes_expired (c : SeenCache) (now : Millis) : + ∀ e ∈ (SeenCache.prune c now).entries, e.at + c.params.ttl > now := by + intro e he + simp only [SeenCache.prune, List.mem_filter] at he + exact of_decide_eq_true he.2 + +/-! ## Backoff table + +The backoff table holds the period after a prune during which the router +does not regraft a peer for a topic. The spec extends the period when a +`GRAFT` arrives inside it, and this table's `add` is the write that both +sets and extends the period. +-/ + +/-- One backoff period: the pair and the time it ends. -/ +structure BackoffEntry where + /-- The peer. -/ + peer : Peer + /-- The topic. -/ + topic : TopicId + /-- The time the period ends. -/ + «until» : Millis + deriving DecidableEq, Repr + +/-- The backoff table: one entry per pair, newest first. -/ +abbrev BackoffTable := List BackoffEntry + +/-- An empty table. -/ +def Backoff.empty : BackoffTable := [] + +/-- Set the backoff of the pair to end at `now + d`, and replace an older +period for the same pair. A prune, and a `GRAFT` inside the period, both +call this. -/ +def Backoff.add (b : BackoffTable) (peer : Peer) (topic : TopicId) + (d now : Millis) : BackoffTable := + { peer := peer, topic := topic, «until» := now + d } :: + b.filter (fun e => decide (e.peer ≠ peer ∨ e.topic ≠ topic)) + +/-- Whether the pair is in backoff at `now`. The newest entry of the pair +decides, because `add` removes older entries of the same pair. -/ +def Backoff.active (b : BackoffTable) (peer : Peer) (topic : TopicId) + (now : Millis) : Bool := + match b.find? (fun e => decide (e.peer = peer ∧ e.topic = topic)) with + | some e => decide (now < e.until) + | none => false + +/-- Drop the entries whose period has ended. -/ +def Backoff.clear (b : BackoffTable) (now : Millis) : BackoffTable := + b.filter (fun e => decide (now < e.until)) + +/-- A pair added at `t` for `d` is active at every time below `t + d`. -/ +theorem add_active_below (b : BackoffTable) (peer : Peer) (topic : TopicId) + (d t now : Millis) (_hle : t ≤ now) (hlt : now < t + d) : + Backoff.active (Backoff.add b peer topic d t) peer topic now = true := by + simp [Backoff.active, Backoff.add, hlt] + +/-- A pair added at `t` for `d` is not active at `t + d`. -/ +theorem add_inactive_at_expiry (b : BackoffTable) (peer : Peer) (topic : TopicId) + (d t : Millis) : + Backoff.active (Backoff.add b peer topic d t) peer topic (t + d) = false := by + simp [Backoff.active, Backoff.add] + +/-- A second add for the same pair replaces the first period. -/ +theorem add_overwrites (b : BackoffTable) (peer : Peer) (topic : TopicId) + (d t d' t' now : Millis) (_hle : t' ≤ now) (hlt : now < t' + d') : + Backoff.active (Backoff.add (Backoff.add b peer topic d t) peer topic d' t') peer topic now + = true := by + simp [Backoff.active, Backoff.add, hlt] + +/-- If a `find?` finds an entry, and that entry satisfies `q`, then the +search for `q` together with the original predicate finds the same entry. -/ +private theorem find?_and_of_find? {α : Type} (l : List α) (p q : α → Bool) {e : α} + (h : l.find? p = some e) (hq : q e = true) : + l.find? (fun x => decide (q x = true ∧ p x = true)) = some e := by + induction l with + | nil => simp at h + | cons a as ih => + by_cases hpa : p a = true + · rw [List.find?_cons_of_pos hpa] at h + have hae : a = e := by injection h + have hqa : q a = true := by rw [hae]; exact hq + rw [List.find?_cons_of_pos (by simp [hpa, hqa]), hae] + · rw [List.find?_cons_of_neg hpa] at h + rw [List.find?_cons_of_neg (by simp [hpa])] + exact ih h + +/-- Clearing the expired entries keeps the active ones. -/ +theorem clear_keeps_active (b : BackoffTable) (peer : Peer) (topic : TopicId) + (now : Millis) (h : Backoff.active b peer topic now = true) : + Backoff.active (Backoff.clear b now) peer topic now = true := by + simp only [Backoff.active] at h + split at h + · rename_i e hfind + have hlive : decide (now < e.until) = true := h + have hand := find?_and_of_find? b (fun e => decide (e.peer = peer ∧ e.topic = topic)) + (fun e => decide (now < e.until)) hfind hlive + have hfilter : (b.filter (fun e => decide (now < e.until))).find? + (fun e => decide (e.peer = peer ∧ e.topic = topic)) = some e := by + rw [List.find?_filter] + exact hand + simp only [Backoff.clear, Backoff.active, hfilter, hlive] + · simp at h + +/-- A clear leaves only entries whose period has not ended. -/ +theorem clear_removes_expired (b : BackoffTable) (now : Millis) : + ∀ e ∈ Backoff.clear b now, now < e.until := by + intro e he + simp only [Backoff.clear, List.mem_filter] at he + exact of_decide_eq_true he.2 + +/-! ## IWANT promises + +The router promises to answer an `IWANT` within a deadline. A delivery +of the id removes every promise for it. A promise whose deadline passed +without a delivery counts against the advertising peer, the input of the +`P₇` behavior penalty of the score part. +-/ + +/-- One in-flight promise: the peer that asked, the id it asked for, and +the time the answer is due. -/ +structure IwantPromise where + /-- The peer that asked. -/ + peer : Peer + /-- The message id it asked for. -/ + id : MessageId + /-- The time the answer is due. -/ + deadline : Millis + deriving DecidableEq, Repr + +/-- The promise table: one entry per pair, newest first. -/ +abbrev PromiseTable := List IwantPromise + +/-- Record a promise to `peer` for `id`, due at `deadline`. A second +promise for the same pair replaces the first, so the table holds one +deadline per pair. -/ +def PromiseTable.promise (ps : PromiseTable) (peer : Peer) (id : MessageId) + (deadline : Millis) : PromiseTable := + { peer := peer, id := id, deadline := deadline } :: + ps.filter (fun e => decide (e.peer ≠ peer ∨ e.id ≠ id)) + +/-- The id arrived, so every promise for it is kept. -/ +def PromiseTable.deliver (ps : PromiseTable) (id : MessageId) : PromiseTable := + ps.filter (fun e => decide (e.id ≠ id)) + +/-- The number of promises of one peer whose deadline is at or before +`now`. -/ +private def countFor (ps : PromiseTable) (peer : Peer) (now : Millis) : Nat := + (ps.filter (fun e => decide (e.peer = peer ∧ e.deadline ≤ now))).length + +/-- The peers of the promises whose deadline is at or before `now`, in +table order, with duplicates. -/ +private def expiredPeers (ps : PromiseTable) (now : Millis) : List Peer := + (ps.filter (fun e => decide (e.deadline ≤ now))).map (fun e => e.peer) + +/-- Boolean membership on a peer list, decided by `DecidableEq`. Core +`List` has no decidable membership instance, so the deduplication below +carries its own. -/ +private def memPeer (p : Peer) (l : List Peer) : Bool := + l.any (fun q => decide (q = p)) + +private theorem memPeer_eq_true {p : Peer} {l : List Peer} : + memPeer p l = true ↔ p ∈ l := by + rw [memPeer, List.any_eq_true] + constructor + · rintro ⟨q, hq, hqp⟩ + rw [← of_decide_eq_true hqp] + exact hq + · intro hp + exact ⟨p, hp, by simp⟩ + +/-- Deduplicate a peer list, keeping the first occurrence of each peer. -/ +private def dedupGo (seen : List Peer) : List Peer → List Peer + | [] => [] + | p :: ps => if memPeer p seen then dedupGo seen ps else p :: dedupGo (p :: seen) ps + +/-- Deduplicate a peer list. -/ +private def dedupPeers (ps : List Peer) : List Peer := dedupGo [] ps + +private theorem mem_dedupGo {p : Peer} {seen ps : List Peer} : + p ∈ dedupGo seen ps ↔ p ∈ ps ∧ p ∉ seen := by + induction ps generalizing seen with + | nil => simp [dedupGo] + | cons q qs ih => + simp only [dedupGo] + split + · rename_i hq + have hqmem : q ∈ seen := memPeer_eq_true.mp hq + rw [ih, List.mem_cons] + constructor + · rintro ⟨h1, h2⟩; exact ⟨Or.inr h1, h2⟩ + · rintro ⟨h1 | h1, h2⟩ + · subst h1; exact absurd hqmem h2 + · exact ⟨h1, h2⟩ + · rename_i hq + have hqmem : q ∉ seen := fun hc => hq (memPeer_eq_true.mpr hc) + rw [List.mem_cons, ih] + simp only [List.mem_cons, not_or] + constructor + · rintro (h | ⟨h1, hne, h2⟩) + · exact ⟨Or.inl h, fun hs => hqmem (h ▸ hs)⟩ + · exact ⟨Or.inr h1, h2⟩ + · rintro ⟨h1 | h1, h2⟩ + · exact Or.inl h1 + · by_cases hpq : p = q + · exact Or.inl hpq + · exact Or.inr ⟨h1, hpq, h2⟩ + +private theorem mem_dedupPeers {p : Peer} {ps : List Peer} : + p ∈ dedupPeers ps ↔ p ∈ ps := by + rw [dedupPeers, mem_dedupGo] + simp + +/-- The peers with an expired promise, one pair per peer, with the number +of that peer's expired promises. A peer appears only when it has at least +one expired promise, and it appears once. This is the input of the `P₇` +behavior penalty. -/ +def PromiseTable.broken (ps : PromiseTable) (now : Millis) : List (Peer × Nat) := + (dedupPeers (expiredPeers ps now)).map (fun p => (p, countFor ps p now)) + +/-- A promise records its pair and its deadline. -/ +theorem promise_mem (ps : PromiseTable) (peer : Peer) (id : MessageId) (d : Millis) : + ∃ e ∈ PromiseTable.promise ps peer id d, e.peer = peer ∧ e.id = id ∧ e.deadline = d := + ⟨_, List.mem_cons_self, rfl, rfl, rfl⟩ + +/-- A delivery leaves no promise that names the delivered id. -/ +theorem deliver_removes (ps : PromiseTable) (id : MessageId) : + ∀ e ∈ PromiseTable.deliver ps id, e.id ≠ id := by + intro e he + simp only [PromiseTable.deliver, List.mem_filter] at he + exact of_decide_eq_true he.2 + +/-- A delivery keeps every promise of another id. -/ +theorem deliver_keeps_others (ps : PromiseTable) (id : MessageId) : + ∀ e, e ∈ ps → e.id ≠ id → e ∈ PromiseTable.deliver ps id := by + intro e he hne + rw [PromiseTable.deliver, List.mem_filter] + exact ⟨he, by simpa using hne⟩ + +/-- `broken` names exactly the peers with an expired promise, with the +exact count. -/ +theorem broken_mem_iff (ps : PromiseTable) (now : Millis) (peer : Peer) (n : Nat) : + (peer, n) ∈ PromiseTable.broken ps now ↔ + n = (ps.filter (fun e => decide (e.peer = peer ∧ e.deadline ≤ now))).length ∧ 0 < n := by + constructor + · intro h + rw [PromiseTable.broken, List.mem_map] at h + obtain ⟨p, hp, hpn⟩ := h + rw [Prod.mk.injEq] at hpn + obtain ⟨hpe, hcnt⟩ := hpn + subst hpe + refine ⟨hcnt.symm, ?_⟩ + have hexp : p ∈ expiredPeers ps now := mem_dedupPeers.mp hp + rw [expiredPeers, List.mem_map] at hexp + obtain ⟨e, he, hep⟩ := hexp + rw [List.mem_filter] at he + have hle : e.deadline ≤ now := of_decide_eq_true he.2 + have hemem : e ∈ ps.filter (fun e => decide (e.peer = p ∧ e.deadline ≤ now)) := by + rw [List.mem_filter] + exact ⟨he.1, by simp [hep, hle]⟩ + have hpos : 0 < (ps.filter (fun e => decide (e.peer = p ∧ e.deadline ≤ now))).length := + List.length_pos_of_mem hemem + rw [← hcnt] + simpa [countFor] using hpos + · rintro ⟨hn, hpos⟩ + rw [PromiseTable.broken, List.mem_map] + have hpos' : 0 < (ps.filter (fun e => decide (e.peer = peer ∧ e.deadline ≤ now))).length := by + rw [← hn] + exact hpos + obtain ⟨e, hemem⟩ := List.exists_mem_of_length_pos hpos' + rw [List.mem_filter] at hemem + obtain ⟨he, hcond⟩ := hemem + obtain ⟨hpe, hle⟩ := of_decide_eq_true hcond + have hexp : peer ∈ expiredPeers ps now := by + rw [expiredPeers, List.mem_map] + exact ⟨e, by rw [List.mem_filter]; exact ⟨he, by simp [hle]⟩, hpe⟩ + refine ⟨peer, mem_dedupPeers.mpr hexp, ?_⟩ + rw [Prod.mk.injEq] + exact ⟨rfl, by simpa [countFor] using hn.symm⟩ + +/-- After a delivery every count that `broken` reports excludes the +delivered id: a peer's count is exactly its expired promises for other +ids. -/ +theorem broken_after_deliver_ignores_id (ps : PromiseTable) (id : MessageId) (now : Millis) : + ∀ (peer : Peer) (n : Nat), + (peer, n) ∈ PromiseTable.broken (PromiseTable.deliver ps id) now → + n = (ps.filter (fun e => + decide (e.peer = peer ∧ e.id ≠ id ∧ e.deadline ≤ now))).length := by + intro peer n hn + have h := (broken_mem_iff (PromiseTable.deliver ps id) now peer n).mp hn + obtain ⟨hcount, _⟩ := h + rw [hcount] + simp only [PromiseTable.deliver, List.filter_filter] + congr 1 + apply List.filter_congr + intro a _ + by_cases h1 : a.peer = peer <;> by_cases h2 : a.id = id <;> by_cases h3 : a.deadline ≤ now <;> + simp [h1, h2, h3] + +end Libp2p.Protocols.Gossipsub.Cache diff --git a/packages/libp2p/protocols/Libp2p/Protocols/Gossipsub/Core.lean b/packages/libp2p/protocols/Libp2p/Protocols/Gossipsub/Core.lean new file mode 100644 index 00000000..01f006cc --- /dev/null +++ b/packages/libp2p/protocols/Libp2p/Protocols/Gossipsub/Core.lean @@ -0,0 +1,2056 @@ +import Libp2p.Protocols.Gossipsub.Types +import Libp2p.Protocols.Gossipsub.Rpc +import Libp2p.Protocols.Gossipsub.Score +import Libp2p.Protocols.Gossipsub.Cache +import Libp2p.Protocols.Gossipsub.Sign + +/-! +# Gossipsub router core + +The pure state machine of the gossipsub v1.1 router: the state, the +external events, the outgoing actions, the one error constructor, and +the total `step`. Time and the peer order enter as event data, so the +core reads no clock and makes no random choice. `run` drives a list of +events. + +The core owns the mesh, the fanout, the score accounting, and the +heartbeat. It calls the public surface of the score, the cache, and the +sign parts. It decodes an inbound RPC at its own boundary and maps a +decode failure to the one error constructor. + +An inbound message that passes validation and is not already seen is +delivered to the application and forwarded to every other peer of +`mesh[topic]`. The sender already holds it, so the forward list drops +the sender. A local publish records the message id in the seen cache, +so a later echo of the same id is not forwarded again. + +The heartbeat emits gossip after it settles the mesh and before it +shifts the message cache. The recipients are peers of the supplied order +at or above the gossip threshold that are in neither the mesh nor the +fanout of the topic, up to `dLazy` per topic. `gossipFactor` is stored +and never read, so the emission count is not adaptive. + +Several fields of the spec are stored and never read, and several +behaviors are absent: + +- `PubsubProto` is stored and no path reads it, so floodsub peers are + not a separate forward set. +- `ControlPrune.peers` decodes and `handlePrune` ignores it, so peer + exchange is absent. `acceptPXThreshold` is never read. +- `opportunisticGraftThreshold` is never read. +- `fanoutTtl` is never read, so a fanout entry does not expire. +- `handleIhave` records a promise and nothing calls + `PromiseTable.broken`, so an expired promise changes no score. +- No event writes `appScore`, `ipColocation`, or `behaviourPenalty`, so + P5, P6, and P7 have no producer. `PeerData` carries no address. +- `meshMessageDeliveriesWindow` is never read. A mesh peer counts + toward P3 for every accepted message. +- `leave` deletes the peer record, so a score does not survive a + disconnect. +- `planTopic` drops the oldest mesh entries. `dScore` is never read and + there is no outbound flag, so the `D_out` quota is absent. +- `validationOk` uses `strictNoSign`. There is no per-topic policy and + no Ignore outcome. `defaults.msgId` is the payload, because that + policy leaves `from` and `seqno` absent and the spec's default id + would be empty for every accepted message. +- IHAVE and IWANT caps and flood publishing are absent. + `publishThreshold` is stored and never read. +- A graft for a topic the router does not subscribe to is ignored. The + v1.0 spec answers with a prune, and the v1.1 spam measures say to + ignore it. This step follows v1.1. +- `unsubscribe` writes `pruneBackoff`. The spec's shorter unsubscribe + backoff is absent, and `subscribe` does not consult the backoff. + +The spec sections are `vendor/specs/pubsub/gossipsub/gossipsub-v1.0.md` +(the router, the message cache, the mesh, the fanout, the heartbeat) and +`gossipsub-v1.1.md` (the score gates, the backoff, the spam measures). +The root `docs/ARCHITECTURE.md` section 3.3 fixes the shape: one total +`step` that returns `Except ProtocolError (State × List Action)`, with +time as an input. +-/ + +set_option autoImplicit false + +namespace Libp2p.Protocols.Gossipsub.Core + +open Libp2p.Protocols.Gossipsub + +/-! ## Types -/ + +/-- The pubsub protocol a peer negotiated. -/ +inductive PubsubProto where + /-- The gossipsub mesh protocol. -/ + | meshsub + /-- The floodsub protocol. -/ + | floodsub + deriving DecidableEq, BEq, Repr + +/-- The per-peer record: the score state, the pubsub protocol, and the +topics the peer announced. `Peer` keeps its one meaning, the peer id of +`Types.lean`. -/ +structure PeerData where + /-- The score state of the peer. -/ + score : Score.PeerState + /-- The pubsub protocol of the peer. -/ + proto : PubsubProto + /-- The announced topics. -/ + subs : List TopicId + +/-- The mesh degree parameters: `d`, `dLow`, `dHigh`, `dScore`, `dLazy`. -/ +structure MeshParams where + /-- The target mesh degree. -/ + d : Nat + /-- The low mesh water mark. -/ + dLow : Nat + /-- The high mesh water mark. -/ + dHigh : Nat + /-- The number of score-best peers the mesh keeps. -/ + dScore : Nat + /-- The number of peers that receive gossip. -/ + dLazy : Nat + deriving DecidableEq, Repr + +/-- The router parameters. -/ +structure Params where + /-- The score parameters. -/ + score : Score.Params + /-- The score thresholds. -/ + thresholds : Thresholds + /-- The mesh degree parameters. -/ + mesh : MeshParams + /-- The heartbeat interval in milliseconds. -/ + heartbeatInterval : Millis + /-- The fanout time to live in milliseconds. -/ + fanoutTtl : Millis + /-- The prune backoff period in milliseconds. -/ + pruneBackoff : Millis + /-- The gossip factor in percent. -/ + gossipFactor : Nat + /-- The IWANT answer deadline in milliseconds. -/ + iwantDeadline : Millis + /-- The message cache window parameters. -/ + window : Cache.WindowParams + /-- The seen cache parameters. -/ + seen : Cache.SeenParams + /-- The signature primitives. -/ + prims : Sign.Prims + /-- The message id function. -/ + msgId : Sign.MsgIdFn + +/-- The parameter bound the mesh theorems consume: the target degree is +at most the high water mark. -/ +structure Params.Valid (p : Params) : Prop where + /-- The target degree is at most the high water mark. -/ + d_le_dHigh : p.mesh.d ≤ p.mesh.dHigh + +/-- One fanout record: the peers and the last publish time. -/ +structure Fanout where + /-- The fanout peers. -/ + peers : List Peer + /-- The last publish time. -/ + lastPublished : Millis + deriving Repr + +/-- The router state. -/ +structure State where + /-- The known peers and their records. -/ + peers : List (Peer × PeerData) + /-- The mesh of each topic. -/ + mesh : List (TopicId × List Peer) + /-- The fanout of each topic. -/ + fanout : List (TopicId × Fanout) + /-- The message cache. -/ + mcache : Cache.MessageCache + /-- The seen cache. -/ + seen : Cache.SeenCache + /-- The backoff table. -/ + backoff : Cache.BackoffTable + /-- The IWANT promises. -/ + iwant : Cache.PromiseTable + /-- The topics the router subscribes to. -/ + topics : List TopicId + /-- The current time. -/ + now : Millis + /-- The time of the last heartbeat. -/ + lastHeartbeat : Millis + +/-- The external events. -/ +inductive Event where + /-- A peer joined with a protocol. -/ + | peerJoined (peer : Peer) (proto : PubsubProto) + /-- A peer left. -/ + | peerLeft (peer : Peer) + /-- An RPC arrived from a peer, as bytes. -/ + | rpc (sender : Peer) (bytes : List Nat) + /-- The application publishes a message. -/ + | publish (msg : Message) + /-- The application subscribes to a topic. -/ + | subscribe (topic : TopicId) + /-- The application unsubscribes from a topic. -/ + | unsubscribe (topic : TopicId) + /-- The heartbeat fires at `now` with a peer order. -/ + | tick (now : Millis) (order : List Peer) + deriving Repr + +/-- The outgoing actions. -/ +inductive Action where + /-- Send an RPC to a peer. -/ + | sendRpc (to : Peer) (rpc : Rpc.RPC) + /-- Deliver a message to the application. -/ + | deliver (msg : Message) + deriving DecidableEq, BEq, Repr + +/-- The one error constructor: an RPC that does not decode. -/ +inductive ProtocolError where + /-- The RPC bytes do not decode. -/ + | decode (e : Pb.Error) + deriving DecidableEq, BEq, Repr + +/-! ## Boolean membership -/ + +/-- Boolean membership, decided by `DecidableEq`. The core uses it in +the guards of its definitions, because a `Prop` membership has no +decidable instance in the core library. -/ +def memB {α : Type} [DecidableEq α] (a : α) (l : List α) : Bool := + l.any (fun b => decide (b = a)) + +/-- The Boolean membership holds exactly for the `Prop` membership. -/ +theorem memB_eq_true {α : Type} [DecidableEq α] {a : α} {l : List α} : + memB a l = true ↔ a ∈ l := by + rw [memB, List.any_eq_true] + constructor + · rintro ⟨b, hb, hab⟩ + rw [← of_decide_eq_true hab] + exact hb + · intro h + exact ⟨a, h, by simp⟩ + +/-! ## Association list accessors -/ + +/-- The first list of the topic. -/ +def lookupTopic : List (TopicId × List Peer) → TopicId → List Peer + | [], _ => [] + | (t', ps) :: l, t => if t = t' then ps else lookupTopic l t + +/-- Replace the first entry of the topic, or append the topic. -/ +def upsertMesh : List (TopicId × List Peer) → TopicId → List Peer → + List (TopicId × List Peer) + | [], t, ps => [(t, ps)] + | (t', ps') :: l, t, ps => if t = t' then (t, ps) :: l else (t', ps') :: upsertMesh l t ps + +/-- The mesh of the topic. -/ +def meshOf (s : State) (topic : TopicId) : List Peer := lookupTopic s.mesh topic + +/-- Write the mesh of the topic. -/ +def setTopicMesh (s : State) (topic : TopicId) (ps : List Peer) : State := + { s with mesh := upsertMesh s.mesh topic ps } + +/-- The first fanout record of the topic. -/ +def lookupFanout : List (TopicId × Fanout) → TopicId → Option Fanout + | [], _ => none + | (t', f) :: l, t => if t = t' then some f else lookupFanout l t + +/-- The fanout peers of the topic. -/ +def fanoutOf (s : State) (topic : TopicId) : List Peer := + match lookupFanout s.fanout topic with + | some f => f.peers + | none => [] + +/-- Write the fanout record of the topic. -/ +def setTopicFanout (s : State) (topic : TopicId) (f : Fanout) : State := + { s with fanout := (topic, f) :: s.fanout.filter (fun e => decide (e.1 ≠ topic)) } + +/-- The first record of the peer. -/ +def lookupPeer : List (Peer × PeerData) → Peer → Option PeerData + | [], _ => none + | (q', pd) :: l, q => if q = q' then some pd else lookupPeer l q + +/-- Replace the first record of the peer, or append the peer. -/ +def upsertPeer : List (Peer × PeerData) → Peer → PeerData → List (Peer × PeerData) + | [], q, pd => [(q, pd)] + | (q', pd') :: l, q, pd => if q = q' then (q, pd) :: l else (q', pd') :: upsertPeer l q pd + +/-- The record of the peer. -/ +def peerOf (s : State) (q : Peer) : Option PeerData := lookupPeer s.peers q + +/-- Write the record of the peer. -/ +def setPeer (s : State) (q : Peer) (pd : PeerData) : State := + { s with peers := upsertPeer s.peers q pd } + +/-- Update every peer record through `f`. The list order stays. -/ +def mapPeers (s : State) (f : Peer → PeerData → PeerData) : State := + { s with peers := s.peers.map (fun e => (e.1, f e.1 e.2)) } + +/-- Update the score state of one peer and leave every other field. -/ +def updateScoreOf (s : State) (q : Peer) (f : Score.PeerState → Score.PeerState) : State := + mapPeers s (fun r pd => if r = q then { pd with score := f pd.score } else pd) + +/-- The score of a peer at the state time. A missing peer scores 0. -/ +def scoreOf (p : Params) (s : State) (q : Peer) : Score := + match peerOf s q with + | some pd => Score.score p.score pd.score s.now + | none => 0 + +/-- The next heartbeat is due: the interval has passed since the last one. -/ +def heartbeatDue (p : Params) (s : State) (now : Millis) : Prop := + s.lastHeartbeat + p.heartbeatInterval ≤ now + +/-- The heartbeat condition is decidable. -/ +instance heartbeatDueDecidable (p : Params) (s : State) (now : Millis) : + Decidable (heartbeatDue p s now) := + decidable_of_iff (s.lastHeartbeat + p.heartbeatInterval ≤ now) Iff.rfl + +/-- The RPC grafts the topic. -/ +def rpcGrafts (r : Rpc.RPC) (t : TopicId) : Prop := + match r.control with + | some c => ∃ g ∈ c.graft, g.topicId = t.name + | none => False + +/-- The RPC prunes the topic. -/ +def rpcPrunes (r : Rpc.RPC) (t : TopicId) : Prop := + match r.control with + | some c => ∃ p ∈ c.prune, p.topicId = t.name + | none => False + +/-- The RPC carries nonempty ihave gossip. -/ +def rpcHasIhave (r : Rpc.RPC) : Prop := + match r.control with + | some c => c.ihave ≠ [] + | none => False + +/-- The messages the action list delivers to the application. -/ +def delivers : List Action → List Message + | [] => [] + | .deliver m :: as => m :: delivers as + | .sendRpc _ _ :: as => delivers as + +/-! ## Outgoing RPC constructors -/ + +/-- An RPC that carries one control message. -/ +def controlRpc (c : Rpc.ControlMessage) : Rpc.RPC := + { subscriptions := [], publish := [], control := some c } + +/-- The hello RPC: one subscription record per topic. -/ +def helloRpc (topics : List TopicId) : Rpc.RPC := + { subscriptions := topics.map (fun t => ({ subscribe := true, topicid := t.name } : Rpc.SubOpts)), + publish := [], control := none } + +/-- A graft of the topic. -/ +def graftRpc (t : TopicId) : Rpc.RPC := + controlRpc { ihave := [], iwant := [], graft := [{ topicId := t.name }], prune := [] } + +/-- A prune of the topic with the backoff period in seconds. -/ +def pruneRpc (t : TopicId) (backoff : Millis) : Rpc.RPC := + controlRpc + { ihave := [], iwant := [], graft := [], + prune := [{ topicId := t.name, peers := [], backoff := some (backoff / 1000) }] } + +/-- An IHAVE of the topic with the message ids. -/ +def ihaveRpc (t : TopicId) (ids : List MessageId) : Rpc.RPC := + controlRpc + { ihave := [{ topicId := t.name, messageIds := ids }], iwant := [], graft := [], prune := [] } + +/-- An IWANT with the message ids. -/ +def iwantRpc (ids : List MessageId) : Rpc.RPC := + controlRpc { ihave := [], iwant := [{ messageIds := ids }], graft := [], prune := [] } + +/-- An RPC that forwards one published message. -/ +def publishRpc (m : Message) : Rpc.RPC := + { subscriptions := [], publish := [m], control := none } + +/-- The mesh peers that receive a forwarded inbound message. The sender +already holds the message, so the list drops the sender. -/ +def inboundTargets (s : State) (sender : Peer) (topic : TopicId) : List Peer := + (meshOf s topic).filter (fun q => decide (q ≠ sender)) + +/-- One publish RPC per accepted message, to each mesh peer other than +the sender. The targets are read from the pre-state mesh. -/ +def forwardPubs (s : State) (sender : Peer) (ms : List Message) : List Action := + ms.flatMap fun m => + (inboundTargets s sender m.topic).map fun q => Action.sendRpc q (publishRpc m) + +/-! ## The decode boundary -/ + +/-- The one decode entry point of the core: wire bytes to an RPC value. +The protobuf decoder returns the empty suffix, so the core drops it. -/ +def decodeRpc (bytes : List Nat) : Except ProtocolError Rpc.RPC := + match Pb.decodeMessage? (α := Rpc.RPC) bytes with + | .ok (r, _) => .ok r + | .error e => .error (.decode e) + +/-- A decode that the model decoder reports as successful decodes to the +same value. -/ +theorem decodeRpc_of_decodeMessage (bytes : List Nat) (r : Rpc.RPC) + (h : Pb.decodeMessage? (α := Rpc.RPC) bytes = .ok (r, [])) : + decodeRpc bytes = .ok r := by + simp [decodeRpc, h] + +/-- A `strictNoSign` validation as a `Bool`. -/ +def validationOk (p : Params) (m : Message) : Bool := + match Sign.validate p.prims (fun x => Pb.encodeMessage x) .strictNoSign m with + | .ok _ => true + | .error _ => false + +/-! ## Message acceptance -/ + +/-- One run of the acceptance loop: the accepted messages, the ids the +loop saw, and the seen cache after the run. -/ +structure Accept where + /-- The accepted messages, in arrival order. -/ + accepted : List Message + /-- The ids the loop passed. -/ + done : List MessageId + /-- The seen cache after the run. -/ + cache : Cache.SeenCache + +/-- Accept a list of messages. The gate reads the pre-state cache `c0`, +so a message that the pre-state cache holds as seen is never accepted. +The local `done` list rejects a repeated id inside the one event. -/ +def accept (c0 : Cache.SeenCache) (now : Millis) (idf : Sign.MsgIdFn) + (done : List MessageId) (c : Cache.SeenCache) : List Message → Accept + | [] => { accepted := [], done := done, cache := c } + | m :: ms => + if c0.contains (idf m) now || memB (idf m) done then + accept c0 now idf done c ms + else + let r := accept c0 now idf (idf m :: done) (c.add (idf m) now) ms + { accepted := m :: r.accepted, done := r.done, cache := r.cache } + +/-! ## The heartbeat plan -/ + +/-- One per-topic heartbeat plan: the kept mesh, the pruned peers, and +the added peers. -/ +structure TopicPlan where + /-- The topic. -/ + topic : TopicId + /-- The mesh after the heartbeat. -/ + kept : List Peer + /-- The peers that leave the mesh. -/ + pruned : List Peer + /-- The peers that enter the mesh. -/ + added : List Peer + deriving Repr + +/-- The peers that announce the topic. -/ +def announced (s : State) (topic : TopicId) : List Peer := + (s.peers.filter (fun e => memB topic e.2.subs)).map Prod.fst + +/-- One topic plan. An overfull mesh drops its oldest entries down to +`d`, an underfull mesh takes candidates from the order up to `d`, and a +candidate is outside the mesh, outside backoff, known, and at or above +score 0. The kept list is cut at `dHigh`, so the mesh bound holds by +construction. -/ +def planTopic (p : Params) (s : State) (now : Millis) (order : List Peer) + (topic : TopicId) : TopicPlan := + let ps := meshOf s topic + let excess := if ps.length > p.mesh.dHigh then ps.length - p.mesh.d else 0 + let over := ps.take excess + let kept0 := ps.drop excess + let need := if kept0.length < p.mesh.dLow then p.mesh.d - kept0.length else 0 + let cands := order.filter (fun q => + !memB q ps && !(Cache.Backoff.active s.backoff q topic now) && + (peerOf s q).isSome && decide (0 ≤ scoreOf p s q)) + let added := cands.take need + { topic := topic, kept := (kept0 ++ added).take p.mesh.dHigh, pruned := over, added := added } + +/-- The plan of every topic in the mesh. -/ +def planList (p : Params) (s : State) (now : Millis) (order : List Peer) : List TopicPlan := + s.mesh.map (fun e => planTopic p s now order e.1) + +/-- Apply the score updates of the plans: a pruned peer takes the prune +update, an added peer takes the graft update. -/ +def applyPlanScores (p : Params) (s : State) (plans : List TopicPlan) : State := + mapPeers s (fun q pd => + { pd with + score := plans.foldl (fun sc pl => + if memB q pl.pruned then Score.prune p.score sc s.now pl.topic + else if memB q pl.added then Score.graft sc s.now pl.topic + else sc) pd.score }) + +/-- The actions of the plans: a prune to each pruned peer, then a graft +to each added peer. -/ +def planActions (p : Params) (pl : TopicPlan) : List Action := + pl.pruned.map (fun q => Action.sendRpc q (pruneRpc pl.topic p.pruneBackoff)) ++ + pl.added.map (fun q => Action.sendRpc q (graftRpc pl.topic)) + +/-- Write the planned meshes and drop the peers below score 0 from every +mesh. A dropped peer takes the prune score update. -/ +def settleMesh (p : Params) (s : State) (plans : List TopicPlan) : State := + { s with + mesh := plans.map (fun pl => + (pl.topic, pl.kept.filter (fun q => decide (0 ≤ scoreOf p s q)))), + peers := s.peers.map (fun e => + (e.1, if scoreOf p s e.1 < 0 then + { e.2 with + score := s.topics.foldl (fun sc t => Score.prune p.score sc s.now t) e.2.score } + else e.2)) } + +/-- The gossip actions: each topic of the mesh and the fanout sends its +gossip ids to the peers of the order at or above the gossip threshold, +up to `dLazy` targets, skipping the peers that already sit in that +topic's mesh or fanout. The ids come from the cache as it stands, so a +heartbeat emits gossip before it shifts the cache. -/ +def gossipActions (p : Params) (s : State) (order : List Peer) : List Action := + let eligible := + order.filter (fun q => decide (p.thresholds.gossipThreshold ≤ scoreOf p s q)) + let topics := s.mesh.map Prod.fst ++ s.fanout.map Prod.fst + topics.flatMap fun t => + let ids := Cache.MessageCache.gossipIds s.mcache t + if ids = [] then [] + else + let targets := + (eligible.filter (fun q => + !(memB q (meshOf s t)) && !(memB q (fanoutOf s t)))).take p.mesh.dLazy + targets.map fun q => Action.sendRpc q (ihaveRpc t ids) + +/-- The state at the start of the heartbeat: the decayed scores, the new +clock, the cleared backoff table, the pruned seen cache, and the shifted +message cache. -/ +def freshState (p : Params) (s : State) (now : Millis) : State := + { s with + peers := s.peers.map (fun e => (e.1, { e.2 with score := Score.decay p.score e.2.score })), + now := now, + lastHeartbeat := now, + backoff := Cache.Backoff.clear s.backoff now, + seen := Cache.SeenCache.prune s.seen now, + mcache := Cache.MessageCache.shift s.mcache } + +/-- The per-topic plans of the heartbeat. -/ +def heartbeatPlans (p : Params) (s : State) (now : Millis) (order : List Peer) : + List TopicPlan := + planList p (freshState p s now) now order + +/-- The state after the plan score updates. -/ +def planScored (p : Params) (s : State) (now : Millis) (order : List Peer) : State := + applyPlanScores p (freshState p s now) (heartbeatPlans p s now order) + +/-- The state after the heartbeat. -/ +def heartbeatNext (p : Params) (s : State) (now : Millis) (order : List Peer) : State := + settleMesh p (planScored p s now order) (heartbeatPlans p s now order) + +/-- The plan actions of the heartbeat. -/ +def planActionsOf (p : Params) (s : State) (now : Millis) (order : List Peer) : + List Action := + (heartbeatPlans p s now order).flatMap (planActions p) + +/-- The negative prune actions of the heartbeat. -/ +def negativeActions (p : Params) (s : State) (now : Millis) (order : List Peer) : + List Action := + (heartbeatPlans p s now order).flatMap (fun pl => + (pl.kept.filter (fun q => decide (scoreOf p (planScored p s now order) q < 0))).map + (fun q => Action.sendRpc q (pruneRpc pl.topic p.pruneBackoff))) + +/-- The state the gossip reads: the mesh and the scores after the +heartbeat, and the message cache before the shift. The spec emits the +ids, then shifts the cache. -/ +def gossipState (p : Params) (s : State) (now : Millis) (order : List Peer) : State := + { heartbeatNext p s now order with mcache := s.mcache } + +/-- The actions of the heartbeat. Gossip is emitted after the mesh is +settled and before the cache shifts. -/ +def heartbeatActions (p : Params) (s : State) (now : Millis) (order : List Peer) : + List Action := + planActionsOf p s now order ++ negativeActions p s now order ++ + gossipActions p (gossipState p s now order) order + +/-- The heartbeat: decay, the fresh clock and tables, the per-topic plan, +the score updates, the mesh write with the negative prune, and the +actions of the plan, the negative prune, and the gossip. -/ +def heartbeat (p : Params) (s : State) (now : Millis) (order : List Peer) : + State × List Action := + (heartbeatNext p s now order, heartbeatActions p s now order) + +/-! ## The event handlers -/ + +/-- A peer joins: record it, and send the hello RPC with our topics. A +known peer keeps its record. -/ +def join (_p : Params) (s : State) (peer : Peer) (proto : PubsubProto) : + State × List Action := + let s1 := + if (peerOf s peer).isSome then s + else setPeer s peer { score := Score.PeerState.zero, proto := proto, subs := [] } + (s1, [Action.sendRpc peer (helloRpc s.topics)]) + +/-- A peer leaves: drop it from the peers, the meshes, the fanout +records, and the promise table. -/ +def leave (s : State) (peer : Peer) : State × List Action := + ({ s with + peers := s.peers.filter (fun e => decide (e.1 ≠ peer)), + mesh := s.mesh.map (fun e => (e.1, e.2.filter (fun q => decide (q ≠ peer)))), + fanout := s.fanout.map (fun e => + (e.1, { e.2 with peers := e.2.peers.filter (fun q => decide (q ≠ peer)) })), + iwant := s.iwant.filter (fun e => decide (e.peer ≠ peer)) }, + []) + +/-- One subscription option against the announced list. -/ +def applySubOpts (subs : List TopicId) (o : Rpc.SubOpts) : List TopicId := + let t : TopicId := ⟨o.topicid⟩ + if o.subscribe then (if memB t subs then subs else t :: subs) + else subs.filter (fun u => decide (u ≠ t)) + +/-- Update the announced topics of the sender. -/ +def handleSubs (s : State) (sender : Peer) (subs : List Rpc.SubOpts) : State := + match peerOf s sender with + | some pd => setPeer s sender { pd with subs := subs.foldl applySubOpts pd.subs } + | none => s + +/-- Handle the published messages of an RPC: validate, accept, cache, +score, deliver, and forward. -/ +def handlePubs (p : Params) (s : State) (sender : Peer) (ms : List Message) : + State × List Action := + let valid := ms.filter (fun m => validationOk p m) + let invalid := ms.filter (fun m => !(validationOk p m)) + let r := accept s.seen s.now p.msgId [] s.seen valid + let s1 : State := + { s with + seen := r.cache, + mcache := r.accepted.foldl (fun c m => c.put (p.msgId m) m) s.mcache } + let s2 := invalid.foldl (fun st m => + updateScoreOf st sender (fun sc => Score.invalidDelivery p.score sc m.topic)) s1 + let s3 := r.accepted.foldl (fun st m => + updateScoreOf st sender (fun sc => + let sc' := Score.firstDelivery p.score sc m.topic + if memB sender (meshOf s m.topic) then Score.meshDelivery p.score sc' m.topic else sc')) s2 + (s3, r.accepted.map (fun m => Action.deliver m) ++ + forwardPubs s sender r.accepted) + +/-- Whether the promise table holds the pair. -/ +def promised (ps : Cache.PromiseTable) (q : Peer) (id : MessageId) : Bool := + ps.any (fun e => decide (e.peer = q ∧ e.id = id)) + +/-- Handle one IHAVE: request the ids that are unseen, uncached, and not +promised, and record the promise. -/ +def handleIhave (p : Params) (s : State) (sender : Peer) (i : Rpc.ControlIHave) : + State × List Action := + let wanted := i.messageIds.filter (fun id => + !(s.seen.contains id s.now) && (Cache.MessageCache.get s.mcache id).isNone && + !(promised s.iwant sender id)) + let s1 : State := + { s with + iwant := wanted.foldl + (fun ps id => ps.promise sender id (s.now + p.iwantDeadline)) s.iwant } + (s1, if wanted = [] then [] else [Action.sendRpc sender (iwantRpc wanted)]) + +/-- Handle one IWANT: answer each id that the message cache holds under +the same id. -/ +def handleIwant (p : Params) (s : State) (sender : Peer) (w : Rpc.ControlIWant) : + State × List Action := + let answers := w.messageIds.filterMap (fun id => + match Cache.MessageCache.get s.mcache id with + | some m => if p.msgId m = id then some m else none + | none => none) + (s, answers.map (fun m => Action.sendRpc sender (publishRpc m))) + +/-- Handle one GRAFT. An unknown topic is ignored. A member is a no-op. +A peer in backoff, or a mesh at `dHigh`, gets a prune. Otherwise the +peer enters the mesh and takes the graft score update. -/ +def handleGraft (p : Params) (s : State) (sender : Peer) (g : Rpc.ControlGraft) : + State × List Action := + if memB (⟨g.topicId⟩ : TopicId) s.topics then + if memB sender (meshOf s ⟨g.topicId⟩) then (s, []) + else if Cache.Backoff.active s.backoff sender ⟨g.topicId⟩ s.now = true then + (s, [Action.sendRpc sender (pruneRpc ⟨g.topicId⟩ p.pruneBackoff)]) + else if p.mesh.dHigh ≤ (meshOf s ⟨g.topicId⟩).length then + (s, [Action.sendRpc sender (pruneRpc ⟨g.topicId⟩ p.pruneBackoff)]) + else + match peerOf s sender with + | some pd => + (setPeer (setTopicMesh s ⟨g.topicId⟩ (sender :: meshOf s ⟨g.topicId⟩)) sender + { pd with score := Score.graft pd.score s.now ⟨g.topicId⟩ }, []) + | none => (s, []) + else (s, []) + +/-- The state after a prune records the backoff pair. -/ +def pruneState (p : Params) (s : State) (sender : Peer) (q : Rpc.ControlPrune) : State := + { s with + backoff := Cache.Backoff.add s.backoff sender ⟨q.topicId⟩ + (match q.backoff with + | some sec => sec * 1000 + | none => p.pruneBackoff) s.now } + +/-- Handle one PRUNE: record the backoff pair, and remove the sender +from the topic mesh with the prune score update. -/ +def handlePrune (p : Params) (s : State) (sender : Peer) (q : Rpc.ControlPrune) : + State × List Action := + if memB sender (meshOf (pruneState p s sender q) ⟨q.topicId⟩) then + match peerOf (pruneState p s sender q) sender with + | some pd => + (setPeer (setTopicMesh (pruneState p s sender q) ⟨q.topicId⟩ + ((meshOf (pruneState p s sender q) ⟨q.topicId⟩).filter + (fun r => decide (r ≠ sender)))) sender + { pd with + score := Score.prune p.score pd.score (pruneState p s sender q).now ⟨q.topicId⟩ }, []) + | none => (pruneState p s sender q, []) + else (pruneState p s sender q, []) + +/-- Run one handler over a list, threading the state and appending the +actions. -/ +def mapHandlers {β : Type} (f : State → β → State × List Action) : + List β → State → State × List Action + | [], s => (s, []) + | b :: bs, s => + let r := f s b + let r' := mapHandlers f bs r.1 + (r'.1, r.2 ++ r'.2) + +/-- The state after the ihave handlers. -/ +def controlIhaves (p : Params) (s : State) (sender : Peer) (c : Rpc.ControlMessage) : + State := + (mapHandlers (fun st b => handleIhave p st sender b) c.ihave s).1 + +/-- The state after the iwant handlers. -/ +def controlIwants (p : Params) (s : State) (sender : Peer) (c : Rpc.ControlMessage) : + State := + (mapHandlers (fun st b => handleIwant p st sender b) c.iwant (controlIhaves p s sender c)).1 + +/-- The state after the graft handlers. -/ +def controlGrafts (p : Params) (s : State) (sender : Peer) (c : Rpc.ControlMessage) : + State := + (mapHandlers (fun st b => handleGraft p st sender b) c.graft (controlIwants p s sender c)).1 + +/-- The state after the prune handlers. -/ +def controlPrunes (p : Params) (s : State) (sender : Peer) (c : Rpc.ControlMessage) : + State := + (mapHandlers (fun st b => handlePrune p st sender b) c.prune (controlGrafts p s sender c)).1 + +/-- The actions of the control handlers: ihave, iwant, graft, then prune. -/ +def controlActions (p : Params) (s : State) (sender : Peer) (c : Rpc.ControlMessage) : + List Action := + (mapHandlers (fun st b => handleIhave p st sender b) c.ihave s).2 ++ + (mapHandlers (fun st b => handleIwant p st sender b) c.iwant + (controlIhaves p s sender c)).2 ++ + (mapHandlers (fun st b => handleGraft p st sender b) c.graft + (controlIwants p s sender c)).2 ++ + (mapHandlers (fun st b => handlePrune p st sender b) c.prune + (controlGrafts p s sender c)).2 + +/-- Handle the control envelope: ihave, iwant, graft, then prune. -/ +def handleControl (p : Params) (s : State) (sender : Peer) : + Option Rpc.ControlMessage → State × List Action + | none => (s, []) + | some c => (controlPrunes p s sender c, controlActions p s sender c) + +/-- Handle a decoded RPC: subscriptions, published messages, control. -/ +def handleRpc (p : Params) (s : State) (sender : Peer) (rpc : Rpc.RPC) : + State × List Action := + let s1 := handleSubs s sender rpc.subscriptions + let (s2, a2) := handlePubs p s1 sender rpc.publish + let (s3, a3) := handleControl p s2 sender rpc.control + (s3, a2 ++ a3) + +/-- The RPC branch gates: a graylisted peer is ignored, an unknown peer +is ignored, and a decode failure is the one error. -/ +def rpcStep (p : Params) (s : State) (sender : Peer) (bytes : List Nat) : + Except ProtocolError (State × List Action) := + if scoreOf p s sender < p.thresholds.graylistThreshold then .ok (s, []) + else + match peerOf s sender with + | none => .ok (s, []) + | some _ => + match decodeRpc bytes with + | .error e => .error e + | .ok rpc => .ok (handleRpc p s sender rpc) + +/-- The fanout candidates: peers that announce the topic, at or above +score 0. -/ +def fanoutCandidates (p : Params) (s : State) (topic : TopicId) : List Peer := + (announced s topic).filter (fun q => decide (0 ≤ scoreOf p s q)) + +/-- Extend the fanout of the topic to `d`, and set its last publish +time. -/ +def extendFanout (p : Params) (s : State) (topic : TopicId) : State := + match lookupFanout s.fanout topic with + | some f => + let need := if f.peers.length < p.mesh.d then p.mesh.d - f.peers.length else 0 + let cands := (fanoutCandidates p s topic).filter (fun q => !memB q f.peers) + setTopicFanout s topic { peers := f.peers ++ cands.take need, lastPublished := s.now } + | none => + setTopicFanout s topic + { peers := (fanoutCandidates p s topic).take p.mesh.d, lastPublished := s.now } + +/-- The forward targets of a publish: the mesh when we subscribe to the +topic, the fanout otherwise. -/ +def publishTargets (_p : Params) (s : State) (topic : TopicId) : List Peer := + if memB topic s.topics then meshOf s topic else fanoutOf s topic + +/-- The application publishes a message: validate, cache, record the id +as seen, extend the fanout when we do not subscribe, and forward to the +targets. The application already holds its own message, so this path +emits no deliver action. The seen record stops a later echo of the same +id from being forwarded again. -/ +def publishLocal (p : Params) (s : State) (m : Message) : State × List Action := + if validationOk p m then + let s1 : State := + { s with + mcache := s.mcache.put (p.msgId m) m + seen := s.seen.add (p.msgId m) s.now } + let s2 := if memB m.topic s1.topics then s1 else extendFanout p s1 m.topic + (s2, (publishTargets p s2 m.topic).map (fun q => Action.sendRpc q (publishRpc m))) + else (s, []) + +/-- The application subscribes: add the topic, move the fanout peers +into the mesh, fill to `d`, and graft the added peers. -/ +def subscribe (p : Params) (s : State) (topic : TopicId) : State × List Action := + let s1 : State := + if memB topic s.topics then s else { s with topics := topic :: s.topics } + let ps := meshOf s1 topic + let ps1 := ps ++ (fanoutOf s1 topic).filter (fun q => !memB q ps) + let need := if ps1.length < p.mesh.d then p.mesh.d - ps1.length else 0 + let cands := (announced s1 topic).filter (fun q => + !memB q ps1 && decide (0 ≤ scoreOf p s1 q) && + !(Cache.Backoff.active s1.backoff q topic s1.now)) + let added := cands.take need + let s2 := setTopicMesh s1 topic ((ps1 ++ added).take p.mesh.dHigh) + let s3 : State := + { s2 with fanout := s2.fanout.filter (fun e => decide (e.1 ≠ topic)) } + (s3, added.map (fun q => Action.sendRpc q (graftRpc topic))) + +/-- The application unsubscribes: drop the topic, empty its mesh and +fanout, prune the old mesh peers, and record the backoff pairs. -/ +def unsubscribe (p : Params) (s : State) (topic : TopicId) : State × List Action := + let ps := meshOf s topic + let s1 : State := { s with topics := s.topics.filter (fun t => decide (t ≠ topic)) } + let s2 := setTopicMesh s1 topic [] + let s3 : State := + { s2 with fanout := s2.fanout.filter (fun e => decide (e.1 ≠ topic)) } + let s4 := ps.foldl + (fun st q => { st with backoff := Cache.Backoff.add st.backoff q topic p.pruneBackoff s.now }) + s3 + (s4, ps.map (fun q => Action.sendRpc q (pruneRpc topic p.pruneBackoff))) + +/-! ## The transition and the driver -/ + +/-- One transition of the router. Every event is handled, and only an +RPC that does not decode returns an error. -/ +def step (p : Params) (s : State) : Event → Except ProtocolError (State × List Action) + | .peerJoined peer proto => .ok (join p s peer proto) + | .peerLeft peer => .ok (leave s peer) + | .rpc sender bytes => rpcStep p s sender bytes + | .publish msg => .ok (publishLocal p s msg) + | .subscribe topic => .ok (subscribe p s topic) + | .unsubscribe topic => .ok (unsubscribe p s topic) + | .tick now order => + if heartbeatDue p s now then .ok (heartbeat p s now order) else .ok (s, []) + +/-- The transcript driver: the events in order. The first error stops +the run. -/ +def run (p : Params) (s : State) : List Event → Except ProtocolError (State × List Action) + | [] => .ok (s, []) + | e :: es => + match step p s e with + | .error err => .error err + | .ok (s1, a1) => + match run p s1 es with + | .error err => .error err + | .ok (s2, a2) => .ok (s2, a1 ++ a2) + +/-! ## The spec defaults -/ + +/-- The spec defaults: d 6, dLow 4, dHigh 12, dScore 4, dLazy 6, +heartbeat 1000 ms, fanout ttl 60000 ms, prune backoff 60000 ms, gossip +factor 25 percent, iwant deadline 3000 ms, window 5 over 3, seen +capacity 65536 and ttl 120000 ms, and zero score weights with no +configured topic. The numbers follow `gossipsub-v1.0.md` and the v1.1 +recommendations. The seen capacity is a chosen memory bound that the +spec leaves open. `validationOk` uses `strictNoSign`, which requires +`from` and `seqno` to be absent, so the spec's default id `from ++ seqno` +would be empty for every accepted message. The default here is the +payload bytes. `Sign.defaultMsgId` stays available for a caller that +pairs it with `strictSign`. -/ +def defaults : Params where + score := + { topics := fun _ => none, + global := + { appSpecificWeight := 0 + ipColocationFactorWeight := 0 + ipColocationFactorThreshold := 1 + behaviourPenaltyWeight := 0 + behaviourPenaltyThreshold := 0 + behaviourPenaltyDecay := scoreScale + decayInterval := 1000 + decayToZero := 0 + topicScoreCap := 0 } } + thresholds := + { gossipThreshold := 0 + publishThreshold := 0 + graylistThreshold := 0 + acceptPXThreshold := 0 + opportunisticGraftThreshold := 0 } + mesh := { d := 6, dLow := 4, dHigh := 12, dScore := 4, dLazy := 6 } + heartbeatInterval := 1000 + fanoutTtl := 60000 + pruneBackoff := 60000 + gossipFactor := 25 + iwantDeadline := 3000 + window := { historyLength := 5, historyGossip := 3, valid := by decide } + seen := { capacity := 65536, ttl := 120000, valid := by decide } + prims := Sign.Prims.identity + msgId := fun m => ⟨m.data.getD []⟩ + +/-! ## Accessor lemmas -/ + +private theorem lookupTopic_upsertMesh_same (l : List (TopicId × List Peer)) (t : TopicId) + (ps : List Peer) : lookupTopic (upsertMesh l t ps) t = ps := by + induction l with + | nil => simp [upsertMesh, lookupTopic] + | cons e l ih => + obtain ⟨t', ps'⟩ := e + by_cases h : t = t' + · simp [upsertMesh, lookupTopic, h] + · simp [upsertMesh, lookupTopic, h, ih] + +private theorem lookupTopic_upsertMesh_ne (t' t : TopicId) (h : t' ≠ t) + (l : List (TopicId × List Peer)) (ps : List Peer) : + lookupTopic (upsertMesh l t ps) t' = lookupTopic l t' := by + induction l with + | nil => simp [upsertMesh, lookupTopic, h] + | cons e l ih => + obtain ⟨u, us⟩ := e + by_cases h2 : t = u + · subst h2 + simp [upsertMesh, lookupTopic, h] + · simp [upsertMesh, lookupTopic, h2, ih] + +/-- The mesh of the written topic. -/ +theorem meshOf_setTopicMesh_same (s : State) (t : TopicId) (ps : List Peer) : + meshOf (setTopicMesh s t ps) t = ps := by + simp [meshOf, setTopicMesh, lookupTopic_upsertMesh_same] + +/-- A write of one topic leaves the mesh of another topic. -/ +theorem meshOf_setTopicMesh_ne (s : State) (t t' : TopicId) (h : t' ≠ t) + (ps : List Peer) : meshOf (setTopicMesh s t ps) t' = meshOf s t' := by + simp [meshOf, setTopicMesh, lookupTopic_upsertMesh_ne t' t h] + +private theorem lookupPeer_upsertPeer_same (l : List (Peer × PeerData)) (q : Peer) + (pd : PeerData) : lookupPeer (upsertPeer l q pd) q = some pd := by + induction l with + | nil => simp [upsertPeer, lookupPeer] + | cons e l ih => + obtain ⟨q', pd'⟩ := e + by_cases h : q = q' + · simp [upsertPeer, lookupPeer, h] + · simp [upsertPeer, lookupPeer, h, ih] + +private theorem lookupPeer_upsertPeer_ne (q' q : Peer) (h : q' ≠ q) + (l : List (Peer × PeerData)) (pd : PeerData) : + lookupPeer (upsertPeer l q pd) q' = lookupPeer l q' := by + induction l with + | nil => simp [upsertPeer, lookupPeer, h] + | cons e l ih => + obtain ⟨u, ud⟩ := e + by_cases h2 : q = u + · subst h2 + simp [upsertPeer, lookupPeer, h] + · simp [upsertPeer, lookupPeer, h2, ih] + +/-- The record of the written peer. -/ +theorem peerOf_setPeer_same (s : State) (q : Peer) (pd : PeerData) : + peerOf (setPeer s q pd) q = some pd := by + simp [peerOf, setPeer, lookupPeer_upsertPeer_same] + +/-- A write of one peer leaves the record of another peer. -/ +theorem peerOf_setPeer_ne (s : State) (q q' : Peer) (h : q' ≠ q) (pd : PeerData) : + peerOf (setPeer s q pd) q' = peerOf s q' := by + simp [peerOf, setPeer, lookupPeer_upsertPeer_ne q' q h] + +private theorem lookupPeer_map (l : List (Peer × PeerData)) + (f : Peer → PeerData → PeerData) (q : Peer) : + lookupPeer (l.map (fun e => (e.1, f e.1 e.2))) q = (lookupPeer l q).map (f q) := by + induction l with + | nil => simp [lookupPeer] + | cons e l ih => + obtain ⟨q', pd'⟩ := e + simp only [List.map_cons, lookupPeer] + by_cases h : q = q' + · subst h + simp + · simp [h, ih] + +/-- The record of a peer after a map over the whole list. -/ +theorem peerOf_mapPeers (s : State) (f : Peer → PeerData → PeerData) (q : Peer) : + peerOf (mapPeers s f) q = (peerOf s q).map (f q) := by + simp [peerOf, mapPeers, lookupPeer_map] + +/-- The score of a peer after a map over the whole peer list. -/ +theorem scoreOf_mapPeers (p : Params) (s : State) (f : Peer → PeerData → PeerData) + (q : Peer) : + scoreOf p (mapPeers s f) q = + match peerOf s q with + | some pd => Score.score p.score (f q pd).score s.now + | none => 0 := by + simp only [scoreOf, peerOf_mapPeers] + cases h : peerOf s q <;> rfl + +/-- The score reads only the peer list and the time. -/ +theorem scoreOf_congr (p : Params) {s s' : State} (hpeers : s'.peers = s.peers) + (hnow : s'.now = s.now) (q : Peer) : scoreOf p s' q = scoreOf p s q := by + simp [scoreOf, peerOf, hpeers, hnow] + +/-! ## Plan lemmas -/ + +/-- The plan of a topic keeps at most `dHigh` peers. -/ +theorem planTopic_kept_le (p : Params) (s : State) (now : Millis) (order : List Peer) + (t : TopicId) : (planTopic p s now order t).kept.length ≤ p.mesh.dHigh := by + simp only [planTopic] + rw [List.length_take] + omega + +/-- The lookup of a mesh rebuilt from a list of plans. -/ +theorem lookupTopic_map_plans (plans : List TopicPlan) (f : TopicPlan → List Peer) + (t : TopicId) : + lookupTopic (plans.map (fun pl => (pl.topic, f pl))) t = + match plans.find? (fun pl => decide (t = pl.topic)) with + | some pl => f pl + | none => [] := by + induction plans with + | nil => simp [lookupTopic] + | cons pl rest ih => + simp only [List.map_cons] + by_cases h : t = pl.topic + · simp [lookupTopic, List.find?, h] + · simp [lookupTopic, List.find?, h, ih] + + +/-- The mesh of a topic after the heartbeat mesh write. -/ +theorem meshOf_settleMesh (p : Params) (s : State) (plans : List TopicPlan) (topic : TopicId) : + meshOf (settleMesh p s plans) topic = + match plans.find? (fun pl => decide (topic = pl.topic)) with + | some pl => pl.kept.filter (fun q => decide (0 ≤ scoreOf p s q)) + | none => [] := by + simp [meshOf, settleMesh, lookupTopic_map_plans] + +/-! ## The mesh bound theorem -/ + +/-- After a heartbeat, every topic mesh holds at most `dHigh` peers. +The kept list is cut at `dHigh` by construction, and the negative prune +only removes peers. -/ +theorem heartbeatMeshBound (p : Params) (s s' : State) (acts : List Action) + (now : Millis) (order : List Peer) + (hdue : heartbeatDue p s now) + (hstep : step p s (.tick now order) = .ok (s', acts)) : + ∀ topic, (meshOf s' topic).length ≤ p.mesh.dHigh := by + intro topic + simp only [step, ite_eq_left hdue, Except.ok.injEq] at hstep + obtain ⟨rfl, rfl⟩ := hstep + rw [heartbeatNext, meshOf_settleMesh] + split + · next pl hfind => + have hmem : pl ∈ heartbeatPlans p s now order := List.mem_of_find?_eq_some hfind + have hmem' : pl ∈ (freshState p s now).mesh.map + (fun e => planTopic p (freshState p s now) now order e.1) := by + simpa [heartbeatPlans, planList] using hmem + simp only [List.mem_map] at hmem' + obtain ⟨e, _, rfl⟩ := hmem' + exact Nat.le_trans (List.length_filter_le _ _) + (planTopic_kept_le p (freshState p s now) now order e.1) + · exact Nat.zero_le _ + + +/-! ## The graylist gate theorem -/ + +/-- No RPC is processed from a peer below the graylist threshold. The +graylist premise is the only hypothesis, so the gate is what the +theorem proves: the step returns the pre-state and no actions, and it +does not consult the bytes. -/ +theorem rpcGraylistIgnored (p : Params) (s : State) (sender : Peer) + (bytes : List Nat) + (hgray : scoreOf p s sender < p.thresholds.graylistThreshold) : + step p s (.rpc sender bytes) = .ok (s, []) := by + simp [step, rpcStep, hgray] + + +/-! ## The graft handler lemmas -/ + +private theorem handleGraft_unknown (p : Params) (s : State) (sender : Peer) + (g : Rpc.ControlGraft) (h : memB (⟨g.topicId⟩ : TopicId) s.topics = false) : + handleGraft p s sender g = (s, []) := by + simp [handleGraft, h] + +private theorem handleGraft_member (p : Params) (s : State) (sender : Peer) + (g : Rpc.ControlGraft) (hsub : memB (⟨g.topicId⟩ : TopicId) s.topics = true) + (hm : memB sender (meshOf s ⟨g.topicId⟩) = true) : + handleGraft p s sender g = (s, []) := by + simp [handleGraft, hsub, hm] + +private theorem handleGraft_backoff (p : Params) (s : State) (sender : Peer) + (g : Rpc.ControlGraft) (hsub : memB (⟨g.topicId⟩ : TopicId) s.topics = true) + (hm : memB sender (meshOf s ⟨g.topicId⟩) = false) + (hb : Cache.Backoff.active s.backoff sender ⟨g.topicId⟩ s.now = true) : + handleGraft p s sender g = + (s, [Action.sendRpc sender (pruneRpc ⟨g.topicId⟩ p.pruneBackoff)]) := by + simp [handleGraft, hsub, hm, hb] + +private theorem handleGraft_overfull (p : Params) (s : State) (sender : Peer) + (g : Rpc.ControlGraft) (hsub : memB (⟨g.topicId⟩ : TopicId) s.topics = true) + (hm : memB sender (meshOf s ⟨g.topicId⟩) = false) + (hb : Cache.Backoff.active s.backoff sender ⟨g.topicId⟩ s.now = false) + (hd : p.mesh.dHigh ≤ (meshOf s ⟨g.topicId⟩).length) : + handleGraft p s sender g = + (s, [Action.sendRpc sender (pruneRpc ⟨g.topicId⟩ p.pruneBackoff)]) := by + simp [handleGraft, hsub, hm, hb, hd] + +private theorem handleGraft_add (p : Params) (s : State) (sender : Peer) + (g : Rpc.ControlGraft) (pd : PeerData) + (hsub : memB (⟨g.topicId⟩ : TopicId) s.topics = true) + (hm : memB sender (meshOf s ⟨g.topicId⟩) = false) + (hb : Cache.Backoff.active s.backoff sender ⟨g.topicId⟩ s.now = false) + (hd : ¬ p.mesh.dHigh ≤ (meshOf s ⟨g.topicId⟩).length) + (hpd : peerOf s sender = some pd) : + handleGraft p s sender g = + (setPeer (setTopicMesh s ⟨g.topicId⟩ (sender :: meshOf s ⟨g.topicId⟩)) sender + { pd with score := Score.graft pd.score s.now ⟨g.topicId⟩ }, []) := by + simp [handleGraft, hsub, hm, hb, hd, hpd] + +private theorem handleGraft_add_none (p : Params) (s : State) (sender : Peer) + (g : Rpc.ControlGraft) (hsub : memB (⟨g.topicId⟩ : TopicId) s.topics = true) + (hm : memB sender (meshOf s ⟨g.topicId⟩) = false) + (hb : Cache.Backoff.active s.backoff sender ⟨g.topicId⟩ s.now = false) + (hd : ¬ p.mesh.dHigh ≤ (meshOf s ⟨g.topicId⟩).length) + (hpd : peerOf s sender = none) : + handleGraft p s sender g = (s, []) := by + simp [handleGraft, hsub, hm, hb, hd, hpd] + +private theorem meshOf_addPeer (s : State) (sender : Peer) (pd : PeerData) (t topic : TopicId) + (h : topic ≠ t) : + meshOf (setPeer (setTopicMesh s t (sender :: meshOf s t)) sender pd) topic + = meshOf s topic := by + simp [meshOf, setPeer, setTopicMesh, lookupTopic_upsertMesh_ne topic t h] + +private theorem meshOf_addPeer_same (s : State) (sender : Peer) (pd : PeerData) + (t : TopicId) : + meshOf (setPeer (setTopicMesh s t (sender :: meshOf s t)) sender pd) t + = sender :: meshOf s t := by + simp [meshOf, setPeer, setTopicMesh, lookupTopic_upsertMesh_same] + +private theorem handleGraft_mesh_other (p : Params) (s : State) (sender : Peer) + (g : Rpc.ControlGraft) (topic : TopicId) (h : topic ≠ ⟨g.topicId⟩) : + meshOf (handleGraft p s sender g).1 topic = meshOf s topic := by + by_cases hsub : memB (⟨g.topicId⟩ : TopicId) s.topics = true + · by_cases hm : memB sender (meshOf s ⟨g.topicId⟩) = true + · rw [handleGraft_member p s sender g hsub hm] + · have hm' : memB sender (meshOf s ⟨g.topicId⟩) = false := by + cases h' : memB sender (meshOf s ⟨g.topicId⟩) <;> simp_all + by_cases hb : Cache.Backoff.active s.backoff sender ⟨g.topicId⟩ s.now = true + · rw [handleGraft_backoff p s sender g hsub hm' hb] + · have hb' : Cache.Backoff.active s.backoff sender ⟨g.topicId⟩ s.now = false := by + cases h' : Cache.Backoff.active s.backoff sender ⟨g.topicId⟩ s.now <;> simp_all + by_cases hd : p.mesh.dHigh ≤ (meshOf s ⟨g.topicId⟩).length + · rw [handleGraft_overfull p s sender g hsub hm' hb' hd] + · cases hpd : peerOf s sender with + | none => rw [handleGraft_add_none p s sender g hsub hm' hb' hd hpd] + | some pd => + rw [handleGraft_add p s sender g pd hsub hm' hb' hd hpd] + exact meshOf_addPeer s sender _ ⟨g.topicId⟩ topic h + · have hsub' : memB (⟨g.topicId⟩ : TopicId) s.topics = false := by + cases h' : memB (⟨g.topicId⟩ : TopicId) s.topics <;> simp_all + rw [handleGraft_unknown p s sender g hsub'] + +private theorem handleGraft_mesh_same (p : Params) (s : State) (sender : Peer) + (g : Rpc.ControlGraft) (hnot : memB sender (meshOf s ⟨g.topicId⟩) = false) + (hover : p.mesh.dHigh ≤ (meshOf s ⟨g.topicId⟩).length) : + meshOf (handleGraft p s sender g).1 ⟨g.topicId⟩ = meshOf s ⟨g.topicId⟩ := by + by_cases hsub : memB (⟨g.topicId⟩ : TopicId) s.topics = true + · by_cases hm : memB sender (meshOf s ⟨g.topicId⟩) = true + · rw [handleGraft_member p s sender g hsub hm] + · by_cases hb : Cache.Backoff.active s.backoff sender ⟨g.topicId⟩ s.now = true + · rw [handleGraft_backoff p s sender g hsub hnot hb] + · have hb' : Cache.Backoff.active s.backoff sender ⟨g.topicId⟩ s.now = false := by + cases h' : Cache.Backoff.active s.backoff sender ⟨g.topicId⟩ s.now <;> simp_all + by_cases hd : p.mesh.dHigh ≤ (meshOf s ⟨g.topicId⟩).length + · rw [handleGraft_overfull p s sender g hsub hnot hb' hd] + · exfalso + omega + · have hsub' : memB (⟨g.topicId⟩ : TopicId) s.topics = false := by + cases h' : memB (⟨g.topicId⟩ : TopicId) s.topics <;> simp_all + rw [handleGraft_unknown p s sender g hsub'] + +private theorem handleGraft_prune (p : Params) (s : State) (sender : Peer) + (g : Rpc.ControlGraft) (hsub0 : memB (⟨g.topicId⟩ : TopicId) s.topics = true) + (hnot : memB sender (meshOf s ⟨g.topicId⟩) = false) + (hover : p.mesh.dHigh ≤ (meshOf s ⟨g.topicId⟩).length) : + ∃ pr, Action.sendRpc sender pr ∈ (handleGraft p s sender g).2 ∧ + rpcPrunes pr ⟨g.topicId⟩ := by + by_cases hm : memB sender (meshOf s ⟨g.topicId⟩) = true + · exfalso + rw [hnot] at hm + simp at hm + · by_cases hb : Cache.Backoff.active s.backoff sender ⟨g.topicId⟩ s.now = true + · rw [handleGraft_backoff p s sender g hsub0 hnot hb] + exact ⟨pruneRpc ⟨g.topicId⟩ p.pruneBackoff, by simp, + by simp [pruneRpc, controlRpc, rpcPrunes]⟩ + · have hb' : Cache.Backoff.active s.backoff sender ⟨g.topicId⟩ s.now = false := by + cases h' : Cache.Backoff.active s.backoff sender ⟨g.topicId⟩ s.now <;> simp_all + rw [handleGraft_overfull p s sender g hsub0 hnot hb' hover] + exact ⟨pruneRpc ⟨g.topicId⟩ p.pruneBackoff, by simp, + by simp [pruneRpc, controlRpc, rpcPrunes]⟩ + +/-! ## The prune and control handler lemmas -/ + +private theorem filter_ne_self (l : List Peer) (sender : Peer) (h : memB sender l = false) : + l.filter (fun r => decide (r ≠ sender)) = l := by + rw [List.filter_eq_self] + intro x hx + have hx' : x ≠ sender := by + intro hc + subst hc + have hmem : memB x l = true := memB_eq_true.mpr hx + rw [h] at hmem + simp at hmem + simp [hx'] + +private theorem meshOf_pruneState (p : Params) (s : State) (sender : Peer) + (q : Rpc.ControlPrune) (topic : TopicId) : + meshOf (pruneState p s sender q) topic = meshOf s topic := by + simp [pruneState, meshOf] + +private theorem meshOf_setPeer (s : State) (q : Peer) (pd : PeerData) (topic : TopicId) : + meshOf (setPeer s q pd) topic = meshOf s topic := by + simp [meshOf, setPeer] + +private theorem handlePrune_mesh_same (p : Params) (s : State) (sender : Peer) + (q : Rpc.ControlPrune) (hnot : memB sender (meshOf s ⟨q.topicId⟩) = false) : + meshOf (handlePrune p s sender q).1 ⟨q.topicId⟩ = meshOf s ⟨q.topicId⟩ := by + have hst : meshOf (pruneState p s sender q) ⟨q.topicId⟩ = meshOf s ⟨q.topicId⟩ := + meshOf_pruneState p s sender q ⟨q.topicId⟩ + by_cases hm : memB sender (meshOf (pruneState p s sender q) ⟨q.topicId⟩) = true + · cases hpd : peerOf (pruneState p s sender q) sender with + | none => + simp only [handlePrune, hm, ite_true, hpd] + exact hst + | some pd => + have hf : (meshOf (pruneState p s sender q) ⟨q.topicId⟩).filter + (fun r => decide (r ≠ sender)) + = meshOf (pruneState p s sender q) ⟨q.topicId⟩ := by + apply filter_ne_self + rw [hst] + exact hnot + simp only [handlePrune, hm, ite_true, hpd] + rw [meshOf_setPeer, meshOf_setTopicMesh_same, hf, hst] + · simp only [handlePrune, hm] + exact hst + +private theorem handlePrune_mesh_ne (p : Params) (s : State) (sender : Peer) + (q : Rpc.ControlPrune) (topic : TopicId) (h : topic ≠ ⟨q.topicId⟩) : + meshOf (handlePrune p s sender q).1 topic = meshOf s topic := by + have hst : meshOf (pruneState p s sender q) topic = meshOf s topic := + meshOf_pruneState p s sender q topic + by_cases hm : memB sender (meshOf (pruneState p s sender q) ⟨q.topicId⟩) = true + · cases hpd : peerOf (pruneState p s sender q) sender with + | none => + simp only [handlePrune, hm, ite_true, hpd] + exact hst + | some pd => + simp only [handlePrune, hm, ite_true, hpd] + rw [meshOf_setPeer, + meshOf_setTopicMesh_ne (pruneState p s sender q) ⟨q.topicId⟩ topic h, hst] + · simp only [handlePrune, hm] + exact hst + +private theorem handlePrunes_mesh (p : Params) (s : State) (sender : Peer) + (qs : List Rpc.ControlPrune) (topic : TopicId) + (hnot : memB sender (meshOf s topic) = false) : + meshOf (mapHandlers (fun st b => handlePrune p st sender b) qs s).1 topic + = meshOf s topic := by + induction qs generalizing s with + | nil => simp [mapHandlers] + | cons q qs ih => + simp only [mapHandlers] + by_cases h : topic = ⟨q.topicId⟩ + · subst h + rw [ih (s := (handlePrune p s sender q).1) + (by rwa [handlePrune_mesh_same p s sender q hnot]), + handlePrune_mesh_same p s sender q hnot] + · rw [ih (s := (handlePrune p s sender q).1) + (by rwa [handlePrune_mesh_ne p s sender q topic h]), + handlePrune_mesh_ne p s sender q topic h] + +private theorem handleIhave_mesh (p : Params) (s : State) (sender : Peer) + (i : Rpc.ControlIHave) (topic : TopicId) : + meshOf (handleIhave p s sender i).1 topic = meshOf s topic := by + simp [handleIhave, meshOf] + +private theorem handleIwant_mesh (p : Params) (s : State) (sender : Peer) + (w : Rpc.ControlIWant) (topic : TopicId) : + meshOf (handleIwant p s sender w).1 topic = meshOf s topic := by + simp [handleIwant] + +private theorem handleIhaves_mesh (p : Params) (s : State) (sender : Peer) + (is : List Rpc.ControlIHave) (topic : TopicId) : + meshOf (mapHandlers (fun st b => handleIhave p st sender b) is s).1 topic + = meshOf s topic := by + induction is generalizing s with + | nil => simp [mapHandlers] + | cons i is ih => + simp only [mapHandlers] + rw [ih (s := (handleIhave p s sender i).1), handleIhave_mesh] + +private theorem handleIwants_mesh (p : Params) (s : State) (sender : Peer) + (ws : List Rpc.ControlIWant) (topic : TopicId) : + meshOf (mapHandlers (fun st b => handleIwant p st sender b) ws s).1 topic + = meshOf s topic := by + induction ws generalizing s with + | nil => simp [mapHandlers] + | cons w ws ih => + simp only [mapHandlers] + rw [ih (s := (handleIwant p s sender w).1), handleIwant_mesh] + +private theorem topics_mapHandlers {β : Type} (f : State → β → State × List Action) + (hf : ∀ (s : State) (b : β), (f s b).1.topics = s.topics) : + ∀ (bs : List β) (s : State), (mapHandlers f bs s).1.topics = s.topics + | [], s => rfl + | b :: bs, s => by + simp only [mapHandlers] + rw [topics_mapHandlers f hf bs (f s b).1, hf s b] + +private theorem handleIhave_topics (p : Params) (s : State) (sender : Peer) + (i : Rpc.ControlIHave) : (handleIhave p s sender i).1.topics = s.topics := by + simp [handleIhave] + +private theorem handleIwant_topics (p : Params) (s : State) (sender : Peer) + (w : Rpc.ControlIWant) : (handleIwant p s sender w).1.topics = s.topics := by + simp [handleIwant] + +private theorem handleGraft_topics (p : Params) (s : State) (sender : Peer) + (g : Rpc.ControlGraft) : (handleGraft p s sender g).1.topics = s.topics := by + by_cases hsub : memB (⟨g.topicId⟩ : TopicId) s.topics = true + · by_cases hm : memB sender (meshOf s ⟨g.topicId⟩) = true + · simp [handleGraft, hsub, hm] + · by_cases hb : Cache.Backoff.active s.backoff sender ⟨g.topicId⟩ s.now = true + · simp [handleGraft, hsub, hm, hb] + · by_cases hd : p.mesh.dHigh ≤ (meshOf s ⟨g.topicId⟩).length + · simp [handleGraft, hsub, hm, hb, hd] + · cases hpd : peerOf s sender <;> simp [handleGraft, hsub, hm, hb, hd, hpd, setPeer, setTopicMesh] + · simp [handleGraft, hsub] + +private theorem pruneState_topics (p : Params) (s : State) (sender : Peer) + (q : Rpc.ControlPrune) : (pruneState p s sender q).topics = s.topics := by + simp [pruneState] + +private theorem handlePrune_topics (p : Params) (s : State) (sender : Peer) + (q : Rpc.ControlPrune) : (handlePrune p s sender q).1.topics = s.topics := by + by_cases hm : memB sender (meshOf (pruneState p s sender q) ⟨q.topicId⟩) = true + · cases hpd : peerOf (pruneState p s sender q) sender with + | none => + simp only [handlePrune, hm, ite_true, hpd] + exact pruneState_topics p s sender q + | some pd => + simp only [handlePrune, hm, ite_true, hpd] + simp [setPeer, setTopicMesh, pruneState] + · simp only [handlePrune, hm] + exact pruneState_topics p s sender q + +private theorem controlIwants_topics (p : Params) (s : State) (sender : Peer) + (c : Rpc.ControlMessage) : (controlIwants p s sender c).topics = s.topics := by + simp only [controlIwants, controlIhaves] + rw [topics_mapHandlers (fun st b => handleIwant p st sender b) + (fun st b => handleIwant_topics p st sender b), + topics_mapHandlers (fun st b => handleIhave p st sender b) + (fun st b => handleIhave_topics p st sender b)] + +private theorem controlGrafts_topics (p : Params) (s : State) (sender : Peer) + (c : Rpc.ControlMessage) : (controlGrafts p s sender c).topics = s.topics := by + simp only [controlGrafts] + rw [topics_mapHandlers (fun st b => handleGraft p st sender b) + (fun st b => handleGraft_topics p st sender b), + controlIwants_topics] + +private theorem handleGrafts_mesh (p : Params) (s : State) (sender : Peer) + (gs : List Rpc.ControlGraft) (topic : TopicId) + (hsub : memB topic s.topics = true) + (hnot : memB sender (meshOf s topic) = false) + (hover : p.mesh.dHigh ≤ (meshOf s topic).length) : + meshOf (mapHandlers (fun st b => handleGraft p st sender b) gs s).1 topic + = meshOf s topic := by + induction gs generalizing s with + | nil => simp [mapHandlers] + | cons g gs ih => + simp only [mapHandlers] + have htop : (handleGraft p s sender g).1.topics = s.topics := + handleGraft_topics p s sender g + have hsub' : memB topic (handleGraft p s sender g).1.topics = true := by + rw [htop]; exact hsub + by_cases h : topic = ⟨g.topicId⟩ + · subst h + have hm : meshOf (handleGraft p s sender g).1 ⟨g.topicId⟩ = meshOf s ⟨g.topicId⟩ := + handleGraft_mesh_same p s sender g hnot hover + rw [ih (s := (handleGraft p s sender g).1) hsub' (by rw [hm]; exact hnot) + (by rw [hm]; exact hover), hm] + · have hm : meshOf (handleGraft p s sender g).1 topic = meshOf s topic := + handleGraft_mesh_other p s sender g topic h + rw [ih (s := (handleGraft p s sender g).1) hsub' (by rw [hm]; exact hnot) + (by rw [hm]; exact hover), hm] + +private theorem handleGrafts_prune (p : Params) (s : State) (sender : Peer) + (gs : List Rpc.ControlGraft) (topic : TopicId) + (hsub : memB topic s.topics = true) + (hnot : memB sender (meshOf s topic) = false) + (hover : p.mesh.dHigh ≤ (meshOf s topic).length) + (hg : ∃ g ∈ gs, g.topicId = topic.name) : + ∃ pr, Action.sendRpc sender pr ∈ + (mapHandlers (fun st b => handleGraft p st sender b) gs s).2 ∧ + rpcPrunes pr topic := by + induction gs generalizing s with + | nil => simp at hg + | cons g gs ih => + simp only [mapHandlers] + by_cases hsame : g.topicId = topic.name + · have ht : (⟨g.topicId⟩ : TopicId) = topic := by rw [hsame] + have hsub' : memB (⟨g.topicId⟩ : TopicId) s.topics = true := by + rw [ht]; exact hsub + have hnot' : memB sender (meshOf s ⟨g.topicId⟩) = false := by + rw [ht]; exact hnot + have hover' : p.mesh.dHigh ≤ (meshOf s ⟨g.topicId⟩).length := by + rw [ht]; exact hover + obtain ⟨pr, hpr, hrpc⟩ := handleGraft_prune p s sender g hsub' hnot' hover' + refine ⟨pr, List.mem_append_left _ hpr, ?_⟩ + rwa [ht] at hrpc + · have hg' : ∃ g' ∈ gs, g'.topicId = topic.name := by + obtain ⟨g', hg'mem, hg'eq⟩ := hg + rcases List.mem_cons.mp hg'mem with h | h + · rw [h] at hg'eq + exact absurd hg'eq hsame + · exact ⟨g', h, hg'eq⟩ + have htop : (handleGraft p s sender g).1.topics = s.topics := + handleGraft_topics p s sender g + have hmesh : meshOf (handleGraft p s sender g).1 topic = meshOf s topic := by + by_cases h : topic = ⟨g.topicId⟩ + · subst h + exact handleGraft_mesh_same p s sender g hnot hover + · exact handleGraft_mesh_other p s sender g topic h + obtain ⟨pr, hpr, hrpc⟩ := ih (s := (handleGraft p s sender g).1) + (by rw [htop]; exact hsub) (by rw [hmesh]; exact hnot) + (by rw [hmesh]; exact hover) hg' + exact ⟨pr, List.mem_append_right _ hpr, hrpc⟩ + +private theorem handleControl_mesh (p : Params) (s : State) (sender : Peer) + (c : Option Rpc.ControlMessage) (topic : TopicId) + (hsub : memB topic s.topics = true) + (hnot : memB sender (meshOf s topic) = false) + (hover : p.mesh.dHigh ≤ (meshOf s topic).length) : + meshOf (handleControl p s sender c).1 topic = meshOf s topic := by + cases c with + | none => rfl + | some c => + have h1 : meshOf (controlIhaves p s sender c) topic = meshOf s topic := by + simp only [controlIhaves] + exact handleIhaves_mesh p s sender c.ihave topic + have h2 : meshOf (controlIwants p s sender c) topic = meshOf s topic := by + simp only [controlIwants] + rw [handleIwants_mesh, h1] + have h3 : meshOf (controlGrafts p s sender c) topic = meshOf s topic := by + simp only [controlGrafts] + rw [handleGrafts_mesh p (controlIwants p s sender c) sender c.graft topic + (by rw [controlIwants_topics]; exact hsub) + (by rw [h2]; exact hnot) (by rw [h2]; exact hover), h2] + have h4 : meshOf (controlPrunes p s sender c) topic = meshOf s topic := by + simp only [controlPrunes] + rw [handlePrunes_mesh p (controlGrafts p s sender c) sender c.prune topic + (by rw [h3]; exact hnot), h3] + simp only [handleControl] + exact h4 + +private theorem handleControl_prune (p : Params) (s : State) (sender : Peer) + (c : Rpc.ControlMessage) (topic : TopicId) + (hsub : memB topic s.topics = true) + (hnot : memB sender (meshOf s topic) = false) + (hover : p.mesh.dHigh ≤ (meshOf s topic).length) + (hg : ∃ g ∈ c.graft, g.topicId = topic.name) : + ∃ pr, Action.sendRpc sender pr ∈ controlActions p s sender c ∧ rpcPrunes pr topic := by + have h2 : meshOf (controlIwants p s sender c) topic = meshOf s topic := by + simp only [controlIwants, controlIhaves] + rw [handleIwants_mesh, handleIhaves_mesh] + have htop : (controlIwants p s sender c).topics = s.topics := + controlIwants_topics p s sender c + obtain ⟨pr, hpr, hrpc⟩ := + handleGrafts_prune p (controlIwants p s sender c) sender c.graft topic + (by rw [htop]; exact hsub) (by rw [h2]; exact hnot) (by rw [h2]; exact hover) hg + refine ⟨pr, ?_, hrpc⟩ + simp only [controlActions] + exact List.mem_append_left _ (List.mem_append_right _ hpr) + +/-! ## The RPC pipeline lemmas -/ + +private theorem updateScoreOf_mesh (s : State) (q : Peer) + (f : Score.PeerState → Score.PeerState) (topic : TopicId) : + meshOf (updateScoreOf s q f) topic = meshOf s topic := by + simp [updateScoreOf, mapPeers, meshOf] + +private theorem updateScoreOf_topics (s : State) (q : Peer) + (f : Score.PeerState → Score.PeerState) : + (updateScoreOf s q f).topics = s.topics := by + simp [updateScoreOf, mapPeers] + +private theorem foldl_updateScoreOf_mesh (ms : List Message) (s : State) (sender : Peer) + (f : Message → Score.PeerState → Score.PeerState) (topic : TopicId) : + meshOf (ms.foldl (fun st m => updateScoreOf st sender (f m)) s) topic + = meshOf s topic := by + induction ms generalizing s with + | nil => rfl + | cons m ms ih => + rw [List.foldl_cons, ih, updateScoreOf_mesh] + +private theorem foldl_updateScoreOf_topics (ms : List Message) (s : State) (sender : Peer) + (f : Message → Score.PeerState → Score.PeerState) : + (ms.foldl (fun st m => updateScoreOf st sender (f m)) s).topics = s.topics := by + induction ms generalizing s with + | nil => rfl + | cons m ms ih => + rw [List.foldl_cons, ih, updateScoreOf_topics] + +private theorem handleSubs_mesh (s : State) (sender : Peer) (subs : List Rpc.SubOpts) + (topic : TopicId) : meshOf (handleSubs s sender subs) topic = meshOf s topic := by + cases h : peerOf s sender <;> simp [handleSubs, h, setPeer, meshOf] + +private theorem handleSubs_topics (s : State) (sender : Peer) (subs : List Rpc.SubOpts) : + (handleSubs s sender subs).topics = s.topics := by + cases h : peerOf s sender <;> simp [handleSubs, h, setPeer] + +private theorem handlePubs_mesh (p : Params) (s : State) (sender : Peer) + (ms : List Message) (topic : TopicId) : + meshOf (handlePubs p s sender ms).1 topic = meshOf s topic := by + simp only [handlePubs] + rw [foldl_updateScoreOf_mesh, foldl_updateScoreOf_mesh] + simp [meshOf] + +private theorem handlePubs_topics (p : Params) (s : State) (sender : Peer) + (ms : List Message) : (handlePubs p s sender ms).1.topics = s.topics := by + simp only [handlePubs] + rw [foldl_updateScoreOf_topics, foldl_updateScoreOf_topics] + +/-- The record of a peer after the heartbeat mesh write. -/ +theorem peerOf_settleMesh (p : Params) (s : State) (plans : List TopicPlan) (q : Peer) : + peerOf (settleMesh p s plans) q = + (peerOf s q).map (fun pd => + if scoreOf p s q < 0 then + { pd with score := s.topics.foldl (fun sc t => Score.prune p.score sc s.now t) pd.score } + else pd) := by + simp only [peerOf, settleMesh] + induction s.peers with + | nil => simp [lookupPeer] + | cons e l ih => + obtain ⟨a, b⟩ := e + simp only [List.map_cons, lookupPeer] + by_cases h : q = a + · subst h + simp + · simp [h, ih] + +/-- The score of a peer after the heartbeat mesh write. -/ +theorem scoreOf_settleMesh (p : Params) (s : State) (plans : List TopicPlan) (q : Peer) : + scoreOf p (settleMesh p s plans) q = + match peerOf s q with + | some pd => + Score.score p.score + (if scoreOf p s q < 0 then + s.topics.foldl (fun sc t => Score.prune p.score sc s.now t) pd.score + else pd.score) s.now + | none => 0 := by + unfold scoreOf + rw [peerOf_settleMesh] + cases h : peerOf s q with + | none => simp + | some pd => + by_cases hlt : Score.score p.score pd.score s.now < 0 <;> + simp [scoreOf, settleMesh, h, hlt] + +/-- The mesh of a topic after a heartbeat. -/ +theorem meshOf_heartbeatNext (p : Params) (s : State) (now : Millis) + (order : List Peer) (topic : TopicId) : + meshOf (heartbeatNext p s now order) topic = + match (heartbeatPlans p s now order).find? (fun pl => decide (topic = pl.topic)) with + | some pl => + pl.kept.filter (fun q => decide (0 ≤ scoreOf p (planScored p s now order) q)) + | none => [] := by + simp [heartbeatNext, settleMesh, meshOf, lookupTopic_map_plans] + +/-! ## The score gate theorems -/ + +private theorem rpcHasIhave_pruneRpc (t : TopicId) (b : Millis) : + ¬ rpcHasIhave (pruneRpc t b) := by + simp [rpcHasIhave, pruneRpc, controlRpc] + +private theorem rpcHasIhave_graftRpc (t : TopicId) : ¬ rpcHasIhave (graftRpc t) := by + simp [rpcHasIhave, graftRpc, controlRpc] + +/-- A gossip action targets a peer of the order at or above the gossip +threshold. -/ +theorem mem_gossipActions (p : Params) (s : State) (order : List Peer) + (to : Peer) (rpc : Rpc.RPC) + (h : Action.sendRpc to rpc ∈ gossipActions p s order) : + p.thresholds.gossipThreshold ≤ scoreOf p s to := by + simp only [gossipActions, List.mem_flatMap] at h + obtain ⟨t, _, ht⟩ := h + split at ht + · simp at ht + · next ids hne => + simp only [List.mem_map] at ht + obtain ⟨q, hq, hqr⟩ := ht + cases hqr + have htake := List.mem_of_mem_take hq + rw [List.mem_filter] at htake + obtain ⟨hbase, _⟩ := htake + rw [List.mem_filter] at hbase + obtain ⟨_, hscore⟩ := hbase + exact of_decide_eq_true hscore + +/-- The gossip state keeps the settled scores and replaces the cache, so +the score of a peer is the score after the heartbeat. -/ +theorem scoreOf_gossipState (p : Params) (s : State) (now : Millis) (order : List Peer) + (q : Peer) : + scoreOf p (gossipState p s now order) q = scoreOf p (heartbeatNext p s now order) q := by + simp [scoreOf, gossipState, peerOf, heartbeatNext] + +theorem tickPruneNegative (p : Params) (s s' : State) (acts : List Action) + (now : Millis) (order : List Peer) + (hdue : heartbeatDue p s now) + (hstep : step p s (.tick now order) = .ok (s', acts)) : + ∀ topic peer, peer ∈ meshOf s' topic → 0 ≤ scoreOf p s' peer := by + simp only [step, ite_eq_left hdue, Except.ok.injEq] at hstep + obtain ⟨rfl, rfl⟩ := hstep + intro topic peer hmem + rw [meshOf_heartbeatNext] at hmem + split at hmem + · next pl hfind => + simp only [List.mem_filter] at hmem + have hge : 0 ≤ scoreOf p (planScored p s now order) peer := of_decide_eq_true hmem.2 + have hscore : scoreOf p (heartbeatNext p s now order) peer + = scoreOf p (planScored p s now order) peer := by + simp only [heartbeatNext] + rw [scoreOf_settleMesh] + cases hpd : peerOf (planScored p s now order) peer with + | none => simp [scoreOf, hpd] + | some pd => + have hge' : 0 ≤ Score.score p.score pd.score (planScored p s now order).now := by + simpa [scoreOf, hpd] using hge + have hlt : ¬ Score.score p.score pd.score (planScored p s now order).now < 0 := + Int.not_lt_of_ge hge' + simp [scoreOf, hpd, hlt] + rw [hscore] + exact hge + · simp at hmem + +/-- Every gossip action targets a peer at or above the gossip threshold +in the state after the heartbeat. The prune and graft actions fail the +ihave test. -/ +theorem tickGossipGated (p : Params) (s s' : State) (acts : List Action) + (now : Millis) (order : List Peer) + (hdue : heartbeatDue p s now) + (hstep : step p s (.tick now order) = .ok (s', acts)) : + ∀ to rpc, Action.sendRpc to rpc ∈ acts → rpcHasIhave rpc → + p.thresholds.gossipThreshold ≤ scoreOf p s' to := by + simp only [step, ite_eq_left hdue, Except.ok.injEq] at hstep + obtain ⟨rfl, rfl⟩ := hstep + intro to rpc hmem hih + simp only [heartbeatActions, List.mem_append] at hmem + rcases hmem with hmem | hmem + · rcases hmem with hmem | hmem + · simp only [planActionsOf, List.mem_flatMap] at hmem + obtain ⟨pl, _, hpl⟩ := hmem + simp only [planActions, List.mem_append, List.mem_map] at hpl + rcases hpl with ⟨q, _, hq⟩ | ⟨q, _, hq⟩ + · cases hq + exact absurd hih (rpcHasIhave_pruneRpc _ _) + · cases hq + exact absurd hih (rpcHasIhave_graftRpc _) + · simp only [negativeActions, List.mem_flatMap, List.mem_map] at hmem + obtain ⟨pl, _, q, _, hq⟩ := hmem + cases hq + exact absurd hih (rpcHasIhave_pruneRpc _ _) + · rw [← scoreOf_gossipState] + exact mem_gossipActions p (gossipState p s now order) order to rpc hmem + +/-! ## The deduplication theorem -/ + +/-- Every accepted message was absent from the pre-state seen cache. -/ +theorem accept_gate (c0 : Cache.SeenCache) (now : Millis) (idf : Sign.MsgIdFn) + (done : List MessageId) (c : Cache.SeenCache) (ms : List Message) : + ∀ m ∈ (accept c0 now idf done c ms).accepted, c0.contains (idf m) now = false := by + induction ms generalizing done c with + | nil => simp [accept] + | cons m ms ih => + simp only [accept] + split + · exact ih done c + · next hskip => + intro m' hm' + rw [List.mem_cons] at hm' + rcases hm' with hhead | hm' + · rw [hhead] + cases hc : c0.contains (idf m) now with + | false => rfl + | true => exact absurd (by rw [hc]; rfl) hskip + · exact ih (idf m :: done) (c.add (idf m) now) m' hm' + +private theorem forwardPubs_not_deliver (s : State) (sender : Peer) (ms : List Message) + (m : Message) : Action.deliver m ∉ forwardPubs s sender ms := by + intro h + simp only [forwardPubs, List.mem_flatMap, List.mem_map] at h + obtain ⟨_, _, q, _, hq⟩ := h + cases hq + +/-- An inbound forward never goes back to the sender. -/ +theorem forwardPubs_excludes_sender (s : State) (sender : Peer) (ms : List Message) + (rpc : Rpc.RPC) : Action.sendRpc sender rpc ∉ forwardPubs s sender ms := by + intro h + simp only [forwardPubs, List.mem_flatMap, List.mem_map] at h + rcases h with ⟨_, _, _, hqmem, heq⟩ + injection heq with hpeer _ + simp only [inboundTargets, List.mem_filter] at hqmem + have hne : _ ≠ sender := of_decide_eq_true hqmem.2 + exact hne hpeer + +private theorem handleIhave_not_deliver (p : Params) (s : State) (sender : Peer) + (i : Rpc.ControlIHave) (m : Message) : + Action.deliver m ∉ (handleIhave p s sender i).2 := by + simp only [handleIhave] + split <;> simp + +private theorem handleIwant_not_deliver (p : Params) (s : State) (sender : Peer) + (w : Rpc.ControlIWant) (m : Message) : + Action.deliver m ∉ (handleIwant p s sender w).2 := by + simp only [handleIwant] + simp [List.mem_map] + +private theorem handlePrune_not_deliver (p : Params) (s : State) (sender : Peer) + (q : Rpc.ControlPrune) (m : Message) : + Action.deliver m ∉ (handlePrune p s sender q).2 := by + simp only [handlePrune] + split + · split <;> simp + · simp + +private theorem handleGraft_not_deliver (p : Params) (s : State) (sender : Peer) + (g : Rpc.ControlGraft) (m : Message) : + Action.deliver m ∉ (handleGraft p s sender g).2 := by + by_cases hsub : memB (⟨g.topicId⟩ : TopicId) s.topics = true + · by_cases hm : memB sender (meshOf s ⟨g.topicId⟩) = true + · rw [handleGraft_member p s sender g hsub hm] + simp + · have hm' : memB sender (meshOf s ⟨g.topicId⟩) = false := by + cases h' : memB sender (meshOf s ⟨g.topicId⟩) <;> simp_all + by_cases hb : Cache.Backoff.active s.backoff sender ⟨g.topicId⟩ s.now = true + · rw [handleGraft_backoff p s sender g hsub hm' hb] + simp + · have hb' : Cache.Backoff.active s.backoff sender ⟨g.topicId⟩ s.now = false := by + cases h' : Cache.Backoff.active s.backoff sender ⟨g.topicId⟩ s.now <;> simp_all + by_cases hd : p.mesh.dHigh ≤ (meshOf s ⟨g.topicId⟩).length + · rw [handleGraft_overfull p s sender g hsub hm' hb' hd] + simp + · cases hpd : peerOf s sender with + | none => + rw [handleGraft_add_none p s sender g hsub hm' hb' hd hpd] + simp + | some pd => + rw [handleGraft_add p s sender g pd hsub hm' hb' hd hpd] + simp + · have hsub' : memB (⟨g.topicId⟩ : TopicId) s.topics = false := by + cases h' : memB (⟨g.topicId⟩ : TopicId) s.topics <;> simp_all + rw [handleGraft_unknown p s sender g hsub'] + simp + +private theorem mapHandlers_not_deliver {β : Type} (f : State → β → State × List Action) + (hf : ∀ (s : State) (b : β) (m : Message), Action.deliver m ∉ (f s b).2) : + ∀ (bs : List β) (s : State) (m : Message), Action.deliver m ∉ (mapHandlers f bs s).2 := by + intro bs + induction bs with + | nil => + intro s m + simp [mapHandlers] + | cons b bs ih => + intro s m + simp only [mapHandlers, List.mem_append, not_or] + exact ⟨hf s b m, ih (f s b).1 m⟩ + +private theorem controlActions_not_deliver (p : Params) (s : State) (sender : Peer) + (c : Rpc.ControlMessage) (m : Message) : + Action.deliver m ∉ controlActions p s sender c := by + intro h + simp only [controlActions, List.mem_append] at h + rcases h with h | h + · rcases h with h | h + · rcases h with h | h + · exact mapHandlers_not_deliver _ (fun s b => handleIhave_not_deliver p s sender b) _ _ _ h + · exact mapHandlers_not_deliver _ (fun s b => handleIwant_not_deliver p s sender b) _ _ _ h + · exact mapHandlers_not_deliver _ (fun s b => handleGraft_not_deliver p s sender b) _ _ _ h + · exact mapHandlers_not_deliver _ (fun s b => handlePrune_not_deliver p s sender b) _ _ _ h + +/-- A message reaches the application only when the pre-state seen cache +does not hold its id. -/ +private theorem handleSubs_seen (s : State) (sender : Peer) (subs : List Rpc.SubOpts) : + (handleSubs s sender subs).seen = s.seen := by + cases h : peerOf s sender <;> simp [handleSubs, h, setPeer] + +private theorem handleSubs_now (s : State) (sender : Peer) (subs : List Rpc.SubOpts) : + (handleSubs s sender subs).now = s.now := by + cases h : peerOf s sender <;> simp [handleSubs, h, setPeer] + +private theorem join_not_deliver (p : Params) (s : State) (peer : Peer) + (proto : PubsubProto) (m : Message) : Action.deliver m ∉ (join p s peer proto).2 := by + simp [join] + +private theorem leave_not_deliver (s : State) (peer : Peer) (m : Message) : + Action.deliver m ∉ (leave s peer).2 := by + simp [leave] + +private theorem publishLocal_not_deliver (p : Params) (s : State) (msg : Message) + (m : Message) : Action.deliver m ∉ (publishLocal p s msg).2 := by + unfold publishLocal + by_cases h : validationOk p msg + · simp [h, List.mem_map] + · simp [h] + +private theorem subscribe_not_deliver (p : Params) (s : State) (topic : TopicId) + (m : Message) : Action.deliver m ∉ (subscribe p s topic).2 := by + simp only [subscribe] + intro h + rw [List.mem_map] at h + obtain ⟨q, -, hq⟩ := h + cases hq + +private theorem unsubscribe_not_deliver (p : Params) (s : State) (topic : TopicId) + (m : Message) : Action.deliver m ∉ (unsubscribe p s topic).2 := by + simp only [unsubscribe] + intro h + rw [List.mem_map] at h + obtain ⟨q, -, hq⟩ := h + cases hq + +private theorem planActions_not_deliver (p : Params) (pl : TopicPlan) (m : Message) : + Action.deliver m ∉ planActions p pl := by + simp [planActions, List.mem_map] + +private theorem planActionsOf_not_deliver (p : Params) (s : State) (now : Millis) + (order : List Peer) (m : Message) : + Action.deliver m ∉ planActionsOf p s now order := by + intro h + simp only [planActionsOf, List.mem_flatMap] at h + obtain ⟨pl, -, hp⟩ := h + exact planActions_not_deliver p pl m hp + +private theorem negativeActions_not_deliver (p : Params) (s : State) (now : Millis) + (order : List Peer) (m : Message) : + Action.deliver m ∉ negativeActions p s now order := by + intro h + simp only [negativeActions, List.mem_flatMap, List.mem_map] at h + obtain ⟨pl, -, q, -, hq⟩ := h + cases hq + +private theorem gossipActions_not_deliver (p : Params) (s : State) (order : List Peer) + (m : Message) : Action.deliver m ∉ gossipActions p s order := by + intro h + simp only [gossipActions, List.mem_flatMap] at h + obtain ⟨t, -, ht⟩ := h + split at ht + · simp at ht + · simp only [List.mem_map] at ht + obtain ⟨q, -, hq⟩ := ht + cases hq + +private theorem heartbeatActions_not_deliver (p : Params) (s : State) (now : Millis) + (order : List Peer) (m : Message) : + Action.deliver m ∉ heartbeatActions p s now order := by + intro h + simp only [heartbeatActions, List.mem_append] at h + rcases h with h | h + · rcases h with h | h + · exact planActionsOf_not_deliver p s now order m h + · exact negativeActions_not_deliver p s now order m h + · exact gossipActions_not_deliver p (gossipState p s now order) order m h + +/-- A message reaches the application only when the pre-state seen cache +does not hold its id. -/ +theorem stepDeliverUnseen (p : Params) (s s' : State) (acts : List Action) + (e : Event) (m : Message) + (hstep : step p s e = .ok (s', acts)) (hdel : Action.deliver m ∈ acts) : + Cache.SeenCache.contains s.seen (p.msgId m) s.now = false := by + cases e with + | peerJoined peer proto => + have h1 : join p s peer proto = (s', acts) := by + simpa only [step, Except.ok.injEq] using hstep + have hacts : acts = (join p s peer proto).2 := by rw [h1] + rw [hacts] at hdel + exact absurd hdel (join_not_deliver p s peer proto m) + | peerLeft peer => + have h1 : leave s peer = (s', acts) := by + simpa only [step, Except.ok.injEq] using hstep + have hacts : acts = (leave s peer).2 := by rw [h1] + rw [hacts] at hdel + exact absurd hdel (leave_not_deliver s peer m) + | publish msg => + have h1 : publishLocal p s msg = (s', acts) := by + simpa only [step, Except.ok.injEq] using hstep + have hacts : acts = (publishLocal p s msg).2 := by rw [h1] + rw [hacts] at hdel + exact absurd hdel (publishLocal_not_deliver p s msg m) + | subscribe topic => + have h1 : subscribe p s topic = (s', acts) := by + simpa only [step, Except.ok.injEq] using hstep + have hacts : acts = (subscribe p s topic).2 := by rw [h1] + rw [hacts] at hdel + exact absurd hdel (subscribe_not_deliver p s topic m) + | unsubscribe topic => + have h1 : unsubscribe p s topic = (s', acts) := by + simpa only [step, Except.ok.injEq] using hstep + have hacts : acts = (unsubscribe p s topic).2 := by rw [h1] + rw [hacts] at hdel + exact absurd hdel (unsubscribe_not_deliver p s topic m) + | tick now order => + by_cases hdue : heartbeatDue p s now + · have h1 : heartbeat p s now order = (s', acts) := by + simpa only [step, ite_eq_left hdue, Except.ok.injEq] using hstep + have hacts : acts = (heartbeat p s now order).2 := by rw [h1] + simp only [heartbeat] at hacts + rw [hacts] at hdel + exact absurd hdel (heartbeatActions_not_deliver p s now order m) + · have h1 : (s, ([] : List Action)) = (s', acts) := by + simpa only [step, ite_eq_right hdue, Except.ok.injEq] using hstep + have hacts : acts = [] := by + have hs := congrArg Prod.snd h1 + simpa using hs + rw [hacts] at hdel + simp at hdel + | rpc sender bytes => + simp only [step, rpcStep] at hstep + split at hstep + · have h1 : (s, ([] : List Action)) = (s', acts) := by + simpa only [Except.ok.injEq] using hstep + have hacts : acts = [] := by + have hs := congrArg Prod.snd h1 + simpa using hs + rw [hacts] at hdel + simp at hdel + · split at hstep + · have h1 : (s, ([] : List Action)) = (s', acts) := by + simpa only [Except.ok.injEq] using hstep + have hacts : acts = [] := by + have hs := congrArg Prod.snd h1 + simpa using hs + rw [hacts] at hdel + simp at hdel + · split at hstep + · cases hstep + · next rpc hdec => + have h1 : handleRpc p s sender rpc = (s', acts) := by + simpa only [Except.ok.injEq] using hstep + have hacts : acts = (handleRpc p s sender rpc).2 := by rw [h1] + rw [hacts] at hdel + simp only [handleRpc, List.mem_append] at hdel + rcases hdel with hdel | hdel + · simp only [handlePubs, List.mem_append] at hdel + rcases hdel with hdl | hdl + · simp only [List.mem_map] at hdl + obtain ⟨m', hm', heq⟩ := hdl + have hseen : (handleSubs s sender rpc.subscriptions).seen = s.seen := + handleSubs_seen s sender rpc.subscriptions + have hnow : (handleSubs s sender rpc.subscriptions).now = s.now := + handleSubs_now s sender rpc.subscriptions + cases heq + have hgate := accept_gate (handleSubs s sender rpc.subscriptions).seen + (handleSubs s sender rpc.subscriptions).now p.msgId [] + (handleSubs s sender rpc.subscriptions).seen + (List.filter (fun m => validationOk p m) rpc.publish) m hm' + rwa [hseen, hnow] at hgate + · exact absurd hdl + (forwardPubs_not_deliver (handleSubs s sender rpc.subscriptions) sender + ((accept (handleSubs s sender rpc.subscriptions).seen + (handleSubs s sender rpc.subscriptions).now p.msgId [] + (handleSubs s sender rpc.subscriptions).seen + (List.filter (fun m => validationOk p m) rpc.publish)).accepted) m) + · cases hctl : rpc.control with + | none => + rw [hctl] at hdel + simp [handleControl] at hdel + | some c => + rw [hctl] at hdel + simp only [handleControl] at hdel + exact absurd hdel (controlActions_not_deliver p _ sender c m) + +/-! ## The backoff theorem -/ + +private theorem rpcGrafts_graftRpc (t t' : TopicId) (h : rpcGrafts (graftRpc t) t') : t' = t := by + simp only [rpcGrafts, graftRpc, controlRpc, List.mem_cons, List.mem_nil_iff, or_false] at h + obtain ⟨g, hg, hgt⟩ := h + subst hg + obtain ⟨tn⟩ := t + obtain ⟨t'n⟩ := t' + simpa using hgt.symm + +private theorem rpcGrafts_pruneRpc (t t' : TopicId) (b : Millis) : + ¬ rpcGrafts (pruneRpc t b) t' := by + simp [rpcGrafts, pruneRpc, controlRpc] + +private theorem rpcGrafts_ihaveRpc (t t' : TopicId) (ids : List MessageId) : + ¬ rpcGrafts (ihaveRpc t ids) t' := by + simp [rpcGrafts, ihaveRpc, controlRpc] + +private theorem freshState_backoff (p : Params) (s : State) (now : Millis) : + (freshState p s now).backoff = Cache.Backoff.clear s.backoff now := by + simp [freshState] + +private theorem planTopic_added_backoff (p : Params) (s : State) (now : Millis) + (order : List Peer) (topic : TopicId) {q : Peer} + (h : q ∈ (planTopic p s now order topic).added) : + Cache.Backoff.active s.backoff q topic now = false := by + simp only [planTopic] at h + have hc : q ∈ order.filter (fun q => + !memB q (meshOf s topic) && !(Cache.Backoff.active s.backoff q topic now) && + (peerOf s q).isSome && decide (0 ≤ scoreOf p s q)) := List.mem_of_mem_take h + cases hb : Cache.Backoff.active s.backoff q topic now with + | false => rfl + | true => + exfalso + simp only [List.mem_filter] at hc + rw [hb] at hc + simp at hc + +/-- The heartbeat never grafts a peer whose backoff is active. -/ +theorem tickNoBackoffGraft (p : Params) (s s' : State) (acts : List Action) + (now : Millis) (order : List Peer) + (hdue : heartbeatDue p s now) + (hstep : step p s (.tick now order) = .ok (s', acts)) : + ∀ to rpc topic, Action.sendRpc to rpc ∈ acts → rpcGrafts rpc topic → + Cache.Backoff.active s.backoff to topic now = false := by + simp only [step, ite_eq_left hdue, Except.ok.injEq] at hstep + obtain ⟨rfl, rfl⟩ := hstep + intro to rpc topic hmem hgraft + simp only [heartbeatActions, List.mem_append] at hmem + rcases hmem with hmem | hmem + · rcases hmem with hmem | hmem + · simp only [planActionsOf, List.mem_flatMap] at hmem + obtain ⟨pl, hpl, hmem⟩ := hmem + simp only [planActions, List.mem_append, List.mem_map] at hmem + rcases hmem with ⟨q, hq, hqeq⟩ | ⟨q, hq, hqeq⟩ + · cases hqeq + exact absurd hgraft (rpcGrafts_pruneRpc _ _ _) + · cases hqeq + have htopic : topic = pl.topic := rpcGrafts_graftRpc pl.topic topic hgraft + subst htopic + have hpl' : pl ∈ (freshState p s now).mesh.map + (fun e => planTopic p (freshState p s now) now order e.1) := by + simpa [heartbeatPlans, planList] using hpl + simp only [List.mem_map] at hpl' + obtain ⟨e, -, rfl⟩ := hpl' + have hb := planTopic_added_backoff p (freshState p s now) now order e.1 hq + change Cache.Backoff.active s.backoff to e.1 now = false + cases hcase : Cache.Backoff.active s.backoff to e.1 now with + | false => rfl + | true => + exfalso + have hclear := Cache.clear_keeps_active s.backoff to e.1 now hcase + rw [← freshState_backoff p s now] at hclear + rw [hb] at hclear + simp at hclear + · simp only [negativeActions, List.mem_flatMap, List.mem_map] at hmem + obtain ⟨pl, -, q, -, hqeq⟩ := hmem + cases hqeq + exact absurd hgraft (rpcGrafts_pruneRpc _ _ _) + · simp only [gossipActions, List.mem_flatMap] at hmem + obtain ⟨t, -, ht⟩ := hmem + split at ht + · simp at ht + · simp only [List.mem_map] at ht + obtain ⟨q, -, hqeq⟩ := ht + cases hqeq + exact absurd hgraft (rpcGrafts_ihaveRpc _ _ _) + +end Libp2p.Protocols.Gossipsub.Core diff --git a/packages/libp2p/protocols/Libp2p/Protocols/Gossipsub/HostSeam.lean b/packages/libp2p/protocols/Libp2p/Protocols/Gossipsub/HostSeam.lean new file mode 100644 index 00000000..b813e922 --- /dev/null +++ b/packages/libp2p/protocols/Libp2p/Protocols/Gossipsub/HostSeam.lean @@ -0,0 +1,25 @@ +import Libp2p.Host +import Libp2p.Core + +set_option autoImplicit false + +namespace Libp2p.Protocols.Gossipsub + +/-- The host operations used by a gossipsub node. -/ +structure HostSeam where + /-- Open a stream to a peer. -/ + newStream : Libp2p.Identity.PeerId → List Libp2p.Core.ProtocolId → + Conc.Async (Except Libp2p.Core.Error (Libp2p.Core.ProtocolId × Streams.Stream)) + /-- Register a stream handler. -/ + setHandler : Libp2p.Core.StreamHandler → BaseIO Unit + +/-- Adapt a host to the node operations. -/ +def HostSeam.ofHost (host : Libp2p.Host.Host) : HostSeam where + setHandler := host.setHandler + newStream := fun peer ids => do + match ← host.newStream peer ids with + | .ok pair => return .ok pair + | .error (.stream e) => return .error e + | .error e => return .error (.protocol e.render) + +end Libp2p.Protocols.Gossipsub diff --git a/packages/libp2p/protocols/Libp2p/Protocols/Gossipsub/Interop.lean b/packages/libp2p/protocols/Libp2p/Protocols/Gossipsub/Interop.lean new file mode 100644 index 00000000..16f157b4 --- /dev/null +++ b/packages/libp2p/protocols/Libp2p/Protocols/Gossipsub/Interop.lean @@ -0,0 +1,14 @@ +import Libp2p.Protocols.Gossipsub.Interop.Address +import Libp2p.Protocols.Gossipsub.Interop.Hosts +import Libp2p.Protocols.Gossipsub.Interop.MessageId +import Libp2p.Protocols.Gossipsub.Interop.Record +import Libp2p.Protocols.Gossipsub.Interop.Schedule +import Libp2p.Protocols.Gossipsub.Interop.Script + +/-! +# GossipSub interop pure core + +This root re-exports the pure adapter parts. The parser, hosts resolver, virtual +schedule, record encoder, address helpers, and message id functions perform no +network IO. Only the hosts-file edge reads the filesystem. +-/ diff --git a/packages/libp2p/protocols/Libp2p/Protocols/Gossipsub/Interop/Address.lean b/packages/libp2p/protocols/Libp2p/Protocols/Gossipsub/Interop/Address.lean new file mode 100644 index 00000000..d01cd34f --- /dev/null +++ b/packages/libp2p/protocols/Libp2p/Protocols/Gossipsub/Interop/Address.lean @@ -0,0 +1,40 @@ +import Lean + +/-! +# GossipSub interop node addresses + +This module implements specification section 7. It accepts only the exact +lowercase prefix node followed by decimal digits. The suffix must be +non-empty, must not have leading zeroes, and must fit an unsigned 64 bit value. +The peer seed writes the node number into the first eight bytes +in little endian order and fills the remaining 24 bytes with zeroes. +-/ + +set_option autoImplicit false + +namespace Libp2p.Protocols.Gossipsub.Interop + +/-- Parse a simulation hostname into its node number. -/ +def nodeId? (hostname : String) : Option Nat := + if !hostname.startsWith "node" then + none + else + let suffix := (hostname.drop 4).toString + match suffix.toList with + | [] => none + | '0' :: _ :: _ => none + | chars => + if !chars.all (fun c => '0' ≤ c && c ≤ '9') then none + else match suffix.toNat? with + | some value => if value ≤ 18446744073709551615 then some value else none + | none => none + +/-- Return one little endian byte from a natural number. -/ +def littleEndianByte (value index : Nat) : Nat := + (value / (2 ^ (8 * index))) % 256 + +/-- Build the 32 byte seed used by the interop peer id. -/ +def peerSeed (nodeId : Nat) : List Nat := + (List.range 8).map (littleEndianByte nodeId) ++ List.replicate 24 0 + +end Libp2p.Protocols.Gossipsub.Interop diff --git a/packages/libp2p/protocols/Libp2p/Protocols/Gossipsub/Interop/Hosts.lean b/packages/libp2p/protocols/Libp2p/Protocols/Gossipsub/Interop/Hosts.lean new file mode 100644 index 00000000..9502b516 --- /dev/null +++ b/packages/libp2p/protocols/Libp2p/Protocols/Gossipsub/Interop/Hosts.lean @@ -0,0 +1,89 @@ +import Lean +import Multiformats.Multiaddr.Ip + +/-! +# Shadow hosts resolution + +Shadow writes one hosts file per simulated node. Each file keeps the +loopback row and adds one IPv4 row per simulated host, for example +`11.0.0.1 node0`. The probe in GATE-PROBES-2026-11-06.md confirmed the +format, so the adapter resolves `nodeN` from that file instead of calling +`getaddrinfo` through FFI. + +This module keeps the parsing pure. `parseHosts` and `lookup?` run on a +string, so the suite pins them without a filesystem. Only `lookupHostsFile` +reads `/etc/hosts`; it delegates every decision to the pure parser. +-/ + +set_option autoImplicit false + +namespace Libp2p.Protocols.Gossipsub.Interop + +open Multiformats.Multiaddr + +/-- A parsed IPv4 address holds its four octets, the wire shape `parseIp4?` returns. -/ +abbrev IPv4 := List Nat + +/-- Drop the ASCII whitespace a hosts line can carry at either end. -/ +private def trim (text : String) : String := text.trimAscii.toString + +/-- Reuse the strict dotted-quad parser from the multiformats package. -/ +private def parseIPv4? (text : String) : Option IPv4 := + match parseIp4? text with + | .ok bytes => some bytes + | .error _ => none + +/-- Parse one hosts line into an IPv4 address and its exact names. + +A line is ` [...]`. A `#` starts a comment that runs to +the end of the line, spaces and tabs separate fields, and blank fields are +dropped. The line is ignored when the address is not a strict IPv4 value, +when it carries no name, or when it is empty after the comment. IPv6 rows +fail the IPv4 parser and are therefore ignored rather than rejected. -/ +private def parseLine (line : String) : Option (IPv4 × List String) := + let body := (trim line).splitOn "#" |>.headD "" |>.trimAscii.toString + let fields := (body.split (fun c => c == ' ' || c == '\t')).toList + let fields := fields.filter (fun field => !field.isEmpty) + match fields with + | address :: names => + let address := address.toString + let names := names.map String.Slice.toString + if names.isEmpty then none + else parseIPv4? address |>.map (fun ip => (ip, names)) + | [] => none + +/-- Parse hosts content into IPv4 rows with their exact names. + +The parser splits on line feeds only. A carriage return survives a split and +the field trim removes it, so CRLF content parses without a second pass. -/ +def parseHosts (content : String) : List (IPv4 × List String) := + content.splitOn "\n" |>.filterMap parseLine + +/-- Find the IPv4 address of one name across primary names and aliases. + +The comparison is exact. `node1` does not match the row `11.0.0.3 node10`, +and `node1x` matches nothing, because list equality does not accept a +prefix. The first row that carries the name wins, matching the order the +hosts file states. -/ +def lookup? (hosts : List (IPv4 × List String)) (name : String) : Option IPv4 := + hosts.find? (fun entry => entry.2.any (fun candidate => candidate == name)) + |>.map Prod.fst + +/-- Build the transport address prefix for one resolved address. + +The interop transport fixes the port at 9000 (specification section 8). +The caller appends `/p2p/` once the identity package derives it, +so this function returns only the prefix it can build from the address. -/ +def multiaddrPrefix (ip : IPv4) : Option String := + ip4String? ip |>.toOption + |>.map (fun text => s!"/ip4/{text}/tcp/9000") + +/-- Read Shadow's hosts file and resolve one name through the pure parser. + +This is the only definition in the module that touches the filesystem, and +it performs no parsing of its own. -/ +def lookupHostsFile (name : String) : IO (Option IPv4) := do + let content ← IO.FS.readFile "/etc/hosts" + pure (lookup? (parseHosts content) name) + +end Libp2p.Protocols.Gossipsub.Interop diff --git a/packages/libp2p/protocols/Libp2p/Protocols/Gossipsub/Interop/MessageId.lean b/packages/libp2p/protocols/Libp2p/Protocols/Gossipsub/Interop/MessageId.lean new file mode 100644 index 00000000..ab53afcd --- /dev/null +++ b/packages/libp2p/protocols/Libp2p/Protocols/Gossipsub/Interop/MessageId.lean @@ -0,0 +1,44 @@ +import Libp2p.Protocols.Gossipsub.Core + +/-! +# GossipSub interop message ids + +This module implements specification section 6. The harness reads the first +8 payload bytes as one big endian unsigned integer and prints base ten. +A short payload gets zero bytes appended on the right. This gives a total +function and matches the byte-position rule for every available byte. + +The core stores the same padded eight bytes in MessageId. A caller can use +coreMsgId in Core.Params.msgId without changing the core state machine. +-/ + +set_option autoImplicit false + +namespace Libp2p.Protocols.Gossipsub.Interop + +open Libp2p.Protocols.Gossipsub + +/-- Pad a payload on the right so the id always has eight input bytes. -/ +def paddedPayload (payload : List Nat) : List Nat := + let headBytes := payload.take 8; headBytes ++ List.replicate (8 - headBytes.length) 0 + +/-- Read up to eight bytes as a big endian natural number. -/ +def readBigEndian (bytes : List Nat) : Nat := + bytes.foldl (fun acc byte => acc * 256 + byte % 256) 0 + +/-- Read the message id as an unsigned 64 bit value. -/ +def messageIdNumber (payload : List Nat) : UInt64 := + UInt64.ofNat (readBigEndian (paddedPayload payload)) + +/-- Format the message id as an unsigned base ten string. -/ +def messageIdString (payload : List Nat) : String := + (messageIdNumber payload).toNat |> Nat.repr + +/-- Adapt a gossipsub message to the harness message id rule. -/ +def coreMsgId (message : Message) : MessageId := + ⟨paddedPayload ((message.data).getD [])⟩ + +/-- The function value for Core.Params.msgId. -/ +def msgIdFn : Sign.MsgIdFn := coreMsgId + +end Libp2p.Protocols.Gossipsub.Interop diff --git a/packages/libp2p/protocols/Libp2p/Protocols/Gossipsub/Interop/Record.lean b/packages/libp2p/protocols/Libp2p/Protocols/Gossipsub/Interop/Record.lean new file mode 100644 index 00000000..41a85bf2 --- /dev/null +++ b/packages/libp2p/protocols/Libp2p/Protocols/Gossipsub/Interop/Record.lean @@ -0,0 +1,56 @@ +import Lean + +/-! +# GossipSub interop NDJSON records + +This module implements specification section 5. It turns one event and one +RFC 3339 time string into one compact JSON object. The result has no trailing +line feed. The JSON encoder escapes quotes, backslashes, and control bytes, +so each result stays one NDJSON line. Object keys use the core tree-map +ordering. The harness reads named JSON fields, so their order carries no +meaning. The caller supplies an RFC 3339 timestamp. The received event takes +a UInt64 so its id always becomes an unsigned decimal string. Each call +emits one event, including when the caller passes a duplicate message. +-/ + +set_option autoImplicit false + +namespace Libp2p.Protocols.Gossipsub.Interop + +open Lean + +/-- Events required by the interop output contract. -/ +inductive RecordEvent where + | peerId (id : String) (nodeId : Int) + | received (id : UInt64) (sender : Option String) (topic : Option String) + deriving Repr + +/-- Build a JSON object from fields. Lean's encoder owns string escaping. -/ +def jsonObject (fields : List (String × Json)) : Json := + Json.obj (Std.TreeMap.Raw.ofList fields) + +/-- Encode one event without a trailing line feed. -/ +def recordLine (time : String) (event : RecordEvent) : String := + let base := [("time", Json.str time)] + let fields := match event with + | .peerId id nodeId => + base ++ [("msg", Json.str "PeerID"), ("id", Json.str id), + ("node_id", Json.num ⟨nodeId, 0⟩)] + | .received id sender topic => + let optional := (match sender with + | some value => [("from", Json.str value)] + | none => []) ++ (match topic with + | some value => [("topic", Json.str value)] + | none => []) + base ++ [("msg", Json.str "Received Message"), ("id", Json.str (Nat.repr id.toNat))] ++ optional + (jsonObject fields).compress + +/-- Encode the start event with a peer id and node number. -/ +def peerIdLine (time id : String) (nodeId : Int) : String := + recordLine time (.peerId id nodeId) + +/-- Encode one received message event. -/ +def receivedMessageLine (time : String) (id : UInt64) (sender topic : Option String) : String := + recordLine time (.received id sender topic) + +end Libp2p.Protocols.Gossipsub.Interop diff --git a/packages/libp2p/protocols/Libp2p/Protocols/Gossipsub/Interop/Schedule.lean b/packages/libp2p/protocols/Libp2p/Protocols/Gossipsub/Interop/Schedule.lean new file mode 100644 index 00000000..8c3cf122 --- /dev/null +++ b/packages/libp2p/protocols/Libp2p/Protocols/Gossipsub/Interop/Schedule.lean @@ -0,0 +1,111 @@ +import Libp2p.Protocols.Gossipsub.Interop.Script + +/-! +# GossipSub interop virtual clock + +The pure core needs a dry run over the captured 342 second scenario before +any IO driver exists. `plan` is that dry run: it selects one node's +instructions with `prepareScript`, projects each instruction to an `Action`, +and stamps every action with the virtual time a driver would reach it at. + +A `waitUntil` instruction is a barrier, not an action the node sleeps +through. The virtual clock jumps to the deadline and the actions behind the +wait inherit the new time. The IO driver keeps the same contract: it calls +`waitComplete` between its ticks while delivery and forwarding continue. + +`GossipSubParams` carries `Lean.Json` values, which have no `BEq` instance, +so `Action.beq` compares the parsed fields directly. Two scripts that parse +to the same fields therefore produce plans the test can compare for +determinism. +-/ + +set_option autoImplicit false + +namespace Libp2p.Protocols.Gossipsub.Interop + +/-- Virtual time in milliseconds, matching the gossipsub core's `Millis`. -/ +abbrev Millis := Nat + +/-- One instruction the selected node executes, without the node conditions. -/ +inductive Action where + | initGossipSub (params : GossipSubParams) + | connect (peers : List Int) + | subscribeToTopic (topic : String) (isPartial : Bool) + | publish (messageId size : Int) (topic : String) + | setTopicValidationDelay (topic : String) (delayMillis : Nat) + | waitUntil (deadlineSeconds : Int) + +/-- One action and the virtual time it executes at. -/ +structure ScheduledAction where + time : Millis + action : Action + +/-- Compare two parameter objects by their parsed fields, since `Json` has no `BEq`. -/ +private def sameParams : GossipSubParams → GossipSubParams → Bool + | left, right => left.raw == right.raw && left.unmapped == right.unmapped + && left.heartbeatInitialDelay == right.heartbeatInitialDelay + && left.heartbeatInterval == right.heartbeatInterval + && left.fanoutTtl == right.fanoutTtl && left.pruneBackoff == right.pruneBackoff + && left.historyLength == right.historyLength && left.historyGossip == right.historyGossip + && left.gossipFactor == right.gossipFactor && left.d == right.d && left.dLow == right.dLow + && left.dHigh == right.dHigh && left.dScore == right.dScore && left.dLazy == right.dLazy + +/-- Structural equality over actions, field by field. -/ +def Action.beq : Action → Action → Bool + | .initGossipSub left, .initGossipSub right => sameParams left right + | .connect left, .connect right => left == right + | .subscribeToTopic lTopic lPartial, .subscribeToTopic rTopic rPartial => + lTopic == rTopic && lPartial == rPartial + | .publish lId lSize lTopic, .publish rId rSize rTopic => + lId == rId && lSize == rSize && lTopic == rTopic + | .setTopicValidationDelay lTopic lDelay, .setTopicValidationDelay rTopic rDelay => + lTopic == rTopic && lDelay == rDelay + | .waitUntil left, .waitUntil right => left == right + | _, _ => false + +instance : BEq Action := ⟨Action.beq⟩ + +instance : BEq ScheduledAction where + beq left right := left.time == right.time && left.action == right.action + +/-- Flatten node conditions away and convert a validation delay to milliseconds. + +The delay arrives as a `Float` in seconds. The nearest whole millisecond, +floored toward zero, keeps `0.005` at exactly 5 ms. -/ +private def actionOf : Instruction → Action + | .initGossipSub params => .initGossipSub params + | .ifNodeIDEquals _ nested => actionOf nested + | .connect peers => .connect peers + | .subscribeToTopic topic isPartial => .subscribeToTopic topic isPartial + | .publish id size topic => .publish id size topic + | .setTopicValidationDelay topic delay => + .setTopicValidationDelay topic (Float.floor (delay * 1000.0)).toUInt64.toNat + | .waitUntil deadline => .waitUntil deadline + +/-- Stamp each instruction with the virtual time a driver reaches it at. + +The clock starts at zero. A `waitUntil` requests its deadline in milliseconds. +The plan keeps the clock monotonic when a later deadline is already past. A +nonpositive deadline also keeps the current time. -/ +private def expand (clock : Millis) : List Instruction → List ScheduledAction + | [] => [] + | instruction :: rest => + match instruction with + | .waitUntil deadline => + let requested := if deadline ≤ 0 then clock else deadline.toNat * 1000 + let next := max clock requested + { time := next, action := actionOf instruction } :: expand next rest + | _ => { time := clock, action := actionOf instruction } :: expand clock rest + +/-- Build the virtual-clock plan for one node. + +The plan starts with the startup `initGossipSub` at time zero, because the +harness states every node initializes before any other instruction. The +remaining actions keep script order. `waitComplete` is the guard the live +driver pairs with each `waitUntil` in this list. -/ +def plan (script : Script) (nodeId : Int) : Except String (List ScheduledAction) := do + let (params, selected) ← prepareScript nodeId script + let start := { time := 0, action := Action.initGossipSub params } + pure (start :: expand 0 selected) + +end Libp2p.Protocols.Gossipsub.Interop \ No newline at end of file diff --git a/packages/libp2p/protocols/Libp2p/Protocols/Gossipsub/Interop/Script.lean b/packages/libp2p/protocols/Libp2p/Protocols/Gossipsub/Interop/Script.lean new file mode 100644 index 00000000..ff37b1c4 --- /dev/null +++ b/packages/libp2p/protocols/Libp2p/Protocols/Gossipsub/Interop/Script.lean @@ -0,0 +1,345 @@ +import Lean +import Libp2p.Protocols.Gossipsub.Core +import Libp2p.Protocols.Gossipsub.Interop.MessageId + +/-! +# GossipSub interop script parser + +This module implements specification sections 2 and 3. It parses the script +object and all seven v1.1 instruction forms. It reports the instruction path +and field when decoding fails. The two v1.2 partial-message forms return an +explicit out-of-scope error. + +The parser uses a fuel argument for nested instructions. This gives a total +parser with a clear maximum nesting depth and avoids a partial definition. +Missing parameter fields stay as options and therefore select Core.defaults. +The raw field retains every source pair. The unmapped field retains fields +without a Core home, including HeartbeatInitialDelay. Exact decimal integer +division truncates nanoseconds toward zero: 100100000 ns becomes 100 ms. +GossipFactor scales a fraction by 100 and truncates to Core's integer percent. +The harness model declares a float; its JVM adapter defaults it to 0.25 +(ScriptInstruction.kt:89). Negative mapped numbers and invalid cache windows +return errors. The supported nesting depth is 256 instructions. An unknown parameter name +returns an error, so a future harness field stays visible instead of becoming +an unrecorded default. + +prepareScript selects conditional instructions for one node and requires its +initialization first. waitComplete compares elapsed seconds with a deadline. +The later IO driver must keep message processing active while that guard is false. +-/ + +set_option autoImplicit false + +namespace Libp2p.Protocols.Gossipsub.Interop + +open Lean +open Libp2p.Protocols.Gossipsub + +/-- One parsed GossipSub parameter object, with all supplied fields retained. -/ +structure GossipSubParams where + d : Option Nat + dLow : Option Nat + dHigh : Option Nat + dScore : Option Nat + dLazy : Option Nat + historyLength : Option Nat + historyGossip : Option Nat + gossipFactor : Option JsonNumber + heartbeatInitialDelay : Option JsonNumber + heartbeatInterval : Option JsonNumber + fanoutTtl : Option JsonNumber + pruneBackoff : Option JsonNumber + raw : List (String × Json) + unmapped : List (String × Json) + +/-- The seven supported script instruction forms. -/ +inductive Instruction where + | initGossipSub (params : GossipSubParams) + | ifNodeIDEquals (nodeId : Int) (instruction : Instruction) + | connect (connectTo : List Int) + | subscribeToTopic (topicId : String) (isPartial : Bool) + | publish (messageId : Int) (messageSizeBytes : Int) (topicId : String) + | setTopicValidationDelay (topicId : String) (delaySeconds : Float) + | waitUntil (elapsedSeconds : Int) + +/-- The complete script value. -/ +structure Script where + instructions : List Instruction + +/-- Convert a JSON value to object pairs while adding a readable path. -/ +def objectPairs (path : String) (value : Json) : Except String (List (String × Json)) := + match value.getObj? with + | .ok object => .ok object.toList + | .error _ => .error s!"{path}: expected object" + +/-- Find one object field without confusing a missing field with a bad type. -/ +def field? (path : String) (value : Json) (name : String) : Except String (Option Json) := + match objectPairs path value with + | .error error => .error error + | .ok pairs => .ok (pairs.find? (fun pair => pair.1 == name) |>.map (fun pair => pair.2)) + +/-- Read one required field. -/ +def requiredField (path : String) (value : Json) (name : String) : Except String Json := + match field? path value name with + | .error error => .error error + | .ok (some result) => .ok result + | .ok none => .error s!"{path}: missing field {name}" + +/-- Read one optional field. -/ +def optionalField (path : String) (value : Json) (name : String) : Except String (Option Json) := + field? path value name + +/-- Decode a required natural number. -/ +def requiredNat (path : String) (value : Json) (name : String) : Except String Nat := + let fieldPath := s!"{path}.{name}" + match requiredField path value name with + | .error error => .error error + | .ok field => + match field.getNat? with + | .ok result => .ok result + | .error _ => .error s!"{fieldPath}: expected integer" + +/-- Decode an optional natural number. -/ +def optionalNat (path : String) (value : Json) (name : String) : Except String (Option Nat) := + match optionalField path value name with + | .error error => .error error + | .ok none => .ok none + | .ok (some field) => + match field.getNat? with + | .ok result => .ok (some result) + | .error _ => .error s!"{path}.{name}: expected integer" + +/-- Decode a required signed integer. -/ +def requiredInt (path : String) (value : Json) (name : String) : Except String Int := + match requiredField path value name with + | .error error => .error error + | .ok field => + match field.getInt? with + | .ok result => .ok result + | .error _ => .error s!"{path}.{name}: expected integer" + +/-- Decode a required string. -/ +def requiredString (path : String) (value : Json) (name : String) : Except String String := + match requiredField path value name with + | .error error => .error error + | .ok field => + match field.getStr? with + | .ok result => .ok result + | .error _ => .error s!"{path}.{name}: expected string" + +/-- Decode a required boolean. -/ +def requiredBool (path : String) (value : Json) (name : String) : Except String Bool := + match requiredField path value name with + | .error error => .error error + | .ok field => + match field.getBool? with + | .ok result => .ok result + | .error _ => .error s!"{path}.{name}: expected boolean" + +/-- Decode a required JSON number as a Float. -/ +def requiredFloat (path : String) (value : Json) (name : String) : Except String Float := + match requiredField path value name with + | .error error => .error error + | .ok field => + match field.getNum? with + | .ok result => .ok result.toFloat + | .error _ => .error s!"{path}.{name}: expected number" + +/-- Decode a nonnegative decimal parameter without a binary float conversion. -/ +private def optionalNumber (path : String) (value : Json) (name : String) : + Except String (Option JsonNumber) := do + match ← optionalField path value name with + | none => return none + | some field => + let number ← field.getNum?.mapError (fun _ => s!"{path}.{name}: expected number") + if number.mantissa < 0 then + throw s!"{path}.{name}: expected nonnegative number" + return some number + +/-- Decode one parameter object and keep every source pair. -/ +def parseGossipSubParams (path : String) (value : Json) : Except String GossipSubParams := do + let pairs ← objectPairs path value + let getNat (name : String) := optionalNat path value name + let d ← getNat "D" + let dLow ← getNat "Dlo" + let dHigh ← getNat "Dhi" + let dScore ← getNat "Dscore" + let dLazy ← getNat "Dlazy" + let historyLength ← getNat "HistoryLength" + let historyGossip ← getNat "HistoryGossip" + let gossipFactor ← optionalNumber path value "GossipFactor" + let heartbeatInitialDelay ← optionalNumber path value "HeartbeatInitialDelay" + let heartbeatInterval ← optionalNumber path value "HeartbeatInterval" + let fanoutTtl ← optionalNumber path value "FanoutTTL" + let pruneBackoff ← optionalNumber path value "PruneBackoff" + let mapped := ["D", "Dlo", "Dhi", "Dscore", "Dlazy", "HistoryLength", "HistoryGossip", + "GossipFactor", "HeartbeatInterval", "FanoutTTL", "PruneBackoff"] + let retained := ["Dout", "GossipRetransmission", "HeartbeatInitialDelay", "SlowHeartbeatWarning", + "PrunePeers", "UnsubscribeBackoff", "Connectors", "MaxPendingConnections", "ConnectionTimeout", + "DirectConnectTicks", "DirectConnectInitialDelay", "OpportunisticGraftTicks", "OpportunisticGraftPeers", + "GraftFloodThreshold", "MaxIHaveLength", "MaxIHaveMessages", "MaxIDontWantLength", "MaxIDontWantMessages", + "IWantFollowupTime", "IDontWantMessageThreshold", "IDontWantMessageTTL"] + return { + d := d + dLow := dLow + dHigh := dHigh + dScore := dScore + dLazy := dLazy + historyLength := historyLength + historyGossip := historyGossip + gossipFactor := gossipFactor + heartbeatInitialDelay := heartbeatInitialDelay + heartbeatInterval := heartbeatInterval + fanoutTtl := fanoutTtl + pruneBackoff := pruneBackoff + raw := pairs + unmapped := ← pairs.filterMapM fun pair => + if mapped.contains pair.1 then pure none + else if retained.contains pair.1 then pure (some pair) + else throw s!"{path}: unknown parameter {pair.1}" + } + +/-- Convert harness nanoseconds to Core milliseconds by truncating. -/ +def nanosecondsToMillis (value : JsonNumber) : Nat := + value.mantissa.toNat / (10 ^ value.exponent * 1000000) + +/-- Convert the harness fraction to Core's integer percent field. -/ +def gossipFactorToPercent (value : JsonNumber) : Nat := + (value.mantissa.toNat * 100) / 10 ^ value.exponent + +/-- Build a cache window only when its required bound holds. -/ +def validWindow (historyLength historyGossip : Nat) : Option Cache.WindowParams := + if proof : 0 < historyGossip ∧ historyGossip ≤ historyLength then + some { historyLength := historyLength, historyGossip := historyGossip, valid := proof } + else + none + +/-- Apply parsed parameters to Core.defaults and install the interop id rule. -/ +def toCoreParams (params : GossipSubParams) : Except String Core.Params := do + let base := Core.defaults + let historyLength := params.historyLength.getD base.window.historyLength + let historyGossip := params.historyGossip.getD base.window.historyGossip + let window ← match validWindow historyLength historyGossip with + | some value => pure value + | none => throw "gossipSubParams.HistoryGossip: expected 0 < HistoryGossip ≤ HistoryLength" + return { base with + mesh := { base.mesh with + d := params.d.getD base.mesh.d + dLow := params.dLow.getD base.mesh.dLow + dHigh := params.dHigh.getD base.mesh.dHigh + dScore := params.dScore.getD base.mesh.dScore + dLazy := params.dLazy.getD base.mesh.dLazy } + heartbeatInterval := params.heartbeatInterval.map nanosecondsToMillis |>.getD base.heartbeatInterval + fanoutTtl := params.fanoutTtl.map nanosecondsToMillis |>.getD base.fanoutTtl + pruneBackoff := params.pruneBackoff.map nanosecondsToMillis |>.getD base.pruneBackoff + gossipFactor := params.gossipFactor.map gossipFactorToPercent |>.getD base.gossipFactor + window := window + msgId := msgIdFn } + +/-- Decode a required integer list with indexed field errors. -/ +def parseIntList (path : String) (values : List Json) (index : Nat) : Except String (List Int) := + match values with + | [] => .ok [] + | value :: rest => + match value.getInt? with + | .error _ => .error s!"{path}[{index}]: expected integer" + | .ok result => + match parseIntList path rest (index + 1) with + | .error error => .error error + | .ok tail => .ok (result :: tail) + +/-- Decode one instruction with bounded recursive fuel. -/ +def parseInstructionFuel : Nat → String → Json → Except String Instruction + | 0, path, _ => .error s!"{path}: nesting limit exceeded" + | fuel + 1, path, value => do + let kind ← requiredString path value "type" + match kind with + | "initGossipSub" => + let params ← requiredField path value "gossipSubParams" + .initGossipSub <$> parseGossipSubParams s!"{path}.gossipSubParams" params + | "ifNodeIDEquals" => + let nodeId ← requiredInt path value "nodeID" + let nested ← requiredField path value "instruction" + let instruction ← parseInstructionFuel fuel s!"{path}.instruction" nested + .ok (.ifNodeIDEquals nodeId instruction) + | "connect" => + let field ← requiredField path value "connectTo" + match field.getArr? with + | .error _ => .error s!"{path}.connectTo: expected array" + | .ok values => .connect <$> parseIntList s!"{path}.connectTo" values.toList 0 + | "subscribeToTopic" => + let topicId ← requiredString path value "topicID" + let isPartial ← match optionalField path value "partial" with + | .error error => .error error + | .ok none => .ok false + | .ok (some field) => match field.getBool? with + | .ok result => .ok result + | .error _ => .error s!"{path}.partial: expected boolean" + .ok (.subscribeToTopic topicId isPartial) + | "publish" => + let messageId ← requiredInt path value "messageID" + let messageSizeBytes ← requiredInt path value "messageSizeBytes" + let topicId ← requiredString path value "topicID" + .ok (.publish messageId messageSizeBytes topicId) + | "setTopicValidationDelay" => + let topicId ← requiredString path value "topicID" + let delaySeconds ← requiredFloat path value "delaySeconds" + .ok (.setTopicValidationDelay topicId delaySeconds) + | "waitUntil" => + .waitUntil <$> requiredInt path value "elapsedSeconds" + | "addPartialMessage" => + .error s!"{path}: addPartialMessage is out of scope for v1.1" + | "publishPartial" | "publishToNodeIDs" => + .error s!"{path}: {kind} is out of scope for v1.1" + | _ => .error s!"{path}.type: unknown instruction type {kind}" + +/-- Decode one instruction with the module's bounded nesting depth. -/ +def parseInstruction (path : String) (value : Json) : Except String Instruction := + parseInstructionFuel 256 path value + +/-- Decode a JSON script object. -/ +def parseScriptJson (value : Json) : Except String Script := do + let field ← requiredField "root" value "script" + let values ← match field.getArr? with + | .ok result => .ok result.toList + | .error _ => .error "root.script: expected array" + let rec loop (index : Nat) (remaining : List Json) : Except String (List Instruction) := + match remaining with + | [] => .ok [] + | item :: rest => do + let instruction ← parseInstruction s!"script[{index}]" item + let tail ← loop (index + 1) rest + return instruction :: tail + return { instructions := ← loop 0 values } + +/-- Parse a complete params.json string. -/ +def parseScript (source : String) : Except String Script := do + let value ← Json.parse source + parseScriptJson value + +/-- Resolve a chain of node conditions without changing instruction order. -/ +def instructionForNode (nodeId : Int) : Instruction → Option Instruction + | .ifNodeIDEquals expected nested => + if nodeId == expected then instructionForNode nodeId nested else none + | instruction => some instruction + +/-- Separate startup parameters from the remaining instructions for one node. -/ +def prepareScript (nodeId : Int) (script : Script) : + Except String (GossipSubParams × List Instruction) := do + let selected := script.instructions.filterMap (instructionForNode nodeId) + match selected with + | .initGossipSub params :: rest => + if rest.any (fun instruction => match instruction with + | .initGossipSub _ => true + | _ => false) then + throw "script: initGossipSub must occur only at startup" + let _ ← toCoreParams params + return (params, rest) + | _ => throw "script: initGossipSub must precede every other instruction" + +/-- A wait releases the next instruction once elapsed seconds reach its deadline. +The IO driver continues delivery and forwarding between calls to this guard. -/ +def waitComplete (elapsedSeconds deadlineSeconds : Int) : Bool := + decide (elapsedSeconds ≥ deadlineSeconds) + +end Libp2p.Protocols.Gossipsub.Interop diff --git a/packages/libp2p/protocols/Libp2p/Protocols/Gossipsub/Node.lean b/packages/libp2p/protocols/Libp2p/Protocols/Gossipsub/Node.lean new file mode 100644 index 00000000..8a55fcb2 --- /dev/null +++ b/packages/libp2p/protocols/Libp2p/Protocols/Gossipsub/Node.lean @@ -0,0 +1,601 @@ +import Conc +import Streams +import Bytes +import Libp2p.Protocols.Constants +import Libp2p.Core +import Libp2p.Protocols.StreamError +import Libp2p.Protocols.Gossipsub.HostSeam +import Libp2p.Protocols.Gossipsub.Core + +/-! +# Gossipsub node: the loop over the router core + +The core of this unit is the pure `Core.step`. This module holds the one +task that drives it, and the publish and subscribe surface at the seam +level (ARCHITECTURE, "Modules"; root ARCHITECTURE §3.3). + +## The loop + +One task owns the router state. A bounded channel carries application and +host events and the wakes of Conc.Alarm. The alarm has its own task. The +loop uses repeat and parks only on the channel receive. Ready events use +constant stack space. The loop asks the alarm for the next due time after +each input. A TestClock drives the alarm in tests. + +The inbound reader uses repeat with a mutable header, body cursor and chunk +position. Each bounded body allocation follows its decoded length. Each +input byte enters the body once, so small reads use constant stack space +and linear copying. + +## Ownership + +The task set of the node owns the loop, alarm and writer tasks. Stop joins +them. The host owns the handler tasks of the inbound streams. Nothing cancels a task: +`stop` closes the event queue, and the close wakes the parked loop. The +router state and pending gossip are local to the loop. Shared registries, +subscriptions and the closing flag change through OnceRef.update or +updateGet (root ARCHITECTURE §4.1). Only the first stop caller joins. + +## Recorded decisions + +- **The stop field runs in Async.** Joining parks, and TaskSet.joinAll runs + in Async for that reason. So the field keeps that type and leaves the + blocking run to the application edge. +- **Actions never park.** Each send offers a frame to a peer queue with 32 + slots. One writer task owns that peer's stream. A full queue drops the + frame, as go does for a full peer queue. Node.dropped reports these drops. + Actions return the next pending list, and the loop assigns that answer. +- **Delayed gossip has a bound.** At most 256 sends wait in the pending list. + Overflow drops the newest excess sends and adds to Node.dropped. Due sends + leave before the tick shifts the message cache. The delay index increases + across the run. Config.order hashes the heartbeat count with peer bytes. +- **Subscribe registers its callback before the post.** The loop can serve a + delivery between the post and the registration, and that delivery would + miss the callback. A refused post takes back its own registration by id. + Concurrent callers remove their own entries through atomic updates. +- **The handler announces its opener.** It posts peerJoined before reading + frames, so the first RPC reaches a known peer. Repeat joins are no-ops. +- **No signer.** The core validates with `strictNoSign` (`Core.validationOk`), + which requires `from` and `seqno` to be absent. So `publish` signs nothing + and never writes the peer id of the node. +- **A local publish delivers nothing locally.** `Core.publishLocal` emits no + deliver action, because the application already holds its own message. + The callback of the publisher stays silent. +- **The loop announces a subscription change.** `Core.join` sends the topics + only when a peer joins. The core emits nothing for a subscribe after the + join, so that peer would never learn the topic. The go implementation + announces a change in its host registry, outside the router. The loop + does the same with subOptRpc, before the core step. +- **The default gossip delay is zero.** The pinned go router enqueues IHAVEs + in emitGossip (gossipsub.go, lines 2007 to 2076). Its flush sends them at + once (lines 2078 to 2092). The random sleep in announceRetry belongs to a + subscription retry (pubsub.go, lines 1234 to 1235). The loop owns the + emission schedule, so a caller can spread heartbeat gossip across the + interval. The default spreads nothing and matches go. Tests configure an + explicit spread because issue #23 names the trickle regressions. The delay + applies only to heartbeat IHAVEs. Publish, forward, graft and hello sends + leave at once. +- **Each peer has one writer task.** The pinned go pubsub version a01777894f4c + resets duplicate inbound streams (comm.go, lines 55 to 65). Its sender uses + one stream and a bounded queue per peer (lines 181 to 233). Our writer + writes frames in queue order. A failed write retires its outbound entry + and closes its queue. Transport notifications own peerLeft. Stop resets + installed streams to wake parked writes and joins + the writer tasks. The host must end a newStream call when its session ends. +- **The inbound handler serves every frame of a stream.** The pubsub spec + uses separate long-lived inbound and outbound streams (pubsub/README.md, + lines 78 to 83). Go reads messages in a loop (comm.go, lines 75 to 120). + Rust keeps one inbound substream (protocols/gossipsub/src/handler.rs, + lines 78 to 82). Both peers write many RPCs on one stream. The handler + preserves frame order and checks each length before appending body bytes. + A bad prefix, an over-cap frame or EOF inside a frame is a protocol error. + A zero-length frame is legal (Rpc.lean, line 867). Go skips it; this handler + decodes it as an empty RPC that the core accepts. +- **The receipt observer runs before the core step.** The interop harness + logs every received message, including duplicates (gossipsub-interop, + test-specs/implementation.md, lines 71 to 78). Its go participant logs + messages on RECV_RPC (go-libp2p/tracer.go, lines 75 to 88). Core.handlePubs + deduplicates before delivery (Core.lean, lines 603 and 614). Config.onRpc + exposes the sender and unframed body before that step. The observer can + decode the body to log every message. The default hook decodes nothing. + Frame errors fail the handler before they reach the observer. +- **A send failure retires its outbound entry.** The next send starts a fresh + writer and stream. The core keeps the peer and accepts further RPCs on its + live inbound stream. Retirement checks the writer id, so an old writer + cannot remove a newer entry for the same peer. +-/ + +set_option autoImplicit false + +namespace Libp2p.Protocols.Gossipsub.Node + + +/-- A spread a caller may choose for heartbeat gossip. Each delay stays below +one second. The multiplier is prime, so the delays do not repeat with a short +period. Config.trickle defaults to zero. A caller who needs gossip inside +each heartbeat interval sets heartbeatInterval above the largest delay. -/ +def spread (i : Nat) : Millis := (i * 7919 + 1) % 1000 + +/-- Replayable peer order for a heartbeat. The hash combines its counter +with each peer's identity bytes. The loop supplies only known peers. -/ +def defaultOrder (tick : Nat) (peers : List Peer) : List Peer := + let key (peer : Peer) : UInt64 := mixHash (hash tick) (hash (Bytes.toNatList peer.toBytes)) + peers.mergeSort (fun a b => key a ≤ key b) + +/-- The configuration of one node: the router parameters, the clock the +loop reads, and the spread of the heartbeat gossip. -/ +structure Config where + /-- The router parameters. -/ + params : Core.Params := Core.defaults + /-- The clock of the loop's timer. -/ + clock : Conc.Clock := Conc.Clock.system + /-- The delay of the run-wide `i`-th delayed gossip send, in milliseconds. + The caller fixes it, so a test gets exact times. -/ + trickle : Nat → Millis := fun _ => 0 + /-- Replayable peer order for the numbered heartbeat. -/ + order : Nat → List Peer → List Peer := defaultOrder + /-- Called for every inbound RPC with its sender and its bytes, before the + core sees them. A harness reads it to log each received message, including + a duplicate. It runs on the loop task, so it must not park. -/ + onRpc : Peer → List Nat → BaseIO Unit := fun _ _ => pure () + /-- Called for every outbound RPC before it enters the peer queue. -/ + onSend : Peer → Rpc.RPC → BaseIO Unit := fun _ _ => pure () + +/-- A running node. Every field is a seam over the loop task, which is the +only owner of the router state. -/ +structure Node where + /-- Publish the payload on the topic. `false` when the queue of the loop + is full or the node stopped. -/ + publish : TopicId → ByteArray → BaseIO Bool + /-- Register a delivery callback. The callback runs on the loop task, so + it must not park. `false` under the same conditions as `publish`, and the + callback then stays unregistered. -/ + subscribe : TopicId → (Message → BaseIO Unit) → BaseIO Bool + /-- Drop every callback of the topic, and leave the topic. `false` under the + same conditions as `publish`. -/ + unsubscribe : TopicId → BaseIO Bool + /-- Tell the loop that a peer joined. The connection notifiee of the host + is the caller at integration. -/ + peerJoined : Peer → Core.PubsubProto → BaseIO Bool + /-- Tell the loop that a peer left. -/ + peerLeft : Peer → BaseIO Bool + /-- Close the event queue and join the loop task. The loop serves the + queued events first, and drops the gossip that waits for its time. This + field runs in `Async` because a join parks, and a parked `BaseIO` action would + hold a thread. A second call does nothing. -/ + stop : Conc.Async Unit + /-- Frames dropped by a full or closed peer queue, or by the 256-send + pending-gossip bound. -/ + dropped : BaseIO Nat + +/-! ## One outbound RPC -/ + +/-- True when the control message of the RPC holds an IHAVE. The loop +delays these sends. `Core.rpcHasIhave` is a `Prop`, and the loop needs a `Bool`. -/ +def hasIhave (r : Rpc.RPC) : Bool := + match r.control with + | some c => !c.ihave.isEmpty + | none => false + +/-- One subscription record: `true` subscribes, and `false` unsubscribes. +`Core.helloRpc` holds only positive records, so the loop builds this record +beside the code that sends it. -/ +def subOptRpc (t : TopicId) (on : Bool) : Rpc.RPC := + { subscriptions := [{ subscribe := on, topicid := t.name }], publish := [], control := none } + +/-- The one input queue carries core events and alarm wakes. -/ +inductive Input where + | event (e : Core.Event) + | wake (now : Millis) + +/-- One peer's bounded writer queue and its currently open stream. -/ +structure Outbound where + /-- The writer generation used for exact retirement. -/ + id : Nat + /-- At most 32 encoded frames wait for this peer. -/ + queue : Conc.Chan ByteArray + /-- Stop resets this stream to wake a parked write. -/ + stream : Conc.OnceRef (Option Streams.Stream) + +/-- Close a queue and reset its stream to wake its writer. -/ +private def endOutbound (ob : Outbound) : Conc.Async Unit := do + (ob.queue.close : BaseIO Unit) + if let some stream ← (ob.stream.get : BaseIO (Option Streams.Stream)) then + stream.reset + +/-- One writer owns the peer stream. It writes queued frames in order and +closes the stream at the end. A failed open or write retires only this writer +entry and closes its queue. The core peer stays connected. Stop resets an +installed stream; a late open checks closing too. -/ +private def writer (host : HostSeam) (closing : Conc.OnceRef Bool) + (outbound : Conc.OnceRef (List (Peer × Outbound))) (peer : Peer) (ob : Outbound) : Conc.Async Unit := do + let fail : Conc.Async Unit := do + (ob.queue.close : BaseIO Unit) + (outbound.update (fun table => table.filter (fun e => e.2.id != ob.id)) : BaseIO Unit) + if let some stream ← (ob.stream.get : BaseIO (Option Streams.Stream)) then stream.reset + try + match ← host.newStream peer [Constants.meshsubId] with + | .error _ => fail + | .ok (_, stream) => + (ob.stream.set (some stream) : BaseIO Unit) + if ← (closing.get : BaseIO Bool) then stream.reset + repeat + match ← ob.queue.recv with + | none => break + | some frame => + if let .error _ ← stream.write frame then + fail + break + stream.close + catch _ => fail + +/-! ## The inbound handler -/ + +/-- Read consecutive frames with a bounded header and one body allocation. +The header is at most Varint.maxBytes bytes. Its decoded length is capped +before emptyWithCapacity allocates the body. This is the pinned Lean name +for the capacity constructor. Each copySlice writes only the next slice at +cursor, without retaining an earlier body value. EOF between frames is clean. +The repeat loop uses constant stack space for any number of chunks or frames. -/ +private def readFrames (events : Conc.Chan Input) (peer : Peer) + (stream : Streams.Stream) : Conc.Async (Except Libp2p.Core.Error Unit) := do + let mut header := ByteArray.emptyWithCapacity Varint.maxBytes + let mut body := ByteArray.empty + let mut expected : Option Nat := none + let mut cursor := 0 + let mut chunk := ByteArray.empty + let mut pos := 0 + repeat + if let some size := expected then + if cursor = size then + -- Park: the loop frees a slot, or stop closes the event queue. + if let .error _ ← events.send (.event (.rpc peer (Bytes.toNatList body))) then + return .ok () + body := ByteArray.empty + expected := none + cursor := 0 + if pos = chunk.size then + -- Park: the peer writes, half-closes, or resets. + match ← stream.read with + | .error e => return .error (mapStreamError e) + | .ok none => + return if header.isEmpty && expected.isNone then .ok () + else .error (.protocol "the stream ended inside an RPC frame") + | .ok (some bytes) => + chunk := bytes + pos := 0 + else + match expected with + | some size => + let count := Nat.min (size - cursor) (chunk.size - pos) + body := chunk.copySlice pos body cursor count + cursor := cursor + count + pos := pos + count + | none => + if Varint.maxBytes ≤ header.size then + return .error (.protocol "the RPC header is malformed") + header := header.push (chunk.get! pos) + pos := pos + 1 + match Streams.Framing.decodeLength? .uvarint (Bytes.toNatList header) with + | .error .truncated => pure () + | .error .malformed => return .error (.protocol "the RPC header is malformed") + | .ok (size, _) => + if Constants.rpcCap < size then + return .error (.protocol "the RPC exceeds the cap") + -- The allocation runs only after the length passes the cap. + body := ByteArray.emptyWithCapacity size + expected := some size + cursor := 0 + header := ByteArray.emptyWithCapacity Varint.maxBytes + return .ok () + +/-- Serve every RPC of a peer's stream in arrival order. Frame errors reset +both directions. A clean EOF closes our write direction. The host owns and +joins this handler task. -/ +def handler (events : Conc.Chan Input) : Libp2p.Core.StreamHandler where + id := Constants.meshsubId + handle ctx stream := ExceptT.mk do + let peer := ctx.remotePeer + -- The opener joins before its first RPC. Repeat joins are loop no-ops. + if let .error _ ← events.send (.event (.peerJoined peer .meshsub)) then + stream.close + return .ok () + let answer ← readFrames events peer stream + match answer with + | .ok () => stream.close + | .error _ => stream.reset + return answer + +/-! ## The subscription table -/ + +/-- One delivery callback. -/ +private structure Subscription where + /-- The unique registration id used by a refused post. -/ + id : Nat + /-- The topic. -/ + topic : TopicId + /-- The callback. -/ + callback : Message → BaseIO Unit + +/-- The shared table of the callbacks, in registration order. -/ +private structure Subscriptions where + /-- The next unused registration id. -/ + next : Nat + /-- The callbacks. -/ + entries : List Subscription + +/-- Call every callback of the topic of `m`, in registration order. -/ +private def deliver (subs : Conc.OnceRef Subscriptions) (m : Message) : BaseIO Unit := do + for sub in (← subs.get).entries do + if sub.topic = m.topic then + sub.callback m + +/-! ## The loop -/ + +/-- What the loop reads and never changes. -/ +private structure Env where + /-- The host seam. -/ + host : HostSeam + /-- The configuration. -/ + config : Config + /-- The subscription table. -/ + subs : Conc.OnceRef Subscriptions + /-- The event queue. -/ + events : Conc.Chan Input + /-- The owned alarm task posts wakes on events. -/ + alarm : Conc.Alarm + /-- The owner of the loop, alarm and writers. -/ + tasks : Conc.TaskSet + /-- The peer writer registry. The loop inserts and writers retire by id. -/ + outbound : Conc.OnceRef (List (Peer × Outbound)) + /-- The next writer generation, allocated atomically. -/ + nextOutbound : Conc.OnceRef Nat + /-- The first stop caller sets this flag. -/ + closing : Conc.OnceRef Bool + /-- Frames refused by a full or closed outbound queue. -/ + drops : Conc.OnceRef Nat + +/-- Encode and offer one RPC to a peer's queue. This operation never parks. +A full queue drops its frame. Each peer has one writer and one stream. -/ +private def enqueue (env : Env) (peer : Peer) (rpc : Rpc.RPC) : BaseIO Unit := do + if ← env.closing.get then return () + (env.config.onSend peer rpc : BaseIO Unit) + let some frame := Rpc.encodeFrame? rpc | return () + let table ← env.outbound.get + let ob ← match table.find? (fun e => e.1 == peer) with + | some (_, ob) => pure ob + | none => do + let id ← env.nextOutbound.updateGet (fun n => (n, n + 1)) + let ob : Outbound := { id, queue := ← Conc.Chan.new 32, stream := ← Conc.OnceRef.new none } + env.outbound.update (fun table => (peer, ob) :: table) + env.tasks.spawn (writer env.host env.closing env.outbound peer ob) + pure ob + if ← env.closing.get then + ob.queue.close + return () + if !(← ob.queue.trySend (Bytes.ofNatList frame)) then env.drops.update (· + 1) + +/-- Retire a peer writer after its disconnect event. -/ +private def removePeer (env : Env) (peer : Peer) : BaseIO Unit := do + let removed ← env.outbound.updateGet fun table => + (table.filter (fun e => e.1 == peer), table.filter (fun e => e.1 != peer)) + for (_, ob) in removed do + ob.queue.close + env.tasks.spawn (endOutbound ob) + +/-- One gossip send that waits for its due time. -/ +private structure Pending where + /-- The due time. -/ + due : Millis + /-- The receiver. -/ + peer : Peer + /-- The RPC. -/ + rpc : Rpc.RPC + +/-- The local state of the loop. -/ +private structure LoopState where + /-- The router state. -/ + router : Core.State + /-- The gossip sends that wait for their due time. -/ + pending : List Pending + /-- The time of the next heartbeat. -/ + nextTick : Millis + /-- The heartbeat count used for the peer order. -/ + ticks : Nat + /-- The next run-wide delayed gossip index. -/ + gossipIndex : Nat + +/-- Run the actions of one core step, in order, and answer the delayed +sends. `gossipAt` is the time of the tick when a tick made the actions. The +run-wide i-th IHAVE waits for trickle i milliseconds. Every other action +runs now. -/ +private def perform (env : Env) (gossipAt : Option Millis) (index : Nat) + (actions : List Core.Action) : BaseIO (List Pending × Nat) := do + let mut delayed : Array Pending := #[] + let mut gossipIndex := index + for action in actions do + match action with + | .deliver m => (deliver env.subs m : BaseIO Unit) + | .sendRpc to rpc => + match gossipAt with + | some now => + if hasIhave rpc then + let delay := env.config.trickle gossipIndex + gossipIndex := gossipIndex + 1 + if delay = 0 then + enqueue env to rpc + else + delayed := delayed.push { due := now + delay, peer := to, rpc := rpc } + else + enqueue env to rpc + | none => enqueue env to rpc + return (delayed.toList, gossipIndex) + +/-- Tell peers about a subscription change, or tell a new peer about all +current subscriptions. Each offer uses the bounded per-peer queue. -/ +private def announce (env : Env) (st : LoopState) (event : Core.Event) : + BaseIO LoopState := do + match event with + | .subscribe t => + let rpc := subOptRpc t true + for (peer, _) in st.router.peers do + enqueue env peer rpc + | .unsubscribe t => + let rpc := subOptRpc t false + for (peer, _) in st.router.peers do + enqueue env peer rpc + | .peerJoined peer _ => + if !st.router.topics.isEmpty then + enqueue env peer (Core.helloRpc st.router.topics) + | _ => pure () + return st + +/-- Drop the core hello when the node already sent per-topic join records. -/ +private def withoutPeerHello (peer : Peer) (topics : List TopicId) : + List Core.Action → List Core.Action + | [] => [] + | action :: rest => + match action with + | .sendRpc target rpc => + if !topics.isEmpty && target == peer && rpc == Core.helloRpc topics then + withoutPeerHello peer topics rest + else action :: withoutPeerHello peer topics rest + | _ => action :: withoutPeerHello peer topics rest + +/-- One core step, then its actions. Only an RPC that does not decode makes +the core answer `.error`. That answer keeps the state, and the loop runs on. -/ +private def stepAndAct (env : Env) (st : LoopState) (event : Core.Event) : BaseIO LoopState := do + if let .peerJoined peer _ := event then + if (Core.peerOf st.router peer).isSome then return st + let st ← announce env st event + if let .rpc peer bytes := event then + (env.config.onRpc peer bytes : BaseIO Unit) + let st ← match event with + | .peerLeft peer => do + removePeer env peer + pure { st with pending := st.pending.filter (fun p => p.peer != peer) } + | _ => pure st + match Core.step env.config.params st.router event with + | .error (.decode _) => return st + | .ok (router, actions) => + let actions := match event with + | .peerJoined peer _ => withoutPeerHello peer st.router.topics actions + | _ => actions + let gossipAt := match event with + | .tick now _ => some now + | _ => none + let (delayed, gossipIndex) ← perform env gossipAt st.gossipIndex actions + let all := st.pending ++ delayed + let overflow := all.length - 256 + if 0 < overflow then env.drops.update (· + overflow) + return { st with router, gossipIndex, pending := all.take 256 } + +/-- The next wake is the first due send or the next heartbeat. -/ +private def dueOf (st : LoopState) : Millis := + st.pending.foldl (fun due p => Nat.min due p.due) st.nextTick + +/-- Send due gossip before the heartbeat shifts the message cache. -/ +private def onWake (env : Env) (st : LoopState) (now : Millis) : BaseIO LoopState := do + let (due, later) := st.pending.partition (fun p => decide (p.due ≤ now)) + for p in due.mergeSort (fun a b => decide (a.due ≤ b.due)) do + enqueue env p.peer p.rpc + let st := { st with pending := later } + if st.nextTick ≤ now then + let order := env.config.order st.ticks (st.router.peers.map (·.1)) + let st ← stepAndAct env st (.tick now order) + return { st with nextTick := now + env.config.params.heartbeatInterval, ticks := st.ticks + 1 } + return st + +/-- Run the core through one receive park. Each action returns its state. +The closed channel serves queued inputs before it returns none. The tail +closes the alarm and every cached outbound stream. -/ +private def loop (env : Env) (st0 : LoopState) : Conc.Async Unit := do + let mut st := st0 + (env.alarm.set (dueOf st) : BaseIO Unit) + repeat + match ← env.events.recv with + | none => break + | some (.wake now) => st ← (onWake env st now : BaseIO LoopState) + | some (.event e) => + st ← (stepAndAct env st e : BaseIO LoopState) + (env.alarm.set (dueOf st) : BaseIO Unit) + (env.alarm.close : BaseIO Unit) + for (_, ob) in ← (env.outbound.get : BaseIO (List (Peer × Outbound))) do + (ob.queue.close : BaseIO Unit) + +/-- The router state of a node with no peers at time `now`. -/ +private def initialState (p : Core.Params) (now : Millis) : Core.State := + { peers := [] + mesh := [] + fanout := [] + mcache := Cache.MessageCache.empty p.window + seen := Cache.SeenCache.empty p.seen + backoff := [] + iwant := [] + topics := [] + now := now + lastHeartbeat := now } + +/-- The published message of a payload. The four signed fields stay absent, +as `strictNoSign` requires. -/ +private def messageOf (topic : TopicId) (payload : ByteArray) : Message := + { «from» := none + data := some (Bytes.toNatList payload) + seqno := none + topic := topic + signature := none + key := none } + +/-- Build the state, register the handler, and start the loop. The event +queue holds 64 events. A full queue parks an inbound handler, which carries +the back-pressure to the peer, and it refuses a `BaseIO` post of the +application. -/ +def start (host : HostSeam) (config : Config := {}) : BaseIO Node := do + let now ← config.clock.nowMs + let subs : Conc.OnceRef Subscriptions ← Conc.OnceRef.new { next := 0, entries := [] } + let events : Conc.Chan Input ← Conc.Chan.new 64 + host.setHandler (handler events) + let tasks ← Conc.TaskSet.new + let alarm ← Conc.Alarm.new + let outbound ← Conc.OnceRef.new ([] : List (Peer × Outbound)) + let nextOutbound ← Conc.OnceRef.new 0 + let closing ← Conc.OnceRef.new false + let drops ← Conc.OnceRef.new 0 + tasks.spawn <| alarm.run config.clock (fun now => events.send (.wake now)) + tasks.spawn <| loop { host, config, subs, events, alarm, tasks, outbound, nextOutbound, closing, drops } + { router := initialState config.params now + pending := [] + nextTick := now + config.params.heartbeatInterval + ticks := 0 + gossipIndex := 0 } + let post : Core.Event → BaseIO Bool := fun e => do + if ← closing.get then return false + events.trySend (.event e) + return { + publish := fun topic payload => post (.publish (messageOf topic payload)) + subscribe := fun topic callback => do + if ← closing.get then return false + let id ← subs.updateGet fun s => + (s.next, + { next := s.next + 1 + entries := s.entries ++ [{ id := s.next, topic, callback }] }) + let sent ← post (.subscribe topic) + if !sent then + subs.update fun s => { s with entries := s.entries.filter (fun entry => entry.id != id) } + return sent + unsubscribe := fun topic => do + if ← closing.get then return false + subs.update fun s => { s with entries := s.entries.filter (fun e => e.topic != topic) } + post (.unsubscribe topic) + peerJoined := fun peer proto => post (.peerJoined peer proto) + peerLeft := fun peer => post (.peerLeft peer) + dropped := drops.get + stop := do + let first ← (closing.updateGet (fun was => (!was, true)) : BaseIO Bool) + if first then + (events.close : BaseIO Unit) + for (_, ob) in ← (outbound.get : BaseIO (List (Peer × Outbound))) do + endOutbound ob + tasks.joinAll } + +end Libp2p.Protocols.Gossipsub.Node diff --git a/packages/libp2p/protocols/Libp2p/Protocols/Gossipsub/Rpc.lean b/packages/libp2p/protocols/Libp2p/Protocols/Gossipsub/Rpc.lean new file mode 100644 index 00000000..76df1f43 --- /dev/null +++ b/packages/libp2p/protocols/Libp2p/Protocols/Gossipsub/Rpc.lean @@ -0,0 +1,944 @@ +import Proto +import Bytes +import Streams +import Libp2p.Protocols.Constants +import Libp2p.Protocols.Gossipsub.Types + +/-! +# The gossipsub RPC + +The wire messages of gossipsub v1.1, their `ProtoMessage` instances, and +the length-delimited frame step. + +The field numbers come from the vendored specs: `vendor/specs/pubsub/README.md` +for `RPC`, `SubOpts`, and `Message`, `gossipsub-v1.0.md` for the control +envelope and its four messages, and `gossipsub-v1.1.md` for the prune +peers and the backoff. + +Proto has no nested-message value. A nested message goes on the wire as a +`.bytes` entry that holds its encoding, and the decoder decodes that +encoding as the submessage. + +The table is strict on singular fields: a present field with the wrong +wire type, a present string with invalid UTF-8, and a bool varint other +than 0 or 1 are `Pb.Error.badField`. The table is permissive on repeated +fields: an entry of another wire type is skipped. Unknown fields stay in +the wire view and drop out of the value on decode. + +The validity premise of every table law is recursive. A child whose own +entries are invalid has an empty encoding, so a parent entry that holds +that empty encoding is valid. The parent law therefore states the +validity of its children as well. +-/ + +set_option autoImplicit false + +namespace Libp2p.Protocols.Gossipsub + +/-! ## Shared readers + +These readers are the house pattern of `Identify`, plus a strict bool +reader and an optional varint reader. Each one gates on `Pb.hasField`, +so a present field of another wire type is `badField` rather than a +miss. -/ + +/-- A present singular payload must be length-delimited. A miss means +absent. -/ +private def bytesField? (num : Nat) (fs : Pb.Fields) : Except Pb.Error (Option (List Nat)) := + if Pb.hasField num fs then + match Pb.getBytes? num fs with + | some bs => .ok (some bs) + | none => .error .badField + else .ok none + +/-- A present singular string must be length-delimited and valid UTF-8. +A miss means absent. -/ +private def strField? (num : Nat) (fs : Pb.Fields) : Except Pb.Error (Option String) := + if Pb.hasField num fs then + match (Pb.getBytes? num fs).bind (fun bs => Bytes.toString? (Bytes.ofNatList bs)) with + | some s => .ok (some s) + | none => .error .badField + else .ok none + +/-- A strict bool. A miss and varint 0 are `false`, varint 1 is `true`, +and every other present value is `badField`. -/ +private def boolField? (num : Nat) (fs : Pb.Fields) : Except Pb.Error Bool := + if Pb.hasField num fs then + match Pb.getVarint? num fs with + | some 0 => .ok false + | some 1 => .ok true + | _ => .error .badField + else .ok false + +/-- An optional varint. A miss is `none`, and a present value of another +wire type is `badField`. -/ +private def varintField? (num : Nat) (fs : Pb.Fields) : Except Pb.Error (Option Nat) := + if Pb.hasField num fs then + match Pb.getVarint? num fs with + | some n => .ok (some n) + | none => .error .badField + else .ok none + +/-- A required string. A miss is `badField`. -/ +private def requiredStr? (num : Nat) (fs : Pb.Fields) : Except Pb.Error String := do + match ← strField? num fs with + | some s => .ok s + | none => .error .badField + +/-- A singular nested field. A present field of another wire type is +`badField`, and a malformed nested payload keeps its wire error. -/ +private def subField? {α : Type} (num : Nat) (fs : Pb.Fields) + (ofFields? : Pb.Fields → Except Pb.Error α) : + Except Pb.Error (Option α) := + if Pb.hasField num fs then + match Pb.getSub? num fs with + | .error e => .error e + | .ok none => .error .badField + | .ok (some nested) => do + let a ← ofFields? nested + .ok (some a) + else .ok none + +/-- Decode a list of encoded children in wire order. A child that fails +keeps its wire error. -/ +private def decodeAll? {α : Type} [Pb.ProtoMessage α] : + List (List Nat) → Except Pb.Error (List α) + | [] => .ok [] + | bs :: rest => do + let p ← Pb.decodeMessage? (α := α) bs + let as ← decodeAll? rest + .ok (p.1 :: as) + +/-- The length-delimited payloads of one field number, in wire order. An +entry of another wire type is skipped. -/ +private def allBytesOf (num : Nat) : List Pb.Entry → List (List Nat) + | [] => [] + | (n, v) :: es => + (if n = num then + match v with + | .bytes bs => [bs] + | _ => [] + else []) ++ allBytesOf num es + +/-- The repeated nested reader. It takes every length-delimited payload +of the field number, in wire order, and decodes each one. -/ +private def subAll? {α : Type} [Pb.ProtoMessage α] (num : Nat) (fs : Pb.Fields) : + Except Pb.Error (List α) := + decodeAll? (allBytesOf num fs.entries) + +/-- The wire view of a value, checked. Fail closed: a list whose entries +do not fit the wire model gives the empty field table. -/ +private def fieldsOf? (es : List Pb.Entry) : Pb.Fields := + if h : ∀ e ∈ es, e.Valid then ⟨es, h⟩ else Pb.Fields.empty + +/-! ## List lemmas -/ + +/-- A miss lookup through a segment that maps payloads to another field +number. -/ +private theorem lookupMissMap {α : Type} (k n : Nat) (h : k ≠ n) (g : α → Pb.Value) + (l : List α) : + List.lookup k (l.map fun x => (n, g x)).reverse = none := by + induction l with + | nil => rfl + | cons x l ih => + have hb : (k == n) = false := by simpa using h + simp [List.lookup_append, ih, List.lookup, hb] + +/-- The payload reader splits over an append. -/ +private theorem allBytesOfAppend (num : Nat) (a b : List Pb.Entry) : + allBytesOf num (a ++ b) = allBytesOf num a ++ allBytesOf num b := by + induction a with + | nil => simp [allBytesOf] + | cons e a ih => + obtain ⟨n, v⟩ := e + simp [allBytesOf, ih, List.append_assoc] + +/-- A segment of payload entries of one field number keeps exactly those +payloads, in order. -/ +private theorem allBytesOfMapBytes {α : Type} (num k : Nat) (g : α → List Nat) + (l : List α) : + allBytesOf num (l.map fun x => (k, Pb.Value.bytes (g x))) = + if k = num then l.map g else [] := by + induction l with + | nil => simp [allBytesOf] + | cons x l ih => + simp only [List.map_cons, allBytesOf, ih] + split <;> simp_all + +/-- A segment of entries of another field number is empty. -/ +private theorem allBytesOfMapOther {α : Type} (num k : Nat) (h : k ≠ num) (g : α → Pb.Value) + (l : List α) : + allBytesOf num (l.map fun x => (k, g x)) = [] := by + induction l with + | nil => simp [allBytesOf] + | cons x l ih => simp [allBytesOf, h, ih] + +/-- One entry of another field number is empty. -/ +private theorem allBytesOfSingleOther {num k : Nat} (h : k ≠ num) (v : Pb.Value) : + allBytesOf num [(k, v)] = [] := by + simp [allBytesOf, h] + +/-- The child round trips compose over a list of encoded children. -/ +private theorem decodeAll?Encode {α : Type} [Pb.ProtoMessage α] (l : List α) + (h : ∀ a ∈ l, Pb.decodeMessage? (α := α) (Pb.encodeMessage a) = .ok (a, [])) : + decodeAll? (l.map (fun a => Pb.encodeMessage a)) = .ok l := by + induction l with + | nil => rfl + | cons a l ih => + have ha := h a (by simp) + have hl : ∀ b ∈ l, Pb.decodeMessage? (α := α) (Pb.encodeMessage b) = .ok (b, []) := + fun b hb => h b (by simp [hb]) + simp [decodeAll?, ha, ih hl, Bind.bind, Except.bind] + +/-- A `MessageId` is its payload, so a list of ids survives the wrapper. -/ +private theorem mapMessageIdMk (l : List MessageId) : + (l.map (fun m => m.bytes)).map (fun bs => (⟨bs⟩ : MessageId)) = l := by + induction l with + | nil => rfl + | cons m l ih => cases m; simp [ih] + +/-- The same reading with the two maps composed. -/ +private theorem mapMessageIdMkComp (l : List MessageId) : + l.map (fun m => (⟨m.bytes⟩ : MessageId)) = l := by + induction l with + | nil => rfl + | cons m l ih => cases m; simp [ih] + +/-- The repeated payload reader for message ids. -/ +private def messageIdsOf (num : Nat) (fs : Pb.Fields) : List MessageId := + (allBytesOf num fs.entries).map (fun bs => ⟨bs⟩) + +/-! ## The `Message` codec + +`Message` is the shared type of `Gossipsub.Types`. Its codec lives here, +in the parent namespace, so the type stays free of codec code. -/ + +namespace Message + +/-- The wire view: one entry per present field, in the ascending order +1, 2, 3, 4, 5, 6. The topic is required and always present. -/ +def entries (m : Message) : List Pb.Entry := + m.«from».toList.map (fun b => (1, Pb.Value.bytes b)) + ++ m.data.toList.map (fun b => (2, Pb.Value.bytes b)) + ++ m.seqno.toList.map (fun b => (3, Pb.Value.bytes b)) + ++ [(4, Pb.Value.bytes (Bytes.toNatList (Bytes.ofString m.topic.name)))] + ++ m.signature.toList.map (fun b => (5, Pb.Value.bytes b)) + ++ m.key.toList.map (fun b => (6, Pb.Value.bytes b)) + +/-- The wire validity of the entries. -/ +def entriesValid (m : Message) : Prop := ∀ e ∈ m.entries, e.Valid + +instance (m : Message) : Decidable (entriesValid m) := by + unfold entriesValid + infer_instance + +/-- The validity of the message. It is the validity of its entries. -/ +abbrev Valid (m : Message) : Prop := m.entriesValid + +/-- Fail closed. A message whose entries do not fit the wire model emits +no entry; every theorem about the encode takes the validity as a +premise. -/ +def toFields (m : Message) : Pb.Fields := + if h : m.entriesValid then ⟨m.entries, h⟩ else Pb.Fields.empty + +/-- The field table: strict singular fields, permissive repeated fields. +The required topic reads first, so a missing topic fails before the +other fields run. -/ +def ofFields? (fs : Pb.Fields) : Except Pb.Error Message := do + let topic ← requiredStr? 4 fs + let frm ← bytesField? 1 fs + let data ← bytesField? 2 fs + let seqno ← bytesField? 3 fs + let signature ← bytesField? 5 fs + let key ← bytesField? 6 fs + .ok { «from» := frm, data, seqno, topic := ⟨topic⟩, signature, key } + +instance : Pb.ProtoMessage Message where + toFields := toFields + ofFields? := ofFields? + +/-- The table law: a valid message round-trips through its own table. -/ +theorem ofFieldsToFields (m : Message) (h : Valid m) : + Pb.ProtoMessage.ofFields? (α := Message) (Pb.ProtoMessage.toFields m) = .ok m := by + change ofFields? (toFields m) = .ok m + have he : m.entriesValid := h + rw [toFields, dite_eq_left he] + obtain ⟨frm, data, seqno, topic, signature, key⟩ := m + obtain ⟨name⟩ := topic + cases frm <;> cases data <;> cases seqno <;> cases signature <;> cases key <;> + simp [ofFields?, requiredStr?, bytesField?, strField?, entries, Pb.hasField, Pb.getBytes?, + Pb.lookupLast, List.lookup_cons, Bind.bind, Except.bind, Bytes.ofNatListToNatList, + Bytes.toString?OfString] + +/-- An encode decodes back to the message. -/ +theorem decodeEncode (m : Message) (h : Valid m) : + Pb.decodeMessage? (Pb.encodeMessage m) = .ok (m, []) := by + simp only [Pb.decodeMessage?, Pb.encodeMessage, Pb.decodeEncode, ofFieldsToFields m h, + Bind.bind, Except.bind] + +/-- A `Message` without field 4 fails its table. -/ +theorem ofFields?MissingTopic (fs : Pb.Fields) (h : ¬ Pb.hasField 4 fs) : + ofFields? fs = .error .badField := by + have hb : Pb.hasField 4 fs = false := by + cases hm : Pb.hasField 4 fs <;> simp_all + simp [ofFields?, requiredStr?, strField?, hb, Bind.bind, Except.bind] + +end Message + +/-! ## The control messages and the RPC envelope -/ + +namespace Rpc + +/-- One subscription option. Field 1 is the bool, field 2 the topic. -/ +structure SubOpts where + /-- True subscribes, false unsubscribes. -/ + subscribe : Bool -- field 1, bool + /-- The topic name. -/ + topicid : String -- field 2, string + deriving DecidableEq, BEq, Repr + +namespace SubOpts + +/-- The wire view: field 1 as a varint, field 2 as the topic bytes. -/ +def entries (s : SubOpts) : List Pb.Entry := + [(1, .varint (if s.subscribe then 1 else 0)), + (2, .bytes (Bytes.toNatList (Bytes.ofString s.topicid)))] + +def entriesValid (s : SubOpts) : Prop := ∀ e ∈ s.entries, e.Valid + +instance (s : SubOpts) : Decidable (entriesValid s) := by + unfold entriesValid + infer_instance + +/-- The validity of the value. -/ +abbrev Valid (s : SubOpts) : Prop := s.entriesValid + +def toFields (s : SubOpts) : Pb.Fields := + fieldsOf? s.entries + +/-- The field table. A miss reads as false or the empty string. -/ +def ofFields? (fs : Pb.Fields) : Except Pb.Error SubOpts := do + let subscribe ← boolField? 1 fs + let topicid ← strField? 2 fs + .ok { subscribe, topicid := topicid.getD "" } + +instance : Pb.ProtoMessage SubOpts where + toFields := toFields + ofFields? := ofFields? + +/-- The table law. -/ +theorem ofFieldsToFields (s : SubOpts) (h : Valid s) : + Pb.ProtoMessage.ofFields? (α := SubOpts) (Pb.ProtoMessage.toFields s) = .ok s := by + change ofFields? (toFields s) = .ok s + have he : ∀ e ∈ s.entries, e.Valid := h + rw [toFields, fieldsOf?, dite_eq_left he] + obtain ⟨sub, tid⟩ := s + cases sub <;> simp [ofFields?, boolField?, strField?, entries, Pb.hasField, Pb.getBytes?, + Pb.getVarint?, Pb.lookupLast, List.lookup_cons, Bind.bind, Except.bind, + Bytes.ofNatListToNatList, Bytes.toString?OfString] + +/-- An encode decodes back to the value. -/ +theorem decodeEncode (s : SubOpts) (h : Valid s) : + Pb.decodeMessage? (Pb.encodeMessage s) = .ok (s, []) := by + simp only [Pb.decodeMessage?, Pb.encodeMessage, Pb.decodeEncode, ofFieldsToFields s h, + Bind.bind, Except.bind] + +end SubOpts + +/-- A peer record of the v1.1 peer exchange. -/ +structure PeerInfo where + /-- The peer id bytes. -/ + peerId : Option (List Nat) -- field 1, bytes + /-- The signed peer record bytes. -/ + signedPeerRecord : Option (List Nat) -- field 2, bytes + deriving DecidableEq, BEq, Repr + +namespace PeerInfo + +def entries (p : PeerInfo) : List Pb.Entry := + p.peerId.toList.map (fun b => (1, Pb.Value.bytes b)) + ++ p.signedPeerRecord.toList.map (fun b => (2, Pb.Value.bytes b)) + +def entriesValid (p : PeerInfo) : Prop := ∀ e ∈ p.entries, e.Valid + +instance (p : PeerInfo) : Decidable (entriesValid p) := by + unfold entriesValid + infer_instance + +/-- The validity of the value. -/ +abbrev Valid (p : PeerInfo) : Prop := p.entriesValid + +def toFields (p : PeerInfo) : Pb.Fields := + fieldsOf? p.entries + +/-- The field table. A miss reads as an absent payload. -/ +def ofFields? (fs : Pb.Fields) : Except Pb.Error PeerInfo := do + let peerId ← bytesField? 1 fs + let signedPeerRecord ← bytesField? 2 fs + .ok { peerId, signedPeerRecord } + +instance : Pb.ProtoMessage PeerInfo where + toFields := toFields + ofFields? := ofFields? + +/-- The table law. -/ +theorem ofFieldsToFields (p : PeerInfo) (h : Valid p) : + Pb.ProtoMessage.ofFields? (α := PeerInfo) (Pb.ProtoMessage.toFields p) = .ok p := by + change ofFields? (toFields p) = .ok p + have he : ∀ e ∈ p.entries, e.Valid := h + rw [toFields, fieldsOf?, dite_eq_left he] + obtain ⟨pid, spr⟩ := p + cases pid <;> cases spr <;> + simp [ofFields?, bytesField?, entries, Pb.hasField, Pb.getBytes?, Pb.lookupLast, + List.lookup_cons, Bind.bind, Except.bind] + +/-- An encode decodes back to the value. -/ +theorem decodeEncode (p : PeerInfo) (h : Valid p) : + Pb.decodeMessage? (Pb.encodeMessage p) = .ok (p, []) := by + simp only [Pb.decodeMessage?, Pb.encodeMessage, Pb.decodeEncode, ofFieldsToFields p h, + Bind.bind, Except.bind] + +end PeerInfo + +/-- An IHAVE control message. -/ +structure ControlIHave where + /-- The topic of the gossip. -/ + topicId : String -- field 1, string + /-- The ids of the messages on offer. -/ + messageIds : List MessageId -- field 2, repeated bytes + deriving DecidableEq, BEq, Repr + +namespace ControlIHave + +def entries (i : ControlIHave) : List Pb.Entry := + [(1, .bytes (Bytes.toNatList (Bytes.ofString i.topicId)))] + ++ i.messageIds.map (fun m => (2, Pb.Value.bytes m.bytes)) + +def entriesValid (i : ControlIHave) : Prop := ∀ e ∈ i.entries, e.Valid + +instance (i : ControlIHave) : Decidable (entriesValid i) := by + unfold entriesValid + infer_instance + +/-- The validity of the value. -/ +abbrev Valid (i : ControlIHave) : Prop := i.entriesValid + +def toFields (i : ControlIHave) : Pb.Fields := + fieldsOf? i.entries + +/-- The field table. -/ +def ofFields? (fs : Pb.Fields) : Except Pb.Error ControlIHave := do + let topicId ← strField? 1 fs + .ok { topicId := topicId.getD "", messageIds := messageIdsOf 2 fs } + +instance : Pb.ProtoMessage ControlIHave where + toFields := toFields + ofFields? := ofFields? + +/-- The table law. -/ +theorem ofFieldsToFields (i : ControlIHave) (h : Valid i) : + Pb.ProtoMessage.ofFields? (α := ControlIHave) (Pb.ProtoMessage.toFields i) = .ok i := by + change ofFields? (toFields i) = .ok i + have he : ∀ e ∈ i.entries, e.Valid := h + rw [toFields, fieldsOf?, dite_eq_left he] + obtain ⟨tid, ids⟩ := i + have hb : allBytesOf 2 ((1, Pb.Value.bytes (Bytes.toNatList (Bytes.ofString tid))) :: + ids.map (fun m => (2, Pb.Value.bytes m.bytes))) = ids.map (fun m => m.bytes) := by + simp [allBytesOf, allBytesOfMapBytes] + simp [ofFields?, strField?, entries, messageIdsOf, hb, mapMessageIdMkComp, Function.comp_def, + Pb.hasField, Pb.getBytes?, Pb.lookupLast, List.lookup_append, Bind.bind, Except.bind, + Bytes.ofNatListToNatList, Bytes.toString?OfString, lookupMissMap] + +/-- An encode decodes back to the value. -/ +theorem decodeEncode (i : ControlIHave) (h : Valid i) : + Pb.decodeMessage? (Pb.encodeMessage i) = .ok (i, []) := by + simp only [Pb.decodeMessage?, Pb.encodeMessage, Pb.decodeEncode, ofFieldsToFields i h, + Bind.bind, Except.bind] + +end ControlIHave + +/-- An IWANT control message. -/ +structure ControlIWant where + /-- The ids of the wanted messages. -/ + messageIds : List MessageId -- field 1, repeated bytes + deriving DecidableEq, BEq, Repr + +namespace ControlIWant + +def entries (i : ControlIWant) : List Pb.Entry := + i.messageIds.map (fun m => (1, Pb.Value.bytes m.bytes)) + +def entriesValid (i : ControlIWant) : Prop := ∀ e ∈ i.entries, e.Valid + +instance (i : ControlIWant) : Decidable (entriesValid i) := by + unfold entriesValid + infer_instance + +/-- The validity of the value. -/ +abbrev Valid (i : ControlIWant) : Prop := i.entriesValid + +def toFields (i : ControlIWant) : Pb.Fields := + fieldsOf? i.entries + +/-- The field table. -/ +def ofFields? (fs : Pb.Fields) : Except Pb.Error ControlIWant := + .ok { messageIds := messageIdsOf 1 fs } + +instance : Pb.ProtoMessage ControlIWant where + toFields := toFields + ofFields? := ofFields? + +/-- The table law. -/ +theorem ofFieldsToFields (i : ControlIWant) (h : Valid i) : + Pb.ProtoMessage.ofFields? (α := ControlIWant) (Pb.ProtoMessage.toFields i) = .ok i := by + change ofFields? (toFields i) = .ok i + have he : ∀ e ∈ i.entries, e.Valid := h + rw [toFields, fieldsOf?, dite_eq_left he] + obtain ⟨ids⟩ := i + simp [ofFields?, entries, messageIdsOf, allBytesOfMapBytes, mapMessageIdMkComp, Function.comp_def] + +/-- An encode decodes back to the value. -/ +theorem decodeEncode (i : ControlIWant) (h : Valid i) : + Pb.decodeMessage? (Pb.encodeMessage i) = .ok (i, []) := by + simp only [Pb.decodeMessage?, Pb.encodeMessage, Pb.decodeEncode, ofFieldsToFields i h, + Bind.bind, Except.bind] + +end ControlIWant + +/-- A GRAFT control message. -/ +structure ControlGraft where + /-- The topic to graft. -/ + topicId : String -- field 1, string + deriving DecidableEq, BEq, Repr + +namespace ControlGraft + +def entries (g : ControlGraft) : List Pb.Entry := + [(1, .bytes (Bytes.toNatList (Bytes.ofString g.topicId)))] + +def entriesValid (g : ControlGraft) : Prop := ∀ e ∈ g.entries, e.Valid + +instance (g : ControlGraft) : Decidable (entriesValid g) := by + unfold entriesValid + infer_instance + +/-- The validity of the value. -/ +abbrev Valid (g : ControlGraft) : Prop := g.entriesValid + +def toFields (g : ControlGraft) : Pb.Fields := + fieldsOf? g.entries + +/-- The field table. -/ +def ofFields? (fs : Pb.Fields) : Except Pb.Error ControlGraft := do + let topicId ← strField? 1 fs + .ok { topicId := topicId.getD "" } + +instance : Pb.ProtoMessage ControlGraft where + toFields := toFields + ofFields? := ofFields? + +/-- The table law. -/ +theorem ofFieldsToFields (g : ControlGraft) (h : Valid g) : + Pb.ProtoMessage.ofFields? (α := ControlGraft) (Pb.ProtoMessage.toFields g) = .ok g := by + change ofFields? (toFields g) = .ok g + have he : ∀ e ∈ g.entries, e.Valid := h + rw [toFields, fieldsOf?, dite_eq_left he] + obtain ⟨tid⟩ := g + simp [ofFields?, strField?, entries, Pb.hasField, Pb.getBytes?, Pb.lookupLast, + Bind.bind, Except.bind, Bytes.ofNatListToNatList, Bytes.toString?OfString] + +/-- An encode decodes back to the value. -/ +theorem decodeEncode (g : ControlGraft) (h : Valid g) : + Pb.decodeMessage? (Pb.encodeMessage g) = .ok (g, []) := by + simp only [Pb.decodeMessage?, Pb.encodeMessage, Pb.decodeEncode, ofFieldsToFields g h, + Bind.bind, Except.bind] + +end ControlGraft + +/-- A PRUNE control message of gossipsub v1.1. -/ +structure ControlPrune where + /-- The topic to prune. -/ + topicId : String -- field 1, string + /-- The peers of the peer exchange. -/ + peers : List PeerInfo -- field 2, repeated nested + /-- The backoff in seconds. -/ + backoff : Option Nat -- field 3, uint64 varint + deriving DecidableEq, BEq, Repr + +namespace ControlPrune + +def entries (p : ControlPrune) : List Pb.Entry := + [(1, .bytes (Bytes.toNatList (Bytes.ofString p.topicId)))] + ++ p.peers.map (fun q => (2, Pb.Value.bytes (Pb.encodeMessage q))) + ++ p.backoff.toList.map (fun n => (3, Pb.Value.varint n)) + +def entriesValid (p : ControlPrune) : Prop := ∀ e ∈ p.entries, e.Valid + +instance (p : ControlPrune) : Decidable (entriesValid p) := by + unfold entriesValid + infer_instance + +/-- The validity of the value: its entries, and every peer. -/ +abbrev Valid (p : ControlPrune) : Prop := + p.entriesValid ∧ ∀ q ∈ p.peers, q.Valid + +def toFields (p : ControlPrune) : Pb.Fields := + fieldsOf? p.entries + +/-- The field table. The peers decode as nested messages. -/ +def ofFields? (fs : Pb.Fields) : Except Pb.Error ControlPrune := do + let topicId ← strField? 1 fs + let peers ← subAll? (α := PeerInfo) 2 fs + let backoff ← varintField? 3 fs + .ok { topicId := topicId.getD "", peers, backoff } + +instance : Pb.ProtoMessage ControlPrune where + toFields := toFields + ofFields? := ofFields? + +/-- The table law. -/ +theorem ofFieldsToFields (p : ControlPrune) (h : Valid p) : + Pb.ProtoMessage.ofFields? (α := ControlPrune) (Pb.ProtoMessage.toFields p) = .ok p := by + change ofFields? (toFields p) = .ok p + obtain ⟨he, hp⟩ := h + have he' : ∀ e ∈ p.entries, e.Valid := he + rw [toFields, fieldsOf?, dite_eq_left he'] + obtain ⟨tid, peers, backoff⟩ := p + have hdec := decodeAll?Encode peers (fun q hq => PeerInfo.decodeEncode q (hp q hq)) + cases backoff with + | none => + have hb : allBytesOf 2 ((1, Pb.Value.bytes (Bytes.toNatList (Bytes.ofString tid))) :: + peers.map (fun q => (2, Pb.Value.bytes (Pb.encodeMessage q)))) = + peers.map (fun q => Pb.encodeMessage q) := by + simp [allBytesOf, allBytesOfMapBytes] + simp [ofFields?, strField?, varintField?, subAll?, hdec, hb, entries, Pb.hasField, + Pb.getBytes?, Pb.lookupLast, List.lookup_append, lookupMissMap, Bind.bind, + Except.bind, Bytes.ofNatListToNatList, Bytes.toString?OfString] + | some n => + have hb : allBytesOf 2 ((1, Pb.Value.bytes (Bytes.toNatList (Bytes.ofString tid))) :: + (peers.map (fun q => (2, Pb.Value.bytes (Pb.encodeMessage q))) ++ + [(3, Pb.Value.varint n)])) = + peers.map (fun q => Pb.encodeMessage q) := by + simp [allBytesOf, allBytesOfAppend, allBytesOfMapBytes] + simp [ofFields?, strField?, varintField?, subAll?, hdec, hb, entries, Pb.hasField, + Pb.getBytes?, Pb.getVarint?, Pb.lookupLast, List.lookup_append, List.lookup_cons, + List.reverse_append, lookupMissMap, Bind.bind, Except.bind, + Bytes.ofNatListToNatList, Bytes.toString?OfString] + +/-- An encode decodes back to the value. -/ +theorem decodeEncode (p : ControlPrune) (h : Valid p) : + Pb.decodeMessage? (Pb.encodeMessage p) = .ok (p, []) := by + simp only [Pb.decodeMessage?, Pb.encodeMessage, Pb.decodeEncode, ofFieldsToFields p h, + Bind.bind, Except.bind] + +end ControlPrune + +/-- The control envelope. Field 5 (`idontwant`, v1.2) is out of scope. -/ +structure ControlMessage where + /-- The IHAVE messages. -/ + ihave : List ControlIHave -- field 1, repeated nested + /-- The IWANT messages. -/ + iwant : List ControlIWant -- field 2, repeated nested + /-- The GRAFT messages. -/ + graft : List ControlGraft -- field 3, repeated nested + /-- The PRUNE messages. -/ + prune : List ControlPrune -- field 4, repeated nested + deriving DecidableEq, BEq, Repr + +namespace ControlMessage + +def entries (c : ControlMessage) : List Pb.Entry := + c.ihave.map (fun x => (1, Pb.Value.bytes (Pb.encodeMessage x))) + ++ c.iwant.map (fun x => (2, Pb.Value.bytes (Pb.encodeMessage x))) + ++ c.graft.map (fun x => (3, Pb.Value.bytes (Pb.encodeMessage x))) + ++ c.prune.map (fun x => (4, Pb.Value.bytes (Pb.encodeMessage x))) + +def entriesValid (c : ControlMessage) : Prop := ∀ e ∈ c.entries, e.Valid + +instance (c : ControlMessage) : Decidable (entriesValid c) := by + unfold entriesValid + infer_instance + +/-- The validity of the value: its entries and every child. -/ +abbrev Valid (c : ControlMessage) : Prop := + c.entriesValid ∧ + (∀ x ∈ c.ihave, x.Valid) ∧ + (∀ x ∈ c.iwant, x.Valid) ∧ + (∀ x ∈ c.graft, x.Valid) ∧ + (∀ x ∈ c.prune, x.Valid) + +def toFields (c : ControlMessage) : Pb.Fields := + fieldsOf? c.entries + +/-- The field table. Each of the four lists decodes as nested messages. -/ +def ofFields? (fs : Pb.Fields) : Except Pb.Error ControlMessage := do + let ihave ← subAll? (α := ControlIHave) 1 fs + let iwant ← subAll? (α := ControlIWant) 2 fs + let graft ← subAll? (α := ControlGraft) 3 fs + let prune ← subAll? (α := ControlPrune) 4 fs + .ok { ihave, iwant, graft, prune } + +instance : Pb.ProtoMessage ControlMessage where + toFields := toFields + ofFields? := ofFields? + +/-- The table law. -/ +theorem ofFieldsToFields (c : ControlMessage) (h : Valid c) : + Pb.ProtoMessage.ofFields? (α := ControlMessage) (Pb.ProtoMessage.toFields c) = .ok c := by + change ofFields? (toFields c) = .ok c + obtain ⟨he, hi, hw, hg, hr⟩ := h + have he' : ∀ e ∈ c.entries, e.Valid := he + rw [toFields, fieldsOf?, dite_eq_left he'] + obtain ⟨ih, iw, gr, pr⟩ := c + have h1 := decodeAll?Encode ih (fun x hx => ControlIHave.decodeEncode x (hi x hx)) + have h2 := decodeAll?Encode iw (fun x hx => ControlIWant.decodeEncode x (hw x hx)) + have h3 := decodeAll?Encode gr (fun x hx => ControlGraft.decodeEncode x (hg x hx)) + have h4 := decodeAll?Encode pr (fun x hx => ControlPrune.decodeEncode x (hr x hx)) + simp [ofFields?, subAll?, h1, h2, h3, h4, entries, allBytesOfAppend, + allBytesOfMapBytes, Bind.bind, Except.bind] + +/-- An encode decodes back to the value. -/ +theorem decodeEncode (c : ControlMessage) (h : Valid c) : + Pb.decodeMessage? (Pb.encodeMessage c) = .ok (c, []) := by + simp only [Pb.decodeMessage?, Pb.encodeMessage, Pb.decodeEncode, ofFieldsToFields c h, + Bind.bind, Except.bind] + +end ControlMessage + +/-- The control value of an optional control field. The `Option` shape +keeps the predicate decidable over a bounded value. -/ +def controlValid : Option ControlMessage → Prop + | none => True + | some c => c.Valid + +instance (o : Option ControlMessage) : Decidable (controlValid o) := by + cases o with + | none => exact isTrue trivial + | some c => + rw [show controlValid (some c) = c.Valid from rfl] + infer_instance + +/-- The RPC envelope. -/ +structure RPC where + /-- The subscriptions. -/ + subscriptions : List SubOpts -- field 1, repeated nested + /-- The published messages. -/ + publish : List Message -- field 2, repeated nested + /-- The control envelope. -/ + control : Option ControlMessage -- field 3, optional nested + deriving DecidableEq, BEq, Repr + +namespace RPC + +def entries (r : RPC) : List Pb.Entry := + r.subscriptions.map (fun s => (1, Pb.Value.bytes (Pb.encodeMessage s))) + ++ r.publish.map (fun m => (2, Pb.Value.bytes (Pb.encodeMessage m))) + ++ r.control.toList.map (fun c => (3, Pb.Value.bytes (Pb.encodeMessage c))) + +def entriesValid (r : RPC) : Prop := ∀ e ∈ r.entries, e.Valid + +instance (r : RPC) : Decidable (entriesValid r) := by + unfold entriesValid + infer_instance + +/-- The validity of the value: its entries, every child, and the control +value. -/ +abbrev Valid (r : RPC) : Prop := + r.entriesValid ∧ + (∀ s ∈ r.subscriptions, s.Valid) ∧ + (∀ m ∈ r.publish, m.Valid) ∧ + controlValid r.control + +instance (r : RPC) : Decidable (Valid r) := by + unfold Valid + cases r.control with + | none => + rw [show controlValid none = True from rfl] + infer_instance + | some c => + rw [show controlValid (some c) = c.Valid from rfl] + infer_instance + +def toFields (r : RPC) : Pb.Fields := + fieldsOf? r.entries + +/-- The field table. The subscriptions and the messages decode as nested +messages, and the control field as one nested message. -/ +def ofFields? (fs : Pb.Fields) : Except Pb.Error RPC := do + let subscriptions ← subAll? (α := SubOpts) 1 fs + let publish ← subAll? (α := Message) 2 fs + let control ← subField? 3 fs ControlMessage.ofFields? + .ok { subscriptions, publish, control } + +instance : Pb.ProtoMessage RPC where + toFields := toFields + ofFields? := ofFields? + +/-- The table law. -/ +theorem ofFieldsToFields (r : RPC) (h : Valid r) : + Pb.ProtoMessage.ofFields? (α := RPC) (Pb.ProtoMessage.toFields r) = .ok r := by + change ofFields? (toFields r) = .ok r + obtain ⟨he, hsub, hpub, hctl⟩ := h + have he' : ∀ e ∈ r.entries, e.Valid := he + rw [toFields, fieldsOf?, dite_eq_left he'] + obtain ⟨subs, pubs, ctl⟩ := r + have h1 := decodeAll?Encode subs (fun s hmem => SubOpts.decodeEncode s (hsub s hmem)) + have h2 := decodeAll?Encode pubs (fun m hmem => Message.decodeEncode m (hpub m hmem)) + cases ctl with + | none => + simp [ofFields?, subAll?, subField?, h1, h2, entries, allBytesOfAppend, + allBytesOfMapBytes, Pb.hasField, Bind.bind, Except.bind] + | some c => + have hcv : c.Valid := hctl + have h4 : ControlMessage.ofFields? c.toFields = .ok c := + ControlMessage.ofFieldsToFields c hcv + have he₂ : ∀ e : Pb.Entry, + e ∈ (List.map (fun s => (1, Pb.Value.bytes (Pb.encodeMessage s))) subs ++ + (List.map (fun m => (2, Pb.Value.bytes (Pb.encodeMessage m))) pubs ++ + [(3, Pb.Value.bytes (Pb.encodeMessage c))])) → e.Valid := by + intro e hmem + exact he' e (by simpa [entries] using hmem) + have hsub3 : Pb.getSub? 3 ⟨List.map (fun s => (1, Pb.Value.bytes (Pb.encodeMessage s))) subs ++ + (List.map (fun m => (2, Pb.Value.bytes (Pb.encodeMessage m))) pubs ++ + [(3, Pb.Value.bytes (Pb.encodeMessage c))]), he₂⟩ = + .ok (some (ControlMessage.toFields c)) := by + simp [Pb.getSub?, Pb.getBytes?, Pb.lookupLast, Pb.encodeMessage, Pb.decodeEncode, + Bind.bind, Except.bind] + rfl + simp [ofFields?, subAll?, subField?, h1, h2, h4, hsub3, entries, allBytesOfAppend, + allBytesOfMapBytes, allBytesOfSingleOther, Pb.hasField, Bind.bind, Except.bind] + +/-- An encode decodes back to the value. -/ +theorem decodeEncode (r : RPC) (h : Valid r) : + Pb.decodeMessage? (Pb.encodeMessage r) = .ok (r, []) := by + simp only [Pb.decodeMessage?, Pb.encodeMessage, Pb.decodeEncode, ofFieldsToFields r h, + Bind.bind, Except.bind] + +/-- Equal encodings imply equal messages. Without the premise the +statement is false: two different invalid messages both encode to the +empty list. -/ +theorem encodeInjective (a b : RPC) (ha : Valid a) (hb : Valid b) + (h : Pb.encodeMessage a = Pb.encodeMessage b) : a = b := by + have hf : toFields a = toFields b := Pb.encodeInjective h + have h3 : Pb.ProtoMessage.ofFields? (α := RPC) (Pb.ProtoMessage.toFields a) = + Pb.ProtoMessage.ofFields? (α := RPC) (Pb.ProtoMessage.toFields b) := + congrArg (fun fs => Pb.ProtoMessage.ofFields? (α := RPC) fs) hf + rw [ofFieldsToFields a ha, ofFieldsToFields b hb] at h3 + exact Except.ok.inj h3 + +end RPC + +/-- The repeated fields keep their order through the round trip. -/ +theorem roundTripLists (r : RPC) (h : RPC.Valid r) : + ∃ r' : RPC, Pb.decodeMessage? (Pb.encodeMessage r) = .ok (r', []) ∧ + r'.subscriptions = r.subscriptions ∧ r'.publish = r.publish := + ⟨r, RPC.decodeEncode r h, rfl, rfl⟩ + +/-- The four control lists keep their order through the round trip. -/ +theorem roundTripControlLists (c : ControlMessage) (h : ControlMessage.Valid c) : + ∃ c' : ControlMessage, Pb.decodeMessage? (Pb.encodeMessage c) = .ok (c', []) ∧ + c'.ihave = c.ihave ∧ c'.iwant = c.iwant ∧ c'.graft = c.graft ∧ c'.prune = c.prune := + ⟨c, ControlMessage.decodeEncode c h, rfl, rfl, rfl, rfl⟩ + +/-- The message ids of an IHAVE keep their order through the round +trip. -/ +theorem roundTripMessageIdsIHave (i : ControlIHave) (h : ControlIHave.Valid i) : + ∃ i' : ControlIHave, Pb.decodeMessage? (Pb.encodeMessage i) = .ok (i', []) ∧ + i'.messageIds = i.messageIds := + ⟨i, ControlIHave.decodeEncode i h, rfl⟩ + +/-- The message ids of an IWANT keep their order through the round +trip. -/ +theorem roundTripMessageIdsIWant (i : ControlIWant) (h : ControlIWant.Valid i) : + ∃ i' : ControlIWant, Pb.decodeMessage? (Pb.encodeMessage i) = .ok (i', []) ∧ + i'.messageIds = i.messageIds := + ⟨i, ControlIWant.decodeEncode i h, rfl⟩ + +/-- The peers of a PRUNE keep their order through the round trip. -/ +theorem roundTripPeers (p : ControlPrune) (h : ControlPrune.Valid p) : + ∃ p' : ControlPrune, Pb.decodeMessage? (Pb.encodeMessage p) = .ok (p', []) ∧ + p'.peers = p.peers := + ⟨p, ControlPrune.decodeEncode p h, rfl⟩ + +/-! ## The length-delimited frame step + +One RPC frame is a uvarint length and the message bytes, under +`Constants.rpcCap`. The decoder checks the cap before it takes the body, +as `Streams.Framing.Frame.decode?` does. A zero-length frame is a legal +empty RPC; unlike `Identify`, this protocol has no empty-frame rule. -/ + +/-- One length-delimited RPC frame under the cap. `none` when the +payload is over the cap. -/ +def encodeFrame? (r : RPC) : Option (List Nat) := + let payload := Pb.encodeMessage r + if payload.length ≤ Constants.rpcCap then + some (Streams.Framing.encodeLength .uvarint payload.length ++ payload) + else none + +/-- One frame from the front of the buffer. + +* `.ok (some (payload, rest))`: one whole frame, and the bytes after it. +* `.ok none`: the buffer ends inside the prefix or inside the payload. +* `.error (.framing .malformed)`: a bad prefix, or a length over the cap. + +The cap check runs before the payload is taken. -/ +def decodeFrame? (bytes : List Nat) : + Except Pb.Error (Option (List Nat × List Nat)) := + match Streams.Framing.Frame.decode? .uvarint Constants.rpcCap bytes with + | .error .malformed => .error (.framing .malformed) + | .error .truncated => .ok none + | .ok (frame, rest) => .ok (some (frame.payload, rest)) + +/-- The frame step reads a whole frame back from its encoding, with any +rest after it. -/ +theorem decodeFrame?Encode (f : Streams.Framing.Frame .uvarint Constants.rpcCap) + (rest : List Nat) : + decodeFrame? (Streams.Framing.Frame.encode f ++ rest) = + .ok (some (f.payload, rest)) := by + unfold decodeFrame? + rw [Streams.Framing.Frame.decode?EncodeAppend f rest] + +/-- A prefix with a length over the cap is malformed before the body. -/ +theorem decodeFrame?OverCap (len : Nat) (rest : List Nat) + (hlen : len ≤ Streams.Framing.Prefix.maxLen .uvarint) + (hcap : Constants.rpcCap < len) : + decodeFrame? (Streams.Framing.encodeLength .uvarint len ++ rest) = + .error (.framing .malformed) := by + unfold decodeFrame? + rw [Streams.Framing.Frame.decode?OverCap (p := .uvarint) + (cap := Constants.rpcCap) + (l := Streams.Framing.encodeLength .uvarint len ++ rest) + (rest := rest) (len := len) + (Streams.Framing.decodeLength?EncodeLength .uvarint len rest hlen) hcap] + +/-- A prefix cut short needs more bytes. -/ +theorem decodeFrame?Truncated (bytes : List Nat) + (h : Streams.Framing.Frame.decode? .uvarint Constants.rpcCap bytes = + .error .truncated) : + decodeFrame? bytes = .ok none := by + unfold decodeFrame? + rw [h] + +/-- The rpc cap fits the uvarint prefix. -/ +theorem rpcCapLeMaxLen : + Constants.rpcCap ≤ Streams.Framing.Prefix.maxLen .uvarint := by + decide + +/-- The frame steps are inverse for a valid RPC under the cap. -/ +theorem decodeFrame?EncodeRpc (r : RPC) (rest : List Nat) + (_h : ∀ e ∈ r.entries, e.Valid) + (hcap : (Pb.encodeMessage r).length ≤ Constants.rpcCap) : + decodeFrame? (Streams.Framing.encodeLength .uvarint (Pb.encodeMessage r).length + ++ Pb.encodeMessage r ++ rest) = .ok (some (Pb.encodeMessage r, rest)) := by + have hbytes : ∀ b ∈ Pb.encodeMessage r, b < 256 := + Pb.encodeFieldsBytes (Pb.ProtoMessage.toFields r) + have hmax : (Pb.encodeMessage r).length ≤ + Streams.Framing.Prefix.maxLen .uvarint := + Nat.le_trans hcap rpcCapLeMaxLen + let f : Streams.Framing.Frame .uvarint Constants.rpcCap := + ⟨Pb.encodeMessage r, hbytes, ⟨hcap, hmax⟩⟩ + simpa [Streams.Framing.Frame.encode] using decodeFrame?Encode f rest + +end Rpc + +end Libp2p.Protocols.Gossipsub diff --git a/packages/libp2p/protocols/Libp2p/Protocols/Gossipsub/Score.lean b/packages/libp2p/protocols/Libp2p/Protocols/Gossipsub/Score.lean new file mode 100644 index 00000000..9f2fbacd --- /dev/null +++ b/packages/libp2p/protocols/Libp2p/Protocols/Gossipsub/Score.lean @@ -0,0 +1,1028 @@ +import Libp2p.Protocols.Gossipsub.Types + +/-! +# Gossipsub peer scoring + +The peer score function of gossipsub v1.1 over its counters, with decay. The function is pure. +The caller passes the time as `Millis`, and this module reads no clock. + +## The score of the spec + +The spec mixes four per-topic terms with three global terms +(`vendor/specs/pubsub/gossipsub/gossipsub-v1.1.md`, section "Peer Scoring"): + +`Score = TopicCap(sum over topics t * (w1*P1 + w2*P2 + w3*P3 + w3b*P3b + w4*P4)) + + w5*P5 + w6*P6 + w7*P7` + +`P1` is the time in mesh, `P2` the first message deliveries, `P3` the mesh message delivery rate, +`P3b` the mesh message delivery failures, `P4` the invalid message deliveries, `P5` the +application score, `P6` the IP colocation factor, and `P7` the behavior penalty. The topic cap +applies to the sum of the topic terms. A topic with no configured parameters contributes nothing. + +## Fixed point + +The spec computes with float64. This module computes with integers, in `scoreScale` units per 1.0 +(`Gossipsub/Types.lean`). One convention applies to every product: + +- Every weight is an `Int` in `1/scoreScale` units. A weight of 3 means 0.003. +- The `P1` to `P4`, `P6`, and `P7` values are `Nat` counts. +- A weight times a count is an exact `Int` product. +- The one rounded product is the topic contribution: + `(topicWeight * topicScore) tdiv scoreScale`. `Int.tdiv` truncates toward zero. This is the + single rounding site. +- The decay divides with `Nat` division, `counter * decay / scoreScale`, which floors. + +Lean 4.34 defines `/` on `Int` as Euclidean division. For a negative numerator, `/ scoreScale` +rounds down by one unit at the boundary, and `Int.tdiv scoreScale` truncates toward zero. This +module writes `Int.tdiv` to keep the truncation toward zero that the fixed-point convention +states. + +## Parameters and validity + +`Params.Valid` carries the weight signs and the decay bounds that the score theorems consume, +plus three constraints that no theorem consumes. The predicate adds `topicWeight >= 0` and +`timeInMeshQuantum > 0` to the weight and decay list, because a negative topic weight inverts +every term of its topic. It also carries `ipColocationFactorThreshold >= 1` and +`appSpecificWeight >= 0` from the spec's global table. The `TopicParams.Valid` docstring lists +the constraints and names the spec constraint that the predicate does not encode. The predicate +is a sufficient condition for the theorems, and it is not the complete parameter table of the +spec. + +## The behavior penalty threshold + +The vendored spec describes `P7` as the square of a counter with no threshold. The global +parameter table names only `BehaviourPenaltyWeight` and `BehaviourPenaltyDecay`. This module +carries `behaviourPenaltyThreshold`, as the go implementation does, and `P7` is the square of the +excess of the counter over the threshold. A threshold of 0 reproduces the spec's square of the +counter. + +## Application score + +`PeerState.appScore` is an integer count of application points. The spec leaves the unit of `P5` +open. An integer count keeps the product `appSpecificWeight * appScore` exact. + +## Decay + +`decayCounter` zeroes a counter that falls below `decayToZero` and keeps a counter that equals it. +The comparison is strict, as the spec's pseudocode writes it. + +## Time + +`elapsed` computes the mesh time lazily from `timeInMesh`, `graftTime`, and `now`. `Nat` +subtraction saturates at zero. The caller passes `now >= graftTime`; no theorem consumes that +precondition. + +## The Boolean validity mirror + +`TopicParams.validB` and `GlobalParams.validB` repeat the sign lists of the predicates as +`Bool` values. The test runner takes `Bool`, and `Params.Valid` has a `∀` over `TopicId`, so no +`Decidable` instance exists. The reflection theorems tie each `Bool` mirror to its predicate. +-/ + +set_option autoImplicit false + +namespace Libp2p.Protocols.Gossipsub.Score + +/-! ## Parameters -/ + +/-- The per-topic score parameters of gossipsub v1.1. The field order follows the topic +parameter table of the spec. `topicScore` reads the activation. The core stores the window +and never reads it, so every accepted message from a mesh peer counts toward `P3` +(`Core.lean` records the omission). -/ +structure TopicParams where + /-- The weight of this topic in the total score. -/ + topicWeight : Int + /-- The weight of `P1`, the time in mesh. -/ + timeInMeshWeight : Int + /-- The mesh time that earns one `P1` point. -/ + timeInMeshQuantum : Millis + /-- The maximum of `P1`. -/ + timeInMeshCap : Nat + /-- The weight of `P2`, the first message deliveries. -/ + firstMessageDeliveriesWeight : Int + /-- The decay factor of the `P2` counter. -/ + firstMessageDeliveriesDecay : Nat + /-- The maximum of the `P2` counter. The cap applies at increment time. -/ + firstMessageDeliveriesCap : Nat + /-- The weight of `P3`, the mesh message delivery rate. -/ + meshMessageDeliveriesWeight : Int + /-- The decay factor of the `P3` counter. -/ + meshMessageDeliveriesDecay : Nat + /-- The delivery count below which `P3` penalizes the peer. -/ + meshMessageDeliveriesThreshold : Nat + /-- The maximum of the `P3` counter. The cap applies at increment time. -/ + meshMessageDeliveriesCap : Nat + /-- The mesh time a peer needs before `P3` applies. -/ + meshMessageDeliveriesActivation : Millis + /-- The window after a first delivery that counts as a near-first delivery. -/ + meshMessageDeliveriesWindow : Millis + /-- The weight of `P3b`, the mesh failure penalty. -/ + meshFailurePenaltyWeight : Int + /-- The decay factor of the `P3b` counter. -/ + meshFailurePenaltyDecay : Nat + /-- The weight of `P4`, the invalid message deliveries. -/ + invalidMessageDeliveriesWeight : Int + /-- The decay factor of the `P4` counter. -/ + invalidMessageDeliveriesDecay : Nat + +/-- The global score parameters of gossipsub v1.1. The field order follows the global parameter +table of the spec. `decayInterval` feeds the core's heartbeat; `score` does not read it. + +The vendored spec gives `P7` as the square of a counter with no threshold +(`gossipsub-v1.1.md`, "The Score Function"). The global parameter table names only +`BehaviourPenaltyWeight` and `BehaviourPenaltyDecay`. This module carries +`behaviourPenaltyThreshold`, as the go implementation does. -/ +structure GlobalParams where + /-- The weight of `P5`, the application score. -/ + appSpecificWeight : Int + /-- The weight of `P6`, the IP colocation factor. -/ + ipColocationFactorWeight : Int + /-- The colocation count at or below which `P6` is zero. -/ + ipColocationFactorThreshold : Nat + /-- The weight of `P7`, the behavior penalty. -/ + behaviourPenaltyWeight : Int + /-- The behavior penalty count at or below which `P7` is zero. -/ + behaviourPenaltyThreshold : Nat + /-- The decay factor of the behavior penalty counter. -/ + behaviourPenaltyDecay : Nat + /-- The decay period of the router's heartbeat. -/ + decayInterval : Millis + /-- The counter limit below which a decayed counter becomes zero. -/ + decayToZero : Nat + /-- The cap of the summed topic part. A cap of 0 or less means no cap. -/ + topicScoreCap : Int + +/-- The score parameters: the per-topic table and the global table. -/ +structure Params where + /-- The parameters of each configured topic; `none` means the topic is not configured. -/ + topics : TopicId → Option TopicParams + /-- The global parameters. -/ + global : GlobalParams + +/-! ## Validity -/ + +/-- The validity predicate of a topic's parameters. It carries the weight signs that the score +theorems consume, the decay bounds that the decay theorems consume, and one constraint that no +theorem consumes: + +- `topicWeight_nonneg`: a negative topic weight inverts every term of the topic. The invalid + delivery monotonicity theorem consumes it. +- `timeInMeshQuantum_pos`: the quantum divides the elapsed mesh time. No theorem consumes this + constraint, and it is part of the contract the core relies on. +- The four decay bounds keep each factor at or below `scoreScale`, the unit of a full decay. + Each decay theorem consumes the bound of its own counter. + +The spec's `MeshMessageDeliveriesCap >= MeshMessageDeliveriesThreshold` constraint is not +encoded. -/ +structure TopicParams.Valid (tp : TopicParams) : Prop where + /-- The topic weight is at or above 0. -/ + topicWeight_nonneg : 0 ≤ tp.topicWeight + /-- The `P1` weight is at or above 0. -/ + timeInMeshWeight_nonneg : 0 ≤ tp.timeInMeshWeight + /-- The `P2` weight is at or above 0. -/ + firstMessageDeliveriesWeight_nonneg : 0 ≤ tp.firstMessageDeliveriesWeight + /-- The `P3` weight is at or below 0. -/ + meshMessageDeliveriesWeight_nonpos : tp.meshMessageDeliveriesWeight ≤ 0 + /-- The `P3b` weight is at or below 0. -/ + meshFailurePenaltyWeight_nonpos : tp.meshFailurePenaltyWeight ≤ 0 + /-- The `P4` weight is at or below 0. -/ + invalidMessageDeliveriesWeight_nonpos : tp.invalidMessageDeliveriesWeight ≤ 0 + /-- The time quantum is at or above 1 millisecond. -/ + timeInMeshQuantum_pos : 0 < tp.timeInMeshQuantum + /-- The `P2` decay factor is at most a full decay. -/ + firstMessageDeliveriesDecay_le : tp.firstMessageDeliveriesDecay ≤ scoreScale + /-- The `P3` decay factor is at most a full decay. -/ + meshMessageDeliveriesDecay_le : tp.meshMessageDeliveriesDecay ≤ scoreScale + /-- The `P3b` decay factor is at most a full decay. -/ + meshFailurePenaltyDecay_le : tp.meshFailurePenaltyDecay ≤ scoreScale + /-- The `P4` decay factor is at most a full decay. -/ + invalidMessageDeliveriesDecay_le : tp.invalidMessageDeliveriesDecay ≤ scoreScale + +/-- The validity predicate of the global parameters. It carries the weight signs that the score +theorems consume, the colocation threshold of the spec, and the behavior decay bound. + +`appSpecificWeight_nonneg` and `ipColocationFactorThreshold_pos` are spec constraints, and no +theorem consumes them. -/ +structure GlobalParams.Valid (g : GlobalParams) : Prop where + /-- The `P5` weight is at or above 0. -/ + appSpecificWeight_nonneg : 0 ≤ g.appSpecificWeight + /-- The `P6` weight is at or below 0. -/ + ipColocationFactorWeight_nonpos : g.ipColocationFactorWeight ≤ 0 + /-- The colocation threshold is at least 1. -/ + ipColocationFactorThreshold_pos : 1 ≤ g.ipColocationFactorThreshold + /-- The `P7` weight is at or below 0. -/ + behaviourPenaltyWeight_nonpos : g.behaviourPenaltyWeight ≤ 0 + /-- The behavior penalty decay factor is at most a full decay. -/ + behaviourPenaltyDecay_le : g.behaviourPenaltyDecay ≤ scoreScale + +/-- The validity predicate of the parameters: every configured topic passes +`TopicParams.Valid`, and the global parameters pass `GlobalParams.Valid`. An unconfigured topic +passes vacuously. -/ +structure Params.Valid (p : Params) : Prop where + /-- Every configured topic is valid. -/ + topic : ∀ (id : TopicId) (tp : TopicParams), p.topics id = some tp → TopicParams.Valid tp + /-- The global parameters are valid. -/ + global : GlobalParams.Valid p.global + +/-- The `Bool` mirror of `TopicParams.Valid`. The test runner takes `Bool`, and a `Prop` +structure has no `Decidable` instance. `TopicParams.validB_iff` ties the mirror to the +predicate. -/ +def TopicParams.validB (tp : TopicParams) : Bool := + decide (0 ≤ tp.topicWeight) && + decide (0 ≤ tp.timeInMeshWeight) && + decide (0 ≤ tp.firstMessageDeliveriesWeight) && + decide (tp.meshMessageDeliveriesWeight ≤ 0) && + decide (tp.meshFailurePenaltyWeight ≤ 0) && + decide (tp.invalidMessageDeliveriesWeight ≤ 0) && + decide (0 < tp.timeInMeshQuantum) && + decide (tp.firstMessageDeliveriesDecay ≤ scoreScale) && + decide (tp.meshMessageDeliveriesDecay ≤ scoreScale) && + decide (tp.meshFailurePenaltyDecay ≤ scoreScale) && + decide (tp.invalidMessageDeliveriesDecay ≤ scoreScale) + +/-- The `Bool` mirror of `GlobalParams.Valid`. -/ +def GlobalParams.validB (g : GlobalParams) : Bool := + decide (0 ≤ g.appSpecificWeight) && + decide (g.ipColocationFactorWeight ≤ 0) && + decide (1 ≤ g.ipColocationFactorThreshold) && + decide (g.behaviourPenaltyWeight ≤ 0) && + decide (g.behaviourPenaltyDecay ≤ scoreScale) + +/-- The `Bool` mirror accepts exactly the valid topic parameters. -/ +theorem TopicParams.validB_iff (tp : TopicParams) : + tp.validB = true ↔ TopicParams.Valid tp := by + constructor + · intro h + simp [TopicParams.validB, Bool.and_eq_true, decide_eq_true_eq] at h + constructor <;> simp_all + · intro h + cases h + simp [TopicParams.validB, *] + +/-- The `Bool` mirror accepts exactly the valid global parameters. -/ +theorem GlobalParams.validB_iff (g : GlobalParams) : + g.validB = true ↔ GlobalParams.Valid g := by + constructor + · intro h + simp [GlobalParams.validB, Bool.and_eq_true, decide_eq_true_eq] at h + constructor <;> simp_all + · intro h + cases h + simp [GlobalParams.validB, *] + +/-! ## Peer state -/ + +/-- The counters of one peer in one topic. `timeInMesh` holds the accrued mesh time, and +`graftTime` holds the start of the current mesh period. `elapsed` sums the two when `inMesh` is +true. -/ +structure TopicCounters where + /-- The mesh time accrued in earlier mesh periods. -/ + timeInMesh : Millis + /-- The time of the last graft. -/ + graftTime : Millis + /-- Whether the peer is in the mesh of the topic. -/ + inMesh : Bool + /-- The first message deliveries, capped at increment time. -/ + firstMessageDeliveries : Nat + /-- The mesh message deliveries, capped at increment time. -/ + meshMessageDeliveries : Nat + /-- The mesh message delivery failures. -/ + meshFailurePenalty : Nat + /-- The invalid message deliveries. -/ + invalidMessageDeliveries : Nat + +/-- The all-zero counters of a peer that has no history in a topic. -/ +def TopicCounters.zero : TopicCounters where + timeInMesh := 0 + graftTime := 0 + inMesh := false + firstMessageDeliveries := 0 + meshMessageDeliveries := 0 + meshFailurePenalty := 0 + invalidMessageDeliveries := 0 + +/-- The score state of one peer: the per-topic counters and the three global inputs. -/ +structure PeerState where + /-- The counters of each topic the peer has a history in. -/ + topics : List (TopicId × TopicCounters) + /-- The application score `P5`, an integer count of application points. -/ + appScore : Int + /-- The IP colocation count `P6`. -/ + ipColocation : Nat + /-- The behavior penalty counter `P7`. -/ + behaviourPenalty : Nat + +/-- The state of a fresh peer: no topic history, no application score, no colocation, and no +behavior penalty. -/ +def PeerState.zero : PeerState where + topics := [] + appScore := 0 + ipColocation := 0 + behaviourPenalty := 0 + +/-! ## Scoring -/ + +/-- The mesh time of a peer in a topic. The peer accrues time from `graftTime` to `now` while +`inMesh` is true, and the stored `timeInMesh` holds the earlier periods. The `Nat` subtraction +saturates at zero. The caller passes `now >= graftTime`. -/ +def elapsed (now : Millis) (t : TopicCounters) : Millis := + t.timeInMesh + (if t.inMesh then now - t.graftTime else 0) + +/-- `P1`, the time in mesh: the elapsed time in quanta, capped. -/ +def p1 (tp : TopicParams) (now : Millis) (t : TopicCounters) : Nat := + min (elapsed now t / tp.timeInMeshQuantum) tp.timeInMeshCap + +/-- `P3`, the mesh message delivery rate: the square of the delivery deficit below the +threshold, when the activation window has passed. Both comparisons are strict, so equality with +either bound gives 0. -/ +def p3 (tp : TopicParams) (now : Millis) (t : TopicCounters) : Nat := + if tp.meshMessageDeliveriesActivation < elapsed now t ∧ + t.meshMessageDeliveries < tp.meshMessageDeliveriesThreshold then + (tp.meshMessageDeliveriesThreshold - t.meshMessageDeliveries) * + (tp.meshMessageDeliveriesThreshold - t.meshMessageDeliveries) + else 0 + +/-- `P4`, the invalid message deliveries: the square of the counter, uncapped. -/ +def p4 (t : TopicCounters) : Nat := + t.invalidMessageDeliveries * t.invalidMessageDeliveries + +/-- `P6`, the IP colocation factor: the square of the surplus over the threshold. -/ +def p6 (g : GlobalParams) (s : PeerState) : Nat := + if g.ipColocationFactorThreshold < s.ipColocation then + (s.ipColocation - g.ipColocationFactorThreshold) * + (s.ipColocation - g.ipColocationFactorThreshold) + else 0 + +/-- `P7`, the behavior penalty: the square of the excess over the threshold. -/ +def p7 (g : GlobalParams) (s : PeerState) : Nat := + if g.behaviourPenaltyThreshold < s.behaviourPenalty then + (s.behaviourPenalty - g.behaviourPenaltyThreshold) * + (s.behaviourPenalty - g.behaviourPenaltyThreshold) + else 0 + +/-- The per-topic score: the weighted sum of the five terms, in `1/scoreScale` weight units. +The `P2` and `P3b` caps apply at increment time, so this function reads the counters as they +stand. -/ +def topicScore (tp : TopicParams) (now : Millis) (t : TopicCounters) : Int := + tp.timeInMeshWeight * (p1 tp now t : Int) + + tp.firstMessageDeliveriesWeight * (t.firstMessageDeliveries : Int) + + tp.meshMessageDeliveriesWeight * (p3 tp now t : Int) + + tp.meshFailurePenaltyWeight * (t.meshFailurePenalty : Int) + + tp.invalidMessageDeliveriesWeight * (p4 t : Int) + +/-- The contribution of one topic entry. An unconfigured topic contributes 0. A configured +topic contributes `(topicWeight * topicScore) tdiv scoreScale`; this is the one rounding site of +the module. -/ +private def topicContribution (p : Params) (now : Millis) + (entry : TopicId × TopicCounters) : Int := + match p.topics entry.1 with + | some tp => (tp.topicWeight * topicScore tp now entry.2).tdiv scoreScale + | none => 0 + +/-- The sum of the topic contributions, before the cap. -/ +private def rawTopicPart (p : Params) (now : Millis) (s : PeerState) : Int := + (s.topics.map (topicContribution p now)).sum + +/-- The topic part of the score: the sum of the configured topic contributions, capped at +`topicScoreCap` when the cap is positive. A cap of 0 or less means no cap. -/ +def topicPart (p : Params) (now : Millis) (s : PeerState) : Int := + let raw := rawTopicPart p now s + if 0 < p.global.topicScoreCap ∧ p.global.topicScoreCap < raw then + p.global.topicScoreCap + else raw + +/-- The peer score of gossipsub v1.1: the capped topic part, the weighted application score, and +the weighted `P6` and `P7` terms. The last three products are exact. -/ +def score (p : Params) (s : PeerState) (now : Millis) : Score := + topicPart p now s + + p.global.appSpecificWeight * s.appScore + + p.global.ipColocationFactorWeight * (p6 p.global s : Int) + + p.global.behaviourPenaltyWeight * (p7 p.global s : Int) + +/-! ## Decay -/ + +/-- One decay step of a counter: multiply by the decay factor, divide by `scoreScale`, and +zero the result when it falls below `decayToZero`. The comparison is strict. -/ +def decayCounter (d : Nat) (decayToZero : Nat) (c : Nat) : Nat := + if c * d / scoreScale < decayToZero then 0 else c * d / scoreScale + +/-- One decay step of the topic counters. The four decaying counters move; `timeInMesh`, +`graftTime`, and `inMesh` stay. -/ +def decayTopic (tp : TopicParams) (decayToZero : Nat) (t : TopicCounters) : TopicCounters := + { t with + firstMessageDeliveries := + decayCounter tp.firstMessageDeliveriesDecay decayToZero t.firstMessageDeliveries + meshMessageDeliveries := + decayCounter tp.meshMessageDeliveriesDecay decayToZero t.meshMessageDeliveries + meshFailurePenalty := + decayCounter tp.meshFailurePenaltyDecay decayToZero t.meshFailurePenalty + invalidMessageDeliveries := + decayCounter tp.invalidMessageDeliveriesDecay decayToZero t.invalidMessageDeliveries } + +/-- One decay step of a topic entry. An entry whose topic has no configured parameters stays +unchanged. Such an entry contributes 0 to the score. -/ +private def decayEntry (p : Params) (decayToZero : Nat) + (entry : TopicId × TopicCounters) : TopicId × TopicCounters := + match p.topics entry.1 with + | some tp => (entry.1, decayTopic tp decayToZero entry.2) + | none => entry + +/-- One decay interval of a peer state. The four topic counters and the behavior penalty +decay. The application score and the colocation count stay; the application owns their +lifecycle. -/ +def decay (p : Params) (s : PeerState) : PeerState := + { s with + topics := s.topics.map (decayEntry p p.global.decayToZero) + behaviourPenalty := + decayCounter p.global.behaviourPenaltyDecay p.global.decayToZero s.behaviourPenalty } + +/-! ## The counter table -/ + +/-- The first entry of `l` with key `id`. -/ +private def lookupCounters : + List (TopicId × TopicCounters) → TopicId → Option TopicCounters + | [], _ => none + | (k, v) :: rest, id => if k = id then some v else lookupCounters rest id + +/-- The counters of `id`, or the all-zero counters when the state has no entry for the topic. -/ +def countersOf (s : PeerState) (id : TopicId) : TopicCounters := + match lookupCounters s.topics id with + | some t => t + | none => TopicCounters.zero + +/-- Replace the first entry with key `id`, or append a new entry when the key is absent. The +other entries keep their order. -/ +private def upsert : + List (TopicId × TopicCounters) → TopicId → TopicCounters → + List (TopicId × TopicCounters) + | [], id, t => [(id, t)] + | (k, v) :: rest, id, t => + if k = id then (k, t) :: rest else (k, v) :: upsert rest id t + +/-- Replace the counters of `id` in the state. -/ +private def setCounters (s : PeerState) (id : TopicId) (t : TopicCounters) : PeerState := + { s with topics := upsert s.topics id t } + +/-! ## Counter updates -/ + +/-- The graft update: when the peer is not in the mesh, set `inMesh` and record the graft time. +A peer that is already in the mesh is unchanged. -/ +def graft (s : PeerState) (now : Millis) (id : TopicId) : PeerState := + let t := countersOf s id + if t.inMesh then s + else setCounters s id { t with inMesh := true, graftTime := now } + +/-- The counter part of a prune: fold the elapsed mesh time into `timeInMesh`, leave the mesh, +and add the square of the delivery deficit to the failure penalty when the activation window +has passed and the delivery count is below the threshold. -/ +private def pruneCounters (p : Params) (now : Millis) (id : TopicId) + (t : TopicCounters) : TopicCounters := + let e := elapsed now t + let t1 := { t with timeInMesh := e, inMesh := false } + match p.topics id with + | some tp => + if tp.meshMessageDeliveriesActivation < e ∧ + t.meshMessageDeliveries < tp.meshMessageDeliveriesThreshold then + { t1 with meshFailurePenalty := t.meshFailurePenalty + + (tp.meshMessageDeliveriesThreshold - t.meshMessageDeliveries) * + (tp.meshMessageDeliveriesThreshold - t.meshMessageDeliveries) } + else t1 + | none => t1 + +/-- The prune update: a peer in the mesh folds its mesh time and adds the failure penalty, per +spec. A peer that is not in the mesh is unchanged. -/ +def prune (p : Params) (s : PeerState) (now : Millis) (id : TopicId) : PeerState := + let t := countersOf s id + if t.inMesh then setCounters s id (pruneCounters p now id t) else s + +/-- The first message delivery update: increment the counter and cap it. An unconfigured topic +is unchanged. -/ +def firstDelivery (p : Params) (s : PeerState) (id : TopicId) : PeerState := + match p.topics id with + | some tp => + let t := countersOf s id + setCounters s id { t with + firstMessageDeliveries := + min (t.firstMessageDeliveries + 1) tp.firstMessageDeliveriesCap } + | none => s + +/-- The mesh message delivery update: increment the counter and cap it. An unconfigured topic +is unchanged. -/ +def meshDelivery (p : Params) (s : PeerState) (id : TopicId) : PeerState := + match p.topics id with + | some tp => + let t := countersOf s id + setCounters s id { t with + meshMessageDeliveries := + min (t.meshMessageDeliveries + 1) tp.meshMessageDeliveriesCap } + | none => s + +/-- The counter part of an invalid delivery: increment the uncapped `P4` counter. -/ +private def invalidStep (t : TopicCounters) : TopicCounters := + { t with invalidMessageDeliveries := t.invalidMessageDeliveries + 1 } + +/-- The invalid message delivery update: increment the uncapped `P4` counter. An unconfigured +topic is unchanged. -/ +def invalidDelivery (p : Params) (s : PeerState) (id : TopicId) : PeerState := + match p.topics id with + | some _ => setCounters s id (invalidStep (countersOf s id)) + | none => s + +/-- The behavior penalty update: increment the `P7` counter. It is uncapped, and the decay +reduces it later. -/ +def behaviourPenalty (s : PeerState) : PeerState := + { s with behaviourPenalty := s.behaviourPenalty + 1 } + +/-! ## Table lemmas -/ + +/-- Reading the table after a write of the same key returns the written counters. -/ +private theorem lookupCounters_upsert_same (l : List (TopicId × TopicCounters)) + (id : TopicId) (t : TopicCounters) : + lookupCounters (upsert l id t) id = some t := by + induction l with + | nil => simp [upsert, lookupCounters] + | cons head rest ih => + obtain ⟨k, v⟩ := head + by_cases hk : k = id + · simp [upsert, lookupCounters, hk] + · simp [upsert, lookupCounters, hk, ih] + +/-- Reading the table of a mapped list maps the read entry, when the map keeps the keys. -/ +private theorem lookupCounters_map + (f : TopicId × TopicCounters → TopicId × TopicCounters) + (hf : ∀ e, (f e).1 = e.1) : + ∀ (l : List (TopicId × TopicCounters)) (id : TopicId), + lookupCounters (l.map f) id = + (lookupCounters l id).map (fun t => (f (id, t)).2) := by + intro l + induction l with + | nil => intro id; rfl + | cons head rest ih => + intro id + obtain ⟨k, v⟩ := head + by_cases hk : k = id + · simp [lookupCounters, hf, hk] + · simp [lookupCounters, hf, hk, ih] + +/-- The decay map keeps the key of every entry. -/ +private theorem decayEntry_key (p : Params) (z : Nat) (e : TopicId × TopicCounters) : + (decayEntry p z e).1 = e.1 := by + obtain ⟨k, v⟩ := e + unfold decayEntry + split <;> rfl + +/-- Reading the counters of a decayed state. -/ +private theorem countersOf_decay (p : Params) (s : PeerState) (id : TopicId) : + countersOf (decay p s) id = + match lookupCounters s.topics id with + | some t => (decayEntry p p.global.decayToZero (id, t)).2 + | none => TopicCounters.zero := by + unfold countersOf decay + rw [lookupCounters_map (decayEntry p p.global.decayToZero) (decayEntry_key p _) + s.topics id] + cases lookupCounters s.topics id <;> rfl + +/-- Replacing the first matching entry does not increase a sum of contributions, when the +replacement contributes at most the old entry. -/ +private theorem sum_map_upsert_le_of_found (f : TopicId × TopicCounters → Int) + (l : List (TopicId × TopicCounters)) (id : TopicId) + (u t : TopicCounters) (hfound : lookupCounters l id = some t) + (hle : f (id, u) ≤ f (id, t)) : + ((upsert l id u).map f).sum ≤ (l.map f).sum := by + induction l with + | nil => simp [lookupCounters] at hfound + | cons head rest ih => + obtain ⟨k, v⟩ := head + by_cases hk : k = id + · have hvt : v = t := by + rw [hk] at hfound + simpa [lookupCounters] using hfound + subst v + rw [hk] + simp [upsert, List.map_cons, List.sum_cons] + omega + · have hfound' : lookupCounters rest id = some t := by + simpa [lookupCounters, hk] using hfound + have hrec := ih hfound' + simp [upsert, hk, List.map_cons, List.sum_cons] + omega + +/-- Appending an entry does not increase a sum of contributions, when the new entry +contributes at most 0. -/ +private theorem sum_map_upsert_le_of_absent (f : TopicId × TopicCounters → Int) + (l : List (TopicId × TopicCounters)) (id : TopicId) (u : TopicCounters) + (habsent : lookupCounters l id = none) (hle : f (id, u) ≤ 0) : + ((upsert l id u).map f).sum ≤ (l.map f).sum := by + induction l with + | nil => simpa [upsert] using hle + | cons head rest ih => + obtain ⟨k, v⟩ := head + by_cases hk : k = id + · simp [lookupCounters, hk] at habsent + · have habsent' : lookupCounters rest id = none := by + simpa [lookupCounters, hk] using habsent + have hrec := ih habsent' + simp [upsert, hk, List.map_cons, List.sum_cons] + omega + +/-! ## Sign facts -/ + +/-- The `P1` term is at or above 0. -/ +theorem p1Term_nonneg {tp : TopicParams} (h : TopicParams.Valid tp) + (now : Millis) (t : TopicCounters) : + 0 ≤ tp.timeInMeshWeight * (p1 tp now t : Int) := + Int.mul_nonneg h.timeInMeshWeight_nonneg (Int.natCast_nonneg _) + +/-- The `P2` term is at or above 0. -/ +theorem p2Term_nonneg {tp : TopicParams} (h : TopicParams.Valid tp) + (t : TopicCounters) : + 0 ≤ tp.firstMessageDeliveriesWeight * (t.firstMessageDeliveries : Int) := + Int.mul_nonneg h.firstMessageDeliveriesWeight_nonneg (Int.natCast_nonneg _) + +/-- The `P3` term is at or below 0. -/ +theorem p3Term_nonpos {tp : TopicParams} (h : TopicParams.Valid tp) + (now : Millis) (t : TopicCounters) : + tp.meshMessageDeliveriesWeight * (p3 tp now t : Int) ≤ 0 := + Int.mul_nonpos_of_nonpos_of_nonneg h.meshMessageDeliveriesWeight_nonpos + (Int.natCast_nonneg _) + +/-- The `P3b` term is at or below 0. -/ +theorem p3bTerm_nonpos {tp : TopicParams} (h : TopicParams.Valid tp) + (t : TopicCounters) : + tp.meshFailurePenaltyWeight * (t.meshFailurePenalty : Int) ≤ 0 := + Int.mul_nonpos_of_nonpos_of_nonneg h.meshFailurePenaltyWeight_nonpos + (Int.natCast_nonneg _) + +/-- The `P4` term is at or below 0. -/ +theorem p4Term_nonpos {tp : TopicParams} (h : TopicParams.Valid tp) + (t : TopicCounters) : + tp.invalidMessageDeliveriesWeight * (p4 t : Int) ≤ 0 := + Int.mul_nonpos_of_nonpos_of_nonneg h.invalidMessageDeliveriesWeight_nonpos + (Int.natCast_nonneg _) + +/-- The `P6` term is at or below 0. -/ +theorem p6Term_nonpos {g : GlobalParams} (h : GlobalParams.Valid g) (s : PeerState) : + g.ipColocationFactorWeight * (p6 g s : Int) ≤ 0 := + Int.mul_nonpos_of_nonpos_of_nonneg h.ipColocationFactorWeight_nonpos + (Int.natCast_nonneg _) + +/-- The `P7` term is at or below 0. -/ +theorem p7Term_nonpos {g : GlobalParams} (h : GlobalParams.Valid g) (s : PeerState) : + g.behaviourPenaltyWeight * (p7 g s : Int) ≤ 0 := + Int.mul_nonpos_of_nonpos_of_nonneg h.behaviourPenaltyWeight_nonpos + (Int.natCast_nonneg _) + +/-! ## The topic cap -/ + +/-- The topic part never exceeds a positive topic score cap. The statement needs no +`Params.Valid`: the cap holds for any parameters. -/ +theorem topicPart_le_cap (p : Params) (now : Millis) (s : PeerState) + (hcap : 0 < p.global.topicScoreCap) : + topicPart p now s ≤ p.global.topicScoreCap := by + dsimp only [topicPart] + split <;> omega + +/-! ## Decay theorems -/ + +/-- A decay step never increases a counter, when the decay factor is at most `scoreScale`. -/ +theorem decayCounter_le (d : Nat) (z : Nat) (c : Nat) (hd : d ≤ scoreScale) : + decayCounter d z c ≤ c := by + have hdiv : c * d / scoreScale ≤ c := by + exact Nat.div_le_of_le_mul (by + rw [Nat.mul_comm scoreScale c] + exact Nat.mul_le_mul_left c hd) + unfold decayCounter + split <;> omega + +/-- A counter below `decayToZero` becomes 0 after a decay step, when the decay factor is at most +`scoreScale`. -/ +theorem decayCounter_of_lt (d : Nat) (z : Nat) (c : Nat) (hd : d ≤ scoreScale) + (hc : c < z) : + decayCounter d z c = 0 := by + have hdiv : c * d / scoreScale ≤ c := by + exact Nat.div_le_of_le_mul (by + rw [Nat.mul_comm scoreScale c] + exact Nat.mul_le_mul_left c hd) + unfold decayCounter + split + · rfl + · omega + +/-- The behavior penalty never increases under a decay step. -/ +theorem decay_behaviourPenalty_le {p : Params} (h : Params.Valid p) (s : PeerState) : + (decay p s).behaviourPenalty ≤ s.behaviourPenalty := by + simpa [decay] using + decayCounter_le p.global.behaviourPenaltyDecay p.global.decayToZero + s.behaviourPenalty h.global.behaviourPenaltyDecay_le + +/-- A decay step leaves the mesh time, the graft time, and the mesh membership unchanged. -/ +theorem decay_preserves_mesh {p : Params} (s : PeerState) (id : TopicId) : + (countersOf (decay p s) id).timeInMesh = (countersOf s id).timeInMesh ∧ + (countersOf (decay p s) id).graftTime = (countersOf s id).graftTime ∧ + (countersOf (decay p s) id).inMesh = (countersOf s id).inMesh := by + rw [countersOf_decay] + cases hlook : lookupCounters s.topics id with + | none => simp [countersOf, hlook, TopicCounters.zero] + | some t => + simp only [countersOf, hlook] + cases hcfg : p.topics id with + | none => simp [decayEntry, hcfg] + | some tp => simp [decayEntry, decayTopic, hcfg] + +/-- The `P2` counter never increases under a decay step. -/ +theorem decay_firstMessageDeliveries_le {p : Params} (h : Params.Valid p) + (s : PeerState) (id : TopicId) : + (countersOf (decay p s) id).firstMessageDeliveries ≤ + (countersOf s id).firstMessageDeliveries := by + rw [countersOf_decay] + cases hlook : lookupCounters s.topics id with + | none => simp [countersOf, hlook, TopicCounters.zero] + | some t => + simp only [countersOf, hlook] + cases hcfg : p.topics id with + | none => simp [decayEntry, hcfg] + | some tp => + simp only [decayEntry, hcfg] + exact decayCounter_le tp.firstMessageDeliveriesDecay p.global.decayToZero + t.firstMessageDeliveries (h.topic id tp hcfg).firstMessageDeliveriesDecay_le + +/-- The `P3` counter never increases under a decay step. -/ +theorem decay_meshMessageDeliveries_le {p : Params} (h : Params.Valid p) + (s : PeerState) (id : TopicId) : + (countersOf (decay p s) id).meshMessageDeliveries ≤ + (countersOf s id).meshMessageDeliveries := by + rw [countersOf_decay] + cases hlook : lookupCounters s.topics id with + | none => simp [countersOf, hlook, TopicCounters.zero] + | some t => + simp only [countersOf, hlook] + cases hcfg : p.topics id with + | none => simp [decayEntry, hcfg] + | some tp => + simp only [decayEntry, hcfg] + exact decayCounter_le tp.meshMessageDeliveriesDecay p.global.decayToZero + t.meshMessageDeliveries (h.topic id tp hcfg).meshMessageDeliveriesDecay_le + +/-- The `P3b` counter never increases under a decay step. -/ +theorem decay_meshFailurePenalty_le {p : Params} (h : Params.Valid p) + (s : PeerState) (id : TopicId) : + (countersOf (decay p s) id).meshFailurePenalty ≤ + (countersOf s id).meshFailurePenalty := by + rw [countersOf_decay] + cases hlook : lookupCounters s.topics id with + | none => simp [countersOf, hlook, TopicCounters.zero] + | some t => + simp only [countersOf, hlook] + cases hcfg : p.topics id with + | none => simp [decayEntry, hcfg] + | some tp => + simp only [decayEntry, hcfg] + exact decayCounter_le tp.meshFailurePenaltyDecay p.global.decayToZero + t.meshFailurePenalty (h.topic id tp hcfg).meshFailurePenaltyDecay_le + +/-- The `P4` counter never increases under a decay step. -/ +theorem decay_invalidMessageDeliveries_le {p : Params} (h : Params.Valid p) + (s : PeerState) (id : TopicId) : + (countersOf (decay p s) id).invalidMessageDeliveries ≤ + (countersOf s id).invalidMessageDeliveries := by + rw [countersOf_decay] + cases hlook : lookupCounters s.topics id with + | none => simp [countersOf, hlook, TopicCounters.zero] + | some t => + simp only [countersOf, hlook] + cases hcfg : p.topics id with + | none => simp [decayEntry, hcfg] + | some tp => + simp only [decayEntry, hcfg] + exact decayCounter_le tp.invalidMessageDeliveriesDecay p.global.decayToZero + t.invalidMessageDeliveries (h.topic id tp hcfg).invalidMessageDeliveriesDecay_le + +/-- The first message delivery counter stays at or below its cap. The topic must be +configured. -/ +theorem firstDelivery_le_cap {p : Params} {s : PeerState} {id : TopicId} {tp : TopicParams} + (hcfg : p.topics id = some tp) : + (countersOf (firstDelivery p s id) id).firstMessageDeliveries ≤ + tp.firstMessageDeliveriesCap := by + have h : min ((countersOf s id).firstMessageDeliveries + 1) + tp.firstMessageDeliveriesCap ≤ tp.firstMessageDeliveriesCap := + Nat.min_le_right _ _ + simpa [firstDelivery, hcfg, setCounters, countersOf, lookupCounters_upsert_same] using h + +/-- The mesh message delivery counter stays at or below its cap. The topic must be +configured. -/ +theorem meshDelivery_le_cap {p : Params} {s : PeerState} {id : TopicId} {tp : TopicParams} + (hcfg : p.topics id = some tp) : + (countersOf (meshDelivery p s id) id).meshMessageDeliveries ≤ + tp.meshMessageDeliveriesCap := by + have h : min ((countersOf s id).meshMessageDeliveries + 1) + tp.meshMessageDeliveriesCap ≤ tp.meshMessageDeliveriesCap := + Nat.min_le_right _ _ + simpa [meshDelivery, hcfg, setCounters, countersOf, lookupCounters_upsert_same] using h + +/-! ## Invalid delivery monotonicity -/ + +/-- One invalid delivery leaves `P1` unchanged. -/ +private theorem p1_invalidStep (tp : TopicParams) (now : Millis) (t : TopicCounters) : + p1 tp now (invalidStep t) = p1 tp now t := rfl + +/-- One invalid delivery leaves `P3` unchanged. -/ +private theorem p3_invalidStep (tp : TopicParams) (now : Millis) (t : TopicCounters) : + p3 tp now (invalidStep t) = p3 tp now t := rfl + +/-- The `P4` square does not fall when the counter grows. -/ +private theorem p4_invalidStep_mono (t : TopicCounters) : + p4 t ≤ p4 (invalidStep t) := + Nat.mul_le_mul (Nat.le_succ _) (Nat.le_succ _) + +/-- One invalid delivery does not raise the topic score, when the `P4` weight is at or below +0. -/ +private theorem topicScore_invalidStep_le (tp : TopicParams) (now : Millis) + (t : TopicCounters) (hw : tp.invalidMessageDeliveriesWeight ≤ 0) : + topicScore tp now (invalidStep t) ≤ topicScore tp now t := by + have h1 := p1_invalidStep tp now t + have h3 := p3_invalidStep tp now t + have hp4 : (p4 t : Int) ≤ (p4 (invalidStep t) : Int) := + Int.ofNat_le.mpr (p4_invalidStep_mono t) + have hprod := Int.mul_le_mul_of_nonpos_left hw hp4 + unfold topicScore + have h2 : (invalidStep t).firstMessageDeliveries = t.firstMessageDeliveries := rfl + have h4 : (invalidStep t).meshFailurePenalty = t.meshFailurePenalty := rfl + rw [h1, h3, h2, h4] + omega + +/-- One invalid delivery does not raise a topic contribution, when the topic weight is at or +above 0 and the `P4` weight is at or below 0. -/ +private theorem topicContribution_invalidStep_le (p : Params) (now : Millis) + (id : TopicId) {tp : TopicParams} (htp : p.topics id = some tp) + (hW : 0 ≤ tp.topicWeight) (hw : tp.invalidMessageDeliveriesWeight ≤ 0) + (t : TopicCounters) : + topicContribution p now (id, invalidStep t) ≤ topicContribution p now (id, t) := by + have hts := topicScore_invalidStep_le tp now t hw + have hmul : tp.topicWeight * topicScore tp now (invalidStep t) ≤ + tp.topicWeight * topicScore tp now t := + Int.mul_le_mul_of_nonneg_left hts hW + have hdiv := Int.tdiv_le_tdiv + (by decide : (0 : Int) < (scoreScale : Int)) hmul + simpa [topicContribution, htp] using hdiv + +/-- The contribution of a fresh counters value after one invalid delivery is at or below 0, +when the topic weight is at or above 0 and the `P4` weight is at or below 0. -/ +private theorem topicContribution_invalidStep_zero_nonpos (p : Params) (now : Millis) + (id : TopicId) {tp : TopicParams} (htp : p.topics id = some tp) + (hW : 0 ≤ tp.topicWeight) (hw : tp.invalidMessageDeliveriesWeight ≤ 0) : + topicContribution p now (id, invalidStep TopicCounters.zero) ≤ 0 := by + have hts : topicScore tp now (invalidStep TopicCounters.zero) = + tp.invalidMessageDeliveriesWeight := by + simp [topicScore, p1, p3, p4, elapsed, invalidStep, TopicCounters.zero] + have hmul : tp.topicWeight * tp.invalidMessageDeliveriesWeight ≤ 0 := + Int.mul_nonpos_of_nonneg_of_nonpos hW hw + have hdiv : (tp.topicWeight * tp.invalidMessageDeliveriesWeight).tdiv + (scoreScale : Int) ≤ 0 := by + have h0 : (0 : Int).tdiv (scoreScale : Int) = 0 := by decide + have := Int.tdiv_le_tdiv (by decide : (0 : Int) < (scoreScale : Int)) hmul + omega + simpa [topicContribution, htp, hts] using hdiv + +/-- The topic cap preserves the order of the raw topic part. -/ +private theorem topicPart_mono {p : Params} {now : Millis} {s s' : PeerState} + (h : rawTopicPart p now s' ≤ rawTopicPart p now s) : + topicPart p now s' ≤ topicPart p now s := by + dsimp only [topicPart] + split <;> split <;> omega + +/-- The score preserves the order of the topic part, when the three global inputs are equal. -/ +private theorem score_le_of_topicPart_le {p : Params} {now : Millis} {s s' : PeerState} + (h : topicPart p now s' ≤ topicPart p now s) + (happ : s'.appScore = s.appScore) + (hip : s'.ipColocation = s.ipColocation) + (hbp : s'.behaviourPenalty = s.behaviourPenalty) : + score p s' now ≤ score p s now := by + have happ' : p.global.appSpecificWeight * s'.appScore = + p.global.appSpecificWeight * s.appScore := by + rw [happ] + have hp6 : p6 p.global s' = p6 p.global s := by + unfold p6 + rw [hip] + have hp7 : p7 p.global s' = p7 p.global s := by + unfold p7 + rw [hbp] + unfold score + rw [happ', hp6, hp7] + exact Int.add_le_add_right (Int.add_le_add_right (Int.add_le_add_right h _) _) _ + +/-- One more invalid delivery never raises the score of a peer. The proof consumes the topic +weight sign and the `P4` weight sign of the target topic. The other topics do not move. -/ +theorem invalidDelivery_score_mono {p : Params} (h : Params.Valid p) + (now : Millis) (s : PeerState) (id : TopicId) : + score p (invalidDelivery p s id) now ≤ score p s now := by + cases hcfg : p.topics id with + | none => simp [invalidDelivery, hcfg] + | some tp => + simp only [invalidDelivery, hcfg] + have htp := h.topic id tp hcfg + have hraw : + rawTopicPart p now + (setCounters s id (invalidStep (countersOf s id))) ≤ + rawTopicPart p now s := by + cases hlook : lookupCounters s.topics id with + | none => + have hu : countersOf s id = TopicCounters.zero := by + simp [countersOf, hlook] + rw [hu] + exact sum_map_upsert_le_of_absent (topicContribution p now) s.topics id + (invalidStep TopicCounters.zero) hlook + (topicContribution_invalidStep_zero_nonpos p now id hcfg + htp.topicWeight_nonneg htp.invalidMessageDeliveriesWeight_nonpos) + | some t => + have hu : countersOf s id = t := by + simp [countersOf, hlook] + rw [hu] + exact sum_map_upsert_le_of_found (topicContribution p now) s.topics id + (invalidStep t) t hlook + (topicContribution_invalidStep_le p now id hcfg + htp.topicWeight_nonneg htp.invalidMessageDeliveriesWeight_nonpos t) + have hpart := topicPart_mono hraw + have happ : (setCounters s id (invalidStep (countersOf s id))).appScore = + s.appScore := rfl + have hip : (setCounters s id (invalidStep (countersOf s id))).ipColocation = + s.ipColocation := rfl + have hbp : (setCounters s id (invalidStep (countersOf s id))).behaviourPenalty = + s.behaviourPenalty := rfl + exact score_le_of_topicPart_le hpart happ hip hbp + +/-- An invalid delivery to a topic with no configured parameters leaves the score +unchanged. -/ +theorem invalidDelivery_unconfigured_score {p : Params} {s : PeerState} {id : TopicId} + (hcfg : p.topics id = none) (now : Millis) : + score p (invalidDelivery p s id) now = score p s now := by + simp [invalidDelivery, hcfg] + +/-! ## The fresh peer -/ + +/-- The raw topic part of a fresh peer is 0. -/ +private theorem rawTopicPart_zero (p : Params) (now : Millis) : + rawTopicPart p now PeerState.zero = 0 := rfl + +/-- The topic part of a fresh peer is 0, for any parameters. -/ +private theorem topicPart_zero (p : Params) (now : Millis) : + topicPart p now PeerState.zero = 0 := by + have hraw := rawTopicPart_zero p now + dsimp only [topicPart] + rw [hraw] + split <;> omega + +/-- A fresh peer scores 0. The statement needs no `Params.Valid`: it holds for any +parameters. -/ +theorem freshPeer_scores_zero (p : Params) (now : Millis) : + score p PeerState.zero now = 0 := by + have hpart := topicPart_zero p now + unfold score + rw [hpart] + simp [p6, p7, PeerState.zero] + +/-! ## Prune -/ + +/-- A prune with an active delivery deficit adds the square of the deficit to the failure +penalty. The peer is in the mesh, the topic is configured, the activation window has passed, +and the delivery count is below the threshold. -/ +theorem prune_adds_deficit {p : Params} {s : PeerState} {now : Millis} {id : TopicId} + {tp : TopicParams} {t : TopicCounters} + (hcfg : p.topics id = some tp) (ht : countersOf s id = t) (hin : t.inMesh) + (hact : tp.meshMessageDeliveriesActivation < elapsed now t) + (hdef : t.meshMessageDeliveries < tp.meshMessageDeliveriesThreshold) : + (countersOf (prune p s now id) id).meshFailurePenalty = + t.meshFailurePenalty + + (tp.meshMessageDeliveriesThreshold - t.meshMessageDeliveries) * + (tp.meshMessageDeliveriesThreshold - t.meshMessageDeliveries) := by + have hprune : prune p s now id = setCounters s id (pruneCounters p now id t) := by + simp [prune, ht, hin] + rw [hprune] + simp [setCounters, countersOf, lookupCounters_upsert_same, pruneCounters, hcfg, hact, hdef] + +/-- A prune with no active delivery deficit leaves the failure penalty unchanged. -/ +theorem prune_without_deficit_keeps_penalty {p : Params} {s : PeerState} {now : Millis} + {id : TopicId} {tp : TopicParams} {t : TopicCounters} + (hcfg : p.topics id = some tp) (ht : countersOf s id = t) (hin : t.inMesh) + (hno : ¬ (tp.meshMessageDeliveriesActivation < elapsed now t ∧ + t.meshMessageDeliveries < tp.meshMessageDeliveriesThreshold)) : + (countersOf (prune p s now id) id).meshFailurePenalty = t.meshFailurePenalty := by + have hprune : prune p s now id = setCounters s id (pruneCounters p now id t) := by + simp [prune, ht, hin] + rw [hprune] + simp [setCounters, countersOf, lookupCounters_upsert_same, pruneCounters, hcfg, hno] + +/-- A prune removes a peer from the mesh. The statement needs no parameter hypothesis. -/ +theorem prune_leaves_mesh {p : Params} {s : PeerState} {now : Millis} {id : TopicId} + {t : TopicCounters} (ht : countersOf s id = t) (hin : t.inMesh) : + ¬ (countersOf (prune p s now id) id).inMesh := by + have hprune : prune p s now id = setCounters s id (pruneCounters p now id t) := by + simp [prune, ht, hin] + rw [hprune] + simp [setCounters, countersOf, lookupCounters_upsert_same] + cases hcfg : p.topics id with + | none => simp [pruneCounters, hcfg] + | some tp => + simp [pruneCounters, hcfg] + split <;> simp + +end Libp2p.Protocols.Gossipsub.Score diff --git a/packages/libp2p/protocols/Libp2p/Protocols/Gossipsub/Sign.lean b/packages/libp2p/protocols/Libp2p/Protocols/Gossipsub/Sign.lean new file mode 100644 index 00000000..eb357a16 --- /dev/null +++ b/packages/libp2p/protocols/Libp2p/Protocols/Gossipsub/Sign.lean @@ -0,0 +1,318 @@ +import Libp2p.Protocols.Gossipsub.Types +import Libp2p.Identity +import Bytes + +/-! +# Gossipsub signing policy + +This module holds the signing policy check, the bytes a signature +covers, and the default message id of gossipsub v1.1. The vendored +spec is `vendor/specs/pubsub/README.md`, sections Message Signing, +Signature Policy Options, and Message Identification. + +## Policy decision + +The router uses `strictNoSign` by default. The project targets the +Ethereum consensus layer, and that layer requires the StrictNoSign +policy for gossipsub v1.1 messages: +. +The libp2p pubsub spec defines the policy options: +. +Under `strictNoSign` the four signed fields are absent, so the default +message id is empty. The consensus layer supplies a content-addressed +id function through `MsgIdFn`. The hash itself is outside this module. + +The two lax options stay out. The spec marks them legacy, and the +project does not target them. + +## The signature input + +The bytes that a signature covers are the UTF-8 bytes of the string +`libp2p-pubsub:` followed by the encoded message with the `signature` +and `key` fields absent. The vendored spec excludes the signature +field (Message Signing). The go reference implementation also clears +the key field before it signs, so this module clears both fields. +The encoder is a parameter. The rpc module supplies the protobuf +encoder at integration. + +## Check order + +Under `strictSign` the check order is: `from`, `seqno`, `signature`, +key resolution, key match, verify. The key match applies when the +`key` field is present. An absent field produces its own error. Under +`strictNoSign` the check order is: `from`, `seqno`, `signature`, +`key`. The first present field decides the error. Both orders are +fixed here, so the error of a message with several faults is well +defined. + +The key resolution takes a key from the `key` field when that field is +present, and from the peer id otherwise. A `key` field that does not +parse never falls back to the peer id. This rule matches the go +reference. + +## The message id + +The default id is the concatenation of `from` and `seqno`. An absent +field adds no bytes. The spec calls this strategy origin stamping +(Message Identification). `MsgIdFn` is the type of a caller-supplied +id function. +-/ + +set_option autoImplicit false + +namespace Libp2p.Protocols.Gossipsub.Sign + +open Libp2p.Identity + +/-- The two deterministic signature policies of the spec. The spec +names no numeric codes for the policies, so this is a plain inductive +with no code pair. -/ +inductive SignPolicy where + /-- Require `from`, `seqno`, and `signature`, resolve the key, and + verify the signature. -/ + | strictSign + /-- Require the four signed fields to be absent. -/ + | strictNoSign + deriving DecidableEq, BEq, Repr + +/-- The rejection reasons of `validate`. Each arm names one reason. +The first six arms belong to `strictSign`, the last four to +`strictNoSign`. -/ +inductive SignError where + /-- `strictSign`: the `from` field is absent. -/ + | missingFrom + /-- `strictSign`: the `seqno` field is absent. -/ + | missingSeqno + /-- `strictSign`: the `signature` field is absent. -/ + | missingSignature + /-- `strictSign`: the `key` field does not parse, or the `from` + field inlines no key. -/ + | keyUnresolved + /-- `strictSign`: the key of the `key` field derives another peer + id than `from`. -/ + | keyMismatch + /-- `strictSign`: the verifier rejects the signature. -/ + | badSignature + /-- `strictNoSign`: the `from` field is present. -/ + | hasFrom + /-- `strictNoSign`: the `seqno` field is present. -/ + | hasSeqno + /-- `strictNoSign`: the `signature` field is present. -/ + | hasSignature + /-- `strictNoSign`: the `key` field is present. -/ + | hasKey + deriving DecidableEq, BEq, Repr + +/-- The signing prefix of the spec and of the go reference +implementation. -/ +def signPrefix : String := "libp2p-pubsub:" + +/-- The prefix as UTF-8 bytes. -/ +def signPrefixBytes : List Nat := Bytes.toNatList (Bytes.ofString signPrefix) + +/-- The functions that the router wires to the real packages at +integration. + +`unmarshalKey` parses the `key` field. `inlineKey` extracts a key that +the peer id inlines. `idOfKey` derives the peer id bytes of a key. +`verify` checks a signature over the signed bytes. All four functions +are total, and the caller supplies them. -/ +structure Prims where + /-- Parse the `key` field. -/ + unmarshalKey : List Nat → Option PubKey + /-- Extract the key that a peer id inlines. -/ + inlineKey : List Nat → Option PubKey + /-- Derive the peer id bytes of a key. -/ + idOfKey : PubKey → List Nat + /-- Check a signature over the signed bytes. -/ + verify : PubKey → List Nat → List Nat → Bool + +/-- The identity functions at the router boundary. A failed derivation +gives empty bytes, which cannot encode a peer id. The inline path reads +only an identity multihash and checks its public key protobuf. -/ +def Prims.identity : Prims where + unmarshalKey bs := (PubKey.unmarshal? (Bytes.ofNatList bs)).toOption + inlineKey bs := do + let peer ← (PeerId.ofBytes? (Bytes.ofNatList bs)).toOption + if peer.hash.code != .«identity» then none + else (PubKey.unmarshal? (Bytes.ofNatList peer.hash.digest)).toOption + idOfKey k := ((PeerId.ofPubKey k).toOption.map + (fun peer => Bytes.toNatList peer.toBytes)).getD [] + verify k msg sig := k.verify (Bytes.ofNatList msg) (Bytes.ofNatList sig) + +/-- The bytes a signature covers: the prefix, then the encoded message +with the `signature` and `key` fields absent. -/ +def signInput (encode : Message → List Nat) (m : Message) : List Nat := + signPrefixBytes ++ encode { m with signature := none, key := none } + +/-- Check a message against a signature policy. + +Under `strictNoSign`, the first present field of `from`, `seqno`, +`signature`, and `key` produces its error arm. + +Under `strictSign`, the checks run in this order: `from`, `seqno`, +`signature`, key resolution, key match, verify. A `key` field that is +present must parse, and the derived peer id must equal `from`. An +absent `key` field resolves through the peer id. -/ +def validate (p : Prims) (encode : Message → List Nat) (policy : SignPolicy) + (m : Message) : Except SignError Unit := + match policy, m with + | .strictNoSign, m => + if m.from.isSome then .error .hasFrom + else if m.seqno.isSome then .error .hasSeqno + else if m.signature.isSome then .error .hasSignature + else if m.key.isSome then .error .hasKey + else .ok () + | .strictSign, m => + match m.from with + | none => .error .missingFrom + | some fr => + match m.seqno with + | none => .error .missingSeqno + | some _ => + match m.signature with + | none => .error .missingSignature + | some sig => + match m.key with + | some kb => + match p.unmarshalKey kb with + | none => .error .keyUnresolved + | some k => + if p.idOfKey k = fr then + if p.verify k (signInput encode m) sig then .ok () + else .error .badSignature + else .error .keyMismatch + | none => + match p.inlineKey fr with + | none => .error .keyUnresolved + | some k => + if p.verify k (signInput encode m) sig then .ok () + else .error .badSignature + +/-- A caller-supplied message id function. The consensus layer supplies +a content-addressed function. -/ +abbrev MsgIdFn := Message → MessageId + +/-- The default message id: `from` followed by `seqno`. An absent field +adds no bytes. -/ +def defaultMsgId (m : Message) : MessageId := + ⟨m.from.getD [] ++ m.seqno.getD []⟩ + +/-! ## Theorems -/ + +/-- The UTF-8 bytes of the prefix are the expected 14 bytes. -/ +theorem signPrefixBytes_eq : + signPrefixBytes = + [108, 105, 98, 112, 50, 112, 45, 112, 117, 98, 115, 117, 98, 58] := rfl + +/-- `strictNoSign` accepts a message exactly when the four signed fields +are absent. -/ +theorem validate_strictNoSign_iff (p : Prims) (encode : Message → List Nat) + (m : Message) : + validate p encode .strictNoSign m = .ok () ↔ + m.from = none ∧ m.seqno = none ∧ m.signature = none ∧ m.key = none := by + rcases m with ⟨fr, d, sq, t, sig, key⟩ + cases fr <;> cases sq <;> cases sig <;> cases key <;> simp [validate] + +/-- `strictSign` acceptance implies that `from`, `seqno`, and +`signature` are present, and that the verifier accepted the signature +over the signed bytes. -/ +theorem validate_strictSign_sound (p : Prims) (encode : Message → List Nat) + (m : Message) : + validate p encode .strictSign m = .ok () → + m.from.isSome ∧ m.seqno.isSome ∧ + ∃ k sig, m.signature = some sig ∧ + p.verify k (signInput encode m) sig = true := by + intro h + rcases m with ⟨fr, d, sq, t, sig, key⟩ + cases fr with + | none => simp [validate] at h + | some fr => + cases sq with + | none => simp [validate] at h + | some sq => + cases sig with + | none => simp [validate] at h + | some sig => + cases key with + | none => + cases hik : p.inlineKey fr with + | none => simp [validate, hik] at h + | some k => + simp [validate, hik] at h + exact ⟨rfl, rfl, k, sig, rfl, h⟩ + | some kb => + cases huk : p.unmarshalKey kb with + | none => simp [validate, huk] at h + | some k => + by_cases hm : p.idOfKey k = fr + · simp [validate, huk, hm] at h + exact ⟨rfl, rfl, k, sig, rfl, h⟩ + · simp [validate, huk, hm] at h + +/-- Under `strictSign`, an accepted message with a `key` field has a +key that parses and derives the peer id in `from`. This theorem pins +the key match check, which the soundness theorem does not name. -/ +theorem validate_strictSign_keyField_binding (p : Prims) + (encode : Message → List Nat) (m : Message) : + validate p encode .strictSign m = .ok () → m.key.isSome → + ∃ kb k, m.key = some kb ∧ p.unmarshalKey kb = some k ∧ + m.from = some (p.idOfKey k) := by + intro h hsome + rcases m with ⟨fr, d, sq, t, sig, key⟩ + cases fr with + | none => simp [validate] at h + | some fr => + cases sq with + | none => simp [validate] at h + | some sq => + cases sig with + | none => simp [validate] at h + | some sig => + cases key with + | none => simp at hsome + | some kb => + cases huk : p.unmarshalKey kb with + | none => simp [validate, huk] at h + | some k => + by_cases hm : p.idOfKey k = fr + · simp [validate, huk, hm] at h + exact ⟨kb, k, rfl, huk, by simp [hm]⟩ + · simp [validate, huk, hm] at h + +/-- The bytes of the signature input do not depend on the `signature` +and `key` fields. -/ +theorem signInput_signature_key_irrelevant (encode : Message → List Nat) + (m : Message) (s s' k k' : Option (List Nat)) : + signInput encode { m with signature := s, key := k } = + signInput encode { m with signature := s', key := k' } := by + simp [signInput] + +/-- The signature input starts with the prefix bytes. -/ +theorem signInput_prefix (encode : Message → List Nat) (m : Message) : + (signInput encode m).take signPrefixBytes.length = signPrefixBytes := by + simp [signInput] + +/-- The default message id ignores the payload and the topic. -/ +theorem defaultMsgId_ignores_payload (m : Message) (d' : Option (List Nat)) + (t' : TopicId) : + defaultMsgId { m with data := d', topic := t' } = defaultMsgId m := rfl + +/-- An accepted `strictNoSign` message has the empty default message +id. -/ +theorem strictNoSign_ok_defaultMsgId_empty (p : Prims) + (encode : Message → List Nat) (m : Message) : + validate p encode .strictNoSign m = .ok () → defaultMsgId m = ⟨[]⟩ := by + intro h + have h' := (validate_strictNoSign_iff p encode m).mp h + simp [defaultMsgId, h'.1, h'.2.1] + +/-- The ten rejection arms are pairwise distinct. -/ +theorem signError_arms_nodup : + [SignError.missingFrom, SignError.missingSeqno, SignError.missingSignature, + SignError.keyUnresolved, SignError.keyMismatch, SignError.badSignature, + SignError.hasFrom, SignError.hasSeqno, SignError.hasSignature, + SignError.hasKey].Nodup := by + decide + +end Libp2p.Protocols.Gossipsub.Sign diff --git a/packages/libp2p/protocols/Libp2p/Protocols/Gossipsub/Types.lean b/packages/libp2p/protocols/Libp2p/Protocols/Gossipsub/Types.lean new file mode 100644 index 00000000..c2fd2ec9 --- /dev/null +++ b/packages/libp2p/protocols/Libp2p/Protocols/Gossipsub/Types.lean @@ -0,0 +1,83 @@ +import Libp2p.Identity + +/-! +# Gossipsub shared types + +The types that every gossipsub module reads: the topic and message ids, +the published message, the score value and its thresholds, and the time +unit. The RPC codec, the scoring function, the caches, the signing +policy, and the router core all build on these, so they live in one +module that imports nothing else of gossipsub. + +A score is an `Int` in fixed point: `scoreScale` units make 1.0 of the +pubsub spec. The spec states scores as float64. Fixed point keeps the +arithmetic exact, so `omega` can close the score theorems. + +Time enters the router as data. `Millis` is the time of an `Event.tick` +in milliseconds. No gossipsub module reads a clock. +-/ + +set_option autoImplicit false + +namespace Libp2p.Protocols.Gossipsub + +open Libp2p.Identity + +/-- A topic, by its name. -/ +structure TopicId where + name : String + deriving DecidableEq, BEq, Hashable, Repr + +/-- A message id: the bytes the message id function returns. -/ +structure MessageId where + bytes : List Nat + deriving DecidableEq, BEq, Hashable, Repr + +/-- A published message, with the fields of the pubsub spec. Each +optional field of the protobuf is an `Option`, so an absent field and +an empty field stay apart. -/ +structure Message where + /-- The author's peer id bytes. -/ + «from» : Option (List Nat) + /-- The payload. -/ + data : Option (List Nat) + /-- The sequence number bytes. -/ + seqno : Option (List Nat) + /-- The topic, a required field. -/ + topic : TopicId + /-- The signature over the message. -/ + signature : Option (List Nat) + /-- The author's public key, when the peer id does not inline it. -/ + key : Option (List Nat) + deriving DecidableEq, BEq, Repr + +/-- A peer score, in fixed point: `scoreScale` units per 1.0. -/ +abbrev Score := Int + +/-- The fixed-point scale of `Score`. -/ +def scoreScale : Nat := 1000 + +/-- The score thresholds of gossipsub v1.1. The router compares a peer's +score with them. The spec leaves the values to the application. -/ +structure Thresholds where + /-- Below this score, the router emits no gossip to the peer and + ignores its gossip. -/ + gossipThreshold : Score + /-- Below this score, the router does not publish to the peer. -/ + publishThreshold : Score + /-- Below this score, the router ignores every RPC from the peer. -/ + graylistThreshold : Score + /-- At or above this score, the router accepts peer exchange from + the peer's prune. -/ + acceptPXThreshold : Score + /-- Below this median mesh score, the router grafts opportunistically. -/ + opportunisticGraftThreshold : Score + deriving DecidableEq, Repr + +/-- A point in time, in milliseconds. -/ +abbrev Millis := Nat + +/-- The validated peer id of the identity package. -/ +abbrev Peer := Libp2p.Identity.PeerId + +end Libp2p.Protocols.Gossipsub diff --git a/packages/libp2p/protocols/docs/PLAN.md b/packages/libp2p/protocols/docs/PLAN.md index 8adc6850..38dfa427 100644 --- a/packages/libp2p/protocols/docs/PLAN.md +++ b/packages/libp2p/protocols/docs/PLAN.md @@ -122,9 +122,43 @@ in the codec group wrapping upstream `google/snappy`. field tables of the vendored gossipsub spec, both directions. - The router core as transcript tests, `Event.tick` advancing the heartbeat: mesh graft and prune transitions, and iwant/ihave - bookkeeping, at the level the root names (root PLAN W6.4). + bookkeeping (root PLAN W6.4). - The signing-policy tests, once unit 3b records the choice against the two axes of the pubsub spec. +- Unit 3b owns the trickle regressions that issue #23's done-when names. + Tests/Gossipsub/Node.lean pins the IHAVE deadlines on a test clock. + The checks cover configured delays across heartbeats and the default + zero-delay schedule. +- The loop suite, `Tests/Gossipsub/Node.lean`, runs over the in-memory + switch of `Tests/Gossipsub/Switch.lean`, a double of the real + `Libp2p.Core` handler seam: + - One publish crosses two nodes and reaches the callback. + - Subscribe registers the callback before posting its event. A refused + post takes back its own registration by id. + - A subscribe after the join reaches the peer, and an unsubscribe sends + the negative record. + - A bad frame fails its handler and resets its stream. A body that does + not decode leaves the loop running. + - The handler serves every frame of a stream, in any chunking, and resets + the stream on a bad frame. + - Each peer has a writer task and a 32-slot queue. Two publishers run in + parallel with one stream each and zero adjacent inversions per peer. + A full queue drops frames, and the suite reports the drop count. + - A parked peer writer leaves the loop free. Stop resets its stream, + closes the peer queues and alarm, and joins the owned tasks. + - The handler registers its opener before the first RPC. Repeat joins + emit no hello. Subscription records precede grafts. + - The replayable hash order changes across heartbeat counters. + - Pending gossip holds at most 256 sends. Excess sends increment the + drop count. Due sends leave before the heartbeat shifts the cache. + - The receipt observer sees duplicates before validation and delivery. + - The default gossip delay is zero. Tests supply an explicit spread. + - A heartbeat sends the graft, and a later publish reaches the mesh. + - With a configured spread, each IHAVE leaves at its run-wide indexed + delay and no earlier. The check crosses two heartbeat intervals. + - A publish with no peers opens no stream and delivers nothing locally. + - `stop` joins the loop, a later post answers `false`, and the switch + closes. - The gate (unit 3c): the interop suite against go and rust runs with no behavior difference, per root PLAN W6.4. This is a live suite over foreign nodes, and it closes the unit where no vector file can. diff --git a/packages/libp2p/protocols/tools/go-rpc-fixtures/go.mod b/packages/libp2p/protocols/tools/go-rpc-fixtures/go.mod new file mode 100644 index 00000000..75837f52 --- /dev/null +++ b/packages/libp2p/protocols/tools/go-rpc-fixtures/go.mod @@ -0,0 +1,8 @@ +module leanp2p-protocols-rpc-fixtures + +go 1.24.1 + +require ( + github.com/gogo/protobuf v1.3.2 + github.com/libp2p/go-libp2p-pubsub v0.15.1-0.20260305223556-a01777894f4c +) \ No newline at end of file diff --git a/packages/libp2p/protocols/tools/go-rpc-fixtures/go.sum b/packages/libp2p/protocols/tools/go-rpc-fixtures/go.sum new file mode 100644 index 00000000..c668fd58 --- /dev/null +++ b/packages/libp2p/protocols/tools/go-rpc-fixtures/go.sum @@ -0,0 +1,33 @@ +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/libp2p/go-libp2p-pubsub v0.15.1-0.20260305223556-a01777894f4c h1:rcinqzooEODok/JNrp5xpgHbqre3qR3ww7u0fDMkTVw= +github.com/libp2p/go-libp2p-pubsub v0.15.1-0.20260305223556-a01777894f4c/go.mod h1:lr4oE8bFgQaifRcoc2uWhWWiK6tPdOEKpUuR408GFN4= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/packages/libp2p/protocols/tools/go-rpc-fixtures/main.go b/packages/libp2p/protocols/tools/go-rpc-fixtures/main.go new file mode 100644 index 00000000..9c15f3b6 --- /dev/null +++ b/packages/libp2p/protocols/tools/go-rpc-fixtures/main.go @@ -0,0 +1,486 @@ +package main + +import ( + "bytes" + "encoding/hex" + "encoding/json" + "fmt" + "os" + + "github.com/gogo/protobuf/proto" + pb "github.com/libp2p/go-libp2p-pubsub/pb" +) + +func strPtr(s string) *string { return &s } +func boolPtr(b bool) *bool { return &b } +func uint64Ptr(u uint64) *uint64 { return &u } + +type CaseJSON struct { + Name string `json:"name"` + Hex string `json:"hex"` + Fields FieldsJSON `json:"fields"` + Roundtrip bool `json:"roundtrip"` +} + +type FieldsJSON struct { + Subscriptions []SubscriptionFields `json:"subscriptions,omitempty"` + Publish []PublishFields `json:"publish,omitempty"` + Control *ControlFields `json:"control,omitempty"` +} + +type SubscriptionFields struct { + Topicid string `json:"topicid"` + Subscribe bool `json:"subscribe"` +} + +type PublishFields struct { + TopicStr *string `json:"topic_str,omitempty"` + TopicBytesHex *string `json:"topic_bytes_hex,omitempty"` + DataHex *string `json:"data_hex,omitempty"` + SeqnoHex *string `json:"seqno_hex,omitempty"` + SignatureHex *string `json:"signature_hex,omitempty"` + KeyHex *string `json:"key_hex,omitempty"` +} + +type ControlFields struct { + Graft []GraftFields `json:"graft,omitempty"` + Prune []PruneFields `json:"prune,omitempty"` + Ihave []IhaveFields `json:"ihave,omitempty"` + Iwant []IwantFields `json:"iwant,omitempty"` +} + +type GraftFields struct { + TopicBytesHex string `json:"topic_bytes_hex"` +} + +type PruneFields struct { + TopicBytesHex string `json:"topic_bytes_hex"` + PeersHexList []string `json:"peers_hex_list,omitempty"` + Backoff *uint64 `json:"backoff,omitempty"` +} + +type IhaveFields struct { + TopicBytesHex string `json:"topic_bytes_hex"` + MessageIDsHex []string `json:"message_ids_hex"` +} + +type IwantFields struct { + MessageIDsHex []string `json:"message_ids_hex"` +} + +// Command go-rpc-fixtures writes Tests/fixtures/gossipsub-rpc-go.json. +// +// The gossipsub RPC bytes of this tree are pinned against the protobuf +// marshaler of the go implementation that the interop gate runs +// (vendor/test-plans/gossipsub-interop/go-libp2p/go.mod). Each case records the +// marshaled payload, the semantic values the program set, and a roundtrip flag +// that proto.Unmarshal and proto.Equal confirm before the file is written. +// +// Run from this directory with the Go toolchain the gate image carries: +// +// go run . ../../../../../Tests/fixtures/gossipsub-rpc-go.json +func main() { + var cases []CaseJSON + + topicTest := "test-topic" + topicTestHex := hex.EncodeToString([]byte(topicTest)) + dataBasic := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07} + dataBasicHex := hex.EncodeToString(dataBasic) + + msgId1 := []byte{1, 0, 0, 0, 0, 0, 0, 0} + msgId2 := []byte{2, 0, 0, 0, 0, 0, 0, 0} + msgId1Hex := hex.EncodeToString(msgId1) + msgId2Hex := hex.EncodeToString(msgId2) + + // 1. sub-opt-subscribe: two SubOpts, one true, one false, topics "test-topic" and "t" + { + rpc := &pb.RPC{ + Subscriptions: []*pb.RPC_SubOpts{ + {Topicid: strPtr("test-topic"), Subscribe: boolPtr(true)}, + {Topicid: strPtr("t"), Subscribe: boolPtr(false)}, + }, + } + raw, err := proto.Marshal(rpc) + if err != nil { + panic(fmt.Sprintf("case 1 marshal failed: %v", err)) + } + var decoded pb.RPC + if err := proto.Unmarshal(raw, &decoded); err != nil { + panic(fmt.Sprintf("case 1 unmarshal failed: %v", err)) + } + rt := proto.Equal(rpc, &decoded) + if !rt { + panic("case 1 roundtrip failed") + } + cases = append(cases, CaseJSON{ + Name: "sub-opt-subscribe", + Hex: hex.EncodeToString(raw), + Fields: FieldsJSON{ + Subscriptions: []SubscriptionFields{ + {Topicid: "test-topic", Subscribe: true}, + {Topicid: "t", Subscribe: false}, + }, + }, + Roundtrip: rt, + }) + } + + // 2. publish-basic: one Message with data 8 bytes 00..07, topic "test-topic", from/seqno/signature/key absent + var bPublishBasic []byte + var fieldsPublishBasic PublishFields + { + rpc := &pb.RPC{ + Publish: []*pb.Message{ + { + Topic: strPtr(topicTest), + Data: dataBasic, + }, + }, + } + raw, err := proto.Marshal(rpc) + if err != nil { + panic(fmt.Sprintf("case 2 marshal failed: %v", err)) + } + bPublishBasic = raw + var decoded pb.RPC + if err := proto.Unmarshal(raw, &decoded); err != nil { + panic(fmt.Sprintf("case 2 unmarshal failed: %v", err)) + } + rt := proto.Equal(rpc, &decoded) + if !rt { + panic("case 2 roundtrip failed") + } + fieldsPublishBasic = PublishFields{ + TopicStr: strPtr(topicTest), + TopicBytesHex: strPtr(topicTestHex), + DataHex: strPtr(dataBasicHex), + } + cases = append(cases, CaseJSON{ + Name: "publish-basic", + Hex: hex.EncodeToString(raw), + Fields: FieldsJSON{ + Publish: []PublishFields{fieldsPublishBasic}, + }, + Roundtrip: rt, + }) + } + + // 3. publish-empty-data: one Message with data absent, topic "test-topic" + { + rpc := &pb.RPC{ + Publish: []*pb.Message{ + { + Topic: strPtr(topicTest), + Data: nil, + }, + }, + } + raw, err := proto.Marshal(rpc) + if err != nil { + panic(fmt.Sprintf("case 3 marshal failed: %v", err)) + } + var decoded pb.RPC + if err := proto.Unmarshal(raw, &decoded); err != nil { + panic(fmt.Sprintf("case 3 unmarshal failed: %v", err)) + } + rt := proto.Equal(rpc, &decoded) + if !rt { + panic("case 3 roundtrip failed") + } + cases = append(cases, CaseJSON{ + Name: "publish-empty-data", + Hex: hex.EncodeToString(raw), + Fields: FieldsJSON{ + Publish: []PublishFields{ + { + TopicStr: strPtr(topicTest), + TopicBytesHex: strPtr(topicTestHex), + }, + }, + }, + Roundtrip: rt, + }) + } + + // 4. control-graft: control with one Graft for "test-topic" + { + rpc := &pb.RPC{ + Control: &pb.ControlMessage{ + Graft: []*pb.ControlGraft{ + {TopicID: strPtr(topicTest)}, + }, + }, + } + raw, err := proto.Marshal(rpc) + if err != nil { + panic(fmt.Sprintf("case 4 marshal failed: %v", err)) + } + var decoded pb.RPC + if err := proto.Unmarshal(raw, &decoded); err != nil { + panic(fmt.Sprintf("case 4 unmarshal failed: %v", err)) + } + rt := proto.Equal(rpc, &decoded) + if !rt { + panic("case 4 roundtrip failed") + } + cases = append(cases, CaseJSON{ + Name: "control-graft", + Hex: hex.EncodeToString(raw), + Fields: FieldsJSON{ + Control: &ControlFields{ + Graft: []GraftFields{ + {TopicBytesHex: topicTestHex}, + }, + }, + }, + Roundtrip: rt, + }) + } + + // 5. control-prune: control with one Prune for "test-topic", backoff 60, one peer id bytes [1,2,3] + { + peerBytes := []byte{1, 2, 3} + rpc := &pb.RPC{ + Control: &pb.ControlMessage{ + Prune: []*pb.ControlPrune{ + { + TopicID: strPtr(topicTest), + Backoff: uint64Ptr(60), + Peers: []*pb.PeerInfo{ + {PeerID: peerBytes}, + }, + }, + }, + }, + } + raw, err := proto.Marshal(rpc) + if err != nil { + panic(fmt.Sprintf("case 5 marshal failed: %v", err)) + } + var decoded pb.RPC + if err := proto.Unmarshal(raw, &decoded); err != nil { + panic(fmt.Sprintf("case 5 unmarshal failed: %v", err)) + } + rt := proto.Equal(rpc, &decoded) + if !rt { + panic("case 5 roundtrip failed") + } + cases = append(cases, CaseJSON{ + Name: "control-prune", + Hex: hex.EncodeToString(raw), + Fields: FieldsJSON{ + Control: &ControlFields{ + Prune: []PruneFields{ + { + TopicBytesHex: topicTestHex, + Backoff: uint64Ptr(60), + PeersHexList: []string{hex.EncodeToString(peerBytes)}, + }, + }, + }, + }, + Roundtrip: rt, + }) + } + + // 6. control-ihave: control with one IHave for "test-topic" and two message ids of 8 bytes each + { + rpc := &pb.RPC{ + Control: &pb.ControlMessage{ + Ihave: []*pb.ControlIHave{ + { + TopicID: strPtr(topicTest), + MessageIDs: []string{string(msgId1), string(msgId2)}, + }, + }, + }, + } + raw, err := proto.Marshal(rpc) + if err != nil { + panic(fmt.Sprintf("case 6 marshal failed: %v", err)) + } + var decoded pb.RPC + if err := proto.Unmarshal(raw, &decoded); err != nil { + panic(fmt.Sprintf("case 6 unmarshal failed: %v", err)) + } + rt := proto.Equal(rpc, &decoded) + if !rt { + panic("case 6 roundtrip failed") + } + cases = append(cases, CaseJSON{ + Name: "control-ihave", + Hex: hex.EncodeToString(raw), + Fields: FieldsJSON{ + Control: &ControlFields{ + Ihave: []IhaveFields{ + { + TopicBytesHex: topicTestHex, + MessageIDsHex: []string{msgId1Hex, msgId2Hex}, + }, + }, + }, + }, + Roundtrip: rt, + }) + } + + // 7. control-iwant: control with one IWant holding the same two ids + { + rpc := &pb.RPC{ + Control: &pb.ControlMessage{ + Iwant: []*pb.ControlIWant{ + { + MessageIDs: []string{string(msgId1), string(msgId2)}, + }, + }, + }, + } + raw, err := proto.Marshal(rpc) + if err != nil { + panic(fmt.Sprintf("case 7 marshal failed: %v", err)) + } + var decoded pb.RPC + if err := proto.Unmarshal(raw, &decoded); err != nil { + panic(fmt.Sprintf("case 7 unmarshal failed: %v", err)) + } + rt := proto.Equal(rpc, &decoded) + if !rt { + panic("case 7 roundtrip failed") + } + cases = append(cases, CaseJSON{ + Name: "control-iwant", + Hex: hex.EncodeToString(raw), + Fields: FieldsJSON{ + Control: &ControlFields{ + Iwant: []IwantFields{ + { + MessageIDsHex: []string{msgId1Hex, msgId2Hex}, + }, + }, + }, + }, + Roundtrip: rt, + }) + } + + // 8. mixed: subscriptions plus one message plus a control with one Graft, to pin field order + { + rpc := &pb.RPC{ + Subscriptions: []*pb.RPC_SubOpts{ + {Topicid: strPtr("test-topic"), Subscribe: boolPtr(true)}, + {Topicid: strPtr("t"), Subscribe: boolPtr(false)}, + }, + Publish: []*pb.Message{ + { + Topic: strPtr(topicTest), + Data: dataBasic, + }, + }, + Control: &pb.ControlMessage{ + Graft: []*pb.ControlGraft{ + {TopicID: strPtr(topicTest)}, + }, + }, + } + raw, err := proto.Marshal(rpc) + if err != nil { + panic(fmt.Sprintf("case 8 marshal failed: %v", err)) + } + var decoded pb.RPC + if err := proto.Unmarshal(raw, &decoded); err != nil { + panic(fmt.Sprintf("case 8 unmarshal failed: %v", err)) + } + rt := proto.Equal(rpc, &decoded) + if !rt { + panic("case 8 roundtrip failed") + } + cases = append(cases, CaseJSON{ + Name: "mixed", + Hex: hex.EncodeToString(raw), + Fields: FieldsJSON{ + Subscriptions: []SubscriptionFields{ + {Topicid: "test-topic", Subscribe: true}, + {Topicid: "t", Subscribe: false}, + }, + Publish: []PublishFields{fieldsPublishBasic}, + Control: &ControlFields{ + Graft: []GraftFields{ + {TopicBytesHex: topicTestHex}, + }, + }, + }, + Roundtrip: rt, + }) + } + + // 9. unknown-fields: "publish-basic" hex with an extra appended field + // Field number 9, wire type 2 (length-delimited), payload 4 bytes (0xde, 0xad, 0xbe, 0xef) + // Tag = (9 << 3) | 2 = 74 = 0x4a, Length = 0x04 + { + extraField := []byte{0x4a, 0x04, 0xde, 0xad, 0xbe, 0xef} + raw := append(bytes.Clone(bPublishBasic), extraField...) + + var decoded1 pb.RPC + if err := proto.Unmarshal(raw, &decoded1); err != nil { + panic(fmt.Sprintf("case 9 unmarshal failed: %v", err)) + } + var decoded2 pb.RPC + if err := proto.Unmarshal(raw, &decoded2); err != nil { + panic(fmt.Sprintf("case 9 second unmarshal failed: %v", err)) + } + rt := proto.Equal(&decoded1, &decoded2) + if !rt { + panic("case 9 roundtrip failed") + } + // Confirm re-marshaling produces exact original bytes including unknown field + reEncoded, err := proto.Marshal(&decoded1) + if err != nil || !bytes.Equal(reEncoded, raw) { + panic("case 9 re-encoding did not match raw bytes") + } + + cases = append(cases, CaseJSON{ + Name: "unknown-fields", + Hex: hex.EncodeToString(raw), + Fields: FieldsJSON{ + Publish: []PublishFields{fieldsPublishBasic}, + }, + Roundtrip: rt, + }) + } + + // Verify all roundtrips + for _, c := range cases { + if !c.Roundtrip { + fmt.Fprintf(os.Stderr, "ERROR: case %s roundtrip failed\n", c.Name) + os.Exit(1) + } + } + + outPath := "../../../../../Tests/fixtures/gossipsub-rpc-go.json" + if len(os.Args) > 1 { + outPath = os.Args[1] + } + + provenance := map[string]string{ + "generatedBy": "packages/libp2p/protocols/tools/go-rpc-fixtures/main.go", + "module": "github.com/libp2p/go-libp2p-pubsub, pb package", + "version": "v0.15.1-0.20260305223556-a01777894f4c", + "api": "proto.Marshal of pb.RPC values, then proto.Unmarshal and proto.Equal for the roundtrip field", + "format": "the RPC payload of one gossipsub frame, as lowercase hex; the uvarint prefix is Streams.Framing", + "source": "vendor/test-plans/gossipsub-interop/go-libp2p/go.mod:8 pins the module version", + } + out := struct { + Provenance map[string]string `json:"_provenance"` + Cases []CaseJSON `json:"cases"` + }{provenance, cases} + buf, err := json.MarshalIndent(out, "", " ") + if err != nil { + panic(err) + } + buf = append(buf, '\n') + + if err := os.WriteFile(outPath, buf, 0644); err != nil { + panic(err) + } + fmt.Printf("Generated %d cases to %s\n", len(cases), outPath) +}