rust-create-cascade: add support for TLS-style encoding of clubcards

rust-create-cascade/Cargo.toml

@@ -14,6 +14,6 @@ rust_cascade = { version = "1.5.0" , features = ["builder"] }
 statsd = "0.16.0"
 stderrlog = "0.5"
 tempfile = "3.10.1"
-clubcard = { version = "0.3", features = ["builder"] }
-clubcard-crlite = { version = "0.3.2", features = ["builder"] }
+clubcard = { version = "0.3.3", features = ["builder"] }
+clubcard-crlite = { version = "0.4.0", features = ["builder"] }
 serde_json = "1"

rust-create-cascade/src/clubcard_helper.rs

@@ -8,7 +8,9 @@ use crate::{
 };
 use clubcard::{builder::*, Clubcard};
 
-use clubcard_crlite::{builder::*, CRLiteClubcard, CRLiteCoverage, CRLiteKey, CRLiteQuery};
+use clubcard_crlite::{
+    builder::*, CRLiteClubcard, CRLiteCoverage, CRLiteKey, CRLiteQuery, Encoding, IssuerSpkiHash,
+};
 
 use log::*;
 use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
@@ -31,7 +33,7 @@ fn clubcard_do_one_issuer(
     for (_expiry, serial) in known_serials {
         universe_size += 1;
         if revoked_serial_set.contains(&serial) {
-            let key = CRLiteBuilderItem::revoked(*issuer, decode_serial(&serial));
+            let key = CRLiteBuilderItem::revoked(IssuerSpkiHash(*issuer), decode_serial(&serial));
             ribbon_builder.insert(key);
             // Ensure that we do not attempt to include this issuer+serial again.
             revoked_serial_set.remove(&serial);
@@ -98,9 +100,9 @@ impl FilterBuilder for ClubcardBuilder<4, CRLiteBuilderItem> {
         for (_expiry, serial) in known_serials {
             let serial_bytes = decode_serial(&serial);
             let key = if revoked_serial_set.contains(&serial) {
-                CRLiteBuilderItem::revoked(*issuer, serial_bytes)
+                CRLiteBuilderItem::revoked(IssuerSpkiHash(*issuer), serial_bytes)
             } else {
-                CRLiteBuilderItem::not_revoked(*issuer, serial_bytes)
+                CRLiteBuilderItem::not_revoked(IssuerSpkiHash(*issuer), serial_bytes)
             };
             ribbon_builder.insert(key);
         }
@@ -128,9 +130,10 @@ impl CheckableFilter for CRLiteClubcard {
             .map(|iter| iter.into())
             .unwrap_or_default();
 
+        let issuer_hash = IssuerSpkiHash(*issuer);
         for (_expiry, serial) in known_serials {
             let decoded_serial = decode_serial(&serial);
-            let key = CRLiteKey::new(issuer, &decoded_serial);
+            let key = CRLiteKey::new(&issuer_hash, &decoded_serial);
             let query = CRLiteQuery::new(&key, None);
             assert!(
                 Clubcard::unchecked_contains(self.as_ref(), &query)
@@ -146,6 +149,7 @@ pub fn create_clubcard(
     known_dir: &Path,
     coverage_path: &Path,
     reason_set: ReasonSet,
+    encoding: Encoding,
 ) -> Vec<u8> {
     let coverage = CRLiteCoverage::from_mozilla_ct_logs_json(BufReader::new(
         std::fs::File::open(coverage_path).unwrap(),
@@ -165,7 +169,7 @@ pub fn create_clubcard(
     info!("Generated {}", clubcard);
 
     info!("Testing serialization");
-    let clubcard_bytes = clubcard.to_bytes().expect("cannot serialize clubcard");
+    let clubcard_bytes = clubcard.to_bytes(encoding).expect("cannot serialize clubcard");
     info!("Clubcard is {} bytes", clubcard_bytes.len());
 
     let clubcard =

rust-create-cascade/src/main.rs

@@ -682,6 +682,25 @@ enum FilterType {
     Clubcard,
 }
 
+/// Serialization format for clubcard filters. Ignored for cascade filters.
+///
+/// `bincode` is the legacy V3 encoding; `tls` is the V4 encoding that uses a
+/// TLS-presentation-language-style codec.
+#[derive(clap::ValueEnum, Copy, Clone, PartialEq)]
+enum ClubcardEncoding {
+    Bincode,
+    Tls,
+}
+
+impl From<ClubcardEncoding> for clubcard_crlite::Encoding {
+    fn from(encoding: ClubcardEncoding) -> clubcard_crlite::Encoding {
+        match encoding {
+            ClubcardEncoding::Bincode => clubcard_crlite::Encoding::V3,
+            ClubcardEncoding::Tls => clubcard_crlite::Encoding::V4,
+        }
+    }
+}
+
 #[derive(Parser)]
 struct Cli {
     #[clap(long, parse(from_os_str), default_value = "./known/")]
@@ -704,6 +723,8 @@ struct Cli {
     murmurhash3: bool,
     #[clap(long, value_enum, default_value = "cascade")]
     filter_type: FilterType,
+    #[clap(long, value_enum, default_value = "bincode")]
+    encoding: ClubcardEncoding,
     #[clap(long)]
     clobber: bool,
     #[clap(short = 'v', parse(from_occurrences))]
@@ -727,6 +748,7 @@ fn main() {
     let reason_set = args.reason_set;
     let delta_reason_set = args.delta_reason_set;
     let filter_type = args.filter_type;
+    let encoding = args.encoding;
     let ct_logs_json = &args.ct_logs_json;
 
     let out_dir = &args.outdir;
@@ -848,6 +870,7 @@ fn main() {
                 known_dir,
                 ct_logs_json,
                 reason_set,
+                encoding.into(),
             )
         }
         FilterType::Cascade => {
@@ -915,6 +938,7 @@ fn main() {
                 known_dir,
                 ct_logs_json,
                 delta_reason_set,
+                encoding.into(),
             )
         }
         FilterType::Cascade => {
@@ -1006,7 +1030,7 @@ mod tests {
         decode_issuer, decode_serial, write_revset_and_delta, write_stash, CheckableFilter, Reason,
         ReasonSet,
     };
-    use clubcard_crlite::CRLiteClubcard;
+    use clubcard_crlite::{CRLiteClubcard, Encoding};
     use rand::rngs::OsRng;
     use rand::RngCore;
     use rust_cascade::{Cascade, HashAlgorithm};
@@ -1352,6 +1376,7 @@ mod tests {
             &env.known_dir(),
             &env.ct_logs_path(),
             ReasonSet::All,
+            Encoding::V4,
         );
         assert!(
             Cascade::from_bytes(clubcard_bytes).is_err(),

rust-query-crlite/Cargo.toml

@@ -9,7 +9,7 @@ bincode = "1.3"
 byteorder = "1.2.7"
 clap = { version = "4.5", features = ["derive"] }
 clubcard = "0.3"
-clubcard-crlite = "0.3"
+clubcard-crlite = "0.4"
 der-parser = "9.0"
 hex = "0.4"
 log = "0.4"

rust-query-crlite/src/main.rs

@@ -19,7 +19,7 @@ extern crate stderrlog;
 extern crate x509_parser;
 
 use clap::Parser;
-use clubcard_crlite::{CRLiteClubcard, CRLiteStatus};
+use clubcard_crlite::{CRLiteClubcard, CRLiteStatus, IssuerSpkiHash, LogId, Timestamp};
 use der_parser::oid;
 use log::*;
 use serde::Deserialize;
@@ -272,8 +272,12 @@ impl Filter {
     ) -> Status {
         match self {
             Filter::Clubcard((_, clubcard)) => {
-                let crlite_key = clubcard_crlite::CRLiteKey::new(issuer_spki_hash, serial);
-                match clubcard.contains(&crlite_key, timestamps.iter().map(|(x, y)| (x, *y))) {
+                let issuer_spki_hash = IssuerSpkiHash(issuer_spki_hash.clone());
+                let crlite_key = clubcard_crlite::CRLiteKey::new(&issuer_spki_hash, serial);
+                match clubcard.contains(
+                    &crlite_key,
+                    timestamps.iter().map(|(x, y)| (LogId(*x), Timestamp(*y))),
+                ) {
                     CRLiteStatus::Good => Status::Good,
                     CRLiteStatus::NotCovered => Status::NotCovered,
                     CRLiteStatus::NotEnrolled => Status::NotEnrolled,