feat(common): encode the SNARK aggregate signature as bytes in the certificate message

mithril-aggregator/src/database/record/certificate.rs

@@ -215,44 +215,48 @@ impl TryFrom<Certificate> for CertificateRecord {
     fn try_from(other: Certificate) -> Result<Self, Self::Error> {
         let signed_entity_type = other.signed_entity_type();
         let (signature, parent_certificate_id) = match other.signature {
-            CertificateSignature::GenesisSignature(signature) => (signature.to_bytes_hex()?, None),
+            CertificateSignature::GenesisSignature(signature) => {
+                (String::try_from(&signature)?, None)
+            }
             #[cfg(feature = "future_snark")]
             CertificateSignature::GenesisDualSignature(ed_signature, schnorr_signature) => {
                 let payload = PersistedDualGenesisSignature {
-                    ed25519: ed_signature.to_bytes_hex()?,
+                    ed25519: String::try_from(&ed_signature)?,
                     schnorr: schnorr_signature_to_hex(&schnorr_signature),
                 };
                 (serde_json::to_string(&payload)?, None)
             }
             CertificateSignature::MultiSignature(_, signature) => {
-                (signature.to_json_hex()?, Some(other.previous_hash))
+                (String::try_from(&signature)?, Some(other.previous_hash))
             }
         };
 
         #[cfg(feature = "future_snark")]
         let aggregate_verification_key_snark = other
             .aggregate_verification_key_snark
             .as_ref()
-            .map(|avk| avk.to_bytes_hex())
+            .map(String::try_from)
             .transpose()?;
         #[cfg(not(feature = "future_snark"))]
         let aggregate_verification_key_snark: Option<HexEncodedKey> = None;
 
         let ancillary_prover_data = other
             .ancillary_prover_data
-            .map(|data| data.to_bytes_hex())
+            .as_ref()
+            .map(String::try_from)
             .transpose()?;
         let ancillary_verifier_data = other
             .ancillary_verifier_data
-            .map(|data| data.to_bytes_hex())
+            .as_ref()
+            .map(String::try_from)
             .transpose()?;
 
         let certificate_record = CertificateRecord {
             certificate_id: other.hash,
             parent_certificate_id,
             message: other.signed_message,
             signature,
-            aggregate_verification_key: other.aggregate_verification_key.to_json_hex()?,
+            aggregate_verification_key: String::try_from(&other.aggregate_verification_key)?,
             aggregate_verification_key_snark,
             ancillary_prover_data,
             ancillary_verifier_data,
@@ -597,6 +601,37 @@ mod tests {
         assert_eq!(expected_hash, &certificate.hash);
     }
 
+    #[cfg(feature = "future_snark")]
+    mod snark_multi_signature {
+        use super::*;
+
+        #[test]
+        fn snark_multi_signature_is_stored_as_bytes_hex() {
+            let chain = setup_certificate_chain(5, 2);
+            let mut certificate = chain
+                .certificates_chained
+                .iter()
+                .find(|c| matches!(c.signature, CertificateSignature::MultiSignature(..)))
+                .expect("chain should contain a multi-signature certificate")
+                .clone();
+            let signed_entity_type = match &certificate.signature {
+                CertificateSignature::MultiSignature(signed_entity_type, _) => {
+                    signed_entity_type.clone()
+                }
+                _ => unreachable!(),
+            };
+            certificate.signature = CertificateSignature::MultiSignature(
+                signed_entity_type,
+                fake_data::snark_aggregate_signature(),
+            );
+            let expected_bytes_hex = fake_data::snark_aggregate_signature().to_bytes_hex().unwrap();
+
+            let record: CertificateRecord = certificate.try_into().unwrap();
+
+            assert_eq!(expected_bytes_hex, record.signature);
+        }
+    }
+
     #[cfg(feature = "future_snark")]
     mod snark_aggregate_verification_key {
         use super::*;

mithril-common/src/crypto_helper/types/wrappers.rs

@@ -1,13 +1,16 @@
 use kes_summed_ed25519::kes::Sum6KesSig;
 use mithril_stm::{
-    AggregateSignature, AggregateVerificationKey, AggregateVerificationKeyForConcatenation,
-    AncillaryProverData, AncillaryVerifierData, SingleSignature,
-    VerificationKeyProofOfPossessionForConcatenation,
+    AggregateSignature, AggregateSignatureType, AggregateVerificationKey,
+    AggregateVerificationKeyForConcatenation, AncillaryProverData, AncillaryVerifierData,
+    SingleSignature, VerificationKeyProofOfPossessionForConcatenation,
 };
 #[cfg(feature = "future_snark")]
 use mithril_stm::{AggregateVerificationKeyForSnark, VerificationKeyForSnark};
 
-use crate::crypto_helper::{MKMapProof, MKProof, OpCert, ProtocolKey, ProtocolMembershipDigest};
+use crate::StdResult;
+use crate::crypto_helper::{
+    MKMapProof, MKProof, OpCert, ProtocolKey, ProtocolKeyCodec, ProtocolMembershipDigest,
+};
 use crate::entities::BlockRange;
 
 /// Wrapper of [MithrilStm:VerificationKeyProofOfPossessionForConcatenation](type@VerificationKeyProofOfPossessionForConcatenation) to add serialization
@@ -60,14 +63,46 @@ pub type ProtocolAncillaryProverData = ProtocolKey<AncillaryProverData>;
 pub type ProtocolAncillaryVerifierData = ProtocolKey<AncillaryVerifierData>;
 
 impl_codec_and_type_conversions_for_protocol_key!(
-    json_hex_codec => AggregateSignature<ProtocolMembershipDigest>, ed25519_dalek::VerifyingKey, ed25519_dalek::SigningKey, AggregateVerificationKeyForConcatenation<ProtocolMembershipDigest>,
+    json_hex_codec => ed25519_dalek::VerifyingKey, ed25519_dalek::SigningKey, AggregateVerificationKeyForConcatenation<ProtocolMembershipDigest>,
         MKProof, VerificationKeyProofOfPossessionForConcatenation, Sum6KesSig, OpCert, SingleSignature
 );
 
 impl_codec_and_type_conversions_for_protocol_key!(
     bytes_hex_codec => ed25519_dalek::Signature, AncillaryProverData, AncillaryVerifierData
 );
 
+impl_codec_and_type_conversions_for_protocol_key!(
+    no_default_codec => AggregateSignature<ProtocolMembershipDigest>
+);
+
+impl ProtocolKeyCodec<AggregateSignature<ProtocolMembershipDigest>>
+    for AggregateSignature<ProtocolMembershipDigest>
+{
+    fn decode_key(
+        encoded: &str,
+    ) -> StdResult<ProtocolKey<AggregateSignature<ProtocolMembershipDigest>>> {
+        match ProtocolKey::from_json_hex(encoded) {
+            Ok(res) => Ok(res),
+            Err(_) => ProtocolKey::from_bytes_hex(encoded),
+        }
+    }
+
+    fn encode_key(key: &AggregateSignature<ProtocolMembershipDigest>) -> StdResult<String> {
+        // Temporary workaround: encode by aggregate signature type rather than a single codec. The
+        // bytes encoding of an aggregate signature is only decodable by clients from distribution
+        // 2617.0 onwards, so concatenation multi-signatures must stay JSON-hex until the older
+        // distributions (up to 2603.1) are retired. Once they are, register `AggregateSignature`
+        // under `bytes_hex_codec` and drop this custom codec.
+        match AggregateSignatureType::from(key) {
+            AggregateSignatureType::Concatenation => ProtocolKey::key_to_json_hex(key),
+            #[cfg(feature = "future_snark")]
+            AggregateSignatureType::Snark | AggregateSignatureType::IvcSnark => {
+                ProtocolKey::key_to_bytes_hex(key)
+            }
+        }
+    }
+}
+
 #[cfg(feature = "future_snark")]
 impl_codec_and_type_conversions_for_protocol_key!(
     json_hex_codec => VerificationKeyForSnark

mithril-common/src/messages/certificate.rs

@@ -270,7 +270,7 @@ impl TryFrom<Certificate> for CertificateMessage {
         let (multi_signature, genesis_signature) = match certificate.signature {
             CertificateSignature::GenesisSignature(signature) => (
                 String::new(),
-                signature.to_bytes_hex().with_context(|| {
+                String::try_from(&signature).with_context(|| {
                     "Can not convert certificate to message: can not encode the genesis signature"
                 })?,
             ),
@@ -279,13 +279,13 @@ impl TryFrom<Certificate> for CertificateMessage {
                 genesis_schnorr_signature = hex::encode(schnorr_signature.to_bytes());
                 (
                     String::new(),
-                    ed_signature.to_bytes_hex().with_context(|| {
+                    String::try_from(&ed_signature).with_context(|| {
                         "Can not convert certificate to message: can not encode the genesis signature"
                     })?,
                 )
             }
             CertificateSignature::MultiSignature(_, signature) => (
-                signature.to_json_hex().with_context(|| {
+                String::try_from(&signature).with_context(|| {
                     "Can not convert certificate to message: can not encode the multi-signature"
                 })?,
                 String::new(),
@@ -300,30 +300,28 @@ impl TryFrom<Certificate> for CertificateMessage {
             metadata,
             protocol_message: certificate.protocol_message,
             signed_message: certificate.signed_message,
-            aggregate_verification_key: certificate
-                .aggregate_verification_key
-                .to_json_hex()
+            aggregate_verification_key: String::try_from(&certificate.aggregate_verification_key)
                 .with_context(|| {
                     "Can not convert certificate to message: can not encode aggregate verification key for Concatenation"
                 })?,
             #[cfg(feature = "future_snark")]
             aggregate_verification_key_snark: certificate
                 .aggregate_verification_key_snark
-                .map(|avk| avk.to_bytes_hex())
+                .map(String::try_from)
                 .transpose()
                 .with_context(|| {
                     "Can not convert certificate to message: can not encode aggregate verification key for SNARK"
                 })?,
             ancillary_prover_data: certificate
                 .ancillary_prover_data
-                .map(|data| data.to_bytes_hex())
+                .map(String::try_from)
                 .transpose()
                 .with_context(|| {
                     "Can not convert certificate to message: can not encode the ancillary prover data"
                 })?,
             ancillary_verifier_data: certificate
                 .ancillary_verifier_data
-                .map(|data| data.to_bytes_hex())
+                .map(String::try_from)
                 .transpose()
                 .with_context(|| {
                     "Can not convert certificate to message: can not encode the ancillary verifier data"
@@ -694,4 +692,45 @@ mod tests {
             }
         }
     }
+
+    mod multi_signature_encoding {
+        use crate::test::double::fake_data;
+
+        use super::*;
+
+        #[test]
+        fn concatenation_multi_signature_is_encoded_as_json_hex() {
+            let certificate = fake_data::certificate("hash");
+            let expected_json_hex = match &certificate.signature {
+                CertificateSignature::MultiSignature(_, signature) => {
+                    signature.to_json_hex().unwrap()
+                }
+                _ => panic!("expected a multi-signature certificate"),
+            };
+
+            let message = CertificateMessage::try_from(certificate).unwrap();
+
+            assert_eq!(expected_json_hex, message.multi_signature);
+        }
+
+        #[cfg(feature = "future_snark")]
+        #[test]
+        fn snark_multi_signature_is_encoded_as_bytes_hex() {
+            let snark_signature = fake_data::snark_aggregate_signature();
+            let expected_bytes_hex = snark_signature.to_bytes_hex().unwrap();
+            let mut certificate = fake_data::certificate("hash");
+            let signed_entity_type = match &certificate.signature {
+                CertificateSignature::MultiSignature(signed_entity_type, _) => {
+                    signed_entity_type.clone()
+                }
+                _ => panic!("expected a multi-signature certificate"),
+            };
+            certificate.signature =
+                CertificateSignature::MultiSignature(signed_entity_type, snark_signature);
+
+            let message = CertificateMessage::try_from(certificate).unwrap();
+
+            assert_eq!(expected_bytes_hex, message.multi_signature);
+        }
+    }
 }

mithril-stm/Cargo.toml

@@ -10,6 +10,10 @@ description = "A Rust implementation of Mithril Stake-based Threshold Multisigna
 categories = ["cryptography"]
 include = ["**/*.rs", "Cargo.toml", "README.md", ".gitignore"]
 
+[package.metadata.cargo-machete]
+# `serde_bytes` is used for proof bytes serialization
+ignored = ["serde_bytes"]
+
 [lib]
 crate-type = ["lib", "cdylib", "staticlib"]
 
@@ -65,6 +69,7 @@ rand_core = { workspace = true, features = ["std"] }
 rayon = { workspace = true }
 reqwest = { workspace = true, default-features = false, features = ["blocking"], optional = true }
 serde = { workspace = true }
+serde_bytes = "0.11.19"
 sha2 = { version = "0.10.9", optional = true }
 thiserror = { workspace = true }
 

mithril-stm/src/circuits/halo2_ivc/types.rs

@@ -257,7 +257,7 @@ impl CertificateProofBytes {
 
 /// Provisional recursive proof-byte wrapper until `IvcProof` is wired end-to-end.
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
-pub(crate) struct IvcProofBytes(Vec<u8>);
+pub(crate) struct IvcProofBytes(#[serde(with = "serde_bytes")] Vec<u8>);
 
 impl IvcProofBytes {
     pub(crate) fn new(bytes: Vec<u8>) -> Self {

mithril-stm/src/proof_system/halo2_snark/proof.rs

@@ -33,6 +33,7 @@ use crate::circuits::halo2::keys::NonRecursiveCircuitVerifyingKey;
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct SnarkProof<D: MembershipDigest> {
     /// Raw proof bytes produced by the SNARK prover.
+    #[serde(with = "serde_bytes")]
     circuit_proof: Vec<u8>,
     // The protocol parameters
     // Note: this field is temporary and will be removed once the circuit setup is stable