unicode-org · Manishearth · Nov 12, 2024 · Nov 5, 2024 · Nov 11, 2024 · Nov 11, 2024
@@ -3,6 +3,7 @@
 // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
 
 use std::borrow::Cow;
+use std::fmt::Debug;
 
 use zerofrom::ZeroFrom;
 use zerovec::{ule::AsULE, *};
@@ -105,6 +106,8 @@ where
     U: ule::EncodeAsVarULE<T> + serde::Serialize,
     F: Fn(&U, &T),
     for<'a> Box<T>: serde::Deserialize<'a>,
+    for<'a> &'a T: serde::Deserialize<'a>,
+    T: PartialEq + Debug,
 {
     let varzerovec: VarZeroVec<T> = slice.into();
 
@@ -142,6 +145,27 @@ where
     for (stack, zero) in slice.iter().zip(deserialized.iter()) {
         assert(stack, zero)
     }
+
+    if let Some(first) = varzerovec.get(0) {
+        let bincode = bincode::serialize(first).unwrap();
+        let deserialized: &T = bincode::deserialize(&bincode).unwrap();
+        let deserialized_box: Box<T> = bincode::deserialize(&bincode).unwrap();
+        assert_eq!(
+            first, deserialized,
+            "Single element roundtrips with bincode"
+        );
+        assert_eq!(
+            first, &*deserialized_box,
+            "Single element roundtrips with bincode"
+        );
+
+        let json = serde_json::to_string(first).unwrap();
+        let deserialized: Box<T> = serde_json::from_str(&json).unwrap();
+        assert_eq!(
+            first, &*deserialized,
+            "Single element roundtrips with serde"
+        );
+    }
 }
 
 fn main() {

@@ -200,6 +200,19 @@ pub fn make_varule_impl(ule_name: Ident, mut input: DeriveInput) -> TokenStream2
         quote!()
     };
 
+    let maybe_toowned = if !attrs.skip_toowned {
+        quote!(
+            impl zerovec::__zerovec_internal_reexport::borrow::ToOwned for #ule_name {
+                type Owned = zerovec::__zerovec_internal_reexport::boxed::Box<Self>;
+                fn to_owned(&self) -> Self::Owned {
+                    zerovec::ule::encode_varule_to_box(self)
+                }
+            }
+        )
+    } else {
+        quote!()
+    };
+
     let zmkv = if attrs.skip_kv {
         quote!()
     } else {
@@ -219,21 +232,43 @@ pub fn make_varule_impl(ule_name: Ident, mut input: DeriveInput) -> TokenStream2
         quote!(
             impl #serde_path::Serialize for #ule_name {
                 fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error> where S: #serde_path::Serializer {
-                    let this = #zerofrom_fq_path::zero_from(self);
-                    <#name as #serde_path::Serialize>::serialize(&this, serializer)
+                    if serializer.is_human_readable() {
+                        let this = #zerofrom_fq_path::zero_from(self);
+                        <#name as #serde_path::Serialize>::serialize(&this, serializer)
+                    } else {
+                        serializer.serialize_bytes(zerovec::ule::VarULE::as_byte_slice(self))
+                    }
                 }
             }
         )
     } else {
         quote!()
     };
 
+    let deserialize_error = format!("&{ule_name} can only deserialize in zero-copy ways");
+
     let maybe_de = if attrs.deserialize {
         quote!(
             impl<'de> #serde_path::Deserialize<'de> for zerovec::__zerovec_internal_reexport::boxed::Box<#ule_name> {
                 fn deserialize<D>(deserializer: D) -> Result<Self, D::Error> where D: #serde_path::Deserializer<'de> {
-                    let this = <#name as #serde_path::Deserialize>::deserialize(deserializer)?;
-                    Ok(zerovec::ule::encode_varule_to_box(&this))
+                    if deserializer.is_human_readable() {
+                        let this = <#name as #serde_path::Deserialize>::deserialize(deserializer)?;
+                        Ok(zerovec::ule::encode_varule_to_box(&this))
+                    } else {
+                        // This branch should usually not be hit, since Cow-like use cases will hit the Deserialize impl for &'a ULEType instead.
+                        let deserialized = <& #ule_name>::deserialize(deserializer)?;
+                        Ok(zerovec::ule::VarULE::to_boxed(deserialized))
+                    }
+                }
+            }
+            impl<'a, 'de: 'a> #serde_path::Deserialize<'de> for &'a #ule_name {
+                fn deserialize<D>(deserializer: D) -> Result<Self, D::Error> where D: #serde_path::Deserializer<'de> {
+                    if !deserializer.is_human_readable() {
+                        let bytes = <&[u8]>::deserialize(deserializer)?;
+                        <#ule_name as zerovec::ule::VarULE>::parse_byte_slice(bytes).map_err(#serde_path::de::Error::custom)
+                    } else {
+                        Err(#serde_path::de::Error::custom(#deserialize_error))
+                    }
                 }
             }
         )
@@ -289,6 +324,8 @@ pub fn make_varule_impl(ule_name: Ident, mut input: DeriveInput) -> TokenStream2
 
         #maybe_debug
 
+        #maybe_toowned
+
         #maybe_hash
     )
 }

@@ -270,6 +270,7 @@ pub fn extract_field_attributes(attrs: &mut Vec<Attribute>) -> Result<Option<Ide
 pub struct ZeroVecAttrs {
     pub skip_kv: bool,
     pub skip_ord: bool,
+    pub skip_toowned: bool,
     pub serialize: bool,
     pub deserialize: bool,
     pub debug: bool,
@@ -322,6 +323,8 @@ pub fn extract_attributes_common(
             attrs.skip_kv = true;
         } else if ident == "Ord" {
             attrs.skip_ord = true;
+        } else if ident == "ToOwned" && is_var {
+            attrs.skip_toowned = true;
         } else {
             return Err(Error::new(
                 ident.span(),

@@ -241,6 +241,7 @@ pub use crate::zerovec::{ZeroSlice, ZeroVec};
 pub mod __zerovec_internal_reexport {
     pub use zerofrom::ZeroFrom;
 
+    pub use alloc::borrow;
     pub use alloc::boxed;
 
     #[cfg(feature = "serde")]

@@ -20,6 +20,8 @@ mod niche;
 mod option;
 mod plain;
 mod slices;
+#[cfg(test)]
+pub mod test_utils;
 
 pub mod tuple;
 pub mod tuplevar;

@@ -104,6 +104,11 @@ impl<const LEN: usize, Format: VarZeroVecFormat> MultiFieldsULE<LEN, Format> {
         // &Self is transparent over &VZS<..>
         mem::transmute(<VarZeroLengthlessSlice<[u8]>>::from_bytes_unchecked(bytes))
     }
+
+    /// Get the bytes behind this value
+    pub fn as_bytes(&self) -> &[u8] {
+        self.0.as_bytes()
+    }
 }
 
 impl<const LEN: usize, Format: VarZeroVecFormat> fmt::Debug for MultiFieldsULE<LEN, Format> {

@@ -0,0 +1,30 @@
+// This file is part of ICU4X. For terms of use, please see the file
+// called LICENSE at the top level of the ICU4X source tree
+// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
+
+/// Take a VarULE type and serialize it both in human and machine readable contexts,
+/// and ensure it roundtrips correctly
+///
+/// Note that the concrete type may need to be explicitly specified to prevent issues with
+/// https://github.com/rust-lang/rust/issues/130180
+#[cfg(feature = "serde")]
+pub(crate) fn assert_serde_roundtrips<T>(var: &T)
+where
+    T: crate::ule::VarULE + ?Sized + serde::Serialize,
+    for<'a> Box<T>: serde::Deserialize<'a>,
+    for<'a> &'a T: serde::Deserialize<'a>,
+    T: core::fmt::Debug + PartialEq,
+{
+    let bincode = bincode::serialize(var).unwrap();
+    let deserialized: &T = bincode::deserialize(&bincode).unwrap();
+    let deserialized_box: Box<T> = bincode::deserialize(&bincode).unwrap();
+    assert_eq!(var, deserialized, "Single element roundtrips with bincode");
+    assert_eq!(
+        var, &*deserialized_box,
+        "Single element roundtrips with bincode"
+    );
+
+    let json = serde_json::to_string(var).unwrap();
+    let deserialized: Box<T> = serde_json::from_str(&json).unwrap();
+    assert_eq!(var, &*deserialized, "Single element roundtrips with serde");
+}
@@ -167,6 +167,62 @@ macro_rules! tuple_varule {
                 )
             }
         }
+
+        #[cfg(feature = "serde")]
+        impl<$($T: serde::Serialize),+> serde::Serialize for $name<$($T),+>
+        where
+            $($T: VarULE + ?Sized,)+
+            // This impl should be present on almost all VarULE types. if it isn't, that is a bug
+            $(for<'a> &'a $T: ZeroFrom<'a, $T>),+
+        {
+            fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error> where S: serde::Serializer {
+                if serializer.is_human_readable() {
+                    let this = (
+                        $(self.$t()),+
+                    );
+                    <($(&$T),+) as serde::Serialize>::serialize(&this, serializer)
+                } else {
+                    serializer.serialize_bytes(self.multi.as_bytes())
+                }
+            }
+        }
+
+        #[cfg(feature = "serde")]
+        impl<'de, $($T: VarULE + ?Sized),+> serde::Deserialize<'de> for Box<$name<$($T),+>>
+            where
+                // This impl should be present on almost all deserializable VarULE types
+                $( Box<$T>: serde::Deserialize<'de>),+ {
+            fn deserialize<Des>(deserializer: Des) -> Result<Self, Des::Error> where Des: serde::Deserializer<'de> {
+                if deserializer.is_human_readable() {
+                    let this = <( $(Box<$T>),+) as serde::Deserialize>::deserialize(deserializer)?;
+                    let this_ref = (
+                        $(&*this.$i),+
+                    );
+                    Ok(crate::ule::encode_varule_to_box(&this_ref))
+                } else {
+                    // This branch should usually not be hit, since Cow-like use cases will hit the Deserialize impl for &'a TupleNVarULE instead.
+
+                    let deserialized = <&$name<$($T),+>>::deserialize(deserializer)?;
+                    Ok(deserialized.to_boxed())
+                }
+            }
+        }
+
+        #[cfg(feature = "serde")]
+        impl<'a, 'de, $($T: VarULE + ?Sized),+> serde::Deserialize<'de> for &'a $name<$($T),+>
+            where
+                'de: 'a {
+            fn deserialize<Des>(deserializer: Des) -> Result<Self, Des::Error> where Des: serde::Deserializer<'de> {
+                if deserializer.is_human_readable() {
+                    Err(serde::de::Error::custom(
+                        concat!("&", stringify!($name), " can only deserialize in zero-copy ways"),
+                    ))
+                } else {
+                    let bytes = <&[u8]>::deserialize(deserializer)?;
+                    $name::<$($T),+>::parse_byte_slice(bytes).map_err(serde::de::Error::custom)
+                }
+            }
+        }
     };
 }
 
@@ -193,6 +249,12 @@ mod tests {
         // Note: ipsum\xFF is not a valid str
         let zerovec3 = VarZeroVec::<Tuple2VarULE<str, str>>::parse_byte_slice(bytes);
         assert!(zerovec3.is_err());
+
+        #[cfg(feature = "serde")]
+        for val in zerovec.iter() {
+            // Can't use inference due to https://github.com/rust-lang/rust/issues/130180
+            crate::ule::test_utils::assert_serde_roundtrips::<Tuple2VarULE<str, [u8]>>(val);
+        }
     }
     #[test]
     fn test_tripleule_validate() {
@@ -214,5 +276,13 @@ mod tests {
         // Note: the str is unlikely to be a valid varzerovec
         let zerovec3 = VarZeroVec::<Tuple3VarULE<VarZeroSlice<str>, [u8], VarZeroSlice<str>>>::parse_byte_slice(bytes);
         assert!(zerovec3.is_err());
+
+        #[cfg(feature = "serde")]
+        for val in zerovec.iter() {
+            // Can't use inference due to https://github.com/rust-lang/rust/issues/130180
+            crate::ule::test_utils::assert_serde_roundtrips::<
+                Tuple3VarULE<str, [u8], VarZeroSlice<str>>,
+            >(val);
+        }
     }
 }