From d2de4e4a7bd7d6bc14a1aa757dfabb6c3460a1f5 Mon Sep 17 00:00:00 2001 From: Simon Warta Date: Wed, 18 Sep 2024 11:51:43 +0200 Subject: [PATCH 1/3] Add messagepack encoding tests for integer types --- packages/std/src/msgpack.rs | 233 ++++++++++++++++++++++++++++++++++++ 1 file changed, 233 insertions(+) diff --git a/packages/std/src/msgpack.rs b/packages/std/src/msgpack.rs index 5ca264abd..6dd97527c 100644 --- a/packages/std/src/msgpack.rs +++ b/packages/std/src/msgpack.rs @@ -35,6 +35,7 @@ where #[cfg(test)] mod tests { use super::*; + use crate::{Int128, Int256, Int512, Int64, Uint128, Uint256, Uint512, Uint64}; use serde::Deserialize; #[derive(Serialize, Deserialize, Debug, PartialEq)] @@ -224,4 +225,236 @@ mod tests { } ); } + + #[test] + fn msgpack_serialization_for_boolean_types() { + // "Bool format family stores false or true in 1 byte." + let serialized = to_msgpack_vec(&false).unwrap(); + assert_eq!(serialized, [0xc2]); + let serialized = to_msgpack_vec(&true).unwrap(); + assert_eq!(serialized, [0xc3]); + } + + #[test] + fn msgpack_serialization_for_integer_types() { + // primitive integers up to 64bit + // similar to VARINT in protobuf or number in JSON, the encoding does not contain integer size + { + // "positive fixint stores 7-bit positive integer" + let serialized = to_msgpack_vec(&0u8).unwrap(); + assert_eq!(serialized, [0]); + let serialized = to_msgpack_vec(&0u16).unwrap(); + assert_eq!(serialized, [0]); + let serialized = to_msgpack_vec(&0u32).unwrap(); + assert_eq!(serialized, [0]); + let serialized = to_msgpack_vec(&0u64).unwrap(); + assert_eq!(serialized, [0]); + let serialized = to_msgpack_vec(&0i64).unwrap(); + assert_eq!(serialized, [0]); + let serialized = to_msgpack_vec(&7u8).unwrap(); + assert_eq!(serialized, [7]); + let serialized = to_msgpack_vec(&7u16).unwrap(); + assert_eq!(serialized, [7]); + let serialized = to_msgpack_vec(&7u32).unwrap(); + assert_eq!(serialized, [7]); + let serialized = to_msgpack_vec(&7u64).unwrap(); + assert_eq!(serialized, [7]); + let serialized = to_msgpack_vec(&127u32).unwrap(); + assert_eq!(serialized, [127]); + + // "negative fixint stores 5-bit negative integer" + let serialized = to_msgpack_vec(&-1i32).unwrap(); + assert_eq!(serialized, [255]); + let serialized = to_msgpack_vec(&-1i64).unwrap(); + assert_eq!(serialized, [255]); + let serialized = to_msgpack_vec(&-10i64).unwrap(); + assert_eq!(serialized, [246]); + let serialized = to_msgpack_vec(&-24i64).unwrap(); + assert_eq!(serialized, [232]); + + // "uint 8 stores a 8-bit unsigned integer" + let serialized = to_msgpack_vec(&128u32).unwrap(); + assert_eq!(serialized, [0xcc, 128]); + let serialized = to_msgpack_vec(&237u32).unwrap(); + assert_eq!(serialized, [0xcc, 237]); + + // "uint 16 stores a 16-bit big-endian unsigned integer" + let serialized = to_msgpack_vec(&1000u32).unwrap(); + assert_eq!(serialized, [0xcd, 3, 232]); + + // "uint 32 stores a 32-bit big-endian unsigned integer" + let serialized = to_msgpack_vec(&u32::MAX).unwrap(); + assert_eq!(serialized, [0xce, 255, 255, 255, 255]); + + // "uint 64 stores a 64-bit big-endian unsigned integer" + let serialized = to_msgpack_vec(&575747839886u64).unwrap(); + assert_eq!(serialized, [0xcf, 0, 0, 0, 134, 13, 62, 215, 142]); + let serialized = to_msgpack_vec(&u64::MAX).unwrap(); + assert_eq!(serialized, [0xcf, 255, 255, 255, 255, 255, 255, 255, 255]); + + // "int 8 stores a 8-bit signed integer" + let serialized = to_msgpack_vec(&i8::MIN).unwrap(); + assert_eq!(serialized, [0xd0, 128]); + let serialized = to_msgpack_vec(&-111i8).unwrap(); + assert_eq!(serialized, [0xd0, 145]); + + // "int 16 stores a 16-bit big-endian signed integer" + let serialized = to_msgpack_vec(&i16::MIN).unwrap(); + assert_eq!(serialized, [0xd1, 128, 0]); + + // "int 32 stores a 32-bit big-endian signed integer" + let serialized = to_msgpack_vec(&i32::MIN).unwrap(); + assert_eq!(serialized, [0xd2, 128, 0, 0, 0]); + + // "int 64 stores a 64-bit big-endian signed integer" + let serialized = to_msgpack_vec(&i64::MIN).unwrap(); + assert_eq!(serialized, [0xd3, 128, 0, 0, 0, 0, 0, 0, 0]); + } + + // u128/i128 + // cannot be serialized as integers in messagepack due to the limitation + // "a value of an Integer object is limited from -(2^63) upto (2^64)-1" + { + // encoded as 16 bytes big endian + // i.e. takes 18 bytes of storage + assert_eq!( + to_msgpack_vec(&0u128).unwrap(), + [0xc4, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + ); + assert_eq!( + to_msgpack_vec(&1u128).unwrap(), + [0xc4, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1] + ); + assert_eq!( + to_msgpack_vec(&17u128).unwrap(), + [0xc4, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17] + ); + assert_eq!( + to_msgpack_vec(&u128::MAX).unwrap(), + [ + 0xc4, 16, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255 + ] + ); + + assert_eq!( + to_msgpack_vec(&0i128).unwrap(), + [0xc4, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + ); + assert_eq!( + to_msgpack_vec(&1i128).unwrap(), + [0xc4, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1] + ); + assert_eq!( + to_msgpack_vec(&17i128).unwrap(), + [0xc4, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17] + ); + assert_eq!( + to_msgpack_vec(&-1i128).unwrap(), + [ + 0xc4, 16, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255 + ] + ); + assert_eq!( + to_msgpack_vec(&i128::MIN).unwrap(), + [0xc4, 16, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + ); + assert_eq!( + to_msgpack_vec(&i128::MAX).unwrap(), + [ + 0xc4, 16, 127, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255 + ] + ); + } + + // Uint64/Uint128/Uint256/Uint512 + { + let s = to_msgpack_vec(&Uint64::zero()).unwrap(); + assert_eq!(s, [0b10100000 ^ 1, '0' as u8]); // string of lengths 1 with value "0" + let s = to_msgpack_vec(&Uint128::zero()).unwrap(); + assert_eq!(s, [0b10100000 ^ 1, '0' as u8]); // string of lengths 1 with value "0" + let s = to_msgpack_vec(&Uint256::zero()).unwrap(); + assert_eq!(s, [0b10100000 ^ 1, '0' as u8]); // string of lengths 1 with value "0" + let s = to_msgpack_vec(&Uint512::zero()).unwrap(); + assert_eq!(s, [0b10100000 ^ 1, '0' as u8]); // string of lengths 1 with value "0" + + let s = to_msgpack_vec(&Uint64::one()).unwrap(); + assert_eq!(s, [0b10100000 ^ 1, '1' as u8]); // string of lengths 1 with value "1" + let s = to_msgpack_vec(&Uint128::one()).unwrap(); + assert_eq!(s, [0b10100000 ^ 1, '1' as u8]); // string of lengths 1 with value "1" + let s = to_msgpack_vec(&Uint256::one()).unwrap(); + assert_eq!(s, [0b10100000 ^ 1, '1' as u8]); // string of lengths 1 with value "1" + let s = to_msgpack_vec(&Uint512::one()).unwrap(); + assert_eq!(s, [0b10100000 ^ 1, '1' as u8]); // string of lengths 1 with value "1" + + let s = to_msgpack_vec(&Uint64::MAX).unwrap(); + assert_eq!( + s, + [ + 0b10100000 ^ 20, + '1' as u8, + '8' as u8, + '4' as u8, + '4' as u8, + '6' as u8, + '7' as u8, + '4' as u8, + '4' as u8, + '0' as u8, + '7' as u8, + '3' as u8, + '7' as u8, + '0' as u8, + '9' as u8, + '5' as u8, + '5' as u8, + '1' as u8, + '6' as u8, + '1' as u8, + '5' as u8 + ] + ); // string of lengths 1 with value "1" + } + + // Int64/Int128/Int256/Int512 + { + let s = to_msgpack_vec(&Int64::zero()).unwrap(); + assert_eq!(s, [0b10100000 ^ 1, '0' as u8]); // string of lengths 1 with value "0" + let s = to_msgpack_vec(&Int128::zero()).unwrap(); + assert_eq!(s, [0b10100000 ^ 1, '0' as u8]); // string of lengths 1 with value "0" + let s = to_msgpack_vec(&Int256::zero()).unwrap(); + assert_eq!(s, [0b10100000 ^ 1, '0' as u8]); // string of lengths 1 with value "0" + let s = to_msgpack_vec(&Int512::zero()).unwrap(); + assert_eq!(s, [0b10100000 ^ 1, '0' as u8]); // string of lengths 1 with value "0" + + let s = to_msgpack_vec(&Int64::one()).unwrap(); + assert_eq!(s, [0b10100000 ^ 1, '1' as u8]); // string of lengths 1 with value "1" + let s = to_msgpack_vec(&Int128::one()).unwrap(); + assert_eq!(s, [0b10100000 ^ 1, '1' as u8]); // string of lengths 1 with value "1" + let s = to_msgpack_vec(&Int256::one()).unwrap(); + assert_eq!(s, [0b10100000 ^ 1, '1' as u8]); // string of lengths 1 with value "1" + let s = to_msgpack_vec(&Int512::one()).unwrap(); + assert_eq!(s, [0b10100000 ^ 1, '1' as u8]); // string of lengths 1 with value "1" + + let s = to_msgpack_vec(&Int64::from(15i32)).unwrap(); + assert_eq!(s, [0b10100000 ^ 2, '1' as u8, '5' as u8]); // string of lengths 2 with value "15" + let s = to_msgpack_vec(&Int128::from(15i32)).unwrap(); + assert_eq!(s, [0b10100000 ^ 2, '1' as u8, '5' as u8]); // string of lengths 2 with value "15" + let s = to_msgpack_vec(&Int256::from(15i32)).unwrap(); + assert_eq!(s, [0b10100000 ^ 2, '1' as u8, '5' as u8]); // string of lengths 2 with value "15" + let s = to_msgpack_vec(&Int512::from(15i32)).unwrap(); + assert_eq!(s, [0b10100000 ^ 2, '1' as u8, '5' as u8]); // string of lengths 2 with value "15" + + let s = to_msgpack_vec(&Int64::from(-1i64)).unwrap(); + assert_eq!(s, [0b10100000 ^ 2, '-' as u8, '1' as u8]); // string of lengths 2 with value "-1" + let s = to_msgpack_vec(&Int128::from(-1i64)).unwrap(); + assert_eq!(s, [0b10100000 ^ 2, '-' as u8, '1' as u8]); // string of lengths 2 with value "-1" + let s = to_msgpack_vec(&Int256::from(-1i64)).unwrap(); + assert_eq!(s, [0b10100000 ^ 2, '-' as u8, '1' as u8]); // string of lengths 2 with value "-1" + let s = to_msgpack_vec(&Int512::from(-1i64)).unwrap(); + assert_eq!(s, [0b10100000 ^ 2, '-' as u8, '1' as u8]); // string of lengths 2 with value "-1" + } + } } From 2325f3aa503d0d6a5b07ded1c6d118683931acf6 Mon Sep 17 00:00:00 2001 From: Simon Warta Date: Wed, 18 Sep 2024 12:09:51 +0200 Subject: [PATCH 2/3] Use u8 literal notation --- packages/std/src/msgpack.rs | 88 ++++++++++++++++++------------------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/packages/std/src/msgpack.rs b/packages/std/src/msgpack.rs index 6dd97527c..b484697be 100644 --- a/packages/std/src/msgpack.rs +++ b/packages/std/src/msgpack.rs @@ -372,48 +372,48 @@ mod tests { // Uint64/Uint128/Uint256/Uint512 { let s = to_msgpack_vec(&Uint64::zero()).unwrap(); - assert_eq!(s, [0b10100000 ^ 1, '0' as u8]); // string of lengths 1 with value "0" + assert_eq!(s, [0b10100000 ^ 1, b'0']); // string of lengths 1 with value "0" let s = to_msgpack_vec(&Uint128::zero()).unwrap(); - assert_eq!(s, [0b10100000 ^ 1, '0' as u8]); // string of lengths 1 with value "0" + assert_eq!(s, [0b10100000 ^ 1, b'0']); // string of lengths 1 with value "0" let s = to_msgpack_vec(&Uint256::zero()).unwrap(); - assert_eq!(s, [0b10100000 ^ 1, '0' as u8]); // string of lengths 1 with value "0" + assert_eq!(s, [0b10100000 ^ 1, b'0']); // string of lengths 1 with value "0" let s = to_msgpack_vec(&Uint512::zero()).unwrap(); - assert_eq!(s, [0b10100000 ^ 1, '0' as u8]); // string of lengths 1 with value "0" + assert_eq!(s, [0b10100000 ^ 1, b'0']); // string of lengths 1 with value "0" let s = to_msgpack_vec(&Uint64::one()).unwrap(); - assert_eq!(s, [0b10100000 ^ 1, '1' as u8]); // string of lengths 1 with value "1" + assert_eq!(s, [0b10100000 ^ 1, b'1']); // string of lengths 1 with value "1" let s = to_msgpack_vec(&Uint128::one()).unwrap(); - assert_eq!(s, [0b10100000 ^ 1, '1' as u8]); // string of lengths 1 with value "1" + assert_eq!(s, [0b10100000 ^ 1, b'1']); // string of lengths 1 with value "1" let s = to_msgpack_vec(&Uint256::one()).unwrap(); - assert_eq!(s, [0b10100000 ^ 1, '1' as u8]); // string of lengths 1 with value "1" + assert_eq!(s, [0b10100000 ^ 1, b'1']); // string of lengths 1 with value "1" let s = to_msgpack_vec(&Uint512::one()).unwrap(); - assert_eq!(s, [0b10100000 ^ 1, '1' as u8]); // string of lengths 1 with value "1" + assert_eq!(s, [0b10100000 ^ 1, b'1']); // string of lengths 1 with value "1" let s = to_msgpack_vec(&Uint64::MAX).unwrap(); assert_eq!( s, [ 0b10100000 ^ 20, - '1' as u8, - '8' as u8, - '4' as u8, - '4' as u8, - '6' as u8, - '7' as u8, - '4' as u8, - '4' as u8, - '0' as u8, - '7' as u8, - '3' as u8, - '7' as u8, - '0' as u8, - '9' as u8, - '5' as u8, - '5' as u8, - '1' as u8, - '6' as u8, - '1' as u8, - '5' as u8 + b'1', + b'8', + b'4', + b'4', + b'6', + b'7', + b'4', + b'4', + b'0', + b'7', + b'3', + b'7', + b'0', + b'9', + b'5', + b'5', + b'1', + b'6', + b'1', + b'5' ] ); // string of lengths 1 with value "1" } @@ -421,40 +421,40 @@ mod tests { // Int64/Int128/Int256/Int512 { let s = to_msgpack_vec(&Int64::zero()).unwrap(); - assert_eq!(s, [0b10100000 ^ 1, '0' as u8]); // string of lengths 1 with value "0" + assert_eq!(s, [0b10100000 ^ 1, b'0']); // string of lengths 1 with value "0" let s = to_msgpack_vec(&Int128::zero()).unwrap(); - assert_eq!(s, [0b10100000 ^ 1, '0' as u8]); // string of lengths 1 with value "0" + assert_eq!(s, [0b10100000 ^ 1, b'0']); // string of lengths 1 with value "0" let s = to_msgpack_vec(&Int256::zero()).unwrap(); - assert_eq!(s, [0b10100000 ^ 1, '0' as u8]); // string of lengths 1 with value "0" + assert_eq!(s, [0b10100000 ^ 1, b'0']); // string of lengths 1 with value "0" let s = to_msgpack_vec(&Int512::zero()).unwrap(); - assert_eq!(s, [0b10100000 ^ 1, '0' as u8]); // string of lengths 1 with value "0" + assert_eq!(s, [0b10100000 ^ 1, b'0']); // string of lengths 1 with value "0" let s = to_msgpack_vec(&Int64::one()).unwrap(); - assert_eq!(s, [0b10100000 ^ 1, '1' as u8]); // string of lengths 1 with value "1" + assert_eq!(s, [0b10100000 ^ 1, b'1']); // string of lengths 1 with value "1" let s = to_msgpack_vec(&Int128::one()).unwrap(); - assert_eq!(s, [0b10100000 ^ 1, '1' as u8]); // string of lengths 1 with value "1" + assert_eq!(s, [0b10100000 ^ 1, b'1']); // string of lengths 1 with value "1" let s = to_msgpack_vec(&Int256::one()).unwrap(); - assert_eq!(s, [0b10100000 ^ 1, '1' as u8]); // string of lengths 1 with value "1" + assert_eq!(s, [0b10100000 ^ 1, b'1']); // string of lengths 1 with value "1" let s = to_msgpack_vec(&Int512::one()).unwrap(); - assert_eq!(s, [0b10100000 ^ 1, '1' as u8]); // string of lengths 1 with value "1" + assert_eq!(s, [0b10100000 ^ 1, b'1']); // string of lengths 1 with value "1" let s = to_msgpack_vec(&Int64::from(15i32)).unwrap(); - assert_eq!(s, [0b10100000 ^ 2, '1' as u8, '5' as u8]); // string of lengths 2 with value "15" + assert_eq!(s, [0b10100000 ^ 2, b'1', b'5']); // string of lengths 2 with value "15" let s = to_msgpack_vec(&Int128::from(15i32)).unwrap(); - assert_eq!(s, [0b10100000 ^ 2, '1' as u8, '5' as u8]); // string of lengths 2 with value "15" + assert_eq!(s, [0b10100000 ^ 2, b'1', b'5']); // string of lengths 2 with value "15" let s = to_msgpack_vec(&Int256::from(15i32)).unwrap(); - assert_eq!(s, [0b10100000 ^ 2, '1' as u8, '5' as u8]); // string of lengths 2 with value "15" + assert_eq!(s, [0b10100000 ^ 2, b'1', b'5']); // string of lengths 2 with value "15" let s = to_msgpack_vec(&Int512::from(15i32)).unwrap(); - assert_eq!(s, [0b10100000 ^ 2, '1' as u8, '5' as u8]); // string of lengths 2 with value "15" + assert_eq!(s, [0b10100000 ^ 2, b'1', b'5']); // string of lengths 2 with value "15" let s = to_msgpack_vec(&Int64::from(-1i64)).unwrap(); - assert_eq!(s, [0b10100000 ^ 2, '-' as u8, '1' as u8]); // string of lengths 2 with value "-1" + assert_eq!(s, [0b10100000 ^ 2, b'-', b'1']); // string of lengths 2 with value "-1" let s = to_msgpack_vec(&Int128::from(-1i64)).unwrap(); - assert_eq!(s, [0b10100000 ^ 2, '-' as u8, '1' as u8]); // string of lengths 2 with value "-1" + assert_eq!(s, [0b10100000 ^ 2, b'-', b'1']); // string of lengths 2 with value "-1" let s = to_msgpack_vec(&Int256::from(-1i64)).unwrap(); - assert_eq!(s, [0b10100000 ^ 2, '-' as u8, '1' as u8]); // string of lengths 2 with value "-1" + assert_eq!(s, [0b10100000 ^ 2, b'-', b'1']); // string of lengths 2 with value "-1" let s = to_msgpack_vec(&Int512::from(-1i64)).unwrap(); - assert_eq!(s, [0b10100000 ^ 2, '-' as u8, '1' as u8]); // string of lengths 2 with value "-1" + assert_eq!(s, [0b10100000 ^ 2, b'-', b'1']); // string of lengths 2 with value "-1" } } } From 28348706a9c49efd4cf704599e3bfdbcb2cde9cf Mon Sep 17 00:00:00 2001 From: Simon Warta Date: Wed, 18 Sep 2024 14:32:03 +0200 Subject: [PATCH 3/3] Improve serialize/deserialize docs for `UintXXX`/`IntYYY` --- packages/std/src/math/mod.rs | 39 ++++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/packages/std/src/math/mod.rs b/packages/std/src/math/mod.rs index c1c1f2682..dd4d0309c 100644 --- a/packages/std/src/math/mod.rs +++ b/packages/std/src/math/mod.rs @@ -33,7 +33,39 @@ pub use uint64::Uint64; macro_rules! impl_int_serde { ($ty:ty) => { impl ::serde::Serialize for $ty { - /// Serializes as an integer string using base 10 + /// Serializes as an integer string using base 10. + /// + /// We consistently serialize all `UintXXX` and `IntYYY` types as strings in JSON + /// to ensure the best possible compatibility with clients. E.g. JavaScript and jq + /// only support up to ~53bit numbers without losing precision, making it hard to use + /// serialized `u64`s on other systems than Rust or Go. `Uint64`/`Int64` ensure the full + /// 64 bit range is supported. For larger integers, the use of strings is pretty much the + /// only reasonable way to store them in JSON. + /// + /// For binary encodings (notably MessagePack) strings are used too. The reason is that + /// in MessagePack integers are limited to 64 bit and we strive for consistent encoding + /// within the `UintXXX`/`IntYYY` family. Also for small to mid sized values, decimal strings + /// are often more compact than a fixed-length binary encoding. + /// + /// ## Examples + /// + /// Serialize to JSON: + /// + /// ``` + /// # use cosmwasm_std::{to_json_vec, Uint64}; + /// let value = Uint64::new(17); + /// let serialized = to_json_vec(&value).unwrap(); + /// assert_eq!(serialized, b"\"17\""); + /// ``` + /// + /// Serialize to MessagePack: + /// + /// ``` + /// # use cosmwasm_std::{to_msgpack_vec, Uint64}; + /// let value = Uint64::new(17); + /// let serialized = to_msgpack_vec(&value).unwrap(); + /// assert_eq!(serialized, [0b10100000 ^ 2, b'1', b'7']); // string of lengths 2 with value "17" + /// ``` fn serialize(&self, serializer: S) -> Result where S: ::serde::ser::Serializer, @@ -43,7 +75,10 @@ macro_rules! impl_int_serde { } impl<'de> ::serde::Deserialize<'de> for $ty { - /// Deserialized from an integer string using base 10 + /// Deserializes from an integer string using base 10. + /// + /// See the [`Serialize` documentation](#method.serialize) for a few more words + /// on the encoding of the `UintXXX`/`IntYYY` family. fn deserialize(deserializer: D) -> Result where D: ::serde::de::Deserializer<'de>,