From 4273316d8f612f4cef436730ea2635d2bd6a34d3 Mon Sep 17 00:00:00 2001 From: DarkSky <25152247+darkskygit@users.noreply.github.com> Date: Wed, 14 Jun 2023 15:44:20 +0800 Subject: [PATCH] feat: crdt type create & convert (#461) * feat: add convert for any * chore: disable add to project action * feat: add content convert & sub array support * feat: fix item ref for ytype * fix: test case * chore: cleanup type convertor --- .github/workflows/add-to-project.yml | 40 ++-- Cargo.lock | 1 + libs/jwst-codec/Cargo.toml | 3 +- libs/jwst-codec/src/doc/codec/any.rs | 221 ++++++++++++------ libs/jwst-codec/src/doc/codec/content.rs | 41 ++++ libs/jwst-codec/src/doc/document.rs | 45 +++- libs/jwst-codec/src/doc/types/array.rs | 12 +- libs/jwst-codec/src/doc/types/list/mod.rs | 5 + .../src/doc/types/list/search_marker.rs | 4 +- libs/jwst-codec/src/doc/types/mod.rs | 8 +- libs/jwst-codec/src/doc/types/text.rs | 12 +- 11 files changed, 286 insertions(+), 106 deletions(-) diff --git a/.github/workflows/add-to-project.yml b/.github/workflows/add-to-project.yml index 4f50752bc..df04928a4 100644 --- a/.github/workflows/add-to-project.yml +++ b/.github/workflows/add-to-project.yml @@ -1,24 +1,24 @@ name: Add to GitHub projects -on: - issues: - types: - - opened - pull_request_target: - types: - - opened - - reopened +# on: +# issues: +# types: +# - opened +# pull_request_target: +# types: +# - opened +# - reopened jobs: - add-to-project: - name: Add issues and pull requests - runs-on: ubuntu-latest - steps: - - uses: actions/add-to-project@v0.4.0 - with: - # You can target a repository in a different organization - # to the issue - project-url: https://github.com/orgs/toeverything/projects/10 - github-token: ${{ secrets.ADD_TO_PROJECT_PAT }} - # labeled: bug, needs-triage - # label-operator: OR + add-to-project: + name: Add issues and pull requests + runs-on: ubuntu-latest + steps: + - uses: actions/add-to-project@v0.4.0 + with: + # You can target a repository in a different organization + # to the issue + project-url: https://github.com/orgs/toeverything/projects/10 + github-token: ${{ secrets.ADD_TO_PROJECT_PAT }} + # labeled: bug, needs-triage + # label-operator: OR diff --git a/Cargo.lock b/Cargo.lock index 34caaceff..dedc6293e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2583,6 +2583,7 @@ name = "jwst-codec" version = "0.1.0" dependencies = [ "arbitrary", + "assert-json-diff", "bitvec", "byteorder", "criterion", diff --git a/libs/jwst-codec/Cargo.toml b/libs/jwst-codec/Cargo.toml index cc31f92e8..dc7a9d410 100644 --- a/libs/jwst-codec/Cargo.toml +++ b/libs/jwst-codec/Cargo.toml @@ -25,8 +25,9 @@ arbitrary = { version = "1.3.0", features = ["derive"] } ordered-float = { version = "3.6.0", features = ["arbitrary"] } [dev-dependencies] +assert-json-diff = "2.0.2" criterion = { version = "0.5.1", features = ["html_reports"] } -lib0 = "0.16.5" +lib0 = { version = "0.16.5", features = ["lib0-serde"] } ordered-float = { version = "3.6.0", features = ["proptest"] } path-ext = "0.1.0" proptest = "1.1.0" diff --git a/libs/jwst-codec/src/doc/codec/any.rs b/libs/jwst-codec/src/doc/codec/any.rs index 407636d41..c7abf1c99 100644 --- a/libs/jwst-codec/src/doc/codec/any.rs +++ b/libs/jwst-codec/src/doc/codec/any.rs @@ -112,112 +112,142 @@ impl CrdtWrite for Any { } } -// TODO: impl for Any::Undefined -impl From for Any { - fn from(s: String) -> Self { - Any::String(s) +impl Any { + fn read_key_value(reader: &mut R) -> JwstCodecResult<(String, Any)> { + let key = reader.read_var_string()?; + let value = Self::read(reader)?; + + Ok((key, value)) } -} -impl From<&str> for Any { - fn from(s: &str) -> Self { - Any::from(s.to_string()) + fn write_key_value(writer: &mut W, key: &str, value: &Any) -> JwstCodecResult { + writer.write_var_string(key)?; + value.write(writer)?; + + Ok(()) } -} -impl> From> for Any { - fn from(value: Option) -> Self { - if let Some(val) = value { - val.into() - } else { - Any::Null - } + pub(crate) fn read_multiple(reader: &mut R) -> JwstCodecResult> { + let len = reader.read_var_u64()?; + let any = (0..len) + .map(|_| Any::read(reader)) + .collect::, _>>()?; + + Ok(any) } -} -impl From for Any { - fn from(value: u64) -> Self { - Any::Integer(value) + pub(crate) fn write_multiple(writer: &mut W, any: &[Any]) -> JwstCodecResult { + writer.write_var_u64(any.len() as u64)?; + for value in any { + value.write(writer)?; + } + + Ok(()) } } -impl From> for Any { - fn from(value: OrderedFloat) -> Self { - Any::Float32(value) - } +macro_rules! impl_primitive_from { + (unsigned, $($ty: ty),*) => { + $( + impl From<$ty> for Any { + fn from(value: $ty) -> Self { + Self::Integer(value.into()) + } + } + )* + }; + (signed, $($ty: ty),*) => { + $( + impl From<$ty> for Any { + fn from(value: $ty) -> Self { + Self::BigInt64(value.into()) + } + } + )* + }; + (string, $($ty: ty),*) => { + $( + impl From<$ty> for Any { + fn from(value: $ty) -> Self { + Self::String(value.into()) + } + } + )* + }; } -impl From> for Any { - fn from(value: OrderedFloat) -> Self { - Any::Float64(value) +impl_primitive_from!(unsigned, u8, u16, u32, u64); +impl_primitive_from!(signed, i8, i16, i32, i64); +impl_primitive_from!(string, String, &str); + +impl From for Any { + fn from(value: f32) -> Self { + Self::Float32(value.into()) } } -impl From for Any { - fn from(value: i64) -> Self { - Any::BigInt64(value) +impl From for Any { + fn from(value: f64) -> Self { + Self::Float64(value.into()) } } impl From for Any { fn from(value: bool) -> Self { if value { - Any::True + Self::True } else { - Any::False + Self::False } } } -impl From> for Any { - fn from(value: HashMap) -> Self { - Any::Object(value) +impl FromIterator for Any { + fn from_iter>(iter: I) -> Self { + Self::Array(iter.into_iter().collect()) } } -impl From> for Any { - fn from(value: Vec) -> Self { - Any::Array(value) +impl<'a> FromIterator<&'a Any> for Any { + fn from_iter>(iter: I) -> Self { + Self::Array(iter.into_iter().cloned().collect()) } } -impl From> for Any { - fn from(value: Vec) -> Self { - Any::Binary(value) +impl FromIterator<(String, Any)> for Any { + fn from_iter>(iter: I) -> Self { + let mut map = HashMap::new(); + map.extend(iter); + Self::Object(map) } } -impl Any { - fn read_key_value(reader: &mut R) -> JwstCodecResult<(String, Any)> { - let key = reader.read_var_string()?; - let value = Self::read(reader)?; - - Ok((key, value)) +impl From> for Any { + fn from(value: HashMap) -> Self { + Self::Object(value) } +} - fn write_key_value(writer: &mut W, key: &str, value: &Any) -> JwstCodecResult { - writer.write_var_string(key)?; - value.write(writer)?; - - Ok(()) +impl From> for Any { + fn from(value: Vec) -> Self { + Self::Binary(value) } +} - pub(crate) fn read_multiple(reader: &mut R) -> JwstCodecResult> { - let len = reader.read_var_u64()?; - let any = (0..len) - .map(|_| Any::read(reader)) - .collect::, _>>()?; - - Ok(any) +impl From<&[u8]> for Any { + fn from(value: &[u8]) -> Self { + Self::Binary(value.into()) } +} - pub(crate) fn write_multiple(writer: &mut W, any: &[Any]) -> JwstCodecResult { - writer.write_var_u64(any.len() as u64)?; - for value in any { - value.write(writer)?; +// TODO: impl for Any::Undefined +impl> From> for Any { + fn from(value: Option) -> Self { + if let Some(val) = value { + val.into() + } else { + Any::Null } - - Ok(()) } } @@ -310,4 +340,65 @@ mod tests { } } } + + #[test] + fn test_convert_to_any() { + let any: Vec = vec![ + 42u8.into(), + 42u16.into(), + 42u32.into(), + 42u64.into(), + 114.514f32.into(), + 1919.810f64.into(), + (-42i8).into(), + (-42i16).into(), + (-42i32).into(), + (-42i64).into(), + false.into(), + true.into(), + "JWST".to_string().into(), + "OctoBase".into(), + vec![1u8, 9, 1, 9].into(), + (&[8u8, 1, 0][..]).into(), + [Any::True, 42u8.into()].iter().collect(), + ]; + assert_eq!( + any, + vec![ + Any::Integer(42), + Any::Integer(42), + Any::Integer(42), + Any::Integer(42), + Any::Float32(114.514.into()), + Any::Float64(1919.810.into()), + Any::BigInt64(-42), + Any::BigInt64(-42), + Any::BigInt64(-42), + Any::BigInt64(-42), + Any::False, + Any::True, + Any::String("JWST".to_string()), + Any::String("OctoBase".to_string()), + Any::Binary(vec![1, 9, 1, 9]), + Any::Binary(vec![8, 1, 0]), + Any::Array(vec![Any::True, Any::Integer(42)]) + ] + ); + + assert_eq!( + vec![("key".to_string(), 10u64.into())] + .into_iter() + .collect::(), + Any::Object(HashMap::from_iter(vec![( + "key".to_string(), + Any::Integer(10) + )])) + ); + + let any: Any = 10u64.into(); + assert_eq!( + [any].iter().collect::(), + Any::Array(vec![Any::Integer(10)]) + ); + } } diff --git a/libs/jwst-codec/src/doc/codec/content.rs b/libs/jwst-codec/src/doc/codec/content.rs index 9ad9f0f98..ced852a94 100644 --- a/libs/jwst-codec/src/doc/codec/content.rs +++ b/libs/jwst-codec/src/doc/codec/content.rs @@ -329,8 +329,49 @@ impl Content { s.split_at(utf_8_offset) } + + pub fn as_array(&self) -> Option { + if let Self::Type(type_ref) = self { + return Array::try_from(type_ref.clone()).ok(); + } + None + } } +macro_rules! impl_primitive_from { + (any => $($ty:ty),* $(,)?) => { + $( + impl From<$ty> for Content { + fn from(value: $ty) -> Self { + Self::Any(vec![value.into()]) + } + } + )* + }; + (raw => $($ty:ty: $v:ident),* $(,)?) => { + $( + impl From<$ty> for Content { + fn from(value: $ty) -> Self { + Self::$v(value) + } + } + )* + }; + (type_ref => $($ty:ty),* $(,)?) => { + $( + impl From<$ty> for Content { + fn from(type_ref: $ty) -> Self { + Self::Type(type_ref.as_inner().clone()) + } + } + )* + } +} + +impl_primitive_from! { any => u8, u16, u32, u64, i8, i16, i32, i64, String, &str, f32, f64, bool } +impl_primitive_from! { raw => Vec: Binary, YTypeRef: Type } +impl_primitive_from! { type_ref => Array, Text } + #[cfg(test)] mod tests { use super::*; diff --git a/libs/jwst-codec/src/doc/document.rs b/libs/jwst-codec/src/doc/document.rs index 0778efd87..5799e9c01 100644 --- a/libs/jwst-codec/src/doc/document.rs +++ b/libs/jwst-codec/src/doc/document.rs @@ -124,7 +124,7 @@ impl Doc { Ok(()) } - pub fn get_or_crate_text(&self, name: &str) -> JwstCodecResult { + pub fn get_or_create_text(&self, name: &str) -> JwstCodecResult { YTypeBuilder::new(self.store.clone()) .with_kind(YTypeKind::Text) .set_name(name.to_string()) @@ -144,6 +144,12 @@ impl Doc { .build() } + pub fn create_array(&self) -> JwstCodecResult { + YTypeBuilder::new(self.store.clone()) + .with_kind(YTypeKind::Array) + .build() + } + pub fn get_or_create_map(&self, str: &str) -> JwstCodecResult { YTypeBuilder::new(self.store.clone()) .with_kind(YTypeKind::Map) @@ -174,7 +180,7 @@ impl Doc { #[cfg(test)] mod tests { use super::*; - use yrs::{Array, Map, Transact}; + use yrs::{types::ToJson, updates::decoder::Decode, Array, Map, Transact}; #[test] fn double_run_test_with_yrs_basic() { @@ -227,4 +233,39 @@ mod tests { doc_new.encode_update_v1().unwrap() ); } + + #[test] + fn test_array_create() { + let binary = { + let doc = Doc::with_client(0); + let mut array = doc.get_or_create_array("abc").unwrap(); + array.insert(0, 42).unwrap(); + array.insert(1, -42).unwrap(); + array.insert(2, true).unwrap(); + array.insert(3, false).unwrap(); + array.insert(4, "hello").unwrap(); + array.insert(5, "world").unwrap(); + + let mut sub_array = doc.create_array().unwrap(); + array.insert(6, sub_array.clone()).unwrap(); + // FIXME: array need insert first to compatible with yrs + sub_array.insert(0, 1).unwrap(); + + doc.encode_update_v1().unwrap() + }; + + { + use yrs::{Doc, Update}; + + let ydoc = Doc::new(); + let array = ydoc.get_or_insert_array("abc"); + let mut trx = ydoc.transact_mut(); + trx.apply_update(Update::decode_v1(&binary).unwrap()); + + assert_json_diff::assert_json_eq!( + array.to_json(&trx), + serde_json::json!([42, -42, true, false, "hello", "world", [1]]) + ); + } + } } diff --git a/libs/jwst-codec/src/doc/types/array.rs b/libs/jwst-codec/src/doc/types/array.rs index 11e5c791c..75c367825 100644 --- a/libs/jwst-codec/src/doc/types/array.rs +++ b/libs/jwst-codec/src/doc/types/array.rs @@ -39,24 +39,18 @@ impl Array { ArrayIter(self.iter_item()) } - pub fn push>(&mut self, val: V) -> JwstCodecResult { + pub fn push>(&mut self, val: V) -> JwstCodecResult { self.insert(self.len(), val) } - pub fn insert>(&mut self, idx: u64, val: V) -> JwstCodecResult { - let contents = Self::group_content(val); + pub fn insert>(&mut self, idx: u64, val: V) -> JwstCodecResult { + let contents = vec![val.into()]; self.insert_at(idx, contents) } pub fn remove(&mut self, idx: u64, len: u64) -> JwstCodecResult { self.remove_at(idx, len) } - - #[inline(always)] - fn group_content>(val: V) -> Vec { - let any = val.into(); - vec![any.into()] - } } impl Index for Array { diff --git a/libs/jwst-codec/src/doc/types/list/mod.rs b/libs/jwst-codec/src/doc/types/list/mod.rs index 1204dc194..2b7600d83 100644 --- a/libs/jwst-codec/src/doc/types/list/mod.rs +++ b/libs/jwst-codec/src/doc/types/list/mod.rs @@ -164,6 +164,11 @@ pub(crate) trait ListType: AsInner { .parent(Some(Parent::Type(pos.parent.clone()))) .build(), ); + + if let Content::Type(t) = item.content.as_ref() { + t.write().unwrap().set_item(item.clone()); + } + store.integrate(StructInfo::Item(item.clone()), 0, Some(&mut lock))?; pos.right = Some(item); diff --git a/libs/jwst-codec/src/doc/types/list/search_marker.rs b/libs/jwst-codec/src/doc/types/list/search_marker.rs index 59720689b..f42e79dbe 100644 --- a/libs/jwst-codec/src/doc/types/list/search_marker.rs +++ b/libs/jwst-codec/src/doc/types/list/search_marker.rs @@ -235,7 +235,7 @@ mod tests { #[test] fn test_search_marker_flaky() { let doc = Doc::default(); - let mut text = doc.get_or_crate_text("test").unwrap(); + let mut text = doc.get_or_create_text("test").unwrap(); text.insert(0, "0").unwrap(); text.insert(1, "1").unwrap(); text.insert(0, "0").unwrap(); @@ -246,7 +246,7 @@ mod tests { let iteration = 20; let doc = Doc::with_client(1); - let mut text = doc.get_or_crate_text("test").unwrap(); + let mut text = doc.get_or_create_text("test").unwrap(); text.insert(0, "This is a string with length 32.").unwrap(); let mut len = text.len(); diff --git a/libs/jwst-codec/src/doc/types/mod.rs b/libs/jwst-codec/src/doc/types/mod.rs index 50b39bcdd..4e151a29e 100644 --- a/libs/jwst-codec/src/doc/types/mod.rs +++ b/libs/jwst-codec/src/doc/types/mod.rs @@ -79,6 +79,10 @@ impl YType { Ok(()) } + pub fn set_item(&mut self, item: ItemRef) { + self.item = Some(Arc::downgrade(&item)); + } + pub fn store<'a>(&self) -> Option> { if let Some(store) = self.store.upgrade() { let ptr = unsafe { &*Arc::as_ptr(&store) }; @@ -281,7 +285,9 @@ macro_rules! impl_type { inner.set_kind(super::YTypeKind::$name)?; Ok($name::new(value.clone())) } - _ => Err($crate::JwstCodecError::TypeCastError("Text")), + _ => Err($crate::JwstCodecError::TypeCastError(std::stringify!( + $name + ))), } } } diff --git a/libs/jwst-codec/src/doc/types/text.rs b/libs/jwst-codec/src/doc/types/text.rs index 642e0f2c0..dbd74b22e 100644 --- a/libs/jwst-codec/src/doc/types/text.rs +++ b/libs/jwst-codec/src/doc/types/text.rs @@ -81,7 +81,7 @@ mod tests { let mut handles = Vec::new(); let doc = Doc::with_client(1); - let mut text = doc.get_or_crate_text("test").unwrap(); + let mut text = doc.get_or_create_text("test").unwrap(); text.insert(0, "This is a string with length 32.").unwrap(); let added_len = Arc::new(AtomicUsize::new(32)); @@ -108,7 +108,7 @@ mod tests { let mut rand = rand.clone(); let len = added_len.clone(); handles.push(std::thread::spawn(move || { - let mut text = doc.get_or_crate_text("test").unwrap(); + let mut text = doc.get_or_create_text("test").unwrap(); let pos = rand.gen_range(0..text.len()); let string = format!("hello doc{i}"); @@ -129,7 +129,7 @@ mod tests { fn parallel_ins_del_text(seed: u64) { let doc = Doc::with_client(1); let mut rand = ChaCha20Rng::seed_from_u64(seed); - let mut text = doc.get_or_crate_text("test").unwrap(); + let mut text = doc.get_or_create_text("test").unwrap(); text.insert(0, "This is a string with length 32.").unwrap(); let mut handles = Vec::new(); @@ -183,7 +183,7 @@ mod tests { }; let doc = Doc::new_from_binary(binary).unwrap(); - let mut text = doc.get_or_crate_text("greating").unwrap(); + let mut text = doc.get_or_create_text("greating").unwrap(); assert_eq!(text.to_string(), "hello world"); @@ -196,7 +196,7 @@ mod tests { fn test_recover_from_octobase_encoder() { let binary = { let doc = Doc::with_client(1); - let mut text = doc.get_or_crate_text("greating").unwrap(); + let mut text = doc.get_or_create_text("greating").unwrap(); text.insert(0, "hello").unwrap(); text.insert(5, " world!").unwrap(); text.remove(11, 1).unwrap(); @@ -205,7 +205,7 @@ mod tests { }; let doc = Doc::new_from_binary(binary).unwrap(); - let mut text = doc.get_or_crate_text("greating").unwrap(); + let mut text = doc.get_or_create_text("greating").unwrap(); assert_eq!(text.to_string(), "hello world");