From 71a11069379a4b19219a1509067c47597ff59020 Mon Sep 17 00:00:00 2001 From: Micah Date: Wed, 15 Nov 2023 15:15:10 -0800 Subject: [PATCH 001/366] Initial implementation of diffing --- Cargo.toml | 3 + src/snapshot/diff/hash.rs | 118 ++++++++++++++++++++++++++++++++++++++ src/snapshot/diff/mod.rs | 3 + src/snapshot/mod.rs | 2 + 4 files changed, 126 insertions(+) create mode 100644 src/snapshot/diff/hash.rs create mode 100644 src/snapshot/diff/mod.rs diff --git a/Cargo.toml b/Cargo.toml index 0b5ae9e68..0627184d2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -90,6 +90,9 @@ clap = { version = "3.1.18", features = ["derive"] } profiling = "1.0.6" tracy-client = { version = "0.13.2", optional = true } +blake3 = "1.5.0" +float-cmp = "0.9.0" + [target.'cfg(windows)'.dependencies] winreg = "0.10.1" diff --git a/src/snapshot/diff/hash.rs b/src/snapshot/diff/hash.rs new file mode 100644 index 000000000..d60a56956 --- /dev/null +++ b/src/snapshot/diff/hash.rs @@ -0,0 +1,118 @@ +//! Hashing utility for a RojoTree + +use blake3::{Hash, Hasher}; +use rbx_dom_weak::{ + types::{Ref, Variant}, + Instance, WeakDom, +}; + +use crate::snapshot::RojoTree; + +use std::collections::{HashMap, VecDeque}; + +pub fn hash_tree(tree: &RojoTree) -> HashMap { + let dom = tree.inner(); + let mut map: HashMap = HashMap::new(); + let mut order = descendants(dom); + + let mut prop_list = Vec::with_capacity(2); + + // function get_hash_id(inst) + // return hash({ sort(foreach(inst.properties, hash)), sort(foreach(inst.children, get_hash_id)) }) + // end + while let Some(referent) = order.pop() { + log::info!("processing {referent}"); + let inst = dom.get_by_ref(referent).unwrap(); + let hash = hash_inst(&mut prop_list, &map, inst); + + map.insert(referent, hash); + } + + map +} + +fn hash_inst<'map, 'inst>( + prop_list: &mut Vec<(&'inst str, &'inst Variant)>, + map: &'map HashMap, + inst: &'inst Instance, +) -> Hash { + let mut hasher = Hasher::new(); + hasher.update(inst.class.as_bytes()); + hasher.update(inst.name.as_bytes()); + + for (name, value) in &inst.properties { + prop_list.push((name, value)) + } + prop_list.sort_unstable_by_key(|(key, _)| *key); + for (name, value) in prop_list.iter() { + hasher.update(name.as_bytes()); + hash_variant(&mut hasher, value) + } + + let mut child_list = Vec::with_capacity(inst.children().len()); + for child in inst.children() { + if let Some(hash) = map.get(child) { + child_list.push(hash.as_bytes()) + } else { + panic!("Invariant: child {} not hashed before its parent", child); + } + } + child_list.sort_unstable(); + for hash in child_list { + hasher.update(hash); + } + + prop_list.clear(); + + hasher.finalize() +} + +fn descendants(dom: &WeakDom) -> Vec { + let mut queue = VecDeque::new(); + let mut ordered = Vec::new(); + queue.push_front(dom.root_ref()); + + while let Some(referent) = queue.pop_front() { + let inst = dom + .get_by_ref(referent) + .expect("Invariant: WeakDom had a Ref that wasn't inside it"); + ordered.push(referent); + for child in inst.children() { + queue.push_back(*child) + } + } + + ordered +} + +macro_rules! n_hash { + ($hash:ident, $($num:expr),*) => { + {$( + $hash.update(&$num.to_le_bytes()); + )*} + }; +} + +macro_rules! hash { + ($hash:ident, $value:expr) => {{ + $hash.update($value); + }}; +} + +fn hash_variant(hasher: &mut Hasher, value: &Variant) { + // im da joker babeh + match value { + Variant::String(str) => hash!(hasher, str.as_bytes()), + Variant::Bool(bool) => hash!(hasher, &[*bool as u8]), + Variant::Float32(n) => n_hash!(hasher, n), + Variant::Float64(n) => n_hash!(hasher, n), + Variant::Int32(n) => n_hash!(hasher, n), + Variant::Int64(n) => n_hash!(hasher, n), + Variant::BinaryString(bytes) => hash!(hasher, bytes.as_ref()), + Variant::Vector3(v3) => n_hash!(hasher, v3.x, v3.y, v3.z), + Variant::Vector2(v2) => n_hash!(hasher, v2.x, v2.y), + + // TODO: Add the rest + _ => (), + } +} diff --git a/src/snapshot/diff/mod.rs b/src/snapshot/diff/mod.rs new file mode 100644 index 000000000..b820cdbd8 --- /dev/null +++ b/src/snapshot/diff/mod.rs @@ -0,0 +1,3 @@ +mod hash; + +pub use hash::hash_tree; diff --git a/src/snapshot/mod.rs b/src/snapshot/mod.rs index 5aacdad6e..6a36b893b 100644 --- a/src/snapshot/mod.rs +++ b/src/snapshot/mod.rs @@ -48,6 +48,7 @@ #![allow(dead_code)] +mod diff; mod instance_snapshot; mod metadata; mod patch; @@ -55,6 +56,7 @@ mod patch_apply; mod patch_compute; mod tree; +pub use diff::*; pub use instance_snapshot::InstanceSnapshot; pub use metadata::*; pub use patch::*; From da18517b0bcad5061ff1bf522263244cce33bd41 Mon Sep 17 00:00:00 2001 From: Micah Date: Wed, 15 Nov 2023 15:18:09 -0800 Subject: [PATCH 002/366] I am not the joker, baby --- src/snapshot/diff/hash.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/snapshot/diff/hash.rs b/src/snapshot/diff/hash.rs index d60a56956..95c45a607 100644 --- a/src/snapshot/diff/hash.rs +++ b/src/snapshot/diff/hash.rs @@ -99,8 +99,10 @@ macro_rules! hash { }}; } +/// Places `value` into the provided hasher. fn hash_variant(hasher: &mut Hasher, value: &Variant) { - // im da joker babeh + // We need to round floats, though I'm not sure to what degree we can + // realistically do that. match value { Variant::String(str) => hash!(hasher, str.as_bytes()), Variant::Bool(bool) => hash!(hasher, &[*bool as u8]), From 986e327c71278b3b912b738526b8f47daa1402a3 Mon Sep 17 00:00:00 2001 From: Micah Date: Mon, 20 Nov 2023 10:22:00 -0800 Subject: [PATCH 003/366] Updating hasher implementation --- src/snapshot/diff/hash.rs | 60 ++++++++++++++++++++++++++++++++------- 1 file changed, 49 insertions(+), 11 deletions(-) diff --git a/src/snapshot/diff/hash.rs b/src/snapshot/diff/hash.rs index 95c45a607..3c9909ad8 100644 --- a/src/snapshot/diff/hash.rs +++ b/src/snapshot/diff/hash.rs @@ -2,16 +2,13 @@ use blake3::{Hash, Hasher}; use rbx_dom_weak::{ - types::{Ref, Variant}, + types::{Ref, Variant, Vector3}, Instance, WeakDom, }; -use crate::snapshot::RojoTree; - use std::collections::{HashMap, VecDeque}; -pub fn hash_tree(tree: &RojoTree) -> HashMap { - let dom = tree.inner(); +pub fn hash_tree(dom: &WeakDom) -> HashMap { let mut map: HashMap = HashMap::new(); let mut order = descendants(dom); @@ -21,7 +18,6 @@ pub fn hash_tree(tree: &RojoTree) -> HashMap { // return hash({ sort(foreach(inst.properties, hash)), sort(foreach(inst.children, get_hash_id)) }) // end while let Some(referent) = order.pop() { - log::info!("processing {referent}"); let inst = dom.get_by_ref(referent).unwrap(); let hash = hash_inst(&mut prop_list, &map, inst); @@ -88,7 +84,7 @@ fn descendants(dom: &WeakDom) -> Vec { macro_rules! n_hash { ($hash:ident, $($num:expr),*) => { {$( - $hash.update(&$num.to_le_bytes()); + $hash.update(&($num).to_le_bytes()); )*} }; } @@ -106,15 +102,57 @@ fn hash_variant(hasher: &mut Hasher, value: &Variant) { match value { Variant::String(str) => hash!(hasher, str.as_bytes()), Variant::Bool(bool) => hash!(hasher, &[*bool as u8]), - Variant::Float32(n) => n_hash!(hasher, n), + Variant::Float32(n) => n_hash!(hasher, round(*n)), Variant::Float64(n) => n_hash!(hasher, n), Variant::Int32(n) => n_hash!(hasher, n), Variant::Int64(n) => n_hash!(hasher, n), Variant::BinaryString(bytes) => hash!(hasher, bytes.as_ref()), - Variant::Vector3(v3) => n_hash!(hasher, v3.x, v3.y, v3.z), - Variant::Vector2(v2) => n_hash!(hasher, v2.x, v2.y), + Variant::Vector3(v3) => vector_hash(hasher, *v3), + Variant::Vector2(v2) => n_hash!(hasher, round(v2.x), round(v2.y)), + Variant::Axes(a) => hash!(hasher, &[a.bits()]), + Variant::BrickColor(color) => n_hash!(hasher, *color as u16), + Variant::CFrame(cf) => { + vector_hash(hasher, cf.position); + vector_hash(hasher, cf.orientation.x); + vector_hash(hasher, cf.orientation.y); + vector_hash(hasher, cf.orientation.z); + } + Variant::Color3(color) => n_hash!(hasher, round(color.r), round(color.g), round(color.b)), + Variant::Color3uint8(color) => hash!(hasher, &[color.r, color.b, color.g]), + Variant::ColorSequence(seq) => { + let mut new = Vec::with_capacity(seq.keypoints.len()); + for keypoint in &seq.keypoints { + new.push(keypoint); + } + new.sort_unstable_by(|a, b| round(a.time).partial_cmp(&round(b.time)).unwrap()); + for keypoint in new { + n_hash!( + hasher, + round(keypoint.time), + round(keypoint.color.r), + round(keypoint.color.g), + round(keypoint.color.b) + ) + } + } + // TODO: Make this more ergonomic + Variant::Content(content) => { + let s: &str = content.as_ref(); + hash!(hasher, s.as_bytes()) + } + Variant::Enum(e) => n_hash!(hasher, e.to_u32()), + Variant::Faces(f) => hash!(hasher, &[f.bits()]), // TODO: Add the rest - _ => (), + // Hashing UniqueId properties doesn't make sense + Variant::UniqueId(_) | _ => (), } } + +fn vector_hash(hasher: &mut Hasher, vector: Vector3) { + n_hash!(hasher, round(vector.x), round(vector.y), round(vector.z)) +} + +fn round(float: f32) -> f32 { + (float * 10.0).round() / 10.0 +} From 7e7ef62c7a28ee358f803aa22bdb0ec34eda6c6e Mon Sep 17 00:00:00 2001 From: Micah Date: Mon, 20 Nov 2023 11:32:19 -0800 Subject: [PATCH 004/366] ignore default properies for hashing --- src/snapshot/diff/hash.rs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/snapshot/diff/hash.rs b/src/snapshot/diff/hash.rs index 3c9909ad8..aa814152a 100644 --- a/src/snapshot/diff/hash.rs +++ b/src/snapshot/diff/hash.rs @@ -36,9 +36,22 @@ fn hash_inst<'map, 'inst>( hasher.update(inst.class.as_bytes()); hasher.update(inst.name.as_bytes()); + let descriptor = rbx_reflection_database::get() + .classes + .get(inst.class.as_str()) + .expect("class should be known to Rojo"); + for (name, value) in &inst.properties { - prop_list.push((name, value)) + if let Some(default) = descriptor.default_properties.get(name.as_str()) { + // TODO: Float comparison + if value != default { + prop_list.push((name, value)) + } + } else { + prop_list.push((name, value)) + } } + prop_list.sort_unstable_by_key(|(key, _)| *key); for (name, value) in prop_list.iter() { hasher.update(name.as_bytes()); From ddc85098cc2dd00d5d8443a4632fad604012fc60 Mon Sep 17 00:00:00 2001 From: Micah Date: Mon, 20 Nov 2023 11:44:13 -0800 Subject: [PATCH 005/366] Move variant stuff to its own file --- src/snapshot/diff/hash.rs | 80 ++--------------------------------- src/snapshot/diff/mod.rs | 2 + src/snapshot/diff/variant.rs | 81 ++++++++++++++++++++++++++++++++++++ 3 files changed, 86 insertions(+), 77 deletions(-) create mode 100644 src/snapshot/diff/variant.rs diff --git a/src/snapshot/diff/hash.rs b/src/snapshot/diff/hash.rs index aa814152a..b0122214e 100644 --- a/src/snapshot/diff/hash.rs +++ b/src/snapshot/diff/hash.rs @@ -2,12 +2,14 @@ use blake3::{Hash, Hasher}; use rbx_dom_weak::{ - types::{Ref, Variant, Vector3}, + types::{Ref, Variant}, Instance, WeakDom, }; use std::collections::{HashMap, VecDeque}; +use super::hash_variant; + pub fn hash_tree(dom: &WeakDom) -> HashMap { let mut map: HashMap = HashMap::new(); let mut order = descendants(dom); @@ -93,79 +95,3 @@ fn descendants(dom: &WeakDom) -> Vec { ordered } - -macro_rules! n_hash { - ($hash:ident, $($num:expr),*) => { - {$( - $hash.update(&($num).to_le_bytes()); - )*} - }; -} - -macro_rules! hash { - ($hash:ident, $value:expr) => {{ - $hash.update($value); - }}; -} - -/// Places `value` into the provided hasher. -fn hash_variant(hasher: &mut Hasher, value: &Variant) { - // We need to round floats, though I'm not sure to what degree we can - // realistically do that. - match value { - Variant::String(str) => hash!(hasher, str.as_bytes()), - Variant::Bool(bool) => hash!(hasher, &[*bool as u8]), - Variant::Float32(n) => n_hash!(hasher, round(*n)), - Variant::Float64(n) => n_hash!(hasher, n), - Variant::Int32(n) => n_hash!(hasher, n), - Variant::Int64(n) => n_hash!(hasher, n), - Variant::BinaryString(bytes) => hash!(hasher, bytes.as_ref()), - Variant::Vector3(v3) => vector_hash(hasher, *v3), - Variant::Vector2(v2) => n_hash!(hasher, round(v2.x), round(v2.y)), - Variant::Axes(a) => hash!(hasher, &[a.bits()]), - Variant::BrickColor(color) => n_hash!(hasher, *color as u16), - Variant::CFrame(cf) => { - vector_hash(hasher, cf.position); - vector_hash(hasher, cf.orientation.x); - vector_hash(hasher, cf.orientation.y); - vector_hash(hasher, cf.orientation.z); - } - Variant::Color3(color) => n_hash!(hasher, round(color.r), round(color.g), round(color.b)), - Variant::Color3uint8(color) => hash!(hasher, &[color.r, color.b, color.g]), - Variant::ColorSequence(seq) => { - let mut new = Vec::with_capacity(seq.keypoints.len()); - for keypoint in &seq.keypoints { - new.push(keypoint); - } - new.sort_unstable_by(|a, b| round(a.time).partial_cmp(&round(b.time)).unwrap()); - for keypoint in new { - n_hash!( - hasher, - round(keypoint.time), - round(keypoint.color.r), - round(keypoint.color.g), - round(keypoint.color.b) - ) - } - } - // TODO: Make this more ergonomic - Variant::Content(content) => { - let s: &str = content.as_ref(); - hash!(hasher, s.as_bytes()) - } - Variant::Enum(e) => n_hash!(hasher, e.to_u32()), - Variant::Faces(f) => hash!(hasher, &[f.bits()]), - - // TODO: Add the rest - // Hashing UniqueId properties doesn't make sense - Variant::UniqueId(_) | _ => (), - } -} - -fn vector_hash(hasher: &mut Hasher, vector: Vector3) { - n_hash!(hasher, round(vector.x), round(vector.y), round(vector.z)) -} - -fn round(float: f32) -> f32 { - (float * 10.0).round() / 10.0 -} diff --git a/src/snapshot/diff/mod.rs b/src/snapshot/diff/mod.rs index b820cdbd8..0a1e6ec1f 100644 --- a/src/snapshot/diff/mod.rs +++ b/src/snapshot/diff/mod.rs @@ -1,3 +1,5 @@ mod hash; +mod variant; pub use hash::hash_tree; +pub use variant::hash_variant; diff --git a/src/snapshot/diff/variant.rs b/src/snapshot/diff/variant.rs new file mode 100644 index 000000000..8ee7f7189 --- /dev/null +++ b/src/snapshot/diff/variant.rs @@ -0,0 +1,81 @@ +use blake3::Hasher; +use rbx_dom_weak::types::{Variant, Vector3}; + +macro_rules! round { + ($value:expr) => { + (($value * 10.0).round() / 10.0) + }; +} + +macro_rules! n_hash { + ($hash:ident, $($num:expr),*) => { + {$( + $hash.update(&($num).to_le_bytes()); + )*} + }; +} + +macro_rules! hash { + ($hash:ident, $value:expr) => {{ + $hash.update($value); + }}; +} + +/// Places `value` into the provided hasher. +pub fn hash_variant(hasher: &mut Hasher, value: &Variant) { + // We need to round floats, though I'm not sure to what degree we can + // realistically do that. + match value { + Variant::String(str) => hash!(hasher, str.as_bytes()), + Variant::Bool(bool) => hash!(hasher, &[*bool as u8]), + Variant::Float32(n) => n_hash!(hasher, round!(*n)), + Variant::Float64(n) => n_hash!(hasher, round!(n)), + Variant::Int32(n) => n_hash!(hasher, n), + Variant::Int64(n) => n_hash!(hasher, n), + Variant::BinaryString(bytes) => hash!(hasher, bytes.as_ref()), + Variant::Vector3(v3) => vector_hash(hasher, *v3), + Variant::Vector2(v2) => n_hash!(hasher, round!(v2.x), round!(v2.y)), + Variant::Axes(a) => hash!(hasher, &[a.bits()]), + Variant::BrickColor(color) => n_hash!(hasher, *color as u16), + Variant::CFrame(cf) => { + vector_hash(hasher, cf.position); + vector_hash(hasher, cf.orientation.x); + vector_hash(hasher, cf.orientation.y); + vector_hash(hasher, cf.orientation.z); + } + Variant::Color3(color) => { + n_hash!(hasher, round!(color.r), round!(color.g), round!(color.b)) + } + Variant::Color3uint8(color) => hash!(hasher, &[color.r, color.b, color.g]), + Variant::ColorSequence(seq) => { + let mut new = Vec::with_capacity(seq.keypoints.len()); + for keypoint in &seq.keypoints { + new.push(keypoint); + } + new.sort_unstable_by(|a, b| round!(a.time).partial_cmp(&round!(b.time)).unwrap()); + for keypoint in new { + n_hash!( + hasher, + round!(keypoint.time), + round!(keypoint.color.r), + round!(keypoint.color.g), + round!(keypoint.color.b) + ) + } + } + Variant::Content(content) => { + let s: &str = content.as_ref(); + hash!(hasher, s.as_bytes()) + } + Variant::Enum(e) => n_hash!(hasher, e.to_u32()), + Variant::Faces(f) => hash!(hasher, &[f.bits()]), + + // TODO: Add the rest + // Hashing UniqueId properties doesn't make sense + Variant::UniqueId(_) | _ => (), + } +} + +fn vector_hash(hasher: &mut Hasher, vector: Vector3) { + n_hash!(hasher, round!(vector.x), round!(vector.y), round!(vector.z)) +} From 1c70f1678dcad748443822642861c434ee6d40d0 Mon Sep 17 00:00:00 2001 From: Micah Date: Mon, 20 Nov 2023 12:06:52 -0800 Subject: [PATCH 006/366] Add a variant_eq function --- src/snapshot/diff/hash.rs | 4 ++-- src/snapshot/diff/mod.rs | 4 ++-- src/snapshot/diff/variant.rs | 45 ++++++++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 4 deletions(-) diff --git a/src/snapshot/diff/hash.rs b/src/snapshot/diff/hash.rs index b0122214e..2176d3958 100644 --- a/src/snapshot/diff/hash.rs +++ b/src/snapshot/diff/hash.rs @@ -8,7 +8,7 @@ use rbx_dom_weak::{ use std::collections::{HashMap, VecDeque}; -use super::hash_variant; +use super::{hash_variant, variant_eq}; pub fn hash_tree(dom: &WeakDom) -> HashMap { let mut map: HashMap = HashMap::new(); @@ -46,7 +46,7 @@ fn hash_inst<'map, 'inst>( for (name, value) in &inst.properties { if let Some(default) = descriptor.default_properties.get(name.as_str()) { // TODO: Float comparison - if value != default { + if variant_eq(default, value) { prop_list.push((name, value)) } } else { diff --git a/src/snapshot/diff/mod.rs b/src/snapshot/diff/mod.rs index 0a1e6ec1f..28142c825 100644 --- a/src/snapshot/diff/mod.rs +++ b/src/snapshot/diff/mod.rs @@ -1,5 +1,5 @@ mod hash; mod variant; -pub use hash::hash_tree; -pub use variant::hash_variant; +pub use hash::*; +pub use variant::*; diff --git a/src/snapshot/diff/variant.rs b/src/snapshot/diff/variant.rs index 8ee7f7189..ca54251ab 100644 --- a/src/snapshot/diff/variant.rs +++ b/src/snapshot/diff/variant.rs @@ -1,6 +1,8 @@ use blake3::Hasher; use rbx_dom_weak::types::{Variant, Vector3}; +use float_cmp::approx_eq; + macro_rules! round { ($value:expr) => { (($value * 10.0).round() / 10.0) @@ -79,3 +81,46 @@ pub fn hash_variant(hasher: &mut Hasher, value: &Variant) { fn vector_hash(hasher: &mut Hasher, vector: Vector3) { n_hash!(hasher, round!(vector.x), round!(vector.y), round!(vector.z)) } + +/// Compares to variants to determine if they're equal. This correctly takes +/// float comparisons into account. +pub fn variant_eq(variant_a: &Variant, variant_b: &Variant) -> bool { + if variant_a.ty() != variant_b.ty() { + return false; + } + + match (variant_a, variant_b) { + (Variant::Float32(a), Variant::Float32(b)) => { + approx_eq!(f32, *a, *b) + } + (Variant::Float64(a), Variant::Float64(b)) => { + approx_eq!(f64, *a, *b) + } + (Variant::Vector3(a), Variant::Vector3(b)) => vector_eq(a, b), + (Variant::CFrame(a), Variant::CFrame(b)) => { + vector_eq(&a.position, &b.position) + & vector_eq(&a.orientation.x, &b.orientation.x) + & vector_eq(&a.orientation.y, &b.orientation.y) + & vector_eq(&a.orientation.z, &b.orientation.z) + } + + (Variant::String(a), Variant::String(b)) => a == b, + (Variant::BinaryString(a), Variant::BinaryString(b)) => a == b, + (Variant::Bool(a), Variant::Bool(b)) => a == b, + (Variant::Int32(a), Variant::Int32(b)) => a == b, + (Variant::Int64(a), Variant::Int64(b)) => a == b, + (Variant::Axes(a), Variant::Axes(b)) => a == b, + (Variant::Faces(a), Variant::Faces(b)) => a == b, + + (a, b) => panic!( + "unsupport variant comparison: {:?} and {:?}", + a.ty(), + b.ty() + ), + } +} + +#[inline(always)] +fn vector_eq(a: &Vector3, b: &Vector3) -> bool { + approx_eq!(f32, a.x, b.x) & approx_eq!(f32, a.y, b.y) & approx_eq!(f32, a.z, b.z) +} From f879de4f51cf1228e6a64322adc78c5b37d8eba9 Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 21 Nov 2023 13:05:00 -0800 Subject: [PATCH 007/366] Support equality checks for every variant type --- src/snapshot/diff/variant.rs | 191 +++++++++++++++++++++++++++++++---- 1 file changed, 172 insertions(+), 19 deletions(-) diff --git a/src/snapshot/diff/variant.rs b/src/snapshot/diff/variant.rs index ca54251ab..48ea28078 100644 --- a/src/snapshot/diff/variant.rs +++ b/src/snapshot/diff/variant.rs @@ -1,7 +1,8 @@ use blake3::Hasher; -use rbx_dom_weak::types::{Variant, Vector3}; - use float_cmp::approx_eq; +use rbx_dom_weak::types::{PhysicalProperties, Variant, Vector3}; + +use std::collections::HashMap; macro_rules! round { ($value:expr) => { @@ -90,28 +91,180 @@ pub fn variant_eq(variant_a: &Variant, variant_b: &Variant) -> bool { } match (variant_a, variant_b) { - (Variant::Float32(a), Variant::Float32(b)) => { - approx_eq!(f32, *a, *b) - } - (Variant::Float64(a), Variant::Float64(b)) => { - approx_eq!(f64, *a, *b) + (Variant::Attributes(a), Variant::Attributes(b)) => { + // If they're not the same size, we can just abort + if a.iter().count() != b.iter().count() { + return false; + } + // Using a duplicated map, we can determine if we have + // mismatched keys between A and B + let mut b_dupe = HashMap::with_capacity(b.iter().count()); + for (name, value) in b.iter() { + b_dupe.insert(name, value); + } + for (name, a_value) in a.iter() { + if let Some(b_value) = b.get(name.as_str()) { + if variant_eq(a_value, b_value) { + b_dupe.remove(name); + } else { + return false; + } + } else { + return false; + } + } + b_dupe.is_empty() } - (Variant::Vector3(a), Variant::Vector3(b)) => vector_eq(a, b), + (Variant::Axes(a), Variant::Axes(b)) => a == b, + (Variant::BinaryString(a), Variant::BinaryString(b)) => a == b, + (Variant::Bool(a), Variant::Bool(b)) => a == b, + (Variant::BrickColor(a), Variant::BrickColor(b)) => a == b, (Variant::CFrame(a), Variant::CFrame(b)) => { vector_eq(&a.position, &b.position) - & vector_eq(&a.orientation.x, &b.orientation.x) - & vector_eq(&a.orientation.y, &b.orientation.y) - & vector_eq(&a.orientation.z, &b.orientation.z) + && vector_eq(&a.orientation.x, &b.orientation.x) + && vector_eq(&a.orientation.y, &b.orientation.y) + && vector_eq(&a.orientation.z, &b.orientation.z) + } + (Variant::Color3(a), Variant::Color3(b)) => { + approx_eq!(f32, a.r, b.r) && approx_eq!(f32, a.b, b.b) && approx_eq!(f32, a.g, b.g) + } + (Variant::Color3uint8(a), Variant::Color3uint8(b)) => a == b, + (Variant::ColorSequence(a), Variant::ColorSequence(b)) => { + if a.keypoints.len() != b.keypoints.len() { + return false; + } + let mut a_keypoints = Vec::with_capacity(a.keypoints.len()); + let mut b_keypoints = Vec::with_capacity(b.keypoints.len()); + for keypoint in &a.keypoints { + a_keypoints.push(keypoint) + } + for keypoint in &b.keypoints { + b_keypoints.push(keypoint) + } + a_keypoints.sort_unstable_by(|k1, k2| k1.time.partial_cmp(&k2.time).unwrap()); + b_keypoints.sort_unstable_by(|k1, k2| k1.time.partial_cmp(&k2.time).unwrap()); + for (a_kp, b_kp) in a_keypoints.iter().zip(b_keypoints) { + if !(approx_eq!(f32, a_kp.time, b_kp.time) + && approx_eq!(f32, a_kp.color.r, b_kp.color.r) + && approx_eq!(f32, a_kp.color.g, b_kp.color.g) + && approx_eq!(f32, a_kp.color.b, b_kp.color.b)) + { + return false; + } + } + true + } + (Variant::Content(a), Variant::Content(b)) => a == b, + (Variant::Enum(a), Variant::Enum(b)) => a == b, + (Variant::Faces(a), Variant::Faces(b)) => a == b, + (Variant::Float32(a), Variant::Float32(b)) => approx_eq!(f32, *a, *b), + (Variant::Float64(a), Variant::Float64(b)) => approx_eq!(f64, *a, *b), + (Variant::Font(a), Variant::Font(b)) => { + a.weight == b.weight + && a.style == b.style + && a.family == b.family + && a.cached_face_id == b.cached_face_id } - - (Variant::String(a), Variant::String(b)) => a == b, - (Variant::BinaryString(a), Variant::BinaryString(b)) => a == b, - (Variant::Bool(a), Variant::Bool(b)) => a == b, (Variant::Int32(a), Variant::Int32(b)) => a == b, (Variant::Int64(a), Variant::Int64(b)) => a == b, - (Variant::Axes(a), Variant::Axes(b)) => a == b, - (Variant::Faces(a), Variant::Faces(b)) => a == b, - + (Variant::MaterialColors(a), Variant::MaterialColors(b)) => a.encode() == b.encode(), + (Variant::NumberRange(a), Variant::NumberRange(b)) => { + approx_eq!(f32, a.max, b.max) && approx_eq!(f32, a.min, a.max) + } + (Variant::NumberSequence(a), Variant::NumberSequence(b)) => { + if a.keypoints.len() != b.keypoints.len() { + return false; + } + let mut a_keypoints = Vec::with_capacity(a.keypoints.len()); + let mut b_keypoints = Vec::with_capacity(b.keypoints.len()); + for keypoint in &a.keypoints { + a_keypoints.push(keypoint) + } + for keypoint in &b.keypoints { + b_keypoints.push(keypoint) + } + a_keypoints.sort_unstable_by(|k1, k2| k1.time.partial_cmp(&k2.time).unwrap()); + b_keypoints.sort_unstable_by(|k1, k2| k1.time.partial_cmp(&k2.time).unwrap()); + for (a_kp, b_kp) in a_keypoints.iter().zip(b_keypoints) { + if !(approx_eq!(f32, a_kp.time, b_kp.time) + && approx_eq!(f32, a_kp.value, b_kp.value) + && approx_eq!(f32, a_kp.envelope, b_kp.envelope)) + { + return false; + } + } + true + } + (Variant::OptionalCFrame(a), Variant::OptionalCFrame(b)) => { + if let (Some(a2), Some(b2)) = (a, b) { + vector_eq(&a2.position, &b2.position) + && vector_eq(&a2.orientation.x, &b2.orientation.x) + && vector_eq(&a2.orientation.y, &b2.orientation.y) + && vector_eq(&a2.orientation.z, &b2.orientation.z) + } else { + false + } + } + (Variant::PhysicalProperties(a), Variant::PhysicalProperties(b)) => match (a, b) { + (PhysicalProperties::Default, PhysicalProperties::Default) => true, + (PhysicalProperties::Custom(a2), PhysicalProperties::Custom(b2)) => { + approx_eq!(f32, a2.density, b2.density) + && approx_eq!(f32, a2.elasticity, b2.elasticity) + && approx_eq!(f32, a2.friction, b2.friction) + && approx_eq!(f32, a2.elasticity_weight, b2.elasticity_weight) + && approx_eq!(f32, a2.friction_weight, b2.friction_weight) + } + (_, _) => false, + }, + (Variant::Ray(a), Variant::Ray(b)) => { + vector_eq(&a.direction, &b.direction) && vector_eq(&a.origin, &b.origin) + } + (Variant::Rect(a), Variant::Rect(b)) => { + approx_eq!(f32, a.max.x, b.max.x) + && approx_eq!(f32, a.max.y, b.max.y) + && approx_eq!(f32, a.min.x, b.min.x) + && approx_eq!(f32, a.min.y, b.min.y) + } + (Variant::Ref(a), Variant::Ref(b)) => a == b, + (Variant::Region3(a), Variant::Region3(b)) => { + vector_eq(&a.max, &b.max) && vector_eq(&a.min, &b.min) + } + (Variant::Region3int16(a), Variant::Region3int16(b)) => a == b, + (Variant::SecurityCapabilities(a), Variant::SecurityCapabilities(b)) => a == b, + (Variant::SharedString(a), Variant::SharedString(b)) => a == b, + (Variant::Tags(a), Variant::Tags(b)) => { + let mut a_sorted: Vec<&str> = a.iter().collect(); + let mut b_sorted: Vec<&str> = b.iter().collect(); + if a_sorted.len() == b_sorted.len() { + a_sorted.sort_unstable(); + b_sorted.sort_unstable(); + for (a_tag, b_tag) in a_sorted.into_iter().zip(b_sorted) { + if a_tag != b_tag { + return false; + } + } + true + } else { + false + } + } + (Variant::UDim(a), Variant::UDim(b)) => { + approx_eq!(f32, a.scale, b.scale) && a.offset == b.offset + } + (Variant::UDim2(a), Variant::UDim2(b)) => { + approx_eq!(f32, a.x.scale, b.x.scale) + && a.x.offset == b.x.offset + && approx_eq!(f32, a.y.scale, b.y.scale) + && a.y.offset == b.y.offset + } + (Variant::UniqueId(a), Variant::UniqueId(b)) => a == b, + (Variant::String(a), Variant::String(b)) => a == b, + (Variant::Vector2(a), Variant::Vector2(b)) => { + approx_eq!(f32, a.x, b.x) && approx_eq!(f32, a.y, b.y) + } + (Variant::Vector2int16(a), Variant::Vector2int16(b)) => a == b, + (Variant::Vector3(a), Variant::Vector3(b)) => vector_eq(a, b), + (Variant::Vector3int16(a), Variant::Vector3int16(b)) => a == b, (a, b) => panic!( "unsupport variant comparison: {:?} and {:?}", a.ty(), @@ -122,5 +275,5 @@ pub fn variant_eq(variant_a: &Variant, variant_b: &Variant) -> bool { #[inline(always)] fn vector_eq(a: &Vector3, b: &Vector3) -> bool { - approx_eq!(f32, a.x, b.x) & approx_eq!(f32, a.y, b.y) & approx_eq!(f32, a.z, b.z) + approx_eq!(f32, a.x, b.x) && approx_eq!(f32, a.y, b.y) && approx_eq!(f32, a.z, b.z) } From 77152f398bcc3a7b5932ed5040235cdc19a3c624 Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 21 Nov 2023 15:10:30 -0800 Subject: [PATCH 008/366] Don't hash DataModels --- src/snapshot/diff/hash.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/snapshot/diff/hash.rs b/src/snapshot/diff/hash.rs index 2176d3958..4ed2cb267 100644 --- a/src/snapshot/diff/hash.rs +++ b/src/snapshot/diff/hash.rs @@ -21,6 +21,10 @@ pub fn hash_tree(dom: &WeakDom) -> HashMap { // end while let Some(referent) = order.pop() { let inst = dom.get_by_ref(referent).unwrap(); + // We don't really care about the equality of a DataModel. + if inst.class == "DataModel" { + continue; + } let hash = hash_inst(&mut prop_list, &map, inst); map.insert(referent, hash); From cc06f271fb105bd3595c26f521efa7311df90ece Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 21 Nov 2023 15:13:56 -0800 Subject: [PATCH 009/366] Correctly only hash non-default properties --- src/snapshot/diff/hash.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/snapshot/diff/hash.rs b/src/snapshot/diff/hash.rs index 4ed2cb267..a7e9a3391 100644 --- a/src/snapshot/diff/hash.rs +++ b/src/snapshot/diff/hash.rs @@ -49,8 +49,7 @@ fn hash_inst<'map, 'inst>( for (name, value) in &inst.properties { if let Some(default) = descriptor.default_properties.get(name.as_str()) { - // TODO: Float comparison - if variant_eq(default, value) { + if !variant_eq(default, value) { prop_list.push((name, value)) } } else { From 75182ed316bab2c4151fcab7b3d5a0f159b2e711 Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 21 Nov 2023 15:14:37 -0800 Subject: [PATCH 010/366] Implement hashing for all Variant types --- src/snapshot/diff/variant.rs | 135 ++++++++++++++++++++++++++++++++--- 1 file changed, 124 insertions(+), 11 deletions(-) diff --git a/src/snapshot/diff/variant.rs b/src/snapshot/diff/variant.rs index 48ea28078..2d1b3827a 100644 --- a/src/snapshot/diff/variant.rs +++ b/src/snapshot/diff/variant.rs @@ -29,16 +29,17 @@ pub fn hash_variant(hasher: &mut Hasher, value: &Variant) { // We need to round floats, though I'm not sure to what degree we can // realistically do that. match value { - Variant::String(str) => hash!(hasher, str.as_bytes()), - Variant::Bool(bool) => hash!(hasher, &[*bool as u8]), - Variant::Float32(n) => n_hash!(hasher, round!(*n)), - Variant::Float64(n) => n_hash!(hasher, round!(n)), - Variant::Int32(n) => n_hash!(hasher, n), - Variant::Int64(n) => n_hash!(hasher, n), - Variant::BinaryString(bytes) => hash!(hasher, bytes.as_ref()), - Variant::Vector3(v3) => vector_hash(hasher, *v3), - Variant::Vector2(v2) => n_hash!(hasher, round!(v2.x), round!(v2.y)), + Variant::Attributes(attrs) => { + let mut sorted: Vec<(&String, &Variant)> = attrs.iter().collect(); + sorted.sort_unstable_by_key(|(name, _)| *name); + for (name, attribute) in sorted { + hasher.update(name.as_bytes()); + hash_variant(hasher, attribute); + } + } Variant::Axes(a) => hash!(hasher, &[a.bits()]), + Variant::BinaryString(bytes) => hash!(hasher, bytes.as_ref()), + Variant::Bool(bool) => hash!(hasher, &[*bool as u8]), Variant::BrickColor(color) => n_hash!(hasher, *color as u16), Variant::CFrame(cf) => { vector_hash(hasher, cf.position); @@ -72,10 +73,122 @@ pub fn hash_variant(hasher: &mut Hasher, value: &Variant) { } Variant::Enum(e) => n_hash!(hasher, e.to_u32()), Variant::Faces(f) => hash!(hasher, &[f.bits()]), + Variant::Float32(n) => n_hash!(hasher, round!(*n)), + Variant::Float64(n) => n_hash!(hasher, round!(n)), + Variant::Font(f) => { + n_hash!(hasher, f.weight as u16); + n_hash!(hasher, f.style as u8); + hash!(hasher, f.family.as_bytes()); + if let Some(cache) = &f.cached_face_id { + hash!(hasher, &[0x01]); + hash!(hasher, cache.as_bytes()); + } else { + hash!(hasher, &[0x00]); + } + } + Variant::Int32(n) => n_hash!(hasher, n), + Variant::Int64(n) => n_hash!(hasher, n), + Variant::MaterialColors(n) => hash!(hasher, n.encode().as_slice()), + Variant::NumberRange(nr) => n_hash!(hasher, round!(nr.max), round!(nr.min)), + Variant::NumberSequence(seq) => { + let mut new = Vec::with_capacity(seq.keypoints.len()); + for keypoint in &seq.keypoints { + new.push(keypoint); + } + new.sort_unstable_by(|a, b| round!(a.time).partial_cmp(&round!(b.time)).unwrap()); + for keypoint in new { + n_hash!( + hasher, + round!(keypoint.time), + round!(keypoint.value), + round!(keypoint.envelope) + ) + } + } + Variant::OptionalCFrame(maybe_cf) => { + if let Some(cf) = maybe_cf { + hash!(hasher, &[0x01]); + vector_hash(hasher, cf.position); + vector_hash(hasher, cf.orientation.x); + vector_hash(hasher, cf.orientation.y); + vector_hash(hasher, cf.orientation.z); + } else { + hash!(hasher, &[0x00]); + } + } + Variant::PhysicalProperties(properties) => match properties { + PhysicalProperties::Default => hash!(hasher, &[0x00]), + PhysicalProperties::Custom(custom) => { + hash!(hasher, &[0x00]); + n_hash!( + hasher, + round!(custom.density), + round!(custom.friction), + round!(custom.elasticity), + round!(custom.friction_weight), + round!(custom.elasticity_weight) + ) + } + }, + Variant::Ray(ray) => { + vector_hash(hasher, ray.origin); + vector_hash(hasher, ray.direction); + } + Variant::Rect(rect) => n_hash!( + hasher, + round!(rect.max.x), + round!(rect.max.y), + round!(rect.min.x), + round!(rect.min.y) + ), + Variant::Ref(referent) => hash!(hasher, referent.to_string().as_bytes()), + Variant::Region3(region) => { + vector_hash(hasher, region.max); + vector_hash(hasher, region.min); + } + Variant::Region3int16(region) => { + n_hash!( + hasher, + region.max.x, + region.max.y, + region.max.z, + region.min.x, + region.min.y, + region.min.z + ) + } + Variant::SecurityCapabilities(capabilities) => n_hash!(hasher, capabilities.bits()), + Variant::SharedString(sstr) => hash!(hasher, sstr.hash().as_bytes()), + Variant::String(str) => hash!(hasher, str.as_bytes()), + Variant::Tags(tags) => { + let mut dupe: Vec<&str> = tags.iter().collect(); + dupe.sort_unstable(); + for tag in dupe { + hash!(hasher, tag.as_bytes()) + } + } + Variant::UDim(udim) => n_hash!(hasher, round!(udim.scale), udim.offset), + Variant::UDim2(udim) => n_hash!( + hasher, + round!(udim.y.scale), + udim.y.offset, + round!(udim.x.scale), + udim.x.offset + ), + Variant::Vector2(v2) => n_hash!(hasher, round!(v2.x), round!(v2.y)), + Variant::Vector2int16(v2) => n_hash!(hasher, v2.x, v2.y), + Variant::Vector3(v3) => vector_hash(hasher, *v3), + Variant::Vector3int16(v3) => n_hash!(hasher, v3.x, v3.y, v3.z), - // TODO: Add the rest // Hashing UniqueId properties doesn't make sense - Variant::UniqueId(_) | _ => (), + Variant::UniqueId(_) => (), + + unknown => { + log::warn!( + "Encountered unknown Variant {:?} while hashing", + unknown.ty() + ) + } } } From d5635c2dc85c6e89726a6aa081103a74db6f53a7 Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 21 Nov 2023 15:16:31 -0800 Subject: [PATCH 011/366] Make `hash_inst` public --- src/snapshot/diff/hash.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/snapshot/diff/hash.rs b/src/snapshot/diff/hash.rs index a7e9a3391..c91769331 100644 --- a/src/snapshot/diff/hash.rs +++ b/src/snapshot/diff/hash.rs @@ -33,7 +33,7 @@ pub fn hash_tree(dom: &WeakDom) -> HashMap { map } -fn hash_inst<'map, 'inst>( +pub fn hash_inst<'map, 'inst>( prop_list: &mut Vec<(&'inst str, &'inst Variant)>, map: &'map HashMap, inst: &'inst Instance, From 77294d26db77e199f047f173e2458325a0d1eed2 Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 21 Nov 2023 16:01:37 -0800 Subject: [PATCH 012/366] Abstract tree diffing to its own function --- src/snapshot/diff/mod.rs | 44 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/snapshot/diff/mod.rs b/src/snapshot/diff/mod.rs index 28142c825..1b049b4cd 100644 --- a/src/snapshot/diff/mod.rs +++ b/src/snapshot/diff/mod.rs @@ -3,3 +3,47 @@ mod variant; pub use hash::*; pub use variant::*; + +use rbx_dom_weak::{types::Ref, WeakDom}; +use std::{collections::HashMap, hash::Hash}; + +pub fn diff_trees(dom_1: &WeakDom, dom_2: &WeakDom) -> Diff { + let list_1 = hash_tree(dom_1); + let mut list_2 = invert_map(hash_tree(dom_2)); + + let mut removals = Vec::new(); + + for (referent, hash) in list_1 { + // If it's in both lists, we'll pull it out of list_2 so that + // it doesn't get flagged as an addition. + if list_2.contains_key(&hash) { + list_2.remove(&hash); + } else { + removals.push(referent); + } + } + + Diff { + removals, + additions: list_2.into_iter().map(|(_, referent)| referent).collect(), + } +} + +fn invert_map(map: HashMap) -> HashMap { + map.into_iter().map(|(key, value)| (value, key)).collect() +} + +pub struct Diff { + /// Referents that were either removed or changed (in dom 1, not in dom 2) + pub removals: Vec, + /// Referents that were added or changed (in dom 2, not in dom 1) + pub additions: Vec, +} + +impl Diff { + /// Returns the total number of diffs represented by this struct + #[inline] + pub fn total(&self) -> usize { + self.removals.len() + self.additions.len() + } +} From d052d20841175c5b4bfe9fceb097c2c2c6be831d Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 28 Nov 2023 11:42:11 -0800 Subject: [PATCH 013/366] Track middleware in InstanceMetadata --- src/snapshot/metadata.rs | 3 +++ src/snapshot_middleware/mod.rs | 30 +++++++++++++++++------------- src/web/ui.rs | 1 + 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/src/snapshot/metadata.rs b/src/snapshot/metadata.rs index 1578af230..6df36e082 100644 --- a/src/snapshot/metadata.rs +++ b/src/snapshot/metadata.rs @@ -58,6 +58,8 @@ pub struct InstanceMetadata { /// that instance's instigating source is snapshotted directly, the same /// context will be passed into it. pub context: InstanceContext, + + pub middleware: Option, } impl InstanceMetadata { @@ -67,6 +69,7 @@ impl InstanceMetadata { instigating_source: None, relevant_paths: Vec::new(), context: InstanceContext::default(), + middleware: None, } } diff --git a/src/snapshot_middleware/mod.rs b/src/snapshot_middleware/mod.rs index 30ea57061..dd52fc2e4 100644 --- a/src/snapshot_middleware/mod.rs +++ b/src/snapshot_middleware/mod.rs @@ -161,23 +161,27 @@ fn snapshot_from_path( vfs: &Vfs, path: &Path, ) -> anyhow::Result> { - if let Some(rule) = context.get_user_sync_rule(path) { - return rule - .middleware - .snapshot(context, vfs, path, rule.file_name_for_path(path)?); + let mut rule = None; + if let Some(user_rule) = context.get_user_sync_rule(path) { + rule = Some(user_rule); } else { - for rule in default_sync_rules() { - if rule.matches(path) { - return rule.middleware.snapshot( - context, - vfs, - path, - rule.file_name_for_path(path)?, - ); + for default_rule in default_sync_rules() { + if default_rule.matches(path) { + rule = Some(default_rule); } } } - Ok(None) + if let Some(rule) = rule { + Ok(rule + .middleware + .snapshot(context, vfs, path, rule.file_name_for_path(path)?)? + .and_then(|mut snapshot| { + snapshot.metadata.middleware = Some(rule.middleware); + Some(snapshot) + })) + } else { + Ok(None) + } } /// Represents a possible 'transformer' used by Rojo to turn a file system diff --git a/src/web/ui.rs b/src/web/ui.rs index 0199d58d7..6f9d0e077 100644 --- a/src/web/ui.rs +++ b/src/web/ui.rs @@ -165,6 +165,7 @@ impl UiService { <>
"ignore_unknown_instances: " { metadata.ignore_unknown_instances.to_string() }
"instigating source: " { format!("{:?}", metadata.instigating_source) }
+
"middleware: " { format!("{:?}", metadata.middleware) }
{ relevant_paths } }; From 41e95e624ad1b61946790c519760439ac2dc2500 Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 28 Nov 2023 12:02:13 -0800 Subject: [PATCH 014/366] Only add actually relevant paths for dirs --- src/snapshot_middleware/csv.rs | 4 ++++ src/snapshot_middleware/dir.rs | 15 +-------------- src/snapshot_middleware/lua.rs | 4 ++++ 3 files changed, 9 insertions(+), 14 deletions(-) diff --git a/src/snapshot_middleware/csv.rs b/src/snapshot_middleware/csv.rs index 52e8ab0fd..676dc853c 100644 --- a/src/snapshot_middleware/csv.rs +++ b/src/snapshot_middleware/csv.rs @@ -76,6 +76,10 @@ pub fn snapshot_csv_init( init_snapshot.children = dir_snapshot.children; init_snapshot.metadata = dir_snapshot.metadata; + init_snapshot + .metadata + .relevant_paths + .push(init_path.to_owned()); if let Some(mut meta) = dir_meta(vfs, folder_path)? { meta.apply_all(&mut init_snapshot)?; diff --git a/src/snapshot_middleware/dir.rs b/src/snapshot_middleware/dir.rs index 092641e7f..ac3e79efc 100644 --- a/src/snapshot_middleware/dir.rs +++ b/src/snapshot_middleware/dir.rs @@ -75,20 +75,7 @@ pub fn snapshot_dir_no_meta( let meta_path = path.join("init.meta.json"); - let relevant_paths = vec![ - path.to_path_buf(), - meta_path, - // TODO: We shouldn't need to know about Lua existing in this - // middleware. Should we figure out a way for that function to add - // relevant paths to this middleware? - path.join("init.lua"), - path.join("init.luau"), - path.join("init.server.lua"), - path.join("init.server.luau"), - path.join("init.client.lua"), - path.join("init.client.luau"), - path.join("init.csv"), - ]; + let relevant_paths = vec![path.to_path_buf(), meta_path]; let snapshot = InstanceSnapshot::new() .name(instance_name) diff --git a/src/snapshot_middleware/lua.rs b/src/snapshot_middleware/lua.rs index ab7f15a1f..6821dd84b 100644 --- a/src/snapshot_middleware/lua.rs +++ b/src/snapshot_middleware/lua.rs @@ -107,6 +107,10 @@ pub fn snapshot_lua_init( init_snapshot.children = dir_snapshot.children; init_snapshot.metadata = dir_snapshot.metadata; + init_snapshot + .metadata + .relevant_paths + .push(init_path.to_owned()); if let Some(mut meta) = dir_meta(vfs, folder_path)? { meta.apply_all(&mut init_snapshot)?; From 03644632e03c26472fc6ff109d528940e6e04170 Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 28 Nov 2023 12:46:14 -0800 Subject: [PATCH 015/366] Move descendants function to diff module --- src/snapshot/diff/hash.rs | 22 ++-------------------- src/snapshot/diff/mod.rs | 23 ++++++++++++++++++++++- 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/src/snapshot/diff/hash.rs b/src/snapshot/diff/hash.rs index c91769331..c05db9a75 100644 --- a/src/snapshot/diff/hash.rs +++ b/src/snapshot/diff/hash.rs @@ -6,9 +6,9 @@ use rbx_dom_weak::{ Instance, WeakDom, }; -use std::collections::{HashMap, VecDeque}; +use std::collections::HashMap; -use super::{hash_variant, variant_eq}; +use super::{descendants, hash_variant, variant_eq}; pub fn hash_tree(dom: &WeakDom) -> HashMap { let mut map: HashMap = HashMap::new(); @@ -80,21 +80,3 @@ pub fn hash_inst<'map, 'inst>( hasher.finalize() } - -fn descendants(dom: &WeakDom) -> Vec { - let mut queue = VecDeque::new(); - let mut ordered = Vec::new(); - queue.push_front(dom.root_ref()); - - while let Some(referent) = queue.pop_front() { - let inst = dom - .get_by_ref(referent) - .expect("Invariant: WeakDom had a Ref that wasn't inside it"); - ordered.push(referent); - for child in inst.children() { - queue.push_back(*child) - } - } - - ordered -} diff --git a/src/snapshot/diff/mod.rs b/src/snapshot/diff/mod.rs index 1b049b4cd..14651ae8f 100644 --- a/src/snapshot/diff/mod.rs +++ b/src/snapshot/diff/mod.rs @@ -5,7 +5,10 @@ pub use hash::*; pub use variant::*; use rbx_dom_weak::{types::Ref, WeakDom}; -use std::{collections::HashMap, hash::Hash}; +use std::{ + collections::{HashMap, VecDeque}, + hash::Hash, +}; pub fn diff_trees(dom_1: &WeakDom, dom_2: &WeakDom) -> Diff { let list_1 = hash_tree(dom_1); @@ -47,3 +50,21 @@ impl Diff { self.removals.len() + self.additions.len() } } + +pub(crate) fn descendants(dom: &WeakDom) -> Vec { + let mut queue = VecDeque::new(); + let mut ordered = Vec::new(); + queue.push_front(dom.root_ref()); + + while let Some(referent) = queue.pop_front() { + let inst = dom + .get_by_ref(referent) + .expect("Invariant: WeakDom had a Ref that wasn't inside it"); + ordered.push(referent); + for child in inst.children() { + queue.push_back(*child) + } + } + + ordered +} From 8e72b42d81110c2db371e284a3acbbb194b7a21f Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 5 Dec 2023 10:58:05 -0800 Subject: [PATCH 016/366] Use middleware for directories too! --- ..._snapshot__tests__apply__add_property.snap | 1 + ...s__apply__remove_property_after_patch.snap | 1 + ...tests__apply__remove_property_initial.snap | 1 + ...tests__apply__set_name_and_class_name.snap | 1 + ...__snapshot__tests__compute__add_child.snap | 1 + src/snapshot_middleware/csv.rs | 3 +- src/snapshot_middleware/dir.rs | 37 ++-- src/snapshot_middleware/lua.rs | 3 +- src/snapshot_middleware/mod.rs | 166 ++++++++---------- ...t_middleware__csv__test__csv_from_vfs.snap | 1 + ..._middleware__csv__test__csv_with_meta.snap | 1 + ...t_middleware__dir__test__empty_folder.snap | 8 +- ...ddleware__dir__test__folder_in_folder.snap | 16 +- ...leware__json__test__instance_from_vfs.snap | 1 + ...are__json_model__test__model_from_vfs.snap | 2 + ...on_model__test__model_from_vfs_legacy.snap | 2 + ...are__lua__test__class_client_from_vfs.snap | 1 + ...are__lua__test__class_module_from_vfs.snap | 1 + ...re__lua__test__class_module_with_meta.snap | 1 + ...are__lua__test__class_script_disabled.snap | 1 + ...re__lua__test__class_script_with_meta.snap | 1 + ...are__lua__test__class_server_from_vfs.snap | 1 + ...lua__test__runcontext_client_from_vfs.snap | 1 + ...lua__test__runcontext_module_from_vfs.snap | 1 + ...ua__test__runcontext_module_with_meta.snap | 1 + ...lua__test__runcontext_script_disabled.snap | 1 + ...ua__test__runcontext_script_with_meta.snap | 1 + ...lua__test__runcontext_server_from_vfs.snap | 1 + ...oject__test__project_from_direct_file.snap | 1 + ...test__project_path_property_overrides.snap | 1 + ..._project__test__project_with_children.snap | 2 + ...t__test__project_with_path_to_project.snap | 1 + ...ct_with_path_to_project_with_children.snap | 2 + ...oject__test__project_with_path_to_txt.snap | 1 + ...est__project_with_resolved_properties.snap | 1 + ...t__project_with_unresolved_properties.snap | 1 + ...leware__toml__test__instance_from_vfs.snap | 1 + ...dleware__txt__test__instance_from_vfs.snap | 1 + 38 files changed, 139 insertions(+), 130 deletions(-) diff --git a/src/snapshot/tests/snapshots/librojo__snapshot__tests__apply__add_property.snap b/src/snapshot/tests/snapshots/librojo__snapshot__tests__apply__add_property.snap index 0d594dbc0..3c751fdf4 100644 --- a/src/snapshot/tests/snapshots/librojo__snapshot__tests__apply__add_property.snap +++ b/src/snapshot/tests/snapshots/librojo__snapshot__tests__apply__add_property.snap @@ -13,5 +13,6 @@ metadata: relevant_paths: [] context: emit_legacy_scripts: true + middleware: ~ children: [] diff --git a/src/snapshot/tests/snapshots/librojo__snapshot__tests__apply__remove_property_after_patch.snap b/src/snapshot/tests/snapshots/librojo__snapshot__tests__apply__remove_property_after_patch.snap index b6fcbbd08..c4af516ea 100644 --- a/src/snapshot/tests/snapshots/librojo__snapshot__tests__apply__remove_property_after_patch.snap +++ b/src/snapshot/tests/snapshots/librojo__snapshot__tests__apply__remove_property_after_patch.snap @@ -11,5 +11,6 @@ metadata: relevant_paths: [] context: emit_legacy_scripts: true + middleware: ~ children: [] diff --git a/src/snapshot/tests/snapshots/librojo__snapshot__tests__apply__remove_property_initial.snap b/src/snapshot/tests/snapshots/librojo__snapshot__tests__apply__remove_property_initial.snap index 84b3c3d1e..8ed43ccd9 100644 --- a/src/snapshot/tests/snapshots/librojo__snapshot__tests__apply__remove_property_initial.snap +++ b/src/snapshot/tests/snapshots/librojo__snapshot__tests__apply__remove_property_initial.snap @@ -13,5 +13,6 @@ metadata: relevant_paths: [] context: emit_legacy_scripts: true + middleware: ~ children: [] diff --git a/src/snapshot/tests/snapshots/librojo__snapshot__tests__apply__set_name_and_class_name.snap b/src/snapshot/tests/snapshots/librojo__snapshot__tests__apply__set_name_and_class_name.snap index 325c86f3c..5772090e1 100644 --- a/src/snapshot/tests/snapshots/librojo__snapshot__tests__apply__set_name_and_class_name.snap +++ b/src/snapshot/tests/snapshots/librojo__snapshot__tests__apply__set_name_and_class_name.snap @@ -11,5 +11,6 @@ metadata: relevant_paths: [] context: emit_legacy_scripts: true + middleware: ~ children: [] diff --git a/src/snapshot/tests/snapshots/librojo__snapshot__tests__compute__add_child.snap b/src/snapshot/tests/snapshots/librojo__snapshot__tests__compute__add_child.snap index 94ddf12fb..01ae237fa 100644 --- a/src/snapshot/tests/snapshots/librojo__snapshot__tests__compute__add_child.snap +++ b/src/snapshot/tests/snapshots/librojo__snapshot__tests__compute__add_child.snap @@ -12,6 +12,7 @@ added_instances: relevant_paths: [] context: emit_legacy_scripts: true + middleware: ~ name: New class_name: Folder properties: {} diff --git a/src/snapshot_middleware/csv.rs b/src/snapshot_middleware/csv.rs index 676dc853c..594b5d53b 100644 --- a/src/snapshot_middleware/csv.rs +++ b/src/snapshot_middleware/csv.rs @@ -57,9 +57,10 @@ pub fn snapshot_csv_init( context: &InstanceContext, vfs: &Vfs, init_path: &Path, + name: &str, ) -> anyhow::Result> { let folder_path = init_path.parent().unwrap(); - let dir_snapshot = snapshot_dir_no_meta(context, vfs, folder_path)?.unwrap(); + let dir_snapshot = snapshot_dir_no_meta(context, vfs, folder_path, name)?.unwrap(); if dir_snapshot.class_name != "Folder" { anyhow::bail!( diff --git a/src/snapshot_middleware/dir.rs b/src/snapshot_middleware/dir.rs index ac3e79efc..eea248a52 100644 --- a/src/snapshot_middleware/dir.rs +++ b/src/snapshot_middleware/dir.rs @@ -10,8 +10,9 @@ pub fn snapshot_dir( context: &InstanceContext, vfs: &Vfs, path: &Path, + name: &str, ) -> anyhow::Result> { - let mut snapshot = match snapshot_dir_no_meta(context, vfs, path)? { + let mut snapshot = match snapshot_dir_no_meta(context, vfs, path, name)? { Some(snapshot) => snapshot, None => return Ok(None), }; @@ -44,6 +45,7 @@ pub fn snapshot_dir_no_meta( context: &InstanceContext, vfs: &Vfs, path: &Path, + name: &str, ) -> anyhow::Result> { let passes_filter_rules = |child: &DirEntry| { context @@ -66,19 +68,12 @@ pub fn snapshot_dir_no_meta( } } - let instance_name = path - .file_name() - .expect("Could not extract file name") - .to_str() - .ok_or_else(|| anyhow::anyhow!("File name was not valid UTF-8: {}", path.display()))? - .to_string(); - let meta_path = path.join("init.meta.json"); let relevant_paths = vec![path.to_path_buf(), meta_path]; let snapshot = InstanceSnapshot::new() - .name(instance_name) + .name(name) .class_name("Folder") .children(snapshot_children) .metadata( @@ -106,10 +101,14 @@ mod test { let mut vfs = Vfs::new(imfs); - let instance_snapshot = - snapshot_dir(&InstanceContext::default(), &mut vfs, Path::new("/foo")) - .unwrap() - .unwrap(); + let instance_snapshot = snapshot_dir( + &InstanceContext::default(), + &mut vfs, + Path::new("/foo"), + "foo", + ) + .unwrap() + .unwrap(); insta::assert_yaml_snapshot!(instance_snapshot); } @@ -127,10 +126,14 @@ mod test { let mut vfs = Vfs::new(imfs); - let instance_snapshot = - snapshot_dir(&InstanceContext::default(), &mut vfs, Path::new("/foo")) - .unwrap() - .unwrap(); + let instance_snapshot = snapshot_dir( + &InstanceContext::default(), + &mut vfs, + Path::new("/foo"), + "foo", + ) + .unwrap() + .unwrap(); insta::assert_yaml_snapshot!(instance_snapshot); } diff --git a/src/snapshot_middleware/lua.rs b/src/snapshot_middleware/lua.rs index 6821dd84b..650a71eb7 100644 --- a/src/snapshot_middleware/lua.rs +++ b/src/snapshot_middleware/lua.rs @@ -85,10 +85,11 @@ pub fn snapshot_lua_init( context: &InstanceContext, vfs: &Vfs, init_path: &Path, + name: &str, script_type: ScriptType, ) -> anyhow::Result> { let folder_path = init_path.parent().unwrap(); - let dir_snapshot = snapshot_dir_no_meta(context, vfs, folder_path)?.unwrap(); + let dir_snapshot = snapshot_dir_no_meta(context, vfs, folder_path, name)?.unwrap(); if dir_snapshot.class_name != "Folder" { anyhow::bail!( diff --git a/src/snapshot_middleware/mod.rs b/src/snapshot_middleware/mod.rs index dd52fc2e4..5be18eb36 100644 --- a/src/snapshot_middleware/mod.rs +++ b/src/snapshot_middleware/mod.rs @@ -61,33 +61,9 @@ pub fn snapshot_from_vfs( }; if meta.is_dir() { - if let Some(init_path) = get_init_path(vfs, path)? { - // TODO: support user-defined init paths - for rule in default_sync_rules() { - if rule.matches(&init_path) { - return match rule.middleware { - Middleware::Project => snapshot_project(context, vfs, &init_path), - - Middleware::ModuleScript => { - snapshot_lua_init(context, vfs, &init_path, ScriptType::Module) - } - Middleware::ServerScript => { - snapshot_lua_init(context, vfs, &init_path, ScriptType::Server) - } - Middleware::ClientScript => { - snapshot_lua_init(context, vfs, &init_path, ScriptType::Client) - } - - Middleware::Csv => snapshot_csv_init(context, vfs, &init_path), - - _ => snapshot_dir(context, vfs, path), - }; - } - } - snapshot_dir(context, vfs, path) - } else { - snapshot_dir(context, vfs, path) - } + let (middleware, name) = get_dir_middleware(vfs, path)?; + // TODO: Support user defined init paths + middleware.snapshot(context, vfs, path, &name) } else { let file_name = path .file_name() @@ -105,53 +81,39 @@ pub fn snapshot_from_vfs( } } -/// Gets an `init` path for the given directory. -/// This uses an intrinsic priority list and for compatibility, -/// it should not be changed. -fn get_init_path>(vfs: &Vfs, dir: P) -> anyhow::Result> { +/// Gets the appropriate middleware for a directory by checking for `init` +/// files. This uses an intrinsic priority list and for compatibility, +/// that order should be left unchanged. +fn get_dir_middleware>(vfs: &Vfs, dir: P) -> anyhow::Result<(Middleware, String)> { let path = dir.as_ref(); + let dir_name = path + .file_name() + .expect("Could not extract directory name") + .to_str() + .ok_or_else(|| anyhow::anyhow!("File name was not valid UTF-8: {}", path.display()))? + .to_string(); - let project_path = path.join("default.project.json"); - if vfs.metadata(&project_path).with_not_found()?.is_some() { - return Ok(Some(project_path)); - } - - let init_path = path.join("init.luau"); - if vfs.metadata(&init_path).with_not_found()?.is_some() { - return Ok(Some(init_path)); - } - - let init_path = path.join("init.lua"); - if vfs.metadata(&init_path).with_not_found()?.is_some() { - return Ok(Some(init_path)); - } - - let init_path = path.join("init.server.luau"); - if vfs.metadata(&init_path).with_not_found()?.is_some() { - return Ok(Some(init_path)); - } - - let init_path = path.join("init.server.lua"); - if vfs.metadata(&init_path).with_not_found()?.is_some() { - return Ok(Some(init_path)); - } - - let init_path = path.join("init.client.luau"); - if vfs.metadata(&init_path).with_not_found()?.is_some() { - return Ok(Some(init_path)); - } - - let init_path = path.join("init.client.lua"); - if vfs.metadata(&init_path).with_not_found()?.is_some() { - return Ok(Some(init_path)); - } + static INIT_PATHS: OnceLock> = OnceLock::new(); + let order = INIT_PATHS.get_or_init(|| { + vec![ + (Middleware::Project, "default.project.json"), + (Middleware::ModuleScriptDir, "init.luau"), + (Middleware::ModuleScriptDir, "init.lua"), + (Middleware::ServerScriptDir, "init.server.luau"), + (Middleware::ServerScriptDir, "init.server.lua"), + (Middleware::ClientScriptDir, "init.client.luau"), + (Middleware::ClientScriptDir, "init.client.lua"), + (Middleware::CsvDir, "init.csv"), + ] + }); - let init_path = path.join("init.csv"); - if vfs.metadata(&init_path).with_not_found()?.is_some() { - return Ok(Some(init_path)); + for (middleware, name) in order { + if vfs.metadata(path.join(name)).with_not_found()?.is_some() { + return Ok((*middleware, dir_name)); + } } - Ok(None) + Ok((Middleware::Dir, dir_name)) } /// Gets a snapshot for a path given an InstanceContext and Vfs, taking @@ -161,33 +123,30 @@ fn snapshot_from_path( vfs: &Vfs, path: &Path, ) -> anyhow::Result> { - let mut rule = None; - if let Some(user_rule) = context.get_user_sync_rule(path) { - rule = Some(user_rule); + if let Some(rule) = context.get_user_sync_rule(path) { + return rule + .middleware + .snapshot(context, vfs, path, rule.file_name_for_path(path)?); } else { - for default_rule in default_sync_rules() { - if default_rule.matches(path) { - rule = Some(default_rule); + for rule in default_sync_rules() { + if rule.matches(path) { + return rule.middleware.snapshot( + context, + vfs, + path, + rule.file_name_for_path(path)?, + ); } } } - if let Some(rule) = rule { - Ok(rule - .middleware - .snapshot(context, vfs, path, rule.file_name_for_path(path)?)? - .and_then(|mut snapshot| { - snapshot.metadata.middleware = Some(rule.middleware); - Some(snapshot) - })) - } else { - Ok(None) - } + Ok(None) } /// Represents a possible 'transformer' used by Rojo to turn a file system -/// item into a Roblox Instance. Missing from this list are directories and -/// metadata. This is deliberate, as metadata is not a snapshot middleware -/// and directories do not make sense to turn into files. +/// item into a Roblox Instance. Missing from this list is metadata. +/// This is deliberate, as metadata is not a snapshot middleware. +/// +/// Directories cannot be used for sync rules so they're ignored by Serde. #[derive(Debug, Clone, Copy, PartialEq, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] pub enum Middleware { @@ -203,6 +162,17 @@ pub enum Middleware { Toml, Text, Ignore, + + #[serde(skip_deserializing)] + Dir, + #[serde(skip_deserializing)] + ServerScriptDir, + #[serde(skip_deserializing)] + ClientScriptDir, + #[serde(skip_deserializing)] + ModuleScriptDir, + #[serde(skip_deserializing)] + CsvDir, } impl Middleware { @@ -215,7 +185,7 @@ impl Middleware { path: &Path, name: &str, ) -> anyhow::Result> { - match self { + let mut output = match self { Self::Csv => snapshot_csv(context, vfs, path, name), Self::JsonModel => snapshot_json_model(context, vfs, path, name), Self::Json => snapshot_json(context, vfs, path, name), @@ -230,7 +200,23 @@ impl Middleware { Self::Toml => snapshot_toml(context, vfs, path, name), Self::Text => snapshot_txt(context, vfs, path, name), Self::Ignore => Ok(None), + + Self::Dir => snapshot_dir(context, vfs, path, name), + Self::ServerScriptDir => { + snapshot_lua_init(context, vfs, path, name, ScriptType::Server) + } + Self::ClientScriptDir => { + snapshot_lua_init(context, vfs, path, name, ScriptType::Client) + } + Self::ModuleScriptDir => { + snapshot_lua_init(context, vfs, path, name, ScriptType::Module) + } + Self::CsvDir => snapshot_csv_init(context, vfs, path, name), + }; + if let Ok(Some(ref mut snapshot)) = output { + snapshot.metadata.middleware = Some(*self); } + output } } diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__csv__test__csv_from_vfs.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__csv__test__csv_from_vfs.snap index 50ce53c11..18b12d24b 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__csv__test__csv_from_vfs.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__csv__test__csv_from_vfs.snap @@ -12,6 +12,7 @@ metadata: - /foo.meta.json context: emit_legacy_scripts: true + middleware: ~ name: foo class_name: LocalizationTable properties: diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__csv__test__csv_with_meta.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__csv__test__csv_with_meta.snap index c91924f8c..1bddf74c1 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__csv__test__csv_with_meta.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__csv__test__csv_with_meta.snap @@ -12,6 +12,7 @@ metadata: - /foo.meta.json context: emit_legacy_scripts: true + middleware: ~ name: foo class_name: LocalizationTable properties: diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__dir__test__empty_folder.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__dir__test__empty_folder.snap index 4c0dcbbf4..823106566 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__dir__test__empty_folder.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__dir__test__empty_folder.snap @@ -10,15 +10,9 @@ metadata: relevant_paths: - /foo - /foo/init.meta.json - - /foo/init.lua - - /foo/init.luau - - /foo/init.server.lua - - /foo/init.server.luau - - /foo/init.client.lua - - /foo/init.client.luau - - /foo/init.csv context: emit_legacy_scripts: true + middleware: ~ name: foo class_name: Folder properties: {} diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__dir__test__folder_in_folder.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__dir__test__folder_in_folder.snap index 6238fe68f..936ec5176 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__dir__test__folder_in_folder.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__dir__test__folder_in_folder.snap @@ -10,15 +10,9 @@ metadata: relevant_paths: - /foo - /foo/init.meta.json - - /foo/init.lua - - /foo/init.luau - - /foo/init.server.lua - - /foo/init.server.luau - - /foo/init.client.lua - - /foo/init.client.luau - - /foo/init.csv context: emit_legacy_scripts: true + middleware: ~ name: foo class_name: Folder properties: {} @@ -31,15 +25,9 @@ children: relevant_paths: - /foo/Child - /foo/Child/init.meta.json - - /foo/Child/init.lua - - /foo/Child/init.luau - - /foo/Child/init.server.lua - - /foo/Child/init.server.luau - - /foo/Child/init.client.lua - - /foo/Child/init.client.luau - - /foo/Child/init.csv context: emit_legacy_scripts: true + middleware: dir name: Child class_name: Folder properties: {} diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__json__test__instance_from_vfs.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__json__test__instance_from_vfs.snap index 9cc320ebb..e1b3b35a0 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__json__test__instance_from_vfs.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__json__test__instance_from_vfs.snap @@ -12,6 +12,7 @@ metadata: - /foo.meta.json context: emit_legacy_scripts: true + middleware: ~ name: foo class_name: ModuleScript properties: diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__json_model__test__model_from_vfs.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__json_model__test__model_from_vfs.snap index e3ee581e0..f573e51cb 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__json_model__test__model_from_vfs.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__json_model__test__model_from_vfs.snap @@ -11,6 +11,7 @@ metadata: - /foo.model.json context: emit_legacy_scripts: true + middleware: ~ name: foo class_name: IntValue properties: @@ -23,6 +24,7 @@ children: relevant_paths: [] context: emit_legacy_scripts: true + middleware: ~ name: The Child class_name: StringValue properties: {} diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__json_model__test__model_from_vfs_legacy.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__json_model__test__model_from_vfs_legacy.snap index e3ee581e0..f573e51cb 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__json_model__test__model_from_vfs_legacy.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__json_model__test__model_from_vfs_legacy.snap @@ -11,6 +11,7 @@ metadata: - /foo.model.json context: emit_legacy_scripts: true + middleware: ~ name: foo class_name: IntValue properties: @@ -23,6 +24,7 @@ children: relevant_paths: [] context: emit_legacy_scripts: true + middleware: ~ name: The Child class_name: StringValue properties: {} diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_client_from_vfs.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_client_from_vfs.snap index 5f5f6e9a1..1484bc63c 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_client_from_vfs.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_client_from_vfs.snap @@ -12,6 +12,7 @@ metadata: - /foo.meta.json context: emit_legacy_scripts: true + middleware: ~ name: foo class_name: LocalScript properties: diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_module_from_vfs.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_module_from_vfs.snap index de2471b9a..4e62485b9 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_module_from_vfs.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_module_from_vfs.snap @@ -12,6 +12,7 @@ metadata: - /foo.meta.json context: emit_legacy_scripts: true + middleware: ~ name: foo class_name: ModuleScript properties: diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_module_with_meta.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_module_with_meta.snap index 32d70c708..f625bcece 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_module_with_meta.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_module_with_meta.snap @@ -12,6 +12,7 @@ metadata: - /foo.meta.json context: emit_legacy_scripts: true + middleware: ~ name: foo class_name: ModuleScript properties: diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_script_disabled.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_script_disabled.snap index 2c5546a53..6b8c105bb 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_script_disabled.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_script_disabled.snap @@ -12,6 +12,7 @@ metadata: - /bar.meta.json context: emit_legacy_scripts: true + middleware: ~ name: bar class_name: Script properties: diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_script_with_meta.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_script_with_meta.snap index e4e31ff96..3871314b9 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_script_with_meta.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_script_with_meta.snap @@ -12,6 +12,7 @@ metadata: - /foo.meta.json context: emit_legacy_scripts: true + middleware: ~ name: foo class_name: Script properties: diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_server_from_vfs.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_server_from_vfs.snap index b70b9cc2b..dc8377f27 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_server_from_vfs.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_server_from_vfs.snap @@ -12,6 +12,7 @@ metadata: - /foo.meta.json context: emit_legacy_scripts: true + middleware: ~ name: foo class_name: Script properties: diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_client_from_vfs.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_client_from_vfs.snap index 6ad72d084..c167a8209 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_client_from_vfs.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_client_from_vfs.snap @@ -12,6 +12,7 @@ metadata: - /foo.meta.json context: emit_legacy_scripts: false + middleware: ~ name: foo class_name: Script properties: diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_module_from_vfs.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_module_from_vfs.snap index 83be811be..203e917a6 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_module_from_vfs.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_module_from_vfs.snap @@ -12,6 +12,7 @@ metadata: - /foo.meta.json context: emit_legacy_scripts: false + middleware: ~ name: foo class_name: ModuleScript properties: diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_module_with_meta.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_module_with_meta.snap index 98a88ec7f..35ff43ff5 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_module_with_meta.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_module_with_meta.snap @@ -12,6 +12,7 @@ metadata: - /foo.meta.json context: emit_legacy_scripts: false + middleware: ~ name: foo class_name: ModuleScript properties: diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_script_disabled.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_script_disabled.snap index c47b83dc3..d5f710971 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_script_disabled.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_script_disabled.snap @@ -12,6 +12,7 @@ metadata: - /bar.meta.json context: emit_legacy_scripts: false + middleware: ~ name: bar class_name: Script properties: diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_script_with_meta.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_script_with_meta.snap index 72d24206f..a9985c790 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_script_with_meta.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_script_with_meta.snap @@ -12,6 +12,7 @@ metadata: - /foo.meta.json context: emit_legacy_scripts: false + middleware: ~ name: foo class_name: Script properties: diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_server_from_vfs.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_server_from_vfs.snap index 2b902f964..526166265 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_server_from_vfs.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_server_from_vfs.snap @@ -12,6 +12,7 @@ metadata: - /foo.meta.json context: emit_legacy_scripts: false + middleware: ~ name: foo class_name: Script properties: diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_from_direct_file.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_from_direct_file.snap index bd5db23db..5d25824b3 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_from_direct_file.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_from_direct_file.snap @@ -11,6 +11,7 @@ metadata: - /foo/hello.project.json context: emit_legacy_scripts: true + middleware: ~ name: direct-project class_name: Model properties: {} diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_path_property_overrides.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_path_property_overrides.snap index 77cb44c96..5ecaa4477 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_path_property_overrides.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_path_property_overrides.snap @@ -12,6 +12,7 @@ metadata: - /foo/default.project.json context: emit_legacy_scripts: true + middleware: project name: path-property-override class_name: StringValue properties: diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_children.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_children.snap index 914a2f95e..fa5a4d402 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_children.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_children.snap @@ -11,6 +11,7 @@ metadata: - /foo.project.json context: emit_legacy_scripts: true + middleware: ~ name: children class_name: Folder properties: {} @@ -27,6 +28,7 @@ children: relevant_paths: [] context: emit_legacy_scripts: true + middleware: ~ name: Child class_name: Model properties: {} diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_path_to_project.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_path_to_project.snap index 02e6ee0ba..075867f12 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_path_to_project.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_path_to_project.snap @@ -12,6 +12,7 @@ metadata: - /foo/default.project.json context: emit_legacy_scripts: true + middleware: project name: path-project class_name: Model properties: {} diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_path_to_project_with_children.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_path_to_project_with_children.snap index baf643653..e34f5a735 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_path_to_project_with_children.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_path_to_project_with_children.snap @@ -12,6 +12,7 @@ metadata: - /foo/default.project.json context: emit_legacy_scripts: true + middleware: project name: path-child-project class_name: Folder properties: {} @@ -28,6 +29,7 @@ children: relevant_paths: [] context: emit_legacy_scripts: true + middleware: ~ name: SomeChild class_name: Model properties: {} diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_path_to_txt.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_path_to_txt.snap index 223c366cd..7fce0d01c 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_path_to_txt.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_path_to_txt.snap @@ -13,6 +13,7 @@ metadata: - /foo/default.project.json context: emit_legacy_scripts: true + middleware: text name: path-project class_name: StringValue properties: diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_resolved_properties.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_resolved_properties.snap index eddb4e040..30bc6655c 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_resolved_properties.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_resolved_properties.snap @@ -11,6 +11,7 @@ metadata: - /foo.project.json context: emit_legacy_scripts: true + middleware: ~ name: resolved-properties class_name: StringValue properties: diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_unresolved_properties.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_unresolved_properties.snap index 8bd6e28b8..33c0bdaff 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_unresolved_properties.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_unresolved_properties.snap @@ -11,6 +11,7 @@ metadata: - /foo.project.json context: emit_legacy_scripts: true + middleware: ~ name: unresolved-properties class_name: StringValue properties: diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__toml__test__instance_from_vfs.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__toml__test__instance_from_vfs.snap index 329bdbcd3..6737442c1 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__toml__test__instance_from_vfs.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__toml__test__instance_from_vfs.snap @@ -12,6 +12,7 @@ metadata: - /foo.meta.json context: emit_legacy_scripts: true + middleware: ~ name: foo class_name: ModuleScript properties: diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__txt__test__instance_from_vfs.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__txt__test__instance_from_vfs.snap index aa3e0f067..961da5209 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__txt__test__instance_from_vfs.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__txt__test__instance_from_vfs.snap @@ -12,6 +12,7 @@ metadata: - /foo.meta.json context: emit_legacy_scripts: true + middleware: ~ name: foo class_name: StringValue properties: From 6824c2463c0f7af857aa8fffd5fc0ae37b1b60da Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 7 Dec 2023 12:07:13 -0800 Subject: [PATCH 017/366] Add `create_dir` to memofs --- crates/memofs/CHANGELOG.md | 1 + crates/memofs/src/in_memory_fs.rs | 5 +++++ crates/memofs/src/lib.rs | 19 +++++++++++++++++++ crates/memofs/src/noop_backend.rs | 7 +++++++ crates/memofs/src/std_backend.rs | 4 ++++ 5 files changed, 36 insertions(+) diff --git a/crates/memofs/CHANGELOG.md b/crates/memofs/CHANGELOG.md index c9961d0ed..c5ebd95ab 100644 --- a/crates/memofs/CHANGELOG.md +++ b/crates/memofs/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased Changes * Changed `StdBackend` file watching component to use minimal recursive watches. [#830] +* Added `create_dir` to allow creating directories. [#830]: https://github.com/rojo-rbx/rojo/pull/830 diff --git a/crates/memofs/src/in_memory_fs.rs b/crates/memofs/src/in_memory_fs.rs index fc8cf2efe..217899dc4 100644 --- a/crates/memofs/src/in_memory_fs.rs +++ b/crates/memofs/src/in_memory_fs.rs @@ -176,6 +176,11 @@ impl VfsBackend for InMemoryFs { } } + fn create_dir(&mut self, path: &Path) -> io::Result<()> { + let mut inner = self.inner.lock().unwrap(); + inner.load_snapshot(path.to_path_buf(), VfsSnapshot::empty_dir()) + } + fn remove_file(&mut self, path: &Path) -> io::Result<()> { let mut inner = self.inner.lock().unwrap(); diff --git a/crates/memofs/src/lib.rs b/crates/memofs/src/lib.rs index 744ca60fc..1ead8d19f 100644 --- a/crates/memofs/src/lib.rs +++ b/crates/memofs/src/lib.rs @@ -71,6 +71,7 @@ pub trait VfsBackend: sealed::Sealed + Send + 'static { fn read(&mut self, path: &Path) -> io::Result>; fn write(&mut self, path: &Path, data: &[u8]) -> io::Result<()>; fn read_dir(&mut self, path: &Path) -> io::Result; + fn create_dir(&mut self, path: &Path) -> io::Result<()>; fn metadata(&mut self, path: &Path) -> io::Result; fn remove_file(&mut self, path: &Path) -> io::Result<()>; fn remove_dir_all(&mut self, path: &Path) -> io::Result<()>; @@ -172,6 +173,11 @@ impl VfsInner { Ok(dir) } + fn create_dir>(&mut self, path: P) -> io::Result<()> { + let path = path.as_ref(); + self.backend.create_dir(path) + } + fn remove_file>(&mut self, path: P) -> io::Result<()> { let path = path.as_ref(); let _ = self.backend.unwatch(path); @@ -281,6 +287,19 @@ impl Vfs { self.inner.lock().unwrap().read_dir(path) } + /// Creates a directory at the provided location. + /// + /// Roughly equivalent to [`std::fs::create_dir`][std::fs::create_dir]. + /// Similiar to that function, this function will fail if the parent of the + /// path does not exist. + /// + /// [std::fs::create_dir]: https://doc.rust-lang.org/stable/std/fs/fn.create_dir.html + #[inline] + pub fn create_dir>(&self, path: P) -> io::Result<()> { + let path = path.as_ref(); + self.inner.lock().unwrap().create_dir(path) + } + /// Remove a file. /// /// Roughly equivalent to [`std::fs::remove_file`][std::fs::remove_file]. diff --git a/crates/memofs/src/noop_backend.rs b/crates/memofs/src/noop_backend.rs index efc8fd4ae..5a028d4be 100644 --- a/crates/memofs/src/noop_backend.rs +++ b/crates/memofs/src/noop_backend.rs @@ -35,6 +35,13 @@ impl VfsBackend for NoopBackend { )) } + fn create_dir(&mut self, _path: &Path) -> io::Result<()> { + Err(io::Error::new( + io::ErrorKind::Other, + "NoopBackend doesn't do anything", + )) + } + fn remove_file(&mut self, _path: &Path) -> io::Result<()> { Err(io::Error::new( io::ErrorKind::Other, diff --git a/crates/memofs/src/std_backend.rs b/crates/memofs/src/std_backend.rs index 0a8fb3b79..359d023ae 100644 --- a/crates/memofs/src/std_backend.rs +++ b/crates/memofs/src/std_backend.rs @@ -78,6 +78,10 @@ impl VfsBackend for StdBackend { }) } + fn create_dir(&mut self, path: &Path) -> io::Result<()> { + fs_err::create_dir(path) + } + fn remove_file(&mut self, path: &Path) -> io::Result<()> { fs_err::remove_file(path) } From 227865591654303f8d535f1fd6418e46d09fdc6e Mon Sep 17 00:00:00 2001 From: Micah Date: Mon, 11 Dec 2023 14:19:57 -0800 Subject: [PATCH 018/366] Export ScriptType and metadata from middleware --- src/snapshot_middleware/mod.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/snapshot_middleware/mod.rs b/src/snapshot_middleware/mod.rs index 5be18eb36..cfab3af72 100644 --- a/src/snapshot_middleware/mod.rs +++ b/src/snapshot_middleware/mod.rs @@ -35,7 +35,7 @@ use self::{ dir::snapshot_dir, json::snapshot_json, json_model::snapshot_json_model, - lua::{snapshot_lua, snapshot_lua_init, ScriptType}, + lua::{snapshot_lua, snapshot_lua_init}, project::snapshot_project, rbxm::snapshot_rbxm, rbxmx::snapshot_rbxmx, @@ -43,7 +43,12 @@ use self::{ txt::snapshot_txt, }; -pub use self::{project::snapshot_project_node, util::emit_legacy_scripts_default}; +pub use self::{ + lua::ScriptType, + meta_file::{AdjacentMetadata, DirectoryMetadata}, + project::snapshot_project_node, + util::emit_legacy_scripts_default, +}; /// Returns an `InstanceSnapshot` for the provided path. /// This will inspect the path and find the appropriate middleware for it, From b4522e6842cd194995c1bcf7af50dd5103d9bbf8 Mon Sep 17 00:00:00 2001 From: Micah Date: Mon, 11 Dec 2023 14:20:45 -0800 Subject: [PATCH 019/366] Add function for producing InstanceSnapshot from an Instance --- src/snapshot/instance_snapshot.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/snapshot/instance_snapshot.rs b/src/snapshot/instance_snapshot.rs index 0b3a4fca3..5337e529f 100644 --- a/src/snapshot/instance_snapshot.rs +++ b/src/snapshot/instance_snapshot.rs @@ -127,6 +127,17 @@ impl InstanceSnapshot { children, } } + + pub fn from_instance(instance: &Instance) -> Self { + Self { + snapshot_id: instance.referent(), + metadata: InstanceMetadata::new(), + name: Cow::Owned(instance.name.clone()), + class_name: Cow::Owned(instance.class.clone()), + properties: instance.properties.clone(), + children: Vec::new(), + } + } } impl Default for InstanceSnapshot { From 46cf2658037c8984f34b6257818c62000df657bc Mon Sep 17 00:00:00 2001 From: Micah Date: Mon, 11 Dec 2023 14:22:24 -0800 Subject: [PATCH 020/366] Add framework for syncback implementation --- src/lib.rs | 1 + src/syncback/fs_snapshot.rs | 61 +++++++++++++++ src/syncback/middleware.rs | 130 ++++++++++++++++++++++++++++++ src/syncback/mod.rs | 152 ++++++++++++++++++++++++++++++++++++ 4 files changed, 344 insertions(+) create mode 100644 src/syncback/fs_snapshot.rs create mode 100644 src/syncback/middleware.rs create mode 100644 src/syncback/mod.rs diff --git a/src/lib.rs b/src/lib.rs index 51195b6c1..63f25d177 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -20,6 +20,7 @@ mod serve_session; mod session_id; mod snapshot; mod snapshot_middleware; +mod syncback; mod web; pub use project::*; diff --git a/src/syncback/fs_snapshot.rs b/src/syncback/fs_snapshot.rs new file mode 100644 index 000000000..75796a015 --- /dev/null +++ b/src/syncback/fs_snapshot.rs @@ -0,0 +1,61 @@ +use std::{ + collections::{HashMap, HashSet}, + fmt, io, + path::{Path, PathBuf}, + sync::Arc, +}; + +use memofs::Vfs; + +pub struct FsSnapshot { + files: HashMap>>, + dir: HashSet, +} + +impl FsSnapshot { + pub fn new() -> Self { + Self { + files: HashMap::new(), + dir: HashSet::new(), + } + } + + pub fn with_file>(mut self, path: P, data: Vec) -> Self { + self.files + .insert(path.as_ref().to_path_buf(), Arc::new(data)); + self + } + + pub fn with_dir>(mut self, path: P) -> Self { + self.dir.insert(path.as_ref().to_path_buf()); + self + } + + pub fn push_file>(&mut self, path: P, data: Vec) { + self.files + .insert(path.as_ref().to_path_buf(), Arc::new(data)); + } + + pub fn write_to_vfs(&self, vfs: &Vfs) -> io::Result<()> { + for dir_path in &self.dir { + vfs.create_dir(dir_path)?; + } + for (path, contents) in &self.files { + vfs.write(path, contents.as_slice())?; + } + + Ok(()) + } +} + +impl fmt::Debug for FsSnapshot { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let files = self + .files + .iter() + .map(|(k, v)| format!("{}: {} bytes", k.display(), v.len())); + let dirs = self.dir.iter().map(|v| format!("{}", v.display())); + + f.debug_list().entries(files).entries(dirs).finish() + } +} diff --git a/src/syncback/middleware.rs b/src/syncback/middleware.rs new file mode 100644 index 000000000..0f44c843f --- /dev/null +++ b/src/syncback/middleware.rs @@ -0,0 +1,130 @@ +use std::collections::{HashMap, HashSet}; + +use rbx_dom_weak::{ + types::{Ref, Variant}, + Instance, +}; + +use crate::{ + resolution::UnresolvedValue, + snapshot::{InstanceMetadata, InstanceSnapshot, InstanceWithMeta}, + snapshot_middleware::{DirectoryMetadata, Middleware, ScriptType}, +}; + +use super::{FsSnapshot, SyncbackSnapshot}; + +#[derive(Debug)] +pub struct SyncbackReturn<'new, 'old> { + pub inst_snapshot: InstanceSnapshot, + pub fs_snapshot: FsSnapshot, + pub children: Vec>, + pub removed_children: Vec>, +} + +pub fn syncback_middleware<'new, 'old>( + snapshot: &SyncbackSnapshot<'new, 'old>, + middleware: Middleware, +) -> SyncbackReturn<'new, 'old> { + match middleware { + Middleware::ModuleScript => syncback_script(ScriptType::Module, snapshot), + Middleware::ClientScript => syncback_script(ScriptType::Client, snapshot), + Middleware::ServerScript => syncback_script(ScriptType::Server, snapshot), + Middleware::Dir => syncback_dir(snapshot), + _ => panic!("unsupported instance middleware {:?}", middleware), + } +} + +fn syncback_script<'new, 'old>( + script_type: ScriptType, + snapshot: &SyncbackSnapshot<'new, 'old>, +) -> SyncbackReturn<'new, 'old> { + let inst = snapshot.new_inst(); + + let mut path = snapshot.parent_path.clone(); + path.set_file_name(snapshot.name.clone()); + path.set_extension(match script_type { + ScriptType::Module => "lua", + ScriptType::Client => "client.lua", + ScriptType::Server => "server.lua", + }); + let contents = if let Some(Variant::String(source)) = inst.properties.get("Source") { + source.as_bytes().to_vec() + } else { + panic!("Source should be a string") + }; + + SyncbackReturn { + inst_snapshot: InstanceSnapshot::from_instance(inst).metadata(InstanceMetadata::new()), + fs_snapshot: FsSnapshot::new().with_file(path, contents), + // Scripts don't have a child! + children: Vec::new(), + removed_children: Vec::new(), + } +} + +fn syncback_dir<'new, 'old>(snapshot: &SyncbackSnapshot<'new, 'old>) -> SyncbackReturn<'new, 'old> { + let path = snapshot.parent_path.join(snapshot.name.clone()); + + let mut removed_children = Vec::new(); + let mut children = Vec::new(); + + if let Some(old_inst) = snapshot.old_inst() { + let old_children: HashMap<&str, Ref> = old_inst + .children() + .iter() + .map(|old_ref| { + ( + snapshot.get_old_instance(*old_ref).unwrap().name(), + *old_ref, + ) + }) + .collect(); + let new_children: HashSet<&str> = snapshot + .new_inst() + .children() + .iter() + .map(|new_ref| snapshot.get_new_instance(*new_ref).unwrap().name.as_str()) + .collect(); + + for child_ref in old_inst.children() { + let old_child = snapshot.get_old_instance(*child_ref).unwrap(); + // If it exists in the old tree but not the new one, it was removed. + if !new_children.contains(old_child.name()) { + removed_children.push(old_child); + } + } + + for child_ref in snapshot.new_inst().children() { + let new_child = snapshot.get_new_instance(*child_ref).unwrap(); + // If it exists in the new tree but not the old one, it was added. + match old_children.get(new_child.name.as_str()) { + None => children.push(snapshot.from_parent( + &new_child.name, + new_child.name.clone(), + *child_ref, + None, + )), + Some(old_ref) => children.push(snapshot.from_parent( + &new_child.name, + new_child.name.clone(), + *child_ref, + Some(*old_ref), + )), + } + } + } else { + for child_ref in snapshot.new_inst().children() { + let child = snapshot.get_new_instance(*child_ref).unwrap(); + children.push(snapshot.from_parent(&child.name, child.name.clone(), *child_ref, None)) + } + } + let mut fs_snapshot = FsSnapshot::new().with_dir(&path); + // TODO metadata + + SyncbackReturn { + inst_snapshot: InstanceSnapshot::from_instance(snapshot.new_inst()), + fs_snapshot, + children, + removed_children, + } +} diff --git a/src/syncback/mod.rs b/src/syncback/mod.rs new file mode 100644 index 000000000..351c1c855 --- /dev/null +++ b/src/syncback/mod.rs @@ -0,0 +1,152 @@ +mod fs_snapshot; +mod middleware; + +use crate::{ + snapshot::{hash_tree, InstanceSnapshot, InstanceWithMeta, RojoTree}, + snapshot_middleware::Middleware, + Project, +}; +use memofs::Vfs; +use rbx_dom_weak::{types::Ref, Instance, WeakDom}; +use std::path::{Path, PathBuf}; + +pub use fs_snapshot::FsSnapshot; + +use self::middleware::syncback_middleware; + +#[derive(Debug)] +pub struct SyncbackSnapshot<'new, 'old> { + old_tree: &'old RojoTree, + new_tree: &'new WeakDom, + old: Option, + new: Ref, + parent_path: PathBuf, + name: String, +} + +impl<'new, 'old> SyncbackSnapshot<'new, 'old> { + /// Constructs a SyncbackSnapshot from the provided refs + /// while inheriting the parent's trees and path + #[inline] + pub fn from_parent>( + &self, + extension: P, + new_name: String, + new_ref: Ref, + old_ref: Option, + ) -> Self { + Self { + old_tree: self.old_tree, + new_tree: self.new_tree, + old: old_ref, + new: new_ref, + parent_path: self.parent_path.join(extension.as_ref()), + name: new_name, + } + } + + /// The 'old' Instance this snapshot is for, if it exists. + #[inline] + pub fn old_inst(&self) -> Option> { + self.old.and_then(|old| self.old_tree.get_instance(old)) + } + + /// The 'new' Instance this snapshot is for. + #[inline] + pub fn new_inst(&self) -> &'new Instance { + self.new_tree + .get_by_ref(self.new) + .expect("SyncbackSnapshot should not contain invalid referents") + } + + /// Returns an Instance from the old tree with the provided referent, if it + /// exists. + #[inline] + pub fn get_old_instance(&self, referent: Ref) -> Option> { + self.old_tree.get_instance(referent) + } + + /// Returns an Instance from the new tree with the provided referent, if it + /// exists. + #[inline] + pub fn get_new_instance(&self, referent: Ref) -> Option<&'new Instance> { + self.new_tree.get_by_ref(referent) + } +} + +pub fn syncback_loop( + vfs: &Vfs, + old_tree: &RojoTree, + new_tree: &WeakDom, + project: &Project, +) -> anyhow::Result> { + let old_hashes = hash_tree(old_tree.inner()); + let new_hashes = hash_tree(new_tree); + let mut snapshots = vec![SyncbackSnapshot { + old_tree, + new_tree, + old: Some(old_tree.get_root_id()), + new: new_tree.root_ref(), + parent_path: project.file_location.clone(), + name: project.name.clone(), + }]; + + let mut replacements = Vec::new(); + + while let Some(snapshot) = snapshots.pop() { + // We can quickly check that two subtrees are identical and if they are, + // skip reconciling them. + if let Some(old_ref) = snapshot.old { + if old_hashes.get(&old_ref) == new_hashes.get(&snapshot.new) { + continue; + } + } + + let middleware = snapshot + .old_inst() + .and_then(|inst| inst.metadata().middleware) + .unwrap_or_else(|| get_best_middleware(snapshot.new_inst())); + + let syncback = syncback_middleware(&snapshot, middleware); + + if let Some(old_inst) = snapshot.old_inst() { + replacements.push((old_inst.parent(), syncback.inst_snapshot)); + } + + syncback.fs_snapshot.write_to_vfs(vfs)?; + + // TODO handle children + } + + Ok(replacements) +} + +fn get_best_middleware(inst: &Instance) -> Middleware { + match inst.class.as_str() { + "Folder" => Middleware::Dir, + // TODO this should probably just be rbxm + "Model" => Middleware::Rbxmx, + "Script" => { + if inst.children().len() == 0 { + Middleware::ServerScript + } else { + Middleware::ServerScriptDir + } + } + "LocalScript" => { + if inst.children().len() == 0 { + Middleware::ClientScript + } else { + Middleware::ClientScriptDir + } + } + "ModuleScript" => { + if inst.children().len() == 0 { + Middleware::ModuleScript + } else { + Middleware::ModuleScriptDir + } + } + _ => Middleware::Rbxmx, + } +} From 262283b613e3da53d15f5de69f03663003978a0c Mon Sep 17 00:00:00 2001 From: Micah Date: Mon, 11 Dec 2023 14:23:06 -0800 Subject: [PATCH 021/366] Add function for accessing root project to serve session --- src/serve_session.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/serve_session.rs b/src/serve_session.rs index 85ae77934..a99f34c38 100644 --- a/src/serve_session.rs +++ b/src/serve_session.rs @@ -220,6 +220,10 @@ impl ServeSession { pub fn root_dir(&self) -> &Path { self.root_project.folder_location() } + + pub fn root_project(&self) -> &Project { + &self.root_project + } } #[derive(Debug, Error)] From 79ad94a02972ca5b7022fdb81d59c9f4c4ecf83c Mon Sep 17 00:00:00 2001 From: Micah Date: Wed, 13 Dec 2023 11:55:40 -0800 Subject: [PATCH 022/366] Correctly update paths in dir middleware --- src/snapshot_middleware/mod.rs | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/snapshot_middleware/mod.rs b/src/snapshot_middleware/mod.rs index cfab3af72..70baa6616 100644 --- a/src/snapshot_middleware/mod.rs +++ b/src/snapshot_middleware/mod.rs @@ -19,6 +19,7 @@ mod txt; mod util; use std::{ + borrow::Cow, path::{Path, PathBuf}, sync::OnceLock, }; @@ -68,7 +69,13 @@ pub fn snapshot_from_vfs( if meta.is_dir() { let (middleware, name) = get_dir_middleware(vfs, path)?; // TODO: Support user defined init paths - middleware.snapshot(context, vfs, path, &name) + match middleware { + Middleware::Dir => middleware.snapshot(context, vfs, path, &name), + _ => { + let name_as_path: PathBuf = name.as_ref().into(); + middleware.snapshot(context, vfs, &path.join(name_as_path), &name) + } + } } else { let file_name = path .file_name() @@ -89,7 +96,10 @@ pub fn snapshot_from_vfs( /// Gets the appropriate middleware for a directory by checking for `init` /// files. This uses an intrinsic priority list and for compatibility, /// that order should be left unchanged. -fn get_dir_middleware>(vfs: &Vfs, dir: P) -> anyhow::Result<(Middleware, String)> { +fn get_dir_middleware>( + vfs: &Vfs, + dir: P, +) -> anyhow::Result<(Middleware, Cow<'static, str>)> { let path = dir.as_ref(); let dir_name = path .file_name() @@ -114,11 +124,11 @@ fn get_dir_middleware>(vfs: &Vfs, dir: P) -> anyhow::Result<(Midd for (middleware, name) in order { if vfs.metadata(path.join(name)).with_not_found()?.is_some() { - return Ok((*middleware, dir_name)); + return Ok((*middleware, Cow::Borrowed(*name))); } } - Ok((Middleware::Dir, dir_name)) + Ok((Middleware::Dir, Cow::Owned(dir_name))) } /// Gets a snapshot for a path given an InstanceContext and Vfs, taking From 9a5384da392b0e15c49636c82399c6356138f521 Mon Sep 17 00:00:00 2001 From: Micah Date: Wed, 13 Dec 2023 11:55:53 -0800 Subject: [PATCH 023/366] Hash datamodels --- src/snapshot/diff/hash.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/snapshot/diff/hash.rs b/src/snapshot/diff/hash.rs index c05db9a75..baece8ceb 100644 --- a/src/snapshot/diff/hash.rs +++ b/src/snapshot/diff/hash.rs @@ -21,10 +21,6 @@ pub fn hash_tree(dom: &WeakDom) -> HashMap { // end while let Some(referent) = order.pop() { let inst = dom.get_by_ref(referent).unwrap(); - // We don't really care about the equality of a DataModel. - if inst.class == "DataModel" { - continue; - } let hash = hash_inst(&mut prop_list, &map, inst); map.insert(referent, hash); From ccd4edcffa16149e5bdf6c489971b60c41c87247 Mon Sep 17 00:00:00 2001 From: Micah Date: Wed, 13 Dec 2023 12:54:20 -0800 Subject: [PATCH 024/366] Change InstigatingSource::ProjectNode to be a struct variant --- src/change_processor.rs | 17 +++++++++----- src/snapshot/metadata.rs | 23 +++++++++++++------ src/snapshot_middleware/project.rs | 12 +++++----- ..._project__test__project_with_children.snap | 9 ++++---- ...ct_with_path_to_project_with_children.snap | 9 ++++---- 5 files changed, 43 insertions(+), 27 deletions(-) diff --git a/src/change_processor.rs b/src/change_processor.rs index 497ba4e79..fc429190c 100644 --- a/src/change_processor.rs +++ b/src/change_processor.rs @@ -183,7 +183,7 @@ impl JobThreadContext { if let Some(instigating_source) = &instance.metadata().instigating_source { match instigating_source { InstigatingSource::Path(path) => fs::remove_file(path).unwrap(), - InstigatingSource::ProjectNode(_, _, _, _) => { + InstigatingSource::ProjectNode { .. } => { log::warn!( "Cannot remove instance {:?}, it's from a project file", id @@ -231,7 +231,7 @@ impl JobThreadContext { log::warn!("Cannot change Source to non-string value."); } } - InstigatingSource::ProjectNode(_, _, _, _) => { + InstigatingSource::ProjectNode { .. } => { log::warn!( "Cannot remove instance {:?}, it's from a project file", id @@ -317,16 +317,21 @@ fn compute_and_apply_changes(tree: &mut RojoTree, vfs: &Vfs, id: Ref) -> Option< } }, - InstigatingSource::ProjectNode(project_path, instance_name, project_node, parent_class) => { + InstigatingSource::ProjectNode { + path, + name, + node, + parent_class, + } => { // This instance is the direct subject of a project node. Since // there might be information associated with our instance from // the project file, we snapshot the entire project node again. let snapshot_result = snapshot_project_node( &metadata.context, - project_path, - instance_name, - project_node, + path, + name, + node, vfs, parent_class.as_ref().map(|name| name.as_str()), ); diff --git a/src/snapshot/metadata.rs b/src/snapshot/metadata.rs index 6df36e082..fbc70c112 100644 --- a/src/snapshot/metadata.rs +++ b/src/snapshot/metadata.rs @@ -206,22 +206,31 @@ impl PathIgnoreRule { } } +/// Represents where a particular Instance or InstanceSnapshot came from. #[derive(Clone, PartialEq, Serialize, Deserialize)] pub enum InstigatingSource { + /// The path the Instance was made from. Path(#[serde(serialize_with = "path_serializer::serialize_absolute")] PathBuf), - ProjectNode( - #[serde(serialize_with = "path_serializer::serialize_absolute")] PathBuf, - String, - ProjectNode, - Option, - ), + /// The node in a Project that the Instance was made from. + ProjectNode { + #[serde(serialize_with = "path_serializer::serialize_absolute")] + path: PathBuf, + name: String, + node: ProjectNode, + parent_class: Option, + }, } impl fmt::Debug for InstigatingSource { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { match self { InstigatingSource::Path(path) => write!(formatter, "Path({})", path.display()), - InstigatingSource::ProjectNode(path, name, node, parent_class) => write!( + InstigatingSource::ProjectNode { + name, + node, + path, + parent_class, + } => write!( formatter, "ProjectNode({}: {:?}) from path {} and parent class {:?}", name, diff --git a/src/snapshot_middleware/project.rs b/src/snapshot_middleware/project.rs index c92f42d8a..23b5b967b 100644 --- a/src/snapshot_middleware/project.rs +++ b/src/snapshot_middleware/project.rs @@ -277,12 +277,12 @@ pub fn snapshot_project_node( metadata.ignore_unknown_instances = true; } - metadata.instigating_source = Some(InstigatingSource::ProjectNode( - project_path.to_path_buf(), - instance_name.to_string(), - node.clone(), - parent_class.map(|name| name.to_owned()), - )); + metadata.instigating_source = Some(InstigatingSource::ProjectNode { + path: project_path.to_path_buf(), + name: instance_name.to_string(), + node: node.clone(), + parent_class: parent_class.map(|name| name.to_owned()), + }); Ok(Some(InstanceSnapshot { snapshot_id: Ref::none(), diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_children.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_children.snap index fa5a4d402..2f3cd2547 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_children.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_children.snap @@ -21,10 +21,11 @@ children: ignore_unknown_instances: true instigating_source: ProjectNode: - - /foo.project.json - - Child - - $className: Model - - Folder + path: /foo.project.json + name: Child + node: + $className: Model + parent_class: Folder relevant_paths: [] context: emit_legacy_scripts: true diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_path_to_project_with_children.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_path_to_project_with_children.snap index e34f5a735..ea628e4d9 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_path_to_project_with_children.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_path_to_project_with_children.snap @@ -22,10 +22,11 @@ children: ignore_unknown_instances: true instigating_source: ProjectNode: - - /foo/other.project.json - - SomeChild - - $className: Model - - Folder + path: /foo/other.project.json + name: SomeChild + node: + $className: Model + parent_class: Folder relevant_paths: [] context: emit_legacy_scripts: true From 4e71282a5c5058cc2a24d810a649a01c4919e986 Mon Sep 17 00:00:00 2001 From: Micah Date: Fri, 15 Dec 2023 12:46:29 -0800 Subject: [PATCH 025/366] Remove old diff code --- src/snapshot/diff/mod.rs | 46 +--------------------------------------- 1 file changed, 1 insertion(+), 45 deletions(-) diff --git a/src/snapshot/diff/mod.rs b/src/snapshot/diff/mod.rs index 14651ae8f..9c08d23bf 100644 --- a/src/snapshot/diff/mod.rs +++ b/src/snapshot/diff/mod.rs @@ -5,51 +5,7 @@ pub use hash::*; pub use variant::*; use rbx_dom_weak::{types::Ref, WeakDom}; -use std::{ - collections::{HashMap, VecDeque}, - hash::Hash, -}; - -pub fn diff_trees(dom_1: &WeakDom, dom_2: &WeakDom) -> Diff { - let list_1 = hash_tree(dom_1); - let mut list_2 = invert_map(hash_tree(dom_2)); - - let mut removals = Vec::new(); - - for (referent, hash) in list_1 { - // If it's in both lists, we'll pull it out of list_2 so that - // it doesn't get flagged as an addition. - if list_2.contains_key(&hash) { - list_2.remove(&hash); - } else { - removals.push(referent); - } - } - - Diff { - removals, - additions: list_2.into_iter().map(|(_, referent)| referent).collect(), - } -} - -fn invert_map(map: HashMap) -> HashMap { - map.into_iter().map(|(key, value)| (value, key)).collect() -} - -pub struct Diff { - /// Referents that were either removed or changed (in dom 1, not in dom 2) - pub removals: Vec, - /// Referents that were added or changed (in dom 2, not in dom 1) - pub additions: Vec, -} - -impl Diff { - /// Returns the total number of diffs represented by this struct - #[inline] - pub fn total(&self) -> usize { - self.removals.len() + self.additions.len() - } -} +use std::collections::VecDeque; pub(crate) fn descendants(dom: &WeakDom) -> Vec { let mut queue = VecDeque::new(); From 981829151bfd6a849fbab1a0fc718cb247d61711 Mon Sep 17 00:00:00 2001 From: Micah Date: Fri, 15 Dec 2023 12:47:37 -0800 Subject: [PATCH 026/366] Make a myriad of changes to syncback Implement project middleware --- src/syncback/middleware.rs | 134 ++++++++++++++++++++++++++++++++++++- src/syncback/mod.rs | 128 ++++++++++++++++++++--------------- 2 files changed, 204 insertions(+), 58 deletions(-) diff --git a/src/syncback/middleware.rs b/src/syncback/middleware.rs index 0f44c843f..0daeb9269 100644 --- a/src/syncback/middleware.rs +++ b/src/syncback/middleware.rs @@ -7,13 +7,13 @@ use rbx_dom_weak::{ use crate::{ resolution::UnresolvedValue, - snapshot::{InstanceMetadata, InstanceSnapshot, InstanceWithMeta}, + snapshot::{InstanceMetadata, InstanceSnapshot, InstanceWithMeta, InstigatingSource}, snapshot_middleware::{DirectoryMetadata, Middleware, ScriptType}, + Project, ProjectNode, }; use super::{FsSnapshot, SyncbackSnapshot}; -#[derive(Debug)] pub struct SyncbackReturn<'new, 'old> { pub inst_snapshot: InstanceSnapshot, pub fs_snapshot: FsSnapshot, @@ -26,6 +26,7 @@ pub fn syncback_middleware<'new, 'old>( middleware: Middleware, ) -> SyncbackReturn<'new, 'old> { match middleware { + Middleware::Project => syncback_project(snapshot), Middleware::ModuleScript => syncback_script(ScriptType::Module, snapshot), Middleware::ClientScript => syncback_script(ScriptType::Client, snapshot), Middleware::ServerScript => syncback_script(ScriptType::Server, snapshot), @@ -34,6 +35,36 @@ pub fn syncback_middleware<'new, 'old>( } } +pub fn get_best_middleware(inst: &Instance) -> Middleware { + match inst.class.as_str() { + "Folder" => Middleware::Dir, + // TODO this should probably just be rbxm + "Model" => Middleware::Rbxmx, + "Script" => { + if inst.children().len() == 0 { + Middleware::ServerScript + } else { + Middleware::ServerScriptDir + } + } + "LocalScript" => { + if inst.children().len() == 0 { + Middleware::ClientScript + } else { + Middleware::ClientScriptDir + } + } + "ModuleScript" => { + if inst.children().len() == 0 { + Middleware::ModuleScript + } else { + Middleware::ModuleScriptDir + } + } + _ => Middleware::Rbxmx, + } +} + fn syncback_script<'new, 'old>( script_type: ScriptType, snapshot: &SyncbackSnapshot<'new, 'old>, @@ -119,7 +150,7 @@ fn syncback_dir<'new, 'old>(snapshot: &SyncbackSnapshot<'new, 'old>) -> Syncback } } let mut fs_snapshot = FsSnapshot::new().with_dir(&path); - // TODO metadata + // TODO metadata, including classname SyncbackReturn { inst_snapshot: InstanceSnapshot::from_instance(snapshot.new_inst()), @@ -128,3 +159,100 @@ fn syncback_dir<'new, 'old>(snapshot: &SyncbackSnapshot<'new, 'old>) -> Syncback removed_children, } } + +fn syncback_project<'new, 'old>( + snapshot: &SyncbackSnapshot<'new, 'old>, +) -> SyncbackReturn<'new, 'old> { + // We need to build a 'new' project and serialize it using an FsSnapshot. + // It's convenient to start with the old one though, since it means we have + // a thing to iterate through. + let mut project = Project::load_from_slice( + &snapshot.vfs().read(&snapshot.parent_path).unwrap(), + &snapshot.parent_path, + ) + .unwrap(); + + let mut children = Vec::new(); + let mut removed_children = Vec::new(); + + // Projects are special. We won't be adding or removing things from them, + // so we'll simply match Instances on a per-node basis and rebuild the tree + // with the new instance's data. This matching will be done by class and name + // to simplify things. + let mut nodes = vec![( + &mut project.tree, + snapshot.new_inst(), + snapshot.old_inst().unwrap(), + )]; + while let Some((node, new_inst, old_inst)) = nodes.pop() { + let mut old_child_map = HashMap::with_capacity(old_inst.children().len()); + for child_ref in old_inst.children() { + let child = snapshot.get_old_instance(*child_ref).unwrap(); + old_child_map.insert(child.name(), child); + } + let mut new_child_map = HashMap::with_capacity(new_inst.children().len()); + for child_ref in new_inst.children() { + let child = snapshot.get_new_instance(*child_ref).unwrap(); + new_child_map.insert(child.name.as_str(), child); + } + for (child_name, child_node) in &mut node.children { + if let Some(new_child) = new_child_map.get(child_name.as_str()) { + if let Some(old_child) = old_child_map.get(child_name.as_str()) { + // TODO verify class names + for (name, value) in &new_child.properties { + if child_node.properties.contains_key(name) { + child_node + .properties + .insert(name.clone(), UnresolvedValue::from(value.clone())); + } + } + nodes.push((child_node, new_child, *old_child)); + new_child_map.remove(child_name.as_str()); + old_child_map.remove(child_name.as_str()); + } else { + log::error!( + "Node {} was in new tree but not old. How did we get here?", + child_name + ); + } + } else { + panic!("Cannot add or remove children from a project") + } + } + // From this point, both maps contain only children of the current + // instance that aren't in the project. So, we just do some quick and + // dirty matching to identify children that were: + // - added (in new but not old) + // - removed (in old but not new) + for (new_name, new_child) in new_child_map { + if let Some(old_inst) = old_child_map.get(new_name) { + children.push(snapshot.from_parent( + new_name, + new_name.to_string(), + new_child.referent(), + Some(old_inst.id()), + )); + old_child_map.remove(new_name); + } else { + // it's new + children.push(snapshot.from_parent( + new_name, + new_name.to_string(), + new_child.referent(), + None, + )); + } + } + removed_children.extend(old_child_map.into_values()); + } + + SyncbackReturn { + inst_snapshot: InstanceSnapshot::from_instance(snapshot.new_inst()), + fs_snapshot: FsSnapshot::new().with_file( + &project.file_location, + serde_json::to_vec(&project).unwrap(), + ), + children, + removed_children, + } +} diff --git a/src/syncback/mod.rs b/src/syncback/mod.rs index 351c1c855..5b6c6953d 100644 --- a/src/syncback/mod.rs +++ b/src/syncback/mod.rs @@ -3,21 +3,31 @@ mod middleware; use crate::{ snapshot::{hash_tree, InstanceSnapshot, InstanceWithMeta, RojoTree}, - snapshot_middleware::Middleware, Project, }; +use blake3::Hash; use memofs::Vfs; use rbx_dom_weak::{types::Ref, Instance, WeakDom}; -use std::path::{Path, PathBuf}; +use std::{ + collections::{HashMap, VecDeque}, + path::{Path, PathBuf}, + rc::Rc, +}; pub use fs_snapshot::FsSnapshot; -use self::middleware::syncback_middleware; +use self::middleware::{get_best_middleware, syncback_middleware}; -#[derive(Debug)] -pub struct SyncbackSnapshot<'new, 'old> { +struct SyncbackData<'new, 'old> { + vfs: &'old Vfs, old_tree: &'old RojoTree, new_tree: &'new WeakDom, + + old_hashes: Rc>, + new_hashes: Rc>, +} +pub struct SyncbackSnapshot<'new, 'old> { + data: Rc>, old: Option, new: Ref, parent_path: PathBuf, @@ -36,8 +46,7 @@ impl<'new, 'old> SyncbackSnapshot<'new, 'old> { old_ref: Option, ) -> Self { Self { - old_tree: self.old_tree, - new_tree: self.new_tree, + data: Rc::clone(&self.data), old: old_ref, new: new_ref, parent_path: self.parent_path.join(extension.as_ref()), @@ -45,46 +54,64 @@ impl<'new, 'old> SyncbackSnapshot<'new, 'old> { } } + /// Returns an Instance from the old tree with the provided referent, if it + /// exists. + #[inline] + pub fn get_old_instance(&self, referent: Ref) -> Option> { + self.data.old_tree.get_instance(referent) + } + + /// Returns an Instance from the new tree with the provided referent, if it + /// exists. + #[inline] + pub fn get_new_instance(&self, referent: Ref) -> Option<&'new Instance> { + self.data.new_tree.get_by_ref(referent) + } + /// The 'old' Instance this snapshot is for, if it exists. #[inline] pub fn old_inst(&self) -> Option> { - self.old.and_then(|old| self.old_tree.get_instance(old)) + self.old + .and_then(|old| self.data.old_tree.get_instance(old)) } /// The 'new' Instance this snapshot is for. #[inline] pub fn new_inst(&self) -> &'new Instance { - self.new_tree + self.data + .new_tree .get_by_ref(self.new) .expect("SyncbackSnapshot should not contain invalid referents") } - /// Returns an Instance from the old tree with the provided referent, if it - /// exists. - #[inline] - pub fn get_old_instance(&self, referent: Ref) -> Option> { - self.old_tree.get_instance(referent) - } - - /// Returns an Instance from the new tree with the provided referent, if it - /// exists. + /// Returns the underlying VFS being used for syncback. #[inline] - pub fn get_new_instance(&self, referent: Ref) -> Option<&'new Instance> { - self.new_tree.get_by_ref(referent) + pub fn vfs(&self) -> &Vfs { + self.data.vfs } } -pub fn syncback_loop( - vfs: &Vfs, - old_tree: &RojoTree, - new_tree: &WeakDom, +pub fn syncback_loop<'old, 'new>( + vfs: &'old Vfs, + old_tree: &'old RojoTree, + new_tree: &'new WeakDom, project: &Project, ) -> anyhow::Result> { - let old_hashes = hash_tree(old_tree.inner()); - let new_hashes = hash_tree(new_tree); - let mut snapshots = vec![SyncbackSnapshot { + log::debug!("Hashing project DOM"); + let old_hashes = Rc::new(hash_tree(old_tree.inner())); + log::debug!("Hashing file DOM"); + let new_hashes = Rc::new(hash_tree(new_tree)); + + let syncback_data = Rc::new(SyncbackData { + vfs, old_tree, new_tree, + old_hashes: Rc::clone(&old_hashes), + new_hashes: Rc::clone(&new_hashes), + }); + + let mut snapshots = vec![SyncbackSnapshot { + data: syncback_data, old: Some(old_tree.get_root_id()), new: new_tree.root_ref(), parent_path: project.file_location.clone(), @@ -98,6 +125,11 @@ pub fn syncback_loop( // skip reconciling them. if let Some(old_ref) = snapshot.old { if old_hashes.get(&old_ref) == new_hashes.get(&snapshot.new) { + log::debug!( + "Skipping {} due to it being identically hashed as {:?}", + get_inst_path(new_tree, snapshot.new), + old_hashes.get(&old_ref) + ); continue; } } @@ -106,6 +138,11 @@ pub fn syncback_loop( .old_inst() .and_then(|inst| inst.metadata().middleware) .unwrap_or_else(|| get_best_middleware(snapshot.new_inst())); + log::debug!( + "Middleware for {}: {:?}", + get_inst_path(new_tree, snapshot.new), + middleware + ); let syncback = syncback_middleware(&snapshot, middleware); @@ -113,40 +150,21 @@ pub fn syncback_loop( replacements.push((old_inst.parent(), syncback.inst_snapshot)); } + log::debug!("Writing {} to vfs", get_inst_path(new_tree, snapshot.new)); syncback.fs_snapshot.write_to_vfs(vfs)?; - // TODO handle children + snapshots.extend(syncback.children); } Ok(replacements) } -fn get_best_middleware(inst: &Instance) -> Middleware { - match inst.class.as_str() { - "Folder" => Middleware::Dir, - // TODO this should probably just be rbxm - "Model" => Middleware::Rbxmx, - "Script" => { - if inst.children().len() == 0 { - Middleware::ServerScript - } else { - Middleware::ServerScriptDir - } - } - "LocalScript" => { - if inst.children().len() == 0 { - Middleware::ClientScript - } else { - Middleware::ClientScriptDir - } - } - "ModuleScript" => { - if inst.children().len() == 0 { - Middleware::ModuleScript - } else { - Middleware::ModuleScriptDir - } - } - _ => Middleware::Rbxmx, +fn get_inst_path(dom: &WeakDom, referent: Ref) -> String { + let mut path: VecDeque<&str> = VecDeque::new(); + let mut inst = dom.get_by_ref(referent); + while let Some(instance) = inst { + path.push_front(&instance.name); + inst = dom.get_by_ref(instance.parent()); } + path.into_iter().collect::>().join(".") } From f9c85fb257fa4c3c6c198554a57f7b7d196af5dc Mon Sep 17 00:00:00 2001 From: Micah Date: Fri, 15 Dec 2023 12:47:49 -0800 Subject: [PATCH 027/366] Add some logging to FsSnapshot --- src/syncback/fs_snapshot.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/syncback/fs_snapshot.rs b/src/syncback/fs_snapshot.rs index 75796a015..1bf37cce2 100644 --- a/src/syncback/fs_snapshot.rs +++ b/src/syncback/fs_snapshot.rs @@ -37,13 +37,18 @@ impl FsSnapshot { } pub fn write_to_vfs(&self, vfs: &Vfs) -> io::Result<()> { + let mut dirs = 0; + let mut files = 0; for dir_path in &self.dir { vfs.create_dir(dir_path)?; + dirs += 1; } for (path, contents) in &self.files { vfs.write(path, contents.as_slice())?; + files += 1; } + log::debug!("Wrote {dirs} directories and {files} files to the VFS"); Ok(()) } } From 70a4bb38c5612244d6d3fd53a31509852f285096 Mon Sep 17 00:00:00 2001 From: Micah Date: Fri, 15 Dec 2023 12:48:04 -0800 Subject: [PATCH 028/366] Add a barebones syncback command --- src/cli/mod.rs | 4 +++ src/cli/syncback.rs | 79 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+) create mode 100644 src/cli/syncback.rs diff --git a/src/cli/mod.rs b/src/cli/mod.rs index 065128fb6..37ab9af93 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -7,6 +7,7 @@ mod init; mod plugin; mod serve; mod sourcemap; +mod syncback; mod upload; use std::{borrow::Cow, env, path::Path, str::FromStr}; @@ -21,6 +22,7 @@ pub use self::init::{InitCommand, InitKind}; pub use self::plugin::{PluginCommand, PluginSubcommand}; pub use self::serve::ServeCommand; pub use self::sourcemap::SourcemapCommand; +pub use self::syncback::SyncbackCommand; pub use self::upload::UploadCommand; /// Command line options that Rojo accepts, defined using the clap crate. @@ -46,6 +48,7 @@ impl Options { Subcommand::FmtProject(subcommand) => subcommand.run(), Subcommand::Doc(subcommand) => subcommand.run(), Subcommand::Plugin(subcommand) => subcommand.run(), + Subcommand::Syncback(subcommand) => subcommand.run(), } } } @@ -119,6 +122,7 @@ pub enum Subcommand { FmtProject(FmtProjectCommand), Doc(DocCommand), Plugin(PluginCommand), + Syncback(SyncbackCommand), } pub(super) fn resolve_path(path: &Path) -> Cow<'_, Path> { diff --git a/src/cli/syncback.rs b/src/cli/syncback.rs new file mode 100644 index 000000000..f6ac43b35 --- /dev/null +++ b/src/cli/syncback.rs @@ -0,0 +1,79 @@ +use std::{ + collections::HashMap, + fs, + hash::Hash, + path::{Path, PathBuf}, + time::Instant, +}; + +use clap::Parser; +use memofs::Vfs; +use rbx_dom_weak::WeakDom; +use rbx_xml::DecodeOptions; + +use crate::{serve_session::ServeSession, syncback::syncback_loop}; + +use super::resolve_path; + +/// Performs syncback for a project file +#[derive(Debug, Parser)] +pub struct SyncbackCommand { + /// Path to the project to sync back to. + #[clap(default_value = "")] + pub project: PathBuf, + + /// Path to the place to perform syncback on. + #[clap(long, short)] + pub input: PathBuf, +} + +impl SyncbackCommand { + pub fn run(&self) -> anyhow::Result<()> { + let path_old = resolve_path(&self.project); + let path_new = resolve_path(&self.input); + + log::info!("Opening project at {}", path_old.display()); + let session_old = ServeSession::new(Vfs::new_default(), path_old.clone())?; + + let dom_old = session_old.tree(); + log::info!("Reading place file at {}", path_new.display()); + let dom_new = read_dom(&path_new); + + let start = Instant::now(); + log::debug!("Beginning syncback..."); + syncback_loop( + session_old.vfs(), + &dom_old, + &dom_new, + session_old.root_project(), + )?; + log::debug!( + "Syncback finished in {:.02}s!", + start.elapsed().as_secs_f32() + ); + + Ok(()) + } +} + +fn read_dom(path: &Path) -> WeakDom { + let content = fs::read(path).unwrap(); + if &content[0..8] == b"(map: HashMap) -> HashMap { + map.into_iter().map(|(key, value)| (value, key)).collect() +} From 44a15d815695c6d71eeab75132d7a8c3406f634f Mon Sep 17 00:00:00 2001 From: Micah Date: Fri, 15 Dec 2023 12:59:30 -0800 Subject: [PATCH 029/366] Use pretty JSON for project reserialization --- src/syncback/middleware.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/syncback/middleware.rs b/src/syncback/middleware.rs index 0daeb9269..9a6ac5669 100644 --- a/src/syncback/middleware.rs +++ b/src/syncback/middleware.rs @@ -250,7 +250,7 @@ fn syncback_project<'new, 'old>( inst_snapshot: InstanceSnapshot::from_instance(snapshot.new_inst()), fs_snapshot: FsSnapshot::new().with_file( &project.file_location, - serde_json::to_vec(&project).unwrap(), + serde_json::to_vec_pretty(&project).unwrap(), ), children, removed_children, From fa093dfdf87622399ef83103efcb27df4efa520d Mon Sep 17 00:00:00 2001 From: Micah Date: Fri, 15 Dec 2023 12:59:42 -0800 Subject: [PATCH 030/366] Allow Variants to be converted into UnresolvedValue --- src/resolution.rs | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/resolution.rs b/src/resolution.rs index 9a0219eeb..9e7cc3f41 100644 --- a/src/resolution.rs +++ b/src/resolution.rs @@ -190,6 +190,50 @@ impl AmbiguousValue { } } +impl From for UnresolvedValue { + fn from(value: Variant) -> Self { + Self::Ambiguous(match value { + Variant::Bool(bool) => AmbiguousValue::Bool(bool), + Variant::Float32(n) => AmbiguousValue::Number(n as f64), + Variant::Float64(n) => AmbiguousValue::Number(n), + Variant::Int32(n) => AmbiguousValue::Number(n as f64), + Variant::Int64(n) => AmbiguousValue::Number(n as f64), + Variant::String(str) => AmbiguousValue::String(str), + Variant::Tags(tags) => { + AmbiguousValue::StringArray(tags.iter().map(|s| s.to_string()).collect()) + } + Variant::Content(content) => AmbiguousValue::String(content.into_string()), + Variant::Vector2(vector) => AmbiguousValue::Array2([vector.x as f64, vector.y as f64]), + Variant::Vector3(vector) => { + AmbiguousValue::Array3([vector.x as f64, vector.y as f64, vector.z as f64]) + } + Variant::Color3(color) => { + AmbiguousValue::Array3([color.r as f64, color.g as f64, color.b as f64]) + } + Variant::CFrame(cf) => AmbiguousValue::Array12([ + cf.position.x as f64, + cf.position.y as f64, + cf.position.z as f64, + cf.orientation.x.x as f64, + cf.orientation.x.y as f64, + cf.orientation.x.z as f64, + cf.orientation.y.x as f64, + cf.orientation.y.y as f64, + cf.orientation.y.z as f64, + cf.orientation.z.x as f64, + cf.orientation.z.y as f64, + cf.orientation.z.z as f64, + ]), + Variant::Attributes(attr) => AmbiguousValue::Attributes(attr), + Variant::Font(font) => AmbiguousValue::Font(font), + Variant::MaterialColors(colors) => AmbiguousValue::MaterialColors(colors), + _ => { + return Self::FullyQualified(value); + } + }) + } +} + fn find_descriptor( class_name: &str, prop_name: &str, From 93604bd0ea9762b0afec0148ab72c79e757706a5 Mon Sep 17 00:00:00 2001 From: Micah Date: Fri, 15 Dec 2023 13:51:06 -0800 Subject: [PATCH 031/366] Spin SyncbackSnapshot into its own module --- src/syncback/mod.rs | 96 ++++------------------------------------ src/syncback/snapshot.rs | 85 +++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 87 deletions(-) create mode 100644 src/syncback/snapshot.rs diff --git a/src/syncback/mod.rs b/src/syncback/mod.rs index 5b6c6953d..95091bfea 100644 --- a/src/syncback/mod.rs +++ b/src/syncback/mod.rs @@ -1,95 +1,18 @@ mod fs_snapshot; mod middleware; +mod snapshot; use crate::{ - snapshot::{hash_tree, InstanceSnapshot, InstanceWithMeta, RojoTree}, + snapshot::{hash_tree, InstanceSnapshot, RojoTree}, Project, }; -use blake3::Hash; use memofs::Vfs; -use rbx_dom_weak::{types::Ref, Instance, WeakDom}; -use std::{ - collections::{HashMap, VecDeque}, - path::{Path, PathBuf}, - rc::Rc, -}; +use rbx_dom_weak::{types::Ref, WeakDom}; +use std::{collections::VecDeque, rc::Rc}; pub use fs_snapshot::FsSnapshot; - -use self::middleware::{get_best_middleware, syncback_middleware}; - -struct SyncbackData<'new, 'old> { - vfs: &'old Vfs, - old_tree: &'old RojoTree, - new_tree: &'new WeakDom, - - old_hashes: Rc>, - new_hashes: Rc>, -} -pub struct SyncbackSnapshot<'new, 'old> { - data: Rc>, - old: Option, - new: Ref, - parent_path: PathBuf, - name: String, -} - -impl<'new, 'old> SyncbackSnapshot<'new, 'old> { - /// Constructs a SyncbackSnapshot from the provided refs - /// while inheriting the parent's trees and path - #[inline] - pub fn from_parent>( - &self, - extension: P, - new_name: String, - new_ref: Ref, - old_ref: Option, - ) -> Self { - Self { - data: Rc::clone(&self.data), - old: old_ref, - new: new_ref, - parent_path: self.parent_path.join(extension.as_ref()), - name: new_name, - } - } - - /// Returns an Instance from the old tree with the provided referent, if it - /// exists. - #[inline] - pub fn get_old_instance(&self, referent: Ref) -> Option> { - self.data.old_tree.get_instance(referent) - } - - /// Returns an Instance from the new tree with the provided referent, if it - /// exists. - #[inline] - pub fn get_new_instance(&self, referent: Ref) -> Option<&'new Instance> { - self.data.new_tree.get_by_ref(referent) - } - - /// The 'old' Instance this snapshot is for, if it exists. - #[inline] - pub fn old_inst(&self) -> Option> { - self.old - .and_then(|old| self.data.old_tree.get_instance(old)) - } - - /// The 'new' Instance this snapshot is for. - #[inline] - pub fn new_inst(&self) -> &'new Instance { - self.data - .new_tree - .get_by_ref(self.new) - .expect("SyncbackSnapshot should not contain invalid referents") - } - - /// Returns the underlying VFS being used for syncback. - #[inline] - pub fn vfs(&self) -> &Vfs { - self.data.vfs - } -} +pub use middleware::{get_best_middleware, syncback_middleware}; +pub use snapshot::{SyncbackData, SyncbackSnapshot}; pub fn syncback_loop<'old, 'new>( vfs: &'old Vfs, @@ -98,16 +21,14 @@ pub fn syncback_loop<'old, 'new>( project: &Project, ) -> anyhow::Result> { log::debug!("Hashing project DOM"); - let old_hashes = Rc::new(hash_tree(old_tree.inner())); + let old_hashes = hash_tree(old_tree.inner()); log::debug!("Hashing file DOM"); - let new_hashes = Rc::new(hash_tree(new_tree)); + let new_hashes = hash_tree(new_tree); let syncback_data = Rc::new(SyncbackData { vfs, old_tree, new_tree, - old_hashes: Rc::clone(&old_hashes), - new_hashes: Rc::clone(&new_hashes), }); let mut snapshots = vec![SyncbackSnapshot { @@ -150,6 +71,7 @@ pub fn syncback_loop<'old, 'new>( replacements.push((old_inst.parent(), syncback.inst_snapshot)); } + // TODO: Check if file names are valid files log::debug!("Writing {} to vfs", get_inst_path(new_tree, snapshot.new)); syncback.fs_snapshot.write_to_vfs(vfs)?; diff --git a/src/syncback/snapshot.rs b/src/syncback/snapshot.rs new file mode 100644 index 000000000..98ac7db73 --- /dev/null +++ b/src/syncback/snapshot.rs @@ -0,0 +1,85 @@ +use memofs::Vfs; +use std::{ + path::{Path, PathBuf}, + rc::Rc, +}; + +use crate::snapshot::{InstanceWithMeta, RojoTree}; +use rbx_dom_weak::{types::Ref, Instance, WeakDom}; + +pub struct SyncbackData<'new, 'old> { + pub(super) vfs: &'old Vfs, + pub(super) old_tree: &'old RojoTree, + pub(super) new_tree: &'new WeakDom, +} + +pub struct SyncbackSnapshot<'new, 'old> { + pub(super) data: Rc>, + pub(super) old: Option, + pub(super) new: Ref, + pub(super) parent_path: PathBuf, + pub(super) name: String, +} + +impl<'new, 'old> SyncbackSnapshot<'new, 'old> { + /// Constructs a SyncbackSnapshot from the provided refs + /// while inheriting the parent's trees and path + #[inline] + pub fn from_parent>( + &self, + extension: P, + new_name: String, + new_ref: Ref, + old_ref: Option, + ) -> Self { + Self { + data: Rc::clone(&self.data), + old: old_ref, + new: new_ref, + parent_path: self.parent_path.join(extension.as_ref()), + name: new_name, + } + } + + /// Returns an Instance from the old tree with the provided referent, if it + /// exists. + #[inline] + pub fn get_old_instance(&self, referent: Ref) -> Option> { + self.data.old_tree.get_instance(referent) + } + + /// Returns an Instance from the new tree with the provided referent, if it + /// exists. + #[inline] + pub fn get_new_instance(&self, referent: Ref) -> Option<&'new Instance> { + self.data.new_tree.get_by_ref(referent) + } + + /// The 'old' Instance this snapshot is for, if it exists. + #[inline] + pub fn old_inst(&self) -> Option> { + self.old + .and_then(|old| self.data.old_tree.get_instance(old)) + } + + /// The 'new' Instance this snapshot is for. + #[inline] + pub fn new_inst(&self) -> &'new Instance { + self.data + .new_tree + .get_by_ref(self.new) + .expect("SyncbackSnapshot should not contain invalid referents") + } + + /// Returns the underlying VFS being used for syncback. + #[inline] + pub fn vfs(&self) -> &'old Vfs { + self.data.vfs + } + + /// Returns the WeakDom used for the 'new' tree + #[inline] + pub fn new_tree(&self) -> &'new WeakDom { + self.data.new_tree + } +} From 5eb7adf2f01d8998eda0be14b3036f905f78b5ad Mon Sep 17 00:00:00 2001 From: Micah Date: Fri, 15 Dec 2023 13:53:19 -0800 Subject: [PATCH 032/366] Support rbxmx middleware for syncback --- src/syncback/middleware.rs | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/src/syncback/middleware.rs b/src/syncback/middleware.rs index 9a6ac5669..9bc26105b 100644 --- a/src/syncback/middleware.rs +++ b/src/syncback/middleware.rs @@ -30,6 +30,7 @@ pub fn syncback_middleware<'new, 'old>( Middleware::ModuleScript => syncback_script(ScriptType::Module, snapshot), Middleware::ClientScript => syncback_script(ScriptType::Client, snapshot), Middleware::ServerScript => syncback_script(ScriptType::Server, snapshot), + Middleware::Rbxmx => syncback_rbxmx(snapshot), Middleware::Dir => syncback_dir(snapshot), _ => panic!("unsupported instance middleware {:?}", middleware), } @@ -221,9 +222,7 @@ fn syncback_project<'new, 'old>( } // From this point, both maps contain only children of the current // instance that aren't in the project. So, we just do some quick and - // dirty matching to identify children that were: - // - added (in new but not old) - // - removed (in old but not new) + // dirty matching to identify children that were added and removed. for (new_name, new_child) in new_child_map { if let Some(old_inst) = old_child_map.get(new_name) { children.push(snapshot.from_parent( @@ -256,3 +255,23 @@ fn syncback_project<'new, 'old>( removed_children, } } + +pub fn syncback_rbxmx<'new, 'old>( + snapshot: &SyncbackSnapshot<'new, 'old>, +) -> SyncbackReturn<'new, 'old> { + // If any of the children of this Instance are scripts, we don't want + // include them in the model. So instead, we'll check and then serialize. + + let inst = snapshot.new_inst(); + let mut path = snapshot.parent_path.join(&inst.name); + path.set_extension("rbxmx"); + // Long-term, anyway. Right now we stay silly. + let mut serialized = Vec::new(); + rbx_xml::to_writer_default(&mut serialized, snapshot.new_tree(), &[inst.referent()]).unwrap(); + SyncbackReturn { + inst_snapshot: InstanceSnapshot::from_instance(inst), + fs_snapshot: FsSnapshot::new().with_file(&path, serialized), + children: Vec::new(), + removed_children: Vec::new(), + } +} From 5f82ce2bedbef4d117244c16bafc4add332f93ff Mon Sep 17 00:00:00 2001 From: Micah Date: Fri, 15 Dec 2023 13:56:19 -0800 Subject: [PATCH 033/366] Add spacing to project syncback to make it easier to read --- src/syncback/middleware.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/syncback/middleware.rs b/src/syncback/middleware.rs index 9bc26105b..8e7910b9b 100644 --- a/src/syncback/middleware.rs +++ b/src/syncback/middleware.rs @@ -185,6 +185,7 @@ fn syncback_project<'new, 'old>( snapshot.new_inst(), snapshot.old_inst().unwrap(), )]; + while let Some((node, new_inst, old_inst)) = nodes.pop() { let mut old_child_map = HashMap::with_capacity(old_inst.children().len()); for child_ref in old_inst.children() { @@ -196,6 +197,7 @@ fn syncback_project<'new, 'old>( let child = snapshot.get_new_instance(*child_ref).unwrap(); new_child_map.insert(child.name.as_str(), child); } + for (child_name, child_node) in &mut node.children { if let Some(new_child) = new_child_map.get(child_name.as_str()) { if let Some(old_child) = old_child_map.get(child_name.as_str()) { @@ -220,6 +222,7 @@ fn syncback_project<'new, 'old>( panic!("Cannot add or remove children from a project") } } + // From this point, both maps contain only children of the current // instance that aren't in the project. So, we just do some quick and // dirty matching to identify children that were added and removed. From c770770ad5d3fe00dbb7d9e4b73560f481b8b5e7 Mon Sep 17 00:00:00 2001 From: Micah Date: Fri, 15 Dec 2023 16:04:12 -0800 Subject: [PATCH 034/366] Support script directories --- src/syncback/middleware.rs | 87 +++++++++++++++++++++++++++++++------- 1 file changed, 72 insertions(+), 15 deletions(-) diff --git a/src/syncback/middleware.rs b/src/syncback/middleware.rs index 8e7910b9b..4678ec1ab 100644 --- a/src/syncback/middleware.rs +++ b/src/syncback/middleware.rs @@ -7,9 +7,9 @@ use rbx_dom_weak::{ use crate::{ resolution::UnresolvedValue, - snapshot::{InstanceMetadata, InstanceSnapshot, InstanceWithMeta, InstigatingSource}, - snapshot_middleware::{DirectoryMetadata, Middleware, ScriptType}, - Project, ProjectNode, + snapshot::{InstanceMetadata, InstanceSnapshot, InstanceWithMeta}, + snapshot_middleware::{Middleware, ScriptType}, + Project, }; use super::{FsSnapshot, SyncbackSnapshot}; @@ -32,6 +32,9 @@ pub fn syncback_middleware<'new, 'old>( Middleware::ServerScript => syncback_script(ScriptType::Server, snapshot), Middleware::Rbxmx => syncback_rbxmx(snapshot), Middleware::Dir => syncback_dir(snapshot), + Middleware::ModuleScriptDir => syncback_script_dir(ScriptType::Module, snapshot), + Middleware::ClientScriptDir => syncback_script_dir(ScriptType::Client, snapshot), + Middleware::ServerScriptDir => syncback_script_dir(ScriptType::Server, snapshot), _ => panic!("unsupported instance middleware {:?}", middleware), } } @@ -94,6 +97,38 @@ fn syncback_script<'new, 'old>( } } +fn syncback_script_dir<'new, 'old>( + script_type: ScriptType, + snapshot: &SyncbackSnapshot<'new, 'old>, +) -> SyncbackReturn<'new, 'old> { + let mut path = snapshot.parent_path.join("init"); + path.set_extension(match script_type { + ScriptType::Module => "lua", + ScriptType::Client => "client.lua", + ScriptType::Server => "server.lua", + }); + let contents = + if let Some(Variant::String(source)) = snapshot.new_inst().properties.get("Source") { + source.as_bytes().to_vec() + } else { + panic!("Source should be a string") + }; + + let dir_syncback = syncback_dir(snapshot); + let init_syncback = syncback_script(script_type, snapshot); + + let mut fs_snapshot = FsSnapshot::new(); + fs_snapshot.push_file(path, contents); + fs_snapshot.merge(dir_syncback.fs_snapshot); + + SyncbackReturn { + inst_snapshot: InstanceSnapshot::from_instance(snapshot.new_inst()), + fs_snapshot, + children: dir_syncback.children, + removed_children: dir_syncback.removed_children, + } +} + fn syncback_dir<'new, 'old>(snapshot: &SyncbackSnapshot<'new, 'old>) -> SyncbackReturn<'new, 'old> { let path = snapshot.parent_path.join(snapshot.name.clone()); @@ -186,7 +221,14 @@ fn syncback_project<'new, 'old>( snapshot.old_inst().unwrap(), )]; + // A map of referents from the new tree to the Path that created it, + // if it exists. This is a roundabout way to locate the parents of + // Instances. + let mut ref_to_node = HashMap::new(); + while let Some((node, new_inst, old_inst)) = nodes.pop() { + ref_to_node.insert(new_inst.referent(), node.path.as_ref()); + let mut old_child_map = HashMap::with_capacity(old_inst.children().len()); for child_ref in old_inst.children() { let child = snapshot.get_old_instance(*child_ref).unwrap(); @@ -227,22 +269,37 @@ fn syncback_project<'new, 'old>( // instance that aren't in the project. So, we just do some quick and // dirty matching to identify children that were added and removed. for (new_name, new_child) in new_child_map { + let parent_path = match ref_to_node.get(&new_child.parent()) { + Some(Some(path)) => path.path().to_path_buf(), + Some(None) => { + log::debug!("{new_name} was visited but has no path"); + continue; + } + None => { + log::debug!("{new_name} does not currently exist on FS"); + continue; + } + }; if let Some(old_inst) = old_child_map.get(new_name) { - children.push(snapshot.from_parent( - new_name, - new_name.to_string(), - new_child.referent(), - Some(old_inst.id()), - )); + // All children are descendants of a node of a project + // So we really just need to track which one is which. + children.push(SyncbackSnapshot { + data: snapshot.data.clone(), + old: Some(old_inst.id()), + new: new_child.referent(), + name: new_name.to_string(), + parent_path, + }); old_child_map.remove(new_name); } else { // it's new - children.push(snapshot.from_parent( - new_name, - new_name.to_string(), - new_child.referent(), - None, - )); + children.push(SyncbackSnapshot { + data: snapshot.data.clone(), + old: None, + new: new_child.referent(), + name: new_name.to_string(), + parent_path, + }); } } removed_children.extend(old_child_map.into_values()); From 343b97f34496c99b0598a0ae223819a3c3d4ffb0 Mon Sep 17 00:00:00 2001 From: Micah Date: Fri, 15 Dec 2023 16:04:51 -0800 Subject: [PATCH 035/366] Add merge, pop_dir support to FsSnapshot Make write_to_vfs ignore existing directory errors --- src/syncback/fs_snapshot.rs | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/syncback/fs_snapshot.rs b/src/syncback/fs_snapshot.rs index 1bf37cce2..dab1c6837 100644 --- a/src/syncback/fs_snapshot.rs +++ b/src/syncback/fs_snapshot.rs @@ -31,20 +31,34 @@ impl FsSnapshot { self } + pub fn merge(&mut self, other: Self) { + self.dir.extend(other.dir); + self.files.extend(other.files); + } + pub fn push_file>(&mut self, path: P, data: Vec) { self.files .insert(path.as_ref().to_path_buf(), Arc::new(data)); } - pub fn write_to_vfs(&self, vfs: &Vfs) -> io::Result<()> { + pub fn pop_dir>(&mut self, path: P) -> bool { + self.dir.remove(path.as_ref()) + } + + pub fn write_to_vfs>(&self, base: P, vfs: &Vfs) -> io::Result<()> { + let base_path = base.as_ref(); let mut dirs = 0; let mut files = 0; for dir_path in &self.dir { - vfs.create_dir(dir_path)?; + match vfs.create_dir(base_path.join(dir_path)) { + Ok(_) => (), + Err(err) if err.kind() == io::ErrorKind::AlreadyExists => (), + Err(err) => return Err(err), + }; dirs += 1; } for (path, contents) in &self.files { - vfs.write(path, contents.as_slice())?; + vfs.write(base_path.join(path), contents.as_slice())?; files += 1; } From ffcf0d612652181f619af13594d8b15afcdc0ff3 Mon Sep 17 00:00:00 2001 From: Micah Date: Fri, 15 Dec 2023 16:14:52 -0800 Subject: [PATCH 036/366] Use project instigating source for syncback --- src/syncback/middleware.rs | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/src/syncback/middleware.rs b/src/syncback/middleware.rs index 4678ec1ab..da31bebfb 100644 --- a/src/syncback/middleware.rs +++ b/src/syncback/middleware.rs @@ -7,7 +7,7 @@ use rbx_dom_weak::{ use crate::{ resolution::UnresolvedValue, - snapshot::{InstanceMetadata, InstanceSnapshot, InstanceWithMeta}, + snapshot::{InstanceMetadata, InstanceSnapshot, InstanceWithMeta, InstigatingSource}, snapshot_middleware::{Middleware, ScriptType}, Project, }; @@ -199,14 +199,23 @@ fn syncback_dir<'new, 'old>(snapshot: &SyncbackSnapshot<'new, 'old>) -> Syncback fn syncback_project<'new, 'old>( snapshot: &SyncbackSnapshot<'new, 'old>, ) -> SyncbackReturn<'new, 'old> { + let old_inst = snapshot + .old_inst() + .expect("project middleware shouldn't be used to make new files"); + // This can never be None. + let source = old_inst.metadata().instigating_source.as_ref().unwrap(); + + let project_path = match source { + InstigatingSource::Path(path) => path.as_path(), + InstigatingSource::ProjectNode { path, .. } => path.as_path(), + }; + // We need to build a 'new' project and serialize it using an FsSnapshot. // It's convenient to start with the old one though, since it means we have // a thing to iterate through. - let mut project = Project::load_from_slice( - &snapshot.vfs().read(&snapshot.parent_path).unwrap(), - &snapshot.parent_path, - ) - .unwrap(); + let mut project = + Project::load_from_slice(&snapshot.vfs().read(project_path).unwrap(), project_path) + .unwrap(); let mut children = Vec::new(); let mut removed_children = Vec::new(); @@ -215,11 +224,7 @@ fn syncback_project<'new, 'old>( // so we'll simply match Instances on a per-node basis and rebuild the tree // with the new instance's data. This matching will be done by class and name // to simplify things. - let mut nodes = vec![( - &mut project.tree, - snapshot.new_inst(), - snapshot.old_inst().unwrap(), - )]; + let mut nodes = vec![(&mut project.tree, snapshot.new_inst(), old_inst)]; // A map of referents from the new tree to the Path that created it, // if it exists. This is a roundabout way to locate the parents of From 46ee80faa4046df374d9c8a685ca04e988b1307e Mon Sep 17 00:00:00 2001 From: Micah Date: Fri, 15 Dec 2023 16:17:08 -0800 Subject: [PATCH 037/366] Write to VFS only once for syncback Start at actual project parent --- src/syncback/mod.rs | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/syncback/mod.rs b/src/syncback/mod.rs index 95091bfea..fe7508fe4 100644 --- a/src/syncback/mod.rs +++ b/src/syncback/mod.rs @@ -35,18 +35,24 @@ pub fn syncback_loop<'old, 'new>( data: syncback_data, old: Some(old_tree.get_root_id()), new: new_tree.root_ref(), - parent_path: project.file_location.clone(), + parent_path: project.folder_location().to_path_buf(), name: project.name.clone(), }]; let mut replacements = Vec::new(); + let mut fs_snapshot = FsSnapshot::new(); while let Some(snapshot) = snapshots.pop() { + log::debug!( + "instance {} parent is {}", + snapshot.name, + snapshot.parent_path.display() + ); // We can quickly check that two subtrees are identical and if they are, // skip reconciling them. if let Some(old_ref) = snapshot.old { if old_hashes.get(&old_ref) == new_hashes.get(&snapshot.new) { - log::debug!( + log::trace!( "Skipping {} due to it being identically hashed as {:?}", get_inst_path(new_tree, snapshot.new), old_hashes.get(&old_ref) @@ -59,7 +65,7 @@ pub fn syncback_loop<'old, 'new>( .old_inst() .and_then(|inst| inst.metadata().middleware) .unwrap_or_else(|| get_best_middleware(snapshot.new_inst())); - log::debug!( + log::trace!( "Middleware for {}: {:?}", get_inst_path(new_tree, snapshot.new), middleware @@ -72,12 +78,13 @@ pub fn syncback_loop<'old, 'new>( } // TODO: Check if file names are valid files - log::debug!("Writing {} to vfs", get_inst_path(new_tree, snapshot.new)); - syncback.fs_snapshot.write_to_vfs(vfs)?; + fs_snapshot.merge(syncback.fs_snapshot); snapshots.extend(syncback.children); } + fs_snapshot.write_to_vfs(project.folder_location(), vfs)?; + Ok(replacements) } From 88b2b1c98f5fb12c0ad7edee84ba610b1ec65d8a Mon Sep 17 00:00:00 2001 From: Micah Date: Fri, 15 Dec 2023 16:18:31 -0800 Subject: [PATCH 038/366] Don't superfluously snapshot init.lua files --- src/syncback/middleware.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/syncback/middleware.rs b/src/syncback/middleware.rs index da31bebfb..f85a80e6d 100644 --- a/src/syncback/middleware.rs +++ b/src/syncback/middleware.rs @@ -115,7 +115,6 @@ fn syncback_script_dir<'new, 'old>( }; let dir_syncback = syncback_dir(snapshot); - let init_syncback = syncback_script(script_type, snapshot); let mut fs_snapshot = FsSnapshot::new(); fs_snapshot.push_file(path, contents); From 9166a19df23aae092b04acb7659911b14b3317c9 Mon Sep 17 00:00:00 2001 From: Micah Date: Fri, 15 Dec 2023 16:22:19 -0800 Subject: [PATCH 039/366] Add syncback support for text middleware --- src/syncback/middleware.rs | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/syncback/middleware.rs b/src/syncback/middleware.rs index f85a80e6d..677a60922 100644 --- a/src/syncback/middleware.rs +++ b/src/syncback/middleware.rs @@ -30,6 +30,7 @@ pub fn syncback_middleware<'new, 'old>( Middleware::ModuleScript => syncback_script(ScriptType::Module, snapshot), Middleware::ClientScript => syncback_script(ScriptType::Client, snapshot), Middleware::ServerScript => syncback_script(ScriptType::Server, snapshot), + Middleware::Text => syncback_text(snapshot), Middleware::Rbxmx => syncback_rbxmx(snapshot), Middleware::Dir => syncback_dir(snapshot), Middleware::ModuleScriptDir => syncback_script_dir(ScriptType::Module, snapshot), @@ -339,3 +340,25 @@ pub fn syncback_rbxmx<'new, 'old>( removed_children: Vec::new(), } } + +fn syncback_text<'new, 'old>( + snapshot: &SyncbackSnapshot<'new, 'old>, +) -> SyncbackReturn<'new, 'old> { + let inst = snapshot.new_inst(); + + let mut path = snapshot.parent_path.clone(); + path.set_file_name(snapshot.name.clone()); + path.set_extension("txt"); + let contents = if let Some(Variant::String(source)) = inst.properties.get("Value") { + source.as_bytes().to_vec() + } else { + panic!("Value should be a string") + }; + + SyncbackReturn { + inst_snapshot: InstanceSnapshot::from_instance(inst).metadata(InstanceMetadata::new()), + fs_snapshot: FsSnapshot::new().with_file(path, contents), + children: Vec::new(), + removed_children: Vec::new(), + } +} From 3d37de1732a499de6f1d610f0f7133f56a6741cc Mon Sep 17 00:00:00 2001 From: Micah Date: Mon, 18 Dec 2023 09:12:57 -0800 Subject: [PATCH 040/366] Add `path` method to InstigatingSource --- src/snapshot/metadata.rs | 9 +++++++++ src/syncback/middleware.rs | 7 +------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/snapshot/metadata.rs b/src/snapshot/metadata.rs index fbc70c112..27ec40213 100644 --- a/src/snapshot/metadata.rs +++ b/src/snapshot/metadata.rs @@ -221,6 +221,15 @@ pub enum InstigatingSource { }, } +impl InstigatingSource { + pub fn path(&self) -> &Path { + match self { + Self::Path(path) => path.as_path(), + Self::ProjectNode { path, .. } => path.as_path(), + } + } +} + impl fmt::Debug for InstigatingSource { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { match self { diff --git a/src/syncback/middleware.rs b/src/syncback/middleware.rs index 677a60922..94a223d96 100644 --- a/src/syncback/middleware.rs +++ b/src/syncback/middleware.rs @@ -205,16 +205,11 @@ fn syncback_project<'new, 'old>( // This can never be None. let source = old_inst.metadata().instigating_source.as_ref().unwrap(); - let project_path = match source { - InstigatingSource::Path(path) => path.as_path(), - InstigatingSource::ProjectNode { path, .. } => path.as_path(), - }; - // We need to build a 'new' project and serialize it using an FsSnapshot. // It's convenient to start with the old one though, since it means we have // a thing to iterate through. let mut project = - Project::load_from_slice(&snapshot.vfs().read(project_path).unwrap(), project_path) + Project::load_from_slice(&snapshot.vfs().read(source.path()).unwrap(), source.path()) .unwrap(); let mut children = Vec::new(); From 75e8844a9f99ceb807294a45db7bb7b0e37881b7 Mon Sep 17 00:00:00 2001 From: Micah Date: Mon, 18 Dec 2023 11:31:58 -0800 Subject: [PATCH 041/366] Make JsonModel public and add QoL methods to it --- src/snapshot_middleware/json_model.rs | 36 ++++++++++++++++++++++++--- src/snapshot_middleware/mod.rs | 1 + 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/src/snapshot_middleware/json_model.rs b/src/snapshot_middleware/json_model.rs index 93d566a2e..1fa8c5a37 100644 --- a/src/snapshot_middleware/json_model.rs +++ b/src/snapshot_middleware/json_model.rs @@ -3,7 +3,7 @@ use std::{borrow::Cow, collections::HashMap, path::Path, str}; use anyhow::Context; use memofs::Vfs; use rbx_dom_weak::types::{Attributes, Ref}; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use crate::{ resolution::UnresolvedValue, @@ -54,9 +54,9 @@ pub fn snapshot_json_model( Ok(Some(snapshot)) } -#[derive(Debug, Deserialize)] +#[derive(Debug, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] -struct JsonModel { +pub struct JsonModel { #[serde(alias = "Name")] name: Option, @@ -117,6 +117,36 @@ impl JsonModel { children, }) } + + /// Constructs an empty JSON model with the provided name and class. + #[inline] + pub fn new(name: &str, class_name: &str) -> Self { + Self { + name: Some(name.to_string()), + class_name: class_name.to_string(), + children: Vec::new(), + properties: HashMap::new(), + attributes: HashMap::new(), + } + } + + /// Sets the properties of this `JsonModel`. + #[inline] + pub fn set_properties(&mut self, properties: HashMap) { + self.properties = properties; + } + + /// Sets the attributes of this `JsonModel`. + #[inline] + pub fn set_attributes(&mut self, attributes: HashMap) { + self.attributes = attributes; + } + + /// Pushes the provided `JsonModel` as a child of this one. + #[inline] + pub fn push_child(&mut self, child: Self) { + self.children.push(child); + } } #[cfg(test)] diff --git a/src/snapshot_middleware/mod.rs b/src/snapshot_middleware/mod.rs index 70baa6616..2880ba9ff 100644 --- a/src/snapshot_middleware/mod.rs +++ b/src/snapshot_middleware/mod.rs @@ -45,6 +45,7 @@ use self::{ }; pub use self::{ + json_model::JsonModel, lua::ScriptType, meta_file::{AdjacentMetadata, DirectoryMetadata}, project::snapshot_project_node, From 166f180a395a74fd63fd004d0d03e194de2139f5 Mon Sep 17 00:00:00 2001 From: Micah Date: Mon, 18 Dec 2023 11:32:17 -0800 Subject: [PATCH 042/366] Implement JsonModel syncback --- src/syncback/middleware.rs | 45 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/src/syncback/middleware.rs b/src/syncback/middleware.rs index 94a223d96..9d39cccf3 100644 --- a/src/syncback/middleware.rs +++ b/src/syncback/middleware.rs @@ -7,8 +7,8 @@ use rbx_dom_weak::{ use crate::{ resolution::UnresolvedValue, - snapshot::{InstanceMetadata, InstanceSnapshot, InstanceWithMeta, InstigatingSource}, - snapshot_middleware::{Middleware, ScriptType}, + snapshot::{InstanceMetadata, InstanceSnapshot, InstanceWithMeta}, + snapshot_middleware::{JsonModel, Middleware, ScriptType}, Project, }; @@ -31,6 +31,7 @@ pub fn syncback_middleware<'new, 'old>( Middleware::ClientScript => syncback_script(ScriptType::Client, snapshot), Middleware::ServerScript => syncback_script(ScriptType::Server, snapshot), Middleware::Text => syncback_text(snapshot), + Middleware::JsonModel => syncback_json_model(snapshot), Middleware::Rbxmx => syncback_rbxmx(snapshot), Middleware::Dir => syncback_dir(snapshot), Middleware::ModuleScriptDir => syncback_script_dir(ScriptType::Module, snapshot), @@ -130,6 +131,11 @@ fn syncback_script_dir<'new, 'old>( } fn syncback_dir<'new, 'old>(snapshot: &SyncbackSnapshot<'new, 'old>) -> SyncbackReturn<'new, 'old> { + log::debug!( + "parent of {} is {}", + snapshot.name, + snapshot.parent_path.display() + ); let path = snapshot.parent_path.join(snapshot.name.clone()); let mut removed_children = Vec::new(); @@ -357,3 +363,38 @@ fn syncback_text<'new, 'old>( removed_children: Vec::new(), } } + +fn syncback_json_model<'new, 'old>( + snapshot: &SyncbackSnapshot<'new, 'old>, +) -> SyncbackReturn<'new, 'old> { + let mut path = snapshot.parent_path.clone(); + path.set_file_name(snapshot.name.clone()); + path.set_extension("model.json"); + + let new_inst = snapshot.new_inst(); + let mut model = JsonModel::new(&new_inst.name, &new_inst.class); + let mut properties = HashMap::with_capacity(new_inst.properties.len()); + + // TODO handle attributes separately + if let Some(old_inst) = snapshot.old_inst() { + for (name, value) in &new_inst.properties { + if old_inst.properties().contains_key(name) { + properties.insert(name.clone(), UnresolvedValue::from(value.clone())); + } + } + } else { + for (name, value) in new_inst.properties.clone() { + properties.insert(name, UnresolvedValue::from(value)); + } + } + model.set_properties(properties); + + // TODO children + + SyncbackReturn { + inst_snapshot: InstanceSnapshot::from_instance(new_inst), + fs_snapshot: FsSnapshot::new().with_file(&path, serde_json::to_vec_pretty(&model).unwrap()), + children: Vec::new(), + removed_children: Vec::new(), + } +} From c575a4700fc2e1e9c5008dcae6fc15bca1dc602e Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 19 Dec 2023 08:27:23 -0800 Subject: [PATCH 043/366] Correct directory syncback --- src/syncback/middleware.rs | 19 ++++++++----------- src/syncback/snapshot.rs | 10 ++-------- 2 files changed, 10 insertions(+), 19 deletions(-) diff --git a/src/syncback/middleware.rs b/src/syncback/middleware.rs index 9d39cccf3..a0adcf84f 100644 --- a/src/syncback/middleware.rs +++ b/src/syncback/middleware.rs @@ -131,12 +131,13 @@ fn syncback_script_dir<'new, 'old>( } fn syncback_dir<'new, 'old>(snapshot: &SyncbackSnapshot<'new, 'old>) -> SyncbackReturn<'new, 'old> { + let path = snapshot.parent_path.join(snapshot.name.clone()); log::debug!( - "parent of {} is {}", + "parent of {} is {}, writing to {}", snapshot.name, - snapshot.parent_path.display() + snapshot.parent_path.display(), + path.display() ); - let path = snapshot.parent_path.join(snapshot.name.clone()); let mut removed_children = Vec::new(); let mut children = Vec::new(); @@ -171,14 +172,10 @@ fn syncback_dir<'new, 'old>(snapshot: &SyncbackSnapshot<'new, 'old>) -> Syncback let new_child = snapshot.get_new_instance(*child_ref).unwrap(); // If it exists in the new tree but not the old one, it was added. match old_children.get(new_child.name.as_str()) { - None => children.push(snapshot.from_parent( - &new_child.name, - new_child.name.clone(), - *child_ref, - None, - )), + None => { + children.push(snapshot.from_parent(new_child.name.clone(), *child_ref, None)) + } Some(old_ref) => children.push(snapshot.from_parent( - &new_child.name, new_child.name.clone(), *child_ref, Some(*old_ref), @@ -188,7 +185,7 @@ fn syncback_dir<'new, 'old>(snapshot: &SyncbackSnapshot<'new, 'old>) -> Syncback } else { for child_ref in snapshot.new_inst().children() { let child = snapshot.get_new_instance(*child_ref).unwrap(); - children.push(snapshot.from_parent(&child.name, child.name.clone(), *child_ref, None)) + children.push(snapshot.from_parent(child.name.clone(), *child_ref, None)) } } let mut fs_snapshot = FsSnapshot::new().with_dir(&path); diff --git a/src/syncback/snapshot.rs b/src/syncback/snapshot.rs index 98ac7db73..a4cf86098 100644 --- a/src/syncback/snapshot.rs +++ b/src/syncback/snapshot.rs @@ -25,18 +25,12 @@ impl<'new, 'old> SyncbackSnapshot<'new, 'old> { /// Constructs a SyncbackSnapshot from the provided refs /// while inheriting the parent's trees and path #[inline] - pub fn from_parent>( - &self, - extension: P, - new_name: String, - new_ref: Ref, - old_ref: Option, - ) -> Self { + pub fn from_parent(&self, new_name: String, new_ref: Ref, old_ref: Option) -> Self { Self { data: Rc::clone(&self.data), old: old_ref, new: new_ref, - parent_path: self.parent_path.join(extension.as_ref()), + parent_path: self.parent_path.join(&self.name), name: new_name, } } From 38b1f9bdf08f309e546717ae91bd5730c53c6901 Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 19 Dec 2023 08:47:08 -0800 Subject: [PATCH 044/366] Correct path of project nodes --- src/syncback/middleware.rs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/syncback/middleware.rs b/src/syncback/middleware.rs index a0adcf84f..d1090cdbd 100644 --- a/src/syncback/middleware.rs +++ b/src/syncback/middleware.rs @@ -132,12 +132,6 @@ fn syncback_script_dir<'new, 'old>( fn syncback_dir<'new, 'old>(snapshot: &SyncbackSnapshot<'new, 'old>) -> SyncbackReturn<'new, 'old> { let path = snapshot.parent_path.join(snapshot.name.clone()); - log::debug!( - "parent of {} is {}, writing to {}", - snapshot.name, - snapshot.parent_path.display(), - path.display() - ); let mut removed_children = Vec::new(); let mut children = Vec::new(); @@ -214,6 +208,7 @@ fn syncback_project<'new, 'old>( let mut project = Project::load_from_slice(&snapshot.vfs().read(source.path()).unwrap(), source.path()) .unwrap(); + let base_path = source.path().parent().unwrap(); let mut children = Vec::new(); let mut removed_children = Vec::new(); @@ -273,7 +268,7 @@ fn syncback_project<'new, 'old>( // dirty matching to identify children that were added and removed. for (new_name, new_child) in new_child_map { let parent_path = match ref_to_node.get(&new_child.parent()) { - Some(Some(path)) => path.path().to_path_buf(), + Some(Some(path)) => base_path.join(path.path()), Some(None) => { log::debug!("{new_name} was visited but has no path"); continue; From e17984b0c9537c9a5696cb701c7126192ad4f0a5 Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 19 Dec 2023 09:50:16 -0800 Subject: [PATCH 045/366] Validate file names (badly) --- src/syncback/middleware.rs | 31 ++++++++++++++++++++++--- src/syncback/mod.rs | 46 +++++++++++++++++++++++++++++++------- 2 files changed, 66 insertions(+), 11 deletions(-) diff --git a/src/syncback/middleware.rs b/src/syncback/middleware.rs index d1090cdbd..bfac0b04a 100644 --- a/src/syncback/middleware.rs +++ b/src/syncback/middleware.rs @@ -12,7 +12,7 @@ use crate::{ Project, }; -use super::{FsSnapshot, SyncbackSnapshot}; +use super::{is_valid_file_name, FsSnapshot, SyncbackSnapshot}; pub struct SyncbackReturn<'new, 'old> { pub inst_snapshot: InstanceSnapshot, @@ -75,6 +75,10 @@ fn syncback_script<'new, 'old>( script_type: ScriptType, snapshot: &SyncbackSnapshot<'new, 'old>, ) -> SyncbackReturn<'new, 'old> { + if !is_valid_file_name(&snapshot.name) { + panic!("cannot create a file with name {}", snapshot.name); + } + let inst = snapshot.new_inst(); let mut path = snapshot.parent_path.clone(); @@ -103,6 +107,10 @@ fn syncback_script_dir<'new, 'old>( script_type: ScriptType, snapshot: &SyncbackSnapshot<'new, 'old>, ) -> SyncbackReturn<'new, 'old> { + if !is_valid_file_name(&snapshot.name) { + panic!("cannot create a directory with name {}", snapshot.name); + } + let mut path = snapshot.parent_path.join("init"); path.set_extension(match script_type { ScriptType::Module => "lua", @@ -131,6 +139,10 @@ fn syncback_script_dir<'new, 'old>( } fn syncback_dir<'new, 'old>(snapshot: &SyncbackSnapshot<'new, 'old>) -> SyncbackReturn<'new, 'old> { + if !is_valid_file_name(&snapshot.name) { + panic!("cannot create a directory with name {}", snapshot.name); + } + let path = snapshot.parent_path.join(snapshot.name.clone()); let mut removed_children = Vec::new(); @@ -303,6 +315,9 @@ fn syncback_project<'new, 'old>( removed_children.extend(old_child_map.into_values()); } + // We don't need to validate any file names for the FsSnapshot + // because projects can't ever be made by syncback, so they must + // already exist on the file system SyncbackReturn { inst_snapshot: InstanceSnapshot::from_instance(snapshot.new_inst()), fs_snapshot: FsSnapshot::new().with_file( @@ -319,9 +334,12 @@ pub fn syncback_rbxmx<'new, 'old>( ) -> SyncbackReturn<'new, 'old> { // If any of the children of this Instance are scripts, we don't want // include them in the model. So instead, we'll check and then serialize. + if !is_valid_file_name(&snapshot.name) { + panic!("cannot create a file with name {}", snapshot.name); + } let inst = snapshot.new_inst(); - let mut path = snapshot.parent_path.join(&inst.name); + let mut path = snapshot.parent_path.join(&snapshot.name); path.set_extension("rbxmx"); // Long-term, anyway. Right now we stay silly. let mut serialized = Vec::new(); @@ -337,8 +355,11 @@ pub fn syncback_rbxmx<'new, 'old>( fn syncback_text<'new, 'old>( snapshot: &SyncbackSnapshot<'new, 'old>, ) -> SyncbackReturn<'new, 'old> { - let inst = snapshot.new_inst(); + if !is_valid_file_name(&snapshot.name) { + panic!("cannot create a file with name {}", snapshot.name); + } + let inst = snapshot.new_inst(); let mut path = snapshot.parent_path.clone(); path.set_file_name(snapshot.name.clone()); path.set_extension("txt"); @@ -359,6 +380,10 @@ fn syncback_text<'new, 'old>( fn syncback_json_model<'new, 'old>( snapshot: &SyncbackSnapshot<'new, 'old>, ) -> SyncbackReturn<'new, 'old> { + if !is_valid_file_name(&snapshot.name) { + panic!("cannot create a file with name {}", snapshot.name); + } + let mut path = snapshot.parent_path.clone(); path.set_file_name(snapshot.name.clone()); path.set_extension("model.json"); diff --git a/src/syncback/mod.rs b/src/syncback/mod.rs index fe7508fe4..30eb023e4 100644 --- a/src/syncback/mod.rs +++ b/src/syncback/mod.rs @@ -43,11 +43,6 @@ pub fn syncback_loop<'old, 'new>( let mut fs_snapshot = FsSnapshot::new(); while let Some(snapshot) = snapshots.pop() { - log::debug!( - "instance {} parent is {}", - snapshot.name, - snapshot.parent_path.display() - ); // We can quickly check that two subtrees are identical and if they are, // skip reconciling them. if let Some(old_ref) = snapshot.old { @@ -65,9 +60,10 @@ pub fn syncback_loop<'old, 'new>( .old_inst() .and_then(|inst| inst.metadata().middleware) .unwrap_or_else(|| get_best_middleware(snapshot.new_inst())); - log::trace!( - "Middleware for {}: {:?}", + log::debug!( + "instance {} parent is {} (using middleware {:?})", get_inst_path(new_tree, snapshot.new), + snapshot.parent_path.display(), middleware ); @@ -77,7 +73,6 @@ pub fn syncback_loop<'old, 'new>( replacements.push((old_inst.parent(), syncback.inst_snapshot)); } - // TODO: Check if file names are valid files fs_snapshot.merge(syncback.fs_snapshot); snapshots.extend(syncback.children); @@ -97,3 +92,38 @@ fn get_inst_path(dom: &WeakDom, referent: Ref) -> String { } path.into_iter().collect::>().join(".") } + +/// A list of file names that are not valid on Windows. +const INVALID_WINDOWS_NAMES: [&str; 22] = [ + "CON", "PRN", "AUX", "NUL", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", + "COM9", "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9", +]; + +/// A list of all characters that are outright forbidden to be included +/// in a file's name. +const FORBIDDEN_CHARS: [char; 9] = ['<', '>', ':', '"', '/', '|', '?', '*', '\\']; + +/// Returns whether a given name is a valid file name. This takes into account +/// rules for Windows, MacOS, and Linux. +/// +/// In practice however, these broadly overlap so the only unexpected behavior +/// is Windows, where there are 22 reserved names. +pub fn is_valid_file_name>(name: S) -> bool { + let str = name.as_ref(); + + if str.ends_with(' ') || str.ends_with('.') { + return false; + } + // TODO check control characters + for forbidden in FORBIDDEN_CHARS { + if str.contains(forbidden) { + return false; + } + } + for forbidden in INVALID_WINDOWS_NAMES { + if str == forbidden { + return false; + } + } + true +} From 97bbb203dd5003f0a47fb97513002b636469671e Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 19 Dec 2023 10:51:24 -0800 Subject: [PATCH 046/366] Extend paths correctly in syncback middleware --- src/syncback/middleware.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/syncback/middleware.rs b/src/syncback/middleware.rs index bfac0b04a..2c165f60d 100644 --- a/src/syncback/middleware.rs +++ b/src/syncback/middleware.rs @@ -81,8 +81,7 @@ fn syncback_script<'new, 'old>( let inst = snapshot.new_inst(); - let mut path = snapshot.parent_path.clone(); - path.set_file_name(snapshot.name.clone()); + let mut path = snapshot.parent_path.join(&snapshot.name); path.set_extension(match script_type { ScriptType::Module => "lua", ScriptType::Client => "client.lua", @@ -111,7 +110,8 @@ fn syncback_script_dir<'new, 'old>( panic!("cannot create a directory with name {}", snapshot.name); } - let mut path = snapshot.parent_path.join("init"); + let mut path = snapshot.parent_path.join(&snapshot.name); + path.push("init"); path.set_extension(match script_type { ScriptType::Module => "lua", ScriptType::Client => "client.lua", @@ -360,9 +360,9 @@ fn syncback_text<'new, 'old>( } let inst = snapshot.new_inst(); - let mut path = snapshot.parent_path.clone(); - path.set_file_name(snapshot.name.clone()); + let mut path = snapshot.parent_path.join(&snapshot.name); path.set_extension("txt"); + let contents = if let Some(Variant::String(source)) = inst.properties.get("Value") { source.as_bytes().to_vec() } else { @@ -384,8 +384,7 @@ fn syncback_json_model<'new, 'old>( panic!("cannot create a file with name {}", snapshot.name); } - let mut path = snapshot.parent_path.clone(); - path.set_file_name(snapshot.name.clone()); + let mut path = snapshot.parent_path.join(&snapshot.name); path.set_extension("model.json"); let new_inst = snapshot.new_inst(); From d69130d522cf83da5e97f4c0c584ef5864eb4fd3 Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 19 Dec 2023 10:54:33 -0800 Subject: [PATCH 047/366] Skip serializing name field of JsonModel --- src/snapshot_middleware/json_model.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/snapshot_middleware/json_model.rs b/src/snapshot_middleware/json_model.rs index 1fa8c5a37..76acddca2 100644 --- a/src/snapshot_middleware/json_model.rs +++ b/src/snapshot_middleware/json_model.rs @@ -57,7 +57,7 @@ pub fn snapshot_json_model( #[derive(Debug, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] pub struct JsonModel { - #[serde(alias = "Name")] + #[serde(alias = "Name", skip_serializing)] name: Option, #[serde(alias = "ClassName")] From 28e42325d42235186bab56383392cc9d3d54665a Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 19 Dec 2023 13:49:57 -0800 Subject: [PATCH 048/366] Move syncback middleware to the appropriate modules --- src/snapshot_middleware/dir.rs | 80 ++++- src/snapshot_middleware/json_model.rs | 42 +++ src/snapshot_middleware/lua.rs | 76 ++++- src/snapshot_middleware/mod.rs | 39 ++- src/snapshot_middleware/project.rs | 120 ++++++++ src/snapshot_middleware/rbxmx.rs | 30 +- src/snapshot_middleware/txt.rs | 31 +- src/syncback/middleware.rs | 416 -------------------------- src/syncback/mod.rs | 48 ++- src/syncback/snapshot.rs | 17 +- 10 files changed, 453 insertions(+), 446 deletions(-) delete mode 100644 src/syncback/middleware.rs diff --git a/src/snapshot_middleware/dir.rs b/src/snapshot_middleware/dir.rs index eea248a52..811510af5 100644 --- a/src/snapshot_middleware/dir.rs +++ b/src/snapshot_middleware/dir.rs @@ -1,8 +1,15 @@ -use std::path::Path; +use std::{ + collections::{HashMap, HashSet}, + path::Path, +}; use memofs::{DirEntry, IoResultExt, Vfs}; +use rbx_dom_weak::types::Ref; -use crate::snapshot::{InstanceContext, InstanceMetadata, InstanceSnapshot}; +use crate::{ + snapshot::{InstanceContext, InstanceMetadata, InstanceSnapshot}, + syncback::{is_valid_file_name, FsSnapshot, SyncbackReturn, SyncbackSnapshot}, +}; use super::{meta_file::DirectoryMetadata, snapshot_from_vfs}; @@ -86,6 +93,75 @@ pub fn snapshot_dir_no_meta( Ok(Some(snapshot)) } +pub fn syncback_dir<'new, 'old>( + snapshot: &SyncbackSnapshot<'new, 'old>, +) -> anyhow::Result> { + if !is_valid_file_name(&snapshot.name) { + anyhow::bail!("cannot create a directory with name {}", snapshot.name); + } + + let path = snapshot.parent_path.join(&snapshot.name); + + let mut removed_children = Vec::new(); + let mut children = Vec::new(); + + if let Some(old_inst) = snapshot.old_inst() { + let old_children: HashMap<&str, Ref> = old_inst + .children() + .iter() + .map(|old_ref| { + ( + snapshot.get_old_instance(*old_ref).unwrap().name(), + *old_ref, + ) + }) + .collect(); + let new_children: HashSet<&str> = snapshot + .new_inst() + .children() + .iter() + .map(|new_ref| snapshot.get_new_instance(*new_ref).unwrap().name.as_str()) + .collect(); + + for child_ref in old_inst.children() { + let old_child = snapshot.get_old_instance(*child_ref).unwrap(); + // If it exists in the old tree but not the new one, it was removed. + if !new_children.contains(old_child.name()) { + removed_children.push(old_child); + } + } + + for child_ref in snapshot.new_inst().children() { + let new_child = snapshot.get_new_instance(*child_ref).unwrap(); + // If it exists in the new tree but not the old one, it was added. + match old_children.get(new_child.name.as_str()) { + None => { + children.push(snapshot.from_parent(new_child.name.clone(), *child_ref, None)) + } + Some(old_ref) => children.push(snapshot.from_parent( + new_child.name.clone(), + *child_ref, + Some(*old_ref), + )), + } + } + } else { + for child_ref in snapshot.new_inst().children() { + let child = snapshot.get_new_instance(*child_ref).unwrap(); + children.push(snapshot.from_parent(child.name.clone(), *child_ref, None)) + } + } + let fs_snapshot = FsSnapshot::new().with_dir(&path); + // TODO metadata, including classname + + Ok(SyncbackReturn { + inst_snapshot: InstanceSnapshot::from_instance(snapshot.new_inst()), + fs_snapshot, + children, + removed_children, + }) +} + #[cfg(test)] mod test { use super::*; diff --git a/src/snapshot_middleware/json_model.rs b/src/snapshot_middleware/json_model.rs index 76acddca2..229b94318 100644 --- a/src/snapshot_middleware/json_model.rs +++ b/src/snapshot_middleware/json_model.rs @@ -8,6 +8,7 @@ use serde::{Deserialize, Serialize}; use crate::{ resolution::UnresolvedValue, snapshot::{InstanceContext, InstanceSnapshot}, + syncback::{is_valid_file_name, FsSnapshot, SyncbackReturn, SyncbackSnapshot}, }; pub fn snapshot_json_model( @@ -54,6 +55,47 @@ pub fn snapshot_json_model( Ok(Some(snapshot)) } +pub fn syncback_json_model<'new, 'old>( + snapshot: &SyncbackSnapshot<'new, 'old>, +) -> anyhow::Result> { + if !is_valid_file_name(&snapshot.name) { + anyhow::bail!("cannot create a file with name {}", snapshot.name); + } + + let mut path = snapshot.parent_path.join(&snapshot.name); + path.set_extension("model.json"); + + let new_inst = snapshot.new_inst(); + let mut model = JsonModel::new(&new_inst.name, &new_inst.class); + let mut properties = HashMap::with_capacity(new_inst.properties.len()); + + // TODO handle attributes separately + if let Some(old_inst) = snapshot.old_inst() { + for (name, value) in &new_inst.properties { + if old_inst.properties().contains_key(name) { + properties.insert(name.clone(), UnresolvedValue::from(value.clone())); + } + } + } else { + for (name, value) in new_inst.properties.clone() { + properties.insert(name, UnresolvedValue::from(value)); + } + } + model.set_properties(properties); + + // TODO children + + Ok(SyncbackReturn { + inst_snapshot: InstanceSnapshot::from_instance(new_inst), + fs_snapshot: FsSnapshot::new().with_file( + &path, + serde_json::to_vec_pretty(&model).context("failed to serialize new JSON Model")?, + ), + children: Vec::new(), + removed_children: Vec::new(), + }) +} + #[derive(Debug, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] pub struct JsonModel { diff --git a/src/snapshot_middleware/lua.rs b/src/snapshot_middleware/lua.rs index 650a71eb7..2f7eb3457 100644 --- a/src/snapshot_middleware/lua.rs +++ b/src/snapshot_middleware/lua.rs @@ -2,12 +2,15 @@ use std::{collections::HashMap, path::Path, str}; use anyhow::Context; use memofs::{IoResultExt, Vfs}; -use rbx_dom_weak::types::Enum; +use rbx_dom_weak::types::{Enum, Variant}; -use crate::snapshot::{InstanceContext, InstanceMetadata, InstanceSnapshot}; +use crate::{ + snapshot::{InstanceContext, InstanceMetadata, InstanceSnapshot}, + syncback::{is_valid_file_name, FsSnapshot, SyncbackReturn, SyncbackSnapshot}, +}; use super::{ - dir::{dir_meta, snapshot_dir_no_meta}, + dir::{dir_meta, snapshot_dir_no_meta, syncback_dir}, meta_file::AdjacentMetadata, }; @@ -120,6 +123,73 @@ pub fn snapshot_lua_init( Ok(Some(init_snapshot)) } +pub fn syncback_lua<'new, 'old>( + script_type: ScriptType, + snapshot: &SyncbackSnapshot<'new, 'old>, +) -> anyhow::Result> { + if !is_valid_file_name(&snapshot.name) { + anyhow::bail!("cannot create a file with name {}", snapshot.name); + } + + let inst = snapshot.new_inst(); + + let mut path = snapshot.parent_path.join(&snapshot.name); + path.set_extension(match script_type { + ScriptType::Module => "lua", + ScriptType::Client => "client.lua", + ScriptType::Server => "server.lua", + }); + let contents = if let Some(Variant::String(source)) = inst.properties.get("Source") { + source.as_bytes().to_vec() + } else { + anyhow::bail!("Scripts must have a `Source` property that is a String") + }; + + Ok(SyncbackReturn { + inst_snapshot: InstanceSnapshot::from_instance(inst).metadata(InstanceMetadata::new()), + fs_snapshot: FsSnapshot::new().with_file(path, contents), + // Scripts don't have a child! + children: Vec::new(), + removed_children: Vec::new(), + }) +} + +pub fn syncback_lua_init<'new, 'old>( + script_type: ScriptType, + snapshot: &SyncbackSnapshot<'new, 'old>, +) -> anyhow::Result> { + if !is_valid_file_name(&snapshot.name) { + anyhow::bail!("cannot create a directory with name {}", snapshot.name); + } + + let mut path = snapshot.parent_path.join(&snapshot.name); + path.push("init"); + path.set_extension(match script_type { + ScriptType::Module => "lua", + ScriptType::Client => "client.lua", + ScriptType::Server => "server.lua", + }); + let contents = + if let Some(Variant::String(source)) = snapshot.new_inst().properties.get("Source") { + source.as_bytes().to_vec() + } else { + anyhow::bail!("Scripts must have a `Source` property that is a String") + }; + + let dir_syncback = syncback_dir(snapshot)?; + + let mut fs_snapshot = FsSnapshot::new(); + fs_snapshot.push_file(path, contents); + fs_snapshot.merge(dir_syncback.fs_snapshot); + + Ok(SyncbackReturn { + inst_snapshot: InstanceSnapshot::from_instance(snapshot.new_inst()), + fs_snapshot, + children: dir_syncback.children, + removed_children: dir_syncback.removed_children, + }) +} + #[cfg(test)] mod test { use super::*; diff --git a/src/snapshot_middleware/mod.rs b/src/snapshot_middleware/mod.rs index 2880ba9ff..34b1170e3 100644 --- a/src/snapshot_middleware/mod.rs +++ b/src/snapshot_middleware/mod.rs @@ -28,20 +28,23 @@ use anyhow::Context; use memofs::{IoResultExt, Vfs}; use serde::{Deserialize, Serialize}; -use crate::glob::Glob; use crate::snapshot::{InstanceContext, InstanceSnapshot, SyncRule}; +use crate::{ + glob::Glob, + syncback::{SyncbackReturn, SyncbackSnapshot}, +}; use self::{ csv::{snapshot_csv, snapshot_csv_init}, - dir::snapshot_dir, + dir::{snapshot_dir, syncback_dir}, json::snapshot_json, - json_model::snapshot_json_model, - lua::{snapshot_lua, snapshot_lua_init}, - project::snapshot_project, + json_model::{snapshot_json_model, syncback_json_model}, + lua::{snapshot_lua, snapshot_lua_init, syncback_lua, syncback_lua_init}, + project::{snapshot_project, syncback_project}, rbxm::snapshot_rbxm, - rbxmx::snapshot_rbxmx, + rbxmx::{snapshot_rbxmx, syncback_rbxmx}, toml::snapshot_toml, - txt::snapshot_txt, + txt::{snapshot_txt, syncback_txt}, }; pub use self::{ @@ -234,6 +237,28 @@ impl Middleware { } output } + + /// Runs the syncback mechanism for the provided middleware given a + /// SyncbackSnapshot. + pub fn syncback<'new, 'old>( + &self, + snapshot: &SyncbackSnapshot<'new, 'old>, + ) -> anyhow::Result> { + match self { + Middleware::Project => syncback_project(snapshot), + Middleware::ServerScript => syncback_lua(ScriptType::Server, snapshot), + Middleware::ClientScript => syncback_lua(ScriptType::Client, snapshot), + Middleware::ModuleScript => syncback_lua(ScriptType::Module, snapshot), + Middleware::Rbxmx => syncback_rbxmx(snapshot), + Middleware::Text => syncback_txt(snapshot), + Middleware::JsonModel => syncback_json_model(snapshot), + Middleware::Dir => syncback_dir(snapshot), + Middleware::ServerScriptDir => syncback_lua_init(ScriptType::Server, snapshot), + Middleware::ClientScriptDir => syncback_lua_init(ScriptType::Client, snapshot), + Middleware::ModuleScriptDir => syncback_lua_init(ScriptType::Module, snapshot), + _ => anyhow::bail!("cannot syncback with middleware {:?}", self), + } + } } /// A helper for easily defining a SyncRule. Arguments are passed literally diff --git a/src/snapshot_middleware/project.rs b/src/snapshot_middleware/project.rs index 23b5b967b..47651009d 100644 --- a/src/snapshot_middleware/project.rs +++ b/src/snapshot_middleware/project.rs @@ -7,10 +7,12 @@ use rbx_reflection::ClassTag; use crate::{ project::{PathNode, Project, ProjectNode}, + resolution::UnresolvedValue, snapshot::{ InstanceContext, InstanceMetadata, InstanceSnapshot, InstigatingSource, PathIgnoreRule, SyncRule, }, + syncback::{FsSnapshot, SyncbackReturn, SyncbackSnapshot}, }; use super::{emit_legacy_scripts_default, snapshot_from_vfs}; @@ -294,6 +296,124 @@ pub fn snapshot_project_node( })) } +pub fn syncback_project<'new, 'old>( + snapshot: &SyncbackSnapshot<'new, 'old>, +) -> anyhow::Result> { + let old_inst = snapshot + .old_inst() + .context("project middleware shouldn't be used to make new files")?; + // This can never be None. + let source = old_inst.metadata().instigating_source.as_ref().unwrap(); + + // We need to build a 'new' project and serialize it using an FsSnapshot. + // It's convenient to start with the old one though, since it means we have + // a thing to iterate through. + let mut project = + Project::load_from_slice(&snapshot.vfs().read(source.path()).unwrap(), source.path()) + .context("could not syncback project due to fs error")?; + + let base_path = source.path().parent().unwrap(); + + let mut children = Vec::new(); + let mut removed_children = Vec::new(); + + // Projects are special. We won't be adding or removing things from them, + // so we'll simply match Instances on a per-node basis and rebuild the tree + // with the new instance's data. This matching will be done by class and name + // to simplify things. + let mut nodes = vec![(&mut project.tree, snapshot.new_inst(), old_inst)]; + + // A map of referents from the new tree to the Path that created it, + // if it exists. This is a roundabout way to locate the parents of + // Instances. + let mut ref_to_node = HashMap::new(); + + while let Some((node, new_inst, old_inst)) = nodes.pop() { + ref_to_node.insert(new_inst.referent(), node.path.as_ref()); + + let mut old_child_map = HashMap::with_capacity(old_inst.children().len()); + for child_ref in old_inst.children() { + let child = snapshot.get_old_instance(*child_ref).unwrap(); + old_child_map.insert(child.name(), child); + } + let mut new_child_map = HashMap::with_capacity(new_inst.children().len()); + for child_ref in new_inst.children() { + let child = snapshot.get_new_instance(*child_ref).unwrap(); + new_child_map.insert(child.name.as_str(), child); + } + + for (child_name, child_node) in &mut node.children { + if let Some(new_child) = new_child_map.get(child_name.as_str()) { + if let Some(old_child) = old_child_map.get(child_name.as_str()) { + // TODO verify class names + for (name, value) in &new_child.properties { + if child_node.properties.contains_key(name) { + child_node + .properties + .insert(name.clone(), UnresolvedValue::from(value.clone())); + } + } + nodes.push((child_node, new_child, *old_child)); + new_child_map.remove(child_name.as_str()); + old_child_map.remove(child_name.as_str()); + } + } else { + anyhow::bail!("Cannot add or remove children from a project") + } + } + + // From this point, both maps contain only children of the current + // instance that aren't in the project. So, we just do some quick and + // dirty matching to identify children that were added and removed. + for (new_name, new_child) in new_child_map { + let parent_path = match ref_to_node.get(&new_child.parent()) { + Some(Some(path)) => base_path.join(path.path()), + Some(None) => { + continue; + } + None => { + continue; + } + }; + if let Some(old_inst) = old_child_map.get(new_name) { + // All children are descendants of a node of a project + // So we really just need to track which one is which. + children.push(SyncbackSnapshot { + data: snapshot.data.clone(), + old: Some(old_inst.id()), + new: new_child.referent(), + parent_path, + name: new_name.to_string(), + }); + old_child_map.remove(new_name); + } else { + // it's new + children.push(SyncbackSnapshot { + data: snapshot.data.clone(), + old: None, + new: new_child.referent(), + parent_path, + name: new_name.to_string(), + }); + } + } + removed_children.extend(old_child_map.into_values()); + } + + // We don't need to validate any file names for the FsSnapshot + // because projects can't ever be made by syncback, so they must + // already exist on the file system + Ok(SyncbackReturn { + inst_snapshot: InstanceSnapshot::from_instance(snapshot.new_inst()), + fs_snapshot: FsSnapshot::new().with_file( + &project.file_location, + serde_json::to_vec_pretty(&project).context("failed to serialize new project")?, + ), + children, + removed_children, + }) +} + fn infer_class_name(name: &str, parent_class: Option<&str>) -> Option> { // If className wasn't defined from another source, we may be able // to infer one. diff --git a/src/snapshot_middleware/rbxmx.rs b/src/snapshot_middleware/rbxmx.rs index 4266dc0d7..78bb24984 100644 --- a/src/snapshot_middleware/rbxmx.rs +++ b/src/snapshot_middleware/rbxmx.rs @@ -3,7 +3,10 @@ use std::path::Path; use anyhow::Context; use memofs::Vfs; -use crate::snapshot::{InstanceContext, InstanceMetadata, InstanceSnapshot}; +use crate::{ + snapshot::{InstanceContext, InstanceMetadata, InstanceSnapshot}, + syncback::{is_valid_file_name, FsSnapshot, SyncbackReturn, SyncbackSnapshot}, +}; pub fn snapshot_rbxmx( context: &InstanceContext, @@ -41,6 +44,31 @@ pub fn snapshot_rbxmx( } } +pub fn syncback_rbxmx<'new, 'old>( + snapshot: &SyncbackSnapshot<'new, 'old>, +) -> anyhow::Result> { + // If any of the children of this Instance are scripts, we don't want + // include them in the model. So instead, we'll check and then serialize. + if !is_valid_file_name(&snapshot.name) { + anyhow::bail!("cannot create a file with name {}", snapshot.name); + } + + let inst = snapshot.new_inst(); + let mut path = snapshot.parent_path.join(&snapshot.name); + path.set_extension("rbxmx"); + // Long-term, anyway. Right now we stay silly. + let mut serialized = Vec::new(); + rbx_xml::to_writer_default(&mut serialized, snapshot.new_tree(), &[inst.referent()]) + .context("failed to serialize new rbxmx")?; + + Ok(SyncbackReturn { + inst_snapshot: InstanceSnapshot::from_instance(inst), + fs_snapshot: FsSnapshot::new().with_file(&path, serialized), + children: Vec::new(), + removed_children: Vec::new(), + }) +} + #[cfg(test)] mod test { use super::*; diff --git a/src/snapshot_middleware/txt.rs b/src/snapshot_middleware/txt.rs index 207243f9f..c24ee2a94 100644 --- a/src/snapshot_middleware/txt.rs +++ b/src/snapshot_middleware/txt.rs @@ -3,8 +3,12 @@ use std::{path::Path, str}; use anyhow::Context; use maplit::hashmap; use memofs::{IoResultExt, Vfs}; +use rbx_dom_weak::types::Variant; -use crate::snapshot::{InstanceContext, InstanceMetadata, InstanceSnapshot}; +use crate::{ + snapshot::{InstanceContext, InstanceMetadata, InstanceSnapshot}, + syncback::{is_valid_file_name, FsSnapshot, SyncbackReturn, SyncbackSnapshot}, +}; use super::meta_file::AdjacentMetadata; @@ -44,6 +48,31 @@ pub fn snapshot_txt( Ok(Some(snapshot)) } +pub fn syncback_txt<'new, 'old>( + snapshot: &SyncbackSnapshot<'new, 'old>, +) -> anyhow::Result> { + if !is_valid_file_name(&snapshot.name) { + anyhow::bail!("cannot create a file with name {}", snapshot.name); + } + + let inst = snapshot.new_inst(); + let mut path = snapshot.parent_path.join(&snapshot.name); + path.set_extension("txt"); + + let contents = if let Some(Variant::String(source)) = inst.properties.get("Value") { + source.as_bytes().to_vec() + } else { + anyhow::bail!("StringValues must have a `Value` property that is a String"); + }; + + Ok(SyncbackReturn { + inst_snapshot: InstanceSnapshot::from_instance(inst), + fs_snapshot: FsSnapshot::new().with_file(path, contents), + children: Vec::new(), + removed_children: Vec::new(), + }) +} + #[cfg(test)] mod test { use super::*; diff --git a/src/syncback/middleware.rs b/src/syncback/middleware.rs deleted file mode 100644 index 2c165f60d..000000000 --- a/src/syncback/middleware.rs +++ /dev/null @@ -1,416 +0,0 @@ -use std::collections::{HashMap, HashSet}; - -use rbx_dom_weak::{ - types::{Ref, Variant}, - Instance, -}; - -use crate::{ - resolution::UnresolvedValue, - snapshot::{InstanceMetadata, InstanceSnapshot, InstanceWithMeta}, - snapshot_middleware::{JsonModel, Middleware, ScriptType}, - Project, -}; - -use super::{is_valid_file_name, FsSnapshot, SyncbackSnapshot}; - -pub struct SyncbackReturn<'new, 'old> { - pub inst_snapshot: InstanceSnapshot, - pub fs_snapshot: FsSnapshot, - pub children: Vec>, - pub removed_children: Vec>, -} - -pub fn syncback_middleware<'new, 'old>( - snapshot: &SyncbackSnapshot<'new, 'old>, - middleware: Middleware, -) -> SyncbackReturn<'new, 'old> { - match middleware { - Middleware::Project => syncback_project(snapshot), - Middleware::ModuleScript => syncback_script(ScriptType::Module, snapshot), - Middleware::ClientScript => syncback_script(ScriptType::Client, snapshot), - Middleware::ServerScript => syncback_script(ScriptType::Server, snapshot), - Middleware::Text => syncback_text(snapshot), - Middleware::JsonModel => syncback_json_model(snapshot), - Middleware::Rbxmx => syncback_rbxmx(snapshot), - Middleware::Dir => syncback_dir(snapshot), - Middleware::ModuleScriptDir => syncback_script_dir(ScriptType::Module, snapshot), - Middleware::ClientScriptDir => syncback_script_dir(ScriptType::Client, snapshot), - Middleware::ServerScriptDir => syncback_script_dir(ScriptType::Server, snapshot), - _ => panic!("unsupported instance middleware {:?}", middleware), - } -} - -pub fn get_best_middleware(inst: &Instance) -> Middleware { - match inst.class.as_str() { - "Folder" => Middleware::Dir, - // TODO this should probably just be rbxm - "Model" => Middleware::Rbxmx, - "Script" => { - if inst.children().len() == 0 { - Middleware::ServerScript - } else { - Middleware::ServerScriptDir - } - } - "LocalScript" => { - if inst.children().len() == 0 { - Middleware::ClientScript - } else { - Middleware::ClientScriptDir - } - } - "ModuleScript" => { - if inst.children().len() == 0 { - Middleware::ModuleScript - } else { - Middleware::ModuleScriptDir - } - } - _ => Middleware::Rbxmx, - } -} - -fn syncback_script<'new, 'old>( - script_type: ScriptType, - snapshot: &SyncbackSnapshot<'new, 'old>, -) -> SyncbackReturn<'new, 'old> { - if !is_valid_file_name(&snapshot.name) { - panic!("cannot create a file with name {}", snapshot.name); - } - - let inst = snapshot.new_inst(); - - let mut path = snapshot.parent_path.join(&snapshot.name); - path.set_extension(match script_type { - ScriptType::Module => "lua", - ScriptType::Client => "client.lua", - ScriptType::Server => "server.lua", - }); - let contents = if let Some(Variant::String(source)) = inst.properties.get("Source") { - source.as_bytes().to_vec() - } else { - panic!("Source should be a string") - }; - - SyncbackReturn { - inst_snapshot: InstanceSnapshot::from_instance(inst).metadata(InstanceMetadata::new()), - fs_snapshot: FsSnapshot::new().with_file(path, contents), - // Scripts don't have a child! - children: Vec::new(), - removed_children: Vec::new(), - } -} - -fn syncback_script_dir<'new, 'old>( - script_type: ScriptType, - snapshot: &SyncbackSnapshot<'new, 'old>, -) -> SyncbackReturn<'new, 'old> { - if !is_valid_file_name(&snapshot.name) { - panic!("cannot create a directory with name {}", snapshot.name); - } - - let mut path = snapshot.parent_path.join(&snapshot.name); - path.push("init"); - path.set_extension(match script_type { - ScriptType::Module => "lua", - ScriptType::Client => "client.lua", - ScriptType::Server => "server.lua", - }); - let contents = - if let Some(Variant::String(source)) = snapshot.new_inst().properties.get("Source") { - source.as_bytes().to_vec() - } else { - panic!("Source should be a string") - }; - - let dir_syncback = syncback_dir(snapshot); - - let mut fs_snapshot = FsSnapshot::new(); - fs_snapshot.push_file(path, contents); - fs_snapshot.merge(dir_syncback.fs_snapshot); - - SyncbackReturn { - inst_snapshot: InstanceSnapshot::from_instance(snapshot.new_inst()), - fs_snapshot, - children: dir_syncback.children, - removed_children: dir_syncback.removed_children, - } -} - -fn syncback_dir<'new, 'old>(snapshot: &SyncbackSnapshot<'new, 'old>) -> SyncbackReturn<'new, 'old> { - if !is_valid_file_name(&snapshot.name) { - panic!("cannot create a directory with name {}", snapshot.name); - } - - let path = snapshot.parent_path.join(snapshot.name.clone()); - - let mut removed_children = Vec::new(); - let mut children = Vec::new(); - - if let Some(old_inst) = snapshot.old_inst() { - let old_children: HashMap<&str, Ref> = old_inst - .children() - .iter() - .map(|old_ref| { - ( - snapshot.get_old_instance(*old_ref).unwrap().name(), - *old_ref, - ) - }) - .collect(); - let new_children: HashSet<&str> = snapshot - .new_inst() - .children() - .iter() - .map(|new_ref| snapshot.get_new_instance(*new_ref).unwrap().name.as_str()) - .collect(); - - for child_ref in old_inst.children() { - let old_child = snapshot.get_old_instance(*child_ref).unwrap(); - // If it exists in the old tree but not the new one, it was removed. - if !new_children.contains(old_child.name()) { - removed_children.push(old_child); - } - } - - for child_ref in snapshot.new_inst().children() { - let new_child = snapshot.get_new_instance(*child_ref).unwrap(); - // If it exists in the new tree but not the old one, it was added. - match old_children.get(new_child.name.as_str()) { - None => { - children.push(snapshot.from_parent(new_child.name.clone(), *child_ref, None)) - } - Some(old_ref) => children.push(snapshot.from_parent( - new_child.name.clone(), - *child_ref, - Some(*old_ref), - )), - } - } - } else { - for child_ref in snapshot.new_inst().children() { - let child = snapshot.get_new_instance(*child_ref).unwrap(); - children.push(snapshot.from_parent(child.name.clone(), *child_ref, None)) - } - } - let mut fs_snapshot = FsSnapshot::new().with_dir(&path); - // TODO metadata, including classname - - SyncbackReturn { - inst_snapshot: InstanceSnapshot::from_instance(snapshot.new_inst()), - fs_snapshot, - children, - removed_children, - } -} - -fn syncback_project<'new, 'old>( - snapshot: &SyncbackSnapshot<'new, 'old>, -) -> SyncbackReturn<'new, 'old> { - let old_inst = snapshot - .old_inst() - .expect("project middleware shouldn't be used to make new files"); - // This can never be None. - let source = old_inst.metadata().instigating_source.as_ref().unwrap(); - - // We need to build a 'new' project and serialize it using an FsSnapshot. - // It's convenient to start with the old one though, since it means we have - // a thing to iterate through. - let mut project = - Project::load_from_slice(&snapshot.vfs().read(source.path()).unwrap(), source.path()) - .unwrap(); - let base_path = source.path().parent().unwrap(); - - let mut children = Vec::new(); - let mut removed_children = Vec::new(); - - // Projects are special. We won't be adding or removing things from them, - // so we'll simply match Instances on a per-node basis and rebuild the tree - // with the new instance's data. This matching will be done by class and name - // to simplify things. - let mut nodes = vec![(&mut project.tree, snapshot.new_inst(), old_inst)]; - - // A map of referents from the new tree to the Path that created it, - // if it exists. This is a roundabout way to locate the parents of - // Instances. - let mut ref_to_node = HashMap::new(); - - while let Some((node, new_inst, old_inst)) = nodes.pop() { - ref_to_node.insert(new_inst.referent(), node.path.as_ref()); - - let mut old_child_map = HashMap::with_capacity(old_inst.children().len()); - for child_ref in old_inst.children() { - let child = snapshot.get_old_instance(*child_ref).unwrap(); - old_child_map.insert(child.name(), child); - } - let mut new_child_map = HashMap::with_capacity(new_inst.children().len()); - for child_ref in new_inst.children() { - let child = snapshot.get_new_instance(*child_ref).unwrap(); - new_child_map.insert(child.name.as_str(), child); - } - - for (child_name, child_node) in &mut node.children { - if let Some(new_child) = new_child_map.get(child_name.as_str()) { - if let Some(old_child) = old_child_map.get(child_name.as_str()) { - // TODO verify class names - for (name, value) in &new_child.properties { - if child_node.properties.contains_key(name) { - child_node - .properties - .insert(name.clone(), UnresolvedValue::from(value.clone())); - } - } - nodes.push((child_node, new_child, *old_child)); - new_child_map.remove(child_name.as_str()); - old_child_map.remove(child_name.as_str()); - } else { - log::error!( - "Node {} was in new tree but not old. How did we get here?", - child_name - ); - } - } else { - panic!("Cannot add or remove children from a project") - } - } - - // From this point, both maps contain only children of the current - // instance that aren't in the project. So, we just do some quick and - // dirty matching to identify children that were added and removed. - for (new_name, new_child) in new_child_map { - let parent_path = match ref_to_node.get(&new_child.parent()) { - Some(Some(path)) => base_path.join(path.path()), - Some(None) => { - log::debug!("{new_name} was visited but has no path"); - continue; - } - None => { - log::debug!("{new_name} does not currently exist on FS"); - continue; - } - }; - if let Some(old_inst) = old_child_map.get(new_name) { - // All children are descendants of a node of a project - // So we really just need to track which one is which. - children.push(SyncbackSnapshot { - data: snapshot.data.clone(), - old: Some(old_inst.id()), - new: new_child.referent(), - name: new_name.to_string(), - parent_path, - }); - old_child_map.remove(new_name); - } else { - // it's new - children.push(SyncbackSnapshot { - data: snapshot.data.clone(), - old: None, - new: new_child.referent(), - name: new_name.to_string(), - parent_path, - }); - } - } - removed_children.extend(old_child_map.into_values()); - } - - // We don't need to validate any file names for the FsSnapshot - // because projects can't ever be made by syncback, so they must - // already exist on the file system - SyncbackReturn { - inst_snapshot: InstanceSnapshot::from_instance(snapshot.new_inst()), - fs_snapshot: FsSnapshot::new().with_file( - &project.file_location, - serde_json::to_vec_pretty(&project).unwrap(), - ), - children, - removed_children, - } -} - -pub fn syncback_rbxmx<'new, 'old>( - snapshot: &SyncbackSnapshot<'new, 'old>, -) -> SyncbackReturn<'new, 'old> { - // If any of the children of this Instance are scripts, we don't want - // include them in the model. So instead, we'll check and then serialize. - if !is_valid_file_name(&snapshot.name) { - panic!("cannot create a file with name {}", snapshot.name); - } - - let inst = snapshot.new_inst(); - let mut path = snapshot.parent_path.join(&snapshot.name); - path.set_extension("rbxmx"); - // Long-term, anyway. Right now we stay silly. - let mut serialized = Vec::new(); - rbx_xml::to_writer_default(&mut serialized, snapshot.new_tree(), &[inst.referent()]).unwrap(); - SyncbackReturn { - inst_snapshot: InstanceSnapshot::from_instance(inst), - fs_snapshot: FsSnapshot::new().with_file(&path, serialized), - children: Vec::new(), - removed_children: Vec::new(), - } -} - -fn syncback_text<'new, 'old>( - snapshot: &SyncbackSnapshot<'new, 'old>, -) -> SyncbackReturn<'new, 'old> { - if !is_valid_file_name(&snapshot.name) { - panic!("cannot create a file with name {}", snapshot.name); - } - - let inst = snapshot.new_inst(); - let mut path = snapshot.parent_path.join(&snapshot.name); - path.set_extension("txt"); - - let contents = if let Some(Variant::String(source)) = inst.properties.get("Value") { - source.as_bytes().to_vec() - } else { - panic!("Value should be a string") - }; - - SyncbackReturn { - inst_snapshot: InstanceSnapshot::from_instance(inst).metadata(InstanceMetadata::new()), - fs_snapshot: FsSnapshot::new().with_file(path, contents), - children: Vec::new(), - removed_children: Vec::new(), - } -} - -fn syncback_json_model<'new, 'old>( - snapshot: &SyncbackSnapshot<'new, 'old>, -) -> SyncbackReturn<'new, 'old> { - if !is_valid_file_name(&snapshot.name) { - panic!("cannot create a file with name {}", snapshot.name); - } - - let mut path = snapshot.parent_path.join(&snapshot.name); - path.set_extension("model.json"); - - let new_inst = snapshot.new_inst(); - let mut model = JsonModel::new(&new_inst.name, &new_inst.class); - let mut properties = HashMap::with_capacity(new_inst.properties.len()); - - // TODO handle attributes separately - if let Some(old_inst) = snapshot.old_inst() { - for (name, value) in &new_inst.properties { - if old_inst.properties().contains_key(name) { - properties.insert(name.clone(), UnresolvedValue::from(value.clone())); - } - } - } else { - for (name, value) in new_inst.properties.clone() { - properties.insert(name, UnresolvedValue::from(value)); - } - } - model.set_properties(properties); - - // TODO children - - SyncbackReturn { - inst_snapshot: InstanceSnapshot::from_instance(new_inst), - fs_snapshot: FsSnapshot::new().with_file(&path, serde_json::to_vec_pretty(&model).unwrap()), - children: Vec::new(), - removed_children: Vec::new(), - } -} diff --git a/src/syncback/mod.rs b/src/syncback/mod.rs index 30eb023e4..c1860fa0e 100644 --- a/src/syncback/mod.rs +++ b/src/syncback/mod.rs @@ -1,24 +1,23 @@ mod fs_snapshot; -mod middleware; mod snapshot; use crate::{ - snapshot::{hash_tree, InstanceSnapshot, RojoTree}, + snapshot::{hash_tree, InstanceSnapshot, InstanceWithMeta, RojoTree}, + snapshot_middleware::Middleware, Project, }; use memofs::Vfs; -use rbx_dom_weak::{types::Ref, WeakDom}; +use rbx_dom_weak::{types::Ref, Instance, WeakDom}; use std::{collections::VecDeque, rc::Rc}; pub use fs_snapshot::FsSnapshot; -pub use middleware::{get_best_middleware, syncback_middleware}; pub use snapshot::{SyncbackData, SyncbackSnapshot}; pub fn syncback_loop<'old, 'new>( vfs: &'old Vfs, old_tree: &'old RojoTree, new_tree: &'new WeakDom, - project: &Project, + project: &'old Project, ) -> anyhow::Result> { log::debug!("Hashing project DOM"); let old_hashes = hash_tree(old_tree.inner()); @@ -67,7 +66,7 @@ pub fn syncback_loop<'old, 'new>( middleware ); - let syncback = syncback_middleware(&snapshot, middleware); + let syncback = middleware.syncback(&snapshot)?; if let Some(old_inst) = snapshot.old_inst() { replacements.push((old_inst.parent(), syncback.inst_snapshot)); @@ -83,6 +82,43 @@ pub fn syncback_loop<'old, 'new>( Ok(replacements) } +pub struct SyncbackReturn<'new, 'old> { + pub inst_snapshot: InstanceSnapshot, + pub fs_snapshot: FsSnapshot, + pub children: Vec>, + pub removed_children: Vec>, +} + +pub fn get_best_middleware(inst: &Instance) -> Middleware { + match inst.class.as_str() { + "Folder" => Middleware::Dir, + // TODO this should probably just be rbxm + "Model" => Middleware::Rbxmx, + "Script" => { + if inst.children().len() == 0 { + Middleware::ServerScript + } else { + Middleware::ServerScriptDir + } + } + "LocalScript" => { + if inst.children().len() == 0 { + Middleware::ClientScript + } else { + Middleware::ClientScriptDir + } + } + "ModuleScript" => { + if inst.children().len() == 0 { + Middleware::ModuleScript + } else { + Middleware::ModuleScriptDir + } + } + _ => Middleware::Rbxmx, + } +} + fn get_inst_path(dom: &WeakDom, referent: Ref) -> String { let mut path: VecDeque<&str> = VecDeque::new(); let mut inst = dom.get_by_ref(referent); diff --git a/src/syncback/snapshot.rs b/src/syncback/snapshot.rs index a4cf86098..8590f1511 100644 --- a/src/syncback/snapshot.rs +++ b/src/syncback/snapshot.rs @@ -1,8 +1,5 @@ use memofs::Vfs; -use std::{ - path::{Path, PathBuf}, - rc::Rc, -}; +use std::{path::PathBuf, rc::Rc}; use crate::snapshot::{InstanceWithMeta, RojoTree}; use rbx_dom_weak::{types::Ref, Instance, WeakDom}; @@ -14,11 +11,11 @@ pub struct SyncbackData<'new, 'old> { } pub struct SyncbackSnapshot<'new, 'old> { - pub(super) data: Rc>, - pub(super) old: Option, - pub(super) new: Ref, - pub(super) parent_path: PathBuf, - pub(super) name: String, + pub data: Rc>, + pub old: Option, + pub new: Ref, + pub parent_path: PathBuf, + pub name: String, } impl<'new, 'old> SyncbackSnapshot<'new, 'old> { @@ -71,7 +68,7 @@ impl<'new, 'old> SyncbackSnapshot<'new, 'old> { self.data.vfs } - /// Returns the WeakDom used for the 'new' tree + /// Returns the WeakDom used for the 'new' tree. #[inline] pub fn new_tree(&self) -> &'new WeakDom { self.data.new_tree From 17f0b953e99ed854db68310643cf6b7d6c3fbae5 Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 19 Dec 2023 15:15:09 -0800 Subject: [PATCH 049/366] Use binary models instead of XML --- src/snapshot_middleware/mod.rs | 3 ++- src/snapshot_middleware/rbxm.rs | 30 +++++++++++++++++++++++++++++- src/syncback/mod.rs | 4 ++-- 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/src/snapshot_middleware/mod.rs b/src/snapshot_middleware/mod.rs index 34b1170e3..b9535c544 100644 --- a/src/snapshot_middleware/mod.rs +++ b/src/snapshot_middleware/mod.rs @@ -41,7 +41,7 @@ use self::{ json_model::{snapshot_json_model, syncback_json_model}, lua::{snapshot_lua, snapshot_lua_init, syncback_lua, syncback_lua_init}, project::{snapshot_project, syncback_project}, - rbxm::snapshot_rbxm, + rbxm::{snapshot_rbxm, syncback_rbxm}, rbxmx::{snapshot_rbxmx, syncback_rbxmx}, toml::snapshot_toml, txt::{snapshot_txt, syncback_txt}, @@ -250,6 +250,7 @@ impl Middleware { Middleware::ClientScript => syncback_lua(ScriptType::Client, snapshot), Middleware::ModuleScript => syncback_lua(ScriptType::Module, snapshot), Middleware::Rbxmx => syncback_rbxmx(snapshot), + Middleware::Rbxm => syncback_rbxm(snapshot), Middleware::Text => syncback_txt(snapshot), Middleware::JsonModel => syncback_json_model(snapshot), Middleware::Dir => syncback_dir(snapshot), diff --git a/src/snapshot_middleware/rbxm.rs b/src/snapshot_middleware/rbxm.rs index 7983d0e71..31a22cee1 100644 --- a/src/snapshot_middleware/rbxm.rs +++ b/src/snapshot_middleware/rbxm.rs @@ -3,7 +3,10 @@ use std::path::Path; use anyhow::Context; use memofs::Vfs; -use crate::snapshot::{InstanceContext, InstanceMetadata, InstanceSnapshot}; +use crate::{ + snapshot::{InstanceContext, InstanceMetadata, InstanceSnapshot}, + syncback::{is_valid_file_name, FsSnapshot, SyncbackReturn, SyncbackSnapshot}, +}; #[profiling::function] pub fn snapshot_rbxm( @@ -39,6 +42,31 @@ pub fn snapshot_rbxm( } } +pub fn syncback_rbxm<'new, 'old>( + snapshot: &SyncbackSnapshot<'new, 'old>, +) -> anyhow::Result> { + // If any of the children of this Instance are scripts, we don't want + // include them in the model. So instead, we'll check and then serialize. + if !is_valid_file_name(&snapshot.name) { + anyhow::bail!("cannot create a file with name {}", snapshot.name); + } + + let inst = snapshot.new_inst(); + let mut path = snapshot.parent_path.join(&snapshot.name); + path.set_extension("rbxm"); + // Long-term, anyway. Right now we stay silly. + let mut serialized = Vec::new(); + rbx_binary::to_writer(&mut serialized, snapshot.new_tree(), &[inst.referent()]) + .context("failed to serialize new rbxm")?; + + Ok(SyncbackReturn { + inst_snapshot: InstanceSnapshot::from_instance(inst), + fs_snapshot: FsSnapshot::new().with_file(&path, serialized), + children: Vec::new(), + removed_children: Vec::new(), + }) +} + #[cfg(test)] mod test { use super::*; diff --git a/src/syncback/mod.rs b/src/syncback/mod.rs index c1860fa0e..67543ceef 100644 --- a/src/syncback/mod.rs +++ b/src/syncback/mod.rs @@ -93,7 +93,7 @@ pub fn get_best_middleware(inst: &Instance) -> Middleware { match inst.class.as_str() { "Folder" => Middleware::Dir, // TODO this should probably just be rbxm - "Model" => Middleware::Rbxmx, + "Model" => Middleware::Rbxm, "Script" => { if inst.children().len() == 0 { Middleware::ServerScript @@ -115,7 +115,7 @@ pub fn get_best_middleware(inst: &Instance) -> Middleware { Middleware::ModuleScriptDir } } - _ => Middleware::Rbxmx, + _ => Middleware::Rbxm, } } From bfd59c5f76b534429ffaf508669f7db6d43ddd9f Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 19 Dec 2023 15:15:31 -0800 Subject: [PATCH 050/366] Emit how many files were written as info instead of debug --- src/syncback/fs_snapshot.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/syncback/fs_snapshot.rs b/src/syncback/fs_snapshot.rs index dab1c6837..d9d67afe5 100644 --- a/src/syncback/fs_snapshot.rs +++ b/src/syncback/fs_snapshot.rs @@ -62,7 +62,7 @@ impl FsSnapshot { files += 1; } - log::debug!("Wrote {dirs} directories and {files} files to the VFS"); + log::info!("Wrote {dirs} directories and {files} files to the file system!"); Ok(()) } } From 044e5144ea1bb69a07877e7e2d3bc48beace48c3 Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 19 Dec 2023 15:18:44 -0800 Subject: [PATCH 051/366] Add timers to syncback command --- src/cli/syncback.rs | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/cli/syncback.rs b/src/cli/syncback.rs index f6ac43b35..f3f943b69 100644 --- a/src/cli/syncback.rs +++ b/src/cli/syncback.rs @@ -1,7 +1,5 @@ use std::{ - collections::HashMap, fs, - hash::Hash, path::{Path, PathBuf}, time::Instant, }; @@ -32,22 +30,33 @@ impl SyncbackCommand { let path_old = resolve_path(&self.project); let path_new = resolve_path(&self.input); + let project_start = Instant::now(); log::info!("Opening project at {}", path_old.display()); let session_old = ServeSession::new(Vfs::new_default(), path_old.clone())?; + log::info!( + "Finished opening project in {:0.02}s", + project_start.elapsed().as_secs_f32() + ); let dom_old = session_old.tree(); + + let dom_start = Instant::now(); log::info!("Reading place file at {}", path_new.display()); let dom_new = read_dom(&path_new); + log::info!( + "Finished opening file in {:0.02}s", + dom_start.elapsed().as_secs_f32() + ); let start = Instant::now(); - log::debug!("Beginning syncback..."); + log::info!("Beginning syncback..."); syncback_loop( session_old.vfs(), &dom_old, &dom_new, session_old.root_project(), )?; - log::debug!( + log::info!( "Syncback finished in {:.02}s!", start.elapsed().as_secs_f32() ); @@ -72,8 +81,3 @@ fn read_dom(path: &Path) -> WeakDom { panic!("invalid Roblox file at {}", path.display()) } } - -#[inline] -fn invert_map(map: HashMap) -> HashMap { - map.into_iter().map(|(key, value)| (value, key)).collect() -} From 029a6f6ce38fd6addff953c7ff874ebbd23e49b5 Mon Sep 17 00:00:00 2001 From: Micah Date: Wed, 20 Dec 2023 11:36:17 -0800 Subject: [PATCH 052/366] Add `create_dir_all` to memofs --- crates/memofs/CHANGELOG.md | 2 +- crates/memofs/src/in_memory_fs.rs | 10 ++++++++++ crates/memofs/src/lib.rs | 18 ++++++++++++++++++ crates/memofs/src/noop_backend.rs | 7 +++++++ crates/memofs/src/std_backend.rs | 4 ++++ 5 files changed, 40 insertions(+), 1 deletion(-) diff --git a/crates/memofs/CHANGELOG.md b/crates/memofs/CHANGELOG.md index c5ebd95ab..8a1d580ce 100644 --- a/crates/memofs/CHANGELOG.md +++ b/crates/memofs/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased Changes * Changed `StdBackend` file watching component to use minimal recursive watches. [#830] -* Added `create_dir` to allow creating directories. +* Added `create_dir` and `create_dir_all` to allow creating directories. [#830]: https://github.com/rojo-rbx/rojo/pull/830 diff --git a/crates/memofs/src/in_memory_fs.rs b/crates/memofs/src/in_memory_fs.rs index 217899dc4..2eb09b8a1 100644 --- a/crates/memofs/src/in_memory_fs.rs +++ b/crates/memofs/src/in_memory_fs.rs @@ -181,6 +181,16 @@ impl VfsBackend for InMemoryFs { inner.load_snapshot(path.to_path_buf(), VfsSnapshot::empty_dir()) } + fn create_dir_all(&mut self, path: &Path) -> io::Result<()> { + let mut inner = self.inner.lock().unwrap(); + let mut path_buf = path.to_path_buf(); + while let Some(parent) = path_buf.parent() { + inner.load_snapshot(parent.to_path_buf(), VfsSnapshot::empty_dir())?; + path_buf.pop(); + } + inner.load_snapshot(path.to_path_buf(), VfsSnapshot::empty_dir()) + } + fn remove_file(&mut self, path: &Path) -> io::Result<()> { let mut inner = self.inner.lock().unwrap(); diff --git a/crates/memofs/src/lib.rs b/crates/memofs/src/lib.rs index 1ead8d19f..1dfa41b11 100644 --- a/crates/memofs/src/lib.rs +++ b/crates/memofs/src/lib.rs @@ -72,6 +72,7 @@ pub trait VfsBackend: sealed::Sealed + Send + 'static { fn write(&mut self, path: &Path, data: &[u8]) -> io::Result<()>; fn read_dir(&mut self, path: &Path) -> io::Result; fn create_dir(&mut self, path: &Path) -> io::Result<()>; + fn create_dir_all(&mut self, path: &Path) -> io::Result<()>; fn metadata(&mut self, path: &Path) -> io::Result; fn remove_file(&mut self, path: &Path) -> io::Result<()>; fn remove_dir_all(&mut self, path: &Path) -> io::Result<()>; @@ -178,6 +179,11 @@ impl VfsInner { self.backend.create_dir(path) } + fn create_dir_all>(&mut self, path: P) -> io::Result<()> { + let path = path.as_ref(); + self.backend.create_dir_all(path) + } + fn remove_file>(&mut self, path: P) -> io::Result<()> { let path = path.as_ref(); let _ = self.backend.unwatch(path); @@ -300,6 +306,18 @@ impl Vfs { self.inner.lock().unwrap().create_dir(path) } + /// Creates a directory at the provided location, recursively creating + /// all parent components if they are missing. + /// + /// Roughly equivalent to [`std::fs::create_dir_all`][std::fs::create_dir_all]. + /// + /// [std::fs::create_dir_all]: https://doc.rust-lang.org/stable/std/fs/fn.create_dir_all.html + #[inline] + pub fn create_dir_all>(&self, path: P) -> io::Result<()> { + let path = path.as_ref(); + self.inner.lock().unwrap().create_dir_all(path) + } + /// Remove a file. /// /// Roughly equivalent to [`std::fs::remove_file`][std::fs::remove_file]. diff --git a/crates/memofs/src/noop_backend.rs b/crates/memofs/src/noop_backend.rs index 5a028d4be..8e5a24177 100644 --- a/crates/memofs/src/noop_backend.rs +++ b/crates/memofs/src/noop_backend.rs @@ -42,6 +42,13 @@ impl VfsBackend for NoopBackend { )) } + fn create_dir_all(&mut self, _path: &Path) -> io::Result<()> { + Err(io::Error::new( + io::ErrorKind::Other, + "NoopBackend doesn't do anything", + )) + } + fn remove_file(&mut self, _path: &Path) -> io::Result<()> { Err(io::Error::new( io::ErrorKind::Other, diff --git a/crates/memofs/src/std_backend.rs b/crates/memofs/src/std_backend.rs index 359d023ae..fc2e1d208 100644 --- a/crates/memofs/src/std_backend.rs +++ b/crates/memofs/src/std_backend.rs @@ -82,6 +82,10 @@ impl VfsBackend for StdBackend { fs_err::create_dir(path) } + fn create_dir_all(&mut self, path: &Path) -> io::Result<()> { + fs_err::create_dir_all(path) + } + fn remove_file(&mut self, path: &Path) -> io::Result<()> { fs_err::remove_file(path) } From 9ae87dec8e5362b11c0fd82adc5795498230bb5d Mon Sep 17 00:00:00 2001 From: Micah Date: Wed, 20 Dec 2023 11:36:30 -0800 Subject: [PATCH 053/366] Switch to create_dir_all in fs_snapshot --- src/syncback/fs_snapshot.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/syncback/fs_snapshot.rs b/src/syncback/fs_snapshot.rs index d9d67afe5..a83dbe1c1 100644 --- a/src/syncback/fs_snapshot.rs +++ b/src/syncback/fs_snapshot.rs @@ -50,7 +50,7 @@ impl FsSnapshot { let mut dirs = 0; let mut files = 0; for dir_path in &self.dir { - match vfs.create_dir(base_path.join(dir_path)) { + match vfs.create_dir_all(base_path.join(dir_path)) { Ok(_) => (), Err(err) if err.kind() == io::ErrorKind::AlreadyExists => (), Err(err) => return Err(err), From dcccd76efeb6e6e701b0e8e24095e99fdf58e61f Mon Sep 17 00:00:00 2001 From: Micah Date: Wed, 20 Dec 2023 12:06:57 -0800 Subject: [PATCH 054/366] Support syncback filters --- src/project.rs | 6 +++- src/syncback/mod.rs | 77 +++++++++++++++++++++++++++++++++++++++------ 2 files changed, 72 insertions(+), 11 deletions(-) diff --git a/src/project.rs b/src/project.rs index 1095fba20..768ee045b 100644 --- a/src/project.rs +++ b/src/project.rs @@ -8,7 +8,9 @@ use std::{ use serde::{Deserialize, Serialize}; use thiserror::Error; -use crate::{glob::Glob, resolution::UnresolvedValue, snapshot::SyncRule}; +use crate::{ + glob::Glob, resolution::UnresolvedValue, snapshot::SyncRule, syncback::SyncbackIgnoreRules, +}; static PROJECT_FILENAME: &str = "default.project.json"; @@ -84,6 +86,8 @@ pub struct Project { #[serde(default, skip_serializing_if = "Vec::is_empty")] pub glob_ignore_paths: Vec, + pub syncback_rules: Option, + /// A list of mappings of globs to syncing rules. If a file matches a glob, /// it will be 'transformed' into an Instance following the rule provided. /// Globs are relative to the folder the project file is in. diff --git a/src/syncback/mod.rs b/src/syncback/mod.rs index 67543ceef..786746bac 100644 --- a/src/syncback/mod.rs +++ b/src/syncback/mod.rs @@ -1,14 +1,23 @@ mod fs_snapshot; mod snapshot; +use memofs::Vfs; +use rbx_dom_weak::{ + types::{Ref, Variant}, + Instance, WeakDom, +}; +use serde::{Deserialize, Serialize}; +use std::{ + collections::{HashMap, VecDeque}, + rc::Rc, +}; + use crate::{ + resolution::UnresolvedValue, snapshot::{hash_tree, InstanceSnapshot, InstanceWithMeta, RojoTree}, snapshot_middleware::Middleware, Project, }; -use memofs::Vfs; -use rbx_dom_weak::{types::Ref, Instance, WeakDom}; -use std::{collections::VecDeque, rc::Rc}; pub use fs_snapshot::FsSnapshot; pub use snapshot::{SyncbackData, SyncbackSnapshot}; @@ -55,16 +64,20 @@ pub fn syncback_loop<'old, 'new>( } } + if let Some(syncback_rules) = &project.syncback_rules { + if !syncback_rules.acceptable(new_tree, snapshot.new) { + log::error!( + "Path {} is blocked by project", + get_inst_path(new_tree, snapshot.new) + ); + continue; + } + } + let middleware = snapshot .old_inst() .and_then(|inst| inst.metadata().middleware) .unwrap_or_else(|| get_best_middleware(snapshot.new_inst())); - log::debug!( - "instance {} parent is {} (using middleware {:?})", - get_inst_path(new_tree, snapshot.new), - snapshot.parent_path.display(), - middleware - ); let syncback = middleware.syncback(&snapshot)?; @@ -119,6 +132,50 @@ pub fn get_best_middleware(inst: &Instance) -> Middleware { } } +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SyncbackIgnoreRules { + #[serde(default)] + paths: Vec, + #[serde(default, skip)] + classes: HashMap>, +} + +impl SyncbackIgnoreRules { + /// If possible, resolves all of the properties in the ignore rules so that + /// they're Variants. + pub fn resolve(&self) -> anyhow::Result>> { + let mut resolved = HashMap::with_capacity(self.classes.capacity()); + + for (class_name, properties) in &self.classes { + let mut resolved_props = HashMap::with_capacity(properties.capacity()); + for (prop_name, prop_value) in properties { + resolved_props.insert( + prop_name.as_str(), + prop_value.clone().resolve(class_name, prop_name)?, + ); + } + + resolved.insert(class_name.as_str(), resolved_props); + } + + Ok(resolved) + } + + /// Returns whether the provided Instance is allowed to be handled with + /// syncback. + #[inline] + pub fn acceptable(&self, dom: &WeakDom, inst: Ref) -> bool { + let path = get_inst_path(dom, inst); + for ignored in &self.paths { + if path.starts_with(ignored.as_str()) { + return false; + } + } + true + } +} + fn get_inst_path(dom: &WeakDom, referent: Ref) -> String { let mut path: VecDeque<&str> = VecDeque::new(); let mut inst = dom.get_by_ref(referent); @@ -126,7 +183,7 @@ fn get_inst_path(dom: &WeakDom, referent: Ref) -> String { path.push_front(&instance.name); inst = dom.get_by_ref(instance.parent()); } - path.into_iter().collect::>().join(".") + path.into_iter().collect::>().join("/") } /// A list of file names that are not valid on Windows. From 563decaca0d91d975a61d6a1d8fa6916337a4c76 Mon Sep 17 00:00:00 2001 From: Micah Date: Wed, 20 Dec 2023 12:09:38 -0800 Subject: [PATCH 055/366] Change logging a bit --- src/syncback/mod.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/syncback/mod.rs b/src/syncback/mod.rs index 786746bac..67d623040 100644 --- a/src/syncback/mod.rs +++ b/src/syncback/mod.rs @@ -66,7 +66,7 @@ pub fn syncback_loop<'old, 'new>( if let Some(syncback_rules) = &project.syncback_rules { if !syncback_rules.acceptable(new_tree, snapshot.new) { - log::error!( + log::debug!( "Path {} is blocked by project", get_inst_path(new_tree, snapshot.new) ); @@ -78,6 +78,11 @@ pub fn syncback_loop<'old, 'new>( .old_inst() .and_then(|inst| inst.metadata().middleware) .unwrap_or_else(|| get_best_middleware(snapshot.new_inst())); + log::trace!( + "Middleware for {} is {:?}", + get_inst_path(new_tree, snapshot.new), + middleware + ); let syncback = middleware.syncback(&snapshot)?; From f63d60d332aada6978d28f7958bccc7f7732ed4c Mon Sep 17 00:00:00 2001 From: Micah Date: Wed, 20 Dec 2023 12:54:18 -0800 Subject: [PATCH 056/366] Abstract variant equality function to its own module Rename diff module to hash --- src/lib.rs | 1 + src/snapshot/diff/mod.rs | 26 --- src/snapshot/{diff/hash.rs => hash/mod.rs} | 26 ++- src/snapshot/hash/variant.rs | 194 +++++++++++++++++ src/snapshot/mod.rs | 4 +- .../diff/variant.rs => variant_eq.rs} | 199 +----------------- 6 files changed, 223 insertions(+), 227 deletions(-) delete mode 100644 src/snapshot/diff/mod.rs rename src/snapshot/{diff/hash.rs => hash/mod.rs} (77%) create mode 100644 src/snapshot/hash/variant.rs rename src/{snapshot/diff/variant.rs => variant_eq.rs} (55%) diff --git a/src/lib.rs b/src/lib.rs index 63f25d177..5d4df344a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -21,6 +21,7 @@ mod session_id; mod snapshot; mod snapshot_middleware; mod syncback; +mod variant_eq; mod web; pub use project::*; diff --git a/src/snapshot/diff/mod.rs b/src/snapshot/diff/mod.rs deleted file mode 100644 index 9c08d23bf..000000000 --- a/src/snapshot/diff/mod.rs +++ /dev/null @@ -1,26 +0,0 @@ -mod hash; -mod variant; - -pub use hash::*; -pub use variant::*; - -use rbx_dom_weak::{types::Ref, WeakDom}; -use std::collections::VecDeque; - -pub(crate) fn descendants(dom: &WeakDom) -> Vec { - let mut queue = VecDeque::new(); - let mut ordered = Vec::new(); - queue.push_front(dom.root_ref()); - - while let Some(referent) = queue.pop_front() { - let inst = dom - .get_by_ref(referent) - .expect("Invariant: WeakDom had a Ref that wasn't inside it"); - ordered.push(referent); - for child in inst.children() { - queue.push_back(*child) - } - } - - ordered -} diff --git a/src/snapshot/diff/hash.rs b/src/snapshot/hash/mod.rs similarity index 77% rename from src/snapshot/diff/hash.rs rename to src/snapshot/hash/mod.rs index baece8ceb..129a3819b 100644 --- a/src/snapshot/diff/hash.rs +++ b/src/snapshot/hash/mod.rs @@ -1,14 +1,16 @@ //! Hashing utility for a RojoTree +mod variant; + +pub use variant::*; use blake3::{Hash, Hasher}; use rbx_dom_weak::{ types::{Ref, Variant}, Instance, WeakDom, }; +use std::collections::{HashMap, VecDeque}; -use std::collections::HashMap; - -use super::{descendants, hash_variant, variant_eq}; +use crate::variant_eq::variant_eq; pub fn hash_tree(dom: &WeakDom) -> HashMap { let mut map: HashMap = HashMap::new(); @@ -76,3 +78,21 @@ pub fn hash_inst<'map, 'inst>( hasher.finalize() } + +pub(crate) fn descendants(dom: &WeakDom) -> Vec { + let mut queue = VecDeque::new(); + let mut ordered = Vec::new(); + queue.push_front(dom.root_ref()); + + while let Some(referent) = queue.pop_front() { + let inst = dom + .get_by_ref(referent) + .expect("Invariant: WeakDom had a Ref that wasn't inside it"); + ordered.push(referent); + for child in inst.children() { + queue.push_back(*child) + } + } + + ordered +} diff --git a/src/snapshot/hash/variant.rs b/src/snapshot/hash/variant.rs new file mode 100644 index 000000000..6c41ee7d5 --- /dev/null +++ b/src/snapshot/hash/variant.rs @@ -0,0 +1,194 @@ +use blake3::Hasher; +use rbx_dom_weak::types::{PhysicalProperties, Variant, Vector3}; + +macro_rules! round { + ($value:expr) => { + (($value * 10.0).round() / 10.0) + }; +} + +macro_rules! n_hash { + ($hash:ident, $($num:expr),*) => { + {$( + $hash.update(&($num).to_le_bytes()); + )*} + }; +} + +macro_rules! hash { + ($hash:ident, $value:expr) => {{ + $hash.update($value); + }}; +} + +/// Places `value` into the provided hasher. +pub fn hash_variant(hasher: &mut Hasher, value: &Variant) { + // We need to round floats, though I'm not sure to what degree we can + // realistically do that. + match value { + Variant::Attributes(attrs) => { + let mut sorted: Vec<(&String, &Variant)> = attrs.iter().collect(); + sorted.sort_unstable_by_key(|(name, _)| *name); + for (name, attribute) in sorted { + hasher.update(name.as_bytes()); + hash_variant(hasher, attribute); + } + } + Variant::Axes(a) => hash!(hasher, &[a.bits()]), + Variant::BinaryString(bytes) => hash!(hasher, bytes.as_ref()), + Variant::Bool(bool) => hash!(hasher, &[*bool as u8]), + Variant::BrickColor(color) => n_hash!(hasher, *color as u16), + Variant::CFrame(cf) => { + vector_hash(hasher, cf.position); + vector_hash(hasher, cf.orientation.x); + vector_hash(hasher, cf.orientation.y); + vector_hash(hasher, cf.orientation.z); + } + Variant::Color3(color) => { + n_hash!(hasher, round!(color.r), round!(color.g), round!(color.b)) + } + Variant::Color3uint8(color) => hash!(hasher, &[color.r, color.b, color.g]), + Variant::ColorSequence(seq) => { + let mut new = Vec::with_capacity(seq.keypoints.len()); + for keypoint in &seq.keypoints { + new.push(keypoint); + } + new.sort_unstable_by(|a, b| round!(a.time).partial_cmp(&round!(b.time)).unwrap()); + for keypoint in new { + n_hash!( + hasher, + round!(keypoint.time), + round!(keypoint.color.r), + round!(keypoint.color.g), + round!(keypoint.color.b) + ) + } + } + Variant::Content(content) => { + let s: &str = content.as_ref(); + hash!(hasher, s.as_bytes()) + } + Variant::Enum(e) => n_hash!(hasher, e.to_u32()), + Variant::Faces(f) => hash!(hasher, &[f.bits()]), + Variant::Float32(n) => n_hash!(hasher, round!(*n)), + Variant::Float64(n) => n_hash!(hasher, round!(n)), + Variant::Font(f) => { + n_hash!(hasher, f.weight as u16); + n_hash!(hasher, f.style as u8); + hash!(hasher, f.family.as_bytes()); + if let Some(cache) = &f.cached_face_id { + hash!(hasher, &[0x01]); + hash!(hasher, cache.as_bytes()); + } else { + hash!(hasher, &[0x00]); + } + } + Variant::Int32(n) => n_hash!(hasher, n), + Variant::Int64(n) => n_hash!(hasher, n), + Variant::MaterialColors(n) => hash!(hasher, n.encode().as_slice()), + Variant::NumberRange(nr) => n_hash!(hasher, round!(nr.max), round!(nr.min)), + Variant::NumberSequence(seq) => { + let mut new = Vec::with_capacity(seq.keypoints.len()); + for keypoint in &seq.keypoints { + new.push(keypoint); + } + new.sort_unstable_by(|a, b| round!(a.time).partial_cmp(&round!(b.time)).unwrap()); + for keypoint in new { + n_hash!( + hasher, + round!(keypoint.time), + round!(keypoint.value), + round!(keypoint.envelope) + ) + } + } + Variant::OptionalCFrame(maybe_cf) => { + if let Some(cf) = maybe_cf { + hash!(hasher, &[0x01]); + vector_hash(hasher, cf.position); + vector_hash(hasher, cf.orientation.x); + vector_hash(hasher, cf.orientation.y); + vector_hash(hasher, cf.orientation.z); + } else { + hash!(hasher, &[0x00]); + } + } + Variant::PhysicalProperties(properties) => match properties { + PhysicalProperties::Default => hash!(hasher, &[0x00]), + PhysicalProperties::Custom(custom) => { + hash!(hasher, &[0x00]); + n_hash!( + hasher, + round!(custom.density), + round!(custom.friction), + round!(custom.elasticity), + round!(custom.friction_weight), + round!(custom.elasticity_weight) + ) + } + }, + Variant::Ray(ray) => { + vector_hash(hasher, ray.origin); + vector_hash(hasher, ray.direction); + } + Variant::Rect(rect) => n_hash!( + hasher, + round!(rect.max.x), + round!(rect.max.y), + round!(rect.min.x), + round!(rect.min.y) + ), + Variant::Ref(referent) => hash!(hasher, referent.to_string().as_bytes()), + Variant::Region3(region) => { + vector_hash(hasher, region.max); + vector_hash(hasher, region.min); + } + Variant::Region3int16(region) => { + n_hash!( + hasher, + region.max.x, + region.max.y, + region.max.z, + region.min.x, + region.min.y, + region.min.z + ) + } + Variant::SecurityCapabilities(capabilities) => n_hash!(hasher, capabilities.bits()), + Variant::SharedString(sstr) => hash!(hasher, sstr.hash().as_bytes()), + Variant::String(str) => hash!(hasher, str.as_bytes()), + Variant::Tags(tags) => { + let mut dupe: Vec<&str> = tags.iter().collect(); + dupe.sort_unstable(); + for tag in dupe { + hash!(hasher, tag.as_bytes()) + } + } + Variant::UDim(udim) => n_hash!(hasher, round!(udim.scale), udim.offset), + Variant::UDim2(udim) => n_hash!( + hasher, + round!(udim.y.scale), + udim.y.offset, + round!(udim.x.scale), + udim.x.offset + ), + Variant::Vector2(v2) => n_hash!(hasher, round!(v2.x), round!(v2.y)), + Variant::Vector2int16(v2) => n_hash!(hasher, v2.x, v2.y), + Variant::Vector3(v3) => vector_hash(hasher, *v3), + Variant::Vector3int16(v3) => n_hash!(hasher, v3.x, v3.y, v3.z), + + // Hashing UniqueId properties doesn't make sense + Variant::UniqueId(_) => (), + + unknown => { + log::warn!( + "Encountered unknown Variant {:?} while hashing", + unknown.ty() + ) + } + } +} + +fn vector_hash(hasher: &mut Hasher, vector: Vector3) { + n_hash!(hasher, round!(vector.x), round!(vector.y), round!(vector.z)) +} diff --git a/src/snapshot/mod.rs b/src/snapshot/mod.rs index 6a36b893b..ff6ddb5ba 100644 --- a/src/snapshot/mod.rs +++ b/src/snapshot/mod.rs @@ -48,7 +48,7 @@ #![allow(dead_code)] -mod diff; +mod hash; mod instance_snapshot; mod metadata; mod patch; @@ -56,7 +56,7 @@ mod patch_apply; mod patch_compute; mod tree; -pub use diff::*; +pub use hash::*; pub use instance_snapshot::InstanceSnapshot; pub use metadata::*; pub use patch::*; diff --git a/src/snapshot/diff/variant.rs b/src/variant_eq.rs similarity index 55% rename from src/snapshot/diff/variant.rs rename to src/variant_eq.rs index 2d1b3827a..831f03929 100644 --- a/src/snapshot/diff/variant.rs +++ b/src/variant_eq.rs @@ -1,202 +1,9 @@ -use blake3::Hasher; -use float_cmp::approx_eq; -use rbx_dom_weak::types::{PhysicalProperties, Variant, Vector3}; - use std::collections::HashMap; -macro_rules! round { - ($value:expr) => { - (($value * 10.0).round() / 10.0) - }; -} - -macro_rules! n_hash { - ($hash:ident, $($num:expr),*) => { - {$( - $hash.update(&($num).to_le_bytes()); - )*} - }; -} - -macro_rules! hash { - ($hash:ident, $value:expr) => {{ - $hash.update($value); - }}; -} - -/// Places `value` into the provided hasher. -pub fn hash_variant(hasher: &mut Hasher, value: &Variant) { - // We need to round floats, though I'm not sure to what degree we can - // realistically do that. - match value { - Variant::Attributes(attrs) => { - let mut sorted: Vec<(&String, &Variant)> = attrs.iter().collect(); - sorted.sort_unstable_by_key(|(name, _)| *name); - for (name, attribute) in sorted { - hasher.update(name.as_bytes()); - hash_variant(hasher, attribute); - } - } - Variant::Axes(a) => hash!(hasher, &[a.bits()]), - Variant::BinaryString(bytes) => hash!(hasher, bytes.as_ref()), - Variant::Bool(bool) => hash!(hasher, &[*bool as u8]), - Variant::BrickColor(color) => n_hash!(hasher, *color as u16), - Variant::CFrame(cf) => { - vector_hash(hasher, cf.position); - vector_hash(hasher, cf.orientation.x); - vector_hash(hasher, cf.orientation.y); - vector_hash(hasher, cf.orientation.z); - } - Variant::Color3(color) => { - n_hash!(hasher, round!(color.r), round!(color.g), round!(color.b)) - } - Variant::Color3uint8(color) => hash!(hasher, &[color.r, color.b, color.g]), - Variant::ColorSequence(seq) => { - let mut new = Vec::with_capacity(seq.keypoints.len()); - for keypoint in &seq.keypoints { - new.push(keypoint); - } - new.sort_unstable_by(|a, b| round!(a.time).partial_cmp(&round!(b.time)).unwrap()); - for keypoint in new { - n_hash!( - hasher, - round!(keypoint.time), - round!(keypoint.color.r), - round!(keypoint.color.g), - round!(keypoint.color.b) - ) - } - } - Variant::Content(content) => { - let s: &str = content.as_ref(); - hash!(hasher, s.as_bytes()) - } - Variant::Enum(e) => n_hash!(hasher, e.to_u32()), - Variant::Faces(f) => hash!(hasher, &[f.bits()]), - Variant::Float32(n) => n_hash!(hasher, round!(*n)), - Variant::Float64(n) => n_hash!(hasher, round!(n)), - Variant::Font(f) => { - n_hash!(hasher, f.weight as u16); - n_hash!(hasher, f.style as u8); - hash!(hasher, f.family.as_bytes()); - if let Some(cache) = &f.cached_face_id { - hash!(hasher, &[0x01]); - hash!(hasher, cache.as_bytes()); - } else { - hash!(hasher, &[0x00]); - } - } - Variant::Int32(n) => n_hash!(hasher, n), - Variant::Int64(n) => n_hash!(hasher, n), - Variant::MaterialColors(n) => hash!(hasher, n.encode().as_slice()), - Variant::NumberRange(nr) => n_hash!(hasher, round!(nr.max), round!(nr.min)), - Variant::NumberSequence(seq) => { - let mut new = Vec::with_capacity(seq.keypoints.len()); - for keypoint in &seq.keypoints { - new.push(keypoint); - } - new.sort_unstable_by(|a, b| round!(a.time).partial_cmp(&round!(b.time)).unwrap()); - for keypoint in new { - n_hash!( - hasher, - round!(keypoint.time), - round!(keypoint.value), - round!(keypoint.envelope) - ) - } - } - Variant::OptionalCFrame(maybe_cf) => { - if let Some(cf) = maybe_cf { - hash!(hasher, &[0x01]); - vector_hash(hasher, cf.position); - vector_hash(hasher, cf.orientation.x); - vector_hash(hasher, cf.orientation.y); - vector_hash(hasher, cf.orientation.z); - } else { - hash!(hasher, &[0x00]); - } - } - Variant::PhysicalProperties(properties) => match properties { - PhysicalProperties::Default => hash!(hasher, &[0x00]), - PhysicalProperties::Custom(custom) => { - hash!(hasher, &[0x00]); - n_hash!( - hasher, - round!(custom.density), - round!(custom.friction), - round!(custom.elasticity), - round!(custom.friction_weight), - round!(custom.elasticity_weight) - ) - } - }, - Variant::Ray(ray) => { - vector_hash(hasher, ray.origin); - vector_hash(hasher, ray.direction); - } - Variant::Rect(rect) => n_hash!( - hasher, - round!(rect.max.x), - round!(rect.max.y), - round!(rect.min.x), - round!(rect.min.y) - ), - Variant::Ref(referent) => hash!(hasher, referent.to_string().as_bytes()), - Variant::Region3(region) => { - vector_hash(hasher, region.max); - vector_hash(hasher, region.min); - } - Variant::Region3int16(region) => { - n_hash!( - hasher, - region.max.x, - region.max.y, - region.max.z, - region.min.x, - region.min.y, - region.min.z - ) - } - Variant::SecurityCapabilities(capabilities) => n_hash!(hasher, capabilities.bits()), - Variant::SharedString(sstr) => hash!(hasher, sstr.hash().as_bytes()), - Variant::String(str) => hash!(hasher, str.as_bytes()), - Variant::Tags(tags) => { - let mut dupe: Vec<&str> = tags.iter().collect(); - dupe.sort_unstable(); - for tag in dupe { - hash!(hasher, tag.as_bytes()) - } - } - Variant::UDim(udim) => n_hash!(hasher, round!(udim.scale), udim.offset), - Variant::UDim2(udim) => n_hash!( - hasher, - round!(udim.y.scale), - udim.y.offset, - round!(udim.x.scale), - udim.x.offset - ), - Variant::Vector2(v2) => n_hash!(hasher, round!(v2.x), round!(v2.y)), - Variant::Vector2int16(v2) => n_hash!(hasher, v2.x, v2.y), - Variant::Vector3(v3) => vector_hash(hasher, *v3), - Variant::Vector3int16(v3) => n_hash!(hasher, v3.x, v3.y, v3.z), - - // Hashing UniqueId properties doesn't make sense - Variant::UniqueId(_) => (), - - unknown => { - log::warn!( - "Encountered unknown Variant {:?} while hashing", - unknown.ty() - ) - } - } -} - -fn vector_hash(hasher: &mut Hasher, vector: Vector3) { - n_hash!(hasher, round!(vector.x), round!(vector.y), round!(vector.z)) -} +use float_cmp::approx_eq; +use rbx_dom_weak::types::{PhysicalProperties, Variant, Vector3}; -/// Compares to variants to determine if they're equal. This correctly takes +/// Compares two variants to determine if they're equal. This correctly takes /// float comparisons into account. pub fn variant_eq(variant_a: &Variant, variant_b: &Variant) -> bool { if variant_a.ty() != variant_b.ty() { From bbc926deeb79373ab4e161086ae88968cd6f33b0 Mon Sep 17 00:00:00 2001 From: Micah Date: Wed, 20 Dec 2023 14:13:07 -0800 Subject: [PATCH 057/366] Filter default and ref properties out of json_models --- src/snapshot_middleware/json_model.rs | 34 ++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/src/snapshot_middleware/json_model.rs b/src/snapshot_middleware/json_model.rs index 229b94318..3d2173b93 100644 --- a/src/snapshot_middleware/json_model.rs +++ b/src/snapshot_middleware/json_model.rs @@ -2,13 +2,14 @@ use std::{borrow::Cow, collections::HashMap, path::Path, str}; use anyhow::Context; use memofs::Vfs; -use rbx_dom_weak::types::{Attributes, Ref}; +use rbx_dom_weak::types::{Attributes, Ref, Variant}; use serde::{Deserialize, Serialize}; use crate::{ resolution::UnresolvedValue, snapshot::{InstanceContext, InstanceSnapshot}, syncback::{is_valid_file_name, FsSnapshot, SyncbackReturn, SyncbackSnapshot}, + variant_eq::variant_eq, }; pub fn snapshot_json_model( @@ -69,16 +70,43 @@ pub fn syncback_json_model<'new, 'old>( let mut model = JsonModel::new(&new_inst.name, &new_inst.class); let mut properties = HashMap::with_capacity(new_inst.properties.len()); + let class_data = rbx_reflection_database::get() + .classes + .get(model.class_name.as_str()); + // TODO handle attributes separately if let Some(old_inst) = snapshot.old_inst() { for (name, value) in &new_inst.properties { + // We do not currently support Ref properties. + if matches!(value, Variant::Ref(_)) { + continue; + } if old_inst.properties().contains_key(name) { properties.insert(name.clone(), UnresolvedValue::from(value.clone())); } } } else { - for (name, value) in new_inst.properties.clone() { - properties.insert(name, UnresolvedValue::from(value)); + if let Some(class_data) = class_data { + let default_properties = &class_data.default_properties; + for (name, value) in new_inst.properties.clone() { + // We do not currently support Ref properties. + if matches!(value, Variant::Ref(_)) { + continue; + } + match default_properties.get(name.as_str()) { + Some(default) if variant_eq(&value, default) => {} + _ => { + properties.insert(name, UnresolvedValue::from(value)); + } + } + } + } else { + for (name, value) in new_inst.properties.clone() { + if matches!(value, Variant::Ref(_)) { + continue; + } + properties.insert(name, UnresolvedValue::from(value)); + } } } model.set_properties(properties); From 24e519253fdee333ed8dfac4301c849a405cffbb Mon Sep 17 00:00:00 2001 From: Micah Date: Wed, 20 Dec 2023 14:32:51 -0800 Subject: [PATCH 058/366] Revert making JsonModel public --- src/snapshot_middleware/json_model.rs | 42 ++++++--------------------- src/snapshot_middleware/mod.rs | 1 - 2 files changed, 9 insertions(+), 34 deletions(-) diff --git a/src/snapshot_middleware/json_model.rs b/src/snapshot_middleware/json_model.rs index 3d2173b93..1fa6b6aa4 100644 --- a/src/snapshot_middleware/json_model.rs +++ b/src/snapshot_middleware/json_model.rs @@ -67,7 +67,13 @@ pub fn syncback_json_model<'new, 'old>( path.set_extension("model.json"); let new_inst = snapshot.new_inst(); - let mut model = JsonModel::new(&new_inst.name, &new_inst.class); + let mut model = JsonModel { + name: Some(new_inst.name.clone()), + class_name: new_inst.class.clone(), + children: Vec::new(), + properties: HashMap::new(), + attributes: HashMap::new(), + }; let mut properties = HashMap::with_capacity(new_inst.properties.len()); let class_data = rbx_reflection_database::get() @@ -109,7 +115,7 @@ pub fn syncback_json_model<'new, 'old>( } } } - model.set_properties(properties); + model.properties = properties; // TODO children @@ -126,7 +132,7 @@ pub fn syncback_json_model<'new, 'old>( #[derive(Debug, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] -pub struct JsonModel { +struct JsonModel { #[serde(alias = "Name", skip_serializing)] name: Option, @@ -187,36 +193,6 @@ impl JsonModel { children, }) } - - /// Constructs an empty JSON model with the provided name and class. - #[inline] - pub fn new(name: &str, class_name: &str) -> Self { - Self { - name: Some(name.to_string()), - class_name: class_name.to_string(), - children: Vec::new(), - properties: HashMap::new(), - attributes: HashMap::new(), - } - } - - /// Sets the properties of this `JsonModel`. - #[inline] - pub fn set_properties(&mut self, properties: HashMap) { - self.properties = properties; - } - - /// Sets the attributes of this `JsonModel`. - #[inline] - pub fn set_attributes(&mut self, attributes: HashMap) { - self.attributes = attributes; - } - - /// Pushes the provided `JsonModel` as a child of this one. - #[inline] - pub fn push_child(&mut self, child: Self) { - self.children.push(child); - } } #[cfg(test)] diff --git a/src/snapshot_middleware/mod.rs b/src/snapshot_middleware/mod.rs index b9535c544..febf4092c 100644 --- a/src/snapshot_middleware/mod.rs +++ b/src/snapshot_middleware/mod.rs @@ -48,7 +48,6 @@ use self::{ }; pub use self::{ - json_model::JsonModel, lua::ScriptType, meta_file::{AdjacentMetadata, DirectoryMetadata}, project::snapshot_project_node, From cbb92204f1277f4db3881e900abb16ee7b3931fa Mon Sep 17 00:00:00 2001 From: Micah Date: Wed, 20 Dec 2023 14:51:28 -0800 Subject: [PATCH 059/366] Validate file names in one place instead of in each middleware --- src/snapshot_middleware/dir.rs | 6 +----- src/snapshot_middleware/json_model.rs | 6 +----- src/snapshot_middleware/lua.rs | 10 +--------- src/snapshot_middleware/mod.rs | 11 ++++++++++- src/snapshot_middleware/rbxm.rs | 6 +----- src/snapshot_middleware/rbxmx.rs | 5 +---- src/snapshot_middleware/txt.rs | 6 +----- 7 files changed, 16 insertions(+), 34 deletions(-) diff --git a/src/snapshot_middleware/dir.rs b/src/snapshot_middleware/dir.rs index 811510af5..496a09bda 100644 --- a/src/snapshot_middleware/dir.rs +++ b/src/snapshot_middleware/dir.rs @@ -8,7 +8,7 @@ use rbx_dom_weak::types::Ref; use crate::{ snapshot::{InstanceContext, InstanceMetadata, InstanceSnapshot}, - syncback::{is_valid_file_name, FsSnapshot, SyncbackReturn, SyncbackSnapshot}, + syncback::{FsSnapshot, SyncbackReturn, SyncbackSnapshot}, }; use super::{meta_file::DirectoryMetadata, snapshot_from_vfs}; @@ -96,10 +96,6 @@ pub fn snapshot_dir_no_meta( pub fn syncback_dir<'new, 'old>( snapshot: &SyncbackSnapshot<'new, 'old>, ) -> anyhow::Result> { - if !is_valid_file_name(&snapshot.name) { - anyhow::bail!("cannot create a directory with name {}", snapshot.name); - } - let path = snapshot.parent_path.join(&snapshot.name); let mut removed_children = Vec::new(); diff --git a/src/snapshot_middleware/json_model.rs b/src/snapshot_middleware/json_model.rs index 1fa6b6aa4..78b643054 100644 --- a/src/snapshot_middleware/json_model.rs +++ b/src/snapshot_middleware/json_model.rs @@ -8,7 +8,7 @@ use serde::{Deserialize, Serialize}; use crate::{ resolution::UnresolvedValue, snapshot::{InstanceContext, InstanceSnapshot}, - syncback::{is_valid_file_name, FsSnapshot, SyncbackReturn, SyncbackSnapshot}, + syncback::{FsSnapshot, SyncbackReturn, SyncbackSnapshot}, variant_eq::variant_eq, }; @@ -59,10 +59,6 @@ pub fn snapshot_json_model( pub fn syncback_json_model<'new, 'old>( snapshot: &SyncbackSnapshot<'new, 'old>, ) -> anyhow::Result> { - if !is_valid_file_name(&snapshot.name) { - anyhow::bail!("cannot create a file with name {}", snapshot.name); - } - let mut path = snapshot.parent_path.join(&snapshot.name); path.set_extension("model.json"); diff --git a/src/snapshot_middleware/lua.rs b/src/snapshot_middleware/lua.rs index 2f7eb3457..744883bcd 100644 --- a/src/snapshot_middleware/lua.rs +++ b/src/snapshot_middleware/lua.rs @@ -6,7 +6,7 @@ use rbx_dom_weak::types::{Enum, Variant}; use crate::{ snapshot::{InstanceContext, InstanceMetadata, InstanceSnapshot}, - syncback::{is_valid_file_name, FsSnapshot, SyncbackReturn, SyncbackSnapshot}, + syncback::{FsSnapshot, SyncbackReturn, SyncbackSnapshot}, }; use super::{ @@ -127,10 +127,6 @@ pub fn syncback_lua<'new, 'old>( script_type: ScriptType, snapshot: &SyncbackSnapshot<'new, 'old>, ) -> anyhow::Result> { - if !is_valid_file_name(&snapshot.name) { - anyhow::bail!("cannot create a file with name {}", snapshot.name); - } - let inst = snapshot.new_inst(); let mut path = snapshot.parent_path.join(&snapshot.name); @@ -158,10 +154,6 @@ pub fn syncback_lua_init<'new, 'old>( script_type: ScriptType, snapshot: &SyncbackSnapshot<'new, 'old>, ) -> anyhow::Result> { - if !is_valid_file_name(&snapshot.name) { - anyhow::bail!("cannot create a directory with name {}", snapshot.name); - } - let mut path = snapshot.parent_path.join(&snapshot.name); path.push("init"); path.set_extension(match script_type { diff --git a/src/snapshot_middleware/mod.rs b/src/snapshot_middleware/mod.rs index febf4092c..b0d13a88d 100644 --- a/src/snapshot_middleware/mod.rs +++ b/src/snapshot_middleware/mod.rs @@ -28,11 +28,14 @@ use anyhow::Context; use memofs::{IoResultExt, Vfs}; use serde::{Deserialize, Serialize}; -use crate::snapshot::{InstanceContext, InstanceSnapshot, SyncRule}; use crate::{ glob::Glob, syncback::{SyncbackReturn, SyncbackSnapshot}, }; +use crate::{ + snapshot::{InstanceContext, InstanceSnapshot, SyncRule}, + syncback::is_valid_file_name, +}; use self::{ csv::{snapshot_csv, snapshot_csv_init}, @@ -243,6 +246,12 @@ impl Middleware { &self, snapshot: &SyncbackSnapshot<'new, 'old>, ) -> anyhow::Result> { + if !is_valid_file_name(&snapshot.name) { + anyhow::bail!( + "cannot create a file or directory with name {}", + snapshot.name + ); + } match self { Middleware::Project => syncback_project(snapshot), Middleware::ServerScript => syncback_lua(ScriptType::Server, snapshot), diff --git a/src/snapshot_middleware/rbxm.rs b/src/snapshot_middleware/rbxm.rs index 31a22cee1..b8137b710 100644 --- a/src/snapshot_middleware/rbxm.rs +++ b/src/snapshot_middleware/rbxm.rs @@ -5,7 +5,7 @@ use memofs::Vfs; use crate::{ snapshot::{InstanceContext, InstanceMetadata, InstanceSnapshot}, - syncback::{is_valid_file_name, FsSnapshot, SyncbackReturn, SyncbackSnapshot}, + syncback::{FsSnapshot, SyncbackReturn, SyncbackSnapshot}, }; #[profiling::function] @@ -47,10 +47,6 @@ pub fn syncback_rbxm<'new, 'old>( ) -> anyhow::Result> { // If any of the children of this Instance are scripts, we don't want // include them in the model. So instead, we'll check and then serialize. - if !is_valid_file_name(&snapshot.name) { - anyhow::bail!("cannot create a file with name {}", snapshot.name); - } - let inst = snapshot.new_inst(); let mut path = snapshot.parent_path.join(&snapshot.name); path.set_extension("rbxm"); diff --git a/src/snapshot_middleware/rbxmx.rs b/src/snapshot_middleware/rbxmx.rs index 78bb24984..1b7ecdca7 100644 --- a/src/snapshot_middleware/rbxmx.rs +++ b/src/snapshot_middleware/rbxmx.rs @@ -5,7 +5,7 @@ use memofs::Vfs; use crate::{ snapshot::{InstanceContext, InstanceMetadata, InstanceSnapshot}, - syncback::{is_valid_file_name, FsSnapshot, SyncbackReturn, SyncbackSnapshot}, + syncback::{FsSnapshot, SyncbackReturn, SyncbackSnapshot}, }; pub fn snapshot_rbxmx( @@ -49,9 +49,6 @@ pub fn syncback_rbxmx<'new, 'old>( ) -> anyhow::Result> { // If any of the children of this Instance are scripts, we don't want // include them in the model. So instead, we'll check and then serialize. - if !is_valid_file_name(&snapshot.name) { - anyhow::bail!("cannot create a file with name {}", snapshot.name); - } let inst = snapshot.new_inst(); let mut path = snapshot.parent_path.join(&snapshot.name); diff --git a/src/snapshot_middleware/txt.rs b/src/snapshot_middleware/txt.rs index c24ee2a94..c6fa0b34e 100644 --- a/src/snapshot_middleware/txt.rs +++ b/src/snapshot_middleware/txt.rs @@ -7,7 +7,7 @@ use rbx_dom_weak::types::Variant; use crate::{ snapshot::{InstanceContext, InstanceMetadata, InstanceSnapshot}, - syncback::{is_valid_file_name, FsSnapshot, SyncbackReturn, SyncbackSnapshot}, + syncback::{FsSnapshot, SyncbackReturn, SyncbackSnapshot}, }; use super::meta_file::AdjacentMetadata; @@ -51,10 +51,6 @@ pub fn snapshot_txt( pub fn syncback_txt<'new, 'old>( snapshot: &SyncbackSnapshot<'new, 'old>, ) -> anyhow::Result> { - if !is_valid_file_name(&snapshot.name) { - anyhow::bail!("cannot create a file with name {}", snapshot.name); - } - let inst = snapshot.new_inst(); let mut path = snapshot.parent_path.join(&snapshot.name); path.set_extension("txt"); From 25343a6233b410200fb5f12d1041a645fa8258dd Mon Sep 17 00:00:00 2001 From: Micah Date: Wed, 20 Dec 2023 14:53:49 -0800 Subject: [PATCH 060/366] Add more classes to get_best_middleware --- src/syncback/mod.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/syncback/mod.rs b/src/syncback/mod.rs index 67d623040..a4ae0d48b 100644 --- a/src/syncback/mod.rs +++ b/src/syncback/mod.rs @@ -110,8 +110,15 @@ pub struct SyncbackReturn<'new, 'old> { pub fn get_best_middleware(inst: &Instance) -> Middleware { match inst.class.as_str() { "Folder" => Middleware::Dir, - // TODO this should probably just be rbxm - "Model" => Middleware::Rbxm, + "Sound" + | "SoundGroup" + | "Sky" + | "Atmosphere" + | "BloomEffect" + | "BlurEffect" + | "ColorCorrectionEffect" + | "DepthOfFieldEffect" + | "SunRaysEffect" => Middleware::JsonModel, "Script" => { if inst.children().len() == 0 { Middleware::ServerScript From 78d5d3cea94f2643e9f6230ff69ee666968a5df6 Mon Sep 17 00:00:00 2001 From: Micah Date: Wed, 20 Dec 2023 14:59:40 -0800 Subject: [PATCH 061/366] Check class names in Project syncback --- src/snapshot_middleware/project.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/snapshot_middleware/project.rs b/src/snapshot_middleware/project.rs index 47651009d..4a513cf22 100644 --- a/src/snapshot_middleware/project.rs +++ b/src/snapshot_middleware/project.rs @@ -345,7 +345,9 @@ pub fn syncback_project<'new, 'old>( for (child_name, child_node) in &mut node.children { if let Some(new_child) = new_child_map.get(child_name.as_str()) { if let Some(old_child) = old_child_map.get(child_name.as_str()) { - // TODO verify class names + if &new_child.class != old_child.class_name() { + anyhow::bail!("Cannot change the class of items in a project"); + } for (name, value) in &new_child.properties { if child_node.properties.contains_key(name) { child_node From b776b64632a999eb20b652d33e23061f8cc5a841 Mon Sep 17 00:00:00 2001 From: Micah Date: Wed, 20 Dec 2023 15:52:16 -0800 Subject: [PATCH 062/366] Fix init middleware when syncing --- src/snapshot_middleware/mod.rs | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/snapshot_middleware/mod.rs b/src/snapshot_middleware/mod.rs index b0d13a88d..0e6a427b3 100644 --- a/src/snapshot_middleware/mod.rs +++ b/src/snapshot_middleware/mod.rs @@ -19,7 +19,6 @@ mod txt; mod util; use std::{ - borrow::Cow, path::{Path, PathBuf}, sync::OnceLock, }; @@ -73,14 +72,11 @@ pub fn snapshot_from_vfs( }; if meta.is_dir() { - let (middleware, name) = get_dir_middleware(vfs, path)?; + let (middleware, dir_name, init_path) = get_dir_middleware(vfs, path)?; // TODO: Support user defined init paths match middleware { - Middleware::Dir => middleware.snapshot(context, vfs, path, &name), - _ => { - let name_as_path: PathBuf = name.as_ref().into(); - middleware.snapshot(context, vfs, &path.join(name_as_path), &name) - } + Middleware::Dir => middleware.snapshot(context, vfs, path, &dir_name), + _ => middleware.snapshot(context, vfs, &init_path, &dir_name), } } else { let file_name = path @@ -102,10 +98,13 @@ pub fn snapshot_from_vfs( /// Gets the appropriate middleware for a directory by checking for `init` /// files. This uses an intrinsic priority list and for compatibility, /// that order should be left unchanged. +/// +/// Returns the middleware, the name of the directory, and the path to +/// the init location. fn get_dir_middleware>( vfs: &Vfs, dir: P, -) -> anyhow::Result<(Middleware, Cow<'static, str>)> { +) -> anyhow::Result<(Middleware, String, PathBuf)> { let path = dir.as_ref(); let dir_name = path .file_name() @@ -129,12 +128,13 @@ fn get_dir_middleware>( }); for (middleware, name) in order { - if vfs.metadata(path.join(name)).with_not_found()?.is_some() { - return Ok((*middleware, Cow::Borrowed(*name))); + let test_path = path.join(name); + if vfs.metadata(&test_path).with_not_found()?.is_some() { + return Ok((*middleware, dir_name, test_path)); } } - Ok((Middleware::Dir, Cow::Owned(dir_name))) + Ok((Middleware::Dir, dir_name, path.to_path_buf())) } /// Gets a snapshot for a path given an InstanceContext and Vfs, taking From d7a6f13fe1204eb57768582fc86b00048ebc0e68 Mon Sep 17 00:00:00 2001 From: Micah Date: Wed, 20 Dec 2023 16:56:40 -0800 Subject: [PATCH 063/366] Move Variant -> UnresolvedValue Support ambiguous enum syntax --- src/resolution.rs | 103 +++++++++++++++----------- src/snapshot_middleware/json_model.rs | 17 ++++- src/snapshot_middleware/project.rs | 11 ++- 3 files changed, 80 insertions(+), 51 deletions(-) diff --git a/src/resolution.rs b/src/resolution.rs index 9e7cc3f41..9fcef34d3 100644 --- a/src/resolution.rs +++ b/src/resolution.rs @@ -35,6 +35,65 @@ impl UnresolvedValue { UnresolvedValue::Ambiguous(partial) => partial.resolve_unambiguous(), } } + + pub fn from_variant(variant: Variant, class_name: &str, prop_name: &str) -> Self { + Self::Ambiguous(match variant { + Variant::Enum(rbx_enum) => { + if let Some(property) = find_descriptor(class_name, prop_name) { + if let DataType::Enum(enum_name) = &property.data_type { + let database = rbx_reflection_database::get(); + if let Some(enum_descriptor) = database.enums.get(enum_name) { + for (variant_name, id) in &enum_descriptor.items { + if *id == rbx_enum.to_u32() { + return Self::Ambiguous(AmbiguousValue::String( + variant_name.to_string(), + )); + } + } + } + } + } + return Self::FullyQualified(variant); + } + Variant::Bool(bool) => AmbiguousValue::Bool(bool), + Variant::Float32(n) => AmbiguousValue::Number(n as f64), + Variant::Float64(n) => AmbiguousValue::Number(n), + Variant::Int32(n) => AmbiguousValue::Number(n as f64), + Variant::Int64(n) => AmbiguousValue::Number(n as f64), + Variant::String(str) => AmbiguousValue::String(str), + Variant::Tags(tags) => { + AmbiguousValue::StringArray(tags.iter().map(|s| s.to_string()).collect()) + } + Variant::Content(content) => AmbiguousValue::String(content.into_string()), + Variant::Vector2(vector) => AmbiguousValue::Array2([vector.x as f64, vector.y as f64]), + Variant::Vector3(vector) => { + AmbiguousValue::Array3([vector.x as f64, vector.y as f64, vector.z as f64]) + } + Variant::Color3(color) => { + AmbiguousValue::Array3([color.r as f64, color.g as f64, color.b as f64]) + } + Variant::CFrame(cf) => AmbiguousValue::Array12([ + cf.position.x as f64, + cf.position.y as f64, + cf.position.z as f64, + cf.orientation.x.x as f64, + cf.orientation.x.y as f64, + cf.orientation.x.z as f64, + cf.orientation.y.x as f64, + cf.orientation.y.y as f64, + cf.orientation.y.z as f64, + cf.orientation.z.x as f64, + cf.orientation.z.y as f64, + cf.orientation.z.z as f64, + ]), + Variant::Attributes(attr) => AmbiguousValue::Attributes(attr), + Variant::Font(font) => AmbiguousValue::Font(font), + Variant::MaterialColors(colors) => AmbiguousValue::MaterialColors(colors), + _ => { + return Self::FullyQualified(variant); + } + }) + } } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -190,50 +249,6 @@ impl AmbiguousValue { } } -impl From for UnresolvedValue { - fn from(value: Variant) -> Self { - Self::Ambiguous(match value { - Variant::Bool(bool) => AmbiguousValue::Bool(bool), - Variant::Float32(n) => AmbiguousValue::Number(n as f64), - Variant::Float64(n) => AmbiguousValue::Number(n), - Variant::Int32(n) => AmbiguousValue::Number(n as f64), - Variant::Int64(n) => AmbiguousValue::Number(n as f64), - Variant::String(str) => AmbiguousValue::String(str), - Variant::Tags(tags) => { - AmbiguousValue::StringArray(tags.iter().map(|s| s.to_string()).collect()) - } - Variant::Content(content) => AmbiguousValue::String(content.into_string()), - Variant::Vector2(vector) => AmbiguousValue::Array2([vector.x as f64, vector.y as f64]), - Variant::Vector3(vector) => { - AmbiguousValue::Array3([vector.x as f64, vector.y as f64, vector.z as f64]) - } - Variant::Color3(color) => { - AmbiguousValue::Array3([color.r as f64, color.g as f64, color.b as f64]) - } - Variant::CFrame(cf) => AmbiguousValue::Array12([ - cf.position.x as f64, - cf.position.y as f64, - cf.position.z as f64, - cf.orientation.x.x as f64, - cf.orientation.x.y as f64, - cf.orientation.x.z as f64, - cf.orientation.y.x as f64, - cf.orientation.y.y as f64, - cf.orientation.y.z as f64, - cf.orientation.z.x as f64, - cf.orientation.z.y as f64, - cf.orientation.z.z as f64, - ]), - Variant::Attributes(attr) => AmbiguousValue::Attributes(attr), - Variant::Font(font) => AmbiguousValue::Font(font), - Variant::MaterialColors(colors) => AmbiguousValue::MaterialColors(colors), - _ => { - return Self::FullyQualified(value); - } - }) - } -} - fn find_descriptor( class_name: &str, prop_name: &str, diff --git a/src/snapshot_middleware/json_model.rs b/src/snapshot_middleware/json_model.rs index 78b643054..2dbd5babb 100644 --- a/src/snapshot_middleware/json_model.rs +++ b/src/snapshot_middleware/json_model.rs @@ -84,13 +84,16 @@ pub fn syncback_json_model<'new, 'old>( continue; } if old_inst.properties().contains_key(name) { - properties.insert(name.clone(), UnresolvedValue::from(value.clone())); + properties.insert( + name.clone(), + UnresolvedValue::from_variant(value.clone(), &new_inst.class, &*name), + ); } } } else { if let Some(class_data) = class_data { let default_properties = &class_data.default_properties; - for (name, value) in new_inst.properties.clone() { + for (name, value) in &new_inst.properties { // We do not currently support Ref properties. if matches!(value, Variant::Ref(_)) { continue; @@ -98,7 +101,10 @@ pub fn syncback_json_model<'new, 'old>( match default_properties.get(name.as_str()) { Some(default) if variant_eq(&value, default) => {} _ => { - properties.insert(name, UnresolvedValue::from(value)); + properties.insert( + name.clone(), + UnresolvedValue::from_variant(value.clone(), &new_inst.class, &name), + ); } } } @@ -107,7 +113,10 @@ pub fn syncback_json_model<'new, 'old>( if matches!(value, Variant::Ref(_)) { continue; } - properties.insert(name, UnresolvedValue::from(value)); + // Inserting `name` into the map takes ownership of it, so we + // have to make this first. + let value = UnresolvedValue::from_variant(value, &new_inst.class, &name); + properties.insert(name, value); } } } diff --git a/src/snapshot_middleware/project.rs b/src/snapshot_middleware/project.rs index 4a513cf22..52c55616a 100644 --- a/src/snapshot_middleware/project.rs +++ b/src/snapshot_middleware/project.rs @@ -350,9 +350,14 @@ pub fn syncback_project<'new, 'old>( } for (name, value) in &new_child.properties { if child_node.properties.contains_key(name) { - child_node - .properties - .insert(name.clone(), UnresolvedValue::from(value.clone())); + child_node.properties.insert( + name.clone(), + UnresolvedValue::from_variant( + value.clone(), + &new_child.class, + name.as_str(), + ), + ); } } nodes.push((child_node, new_child, *old_child)); From bfabceb560b3a4b36ddb8a422af9b6abef9389fb Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 21 Dec 2023 14:45:00 -0800 Subject: [PATCH 064/366] Don't bail when project name isn't valid file name --- src/snapshot_middleware/mod.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/snapshot_middleware/mod.rs b/src/snapshot_middleware/mod.rs index 0e6a427b3..818968b64 100644 --- a/src/snapshot_middleware/mod.rs +++ b/src/snapshot_middleware/mod.rs @@ -246,7 +246,8 @@ impl Middleware { &self, snapshot: &SyncbackSnapshot<'new, 'old>, ) -> anyhow::Result> { - if !is_valid_file_name(&snapshot.name) { + // We don't care about the names of projects + if !is_valid_file_name(&snapshot.name) && !matches!(self, Middleware::Project) { anyhow::bail!( "cannot create a file or directory with name {}", snapshot.name From 8692d19b7f577828c673ae228ff401c71e849263 Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 21 Dec 2023 14:48:45 -0800 Subject: [PATCH 065/366] Display child name in project errors --- src/snapshot_middleware/project.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/snapshot_middleware/project.rs b/src/snapshot_middleware/project.rs index 52c55616a..c02de3d69 100644 --- a/src/snapshot_middleware/project.rs +++ b/src/snapshot_middleware/project.rs @@ -346,7 +346,7 @@ pub fn syncback_project<'new, 'old>( if let Some(new_child) = new_child_map.get(child_name.as_str()) { if let Some(old_child) = old_child_map.get(child_name.as_str()) { if &new_child.class != old_child.class_name() { - anyhow::bail!("Cannot change the class of items in a project"); + anyhow::bail!("Cannot change the class of {child_name} in a project"); } for (name, value) in &new_child.properties { if child_node.properties.contains_key(name) { @@ -365,7 +365,7 @@ pub fn syncback_project<'new, 'old>( old_child_map.remove(child_name.as_str()); } } else { - anyhow::bail!("Cannot add or remove children from a project") + anyhow::bail!("Cannot add or remove {child_name} from project") } } From ede795abd1b7d875c3f70c5d06a68041100a839a Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 21 Dec 2023 15:00:22 -0800 Subject: [PATCH 066/366] Don't serialize syncback_rules when it's None --- src/project.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/project.rs b/src/project.rs index 768ee045b..fc9e4581d 100644 --- a/src/project.rs +++ b/src/project.rs @@ -86,6 +86,8 @@ pub struct Project { #[serde(default, skip_serializing_if = "Vec::is_empty")] pub glob_ignore_paths: Vec, + /// A list of rules for syncback with this project file. + #[serde(skip_serializing_if = "Option::is_none")] pub syncback_rules: Option, /// A list of mappings of globs to syncing rules. If a file matches a glob, From 265c128641b096d68c724c7f744a427de91e7763 Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 21 Dec 2023 15:27:39 -0800 Subject: [PATCH 067/366] Redo property serialization of json_model --- src/snapshot_middleware/json_model.rs | 102 ++++++++++++++------------ 1 file changed, 54 insertions(+), 48 deletions(-) diff --git a/src/snapshot_middleware/json_model.rs b/src/snapshot_middleware/json_model.rs index 2dbd5babb..e7f4e5d70 100644 --- a/src/snapshot_middleware/json_model.rs +++ b/src/snapshot_middleware/json_model.rs @@ -63,64 +63,70 @@ pub fn syncback_json_model<'new, 'old>( path.set_extension("model.json"); let new_inst = snapshot.new_inst(); - let mut model = JsonModel { - name: Some(new_inst.name.clone()), - class_name: new_inst.class.clone(), - children: Vec::new(), - properties: HashMap::new(), - attributes: HashMap::new(), - }; - let mut properties = HashMap::with_capacity(new_inst.properties.len()); let class_data = rbx_reflection_database::get() .classes - .get(model.class_name.as_str()); - - // TODO handle attributes separately - if let Some(old_inst) = snapshot.old_inst() { - for (name, value) in &new_inst.properties { - // We do not currently support Ref properties. - if matches!(value, Variant::Ref(_)) { - continue; - } - if old_inst.properties().contains_key(name) { - properties.insert( - name.clone(), - UnresolvedValue::from_variant(value.clone(), &new_inst.class, &*name), - ); - } - } + .get(new_inst.class.as_str()); + + let filtered: HashMap<&String, &Variant> = if let Some(old_inst) = snapshot.old_inst() { + new_inst + .properties + .iter() + .filter(|(name, _)| old_inst.properties().contains_key(*name)) + .collect() } else { if let Some(class_data) = class_data { let default_properties = &class_data.default_properties; - for (name, value) in &new_inst.properties { - // We do not currently support Ref properties. - if matches!(value, Variant::Ref(_)) { - continue; - } - match default_properties.get(name.as_str()) { - Some(default) if variant_eq(&value, default) => {} - _ => { - properties.insert( - name.clone(), - UnresolvedValue::from_variant(value.clone(), &new_inst.class, &name), - ); - } - } - } + new_inst + .properties + .iter() + .filter( + |(name, value)| match default_properties.get(name.as_str()) { + Some(default) => !variant_eq(value, default), + None => true, + }, + ) + .collect() } else { - for (name, value) in new_inst.properties.clone() { - if matches!(value, Variant::Ref(_)) { - continue; - } - // Inserting `name` into the map takes ownership of it, so we - // have to make this first. - let value = UnresolvedValue::from_variant(value, &new_inst.class, &name); - properties.insert(name, value); + new_inst.properties.iter().collect() + } + }; + + let mut properties = HashMap::with_capacity(new_inst.properties.capacity()); + let mut attributes = HashMap::new(); + for (name, value) in filtered { + if name == "Attributes" || name == "AttributesSerialize" { + if let Variant::Attributes(attr) = value { + attributes.extend(attr.iter().map(|(name, value)| { + ( + name.clone(), + UnresolvedValue::from_variant(value.clone(), &new_inst.class, &name), + ) + })) + } else { + log::error!( + "Property {name} should be Attributes but is {:?}", + value.ty() + ); } + } else if let Variant::Ref(_) = value { + // We do not currently support Ref properties + continue; + } else { + properties.insert( + name.clone(), + UnresolvedValue::from_variant(value.clone(), &new_inst.class, &name), + ); } } - model.properties = properties; + + let model = JsonModel { + name: Some(new_inst.name.clone()), + class_name: new_inst.class.clone(), + children: Vec::new(), + properties, + attributes, + }; // TODO children From bd4e96d9a8cc08ada27a82e5bb99b3559af3ead1 Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 28 Dec 2023 09:41:21 -0800 Subject: [PATCH 068/366] Handle SharedString appropriately --- src/resolution.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/resolution.rs b/src/resolution.rs index 9fcef34d3..909e095c9 100644 --- a/src/resolution.rs +++ b/src/resolution.rs @@ -2,8 +2,8 @@ use std::borrow::Borrow; use anyhow::{bail, format_err}; use rbx_dom_weak::types::{ - Attributes, CFrame, Color3, Content, Enum, Font, MaterialColors, Matrix3, Tags, Variant, - VariantType, Vector2, Vector3, + Attributes, BinaryString, CFrame, Color3, Content, Enum, Font, MaterialColors, Matrix3, Tags, + Variant, VariantType, Vector2, Vector3, }; use rbx_reflection::{DataType, PropertyDescriptor}; use serde::{Deserialize, Serialize}; @@ -60,6 +60,15 @@ impl UnresolvedValue { Variant::Float64(n) => AmbiguousValue::Number(n), Variant::Int32(n) => AmbiguousValue::Number(n as f64), Variant::Int64(n) => AmbiguousValue::Number(n as f64), + Variant::SharedString(sstr) => { + if let Ok(str) = std::str::from_utf8(sstr.data()) { + AmbiguousValue::String(str.to_string()) + } else { + return Self::FullyQualified(Variant::BinaryString(BinaryString::from( + sstr.data(), + ))); + } + } Variant::String(str) => AmbiguousValue::String(str), Variant::Tags(tags) => { AmbiguousValue::StringArray(tags.iter().map(|s| s.to_string()).collect()) From 75c3aaa7dde102e1f69a3fef351e4ed2d35018ce Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 28 Dec 2023 09:41:53 -0800 Subject: [PATCH 069/366] Add `push_dir` to FsSnapshot --- src/syncback/fs_snapshot.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/syncback/fs_snapshot.rs b/src/syncback/fs_snapshot.rs index a83dbe1c1..27fb272af 100644 --- a/src/syncback/fs_snapshot.rs +++ b/src/syncback/fs_snapshot.rs @@ -41,6 +41,10 @@ impl FsSnapshot { .insert(path.as_ref().to_path_buf(), Arc::new(data)); } + pub fn push_dir>(&mut self, path: P) { + self.dir.insert(path.as_ref().to_path_buf()); + } + pub fn pop_dir>(&mut self, path: P) -> bool { self.dir.remove(path.as_ref()) } From 6f0028058c4c983ba9d7953f5b34643180d9ccc1 Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 28 Dec 2023 09:42:09 -0800 Subject: [PATCH 070/366] Add method for checking if a meta file is 'empty'. --- src/snapshot_middleware/meta_file.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/snapshot_middleware/meta_file.rs b/src/snapshot_middleware/meta_file.rs index 72930e49e..ed5958627 100644 --- a/src/snapshot_middleware/meta_file.rs +++ b/src/snapshot_middleware/meta_file.rs @@ -174,4 +174,22 @@ impl DirectoryMetadata { Ok(()) } + + /// Returns whether the metadata is 'empty', meaning it doesn't have anything + /// worth persisting in it. Specifically: + /// + /// - The number of properties and attributes is 0 + /// - `ignore_unknown_instances` is None + /// - `class_name` is either None or not Some("Folder") + #[inline] + pub fn is_empty(&self) -> bool { + self.attributes.len() == 0 + && self.properties.len() == 0 + && self.ignore_unknown_instances.is_none() + && if let Some(class) = &self.class_name { + class == "Folder" + } else { + true + } + } } From 952770cf5d00477a24776db21cb5890108fa951c Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 28 Dec 2023 11:05:46 -0800 Subject: [PATCH 071/366] Move property filtering logic elsewhere --- src/snapshot_middleware/json_model.rs | 33 ++--------------- src/syncback/snapshot.rs | 53 +++++++++++++++++++++++++-- 2 files changed, 53 insertions(+), 33 deletions(-) diff --git a/src/snapshot_middleware/json_model.rs b/src/snapshot_middleware/json_model.rs index e7f4e5d70..4a8dfafda 100644 --- a/src/snapshot_middleware/json_model.rs +++ b/src/snapshot_middleware/json_model.rs @@ -9,7 +9,6 @@ use crate::{ resolution::UnresolvedValue, snapshot::{InstanceContext, InstanceSnapshot}, syncback::{FsSnapshot, SyncbackReturn, SyncbackSnapshot}, - variant_eq::variant_eq, }; pub fn snapshot_json_model( @@ -64,33 +63,7 @@ pub fn syncback_json_model<'new, 'old>( let new_inst = snapshot.new_inst(); - let class_data = rbx_reflection_database::get() - .classes - .get(new_inst.class.as_str()); - - let filtered: HashMap<&String, &Variant> = if let Some(old_inst) = snapshot.old_inst() { - new_inst - .properties - .iter() - .filter(|(name, _)| old_inst.properties().contains_key(*name)) - .collect() - } else { - if let Some(class_data) = class_data { - let default_properties = &class_data.default_properties; - new_inst - .properties - .iter() - .filter( - |(name, value)| match default_properties.get(name.as_str()) { - Some(default) => !variant_eq(value, default), - None => true, - }, - ) - .collect() - } else { - new_inst.properties.iter().collect() - } - }; + let filtered = snapshot.get_filtered_properties(); let mut properties = HashMap::with_capacity(new_inst.properties.capacity()); let mut attributes = HashMap::new(); @@ -99,7 +72,7 @@ pub fn syncback_json_model<'new, 'old>( if let Variant::Attributes(attr) = value { attributes.extend(attr.iter().map(|(name, value)| { ( - name.clone(), + name.to_string(), UnresolvedValue::from_variant(value.clone(), &new_inst.class, &name), ) })) @@ -114,7 +87,7 @@ pub fn syncback_json_model<'new, 'old>( continue; } else { properties.insert( - name.clone(), + name.to_string(), UnresolvedValue::from_variant(value.clone(), &new_inst.class, &name), ); } diff --git a/src/syncback/snapshot.rs b/src/syncback/snapshot.rs index 8590f1511..b40a2b26c 100644 --- a/src/syncback/snapshot.rs +++ b/src/syncback/snapshot.rs @@ -1,8 +1,14 @@ use memofs::Vfs; -use std::{path::PathBuf, rc::Rc}; +use std::{collections::HashMap, path::PathBuf, rc::Rc}; -use crate::snapshot::{InstanceWithMeta, RojoTree}; -use rbx_dom_weak::{types::Ref, Instance, WeakDom}; +use crate::{ + snapshot::{InstanceWithMeta, RojoTree}, + variant_eq::variant_eq, +}; +use rbx_dom_weak::{ + types::{Ref, Variant}, + Instance, WeakDom, +}; pub struct SyncbackData<'new, 'old> { pub(super) vfs: &'old Vfs, @@ -32,6 +38,47 @@ impl<'new, 'old> SyncbackSnapshot<'new, 'old> { } } + /// Returns a map of properties for the 'new' Instance with filtering + /// done to avoid noise. + /// + /// Note that the returned map does not filter any properties by name, nor + /// does it clone the values. This is left to the consumer. + pub fn get_filtered_properties(&self) -> HashMap<&'new str, &'new Variant> { + let new_inst = self.new_inst(); + let mut properties: HashMap<&str, &Variant> = + HashMap::with_capacity(new_inst.properties.capacity()); + + if let Some(old_inst) = self.old_inst() { + for (name, value) in &new_inst.properties { + if old_inst.properties().contains_key(name) { + properties.insert(name, value); + } + } + } else { + let class_data = rbx_reflection_database::get() + .classes + .get(new_inst.class.as_str()); + if let Some(class_data) = class_data { + let defaults = &class_data.default_properties; + for (name, value) in &new_inst.properties { + if let Some(default) = defaults.get(name.as_str()) { + if !variant_eq(value, default) { + properties.insert(name, value); + } + } else { + properties.insert(name, value); + } + } + } else { + for (name, value) in &new_inst.properties { + properties.insert(name, value); + } + } + } + + properties + } + /// Returns an Instance from the old tree with the provided referent, if it /// exists. #[inline] From b8488f6e129505d94bb3c76a96901848f068228c Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 28 Dec 2023 11:06:44 -0800 Subject: [PATCH 072/366] Swap dir syncback middleware to use new filtering logic --- src/snapshot_middleware/dir.rs | 68 ++++++++++++++++++++++++++++++---- 1 file changed, 61 insertions(+), 7 deletions(-) diff --git a/src/snapshot_middleware/dir.rs b/src/snapshot_middleware/dir.rs index 496a09bda..700036be9 100644 --- a/src/snapshot_middleware/dir.rs +++ b/src/snapshot_middleware/dir.rs @@ -3,10 +3,12 @@ use std::{ path::Path, }; +use anyhow::Context; use memofs::{DirEntry, IoResultExt, Vfs}; -use rbx_dom_weak::types::Ref; +use rbx_dom_weak::types::{Ref, Variant}; use crate::{ + resolution::UnresolvedValue, snapshot::{InstanceContext, InstanceMetadata, InstanceSnapshot}, syncback::{FsSnapshot, SyncbackReturn, SyncbackSnapshot}, }; @@ -97,6 +99,60 @@ pub fn syncback_dir<'new, 'old>( snapshot: &SyncbackSnapshot<'new, 'old>, ) -> anyhow::Result> { let path = snapshot.parent_path.join(&snapshot.name); + let new_inst = snapshot.new_inst(); + + let mut dir_syncback = syncback_dir_no_meta(snapshot)?; + + let mut meta = if let Some(dir) = dir_meta(snapshot.vfs(), &path)? { + dir + } else { + DirectoryMetadata { + ignore_unknown_instances: None, + properties: HashMap::with_capacity(new_inst.properties.capacity()), + attributes: HashMap::new(), + class_name: if new_inst.class == "Folder" { + None + } else { + Some(new_inst.class.clone()) + }, + path: path.join("init.meta.json"), + } + }; + for (name, value) in snapshot.get_filtered_properties() { + if name == "Attributes" || name == "AttributesSerialize" { + if let Variant::Attributes(attrs) = value { + meta.attributes.extend(attrs.iter().map(|(name, value)| { + ( + name.to_string(), + UnresolvedValue::FullyQualified(value.clone()), + ) + })) + } else { + log::error!("Property {name} should be Attributes but is not"); + } + } else { + meta.properties.insert( + name.to_string(), + UnresolvedValue::from_variant(value.to_owned(), &new_inst.class, name), + ); + } + } + + if !meta.is_empty() { + dir_syncback.fs_snapshot.push_file( + &meta.path, + serde_json::to_vec_pretty(&meta).context("could not serialize new init.meta.json")?, + ); + } + Ok(dir_syncback) +} + +pub fn syncback_dir_no_meta<'new, 'old>( + snapshot: &SyncbackSnapshot<'new, 'old>, +) -> anyhow::Result> { + let path = snapshot.parent_path.join(&snapshot.name); + + let new_inst = snapshot.new_inst(); let mut removed_children = Vec::new(); let mut children = Vec::new(); @@ -127,7 +183,7 @@ pub fn syncback_dir<'new, 'old>( } } - for child_ref in snapshot.new_inst().children() { + for child_ref in new_inst.children() { let new_child = snapshot.get_new_instance(*child_ref).unwrap(); // If it exists in the new tree but not the old one, it was added. match old_children.get(new_child.name.as_str()) { @@ -142,17 +198,15 @@ pub fn syncback_dir<'new, 'old>( } } } else { - for child_ref in snapshot.new_inst().children() { + for child_ref in new_inst.children() { let child = snapshot.get_new_instance(*child_ref).unwrap(); children.push(snapshot.from_parent(child.name.clone(), *child_ref, None)) } } - let fs_snapshot = FsSnapshot::new().with_dir(&path); - // TODO metadata, including classname Ok(SyncbackReturn { - inst_snapshot: InstanceSnapshot::from_instance(snapshot.new_inst()), - fs_snapshot, + inst_snapshot: InstanceSnapshot::from_instance(new_inst), + fs_snapshot: FsSnapshot::new().with_dir(path), children, removed_children, }) From b6bc5b8da1e76f6730d4c100b6bd9b29a6354f56 Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 28 Dec 2023 11:17:31 -0800 Subject: [PATCH 073/366] Don't syncback Ref or SharedString properties --- src/resolution.rs | 18 +++++++++--------- src/syncback/snapshot.rs | 8 ++++++++ 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/resolution.rs b/src/resolution.rs index 909e095c9..379ccc057 100644 --- a/src/resolution.rs +++ b/src/resolution.rs @@ -60,15 +60,15 @@ impl UnresolvedValue { Variant::Float64(n) => AmbiguousValue::Number(n), Variant::Int32(n) => AmbiguousValue::Number(n as f64), Variant::Int64(n) => AmbiguousValue::Number(n as f64), - Variant::SharedString(sstr) => { - if let Ok(str) = std::str::from_utf8(sstr.data()) { - AmbiguousValue::String(str.to_string()) - } else { - return Self::FullyQualified(Variant::BinaryString(BinaryString::from( - sstr.data(), - ))); - } - } + // Variant::SharedString(sstr) => { + // if let Ok(str) = std::str::from_utf8(sstr.data()) { + // AmbiguousValue::String(str.to_string()) + // } else { + // return Self::FullyQualified(Variant::BinaryString(BinaryString::from( + // sstr.data(), + // ))); + // } + // } Variant::String(str) => AmbiguousValue::String(str), Variant::Tags(tags) => { AmbiguousValue::StringArray(tags.iter().map(|s| s.to_string()).collect()) diff --git a/src/syncback/snapshot.rs b/src/syncback/snapshot.rs index b40a2b26c..3a2db8634 100644 --- a/src/syncback/snapshot.rs +++ b/src/syncback/snapshot.rs @@ -61,6 +61,10 @@ impl<'new, 'old> SyncbackSnapshot<'new, 'old> { if let Some(class_data) = class_data { let defaults = &class_data.default_properties; for (name, value) in &new_inst.properties { + // We don't currently support refs or shared strings + if matches!(value, Variant::Ref(_) | Variant::SharedString(_)) { + continue; + } if let Some(default) = defaults.get(name.as_str()) { if !variant_eq(value, default) { properties.insert(name, value); @@ -71,6 +75,10 @@ impl<'new, 'old> SyncbackSnapshot<'new, 'old> { } } else { for (name, value) in &new_inst.properties { + // We don't currently support refs or shared strings + if matches!(value, Variant::Ref(_) | Variant::SharedString(_)) { + continue; + } properties.insert(name, value); } } From 1b02e197f90b7b82cbf8e6cb6a7ea4d3206f8061 Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 28 Dec 2023 11:24:31 -0800 Subject: [PATCH 074/366] Update what classes sync back as what --- src/syncback/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/syncback/mod.rs b/src/syncback/mod.rs index a4ae0d48b..00838232a 100644 --- a/src/syncback/mod.rs +++ b/src/syncback/mod.rs @@ -109,7 +109,7 @@ pub struct SyncbackReturn<'new, 'old> { pub fn get_best_middleware(inst: &Instance) -> Middleware { match inst.class.as_str() { - "Folder" => Middleware::Dir, + "Folder" | "Configuration" | "Tool" | "ScreenGui" => Middleware::Dir, "Sound" | "SoundGroup" | "Sky" @@ -120,14 +120,14 @@ pub fn get_best_middleware(inst: &Instance) -> Middleware { | "DepthOfFieldEffect" | "SunRaysEffect" => Middleware::JsonModel, "Script" => { - if inst.children().len() == 0 { + if inst.children().is_empty() { Middleware::ServerScript } else { Middleware::ServerScriptDir } } "LocalScript" => { - if inst.children().len() == 0 { + if inst.children().is_empty() { Middleware::ClientScript } else { Middleware::ClientScriptDir From 0f79f414aa99ca222fe61f387e9dee4a477f3000 Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 28 Dec 2023 11:34:47 -0800 Subject: [PATCH 075/366] Support metadata for Lua dirs --- src/snapshot_middleware/lua.rs | 64 +++++++++++++++++++++++++++++----- 1 file changed, 55 insertions(+), 9 deletions(-) diff --git a/src/snapshot_middleware/lua.rs b/src/snapshot_middleware/lua.rs index 744883bcd..f57d39bcf 100644 --- a/src/snapshot_middleware/lua.rs +++ b/src/snapshot_middleware/lua.rs @@ -5,13 +5,15 @@ use memofs::{IoResultExt, Vfs}; use rbx_dom_weak::types::{Enum, Variant}; use crate::{ + resolution::UnresolvedValue, snapshot::{InstanceContext, InstanceMetadata, InstanceSnapshot}, syncback::{FsSnapshot, SyncbackReturn, SyncbackSnapshot}, }; use super::{ - dir::{dir_meta, snapshot_dir_no_meta, syncback_dir}, + dir::{dir_meta, snapshot_dir_no_meta, syncback_dir_no_meta}, meta_file::AdjacentMetadata, + DirectoryMetadata, }; #[derive(Debug)] @@ -142,7 +144,7 @@ pub fn syncback_lua<'new, 'old>( }; Ok(SyncbackReturn { - inst_snapshot: InstanceSnapshot::from_instance(inst).metadata(InstanceMetadata::new()), + inst_snapshot: InstanceSnapshot::from_instance(inst), fs_snapshot: FsSnapshot::new().with_file(path, contents), // Scripts don't have a child! children: Vec::new(), @@ -154,6 +156,8 @@ pub fn syncback_lua_init<'new, 'old>( script_type: ScriptType, snapshot: &SyncbackSnapshot<'new, 'old>, ) -> anyhow::Result> { + let new_inst = snapshot.new_inst(); + let mut path = snapshot.parent_path.join(&snapshot.name); path.push("init"); path.set_extension(match script_type { @@ -161,19 +165,61 @@ pub fn syncback_lua_init<'new, 'old>( ScriptType::Client => "client.lua", ScriptType::Server => "server.lua", }); - let contents = - if let Some(Variant::String(source)) = snapshot.new_inst().properties.get("Source") { - source.as_bytes().to_vec() - } else { - anyhow::bail!("Scripts must have a `Source` property that is a String") - }; + let contents = if let Some(Variant::String(source)) = new_inst.properties.get("Source") { + source.as_bytes().to_vec() + } else { + anyhow::bail!("Scripts must have a `Source` property that is a String") + }; + + let dir_syncback = syncback_dir_no_meta(snapshot)?; - let dir_syncback = syncback_dir(snapshot)?; + let mut meta = if let Some(dir) = dir_meta(snapshot.vfs(), &path)? { + dir + } else { + DirectoryMetadata { + ignore_unknown_instances: None, + class_name: None, + properties: HashMap::with_capacity(new_inst.properties.capacity()), + attributes: HashMap::new(), + path: snapshot + .parent_path + .join(&snapshot.name) + .join("init.meta.json"), + } + }; + for (name, value) in snapshot.get_filtered_properties() { + if name == "Source" { + continue; + } else if name == "Attributes" || name == "AttributesSerialize" { + if let Variant::Attributes(attrs) = value { + meta.attributes.extend(attrs.iter().map(|(name, value)| { + ( + name.to_string(), + UnresolvedValue::FullyQualified(value.clone()), + ) + })) + } else { + log::error!("Property {name} should be Attributes but is not"); + } + } else { + meta.properties.insert( + name.to_string(), + UnresolvedValue::from_variant(value.to_owned(), &new_inst.class, name), + ); + } + } let mut fs_snapshot = FsSnapshot::new(); fs_snapshot.push_file(path, contents); fs_snapshot.merge(dir_syncback.fs_snapshot); + if !meta.is_empty() { + fs_snapshot.push_file( + &meta.path, + serde_json::to_vec_pretty(&meta).context("could not serialize new init.meta.json")?, + ); + } + Ok(SyncbackReturn { inst_snapshot: InstanceSnapshot::from_instance(snapshot.new_inst()), fs_snapshot, From 6f7cac8439fc818295358cb9eba55fc94b7c0a8a Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 28 Dec 2023 12:06:54 -0800 Subject: [PATCH 076/366] Correct small clippy lint in get_best_middleware --- src/syncback/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/syncback/mod.rs b/src/syncback/mod.rs index 00838232a..9508ceb11 100644 --- a/src/syncback/mod.rs +++ b/src/syncback/mod.rs @@ -134,7 +134,7 @@ pub fn get_best_middleware(inst: &Instance) -> Middleware { } } "ModuleScript" => { - if inst.children().len() == 0 { + if inst.children().is_empty() { Middleware::ModuleScript } else { Middleware::ModuleScriptDir From 4238bc4a17fe1b6a1ceda148f52a38c7f1831396 Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 28 Dec 2023 12:07:14 -0800 Subject: [PATCH 077/366] Add functions in meta_file middleware for getting metadata --- src/snapshot_middleware/meta_file.rs | 37 +++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/src/snapshot_middleware/meta_file.rs b/src/snapshot_middleware/meta_file.rs index ed5958627..9eb3559c2 100644 --- a/src/snapshot_middleware/meta_file.rs +++ b/src/snapshot_middleware/meta_file.rs @@ -1,6 +1,11 @@ -use std::{borrow::Cow, collections::HashMap, path::PathBuf}; +use std::{ + borrow::Cow, + collections::HashMap, + path::{Path, PathBuf}, +}; use anyhow::{format_err, Context}; +use memofs::{IoResultExt as _, Vfs}; use rbx_dom_weak::types::Attributes; use serde::{Deserialize, Serialize}; @@ -193,3 +198,33 @@ impl DirectoryMetadata { } } } + +/// Retrieves the meta file that should be applied for the provided directory, +/// if it exists. +pub fn dir_meta(vfs: &Vfs, path: &Path) -> anyhow::Result> { + let meta_path = path.join("init.meta.json"); + + if let Some(meta_contents) = vfs.read(&meta_path).with_not_found()? { + let metadata = DirectoryMetadata::from_slice(&meta_contents, meta_path)?; + Ok(Some(metadata)) + } else { + Ok(None) + } +} + +/// Retrieves the meta file that should be applied for the provided file, +/// if it exists. +/// +/// The `name` field should be the name the metadata should have. +pub fn file_meta(vfs: &Vfs, path: &Path, name: &str) -> anyhow::Result> { + let mut meta_path = path.with_file_name(name); + meta_path.set_extension("meta.json"); + + log::debug!("metadata: {}", meta_path.display()); + if let Some(meta_contents) = vfs.read(&meta_path).with_not_found()? { + let metadata = AdjacentMetadata::from_slice(&meta_contents, meta_path)?; + Ok(Some(metadata)) + } else { + Ok(None) + } +} From 23b50696370f127a1550057e0219b50e8f41498c Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 28 Dec 2023 12:13:54 -0800 Subject: [PATCH 078/366] Minor cleanup to json_model syncback --- src/snapshot_middleware/json_model.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/snapshot_middleware/json_model.rs b/src/snapshot_middleware/json_model.rs index 4a8dfafda..6cb131ef4 100644 --- a/src/snapshot_middleware/json_model.rs +++ b/src/snapshot_middleware/json_model.rs @@ -63,17 +63,15 @@ pub fn syncback_json_model<'new, 'old>( let new_inst = snapshot.new_inst(); - let filtered = snapshot.get_filtered_properties(); - let mut properties = HashMap::with_capacity(new_inst.properties.capacity()); let mut attributes = HashMap::new(); - for (name, value) in filtered { + for (name, value) in snapshot.get_filtered_properties() { if name == "Attributes" || name == "AttributesSerialize" { if let Variant::Attributes(attr) = value { attributes.extend(attr.iter().map(|(name, value)| { ( name.to_string(), - UnresolvedValue::from_variant(value.clone(), &new_inst.class, &name), + UnresolvedValue::from_variant(value.clone(), &new_inst.class, name), ) })) } else { @@ -88,7 +86,7 @@ pub fn syncback_json_model<'new, 'old>( } else { properties.insert( name.to_string(), - UnresolvedValue::from_variant(value.clone(), &new_inst.class, &name), + UnresolvedValue::from_variant(value.clone(), &new_inst.class, name), ); } } From ffafc713b493888aaa60243d8f5fc5c11f237798 Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 28 Dec 2023 12:15:38 -0800 Subject: [PATCH 079/366] Support non-dir metadata also --- src/snapshot_middleware/lua.rs | 42 ++++++++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/src/snapshot_middleware/lua.rs b/src/snapshot_middleware/lua.rs index f57d39bcf..a87ca3ee8 100644 --- a/src/snapshot_middleware/lua.rs +++ b/src/snapshot_middleware/lua.rs @@ -12,7 +12,7 @@ use crate::{ use super::{ dir::{dir_meta, snapshot_dir_no_meta, syncback_dir_no_meta}, - meta_file::AdjacentMetadata, + meta_file::{file_meta, AdjacentMetadata}, DirectoryMetadata, }; @@ -129,7 +129,7 @@ pub fn syncback_lua<'new, 'old>( script_type: ScriptType, snapshot: &SyncbackSnapshot<'new, 'old>, ) -> anyhow::Result> { - let inst = snapshot.new_inst(); + let new_inst = snapshot.new_inst(); let mut path = snapshot.parent_path.join(&snapshot.name); path.set_extension(match script_type { @@ -137,14 +137,48 @@ pub fn syncback_lua<'new, 'old>( ScriptType::Client => "client.lua", ScriptType::Server => "server.lua", }); - let contents = if let Some(Variant::String(source)) = inst.properties.get("Source") { + let contents = if let Some(Variant::String(source)) = new_inst.properties.get("Source") { source.as_bytes().to_vec() } else { anyhow::bail!("Scripts must have a `Source` property that is a String") }; + let mut meta = if let Some(meta) = file_meta(snapshot.vfs(), &path, &snapshot.name)? { + meta + } else { + AdjacentMetadata { + ignore_unknown_instances: None, + properties: HashMap::with_capacity(new_inst.properties.capacity()), + attributes: HashMap::new(), + path: path + .with_file_name(&snapshot.name) + .with_extension("meta.json"), + } + }; + for (name, value) in snapshot.get_filtered_properties() { + if name == "Source" { + continue; + } else if name == "Attributes" || name == "AttributesSerialize" { + if let Variant::Attributes(attrs) = value { + meta.attributes.extend(attrs.iter().map(|(name, value)| { + ( + name.to_string(), + UnresolvedValue::FullyQualified(value.clone()), + ) + })) + } else { + log::error!("Property {name} should be Attributes but is not"); + } + } else { + meta.properties.insert( + name.to_string(), + UnresolvedValue::from_variant(value.to_owned(), &new_inst.class, name), + ); + } + } + Ok(SyncbackReturn { - inst_snapshot: InstanceSnapshot::from_instance(inst), + inst_snapshot: InstanceSnapshot::from_instance(new_inst), fs_snapshot: FsSnapshot::new().with_file(path, contents), // Scripts don't have a child! children: Vec::new(), From 01bcd76f81aa3234759fd95f0c22bf8d14cadf02 Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 28 Dec 2023 12:16:06 -0800 Subject: [PATCH 080/366] Remove logging from file_meta function --- src/snapshot_middleware/meta_file.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/snapshot_middleware/meta_file.rs b/src/snapshot_middleware/meta_file.rs index 9eb3559c2..c2969c06c 100644 --- a/src/snapshot_middleware/meta_file.rs +++ b/src/snapshot_middleware/meta_file.rs @@ -220,7 +220,6 @@ pub fn file_meta(vfs: &Vfs, path: &Path, name: &str) -> anyhow::Result Date: Thu, 28 Dec 2023 12:21:22 -0800 Subject: [PATCH 081/366] Why did I even have an Rc? --- src/snapshot_middleware/project.rs | 6 +++--- src/syncback/mod.rs | 9 +++------ src/syncback/snapshot.rs | 7 ++++--- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/snapshot_middleware/project.rs b/src/snapshot_middleware/project.rs index c02de3d69..9a5cda0b9 100644 --- a/src/snapshot_middleware/project.rs +++ b/src/snapshot_middleware/project.rs @@ -345,7 +345,7 @@ pub fn syncback_project<'new, 'old>( for (child_name, child_node) in &mut node.children { if let Some(new_child) = new_child_map.get(child_name.as_str()) { if let Some(old_child) = old_child_map.get(child_name.as_str()) { - if &new_child.class != old_child.class_name() { + if new_child.class != old_child.class_name() { anyhow::bail!("Cannot change the class of {child_name} in a project"); } for (name, value) in &new_child.properties { @@ -386,7 +386,7 @@ pub fn syncback_project<'new, 'old>( // All children are descendants of a node of a project // So we really just need to track which one is which. children.push(SyncbackSnapshot { - data: snapshot.data.clone(), + data: snapshot.data, old: Some(old_inst.id()), new: new_child.referent(), parent_path, @@ -396,7 +396,7 @@ pub fn syncback_project<'new, 'old>( } else { // it's new children.push(SyncbackSnapshot { - data: snapshot.data.clone(), + data: snapshot.data, old: None, new: new_child.referent(), parent_path, diff --git a/src/syncback/mod.rs b/src/syncback/mod.rs index 9508ceb11..59fa08d55 100644 --- a/src/syncback/mod.rs +++ b/src/syncback/mod.rs @@ -7,10 +7,7 @@ use rbx_dom_weak::{ Instance, WeakDom, }; use serde::{Deserialize, Serialize}; -use std::{ - collections::{HashMap, VecDeque}, - rc::Rc, -}; +use std::collections::{HashMap, VecDeque}; use crate::{ resolution::UnresolvedValue, @@ -33,11 +30,11 @@ pub fn syncback_loop<'old, 'new>( log::debug!("Hashing file DOM"); let new_hashes = hash_tree(new_tree); - let syncback_data = Rc::new(SyncbackData { + let syncback_data = SyncbackData { vfs, old_tree, new_tree, - }); + }; let mut snapshots = vec![SyncbackSnapshot { data: syncback_data, diff --git a/src/syncback/snapshot.rs b/src/syncback/snapshot.rs index 3a2db8634..cfe782730 100644 --- a/src/syncback/snapshot.rs +++ b/src/syncback/snapshot.rs @@ -1,5 +1,5 @@ use memofs::Vfs; -use std::{collections::HashMap, path::PathBuf, rc::Rc}; +use std::{collections::HashMap, path::PathBuf}; use crate::{ snapshot::{InstanceWithMeta, RojoTree}, @@ -10,6 +10,7 @@ use rbx_dom_weak::{ Instance, WeakDom, }; +#[derive(Clone, Copy)] pub struct SyncbackData<'new, 'old> { pub(super) vfs: &'old Vfs, pub(super) old_tree: &'old RojoTree, @@ -17,7 +18,7 @@ pub struct SyncbackData<'new, 'old> { } pub struct SyncbackSnapshot<'new, 'old> { - pub data: Rc>, + pub data: SyncbackData<'new, 'old>, pub old: Option, pub new: Ref, pub parent_path: PathBuf, @@ -30,7 +31,7 @@ impl<'new, 'old> SyncbackSnapshot<'new, 'old> { #[inline] pub fn from_parent(&self, new_name: String, new_ref: Ref, old_ref: Option) -> Self { Self { - data: Rc::clone(&self.data), + data: self.data, old: old_ref, new: new_ref, parent_path: self.parent_path.join(&self.name), From 1b4e992482f2765c9a9b566b8636b8cef2e077c3 Mon Sep 17 00:00:00 2001 From: Micah Date: Fri, 29 Dec 2023 11:09:53 -0800 Subject: [PATCH 082/366] Add method for checking if AdjacentMetadata is empty --- src/snapshot_middleware/meta_file.rs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/snapshot_middleware/meta_file.rs b/src/snapshot_middleware/meta_file.rs index c2969c06c..3cd31ea4a 100644 --- a/src/snapshot_middleware/meta_file.rs +++ b/src/snapshot_middleware/meta_file.rs @@ -83,6 +83,18 @@ impl AdjacentMetadata { Ok(()) } + /// Returns whether the metadata is 'empty', meaning it doesn't have anything + /// worth persisting in it. Specifically: + /// + /// - The number of properties and attributes is 0 + /// - `ignore_unknown_instances` is None + #[inline] + pub fn is_empty(&self) -> bool { + self.attributes.is_empty() + && self.properties.is_empty() + && self.ignore_unknown_instances.is_none() + } + // TODO: Add method to allow selectively applying parts of metadata and // throwing errors if invalid parts are specified. } @@ -188,8 +200,8 @@ impl DirectoryMetadata { /// - `class_name` is either None or not Some("Folder") #[inline] pub fn is_empty(&self) -> bool { - self.attributes.len() == 0 - && self.properties.len() == 0 + self.attributes.is_empty() + && self.properties.is_empty() && self.ignore_unknown_instances.is_none() && if let Some(class) = &self.class_name { class == "Folder" From 1e29dfed72cd5d0bd2bb2950b552ee6a42c2aa51 Mon Sep 17 00:00:00 2001 From: Micah Date: Fri, 29 Dec 2023 11:10:22 -0800 Subject: [PATCH 083/366] Add (incomplete) implementation of CSV syncback --- src/snapshot_middleware/csv.rs | 141 +++++++++++++++++++++++++++++++-- src/snapshot_middleware/mod.rs | 3 +- src/syncback/mod.rs | 7 ++ 3 files changed, 145 insertions(+), 6 deletions(-) diff --git a/src/snapshot_middleware/csv.rs b/src/snapshot_middleware/csv.rs index 594b5d53b..05ae716b5 100644 --- a/src/snapshot_middleware/csv.rs +++ b/src/snapshot_middleware/csv.rs @@ -1,15 +1,23 @@ -use std::{collections::BTreeMap, path::Path}; +use std::{ + collections::{BTreeMap, BTreeSet, HashMap}, + path::Path, +}; use anyhow::Context; use maplit::hashmap; use memofs::{IoResultExt, Vfs}; -use serde::Serialize; +use rbx_dom_weak::types::Variant; +use serde::{Deserialize, Serialize}; -use crate::snapshot::{InstanceContext, InstanceMetadata, InstanceSnapshot}; +use crate::{ + resolution::UnresolvedValue, + snapshot::{InstanceContext, InstanceMetadata, InstanceSnapshot}, + syncback::{FsSnapshot, SyncbackReturn, SyncbackSnapshot}, +}; use super::{ dir::{dir_meta, snapshot_dir_no_meta}, - meta_file::AdjacentMetadata, + meta_file::{file_meta, AdjacentMetadata}, }; pub fn snapshot_csv( @@ -89,11 +97,78 @@ pub fn snapshot_csv_init( Ok(Some(init_snapshot)) } +pub fn syncback_csv<'new, 'old>( + snapshot: &SyncbackSnapshot<'new, 'old>, +) -> anyhow::Result> { + let new_inst = snapshot.new_inst(); + + let mut path = snapshot.parent_path.join(&snapshot.name); + path.set_extension("csv"); + + let contents = if let Some(Variant::String(content)) = new_inst.properties.get("Contents") { + content.as_str() + } else { + anyhow::bail!("LocalizationTables must have a `Contents` property that is a String") + }; + + let mut meta = if let Some(meta) = file_meta(snapshot.vfs(), &path, &snapshot.name)? { + meta + } else { + AdjacentMetadata { + ignore_unknown_instances: None, + properties: HashMap::with_capacity(new_inst.properties.len()), + attributes: HashMap::new(), + path: path + .with_file_name(&snapshot.name) + .with_extension("meta.json"), + } + }; + for (name, value) in snapshot.get_filtered_properties() { + log::debug!("property: {name}"); + if name == "Contents" { + continue; + } else if name == "Attributes" || name == "AttributesSerialize" { + if let Variant::Attributes(attrs) = value { + meta.attributes.extend(attrs.iter().map(|(name, value)| { + ( + name.to_string(), + UnresolvedValue::FullyQualified(value.clone()), + ) + })) + } else { + log::error!("Property {name} should be Attributes but is not"); + } + } else { + meta.properties.insert( + name.to_string(), + UnresolvedValue::from_variant(value.to_owned(), &new_inst.class, name), + ); + } + } + + // TODO tags don't work, why? + let mut fs_snapshot = FsSnapshot::new(); + fs_snapshot.push_file(path, localization_to_csv(contents)?); + if !meta.is_empty() { + fs_snapshot.push_file( + &meta.path, + serde_json::to_vec_pretty(&meta).context("failed to reserialize metadata")?, + ) + } + + Ok(SyncbackReturn { + inst_snapshot: InstanceSnapshot::from_instance(new_inst), + fs_snapshot, + children: Vec::new(), + removed_children: Vec::new(), + }) +} + /// Struct that holds any valid row from a Roblox CSV translation table. /// /// We manually deserialize into this table from CSV, but let serde_json handle /// serialization. -#[derive(Debug, Default, Serialize)] +#[derive(Debug, Default, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] struct LocalizationEntry<'a> { #[serde(skip_serializing_if = "Option::is_none")] @@ -166,6 +241,62 @@ fn convert_localization_csv(contents: &[u8]) -> Result { Ok(encoded) } +/// Takes a localization table (as a string) and converts it into a CSV file. +/// +/// The CSV file is ordered, so it should be deterministic. +fn localization_to_csv(csv_contents: &str) -> anyhow::Result> { + let mut out = Vec::new(); + let mut writer = csv::Writer::from_writer(&mut out); + + // TODO deserialize this properly to dodge the bug listed above + let mut csv: Vec = + serde_json::from_str(csv_contents).context("cannot decode JSON from localization table")?; + + // TODO sort this better + csv.sort_unstable_by_key(|entry| entry.source); + + let mut headers = vec!["Key", "Source", "Context", "Example"]; + // We want both order and a lack of duplicates, so we use a BTreeSet. + let mut extra_headers = BTreeSet::new(); + for entry in &csv { + for lang in entry.values.keys() { + extra_headers.insert(*lang); + } + } + headers.extend(extra_headers.iter()); + + writer + .write_record(&headers) + .context("could not write headers for localization table")?; + + let mut record = Vec::with_capacity(headers.len()); + for entry in csv { + record.push(entry.key.unwrap_or_default()); + record.push(entry.source.unwrap_or_default()); + record.push(entry.context.unwrap_or_default()); + record.push(entry.example.unwrap_or_default()); + + let values = entry.values; + for header in &extra_headers { + record.push( + values + .get(header) + .context("missing header for localization table record")?, + ); + } + + writer + .write_record(&record) + .context("cannot write record for localization table")?; + record.clear(); + } + + // We must drop `writer`` here to regain access to `out`. + drop(writer); + + Ok(out) +} + #[cfg(test)] mod test { use super::*; diff --git a/src/snapshot_middleware/mod.rs b/src/snapshot_middleware/mod.rs index 818968b64..bb1fd3688 100644 --- a/src/snapshot_middleware/mod.rs +++ b/src/snapshot_middleware/mod.rs @@ -37,7 +37,7 @@ use crate::{ }; use self::{ - csv::{snapshot_csv, snapshot_csv_init}, + csv::{snapshot_csv, snapshot_csv_init, syncback_csv}, dir::{snapshot_dir, syncback_dir}, json::snapshot_json, json_model::{snapshot_json_model, syncback_json_model}, @@ -254,6 +254,7 @@ impl Middleware { ); } match self { + Middleware::Csv => syncback_csv(snapshot), Middleware::Project => syncback_project(snapshot), Middleware::ServerScript => syncback_lua(ScriptType::Server, snapshot), Middleware::ClientScript => syncback_lua(ScriptType::Client, snapshot), diff --git a/src/syncback/mod.rs b/src/syncback/mod.rs index 59fa08d55..c704ecb3c 100644 --- a/src/syncback/mod.rs +++ b/src/syncback/mod.rs @@ -137,6 +137,13 @@ pub fn get_best_middleware(inst: &Instance) -> Middleware { Middleware::ModuleScriptDir } } + "LocalizationTable" => { + if inst.children().is_empty() { + Middleware::Csv + } else { + Middleware::CsvDir + } + } _ => Middleware::Rbxm, } } From 37d9d727b2a803277f75a9c86dbc2c58e8aa1faa Mon Sep 17 00:00:00 2001 From: Micah Date: Fri, 29 Dec 2023 11:29:21 -0800 Subject: [PATCH 084/366] Use dir middleware for json models with children --- src/syncback/mod.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/syncback/mod.rs b/src/syncback/mod.rs index c704ecb3c..7833c9721 100644 --- a/src/syncback/mod.rs +++ b/src/syncback/mod.rs @@ -115,7 +115,15 @@ pub fn get_best_middleware(inst: &Instance) -> Middleware { | "BlurEffect" | "ColorCorrectionEffect" | "DepthOfFieldEffect" - | "SunRaysEffect" => Middleware::JsonModel, + | "SunRaysEffect" => { + if inst.children().is_empty() { + Middleware::JsonModel + } else { + // This begs the question of an init.model.json but we'll leave + // that for another day. + Middleware::Dir + } + } "Script" => { if inst.children().is_empty() { Middleware::ServerScript From 6e5c6b35256bae7f30e6c10a5f684e53e956dd3e Mon Sep 17 00:00:00 2001 From: Micah Date: Fri, 29 Dec 2023 11:33:54 -0800 Subject: [PATCH 085/366] Reorder and map out middleware matching --- src/snapshot_middleware/mod.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/snapshot_middleware/mod.rs b/src/snapshot_middleware/mod.rs index bb1fd3688..82bd25848 100644 --- a/src/snapshot_middleware/mod.rs +++ b/src/snapshot_middleware/mod.rs @@ -255,19 +255,22 @@ impl Middleware { } match self { Middleware::Csv => syncback_csv(snapshot), + Middleware::JsonModel => syncback_json_model(snapshot), + Middleware::Json => unimplemented!(), Middleware::Project => syncback_project(snapshot), Middleware::ServerScript => syncback_lua(ScriptType::Server, snapshot), Middleware::ClientScript => syncback_lua(ScriptType::Client, snapshot), Middleware::ModuleScript => syncback_lua(ScriptType::Module, snapshot), - Middleware::Rbxmx => syncback_rbxmx(snapshot), Middleware::Rbxm => syncback_rbxm(snapshot), + Middleware::Rbxmx => syncback_rbxmx(snapshot), + Middleware::Toml => unimplemented!(), Middleware::Text => syncback_txt(snapshot), - Middleware::JsonModel => syncback_json_model(snapshot), + Middleware::Ignore => anyhow::bail!("cannot syncback Ignore middleware"), Middleware::Dir => syncback_dir(snapshot), Middleware::ServerScriptDir => syncback_lua_init(ScriptType::Server, snapshot), Middleware::ClientScriptDir => syncback_lua_init(ScriptType::Client, snapshot), Middleware::ModuleScriptDir => syncback_lua_init(ScriptType::Module, snapshot), - _ => anyhow::bail!("cannot syncback with middleware {:?}", self), + Middleware::CsvDir => unimplemented!(), } } } From 27b4da48a6d585debf5e24eaec572c5754d8f3dd Mon Sep 17 00:00:00 2001 From: Micah Date: Fri, 29 Dec 2023 12:32:45 -0800 Subject: [PATCH 086/366] Use BTreeMaps instead of HashMaps when appropriate This ensures that the order of properties and attributes is consistent --- src/project.rs | 10 ++++----- src/snapshot_middleware/csv.rs | 6 +++--- src/snapshot_middleware/dir.rs | 6 +++--- src/snapshot_middleware/json_model.rs | 21 +++++++++++-------- src/snapshot_middleware/lua.rs | 14 ++++++++----- src/snapshot_middleware/meta_file.rs | 29 +++++++++++++++------------ 6 files changed, 49 insertions(+), 37 deletions(-) diff --git a/src/project.rs b/src/project.rs index fc9e4581d..660bfe31c 100644 --- a/src/project.rs +++ b/src/project.rs @@ -1,5 +1,5 @@ use std::{ - collections::{BTreeMap, HashMap, HashSet}, + collections::{BTreeMap, HashSet}, fs, io, net::IpAddr, path::{Path, PathBuf}, @@ -242,16 +242,16 @@ pub struct ProjectNode { #[serde( rename = "$properties", default, - skip_serializing_if = "HashMap::is_empty" + skip_serializing_if = "BTreeMap::is_empty" )] - pub properties: HashMap, + pub properties: BTreeMap, #[serde( rename = "$attributes", default, - skip_serializing_if = "HashMap::is_empty" + skip_serializing_if = "BTreeMap::is_empty" )] - pub attributes: HashMap, + pub attributes: BTreeMap, /// Defines the behavior when Rojo encounters unknown instances in Roblox /// Studio during live sync. `$ignoreUnknownInstances` should be considered diff --git a/src/snapshot_middleware/csv.rs b/src/snapshot_middleware/csv.rs index 05ae716b5..f5faa4fff 100644 --- a/src/snapshot_middleware/csv.rs +++ b/src/snapshot_middleware/csv.rs @@ -1,5 +1,5 @@ use std::{ - collections::{BTreeMap, BTreeSet, HashMap}, + collections::{BTreeMap, BTreeSet}, path::Path, }; @@ -116,8 +116,8 @@ pub fn syncback_csv<'new, 'old>( } else { AdjacentMetadata { ignore_unknown_instances: None, - properties: HashMap::with_capacity(new_inst.properties.len()), - attributes: HashMap::new(), + properties: BTreeMap::new(), + attributes: BTreeMap::new(), path: path .with_file_name(&snapshot.name) .with_extension("meta.json"), diff --git a/src/snapshot_middleware/dir.rs b/src/snapshot_middleware/dir.rs index 700036be9..d276b85f7 100644 --- a/src/snapshot_middleware/dir.rs +++ b/src/snapshot_middleware/dir.rs @@ -1,5 +1,5 @@ use std::{ - collections::{HashMap, HashSet}, + collections::{BTreeMap, HashMap, HashSet}, path::Path, }; @@ -108,8 +108,8 @@ pub fn syncback_dir<'new, 'old>( } else { DirectoryMetadata { ignore_unknown_instances: None, - properties: HashMap::with_capacity(new_inst.properties.capacity()), - attributes: HashMap::new(), + properties: BTreeMap::new(), + attributes: BTreeMap::new(), class_name: if new_inst.class == "Folder" { None } else { diff --git a/src/snapshot_middleware/json_model.rs b/src/snapshot_middleware/json_model.rs index 6cb131ef4..261444977 100644 --- a/src/snapshot_middleware/json_model.rs +++ b/src/snapshot_middleware/json_model.rs @@ -1,4 +1,9 @@ -use std::{borrow::Cow, collections::HashMap, path::Path, str}; +use std::{ + borrow::Cow, + collections::{BTreeMap, HashMap}, + path::Path, + str, +}; use anyhow::Context; use memofs::Vfs; @@ -63,8 +68,8 @@ pub fn syncback_json_model<'new, 'old>( let new_inst = snapshot.new_inst(); - let mut properties = HashMap::with_capacity(new_inst.properties.capacity()); - let mut attributes = HashMap::new(); + let mut properties = BTreeMap::new(); + let mut attributes = BTreeMap::new(); for (name, value) in snapshot.get_filtered_properties() { if name == "Attributes" || name == "AttributesSerialize" { if let Variant::Attributes(attr) = value { @@ -130,13 +135,13 @@ struct JsonModel { #[serde( alias = "Properties", - default = "HashMap::new", - skip_serializing_if = "HashMap::is_empty" + default = "BTreeMap::new", + skip_serializing_if = "BTreeMap::is_empty" )] - properties: HashMap, + properties: BTreeMap, - #[serde(default = "HashMap::new", skip_serializing_if = "HashMap::is_empty")] - attributes: HashMap, + #[serde(default = "BTreeMap::new", skip_serializing_if = "BTreeMap::is_empty")] + attributes: BTreeMap, } impl JsonModel { diff --git a/src/snapshot_middleware/lua.rs b/src/snapshot_middleware/lua.rs index a87ca3ee8..3fb5382c6 100644 --- a/src/snapshot_middleware/lua.rs +++ b/src/snapshot_middleware/lua.rs @@ -1,4 +1,8 @@ -use std::{collections::HashMap, path::Path, str}; +use std::{ + collections::{BTreeMap, HashMap}, + path::Path, + str, +}; use anyhow::Context; use memofs::{IoResultExt, Vfs}; @@ -148,8 +152,8 @@ pub fn syncback_lua<'new, 'old>( } else { AdjacentMetadata { ignore_unknown_instances: None, - properties: HashMap::with_capacity(new_inst.properties.capacity()), - attributes: HashMap::new(), + properties: BTreeMap::new(), + attributes: BTreeMap::new(), path: path .with_file_name(&snapshot.name) .with_extension("meta.json"), @@ -213,8 +217,8 @@ pub fn syncback_lua_init<'new, 'old>( DirectoryMetadata { ignore_unknown_instances: None, class_name: None, - properties: HashMap::with_capacity(new_inst.properties.capacity()), - attributes: HashMap::new(), + properties: BTreeMap::new(), + attributes: BTreeMap::new(), path: snapshot .parent_path .join(&snapshot.name) diff --git a/src/snapshot_middleware/meta_file.rs b/src/snapshot_middleware/meta_file.rs index 3cd31ea4a..058c10e11 100644 --- a/src/snapshot_middleware/meta_file.rs +++ b/src/snapshot_middleware/meta_file.rs @@ -1,6 +1,6 @@ use std::{ borrow::Cow, - collections::HashMap, + collections::BTreeMap, path::{Path, PathBuf}, }; @@ -21,11 +21,11 @@ pub struct AdjacentMetadata { #[serde(skip_serializing_if = "Option::is_none")] pub ignore_unknown_instances: Option, - #[serde(default, skip_serializing_if = "HashMap::is_empty")] - pub properties: HashMap, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub properties: BTreeMap, - #[serde(default, skip_serializing_if = "HashMap::is_empty")] - pub attributes: HashMap, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub attributes: BTreeMap, #[serde(skip)] pub path: PathBuf, @@ -53,7 +53,10 @@ impl AdjacentMetadata { pub fn apply_properties(&mut self, snapshot: &mut InstanceSnapshot) -> anyhow::Result<()> { let path = &self.path; - for (key, unresolved) in self.properties.drain() { + // BTreeMaps don't have an equivalent to HashMap::drain, so the next + // best option is to take ownership of the entire map. Not free, but + // very cheap. + for (key, unresolved) in std::mem::take(&mut self.properties) { let value = unresolved .resolve(&snapshot.class_name, &key) .with_context(|| format!("error applying meta file {}", path.display()))?; @@ -64,7 +67,7 @@ impl AdjacentMetadata { if !self.attributes.is_empty() { let mut attributes = Attributes::new(); - for (key, unresolved) in self.attributes.drain() { + for (key, unresolved) in std::mem::take(&mut self.attributes) { let value = unresolved.resolve_unambiguous()?; attributes.insert(key, value); } @@ -109,11 +112,11 @@ pub struct DirectoryMetadata { #[serde(skip_serializing_if = "Option::is_none")] pub ignore_unknown_instances: Option, - #[serde(default, skip_serializing_if = "HashMap::is_empty")] - pub properties: HashMap, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub properties: BTreeMap, - #[serde(default, skip_serializing_if = "HashMap::is_empty")] - pub attributes: HashMap, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub attributes: BTreeMap, #[serde(skip_serializing_if = "Option::is_none")] pub class_name: Option, @@ -168,7 +171,7 @@ impl DirectoryMetadata { fn apply_properties(&mut self, snapshot: &mut InstanceSnapshot) -> anyhow::Result<()> { let path = &self.path; - for (key, unresolved) in self.properties.drain() { + for (key, unresolved) in std::mem::take(&mut self.properties) { let value = unresolved .resolve(&snapshot.class_name, &key) .with_context(|| format!("error applying meta file {}", path.display()))?; @@ -179,7 +182,7 @@ impl DirectoryMetadata { if !self.attributes.is_empty() { let mut attributes = Attributes::new(); - for (key, unresolved) in self.attributes.drain() { + for (key, unresolved) in std::mem::take(&mut self.attributes) { let value = unresolved.resolve_unambiguous()?; attributes.insert(key, value); } From d6e7550643ee5a837a7864bd1683e945d6804726 Mon Sep 17 00:00:00 2001 From: Micah Date: Fri, 29 Dec 2023 16:31:39 -0800 Subject: [PATCH 087/366] Introduce cows to the CSV middleware syncback (moo) --- src/snapshot_middleware/csv.rs | 48 ++++++++++++++++------------------ 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/src/snapshot_middleware/csv.rs b/src/snapshot_middleware/csv.rs index f5faa4fff..978169ba6 100644 --- a/src/snapshot_middleware/csv.rs +++ b/src/snapshot_middleware/csv.rs @@ -1,4 +1,5 @@ use std::{ + borrow::Cow, collections::{BTreeMap, BTreeSet}, path::Path, }; @@ -172,19 +173,19 @@ pub fn syncback_csv<'new, 'old>( #[serde(rename_all = "camelCase")] struct LocalizationEntry<'a> { #[serde(skip_serializing_if = "Option::is_none")] - key: Option<&'a str>, + key: Option>, #[serde(skip_serializing_if = "Option::is_none")] - context: Option<&'a str>, + context: Option>, #[serde(skip_serializing_if = "Option::is_none")] - example: Option<&'a str>, + example: Option>, #[serde(skip_serializing_if = "Option::is_none")] - source: Option<&'a str>, + source: Option>, // We use a BTreeMap here to get deterministic output order. - values: BTreeMap<&'a str, &'a str>, + values: BTreeMap, Cow<'a, str>>, } /// Normally, we'd be able to let the csv crate construct our struct for us. @@ -218,12 +219,14 @@ fn convert_localization_csv(contents: &[u8]) -> Result { } match header { - "Key" => entry.key = Some(value), - "Source" => entry.source = Some(value), - "Context" => entry.context = Some(value), - "Example" => entry.example = Some(value), + "Key" => entry.key = Some(Cow::Borrowed(value)), + "Source" => entry.source = Some(Cow::Borrowed(value)), + "Context" => entry.context = Some(Cow::Borrowed(value)), + "Example" => entry.example = Some(Cow::Borrowed(value)), _ => { - entry.values.insert(header, value); + entry + .values + .insert(Cow::Borrowed(value), Cow::Borrowed(value)); } } } @@ -248,19 +251,18 @@ fn localization_to_csv(csv_contents: &str) -> anyhow::Result> { let mut out = Vec::new(); let mut writer = csv::Writer::from_writer(&mut out); - // TODO deserialize this properly to dodge the bug listed above let mut csv: Vec = serde_json::from_str(csv_contents).context("cannot decode JSON from localization table")?; // TODO sort this better - csv.sort_unstable_by_key(|entry| entry.source); + csv.sort_unstable_by(|a, b| a.source.partial_cmp(&b.source).unwrap()); let mut headers = vec!["Key", "Source", "Context", "Example"]; // We want both order and a lack of duplicates, so we use a BTreeSet. let mut extra_headers = BTreeSet::new(); for entry in &csv { for lang in entry.values.keys() { - extra_headers.insert(*lang); + extra_headers.insert(lang.as_ref()); } } headers.extend(extra_headers.iter()); @@ -269,20 +271,16 @@ fn localization_to_csv(csv_contents: &str) -> anyhow::Result> { .write_record(&headers) .context("could not write headers for localization table")?; - let mut record = Vec::with_capacity(headers.len()); - for entry in csv { - record.push(entry.key.unwrap_or_default()); - record.push(entry.source.unwrap_or_default()); - record.push(entry.context.unwrap_or_default()); - record.push(entry.example.unwrap_or_default()); + let mut record: Vec<&str> = Vec::with_capacity(headers.len()); + for entry in &csv { + record.push(entry.key.as_ref().unwrap_or(&Cow::Borrowed(""))); + record.push(entry.source.as_ref().unwrap_or(&Cow::Borrowed(""))); + record.push(entry.context.as_ref().unwrap_or(&Cow::Borrowed(""))); + record.push(entry.example.as_ref().unwrap_or(&Cow::Borrowed(""))); - let values = entry.values; + let values = &entry.values; for header in &extra_headers { - record.push( - values - .get(header) - .context("missing header for localization table record")?, - ); + record.push(values.get(*header).unwrap_or(&Cow::Borrowed(""))); } writer From 869aa4bf66ba23d4c41565e0a08e21ab79ddc764 Mon Sep 17 00:00:00 2001 From: Micah Date: Fri, 29 Dec 2023 16:38:58 -0800 Subject: [PATCH 088/366] Support metadata for text middleware --- src/snapshot_middleware/txt.rs | 45 ++++++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/src/snapshot_middleware/txt.rs b/src/snapshot_middleware/txt.rs index c6fa0b34e..53928d59c 100644 --- a/src/snapshot_middleware/txt.rs +++ b/src/snapshot_middleware/txt.rs @@ -1,4 +1,4 @@ -use std::{path::Path, str}; +use std::{collections::BTreeMap, path::Path, str}; use anyhow::Context; use maplit::hashmap; @@ -6,11 +6,12 @@ use memofs::{IoResultExt, Vfs}; use rbx_dom_weak::types::Variant; use crate::{ + resolution::UnresolvedValue, snapshot::{InstanceContext, InstanceMetadata, InstanceSnapshot}, syncback::{FsSnapshot, SyncbackReturn, SyncbackSnapshot}, }; -use super::meta_file::AdjacentMetadata; +use super::meta_file::{file_meta, AdjacentMetadata}; pub fn snapshot_txt( context: &InstanceContext, @@ -51,18 +52,52 @@ pub fn snapshot_txt( pub fn syncback_txt<'new, 'old>( snapshot: &SyncbackSnapshot<'new, 'old>, ) -> anyhow::Result> { - let inst = snapshot.new_inst(); + let new_inst = snapshot.new_inst(); let mut path = snapshot.parent_path.join(&snapshot.name); path.set_extension("txt"); - let contents = if let Some(Variant::String(source)) = inst.properties.get("Value") { + let contents = if let Some(Variant::String(source)) = new_inst.properties.get("Value") { source.as_bytes().to_vec() } else { anyhow::bail!("StringValues must have a `Value` property that is a String"); }; + let mut meta = if let Some(meta) = file_meta(snapshot.vfs(), &path, &snapshot.name)? { + meta + } else { + AdjacentMetadata { + ignore_unknown_instances: None, + properties: BTreeMap::new(), + attributes: BTreeMap::new(), + path: path + .with_file_name(&snapshot.name) + .with_extension("meta.json"), + } + }; + for (name, value) in snapshot.get_filtered_properties() { + if name == "Value" { + continue; + } else if name == "Attributes" || name == "AttributesSerialize" { + if let Variant::Attributes(attrs) = value { + meta.attributes.extend(attrs.iter().map(|(name, value)| { + ( + name.to_string(), + UnresolvedValue::FullyQualified(value.clone()), + ) + })) + } else { + log::error!("Property {name} should be Attributes but is not"); + } + } else { + meta.properties.insert( + name.to_string(), + UnresolvedValue::from_variant(value.to_owned(), &new_inst.class, name), + ); + } + } + Ok(SyncbackReturn { - inst_snapshot: InstanceSnapshot::from_instance(inst), + inst_snapshot: InstanceSnapshot::from_instance(new_inst), fs_snapshot: FsSnapshot::new().with_file(path, contents), children: Vec::new(), removed_children: Vec::new(), From 408267c48a318b40ce4438c39169930bc252408d Mon Sep 17 00:00:00 2001 From: Micah Date: Fri, 29 Dec 2023 16:40:12 -0800 Subject: [PATCH 089/366] Add StringValue to get_best_middleware --- src/syncback/mod.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/syncback/mod.rs b/src/syncback/mod.rs index 7833c9721..c389389e1 100644 --- a/src/syncback/mod.rs +++ b/src/syncback/mod.rs @@ -124,6 +124,13 @@ pub fn get_best_middleware(inst: &Instance) -> Middleware { Middleware::Dir } } + "StringValue" => { + if inst.children().is_empty() { + Middleware::Text + } else { + Middleware::Dir + } + } "Script" => { if inst.children().is_empty() { Middleware::ServerScript From 003da67e9f1fca4ac439e5f290a0c021b7b3394d Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 2 Jan 2024 08:15:55 -0800 Subject: [PATCH 090/366] Add basic changelog entry for syncback --- CHANGELOG.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70a7d5a81..7e4021a7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,30 @@ # Rojo Changelog ## Unreleased Changes +* A new command `rojo syncback` has been added. It can be used as `rojo syncback [path to project] --input [path to file]`. + This command takes a Roblox file and pulls Instances out of it and places them in the correct position in the provided project. + Syncback is primarily controlled by the project file. Any Instances who are either referenced in the project file or a descendant + of one that is will be placed in an appropriate location. + + In addition, a new field has been added to project files, `syncbackRules` to control how it behaves: + + ```json + { + "syncbackRules": { + "ignorePaths": [ + "Workspace/Camera", + "ServerStorage/ImportantSecrets/**", + ], + "ignoreProperties": { + "Part": ["Color"] + }, + "defaultOverrides": { + "Part": + } + } + } + ``` + * Added popout diff visualizer for table properties like Attributes and Tags ([#834]) * Updated Theme to use Studio colors ([#838]) * Projects may now specify rules for syncing files as if they had a different file extension. ([#813]) From da9a81489517f9dd847cdac5eb4dc6688a5d0eaa Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 2 Jan 2024 09:44:45 -0800 Subject: [PATCH 091/366] Derive Default + document FsSnapshot --- src/syncback/fs_snapshot.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/syncback/fs_snapshot.rs b/src/syncback/fs_snapshot.rs index 27fb272af..6aa05e2cf 100644 --- a/src/syncback/fs_snapshot.rs +++ b/src/syncback/fs_snapshot.rs @@ -7,6 +7,12 @@ use std::{ use memofs::Vfs; +/// A simple representation of a subsection of a file system. +/// +/// This is distinct from the snapshot provided by memofs (`VfsSnapshot`) due +/// to the need to support writing to the file system instead of an in-memory +/// file system. +#[derive(Default)] pub struct FsSnapshot { files: HashMap>>, dir: HashSet, From d35c4bbebced5276e983cbac73e1424030d3a30f Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 2 Jan 2024 09:48:44 -0800 Subject: [PATCH 092/366] Implement CsvDir snapshot --- src/snapshot_middleware/csv.rs | 74 ++++++++++++++++++++++++++++++++-- src/snapshot_middleware/mod.rs | 8 ++-- 2 files changed, 74 insertions(+), 8 deletions(-) diff --git a/src/snapshot_middleware/csv.rs b/src/snapshot_middleware/csv.rs index 978169ba6..c60c1d648 100644 --- a/src/snapshot_middleware/csv.rs +++ b/src/snapshot_middleware/csv.rs @@ -17,8 +17,9 @@ use crate::{ }; use super::{ - dir::{dir_meta, snapshot_dir_no_meta}, + dir::{dir_meta, snapshot_dir_no_meta, syncback_dir_no_meta}, meta_file::{file_meta, AdjacentMetadata}, + DirectoryMetadata, }; pub fn snapshot_csv( @@ -125,7 +126,6 @@ pub fn syncback_csv<'new, 'old>( } }; for (name, value) in snapshot.get_filtered_properties() { - log::debug!("property: {name}"); if name == "Contents" { continue; } else if name == "Attributes" || name == "AttributesSerialize" { @@ -165,6 +165,74 @@ pub fn syncback_csv<'new, 'old>( }) } +pub fn syncback_csv_init<'new, 'old>( + snapshot: &SyncbackSnapshot<'new, 'old>, +) -> anyhow::Result> { + let new_inst = snapshot.new_inst(); + + let mut path = snapshot.parent_path.join(&snapshot.name); + path.push("init.csv"); + + let contents = if let Some(Variant::String(content)) = new_inst.properties.get("Contents") { + content.as_str() + } else { + anyhow::bail!("LocalizationTables must have a `Contents` property that is a String") + }; + + let mut dir_syncback = syncback_dir_no_meta(snapshot)?; + let mut meta = if let Some(dir) = dir_meta(snapshot.vfs(), &path)? { + dir + } else { + DirectoryMetadata { + ignore_unknown_instances: None, + class_name: None, + properties: BTreeMap::new(), + attributes: BTreeMap::new(), + path: snapshot + .parent_path + .join(&snapshot.name) + .join("init.meta.json"), + } + }; + for (name, value) in snapshot.get_filtered_properties() { + if name == "Contents" { + continue; + } else if name == "Attributes" || name == "AttributesSerialize" { + if let Variant::Attributes(attrs) = value { + meta.attributes.extend(attrs.iter().map(|(name, value)| { + ( + name.to_string(), + UnresolvedValue::FullyQualified(value.clone()), + ) + })) + } else { + log::error!("Property {name} should be Attributes but is not"); + } + } else { + meta.properties.insert( + name.to_string(), + UnresolvedValue::from_variant(value.to_owned(), &new_inst.class, name), + ); + } + } + + let mut fs_snapshot = std::mem::take(&mut dir_syncback.fs_snapshot); + fs_snapshot.push_file(&path, localization_to_csv(contents)?); + if !meta.is_empty() { + fs_snapshot.push_file( + &meta.path, + serde_json::to_vec_pretty(&meta).context("could not serialize new init.meta.json")?, + ); + } + + Ok(SyncbackReturn { + inst_snapshot: InstanceSnapshot::from_instance(new_inst), + fs_snapshot, + children: dir_syncback.children, + removed_children: dir_syncback.removed_children, + }) +} + /// Struct that holds any valid row from a Roblox CSV translation table. /// /// We manually deserialize into this table from CSV, but let serde_json handle @@ -289,7 +357,7 @@ fn localization_to_csv(csv_contents: &str) -> anyhow::Result> { record.clear(); } - // We must drop `writer`` here to regain access to `out`. + // We must drop `writer` here to regain access to `out`. drop(writer); Ok(out) diff --git a/src/snapshot_middleware/mod.rs b/src/snapshot_middleware/mod.rs index 82bd25848..9e66991a4 100644 --- a/src/snapshot_middleware/mod.rs +++ b/src/snapshot_middleware/mod.rs @@ -37,7 +37,7 @@ use crate::{ }; use self::{ - csv::{snapshot_csv, snapshot_csv_init, syncback_csv}, + csv::{snapshot_csv, snapshot_csv_init, syncback_csv, syncback_csv_init}, dir::{snapshot_dir, syncback_dir}, json::snapshot_json, json_model::{snapshot_json_model, syncback_json_model}, @@ -50,9 +50,7 @@ use self::{ }; pub use self::{ - lua::ScriptType, - meta_file::{AdjacentMetadata, DirectoryMetadata}, - project::snapshot_project_node, + lua::ScriptType, meta_file::DirectoryMetadata, project::snapshot_project_node, util::emit_legacy_scripts_default, }; @@ -270,7 +268,7 @@ impl Middleware { Middleware::ServerScriptDir => syncback_lua_init(ScriptType::Server, snapshot), Middleware::ClientScriptDir => syncback_lua_init(ScriptType::Client, snapshot), Middleware::ModuleScriptDir => syncback_lua_init(ScriptType::Module, snapshot), - Middleware::CsvDir => unimplemented!(), + Middleware::CsvDir => syncback_csv_init(snapshot), } } } From 5ffd7a067c051e634bfcb05544dfcb9f706fdc2e Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 2 Jan 2024 10:15:37 -0800 Subject: [PATCH 093/366] Properly handle Json and Toml for the moment --- src/snapshot_middleware/mod.rs | 4 ++-- src/syncback/mod.rs | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/snapshot_middleware/mod.rs b/src/snapshot_middleware/mod.rs index 9e66991a4..8e8022cfe 100644 --- a/src/snapshot_middleware/mod.rs +++ b/src/snapshot_middleware/mod.rs @@ -254,14 +254,14 @@ impl Middleware { match self { Middleware::Csv => syncback_csv(snapshot), Middleware::JsonModel => syncback_json_model(snapshot), - Middleware::Json => unimplemented!(), + Middleware::Json => unimplemented!("cannot syncback Json middleware"), Middleware::Project => syncback_project(snapshot), Middleware::ServerScript => syncback_lua(ScriptType::Server, snapshot), Middleware::ClientScript => syncback_lua(ScriptType::Client, snapshot), Middleware::ModuleScript => syncback_lua(ScriptType::Module, snapshot), Middleware::Rbxm => syncback_rbxm(snapshot), Middleware::Rbxmx => syncback_rbxmx(snapshot), - Middleware::Toml => unimplemented!(), + Middleware::Toml => unimplemented!("cannot syncback Toml middleware"), Middleware::Text => syncback_txt(snapshot), Middleware::Ignore => anyhow::bail!("cannot syncback Ignore middleware"), Middleware::Dir => syncback_dir(snapshot), diff --git a/src/syncback/mod.rs b/src/syncback/mod.rs index c389389e1..2843bbaa2 100644 --- a/src/syncback/mod.rs +++ b/src/syncback/mod.rs @@ -81,6 +81,14 @@ pub fn syncback_loop<'old, 'new>( middleware ); + if matches!(middleware, Middleware::Json | Middleware::Toml) { + log::warn!( + "Cannot syncback {middleware:?} at {}, skipping", + get_inst_path(new_tree, snapshot.new) + ); + continue; + } + let syncback = middleware.syncback(&snapshot)?; if let Some(old_inst) = snapshot.old_inst() { From 18db6a1a6d1b05d1386043c5cf054c383445a5a8 Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 2 Jan 2024 11:47:12 -0800 Subject: [PATCH 094/366] More properly handle syncbackRules --- src/project.rs | 6 ++--- src/syncback/mod.rs | 47 +++++++++++++--------------------------- src/syncback/snapshot.rs | 32 ++++++++++++++++++++++++--- 3 files changed, 46 insertions(+), 39 deletions(-) diff --git a/src/project.rs b/src/project.rs index 660bfe31c..768335f64 100644 --- a/src/project.rs +++ b/src/project.rs @@ -8,9 +8,7 @@ use std::{ use serde::{Deserialize, Serialize}; use thiserror::Error; -use crate::{ - glob::Glob, resolution::UnresolvedValue, snapshot::SyncRule, syncback::SyncbackIgnoreRules, -}; +use crate::{glob::Glob, resolution::UnresolvedValue, snapshot::SyncRule, syncback::SyncbackRules}; static PROJECT_FILENAME: &str = "default.project.json"; @@ -88,7 +86,7 @@ pub struct Project { /// A list of rules for syncback with this project file. #[serde(skip_serializing_if = "Option::is_none")] - pub syncback_rules: Option, + pub syncback_rules: Option, /// A list of mappings of globs to syncing rules. If a file matches a glob, /// it will be 'transformed' into an Instance following the rule provided. diff --git a/src/syncback/mod.rs b/src/syncback/mod.rs index 2843bbaa2..69fdd2731 100644 --- a/src/syncback/mod.rs +++ b/src/syncback/mod.rs @@ -2,15 +2,11 @@ mod fs_snapshot; mod snapshot; use memofs::Vfs; -use rbx_dom_weak::{ - types::{Ref, Variant}, - Instance, WeakDom, -}; +use rbx_dom_weak::{types::Ref, Instance, WeakDom}; use serde::{Deserialize, Serialize}; use std::collections::{HashMap, VecDeque}; use crate::{ - resolution::UnresolvedValue, snapshot::{hash_tree, InstanceSnapshot, InstanceWithMeta, RojoTree}, snapshot_middleware::Middleware, Project, @@ -34,6 +30,10 @@ pub fn syncback_loop<'old, 'new>( vfs, old_tree, new_tree, + ignore_props: project + .syncback_rules + .as_ref() + .map(|rules| &rules.ignore_properties), }; let mut snapshots = vec![SyncbackSnapshot { @@ -172,41 +172,24 @@ pub fn get_best_middleware(inst: &Instance) -> Middleware { } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct SyncbackIgnoreRules { +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct SyncbackRules { + /// A list of paths in a file that will be ignored by Syncback. + #[serde(default)] + ignore_paths: Vec, + /// A map of classes to properties to ignore for that class when doing + /// syncback. #[serde(default)] - paths: Vec, - #[serde(default, skip)] - classes: HashMap>, + ignore_properties: HashMap>, } -impl SyncbackIgnoreRules { - /// If possible, resolves all of the properties in the ignore rules so that - /// they're Variants. - pub fn resolve(&self) -> anyhow::Result>> { - let mut resolved = HashMap::with_capacity(self.classes.capacity()); - - for (class_name, properties) in &self.classes { - let mut resolved_props = HashMap::with_capacity(properties.capacity()); - for (prop_name, prop_value) in properties { - resolved_props.insert( - prop_name.as_str(), - prop_value.clone().resolve(class_name, prop_name)?, - ); - } - - resolved.insert(class_name.as_str(), resolved_props); - } - - Ok(resolved) - } - +impl SyncbackRules { /// Returns whether the provided Instance is allowed to be handled with /// syncback. #[inline] pub fn acceptable(&self, dom: &WeakDom, inst: Ref) -> bool { let path = get_inst_path(dom, inst); - for ignored in &self.paths { + for ignored in &self.ignore_paths { if path.starts_with(ignored.as_str()) { return false; } diff --git a/src/syncback/snapshot.rs b/src/syncback/snapshot.rs index cfe782730..1c473a298 100644 --- a/src/syncback/snapshot.rs +++ b/src/syncback/snapshot.rs @@ -1,5 +1,8 @@ use memofs::Vfs; -use std::{collections::HashMap, path::PathBuf}; +use std::{ + collections::{BTreeSet, HashMap}, + path::PathBuf, +}; use crate::{ snapshot::{InstanceWithMeta, RojoTree}, @@ -15,6 +18,7 @@ pub struct SyncbackData<'new, 'old> { pub(super) vfs: &'old Vfs, pub(super) old_tree: &'old RojoTree, pub(super) new_tree: &'new WeakDom, + pub(super) ignore_props: Option<&'old HashMap>>, } pub struct SyncbackSnapshot<'new, 'old> { @@ -42,13 +46,16 @@ impl<'new, 'old> SyncbackSnapshot<'new, 'old> { /// Returns a map of properties for the 'new' Instance with filtering /// done to avoid noise. /// - /// Note that the returned map does not filter any properties by name, nor - /// does it clone the values. This is left to the consumer. + /// Note that while the returned map does filter based on the user's + /// `ignore_props` field, it does not do any other filtering and doesn't + /// clone any data. This is left to the consumer. pub fn get_filtered_properties(&self) -> HashMap<&'new str, &'new Variant> { let new_inst = self.new_inst(); let mut properties: HashMap<&str, &Variant> = HashMap::with_capacity(new_inst.properties.capacity()); + let filter = self.get_property_filter(); + if let Some(old_inst) = self.old_inst() { for (name, value) in &new_inst.properties { if old_inst.properties().contains_key(name) { @@ -66,6 +73,11 @@ impl<'new, 'old> SyncbackSnapshot<'new, 'old> { if matches!(value, Variant::Ref(_) | Variant::SharedString(_)) { continue; } + if let Some(list) = &filter { + if list.contains(name) { + continue; + } + } if let Some(default) = defaults.get(name.as_str()) { if !variant_eq(value, default) { properties.insert(name, value); @@ -80,6 +92,11 @@ impl<'new, 'old> SyncbackSnapshot<'new, 'old> { if matches!(value, Variant::Ref(_) | Variant::SharedString(_)) { continue; } + if let Some(list) = &filter { + if list.contains(name) { + continue; + } + } properties.insert(name, value); } } @@ -88,6 +105,15 @@ impl<'new, 'old> SyncbackSnapshot<'new, 'old> { properties } + /// Returns a set of properties that should not be written with syncback if + /// one exists. + fn get_property_filter(&self) -> Option> { + self.data + .ignore_props + .and_then(|filter| filter.get(&self.new_inst().class)) + .map(|list| list.iter().collect()) + } + /// Returns an Instance from the old tree with the provided referent, if it /// exists. #[inline] From 858ea474fe9485a08a3a9681088dd3af77cb12a3 Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 2 Jan 2024 11:47:36 -0800 Subject: [PATCH 095/366] Update changelog entry --- CHANGELOG.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e4021a7a..8d6a254c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,9 +18,6 @@ "ignoreProperties": { "Part": ["Color"] }, - "defaultOverrides": { - "Part": - } } } ``` From 13392ba27fdf21505058a6f79ea90d8d526eed96 Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 2 Jan 2024 12:19:06 -0800 Subject: [PATCH 096/366] Make `ignoreProperties` respect inheritance --- src/syncback/snapshot.rs | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/src/syncback/snapshot.rs b/src/syncback/snapshot.rs index 1c473a298..0f668e108 100644 --- a/src/syncback/snapshot.rs +++ b/src/syncback/snapshot.rs @@ -108,10 +108,26 @@ impl<'new, 'old> SyncbackSnapshot<'new, 'old> { /// Returns a set of properties that should not be written with syncback if /// one exists. fn get_property_filter(&self) -> Option> { - self.data - .ignore_props - .and_then(|filter| filter.get(&self.new_inst().class)) - .map(|list| list.iter().collect()) + let filter = self.data.ignore_props?; + let mut set = BTreeSet::new(); + + let database = rbx_reflection_database::get(); + let mut current_class_name = self.new_inst().class.as_str(); + + loop { + if let Some(list) = filter.get(current_class_name) { + set.extend(list) + } + + let class = database.classes.get(current_class_name)?; + if let Some(super_class) = class.superclass.as_ref() { + current_class_name = &super_class; + } else { + break; + } + } + + Some(set) } /// Returns an Instance from the old tree with the provided referent, if it From 20af38292672d212ff48464364818798ca5b34bb Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 2 Jan 2024 12:40:12 -0800 Subject: [PATCH 097/366] Give FsSnapshot a facelift --- src/snapshot_middleware/csv.rs | 8 +-- src/snapshot_middleware/dir.rs | 4 +- src/snapshot_middleware/json_model.rs | 2 +- src/snapshot_middleware/lua.rs | 6 +-- src/snapshot_middleware/project.rs | 2 +- src/snapshot_middleware/rbxm.rs | 2 +- src/snapshot_middleware/rbxmx.rs | 2 +- src/snapshot_middleware/txt.rs | 2 +- src/syncback/fs_snapshot.rs | 70 +++++++++++++++------------ 9 files changed, 54 insertions(+), 44 deletions(-) diff --git a/src/snapshot_middleware/csv.rs b/src/snapshot_middleware/csv.rs index c60c1d648..1867fe8f6 100644 --- a/src/snapshot_middleware/csv.rs +++ b/src/snapshot_middleware/csv.rs @@ -149,9 +149,9 @@ pub fn syncback_csv<'new, 'old>( // TODO tags don't work, why? let mut fs_snapshot = FsSnapshot::new(); - fs_snapshot.push_file(path, localization_to_csv(contents)?); + fs_snapshot.add_file(path, localization_to_csv(contents)?); if !meta.is_empty() { - fs_snapshot.push_file( + fs_snapshot.add_file( &meta.path, serde_json::to_vec_pretty(&meta).context("failed to reserialize metadata")?, ) @@ -217,9 +217,9 @@ pub fn syncback_csv_init<'new, 'old>( } let mut fs_snapshot = std::mem::take(&mut dir_syncback.fs_snapshot); - fs_snapshot.push_file(&path, localization_to_csv(contents)?); + fs_snapshot.add_file(&path, localization_to_csv(contents)?); if !meta.is_empty() { - fs_snapshot.push_file( + fs_snapshot.add_file( &meta.path, serde_json::to_vec_pretty(&meta).context("could not serialize new init.meta.json")?, ); diff --git a/src/snapshot_middleware/dir.rs b/src/snapshot_middleware/dir.rs index d276b85f7..daff31ca6 100644 --- a/src/snapshot_middleware/dir.rs +++ b/src/snapshot_middleware/dir.rs @@ -139,7 +139,7 @@ pub fn syncback_dir<'new, 'old>( } if !meta.is_empty() { - dir_syncback.fs_snapshot.push_file( + dir_syncback.fs_snapshot.add_file( &meta.path, serde_json::to_vec_pretty(&meta).context("could not serialize new init.meta.json")?, ); @@ -206,7 +206,7 @@ pub fn syncback_dir_no_meta<'new, 'old>( Ok(SyncbackReturn { inst_snapshot: InstanceSnapshot::from_instance(new_inst), - fs_snapshot: FsSnapshot::new().with_dir(path), + fs_snapshot: FsSnapshot::new().with_added_dir(path), children, removed_children, }) diff --git a/src/snapshot_middleware/json_model.rs b/src/snapshot_middleware/json_model.rs index 261444977..97a1ed97c 100644 --- a/src/snapshot_middleware/json_model.rs +++ b/src/snapshot_middleware/json_model.rs @@ -108,7 +108,7 @@ pub fn syncback_json_model<'new, 'old>( Ok(SyncbackReturn { inst_snapshot: InstanceSnapshot::from_instance(new_inst), - fs_snapshot: FsSnapshot::new().with_file( + fs_snapshot: FsSnapshot::new().with_added_file( &path, serde_json::to_vec_pretty(&model).context("failed to serialize new JSON Model")?, ), diff --git a/src/snapshot_middleware/lua.rs b/src/snapshot_middleware/lua.rs index 3fb5382c6..ead6ecb54 100644 --- a/src/snapshot_middleware/lua.rs +++ b/src/snapshot_middleware/lua.rs @@ -183,7 +183,7 @@ pub fn syncback_lua<'new, 'old>( Ok(SyncbackReturn { inst_snapshot: InstanceSnapshot::from_instance(new_inst), - fs_snapshot: FsSnapshot::new().with_file(path, contents), + fs_snapshot: FsSnapshot::new().with_added_file(path, contents), // Scripts don't have a child! children: Vec::new(), removed_children: Vec::new(), @@ -248,11 +248,11 @@ pub fn syncback_lua_init<'new, 'old>( } let mut fs_snapshot = FsSnapshot::new(); - fs_snapshot.push_file(path, contents); + fs_snapshot.add_file(path, contents); fs_snapshot.merge(dir_syncback.fs_snapshot); if !meta.is_empty() { - fs_snapshot.push_file( + fs_snapshot.add_file( &meta.path, serde_json::to_vec_pretty(&meta).context("could not serialize new init.meta.json")?, ); diff --git a/src/snapshot_middleware/project.rs b/src/snapshot_middleware/project.rs index 9a5cda0b9..b42a2a938 100644 --- a/src/snapshot_middleware/project.rs +++ b/src/snapshot_middleware/project.rs @@ -412,7 +412,7 @@ pub fn syncback_project<'new, 'old>( // already exist on the file system Ok(SyncbackReturn { inst_snapshot: InstanceSnapshot::from_instance(snapshot.new_inst()), - fs_snapshot: FsSnapshot::new().with_file( + fs_snapshot: FsSnapshot::new().with_added_file( &project.file_location, serde_json::to_vec_pretty(&project).context("failed to serialize new project")?, ), diff --git a/src/snapshot_middleware/rbxm.rs b/src/snapshot_middleware/rbxm.rs index b8137b710..5d392a5d2 100644 --- a/src/snapshot_middleware/rbxm.rs +++ b/src/snapshot_middleware/rbxm.rs @@ -57,7 +57,7 @@ pub fn syncback_rbxm<'new, 'old>( Ok(SyncbackReturn { inst_snapshot: InstanceSnapshot::from_instance(inst), - fs_snapshot: FsSnapshot::new().with_file(&path, serialized), + fs_snapshot: FsSnapshot::new().with_added_file(&path, serialized), children: Vec::new(), removed_children: Vec::new(), }) diff --git a/src/snapshot_middleware/rbxmx.rs b/src/snapshot_middleware/rbxmx.rs index 1b7ecdca7..3aa3931fb 100644 --- a/src/snapshot_middleware/rbxmx.rs +++ b/src/snapshot_middleware/rbxmx.rs @@ -60,7 +60,7 @@ pub fn syncback_rbxmx<'new, 'old>( Ok(SyncbackReturn { inst_snapshot: InstanceSnapshot::from_instance(inst), - fs_snapshot: FsSnapshot::new().with_file(&path, serialized), + fs_snapshot: FsSnapshot::new().with_added_file(&path, serialized), children: Vec::new(), removed_children: Vec::new(), }) diff --git a/src/snapshot_middleware/txt.rs b/src/snapshot_middleware/txt.rs index 53928d59c..b4827421d 100644 --- a/src/snapshot_middleware/txt.rs +++ b/src/snapshot_middleware/txt.rs @@ -98,7 +98,7 @@ pub fn syncback_txt<'new, 'old>( Ok(SyncbackReturn { inst_snapshot: InstanceSnapshot::from_instance(new_inst), - fs_snapshot: FsSnapshot::new().with_file(path, contents), + fs_snapshot: FsSnapshot::new().with_added_file(path, contents), children: Vec::new(), removed_children: Vec::new(), }) diff --git a/src/syncback/fs_snapshot.rs b/src/syncback/fs_snapshot.rs index 6aa05e2cf..22c4b6583 100644 --- a/src/syncback/fs_snapshot.rs +++ b/src/syncback/fs_snapshot.rs @@ -2,64 +2,74 @@ use std::{ collections::{HashMap, HashSet}, fmt, io, path::{Path, PathBuf}, - sync::Arc, }; use memofs::Vfs; /// A simple representation of a subsection of a file system. -/// -/// This is distinct from the snapshot provided by memofs (`VfsSnapshot`) due -/// to the need to support writing to the file system instead of an in-memory -/// file system. #[derive(Default)] pub struct FsSnapshot { - files: HashMap>>, - dir: HashSet, + /// Paths representing new files mapped to their contents. + add_files: HashMap>, + /// Paths representing new directories. + add_dirs: HashSet, + /// Paths representing removed files. + removed_files: HashSet, + /// Paths representing removed directories. + removed_dirs: HashSet, } impl FsSnapshot { + /// Creates a new `FsSnapshot`. pub fn new() -> Self { Self { - files: HashMap::new(), - dir: HashSet::new(), + add_files: HashMap::new(), + add_dirs: HashSet::new(), + removed_files: HashSet::new(), + removed_dirs: HashSet::new(), } } - pub fn with_file>(mut self, path: P, data: Vec) -> Self { - self.files - .insert(path.as_ref().to_path_buf(), Arc::new(data)); + /// Adds the given path to the `FsSnapshot` as a file with the given + /// contents, then returns it. + pub fn with_added_file>(mut self, path: P, data: Vec) -> Self { + self.add_files.insert(path.as_ref().to_path_buf(), data); self } - pub fn with_dir>(mut self, path: P) -> Self { - self.dir.insert(path.as_ref().to_path_buf()); + /// Adds the given path to the `FsSnapshot` as a file with the given + /// then returns it. + pub fn with_added_dir>(mut self, path: P) -> Self { + self.add_dirs.insert(path.as_ref().to_path_buf()); self } + /// Merges two `FsSnapshot`s together. + #[inline] pub fn merge(&mut self, other: Self) { - self.dir.extend(other.dir); - self.files.extend(other.files); + self.add_files.extend(other.add_files); + self.add_dirs.extend(other.add_dirs); + self.removed_files.extend(other.removed_files); + self.removed_dirs.extend(other.removed_dirs); } - pub fn push_file>(&mut self, path: P, data: Vec) { - self.files - .insert(path.as_ref().to_path_buf(), Arc::new(data)); + /// Adds the provided path as a file with the given contents. + pub fn add_file>(&mut self, path: P, data: Vec) { + self.add_files.insert(path.as_ref().to_path_buf(), data); } - pub fn push_dir>(&mut self, path: P) { - self.dir.insert(path.as_ref().to_path_buf()); - } - - pub fn pop_dir>(&mut self, path: P) -> bool { - self.dir.remove(path.as_ref()) + /// Adds the provided path as a directory. + pub fn add_dir>(&mut self, path: P) { + self.add_dirs.insert(path.as_ref().to_path_buf()); } + /// Writes the `FsSnapshot` to the provided VFS, using the provided `base` + /// as a root for the other paths in the `FsSnapshot`. pub fn write_to_vfs>(&self, base: P, vfs: &Vfs) -> io::Result<()> { let base_path = base.as_ref(); let mut dirs = 0; let mut files = 0; - for dir_path in &self.dir { + for dir_path in &self.add_dirs { match vfs.create_dir_all(base_path.join(dir_path)) { Ok(_) => (), Err(err) if err.kind() == io::ErrorKind::AlreadyExists => (), @@ -67,8 +77,8 @@ impl FsSnapshot { }; dirs += 1; } - for (path, contents) in &self.files { - vfs.write(base_path.join(path), contents.as_slice())?; + for (path, contents) in &self.add_files { + vfs.write(base_path.join(path), contents)?; files += 1; } @@ -80,10 +90,10 @@ impl FsSnapshot { impl fmt::Debug for FsSnapshot { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let files = self - .files + .add_files .iter() .map(|(k, v)| format!("{}: {} bytes", k.display(), v.len())); - let dirs = self.dir.iter().map(|v| format!("{}", v.display())); + let dirs = self.add_dirs.iter().map(|v| format!("{}", v.display())); f.debug_list().entries(files).entries(dirs).finish() } From fa6d295145ed659e77637a32cda28652caa94304 Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 2 Jan 2024 12:42:14 -0800 Subject: [PATCH 098/366] Actually write metadata for scripts and text middleware --- src/snapshot_middleware/lua.rs | 10 +++++++++- src/snapshot_middleware/txt.rs | 10 +++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/snapshot_middleware/lua.rs b/src/snapshot_middleware/lua.rs index ead6ecb54..d08a02267 100644 --- a/src/snapshot_middleware/lua.rs +++ b/src/snapshot_middleware/lua.rs @@ -180,10 +180,18 @@ pub fn syncback_lua<'new, 'old>( ); } } + let mut fs_snapshot = FsSnapshot::new(); + fs_snapshot.add_file(path, contents); + if !meta.is_empty() { + fs_snapshot.add_file( + &meta.path, + serde_json::to_vec_pretty(&meta).context("could not serialize new init.meta.json")?, + ); + } Ok(SyncbackReturn { inst_snapshot: InstanceSnapshot::from_instance(new_inst), - fs_snapshot: FsSnapshot::new().with_added_file(path, contents), + fs_snapshot, // Scripts don't have a child! children: Vec::new(), removed_children: Vec::new(), diff --git a/src/snapshot_middleware/txt.rs b/src/snapshot_middleware/txt.rs index b4827421d..8bb110161 100644 --- a/src/snapshot_middleware/txt.rs +++ b/src/snapshot_middleware/txt.rs @@ -95,10 +95,18 @@ pub fn syncback_txt<'new, 'old>( ); } } + let mut fs_snapshot = FsSnapshot::new(); + fs_snapshot.add_file(path, contents); + if !meta.is_empty() { + fs_snapshot.add_file( + &meta.path, + serde_json::to_vec_pretty(&meta).context("could not serialize new init.meta.json")?, + ); + } Ok(SyncbackReturn { inst_snapshot: InstanceSnapshot::from_instance(new_inst), - fs_snapshot: FsSnapshot::new().with_added_file(path, contents), + fs_snapshot, children: Vec::new(), removed_children: Vec::new(), }) From df1d0fb6894afaf1fc055aa006bb2f00cc9f2efd Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 2 Jan 2024 12:56:58 -0800 Subject: [PATCH 099/366] Support removing things with FsSnapshot --- src/syncback/fs_snapshot.rs | 65 ++++++++++++++++++++++++++----------- 1 file changed, 46 insertions(+), 19 deletions(-) diff --git a/src/syncback/fs_snapshot.rs b/src/syncback/fs_snapshot.rs index 22c4b6583..ad9a6fca4 100644 --- a/src/syncback/fs_snapshot.rs +++ b/src/syncback/fs_snapshot.rs @@ -10,9 +10,9 @@ use memofs::Vfs; #[derive(Default)] pub struct FsSnapshot { /// Paths representing new files mapped to their contents. - add_files: HashMap>, + added_files: HashMap>, /// Paths representing new directories. - add_dirs: HashSet, + added_dirs: HashSet, /// Paths representing removed files. removed_files: HashSet, /// Paths representing removed directories. @@ -23,8 +23,8 @@ impl FsSnapshot { /// Creates a new `FsSnapshot`. pub fn new() -> Self { Self { - add_files: HashMap::new(), - add_dirs: HashSet::new(), + added_files: HashMap::new(), + added_dirs: HashSet::new(), removed_files: HashSet::new(), removed_dirs: HashSet::new(), } @@ -33,56 +33,83 @@ impl FsSnapshot { /// Adds the given path to the `FsSnapshot` as a file with the given /// contents, then returns it. pub fn with_added_file>(mut self, path: P, data: Vec) -> Self { - self.add_files.insert(path.as_ref().to_path_buf(), data); + self.added_files.insert(path.as_ref().to_path_buf(), data); self } /// Adds the given path to the `FsSnapshot` as a file with the given /// then returns it. pub fn with_added_dir>(mut self, path: P) -> Self { - self.add_dirs.insert(path.as_ref().to_path_buf()); + self.added_dirs.insert(path.as_ref().to_path_buf()); self } /// Merges two `FsSnapshot`s together. #[inline] pub fn merge(&mut self, other: Self) { - self.add_files.extend(other.add_files); - self.add_dirs.extend(other.add_dirs); + self.added_files.extend(other.added_files); + self.added_dirs.extend(other.added_dirs); self.removed_files.extend(other.removed_files); self.removed_dirs.extend(other.removed_dirs); } /// Adds the provided path as a file with the given contents. pub fn add_file>(&mut self, path: P, data: Vec) { - self.add_files.insert(path.as_ref().to_path_buf(), data); + self.added_files.insert(path.as_ref().to_path_buf(), data); } /// Adds the provided path as a directory. pub fn add_dir>(&mut self, path: P) { - self.add_dirs.insert(path.as_ref().to_path_buf()); + self.added_dirs.insert(path.as_ref().to_path_buf()); + } + + /// Removes the provided path, as a file. + pub fn remove_file>(&mut self, path: P) { + self.removed_files.insert(path.as_ref().to_path_buf()); + } + + /// Removes the provided path, as a directory. + pub fn remove_dir>(&mut self, path: P) { + self.removed_dirs.insert(path.as_ref().to_path_buf()); } /// Writes the `FsSnapshot` to the provided VFS, using the provided `base` /// as a root for the other paths in the `FsSnapshot`. + /// + /// This includes removals, but makes no effort to minimize work done. pub fn write_to_vfs>(&self, base: P, vfs: &Vfs) -> io::Result<()> { let base_path = base.as_ref(); - let mut dirs = 0; - let mut files = 0; - for dir_path in &self.add_dirs { + for dir_path in &self.added_dirs { match vfs.create_dir_all(base_path.join(dir_path)) { Ok(_) => (), Err(err) if err.kind() == io::ErrorKind::AlreadyExists => (), Err(err) => return Err(err), }; - dirs += 1; } - for (path, contents) in &self.add_files { + for (path, contents) in &self.added_files { vfs.write(base_path.join(path), contents)?; - files += 1; + } + for dir_path in &self.removed_dirs { + vfs.remove_dir_all(base_path.join(dir_path))?; + } + for path in &self.removed_files { + vfs.remove_file(base_path.join(path))?; } - log::info!("Wrote {dirs} directories and {files} files to the file system!"); + if self.added_dirs.len() + self.added_files.len() > 0 { + log::info!( + "Wrote {} directories and {} files to the file system!", + self.added_dirs.len(), + self.added_files.len() + ); + } + if self.removed_dirs.len() + self.removed_files.len() > 0 { + log::info!( + "Removed {} directories and {} files from the file system. Yikes!", + self.removed_dirs.len(), + self.removed_files.len() + ); + } Ok(()) } } @@ -90,10 +117,10 @@ impl FsSnapshot { impl fmt::Debug for FsSnapshot { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let files = self - .add_files + .added_files .iter() .map(|(k, v)| format!("{}: {} bytes", k.display(), v.len())); - let dirs = self.add_dirs.iter().map(|v| format!("{}", v.display())); + let dirs = self.added_dirs.iter().map(|v| format!("{}", v.display())); f.debug_list().entries(files).entries(dirs).finish() } From fce6964741eb7eb760b94b991f151470b1808032 Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 4 Jan 2024 12:23:06 -0800 Subject: [PATCH 100/366] Actually remove files that are marked for removal --- src/syncback/mod.rs | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/syncback/mod.rs b/src/syncback/mod.rs index 69fdd2731..1be73b318 100644 --- a/src/syncback/mod.rs +++ b/src/syncback/mod.rs @@ -15,10 +15,10 @@ use crate::{ pub use fs_snapshot::FsSnapshot; pub use snapshot::{SyncbackData, SyncbackSnapshot}; -pub fn syncback_loop<'old, 'new>( +pub fn syncback_loop<'old>( vfs: &'old Vfs, old_tree: &'old RojoTree, - new_tree: &'new WeakDom, + new_tree: &WeakDom, project: &'old Project, ) -> anyhow::Result> { log::debug!("Hashing project DOM"); @@ -90,6 +90,21 @@ pub fn syncback_loop<'old, 'new>( } let syncback = middleware.syncback(&snapshot)?; + if !syncback.removed_children.is_empty() { + log::debug!( + "removed children for {}: {}", + get_inst_path(new_tree, snapshot.new), + syncback.removed_children.len() + ); + for inst in &syncback.removed_children { + let path = inst.metadata().instigating_source.as_ref().unwrap().path(); + if path.is_dir() { + fs_snapshot.remove_dir(path) + } else { + fs_snapshot.remove_file(path) + } + } + } if let Some(old_inst) = snapshot.old_inst() { replacements.push((old_inst.parent(), syncback.inst_snapshot)); From 7eaa362300eb48ffd018f4380c6f265c273ce68e Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 4 Jan 2024 12:24:53 -0800 Subject: [PATCH 101/366] Rename `from_parent` to `with_parent` --- src/snapshot_middleware/dir.rs | 6 +++--- src/syncback/snapshot.rs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/snapshot_middleware/dir.rs b/src/snapshot_middleware/dir.rs index daff31ca6..c27e54803 100644 --- a/src/snapshot_middleware/dir.rs +++ b/src/snapshot_middleware/dir.rs @@ -188,9 +188,9 @@ pub fn syncback_dir_no_meta<'new, 'old>( // If it exists in the new tree but not the old one, it was added. match old_children.get(new_child.name.as_str()) { None => { - children.push(snapshot.from_parent(new_child.name.clone(), *child_ref, None)) + children.push(snapshot.with_parent(new_child.name.clone(), *child_ref, None)) } - Some(old_ref) => children.push(snapshot.from_parent( + Some(old_ref) => children.push(snapshot.with_parent( new_child.name.clone(), *child_ref, Some(*old_ref), @@ -200,7 +200,7 @@ pub fn syncback_dir_no_meta<'new, 'old>( } else { for child_ref in new_inst.children() { let child = snapshot.get_new_instance(*child_ref).unwrap(); - children.push(snapshot.from_parent(child.name.clone(), *child_ref, None)) + children.push(snapshot.with_parent(child.name.clone(), *child_ref, None)) } } diff --git a/src/syncback/snapshot.rs b/src/syncback/snapshot.rs index 0f668e108..c5a5928e4 100644 --- a/src/syncback/snapshot.rs +++ b/src/syncback/snapshot.rs @@ -33,7 +33,7 @@ impl<'new, 'old> SyncbackSnapshot<'new, 'old> { /// Constructs a SyncbackSnapshot from the provided refs /// while inheriting the parent's trees and path #[inline] - pub fn from_parent(&self, new_name: String, new_ref: Ref, old_ref: Option) -> Self { + pub fn with_parent(&self, new_name: String, new_ref: Ref, old_ref: Option) -> Self { Self { data: self.data, old: old_ref, From 8ae087690dbefb4e3184589e32e68b3ff7336f77 Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 4 Jan 2024 12:40:22 -0800 Subject: [PATCH 102/366] Cleanup resolution.rs --- src/resolution.rs | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/src/resolution.rs b/src/resolution.rs index 379ccc057..9fcef34d3 100644 --- a/src/resolution.rs +++ b/src/resolution.rs @@ -2,8 +2,8 @@ use std::borrow::Borrow; use anyhow::{bail, format_err}; use rbx_dom_weak::types::{ - Attributes, BinaryString, CFrame, Color3, Content, Enum, Font, MaterialColors, Matrix3, Tags, - Variant, VariantType, Vector2, Vector3, + Attributes, CFrame, Color3, Content, Enum, Font, MaterialColors, Matrix3, Tags, Variant, + VariantType, Vector2, Vector3, }; use rbx_reflection::{DataType, PropertyDescriptor}; use serde::{Deserialize, Serialize}; @@ -60,15 +60,6 @@ impl UnresolvedValue { Variant::Float64(n) => AmbiguousValue::Number(n), Variant::Int32(n) => AmbiguousValue::Number(n as f64), Variant::Int64(n) => AmbiguousValue::Number(n as f64), - // Variant::SharedString(sstr) => { - // if let Ok(str) = std::str::from_utf8(sstr.data()) { - // AmbiguousValue::String(str.to_string()) - // } else { - // return Self::FullyQualified(Variant::BinaryString(BinaryString::from( - // sstr.data(), - // ))); - // } - // } Variant::String(str) => AmbiguousValue::String(str), Variant::Tags(tags) => { AmbiguousValue::StringArray(tags.iter().map(|s| s.to_string()).collect()) From 514f05d364faf514377a03a601a9dc01d7ad2220 Mon Sep 17 00:00:00 2001 From: Micah Date: Mon, 8 Jan 2024 12:18:51 -0800 Subject: [PATCH 103/366] Correct CSV middleware --- src/snapshot_middleware/csv.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/snapshot_middleware/csv.rs b/src/snapshot_middleware/csv.rs index 1867fe8f6..1e7bf02c0 100644 --- a/src/snapshot_middleware/csv.rs +++ b/src/snapshot_middleware/csv.rs @@ -294,7 +294,7 @@ fn convert_localization_csv(contents: &[u8]) -> Result { _ => { entry .values - .insert(Cow::Borrowed(value), Cow::Borrowed(value)); + .insert(Cow::Borrowed(header), Cow::Borrowed(value)); } } } From 3e7f30773a7eb4673ff10db17e9955cc397e0c2a Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 11 Jan 2024 11:31:51 -0800 Subject: [PATCH 104/366] If a directory is empty, add a file so it gets preserved in source control --- src/snapshot_middleware/dir.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/snapshot_middleware/dir.rs b/src/snapshot_middleware/dir.rs index c27e54803..ed4567f2b 100644 --- a/src/snapshot_middleware/dir.rs +++ b/src/snapshot_middleware/dir.rs @@ -15,6 +15,8 @@ use crate::{ use super::{meta_file::DirectoryMetadata, snapshot_from_vfs}; +const EMPTY_DIR_KEEP_NAME: &str = ".gitkeep"; + pub fn snapshot_dir( context: &InstanceContext, vfs: &Vfs, @@ -204,9 +206,15 @@ pub fn syncback_dir_no_meta<'new, 'old>( } } + let mut fs_snapshot = FsSnapshot::new(); + fs_snapshot.add_dir(&path); + if new_inst.children().is_empty() { + fs_snapshot.add_file(path.join(EMPTY_DIR_KEEP_NAME), Vec::new()) + } + Ok(SyncbackReturn { inst_snapshot: InstanceSnapshot::from_instance(new_inst), - fs_snapshot: FsSnapshot::new().with_added_dir(path), + fs_snapshot, children, removed_children, }) From 93adc8d9ce93b08501394fac865c2900bbe3a05d Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 11 Jan 2024 13:52:34 -0800 Subject: [PATCH 105/366] Finally commit an updated Cargo.lock --- Cargo.lock | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 3e2556f20..c5c14d30d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -513,6 +513,15 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "float-cmp" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4" +dependencies = [ + "num-traits", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1836,12 +1845,14 @@ dependencies = [ "anyhow", "backtrace", "bincode", + "blake3", "clap 3.2.25", "criterion", "crossbeam-channel", "csv", "embed-resource", "env_logger", + "float-cmp", "fs-err", "futures", "globset", From 0d34cbbc786f5ad4b0c431eff05e669e36eb6abb Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 11 Jan 2024 14:02:01 -0800 Subject: [PATCH 106/366] Rename `get_filtered_properties` to `new_filtered_properties` --- src/snapshot_middleware/csv.rs | 4 ++-- src/snapshot_middleware/dir.rs | 2 +- src/snapshot_middleware/json_model.rs | 2 +- src/snapshot_middleware/lua.rs | 4 ++-- src/snapshot_middleware/txt.rs | 2 +- src/syncback/snapshot.rs | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/snapshot_middleware/csv.rs b/src/snapshot_middleware/csv.rs index 1e7bf02c0..846c4edec 100644 --- a/src/snapshot_middleware/csv.rs +++ b/src/snapshot_middleware/csv.rs @@ -125,7 +125,7 @@ pub fn syncback_csv<'new, 'old>( .with_extension("meta.json"), } }; - for (name, value) in snapshot.get_filtered_properties() { + for (name, value) in snapshot.new_filtered_properties() { if name == "Contents" { continue; } else if name == "Attributes" || name == "AttributesSerialize" { @@ -194,7 +194,7 @@ pub fn syncback_csv_init<'new, 'old>( .join("init.meta.json"), } }; - for (name, value) in snapshot.get_filtered_properties() { + for (name, value) in snapshot.new_filtered_properties() { if name == "Contents" { continue; } else if name == "Attributes" || name == "AttributesSerialize" { diff --git a/src/snapshot_middleware/dir.rs b/src/snapshot_middleware/dir.rs index ed4567f2b..408aedb5b 100644 --- a/src/snapshot_middleware/dir.rs +++ b/src/snapshot_middleware/dir.rs @@ -120,7 +120,7 @@ pub fn syncback_dir<'new, 'old>( path: path.join("init.meta.json"), } }; - for (name, value) in snapshot.get_filtered_properties() { + for (name, value) in snapshot.new_filtered_properties() { if name == "Attributes" || name == "AttributesSerialize" { if let Variant::Attributes(attrs) = value { meta.attributes.extend(attrs.iter().map(|(name, value)| { diff --git a/src/snapshot_middleware/json_model.rs b/src/snapshot_middleware/json_model.rs index 97a1ed97c..f2a2a99fa 100644 --- a/src/snapshot_middleware/json_model.rs +++ b/src/snapshot_middleware/json_model.rs @@ -70,7 +70,7 @@ pub fn syncback_json_model<'new, 'old>( let mut properties = BTreeMap::new(); let mut attributes = BTreeMap::new(); - for (name, value) in snapshot.get_filtered_properties() { + for (name, value) in snapshot.new_filtered_properties() { if name == "Attributes" || name == "AttributesSerialize" { if let Variant::Attributes(attr) = value { attributes.extend(attr.iter().map(|(name, value)| { diff --git a/src/snapshot_middleware/lua.rs b/src/snapshot_middleware/lua.rs index d08a02267..213bd10a4 100644 --- a/src/snapshot_middleware/lua.rs +++ b/src/snapshot_middleware/lua.rs @@ -159,7 +159,7 @@ pub fn syncback_lua<'new, 'old>( .with_extension("meta.json"), } }; - for (name, value) in snapshot.get_filtered_properties() { + for (name, value) in snapshot.new_filtered_properties() { if name == "Source" { continue; } else if name == "Attributes" || name == "AttributesSerialize" { @@ -233,7 +233,7 @@ pub fn syncback_lua_init<'new, 'old>( .join("init.meta.json"), } }; - for (name, value) in snapshot.get_filtered_properties() { + for (name, value) in snapshot.new_filtered_properties() { if name == "Source" { continue; } else if name == "Attributes" || name == "AttributesSerialize" { diff --git a/src/snapshot_middleware/txt.rs b/src/snapshot_middleware/txt.rs index 8bb110161..7db7c9873 100644 --- a/src/snapshot_middleware/txt.rs +++ b/src/snapshot_middleware/txt.rs @@ -74,7 +74,7 @@ pub fn syncback_txt<'new, 'old>( .with_extension("meta.json"), } }; - for (name, value) in snapshot.get_filtered_properties() { + for (name, value) in snapshot.new_filtered_properties() { if name == "Value" { continue; } else if name == "Attributes" || name == "AttributesSerialize" { diff --git a/src/syncback/snapshot.rs b/src/syncback/snapshot.rs index c5a5928e4..3c40ece53 100644 --- a/src/syncback/snapshot.rs +++ b/src/syncback/snapshot.rs @@ -49,7 +49,7 @@ impl<'new, 'old> SyncbackSnapshot<'new, 'old> { /// Note that while the returned map does filter based on the user's /// `ignore_props` field, it does not do any other filtering and doesn't /// clone any data. This is left to the consumer. - pub fn get_filtered_properties(&self) -> HashMap<&'new str, &'new Variant> { + pub fn new_filtered_properties(&self) -> HashMap<&'new str, &'new Variant> { let new_inst = self.new_inst(); let mut properties: HashMap<&str, &Variant> = HashMap::with_capacity(new_inst.properties.capacity()); From 01551f6662e67da5d70f2662fa5555cbb1a97a02 Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 11 Jan 2024 14:04:34 -0800 Subject: [PATCH 107/366] Add `get_filtered_properties` to get filtered properties for any instance --- src/syncback/snapshot.rs | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/src/syncback/snapshot.rs b/src/syncback/snapshot.rs index 3c40ece53..780b78eab 100644 --- a/src/syncback/snapshot.rs +++ b/src/syncback/snapshot.rs @@ -49,15 +49,29 @@ impl<'new, 'old> SyncbackSnapshot<'new, 'old> { /// Note that while the returned map does filter based on the user's /// `ignore_props` field, it does not do any other filtering and doesn't /// clone any data. This is left to the consumer. + #[inline] pub fn new_filtered_properties(&self) -> HashMap<&'new str, &'new Variant> { - let new_inst = self.new_inst(); + self.get_filtered_properties(self.new).unwrap() + } + + /// Returns a map of properties for an Instance from the 'new' tree + /// with filtering done to avoid noise. + /// + /// Note that while the returned map does filter based on the user's + /// `ignore_props` field, it does not do any other filtering and doesn't + /// clone any data. This is left to the consumer. + pub fn get_filtered_properties( + &self, + new_ref: Ref, + ) -> Option> { + let inst = self.get_new_instance(new_ref)?; let mut properties: HashMap<&str, &Variant> = - HashMap::with_capacity(new_inst.properties.capacity()); + HashMap::with_capacity(inst.properties.capacity()); let filter = self.get_property_filter(); if let Some(old_inst) = self.old_inst() { - for (name, value) in &new_inst.properties { + for (name, value) in &inst.properties { if old_inst.properties().contains_key(name) { properties.insert(name, value); } @@ -65,10 +79,10 @@ impl<'new, 'old> SyncbackSnapshot<'new, 'old> { } else { let class_data = rbx_reflection_database::get() .classes - .get(new_inst.class.as_str()); + .get(inst.class.as_str()); if let Some(class_data) = class_data { let defaults = &class_data.default_properties; - for (name, value) in &new_inst.properties { + for (name, value) in &inst.properties { // We don't currently support refs or shared strings if matches!(value, Variant::Ref(_) | Variant::SharedString(_)) { continue; @@ -87,7 +101,7 @@ impl<'new, 'old> SyncbackSnapshot<'new, 'old> { } } } else { - for (name, value) in &new_inst.properties { + for (name, value) in &inst.properties { // We don't currently support refs or shared strings if matches!(value, Variant::Ref(_) | Variant::SharedString(_)) { continue; @@ -102,7 +116,7 @@ impl<'new, 'old> SyncbackSnapshot<'new, 'old> { } } - properties + Some(properties) } /// Returns a set of properties that should not be written with syncback if From cc1d68d7bbe95725ddfb09bba85df377084a2f8e Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 11 Jan 2024 15:02:34 -0800 Subject: [PATCH 108/366] Optimistically serialize any non-default property in a ProjectNode --- src/snapshot_middleware/project.rs | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/src/snapshot_middleware/project.rs b/src/snapshot_middleware/project.rs index b42a2a938..73fff845c 100644 --- a/src/snapshot_middleware/project.rs +++ b/src/snapshot_middleware/project.rs @@ -348,17 +348,14 @@ pub fn syncback_project<'new, 'old>( if new_child.class != old_child.class_name() { anyhow::bail!("Cannot change the class of {child_name} in a project"); } - for (name, value) in &new_child.properties { - if child_node.properties.contains_key(name) { - child_node.properties.insert( - name.clone(), - UnresolvedValue::from_variant( - value.clone(), - &new_child.class, - name.as_str(), - ), - ); - } + let properties = snapshot + .get_filtered_properties(new_child.referent()) + .expect("all project nodes should exist in the DOM"); + for (name, value) in properties { + child_node.properties.insert( + name.to_owned(), + UnresolvedValue::from_variant(value.clone(), &new_child.class, name), + ); } nodes.push((child_node, new_child, *old_child)); new_child_map.remove(child_name.as_str()); From 4a83c4c13d6e656d0d9ebb64cc53a7b23b4dc445 Mon Sep 17 00:00:00 2001 From: Micah Date: Fri, 12 Jan 2024 14:05:23 -0800 Subject: [PATCH 109/366] Rewrite syncback middleware for Project --- src/snapshot_middleware/project.rs | 144 +++++++++++++++-------------- 1 file changed, 75 insertions(+), 69 deletions(-) diff --git a/src/snapshot_middleware/project.rs b/src/snapshot_middleware/project.rs index 73fff845c..a2fd9b782 100644 --- a/src/snapshot_middleware/project.rs +++ b/src/snapshot_middleware/project.rs @@ -301,117 +301,123 @@ pub fn syncback_project<'new, 'old>( ) -> anyhow::Result> { let old_inst = snapshot .old_inst() - .context("project middleware shouldn't be used to make new files")?; - // This can never be None. - let source = old_inst.metadata().instigating_source.as_ref().unwrap(); + .expect("projects should always exist in both trees"); + // Project roots have to come from somewhere! + let path = old_inst + .metadata() + .instigating_source + .as_ref() + .unwrap() + .path(); + + let base_path = path.parent().expect("project did not have a parent"); - // We need to build a 'new' project and serialize it using an FsSnapshot. - // It's convenient to start with the old one though, since it means we have - // a thing to iterate through. - let mut project = - Project::load_from_slice(&snapshot.vfs().read(source.path()).unwrap(), source.path()) - .context("could not syncback project due to fs error")?; + let vfs = snapshot.vfs(); - let base_path = source.path().parent().unwrap(); + let mut project = Project::load_from_slice(&vfs.read(path)?, path)?; let mut children = Vec::new(); let mut removed_children = Vec::new(); - // Projects are special. We won't be adding or removing things from them, - // so we'll simply match Instances on a per-node basis and rebuild the tree - // with the new instance's data. This matching will be done by class and name - // to simplify things. - let mut nodes = vec![(&mut project.tree, snapshot.new_inst(), old_inst)]; - - // A map of referents from the new tree to the Path that created it, - // if it exists. This is a roundabout way to locate the parents of - // Instances. - let mut ref_to_node = HashMap::new(); - - while let Some((node, new_inst, old_inst)) = nodes.pop() { - ref_to_node.insert(new_inst.referent(), node.path.as_ref()); - - let mut old_child_map = HashMap::with_capacity(old_inst.children().len()); - for child_ref in old_inst.children() { - let child = snapshot.get_old_instance(*child_ref).unwrap(); - old_child_map.insert(child.name(), child); - } - let mut new_child_map = HashMap::with_capacity(new_inst.children().len()); - for child_ref in new_inst.children() { - let child = snapshot.get_new_instance(*child_ref).unwrap(); - new_child_map.insert(child.name.as_str(), child); + // A map of every node and the Instances they map to. This is fine because + // we don't add or remove from projects, so every node must map somewhere. + let mut project_nodes = vec![(&mut project.tree, snapshot.new_inst(), old_inst)]; + + // A map of refs to the path of a node they represent. This is used later to + // match children to their parent's path. + let mut ref_to_path = HashMap::new(); + + // These map children of a node by name to the actual Instance + let mut old_child_map = HashMap::new(); + let mut new_child_map = HashMap::new(); + + while let Some((node, new_inst, old_inst)) = project_nodes.pop() { + log::trace!("Processing node '{}' of project", old_inst.name()); + ref_to_path.insert(new_inst.referent(), node.path.as_ref()); + + old_child_map.extend(old_inst.children().iter().map(|referent| { + let child = snapshot.get_old_instance(*referent).unwrap(); + (child.name(), child) + })); + new_child_map.extend(new_inst.children().iter().map(|referent| { + let child = snapshot.get_new_instance(*referent).unwrap(); + (&child.name, child) + })); + + let properties = &mut node.properties; + + let filtered_properties = snapshot + .get_filtered_properties(new_inst.referent()) + .expect("all project nodes should exist in both trees when in queue"); + for (name, value) in filtered_properties { + properties.insert( + name.to_owned(), + UnresolvedValue::from_variant(value.clone(), &new_inst.class, name), + ); } for (child_name, child_node) in &mut node.children { - if let Some(new_child) = new_child_map.get(child_name.as_str()) { - if let Some(old_child) = old_child_map.get(child_name.as_str()) { + // There could be no old_child if the node is optional and not + // present on the file system, which is why this isn't an `unwrap`. + if let Some(old_child) = old_child_map.get(child_name.as_str()) { + if let Some(new_child) = new_child_map.get(child_name) { if new_child.class != old_child.class_name() { - anyhow::bail!("Cannot change the class of {child_name} in a project"); - } - let properties = snapshot - .get_filtered_properties(new_child.referent()) - .expect("all project nodes should exist in the DOM"); - for (name, value) in properties { - child_node.properties.insert( - name.to_owned(), - UnresolvedValue::from_variant(value.clone(), &new_child.class, name), - ); + anyhow::bail!("cannot change the class of {child_name} in project"); } - nodes.push((child_node, new_child, *old_child)); - new_child_map.remove(child_name.as_str()); + project_nodes.push((child_node, new_child, *old_child)); old_child_map.remove(child_name.as_str()); + new_child_map.remove(child_name); + } else { + anyhow::bail!("cannot add or remove {child_name} from project"); } } else { - anyhow::bail!("Cannot add or remove {child_name} from project") + log::warn!( + "Project node '{child_name}' is optional and does not \ + exist on the file system. It will be skipped during syncback." + ) } } - // From this point, both maps contain only children of the current - // instance that aren't in the project. So, we just do some quick and - // dirty matching to identify children that were added and removed. - for (new_name, new_child) in new_child_map { - let parent_path = match ref_to_node.get(&new_child.parent()) { + // After matching children above, the child maps only contain children + // of this node that aren't in the project file. + for (new_name, new_child) in new_child_map.drain() { + let parent_path = match ref_to_path.get(&new_child.parent()) { Some(Some(path)) => base_path.join(path.path()), - Some(None) => { - continue; - } - None => { + _ => { + // For this to happen, the instance isn't a part of the + // project at all, so we need to just skip it. continue; } }; - if let Some(old_inst) = old_child_map.get(new_name) { - // All children are descendants of a node of a project - // So we really just need to track which one is which. + if let Some(old_inst) = old_child_map.get(new_name.as_str()) { + // This new instance represents an older one! children.push(SyncbackSnapshot { data: snapshot.data, old: Some(old_inst.id()), new: new_child.referent(), parent_path, - name: new_name.to_string(), + name: new_name.to_owned(), }); - old_child_map.remove(new_name); + old_child_map.remove(new_name.as_str()); } else { - // it's new + // This new instance is... new. children.push(SyncbackSnapshot { data: snapshot.data, old: None, new: new_child.referent(), parent_path, - name: new_name.to_string(), + name: new_name.to_owned(), }); } } - removed_children.extend(old_child_map.into_values()); + removed_children.extend(old_child_map.drain().map(|(_, inst)| inst)); } - // We don't need to validate any file names for the FsSnapshot - // because projects can't ever be made by syncback, so they must - // already exist on the file system Ok(SyncbackReturn { inst_snapshot: InstanceSnapshot::from_instance(snapshot.new_inst()), fs_snapshot: FsSnapshot::new().with_added_file( &project.file_location, - serde_json::to_vec_pretty(&project).context("failed to serialize new project")?, + serde_json::to_vec_pretty(&project).context("failed to serialize updated project")?, ), children, removed_children, From 6f447ec35fdbe37ba410db4a6a27a3668414b886 Mon Sep 17 00:00:00 2001 From: Micah Date: Fri, 12 Jan 2024 15:34:17 -0800 Subject: [PATCH 110/366] Properly warn and skip missing project nodes --- src/snapshot_middleware/project.rs | 40 +++++++++++++++++++----------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/src/snapshot_middleware/project.rs b/src/snapshot_middleware/project.rs index a2fd9b782..7a724b6db 100644 --- a/src/snapshot_middleware/project.rs +++ b/src/snapshot_middleware/project.rs @@ -344,6 +344,9 @@ pub fn syncback_project<'new, 'old>( (&child.name, child) })); + // The fact we are not removing old properties might be a problem but + // removing them is worse! Consider a property that doesn't serialize, + // like HttpEnabled. let properties = &mut node.properties; let filtered_properties = snapshot @@ -355,26 +358,33 @@ pub fn syncback_project<'new, 'old>( UnresolvedValue::from_variant(value.clone(), &new_inst.class, name), ); } - for (child_name, child_node) in &mut node.children { - // There could be no old_child if the node is optional and not - // present on the file system, which is why this isn't an `unwrap`. - if let Some(old_child) = old_child_map.get(child_name.as_str()) { - if let Some(new_child) = new_child_map.get(child_name) { - if new_child.class != old_child.class_name() { - anyhow::bail!("cannot change the class of {child_name} in project"); - } - project_nodes.push((child_node, new_child, *old_child)); + if let Some(path_node) = &child_node.path { + if let Ok(false) = base_path.join(path_node.path()).try_exists() { + log::warn!( + "The project refers to '{child_name}' with path '{}' \ + which does not exist in the project directory.", + path_node.path().display() + ); old_child_map.remove(child_name.as_str()); new_child_map.remove(child_name); - } else { - anyhow::bail!("cannot add or remove {child_name} from project"); + continue; } + } + let old_child = old_child_map + .get(child_name.as_str()) + .expect("all nodes in queue should have old instances"); + + if let Some(new_child) = new_child_map.get(child_name) { + if new_child.class != old_child.class_name() { + anyhow::bail!("cannot change the class of {child_name} in project"); + } + + project_nodes.push((child_node, new_child, *old_child)); + old_child_map.remove(child_name.as_str()); + new_child_map.remove(child_name); } else { - log::warn!( - "Project node '{child_name}' is optional and does not \ - exist on the file system. It will be skipped during syncback." - ) + anyhow::bail!("cannot add or remove {child_name} from project"); } } From aad9b8a3b0589363dbd80842833befd7ded2b3a9 Mon Sep 17 00:00:00 2001 From: Micah Date: Mon, 15 Jan 2024 10:50:33 -0800 Subject: [PATCH 111/366] Rewrite directory syncback middleware --- src/snapshot_middleware/dir.rs | 62 ++++++++++++---------------------- 1 file changed, 22 insertions(+), 40 deletions(-) diff --git a/src/snapshot_middleware/dir.rs b/src/snapshot_middleware/dir.rs index 408aedb5b..46ff7bf16 100644 --- a/src/snapshot_middleware/dir.rs +++ b/src/snapshot_middleware/dir.rs @@ -153,59 +153,41 @@ pub fn syncback_dir_no_meta<'new, 'old>( snapshot: &SyncbackSnapshot<'new, 'old>, ) -> anyhow::Result> { let path = snapshot.parent_path.join(&snapshot.name); - let new_inst = snapshot.new_inst(); - let mut removed_children = Vec::new(); let mut children = Vec::new(); + let mut removed_children = Vec::new(); if let Some(old_inst) = snapshot.old_inst() { - let old_children: HashMap<&str, Ref> = old_inst - .children() - .iter() - .map(|old_ref| { - ( - snapshot.get_old_instance(*old_ref).unwrap().name(), - *old_ref, - ) - }) - .collect(); - let new_children: HashSet<&str> = snapshot - .new_inst() - .children() - .iter() - .map(|new_ref| snapshot.get_new_instance(*new_ref).unwrap().name.as_str()) - .collect(); - - for child_ref in old_inst.children() { - let old_child = snapshot.get_old_instance(*child_ref).unwrap(); - // If it exists in the old tree but not the new one, it was removed. - if !new_children.contains(old_child.name()) { - removed_children.push(old_child); - } + let mut old_child_map = HashMap::new(); + for child in old_inst.children() { + let inst = snapshot.get_old_instance(*child).unwrap(); + old_child_map.insert(inst.name(), inst); } - for child_ref in new_inst.children() { - let new_child = snapshot.get_new_instance(*child_ref).unwrap(); - // If it exists in the new tree but not the old one, it was added. - match old_children.get(new_child.name.as_str()) { - None => { - children.push(snapshot.with_parent(new_child.name.clone(), *child_ref, None)) - } - Some(old_ref) => children.push(snapshot.with_parent( + for new_child_ref in new_inst.children() { + let new_child = snapshot.get_new_instance(*new_child_ref).unwrap(); + if let Some(old_child) = old_child_map.remove(new_child.name.as_str()) { + // This child exists in both doms. Pass it on. + children.push(snapshot.with_parent( new_child.name.clone(), - *child_ref, - Some(*old_ref), - )), + *new_child_ref, + Some(old_child.id()), + )); + } else { + // The child only exists in the the new dom + children.push(snapshot.with_parent(new_child.name.clone(), *new_child_ref, None)); } } + // Any children that are in the old dom but not the new one are removed. + removed_children.extend(old_child_map.into_values()); } else { - for child_ref in new_inst.children() { - let child = snapshot.get_new_instance(*child_ref).unwrap(); - children.push(snapshot.with_parent(child.name.clone(), *child_ref, None)) + // There is no old instance. Just add every child. + for new_child_ref in new_inst.children() { + let new_child = snapshot.get_new_instance(*new_child_ref).unwrap(); + children.push(snapshot.with_parent(new_child.name.clone(), *new_child_ref, None)); } } - let mut fs_snapshot = FsSnapshot::new(); fs_snapshot.add_dir(&path); if new_inst.children().is_empty() { From c04893673aa9f5cbc2343c089e7e516d2ff702e8 Mon Sep 17 00:00:00 2001 From: Micah Date: Mon, 15 Jan 2024 11:47:50 -0800 Subject: [PATCH 112/366] Remove unnecessary use in dir middleware --- src/snapshot_middleware/dir.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/snapshot_middleware/dir.rs b/src/snapshot_middleware/dir.rs index 46ff7bf16..55f9922c9 100644 --- a/src/snapshot_middleware/dir.rs +++ b/src/snapshot_middleware/dir.rs @@ -1,11 +1,11 @@ use std::{ - collections::{BTreeMap, HashMap, HashSet}, + collections::{BTreeMap, HashMap}, path::Path, }; use anyhow::Context; use memofs::{DirEntry, IoResultExt, Vfs}; -use rbx_dom_weak::types::{Ref, Variant}; +use rbx_dom_weak::types::Variant; use crate::{ resolution::UnresolvedValue, From a2f173976c470b47f8a39fcf0c63c4a679336411 Mon Sep 17 00:00:00 2001 From: Micah Date: Mon, 15 Jan 2024 11:48:09 -0800 Subject: [PATCH 113/366] Make syncback command less panic-y --- src/cli/syncback.rs | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/cli/syncback.rs b/src/cli/syncback.rs index f3f943b69..bb32b7f90 100644 --- a/src/cli/syncback.rs +++ b/src/cli/syncback.rs @@ -1,5 +1,4 @@ use std::{ - fs, path::{Path, PathBuf}, time::Instant, }; @@ -42,7 +41,7 @@ impl SyncbackCommand { let dom_start = Instant::now(); log::info!("Reading place file at {}", path_new.display()); - let dom_new = read_dom(&path_new); + let dom_new = read_dom(&path_new)?; log::info!( "Finished opening file in {:0.02}s", dom_start.elapsed().as_secs_f32() @@ -65,19 +64,18 @@ impl SyncbackCommand { } } -fn read_dom(path: &Path) -> WeakDom { - let content = fs::read(path).unwrap(); - if &content[0..8] == b" anyhow::Result { + let content = fs_err::read(path)?; + Ok(if &content[0..8] == b" Date: Mon, 15 Jan 2024 12:19:36 -0800 Subject: [PATCH 114/366] Add context to error when syncback fails --- src/syncback/mod.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/syncback/mod.rs b/src/syncback/mod.rs index 1be73b318..85e067927 100644 --- a/src/syncback/mod.rs +++ b/src/syncback/mod.rs @@ -1,6 +1,7 @@ mod fs_snapshot; mod snapshot; +use anyhow::Context; use memofs::Vfs; use rbx_dom_weak::{types::Ref, Instance, WeakDom}; use serde::{Deserialize, Serialize}; @@ -89,7 +90,12 @@ pub fn syncback_loop<'old>( continue; } - let syncback = middleware.syncback(&snapshot)?; + let syncback = middleware.syncback(&snapshot).with_context(|| { + format!( + "Failed to syncback {}", + get_inst_path(new_tree, snapshot.new) + ) + })?; if !syncback.removed_children.is_empty() { log::debug!( "removed children for {}: {}", From bd53ac96243d653f4cbf3478a9f5bbbfdd9a8bd2 Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 16 Jan 2024 08:52:19 -0800 Subject: [PATCH 115/366] Overwrite existing file if it exists, supporting sync rules --- src/snapshot_middleware/csv.rs | 25 ++++++++++++---- src/snapshot_middleware/json_model.rs | 16 ++++++++-- src/snapshot_middleware/lua.rs | 42 ++++++++++++++++++--------- src/snapshot_middleware/rbxm.rs | 22 ++++++++++---- src/snapshot_middleware/rbxmx.rs | 25 +++++++++++----- src/snapshot_middleware/txt.rs | 14 +++++++-- 6 files changed, 109 insertions(+), 35 deletions(-) diff --git a/src/snapshot_middleware/csv.rs b/src/snapshot_middleware/csv.rs index 846c4edec..2317927d2 100644 --- a/src/snapshot_middleware/csv.rs +++ b/src/snapshot_middleware/csv.rs @@ -103,9 +103,18 @@ pub fn syncback_csv<'new, 'old>( snapshot: &SyncbackSnapshot<'new, 'old>, ) -> anyhow::Result> { let new_inst = snapshot.new_inst(); - - let mut path = snapshot.parent_path.join(&snapshot.name); - path.set_extension("csv"); + let path = snapshot + .old_inst() + .and_then(|inst| inst.metadata().instigating_source.as_ref()) + .map_or_else( + || { + // Since Roblox instances may or may not a `.` character in + // their names, we can't just use `.set_file_name` and + // `.set_extension`. + snapshot.parent_path.join(format!("{}.csv", snapshot.name)) + }, + |source| source.path().to_path_buf(), + ); let contents = if let Some(Variant::String(content)) = new_inst.properties.get("Contents") { content.as_str() @@ -170,8 +179,14 @@ pub fn syncback_csv_init<'new, 'old>( ) -> anyhow::Result> { let new_inst = snapshot.new_inst(); - let mut path = snapshot.parent_path.join(&snapshot.name); - path.push("init.csv"); + let path = snapshot + .old_inst() + .and_then(|inst| inst.metadata().instigating_source.as_ref()) + .map_or_else(|| snapshot.parent_path.as_path(), |source| source.path()) + // Instigating source for 'init' middleware is the directory they point + // to, not the `init.*` file, which is good since that's what we rerun + // the snapshot middleware on, but it means we have to do this: + .join("init.csv"); let contents = if let Some(Variant::String(content)) = new_inst.properties.get("Contents") { content.as_str() diff --git a/src/snapshot_middleware/json_model.rs b/src/snapshot_middleware/json_model.rs index f2a2a99fa..23ff61627 100644 --- a/src/snapshot_middleware/json_model.rs +++ b/src/snapshot_middleware/json_model.rs @@ -63,8 +63,20 @@ pub fn snapshot_json_model( pub fn syncback_json_model<'new, 'old>( snapshot: &SyncbackSnapshot<'new, 'old>, ) -> anyhow::Result> { - let mut path = snapshot.parent_path.join(&snapshot.name); - path.set_extension("model.json"); + let path = snapshot + .old_inst() + .and_then(|inst| inst.metadata().instigating_source.as_ref()) + .map_or_else( + || { + // Since Roblox instances may or may not a `.` character in + // their names, we can't just use `.set_file_name` and + // `.set_extension`. + snapshot + .parent_path + .join(format!("{}.model.json", snapshot.name)) + }, + |source| source.path().to_path_buf(), + ); let new_inst = snapshot.new_inst(); diff --git a/src/snapshot_middleware/lua.rs b/src/snapshot_middleware/lua.rs index 213bd10a4..91e42c255 100644 --- a/src/snapshot_middleware/lua.rs +++ b/src/snapshot_middleware/lua.rs @@ -134,13 +134,24 @@ pub fn syncback_lua<'new, 'old>( snapshot: &SyncbackSnapshot<'new, 'old>, ) -> anyhow::Result> { let new_inst = snapshot.new_inst(); + let path = snapshot + .old_inst() + .and_then(|inst| inst.metadata().instigating_source.as_ref()) + .map_or_else( + || { + // Since Roblox instances may or may not a `.` character in + // their names, we can't just use `.set_file_name` and + // `.set_extension`. + snapshot.parent_path.join(match script_type { + ScriptType::Server => format!("{}.server.lua", snapshot.name), + ScriptType::Client => format!("{}.client.lua", snapshot.name), + ScriptType::Module => format!("{}.lua", snapshot.name), + }) + }, + |source| source.path().to_path_buf(), + ); + log::debug!("Luau middleware: {}", path.display()); - let mut path = snapshot.parent_path.join(&snapshot.name); - path.set_extension(match script_type { - ScriptType::Module => "lua", - ScriptType::Client => "client.lua", - ScriptType::Server => "server.lua", - }); let contents = if let Some(Variant::String(source)) = new_inst.properties.get("Source") { source.as_bytes().to_vec() } else { @@ -203,14 +214,19 @@ pub fn syncback_lua_init<'new, 'old>( snapshot: &SyncbackSnapshot<'new, 'old>, ) -> anyhow::Result> { let new_inst = snapshot.new_inst(); + let path = snapshot + .old_inst() + .and_then(|inst| inst.metadata().instigating_source.as_ref()) + .map_or_else(|| snapshot.parent_path.as_path(), |source| source.path()) + // Instigating source for 'init' middleware is the directory they point + // to, not the `init.*` file, which is good since that's what we rerun + // the snapshot middleware on, but it means we have to do this: + .join(match script_type { + ScriptType::Server => "init.server.lua", + ScriptType::Client => "init.client.lua", + ScriptType::Module => "init.lua", + }); - let mut path = snapshot.parent_path.join(&snapshot.name); - path.push("init"); - path.set_extension(match script_type { - ScriptType::Module => "lua", - ScriptType::Client => "client.lua", - ScriptType::Server => "server.lua", - }); let contents = if let Some(Variant::String(source)) = new_inst.properties.get("Source") { source.as_bytes().to_vec() } else { diff --git a/src/snapshot_middleware/rbxm.rs b/src/snapshot_middleware/rbxm.rs index 5d392a5d2..4606c604a 100644 --- a/src/snapshot_middleware/rbxm.rs +++ b/src/snapshot_middleware/rbxm.rs @@ -45,19 +45,29 @@ pub fn snapshot_rbxm( pub fn syncback_rbxm<'new, 'old>( snapshot: &SyncbackSnapshot<'new, 'old>, ) -> anyhow::Result> { - // If any of the children of this Instance are scripts, we don't want - // include them in the model. So instead, we'll check and then serialize. let inst = snapshot.new_inst(); - let mut path = snapshot.parent_path.join(&snapshot.name); - path.set_extension("rbxm"); - // Long-term, anyway. Right now we stay silly. + let path = snapshot + .old_inst() + .and_then(|inst| inst.metadata().instigating_source.as_ref()) + .map_or_else( + || { + // Since Roblox instances may or may not a `.` character in + // their names, we can't just use `.set_file_name` and + // `.set_extension`. + snapshot.parent_path.join(format!("{}.rbxm", snapshot.name)) + }, + |source| source.path().to_path_buf(), + ); + + // Long-term, we probably want to have some logic for if this contains a + // script. That's a future endeavor though. let mut serialized = Vec::new(); rbx_binary::to_writer(&mut serialized, snapshot.new_tree(), &[inst.referent()]) .context("failed to serialize new rbxm")?; Ok(SyncbackReturn { inst_snapshot: InstanceSnapshot::from_instance(inst), - fs_snapshot: FsSnapshot::new().with_added_file(&path, serialized), + fs_snapshot: FsSnapshot::new().with_added_file(path, serialized), children: Vec::new(), removed_children: Vec::new(), }) diff --git a/src/snapshot_middleware/rbxmx.rs b/src/snapshot_middleware/rbxmx.rs index 3aa3931fb..053b444bb 100644 --- a/src/snapshot_middleware/rbxmx.rs +++ b/src/snapshot_middleware/rbxmx.rs @@ -47,20 +47,31 @@ pub fn snapshot_rbxmx( pub fn syncback_rbxmx<'new, 'old>( snapshot: &SyncbackSnapshot<'new, 'old>, ) -> anyhow::Result> { - // If any of the children of this Instance are scripts, we don't want - // include them in the model. So instead, we'll check and then serialize. - let inst = snapshot.new_inst(); - let mut path = snapshot.parent_path.join(&snapshot.name); - path.set_extension("rbxmx"); - // Long-term, anyway. Right now we stay silly. + let path = snapshot + .old_inst() + .and_then(|inst| inst.metadata().instigating_source.as_ref()) + .map_or_else( + || { + // Since Roblox instances may or may not a `.` character in + // their names, we can't just use `.set_file_name` and + // `.set_extension`. + snapshot + .parent_path + .join(format!("{}.rbxmx", &snapshot.name)) + }, + |source| source.path().to_path_buf(), + ); + + // Long-term, we probably want to have some logic for if this contains a + // script. That's a future endeavor though. let mut serialized = Vec::new(); rbx_xml::to_writer_default(&mut serialized, snapshot.new_tree(), &[inst.referent()]) .context("failed to serialize new rbxmx")?; Ok(SyncbackReturn { inst_snapshot: InstanceSnapshot::from_instance(inst), - fs_snapshot: FsSnapshot::new().with_added_file(&path, serialized), + fs_snapshot: FsSnapshot::new().with_added_file(path, serialized), children: Vec::new(), removed_children: Vec::new(), }) diff --git a/src/snapshot_middleware/txt.rs b/src/snapshot_middleware/txt.rs index 7db7c9873..dd37648ee 100644 --- a/src/snapshot_middleware/txt.rs +++ b/src/snapshot_middleware/txt.rs @@ -53,8 +53,18 @@ pub fn syncback_txt<'new, 'old>( snapshot: &SyncbackSnapshot<'new, 'old>, ) -> anyhow::Result> { let new_inst = snapshot.new_inst(); - let mut path = snapshot.parent_path.join(&snapshot.name); - path.set_extension("txt"); + let path = snapshot + .old_inst() + .and_then(|inst| inst.metadata().instigating_source.as_ref()) + .map_or_else( + || { + // Since Roblox instances may or may not a `.` character in + // their names, we can't just use `.set_file_name` and + // `.set_extension`. + snapshot.parent_path.join(format!("{}.txt", &snapshot.name)) + }, + |source| source.path().to_path_buf(), + ); let contents = if let Some(Variant::String(source)) = new_inst.properties.get("Value") { source.as_bytes().to_vec() From 2b57774eaaf152897af35af7b0a598950335a2cf Mon Sep 17 00:00:00 2001 From: Micah Date: Tue, 16 Jan 2024 14:58:33 -0800 Subject: [PATCH 116/366] Make syncback into .git illegal --- src/syncback/mod.rs | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/syncback/mod.rs b/src/syncback/mod.rs index 85e067927..04572ba90 100644 --- a/src/syncback/mod.rs +++ b/src/syncback/mod.rs @@ -5,9 +5,13 @@ use anyhow::Context; use memofs::Vfs; use rbx_dom_weak::{types::Ref, Instance, WeakDom}; use serde::{Deserialize, Serialize}; -use std::collections::{HashMap, VecDeque}; +use std::{ + collections::{HashMap, VecDeque}, + sync::OnceLock, +}; use crate::{ + glob::Glob, snapshot::{hash_tree, InstanceSnapshot, InstanceWithMeta, RojoTree}, snapshot_middleware::Middleware, Project, @@ -16,6 +20,9 @@ use crate::{ pub use fs_snapshot::FsSnapshot; pub use snapshot::{SyncbackData, SyncbackSnapshot}; +/// A glob that can be used to tell if a path contains a `.git` folder. +static GIT_IGNORE_GLOB: OnceLock = OnceLock::new(); + pub fn syncback_loop<'old>( vfs: &'old Vfs, old_tree: &'old RojoTree, @@ -27,6 +34,9 @@ pub fn syncback_loop<'old>( log::debug!("Hashing file DOM"); let new_hashes = hash_tree(new_tree); + let git_glob = GIT_IGNORE_GLOB.get_or_init(|| Glob::new(".git/**").unwrap()); + let project_path = project.folder_location(); + let syncback_data = SyncbackData { vfs, old_tree, @@ -49,6 +59,16 @@ pub fn syncback_loop<'old>( let mut fs_snapshot = FsSnapshot::new(); while let Some(snapshot) = snapshots.pop() { + if let Ok(suffix) = snapshot.parent_path.strip_prefix(project_path) { + if git_glob.is_match(suffix) { + log::warn!( + "Cannot syncback into `.git`, {} will be skipped.", + get_inst_path(new_tree, snapshot.new) + ); + continue; + } + } + // We can quickly check that two subtrees are identical and if they are, // skip reconciling them. if let Some(old_ref) = snapshot.old { @@ -96,6 +116,7 @@ pub fn syncback_loop<'old>( get_inst_path(new_tree, snapshot.new) ) })?; + if !syncback.removed_children.is_empty() { log::debug!( "removed children for {}: {}", From d9580532a1d3db288c13f459d6302c2174b777f6 Mon Sep 17 00:00:00 2001 From: Micah Date: Wed, 17 Jan 2024 13:06:15 -0800 Subject: [PATCH 117/366] Remove debug log in luau syncback middleware --- src/snapshot_middleware/lua.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/snapshot_middleware/lua.rs b/src/snapshot_middleware/lua.rs index 91e42c255..e452ab915 100644 --- a/src/snapshot_middleware/lua.rs +++ b/src/snapshot_middleware/lua.rs @@ -150,7 +150,6 @@ pub fn syncback_lua<'new, 'old>( }, |source| source.path().to_path_buf(), ); - log::debug!("Luau middleware: {}", path.display()); let contents = if let Some(Variant::String(source)) = new_inst.properties.get("Source") { source.as_bytes().to_vec() From fb82551ee90ef6b9d7b4dce26332470c1635d271 Mon Sep 17 00:00:00 2001 From: Micah Date: Wed, 17 Jan 2024 14:23:02 -0800 Subject: [PATCH 118/366] Support ignoring via destination with globs --- src/syncback/mod.rs | 87 ++++++++++++++++++--------------------------- 1 file changed, 35 insertions(+), 52 deletions(-) diff --git a/src/syncback/mod.rs b/src/syncback/mod.rs index 04572ba90..fb44a37d2 100644 --- a/src/syncback/mod.rs +++ b/src/syncback/mod.rs @@ -58,24 +58,14 @@ pub fn syncback_loop<'old>( let mut replacements = Vec::new(); let mut fs_snapshot = FsSnapshot::new(); - while let Some(snapshot) = snapshots.pop() { - if let Ok(suffix) = snapshot.parent_path.strip_prefix(project_path) { - if git_glob.is_match(suffix) { - log::warn!( - "Cannot syncback into `.git`, {} will be skipped.", - get_inst_path(new_tree, snapshot.new) - ); - continue; - } - } - + 'syncback: while let Some(snapshot) = snapshots.pop() { + let inst_path = get_inst_path(new_tree, snapshot.new); // We can quickly check that two subtrees are identical and if they are, // skip reconciling them. if let Some(old_ref) = snapshot.old { if old_hashes.get(&old_ref) == new_hashes.get(&snapshot.new) { log::trace!( - "Skipping {} due to it being identically hashed as {:?}", - get_inst_path(new_tree, snapshot.new), + "Skipping {inst_path} due to it being identically hashed as {:?}", old_hashes.get(&old_ref) ); continue; @@ -83,11 +73,26 @@ pub fn syncback_loop<'old>( } if let Some(syncback_rules) = &project.syncback_rules { - if !syncback_rules.acceptable(new_tree, snapshot.new) { - log::debug!( - "Path {} is blocked by project", - get_inst_path(new_tree, snapshot.new) - ); + // Ignore paths + if let Ok(suffix) = snapshot.parent_path.strip_prefix(project_path) { + for glob in &syncback_rules.ignore_paths { + if glob.is_match(suffix) { + log::debug!("Skipping {inst_path} because its path matches ignore pattern"); + continue 'syncback; + } + } + } + // Ignore trees; + for ignored in &syncback_rules.ignore_trees { + if inst_path.starts_with(ignored.as_str()) { + log::debug!("Tree {inst_path} is blocked by project"); + continue 'syncback; + } + } + } + if let Ok(suffix) = snapshot.parent_path.strip_prefix(project_path) { + if git_glob.is_match(suffix) { + log::warn!("Cannot syncback into `.git`, {inst_path} will be skipped."); continue; } } @@ -96,31 +101,20 @@ pub fn syncback_loop<'old>( .old_inst() .and_then(|inst| inst.metadata().middleware) .unwrap_or_else(|| get_best_middleware(snapshot.new_inst())); - log::trace!( - "Middleware for {} is {:?}", - get_inst_path(new_tree, snapshot.new), - middleware - ); + log::trace!("Middleware for {inst_path} is {:?}", middleware); if matches!(middleware, Middleware::Json | Middleware::Toml) { - log::warn!( - "Cannot syncback {middleware:?} at {}, skipping", - get_inst_path(new_tree, snapshot.new) - ); + log::warn!("Cannot syncback {middleware:?} at {inst_path}, skipping"); continue; } - let syncback = middleware.syncback(&snapshot).with_context(|| { - format!( - "Failed to syncback {}", - get_inst_path(new_tree, snapshot.new) - ) - })?; + let syncback = middleware + .syncback(&snapshot) + .with_context(|| format!("Failed to syncback {inst_path}"))?; if !syncback.removed_children.is_empty() { log::debug!( - "removed children for {}: {}", - get_inst_path(new_tree, snapshot.new), + "removed children for {inst_path}: {}", syncback.removed_children.len() ); for inst in &syncback.removed_children { @@ -216,30 +210,19 @@ pub fn get_best_middleware(inst: &Instance) -> Middleware { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields, rename_all = "camelCase")] pub struct SyncbackRules { - /// A list of paths in a file that will be ignored by Syncback. + /// A list of subtrees in a file that will be ignored by Syncback. + #[serde(default)] + ignore_trees: Vec, + /// A list of patterns to check against the path an Instance would serialize + /// to. If a path matches one of these, the Instance won't be syncbacked. #[serde(default)] - ignore_paths: Vec, + ignore_paths: Vec, /// A map of classes to properties to ignore for that class when doing /// syncback. #[serde(default)] ignore_properties: HashMap>, } -impl SyncbackRules { - /// Returns whether the provided Instance is allowed to be handled with - /// syncback. - #[inline] - pub fn acceptable(&self, dom: &WeakDom, inst: Ref) -> bool { - let path = get_inst_path(dom, inst); - for ignored in &self.ignore_paths { - if path.starts_with(ignored.as_str()) { - return false; - } - } - true - } -} - fn get_inst_path(dom: &WeakDom, referent: Ref) -> String { let mut path: VecDeque<&str> = VecDeque::new(); let mut inst = dom.get_by_ref(referent); From 01eee8a14a96c729d39ea0ea1018a49c6beeb36a Mon Sep 17 00:00:00 2001 From: Micah Date: Wed, 17 Jan 2024 14:23:50 -0800 Subject: [PATCH 119/366] Move file name validation to own module Check if there are control characters in a name --- src/syncback/file_names.rs | 36 ++++++++++++++++++++++++++++++++++++ src/syncback/mod.rs | 37 ++----------------------------------- 2 files changed, 38 insertions(+), 35 deletions(-) create mode 100644 src/syncback/file_names.rs diff --git a/src/syncback/file_names.rs b/src/syncback/file_names.rs new file mode 100644 index 000000000..2a7d5c72a --- /dev/null +++ b/src/syncback/file_names.rs @@ -0,0 +1,36 @@ +/// A list of file names that are not valid on Windows. +const INVALID_WINDOWS_NAMES: [&str; 22] = [ + "CON", "PRN", "AUX", "NUL", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", + "COM9", "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9", +]; + +/// A list of all characters that are outright forbidden to be included +/// in a file's name. +const FORBIDDEN_CHARS: [char; 9] = ['<', '>', ':', '"', '/', '|', '?', '*', '\\']; + +/// Returns whether a given name is a valid file name. This takes into account +/// rules for Windows, MacOS, and Linux. +/// +/// In practice however, these broadly overlap so the only unexpected behavior +/// is Windows, where there are 22 reserved names. +pub fn is_valid_file_name>(name: S) -> bool { + let str = name.as_ref(); + + if str.ends_with(' ') || str.ends_with('.') { + return false; + } + + for char in str.chars() { + if char.is_control() || FORBIDDEN_CHARS.contains(&char) { + return false; + } + } + + for forbidden in INVALID_WINDOWS_NAMES { + if str == forbidden { + return false; + } + } + + true +} diff --git a/src/syncback/mod.rs b/src/syncback/mod.rs index fb44a37d2..b2940689a 100644 --- a/src/syncback/mod.rs +++ b/src/syncback/mod.rs @@ -1,3 +1,4 @@ +mod file_names; mod fs_snapshot; mod snapshot; @@ -17,6 +18,7 @@ use crate::{ Project, }; +pub use file_names::is_valid_file_name; pub use fs_snapshot::FsSnapshot; pub use snapshot::{SyncbackData, SyncbackSnapshot}; @@ -232,38 +234,3 @@ fn get_inst_path(dom: &WeakDom, referent: Ref) -> String { } path.into_iter().collect::>().join("/") } - -/// A list of file names that are not valid on Windows. -const INVALID_WINDOWS_NAMES: [&str; 22] = [ - "CON", "PRN", "AUX", "NUL", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", - "COM9", "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9", -]; - -/// A list of all characters that are outright forbidden to be included -/// in a file's name. -const FORBIDDEN_CHARS: [char; 9] = ['<', '>', ':', '"', '/', '|', '?', '*', '\\']; - -/// Returns whether a given name is a valid file name. This takes into account -/// rules for Windows, MacOS, and Linux. -/// -/// In practice however, these broadly overlap so the only unexpected behavior -/// is Windows, where there are 22 reserved names. -pub fn is_valid_file_name>(name: S) -> bool { - let str = name.as_ref(); - - if str.ends_with(' ') || str.ends_with('.') { - return false; - } - // TODO check control characters - for forbidden in FORBIDDEN_CHARS { - if str.contains(forbidden) { - return false; - } - } - for forbidden in INVALID_WINDOWS_NAMES { - if str == forbidden { - return false; - } - } - true -} From dd4f614c21de3edac0ebe501850c00995c09d1bf Mon Sep 17 00:00:00 2001 From: Micah Date: Wed, 17 Jan 2024 16:21:19 -0800 Subject: [PATCH 120/366] Add function for getting a file-system safe name for Instances --- src/syncback/file_names.rs | 73 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/src/syncback/file_names.rs b/src/syncback/file_names.rs index 2a7d5c72a..ef20f3ed1 100644 --- a/src/syncback/file_names.rs +++ b/src/syncback/file_names.rs @@ -1,3 +1,76 @@ +//! Contains logic for generating new file names for Instances based on their +//! middleware. + +use std::borrow::Cow; + +use anyhow::Context; +use rbx_dom_weak::Instance; + +use crate::{snapshot::InstanceWithMeta, snapshot_middleware::Middleware}; + +pub fn name_for_inst<'old>( + middleware: Middleware, + new_inst: &Instance, + old_inst: Option>, +) -> anyhow::Result> { + if let Some(old_inst) = old_inst { + if let Some(source) = &old_inst.metadata().instigating_source { + source + .path() + .file_name() + .and_then(|s| s.to_str()) + .map(Cow::Borrowed) + .context("sources on the file system should be valid unicode and not be stubs") + } else { + anyhow::bail!("members of 'old' trees should have an instigating source!"); + } + } else { + Ok(match middleware { + Middleware::Dir + | Middleware::CsvDir + | Middleware::ServerScriptDir + | Middleware::ClientScriptDir + | Middleware::ModuleScriptDir => Cow::Owned(new_inst.name.clone()), + _ => { + let extension = extension_for_middleware(middleware); + let name = &new_inst.name; + if is_valid_file_name(name) { + Cow::Owned(format!("{name}.{extension}")) + } else { + anyhow::bail!("name '{name}' is not legal to write to the file system") + } + } + }) + } +} + +/// Returns the extension a provided piece of middleware is supposed to use. +fn extension_for_middleware(middleware: Middleware) -> &'static str { + match middleware { + Middleware::Csv => "csv", + Middleware::JsonModel => "model.json", + Middleware::Json => "json", + Middleware::ServerScript => "server.luau", + Middleware::ClientScript => "client.luau", + Middleware::ModuleScript => "luau", + Middleware::Project => "project.json", + Middleware::Rbxm => "rbxm", + Middleware::Rbxmx => "rbxmx", + Middleware::Toml => "toml", + Middleware::Text => "txt", + // These are manually specified and not `_` to guard against future + // middleware additions missing this function. + Middleware::Ignore => unimplemented!("syncback does not work on Ignore middleware"), + Middleware::Dir + | Middleware::CsvDir + | Middleware::ServerScriptDir + | Middleware::ClientScriptDir + | Middleware::ModuleScriptDir => { + unimplemented!("directory middleware requires special treatment") + } + } +} + /// A list of file names that are not valid on Windows. const INVALID_WINDOWS_NAMES: [&str; 22] = [ "CON", "PRN", "AUX", "NUL", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", From e4b37ee5f33bb0f1f7a7d7b783863071f32fc79e Mon Sep 17 00:00:00 2001 From: Micah Date: Wed, 17 Jan 2024 16:28:41 -0800 Subject: [PATCH 121/366] Store syncback rules in data struct --- src/syncback/mod.rs | 5 +---- src/syncback/snapshot.rs | 31 +++++++++++++++++++++++++++++-- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/src/syncback/mod.rs b/src/syncback/mod.rs index b2940689a..92a10a7e4 100644 --- a/src/syncback/mod.rs +++ b/src/syncback/mod.rs @@ -43,10 +43,7 @@ pub fn syncback_loop<'old>( vfs, old_tree, new_tree, - ignore_props: project - .syncback_rules - .as_ref() - .map(|rules| &rules.ignore_properties), + syncback_rules: project.syncback_rules.as_ref(), }; let mut snapshots = vec![SyncbackSnapshot { diff --git a/src/syncback/snapshot.rs b/src/syncback/snapshot.rs index 780b78eab..ecac671c9 100644 --- a/src/syncback/snapshot.rs +++ b/src/syncback/snapshot.rs @@ -5,6 +5,7 @@ use std::{ }; use crate::{ + glob::Glob, snapshot::{InstanceWithMeta, RojoTree}, variant_eq::variant_eq, }; @@ -13,12 +14,14 @@ use rbx_dom_weak::{ Instance, WeakDom, }; +use super::SyncbackRules; + #[derive(Clone, Copy)] pub struct SyncbackData<'new, 'old> { pub(super) vfs: &'old Vfs, pub(super) old_tree: &'old RojoTree, pub(super) new_tree: &'new WeakDom, - pub(super) ignore_props: Option<&'old HashMap>>, + pub(super) syncback_rules: Option<&'old SyncbackRules>, } pub struct SyncbackSnapshot<'new, 'old> { @@ -122,7 +125,7 @@ impl<'new, 'old> SyncbackSnapshot<'new, 'old> { /// Returns a set of properties that should not be written with syncback if /// one exists. fn get_property_filter(&self) -> Option> { - let filter = self.data.ignore_props?; + let filter = self.ignore_props()?; let mut set = BTreeSet::new(); let database = rbx_reflection_database::get(); @@ -185,4 +188,28 @@ impl<'new, 'old> SyncbackSnapshot<'new, 'old> { pub fn new_tree(&self) -> &'new WeakDom { self.data.new_tree } + + /// Returns user-specified property ignore rules. + #[inline] + pub fn ignore_props(&self) -> Option<&HashMap>> { + self.data + .syncback_rules + .map(|rules| &rules.ignore_properties) + } + + /// Returns user-specified ignore paths. + #[inline] + pub fn ignore_paths(&self) -> Option<&[Glob]> { + self.data + .syncback_rules + .map(|rules| rules.ignore_paths.as_slice()) + } + + /// Returns user-specified ignore tree. + #[inline] + pub fn ignore_tree(&self) -> Option<&[String]> { + self.data + .syncback_rules + .map(|rules| rules.ignore_trees.as_slice()) + } } From 3717034996b711b796c31d5fffa4ce9ccb3a2696 Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 18 Jan 2024 11:25:35 -0800 Subject: [PATCH 122/366] Add method for validating paths for syncback --- src/syncback/snapshot.rs | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/src/syncback/snapshot.rs b/src/syncback/snapshot.rs index ecac671c9..110354167 100644 --- a/src/syncback/snapshot.rs +++ b/src/syncback/snapshot.rs @@ -1,7 +1,7 @@ use memofs::Vfs; use std::{ collections::{BTreeSet, HashMap}, - path::PathBuf, + path::{Path, PathBuf}, }; use crate::{ @@ -147,6 +147,33 @@ impl<'new, 'old> SyncbackSnapshot<'new, 'old> { Some(set) } + /// Returns whether a given path is allowed for syncback by matching `path` + /// against every user specified glob for ignoring. + /// + /// If the provided `path` is absolute, it has `base_path` stripped from it + /// to allow globs to operate as if it were local. + #[inline] + pub fn is_valid_path(&self, base_path: &Path, path: &Path) -> bool { + if let Some(ignore_paths) = self.ignore_paths() { + if path.is_absolute() { + if let Ok(suffix) = path.strip_prefix(base_path) { + for glob in ignore_paths { + if glob.is_match(suffix) { + return false; + } + } + } + } else { + for glob in ignore_paths { + if glob.is_match(path) { + return false; + } + } + } + } + true + } + /// Returns an Instance from the old tree with the provided referent, if it /// exists. #[inline] From 4d7142ce6faa1588d7a7d1bbe52fd8f54b7a283f Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 18 Jan 2024 11:39:35 -0800 Subject: [PATCH 123/366] Move .git filtering to `is_valid_path` --- src/syncback/snapshot.rs | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/syncback/snapshot.rs b/src/syncback/snapshot.rs index 110354167..5c9a2e36a 100644 --- a/src/syncback/snapshot.rs +++ b/src/syncback/snapshot.rs @@ -2,6 +2,7 @@ use memofs::Vfs; use std::{ collections::{BTreeSet, HashMap}, path::{Path, PathBuf}, + sync::OnceLock, }; use crate::{ @@ -16,6 +17,9 @@ use rbx_dom_weak::{ use super::SyncbackRules; +/// A glob that can be used to tell if a path contains a `.git` folder. +static GIT_IGNORE_GLOB: OnceLock = OnceLock::new(); + #[derive(Clone, Copy)] pub struct SyncbackData<'new, 'old> { pub(super) vfs: &'old Vfs, @@ -154,20 +158,18 @@ impl<'new, 'old> SyncbackSnapshot<'new, 'old> { /// to allow globs to operate as if it were local. #[inline] pub fn is_valid_path(&self, base_path: &Path, path: &Path) -> bool { + let git_glob = GIT_IGNORE_GLOB.get_or_init(|| Glob::new(".git/**").unwrap()); + let test_path = match path.strip_prefix(base_path) { + Ok(suffix) => suffix, + Err(_) => path, + }; + if git_glob.is_match(test_path) { + return false; + } if let Some(ignore_paths) = self.ignore_paths() { - if path.is_absolute() { - if let Ok(suffix) = path.strip_prefix(base_path) { - for glob in ignore_paths { - if glob.is_match(suffix) { - return false; - } - } - } - } else { - for glob in ignore_paths { - if glob.is_match(path) { - return false; - } + for glob in ignore_paths { + if glob.is_match(test_path) { + return false; } } } From 89deb6e6eabcf45966c139ce3b098cfea2f52f97 Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 18 Jan 2024 11:40:03 -0800 Subject: [PATCH 124/366] Filter project node paths --- src/snapshot_middleware/project.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/snapshot_middleware/project.rs b/src/snapshot_middleware/project.rs index 7a724b6db..48c8ac004 100644 --- a/src/snapshot_middleware/project.rs +++ b/src/snapshot_middleware/project.rs @@ -333,6 +333,16 @@ pub fn syncback_project<'new, 'old>( while let Some((node, new_inst, old_inst)) = project_nodes.pop() { log::trace!("Processing node '{}' of project", old_inst.name()); + if let Some(node_path) = &node.path { + let node_path = node_path.path(); + if !snapshot.is_valid_path(base_path, node_path) { + log::debug!( + "Skipping {} because its path matches ignore pattern", + new_inst.name, + ); + continue; + } + } ref_to_path.insert(new_inst.referent(), node.path.as_ref()); old_child_map.extend(old_inst.children().iter().map(|referent| { @@ -379,7 +389,6 @@ pub fn syncback_project<'new, 'old>( if new_child.class != old_child.class_name() { anyhow::bail!("cannot change the class of {child_name} in project"); } - project_nodes.push((child_node, new_child, *old_child)); old_child_map.remove(child_name.as_str()); new_child_map.remove(child_name); From 5888295f43fc93661791fba2ec231c9a1feef251 Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 18 Jan 2024 11:41:22 -0800 Subject: [PATCH 125/366] Use `is_valid_path` for filtering syncback middleware --- src/syncback/mod.rs | 54 +++++++++++++++++---------------------------- 1 file changed, 20 insertions(+), 34 deletions(-) diff --git a/src/syncback/mod.rs b/src/syncback/mod.rs index 92a10a7e4..8b63af61b 100644 --- a/src/syncback/mod.rs +++ b/src/syncback/mod.rs @@ -6,10 +6,7 @@ use anyhow::Context; use memofs::Vfs; use rbx_dom_weak::{types::Ref, Instance, WeakDom}; use serde::{Deserialize, Serialize}; -use std::{ - collections::{HashMap, VecDeque}, - sync::OnceLock, -}; +use std::collections::{HashMap, VecDeque}; use crate::{ glob::Glob, @@ -18,13 +15,10 @@ use crate::{ Project, }; -pub use file_names::is_valid_file_name; +pub use file_names::{is_valid_file_name, name_for_inst}; pub use fs_snapshot::FsSnapshot; pub use snapshot::{SyncbackData, SyncbackSnapshot}; -/// A glob that can be used to tell if a path contains a `.git` folder. -static GIT_IGNORE_GLOB: OnceLock = OnceLock::new(); - pub fn syncback_loop<'old>( vfs: &'old Vfs, old_tree: &'old RojoTree, @@ -36,7 +30,6 @@ pub fn syncback_loop<'old>( log::debug!("Hashing file DOM"); let new_hashes = hash_tree(new_tree); - let git_glob = GIT_IGNORE_GLOB.get_or_init(|| Glob::new(".git/**").unwrap()); let project_path = project.folder_location(); let syncback_data = SyncbackData { @@ -71,31 +64,6 @@ pub fn syncback_loop<'old>( } } - if let Some(syncback_rules) = &project.syncback_rules { - // Ignore paths - if let Ok(suffix) = snapshot.parent_path.strip_prefix(project_path) { - for glob in &syncback_rules.ignore_paths { - if glob.is_match(suffix) { - log::debug!("Skipping {inst_path} because its path matches ignore pattern"); - continue 'syncback; - } - } - } - // Ignore trees; - for ignored in &syncback_rules.ignore_trees { - if inst_path.starts_with(ignored.as_str()) { - log::debug!("Tree {inst_path} is blocked by project"); - continue 'syncback; - } - } - } - if let Ok(suffix) = snapshot.parent_path.strip_prefix(project_path) { - if git_glob.is_match(suffix) { - log::warn!("Cannot syncback into `.git`, {inst_path} will be skipped."); - continue; - } - } - let middleware = snapshot .old_inst() .and_then(|inst| inst.metadata().middleware) @@ -107,6 +75,24 @@ pub fn syncback_loop<'old>( continue; } + let appended_name = name_for_inst(middleware, snapshot.new_inst(), snapshot.old_inst())?; + let working_path = snapshot.parent_path.join(appended_name.as_ref()); + + if !snapshot.is_valid_path(project_path, &working_path) { + log::debug!("Skipping {inst_path} because its path matches ignore pattern"); + continue; + } + + if let Some(syncback_rules) = &project.syncback_rules { + // Ignore trees; + for ignored in &syncback_rules.ignore_trees { + if inst_path.starts_with(ignored.as_str()) { + log::debug!("Tree {inst_path} is blocked by project"); + continue 'syncback; + } + } + } + let syncback = middleware .syncback(&snapshot) .with_context(|| format!("Failed to syncback {inst_path}"))?; From e0c8419196c8de093145af0366c013b7307bf8ec Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 18 Jan 2024 12:54:49 -0800 Subject: [PATCH 126/366] Use a more reasonable method for getting file names --- src/snapshot_middleware/csv.rs | 26 ++++-------------- src/snapshot_middleware/dir.rs | 8 +++--- src/snapshot_middleware/json_model.rs | 16 ++--------- src/snapshot_middleware/lua.rs | 39 +++++++-------------------- src/snapshot_middleware/mod.rs | 35 +++++++++++++++--------- src/snapshot_middleware/rbxm.rs | 14 ++-------- src/snapshot_middleware/rbxmx.rs | 16 ++--------- src/snapshot_middleware/txt.rs | 14 ++-------- src/syncback/mod.rs | 2 +- 9 files changed, 50 insertions(+), 120 deletions(-) diff --git a/src/snapshot_middleware/csv.rs b/src/snapshot_middleware/csv.rs index 2317927d2..73296f39a 100644 --- a/src/snapshot_middleware/csv.rs +++ b/src/snapshot_middleware/csv.rs @@ -101,20 +101,10 @@ pub fn snapshot_csv_init( pub fn syncback_csv<'new, 'old>( snapshot: &SyncbackSnapshot<'new, 'old>, + file_name: &str, ) -> anyhow::Result> { let new_inst = snapshot.new_inst(); - let path = snapshot - .old_inst() - .and_then(|inst| inst.metadata().instigating_source.as_ref()) - .map_or_else( - || { - // Since Roblox instances may or may not a `.` character in - // their names, we can't just use `.set_file_name` and - // `.set_extension`. - snapshot.parent_path.join(format!("{}.csv", snapshot.name)) - }, - |source| source.path().to_path_buf(), - ); + let path = snapshot.parent_path.join(file_name); let contents = if let Some(Variant::String(content)) = new_inst.properties.get("Contents") { content.as_str() @@ -176,17 +166,11 @@ pub fn syncback_csv<'new, 'old>( pub fn syncback_csv_init<'new, 'old>( snapshot: &SyncbackSnapshot<'new, 'old>, + dir_name: &str, ) -> anyhow::Result> { let new_inst = snapshot.new_inst(); - let path = snapshot - .old_inst() - .and_then(|inst| inst.metadata().instigating_source.as_ref()) - .map_or_else(|| snapshot.parent_path.as_path(), |source| source.path()) - // Instigating source for 'init' middleware is the directory they point - // to, not the `init.*` file, which is good since that's what we rerun - // the snapshot middleware on, but it means we have to do this: - .join("init.csv"); + let path = snapshot.parent_path.join(dir_name).join("init.csv"); let contents = if let Some(Variant::String(content)) = new_inst.properties.get("Contents") { content.as_str() @@ -194,7 +178,7 @@ pub fn syncback_csv_init<'new, 'old>( anyhow::bail!("LocalizationTables must have a `Contents` property that is a String") }; - let mut dir_syncback = syncback_dir_no_meta(snapshot)?; + let mut dir_syncback = syncback_dir_no_meta(snapshot, dir_name)?; let mut meta = if let Some(dir) = dir_meta(snapshot.vfs(), &path)? { dir } else { diff --git a/src/snapshot_middleware/dir.rs b/src/snapshot_middleware/dir.rs index 55f9922c9..be6ce1e25 100644 --- a/src/snapshot_middleware/dir.rs +++ b/src/snapshot_middleware/dir.rs @@ -99,11 +99,12 @@ pub fn snapshot_dir_no_meta( pub fn syncback_dir<'new, 'old>( snapshot: &SyncbackSnapshot<'new, 'old>, + dir_name: &str, ) -> anyhow::Result> { - let path = snapshot.parent_path.join(&snapshot.name); + let path = snapshot.parent_path.join(&dir_name); let new_inst = snapshot.new_inst(); - let mut dir_syncback = syncback_dir_no_meta(snapshot)?; + let mut dir_syncback = syncback_dir_no_meta(snapshot, dir_name)?; let mut meta = if let Some(dir) = dir_meta(snapshot.vfs(), &path)? { dir @@ -151,8 +152,9 @@ pub fn syncback_dir<'new, 'old>( pub fn syncback_dir_no_meta<'new, 'old>( snapshot: &SyncbackSnapshot<'new, 'old>, + dir_name: &str, ) -> anyhow::Result> { - let path = snapshot.parent_path.join(&snapshot.name); + let path = snapshot.parent_path.join(dir_name); let new_inst = snapshot.new_inst(); let mut children = Vec::new(); diff --git a/src/snapshot_middleware/json_model.rs b/src/snapshot_middleware/json_model.rs index 23ff61627..fce7c57ac 100644 --- a/src/snapshot_middleware/json_model.rs +++ b/src/snapshot_middleware/json_model.rs @@ -62,21 +62,9 @@ pub fn snapshot_json_model( pub fn syncback_json_model<'new, 'old>( snapshot: &SyncbackSnapshot<'new, 'old>, + file_name: &str, ) -> anyhow::Result> { - let path = snapshot - .old_inst() - .and_then(|inst| inst.metadata().instigating_source.as_ref()) - .map_or_else( - || { - // Since Roblox instances may or may not a `.` character in - // their names, we can't just use `.set_file_name` and - // `.set_extension`. - snapshot - .parent_path - .join(format!("{}.model.json", snapshot.name)) - }, - |source| source.path().to_path_buf(), - ); + let path = snapshot.parent_path.join(file_name); let new_inst = snapshot.new_inst(); diff --git a/src/snapshot_middleware/lua.rs b/src/snapshot_middleware/lua.rs index e452ab915..8a0743c85 100644 --- a/src/snapshot_middleware/lua.rs +++ b/src/snapshot_middleware/lua.rs @@ -130,26 +130,11 @@ pub fn snapshot_lua_init( } pub fn syncback_lua<'new, 'old>( - script_type: ScriptType, snapshot: &SyncbackSnapshot<'new, 'old>, + file_name: &str, ) -> anyhow::Result> { let new_inst = snapshot.new_inst(); - let path = snapshot - .old_inst() - .and_then(|inst| inst.metadata().instigating_source.as_ref()) - .map_or_else( - || { - // Since Roblox instances may or may not a `.` character in - // their names, we can't just use `.set_file_name` and - // `.set_extension`. - snapshot.parent_path.join(match script_type { - ScriptType::Server => format!("{}.server.lua", snapshot.name), - ScriptType::Client => format!("{}.client.lua", snapshot.name), - ScriptType::Module => format!("{}.lua", snapshot.name), - }) - }, - |source| source.path().to_path_buf(), - ); + let path = snapshot.parent_path.join(file_name); let contents = if let Some(Variant::String(source)) = new_inst.properties.get("Source") { source.as_bytes().to_vec() @@ -211,20 +196,14 @@ pub fn syncback_lua<'new, 'old>( pub fn syncback_lua_init<'new, 'old>( script_type: ScriptType, snapshot: &SyncbackSnapshot<'new, 'old>, + dir_name: &str, ) -> anyhow::Result> { let new_inst = snapshot.new_inst(); - let path = snapshot - .old_inst() - .and_then(|inst| inst.metadata().instigating_source.as_ref()) - .map_or_else(|| snapshot.parent_path.as_path(), |source| source.path()) - // Instigating source for 'init' middleware is the directory they point - // to, not the `init.*` file, which is good since that's what we rerun - // the snapshot middleware on, but it means we have to do this: - .join(match script_type { - ScriptType::Server => "init.server.lua", - ScriptType::Client => "init.client.lua", - ScriptType::Module => "init.lua", - }); + let path = snapshot.parent_path.join(dir_name).join(match script_type { + ScriptType::Server => "init.server.lua", + ScriptType::Client => "init.client.lua", + ScriptType::Module => "init.lua", + }); let contents = if let Some(Variant::String(source)) = new_inst.properties.get("Source") { source.as_bytes().to_vec() @@ -232,7 +211,7 @@ pub fn syncback_lua_init<'new, 'old>( anyhow::bail!("Scripts must have a `Source` property that is a String") }; - let dir_syncback = syncback_dir_no_meta(snapshot)?; + let dir_syncback = syncback_dir_no_meta(snapshot, dir_name)?; let mut meta = if let Some(dir) = dir_meta(snapshot.vfs(), &path)? { dir diff --git a/src/snapshot_middleware/mod.rs b/src/snapshot_middleware/mod.rs index 8e8022cfe..137fe5bb1 100644 --- a/src/snapshot_middleware/mod.rs +++ b/src/snapshot_middleware/mod.rs @@ -243,6 +243,7 @@ impl Middleware { pub fn syncback<'new, 'old>( &self, snapshot: &SyncbackSnapshot<'new, 'old>, + file_name: &str, ) -> anyhow::Result> { // We don't care about the names of projects if !is_valid_file_name(&snapshot.name) && !matches!(self, Middleware::Project) { @@ -252,23 +253,31 @@ impl Middleware { ); } match self { - Middleware::Csv => syncback_csv(snapshot), - Middleware::JsonModel => syncback_json_model(snapshot), + Middleware::Csv => syncback_csv(snapshot, file_name), + Middleware::JsonModel => syncback_json_model(snapshot, file_name), Middleware::Json => unimplemented!("cannot syncback Json middleware"), + // Projects are only generated from files that already exist on the + // file system, so we don't need to pass a file name. Middleware::Project => syncback_project(snapshot), - Middleware::ServerScript => syncback_lua(ScriptType::Server, snapshot), - Middleware::ClientScript => syncback_lua(ScriptType::Client, snapshot), - Middleware::ModuleScript => syncback_lua(ScriptType::Module, snapshot), - Middleware::Rbxm => syncback_rbxm(snapshot), - Middleware::Rbxmx => syncback_rbxmx(snapshot), + Middleware::ServerScript => syncback_lua(snapshot, file_name), + Middleware::ClientScript => syncback_lua(snapshot, file_name), + Middleware::ModuleScript => syncback_lua(snapshot, file_name), + Middleware::Rbxm => syncback_rbxm(snapshot, file_name), + Middleware::Rbxmx => syncback_rbxmx(snapshot, file_name), Middleware::Toml => unimplemented!("cannot syncback Toml middleware"), - Middleware::Text => syncback_txt(snapshot), + Middleware::Text => syncback_txt(snapshot, file_name), Middleware::Ignore => anyhow::bail!("cannot syncback Ignore middleware"), - Middleware::Dir => syncback_dir(snapshot), - Middleware::ServerScriptDir => syncback_lua_init(ScriptType::Server, snapshot), - Middleware::ClientScriptDir => syncback_lua_init(ScriptType::Client, snapshot), - Middleware::ModuleScriptDir => syncback_lua_init(ScriptType::Module, snapshot), - Middleware::CsvDir => syncback_csv_init(snapshot), + Middleware::Dir => syncback_dir(snapshot, file_name), + Middleware::ServerScriptDir => { + syncback_lua_init(ScriptType::Server, snapshot, file_name) + } + Middleware::ClientScriptDir => { + syncback_lua_init(ScriptType::Client, snapshot, file_name) + } + Middleware::ModuleScriptDir => { + syncback_lua_init(ScriptType::Module, snapshot, file_name) + } + Middleware::CsvDir => syncback_csv_init(snapshot, file_name), } } } diff --git a/src/snapshot_middleware/rbxm.rs b/src/snapshot_middleware/rbxm.rs index 4606c604a..4cbda433b 100644 --- a/src/snapshot_middleware/rbxm.rs +++ b/src/snapshot_middleware/rbxm.rs @@ -44,20 +44,10 @@ pub fn snapshot_rbxm( pub fn syncback_rbxm<'new, 'old>( snapshot: &SyncbackSnapshot<'new, 'old>, + file_name: &str, ) -> anyhow::Result> { let inst = snapshot.new_inst(); - let path = snapshot - .old_inst() - .and_then(|inst| inst.metadata().instigating_source.as_ref()) - .map_or_else( - || { - // Since Roblox instances may or may not a `.` character in - // their names, we can't just use `.set_file_name` and - // `.set_extension`. - snapshot.parent_path.join(format!("{}.rbxm", snapshot.name)) - }, - |source| source.path().to_path_buf(), - ); + let path = snapshot.parent_path.join(file_name); // Long-term, we probably want to have some logic for if this contains a // script. That's a future endeavor though. diff --git a/src/snapshot_middleware/rbxmx.rs b/src/snapshot_middleware/rbxmx.rs index 053b444bb..feb9198fe 100644 --- a/src/snapshot_middleware/rbxmx.rs +++ b/src/snapshot_middleware/rbxmx.rs @@ -46,22 +46,10 @@ pub fn snapshot_rbxmx( pub fn syncback_rbxmx<'new, 'old>( snapshot: &SyncbackSnapshot<'new, 'old>, + file_name: &str, ) -> anyhow::Result> { let inst = snapshot.new_inst(); - let path = snapshot - .old_inst() - .and_then(|inst| inst.metadata().instigating_source.as_ref()) - .map_or_else( - || { - // Since Roblox instances may or may not a `.` character in - // their names, we can't just use `.set_file_name` and - // `.set_extension`. - snapshot - .parent_path - .join(format!("{}.rbxmx", &snapshot.name)) - }, - |source| source.path().to_path_buf(), - ); + let path = snapshot.parent_path.join(file_name); // Long-term, we probably want to have some logic for if this contains a // script. That's a future endeavor though. diff --git a/src/snapshot_middleware/txt.rs b/src/snapshot_middleware/txt.rs index dd37648ee..2c9b91c93 100644 --- a/src/snapshot_middleware/txt.rs +++ b/src/snapshot_middleware/txt.rs @@ -51,20 +51,10 @@ pub fn snapshot_txt( pub fn syncback_txt<'new, 'old>( snapshot: &SyncbackSnapshot<'new, 'old>, + file_name: &str, ) -> anyhow::Result> { let new_inst = snapshot.new_inst(); - let path = snapshot - .old_inst() - .and_then(|inst| inst.metadata().instigating_source.as_ref()) - .map_or_else( - || { - // Since Roblox instances may or may not a `.` character in - // their names, we can't just use `.set_file_name` and - // `.set_extension`. - snapshot.parent_path.join(format!("{}.txt", &snapshot.name)) - }, - |source| source.path().to_path_buf(), - ); + let path = snapshot.parent_path.join(file_name); let contents = if let Some(Variant::String(source)) = new_inst.properties.get("Value") { source.as_bytes().to_vec() diff --git a/src/syncback/mod.rs b/src/syncback/mod.rs index 8b63af61b..78e433e9c 100644 --- a/src/syncback/mod.rs +++ b/src/syncback/mod.rs @@ -94,7 +94,7 @@ pub fn syncback_loop<'old>( } let syncback = middleware - .syncback(&snapshot) + .syncback(&snapshot, &appended_name) .with_context(|| format!("Failed to syncback {inst_path}"))?; if !syncback.removed_children.is_empty() { From 10d81d65337fc04f480e38dd18ca492f98ad6654 Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 18 Jan 2024 12:59:08 -0800 Subject: [PATCH 127/366] Remove unnecessary borrow in dir syncback middleware --- src/snapshot_middleware/dir.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/snapshot_middleware/dir.rs b/src/snapshot_middleware/dir.rs index be6ce1e25..7c1fa0ca3 100644 --- a/src/snapshot_middleware/dir.rs +++ b/src/snapshot_middleware/dir.rs @@ -101,7 +101,7 @@ pub fn syncback_dir<'new, 'old>( snapshot: &SyncbackSnapshot<'new, 'old>, dir_name: &str, ) -> anyhow::Result> { - let path = snapshot.parent_path.join(&dir_name); + let path = snapshot.parent_path.join(dir_name); let new_inst = snapshot.new_inst(); let mut dir_syncback = syncback_dir_no_meta(snapshot, dir_name)?; From aaa422dc606907412716df8e680e2072df84f7e4 Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 18 Jan 2024 14:03:47 -0800 Subject: [PATCH 128/366] Enforce unique child names for folders --- src/snapshot_middleware/dir.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/snapshot_middleware/dir.rs b/src/snapshot_middleware/dir.rs index 7c1fa0ca3..4724d5d4f 100644 --- a/src/snapshot_middleware/dir.rs +++ b/src/snapshot_middleware/dir.rs @@ -1,5 +1,5 @@ use std::{ - collections::{BTreeMap, HashMap}, + collections::{BTreeMap, HashMap, HashSet}, path::Path, }; @@ -160,8 +160,17 @@ pub fn syncback_dir_no_meta<'new, 'old>( let mut children = Vec::new(); let mut removed_children = Vec::new(); + // We have to enforce unique child names for the file system. + let mut duplicate_check = HashSet::with_capacity(new_inst.children().len()); + for child in new_inst.children() { + let child = snapshot.get_new_instance(*child).unwrap(); + if !duplicate_check.insert(&child.name) { + anyhow::bail!("Instance has duplicate child {}", child.name); + } + } + if let Some(old_inst) = snapshot.old_inst() { - let mut old_child_map = HashMap::new(); + let mut old_child_map = HashMap::with_capacity(old_inst.children().len()); for child in old_inst.children() { let inst = snapshot.get_old_instance(*child).unwrap(); old_child_map.insert(inst.name(), inst); From cd10c0d21ac27751f1d6e18ec3c4680895e1ee36 Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 18 Jan 2024 14:11:55 -0800 Subject: [PATCH 129/366] Make dedup checking case-insensitive --- src/snapshot_middleware/dir.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/snapshot_middleware/dir.rs b/src/snapshot_middleware/dir.rs index 4724d5d4f..6b18b757f 100644 --- a/src/snapshot_middleware/dir.rs +++ b/src/snapshot_middleware/dir.rs @@ -164,8 +164,8 @@ pub fn syncback_dir_no_meta<'new, 'old>( let mut duplicate_check = HashSet::with_capacity(new_inst.children().len()); for child in new_inst.children() { let child = snapshot.get_new_instance(*child).unwrap(); - if !duplicate_check.insert(&child.name) { - anyhow::bail!("Instance has duplicate child {}", child.name); + if !duplicate_check.insert(child.name.to_lowercase()) { + anyhow::bail!("Instance has duplicate child {}", child.name.to_lowercase()); } } From 5860a46b463d25fe509cd6f65264dfd69405be9e Mon Sep 17 00:00:00 2001 From: Micah Date: Mon, 22 Jan 2024 16:11:03 -0800 Subject: [PATCH 130/366] Correct filtering of properties to merge new and old properties --- src/snapshot_middleware/project.rs | 3 -- src/syncback/snapshot.rs | 57 +++++++++++++++--------------- 2 files changed, 28 insertions(+), 32 deletions(-) diff --git a/src/snapshot_middleware/project.rs b/src/snapshot_middleware/project.rs index 48c8ac004..a40e8195a 100644 --- a/src/snapshot_middleware/project.rs +++ b/src/snapshot_middleware/project.rs @@ -354,9 +354,6 @@ pub fn syncback_project<'new, 'old>( (&child.name, child) })); - // The fact we are not removing old properties might be a problem but - // removing them is worse! Consider a property that doesn't serialize, - // like HttpEnabled. let properties = &mut node.properties; let filtered_properties = snapshot diff --git a/src/syncback/snapshot.rs b/src/syncback/snapshot.rs index 5c9a2e36a..746cbb0d3 100644 --- a/src/syncback/snapshot.rs +++ b/src/syncback/snapshot.rs @@ -83,43 +83,42 @@ impl<'new, 'old> SyncbackSnapshot<'new, 'old> { properties.insert(name, value); } } - } else { - let class_data = rbx_reflection_database::get() - .classes - .get(inst.class.as_str()); - if let Some(class_data) = class_data { - let defaults = &class_data.default_properties; - for (name, value) in &inst.properties { - // We don't currently support refs or shared strings - if matches!(value, Variant::Ref(_) | Variant::SharedString(_)) { + } + let class_data = rbx_reflection_database::get() + .classes + .get(inst.class.as_str()); + if let Some(class_data) = class_data { + let defaults = &class_data.default_properties; + for (name, value) in &inst.properties { + // We don't currently support refs or shared strings + if matches!(value, Variant::Ref(_) | Variant::SharedString(_)) { + continue; + } + if let Some(list) = &filter { + if list.contains(name) { continue; } - if let Some(list) = &filter { - if list.contains(name) { - continue; - } - } - if let Some(default) = defaults.get(name.as_str()) { - if !variant_eq(value, default) { - properties.insert(name, value); - } - } else { + } + if let Some(default) = defaults.get(name.as_str()) { + if !variant_eq(value, default) { properties.insert(name, value); } + } else { + properties.insert(name, value); } - } else { - for (name, value) in &inst.properties { - // We don't currently support refs or shared strings - if matches!(value, Variant::Ref(_) | Variant::SharedString(_)) { + } + } else { + for (name, value) in &inst.properties { + // We don't currently support refs or shared strings + if matches!(value, Variant::Ref(_) | Variant::SharedString(_)) { + continue; + } + if let Some(list) = &filter { + if list.contains(name) { continue; } - if let Some(list) = &filter { - if list.contains(name) { - continue; - } - } - properties.insert(name, value); } + properties.insert(name, value); } } From ffb3da9edc5574a5e40e706ee522278f2cc6cd55 Mon Sep 17 00:00:00 2001 From: Micah Date: Mon, 22 Jan 2024 16:23:46 -0800 Subject: [PATCH 131/366] Filter out properties by class and name matching --- src/syncback/snapshot.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/syncback/snapshot.rs b/src/syncback/snapshot.rs index 746cbb0d3..e4551ed1c 100644 --- a/src/syncback/snapshot.rs +++ b/src/syncback/snapshot.rs @@ -90,6 +90,9 @@ impl<'new, 'old> SyncbackSnapshot<'new, 'old> { if let Some(class_data) = class_data { let defaults = &class_data.default_properties; for (name, value) in &inst.properties { + if filter_out_property(inst, name.as_str()) { + continue; + } // We don't currently support refs or shared strings if matches!(value, Variant::Ref(_) | Variant::SharedString(_)) { continue; @@ -109,6 +112,9 @@ impl<'new, 'old> SyncbackSnapshot<'new, 'old> { } } else { for (name, value) in &inst.properties { + if filter_out_property(inst, name.as_str()) { + continue; + } // We don't currently support refs or shared strings if matches!(value, Variant::Ref(_) | Variant::SharedString(_)) { continue; @@ -241,3 +247,15 @@ impl<'new, 'old> SyncbackSnapshot<'new, 'old> { .map(|rules| rules.ignore_trees.as_slice()) } } + +fn filter_out_property(inst: &Instance, prop_name: &str) -> bool { + // We don't need SourceAssetId. + if prop_name == "Tags" || prop_name == "Attributes" || prop_name == "SourceAssetId" { + return true; + } + match inst.class.as_str() { + "Script" | "LocalScript" | "ModuleScript" => prop_name == "Source", + "LocalizationTable" => prop_name == "Contents", + _ => false, + } +} From 2b0c4043767b36537232bdea9f2866c1d6883445 Mon Sep 17 00:00:00 2001 From: Micah Date: Mon, 22 Jan 2024 16:31:39 -0800 Subject: [PATCH 132/366] Remove unnecessary borrow in json_model syncback --- src/snapshot_middleware/json_model.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/snapshot_middleware/json_model.rs b/src/snapshot_middleware/json_model.rs index fce7c57ac..4fca94fee 100644 --- a/src/snapshot_middleware/json_model.rs +++ b/src/snapshot_middleware/json_model.rs @@ -104,12 +104,13 @@ pub fn syncback_json_model<'new, 'old>( attributes, }; - // TODO children + // TODO Do we want children to be included in a JSON model? + // Feels unlikely, since it could be very dense in the case of e.g. Tools Ok(SyncbackReturn { inst_snapshot: InstanceSnapshot::from_instance(new_inst), fs_snapshot: FsSnapshot::new().with_added_file( - &path, + path, serde_json::to_vec_pretty(&model).context("failed to serialize new JSON Model")?, ), children: Vec::new(), From 53355f0468f179d49d01ddf8ad1c2358af5e5b3b Mon Sep 17 00:00:00 2001 From: Micah Date: Mon, 22 Jan 2024 16:56:46 -0800 Subject: [PATCH 133/366] Move towards syncback command not sucking --- src/cli/syncback.rs | 105 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 84 insertions(+), 21 deletions(-) diff --git a/src/cli/syncback.rs b/src/cli/syncback.rs index bb32b7f90..d8ffccb23 100644 --- a/src/cli/syncback.rs +++ b/src/cli/syncback.rs @@ -3,15 +3,18 @@ use std::{ time::Instant, }; +use anyhow::Context; use clap::Parser; use memofs::Vfs; -use rbx_dom_weak::WeakDom; -use rbx_xml::DecodeOptions; +use rbx_dom_weak::{InstanceBuilder, WeakDom}; use crate::{serve_session::ServeSession, syncback::syncback_loop}; use super::resolve_path; +const UNKNOWN_INPUT_KIND_ERR: &str = "Could not detect what kind of file was inputted. \ + Expected input file to end in .rbxl, .rbxlx, .rbxm, or .rbxmx."; + /// Performs syncback for a project file #[derive(Debug, Parser)] pub struct SyncbackCommand { @@ -29,6 +32,15 @@ impl SyncbackCommand { let path_old = resolve_path(&self.project); let path_new = resolve_path(&self.input); + let input_kind = FileKind::from_path(&path_new).context(UNKNOWN_INPUT_KIND_ERR)?; + let dom_start = Instant::now(); + log::info!("Reading place file at {}", path_new.display()); + let dom_new = read_dom(&path_new, input_kind)?; + log::info!( + "Finished opening file in {:0.02}s", + dom_start.elapsed().as_secs_f32() + ); + let project_start = Instant::now(); log::info!("Opening project at {}", path_old.display()); let session_old = ServeSession::new(Vfs::new_default(), path_old.clone())?; @@ -39,13 +51,8 @@ impl SyncbackCommand { let dom_old = session_old.tree(); - let dom_start = Instant::now(); - log::info!("Reading place file at {}", path_new.display()); - let dom_new = read_dom(&path_new)?; - log::info!( - "Finished opening file in {:0.02}s", - dom_start.elapsed().as_secs_f32() - ); + log::debug!("Old root: {}", dom_old.inner().root().class); + log::debug!("New root: {}", dom_new.root().class); let start = Instant::now(); log::info!("Beginning syncback..."); @@ -64,18 +71,74 @@ impl SyncbackCommand { } } -fn read_dom(path: &Path) -> anyhow::Result { +fn read_dom(path: &Path, file_kind: FileKind) -> anyhow::Result { let content = fs_err::read(path)?; - Ok(if &content[0..8] == b" rbx_binary::from_reader(content.as_slice())?, + FileKind::Rbxlx => rbx_xml::from_reader(content.as_slice(), xml_decode_config())?, + FileKind::Rbxm => { + let temp_tree = rbx_binary::from_reader(content.as_slice())?; + let root_children = temp_tree.root().children(); + if root_children.len() != 1 { + anyhow::bail!( + "Rojo does not currently support models with more \ + than one Instance at the Root!" + ); + } + let real_root = temp_tree.get_by_ref(root_children[0]).unwrap(); + let mut new_tree = WeakDom::new(InstanceBuilder::new(&real_root.class)); + temp_tree.clone_multiple_into_external(real_root.children(), &mut new_tree); + + new_tree + } + FileKind::Rbxmx => { + let temp_tree = rbx_xml::from_reader(content.as_slice(), xml_decode_config())?; + let root_children = temp_tree.root().children(); + if root_children.len() != 1 { + anyhow::bail!( + "Rojo does not currently support models with more \ + than one Instance at the Root!" + ); + } + let real_root = temp_tree.get_by_ref(root_children[0]).unwrap(); + let mut new_tree = WeakDom::new(InstanceBuilder::new(&real_root.class)); + temp_tree.clone_multiple_into_external(real_root.children(), &mut new_tree); + + new_tree + } }) } + +fn xml_decode_config() -> rbx_xml::DecodeOptions<'static> { + rbx_xml::DecodeOptions::new().property_behavior(rbx_xml::DecodePropertyBehavior::ReadUnknown) +} + +/// The different kinds of input that Rojo can syncback. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum FileKind { + /// An XML model file. + Rbxmx, + + /// An XML place file. + Rbxlx, + + /// A binary model file. + Rbxm, + + /// A binary place file. + Rbxl, +} + +impl FileKind { + fn from_path(output: &Path) -> Option { + let extension = output.extension()?.to_str()?; + + match extension { + "rbxlx" => Some(FileKind::Rbxlx), + "rbxmx" => Some(FileKind::Rbxmx), + "rbxl" => Some(FileKind::Rbxl), + "rbxm" => Some(FileKind::Rbxm), + _ => None, + } + } +} From 8c1851f16ee00a8940fd33a5e2905fc3b4a7b08e Mon Sep 17 00:00:00 2001 From: Micah Date: Wed, 24 Jan 2024 19:59:58 -0800 Subject: [PATCH 134/366] Add function for hashing a tree without using descendants --- src/snapshot/hash/mod.rs | 65 +++++++++++++++++++++++++++++++++------- 1 file changed, 54 insertions(+), 11 deletions(-) diff --git a/src/snapshot/hash/mod.rs b/src/snapshot/hash/mod.rs index 129a3819b..82f17a295 100644 --- a/src/snapshot/hash/mod.rs +++ b/src/snapshot/hash/mod.rs @@ -1,4 +1,4 @@ -//! Hashing utility for a RojoTree +//! Hashing utilities for a WeakDom. mod variant; pub use variant::*; @@ -12,18 +12,45 @@ use std::collections::{HashMap, VecDeque}; use crate::variant_eq::variant_eq; +/// Returns a map of every `Ref` in the `WeakDom` to a hashed version of the +/// `Instance` it points to, including the properties but not including the +/// descendants of the Instance. +/// +/// The hashes **do not** include the descendants of the Instances in them, +/// so they should only be used for comparing Instances directly. To compare a +/// subtree, use `hash_tree`. +pub fn hash_tree_no_descendants(dom: &WeakDom) -> HashMap { + let mut map: HashMap = HashMap::new(); + let mut order = descendants(dom); + + let mut prop_list = Vec::with_capacity(2); + + while let Some(referent) = order.pop() { + let inst = dom.get_by_ref(referent).unwrap(); + let hash = hash_inst_no_descendants(inst, &mut prop_list); + + map.insert(referent, hash.finalize()); + } + + map +} + +/// Returns a map of every `Ref` in the `WeakDom` to a hashed version of the +/// `Instance` it points to, including the properties and descendants of the +/// `Instance`. +/// +/// The hashes **do** include the descendants of the Instances in them, +/// so they should only be used for comparing subtrees directly. To compare an +/// `Instance` directly, use `hash_tree_no_descendants`. pub fn hash_tree(dom: &WeakDom) -> HashMap { let mut map: HashMap = HashMap::new(); let mut order = descendants(dom); let mut prop_list = Vec::with_capacity(2); - // function get_hash_id(inst) - // return hash({ sort(foreach(inst.properties, hash)), sort(foreach(inst.children, get_hash_id)) }) - // end while let Some(referent) = order.pop() { let inst = dom.get_by_ref(referent).unwrap(); - let hash = hash_inst(&mut prop_list, &map, inst); + let hash = hash_inst(inst, &mut prop_list, &map); map.insert(referent, hash); } @@ -31,11 +58,12 @@ pub fn hash_tree(dom: &WeakDom) -> HashMap { map } -pub fn hash_inst<'map, 'inst>( - prop_list: &mut Vec<(&'inst str, &'inst Variant)>, - map: &'map HashMap, +/// Hashes an Instance using its class, name, and properties. The passed +/// `prop_list` is used to sort properties before hashing them. +fn hash_inst_no_descendants<'inst>( inst: &'inst Instance, -) -> Hash { + prop_list: &mut Vec<(&'inst str, &'inst Variant)>, +) -> Hasher { let mut hasher = Hasher::new(); hasher.update(inst.class.as_bytes()); hasher.update(inst.name.as_bytes()); @@ -61,7 +89,24 @@ pub fn hash_inst<'map, 'inst>( hash_variant(&mut hasher, value) } + prop_list.clear(); + + hasher +} + +/// Hashes an Instance using its class, name, properties, and descendants. +/// The passed `prop_list` is used to sort properties before hashing them. +/// +/// # Panics +/// If any children of the Instance are inside `map`, this function will panic. +fn hash_inst<'inst>( + inst: &'inst Instance, + prop_list: &mut Vec<(&'inst str, &'inst Variant)>, + map: &HashMap, +) -> Hash { + let mut hasher = hash_inst_no_descendants(inst, prop_list); let mut child_list = Vec::with_capacity(inst.children().len()); + for child in inst.children() { if let Some(hash) = map.get(child) { child_list.push(hash.as_bytes()) @@ -74,8 +119,6 @@ pub fn hash_inst<'map, 'inst>( hasher.update(hash); } - prop_list.clear(); - hasher.finalize() } From 186a386ade38c4818e29c7b37a174c92e8d5aff8 Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 25 Jan 2024 11:21:27 -0800 Subject: [PATCH 135/366] Add function for doing both hash types at once, use it --- src/snapshot/hash/mod.rs | 81 +++++++++++++++++++----------- src/snapshot_middleware/project.rs | 6 +-- src/syncback/mod.rs | 28 +++++------ src/syncback/snapshot.rs | 39 ++++++++++++-- 4 files changed, 104 insertions(+), 50 deletions(-) diff --git a/src/snapshot/hash/mod.rs b/src/snapshot/hash/mod.rs index 82f17a295..d9c87f67b 100644 --- a/src/snapshot/hash/mod.rs +++ b/src/snapshot/hash/mod.rs @@ -12,6 +12,42 @@ use std::collections::{HashMap, VecDeque}; use crate::variant_eq::variant_eq; +/// Returns a map of hashes for every Instance contained in the DOM. +/// Hashes are mapped to the Instance's referent. +/// +/// The first hash in each tuple is the Instance's hash with no descendants, +/// the second is one using descendants. +pub fn hash_tree_both(dom: &WeakDom) -> HashMap { + let mut map: HashMap = HashMap::new(); + let mut order = descendants(dom); + + let mut prop_list = Vec::with_capacity(2); + + while let Some(referent) = order.pop() { + let inst = dom.get_by_ref(referent).unwrap(); + let mut hasher = hash_inst_no_descendants(inst, &mut prop_list); + let no_descendants = hasher.finalize(); + + let mut child_list = Vec::with_capacity(inst.children().len()); + + for child in inst.children() { + if let Some((_, descendant)) = map.get(child) { + child_list.push(descendant.as_bytes()) + } else { + panic!("Invariant: child {} not hashed before its parent", child); + } + } + child_list.sort_unstable(); + for hash in child_list { + hasher.update(hash); + } + + map.insert(referent, (no_descendants, hasher.finalize())); + } + + map +} + /// Returns a map of every `Ref` in the `WeakDom` to a hashed version of the /// `Instance` it points to, including the properties but not including the /// descendants of the Instance. @@ -50,9 +86,22 @@ pub fn hash_tree(dom: &WeakDom) -> HashMap { while let Some(referent) = order.pop() { let inst = dom.get_by_ref(referent).unwrap(); - let hash = hash_inst(inst, &mut prop_list, &map); + let mut hasher = hash_inst_no_descendants(inst, &mut prop_list); - map.insert(referent, hash); + let mut child_list = Vec::with_capacity(inst.children().len()); + for child in inst.children() { + if let Some(hash) = map.get(child) { + child_list.push(hash.as_bytes()) + } else { + panic!("Invariant: child {} not hashed before its parent", child); + } + } + child_list.sort_unstable(); + for hash in child_list { + hasher.update(hash); + } + + map.insert(referent, hasher.finalize()); } map @@ -94,34 +143,6 @@ fn hash_inst_no_descendants<'inst>( hasher } -/// Hashes an Instance using its class, name, properties, and descendants. -/// The passed `prop_list` is used to sort properties before hashing them. -/// -/// # Panics -/// If any children of the Instance are inside `map`, this function will panic. -fn hash_inst<'inst>( - inst: &'inst Instance, - prop_list: &mut Vec<(&'inst str, &'inst Variant)>, - map: &HashMap, -) -> Hash { - let mut hasher = hash_inst_no_descendants(inst, prop_list); - let mut child_list = Vec::with_capacity(inst.children().len()); - - for child in inst.children() { - if let Some(hash) = map.get(child) { - child_list.push(hash.as_bytes()) - } else { - panic!("Invariant: child {} not hashed before its parent", child); - } - } - child_list.sort_unstable(); - for hash in child_list { - hasher.update(hash); - } - - hasher.finalize() -} - pub(crate) fn descendants(dom: &WeakDom) -> Vec { let mut queue = VecDeque::new(); let mut ordered = Vec::new(); diff --git a/src/snapshot_middleware/project.rs b/src/snapshot_middleware/project.rs index a40e8195a..d7295ad28 100644 --- a/src/snapshot_middleware/project.rs +++ b/src/snapshot_middleware/project.rs @@ -1,4 +1,4 @@ -use std::{borrow::Cow, collections::HashMap, path::Path}; +use std::{borrow::Cow, collections::HashMap, path::Path, rc::Rc}; use anyhow::{bail, Context}; use memofs::Vfs; @@ -408,7 +408,7 @@ pub fn syncback_project<'new, 'old>( if let Some(old_inst) = old_child_map.get(new_name.as_str()) { // This new instance represents an older one! children.push(SyncbackSnapshot { - data: snapshot.data, + data: Rc::clone(&snapshot.data), old: Some(old_inst.id()), new: new_child.referent(), parent_path, @@ -418,7 +418,7 @@ pub fn syncback_project<'new, 'old>( } else { // This new instance is... new. children.push(SyncbackSnapshot { - data: snapshot.data, + data: Rc::clone(&snapshot.data), old: None, new: new_child.referent(), parent_path, diff --git a/src/syncback/mod.rs b/src/syncback/mod.rs index 78e433e9c..30ab75901 100644 --- a/src/syncback/mod.rs +++ b/src/syncback/mod.rs @@ -6,11 +6,14 @@ use anyhow::Context; use memofs::Vfs; use rbx_dom_weak::{types::Ref, Instance, WeakDom}; use serde::{Deserialize, Serialize}; -use std::collections::{HashMap, VecDeque}; +use std::{ + collections::{HashMap, VecDeque}, + rc::Rc, +}; use crate::{ glob::Glob, - snapshot::{hash_tree, InstanceSnapshot, InstanceWithMeta, RojoTree}, + snapshot::{hash_tree_both, InstanceSnapshot, InstanceWithMeta, RojoTree}, snapshot_middleware::Middleware, Project, }; @@ -26,18 +29,20 @@ pub fn syncback_loop<'old>( project: &'old Project, ) -> anyhow::Result> { log::debug!("Hashing project DOM"); - let old_hashes = hash_tree(old_tree.inner()); + let old_hashes = hash_tree_both(old_tree.inner()); log::debug!("Hashing file DOM"); - let new_hashes = hash_tree(new_tree); + let new_hashes = hash_tree_both(new_tree); let project_path = project.folder_location(); - let syncback_data = SyncbackData { + let syncback_data = Rc::new(SyncbackData { vfs, old_tree, new_tree, syncback_rules: project.syncback_rules.as_ref(), - }; + old_hashes, + new_hashes, + }); let mut snapshots = vec![SyncbackSnapshot { data: syncback_data, @@ -54,14 +59,9 @@ pub fn syncback_loop<'old>( let inst_path = get_inst_path(new_tree, snapshot.new); // We can quickly check that two subtrees are identical and if they are, // skip reconciling them. - if let Some(old_ref) = snapshot.old { - if old_hashes.get(&old_ref) == new_hashes.get(&snapshot.new) { - log::trace!( - "Skipping {inst_path} due to it being identically hashed as {:?}", - old_hashes.get(&old_ref) - ); - continue; - } + if snapshot.trees_are_deep_eq() { + log::trace!("Skipping {inst_path} due to it being identically hashed"); + continue; } let middleware = snapshot diff --git a/src/syncback/snapshot.rs b/src/syncback/snapshot.rs index e4551ed1c..d2ebf747e 100644 --- a/src/syncback/snapshot.rs +++ b/src/syncback/snapshot.rs @@ -1,7 +1,9 @@ +use blake3::Hash; use memofs::Vfs; use std::{ collections::{BTreeSet, HashMap}, path::{Path, PathBuf}, + rc::Rc, sync::OnceLock, }; @@ -20,16 +22,17 @@ use super::SyncbackRules; /// A glob that can be used to tell if a path contains a `.git` folder. static GIT_IGNORE_GLOB: OnceLock = OnceLock::new(); -#[derive(Clone, Copy)] pub struct SyncbackData<'new, 'old> { pub(super) vfs: &'old Vfs, pub(super) old_tree: &'old RojoTree, pub(super) new_tree: &'new WeakDom, pub(super) syncback_rules: Option<&'old SyncbackRules>, + pub(super) new_hashes: HashMap, + pub(super) old_hashes: HashMap, } pub struct SyncbackSnapshot<'new, 'old> { - pub data: SyncbackData<'new, 'old>, + pub data: Rc>, pub old: Option, pub new: Ref, pub parent_path: PathBuf, @@ -42,7 +45,7 @@ impl<'new, 'old> SyncbackSnapshot<'new, 'old> { #[inline] pub fn with_parent(&self, new_name: String, new_ref: Ref, old_ref: Option) -> Self { Self { - data: self.data, + data: Rc::clone(&self.data), old: old_ref, new: new_ref, parent_path: self.parent_path.join(&self.name), @@ -181,6 +184,36 @@ impl<'new, 'old> SyncbackSnapshot<'new, 'old> { true } + /// Returns whether two Instances are equal, using their hash for O(1) + /// comparisons. This comparison **does not** take descendants into account. + #[inline] + pub fn trees_are_eq(&self) -> bool { + let old_hashes = self + .old + .and_then(|referent| self.data.old_hashes.get(&referent)); + let new_hashes = self.data.new_hashes.get(&self.new); + + match (old_hashes, new_hashes) { + (Some((old_hash, _)), Some((new_hash, _))) => old_hash == new_hash, + _ => false, + } + } + + /// Returns whether two Instance trees are equal, using their hash for O(1) + /// comparisons. This comparison **does** take descendants into account. + #[inline] + pub fn trees_are_deep_eq(&self) -> bool { + let old_hashes = self + .old + .and_then(|referent| self.data.old_hashes.get(&referent)); + let new_hashes = self.data.new_hashes.get(&self.new); + + match (old_hashes, new_hashes) { + (Some((_, old_hash)), Some((_, new_hash))) => old_hash == new_hash, + _ => false, + } + } + /// Returns an Instance from the old tree with the provided referent, if it /// exists. #[inline] From 36b288ba00070a86bfbe75766aaad46178f43f1c Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 25 Jan 2024 14:30:34 -0800 Subject: [PATCH 136/366] Slightly improve UX of duplicate name checking --- src/snapshot_middleware/dir.rs | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/snapshot_middleware/dir.rs b/src/snapshot_middleware/dir.rs index 6b18b757f..7fb49267a 100644 --- a/src/snapshot_middleware/dir.rs +++ b/src/snapshot_middleware/dir.rs @@ -161,11 +161,22 @@ pub fn syncback_dir_no_meta<'new, 'old>( let mut removed_children = Vec::new(); // We have to enforce unique child names for the file system. - let mut duplicate_check = HashSet::with_capacity(new_inst.children().len()); - for child in new_inst.children() { - let child = snapshot.get_new_instance(*child).unwrap(); - if !duplicate_check.insert(child.name.to_lowercase()) { - anyhow::bail!("Instance has duplicate child {}", child.name.to_lowercase()); + let mut child_names = HashSet::with_capacity(new_inst.children().len()); + let mut duplicate_set = HashSet::new(); + for child_ref in new_inst.children() { + let child = snapshot.get_new_instance(*child_ref).unwrap(); + if !child_names.insert(child.name.to_lowercase()) { + duplicate_set.insert(child.name.as_str()); + } + } + if !duplicate_set.is_empty() { + if duplicate_set.len() <= 25 { + anyhow::bail!( + "Instance has children with duplicate name (case may not exactly match):\n {}", + duplicate_set.into_iter().collect::>().join(", ") + ); + } else { + anyhow::bail!("Instance has more than 25 children with duplicate names"); } } From 09e8ac18dc75caeaf780397d7ebb9be4ecbc9d4e Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 25 Jan 2024 14:31:40 -0800 Subject: [PATCH 137/366] Mildly improve performance of checking classes for JSON models Also add ParticleEmitter as modeljson and remove screengui exceptionalism --- src/syncback/mod.rs | 43 ++++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/src/syncback/mod.rs b/src/syncback/mod.rs index 30ab75901..c3a15d908 100644 --- a/src/syncback/mod.rs +++ b/src/syncback/mod.rs @@ -7,8 +7,9 @@ use memofs::Vfs; use rbx_dom_weak::{types::Ref, Instance, WeakDom}; use serde::{Deserialize, Serialize}; use std::{ - collections::{HashMap, VecDeque}, + collections::{HashMap, HashSet, VecDeque}, rc::Rc, + sync::OnceLock, }; use crate::{ @@ -134,25 +135,29 @@ pub struct SyncbackReturn<'new, 'old> { } pub fn get_best_middleware(inst: &Instance) -> Middleware { - match inst.class.as_str() { - "Folder" | "Configuration" | "Tool" | "ScreenGui" => Middleware::Dir, - "Sound" - | "SoundGroup" - | "Sky" - | "Atmosphere" - | "BloomEffect" - | "BlurEffect" - | "ColorCorrectionEffect" - | "DepthOfFieldEffect" - | "SunRaysEffect" => { - if inst.children().is_empty() { - Middleware::JsonModel - } else { - // This begs the question of an init.model.json but we'll leave - // that for another day. - Middleware::Dir - } + // At some point, we're better off using an O(1) method for checking + // equality for classes like this. + static JSON_MODEL_CLASSES: OnceLock> = OnceLock::new(); + let json_model_classes = JSON_MODEL_CLASSES.get_or_init(|| { + maplit::hashset! { + "Sound", "SoundGroup", "Sky", "Atmosphere", "BloomEffect", + "BlurEffect", "ColorCorrectionEffect", "DepthOfFieldEffect", + "SunRaysEffect", "ParticleEmitter" } + }); + + if json_model_classes.contains(inst.class.as_str()) { + if inst.children().is_empty() { + return Middleware::JsonModel; + } else { + // This begs the question of an init.model.json but we'll leave + // that for another day. + return Middleware::Dir; + } + } + + match inst.class.as_str() { + "Folder" | "Configuration" | "Tool" => Middleware::Dir, "StringValue" => { if inst.children().is_empty() { Middleware::Text From 55354cfeaa1c6948e69688a42579da7460933e76 Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 25 Jan 2024 14:51:25 -0800 Subject: [PATCH 138/366] Don't publicly export DirectoryMetadata --- src/snapshot_middleware/csv.rs | 3 +-- src/snapshot_middleware/lua.rs | 3 +-- src/snapshot_middleware/mod.rs | 3 +-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/snapshot_middleware/csv.rs b/src/snapshot_middleware/csv.rs index 73296f39a..188dc5483 100644 --- a/src/snapshot_middleware/csv.rs +++ b/src/snapshot_middleware/csv.rs @@ -18,8 +18,7 @@ use crate::{ use super::{ dir::{dir_meta, snapshot_dir_no_meta, syncback_dir_no_meta}, - meta_file::{file_meta, AdjacentMetadata}, - DirectoryMetadata, + meta_file::{file_meta, AdjacentMetadata, DirectoryMetadata}, }; pub fn snapshot_csv( diff --git a/src/snapshot_middleware/lua.rs b/src/snapshot_middleware/lua.rs index 8a0743c85..ee3c5bea8 100644 --- a/src/snapshot_middleware/lua.rs +++ b/src/snapshot_middleware/lua.rs @@ -16,8 +16,7 @@ use crate::{ use super::{ dir::{dir_meta, snapshot_dir_no_meta, syncback_dir_no_meta}, - meta_file::{file_meta, AdjacentMetadata}, - DirectoryMetadata, + meta_file::{file_meta, AdjacentMetadata, DirectoryMetadata}, }; #[derive(Debug)] diff --git a/src/snapshot_middleware/mod.rs b/src/snapshot_middleware/mod.rs index 137fe5bb1..60587877e 100644 --- a/src/snapshot_middleware/mod.rs +++ b/src/snapshot_middleware/mod.rs @@ -50,8 +50,7 @@ use self::{ }; pub use self::{ - lua::ScriptType, meta_file::DirectoryMetadata, project::snapshot_project_node, - util::emit_legacy_scripts_default, + lua::ScriptType, project::snapshot_project_node, util::emit_legacy_scripts_default, }; /// Returns an `InstanceSnapshot` for the provided path. From bbb59771868a7f7222a205feb1458806c37a5649 Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 25 Jan 2024 14:58:05 -0800 Subject: [PATCH 139/366] Correct problem with using old_inst properties where we shouldn't --- src/snapshot_middleware/project.rs | 2 +- src/syncback/snapshot.rs | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/snapshot_middleware/project.rs b/src/snapshot_middleware/project.rs index d7295ad28..59091308e 100644 --- a/src/snapshot_middleware/project.rs +++ b/src/snapshot_middleware/project.rs @@ -357,7 +357,7 @@ pub fn syncback_project<'new, 'old>( let properties = &mut node.properties; let filtered_properties = snapshot - .get_filtered_properties(new_inst.referent()) + .get_filtered_properties(new_inst.referent(), Some(old_inst.id())) .expect("all project nodes should exist in both trees when in queue"); for (name, value) in filtered_properties { properties.insert( diff --git a/src/syncback/snapshot.rs b/src/syncback/snapshot.rs index d2ebf747e..236d223b9 100644 --- a/src/syncback/snapshot.rs +++ b/src/syncback/snapshot.rs @@ -61,7 +61,7 @@ impl<'new, 'old> SyncbackSnapshot<'new, 'old> { /// clone any data. This is left to the consumer. #[inline] pub fn new_filtered_properties(&self) -> HashMap<&'new str, &'new Variant> { - self.get_filtered_properties(self.new).unwrap() + self.get_filtered_properties(self.new, self.old).unwrap() } /// Returns a map of properties for an Instance from the 'new' tree @@ -73,6 +73,7 @@ impl<'new, 'old> SyncbackSnapshot<'new, 'old> { pub fn get_filtered_properties( &self, new_ref: Ref, + old_ref: Option, ) -> Option> { let inst = self.get_new_instance(new_ref)?; let mut properties: HashMap<&str, &Variant> = @@ -80,7 +81,7 @@ impl<'new, 'old> SyncbackSnapshot<'new, 'old> { let filter = self.get_property_filter(); - if let Some(old_inst) = self.old_inst() { + if let Some(old_inst) = old_ref.and_then(|referent| self.get_old_instance(referent)) { for (name, value) in &inst.properties { if old_inst.properties().contains_key(name) { properties.insert(name, value); From fae849e7ac1ef082480a13e1ba95d7dc4ea2bdd8 Mon Sep 17 00:00:00 2001 From: Micah Date: Thu, 25 Jan 2024 15:54:52 -0800 Subject: [PATCH 140/366] Add baseline for better metadata support --- src/snapshot_middleware/lua.rs | 41 +++---------------- src/snapshot_middleware/meta_file.rs | 59 +++++++++++++++++++++++++++- 2 files changed, 63 insertions(+), 37 deletions(-) diff --git a/src/snapshot_middleware/lua.rs b/src/snapshot_middleware/lua.rs index ee3c5bea8..f8ef52753 100644 --- a/src/snapshot_middleware/lua.rs +++ b/src/snapshot_middleware/lua.rs @@ -141,45 +141,16 @@ pub fn syncback_lua<'new, 'old>( anyhow::bail!("Scripts must have a `Source` property that is a String") }; - let mut meta = if let Some(meta) = file_meta(snapshot.vfs(), &path, &snapshot.name)? { - meta - } else { - AdjacentMetadata { - ignore_unknown_instances: None, - properties: BTreeMap::new(), - attributes: BTreeMap::new(), - path: path - .with_file_name(&snapshot.name) - .with_extension("meta.json"), - } - }; - for (name, value) in snapshot.new_filtered_properties() { - if name == "Source" { - continue; - } else if name == "Attributes" || name == "AttributesSerialize" { - if let Variant::Attributes(attrs) = value { - meta.attributes.extend(attrs.iter().map(|(name, value)| { - ( - name.to_string(), - UnresolvedValue::FullyQualified(value.clone()), - ) - })) - } else { - log::error!("Property {name} should be Attributes but is not"); - } - } else { - meta.properties.insert( - name.to_string(), - UnresolvedValue::from_variant(value.to_owned(), &new_inst.class, name), - ); - } - } + let meta = AdjacentMetadata::from_syncback_snapshot(snapshot, path.clone())?; + let mut fs_snapshot = FsSnapshot::new(); fs_snapshot.add_file(path, contents); if !meta.is_empty() { fs_snapshot.add_file( - &meta.path, - serde_json::to_vec_pretty(&meta).context("could not serialize new init.meta.json")?, + snapshot + .parent_path + .join(format!("{}.meta.json", new_inst.name)), + serde_json::to_vec_pretty(&meta).context("cannot serialize metadata")?, ); } diff --git a/src/snapshot_middleware/meta_file.rs b/src/snapshot_middleware/meta_file.rs index 058c10e11..a3b8fdaa9 100644 --- a/src/snapshot_middleware/meta_file.rs +++ b/src/snapshot_middleware/meta_file.rs @@ -6,10 +6,10 @@ use std::{ use anyhow::{format_err, Context}; use memofs::{IoResultExt as _, Vfs}; -use rbx_dom_weak::types::Attributes; +use rbx_dom_weak::types::{Attributes, Variant}; use serde::{Deserialize, Serialize}; -use crate::{resolution::UnresolvedValue, snapshot::InstanceSnapshot}; +use crate::{resolution::UnresolvedValue, snapshot::InstanceSnapshot, syncback::SyncbackSnapshot}; /// Represents metadata in a sibling file with the same basename. /// @@ -44,6 +44,61 @@ impl AdjacentMetadata { Ok(meta) } + /// Constructs an `AdjacentMetadata` from the provided snapshot, assuming it + /// will be at the provided path. + pub fn from_syncback_snapshot( + snapshot: &SyncbackSnapshot, + path: PathBuf, + ) -> anyhow::Result { + let mut properties = BTreeMap::new(); + let mut attributes = BTreeMap::new(); + + let ignore_unknown_instances = if let Some(old_inst) = snapshot.old_inst() { + if old_inst.metadata().ignore_unknown_instances { + Some(true) + } else { + None + } + } else { + None + }; + + let class = &snapshot.new_inst().class; + for (name, value) in snapshot.new_filtered_properties() { + properties.insert( + name.to_owned(), + UnresolvedValue::from_variant(value.clone(), class, name), + ); + } + if let Some(value) = snapshot.new_inst().properties.get("Attributes") { + if let Variant::Attributes(attrs) = value { + for (name, value) in attrs.iter() { + let value = if let Variant::BinaryString(bstr) = value { + match std::str::from_utf8(bstr.as_ref()) { + Ok(str) => Variant::String(str.to_owned()), + Err(_) => value.clone(), + } + } else { + value.clone() + }; + attributes.insert(name.to_owned(), UnresolvedValue::FullyQualified(value)); + } + } else { + anyhow::bail!( + "expected Attributes to be of type Attributes but it was {:?}", + value.ty() + ); + } + } + + Ok(Self { + ignore_unknown_instances, + properties, + attributes, + path, + }) + } + pub fn apply_ignore_unknown_instances(&mut self, snapshot: &mut InstanceSnapshot) { if let Some(ignore) = self.ignore_unknown_instances.take() { snapshot.metadata.ignore_unknown_instances = ignore; From e670d8f63bef8fe30117a9925c6721685797b3e6 Mon Sep 17 00:00:00 2001 From: Micah Date: Fri, 26 Jan 2024 11:47:26 -0800 Subject: [PATCH 141/366] Use convenience method for metadata when possible --- src/snapshot_middleware/csv.rs | 39 +++---------------------------- src/snapshot_middleware/lua.rs | 2 +- src/snapshot_middleware/txt.rs | 42 ++++------------------------------ src/syncback/snapshot.rs | 1 + 4 files changed, 10 insertions(+), 74 deletions(-) diff --git a/src/snapshot_middleware/csv.rs b/src/snapshot_middleware/csv.rs index 188dc5483..8afa4ade2 100644 --- a/src/snapshot_middleware/csv.rs +++ b/src/snapshot_middleware/csv.rs @@ -18,7 +18,7 @@ use crate::{ use super::{ dir::{dir_meta, snapshot_dir_no_meta, syncback_dir_no_meta}, - meta_file::{file_meta, AdjacentMetadata, DirectoryMetadata}, + meta_file::{AdjacentMetadata, DirectoryMetadata}, }; pub fn snapshot_csv( @@ -111,47 +111,14 @@ pub fn syncback_csv<'new, 'old>( anyhow::bail!("LocalizationTables must have a `Contents` property that is a String") }; - let mut meta = if let Some(meta) = file_meta(snapshot.vfs(), &path, &snapshot.name)? { - meta - } else { - AdjacentMetadata { - ignore_unknown_instances: None, - properties: BTreeMap::new(), - attributes: BTreeMap::new(), - path: path - .with_file_name(&snapshot.name) - .with_extension("meta.json"), - } - }; - for (name, value) in snapshot.new_filtered_properties() { - if name == "Contents" { - continue; - } else if name == "Attributes" || name == "AttributesSerialize" { - if let Variant::Attributes(attrs) = value { - meta.attributes.extend(attrs.iter().map(|(name, value)| { - ( - name.to_string(), - UnresolvedValue::FullyQualified(value.clone()), - ) - })) - } else { - log::error!("Property {name} should be Attributes but is not"); - } - } else { - meta.properties.insert( - name.to_string(), - UnresolvedValue::from_variant(value.to_owned(), &new_inst.class, name), - ); - } - } + let meta = AdjacentMetadata::from_syncback_snapshot(snapshot, path.clone())?; - // TODO tags don't work, why? let mut fs_snapshot = FsSnapshot::new(); fs_snapshot.add_file(path, localization_to_csv(contents)?); if !meta.is_empty() { fs_snapshot.add_file( &meta.path, - serde_json::to_vec_pretty(&meta).context("failed to reserialize metadata")?, + serde_json::to_vec_pretty(&meta).context("cannot serialize metadata")?, ) } diff --git a/src/snapshot_middleware/lua.rs b/src/snapshot_middleware/lua.rs index f8ef52753..664ba200c 100644 --- a/src/snapshot_middleware/lua.rs +++ b/src/snapshot_middleware/lua.rs @@ -16,7 +16,7 @@ use crate::{ use super::{ dir::{dir_meta, snapshot_dir_no_meta, syncback_dir_no_meta}, - meta_file::{file_meta, AdjacentMetadata, DirectoryMetadata}, + meta_file::{AdjacentMetadata, DirectoryMetadata}, }; #[derive(Debug)] diff --git a/src/snapshot_middleware/txt.rs b/src/snapshot_middleware/txt.rs index 2c9b91c93..f3d15fbb9 100644 --- a/src/snapshot_middleware/txt.rs +++ b/src/snapshot_middleware/txt.rs @@ -1,4 +1,4 @@ -use std::{collections::BTreeMap, path::Path, str}; +use std::{path::Path, str}; use anyhow::Context; use maplit::hashmap; @@ -6,12 +6,11 @@ use memofs::{IoResultExt, Vfs}; use rbx_dom_weak::types::Variant; use crate::{ - resolution::UnresolvedValue, snapshot::{InstanceContext, InstanceMetadata, InstanceSnapshot}, syncback::{FsSnapshot, SyncbackReturn, SyncbackSnapshot}, }; -use super::meta_file::{file_meta, AdjacentMetadata}; +use super::meta_file::AdjacentMetadata; pub fn snapshot_txt( context: &InstanceContext, @@ -62,45 +61,14 @@ pub fn syncback_txt<'new, 'old>( anyhow::bail!("StringValues must have a `Value` property that is a String"); }; - let mut meta = if let Some(meta) = file_meta(snapshot.vfs(), &path, &snapshot.name)? { - meta - } else { - AdjacentMetadata { - ignore_unknown_instances: None, - properties: BTreeMap::new(), - attributes: BTreeMap::new(), - path: path - .with_file_name(&snapshot.name) - .with_extension("meta.json"), - } - }; - for (name, value) in snapshot.new_filtered_properties() { - if name == "Value" { - continue; - } else if name == "Attributes" || name == "AttributesSerialize" { - if let Variant::Attributes(attrs) = value { - meta.attributes.extend(attrs.iter().map(|(name, value)| { - ( - name.to_string(), - UnresolvedValue::FullyQualified(value.clone()), - ) - })) - } else { - log::error!("Property {name} should be Attributes but is not"); - } - } else { - meta.properties.insert( - name.to_string(), - UnresolvedValue::from_variant(value.to_owned(), &new_inst.class, name), - ); - } - } + let meta = AdjacentMetadata::from_syncback_snapshot(snapshot, path.clone())?; + let mut fs_snapshot = FsSnapshot::new(); fs_snapshot.add_file(path, contents); if !meta.is_empty() { fs_snapshot.add_file( &meta.path, - serde_json::to_vec_pretty(&meta).context("could not serialize new init.meta.json")?, + serde_json::to_vec_pretty(&meta).context("could not serialize metadata")?, ); } diff --git a/src/syncback/snapshot.rs b/src/syncback/snapshot.rs index 236d223b9..e627e758e 100644 --- a/src/syncback/snapshot.rs +++ b/src/syncback/snapshot.rs @@ -290,6 +290,7 @@ fn filter_out_property(inst: &Instance, prop_name: &str) -> bool { match inst.class.as_str() { "Script" | "LocalScript" | "ModuleScript" => prop_name == "Source", "LocalizationTable" => prop_name == "Contents", + "StringValue" => prop_name == "Value", _ => false, } } From ec7d4ed6bbab6d66c607a5d8aabf7d31bf852c3d Mon Sep 17 00:00:00 2001 From: Micah Date: Fri, 26 Jan 2024 11:48:40 -0800 Subject: [PATCH 142/366] Make gitkeep file only in full dir syncback (not no_meta variant) --- src/snapshot_middleware/dir.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/snapshot_middleware/dir.rs b/src/snapshot_middleware/dir.rs index 7fb49267a..a3fa5f3ba 100644 --- a/src/snapshot_middleware/dir.rs +++ b/src/snapshot_middleware/dir.rs @@ -147,6 +147,12 @@ pub fn syncback_dir<'new, 'old>( serde_json::to_vec_pretty(&meta).context("could not serialize new init.meta.json")?, ); } + if new_inst.children().is_empty() { + dir_syncback + .fs_snapshot + .add_file(path.join(EMPTY_DIR_KEEP_NAME), Vec::new()) + } + Ok(dir_syncback) } @@ -212,9 +218,6 @@ pub fn syncback_dir_no_meta<'new, 'old>( } let mut fs_snapshot = FsSnapshot::new(); fs_snapshot.add_dir(&path); - if new_inst.children().is_empty() { - fs_snapshot.add_file(path.join(EMPTY_DIR_KEEP_NAME), Vec::new()) - } Ok(SyncbackReturn { inst_snapshot: InstanceSnapshot::from_instance(new_inst), From 9ba191a7f74f781738f652ade7ad18ad0791c590 Mon Sep 17 00:00:00 2001 From: Micah Date: Fri, 26 Jan 2024 11:58:45 -0800 Subject: [PATCH 143/366] Correct metadata location for text and csv --- src/snapshot_middleware/csv.rs | 4 +++- src/snapshot_middleware/txt.rs | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/snapshot_middleware/csv.rs b/src/snapshot_middleware/csv.rs index 8afa4ade2..b372e24a3 100644 --- a/src/snapshot_middleware/csv.rs +++ b/src/snapshot_middleware/csv.rs @@ -117,7 +117,9 @@ pub fn syncback_csv<'new, 'old>( fs_snapshot.add_file(path, localization_to_csv(contents)?); if !meta.is_empty() { fs_snapshot.add_file( - &meta.path, + snapshot + .parent_path + .join(format!("{}.meta.json", new_inst.name)), serde_json::to_vec_pretty(&meta).context("cannot serialize metadata")?, ) } diff --git a/src/snapshot_middleware/txt.rs b/src/snapshot_middleware/txt.rs index f3d15fbb9..d37782b8b 100644 --- a/src/snapshot_middleware/txt.rs +++ b/src/snapshot_middleware/txt.rs @@ -67,7 +67,9 @@ pub fn syncback_txt<'new, 'old>( fs_snapshot.add_file(path, contents); if !meta.is_empty() { fs_snapshot.add_file( - &meta.path, + snapshot + .parent_path + .join(format!("{}.meta.json", new_inst.name)), serde_json::to_vec_pretty(&meta).context("could not serialize metadata")?, ); } From 80d19ca8050eec009d8e2f05dac15d7c72ec597b Mon Sep 17 00:00:00 2001 From: Micah Date: Fri, 26 Jan 2024 13:10:54 -0800 Subject: [PATCH 144/366] Use DirectoryMeta for syncing dir metadata --- src/snapshot_middleware/csv.rs | 39 +----------------- src/snapshot_middleware/dir.rs | 44 +++------------------ src/snapshot_middleware/lua.rs | 45 ++------------------- src/snapshot_middleware/meta_file.rs | 59 ++++++++++++++++++++++++++++ 4 files changed, 70 insertions(+), 117 deletions(-) diff --git a/src/snapshot_middleware/csv.rs b/src/snapshot_middleware/csv.rs index b372e24a3..88b8620c7 100644 --- a/src/snapshot_middleware/csv.rs +++ b/src/snapshot_middleware/csv.rs @@ -11,7 +11,6 @@ use rbx_dom_weak::types::Variant; use serde::{Deserialize, Serialize}; use crate::{ - resolution::UnresolvedValue, snapshot::{InstanceContext, InstanceMetadata, InstanceSnapshot}, syncback::{FsSnapshot, SyncbackReturn, SyncbackSnapshot}, }; @@ -147,47 +146,13 @@ pub fn syncback_csv_init<'new, 'old>( }; let mut dir_syncback = syncback_dir_no_meta(snapshot, dir_name)?; - let mut meta = if let Some(dir) = dir_meta(snapshot.vfs(), &path)? { - dir - } else { - DirectoryMetadata { - ignore_unknown_instances: None, - class_name: None, - properties: BTreeMap::new(), - attributes: BTreeMap::new(), - path: snapshot - .parent_path - .join(&snapshot.name) - .join("init.meta.json"), - } - }; - for (name, value) in snapshot.new_filtered_properties() { - if name == "Contents" { - continue; - } else if name == "Attributes" || name == "AttributesSerialize" { - if let Variant::Attributes(attrs) = value { - meta.attributes.extend(attrs.iter().map(|(name, value)| { - ( - name.to_string(), - UnresolvedValue::FullyQualified(value.clone()), - ) - })) - } else { - log::error!("Property {name} should be Attributes but is not"); - } - } else { - meta.properties.insert( - name.to_string(), - UnresolvedValue::from_variant(value.to_owned(), &new_inst.class, name), - ); - } - } + let meta = DirectoryMetadata::from_syncback_snapshot(snapshot, path.clone())?; let mut fs_snapshot = std::mem::take(&mut dir_syncback.fs_snapshot); fs_snapshot.add_file(&path, localization_to_csv(contents)?); if !meta.is_empty() { fs_snapshot.add_file( - &meta.path, + snapshot.parent_path.join(dir_name).join("init.meta.json"), serde_json::to_vec_pretty(&meta).context("could not serialize new init.meta.json")?, ); } diff --git a/src/snapshot_middleware/dir.rs b/src/snapshot_middleware/dir.rs index a3fa5f3ba..8da1061ae 100644 --- a/src/snapshot_middleware/dir.rs +++ b/src/snapshot_middleware/dir.rs @@ -1,14 +1,12 @@ use std::{ - collections::{BTreeMap, HashMap, HashSet}, + collections::{HashMap, HashSet}, path::Path, }; use anyhow::Context; use memofs::{DirEntry, IoResultExt, Vfs}; -use rbx_dom_weak::types::Variant; use crate::{ - resolution::UnresolvedValue, snapshot::{InstanceContext, InstanceMetadata, InstanceSnapshot}, syncback::{FsSnapshot, SyncbackReturn, SyncbackSnapshot}, }; @@ -106,48 +104,18 @@ pub fn syncback_dir<'new, 'old>( let mut dir_syncback = syncback_dir_no_meta(snapshot, dir_name)?; - let mut meta = if let Some(dir) = dir_meta(snapshot.vfs(), &path)? { - dir - } else { - DirectoryMetadata { - ignore_unknown_instances: None, - properties: BTreeMap::new(), - attributes: BTreeMap::new(), - class_name: if new_inst.class == "Folder" { - None - } else { - Some(new_inst.class.clone()) - }, - path: path.join("init.meta.json"), - } - }; - for (name, value) in snapshot.new_filtered_properties() { - if name == "Attributes" || name == "AttributesSerialize" { - if let Variant::Attributes(attrs) = value { - meta.attributes.extend(attrs.iter().map(|(name, value)| { - ( - name.to_string(), - UnresolvedValue::FullyQualified(value.clone()), - ) - })) - } else { - log::error!("Property {name} should be Attributes but is not"); - } - } else { - meta.properties.insert( - name.to_string(), - UnresolvedValue::from_variant(value.to_owned(), &new_inst.class, name), - ); - } + let mut meta = DirectoryMetadata::from_syncback_snapshot(snapshot, path.clone())?; + if new_inst.class != "Folder" { + meta.class_name = Some(new_inst.class.clone()); } if !meta.is_empty() { dir_syncback.fs_snapshot.add_file( - &meta.path, + path.join("init.meta.json"), serde_json::to_vec_pretty(&meta).context("could not serialize new init.meta.json")?, ); } - if new_inst.children().is_empty() { + if new_inst.children().is_empty() && meta.is_empty() { dir_syncback .fs_snapshot .add_file(path.join(EMPTY_DIR_KEEP_NAME), Vec::new()) diff --git a/src/snapshot_middleware/lua.rs b/src/snapshot_middleware/lua.rs index 664ba200c..195443e16 100644 --- a/src/snapshot_middleware/lua.rs +++ b/src/snapshot_middleware/lua.rs @@ -1,15 +1,10 @@ -use std::{ - collections::{BTreeMap, HashMap}, - path::Path, - str, -}; +use std::{collections::HashMap, path::Path, str}; use anyhow::Context; use memofs::{IoResultExt, Vfs}; use rbx_dom_weak::types::{Enum, Variant}; use crate::{ - resolution::UnresolvedValue, snapshot::{InstanceContext, InstanceMetadata, InstanceSnapshot}, syncback::{FsSnapshot, SyncbackReturn, SyncbackSnapshot}, }; @@ -183,41 +178,7 @@ pub fn syncback_lua_init<'new, 'old>( let dir_syncback = syncback_dir_no_meta(snapshot, dir_name)?; - let mut meta = if let Some(dir) = dir_meta(snapshot.vfs(), &path)? { - dir - } else { - DirectoryMetadata { - ignore_unknown_instances: None, - class_name: None, - properties: BTreeMap::new(), - attributes: BTreeMap::new(), - path: snapshot - .parent_path - .join(&snapshot.name) - .join("init.meta.json"), - } - }; - for (name, value) in snapshot.new_filtered_properties() { - if name == "Source" { - continue; - } else if name == "Attributes" || name == "AttributesSerialize" { - if let Variant::Attributes(attrs) = value { - meta.attributes.extend(attrs.iter().map(|(name, value)| { - ( - name.to_string(), - UnresolvedValue::FullyQualified(value.clone()), - ) - })) - } else { - log::error!("Property {name} should be Attributes but is not"); - } - } else { - meta.properties.insert( - name.to_string(), - UnresolvedValue::from_variant(value.to_owned(), &new_inst.class, name), - ); - } - } + let meta = DirectoryMetadata::from_syncback_snapshot(snapshot, path.clone())?; let mut fs_snapshot = FsSnapshot::new(); fs_snapshot.add_file(path, contents); @@ -225,7 +186,7 @@ pub fn syncback_lua_init<'new, 'old>( if !meta.is_empty() { fs_snapshot.add_file( - &meta.path, + snapshot.parent_path.join(dir_name).join("init.meta.json"), serde_json::to_vec_pretty(&meta).context("could not serialize new init.meta.json")?, ); } diff --git a/src/snapshot_middleware/meta_file.rs b/src/snapshot_middleware/meta_file.rs index a3b8fdaa9..0f2519398 100644 --- a/src/snapshot_middleware/meta_file.rs +++ b/src/snapshot_middleware/meta_file.rs @@ -193,6 +193,65 @@ impl DirectoryMetadata { Ok(meta) } + /// Constructs a `DirectoryMetadata` from the provided snapshot, assuming it + /// will be at the provided path. + /// + /// This function does not set `ClassName` manually as most uses won't + /// want it set. + pub fn from_syncback_snapshot( + snapshot: &SyncbackSnapshot, + path: PathBuf, + ) -> anyhow::Result { + let mut properties = BTreeMap::new(); + let mut attributes = BTreeMap::new(); + + let ignore_unknown_instances = if let Some(old_inst) = snapshot.old_inst() { + if old_inst.metadata().ignore_unknown_instances { + Some(true) + } else { + None + } + } else { + None + }; + + let class = &snapshot.new_inst().class; + for (name, value) in snapshot.new_filtered_properties() { + properties.insert( + name.to_owned(), + UnresolvedValue::from_variant(value.clone(), class, name), + ); + } + if let Some(value) = snapshot.new_inst().properties.get("Attributes") { + if let Variant::Attributes(attrs) = value { + for (name, value) in attrs.iter() { + let value = if let Variant::BinaryString(bstr) = value { + match std::str::from_utf8(bstr.as_ref()) { + Ok(str) => Variant::String(str.to_owned()), + Err(_) => value.clone(), + } + } else { + value.clone() + }; + attributes.insert(name.to_owned(), UnresolvedValue::FullyQualified(value)); + } + } else { + anyhow::bail!( + "expected Attributes to be of type Attributes but it was {:?}", + value.ty() + ); + } + } + + Ok(Self { + ignore_unknown_instances, + properties, + attributes, + class_name: None, + path, + }) + } + pub fn apply_all(&mut self, snapshot: &mut InstanceSnapshot) -> anyhow::Result<()> { self.apply_ignore_unknown_instances(snapshot); self.apply_class_name(snapshot)?; From 670e5413fad6a4f99318e4a29c02ccffcc760f8f Mon Sep 17 00:00:00 2001 From: Micah Date: Fri, 26 Jan 2024 13:34:37 -0800 Subject: [PATCH 145/366] Filter old_inst properties too --- src/syncback/snapshot.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/syncback/snapshot.rs b/src/syncback/snapshot.rs index e627e758e..21d9bd366 100644 --- a/src/syncback/snapshot.rs +++ b/src/syncback/snapshot.rs @@ -83,6 +83,9 @@ impl<'new, 'old> SyncbackSnapshot<'new, 'old> { if let Some(old_inst) = old_ref.and_then(|referent| self.get_old_instance(referent)) { for (name, value) in &inst.properties { + if filter_out_property(inst, name.as_str()) { + continue; + } if old_inst.properties().contains_key(name) { properties.insert(name, value); } From 7b97520d6cc0c88415028768e3d8d794a1a4a105 Mon Sep 17 00:00:00 2001 From: Micah Date: Wed, 31 Jan 2024 08:25:52 -0800 Subject: [PATCH 146/366] Revert "Add function for doing both hash types at once, use it" This reverts commit 186a386ade38c4818e29c7b37a174c92e8d5aff8. --- src/snapshot/hash/mod.rs | 81 +++++++++++------------------- src/snapshot_middleware/project.rs | 6 +-- src/syncback/mod.rs | 29 ++++++----- src/syncback/snapshot.rs | 39 ++------------ 4 files changed, 50 insertions(+), 105 deletions(-) diff --git a/src/snapshot/hash/mod.rs b/src/snapshot/hash/mod.rs index d9c87f67b..82f17a295 100644 --- a/src/snapshot/hash/mod.rs +++ b/src/snapshot/hash/mod.rs @@ -12,42 +12,6 @@ use std::collections::{HashMap, VecDeque}; use crate::variant_eq::variant_eq; -/// Returns a map of hashes for every Instance contained in the DOM. -/// Hashes are mapped to the Instance's referent. -/// -/// The first hash in each tuple is the Instance's hash with no descendants, -/// the second is one using descendants. -pub fn hash_tree_both(dom: &WeakDom) -> HashMap { - let mut map: HashMap = HashMap::new(); - let mut order = descendants(dom); - - let mut prop_list = Vec::with_capacity(2); - - while let Some(referent) = order.pop() { - let inst = dom.get_by_ref(referent).unwrap(); - let mut hasher = hash_inst_no_descendants(inst, &mut prop_list); - let no_descendants = hasher.finalize(); - - let mut child_list = Vec::with_capacity(inst.children().len()); - - for child in inst.children() { - if let Some((_, descendant)) = map.get(child) { - child_list.push(descendant.as_bytes()) - } else { - panic!("Invariant: child {} not hashed before its parent", child); - } - } - child_list.sort_unstable(); - for hash in child_list { - hasher.update(hash); - } - - map.insert(referent, (no_descendants, hasher.finalize())); - } - - map -} - /// Returns a map of every `Ref` in the `WeakDom` to a hashed version of the /// `Instance` it points to, including the properties but not including the /// descendants of the Instance. @@ -86,22 +50,9 @@ pub fn hash_tree(dom: &WeakDom) -> HashMap { while let Some(referent) = order.pop() { let inst = dom.get_by_ref(referent).unwrap(); - let mut hasher = hash_inst_no_descendants(inst, &mut prop_list); + let hash = hash_inst(inst, &mut prop_list, &map); - let mut child_list = Vec::with_capacity(inst.children().len()); - for child in inst.children() { - if let Some(hash) = map.get(child) { - child_list.push(hash.as_bytes()) - } else { - panic!("Invariant: child {} not hashed before its parent", child); - } - } - child_list.sort_unstable(); - for hash in child_list { - hasher.update(hash); - } - - map.insert(referent, hasher.finalize()); + map.insert(referent, hash); } map @@ -143,6 +94,34 @@ fn hash_inst_no_descendants<'inst>( hasher } +/// Hashes an Instance using its class, name, properties, and descendants. +/// The passed `prop_list` is used to sort properties before hashing them. +/// +/// # Panics +/// If any children of the Instance are inside `map`, this function will panic. +fn hash_inst<'inst>( + inst: &'inst Instance, + prop_list: &mut Vec<(&'inst str, &'inst Variant)>, + map: &HashMap, +) -> Hash { + let mut hasher = hash_inst_no_descendants(inst, prop_list); + let mut child_list = Vec::with_capacity(inst.children().len()); + + for child in inst.children() { + if let Some(hash) = map.get(child) { + child_list.push(hash.as_bytes()) + } else { + panic!("Invariant: child {} not hashed before its parent", child); + } + } + child_list.sort_unstable(); + for hash in child_list { + hasher.update(hash); + } + + hasher.finalize() +} + pub(crate) fn descendants(dom: &WeakDom) -> Vec { let mut queue = VecDeque::new(); let mut ordered = Vec::new(); diff --git a/src/snapshot_middleware/project.rs b/src/snapshot_middleware/project.rs index 59091308e..d9144b65b 100644 --- a/src/snapshot_middleware/project.rs +++ b/src/snapshot_middleware/project.rs @@ -1,4 +1,4 @@ -use std::{borrow::Cow, collections::HashMap, path::Path, rc::Rc}; +use std::{borrow::Cow, collections::HashMap, path::Path}; use anyhow::{bail, Context}; use memofs::Vfs; @@ -408,7 +408,7 @@ pub fn syncback_project<'new, 'old>( if let Some(old_inst) = old_child_map.get(new_name.as_str()) { // This new instance represents an older one! children.push(SyncbackSnapshot { - data: Rc::clone(&snapshot.data), + data: snapshot.data, old: Some(old_inst.id()), new: new_child.referent(), parent_path, @@ -418,7 +418,7 @@ pub fn syncback_project<'new, 'old>( } else { // This new instance is... new. children.push(SyncbackSnapshot { - data: Rc::clone(&snapshot.data), + data: snapshot.data, old: None, new: new_child.referent(), parent_path, diff --git a/src/syncback/mod.rs b/src/syncback/mod.rs index c3a15d908..ea4484908 100644 --- a/src/syncback/mod.rs +++ b/src/syncback/mod.rs @@ -6,15 +6,11 @@ use anyhow::Context; use memofs::Vfs; use rbx_dom_weak::{types::Ref, Instance, WeakDom}; use serde::{Deserialize, Serialize}; -use std::{ - collections::{HashMap, HashSet, VecDeque}, - rc::Rc, - sync::OnceLock, -}; +use std::collections::{HashMap, VecDeque}; use crate::{ glob::Glob, - snapshot::{hash_tree_both, InstanceSnapshot, InstanceWithMeta, RojoTree}, + snapshot::{hash_tree, InstanceSnapshot, InstanceWithMeta, RojoTree}, snapshot_middleware::Middleware, Project, }; @@ -30,20 +26,18 @@ pub fn syncback_loop<'old>( project: &'old Project, ) -> anyhow::Result> { log::debug!("Hashing project DOM"); - let old_hashes = hash_tree_both(old_tree.inner()); + let old_hashes = hash_tree(old_tree.inner()); log::debug!("Hashing file DOM"); - let new_hashes = hash_tree_both(new_tree); + let new_hashes = hash_tree(new_tree); let project_path = project.folder_location(); - let syncback_data = Rc::new(SyncbackData { + let syncback_data = SyncbackData { vfs, old_tree, new_tree, syncback_rules: project.syncback_rules.as_ref(), - old_hashes, - new_hashes, - }); + }; let mut snapshots = vec![SyncbackSnapshot { data: syncback_data, @@ -60,9 +54,14 @@ pub fn syncback_loop<'old>( let inst_path = get_inst_path(new_tree, snapshot.new); // We can quickly check that two subtrees are identical and if they are, // skip reconciling them. - if snapshot.trees_are_deep_eq() { - log::trace!("Skipping {inst_path} due to it being identically hashed"); - continue; + if let Some(old_ref) = snapshot.old { + if old_hashes.get(&old_ref) == new_hashes.get(&snapshot.new) { + log::trace!( + "Skipping {inst_path} due to it being identically hashed as {:?}", + old_hashes.get(&old_ref) + ); + continue; + } } let middleware = snapshot diff --git a/src/syncback/snapshot.rs b/src/syncback/snapshot.rs index 21d9bd366..08e6ec669 100644 --- a/src/syncback/snapshot.rs +++ b/src/syncback/snapshot.rs @@ -1,9 +1,7 @@ -use blake3::Hash; use memofs::Vfs; use std::{ collections::{BTreeSet, HashMap}, path::{Path, PathBuf}, - rc::Rc, sync::OnceLock, }; @@ -22,17 +20,16 @@ use super::SyncbackRules; /// A glob that can be used to tell if a path contains a `.git` folder. static GIT_IGNORE_GLOB: OnceLock = OnceLock::new(); +#[derive(Clone, Copy)] pub struct SyncbackData<'new, 'old> { pub(super) vfs: &'old Vfs, pub(super) old_tree: &'old RojoTree, pub(super) new_tree: &'new WeakDom, pub(super) syncback_rules: Option<&'old SyncbackRules>, - pub(super) new_hashes: HashMap, - pub(super) old_hashes: HashMap, } pub struct SyncbackSnapshot<'new, 'old> { - pub data: Rc>, + pub data: SyncbackData<'new, 'old>, pub old: Option, pub new: Ref, pub parent_path: PathBuf, @@ -45,7 +42,7 @@ impl<'new, 'old> SyncbackSnapshot<'new, 'old> { #[inline] pub fn with_parent(&self, new_name: String, new_ref: Ref, old_ref: Option) -> Self { Self { - data: Rc::clone(&self.data), + data: self.data, old: old_ref, new: new_ref, parent_path: self.parent_path.join(&self.name), @@ -188,36 +185,6 @@ impl<'new, 'old> SyncbackSnapshot<'new, 'old> { true } - /// Returns whether two Instances are equal, using their hash for O(1) - /// comparisons. This comparison **does not** take descendants into account. - #[inline] - pub fn trees_are_eq(&self) -> bool { - let old_hashes = self - .old - .and_then(|referent| self.data.old_hashes.get(&referent)); - let new_hashes = self.data.new_hashes.get(&self.new); - - match (old_hashes, new_hashes) { - (Some((old_hash, _)), Some((new_hash, _))) => old_hash == new_hash, - _ => false, - } - } - - /// Returns whether two Instance trees are equal, using their hash for O(1) - /// comparisons. This comparison **does** take descendants into account. - #[inline] - pub fn trees_are_deep_eq(&self) -> bool { - let old_hashes = self - .old - .and_then(|referent| self.data.old_hashes.get(&referent)); - let new_hashes = self.data.new_hashes.get(&self.new); - - match (old_hashes, new_hashes) { - (Some((_, old_hash)), Some((_, new_hash))) => old_hash == new_hash, - _ => false, - } - } - /// Returns an Instance from the old tree with the provided referent, if it /// exists. #[inline] From 8478a9af1827c2fd8085735018adeb9a2d3f9a8c Mon Sep 17 00:00:00 2001 From: Micah Date: Wed, 31 Jan 2024 08:28:08 -0800 Subject: [PATCH 147/366] Fix issue caused by bad revert --- src/syncback/mod.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/syncback/mod.rs b/src/syncback/mod.rs index ea4484908..1ac992916 100644 --- a/src/syncback/mod.rs +++ b/src/syncback/mod.rs @@ -6,7 +6,10 @@ use anyhow::Context; use memofs::Vfs; use rbx_dom_weak::{types::Ref, Instance, WeakDom}; use serde::{Deserialize, Serialize}; -use std::collections::{HashMap, VecDeque}; +use std::{ + collections::{HashMap, HashSet, VecDeque}, + sync::OnceLock, +}; use crate::{ glob::Glob, From b3d2e3157bd6d3e36acb76220a0bcbf57492b1af Mon Sep 17 00:00:00 2001 From: Micah Date: Wed, 31 Jan 2024 08:56:58 -0800 Subject: [PATCH 148/366] Refactor json model middleware to handle children --- src/snapshot_middleware/json_model.rs | 91 +++++++++++++++++---------- 1 file changed, 58 insertions(+), 33 deletions(-) diff --git a/src/snapshot_middleware/json_model.rs b/src/snapshot_middleware/json_model.rs index 4fca94fee..b17a4c3aa 100644 --- a/src/snapshot_middleware/json_model.rs +++ b/src/snapshot_middleware/json_model.rs @@ -66,26 +66,29 @@ pub fn syncback_json_model<'new, 'old>( ) -> anyhow::Result> { let path = snapshot.parent_path.join(file_name); - let new_inst = snapshot.new_inst(); + let model = json_model_from_pair(snapshot, snapshot.new, snapshot.old); + + Ok(SyncbackReturn { + inst_snapshot: InstanceSnapshot::from_instance(snapshot.new_inst()), + fs_snapshot: FsSnapshot::new().with_added_file( + path, + serde_json::to_vec_pretty(&model).context("failed to serialize new JSON Model")?, + ), + children: Vec::new(), + removed_children: Vec::new(), + }) +} + +fn json_model_from_pair(snapshot: &SyncbackSnapshot, new: Ref, old: Option) -> JsonModel { + let new_inst = snapshot + .get_new_instance(new) + .expect("all new referents passed to json_model_from_pair should exist"); + let old_inst = old.and_then(|r| snapshot.get_old_instance(r)); let mut properties = BTreeMap::new(); let mut attributes = BTreeMap::new(); - for (name, value) in snapshot.new_filtered_properties() { - if name == "Attributes" || name == "AttributesSerialize" { - if let Variant::Attributes(attr) = value { - attributes.extend(attr.iter().map(|(name, value)| { - ( - name.to_string(), - UnresolvedValue::from_variant(value.clone(), &new_inst.class, name), - ) - })) - } else { - log::error!( - "Property {name} should be Attributes but is {:?}", - value.ty() - ); - } - } else if let Variant::Ref(_) = value { + for (name, value) in snapshot.get_filtered_properties(new, old).unwrap() { + if let Variant::Ref(_) = value { // We do not currently support Ref properties continue; } else { @@ -95,27 +98,49 @@ pub fn syncback_json_model<'new, 'old>( ); } } + if let Some(Variant::Attributes(attrs)) = new_inst.properties.get("Attributes") { + for (attr_name, attr_value) in attrs.iter() { + attributes.insert( + attr_name.clone(), + UnresolvedValue::FullyQualified(attr_value.clone()), + ); + } + } + + let mut children = Vec::with_capacity(new_inst.children().len()); + + if let Some(old_inst) = old_inst { + let mut old_child_map = HashMap::with_capacity(old_inst.children().len()); + for child_ref in old_inst.children() { + let inst = snapshot.get_old_instance(*child_ref).unwrap(); + old_child_map.insert(inst.name(), *child_ref); + } - let model = JsonModel { + for new_child_ref in new_inst.children() { + let new_child = snapshot.get_new_instance(*new_child_ref).unwrap(); + if let Some(old_child) = old_child_map.remove(new_child.name.as_str()) { + children.push(json_model_from_pair( + snapshot, + *new_child_ref, + Some(old_child), + )) + } else { + children.push(json_model_from_pair(snapshot, new, None)) + } + } + } else { + for new_child_ref in new_inst.children() { + children.push(json_model_from_pair(snapshot, *new_child_ref, None)) + } + } + + JsonModel { name: Some(new_inst.name.clone()), class_name: new_inst.class.clone(), - children: Vec::new(), + children, properties, attributes, - }; - - // TODO Do we want children to be included in a JSON model? - // Feels unlikely, since it could be very dense in the case of e.g. Tools - - Ok(SyncbackReturn { - inst_snapshot: InstanceSnapshot::from_instance(new_inst), - fs_snapshot: FsSnapshot::new().with_added_file( - path, - serde_json::to_vec_pretty(&model).context("failed to serialize new JSON Model")?, - ), - children: Vec::new(), - removed_children: Vec::new(), - }) + } } #[derive(Debug, Deserialize, Serialize)] From 3799e949535b15769e00d73158bdc22878981afa Mon Sep 17 00:00:00 2001 From: Micah Date: Mon, 5 Feb 2024 11:23:46 -0800 Subject: [PATCH 149/366] Merge branch 'syncback-refs' into syncback-incremental --- CHANGELOG.md | 22 +++ plugin/src/App/StatusPages/Settings/init.lua | 35 +++-- plugin/src/App/init.lua | 76 +++++++++- plugin/src/Settings.lua | 1 + ...d__tests__serve__ref_properties_all-2.snap | 141 ++++++++++++++++++ ...end__tests__serve__ref_properties_all.snap | 121 +++++++++++++++ ...nd__tests__serve__ref_properties_info.snap | 13 ++ ...s__serve__ref_properties_remove_all-2.snap | 33 ++++ ...sts__serve__ref_properties_remove_all.snap | 47 ++++++ ...ts__serve__ref_properties_remove_info.snap | 13 ++ ...erve__ref_properties_remove_subscribe.snap | 12 ++ ...ests__serve__ref_properties_subscribe.snap | 46 ++++++ .../FolderTarget/FolderPointer.model.json | 6 + .../FolderTarget/init.meta.json | 3 + .../ref_properties/ModelTarget.model.json | 13 ++ .../ref_properties/default.project.json | 32 ++++ .../default.project.json | 6 + .../src/pointer.model.json | 6 + .../src/target.model.json | 6 + src/cli/syncback.rs | 2 +- src/lib.rs | 2 + src/multimap.rs | 32 +++- src/project.rs | 5 + src/resolution.rs | 6 + src/rojo_ref.rs | 40 +++++ src/snapshot/metadata.rs | 12 ++ src/snapshot/mod.rs | 2 - src/snapshot/patch_apply.rs | 79 ++++++++++ ..._snapshot__tests__apply__add_property.snap | 1 + ...s__apply__remove_property_after_patch.snap | 1 + ...tests__apply__remove_property_initial.snap | 1 + ...tests__apply__set_name_and_class_name.snap | 1 + ...__snapshot__tests__compute__add_child.snap | 1 + src/snapshot/tree.rs | 39 ++++- src/snapshot_middleware/json_model.rs | 10 +- src/snapshot_middleware/meta_file.rs | 36 ++++- src/snapshot_middleware/project.rs | 5 + ...t_middleware__csv__test__csv_from_vfs.snap | 1 + ..._middleware__csv__test__csv_with_meta.snap | 1 + ...t_middleware__dir__test__empty_folder.snap | 1 + ...ddleware__dir__test__folder_in_folder.snap | 2 + ...leware__json__test__instance_from_vfs.snap | 1 + ...are__json_model__test__model_from_vfs.snap | 2 + ...on_model__test__model_from_vfs_legacy.snap | 2 + ...are__lua__test__class_client_from_vfs.snap | 1 + ...are__lua__test__class_module_from_vfs.snap | 1 + ...re__lua__test__class_module_with_meta.snap | 1 + ...are__lua__test__class_script_disabled.snap | 1 + ...re__lua__test__class_script_with_meta.snap | 1 + ...are__lua__test__class_server_from_vfs.snap | 1 + ...lua__test__runcontext_client_from_vfs.snap | 1 + ...lua__test__runcontext_module_from_vfs.snap | 1 + ...ua__test__runcontext_module_with_meta.snap | 1 + ...lua__test__runcontext_script_disabled.snap | 1 + ...ua__test__runcontext_script_with_meta.snap | 1 + ...lua__test__runcontext_server_from_vfs.snap | 1 + ...oject__test__project_from_direct_file.snap | 1 + ...test__project_path_property_overrides.snap | 1 + ..._project__test__project_with_children.snap | 2 + ...t__test__project_with_path_to_project.snap | 1 + ...ct_with_path_to_project_with_children.snap | 2 + ...oject__test__project_with_path_to_txt.snap | 1 + ...est__project_with_resolved_properties.snap | 1 + ...t__project_with_unresolved_properties.snap | 1 + ...leware__toml__test__instance_from_vfs.snap | 1 + ...dleware__txt__test__instance_from_vfs.snap | 1 + src/{snapshot => syncback}/hash/mod.rs | 21 +-- src/{snapshot => syncback}/hash/variant.rs | 0 src/syncback/mod.rs | 37 ++++- src/syncback/ref_properties.rs | 88 +++++++++++ src/syncback/snapshot.rs | 4 +- src/web/ui.rs | 3 +- tests/tests/serve.rs | 85 +++++++++++ 73 files changed, 1131 insertions(+), 48 deletions(-) create mode 100644 rojo-test/serve-test-snapshots/end_to_end__tests__serve__ref_properties_all-2.snap create mode 100644 rojo-test/serve-test-snapshots/end_to_end__tests__serve__ref_properties_all.snap create mode 100644 rojo-test/serve-test-snapshots/end_to_end__tests__serve__ref_properties_info.snap create mode 100644 rojo-test/serve-test-snapshots/end_to_end__tests__serve__ref_properties_remove_all-2.snap create mode 100644 rojo-test/serve-test-snapshots/end_to_end__tests__serve__ref_properties_remove_all.snap create mode 100644 rojo-test/serve-test-snapshots/end_to_end__tests__serve__ref_properties_remove_info.snap create mode 100644 rojo-test/serve-test-snapshots/end_to_end__tests__serve__ref_properties_remove_subscribe.snap create mode 100644 rojo-test/serve-test-snapshots/end_to_end__tests__serve__ref_properties_subscribe.snap create mode 100644 rojo-test/serve-tests/ref_properties/FolderTarget/FolderPointer.model.json create mode 100644 rojo-test/serve-tests/ref_properties/FolderTarget/init.meta.json create mode 100644 rojo-test/serve-tests/ref_properties/ModelTarget.model.json create mode 100644 rojo-test/serve-tests/ref_properties/default.project.json create mode 100644 rojo-test/serve-tests/ref_properties_remove/default.project.json create mode 100644 rojo-test/serve-tests/ref_properties_remove/src/pointer.model.json create mode 100644 rojo-test/serve-tests/ref_properties_remove/src/target.model.json create mode 100644 src/rojo_ref.rs rename src/{snapshot => syncback}/hash/mod.rs (87%) rename src/{snapshot => syncback}/hash/variant.rs (100%) create mode 100644 src/syncback/ref_properties.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d6a254c4..9785f7963 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,8 +22,28 @@ } ``` +* Projects may now manually link `Ref` properties together using `Attributes`. ([#843]) + This has two parts: using `id` or `$id` in JSON files or a `Rojo_Target` attribute, an Instance + is given an ID. Then, that ID may be used elsewhere in the project to point to an Instance + using an attribute named `Rojo_Target_PROP_NAME`, where `PROP_NAME` is the name of a property. + + As an example, here is a `model.json` for an ObjectValue that refers to itself: + + ```json + { + "id": "arbitrary string", + "attributes": { + "Rojo_Target_Value": "arbitrary string" + } + } + ``` + + This is a very rough implementation and the usage will become more ergonomic + over time. + * Added popout diff visualizer for table properties like Attributes and Tags ([#834]) * Updated Theme to use Studio colors ([#838]) +* Added experimental setting for Auto Connect in playtests ([#840]) * Projects may now specify rules for syncing files as if they had a different file extension. ([#813]) This is specified via a new field on project files, `syncRules`: @@ -74,6 +94,8 @@ [#813]: https://github.com/rojo-rbx/rojo/pull/813 [#834]: https://github.com/rojo-rbx/rojo/pull/834 [#838]: https://github.com/rojo-rbx/rojo/pull/838 +[#840]: https://github.com/rojo-rbx/rojo/pull/840 +[#843]: https://github.com/rojo-rbx/rojo/pull/843 ## [7.4.0] - January 16, 2024 * Improved the visualization for array properties like Tags ([#829]) diff --git a/plugin/src/App/StatusPages/Settings/init.lua b/plugin/src/App/StatusPages/Settings/init.lua index 688be85a3..a33eb1b54 100644 --- a/plugin/src/App/StatusPages/Settings/init.lua +++ b/plugin/src/App/StatusPages/Settings/init.lua @@ -75,6 +75,12 @@ function SettingsPage:init() end function SettingsPage:render() + local layoutOrder = 0 + local function layoutIncrement() + layoutOrder += 1 + return layoutOrder + end + return Theme.with(function(theme) theme = theme.Settings @@ -86,7 +92,7 @@ function SettingsPage:render() Navbar = e(Navbar, { onBack = self.props.onBack, transparency = self.props.transparency, - layoutOrder = 0, + layoutOrder = layoutIncrement(), }), ShowNotifications = e(Setting, { @@ -94,7 +100,7 @@ function SettingsPage:render() name = "Show Notifications", description = "Popup notifications in viewport", transparency = self.props.transparency, - layoutOrder = 1, + layoutOrder = layoutIncrement(), }), SyncReminder = e(Setting, { @@ -103,7 +109,7 @@ function SettingsPage:render() description = "Notify to sync when opening a place that has previously been synced", transparency = self.props.transparency, visible = Settings:getBinding("showNotifications"), - layoutOrder = 2, + layoutOrder = layoutIncrement(), }), ConfirmationBehavior = e(Setting, { @@ -111,7 +117,7 @@ function SettingsPage:render() name = "Confirmation Behavior", description = "When to prompt for confirmation before syncing", transparency = self.props.transparency, - layoutOrder = 3, + layoutOrder = layoutIncrement(), options = confirmationBehaviors, }), @@ -121,7 +127,7 @@ function SettingsPage:render() name = "Confirmation Threshold", description = "How many modified instances to be considered a large change", transparency = self.props.transparency, - layoutOrder = 4, + layoutOrder = layoutIncrement(), visible = Settings:getBinding("confirmationBehavior"):map(function(value) return value == "Large Changes" end), @@ -152,7 +158,16 @@ function SettingsPage:render() name = "Play Sounds", description = "Toggle sound effects", transparency = self.props.transparency, - layoutOrder = 5, + layoutOrder = layoutIncrement(), + }), + + AutoConnectPlaytestServer = e(Setting, { + id = "autoConnectPlaytestServer", + name = "Auto Connect Playtest Server", + description = "Automatically connect game server to Rojo when playtesting while connected in Edit", + experimental = true, + transparency = self.props.transparency, + layoutOrder = layoutIncrement(), }), OpenScriptsExternally = e(Setting, { @@ -162,7 +177,7 @@ function SettingsPage:render() locked = self.props.syncActive, experimental = true, transparency = self.props.transparency, - layoutOrder = 6, + layoutOrder = layoutIncrement(), }), TwoWaySync = e(Setting, { @@ -172,7 +187,7 @@ function SettingsPage:render() locked = self.props.syncActive, experimental = true, transparency = self.props.transparency, - layoutOrder = 7, + layoutOrder = layoutIncrement(), }), LogLevel = e(Setting, { @@ -180,7 +195,7 @@ function SettingsPage:render() name = "Log Level", description = "Plugin output verbosity level", transparency = self.props.transparency, - layoutOrder = 100, + layoutOrder = layoutIncrement(), options = invertedLevels, showReset = Settings:getBinding("logLevel"):map(function(value) @@ -196,7 +211,7 @@ function SettingsPage:render() name = "Typechecking", description = "Toggle typechecking on the API surface", transparency = self.props.transparency, - layoutOrder = 101, + layoutOrder = layoutIncrement(), }), Layout = e("UIListLayout", { diff --git a/plugin/src/App/init.lua b/plugin/src/App/init.lua index 22f277f9f..ecb807f05 100644 --- a/plugin/src/App/init.lua +++ b/plugin/src/App/init.lua @@ -158,11 +158,29 @@ function App:init() }, }) end + + if self:isAutoConnectPlaytestServerAvailable() then + self:useRunningConnectionInfo() + self:startSession() + end + self.autoConnectPlaytestServerListener = Settings:onChanged("autoConnectPlaytestServer", function(enabled) + if enabled then + if self:isAutoConnectPlaytestServerWriteable() and self.serveSession ~= nil then + -- Write the existing session + local baseUrl = self.serveSession.__apiContext.__baseUrl + self:setRunningConnectionInfo(baseUrl) + end + else + self:clearRunningConnectionInfo() + end + end) end function App:willUnmount() self.waypointConnection:Disconnect() self.confirmationBindable:Destroy() + self.autoConnectPlaytestServerListener() + self:clearRunningConnectionInfo() end function App:addNotification( @@ -278,10 +296,7 @@ function App:getHostAndPort() local host = self.host:getValue() local port = self.port:getValue() - local host = if #host > 0 then host else Config.defaultHost - local port = if #port > 0 then port else Config.defaultPort - - return host, port + return if #host > 0 then host else Config.defaultHost, if #port > 0 then port else Config.defaultPort end function App:isSyncLockAvailable() @@ -349,6 +364,49 @@ function App:releaseSyncLock() Log.trace("Could not relase sync lock because it is owned by {}", lock.Value) end +function App:isAutoConnectPlaytestServerAvailable() + return RunService:IsRunMode() + and RunService:IsServer() + and Settings:get("autoConnectPlaytestServer") + and workspace:GetAttribute("__Rojo_ConnectionUrl") +end + +function App:isAutoConnectPlaytestServerWriteable() + return RunService:IsEdit() and Settings:get("autoConnectPlaytestServer") +end + +function App:setRunningConnectionInfo(baseUrl: string) + if not self:isAutoConnectPlaytestServerWriteable() then + return + end + + Log.trace("Setting connection info for play solo auto-connect") + workspace:SetAttribute("__Rojo_ConnectionUrl", baseUrl) +end + +function App:clearRunningConnectionInfo() + if not RunService:IsEdit() then + -- Only write connection info from edit mode + return + end + + Log.trace("Clearing connection info for play solo auto-connect") + workspace:SetAttribute("__Rojo_ConnectionUrl", nil) +end + +function App:useRunningConnectionInfo() + local connectionInfo = workspace:GetAttribute("__Rojo_ConnectionUrl") + if not connectionInfo then + return + end + + Log.trace("Using connection info for play solo auto-connect") + local host, port = string.match(connectionInfo, "^(.+):(.-)$") + + self.setHost(host) + self.setPort(port) +end + function App:startSession() local claimedLock, priorOwner = self:claimSyncLock() if not claimedLock then @@ -441,6 +499,7 @@ function App:startSession() self:addNotification("Connecting to session...") elseif status == ServeSession.Status.Connected then self.knownProjects[details] = true + self:setRunningConnectionInfo(baseUrl) local address = ("%s:%s"):format(host, port) self:setState({ @@ -453,6 +512,7 @@ function App:startSession() elseif status == ServeSession.Status.Disconnected then self.serveSession = nil self:releaseSyncLock() + self:clearRunningConnectionInfo() self:setState({ patchData = { patch = PatchSet.newEmpty(), @@ -488,6 +548,12 @@ function App:startSession() return "Accept" end + -- Play solo auto-connect does not require confirmation + if self:isAutoConnectPlaytestServerAvailable() then + Log.trace("Accepting patch without confirmation because play solo auto-connect is enabled") + return "Accept" + end + local confirmationBehavior = Settings:get("confirmationBehavior") if confirmationBehavior == "Initial" then -- Only confirm if we haven't synced this project yet this session @@ -603,7 +669,7 @@ function App:render() value = self.props.plugin, }, { e(Theme.StudioProvider, nil, { - e(Tooltip.Provider, nil, { + tooltip = e(Tooltip.Provider, nil, { gui = e(StudioPluginGui, { id = pluginName, title = pluginName, diff --git a/plugin/src/Settings.lua b/plugin/src/Settings.lua index 194dd4c7a..b5be22a76 100644 --- a/plugin/src/Settings.lua +++ b/plugin/src/Settings.lua @@ -14,6 +14,7 @@ local defaultSettings = { twoWaySync = false, showNotifications = true, syncReminder = true, + autoConnectPlaytestServer = false, confirmationBehavior = "Initial", largeChangesConfirmationThreshold = 5, playSounds = true, diff --git a/rojo-test/serve-test-snapshots/end_to_end__tests__serve__ref_properties_all-2.snap b/rojo-test/serve-test-snapshots/end_to_end__tests__serve__ref_properties_all-2.snap new file mode 100644 index 000000000..63f6c5504 --- /dev/null +++ b/rojo-test/serve-test-snapshots/end_to_end__tests__serve__ref_properties_all-2.snap @@ -0,0 +1,141 @@ +--- +source: tests/tests/serve.rs +expression: "read_response.intern_and_redact(&mut redactions, root_id)" +--- +instances: + id-10: + Children: [] + ClassName: ObjectValue + Id: id-10 + Metadata: + ignoreUnknownInstances: true + Name: ProjectPointer + Parent: id-9 + Properties: + Attributes: + Attributes: + Rojo_Target_Value: + String: project target + Value: + Ref: id-9 + id-11: + Children: [] + ClassName: Model + Id: id-11 + Metadata: + ignoreUnknownInstances: false + Name: ProjectPointer + Parent: id-7 + Properties: + Attributes: + Attributes: + Rojo_Target_PrimaryPart: + String: project target + PrimaryPart: + Ref: id-9 + id-2: + Children: + - id-3 + ClassName: DataModel + Id: id-2 + Metadata: + ignoreUnknownInstances: true + Name: ref_properties + Parent: "00000000000000000000000000000000" + Properties: {} + id-3: + Children: + - id-4 + - id-5 + - id-7 + - id-9 + ClassName: Workspace + Id: id-3 + Metadata: + ignoreUnknownInstances: true + Name: Workspace + Parent: id-2 + Properties: {} + id-4: + Children: [] + ClassName: ObjectValue + Id: id-4 + Metadata: + ignoreUnknownInstances: true + Name: CrossFormatPointer + Parent: id-3 + Properties: + Attributes: + Attributes: + Rojo_Target_Value: + String: folder target + Value: + Ref: id-5 + id-5: + Children: + - id-6 + ClassName: Folder + Id: id-5 + Metadata: + ignoreUnknownInstances: false + Name: FolderTarget + Parent: id-3 + Properties: {} + id-6: + Children: [] + ClassName: ObjectValue + Id: id-6 + Metadata: + ignoreUnknownInstances: false + Name: FolderPointer + Parent: id-5 + Properties: + Attributes: + Attributes: + Rojo_Target_Value: + String: folder target + Value: + Ref: id-5 + id-7: + Children: + - id-8 + - id-11 + ClassName: Folder + Id: id-7 + Metadata: + ignoreUnknownInstances: false + Name: ModelTarget + Parent: id-3 + Properties: + Attributes: + Attributes: + Rojo_Id: + String: model target 2 + id-8: + Children: [] + ClassName: Model + Id: id-8 + Metadata: + ignoreUnknownInstances: false + Name: ModelPointer + Parent: id-7 + Properties: + Attributes: + Attributes: + Rojo_Target_PrimaryPart: + String: model target 2 + PrimaryPart: + Ref: id-7 + id-9: + Children: + - id-10 + ClassName: Folder + Id: id-9 + Metadata: + ignoreUnknownInstances: true + Name: ProjectTarget + Parent: id-3 + Properties: {} +messageCursor: 1 +sessionId: id-1 + diff --git a/rojo-test/serve-test-snapshots/end_to_end__tests__serve__ref_properties_all.snap b/rojo-test/serve-test-snapshots/end_to_end__tests__serve__ref_properties_all.snap new file mode 100644 index 000000000..7fe614cd8 --- /dev/null +++ b/rojo-test/serve-test-snapshots/end_to_end__tests__serve__ref_properties_all.snap @@ -0,0 +1,121 @@ +--- +source: tests/tests/serve.rs +expression: "read_response.intern_and_redact(&mut redactions, root_id)" +--- +instances: + id-10: + Children: [] + ClassName: ObjectValue + Id: id-10 + Metadata: + ignoreUnknownInstances: true + Name: ProjectPointer + Parent: id-9 + Properties: + Attributes: + Attributes: + Rojo_Target_Value: + String: project target + Value: + Ref: id-9 + id-2: + Children: + - id-3 + ClassName: DataModel + Id: id-2 + Metadata: + ignoreUnknownInstances: true + Name: ref_properties + Parent: "00000000000000000000000000000000" + Properties: {} + id-3: + Children: + - id-4 + - id-5 + - id-7 + - id-9 + ClassName: Workspace + Id: id-3 + Metadata: + ignoreUnknownInstances: true + Name: Workspace + Parent: id-2 + Properties: {} + id-4: + Children: [] + ClassName: ObjectValue + Id: id-4 + Metadata: + ignoreUnknownInstances: true + Name: CrossFormatPointer + Parent: id-3 + Properties: + Attributes: + Attributes: + Rojo_Target_Value: + String: folder target + Value: + Ref: id-5 + id-5: + Children: + - id-6 + ClassName: Folder + Id: id-5 + Metadata: + ignoreUnknownInstances: false + Name: FolderTarget + Parent: id-3 + Properties: {} + id-6: + Children: [] + ClassName: ObjectValue + Id: id-6 + Metadata: + ignoreUnknownInstances: false + Name: FolderPointer + Parent: id-5 + Properties: + Attributes: + Attributes: + Rojo_Target_Value: + String: folder target + Value: + Ref: id-5 + id-7: + Children: + - id-8 + ClassName: Folder + Id: id-7 + Metadata: + ignoreUnknownInstances: false + Name: ModelTarget + Parent: id-3 + Properties: {} + id-8: + Children: [] + ClassName: Model + Id: id-8 + Metadata: + ignoreUnknownInstances: false + Name: ModelPointer + Parent: id-7 + Properties: + Attributes: + Attributes: + Rojo_Target_PrimaryPart: + String: model target + PrimaryPart: + Ref: id-7 + id-9: + Children: + - id-10 + ClassName: Folder + Id: id-9 + Metadata: + ignoreUnknownInstances: true + Name: ProjectTarget + Parent: id-3 + Properties: {} +messageCursor: 0 +sessionId: id-1 + diff --git a/rojo-test/serve-test-snapshots/end_to_end__tests__serve__ref_properties_info.snap b/rojo-test/serve-test-snapshots/end_to_end__tests__serve__ref_properties_info.snap new file mode 100644 index 000000000..46ee7ea0a --- /dev/null +++ b/rojo-test/serve-test-snapshots/end_to_end__tests__serve__ref_properties_info.snap @@ -0,0 +1,13 @@ +--- +source: tests/tests/serve.rs +expression: redactions.redacted_yaml(info) +--- +expectedPlaceIds: ~ +gameId: ~ +placeId: ~ +projectName: ref_properties +protocolVersion: 4 +rootInstanceId: id-2 +serverVersion: "[server-version]" +sessionId: id-1 + diff --git a/rojo-test/serve-test-snapshots/end_to_end__tests__serve__ref_properties_remove_all-2.snap b/rojo-test/serve-test-snapshots/end_to_end__tests__serve__ref_properties_remove_all-2.snap new file mode 100644 index 000000000..e548f0108 --- /dev/null +++ b/rojo-test/serve-test-snapshots/end_to_end__tests__serve__ref_properties_remove_all-2.snap @@ -0,0 +1,33 @@ +--- +source: tests/tests/serve.rs +expression: "read_response.intern_and_redact(&mut redactions, root_id)" +--- +instances: + id-2: + Children: + - id-3 + ClassName: Folder + Id: id-2 + Metadata: + ignoreUnknownInstances: false + Name: ref_properties_remove + Parent: "00000000000000000000000000000000" + Properties: {} + id-3: + Children: [] + ClassName: ObjectValue + Id: id-3 + Metadata: + ignoreUnknownInstances: false + Name: pointer + Parent: id-2 + Properties: + Attributes: + Attributes: + Rojo_Target_Value: + String: test pointer + Value: + Ref: id-4 +messageCursor: 1 +sessionId: id-1 + diff --git a/rojo-test/serve-test-snapshots/end_to_end__tests__serve__ref_properties_remove_all.snap b/rojo-test/serve-test-snapshots/end_to_end__tests__serve__ref_properties_remove_all.snap new file mode 100644 index 000000000..407670daa --- /dev/null +++ b/rojo-test/serve-test-snapshots/end_to_end__tests__serve__ref_properties_remove_all.snap @@ -0,0 +1,47 @@ +--- +source: tests/tests/serve.rs +expression: "read_response.intern_and_redact(&mut redactions, root_id)" +--- +instances: + id-2: + Children: + - id-3 + - id-4 + ClassName: Folder + Id: id-2 + Metadata: + ignoreUnknownInstances: false + Name: ref_properties_remove + Parent: "00000000000000000000000000000000" + Properties: {} + id-3: + Children: [] + ClassName: ObjectValue + Id: id-3 + Metadata: + ignoreUnknownInstances: false + Name: pointer + Parent: id-2 + Properties: + Attributes: + Attributes: + Rojo_Target_Value: + String: test pointer + Value: + Ref: id-4 + id-4: + Children: [] + ClassName: ObjectValue + Id: id-4 + Metadata: + ignoreUnknownInstances: false + Name: target + Parent: id-2 + Properties: + Attributes: + Attributes: + Rojo_Id: + String: test pointer +messageCursor: 0 +sessionId: id-1 + diff --git a/rojo-test/serve-test-snapshots/end_to_end__tests__serve__ref_properties_remove_info.snap b/rojo-test/serve-test-snapshots/end_to_end__tests__serve__ref_properties_remove_info.snap new file mode 100644 index 000000000..b51fd1f80 --- /dev/null +++ b/rojo-test/serve-test-snapshots/end_to_end__tests__serve__ref_properties_remove_info.snap @@ -0,0 +1,13 @@ +--- +source: tests/tests/serve.rs +expression: redactions.redacted_yaml(info) +--- +expectedPlaceIds: ~ +gameId: ~ +placeId: ~ +projectName: ref_properties_remove +protocolVersion: 4 +rootInstanceId: id-2 +serverVersion: "[server-version]" +sessionId: id-1 + diff --git a/rojo-test/serve-test-snapshots/end_to_end__tests__serve__ref_properties_remove_subscribe.snap b/rojo-test/serve-test-snapshots/end_to_end__tests__serve__ref_properties_remove_subscribe.snap new file mode 100644 index 000000000..92432570f --- /dev/null +++ b/rojo-test/serve-test-snapshots/end_to_end__tests__serve__ref_properties_remove_subscribe.snap @@ -0,0 +1,12 @@ +--- +source: tests/tests/serve.rs +expression: "subscribe_response.intern_and_redact(&mut redactions, ())" +--- +messageCursor: 1 +messages: + - added: {} + removed: + - id-4 + updated: [] +sessionId: id-1 + diff --git a/rojo-test/serve-test-snapshots/end_to_end__tests__serve__ref_properties_subscribe.snap b/rojo-test/serve-test-snapshots/end_to_end__tests__serve__ref_properties_subscribe.snap new file mode 100644 index 000000000..4500894aa --- /dev/null +++ b/rojo-test/serve-test-snapshots/end_to_end__tests__serve__ref_properties_subscribe.snap @@ -0,0 +1,46 @@ +--- +source: tests/tests/serve.rs +expression: "subscribe_response.intern_and_redact(&mut redactions, ())" +--- +messageCursor: 1 +messages: + - added: + id-11: + Children: [] + ClassName: Model + Id: id-11 + Metadata: + ignoreUnknownInstances: false + Name: ProjectPointer + Parent: id-7 + Properties: + Attributes: + Attributes: + Rojo_Target_PrimaryPart: + String: project target + PrimaryPart: + Ref: id-9 + removed: [] + updated: + - changedClassName: ~ + changedMetadata: + ignoreUnknownInstances: false + changedName: ~ + changedProperties: + Attributes: + Attributes: + Rojo_Id: + String: model target 2 + id: id-7 + - changedClassName: ~ + changedMetadata: ~ + changedName: ~ + changedProperties: + Attributes: + Attributes: + Rojo_Target_PrimaryPart: + String: model target 2 + PrimaryPart: ~ + id: id-8 +sessionId: id-1 + diff --git a/rojo-test/serve-tests/ref_properties/FolderTarget/FolderPointer.model.json b/rojo-test/serve-tests/ref_properties/FolderTarget/FolderPointer.model.json new file mode 100644 index 000000000..a4543ff4d --- /dev/null +++ b/rojo-test/serve-tests/ref_properties/FolderTarget/FolderPointer.model.json @@ -0,0 +1,6 @@ +{ + "className": "ObjectValue", + "attributes": { + "Rojo_Target_Value": "folder target" + } +} \ No newline at end of file diff --git a/rojo-test/serve-tests/ref_properties/FolderTarget/init.meta.json b/rojo-test/serve-tests/ref_properties/FolderTarget/init.meta.json new file mode 100644 index 000000000..1705a7be2 --- /dev/null +++ b/rojo-test/serve-tests/ref_properties/FolderTarget/init.meta.json @@ -0,0 +1,3 @@ +{ + "id": "folder target" +} \ No newline at end of file diff --git a/rojo-test/serve-tests/ref_properties/ModelTarget.model.json b/rojo-test/serve-tests/ref_properties/ModelTarget.model.json new file mode 100644 index 000000000..367103658 --- /dev/null +++ b/rojo-test/serve-tests/ref_properties/ModelTarget.model.json @@ -0,0 +1,13 @@ +{ + "id": "model target", + "className": "Folder", + "children": [ + { + "name": "ModelPointer", + "className": "Model", + "attributes": { + "Rojo_Target_PrimaryPart": "model target" + } + } + ] +} \ No newline at end of file diff --git a/rojo-test/serve-tests/ref_properties/default.project.json b/rojo-test/serve-tests/ref_properties/default.project.json new file mode 100644 index 000000000..8ffa1908f --- /dev/null +++ b/rojo-test/serve-tests/ref_properties/default.project.json @@ -0,0 +1,32 @@ +{ + "name": "ref_properties", + "tree": { + "$className": "DataModel", + "Workspace": { + "ProjectTarget": { + "$className": "Folder", + "$id": "project target", + "ProjectPointer": { + "$className": "ObjectValue", + "$attributes": { + "Rojo_Target_Value": { + "String": "project target" + } + } + } + }, + "ModelTarget": { + "$path": "ModelTarget.model.json" + }, + "FolderTarget": { + "$path": "FolderTarget" + }, + "CrossFormatPointer": { + "$className": "ObjectValue", + "$attributes": { + "Rojo_Target_Value": "folder target" + } + } + } + } +} \ No newline at end of file diff --git a/rojo-test/serve-tests/ref_properties_remove/default.project.json b/rojo-test/serve-tests/ref_properties_remove/default.project.json new file mode 100644 index 000000000..43f94765b --- /dev/null +++ b/rojo-test/serve-tests/ref_properties_remove/default.project.json @@ -0,0 +1,6 @@ +{ + "name": "ref_properties_remove", + "tree": { + "$path": "src" + } +} \ No newline at end of file diff --git a/rojo-test/serve-tests/ref_properties_remove/src/pointer.model.json b/rojo-test/serve-tests/ref_properties_remove/src/pointer.model.json new file mode 100644 index 000000000..4add638b9 --- /dev/null +++ b/rojo-test/serve-tests/ref_properties_remove/src/pointer.model.json @@ -0,0 +1,6 @@ +{ + "className": "ObjectValue", + "attributes": { + "Rojo_Target_Value": "test pointer" + } +} \ No newline at end of file diff --git a/rojo-test/serve-tests/ref_properties_remove/src/target.model.json b/rojo-test/serve-tests/ref_properties_remove/src/target.model.json new file mode 100644 index 000000000..8069631e4 --- /dev/null +++ b/rojo-test/serve-tests/ref_properties_remove/src/target.model.json @@ -0,0 +1,6 @@ +{ + "className": "ObjectValue", + "attributes": { + "Rojo_Id": "test pointer" + } +} \ No newline at end of file diff --git a/src/cli/syncback.rs b/src/cli/syncback.rs index d8ffccb23..5d56c56ca 100644 --- a/src/cli/syncback.rs +++ b/src/cli/syncback.rs @@ -59,7 +59,7 @@ impl SyncbackCommand { syncback_loop( session_old.vfs(), &dom_old, - &dom_new, + dom_new, session_old.root_project(), )?; log::info!( diff --git a/src/lib.rs b/src/lib.rs index 5d4df344a..19851a8a6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,6 +16,7 @@ mod multimap; mod path_serializer; mod project; mod resolution; +mod rojo_ref; mod serve_session; mod session_id; mod snapshot; @@ -25,5 +26,6 @@ mod variant_eq; mod web; pub use project::*; +pub use rojo_ref::*; pub use session_id::SessionId; pub use web::interface as web_api; diff --git a/src/multimap.rs b/src/multimap.rs index 3a243af1a..9c399c24d 100644 --- a/src/multimap.rs +++ b/src/multimap.rs @@ -1,6 +1,6 @@ use std::{ borrow::Borrow, - collections::HashMap, + collections::{hash_map, HashMap}, fmt::{self, Debug}, hash::Hash, }; @@ -71,3 +71,33 @@ impl PartialEq for MultiMap { self.inner == other.inner } } + +impl Default for MultiMap { + fn default() -> Self { + Self { + inner: Default::default(), + } + } +} + +impl IntoIterator for MultiMap { + type IntoIter = MultiMapIntoIter; + type Item = (K, Vec); + fn into_iter(self) -> Self::IntoIter { + Self::IntoIter { + inner: self.inner.into_iter(), + } + } +} + +pub struct MultiMapIntoIter { + inner: hash_map::IntoIter>, +} + +impl Iterator for MultiMapIntoIter { + type Item = (K, Vec); + + fn next(&mut self) -> Option { + self.inner.next() + } +} diff --git a/src/project.rs b/src/project.rs index 768335f64..35d78426f 100644 --- a/src/project.rs +++ b/src/project.rs @@ -230,6 +230,11 @@ pub struct ProjectNode { #[serde(rename = "$className", skip_serializing_if = "Option::is_none")] pub class_name: Option, + /// If set, defines an ID for the described Instance that can be used + /// to refer to it for the purpose of referent properties. + #[serde(rename = "$id", skip_serializing_if = "Option::is_none")] + pub id: Option, + /// Contains all of the children of the described instance. #[serde(flatten)] pub children: BTreeMap, diff --git a/src/resolution.rs b/src/resolution.rs index 9fcef34d3..4c7fcaacd 100644 --- a/src/resolution.rs +++ b/src/resolution.rs @@ -8,6 +8,8 @@ use rbx_dom_weak::types::{ use rbx_reflection::{DataType, PropertyDescriptor}; use serde::{Deserialize, Serialize}; +use crate::REF_POINTER_ATTRIBUTE_PREFIX; + /// A user-friendly version of `Variant` that supports specifying ambiguous /// values. Ambiguous values need a reflection database to be resolved to a /// usable value. @@ -206,6 +208,10 @@ impl AmbiguousValue { Ok(value.into()) } + (VariantType::Ref, AmbiguousValue::String(_)) => Err(format_err!( + "Cannot resolve Ref properties as a String.\ + Use an attribute named `{REF_POINTER_ATTRIBUTE_PREFIX}{prop_name}" + )), (_, unresolved) => Err(format_err!( "Wrong type of value for property {}.{}. Expected {:?}, got {}", class_name, diff --git a/src/rojo_ref.rs b/src/rojo_ref.rs new file mode 100644 index 000000000..c22fe6227 --- /dev/null +++ b/src/rojo_ref.rs @@ -0,0 +1,40 @@ +use std::{fmt, sync::Arc}; + +use serde::{Deserialize, Serialize}; + +pub const REF_ID_ATTRIBUTE_NAME: &str = "Rojo_Id"; +pub const REF_POINTER_ATTRIBUTE_PREFIX: &str = "Rojo_Target_"; + +// TODO add an internment strategy for RojoRefs +// Something like what rbx-dom does for SharedStrings probably works + +#[derive(Debug, Default, PartialEq, Hash, Clone, Serialize, Deserialize, Eq)] +pub struct RojoRef(Arc>); + +impl RojoRef { + #[inline] + pub fn new(id: Vec) -> Self { + Self(Arc::from(id)) + } + + #[inline] + pub fn from_string(id: String) -> Self { + Self(Arc::from(id.into_bytes())) + } + + #[inline] + pub fn as_str(&self) -> Option<&str> { + std::str::from_utf8(&self.0).ok() + } +} + +impl fmt::Display for RojoRef { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.as_str() { + Some(str) => write!(f, "{str}"), + None => { + write!(f, "Binary({:?})", self.0.as_slice()) + } + } + } +} diff --git a/src/snapshot/metadata.rs b/src/snapshot/metadata.rs index 27ec40213..55158518a 100644 --- a/src/snapshot/metadata.rs +++ b/src/snapshot/metadata.rs @@ -12,6 +12,7 @@ use crate::{ path_serializer, project::ProjectNode, snapshot_middleware::{emit_legacy_scripts_default, Middleware}, + RojoRef, }; /// Rojo-specific metadata that can be associated with an instance or a snapshot @@ -59,6 +60,9 @@ pub struct InstanceMetadata { /// context will be passed into it. pub context: InstanceContext, + /// Indicates the ID used for Ref properties pointing to this Instance. + pub specified_id: Option, + pub middleware: Option, } @@ -69,6 +73,7 @@ impl InstanceMetadata { instigating_source: None, relevant_paths: Vec::new(), context: InstanceContext::default(), + specified_id: None, middleware: None, } } @@ -100,6 +105,13 @@ impl InstanceMetadata { ..self } } + + pub fn specified_id(self, id: Option) -> Self { + Self { + specified_id: id, + ..self + } + } } impl Default for InstanceMetadata { diff --git a/src/snapshot/mod.rs b/src/snapshot/mod.rs index ff6ddb5ba..5aacdad6e 100644 --- a/src/snapshot/mod.rs +++ b/src/snapshot/mod.rs @@ -48,7 +48,6 @@ #![allow(dead_code)] -mod hash; mod instance_snapshot; mod metadata; mod patch; @@ -56,7 +55,6 @@ mod patch_apply; mod patch_compute; mod tree; -pub use hash::*; pub use instance_snapshot::InstanceSnapshot; pub use metadata::*; pub use patch::*; diff --git a/src/snapshot/patch_apply.rs b/src/snapshot/patch_apply.rs index 572b8455e..b1cbb3567 100644 --- a/src/snapshot/patch_apply.rs +++ b/src/snapshot/patch_apply.rs @@ -11,6 +11,7 @@ use super::{ patch::{AppliedPatchSet, AppliedPatchUpdate, PatchSet, PatchUpdate}, InstanceSnapshot, RojoTree, }; +use crate::{multimap::MultiMap, RojoRef, REF_ID_ATTRIBUTE_NAME, REF_POINTER_ATTRIBUTE_PREFIX}; /// Consumes the input `PatchSet`, applying all of its prescribed changes to the /// tree and returns an `AppliedPatchSet`, which can be used to keep another @@ -72,6 +73,11 @@ struct PatchApplyContext { /// to be rewritten. has_refs_to_rewrite: HashSet, + /// Tracks all ref properties that were specified using attributes. This has + /// to be handled after everything else is done just like normal referent + /// properties. + attribute_refs_to_rewrite: MultiMap)>, + /// The current applied patch result, describing changes made to the tree. applied_patch_set: AppliedPatchSet, } @@ -104,6 +110,22 @@ fn finalize_patch_application(context: PatchApplyContext, tree: &mut RojoTree) - } } + // This is to get around the fact that `RojoTre::get_specified_id` borrows + // the tree as immutable, but we need to hold a mutable reference to it. + // Not exactly elegant, but it does the job. + let mut real_rewrites = Vec::new(); + for (id, map) in context.attribute_refs_to_rewrite { + for (prop_name, prop_value) in map { + if let Some(target) = tree.get_specified_id(&RojoRef::new(prop_value)) { + real_rewrites.push((prop_name, Variant::Ref(target))) + } + } + let mut instance = tree + .get_instance_mut(id) + .expect("Invalid instance ID in deferred attribute ref map"); + instance.properties_mut().extend(real_rewrites.drain(..)); + } + context.applied_patch_set } @@ -142,6 +164,8 @@ fn apply_add_child( for child in children { apply_add_child(context, tree, id, child); } + + defer_ref_properties(tree, id, context); } fn apply_update_child(context: &mut PatchApplyContext, tree: &mut RojoTree, patch: PatchUpdate) { @@ -208,9 +232,64 @@ fn apply_update_child(context: &mut PatchApplyContext, tree: &mut RojoTree, patc applied_patch.changed_properties.insert(key, property_entry); } + defer_ref_properties(tree, patch.id, context); + context.applied_patch_set.updated.push(applied_patch) } +/// Calculates manually-specified Ref properties and marks them in the provided +/// `PatchApplyContext` to be rewritten at the end of the patch application +/// process. +/// +/// Currently, this only uses attributes but it can easily handle rewriting +/// referents in other ways too! +fn defer_ref_properties(tree: &mut RojoTree, id: Ref, context: &mut PatchApplyContext) { + let instance = tree + .get_instance(id) + .expect("Instances should exist when calculating deferred refs"); + let attributes = match instance.properties().get("Attributes") { + Some(Variant::Attributes(attrs)) => attrs, + _ => return, + }; + + let mut attr_id = None; + for (attr_name, attr_value) in attributes.iter() { + if attr_name == REF_ID_ATTRIBUTE_NAME { + if let Variant::String(specified_id) = attr_value { + attr_id = Some(RojoRef::from_string(specified_id.clone())); + } else if let Variant::BinaryString(specified_id) = attr_value { + attr_id = Some(RojoRef::new(specified_id.clone().into_vec())) + } else { + log::warn!( + "Attribute {attr_name} is of type {:?} when it was \ + expected to be a String", + attr_value.ty() + ) + } + } + if let Some(prop_name) = attr_name.strip_prefix(REF_POINTER_ATTRIBUTE_PREFIX) { + if let Variant::String(prop_value) = attr_value { + context + .attribute_refs_to_rewrite + .insert(id, (prop_name.to_owned(), prop_value.clone().into_bytes())); + } else if let Variant::BinaryString(specified_id) = attr_value { + context + .attribute_refs_to_rewrite + .insert(id, (prop_name.to_owned(), specified_id.clone().into_vec())); + } else { + log::warn!( + "Attribute {attr_name} is of type {:?} when it was \ + expected to be a String", + attr_value.ty() + ) + } + } + } + if let Some(specified_id) = attr_id { + tree.set_specified_id(id, specified_id); + } +} + #[cfg(test)] mod test { use super::*; diff --git a/src/snapshot/tests/snapshots/librojo__snapshot__tests__apply__add_property.snap b/src/snapshot/tests/snapshots/librojo__snapshot__tests__apply__add_property.snap index 3c751fdf4..c4390b6da 100644 --- a/src/snapshot/tests/snapshots/librojo__snapshot__tests__apply__add_property.snap +++ b/src/snapshot/tests/snapshots/librojo__snapshot__tests__apply__add_property.snap @@ -13,6 +13,7 @@ metadata: relevant_paths: [] context: emit_legacy_scripts: true + specified_id: ~ middleware: ~ children: [] diff --git a/src/snapshot/tests/snapshots/librojo__snapshot__tests__apply__remove_property_after_patch.snap b/src/snapshot/tests/snapshots/librojo__snapshot__tests__apply__remove_property_after_patch.snap index c4af516ea..fa4d3ebc2 100644 --- a/src/snapshot/tests/snapshots/librojo__snapshot__tests__apply__remove_property_after_patch.snap +++ b/src/snapshot/tests/snapshots/librojo__snapshot__tests__apply__remove_property_after_patch.snap @@ -11,6 +11,7 @@ metadata: relevant_paths: [] context: emit_legacy_scripts: true + specified_id: ~ middleware: ~ children: [] diff --git a/src/snapshot/tests/snapshots/librojo__snapshot__tests__apply__remove_property_initial.snap b/src/snapshot/tests/snapshots/librojo__snapshot__tests__apply__remove_property_initial.snap index 8ed43ccd9..f07da090e 100644 --- a/src/snapshot/tests/snapshots/librojo__snapshot__tests__apply__remove_property_initial.snap +++ b/src/snapshot/tests/snapshots/librojo__snapshot__tests__apply__remove_property_initial.snap @@ -13,6 +13,7 @@ metadata: relevant_paths: [] context: emit_legacy_scripts: true + specified_id: ~ middleware: ~ children: [] diff --git a/src/snapshot/tests/snapshots/librojo__snapshot__tests__apply__set_name_and_class_name.snap b/src/snapshot/tests/snapshots/librojo__snapshot__tests__apply__set_name_and_class_name.snap index 5772090e1..cc582633d 100644 --- a/src/snapshot/tests/snapshots/librojo__snapshot__tests__apply__set_name_and_class_name.snap +++ b/src/snapshot/tests/snapshots/librojo__snapshot__tests__apply__set_name_and_class_name.snap @@ -11,6 +11,7 @@ metadata: relevant_paths: [] context: emit_legacy_scripts: true + specified_id: ~ middleware: ~ children: [] diff --git a/src/snapshot/tests/snapshots/librojo__snapshot__tests__compute__add_child.snap b/src/snapshot/tests/snapshots/librojo__snapshot__tests__compute__add_child.snap index 01ae237fa..4c4046047 100644 --- a/src/snapshot/tests/snapshots/librojo__snapshot__tests__compute__add_child.snap +++ b/src/snapshot/tests/snapshots/librojo__snapshot__tests__compute__add_child.snap @@ -12,6 +12,7 @@ added_instances: relevant_paths: [] context: emit_legacy_scripts: true + specified_id: ~ middleware: ~ name: New class_name: Folder diff --git a/src/snapshot/tree.rs b/src/snapshot/tree.rs index e4379bc2d..14c6593ec 100644 --- a/src/snapshot/tree.rs +++ b/src/snapshot/tree.rs @@ -8,7 +8,7 @@ use rbx_dom_weak::{ Instance, InstanceBuilder, WeakDom, }; -use crate::multimap::MultiMap; +use crate::{multimap::MultiMap, RojoRef}; use super::{InstanceMetadata, InstanceSnapshot}; @@ -33,6 +33,9 @@ pub struct RojoTree { /// appearing multiple times in the same Rojo project. This is sometimes /// called "path aliasing" in various Rojo documentation. path_to_ids: MultiMap, + + /// A map of specified RojoRefs to the actual underlying Ref they represent. + specified_id_to_refs: HashMap, } impl RojoTree { @@ -45,6 +48,7 @@ impl RojoTree { inner: WeakDom::new(root_builder), metadata_map: HashMap::new(), path_to_ids: MultiMap::new(), + specified_id_to_refs: HashMap::new(), }; let root_ref = tree.inner.root_ref(); @@ -137,6 +141,14 @@ impl RojoTree { self.path_to_ids.insert(new_path.clone(), id); } } + if existing_metadata.specified_id != metadata.specified_id { + if let Some(old) = &existing_metadata.specified_id { + self.specified_id_to_refs.remove(old); + } + if let Some(new) = &metadata.specified_id { + self.specified_id_to_refs.insert(new.clone(), id); + } + } entry.insert(metadata); } @@ -161,11 +173,32 @@ impl RojoTree { self.metadata_map.get(&id) } + pub fn get_specified_id(&self, specified: &RojoRef) -> Option { + self.specified_id_to_refs.get(specified).copied() + } + + pub fn set_specified_id(&mut self, id: Ref, specified: RojoRef) { + if let Some(metadata) = self.metadata_map.get_mut(&id) { + if let Some(old) = metadata.specified_id.replace(specified.clone()) { + self.specified_id_to_refs.remove(&old); + } + } + self.specified_id_to_refs.insert(specified, id); + } + fn insert_metadata(&mut self, id: Ref, metadata: InstanceMetadata) { for path in &metadata.relevant_paths { self.path_to_ids.insert(path.clone(), id); } + if let Some(specified_id) = &metadata.specified_id { + if self.get_specified_id(specified_id).is_some() { + log::warn!("Duplicate user-specified referent {specified_id}") + } else { + self.set_specified_id(id, specified_id.clone()); + } + } + self.metadata_map.insert(id, metadata); } @@ -174,6 +207,10 @@ impl RojoTree { fn remove_metadata(&mut self, id: Ref) { let metadata = self.metadata_map.remove(&id).unwrap(); + if let Some(specified) = metadata.specified_id { + self.specified_id_to_refs.remove(&specified); + } + for path in &metadata.relevant_paths { self.path_to_ids.remove(path, id); } diff --git a/src/snapshot_middleware/json_model.rs b/src/snapshot_middleware/json_model.rs index b17a4c3aa..07b1ca905 100644 --- a/src/snapshot_middleware/json_model.rs +++ b/src/snapshot_middleware/json_model.rs @@ -14,6 +14,7 @@ use crate::{ resolution::UnresolvedValue, snapshot::{InstanceContext, InstanceSnapshot}, syncback::{FsSnapshot, SyncbackReturn, SyncbackSnapshot}, + RojoRef, }; pub fn snapshot_json_model( @@ -47,6 +48,8 @@ pub fn snapshot_json_model( instance.name = Some(name.to_owned()); + let id = instance.id.take().map(RojoRef::from_string); + let mut snapshot = instance .into_snapshot() .with_context(|| format!("Could not load JSON model: {}", path.display()))?; @@ -55,7 +58,8 @@ pub fn snapshot_json_model( .metadata .instigating_source(path) .relevant_paths(vec![path.to_path_buf()]) - .context(context); + .context(context) + .specified_id(id); Ok(Some(snapshot)) } @@ -140,6 +144,7 @@ fn json_model_from_pair(snapshot: &SyncbackSnapshot, new: Ref, old: Option) children, properties, attributes, + id: None, } } @@ -152,6 +157,9 @@ struct JsonModel { #[serde(alias = "ClassName")] class_name: String, + #[serde(skip_serializing_if = "Option::is_none")] + id: Option, + #[serde( alias = "Children", default = "Vec::new", diff --git a/src/snapshot_middleware/meta_file.rs b/src/snapshot_middleware/meta_file.rs index 0f2519398..1ac4f84f4 100644 --- a/src/snapshot_middleware/meta_file.rs +++ b/src/snapshot_middleware/meta_file.rs @@ -9,7 +9,9 @@ use memofs::{IoResultExt as _, Vfs}; use rbx_dom_weak::types::{Attributes, Variant}; use serde::{Deserialize, Serialize}; -use crate::{resolution::UnresolvedValue, snapshot::InstanceSnapshot, syncback::SyncbackSnapshot}; +use crate::{ + resolution::UnresolvedValue, snapshot::InstanceSnapshot, syncback::SyncbackSnapshot, RojoRef, +}; /// Represents metadata in a sibling file with the same basename. /// @@ -18,6 +20,9 @@ use crate::{resolution::UnresolvedValue, snapshot::InstanceSnapshot, syncback::S #[derive(Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct AdjacentMetadata { + #[serde(skip_serializing_if = "Option::is_none")] + pub id: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub ignore_unknown_instances: Option, @@ -96,6 +101,7 @@ impl AdjacentMetadata { properties, attributes, path, + id: None, }) } @@ -135,9 +141,21 @@ impl AdjacentMetadata { Ok(()) } + fn apply_id(&mut self, snapshot: &mut InstanceSnapshot) -> anyhow::Result<()> { + if self.id.is_some() && snapshot.metadata.specified_id.is_some() { + anyhow::bail!( + "cannot specify an ID using {} (instance has an ID from somewhere else)", + self.path.display() + ); + } + snapshot.metadata.specified_id = self.id.take().map(RojoRef::from_string); + Ok(()) + } + pub fn apply_all(&mut self, snapshot: &mut InstanceSnapshot) -> anyhow::Result<()> { self.apply_ignore_unknown_instances(snapshot); self.apply_properties(snapshot)?; + self.apply_id(snapshot)?; Ok(()) } @@ -164,6 +182,9 @@ impl AdjacentMetadata { #[derive(Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct DirectoryMetadata { + #[serde(skip_serializing_if = "Option::is_none")] + pub id: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub ignore_unknown_instances: Option, @@ -249,6 +270,7 @@ impl DirectoryMetadata { attributes, class_name: None, path, + id: None, }) } @@ -256,6 +278,7 @@ impl DirectoryMetadata { self.apply_ignore_unknown_instances(snapshot); self.apply_class_name(snapshot)?; self.apply_properties(snapshot)?; + self.apply_id(snapshot)?; Ok(()) } @@ -309,6 +332,17 @@ impl DirectoryMetadata { Ok(()) } + fn apply_id(&mut self, snapshot: &mut InstanceSnapshot) -> anyhow::Result<()> { + if self.id.is_some() && snapshot.metadata.specified_id.is_some() { + anyhow::bail!( + "cannot specify an ID using {} (instance has an ID from somewhere else)", + self.path.display() + ); + } + snapshot.metadata.specified_id = self.id.take().map(RojoRef::from_string); + Ok(()) + } + /// Returns whether the metadata is 'empty', meaning it doesn't have anything /// worth persisting in it. Specifically: /// diff --git a/src/snapshot_middleware/project.rs b/src/snapshot_middleware/project.rs index d9144b65b..643ea91e3 100644 --- a/src/snapshot_middleware/project.rs +++ b/src/snapshot_middleware/project.rs @@ -13,6 +13,7 @@ use crate::{ SyncRule, }, syncback::{FsSnapshot, SyncbackReturn, SyncbackSnapshot}, + RojoRef, }; use super::{emit_legacy_scripts_default, snapshot_from_vfs}; @@ -279,6 +280,10 @@ pub fn snapshot_project_node( metadata.ignore_unknown_instances = true; } + if let Some(id) = &node.id { + metadata.specified_id = Some(RojoRef::from_string(id.clone())) + } + metadata.instigating_source = Some(InstigatingSource::ProjectNode { path: project_path.to_path_buf(), name: instance_name.to_string(), diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__csv__test__csv_from_vfs.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__csv__test__csv_from_vfs.snap index 18b12d24b..df634792f 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__csv__test__csv_from_vfs.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__csv__test__csv_from_vfs.snap @@ -12,6 +12,7 @@ metadata: - /foo.meta.json context: emit_legacy_scripts: true + specified_id: ~ middleware: ~ name: foo class_name: LocalizationTable diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__csv__test__csv_with_meta.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__csv__test__csv_with_meta.snap index 1bddf74c1..8e97b5050 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__csv__test__csv_with_meta.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__csv__test__csv_with_meta.snap @@ -12,6 +12,7 @@ metadata: - /foo.meta.json context: emit_legacy_scripts: true + specified_id: ~ middleware: ~ name: foo class_name: LocalizationTable diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__dir__test__empty_folder.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__dir__test__empty_folder.snap index 823106566..224bbfac7 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__dir__test__empty_folder.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__dir__test__empty_folder.snap @@ -12,6 +12,7 @@ metadata: - /foo/init.meta.json context: emit_legacy_scripts: true + specified_id: ~ middleware: ~ name: foo class_name: Folder diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__dir__test__folder_in_folder.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__dir__test__folder_in_folder.snap index 936ec5176..425b5bdad 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__dir__test__folder_in_folder.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__dir__test__folder_in_folder.snap @@ -12,6 +12,7 @@ metadata: - /foo/init.meta.json context: emit_legacy_scripts: true + specified_id: ~ middleware: ~ name: foo class_name: Folder @@ -27,6 +28,7 @@ children: - /foo/Child/init.meta.json context: emit_legacy_scripts: true + specified_id: ~ middleware: dir name: Child class_name: Folder diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__json__test__instance_from_vfs.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__json__test__instance_from_vfs.snap index e1b3b35a0..1ccc79b99 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__json__test__instance_from_vfs.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__json__test__instance_from_vfs.snap @@ -12,6 +12,7 @@ metadata: - /foo.meta.json context: emit_legacy_scripts: true + specified_id: ~ middleware: ~ name: foo class_name: ModuleScript diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__json_model__test__model_from_vfs.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__json_model__test__model_from_vfs.snap index f573e51cb..bf7689775 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__json_model__test__model_from_vfs.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__json_model__test__model_from_vfs.snap @@ -11,6 +11,7 @@ metadata: - /foo.model.json context: emit_legacy_scripts: true + specified_id: ~ middleware: ~ name: foo class_name: IntValue @@ -24,6 +25,7 @@ children: relevant_paths: [] context: emit_legacy_scripts: true + specified_id: ~ middleware: ~ name: The Child class_name: StringValue diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__json_model__test__model_from_vfs_legacy.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__json_model__test__model_from_vfs_legacy.snap index f573e51cb..bf7689775 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__json_model__test__model_from_vfs_legacy.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__json_model__test__model_from_vfs_legacy.snap @@ -11,6 +11,7 @@ metadata: - /foo.model.json context: emit_legacy_scripts: true + specified_id: ~ middleware: ~ name: foo class_name: IntValue @@ -24,6 +25,7 @@ children: relevant_paths: [] context: emit_legacy_scripts: true + specified_id: ~ middleware: ~ name: The Child class_name: StringValue diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_client_from_vfs.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_client_from_vfs.snap index 1484bc63c..2977ec9a1 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_client_from_vfs.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_client_from_vfs.snap @@ -12,6 +12,7 @@ metadata: - /foo.meta.json context: emit_legacy_scripts: true + specified_id: ~ middleware: ~ name: foo class_name: LocalScript diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_module_from_vfs.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_module_from_vfs.snap index 4e62485b9..7f604b3a2 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_module_from_vfs.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_module_from_vfs.snap @@ -12,6 +12,7 @@ metadata: - /foo.meta.json context: emit_legacy_scripts: true + specified_id: ~ middleware: ~ name: foo class_name: ModuleScript diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_module_with_meta.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_module_with_meta.snap index f625bcece..1a621ecd7 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_module_with_meta.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_module_with_meta.snap @@ -12,6 +12,7 @@ metadata: - /foo.meta.json context: emit_legacy_scripts: true + specified_id: ~ middleware: ~ name: foo class_name: ModuleScript diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_script_disabled.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_script_disabled.snap index 6b8c105bb..fd50ee16f 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_script_disabled.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_script_disabled.snap @@ -12,6 +12,7 @@ metadata: - /bar.meta.json context: emit_legacy_scripts: true + specified_id: ~ middleware: ~ name: bar class_name: Script diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_script_with_meta.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_script_with_meta.snap index 3871314b9..8476359fd 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_script_with_meta.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_script_with_meta.snap @@ -12,6 +12,7 @@ metadata: - /foo.meta.json context: emit_legacy_scripts: true + specified_id: ~ middleware: ~ name: foo class_name: Script diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_server_from_vfs.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_server_from_vfs.snap index dc8377f27..59f84aba8 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_server_from_vfs.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__class_server_from_vfs.snap @@ -12,6 +12,7 @@ metadata: - /foo.meta.json context: emit_legacy_scripts: true + specified_id: ~ middleware: ~ name: foo class_name: Script diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_client_from_vfs.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_client_from_vfs.snap index c167a8209..28ddeef7c 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_client_from_vfs.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_client_from_vfs.snap @@ -12,6 +12,7 @@ metadata: - /foo.meta.json context: emit_legacy_scripts: false + specified_id: ~ middleware: ~ name: foo class_name: Script diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_module_from_vfs.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_module_from_vfs.snap index 203e917a6..3645366e8 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_module_from_vfs.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_module_from_vfs.snap @@ -12,6 +12,7 @@ metadata: - /foo.meta.json context: emit_legacy_scripts: false + specified_id: ~ middleware: ~ name: foo class_name: ModuleScript diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_module_with_meta.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_module_with_meta.snap index 35ff43ff5..d1881f320 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_module_with_meta.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_module_with_meta.snap @@ -12,6 +12,7 @@ metadata: - /foo.meta.json context: emit_legacy_scripts: false + specified_id: ~ middleware: ~ name: foo class_name: ModuleScript diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_script_disabled.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_script_disabled.snap index d5f710971..b4f3a6c74 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_script_disabled.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_script_disabled.snap @@ -12,6 +12,7 @@ metadata: - /bar.meta.json context: emit_legacy_scripts: false + specified_id: ~ middleware: ~ name: bar class_name: Script diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_script_with_meta.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_script_with_meta.snap index a9985c790..a87dcf6b0 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_script_with_meta.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_script_with_meta.snap @@ -12,6 +12,7 @@ metadata: - /foo.meta.json context: emit_legacy_scripts: false + specified_id: ~ middleware: ~ name: foo class_name: Script diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_server_from_vfs.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_server_from_vfs.snap index 526166265..5bc36a9df 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_server_from_vfs.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__lua__test__runcontext_server_from_vfs.snap @@ -12,6 +12,7 @@ metadata: - /foo.meta.json context: emit_legacy_scripts: false + specified_id: ~ middleware: ~ name: foo class_name: Script diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_from_direct_file.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_from_direct_file.snap index 5d25824b3..1b87aab8a 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_from_direct_file.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_from_direct_file.snap @@ -11,6 +11,7 @@ metadata: - /foo/hello.project.json context: emit_legacy_scripts: true + specified_id: ~ middleware: ~ name: direct-project class_name: Model diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_path_property_overrides.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_path_property_overrides.snap index 5ecaa4477..cd32e81d0 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_path_property_overrides.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_path_property_overrides.snap @@ -12,6 +12,7 @@ metadata: - /foo/default.project.json context: emit_legacy_scripts: true + specified_id: ~ middleware: project name: path-property-override class_name: StringValue diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_children.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_children.snap index 2f3cd2547..f9d6e4a44 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_children.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_children.snap @@ -11,6 +11,7 @@ metadata: - /foo.project.json context: emit_legacy_scripts: true + specified_id: ~ middleware: ~ name: children class_name: Folder @@ -29,6 +30,7 @@ children: relevant_paths: [] context: emit_legacy_scripts: true + specified_id: ~ middleware: ~ name: Child class_name: Model diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_path_to_project.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_path_to_project.snap index 075867f12..da28d1ce2 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_path_to_project.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_path_to_project.snap @@ -12,6 +12,7 @@ metadata: - /foo/default.project.json context: emit_legacy_scripts: true + specified_id: ~ middleware: project name: path-project class_name: Model diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_path_to_project_with_children.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_path_to_project_with_children.snap index ea628e4d9..077ebb153 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_path_to_project_with_children.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_path_to_project_with_children.snap @@ -12,6 +12,7 @@ metadata: - /foo/default.project.json context: emit_legacy_scripts: true + specified_id: ~ middleware: project name: path-child-project class_name: Folder @@ -30,6 +31,7 @@ children: relevant_paths: [] context: emit_legacy_scripts: true + specified_id: ~ middleware: ~ name: SomeChild class_name: Model diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_path_to_txt.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_path_to_txt.snap index 7fce0d01c..36e451fa2 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_path_to_txt.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_path_to_txt.snap @@ -13,6 +13,7 @@ metadata: - /foo/default.project.json context: emit_legacy_scripts: true + specified_id: ~ middleware: text name: path-project class_name: StringValue diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_resolved_properties.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_resolved_properties.snap index 30bc6655c..27d7004aa 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_resolved_properties.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_resolved_properties.snap @@ -11,6 +11,7 @@ metadata: - /foo.project.json context: emit_legacy_scripts: true + specified_id: ~ middleware: ~ name: resolved-properties class_name: StringValue diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_unresolved_properties.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_unresolved_properties.snap index 33c0bdaff..c6349bae7 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_unresolved_properties.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__project__test__project_with_unresolved_properties.snap @@ -11,6 +11,7 @@ metadata: - /foo.project.json context: emit_legacy_scripts: true + specified_id: ~ middleware: ~ name: unresolved-properties class_name: StringValue diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__toml__test__instance_from_vfs.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__toml__test__instance_from_vfs.snap index 6737442c1..9e72ad999 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__toml__test__instance_from_vfs.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__toml__test__instance_from_vfs.snap @@ -12,6 +12,7 @@ metadata: - /foo.meta.json context: emit_legacy_scripts: true + specified_id: ~ middleware: ~ name: foo class_name: ModuleScript diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__txt__test__instance_from_vfs.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__txt__test__instance_from_vfs.snap index 961da5209..b9c07046a 100644 --- a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__txt__test__instance_from_vfs.snap +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__txt__test__instance_from_vfs.snap @@ -12,6 +12,7 @@ metadata: - /foo.meta.json context: emit_legacy_scripts: true + specified_id: ~ middleware: ~ name: foo class_name: StringValue diff --git a/src/snapshot/hash/mod.rs b/src/syncback/hash/mod.rs similarity index 87% rename from src/snapshot/hash/mod.rs rename to src/syncback/hash/mod.rs index 82f17a295..0cd187b78 100644 --- a/src/snapshot/hash/mod.rs +++ b/src/syncback/hash/mod.rs @@ -8,8 +8,9 @@ use rbx_dom_weak::{ types::{Ref, Variant}, Instance, WeakDom, }; -use std::collections::{HashMap, VecDeque}; +use std::collections::HashMap; +use super::descendants; use crate::variant_eq::variant_eq; /// Returns a map of every `Ref` in the `WeakDom` to a hashed version of the @@ -121,21 +122,3 @@ fn hash_inst<'inst>( hasher.finalize() } - -pub(crate) fn descendants(dom: &WeakDom) -> Vec { - let mut queue = VecDeque::new(); - let mut ordered = Vec::new(); - queue.push_front(dom.root_ref()); - - while let Some(referent) = queue.pop_front() { - let inst = dom - .get_by_ref(referent) - .expect("Invariant: WeakDom had a Ref that wasn't inside it"); - ordered.push(referent); - for child in inst.children() { - queue.push_back(*child) - } - } - - ordered -} diff --git a/src/snapshot/hash/variant.rs b/src/syncback/hash/variant.rs similarity index 100% rename from src/snapshot/hash/variant.rs rename to src/syncback/hash/variant.rs diff --git a/src/syncback/mod.rs b/src/syncback/mod.rs index 1ac992916..ee2eecc1a 100644 --- a/src/syncback/mod.rs +++ b/src/syncback/mod.rs @@ -1,5 +1,7 @@ mod file_names; mod fs_snapshot; +mod hash; +mod ref_properties; mod snapshot; use anyhow::Context; @@ -13,32 +15,37 @@ use std::{ use crate::{ glob::Glob, - snapshot::{hash_tree, InstanceSnapshot, InstanceWithMeta, RojoTree}, + snapshot::{InstanceSnapshot, InstanceWithMeta, RojoTree}, snapshot_middleware::Middleware, Project, }; pub use file_names::{is_valid_file_name, name_for_inst}; pub use fs_snapshot::FsSnapshot; +pub use hash::*; +pub use ref_properties::link_referents; pub use snapshot::{SyncbackData, SyncbackSnapshot}; pub fn syncback_loop<'old>( vfs: &'old Vfs, old_tree: &'old RojoTree, - new_tree: &WeakDom, + mut new_tree: WeakDom, project: &'old Project, ) -> anyhow::Result> { log::debug!("Hashing project DOM"); let old_hashes = hash_tree(old_tree.inner()); log::debug!("Hashing file DOM"); - let new_hashes = hash_tree(new_tree); + let new_hashes = hash_tree(&new_tree); + + log::debug!("Linking referents for new DOM..."); + link_referents(&mut new_tree)?; let project_path = project.folder_location(); let syncback_data = SyncbackData { vfs, old_tree, - new_tree, + new_tree: &new_tree, syncback_rules: project.syncback_rules.as_ref(), }; @@ -54,7 +61,7 @@ pub fn syncback_loop<'old>( let mut fs_snapshot = FsSnapshot::new(); 'syncback: while let Some(snapshot) = snapshots.pop() { - let inst_path = get_inst_path(new_tree, snapshot.new); + let inst_path = get_inst_path(&new_tree, snapshot.new); // We can quickly check that two subtrees are identical and if they are, // skip reconciling them. if let Some(old_ref) = snapshot.old { @@ -224,3 +231,23 @@ fn get_inst_path(dom: &WeakDom, referent: Ref) -> String { } path.into_iter().collect::>().join("/") } + +/// Produces a list of descendants in the WeakDom such that all children come +/// before their parents. +pub(crate) fn descendants(dom: &WeakDom) -> Vec { + let mut queue = VecDeque::new(); + let mut ordered = Vec::new(); + queue.push_front(dom.root_ref()); + + while let Some(referent) = queue.pop_front() { + let inst = dom + .get_by_ref(referent) + .expect("Invariant: WeakDom had a Ref that wasn't inside it"); + ordered.push(referent); + for child in inst.children() { + queue.push_back(*child) + } + } + + ordered +} diff --git a/src/syncback/ref_properties.rs b/src/syncback/ref_properties.rs new file mode 100644 index 000000000..98c4a1b2b --- /dev/null +++ b/src/syncback/ref_properties.rs @@ -0,0 +1,88 @@ +//! Implements iterating through an entire WeakDom and linking all Ref +//! properties using attributes. + +use std::collections::VecDeque; + +use rbx_dom_weak::{ + types::{Attributes, BinaryString, Variant}, + Instance, WeakDom, +}; + +use crate::{multimap::MultiMap, REF_ID_ATTRIBUTE_NAME, REF_POINTER_ATTRIBUTE_PREFIX}; + +/// Iterates through a WeakDom and links referent properties using attributes. +pub fn link_referents(dom: &mut WeakDom) -> anyhow::Result<()> { + let mut links = MultiMap::new(); + let mut queue = VecDeque::new(); + + // Note that this is back-in, front-out. This is important because + // VecDeque::extend is the equivalent to using push_back. + queue.push_back(dom.root_ref()); + while let Some(referent) = queue.pop_front() { + let instance = dom.get_by_ref(referent).unwrap(); + + queue.extend(instance.children().iter().copied()); + for (name, value) in &instance.properties { + if let Variant::Ref(prop_value) = value { + if dom.get_by_ref(*prop_value).is_some() { + links.insert(referent, (name.clone(), *prop_value)) + } + } + } + } + let mut rewrites = Vec::new(); + + for (referent, ref_properties) in links { + for (prop_name, target_ref) in ref_properties { + log::debug!( + "Linking {} to {}.{prop_name}", + dom.get_by_ref(target_ref).unwrap().name, + dom.get_by_ref(referent).unwrap().name, + ); + let target_inst = dom + .get_by_ref_mut(target_ref) + .expect("Ref properties that aren't in DOM should be filtered"); + + let attributes = get_or_insert_attributes(target_inst)?; + if attributes.get(REF_ID_ATTRIBUTE_NAME).is_none() { + attributes.insert( + REF_ID_ATTRIBUTE_NAME.to_owned(), + Variant::BinaryString(referent.to_string().into_bytes().into()), + ); + } + + let target_id = attributes + .get(REF_ID_ATTRIBUTE_NAME) + .expect("every Instance to have an ID"); + if let Variant::BinaryString(target_id) = target_id { + rewrites.push((prop_name, target_id.clone())); + } + } + + let inst = dom.get_by_ref_mut(referent).unwrap(); + let attrs = get_or_insert_attributes(inst)?; + for (name, id) in rewrites.drain(..) { + attrs.insert( + format!("{REF_POINTER_ATTRIBUTE_PREFIX}{name}"), + BinaryString::from(id.into_vec()).into(), + ); + } + } + + Ok(()) +} + +fn get_or_insert_attributes(inst: &mut Instance) -> anyhow::Result<&mut Attributes> { + if !inst.properties.contains_key("Attributes") { + inst.properties + .insert("Attributes".into(), Attributes::new().into()); + } + match inst.properties.get_mut("Attributes") { + Some(Variant::Attributes(attrs)) => Ok(attrs), + Some(ty) => Err(anyhow::format_err!( + "expected property Attributes to be an Attributes but it was {:?}", + ty.ty() + )), + None => unreachable!(), + } +} diff --git a/src/syncback/snapshot.rs b/src/syncback/snapshot.rs index 08e6ec669..099a85f1f 100644 --- a/src/syncback/snapshot.rs +++ b/src/syncback/snapshot.rs @@ -97,7 +97,9 @@ impl<'new, 'old> SyncbackSnapshot<'new, 'old> { if filter_out_property(inst, name.as_str()) { continue; } - // We don't currently support refs or shared strings + // We don't currently support refs or shared strings as properties. + // Technically, Refs are supported as attributes but we don't want + // those handled as properties. if matches!(value, Variant::Ref(_) | Variant::SharedString(_)) { continue; } diff --git a/src/web/ui.rs b/src/web/ui.rs index 6f9d0e077..a0b4f8826 100644 --- a/src/web/ui.rs +++ b/src/web/ui.rs @@ -163,6 +163,7 @@ impl UiService { let content = html! { <> +
"specified_id: " { format!("{:?}", metadata.specified_id) }
"ignore_unknown_instances: " { metadata.ignore_unknown_instances.to_string() }
"instigating source: " { format!("{:?}", metadata.instigating_source) }
"middleware: " { format!("{:?}", metadata.middleware) }
@@ -193,7 +194,7 @@ impl UiService { html! {
-