Implement Syncback to support converting Roblox files to a Rojo project

CHANGELOG.md

@@ -1,6 +1,39 @@
 # Rojo Changelog
 
 ## Unreleased Changes
+* A new command `rojo syncback` has been added. It can be used as `rojo syncback [path to project] --input [path to file]`.
+ 	This command takes a Roblox file and pulls Instances out of it and places them in the correct position in the provided project.
+    Syncback is primarily controlled by the project file. Any Instances who are either referenced in the project file or a descendant
+    of one that is will be placed in an appropriate location.
+
+    In addition, a new field has been added to project files, `syncbackRules` to control how it behaves:
+
+    ```json
+    {
+        "syncbackRules": {
+            "ignoreTrees": [
+                "DataModel/ServerStorage/ImportantSecrets",
+            ],
+            "ignorePaths": [
+                "src/ServerStorage/Secrets/*"
+            ],
+            "ignoreProperties": {
+                "BasePart": ["Color"]
+            },
+            "syncCurrentCamera": false,
+            "syncUnscriptable": true,
+        }
+    }
+    ```
+
+    A brief explanation of each field:
+
+    - `ignoreTrees` is a list of paths in the **roblox file** that should be ignored
+    - `ignorePaths` is a list of paths in the **file system** that should be ignored
+    - `ignoreProperties` is a list of properties that won't be synced back
+    - `syncCurrentCamera` is a toggle for whether to sync back the Workspace's CurrentCamera. Defaults to `false`.
+    - `syncUnscriptable` is a toggle for whether to sync back properties that cannot be set by the Roblox Studio plugin. Defaults to `false`.
+
 * Projects may now manually link `Ref` properties together using `Attributes`. ([#843])
  	This has two parts: using `id` or `$id` in JSON files or a `Rojo_Target` attribute, an Instance
     is given an ID. Then, that ID may be used elsewhere in the project to point to an Instance

Cargo.toml

@@ -91,6 +91,9 @@ uuid = { version = "1.7.0", features = ["v4", "serde"] }
 clap = { version = "3.2.25", features = ["derive"] }
 profiling = "1.0.15"
 
+blake3 = "1.5.0"
+float-cmp = "0.9.0"
+
 [target.'cfg(windows)'.dependencies]
 winreg = "0.10.1"
 

crates/memofs/CHANGELOG.md

@@ -5,6 +5,7 @@
 ## 0.3.0 (2024-03-15)
 * Changed `StdBackend` file watching component to use minimal recursive watches. [#830]
 * Added `Vfs::read_to_string` and `Vfs::read_to_string_lf_normalized` [#854]
+* Added `create_dir` and `create_dir_all` to allow creating directories.
 
 [#830]: https://github.com/rojo-rbx/rojo/pull/830
 [#854]: https://github.com/rojo-rbx/rojo/pull/854

crates/memofs/src/in_memory_fs.rs

@@ -176,6 +176,21 @@ impl VfsBackend for InMemoryFs {
         }
     }
 
+    fn create_dir(&mut self, path: &Path) -> io::Result<()> {
+        let mut inner = self.inner.lock().unwrap();
+        inner.load_snapshot(path.to_path_buf(), VfsSnapshot::empty_dir())
+    }
+
+    fn create_dir_all(&mut self, path: &Path) -> io::Result<()> {
+        let mut inner = self.inner.lock().unwrap();
+        let mut path_buf = path.to_path_buf();
+        while let Some(parent) = path_buf.parent() {
+            inner.load_snapshot(parent.to_path_buf(), VfsSnapshot::empty_dir())?;
+            path_buf.pop();
+        }
+        inner.load_snapshot(path.to_path_buf(), VfsSnapshot::empty_dir())
+    }
+
     fn remove_file(&mut self, path: &Path) -> io::Result<()> {
         let mut inner = self.inner.lock().unwrap();
 

crates/memofs/src/lib.rs

@@ -71,6 +71,8 @@ pub trait VfsBackend: sealed::Sealed + Send + 'static {
     fn read(&mut self, path: &Path) -> io::Result<Vec<u8>>;
     fn write(&mut self, path: &Path, data: &[u8]) -> io::Result<()>;
     fn read_dir(&mut self, path: &Path) -> io::Result<ReadDir>;
+    fn create_dir(&mut self, path: &Path) -> io::Result<()>;
+    fn create_dir_all(&mut self, path: &Path) -> io::Result<()>;
     fn metadata(&mut self, path: &Path) -> io::Result<Metadata>;
     fn remove_file(&mut self, path: &Path) -> io::Result<()>;
     fn remove_dir_all(&mut self, path: &Path) -> io::Result<()>;
@@ -190,6 +192,16 @@ impl VfsInner {
         Ok(dir)
     }
 
+    fn create_dir<P: AsRef<Path>>(&mut self, path: P) -> io::Result<()> {
+        let path = path.as_ref();
+        self.backend.create_dir(path)
+    }
+
+    fn create_dir_all<P: AsRef<Path>>(&mut self, path: P) -> io::Result<()> {
+        let path = path.as_ref();
+        self.backend.create_dir_all(path)
+    }
+
     fn remove_file<P: AsRef<Path>>(&mut self, path: P) -> io::Result<()> {
         let path = path.as_ref();
         let _ = self.backend.unwatch(path);
@@ -326,6 +338,31 @@ impl Vfs {
         self.inner.lock().unwrap().read_dir(path)
     }
 
+    /// Creates a directory at the provided location.
+    ///
+    /// Roughly equivalent to [`std::fs::create_dir`][std::fs::create_dir].
+    /// Similiar to that function, this function will fail if the parent of the
+    /// path does not exist.
+    ///
+    /// [std::fs::create_dir]: https://doc.rust-lang.org/stable/std/fs/fn.create_dir.html
+    #[inline]
+    pub fn create_dir<P: AsRef<Path>>(&self, path: P) -> io::Result<()> {
+        let path = path.as_ref();
+        self.inner.lock().unwrap().create_dir(path)
+    }
+
+    /// Creates a directory at the provided location, recursively creating
+    /// all parent components if they are missing.
+    ///
+    /// Roughly equivalent to [`std::fs::create_dir_all`][std::fs::create_dir_all].
+    ///
+    /// [std::fs::create_dir_all]: https://doc.rust-lang.org/stable/std/fs/fn.create_dir_all.html
+    #[inline]
+    pub fn create_dir_all<P: AsRef<Path>>(&self, path: P) -> io::Result<()> {
+        let path = path.as_ref();
+        self.inner.lock().unwrap().create_dir_all(path)
+    }
+
     /// Remove a file.
     ///
     /// Roughly equivalent to [`std::fs::remove_file`][std::fs::remove_file].
@@ -428,6 +465,31 @@ impl VfsLock<'_> {
         self.inner.read_dir(path)
     }
 
+    /// Creates a directory at the provided location.
+    ///
+    /// Roughly equivalent to [`std::fs::create_dir`][std::fs::create_dir].
+    /// Similiar to that function, this function will fail if the parent of the
+    /// path does not exist.
+    ///
+    /// [std::fs::create_dir]: https://doc.rust-lang.org/stable/std/fs/fn.create_dir.html
+    #[inline]
+    pub fn create_dir<P: AsRef<Path>>(&mut self, path: P) -> io::Result<()> {
+        let path = path.as_ref();
+        self.inner.create_dir(path)
+    }
+
+    /// Creates a directory at the provided location, recursively creating
+    /// all parent components if they are missing.
+    ///
+    /// Roughly equivalent to [`std::fs::create_dir_all`][std::fs::create_dir_all].
+    ///
+    /// [std::fs::create_dir_all]: https://doc.rust-lang.org/stable/std/fs/fn.create_dir_all.html
+    #[inline]
+    pub fn create_dir_all<P: AsRef<Path>>(&mut self, path: P) -> io::Result<()> {
+        let path = path.as_ref();
+        self.inner.create_dir_all(path)
+    }
+
     /// Remove a file.
     ///
     /// Roughly equivalent to [`std::fs::remove_file`][std::fs::remove_file].

crates/memofs/src/noop_backend.rs

@@ -35,6 +35,20 @@ impl VfsBackend for NoopBackend {
         ))
     }
 
+    fn create_dir(&mut self, _path: &Path) -> io::Result<()> {
+        Err(io::Error::new(
+            io::ErrorKind::Other,
+            "NoopBackend doesn't do anything",
+        ))
+    }
+
+    fn create_dir_all(&mut self, _path: &Path) -> io::Result<()> {
+        Err(io::Error::new(
+            io::ErrorKind::Other,
+            "NoopBackend doesn't do anything",
+        ))
+    }
+
     fn remove_file(&mut self, _path: &Path) -> io::Result<()> {
         Err(io::Error::new(
             io::ErrorKind::Other,

crates/memofs/src/std_backend.rs

@@ -78,6 +78,14 @@ impl VfsBackend for StdBackend {
         })
     }
 
+    fn create_dir(&mut self, path: &Path) -> io::Result<()> {
+        fs_err::create_dir(path)
+    }
+
+    fn create_dir_all(&mut self, path: &Path) -> io::Result<()> {
+        fs_err::create_dir_all(path)
+    }
+
     fn remove_file(&mut self, path: &Path) -> io::Result<()> {
         fs_err::remove_file(path)
     }

rojo-test/syncback-test-snapshots/end_to_end__rojo_test__syncback_util__all_middleware.snap

@@ -0,0 +1,27 @@
+---
+source: tests/rojo_test/syncback_util.rs
+expression: "visualize_fs_snapshot(&fs_snapshot, &output_path)"
+---
+added_files:
+  - src/csv.csv
+  - src/csv_init/init.csv
+  - src/dir/client_script.client.luau
+  - src/dir/init_client_script/init.client.luau
+  - src/dir/init_module_script/init.luau
+  - src/dir/init_server_script/init.server.luau
+  - src/dir/module_script.luau
+  - src/dir/server_script.server.luau
+  - src/model_json.model.json
+  - src/project_json.project.json
+  - src/rbxm.rbxm
+  - src/rbxmx.rbxmx
+  - src/text.txt
+added_dirs:
+  - src
+  - src/csv_init
+  - src/dir
+  - src/dir/init_client_script
+  - src/dir/init_module_script
+  - src/dir/init_server_script
+removed_files: []
+removed_dirs: []

rojo-test/syncback-test-snapshots/end_to_end__rojo_test__syncback_util__baseplate.snap

@@ -0,0 +1,20 @@
+---
+source: tests/rojo_test/syncback_util.rs
+expression: "visualize_fs_snapshot(&fs_snapshot, &output_path)"
+---
+added_files:
+  - Lighting/Atmosphere.model.json
+  - Lighting/Bloom.model.json
+  - Lighting/DepthOfField.model.json
+  - Lighting/Sky.model.json
+  - Lighting/SunRays.model.json
+  - Workspace/Baseplate.rbxm
+  - Workspace/Camera.rbxm
+  - Workspace/SpawnLocation.rbxm
+  - Workspace/Terrain.rbxm
+  - default.project.json
+added_dirs:
+  - Lighting
+  - Workspace
+removed_files: []
+removed_dirs: []

rojo-test/syncback-test-snapshots/end_to_end__rojo_test__syncback_util__child_but_not.snap

@@ -0,0 +1,12 @@
+---
+source: tests/rojo_test/syncback_util.rs
+expression: "visualize_fs_snapshot(&fs_snapshot, &output_path)"
+---
+added_files:
+  - OnlyOneCopy/child_of_one.luau
+  - ReplicatedStorage/child_replicated_storage.luau
+added_dirs:
+  - OnlyOneCopy
+  - ReplicatedStorage
+removed_files: []
+removed_dirs: []

rojo-test/syncback-test-snapshots/end_to_end__rojo_test__syncback_util__duplicate_rojo_id.snap

@@ -0,0 +1,11 @@
+---
+source: tests/rojo_test/syncback_util.rs
+assertion_line: 48
+expression: "visualize_fs_snapshot(&fs_snapshot, &output_path)"
+---
+added_files:
+  - container.model.json
+added_dirs: []
+removed_files: []
+removed_dirs: []
+

rojo-test/syncback-test-snapshots/end_to_end__rojo_test__syncback_util__ignore_paths.snap

@@ -0,0 +1,11 @@
+---
+source: tests/rojo_test/syncback_util.rs
+expression: "visualize_fs_snapshot(&fs_snapshot, &output_path)"
+---
+added_files:
+  - default.project.json
+  - src/integer.model.json
+added_dirs:
+  - src
+removed_files: []
+removed_dirs: []

rojo-test/syncback-test-snapshots/end_to_end__rojo_test__syncback_util__nested_projects.snap

@@ -0,0 +1,12 @@
+---
+source: tests/rojo_test/syncback_util.rs
+assertion_line: 48
+expression: "visualize_fs_snapshot(&fs_snapshot, &output_path)"
+---
+added_files:
+  - nested.project.json
+  - string_value.txt
+added_dirs: []
+removed_files: []
+removed_dirs: []
+

rojo-test/syncback-test-snapshots/end_to_end__rojo_test__syncback_util__nested_projects_weird.snap

@@ -0,0 +1,11 @@
+---
+source: tests/rojo_test/syncback_util.rs
+expression: "visualize_fs_snapshot(&fs_snapshot, &output_path)"
+---
+added_files:
+  - src/modules/ClientModule.luau
+  - src/modules/ServerModule.luau
+added_dirs:
+  - src/modules
+removed_files: []
+removed_dirs: []

rojo-test/syncback-test-snapshots/end_to_end__rojo_test__syncback_util__project_all_middleware.snap

@@ -0,0 +1,24 @@
+---
+source: tests/rojo_test/syncback_util.rs
+expression: "visualize_fs_snapshot(&fs_snapshot, &output_path)"
+---
+added_files:
+  - src/client_script.client.luau
+  - src/csv.csv
+  - src/csv_init/init.csv
+  - src/init_client_script/init.client.luau
+  - src/init_module_script/init.luau
+  - src/init_server_script/init.server.luau
+  - src/model_json.model.json
+  - src/module_script.luau
+  - src/project_json.project.json
+  - src/rbxmx.rbxmx
+  - src/server_script.server.luau
+  - src/text.txt
+added_dirs:
+  - src/csv_init
+  - src/init_client_script
+  - src/init_module_script
+  - src/init_server_script
+removed_files: []
+removed_dirs: []

rojo-test/syncback-test-snapshots/end_to_end__rojo_test__syncback_util__project_init.snap

@@ -0,0 +1,10 @@
+---
+source: tests/rojo_test/syncback_util.rs
+expression: "visualize_fs_snapshot(&fs_snapshot, &output_path)"
+---
+added_files:
+  - src/init.luau
+added_dirs:
+  - src
+removed_files: []
+removed_dirs: []

rojo-test/syncback-test-snapshots/end_to_end__rojo_test__syncback_util__project_reserialize.snap

@@ -0,0 +1,12 @@
+---
+source: tests/rojo_test/syncback_util.rs
+assertion_line: 48
+expression: "visualize_fs_snapshot(&fs_snapshot, &output_path)"
+---
+added_files:
+  - attribute_mismatch.luau
+  - property_mismatch.project.json
+added_dirs: []
+removed_files: []
+removed_dirs: []
+