refactor: JaggedArray replaces DictIndexPointerArray as a more general struct

Cargo.toml

@@ -18,4 +18,37 @@ opt-level = "z"
 needless_borrows_for_generic_args = "allow"
 
 [workspace.dependencies]
+lindera-core = { git = "https://github.com/BlueGreenMagick/lindera.git", branch = "yomikiri" }
+lindera-dictionary = { git = "https://github.com/BlueGreenMagick/lindera.git", branch = "yomikiri" }
+lindera-tokenizer = { git = "https://github.com/BlueGreenMagick/lindera.git", branch = "yomikiri" }
+lindera-unidic-builder = { git = "https://github.com/BlueGreenMagick/lindera.git", branch = "yomikiri" }
+
+anyhow = "1"
+bincode = { version = "2.0.0-rc.3", features = ["serde"] }
+byteorder = "1"
+cfg_aliases = "0.2"
+chrono = { version = "0.4", features = ["alloc"] }
+clap = { version = "4.5", features = ["derive"] }
+csv = "1.2"
+filetime = "0.2"
+flate2 = "1"
+fst = "0.4"
+itertools = "0.11"
+log = "0.4"
+ouroboros = "0.8"
+tempfile = "3.8"
+thiserror = "1"
+regex = "1"
+rustyxml = { version = "0.3.0", git = "https://github.com/Florob/RustyXML.git", package = "RustyXML" }
+unicode-normalization = "0.1"
+unicode-segmentation = "1.9"
 uniffi = { git = "https://github.com/BlueGreenMagick/uniffi-rs.git", branch = "custom" }
+ureq = "2.9"
+walkdir = "2.3"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1"
+zip = "0.6"
+
+yomikiri-dictionary = { path = "./crates/yomikiri-dictionary" }
+yomikiri-jmdict = { path = "./crates/jmdict" }
+yomikiri-unidic-types = { path = "./crates/unidic-types" }

crates/jmdict/Cargo.toml

@@ -6,10 +6,10 @@ description = "JMDict XML Types"
 license = "MIT"
 
 [dependencies]
-rustyxml = { version = "0.3.0", git = "https://github.com/Florob/RustyXML.git", package = "RustyXML" }
-regex = "1"
-bincode = "1"
-thiserror = "1"
+rustyxml = { workspace = true }
+regex = { workspace = true }
+bincode = { workspace = true }
+thiserror = { workspace = true }
 
 [lints]
 workspace = true

crates/jmdict/src/error.rs

@@ -9,7 +9,7 @@ pub enum Error {
     #[error("IO {0}")]
     Io(#[from] io::Error),
     #[error("Bincode {0}")]
-    Bincode(#[from] bincode::Error),
+    BincodeDecode(#[from] bincode::error::DecodeError),
     #[error("Xml parse error: {0}")]
     RustyXml(#[from] rustyxml::ParserError),
     #[error("Expected {expected}, encountered {actual}")]

crates/unidic/Cargo.toml

@@ -6,18 +6,18 @@ edition = "2021"
 license = "MIT"
 
 [dependencies]
-lindera-unidic-builder = { git = "https://github.com/BlueGreenMagick/lindera.git", branch = "yomikiri" }
-lindera-core = { git = "https://github.com/BlueGreenMagick/lindera.git", branch = "yomikiri" }
-yomikiri-jmdict = { path = "../jmdict", default-features = false }
-yomikiri-dictionary = { path = "../yomikiri-dictionary" }
-yomikiri-unidic-types = { path = "../unidic-types" }
-csv = "1.2"
-walkdir = "2.3"
-filetime = "0.2"
-ureq = "2.7"
-tempfile = "3.8"
-zip = "0.6"
-anyhow = "1"
+lindera-unidic-builder = { workspace = true }
+lindera-core = { workspace = true }
+yomikiri-jmdict = { workspace = true, default-features = false }
+yomikiri-dictionary = { workspace = true }
+yomikiri-unidic-types = { workspace = true }
+csv = { workspace = true }
+walkdir = { workspace = true }
+filetime = { workspace = true }
+ureq = { workspace = true }
+tempfile = { workspace = true }
+zip = { workspace = true }
+anyhow = { workspace = true }
 
 [lints]
 workspace = true

crates/yomikiri-backend/Cargo.toml

@@ -25,21 +25,21 @@ path = "chunk_wasm.rs"
 uniffi-bindgen = []
 
 [dependencies]
-lindera-tokenizer = { git = "https://github.com/BlueGreenMagick/lindera.git", branch = "yomikiri" }
-lindera-dictionary = { git = "https://github.com/BlueGreenMagick/lindera.git", branch = "yomikiri" }
-lindera-core = { git = "https://github.com/BlueGreenMagick/lindera.git", branch = "yomikiri" }
-yomikiri-dictionary = { path = "../yomikiri-dictionary" }
-yomikiri-unidic-types = { path = "../unidic-types" }
-unicode-normalization = "0.1"
-unicode-segmentation = "1.9"
-log = { version = "0.4" }
-serde = { version = "1.0", features = ["derive"] }
-serde_json = "1"
-bincode = "1"
-thiserror = "1.0"
-flate2 = "1"
-fst = "0.4"
-anyhow = "1"
+lindera-tokenizer = { workspace = true }
+lindera-dictionary = { workspace = true }
+lindera-core = { workspace = true }
+yomikiri-dictionary = { workspace = true }
+yomikiri-unidic-types = { workspace = true }
+unicode-normalization = { workspace = true }
+unicode-segmentation = { workspace = true }
+log = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+bincode = { workspace = true }
+thiserror = { workspace = true }
+flate2 = { workspace = true }
+fst = { workspace = true }
+anyhow = { workspace = true }
 
 [target.'cfg(target_family="wasm")'.dependencies]
 serde-wasm-bindgen = { version = "0.6" }
@@ -51,8 +51,8 @@ console_error_panic_hook = { version = "0.1" }
 
 [target.'cfg(not(target_family="wasm"))'.dependencies]
 uniffi = { workspace = true }
-ureq = "2.9"
-tempfile = "3"
+ureq = { workspace = true }
+tempfile = { workspace = true }
 
 [target.'cfg(any(target_os="macos", target_os="ios"))'.dependencies]
 oslog = { version = "0.2" }
@@ -62,12 +62,10 @@ path = "../yomikiri-dictionary"
 features = ["uniffi"]
 
 [build-dependencies]
-cfg_aliases = "0.2"
+cfg_aliases = { workspace = true }
 
 [target.'cfg(not(target_family="wasm"))'.build-dependencies]
-uniffi = { git = "https://github.com/BlueGreenMagick/uniffi-rs.git", branch = "custom", features = [
-  "build",
-] }
+uniffi = { workspace = true, features = ["build"] }
 
 [dev-dependencies]
 wasm-bindgen-test = "0.3"

crates/yomikiri-dictionary-generator/Cargo.toml

@@ -7,13 +7,13 @@ description = "Generate or download yomikiri dictionary files"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-clap = { version = "4.5", features = ["derive"] }
-ureq = "2.7"
-tempfile = "3.8"
-anyhow = "1"
-flate2 = "1"
-serde_json = "1"
-yomikiri-dictionary = { path = "../yomikiri-dictionary" }
+clap = { workspace = true }
+ureq = { workspace = true }
+tempfile = { workspace = true }
+anyhow = { workspace = true }
+flate2 = { workspace = true }
+serde_json = { workspace = true }
+yomikiri-dictionary = { workspace = true }
 
 [lints]
 workspace = true

crates/yomikiri-dictionary/Cargo.toml

@@ -7,18 +7,18 @@ license = "MIT"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-yomikiri-jmdict = { path = "../jmdict" }
-yomikiri-unidic-types = { path = "../unidic-types" }
-serde = { version = "1.0", features = ["derive"] }
-serde_json = "1"
-thiserror = "1"
-flate2 = "1"
-byteorder = "1"
-bincode = "1"
-itertools = "0.11"
-log = "0.4"
-fst = "0.4"
-ouroboros = "0.8"
+yomikiri-jmdict = { workspace = true }
+yomikiri-unidic-types = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+thiserror = { workspace = true }
+flate2 = { workspace = true }
+byteorder = { workspace = true }
+bincode = { workspace = true }
+itertools = { workspace = true }
+log = { workspace = true }
+fst = { workspace = true }
+ouroboros = { workspace = true }
 chrono = { version = "0.4", features = ["alloc"] }
 uniffi = { workspace = true, optional = true }
 

crates/yomikiri-dictionary/src/error.rs

@@ -13,12 +13,16 @@ pub enum Error {
     Io(#[from] io::Error),
     #[error("TryFromInt {0}")]
     TryFromInt(#[from] TryFromIntError),
-    #[error("Bincode {0}")]
-    Bincode(#[from] bincode::Error),
+    #[error("Bincode encode: {0}")]
+    BincodeEncode(#[from] bincode::error::EncodeError),
+    #[error("Bincode decode: {0}")]
+    BincodeDecode(#[from] bincode::error::DecodeError),
     #[error("JMDict parse error {0}")]
     JMDict(#[from] yomikiri_jmdict::Error),
     #[error("FST error: {0}")]
     Fst(#[from] fst::Error),
     #[error("Out of range")]
     OutOfRange,
+    #[error("Could not reserve space for vector")]
+    CouldNotReserve(#[from] std::collections::TryReserveError),
 }

crates/yomikiri-dictionary/src/file.rs

@@ -9,7 +9,7 @@ use serde::{Deserialize, Serialize};
 
 use crate::entry::Entry;
 use crate::error::Result;
-use crate::index::DictIndexPointerArray;
+use crate::jagged_array::JaggedArray;
 
 /// Separate chunk when it gets bigger than this size
 pub const CHUNK_CUTOFF_SIZE: usize = 16 * 1024;
@@ -224,10 +224,9 @@ pub fn write_indexes<W: Write>(writer: &mut W, terms: &[DictTermIndex]) -> Resul
     writer.write_u32::<LittleEndian>(fst_bytes.len() as u32)?;
     writer.write(&fst_bytes)?;
 
-    let pointers_bytes = DictIndexPointerArray::create_bytes(&pointers)?;
-
-    writer.write_u32::<LittleEndian>(pointers_bytes.len() as u32)?;
-    writer.write(&pointers_bytes)?;
+    let mut pointers_bytes_buffer = Vec::with_capacity(1000);
+    let pointers_bytes = JaggedArray::from_vec_with_buffer(&pointers, &mut pointers_bytes_buffer)?;
+    pointers_bytes.encode_to(writer)?;
     Ok(())
 }