support customize log,index,crates etc. path in config file (#35)

Signed-off-by: benjamin.747 <[email protected]>

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "freighter-registry"
-version = "0.3.0"
+version = "0.4.0"
 edition = "2021"
 license = "MIT"
 homepage = "https://github.com/open-rust-Initiative/freighter"
@@ -10,31 +10,31 @@ description = "The Freighter's purpose is to help the community and company to b
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-anyhow = "1.0.71"
-thiserror = "1.0.36"
-clap = "4.3.4"
+anyhow = "1.0.79"
+thiserror = "1.0.56"
+clap = {version = "4.4.13", features = ["cargo"]}
 git2 = "0.18.1"
 git2-curl = "0.19"
-url = "2.3.1"
-serde = { version = "1.0.164", features = ["derive"] }
-serde_json = "1.0.97"
-walkdir = "2.3.2"
+url = "2.5.0"
+serde = { version = "1.0.195", features = ["derive"] }
+serde_json = "1.0.111"
+walkdir = "2.4.0"
 reqwest = { version = "0.11.13", features = ["blocking"] }
-openssl = { version = "0.10.54", features = ["vendored"] }
-chrono = "0.4.26"
-sha2 = "0.10.7"
-dirs = "5.0.0"
+openssl = { version = "0.10.62", features = ["vendored"] }
+chrono = "0.4.31"
+sha2 = "0.10.8"
+dirs = "5.0.1"
 toml = "0.8.8"
 log4rs = {version = "1.2.0", features = ["toml_format"] }
-tokio = { version = "1.28.2", features = ["full"] }
-warp = { version = "0.3.3", features = ["tls"] }
-tokio-util = { version = "0.7.4", features = ["codec"] }
-tracing = "0.1"
-tracing-subscriber = "0.3.17"
-bytes = "1.1.0"
-async-trait = "0.1.64"
-tokio-test = "0.4.2"
-rayon = "1.7.0"
+tokio = { version = "1.35.1", features = ["macros", "process"] }
+warp = { version = "0.3.6", features = ["tls"] }
+tokio-util = { version = "0.7.10", features = ["codec"] }
+tracing = "0.1.40"
+tracing-subscriber = "0.3.18"
+bytes = "1.5.0"
+async-trait = "0.1.77"
+tokio-test = "0.4.3"
+rayon = "1.8.0"
 
 
 [dev-dependencies]

README.md

@@ -22,35 +22,41 @@ Freighter's functionality mainly consists of four parts: synchronizing crates in
 
 Freighter can be executed as a standalone executable program. You can build it using the **cargo build --release** command and then copy it to your `/usr/local/bin directory`.
 
-#### 1. Synchronizing Crates Index and Crates
+#### 1. Prerequisite
+* Freighter defaults to storing data in the **default directory /Users/${USERNAME}/freighter**. To customize the storage directory, modify the config.toml file.
+* The config file can be obtained by executing any command; Freighter copies the config.default.toml file to the default directory.
+* Alternatively, the config file can be manually copied from the source code directory src which named **config.default.toml**.
+* Customize the storage path for data by modifying configurations like **log_path, index_path, crates_path**, etc., in the config file.
+* Freighter uses the config.toml configuration in the default directory for its operations. To use a custom config path, add the -c parameter when executing a command. For example:
+  ```bash
+  freighter -c /path/to/config.toml <subcommand>
+  ```
+    You can specify paths like /tmp/freighter/config.toml, /tmp/freighter, or /tmp, and Freighter will automatically interpret them as /tmp/freighter/config.toml.
+
+#### 2. Synchronizing Crates Index and Crates
 To sync crate files, Freighter needs to first sync the crates index. You can use the following command to sync the index file:
 
 ```bash
 freighter crates pull
 ```
 
 This command will create a crates.io-index directory in the default path **/Users/${USERNAME}/freighter** and fetch the index. If the index already exists, it will attempt to update it. 
-You can also use the **-c** parameter to specify a **working directory** to change the storage location of the index and crates:
-
-```bash
-freighter -c /path/to/wokring_dir crates pull
-```
 
 **Full download**: Next, you can use the download command with the init parameter to download the full set of crates files:
 
 ```bash
-freighter -c /path/to/wokring_dir crates download --init
+freighter crates download --init
 ```
 
 **Incremental update**: Without the init parameter, Freighter will compare log records in the **working directory** to determine the index and crates that need incremental updates:
 
 ```bash
-freighter -c /path/to/wokring_dir crates download
+freighter crates download
 ```
 
-#### 2.Syncing the rustup-init Files
-#### 3.Syncing the Rust Toolchain Files
-#### 4.Http Server
+#### 3.Syncing the rustup-init Files
+#### 4.Syncing the Rust Toolchain Files
+#### 5.Http Server
 
 ### How to contribute?
 

src/cli.rs

@@ -13,10 +13,10 @@
 //!   ```
 //!
 
-use std::path::PathBuf;
+use std::path::{PathBuf, Path};
 use std::str::FromStr;
 
-use clap::{arg, ArgMatches};
+use clap::{arg, crate_version, ArgMatches};
 use log4rs::append::console::ConsoleAppender;
 use log4rs::append::rolling_file::policy::compound::roll::delete::DeleteRoller;
 use log4rs::append::rolling_file::policy::compound::trigger::size::SizeTrigger;
@@ -28,7 +28,7 @@ use log4rs::config::{Appender, Root};
 use log4rs::encode::pattern::PatternEncoder;
 use tracing::log::LevelFilter;
 
-use crate::commands;
+use crate::commands::{self};
 use crate::config::{Config, LogConfig};
 use crate::errors::{FreightResult, FreighterError};
 
@@ -47,11 +47,8 @@ pub fn main(config: &mut Config) -> FreightResult {
 
     let args = cli().try_get_matches().unwrap_or_else(|e| e.exit());
 
-    let root = match args.get_one::<String>("work-dir").cloned() {
-        Some(root) => PathBuf::from(root),
-        None => dirs::home_dir().unwrap(),
-    };
-    let mut config = config.load(&root);
+    let config_parent = args.get_one::<String>("config-path").cloned().map(PathBuf::from);
+    let mut config = config.load(config_parent);
 
     let (cmd, subcommand_args) = match args.subcommand() {
         Some((cmd, args)) => (cmd, args),
@@ -71,13 +68,12 @@ fn cli() -> App {
     let usage = "freighter [SUBCOMMAND]";
 
     App::new("freighter")
-        .version("0.1.0")
+        .version(crate_version!())
         .disable_colored_help(true)
         .disable_help_subcommand(true)
         .override_usage(usage)
         .author("Open Rust Initiative")
-        .arg(arg!(-c --"work-dir" <FILE> "specify the work dir,
-             where to download crates, rust toolchains and storage logs, default: $HOME/.freighter")
+        .arg(arg!(-c --"config-path" <FILE> "specify the config path, default: $HOME/freighter/config.toml")
         )
         .help_template(
             "\
@@ -107,9 +103,10 @@ pub fn execute_subcommand(config: &mut Config, cmd: &str, args: &ArgMatches) ->
     }
 }
 /// read values(log format encoder, log limit and level) from config file
-/// and then initialize config for log4rs, log will preserve in /work_dir/log by default
-pub fn init_log(config: &LogConfig, work_dir: PathBuf, sub_command: &str) -> FreightResult {
-    let binding = work_dir.join(format!("log/{}.log", sub_command));
+/// and then initialize config for log4rs, log will preserve in /log_path/log by default
+pub fn init_log(config: &LogConfig, log_path: &Path, sub_command: &str) -> FreightResult {
+    // attach file name
+    let binding = log_path.join(format!("{}.log", sub_command));
     let log_path = binding.to_str().unwrap();
     let level = LevelFilter::from_str(&config.level).unwrap();
 

src/commands/channel.rs

@@ -107,17 +107,12 @@ with 64 download threads and then clean historical files
 ///
 ///
 pub fn exec(config: &mut Config, args: &ArgMatches) -> FreightResult {
-    let work_dir = config
-        .work_dir
-        .as_ref()
-        .expect("something bad happened because work_dir is none");
-
-    crate::cli::init_log(&config.log, work_dir.to_path_buf(), "channel").unwrap();
+    crate::cli::init_log(&config.log, &config.log_path, "channel").unwrap();
 
     let mut opts = ChannelOptions {
         config: config.rustup.to_owned(),
         proxy: config.proxy.to_owned(),
-        dist_path: work_dir.join("dist"),
+        dist_path: config.dist_path.to_owned(),
         ..Default::default()
     };
 

src/commands/command_prelude.rs

@@ -4,7 +4,7 @@
 //!
 //!
 use clap::Command;
-pub use clap::{value_parser, Arg, ArgAction, ArgMatches};
+pub use clap::{value_parser, Arg, ArgAction};
 
 /// Add a custom flag to subcommand
 pub fn flag(name: &'static str, help: &'static str) -> Arg {

src/commands/crates.rs

@@ -101,17 +101,17 @@ OPTIONS:
 {options}
 
 EXAMPLES
-1. Sync the crates index with specify directory
+1. Sync the crates index with specify config path
 
-       freighter -c /mnt/volume_fra1_01 crates pull
+       freighter -c /mnt/freighter/config.toml crates pull
 
 2. Download all crates file and unload:
 
        freighter crates download --init --upload --bucket crates
 
-3. Download crates file with multi-thread to specify directory:
+3. Download crates file with multi-thread:
 
-       freighter -c /mnt/volume_fra1_01 crates -t 32 download --init
+       freighter crates -t 32 download --init
 
 \n")
 }
@@ -120,20 +120,15 @@ EXAMPLES
 ///
 ///
 pub fn exec(config: &mut Config, args: &ArgMatches) -> FreightResult {
-    let work_dir = config
-        .work_dir
-        .as_ref()
-        .expect("something bad happened because work_dir is none");
-
-    crate::cli::init_log(&config.log, work_dir.to_path_buf(), "crates").unwrap();
+    crate::cli::init_log(&config.log, &config.log_path, "crates").unwrap();
 
     let opts = &mut CratesOptions {
         config: config.crates.to_owned(),
         proxy: config.proxy.to_owned(),
-        index: CrateIndex::new(&config.crates.index_domain, work_dir.to_owned()),
+        index: CrateIndex::new(&config.crates.index_domain, config.index_path.to_owned()),
         no_progressbar: args.get_flag("no-progressbar"),
-        crates_path: work_dir.join("crates"),
-        log_path: work_dir.join("log"),
+        crates_path: config.crates_path.to_owned(),
+        log_path: config.log_path.to_owned(),
         ..Default::default()
     };
     let domain = args.get_one::<String>("domain").cloned();

src/commands/rustup_init.rs

@@ -84,17 +84,12 @@ with 64 download threads
 ///
 ///
 pub fn exec(config: &mut Config, args: &ArgMatches) -> FreightResult {
-    let work_dir = config
-        .work_dir
-        .as_ref()
-        .expect("something bad happened because work_dir is none");
-
-    crate::cli::init_log(&config.log, work_dir.to_path_buf(), "rustup").unwrap();
+    crate::cli::init_log(&config.log, &config.log_path, "rustup").unwrap();
 
     let mut opts = RustUpOptions {
         proxy: config.proxy.to_owned(),
         config: config.rustup.to_owned(),
-        rustup_path: work_dir.join("rustup"),
+        rustup_path: config.rustup_path.to_owned(),
         ..Default::default()
     };
 

src/config.default.toml

@@ -1,18 +1,27 @@
 
 
 [log]
+# The path which log file is saved
+log_path = ""
+
 # log format encoder, see https://docs.rs/log4rs/1.2.0/log4rs/append/file/struct.FileAppenderDeserializer.html#configuration
 # for more information
 encoder = "{d}:{l} - {m}{n}"
 
 # log level
 level = "info"
 
-# will delete and regenerate log file if exceeded the limit, unit is MB
+# log file will be deleted and regenerated if exceeded the threshold, unit is MB
 limit = 100
 
 
 [crates]
+# The path which the crates index file is saved
+index_path = ""
+
+# The path which the crates file is saved
+crates_path = ""
+
 # download index from domain
 index_domain = "https://github.com/rust-lang/crates.io-index.git"
 
@@ -22,9 +31,8 @@ domain = "https://static.crates.io/crates"
 # Number of crates download threads
 download_threads = 16
 
-# A list of domains that provide service in order
-# localhost means read from filesystem
-# It is better to add s3 domain at the first line
+# When providing services, freighter will retrieve files from the specified location in the following sequence and 
+# return them to the requesting client. localhost means read from local filesystem
 serve_domains = [
     "https://rust-proxy.obs.cn-east-3.myhuaweicloud.com",
     # "localhost",
@@ -36,13 +44,19 @@ serve_domains = [
 serve_index = "/opt/rust/"
 
 [rustup]
-# download rustup from domain
+# The path which the rustup file is saved
+rustup_path = ""
+
+# The path which the rustup release sha file is saved
+dist_path = ""
+
+# which domain to download rustup from
 domain = "https://static.rust-lang.org"
 
 # Number of rust toolchain download threads
 download_threads = 16
 
-# stable versions that you want to sync with
+# released rust versions that you want to sync with
 sync_stable_versions = [
     "1.9",
     "1.10",
@@ -132,7 +146,15 @@ sync_stable_versions = [
     "1.68.1",
     "1.68.2",
     "1.69",
-    "1.70"
+    "1.70",
+    "1.71",
+    "1.71.1",
+    "1.72",
+    "1.72.1",
+    "1.73",
+    "1.74",
+    "1.74.1",
+    "1.75",
 ]
 
 # days you want to keep for historical nightly version
@@ -141,8 +163,8 @@ sync_nightly_days = 30
 # days you want to keep for historical beta version
 sync_beta_days = 30
 
-# A list of domains that provide service in order
-# localhost means read from filesystem
+# When providing services, freighter will retrieve files from the specified location in the following sequence and 
+# return them to the requesting client. localhost means read from local filesystem
 serve_domains = [
     "https://rust-proxy.obs.cn-east-3.myhuaweicloud.com",
     # "localhost",