Update readme doc, add crates publish api in http server (#31)

* added crates publsh api Signed-off-by: benjamin.747 <[email protected]> * update publish progress: added index and file save Signed-off-by: benjamin.747 <[email protected]> * update readme doc Signed-off-by: benjamin.747 <[email protected]> --------- Signed-off-by: benjamin.747 <[email protected]>
open-rust-initiative · Dec 18, 2023 · b8f094b · b8f094b
1 parent b1279fb
commit b8f094b
Show file tree

Hide file tree

Showing 12 changed files with 321 additions and 36 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -1,5 +1,8 @@
 name: Continuous Integration (CI)
-on: [push]
+on:
+  push:
+    branches:
+      - main
 
 jobs:
   test:

diff --git a/Cargo.toml b/Cargo.toml
@@ -1,34 +1,31 @@
 [package]
 name = "freighter"
-version = "0.2.0"
+version = "0.3.0"
 edition = "2021"
 license = "MIT"
 homepage = ""
 repository = "https://github.com/open-rust-Initiative/freighter"
 documentation = ""
-readme = "README.md"
-description = """
-
-"""
+description = ""
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-anyhow = "1.0.66"
+anyhow = "1.0.71"
 thiserror = "1.0.36"
-clap = "4.3.1"
-git2 = "0.17.2"
-git2-curl = "0.18"
+clap = "4.3.4"
+git2 = "0.18.1"
+git2-curl = "0.19"
 url = "2.3.1"
-serde = { version = "1.0.159", features = ["derive"] }
-serde_json = "1.0.96"
+serde = { version = "1.0.164", features = ["derive"] }
+serde_json = "1.0.97"
 walkdir = "2.3.2"
 reqwest = { version = "0.11.13", features = ["blocking"] }
 openssl = { version = "0.10.54", features = ["vendored"] }
 chrono = "0.4.26"
-sha2 = "0.10.6"
+sha2 = "0.10.7"
 dirs = "5.0.0"
-toml = "0.7.1"
+toml = "0.8.8"
 log4rs = {version = "1.2.0", features = ["toml_format"] }
 tokio = { version = "1.28.2", features = ["full"] }
 warp = { version = "0.3.3", features = ["tls"] }

diff --git a/README.md b/README.md
@@ -18,6 +18,39 @@ When developing a program using Rust in a company, we need to host a proxy for c
 
 ### How to use the Freighter?
 
+Freighter's functionality mainly consists of four parts: synchronizing crates index and crates; syncing the rustup-init files; syncing the rust toolchain files; providing a HTTP server that support static file server, parse the Git protocol, and offering API services such as crate publication.
+
+Freighter can be executed as a standalone executable program. You can build it using the **cargo build --release** command and then copy it to your `/usr/local/bin directory`.
+
+#### 1. Synchronizing Crates Index and Crates
+To sync crate files, Freighter needs to first sync the crates index. You can use the following command to sync the index file:
+
+```bash
+freighter crates pull
+```
+
+This command will create a crates.io-index directory in the default path **/Users/${USERNAME}/freighter** and fetch the index. If the index already exists, it will attempt to update it. 
+You can also use the **-c** parameter to specify a **working directory** to change the storage location of the index and crates:
+
+```bash
+freighter -c /path/to/wokring_dir crates pull
+```
+
+**Full download**: Next, you can use the download command with the init parameter to download the full set of crates files:
+
+```bash
+freighter -c /path/to/wokring_dir crates download --init
+```
+
+**Incremental update**: Without the init parameter, Freighter will compare log records in the **working directory** to determine the index and crates that need incremental updates:
+
+```bash
+freighter -c /path/to/wokring_dir crates download
+```
+
+#### 2.Syncing the rustup-init Files
+#### 3.Syncing the Rust Toolchain Files
+#### 4.Http Server
 
 ### How to contribute?
 

diff --git a/src/commands/mod.rs b/src/commands/mod.rs
@@ -25,7 +25,12 @@ pub mod server;
 /// point and the `exec` function is logic entry.
 ///
 pub fn builtin() -> Vec<App> {
-    vec![crates::cli(), rustup_init::cli(), channel::cli(), server::cli()]
+    vec![
+        crates::cli(),
+        rustup_init::cli(),
+        channel::cli(),
+        server::cli(),
+    ]
 }
 
 ///

diff --git a/src/commands/server.rs b/src/commands/server.rs
@@ -32,9 +32,15 @@ use crate::server::file_server::{self, FileServer};
 
 pub fn cli() -> clap::Command {
     clap::Command::new("server")
-        .arg(arg!(-i --"ip" <VALUE> "spcify the ip address").value_parser(value_parser!(IpAddr)).default_value("127.0.0.1"))
         .arg(
-            arg!(-p --"port" <VALUE> "specify the listening port").value_parser(value_parser!(u16)).default_value("8000"),
+            arg!(-i --"ip" <VALUE> "specify the ip address")
+                .value_parser(value_parser!(IpAddr))
+                .default_value("127.0.0.1"),
+        )
+        .arg(
+            arg!(-p --"port" <VALUE> "specify the listening port")
+                .value_parser(value_parser!(u16))
+                .default_value("8000"),
         )
         .arg(
             arg!(-c --"cert-path" <VALUE> "Path to a TLS certificate file")

diff --git a/src/download.rs b/src/download.rs
@@ -123,7 +123,7 @@ pub fn download_and_check_hash(
         let hex = format!("{:x}", result);
 
         //if need to calculate hash
-        if let Some(..) = check_sum {
+        if check_sum.is_some() {
             return if hex == check_sum.unwrap() {
                 tracing::info!("###[ALREADY] \t{:?}", file);
                 Ok(false)
@@ -145,7 +145,7 @@ pub fn download_and_check_hash(
 
 pub fn encode_huaweicloud_url(url: &mut Url) {
     if let Some(domain) = url.domain() {
-        if domain.contains("myhuaweicloud.com") && url.path().starts_with("/crates"){
+        if domain.contains("myhuaweicloud.com") && url.path().starts_with("/crates") {
             let mut path = PathBuf::from(url.path());
             let encode_path: String =
                 byte_serialize(path.file_name().unwrap().to_str().unwrap().as_bytes()).collect();

diff --git a/src/handler/crates_file.rs b/src/handler/crates_file.rs
@@ -28,7 +28,7 @@ use crate::errors::FreightResult;
 use crate::handler::index;
 
 use super::index::CrateIndex;
-use super::DownloadMode;
+use super::{utils, DownloadMode};
 
 /// CratesOptions preserve the sync subcommand config
 #[derive(Clone, Debug)]
@@ -84,22 +84,19 @@ impl Default for CratesOptions {
 impl CratesOptions {
     // the path rules of craes index file
     pub fn get_index_path(&self, name: &str) -> PathBuf {
-        let suffix = match name.len() {
-            1..=2 => format!("{}/{}", name.len(), name),
-            3 => format!("{}/{}/{}", name.len(), &name[0..1], name),
-            _ => format!("{}/{}/{}", &name[0..2], &name[2..4], name),
-        };
+        let suffix = utils::index_suffix(name);
         self.index.path.join(suffix)
     }
 }
 
 /// Crate preserve the crates info parse from registry json file
 #[derive(Serialize, Deserialize, Debug)]
-pub struct Crate {
+pub struct IndexFile {
     pub name: String,
     pub vers: String,
     pub deps: Vec<Dependency>,
-    pub cksum: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub cksum: Option<String>,
     pub features: BTreeMap<String, Vec<String>>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub features2: Option<BTreeMap<String, Vec<String>>>,
@@ -316,14 +313,15 @@ pub fn parse_index_and_download(
 
             for line in buffered.lines() {
                 let line = line.unwrap();
-                let c: Crate = serde_json::from_str(&line).unwrap();
+                let c: IndexFile = serde_json::from_str(&line).unwrap();
                 let err_record = Arc::clone(err_record);
                 let opts = opts.clone();
 
                 let url = Url::parse(&format!(
                     "{}/{}/{}-{}.crate",
                     opts.config.domain, &c.name, &c.name, &c.vers
-                )).unwrap();
+                ))
+                .unwrap();
 
                 let file = opts
                     .crates_path
@@ -352,7 +350,7 @@ pub fn download_crates_with_log(
     path: PathBuf,
     opts: &CratesOptions,
     url: Url,
-    c: Crate,
+    index_file: IndexFile,
     err_record: Arc<Mutex<File>>,
 ) -> FreightResult {
     let down_opts = &DownloadOptions {
@@ -361,7 +359,7 @@ pub fn download_crates_with_log(
         path,
     };
 
-    match download_and_check_hash(down_opts, Some(&c.cksum), false) {
+    match download_and_check_hash(down_opts, Some(&index_file.cksum.unwrap()), false) {
         Ok(download_succ) => {
             let path = &down_opts.path;
             if download_succ && opts.upload {
@@ -383,8 +381,8 @@ pub fn download_crates_with_log(
         Err(err) => {
             let mut err_file = err_record.lock().unwrap();
             let err_crate = ErrorCrate {
-                name: c.name,
-                vers: c.vers,
+                name: index_file.name,
+                vers: index_file.vers,
                 time: Utc::now().timestamp().to_string(),
             };
             let json = serde_json::to_string(&err_crate).unwrap();

diff --git a/src/handler/mod.rs b/src/handler/mod.rs
@@ -29,3 +29,15 @@ impl DownloadMode {
         }
     }
 }
+
+pub mod utils {
+
+    // the path rules of crates index file
+    pub fn index_suffix(name: &str) -> String {
+        match name.len() {
+            1..=2 => format!("{}/{}", name.len(), name),
+            3 => format!("{}/{}/{}", name.len(), &name[0..1], name),
+            _ => format!("{}/{}/{}", &name[0..2], &name[2..4], name),
+        }
+    }
+}
diff --git a/src/handler/rustup.rs b/src/handler/rustup.rs
@@ -6,8 +6,8 @@
 //!
 
 use rayon::{ThreadPool, ThreadPoolBuilder};
-use url::Url;
 use std::{path::PathBuf, sync::Arc};
+use url::Url;
 
 use crate::{
     config::ProxyConfig,

diff --git a/src/server/file_server.rs b/src/server/file_server.rs
@@ -32,6 +32,7 @@ pub struct FileServer {
 #[tokio::main]
 pub async fn start(config: &Config, file_server: &FileServer) {
     tracing_subscriber::fmt::init();
+    // storage::init().await;
     let routes = filters::build_route(config.to_owned())
         .recover(handlers::handle_rejection)
         .with(warp::trace::request());
@@ -68,9 +69,17 @@ pub async fn start(config: &Config, file_server: &FileServer) {
 mod filters {
     use std::path::PathBuf;
 
+    use bytes::{Buf, Bytes};
     use warp::{Filter, Rejection};
 
-    use crate::{config::Config, server::git_protocol::GitCommand};
+    use crate::{
+        config::Config,
+        server::{
+            file_server::utils,
+            git_protocol::GitCommand,
+            model::{CratesPublish, Errors, PublishRsp},
+        },
+    };
 
     use super::handlers;
 
@@ -86,8 +95,64 @@ mod filters {
         // GET /dist/... => ./dist/..
         dist(config.clone())
             .or(rustup(config.clone()))
-            .or(crates(config))
+            .or(crates(config.clone()))
             .or(git(git_work_dir))
+            .or(publish(config.clone()))
+            .or(sparse_index(config))
+    }
+
+    pub fn publish(
+        config: Config,
+    ) -> impl Filter<Extract = (impl warp::Reply,), Error = warp::Rejection> + Clone {
+        warp::path!("api" / "v1" / "crates" / "new")
+            .and(warp::body::bytes())
+            .and(with_config(config))
+            .map(|mut body: Bytes, config: Config| {
+                let json_len = utils::get_usize_from_bytes(body.copy_to_bytes(4));
+
+                tracing::info!("json_len: {:?}", json_len);
+                let json = body.copy_to_bytes(json_len);
+                tracing::info!("raw json: {:?}", json);
+
+                let parse_result = serde_json::from_slice::<CratesPublish>(json.as_ref());
+                let crate_len = utils::get_usize_from_bytes(body.copy_to_bytes(4));
+                let file_content = body.copy_to_bytes(crate_len);
+
+                match parse_result {
+                    Ok(result) => {
+                        println!("JSON: {:?}", result);
+                        let work_dir = config.work_dir.unwrap();
+                        utils::save_crate_index(
+                            &result,
+                            &file_content,
+                            work_dir.join("crates.io-index"),
+                        );
+                        utils::save_crate_file(&result, &file_content, work_dir.join("crates"));
+                        // let std::fs::write();
+                        // 1.verify name and version from local db
+                        // 2.call remote server to check info in crates.io
+                        warp::reply::json(&PublishRsp::default())
+                    }
+                    Err(err) => warp::reply::json(&Errors::new(err.to_string())),
+                }
+            })
+    }
+
+    pub fn sparse_index(
+        config: Config,
+    ) -> impl Filter<Extract = (impl warp::Reply,), Error = warp::Rejection> + Clone {
+        warp::path("index")
+            .and(warp::path::tail())
+            .and(with_config(config))
+            .and_then(|tail: warp::path::Tail, config: Config| async move {
+                handlers::return_files(
+                    config.rustup.serve_domains.unwrap(),
+                    config.work_dir.unwrap(),
+                    PathBuf::from("crates.io-index").join(tail.as_str()),
+                    false,
+                )
+                .await
+            })
     }
 
     // build '/dist/*' route, this route handle rust toolchian files request
@@ -370,3 +435,42 @@ mod handlers {
         Ok(())
     }
 }
+
+mod utils {
+    use std::{fs, path::PathBuf};
+
+    use crate::{
+        handler::{crates_file::IndexFile, utils},
+        server::model::CratesPublish,
+    };
+    use bytes::Bytes;
+    use sha2::{Digest, Sha256};
+
+    pub fn get_usize_from_bytes(bytes: Bytes) -> usize {
+        let mut fixed_array = [0u8; 8];
+        fixed_array[..4].copy_from_slice(&bytes[..4]);
+        usize::from_le_bytes(fixed_array)
+    }
+
+    pub fn save_crate_index(json: &CratesPublish, content: &Bytes, work_dir: PathBuf) {
+        let suffix = utils::index_suffix(&json.name);
+        let index_path = work_dir.join(suffix);
+        //convert publish json to index file
+        let mut index_file: IndexFile =
+            serde_json::from_str(&serde_json::to_string(&json).unwrap()).unwrap();
+
+        let mut hasher = Sha256::new();
+        hasher.update(content);
+        index_file.cksum = Some(format!("{:x}", hasher.finalize()));
+        fs::write(index_path, serde_json::to_string(&index_file).unwrap()).unwrap();
+    }
+
+    pub fn save_crate_file(json: &CratesPublish, content: &Bytes, work_dir: PathBuf) {
+        let crates_dir = work_dir.join(&json.name);
+        if !crates_dir.exists() {
+            fs::create_dir_all(&crates_dir).unwrap();
+        }
+        let crates_file = crates_dir.join(format!("{}-{}.crate", json.name, json.vers));
+        fs::write(crates_file, content).unwrap();
+    }
+}
diff --git a/src/server/mod.rs b/src/server/mod.rs
@@ -7,3 +7,4 @@
 
 pub mod file_server;
 pub mod git_protocol;
+mod model;
Original file line number	Diff line number	Diff line change
Expand Up		@@ -7,3 +7,4 @@

		pub mod file_server;
		pub mod git_protocol;
		mod model;