From 9d9560af923319ff50b84e549d0c7134aa08d790 Mon Sep 17 00:00:00 2001 From: Tim Hoiberg Date: Tue, 4 Jul 2023 00:13:33 +0900 Subject: [PATCH 1/5] Add Satori requests Unfortunately Satori doesn't have a public API, but using a session token there is a "hidden" api that returns the data I want. This means that they'll probably change the API without warning, but I think this solution is still more reliable than scraping the HTML. The cookies have a Max expiration set to 2038, so I shouldn't have to update the cookie for quite a while (assuming that it doesn't get destroyed if I log out). --- .env.development.sample | 3 +- backend/src/api.rs | 1 + backend/src/api/satori.rs | 4 ++ backend/src/api/satori/data.rs | 24 ++++++++ .../current_cards_with_no_reviews.json | 6 ++ .../current_cards_with_pending_reviews.json | 6 ++ backend/src/api/satori/request.rs | 56 +++++++++++++++++++ backend/src/main.rs | 3 +- 8 files changed, 101 insertions(+), 2 deletions(-) create mode 100644 backend/src/api/satori.rs create mode 100644 backend/src/api/satori/data.rs create mode 100644 backend/src/api/satori/fixtures/current_cards_with_no_reviews.json create mode 100644 backend/src/api/satori/fixtures/current_cards_with_pending_reviews.json create mode 100644 backend/src/api/satori/request.rs diff --git a/.env.development.sample b/.env.development.sample index 5d2515c..3159b00 100644 --- a/.env.development.sample +++ b/.env.development.sample @@ -1,2 +1,3 @@ WANIKANI_API_TOKEN="" -BUNPRO_API_TOKEN="" \ No newline at end of file +BUNPRO_API_TOKEN="" +SATORI_COOKIE=" \ No newline at end of file diff --git a/backend/src/api.rs b/backend/src/api.rs index 06a5275..a0a77ac 100644 --- a/backend/src/api.rs +++ b/backend/src/api.rs @@ -1,6 +1,7 @@ use axum::{http::StatusCode, Json}; pub mod bunpro; +pub mod satori; pub mod wanikani; #[derive(serde::Serialize)] diff --git a/backend/src/api/satori.rs b/backend/src/api/satori.rs new file mode 100644 index 0000000..2a6a16b --- /dev/null +++ b/backend/src/api/satori.rs @@ -0,0 +1,4 @@ +pub mod data; +pub mod request; + +pub use request::satori_handler; diff --git a/backend/src/api/satori/data.rs b/backend/src/api/satori/data.rs new file mode 100644 index 0000000..c616bcb --- /dev/null +++ b/backend/src/api/satori/data.rs @@ -0,0 +1,24 @@ +use chrono::{DateTime, Utc}; + +#[derive(serde::Serialize)] +pub struct SatoriData { + data_updated_at: DateTime, + active_review_count: u32, +} + +impl SatoriData { + pub fn new(current_count: SatoriCurrentCardsResponse) -> Self { + Self { + data_updated_at: Utc::now(), + active_review_count: current_count.result, + } + } +} + +#[derive(serde::Deserialize)] +pub struct SatoriCurrentCardsResponse { + result: u32, + success: bool, + message: Option, + exception: Option, +} diff --git a/backend/src/api/satori/fixtures/current_cards_with_no_reviews.json b/backend/src/api/satori/fixtures/current_cards_with_no_reviews.json new file mode 100644 index 0000000..a994214 --- /dev/null +++ b/backend/src/api/satori/fixtures/current_cards_with_no_reviews.json @@ -0,0 +1,6 @@ +{ + "result": 0, + "success": true, + "message": null, + "exception": null +} \ No newline at end of file diff --git a/backend/src/api/satori/fixtures/current_cards_with_pending_reviews.json b/backend/src/api/satori/fixtures/current_cards_with_pending_reviews.json new file mode 100644 index 0000000..9a865d5 --- /dev/null +++ b/backend/src/api/satori/fixtures/current_cards_with_pending_reviews.json @@ -0,0 +1,6 @@ +{ + "result": 6, + "success": true, + "message": null, + "exception": null +} \ No newline at end of file diff --git a/backend/src/api/satori/request.rs b/backend/src/api/satori/request.rs new file mode 100644 index 0000000..c0b735d --- /dev/null +++ b/backend/src/api/satori/request.rs @@ -0,0 +1,56 @@ +use std::env; + +use axum::Json; +use reqwest::{Client, StatusCode}; + +use crate::api::{internal_error, ErrorResponse}; + +use super::data::{SatoriCurrentCardsResponse, SatoriData}; + +pub async fn satori_handler() -> Result, (StatusCode, Json)> { + let current_cards = get_current_cards().await.map_err(internal_error)?; + + let satori_data = SatoriData::new(current_cards); + + Ok(Json(satori_data)) +} + +async fn get_current_cards() -> anyhow::Result { + let satori_cookie = env::var("SATORI_COOKIE")?; + + Client::new() + .get("https://www.satorireader.com/api/studylist/due/count") + .header("Cookie", format!("SessionToken={}", satori_cookie)) + .send() + .await? + .text() + .await + .map(|body| serialize_current_cards_response(&body))? +} + +fn serialize_current_cards_response(body: &str) -> anyhow::Result { + let json_data: SatoriCurrentCardsResponse = serde_json::from_str(body)?; + + Ok(json_data) +} + +#[cfg(test)] +mod test_super { + use super::*; + + #[test] + fn test_current_cards_with_pending_reviews() { + let json_string = include_str!("./fixtures/current_cards_with_pending_reviews.json"); + let serialize_result = serialize_current_cards_response(json_string); + + assert!(serialize_result.is_ok()); + } + + #[test] + fn test_current_cards_with_no_reviews() { + let json_string = include_str!("./fixtures/current_cards_with_no_reviews.json"); + let serialize_result = serialize_current_cards_response(json_string); + + assert!(serialize_result.is_ok()); + } +} diff --git a/backend/src/main.rs b/backend/src/main.rs index a1310bb..6b36be5 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -5,7 +5,7 @@ use tokio::signal; use tower_http::{services::ServeDir, trace::TraceLayer}; use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; -use crate::api::{bunpro::bunpro_handler, wanikani::wanikani_handler}; +use crate::api::{bunpro::bunpro_handler, satori::satori_handler, wanikani::wanikani_handler}; pub mod api; @@ -24,6 +24,7 @@ async fn main() { .route("/", get(root_handler)) .route("/api/wanikani", get(wanikani_handler)) .route("/api/bunpro", get(bunpro_handler)) + .route("/api/satori", get(satori_handler)) .layer(TraceLayer::new_for_http()); let address = SocketAddr::from(([0, 0, 0, 0], 3000)); From 3b4779f9914beb6b5cd57061535c346d5dba2d91 Mon Sep 17 00:00:00 2001 From: Tim Hoiberg Date: Tue, 4 Jul 2023 00:20:25 +0900 Subject: [PATCH 2/5] Correct " in the sample env file --- .env.development.sample | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.development.sample b/.env.development.sample index 3159b00..6f309a6 100644 --- a/.env.development.sample +++ b/.env.development.sample @@ -1,3 +1,3 @@ WANIKANI_API_TOKEN="" BUNPRO_API_TOKEN="" -SATORI_COOKIE=" \ No newline at end of file +SATORI_COOKIE="" \ No newline at end of file From 279f5a7a056c9e4a41710e4a79b6a74a15cccad3 Mon Sep 17 00:00:00 2001 From: Tim Hoiberg Date: Tue, 4 Jul 2023 23:18:37 +0900 Subject: [PATCH 3/5] Fetch new card count from Satori --- backend/src/api/satori/data.rs | 17 +++++- .../fixtures/new_cards_with_no_cards.json | 6 ++ .../new_cards_with_pending_cards.json | 6 ++ backend/src/api/satori/request.rs | 58 +++++++++++++++++-- 4 files changed, 79 insertions(+), 8 deletions(-) create mode 100644 backend/src/api/satori/fixtures/new_cards_with_no_cards.json create mode 100644 backend/src/api/satori/fixtures/new_cards_with_pending_cards.json diff --git a/backend/src/api/satori/data.rs b/backend/src/api/satori/data.rs index c616bcb..4be6c6a 100644 --- a/backend/src/api/satori/data.rs +++ b/backend/src/api/satori/data.rs @@ -4,13 +4,18 @@ use chrono::{DateTime, Utc}; pub struct SatoriData { data_updated_at: DateTime, active_review_count: u32, + new_card_count: u32, } impl SatoriData { - pub fn new(current_count: SatoriCurrentCardsResponse) -> Self { + pub fn new( + current_cards: SatoriCurrentCardsResponse, + new_cards: SatoriNewCardsResponse, + ) -> Self { Self { data_updated_at: Utc::now(), - active_review_count: current_count.result, + active_review_count: current_cards.result, + new_card_count: new_cards.result, } } } @@ -22,3 +27,11 @@ pub struct SatoriCurrentCardsResponse { message: Option, exception: Option, } + +#[derive(serde::Deserialize)] +pub struct SatoriNewCardsResponse { + result: u32, + success: bool, + message: Option, + exception: Option, +} diff --git a/backend/src/api/satori/fixtures/new_cards_with_no_cards.json b/backend/src/api/satori/fixtures/new_cards_with_no_cards.json new file mode 100644 index 0000000..a994214 --- /dev/null +++ b/backend/src/api/satori/fixtures/new_cards_with_no_cards.json @@ -0,0 +1,6 @@ +{ + "result": 0, + "success": true, + "message": null, + "exception": null +} \ No newline at end of file diff --git a/backend/src/api/satori/fixtures/new_cards_with_pending_cards.json b/backend/src/api/satori/fixtures/new_cards_with_pending_cards.json new file mode 100644 index 0000000..1cdb83d --- /dev/null +++ b/backend/src/api/satori/fixtures/new_cards_with_pending_cards.json @@ -0,0 +1,6 @@ +{ + "result": 20, + "success": true, + "message": null, + "exception": null +} \ No newline at end of file diff --git a/backend/src/api/satori/request.rs b/backend/src/api/satori/request.rs index c0b735d..306bf27 100644 --- a/backend/src/api/satori/request.rs +++ b/backend/src/api/satori/request.rs @@ -1,26 +1,26 @@ use std::env; use axum::Json; -use reqwest::{Client, StatusCode}; +use reqwest::{Client, ClientBuilder, StatusCode}; use crate::api::{internal_error, ErrorResponse}; -use super::data::{SatoriCurrentCardsResponse, SatoriData}; +use super::data::{SatoriCurrentCardsResponse, SatoriData, SatoriNewCardsResponse}; pub async fn satori_handler() -> Result, (StatusCode, Json)> { let current_cards = get_current_cards().await.map_err(internal_error)?; + let new_cards = get_new_cards().await.map_err(internal_error)?; - let satori_data = SatoriData::new(current_cards); + let satori_data = SatoriData::new(current_cards, new_cards); Ok(Json(satori_data)) } async fn get_current_cards() -> anyhow::Result { - let satori_cookie = env::var("SATORI_COOKIE")?; + let client = satori_client()?.build()?; - Client::new() + client .get("https://www.satorireader.com/api/studylist/due/count") - .header("Cookie", format!("SessionToken={}", satori_cookie)) .send() .await? .text() @@ -34,6 +34,36 @@ fn serialize_current_cards_response(body: &str) -> anyhow::Result anyhow::Result { + let client = satori_client()?; + + client + .build()? + .get("https://www.satorireader.com/api/studylist/pending-auto-importable/count") + .send() + .await? + .text() + .await + .map(|body| serialize_new_cards_response(&body))? +} + +fn serialize_new_cards_response(body: &str) -> anyhow::Result { + let json_data: SatoriNewCardsResponse = serde_json::from_str(body)?; + + Ok(json_data) +} + +fn satori_client() -> anyhow::Result { + let satori_cookie = env::var("SATORI_COOKIE")?; + + let mut headers = reqwest::header::HeaderMap::new(); + headers.insert( + "Cookie", + format!("SessionToken={}", satori_cookie).parse().unwrap(), + ); + Ok(Client::builder().default_headers(headers)) +} + #[cfg(test)] mod test_super { use super::*; @@ -53,4 +83,20 @@ mod test_super { assert!(serialize_result.is_ok()); } + + #[test] + fn test_new_card_with_pending_cards() { + let json_string = include_str!("./fixtures/new_cards_with_pending_cards.json"); + let serialized_result = serialize_new_cards_response(json_string); + + assert!(serialized_result.is_ok()); + } + + #[test] + fn test_new_card_with_no_cards() { + let json_string = include_str!("./fixtures/new_cards_with_no_cards.json"); + let serialized_result = serialize_new_cards_response(json_string); + + assert!(serialized_result.is_ok()); + } } From 6be5aa08776db256bf19ab143b4341836588741e Mon Sep 17 00:00:00 2001 From: Tim Hoiberg Date: Tue, 4 Jul 2023 23:35:08 +0900 Subject: [PATCH 4/5] Return a client instead of the builder --- backend/src/api/satori/request.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/backend/src/api/satori/request.rs b/backend/src/api/satori/request.rs index 306bf27..5dbbee5 100644 --- a/backend/src/api/satori/request.rs +++ b/backend/src/api/satori/request.rs @@ -1,7 +1,7 @@ use std::env; use axum::Json; -use reqwest::{Client, ClientBuilder, StatusCode}; +use reqwest::{Client, StatusCode}; use crate::api::{internal_error, ErrorResponse}; @@ -17,7 +17,7 @@ pub async fn satori_handler() -> Result, (StatusCode, Json anyhow::Result { - let client = satori_client()?.build()?; + let client = satori_client()?; client .get("https://www.satorireader.com/api/studylist/due/count") @@ -38,7 +38,6 @@ async fn get_new_cards() -> anyhow::Result { let client = satori_client()?; client - .build()? .get("https://www.satorireader.com/api/studylist/pending-auto-importable/count") .send() .await? @@ -53,7 +52,7 @@ fn serialize_new_cards_response(body: &str) -> anyhow::Result anyhow::Result { +fn satori_client() -> anyhow::Result { let satori_cookie = env::var("SATORI_COOKIE")?; let mut headers = reqwest::header::HeaderMap::new(); @@ -61,7 +60,8 @@ fn satori_client() -> anyhow::Result { "Cookie", format!("SessionToken={}", satori_cookie).parse().unwrap(), ); - Ok(Client::builder().default_headers(headers)) + + Ok(Client::builder().default_headers(headers).build()?) } #[cfg(test)] From c595c94cd92b1dfdd2cd32610b160d712ea28aff Mon Sep 17 00:00:00 2001 From: Tim Hoiberg Date: Tue, 4 Jul 2023 23:40:07 +0900 Subject: [PATCH 5/5] Use try_join! Without using join! or try_join! it will execute the requests synchronously, rather than firing both of them off at the same time and waiting for the responses. --- backend/src/api/satori/request.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/backend/src/api/satori/request.rs b/backend/src/api/satori/request.rs index 5dbbee5..b90ce54 100644 --- a/backend/src/api/satori/request.rs +++ b/backend/src/api/satori/request.rs @@ -2,14 +2,17 @@ use std::env; use axum::Json; use reqwest::{Client, StatusCode}; +use tokio::try_join; use crate::api::{internal_error, ErrorResponse}; use super::data::{SatoriCurrentCardsResponse, SatoriData, SatoriNewCardsResponse}; pub async fn satori_handler() -> Result, (StatusCode, Json)> { - let current_cards = get_current_cards().await.map_err(internal_error)?; - let new_cards = get_new_cards().await.map_err(internal_error)?; + let current_cards = get_current_cards(); + let new_cards = get_new_cards(); + + let (current_cards, new_cards) = try_join!(current_cards, new_cards).map_err(internal_error)?; let satori_data = SatoriData::new(current_cards, new_cards);