From 15581b632e2fa072da3fd66d379371507dadf71f Mon Sep 17 00:00:00 2001 From: fuyutarow Date: Mon, 7 Jun 2021 17:49:03 +0900 Subject: [PATCH 1/4] Fix tests-make --- Makefile.toml | 14 +++++---- rust-toolchain.toml | 2 ++ tests-make.toml | 9 +++--- tests-make/convert.toml | 16 +++++------ tests-make/float.toml | 55 ------------------------------------ tests-make/package-json.toml | 8 +++--- 6 files changed, 27 insertions(+), 77 deletions(-) create mode 100644 rust-toolchain.toml delete mode 100644 tests-make/float.toml diff --git a/Makefile.toml b/Makefile.toml index dac85b6..66099eb 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -22,9 +22,15 @@ args = ["build", "--release"] command = "cargo" args = ["build", "--bin", "pq", "--release"] -# not working + +[tasks.test] +alias = "test:all" + [tasks."test:all"] -run_task = [{ name = "test:lib" }, { name = "test:pq" }] +script = ''' +makers test:lib +makers test:pq +''' [tasks."test:lib"] command = "cargo" @@ -71,7 +77,5 @@ script = ''' alias pc="./target/debug/partiql-cli" alias pq="./target/debug/pq" for i in $(seq 1 18) -do - cat samples/q$i.env | pc from --to json | pq -S > samples/q$i.json -done +cat samples/q$i.env | pc from --to json | pq -S > samples/q$i.json ''' diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..271800c --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,2 @@ +[toolchain] +channel = "nightly" \ No newline at end of file diff --git a/tests-make.toml b/tests-make.toml index ac1a49b..19d36dd 100644 --- a/tests-make.toml +++ b/tests-make.toml @@ -1,7 +1,6 @@ includes = [ - "tests-make/convert.toml", - "tests-make/float.toml", - "tests-make/package-json.toml", - "tests-make/se_csv.toml", - "tests-make/se_toml.toml", + "tests-make/convert.toml", + "tests-make/package-json.toml", + "tests-make/se_csv.toml", + "tests-make/se_toml.toml", ] diff --git a/tests-make/convert.toml b/tests-make/convert.toml index 16495b3..7274cec 100644 --- a/tests-make/convert.toml +++ b/tests-make/convert.toml @@ -7,10 +7,10 @@ INPUT = ''' } ''' -[tests.to_json] +[tests.to-json] script = ''' alias pq ="./target/debug/pq" -echo ${INPUT} | pq -t json +echo "${INPUT}" | pq -t json ''' tobe = ''' { @@ -20,10 +20,10 @@ tobe = ''' } ''' -[tests.to_toml] +[tests.to-toml] script = ''' alias pq ="./target/debug/pq" -echo ${INPUT} | pq -t toml +echo "${INPUT}" | pq -t toml ''' tobe = ''' name = 'Mew' @@ -32,10 +32,10 @@ fleeRate = 0.1 ''' -[tests.to_yaml] +[tests.to-yaml] script = ''' alias pq ="./target/debug/pq" -echo ${INPUT} | pq -t yaml +echo "${INPUT}" | pq -t yaml ''' tobe = ''' --- @@ -45,10 +45,10 @@ fleeRate: 0.1 ''' -[tests.to_xml] +[tests.to-xml] script = ''' alias pq ="./target/debug/pq" -echo ${INPUT} | pq -t xml +echo "${INPUT}" | pq -t xml ''' tobe = ''' Mew1510.1 diff --git a/tests-make/float.toml b/tests-make/float.toml deleted file mode 100644 index 502628b..0000000 --- a/tests-make/float.toml +++ /dev/null @@ -1,55 +0,0 @@ -[env] -INPUT = ''' -{ - "name": "Mew", - "id": 151, - "fleeRate": 0.1 -} -''' - -[tests.to-json] -script = ''' -alias pq ="./target/debug/pq" -echo ${INPUT} | pq -t json -''' -tobe = ''' -{ - "name": "Mew", - "id": 151, - "fleeRate": 0.1 -} -''' - -[tests.to-toml] -script = ''' -alias pq ="./target/debug/pq" -echo ${INPUT} | pq -t toml -''' -tobe = ''' -name = 'Mew' -id = 151 -fleeRate = 0.1 - -''' - -[tests.to-yaml] -script = ''' -alias pq ="./target/debug/pq" -echo ${INPUT} | pq -t yaml -''' -tobe = ''' ---- -name: Mew -id: 151 -fleeRate: 0.1 - -''' - -[tests.to-xml] -script = ''' -alias pq ="./target/debug/pq" -echo ${INPUT} | pq -t xml -''' -tobe = ''' -Mew1510.1 -''' diff --git a/tests-make/package-json.toml b/tests-make/package-json.toml index e4acc0a..b5c164c 100644 --- a/tests-make/package-json.toml +++ b/tests-make/package-json.toml @@ -36,7 +36,7 @@ INPUT = ''' [tests.to-json] script = ''' alias pq ="./target/debug/pq" -echo ${INPUT} | pq -t json +echo "${INPUT}" | pq -t json ''' tobe = ''' { @@ -75,7 +75,7 @@ tobe = ''' [tests.to-toml] script = ''' alias pq ="./target/debug/pq" -echo ${INPUT} | pq -t toml +echo "${INPUT}" | pq -t toml ''' tobe = ''' name = 'partiql-pokemon' @@ -113,7 +113,7 @@ typescript = '^4.2.4' [tests.to-yaml] script = ''' alias pq ="./target/debug/pq" -echo ${INPUT} | pq -t yaml +echo "${INPUT}" | pq -t yaml ''' tobe = ''' --- @@ -149,7 +149,7 @@ license: MIT [tests.to-xml] script = ''' alias pq ="./target/debug/pq" -echo ${INPUT} | pq -t xml +echo "${INPUT}" | pq -t xml ''' tobe = ''' partiql-pokemon0.202105.0truenextnext buildnext startnext build && next starteslint . --fix -c .eslintrc.js --ext js,jsx,ts,tsx --ignore-pattern='!.*'tsc<@material-ui/core>^4.11.3<@material-ui/icons>^4.11.2^9.4.4^16.13.1^16.13.1<@types/node>^14.0.13<@types/react>^17.0.3<@types/react-dom>^17.0.3<@typescript-eslint/eslint-plugin>^4.22.0<@typescript-eslint/parser>^4.22.0^2.22.1^4.2.0^4.2.4MIT From ea422fcad765691193d980cd26aac756cec57b3e Mon Sep 17 00:00:00 2001 From: fuyutarow Date: Mon, 7 Jun 2021 20:08:01 +0900 Subject: [PATCH 2/4] Fix serialization to YAML --- Makefile.toml | 2 +- src/lib/lang.rs | 28 ++++++++++++++++++---------- tests-make/convert.toml | 13 ++++--------- tests-make/package-json.toml | 13 ++++--------- 4 files changed, 27 insertions(+), 29 deletions(-) diff --git a/Makefile.toml b/Makefile.toml index 66099eb..e8177e3 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -33,9 +33,9 @@ makers test:pq ''' [tasks."test:lib"] +dependencies = ["build"] command = "cargo" args = ["test"] -dependencies = ["build"] [tasks."test:pq"] dependencies = ["build:pq"] diff --git a/src/lib/lang.rs b/src/lib/lang.rs index 0f42db6..59a3c50 100644 --- a/src/lib/lang.rs +++ b/src/lib/lang.rs @@ -32,15 +32,16 @@ impl FromStr for Lang { type Err = anyhow::Error; fn from_str(input: &str) -> anyhow::Result { - if let Ok(data) = csvstr_to_pqlv(&input) { - Ok(Self { - data, - text: input.to_string(), - from: LangType::Csv, - to: LangType::Csv, - colnames: Vec::default(), - }) - } else if let Ok(data) = serde_json::from_str::(&input) { + // if let Ok(data) = csvstr_to_pqlv(&input) { + // Ok(Self { + // data, + // text: input.to_string(), + // from: LangType::Csv, + // to: LangType::Csv, + // colnames: Vec::default(), + // }) + // } else + if let Ok(data) = serde_json::from_str::(&input) { // Json does not distinguish between Float and Int. For this reason, it it parsed once with serde_json::value::Value, not crate::value::PqlValue. Ok(Self { data: crate::value::json_value::to_pqlvalue(data), @@ -117,7 +118,14 @@ impl Lang { let v = TomlValue::from(self.data.to_owned()); toml::to_string_pretty(&v).unwrap() } - (LangType::Yaml, _) => serde_yaml::to_string(&self.data).unwrap(), + (LangType::Yaml, _) => { + dbg!("yaml"); + + serde_yaml::to_string(&self.data) + .unwrap() + .trim_start_matches("---\n") + .to_string() + } (LangType::Xml, _) => quick_xml::se::to_string(&self.data).unwrap(), }; diff --git a/tests-make/convert.toml b/tests-make/convert.toml index 7274cec..4756b3e 100644 --- a/tests-make/convert.toml +++ b/tests-make/convert.toml @@ -9,8 +9,7 @@ INPUT = ''' [tests.to-json] script = ''' -alias pq ="./target/debug/pq" -echo "${INPUT}" | pq -t json +echo $INPUT | ./target/debug/pq -t json ''' tobe = ''' { @@ -22,8 +21,7 @@ tobe = ''' [tests.to-toml] script = ''' -alias pq ="./target/debug/pq" -echo "${INPUT}" | pq -t toml +echo $INPUT | ./target/debug/pq -t toml ''' tobe = ''' name = 'Mew' @@ -34,11 +32,9 @@ fleeRate = 0.1 [tests.to-yaml] script = ''' -alias pq ="./target/debug/pq" -echo "${INPUT}" | pq -t yaml +echo $INPUT | ./target/debug/pq -t yaml ''' tobe = ''' ---- name: Mew id: 151 fleeRate: 0.1 @@ -47,8 +43,7 @@ fleeRate: 0.1 [tests.to-xml] script = ''' -alias pq ="./target/debug/pq" -echo "${INPUT}" | pq -t xml +echo $INPUT | ./target/debug/pq -t xml ''' tobe = ''' Mew1510.1 diff --git a/tests-make/package-json.toml b/tests-make/package-json.toml index b5c164c..15f79bf 100644 --- a/tests-make/package-json.toml +++ b/tests-make/package-json.toml @@ -35,8 +35,7 @@ INPUT = ''' [tests.to-json] script = ''' -alias pq ="./target/debug/pq" -echo "${INPUT}" | pq -t json +echo $INPUT | ./target/debug/pq -t json ''' tobe = ''' { @@ -74,8 +73,7 @@ tobe = ''' [tests.to-toml] script = ''' -alias pq ="./target/debug/pq" -echo "${INPUT}" | pq -t toml +echo $INPUT | ./target/debug/pq -t toml ''' tobe = ''' name = 'partiql-pokemon' @@ -112,11 +110,9 @@ typescript = '^4.2.4' [tests.to-yaml] script = ''' -alias pq ="./target/debug/pq" -echo "${INPUT}" | pq -t yaml +echo $INPUT | ./target/debug/pq -t yaml ''' tobe = ''' ---- name: partiql-pokemon version: 0.202105.0 private: true @@ -148,8 +144,7 @@ license: MIT [tests.to-xml] script = ''' -alias pq ="./target/debug/pq" -echo "${INPUT}" | pq -t xml +echo $INPUT | ./target/debug/pq -t xml ''' tobe = ''' partiql-pokemon0.202105.0truenextnext buildnext startnext build && next starteslint . --fix -c .eslintrc.js --ext js,jsx,ts,tsx --ignore-pattern='!.*'tsc<@material-ui/core>^4.11.3<@material-ui/icons>^4.11.2^9.4.4^16.13.1^16.13.1<@types/node>^14.0.13<@types/react>^17.0.3<@types/react-dom>^17.0.3<@typescript-eslint/eslint-plugin>^4.22.0<@typescript-eslint/parser>^4.22.0^2.22.1^4.2.0^4.2.4MIT From 194763f9001a4d895dbfe48fff2fb7bd2c99a38b Mon Sep 17 00:00:00 2001 From: fuyutarow Date: Mon, 7 Jun 2021 20:31:39 +0900 Subject: [PATCH 3/4] Add option: --from --- samples/mew-c.json | 1 + samples/mew.json | 5 +++ src/bin/pq.rs | 14 ++++++- src/lib/lang.rs | 81 ++++++++++++++++++++++++++++++++--------- tests-make/se_csv.toml | 3 +- tests-make/se_toml.toml | 6 +-- 6 files changed, 86 insertions(+), 24 deletions(-) create mode 100644 samples/mew-c.json create mode 100644 samples/mew.json diff --git a/samples/mew-c.json b/samples/mew-c.json new file mode 100644 index 0000000..28cce86 --- /dev/null +++ b/samples/mew-c.json @@ -0,0 +1 @@ +{"name":"Mew","id":151,"fleeRate":0.1} diff --git a/samples/mew.json b/samples/mew.json new file mode 100644 index 0000000..deffd39 --- /dev/null +++ b/samples/mew.json @@ -0,0 +1,5 @@ +{ + "name": "Mew", + "id": 151, + "fleeRate": 0.1 +} \ No newline at end of file diff --git a/src/bin/pq.rs b/src/bin/pq.rs index b76c0fc..60a2f59 100644 --- a/src/bin/pq.rs +++ b/src/bin/pq.rs @@ -1,3 +1,4 @@ +use std::convert::TryFrom; use std::io::{self, Read}; use std::path::PathBuf; use std::str::FromStr; @@ -27,6 +28,10 @@ struct Opt { #[structopt(short, long)] query: Option, + /// target config file + #[structopt(short, long, possible_values(&["csv", "json", "toml", "yaml", "xml"]))] + from: Option, + /// target config file #[structopt(short, long, possible_values(&["csv", "json", "toml", "yaml", "xml"]))] to: Option, @@ -40,6 +45,7 @@ fn main() -> anyhow::Result<()> { let Opt { file_or_stdin, query, + from, to, sort_keys, } = Opt::from_args(); @@ -50,7 +56,13 @@ fn main() -> anyhow::Result<()> { read_from_stdin()? }; - let mut lang = Lang::from_str(&input)?; + let mut lang = if let Some(s_lang_type) = from { + let lang_type = LangType::from_str(&s_lang_type)?; + Lang::from_as(&input, lang_type)? + } else { + Lang::from_str(&input)? + }; + if let Some(t) = to { match LangType::from_str(&t) { Ok(lang_type) => lang.to = lang_type, diff --git a/src/lib/lang.rs b/src/lib/lang.rs index 59a3c50..0e398f5 100644 --- a/src/lib/lang.rs +++ b/src/lib/lang.rs @@ -12,9 +12,9 @@ use crate::value::{BPqlValue, PqlValue, TomlValue}; #[derive(Display, FromStr, PartialEq, Clone, Debug)] #[display(style = "snake_case")] pub enum LangType { + Toml, Csv, Json, - Toml, Yaml, Xml, } @@ -32,16 +32,47 @@ impl FromStr for Lang { type Err = anyhow::Error; fn from_str(input: &str) -> anyhow::Result { - // if let Ok(data) = csvstr_to_pqlv(&input) { - // Ok(Self { - // data, - // text: input.to_string(), - // from: LangType::Csv, - // to: LangType::Csv, - // colnames: Vec::default(), - // }) - // } else - if let Ok(data) = serde_json::from_str::(&input) { + if let Ok(this) = Self::from_as_json(input) { + Ok(this) + } else if let Ok(this) = Self::from_as_toml(input) { + Ok(this) + } else if let Ok(this) = Self::from_as_xml(input) { + Ok(this) + } else if let Ok(this) = Self::from_as_xml(input) { + Ok(this) + } else { + anyhow::bail!("not supported") + } + } +} + +impl Lang { + pub fn from_as(input: &str, lnag_type: LangType) -> anyhow::Result { + match lnag_type { + LangType::Csv => Self::from_as_csv(input), + LangType::Json => Self::from_as_json(input), + LangType::Toml => Self::from_as_toml(input), + LangType::Yaml => Self::from_as_yaml(input), + LangType::Xml => Self::from_as_xml(input), + } + } + + pub fn from_as_csv(input: &str) -> anyhow::Result { + if let Ok(data) = csvstr_to_pqlv(input) { + Ok(Self { + data, + text: input.to_string(), + from: LangType::Csv, + to: LangType::Csv, + colnames: Vec::default(), + }) + } else { + anyhow::bail!("fail to parse input as csv"); + } + } + + pub fn from_as_json(input: &str) -> anyhow::Result { + if let Ok(data) = serde_json::from_str::(input) { // Json does not distinguish between Float and Int. For this reason, it it parsed once with serde_json::value::Value, not crate::value::PqlValue. Ok(Self { data: crate::value::json_value::to_pqlvalue(data), @@ -50,7 +81,13 @@ impl FromStr for Lang { to: LangType::Json, colnames: Vec::default(), }) - } else if let Ok(data) = toml::from_str::(&input) { + } else { + anyhow::bail!("fail to parse input as json"); + } + } + + pub fn from_as_toml(input: &str) -> anyhow::Result { + if let Ok(data) = toml::from_str::(input) { Ok(Self { data, text: input.to_string(), @@ -58,7 +95,13 @@ impl FromStr for Lang { to: LangType::Toml, colnames: Vec::default(), }) - } else if let Ok(data) = quick_xml::de::from_str::(&input) { + } else { + anyhow::bail!("fail to parse input as toml"); + } + } + + pub fn from_as_xml(input: &str) -> anyhow::Result { + if let Ok(data) = quick_xml::de::from_str::(input) { Ok(Self { data, text: input.to_string(), @@ -66,7 +109,13 @@ impl FromStr for Lang { to: LangType::Xml, colnames: Vec::default(), }) - } else if let Ok(data) = serde_yaml::from_str::(&input) { + } else { + anyhow::bail!("fail to parse input as xml"); + } + } + + pub fn from_as_yaml(input: &str) -> anyhow::Result { + if let Ok(data) = serde_yaml::from_str::(input) { Ok(Self { data, text: input.to_string(), @@ -75,12 +124,10 @@ impl FromStr for Lang { colnames: Vec::default(), }) } else { - anyhow::bail!("not supported") + anyhow::bail!("fail to parse input as yaml"); } } -} -impl Lang { pub fn sort_keys(&mut self) { let json = serde_json::to_string(&self.data).unwrap(); let bdata = serde_json::from_str::(&json).unwrap(); diff --git a/tests-make/se_csv.toml b/tests-make/se_csv.toml index 827939a..2cbe945 100644 --- a/tests-make/se_csv.toml +++ b/tests-make/se_csv.toml @@ -1,8 +1,7 @@ [tests.as-name] script = ''' -alias pq ="./target/debug/pq" -cat samples/ip_addr.json | pq -q "SELECT address, ifname, addr_info.family" -t csv +cat samples/ip_addr.json | ./target/debug/pq -q "SELECT address, ifname, addr_info.family" -t csv ''' tobe = ''' address,ifname,family diff --git a/tests-make/se_toml.toml b/tests-make/se_toml.toml index 83e1eaa..85a7850 100644 --- a/tests-make/se_toml.toml +++ b/tests-make/se_toml.toml @@ -1,8 +1,7 @@ [tests.nonamed-table] script = ''' -alias pq ="./target/debug/pq" -cat< Date: Mon, 7 Jun 2021 20:36:48 +0900 Subject: [PATCH 4/4] Add option: -c --- src/bin/pq.rs | 7 ++++++- src/lib/lang.rs | 3 ++- tests-make/convert.toml | 9 +++++++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/bin/pq.rs b/src/bin/pq.rs index 60a2f59..e6e2bbc 100644 --- a/src/bin/pq.rs +++ b/src/bin/pq.rs @@ -39,6 +39,10 @@ struct Opt { /// sort keys of objects on output. it on works when --to option is json, currently #[structopt(short = "S", long)] sort_keys: bool, + + /// compact instead of pretty-printed output, only when outputting in JSON + #[structopt(short, long)] + compact: bool, } fn main() -> anyhow::Result<()> { @@ -48,6 +52,7 @@ fn main() -> anyhow::Result<()> { from, to, sort_keys, + compact, } = Opt::from_args(); let _ = { let input = if let Some(file) = file_or_stdin { @@ -81,7 +86,7 @@ fn main() -> anyhow::Result<()> { lang.sort_keys(); } - lang.print(); + lang.print(compact); }; Ok(()) diff --git a/src/lib/lang.rs b/src/lib/lang.rs index 0e398f5..7ebd7cb 100644 --- a/src/lib/lang.rs +++ b/src/lib/lang.rs @@ -136,7 +136,7 @@ impl Lang { self.data = data; } - pub fn print(&self) -> anyhow::Result<()> { + pub fn print(&self, compact: bool) -> anyhow::Result<()> { let output = match (&self.to, &self.from == &self.to) { (LangType::Csv, _) => { // To pad missing values with null, serialize them to json, deserialize them with polars, and write them to csv from there. @@ -159,6 +159,7 @@ impl Lang { let s = String::from_utf8(v)?; s } + (LangType::Json, _) if compact => serde_json::to_string(&self.data).unwrap(), (LangType::Json, _) => serde_json::to_string_pretty(&self.data).unwrap(), (_, true) => self.text.to_owned(), (LangType::Toml, _) => { diff --git a/tests-make/convert.toml b/tests-make/convert.toml index 4756b3e..aa297d1 100644 --- a/tests-make/convert.toml +++ b/tests-make/convert.toml @@ -19,6 +19,15 @@ tobe = ''' } ''' +[tests.to-json-compact] +script = ''' +echo $INPUT | ./target/debug/pq -t json -c +''' +tobe = ''' +{"name":"Mew","id":151,"fleeRate":0.1} +''' + + [tests.to-toml] script = ''' echo $INPUT | ./target/debug/pq -t toml