csv2json arguments for changing default delimiter, quote & escape cha…

…racters

CHANGELOG.md

@@ -1,6 +1,10 @@
 Change Log of convert2json utilities
 ====================================
 
+Version 0.4.0 / 2023-07-02
+--------------------------
+- added csv2json arguments for changing default delimiter, quote & escape characters
+
 Version 0.3.0 / 2023-07-02
 --------------------------
 - added support for files in arguments to *2json tools

Cargo.toml

@@ -2,7 +2,7 @@
 name = "convert2json"
 description = "CLI utilities to convert CSV, TOML, XML & YAML into JSON and for use with jq."
 authors = ["Simon Rupf <[email protected]>"]
-version = "0.3.0"
+version = "0.4.0"
 edition = "2021"
 license = "MIT"
 repository = "https://github.com/simonrupf/convert2json"
@@ -29,6 +29,7 @@ yaml2json = []
 yq = []
 
 [dependencies]
+argh = "0.1.10"
 csv = "1.2.2"
 is-terminal = "0.4.7"
 serde = { version = "1.0.164", features = ["serde_derive"] }

LICENSE.md

@@ -6,6 +6,7 @@ This document lists the licenses of the projects used in convert2json.
 
 - [MIT License](#MIT) (7)
 - [Apache License 2.0](#Apache-2.0) (4)
+- [BSD 3-Clause "New" or "Revised" License](#BSD-3-Clause) (1)
 - [Unicode License Agreement - Data Files and Software (2016)](#Unicode-DFS-2016) (1)
 
 ## Full license texts
@@ -439,6 +440,7 @@ END OF TERMS AND CONDITIONS
 - [log](https://github.com/rust-lang/log) 0.4.19
 - [rustix](https://github.com/bytecodealliance/rustix) 0.37.20
 - [serde_spanned](https://github.com/toml-rs/toml) 0.6.3
+- [syn](https://github.com/dtolnay/syn) 1.0.109
 - [toml](https://github.com/toml-rs/toml) 0.7.5
 - [toml_datetime](https://github.com/toml-rs/toml) 0.6.3
 - [toml_edit](https://github.com/toml-rs/toml) 0.19.11
@@ -728,6 +730,44 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 
+```
+### <a id="BSD-3-Clause"></a>BSD 3-Clause &quot;New&quot; or &quot;Revised&quot; License
+
+#### Used by
+
+- [argh](https://github.com/google/argh) 0.1.10
+- [argh_derive](https://github.com/google/argh) 0.1.10
+- [argh_shared](https://github.com/google/argh) 0.1.10
+
+```
+Copyright 2019 The Fuchsia Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+&quot;AS IS&quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
 ```
 ### <a id="MIT"></a>MIT License
 

README.md

@@ -36,7 +36,7 @@ Alternatives:
 * [Python 🐍](https://pypi.org/search/?q=yaml2json)
 
 To Do:
-- [ ] csv arguments for delimiter, flexible fields, quote, terminator, escape characters
+- [ ] in jq arguments, ignore filenames if preceeded by certain flags (i.e. --from-file)
 
 Feature Matrix
 --------------

about.toml

@@ -1,5 +1,6 @@
 accepted = [
     "Apache-2.0",
+    "BSD-3-Clause",
     "MIT",
     "Unicode-DFS-2016",
 ]

src/bin/cq.rs

@@ -1,13 +1,14 @@
-use convert2json::csv::{append, CsvMap};
+use convert2json::csv::{CsvMap, CsvReader};
 use convert2json::jq::{parse_args, readers, Jq};
 
 #[cfg(feature = "cq")]
 fn main() {
     let (arguments, files) = parse_args();
     let mut jq = Jq::new(&arguments);
+    let mut csv: CsvReader = Default::default();
     let mut results: Vec<CsvMap> = vec![];
     for reader in readers(&files) {
-        append(&mut results, reader);
+        csv.append(&mut results, reader);
     }
     jq.write(&results);
 }

src/bin/csv2json.rs

@@ -1,11 +1,12 @@
-use convert2json::csv::{append, CsvMap};
+use convert2json::csv::{CsvMap, CsvReader};
 use convert2json::json::{parse_args, stdout_writer};
 
 #[cfg(feature = "csv2json")]
 fn main() {
+    let mut csv = CsvReader::new();
     let mut results: Vec<CsvMap> = vec![];
     for reader in parse_args() {
-        append(&mut results, reader);
+        csv.append(&mut results, reader);
     }
     stdout_writer(&results);
 }

src/csv.rs

@@ -1,7 +1,8 @@
 extern crate csv;
 
 use super::{exit, Error};
-use csv::Reader;
+use argh::FromArgs;
+use csv::ReaderBuilder;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::io::Read;
@@ -12,15 +13,65 @@ pub struct CsvMap {
     values: HashMap<String, String>,
 }
 
-pub fn append<R: Read>(results: &mut Vec<CsvMap>, reader: R) {
-    for row in Reader::from_reader(reader).deserialize() {
-        let record: CsvMap = match row {
-            Ok(values) => values,
-            Err(e) => {
-                eprintln!("Error parsing input: {e}");
-                exit(Error::InputParsing as i32);
-            }
-        };
-        results.push(record);
+#[derive(FromArgs)]
+/// Reads CSV from files or standard input and converts this to JSON, emitted on
+/// standard output. Any errors are reported to standard error and result in a
+/// non-zero exit code.
+struct CsvParameters {
+    /// field delimiter to use when parsing CSV, defaults to: , (comma)
+    #[argh(option, short = 'd')]
+    delimiter: Option<char>,
+
+    /// quote character to use when parsing CSV, defaults to: " (double quote)
+    #[argh(option, short = 'q')]
+    quote: Option<char>,
+
+    /// escape character to use when parsing CSV, to escape quote characters
+    /// within a field. By default, quotes get escaped by doubling them.
+    #[argh(option, short = 'E')]
+    escape: Option<char>,
+}
+
+pub struct CsvReader {
+    read: ReaderBuilder,
+}
+
+impl CsvReader {
+    pub fn new() -> Self {
+        let arguments: CsvParameters = argh::from_env();
+        let mut read = ReaderBuilder::new();
+        read.flexible(true);
+        if let Some(delimiter) = arguments.delimiter {
+            read.delimiter(delimiter as u8);
+        }
+        if let Some(quote) = arguments.quote {
+            read.quote(quote as u8);
+        }
+        if let Some(escape) = arguments.escape {
+            // note that setting this to None would disable escape sequences entirely
+            read.escape(Some(escape as u8)).double_quote(false);
+        }
+        Self { read }
+    }
+
+    pub fn append<R: Read>(&mut self, results: &mut Vec<CsvMap>, reader: R) {
+        for row in self.read.from_reader(reader).deserialize() {
+            let record: CsvMap = match row {
+                Ok(values) => values,
+                Err(e) => {
+                    eprintln!("Error parsing input: {e}");
+                    exit(Error::InputParsing as i32);
+                }
+            };
+            results.push(record);
+        }
+    }
+}
+
+impl Default for CsvReader {
+    fn default() -> Self {
+        let mut read = ReaderBuilder::new();
+        read.flexible(true);
+        Self { read }
     }
 }