Merge pull request #644 from lognaturel/issue-616

Use Apache Commons CSV for parsing external secondary instances

build.gradle

@@ -33,6 +33,8 @@ dependencies {
     // Be sure to update dependencies in pom.xml to match
     implementation 'joda-time:joda-time:2.10.10'
     implementation 'org.slf4j:slf4j-api:1.7.31'
+    // Upgrade to version higher than 1.4 when Collect minSDK >= 26
+    implementation 'org.apache.commons:commons-csv:1.4'
     compileOnly 'net.sf.kxml:kxml2:2.3.0'
     testImplementation 'ch.qos.logback:logback-classic:1.2.3'
     testImplementation 'junit:junit:4.13.2'

pom.xml

@@ -57,6 +57,11 @@
        <scope>compile</scope>
     </dependency>
     <dependency>
+       <groupId>org.apache.commons</groupId>
+       <artifactId>commons-csv</artifactId>
+       <version>1.4</version>
+    </dependency>
+      <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
       <version>1.7.31</version>

src/main/java/org/javarosa/core/model/instance/CsvExternalInstance.java

@@ -1,38 +1,52 @@
 package org.javarosa.core.model.instance;
 
 import java.io.BufferedReader;
+import java.io.File;
 import java.io.FileReader;
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVParser;
+import org.apache.commons.csv.CSVRecord;
 import org.javarosa.core.model.data.UncastData;
 
 public class CsvExternalInstance {
     public static TreeElement parse(String instanceId, String path) throws IOException {
-        TreeElement root = new TreeElement("root", 0);
+        final TreeElement root = new TreeElement("root", 0);
         root.setInstanceName(instanceId);
 
-        try (BufferedReader br = new BufferedReader(new FileReader(path))) {
-            String csvLine = br.readLine();
+        final CSVFormat csvFormat = CSVFormat.DEFAULT
+            .withDelimiter(getDelimiter(path))
+            .withFirstRecordAsHeader();
+        final CSVParser csvParser = CSVParser.parse(new File(path), StandardCharsets.UTF_8, csvFormat);
+        final String[] fieldNames = csvParser.getHeaderMap().keySet().toArray(new String[0]);
+        int multiplicity = 0;
 
-            if (csvLine != null) {
-                String[] fieldNames = csvLine.split(",");
-                int multiplicity = 0;
+        for (CSVRecord csvRecord : csvParser.getRecords()) {
+            TreeElement item = new TreeElement("item", multiplicity);
 
-                while ((csvLine = br.readLine()) != null) {
-                    TreeElement item = new TreeElement("item", multiplicity);
-                    String[] data = csvLine.split(",");
-                    for (int i = 0; i < fieldNames.length; ++i) {
-                        TreeElement field = new TreeElement(fieldNames[i], 0);
-                        field.setValue(new UncastData(i < data.length ? data[i] : ""));
-
-                        item.addChild(field);
-                    }
-
-                    root.addChild(item);
-                    multiplicity++;
-                }
+            for (int i = 0; i < fieldNames.length; ++i) {
+                TreeElement field = new TreeElement(fieldNames[i], 0);
+                field.setValue(new UncastData(i < csvRecord.size() ? csvRecord.get(i) : ""));
+                item.addChild(field);
             }
+
+            root.addChild(item);
+            multiplicity++;
         }
 
         return root;
     }
+
+    private static char getDelimiter(String path) throws IOException {
+        char delimiter = ',';
+        try (BufferedReader reader = new BufferedReader(new FileReader(path))) {
+            String header = reader.readLine();
+
+            if (header.contains(";")) {
+                delimiter = ';';
+            }
+        }
+        return delimiter;
+    }
 }

src/test/java/org/javarosa/core/model/instance/CsvExternalInstanceTest.java

@@ -0,0 +1,51 @@
+package org.javarosa.core.model.instance;
+
+import static org.javarosa.test.utils.ResourcePathHelper.r;
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import org.junit.Before;
+import org.junit.Test;
+
+public class CsvExternalInstanceTest {
+    private TreeElement commaSeparated;
+    private TreeElement semiColonSeparated;
+
+    @Before
+    public void setUp() throws IOException {
+        commaSeparated = CsvExternalInstance.parse("id", r("external-secondary-comma-complex.csv").toString());
+        semiColonSeparated = CsvExternalInstance.parse("id", r("external-secondary-semicolon-complex.csv").toString());
+    }
+
+    @Test
+    public void heading_has_no_extra_quotes() {
+        assertEquals("label", commaSeparated.getChildAt(0).getChildAt(0).getName());
+        assertEquals("label", semiColonSeparated.getChildAt(0).getChildAt(0).getName());
+    }
+
+    @Test
+    public void value_has_no_extra_quotes() {
+        assertEquals("A", commaSeparated.getChildAt(0).getChildAt(0).getValue().getValue());
+        assertEquals("A", semiColonSeparated.getChildAt(0).getChildAt(0).getValue().getValue());
+    }
+
+    @Test
+    public void quoted_string_with_comma() {
+        assertEquals("121 Main St, NE", commaSeparated.getChildAt(6).getChildAt(0).getValue().getValue());
+        assertEquals("121 Main St, NE", semiColonSeparated.getChildAt(6).getChildAt(0).getValue().getValue());
+    }
+
+    @Test
+    public void quoted_string_with_semicolon() {
+        assertEquals("text; more text", commaSeparated.getChildAt(7).getChildAt(0).getValue().getValue());
+        assertEquals("text; more text", semiColonSeparated.getChildAt(7).getChildAt(0).getValue().getValue());
+    }
+
+    @Test
+    public void missing_fields_replaced_with_spaces() {
+        for (int fieldIndex = 1; fieldIndex < 2; fieldIndex++) {
+            assertEquals("", commaSeparated.getChildAt(5).getChildAt(fieldIndex).getValue().getValue());
+            assertEquals("", semiColonSeparated.getChildAt(5).getChildAt(fieldIndex).getValue().getValue());
+        }
+    }
+}

src/test/resources/org/javarosa/core/model/instance/external-secondary-comma-complex.csv

@@ -0,0 +1,9 @@
+"label","name","first"
+"A","a",
+B,b,
+C,c
+AA,aa,a
+AB,ab,a
+AC
+"121 Main St, NE",main,m
+"text; more text",foo,bar

src/test/resources/org/javarosa/core/model/instance/external-secondary-semicolon-complex.csv

@@ -0,0 +1,9 @@
+"label";"name";"first"
+"A";"a";
+B;b;
+C;c
+AA;aa;a
+AB;ab;a
+AC
+"121 Main St, NE";main;m
+"text; more text";foo;bar