Init loadCSV improvements

pom.xml

@@ -251,6 +251,14 @@
                     </execution>
                 </executions>
             </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <version>3.1.2</version>
+                <configuration>
+                    <argLine>--add-exports java.base/sun.nio.ch=ALL-UNNAMED</argLine>
+                </configuration>
+            </plugin>
         </plugins>
     </build>
 </project>

src/main/java/fr/insee/trevas/jupyter/SparkUtils.java

@@ -3,16 +3,18 @@
 
 import fr.insee.vtl.engine.VtlScriptEngine;
 import fr.insee.vtl.spark.SparkDataset;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import javax.script.ScriptEngine;
-import javax.script.ScriptEngineManager;
 import org.apache.spark.SparkConf;
 import org.apache.spark.sql.Dataset;
 import org.apache.spark.sql.Row;
 import org.apache.spark.sql.SaveMode;
 import org.apache.spark.sql.SparkSession;
 
+import javax.script.ScriptEngine;
+import javax.script.ScriptEngineManager;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Map;
+
 public class SparkUtils {
 
     public static VtlScriptEngine buildSparkEngine(SparkSession spark) {
@@ -70,8 +72,15 @@ public static SparkDataset readSasDataset(SparkSession spark, String path) throw
 
     public static SparkDataset readCSVDataset(SparkSession spark, String path) throws Exception {
         Dataset<Row> dataset;
+
+        String url = Utils.getURL(path);
+        Map<String, String> options = Utils.getQueryMap(path);
+
         try {
-            dataset = spark.read().option("sep", ";").option("header", "true").csv(path);
+            dataset = spark.read().option("sep", ";")
+                    .option("header", "true")
+                    .options(options)
+                    .csv(url);
         } catch (Exception e) {
             throw new Exception(e);
         }

src/main/java/fr/insee/trevas/jupyter/Utils.java

@@ -0,0 +1,26 @@
+package fr.insee.trevas.jupyter;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class Utils {
+
+    public static String getURL(String path) {
+        return path.split("\\?")[0];
+    }
+
+    public static Map<String, String> getQueryMap(String path) {
+        if (path == null || !path.contains("\\?")) {
+            return Map.of();
+        }
+        String[] params = path.split("\\?")[1].split("&");
+        Map<String, String> map = new HashMap<>();
+
+        for (String param : params) {
+            String name = param.split("=")[0];
+            String value = param.split("=")[1];
+            map.put(name, value);
+        }
+        return map;
+    }
+}

src/test/java/CSVTest.java

@@ -0,0 +1,28 @@
+import fr.insee.trevas.jupyter.SparkUtils;
+import fr.insee.vtl.model.Dataset;
+import org.apache.spark.sql.SparkSession;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class CSVTest {
+
+    private SparkSession spark;
+
+    @BeforeEach
+    public void setUp() {
+        spark = SparkSession.builder()
+                .appName("test")
+                .master("local")
+                .getOrCreate();
+    }
+
+    @Test
+    public void readCSVDatasetTest() throws Exception {
+        Dataset ds1 = SparkUtils.readCSVDataset(spark, "src/test/resources/ds1.csv");
+        assertThat(ds1.getDataPoints().get(1).get("name")).isEqualTo("B");
+        Dataset ds2 = SparkUtils.readCSVDataset(spark, "src/test/resources/ds2.csv?sep=\"|\"&delimiter=\"\"");
+        assertThat(ds2.getDataPoints().get(1).get("name")).isEqualTo("B");
+    }
+}

src/test/resources/ds1.csv

@@ -0,0 +1,6 @@
+"id";"name"
+"1";"A"
+"2";"B"
+"3";"C"
+"4";"D"
+"5";"E"

src/test/resources/ds2.csv

@@ -0,0 +1,6 @@
+id|name
+6|F
+7|G
+8|H
+9|I
+10|J