Skip to content

Commit

Permalink
Fix format
Browse files Browse the repository at this point in the history
  • Loading branch information
@NicoLaval
NicoLaval committed Jul 1, 2024
1 parent 78bfd7d commit 42935cb
Show file tree
Hide file tree
Showing 3 changed files with 72 additions and 66 deletions.
16 changes: 7 additions & 9 deletions src/main/java/fr/insee/trevas/jupyter/DatasetUtils.java
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
/* (C)2024 */
package fr.insee.trevas.jupyter;

import fr.insee.vtl.model.Dataset;
Expand Down Expand Up @@ -41,10 +42,10 @@ public static String datasetToDisplay(Dataset dataset) {
+ " crossorigin=\"anonymous\"></script>");
b.append(
"<link rel=\"stylesheet\" type=\"text/css\""
+ " href=\"https://cdn.datatables.net/1.12.1/css/jquery.dataTables.css\">\n"
+ " \n"
+ "<script type=\"text/javascript\" charset=\"utf8\""
+ " src=\"https://cdn.datatables.net/1.12.1/js/jquery.dataTables.js\"></script>\n");
+ " href=\"https://cdn.datatables.net/1.12.1/css/jquery.dataTables.css\">\n"
+ " \n"
+ "<script type=\"text/javascript\" charset=\"utf8\""
+ " src=\"https://cdn.datatables.net/1.12.1/js/jquery.dataTables.js\"></script>\n");
b.append(
"<script type=\"text/javascript\">"
+ "$(document).ready( function () {\n"
Expand All @@ -71,12 +72,9 @@ public static String datasetMetadataToDisplay(Dataset dataset) {
.append(value.getType().getSimpleName());
String valuedomain = value.getValuedomain();
if (null != valuedomain) {
sb.append(" - ")
.append(valuedomain);
sb.append(" - ").append(valuedomain);
}
sb.append(")")
.append("</li>")
.append("\n");
sb.append(")").append("</li>").append("\n");
});
sb.append("</ul>");
return sb.toString();
Expand Down
82 changes: 45 additions & 37 deletions src/main/java/fr/insee/trevas/jupyter/VtlKernel.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,6 @@
import io.github.spencerpark.jupyter.kernel.display.DisplayData;
import io.sdmx.api.io.ReadableDataLocation;
import io.sdmx.utils.core.io.ReadableDataLocationTmp;
import org.apache.spark.sql.SparkSession;

import javax.script.ScriptContext;
import javax.script.ScriptEngineFactory;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
Expand All @@ -30,6 +26,9 @@
import java.util.Map;
import java.util.logging.Level;
import java.util.stream.Collectors;
import javax.script.ScriptContext;
import javax.script.ScriptEngineFactory;
import org.apache.spark.sql.SparkSession;

public class VtlKernel extends BaseKernel {

Expand Down Expand Up @@ -147,8 +146,7 @@ public static Dataset loadSDMXSource(String path, String id, String dataPath) {
.option("delimiter", ";")
.option("quote", "\"")
.csv(dataPath),
structure
);
structure);
}

public static void runSDMXPreview(String path) {
Expand All @@ -163,41 +161,50 @@ public static void runSDMXPreview(String path) {

var result = new StringBuilder();

results.forEach((k, v) -> {
result.append("<h2>").append(k).append("</h2>")
.append(DatasetUtils.datasetMetadataToDisplay(v));
});
results.forEach(
(k, v) -> {
result.append("<h2>")
.append(k)
.append("</h2>")
.append(DatasetUtils.datasetMetadataToDisplay(v));
});

displayData.putText(result.toString());
}

public static void runSDMX(String path, Map<String, String> data) {

Map<String, Dataset> inputs = data.entrySet().stream().collect(Collectors.toMap(
Map.Entry::getKey,
e -> {
Structured.DataStructure structure = TrevasSDMXUtils.buildStructureFromSDMX3(path, e.getKey());
return new SparkDataset(
spark.read()
.option("header", "true")
.option("delimiter", ";")
.option("quote", "\"")
.csv(e.getValue()),
structure
);
}
));
Map<String, Dataset> inputs =
data.entrySet().stream()
.collect(
Collectors.toMap(
Map.Entry::getKey,
e -> {
Structured.DataStructure structure =
TrevasSDMXUtils.buildStructureFromSDMX3(
path, e.getKey());
return new SparkDataset(
spark.read()
.option("header", "true")
.option("delimiter", ";")
.option("quote", "\"")
.csv(e.getValue()),
structure);
}));

ReadableDataLocation rdl = new ReadableDataLocationTmp(path);
SDMXVTLWorkflow sdmxVtlWorkflow = new SDMXVTLWorkflow(engine, rdl, inputs);
Map<String, PersistentDataset> results = sdmxVtlWorkflow.run();

var result = new StringBuilder();

results.forEach((k, v) -> {
result.append("<h2>").append(k).append("</h2>")
.append(DatasetUtils.datasetToDisplay(v));
});
results.forEach(
(k, v) -> {
result.append("<h2>")
.append(k)
.append("</h2>")
.append(DatasetUtils.datasetToDisplay(v));
});

displayData.putText(result.toString());
}
Expand Down Expand Up @@ -263,19 +270,20 @@ private void registerGlobalMethods() throws NoSuchMethodException {

// SDMX
this.engine.registerGlobalMethod(
"loadSDMXSource", VtlKernel.class.getMethod("loadSDMXSource", String.class, String.class)
);
"loadSDMXSource",
VtlKernel.class.getMethod("loadSDMXSource", String.class, String.class));
this.engine.registerGlobalMethod(
"loadSDMXSource", VtlKernel.class.getMethod("loadSDMXSource", String.class, String.class, String.class)
);
"loadSDMXSource",
VtlKernel.class.getMethod(
"loadSDMXSource", String.class, String.class, String.class));
this.engine.registerGlobalMethod(
"runSDMXPreview", VtlKernel.class.getMethod("runSDMXPreview", String.class)
);
//this.engine.registerGlobalMethod(
"runSDMXPreview", VtlKernel.class.getMethod("runSDMXPreview", String.class));
// this.engine.registerGlobalMethod(
// "runSDMX", VtlKernel.class.getMethod("runSDMX", String.class, Map<String, String>)
//);
// );
this.engine.registerGlobalMethod(
"getTransformationsVTL", VtlKernel.class.getMethod("getTransformationsVTL", String.class));
"getTransformationsVTL",
VtlKernel.class.getMethod("getTransformationsVTL", String.class));
this.engine.registerGlobalMethod(
"getRulesetsVTL", VtlKernel.class.getMethod("getRulesetsVTL", String.class));
}
Expand Down
40 changes: 20 additions & 20 deletions src/test/java/fr/insee/trevas/jupyter/SDMXTest.java
View file
Original file line number Diff line number Diff line change
@@ -1,60 +1,60 @@
/* (C)2024 */
package fr.insee.trevas.jupyter;

import fr.insee.vtl.model.Dataset;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;

import fr.insee.vtl.model.Dataset;
import java.util.Map;

import static org.assertj.core.api.Assertions.assertThat;
import org.junit.jupiter.api.Test;

public class SDMXTest {

@Test
public void testLoadSDMXSource() {
Dataset ds = VtlKernel.loadSDMXSource(
"src/test/resources/sdmx/DSD_BPE_CENSUS.xml",
"BPE_DETAIL_VTL");
Dataset ds =
VtlKernel.loadSDMXSource(
"src/test/resources/sdmx/DSD_BPE_CENSUS.xml", "BPE_DETAIL_VTL");
assertThat(ds.getDataStructure().size()).isEqualTo(6);
}

@Test
public void testLoadSDMXSourceWithData() throws Exception {
new VtlKernel();
Dataset ds = VtlKernel.loadSDMXSource(
"src/test/resources/sdmx/DSD_BPE_CENSUS.xml",
"BPE_DETAIL_VTL",
"src/test/resources/sdmx/BPE_DETAIL_SAMPLE.csv");
Dataset ds =
VtlKernel.loadSDMXSource(
"src/test/resources/sdmx/DSD_BPE_CENSUS.xml",
"BPE_DETAIL_VTL",
"src/test/resources/sdmx/BPE_DETAIL_SAMPLE.csv");
assertThat(ds.getDataStructure().size()).isEqualTo(6);
}

@Test
public void testRunSDMXPreview() throws Exception {
new VtlKernel();
VtlKernel.runSDMXPreview(
"src/test/resources/sdmx/DSD_BPE_CENSUS.xml");
VtlKernel.runSDMXPreview("src/test/resources/sdmx/DSD_BPE_CENSUS.xml");
}

@Test
public void testRunSDMX() throws Exception {
new VtlKernel();
VtlKernel.runSDMX(
"src/test/resources/sdmx/DSD_BPE_CENSUS.xml",
Map.of("BPE_DETAIL_VTL", "src/test/resources/sdmx/BPE_DETAIL_SAMPLE.csv",
"LEGAL_POP", "src/test/resources/sdmx/LEGAL_POP_NUTS3.csv"
));
Map.of(
"BPE_DETAIL_VTL",
"src/test/resources/sdmx/BPE_DETAIL_SAMPLE.csv",
"LEGAL_POP",
"src/test/resources/sdmx/LEGAL_POP_NUTS3.csv"));
}

@Test
public void testGetTransformationsVTL() throws Exception {
new VtlKernel();
VtlKernel.getTransformationsVTL(
"src/test/resources/sdmx/DSD_BPE_CENSUS.xml");
VtlKernel.getTransformationsVTL("src/test/resources/sdmx/DSD_BPE_CENSUS.xml");
}

@Test
public void testGetRulesetsVTL() throws Exception {
new VtlKernel();
VtlKernel.getRulesetsVTL(
"src/test/resources/sdmx/DSD_BPE_CENSUS.xml");
VtlKernel.getRulesetsVTL("src/test/resources/sdmx/DSD_BPE_CENSUS.xml");
}
}

0 comments on commit 42935cb

Please sign in to comment.