Merge pull request #91 from InseeFrLab/0.5.0

v0.5.0
InseeFrLab · Feb 12, 2024 · d3170fe · d3170fe
2 parents 0ffb93b + 84cefd3
commit d3170fe
Show file tree

Hide file tree

Showing 4 changed files with 28 additions and 14 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -14,9 +14,10 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Set up JDK 11
-        uses: actions/setup-java@v1
+        uses: actions/setup-java@v4
         with:
-          java-version: 11
+          java-version: 17
+          distribution: "adopt"
       - name: Set current version
         run: mvn -B versions:set -DnewVersion=${GITHUB_REF##*/} -DprocessAllModules -DgenerateBackupPoms=false
       - uses: s4u/[email protected]

diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM inseefrlab/onyxia-jupyter-pyspark:py3.10.9-spark3.3.1
+FROM --platform=linux/amd64 inseefrlab/onyxia-jupyter-pyspark:py3.11.6-spark3.5.0
 
 # Allows the kernel to load the Spark and Hadoop config.
 ENV CLASSPATH_PREFIX "/opt/hadoop/etc/hadoop:/opt/spark/conf"
@@ -11,6 +11,10 @@ COPY target/appassembler/repo/fr/insee/trevas/vtl-model/*/vtl-model-*.jar /vtl-m
 COPY target/appassembler/repo/fr/insee/trevas/vtl-engine/*/vtl-engine-*.jar /vtl-engine.jar
 COPY target/appassembler/repo/fr/insee/trevas/vtl-parser/*/vtl-parser-*.jar /vtl-parser.jar
 
-RUN mamba install -y -c conda-forge "elyra[all]"
+USER root
+
+RUN pip3 install --upgrade elyra-pipeline-editor-extension
+
+USER 1000
 
 CMD ["jupyter", "lab", "--no-browser", "--ip", "0.0.0.0"]
diff --git a/pom.xml b/pom.xml
@@ -8,7 +8,7 @@
     <description>Jupyter notebook providing VTL support through Trevas engine</description>
     <artifactId>trevas-jupyter</artifactId>
     <groupId>fr.insee</groupId>
-    <version>0.4.2</version>
+    <version>0.5.0</version>
 
     <licenses>
         <license>
@@ -18,9 +18,9 @@
     </licenses>
 
     <properties>
-        <jdk.version>11</jdk.version>
+        <java.version>17</java.version>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-        <trevas.version>1.1.1</trevas.version>
+        <trevas.version>1.3.0</trevas.version>
     </properties>
 
     <repositories>
@@ -98,7 +98,7 @@
         <dependency>
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-kubernetes_2.12</artifactId>
-            <version>3.3.1</version>
+            <version>3.5.0</version>
         </dependency>
 
         <dependency>
@@ -153,6 +153,7 @@
                     </execution>
                 </executions>
                 <configuration>
+                    <extraJvmArguments>--add-opens=java.base/sun.nio.ch=ALL-UNNAMED</extraJvmArguments>
                     <programs>
                         <program>
                             <mainClass>fr.insee.trevas.jupyter.VtlKernel</mainClass>
@@ -184,10 +185,10 @@
                 <artifactId>maven-compiler-plugin</artifactId>
                 <version>3.12.1</version>
                 <configuration>
-                    <release>${jdk.version}</release>
-                    <testRelease>${jdk.version}</testRelease>
-                    <target>${jdk.version}</target>
-                    <source>${jdk.version}</source>
+                    <release>${java.version}</release>
+                    <testRelease>${java.version}</testRelease>
+                    <target>${java.version}</target>
+                    <source>${java.version}</source>
                 </configuration>
             </plugin>
         </plugins>

diff --git a/src/main/java/fr/insee/trevas/jupyter/VtlKernel.java b/src/main/java/fr/insee/trevas/jupyter/VtlKernel.java
@@ -2,6 +2,7 @@
 
 import fr.insee.vtl.engine.VtlScriptEngine;
 import fr.insee.vtl.model.Dataset;
+import fr.insee.vtl.model.PersistentDataset;
 import fr.insee.vtl.model.Structured;
 import fr.insee.vtl.spark.SparkDataset;
 import io.github.spencerpark.jupyter.channels.JupyterConnection;
@@ -58,9 +59,16 @@ private static Map<String, Dataset.Role> getRoleMap(fr.insee.vtl.model.Dataset d
     private static SparkDataset asSparkDataset(Dataset dataset) {
         if (dataset instanceof SparkDataset) {
             return (SparkDataset) dataset;
-        } else {
-            return new SparkDataset(dataset, getRoleMap(dataset), spark);
         }
+        if (dataset instanceof PersistentDataset) {
+            fr.insee.vtl.model.Dataset ds = ((PersistentDataset) dataset).getDelegate();
+            if (ds instanceof SparkDataset) {
+                return (SparkDataset) ds;
+            } else {
+                return new SparkDataset(ds, getRoleMap(dataset), spark);
+            }
+        }
+        throw new IllegalArgumentException("Unknow dataset type");
     }
 
     public static SparkDataset loadParquet(String path) throws Exception {