Added VOTable to parquet.

opencadc · Dec 17, 2024 · febce06 · febce06
1 parent a9d5dc2
commit febce06
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 1 deletion.
diff --git a/cadc-dali/src/main/java/ca/nrc/cadc/dali/tables/parquet/ParquetReader.java b/cadc-dali/src/main/java/ca/nrc/cadc/dali/tables/parquet/ParquetReader.java
@@ -37,6 +37,9 @@ public TableShape read(InputStream inputStream) throws IOException {
         MessageType schema = metadata.getFileMetaData().getSchema();
         columnCount = schema.getFieldCount();
 
+        String votable = metadata.getFileMetaData().getKeyValueMetaData().get("votable");
+        log.debug("VOTable: " + votable);
+
         try (org.apache.parquet.hadoop.ParquetReader<GenericRecord> reader = AvroParquetReader.<GenericRecord>builder(inputFile).build()) {
             GenericRecord record;
 

diff --git a/cadc-dali/src/main/java/ca/nrc/cadc/dali/tables/parquet/ParquetWriter.java b/cadc-dali/src/main/java/ca/nrc/cadc/dali/tables/parquet/ParquetWriter.java
@@ -1,16 +1,21 @@
 package ca.nrc.cadc.dali.tables.parquet;
 
+import ca.nrc.cadc.dali.tables.TableData;
 import ca.nrc.cadc.dali.tables.TableWriter;
 import ca.nrc.cadc.dali.tables.votable.VOTableDocument;
 import ca.nrc.cadc.dali.tables.votable.VOTableResource;
+import ca.nrc.cadc.dali.tables.votable.VOTableWriter;
 import ca.nrc.cadc.dali.util.FormatFactory;
 
 import java.io.IOException;
 import java.io.OutputStream;
+import java.io.StringWriter;
 import java.io.Writer;
 
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 
 import org.apache.avro.Schema;
 import org.apache.avro.generic.GenericData;
@@ -65,6 +70,16 @@ public void write(VOTableDocument voTableDocument, OutputStream out, Long maxRec
             Schema schema = DynamicSchemaGenerator.generateSchema(resource.getTable().getFields());
             OutputFile outputFile = outputFileFromStream(out);
 
+            TableData tableData = resource.getTable().getTableData();
+            resource.getTable().setTableData(null);
+
+            StringWriter stringWriter = new StringWriter();
+            VOTableWriter votableWriter = new VOTableWriter();
+            votableWriter.write(voTableDocument, stringWriter, maxRec);
+
+            Map<String, String> customMetaData = new HashMap<>();
+            customMetaData.put("votable", stringWriter.toString());
+
             try (org.apache.parquet.hadoop.ParquetWriter<GenericRecord> writer = AvroParquetWriter.<GenericRecord>builder(outputFile)
                     .withSchema(schema)
                     .withCompressionCodec(CompressionCodecName.SNAPPY)
@@ -74,9 +89,10 @@ public void write(VOTableDocument voTableDocument, OutputStream out, Long maxRec
                     .withWriteMode(ParquetFileWriter.Mode.OVERWRITE)
                     .withValidation(false)
                     .withDictionaryEncoding(false)
+                    .withExtraMetaData(customMetaData)
                     .build()) {
 
-                Iterator<List<Object>> iterator = resource.getTable().getTableData().iterator();
+                Iterator<List<Object>> iterator = tableData.iterator();
                 int recordCount = 1;
 
                 while (iterator.hasNext() && recordCount <= maxRec) {