Add support for FlatGeobuf files (import and export)

* Allow .fgb as import file extension (otherwise just works, because GDAL) * Support .fgb export (format has no support for array columns) * Import and export tests
yma11 · Aug 12, 2021 · 73571ae · 73571ae
1 parent ddae962
commit 73571ae
Show file tree

Hide file tree

Showing 12 changed files with 89 additions and 17 deletions.
diff --git a/ImportExport/QueryExporter.cpp b/ImportExport/QueryExporter.cpp
@@ -32,6 +32,7 @@ std::unique_ptr<QueryExporter> QueryExporter::create(FileType file_type) {
     case FileType::kGeoJSON:
     case FileType::kGeoJSONL:
     case FileType::kShapefile:
+    case FileType::kFlatGeobuf:
       return std::make_unique<QueryExporterGDAL>(file_type);
   }
   CHECK(false);

diff --git a/ImportExport/QueryExporter.h b/ImportExport/QueryExporter.h
@@ -26,7 +26,7 @@ namespace import_export {
 
 class QueryExporter {
  public:
-  enum class FileType { kCSV, kGeoJSON, kGeoJSONL, kShapefile };
+  enum class FileType { kCSV, kGeoJSON, kGeoJSONL, kShapefile, kFlatGeobuf };
   enum class FileCompression { kNone, kGZip, kZip };
   enum class ArrayNullHandling {
     kAbortWithWarning,

diff --git a/ImportExport/QueryExporterGDAL.cpp b/ImportExport/QueryExporterGDAL.cpp
@@ -61,15 +61,17 @@ void QueryExporterGDAL::cleanUp() {
 
 namespace {
 
-static constexpr std::array<const char*, 4> driver_names = {"INVALID",
+static constexpr std::array<const char*, 5> driver_names = {"INVALID",
                                                             "GeoJSON",
                                                             "GeoJSONSeq",
-                                                            "ESRI Shapefile"};
+                                                            "ESRI Shapefile",
+                                                            "FlatGeobuf"};
 
-static constexpr std::array<const char*, 4> file_type_names = {"CSV",
+static constexpr std::array<const char*, 5> file_type_names = {"CSV",
                                                                "GeoJSON",
                                                                "GeoJSONL",
-                                                               "Shapefile"};
+                                                               "Shapefile",
+                                                               "FlatGeobuf"};
 
 static constexpr std::array<const char*, 3> compression_prefix = {"",
                                                                   "/vsigzip/",
@@ -79,14 +81,15 @@ static constexpr std::array<const char*, 3> compression_suffix = {"", ".gz", ".z
 
 // this table is by file type then by compression type
 // @TODO(se) implement more compression options
-static constexpr std::array<std::array<bool, 3>, 4> compression_implemented = {
+static constexpr std::array<std::array<bool, 3>, 5> compression_implemented = {
     {{true, false, false},    // CSV: none
      {true, true, false},     // GeoJSON: on-the-fly GZip only
      {true, true, false},     // GeoJSONL: on-the-fly GZip only
-     {true, false, false}}};  // Shapefile: none
+     {true, false, false},    // Shapefile: none
+     {true, false, false}}};  // FlatGeobuf: none
 
-static std::array<std::unordered_set<std::string>, 4> file_type_valid_extensions = {
-    {{".csv", ".tsv"}, {".geojson", ".json"}, {".geojson", ".json"}, {".shp"}}};
+static std::array<std::unordered_set<std::string>, 5> file_type_valid_extensions = {
+    {{".csv", ".tsv"}, {".geojson", ".json"}, {".geojson", ".json"}, {".shp"}, {".fgb"}}};
 
 OGRFieldType sql_type_info_to_ogr_field_type(const std::string& name,
                                              const SQLTypeInfo& type_info,
@@ -118,7 +121,8 @@ OGRFieldType sql_type_info_to_ogr_field_type(const std::string& name,
     case kINTERVAL_YEAR_MONTH:
       return OFTInteger64;
     case kARRAY:
-      if (file_type != QueryExporter::FileType::kShapefile) {
+      if (file_type != QueryExporter::FileType::kShapefile &&
+          file_type != QueryExporter::FileType::kFlatGeobuf) {
         switch (type_info.get_subtype()) {
           case kBOOLEAN:
           case kTINYINT:

diff --git a/Parser/ParserNode.cpp b/Parser/ParserNode.cpp
@@ -5575,10 +5575,12 @@ void ExportQueryStmt::parseOptions(
           file_type = import_export::QueryExporter::FileType::kGeoJSONL;
         } else if (file_type_str == "shapefile") {
           file_type = import_export::QueryExporter::FileType::kShapefile;
+        } else if (file_type_str == "flatgeobuf") {
+          file_type = import_export::QueryExporter::FileType::kFlatGeobuf;
         } else {
           throw std::runtime_error(
-              "File Type option must be 'CSV', 'GeoJSON', 'GeoJSONL' or "
-              "'Shapefile'");
+              "File Type option must be 'CSV', 'GeoJSON', 'GeoJSONL', "
+              "'Shapefile', or 'FlatGeobuf'");
         }
       } else if (boost::iequals(*p->get_name(), "layer_name")) {
         const StringLiteral* str_literal =

diff --git a/Tests/Export/QueryExport/datafiles/query_export_test_fgb_linestring.fgb b/Tests/Export/QueryExport/datafiles/query_export_test_fgb_linestring.fgb
diff --git a/Tests/Export/QueryExport/datafiles/query_export_test_fgb_multipolygon.fgb b/Tests/Export/QueryExport/datafiles/query_export_test_fgb_multipolygon.fgb
diff --git a/Tests/Export/QueryExport/datafiles/query_export_test_fgb_point.fgb b/Tests/Export/QueryExport/datafiles/query_export_test_fgb_point.fgb
diff --git a/Tests/Export/QueryExport/datafiles/query_export_test_fgb_polygon.fgb b/Tests/Export/QueryExport/datafiles/query_export_test_fgb_polygon.fgb
diff --git a/Tests/Import/datafiles/geospatial_mpoly/geospatial_mpoly.fgb b/Tests/Import/datafiles/geospatial_mpoly/geospatial_mpoly.fgb
diff --git a/Tests/Import/datafiles/geospatial_point/geospatial_point.fgb b/Tests/Import/datafiles/geospatial_point/geospatial_point.fgb
diff --git a/Tests/ImportExportTest.cpp b/Tests/ImportExportTest.cpp
@@ -1627,6 +1627,22 @@ TEST_F(ImportTestGDAL, KML_Simple) {
   }
 }
 
+TEST_F(ImportTestGDAL, FlatGeobuf_Point_Import) {
+  SKIP_ALL_ON_AGGREGATOR();
+  const auto file_path = boost::filesystem::path("geospatial_point/geospatial_point.fgb");
+  import_test_geofile_importer(file_path.string(), "geospatial", false, true, false);
+  check_geo_gdal_point_import();
+  check_geo_num_rows("omnisci_geo, trip", 10);
+}
+
+TEST_F(ImportTestGDAL, FlatGeobuf_MultiPolygon_Import) {
+  SKIP_ALL_ON_AGGREGATOR();
+  const auto file_path = boost::filesystem::path("geospatial_mpoly/geospatial_mpoly.fgb");
+  import_test_geofile_importer(file_path.string(), "geospatial", false, true, false);
+  check_geo_gdal_poly_or_mpoly_import(false, false);  // poly, not exploded
+  check_geo_num_rows("omnisci_geo, trip", 10);
+}
+
 #ifdef HAVE_AWS_S3
 // s3 compressed (non-parquet) test cases
 TEST_F(ImportTest, S3_One_csv_file) {
@@ -2010,17 +2026,17 @@ class ExportTest : public ::testing::Test {
     }
 
     // select a comparable value from the first row
-    // hopefully immune to any re-ordering due to export query non-determinism
+    // tolerate any re-ordering due to export query non-determinism
     // scope this block so that the ResultSet is destroyed before the table is dropped
-    // @TODO(se) make this mo better
-    // push scope to ensure ResultSet deletion before table drop
     {
       auto rows =
           run_query("SELECT col_big FROM query_export_test_reimport WHERE rowid=0");
       rows->setGeoReturnType(ResultSet::GeoReturnType::GeoTargetValue);
       auto crt_row = rows->getNextRow(true, true);
       const auto col_big = v<int64_t>(crt_row[0]);
-      ASSERT_EQ(20395569495LL, col_big);
+      constexpr std::array<int64_t, 5> values{
+          84212876526LL, 53000912292LL, 31851544292LL, 31334726270LL, 20395569495LL};
+      ASSERT_NE(std::find(values.begin(), values.end(), col_big), values.end());
     }
 
     // drop the table
@@ -2608,6 +2624,55 @@ TEST_F(ExportTest, Shapefile_Zip_Unimplemented) {
   RUN_TEST_ON_ALL_GEO_TYPES();
 }
 
+TEST_F(ExportTest, FlatGeobuf) {
+  SKIP_ALL_ON_AGGREGATOR();
+  doCreateAndImport();
+  auto run_test = [&](const std::string& geo_type) {
+    std::string exp_file = "query_export_test_fgb_" + geo_type + ".fgb";
+    ASSERT_NO_THROW(
+        doExport(exp_file, "FlatGeobuf", "", geo_type, NO_ARRAYS, DEFAULT_SRID));
+    std::string layer_name = "query_export_test_fgb_" + geo_type;
+    ASSERT_NO_THROW(doCompareWithOGRInfo(exp_file, layer_name, COMPARE_EXPLICIT));
+    doImportAgainAndCompare(exp_file, "FlatGeobuf", geo_type, NO_ARRAYS);
+    removeExportedFile(exp_file);
+  };
+  RUN_TEST_ON_ALL_GEO_TYPES();
+}
+
+TEST_F(ExportTest, FlatGeobuf_Overwrite) {
+  SKIP_ALL_ON_AGGREGATOR();
+  doCreateAndImport();
+  auto run_test = [&](const std::string& geo_type) {
+    std::string exp_file = "query_export_test_fgb_" + geo_type + ".fgb";
+    ASSERT_NO_THROW(
+        doExport(exp_file, "FlatGeobuf", "", geo_type, NO_ARRAYS, DEFAULT_SRID));
+    ASSERT_NO_THROW(
+        doExport(exp_file, "FlatGeobuf", "", geo_type, NO_ARRAYS, DEFAULT_SRID));
+    removeExportedFile(exp_file);
+  };
+  RUN_TEST_ON_ALL_GEO_TYPES();
+}
+
+TEST_F(ExportTest, FlatGeobuf_InvalidName) {
+  SKIP_ALL_ON_AGGREGATOR();
+  doCreateAndImport();
+  std::string geo_type = "point";
+  std::string exp_file = "query_export_test_fgb_" + geo_type + ".jpg";
+  EXPECT_THROW(doExport(exp_file, "FlatGeobuf", "", geo_type, NO_ARRAYS, DEFAULT_SRID),
+               std::runtime_error);
+}
+
+TEST_F(ExportTest, FlatGeobuf_Invalid_SRID) {
+  SKIP_ALL_ON_AGGREGATOR();
+  doCreateAndImport();
+  auto run_test = [&](const std::string& geo_type) {
+    std::string exp_file = "query_export_test_geojsonl_" + geo_type + ".fgb";
+    EXPECT_THROW(doExport(exp_file, "FlatGeobuf", "", geo_type, NO_ARRAYS, INVALID_SRID),
+                 std::runtime_error);
+  };
+  RUN_TEST_ON_ALL_GEO_TYPES();
+}
+
 TEST_F(ExportTest, Array_Null_Handling_Default) {
   SKIP_ALL_ON_AGGREGATOR();
   EXPECT_THROW(doTestArrayNullHandling(

diff --git a/ThriftHandler/DBHandler.cpp b/ThriftHandler/DBHandler.cpp
@@ -3807,7 +3807,7 @@ bool is_a_supported_geo_file(const std::string& path, bool include_gz) {
   if (boost::iends_with(path, ".shp") || boost::iends_with(path, ".geojson") ||
       boost::iends_with(path, ".json") || boost::iends_with(path, ".kml") ||
       boost::iends_with(path, ".kmz") || boost::iends_with(path, ".gdb") ||
-      boost::iends_with(path, ".gdb.zip")) {
+      boost::iends_with(path, ".gdb.zip") || boost::iends_with(path, ".fgb")) {
     return true;
   }
   return false;