diff --git a/ImportExport/QueryExporter.cpp b/ImportExport/QueryExporter.cpp index 322d0ae5f8..e3d29e5786 100644 --- a/ImportExport/QueryExporter.cpp +++ b/ImportExport/QueryExporter.cpp @@ -32,6 +32,7 @@ std::unique_ptr QueryExporter::create(FileType file_type) { case FileType::kGeoJSON: case FileType::kGeoJSONL: case FileType::kShapefile: + case FileType::kFlatGeobuf: return std::make_unique(file_type); } CHECK(false); diff --git a/ImportExport/QueryExporter.h b/ImportExport/QueryExporter.h index bdbcdb1e8e..ac3b078dcf 100644 --- a/ImportExport/QueryExporter.h +++ b/ImportExport/QueryExporter.h @@ -26,7 +26,7 @@ namespace import_export { class QueryExporter { public: - enum class FileType { kCSV, kGeoJSON, kGeoJSONL, kShapefile }; + enum class FileType { kCSV, kGeoJSON, kGeoJSONL, kShapefile, kFlatGeobuf }; enum class FileCompression { kNone, kGZip, kZip }; enum class ArrayNullHandling { kAbortWithWarning, diff --git a/ImportExport/QueryExporterGDAL.cpp b/ImportExport/QueryExporterGDAL.cpp index 86ec873a78..1d3b34a1fe 100644 --- a/ImportExport/QueryExporterGDAL.cpp +++ b/ImportExport/QueryExporterGDAL.cpp @@ -61,15 +61,17 @@ void QueryExporterGDAL::cleanUp() { namespace { -static constexpr std::array driver_names = {"INVALID", +static constexpr std::array driver_names = {"INVALID", "GeoJSON", "GeoJSONSeq", - "ESRI Shapefile"}; + "ESRI Shapefile", + "FlatGeobuf"}; -static constexpr std::array file_type_names = {"CSV", +static constexpr std::array file_type_names = {"CSV", "GeoJSON", "GeoJSONL", - "Shapefile"}; + "Shapefile", + "FlatGeobuf"}; static constexpr std::array compression_prefix = {"", "/vsigzip/", @@ -79,14 +81,15 @@ static constexpr std::array compression_suffix = {"", ".gz", ".z // this table is by file type then by compression type // @TODO(se) implement more compression options -static constexpr std::array, 4> compression_implemented = { +static constexpr std::array, 5> compression_implemented = { {{true, false, false}, // CSV: none {true, true, false}, // GeoJSON: on-the-fly GZip only {true, true, false}, // GeoJSONL: on-the-fly GZip only - {true, false, false}}}; // Shapefile: none + {true, false, false}, // Shapefile: none + {true, false, false}}}; // FlatGeobuf: none -static std::array, 4> file_type_valid_extensions = { - {{".csv", ".tsv"}, {".geojson", ".json"}, {".geojson", ".json"}, {".shp"}}}; +static std::array, 5> file_type_valid_extensions = { + {{".csv", ".tsv"}, {".geojson", ".json"}, {".geojson", ".json"}, {".shp"}, {".fgb"}}}; OGRFieldType sql_type_info_to_ogr_field_type(const std::string& name, const SQLTypeInfo& type_info, @@ -118,7 +121,8 @@ OGRFieldType sql_type_info_to_ogr_field_type(const std::string& name, case kINTERVAL_YEAR_MONTH: return OFTInteger64; case kARRAY: - if (file_type != QueryExporter::FileType::kShapefile) { + if (file_type != QueryExporter::FileType::kShapefile && + file_type != QueryExporter::FileType::kFlatGeobuf) { switch (type_info.get_subtype()) { case kBOOLEAN: case kTINYINT: diff --git a/Parser/ParserNode.cpp b/Parser/ParserNode.cpp index aa66791f8a..7f066bc27f 100644 --- a/Parser/ParserNode.cpp +++ b/Parser/ParserNode.cpp @@ -5575,10 +5575,12 @@ void ExportQueryStmt::parseOptions( file_type = import_export::QueryExporter::FileType::kGeoJSONL; } else if (file_type_str == "shapefile") { file_type = import_export::QueryExporter::FileType::kShapefile; + } else if (file_type_str == "flatgeobuf") { + file_type = import_export::QueryExporter::FileType::kFlatGeobuf; } else { throw std::runtime_error( - "File Type option must be 'CSV', 'GeoJSON', 'GeoJSONL' or " - "'Shapefile'"); + "File Type option must be 'CSV', 'GeoJSON', 'GeoJSONL', " + "'Shapefile', or 'FlatGeobuf'"); } } else if (boost::iequals(*p->get_name(), "layer_name")) { const StringLiteral* str_literal = diff --git a/Tests/Export/QueryExport/datafiles/query_export_test_fgb_linestring.fgb b/Tests/Export/QueryExport/datafiles/query_export_test_fgb_linestring.fgb new file mode 100644 index 0000000000..13e5ee086d Binary files /dev/null and b/Tests/Export/QueryExport/datafiles/query_export_test_fgb_linestring.fgb differ diff --git a/Tests/Export/QueryExport/datafiles/query_export_test_fgb_multipolygon.fgb b/Tests/Export/QueryExport/datafiles/query_export_test_fgb_multipolygon.fgb new file mode 100644 index 0000000000..82c56e0b06 Binary files /dev/null and b/Tests/Export/QueryExport/datafiles/query_export_test_fgb_multipolygon.fgb differ diff --git a/Tests/Export/QueryExport/datafiles/query_export_test_fgb_point.fgb b/Tests/Export/QueryExport/datafiles/query_export_test_fgb_point.fgb new file mode 100644 index 0000000000..b36bc4c1d8 Binary files /dev/null and b/Tests/Export/QueryExport/datafiles/query_export_test_fgb_point.fgb differ diff --git a/Tests/Export/QueryExport/datafiles/query_export_test_fgb_polygon.fgb b/Tests/Export/QueryExport/datafiles/query_export_test_fgb_polygon.fgb new file mode 100644 index 0000000000..b66954c54d Binary files /dev/null and b/Tests/Export/QueryExport/datafiles/query_export_test_fgb_polygon.fgb differ diff --git a/Tests/Import/datafiles/geospatial_mpoly/geospatial_mpoly.fgb b/Tests/Import/datafiles/geospatial_mpoly/geospatial_mpoly.fgb new file mode 100644 index 0000000000..549393185e Binary files /dev/null and b/Tests/Import/datafiles/geospatial_mpoly/geospatial_mpoly.fgb differ diff --git a/Tests/Import/datafiles/geospatial_point/geospatial_point.fgb b/Tests/Import/datafiles/geospatial_point/geospatial_point.fgb new file mode 100644 index 0000000000..da2b08775e Binary files /dev/null and b/Tests/Import/datafiles/geospatial_point/geospatial_point.fgb differ diff --git a/Tests/ImportExportTest.cpp b/Tests/ImportExportTest.cpp index 270b6eec22..1f86e52d43 100644 --- a/Tests/ImportExportTest.cpp +++ b/Tests/ImportExportTest.cpp @@ -1627,6 +1627,22 @@ TEST_F(ImportTestGDAL, KML_Simple) { } } +TEST_F(ImportTestGDAL, FlatGeobuf_Point_Import) { + SKIP_ALL_ON_AGGREGATOR(); + const auto file_path = boost::filesystem::path("geospatial_point/geospatial_point.fgb"); + import_test_geofile_importer(file_path.string(), "geospatial", false, true, false); + check_geo_gdal_point_import(); + check_geo_num_rows("omnisci_geo, trip", 10); +} + +TEST_F(ImportTestGDAL, FlatGeobuf_MultiPolygon_Import) { + SKIP_ALL_ON_AGGREGATOR(); + const auto file_path = boost::filesystem::path("geospatial_mpoly/geospatial_mpoly.fgb"); + import_test_geofile_importer(file_path.string(), "geospatial", false, true, false); + check_geo_gdal_poly_or_mpoly_import(false, false); // poly, not exploded + check_geo_num_rows("omnisci_geo, trip", 10); +} + #ifdef HAVE_AWS_S3 // s3 compressed (non-parquet) test cases TEST_F(ImportTest, S3_One_csv_file) { @@ -2010,17 +2026,17 @@ class ExportTest : public ::testing::Test { } // select a comparable value from the first row - // hopefully immune to any re-ordering due to export query non-determinism + // tolerate any re-ordering due to export query non-determinism // scope this block so that the ResultSet is destroyed before the table is dropped - // @TODO(se) make this mo better - // push scope to ensure ResultSet deletion before table drop { auto rows = run_query("SELECT col_big FROM query_export_test_reimport WHERE rowid=0"); rows->setGeoReturnType(ResultSet::GeoReturnType::GeoTargetValue); auto crt_row = rows->getNextRow(true, true); const auto col_big = v(crt_row[0]); - ASSERT_EQ(20395569495LL, col_big); + constexpr std::array values{ + 84212876526LL, 53000912292LL, 31851544292LL, 31334726270LL, 20395569495LL}; + ASSERT_NE(std::find(values.begin(), values.end(), col_big), values.end()); } // drop the table @@ -2608,6 +2624,55 @@ TEST_F(ExportTest, Shapefile_Zip_Unimplemented) { RUN_TEST_ON_ALL_GEO_TYPES(); } +TEST_F(ExportTest, FlatGeobuf) { + SKIP_ALL_ON_AGGREGATOR(); + doCreateAndImport(); + auto run_test = [&](const std::string& geo_type) { + std::string exp_file = "query_export_test_fgb_" + geo_type + ".fgb"; + ASSERT_NO_THROW( + doExport(exp_file, "FlatGeobuf", "", geo_type, NO_ARRAYS, DEFAULT_SRID)); + std::string layer_name = "query_export_test_fgb_" + geo_type; + ASSERT_NO_THROW(doCompareWithOGRInfo(exp_file, layer_name, COMPARE_EXPLICIT)); + doImportAgainAndCompare(exp_file, "FlatGeobuf", geo_type, NO_ARRAYS); + removeExportedFile(exp_file); + }; + RUN_TEST_ON_ALL_GEO_TYPES(); +} + +TEST_F(ExportTest, FlatGeobuf_Overwrite) { + SKIP_ALL_ON_AGGREGATOR(); + doCreateAndImport(); + auto run_test = [&](const std::string& geo_type) { + std::string exp_file = "query_export_test_fgb_" + geo_type + ".fgb"; + ASSERT_NO_THROW( + doExport(exp_file, "FlatGeobuf", "", geo_type, NO_ARRAYS, DEFAULT_SRID)); + ASSERT_NO_THROW( + doExport(exp_file, "FlatGeobuf", "", geo_type, NO_ARRAYS, DEFAULT_SRID)); + removeExportedFile(exp_file); + }; + RUN_TEST_ON_ALL_GEO_TYPES(); +} + +TEST_F(ExportTest, FlatGeobuf_InvalidName) { + SKIP_ALL_ON_AGGREGATOR(); + doCreateAndImport(); + std::string geo_type = "point"; + std::string exp_file = "query_export_test_fgb_" + geo_type + ".jpg"; + EXPECT_THROW(doExport(exp_file, "FlatGeobuf", "", geo_type, NO_ARRAYS, DEFAULT_SRID), + std::runtime_error); +} + +TEST_F(ExportTest, FlatGeobuf_Invalid_SRID) { + SKIP_ALL_ON_AGGREGATOR(); + doCreateAndImport(); + auto run_test = [&](const std::string& geo_type) { + std::string exp_file = "query_export_test_geojsonl_" + geo_type + ".fgb"; + EXPECT_THROW(doExport(exp_file, "FlatGeobuf", "", geo_type, NO_ARRAYS, INVALID_SRID), + std::runtime_error); + }; + RUN_TEST_ON_ALL_GEO_TYPES(); +} + TEST_F(ExportTest, Array_Null_Handling_Default) { SKIP_ALL_ON_AGGREGATOR(); EXPECT_THROW(doTestArrayNullHandling( diff --git a/ThriftHandler/DBHandler.cpp b/ThriftHandler/DBHandler.cpp index 0724cba358..00bd826e08 100644 --- a/ThriftHandler/DBHandler.cpp +++ b/ThriftHandler/DBHandler.cpp @@ -3807,7 +3807,7 @@ bool is_a_supported_geo_file(const std::string& path, bool include_gz) { if (boost::iends_with(path, ".shp") || boost::iends_with(path, ".geojson") || boost::iends_with(path, ".json") || boost::iends_with(path, ".kml") || boost::iends_with(path, ".kmz") || boost::iends_with(path, ".gdb") || - boost::iends_with(path, ".gdb.zip")) { + boost::iends_with(path, ".gdb.zip") || boost::iends_with(path, ".fgb")) { return true; } return false;