Split UdfCompiler into separate library

Removes clang dependence for QueryEngine, and separates the compiler from the registration of the module IR, taking a step towards unifying compiled and runtime UDFs.
yma11 · Jul 19, 2021 · 09b6b02 · 09b6b02
1 parent d58aca1
commit 09b6b02
Show file tree

Hide file tree

Showing 18 changed files with 455 additions and 459 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -456,11 +456,6 @@ else()
   set(llvm_libs ${LLVM_LIB})
 endif()
 
-# Clang (UDF Compiler)
-find_package(Clang REQUIRED)
-include_directories(${CLANG_INCLUDE_DIRS})
-add_definitions(${CLANG_DEFINITIONS})
-
 
 # Boost
 find_package(Boost COMPONENTS log log_setup filesystem program_options regex system thread timer locale iostreams REQUIRED)
@@ -692,6 +687,7 @@ add_subdirectory(TableArchiver)
 add_subdirectory(ThriftHandler)
 add_subdirectory(Geospatial)
 add_subdirectory(Distributed)
+add_subdirectory(UdfCompiler)
 
 if(ENABLE_DBE)
   add_subdirectory(Embedded)

diff --git a/QueryEngine/CMakeLists.txt b/QueryEngine/CMakeLists.txt
@@ -114,7 +114,6 @@ set(query_engine_source_files
     TableGenerations.cpp
     TableOptimizer.cpp
     TargetExprBuilder.cpp
-    UDFCompiler.cpp
     Utils/DiamondCodegen.cpp
     StringFunctions.cpp
     StringOpsIR.cpp
@@ -275,7 +274,6 @@ set(QUERY_ENGINE_LIBS
   ${Arrow_LIBRARIES}
 )
 
-list(APPEND QUERY_ENGINE_LIBS ${clang_libs})
 list(APPEND QUERY_ENGINE_LIBS ${llvm_libs} ${ZLIB_LIBRARIES})
 
 target_link_libraries(QueryEngine ${QUERY_ENGINE_LIBS})

diff --git a/QueryEngine/Execute.h b/QueryEngine/Execute.h
@@ -148,9 +148,7 @@ class QuerySessionStatus {
 };
 using QuerySessionMap =
     std::map<const QuerySessionId, std::map<std::string, QuerySessionStatus>>;
-extern void read_udf_gpu_module(const std::string& udf_ir_filename);
-extern void read_udf_cpu_module(const std::string& udf_ir_filename);
-extern bool is_udf_module_present(bool cpu_only = false);
+
 extern void read_rt_udf_gpu_module(const std::string& udf_ir);
 extern void read_rt_udf_cpu_module(const std::string& udf_ir);
 extern bool is_rt_udf_module_present(bool cpu_only = false);
@@ -389,6 +387,8 @@ class Executor {
 
   static size_t getArenaBlockSize();
 
+  static void addUdfIrToModule(const std::string& udf_ir_filename, const bool is_cuda_ir);
+
   /**
    * Returns pointer to the intermediate tables vector currently stored by this executor.
    */

diff --git a/QueryEngine/ExtensionsIR.cpp b/QueryEngine/ExtensionsIR.cpp
diff --git a/QueryEngine/NativeCodegen.cpp b/QueryEngine/NativeCodegen.cpp
@@ -985,6 +985,14 @@ std::map<std::string, std::string> get_device_parameters(bool cpu_only) {
   return result;
 }
 
+namespace {
+
+bool is_udf_module_present(bool cpu_only = false) {
+  return (cpu_only || udf_gpu_module != nullptr) && (udf_cpu_module != nullptr);
+}
+
+}  // namespace
+
 std::shared_ptr<GpuCompilationContext> CodeGenerator::generateNativeGPUCode(
     llvm::Function* func,
     llvm::Function* wrapper_func,
@@ -1725,14 +1733,12 @@ std::unique_ptr<llvm::Module> g_rt_libdevice_module(
     read_libdevice_module(getGlobalLLVMContext()));
 #endif
 
-bool is_udf_module_present(bool cpu_only) {
-  return (cpu_only || udf_gpu_module != nullptr) && (udf_cpu_module != nullptr);
-}
-
 bool is_rt_udf_module_present(bool cpu_only) {
   return (cpu_only || rt_udf_gpu_module != nullptr) && (rt_udf_cpu_module != nullptr);
 }
 
+namespace {
+
 void read_udf_gpu_module(const std::string& udf_ir_filename) {
   llvm::SMDiagnostic parse_error;
 
@@ -1763,6 +1769,17 @@ void read_udf_cpu_module(const std::string& udf_ir_filename) {
   }
 }
 
+}  // namespace
+
+void Executor::addUdfIrToModule(const std::string& udf_ir_filename,
+                                const bool is_cuda_ir) {
+  if (is_cuda_ir) {
+    read_udf_gpu_module(udf_ir_filename);
+  } else {
+    read_udf_cpu_module(udf_ir_filename);
+  }
+}
+
 void read_rt_udf_gpu_module(const std::string& udf_ir_string) {
   llvm::SMDiagnostic parse_error;
 

diff --git a/QueryEngine/OmniSciTypes.h b/QueryEngine/OmniSciTypes.h
@@ -17,6 +17,7 @@
 #pragma once
 
 #include <limits>
+#include <stdexcept>
 #include <type_traits>
 
 /* `../` is required for UDFCompiler */
@@ -69,9 +70,9 @@ struct Array {
   }
 };
 
-struct GeoLineString {
+struct GeoPoint {
   int8_t* ptr;
-  int64_t sz;
+  int32_t sz;
   int32_t compression;
   int32_t input_srid;
   int32_t output_srid;
@@ -85,14 +86,14 @@ struct GeoLineString {
   DEVICE int32_t getOutputSrid() const { return output_srid; }
 };
 
-struct GeoPoint {
+struct GeoLineString {
   int8_t* ptr;
-  int64_t sz;
+  int32_t sz;
   int32_t compression;
   int32_t input_srid;
   int32_t output_srid;
 
-  DEVICE int64_t getSize() const { return sz; }
+  DEVICE int32_t getSize() const { return sz; }
 
   DEVICE int32_t getCompression() const { return compression; }
 
@@ -103,17 +104,17 @@ struct GeoPoint {
 
 struct GeoPolygon {
   int8_t* ptr_coords;
-  int64_t coords_size;
-  int32_t* ring_sizes;
-  int64_t num_rings;
+  int32_t coords_size;
+  int8_t* ring_sizes;
+  int32_t num_rings;
   int32_t compression;
   int32_t input_srid;
   int32_t output_srid;
 
-  DEVICE int32_t* getRingSizes() { return ring_sizes; }
-  DEVICE int64_t getCoordsSize() const { return coords_size; }
+  DEVICE int8_t* getRingSizes() { return ring_sizes; }
+  DEVICE int32_t getCoordsSize() const { return coords_size; }
 
-  DEVICE int64_t getNumRings() const { return num_rings; }
+  DEVICE int32_t getNumRings() const { return num_rings; }
 
   DEVICE int32_t getCompression() const { return compression; }
 
@@ -124,23 +125,23 @@ struct GeoPolygon {
 
 struct GeoMultiPolygon {
   int8_t* ptr_coords;
-  int64_t coords_size;
-  int32_t* ring_sizes;
-  int64_t num_rings;
-  int32_t* poly_sizes;
-  int64_t num_polys;
+  int32_t coords_size;
+  int8_t* ring_sizes;
+  int32_t num_rings;
+  int8_t* poly_sizes;
+  int32_t num_polys;
   int32_t compression;
   int32_t input_srid;
   int32_t output_srid;
 
-  DEVICE int32_t* getRingSizes() { return ring_sizes; }
-  DEVICE int64_t getCoordsSize() const { return coords_size; }
+  DEVICE int8_t* getRingSizes() { return ring_sizes; }
+  DEVICE int32_t getCoordsSize() const { return coords_size; }
 
-  DEVICE int64_t getNumRings() const { return num_rings; }
+  DEVICE int32_t getNumRings() const { return num_rings; }
 
-  DEVICE int32_t* getPolygonSizes() { return poly_sizes; }
+  DEVICE int8_t* getPolygonSizes() { return poly_sizes; }
 
-  DEVICE int64_t getNumPolygons() const { return num_polys; }
+  DEVICE int32_t getNumPolygons() const { return num_polys; }
 
   DEVICE int32_t getCompression() const { return compression; }
 
@@ -159,7 +160,7 @@ struct Column {
 #ifndef __CUDACC__
       throw std::runtime_error("column buffer index is out of range");
 #else
-      static T null_value;
+      static DEVICE T null_value;
       set_null(null_value);
       return null_value;
 #endif

diff --git a/QueryEngine/UDFCompiler.h b/QueryEngine/UDFCompiler.h
diff --git a/Shared/InlineNullValues.h b/Shared/InlineNullValues.h
@@ -23,6 +23,7 @@
 #include <cassert>
 #include <cfloat>
 #include <cstdint>
+#include <cstdlib>
 #include <limits>
 
 #define NULL_BOOLEAN INT8_MIN
@@ -100,12 +101,12 @@ DEVICE T inline_fp_null_array_value() {
 }
 
 template <>
-constexpr inline float inline_fp_null_array_value<float>() {
+DEVICE inline float inline_fp_null_array_value<float>() {
   return NULL_ARRAY_FLOAT;
 }
 
 template <>
-constexpr inline double inline_fp_null_array_value<double>() {
+DEVICE inline double inline_fp_null_array_value<double>() {
   return NULL_ARRAY_DOUBLE;
 }
 

diff --git a/Tests/CMakeLists.txt b/Tests/CMakeLists.txt
@@ -203,7 +203,7 @@ target_link_libraries(LoadTableTest ${THRIFT_HANDLER_TEST_LIBRARIES})
 target_link_libraries(JSONTest gtest Logger Shared)
 
 if(NOT ${CMAKE_SYSTEM_NAME} STREQUAL "Darwin")
-  target_link_libraries(UdfTest gtest ${EXECUTE_TEST_LIBS})
+  target_link_libraries(UdfTest gtest UdfCompiler ${EXECUTE_TEST_LIBS})
 endif()
 
 target_link_libraries(TableUpdateDeleteBenchmark benchmark ${EXECUTE_TEST_LIBS})

diff --git a/Tests/CachedHashTableTest.cpp b/Tests/CachedHashTableTest.cpp
@@ -31,7 +31,6 @@
 #include "QueryEngine/Execute.h"
 #include "QueryEngine/MurmurHash1Inl.h"
 #include "QueryEngine/ResultSet.h"
-#include "QueryEngine/UDFCompiler.h"
 #include "QueryRunner/QueryRunner.h"
 #include "Shared/SystemParameters.h"
 #include "TestHelpers.h"

diff --git a/Tests/JoinHashTableTest.cpp b/Tests/JoinHashTableTest.cpp
@@ -33,7 +33,6 @@
 #include "QueryEngine/ExternalCacheInvalidators.h"
 #include "QueryEngine/JoinHashTable/OverlapsJoinHashTable.h"
 #include "QueryEngine/ResultSet.h"
-#include "QueryEngine/UDFCompiler.h"
 #include "QueryRunner/QueryRunner.h"
 #include "Shared/thread_count.h"
 #include "TestHelpers.h"