From cf7e1fcdd4c1a22622ec4e469e0b77f1724efaa0 Mon Sep 17 00:00:00 2001 From: mattpulver <47544175+mattpulver@users.noreply.github.com> Date: Thu, 20 Jun 2019 03:50:59 -0700 Subject: [PATCH] Replace glog with boost.log - Investigate alternative logging frameworks to glog: Boost.Log. - Implement in Shared/Logger.{h,cpp} - Support both file and console (std::clog) sinks. - Replaced #include with #include "Shared/Logger.h" - Removed logging from Calcite destructor which caused problems on shutdown. - Move std::ostream operator<<() overloads into their namespaces to allow for ADL. - Print Boost version in cmake. - Don't log when __CUDACC__ is defined. - Move always_assert.h logic into Logger.h and delete file. - Move unreachable.h logic into Logger.h and delete file. - Fix casting in numerous places. Warnings were previously suppressed by glog. --- Analyzer/Analyzer.cpp | 2 - Analyzer/Analyzer.h | 4 +- Archive/S3Archive.cpp | 2 +- CMakeLists.txt | 14 +- Calcite/Calcite.cpp | 16 +- Calcite/Calcite.h | 2 +- Catalog/DBObject.h | 2 +- Catalog/Grantee.h | 2 +- Catalog/SessionInfo.cpp | 7 +- Catalog/SessionInfo.h | 5 +- Catalog/SharedDictionaryValidator.cpp | 2 +- CudaMgr/CudaMgr.cpp | 2 +- CudaMgr/CudaMgrNoCuda.cpp | 2 +- DataMgr/AbstractBuffer.h | 6 +- DataMgr/ArrayNoneEncoder.h | 3 +- DataMgr/BufferMgr/Buffer.cpp | 2 +- DataMgr/BufferMgr/BufferMgr.cpp | 2 +- DataMgr/BufferMgr/CpuBufferMgr/CpuBuffer.cpp | 2 +- .../BufferMgr/CpuBufferMgr/CpuBufferMgr.cpp | 1 - .../GpuCudaBufferMgr/GpuCudaBuffer.cpp | 2 +- .../GpuCudaBufferMgr/GpuCudaBufferMgr.cpp | 2 +- DataMgr/CMakeLists.txt | 2 +- DataMgr/ChunkMetadata.h | 2 +- DataMgr/DateDaysEncoder.h | 5 +- DataMgr/Encoder.cpp | 2 +- DataMgr/FileMgr/FileBuffer.cpp | 1 - DataMgr/FileMgr/FileBuffer.h | 6 +- DataMgr/FileMgr/FileInfo.cpp | 3 +- DataMgr/FileMgr/FileInfo.h | 1 + DataMgr/FileMgr/FileMgr.cpp | 1 + DataMgr/FileMgr/GlobalFileMgr.h | 1 + DataMgr/FileMgr/Page.h | 3 +- DataMgr/FixedLengthArrayNoneEncoder.h | 3 +- DataMgr/FixedLengthEncoder.h | 3 +- DataMgr/StringNoneEncoder.h | 2 +- Distributed/os/LeafAggregator.h | 2 +- Fragmenter/InsertOrderFragmenter.cpp | 2 +- Fragmenter/SortedOrderFragmenter.cpp | 2 +- Fragmenter/UpdelStorage.cpp | 2 +- Import/CMakeLists.txt | 6 +- Import/Importer.cpp | 38 +-- Import/Importer.h | 6 +- Import/KafkaImporter.cpp | 10 +- Import/RowToColumnLoader.cpp | 27 +- Import/RowToColumnLoader.h | 3 +- Import/StreamImporter.cpp | 10 +- MapDServer.cpp | 136 ++++---- Parser/ParserNode.h | 1 - Parser/ParserWrapper.cpp | 2 - QueryEngine/AggregateUtils.h | 2 +- QueryEngine/Allocators/CudaAllocator.cpp | 3 +- QueryEngine/Allocators/DeviceAllocator.h | 2 +- QueryEngine/Allocators/ThrustAllocator.cpp | 5 +- QueryEngine/BaselineJoinHashTable.cpp | 4 +- QueryEngine/CMakeLists.txt | 10 +- QueryEngine/CalciteDeserializerUtils.cpp | 1 + QueryEngine/CalciteDeserializerUtils.h | 3 +- QueryEngine/CardinalityEstimator.h | 3 +- QueryEngine/Codec.cpp | 2 +- QueryEngine/ColumnIR.cpp | 7 +- QueryEngine/ConstantIR.cpp | 6 +- QueryEngine/DateAdd.cpp | 2 +- QueryEngine/DateTimePlusRewrite.cpp | 3 +- QueryEngine/DateTimeUtils.h | 16 +- QueryEngine/DateTruncate.cpp | 3 +- QueryEngine/Descriptors/ColSlotContext.cpp | 2 +- QueryEngine/Descriptors/ColSlotContext.h | 2 +- .../Descriptors/CountDistinctDescriptor.h | 3 +- QueryEngine/Descriptors/InputDescriptors.h | 2 +- .../Descriptors/QueryFragmentDescriptor.h | 2 +- .../Descriptors/QueryMemoryDescriptor.h | 3 +- QueryEngine/Descriptors/RowSetMemoryOwner.h | 2 +- QueryEngine/DynamicWatchdog.cpp | 2 +- QueryEngine/Execute.cpp | 2 + QueryEngine/ExecutionDispatch.cpp | 2 +- QueryEngine/ExpressionRange.h | 1 - QueryEngine/ExpressionRewrite.cpp | 2 +- QueryEngine/ExtensionsIR.cpp | 2 +- QueryEngine/ExtractFromTime.cpp | 2 +- QueryEngine/GpuMemUtils.cpp | 3 +- QueryEngine/GroupByAndAggregate.cpp | 5 +- QueryEngine/GroupByAndAggregate.h | 2 +- QueryEngine/HashJoinKeyHandlers.h | 2 +- QueryEngine/HashJoinRuntime.cpp | 11 +- QueryEngine/IRCodegenUtils.h | 2 +- QueryEngine/InPlaceSort.cpp | 2 +- QueryEngine/InValuesBitmap.cpp | 2 +- QueryEngine/JoinFilterPushDown.cpp | 5 +- QueryEngine/JoinHashTable.cpp | 8 +- QueryEngine/JsonAccessors.h | 2 +- QueryEngine/LoopControlFlow/JoinLoop.cpp | 1 + QueryEngine/LoopControlFlow/JoinLoop.h | 2 +- QueryEngine/LoopControlFlow/JoinLoopTest.cpp | 2 +- QueryEngine/NvidiaKernel.cpp | 2 +- QueryEngine/OverlapsJoinHashTable.cpp | 10 +- QueryEngine/QueryExecutionContext.cpp | 2 +- QueryEngine/QueryMemoryInitializer.cpp | 6 +- QueryEngine/QueryRewrite.cpp | 3 +- QueryEngine/QueryTemplateGenerator.cpp | 2 +- QueryEngine/RelAlgAbstractInterpreter.cpp | 1 - QueryEngine/RelAlgExecutor.cpp | 10 +- QueryEngine/RelAlgOptimizer.cpp | 3 +- QueryEngine/RelAlgTranslator.cpp | 10 +- QueryEngine/RelAlgTranslatorGeo.cpp | 16 +- QueryEngine/RelLeftDeepInnerJoin.cpp | 3 +- QueryEngine/Rendering/os/RenderAllocator.cpp | 2 +- QueryEngine/ResultSetBufferAccessors.h | 2 - QueryEngine/ResultSetIteration.cpp | 10 +- QueryEngine/ResultSetReduction.cpp | 8 +- QueryEngine/ScalarCodeGenerator.cpp | 4 +- QueryEngine/SpeculativeTopN.cpp | 3 +- QueryEngine/StringDictionaryGenerations.cpp | 2 - QueryEngine/TableGenerations.cpp | 3 +- QueryEngine/TableOptimizer.h | 2 +- QueryEngine/TargetExprBuilder.cpp | 6 +- QueryEngine/TargetValue.h | 2 +- QueryEngine/UDFCompiler.cpp | 2 +- QueryEngine/WindowContext.cpp | 2 +- QueryEngine/invoke_example/sample.cpp | 2 +- QueryRunner/QueryRunner.cpp | 6 +- SQLFrontend/CMakeLists.txt | 2 +- SQLFrontend/CommandFunctors.h | 26 +- SQLFrontend/omnisql.cpp | 2 +- Shared/CMakeLists.txt | 2 +- Shared/Compressor.cpp | 2 +- Shared/Datum.cpp | 2 +- Shared/File.cpp | 2 +- Shared/InlineNullValues.h | 8 +- Shared/Logger.cpp | 290 +++++++++++++++++ Shared/Logger.h | 278 +++++++++++++++++ Shared/SqlTypesLayout.h | 6 +- Shared/StringTransform.h | 3 +- Shared/ThriftTypesConvert.h | 2 +- Shared/TypedDataAccessors.h | 6 +- Shared/always_assert.h | 41 --- Shared/checked_alloc.h | 2 +- Shared/geo_types.cpp | 4 +- Shared/mapdpath.h | 2 +- Shared/measure.h | 3 +- Shared/sql_window_function_to_string.h | 2 +- Shared/unreachable.h | 28 -- StringDictionary/CMakeLists.txt | 4 +- StringDictionary/StringDictionary.cpp | 8 +- StringDictionary/StringDictionary.h | 6 +- StringDictionary/StringDictionaryProxy.cpp | 2 +- Tests/AlterColumnTest.cpp | 3 +- Tests/CMakeLists.txt | 29 +- Tests/CalciteOptimizeTest.cpp | 4 +- Tests/CodeGeneratorTest.cpp | 4 +- Tests/ComputeMetadataTest.cpp | 3 +- Tests/CtasIntegrationTest.cpp | 4 +- Tests/CtasUpdateTest.cpp | 4 +- Tests/DBObjectPrivilegesTest.cpp | 4 +- Tests/ExecuteTest.cpp | 10 +- Tests/ExperimentalTest.cpp | 4 +- Tests/FromTableReorderingTest.cpp | 3 +- Tests/ImportTest.cpp | 8 +- Tests/PlanTest.cpp | 4 +- Tests/PopulateTableRandom.cpp | 3 +- Tests/ProfileTest.cpp | 5 +- Tests/ProfileTest.h | 6 +- Tests/ResultSetBaselineRadixSortTest.cpp | 4 +- Tests/ResultSetTest.cpp | 4 +- Tests/Shared/DateTimeUtilsTest.cpp | 4 +- Tests/Shared/GeoTypesTest.cpp | 4 +- Tests/StoragePerfTest.cpp | 4 +- Tests/StorageTest.cpp | 4 +- Tests/StringDictionaryTest.cpp | 4 +- Tests/StringTransformTest.cpp | 4 +- Tests/TestHelpers.h | 9 + Tests/TokenCompletionHintsTest.cpp | 4 +- Tests/TopKTest.cpp | 4 +- Tests/UdfTest.cpp | 7 +- Tests/UpdateMetadataTest.cpp | 8 +- Tests/UpdelStorageTest.cpp | 4 +- Tests/UtilTest.cpp | 4 +- .../muparserx/muparserx/mpFuncCommon.cpp | 2 +- ThirdParty/muparserx/muparserx/mpFuncStr.cpp | 2 +- .../muparserx/muparserx/mpOprtCmplx.cpp | 2 +- .../muparserx/muparserx/mpOprtMatrix.cpp | 2 +- .../muparserx/muparserx/mpOprtNonCmplx.cpp | 2 +- ThriftHandler/CMakeLists.txt | 2 +- ThriftHandler/DistributedValidate.h | 2 +- ThriftHandler/MapDHandler.cpp | 293 +++++++++--------- ThriftHandler/MapDHandler.h | 73 +++-- initdb.cpp | 9 +- 186 files changed, 1207 insertions(+), 723 deletions(-) create mode 100644 Shared/Logger.cpp create mode 100644 Shared/Logger.h delete mode 100644 Shared/always_assert.h delete mode 100644 Shared/unreachable.h diff --git a/Analyzer/Analyzer.cpp b/Analyzer/Analyzer.cpp index f2bae4e41a..7aec791484 100644 --- a/Analyzer/Analyzer.cpp +++ b/Analyzer/Analyzer.cpp @@ -29,10 +29,8 @@ #include "../Shared/sql_type_to_string.h" #include "../Shared/sql_window_function_to_string.h" #include "../Shared/sqltypes.h" -#include "../Shared/unreachable.h" #include "RangeTableEntry.h" -#include #include #include #include diff --git a/Analyzer/Analyzer.h b/Analyzer/Analyzer.h index 0abdb836a9..44fa59933c 100644 --- a/Analyzer/Analyzer.h +++ b/Analyzer/Analyzer.h @@ -24,7 +24,8 @@ #ifndef ANALYZER_H #define ANALYZER_H -#include +#include "../Shared/Logger.h" + #include #include #include @@ -33,6 +34,7 @@ #include #include #include + #include "../Shared/sqldefs.h" #include "../Shared/sqltypes.h" diff --git a/Archive/S3Archive.cpp b/Archive/S3Archive.cpp index 805991ec0a..9331d78dc5 100644 --- a/Archive/S3Archive.cpp +++ b/Archive/S3Archive.cpp @@ -14,9 +14,9 @@ * limitations under the License. */ #include "S3Archive.h" -#include #include #include +#include "Shared/Logger.h" #include #include diff --git a/CMakeLists.txt b/CMakeLists.txt index 755554edf5..a827ac2812 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -71,6 +71,8 @@ if(PREFER_STATIC_LIBS) endif() set(CUDA_USE_STATIC_CUDA_RUNTIME ON CACHE STRING "Use static CUDA runtime") +else() + add_definitions("-DBOOST_LOG_DYN_LINK") endif() option(ENABLE_JAVA_REMOTE_DEBUG "Enable Java Remote Debug" OFF ) @@ -109,6 +111,11 @@ else() set(MAPD_PACKAGE_FLAGS "${MAPD_PACKAGE_FLAGS}-cpu") endif() +option(SUPPRESS_NULL_LOGGER_DEPRECATION_WARNINGS "Suppress NullLogger deprecated warnings.") +if (SUPPRESS_NULL_LOGGER_DEPRECATION_WARNINGS) + add_definitions("-DSUPPRESS_NULL_LOGGER_DEPRECATION_WARNINGS") +endif() + option(ENABLE_CUDA_KERNEL_DEBUG "Enable debugging symbols for CUDA device Kernels" OFF) option(ENABLE_JIT_DEBUG "Enable debugging symbols for the JIT" OFF) @@ -155,6 +162,7 @@ if(ENABLE_FOLLY) else() include_directories(${Folly_INCLUDE_DIRS}) add_definitions("-DHAVE_FOLLY") + list(APPEND Folly_LIBRARIES ${Glog_LIBRARIES}) endif() endif() @@ -168,7 +176,7 @@ set(EXECUTABLE_OUTPUT_PATH ${CMAKE_BINARY_DIR}/bin) if(APPLE OR ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" AND (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 3.6 OR CMAKE_CXX_COMPILER_VERSION VERSION_EQUAL 3.6)) OR ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 4.9)) - find_package( Boost COMPONENTS filesystem program_options regex system timer REQUIRED QUIET ) + find_package( Boost COMPONENTS log log_setup filesystem program_options regex system thread timer REQUIRED ) else() message(FATAL_ERROR, "GCC v4.9 or clang v3.6 (or newer) is required") endif() @@ -564,9 +572,9 @@ add_custom_target(rerun_cmake ALL ) add_dependencies(omnisci_server rerun_cmake) -target_link_libraries(omnisci_server mapd_thrift thrift_handler ${MAPD_LIBRARIES} ${Boost_LIBRARIES} ${Glog_LIBRARIES} ${CMAKE_DL_LIBS} ${CUDA_LIBRARIES} ${LLVM_LINKER_FLAGS} ${PROFILER_LIBS} ${CURSES_LIBRARIES} ${ZLIB_LIBRARIES}) +target_link_libraries(omnisci_server mapd_thrift thrift_handler ${MAPD_LIBRARIES} ${Boost_LIBRARIES} ${CMAKE_DL_LIBS} ${CUDA_LIBRARIES} ${LLVM_LINKER_FLAGS} ${PROFILER_LIBS} ${CURSES_LIBRARIES} ${ZLIB_LIBRARIES}) -target_link_libraries(initdb mapd_thrift DataMgr ${MAPD_LIBRARIES} ${Boost_LIBRARIES} ${Glog_LIBRARIES} ${CMAKE_DL_LIBS} ${CUDA_LIBRARIES} ${LLVM_LINKER_FLAGS} ${CURSES_LIBRARIES} ${ZLIB_LIBRARIES}) +target_link_libraries(initdb mapd_thrift DataMgr ${MAPD_LIBRARIES} ${Boost_LIBRARIES} ${CMAKE_DL_LIBS} ${CUDA_LIBRARIES} ${LLVM_LINKER_FLAGS} ${CURSES_LIBRARIES} ${ZLIB_LIBRARIES}) macro(set_dpkg_arch arch_in arch_out) if("${arch_in}" STREQUAL "x86_64") diff --git a/Calcite/Calcite.cpp b/Calcite/Calcite.cpp index 81cf75219f..2fee6e6d87 100644 --- a/Calcite/Calcite.cpp +++ b/Calcite/Calcite.cpp @@ -22,15 +22,14 @@ */ #include "Calcite.h" -#include #include #include #include "Catalog/Catalog.h" #include "Shared/ConfigResolve.h" +#include "Shared/Logger.h" #include "Shared/mapd_shared_ptr.h" #include "Shared/mapdpath.h" #include "Shared/measure.h" -#include "Shared/unreachable.h" #include "Shared/fixautotools.h" @@ -459,6 +458,7 @@ TPlanResult Calcite::processImpl( LOG(FATAL) << "Error occurred trying to communicate with calcite server, the error was: '" << ex.what() << "', omnisci_server restart will be required"; + return ret; // satisfy return-type warning } } else { LOG(INFO) << "Not routing to Calcite, server is not up"; @@ -505,24 +505,22 @@ std::string Calcite::getUserDefinedFunctionWhitelist() { return ""; } -void Calcite::close_calcite_server() { +void Calcite::close_calcite_server(bool log) { if (server_available_) { - LOG(INFO) << "Shutting down Calcite server"; + LOG_IF(INFO, log) << "Shutting down Calcite server"; try { std::pair, mapd::shared_ptr> clientP = get_client(remote_calcite_port_); clientP.first->shutdown(); clientP.second->close(); } catch (const std::exception& e) { - LOG(ERROR) << "Error shutting down Calcite server: " << e.what(); + std::cerr << "Error shutting down Calcite server: " << e.what(); } - LOG(INFO) << "shut down Calcite"; + LOG_IF(INFO, log) << "shut down Calcite"; server_available_ = false; } } Calcite::~Calcite() { - LOG(INFO) << "Destroy Calcite Class"; - close_calcite_server(); - LOG(INFO) << "End of Calcite Destructor "; + close_calcite_server(false); } diff --git a/Calcite/Calcite.h b/Calcite/Calcite.h index 17a9d5e742..0494147628 100644 --- a/Calcite/Calcite.h +++ b/Calcite/Calcite.h @@ -71,7 +71,7 @@ class Calcite { std::string getExtensionFunctionWhitelist(); std::string getUserDefinedFunctionWhitelist(); void updateMetadata(std::string catalog, std::string table); - void close_calcite_server(); + void close_calcite_server(bool log = true); virtual ~Calcite(); std::string& get_session_prefix() { return session_prefix_; } diff --git a/Catalog/DBObject.h b/Catalog/DBObject.h index 73b7bed1ad..2ef2db377f 100644 --- a/Catalog/DBObject.h +++ b/Catalog/DBObject.h @@ -29,9 +29,9 @@ #ifndef DBOBJECT_H #define DBOBJECT_H -#include #include #include +#include "Shared/Logger.h" namespace Catalog_Namespace { class Catalog; diff --git a/Catalog/Grantee.h b/Catalog/Grantee.h index d9911b41e6..490e640856 100644 --- a/Catalog/Grantee.h +++ b/Catalog/Grantee.h @@ -19,12 +19,12 @@ #include "DBObject.h" -#include #include #include #include #include #include +#include "Shared/Logger.h" class User; class Role; diff --git a/Catalog/SessionInfo.cpp b/Catalog/SessionInfo.cpp index 259c9741af..6384600a81 100644 --- a/Catalog/SessionInfo.cpp +++ b/Catalog/SessionInfo.cpp @@ -50,10 +50,9 @@ std::string SessionInfo::public_session_id() const { return ss.str(); } -} // namespace Catalog_Namespace - -std::ostream& operator<<(std::ostream& os, - const Catalog_Namespace::SessionInfo& session_info) { +std::ostream& operator<<(std::ostream& os, const SessionInfo& session_info) { os << session_info.get_public_session_id(); return os; } + +} // namespace Catalog_Namespace diff --git a/Catalog/SessionInfo.h b/Catalog/SessionInfo.h index ec646b2545..41b9862637 100644 --- a/Catalog/SessionInfo.h +++ b/Catalog/SessionInfo.h @@ -119,9 +119,8 @@ class SessionInfo { std::string public_session_id() const; }; -} // namespace Catalog_Namespace +std::ostream& operator<<(std::ostream& os, const SessionInfo& session_info); -std::ostream& operator<<(std::ostream& os, - const Catalog_Namespace::SessionInfo& session_info); +} // namespace Catalog_Namespace #endif /* SESSION_INFO_H */ diff --git a/Catalog/SharedDictionaryValidator.cpp b/Catalog/SharedDictionaryValidator.cpp index 50a6b926a3..4209049915 100644 --- a/Catalog/SharedDictionaryValidator.cpp +++ b/Catalog/SharedDictionaryValidator.cpp @@ -18,7 +18,7 @@ #include -#include "../Shared/unreachable.h" +#include "Shared/Logger.h" const Parser::SharedDictionaryDef compress_reference_path( Parser::SharedDictionaryDef cur_node, diff --git a/CudaMgr/CudaMgr.cpp b/CudaMgr/CudaMgr.cpp index 371a3d921f..dff36453ac 100644 --- a/CudaMgr/CudaMgr.cpp +++ b/CudaMgr/CudaMgr.cpp @@ -16,11 +16,11 @@ #include "CudaMgr.h" #include -#include #include #include #include #include +#include "Shared/Logger.h" namespace CudaMgr_Namespace { diff --git a/CudaMgr/CudaMgrNoCuda.cpp b/CudaMgr/CudaMgrNoCuda.cpp index ccf59cada0..a6e3fd8c41 100644 --- a/CudaMgr/CudaMgrNoCuda.cpp +++ b/CudaMgr/CudaMgrNoCuda.cpp @@ -14,8 +14,8 @@ * limitations under the License. */ -#include #include "CudaMgr.h" +#include "Shared/Logger.h" namespace CudaMgr_Namespace { diff --git a/DataMgr/AbstractBuffer.h b/DataMgr/AbstractBuffer.h index 11e5ee2a50..481116957e 100644 --- a/DataMgr/AbstractBuffer.h +++ b/DataMgr/AbstractBuffer.h @@ -22,11 +22,11 @@ #ifndef DATAMGR_MEMORY_ABSTRACTBUFFER_H #define DATAMGR_MEMORY_ABSTRACTBUFFER_H -#include -#include "../Shared/sqltypes.h" -#include "../Shared/types.h" #include "Encoder.h" #include "MemoryLevel.h" +#include "Shared/Logger.h" +#include "Shared/sqltypes.h" +#include "Shared/types.h" #ifdef BUFFER_MUTEX #include diff --git a/DataMgr/ArrayNoneEncoder.h b/DataMgr/ArrayNoneEncoder.h index 91357b3e9e..49c67622e4 100644 --- a/DataMgr/ArrayNoneEncoder.h +++ b/DataMgr/ArrayNoneEncoder.h @@ -24,7 +24,8 @@ #ifndef ARRAY_NONE_ENCODER_H #define ARRAY_NONE_ENCODER_H -#include +#include "Shared/Logger.h" + #include #include #include diff --git a/DataMgr/BufferMgr/Buffer.cpp b/DataMgr/BufferMgr/Buffer.cpp index 00c7e8f231..fa919c381a 100644 --- a/DataMgr/BufferMgr/Buffer.cpp +++ b/DataMgr/BufferMgr/Buffer.cpp @@ -23,12 +23,12 @@ // // Copyright (c) 2014 MapD Technologies, Inc. All rights reserved. // -#include #include #include #include "Buffer.h" #include "BufferMgr.h" +#include "Shared/Logger.h" namespace Buffer_Namespace { diff --git a/DataMgr/BufferMgr/BufferMgr.cpp b/DataMgr/BufferMgr/BufferMgr.cpp index 6ee53cd431..50a02ecc1a 100644 --- a/DataMgr/BufferMgr/BufferMgr.cpp +++ b/DataMgr/BufferMgr/BufferMgr.cpp @@ -21,9 +21,9 @@ */ #include "BufferMgr.h" #include "Buffer.h" +#include "Shared/Logger.h" #include "Shared/measure.h" -#include #include #include #include diff --git a/DataMgr/BufferMgr/CpuBufferMgr/CpuBuffer.cpp b/DataMgr/BufferMgr/CpuBufferMgr/CpuBuffer.cpp index d53652f07e..9a254935d9 100644 --- a/DataMgr/BufferMgr/CpuBufferMgr/CpuBuffer.cpp +++ b/DataMgr/BufferMgr/CpuBufferMgr/CpuBuffer.cpp @@ -15,10 +15,10 @@ */ #include "CpuBuffer.h" -#include #include #include #include "../../../CudaMgr/CudaMgr.h" +#include "Shared/Logger.h" namespace Buffer_Namespace { diff --git a/DataMgr/BufferMgr/CpuBufferMgr/CpuBufferMgr.cpp b/DataMgr/BufferMgr/CpuBufferMgr/CpuBufferMgr.cpp index 2837e47a65..f7520d9013 100644 --- a/DataMgr/BufferMgr/CpuBufferMgr/CpuBufferMgr.cpp +++ b/DataMgr/BufferMgr/CpuBufferMgr/CpuBufferMgr.cpp @@ -15,7 +15,6 @@ */ #include "CpuBufferMgr.h" -#include #include "../../../CudaMgr/CudaMgr.h" #include "CpuBuffer.h" diff --git a/DataMgr/BufferMgr/GpuCudaBufferMgr/GpuCudaBuffer.cpp b/DataMgr/BufferMgr/GpuCudaBufferMgr/GpuCudaBuffer.cpp index 9abc5cceb7..e02ff5fde2 100644 --- a/DataMgr/BufferMgr/GpuCudaBufferMgr/GpuCudaBuffer.cpp +++ b/DataMgr/BufferMgr/GpuCudaBufferMgr/GpuCudaBuffer.cpp @@ -16,8 +16,8 @@ #include "GpuCudaBuffer.h" #include "../../../CudaMgr/CudaMgr.h" +#include "Shared/Logger.h" -#include #include namespace Buffer_Namespace { diff --git a/DataMgr/BufferMgr/GpuCudaBufferMgr/GpuCudaBufferMgr.cpp b/DataMgr/BufferMgr/GpuCudaBufferMgr/GpuCudaBufferMgr.cpp index 349b197e4d..2ab26a3bbe 100644 --- a/DataMgr/BufferMgr/GpuCudaBufferMgr/GpuCudaBufferMgr.cpp +++ b/DataMgr/BufferMgr/GpuCudaBufferMgr/GpuCudaBufferMgr.cpp @@ -15,9 +15,9 @@ */ #include "GpuCudaBufferMgr.h" -#include #include "../../../CudaMgr/CudaMgr.h" #include "GpuCudaBuffer.h" +#include "Shared/Logger.h" //#include "../CudaUtils.h" namespace Buffer_Namespace { diff --git a/DataMgr/CMakeLists.txt b/DataMgr/CMakeLists.txt index e61198f149..3f3e7f3224 100644 --- a/DataMgr/CMakeLists.txt +++ b/DataMgr/CMakeLists.txt @@ -18,7 +18,7 @@ set(datamgr_source_files add_library(DataMgr ${datamgr_source_files}) -target_link_libraries(DataMgr CudaMgr Shared ${Boost_THREAD_LIBRARY} ${Glog_LIBRARIES}) +target_link_libraries(DataMgr CudaMgr Shared ${Boost_THREAD_LIBRARY}) option(ENABLE_CRASH_CORRUPTION_TEST "Enable crash using SIGUSR2 during page deletion to faster and affirmative test/repro db corruption" OFF) if(ENABLE_CRASH_CORRUPTION_TEST) diff --git a/DataMgr/ChunkMetadata.h b/DataMgr/ChunkMetadata.h index 676272135d..0eb3b24b20 100644 --- a/DataMgr/ChunkMetadata.h +++ b/DataMgr/ChunkMetadata.h @@ -20,7 +20,7 @@ #include #include "../Shared/sqltypes.h" -#include +#include "Shared/Logger.h" struct ChunkStats { Datum min; diff --git a/DataMgr/DateDaysEncoder.h b/DataMgr/DateDaysEncoder.h index b8d8b2fba5..14c0ea2048 100644 --- a/DataMgr/DateDaysEncoder.h +++ b/DataMgr/DateDaysEncoder.h @@ -17,7 +17,8 @@ #ifndef DATE_DAYS_ENCODER_H #define DATE_DAYS_ENCODER_H -#include +#include "Shared/Logger.h" + #include #include #include "AbstractBuffer.h" @@ -154,4 +155,4 @@ class DateDaysEncoder : public Encoder { }; // DateDaysEncoder -#endif // DATE_DAYS_ENCODER_H \ No newline at end of file +#endif // DATE_DAYS_ENCODER_H diff --git a/DataMgr/Encoder.cpp b/DataMgr/Encoder.cpp index ea9f982602..c61dc5d439 100644 --- a/DataMgr/Encoder.cpp +++ b/DataMgr/Encoder.cpp @@ -15,12 +15,12 @@ */ #include "Encoder.h" -#include #include "ArrayNoneEncoder.h" #include "DateDaysEncoder.h" #include "FixedLengthArrayNoneEncoder.h" #include "FixedLengthEncoder.h" #include "NoneEncoder.h" +#include "Shared/Logger.h" #include "StringNoneEncoder.h" Encoder* Encoder::Create(Data_Namespace::AbstractBuffer* buffer, diff --git a/DataMgr/FileMgr/FileBuffer.cpp b/DataMgr/FileMgr/FileBuffer.cpp index d1913291b6..4475ff3b12 100644 --- a/DataMgr/FileMgr/FileBuffer.cpp +++ b/DataMgr/FileMgr/FileBuffer.cpp @@ -21,7 +21,6 @@ */ #include "FileBuffer.h" -#include #include #include #include diff --git a/DataMgr/FileMgr/FileBuffer.h b/DataMgr/FileMgr/FileBuffer.h index 78811c5032..d9c09370c5 100644 --- a/DataMgr/FileMgr/FileBuffer.h +++ b/DataMgr/FileMgr/FileBuffer.h @@ -25,6 +25,7 @@ #include "../AbstractBuffer.h" #include "Page.h" +#include "Shared/Logger.h" #include #include @@ -114,7 +115,10 @@ class FileBuffer : public AbstractBuffer { inline Data_Namespace::MemoryLevel getType() const override { return DISK_LEVEL; } /// Not implemented for FileMgr -- throws a runtime_error - int8_t* getMemoryPtr() override { LOG(FATAL) << "Operation not supported."; } + int8_t* getMemoryPtr() override { + LOG(FATAL) << "Operation not supported."; + return nullptr; // satisfy return-type warning + } /// Returns the number of pages in the FileBuffer. inline size_t pageCount() const override { return multiPages_.size(); } diff --git a/DataMgr/FileMgr/FileInfo.cpp b/DataMgr/FileMgr/FileInfo.cpp index b49afe60bf..66152dcd3d 100644 --- a/DataMgr/FileMgr/FileInfo.cpp +++ b/DataMgr/FileMgr/FileInfo.cpp @@ -15,7 +15,6 @@ */ #include "FileInfo.h" -#include #include #include "../../Shared/File.h" #include "FileMgr.h" @@ -77,7 +76,7 @@ void FileInfo::openExistingFile(std::vector& headerVec, for (size_t pageNum = 0; pageNum < numPages; ++pageNum) { int headerSize; -#define MAX_INTS_TO_READ 10 // currently use 1+6 ints + constexpr size_t MAX_INTS_TO_READ{10}; // currently use 1+6 ints int ints[MAX_INTS_TO_READ]; CHECK_EQ(fseek(f, pageNum * pageSize, SEEK_SET), 0); CHECK_EQ(fread(ints, sizeof(int), MAX_INTS_TO_READ, f), MAX_INTS_TO_READ); diff --git a/DataMgr/FileMgr/FileInfo.h b/DataMgr/FileMgr/FileInfo.h index 7ee45a0dab..601014ce35 100644 --- a/DataMgr/FileMgr/FileInfo.h +++ b/DataMgr/FileMgr/FileInfo.h @@ -26,6 +26,7 @@ #include #include "../../Shared/types.h" #include "Page.h" +#include "Shared/Logger.h" namespace File_Namespace { diff --git a/DataMgr/FileMgr/FileMgr.cpp b/DataMgr/FileMgr/FileMgr.cpp index b7106bc39a..c29e255d56 100644 --- a/DataMgr/FileMgr/FileMgr.cpp +++ b/DataMgr/FileMgr/FileMgr.cpp @@ -787,6 +787,7 @@ AbstractBuffer* FileMgr::putBuffer(const ChunkKey& key, AbstractBuffer* FileMgr::alloc(const size_t numBytes = 0) { LOG(FATAL) << "Operation not supported"; + return nullptr; // satisfy return-type warning } void FileMgr::free(AbstractBuffer* buffer) { diff --git a/DataMgr/FileMgr/GlobalFileMgr.h b/DataMgr/FileMgr/GlobalFileMgr.h index b73d18ec48..2224ec4c07 100644 --- a/DataMgr/FileMgr/GlobalFileMgr.h +++ b/DataMgr/FileMgr/GlobalFileMgr.h @@ -104,6 +104,7 @@ class GlobalFileMgr : public AbstractBufferMgr { // implements // Buffer API AbstractBuffer* alloc(const size_t numBytes) override { LOG(FATAL) << "Operation not supported"; + return nullptr; // satisfy return-type warning } void free(AbstractBuffer* buffer) override { LOG(FATAL) << "Operation not supported"; } diff --git a/DataMgr/FileMgr/Page.h b/DataMgr/FileMgr/Page.h index 31cd097e6b..e4b7c43e9e 100644 --- a/DataMgr/FileMgr/Page.h +++ b/DataMgr/FileMgr/Page.h @@ -23,7 +23,8 @@ #ifndef DATAMGR_MEMORY_FILE_PAGE_H #define DATAMGR_MEMORY_FILE_PAGE_H -#include +#include "Shared/Logger.h" + #include #include #include diff --git a/DataMgr/FixedLengthArrayNoneEncoder.h b/DataMgr/FixedLengthArrayNoneEncoder.h index 44db083fea..bf51cd4abd 100644 --- a/DataMgr/FixedLengthArrayNoneEncoder.h +++ b/DataMgr/FixedLengthArrayNoneEncoder.h @@ -24,7 +24,8 @@ #ifndef FIXED_LENGTH_ARRAY_NONE_ENCODER_H #define FIXED_LENGTH_ARRAY_NONE_ENCODER_H -#include +#include "Shared/Logger.h" + #include #include #include diff --git a/DataMgr/FixedLengthEncoder.h b/DataMgr/FixedLengthEncoder.h index 263b7b6a9f..070772da8a 100644 --- a/DataMgr/FixedLengthEncoder.h +++ b/DataMgr/FixedLengthEncoder.h @@ -16,7 +16,8 @@ #ifndef FIXED_LENGTH_ENCODER_H #define FIXED_LENGTH_ENCODER_H -#include +#include "Shared/Logger.h" + #include #include #include diff --git a/DataMgr/StringNoneEncoder.h b/DataMgr/StringNoneEncoder.h index cb80f89a9c..ff0e9130f0 100644 --- a/DataMgr/StringNoneEncoder.h +++ b/DataMgr/StringNoneEncoder.h @@ -23,8 +23,8 @@ **/ #ifndef STRING_NONE_ENCODER_H #define STRING_NONE_ENCODER_H +#include "Shared/Logger.h" -#include #include #include #include diff --git a/Distributed/os/LeafAggregator.h b/Distributed/os/LeafAggregator.h index 3024b59c4d..7f7be0fc4d 100644 --- a/Distributed/os/LeafAggregator.h +++ b/Distributed/os/LeafAggregator.h @@ -24,7 +24,7 @@ #include "QueryEngine/TargetMetaInfo.h" #include "gen-cpp/MapD.h" -#include +#include "Shared/Logger.h" namespace Catalog_Namespace { class SessionInfo; diff --git a/Fragmenter/InsertOrderFragmenter.cpp b/Fragmenter/InsertOrderFragmenter.cpp index 11533e4c4f..109d23f7be 100644 --- a/Fragmenter/InsertOrderFragmenter.cpp +++ b/Fragmenter/InsertOrderFragmenter.cpp @@ -15,7 +15,6 @@ */ #include "InsertOrderFragmenter.h" -#include #include #include #include @@ -28,6 +27,7 @@ #include "../LockMgr/LockMgr.h" #include "../Shared/checked_alloc.h" #include "../Shared/thread_count.h" +#include "Shared/Logger.h" #include diff --git a/Fragmenter/SortedOrderFragmenter.cpp b/Fragmenter/SortedOrderFragmenter.cpp index 6437e7973d..755435bf11 100644 --- a/Fragmenter/SortedOrderFragmenter.cpp +++ b/Fragmenter/SortedOrderFragmenter.cpp @@ -214,7 +214,7 @@ void SortedOrderFragmenter::sortData(InsertData& insertDataStruct) { std::vector indexes(insertDataStruct.numRows); std::iota(indexes.begin(), indexes.end(), 0); const auto dist = std::distance(insertDataStruct.columnIds.begin(), it); - CHECK_LT(dist, insertDataStruct.data.size()); + CHECK_LT(static_cast(dist), insertDataStruct.data.size()); sortIndexes(physical_cd, indexes, insertDataStruct.data[dist]); // shuffle rows of all columns for (size_t i = 0; i < insertDataStruct.columnIds.size(); ++i) { diff --git a/Fragmenter/UpdelStorage.cpp b/Fragmenter/UpdelStorage.cpp index b993bf4681..e03c429693 100644 --- a/Fragmenter/UpdelStorage.cpp +++ b/Fragmenter/UpdelStorage.cpp @@ -28,9 +28,9 @@ #include "QueryEngine/TargetValue.h" #include "Shared/ConfigResolve.h" #include "Shared/DateConverters.h" +#include "Shared/Logger.h" #include "Shared/TypedDataAccessors.h" #include "Shared/thread_count.h" -#include "Shared/unreachable.h" #include "TargetValueConvertersFactories.h" namespace Fragmenter_Namespace { diff --git a/Import/CMakeLists.txt b/Import/CMakeLists.txt index 30c650833d..f9ca190f76 100644 --- a/Import/CMakeLists.txt +++ b/Import/CMakeLists.txt @@ -14,7 +14,7 @@ endif() add_library(CsvImport Importer.cpp Importer.h ${S3Archive}) -target_link_libraries(CsvImport mapd_thrift Shared Catalog Chunk DataMgr StringDictionary ${GDAL_LIBRARIES} ${Glog_LIBRARIES} ${CMAKE_DL_LIBS} +target_link_libraries(CsvImport mapd_thrift Shared Catalog Chunk DataMgr StringDictionary ${GDAL_LIBRARIES} ${CMAKE_DL_LIBS} ${LibArchive_LIBRARIES} ${IMPORT_LIBRARIES} ${Arrow_LIBRARIES}) install(DIRECTORY ${CMAKE_SOURCE_DIR}/ThirdParty/gdal-data DESTINATION "ThirdParty") @@ -27,9 +27,9 @@ add_library(RowToColumn RowToColumnLoader.cpp RowToColumnLoader.h) target_link_libraries(RowToColumn ThriftClient) add_executable(StreamImporter StreamImporter.cpp) -target_link_libraries(StreamImporter RowToColumn mapd_thrift Shared ${Glog_LIBRARIES} ${CMAKE_DL_LIBS} ${Boost_LIBRARIES} ${PROFILER_LIBS}) +target_link_libraries(StreamImporter RowToColumn mapd_thrift Shared ${CMAKE_DL_LIBS} ${Boost_LIBRARIES} ${PROFILER_LIBS}) add_executable(KafkaImporter KafkaImporter.cpp) -target_link_libraries(KafkaImporter RowToColumn mapd_thrift rdkafka++ Shared ${Glog_LIBRARIES} ${CMAKE_DL_LIBS} ${Boost_LIBRARIES} ${PROFILER_LIBS}) +target_link_libraries(KafkaImporter RowToColumn mapd_thrift rdkafka++ Shared ${CMAKE_DL_LIBS} ${Boost_LIBRARIES} ${PROFILER_LIBS}) install(TARGETS StreamImporter KafkaImporter DESTINATION bin) diff --git a/Import/Importer.cpp b/Import/Importer.cpp index c65f1f619a..56156b58c6 100644 --- a/Import/Importer.cpp +++ b/Import/Importer.cpp @@ -21,7 +21,6 @@ */ #include -#include #include #include #include @@ -34,7 +33,6 @@ #include #include #include -#include #include #include #include @@ -58,11 +56,9 @@ #include "../Shared/scope.h" #include "../Shared/shard_key.h" #include "../Shared/thread_count.h" -#include "../Shared/unreachable.h" +#include "Shared/Logger.h" #include "Shared/SqlTypesLayout.h" -#include -#include #include "Importer.h" #include "QueryRunner/QueryRunner.h" #include "Utils/ChunkAccessorTable.h" @@ -75,8 +71,6 @@ #include "../Archive/S3Archive.h" #include "ArrowImporter.h" -using std::ostream; - inline auto get_filesize(const std::string& file_path) { boost::filesystem::path boost_file_path{file_path}; boost::system::error_code ec; @@ -116,6 +110,20 @@ using OGRSpatialReferenceUqPtr = } // namespace +// For logging std::vector row. +namespace boost { +namespace log { +formatting_ostream& operator<<(formatting_ostream& out, std::vector& row) { + out << '['; + for (size_t i = 0; i < row.size(); ++i) { + out << (i ? ", " : "") << row[i]; + } + out << ']'; + return out; +} +} // namespace log +} // namespace boost + namespace Importer_NS { using FieldNameToIndexMapType = std::map; @@ -1467,20 +1475,6 @@ void TypedImportBuffer::add_value(const ColumnDescriptor* cd, } } -template -ostream& operator<<(ostream& out, const std::vector& v) { - out << "["; - size_t last = v.size() - 1; - for (size_t i = 0; i < v.size(); ++i) { - out << v[i]; - if (i != last) { - out << ", "; - } - } - out << "]"; - return out; -} - bool importGeoFromLonLat(double lon, double lat, std::vector& coords) { if (std::isinf(lat) || std::isnan(lat) || std::isinf(lon) || std::isnan(lon)) { return false; @@ -4522,7 +4516,7 @@ ImportStatus Importer::importGDAL( #endif // make an import buffer for each thread - CHECK_EQ(import_buffers_vec.size(), 0); + CHECK_EQ(import_buffers_vec.size(), 0u); import_buffers_vec.resize(max_threads); for (size_t i = 0; i < max_threads; i++) { for (const auto cd : loader->get_column_descs()) { diff --git a/Import/Importer.h b/Import/Importer.h index 7dd9db5b0a..c8314e72e5 100644 --- a/Import/Importer.h +++ b/Import/Importer.h @@ -22,14 +22,12 @@ #ifndef _IMPORTER_H_ #define _IMPORTER_H_ -#include "../Shared/fixautotools.h" +#include "Shared/Logger.h" +#include "Shared/fixautotools.h" #include -#include #include -#include "../Shared/fixautotools.h" - #include #include #include diff --git a/Import/KafkaImporter.cpp b/Import/KafkaImporter.cpp index 24f9aa36e5..8503eefcab 100644 --- a/Import/KafkaImporter.cpp +++ b/Import/KafkaImporter.cpp @@ -23,7 +23,6 @@ * Copyright (c) 2017 MapD Technologies, Inc. All rights reserved. **/ -#include #include #include #include @@ -33,6 +32,7 @@ #include #include "RowToColumnLoader.h" +#include "Shared/Logger.h" #include "Shared/ThriftClient.h" #include "Shared/sqltypes.h" @@ -499,8 +499,6 @@ int main(int argc, char** argv) { transformations; ThriftConnectionType conn_type; - google::InitGoogleLogging(argv[0]); - namespace po = boost::program_options; po::options_description desc("Options"); @@ -570,6 +568,10 @@ int main(int argc, char** argv) { positionalOptions.add("table", 1); positionalOptions.add("database", 1); + logger::LogOptions log_options(argv[0]); + log_options.max_files_ = 0; // stderr only + desc.add(log_options.get_options()); + po::variables_map vm; try { @@ -603,6 +605,8 @@ int main(int argc, char** argv) { return 1; } + logger::init(log_options); + if (http) { conn_type = ThriftConnectionType::HTTP; } else if (https) { diff --git a/Import/RowToColumnLoader.cpp b/Import/RowToColumnLoader.cpp index 8a648944a0..2b030a86ca 100644 --- a/Import/RowToColumnLoader.cpp +++ b/Import/RowToColumnLoader.cpp @@ -23,31 +23,8 @@ * Copyright (c) 2017 MapD Technologies, Inc. All rights reserved. **/ -#include -#include -#include -#include -#include -#include -#include -#include - -#include "Shared/mapd_shared_ptr.h" -#include "Shared/sqltypes.h" - -#include -#include - -#include - -// include files for Thrift and MapD Thrift Services -#include -#include -#include -#include "Importer.h" #include "RowToColumnLoader.h" -#include "gen-cpp/MapD.h" -#include "gen-cpp/mapd_types.h" +#include "Shared/Logger.h" using namespace ::apache::thrift; using namespace ::apache::thrift::protocol; @@ -84,6 +61,7 @@ SQLTypes get_sql_types(const TColumnType& ct) { return SQLTypes::kSMALLINT; default: LOG(FATAL) << "Unsupported TColumnType found, should not be possible"; + return SQLTypes::kNULLT; // satisfy return-type warning } } @@ -181,7 +159,6 @@ void populate_TColumn(TStringValue ts, TColumn& input_col, const Importer_NS::CopyParams& copy_params) { // create datum and push data to column structure from row data - switch (column_type_info.get_type()) { case SQLTypes::kARRAY: LOG(FATAL) << "Trying to process ARRAY at item level something is wrong"; diff --git a/Import/RowToColumnLoader.h b/Import/RowToColumnLoader.h index 6755bd95ff..56f9a8f715 100644 --- a/Import/RowToColumnLoader.h +++ b/Import/RowToColumnLoader.h @@ -26,7 +26,8 @@ * Copyright (c) 2017 MapD Technologies, Inc. All rights reserved. **/ -#include +#include "Shared/Logger.h" + #include #include #include diff --git a/Import/StreamImporter.cpp b/Import/StreamImporter.cpp index 66deea3a48..f432a2f13c 100644 --- a/Import/StreamImporter.cpp +++ b/Import/StreamImporter.cpp @@ -23,7 +23,6 @@ * Copyright (c) 2017 MapD Technologies, Inc. All rights reserved. **/ -#include #include #include #include @@ -33,6 +32,7 @@ #include #include "RowToColumnLoader.h" +#include "Shared/Logger.h" #include "Shared/ThriftClient.h" #include "Shared/sqltypes.h" @@ -209,8 +209,6 @@ int main(int argc, char** argv) { transformations; ThriftConnectionType conn_type; - google::InitGoogleLogging(argv[0]); - namespace po = boost::program_options; po::options_description desc("Options"); @@ -271,6 +269,10 @@ int main(int argc, char** argv) { positionalOptions.add("table", 1); positionalOptions.add("database", 1); + logger::LogOptions log_options(argv[0]); + log_options.max_files_ = 0; // stderr only + desc.add(log_options.get_options()); + po::variables_map vm; try { @@ -304,6 +306,8 @@ int main(int argc, char** argv) { return 1; } + logger::init(log_options); + if (http) { conn_type = ThriftConnectionType::HTTP; } else if (https) { diff --git a/MapDServer.cpp b/MapDServer.cpp index 31f29049f3..b717d5fea5 100644 --- a/MapDServer.cpp +++ b/MapDServer.cpp @@ -30,6 +30,7 @@ #include "MapDRelease.h" +#include "Shared/Logger.h" #include "Shared/MapDParameters.h" #include "Shared/file_delete.h" #include "Shared/mapd_shared_ptr.h" @@ -87,7 +88,7 @@ void mapd_signal_handler(int signal_number) { LOG(INFO) << "Interrupt signal (" << signal_number << ") received.\n"; shutdown_handler(); // shut down logging force a flush - google::ShutdownGoogleLogging(); + logger::shutdown(); // terminate program if (signal_number == SIGTERM) { std::exit(EXIT_SUCCESS); @@ -191,11 +192,10 @@ namespace po = boost::program_options; class MapDProgramOptions { public: - MapDProgramOptions() { + MapDProgramOptions(char const* argv0) : log_options_(argv0) { fillOptions(); fillAdvancedOptions(); } - int http_port = 6278; size_t reserved_gpu_mem = 1 << 27; std::string base_path; @@ -249,16 +249,20 @@ class MapDProgramOptions { private: void fillOptions(); void fillAdvancedOptions(); + [[deprecated( + "Glog was replaced by boost.log. See source comments for more info.")]] void + temporarily_support_deprecated_log_options_201904(); po::options_description help_desc; po::options_description developer_desc; + logger::LogOptions log_options_; po::variables_map vm; public: std::vector db_leaves; std::vector string_leaves; - bool parse_command_line(int argc, char** argv, int& return_code); + boost::optional parse_command_line(int argc, char const* const* argv); }; void MapDProgramOptions::fillOptions() { @@ -375,11 +379,6 @@ void MapDProgramOptions::fillOptions() { "Upperbound on the number of rows that should pass the filter " "if the selectivity is less than " "the high fraction threshold."); - help_desc.add_options()( - "flush-log", - po::value(&flush_log)->default_value(flush_log)->implicit_value(true), - "Immediately flush logs to disk. Set to false if this is a performance " - "bottleneck."); help_desc.add_options()("from-table-reordering", po::value(&g_from_table_reordering) ->default_value(g_from_table_reordering) @@ -466,6 +465,13 @@ void MapDProgramOptions::fillOptions() { ->implicit_value(true), "Write additional debug log messages to server logs."); help_desc.add_options()("version,v", "Print Version Number."); + + help_desc.add_options()( + "flush-log", + po::value(&flush_log)->default_value(flush_log)->implicit_value(true), + R"(DEPRECATED - Immediately flush logs to disk. Set to false if this is a performance bottleneck.)" + " Replaced by log-auto-flush."); + help_desc.add(log_options_.get_options()); } void MapDProgramOptions::fillAdvancedOptions() { @@ -564,25 +570,21 @@ std::stringstream sanitize_config_file(std::ifstream& in) { return ss; } -} // namespace - -bool trim_and_check_file_exists(std::string& filename, - const std::string desc, - int& return_code) { +bool trim_and_check_file_exists(std::string& filename, const std::string desc) { if (!filename.empty()) { boost::algorithm::trim_if(filename, boost::is_any_of("\"'")); if (!boost::filesystem::exists(filename)) { LOG(ERROR) << desc << " " << filename << " does not exist."; - return_code = 1; return false; } } return true; } -bool MapDProgramOptions::parse_command_line(int argc, char** argv, int& return_code) { - return_code = 0; +} // namespace +boost::optional MapDProgramOptions::parse_command_line(int argc, + char const* const* argv) { po::positional_options_description positional_options; positional_options.add("data", 1); @@ -607,17 +609,14 @@ bool MapDProgramOptions::parse_command_line(int argc, char** argv, int& return_c settings_file.close(); } - if (!trim_and_check_file_exists( - mapd_parameters.ssl_cert_file, "ssl cert file", return_code)) { - return false; + if (!trim_and_check_file_exists(mapd_parameters.ssl_cert_file, "ssl cert file")) { + return 1; } - if (!trim_and_check_file_exists( - mapd_parameters.ssl_trust_store, "ssl trust store", return_code)) { - return false; + if (!trim_and_check_file_exists(mapd_parameters.ssl_trust_store, "ssl trust store")) { + return 1; } - if (!trim_and_check_file_exists( - mapd_parameters.ssl_key_file, "ssl key file", return_code)) { - return false; + if (!trim_and_check_file_exists(mapd_parameters.ssl_key_file, "ssl key file")) { + return 1; } if (vm.count("help")) { @@ -626,8 +625,7 @@ bool MapDProgramOptions::parse_command_line(int argc, char** argv, int& return_c << std::endl << std::endl; std::cout << help_desc << std::endl; - return_code = 0; - return false; + return 0; } if (vm.count("dev-options")) { std::cout << "Usage: omnisci_server [-p ] " @@ -635,8 +633,7 @@ bool MapDProgramOptions::parse_command_line(int argc, char** argv, int& return_c << std::endl << std::endl; std::cout << developer_desc << std::endl; - return_code = 0; - return false; + return 0; } if (vm.count("version")) { std::cout << "OmniSci Version: " << MAPD_RELEASE << std::endl; @@ -648,14 +645,12 @@ bool MapDProgramOptions::parse_command_line(int argc, char** argv, int& return_c g_dynamic_watchdog_time_limit = dynamic_watchdog_time_limit; } catch (po::error& e) { std::cerr << "Usage Error: " << e.what() << std::endl; - return_code = 1; - return false; + return 1; } if (g_hll_precision_bits < 1 || g_hll_precision_bits > 16) { std::cerr << "hll-precision-bits must be between 1 and 16." << std::endl; - return_code = 1; - return false; + return 1; } boost::algorithm::trim_if(base_path, boost::is_any_of("\"'")); @@ -663,8 +658,7 @@ bool MapDProgramOptions::parse_command_line(int argc, char** argv, int& return_c if (!boost::filesystem::exists(data_path)) { std::cerr << "OmniSci data directory does not exist at '" << base_path << "'. Run initdb " << base_path << std::endl; - return_code = 1; - return false; + return 1; } const auto lock_file = boost::filesystem::path(base_path) / "omnisci_server_pid.lck"; @@ -674,16 +668,14 @@ bool MapDProgramOptions::parse_command_line(int argc, char** argv, int& return_c auto err = std::string("Failed to open PID file ") + std::string(lock_file.c_str()) + std::string(". ") + strerror(errno) + "."; std::cerr << err << std::endl; - return_code = 1; - return false; + return 1; } if (lockf(pid_fd, F_TLOCK, 0) == -1) { auto err = std::string("Another OmniSci Server is using data directory ") + base_path + std::string("."); std::cerr << err << std::endl; close(pid_fd); - return_code = 1; - return false; + return 1; } if (ftruncate(pid_fd, 0) == -1) { auto err = std::string("Failed to truncate PID file ") + @@ -691,35 +683,27 @@ bool MapDProgramOptions::parse_command_line(int argc, char** argv, int& return_c std::string("."); std::cerr << err << std::endl; close(pid_fd); - return_code = 1; - return false; + return 1; } if (write(pid_fd, pid.c_str(), pid.length()) == -1) { auto err = std::string("Failed to write PID file ") + std::string(lock_file.c_str()) + ". " + strerror(errno) + "."; std::cerr << err << std::endl; close(pid_fd); - return_code = 1; - return false; + return 1; } - const auto log_path = boost::filesystem::path(base_path) / "mapd_log"; - (void)boost::filesystem::create_directory(log_path); - FLAGS_log_dir = log_path.c_str(); - if (flush_log) { - FLAGS_logbuflevel = -1; - } - if (verbose_logging) { - FLAGS_v = 1; + temporarily_support_deprecated_log_options_201904(); + if (verbose_logging && logger::Severity::DEBUG1 < log_options_.severity_) { + log_options_.severity_ = logger::Severity::DEBUG1; } - // Initialize Google's logging library. - google::InitGoogleLogging(argv[0]); + log_options_.set_base_path(base_path); + logger::init(log_options_); boost::algorithm::trim_if(db_query_file, boost::is_any_of("\"'")); if (db_query_file.length() > 0 && !boost::filesystem::exists(db_query_file)) { LOG(ERROR) << "File containing DB queries " << db_query_file << " does not exist."; - return_code = 1; - return false; + return 1; } const auto db_file = boost::filesystem::path(base_path) / "mapd_catalogs" / OMNISCI_SYSTEM_CATALOG; @@ -729,8 +713,7 @@ bool MapDProgramOptions::parse_command_line(int argc, char** argv, int& return_c if (!boost::filesystem::exists(db_file)) { LOG(ERROR) << "OmniSci system catalog " << OMNISCI_SYSTEM_CATALOG << " does not exist."; - return_code = 1; - return false; + return 1; } } } @@ -761,8 +744,7 @@ bool MapDProgramOptions::parse_command_line(int argc, char** argv, int& return_c if (!boost::filesystem::exists(udf_file_name)) { LOG(ERROR) << "User defined function file " << udf_file_name << " does not exist."; - return_code = 1; - return false; + return 1; } LOG(INFO) << "User provided extension functions loaded from " << udf_file_name; @@ -777,22 +759,19 @@ bool MapDProgramOptions::parse_command_line(int argc, char** argv, int& return_c LOG(INFO) << " HA group id " << mapd_parameters.ha_group_id; if (mapd_parameters.ha_unique_server_id.empty()) { LOG(ERROR) << "Starting server in HA mode --ha-unique-server-id must be set "; - return_code = 5; - return false; + return 5; } else { LOG(INFO) << " HA unique server id " << mapd_parameters.ha_unique_server_id; } if (mapd_parameters.ha_brokers.empty()) { LOG(ERROR) << "Starting server in HA mode --ha-brokers must be set "; - return_code = 6; - return false; + return 6; } else { LOG(INFO) << " HA brokers " << mapd_parameters.ha_brokers; } if (mapd_parameters.ha_shared_data.empty()) { LOG(ERROR) << "Starting server in HA mode --ha-shared-data must be set "; - return_code = 7; - return false; + return 7; } else { LOG(INFO) << " HA shared data is " << mapd_parameters.ha_shared_data; } @@ -816,20 +795,31 @@ bool MapDProgramOptions::parse_command_line(int argc, char** argv, int& return_c boost::algorithm::trim_if(authMetadata.restToken, boost::is_any_of("\"'")); boost::algorithm::trim_if(authMetadata.restUrl, boost::is_any_of("\"'")); - return true; + return boost::none; +} + +// Deprecation plan. +// Once users have updated their options to not use flush_log: +// - Delete this function. +// - Delete MapDProgramOptions::flush_log and all references to it. +// (Replaced by LogOptions::auto_flush_.) +void MapDProgramOptions::temporarily_support_deprecated_log_options_201904() { + if (!flush_log) { + log_options_.auto_flush_ = false; + } } int main(int argc, char** argv) { - MapDProgramOptions prog_config_opts; - int return_code = 0; + MapDProgramOptions prog_config_opts(argv[0]); + + if (auto return_code = prog_config_opts.parse_command_line(argc, argv)) { + return *return_code; + } - if (!prog_config_opts.parse_command_line(argc, argv, return_code)) { - return return_code; - }; // rudimetary signal handling to try to guarantee the logging gets flushed to // files on shutdown register_signal_handler(); - google::InstallFailureFunction(&shutdown_handler); + logger::set_once_fatal_func(&shutdown_handler); // start background thread to clean up _DELETE_ME files std::atomic running{true}; diff --git a/Parser/ParserNode.h b/Parser/ParserNode.h index 5e358ca5bc..40501ed9ce 100644 --- a/Parser/ParserNode.h +++ b/Parser/ParserNode.h @@ -33,7 +33,6 @@ #include "../Distributed/AggregatedResult.h" #include "../Shared/sqldefs.h" #include "../Shared/sqltypes.h" -#include "../Shared/unreachable.h" #include "../Fragmenter/InsertDataLoader.h" diff --git a/Parser/ParserWrapper.cpp b/Parser/ParserWrapper.cpp index 8c94c132a8..cba4b4d92d 100644 --- a/Parser/ParserWrapper.cpp +++ b/Parser/ParserWrapper.cpp @@ -26,8 +26,6 @@ #include -#include - using namespace std; const std::vector ParserWrapper::ddl_cmd = {"ALTER", diff --git a/QueryEngine/AggregateUtils.h b/QueryEngine/AggregateUtils.h index 84c4bc9da0..2e68f80f66 100644 --- a/QueryEngine/AggregateUtils.h +++ b/QueryEngine/AggregateUtils.h @@ -21,7 +21,7 @@ #include "BufferCompaction.h" #include "TypePunning.h" -#include +#include "Shared/Logger.h" inline void set_component(int8_t* group_by_buffer, const size_t comp_sz, diff --git a/QueryEngine/Allocators/CudaAllocator.cpp b/QueryEngine/Allocators/CudaAllocator.cpp index e20b451122..ea4ed13c4a 100644 --- a/QueryEngine/Allocators/CudaAllocator.cpp +++ b/QueryEngine/Allocators/CudaAllocator.cpp @@ -20,8 +20,7 @@ #include #include #include "../Rendering/RenderAllocator.h" - -#include +#include "Shared/Logger.h" CudaAllocator::CudaAllocator(Data_Namespace::DataMgr* data_mgr, const int device_id) : data_mgr_(data_mgr), device_id_(device_id) { diff --git a/QueryEngine/Allocators/DeviceAllocator.h b/QueryEngine/Allocators/DeviceAllocator.h index 417645be0b..104cd69055 100644 --- a/QueryEngine/Allocators/DeviceAllocator.h +++ b/QueryEngine/Allocators/DeviceAllocator.h @@ -22,7 +22,7 @@ #pragma once -#include +#include "Shared/Logger.h" #ifdef HAVE_CUDA #include diff --git a/QueryEngine/Allocators/ThrustAllocator.cpp b/QueryEngine/Allocators/ThrustAllocator.cpp index 372b73a132..996d80054f 100644 --- a/QueryEngine/Allocators/ThrustAllocator.cpp +++ b/QueryEngine/Allocators/ThrustAllocator.cpp @@ -16,12 +16,11 @@ #include "ThrustAllocator.h" #include "CudaAllocator.h" +#include "Shared/Logger.h" #include #include -#include - int8_t* ThrustAllocator::allocate(std::ptrdiff_t num_bytes) { #ifdef HAVE_CUDA if (!data_mgr_) { // only for unit tests @@ -79,4 +78,4 @@ ThrustAllocator::~ThrustAllocator() { CHECK_EQ(CUDA_SUCCESS, err); } #endif // HAVE_CUDA -} \ No newline at end of file +} diff --git a/QueryEngine/BaselineJoinHashTable.cpp b/QueryEngine/BaselineJoinHashTable.cpp index d85ae665ca..1e4724cda7 100644 --- a/QueryEngine/BaselineJoinHashTable.cpp +++ b/QueryEngine/BaselineJoinHashTable.cpp @@ -22,8 +22,6 @@ #include "HashJoinKeyHandlers.h" #include "JoinHashTableGpuUtils.h" -#include "Shared/unreachable.h" - #include std::vectorget_type_info(); if (inner_col_ti.get_logical_size() > 4) { - CHECK_EQ(size_t(8), inner_col_ti.get_logical_size()); + CHECK_EQ(8, inner_col_ti.get_logical_size()); return 8; } } diff --git a/QueryEngine/CMakeLists.txt b/QueryEngine/CMakeLists.txt index 44bd43417b..c1135a7194 100644 --- a/QueryEngine/CMakeLists.txt +++ b/QueryEngine/CMakeLists.txt @@ -145,6 +145,14 @@ if(ENABLE_DECODERS_BOUNDS_CHECKING) list(APPEND MAPD_DEFINITIONS "-DWITH_DECODERS_BOUNDS_CHECKING") endif() +if(NOT PREFER_STATIC_LIBS) + list(APPEND MAPD_DEFINITIONS "-DBOOST_LOG_DYN_LINK") +endif() + +if (SUPPRESS_NULL_LOGGER_DEPRECATION_WARNINGS) + list(APPEND MAPD_DEFINITIONS "-DSUPPRESS_NULL_LOGGER_DEPRECATION_WARNINGS") +endif() + if(ENABLE_LLVM_DBG) set(CLANG_SDK_INC "-I/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.11.sdk/usr/include/") set(CLANG_CRT_INC "-I/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/include/c++/v1/") @@ -380,4 +388,4 @@ add_custom_command( ) add_executable(group_by_hash_test ${group_by_hash_test_files}) -target_link_libraries(group_by_hash_test gtest ${Glog_LIBRARIES} ${Boost_LIBRARIES} ${PROFILER_LIBS}) +target_link_libraries(group_by_hash_test gtest Shared ${Boost_LIBRARIES} ${PROFILER_LIBS}) diff --git a/QueryEngine/CalciteDeserializerUtils.cpp b/QueryEngine/CalciteDeserializerUtils.cpp index 3ed6e337cd..02bc4ca03f 100644 --- a/QueryEngine/CalciteDeserializerUtils.cpp +++ b/QueryEngine/CalciteDeserializerUtils.cpp @@ -17,6 +17,7 @@ #include "CalciteDeserializerUtils.h" #include "../Analyzer/Analyzer.h" +#include "Shared/Logger.h" #include diff --git a/QueryEngine/CalciteDeserializerUtils.h b/QueryEngine/CalciteDeserializerUtils.h index ba472c5866..4cea9106e6 100644 --- a/QueryEngine/CalciteDeserializerUtils.h +++ b/QueryEngine/CalciteDeserializerUtils.h @@ -22,8 +22,7 @@ #include "../Shared/sqldefs.h" #include "../Shared/sqltypes.h" - -#include +#include "Shared/Logger.h" inline SQLOps to_sql_op(const std::string& op_str) { if (op_str == std::string(">")) { diff --git a/QueryEngine/CardinalityEstimator.h b/QueryEngine/CardinalityEstimator.h index 13e806322e..db4ae6acac 100644 --- a/QueryEngine/CardinalityEstimator.h +++ b/QueryEngine/CardinalityEstimator.h @@ -28,8 +28,7 @@ #include "RelAlgExecutionUnit.h" #include "../Analyzer/Analyzer.h" - -#include +#include "Shared/Logger.h" class CardinalityEstimationRequired : public std::runtime_error { public: diff --git a/QueryEngine/Codec.cpp b/QueryEngine/Codec.cpp index 9ea8310415..6d07a0350a 100644 --- a/QueryEngine/Codec.cpp +++ b/QueryEngine/Codec.cpp @@ -16,8 +16,8 @@ #include "Codec.h" #include "LLVMGlobalContext.h" +#include "Shared/Logger.h" -#include #include #include #include diff --git a/QueryEngine/ColumnIR.cpp b/QueryEngine/ColumnIR.cpp index cc64f19197..5bce093111 100644 --- a/QueryEngine/ColumnIR.cpp +++ b/QueryEngine/ColumnIR.cpp @@ -107,7 +107,7 @@ std::vector CodeGenerator::codegenColVar(const Analyzer::ColumnVar const bool hoist_literals = co.hoist_literals_; auto col_id = col_var->get_column_id(); const int rte_idx = adjusted_range_table_index(col_var); - CHECK_LT(rte_idx, cgen_state_->frag_offsets_.size()); + CHECK_LT(static_cast(rte_idx), cgen_state_->frag_offsets_.size()); const auto catalog = executor()->getCatalog(); CHECK(catalog); if (col_var->get_table_id() > 0) { @@ -403,8 +403,9 @@ llvm::Value* CodeGenerator::codgenAdjustFixedEncNull(llvm::Value* val, } llvm::Value* CodeGenerator::foundOuterJoinMatch(const ssize_t nesting_level) const { - CHECK_GE(nesting_level, size_t(0)); - CHECK_LE(nesting_level, cgen_state_->outer_join_match_found_per_level_.size()); + CHECK_GE(nesting_level, ssize_t(0)); + CHECK_LE(nesting_level, + static_cast(cgen_state_->outer_join_match_found_per_level_.size())); return cgen_state_->outer_join_match_found_per_level_[nesting_level - 1]; } diff --git a/QueryEngine/ConstantIR.cpp b/QueryEngine/ConstantIR.cpp index 359ce298b5..b1adb7b3d9 100644 --- a/QueryEngine/ConstantIR.cpp +++ b/QueryEngine/ConstantIR.cpp @@ -218,7 +218,7 @@ std::vector CodeGenerator::codegenHoistedConstantsPlaceholders( std::string literal_name = "literal_" + std::to_string(lit_off); if (type_info.is_string() && enc_type != kENCODING_DICT) { - CHECK_EQ(literal_loads.size(), 3); + CHECK_EQ(literal_loads.size(), 3u); llvm::Value* var_start = literal_loads[0]; llvm::Value* var_start_address = literal_loads[1]; @@ -249,7 +249,7 @@ std::vector CodeGenerator::codegenHoistedConstantsPlaceholders( if (type_info.is_array() && (enc_type == kENCODING_NONE || enc_type == kENCODING_GEOINT)) { - CHECK_EQ(literal_loads.size(), 2); + CHECK_EQ(literal_loads.size(), 2u); llvm::Value* var_start_address = literal_loads[0]; llvm::Value* var_length = literal_loads[1]; @@ -271,7 +271,7 @@ std::vector CodeGenerator::codegenHoistedConstantsPlaceholders( return {placeholder0, placeholder1}; } - CHECK_EQ(literal_loads.size(), 1); + CHECK_EQ(literal_loads.size(), 1u); llvm::Value* to_return_lv = literal_loads[0]; auto placeholder0 = cgen_state_->ir_builder_.CreateLoad( diff --git a/QueryEngine/DateAdd.cpp b/QueryEngine/DateAdd.cpp index 5e36d32bb8..ac15764ddd 100644 --- a/QueryEngine/DateAdd.cpp +++ b/QueryEngine/DateAdd.cpp @@ -20,7 +20,7 @@ #ifdef EXECUTE_INCLUDE #ifndef __CUDACC__ -#include +#include // abort() #endif DEVICE diff --git a/QueryEngine/DateTimePlusRewrite.cpp b/QueryEngine/DateTimePlusRewrite.cpp index 3803261b17..06ce6c5e31 100644 --- a/QueryEngine/DateTimePlusRewrite.cpp +++ b/QueryEngine/DateTimePlusRewrite.cpp @@ -19,11 +19,10 @@ #include "../Analyzer/Analyzer.h" #include "../Parser/ParserNode.h" +#include "Shared/Logger.h" #include "DateTimeTranslator.h" -#include - namespace { const Analyzer::Expr* remove_truncate_int(const Analyzer::Expr* expr) { diff --git a/QueryEngine/DateTimeUtils.h b/QueryEngine/DateTimeUtils.h index 64e01525e2..76b488caa3 100644 --- a/QueryEngine/DateTimeUtils.h +++ b/QueryEngine/DateTimeUtils.h @@ -22,9 +22,9 @@ #include #include #include +#include #include "../Shared/sqldefs.h" -#include "../Shared/unreachable.h" namespace { @@ -56,7 +56,7 @@ constexpr inline int64_t get_timestamp_precision_scale(const int32_t dimen) { case 9: return kNanoSecsPerSec; default: - UNREACHABLE(); + throw std::runtime_error("Unknown dimen = " + std::to_string(dimen)); } return -1; } @@ -70,7 +70,7 @@ constexpr inline int64_t get_dateadd_timestamp_precision_scale(const DateaddFiel case daNANOSECOND: return kNanoSecsPerSec; default: - UNREACHABLE(); + throw std::runtime_error("Unknown field = " + std::to_string(field)); } return -1; } @@ -84,7 +84,7 @@ constexpr inline int64_t get_extract_timestamp_precision_scale(const ExtractFiel case kNANOSECOND: return kNanoSecsPerSec; default: - UNREACHABLE(); + throw std::runtime_error("Unknown field = " + std::to_string(field)); } return -1; } @@ -114,7 +114,7 @@ const inline std::pair get_dateadd_high_precision_adjusted_scal case 3: return {kDIVIDE, kMicroSecsPerSec}; default: - UNREACHABLE(); + throw std::runtime_error("Unknown dimen = " + std::to_string(dimen)); } case daMICROSECOND: switch (dimen) { @@ -125,7 +125,7 @@ const inline std::pair get_dateadd_high_precision_adjusted_scal case 3: return {kDIVIDE, kMilliSecsPerSec}; default: - UNREACHABLE(); + throw std::runtime_error("Unknown dimen = " + std::to_string(dimen)); } case daMILLISECOND: switch (dimen) { @@ -136,10 +136,10 @@ const inline std::pair get_dateadd_high_precision_adjusted_scal case 3: return {}; default: - UNREACHABLE(); + throw std::runtime_error("Unknown dimen = " + std::to_string(dimen)); } default: - UNREACHABLE(); + throw std::runtime_error("Unknown field = " + std::to_string(field)); } return {}; } diff --git a/QueryEngine/DateTruncate.cpp b/QueryEngine/DateTruncate.cpp index 8ceba192bd..74adb91379 100644 --- a/QueryEngine/DateTruncate.cpp +++ b/QueryEngine/DateTruncate.cpp @@ -15,11 +15,10 @@ */ #include "DateTruncate.h" -#include "../Shared/unreachable.h" #include "ExtractFromTime.h" #ifndef __CUDACC__ -#include +#include // abort() #endif #include diff --git a/QueryEngine/Descriptors/ColSlotContext.cpp b/QueryEngine/Descriptors/ColSlotContext.cpp index 3592bdadc5..112adf12d3 100644 --- a/QueryEngine/Descriptors/ColSlotContext.cpp +++ b/QueryEngine/Descriptors/ColSlotContext.cpp @@ -195,7 +195,7 @@ size_t ColSlotContext::getCompactByteWidth() const { if (slot_size.padded_size == 0) { continue; } - CHECK_EQ(slot_size.padded_size, compact_width); + CHECK_EQ(static_cast(slot_size.padded_size), compact_width); } return compact_width; } diff --git a/QueryEngine/Descriptors/ColSlotContext.h b/QueryEngine/Descriptors/ColSlotContext.h index 47feedf30e..ca0bc3fd08 100644 --- a/QueryEngine/Descriptors/ColSlotContext.h +++ b/QueryEngine/Descriptors/ColSlotContext.h @@ -24,7 +24,7 @@ #pragma once -#include +#include "Shared/Logger.h" #include #include diff --git a/QueryEngine/Descriptors/CountDistinctDescriptor.h b/QueryEngine/Descriptors/CountDistinctDescriptor.h index 090f83352e..f8053944f8 100644 --- a/QueryEngine/Descriptors/CountDistinctDescriptor.h +++ b/QueryEngine/Descriptors/CountDistinctDescriptor.h @@ -28,8 +28,7 @@ #include "../BufferCompaction.h" #include "../CompilationOptions.h" - -#include +#include "Shared/Logger.h" inline size_t bitmap_bits_to_bytes(const size_t bitmap_sz) { size_t bitmap_byte_sz = bitmap_sz / 8; diff --git a/QueryEngine/Descriptors/InputDescriptors.h b/QueryEngine/Descriptors/InputDescriptors.h index 1bd73fcf42..3b7e79a7c8 100644 --- a/QueryEngine/Descriptors/InputDescriptors.h +++ b/QueryEngine/Descriptors/InputDescriptors.h @@ -18,8 +18,8 @@ #define QUERYENGINE_INPUTDESCRIPTORS_H #include "../Catalog/TableDescriptor.h" +#include "Shared/Logger.h" -#include #include enum class InputSourceType { TABLE, RESULT }; diff --git a/QueryEngine/Descriptors/QueryFragmentDescriptor.h b/QueryEngine/Descriptors/QueryFragmentDescriptor.h index 647649dee5..2ab86589f8 100644 --- a/QueryEngine/Descriptors/QueryFragmentDescriptor.h +++ b/QueryEngine/Descriptors/QueryFragmentDescriptor.h @@ -29,8 +29,8 @@ #include #include -#include #include "../CompilationOptions.h" +#include "Shared/Logger.h" namespace Fragmenter_Namespace { class FragmentInfo; diff --git a/QueryEngine/Descriptors/QueryMemoryDescriptor.h b/QueryEngine/Descriptors/QueryMemoryDescriptor.h index 63a4bdef35..06d821cb12 100644 --- a/QueryEngine/Descriptors/QueryMemoryDescriptor.h +++ b/QueryEngine/Descriptors/QueryMemoryDescriptor.h @@ -30,8 +30,8 @@ #include "ColSlotContext.h" #include "Types.h" -#include #include +#include "Shared/Logger.h" #include #include @@ -43,7 +43,6 @@ #include #include -#include extern bool g_cluster; diff --git a/QueryEngine/Descriptors/RowSetMemoryOwner.h b/QueryEngine/Descriptors/RowSetMemoryOwner.h index affcaff277..347ff69894 100644 --- a/QueryEngine/Descriptors/RowSetMemoryOwner.h +++ b/QueryEngine/Descriptors/RowSetMemoryOwner.h @@ -17,8 +17,8 @@ #pragma once #include "../StringDictionary/StringDictionaryProxy.h" +#include "Shared/Logger.h" -#include #include #include diff --git a/QueryEngine/DynamicWatchdog.cpp b/QueryEngine/DynamicWatchdog.cpp index e274582761..72c42835b9 100644 --- a/QueryEngine/DynamicWatchdog.cpp +++ b/QueryEngine/DynamicWatchdog.cpp @@ -18,8 +18,8 @@ #include #include -#include #include "DynamicWatchdog.h" +#include "Shared/Logger.h" static __inline__ uint64_t read_cycle_counter(void) { #if (defined(__x86_64__) || defined(__x86_64)) diff --git a/QueryEngine/Execute.cpp b/QueryEngine/Execute.cpp index 0e206d547c..6ff590bab5 100644 --- a/QueryEngine/Execute.cpp +++ b/QueryEngine/Execute.cpp @@ -87,6 +87,8 @@ size_t g_constrained_by_in_threshold{10}; size_t g_big_group_threshold{20000}; bool g_enable_window_functions{true}; +int const Executor::max_gpu_count; + Executor::Executor(const int db_id, const size_t block_size_x, const size_t grid_size_x, diff --git a/QueryEngine/ExecutionDispatch.cpp b/QueryEngine/ExecutionDispatch.cpp index 67851fb103..4dc58e2004 100644 --- a/QueryEngine/ExecutionDispatch.cpp +++ b/QueryEngine/ExecutionDispatch.cpp @@ -99,7 +99,7 @@ void Executor::ExecutionDispatch::runImpl( } if (eo.with_dynamic_watchdog && !dynamic_watchdog_set_.test_and_set(std::memory_order_acquire)) { - CHECK_GT(eo.dynamic_watchdog_time_limit, 0); + CHECK_GT(eo.dynamic_watchdog_time_limit, 0u); auto cycle_budget = dynamic_watchdog_init(eo.dynamic_watchdog_time_limit); LOG(INFO) << "Dynamic Watchdog budget: CPU: " << std::to_string(eo.dynamic_watchdog_time_limit) << "ms, " diff --git a/QueryEngine/ExpressionRange.h b/QueryEngine/ExpressionRange.h index 61d8523db2..542e6504f6 100644 --- a/QueryEngine/ExpressionRange.h +++ b/QueryEngine/ExpressionRange.h @@ -17,7 +17,6 @@ #define QUERYENGINE_EXPRESSIONRANGE_H #include "../Analyzer/Analyzer.h" -#include "../Shared/unreachable.h" #include //#include #include diff --git a/QueryEngine/ExpressionRewrite.cpp b/QueryEngine/ExpressionRewrite.cpp index 8ab4df1930..f19501b72d 100644 --- a/QueryEngine/ExpressionRewrite.cpp +++ b/QueryEngine/ExpressionRewrite.cpp @@ -21,9 +21,9 @@ #include "Execute.h" #include "RelAlgTranslator.h" #include "ScalarExprVisitor.h" +#include "Shared/Logger.h" #include "WindowExpressionRewrite.h" -#include #include namespace { diff --git a/QueryEngine/ExtensionsIR.cpp b/QueryEngine/ExtensionsIR.cpp index 1fbed1a710..a05f36b2d4 100644 --- a/QueryEngine/ExtensionsIR.cpp +++ b/QueryEngine/ExtensionsIR.cpp @@ -106,7 +106,7 @@ llvm::Value* CodeGenerator::codegenFunctionOper( const auto arg_lvs = codegen(arg, true, co); // TODO(adb / d): Assuming no const array cols for geo (for now) if (arg_ti.is_geometry()) { - CHECK_EQ(arg_ti.get_physical_coord_cols(), arg_lvs.size()); + CHECK_EQ(static_cast(arg_ti.get_physical_coord_cols()), arg_lvs.size()); for (size_t i = 0; i < arg_lvs.size(); i++) { orig_arg_lvs.push_back(arg_lvs[i]); } diff --git a/QueryEngine/ExtractFromTime.cpp b/QueryEngine/ExtractFromTime.cpp index daad93c2ac..352bc28fa3 100644 --- a/QueryEngine/ExtractFromTime.cpp +++ b/QueryEngine/ExtractFromTime.cpp @@ -18,7 +18,7 @@ #include "../Shared/funcannotations.h" #ifndef __CUDACC__ -#include +#include // abort() #endif extern "C" NEVER_INLINE DEVICE int32_t extract_hour(const int64_t lcltime) { diff --git a/QueryEngine/GpuMemUtils.cpp b/QueryEngine/GpuMemUtils.cpp index ac06b7808e..484dbfac51 100644 --- a/QueryEngine/GpuMemUtils.cpp +++ b/QueryEngine/GpuMemUtils.cpp @@ -18,13 +18,12 @@ #include "Allocators/CudaAllocator.h" #include "Allocators/ThrustAllocator.h" #include "GpuInitGroups.h" +#include "Shared/Logger.h" #include "StreamingTopN.h" #include "../CudaMgr/CudaMgr.h" #include "GroupByAndAggregate.h" -#include - void copy_to_gpu(Data_Namespace::DataMgr* data_mgr, CUdeviceptr dst, const void* src, diff --git a/QueryEngine/GroupByAndAggregate.cpp b/QueryEngine/GroupByAndAggregate.cpp index f9579158b1..d7e95f3c09 100644 --- a/QueryEngine/GroupByAndAggregate.cpp +++ b/QueryEngine/GroupByAndAggregate.cpp @@ -255,7 +255,7 @@ int64_t GroupByAndAggregate::getShardedTopBucket(const ColRangeInfo& col_range_i size_t device_count{0}; if (device_type_ == ExecutorDeviceType::GPU) { device_count = executor_->getCatalog()->getDataMgr().getCudaMgr()->getDeviceCount(); - CHECK_GT(device_count, 0); + CHECK_GT(device_count, 0u); } int64_t bucket{col_range_info.bucket}; @@ -1741,7 +1741,8 @@ std::vector GroupByAndAggregate::codegenAggArg( bool const fetch_columns) -> LLVMValueVector { const auto target_lvs = code_generator.codegen(selected_target_expr, fetch_columns, co); - CHECK_EQ(target_ti.get_physical_coord_cols(), target_lvs.size()); + CHECK_EQ(static_cast(target_ti.get_physical_coord_cols()), + target_lvs.size()); const auto i32_ty = get_int_type(32, executor_->cgen_state_->context_); const auto i8p_ty = diff --git a/QueryEngine/GroupByAndAggregate.h b/QueryEngine/GroupByAndAggregate.h index ce58b96fd4..db837b59b5 100644 --- a/QueryEngine/GroupByAndAggregate.h +++ b/QueryEngine/GroupByAndAggregate.h @@ -28,8 +28,8 @@ #include "../Planner/Planner.h" #include "../Shared/sqltypes.h" +#include "Shared/Logger.h" -#include #include #include #include diff --git a/QueryEngine/HashJoinKeyHandlers.h b/QueryEngine/HashJoinKeyHandlers.h index f79f6f4ec6..01e9178339 100644 --- a/QueryEngine/HashJoinKeyHandlers.h +++ b/QueryEngine/HashJoinKeyHandlers.h @@ -23,10 +23,10 @@ #ifdef __CUDACC__ #include "DecodersImpl.h" #else -#include #include "../StringDictionary/StringDictionary.h" #include "../StringDictionary/StringDictionaryProxy.h" #include "RuntimeFunctions.h" +#include "Shared/Logger.h" #endif #include diff --git a/QueryEngine/HashJoinRuntime.cpp b/QueryEngine/HashJoinRuntime.cpp index 6fcf92d562..cf4d5ad129 100644 --- a/QueryEngine/HashJoinRuntime.cpp +++ b/QueryEngine/HashJoinRuntime.cpp @@ -25,11 +25,12 @@ #include "GpuRtConstants.h" #include "JoinHashImpl.h" #else -#include -#include "../Shared/likely.h" -#include "../StringDictionary/StringDictionary.h" -#include "../StringDictionary/StringDictionaryProxy.h" +#include "Shared/Logger.h" + #include "RuntimeFunctions.h" +#include "Shared/likely.h" +#include "StringDictionary/StringDictionary.h" +#include "StringDictionary/StringDictionaryProxy.h" #include #endif @@ -1371,7 +1372,7 @@ void fill_one_to_many_baseline_hash_table( } std::vector count_copy(hash_entry_count, 0); - CHECK_GT(hash_entry_count, int32_t(0)); + CHECK_GT(hash_entry_count, 0u); memcpy(&count_copy[1], count_buff, (hash_entry_count - 1) * sizeof(int32_t)); inclusive_scan( count_copy.begin(), count_copy.end(), count_copy.begin(), cpu_thread_count); diff --git a/QueryEngine/IRCodegenUtils.h b/QueryEngine/IRCodegenUtils.h index a78a52d10d..63044f56cb 100644 --- a/QueryEngine/IRCodegenUtils.h +++ b/QueryEngine/IRCodegenUtils.h @@ -16,10 +16,10 @@ #pragma once -#include #include #include #include +#include "Shared/Logger.h" inline llvm::ArrayType* get_int_array_type(int const width, int count, diff --git a/QueryEngine/InPlaceSort.cpp b/QueryEngine/InPlaceSort.cpp index 264c8fd038..6b168081cc 100644 --- a/QueryEngine/InPlaceSort.cpp +++ b/QueryEngine/InPlaceSort.cpp @@ -20,8 +20,8 @@ #include #include "Allocators/ThrustAllocator.h" #include "Descriptors/QueryMemoryDescriptor.h" +#include "Shared/Logger.h" -#include #include void sort_groups_cpu(int64_t* val_buff, diff --git a/QueryEngine/InValuesBitmap.cpp b/QueryEngine/InValuesBitmap.cpp index 27b0846f1a..a80323bc96 100644 --- a/QueryEngine/InValuesBitmap.cpp +++ b/QueryEngine/InValuesBitmap.cpp @@ -24,8 +24,8 @@ #include "../Shared/checked_alloc.h" #include "GroupByAndAggregate.h" #include "RuntimeFunctions.h" +#include "Shared/Logger.h" -#include #include #include diff --git a/QueryEngine/JoinFilterPushDown.cpp b/QueryEngine/JoinFilterPushDown.cpp index 1bb65b7147..35ab4bf719 100644 --- a/QueryEngine/JoinFilterPushDown.cpp +++ b/QueryEngine/JoinFilterPushDown.cpp @@ -252,7 +252,8 @@ std::vector find_push_down_filters( CHECK_EQ(to_original_rte_idx.size(), input_permutation.size()); for (size_t i = 0; i < input_permutation.size(); ++i) { CHECK_LT(input_permutation[i], to_original_rte_idx.size()); - CHECK_EQ(to_original_rte_idx[input_permutation[i]], to_original_rte_idx.size()); + CHECK_EQ(static_cast(to_original_rte_idx[input_permutation[i]]), + to_original_rte_idx.size()); to_original_rte_idx[input_permutation[i]] = i; } } else { @@ -278,7 +279,7 @@ std::vector find_push_down_filters( } for (const auto& kv : filters_per_nesting_level) { CHECK_GE(kv.first, 0); - CHECK_LT(kv.first, input_size_prefix_sums.size()); + CHECK_LT(static_cast(kv.first), input_size_prefix_sums.size()); size_t input_prev = (kv.first > 1) ? input_size_prefix_sums[kv.first - 2] : 0; size_t input_start = kv.first ? input_size_prefix_sums[kv.first - 1] : 0; size_t input_next = input_size_prefix_sums[kv.first]; diff --git a/QueryEngine/JoinHashTable.cpp b/QueryEngine/JoinHashTable.cpp index 16bc172670..72d27fa7ee 100644 --- a/QueryEngine/JoinHashTable.cpp +++ b/QueryEngine/JoinHashTable.cpp @@ -22,8 +22,8 @@ #include "HashJoinRuntime.h" #include "RangeTableIndexVisitor.h" #include "RuntimeFunctions.h" +#include "Shared/Logger.h" -#include #include #include #include @@ -914,7 +914,7 @@ void JoinHashTable::initHashTableForDevice( auto& data_mgr = catalog->getDataMgr(); if (shard_count) { const auto shards_per_device = (shard_count + device_count_ - 1) / device_count_; - CHECK_GT(shards_per_device, 0); + CHECK_GT(shards_per_device, 0u); hash_entry_info.hash_entry_count = entries_per_shard * shards_per_device; } gpu_hash_table_buff_[device_id] = CudaAllocator::allocGpuAbstractBuffer( @@ -1039,7 +1039,7 @@ void JoinHashTable::initOneToManyHashTable( // needed once the join hash table has been built on the CPU. if (memory_level_ == Data_Namespace::GPU_LEVEL && shard_count) { const auto shards_per_device = (shard_count + device_count_ - 1) / device_count_; - CHECK_GT(shards_per_device, 0); + CHECK_GT(shards_per_device, 0u); hash_entry_count = entries_per_shard * shards_per_device; } #else @@ -1417,7 +1417,7 @@ size_t get_entries_per_device(const size_t total_entries, size_t entries_per_device = entries_per_shard; if (memory_level == Data_Namespace::GPU_LEVEL && shard_count) { const auto shards_per_device = (shard_count + device_count - 1) / device_count; - CHECK_GT(shards_per_device, 0); + CHECK_GT(shards_per_device, 0u); entries_per_device = entries_per_shard * shards_per_device; } return entries_per_device; diff --git a/QueryEngine/JsonAccessors.h b/QueryEngine/JsonAccessors.h index 2462bc9ec8..61a6d524f8 100644 --- a/QueryEngine/JsonAccessors.h +++ b/QueryEngine/JsonAccessors.h @@ -25,8 +25,8 @@ #ifndef QUERYENGINE_JSONACCESSORS_H #define QUERYENGINE_JSONACCESSORS_H -#include #include +#include "Shared/Logger.h" inline const rapidjson::Value& field(const rapidjson::Value& obj, const char field[]) noexcept { diff --git a/QueryEngine/LoopControlFlow/JoinLoop.cpp b/QueryEngine/LoopControlFlow/JoinLoop.cpp index 5e01714e17..b1d769da1b 100644 --- a/QueryEngine/LoopControlFlow/JoinLoop.cpp +++ b/QueryEngine/LoopControlFlow/JoinLoop.cpp @@ -15,6 +15,7 @@ */ #include "JoinLoop.h" +#include "Shared/Logger.h" #include diff --git a/QueryEngine/LoopControlFlow/JoinLoop.h b/QueryEngine/LoopControlFlow/JoinLoop.h index f6643228b6..83afc9fdca 100644 --- a/QueryEngine/LoopControlFlow/JoinLoop.h +++ b/QueryEngine/LoopControlFlow/JoinLoop.h @@ -16,10 +16,10 @@ #pragma once -#include #include #include #include +#include "Shared/Logger.h" #include "../../Shared/sqldefs.h" #include "../IRCodegenUtils.h" diff --git a/QueryEngine/LoopControlFlow/JoinLoopTest.cpp b/QueryEngine/LoopControlFlow/JoinLoopTest.cpp index c1db8962d8..4b81b347b3 100644 --- a/QueryEngine/LoopControlFlow/JoinLoopTest.cpp +++ b/QueryEngine/LoopControlFlow/JoinLoopTest.cpp @@ -18,8 +18,8 @@ // with the one generated by the `generate_loop_ref.py` script. #include "JoinLoop.h" +#include "Shared/Logger.h" -#include #include #include #include diff --git a/QueryEngine/NvidiaKernel.cpp b/QueryEngine/NvidiaKernel.cpp index 2b31f5532d..98c762a83b 100644 --- a/QueryEngine/NvidiaKernel.cpp +++ b/QueryEngine/NvidiaKernel.cpp @@ -17,8 +17,8 @@ #include "NvidiaKernel.h" #include "../Shared/mapdpath.h" +#include "Shared/Logger.h" -#include #include #ifdef HAVE_CUDA diff --git a/QueryEngine/OverlapsJoinHashTable.cpp b/QueryEngine/OverlapsJoinHashTable.cpp index b8c5faa7fb..f73989d8b5 100644 --- a/QueryEngine/OverlapsJoinHashTable.cpp +++ b/QueryEngine/OverlapsJoinHashTable.cpp @@ -23,8 +23,6 @@ #include "Execute.h" -#include "Shared/unreachable.h" - std::shared_ptr OverlapsJoinHashTable::getInstance( const std::shared_ptr condition, const std::vector& query_infos, @@ -199,7 +197,7 @@ std::pair OverlapsJoinHashTable::approximateTupleCount( std::vector num_keys_for_row; // TODO(adb): support multi-column overlaps join - CHECK_EQ(columns_per_device.size(), 1); + CHECK_EQ(columns_per_device.size(), 1u); num_keys_for_row.resize(columns_per_device.front().join_columns[0].num_elems); approximate_distinct_tuples_overlaps(hll_result, @@ -246,7 +244,7 @@ std::pair OverlapsJoinHashTable::approximateTupleCount( auto join_columns_gpu = transfer_pod_vector_to_gpu(columns_for_device.join_columns, allocator); - CHECK_GT(columns_for_device.join_buckets.size(), 0); + CHECK_GT(columns_for_device.join_buckets.size(), 0u); const auto& bucket_sizes_for_dimension = columns_for_device.join_buckets[0].bucket_sizes_for_dimension; auto bucket_sizes_gpu = allocator.allocateScopedBuffer( @@ -551,7 +549,7 @@ int OverlapsJoinHashTable::initHashTableOnGpu( auto join_columns_gpu = transfer_pod_vector_to_gpu(join_columns, allocator); auto hash_buff = reinterpret_cast(gpu_hash_table_buff_[device_id]->getMemoryPtr()); - CHECK_EQ(join_columns.size(), 1); + CHECK_EQ(join_columns.size(), 1u); auto& bucket_sizes_for_dimension = join_bucket_info[0].bucket_sizes_for_dimension; auto bucket_sizes_gpu = transfer_pod_vector_to_gpu(bucket_sizes_for_dimension, allocator); @@ -698,7 +696,7 @@ void OverlapsJoinHashTable::computeBucketSizes( const std::vector& inner_outer_pairs, const size_t row_count) { // No coalesced keys for overlaps joins yet - CHECK_EQ(inner_outer_pairs.size(), 1); + CHECK_EQ(inner_outer_pairs.size(), 1u); const auto col = inner_outer_pairs[0].first; CHECK(col); diff --git a/QueryEngine/QueryExecutionContext.cpp b/QueryEngine/QueryExecutionContext.cpp index 67381a3b81..2f05277dd0 100644 --- a/QueryEngine/QueryExecutionContext.cpp +++ b/QueryEngine/QueryExecutionContext.cpp @@ -768,7 +768,7 @@ std::vector QueryExecutionContext::prepareKernelParams( data_mgr, params[NUM_FRAGMENTS], &num_fragments, sizeof(uint64_t), device_id); CUdeviceptr literals_and_addr_mapping = gpu_allocator_->alloc(literal_buff.size() + 2 * sizeof(int64_t), nullptr); - CHECK_EQ(0, literals_and_addr_mapping % 8); + CHECK_EQ(static_cast(0), literals_and_addr_mapping % 8); std::vector additional_literal_bytes; const auto count_distinct_bitmap_mem = query_buffers_->getCountDistinctBitmapPtr(); if (count_distinct_bitmap_mem) { diff --git a/QueryEngine/QueryMemoryInitializer.cpp b/QueryEngine/QueryMemoryInitializer.cpp index 4d9694350a..4e1e489477 100644 --- a/QueryEngine/QueryMemoryInitializer.cpp +++ b/QueryEngine/QueryMemoryInitializer.cpp @@ -20,10 +20,10 @@ #include "GpuInitGroups.h" #include "GpuMemUtils.h" #include "ResultSet.h" +#include "Shared/Logger.h" #include "StreamingTopN.h" #include -#include namespace { @@ -372,7 +372,7 @@ void QueryMemoryInitializer::initColumnarGroups( int32_t init_val_idx = 0; for (int32_t i = 0; i < agg_col_count; ++i) { if (query_mem_desc.getPaddedSlotWidthBytes(i) > 0) { - CHECK_LT(init_val_idx, init_vals.size()); + CHECK_LT(static_cast(init_val_idx), init_vals.size()); switch (query_mem_desc.getPaddedSlotWidthBytes(i)) { case 1: buffer_ptr = initColumnarBuffer( @@ -500,7 +500,7 @@ std::vector QueryMemoryInitializer::allocateCountDistinctBuffers( CHECK(!agg_info.sql_type.is_varlen()); const auto agg_col_idx = query_mem_desc.getSlotIndexForSingleSlotCol(target_idx); - CHECK_LT(agg_col_idx, agg_col_count); + CHECK_LT(static_cast(agg_col_idx), agg_col_count); CHECK_EQ(static_cast(query_mem_desc.getLogicalSlotWidthBytes(agg_col_idx)), sizeof(int64_t)); diff --git a/QueryEngine/QueryRewrite.cpp b/QueryEngine/QueryRewrite.cpp index 5eceb80acd..e4baf57440 100644 --- a/QueryEngine/QueryRewrite.cpp +++ b/QueryEngine/QueryRewrite.cpp @@ -17,8 +17,7 @@ #include "QueryRewrite.h" #include "ExpressionRange.h" #include "ExpressionRewrite.h" - -#include +#include "Shared/Logger.h" RelAlgExecutionUnit QueryRewriter::rewrite( const RelAlgExecutionUnit& ra_exe_unit_in) const { diff --git a/QueryEngine/QueryTemplateGenerator.cpp b/QueryEngine/QueryTemplateGenerator.cpp index 1dfcfb3bfa..f34fc3eba3 100644 --- a/QueryEngine/QueryTemplateGenerator.cpp +++ b/QueryEngine/QueryTemplateGenerator.cpp @@ -15,8 +15,8 @@ */ #include "QueryTemplateGenerator.h" +#include "Shared/Logger.h" -#include #include #include #include diff --git a/QueryEngine/RelAlgAbstractInterpreter.cpp b/QueryEngine/RelAlgAbstractInterpreter.cpp index 49e0206a8e..62fef2dba4 100644 --- a/QueryEngine/RelAlgAbstractInterpreter.cpp +++ b/QueryEngine/RelAlgAbstractInterpreter.cpp @@ -23,7 +23,6 @@ #include "RelLeftDeepInnerJoin.h" #include "RexVisitor.h" -#include #include #include diff --git a/QueryEngine/RelAlgExecutor.cpp b/QueryEngine/RelAlgExecutor.cpp index 950bf5df57..0589ce6626 100644 --- a/QueryEngine/RelAlgExecutor.cpp +++ b/QueryEngine/RelAlgExecutor.cpp @@ -648,7 +648,7 @@ get_join_source_used_inputs(const RelAlgNode* ra_node, const Catalog_Namespace::Catalog& cat) { const auto data_sink_node = get_data_sink(ra_node); if (auto join = dynamic_cast(data_sink_node)) { - CHECK_EQ(join->inputCount(), 2); + CHECK_EQ(join->inputCount(), 2u); const auto condition = join->getCondition(); RexUsedInputsVisitor visitor(cat); auto condition_inputs = visitor.visit(condition); @@ -658,7 +658,7 @@ get_join_source_used_inputs(const RelAlgNode* ra_node, } if (auto left_deep_join = dynamic_cast(data_sink_node)) { - CHECK_GE(left_deep_join->inputCount(), 2); + CHECK_GE(left_deep_join->inputCount(), 2u); const auto condition = left_deep_join->getInnerCondition(); RexUsedInputsVisitor visitor(cat); auto result = visitor.visit(condition); @@ -674,7 +674,7 @@ get_join_source_used_inputs(const RelAlgNode* ra_node, return std::make_pair(result, used_inputs_owned); } - CHECK_EQ(ra_node->inputCount(), 1); + CHECK_EQ(ra_node->inputCount(), 1u); return std::make_pair(std::unordered_set{}, std::vector>{}); } @@ -2451,7 +2451,7 @@ RelAlgExecutor::WorkUnit RelAlgExecutor::createModifyCompoundWorkUnit( query_features); size_t starting_projection_column_idx = get_scalar_sources_size(compound) - compound->getTargetColumnCount() - 1; - CHECK_GT(starting_projection_column_idx, 0); + CHECK_GT(starting_projection_column_idx, 0u); const auto scalar_sources = translate_scalar_sources_for_update(compound, translator, @@ -2873,7 +2873,7 @@ RelAlgExecutor::WorkUnit RelAlgExecutor::createModifyProjectWorkUnit( query_features); size_t starting_projection_column_idx = get_scalar_sources_size(project) - project->getTargetColumnCount() - 1; - CHECK_GT(starting_projection_column_idx, 0); + CHECK_GT(starting_projection_column_idx, 0u); auto target_exprs_owned = translate_scalar_sources_for_update(project, translator, diff --git a/QueryEngine/RelAlgOptimizer.cpp b/QueryEngine/RelAlgOptimizer.cpp index 4391287513..0e01077830 100644 --- a/QueryEngine/RelAlgOptimizer.cpp +++ b/QueryEngine/RelAlgOptimizer.cpp @@ -16,8 +16,7 @@ #include "RelAlgOptimizer.h" #include "RexVisitor.h" - -#include +#include "Shared/Logger.h" #include #include diff --git a/QueryEngine/RelAlgTranslator.cpp b/QueryEngine/RelAlgTranslator.cpp index 56bac079f8..2ecfab57a9 100644 --- a/QueryEngine/RelAlgTranslator.cpp +++ b/QueryEngine/RelAlgTranslator.cpp @@ -204,8 +204,8 @@ std::shared_ptr RelAlgTranslator::translateAggregateRex( std::shared_ptr err_rate; if (takes_arg) { const auto operand = rex->getOperand(0); - CHECK_LT(operand, static_cast(scalar_sources.size())); - CHECK_LE(rex->size(), 2); + CHECK_LT(operand, scalar_sources.size()); + CHECK_LE(rex->size(), 2u); arg_expr = scalar_sources[operand]; if (agg_kind == kAPPROX_COUNT_DISTINCT && rex->size() == 2) { err_rate = std::dynamic_pointer_cast( @@ -342,7 +342,7 @@ std::shared_ptr RelAlgTranslator::translateInput( CHECK_EQ("rowid", cd->columnName); col_ti.set_size(8); } - CHECK_LE(rte_idx, join_types_.size()); + CHECK_LE(static_cast(rte_idx), join_types_.size()); if (rte_idx > 0 && join_types_[rte_idx - 1] == JoinType::LEFT) { col_ti.set_notnull(false); } @@ -354,7 +354,7 @@ std::shared_ptr RelAlgTranslator::translateInput( const size_t col_id = rex_input->getIndex(); CHECK_LT(col_id, in_metainfo.size()); auto col_ti = in_metainfo[col_id].get_type_info(); - CHECK_LE(rte_idx, join_types_.size()); + CHECK_LE(static_cast(rte_idx), join_types_.size()); if (rte_idx > 0 && join_types_[rte_idx - 1] == JoinType::LEFT) { col_ti.set_notnull(false); } @@ -1571,7 +1571,7 @@ std::shared_ptr RelAlgTranslator::translateWindowFunction( } auto ti = rex_window_function->getType(); if (window_function_is_value(rex_window_function->getKind())) { - CHECK_GE(args.size(), 1); + CHECK_GE(args.size(), 1u); ti = args.front()->get_type_info(); } return makeExpr( diff --git a/QueryEngine/RelAlgTranslatorGeo.cpp b/QueryEngine/RelAlgTranslatorGeo.cpp index 6d68120dc0..b5b1b648ff 100644 --- a/QueryEngine/RelAlgTranslatorGeo.cpp +++ b/QueryEngine/RelAlgTranslatorGeo.cpp @@ -57,7 +57,7 @@ std::vector> RelAlgTranslator::translateGeoColum CHECK(!in_metainfo.empty()); CHECK_GE(rte_idx, 0); column_id = rex_input->getIndex(); - CHECK_LT(column_id, in_metainfo.size()); + CHECK_LT(static_cast(column_id), in_metainfo.size()); ti = in_metainfo[column_id].get_type_info(); } CHECK(IS_GEO(ti.get_type())); @@ -512,12 +512,12 @@ std::shared_ptr RelAlgTranslator::translateUnaryGeoFunction( auto geoargs = translateGeoFunctionArg( rex_function->getOperand(0), arg_ti, lindex, false, false, true); if (arg_ti.get_type() == kPOLYGON) { - CHECK_EQ(geoargs.size(), 2); + CHECK_EQ(geoargs.size(), 2u); geoargs.erase(geoargs.begin(), geoargs.begin() + 1); // remove the coords return makeExpr( rex_function->getType(), specialized_geofunc, geoargs); } else if (arg_ti.get_type() == kMULTIPOLYGON) { - CHECK_EQ(geoargs.size(), 3); + CHECK_EQ(geoargs.size(), 3u); geoargs.erase(geoargs.begin(), geoargs.begin() + 1); // remove the coords geoargs.erase(geoargs.begin() + 1, geoargs.end()); // remove the poly_rings return makeExpr( @@ -902,7 +902,7 @@ std::shared_ptr RelAlgTranslator::translateFunctionWithGeoArg( std::string specialized_geofunc{rex_function->getName()}; if (rex_function->getName() == std::string("convert_meters_to_pixel_width") || rex_function->getName() == std::string("convert_meters_to_pixel_height")) { - CHECK_EQ(rex_function->size(), 6); + CHECK_EQ(rex_function->size(), 6u); SQLTypeInfo arg_ti; std::vector> args; args.push_back(translateScalarRex(rex_function->getOperand(0))); @@ -945,7 +945,7 @@ std::shared_ptr RelAlgTranslator::translateFunctionWithGeoArg( return makeExpr( rex_function->getType(), specialized_geofunc, args); } else if (rex_function->getName() == std::string("is_point_in_view")) { - CHECK_EQ(rex_function->size(), 5); + CHECK_EQ(rex_function->size(), 5u); SQLTypeInfo arg_ti; std::vector> args; auto geoargs = translateGeoFunctionArg( @@ -977,7 +977,7 @@ std::shared_ptr RelAlgTranslator::translateFunctionWithGeoArg( return makeExpr( rex_function->getType(), specialized_geofunc, args); } else if (rex_function->getName() == std::string("is_point_size_in_view")) { - CHECK_EQ(rex_function->size(), 6); + CHECK_EQ(rex_function->size(), 6u); SQLTypeInfo arg_ti; std::vector> args; auto geoargs = translateGeoFunctionArg( @@ -1016,7 +1016,7 @@ std::shared_ptr RelAlgTranslator::translateFunctionWithGeoArg( std::shared_ptr RelAlgTranslator::translateGeoOverlapsOper( const RexOperator* rex_operator) const { - CHECK_EQ(rex_operator->size(), 2); + CHECK_EQ(rex_operator->size(), 2u); auto translate_input = [&](const RexScalar* operand) -> std::shared_ptr { @@ -1025,7 +1025,7 @@ std::shared_ptr RelAlgTranslator::translateGeoOverlapsOper( SQLTypeInfo ti; const auto exprs = translateGeoColumn(input, ti, true, false, false); - CHECK_GT(exprs.size(), 0); + CHECK_GT(exprs.size(), 0u); if (ti.get_type() == kPOINT) { return exprs.front(); } else { diff --git a/QueryEngine/RelLeftDeepInnerJoin.cpp b/QueryEngine/RelLeftDeepInnerJoin.cpp index 8eda422971..f85e96813a 100644 --- a/QueryEngine/RelLeftDeepInnerJoin.cpp +++ b/QueryEngine/RelLeftDeepInnerJoin.cpp @@ -17,8 +17,7 @@ #include "RelLeftDeepInnerJoin.h" #include "RelAlgAbstractInterpreter.h" #include "RexVisitor.h" - -#include +#include "Shared/Logger.h" #include diff --git a/QueryEngine/Rendering/os/RenderAllocator.cpp b/QueryEngine/Rendering/os/RenderAllocator.cpp index 3c9aa24eee..afb59c00df 100644 --- a/QueryEngine/Rendering/os/RenderAllocator.cpp +++ b/QueryEngine/Rendering/os/RenderAllocator.cpp @@ -15,8 +15,8 @@ */ #include "../RenderAllocator.h" -#include #include "../../GpuInitGroups.h" +#include "Shared/Logger.h" RenderAllocator::RenderAllocator(int8_t* preallocated_ptr, const size_t preallocated_size, diff --git a/QueryEngine/ResultSetBufferAccessors.h b/QueryEngine/ResultSetBufferAccessors.h index 195dee519a..5714dcee34 100644 --- a/QueryEngine/ResultSetBufferAccessors.h +++ b/QueryEngine/ResultSetBufferAccessors.h @@ -29,8 +29,6 @@ #include "BufferCompaction.h" #include "TypePunning.h" -#include "../Shared/unreachable.h" - #ifndef __CUDACC__ #include "Descriptors/QueryMemoryDescriptor.h" diff --git a/QueryEngine/ResultSetIteration.cpp b/QueryEngine/ResultSetIteration.cpp index 8f2c6acd7c..1fa014ac6e 100644 --- a/QueryEngine/ResultSetIteration.cpp +++ b/QueryEngine/ResultSetIteration.cpp @@ -463,7 +463,7 @@ InternalTargetValue ResultSet::RowWiseTargetAccessor::getColumnInternal( result_set_->serialized_varlen_buffer_.size()); const auto& varlen_buffer_for_fragment = result_set_->serialized_varlen_buffer_[storage_lookup_result.storage_idx]; - CHECK_LT(i1, varlen_buffer_for_fragment.size()); + CHECK_LT(static_cast(i1), varlen_buffer_for_fragment.size()); return InternalTargetValue(&varlen_buffer_for_fragment[i1]); } CHECK(offsets_for_target.ptr2); @@ -593,7 +593,7 @@ InternalTargetValue ResultSet::ColumnWiseTargetAccessor::getColumnInternal( result_set_->serialized_varlen_buffer_.size()); const auto& varlen_buffer_for_fragment = result_set_->serialized_varlen_buffer_[storage_lookup_result.storage_idx]; - CHECK_LT(i1, varlen_buffer_for_fragment.size()); + CHECK_LT(static_cast(i1), varlen_buffer_for_fragment.size()); return InternalTargetValue(&varlen_buffer_for_fragment[i1]); } CHECK(offsets_for_target.ptr2); @@ -1090,7 +1090,7 @@ const std::vector& ResultSet::getColumnFrag(const size_t storage_ CHECK_LE(local_idx, global_idx); } CHECK_GE(frag_id, int64_t(0)); - CHECK_LT(frag_id, col_buffers_[storage_idx].size()); + CHECK_LT(static_cast(frag_id), col_buffers_[storage_idx].size()); global_idx = local_idx; return col_buffers_[storage_idx][frag_id]; } else { @@ -1204,7 +1204,7 @@ TargetValue ResultSet::makeVarlenTargetValue(const int8_t* ptr1, return TargetValue(nullptr); } CHECK(vd.pointer); - CHECK_GT(vd.length, 0); + CHECK_GT(vd.length, 0u); std::string fetched_str(reinterpret_cast(vd.pointer), vd.length); return fetched_str; } else { @@ -1219,7 +1219,7 @@ TargetValue ResultSet::makeVarlenTargetValue(const int8_t* ptr1, if (ad.is_null) { return ArrayTargetValue(boost::optional>{}); } - CHECK_GE(ad.length, 0); + CHECK_GE(ad.length, 0u); if (ad.length > 0) { CHECK(ad.pointer); } diff --git a/QueryEngine/ResultSetReduction.cpp b/QueryEngine/ResultSetReduction.cpp index 07a2bc3b2d..08d5055685 100644 --- a/QueryEngine/ResultSetReduction.cpp +++ b/QueryEngine/ResultSetReduction.cpp @@ -462,7 +462,7 @@ void ResultSetStorage::rewriteAggregateBufferOffsets( slot_idx += 2; length_to_elems = 4; } - CHECK_LT(offset, serialized_varlen_buffer.size()); + CHECK_LT(static_cast(offset), serialized_varlen_buffer.size()); const auto& varlen_bytes_str = serialized_varlen_buffer[offset++]; const auto str_ptr = reinterpret_cast(varlen_bytes_str.c_str()); @@ -473,7 +473,7 @@ void ResultSetStorage::rewriteAggregateBufferOffsets( static_cast(varlen_bytes_str.size() / length_to_elems); } } else { - CHECK_LT(offset, serialized_varlen_buffer.size()); + CHECK_LT(static_cast(offset), serialized_varlen_buffer.size()); const auto& varlen_bytes_str = serialized_varlen_buffer[offset]; const auto str_ptr = reinterpret_cast(varlen_bytes_str.c_str()); CHECK(ptr1); @@ -1129,7 +1129,7 @@ void ResultSetStorage::fillOneEntryRowWise(const std::vector& entry) { void ResultSetStorage::initializeRowWise() const { const auto key_count = query_mem_desc_.getGroupbyColCount(); const auto row_size = get_row_bytes(query_mem_desc_); - CHECK_EQ(row_size % 8, 0); + CHECK_EQ(row_size % 8, 0u); const auto key_bytes_with_padding = align_to_int64(get_key_bytes_rowwise(query_mem_desc_)); CHECK(!query_mem_desc_.hasKeylessHash()); @@ -1420,7 +1420,7 @@ void ResultSetStorage::reduceOneSlot( length_to_elems = target_info.sql_type.is_string() ? 1 : elem_ti.get_size(); } - CHECK_LT(rhs_proj_col, serialized_varlen_buffer.size()); + CHECK_LT(static_cast(rhs_proj_col), serialized_varlen_buffer.size()); const auto& varlen_bytes_str = serialized_varlen_buffer[rhs_proj_col]; const auto str_ptr = reinterpret_cast(varlen_bytes_str.c_str()); *reinterpret_cast(this_ptr1) = diff --git a/QueryEngine/ScalarCodeGenerator.cpp b/QueryEngine/ScalarCodeGenerator.cpp index 75c32aee8d..0bbba3c788 100644 --- a/QueryEngine/ScalarCodeGenerator.cpp +++ b/QueryEngine/ScalarCodeGenerator.cpp @@ -49,6 +49,7 @@ llvm::Type* llvm_type_from_sql(const SQLTypeInfo& ti, llvm::LLVMContext& ctx) { } default: { LOG(FATAL) << "Unsupported type"; + return nullptr; // satisfy -Wreturn-type } } } @@ -151,6 +152,7 @@ std::vector ScalarCodeGenerator::generateNativeCode( } default: { LOG(FATAL) << "Invalid device type"; + return {}; // satisfy -Wreturn-type } } } @@ -160,7 +162,7 @@ std::vector ScalarCodeGenerator::codegenColumn( const bool fetch_column, const CompilationOptions& co) { int arg_idx = plan_state_->getLocalColumnId(column, fetch_column); - CHECK_LT(arg_idx, cgen_state_->row_func_->arg_size()); + CHECK_LT(static_cast(arg_idx), cgen_state_->row_func_->arg_size()); llvm::Value* arg = cgen_state_->row_func_->arg_begin() + arg_idx + 1; return {arg}; } diff --git a/QueryEngine/SpeculativeTopN.cpp b/QueryEngine/SpeculativeTopN.cpp index dde6d1a335..2ce14c047d 100644 --- a/QueryEngine/SpeculativeTopN.cpp +++ b/QueryEngine/SpeculativeTopN.cpp @@ -18,8 +18,7 @@ #include "RelAlgExecutor.h" #include "ResultSet.h" - -#include +#include "Shared/Logger.h" SpeculativeTopNMap::SpeculativeTopNMap() : unknown_(0) {} diff --git a/QueryEngine/StringDictionaryGenerations.cpp b/QueryEngine/StringDictionaryGenerations.cpp index 36a24b3f33..8a54a4647c 100644 --- a/QueryEngine/StringDictionaryGenerations.cpp +++ b/QueryEngine/StringDictionaryGenerations.cpp @@ -16,8 +16,6 @@ #include "StringDictionaryGenerations.h" -#include - void StringDictionaryGenerations::setGeneration(const uint32_t id, const size_t generation) { id_to_generation_.emplace(id, generation); diff --git a/QueryEngine/TableGenerations.cpp b/QueryEngine/TableGenerations.cpp index bfd75ff097..79dbcb60ce 100644 --- a/QueryEngine/TableGenerations.cpp +++ b/QueryEngine/TableGenerations.cpp @@ -15,8 +15,7 @@ */ #include "TableGenerations.h" - -#include +#include "Shared/Logger.h" void TableGenerations::setGeneration(const uint32_t id, const TableGeneration& generation) { diff --git a/QueryEngine/TableOptimizer.h b/QueryEngine/TableOptimizer.h index e8f436a37d..76da2f5aa6 100644 --- a/QueryEngine/TableOptimizer.h +++ b/QueryEngine/TableOptimizer.h @@ -19,7 +19,7 @@ #include "Execute.h" #include -#include +#include "Shared/Logger.h" /** * @brief Driver for running cleanup processes on a table. diff --git a/QueryEngine/TargetExprBuilder.cpp b/QueryEngine/TargetExprBuilder.cpp index 228dd5c149..392cfe9660 100644 --- a/QueryEngine/TargetExprBuilder.cpp +++ b/QueryEngine/TargetExprBuilder.cpp @@ -27,8 +27,7 @@ #include "GroupByAndAggregate.h" #include "MaxwellCodegenPatch.h" #include "OutputBufferInitialization.h" - -#include +#include "Shared/Logger.h" #define LL_CONTEXT executor->cgen_state_->context_ #define LL_BUILDER executor->cgen_state_->ir_builder_ @@ -562,7 +561,8 @@ void TargetExprCodegenBuilder::codegen( first_sample_expr.base_slot_index, first_sample_expr.target_idx); } else { - CHECK_LT(first_sample_expr.base_slot_index, agg_out_vec.size()); + CHECK_LT(static_cast(first_sample_expr.base_slot_index), + agg_out_vec.size()); agg_col_ptr = executor->castToIntPtrTyIn(agg_out_vec[first_sample_expr.base_slot_index], 64); } diff --git a/QueryEngine/TargetValue.h b/QueryEngine/TargetValue.h index 042dbd2d81..e5e56b3433 100644 --- a/QueryEngine/TargetValue.h +++ b/QueryEngine/TargetValue.h @@ -25,9 +25,9 @@ #ifndef QUERYENGINE_TARGETVALUE_H #define QUERYENGINE_TARGETVALUE_H -#include #include #include +#include "Shared/Logger.h" #include diff --git a/QueryEngine/UDFCompiler.cpp b/QueryEngine/UDFCompiler.cpp index 28bbdc9a57..2743af38e6 100644 --- a/QueryEngine/UDFCompiler.cpp +++ b/QueryEngine/UDFCompiler.cpp @@ -26,12 +26,12 @@ #include #include #include -#include #include #include #include #include #include "Execute.h" +#include "Shared/Logger.h" using namespace clang; using namespace clang::tooling; diff --git a/QueryEngine/WindowContext.cpp b/QueryEngine/WindowContext.cpp index 07c989e47a..1cb7ad76e5 100644 --- a/QueryEngine/WindowContext.cpp +++ b/QueryEngine/WindowContext.cpp @@ -464,7 +464,7 @@ void WindowFunctionContext::compute() { } if (window_function_is_value(window_func_->getKind()) || window_function_is_aggregate(window_func_->getKind())) { - CHECK_EQ(off, elem_count_); + CHECK_EQ(static_cast(off), elem_count_); } auto output_i64 = reinterpret_cast(output_); if (window_function_is_aggregate(window_func_->getKind())) { diff --git a/QueryEngine/invoke_example/sample.cpp b/QueryEngine/invoke_example/sample.cpp index 76b5d70ea5..b8cb0a0b6b 100644 --- a/QueryEngine/invoke_example/sample.cpp +++ b/QueryEngine/invoke_example/sample.cpp @@ -14,13 +14,13 @@ * limitations under the License. */ -#include #include #include #include #include #include #include +#include "Shared/Logger.h" #include "cuda.h" template diff --git a/QueryRunner/QueryRunner.cpp b/QueryRunner/QueryRunner.cpp index 8139a8598a..8f44e0a088 100644 --- a/QueryRunner/QueryRunner.cpp +++ b/QueryRunner/QueryRunner.cpp @@ -22,6 +22,7 @@ #include "Parser/parser.h" #include "QueryEngine/CalciteAdapter.h" #include "Shared/ConfigResolve.h" +#include "Shared/Logger.h" #include "Shared/MapDParameters.h" #include "Shared/StringTransform.h" #include "bcrypt.h" @@ -32,7 +33,6 @@ #include "DistributedLoader.h" -#include #include #include #include @@ -117,7 +117,7 @@ void mapd_signal_handler(int signal_number) { LOG(ERROR) << "Interrupt signal (" << signal_number << ") received."; calcite_shutdown_handler(); // shut down logging force a flush - google::ShutdownGoogleLogging(); + logger::shutdown(); // terminate program if (signal_number == SIGTERM) { std::exit(EXIT_SUCCESS); @@ -152,7 +152,7 @@ Catalog_Namespace::SessionInfo* get_session( g_leaf_count = leaf_servers.size(); register_signal_handler(); - google::InstallFailureFunction(&calcite_shutdown_handler); + logger::set_once_fatal_func(&calcite_shutdown_handler); g_calcite = std::make_shared(-1, CALCITEPORT, db_path, 1024); diff --git a/SQLFrontend/CMakeLists.txt b/SQLFrontend/CMakeLists.txt index d4c73ec713..21e9040add 100644 --- a/SQLFrontend/CMakeLists.txt +++ b/SQLFrontend/CMakeLists.txt @@ -2,6 +2,6 @@ find_package(Boost COMPONENTS program_options filesystem regex system REQUIRED Q add_executable(omnisql omnisql.cpp) -target_link_libraries(omnisql mapd_thrift Shared ThriftClient linenoise ${Boost_LIBRARIES} ${Glog_LIBRARIES} ${CMAKE_DL_LIBS} ${PROFILER_LIBS} ${Thrift_LIBRARIES}) +target_link_libraries(omnisql mapd_thrift Shared ThriftClient linenoise ${Boost_LIBRARIES} ${CMAKE_DL_LIBS} ${PROFILER_LIBS} ${Thrift_LIBRARIES}) install(TARGETS omnisql DESTINATION bin) diff --git a/SQLFrontend/CommandFunctors.h b/SQLFrontend/CommandFunctors.h index 07bc0698e2..9418ab7f0a 100644 --- a/SQLFrontend/CommandFunctors.h +++ b/SQLFrontend/CommandFunctors.h @@ -25,17 +25,6 @@ #include "Shared/base64.h" -// NOTE: These alternative checks are required because of changes to glog -template -inline auto NESTED_CHECK(T1&& t1) { - CHECK(std::forward(t1)); -} - -template -inline auto NESTED_CHECK_EQ(T1&& t1, T2&& t2) { - CHECK_EQ(std::forward(t1), std::forward(t2)); -} - template class ContextOperations { public: @@ -314,11 +303,11 @@ StandardCommand(ListColumns, { std::vector keys_with_spec; rapidjson::Document document; document.Parse(key_metainfo.c_str()); - NESTED_CHECK(!document.HasParseError()); - NESTED_CHECK(document.IsArray()); + CHECK(!document.HasParseError()); + CHECK(document.IsArray()); for (auto it = document.Begin(); it != document.End(); ++it) { const auto& key_with_spec_json = *it; - NESTED_CHECK(key_with_spec_json.IsObject()); + CHECK(key_with_spec_json.IsObject()); const std::string type = key_with_spec_json["type"].GetString(); const std::string name = key_with_spec_json["name"].GetString(); auto key_with_spec = type + " (" + name + ")"; @@ -329,7 +318,7 @@ StandardCommand(ListColumns, { key_with_spec_json["foreign_column"].GetString(); key_with_spec += foreign_table + "(" + foreign_column + ")"; } else { - NESTED_CHECK(type == "SHARD KEY"); + CHECK(type == "SHARD KEY"); } keys_with_spec.push_back(key_with_spec); } @@ -558,19 +547,18 @@ StandardCommand(GetOptimizedSchema, { decltype(p[1])& table_name(p[1]); auto get_row_count = [](const TQueryResult& query_result) -> size_t { - NESTED_CHECK(!query_result.row_set.row_desc.empty()); + CHECK(!query_result.row_set.row_desc.empty()); if (query_result.row_set.columns.empty()) { return 0; } - NESTED_CHECK_EQ(query_result.row_set.columns.size(), - query_result.row_set.row_desc.size()); + CHECK_EQ(query_result.row_set.columns.size(), query_result.row_set.row_desc.size()); return query_result.row_set.columns.front().nulls.size(); }; // runs a simple single integer value query and returns that single int value returned auto run_query = [get_row_count](ContextType& context, std::string query) -> int { thrift_op(context, query.c_str()); - NESTED_CHECK(get_row_count(context.query_return)); + CHECK(get_row_count(context.query_return)); // std::cerr << "return value is " << // context.query_return.row_set.columns[0].data.int_col[0]; return context.query_return.row_set.columns[0].data.int_col[0]; diff --git a/SQLFrontend/omnisql.cpp b/SQLFrontend/omnisql.cpp index 308f7e805a..bf7be14129 100644 --- a/SQLFrontend/omnisql.cpp +++ b/SQLFrontend/omnisql.cpp @@ -21,7 +21,6 @@ * **/ -#include #include #include #include @@ -49,6 +48,7 @@ #include "../Fragmenter/InsertOrderFragmenter.h" #include "MapDRelease.h" #include "MapDServer.h" +#include "Shared/Logger.h" #include "Shared/StringTransform.h" #include "Shared/ThriftClient.h" #include "Shared/ThriftTypesConvert.h" diff --git a/Shared/CMakeLists.txt b/Shared/CMakeLists.txt index 9c630125d0..1e48d04369 100644 --- a/Shared/CMakeLists.txt +++ b/Shared/CMakeLists.txt @@ -8,11 +8,11 @@ set(shared_source_files File.cpp StackTrace.cpp base64.cpp + Logger.cpp ) add_library(Shared ${shared_source_files}) -find_package(Boost COMPONENTS filesystem system REQUIRED QUIET) # Required by ThriftClient.cpp add_definitions("-DTHRIFT_PACKAGE_VERSION=\"${Thrift_VERSION}\"") add_library(ThriftClient ThriftClient.cpp) diff --git a/Shared/Compressor.cpp b/Shared/Compressor.cpp index 24e7493f72..013cb40907 100644 --- a/Shared/Compressor.cpp +++ b/Shared/Compressor.cpp @@ -24,9 +24,9 @@ #include "Compressor.h" #include -#include #include #include +#include "Logger.h" // we only compress data if the payload size is greater than 512 MB size_t g_compression_limit_bytes{512 * 1024 * 1024}; diff --git a/Shared/Datum.cpp b/Shared/Datum.cpp index fc68b6bea4..88bb7a8efa 100644 --- a/Shared/Datum.cpp +++ b/Shared/Datum.cpp @@ -27,13 +27,13 @@ #endif #include -#include #include #include #include #include #include #include +#include "Logger.h" #include "StringTransform.h" #include "DateConverters.h" diff --git a/Shared/File.cpp b/Shared/File.cpp index 72b7e429ef..aff2ebdb98 100644 --- a/Shared/File.cpp +++ b/Shared/File.cpp @@ -21,7 +21,6 @@ * */ #include "File.h" -#include #include #include #include @@ -29,6 +28,7 @@ #include #include #include +#include "Logger.h" namespace File_Namespace { diff --git a/Shared/InlineNullValues.h b/Shared/InlineNullValues.h index 8fd3c58eb6..11fcd22aea 100644 --- a/Shared/InlineNullValues.h +++ b/Shared/InlineNullValues.h @@ -17,13 +17,7 @@ #ifndef INLINENULLVALUES_H #define INLINENULLVALUES_H -#ifndef __CUDACC__ -#include -#else -#ifndef CHECK -#include "../Shared/always_assert.h" -#endif -#endif // __CUDACC__ +#include "Logger.h" template T inline_fp_null_value(); diff --git a/Shared/Logger.cpp b/Shared/Logger.cpp new file mode 100644 index 0000000000..07c6dc90f1 --- /dev/null +++ b/Shared/Logger.cpp @@ -0,0 +1,290 @@ +/* + * Copyright 2019 OmniSci, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __CUDACC__ + +#include "Logger.h" +#include +#include +#include +#include + +namespace logger { + +// Must match enum Severity in Logger.h. +std::array const SeverityNames{ + {"DEBUG4", "DEBUG3", "DEBUG2", "DEBUG1", "INFO", "WARNING", "ERROR", "FATAL"}}; + +std::array const SeveritySymbols{{'4', '3', '2', '1', 'I', 'W', 'E', 'F'}}; + +static_assert(Severity::NLEVELS == SeverityNames.size(), + "Size of SeverityNames must equal number of Severity levels."); +static_assert(Severity::NLEVELS == SeveritySymbols.size(), + "Size of SeveritySymbols must equal number of Severity levels."); + +BOOST_LOG_ATTRIBUTE_KEYWORD(process_id, "ProcessID", attr::current_process_id::value_type) +BOOST_LOG_ATTRIBUTE_KEYWORD(severity, "Severity", Severity) + +BOOST_LOG_GLOBAL_LOGGER_DEFAULT(g_logger, logger_t) + +LogOptions::LogOptions(char const* argv0) : options_("Logging") { + // Log file base_name matches name of program. + std::string const base_name = argv0 == nullptr + ? std::string("omnisci_server") + : boost::filesystem::path(argv0).filename().string(); + file_name_pattern_ = base_name + file_name_pattern_; + symlink_ = base_name + symlink_; + // Filter out DEBUG[1-4] severities from --help options + std::string severities; + for (auto const& name : SeverityNames) { + if (!boost::algorithm::starts_with(name, "DEBUG")) { + (severities += (severities.empty() ? "" : " ")) += name; + } + } + options_.add_options()( + "log-directory", + po::value(&log_dir_)->default_value(log_dir_), + "Logging directory. May be relative to data directory, or absolute."); + options_.add_options()( + "log-file-name", + po::value(&file_name_pattern_)->default_value(file_name_pattern_), + "Log file name relative to log-directory."); + options_.add_options()("log-symlink", + po::value(&symlink_)->default_value(symlink_), + "Symlink to active log."); + options_.add_options()("log-severity", + po::value(&severity_)->default_value(severity_), + ("Log to file severity level: " + severities).c_str()); + options_.add_options()( + "log-severity-clog", + po::value(&severity_clog_)->default_value(severity_clog_), + ("Log to console severity level: " + severities).c_str()); + options_.add_options()("log-auto-flush", + po::value(&auto_flush_)->default_value(auto_flush_), + "Flush logging buffer to file after each message."); + options_.add_options()("log-max-files", + po::value(&max_files_)->default_value(max_files_), + "Maximum number of log files to keep."); + options_.add_options()( + "log-min-free-space", + po::value(&min_free_space_)->default_value(20 << 20), + "Minimum number of bytes left on device before oldest log files are deleted."); + options_.add_options()("log-rotate-daily", + po::value(&rotate_daily_)->default_value(true), + "Start new log files at midnight."); + options_.add_options()("log-rotation-size", + po::value(&rotation_size_)->default_value(10 << 20), + "Maximum file size in bytes before new log files are started."); +} + +boost::filesystem::path LogOptions::full_log_dir() const { + if (log_dir_.has_root_directory()) { + return log_dir_; + } else { + return base_path_ / log_dir_; + } +} + +po::options_description const& LogOptions::get_options() const { + return options_; +} + +// Typical usage calls either get_options() or parse_command_line() but not both. +void LogOptions::parse_command_line(int argc, char const* const* argv) { + po::variables_map vm; + po::store( + po::command_line_parser(argc, argv).options(options_).allow_unregistered().run(), + vm); + po::notify(vm); // Sets LogOptions member variables. +} + +// Must be called before init() to take effect. +void LogOptions::set_base_path(std::string const& base_path) { + base_path_ = base_path; +} + +std::string replace_braces(std::string const& str, Severity const level) { + constexpr std::regex::flag_type flags = std::regex::ECMAScript | std::regex::optimize; + static std::regex const regex(R"(\{SEVERITY\})", flags); + return std::regex_replace(str, regex, SeverityNames[level]); +} + +// Print decimal value for process_id (14620) instead of hex (0x0000391c) +boost::log::attributes::current_process_id::value_type::native_type get_native_process_id( + boost::log::value_ref const& pid) { + return pid ? pid->native_id() : 0; +} + +template +sinks::text_file_backend::open_handler_type create_or_replace_symlink( + boost::weak_ptr weak_ptr, + std::string&& symlink) { + namespace fs = boost::filesystem; + return [weak_ptr, + symlink = std::move(symlink)](sinks::text_file_backend::stream_type& stream) { + if (boost::shared_ptr sink = weak_ptr.lock()) { + boost::system::error_code ec; + fs::path const& file_name = sink->locked_backend()->get_current_file_name(); + fs::path const symlink_path = file_name.parent_path() / symlink; + fs::remove(symlink_path, ec); + if (ec) { + stream << fs::path(__FILE__).filename().native() << ':' << __LINE__ << ' ' + << ec.message() << std::endl; + } + fs::create_symlink(file_name.filename(), symlink_path, ec); + if (ec) { + stream << fs::path(__FILE__).filename().native() << ':' << __LINE__ << ' ' + << ec.message() << std::endl; + } + } + }; +} + +boost::log::formatting_ostream& operator<<( + boost::log::formatting_ostream& strm, + boost::log::to_log_manip const& manip) { + return strm << SeveritySymbols[manip.get()]; +} + +template +void set_formatter(SINK& sink) { + sink->set_formatter( + expr::stream << expr::format_date_time( + "TimeStamp", "%Y-%m-%dT%H:%M:%S.%f") + << ' ' << severity << ' ' + << boost::phoenix::bind(&get_native_process_id, process_id.or_none()) + << ' ' << expr::smessage); +} + +template +boost::shared_ptr make_sink(LogOptions const& log_opts, + boost::filesystem::path const& full_log_dir, + Severity const level) { + auto sink = boost::make_shared( + keywords::file_name = + full_log_dir / replace_braces(log_opts.file_name_pattern_, level), + keywords::auto_flush = log_opts.auto_flush_, + keywords::rotation_size = log_opts.rotation_size_); + // INFO sink logs all other levels. Other sinks only log at their level or higher. + Severity const min_filter_level = level == Severity::INFO ? log_opts.severity_ : level; + sink->set_filter(min_filter_level <= severity); + set_formatter(sink); + typename FILE_SINK::locked_backend_ptr backend = sink->locked_backend(); + if (log_opts.rotate_daily_) { + backend->set_time_based_rotation(sinks::file::rotation_at_time_point(0, 0, 0)); + } + backend->set_file_collector( + sinks::file::make_collector(keywords::target = full_log_dir, + keywords::max_files = log_opts.max_files_, + keywords::min_free_space = log_opts.min_free_space_)); + backend->set_open_handler(create_or_replace_symlink( + boost::weak_ptr(sink), replace_braces(log_opts.symlink_, level))); + backend->scan_for_files(); + return sink; +} + +// Pointer to function to optionally call on LOG(FATAL). +std::atomic g_fatal_func{nullptr}; +std::once_flag g_fatal_func_flag; + +template <> +BOOST_NORETURN Logger::~Logger() { + if (stream_) { + g_logger::get().push_record( + boost::move(stream_->get_record())); // flushes stream first + } + if (FatalFunc fatal_func = g_fatal_func.load()) { + // set_once_fatal_func() prevents race condition. + // Exceptions thrown by (*fatal_func)() are propagated here. + std::call_once(g_fatal_func_flag, *fatal_func); + } + shutdown(); + abort(); +} + +using ClogSync = sinks::synchronous_sink; +using FileSync = sinks::synchronous_sink; + +template +boost::shared_ptr make_sink(LogOptions const& log_opts) { + auto sink = boost::make_shared(); + boost::shared_ptr clog(&std::clog, boost::null_deleter()); + sink->locked_backend()->add_stream(clog); + sink->set_filter(log_opts.severity_clog_ <= severity); + set_formatter(sink); + return sink; +} + +// Locking/atomicity not needed for g_min_active_severity as it is only +// modifed by init() once. +Severity g_min_active_severity{Severity::FATAL}; + +void init(LogOptions const& log_opts) { + boost::shared_ptr core = boost::log::core::get(); + // boost::log::add_common_attributes(); // LineID TimeStamp ProcessID ThreadID + core->add_global_attribute("TimeStamp", attr::local_clock()); + core->add_global_attribute("ProcessID", attr::current_process_id()); + if (0 < log_opts.max_files_) { + boost::filesystem::path const full_log_dir = log_opts.full_log_dir(); + bool const log_dir_was_created = boost::filesystem::create_directory(full_log_dir); + // Don't create separate log sinks for anything less than Severity::INFO. + Severity const min_sink_level = std::max(Severity::INFO, log_opts.severity_); + for (int i = min_sink_level; i < Severity::NLEVELS; ++i) { + Severity const level = static_cast(i); + core->add_sink(make_sink(log_opts, full_log_dir, level)); + } + g_min_active_severity = std::min(g_min_active_severity, log_opts.severity_); + if (log_dir_was_created) { + LOG(INFO) << "Log directory(" << full_log_dir.native() << ") created."; + } + } + core->add_sink(make_sink(log_opts)); + g_min_active_severity = std::min(g_min_active_severity, log_opts.severity_clog_); +} + +void set_once_fatal_func(FatalFunc fatal_func) { + if (g_fatal_func.exchange(fatal_func)) { + throw std::runtime_error( + "logger::set_once_fatal_func() should not be called more than once."); + } +} + +void shutdown() { + boost::log::core::get()->remove_all_sinks(); +} + +// Used by boost::program_options when parsing enum Severity. +std::istream& operator>>(std::istream& in, Severity& sev) { + std::string token; + in >> token; + auto itr = std::find(SeverityNames.cbegin(), SeverityNames.cend(), token); + if (itr == SeverityNames.cend()) { + in.setstate(std::ios_base::failbit); + } else { + sev = static_cast(itr - SeverityNames.cbegin()); + } + return in; +} + +// Used by boost::program_options when stringifying enum Severity. +std::ostream& operator<<(std::ostream& out, Severity const& sev) { + return out << SeverityNames[sev]; +} + +} // namespace logger + +#endif // #ifndef __CUDACC__ diff --git a/Shared/Logger.h b/Shared/Logger.h new file mode 100644 index 0000000000..8177acb8c6 --- /dev/null +++ b/Shared/Logger.h @@ -0,0 +1,278 @@ +/* + * Copyright (c) 2019 OmniSci, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * @file Logger.h + * @author Matt Pulver + * @description Use Boost.Log for logging data compatible with previous API. + * + * Usage: + * - Initialize a LogOptions object. E.g. + * logger::LogOptions log_options(argv[0]); + * - LogOptions can optionally be added to boost::program_options: + * help_desc.add(log_options.get_options()); + * - Initialize global logger once per application: + * logger::init(log_options); + * - From anywhere in the program: + * - LOG(INFO) << "Nice query!"; + * - LOG(DEBUG4) << "x = " << x; + * - CHECK(condition); + * - CHECK_LE(x, xmax); + * Newlines are automatically appended to log messages. + */ + +#ifndef SHARED_LOGGER_H +#define SHARED_LOGGER_H + +#ifndef __CUDACC__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#endif + +#include + +namespace logger { + +// Must match std::array<> SeverityNames and SeveritySymbols in Logger.cpp. +enum Severity { + DEBUG4 = 0, + DEBUG3, + DEBUG2, + DEBUG1, + INFO, + WARNING, + ERROR, + FATAL, + NLEVELS // number of severity levels +}; + +#ifndef __CUDACC__ + +namespace attr = boost::log::attributes; +namespace expr = boost::log::expressions; +namespace keywords = boost::log::keywords; +namespace sinks = boost::log::sinks; +namespace sources = boost::log::sources; +namespace po = boost::program_options; + +// Filled by boost::program_options +class LogOptions { + std::string base_path_{"."}; // ignored if log_dir_ is absolute. + po::options_description options_; + + public: + // Initialize to default values + boost::filesystem::path log_dir_{"omnisci_log"}; + // file_name_pattern and symlink are prepended with base_name. + std::string file_name_pattern_{".{SEVERITY}.%Y%m%d-%H%M%S.log"}; + std::string symlink_{".{SEVERITY}.log"}; + Severity severity_{Severity::INFO}; + Severity severity_clog_{Severity::WARNING}; + bool auto_flush_{true}; + size_t max_files_{100}; + size_t min_free_space_{20 << 20}; + bool rotate_daily_{true}; + size_t rotation_size_{10 << 20}; + + LogOptions(char const* argv0); + boost::filesystem::path full_log_dir() const; + po::options_description const& get_options() const; + void parse_command_line(int, char const* const*); + void set_base_path(std::string const& base_path); +}; + +// Execute once in main() to initialize boost loggers. +void init(LogOptions const&); + +// Flush all sinks. +// https://www.boost.org/libs/log/doc/html/log/rationale/why_crash_on_term.html +void shutdown(); + +struct LogShutdown { + inline ~LogShutdown() { shutdown(); } +}; + +// Optional pointer to function to call on LOG(FATAL). +typedef void (*FatalFunc)(); +void set_once_fatal_func(FatalFunc); + +using logger_t = boost::log::sources::severity_logger_mt; + +BOOST_LOG_GLOBAL_LOGGER(g_logger, logger_t) + +// Lifetime of Logger is each call to LOG(). +template +class Logger { + // Pointers are used to minimize size of inline objects. + std::unique_ptr record_; + std::unique_ptr stream_; + + public: + Logger(); + Logger(Logger&&) = default; + ~Logger(); + inline operator bool() const { return static_cast(stream_); } + // Must check operator bool() first before calling stream(). + boost::log::record_ostream& stream(char const* file, int line); +}; + +template +Logger::Logger() + : record_(std::make_unique( + g_logger::get().open_record(boost::log::keywords::severity = SEVERITY))) { + if (*record_) { + stream_ = std::make_unique(*record_); + } +} + +template +Logger::~Logger() { + if (stream_) { + g_logger::get().push_record( + boost::move(stream_->get_record())); // flushes stream first + } +} + +template <> +BOOST_NORETURN Logger::~Logger(); + +template +boost::log::record_ostream& Logger::stream(char const* file, int line) { + return *stream_ << boost::filesystem::path(file).filename().native() << ':' << line + << ' '; +} + +extern Severity g_min_active_severity; + +// These macros risk inadvertent else-matching to the if statements, +// which are fortunately prevented by our clang-tidy requirements. +// These can be changed to for/while loops with slight performance degradation. + +#define LOG(severity) \ + if (logger::g_min_active_severity <= logger::Severity::severity) \ + if (auto _omnisci_logger_ = logger::Logger()) \ + _omnisci_logger_.stream(__FILE__, __LINE__) + +#define CHECK(condition) \ + if (BOOST_UNLIKELY(!(condition))) \ + LOG(FATAL) << "Check failed: " #condition " " + +#define CHECK_OP(OP, x, y) \ + if (std::string* fatal_msg = logger::Check##OP(x, y, #x, #y)) \ + LOG(FATAL) << *std::unique_ptr(fatal_msg) + +#define CHECK_EQ(x, y) CHECK_OP(EQ, x, y) +#define CHECK_NE(x, y) CHECK_OP(NE, x, y) +#define CHECK_LT(x, y) CHECK_OP(LT, x, y) +#define CHECK_LE(x, y) CHECK_OP(LE, x, y) +#define CHECK_GT(x, y) CHECK_OP(GT, x, y) +#define CHECK_GE(x, y) CHECK_OP(GE, x, y) + +template +BOOST_NOINLINE std::string* check_failed(X const& x, + Y const& y, + char const* xstr, + char const* ystr, + char const* op_str) { + std::stringstream ss; + ss << "Check failed: " << xstr << op_str << ystr << " (" << x << op_str << y << ") "; + return new std::string(ss.str()); // Deleted by CHECK_OP macro. +} + +// Complexity comes from requirement that x and y be evaluated only once. +#define OMINSCI_CHECKOP_FUNCTION(name, op) \ + template \ + inline std::string* Check##name( \ + X const& x, Y const& y, char const* xstr, char const* ystr) { \ + if (BOOST_LIKELY(x op y)) \ + return nullptr; \ + else \ + return logger::check_failed(x, y, xstr, ystr, " " #op " "); \ + } +OMINSCI_CHECKOP_FUNCTION(EQ, ==) +OMINSCI_CHECKOP_FUNCTION(NE, !=) +OMINSCI_CHECKOP_FUNCTION(LT, <) +OMINSCI_CHECKOP_FUNCTION(LE, <=) +OMINSCI_CHECKOP_FUNCTION(GT, >) +OMINSCI_CHECKOP_FUNCTION(GE, >=) +#undef OMINSCI_CHECKOP_FUNCTION + +#define UNREACHABLE() LOG(FATAL) << "UNREACHABLE " + +#else // __CUDACC__ + +// Provided for backward compatibility to allow code to compile. +// No logging is actually done, since cuda code should not log. +template +class NullLogger { + public: + NullLogger() { + if /* constexpr */ (severity == Severity::FATAL) { + abort(); + } + } + template +#ifndef SUPPRESS_NULL_LOGGER_DEPRECATION_WARNINGS + [[deprecated]] +#endif + // If you are here because of a deprecation warning, that is because the code + // is attempting to log something in cuda (e.g. CHECK macros). It should + // probably be fixed there. + NullLogger& + operator<<(const T&) { + return *this; + } +}; + +#define LOG(severity) logger::NullLogger() + +#define CHECK(condition) LOG_IF(FATAL, !(condition)) + +#define CHECK_EQ(x, y) CHECK((x) == (y)) +#define CHECK_NE(x, y) CHECK((x) != (y)) +#define CHECK_LT(x, y) CHECK((x) < (y)) +#define CHECK_LE(x, y) CHECK((x) <= (y)) +#define CHECK_GT(x, y) CHECK((x) > (y)) +#define CHECK_GE(x, y) CHECK((x) >= (y)) + +#define UNREACHABLE() LOG(FATAL) + +#endif // __CUDACC__ + +#define LOG_IF(severity, condition) \ + if (condition) \ + LOG(severity) + +#define VLOG(n) LOG(DEBUG##n) + +} // namespace logger + +#endif // SHARED_LOGGER_H diff --git a/Shared/SqlTypesLayout.h b/Shared/SqlTypesLayout.h index 9b0f44b9a0..b547a12eb7 100644 --- a/Shared/SqlTypesLayout.h +++ b/Shared/SqlTypesLayout.h @@ -24,11 +24,7 @@ #include "../Shared/TargetInfo.h" -#ifndef __CUDACC__ -#include -#else -#include "../Shared/always_assert.h" -#endif // __CUDACC__ +#include "Logger.h" #include diff --git a/Shared/StringTransform.h b/Shared/StringTransform.h index 36d3151ccc..9a1d01600a 100644 --- a/Shared/StringTransform.h +++ b/Shared/StringTransform.h @@ -17,7 +17,8 @@ #ifndef SHARED_STRINGTRANSFORM_H #define SHARED_STRINGTRANSFORM_H -#include +#include "Logger.h" + #include #include #include diff --git a/Shared/ThriftTypesConvert.h b/Shared/ThriftTypesConvert.h index 959594fe3b..54e3eaf418 100644 --- a/Shared/ThriftTypesConvert.h +++ b/Shared/ThriftTypesConvert.h @@ -19,7 +19,7 @@ #include "gen-cpp/mapd_types.h" -#include +#include "Logger.h" #include diff --git a/Shared/TypedDataAccessors.h b/Shared/TypedDataAccessors.h index f20d30e47f..6d42d04605 100644 --- a/Shared/TypedDataAccessors.h +++ b/Shared/TypedDataAccessors.h @@ -20,12 +20,8 @@ #include #include "Shared/DateConverters.h" #include "Shared/InlineNullValues.h" +#include "Shared/Logger.h" #include "Shared/sqltypes.h" -#include "Shared/unreachable.h" - -#ifndef CHECK // if not collide with the one in glog/logging.h -#include "always_assert.h" -#endif namespace { diff --git a/Shared/always_assert.h b/Shared/always_assert.h deleted file mode 100644 index f24a1761b5..0000000000 --- a/Shared/always_assert.h +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2017 MapD Technologies, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef SHARED_ALWAYS_ASSERT_H -#define SHARED_ALWAYS_ASSERT_H - -#define CHECK(cond) \ - { \ - if (!(cond)) { \ - abort(); \ - } \ - } - -#define CHECK_EQ(ref, val) \ - { \ - if ((val) != (ref)) { \ - abort(); \ - } \ - } - -#define CHECK_GT(val, ref) \ - { \ - if ((val) <= (ref)) { \ - abort(); \ - } \ - } - -#endif // SHARED_ALWAYS_ASSERT_H diff --git a/Shared/checked_alloc.h b/Shared/checked_alloc.h index f142c9179f..a7792a7c93 100644 --- a/Shared/checked_alloc.h +++ b/Shared/checked_alloc.h @@ -19,13 +19,13 @@ #define BOOST_STACKTRACE_GNU_SOURCE_NOT_REQUIRED 1 -#include #include #include #include #include #include #include "../Shared/types.h" +#include "Shared/Logger.h" class OutOfHostMemory : public std::runtime_error { public: diff --git a/Shared/geo_types.cpp b/Shared/geo_types.cpp index 138b7feaf4..fb4f388d95 100644 --- a/Shared/geo_types.cpp +++ b/Shared/geo_types.cpp @@ -15,10 +15,10 @@ */ #include "geo_types.h" +#include "Logger.h" #include "sqltypes.h" #include -#include #include #include @@ -596,7 +596,7 @@ void GeoTypesFactory::getGeoColumnsImpl(const std::unique_ptr& geospati geospatial_poly->getColumns(coords, ring_sizes, bounds); if (promote_poly_to_mpoly) { if (ring_sizes.size()) { - CHECK_GT(coords.size(), 0); + CHECK_GT(coords.size(), 0u); poly_rings.push_back(1 + geospatial_poly->getNumInteriorRings()); } } diff --git a/Shared/mapdpath.h b/Shared/mapdpath.h index a762cf232d..1f851057d9 100644 --- a/Shared/mapdpath.h +++ b/Shared/mapdpath.h @@ -17,8 +17,8 @@ #ifndef _MAPDPATH_H #define _MAPDPATH_H -#include #include +#include "Logger.h" #ifdef __APPLE__ #include diff --git a/Shared/measure.h b/Shared/measure.h index bf0ef5f31f..b19fbce53d 100644 --- a/Shared/measure.h +++ b/Shared/measure.h @@ -20,8 +20,7 @@ #include #include #include - -#include +#include "Logger.h" extern bool g_enable_debug_timer; diff --git a/Shared/sql_window_function_to_string.h b/Shared/sql_window_function_to_string.h index 801e1a352c..01457eb20b 100644 --- a/Shared/sql_window_function_to_string.h +++ b/Shared/sql_window_function_to_string.h @@ -18,8 +18,8 @@ #include "sqldefs.h" -#include #include +#include "Logger.h" inline std::string sql_window_function_to_str(const SqlWindowFunctionKind kind) { switch (kind) { diff --git a/Shared/unreachable.h b/Shared/unreachable.h deleted file mode 100644 index 6e96149503..0000000000 --- a/Shared/unreachable.h +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2017 MapD Technologies, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef SHARED_UNREACHABLE_H -#define SHARED_UNREACHABLE_H - -#ifndef __CUDACC__ -#include - -#define UNREACHABLE() CHECK(false) -#else -#define UNREACHABLE() abort() -#endif // __CUDACC__ - -#endif // SHARED_UNREACHABLE_H diff --git a/StringDictionary/CMakeLists.txt b/StringDictionary/CMakeLists.txt index 56f3eebf51..b2c53e59c9 100644 --- a/StringDictionary/CMakeLists.txt +++ b/StringDictionary/CMakeLists.txt @@ -1,7 +1,7 @@ add_library(StringDictionary StringDictionary.cpp StringDictionaryProxy.cpp) if(ENABLE_FOLLY) - target_link_libraries(StringDictionary Utils ${Glog_LIBRARIES} ${Thrift_LIBRARIES} ${PROFILER_LIBS} ${Folly_LIBRARIES}) + target_link_libraries(StringDictionary Utils ${Boost_LIBRARIES} ${Thrift_LIBRARIES} ${PROFILER_LIBS} ${Folly_LIBRARIES}) else() - target_link_libraries(StringDictionary Utils ${Glog_LIBRARIES} ${Thrift_LIBRARIES} ${PROFILER_LIBS}) + target_link_libraries(StringDictionary Utils ${Boost_LIBRARIES} ${Thrift_LIBRARIES} ${PROFILER_LIBS}) endif() diff --git a/StringDictionary/StringDictionary.cpp b/StringDictionary/StringDictionary.cpp index a06d238d28..d718ada53a 100644 --- a/StringDictionary/StringDictionary.cpp +++ b/StringDictionary/StringDictionary.cpp @@ -19,10 +19,10 @@ #include "../Utils/Regexp.h" #include "../Utils/StringLike.h" #include "LeafHostInfo.h" +#include "Shared/Logger.h" #include "Shared/thread_count.h" #include "StringDictionaryClient.h" -#include #include #include #include @@ -96,7 +96,9 @@ uint32_t rk_hash(const std::string& str) { } } // namespace -const int32_t StringDictionary::INVALID_STR_ID{-1}; +constexpr int32_t StringDictionary::INVALID_STR_ID; +constexpr size_t StringDictionary::MAX_STRLEN; +constexpr size_t StringDictionary::MAX_STRCOUNT; StringDictionary::StringDictionary(const std::string& folder, const bool isTemp, @@ -777,7 +779,7 @@ std::shared_ptr> StringDictionary::copyStrings() const bool multithreaded = str_count_ > 10000; const auto worker_count = multithreaded ? static_cast(cpu_threads()) : size_t(1); - CHECK_GT(worker_count, 0); + CHECK_GT(worker_count, 0UL); std::vector> worker_results(worker_count); auto copy = [this](std::vector& str_list, const size_t start_id, diff --git a/StringDictionary/StringDictionary.h b/StringDictionary/StringDictionary.h index b12ae4db13..be861d0710 100644 --- a/StringDictionary/StringDictionary.h +++ b/StringDictionary/StringDictionary.h @@ -92,9 +92,9 @@ class StringDictionary { const std::vector>& source_array_ids, const StringDictionary* source_dict); - static const int32_t INVALID_STR_ID; - static const size_t MAX_STRLEN = (1 << 15) - 1; - static const size_t MAX_STRCOUNT = (1U << 31) - 1; + static constexpr int32_t INVALID_STR_ID = -1; + static constexpr size_t MAX_STRLEN = (1 << 15) - 1; + static constexpr size_t MAX_STRCOUNT = (1U << 31) - 1; private: struct StringIdxEntry { diff --git a/StringDictionary/StringDictionaryProxy.cpp b/StringDictionary/StringDictionaryProxy.cpp index 819ea29be3..129da4d3b3 100644 --- a/StringDictionary/StringDictionaryProxy.cpp +++ b/StringDictionary/StringDictionaryProxy.cpp @@ -18,10 +18,10 @@ #include "../Shared/sqltypes.h" #include "../Utils/Regexp.h" #include "../Utils/StringLike.h" +#include "Shared/Logger.h" #include "Shared/thread_count.h" #include "StringDictionary.h" -#include #include #include diff --git a/Tests/AlterColumnTest.cpp b/Tests/AlterColumnTest.cpp index e92399fba7..fffc423efe 100644 --- a/Tests/AlterColumnTest.cpp +++ b/Tests/AlterColumnTest.cpp @@ -14,7 +14,6 @@ * limitations under the License. */ -#include #include #include @@ -255,7 +254,7 @@ TEST(AlterColumnTest2, Drop_after_fail_to_add) { } // namespace int main(int argc, char** argv) { - google::InitGoogleLogging(argv[0]); + TestHelpers::init_logger_stderr_only(argc, argv); testing::InitGoogleTest(&argc, argv); g_session.reset(QueryRunner::get_session(BASE_PATH)); diff --git a/Tests/CMakeLists.txt b/Tests/CMakeLists.txt index 9bf0eabf00..b26a9681a3 100644 --- a/Tests/CMakeLists.txt +++ b/Tests/CMakeLists.txt @@ -15,6 +15,9 @@ add_executable(ExperimentalTest ExperimentalTest.cpp) if(ENABLE_CUDA) set(MAPD_DEFINITIONS -DHAVE_CUDA) + if (SUPPRESS_NULL_LOGGER_DEPRECATION_WARNINGS) + list(APPEND MAPD_DEFINITIONS -DSUPPRESS_NULL_LOGGER_DEPRECATION_WARNINGS) + endif() add_custom_command( DEPENDS ProfileUtils.cu ProfileTest.h OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/libProfileUtils.so @@ -61,15 +64,15 @@ if(NOT ${CMAKE_SYSTEM_NAME} STREQUAL "Darwin") add_executable(UdfTest UdfTest.cpp) endif() -target_link_libraries(ProfileTest gtest Shared Calcite QueryEngine ${MAPD_RENDERING_LIBRARIES} CsvImport QueryRunner Parser ${Boost_LIBRARIES} ${Glog_LIBRARIES} ${CMAKE_DL_LIBS} ${CUDA_LIBRARIES} ${PROF_LIBRARIES} ${LLVM_LINKER_FLAGS} ${CURSES_LIBRARIES}) -target_link_libraries(ResultSetTest gtest QueryEngine ${MAPD_RENDERING_LIBRARIES} ${Boost_LIBRARIES} CsvImport QueryRunner Parser DataMgr Chunk ${Boost_LIBRARIES} ${Glog_LIBRARIES} ${CMAKE_DL_LIBS} ${CUDA_LIBRARIES} ${LLVM_LINKER_FLAGS} ${CURSES_LIBRARIES}) -target_link_libraries(FromTableReorderingTest gtest QueryEngine ${MAPD_RENDERING_LIBRARIES} ${Boost_LIBRARIES} CsvImport QueryRunner Parser DataMgr Chunk ${Boost_LIBRARIES} ${Glog_LIBRARIES} ${CMAKE_DL_LIBS} ${CUDA_LIBRARIES} ${LLVM_LINKER_FLAGS} ${CURSES_LIBRARIES}) -target_link_libraries(ResultSetBaselineRadixSortTest gtest QueryEngine ${MAPD_RENDERING_LIBRARIES} CsvImport QueryRunner Parser DataMgr Chunk ${Boost_LIBRARIES} ${Glog_LIBRARIES} ${CMAKE_DL_LIBS} ${CUDA_LIBRARIES} ${LLVM_LINKER_FLAGS} ${CURSES_LIBRARIES}) -target_link_libraries(UtilTest Utils gtest ${Glog_LIBRARIES} ${Boost_LIBRARIES}) -target_link_libraries(StringDictionaryTest StringDictionary gtest ${Glog_LIBRARIES} ${Boost_LIBRARIES}) -target_link_libraries(StringTransformTest Shared gtest ${Glog_LIBRARIES} ${Boost_LIBRARIES}) -target_link_libraries(TokenCompletionHintsTest token_completion_hints gtest mapd_thrift ${Glog_LIBRARIES} ${Boost_LIBRARIES}) -set(EXECUTE_TEST_LIBS gtest QueryRunner ${MAPD_LIBRARIES} ${Boost_LIBRARIES} ${Glog_LIBRARIES} ${CMAKE_DL_LIBS} ${CUDA_LIBRARIES} ${LLVM_LINKER_FLAGS} ${CURSES_LIBRARIES}) +target_link_libraries(ProfileTest gtest Shared Calcite QueryEngine ${MAPD_RENDERING_LIBRARIES} CsvImport QueryRunner Parser ${Boost_LIBRARIES} ${CMAKE_DL_LIBS} ${CUDA_LIBRARIES} ${PROF_LIBRARIES} ${LLVM_LINKER_FLAGS} ${CURSES_LIBRARIES}) +target_link_libraries(ResultSetTest gtest QueryEngine ${MAPD_RENDERING_LIBRARIES} CsvImport QueryRunner Parser DataMgr Chunk Shared ${Boost_LIBRARIES} ${CMAKE_DL_LIBS} ${CUDA_LIBRARIES} ${LLVM_LINKER_FLAGS} ${CURSES_LIBRARIES}) +target_link_libraries(FromTableReorderingTest gtest QueryEngine ${MAPD_RENDERING_LIBRARIES} CsvImport QueryRunner Parser DataMgr Chunk ${Boost_LIBRARIES} ${CMAKE_DL_LIBS} ${CUDA_LIBRARIES} ${LLVM_LINKER_FLAGS} ${CURSES_LIBRARIES}) +target_link_libraries(ResultSetBaselineRadixSortTest gtest QueryEngine ${MAPD_RENDERING_LIBRARIES} CsvImport QueryRunner Parser DataMgr Chunk Shared ${Boost_LIBRARIES} ${CMAKE_DL_LIBS} ${CUDA_LIBRARIES} ${LLVM_LINKER_FLAGS} ${CURSES_LIBRARIES}) +target_link_libraries(UtilTest Utils gtest Shared ${Boost_LIBRARIES}) +target_link_libraries(StringDictionaryTest StringDictionary gtest Shared ${Boost_LIBRARIES}) +target_link_libraries(StringTransformTest Shared gtest ${Boost_LIBRARIES}) +target_link_libraries(TokenCompletionHintsTest token_completion_hints gtest mapd_thrift Shared ${Boost_LIBRARIES}) +set(EXECUTE_TEST_LIBS gtest QueryRunner ${MAPD_LIBRARIES} ${CMAKE_DL_LIBS} ${CUDA_LIBRARIES} ${LLVM_LINKER_FLAGS} ${CURSES_LIBRARIES} ${Boost_LIBRARIES}) list(APPEND EXECUTE_TEST_LIBS Calcite) list(APPEND EXECUTE_TEST_LIBS Calcite mapd_thrift ${PROFILER_LIBS}) @@ -85,13 +88,13 @@ target_link_libraries(ComputeMetadataTest ${EXECUTE_TEST_LIBS}) target_link_libraries(UpdateMetadataTest ${EXECUTE_TEST_LIBS}) target_link_libraries(StoragePerfTest gtest ${EXECUTE_TEST_LIBS}) target_link_libraries(TopKTest ${EXECUTE_TEST_LIBS}) -target_link_libraries(OmniSQLCommandTest gtest ${EXECUTE_TEST_LIBS} ${Boost_LIBRARIES}) +target_link_libraries(OmniSQLCommandTest gtest ${EXECUTE_TEST_LIBS}) target_link_libraries(OmniSQLUtilitiesTest gtest ${Boost_LIBRARIES}) -target_link_libraries(ExperimentalTest gtest ${EXECUTE_TEST_LIBS} ${Boost_LIBRARIES}) -target_link_libraries(DBObjectPrivilegesTest gtest ${EXECUTE_TEST_LIBS} ${Boost_LIBRARIES}) +target_link_libraries(ExperimentalTest gtest ${EXECUTE_TEST_LIBS}) +target_link_libraries(DBObjectPrivilegesTest gtest ${EXECUTE_TEST_LIBS}) target_link_libraries(GeoTypesTest gtest ${EXECUTE_TEST_LIBS}) target_link_libraries(CtasUpdateTest gtest ${EXECUTE_TEST_LIBS}) -target_link_libraries(CtasIntegrationTest gtest ${MAPD_LIBRARIES} ${Glog_LIBRARIES} ${Boost_LIBRARIES} Shared mapd_thrift ThriftClient ${LLVM_LINKER_FLAGS}) +target_link_libraries(CtasIntegrationTest gtest ${MAPD_LIBRARIES} Shared mapd_thrift ThriftClient ${LLVM_LINKER_FLAGS} ${Boost_LIBRARIES}) target_link_libraries(DateTimeUtilsTest gtest ${EXECUTE_TEST_LIBS}) target_link_libraries(CalciteOptimizeTest gtest ${EXECUTE_TEST_LIBS} ${Boost_LIBRARIES}) if(NOT ${CMAKE_SYSTEM_NAME} STREQUAL "Darwin") diff --git a/Tests/CalciteOptimizeTest.cpp b/Tests/CalciteOptimizeTest.cpp index 9ada9112f2..ca92f19a79 100644 --- a/Tests/CalciteOptimizeTest.cpp +++ b/Tests/CalciteOptimizeTest.cpp @@ -1,4 +1,3 @@ -#include #include #include #include @@ -10,6 +9,7 @@ #include "../Parser/parser.h" #include "../QueryRunner/QueryRunner.h" #include "Shared/MapDParameters.h" +#include "TestHelpers.h" #include "gen-cpp/CalciteServer.h" namespace { @@ -85,8 +85,8 @@ TEST_F(ViewObject, BasicTest) { } int main(int argc, char* argv[]) { + TestHelpers::init_logger_stderr_only(argc, argv); testing::InitGoogleTest(&argc, argv); - google::InitGoogleLogging(argv[0]); g_session.reset(QueryRunner::get_session(BASE_PATH)); g_calcite = g_session->getCatalog().getCalciteMgr(); diff --git a/Tests/CodeGeneratorTest.cpp b/Tests/CodeGeneratorTest.cpp index 99e26c053e..19e01e876b 100644 --- a/Tests/CodeGeneratorTest.cpp +++ b/Tests/CodeGeneratorTest.cpp @@ -14,7 +14,6 @@ * limitations under the License. */ -#include #include #include #include @@ -29,6 +28,7 @@ #include "../QueryEngine/IRCodegenUtils.h" #include "../QueryEngine/LLVMGlobalContext.h" #include "../Shared/mapdpath.h" +#include "TestHelpers.h" namespace { @@ -385,7 +385,7 @@ TEST(CodeGeneratorTest, IntegerExprGPU) { #endif // HAVE_CUDA int main(int argc, char** argv) { - google::InitGoogleLogging(argv[0]); + TestHelpers::init_logger_stderr_only(argc, argv); testing::InitGoogleTest(&argc, argv); int err = RUN_ALL_TESTS(); return err; diff --git a/Tests/ComputeMetadataTest.cpp b/Tests/ComputeMetadataTest.cpp index 4b342738c6..c52b034591 100644 --- a/Tests/ComputeMetadataTest.cpp +++ b/Tests/ComputeMetadataTest.cpp @@ -21,7 +21,6 @@ #include "../QueryEngine/TableOptimizer.h" #include "../QueryRunner/QueryRunner.h" -#include #include #include #include @@ -485,7 +484,7 @@ TEST_UNSHARDED_AND_SHARDED(MetadataUpdate, DeleteReset) TEST_UNSHARDED_AND_SHARDED(MetadataUpdate, EncodedStringNull) int main(int argc, char** argv) { - google::InitGoogleLogging(argv[0]); + TestHelpers::init_logger_stderr_only(argc, argv); testing::InitGoogleTest(&argc, argv); g_session.reset(QueryRunner::get_session(BASE_PATH)); diff --git a/Tests/CtasIntegrationTest.cpp b/Tests/CtasIntegrationTest.cpp index 76c8679da1..9c6410cd38 100644 --- a/Tests/CtasIntegrationTest.cpp +++ b/Tests/CtasIntegrationTest.cpp @@ -14,7 +14,6 @@ * limitations under the License. */ -#include #include #include @@ -31,6 +30,7 @@ using namespace ::apache::thrift::transport; #include "QueryEngine/TargetValue.h" #include "Shared/ThriftClient.h" #include "Shared/sqltypes.h" +#include "TestHelpers.h" #include "gen-cpp/MapD.h" #include @@ -1039,10 +1039,10 @@ INSTANTIATE_TEST_CASE_P( int main(int argc, char* argv[]) { int err = 0; + TestHelpers::init_logger_stderr_only(argc, argv); try { testing::InitGoogleTest(&argc, argv); - google::InitGoogleLogging(argv[0]); namespace po = boost::program_options; diff --git a/Tests/CtasUpdateTest.cpp b/Tests/CtasUpdateTest.cpp index c1c0456c64..025048ba9b 100644 --- a/Tests/CtasUpdateTest.cpp +++ b/Tests/CtasUpdateTest.cpp @@ -16,7 +16,6 @@ #include "../QueryRunner/QueryRunner.h" -#include #include #include @@ -25,6 +24,7 @@ #include "../QueryEngine/ArrowResultSet.h" #include "../QueryEngine/Execute.h" #include "../Shared/file_delete.h" +#include "TestHelpers.h" #include "../Shared/ConfigResolve.h" @@ -1494,10 +1494,10 @@ INSTANTIATE_TEST_CASE_P( int main(int argc, char* argv[]) { int err = 0; + TestHelpers::init_logger_stderr_only(argc, argv); try { testing::InitGoogleTest(&argc, argv); - google::InitGoogleLogging(argv[0]); g_session.reset(QueryRunner::get_session(BASE_PATH)); err = RUN_ALL_TESTS(); diff --git a/Tests/DBObjectPrivilegesTest.cpp b/Tests/DBObjectPrivilegesTest.cpp index f315f34125..00ba05d1e5 100644 --- a/Tests/DBObjectPrivilegesTest.cpp +++ b/Tests/DBObjectPrivilegesTest.cpp @@ -1,4 +1,3 @@ -#include #include #include #include @@ -14,6 +13,7 @@ #include "../QueryRunner/QueryRunner.h" #include "Shared/MapDParameters.h" #include "Shared/scope.h" +#include "TestHelpers.h" #include "gen-cpp/CalciteServer.h" #ifndef BASE_PATH @@ -2317,8 +2317,8 @@ TEST(SysCatalog, RecursiveRolesUserMetaData) { } int main(int argc, char* argv[]) { + TestHelpers::init_logger_stderr_only(argc, argv); testing::InitGoogleTest(&argc, argv); - google::InitGoogleLogging(argv[0]); g_session.reset(QueryRunner::get_session(BASE_PATH)); g_calcite = g_session->getCatalog().getCalciteMgr(); diff --git a/Tests/ExecuteTest.cpp b/Tests/ExecuteTest.cpp index b6da0cc671..21f1b36c40 100644 --- a/Tests/ExecuteTest.cpp +++ b/Tests/ExecuteTest.cpp @@ -28,7 +28,6 @@ #include "../SqliteConnector/SqliteConnector.h" #include "DistributedLoader.h" -#include #include #include #include @@ -1746,7 +1745,7 @@ TEST(Select, OrderBy) { SKIP_NO_GPU(); const auto rows = run_multiple_agg( "SELECT x, y, z + t, x * y AS m FROM test ORDER BY 3 desc LIMIT 5;", dt); - CHECK_EQ(rows->rowCount(), std::min(size_t(5), static_cast(g_num_rows))); + CHECK_EQ(rows->rowCount(), std::min(size_t(5), static_cast(g_num_rows)) + 0); CHECK_EQ(rows->colCount(), size_t(4)); for (size_t row_idx = 0; row_idx < rows->rowCount(); ++row_idx) { ASSERT_TRUE(v(rows->getRowAt(row_idx, 0, true)) == 8 || @@ -15958,7 +15957,6 @@ void drop_views() { } // namespace int main(int argc, char** argv) { - google::InitGoogleLogging(argv[0]); testing::InitGoogleTest(&argc, argv); namespace po = boost::program_options; @@ -16002,6 +16000,10 @@ int main(int argc, char** argv) { "test-help", "Print all ExecuteTest specific options (for gtest options use `--help`)."); + logger::LogOptions log_options(argv[0]); + log_options.max_files_ = 0; // stderr only by default + desc.add(log_options.get_options()); + po::variables_map vm; po::store(po::command_line_parser(argc, argv).options(desc).run(), vm); po::notify(vm); @@ -16012,6 +16014,8 @@ int main(int argc, char** argv) { return 0; } + logger::init(log_options); + if (vm.count("disable-literal-hoisting")) { g_hoist_literals = false; } diff --git a/Tests/ExperimentalTest.cpp b/Tests/ExperimentalTest.cpp index a370281f9b..6ff8c722a1 100644 --- a/Tests/ExperimentalTest.cpp +++ b/Tests/ExperimentalTest.cpp @@ -1,7 +1,7 @@ #include "../Shared/ConstExprLib.h" #include "../Shared/ExperimentalTypeUtilities.h" +#include "TestHelpers.h" -#include #include #include @@ -55,7 +55,7 @@ TEST(ExperimentalTest, ConstExprLib_Pair) { } int main(int argc, char* argv[]) { - google::InitGoogleLogging(argv[0]); + TestHelpers::init_logger_stderr_only(argc, argv); ::testing::InitGoogleTest(&argc, argv); int err{0}; diff --git a/Tests/FromTableReorderingTest.cpp b/Tests/FromTableReorderingTest.cpp index aae6b3b727..dca62f98ef 100644 --- a/Tests/FromTableReorderingTest.cpp +++ b/Tests/FromTableReorderingTest.cpp @@ -30,7 +30,6 @@ #include "../SqliteConnector/SqliteConnector.h" #include "DistributedLoader.h" -#include #include TEST(Ordering, Basic) { @@ -365,7 +364,7 @@ TEST(Ordering, Triple) { } int main(int argc, char** argv) { - google::InitGoogleLogging(argv[0]); + TestHelpers::init_logger_stderr_only(argc, argv); testing::InitGoogleTest(&argc, argv); int err{0}; diff --git a/Tests/ImportTest.cpp b/Tests/ImportTest.cpp index c83995d043..5af36c9c2d 100644 --- a/Tests/ImportTest.cpp +++ b/Tests/ImportTest.cpp @@ -22,7 +22,6 @@ #include #include -#include #include #include @@ -1415,7 +1414,6 @@ TEST_F(ImportTest, S3_GCS_One_geo_file) { } // namespace int main(int argc, char** argv) { - google::InitGoogleLogging(argv[0]); testing::InitGoogleTest(&argc, argv); namespace po = boost::program_options; @@ -1430,6 +1428,10 @@ int main(int argc, char** argv) { "test-help", "Print all ExecuteTest specific options (for gtest options use `--help`)."); + logger::LogOptions log_options(argv[0]); + log_options.max_files_ = 0; // stderr only by default + desc.add(log_options.get_options()); + po::variables_map vm; po::store(po::command_line_parser(argc, argv).options(desc).run(), vm); po::notify(vm); @@ -1440,6 +1442,8 @@ int main(int argc, char** argv) { return 0; } + logger::init(log_options); + g_session.reset(QueryRunner::get_session(BASE_PATH)); int err{0}; diff --git a/Tests/PlanTest.cpp b/Tests/PlanTest.cpp index e2cfcf0bf8..8c64ffd427 100644 --- a/Tests/PlanTest.cpp +++ b/Tests/PlanTest.cpp @@ -29,9 +29,9 @@ #include "../Planner/Planner.h" #include "../QueryRunner/QueryRunner.h" #include "Shared/MapDParameters.h" +#include "TestHelpers.h" #include "boost/filesystem.hpp" #include "boost/program_options.hpp" -#include "glog/logging.h" #include "gtest/gtest.h" using namespace std; @@ -337,7 +337,7 @@ void drop_views_and_tables() { } int main(int argc, char* argv[]) { - google::InitGoogleLogging(argv[0]); + TestHelpers::init_logger_stderr_only(argc, argv); ::testing::InitGoogleTest(&argc, argv); g_session.reset(QueryRunner::get_session(BASE_PATH)); diff --git a/Tests/PopulateTableRandom.cpp b/Tests/PopulateTableRandom.cpp index aba819d49e..c4c04b7f49 100644 --- a/Tests/PopulateTableRandom.cpp +++ b/Tests/PopulateTableRandom.cpp @@ -33,14 +33,13 @@ #include #include -#include - #include "../Catalog/Catalog.h" #include "../DataMgr/DataMgr.h" #include "../Fragmenter/Fragmenter.h" #include "../Shared/DateConverters.h" #include "../Shared/measure.h" #include "../Shared/sqltypes.h" +#include "Shared/Logger.h" using namespace Catalog_Namespace; using namespace Fragmenter_Namespace; diff --git a/Tests/ProfileTest.cpp b/Tests/ProfileTest.cpp index a686855736..3c42342ed4 100644 --- a/Tests/ProfileTest.cpp +++ b/Tests/ProfileTest.cpp @@ -25,6 +25,7 @@ #include "../QueryEngine/Descriptors/RowSetMemoryOwner.h" #include "../QueryEngine/ResultSet.h" #include "Shared/measure.h" +#include "TestHelpers.h" #if defined(HAVE_CUDA) && CUDA_VERSION >= 8000 #include @@ -1280,7 +1281,7 @@ TEST(Reduction, Baseline) { std::reverse(target_infos.begin(), target_infos.end()); const auto device_type = ExecutorDeviceType::CPU; - CHECK_GT(key_count, 1); + CHECK_GT(key_count, 1u); size_t row_size = key_count * sizeof(int64_t); std::vector group_col_widths(key_count, sizeof(int64_t)); QueryMemoryDescriptor query_mem_desc( @@ -1832,7 +1833,7 @@ TEST(Reduction, PerfectHash) { } int main(int argc, char** argv) { - google::InitGoogleLogging(argv[0]); + TestHelpers::init_logger_stderr_only(argc, argv); testing::InitGoogleTest(&argc, argv); g_gpus_present = is_gpu_present(); #ifndef HAVE_CUDA diff --git a/Tests/ProfileTest.h b/Tests/ProfileTest.h index 44d6fb3c37..3176ec62a8 100644 --- a/Tests/ProfileTest.h +++ b/Tests/ProfileTest.h @@ -27,11 +27,7 @@ #include "../QueryEngine/GpuRtConstants.h" -#ifndef __CUDACC__ -#include -#else -#include "../Shared/always_assert.h" -#endif // __CUDACC__ +#include "../Shared/Logger.h" #ifdef HAVE_CUDA #include diff --git a/Tests/ResultSetBaselineRadixSortTest.cpp b/Tests/ResultSetBaselineRadixSortTest.cpp index dc1f12e9ea..e82667d44e 100644 --- a/Tests/ResultSetBaselineRadixSortTest.cpp +++ b/Tests/ResultSetBaselineRadixSortTest.cpp @@ -25,6 +25,7 @@ #include "../QueryEngine/ResultSet.h" #include "../QueryEngine/RuntimeFunctions.h" #include "ResultSetTestUtils.h" +#include "TestHelpers.h" #ifdef HAVE_CUDA #include "../CudaMgr/CudaMgr.h" @@ -32,7 +33,6 @@ extern std::unique_ptr g_cuda_mgr; #endif // HAVE_CUDA -#include #include #include @@ -258,7 +258,7 @@ TEST(SortBaseline, Floats) { } int main(int argc, char** argv) { - google::InitGoogleLogging(argv[0]); + TestHelpers::init_logger_stderr_only(argc, argv); testing::InitGoogleTest(&argc, argv); #ifdef HAVE_CUDA diff --git a/Tests/ResultSetTest.cpp b/Tests/ResultSetTest.cpp index 830f84b49a..e6467a2d4e 100644 --- a/Tests/ResultSetTest.cpp +++ b/Tests/ResultSetTest.cpp @@ -27,8 +27,8 @@ #include "../QueryEngine/ResultSet.h" #include "../QueryEngine/RuntimeFunctions.h" #include "../StringDictionary/StringDictionary.h" +#include "TestHelpers.h" -#include #include #include @@ -3314,7 +3314,7 @@ TEST(ReduceRandomGroups, BaselineHashColumnar_Large_NullVal_0075) { } int main(int argc, char** argv) { - google::InitGoogleLogging(argv[0]); + TestHelpers::init_logger_stderr_only(argc, argv); testing::InitGoogleTest(&argc, argv); int err{0}; diff --git a/Tests/Shared/DateTimeUtilsTest.cpp b/Tests/Shared/DateTimeUtilsTest.cpp index c6cd7ff919..578bb8c939 100644 --- a/Tests/Shared/DateTimeUtilsTest.cpp +++ b/Tests/Shared/DateTimeUtilsTest.cpp @@ -15,8 +15,8 @@ */ #include "../../Shared/DateConverters.h" +#include "Tests/TestHelpers.h" -#include #include namespace { @@ -56,7 +56,7 @@ TEST(DATE, EpochDaysToSecondsTest) { } int main(int argc, char** argv) { - google::InitGoogleLogging(argv[0]); + TestHelpers::init_logger_stderr_only(argc, argv); testing::InitGoogleTest(&argc, argv); int err{0}; diff --git a/Tests/Shared/GeoTypesTest.cpp b/Tests/Shared/GeoTypesTest.cpp index 9f80cc6685..76a15f396f 100644 --- a/Tests/Shared/GeoTypesTest.cpp +++ b/Tests/Shared/GeoTypesTest.cpp @@ -15,8 +15,8 @@ */ #include +#include "Tests/TestHelpers.h" -#include #include using namespace Geo_namespace; @@ -307,7 +307,7 @@ TEST(GeoMisc, Inequality) { } int main(int argc, char** argv) { - google::InitGoogleLogging(argv[0]); + TestHelpers::init_logger_stderr_only(argc, argv); testing::InitGoogleTest(&argc, argv); int err{0}; diff --git a/Tests/StoragePerfTest.cpp b/Tests/StoragePerfTest.cpp index 6e5760845a..7736b2f3bb 100644 --- a/Tests/StoragePerfTest.cpp +++ b/Tests/StoragePerfTest.cpp @@ -34,9 +34,9 @@ #include "PopulateTableRandom.h" #include "ScanTable.h" #include "Shared/MapDParameters.h" +#include "TestHelpers.h" #include "boost/filesystem.hpp" #include "boost/program_options.hpp" -#include "glog/logging.h" #include "gtest/gtest.h" using namespace std; @@ -366,7 +366,7 @@ TEST(DataLoad, NumbersTable_Parallel_CreateDropCreateTable_InsertRows) { } int main(int argc, char* argv[]) { - google::InitGoogleLogging(argv[0]); + TestHelpers::init_logger_stderr_only(argc, argv); ::testing::InitGoogleTest(&argc, argv); g_session.reset(QueryRunner::get_session(BASE_PATH)); diff --git a/Tests/StorageTest.cpp b/Tests/StorageTest.cpp index 0435512612..c088d375b7 100644 --- a/Tests/StorageTest.cpp +++ b/Tests/StorageTest.cpp @@ -36,9 +36,9 @@ #include "PopulateTableRandom.h" #include "ScanTable.h" #include "Shared/MapDParameters.h" +#include "TestHelpers.h" #include "boost/filesystem.hpp" #include "boost/program_options.hpp" -#include "glog/logging.h" #include "gtest/gtest.h" using namespace std; @@ -174,7 +174,7 @@ TEST(StorageSmallParallel, AllTypes) { } int main(int argc, char* argv[]) { - google::InitGoogleLogging(argv[0]); + TestHelpers::init_logger_stderr_only(argc, argv); ::testing::InitGoogleTest(&argc, argv); g_session.reset(QueryRunner::get_session(BASE_PATH)); diff --git a/Tests/StringDictionaryTest.cpp b/Tests/StringDictionaryTest.cpp index 77c90f9350..a124bb8cd8 100644 --- a/Tests/StringDictionaryTest.cpp +++ b/Tests/StringDictionaryTest.cpp @@ -15,10 +15,10 @@ */ #include "../StringDictionary/StringDictionary.h" +#include "TestHelpers.h" #include -#include #include #ifndef BASE_PATH @@ -87,7 +87,7 @@ TEST(StringDictionary, RecoverMany) { } int main(int argc, char** argv) { - google::InitGoogleLogging(argv[0]); + TestHelpers::init_logger_stderr_only(argc, argv); testing::InitGoogleTest(&argc, argv); int err{0}; diff --git a/Tests/StringTransformTest.cpp b/Tests/StringTransformTest.cpp index 6f6efc7adc..43cb9f9b2c 100644 --- a/Tests/StringTransformTest.cpp +++ b/Tests/StringTransformTest.cpp @@ -15,8 +15,8 @@ */ #include "../Shared/StringTransform.h" +#include "TestHelpers.h" -#include #include TEST(StringTransform, CsvQuote) { @@ -68,7 +68,7 @@ TEST(StringTransform, HideSensitiveDataFromQuery) { } int main(int argc, char* argv[]) { - google::InitGoogleLogging(argv[0]); + TestHelpers::init_logger_stderr_only(argc, argv); ::testing::InitGoogleTest(&argc, argv); int err{0}; diff --git a/Tests/TestHelpers.h b/Tests/TestHelpers.h index 81580ed2f2..57ce587c9f 100644 --- a/Tests/TestHelpers.h +++ b/Tests/TestHelpers.h @@ -18,11 +18,13 @@ #define TEST_HELPERS_H_ #include "../QueryEngine/TargetValue.h" +#include "Shared/Logger.h" #include "LeafHostInfo.h" #include #include +#include #include namespace TestHelpers { @@ -189,6 +191,13 @@ std::vector to_leaf_host_info(std::vector& server_inf return host_infos; } +void init_logger_stderr_only(int argc, char const* const* argv) { + logger::LogOptions log_options(argv[0]); + log_options.max_files_ = 0; // stderr only by default + log_options.parse_command_line(argc, argv); + logger::init(log_options); +} + } // namespace TestHelpers #endif // TEST_HELPERS_H_ diff --git a/Tests/TokenCompletionHintsTest.cpp b/Tests/TokenCompletionHintsTest.cpp index 943d83b3e4..5f4df6f03b 100644 --- a/Tests/TokenCompletionHintsTest.cpp +++ b/Tests/TokenCompletionHintsTest.cpp @@ -15,8 +15,8 @@ */ #include "../ThriftHandler/TokenCompletionHints.h" +#include "TestHelpers.h" -#include #include TEST(FindLastWord, SimpleId) { @@ -183,7 +183,7 @@ TEST(Completion, ShouldSuggestColumnHints) { } int main(int argc, char** argv) { - google::InitGoogleLogging(argv[0]); + TestHelpers::init_logger_stderr_only(argc, argv); testing::InitGoogleTest(&argc, argv); int err{0}; diff --git a/Tests/TopKTest.cpp b/Tests/TopKTest.cpp index 6e7f32ffcb..41e1672951 100644 --- a/Tests/TopKTest.cpp +++ b/Tests/TopKTest.cpp @@ -22,8 +22,8 @@ #include "../QueryEngine/Execute.h" #include "../QueryRunner/QueryRunner.h" #include "../SqliteConnector/SqliteConnector.h" +#include "TestHelpers.h" -#include #include #include #include @@ -717,7 +717,7 @@ TEST(Select, DISABLED_TopK_LIMIT_OFFSET_DifferentOrders) { } int main(int argc, char* argv[]) { - google::InitGoogleLogging(argv[0]); + TestHelpers::init_logger_stderr_only(argc, argv); ::testing::InitGoogleTest(&argc, argv); g_session.reset(QueryRunner::get_session(BASE_PATH)); diff --git a/Tests/UdfTest.cpp b/Tests/UdfTest.cpp index c712220168..9288230d19 100644 --- a/Tests/UdfTest.cpp +++ b/Tests/UdfTest.cpp @@ -14,7 +14,6 @@ * limitations under the License. */ -#include #include #include #include @@ -30,6 +29,7 @@ #include "QueryEngine/ResultSet.h" #include "QueryEngine/UDFCompiler.h" #include "QueryRunner/QueryRunner.h" +#include "Shared/Logger.h" #include "Shared/MapDParameters.h" #include "TestHelpers.h" @@ -87,7 +87,7 @@ void mapd_signal_handler(int signal_number) { LOG(ERROR) << "Interrupt signal (" << signal_number << ") received."; calcite_shutdown_handler(); // shut down logging force a flush - google::ShutdownGoogleLogging(); + logger::shutdown(); // terminate program if (signal_number == SIGTERM) { std::exit(EXIT_SUCCESS); @@ -142,7 +142,6 @@ class SQLTestEnv : public ::testing::Environment { DBMetadata db; register_signal_handler(); - google::InstallFailureFunction(&calcite_shutdown_handler); boost::filesystem::path udf_file((get_udf_filename())); if (!boost::filesystem::exists(udf_file)) { @@ -317,7 +316,7 @@ TEST_F(UDFCompilerTest, UdfQuery) { } int main(int argc, char** argv) { - google::InitGoogleLogging(argv[0]); + TestHelpers::init_logger_stderr_only(argc, argv); ::testing::InitGoogleTest(&argc, argv); ::testing::AddGlobalTestEnvironment(new SQLTestEnv); diff --git a/Tests/UpdateMetadataTest.cpp b/Tests/UpdateMetadataTest.cpp index 48b92003a6..2df19d3096 100644 --- a/Tests/UpdateMetadataTest.cpp +++ b/Tests/UpdateMetadataTest.cpp @@ -21,7 +21,6 @@ #include "../QueryEngine/TableOptimizer.h" #include "../QueryRunner/QueryRunner.h" -#include #include #include @@ -1485,17 +1484,22 @@ TEST_F(MetadataUpdate, MetadataSmallIntEncodedNotNull) { }; int main(int argc, char** argv) { - google::InitGoogleLogging(argv[0]); testing::InitGoogleTest(&argc, argv); namespace po = boost::program_options; po::options_description desc("Options"); desc.add_options()("keep-data", "Don't drop tables at the end of the tests"); + logger::LogOptions log_options(argv[0]); + log_options.max_files_ = 0; // stderr only by default + desc.add(log_options.get_options()); + po::variables_map vm; po::store(po::command_line_parser(argc, argv).options(desc).run(), vm); po::notify(vm); + logger::init(log_options); + if (vm.count("keep-data")) { g_keep_test_data = true; } diff --git a/Tests/UpdelStorageTest.cpp b/Tests/UpdelStorageTest.cpp index 96e1b98128..7467f1ba91 100644 --- a/Tests/UpdelStorageTest.cpp +++ b/Tests/UpdelStorageTest.cpp @@ -17,7 +17,6 @@ #include #include -#include #include #include @@ -35,6 +34,7 @@ #include "Shared/MapDParameters.h" #include "Shared/UpdelRoll.h" #include "Shared/measure.h" +#include "TestHelpers.h" #ifndef BASE_PATH #define BASE_PATH "./tmp" @@ -1081,7 +1081,7 @@ TEST_F(UpdateStorageTest, Half_boolean_deleted_rollback) { } // namespace int main(int argc, char** argv) { - google::InitGoogleLogging(argv[0]); + TestHelpers::init_logger_stderr_only(argc, argv); testing::InitGoogleTest(&argc, argv); g_session.reset(QueryRunner::get_session(BASE_PATH)); diff --git a/Tests/UtilTest.cpp b/Tests/UtilTest.cpp index 366c75c5f1..5e98e70540 100644 --- a/Tests/UtilTest.cpp +++ b/Tests/UtilTest.cpp @@ -16,8 +16,8 @@ #include "../Utils/Regexp.h" #include "../Utils/StringLike.h" +#include "TestHelpers.h" -#include #include TEST(Utils, StringLike) { @@ -46,7 +46,7 @@ TEST(Utils, Regexp) { } int main(int argc, char* argv[]) { - google::InitGoogleLogging(argv[0]); + TestHelpers::init_logger_stderr_only(argc, argv); ::testing::InitGoogleTest(&argc, argv); int err{0}; diff --git a/ThirdParty/muparserx/muparserx/mpFuncCommon.cpp b/ThirdParty/muparserx/muparserx/mpFuncCommon.cpp index 726d98a8ee..5d9a291d76 100644 --- a/ThirdParty/muparserx/muparserx/mpFuncCommon.cpp +++ b/ThirdParty/muparserx/muparserx/mpFuncCommon.cpp @@ -40,7 +40,7 @@ #include "mpValue.h" #include "mpParserBase.h" -#include +#include "Shared/Logger.h" MUP_NAMESPACE_START diff --git a/ThirdParty/muparserx/muparserx/mpFuncStr.cpp b/ThirdParty/muparserx/muparserx/mpFuncStr.cpp index 1b7c301e07..c7493a4f88 100644 --- a/ThirdParty/muparserx/muparserx/mpFuncStr.cpp +++ b/ThirdParty/muparserx/muparserx/mpFuncStr.cpp @@ -38,7 +38,7 @@ #include "mpValue.h" #include "mpError.h" -#include +#include "Shared/Logger.h" MUP_NAMESPACE_START diff --git a/ThirdParty/muparserx/muparserx/mpOprtCmplx.cpp b/ThirdParty/muparserx/muparserx/mpOprtCmplx.cpp index 0efe6ad8dd..1a9d0710ed 100644 --- a/ThirdParty/muparserx/muparserx/mpOprtCmplx.cpp +++ b/ThirdParty/muparserx/muparserx/mpOprtCmplx.cpp @@ -35,7 +35,7 @@ #include "mpOprtCmplx.h" #include #include -#include +#include "Shared/Logger.h" MUP_NAMESPACE_START diff --git a/ThirdParty/muparserx/muparserx/mpOprtMatrix.cpp b/ThirdParty/muparserx/muparserx/mpOprtMatrix.cpp index aaff9e8bc0..48accdb7a4 100644 --- a/ThirdParty/muparserx/muparserx/mpOprtMatrix.cpp +++ b/ThirdParty/muparserx/muparserx/mpOprtMatrix.cpp @@ -29,7 +29,7 @@ POSSIBILITY OF SUCH DAMAGE. */ #include "mpOprtMatrix.h" -#include +#include "Shared/Logger.h" MUP_NAMESPACE_START diff --git a/ThirdParty/muparserx/muparserx/mpOprtNonCmplx.cpp b/ThirdParty/muparserx/muparserx/mpOprtNonCmplx.cpp index b9bbc572a7..1960a9594e 100644 --- a/ThirdParty/muparserx/muparserx/mpOprtNonCmplx.cpp +++ b/ThirdParty/muparserx/muparserx/mpOprtNonCmplx.cpp @@ -33,7 +33,7 @@ POSSIBILITY OF SUCH DAMAGE. */ #include "mpOprtNonCmplx.h" -#include +#include "Shared/Logger.h" MUP_NAMESPACE_START diff --git a/ThriftHandler/CMakeLists.txt b/ThriftHandler/CMakeLists.txt index 4253f3a5e0..a3aef10e4e 100644 --- a/ThriftHandler/CMakeLists.txt +++ b/ThriftHandler/CMakeLists.txt @@ -1,5 +1,5 @@ set(THRIFT_HANDLER_SOURCES MapDHandler.cpp TokenCompletionHints.cpp) -set(THRIFT_HANDLER_LIBS mapd_thrift Shared ${Glog_LIBRARIES} ${CMAKE_DL_LIBS}) +set(THRIFT_HANDLER_LIBS mapd_thrift Shared ${CMAKE_DL_LIBS}) if("${MAPD_EDITION_LOWER}" STREQUAL "ee") list(APPEND THRIFT_HANDLER_LIBS rdkafka++ StringDictionary) diff --git a/ThriftHandler/DistributedValidate.h b/ThriftHandler/DistributedValidate.h index d491c96fd8..1e2eb00852 100644 --- a/ThriftHandler/DistributedValidate.h +++ b/ThriftHandler/DistributedValidate.h @@ -17,8 +17,8 @@ #pragma once #include -#include #include "Catalog/Catalog.h" +#include "Shared/Logger.h" #include "LeafAggregator.h" #include "MapDServer.h" diff --git a/ThriftHandler/MapDHandler.cpp b/ThriftHandler/MapDHandler.cpp index 0b45ef8d64..ba12d07dbf 100644 --- a/ThriftHandler/MapDHandler.cpp +++ b/ThriftHandler/MapDHandler.cpp @@ -75,7 +75,6 @@ #include "Shared/scope.h" #include -#include #include #include #include @@ -288,7 +287,7 @@ void MapDHandler::check_read_only(const std::string& str) { void MapDHandler::internal_connect(TSessionId& session, const std::string& username, const std::string& dbname) { - LOG_ON_RETURN(session); // session not assumed valid here + LOG_SESSION(session); // session not assumed valid here mapd_lock_guard write_lock(sessions_mutex_); std::string username2 = username; // login() may reset username given as argument std::string dbname2 = dbname; // login() may reset dbname given as argument @@ -310,14 +309,15 @@ void MapDHandler::internal_connect(TSessionId& session, THROW_MAPD_EXCEPTION("Unauthorized Access: user " + username + " is not allowed to access database " + dbname2 + "."); } - connect_impl(session, std::string(""), dbname2, user_meta, cat, log_on_return); + connect_impl(session, std::string(""), dbname2, user_meta, cat, log_session); } void MapDHandler::connect(TSessionId& session, const std::string& username, const std::string& passwd, const std::string& dbname) { - LOG_ON_RETURN(session); // session not assumed valid here + LOG_SESSION(session); // session not assumed valid here + mapd_lock_guard write_lock(sessions_mutex_); std::string username2 = username; // login() may reset username given as argument std::string dbname2 = dbname; // login() may reset dbname given as argument @@ -339,7 +339,7 @@ void MapDHandler::connect(TSessionId& session, THROW_MAPD_EXCEPTION("Unauthorized Access: user " + username + " is not allowed to access database " + dbname2 + "."); } - connect_impl(session, passwd, dbname2, user_meta, cat, log_on_return); + connect_impl(session, passwd, dbname2, user_meta, cat, log_session); } void MapDHandler::connect_impl(TSessionId& session, @@ -347,7 +347,7 @@ void MapDHandler::connect_impl(TSessionId& session, const std::string& dbname, Catalog_Namespace::UserMetadata& user_meta, std::shared_ptr cat, - LogOnReturn& log_on_return) { + LogSession& log_session) { do { session = generate_random_string(32); } while (sessions_.find(session) != sessions_.end()); @@ -356,11 +356,12 @@ void MapDHandler::connect_impl(TSessionId& session, std::make_shared( cat, user_meta, executor_device_type_, session)); CHECK(emplace_retval.second); - log_on_return.set_session(emplace_retval.first->second); + SessionMap::mapped_type const session_ptr = emplace_retval.first->second; + log_session.set_session(session_ptr); if (!super_user_rights_) { // no need to connect to leaf_aggregator_ at this time while // doing warmup if (leaf_aggregator_.leafCount() > 0) { - const auto parent_session_info_ptr = emplace_retval.first->second; + const auto parent_session_info_ptr = session_ptr; CHECK(parent_session_info_ptr); leaf_aggregator_.connect( *parent_session_info_ptr, user_meta.userName, passwd, dbname); @@ -368,11 +369,11 @@ void MapDHandler::connect_impl(TSessionId& session, } } LOG(INFO) << "User " << user_meta.userName << " connected to database " << dbname - << std::endl; + << " with public_session_id " << session_ptr->get_public_session_id(); } void MapDHandler::disconnect(const TSessionId& session) { - LOG_ON_RETURN(session); + LOG_SESSION(session); mapd_lock_guard write_lock(sessions_mutex_); auto session_it = get_session_it_unsafe(session); const auto dbname = session_it->second->getCatalog().getCurrentDB().dbName; @@ -410,7 +411,7 @@ void MapDHandler::switch_database(const TSessionId& session, const std::string& } void MapDHandler::interrupt(const TSessionId& session) { - LOG_ON_RETURN(session); + LOG_SESSION(session); if (g_enable_dynamic_watchdog) { // Shared lock to allow simultaneous interrupts of multiple sessions mapd_shared_lock read_lock(sessions_mutex_); @@ -442,7 +443,7 @@ void MapDHandler::interrupt(const TSessionId& session) { } void MapDHandler::get_server_status(TServerStatus& _return, const TSessionId& session) { - LOG_ON_RETURN(session); + LOG_SESSION(session); const auto rendering_enabled = bool(render_handler_); _return.read_only = read_only_; _return.version = MAPD_RELEASE; @@ -455,7 +456,7 @@ void MapDHandler::get_server_status(TServerStatus& _return, const TSessionId& se void MapDHandler::get_status(std::vector& _return, const TSessionId& session) { - LOG_ON_RETURN(session); + LOG_SESSION(session); const auto rendering_enabled = bool(render_handler_); TServerStatus ret; ret.read_only = read_only_; @@ -484,7 +485,7 @@ void MapDHandler::get_status(std::vector& _return, void MapDHandler::get_hardware_info(TClusterHardwareInfo& _return, const TSessionId& session) { - LOG_ON_RETURN(session); + LOG_SESSION(session); THardwareInfo ret; const auto cuda_mgr = data_mgr_->getCudaMgr(); if (cuda_mgr) { @@ -522,7 +523,7 @@ void MapDHandler::get_hardware_info(TClusterHardwareInfo& _return, } void MapDHandler::get_session_info(TSessionInfo& _return, const TSessionId& session) { - LOG_ON_RETURN(session); + LOG_SESSION(session); auto session_info = get_session_copy(session); _return.user = session_info.get_currentUser().userName; _return.database = session_info.getCatalog().getCurrentDB().dbName; @@ -746,7 +747,7 @@ void MapDHandler::sql_execute(TQueryResult& _return, const std::string& nonce, const int32_t first_n, const int32_t at_most_n) { - LOG_ON_RETURN(session, "query_str", hide_sensitive_data_from_query(query_str)); + LOG_SESSION(session, "query_str", hide_sensitive_data_from_query(query_str)); ScopeGuard reset_was_geo_copy_from = [&] { _was_geo_copy_from = false; }; if (first_n >= 0 && at_most_n >= 0) { THROW_MAPD_EXCEPTION(std::string("At most one of first_n and at_most_n can be set")); @@ -815,11 +816,11 @@ void MapDHandler::sql_execute(TQueryResult& _return, create_params); }); } - log_on_return.append_name_value_pairs( + log_session.append_name_value_pairs( "execution_time_ms", _return.execution_time_ms, "total_time_ms", // BE-3420 - Redundant with duration field - log_on_return.duration()); + log_session.duration()); } void MapDHandler::sql_execute_df(TDataFrame& _return, @@ -828,7 +829,7 @@ void MapDHandler::sql_execute_df(TDataFrame& _return, const TDeviceType::type device_type, const int32_t device_id, const int32_t first_n) { - LOG_ON_RETURN(session, "query_str", hide_sensitive_data_from_query(query_str)); + LOG_SESSION(session, "query_str", hide_sensitive_data_from_query(query_str)); const auto session_info = get_session_copy(session); if (device_type == TDeviceType::GPU) { const auto executor_device_type = session_info.get_executor_device_type(); @@ -892,7 +893,7 @@ void MapDHandler::sql_execute_gdf(TDataFrame& _return, const std::string& query_str, const int32_t device_id, const int32_t first_n) { - LOG_ON_RETURN(session); + LOG_SESSION(session); sql_execute_df(_return, session, query_str, TDeviceType::GPU, device_id, first_n); } @@ -901,7 +902,7 @@ void MapDHandler::deallocate_df(const TSessionId& session, const TDataFrame& df, const TDeviceType::type device_type, const int32_t device_id) { - LOG_ON_RETURN(session); + LOG_SESSION(session); const auto session_info = get_session_copy(session); int8_t* dev_ptr{0}; if (device_type == TDeviceType::GPU) { @@ -944,7 +945,7 @@ std::string MapDHandler::apply_copy_to_shim(const std::string& query_str) { void MapDHandler::sql_validate(TTableDescriptor& _return, const TSessionId& session, const std::string& query_str) { - LOG_ON_RETURN(session); + LOG_SESSION(session); std::unique_ptr root_plan; const auto session_info = get_session_copy(session); ParserWrapper pw{query_str}; @@ -993,7 +994,7 @@ void MapDHandler::get_completion_hints(std::vector& hints, const TSessionId& session, const std::string& sql, const int cursor) { - LOG_ON_RETURN(session); + LOG_SESSION(session); std::vector visible_tables; // Tables allowed for the given session. get_completion_hints_unsorted(hints, visible_tables, session, sql, cursor); const auto proj_tokens = extract_projection_tokens_for_completion(sql); @@ -1159,7 +1160,7 @@ void MapDHandler::validate_rel_alg(TTableDescriptor& _return, } void MapDHandler::get_roles(std::vector& roles, const TSessionId& session) { - LOG_ON_RETURN(session); + LOG_SESSION(session); auto session_info = get_session_copy(session); if (!session_info.get_currentUser().isSuper) { roles = @@ -1298,7 +1299,7 @@ bool MapDHandler::has_object_privilege(const TSessionId& sessionId, const std::string& objectName, const TDBObjectType::type objectType, const TDBObjectPermissions& permissions) { - LOG_ON_RETURN(sessionId); + LOG_SESSION(sessionId); auto session = get_session_copy(sessionId); auto& cat = session.getCatalog(); auto current_user = session.get_currentUser(); @@ -1356,7 +1357,7 @@ bool MapDHandler::has_object_privilege(const TSessionId& sessionId, void MapDHandler::get_db_objects_for_grantee(std::vector& TDBObjectsForRole, const TSessionId& sessionId, const std::string& roleName) { - LOG_ON_RETURN(sessionId); + LOG_SESSION(sessionId); auto session = get_session_copy(sessionId); auto user = session.get_currentUser(); if (!user.isSuper && @@ -1384,7 +1385,7 @@ void MapDHandler::get_db_object_privs(std::vector& TDBObjects, const TSessionId& sessionId, const std::string& objectName, const TDBObjectType::type type) { - LOG_ON_RETURN(sessionId); + LOG_SESSION(sessionId); auto session = get_session_copy(sessionId); DBObjectType object_type; switch (type) { @@ -1460,7 +1461,7 @@ void MapDHandler::get_db_object_privs(std::vector& TDBObjects, void MapDHandler::get_all_roles_for_user(std::vector& roles, const TSessionId& sessionId, const std::string& granteeName) { - LOG_ON_RETURN(sessionId); + LOG_SESSION(sessionId); auto session = get_session_copy(sessionId); auto* grantee = SysCatalog::instance().getGrantee(granteeName); if (grantee) { @@ -1499,21 +1500,21 @@ void MapDHandler::get_result_row_for_pixel( const bool column_format, const int32_t pixel_radius, const std::string& nonce) { - LOG_ON_RETURN(session, - "widget_id", - widget_id, - "pixel.x", - pixel.x, - "pixel.y", - pixel.y, - "column_format", - column_format, - "pixel_radius", - pixel_radius, - "table_col_names", - MapDRenderHandler::dump_table_col_names(table_col_names), - "nonce", - nonce); + LOG_SESSION(session, + "widget_id", + widget_id, + "pixel.x", + pixel.x, + "pixel.y", + pixel.y, + "column_format", + column_format, + "pixel_radius", + pixel_radius, + "table_col_names", + MapDRenderHandler::dump_table_col_names(table_col_names), + "nonce", + nonce); if (!render_handler_) { THROW_MAPD_EXCEPTION("Backend rendering is disabled."); } @@ -1592,21 +1593,21 @@ TColumnType MapDHandler::populateThriftColumnType(const Catalog* cat, void MapDHandler::get_table_descriptor(TTableDescriptor& _return, const TSessionId& session, const std::string& table_name) { - LOG_ON_RETURN(session, "table_name", table_name); + LOG_SESSION(session, "table_name", table_name); LOG(ERROR) << "get_table_descriptor is deprecated, please fix application"; } void MapDHandler::get_internal_table_details(TTableDetails& _return, const TSessionId& session, const std::string& table_name) { - LOG_ON_RETURN(session, "table_name", table_name); + LOG_SESSION(session, "table_name", table_name); get_table_details_impl(_return, session, table_name, true, false); } void MapDHandler::get_table_details(TTableDetails& _return, const TSessionId& session, const std::string& table_name) { - LOG_ON_RETURN(session, "table_name", table_name); + LOG_SESSION(session, "table_name", table_name); get_table_details_impl(_return, session, table_name, false, false); } @@ -1690,7 +1691,7 @@ void MapDHandler::get_table_details_impl(TTableDetails& _return, void MapDHandler::get_row_descriptor(TRowDescriptor& _return, const TSessionId& session, const std::string& table_name) { - LOG_ON_RETURN(session, "table_name", table_name); + LOG_SESSION(session, "table_name", table_name); LOG(ERROR) << "get_row_descriptor is deprecated, please fix application"; } @@ -1698,14 +1699,14 @@ void MapDHandler::get_row_descriptor(TRowDescriptor& _return, void MapDHandler::get_frontend_view(TFrontendView& _return, const TSessionId& session, const std::string& view_name) { - LOG_ON_RETURN(session); + LOG_SESSION(session); LOG(ERROR) << "'get_frontend_view' is deprecated, please use 'get_dashboard'."; } void MapDHandler::get_link_view(TFrontendView& _return, const TSessionId& session, const std::string& link) { - LOG_ON_RETURN(session); + LOG_SESSION(session); const auto session_info = get_session_copy(session); auto& cat = session_info.getCatalog(); auto ld = cat.getMetadataForLink(std::to_string(cat.getCurrentDB().dbId) + link); @@ -1772,25 +1773,25 @@ void MapDHandler::get_tables_impl(std::vector& table_names, void MapDHandler::get_tables(std::vector& table_names, const TSessionId& session) { - LOG_ON_RETURN(session); + LOG_SESSION(session); get_tables_impl(table_names, session, GET_PHYSICAL_TABLES_AND_VIEWS); } void MapDHandler::get_physical_tables(std::vector& table_names, const TSessionId& session) { - LOG_ON_RETURN(session); + LOG_SESSION(session); get_tables_impl(table_names, session, GET_PHYSICAL_TABLES); } void MapDHandler::get_views(std::vector& table_names, const TSessionId& session) { - LOG_ON_RETURN(session); + LOG_SESSION(session); get_tables_impl(table_names, session, GET_VIEWS); } void MapDHandler::get_tables_meta(std::vector& _return, const TSessionId& session) { - LOG_ON_RETURN(session); + LOG_SESSION(session); const auto session_info = get_session_copy(session); auto& cat = session_info.getCatalog(); const auto tables = cat.getAllTableMetadata(); @@ -1878,7 +1879,7 @@ void MapDHandler::get_tables_meta(std::vector& _return, void MapDHandler::get_users(std::vector& user_names, const TSessionId& session) { - LOG_ON_RETURN(session); + LOG_SESSION(session); std::list user_list; const auto session_info = get_session_copy(session); @@ -1898,7 +1899,7 @@ void MapDHandler::get_version(std::string& version) { } void MapDHandler::clear_gpu_memory(const TSessionId& session) { - LOG_ON_RETURN(session); + LOG_SESSION(session); const auto session_info = get_session_copy(session); if (!session_info.get_currentUser().isSuper) { THROW_MAPD_EXCEPTION("Superuser privilege is required to run clear_gpu_memory"); @@ -1918,7 +1919,7 @@ void MapDHandler::clear_gpu_memory(const TSessionId& session) { } void MapDHandler::clear_cpu_memory(const TSessionId& session) { - LOG_ON_RETURN(session); + LOG_SESSION(session); const auto session_info = get_session_copy(session); if (!session_info.get_currentUser().isSuper) { THROW_MAPD_EXCEPTION("Superuser privilege is required to run clear_cpu_memory"); @@ -1944,7 +1945,7 @@ TSessionId MapDHandler::getInvalidSessionId() const { void MapDHandler::get_memory(std::vector& _return, const TSessionId& session, const std::string& memory_level) { - LOG_ON_RETURN(session); + LOG_SESSION(session); const auto session_info = get_session_copy(session); std::vector internal_memory; Data_Namespace::MemoryLevel mem_level; @@ -1988,7 +1989,7 @@ void MapDHandler::get_memory(std::vector& _return, void MapDHandler::get_databases(std::vector& dbinfos, const TSessionId& session) { - LOG_ON_RETURN(session); + LOG_SESSION(session); const auto session_info = get_session_copy(session); const auto& user = session_info.get_currentUser(); Catalog_Namespace::DBSummaryList dbs = @@ -2004,13 +2005,13 @@ void MapDHandler::get_databases(std::vector& dbinfos, // DEPRECATED(2019-04-01) - use get_dashboards() void MapDHandler::get_frontend_views(std::vector& view_names, const TSessionId& session) { - LOG_ON_RETURN(session); + LOG_SESSION(session); LOG(ERROR) << "'get_frontend_views' is deprecated, please use 'get_dashboards'."; } void MapDHandler::set_execution_mode(const TSessionId& session, const TExecuteMode::type mode) { - LOG_ON_RETURN(session); + LOG_SESSION(session); mapd_lock_guard write_lock(sessions_mutex_); auto session_it = get_session_it_unsafe(session); if (leaf_aggregator_.leafCount() > 0) { @@ -2039,7 +2040,7 @@ void check_table_not_sharded(const Catalog& cat, const std::string& table_name) void MapDHandler::load_table_binary(const TSessionId& session, const std::string& table_name, const std::vector& rows) { - LOG_ON_RETURN(session, "table_name", table_name); + LOG_SESSION(session, "table_name", table_name); check_read_only("load_table_binary"); const auto session_info = get_session_copy(session); auto& cat = session_info.getCatalog(); @@ -2135,7 +2136,7 @@ void MapDHandler::prepare_columnar_loader( void MapDHandler::load_table_binary_columnar(const TSessionId& session, const std::string& table_name, const std::vector& cols) { - LOG_ON_RETURN(session, "table_name", table_name); + LOG_SESSION(session, "table_name", table_name); check_read_only("load_table_binary_columnar"); std::unique_ptr loader; @@ -2266,7 +2267,7 @@ RecordBatchVector loadArrowStream(const std::string& stream) { void MapDHandler::load_table_binary_arrow(const TSessionId& session, const std::string& table_name, const std::string& arrow_stream) { - LOG_ON_RETURN(session, "table_name", table_name); + LOG_SESSION(session, "table_name", table_name); check_read_only("load_table_binary_arrow"); RecordBatchVector batches = loadArrowStream(arrow_stream); @@ -2312,7 +2313,7 @@ void MapDHandler::load_table_binary_arrow(const TSessionId& session, void MapDHandler::load_table(const TSessionId& session, const std::string& table_name, const std::vector& rows) { - LOG_ON_RETURN(session, "table_name", table_name); + LOG_SESSION(session, "table_name", table_name); check_read_only("load_table"); const auto session_info = get_session_copy(session); auto& cat = session_info.getCatalog(); @@ -2767,7 +2768,7 @@ void MapDHandler::detect_column_types(TDetectResult& _return, const TSessionId& session, const std::string& file_name_in, const TCopyParams& cp) { - LOG_ON_RETURN(session); + LOG_SESSION(session); check_read_only("detect_column_types"); get_session_copy(session); @@ -2953,15 +2954,15 @@ void MapDHandler::render_vega(TRenderResult& _return, const std::string& vega_json, const int compression_level, const std::string& nonce) { - LOG_ON_RETURN(session, - "widget_id", - widget_id, - "compression_level", - compression_level, - "vega_json", - vega_json, - "nonce", - nonce); + LOG_SESSION(session, + "widget_id", + widget_id, + "compression_level", + compression_level, + "vega_json", + vega_json, + "nonce", + nonce); if (!render_handler_) { THROW_MAPD_EXCEPTION("Backend rendering is disabled."); } @@ -2993,7 +2994,7 @@ static bool is_allowed_on_dashboard(const Catalog_Namespace::SessionInfo& sessio void MapDHandler::get_dashboard(TDashboard& dashboard, const TSessionId& session, const int32_t dashboard_id) { - LOG_ON_RETURN(session); + LOG_SESSION(session); const auto session_info = get_session_copy(session); auto& cat = session_info.getCatalog(); Catalog_Namespace::UserMetadata user_meta; @@ -3030,7 +3031,7 @@ void MapDHandler::get_dashboard(TDashboard& dashboard, void MapDHandler::get_dashboards(std::vector& dashboards, const TSessionId& session) { - LOG_ON_RETURN(session); + LOG_SESSION(session); const auto session_info = get_session_copy(session); auto& cat = session_info.getCatalog(); Catalog_Namespace::UserMetadata user_meta; @@ -3070,7 +3071,7 @@ int32_t MapDHandler::create_dashboard(const TSessionId& session, const std::string& dashboard_state, const std::string& image_hash, const std::string& dashboard_metadata) { - LOG_ON_RETURN(session); + LOG_SESSION(session); check_read_only("create_dashboard"); const auto session_info = get_session_copy(session); auto& cat = session_info.getCatalog(); @@ -3112,7 +3113,7 @@ void MapDHandler::replace_dashboard(const TSessionId& session, const std::string& dashboard_state, const std::string& image_hash, const std::string& dashboard_metadata) { - LOG_ON_RETURN(session); + LOG_SESSION(session); check_read_only("replace_dashboard"); const auto session_info = get_session_copy(session); auto& cat = session_info.getCatalog(); @@ -3145,7 +3146,7 @@ void MapDHandler::replace_dashboard(const TSessionId& session, void MapDHandler::delete_dashboard(const TSessionId& session, const int32_t dashboard_id) { - LOG_ON_RETURN(session); + LOG_SESSION(session); check_read_only("delete_dashboard"); const auto session_info = get_session_copy(session); auto& cat = session_info.getCatalog(); @@ -3199,7 +3200,7 @@ void MapDHandler::share_dashboard(const TSessionId& session, const std::vector& objects, const TDashboardPermissions& permissions, const bool grant_role = false) { - LOG_ON_RETURN(session); + LOG_SESSION(session); check_read_only("share_dashboard"); std::vector valid_groups; valid_groups = get_valid_groups(session, dashboard_id, groups); @@ -3246,7 +3247,7 @@ void MapDHandler::unshare_dashboard(const TSessionId& session, const std::vector& groups, const std::vector& objects, const TDashboardPermissions& permissions) { - LOG_ON_RETURN(session); + LOG_SESSION(session); check_read_only("unshare_dashboard"); std::vector valid_groups; valid_groups = get_valid_groups(session, dashboard_id, groups); @@ -3291,7 +3292,7 @@ void MapDHandler::get_dashboard_grantees( std::vector& dashboard_grantees, const TSessionId& session, int32_t dashboard_id) { - LOG_ON_RETURN(session); + LOG_SESSION(session); const auto session_info = get_session_copy(session); auto& cat = session_info.getCatalog(); Catalog_Namespace::UserMetadata user_meta; @@ -3336,7 +3337,7 @@ void MapDHandler::create_frontend_view(const TSessionId& session, const std::string& view_state, const std::string& image_hash, const std::string& view_metadata) { - LOG_ON_RETURN(session); + LOG_SESSION(session); LOG(ERROR) << "'create_frontend_view' is deprecated, please use " "'create_dashboard'."; } @@ -3344,7 +3345,7 @@ void MapDHandler::create_frontend_view(const TSessionId& session, // DEPRECATED(2019-04-01) - use delete_dashboard() void MapDHandler::delete_frontend_view(const TSessionId& session, const std::string& view_name) { - LOG_ON_RETURN(session); + LOG_SESSION(session); LOG(ERROR) << "'delete_frontend_view' is deprecated, please use " "'delete_dashboard'."; } @@ -3353,7 +3354,7 @@ void MapDHandler::create_link(std::string& _return, const TSessionId& session, const std::string& view_state, const std::string& view_metadata) { - LOG_ON_RETURN(session); + LOG_SESSION(session); // check_read_only("create_link"); const auto session_info = get_session_copy(session); auto& cat = session_info.getCatalog(); @@ -3407,7 +3408,7 @@ void MapDHandler::create_table(const TSessionId& session, const TFileType::type file_type, const TCreateParams& create_params) { // sql_execute() below also logs session info. - LOG_ON_RETURN(session, "table_name", table_name); + LOG_SESSION(session, "table_name", table_name); check_read_only("create_table"); if (ImportHelpers::is_reserved_name(table_name)) { @@ -3490,7 +3491,7 @@ void MapDHandler::import_table(const TSessionId& session, const std::string& table_name, const std::string& file_name_in, const TCopyParams& cp) { - LOG_ON_RETURN(session, "table_name", table_name); + LOG_SESSION(session, "table_name", table_name); check_read_only("import_table"); LOG(INFO) << "import_table " << table_name << " from " << file_name_in; const auto session_info = get_session_copy(session); @@ -3598,7 +3599,7 @@ void MapDHandler::import_geo_table(const TSessionId& session, const TCopyParams& cp, const TRowDescriptor& row_desc, const TCreateParams& create_params) { - LOG_ON_RETURN(session, "table_name", table_name); + LOG_SESSION(session, "table_name", table_name); check_read_only("import_table"); const auto session_info = get_session_copy(session); auto& cat = session_info.getCatalog(); @@ -4041,7 +4042,7 @@ void MapDHandler::import_geo_table(const TSessionId& session, void MapDHandler::import_table_status(TImportStatus& _return, const TSessionId& session, const std::string& import_id) { - LOG_ON_RETURN(session, "import_table_status", import_id); + LOG_SESSION(session, "import_table_status", import_id); auto is = Importer_NS::Importer::get_import_status(import_id); _return.elapsed = is.elapsed.count(); _return.rows_completed = is.rows_completed; @@ -4053,7 +4054,7 @@ void MapDHandler::get_first_geo_file_in_archive(std::string& _return, const TSessionId& session, const std::string& archive_path_in, const TCopyParams& copy_params) { - LOG_ON_RETURN(session, "get_first_geo_file_in_archive", archive_path_in); + LOG_SESSION(session, "get_first_geo_file_in_archive", archive_path_in); std::string archive_path(archive_path_in); if (path_is_relative(archive_path)) { @@ -4092,7 +4093,7 @@ void MapDHandler::get_all_files_in_archive(std::vector& _return, const TSessionId& session, const std::string& archive_path_in, const TCopyParams& copy_params) { - LOG_ON_RETURN(session, "get_all_files_in_archive", archive_path_in); + LOG_SESSION(session, "get_all_files_in_archive", archive_path_in); std::string archive_path(archive_path_in); if (path_is_relative(archive_path)) { @@ -4124,7 +4125,7 @@ void MapDHandler::get_layers_in_geo_file(std::vector& _return const TSessionId& session, const std::string& file_name_in, const TCopyParams& cp) { - LOG_ON_RETURN(session, "get_layers_in_geo_file", file_name_in); + LOG_SESSION(session, "get_layers_in_geo_file", file_name_in); std::string file_name(file_name_in); Importer_NS::CopyParams copy_params = thrift_to_copyparams(cp); @@ -4194,7 +4195,7 @@ void MapDHandler::get_layers_in_geo_file(std::vector& _return } void MapDHandler::start_heap_profile(const TSessionId& session) { - LOG_ON_RETURN(session); + LOG_SESSION(session); const auto session_info = get_session_copy(session); #ifdef HAVE_PROFILER if (IsHeapProfilerRunning()) { @@ -4207,7 +4208,7 @@ void MapDHandler::start_heap_profile(const TSessionId& session) { } void MapDHandler::stop_heap_profile(const TSessionId& session) { - LOG_ON_RETURN(session); + LOG_SESSION(session); const auto session_info = get_session_copy(session); #ifdef HAVE_PROFILER if (!IsHeapProfilerRunning()) { @@ -4220,7 +4221,7 @@ void MapDHandler::stop_heap_profile(const TSessionId& session) { } void MapDHandler::get_heap_profile(std::string& profile, const TSessionId& session) { - LOG_ON_RETURN(session); + LOG_SESSION(session); const auto session_info = get_session_copy(session); #ifdef HAVE_PROFILER if (!IsHeapProfilerRunning()) { @@ -4844,7 +4845,7 @@ void MapDHandler::sql_execute_impl(TQueryResult& _return, if (num_parse_errors > 0) { throw std::runtime_error("Syntax error at: " + last_parsed); } - CHECK_EQ(parse_trees.size(), 1); + CHECK_EQ(parse_trees.size(), 1u); if (pw.is_optimize) { const auto optimize_stmt = @@ -5238,7 +5239,7 @@ std::string MapDHandler::parse_to_ra( void MapDHandler::check_table_consistency(TTableMeta& _return, const TSessionId& session, const int32_t table_id) { - LOG_ON_RETURN(session); + LOG_SESSION(session); if (!leaf_handler_) { THROW_MAPD_EXCEPTION("Distributed support is disabled."); } @@ -5253,7 +5254,7 @@ void MapDHandler::start_query(TPendingQuery& _return, const TSessionId& session, const std::string& query_ra, const bool just_explain) { - LOG_ON_RETURN(session); + LOG_SESSION(session); if (!leaf_handler_) { THROW_MAPD_EXCEPTION("Distributed support is disabled."); } @@ -5305,7 +5306,7 @@ void MapDHandler::broadcast_serialized_rows(const TSerializedRows& serialized_ro void MapDHandler::insert_data(const TSessionId& session, const TInsertData& thrift_insert_data) { - LOG_ON_RETURN(session); + LOG_SESSION(session); CHECK_EQ(thrift_insert_data.column_ids.size(), thrift_insert_data.data.size()); const auto session_info = get_session_copy(session); auto& cat = session_info.getCatalog(); @@ -5392,7 +5393,7 @@ void MapDHandler::start_render_query(TPendingRenderQuery& _return, const int64_t widget_id, const int16_t node_idx, const std::string& vega_json) { - LOG_ON_RETURN(session); + LOG_SESSION(session); if (!render_handler_) { THROW_MAPD_EXCEPTION("Backend rendering is disabled."); } @@ -5433,7 +5434,7 @@ void MapDHandler::execute_next_render_step(TRenderStepResult& _return, void MapDHandler::checkpoint(const TSessionId& session, const int32_t db_id, const int32_t table_id) { - LOG_ON_RETURN(session); + LOG_SESSION(session); const auto session_info = get_session_copy(session); auto& cat = session_info.getCatalog(); cat.getDataMgr().checkpoint(db_id, table_id); @@ -5444,7 +5445,7 @@ void MapDHandler::set_table_epoch(const TSessionId& session, const int db_id, const int table_id, const int new_epoch) { - LOG_ON_RETURN(session); + LOG_SESSION(session); const auto session_info = get_session_copy(session); if (!session_info.get_currentUser().isSuper) { throw std::runtime_error("Only superuser can set_table_epoch"); @@ -5461,7 +5462,7 @@ void MapDHandler::set_table_epoch(const TSessionId& session, void MapDHandler::set_table_epoch_by_name(const TSessionId& session, const std::string& table_name, const int new_epoch) { - LOG_ON_RETURN(session, "table_name", table_name); + LOG_SESSION(session, "table_name", table_name); const auto session_info = get_session_copy(session); if (!session_info.get_currentUser().isSuper) { throw std::runtime_error("Only superuser can set_table_epoch"); @@ -5481,7 +5482,7 @@ void MapDHandler::set_table_epoch_by_name(const TSessionId& session, int32_t MapDHandler::get_table_epoch(const TSessionId& session, const int32_t db_id, const int32_t table_id) { - LOG_ON_RETURN(session); + LOG_SESSION(session); const auto session_info = get_session_copy(session); auto& cat = session_info.getCatalog(); @@ -5493,7 +5494,7 @@ int32_t MapDHandler::get_table_epoch(const TSessionId& session, int32_t MapDHandler::get_table_epoch_by_name(const TSessionId& session, const std::string& table_name) { - LOG_ON_RETURN(session, "table_name", table_name); + LOG_SESSION(session, "table_name", table_name); const auto session_info = get_session_copy(session); auto& cat = session_info.getCatalog(); auto td = cat.getMetadataForTable( @@ -5510,7 +5511,7 @@ void MapDHandler::set_license_key(TLicenseInfo& _return, const TSessionId& session, const std::string& key, const std::string& nonce) { - LOG_ON_RETURN(session); + LOG_SESSION(session); check_read_only("set_license_key"); const auto session_info = get_session_copy(session); THROW_MAPD_EXCEPTION(std::string("Licensing not supported.")); @@ -5519,7 +5520,7 @@ void MapDHandler::set_license_key(TLicenseInfo& _return, void MapDHandler::get_license_claims(TLicenseInfo& _return, const TSessionId& session, const std::string& nonce) { - LOG_ON_RETURN(session); + LOG_SESSION(session); const auto session_info = get_session_copy(session); _return.claims.emplace_back(""); } @@ -5534,46 +5535,52 @@ void MapDHandler::shutdown() { } } -void LogOnReturn::set_session(SessionMap::mapped_type& session_ptr) { - session_ptr_ = session_ptr; -} +std::atomic LogSession::s_match{0}; -LogOnReturn::~LogOnReturn() { - std::stringstream ss; - ss << "stdlog " << file_.filename().c_str() << ' ' << line_ << ' ' << func_ << ' ' - << duration() << ' '; - if (session_ptr_) { - std::string const& dbname = session_ptr_->getCatalog().getCurrentDB().dbName; - if (dbname.find_first_of(" \"") == std::string::npos) { - ss << dbname << ' '; +void LogSession::stdlog(logger::Severity severity, char const* label) { + if (logger::g_min_active_severity <= severity) { + std::stringstream ss; + ss << file_ << ':' << line_ << ' ' << label << ' ' << func_ << ' ' << match_ << ' ' + << duration() << ' '; + if (session_ptr_) { + std::string const& dbname = session_ptr_->getCatalog().getCurrentDB().dbName; + if (dbname.find_first_of(" \"") == std::string::npos) { + ss << dbname << ' '; + } else { + ss << std::quoted(dbname, '"', '"') << ' '; + } + std::string const& username = session_ptr_->get_currentUser().userName; + if (username.find_first_of(" \"") == std::string::npos) { + ss << username << ' '; + } else { + ss << std::quoted(username, '"', '"') << ' '; + } + ss << session_ptr_->get_public_session_id() << ' '; } else { - ss << std::quoted(dbname, '"', '"') << ' '; + ss << " "; // 3 spaces for 3 empty strings } - std::string const& username = session_ptr_->get_currentUser().userName; - if (username.find_first_of(" \"") == std::string::npos) { - ss << username << ' '; + auto const& nv = name_value_pairs_; + if (nv.empty()) { + ss << ' '; // 1 space for final empty names/values arrays } else { - ss << std::quoted(username, '"', '"') << ' '; - } - ss << session_ptr_->get_public_session_id() << ' '; - } else { - ss << " "; // 3 spaces for 3 empty strings - } - if (name_value_pairs_.empty()) { - ss << ' '; // 1 space for final empty names/values arrays - } else { - ss << '{'; // names - for (auto itr = name_value_pairs_.cbegin(); itr != name_value_pairs_.cend(); - std::advance(itr, 2)) { - ss << (itr == name_value_pairs_.cbegin() ? "" : ",") << std::quoted(*itr, '"', '"'); - } - ss << "} {"; // values - for (auto itr = name_value_pairs_.cbegin(); itr != name_value_pairs_.cend(); - std::advance(itr, 2)) { - ss << (itr == name_value_pairs_.cbegin() ? "" : ",") - << std::quoted(*std::next(itr), '"', '"'); + ss << '{' << std::quoted(*nv.cbegin(), '"', '"'); // names + for (auto itr = std::next(nv.cbegin(), 2); itr != nv.cend(); std::advance(itr, 2)) { + ss << ',' << std::quoted(*itr, '"', '"'); + } + ss << "} {" << std::quoted(*std::next(nv.cbegin()), '"', '"'); // values + for (auto itr = std::next(nv.cbegin(), 2); itr != nv.cend(); std::advance(itr, 2)) { + ss << ',' << std::quoted(*std::next(itr), '"', '"'); + } + ss << '}'; } - ss << '}'; + BOOST_LOG_SEV(logger::g_logger::get(), severity) << ss.rdbuf(); } - LOG(INFO) << ss.rdbuf(); +} + +LogSession::~LogSession() { + stdlog(logger::Severity::INFO, "stdlog"); +} + +void LogSession::set_session(SessionMap::mapped_type const& session_ptr) { + session_ptr_ = session_ptr; } diff --git a/ThriftHandler/MapDHandler.h b/ThriftHandler/MapDHandler.h index 327084ed08..6b474a13d5 100644 --- a/ThriftHandler/MapDHandler.h +++ b/ThriftHandler/MapDHandler.h @@ -57,6 +57,7 @@ #include "QueryEngine/TableGenerations.h" #include "Shared/ConfigResolve.h" #include "Shared/GenericTypeUtilities.h" +#include "Shared/Logger.h" #include "Shared/MapDParameters.h" #include "Shared/StringTransform.h" #include "Shared/geosupport.h" @@ -68,11 +69,11 @@ #include "ThriftHandler/DistributedValidate.h" #include -#include #include #include #include #include +#include #include #include #include @@ -101,7 +102,7 @@ using namespace std::string_literals; -class LogOnReturn; +class LogSession; class MapDRenderHandler; class MapDAggHandler; class MapDLeafHandler; @@ -464,7 +465,7 @@ class MapDHandler : public MapDIf { const std::string& dbname, Catalog_Namespace::UserMetadata& user_meta, std::shared_ptr cat, - LogOnReturn&); + LogSession&); void disconnect_impl(const SessionMap::iterator& session_it); void check_table_load_privileges(const TSessionId& session, const std::string& table_name); @@ -710,62 +711,70 @@ class MapDHandler : public MapDIf { }; // Log Format: -// stdlog [file] [line] [func] [milliseconds] [database] [user] [public_session_id] -// {[names]} {[values]} -// Call at beginning of Thrift call. Wait until destructor to LOG(INFO), with timing. +// YYYY-MM-DDTHH:MM::SS.FFFFFF [pid] [file]:[line] [label] [func] [match] [dur_ms] +// [dbname] [user] [pubsessid] {[names]} {[values]} +// Call at both beginning(label="stdlog_begin") and end(label="stdlog") of Thrift call, +// with dur_ms = current age of LogSession object in milliseconds. +// stdlog_begin is logged at DEBUG1 level, stdlog is logged at INFO level. // The only required parameter is session, which can be either: // * std::shared_ptr - No locking is done. // * TSessionId string - will call get_session_copy_ptr() to get shared_ptr. // All remaining optional parameters are name,value pairs that will be included in log. -#define LOG_ON_RETURN(session, ...) \ - LogOnReturn log_on_return(*this, session, __FILE__, __LINE__, __func__, ##__VA_ARGS__) +#define LOG_SESSION(session, ...) \ + LogSession log_session(*this, session, __FILE__, __LINE__, __func__, ##__VA_ARGS__) -class LogOnReturn : boost::noncopyable { - boost::filesystem::path const file_; +class LogSession : boost::noncopyable { + std::string const file_; size_t const line_; char const* const func_; std::list name_value_pairs_; std::chrono::steady_clock::time_point const start_; SessionMap::mapped_type session_ptr_; + static std::atomic s_match; + int64_t const match_; // Unique to each begin/end pair to match them together. template - LogOnReturn(char const* file, size_t line, char const* func, Pairs&&... pairs) - : file_(file) + LogSession(char const* file, size_t line, char const* func, Pairs&&... pairs) + : file_(boost::filesystem::path(file).filename().string()) , line_(line) , func_(func) , name_value_pairs_{to_string(std::forward(pairs))...} - , start_(std::chrono::steady_clock::now()) { + , start_(std::chrono::steady_clock::now()) + , match_(s_match++) { static_assert(sizeof...(Pairs) % 2 == 0, - "LogOnReturn() requires an even number of name/value parameters."); + "LogSession() requires an even number of name/value parameters."); } + void stdlog(logger::Severity, char const* label); public: template - LogOnReturn(MapDHandler&, - SessionMap::mapped_type& session_ptr, - char const* file, - size_t line, - char const* func, - Pairs&&... pairs) - : LogOnReturn(file, line, func, std::forward(pairs)...) { + LogSession(MapDHandler& mh, + SessionMap::mapped_type const& session_ptr, + char const* file, + size_t line, + char const* func, + Pairs&&... pairs) + : LogSession(file, line, func, std::forward(pairs)...) { session_ptr_ = session_ptr; + stdlog(logger::Severity::DEBUG1, "stdlog_begin"); } template - LogOnReturn(MapDHandler& mapd_handler, - TSessionId const& session_id, - char const* file, - size_t line, - char const* func, - Pairs&&... pairs) - : LogOnReturn(file, line, func, std::forward(pairs)...) { + LogSession(MapDHandler& mh, + TSessionId const& session_id, + char const* file, + size_t line, + char const* func, + Pairs&&... pairs) + : LogSession(file, line, func, std::forward(pairs)...) { if (!session_id.empty()) { try { - session_ptr_ = mapd_handler.get_session_copy_ptr(session_id); + session_ptr_ = mh.get_session_copy_ptr(session_id); } catch (...) { session_ptr_.reset(); } } + stdlog(logger::Severity::DEBUG1, "stdlog_begin"); } - ~LogOnReturn(); + ~LogSession(); template void append_name_value_pairs(Pairs&&... pairs) { static_assert(sizeof...(Pairs) % 2 == 0, @@ -774,11 +783,11 @@ class LogOnReturn : boost::noncopyable { {to_string(std::forward(pairs))...}); } template - size_t duration() const { + typename Units::rep duration() const { using namespace std::chrono; return duration_cast(steady_clock::now() - start_).count(); } - void set_session(SessionMap::mapped_type&); + void set_session(SessionMap::mapped_type const& session_ptr); }; #endif /* MAPDHANDLER_H */ diff --git a/initdb.cpp b/initdb.cpp index a72dae77ff..781127a326 100644 --- a/initdb.cpp +++ b/initdb.cpp @@ -23,6 +23,7 @@ #include #include "Catalog/Catalog.h" #include "Import/Importer.h" +#include "Shared/Logger.h" #include "Shared/mapdpath.h" #include "boost/program_options.hpp" @@ -43,8 +44,6 @@ int main(int argc, char* argv[]) { bool skip_geo = false; namespace po = boost::program_options; - google::InitGoogleLogging(argv[0]); - po::options_description desc("Options"); desc.add_options()("help,h", "Print help messages ")( "data", @@ -59,6 +58,9 @@ int main(int argc, char* argv[]) { ->implicit_value(true), "Enable writing messages directly from thrift to stdout/stderr."); + logger::LogOptions log_options(argv[0]); + desc.add(log_options.get_options()); + po::positional_options_description positionalOptions; positionalOptions.add("data", 1); @@ -133,6 +135,9 @@ int main(int argc, char* argv[]) { << std::endl; } + log_options.set_base_path(base_path); + logger::init(log_options); + try { MapDParameters mapd_parms; auto dummy =