diff --git a/CMakeLists.txt b/CMakeLists.txt index 8537bccb18..a8cb08f694 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -490,7 +490,6 @@ add_subdirectory(CudaMgr) add_subdirectory(LockMgr) add_subdirectory(MigrationMgr) add_subdirectory(Fragmenter) -add_subdirectory(Chunk) add_subdirectory(Shared) add_subdirectory(Utils) add_subdirectory(QueryRunner) @@ -509,7 +508,7 @@ if(ENABLE_ODBC) add_subdirectory(ODBC) endif() -set(MAPD_LIBRARIES Shared Catalog SqliteConnector MigrationMgr TableArchiver Parser Analyzer CsvImport QueryRunner QueryEngine QueryState LockMgr DataMgr Fragmenter Chunk) +set(MAPD_LIBRARIES Shared Catalog SqliteConnector MigrationMgr TableArchiver Parser Analyzer CsvImport QueryRunner QueryEngine QueryState LockMgr DataMgr Fragmenter) if("${MAPD_EDITION_LOWER}" STREQUAL "ee") list(APPEND MAPD_LIBRARIES Distributed) diff --git a/Catalog/CMakeLists.txt b/Catalog/CMakeLists.txt index 4fe8f47865..7311ab5ac8 100644 --- a/Catalog/CMakeLists.txt +++ b/Catalog/CMakeLists.txt @@ -22,7 +22,7 @@ if("${MAPD_EDITION_LOWER}" STREQUAL "ee") add_subdirectory(ee) endif() -target_link_libraries(Catalog SqliteConnector StringDictionary Fragmenter MigrationMgr Chunk ${AUTH_LIBRARIES} Calcite bcrypt) +target_link_libraries(Catalog SqliteConnector StringDictionary Fragmenter MigrationMgr ${AUTH_LIBRARIES} Calcite bcrypt) if(ENABLE_KRB5) target_link_libraries(Catalog krb5_gss) endif() diff --git a/Chunk/CMakeLists.txt b/Chunk/CMakeLists.txt deleted file mode 100644 index 8f7fe31eff..0000000000 --- a/Chunk/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-result") - -add_library(Chunk Chunk.cpp) -target_link_libraries(Chunk DataMgr) diff --git a/DataMgr/ArrayNoneEncoder.h b/DataMgr/ArrayNoneEncoder.h index 5ec41ac1c7..5b741a4d16 100644 --- a/DataMgr/ArrayNoneEncoder.h +++ b/DataMgr/ArrayNoneEncoder.h @@ -244,13 +244,13 @@ class ArrayNoneEncoder : public Encoder { initialized = array_encoder->initialized; } - AbstractBuffer* get_index_buf() const { return index_buf; } + AbstractBuffer* getIndexBuf() const { return index_buf; } Datum elem_min; Datum elem_max; bool has_nulls; bool initialized; - void set_index_buf(AbstractBuffer* buf) { + void setIndexBuffer(AbstractBuffer* buf) { std::unique_lock lock(EncoderMutex_); index_buf = buf; } diff --git a/DataMgr/CMakeLists.txt b/DataMgr/CMakeLists.txt index 63be92e828..4de019bb27 100644 --- a/DataMgr/CMakeLists.txt +++ b/DataMgr/CMakeLists.txt @@ -1,6 +1,7 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-result") set(datamgr_source_files + Chunk/Chunk.cpp DataMgr.cpp Encoder.cpp StringNoneEncoder.cpp diff --git a/Chunk/Chunk.cpp b/DataMgr/Chunk/Chunk.cpp similarity index 62% rename from Chunk/Chunk.cpp rename to DataMgr/Chunk/Chunk.cpp index a1e1f8efe9..a248c3bc03 100644 --- a/Chunk/Chunk.cpp +++ b/DataMgr/Chunk/Chunk.cpp @@ -1,5 +1,5 @@ /* - * Copyright 2017 MapD Technologies, Inc. + * Copyright 2020 OmniSci, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,10 +19,10 @@ * @author Wei Hong */ -#include "Chunk.h" -#include "../DataMgr/ArrayNoneEncoder.h" -#include "../DataMgr/FixedLengthArrayNoneEncoder.h" -#include "../DataMgr/StringNoneEncoder.h" +#include "DataMgr/Chunk/Chunk.h" +#include "DataMgr/ArrayNoneEncoder.h" +#include "DataMgr/FixedLengthArrayNoneEncoder.h" +#include "DataMgr/StringNoneEncoder.h" namespace Chunk_NS { std::shared_ptr Chunk::getChunk(const ColumnDescriptor* cd, @@ -41,7 +41,8 @@ bool Chunk::isChunkOnDevice(DataMgr* data_mgr, const ChunkKey& key, const MemoryLevel mem_level, const int device_id) { - if (column_desc->columnType.is_varlen() && !column_desc->columnType.is_fixlen_array()) { + if (column_desc_->columnType.is_varlen() && + !column_desc_->columnType.is_fixlen_array()) { ChunkKey subKey = key; ChunkKey indexKey(subKey); indexKey.push_back(1); @@ -60,48 +61,49 @@ void Chunk::getChunkBuffer(DataMgr* data_mgr, const int device_id, const size_t num_bytes, const size_t num_elems) { - if (column_desc->columnType.is_varlen() && !column_desc->columnType.is_fixlen_array()) { + if (column_desc_->columnType.is_varlen() && + !column_desc_->columnType.is_fixlen_array()) { ChunkKey subKey = key; - subKey.push_back(1); // 1 for the main buffer - buffer = data_mgr->getChunkBuffer(subKey, mem_level, device_id, num_bytes); + subKey.push_back(1); // 1 for the main buffer_ + buffer_ = data_mgr->getChunkBuffer(subKey, mem_level, device_id, num_bytes); subKey.pop_back(); - subKey.push_back(2); // 2 for the index buffer - index_buf = data_mgr->getChunkBuffer( + subKey.push_back(2); // 2 for the index buffer_ + index_buf_ = data_mgr->getChunkBuffer( subKey, mem_level, device_id, (num_elems + 1) * sizeof(StringOffsetT)); // always record n+1 offsets so string // length can be calculated - switch (column_desc->columnType.get_type()) { + switch (column_desc_->columnType.get_type()) { case kARRAY: { - ArrayNoneEncoder* array_encoder = - dynamic_cast(buffer->encoder.get()); - array_encoder->set_index_buf(index_buf); + auto array_encoder = dynamic_cast(buffer_->encoder.get()); + CHECK(array_encoder); + array_encoder->setIndexBuffer(index_buf_); break; } case kTEXT: case kVARCHAR: case kCHAR: { - CHECK_EQ(kENCODING_NONE, column_desc->columnType.get_compression()); - StringNoneEncoder* str_encoder = - dynamic_cast(buffer->encoder.get()); - str_encoder->set_index_buf(index_buf); + CHECK_EQ(kENCODING_NONE, column_desc_->columnType.get_compression()); + auto str_encoder = dynamic_cast(buffer_->encoder.get()); + CHECK(str_encoder); + str_encoder->setIndexBuffer(index_buf_); break; } case kPOINT: case kLINESTRING: case kPOLYGON: case kMULTIPOLYGON: { - StringNoneEncoder* str_encoder = - dynamic_cast(buffer->encoder.get()); - str_encoder->set_index_buf(index_buf); + auto str_encoder = dynamic_cast(buffer_->encoder.get()); + CHECK(str_encoder); + str_encoder->setIndexBuffer(index_buf_); break; } default: - CHECK(false); + UNREACHABLE(); } } else { - buffer = data_mgr->getChunkBuffer(key, mem_level, device_id, num_bytes); + buffer_ = data_mgr->getChunkBuffer(key, mem_level, device_id, num_bytes); } } @@ -110,15 +112,16 @@ void Chunk::createChunkBuffer(DataMgr* data_mgr, const MemoryLevel mem_level, const int device_id, const size_t page_size) { - if (column_desc->columnType.is_varlen() && !column_desc->columnType.is_fixlen_array()) { + if (column_desc_->columnType.is_varlen() && + !column_desc_->columnType.is_fixlen_array()) { ChunkKey subKey = key; - subKey.push_back(1); // 1 for the main buffer - buffer = data_mgr->createChunkBuffer(subKey, mem_level, device_id, page_size); + subKey.push_back(1); // 1 for the main buffer_ + buffer_ = data_mgr->createChunkBuffer(subKey, mem_level, device_id, page_size); subKey.pop_back(); - subKey.push_back(2); // 2 for the index buffer - index_buf = data_mgr->createChunkBuffer(subKey, mem_level, device_id, page_size); + subKey.push_back(2); // 2 for the index buffer_ + index_buf_ = data_mgr->createChunkBuffer(subKey, mem_level, device_id, page_size); } else { - buffer = data_mgr->createChunkBuffer(key, mem_level, device_id, page_size); + buffer_ = data_mgr->createChunkBuffer(key, mem_level, device_id, page_size); } } @@ -127,26 +130,26 @@ size_t Chunk::getNumElemsForBytesInsertData(const DataBlockPtr& src_data, const size_t start_idx, const size_t byte_limit, const bool replicating) { - CHECK(column_desc->columnType.is_varlen()); - switch (column_desc->columnType.get_type()) { + CHECK(column_desc_->columnType.is_varlen()); + switch (column_desc_->columnType.get_type()) { case kARRAY: { - if (column_desc->columnType.get_size() > 0) { + if (column_desc_->columnType.get_size() > 0) { FixedLengthArrayNoneEncoder* array_encoder = - dynamic_cast(buffer->encoder.get()); + dynamic_cast(buffer_->encoder.get()); return array_encoder->getNumElemsForBytesInsertData( src_data.arraysPtr, start_idx, num_elems, byte_limit, replicating); } ArrayNoneEncoder* array_encoder = - dynamic_cast(buffer->encoder.get()); + dynamic_cast(buffer_->encoder.get()); return array_encoder->getNumElemsForBytesInsertData( src_data.arraysPtr, start_idx, num_elems, byte_limit, replicating); } case kTEXT: case kVARCHAR: case kCHAR: { - CHECK_EQ(kENCODING_NONE, column_desc->columnType.get_compression()); + CHECK_EQ(kENCODING_NONE, column_desc_->columnType.get_compression()); StringNoneEncoder* str_encoder = - dynamic_cast(buffer->encoder.get()); + dynamic_cast(buffer_->encoder.get()); return str_encoder->getNumElemsForBytesInsertData( src_data.stringsPtr, start_idx, num_elems, byte_limit, replicating); } @@ -155,7 +158,7 @@ size_t Chunk::getNumElemsForBytesInsertData(const DataBlockPtr& src_data, case kPOLYGON: case kMULTIPOLYGON: { StringNoneEncoder* str_encoder = - dynamic_cast(buffer->encoder.get()); + dynamic_cast(buffer_->encoder.get()); return str_encoder->getNumElemsForBytesInsertData( src_data.stringsPtr, start_idx, num_elems, byte_limit, replicating); } @@ -169,18 +172,18 @@ std::shared_ptr Chunk::appendData(DataBlockPtr& src_data, const size_t num_elems, const size_t start_idx, const bool replicating) { - const auto& ti = column_desc->columnType; + const auto& ti = column_desc_->columnType; if (ti.is_varlen()) { switch (ti.get_type()) { case kARRAY: { if (ti.get_size() > 0) { FixedLengthArrayNoneEncoder* array_encoder = - dynamic_cast(buffer->encoder.get()); + dynamic_cast(buffer_->encoder.get()); return array_encoder->appendData( src_data.arraysPtr, start_idx, num_elems, replicating); } ArrayNoneEncoder* array_encoder = - dynamic_cast(buffer->encoder.get()); + dynamic_cast(buffer_->encoder.get()); return array_encoder->appendData( src_data.arraysPtr, start_idx, num_elems, replicating); } @@ -189,7 +192,7 @@ std::shared_ptr Chunk::appendData(DataBlockPtr& src_data, case kCHAR: { CHECK_EQ(kENCODING_NONE, ti.get_compression()); StringNoneEncoder* str_encoder = - dynamic_cast(buffer->encoder.get()); + dynamic_cast(buffer_->encoder.get()); return str_encoder->appendData( src_data.stringsPtr, start_idx, num_elems, replicating); } @@ -198,7 +201,7 @@ std::shared_ptr Chunk::appendData(DataBlockPtr& src_data, case kPOLYGON: case kMULTIPOLYGON: { StringNoneEncoder* str_encoder = - dynamic_cast(buffer->encoder.get()); + dynamic_cast(buffer_->encoder.get()); return str_encoder->appendData( src_data.stringsPtr, start_idx, num_elems, replicating); } @@ -206,35 +209,36 @@ std::shared_ptr Chunk::appendData(DataBlockPtr& src_data, CHECK(false); } } - return buffer->encoder->appendData(src_data.numbersPtr, num_elems, ti, replicating); + return buffer_->encoder->appendData(src_data.numbersPtr, num_elems, ti, replicating); } -void Chunk::unpin_buffer() { - if (buffer != nullptr) { - buffer->unPin(); +void Chunk::unpinBuffer() { + if (buffer_) { + buffer_->unPin(); } - if (index_buf != nullptr) { - index_buf->unPin(); + if (index_buf_) { + index_buf_->unPin(); } } -void Chunk::init_encoder() { - buffer->initEncoder(column_desc->columnType); - if (column_desc->columnType.is_varlen() && !column_desc->columnType.is_fixlen_array()) { - switch (column_desc->columnType.get_type()) { +void Chunk::initEncoder() { + buffer_->initEncoder(column_desc_->columnType); + if (column_desc_->columnType.is_varlen() && + !column_desc_->columnType.is_fixlen_array()) { + switch (column_desc_->columnType.get_type()) { case kARRAY: { ArrayNoneEncoder* array_encoder = - dynamic_cast(buffer->encoder.get()); - array_encoder->set_index_buf(index_buf); + dynamic_cast(buffer_->encoder.get()); + array_encoder->setIndexBuffer(index_buf_); break; } case kTEXT: case kVARCHAR: case kCHAR: { - CHECK_EQ(kENCODING_NONE, column_desc->columnType.get_compression()); + CHECK_EQ(kENCODING_NONE, column_desc_->columnType.get_compression()); StringNoneEncoder* str_encoder = - dynamic_cast(buffer->encoder.get()); - str_encoder->set_index_buf(index_buf); + dynamic_cast(buffer_->encoder.get()); + str_encoder->setIndexBuffer(index_buf_); break; } case kPOINT: @@ -242,8 +246,8 @@ void Chunk::init_encoder() { case kPOLYGON: case kMULTIPOLYGON: { StringNoneEncoder* str_encoder = - dynamic_cast(buffer->encoder.get()); - str_encoder->set_index_buf(index_buf); + dynamic_cast(buffer_->encoder.get()); + str_encoder->setIndexBuffer(index_buf_); break; } default: @@ -256,17 +260,17 @@ ChunkIter Chunk::begin_iterator(const std::shared_ptr& chunk_meta int start_idx, int skip) const { ChunkIter it; - it.type_info = column_desc->columnType; + it.type_info = column_desc_->columnType; it.skip = skip; - it.skip_size = column_desc->columnType.get_size(); + it.skip_size = column_desc_->columnType.get_size(); if (it.skip_size < 0) { // if it's variable length it.current_pos = it.start_pos = - index_buf->getMemoryPtr() + start_idx * sizeof(StringOffsetT); - it.end_pos = index_buf->getMemoryPtr() + index_buf->size() - sizeof(StringOffsetT); - it.second_buf = buffer->getMemoryPtr(); + index_buf_->getMemoryPtr() + start_idx * sizeof(StringOffsetT); + it.end_pos = index_buf_->getMemoryPtr() + index_buf_->size() - sizeof(StringOffsetT); + it.second_buf = buffer_->getMemoryPtr(); } else { - it.current_pos = it.start_pos = buffer->getMemoryPtr() + start_idx * it.skip_size; - it.end_pos = buffer->getMemoryPtr() + buffer->size(); + it.current_pos = it.start_pos = buffer_->getMemoryPtr() + start_idx * it.skip_size; + it.end_pos = buffer_->getMemoryPtr() + buffer_->size(); it.second_buf = nullptr; } it.num_elems = chunk_metadata->numElements; diff --git a/Chunk/Chunk.h b/DataMgr/Chunk/Chunk.h similarity index 75% rename from Chunk/Chunk.h rename to DataMgr/Chunk/Chunk.h index 4f8cd0a35f..142ce987d8 100644 --- a/Chunk/Chunk.h +++ b/DataMgr/Chunk/Chunk.h @@ -1,5 +1,5 @@ /* - * Copyright 2017 MapD Technologies, Inc. + * Copyright 2020 OmniSci, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,37 +19,39 @@ * @author Wei Hong * */ -#ifndef _CHUNK_H_ -#define _CHUNK_H_ + +#pragma once #include #include -#include "../Catalog/ColumnDescriptor.h" -#include "../DataMgr/AbstractBuffer.h" -#include "../DataMgr/ChunkMetadata.h" -#include "../DataMgr/DataMgr.h" -#include "../Shared/sqltypes.h" -#include "../Utils/ChunkIter.h" + +#include "Catalog/ColumnDescriptor.h" +#include "DataMgr/AbstractBuffer.h" +#include "DataMgr/ChunkMetadata.h" +#include "DataMgr/DataMgr.h" +#include "Shared/sqltypes.h" +#include "Utils/ChunkIter.h" using Data_Namespace::AbstractBuffer; using Data_Namespace::DataMgr; using Data_Namespace::MemoryLevel; -namespace Chunk_NS { -class Chunk; -}; - namespace Chunk_NS { class Chunk { public: - Chunk() : buffer(nullptr), index_buf(nullptr), column_desc(nullptr) {} + Chunk() : buffer_(nullptr), index_buf_(nullptr), column_desc_(nullptr) {} + explicit Chunk(const ColumnDescriptor* td) - : buffer(nullptr), index_buf(nullptr), column_desc(td) {} + : buffer_(nullptr), index_buf_(nullptr), column_desc_(td) {} + Chunk(AbstractBuffer* b, AbstractBuffer* ib, const ColumnDescriptor* td) - : buffer(b), index_buf(ib), column_desc(td){}; - ~Chunk() { unpin_buffer(); } - const ColumnDescriptor* get_column_desc() const { return column_desc; } + : buffer_(b), index_buf_(ib), column_desc_(td){}; + + ~Chunk() { unpinBuffer(); } + + const ColumnDescriptor* getColumnDesc() const { return column_desc_; } + static void translateColumnDescriptorsToChunkVec( const std::list& colDescs, std::vector& chunkVec) { @@ -57,29 +59,35 @@ class Chunk { chunkVec.emplace_back(cd); } } + ChunkIter begin_iterator(const std::shared_ptr&, int start_idx = 0, int skip = 1) const; + size_t getNumElemsForBytesInsertData(const DataBlockPtr& src_data, const size_t num_elems, const size_t start_idx, const size_t byte_limit, const bool replicating = false); + std::shared_ptr appendData(DataBlockPtr& srcData, const size_t numAppendElems, const size_t startIdx, const bool replicating = false); + void createChunkBuffer(DataMgr* data_mgr, const ChunkKey& key, const MemoryLevel mem_level, const int deviceId = 0, const size_t page_size = 0); + void getChunkBuffer(DataMgr* data_mgr, const ChunkKey& key, const MemoryLevel mem_level, const int deviceId = 0, const size_t num_bytes = 0, const size_t num_elems = 0); + static std::shared_ptr getChunk(const ColumnDescriptor* cd, DataMgr* data_mgr, const ChunkKey& key, @@ -87,25 +95,30 @@ class Chunk { const int deviceId, const size_t num_bytes, const size_t num_elems); + bool isChunkOnDevice(DataMgr* data_mgr, const ChunkKey& key, const MemoryLevel mem_level, const int device_id); - // protected: - AbstractBuffer* get_buffer() const { return buffer; } - AbstractBuffer* get_index_buf() const { return index_buf; } - void set_buffer(AbstractBuffer* b) { buffer = b; } - void set_index_buf(AbstractBuffer* ib) { index_buf = ib; } - void init_encoder(); + AbstractBuffer* getBuffer() const { return buffer_; } + + AbstractBuffer* getIndexBuf() const { return index_buf_; } + + void setBuffer(AbstractBuffer* b) { buffer_ = b; } + + void setIndexBuffer(AbstractBuffer* ib) { index_buf_ = ib; } + + void initEncoder(); + void decompress(int8_t* compressed, VarlenDatum* result, Datum* datum) const; private: - AbstractBuffer* buffer; - AbstractBuffer* index_buf; - const ColumnDescriptor* column_desc; - void unpin_buffer(); + AbstractBuffer* buffer_; + AbstractBuffer* index_buf_; + const ColumnDescriptor* column_desc_; + + void unpinBuffer(); }; -} // namespace Chunk_NS -#endif // _CHUNK_H_ +} // namespace Chunk_NS diff --git a/DataMgr/ForeignStorage/CsvDataWrapper.cpp b/DataMgr/ForeignStorage/CsvDataWrapper.cpp index 326953dbed..be53e8b62c 100644 --- a/DataMgr/ForeignStorage/CsvDataWrapper.cpp +++ b/DataMgr/ForeignStorage/CsvDataWrapper.cpp @@ -58,21 +58,21 @@ void CsvDataWrapper::initializeChunkBuffers(const int fragment_index) { db_id_, foreign_table_->tableId, column->columnId, fragment_index, 1}; CHECK(chunk_buffer_map_.find(data_chunk_key) == chunk_buffer_map_.end()); chunk_buffer_map_[data_chunk_key] = std::make_unique(); - chunk.set_buffer(chunk_buffer_map_[data_chunk_key].get()); + chunk.setBuffer(chunk_buffer_map_[data_chunk_key].get()); ChunkKey index_chunk_key{ db_id_, foreign_table_->tableId, column->columnId, fragment_index, 2}; CHECK(chunk_buffer_map_.find(index_chunk_key) == chunk_buffer_map_.end()); chunk_buffer_map_[index_chunk_key] = std::make_unique(); - chunk.set_index_buf(chunk_buffer_map_[index_chunk_key].get()); + chunk.setIndexBuffer(chunk_buffer_map_[index_chunk_key].get()); } else { ChunkKey data_chunk_key{ db_id_, foreign_table_->tableId, column->columnId, fragment_index}; CHECK(chunk_buffer_map_.find(data_chunk_key) == chunk_buffer_map_.end()); chunk_buffer_map_[data_chunk_key] = std::make_unique(); - chunk.set_buffer(chunk_buffer_map_[data_chunk_key].get()); + chunk.setBuffer(chunk_buffer_map_[data_chunk_key].get()); } - chunk.init_encoder(); + chunk.initEncoder(); } } @@ -217,22 +217,22 @@ Importer_NS::Loader* CsvDataWrapper::getLoader(Catalog_Namespace::Catalog& catal ChunkKey data_chunk_key{ db_id_, foreign_table_->tableId, column_id, fragment_index, 1}; CHECK(chunk_buffer_map_.find(data_chunk_key) != chunk_buffer_map_.end()); - chunk.set_buffer(chunk_buffer_map_[data_chunk_key].get()); + chunk.setBuffer(chunk_buffer_map_[data_chunk_key].get()); ChunkKey index_chunk_key{ db_id_, foreign_table_->tableId, column_id, fragment_index, 2}; CHECK(chunk_buffer_map_.find(index_chunk_key) != chunk_buffer_map_.end()); - chunk.set_index_buf(chunk_buffer_map_[index_chunk_key].get()); + chunk.setIndexBuffer(chunk_buffer_map_[index_chunk_key].get()); } else { ChunkKey data_chunk_key{ db_id_, foreign_table_->tableId, column_id, fragment_index}; CHECK(chunk_buffer_map_.find(data_chunk_key) != chunk_buffer_map_.end()); - chunk.set_buffer(chunk_buffer_map_[data_chunk_key].get()); + chunk.setBuffer(chunk_buffer_map_[data_chunk_key].get()); } chunk.appendData( data_blocks[i], row_count_for_fragment, processed_import_row_count); - chunk.set_buffer(nullptr); - chunk.set_index_buf(nullptr); + chunk.setBuffer(nullptr); + chunk.setIndexBuffer(nullptr); } row_count_ += row_count_for_fragment; processed_import_row_count += row_count_for_fragment; diff --git a/DataMgr/ForeignStorage/CsvDataWrapper.h b/DataMgr/ForeignStorage/CsvDataWrapper.h index af4d46de5b..91ae9acaf3 100644 --- a/DataMgr/ForeignStorage/CsvDataWrapper.h +++ b/DataMgr/ForeignStorage/CsvDataWrapper.h @@ -22,7 +22,7 @@ #include "Catalog/Catalog.h" #include "Catalog/ForeignTable.h" -#include "Chunk/Chunk.h" +#include "DataMgr/Chunk/Chunk.h" #include "ForeignDataWrapper.h" #include "Import/Importer.h" diff --git a/DataMgr/StringNoneEncoder.h b/DataMgr/StringNoneEncoder.h index 963d6ed494..160e5cd8cd 100644 --- a/DataMgr/StringNoneEncoder.h +++ b/DataMgr/StringNoneEncoder.h @@ -102,8 +102,8 @@ class StringNoneEncoder : public Encoder { has_nulls = static_cast(copyFromEncoder)->has_nulls; } - AbstractBuffer* get_index_buf() const { return index_buf; } - void set_index_buf(AbstractBuffer* buf) { index_buf = buf; } + AbstractBuffer* getIndexBuf() const { return index_buf; } + void setIndexBuffer(AbstractBuffer* buf) { index_buf = buf; } private: AbstractBuffer* index_buf; diff --git a/Fragmenter/InsertOrderFragmenter.cpp b/Fragmenter/InsertOrderFragmenter.cpp index bc48a4fc65..e213fd235c 100644 --- a/Fragmenter/InsertOrderFragmenter.cpp +++ b/Fragmenter/InsertOrderFragmenter.cpp @@ -77,9 +77,9 @@ InsertOrderFragmenter::InsertOrderFragmenter( // find row id column if it is non virtual for (auto colIt = chunkVec.begin(); colIt != chunkVec.end(); ++colIt) { - int columnId = colIt->get_column_desc()->columnId; + int columnId = colIt->getColumnDesc()->columnId; columnMap_[columnId] = *colIt; - if (colIt->get_column_desc()->columnName == "rowid") { + if (colIt->getColumnDesc()->columnName == "rowid") { hasMaterializedRowId_ = true; rowIdColId_ = columnId; } @@ -146,7 +146,7 @@ void InsertOrderFragmenter::getChunkMetadata() { ssize_t maxFixedColSize = 0; for (auto colIt = columnMap_.begin(); colIt != columnMap_.end(); ++colIt) { - ssize_t size = colIt->second.get_column_desc()->columnType.get_size(); + ssize_t size = colIt->second.getColumnDesc()->columnType.get_size(); if (size == -1) { // variable length varLenColInfo_.insert(std::make_pair(colIt->first, 0)); size = 8; // b/c we use this for string and array indices - gross to have magic @@ -172,7 +172,7 @@ void InsertOrderFragmenter::getChunkMetadata() { colIt->second.getChunkBuffer(dataMgr_, insertKey, defaultInsertLevel_, deviceId); auto varLenColInfoIt = varLenColInfo_.find(colIt->first); if (varLenColInfoIt != varLenColInfo_.end()) { - varLenColInfoIt->second = colIt->second.get_buffer()->size(); + varLenColInfoIt->second = colIt->second.getBuffer()->size(); } } } @@ -273,7 +273,7 @@ void InsertOrderFragmenter::updateChunkStats( 0, chunk_meta_it->second->numBytes, chunk_meta_it->second->numElements); - auto buf = chunk->get_buffer(); + auto buf = chunk->getBuffer(); CHECK(buf); auto encoder = buf->encoder.get(); if (!encoder) { @@ -409,7 +409,7 @@ void InsertOrderFragmenter::replicateData(const InsertData& insertDataStruct) { chunkKey, defaultInsertLevel_, fragmentInfo->deviceIds[static_cast(defaultInsertLevel_)]); - chunk.init_encoder(); + chunk.initEncoder(); try { DataBlockPtr dataCopy = insertDataStruct.data[i]; @@ -435,7 +435,7 @@ void InsertOrderFragmenter::replicateData(const InsertData& insertDataStruct) { // update total size of var-len column in (actually the last) fragment if (0 > size) { std::unique_lock lck(*mutex_access_inmem_states); - varLenColInfo_[columnId] = chunk.get_buffer()->size(); + varLenColInfo_[columnId] = chunk.getBuffer()->size(); } } catch (...) { dataMgr_->deleteChunksWithPrefix(chunkKey); @@ -502,12 +502,12 @@ void InsertOrderFragmenter::insertDataImpl(InsertData& insertDataStruct) { // it is not needed and will cause issues std::unique_ptr data_for_deleted_column; for (const auto& cit : columnMap_) { - if (cit.second.get_column_desc()->isDeletedCol && + if (cit.second.getColumnDesc()->isDeletedCol && insertDataStruct.replicate_count == 0) { data_for_deleted_column.reset(new int8_t[insertDataStruct.numRows]); memset(data_for_deleted_column.get(), 0, insertDataStruct.numRows); insertDataStruct.data.emplace_back(DataBlockPtr{data_for_deleted_column.get()}); - insertDataStruct.columnIds.push_back(cit.second.get_column_desc()->columnId); + insertDataStruct.columnIds.push_back(cit.second.getColumnDesc()->columnId); break; } } @@ -616,7 +616,7 @@ void InsertOrderFragmenter::insertDataImpl(InsertData& insertDataStruct) { colMapIt->second.appendData(dataCopy[i], numRowsToInsert, numRowsInserted); auto varLenColInfoIt = varLenColInfo_.find(columnId); if (varLenColInfoIt != varLenColInfo_.end()) { - varLenColInfoIt->second = colMapIt->second.get_buffer()->size(); + varLenColInfoIt->second = colMapIt->second.getBuffer()->size(); } } if (hasMaterializedRowId_) { @@ -674,9 +674,8 @@ FragmentInfo* InsertOrderFragmenter::createNewFragment( for (map::iterator colMapIt = columnMap_.begin(); colMapIt != columnMap_.end(); ++colMapIt) { - // colMapIt->second.unpin_buffer(); ChunkKey chunkKey = chunkKeyPrefix_; - chunkKey.push_back(colMapIt->second.get_column_desc()->columnId); + chunkKey.push_back(colMapIt->second.getColumnDesc()->columnId); chunkKey.push_back(maxFragmentId_); colMapIt->second.createChunkBuffer( dataMgr_, @@ -684,7 +683,7 @@ FragmentInfo* InsertOrderFragmenter::createNewFragment( memoryLevel, newFragmentInfo->deviceIds[static_cast(memoryLevel)], pageSize_); - colMapIt->second.init_encoder(); + colMapIt->second.initEncoder(); } mapd_lock_guard writeLock(fragmentInfoMutex_); diff --git a/Fragmenter/InsertOrderFragmenter.h b/Fragmenter/InsertOrderFragmenter.h index caf490b768..88be184d48 100644 --- a/Fragmenter/InsertOrderFragmenter.h +++ b/Fragmenter/InsertOrderFragmenter.h @@ -26,7 +26,7 @@ #include #include -#include "Chunk/Chunk.h" +#include "DataMgr/Chunk/Chunk.h" #include "DataMgr/MemoryLevel.h" #include "FragmentDefaultValues.h" #include "Fragmenter/AbstractFragmenter.h" diff --git a/Fragmenter/UpdelStorage.cpp b/Fragmenter/UpdelStorage.cpp index fbe8ede0f3..6a3d15b372 100644 --- a/Fragmenter/UpdelStorage.cpp +++ b/Fragmenter/UpdelStorage.cpp @@ -142,10 +142,10 @@ struct ScalarChunkConverter : public ChunkToInsertDataConverter { const BUFFER_DATA_TYPE* data_buffer_addr_; ScalarChunkConverter(const size_t num_rows, const Chunk_NS::Chunk* chunk) - : chunk_(chunk), column_descriptor_(chunk->get_column_desc()) { + : chunk_(chunk), column_descriptor_(chunk->getColumnDesc()) { column_data_ = ColumnDataPtr(reinterpret_cast( checked_malloc(num_rows * sizeof(INSERT_DATA_TYPE)))); - data_buffer_addr_ = (BUFFER_DATA_TYPE*)chunk->get_buffer()->getMemoryPtr(); + data_buffer_addr_ = (BUFFER_DATA_TYPE*)chunk->getBuffer()->getMemoryPtr(); } ~ScalarChunkConverter() override {} @@ -173,10 +173,10 @@ struct FixedLenArrayChunkConverter : public ChunkToInsertDataConverter { size_t fixed_array_length_; FixedLenArrayChunkConverter(const size_t num_rows, const Chunk_NS::Chunk* chunk) - : chunk_(chunk), column_descriptor_(chunk->get_column_desc()) { + : chunk_(chunk), column_descriptor_(chunk->getColumnDesc()) { column_data_ = std::make_unique>(num_rows); - data_buffer_addr_ = chunk->get_buffer()->getMemoryPtr(); - fixed_array_length_ = chunk->get_column_desc()->columnType.get_size(); + data_buffer_addr_ = chunk->getBuffer()->getMemoryPtr(); + fixed_array_length_ = chunk->getColumnDesc()->columnType.get_size(); } ~FixedLenArrayChunkConverter() override {} @@ -205,8 +205,8 @@ struct ArrayChunkConverter : public FixedLenArrayChunkConverter { ArrayChunkConverter(const size_t num_rows, const Chunk_NS::Chunk* chunk) : FixedLenArrayChunkConverter(num_rows, chunk) { index_buffer_addr_ = - (StringOffsetT*)(chunk->get_index_buf() ? chunk->get_index_buf()->getMemoryPtr() - : nullptr); + (StringOffsetT*)(chunk->getIndexBuf() ? chunk->getIndexBuf()->getMemoryPtr() + : nullptr); } ~ArrayChunkConverter() override {} @@ -230,12 +230,12 @@ struct StringChunkConverter : public ChunkToInsertDataConverter { const StringOffsetT* index_buffer_addr_; StringChunkConverter(size_t num_rows, const Chunk_NS::Chunk* chunk) - : chunk_(chunk), column_descriptor_(chunk->get_column_desc()) { + : chunk_(chunk), column_descriptor_(chunk->getColumnDesc()) { column_data_ = std::make_unique>(num_rows); - data_buffer_addr_ = chunk->get_buffer()->getMemoryPtr(); + data_buffer_addr_ = chunk->getBuffer()->getMemoryPtr(); index_buffer_addr_ = - (StringOffsetT*)(chunk->get_index_buf() ? chunk->get_index_buf()->getMemoryPtr() - : nullptr); + (StringOffsetT*)(chunk->getIndexBuf() ? chunk->getIndexBuf()->getMemoryPtr() + : nullptr); } ~StringChunkConverter() override {} @@ -265,10 +265,10 @@ struct DateChunkConverter : public ChunkToInsertDataConverter { const BUFFER_DATA_TYPE* data_buffer_addr_; DateChunkConverter(const size_t num_rows, const Chunk_NS::Chunk* chunk) - : chunk_(chunk), column_descriptor_(chunk->get_column_desc()) { + : chunk_(chunk), column_descriptor_(chunk->getColumnDesc()) { column_data_ = ColumnDataPtr( reinterpret_cast(checked_malloc(num_rows * sizeof(int64_t)))); - data_buffer_addr_ = (BUFFER_DATA_TYPE*)chunk->get_buffer()->getMemoryPtr(); + data_buffer_addr_ = (BUFFER_DATA_TYPE*)chunk->getBuffer()->getMemoryPtr(); } ~DateChunkConverter() override {} @@ -328,7 +328,7 @@ void InsertOrderFragmenter::updateColumns( std::shared_ptr deletedChunk; for (size_t indexOfChunk = 0; indexOfChunk < chunks.size(); indexOfChunk++) { auto chunk = chunks[indexOfChunk]; - const auto chunk_cd = chunk->get_column_desc(); + const auto chunk_cd = chunk->getColumnDesc(); if (chunk_cd->isDeletedCol) { deletedChunk = chunk; @@ -483,12 +483,12 @@ void InsertOrderFragmenter::updateColumns( updelRoll.dirtyChunks[deletedChunk.get()] = deletedChunk; ChunkKey chunkey{updelRoll.catalog->getCurrentDB().dbId, - deletedChunk->get_column_desc()->tableId, - deletedChunk->get_column_desc()->columnId, + deletedChunk->getColumnDesc()->tableId, + deletedChunk->getColumnDesc()->columnId, fragment.fragmentId}; updelRoll.dirtyChunkeys.insert(chunkey); bool* deletedChunkBuffer = - reinterpret_cast(deletedChunk->get_buffer()->getMemoryPtr()); + reinterpret_cast(deletedChunk->getBuffer()->getMemoryPtr()); std::atomic row_idx{0}; @@ -578,20 +578,20 @@ void InsertOrderFragmenter::updateColumns( insertDataNoCheckpoint(insert_data); // update metdata - if (!deletedChunk->get_buffer()->has_encoder) { - deletedChunk->init_encoder(); + if (!deletedChunk->getBuffer()->has_encoder) { + deletedChunk->initEncoder(); } - deletedChunk->get_buffer()->encoder->updateStats(static_cast(true), false); + deletedChunk->getBuffer()->encoder->updateStats(static_cast(true), false); auto& shadowDeletedChunkMeta = - fragment.shadowChunkMetadataMap[deletedChunk->get_column_desc()->columnId]; + fragment.shadowChunkMetadataMap[deletedChunk->getColumnDesc()->columnId]; if (shadowDeletedChunkMeta->numElements > - deletedChunk->get_buffer()->encoder->getNumElems()) { + deletedChunk->getBuffer()->encoder->getNumElems()) { // the append will have populated shadow meta data, otherwise use existing num // elements - deletedChunk->get_buffer()->encoder->setNumElems(shadowDeletedChunkMeta->numElements); + deletedChunk->getBuffer()->encoder->setNumElems(shadowDeletedChunkMeta->numElements); } - deletedChunk->get_buffer()->setUpdated(); + deletedChunk->getBuffer()->setUpdated(); } void InsertOrderFragmenter::updateColumn(const Catalog_Namespace::Catalog* catalog, @@ -639,7 +639,7 @@ void InsertOrderFragmenter::updateColumn(const Catalog_Namespace::Catalog* catal std::vector> threads; const auto segsz = (nrow + ncore - 1) / ncore; - auto dbuf = chunk->get_buffer(); + auto dbuf = chunk->getBuffer(); auto dbuf_addr = dbuf->getMemoryPtr(); dbuf->setUpdated(); { @@ -946,7 +946,7 @@ void InsertOrderFragmenter::updateColumnMetadata(const ColumnDescriptor* cd, } auto& chunkMetadata = updel_roll.chunkMetadata[key]; - auto buffer = chunk->get_buffer(); + auto buffer = chunk->getBuffer(); const auto& lhs_type = cd->columnType; auto encoder = buffer->encoder.get(); @@ -1026,7 +1026,7 @@ auto InsertOrderFragmenter::getChunksForAllColumns( // get a sorted vector of offsets of rows to vacuum const std::vector InsertOrderFragmenter::getVacuumOffsets( const std::shared_ptr& chunk) { - const auto data_buffer = chunk->get_buffer(); + const auto data_buffer = chunk->getBuffer(); const auto data_addr = data_buffer->getMemoryPtr(); const size_t nrows_in_chunk = data_buffer->size(); const size_t ncore = cpu_threads(); @@ -1077,9 +1077,9 @@ static void set_chunk_metadata(const Catalog_Namespace::Catalog* catalog, const std::shared_ptr& chunk, const size_t nrows_to_keep, UpdelRoll& updel_roll) { - auto cd = chunk->get_column_desc(); + auto cd = chunk->getColumnDesc(); auto td = catalog->getMetadataForTable(cd->tableId); - auto data_buffer = chunk->get_buffer(); + auto data_buffer = chunk->getBuffer(); std::lock_guard lck(updel_roll.mutex); const auto key = std::make_pair(td, &fragment); if (0 == updel_roll.chunkMetadata.count(key)) { @@ -1097,9 +1097,9 @@ auto InsertOrderFragmenter::vacuum_fixlen_rows( const FragmentInfo& fragment, const std::shared_ptr& chunk, const std::vector& frag_offsets) { - const auto cd = chunk->get_column_desc(); + const auto cd = chunk->getColumnDesc(); const auto& col_type = cd->columnType; - auto data_buffer = chunk->get_buffer(); + auto data_buffer = chunk->getBuffer(); auto data_addr = data_buffer->getMemoryPtr(); auto element_size = col_type.is_fixlen_array() ? col_type.get_size() : get_element_size(col_type); @@ -1133,8 +1133,8 @@ auto InsertOrderFragmenter::vacuum_varlen_rows( const FragmentInfo& fragment, const std::shared_ptr& chunk, const std::vector& frag_offsets) { - auto data_buffer = chunk->get_buffer(); - auto index_buffer = chunk->get_index_buf(); + auto data_buffer = chunk->getBuffer(); + auto index_buffer = chunk->getIndexBuf(); auto data_addr = data_buffer->getMemoryPtr(); auto indices_addr = index_buffer ? index_buffer->getMemoryPtr() : nullptr; auto index_array = (StringOffsetT*)indices_addr; @@ -1210,10 +1210,10 @@ void InsertOrderFragmenter::compactRows(const Catalog_Namespace::Catalog* catalo for (size_t ci = 0; ci < chunks.size(); ++ci) { auto chunk = chunks[ci]; - const auto cd = chunk->get_column_desc(); + const auto cd = chunk->getColumnDesc(); const auto& col_type = cd->columnType; - auto data_buffer = chunk->get_buffer(); - auto index_buffer = chunk->get_index_buf(); + auto data_buffer = chunk->getBuffer(); + auto index_buffer = chunk->getIndexBuf(); auto data_addr = data_buffer->getMemoryPtr(); auto indices_addr = index_buffer ? index_buffer->getMemoryPtr() : nullptr; auto index_array = (StringOffsetT*)indices_addr; @@ -1294,7 +1294,7 @@ void InsertOrderFragmenter::compactRows(const Catalog_Namespace::Catalog* catalo updel_roll.numTuples[key] = nrows_to_keep; for (size_t ci = 0; ci < chunks.size(); ++ci) { auto chunk = chunks[ci]; - auto cd = chunk->get_column_desc(); + auto cd = chunk->getColumnDesc(); if (!cd->columnType.is_fixlen_array()) { updateColumnMetadata(cd, fragment, @@ -1353,8 +1353,8 @@ void UpdelRoll::cancelUpdate() { CHECK(td); if (td->persistenceLevel != memoryLevel) { for (auto dit : dirtyChunks) { - catalog->getDataMgr().free(dit.first->get_buffer()); - dit.first->set_buffer(nullptr); + catalog->getDataMgr().free(dit.first->getBuffer()); + dit.first->setBuffer(nullptr); } } } diff --git a/Import/CMakeLists.txt b/Import/CMakeLists.txt index b2aad3db19..27847d2ab9 100644 --- a/Import/CMakeLists.txt +++ b/Import/CMakeLists.txt @@ -10,7 +10,7 @@ endif() add_library(CsvImport Importer.cpp Importer.h DelimitedParserUtils.cpp DelimitedParserUtils.h ${S3Archive}) -target_link_libraries(CsvImport mapd_thrift Shared Catalog Chunk DataMgr StringDictionary ${GDAL_LIBRARIES} ${CMAKE_DL_LIBS} +target_link_libraries(CsvImport mapd_thrift Shared Catalog DataMgr StringDictionary ${GDAL_LIBRARIES} ${CMAKE_DL_LIBS} ${LibArchive_LIBRARIES} ${IMPORT_LIBRARIES} ${Arrow_LIBRARIES}) install(DIRECTORY ${CMAKE_SOURCE_DIR}/ThirdParty/gdal-data DESTINATION "ThirdParty") diff --git a/Import/Importer.h b/Import/Importer.h index b2533e5540..c49766c165 100644 --- a/Import/Importer.h +++ b/Import/Importer.h @@ -44,7 +44,7 @@ #include "Catalog/Catalog.h" #include "Catalog/TableDescriptor.h" -#include "Chunk/Chunk.h" +#include "DataMgr/Chunk/Chunk.h" #include "Fragmenter/Fragmenter.h" #include "Import/CopyParams.h" #include "Shared/Logger.h" diff --git a/QueryEngine/CMakeLists.txt b/QueryEngine/CMakeLists.txt index 18bf1f0a59..f4781ed996 100644 --- a/QueryEngine/CMakeLists.txt +++ b/QueryEngine/CMakeLists.txt @@ -210,7 +210,6 @@ else() endif() set(QUERY_ENGINE_LIBS - Chunk Analyzer StringDictionary Utils diff --git a/QueryEngine/ColumnFetcher.cpp b/QueryEngine/ColumnFetcher.cpp index a3b2a26189..f40d66908a 100644 --- a/QueryEngine/ColumnFetcher.cpp +++ b/QueryEngine/ColumnFetcher.cpp @@ -60,7 +60,7 @@ std::pair ColumnFetcher::getOneColumnFragment( chunk_meta_it->second->numElements); chunks_owner.push_back(chunk); CHECK(chunk); - auto ab = chunk->get_buffer(); + auto ab = chunk->getBuffer(); CHECK(ab->getMemoryPtr()); col_buff = reinterpret_cast(ab->getMemoryPtr()); } else { // temporary table @@ -208,7 +208,7 @@ const int8_t* ColumnFetcher::getOneTableColumnFragment( if (memory_level == Data_Namespace::CPU_LEVEL) { return reinterpret_cast(&chunk_iter); } else { - auto ab = chunk->get_buffer(); + auto ab = chunk->getBuffer(); ab->pin(); auto& row_set_mem_owner = executor_->getRowSetMemoryOwner(); row_set_mem_owner->addVarlenInputBuffer(ab); @@ -223,7 +223,7 @@ const int8_t* ColumnFetcher::getOneTableColumnFragment( return chunk_iter_gpu; } } else { - auto ab = chunk->get_buffer(); + auto ab = chunk->getBuffer(); CHECK(ab->getMemoryPtr()); return ab->getMemoryPtr(); // @TODO(alex) change to use ChunkIter } diff --git a/QueryEngine/Execute.h b/QueryEngine/Execute.h index 63793a9275..5b3d007f42 100644 --- a/QueryEngine/Execute.h +++ b/QueryEngine/Execute.h @@ -37,7 +37,6 @@ #include "TargetMetaInfo.h" #include "WindowContext.h" -#include "../Chunk/Chunk.h" #include "../Shared/Logger.h" #include "../Shared/SystemParameters.h" #include "../Shared/mapd_shared_mutex.h" @@ -46,6 +45,7 @@ #include "../StringDictionary/LruCache.hpp" #include "../StringDictionary/StringDictionary.h" #include "../StringDictionary/StringDictionaryProxy.h" +#include "DataMgr/Chunk/Chunk.h" #include #include diff --git a/QueryEngine/ExecutionDispatch.cpp b/QueryEngine/ExecutionDispatch.cpp index 6408f0ff16..ece41aeb89 100644 --- a/QueryEngine/ExecutionDispatch.cpp +++ b/QueryEngine/ExecutionDispatch.cpp @@ -40,14 +40,14 @@ bool needs_skip_result(const ResultSetPtr& res) { // column is part of the target expressions, result set iteration needs it alive. bool need_to_hold_chunk(const Chunk_NS::Chunk* chunk, const RelAlgExecutionUnit& ra_exe_unit) { - CHECK(chunk->get_column_desc()); - const auto chunk_ti = chunk->get_column_desc()->columnType; + CHECK(chunk->getColumnDesc()); + const auto chunk_ti = chunk->getColumnDesc()->columnType; if (chunk_ti.is_array() || (chunk_ti.is_string() && chunk_ti.get_compression() == kENCODING_NONE)) { for (const auto target_expr : ra_exe_unit.target_exprs) { const auto col_var = dynamic_cast(target_expr); - if (col_var && col_var->get_column_id() == chunk->get_column_desc()->columnId && - col_var->get_table_id() == chunk->get_column_desc()->tableId) { + if (col_var && col_var->get_column_id() == chunk->getColumnDesc()->columnId && + col_var->get_table_id() == chunk->getColumnDesc()->tableId) { return true; } } diff --git a/QueryEngine/JoinHashTable.h b/QueryEngine/JoinHashTable.h index 090e65aa87..246f296b9f 100644 --- a/QueryEngine/JoinHashTable.h +++ b/QueryEngine/JoinHashTable.h @@ -24,11 +24,11 @@ #ifndef QUERYENGINE_JOINHASHTABLE_H #define QUERYENGINE_JOINHASHTABLE_H -#include "../Analyzer/Analyzer.h" -#include "../Catalog/Catalog.h" -#include "../Chunk/Chunk.h" #include "Allocators/ThrustAllocator.h" +#include "Analyzer/Analyzer.h" +#include "Catalog/Catalog.h" #include "ColumnarResults.h" +#include "DataMgr/Chunk/Chunk.h" #include "Descriptors/InputDescriptors.h" #include "Descriptors/RowSetMemoryOwner.h" #include "ExpressionRange.h" diff --git a/QueryEngine/JoinHashTableQueryRuntime.cpp b/QueryEngine/JoinHashTableQueryRuntime.cpp index 8a2a880f4c..23d575d61f 100644 --- a/QueryEngine/JoinHashTableQueryRuntime.cpp +++ b/QueryEngine/JoinHashTableQueryRuntime.cpp @@ -213,12 +213,13 @@ struct BufferRange { const int64_t element_count = 0; }; -ALWAYS_INLINE DEVICE BufferRange get_row_id_buffer_ptr(int64_t* hash_table_ptr, - const int64_t* key, - const int64_t key_component_count, - const int64_t entry_count, - const int64_t offset_buffer_off, - const int64_t sub_buff_size) { +ALWAYS_INLINE DEVICE BufferRange +get_row_id_buffer_ptr(int64_t* hash_table_ptr, + const int64_t* key, + const int64_t key_component_count, + const int64_t entry_count, + const int64_t offset_buffer_ptr_offset, + const int64_t sub_buff_size) { const int64_t min_key = 0; const int64_t max_key = entry_count - 1; @@ -230,7 +231,7 @@ ALWAYS_INLINE DEVICE BufferRange get_row_id_buffer_ptr(int64_t* hash_table_ptr, } int8_t* one_to_many_ptr = reinterpret_cast(hash_table_ptr); - one_to_many_ptr += offset_buffer_off; + one_to_many_ptr += offset_buffer_ptr_offset; // Returns an index used to fetch row count and row ids. const auto slot = overlaps_hash_join_idx( @@ -294,7 +295,7 @@ get_candidate_rows(int32_t* out_arr, const int64_t key_component_count, int64_t* hash_table_ptr, const int64_t entry_count, - const int64_t offset_buffer_off, + const int64_t offset_buffer_ptr_offset, const int64_t sub_buff_size) { const auto range = reinterpret_cast(range_bytes); @@ -317,7 +318,7 @@ get_candidate_rows(int32_t* out_arr, cur_key, key_component_count, entry_count, - offset_buffer_off, + offset_buffer_ptr_offset, sub_buff_size); for (int64_t j = 0; j < buffer_range.element_count; j++) { diff --git a/QueryEngine/OverlapsJoinHashTable.cpp b/QueryEngine/OverlapsJoinHashTable.cpp index 1b7d366e21..2328560420 100644 --- a/QueryEngine/OverlapsJoinHashTable.cpp +++ b/QueryEngine/OverlapsJoinHashTable.cpp @@ -949,7 +949,7 @@ HashJoinMatchingSet OverlapsJoinHashTable::codegenMatchingSet( LL_INT(key_component_count), // key_component_count composite_key_dict, // ptr to hash table LL_INT(entry_count_), // entry_count - LL_INT(composite_key_dict_size), // offset_buffer_off + LL_INT(composite_key_dict_size), // offset_buffer_ptr_offset LL_INT(entry_count_ * sizeof(int32_t)) // sub_buff_size }); diff --git a/QueryEngine/ResultSet.h b/QueryEngine/ResultSet.h index 17950adfae..70c7538707 100644 --- a/QueryEngine/ResultSet.h +++ b/QueryEngine/ResultSet.h @@ -25,8 +25,8 @@ #ifndef QUERYENGINE_RESULTSET_H #define QUERYENGINE_RESULTSET_H -#include "../Chunk/Chunk.h" #include "CardinalityEstimator.h" +#include "DataMgr/Chunk/Chunk.h" #include "ResultSetBufferAccessors.h" #include "TargetValue.h" diff --git a/QueryEngine/ResultSetReductionJIT.cpp b/QueryEngine/ResultSetReductionJIT.cpp index a272039c68..214f18057a 100644 --- a/QueryEngine/ResultSetReductionJIT.cpp +++ b/QueryEngine/ResultSetReductionJIT.cpp @@ -521,7 +521,7 @@ ResultSetReductionJIT::ResultSetReductionJIT(const QueryMemoryDescriptor& query_ // # Retrieval of 'this_ptr' is different between perfect hash and baseline. // this_ptr = this_result_set[that_entry_index] // or -// get_row(key(that_row_ptr), this_result_set_buffer) +// get_row(key(that_row_ptr), this_result_setBuffer) // reduce_func_[baseline|perfect_hash](this_ptr, that_ptr) // // func reduce_loop(this_buff, that_buff, start_entry_index, end_entry_index): @@ -1278,4 +1278,4 @@ ReductionCode GpuReductionHelperJIT::codegen() const { reduction_code.llvm_reduce_loop = ir_reduce_loop; reduction_code.module = std::move(module); return reduction_code; -} \ No newline at end of file +} diff --git a/QueryEngine/StorageIOFacility.h b/QueryEngine/StorageIOFacility.h index a2da8eef55..6f09fe2c09 100644 --- a/QueryEngine/StorageIOFacility.h +++ b/QueryEngine/StorageIOFacility.h @@ -189,7 +189,7 @@ StorageIOFacility::yieldUpdateCallback( chunk_metadata->second->numBytes, chunk_metadata->second->numElements); CHECK(chunk); - auto chunk_buffer = chunk->get_buffer(); + auto chunk_buffer = chunk->getBuffer(); CHECK(chunk_buffer && chunk_buffer->has_encoder); auto encoder = chunk_buffer->encoder.get(); @@ -423,7 +423,7 @@ StorageIOFacility::yieldDeleteCallback( chunk_metadata->second->numBytes, chunk_metadata->second->numElements); CHECK(chunk); - auto chunk_buffer = chunk->get_buffer(); + auto chunk_buffer = chunk->getBuffer(); CHECK(chunk_buffer && chunk_buffer->has_encoder); auto encoder = chunk_buffer->encoder.get(); diff --git a/QueryEngine/WindowContext.h b/QueryEngine/WindowContext.h index 71493b7b95..dfe74724ba 100644 --- a/QueryEngine/WindowContext.h +++ b/QueryEngine/WindowContext.h @@ -16,8 +16,8 @@ #pragma once -#include "../Analyzer/Analyzer.h" -#include "../Chunk/Chunk.h" +#include "Analyzer/Analyzer.h" +#include "DataMgr/Chunk/Chunk.h" #include "JoinHashTableInterface.h" #include diff --git a/Shared/UpdelRoll.h b/Shared/UpdelRoll.h index 04c42fcfd4..69a9c2bb2a 100644 --- a/Shared/UpdelRoll.h +++ b/Shared/UpdelRoll.h @@ -21,9 +21,9 @@ #include #include -#include "../Chunk/Chunk.h" -#include "../DataMgr/ChunkMetadata.h" -#include "../DataMgr/MemoryLevel.h" +#include "DataMgr/Chunk/Chunk.h" +#include "DataMgr/ChunkMetadata.h" +#include "DataMgr/MemoryLevel.h" namespace Fragmenter_Namespace { class InsertOrderFragmenter; diff --git a/Tests/ScanTable.cpp b/Tests/ScanTable.cpp index fb2d96c208..95e15db1f5 100644 --- a/Tests/ScanTable.cpp +++ b/Tests/ScanTable.cpp @@ -32,12 +32,13 @@ #include #include #include -#include "../Catalog/Catalog.h" -#include "../Chunk/Chunk.h" -#include "../DataMgr/DataMgr.h" -#include "../Fragmenter/Fragmenter.h" -#include "../Shared/measure.h" -#include "../Shared/sqltypes.h" + +#include "Catalog/Catalog.h" +#include "DataMgr/Chunk/Chunk.h" +#include "DataMgr/DataMgr.h" +#include "Fragmenter/Fragmenter.h" +#include "Shared/measure.h" +#include "Shared/sqltypes.h" using namespace std; using namespace Catalog_Namespace; @@ -52,7 +53,7 @@ void scan_chunk(const std::shared_ptr& chunk_metadata, ChunkIter cit = chunk.begin_iterator(chunk_metadata, 0, 1); VarlenDatum vd; bool is_end; - const ColumnDescriptor* cd = chunk.get_column_desc(); + const ColumnDescriptor* cd = chunk.getColumnDesc(); std::hash string_hash; int nth = 0; while (true) { diff --git a/Utils/ChunkAccessorTable.h b/Utils/ChunkAccessorTable.h index a042951033..6d4a2421e4 100644 --- a/Utils/ChunkAccessorTable.h +++ b/Utils/ChunkAccessorTable.h @@ -22,8 +22,8 @@ #ifndef _CHUNK_ACCESSOR_TABLE_H_ #define _CHUNK_ACCESSOR_TABLE_H_ -#include -#include +#include "Catalog/Catalog.h" +#include "DataMgr/Chunk/Chunk.h" #include