diff --git a/internal/core/src/common/Array.h b/internal/core/src/common/Array.h index 157fe78702818..4d617a1145ba4 100644 --- a/internal/core/src/common/Array.h +++ b/internal/core/src/common/Array.h @@ -438,19 +438,42 @@ class Array { int size_ = 0; std::vector offsets_{}; DataType element_type_ = DataType::NONE; + + //offsets for mmap, padding to be consistent with ArrayView + const uint64_t* offsets_ptr_; }; class ArrayView { public: ArrayView() = default; + ArrayView(char* data, + int len, + size_t size, + DataType element_type, + const uint64_t* offsets_ptr) + : data_(data), + length_(len), + size_(size), + element_type_(element_type), + offsets_ptr_(offsets_ptr) { + AssertInfo(data != nullptr, + "data pointer for ArrayView cannot be nullptr"); + if (IsVariableDataType(element_type_)) { + AssertInfo(offsets_ptr != nullptr, + "for variable data type, offsets_ptr for array view " + "must not be nullptr"); + } + } + ArrayView(char* data, size_t size, DataType element_type, std::vector&& element_offsets) : size_(size), offsets_(std::move(element_offsets)), - element_type_(element_type) { + element_type_(element_type), + offsets_ptr_(nullptr) { data_ = data; if (IsVariableDataType(element_type_)) { length_ = offsets_.size(); @@ -475,10 +498,19 @@ class ArrayView { if constexpr (std::is_same_v || std::is_same_v) { - size_t element_length = (index == length_ - 1) - ? size_ - offsets_.back() - : offsets_[index + 1] - offsets_[index]; - return T(data_ + offsets_[index], element_length); + if (offsets_ptr_) { + size_t element_length = + (index == length_ - 1) + ? size_ - offsets_ptr_[length_ - 1] + : offsets_ptr_[index + 1] - offsets_ptr_[index]; + return T(data_ + offsets_ptr_[index], element_length); + } else { + size_t element_length = + (index == length_ - 1) + ? size_ - offsets_.back() + : offsets_[index + 1] - offsets_[index]; + return T(data_ + offsets_[index], element_length); + } } if constexpr (std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v) { @@ -583,6 +615,10 @@ class ArrayView { // copy to result std::vector get_offsets_in_copy() const { + if (offsets_ptr_) { + return std::vector( + offsets_ptr_, offsets_ptr_ + sizeof(uint64_t) * length_); + } return offsets_; } @@ -663,6 +699,9 @@ class ArrayView { int size_ = 0; std::vector offsets_{}; DataType element_type_ = DataType::NONE; + + //offsets for mmap + const uint64_t* offsets_ptr_; }; } // namespace milvus diff --git a/internal/core/src/common/Chunk.cpp b/internal/core/src/common/Chunk.cpp index 4b5c17ac1df6e..b1de873d40f2f 100644 --- a/internal/core/src/common/Chunk.cpp +++ b/internal/core/src/common/Chunk.cpp @@ -51,21 +51,18 @@ ArrayChunk::ConstructViews() { int offset = offsets_lens_[2 * i]; int next_offset = offsets_lens_[2 * (i + 1)]; int len = offsets_lens_[2 * i + 1]; - auto data_ptr = data_ + offset; - auto offsets_len = 0; - std::vector element_indices = {}; + auto offsets_bytes_len = 0; + uint64_t* offsets_ptr = nullptr; if (IsStringDataType(element_type_)) { - offsets_len = len * sizeof(uint64_t); - std::vector tmp( - reinterpret_cast(data_ptr), - reinterpret_cast(data_ptr + offsets_len)); - element_indices = std::move(tmp); + offsets_bytes_len = len * sizeof(uint64_t); + offsets_ptr = reinterpret_cast(data_ptr); } - views_.emplace_back(data_ptr + offsets_len, - next_offset - offset - offsets_len, + views_.emplace_back(data_ptr + offsets_bytes_len, + len, + next_offset - offset - offsets_bytes_len, element_type_, - std::move(element_indices)); + offsets_ptr); } } diff --git a/internal/core/src/storage/MmapChunkManager.cpp b/internal/core/src/storage/MmapChunkManager.cpp index 935e9db2e8260..44547491b4e0a 100644 --- a/internal/core/src/storage/MmapChunkManager.cpp +++ b/internal/core/src/storage/MmapChunkManager.cpp @@ -169,8 +169,10 @@ MmapBlocksHandler::AllocateLargeBlock(const uint64_t size) { if (size + Size() > max_disk_limit_) { PanicInfo(ErrorCode::MemAllocateSizeNotMatch, "Failed to create a new mmap_block, not enough disk for " - "create a new mmap block. Allocated size: {}, Max size: {} " + "create a new mmap block. To Allocate:{} Allocated size: {}, " + "Max size: {} " "under mmap file_prefix: {}", + size, Size(), max_disk_limit_, mmap_file_prefix_);