Skip to content

Commit

Permalink
Introduce BaseFileReader::get_mmap_buffer.
Browse files Browse the repository at this point in the history
BaseFileReader is now responsible for trying to get a mmap and then
fallback to reading in file.
  • Loading branch information
mgautierfr committed Mar 29, 2024
1 parent 5eba7f1 commit 14b2306
Show file tree
Hide file tree
Showing 2 changed files with 42 additions and 38 deletions.
60 changes: 26 additions & 34 deletions src/file_reader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -177,24 +177,11 @@ makeMmappedBuffer(int fd, offset_t offset, zsize_t size)
} // unnamed namespace
#endif // ENABLE_USE_MMAP

const Buffer MultiPartFileReader::get_buffer(offset_t offset, zsize_t size) const {
const Buffer BaseFileReader::get_buffer(offset_t offset, zsize_t size) const {
ASSERT(size, <=, _size);
#ifdef ENABLE_USE_MMAP
try {
auto found_range = source->locate(_offset+offset, size);
auto first_part_containing_it = found_range.first;
if (++first_part_containing_it != found_range.second) {
throw MMapException();
}

// The range is in only one part
auto range = found_range.first->first;
auto part = found_range.first->second;
auto logical_local_offset = offset + _offset - range.min;
ASSERT(size, <=, part->size());
int fd = part->fhandle().getNativeHandle();
auto physical_local_offset = logical_local_offset + part->offset();
return Buffer::makeBuffer(makeMmappedBuffer(fd, physical_local_offset, size), size);
return get_mmap_buffer(offset, size);
} catch(MMapException& e)
#endif
{
Expand All @@ -210,6 +197,25 @@ const Buffer MultiPartFileReader::get_buffer(offset_t offset, zsize_t size) cons
}
}

#ifdef ENABLE_USE_MMAP
const Buffer MultiPartFileReader::get_mmap_buffer(offset_t offset, zsize_t size) const {
auto found_range = source->locate(_offset + offset, size);
auto first_part_containing_it = found_range.first;
if (++first_part_containing_it != found_range.second) {
throw MMapException();
}

// The range is in only one part
auto range = found_range.first->first;
auto part = found_range.first->second;
auto logical_local_offset = offset + _offset - range.min;
ASSERT(size, <=, part->size());
int fd = part->fhandle().getNativeHandle();
auto physical_local_offset = logical_local_offset + part->offset();
return Buffer::makeBuffer(makeMmappedBuffer(fd, physical_local_offset, size), size);
}
#endif

bool Reader::can_read(offset_t offset, zsize_t size) const
{
return (offset.v <= this->size().v && (offset.v+size.v) <= this->size().v);
Expand Down Expand Up @@ -273,27 +279,13 @@ void FileReader::read(char* dest, offset_t offset, zsize_t size) const
};
}

const Buffer FileReader::get_buffer(offset_t offset, zsize_t size) const
{
ASSERT(size, <=, _size);
#ifdef ENABLE_USE_MMAP
try {
auto local_offset = offset + _offset;
int fd = _fhandle->getNativeHandle();
return Buffer::makeBuffer(makeMmappedBuffer(fd, local_offset, size), size);
} catch(MMapException& e)
#endif
{
// We cannot do the mmap, for several possible reasons:
// - Mmap offset is too big (>4GB on 32 bits)
// - We are on Windows.
// We will have to do some memory copies :/
// [TODO] Use Windows equivalent for mmap.
auto ret_buffer = Buffer::makeBuffer(size);
read(const_cast<char*>(ret_buffer.data()), offset, size);
return ret_buffer;
}
const Buffer FileReader::get_mmap_buffer(offset_t offset, zsize_t size) const {
auto local_offset = offset + _offset;
int fd = _fhandle->getNativeHandle();
return Buffer::makeBuffer(makeMmappedBuffer(fd, local_offset, size), size);
}
#endif

std::unique_ptr<const Reader>
FileReader::sub_reader(offset_t offset, zsize_t size) const
Expand Down
20 changes: 16 additions & 4 deletions src/file_reader.h
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,12 @@ class BaseFileReader : public Reader {
zsize_t size() const { return _size; };
offset_t offset() const { return _offset; };

#ifdef ENABLE_USE_MMAP
virtual const Buffer get_mmap_buffer(offset_t offset,
zsize_t size) const = 0;
#endif
const Buffer get_buffer(offset_t offset, zsize_t size) const;

protected: // data
offset_t _offset;
zsize_t _size;
Expand All @@ -50,8 +56,11 @@ class FileReader : public BaseFileReader {
~FileReader() = default;

char read(offset_t offset) const;
void read(char* dest, offset_t offset, zsize_t size) const;
const Buffer get_buffer(offset_t offset, zsize_t size) const;
void read(char *dest, offset_t offset, zsize_t size) const;

#ifdef ENABLE_USE_MMAP
const Buffer get_mmap_buffer(offset_t offset, zsize_t size) const;
#endif

std::unique_ptr<const Reader> sub_reader(offset_t offset, zsize_t size) const;

Expand All @@ -68,8 +77,11 @@ class MultiPartFileReader : public BaseFileReader {
~MultiPartFileReader() {};

char read(offset_t offset) const;
void read(char* dest, offset_t offset, zsize_t size) const;
const Buffer get_buffer(offset_t offset, zsize_t size) const;
void read(char *dest, offset_t offset, zsize_t size) const;

#ifdef ENABLE_USE_MMAP
const Buffer get_mmap_buffer(offset_t offset, zsize_t size) const;
#endif

std::unique_ptr<const Reader> sub_reader(offset_t offset, zsize_t size) const;

Expand Down

0 comments on commit 14b2306

Please sign in to comment.