From 47fefafe1bab0a471c3a3e4451b340204991da99 Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Sun, 20 Oct 2024 22:08:33 -0700 Subject: [PATCH 1/3] Fix propagating ingest level Signed-off-by: Yang Zhang --- db/db_impl/db_impl.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/db/db_impl/db_impl.cc b/db/db_impl/db_impl.cc index 417304e1f17..64a68bb3fa6 100644 --- a/db/db_impl/db_impl.cc +++ b/db/db_impl/db_impl.cc @@ -6458,6 +6458,7 @@ void DBImpl::NotifyOnExternalFileIngested( info.internal_file_path = f.internal_file_path; info.global_seqno = f.assigned_seqno; info.table_properties = f.table_properties; + info.picked_level = f.picked_level; for (auto listener : immutable_db_options_.listeners) { listener->OnExternalFileIngested(this, info); } From 43ee15f40e36a650d1a8f8cad7c18cc07d11b5cc Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Tue, 22 Oct 2024 16:08:53 -0700 Subject: [PATCH 2/3] Revert ingest check Signed-off-by: Yang Zhang --- db/external_sst_file_ingestion_job.cc | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/db/external_sst_file_ingestion_job.cc b/db/external_sst_file_ingestion_job.cc index a4a19471456..3fd9ba9a10d 100644 --- a/db/external_sst_file_ingestion_job.cc +++ b/db/external_sst_file_ingestion_job.cc @@ -3,7 +3,6 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). - #include "db/external_sst_file_ingestion_job.h" #include @@ -912,16 +911,19 @@ Status ExternalSstFileIngestionJob::AssignLevelAndSeqnoForIngestedFile( if (lvl > 0 && lvl < vstorage->base_level()) { continue; } - if (cfd_->RangeOverlapWithCompaction( - file_to_ingest->smallest_internal_key.user_key(), - file_to_ingest->largest_internal_key.user_key(), lvl)) { - // We must use L0 or any level higher than `lvl` to be able to overwrite - // the compaction output keys that we overlap with in this level, We also - // need to assign this file a seqno to overwrite the compaction output - // keys in level `lvl` - overlap_with_db = true; - break; - } else if (vstorage->NumLevelFiles(lvl) > 0) { + // if (cfd_->RangeOverlapWithCompaction( + // file_to_ingest->smallest_internal_key.user_key(), + // file_to_ingest->largest_internal_key.user_key(), lvl)) { + // // We must use L0 or any level higher than `lvl` to be able to + // overwrite + // // the compaction output keys that we overlap with in this level, We + // also + // // need to assign this file a seqno to overwrite the compaction output + // // keys in level `lvl` + // overlap_with_db = true; + // break; + // } else if (vstorage->NumLevelFiles(lvl) > 0) { + if (vstorage->NumLevelFiles(lvl) > 0) { bool overlap_with_level = false; status = sv->current->OverlapWithLevelIterator( ro, env_options_, file_to_ingest->smallest_internal_key.user_key(), From a9ecdce5dcc122582d2106d04854108b2262aae4 Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Wed, 23 Oct 2024 13:39:32 -0700 Subject: [PATCH 3/3] Distinguish overlap w/ compactino from w/ db Signed-off-by: Yang Zhang --- db/external_sst_file_ingestion_job.cc | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/db/external_sst_file_ingestion_job.cc b/db/external_sst_file_ingestion_job.cc index 3fd9ba9a10d..3747aed252f 100644 --- a/db/external_sst_file_ingestion_job.cc +++ b/db/external_sst_file_ingestion_job.cc @@ -911,18 +911,6 @@ Status ExternalSstFileIngestionJob::AssignLevelAndSeqnoForIngestedFile( if (lvl > 0 && lvl < vstorage->base_level()) { continue; } - // if (cfd_->RangeOverlapWithCompaction( - // file_to_ingest->smallest_internal_key.user_key(), - // file_to_ingest->largest_internal_key.user_key(), lvl)) { - // // We must use L0 or any level higher than `lvl` to be able to - // overwrite - // // the compaction output keys that we overlap with in this level, We - // also - // // need to assign this file a seqno to overwrite the compaction output - // // keys in level `lvl` - // overlap_with_db = true; - // break; - // } else if (vstorage->NumLevelFiles(lvl) > 0) { if (vstorage->NumLevelFiles(lvl) > 0) { bool overlap_with_level = false; status = sv->current->OverlapWithLevelIterator( @@ -1125,6 +1113,13 @@ bool ExternalSstFileIngestionJob::IngestedFileFitInLevel( return false; } + if (cfd_->RangeOverlapWithCompaction(file_smallest_user_key, + file_largest_user_key, level)) { + // File overlap with a running compaction output that will be stored + // in this level, we cannot add this file to this level + return false; + } + // File did not overlap with level files, nor compaction output return true; }