maokaihe/mini_lsm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

implement mvcc compaction + snapshot

Browse Source 
Signed-off-by: Alex Chi <iskyzh@gmail.com>
This commit is contained in:
Alex Chi
2024-01-26 16:52:37 +08:00
parent 14c3be390c
commit 6025bb8dca
17 changed files with 300 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
mini-lsm-mvcc/src/compact.rs
View File

@@ -115,20 +115,45 @@ impl LsmStorageInner {
fn compact_generate_sst_from_iter(
&self,
mut iter: impl for<'a> StorageIterator<KeyType<'a> = KeySlice<'a>>,
_compact_to_bottom_level: bool,
compact_to_bottom_level: bool,
) -> Result<Vec<Arc<SsTable>>> {
let mut builder = None;
let mut new_sst = Vec::new();
let watermark = self.mvcc().watermark();
let mut last_key = Vec::<u8>::new();
let mut first_key_below_watermark = false;
while iter.is_valid() {
if builder.is_none() {
builder = Some(SsTableBuilder::new(self.options.block_size));
}
let builder_inner = builder.as_mut().unwrap();
builder_inner.add(iter.key(), iter.value());
let same_as_last_key = iter.key().key_ref() == last_key;
if !same_as_last_key {
first_key_below_watermark = true;
}
if compact_to_bottom_level
&& !same_as_last_key
&& iter.key().ts() <= watermark
&& iter.value().is_empty()
{
last_key.clear();
last_key.extend(iter.key().key_ref());
iter.next()?;
first_key_below_watermark = false;
continue;
}
if same_as_last_key && iter.key().ts() < watermark {
if !first_key_below_watermark {
iter.next()?;
continue;
}
first_key_below_watermark = false;
}
let builder_inner = builder.as_mut().unwrap();
builder_inner.add(iter.key(), iter.value());
if builder_inner.estimated_size() >= self.options.target_sst_size && !same_as_last_key {
let sst_id = self.next_sst_id();
@@ -145,6 +170,7 @@ impl LsmStorageInner {
last_key.clear();
last_key.extend(iter.key().key_ref());
}
iter.next()?;
}
if let Some(builder) = builder {
@@ -343,9 +369,7 @@ impl LsmStorageInner {
*state = Arc::new(snapshot);
drop(state);
self.sync_dir()?;
self.manifest
.as_ref()
.unwrap()
self.manifest()
.add_record(&state_lock, ManifestRecord::Compaction(task, new_sst_ids))?;
ssts_to_remove
};