maokaihe/mini_lsm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add compaction tests and fix bugs in compaction

Browse Source 
Signed-off-by: Alex Chi <iskyzh@gmail.com>
This commit is contained in:
Alex Chi
2024-01-25 15:25:23 +08:00
parent 971d0b1c81
commit 8dbaf54e38
23 changed files with 379 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
mini-lsm-mvcc/src/compact.rs
View File

@@ -98,6 +98,7 @@ impl CompactionController {
}
}
#[derive(Debug, Clone)]
pub enum CompactionOptions {
/// Leveled compaction with partial compaction + dynamic level support (= RocksDB's Leveled
/// Compaction)
@@ -309,25 +310,25 @@ impl LsmStorageInner {
};
println!("running compaction task: {:?}", task);
let sstables = self.compact(&task)?;
let files_added = sstables.len();
let output = sstables.iter().map(|x| x.sst_id()).collect::<Vec<_>>();
let ssts_to_remove = {
let state_lock = self.state_lock.lock();
let (mut snapshot, files_to_remove) = self
.compaction_controller
.apply_compaction_result(&self.state.read(), &task, &output);
let mut ssts_to_remove = Vec::with_capacity(files_to_remove.len());
for file_to_remove in &files_to_remove {
let result = snapshot.sstables.remove(file_to_remove);
assert!(result.is_some(), "cannot remove {}.sst", file_to_remove);
ssts_to_remove.push(result.unwrap());
}
let mut snapshot = self.state.read().as_ref().clone();
let mut new_sst_ids = Vec::new();
for file_to_add in sstables {
new_sst_ids.push(file_to_add.sst_id());
let result = snapshot.sstables.insert(file_to_add.sst_id(), file_to_add);
assert!(result.is_none());
}
let (mut snapshot, files_to_remove) = self
.compaction_controller
.apply_compaction_result(&snapshot, &task, &output);
let mut ssts_to_remove = Vec::with_capacity(files_to_remove.len());
for file_to_remove in &files_to_remove {
let result = snapshot.sstables.remove(file_to_remove);
assert!(result.is_some(), "cannot remove {}.sst", file_to_remove);
ssts_to_remove.push(result.unwrap());
}
let mut state = self.state.write();
*state = Arc::new(snapshot);
drop(state);
@@ -339,9 +340,10 @@ impl LsmStorageInner {
ssts_to_remove
};
println!(
"compaction finished: {} files removed, {} files added",
"compaction finished: {} files removed, {} files added, output={:?}",
ssts_to_remove.len(),
files_added
output.len(),
output
);
for sst in ssts_to_remove {
std::fs::remove_file(self.path_of_sst(sst.sst_id()))?;