maokaihe/mini_lsm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

consider merge width for tiered trigger + mvcc compaction

Browse Source 
Signed-off-by: Alex Chi <iskyzh@gmail.com>
This commit is contained in:
Alex Chi
2024-01-25 23:38:26 +08:00
parent 218c73f384
commit 63429b50d5
7 changed files with 153 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
mini-lsm/src/debug.rs
View File

@@ -1,8 +1,8 @@
use crate::lsm_storage::MiniLsm;
use crate::lsm_storage::{LsmStorageInner, MiniLsm};
impl MiniLsm {
impl LsmStorageInner {
pub fn dump_structure(&self) {
let snapshot = self.inner.state.read();
let snapshot = self.state.read();
if !snapshot.l0_sstables.is_empty() {
println!(
"L0 ({}): {:?}",
@@ -15,3 +15,9 @@ impl MiniLsm {
}
}
}
impl MiniLsm {
pub fn dump_structure(&self) {
self.inner.dump_structure()
}
}

24
mini-lsm/src/tests/harness.rs
View File

@@ -288,7 +288,7 @@ pub fn check_compaction_ratio(storage: Arc<MiniLsm>) {
num_tiers,
max_size_amplification_percent,
size_ratio,
..
min_merge_width,
}) => {
let size_ratio_trigger = (100.0 + size_ratio as f64) / 100.0;
assert_eq!(l0_sst_num, 0);
@@ -296,20 +296,22 @@ pub fn check_compaction_ratio(storage: Arc<MiniLsm>) {
let mut sum_size = level_size[0];
for idx in 1..level_size.len() {
let this_size = level_size[idx];
assert!(
sum_size as f64 / this_size as f64 <= size_ratio_trigger,
"sum(⬆️L{})/L{}, {}/{}>{}",
state.levels[idx - 1].0,
state.levels[idx].0,
sum_size,
this_size,
size_ratio_trigger
);
if level_size.len() > min_merge_width {
assert!(
sum_size as f64 / this_size as f64 <= size_ratio_trigger,
"violation of size ratio: sum(⬆️L{})/L{}, {}/{}>{}",
state.levels[idx - 1].0,
state.levels[idx].0,
sum_size,
this_size,
size_ratio_trigger
);
}
if idx + 1 == level_size.len() {
assert!(
sum_size as f64 / this_size as f64
<= max_size_amplification_percent as f64 / 100.0,
"sum(⬆️L{})/L{}, {}/{}>{}%",
"violation of space amp: sum(⬆️L{})/L{}, {}/{}>{}%",
state.levels[idx - 1].0,
state.levels[idx].0,
sum_size,

119
mini-lsm/src/tests/week2_day1.rs
View File

@@ -9,7 +9,7 @@ use crate::{
iterators::{
concat_iterator::SstConcatIterator, merge_iterator::MergeIterator, StorageIterator,
},
key::KeySlice,
key::{KeySlice, TS_ENABLED},
lsm_storage::{LsmStorageInner, LsmStorageOptions, LsmStorageState},
table::{SsTable, SsTableBuilder, SsTableIterator},
};
@@ -37,6 +37,7 @@ fn construct_merge_iterator_over_storage(
#[test]
fn test_task1_full_compaction() {
// We do not use LSM iterator in this test because it's implemented as part of task 3
let dir = tempdir().unwrap();
let storage = LsmStorageInner::open(&dir, LsmStorageOptions::default_for_week1_test()).unwrap();
storage.put(b"0", b"v1").unwrap();
@@ -50,45 +51,107 @@ fn test_task1_full_compaction() {
sync(&storage);
assert_eq!(storage.state.read().l0_sstables.len(), 3);
let mut iter = construct_merge_iterator_over_storage(&storage.state.read());
check_iter_result_by_key(
&mut iter,
vec![
(Bytes::from_static(b"0"), Bytes::from_static(b"")),
(Bytes::from_static(b"1"), Bytes::from_static(b"v2")),
(Bytes::from_static(b"2"), Bytes::from_static(b"")),
],
);
if TS_ENABLED {
check_iter_result_by_key(
&mut iter,
vec![
(Bytes::from_static(b"0"), Bytes::from_static(b"")),
(Bytes::from_static(b"0"), Bytes::from_static(b"v2")),
(Bytes::from_static(b"0"), Bytes::from_static(b"v1")),
(Bytes::from_static(b"1"), Bytes::from_static(b"v2")),
(Bytes::from_static(b"2"), Bytes::from_static(b"")),
(Bytes::from_static(b"2"), Bytes::from_static(b"v2")),
],
);
} else {
check_iter_result_by_key(
&mut iter,
vec![
(Bytes::from_static(b"0"), Bytes::from_static(b"")),
(Bytes::from_static(b"1"), Bytes::from_static(b"v2")),
(Bytes::from_static(b"2"), Bytes::from_static(b"")),
],
);
}
storage.force_full_compaction().unwrap();
assert!(storage.state.read().l0_sstables.is_empty());
let mut iter = construct_merge_iterator_over_storage(&storage.state.read());
check_iter_result_by_key(
&mut iter,
vec![(Bytes::from_static(b"1"), Bytes::from_static(b"v2"))],
);
if TS_ENABLED {
check_iter_result_by_key(
&mut iter,
vec![
(Bytes::from_static(b"0"), Bytes::from_static(b"")),
(Bytes::from_static(b"0"), Bytes::from_static(b"v2")),
(Bytes::from_static(b"0"), Bytes::from_static(b"v1")),
(Bytes::from_static(b"1"), Bytes::from_static(b"v2")),
(Bytes::from_static(b"2"), Bytes::from_static(b"")),
(Bytes::from_static(b"2"), Bytes::from_static(b"v2")),
],
);
} else {
check_iter_result_by_key(
&mut iter,
vec![(Bytes::from_static(b"1"), Bytes::from_static(b"v2"))],
);
}
storage.put(b"0", b"v3").unwrap();
storage.put(b"2", b"v3").unwrap();
sync(&storage);
storage.delete(b"1").unwrap();
sync(&storage);
let mut iter = construct_merge_iterator_over_storage(&storage.state.read());
check_iter_result_by_key(
&mut iter,
vec![
(Bytes::from_static(b"0"), Bytes::from_static(b"v3")),
(Bytes::from_static(b"1"), Bytes::from_static(b"")),
(Bytes::from_static(b"2"), Bytes::from_static(b"v3")),
],
);
if TS_ENABLED {
check_iter_result_by_key(
&mut iter,
vec![
(Bytes::from_static(b"0"), Bytes::from_static(b"v3")),
(Bytes::from_static(b"0"), Bytes::from_static(b"")),
(Bytes::from_static(b"0"), Bytes::from_static(b"v2")),
(Bytes::from_static(b"0"), Bytes::from_static(b"v1")),
(Bytes::from_static(b"1"), Bytes::from_static(b"")),
(Bytes::from_static(b"1"), Bytes::from_static(b"v2")),
(Bytes::from_static(b"2"), Bytes::from_static(b"v3")),
(Bytes::from_static(b"2"), Bytes::from_static(b"")),
(Bytes::from_static(b"2"), Bytes::from_static(b"v2")),
],
);
} else {
check_iter_result_by_key(
&mut iter,
vec![
(Bytes::from_static(b"0"), Bytes::from_static(b"v3")),
(Bytes::from_static(b"1"), Bytes::from_static(b"")),
(Bytes::from_static(b"2"), Bytes::from_static(b"v3")),
],
);
}
storage.force_full_compaction().unwrap();
assert!(storage.state.read().l0_sstables.is_empty());
let mut iter = construct_merge_iterator_over_storage(&storage.state.read());
check_iter_result_by_key(
&mut iter,
vec![
(Bytes::from_static(b"0"), Bytes::from_static(b"v3")),
(Bytes::from_static(b"2"), Bytes::from_static(b"v3")),
],
);
if TS_ENABLED {
check_iter_result_by_key(
&mut iter,
vec![
(Bytes::from_static(b"0"), Bytes::from_static(b"v3")),
(Bytes::from_static(b"0"), Bytes::from_static(b"")),
(Bytes::from_static(b"0"), Bytes::from_static(b"v2")),
(Bytes::from_static(b"0"), Bytes::from_static(b"v1")),
(Bytes::from_static(b"1"), Bytes::from_static(b"")),
(Bytes::from_static(b"1"), Bytes::from_static(b"v2")),
(Bytes::from_static(b"2"), Bytes::from_static(b"v3")),
(Bytes::from_static(b"2"), Bytes::from_static(b"")),
(Bytes::from_static(b"2"), Bytes::from_static(b"v2")),
],
);
} else {
check_iter_result_by_key(
&mut iter,
vec![
(Bytes::from_static(b"0"), Bytes::from_static(b"v3")),
(Bytes::from_static(b"2"), Bytes::from_static(b"v3")),
],
);
}
}
fn generate_concat_sst(