maokaihe/mini_lsm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

use symlink when possible

Browse Source 
Signed-off-by: Alex Chi <iskyzh@gmail.com>
This commit is contained in:
Alex Chi
2024-01-28 15:17:53 +08:00
parent bfeef9d25d
commit 5f1b10b03b
21 changed files with 250 additions and 598 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
mini-lsm/src/debug.rs
View File

@@ -1,23 +0,0 @@
use crate::lsm_storage::{LsmStorageInner, MiniLsm};
impl LsmStorageInner {
pub fn dump_structure(&self) {
let snapshot = self.state.read();
if !snapshot.l0_sstables.is_empty() {
println!(
"L0 ({}): {:?}",
snapshot.l0_sstables.len(),
snapshot.l0_sstables,
);
}
for (level, files) in &snapshot.levels {
println!("L{level} ({}): {:?}", files.len(), files);
}
}
}
impl MiniLsm {
pub fn dump_structure(&self) {
self.inner.dump_structure()
}
}

1
mini-lsm/src/debug.rs Symbolic link
View File

@@ -0,0 +1 @@
../../mini-lsm-starter/src/debug.rs

137
mini-lsm/src/iterators/tests/merge_iterator_test.rs
View File

@@ -1,137 +0,0 @@
use super::*;
use crate::iterators::merge_iterator::MergeIterator;
fn as_bytes(x: &[u8]) -> Bytes {
Bytes::copy_from_slice(x)
}
fn check_iter_result(iter: impl StorageIterator, expected: Vec<(Bytes, Bytes)>) {
let mut iter = iter;
for (k, v) in expected {
assert!(iter.is_valid());
assert_eq!(
k,
iter.key(),
"expected key: {:?}, actual key: {:?}",
k,
as_bytes(iter.key()),
);
assert_eq!(
v,
iter.value(),
"expected value: {:?}, actual value: {:?}",
v,
as_bytes(iter.value()),
);
iter.next().unwrap();
}
assert!(!iter.is_valid());
}
#[test]
fn test_merge_1() {
let i1 = MockIterator::new(vec![
(Bytes::from("a"), Bytes::from("1.1")),
(Bytes::from("b"), Bytes::from("2.1")),
(Bytes::from("c"), Bytes::from("3.1")),
]);
let i2 = MockIterator::new(vec![
(Bytes::from("a"), Bytes::from("1.2")),
(Bytes::from("b"), Bytes::from("2.2")),
(Bytes::from("c"), Bytes::from("3.2")),
(Bytes::from("d"), Bytes::from("4.2")),
]);
let i3 = MockIterator::new(vec![
(Bytes::from("b"), Bytes::from("2.3")),
(Bytes::from("c"), Bytes::from("3.3")),
(Bytes::from("d"), Bytes::from("4.3")),
]);
let iter = MergeIterator::create(vec![
Box::new(i1.clone()),
Box::new(i2.clone()),
Box::new(i3.clone()),
]);
check_iter_result(
iter,
vec![
(Bytes::from("a"), Bytes::from("1.1")),
(Bytes::from("b"), Bytes::from("2.1")),
(Bytes::from("c"), Bytes::from("3.1")),
(Bytes::from("d"), Bytes::from("4.2")),
],
);
let iter = MergeIterator::create(vec![Box::new(i3), Box::new(i1), Box::new(i2)]);
check_iter_result(
iter,
vec![
(Bytes::from("a"), Bytes::from("1.1")),
(Bytes::from("b"), Bytes::from("2.3")),
(Bytes::from("c"), Bytes::from("3.3")),
(Bytes::from("d"), Bytes::from("4.3")),
],
);
}
#[test]
fn test_merge_2() {
let i1 = MockIterator::new(vec![
(Bytes::from("a"), Bytes::from("1.1")),
(Bytes::from("b"), Bytes::from("2.1")),
(Bytes::from("c"), Bytes::from("3.1")),
]);
let i2 = MockIterator::new(vec![
(Bytes::from("d"), Bytes::from("1.2")),
(Bytes::from("e"), Bytes::from("2.2")),
(Bytes::from("f"), Bytes::from("3.2")),
(Bytes::from("g"), Bytes::from("4.2")),
]);
let i3 = MockIterator::new(vec![
(Bytes::from("h"), Bytes::from("1.3")),
(Bytes::from("i"), Bytes::from("2.3")),
(Bytes::from("j"), Bytes::from("3.3")),
(Bytes::from("k"), Bytes::from("4.3")),
]);
let i4 = MockIterator::new(vec![]);
let result = vec![
(Bytes::from("a"), Bytes::from("1.1")),
(Bytes::from("b"), Bytes::from("2.1")),
(Bytes::from("c"), Bytes::from("3.1")),
(Bytes::from("d"), Bytes::from("1.2")),
(Bytes::from("e"), Bytes::from("2.2")),
(Bytes::from("f"), Bytes::from("3.2")),
(Bytes::from("g"), Bytes::from("4.2")),
(Bytes::from("h"), Bytes::from("1.3")),
(Bytes::from("i"), Bytes::from("2.3")),
(Bytes::from("j"), Bytes::from("3.3")),
(Bytes::from("k"), Bytes::from("4.3")),
];
let iter = MergeIterator::create(vec![
Box::new(i1.clone()),
Box::new(i2.clone()),
Box::new(i3.clone()),
Box::new(i4.clone()),
]);
check_iter_result(iter, result.clone());
let iter = MergeIterator::create(vec![
Box::new(i2.clone()),
Box::new(i4.clone()),
Box::new(i3.clone()),
Box::new(i1.clone()),
]);
check_iter_result(iter, result.clone());
let iter = MergeIterator::create(vec![Box::new(i4), Box::new(i3), Box::new(i2), Box::new(i1)]);
check_iter_result(iter, result);
}
#[test]
fn test_merge_empty() {
let iter = MergeIterator::<MockIterator>::create(vec![]);
check_iter_result(iter, vec![]);
}

129
mini-lsm/src/iterators/tests/two_merge_iterator_test.rs
View File

@@ -1,129 +0,0 @@
use super::*;
use crate::iterators::two_merge_iterator::TwoMergeIterator;
fn check_iter_result(iter: impl StorageIterator, expected: Vec<(Bytes, Bytes)>) {
let mut iter = iter;
for (k, v) in expected {
assert!(iter.is_valid());
assert_eq!(iter.key(), k.as_ref());
assert_eq!(iter.value(), v.as_ref());
iter.next().unwrap();
}
assert!(!iter.is_valid());
}
#[test]
fn test_merge_1() {
let i1 = MockIterator::new(vec![
(Bytes::from("a"), Bytes::from("1.1")),
(Bytes::from("b"), Bytes::from("2.1")),
(Bytes::from("c"), Bytes::from("3.1")),
]);
let i2 = MockIterator::new(vec![
(Bytes::from("a"), Bytes::from("1.2")),
(Bytes::from("b"), Bytes::from("2.2")),
(Bytes::from("c"), Bytes::from("3.2")),
(Bytes::from("d"), Bytes::from("4.2")),
]);
let iter = TwoMergeIterator::create(i1, i2).unwrap();
check_iter_result(
iter,
vec![
(Bytes::from("a"), Bytes::from("1.1")),
(Bytes::from("b"), Bytes::from("2.1")),
(Bytes::from("c"), Bytes::from("3.1")),
(Bytes::from("d"), Bytes::from("4.2")),
],
)
}
#[test]
fn test_merge_2() {
let i2 = MockIterator::new(vec![
(Bytes::from("a"), Bytes::from("1.1")),
(Bytes::from("b"), Bytes::from("2.1")),
(Bytes::from("c"), Bytes::from("3.1")),
]);
let i1 = MockIterator::new(vec![
(Bytes::from("a"), Bytes::from("1.2")),
(Bytes::from("b"), Bytes::from("2.2")),
(Bytes::from("c"), Bytes::from("3.2")),
(Bytes::from("d"), Bytes::from("4.2")),
]);
let iter = TwoMergeIterator::create(i1, i2).unwrap();
check_iter_result(
iter,
vec![
(Bytes::from("a"), Bytes::from("1.2")),
(Bytes::from("b"), Bytes::from("2.2")),
(Bytes::from("c"), Bytes::from("3.2")),
(Bytes::from("d"), Bytes::from("4.2")),
],
)
}
#[test]
fn test_merge_3() {
let i2 = MockIterator::new(vec![
(Bytes::from("a"), Bytes::from("1.1")),
(Bytes::from("b"), Bytes::from("2.1")),
(Bytes::from("c"), Bytes::from("3.1")),
]);
let i1 = MockIterator::new(vec![
(Bytes::from("b"), Bytes::from("2.2")),
(Bytes::from("c"), Bytes::from("3.2")),
(Bytes::from("d"), Bytes::from("4.2")),
]);
let iter = TwoMergeIterator::create(i1, i2).unwrap();
check_iter_result(
iter,
vec![
(Bytes::from("a"), Bytes::from("1.1")),
(Bytes::from("b"), Bytes::from("2.2")),
(Bytes::from("c"), Bytes::from("3.2")),
(Bytes::from("d"), Bytes::from("4.2")),
],
)
}
#[test]
fn test_merge_4() {
let i2 = MockIterator::new(vec![]);
let i1 = MockIterator::new(vec![
(Bytes::from("b"), Bytes::from("2.2")),
(Bytes::from("c"), Bytes::from("3.2")),
(Bytes::from("d"), Bytes::from("4.2")),
]);
let iter = TwoMergeIterator::create(i1, i2).unwrap();
check_iter_result(
iter,
vec![
(Bytes::from("b"), Bytes::from("2.2")),
(Bytes::from("c"), Bytes::from("3.2")),
(Bytes::from("d"), Bytes::from("4.2")),
],
);
let i1 = MockIterator::new(vec![]);
let i2 = MockIterator::new(vec![
(Bytes::from("b"), Bytes::from("2.2")),
(Bytes::from("c"), Bytes::from("3.2")),
(Bytes::from("d"), Bytes::from("4.2")),
]);
let iter = TwoMergeIterator::create(i1, i2).unwrap();
check_iter_result(
iter,
vec![
(Bytes::from("b"), Bytes::from("2.2")),
(Bytes::from("c"), Bytes::from("3.2")),
(Bytes::from("d"), Bytes::from("4.2")),
],
);
}
#[test]
fn test_merge_5() {
let i2 = MockIterator::new(vec![]);
let i1 = MockIterator::new(vec![]);
let iter = TwoMergeIterator::create(i1, i2).unwrap();
check_iter_result(iter, vec![])
}

14
mini-lsm/src/iterators/two_merge_iterator.rs
View File

@@ -1,7 +1,5 @@
use anyhow::Result;
use crate::key::KeySlice;
use super::StorageIterator;
/// Merges two iterators of different types into one. If the two iterators have the same key, only
@@ -13,8 +11,8 @@ pub struct TwoMergeIterator<A: StorageIterator, B: StorageIterator> {
}
impl<
A: 'static + for<'a> StorageIterator<KeyType<'a> = KeySlice<'a>>,
B: 'static + for<'a> StorageIterator<KeyType<'a> = KeySlice<'a>>,
A: 'static + StorageIterator,
B: 'static + for<'a> StorageIterator<KeyType<'a> = A::KeyType<'a>>,
> TwoMergeIterator<A, B>
{
fn choose_a(a: &A, b: &B) -> bool {
@@ -47,13 +45,13 @@ impl<
}
impl<
A: 'static + for<'a> StorageIterator<KeyType<'a> = KeySlice<'a>>,
B: 'static + for<'a> StorageIterator<KeyType<'a> = KeySlice<'a>>,
A: 'static + StorageIterator,
B: 'static + for<'a> StorageIterator<KeyType<'a> = A::KeyType<'a>>,
> StorageIterator for TwoMergeIterator<A, B>
{
type KeyType<'a> = KeySlice<'a>;
type KeyType<'a> = A::KeyType<'a>;
fn key(&self) -> KeySlice {
fn key(&self) -> Self::KeyType<'_> {
if self.choose_a {
self.a.key()
} else {

1
mini-lsm/src/lib.rs
View File

@@ -7,6 +7,7 @@ pub mod lsm_iterator;
pub mod lsm_storage;
pub mod manifest;
pub mod mem_table;
pub mod mvcc;
pub mod table;
pub mod wal;

4
mini-lsm/src/lsm_storage.rs
View File

@@ -22,6 +22,7 @@ use crate::key::KeySlice;
use crate::lsm_iterator::{FusedIterator, LsmIterator};
use crate::manifest::{Manifest, ManifestRecord};
use crate::mem_table::{map_bound, MemTable};
use crate::mvcc::LsmMvccInner;
use crate::table::{FileObject, SsTable, SsTableBuilder, SsTableIterator};
pub type BlockCache = moka::sync::Cache<(usize, usize), Arc<Block>>;
@@ -157,6 +158,8 @@ pub(crate) struct LsmStorageInner {
pub(crate) options: Arc<LsmStorageOptions>,
pub(crate) compaction_controller: CompactionController,
pub(crate) manifest: Option<Manifest>,
#[allow(dead_code)]
pub(crate) mvcc: Option<LsmMvccInner>,
}
/// A thin wrapper for `LsmStorageInner` and the user interface for MiniLSM.
@@ -414,6 +417,7 @@ impl LsmStorageInner {
compaction_controller,
manifest: Some(manifest),
options: options.into(),
mvcc: None,
};
storage.sync_dir()?;

1
mini-lsm/src/mvcc.rs Symbolic link
View File

@@ -0,0 +1 @@
../../mini-lsm-starter/src/mvcc.rs

1
mini-lsm/src/mvcc/txn.rs Symbolic link
View File

@@ -0,0 +1 @@
../../../mini-lsm-starter/src/mvcc/txn.rs

1
mini-lsm/src/mvcc/watermark.rs Symbolic link
View File

@@ -0,0 +1 @@
../../../mini-lsm-starter/src/mvcc/watermark.rs