maokaihe/mini_lsm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add key abstraction and prepare for MVCC (#28)

Browse Source 
* add key abstraction and prepare for MVCC

Signed-off-by: Alex Chi <iskyzh@gmail.com>

* a little bit type exercise

Signed-off-by: Alex Chi <iskyzh@gmail.com>

* refactor tests

Signed-off-by: Alex Chi <iskyzh@gmail.com>

* fix clippy warnings

Signed-off-by: Alex Chi <iskyzh@gmail.com>

* refactor starter code

Signed-off-by: Alex Chi <iskyzh@gmail.com>

* final touch docs

Signed-off-by: Alex Chi <iskyzh@gmail.com>

---------

Signed-off-by: Alex Chi <iskyzh@gmail.com>
This commit is contained in:
Alex Chi Z
2024-01-25 10:59:08 +08:00
committed by GitHub
parent 2fefe3e505
commit a3a92359e1
42 changed files with 824 additions and 281 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
mini-lsm/src/lsm_storage.rs
View File

@@ -18,6 +18,7 @@ use crate::iterators::concat_iterator::SstConcatIterator;
use crate::iterators::merge_iterator::MergeIterator;
use crate::iterators::two_merge_iterator::TwoMergeIterator;
use crate::iterators::StorageIterator;
use crate::key::KeySlice;
use crate::lsm_iterator::{FusedIterator, LsmIterator};
use crate::manifest::{Manifest, ManifestRecord};
use crate::mem_table::{map_bound, MemTable};
@@ -98,23 +99,23 @@ impl LsmStorageOptions {
fn range_overlap(
user_begin: Bound<&[u8]>,
user_end: Bound<&[u8]>,
table_begin: &[u8],
table_end: &[u8],
table_begin: KeySlice,
table_end: KeySlice,
) -> bool {
match user_end {
Bound::Excluded(key) if key <= table_begin => {
Bound::Excluded(key) if key <= table_begin.raw_ref() => {
return false;
}
Bound::Included(key) if key < table_begin => {
Bound::Included(key) if key < table_begin.raw_ref() => {
return false;
}
_ => {}
}
match user_begin {
Bound::Excluded(key) if key >= table_end => {
Bound::Excluded(key) if key >= table_end.raw_ref() => {
return false;
}
Bound::Included(key) if key > table_end => {
Bound::Included(key) if key > table_end.raw_ref() => {
return false;
}
_ => {}
@@ -122,8 +123,8 @@ fn range_overlap(
true
}
fn key_within(user_key: &[u8], table_begin: &[u8], table_end: &[u8]) -> bool {
table_begin <= user_key && user_key <= table_end
fn key_within(user_key: &[u8], table_begin: KeySlice, table_end: KeySlice) -> bool {
table_begin.raw_ref() <= user_key && user_key <= table_end.raw_ref()
}
/// The storage interface of the LSM tree.
@@ -425,7 +426,11 @@ impl LsmStorageInner {
let mut l0_iters = Vec::with_capacity(snapshot.l0_sstables.len());
let keep_table = |key: &[u8], table: &SsTable| {
if key_within(key, table.first_key(), table.last_key()) {
if key_within(
key,
table.first_key().as_key_slice(),
table.last_key().as_key_slice(),
) {
if let Some(bloom) = &table.bloom {
if bloom.may_contain(farmhash::fingerprint32(key)) {
return true;
@@ -441,7 +446,8 @@ impl LsmStorageInner {
let table = snapshot.sstables[table].clone();
if keep_table(key, &table) {
l0_iters.push(Box::new(SsTableIterator::create_and_seek_to_key(
table, key,
table,
KeySlice::from_slice(key),
)?));
}
}
@@ -455,13 +461,14 @@ impl LsmStorageInner {
level_ssts.push(table);
}
}
let level_iter = SstConcatIterator::create_and_seek_to_key(level_ssts, key)?;
let level_iter =
SstConcatIterator::create_and_seek_to_key(level_ssts, KeySlice::from_slice(key))?;
level_iters.push(Box::new(level_iter));
}
let iter = TwoMergeIterator::create(l0_iter, MergeIterator::create(level_iters))?;
if iter.is_valid() && iter.key() == key && !iter.value().is_empty() {
if iter.is_valid() && iter.key().raw_ref() == key && !iter.value().is_empty() {
return Ok(Some(Bytes::copy_from_slice(iter.value())));
}
Ok(None)
@@ -653,12 +660,22 @@ impl LsmStorageInner {
let mut table_iters = Vec::with_capacity(snapshot.l0_sstables.len());
for table_id in snapshot.l0_sstables.iter() {
let table = snapshot.sstables[table_id].clone();
if range_overlap(lower, upper, table.first_key(), table.last_key()) {
if range_overlap(
lower,
upper,
table.first_key().as_key_slice(),
table.last_key().as_key_slice(),
) {
let iter = match lower {
Bound::Included(key) => SsTableIterator::create_and_seek_to_key(table, key)?,
Bound::Included(key) => {
SsTableIterator::create_and_seek_to_key(table, KeySlice::from_slice(key))?
}
Bound::Excluded(key) => {
let mut iter = SsTableIterator::create_and_seek_to_key(table, key)?;
if iter.is_valid() && iter.key() == key {
let mut iter = SsTableIterator::create_and_seek_to_key(
table,
KeySlice::from_slice(key),
)?;
if iter.is_valid() && iter.key().raw_ref() == key {
iter.next()?;
}
iter
@@ -676,16 +693,27 @@ impl LsmStorageInner {
let mut level_ssts = Vec::with_capacity(level_sst_ids.len());
for table in level_sst_ids {
let table = snapshot.sstables[table].clone();
if range_overlap(lower, upper, table.first_key(), table.last_key()) {
if range_overlap(
lower,
upper,
table.first_key().as_key_slice(),
table.last_key().as_key_slice(),
) {
level_ssts.push(table);
}
}
let level_iter = match lower {
Bound::Included(key) => SstConcatIterator::create_and_seek_to_key(level_ssts, key)?,
Bound::Included(key) => SstConcatIterator::create_and_seek_to_key(
level_ssts,
KeySlice::from_slice(key),
)?,
Bound::Excluded(key) => {
let mut iter = SstConcatIterator::create_and_seek_to_key(level_ssts, key)?;
if iter.is_valid() && iter.key() == key {
let mut iter = SstConcatIterator::create_and_seek_to_key(
level_ssts,
KeySlice::from_slice(key),
)?;
if iter.is_valid() && iter.key().raw_ref() == key {
iter.next()?;
}
iter