maokaihe/mini_lsm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

patch memtable and add ts for wal

Browse Source 
Signed-off-by: Alex Chi <iskyzh@gmail.com>
This commit is contained in:
Alex Chi
2024-01-25 23:09:16 +08:00
parent 89acc23208
commit 218c73f384
11 changed files with 256 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
mini-lsm-mvcc/src/mem_table.rs
View File

@@ -10,7 +10,7 @@ use crossbeam_skiplist::SkipMap;
use ouroboros::self_referencing;
use crate::iterators::StorageIterator;
use crate::key::{self, KeySlice};
use crate::key::{KeyBytes, KeySlice, TS_DEFAULT};
use crate::table::SsTableBuilder;
use crate::wal::Wal;
@@ -19,7 +19,7 @@ use crate::wal::Wal;
/// An initial implementation of memtable is part of week 1, day 1. It will be incrementally implemented in other
/// chapters of week 1 and week 2.
pub struct MemTable {
map: Arc<SkipMap<Bytes, Bytes>>,
map: Arc<SkipMap<KeyBytes, Bytes>>,
wal: Option<Wal>,
id: usize,
approximate_size: Arc<AtomicUsize>,
@@ -34,6 +34,30 @@ pub(crate) fn map_bound(bound: Bound<&[u8]>) -> Bound<Bytes> {
}
}
/// Create a bound of `Bytes` from a bound of `KeySlice`.
pub(crate) fn map_key_bound(bound: Bound<KeySlice>) -> Bound<KeyBytes> {
match bound {
Bound::Included(x) => Bound::Included(KeyBytes::from_bytes_with_ts(
Bytes::copy_from_slice(x.key_ref()),
x.ts(),
)),
Bound::Excluded(x) => Bound::Excluded(KeyBytes::from_bytes_with_ts(
Bytes::copy_from_slice(x.key_ref()),
x.ts(),
)),
Bound::Unbounded => Bound::Unbounded,
}
}
/// Create a bound of `Bytes` from a bound of `KeySlice`.
pub(crate) fn map_key_bound_plus_ts(bound: Bound<&[u8]>, ts: u64) -> Bound<KeySlice> {
match bound {
Bound::Included(x) => Bound::Included(KeySlice::from_slice(x, ts)),
Bound::Excluded(x) => Bound::Excluded(KeySlice::from_slice(x, ts)),
Bound::Unbounded => Bound::Unbounded,
}
}
impl MemTable {
/// Create a new mem-table.
pub fn create(id: usize) -> Self {
@@ -66,19 +90,44 @@ impl MemTable {
})
}
/// Get a value by key.
pub fn get(&self, key: &[u8]) -> Option<Bytes> {
self.map.get(key).map(|e| e.value().clone())
/// Get a value by key. Should not be used in week 3.
pub fn get(&self, key: KeySlice) -> Option<Bytes> {
let key_bytes = KeyBytes::from_bytes_with_ts(
Bytes::from_static(unsafe { std::mem::transmute(key.key_ref()) }),
key.ts(),
);
self.map.get(&key_bytes).map(|e| e.value().clone())
}
pub fn for_testing_put_slice(&self, key: &[u8], value: &[u8]) -> Result<()> {
self.put(KeySlice::from_slice(key, TS_DEFAULT), value)
}
pub fn for_testing_get_slice(&self, key: &[u8]) -> Option<Bytes> {
self.get(KeySlice::from_slice(key, TS_DEFAULT))
}
pub fn for_testing_scan_slice(
&self,
lower: Bound<&[u8]>,
upper: Bound<&[u8]>,
) -> MemTableIterator {
self.scan(
map_key_bound_plus_ts(lower, TS_DEFAULT),
map_key_bound_plus_ts(upper, TS_DEFAULT),
)
}
/// Put a key-value pair into the mem-table.
///
/// In week 1, day 1, simply put the key-value pair into the skipmap.
/// In week 2, day 6, also flush the data to WAL.
pub fn put(&self, key: &[u8], value: &[u8]) -> Result<()> {
let estimated_size = key.len() + value.len();
self.map
.insert(Bytes::copy_from_slice(key), Bytes::copy_from_slice(value));
pub fn put(&self, key: KeySlice, value: &[u8]) -> Result<()> {
let estimated_size = key.raw_len() + value.len();
self.map.insert(
key.to_key_vec().into_key_bytes(),
Bytes::copy_from_slice(value),
);
self.approximate_size
.fetch_add(estimated_size, std::sync::atomic::Ordering::Relaxed);
if let Some(ref wal) = self.wal {
@@ -95,12 +144,12 @@ impl MemTable {
}
/// Get an iterator over a range of keys.
pub fn scan(&self, lower: Bound<&[u8]>, upper: Bound<&[u8]>) -> MemTableIterator {
let (lower, upper) = (map_bound(lower), map_bound(upper));
pub fn scan(&self, lower: Bound<KeySlice>, upper: Bound<KeySlice>) -> MemTableIterator {
let (lower, upper) = (map_key_bound(lower), map_key_bound(upper));
let mut iter = MemTableIteratorBuilder {
map: self.map.clone(),
iter_builder: |map| map.range((lower, upper)),
item: (Bytes::new(), Bytes::new()),
item: (KeyBytes::new(), Bytes::new()),
}
.build();
let entry = iter.with_iter_mut(|iter| MemTableIterator::entry_to_item(iter.next()));
@@ -111,10 +160,7 @@ impl MemTable {
/// Flush the mem-table to SSTable. Implement in week 1 day 6.
pub fn flush(&self, builder: &mut SsTableBuilder) -> Result<()> {
for entry in self.map.iter() {
builder.add(
KeySlice::from_slice(&entry.key()[..], key::TS_DEFAULT),
&entry.value()[..],
);
builder.add(entry.key().as_key_slice(), &entry.value()[..]);
}
Ok(())
}
@@ -134,8 +180,13 @@ impl MemTable {
}
}
type SkipMapRangeIter<'a> =
crossbeam_skiplist::map::Range<'a, Bytes, (Bound<Bytes>, Bound<Bytes>), Bytes, Bytes>;
type SkipMapRangeIter<'a> = crossbeam_skiplist::map::Range<
'a,
KeyBytes,
(Bound<KeyBytes>, Bound<KeyBytes>),
KeyBytes,
Bytes,
>;
/// An iterator over a range of `SkipMap`. This is a self-referential structure and please refer to week 1, day 2
/// chapter for more information.
@@ -144,20 +195,20 @@ type SkipMapRangeIter<'a> =
#[self_referencing]
pub struct MemTableIterator {
/// Stores a reference to the skipmap.
map: Arc<SkipMap<Bytes, Bytes>>,
map: Arc<SkipMap<KeyBytes, Bytes>>,
/// Stores a skipmap iterator that refers to the lifetime of `MemTableIterator` itself.
#[borrows(map)]
#[not_covariant]
iter: SkipMapRangeIter<'this>,
/// Stores the current key-value pair.
item: (Bytes, Bytes),
item: (KeyBytes, Bytes),
}
impl MemTableIterator {
fn entry_to_item(entry: Option<Entry<'_, Bytes, Bytes>>) -> (Bytes, Bytes) {
fn entry_to_item(entry: Option<Entry<'_, KeyBytes, Bytes>>) -> (KeyBytes, Bytes) {
entry
.map(|x| (x.key().clone(), x.value().clone()))
.unwrap_or_else(|| (Bytes::from_static(&[]), Bytes::from_static(&[])))
.unwrap_or_else(|| (KeyBytes::new(), Bytes::new()))
}
}
@@ -169,7 +220,7 @@ impl StorageIterator for MemTableIterator {
}
fn key(&self) -> KeySlice {
KeySlice::from_slice(&self.borrow_item().0[..], key::TS_DEFAULT)
self.borrow_item().0.as_key_slice()
}
fn is_valid(&self) -> bool {