patch memtable and add ts for wal
Signed-off-by: Alex Chi <iskyzh@gmail.com>
This commit is contained in:
@@ -94,6 +94,10 @@ impl Key<Vec<u8>> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl Key<Bytes> {
|
impl Key<Bytes> {
|
||||||
|
pub fn new() -> Self {
|
||||||
|
Self(Bytes::new(), TS_DEFAULT)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn as_key_slice(&self) -> KeySlice {
|
pub fn as_key_slice(&self) -> KeySlice {
|
||||||
Key(&self.0, self.1)
|
Key(&self.0, self.1)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -20,16 +20,24 @@ pub struct LsmIterator {
|
|||||||
inner: LsmIteratorInner,
|
inner: LsmIteratorInner,
|
||||||
end_bound: Bound<Bytes>,
|
end_bound: Bound<Bytes>,
|
||||||
is_valid: bool,
|
is_valid: bool,
|
||||||
|
read_ts: u64,
|
||||||
|
prev_key: Vec<u8>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl LsmIterator {
|
impl LsmIterator {
|
||||||
pub(crate) fn new(iter: LsmIteratorInner, end_bound: Bound<Bytes>) -> Result<Self> {
|
pub(crate) fn new(
|
||||||
|
iter: LsmIteratorInner,
|
||||||
|
end_bound: Bound<Bytes>,
|
||||||
|
read_ts: u64,
|
||||||
|
) -> Result<Self> {
|
||||||
let mut iter = Self {
|
let mut iter = Self {
|
||||||
is_valid: iter.is_valid(),
|
is_valid: iter.is_valid(),
|
||||||
inner: iter,
|
inner: iter,
|
||||||
end_bound,
|
end_bound,
|
||||||
|
read_ts,
|
||||||
|
prev_key: Vec::new(),
|
||||||
};
|
};
|
||||||
iter.move_to_non_delete()?;
|
iter.move_to_key()?;
|
||||||
Ok(iter)
|
Ok(iter)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -47,9 +55,31 @@ impl LsmIterator {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn move_to_non_delete(&mut self) -> Result<()> {
|
fn move_to_key(&mut self) -> Result<()> {
|
||||||
while self.is_valid() && self.inner.value().is_empty() {
|
loop {
|
||||||
self.next_inner()?;
|
while self.inner.is_valid() && self.inner.key().key_ref() == self.prev_key {
|
||||||
|
self.next_inner()?;
|
||||||
|
}
|
||||||
|
if !self.inner.is_valid() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
self.prev_key.clear();
|
||||||
|
self.prev_key.extend(self.inner.key().key_ref());
|
||||||
|
while self.inner.is_valid()
|
||||||
|
&& self.inner.key().key_ref() == self.prev_key
|
||||||
|
&& self.inner.key().ts() > self.read_ts
|
||||||
|
{
|
||||||
|
self.next_inner()?;
|
||||||
|
}
|
||||||
|
if !self.inner.is_valid() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if self.inner.key().key_ref() != self.prev_key {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if !self.inner.value().is_empty() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -72,7 +102,7 @@ impl StorageIterator for LsmIterator {
|
|||||||
|
|
||||||
fn next(&mut self) -> Result<()> {
|
fn next(&mut self) -> Result<()> {
|
||||||
self.next_inner()?;
|
self.next_inner()?;
|
||||||
self.move_to_non_delete()?;
|
self.move_to_key()?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ use std::collections::{BTreeSet, HashMap};
|
|||||||
use std::fs::File;
|
use std::fs::File;
|
||||||
use std::ops::Bound;
|
use std::ops::Bound;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::sync::atomic::AtomicUsize;
|
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use anyhow::{Context, Result};
|
use anyhow::{Context, Result};
|
||||||
@@ -21,7 +21,7 @@ use crate::iterators::StorageIterator;
|
|||||||
use crate::key::{self, KeySlice};
|
use crate::key::{self, KeySlice};
|
||||||
use crate::lsm_iterator::{FusedIterator, LsmIterator};
|
use crate::lsm_iterator::{FusedIterator, LsmIterator};
|
||||||
use crate::manifest::{Manifest, ManifestRecord};
|
use crate::manifest::{Manifest, ManifestRecord};
|
||||||
use crate::mem_table::{map_bound, MemTable};
|
use crate::mem_table::{map_bound, map_key_bound_plus_ts, MemTable};
|
||||||
use crate::table::{FileObject, SsTable, SsTableBuilder, SsTableIterator};
|
use crate::table::{FileObject, SsTable, SsTableBuilder, SsTableIterator};
|
||||||
|
|
||||||
pub type BlockCache = moka::sync::Cache<(usize, usize), Arc<Block>>;
|
pub type BlockCache = moka::sync::Cache<(usize, usize), Arc<Block>>;
|
||||||
@@ -152,6 +152,8 @@ pub(crate) struct LsmStorageInner {
|
|||||||
pub(crate) options: Arc<LsmStorageOptions>,
|
pub(crate) options: Arc<LsmStorageOptions>,
|
||||||
pub(crate) compaction_controller: CompactionController,
|
pub(crate) compaction_controller: CompactionController,
|
||||||
pub(crate) manifest: Option<Manifest>,
|
pub(crate) manifest: Option<Manifest>,
|
||||||
|
pub(crate) ts: Arc<AtomicU64>,
|
||||||
|
pub(crate) write_lock: Mutex<()>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A thin wrapper for `LsmStorageInner` and the user interface for MiniLSM.
|
/// A thin wrapper for `LsmStorageInner` and the user interface for MiniLSM.
|
||||||
@@ -405,6 +407,8 @@ impl LsmStorageInner {
|
|||||||
compaction_controller,
|
compaction_controller,
|
||||||
manifest: Some(manifest),
|
manifest: Some(manifest),
|
||||||
options: options.into(),
|
options: options.into(),
|
||||||
|
ts: Arc::new(AtomicU64::new(0)),
|
||||||
|
write_lock: Mutex::new(()),
|
||||||
};
|
};
|
||||||
storage.sync_dir()?;
|
storage.sync_dir()?;
|
||||||
|
|
||||||
@@ -422,25 +426,18 @@ impl LsmStorageInner {
|
|||||||
Arc::clone(&guard)
|
Arc::clone(&guard)
|
||||||
}; // drop global lock here
|
}; // drop global lock here
|
||||||
|
|
||||||
// Search on the current memtable.
|
let mut memtable_iters = Vec::with_capacity(snapshot.imm_memtables.len() + 1);
|
||||||
if let Some(value) = snapshot.memtable.get(key) {
|
memtable_iters.push(Box::new(snapshot.memtable.scan(
|
||||||
if value.is_empty() {
|
Bound::Included(KeySlice::from_slice(key, key::TS_RANGE_BEGIN)),
|
||||||
// found tomestone, return key not exists
|
Bound::Included(KeySlice::from_slice(key, key::TS_RANGE_END)),
|
||||||
return Ok(None);
|
)));
|
||||||
}
|
|
||||||
return Ok(Some(value));
|
|
||||||
}
|
|
||||||
|
|
||||||
// Search on immutable memtables.
|
|
||||||
for memtable in snapshot.imm_memtables.iter() {
|
for memtable in snapshot.imm_memtables.iter() {
|
||||||
if let Some(value) = memtable.get(key) {
|
memtable_iters.push(Box::new(memtable.scan(
|
||||||
if value.is_empty() {
|
Bound::Included(KeySlice::from_slice(key, key::TS_RANGE_BEGIN)),
|
||||||
// found tomestone, return key not exists
|
Bound::Included(KeySlice::from_slice(key, key::TS_RANGE_END)),
|
||||||
return Ok(None);
|
)));
|
||||||
}
|
|
||||||
return Ok(Some(value));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
let memtable_iter = MergeIterator::create(memtable_iters);
|
||||||
|
|
||||||
let mut l0_iters = Vec::with_capacity(snapshot.l0_sstables.len());
|
let mut l0_iters = Vec::with_capacity(snapshot.l0_sstables.len());
|
||||||
|
|
||||||
@@ -466,7 +463,7 @@ impl LsmStorageInner {
|
|||||||
if keep_table(key, &table) {
|
if keep_table(key, &table) {
|
||||||
l0_iters.push(Box::new(SsTableIterator::create_and_seek_to_key(
|
l0_iters.push(Box::new(SsTableIterator::create_and_seek_to_key(
|
||||||
table,
|
table,
|
||||||
KeySlice::from_slice(key, key::TS_DEFAULT),
|
KeySlice::from_slice(key, key::TS_RANGE_BEGIN),
|
||||||
)?));
|
)?));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -482,20 +479,29 @@ impl LsmStorageInner {
|
|||||||
}
|
}
|
||||||
let level_iter = SstConcatIterator::create_and_seek_to_key(
|
let level_iter = SstConcatIterator::create_and_seek_to_key(
|
||||||
level_ssts,
|
level_ssts,
|
||||||
KeySlice::from_slice(key, key::TS_DEFAULT),
|
KeySlice::from_slice(key, key::TS_RANGE_BEGIN),
|
||||||
)?;
|
)?;
|
||||||
level_iters.push(Box::new(level_iter));
|
level_iters.push(Box::new(level_iter));
|
||||||
}
|
}
|
||||||
|
|
||||||
let iter = TwoMergeIterator::create(l0_iter, MergeIterator::create(level_iters))?;
|
let iter = LsmIterator::new(
|
||||||
|
TwoMergeIterator::create(
|
||||||
|
TwoMergeIterator::create(memtable_iter, l0_iter)?,
|
||||||
|
MergeIterator::create(level_iters),
|
||||||
|
)?,
|
||||||
|
Bound::Unbounded,
|
||||||
|
self.ts.load(Ordering::SeqCst),
|
||||||
|
)?;
|
||||||
|
|
||||||
if iter.is_valid() && iter.key().key_ref() == key && !iter.value().is_empty() {
|
if iter.is_valid() && iter.key() == key && !iter.value().is_empty() {
|
||||||
return Ok(Some(Bytes::copy_from_slice(iter.value())));
|
return Ok(Some(Bytes::copy_from_slice(iter.value())));
|
||||||
}
|
}
|
||||||
Ok(None)
|
Ok(None)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn write_batch<T: AsRef<[u8]>>(&self, batch: &[WriteBatchRecord<T>]) -> Result<()> {
|
pub fn write_batch<T: AsRef<[u8]>>(&self, batch: &[WriteBatchRecord<T>]) -> Result<()> {
|
||||||
|
let _lck = self.write_lock.lock();
|
||||||
|
let ts = self.ts.fetch_add(1, Ordering::Relaxed);
|
||||||
for record in batch {
|
for record in batch {
|
||||||
match record {
|
match record {
|
||||||
WriteBatchRecord::Del(key) => {
|
WriteBatchRecord::Del(key) => {
|
||||||
@@ -504,7 +510,7 @@ impl LsmStorageInner {
|
|||||||
let size;
|
let size;
|
||||||
{
|
{
|
||||||
let guard = self.state.read();
|
let guard = self.state.read();
|
||||||
guard.memtable.put(key, b"")?;
|
guard.memtable.put(KeySlice::from_slice(key, ts), b"")?;
|
||||||
size = guard.memtable.approximate_size();
|
size = guard.memtable.approximate_size();
|
||||||
}
|
}
|
||||||
self.try_freeze(size)?;
|
self.try_freeze(size)?;
|
||||||
@@ -517,7 +523,7 @@ impl LsmStorageInner {
|
|||||||
let size;
|
let size;
|
||||||
{
|
{
|
||||||
let guard = self.state.read();
|
let guard = self.state.read();
|
||||||
guard.memtable.put(key, value)?;
|
guard.memtable.put(KeySlice::from_slice(key, ts), value)?;
|
||||||
size = guard.memtable.approximate_size();
|
size = guard.memtable.approximate_size();
|
||||||
}
|
}
|
||||||
self.try_freeze(size)?;
|
self.try_freeze(size)?;
|
||||||
@@ -681,9 +687,15 @@ impl LsmStorageInner {
|
|||||||
}; // drop global lock here
|
}; // drop global lock here
|
||||||
|
|
||||||
let mut memtable_iters = Vec::with_capacity(snapshot.imm_memtables.len() + 1);
|
let mut memtable_iters = Vec::with_capacity(snapshot.imm_memtables.len() + 1);
|
||||||
memtable_iters.push(Box::new(snapshot.memtable.scan(lower, upper)));
|
memtable_iters.push(Box::new(snapshot.memtable.scan(
|
||||||
|
map_key_bound_plus_ts(lower, key::TS_RANGE_BEGIN),
|
||||||
|
map_key_bound_plus_ts(upper, key::TS_RANGE_END),
|
||||||
|
)));
|
||||||
for memtable in snapshot.imm_memtables.iter() {
|
for memtable in snapshot.imm_memtables.iter() {
|
||||||
memtable_iters.push(Box::new(memtable.scan(lower, upper)));
|
memtable_iters.push(Box::new(memtable.scan(
|
||||||
|
map_key_bound_plus_ts(lower, key::TS_RANGE_BEGIN),
|
||||||
|
map_key_bound_plus_ts(upper, key::TS_RANGE_END),
|
||||||
|
)));
|
||||||
}
|
}
|
||||||
let memtable_iter = MergeIterator::create(memtable_iters);
|
let memtable_iter = MergeIterator::create(memtable_iters);
|
||||||
|
|
||||||
@@ -699,12 +711,12 @@ impl LsmStorageInner {
|
|||||||
let iter = match lower {
|
let iter = match lower {
|
||||||
Bound::Included(key) => SsTableIterator::create_and_seek_to_key(
|
Bound::Included(key) => SsTableIterator::create_and_seek_to_key(
|
||||||
table,
|
table,
|
||||||
KeySlice::from_slice(key, key::TS_DEFAULT),
|
KeySlice::from_slice(key, key::TS_RANGE_BEGIN),
|
||||||
)?,
|
)?,
|
||||||
Bound::Excluded(key) => {
|
Bound::Excluded(key) => {
|
||||||
let mut iter = SsTableIterator::create_and_seek_to_key(
|
let mut iter = SsTableIterator::create_and_seek_to_key(
|
||||||
table,
|
table,
|
||||||
KeySlice::from_slice(key, key::TS_DEFAULT),
|
KeySlice::from_slice(key, key::TS_RANGE_BEGIN),
|
||||||
)?;
|
)?;
|
||||||
if iter.is_valid() && iter.key().key_ref() == key {
|
if iter.is_valid() && iter.key().key_ref() == key {
|
||||||
iter.next()?;
|
iter.next()?;
|
||||||
@@ -737,12 +749,12 @@ impl LsmStorageInner {
|
|||||||
let level_iter = match lower {
|
let level_iter = match lower {
|
||||||
Bound::Included(key) => SstConcatIterator::create_and_seek_to_key(
|
Bound::Included(key) => SstConcatIterator::create_and_seek_to_key(
|
||||||
level_ssts,
|
level_ssts,
|
||||||
KeySlice::from_slice(key, key::TS_DEFAULT),
|
KeySlice::from_slice(key, key::TS_RANGE_BEGIN),
|
||||||
)?,
|
)?,
|
||||||
Bound::Excluded(key) => {
|
Bound::Excluded(key) => {
|
||||||
let mut iter = SstConcatIterator::create_and_seek_to_key(
|
let mut iter = SstConcatIterator::create_and_seek_to_key(
|
||||||
level_ssts,
|
level_ssts,
|
||||||
KeySlice::from_slice(key, key::TS_DEFAULT),
|
KeySlice::from_slice(key, key::TS_RANGE_BEGIN),
|
||||||
)?;
|
)?;
|
||||||
if iter.is_valid() && iter.key().key_ref() == key {
|
if iter.is_valid() && iter.key().key_ref() == key {
|
||||||
iter.next()?;
|
iter.next()?;
|
||||||
@@ -760,6 +772,7 @@ impl LsmStorageInner {
|
|||||||
Ok(FusedIterator::new(LsmIterator::new(
|
Ok(FusedIterator::new(LsmIterator::new(
|
||||||
iter,
|
iter,
|
||||||
map_bound(upper),
|
map_bound(upper),
|
||||||
|
self.ts.load(Ordering::SeqCst),
|
||||||
)?))
|
)?))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ use crossbeam_skiplist::SkipMap;
|
|||||||
use ouroboros::self_referencing;
|
use ouroboros::self_referencing;
|
||||||
|
|
||||||
use crate::iterators::StorageIterator;
|
use crate::iterators::StorageIterator;
|
||||||
use crate::key::{self, KeySlice};
|
use crate::key::{KeyBytes, KeySlice, TS_DEFAULT};
|
||||||
use crate::table::SsTableBuilder;
|
use crate::table::SsTableBuilder;
|
||||||
use crate::wal::Wal;
|
use crate::wal::Wal;
|
||||||
|
|
||||||
@@ -19,7 +19,7 @@ use crate::wal::Wal;
|
|||||||
/// An initial implementation of memtable is part of week 1, day 1. It will be incrementally implemented in other
|
/// An initial implementation of memtable is part of week 1, day 1. It will be incrementally implemented in other
|
||||||
/// chapters of week 1 and week 2.
|
/// chapters of week 1 and week 2.
|
||||||
pub struct MemTable {
|
pub struct MemTable {
|
||||||
map: Arc<SkipMap<Bytes, Bytes>>,
|
map: Arc<SkipMap<KeyBytes, Bytes>>,
|
||||||
wal: Option<Wal>,
|
wal: Option<Wal>,
|
||||||
id: usize,
|
id: usize,
|
||||||
approximate_size: Arc<AtomicUsize>,
|
approximate_size: Arc<AtomicUsize>,
|
||||||
@@ -34,6 +34,30 @@ pub(crate) fn map_bound(bound: Bound<&[u8]>) -> Bound<Bytes> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Create a bound of `Bytes` from a bound of `KeySlice`.
|
||||||
|
pub(crate) fn map_key_bound(bound: Bound<KeySlice>) -> Bound<KeyBytes> {
|
||||||
|
match bound {
|
||||||
|
Bound::Included(x) => Bound::Included(KeyBytes::from_bytes_with_ts(
|
||||||
|
Bytes::copy_from_slice(x.key_ref()),
|
||||||
|
x.ts(),
|
||||||
|
)),
|
||||||
|
Bound::Excluded(x) => Bound::Excluded(KeyBytes::from_bytes_with_ts(
|
||||||
|
Bytes::copy_from_slice(x.key_ref()),
|
||||||
|
x.ts(),
|
||||||
|
)),
|
||||||
|
Bound::Unbounded => Bound::Unbounded,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Create a bound of `Bytes` from a bound of `KeySlice`.
|
||||||
|
pub(crate) fn map_key_bound_plus_ts(bound: Bound<&[u8]>, ts: u64) -> Bound<KeySlice> {
|
||||||
|
match bound {
|
||||||
|
Bound::Included(x) => Bound::Included(KeySlice::from_slice(x, ts)),
|
||||||
|
Bound::Excluded(x) => Bound::Excluded(KeySlice::from_slice(x, ts)),
|
||||||
|
Bound::Unbounded => Bound::Unbounded,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
impl MemTable {
|
impl MemTable {
|
||||||
/// Create a new mem-table.
|
/// Create a new mem-table.
|
||||||
pub fn create(id: usize) -> Self {
|
pub fn create(id: usize) -> Self {
|
||||||
@@ -66,19 +90,44 @@ impl MemTable {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Get a value by key.
|
/// Get a value by key. Should not be used in week 3.
|
||||||
pub fn get(&self, key: &[u8]) -> Option<Bytes> {
|
pub fn get(&self, key: KeySlice) -> Option<Bytes> {
|
||||||
self.map.get(key).map(|e| e.value().clone())
|
let key_bytes = KeyBytes::from_bytes_with_ts(
|
||||||
|
Bytes::from_static(unsafe { std::mem::transmute(key.key_ref()) }),
|
||||||
|
key.ts(),
|
||||||
|
);
|
||||||
|
self.map.get(&key_bytes).map(|e| e.value().clone())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn for_testing_put_slice(&self, key: &[u8], value: &[u8]) -> Result<()> {
|
||||||
|
self.put(KeySlice::from_slice(key, TS_DEFAULT), value)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn for_testing_get_slice(&self, key: &[u8]) -> Option<Bytes> {
|
||||||
|
self.get(KeySlice::from_slice(key, TS_DEFAULT))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn for_testing_scan_slice(
|
||||||
|
&self,
|
||||||
|
lower: Bound<&[u8]>,
|
||||||
|
upper: Bound<&[u8]>,
|
||||||
|
) -> MemTableIterator {
|
||||||
|
self.scan(
|
||||||
|
map_key_bound_plus_ts(lower, TS_DEFAULT),
|
||||||
|
map_key_bound_plus_ts(upper, TS_DEFAULT),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Put a key-value pair into the mem-table.
|
/// Put a key-value pair into the mem-table.
|
||||||
///
|
///
|
||||||
/// In week 1, day 1, simply put the key-value pair into the skipmap.
|
/// In week 1, day 1, simply put the key-value pair into the skipmap.
|
||||||
/// In week 2, day 6, also flush the data to WAL.
|
/// In week 2, day 6, also flush the data to WAL.
|
||||||
pub fn put(&self, key: &[u8], value: &[u8]) -> Result<()> {
|
pub fn put(&self, key: KeySlice, value: &[u8]) -> Result<()> {
|
||||||
let estimated_size = key.len() + value.len();
|
let estimated_size = key.raw_len() + value.len();
|
||||||
self.map
|
self.map.insert(
|
||||||
.insert(Bytes::copy_from_slice(key), Bytes::copy_from_slice(value));
|
key.to_key_vec().into_key_bytes(),
|
||||||
|
Bytes::copy_from_slice(value),
|
||||||
|
);
|
||||||
self.approximate_size
|
self.approximate_size
|
||||||
.fetch_add(estimated_size, std::sync::atomic::Ordering::Relaxed);
|
.fetch_add(estimated_size, std::sync::atomic::Ordering::Relaxed);
|
||||||
if let Some(ref wal) = self.wal {
|
if let Some(ref wal) = self.wal {
|
||||||
@@ -95,12 +144,12 @@ impl MemTable {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Get an iterator over a range of keys.
|
/// Get an iterator over a range of keys.
|
||||||
pub fn scan(&self, lower: Bound<&[u8]>, upper: Bound<&[u8]>) -> MemTableIterator {
|
pub fn scan(&self, lower: Bound<KeySlice>, upper: Bound<KeySlice>) -> MemTableIterator {
|
||||||
let (lower, upper) = (map_bound(lower), map_bound(upper));
|
let (lower, upper) = (map_key_bound(lower), map_key_bound(upper));
|
||||||
let mut iter = MemTableIteratorBuilder {
|
let mut iter = MemTableIteratorBuilder {
|
||||||
map: self.map.clone(),
|
map: self.map.clone(),
|
||||||
iter_builder: |map| map.range((lower, upper)),
|
iter_builder: |map| map.range((lower, upper)),
|
||||||
item: (Bytes::new(), Bytes::new()),
|
item: (KeyBytes::new(), Bytes::new()),
|
||||||
}
|
}
|
||||||
.build();
|
.build();
|
||||||
let entry = iter.with_iter_mut(|iter| MemTableIterator::entry_to_item(iter.next()));
|
let entry = iter.with_iter_mut(|iter| MemTableIterator::entry_to_item(iter.next()));
|
||||||
@@ -111,10 +160,7 @@ impl MemTable {
|
|||||||
/// Flush the mem-table to SSTable. Implement in week 1 day 6.
|
/// Flush the mem-table to SSTable. Implement in week 1 day 6.
|
||||||
pub fn flush(&self, builder: &mut SsTableBuilder) -> Result<()> {
|
pub fn flush(&self, builder: &mut SsTableBuilder) -> Result<()> {
|
||||||
for entry in self.map.iter() {
|
for entry in self.map.iter() {
|
||||||
builder.add(
|
builder.add(entry.key().as_key_slice(), &entry.value()[..]);
|
||||||
KeySlice::from_slice(&entry.key()[..], key::TS_DEFAULT),
|
|
||||||
&entry.value()[..],
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -134,8 +180,13 @@ impl MemTable {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type SkipMapRangeIter<'a> =
|
type SkipMapRangeIter<'a> = crossbeam_skiplist::map::Range<
|
||||||
crossbeam_skiplist::map::Range<'a, Bytes, (Bound<Bytes>, Bound<Bytes>), Bytes, Bytes>;
|
'a,
|
||||||
|
KeyBytes,
|
||||||
|
(Bound<KeyBytes>, Bound<KeyBytes>),
|
||||||
|
KeyBytes,
|
||||||
|
Bytes,
|
||||||
|
>;
|
||||||
|
|
||||||
/// An iterator over a range of `SkipMap`. This is a self-referential structure and please refer to week 1, day 2
|
/// An iterator over a range of `SkipMap`. This is a self-referential structure and please refer to week 1, day 2
|
||||||
/// chapter for more information.
|
/// chapter for more information.
|
||||||
@@ -144,20 +195,20 @@ type SkipMapRangeIter<'a> =
|
|||||||
#[self_referencing]
|
#[self_referencing]
|
||||||
pub struct MemTableIterator {
|
pub struct MemTableIterator {
|
||||||
/// Stores a reference to the skipmap.
|
/// Stores a reference to the skipmap.
|
||||||
map: Arc<SkipMap<Bytes, Bytes>>,
|
map: Arc<SkipMap<KeyBytes, Bytes>>,
|
||||||
/// Stores a skipmap iterator that refers to the lifetime of `MemTableIterator` itself.
|
/// Stores a skipmap iterator that refers to the lifetime of `MemTableIterator` itself.
|
||||||
#[borrows(map)]
|
#[borrows(map)]
|
||||||
#[not_covariant]
|
#[not_covariant]
|
||||||
iter: SkipMapRangeIter<'this>,
|
iter: SkipMapRangeIter<'this>,
|
||||||
/// Stores the current key-value pair.
|
/// Stores the current key-value pair.
|
||||||
item: (Bytes, Bytes),
|
item: (KeyBytes, Bytes),
|
||||||
}
|
}
|
||||||
|
|
||||||
impl MemTableIterator {
|
impl MemTableIterator {
|
||||||
fn entry_to_item(entry: Option<Entry<'_, Bytes, Bytes>>) -> (Bytes, Bytes) {
|
fn entry_to_item(entry: Option<Entry<'_, KeyBytes, Bytes>>) -> (KeyBytes, Bytes) {
|
||||||
entry
|
entry
|
||||||
.map(|x| (x.key().clone(), x.value().clone()))
|
.map(|x| (x.key().clone(), x.value().clone()))
|
||||||
.unwrap_or_else(|| (Bytes::from_static(&[]), Bytes::from_static(&[])))
|
.unwrap_or_else(|| (KeyBytes::new(), Bytes::new()))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -169,7 +220,7 @@ impl StorageIterator for MemTableIterator {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn key(&self) -> KeySlice {
|
fn key(&self) -> KeySlice {
|
||||||
KeySlice::from_slice(&self.borrow_item().0[..], key::TS_DEFAULT)
|
self.borrow_item().0.as_key_slice()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn is_valid(&self) -> bool {
|
fn is_valid(&self) -> bool {
|
||||||
|
|||||||
@@ -70,7 +70,9 @@ impl SsTableBuilder {
|
|||||||
first_key: std::mem::take(&mut self.first_key).into_key_bytes(),
|
first_key: std::mem::take(&mut self.first_key).into_key_bytes(),
|
||||||
last_key: std::mem::take(&mut self.last_key).into_key_bytes(),
|
last_key: std::mem::take(&mut self.last_key).into_key_bytes(),
|
||||||
});
|
});
|
||||||
|
let checksum = crc32fast::hash(&encoded_block);
|
||||||
self.data.extend(encoded_block);
|
self.data.extend(encoded_block);
|
||||||
|
self.data.put_u32(checksum);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Builds the SSTable and writes it to the given path. Use the `FileObject` structure to manipulate the disk objects.
|
/// Builds the SSTable and writes it to the given path. Use the `FileObject` structure to manipulate the disk objects.
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ mod week1_day4;
|
|||||||
mod week1_day5;
|
mod week1_day5;
|
||||||
mod week1_day6;
|
mod week1_day6;
|
||||||
mod week1_day7;
|
mod week1_day7;
|
||||||
mod week2_day1;
|
// mod week2_day1;
|
||||||
mod week2_day2;
|
// mod week2_day2;
|
||||||
mod week2_day3;
|
// mod week2_day3;
|
||||||
mod week2_day4;
|
// mod week2_day4;
|
||||||
|
|||||||
@@ -9,6 +9,8 @@ use bytes::{Buf, BufMut, Bytes};
|
|||||||
use crossbeam_skiplist::SkipMap;
|
use crossbeam_skiplist::SkipMap;
|
||||||
use parking_lot::Mutex;
|
use parking_lot::Mutex;
|
||||||
|
|
||||||
|
use crate::key::{KeyBytes, KeySlice};
|
||||||
|
|
||||||
pub struct Wal {
|
pub struct Wal {
|
||||||
file: Arc<Mutex<File>>,
|
file: Arc<Mutex<File>>,
|
||||||
}
|
}
|
||||||
@@ -27,7 +29,7 @@ impl Wal {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn recover(path: impl AsRef<Path>, skiplist: &SkipMap<Bytes, Bytes>) -> Result<Self> {
|
pub fn recover(path: impl AsRef<Path>, skiplist: &SkipMap<KeyBytes, Bytes>) -> Result<Self> {
|
||||||
let path = path.as_ref();
|
let path = path.as_ref();
|
||||||
let mut file = OpenOptions::new()
|
let mut file = OpenOptions::new()
|
||||||
.read(true)
|
.read(true)
|
||||||
@@ -44,6 +46,8 @@ impl Wal {
|
|||||||
let key = Bytes::copy_from_slice(&rbuf[..key_len]);
|
let key = Bytes::copy_from_slice(&rbuf[..key_len]);
|
||||||
hasher.write(&key);
|
hasher.write(&key);
|
||||||
rbuf.advance(key_len);
|
rbuf.advance(key_len);
|
||||||
|
let ts = rbuf.get_u64();
|
||||||
|
hasher.write_u64(ts);
|
||||||
let value_len = rbuf.get_u16() as usize;
|
let value_len = rbuf.get_u16() as usize;
|
||||||
hasher.write_u16(value_len as u16);
|
hasher.write_u16(value_len as u16);
|
||||||
let value = Bytes::copy_from_slice(&rbuf[..value_len]);
|
let value = Bytes::copy_from_slice(&rbuf[..value_len]);
|
||||||
@@ -53,22 +57,24 @@ impl Wal {
|
|||||||
if hasher.finalize() != checksum {
|
if hasher.finalize() != checksum {
|
||||||
bail!("checksum mismatch");
|
bail!("checksum mismatch");
|
||||||
}
|
}
|
||||||
skiplist.insert(key, value);
|
skiplist.insert(KeyBytes::from_bytes_with_ts(key, ts), value);
|
||||||
}
|
}
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
file: Arc::new(Mutex::new(file)),
|
file: Arc::new(Mutex::new(file)),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn put(&self, key: &[u8], value: &[u8]) -> Result<()> {
|
pub fn put(&self, key: KeySlice, value: &[u8]) -> Result<()> {
|
||||||
let mut file = self.file.lock();
|
let mut file = self.file.lock();
|
||||||
let mut buf: Vec<u8> =
|
let mut buf: Vec<u8> =
|
||||||
Vec::with_capacity(key.len() + value.len() + std::mem::size_of::<u16>());
|
Vec::with_capacity(key.raw_len() + value.len() + std::mem::size_of::<u16>());
|
||||||
let mut hasher = crc32fast::Hasher::new();
|
let mut hasher = crc32fast::Hasher::new();
|
||||||
hasher.write_u16(key.len() as u16);
|
hasher.write_u16(key.key_len() as u16);
|
||||||
buf.put_u16(key.len() as u16);
|
buf.put_u16(key.key_len() as u16);
|
||||||
hasher.write(key);
|
hasher.write(key.key_ref());
|
||||||
buf.put_slice(key);
|
buf.put_slice(key.key_ref());
|
||||||
|
hasher.write_u64(key.ts());
|
||||||
|
buf.put_u64(key.ts());
|
||||||
hasher.write_u16(value.len() as u16);
|
hasher.write_u16(value.len() as u16);
|
||||||
buf.put_u16(value.len() as u16);
|
buf.put_u16(value.len() as u16);
|
||||||
buf.put_slice(value);
|
buf.put_slice(value);
|
||||||
|
|||||||
@@ -51,6 +51,22 @@ impl MemTable {
|
|||||||
unimplemented!()
|
unimplemented!()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn for_testing_put_slice(&self, key: &[u8], value: &[u8]) -> Result<()> {
|
||||||
|
self.put(key, value)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn for_testing_get_slice(&self, key: &[u8]) -> Option<Bytes> {
|
||||||
|
self.get(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn for_testing_scan_slice(
|
||||||
|
&self,
|
||||||
|
lower: Bound<&[u8]>,
|
||||||
|
upper: Bound<&[u8]>,
|
||||||
|
) -> MemTableIterator {
|
||||||
|
self.scan(lower, upper)
|
||||||
|
}
|
||||||
|
|
||||||
/// Get a value by key.
|
/// Get a value by key.
|
||||||
pub fn get(&self, _key: &[u8]) -> Option<Bytes> {
|
pub fn get(&self, _key: &[u8]) -> Option<Bytes> {
|
||||||
unimplemented!()
|
unimplemented!()
|
||||||
|
|||||||
@@ -66,6 +66,22 @@ impl MemTable {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn for_testing_put_slice(&self, key: &[u8], value: &[u8]) -> Result<()> {
|
||||||
|
self.put(key, value)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn for_testing_get_slice(&self, key: &[u8]) -> Option<Bytes> {
|
||||||
|
self.get(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn for_testing_scan_slice(
|
||||||
|
&self,
|
||||||
|
lower: Bound<&[u8]>,
|
||||||
|
upper: Bound<&[u8]>,
|
||||||
|
) -> MemTableIterator {
|
||||||
|
self.scan(lower, upper)
|
||||||
|
}
|
||||||
|
|
||||||
/// Get a value by key.
|
/// Get a value by key.
|
||||||
pub fn get(&self, key: &[u8]) -> Option<Bytes> {
|
pub fn get(&self, key: &[u8]) -> Option<Bytes> {
|
||||||
self.map.get(key).map(|e| e.value().clone())
|
self.map.get(key).map(|e| e.value().clone())
|
||||||
|
|||||||
@@ -8,26 +8,44 @@ use crate::{
|
|||||||
#[test]
|
#[test]
|
||||||
fn test_task1_memtable_get() {
|
fn test_task1_memtable_get() {
|
||||||
let memtable = MemTable::create(0);
|
let memtable = MemTable::create(0);
|
||||||
memtable.put(b"key1", b"value1").unwrap();
|
memtable.for_testing_put_slice(b"key1", b"value1").unwrap();
|
||||||
memtable.put(b"key2", b"value2").unwrap();
|
memtable.for_testing_put_slice(b"key2", b"value2").unwrap();
|
||||||
memtable.put(b"key3", b"value3").unwrap();
|
memtable.for_testing_put_slice(b"key3", b"value3").unwrap();
|
||||||
assert_eq!(&memtable.get(b"key1").unwrap()[..], b"value1");
|
assert_eq!(
|
||||||
assert_eq!(&memtable.get(b"key2").unwrap()[..], b"value2");
|
&memtable.for_testing_get_slice(b"key1").unwrap()[..],
|
||||||
assert_eq!(&memtable.get(b"key3").unwrap()[..], b"value3");
|
b"value1"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
&memtable.for_testing_get_slice(b"key2").unwrap()[..],
|
||||||
|
b"value2"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
&memtable.for_testing_get_slice(b"key3").unwrap()[..],
|
||||||
|
b"value3"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_task1_memtable_overwrite() {
|
fn test_task1_memtable_overwrite() {
|
||||||
let memtable = MemTable::create(0);
|
let memtable = MemTable::create(0);
|
||||||
memtable.put(b"key1", b"value1").unwrap();
|
memtable.for_testing_put_slice(b"key1", b"value1").unwrap();
|
||||||
memtable.put(b"key2", b"value2").unwrap();
|
memtable.for_testing_put_slice(b"key2", b"value2").unwrap();
|
||||||
memtable.put(b"key3", b"value3").unwrap();
|
memtable.for_testing_put_slice(b"key3", b"value3").unwrap();
|
||||||
memtable.put(b"key1", b"value11").unwrap();
|
memtable.for_testing_put_slice(b"key1", b"value11").unwrap();
|
||||||
memtable.put(b"key2", b"value22").unwrap();
|
memtable.for_testing_put_slice(b"key2", b"value22").unwrap();
|
||||||
memtable.put(b"key3", b"value33").unwrap();
|
memtable.for_testing_put_slice(b"key3", b"value33").unwrap();
|
||||||
assert_eq!(&memtable.get(b"key1").unwrap()[..], b"value11");
|
assert_eq!(
|
||||||
assert_eq!(&memtable.get(b"key2").unwrap()[..], b"value22");
|
&memtable.for_testing_get_slice(b"key1").unwrap()[..],
|
||||||
assert_eq!(&memtable.get(b"key3").unwrap()[..], b"value33");
|
b"value11"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
&memtable.for_testing_get_slice(b"key2").unwrap()[..],
|
||||||
|
b"value22"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
&memtable.for_testing_get_slice(b"key3").unwrap()[..],
|
||||||
|
b"value33"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -17,12 +17,12 @@ use super::harness::{check_iter_result_by_key, expect_iter_error, MockIterator};
|
|||||||
fn test_task1_memtable_iter() {
|
fn test_task1_memtable_iter() {
|
||||||
use std::ops::Bound;
|
use std::ops::Bound;
|
||||||
let memtable = MemTable::create(0);
|
let memtable = MemTable::create(0);
|
||||||
memtable.put(b"key1", b"value1").unwrap();
|
memtable.for_testing_put_slice(b"key1", b"value1").unwrap();
|
||||||
memtable.put(b"key2", b"value2").unwrap();
|
memtable.for_testing_put_slice(b"key2", b"value2").unwrap();
|
||||||
memtable.put(b"key3", b"value3").unwrap();
|
memtable.for_testing_put_slice(b"key3", b"value3").unwrap();
|
||||||
|
|
||||||
{
|
{
|
||||||
let mut iter = memtable.scan(Bound::Unbounded, Bound::Unbounded);
|
let mut iter = memtable.for_testing_scan_slice(Bound::Unbounded, Bound::Unbounded);
|
||||||
assert_eq!(iter.key().for_testing_key_ref(), b"key1");
|
assert_eq!(iter.key().for_testing_key_ref(), b"key1");
|
||||||
assert_eq!(iter.value(), b"value1");
|
assert_eq!(iter.value(), b"value1");
|
||||||
assert!(iter.is_valid());
|
assert!(iter.is_valid());
|
||||||
@@ -39,7 +39,8 @@ fn test_task1_memtable_iter() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
{
|
{
|
||||||
let mut iter = memtable.scan(Bound::Included(b"key1"), Bound::Included(b"key2"));
|
let mut iter =
|
||||||
|
memtable.for_testing_scan_slice(Bound::Included(b"key1"), Bound::Included(b"key2"));
|
||||||
assert_eq!(iter.key().for_testing_key_ref(), b"key1");
|
assert_eq!(iter.key().for_testing_key_ref(), b"key1");
|
||||||
assert_eq!(iter.value(), b"value1");
|
assert_eq!(iter.value(), b"value1");
|
||||||
assert!(iter.is_valid());
|
assert!(iter.is_valid());
|
||||||
@@ -52,7 +53,8 @@ fn test_task1_memtable_iter() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
{
|
{
|
||||||
let mut iter = memtable.scan(Bound::Excluded(b"key1"), Bound::Excluded(b"key3"));
|
let mut iter =
|
||||||
|
memtable.for_testing_scan_slice(Bound::Excluded(b"key1"), Bound::Excluded(b"key3"));
|
||||||
assert_eq!(iter.key().for_testing_key_ref(), b"key2");
|
assert_eq!(iter.key().for_testing_key_ref(), b"key2");
|
||||||
assert_eq!(iter.value(), b"value2");
|
assert_eq!(iter.value(), b"value2");
|
||||||
assert!(iter.is_valid());
|
assert!(iter.is_valid());
|
||||||
@@ -66,15 +68,17 @@ fn test_task1_empty_memtable_iter() {
|
|||||||
use std::ops::Bound;
|
use std::ops::Bound;
|
||||||
let memtable = MemTable::create(0);
|
let memtable = MemTable::create(0);
|
||||||
{
|
{
|
||||||
let iter = memtable.scan(Bound::Excluded(b"key1"), Bound::Excluded(b"key3"));
|
let iter =
|
||||||
|
memtable.for_testing_scan_slice(Bound::Excluded(b"key1"), Bound::Excluded(b"key3"));
|
||||||
assert!(!iter.is_valid());
|
assert!(!iter.is_valid());
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
let iter = memtable.scan(Bound::Included(b"key1"), Bound::Included(b"key2"));
|
let iter =
|
||||||
|
memtable.for_testing_scan_slice(Bound::Included(b"key1"), Bound::Included(b"key2"));
|
||||||
assert!(!iter.is_valid());
|
assert!(!iter.is_valid());
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
let iter = memtable.scan(Bound::Unbounded, Bound::Unbounded);
|
let iter = memtable.for_testing_scan_slice(Bound::Unbounded, Bound::Unbounded);
|
||||||
assert!(!iter.is_valid());
|
assert!(!iter.is_valid());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user