diff --git a/mini-lsm-book/src/04-engine.md b/mini-lsm-book/src/04-engine.md index 83a2f96..093e8fd 100644 --- a/mini-lsm-book/src/04-engine.md +++ b/mini-lsm-book/src/04-engine.md @@ -1 +1,25 @@ # Storage Engine and Block Cache + + + + +In this part, you will need to modify: + +* `src/lsm_iterator.rs` +* `src/lsm_storage.rs` +* Other parts that use `SsTable::read_block` + +You can use `cargo x copy-test day4` to copy our provided test cases to the starter code directory. After you have +finished this part, use `cargo x scheck` to check the style and run all test cases. If you want to write your own +test cases, write a new module `#[cfg(test)] mod user_tests { /* your test cases */ }` in `table.rs`. Remember to remove +`#![allow(...)]` at the top of the modules you modified so that cargo clippy can actually check the styles. + +## Task 1 - Put and Delete + +## Task 2 - Get + +## Task 3 - Scan + +## Task 4 - Sync + +## Task 5 - Block Cache diff --git a/mini-lsm-book/src/SUMMARY.md b/mini-lsm-book/src/SUMMARY.md index 1e5568a..88a8e39 100644 --- a/mini-lsm-book/src/SUMMARY.md +++ b/mini-lsm-book/src/SUMMARY.md @@ -6,8 +6,8 @@ - [Store key-value pairs in little blocks](./01-block.md) - [And make them into an SST](./02-sst.md) -- [Now it's time for merging everything](./03-memtable.md) -- [Block cache, the good way](./04-engine.md) +- [Now it's time to merge everything](./03-memtable.md) +- [The engine starts](./04-engine.md) - [Let's do something in the background](./05-compaction.md) - [Be careful when the system crashes](./06-recovery.md) - [A good bloom filter makes life easier](./07-bloom-filter.md) diff --git a/mini-lsm-starter/src/iterators.rs b/mini-lsm-starter/src/iterators.rs index 19f6c65..bffb820 100644 --- a/mini-lsm-starter/src/iterators.rs +++ b/mini-lsm-starter/src/iterators.rs @@ -1,6 +1,19 @@ -pub mod impls; pub mod merge_iterator; pub mod two_merge_iterator; +pub trait StorageIterator { + /// Get the current value. + fn value(&self) -> &[u8]; + + /// Get the current key. + fn key(&self) -> &[u8]; + + /// Check if the current iterator is valid. + fn is_valid(&self) -> bool; + + /// Move to the next position. + fn next(&mut self) -> anyhow::Result<()>; +} + #[cfg(test)] mod tests; diff --git a/mini-lsm-starter/src/iterators/impls.rs b/mini-lsm-starter/src/iterators/impls.rs deleted file mode 100644 index a05c75c..0000000 --- a/mini-lsm-starter/src/iterators/impls.rs +++ /dev/null @@ -1,15 +0,0 @@ -use anyhow::Result; - -pub trait StorageIterator { - /// Get the current value. - fn value(&self) -> &[u8]; - - /// Get the current key. - fn key(&self) -> &[u8]; - - /// Check if the current iterator is valid. - fn is_valid(&self) -> bool; - - /// Move to the next position. - fn next(&mut self) -> Result<()>; -} diff --git a/mini-lsm-starter/src/iterators/merge_iterator.rs b/mini-lsm-starter/src/iterators/merge_iterator.rs index fa798b9..c62fcc1 100644 --- a/mini-lsm-starter/src/iterators/merge_iterator.rs +++ b/mini-lsm-starter/src/iterators/merge_iterator.rs @@ -6,7 +6,7 @@ use std::collections::BinaryHeap; use anyhow::Result; -use super::impls::StorageIterator; +use super::StorageIterator; struct HeapWrapper(pub usize, pub Box); diff --git a/mini-lsm-starter/src/iterators/two_merge_iterator.rs b/mini-lsm-starter/src/iterators/two_merge_iterator.rs index 3b9d682..43cac61 100644 --- a/mini-lsm-starter/src/iterators/two_merge_iterator.rs +++ b/mini-lsm-starter/src/iterators/two_merge_iterator.rs @@ -3,7 +3,7 @@ use anyhow::Result; -use super::impls::StorageIterator; +use super::StorageIterator; /// Merges two iterators of different types into one. If the two iterators have the same key, only /// produce the key once and prefer the entry from A. diff --git a/mini-lsm-starter/src/lsm_iterator.rs b/mini-lsm-starter/src/lsm_iterator.rs index fb5aef0..c93b1cc 100644 --- a/mini-lsm-starter/src/lsm_iterator.rs +++ b/mini-lsm-starter/src/lsm_iterator.rs @@ -3,7 +3,7 @@ use anyhow::Result; -use crate::iterators::impls::StorageIterator; +use crate::iterators::StorageIterator; pub struct LsmIterator {} diff --git a/mini-lsm-starter/src/lsm_storage.rs b/mini-lsm-starter/src/lsm_storage.rs index 930f1cd..2322c3b 100644 --- a/mini-lsm-starter/src/lsm_storage.rs +++ b/mini-lsm-starter/src/lsm_storage.rs @@ -45,24 +45,32 @@ impl LsmStorage { }) } + /// Get a key from the storage. In day 7, this can be further optimized by using a bloom filter. pub fn get(&self, key: &[u8]) -> Result> { unimplemented!() } + /// Put a key-value pair into the storage by writing into the current memtable. pub fn put(&self, key: &[u8], value: &[u8]) -> Result<()> { assert!(!value.is_empty(), "value cannot be empty"); assert!(!key.is_empty(), "key cannot be empty"); unimplemented!() } + /// Remove a key from the storage by writing an empty value. pub fn delete(&self, _key: &[u8]) -> Result<()> { unimplemented!() } + /// Persist data to disk. + /// + /// In day 3: flush the current memtable to disk as L0 SST. + /// In day 6: call `fsync` on WAL. pub fn sync(&self) -> Result<()> { unimplemented!() } + /// Create an iterator over a range of keys. pub fn scan( &self, _lower: Bound<&[u8]>, diff --git a/mini-lsm-starter/src/mem_table.rs b/mini-lsm-starter/src/mem_table.rs index f77e50e..d79cd9f 100644 --- a/mini-lsm-starter/src/mem_table.rs +++ b/mini-lsm-starter/src/mem_table.rs @@ -9,7 +9,7 @@ use bytes::Bytes; use crossbeam_skiplist::SkipMap; use ouroboros::self_referencing; -use crate::iterators::impls::StorageIterator; +use crate::iterators::StorageIterator; use crate::table::SsTableBuilder; /// A basic mem-table based on crossbeam-skiplist diff --git a/mini-lsm-starter/src/table/iterator.rs b/mini-lsm-starter/src/table/iterator.rs index 6bc52b9..ae3f66c 100644 --- a/mini-lsm-starter/src/table/iterator.rs +++ b/mini-lsm-starter/src/table/iterator.rs @@ -6,7 +6,7 @@ use std::sync::Arc; use anyhow::Result; use super::SsTable; -use crate::iterators::impls::StorageIterator; +use crate::iterators::StorageIterator; /// An iterator over the contents of an SSTable. pub struct SsTableIterator {} diff --git a/mini-lsm/src/iterators.rs b/mini-lsm/src/iterators.rs index 19f6c65..bffb820 100644 --- a/mini-lsm/src/iterators.rs +++ b/mini-lsm/src/iterators.rs @@ -1,6 +1,19 @@ -pub mod impls; pub mod merge_iterator; pub mod two_merge_iterator; +pub trait StorageIterator { + /// Get the current value. + fn value(&self) -> &[u8]; + + /// Get the current key. + fn key(&self) -> &[u8]; + + /// Check if the current iterator is valid. + fn is_valid(&self) -> bool; + + /// Move to the next position. + fn next(&mut self) -> anyhow::Result<()>; +} + #[cfg(test)] mod tests; diff --git a/mini-lsm/src/iterators/impls.rs b/mini-lsm/src/iterators/impls.rs deleted file mode 100644 index a05c75c..0000000 --- a/mini-lsm/src/iterators/impls.rs +++ /dev/null @@ -1,15 +0,0 @@ -use anyhow::Result; - -pub trait StorageIterator { - /// Get the current value. - fn value(&self) -> &[u8]; - - /// Get the current key. - fn key(&self) -> &[u8]; - - /// Check if the current iterator is valid. - fn is_valid(&self) -> bool; - - /// Move to the next position. - fn next(&mut self) -> Result<()>; -} diff --git a/mini-lsm/src/iterators/merge_iterator.rs b/mini-lsm/src/iterators/merge_iterator.rs index b68e461..b5b990d 100644 --- a/mini-lsm/src/iterators/merge_iterator.rs +++ b/mini-lsm/src/iterators/merge_iterator.rs @@ -4,7 +4,7 @@ use std::collections::BinaryHeap; use anyhow::Result; -use super::impls::StorageIterator; +use super::StorageIterator; struct HeapWrapper(pub usize, pub Box); diff --git a/mini-lsm/src/iterators/tests.rs b/mini-lsm/src/iterators/tests.rs index e0c9f90..2c2963c 100644 --- a/mini-lsm/src/iterators/tests.rs +++ b/mini-lsm/src/iterators/tests.rs @@ -1,7 +1,7 @@ use anyhow::Result; use bytes::Bytes; -use super::impls::StorageIterator; +use super::StorageIterator; pub mod merge_iterator_test; pub mod two_merge_iterator_test; diff --git a/mini-lsm/src/iterators/two_merge_iterator.rs b/mini-lsm/src/iterators/two_merge_iterator.rs index 7789ace..89d7c43 100644 --- a/mini-lsm/src/iterators/two_merge_iterator.rs +++ b/mini-lsm/src/iterators/two_merge_iterator.rs @@ -1,6 +1,6 @@ use anyhow::Result; -use super::impls::StorageIterator; +use super::StorageIterator; /// Merges two iterators of different types into one. If the two iterators have the same key, only /// produce the key once and prefer the entry from A. diff --git a/mini-lsm/src/lsm_iterator.rs b/mini-lsm/src/lsm_iterator.rs index 3dd89ae..df49609 100644 --- a/mini-lsm/src/lsm_iterator.rs +++ b/mini-lsm/src/lsm_iterator.rs @@ -3,9 +3,9 @@ use std::ops::Bound; use anyhow::Result; use bytes::Bytes; -use crate::iterators::impls::StorageIterator; use crate::iterators::merge_iterator::MergeIterator; use crate::iterators::two_merge_iterator::TwoMergeIterator; +use crate::iterators::StorageIterator; use crate::mem_table::MemTableIterator; use crate::table::SsTableIterator; diff --git a/mini-lsm/src/lsm_storage.rs b/mini-lsm/src/lsm_storage.rs index 6fc0722..be54338 100644 --- a/mini-lsm/src/lsm_storage.rs +++ b/mini-lsm/src/lsm_storage.rs @@ -7,9 +7,9 @@ use bytes::Bytes; use parking_lot::{Mutex, RwLock}; use crate::block::Block; -use crate::iterators::impls::StorageIterator; use crate::iterators::merge_iterator::MergeIterator; use crate::iterators::two_merge_iterator::TwoMergeIterator; +use crate::iterators::StorageIterator; use crate::lsm_iterator::{FusedIterator, LsmIterator}; use crate::mem_table::{map_bound, MemTable}; use crate::table::{SsTable, SsTableBuilder, SsTableIterator}; @@ -126,6 +126,8 @@ impl LsmStorage { self.path.join(format!("{:05}.sst", id)) } + /// Persist data to disk. + /// /// In day 3: flush the current memtable to disk as L0 SST. /// In day 6: call `fsync` on WAL. pub fn sync(&self) -> Result<()> { diff --git a/mini-lsm/src/mem_table.rs b/mini-lsm/src/mem_table.rs index b88b900..637b762 100644 --- a/mini-lsm/src/mem_table.rs +++ b/mini-lsm/src/mem_table.rs @@ -7,7 +7,7 @@ use crossbeam_skiplist::map::Entry; use crossbeam_skiplist::SkipMap; use ouroboros::self_referencing; -use crate::iterators::impls::StorageIterator; +use crate::iterators::StorageIterator; use crate::table::SsTableBuilder; /// A basic mem-table based on crossbeam-skiplist diff --git a/mini-lsm/src/mem_table/tests.rs b/mini-lsm/src/mem_table/tests.rs index 6010e84..8371ecb 100644 --- a/mini-lsm/src/mem_table/tests.rs +++ b/mini-lsm/src/mem_table/tests.rs @@ -1,7 +1,7 @@ use tempfile::tempdir; use super::MemTable; -use crate::iterators::impls::StorageIterator; +use crate::iterators::StorageIterator; use crate::table::{SsTableBuilder, SsTableIterator}; #[test] diff --git a/mini-lsm/src/table/iterator.rs b/mini-lsm/src/table/iterator.rs index feb9b79..08b135f 100644 --- a/mini-lsm/src/table/iterator.rs +++ b/mini-lsm/src/table/iterator.rs @@ -4,7 +4,7 @@ use anyhow::Result; use super::SsTable; use crate::block::BlockIterator; -use crate::iterators::impls::StorageIterator; +use crate::iterators::StorageIterator; /// An iterator over the contents of an SSTable. pub struct SsTableIterator { diff --git a/mini-lsm/src/table/tests.rs b/mini-lsm/src/table/tests.rs index 349ce6c..c393181 100644 --- a/mini-lsm/src/table/tests.rs +++ b/mini-lsm/src/table/tests.rs @@ -4,7 +4,7 @@ use bytes::Bytes; use tempfile::{tempdir, TempDir}; use super::*; -use crate::iterators::impls::StorageIterator; +use crate::iterators::StorageIterator; use crate::table::SsTableBuilder; #[test] diff --git a/mini-lsm/src/tests/day4_tests.rs b/mini-lsm/src/tests/day4_tests.rs index c4e041e..d6c4d86 100644 --- a/mini-lsm/src/tests/day4_tests.rs +++ b/mini-lsm/src/tests/day4_tests.rs @@ -3,7 +3,7 @@ use std::ops::Bound; use bytes::Bytes; use tempfile::tempdir; -use crate::iterators::impls::StorageIterator; +use crate::iterators::StorageIterator; fn as_bytes(x: &[u8]) -> Bytes { Bytes::copy_from_slice(x)