From 892e6ab8f73eb35e284ed7e1838675aba58df150 Mon Sep 17 00:00:00 2001 From: Alex Chi Date: Sun, 21 Jan 2024 11:56:09 +0800 Subject: [PATCH] add week 1 day 2 tutorial Signed-off-by: Alex Chi --- Cargo.lock | 75 +++- mini-lsm-book/src/week1-01-memtable.md | 2 +- mini-lsm-book/src/week1-02-merge-iterator.md | 97 +++++- mini-lsm-book/src/week1-03-block.md | 5 +- mini-lsm-book/src/week1-04-sst.md | 5 +- mini-lsm-book/src/week2-01-compaction.md | 2 + mini-lsm-book/src/week2-02-simple.md | 1 + mini-lsm-starter/Cargo.toml | 2 +- mini-lsm-starter/src/lsm_iterator.rs | 21 +- mini-lsm/src/tests/day2.rs | 340 +++++++++++++++++++ mini-lsm/src/tests/harness.rs | 70 ++++ 11 files changed, 605 insertions(+), 15 deletions(-) create mode 100644 mini-lsm/src/tests/day2.rs create mode 100644 mini-lsm/src/tests/harness.rs diff --git a/Cargo.lock b/Cargo.lock index de70b5b..9684f2c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -253,6 +253,12 @@ dependencies = [ "shared_child", ] +[[package]] +name = "either" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" + [[package]] name = "encode_unicode" version = "0.3.6" @@ -296,9 +302,9 @@ checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" [[package]] name = "heck" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" @@ -318,6 +324,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "itertools" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25db6b064527c5d482d0423354fcd07a89a2dfe07b67892e62411946db7f07b0" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.5" @@ -400,7 +415,7 @@ dependencies = [ "crossbeam-epoch", "crossbeam-skiplist", "moka", - "ouroboros", + "ouroboros 0.15.5", "parking_lot", "rand", "serde", @@ -420,7 +435,7 @@ dependencies = [ "crossbeam-epoch", "crossbeam-skiplist", "moka", - "ouroboros", + "ouroboros 0.18.2", "parking_lot", "rand", "serde", @@ -494,7 +509,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfbb50b356159620db6ac971c6d5c9ab788c9cc38a6f49619fca2a27acb062ca" dependencies = [ "aliasable", - "ouroboros_macro", + "ouroboros_macro 0.15.5", +] + +[[package]] +name = "ouroboros" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a50b637ffd883b2733a8483599fb6136b9dcedaa1850f7ac08b9b6f9f2061208" +dependencies = [ + "aliasable", + "ouroboros_macro 0.18.2", + "static_assertions", ] [[package]] @@ -510,6 +536,20 @@ dependencies = [ "syn 1.0.107", ] +[[package]] +name = "ouroboros_macro" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3633d65683f13b9bcfaa3150880b018899fb0e5d0542f4adaea4f503fdb5eabf" +dependencies = [ + "heck", + "itertools", + "proc-macro2", + "proc-macro2-diagnostics", + "quote", + "syn 2.0.48", +] + [[package]] name = "parking_lot" version = "0.12.1" @@ -572,6 +612,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "proc-macro2-diagnostics" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", + "version_check", + "yansi", +] + [[package]] name = "pulldown-cmark" version = "0.9.2" @@ -775,6 +828,12 @@ version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "strsim" version = "0.10.0" @@ -1141,3 +1200,9 @@ name = "windows_x86_64_msvc" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" + +[[package]] +name = "yansi" +version = "1.0.0-rc.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1367295b8f788d371ce2dbc842c7b709c73ee1364d30351dd300ec2203b12377" diff --git a/mini-lsm-book/src/week1-01-memtable.md b/mini-lsm-book/src/week1-01-memtable.md index a6fe628..67fb481 100644 --- a/mini-lsm-book/src/week1-01-memtable.md +++ b/mini-lsm-book/src/week1-01-memtable.md @@ -149,7 +149,7 @@ Now that you have multiple memtables, you may modify your read path `get` functi * Why doesn't the memtable provide a `delete` API? * Is it possible to use other data structures as the memtable in LSM? What are the pros/cons of using the skiplist? * Why do we need a combination of `state` and `state_lock`? Can we only use `state.read()` and `state.write()`? -* Why does the order to store and to probe the memtables matter? +* Why does the order to store and to probe the memtables matter? If a key appears in multiple memtables, which version should you return to the user? * Is the memory layout of the memtable efficient / does it have good data locality? (Think of how `Byte` is implemented and stored in the skiplist...) What are the possible optimizations to make the memtable more efficient? * So we are using `parking_lot` locks in this tutorial. Is its read-write lock a fair lock? What might happen to the readers trying to acquire the lock if there is one writer waiting for existing readers to stop? * After freezing the memtable, is it possible that some threads still hold the old LSM state and wrote into these immutable memtables? How does your solution prevent it from happening? diff --git a/mini-lsm-book/src/week1-02-merge-iterator.md b/mini-lsm-book/src/week1-02-merge-iterator.md index a97509f..54d51a8 100644 --- a/mini-lsm-book/src/week1-02-merge-iterator.md +++ b/mini-lsm-book/src/week1-02-merge-iterator.md @@ -10,19 +10,110 @@ In this chapter, you will: ## Task 1: Memtable Iterator -self-referential struct +In this chapter, we will implement the LSM `scan` interface. `scan` returns a range of key-value pairs in order using an iterator API. In the previous chapter, you have implemented the `get` API and the logic to create immutable memtables, and your LSM state should now have multiple memtables. You will need to first create iterators on a single memtable, then create a merge iterator on all memtables, and finally implement the range limit for the iterators. + +In this task, you will need to modify: + +``` +src/mem_table.rs +``` + +All LSM iterators implement the `StorageIterator` trait. It has 4 functions: `key`, `value`, `next`, and `is_valid`. When the iterator is created, its cursor will stop on some element, and `key` / `value` will return the first key in the memtable/block/SST satisfying the start condition (i.e., start key). These two interfaces will return a `&[u8]` to avoid copy. Note that this iterator interface is different from the Rust-style iterator. + +`next` moves the cursor to the next place. `is_valid` returns if the iterator has reached the end or errored. You can assume `next` will only be called when `is_valid` returns true. There will be a `FusedIterator` wrapper for iterators that block calls to `next` when the iterator is not valid to avoid users from misusing the iterators. + +Back to the memtable iterator. You should have found out that the iterator does not have any lifetime associated with that. Imagine that you create a `Vec` and call `vec.iter()`, the iterator type will be something like `VecIterator<'a>`, where `'a` is the lifetime of the `vec` object. The same applies to `SkipMap`, where its `iter` API returns an iterator with a lifetime. However, in our case, we do not want to have such lifetimes on our iterators to avoid making the system overcomplicated (and hard to compile...). + +If the iterator does not have a lifetime generics parameter, we should ensure that *whenever the iterator is being used, the underlying skiplist object is not freed*. The only way to achieve that is to put the `Arc` object into the iterator itself. To define such a structure, + +```rust,no_run +pub struct MemtableIterator { + map: Arc>, + iter: SkipMapRangeIter<'???>, +} +``` + +Okay, here is the problem: we want to express that the lifetime of the iterator is the same as the `map` in the structure. How can we do that? + +This is the first and most tricky Rust language thing that you will ever meet in this tutorial -- self-referential structure. If it is possible to write something like: + +```rust,no_run +pub struct MemtableIterator { // <- with lifetime 'this + map: Arc>, + iter: SkipMapRangeIter<'this>, +} +``` + +Then the problem is solved! You can do this with the help of some third-party libraries like `ouroboros`. It provides an easy way to define self-referential structure. It is also possible to do this with unsafe Rust (and indeed, `ouroboros` itself uses unsafe Rust internally...) + +We have already defined the self-referential `MemtableIterator` fields for you, and you will need to implement `MemtableIterator` and the `Memtable::scan` API. ## Task 2: Merge Iterator -error handling, order requirement +In this task, you will need to modify: -## Task 3: LSM Iterator +``` +src/iterators/merge_iterator.rs +``` + +Now that you have multiple memtables and you will create multiple memtable iterators. You will need to merge the results from the memtables and return the latest version of each key to the user. + +`MergeIterator` maintains a binary heap internally. Note that you will need to handle errors (i.e., when an iterator is not valid) and ensure that the latest version of a key-value pair comes out. + +For example, if we have the following data: + +``` +iter1: b->del, c->4, d->5 +iter2: a->1, b->2, c->3 +iter3: e->4 +``` + +The sequence that the merge iterator outputs should be: + +``` +a->1, b->del, c->4, d->5, e->4 +``` + +The constructor of the merge iterator takes a vector of iterators. We assume the one with a lower index (i.e., the first one) has the latest data. + +We want to avoid dynamic dispatch as much as possible, and therefore we do not use `Box` in the system. Instead, we prefer static dispatch using generics. + +## Task 3: LSM Iterator + Fused Iterator + +In this task, you will need to modify: + +``` +src/iterators/lsm_iterator.rs +``` + +We use the `LsmIterator` structure to represent the internal LSM iterators. You will need to modify this structure multiple times throughout the tutorial when more iterators are added into the system. For now, because we only have multiple memtables, it should be defined as: + +```rust,no_run +type LsmIteratorInner = MergeIterator; +``` + +You may go ahead and implement the `LsmIterator` structure, which calls the corresponding inner iterator, and also skip deleted keys. + +We do not test `LsmIterator` in this task. There will be an integration test in task 4. + +Then, we want to provide extra safety on the iterator to avoid users from misusing them. Users should not call `key`, `value`, or `next` when the iterator is not valid. At the same time, they should not use the iterator anymore if `next` returns an error. `FusedIterator` is a wrapper around an iterator to normalize the behaviors across all iterators. You can go ahead and implement it by yourself. ## Task 4: Read Path - Scan +In this task, you will need to modify: + +``` +src/iterators/lsm_storage.rs +``` + +We are finally there -- with all iterators you have implemented, you can finally implement the `scan` interface of the LSM engine. + ## Test Your Understanding +* What is the time/space complexity of using your merge iterator? * Why do we need a self-referential structure for memtable iterator? +* If a key is removed (there is a delete tombstone), do you need to return it to the user? Where did you handle this logic? +* If a key has multiple versions, will the user see all of them? Where did you handle this logic? * If we want to get rid of self-referential structure and have a lifetime on the memtable iterator (i.e., `MemtableIterator<'a>`, where `'a` = memtable or `LsmStorageInner` lifetime), is it still possible to implement the `scan` functionality? * What happens if (1) we create an iterator on the skiplist memtable (2) someone inserts new keys into the memtable (3) will the iterator see the new key? * What happens if your key comparator cannot give the binary heap implementation a stable order? diff --git a/mini-lsm-book/src/week1-03-block.md b/mini-lsm-book/src/week1-03-block.md index bfa4d88..8ff0c4e 100644 --- a/mini-lsm-book/src/week1-03-block.md +++ b/mini-lsm-book/src/week1-03-block.md @@ -13,9 +13,12 @@ In this chapter, you will: ## Test Your Understanding -* So `Block` is simply a vector of raw data and a vector of offsets. Can we change them to `Byte` and `Arc<[u16]>`, and change all the iterator interfaces to return `Byte` instead of `&[u8]`? What are the pros/cons? +* What is the time complexity of seeking a key in the block? +* Where does the cursor stop when you seek a non-existent key in your implementation? +* So `Block` is simply a vector of raw data and a vector of offsets. Can we change them to `Byte` and `Arc<[u16]>`, and change all the iterator interfaces to return `Byte` instead of `&[u8]`? (Assume that we use `Byte::slice` to return a slice of the block without copying.) What are the pros/cons? * What is the endian of the numbers written into the blocks in your implementation? * Is your implementation prune to a maliciously-built block? Will there be invalid memory access, or OOMs, if a user deliberately construct an invalid block? +* Can a block contain duplicated keys? * What happens if the user adds a key larger than the target block size? * Consider the case that the LSM engine is built on object store services (S3). How would you optimize/change the block format and parameters to make it suitable for such services? * Do you love bubble tea? Why or why not? diff --git a/mini-lsm-book/src/week1-04-sst.md b/mini-lsm-book/src/week1-04-sst.md index 948d549..3b83c56 100644 --- a/mini-lsm-book/src/week1-04-sst.md +++ b/mini-lsm-book/src/week1-04-sst.md @@ -15,11 +15,14 @@ In this chapter, you will: ## Test Your Understanding +* What is the time complexity of seeking a key in the SST? +* Where does the cursor stop when you seek a non-existent key in your implementation? +* Is it possible (or necessary) to do in-place updates of SST files? * An SST is usually large (i.e., 256MB). In this case, the cost of copying/expanding the `Vec` would be significant. Does your implementation allocate enough space for your SST builder in advance? How did you implement it? * Looking at the `moka` block cache, why does it return `Arc` instead of the original `Error`? * Does the usage of a block cache guarantee that there will be at most a fixed number of blocks in memory? For example, if you have a `moka` block cache of 4GB and block size of 4KB, will there be more than 4GB/4KB number of blocks in memory at the same time? * Is it possible to store columnar data (i.e., a table of 100 integer columns) in an LSM engine? Is the current SST format still a good choice? -* Consider the case that the LSM engine is built on object store services (S3). How would you optimize/change the SST format/parameters and the block cache to make it suitable for such services? +* Consider the case that the LSM engine is built on object store services (i.e., S3). How would you optimize/change the SST format/parameters and the block cache to make it suitable for such services? We do not provide reference answers to the questions, and feel free to discuss about them in the Discord community. diff --git a/mini-lsm-book/src/week2-01-compaction.md b/mini-lsm-book/src/week2-01-compaction.md index c68fd6b..6bc318e 100644 --- a/mini-lsm-book/src/week2-01-compaction.md +++ b/mini-lsm-book/src/week2-01-compaction.md @@ -12,6 +12,8 @@ In this chapter, you will: * Is it correct that a key will take some storage space even if a user requests to delete it? * Given that compaction takes a lot of write bandwidth and read bandwidth and may interfere with foreground operations, it is a good idea to postpone compaction when there are large write flow. It is even beneficial to stop/pause existing compaction tasks in this situation. What do you think of this idea? (Read the Slik paper!) +* Is it a good idea to use/fill the block cache for compactions? Or is it better to fully bypass the block cache when compaction? +* Some researchers/engineers propose to offload compaction to a remote server or a serverless lambda function. What are the benefits, and what might be the potential challenges and performance impacts of doing remote compaction? (Think of the point when a compaction completes and the block cache...) We do not provide reference answers to the questions, and feel free to discuss about them in the Discord community. diff --git a/mini-lsm-book/src/week2-02-simple.md b/mini-lsm-book/src/week2-02-simple.md index 09c8221..d5a848b 100644 --- a/mini-lsm-book/src/week2-02-simple.md +++ b/mini-lsm-book/src/week2-02-simple.md @@ -9,6 +9,7 @@ In this chapter, you will: ## Test Your Understanding +* (I know this is stupid but) could you please repeat the definition of read/write/space amplifications? What are the ways to accurately compute them, and what are the ways to estimate them? * Is it correct that a key will only be purged from the LSM tree if the user requests to delete it and it has been compacted in the bottom-most level? * Is it a good strategy to periodically do a full compaction on the LSM tree? Why or why not? * Actively choosing some old files/levels to compact even if they do not violate the level amplifier would be a good choice, is it true? (Look at the Lethe paper!) diff --git a/mini-lsm-starter/Cargo.toml b/mini-lsm-starter/Cargo.toml index 3d4b8fc..611436d 100644 --- a/mini-lsm-starter/Cargo.toml +++ b/mini-lsm-starter/Cargo.toml @@ -11,7 +11,7 @@ bytes = "1" crossbeam-epoch = "0.9" crossbeam-skiplist = "0.1" parking_lot = "0.12" -ouroboros = "0.15" +ouroboros = "0.18" moka = "0.9" clap = { version = "4.4.17", features = ["derive"] } rand = "0.8.5" diff --git a/mini-lsm-starter/src/lsm_iterator.rs b/mini-lsm-starter/src/lsm_iterator.rs index c93b1cc..d656495 100644 --- a/mini-lsm-starter/src/lsm_iterator.rs +++ b/mini-lsm-starter/src/lsm_iterator.rs @@ -3,9 +3,23 @@ use anyhow::Result; -use crate::iterators::StorageIterator; +use crate::{ + iterators::{merge_iterator::MergeIterator, StorageIterator}, + mem_table::MemTableIterator, +}; -pub struct LsmIterator {} +/// Represents the internal type for an LSM iterator. This type will be changed across the tutorial for multiple times. +type LsmIteratorInner = MergeIterator; + +pub struct LsmIterator { + inner: LsmIteratorInner, +} + +impl LsmIterator { + pub(crate) fn new(iter: LsmIteratorInner) -> Result { + Ok(Self { inner: iter }) + } +} impl StorageIterator for LsmIterator { fn is_valid(&self) -> bool { @@ -26,7 +40,8 @@ impl StorageIterator for LsmIterator { } /// A wrapper around existing iterator, will prevent users from calling `next` when the iterator is -/// invalid. +/// invalid. If an iterator is already invalid, `next` does not do anything. If `next` returns an error, +/// `is_valid` should return false, and `next` should always return an error. pub struct FusedIterator { iter: I, } diff --git a/mini-lsm/src/tests/day2.rs b/mini-lsm/src/tests/day2.rs new file mode 100644 index 0000000..1d0f72c --- /dev/null +++ b/mini-lsm/src/tests/day2.rs @@ -0,0 +1,340 @@ +use std::ops::Bound; + +use bytes::Bytes; +use tempfile::tempdir; + +use crate::{ + iterators::{merge_iterator::MergeIterator, StorageIterator}, + lsm_iterator::FusedIterator, + lsm_storage::{LsmStorageInner, LsmStorageOptions}, + mem_table::MemTable, +}; + +use super::harness::MockIterator; + +#[test] +fn test_task1_memtable_iter() { + use std::ops::Bound; + let memtable = MemTable::create(0); + memtable.put(b"key1", b"value1").unwrap(); + memtable.put(b"key2", b"value2").unwrap(); + memtable.put(b"key3", b"value3").unwrap(); + + { + let mut iter = memtable.scan(Bound::Unbounded, Bound::Unbounded); + assert_eq!(iter.key(), b"key1"); + assert_eq!(iter.value(), b"value1"); + assert!(iter.is_valid()); + iter.next().unwrap(); + assert_eq!(iter.key(), b"key2"); + assert_eq!(iter.value(), b"value2"); + assert!(iter.is_valid()); + iter.next().unwrap(); + assert_eq!(iter.key(), b"key3"); + assert_eq!(iter.value(), b"value3"); + assert!(iter.is_valid()); + iter.next().unwrap(); + assert!(!iter.is_valid()); + } + + { + let mut iter = memtable.scan(Bound::Included(b"key1"), Bound::Included(b"key2")); + assert_eq!(iter.key(), b"key1"); + assert_eq!(iter.value(), b"value1"); + assert!(iter.is_valid()); + iter.next().unwrap(); + assert_eq!(iter.key(), b"key2"); + assert_eq!(iter.value(), b"value2"); + assert!(iter.is_valid()); + iter.next().unwrap(); + assert!(!iter.is_valid()); + } + + { + let mut iter = memtable.scan(Bound::Excluded(b"key1"), Bound::Excluded(b"key3")); + assert_eq!(iter.key(), b"key2"); + assert_eq!(iter.value(), b"value2"); + assert!(iter.is_valid()); + iter.next().unwrap(); + assert!(!iter.is_valid()); + } +} + +#[test] +fn test_task1_empty_memtable_iter() { + use std::ops::Bound; + let memtable = MemTable::create(0); + { + let iter = memtable.scan(Bound::Excluded(b"key1"), Bound::Excluded(b"key3")); + assert!(!iter.is_valid()); + } + { + let iter = memtable.scan(Bound::Included(b"key1"), Bound::Included(b"key2")); + assert!(!iter.is_valid()); + } + { + let iter = memtable.scan(Bound::Unbounded, Bound::Unbounded); + assert!(!iter.is_valid()); + } +} + +fn as_bytes(x: &[u8]) -> Bytes { + Bytes::copy_from_slice(x) +} + +fn check_iter_result(iter: &mut impl StorageIterator, expected: Vec<(Bytes, Bytes)>) { + for (k, v) in expected { + assert!(iter.is_valid()); + assert_eq!( + k, + iter.key(), + "expected key: {:?}, actual key: {:?}", + k, + as_bytes(iter.key()), + ); + assert_eq!( + v, + iter.value(), + "expected value: {:?}, actual value: {:?}", + v, + as_bytes(iter.value()), + ); + iter.next().unwrap(); + } + assert!(!iter.is_valid()); +} + +fn expect_iter_error(mut iter: impl StorageIterator) { + loop { + match iter.next() { + Ok(_) if iter.is_valid() => continue, + Ok(_) => panic!("expect an error"), + Err(_) => break, + } + } +} + +#[test] +fn test_task2_merge_1() { + let i1 = MockIterator::new(vec![ + (Bytes::from("a"), Bytes::from("1.1")), + (Bytes::from("b"), Bytes::from("2.1")), + (Bytes::from("c"), Bytes::from("3.1")), + ]); + let i2 = MockIterator::new(vec![ + (Bytes::from("a"), Bytes::from("1.2")), + (Bytes::from("b"), Bytes::from("2.2")), + (Bytes::from("c"), Bytes::from("3.2")), + (Bytes::from("d"), Bytes::from("4.2")), + ]); + let i3 = MockIterator::new(vec![ + (Bytes::from("b"), Bytes::from("2.3")), + (Bytes::from("c"), Bytes::from("3.3")), + (Bytes::from("d"), Bytes::from("4.3")), + ]); + + let mut iter = MergeIterator::create(vec![ + Box::new(i1.clone()), + Box::new(i2.clone()), + Box::new(i3.clone()), + ]); + + check_iter_result( + &mut iter, + vec![ + (Bytes::from("a"), Bytes::from("1.1")), + (Bytes::from("b"), Bytes::from("2.1")), + (Bytes::from("c"), Bytes::from("3.1")), + (Bytes::from("d"), Bytes::from("4.2")), + ], + ); + + let mut iter = MergeIterator::create(vec![Box::new(i3), Box::new(i1), Box::new(i2)]); + + check_iter_result( + &mut iter, + vec![ + (Bytes::from("a"), Bytes::from("1.1")), + (Bytes::from("b"), Bytes::from("2.3")), + (Bytes::from("c"), Bytes::from("3.3")), + (Bytes::from("d"), Bytes::from("4.3")), + ], + ); +} + +#[test] +fn test_task2_merge_2() { + let i1 = MockIterator::new(vec![ + (Bytes::from("a"), Bytes::from("1.1")), + (Bytes::from("b"), Bytes::from("2.1")), + (Bytes::from("c"), Bytes::from("3.1")), + ]); + let i2 = MockIterator::new(vec![ + (Bytes::from("d"), Bytes::from("1.2")), + (Bytes::from("e"), Bytes::from("2.2")), + (Bytes::from("f"), Bytes::from("3.2")), + (Bytes::from("g"), Bytes::from("4.2")), + ]); + let i3 = MockIterator::new(vec![ + (Bytes::from("h"), Bytes::from("1.3")), + (Bytes::from("i"), Bytes::from("2.3")), + (Bytes::from("j"), Bytes::from("3.3")), + (Bytes::from("k"), Bytes::from("4.3")), + ]); + let i4 = MockIterator::new(vec![]); + let result = vec![ + (Bytes::from("a"), Bytes::from("1.1")), + (Bytes::from("b"), Bytes::from("2.1")), + (Bytes::from("c"), Bytes::from("3.1")), + (Bytes::from("d"), Bytes::from("1.2")), + (Bytes::from("e"), Bytes::from("2.2")), + (Bytes::from("f"), Bytes::from("3.2")), + (Bytes::from("g"), Bytes::from("4.2")), + (Bytes::from("h"), Bytes::from("1.3")), + (Bytes::from("i"), Bytes::from("2.3")), + (Bytes::from("j"), Bytes::from("3.3")), + (Bytes::from("k"), Bytes::from("4.3")), + ]; + + let mut iter = MergeIterator::create(vec![ + Box::new(i1.clone()), + Box::new(i2.clone()), + Box::new(i3.clone()), + Box::new(i4.clone()), + ]); + check_iter_result(&mut iter, result.clone()); + + let mut iter = MergeIterator::create(vec![ + Box::new(i2.clone()), + Box::new(i4.clone()), + Box::new(i3.clone()), + Box::new(i1.clone()), + ]); + check_iter_result(&mut iter, result.clone()); + + let mut iter = + MergeIterator::create(vec![Box::new(i4), Box::new(i3), Box::new(i2), Box::new(i1)]); + check_iter_result(&mut iter, result); +} + +#[test] +fn test_task2_merge_empty() { + let mut iter = MergeIterator::::create(vec![]); + check_iter_result(&mut iter, vec![]); + + let i1 = MockIterator::new(vec![ + (Bytes::from("a"), Bytes::from("1.1")), + (Bytes::from("b"), Bytes::from("2.1")), + (Bytes::from("c"), Bytes::from("3.1")), + ]); + let i2 = MockIterator::new(vec![]); + let mut iter = MergeIterator::::create(vec![Box::new(i1), Box::new(i2)]); + check_iter_result( + &mut iter, + vec![ + (Bytes::from("a"), Bytes::from("1.1")), + (Bytes::from("b"), Bytes::from("2.1")), + (Bytes::from("c"), Bytes::from("3.1")), + ], + ); +} + +#[test] +fn test_task2_merge_error() { + let mut iter = MergeIterator::::create(vec![]); + check_iter_result(&mut iter, vec![]); + + let i1 = MockIterator::new(vec![ + (Bytes::from("a"), Bytes::from("1.1")), + (Bytes::from("b"), Bytes::from("2.1")), + (Bytes::from("c"), Bytes::from("3.1")), + ]); + let i2 = MockIterator::new_with_error( + vec![ + (Bytes::from("a"), Bytes::from("1.1")), + (Bytes::from("b"), Bytes::from("2.1")), + (Bytes::from("c"), Bytes::from("3.1")), + ], + 1, + ); + let iter = MergeIterator::::create(vec![Box::new(i1), Box::new(i2)]); + // your implementation should correctly throw an error instead of panic + expect_iter_error(iter); +} + +#[test] +fn test_task3_fused_iterator() { + let iter = MockIterator::new(vec![]); + let mut fused_iter = FusedIterator::new(iter); + assert!(!fused_iter.is_valid()); + fused_iter.next().unwrap(); + fused_iter.next().unwrap(); + fused_iter.next().unwrap(); + assert!(!fused_iter.is_valid()); + + let iter = MockIterator::new_with_error( + vec![ + (Bytes::from("a"), Bytes::from("1.1")), + (Bytes::from("a"), Bytes::from("1.1")), + ], + 1, + ); + let mut fused_iter = FusedIterator::new(iter); + assert!(fused_iter.is_valid()); + assert!(fused_iter.next().is_err()); + assert!(!fused_iter.is_valid()); + assert!(fused_iter.next().is_err()); + assert!(fused_iter.next().is_err()); +} + +#[test] +fn test_task4_integration() { + let dir = tempdir().unwrap(); + let storage = + LsmStorageInner::open(dir.path(), LsmStorageOptions::default_for_week1_test()).unwrap(); + storage.put(b"1", b"233").unwrap(); + storage.put(b"2", b"2333").unwrap(); + storage.put(b"3", b"23333").unwrap(); + storage + .force_freeze_memtable(&storage.state_lock.lock()) + .unwrap(); + storage.delete(b"1").unwrap(); + storage.delete(b"2").unwrap(); + storage.put(b"3", b"2333").unwrap(); + storage.put(b"4", b"23333").unwrap(); + storage + .force_freeze_memtable(&storage.state_lock.lock()) + .unwrap(); + storage.put(b"1", b"233333").unwrap(); + storage.put(b"3", b"233333").unwrap(); + { + let mut iter = storage.scan(Bound::Unbounded, Bound::Unbounded).unwrap(); + check_iter_result( + &mut iter, + vec![ + (Bytes::from_static(b"1"), Bytes::from_static(b"233333")), + (Bytes::from_static(b"3"), Bytes::from_static(b"233333")), + (Bytes::from_static(b"4"), Bytes::from_static(b"23333")), + ], + ); + assert!(!iter.is_valid()); + iter.next().unwrap(); + iter.next().unwrap(); + iter.next().unwrap(); + assert!(!iter.is_valid()); + } + { + let mut iter = storage + .scan(Bound::Included(b"2"), Bound::Included(b"3")) + .unwrap(); + check_iter_result( + &mut iter, + vec![(Bytes::from_static(b"3"), Bytes::from_static(b"233333"))], + ); + assert!(!iter.is_valid()); + iter.next().unwrap(); + iter.next().unwrap(); + iter.next().unwrap(); + assert!(!iter.is_valid()); + } +} diff --git a/mini-lsm/src/tests/harness.rs b/mini-lsm/src/tests/harness.rs new file mode 100644 index 0000000..8e0f693 --- /dev/null +++ b/mini-lsm/src/tests/harness.rs @@ -0,0 +1,70 @@ +use anyhow::{bail, Result}; +use bytes::Bytes; + +use crate::iterators::StorageIterator; + +#[derive(Clone)] +pub struct MockIterator { + pub data: Vec<(Bytes, Bytes)>, + pub error_when: Option, + pub index: usize, +} + +impl MockIterator { + pub fn new(data: Vec<(Bytes, Bytes)>) -> Self { + Self { + data, + index: 0, + error_when: None, + } + } + + pub fn new_with_error(data: Vec<(Bytes, Bytes)>, error_when: usize) -> Self { + Self { + data, + index: 0, + error_when: Some(error_when), + } + } +} + +impl StorageIterator for MockIterator { + fn next(&mut self) -> Result<()> { + if self.index < self.data.len() { + self.index += 1; + } + if let Some(error_when) = self.error_when { + if self.index == error_when { + bail!("fake error!"); + } + } + Ok(()) + } + + fn key(&self) -> &[u8] { + if let Some(error_when) = self.error_when { + if self.index >= error_when { + panic!("invalid access after next returns an error!"); + } + } + self.data[self.index].0.as_ref() + } + + fn value(&self) -> &[u8] { + if let Some(error_when) = self.error_when { + if self.index >= error_when { + panic!("invalid access after next returns an error!"); + } + } + self.data[self.index].1.as_ref() + } + + fn is_valid(&self) -> bool { + if let Some(error_when) = self.error_when { + if self.index >= error_when { + panic!("invalid access after next returns an error!"); + } + } + self.index < self.data.len() + } +}