From 64bb89eee802474946ab28a91376c011d2d038ae Mon Sep 17 00:00:00 2001 From: Alex Chi Date: Wed, 24 Jan 2024 17:34:25 +0800 Subject: [PATCH] finish part 2 Signed-off-by: Alex Chi --- .../src/lsm-tutorial/00-full-overview.svg | 2 +- .../src/lsm-tutorial/00-lsm-read-flow.svg | 2 +- .../src/lsm-tutorial/00-lsm-write-flow.svg | 2 +- .../src/lsm-tutorial/week1-01-frozen.svg | 2 +- .../src/lsm-tutorial/week1-01-overview.svg | 2 +- .../src/lsm-tutorial/week1-01-single.svg | 2 +- .../src/lsm-tutorial/week1-02-overview.svg | 2 +- .../src/lsm-tutorial/week1-03-overview.svg | 2 +- .../src/lsm-tutorial/week1-04-overview.svg | 2 +- .../src/lsm-tutorial/week1-05-overview.svg | 2 +- .../src/lsm-tutorial/week1-07-overview.svg | 2 +- .../src/lsm-tutorial/week1-overview.svg | 2 +- .../src/lsm-tutorial/week2-00-leveled.svg | 2 +- .../src/lsm-tutorial/week2-00-tiered.svg | 2 +- .../lsm-tutorial/week2-00-two-extremes-1.svg | 2 +- .../lsm-tutorial/week2-00-two-extremes-2.svg | 2 +- .../src/lsm-tutorial/week2-01-full.svg | 94 +++++++++ .../src/lsm-tutorial/week2-01-overview.svg | 2 +- .../src/lsm-tutorial/week2-02-simple.svg | 141 +++++++++++++ .../src/lsm-tutorial/week2-04-leveled.svg | 2 +- .../src/lsm-tutorial/week2-05-overview.svg | 2 +- .../src/lsm-tutorial/week2-06-overview.svg | 2 +- .../src/lsm-tutorial/week2-overview.svg | 2 +- mini-lsm-book/src/week2-01-compaction.md | 2 +- mini-lsm-book/src/week2-02-simple.md | 2 +- mini-lsm-book/src/week2-05-manifest.md | 60 ++++++ mini-lsm-book/src/week2-06-wal.md | 41 +++- mini-lsm-starter/src/bin/mini-lsm-cli.rs | 2 +- mini-lsm/src/tests/day4_tests.rs | 193 ------------------ 29 files changed, 359 insertions(+), 218 deletions(-) create mode 100644 mini-lsm-book/src/lsm-tutorial/week2-01-full.svg create mode 100644 mini-lsm-book/src/lsm-tutorial/week2-02-simple.svg delete mode 100644 mini-lsm/src/tests/day4_tests.rs diff --git a/mini-lsm-book/src/lsm-tutorial/00-full-overview.svg b/mini-lsm-book/src/lsm-tutorial/00-full-overview.svg index 0e6bb12..07efd36 100644 --- a/mini-lsm-book/src/lsm-tutorial/00-full-overview.svg +++ b/mini-lsm-book/src/lsm-tutorial/00-full-overview.svg @@ -8,7 +8,7 @@ - + 00-full-overview diff --git a/mini-lsm-book/src/lsm-tutorial/00-lsm-read-flow.svg b/mini-lsm-book/src/lsm-tutorial/00-lsm-read-flow.svg index f319924..077774b 100644 --- a/mini-lsm-book/src/lsm-tutorial/00-lsm-read-flow.svg +++ b/mini-lsm-book/src/lsm-tutorial/00-lsm-read-flow.svg @@ -18,7 +18,7 @@ - + 00-lsm-read-flow diff --git a/mini-lsm-book/src/lsm-tutorial/00-lsm-write-flow.svg b/mini-lsm-book/src/lsm-tutorial/00-lsm-write-flow.svg index e627165..00f3ed6 100644 --- a/mini-lsm-book/src/lsm-tutorial/00-lsm-write-flow.svg +++ b/mini-lsm-book/src/lsm-tutorial/00-lsm-write-flow.svg @@ -13,7 +13,7 @@ - + 00-lsm-write-flow diff --git a/mini-lsm-book/src/lsm-tutorial/week1-01-frozen.svg b/mini-lsm-book/src/lsm-tutorial/week1-01-frozen.svg index 582b1fd..5ec71cc 100644 --- a/mini-lsm-book/src/lsm-tutorial/week1-01-frozen.svg +++ b/mini-lsm-book/src/lsm-tutorial/week1-01-frozen.svg @@ -13,7 +13,7 @@ - + week1-01-frozen diff --git a/mini-lsm-book/src/lsm-tutorial/week1-01-overview.svg b/mini-lsm-book/src/lsm-tutorial/week1-01-overview.svg index 118d76b..3f840b7 100644 --- a/mini-lsm-book/src/lsm-tutorial/week1-01-overview.svg +++ b/mini-lsm-book/src/lsm-tutorial/week1-01-overview.svg @@ -8,7 +8,7 @@ - + week1-01-overview diff --git a/mini-lsm-book/src/lsm-tutorial/week1-01-single.svg b/mini-lsm-book/src/lsm-tutorial/week1-01-single.svg index 93785da..7ab3046 100644 --- a/mini-lsm-book/src/lsm-tutorial/week1-01-single.svg +++ b/mini-lsm-book/src/lsm-tutorial/week1-01-single.svg @@ -8,7 +8,7 @@ - + week1-01-single diff --git a/mini-lsm-book/src/lsm-tutorial/week1-02-overview.svg b/mini-lsm-book/src/lsm-tutorial/week1-02-overview.svg index 71b2b2e..4754d93 100644 --- a/mini-lsm-book/src/lsm-tutorial/week1-02-overview.svg +++ b/mini-lsm-book/src/lsm-tutorial/week1-02-overview.svg @@ -8,7 +8,7 @@ - + week1-02-overview diff --git a/mini-lsm-book/src/lsm-tutorial/week1-03-overview.svg b/mini-lsm-book/src/lsm-tutorial/week1-03-overview.svg index 247b78f..ae6182c 100644 --- a/mini-lsm-book/src/lsm-tutorial/week1-03-overview.svg +++ b/mini-lsm-book/src/lsm-tutorial/week1-03-overview.svg @@ -8,7 +8,7 @@ - + week1-03-overview diff --git a/mini-lsm-book/src/lsm-tutorial/week1-04-overview.svg b/mini-lsm-book/src/lsm-tutorial/week1-04-overview.svg index 69fdac4..8b00f07 100644 --- a/mini-lsm-book/src/lsm-tutorial/week1-04-overview.svg +++ b/mini-lsm-book/src/lsm-tutorial/week1-04-overview.svg @@ -8,7 +8,7 @@ - + week1-04-overview diff --git a/mini-lsm-book/src/lsm-tutorial/week1-05-overview.svg b/mini-lsm-book/src/lsm-tutorial/week1-05-overview.svg index 935f21a..512b37b 100644 --- a/mini-lsm-book/src/lsm-tutorial/week1-05-overview.svg +++ b/mini-lsm-book/src/lsm-tutorial/week1-05-overview.svg @@ -8,7 +8,7 @@ - + week1-05-overview diff --git a/mini-lsm-book/src/lsm-tutorial/week1-07-overview.svg b/mini-lsm-book/src/lsm-tutorial/week1-07-overview.svg index 85fd8f7..d8dd446 100644 --- a/mini-lsm-book/src/lsm-tutorial/week1-07-overview.svg +++ b/mini-lsm-book/src/lsm-tutorial/week1-07-overview.svg @@ -8,7 +8,7 @@ - + week1-07-overview diff --git a/mini-lsm-book/src/lsm-tutorial/week1-overview.svg b/mini-lsm-book/src/lsm-tutorial/week1-overview.svg index 9995e5c..933941f 100644 --- a/mini-lsm-book/src/lsm-tutorial/week1-overview.svg +++ b/mini-lsm-book/src/lsm-tutorial/week1-overview.svg @@ -13,7 +13,7 @@ - + week1-overview diff --git a/mini-lsm-book/src/lsm-tutorial/week2-00-leveled.svg b/mini-lsm-book/src/lsm-tutorial/week2-00-leveled.svg index be9617b..1b7a2bf 100644 --- a/mini-lsm-book/src/lsm-tutorial/week2-00-leveled.svg +++ b/mini-lsm-book/src/lsm-tutorial/week2-00-leveled.svg @@ -8,7 +8,7 @@ - + week2-00-leveled diff --git a/mini-lsm-book/src/lsm-tutorial/week2-00-tiered.svg b/mini-lsm-book/src/lsm-tutorial/week2-00-tiered.svg index cb6b42a..01bde56 100644 --- a/mini-lsm-book/src/lsm-tutorial/week2-00-tiered.svg +++ b/mini-lsm-book/src/lsm-tutorial/week2-00-tiered.svg @@ -8,7 +8,7 @@ - + week2-00-tiered diff --git a/mini-lsm-book/src/lsm-tutorial/week2-00-two-extremes-1.svg b/mini-lsm-book/src/lsm-tutorial/week2-00-two-extremes-1.svg index 4e8168d..d493570 100644 --- a/mini-lsm-book/src/lsm-tutorial/week2-00-two-extremes-1.svg +++ b/mini-lsm-book/src/lsm-tutorial/week2-00-two-extremes-1.svg @@ -8,7 +8,7 @@ - + week2-00-two-extremes-1 diff --git a/mini-lsm-book/src/lsm-tutorial/week2-00-two-extremes-2.svg b/mini-lsm-book/src/lsm-tutorial/week2-00-two-extremes-2.svg index eb7950c..75937cf 100644 --- a/mini-lsm-book/src/lsm-tutorial/week2-00-two-extremes-2.svg +++ b/mini-lsm-book/src/lsm-tutorial/week2-00-two-extremes-2.svg @@ -8,7 +8,7 @@ - + week2-00-two-extremes-2 diff --git a/mini-lsm-book/src/lsm-tutorial/week2-01-full.svg b/mini-lsm-book/src/lsm-tutorial/week2-01-full.svg new file mode 100644 index 0000000..d2f4ea1 --- /dev/null +++ b/mini-lsm-book/src/lsm-tutorial/week2-01-full.svg @@ -0,0 +1,94 @@ + + + + + + + + + + + + week2-01-full + + + Layer 1 + + + + + + + + + SST + + + + + L0 + + + + + + + SST + + + + + L1 + + + + + + + + + + SST + + + + + + + + + + + SST + + + + + + + SST + + + + + + + SST + + + + + + + + Full Compaction + + + + + L1 + + + + + diff --git a/mini-lsm-book/src/lsm-tutorial/week2-01-overview.svg b/mini-lsm-book/src/lsm-tutorial/week2-01-overview.svg index 3cc38f0..825f0fb 100644 --- a/mini-lsm-book/src/lsm-tutorial/week2-01-overview.svg +++ b/mini-lsm-book/src/lsm-tutorial/week2-01-overview.svg @@ -8,7 +8,7 @@ - + week2-01-overview diff --git a/mini-lsm-book/src/lsm-tutorial/week2-02-simple.svg b/mini-lsm-book/src/lsm-tutorial/week2-02-simple.svg new file mode 100644 index 0000000..e02cec3 --- /dev/null +++ b/mini-lsm-book/src/lsm-tutorial/week2-02-simple.svg @@ -0,0 +1,141 @@ + + + + + + + + + + + + week2-02-simple + + + Layer 1 + + + + + + + + + SST + + + + + L0 + + + + + + + SST + + + + + L1 + + + + + + + SST + + + + + + + SST + + + + + L2 + + + + + + + SST + + + + + + + SST + + + + + + + + + + SST + + + + + + + + + + + SST + + + + + + + SST + + + + + + + SST + + + + + + + + Simple Leveled Compaction + + + + + L1 + + + + + + + SST + + + + + + + SST + + + + + diff --git a/mini-lsm-book/src/lsm-tutorial/week2-04-leveled.svg b/mini-lsm-book/src/lsm-tutorial/week2-04-leveled.svg index 3934385..e733341 100644 --- a/mini-lsm-book/src/lsm-tutorial/week2-04-leveled.svg +++ b/mini-lsm-book/src/lsm-tutorial/week2-04-leveled.svg @@ -8,7 +8,7 @@ - + week2-04-leveled diff --git a/mini-lsm-book/src/lsm-tutorial/week2-05-overview.svg b/mini-lsm-book/src/lsm-tutorial/week2-05-overview.svg index ed0f7f3..6c1a48c 100644 --- a/mini-lsm-book/src/lsm-tutorial/week2-05-overview.svg +++ b/mini-lsm-book/src/lsm-tutorial/week2-05-overview.svg @@ -13,7 +13,7 @@ - + week2-05-overview diff --git a/mini-lsm-book/src/lsm-tutorial/week2-06-overview.svg b/mini-lsm-book/src/lsm-tutorial/week2-06-overview.svg index 473f74b..c4c92bc 100644 --- a/mini-lsm-book/src/lsm-tutorial/week2-06-overview.svg +++ b/mini-lsm-book/src/lsm-tutorial/week2-06-overview.svg @@ -8,7 +8,7 @@ - + week2-06-overview diff --git a/mini-lsm-book/src/lsm-tutorial/week2-overview.svg b/mini-lsm-book/src/lsm-tutorial/week2-overview.svg index 684d5b2..0c0ed60 100644 --- a/mini-lsm-book/src/lsm-tutorial/week2-overview.svg +++ b/mini-lsm-book/src/lsm-tutorial/week2-overview.svg @@ -13,7 +13,7 @@ - + week2-overview diff --git a/mini-lsm-book/src/week2-01-compaction.md b/mini-lsm-book/src/week2-01-compaction.md index e7713de..ab8397a 100644 --- a/mini-lsm-book/src/week2-01-compaction.md +++ b/mini-lsm-book/src/week2-01-compaction.md @@ -1,6 +1,6 @@ # Compaction Implementation -![Chapter Overview](./lsm-tutorial/week2-01-overview.svg) +![Chapter Overview](./lsm-tutorial/week2-01-full.svg) In this chapter, you will: diff --git a/mini-lsm-book/src/week2-02-simple.md b/mini-lsm-book/src/week2-02-simple.md index 8ddc6e4..62dfd72 100644 --- a/mini-lsm-book/src/week2-02-simple.md +++ b/mini-lsm-book/src/week2-02-simple.md @@ -1,6 +1,6 @@ # Simple Compaction Strategy -![Chapter Overview](./lsm-tutorial/week2-00-leveled.svg) +![Chapter Overview](./lsm-tutorial/week2-02-simple.svg) In this chapter, you will: diff --git a/mini-lsm-book/src/week2-05-manifest.md b/mini-lsm-book/src/week2-05-manifest.md index d74df7b..eb111d1 100644 --- a/mini-lsm-book/src/week2-05-manifest.md +++ b/mini-lsm-book/src/week2-05-manifest.md @@ -9,10 +9,70 @@ In this chapter, you will: ## Task 1: Manifest Encoding +The system uses a manifest file to record all operations happened in the engine. Currently, there are only two types of them: compaction and SST flush. When the engine restarts, it will read the manifest file, reconstruct the state, and load the SST files on the disk. + +There are many approaches to storing the LSM state. One of the easiest way is to simply store the full state into a JSON file. Every time we do a compaction or flush a new SST, we can serialize the entire LSM state into a file. The problem with this approach is that when the database gets super large (i.e., 10k SSTs), writing the manifest to the disk would be super slow. Therefore, we designed the manifest to be a append-only file. + +In this task, you will need to modify: + +``` +src/manifest.rs +``` + +We encode the manifest records using JSON. You may use `serde_json::to_vec` to encode a manifest record to a json, write it to the manifest file, and do a fsync. When you read from the manifest file, you may use `serde_json::Deserializer::from_slice` and it will return a stream of records. You do not need to store the record length or so, as `serde_json` can automatically find the split of the records. + +After the engine runs for several hours, the manifest file might get very large. At that time, you may periodically compact the manifest file to store the current snapshot and truncate the logs. This is an optimization you may implement as part of bonus tasks. + ## Task 2: Write Manifests +You can now go ahead and modify your LSM engine to write manifests when necessary. In this task, you will need to modify: + +``` +src/lsm_storage.rs +src/compact.rs +``` + +For now, we only use two types of the manifest records: SST flush and compaction. SST flush record stores the SST id that gets flushed to the disk. Compaction record stores the compaction task and the produced SST ids. Every time you write some new files the the disk, first sync the files and the storage directory, and then write to the manifest and sync the manifest. + +To sync the directory, you may implement the `sync_dir` function, where you can use `File::open(dir).sync_all()?` to sync it. On Linux, directory is a file that contains the list of files in the directory. By doing fsync on the directory, you will ensure that the newly-written (or removed) files can be visible to the user if the power goes off. + ## Task 3: Flush on Close +In this task, you will need to modify: + +``` +src/lsm_storage.rs +``` + +You will need to implement the `close` function. If `self.options.enable_wal = false` (we will cover WAL in the next chapter), you should flush all memtables to the disk before stopping the storage engine, so that all user changes will be persisted. + ## Task 4: Recover from the State +In this task, you will need to modify: + +``` +src/lsm_storage.rs +``` + +Now, you may modify the `open` function to recover the engine state from the manifest file. To recover it, you will need to first generate the list of SSTs you will need to load. You can do this by calling `apply_compaction_result` and recover SST ids in the LSM state. After that, you may iterate the state and load all SSTs (update the sstables hash map). During the process, you will need to compute the maximum SST id and update the `next_sst_id` field. After that, you may create a new memtable using that id and increment the id by one. + +You may use the mini-lsm-cli to test your implementation. + +``` +cargo run --bin mini-lsm-cli +fill 1000 2000 +close +cargo run --bin mini-lsm-cli +get 1500 +``` + +## Test Your Understanding + +* When do you need to call `fsync`? Why do you need to fsync the directory? +* What are the places you will need to write to the manifest? + +## Bonus Tasks + +* **Manifest Compaction.** When the number of logs in the manifest file gets too large, you can rewrite the manifest file to only store the current snapshot and append new logs to that file. + {{#include copyright.md}} diff --git a/mini-lsm-book/src/week2-06-wal.md b/mini-lsm-book/src/week2-06-wal.md index ecc4c59..b489683 100644 --- a/mini-lsm-book/src/week2-06-wal.md +++ b/mini-lsm-book/src/week2-06-wal.md @@ -9,13 +9,52 @@ In this chapter, you will: ## Task 1: WAL Encoding -## Task 2: Write WALs +In this task, you will need to modify: + +``` +src/wal.rs +``` + +In the previous chapter, we have implemented the manifest file, so that the LSM state can be persisted. And we implemented the `close` function to flush all memtables to SSTs before stopping the engine. Now, what if the system crashes (i.e., powered off)? We can log memtable modifications to WAL (write-ahead log), and recover WALs when restarting the database. WAL is only enabled when `self.options.enable_wal = true`. + +The WAL encoding is simply a list of key-value pairs. + +``` +| key_len | key | value_len | value | +``` + +You will also need to implement the `recover` function to read the WAL and recover the state of a memtable. + +## Task 2: Integrate WALs + +In this task, you will need to modify: + +``` +src/mem_table.rs +src/wal.rs +src/lsm_storage.rs +``` + +`MemTable` has a WAL field. If the `wal` field is set to `Some(wal)`, you will need to append to the WAL when updating the memtable. In your LSM engine, you will need to create WALs if `enable_wal = true`. You will also need update the manifest using the `ManifestRecord::NewMemtable` record when new memtable is created. ## Task 3: Recover from the WALs +In this task, you will need to modify: + +``` +src/lsm_storage.rs +``` + +If WAL is enabled, you will need to recover the memtables based on WALs when loading the database. You will also need to implement the `sync` function of the database. The basic guarantee of `sync` is that the engine is sure that the data is persisted to the disk (and will be recovered when it restarts). To achieve this, you can simply sync the WAL corresponding to the current memtable. + +``` +cargo run --bin mini-lsm-cli -- --enable-wal +``` + ## Test Your Understanding * When can you tell the user that their modifications (put/delete) have been persisted? +* How can you handle corrupted data in WAL? We do not provide reference answers to the questions, and feel free to discuss about them in the Discord community. diff --git a/mini-lsm-starter/src/bin/mini-lsm-cli.rs b/mini-lsm-starter/src/bin/mini-lsm-cli.rs index ba4e0c5..d210046 100644 --- a/mini-lsm-starter/src/bin/mini-lsm-cli.rs +++ b/mini-lsm-starter/src/bin/mini-lsm-cli.rs @@ -27,7 +27,7 @@ struct Args { path: PathBuf, #[arg(long, default_value = "leveled")] compaction: CompactionStrategy, - #[arg(long, default_value = "true")] + #[arg(long)] enable_wal: bool, } diff --git a/mini-lsm/src/tests/day4_tests.rs b/mini-lsm/src/tests/day4_tests.rs deleted file mode 100644 index 4c26eaa..0000000 --- a/mini-lsm/src/tests/day4_tests.rs +++ /dev/null @@ -1,193 +0,0 @@ -use std::ops::Bound; - -use bytes::Bytes; -use tempfile::tempdir; - -use crate::iterators::StorageIterator; -use crate::lsm_storage::{LsmStorageInner, LsmStorageOptions}; - -fn as_bytes(x: &[u8]) -> Bytes { - Bytes::copy_from_slice(x) -} - -fn check_iter_result(iter: impl StorageIterator, expected: Vec<(Bytes, Bytes)>) { - let mut iter = iter; - for (k, v) in expected { - assert!(iter.is_valid()); - assert_eq!( - k, - iter.key(), - "expected key: {:?}, actual key: {:?}", - k, - as_bytes(iter.key()), - ); - assert_eq!( - v, - iter.value(), - "expected value: {:?}, actual value: {:?}", - v, - as_bytes(iter.value()), - ); - iter.next().unwrap(); - } - assert!(!iter.is_valid()); -} - -fn sync(storage: &LsmStorageInner) { - storage.force_freeze_memtable().unwrap(); - storage.force_flush_next_imm_memtable().unwrap(); -} - -#[test] -fn test_storage_get() { - use crate::lsm_storage::LsmStorageInner; - let dir = tempdir().unwrap(); - let storage = LsmStorageInner::open(&dir, LsmStorageOptions::default_for_week1_test()).unwrap(); - storage.put(b"1", b"233").unwrap(); - storage.put(b"2", b"2333").unwrap(); - storage.put(b"3", b"23333").unwrap(); - assert_eq!(&storage.get(b"1").unwrap().unwrap()[..], b"233"); - assert_eq!(&storage.get(b"2").unwrap().unwrap()[..], b"2333"); - assert_eq!(&storage.get(b"3").unwrap().unwrap()[..], b"23333"); - storage.delete(b"2").unwrap(); - assert!(storage.get(b"2").unwrap().is_none()); -} - -#[test] -fn test_storage_scan_memtable_1() { - use crate::lsm_storage::LsmStorageInner; - let dir = tempdir().unwrap(); - let storage = LsmStorageInner::open(&dir, LsmStorageOptions::default_for_week1_test()).unwrap(); - storage.put(b"1", b"233").unwrap(); - storage.put(b"2", b"2333").unwrap(); - storage.put(b"3", b"23333").unwrap(); - storage.delete(b"2").unwrap(); - check_iter_result( - storage.scan(Bound::Unbounded, Bound::Unbounded).unwrap(), - vec![ - (Bytes::from("1"), Bytes::from("233")), - (Bytes::from("3"), Bytes::from("23333")), - ], - ); - check_iter_result( - storage - .scan(Bound::Included(b"1"), Bound::Included(b"2")) - .unwrap(), - vec![(Bytes::from("1"), Bytes::from("233"))], - ); - check_iter_result( - storage - .scan(Bound::Excluded(b"1"), Bound::Excluded(b"3")) - .unwrap(), - vec![], - ); -} - -#[test] -fn test_storage_scan_memtable_2() { - use crate::lsm_storage::LsmStorageInner; - let dir = tempdir().unwrap(); - let storage = LsmStorageInner::open(&dir, LsmStorageOptions::default_for_week1_test()).unwrap(); - storage.put(b"1", b"233").unwrap(); - storage.put(b"2", b"2333").unwrap(); - storage.put(b"3", b"23333").unwrap(); - storage.delete(b"1").unwrap(); - check_iter_result( - storage.scan(Bound::Unbounded, Bound::Unbounded).unwrap(), - vec![ - (Bytes::from("2"), Bytes::from("2333")), - (Bytes::from("3"), Bytes::from("23333")), - ], - ); - check_iter_result( - storage - .scan(Bound::Included(b"1"), Bound::Included(b"2")) - .unwrap(), - vec![(Bytes::from("2"), Bytes::from("2333"))], - ); - check_iter_result( - storage - .scan(Bound::Excluded(b"1"), Bound::Excluded(b"3")) - .unwrap(), - vec![(Bytes::from("2"), Bytes::from("2333"))], - ); -} - -#[test] -fn test_storage_get_after_sync() { - use crate::lsm_storage::LsmStorageInner; - let dir = tempdir().unwrap(); - let storage = LsmStorageInner::open(&dir, LsmStorageOptions::default_for_week1_test()).unwrap(); - storage.put(b"1", b"233").unwrap(); - storage.put(b"2", b"2333").unwrap(); - sync(&storage); - storage.put(b"3", b"23333").unwrap(); - assert_eq!(&storage.get(b"1").unwrap().unwrap()[..], b"233"); - assert_eq!(&storage.get(b"2").unwrap().unwrap()[..], b"2333"); - assert_eq!(&storage.get(b"3").unwrap().unwrap()[..], b"23333"); - storage.delete(b"2").unwrap(); - assert!(storage.get(b"2").unwrap().is_none()); -} - -#[test] -fn test_storage_scan_memtable_1_after_sync() { - use crate::lsm_storage::LsmStorageInner; - let dir = tempdir().unwrap(); - let storage = LsmStorageInner::open(&dir, LsmStorageOptions::default_for_week1_test()).unwrap(); - storage.put(b"1", b"233").unwrap(); - storage.put(b"2", b"2333").unwrap(); - sync(&storage); - storage.put(b"3", b"23333").unwrap(); - storage.delete(b"2").unwrap(); - check_iter_result( - storage.scan(Bound::Unbounded, Bound::Unbounded).unwrap(), - vec![ - (Bytes::from("1"), Bytes::from("233")), - (Bytes::from("3"), Bytes::from("23333")), - ], - ); - check_iter_result( - storage - .scan(Bound::Included(b"1"), Bound::Included(b"2")) - .unwrap(), - vec![(Bytes::from("1"), Bytes::from("233"))], - ); - check_iter_result( - storage - .scan(Bound::Excluded(b"1"), Bound::Excluded(b"3")) - .unwrap(), - vec![], - ); -} - -#[test] -fn test_storage_scan_memtable_2_after_sync() { - use crate::lsm_storage::LsmStorageInner; - let dir = tempdir().unwrap(); - let storage = LsmStorageInner::open(&dir, LsmStorageOptions::default_for_week1_test()).unwrap(); - storage.put(b"1", b"233").unwrap(); - storage.put(b"2", b"2333").unwrap(); - sync(&storage); - storage.put(b"3", b"23333").unwrap(); - sync(&storage); - storage.delete(b"1").unwrap(); - check_iter_result( - storage.scan(Bound::Unbounded, Bound::Unbounded).unwrap(), - vec![ - (Bytes::from("2"), Bytes::from("2333")), - (Bytes::from("3"), Bytes::from("23333")), - ], - ); - check_iter_result( - storage - .scan(Bound::Included(b"1"), Bound::Included(b"2")) - .unwrap(), - vec![(Bytes::from("2"), Bytes::from("2333"))], - ); - check_iter_result( - storage - .scan(Bound::Excluded(b"1"), Bound::Excluded(b"3")) - .unwrap(), - vec![(Bytes::from("2"), Bytes::from("2333"))], - ); -}