diff --git a/mini-lsm-book/src/week3-01-ts-key-refactor.md b/mini-lsm-book/src/week3-01-ts-key-refactor.md index 573f0ed..caf6bd6 100644 --- a/mini-lsm-book/src/week3-01-ts-key-refactor.md +++ b/mini-lsm-book/src/week3-01-ts-key-refactor.md @@ -105,7 +105,6 @@ Now that we have a timestamp in the key, and when creating the iterators, we wil When you check if a user key is in a table, you can simply compare the user key without comparing the timestamp. -At this point, you should build your implementation and pass all week 1 test cases. We will make the engine fully multi-version and pass all test cases in the next two chapters. - +At this point, you should build your implementation and pass all week 1 test cases. All keys stored in the system will use `TS_DEFAULT` (which is timestamp 0). We will make the engine fully multi-version and pass all test cases in the next two chapters. {{#include copyright.md}} diff --git a/mini-lsm-book/src/week3-03-snapshot-read-part-2.md b/mini-lsm-book/src/week3-03-snapshot-read-part-2.md index 98cd692..8dcfb41 100644 --- a/mini-lsm-book/src/week3-03-snapshot-read-part-2.md +++ b/mini-lsm-book/src/week3-03-snapshot-read-part-2.md @@ -22,6 +22,8 @@ do not implement put and delete ## Task 4: Recover Commit Timestamp +We do not have test cases for this section. You should pass all persistence tests from previous chapters (2.5 and 2.6) after finishing this section. + ## Test Your Understanding * So far, we have assumed that our SST files use a monotonically increasing id as the file name. Is it okay to use `___.sst` as the SST file name? What might be the potential problems with that? diff --git a/mini-lsm-mvcc/src/table.rs b/mini-lsm-mvcc/src/table.rs index b1b757b..60f2f3d 100644 --- a/mini-lsm-mvcc/src/table.rs +++ b/mini-lsm-mvcc/src/table.rs @@ -244,4 +244,10 @@ impl SsTable { pub fn sst_id(&self) -> usize { self.id } + + pub fn max_ts(&self) -> u64 { + // TODO(you): implement me + // self.max_ts + 0 + } } diff --git a/mini-lsm-mvcc/src/tests.rs b/mini-lsm-mvcc/src/tests.rs index 010a21f..3861883 100644 --- a/mini-lsm-mvcc/src/tests.rs +++ b/mini-lsm-mvcc/src/tests.rs @@ -14,3 +14,4 @@ mod week2_day4; // mod week2_day6; mod week3_day1; mod week3_day2; +mod week3_day3; diff --git a/mini-lsm-mvcc/src/tests/week3_day2.rs b/mini-lsm-mvcc/src/tests/week3_day2.rs index 8acca57..df1f3ce 100644 --- a/mini-lsm-mvcc/src/tests/week3_day2.rs +++ b/mini-lsm-mvcc/src/tests/week3_day2.rs @@ -9,7 +9,7 @@ use crate::{ }; #[test] -fn test_task_1_2_integration() { +fn test_task3_compaction_integration() { let dir = tempdir().unwrap(); let mut options = LsmStorageOptions::default_for_week2_test(CompactionOptions::NoCompaction); options.enable_wal = true; diff --git a/mini-lsm-mvcc/src/tests/week3_day3.rs b/mini-lsm-mvcc/src/tests/week3_day3.rs new file mode 100644 index 0000000..391ae17 --- /dev/null +++ b/mini-lsm-mvcc/src/tests/week3_day3.rs @@ -0,0 +1,264 @@ +use std::ops::Bound; + +use bytes::Bytes; +use tempfile::tempdir; + +use crate::{ + compact::CompactionOptions, + key::KeySlice, + lsm_storage::{LsmStorageOptions, MiniLsm}, + table::SsTableBuilder, + tests::harness::check_lsm_iter_result_by_key, +}; + +#[test] +fn test_task2_memtable_mvcc() { + let dir = tempdir().unwrap(); + let mut options = LsmStorageOptions::default_for_week2_test(CompactionOptions::NoCompaction); + options.enable_wal = true; + let storage = MiniLsm::open(&dir, options.clone()).unwrap(); + storage.put(b"a", b"1").unwrap(); + storage.put(b"b", b"1").unwrap(); + let snapshot1 = storage.new_txn().unwrap(); + storage.put(b"a", b"2").unwrap(); + let snapshot2 = storage.new_txn().unwrap(); + storage.delete(b"b").unwrap(); + storage.put(b"c", b"1").unwrap(); + let snapshot3 = storage.new_txn().unwrap(); + assert_eq!(snapshot1.get(b"a").unwrap(), Some(Bytes::from_static(b"1"))); + assert_eq!(snapshot1.get(b"b").unwrap(), Some(Bytes::from_static(b"1"))); + assert_eq!(snapshot1.get(b"c").unwrap(), None); + check_lsm_iter_result_by_key( + &mut snapshot1.scan(Bound::Unbounded, Bound::Unbounded).unwrap(), + vec![ + (Bytes::from("a"), Bytes::from("1")), + (Bytes::from("b"), Bytes::from("1")), + ], + ); + assert_eq!(snapshot2.get(b"a").unwrap(), Some(Bytes::from_static(b"2"))); + assert_eq!(snapshot2.get(b"b").unwrap(), Some(Bytes::from_static(b"1"))); + assert_eq!(snapshot2.get(b"c").unwrap(), None); + check_lsm_iter_result_by_key( + &mut snapshot2.scan(Bound::Unbounded, Bound::Unbounded).unwrap(), + vec![ + (Bytes::from("a"), Bytes::from("2")), + (Bytes::from("b"), Bytes::from("1")), + ], + ); + assert_eq!(snapshot3.get(b"a").unwrap(), Some(Bytes::from_static(b"2"))); + assert_eq!(snapshot3.get(b"b").unwrap(), None); + assert_eq!(snapshot3.get(b"c").unwrap(), Some(Bytes::from_static(b"1"))); + check_lsm_iter_result_by_key( + &mut snapshot3.scan(Bound::Unbounded, Bound::Unbounded).unwrap(), + vec![ + (Bytes::from("a"), Bytes::from("2")), + (Bytes::from("c"), Bytes::from("1")), + ], + ); + storage + .inner + .force_freeze_memtable(&storage.inner.state_lock.lock()) + .unwrap(); + storage.put(b"a", b"3").unwrap(); + storage.put(b"b", b"3").unwrap(); + let snapshot4 = storage.new_txn().unwrap(); + storage.put(b"a", b"4").unwrap(); + let snapshot5 = storage.new_txn().unwrap(); + storage.delete(b"b").unwrap(); + storage.put(b"c", b"5").unwrap(); + let snapshot6 = storage.new_txn().unwrap(); + assert_eq!(snapshot1.get(b"a").unwrap(), Some(Bytes::from_static(b"1"))); + assert_eq!(snapshot1.get(b"b").unwrap(), Some(Bytes::from_static(b"1"))); + assert_eq!(snapshot1.get(b"c").unwrap(), None); + check_lsm_iter_result_by_key( + &mut snapshot1.scan(Bound::Unbounded, Bound::Unbounded).unwrap(), + vec![ + (Bytes::from("a"), Bytes::from("1")), + (Bytes::from("b"), Bytes::from("1")), + ], + ); + assert_eq!(snapshot2.get(b"a").unwrap(), Some(Bytes::from_static(b"2"))); + assert_eq!(snapshot2.get(b"b").unwrap(), Some(Bytes::from_static(b"1"))); + assert_eq!(snapshot2.get(b"c").unwrap(), None); + check_lsm_iter_result_by_key( + &mut snapshot2.scan(Bound::Unbounded, Bound::Unbounded).unwrap(), + vec![ + (Bytes::from("a"), Bytes::from("2")), + (Bytes::from("b"), Bytes::from("1")), + ], + ); + assert_eq!(snapshot3.get(b"a").unwrap(), Some(Bytes::from_static(b"2"))); + assert_eq!(snapshot3.get(b"b").unwrap(), None); + assert_eq!(snapshot3.get(b"c").unwrap(), Some(Bytes::from_static(b"1"))); + check_lsm_iter_result_by_key( + &mut snapshot3.scan(Bound::Unbounded, Bound::Unbounded).unwrap(), + vec![ + (Bytes::from("a"), Bytes::from("2")), + (Bytes::from("c"), Bytes::from("1")), + ], + ); + assert_eq!(snapshot4.get(b"a").unwrap(), Some(Bytes::from_static(b"3"))); + assert_eq!(snapshot4.get(b"b").unwrap(), Some(Bytes::from_static(b"3"))); + assert_eq!(snapshot4.get(b"c").unwrap(), Some(Bytes::from_static(b"1"))); + check_lsm_iter_result_by_key( + &mut snapshot4.scan(Bound::Unbounded, Bound::Unbounded).unwrap(), + vec![ + (Bytes::from("a"), Bytes::from("3")), + (Bytes::from("b"), Bytes::from("3")), + (Bytes::from("c"), Bytes::from("1")), + ], + ); + assert_eq!(snapshot5.get(b"a").unwrap(), Some(Bytes::from_static(b"4"))); + assert_eq!(snapshot5.get(b"b").unwrap(), Some(Bytes::from_static(b"3"))); + assert_eq!(snapshot5.get(b"c").unwrap(), Some(Bytes::from_static(b"1"))); + check_lsm_iter_result_by_key( + &mut snapshot5.scan(Bound::Unbounded, Bound::Unbounded).unwrap(), + vec![ + (Bytes::from("a"), Bytes::from("4")), + (Bytes::from("b"), Bytes::from("3")), + (Bytes::from("c"), Bytes::from("1")), + ], + ); + assert_eq!(snapshot6.get(b"a").unwrap(), Some(Bytes::from_static(b"4"))); + assert_eq!(snapshot6.get(b"b").unwrap(), None); + assert_eq!(snapshot6.get(b"c").unwrap(), Some(Bytes::from_static(b"5"))); + check_lsm_iter_result_by_key( + &mut snapshot6.scan(Bound::Unbounded, Bound::Unbounded).unwrap(), + vec![ + (Bytes::from("a"), Bytes::from("4")), + (Bytes::from("c"), Bytes::from("5")), + ], + ); +} + +#[test] +fn test_task2_lsm_iterator_mvcc() { + let dir = tempdir().unwrap(); + let mut options = LsmStorageOptions::default_for_week2_test(CompactionOptions::NoCompaction); + options.enable_wal = true; + let storage = MiniLsm::open(&dir, options.clone()).unwrap(); + storage.put(b"a", b"1").unwrap(); + storage.put(b"b", b"1").unwrap(); + let snapshot1 = storage.new_txn().unwrap(); + storage.put(b"a", b"2").unwrap(); + let snapshot2 = storage.new_txn().unwrap(); + storage.delete(b"b").unwrap(); + storage.put(b"c", b"1").unwrap(); + let snapshot3 = storage.new_txn().unwrap(); + storage.force_flush().unwrap(); + assert_eq!(snapshot1.get(b"a").unwrap(), Some(Bytes::from_static(b"1"))); + assert_eq!(snapshot1.get(b"b").unwrap(), Some(Bytes::from_static(b"1"))); + assert_eq!(snapshot1.get(b"c").unwrap(), None); + check_lsm_iter_result_by_key( + &mut snapshot1.scan(Bound::Unbounded, Bound::Unbounded).unwrap(), + vec![ + (Bytes::from("a"), Bytes::from("1")), + (Bytes::from("b"), Bytes::from("1")), + ], + ); + assert_eq!(snapshot2.get(b"a").unwrap(), Some(Bytes::from_static(b"2"))); + assert_eq!(snapshot2.get(b"b").unwrap(), Some(Bytes::from_static(b"1"))); + assert_eq!(snapshot2.get(b"c").unwrap(), None); + check_lsm_iter_result_by_key( + &mut snapshot2.scan(Bound::Unbounded, Bound::Unbounded).unwrap(), + vec![ + (Bytes::from("a"), Bytes::from("2")), + (Bytes::from("b"), Bytes::from("1")), + ], + ); + assert_eq!(snapshot3.get(b"a").unwrap(), Some(Bytes::from_static(b"2"))); + assert_eq!(snapshot3.get(b"b").unwrap(), None); + assert_eq!(snapshot3.get(b"c").unwrap(), Some(Bytes::from_static(b"1"))); + check_lsm_iter_result_by_key( + &mut snapshot3.scan(Bound::Unbounded, Bound::Unbounded).unwrap(), + vec![ + (Bytes::from("a"), Bytes::from("2")), + (Bytes::from("c"), Bytes::from("1")), + ], + ); + storage.put(b"a", b"3").unwrap(); + storage.put(b"b", b"3").unwrap(); + let snapshot4 = storage.new_txn().unwrap(); + storage.put(b"a", b"4").unwrap(); + let snapshot5 = storage.new_txn().unwrap(); + storage.delete(b"b").unwrap(); + storage.put(b"c", b"5").unwrap(); + let snapshot6 = storage.new_txn().unwrap(); + storage.force_flush().unwrap(); + assert_eq!(snapshot1.get(b"a").unwrap(), Some(Bytes::from_static(b"1"))); + assert_eq!(snapshot1.get(b"b").unwrap(), Some(Bytes::from_static(b"1"))); + assert_eq!(snapshot1.get(b"c").unwrap(), None); + check_lsm_iter_result_by_key( + &mut snapshot1.scan(Bound::Unbounded, Bound::Unbounded).unwrap(), + vec![ + (Bytes::from("a"), Bytes::from("1")), + (Bytes::from("b"), Bytes::from("1")), + ], + ); + assert_eq!(snapshot2.get(b"a").unwrap(), Some(Bytes::from_static(b"2"))); + assert_eq!(snapshot2.get(b"b").unwrap(), Some(Bytes::from_static(b"1"))); + assert_eq!(snapshot2.get(b"c").unwrap(), None); + check_lsm_iter_result_by_key( + &mut snapshot2.scan(Bound::Unbounded, Bound::Unbounded).unwrap(), + vec![ + (Bytes::from("a"), Bytes::from("2")), + (Bytes::from("b"), Bytes::from("1")), + ], + ); + assert_eq!(snapshot3.get(b"a").unwrap(), Some(Bytes::from_static(b"2"))); + assert_eq!(snapshot3.get(b"b").unwrap(), None); + assert_eq!(snapshot3.get(b"c").unwrap(), Some(Bytes::from_static(b"1"))); + check_lsm_iter_result_by_key( + &mut snapshot3.scan(Bound::Unbounded, Bound::Unbounded).unwrap(), + vec![ + (Bytes::from("a"), Bytes::from("2")), + (Bytes::from("c"), Bytes::from("1")), + ], + ); + assert_eq!(snapshot4.get(b"a").unwrap(), Some(Bytes::from_static(b"3"))); + assert_eq!(snapshot4.get(b"b").unwrap(), Some(Bytes::from_static(b"3"))); + assert_eq!(snapshot4.get(b"c").unwrap(), Some(Bytes::from_static(b"1"))); + check_lsm_iter_result_by_key( + &mut snapshot4.scan(Bound::Unbounded, Bound::Unbounded).unwrap(), + vec![ + (Bytes::from("a"), Bytes::from("3")), + (Bytes::from("b"), Bytes::from("3")), + (Bytes::from("c"), Bytes::from("1")), + ], + ); + assert_eq!(snapshot5.get(b"a").unwrap(), Some(Bytes::from_static(b"4"))); + assert_eq!(snapshot5.get(b"b").unwrap(), Some(Bytes::from_static(b"3"))); + assert_eq!(snapshot5.get(b"c").unwrap(), Some(Bytes::from_static(b"1"))); + check_lsm_iter_result_by_key( + &mut snapshot5.scan(Bound::Unbounded, Bound::Unbounded).unwrap(), + vec![ + (Bytes::from("a"), Bytes::from("4")), + (Bytes::from("b"), Bytes::from("3")), + (Bytes::from("c"), Bytes::from("1")), + ], + ); + assert_eq!(snapshot6.get(b"a").unwrap(), Some(Bytes::from_static(b"4"))); + assert_eq!(snapshot6.get(b"b").unwrap(), None); + assert_eq!(snapshot6.get(b"c").unwrap(), Some(Bytes::from_static(b"5"))); + check_lsm_iter_result_by_key( + &mut snapshot6.scan(Bound::Unbounded, Bound::Unbounded).unwrap(), + vec![ + (Bytes::from("a"), Bytes::from("4")), + (Bytes::from("c"), Bytes::from("5")), + ], + ); +} + +#[test] +fn test_task3_sst_ts() { + let mut builder = SsTableBuilder::new(16); + builder.add(KeySlice::for_testing_from_slice_with_ts(b"11", 1), b"11"); + builder.add(KeySlice::for_testing_from_slice_with_ts(b"22", 2), b"22"); + builder.add(KeySlice::for_testing_from_slice_with_ts(b"33", 3), b"11"); + builder.add(KeySlice::for_testing_from_slice_with_ts(b"44", 4), b"22"); + builder.add(KeySlice::for_testing_from_slice_with_ts(b"55", 5), b"11"); + builder.add(KeySlice::for_testing_from_slice_with_ts(b"66", 6), b"22"); + let dir = tempdir().unwrap(); + let sst = builder.build_for_test(dir.path().join("1.sst")).unwrap(); + assert_eq!(sst.max_ts(), 6); +} diff --git a/mini-lsm-starter/src/table.rs b/mini-lsm-starter/src/table.rs index 58188a5..02baaa2 100644 --- a/mini-lsm-starter/src/table.rs +++ b/mini-lsm-starter/src/table.rs @@ -96,6 +96,8 @@ pub struct SsTable { first_key: KeyBytes, last_key: KeyBytes, pub(crate) bloom: Option, + /// The maximum timestamp stored in this SST, implemented in week 3. + max_ts: u64, } impl SsTable { @@ -125,6 +127,7 @@ impl SsTable { first_key, last_key, bloom: None, + max_ts: 0, } } @@ -165,4 +168,8 @@ impl SsTable { pub fn sst_id(&self) -> usize { self.id } + + pub fn max_ts(&self) -> u64 { + self.max_ts + } } diff --git a/mini-lsm/src/table.rs b/mini-lsm/src/table.rs index 21a0052..34e2287 100644 --- a/mini-lsm/src/table.rs +++ b/mini-lsm/src/table.rs @@ -133,6 +133,7 @@ pub struct SsTable { first_key: KeyBytes, last_key: KeyBytes, pub(crate) bloom: Option, + max_ts: u64, } impl SsTable { #[cfg(test)] @@ -160,6 +161,7 @@ impl SsTable { id, block_cache, bloom: Some(bloom_filter), + max_ts: 0, }) } @@ -179,6 +181,7 @@ impl SsTable { first_key, last_key, bloom: None, + max_ts: 0, } } @@ -240,4 +243,8 @@ impl SsTable { pub fn sst_id(&self) -> usize { self.id } + + pub fn max_ts(&self) -> u64 { + self.max_ts + } }