add 3.3 test

Signed-off-by: Alex Chi <iskyzh@gmail.com>

mini-lsm-book/src/week3-01-ts-key-refactor.md

@@ -105,7 +105,6 @@ Now that we have a timestamp in the key, and when creating the iterators, we wil
 
 When you check if a user key is in a table, you can simply compare the user key without comparing the timestamp.
 
-At this point, you should build your implementation and pass all week 1 test cases. We will make the engine fully multi-version and pass all test cases in the next two chapters.
+At this point, you should build your implementation and pass all week 1 test cases. All keys stored in the system will use `TS_DEFAULT` (which is timestamp 0). We will make the engine fully multi-version and pass all test cases in the next two chapters.
-
 
 {{#include copyright.md}}

mini-lsm-book/src/week3-03-snapshot-read-part-2.md

@@ -22,6 +22,8 @@ do not implement put and delete
 
 ## Task 4: Recover Commit Timestamp
 
+We do not have test cases for this section. You should pass all persistence tests from previous chapters (2.5 and 2.6) after finishing this section.
+
 ## Test Your Understanding
 
 * So far, we have assumed that our SST files use a monotonically increasing id as the file name. Is it okay to use `<level>_<begin_key>_<end_key>_<max_ts>.sst` as the SST file name? What might be the potential problems with that?

mini-lsm-mvcc/src/table.rs

@@ -244,4 +244,10 @@ impl SsTable {
     pub fn sst_id(&self) -> usize {
         self.id
     }
+
+    pub fn max_ts(&self) -> u64 {
+        // TODO(you): implement me
+        // self.max_ts
+        0
+    }
 }

mini-lsm-mvcc/src/tests.rs

@@ -14,3 +14,4 @@ mod week2_day4;
 // mod week2_day6;
 mod week3_day1;
 mod week3_day2;
+mod week3_day3;

mini-lsm-mvcc/src/tests/week3_day2.rs

@@ -9,7 +9,7 @@ use crate::{
 };
 
 #[test]
-fn test_task_1_2_integration() {
+fn test_task3_compaction_integration() {
     let dir = tempdir().unwrap();
     let mut options = LsmStorageOptions::default_for_week2_test(CompactionOptions::NoCompaction);
     options.enable_wal = true;

mini-lsm-mvcc/src/tests/week3_day3.rs

@@ -0,0 +1,264 @@
+use std::ops::Bound;
+
+use bytes::Bytes;
+use tempfile::tempdir;
+
+use crate::{
+    compact::CompactionOptions,
+    key::KeySlice,
+    lsm_storage::{LsmStorageOptions, MiniLsm},
+    table::SsTableBuilder,
+    tests::harness::check_lsm_iter_result_by_key,
+};
+
+#[test]
+fn test_task2_memtable_mvcc() {
+    let dir = tempdir().unwrap();
+    let mut options = LsmStorageOptions::default_for_week2_test(CompactionOptions::NoCompaction);
+    options.enable_wal = true;
+    let storage = MiniLsm::open(&dir, options.clone()).unwrap();
+    storage.put(b"a", b"1").unwrap();
+    storage.put(b"b", b"1").unwrap();
+    let snapshot1 = storage.new_txn().unwrap();
+    storage.put(b"a", b"2").unwrap();
+    let snapshot2 = storage.new_txn().unwrap();
+    storage.delete(b"b").unwrap();
+    storage.put(b"c", b"1").unwrap();
+    let snapshot3 = storage.new_txn().unwrap();
+    assert_eq!(snapshot1.get(b"a").unwrap(), Some(Bytes::from_static(b"1")));
+    assert_eq!(snapshot1.get(b"b").unwrap(), Some(Bytes::from_static(b"1")));
+    assert_eq!(snapshot1.get(b"c").unwrap(), None);
+    check_lsm_iter_result_by_key(
+        &mut snapshot1.scan(Bound::Unbounded, Bound::Unbounded).unwrap(),
+        vec![
+            (Bytes::from("a"), Bytes::from("1")),
+            (Bytes::from("b"), Bytes::from("1")),
+        ],
+    );
+    assert_eq!(snapshot2.get(b"a").unwrap(), Some(Bytes::from_static(b"2")));
+    assert_eq!(snapshot2.get(b"b").unwrap(), Some(Bytes::from_static(b"1")));
+    assert_eq!(snapshot2.get(b"c").unwrap(), None);
+    check_lsm_iter_result_by_key(
+        &mut snapshot2.scan(Bound::Unbounded, Bound::Unbounded).unwrap(),
+        vec![
+            (Bytes::from("a"), Bytes::from("2")),
+            (Bytes::from("b"), Bytes::from("1")),
+        ],
+    );
+    assert_eq!(snapshot3.get(b"a").unwrap(), Some(Bytes::from_static(b"2")));
+    assert_eq!(snapshot3.get(b"b").unwrap(), None);
+    assert_eq!(snapshot3.get(b"c").unwrap(), Some(Bytes::from_static(b"1")));
+    check_lsm_iter_result_by_key(
+        &mut snapshot3.scan(Bound::Unbounded, Bound::Unbounded).unwrap(),
+        vec![
+            (Bytes::from("a"), Bytes::from("2")),
+            (Bytes::from("c"), Bytes::from("1")),
+        ],
+    );
+    storage
+        .inner
+        .force_freeze_memtable(&storage.inner.state_lock.lock())
+        .unwrap();
+    storage.put(b"a", b"3").unwrap();
+    storage.put(b"b", b"3").unwrap();
+    let snapshot4 = storage.new_txn().unwrap();
+    storage.put(b"a", b"4").unwrap();
+    let snapshot5 = storage.new_txn().unwrap();
+    storage.delete(b"b").unwrap();
+    storage.put(b"c", b"5").unwrap();
+    let snapshot6 = storage.new_txn().unwrap();
+    assert_eq!(snapshot1.get(b"a").unwrap(), Some(Bytes::from_static(b"1")));
+    assert_eq!(snapshot1.get(b"b").unwrap(), Some(Bytes::from_static(b"1")));
+    assert_eq!(snapshot1.get(b"c").unwrap(), None);
+    check_lsm_iter_result_by_key(
+        &mut snapshot1.scan(Bound::Unbounded, Bound::Unbounded).unwrap(),
+        vec![
+            (Bytes::from("a"), Bytes::from("1")),
+            (Bytes::from("b"), Bytes::from("1")),
+        ],
+    );
+    assert_eq!(snapshot2.get(b"a").unwrap(), Some(Bytes::from_static(b"2")));
+    assert_eq!(snapshot2.get(b"b").unwrap(), Some(Bytes::from_static(b"1")));
+    assert_eq!(snapshot2.get(b"c").unwrap(), None);
+    check_lsm_iter_result_by_key(
+        &mut snapshot2.scan(Bound::Unbounded, Bound::Unbounded).unwrap(),
+        vec![
+            (Bytes::from("a"), Bytes::from("2")),
+            (Bytes::from("b"), Bytes::from("1")),
+        ],
+    );
+    assert_eq!(snapshot3.get(b"a").unwrap(), Some(Bytes::from_static(b"2")));
+    assert_eq!(snapshot3.get(b"b").unwrap(), None);
+    assert_eq!(snapshot3.get(b"c").unwrap(), Some(Bytes::from_static(b"1")));
+    check_lsm_iter_result_by_key(
+        &mut snapshot3.scan(Bound::Unbounded, Bound::Unbounded).unwrap(),
+        vec![
+            (Bytes::from("a"), Bytes::from("2")),
+            (Bytes::from("c"), Bytes::from("1")),
+        ],
+    );
+    assert_eq!(snapshot4.get(b"a").unwrap(), Some(Bytes::from_static(b"3")));
+    assert_eq!(snapshot4.get(b"b").unwrap(), Some(Bytes::from_static(b"3")));
+    assert_eq!(snapshot4.get(b"c").unwrap(), Some(Bytes::from_static(b"1")));
+    check_lsm_iter_result_by_key(
+        &mut snapshot4.scan(Bound::Unbounded, Bound::Unbounded).unwrap(),
+        vec![
+            (Bytes::from("a"), Bytes::from("3")),
+            (Bytes::from("b"), Bytes::from("3")),
+            (Bytes::from("c"), Bytes::from("1")),
+        ],
+    );
+    assert_eq!(snapshot5.get(b"a").unwrap(), Some(Bytes::from_static(b"4")));
+    assert_eq!(snapshot5.get(b"b").unwrap(), Some(Bytes::from_static(b"3")));
+    assert_eq!(snapshot5.get(b"c").unwrap(), Some(Bytes::from_static(b"1")));
+    check_lsm_iter_result_by_key(
+        &mut snapshot5.scan(Bound::Unbounded, Bound::Unbounded).unwrap(),
+        vec![
+            (Bytes::from("a"), Bytes::from("4")),
+            (Bytes::from("b"), Bytes::from("3")),
+            (Bytes::from("c"), Bytes::from("1")),
+        ],
+    );
+    assert_eq!(snapshot6.get(b"a").unwrap(), Some(Bytes::from_static(b"4")));
+    assert_eq!(snapshot6.get(b"b").unwrap(), None);
+    assert_eq!(snapshot6.get(b"c").unwrap(), Some(Bytes::from_static(b"5")));
+    check_lsm_iter_result_by_key(
+        &mut snapshot6.scan(Bound::Unbounded, Bound::Unbounded).unwrap(),
+        vec![
+            (Bytes::from("a"), Bytes::from("4")),
+            (Bytes::from("c"), Bytes::from("5")),
+        ],
+    );
+}
+
+#[test]
+fn test_task2_lsm_iterator_mvcc() {
+    let dir = tempdir().unwrap();
+    let mut options = LsmStorageOptions::default_for_week2_test(CompactionOptions::NoCompaction);
+    options.enable_wal = true;
+    let storage = MiniLsm::open(&dir, options.clone()).unwrap();
+    storage.put(b"a", b"1").unwrap();
+    storage.put(b"b", b"1").unwrap();
+    let snapshot1 = storage.new_txn().unwrap();
+    storage.put(b"a", b"2").unwrap();
+    let snapshot2 = storage.new_txn().unwrap();
+    storage.delete(b"b").unwrap();
+    storage.put(b"c", b"1").unwrap();
+    let snapshot3 = storage.new_txn().unwrap();
+    storage.force_flush().unwrap();
+    assert_eq!(snapshot1.get(b"a").unwrap(), Some(Bytes::from_static(b"1")));
+    assert_eq!(snapshot1.get(b"b").unwrap(), Some(Bytes::from_static(b"1")));
+    assert_eq!(snapshot1.get(b"c").unwrap(), None);
+    check_lsm_iter_result_by_key(
+        &mut snapshot1.scan(Bound::Unbounded, Bound::Unbounded).unwrap(),
+        vec![
+            (Bytes::from("a"), Bytes::from("1")),
+            (Bytes::from("b"), Bytes::from("1")),
+        ],
+    );
+    assert_eq!(snapshot2.get(b"a").unwrap(), Some(Bytes::from_static(b"2")));
+    assert_eq!(snapshot2.get(b"b").unwrap(), Some(Bytes::from_static(b"1")));
+    assert_eq!(snapshot2.get(b"c").unwrap(), None);
+    check_lsm_iter_result_by_key(
+        &mut snapshot2.scan(Bound::Unbounded, Bound::Unbounded).unwrap(),
+        vec![
+            (Bytes::from("a"), Bytes::from("2")),
+            (Bytes::from("b"), Bytes::from("1")),
+        ],
+    );
+    assert_eq!(snapshot3.get(b"a").unwrap(), Some(Bytes::from_static(b"2")));
+    assert_eq!(snapshot3.get(b"b").unwrap(), None);
+    assert_eq!(snapshot3.get(b"c").unwrap(), Some(Bytes::from_static(b"1")));
+    check_lsm_iter_result_by_key(
+        &mut snapshot3.scan(Bound::Unbounded, Bound::Unbounded).unwrap(),
+        vec![
+            (Bytes::from("a"), Bytes::from("2")),
+            (Bytes::from("c"), Bytes::from("1")),
+        ],
+    );
+    storage.put(b"a", b"3").unwrap();
+    storage.put(b"b", b"3").unwrap();
+    let snapshot4 = storage.new_txn().unwrap();
+    storage.put(b"a", b"4").unwrap();
+    let snapshot5 = storage.new_txn().unwrap();
+    storage.delete(b"b").unwrap();
+    storage.put(b"c", b"5").unwrap();
+    let snapshot6 = storage.new_txn().unwrap();
+    storage.force_flush().unwrap();
+    assert_eq!(snapshot1.get(b"a").unwrap(), Some(Bytes::from_static(b"1")));
+    assert_eq!(snapshot1.get(b"b").unwrap(), Some(Bytes::from_static(b"1")));
+    assert_eq!(snapshot1.get(b"c").unwrap(), None);
+    check_lsm_iter_result_by_key(
+        &mut snapshot1.scan(Bound::Unbounded, Bound::Unbounded).unwrap(),
+        vec![
+            (Bytes::from("a"), Bytes::from("1")),
+            (Bytes::from("b"), Bytes::from("1")),
+        ],
+    );
+    assert_eq!(snapshot2.get(b"a").unwrap(), Some(Bytes::from_static(b"2")));
+    assert_eq!(snapshot2.get(b"b").unwrap(), Some(Bytes::from_static(b"1")));
+    assert_eq!(snapshot2.get(b"c").unwrap(), None);
+    check_lsm_iter_result_by_key(
+        &mut snapshot2.scan(Bound::Unbounded, Bound::Unbounded).unwrap(),
+        vec![
+            (Bytes::from("a"), Bytes::from("2")),
+            (Bytes::from("b"), Bytes::from("1")),
+        ],
+    );
+    assert_eq!(snapshot3.get(b"a").unwrap(), Some(Bytes::from_static(b"2")));
+    assert_eq!(snapshot3.get(b"b").unwrap(), None);
+    assert_eq!(snapshot3.get(b"c").unwrap(), Some(Bytes::from_static(b"1")));
+    check_lsm_iter_result_by_key(
+        &mut snapshot3.scan(Bound::Unbounded, Bound::Unbounded).unwrap(),
+        vec![
+            (Bytes::from("a"), Bytes::from("2")),
+            (Bytes::from("c"), Bytes::from("1")),
+        ],
+    );
+    assert_eq!(snapshot4.get(b"a").unwrap(), Some(Bytes::from_static(b"3")));
+    assert_eq!(snapshot4.get(b"b").unwrap(), Some(Bytes::from_static(b"3")));
+    assert_eq!(snapshot4.get(b"c").unwrap(), Some(Bytes::from_static(b"1")));
+    check_lsm_iter_result_by_key(
+        &mut snapshot4.scan(Bound::Unbounded, Bound::Unbounded).unwrap(),
+        vec![
+            (Bytes::from("a"), Bytes::from("3")),
+            (Bytes::from("b"), Bytes::from("3")),
+            (Bytes::from("c"), Bytes::from("1")),
+        ],
+    );
+    assert_eq!(snapshot5.get(b"a").unwrap(), Some(Bytes::from_static(b"4")));
+    assert_eq!(snapshot5.get(b"b").unwrap(), Some(Bytes::from_static(b"3")));
+    assert_eq!(snapshot5.get(b"c").unwrap(), Some(Bytes::from_static(b"1")));
+    check_lsm_iter_result_by_key(
+        &mut snapshot5.scan(Bound::Unbounded, Bound::Unbounded).unwrap(),
+        vec![
+            (Bytes::from("a"), Bytes::from("4")),
+            (Bytes::from("b"), Bytes::from("3")),
+            (Bytes::from("c"), Bytes::from("1")),
+        ],
+    );
+    assert_eq!(snapshot6.get(b"a").unwrap(), Some(Bytes::from_static(b"4")));
+    assert_eq!(snapshot6.get(b"b").unwrap(), None);
+    assert_eq!(snapshot6.get(b"c").unwrap(), Some(Bytes::from_static(b"5")));
+    check_lsm_iter_result_by_key(
+        &mut snapshot6.scan(Bound::Unbounded, Bound::Unbounded).unwrap(),
+        vec![
+            (Bytes::from("a"), Bytes::from("4")),
+            (Bytes::from("c"), Bytes::from("5")),
+        ],
+    );
+}
+
+#[test]
+fn test_task3_sst_ts() {
+    let mut builder = SsTableBuilder::new(16);
+    builder.add(KeySlice::for_testing_from_slice_with_ts(b"11", 1), b"11");
+    builder.add(KeySlice::for_testing_from_slice_with_ts(b"22", 2), b"22");
+    builder.add(KeySlice::for_testing_from_slice_with_ts(b"33", 3), b"11");
+    builder.add(KeySlice::for_testing_from_slice_with_ts(b"44", 4), b"22");
+    builder.add(KeySlice::for_testing_from_slice_with_ts(b"55", 5), b"11");
+    builder.add(KeySlice::for_testing_from_slice_with_ts(b"66", 6), b"22");
+    let dir = tempdir().unwrap();
+    let sst = builder.build_for_test(dir.path().join("1.sst")).unwrap();
+    assert_eq!(sst.max_ts(), 6);
+}

mini-lsm-starter/src/table.rs

@@ -96,6 +96,8 @@ pub struct SsTable {
     first_key: KeyBytes,
     last_key: KeyBytes,
     pub(crate) bloom: Option<Bloom>,
+    /// The maximum timestamp stored in this SST, implemented in week 3.
+    max_ts: u64,
 }
 
 impl SsTable {
@@ -125,6 +127,7 @@ impl SsTable {
             first_key,
             last_key,
             bloom: None,
+            max_ts: 0,
         }
     }
 
@@ -165,4 +168,8 @@ impl SsTable {
     pub fn sst_id(&self) -> usize {
         self.id
     }
+
+    pub fn max_ts(&self) -> u64 {
+        self.max_ts
+    }
 }

mini-lsm/src/table.rs

@@ -133,6 +133,7 @@ pub struct SsTable {
     first_key: KeyBytes,
     last_key: KeyBytes,
     pub(crate) bloom: Option<Bloom>,
+    max_ts: u64,
 }
 impl SsTable {
     #[cfg(test)]
@@ -160,6 +161,7 @@ impl SsTable {
             id,
             block_cache,
             bloom: Some(bloom_filter),
+            max_ts: 0,
         })
     }
 
@@ -179,6 +181,7 @@ impl SsTable {
             first_key,
             last_key,
             bloom: None,
+            max_ts: 0,
         }
     }
 
@@ -240,4 +243,8 @@ impl SsTable {
     pub fn sst_id(&self) -> usize {
         self.id
     }
+
+    pub fn max_ts(&self) -> u64 {
+        self.max_ts
+    }
 }