From 9eab75ec1a8331c817a712f0b8f5c1d3130ba14f Mon Sep 17 00:00:00 2001 From: Alex Chi Z Date: Tue, 30 Jan 2024 13:39:58 +0800 Subject: [PATCH] add 3.5 tests Signed-off-by: Alex Chi Z --- mini-lsm-book/src/week3-05-txn-occ.md | 3 +- mini-lsm-mvcc/src/mvcc/watermark.rs | 6 +++ mini-lsm-mvcc/src/tests.rs | 1 + mini-lsm-mvcc/src/tests/week3_day4.rs | 5 --- mini-lsm-mvcc/src/tests/week3_day5.rs | 56 +++++++++++++++++++++++++++ mini-lsm/src/tests/harness.rs | 2 + mini-lsm/src/tests/week2_day1.rs | 8 ++-- 7 files changed, 70 insertions(+), 11 deletions(-) create mode 100644 mini-lsm-mvcc/src/tests/week3_day5.rs diff --git a/mini-lsm-book/src/week3-05-txn-occ.md b/mini-lsm-book/src/week3-05-txn-occ.md index 0a214a1..ea5403a 100644 --- a/mini-lsm-book/src/week3-05-txn-occ.md +++ b/mini-lsm-book/src/week3-05-txn-occ.md @@ -1,6 +1,6 @@ # Transaction and Optimistic Concurrency Control -In this chapter, you will implement all interfaces of `Transaction`. Your implementation will maintain a private workspace for modifications inside a transaction, and commit them in batch, so that all modifications within the transaction will only be visible to the transaction itself until commit. +In this chapter, you will implement all interfaces of `Transaction`. Your implementation will maintain a private workspace for modifications inside a transaction, and commit them in batch, so that all modifications within the transaction will only be visible to the transaction itself until commit. We only check for conflicts (i.e., serializable conflicts) when commit, and this is optimistic concurrency control. To run test cases, @@ -45,6 +45,7 @@ Your commit implementation should simply collect all key-value pairs from the lo * With all the things we have implemented up to this point, does the system satisfy snapshot isolation? If not, what else do we need to do to support snapshot isolation? (Note: snapshot isolation is different from serializable snapshot isolation we will talk about in the next chapter) * What if the user wants to batch import data (i.e., 1TB?) If they use the transaction API to do that, will you give them some advice? Is there any opportunity to optimize for this case? +* What is optimistic concurrency control? What would the system be like if we implement pessimistic concurrency control instead in Mini-LSM? ## Bonus Tasks diff --git a/mini-lsm-mvcc/src/mvcc/watermark.rs b/mini-lsm-mvcc/src/mvcc/watermark.rs index 069ed4f..668c919 100644 --- a/mini-lsm-mvcc/src/mvcc/watermark.rs +++ b/mini-lsm-mvcc/src/mvcc/watermark.rs @@ -4,6 +4,12 @@ pub struct Watermark { readers: BTreeMap, } +impl Default for Watermark { + fn default() -> Self { + Self::new() + } +} + impl Watermark { pub fn new() -> Self { Self { diff --git a/mini-lsm-mvcc/src/tests.rs b/mini-lsm-mvcc/src/tests.rs index f4851f6..350910e 100644 --- a/mini-lsm-mvcc/src/tests.rs +++ b/mini-lsm-mvcc/src/tests.rs @@ -16,3 +16,4 @@ mod week3_day1; mod week3_day2; mod week3_day3; mod week3_day4; +mod week3_day5; diff --git a/mini-lsm-mvcc/src/tests/week3_day4.rs b/mini-lsm-mvcc/src/tests/week3_day4.rs index f0e6326..edab25f 100644 --- a/mini-lsm-mvcc/src/tests/week3_day4.rs +++ b/mini-lsm-mvcc/src/tests/week3_day4.rs @@ -1,15 +1,10 @@ -use std::ops::Bound; - use bytes::Bytes; use tempfile::tempdir; use crate::{ compact::CompactionOptions, - key::KeySlice, lsm_storage::{LsmStorageOptions, MiniLsm, WriteBatchRecord}, mvcc::watermark::Watermark, - table::SsTableBuilder, - tests::harness::check_lsm_iter_result_by_key, }; use super::harness::{check_iter_result_by_key, construct_merge_iterator_over_storage}; diff --git a/mini-lsm-mvcc/src/tests/week3_day5.rs b/mini-lsm-mvcc/src/tests/week3_day5.rs new file mode 100644 index 0000000..b1e2ab9 --- /dev/null +++ b/mini-lsm-mvcc/src/tests/week3_day5.rs @@ -0,0 +1,56 @@ +use std::ops::Bound; + +use bytes::Bytes; +use tempfile::tempdir; + +use crate::{ + compact::CompactionOptions, + lsm_storage::{LsmStorageOptions, MiniLsm}, + tests::harness::check_lsm_iter_result_by_key, +}; + +#[test] +fn test_txn_integration() { + let dir = tempdir().unwrap(); + let options = LsmStorageOptions::default_for_week2_test(CompactionOptions::NoCompaction); + let storage = MiniLsm::open(&dir, options.clone()).unwrap(); + let txn1 = storage.new_txn().unwrap(); + let txn2 = storage.new_txn().unwrap(); + txn1.put(b"test1", b"233"); + txn2.put(b"test2", b"233"); + check_lsm_iter_result_by_key( + &mut txn1.scan(Bound::Unbounded, Bound::Unbounded).unwrap(), + vec![(Bytes::from("test1"), Bytes::from("233"))], + ); + check_lsm_iter_result_by_key( + &mut txn2.scan(Bound::Unbounded, Bound::Unbounded).unwrap(), + vec![(Bytes::from("test2"), Bytes::from("233"))], + ); + let txn3 = storage.new_txn().unwrap(); + check_lsm_iter_result_by_key( + &mut txn3.scan(Bound::Unbounded, Bound::Unbounded).unwrap(), + vec![], + ); + txn1.commit().unwrap(); + txn2.commit().unwrap(); + check_lsm_iter_result_by_key( + &mut txn3.scan(Bound::Unbounded, Bound::Unbounded).unwrap(), + vec![], + ); + drop(txn3); + check_lsm_iter_result_by_key( + &mut storage.scan(Bound::Unbounded, Bound::Unbounded).unwrap(), + vec![ + (Bytes::from("test1"), Bytes::from("233")), + (Bytes::from("test2"), Bytes::from("233")), + ], + ); + let txn4 = storage.new_txn().unwrap(); + check_lsm_iter_result_by_key( + &mut txn4.scan(Bound::Unbounded, Bound::Unbounded).unwrap(), + vec![ + (Bytes::from("test1"), Bytes::from("233")), + (Bytes::from("test2"), Bytes::from("233")), + ], + ); +} diff --git a/mini-lsm/src/tests/harness.rs b/mini-lsm/src/tests/harness.rs index 6fbb19f..4b0d7e3 100644 --- a/mini-lsm/src/tests/harness.rs +++ b/mini-lsm/src/tests/harness.rs @@ -114,6 +114,7 @@ where assert!(!iter.is_valid()); } +#[allow(dead_code)] pub fn check_iter_result_by_key_and_ts(iter: &mut I, expected: Vec<((Bytes, u64), Bytes)>) where I: for<'a> StorageIterator = KeySlice<'a>>, @@ -192,6 +193,7 @@ pub fn generate_sst( builder.build(id, block_cache, path.as_ref()).unwrap() } +#[allow(dead_code)] pub fn generate_sst_with_ts( id: usize, path: impl AsRef, diff --git a/mini-lsm/src/tests/week2_day1.rs b/mini-lsm/src/tests/week2_day1.rs index 0124968..15128f2 100644 --- a/mini-lsm/src/tests/week2_day1.rs +++ b/mini-lsm/src/tests/week2_day1.rs @@ -7,12 +7,10 @@ use week2_day1::harness::construct_merge_iterator_over_storage; use super::*; use crate::{ - iterators::{ - concat_iterator::SstConcatIterator, merge_iterator::MergeIterator, StorageIterator, - }, + iterators::{concat_iterator::SstConcatIterator, StorageIterator}, key::{KeySlice, TS_ENABLED}, - lsm_storage::{LsmStorageInner, LsmStorageOptions, LsmStorageState}, - table::{SsTable, SsTableBuilder, SsTableIterator}, + lsm_storage::{LsmStorageInner, LsmStorageOptions}, + table::{SsTable, SsTableBuilder}, }; #[test]