add compaction tests and fix bugs in compaction

Signed-off-by: Alex Chi <iskyzh@gmail.com>

mini-lsm-mvcc/src/compact.rs

@@ -98,6 +98,7 @@ impl CompactionController {
     }
 }
 
+#[derive(Debug, Clone)]
 pub enum CompactionOptions {
     /// Leveled compaction with partial compaction + dynamic level support (= RocksDB's Leveled
     /// Compaction)
@@ -309,25 +310,25 @@ impl LsmStorageInner {
         };
         println!("running compaction task: {:?}", task);
         let sstables = self.compact(&task)?;
-        let files_added = sstables.len();
         let output = sstables.iter().map(|x| x.sst_id()).collect::<Vec<_>>();
         let ssts_to_remove = {
             let state_lock = self.state_lock.lock();
-            let (mut snapshot, files_to_remove) = self
-                .compaction_controller
-                .apply_compaction_result(&self.state.read(), &task, &output);
-            let mut ssts_to_remove = Vec::with_capacity(files_to_remove.len());
-            for file_to_remove in &files_to_remove {
-                let result = snapshot.sstables.remove(file_to_remove);
-                assert!(result.is_some(), "cannot remove {}.sst", file_to_remove);
-                ssts_to_remove.push(result.unwrap());
-            }
+            let mut snapshot = self.state.read().as_ref().clone();
             let mut new_sst_ids = Vec::new();
             for file_to_add in sstables {
                 new_sst_ids.push(file_to_add.sst_id());
                 let result = snapshot.sstables.insert(file_to_add.sst_id(), file_to_add);
                 assert!(result.is_none());
             }
+            let (mut snapshot, files_to_remove) = self
+                .compaction_controller
+                .apply_compaction_result(&snapshot, &task, &output);
+            let mut ssts_to_remove = Vec::with_capacity(files_to_remove.len());
+            for file_to_remove in &files_to_remove {
+                let result = snapshot.sstables.remove(file_to_remove);
+                assert!(result.is_some(), "cannot remove {}.sst", file_to_remove);
+                ssts_to_remove.push(result.unwrap());
+            }
             let mut state = self.state.write();
             *state = Arc::new(snapshot);
             drop(state);
@@ -339,9 +340,10 @@ impl LsmStorageInner {
             ssts_to_remove
         };
         println!(
-            "compaction finished: {} files removed, {} files added",
+            "compaction finished: {} files removed, {} files added, output={:?}",
             ssts_to_remove.len(),
-            files_added
+            output.len(),
+            output
         );
         for sst in ssts_to_remove {
             std::fs::remove_file(self.path_of_sst(sst.sst_id()))?;

mini-lsm-mvcc/src/compact/leveled.rs

@@ -124,11 +124,11 @@ impl LeveledCompactionController {
                 "target level sizes: {:?}, real level sizes: {:?}, base_level: {}",
                 target_level_size
                     .iter()
-                    .map(|x| format!("{}MB", x / 1024 / 1024))
+                    .map(|x| format!("{:.3}MB", *x as f64 / 1024.0 / 1024.0))
                     .collect::<Vec<_>>(),
                 real_level_size
                     .iter()
-                    .map(|x| format!("{}MB", x / 1024 / 1024))
+                    .map(|x| format!("{:.3}MB", *x as f64 / 1024.0 / 1024.0))
                     .collect::<Vec<_>>(),
                 base_level,
             );

mini-lsm-mvcc/src/compact/simple_leveled.rs

@@ -1,3 +1,5 @@
+use std::collections::HashSet;
+
 use serde::{Deserialize, Serialize};
 
 use crate::lsm_storage::LsmStorageState;
@@ -95,12 +97,20 @@ impl SimpleLeveledCompactionController {
             files_to_remove.extend(&snapshot.levels[upper_level - 1].1);
             snapshot.levels[upper_level - 1].1.clear();
         } else {
-            assert_eq!(
-                task.upper_level_sst_ids, snapshot.l0_sstables,
-                "sst mismatched"
-            );
-            files_to_remove.extend(&snapshot.l0_sstables);
-            snapshot.l0_sstables.clear();
+            files_to_remove.extend(&task.upper_level_sst_ids);
+            let mut l0_ssts_compacted = task
+                .upper_level_sst_ids
+                .iter()
+                .copied()
+                .collect::<HashSet<_>>();
+            let new_l0_sstables = snapshot
+                .l0_sstables
+                .iter()
+                .copied()
+                .filter(|x| !l0_ssts_compacted.remove(x))
+                .collect::<Vec<_>>();
+            assert!(l0_ssts_compacted.is_empty());
+            snapshot.l0_sstables = new_l0_sstables;
         }
         assert_eq!(
             task.lower_level_sst_ids,

mini-lsm-mvcc/src/lsm_storage.rs

@@ -94,6 +94,16 @@ impl LsmStorageOptions {
             num_memtable_limit: 2,
         }
     }
+
+    pub fn default_for_week2_test(compaction_options: CompactionOptions) -> Self {
+        Self {
+            block_size: 4096,
+            target_sst_size: 1 << 20, // 1MB
+            compaction_options,
+            enable_wal: false,
+            num_memtable_limit: 2,
+        }
+    }
 }
 
 fn range_overlap(

mini-lsm-mvcc/src/tests.rs

@@ -7,3 +7,6 @@ mod week1_day5;
 mod week1_day6;
 mod week1_day7;
 mod week2_day1;
+mod week2_day2;
+mod week2_day3;
+mod week2_day4;

mini-lsm-mvcc/src/tests/week2_day2.rs

@@ -0,0 +1 @@
+../../../mini-lsm/src/tests/week2_day2.rs

mini-lsm-mvcc/src/tests/week2_day3.rs

@@ -0,0 +1 @@
+../../../mini-lsm/src/tests/week2_day3.rs

mini-lsm-mvcc/src/tests/week2_day4.rs

@@ -0,0 +1 @@
+../../../mini-lsm/src/tests/week2_day4.rs