maokaihe/mini_lsm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add compaction thread to lsm storage

Browse Source 
Signed-off-by: Alex Chi <iskyzh@gmail.com>
This commit is contained in:
Alex Chi
2024-01-18 17:51:24 +08:00
parent 9620a57211
commit ce33f62be6
7 changed files with 229 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
mini-lsm/src/compact/leveled.rs
View File

@@ -1,6 +1,6 @@
use std::collections::HashSet;
use crate::lsm_storage::LsmStorageInner;
use crate::lsm_storage::LsmStorageState;
pub struct LeveledCompactionTask {
// if upper_level is `None`, then it is L0 compaction
@@ -10,6 +10,7 @@ pub struct LeveledCompactionTask {
pub lower_level_sst_ids: Vec<usize>,
}
#[derive(Debug, Clone)]
pub struct LeveledCompactionOptions {
pub level_size_multiplier: usize,
pub level0_file_num_compaction_trigger: usize,
@@ -28,7 +29,7 @@ impl LeveledCompactionController {
fn find_overlapping_ssts(
&self,
snapshot: &LsmStorageInner,
snapshot: &LsmStorageState,
sst_ids: &[usize],
in_level: usize,
) -> Vec<usize> {
@@ -58,7 +59,7 @@ impl LeveledCompactionController {
pub fn generate_compaction_task(
&self,
snapshot: &LsmStorageInner,
snapshot: &LsmStorageState,
) -> Option<LeveledCompactionTask> {
// step 1: compute target level size
let mut target_level_size = (0..self.options.max_levels).map(|_| 0).collect::<Vec<_>>(); // exclude level 0
@@ -149,10 +150,10 @@ impl LeveledCompactionController {
pub fn apply_compaction_result(
&self,
snapshot: &LsmStorageInner,
snapshot: &LsmStorageState,
task: &LeveledCompactionTask,
output: &[usize],
) -> (LsmStorageInner, Vec<usize>) {
) -> (LsmStorageState, Vec<usize>) {
let mut snapshot = snapshot.clone();
let mut files_to_remove = Vec::new();
let mut upper_level_sst_ids_set = task

9
mini-lsm/src/compact/simple_leveled.rs
View File

@@ -1,5 +1,6 @@
use crate::lsm_storage::LsmStorageInner;
use crate::lsm_storage::LsmStorageState;
#[derive(Debug, Clone)]
pub struct SimpleLeveledCompactionOptions {
pub size_ratio_percent: usize,
pub level0_file_num_compaction_trigger: usize,
@@ -25,7 +26,7 @@ impl SimpleLeveledCompactionController {
pub fn generate_compaction_task(
&self,
snapshot: &LsmStorageInner,
snapshot: &LsmStorageState,
) -> Option<SimpleLeveledCompactionTask> {
let mut level_sizes = Vec::new();
level_sizes.push(snapshot.l0_sstables.len());
@@ -64,10 +65,10 @@ impl SimpleLeveledCompactionController {
pub fn apply_compaction_result(
&self,
snapshot: &LsmStorageInner,
snapshot: &LsmStorageState,
task: &SimpleLeveledCompactionTask,
output: &[usize],
) -> (LsmStorageInner, Vec<usize>) {
) -> (LsmStorageState, Vec<usize>) {
let mut snapshot = snapshot.clone();
let mut files_to_remove = Vec::new();
if let Some(upper_level) = task.upper_level {

9
mini-lsm/src/compact/tiered.rs
View File

@@ -1,11 +1,12 @@
use std::collections::HashMap;
use crate::lsm_storage::LsmStorageInner;
use crate::lsm_storage::LsmStorageState;
pub struct TieredCompactionTask {
pub tiers: Vec<(usize, Vec<usize>)>,
}
#[derive(Debug, Clone)]
pub struct TieredCompactionOptions {
pub level0_file_num_compaction_trigger: usize,
pub max_size_amplification_percent: usize,
@@ -24,7 +25,7 @@ impl TieredCompactionController {
pub fn generate_compaction_task(
&self,
snapshot: &LsmStorageInner,
snapshot: &LsmStorageState,
) -> Option<TieredCompactionTask> {
assert!(
snapshot.l0_sstables.is_empty(),
@@ -87,10 +88,10 @@ impl TieredCompactionController {
pub fn apply_compaction_result(
&self,
snapshot: &LsmStorageInner,
snapshot: &LsmStorageState,
task: &TieredCompactionTask,
output: &[usize],
) -> (LsmStorageInner, Vec<usize>) {
) -> (LsmStorageState, Vec<usize>) {
assert!(
snapshot.l0_sstables.is_empty(),
"should not add l0 ssts in tiered compaction"