diff --git a/mini-lsm-book-wip/.gitignore b/mini-lsm-book-wip/.gitignore deleted file mode 100644 index 7585238..0000000 --- a/mini-lsm-book-wip/.gitignore +++ /dev/null @@ -1 +0,0 @@ -book diff --git a/mini-lsm-book-wip/book.toml b/mini-lsm-book-wip/book.toml deleted file mode 100644 index 07a7670..0000000 --- a/mini-lsm-book-wip/book.toml +++ /dev/null @@ -1,6 +0,0 @@ -[book] -authors = ["Alex Chi"] -language = "en" -multilingual = false -src = "src" -title = "LSM in a Week" diff --git a/mini-lsm-book-wip/src/00-get-started.md b/mini-lsm-book-wip/src/00-get-started.md deleted file mode 100644 index 7e0bdca..0000000 --- a/mini-lsm-book-wip/src/00-get-started.md +++ /dev/null @@ -1 +0,0 @@ -# Get Started diff --git a/mini-lsm-book-wip/src/00-overview.md b/mini-lsm-book-wip/src/00-overview.md deleted file mode 100644 index 8268222..0000000 --- a/mini-lsm-book-wip/src/00-overview.md +++ /dev/null @@ -1,41 +0,0 @@ -# Overview - -## Structure - -chapters + snacks, clear goal - -implement, think, try by yourself - -required tasks, check your understanding questions, bonus tasks - -## Testing - -exploring and understanding is more important than passing all the test cases - -testing basic requirements, not the internal structure or something - -## Solution - -### Checkpoints - -the final version, but many things can be simplified, read the docs - -comments / tests / not up-to-date with the starter code - -### How to use the solutions - -## Feedbacks - -join the Discord server, your feedback is important, thank GitHub users - -## License - -### Free forever? - -### Video lectures + Review Service + Office Hour? - -should have a separate preface (before you start) chapter? and what's new with v2? - -## Target audience? - -## What will you get after taking this course... diff --git a/mini-lsm-book-wip/src/00-v1.md b/mini-lsm-book-wip/src/00-v1.md deleted file mode 100644 index 938b235..0000000 --- a/mini-lsm-book-wip/src/00-v1.md +++ /dev/null @@ -1 +0,0 @@ -# Mini-LSM v1 diff --git a/mini-lsm-book-wip/src/01-block.md b/mini-lsm-book-wip/src/01-block.md deleted file mode 100644 index ab54542..0000000 --- a/mini-lsm-book-wip/src/01-block.md +++ /dev/null @@ -1,3 +0,0 @@ -# Store key-value pairs in little blocks - - diff --git a/mini-lsm-book-wip/src/02-sst.md b/mini-lsm-book-wip/src/02-sst.md deleted file mode 100644 index ae692a7..0000000 --- a/mini-lsm-book-wip/src/02-sst.md +++ /dev/null @@ -1,3 +0,0 @@ -# And make them into an SST - - \ No newline at end of file diff --git a/mini-lsm-book-wip/src/03-memtable.md b/mini-lsm-book-wip/src/03-memtable.md deleted file mode 100644 index f05503a..0000000 --- a/mini-lsm-book-wip/src/03-memtable.md +++ /dev/null @@ -1,3 +0,0 @@ -# Now it's time to merge everything - - \ No newline at end of file diff --git a/mini-lsm-book-wip/src/04-engine.md b/mini-lsm-book-wip/src/04-engine.md deleted file mode 100644 index 15732c3..0000000 --- a/mini-lsm-book-wip/src/04-engine.md +++ /dev/null @@ -1,3 +0,0 @@ -# The engine is on fire - - \ No newline at end of file diff --git a/mini-lsm-book-wip/src/05-compaction.md b/mini-lsm-book-wip/src/05-compaction.md deleted file mode 100644 index 3832c60..0000000 --- a/mini-lsm-book-wip/src/05-compaction.md +++ /dev/null @@ -1,3 +0,0 @@ -# Let's do something in the background - - \ No newline at end of file diff --git a/mini-lsm-book-wip/src/06-recovery.md b/mini-lsm-book-wip/src/06-recovery.md deleted file mode 100644 index bce3a2b..0000000 --- a/mini-lsm-book-wip/src/06-recovery.md +++ /dev/null @@ -1,3 +0,0 @@ -# Be careful when the system crashes - - \ No newline at end of file diff --git a/mini-lsm-book-wip/src/07-bloom-filter.md b/mini-lsm-book-wip/src/07-bloom-filter.md deleted file mode 100644 index 5e4f6e2..0000000 --- a/mini-lsm-book-wip/src/07-bloom-filter.md +++ /dev/null @@ -1,3 +0,0 @@ -# A good bloom filter makes life easier - - \ No newline at end of file diff --git a/mini-lsm-book-wip/src/08-key-compression.md b/mini-lsm-book-wip/src/08-key-compression.md deleted file mode 100644 index a282c22..0000000 --- a/mini-lsm-book-wip/src/08-key-compression.md +++ /dev/null @@ -1,3 +0,0 @@ -# Save some space, hopefully - - \ No newline at end of file diff --git a/mini-lsm-book-wip/src/09-whats-next.md b/mini-lsm-book-wip/src/09-whats-next.md deleted file mode 100644 index 4a10510..0000000 --- a/mini-lsm-book-wip/src/09-whats-next.md +++ /dev/null @@ -1,3 +0,0 @@ -# What's next - - \ No newline at end of file diff --git a/mini-lsm-book-wip/src/SUMMARY.md b/mini-lsm-book-wip/src/SUMMARY.md deleted file mode 100644 index 1c41ac2..0000000 --- a/mini-lsm-book-wip/src/SUMMARY.md +++ /dev/null @@ -1,41 +0,0 @@ -# LSM in a Week - -[Overview](./00-overview.md) -[Get Started](./00-get-started.md) - -- [Week 1: Mini-LSM](./week1-overview.md) - - [Memtables](./week1-01-memtable.md) - - [Blocks](./week1-02-block.md) - - [Sorted String Table (SST)](./week1-03-sst.md) - - [Merge Iterators](./week1-04-merge-iterator.md) - - [Read Path](./week1-05-read-path.md) - - [Write Path](./week1-06-write-path.md) - - [Snack Time: SST Optimizations](./week1-07-sst-optimizations.md) - -- [Week 2: Compaction and Persistence](./week2-overview.md) - - [Compaction Implementation](./week2-01-compaction.md) - - [Simple Compaction Strategy](./week2-02-simple.md) - - [Tiered Compaction Strategy](./week2-03-tiered.md) - - [Leveled Compaction Strategy](./week2-04-leveled.md) - - [Manifest](./week2-05-manifest.md) - - [Write-Ahead Log (WAL)](./week2-06-wal.md) - - [Snack Time: Batch Write](./week2-07-batch-write.md) - -- [Week 3: MVCC](./week3-overview.md) - -# The Rest of Your Life (TBD) - ---- - -# Mini-LSM v1 - -- [Overview](./00-v1.md) - - [Store key-value pairs in little blocks](./01-block.md) - - [And make them into an SST](./02-sst.md) - - [Now it's time to merge everything](./03-memtable.md) - - [The engine is on fire](./04-engine.md) - - [Let's do something in the background](./05-compaction.md) - - [Be careful when the system crashes](./06-recovery.md) - - [A good bloom filter makes life easier](./07-bloom-filter.md) - - [Save some space, hopefully](./08-key-compression.md) - - [What's next](./09-whats-next.md) diff --git a/mini-lsm-book-wip/src/week1-01-memtable.md b/mini-lsm-book-wip/src/week1-01-memtable.md deleted file mode 100644 index 3abae24..0000000 --- a/mini-lsm-book-wip/src/week1-01-memtable.md +++ /dev/null @@ -1 +0,0 @@ -# Memtables diff --git a/mini-lsm-book-wip/src/week1-02-block.md b/mini-lsm-book-wip/src/week1-02-block.md deleted file mode 100644 index 54e0a5f..0000000 --- a/mini-lsm-book-wip/src/week1-02-block.md +++ /dev/null @@ -1 +0,0 @@ -# Blocks diff --git a/mini-lsm-book-wip/src/week1-03-sst.md b/mini-lsm-book-wip/src/week1-03-sst.md deleted file mode 100644 index 0794bfd..0000000 --- a/mini-lsm-book-wip/src/week1-03-sst.md +++ /dev/null @@ -1 +0,0 @@ -# Sorted String Table (SST) diff --git a/mini-lsm-book-wip/src/week1-04-merge-iterator.md b/mini-lsm-book-wip/src/week1-04-merge-iterator.md deleted file mode 100644 index 43e07b1..0000000 --- a/mini-lsm-book-wip/src/week1-04-merge-iterator.md +++ /dev/null @@ -1 +0,0 @@ -# Merge Iterators diff --git a/mini-lsm-book-wip/src/week1-05-read-path.md b/mini-lsm-book-wip/src/week1-05-read-path.md deleted file mode 100644 index 3c3129c..0000000 --- a/mini-lsm-book-wip/src/week1-05-read-path.md +++ /dev/null @@ -1 +0,0 @@ -# Read Path diff --git a/mini-lsm-book-wip/src/week1-06-write-path.md b/mini-lsm-book-wip/src/week1-06-write-path.md deleted file mode 100644 index a35123e..0000000 --- a/mini-lsm-book-wip/src/week1-06-write-path.md +++ /dev/null @@ -1 +0,0 @@ -# Write Path diff --git a/mini-lsm-book-wip/src/week1-overview.md b/mini-lsm-book-wip/src/week1-overview.md deleted file mode 100644 index 339129b..0000000 --- a/mini-lsm-book-wip/src/week1-overview.md +++ /dev/null @@ -1,4 +0,0 @@ -# Overview - -key functionalities index - diff --git a/mini-lsm-book-wip/src/week2-01-compaction.md b/mini-lsm-book-wip/src/week2-01-compaction.md deleted file mode 100644 index c728713..0000000 --- a/mini-lsm-book-wip/src/week2-01-compaction.md +++ /dev/null @@ -1 +0,0 @@ -# Compaction Implementation diff --git a/mini-lsm-book-wip/src/week2-02-simple.md b/mini-lsm-book-wip/src/week2-02-simple.md deleted file mode 100644 index cc80245..0000000 --- a/mini-lsm-book-wip/src/week2-02-simple.md +++ /dev/null @@ -1 +0,0 @@ -# Simple Compaction Strategy diff --git a/mini-lsm-book-wip/src/week2-03-tiered.md b/mini-lsm-book-wip/src/week2-03-tiered.md deleted file mode 100644 index e559756..0000000 --- a/mini-lsm-book-wip/src/week2-03-tiered.md +++ /dev/null @@ -1 +0,0 @@ -# Tiered Compaction Strategy diff --git a/mini-lsm-book-wip/src/week2-04-leveled.md b/mini-lsm-book-wip/src/week2-04-leveled.md deleted file mode 100644 index 873534e..0000000 --- a/mini-lsm-book-wip/src/week2-04-leveled.md +++ /dev/null @@ -1 +0,0 @@ -# Leveled Compaction Strategy diff --git a/mini-lsm-book-wip/src/week2-05-manifest.md b/mini-lsm-book-wip/src/week2-05-manifest.md deleted file mode 100644 index 243e07a..0000000 --- a/mini-lsm-book-wip/src/week2-05-manifest.md +++ /dev/null @@ -1 +0,0 @@ -# Manifest diff --git a/mini-lsm-book-wip/src/week2-06-wal.md b/mini-lsm-book-wip/src/week2-06-wal.md deleted file mode 100644 index 9e743f4..0000000 --- a/mini-lsm-book-wip/src/week2-06-wal.md +++ /dev/null @@ -1 +0,0 @@ -# Write-Ahead Log (WAL) diff --git a/mini-lsm-book-wip/src/week2-overview.md b/mini-lsm-book-wip/src/week2-overview.md deleted file mode 100644 index ecb6496..0000000 --- a/mini-lsm-book-wip/src/week2-overview.md +++ /dev/null @@ -1,8 +0,0 @@ -# Overview - -compaction is important - -ask 2 what if questions - -LSM is about tradeoffs - diff --git a/mini-lsm-book/book.toml b/mini-lsm-book/book.toml index 7c7ce5f..ad25ce5 100644 --- a/mini-lsm-book/book.toml +++ b/mini-lsm-book/book.toml @@ -1,5 +1,5 @@ [book] -authors = ["Alex Chi"] +authors = ["Alex Chi Z"] language = "en" multilingual = false src = "src" @@ -8,3 +8,7 @@ title = "LSM in a Week" [preprocessor.toc] command = "mdbook-toc" renderer = ["html"] + +[output.html] +additional-css = ["custom.css"] +git-repository-url = "https://github.com/skyzh/mini-lsm" diff --git a/mini-lsm-book/custom.css b/mini-lsm-book/custom.css new file mode 100644 index 0000000..8792ecb --- /dev/null +++ b/mini-lsm-book/custom.css @@ -0,0 +1,5 @@ +.content img { + margin-left: auto; + margin-right: auto; + display: block; +} \ No newline at end of file diff --git a/mini-lsm-book/src/00-overview.md b/mini-lsm-book/src/00-overview.md index 0a64587..4d9386d 100644 --- a/mini-lsm-book/src/00-overview.md +++ b/mini-lsm-book/src/00-overview.md @@ -67,7 +67,7 @@ systems. ## Write Flow -![Write Flow](figures/lsm-tutorial/00-lsm-write-flow.svg) +![Write Flow](lsm-tutorial/00-lsm-write-flow.svg) The write flow of LSM contains 4 steps: @@ -80,38 +80,13 @@ The write flow of LSM contains 4 steps: ## Read Flow -![Read Flow](figures/lsm-tutorial/00-lsm-read-flow.svg) +![Read Flow](lsm-tutorial/00-lsm-read-flow.svg) When we want to read a key, 1. We will first probe all the memtables from latest to oldest. 2. If the key is not found, we will then search the entire LSM tree containing SSTs to find the data. -## Tutorial Overview - -![Tutorial Overview](figures/lsm-tutorial/00-lsm-tutorial-overview.svg) - -In this tutorial, we will build the LSM tree structure in 7 days: - -* Day 1: Block encoding. SSTs are composed of multiple data blocks. We will implement the block encoding. -* Day 2: SST encoding. -* Day 3: MemTable and Merge Iterators. -* Day 4: Block cache and Engine. To reduce disk I/O and maximize performance, we will use moka-rs to build a block cache - for the LSM tree. In this day we will get a functional (but not persistent) key-value engine with `get`, `put`, `scan`, - `delete` API. -* Day 5: Compaction. Now it's time to maintain a leveled structure for SSTs. -* Day 6: Recovery. We will implement WAL and manifest so that the engine can recover after restart. -* Day 7: Bloom filter and key compression. They are widely-used optimizations in LSM tree structures. - -## Development Guide - -We provide you starter code (see `mini-lsm-starter` crate), where we simply replace all function body with -`unimplemented!()`. You can start your project based on this starter code. We provide test cases, but they are very -simple. We recommend you to think carefully about your implementation and write test cases by yourself. - -* You can use `cargo x scheck` to run all test cases and do style check in your codebase. -* You can use `cargo x copy-test dayX` to copy test cases to the starter code. - ## Community You may join skyzh's Discord server and study with the mini-lsm community. @@ -138,3 +113,45 @@ vectorized expression framework if you are also interested in that topic. [tweet]: https://twitter.com/andy_pavlo/status/1598137241016360961 [type-exercise]: https://github.com/skyzh/type-exercise-in-rust [bustub]: https://github.com/cmu-db/bustub + + diff --git a/mini-lsm-book/src/00-v1.md b/mini-lsm-book/src/00-v1.md new file mode 100644 index 0000000..d337137 --- /dev/null +++ b/mini-lsm-book/src/00-v1.md @@ -0,0 +1,28 @@ +# Mini-LSM v1 + +This is a legacy version of the Mini-LSM tutorial and we will not maintain it anymore. We are working on a new version of the tutorial. We keep the legacy version in this book so that the search engine can keep the pages in the index and users can follow the links to the new version of the tutorial. + +## V1 Tutorial Overview + +![Tutorial Overview](legacy-lsm-tutorial/00-lsm-tutorial-overview.svg) + +In this tutorial, we will build the LSM tree structure in 7 days: + +* Day 1: Block encoding. SSTs are composed of multiple data blocks. We will implement the block encoding. +* Day 2: SST encoding. +* Day 3: MemTable and Merge Iterators. +* Day 4: Block cache and Engine. To reduce disk I/O and maximize performance, we will use moka-rs to build a block cache + for the LSM tree. In this day we will get a functional (but not persistent) key-value engine with `get`, `put`, `scan`, + `delete` API. +* Day 5: Compaction. Now it's time to maintain a leveled structure for SSTs. +* Day 6: Recovery. We will implement WAL and manifest so that the engine can recover after restart. +* Day 7: Bloom filter and key compression. They are widely-used optimizations in LSM tree structures. + +## Development Guide + +We provide you starter code (see `mini-lsm-starter` crate), where we simply replace all function body with +`unimplemented!()`. You can start your project based on this starter code. We provide test cases, but they are very +simple. We recommend you to think carefully about your implementation and write test cases by yourself. + +* You can use `cargo x scheck` to run all test cases and do style check in your codebase. +* You can use `cargo x copy-test dayX` to copy test cases to the starter code. diff --git a/mini-lsm-book/src/01-block.md b/mini-lsm-book/src/01-block.md index 9b58e65..7049dec 100644 --- a/mini-lsm-book/src/01-block.md +++ b/mini-lsm-book/src/01-block.md @@ -1,5 +1,11 @@ # Block Builder and Block Iterator +
+ +This is a legacy version of the Mini-LSM tutorial and we will not maintain it anymore. We are working on a new version of this tutorial and this chapter is now part of [Mini-LSM Week 1 Day 2: Blocks](./week1-02-block.md). + +
+ In this part, you will need to modify: @@ -23,9 +29,9 @@ The block contains two parts: data and offsets. ``` --------------------------------------------------------------------- -| data | offsets | meta | -|-----------------------|---------------------------|---------------| -|entry|entry|entry|entry|offset|offset|offset|offset|num_of_elements| +| data | offsets | meta | +| ----- | ------- | ----- | +| entry | entry | entry | entry | offset | offset | offset | offset | num_of_elements | --------------------------------------------------------------------- ``` diff --git a/mini-lsm-book/src/02-sst.md b/mini-lsm-book/src/02-sst.md index db821b6..261c420 100644 --- a/mini-lsm-book/src/02-sst.md +++ b/mini-lsm-book/src/02-sst.md @@ -1,5 +1,11 @@ # SST Builder and SST Iterator +
+ +This is a legacy version of the Mini-LSM tutorial and we will not maintain it anymore. We are working on a new version of this tutorial and this chapter is now part of [Mini-LSM Week 1 Day 3: Sorted String Table (SST)](./week1-03-sst.md). + +
+ In this part, you will need to modify: diff --git a/mini-lsm-book/src/03-memtable.md b/mini-lsm-book/src/03-memtable.md index 1ba093f..663861e 100644 --- a/mini-lsm-book/src/03-memtable.md +++ b/mini-lsm-book/src/03-memtable.md @@ -1,5 +1,11 @@ # Mem Table and Merge Iterators +
+ +This is a legacy version of the Mini-LSM tutorial and we will not maintain it anymore. We are working on a new version of this tutorial and this chapter is now part of [Mini-LSM Week 1 Day 1: Memtable](./week1-01-memtable.md) and [Mini-LSM Week 1 Day 4: Merge Iterator](./week1-04-merge-iterator.md) + +
+ In this part, you will need to modify: diff --git a/mini-lsm-book/src/04-engine.md b/mini-lsm-book/src/04-engine.md index 70225e7..e947afa 100644 --- a/mini-lsm-book/src/04-engine.md +++ b/mini-lsm-book/src/04-engine.md @@ -1,5 +1,10 @@ # Storage Engine and Block Cache +
+ +This is a legacy version of the Mini-LSM tutorial and we will not maintain it anymore. We are working on a new version of this tutorial and this chapter is now part of [Mini-LSM Week 1 Day 5: Read Path](./week1-05-read-path.md) and [Mini-LSM Week 1 Day 6: Write Path](./week1-06-write-path.md) + +
diff --git a/mini-lsm-book/src/05-compaction.md b/mini-lsm-book/src/05-compaction.md index 6e6892f..23609ae 100644 --- a/mini-lsm-book/src/05-compaction.md +++ b/mini-lsm-book/src/05-compaction.md @@ -1 +1,16 @@ # Leveled Compaction + + +
+ +This is a legacy version of the Mini-LSM tutorial and we will not maintain it anymore. We are working on a new version of this tutorial +and this chapter is now part of: + +- [Mini-LSM Week 2 Day 1: Compaction Implementation](./week2-01-compaction.md) +- [Mini-LSM Week 2 Day 2: Simple Compaction Strategy](./week2-02-simple.md) +- [Mini-LSM Week 2 Day 3: Tiered Compaction Strategy](./week2-03-tiered.md) +- [Mini-LSM Week 2 Day 4: Leveled Compaction Strategy](./week2-04-leveled.md) + +
+ +We did not finish this chapter as part of Mini-LSM v1. diff --git a/mini-lsm-book/src/06-recovery.md b/mini-lsm-book/src/06-recovery.md index af085ec..57fb4d2 100644 --- a/mini-lsm-book/src/06-recovery.md +++ b/mini-lsm-book/src/06-recovery.md @@ -1 +1,13 @@ # Write-Ahead Log for Recovery + +
+ +This is a legacy version of the Mini-LSM tutorial and we will not maintain it anymore. We are working on a new version of this tutorial +and this chapter is now part of: + +- [Mini-LSM Week 2 Day 5: Manifest](./week2-05-manifest.md) +- [Mini-LSM Week 2 Day 6: Write-Ahead Log (WAL)](./week2-06-wal.md) + +
+ +We did not finish this chapter as part of Mini-LSM v1. diff --git a/mini-lsm-book/src/07-bloom-filter.md b/mini-lsm-book/src/07-bloom-filter.md index c48ca57..3607f5e 100644 --- a/mini-lsm-book/src/07-bloom-filter.md +++ b/mini-lsm-book/src/07-bloom-filter.md @@ -1 +1,11 @@ # Bloom Filters + + +
+ +This is a legacy version of the Mini-LSM tutorial and we will not maintain it anymore. We are working on a new version of this tutorial +and this chapter is now part of [Mini LSM Week 1 Day 7: SST Optimizations](./week1-07-sst-optimizations.md). + +
+ +We did not finish this chapter as part of Mini-LSM v1. diff --git a/mini-lsm-book/src/08-key-compression.md b/mini-lsm-book/src/08-key-compression.md index 65b3f02..d3f28f8 100644 --- a/mini-lsm-book/src/08-key-compression.md +++ b/mini-lsm-book/src/08-key-compression.md @@ -1 +1,10 @@ # Key Compression + +
+ +This is a legacy version of the Mini-LSM tutorial and we will not maintain it anymore. We are working on a new version of this tutorial +and this chapter is now part of [Mini LSM Week 1 Day 7: SST Optimizations](./week1-07-sst-optimizations.md). + +
+ +We did not finish this chapter as part of Mini-LSM v1. diff --git a/mini-lsm-book/src/09-whats-next.md b/mini-lsm-book/src/09-whats-next.md index 6e3eb22..4ce2706 100644 --- a/mini-lsm-book/src/09-whats-next.md +++ b/mini-lsm-book/src/09-whats-next.md @@ -1 +1,3 @@ # What's Next + +We did not finish this chapter as part of Mini-LSM v1. diff --git a/mini-lsm-book/src/SUMMARY.md b/mini-lsm-book/src/SUMMARY.md index 049345e..29d3216 100644 --- a/mini-lsm-book/src/SUMMARY.md +++ b/mini-lsm-book/src/SUMMARY.md @@ -5,22 +5,42 @@ --- -# Checkpoint 1 +# Mini-LSM v1 -- [Store key-value pairs in little blocks](./01-block.md) -- [And make them into an SST](./02-sst.md) -- [Now it's time to merge everything](./03-memtable.md) -- [The engine is on fire](./04-engine.md) - -# Checkpoint 2 - -- [Let's do something in the background](./05-compaction.md) +- [Overview](./00-v1.md) + - [Store key-value pairs in little blocks](./01-block.md) + - [And make them into an SST](./02-sst.md) + - [Now it's time to merge everything](./03-memtable.md) + - [The engine is on fire](./04-engine.md) + - [Let's do something in the background](./05-compaction.md) + - [Be careful when the system crashes](./06-recovery.md) + - [A good bloom filter makes life easier](./07-bloom-filter.md) + - [Save some space, hopefully](./08-key-compression.md) + - [What's next](./09-whats-next.md) --- -# WIP Chapters +# Mini-LSM v2 + +- [Week 1: Mini-LSM](./week1-overview.md) + - [Memtable](./week1-01-memtable.md) + - [Block](./week1-02-block.md) + - [Sorted String Table (SST)](./week1-03-sst.md) + - [Merge Iterator](./week1-04-merge-iterator.md) + - [Read Path](./week1-05-read-path.md) + - [Write Path](./week1-06-write-path.md) + - [Snack Time: SST Optimizations](./week1-07-sst-optimizations.md) + +- [Week 2: Compaction and Persistence](./week2-overview.md) + - [Compaction Implementation](./week2-01-compaction.md) + - [Simple Compaction Strategy](./week2-02-simple.md) + - [Tiered Compaction Strategy](./week2-03-tiered.md) + - [Leveled Compaction Strategy](./week2-04-leveled.md) + - [Manifest](./week2-05-manifest.md) + - [Write-Ahead Log (WAL)](./week2-06-wal.md) + - [Snack Time: Batch Write](./week2-07-batch-write.md) + +- [Week 3: MVCC](./week3-overview.md) + +# The Rest of Your Life (TBD) -- [Be careful when the system crashes](./06-recovery.md) -- [A good bloom filter makes life easier](./07-bloom-filter.md) -- [Save some space, hopefully](./08-key-compression.md) -- [What's next](./09-whats-next.md) diff --git a/mini-lsm-book/src/figures/lsm-tutorial/00-lsm-tutorial-overview.svg b/mini-lsm-book/src/legacy-lsm-tutorial/00-lsm-tutorial-overview.svg similarity index 100% rename from mini-lsm-book/src/figures/lsm-tutorial/00-lsm-tutorial-overview.svg rename to mini-lsm-book/src/legacy-lsm-tutorial/00-lsm-tutorial-overview.svg diff --git a/mini-lsm-book/src/figures/lsm-tutorial/00-lsm-read-flow.svg b/mini-lsm-book/src/lsm-tutorial/00-lsm-read-flow.svg similarity index 97% rename from mini-lsm-book/src/figures/lsm-tutorial/00-lsm-read-flow.svg rename to mini-lsm-book/src/lsm-tutorial/00-lsm-read-flow.svg index f15c1fe..702d92b 100644 --- a/mini-lsm-book/src/figures/lsm-tutorial/00-lsm-read-flow.svg +++ b/mini-lsm-book/src/lsm-tutorial/00-lsm-read-flow.svg @@ -1,6 +1,6 @@ - + @@ -18,7 +18,7 @@ - + 00-lsm-read-flow diff --git a/mini-lsm-book/src/figures/lsm-tutorial/00-lsm-write-flow.svg b/mini-lsm-book/src/lsm-tutorial/00-lsm-write-flow.svg similarity index 97% rename from mini-lsm-book/src/figures/lsm-tutorial/00-lsm-write-flow.svg rename to mini-lsm-book/src/lsm-tutorial/00-lsm-write-flow.svg index 04d459e..4977cc7 100644 --- a/mini-lsm-book/src/figures/lsm-tutorial/00-lsm-write-flow.svg +++ b/mini-lsm-book/src/lsm-tutorial/00-lsm-write-flow.svg @@ -1,6 +1,6 @@ - + @@ -13,7 +13,7 @@ - + 00-lsm-write-flow diff --git a/mini-lsm-book/src/lsm-tutorial/week1-01-overview.svg b/mini-lsm-book/src/lsm-tutorial/week1-01-overview.svg new file mode 100644 index 0000000..795ad09 --- /dev/null +++ b/mini-lsm-book/src/lsm-tutorial/week1-01-overview.svg @@ -0,0 +1,74 @@ + + + + + + + + + + + + week1-01-overview + + + Layer 1 + + + + + + + + + + On Disk + + + + + In Memory + + + + + + + Mem + Table + + + + + + + Mem + Table + + + + + + + Mem + Table + + + + + + + key + value + + + + + + + + Day 1: Memtables + + + + + diff --git a/mini-lsm-book/src/lsm-tutorial/week1-02-overview.svg b/mini-lsm-book/src/lsm-tutorial/week1-02-overview.svg new file mode 100644 index 0000000..d6eaad1 --- /dev/null +++ b/mini-lsm-book/src/lsm-tutorial/week1-02-overview.svg @@ -0,0 +1,270 @@ + + + + + + + + + + + + week1-02-overview + + + Layer 1 + + + + + + + + + SST + + + + + L0 + + + + + + + SST + + + + + + + SST + + + + + L1 + + + + + + + SST + + + + + + + SST + + + + + Ln + + + + + + + SST + + + + + + + SST + + + + + + + + On Disk + + + + + In Memory + + + + + + + Mem + Table + + + + + + + Mem + Table + + + + + + + Mem + Table + + + + + + + key + value + + + + + + + + + + + + + Data + Block + + + + + + + Data + Block + + + + + + + Data + Block + + + + + + + Data + Block + + + + + + + Data + Block + + + + + + + Index + + + + + + + + + + + + + Key + + + + + + + Value + + + + + + + Key + + + + + + + Value + + + + + + + Key + + + + + + + Value + + + + + + + Key + + + + + + + Value + + + + + + + Key + + + + + + + Value + + + + + + + + + + + Day 2: Block Encoding + + + + + diff --git a/mini-lsm-book/src/lsm-tutorial/week1-03-overview.svg b/mini-lsm-book/src/lsm-tutorial/week1-03-overview.svg new file mode 100644 index 0000000..e7aaeb4 --- /dev/null +++ b/mini-lsm-book/src/lsm-tutorial/week1-03-overview.svg @@ -0,0 +1,270 @@ + + + + + + + + + + + + week1-03-overview + + + Layer 1 + + + + + + + Day 3: SST Encoding + + + + + + + SST + + + + + L0 + + + + + + + SST + + + + + + + SST + + + + + L1 + + + + + + + SST + + + + + + + SST + + + + + Ln + + + + + + + SST + + + + + + + SST + + + + + + + + On Disk + + + + + In Memory + + + + + + + Mem + Table + + + + + + + Mem + Table + + + + + + + Mem + Table + + + + + + + key + value + + + + + + + + + + + + + Data + Block + + + + + + + Data + Block + + + + + + + Data + Block + + + + + + + Data + Block + + + + + + + Data + Block + + + + + + + Index + + + + + + + + + + + + + Key + + + + + + + Value + + + + + + + Key + + + + + + + Value + + + + + + + Key + + + + + + + Value + + + + + + + Key + + + + + + + Value + + + + + + + Key + + + + + + + Value + + + + + + + + + + + diff --git a/mini-lsm-book/src/lsm-tutorial/week1-04-overview.svg b/mini-lsm-book/src/lsm-tutorial/week1-04-overview.svg new file mode 100644 index 0000000..9867ce0 --- /dev/null +++ b/mini-lsm-book/src/lsm-tutorial/week1-04-overview.svg @@ -0,0 +1,111 @@ + + + + + + + + + + + + week1-04-overview + + + Layer 1 + + + + + + + L0 + + + + + + + SST + + + + + + + + On Disk + + + + + In Memory + + + + + + + Mem + Table + + + + + + + Mem + Table + + + + + + + Mem + Table + + + + + + + key + value + + + + + + + + + + + Day 4+5+6: Storage Engine + Merge Iterators + Read + Write Path + + + + + + + SST + + + + + + + SST + + + + + + + SST + + + + + diff --git a/mini-lsm-book/src/lsm-tutorial/week1-07-overview.svg b/mini-lsm-book/src/lsm-tutorial/week1-07-overview.svg new file mode 100644 index 0000000..8603609 --- /dev/null +++ b/mini-lsm-book/src/lsm-tutorial/week1-07-overview.svg @@ -0,0 +1,186 @@ + + + + + + + + + + + + week1-07-overview + + + Layer 1 + + + + + + + + + + + + + SST + + + + + + + Data + Block + + + + + + + Data + Block + + + + + + + Data + Block + + + + + + + Data + Block + + + + + + + Data + Block + + + + + + + Index + + + + + + + + + + + + + Key + + + + + + + Value + + + + + + + Key + + + + + + + Value + + + + + + + Key + + + + + + + Value + + + + + + + Key + + + + + + + Value + + + + + + + Key + + + + + + + Value + + + + + + + + + + + Day 7.1: Bloom Filter + + + + + + + + Day 7.2: Key Compression + + + + + + + + + + Bloom + Filter + + + + + diff --git a/mini-lsm-book/src/lsm-tutorial/week1-overview.svg b/mini-lsm-book/src/lsm-tutorial/week1-overview.svg new file mode 100644 index 0000000..7a44b5c --- /dev/null +++ b/mini-lsm-book/src/lsm-tutorial/week1-overview.svg @@ -0,0 +1,338 @@ + + + + + + + + + + + + + + + + + week1-overview + + + Layer 1 + + + + + + + + + + + + + + + Day 3: SST Encoding + + + + + + + + + + + SST + + + + + L0 + + + + + + + SST + + + + + + + SST + + + + + L1 + + + + + + + SST + + + + + + + SST + + + + + Ln + + + + + + + SST + + + + + + + SST + + + + + + + + On Disk + + + + + In Memory + + + + + + + Mem + Table + + + + + + + Mem + Table + + + + + + + Mem + Table + + + + + + + WAL + + + + + + + key + value + + + + + + + + + + + + + + + + + + + Data + Block + + + + + + + Data + Block + + + + + + + Data + Block + + + + + + + Data + Block + + + + + + + Data + Block + + + + + + + Index + + + + + + + + + + + + + Key + + + + + + + Value + + + + + + + Key + + + + + + + Value + + + + + + + Key + + + + + + + Value + + + + + + + Key + + + + + + + Value + + + + + + + Key + + + + + + + Value + + + + + + + + + + + Day 2: Block Encoding + + + + + + + Manifest + + + + + Day 1: Memtables + + + + + Day 4+5+6: Storage Engine + + + + + Day 7.1: Bloom Filter + + + + + + + + Day 7.2: Key Compression + + + + + + + + diff --git a/mini-lsm-book/src/lsm-tutorial/week2-01-overview.svg b/mini-lsm-book/src/lsm-tutorial/week2-01-overview.svg new file mode 100644 index 0000000..ecfb2d0 --- /dev/null +++ b/mini-lsm-book/src/lsm-tutorial/week2-01-overview.svg @@ -0,0 +1,144 @@ + + + + + + + + + + + + week2-01-overview + + + Layer 1 + + + + + + + + + SST + + + + + L0 + + + + + + + SST + + + + + + + SST + + + + + L1 + + + + + + + SST + + + + + + + SST + + + + + Ln + + + + + + + SST + + + + + + + SST + + + + + + + + On Disk + + + + + In Memory + + + + + + + Mem + Table + + + + + + + Mem + Table + + + + + + + Mem + Table + + + + + + + key + value + + + + + + + + + + + + + + Day 1+2+3+4: Compaction + + + + + diff --git a/mini-lsm-book/src/lsm-tutorial/week2-05-overview.svg b/mini-lsm-book/src/lsm-tutorial/week2-05-overview.svg new file mode 100644 index 0000000..d31e0dc --- /dev/null +++ b/mini-lsm-book/src/lsm-tutorial/week2-05-overview.svg @@ -0,0 +1,155 @@ + + + + + + + + + + + + + + + + + week2-05-overview + + + Layer 1 + + + + + SST + + + + + L0 + + + + + + + SST + + + + + + + SST + + + + + L1 + + + + + + + SST + + + + + + + SST + + + + + Ln + + + + + + + SST + + + + + + + SST + + + + + + + + On Disk + + + + + In Memory + + + + + + + Mem + Table + + + + + + + Mem + Table + + + + + + + Mem + Table + + + + + + + key + value + + + + + + + + + + + + + + Day 5: Manifest + + + + + + + Manifest + + + + + + + + diff --git a/mini-lsm-book/src/lsm-tutorial/week2-06-overview.svg b/mini-lsm-book/src/lsm-tutorial/week2-06-overview.svg new file mode 100644 index 0000000..ea47ee3 --- /dev/null +++ b/mini-lsm-book/src/lsm-tutorial/week2-06-overview.svg @@ -0,0 +1,161 @@ + + + + + + + + + + + + week2-06-overview + + + Layer 1 + + + + + + + + + SST + + + + + L0 + + + + + + + SST + + + + + + + SST + + + + + L1 + + + + + + + SST + + + + + + + SST + + + + + Ln + + + + + + + SST + + + + + + + SST + + + + + + + + On Disk + + + + + In Memory + + + + + + + Mem + Table + + + + + + + Mem + Table + + + + + + + Mem + Table + + + + + + + WAL + + + + + + + key + value + + + + + + + + + + + + + + + + + + + Manifest + + + + + Day 6: WAL + + + + + diff --git a/mini-lsm-book/src/lsm-tutorial/week2-overview.svg b/mini-lsm-book/src/lsm-tutorial/week2-overview.svg new file mode 100644 index 0000000..090fbca --- /dev/null +++ b/mini-lsm-book/src/lsm-tutorial/week2-overview.svg @@ -0,0 +1,191 @@ + + + + + + + + + + + + + + + + + week2-overview + + + Layer 1 + + + + + + + + + + + + + SST + + + + + L0 + + + + + + + SST + + + + + + + SST + + + + + L1 + + + + + + + SST + + + + + + + SST + + + + + Ln + + + + + + + SST + + + + + + + SST + + + + + + + + On Disk + + + + + In Memory + + + + + + + Mem + Table + + + + + + + Mem + Table + + + + + + + Mem + Table + + + + + + + WAL + + + + + + + key + value + + + + + + + + + + + + + + + + + Day 5: Manifest + + + + + + + Manifest + + + + + Day 6: WAL + + + + + Day 1+2+3+4: Compaction + + + + + + + + Day 7: Batch Write and Preparations for MVCC + + + + + + + + diff --git a/mini-lsm-book/src/week1-01-memtable.md b/mini-lsm-book/src/week1-01-memtable.md new file mode 100644 index 0000000..0d7dc34 --- /dev/null +++ b/mini-lsm-book/src/week1-01-memtable.md @@ -0,0 +1,3 @@ +# Memtables + +![Chapter Overview](./lsm-tutorial/week1-01-overview.svg) diff --git a/mini-lsm-book/src/week1-02-block.md b/mini-lsm-book/src/week1-02-block.md new file mode 100644 index 0000000..5879d3d --- /dev/null +++ b/mini-lsm-book/src/week1-02-block.md @@ -0,0 +1,3 @@ +# Blocks + +![Chapter Overview](./lsm-tutorial/week1-02-overview.svg) diff --git a/mini-lsm-book/src/week1-03-sst.md b/mini-lsm-book/src/week1-03-sst.md new file mode 100644 index 0000000..8ac9ca5 --- /dev/null +++ b/mini-lsm-book/src/week1-03-sst.md @@ -0,0 +1,3 @@ +# Sorted String Table (SST) + +![Chapter Overview](./lsm-tutorial/week1-03-overview.svg) diff --git a/mini-lsm-book/src/week1-04-merge-iterator.md b/mini-lsm-book/src/week1-04-merge-iterator.md new file mode 100644 index 0000000..971aa18 --- /dev/null +++ b/mini-lsm-book/src/week1-04-merge-iterator.md @@ -0,0 +1,3 @@ +# Merge Iterators + +![Chapter Overview](./lsm-tutorial/week1-04-overview.svg) diff --git a/mini-lsm-book/src/week1-05-read-path.md b/mini-lsm-book/src/week1-05-read-path.md new file mode 100644 index 0000000..fb8108b --- /dev/null +++ b/mini-lsm-book/src/week1-05-read-path.md @@ -0,0 +1,3 @@ +# Read Path + +![Chapter Overview](./lsm-tutorial/week1-04-overview.svg) diff --git a/mini-lsm-book/src/week1-06-write-path.md b/mini-lsm-book/src/week1-06-write-path.md new file mode 100644 index 0000000..10aeacb --- /dev/null +++ b/mini-lsm-book/src/week1-06-write-path.md @@ -0,0 +1,3 @@ +# Write Path + +![Chapter Overview](./lsm-tutorial/week1-04-overview.svg) diff --git a/mini-lsm-book-wip/src/week1-07-sst-optimizations.md b/mini-lsm-book/src/week1-07-sst-optimizations.md similarity index 65% rename from mini-lsm-book-wip/src/week1-07-sst-optimizations.md rename to mini-lsm-book/src/week1-07-sst-optimizations.md index 3c50e9b..6d7cc62 100644 --- a/mini-lsm-book-wip/src/week1-07-sst-optimizations.md +++ b/mini-lsm-book/src/week1-07-sst-optimizations.md @@ -1,3 +1,5 @@ # SST Optimizations +![Chapter Overview](./lsm-tutorial/week1-07-overview.svg) + at the end of each week, we will have some easy, not important, while interesting things diff --git a/mini-lsm-book/src/week1-overview.md b/mini-lsm-book/src/week1-overview.md new file mode 100644 index 0000000..be636f6 --- /dev/null +++ b/mini-lsm-book/src/week1-overview.md @@ -0,0 +1,6 @@ +# Overview + +![Chapter Overview](./lsm-tutorial/week1-overview.svg) + +key functionalities index + diff --git a/mini-lsm-book/src/week2-01-compaction.md b/mini-lsm-book/src/week2-01-compaction.md new file mode 100644 index 0000000..9800891 --- /dev/null +++ b/mini-lsm-book/src/week2-01-compaction.md @@ -0,0 +1,3 @@ +# Compaction Implementation + +![Chapter Overview](./lsm-tutorial/week2-01-overview.svg) diff --git a/mini-lsm-book/src/week2-02-simple.md b/mini-lsm-book/src/week2-02-simple.md new file mode 100644 index 0000000..0ff7ff5 --- /dev/null +++ b/mini-lsm-book/src/week2-02-simple.md @@ -0,0 +1,3 @@ +# Simple Compaction Strategy + +![Chapter Overview](./lsm-tutorial/week2-01-overview.svg) diff --git a/mini-lsm-book/src/week2-03-tiered.md b/mini-lsm-book/src/week2-03-tiered.md new file mode 100644 index 0000000..3ad6d14 --- /dev/null +++ b/mini-lsm-book/src/week2-03-tiered.md @@ -0,0 +1,3 @@ +# Tiered Compaction Strategy + +![Chapter Overview](./lsm-tutorial/week2-01-overview.svg) diff --git a/mini-lsm-book/src/week2-04-leveled.md b/mini-lsm-book/src/week2-04-leveled.md new file mode 100644 index 0000000..b1df435 --- /dev/null +++ b/mini-lsm-book/src/week2-04-leveled.md @@ -0,0 +1,3 @@ +# Leveled Compaction Strategy + +![Chapter Overview](./lsm-tutorial/week2-01-overview.svg) diff --git a/mini-lsm-book/src/week2-05-manifest.md b/mini-lsm-book/src/week2-05-manifest.md new file mode 100644 index 0000000..40d42b4 --- /dev/null +++ b/mini-lsm-book/src/week2-05-manifest.md @@ -0,0 +1,3 @@ +# Manifest + +![Chapter Overview](./lsm-tutorial/week2-05-overview.svg) diff --git a/mini-lsm-book/src/week2-06-wal.md b/mini-lsm-book/src/week2-06-wal.md new file mode 100644 index 0000000..e3247cd --- /dev/null +++ b/mini-lsm-book/src/week2-06-wal.md @@ -0,0 +1,3 @@ +# Write-Ahead Log (WAL) + +![Chapter Overview](./lsm-tutorial/week2-06-overview.svg) diff --git a/mini-lsm-book-wip/src/week2-07-batch-write.md b/mini-lsm-book/src/week2-07-batch-write.md similarity index 100% rename from mini-lsm-book-wip/src/week2-07-batch-write.md rename to mini-lsm-book/src/week2-07-batch-write.md diff --git a/mini-lsm-book/src/week2-overview.md b/mini-lsm-book/src/week2-overview.md new file mode 100644 index 0000000..d488fa5 --- /dev/null +++ b/mini-lsm-book/src/week2-overview.md @@ -0,0 +1,11 @@ +# Overview + +![Chapter Overview](./lsm-tutorial/week2-overview.svg) + +compaction is important + +ask 2 what if questions + +LSM is about tradeoffs + +learning path: feel free to skip compactions? diff --git a/mini-lsm-book-wip/src/week3-overview.md b/mini-lsm-book/src/week3-overview.md similarity index 100% rename from mini-lsm-book-wip/src/week3-overview.md rename to mini-lsm-book/src/week3-overview.md