feat: bump to Rust edition 2024 (#132)
* feat: bump to Rust edition 2024 Signed-off-by: Alex Chi <iskyzh@gmail.com> * allow deadcode Signed-off-by: Alex Chi <iskyzh@gmail.com> * fix deploy to pages Signed-off-by: Alex Chi <iskyzh@gmail.com> * fix fmt Signed-off-by: Alex Chi <iskyzh@gmail.com> * update actions Signed-off-by: Alex Chi <iskyzh@gmail.com> --------- Signed-off-by: Alex Chi <iskyzh@gmail.com>
This commit is contained in:
6
.github/workflows/main.yml
vendored
6
.github/workflows/main.yml
vendored
@@ -14,14 +14,14 @@ jobs:
|
|||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v2
|
- uses: actions/checkout@v2
|
||||||
- name: setup rust toolchain
|
- name: setup rust toolchain
|
||||||
run: rustup show
|
run: rustup toolchain install
|
||||||
- uses: taiki-e/install-action@nextest
|
- uses: taiki-e/install-action@nextest
|
||||||
- uses: taiki-e/install-action@mdbook
|
- uses: taiki-e/install-action@mdbook
|
||||||
- name: patch for gh-pages build
|
- name: patch for gh-pages build
|
||||||
run: mv mini-lsm-book/theme/head.hbs._ mini-lsm-book/theme/head.hbs
|
run: mv mini-lsm-book/theme/head.hbs._ mini-lsm-book/theme/head.hbs
|
||||||
- name: check and build
|
- name: check and build
|
||||||
run: cargo x ci
|
run: cargo x ci
|
||||||
- uses: actions/upload-pages-artifact@v1
|
- uses: actions/upload-pages-artifact@v3
|
||||||
with:
|
with:
|
||||||
path: mini-lsm-book/book
|
path: mini-lsm-book/book
|
||||||
|
|
||||||
@@ -38,4 +38,4 @@ jobs:
|
|||||||
steps:
|
steps:
|
||||||
- name: Deploy to GitHub Pages
|
- name: Deploy to GitHub Pages
|
||||||
id: deployment
|
id: deployment
|
||||||
uses: actions/deploy-pages@v1
|
uses: actions/deploy-pages@v4
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ resolver = "2"
|
|||||||
|
|
||||||
[workspace.package]
|
[workspace.package]
|
||||||
version = "0.2.0"
|
version = "0.2.0"
|
||||||
edition = "2021"
|
edition = "2024"
|
||||||
homepage = "https://github.com/skyzh/mini-lsm"
|
homepage = "https://github.com/skyzh/mini-lsm"
|
||||||
keywords = ["storage", "database", "tutorial", "course"]
|
keywords = ["storage", "database", "tutorial", "course"]
|
||||||
license = "Apache-2.0"
|
license = "Apache-2.0"
|
||||||
|
|||||||
@@ -33,7 +33,7 @@ crossbeam-skiplist provides similar interfaces to the Rust std's `BTreeMap`: ins
|
|||||||
|
|
||||||
You will also notice that the `MemTable` structure does not have a `delete` interface. In the mini-lsm implementation, deletion is represented as a key corresponding to an empty value.
|
You will also notice that the `MemTable` structure does not have a `delete` interface. In the mini-lsm implementation, deletion is represented as a key corresponding to an empty value.
|
||||||
|
|
||||||
In this task, you will need to implement `MemTable::get` and `MemTable::put` to enable modifications of the memtable.
|
In this task, you will need to implement `MemTable::get` and `MemTable::put` to enable modifications of the memtable. Note that `put` should always overwrite a key if it already exists. You won't have mutiple entries of the same key in a single memtable.
|
||||||
|
|
||||||
We use the `bytes` crate for storing the data in the memtable. `bytes::Byte` is similar to `Arc<[u8]>`. When you clone the `Bytes`, or get a slice of `Bytes`, the underlying data will not be copied, and therefore cloning it is cheap. Instead, it simply creates a new reference to the storage area and the storage area will be freed when there are no reference to that area.
|
We use the `bytes` crate for storing the data in the memtable. `bytes::Byte` is similar to `Arc<[u8]>`. When you clone the `Bytes`, or get a slice of `Bytes`, the underlying data will not be copied, and therefore cloning it is cheap. Instead, it simply creates a new reference to the storage area and the storage area will be freed when there are no reference to that area.
|
||||||
|
|
||||||
@@ -157,6 +157,7 @@ Now that you have multiple memtables, you may modify your read path `get` functi
|
|||||||
## Test Your Understanding
|
## Test Your Understanding
|
||||||
|
|
||||||
* Why doesn't the memtable provide a `delete` API?
|
* Why doesn't the memtable provide a `delete` API?
|
||||||
|
* Does it make sense for the memtable to store all write operations instead of only the latest version of a key? For example, the user puts a->1, a->2, and a->3 into the same memtable.
|
||||||
* Is it possible to use other data structures as the memtable in LSM? What are the pros/cons of using the skiplist?
|
* Is it possible to use other data structures as the memtable in LSM? What are the pros/cons of using the skiplist?
|
||||||
* Why do we need a combination of `state` and `state_lock`? Can we only use `state.read()` and `state.write()`?
|
* Why do we need a combination of `state` and `state_lock`? Can we only use `state.read()` and `state.write()`?
|
||||||
* Why does the order to store and to probe the memtables matter? If a key appears in multiple memtables, which version should you return to the user?
|
* Why does the order to store and to probe the memtables matter? If a key appears in multiple memtables, which version should you return to the user?
|
||||||
|
|||||||
@@ -28,10 +28,10 @@ pub use simple_leveled::{
|
|||||||
};
|
};
|
||||||
pub use tiered::{TieredCompactionController, TieredCompactionOptions, TieredCompactionTask};
|
pub use tiered::{TieredCompactionController, TieredCompactionOptions, TieredCompactionTask};
|
||||||
|
|
||||||
|
use crate::iterators::StorageIterator;
|
||||||
use crate::iterators::concat_iterator::SstConcatIterator;
|
use crate::iterators::concat_iterator::SstConcatIterator;
|
||||||
use crate::iterators::merge_iterator::MergeIterator;
|
use crate::iterators::merge_iterator::MergeIterator;
|
||||||
use crate::iterators::two_merge_iterator::TwoMergeIterator;
|
use crate::iterators::two_merge_iterator::TwoMergeIterator;
|
||||||
use crate::iterators::StorageIterator;
|
|
||||||
use crate::key::KeySlice;
|
use crate::key::KeySlice;
|
||||||
use crate::lsm_storage::{CompactionFilter, LsmStorageInner, LsmStorageState};
|
use crate::lsm_storage::{CompactionFilter, LsmStorageInner, LsmStorageState};
|
||||||
use crate::manifest::ManifestRecord;
|
use crate::manifest::ManifestRecord;
|
||||||
|
|||||||
@@ -13,8 +13,8 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
use std::cmp::{self};
|
use std::cmp::{self};
|
||||||
use std::collections::binary_heap::PeekMut;
|
|
||||||
use std::collections::BinaryHeap;
|
use std::collections::BinaryHeap;
|
||||||
|
use std::collections::binary_heap::PeekMut;
|
||||||
|
|
||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
|
|
||||||
|
|||||||
@@ -25,9 +25,9 @@ pub struct TwoMergeIterator<A: StorageIterator, B: StorageIterator> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl<
|
impl<
|
||||||
A: 'static + StorageIterator,
|
A: 'static + StorageIterator,
|
||||||
B: 'static + for<'a> StorageIterator<KeyType<'a> = A::KeyType<'a>>,
|
B: 'static + for<'a> StorageIterator<KeyType<'a> = A::KeyType<'a>>,
|
||||||
> TwoMergeIterator<A, B>
|
> TwoMergeIterator<A, B>
|
||||||
{
|
{
|
||||||
fn choose_a(a: &A, b: &B) -> bool {
|
fn choose_a(a: &A, b: &B) -> bool {
|
||||||
if !a.is_valid() {
|
if !a.is_valid() {
|
||||||
@@ -59,9 +59,9 @@ impl<
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl<
|
impl<
|
||||||
A: 'static + StorageIterator,
|
A: 'static + StorageIterator,
|
||||||
B: 'static + for<'a> StorageIterator<KeyType<'a> = A::KeyType<'a>>,
|
B: 'static + for<'a> StorageIterator<KeyType<'a> = A::KeyType<'a>>,
|
||||||
> StorageIterator for TwoMergeIterator<A, B>
|
> StorageIterator for TwoMergeIterator<A, B>
|
||||||
{
|
{
|
||||||
type KeyType<'a> = A::KeyType<'a>;
|
type KeyType<'a> = A::KeyType<'a>;
|
||||||
|
|
||||||
|
|||||||
@@ -14,13 +14,13 @@
|
|||||||
|
|
||||||
use std::ops::Bound;
|
use std::ops::Bound;
|
||||||
|
|
||||||
use anyhow::{bail, Result};
|
use anyhow::{Result, bail};
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
|
|
||||||
|
use crate::iterators::StorageIterator;
|
||||||
use crate::iterators::concat_iterator::SstConcatIterator;
|
use crate::iterators::concat_iterator::SstConcatIterator;
|
||||||
use crate::iterators::merge_iterator::MergeIterator;
|
use crate::iterators::merge_iterator::MergeIterator;
|
||||||
use crate::iterators::two_merge_iterator::TwoMergeIterator;
|
use crate::iterators::two_merge_iterator::TwoMergeIterator;
|
||||||
use crate::iterators::StorageIterator;
|
|
||||||
use crate::mem_table::MemTableIterator;
|
use crate::mem_table::MemTableIterator;
|
||||||
use crate::table::SsTableIterator;
|
use crate::table::SsTableIterator;
|
||||||
|
|
||||||
|
|||||||
@@ -16,8 +16,8 @@ use std::collections::{BTreeSet, HashMap};
|
|||||||
use std::fs::File;
|
use std::fs::File;
|
||||||
use std::ops::Bound;
|
use std::ops::Bound;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::sync::atomic::AtomicUsize;
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
use std::sync::atomic::AtomicUsize;
|
||||||
|
|
||||||
use anyhow::{Context, Result};
|
use anyhow::{Context, Result};
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
@@ -28,16 +28,16 @@ use crate::compact::{
|
|||||||
CompactionController, CompactionOptions, LeveledCompactionController, LeveledCompactionOptions,
|
CompactionController, CompactionOptions, LeveledCompactionController, LeveledCompactionOptions,
|
||||||
SimpleLeveledCompactionController, SimpleLeveledCompactionOptions, TieredCompactionController,
|
SimpleLeveledCompactionController, SimpleLeveledCompactionOptions, TieredCompactionController,
|
||||||
};
|
};
|
||||||
|
use crate::iterators::StorageIterator;
|
||||||
use crate::iterators::concat_iterator::SstConcatIterator;
|
use crate::iterators::concat_iterator::SstConcatIterator;
|
||||||
use crate::iterators::merge_iterator::MergeIterator;
|
use crate::iterators::merge_iterator::MergeIterator;
|
||||||
use crate::iterators::two_merge_iterator::TwoMergeIterator;
|
use crate::iterators::two_merge_iterator::TwoMergeIterator;
|
||||||
use crate::iterators::StorageIterator;
|
|
||||||
use crate::key::{self, KeySlice};
|
use crate::key::{self, KeySlice};
|
||||||
use crate::lsm_iterator::{FusedIterator, LsmIterator};
|
use crate::lsm_iterator::{FusedIterator, LsmIterator};
|
||||||
use crate::manifest::{Manifest, ManifestRecord};
|
use crate::manifest::{Manifest, ManifestRecord};
|
||||||
use crate::mem_table::{map_bound, map_key_bound_plus_ts, MemTable};
|
use crate::mem_table::{MemTable, map_bound, map_key_bound_plus_ts};
|
||||||
use crate::mvcc::txn::{Transaction, TxnIterator};
|
|
||||||
use crate::mvcc::LsmMvccInner;
|
use crate::mvcc::LsmMvccInner;
|
||||||
|
use crate::mvcc::txn::{Transaction, TxnIterator};
|
||||||
use crate::table::{FileObject, SsTable, SsTableBuilder, SsTableIterator};
|
use crate::table::{FileObject, SsTable, SsTableBuilder, SsTableIterator};
|
||||||
|
|
||||||
pub type BlockCache = moka::sync::Cache<(usize, usize), Arc<Block>>;
|
pub type BlockCache = moka::sync::Cache<(usize, usize), Arc<Block>>;
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ use std::io::{Read, Write};
|
|||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use anyhow::{bail, Context, Result};
|
use anyhow::{Context, Result, bail};
|
||||||
use bytes::{Buf, BufMut};
|
use bytes::{Buf, BufMut};
|
||||||
use parking_lot::{Mutex, MutexGuard};
|
use parking_lot::{Mutex, MutexGuard};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|||||||
@@ -14,13 +14,13 @@
|
|||||||
|
|
||||||
use std::ops::Bound;
|
use std::ops::Bound;
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::sync::atomic::AtomicUsize;
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
use std::sync::atomic::AtomicUsize;
|
||||||
|
|
||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
use crossbeam_skiplist::map::Entry;
|
|
||||||
use crossbeam_skiplist::SkipMap;
|
use crossbeam_skiplist::SkipMap;
|
||||||
|
use crossbeam_skiplist::map::Entry;
|
||||||
use ouroboros::self_referencing;
|
use ouroboros::self_referencing;
|
||||||
|
|
||||||
use crate::iterators::StorageIterator;
|
use crate::iterators::StorageIterator;
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ pub mod watermark;
|
|||||||
|
|
||||||
use std::{
|
use std::{
|
||||||
collections::{BTreeMap, HashSet},
|
collections::{BTreeMap, HashSet},
|
||||||
sync::{atomic::AtomicBool, Arc},
|
sync::{Arc, atomic::AtomicBool},
|
||||||
};
|
};
|
||||||
|
|
||||||
use crossbeam_skiplist::SkipMap;
|
use crossbeam_skiplist::SkipMap;
|
||||||
|
|||||||
@@ -16,19 +16,19 @@ use std::{
|
|||||||
collections::HashSet,
|
collections::HashSet,
|
||||||
ops::Bound,
|
ops::Bound,
|
||||||
sync::{
|
sync::{
|
||||||
atomic::{AtomicBool, Ordering},
|
|
||||||
Arc,
|
Arc,
|
||||||
|
atomic::{AtomicBool, Ordering},
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
use anyhow::{bail, Result};
|
use anyhow::{Result, bail};
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
use crossbeam_skiplist::{map::Entry, SkipMap};
|
use crossbeam_skiplist::{SkipMap, map::Entry};
|
||||||
use ouroboros::self_referencing;
|
use ouroboros::self_referencing;
|
||||||
use parking_lot::Mutex;
|
use parking_lot::Mutex;
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
iterators::{two_merge_iterator::TwoMergeIterator, StorageIterator},
|
iterators::{StorageIterator, two_merge_iterator::TwoMergeIterator},
|
||||||
lsm_iterator::{FusedIterator, LsmIterator},
|
lsm_iterator::{FusedIterator, LsmIterator},
|
||||||
lsm_storage::{LsmStorageInner, WriteBatchRecord},
|
lsm_storage::{LsmStorageInner, WriteBatchRecord},
|
||||||
mem_table::map_bound,
|
mem_table::map_bound,
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ use std::fs::File;
|
|||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use anyhow::{anyhow, bail, Result};
|
use anyhow::{Result, anyhow, bail};
|
||||||
pub use builder::SsTableBuilder;
|
pub use builder::SsTableBuilder;
|
||||||
use bytes::{Buf, BufMut};
|
use bytes::{Buf, BufMut};
|
||||||
pub use iterator::SsTableIterator;
|
pub use iterator::SsTableIterator;
|
||||||
|
|||||||
@@ -14,7 +14,7 @@
|
|||||||
|
|
||||||
// Copyright 2021 TiKV Project Authors. Licensed under Apache-2.0.
|
// Copyright 2021 TiKV Project Authors. Licensed under Apache-2.0.
|
||||||
|
|
||||||
use anyhow::{bail, Result};
|
use anyhow::{Result, bail};
|
||||||
use bytes::{Buf, BufMut, Bytes, BytesMut};
|
use bytes::{Buf, BufMut, Bytes, BytesMut};
|
||||||
|
|
||||||
/// Implements a bloom filter
|
/// Implements a bloom filter
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ use std::io::{BufWriter, Read, Write};
|
|||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use anyhow::{bail, Context, Result};
|
use anyhow::{Context, Result, bail};
|
||||||
use bytes::{Buf, BufMut, Bytes};
|
use bytes::{Buf, BufMut, Bytes};
|
||||||
use crossbeam_skiplist::SkipMap;
|
use crossbeam_skiplist::SkipMap;
|
||||||
use parking_lot::Mutex;
|
use parking_lot::Mutex;
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "mini-lsm-starter"
|
name = "mini-lsm-starter"
|
||||||
version = "0.2.0"
|
version = "0.2.0"
|
||||||
edition = "2021"
|
edition = "2024"
|
||||||
publish = false
|
publish = false
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
|
|||||||
@@ -28,9 +28,9 @@ pub struct TwoMergeIterator<A: StorageIterator, B: StorageIterator> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl<
|
impl<
|
||||||
A: 'static + StorageIterator,
|
A: 'static + StorageIterator,
|
||||||
B: 'static + for<'a> StorageIterator<KeyType<'a> = A::KeyType<'a>>,
|
B: 'static + for<'a> StorageIterator<KeyType<'a> = A::KeyType<'a>>,
|
||||||
> TwoMergeIterator<A, B>
|
> TwoMergeIterator<A, B>
|
||||||
{
|
{
|
||||||
pub fn create(a: A, b: B) -> Result<Self> {
|
pub fn create(a: A, b: B) -> Result<Self> {
|
||||||
unimplemented!()
|
unimplemented!()
|
||||||
@@ -38,9 +38,9 @@ impl<
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl<
|
impl<
|
||||||
A: 'static + StorageIterator,
|
A: 'static + StorageIterator,
|
||||||
B: 'static + for<'a> StorageIterator<KeyType<'a> = A::KeyType<'a>>,
|
B: 'static + for<'a> StorageIterator<KeyType<'a> = A::KeyType<'a>>,
|
||||||
> StorageIterator for TwoMergeIterator<A, B>
|
> StorageIterator for TwoMergeIterator<A, B>
|
||||||
{
|
{
|
||||||
type KeyType<'a> = A::KeyType<'a>;
|
type KeyType<'a> = A::KeyType<'a>;
|
||||||
|
|
||||||
|
|||||||
@@ -18,7 +18,7 @@
|
|||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
iterators::{merge_iterator::MergeIterator, StorageIterator},
|
iterators::{StorageIterator, merge_iterator::MergeIterator},
|
||||||
mem_table::MemTableIterator,
|
mem_table::MemTableIterator,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -18,8 +18,8 @@
|
|||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::ops::Bound;
|
use std::ops::Bound;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::sync::atomic::AtomicUsize;
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
use std::sync::atomic::AtomicUsize;
|
||||||
|
|
||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
|
|||||||
@@ -17,8 +17,8 @@
|
|||||||
|
|
||||||
use std::ops::Bound;
|
use std::ops::Bound;
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::sync::atomic::AtomicUsize;
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
use std::sync::atomic::AtomicUsize;
|
||||||
|
|
||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
|
|||||||
@@ -18,7 +18,7 @@
|
|||||||
use std::{
|
use std::{
|
||||||
collections::HashSet,
|
collections::HashSet,
|
||||||
ops::Bound,
|
ops::Bound,
|
||||||
sync::{atomic::AtomicBool, Arc},
|
sync::{Arc, atomic::AtomicBool},
|
||||||
};
|
};
|
||||||
|
|
||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
@@ -28,7 +28,7 @@ use ouroboros::self_referencing;
|
|||||||
use parking_lot::Mutex;
|
use parking_lot::Mutex;
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
iterators::{two_merge_iterator::TwoMergeIterator, StorageIterator},
|
iterators::{StorageIterator, two_merge_iterator::TwoMergeIterator},
|
||||||
lsm_iterator::{FusedIterator, LsmIterator},
|
lsm_iterator::{FusedIterator, LsmIterator},
|
||||||
lsm_storage::LsmStorageInner,
|
lsm_storage::LsmStorageInner,
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -28,10 +28,10 @@ pub use simple_leveled::{
|
|||||||
};
|
};
|
||||||
pub use tiered::{TieredCompactionController, TieredCompactionOptions, TieredCompactionTask};
|
pub use tiered::{TieredCompactionController, TieredCompactionOptions, TieredCompactionTask};
|
||||||
|
|
||||||
|
use crate::iterators::StorageIterator;
|
||||||
use crate::iterators::concat_iterator::SstConcatIterator;
|
use crate::iterators::concat_iterator::SstConcatIterator;
|
||||||
use crate::iterators::merge_iterator::MergeIterator;
|
use crate::iterators::merge_iterator::MergeIterator;
|
||||||
use crate::iterators::two_merge_iterator::TwoMergeIterator;
|
use crate::iterators::two_merge_iterator::TwoMergeIterator;
|
||||||
use crate::iterators::StorageIterator;
|
|
||||||
use crate::key::KeySlice;
|
use crate::key::KeySlice;
|
||||||
use crate::lsm_storage::{LsmStorageInner, LsmStorageState};
|
use crate::lsm_storage::{LsmStorageInner, LsmStorageState};
|
||||||
use crate::manifest::ManifestRecord;
|
use crate::manifest::ManifestRecord;
|
||||||
|
|||||||
@@ -13,8 +13,8 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
use std::cmp::{self};
|
use std::cmp::{self};
|
||||||
use std::collections::binary_heap::PeekMut;
|
|
||||||
use std::collections::BinaryHeap;
|
use std::collections::BinaryHeap;
|
||||||
|
use std::collections::binary_heap::PeekMut;
|
||||||
|
|
||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
|
|
||||||
|
|||||||
@@ -25,9 +25,9 @@ pub struct TwoMergeIterator<A: StorageIterator, B: StorageIterator> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl<
|
impl<
|
||||||
A: 'static + StorageIterator,
|
A: 'static + StorageIterator,
|
||||||
B: 'static + for<'a> StorageIterator<KeyType<'a> = A::KeyType<'a>>,
|
B: 'static + for<'a> StorageIterator<KeyType<'a> = A::KeyType<'a>>,
|
||||||
> TwoMergeIterator<A, B>
|
> TwoMergeIterator<A, B>
|
||||||
{
|
{
|
||||||
fn choose_a(a: &A, b: &B) -> bool {
|
fn choose_a(a: &A, b: &B) -> bool {
|
||||||
if !a.is_valid() {
|
if !a.is_valid() {
|
||||||
@@ -59,9 +59,9 @@ impl<
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl<
|
impl<
|
||||||
A: 'static + StorageIterator,
|
A: 'static + StorageIterator,
|
||||||
B: 'static + for<'a> StorageIterator<KeyType<'a> = A::KeyType<'a>>,
|
B: 'static + for<'a> StorageIterator<KeyType<'a> = A::KeyType<'a>>,
|
||||||
> StorageIterator for TwoMergeIterator<A, B>
|
> StorageIterator for TwoMergeIterator<A, B>
|
||||||
{
|
{
|
||||||
type KeyType<'a> = A::KeyType<'a>;
|
type KeyType<'a> = A::KeyType<'a>;
|
||||||
|
|
||||||
|
|||||||
@@ -14,13 +14,13 @@
|
|||||||
|
|
||||||
use std::ops::Bound;
|
use std::ops::Bound;
|
||||||
|
|
||||||
use anyhow::{bail, Result};
|
use anyhow::{Result, bail};
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
|
|
||||||
|
use crate::iterators::StorageIterator;
|
||||||
use crate::iterators::concat_iterator::SstConcatIterator;
|
use crate::iterators::concat_iterator::SstConcatIterator;
|
||||||
use crate::iterators::merge_iterator::MergeIterator;
|
use crate::iterators::merge_iterator::MergeIterator;
|
||||||
use crate::iterators::two_merge_iterator::TwoMergeIterator;
|
use crate::iterators::two_merge_iterator::TwoMergeIterator;
|
||||||
use crate::iterators::StorageIterator;
|
|
||||||
use crate::mem_table::MemTableIterator;
|
use crate::mem_table::MemTableIterator;
|
||||||
use crate::table::SsTableIterator;
|
use crate::table::SsTableIterator;
|
||||||
|
|
||||||
|
|||||||
@@ -16,8 +16,8 @@ use std::collections::{BTreeSet, HashMap};
|
|||||||
use std::fs::File;
|
use std::fs::File;
|
||||||
use std::ops::Bound;
|
use std::ops::Bound;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::sync::atomic::AtomicUsize;
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
use std::sync::atomic::AtomicUsize;
|
||||||
|
|
||||||
use anyhow::{Context, Result};
|
use anyhow::{Context, Result};
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
@@ -28,14 +28,14 @@ use crate::compact::{
|
|||||||
CompactionController, CompactionOptions, LeveledCompactionController, LeveledCompactionOptions,
|
CompactionController, CompactionOptions, LeveledCompactionController, LeveledCompactionOptions,
|
||||||
SimpleLeveledCompactionController, SimpleLeveledCompactionOptions, TieredCompactionController,
|
SimpleLeveledCompactionController, SimpleLeveledCompactionOptions, TieredCompactionController,
|
||||||
};
|
};
|
||||||
|
use crate::iterators::StorageIterator;
|
||||||
use crate::iterators::concat_iterator::SstConcatIterator;
|
use crate::iterators::concat_iterator::SstConcatIterator;
|
||||||
use crate::iterators::merge_iterator::MergeIterator;
|
use crate::iterators::merge_iterator::MergeIterator;
|
||||||
use crate::iterators::two_merge_iterator::TwoMergeIterator;
|
use crate::iterators::two_merge_iterator::TwoMergeIterator;
|
||||||
use crate::iterators::StorageIterator;
|
|
||||||
use crate::key::KeySlice;
|
use crate::key::KeySlice;
|
||||||
use crate::lsm_iterator::{FusedIterator, LsmIterator};
|
use crate::lsm_iterator::{FusedIterator, LsmIterator};
|
||||||
use crate::manifest::{Manifest, ManifestRecord};
|
use crate::manifest::{Manifest, ManifestRecord};
|
||||||
use crate::mem_table::{map_bound, MemTable};
|
use crate::mem_table::{MemTable, map_bound};
|
||||||
use crate::mvcc::LsmMvccInner;
|
use crate::mvcc::LsmMvccInner;
|
||||||
use crate::table::{FileObject, SsTable, SsTableBuilder, SsTableIterator};
|
use crate::table::{FileObject, SsTable, SsTableBuilder, SsTableIterator};
|
||||||
|
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ use std::io::{Read, Write};
|
|||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use anyhow::{bail, Context, Result};
|
use anyhow::{Context, Result, bail};
|
||||||
use bytes::{Buf, BufMut};
|
use bytes::{Buf, BufMut};
|
||||||
use parking_lot::{Mutex, MutexGuard};
|
use parking_lot::{Mutex, MutexGuard};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|||||||
@@ -14,13 +14,13 @@
|
|||||||
|
|
||||||
use std::ops::Bound;
|
use std::ops::Bound;
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::sync::atomic::AtomicUsize;
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
use std::sync::atomic::AtomicUsize;
|
||||||
|
|
||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
use crossbeam_skiplist::map::Entry;
|
|
||||||
use crossbeam_skiplist::SkipMap;
|
use crossbeam_skiplist::SkipMap;
|
||||||
|
use crossbeam_skiplist::map::Entry;
|
||||||
use ouroboros::self_referencing;
|
use ouroboros::self_referencing;
|
||||||
|
|
||||||
use crate::iterators::StorageIterator;
|
use crate::iterators::StorageIterator;
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ use std::fs::File;
|
|||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use anyhow::{anyhow, bail, Result};
|
use anyhow::{Result, anyhow, bail};
|
||||||
pub use builder::SsTableBuilder;
|
pub use builder::SsTableBuilder;
|
||||||
use bytes::{Buf, BufMut};
|
use bytes::{Buf, BufMut};
|
||||||
pub use iterator::SsTableIterator;
|
pub use iterator::SsTableIterator;
|
||||||
|
|||||||
@@ -14,7 +14,7 @@
|
|||||||
|
|
||||||
// Copyright 2021 TiKV Project Authors. Licensed under Apache-2.0.
|
// Copyright 2021 TiKV Project Authors. Licensed under Apache-2.0.
|
||||||
|
|
||||||
use anyhow::{bail, Result};
|
use anyhow::{Result, bail};
|
||||||
use bytes::{Buf, BufMut, Bytes, BytesMut};
|
use bytes::{Buf, BufMut, Bytes, BytesMut};
|
||||||
|
|
||||||
/// Implements a bloom filter
|
/// Implements a bloom filter
|
||||||
|
|||||||
@@ -1,3 +1,5 @@
|
|||||||
|
#![allow(dead_code)] // REMOVE THIS LINE once all modules are complete
|
||||||
|
|
||||||
// Copyright (c) 2022-2025 Alex Chi Z
|
// Copyright (c) 2022-2025 Alex Chi Z
|
||||||
//
|
//
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
@@ -17,7 +19,7 @@ use std::{
|
|||||||
time::Duration,
|
time::Duration,
|
||||||
};
|
};
|
||||||
|
|
||||||
use anyhow::{bail, Result};
|
use anyhow::{Result, bail};
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
@@ -25,7 +27,7 @@ use crate::{
|
|||||||
CompactionOptions, LeveledCompactionOptions, SimpleLeveledCompactionOptions,
|
CompactionOptions, LeveledCompactionOptions, SimpleLeveledCompactionOptions,
|
||||||
TieredCompactionOptions,
|
TieredCompactionOptions,
|
||||||
},
|
},
|
||||||
iterators::{merge_iterator::MergeIterator, StorageIterator},
|
iterators::{StorageIterator, merge_iterator::MergeIterator},
|
||||||
key::{KeySlice, TS_ENABLED},
|
key::{KeySlice, TS_ENABLED},
|
||||||
lsm_storage::{BlockCache, LsmStorageInner, LsmStorageState, MiniLsm},
|
lsm_storage::{BlockCache, LsmStorageInner, LsmStorageState, MiniLsm},
|
||||||
table::{SsTable, SsTableBuilder, SsTableIterator},
|
table::{SsTable, SsTableBuilder, SsTableIterator},
|
||||||
@@ -128,7 +130,6 @@ where
|
|||||||
assert!(!iter.is_valid());
|
assert!(!iter.is_valid());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code)]
|
|
||||||
pub fn check_iter_result_by_key_and_ts<I>(iter: &mut I, expected: Vec<((Bytes, u64), Bytes)>)
|
pub fn check_iter_result_by_key_and_ts<I>(iter: &mut I, expected: Vec<((Bytes, u64), Bytes)>)
|
||||||
where
|
where
|
||||||
I: for<'a> StorageIterator<KeyType<'a> = KeySlice<'a>>,
|
I: for<'a> StorageIterator<KeyType<'a> = KeySlice<'a>>,
|
||||||
@@ -207,7 +208,6 @@ pub fn generate_sst(
|
|||||||
builder.build(id, block_cache, path.as_ref()).unwrap()
|
builder.build(id, block_cache, path.as_ref()).unwrap()
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code)]
|
|
||||||
pub fn generate_sst_with_ts(
|
pub fn generate_sst_with_ts(
|
||||||
id: usize,
|
id: usize,
|
||||||
path: impl AsRef<Path>,
|
path: impl AsRef<Path>,
|
||||||
@@ -290,7 +290,9 @@ pub fn compaction_bench(storage: Arc<MiniLsm>) {
|
|||||||
|
|
||||||
storage.dump_structure();
|
storage.dump_structure();
|
||||||
|
|
||||||
println!("This test case does not guarantee your compaction algorithm produces a LSM state as expected. It only does minimal checks on the size of the levels. Please use the compaction simulator to check if the compaction is correctly going on.");
|
println!(
|
||||||
|
"This test case does not guarantee your compaction algorithm produces a LSM state as expected. It only does minimal checks on the size of the levels. Please use the compaction simulator to check if the compaction is correctly going on."
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn check_compaction_ratio(storage: Arc<MiniLsm>) {
|
pub fn check_compaction_ratio(storage: Arc<MiniLsm>) {
|
||||||
|
|||||||
@@ -18,14 +18,14 @@ use bytes::Bytes;
|
|||||||
use tempfile::tempdir;
|
use tempfile::tempdir;
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
iterators::{merge_iterator::MergeIterator, StorageIterator},
|
iterators::{StorageIterator, merge_iterator::MergeIterator},
|
||||||
lsm_iterator::FusedIterator,
|
lsm_iterator::FusedIterator,
|
||||||
lsm_storage::{LsmStorageInner, LsmStorageOptions},
|
lsm_storage::{LsmStorageInner, LsmStorageOptions},
|
||||||
mem_table::MemTable,
|
mem_table::MemTable,
|
||||||
tests::harness::check_lsm_iter_result_by_key,
|
tests::harness::check_lsm_iter_result_by_key,
|
||||||
};
|
};
|
||||||
|
|
||||||
use super::harness::{check_iter_result_by_key, expect_iter_error, MockIterator};
|
use super::harness::{MockIterator, check_iter_result_by_key, expect_iter_error};
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_task1_memtable_iter() {
|
fn test_task1_memtable_iter() {
|
||||||
|
|||||||
@@ -15,7 +15,7 @@
|
|||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
use tempfile::{tempdir, TempDir};
|
use tempfile::{TempDir, tempdir};
|
||||||
|
|
||||||
use crate::iterators::StorageIterator;
|
use crate::iterators::StorageIterator;
|
||||||
use crate::key::{KeySlice, KeyVec};
|
use crate::key::{KeySlice, KeyVec};
|
||||||
|
|||||||
@@ -15,7 +15,7 @@
|
|||||||
use std::ops::Bound;
|
use std::ops::Bound;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use self::harness::{check_iter_result_by_key, MockIterator};
|
use self::harness::{MockIterator, check_iter_result_by_key};
|
||||||
use self::harness::{check_lsm_iter_result_by_key, generate_sst};
|
use self::harness::{check_lsm_iter_result_by_key, generate_sst};
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
use tempfile::tempdir;
|
use tempfile::tempdir;
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ use tempfile::tempdir;
|
|||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
key::{KeySlice, TS_ENABLED},
|
key::{KeySlice, TS_ENABLED},
|
||||||
table::{bloom::Bloom, FileObject, SsTable, SsTableBuilder},
|
table::{FileObject, SsTable, SsTableBuilder, bloom::Bloom},
|
||||||
};
|
};
|
||||||
|
|
||||||
fn key_of(idx: usize) -> Vec<u8> {
|
fn key_of(idx: usize) -> Vec<u8> {
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ use week2_day1::harness::construct_merge_iterator_over_storage;
|
|||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::{
|
use crate::{
|
||||||
iterators::{concat_iterator::SstConcatIterator, StorageIterator},
|
iterators::{StorageIterator, concat_iterator::SstConcatIterator},
|
||||||
key::{KeySlice, TS_ENABLED},
|
key::{KeySlice, TS_ENABLED},
|
||||||
lsm_storage::{LsmStorageInner, LsmStorageOptions},
|
lsm_storage::{LsmStorageInner, LsmStorageOptions},
|
||||||
table::{SsTable, SsTableBuilder},
|
table::{SsTable, SsTableBuilder},
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ use std::io::{BufWriter, Read, Write};
|
|||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use anyhow::{bail, Context, Result};
|
use anyhow::{Context, Result, bail};
|
||||||
use bytes::{Buf, BufMut, Bytes};
|
use bytes::{Buf, BufMut, Bytes};
|
||||||
use crossbeam_skiplist::SkipMap;
|
use crossbeam_skiplist::SkipMap;
|
||||||
use parking_lot::Mutex;
|
use parking_lot::Mutex;
|
||||||
|
|||||||
@@ -14,7 +14,7 @@
|
|||||||
|
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
use anyhow::{anyhow, Result};
|
use anyhow::{Result, anyhow};
|
||||||
use clap::Parser;
|
use clap::Parser;
|
||||||
use console::style;
|
use console::style;
|
||||||
use duct::cmd;
|
use duct::cmd;
|
||||||
|
|||||||
Reference in New Issue
Block a user