зеркало из https://github.com/mozilla/gecko-dev.git
Bug 1543861 - Add auto resizing for kvstore r=myk
This patch implements the auto resizing to handle the MDB_MAP_FULL error. It adds two resizing strategies: 1). Active resizing to increase the store size when it gets opened; 2). Passive resizing to increase the store size when the write transaction encounters the MDB_MAP_FULL error. There are no changes to kvstore APIs, existing consumers will get this feature working without any changes to their code. Differential Revision: https://phabricator.services.mozilla.com/D32216 --HG-- extra : moz-landing-system : lando
This commit is contained in:
Nan Jiang
Родитель
8ea119cb58
Коммит
ac0e950c6e
|
|
@ -461,12 +461,12 @@ dependencies = [
|
|||
"base64 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"crossbeam-utils 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"lmdb-rkv 0.11.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"lmdb-rkv 0.11.4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"moz_task 0.1.0",
|
||||
"nserror 0.1.0",
|
||||
"nsstring 0.1.0",
|
||||
"rkv 0.9.4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rkv 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rust_cascade 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"sha2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"storage_variant 0.1.0",
|
||||
|
|
@ -1542,12 +1542,12 @@ dependencies = [
|
|||
"failure 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"libc 0.2.51 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"lmdb-rkv 0.11.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"lmdb-rkv 0.11.4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"moz_task 0.1.0",
|
||||
"nserror 0.1.0",
|
||||
"nsstring 0.1.0",
|
||||
"rkv 0.9.4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rkv 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"storage_variant 0.1.0",
|
||||
"thin-vec 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"xpcom 0.1.0",
|
||||
|
|
@ -1646,10 +1646,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
|
||||
[[package]]
|
||||
name = "lmdb-rkv"
|
||||
version = "0.11.2"
|
||||
version = "0.11.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"libc 0.2.51 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"lmdb-rkv-sys 0.8.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
|
@ -2529,14 +2530,14 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "rkv"
|
||||
version = "0.9.4"
|
||||
version = "0.9.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"arrayref 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"bincode 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"failure 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"lmdb-rkv 0.11.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"lmdb-rkv 0.11.4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"ordered-float 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde 1.0.88 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde_derive 1.0.88 (git+https://github.com/servo/serde?branch=deserialize_from_enums10)",
|
||||
|
|
@ -3679,12 +3680,12 @@ dependencies = [
|
|||
"failure 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"libc 0.2.51 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"lmdb-rkv 0.11.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"lmdb-rkv 0.11.4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"moz_task 0.1.0",
|
||||
"nserror 0.1.0",
|
||||
"nsstring 0.1.0",
|
||||
"rkv 0.9.4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rkv 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde_json 1.0.26 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"xpcom 0.1.0",
|
||||
]
|
||||
|
|
@ -3856,7 +3857,7 @@ dependencies = [
|
|||
"checksum libudev 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ea626d3bdf40a1c5aee3bcd4f40826970cae8d80a8fec934c82a63840094dcfe"
|
||||
"checksum libz-sys 1.0.16 (registry+https://github.com/rust-lang/crates.io-index)" = "3fdd64ef8ee652185674455c1d450b83cbc8ad895625d543b5324d923f82e4d8"
|
||||
"checksum linked-hash-map 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "70fb39025bc7cdd76305867c4eccf2f2dcf6e9a57f5b21a93e1c2d86cd03ec9e"
|
||||
"checksum lmdb-rkv 0.11.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1452294309db7977dc75e1e8135a8c654d9e52e04ff0c0bd06c880897a91defd"
|
||||
"checksum lmdb-rkv 0.11.4 (registry+https://github.com/rust-lang/crates.io-index)" = "e25b4069789bf7ac069d6fd58229f18aec20c6f7cc9173cb731d11c10dbb6b6e"
|
||||
"checksum lmdb-rkv-sys 0.8.3 (registry+https://github.com/rust-lang/crates.io-index)" = "1470e0168f1832e35afd6d0931ae60db625685332837b97aa156773ec9c5e393"
|
||||
"checksum lock_api 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ed946d4529956a20f2d63ebe1b69996d5a2137c91913fe3ebbeff957f5bca7ff"
|
||||
"checksum log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b"
|
||||
|
|
@ -3940,7 +3941,7 @@ dependencies = [
|
|||
"checksum regex-syntax 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ad890a5eef7953f55427c50575c680c42841653abd2b028b68cd223d157f62db"
|
||||
"checksum regex-syntax 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8f1ac0f60d675cc6cf13a20ec076568254472551051ad5dd050364d70671bf6b"
|
||||
"checksum remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3488ba1b9a2084d38645c4c08276a1752dcbf2c7130d74f1569681ad5d2799c5"
|
||||
"checksum rkv 0.9.4 (registry+https://github.com/rust-lang/crates.io-index)" = "238764bd8750927754d91e4a27155ac672ba88934a2bf698c992d55e5ae25e5b"
|
||||
"checksum rkv 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)" = "2c1b8d667bf149bfac7c47bb728dfb7246f35fdf61c2f16f9f588194f087d23c"
|
||||
"checksum ron 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "da06feaa07f69125ab9ddc769b11de29090122170b402547f64b86fe16ebc399"
|
||||
"checksum runloop 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5d79b4b604167921892e84afbbaad9d5ad74e091bf6c511d9dbfb0593f09fabd"
|
||||
"checksum rust-ini 0.10.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8a654c5bda722c699be6b0fe4c0d90de218928da5b724c3e467fc48865c37263"
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
{"files":{"Cargo.toml":"ba319c56424eed8ac65e6add74a59e9610c2a1e30701801b0ede6313020cd596","LICENSE":"db6d163be642e3b568c5fb2104013da632316ecd4e75935df1613af8b0b37933","README.md":"97b61d73ff27afb03bde9ae960f12651093558214851303c8ae82f567abfe992","azure-pipelines-template.yml":"a75d2421df0feefcb2ea17072c12795e58f9ac6322bacfbea90890fbb5e09cb7","azure-pipelines.yml":"c5a206822c22921bd07ea0a469db734875361457400427991319808cfd13ffef","src/cursor.rs":"f51184cbf015d1aef1a45f0cc1a950524114e1d4aadc50bde7bdb712030a839a","src/database.rs":"003a214f53acd632bc70f2f02d01dcb0bc5bf7e777e1781ef1ff19246f0157d0","src/environment.rs":"774241ef807892b0ebede3392113560563dc8b197c87d97756c47811716a8b4c","src/error.rs":"0ea99c8bc1619f3789eff7734341efa7f48fcd8732dc9f3141804e0a802f5d71","src/flags.rs":"40fd3d4d72c8db8f9ecb893420300a3585e2ca4c49073065ec9ebf24fe23c064","src/lib.rs":"e8c93f6501c58f2d325998730e7991c156a8f53459130181005569ede9d2d871","src/transaction.rs":"09ac6501e3dd1a714c635acb46625c4a4d96d9296cd30bcda31d50224e27fd05"},"package":"1452294309db7977dc75e1e8135a8c654d9e52e04ff0c0bd06c880897a91defd"}
|
||||
{"files":{"Cargo.toml":"aafdb8200f4d1db63ef096c47e1a43a156f4e1690133508e43e9e21b90b15f93","LICENSE":"db6d163be642e3b568c5fb2104013da632316ecd4e75935df1613af8b0b37933","README.md":"97b61d73ff27afb03bde9ae960f12651093558214851303c8ae82f567abfe992","src/cursor.rs":"f51184cbf015d1aef1a45f0cc1a950524114e1d4aadc50bde7bdb712030a839a","src/database.rs":"f0a750497ff32cf1fd4ea21ca0da9234630a6338bb5b4887f72f0abfc6316135","src/environment.rs":"b07df0fe38186e239980904f03db25ca0b1c14f2593dc1bfdefc418b0f26c192","src/error.rs":"0ea99c8bc1619f3789eff7734341efa7f48fcd8732dc9f3141804e0a802f5d71","src/flags.rs":"40fd3d4d72c8db8f9ecb893420300a3585e2ca4c49073065ec9ebf24fe23c064","src/lib.rs":"858e28ebbd9613e0b49b0e8a0ec48a837bc9b2401c0517b5ceabe66b2f0f2a85","src/transaction.rs":"23394768fa7b8603e1f9c3312ba10b8be9fbd74fc26b0300ab93424c3277a400"},"package":"e25b4069789bf7ac069d6fd58229f18aec20c6f7cc9173cb731d11c10dbb6b6e"}
|
||||
|
|
@ -3,7 +3,7 @@
|
|||
# When uploading crates to the registry Cargo will automatically
|
||||
# "normalize" Cargo.toml files for maximal compatibility
|
||||
# with all versions of Cargo and also rewrite `path` dependencies
|
||||
# to registry (e.g. crates.io) dependencies
|
||||
# to registry (e.g., crates.io) dependencies
|
||||
#
|
||||
# If you believe there's an error in this file please file an
|
||||
# issue against the rust-lang/cargo repository. If you're
|
||||
|
|
@ -12,8 +12,9 @@
|
|||
|
||||
[package]
|
||||
name = "lmdb-rkv"
|
||||
version = "0.11.2"
|
||||
version = "0.11.4"
|
||||
authors = ["Dan Burkert <dan@danburkert.com>"]
|
||||
exclude = ["/.appveyor.yml", "/.travis.yml", "/azure-pipelines-template.yml", "/azure-pipelines.yml"]
|
||||
description = "Idiomatic and safe LMDB wrapper."
|
||||
documentation = "https://docs.rs/lmdb-rkv"
|
||||
readme = "README.md"
|
||||
|
|
@ -27,14 +28,14 @@ name = "lmdb"
|
|||
[dependencies.bitflags]
|
||||
version = "1"
|
||||
|
||||
[dependencies.byteorder]
|
||||
version = "1.0"
|
||||
|
||||
[dependencies.libc]
|
||||
version = "0.2"
|
||||
|
||||
[dependencies.lmdb-rkv-sys]
|
||||
version = "0.8.2"
|
||||
[dev-dependencies.byteorder]
|
||||
version = "1.0"
|
||||
|
||||
version = "0.8.3"
|
||||
[dev-dependencies.rand]
|
||||
version = "0.4"
|
||||
|
||||
|
|
|
|||
|
|
@ -1,49 +0,0 @@
|
|||
jobs:
|
||||
- job: ${{ parameters.name }}
|
||||
pool:
|
||||
vmImage: ${{ parameters.vmImage }}
|
||||
strategy:
|
||||
matrix:
|
||||
stable:
|
||||
rustup_toolchain: stable
|
||||
beta:
|
||||
rustup_toolchain: beta
|
||||
nightly:
|
||||
rustup_toolchain: nightly
|
||||
steps:
|
||||
# Linux and macOS.
|
||||
- ${{ if ne(parameters.name, 'Windows') }}:
|
||||
- script: |
|
||||
curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain $RUSTUP_TOOLCHAIN
|
||||
echo "##vso[task.setvariable variable=PATH;]$PATH:$HOME/.cargo/bin"
|
||||
displayName: Install rust
|
||||
# Windows.
|
||||
- ${{ if eq(parameters.name, 'Windows') }}:
|
||||
- script: |
|
||||
curl -sSf -o rustup-init.exe https://win.rustup.rs
|
||||
rustup-init.exe -y --default-toolchain %RUSTUP_TOOLCHAIN%
|
||||
set PATH=%PATH%;%USERPROFILE%\.cargo\bin
|
||||
echo "##vso[task.setvariable variable=PATH;]%PATH%;%USERPROFILE%\.cargo\bin"
|
||||
displayName: Install rust (windows)
|
||||
# All platforms.
|
||||
- script: |
|
||||
rustc -Vv
|
||||
cargo -V
|
||||
displayName: Query rust and cargo versions
|
||||
- script: cargo build
|
||||
displayName: Build
|
||||
# Linux and macOS w/nightly toolchain.
|
||||
# Ideally we'd only run the script for the nightly toolchain, but I can't
|
||||
# figure out how to determine that within the Azure Pipelines conditional.
|
||||
- ${{ if ne(parameters.name, 'Windows') }}:
|
||||
- script: |
|
||||
if [ "$RUSTUP_TOOLCHAIN" = 'nightly' ]
|
||||
then cargo test
|
||||
fi
|
||||
displayName: Test
|
||||
# Windows w/nightly toolchain.
|
||||
# Ideally we'd only run the script for the nightly toolchain, but I can't
|
||||
# figure out how to determine that within the Azure Pipelines conditional.
|
||||
- ${{ if eq(parameters.name, 'Windows') }}:
|
||||
- script: if "%RUSTUP_TOOLCHAIN%" == "nightly" cargo test
|
||||
displayName: Test
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
jobs:
|
||||
- template: azure-pipelines-template.yml
|
||||
parameters:
|
||||
name: macOS
|
||||
vmImage: macOS-10.13
|
||||
|
||||
- template: azure-pipelines-template.yml
|
||||
parameters:
|
||||
name: Linux
|
||||
vmImage: ubuntu-16.04
|
||||
|
||||
- template: azure-pipelines-template.yml
|
||||
parameters:
|
||||
name: Windows
|
||||
vmImage: vs2017-win2016
|
||||
|
|
@ -31,6 +31,12 @@ impl Database {
|
|||
Ok(Database { dbi: dbi })
|
||||
}
|
||||
|
||||
pub(crate) fn freelist_db() -> Database {
|
||||
Database {
|
||||
dbi: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the underlying LMDB database handle.
|
||||
///
|
||||
/// The caller **must** ensure that the handle is not used after the lifetime of the
|
||||
|
|
|
|||
|
|
@ -10,7 +10,10 @@ use std::sync::Mutex;
|
|||
|
||||
use ffi;
|
||||
|
||||
use error::{Result, lmdb_result};
|
||||
use byteorder::{ByteOrder, NativeEndian};
|
||||
|
||||
use cursor::Cursor;
|
||||
use error::{Error, Result, lmdb_result};
|
||||
use database::Database;
|
||||
use transaction::{RoTransaction, RwTransaction, Transaction};
|
||||
use flags::{DatabaseFlags, EnvironmentFlags};
|
||||
|
|
@ -158,18 +161,109 @@ impl Environment {
|
|||
/// Retrieves statistics about this environment.
|
||||
pub fn stat(&self) -> Result<Stat> {
|
||||
unsafe {
|
||||
let mut stat = Stat(mem::zeroed());
|
||||
lmdb_try!(ffi::mdb_env_stat(self.env(), &mut stat.0));
|
||||
let mut stat = Stat::new();
|
||||
lmdb_try!(ffi::mdb_env_stat(self.env(), stat.mdb_stat()));
|
||||
Ok(stat)
|
||||
}
|
||||
}
|
||||
|
||||
/// Retrieves info about this environment.
|
||||
pub fn info(&self) -> Result<Info> {
|
||||
unsafe {
|
||||
let mut info = Info(mem::zeroed());
|
||||
lmdb_try!(ffi::mdb_env_info(self.env(), &mut info.0));
|
||||
Ok(info)
|
||||
}
|
||||
}
|
||||
|
||||
/// Retrieves the total number of pages on the freelist.
|
||||
///
|
||||
/// Along with `Environment::info()`, this can be used to calculate the exact number
|
||||
/// of used pages as well as free pages in this environment.
|
||||
///
|
||||
/// ```ignore
|
||||
/// let env = Environment::new().open("/tmp/test").unwrap();
|
||||
/// let info = env.info().unwrap();
|
||||
/// let stat = env.stat().unwrap();
|
||||
/// let freelist = env.freelist().unwrap();
|
||||
/// let last_pgno = info.last_pgno() + 1; // pgno is 0 based.
|
||||
/// let total_pgs = info.map_size() / stat.page_size() as usize;
|
||||
/// let pgs_in_use = last_pgno - freelist;
|
||||
/// let pgs_free = total_pgs - pgs_in_use;
|
||||
/// ```
|
||||
///
|
||||
/// Note:
|
||||
///
|
||||
/// * LMDB stores all the freelists in the designated database 0 in each environment,
|
||||
/// and the freelist count is stored at the beginning of the value as `libc::size_t`
|
||||
/// in the native byte order.
|
||||
///
|
||||
/// * It will create a read transaction to traverse the freelist database.
|
||||
pub fn freelist(&self) -> Result<size_t> {
|
||||
let mut freelist: size_t = 0;
|
||||
let db = Database::freelist_db();
|
||||
let txn = self.begin_ro_txn()?;
|
||||
let mut cursor = txn.open_ro_cursor(db)?;
|
||||
|
||||
for result in cursor.iter() {
|
||||
let (_key, value) = result?;
|
||||
if value.len() < mem::size_of::<size_t>() {
|
||||
return Err(Error::Corrupted);
|
||||
}
|
||||
|
||||
let s = &value[..mem::size_of::<size_t>()];
|
||||
if cfg!(target_pointer_width = "64") {
|
||||
freelist += NativeEndian::read_u64(s) as size_t;
|
||||
} else {
|
||||
freelist += NativeEndian::read_u32(s) as size_t;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(freelist)
|
||||
}
|
||||
|
||||
/// Sets the size of the memory map to use for the environment.
|
||||
///
|
||||
/// This could be used to resize the map when the environment is already open.
|
||||
///
|
||||
/// Note:
|
||||
///
|
||||
/// * No active transactions allowed when performing resizing in this process.
|
||||
///
|
||||
/// * The size should be a multiple of the OS page size. Any attempt to set
|
||||
/// a size smaller than the space already consumed by the environment will
|
||||
/// be silently changed to the current size of the used space.
|
||||
///
|
||||
/// * In the multi-process case, once a process resizes the map, other
|
||||
/// processes need to either re-open the environment, or call set_map_size
|
||||
/// with size 0 to update the environment. Otherwise, new transaction creation
|
||||
/// will fail with `Error::MapResized`.
|
||||
pub fn set_map_size(&self, size: size_t) -> Result<()> {
|
||||
unsafe {
|
||||
lmdb_result(ffi::mdb_env_set_mapsize(self.env(), size))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Environment statistics.
|
||||
///
|
||||
/// Contains information about the size and layout of an LMDB environment.
|
||||
/// Contains information about the size and layout of an LMDB environment or database.
|
||||
pub struct Stat(ffi::MDB_stat);
|
||||
|
||||
impl Stat {
|
||||
/// Create a new Stat with zero'd inner struct `ffi::MDB_stat`.
|
||||
pub(crate) fn new() -> Stat {
|
||||
unsafe {
|
||||
Stat(mem::zeroed())
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns a mut pointer to `ffi::MDB_stat`.
|
||||
pub(crate) fn mdb_stat(&mut self) -> *mut ffi::MDB_stat {
|
||||
&mut self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl Stat {
|
||||
/// Size of a database page. This is the same for all databases in the environment.
|
||||
#[inline]
|
||||
|
|
@ -208,6 +302,43 @@ impl Stat {
|
|||
}
|
||||
}
|
||||
|
||||
/// Environment information.
|
||||
///
|
||||
/// Contains environment information about the map size, readers, last txn id etc.
|
||||
pub struct Info(ffi::MDB_envinfo);
|
||||
|
||||
impl Info {
|
||||
/// Size of memory map.
|
||||
#[inline]
|
||||
pub fn map_size(&self) -> usize {
|
||||
self.0.me_mapsize
|
||||
}
|
||||
|
||||
/// Last used page number
|
||||
#[inline]
|
||||
pub fn last_pgno(&self) -> usize {
|
||||
self.0.me_last_pgno
|
||||
}
|
||||
|
||||
/// Last transaction ID
|
||||
#[inline]
|
||||
pub fn last_txnid(&self) -> usize {
|
||||
self.0.me_last_txnid
|
||||
}
|
||||
|
||||
/// Max reader slots in the environment
|
||||
#[inline]
|
||||
pub fn max_readers(&self) -> u32 {
|
||||
self.0.me_maxreaders
|
||||
}
|
||||
|
||||
/// Max reader slots used in the environment
|
||||
#[inline]
|
||||
pub fn num_readers(&self) -> u32 {
|
||||
self.0.me_numreaders
|
||||
}
|
||||
}
|
||||
|
||||
unsafe impl Send for Environment {}
|
||||
unsafe impl Sync for Environment {}
|
||||
|
||||
|
|
@ -462,4 +593,75 @@ mod test {
|
|||
assert_eq!(stat.overflow_pages(), 0);
|
||||
assert_eq!(stat.entries(), 64);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_info() {
|
||||
let map_size = 1024 * 1024;
|
||||
let dir = TempDir::new("test").unwrap();
|
||||
let env = Environment::new()
|
||||
.set_map_size(map_size)
|
||||
.open(dir.path())
|
||||
.unwrap();
|
||||
|
||||
let info = env.info().unwrap();
|
||||
assert_eq!(info.map_size(), map_size);
|
||||
assert_eq!(info.last_pgno(), 1);
|
||||
assert_eq!(info.last_txnid(), 0);
|
||||
// The default max readers is 126.
|
||||
assert_eq!(info.max_readers(), 126);
|
||||
assert_eq!(info.num_readers(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_freelist() {
|
||||
let dir = TempDir::new("test").unwrap();
|
||||
let env = Environment::new().open(dir.path()).unwrap();
|
||||
|
||||
let db = env.open_db(None).unwrap();
|
||||
let mut freelist = env.freelist().unwrap();
|
||||
assert_eq!(freelist, 0);
|
||||
|
||||
// Write a few small values.
|
||||
for i in 0..64 {
|
||||
let mut value = [0u8; 8];
|
||||
LittleEndian::write_u64(&mut value, i);
|
||||
let mut tx = env.begin_rw_txn().expect("begin_rw_txn");
|
||||
tx.put(db, &value, &value, WriteFlags::default()).expect("tx.put");
|
||||
tx.commit().expect("tx.commit")
|
||||
}
|
||||
let mut tx = env.begin_rw_txn().expect("begin_rw_txn");
|
||||
tx.clear_db(db).expect("clear");
|
||||
tx.commit().expect("tx.commit");
|
||||
|
||||
// Freelist should not be empty after clear_db.
|
||||
freelist = env.freelist().unwrap();
|
||||
assert!(freelist > 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_set_map_size() {
|
||||
let dir = TempDir::new("test").unwrap();
|
||||
let env = Environment::new().open(dir.path()).unwrap();
|
||||
|
||||
let mut info = env.info().unwrap();
|
||||
let default_size = info.map_size();
|
||||
|
||||
// Resizing to 0 merely reloads the map size
|
||||
env.set_map_size(0).unwrap();
|
||||
info = env.info().unwrap();
|
||||
assert_eq!(info.map_size(), default_size);
|
||||
|
||||
env.set_map_size(2 * default_size).unwrap();
|
||||
info = env.info().unwrap();
|
||||
assert_eq!(info.map_size(), 2 * default_size);
|
||||
|
||||
env.set_map_size(4 * default_size).unwrap();
|
||||
info = env.info().unwrap();
|
||||
assert_eq!(info.map_size(), 4 * default_size);
|
||||
|
||||
// Decreasing is also fine if the space hasn't been consumed.
|
||||
env.set_map_size(2 * default_size).unwrap();
|
||||
info = env.info().unwrap();
|
||||
assert_eq!(info.map_size(), 2 * default_size);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,10 +3,11 @@
|
|||
|
||||
#![cfg_attr(test, feature(test))]
|
||||
#![deny(missing_docs)]
|
||||
#![doc(html_root_url = "https://docs.rs/lmdb-rkv/0.11.2")]
|
||||
#![doc(html_root_url = "https://docs.rs/lmdb-rkv/0.11.4")]
|
||||
|
||||
extern crate libc;
|
||||
extern crate lmdb_rkv_sys as ffi;
|
||||
extern crate byteorder;
|
||||
|
||||
#[cfg(test)] extern crate rand;
|
||||
#[cfg(test)] extern crate tempdir;
|
||||
|
|
@ -21,7 +22,12 @@ pub use cursor::{
|
|||
IterDup,
|
||||
};
|
||||
pub use database::Database;
|
||||
pub use environment::{Environment, Stat, EnvironmentBuilder};
|
||||
pub use environment::{
|
||||
Environment,
|
||||
EnvironmentBuilder,
|
||||
Info,
|
||||
Stat,
|
||||
};
|
||||
pub use error::{Error, Result};
|
||||
pub use flags::*;
|
||||
pub use transaction::{
|
||||
|
|
@ -62,9 +68,7 @@ mod transaction;
|
|||
#[cfg(test)]
|
||||
mod test_utils {
|
||||
|
||||
extern crate byteorder;
|
||||
|
||||
use self::byteorder::{ByteOrder, LittleEndian};
|
||||
use byteorder::{ByteOrder, LittleEndian};
|
||||
use tempdir::TempDir;
|
||||
|
||||
use super::*;
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ use std::marker::PhantomData ;
|
|||
use ffi;
|
||||
|
||||
use cursor::{RoCursor, RwCursor};
|
||||
use environment::Environment;
|
||||
use environment::{Environment, Stat};
|
||||
use database::Database;
|
||||
use error::{Error, Result, lmdb_result};
|
||||
use flags::{DatabaseFlags, EnvironmentFlags, WriteFlags};
|
||||
|
|
@ -102,6 +102,15 @@ pub trait Transaction : Sized {
|
|||
}
|
||||
Ok(DatabaseFlags::from_bits_truncate(flags))
|
||||
}
|
||||
|
||||
/// Retrieves database statistics.
|
||||
fn stat(&self, db: Database) -> Result<Stat> {
|
||||
unsafe {
|
||||
let mut stat = Stat::new();
|
||||
lmdb_try!(ffi::mdb_stat(self.txn(), db.dbi(), stat.mdb_stat()));
|
||||
Ok(stat)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// An LMDB read-only transaction.
|
||||
|
|
@ -654,6 +663,96 @@ mod test {
|
|||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_stat() {
|
||||
let dir = TempDir::new("test").unwrap();
|
||||
let env = Environment::new().open(dir.path()).unwrap();
|
||||
let db = env.create_db(None, DatabaseFlags::empty()).unwrap();
|
||||
|
||||
let mut txn = env.begin_rw_txn().unwrap();
|
||||
txn.put(db, b"key1", b"val1", WriteFlags::empty()).unwrap();
|
||||
txn.put(db, b"key2", b"val2", WriteFlags::empty()).unwrap();
|
||||
txn.put(db, b"key3", b"val3", WriteFlags::empty()).unwrap();
|
||||
txn.commit().unwrap();
|
||||
|
||||
{
|
||||
let txn = env.begin_ro_txn().unwrap();
|
||||
let stat = txn.stat(db).unwrap();
|
||||
assert_eq!(stat.entries(), 3);
|
||||
}
|
||||
|
||||
let mut txn = env.begin_rw_txn().unwrap();
|
||||
txn.del(db, b"key1", None).unwrap();
|
||||
txn.del(db, b"key2", None).unwrap();
|
||||
txn.commit().unwrap();
|
||||
|
||||
{
|
||||
let txn = env.begin_ro_txn().unwrap();
|
||||
let stat = txn.stat(db).unwrap();
|
||||
assert_eq!(stat.entries(), 1);
|
||||
}
|
||||
|
||||
let mut txn = env.begin_rw_txn().unwrap();
|
||||
txn.put(db, b"key4", b"val4", WriteFlags::empty()).unwrap();
|
||||
txn.put(db, b"key5", b"val5", WriteFlags::empty()).unwrap();
|
||||
txn.put(db, b"key6", b"val6", WriteFlags::empty()).unwrap();
|
||||
txn.commit().unwrap();
|
||||
|
||||
{
|
||||
let txn = env.begin_ro_txn().unwrap();
|
||||
let stat = txn.stat(db).unwrap();
|
||||
assert_eq!(stat.entries(), 4);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_stat_dupsort() {
|
||||
let dir = TempDir::new("test").unwrap();
|
||||
let env = Environment::new().open(dir.path()).unwrap();
|
||||
let db = env.create_db(None, DatabaseFlags::DUP_SORT).unwrap();
|
||||
|
||||
let mut txn = env.begin_rw_txn().unwrap();
|
||||
txn.put(db, b"key1", b"val1", WriteFlags::empty()).unwrap();
|
||||
txn.put(db, b"key1", b"val2", WriteFlags::empty()).unwrap();
|
||||
txn.put(db, b"key1", b"val3", WriteFlags::empty()).unwrap();
|
||||
txn.put(db, b"key2", b"val1", WriteFlags::empty()).unwrap();
|
||||
txn.put(db, b"key2", b"val2", WriteFlags::empty()).unwrap();
|
||||
txn.put(db, b"key2", b"val3", WriteFlags::empty()).unwrap();
|
||||
txn.put(db, b"key3", b"val1", WriteFlags::empty()).unwrap();
|
||||
txn.put(db, b"key3", b"val2", WriteFlags::empty()).unwrap();
|
||||
txn.put(db, b"key3", b"val3", WriteFlags::empty()).unwrap();
|
||||
txn.commit().unwrap();
|
||||
|
||||
{
|
||||
let txn = env.begin_ro_txn().unwrap();
|
||||
let stat = txn.stat(db).unwrap();
|
||||
assert_eq!(stat.entries(), 9);
|
||||
}
|
||||
|
||||
let mut txn = env.begin_rw_txn().unwrap();
|
||||
txn.del(db, b"key1", Some(b"val2")).unwrap();
|
||||
txn.del(db, b"key2", None).unwrap();
|
||||
txn.commit().unwrap();
|
||||
|
||||
{
|
||||
let txn = env.begin_ro_txn().unwrap();
|
||||
let stat = txn.stat(db).unwrap();
|
||||
assert_eq!(stat.entries(), 5);
|
||||
}
|
||||
|
||||
let mut txn = env.begin_rw_txn().unwrap();
|
||||
txn.put(db, b"key4", b"val1", WriteFlags::empty()).unwrap();
|
||||
txn.put(db, b"key4", b"val2", WriteFlags::empty()).unwrap();
|
||||
txn.put(db, b"key4", b"val3", WriteFlags::empty()).unwrap();
|
||||
txn.commit().unwrap();
|
||||
|
||||
{
|
||||
let txn = env.begin_ro_txn().unwrap();
|
||||
let stat = txn.stat(db).unwrap();
|
||||
assert_eq!(stat.entries(), 8);
|
||||
}
|
||||
}
|
||||
|
||||
#[bench]
|
||||
fn bench_get_rand(b: &mut Bencher) {
|
||||
let n = 100u32;
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
{"files":{"Cargo.toml":"bb25bb1f8a98037fac1f33ffef7244b6363396e6220af4b856b9fe0616c71b81","LICENSE":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30","README.md":"9dc24375b49fef42f35dec42e316e21827d7337622f9e7cf36243cd28808797a","examples/README.md":"143767fc145bf167ce269a65138cb3f7086cb715b8bc4f73626da82966e646f4","examples/iterator.rs":"ddc3997e394a30ad82d78d2675a48c4617353f88b89bb9a3df5a3804d59b8ef9","examples/simple-store.rs":"cae63e39f2f98ee6ac2f387dcb02d6b929828a74f32f7d18d69c7fc9c3cce765","run-all-examples.sh":"7f9d11d01017f77e1c9d26e3e82dfca8c6930deaec85e864458e33a7fa267de0","src/env.rs":"f886c42b8ea0633ed001d24fee2edcc246b7b0dd2b56dcfbdebf4aef4a36f7a0","src/error.rs":"46632b8fcb1070a1860247e09a59d39772079ebfba5d3d1bbee03d08e1252275","src/lib.rs":"67a1970626fcecf35c0a9ccb0305afbfb12b8a85e3d5060bff4c6617a3d1de78","src/manager.rs":"f06b14ee64f2e58d890a3b37677790b707a02d328242c1af0ce3c74e9028edd8","src/readwrite.rs":"fde695333e4845f4f53d63da6281f585919e2a3ac5cfe00d173cc139bc822763","src/store.rs":"409d13b1ea0d1254dae947ecbce50e741fb71c3ca118a78803b734336dce6a8f","src/store/integer.rs":"f386474c971f671c9b316a16ebff5b586be6837c886f443753ae13277a7e0070","src/store/integermulti.rs":"1a0912f97619297da31cc8c146e38941b88539d2857df81191a49c8dbd18625d","src/store/multi.rs":"2dec01c2202a2c9069cced4e1e42906b01d0b85df25d17e0ea810c05fa8395d0","src/store/single.rs":"c55c3600714f5ed9e820b16c2335ae00a0071174e0a32b9df89a34182a4b908c","src/value.rs":"ad74ba4c9ab0a77f1c4f8ee2650ceeb148e4036b017d804affc35085e97944fb","tests/integer-store.rs":"f7e06c71b0dead2323c7c61fc8bcbffbdd3a4796eebf6138db9cce3dbba716a3","tests/manager.rs":"97ec61145dc227f4f5fbcb6449c096bbe5b9a09db4e61ff4491c0443fe9adf26","tests/multi-integer-store.rs":"83295b0135c502321304aa06b05d5a9eeab41b1438ed7ddf2cb1a3613dfef4d9"},"package":"238764bd8750927754d91e4a27155ac672ba88934a2bf698c992d55e5ae25e5b"}
|
||||
{"files":{"CODE_OF_CONDUCT.md":"902d5357af363426631d907e641e220b3ec89039164743f8442b3f120479b7cf","Cargo.toml":"27ac84d5a81e9b054e79f7615086b5625071aa88803202f7c8de4a1e53e32e83","LICENSE":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30","README.md":"9dc24375b49fef42f35dec42e316e21827d7337622f9e7cf36243cd28808797a","examples/README.md":"143767fc145bf167ce269a65138cb3f7086cb715b8bc4f73626da82966e646f4","examples/iterator.rs":"ddc3997e394a30ad82d78d2675a48c4617353f88b89bb9a3df5a3804d59b8ef9","examples/simple-store.rs":"cae63e39f2f98ee6ac2f387dcb02d6b929828a74f32f7d18d69c7fc9c3cce765","run-all-examples.sh":"7f9d11d01017f77e1c9d26e3e82dfca8c6930deaec85e864458e33a7fa267de0","src/env.rs":"67b4a98d9631bb61ca6181afc557b7fda0876de2959cfb78bd7049da3cd0a623","src/error.rs":"46632b8fcb1070a1860247e09a59d39772079ebfba5d3d1bbee03d08e1252275","src/lib.rs":"6b21c99c067f550124986b3c15d018b1a090351c339758c0a76ba4f58f818f5d","src/manager.rs":"6142e31ec0d31fa18ed0fe29b123de020ad81ecaec21263c374742548f842ee8","src/readwrite.rs":"fde695333e4845f4f53d63da6281f585919e2a3ac5cfe00d173cc139bc822763","src/store.rs":"409d13b1ea0d1254dae947ecbce50e741fb71c3ca118a78803b734336dce6a8f","src/store/integer.rs":"f386474c971f671c9b316a16ebff5b586be6837c886f443753ae13277a7e0070","src/store/integermulti.rs":"1a0912f97619297da31cc8c146e38941b88539d2857df81191a49c8dbd18625d","src/store/multi.rs":"2dec01c2202a2c9069cced4e1e42906b01d0b85df25d17e0ea810c05fa8395d0","src/store/single.rs":"c55c3600714f5ed9e820b16c2335ae00a0071174e0a32b9df89a34182a4b908c","src/value.rs":"7fae77a8291b951591e557ec694bfdadc9eb78557dad36a970cfcdcfb83fd238","tests/integer-store.rs":"f7e06c71b0dead2323c7c61fc8bcbffbdd3a4796eebf6138db9cce3dbba716a3","tests/manager.rs":"97ec61145dc227f4f5fbcb6449c096bbe5b9a09db4e61ff4491c0443fe9adf26","tests/multi-integer-store.rs":"83295b0135c502321304aa06b05d5a9eeab41b1438ed7ddf2cb1a3613dfef4d9"},"package":"2c1b8d667bf149bfac7c47bb728dfb7246f35fdf61c2f16f9f588194f087d23c"}
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
# Community Participation Guidelines
|
||||
|
||||
This repository is governed by Mozilla's code of conduct and etiquette guidelines.
|
||||
For more details, please read the
|
||||
[Mozilla Community Participation Guidelines](https://www.mozilla.org/about/governance/policies/participation/).
|
||||
|
||||
## How to Report
|
||||
For more information on how to report violations of the Community Participation Guidelines, please read our '[How to Report](https://www.mozilla.org/about/governance/policies/participation/reporting/)' page.
|
||||
|
||||
<!--
|
||||
## Project Specific Etiquette
|
||||
|
||||
In some cases, there will be additional project etiquette i.e.: (https://bugzilla.mozilla.org/page.cgi?id=etiquette.html).
|
||||
Please update for your project.
|
||||
-->
|
||||
|
|
@ -3,7 +3,7 @@
|
|||
# When uploading crates to the registry Cargo will automatically
|
||||
# "normalize" Cargo.toml files for maximal compatibility
|
||||
# with all versions of Cargo and also rewrite `path` dependencies
|
||||
# to registry (e.g. crates.io) dependencies
|
||||
# to registry (e.g., crates.io) dependencies
|
||||
#
|
||||
# If you believe there's an error in this file please file an
|
||||
# issue against the rust-lang/cargo repository. If you're
|
||||
|
|
@ -13,7 +13,7 @@
|
|||
[package]
|
||||
edition = "2018"
|
||||
name = "rkv"
|
||||
version = "0.9.4"
|
||||
version = "0.9.5"
|
||||
authors = ["Richard Newman <rnewman@twinql.com>", "Nan Jiang <najiang@mozilla.com>", "Myk Melez <myk@mykzilla.org>"]
|
||||
description = "a simple, humane, typed Rust interface to LMDB"
|
||||
homepage = "https://github.com/mozilla/rkv"
|
||||
|
|
@ -37,7 +37,7 @@ default_features = false
|
|||
version = "1.0.2"
|
||||
|
||||
[dependencies.lmdb-rkv]
|
||||
version = "0.11.2"
|
||||
version = "0.11.4"
|
||||
|
||||
[dependencies.ordered-float]
|
||||
version = "1.0"
|
||||
|
|
|
|||
|
|
@ -22,6 +22,8 @@ use lmdb::{
|
|||
DatabaseFlags,
|
||||
Environment,
|
||||
EnvironmentBuilder,
|
||||
Error,
|
||||
Info,
|
||||
Stat,
|
||||
};
|
||||
|
||||
|
|
@ -197,13 +199,75 @@ impl Rkv {
|
|||
self.env.sync(force).map_err(Into::into)
|
||||
}
|
||||
|
||||
/// Retrieves statistics about this environment.
|
||||
/// Retrieve statistics about this environment.
|
||||
///
|
||||
/// It includes:
|
||||
/// * Page size in bytes
|
||||
/// * B-tree depth
|
||||
/// * Number of internal (non-leaf) pages
|
||||
/// * Number of leaf pages
|
||||
/// * Number of overflow pages
|
||||
/// * Number of data entries
|
||||
pub fn stat(&self) -> Result<Stat, StoreError> {
|
||||
self.env.stat().map_err(Into::into)
|
||||
}
|
||||
|
||||
/// Retrieve information about this environment.
|
||||
///
|
||||
/// It includes:
|
||||
/// * Map size in bytes
|
||||
/// * The last used page number
|
||||
/// * The last transaction ID
|
||||
/// * Max number of readers allowed
|
||||
/// * Number of readers in use
|
||||
pub fn info(&self) -> Result<Info, StoreError> {
|
||||
self.env.info().map_err(Into::into)
|
||||
}
|
||||
|
||||
/// Retrieve the load ratio (# of used pages / total pages) about this environment.
|
||||
///
|
||||
/// With the formular: (last_page_no - freelist_pages) / total_pages
|
||||
pub fn load_ratio(&self) -> Result<f32, StoreError> {
|
||||
let stat = self.stat()?;
|
||||
let info = self.info()?;
|
||||
let freelist = self.env.freelist()?;
|
||||
|
||||
let last_pgno = info.last_pgno() + 1; // pgno is 0 based.
|
||||
let total_pgs = info.map_size() / stat.page_size() as usize;
|
||||
if freelist > last_pgno {
|
||||
return Err(StoreError::LmdbError(Error::Corrupted));
|
||||
}
|
||||
let used_pgs = last_pgno - freelist;
|
||||
Ok(used_pgs as f32 / total_pgs as f32)
|
||||
}
|
||||
|
||||
/// Sets the size of the memory map to use for the environment.
|
||||
///
|
||||
/// This can be used to resize the map when the environment is already open.
|
||||
/// You can also use `Rkv::environment_builder()` to set the map size during
|
||||
/// the `Rkv` initialization.
|
||||
///
|
||||
/// Note:
|
||||
///
|
||||
/// * No active transactions allowed when performing resizing in this process.
|
||||
/// It's up to the consumer to enforce that.
|
||||
///
|
||||
/// * The size should be a multiple of the OS page size. Any attempt to set
|
||||
/// a size smaller than the space already consumed by the environment will
|
||||
/// be silently changed to the current size of the used space.
|
||||
///
|
||||
/// * In the multi-process case, once a process resizes the map, other
|
||||
/// processes need to either re-open the environment, or call set_map_size
|
||||
/// with size 0 to update the environment. Otherwise, new transaction creation
|
||||
/// will fail with `LmdbError::MapResized`.
|
||||
pub fn set_map_size(&self, size: usize) -> Result<(), StoreError> {
|
||||
self.env.set_map_size(size).map_err(Into::into)
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::cyclomatic_complexity)]
|
||||
// TODO: change this back to `clippy::cognitive_complexity` when Clippy stable
|
||||
// deprecates `clippy::cyclomatic_complexity`.
|
||||
#[allow(clippy::complexity)]
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use byteorder::{
|
||||
|
|
@ -224,6 +288,9 @@ mod tests {
|
|||
use super::*;
|
||||
use crate::*;
|
||||
|
||||
// The default size is 1MB.
|
||||
const DEFAULT_SIZE: usize = 1024 * 1024;
|
||||
|
||||
/// We can't open a directory that doesn't exist.
|
||||
#[test]
|
||||
fn test_open_fails() {
|
||||
|
|
@ -308,7 +375,7 @@ mod tests {
|
|||
// https://github.com/LMDB/lmdb/blob/26c7df88e44e31623d0802a564f24781acdefde3/libraries/liblmdb/mdb.c#L729
|
||||
// sets the default map size to 1,048,576 bytes, i.e. 1MiB.
|
||||
//
|
||||
1024 * 1024 + 1 /* 1,048,576 + 1 bytes, i.e. 1MiB + 1 byte */
|
||||
DEFAULT_SIZE + 1 /* 1,048,576 + 1 bytes, i.e. 1MiB + 1 byte */
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -773,7 +840,7 @@ mod tests {
|
|||
|
||||
#[test]
|
||||
fn test_stat() {
|
||||
let root = Builder::new().prefix("test_sync").tempdir().expect("tempdir");
|
||||
let root = Builder::new().prefix("test_stat").tempdir().expect("tempdir");
|
||||
fs::create_dir_all(root.path()).expect("dir created");
|
||||
let k = Rkv::new(root.path()).expect("new succeeded");
|
||||
for i in 0..5 {
|
||||
|
|
@ -791,6 +858,84 @@ mod tests {
|
|||
assert_eq!(k.stat().expect("stat").leaf_pages(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_info() {
|
||||
let root = Builder::new().prefix("test_info").tempdir().expect("tempdir");
|
||||
fs::create_dir_all(root.path()).expect("dir created");
|
||||
let k = Rkv::new(root.path()).expect("new succeeded");
|
||||
let sk: SingleStore = k.open_single("sk", StoreOptions::create()).expect("opened");
|
||||
let mut writer = k.write().expect("writer");
|
||||
|
||||
sk.put(&mut writer, "foo", &Value::Str("bar")).expect("wrote");
|
||||
writer.commit().expect("commited");
|
||||
|
||||
let info = k.info().expect("info");
|
||||
|
||||
// The default size is 1MB.
|
||||
assert_eq!(info.map_size(), DEFAULT_SIZE);
|
||||
// Should greater than 0 after the write txn.
|
||||
assert!(info.last_pgno() > 0);
|
||||
// A txn to open_single + a txn to write.
|
||||
assert_eq!(info.last_txnid(), 2);
|
||||
// The default max readers is 126.
|
||||
assert_eq!(info.max_readers(), 126);
|
||||
assert_eq!(info.num_readers(), 0);
|
||||
|
||||
// A new reader should increment the reader counter.
|
||||
let _reader = k.read().expect("reader");
|
||||
let info = k.info().expect("info");
|
||||
|
||||
assert_eq!(info.num_readers(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_load_ratio() {
|
||||
let root = Builder::new().prefix("test_load_ratio").tempdir().expect("tempdir");
|
||||
fs::create_dir_all(root.path()).expect("dir created");
|
||||
let k = Rkv::new(root.path()).expect("new succeeded");
|
||||
let sk: SingleStore = k.open_single("sk", StoreOptions::create()).expect("opened");
|
||||
let mut writer = k.write().expect("writer");
|
||||
sk.put(&mut writer, "foo", &Value::Str("bar")).expect("wrote");
|
||||
writer.commit().expect("commited");
|
||||
|
||||
let ratio = k.load_ratio().expect("ratio");
|
||||
assert!(ratio > 0.0_f32 && ratio < 1.0_f32);
|
||||
|
||||
// Put data to database should increase the load ratio.
|
||||
let mut writer = k.write().expect("writer");
|
||||
sk.put(&mut writer, "bar", &Value::Str(&"more-than-4KB".repeat(1000))).expect("wrote");
|
||||
writer.commit().expect("commited");
|
||||
let new_ratio = k.load_ratio().expect("ratio");
|
||||
assert!(new_ratio > ratio);
|
||||
|
||||
// Clear the database so that all the used pages should go to freelist, hence the ratio
|
||||
// should decrease.
|
||||
let mut writer = k.write().expect("writer");
|
||||
sk.clear(&mut writer).expect("clear");
|
||||
writer.commit().expect("commited");
|
||||
let after_clear_ratio = k.load_ratio().expect("ratio");
|
||||
assert!(after_clear_ratio < new_ratio);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_set_map_size() {
|
||||
let root = Builder::new().prefix("test_size_map_size").tempdir().expect("tempdir");
|
||||
fs::create_dir_all(root.path()).expect("dir created");
|
||||
let k = Rkv::new(root.path()).expect("new succeeded");
|
||||
let sk: SingleStore = k.open_single("sk", StoreOptions::create()).expect("opened");
|
||||
|
||||
assert_eq!(k.info().expect("info").map_size(), DEFAULT_SIZE);
|
||||
|
||||
k.set_map_size(2 * DEFAULT_SIZE).expect("resized");
|
||||
|
||||
// Should be able to write.
|
||||
let mut writer = k.write().expect("writer");
|
||||
sk.put(&mut writer, "foo", &Value::Str("bar")).expect("wrote");
|
||||
writer.commit().expect("commited");
|
||||
|
||||
assert_eq!(k.info().expect("info").map_size(), 2 * DEFAULT_SIZE);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_iter() {
|
||||
let root = Builder::new().prefix("test_iter").tempdir().expect("tempdir");
|
||||
|
|
|
|||
|
|
@ -216,6 +216,7 @@ pub mod value;
|
|||
pub use lmdb::{
|
||||
Cursor,
|
||||
Database,
|
||||
Info,
|
||||
Iter as LmdbIter,
|
||||
RoCursor,
|
||||
Stat,
|
||||
|
|
|
|||
|
|
@ -37,8 +37,6 @@ use crate::error::StoreError;
|
|||
|
||||
use crate::Rkv;
|
||||
|
||||
/// A process is only permitted to have one open handle to each Rkv environment.
|
||||
/// This manager exists to enforce that constraint: don't open environments directly.
|
||||
lazy_static! {
|
||||
static ref MANAGER: RwLock<Manager> = RwLock::new(Manager::new());
|
||||
}
|
||||
|
|
@ -57,6 +55,8 @@ where
|
|||
Ok(canonical)
|
||||
}
|
||||
|
||||
/// A process is only permitted to have one open handle to each Rkv environment.
|
||||
/// This manager exists to enforce that constraint: don't open environments directly.
|
||||
pub struct Manager {
|
||||
environments: BTreeMap<PathBuf, Arc<RwLock<Rkv>>>,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ use arrayref::array_ref;
|
|||
use bincode::{
|
||||
deserialize,
|
||||
serialize,
|
||||
serialized_size,
|
||||
};
|
||||
use ordered_float::OrderedFloat;
|
||||
|
||||
|
|
@ -176,10 +177,22 @@ impl<'s> Value<'s> {
|
|||
Value::Str(v) => serialize(&(Type::Str.to_tag(), v)),
|
||||
Value::Json(v) => serialize(&(Type::Json.to_tag(), v)),
|
||||
Value::Blob(v) => serialize(&(Type::Blob.to_tag(), v)),
|
||||
Value::Uuid(v) => {
|
||||
// Processed above to avoid verbose duplication of error transforms.
|
||||
serialize(&(Type::Uuid.to_tag(), v))
|
||||
},
|
||||
Value::Uuid(v) => serialize(&(Type::Uuid.to_tag(), v)),
|
||||
}
|
||||
.map_err(DataError::EncodingError)
|
||||
}
|
||||
|
||||
pub fn serialized_size(&self) -> Result<u64, DataError> {
|
||||
match self {
|
||||
Value::Bool(v) => serialized_size(&(Type::Bool.to_tag(), *v)),
|
||||
Value::U64(v) => serialized_size(&(Type::U64.to_tag(), *v)),
|
||||
Value::I64(v) => serialized_size(&(Type::I64.to_tag(), *v)),
|
||||
Value::F64(v) => serialized_size(&(Type::F64.to_tag(), v.0)),
|
||||
Value::Instant(v) => serialized_size(&(Type::Instant.to_tag(), *v)),
|
||||
Value::Str(v) => serialized_size(&(Type::Str.to_tag(), v)),
|
||||
Value::Json(v) => serialized_size(&(Type::Json.to_tag(), v)),
|
||||
Value::Blob(v) => serialized_size(&(Type::Blob.to_tag(), v)),
|
||||
Value::Uuid(v) => serialized_size(&(Type::Uuid.to_tag(), v)),
|
||||
}
|
||||
.map_err(DataError::EncodingError)
|
||||
}
|
||||
|
|
@ -216,3 +229,37 @@ impl<'s> From<&'s OwnedValue> for Value<'s> {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use ordered_float::OrderedFloat;
|
||||
|
||||
#[test]
|
||||
fn test_value_serialized_size() {
|
||||
// | Value enum | tag: 1 byte | value_payload |
|
||||
// |----------------------------------------------------------|
|
||||
// | I64 | 1 | 8 |
|
||||
// | U64 | 1 | 8 |
|
||||
// | Bool | 1 | 1 |
|
||||
// | Instant | 1 | 8 |
|
||||
// | F64 | 1 | 8 |
|
||||
// | Uuid | 1 | 16 |
|
||||
// | Str/Blob/Json | 1 |(8: len + sizeof(payload))|
|
||||
assert_eq!(Value::I64(-1000).serialized_size().unwrap(), 9);
|
||||
assert_eq!(Value::U64(1000u64).serialized_size().unwrap(), 9);
|
||||
assert_eq!(Value::Bool(true).serialized_size().unwrap(), 2);
|
||||
assert_eq!(Value::Instant(1_558_020_865_224).serialized_size().unwrap(), 9);
|
||||
assert_eq!(Value::F64(OrderedFloat(10000.1)).serialized_size().unwrap(), 9);
|
||||
assert_eq!(Value::Str("hello!").serialized_size().unwrap(), 15);
|
||||
assert_eq!(Value::Str("¡Hola").serialized_size().unwrap(), 15);
|
||||
assert_eq!(Value::Blob(b"hello!").serialized_size().unwrap(), 15);
|
||||
assert_eq!(
|
||||
uuid(b"\x9f\xe2\xc4\xe9\x3f\x65\x4f\xdb\xb2\x4c\x02\xb1\x52\x59\x71\x6c")
|
||||
.unwrap()
|
||||
.serialized_size()
|
||||
.unwrap(),
|
||||
17
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -86,6 +86,76 @@ macro_rules! task_done {
|
|||
/// declare the type of `GetOrCreateTask::result`.
|
||||
type RkvStoreTuple = (Arc<RwLock<Rkv>>, SingleStore);
|
||||
|
||||
// The threshold for active resizing.
|
||||
const RESIZE_RATIO: f32 = 0.85;
|
||||
|
||||
/// The threshold (50 MB) to switch the resizing policy from the double size to
|
||||
/// the constant increment for active resizing.
|
||||
const INCREMENTAL_RESIZE_THRESHOLD: usize = 52_428_800;
|
||||
|
||||
/// The incremental resize step (5 MB)
|
||||
const INCREMENTAL_RESIZE_STEP: usize = 5_242_880;
|
||||
|
||||
/// The LMDB disk page size and mask.
|
||||
const PAGE_SIZE: usize = 4096;
|
||||
const PAGE_SIZE_MASK: usize = 0b_1111_1111_1111;
|
||||
|
||||
/// Round the non-zero size to the multiple of page size greater or equal.
|
||||
///
|
||||
/// It does not handle the special cases such as size zero and overflow,
|
||||
/// because even if that happens (extremely unlikely though), LMDB will
|
||||
/// ignore the new size if it's smaller than the current size.
|
||||
///
|
||||
/// E.g:
|
||||
/// [ 1 - 4096] -> 4096,
|
||||
/// [4097 - 8192] -> 8192,
|
||||
/// [8193 - 12286] -> 12286,
|
||||
fn round_to_pagesize(size: usize) -> usize {
|
||||
if size & PAGE_SIZE_MASK == 0 {
|
||||
size
|
||||
} else {
|
||||
(size & !PAGE_SIZE_MASK) + PAGE_SIZE
|
||||
}
|
||||
}
|
||||
|
||||
/// Kvstore employes two auto resizing strategies: active and passive resizing.
|
||||
/// They work together to liberate consumers from having to guess the "proper"
|
||||
/// size of the store upfront. See more detail about this in Bug 1543861.
|
||||
///
|
||||
/// Active resizing that is performed at the store startup.
|
||||
///
|
||||
/// It either increases the size in double, or by a constant size if its size
|
||||
/// reaches INCREMENTAL_RESIZE_THRESHOLD.
|
||||
///
|
||||
/// Note that on Linux / MAC OSX, the increased size would only take effect if
|
||||
/// there is a write transaction committed afterwards.
|
||||
fn active_resize(env: &Rkv) -> Result<(), StoreError> {
|
||||
let info = env.info()?;
|
||||
let current_size = info.map_size();
|
||||
let size;
|
||||
|
||||
if current_size < INCREMENTAL_RESIZE_THRESHOLD {
|
||||
size = current_size << 1;
|
||||
} else {
|
||||
size = current_size + INCREMENTAL_RESIZE_STEP;
|
||||
}
|
||||
|
||||
env.set_map_size(size)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Passive resizing that is performed when the MAP_FULL error occurs. It
|
||||
/// increases the store with a `wanted` size.
|
||||
///
|
||||
/// Note that the `wanted` size must be rounded to a multiple of page size
|
||||
/// by using `round_to_pagesize`.
|
||||
fn passive_resize(env: &Rkv, wanted: usize) -> Result<(), StoreError> {
|
||||
let info = env.info()?;
|
||||
let current_size = info.map_size();
|
||||
env.set_map_size(current_size + wanted)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub struct GetOrCreateTask {
|
||||
callback: AtomicCell<Option<ThreadBoundRefPtr<nsIKeyValueDatabaseCallback>>>,
|
||||
thread: AtomicCell<Option<ThreadBoundRefPtr<nsIThread>>>,
|
||||
|
|
@ -122,11 +192,17 @@ impl Task for GetOrCreateTask {
|
|||
// use the ? operator to simplify the implementation.
|
||||
self.result
|
||||
.store(Some(|| -> Result<RkvStoreTuple, KeyValueError> {
|
||||
let store;
|
||||
let mut writer = Manager::singleton().write()?;
|
||||
let rkv = writer.get_or_create(Path::new(str::from_utf8(&self.path)?), Rkv::new)?;
|
||||
let store = rkv
|
||||
.write()?
|
||||
.open_single(str::from_utf8(&self.name)?, StoreOptions::create())?;
|
||||
{
|
||||
let env = rkv.read()?;
|
||||
let load_ratio = env.load_ratio()?;
|
||||
if load_ratio > RESIZE_RATIO {
|
||||
active_resize(&env)?;
|
||||
}
|
||||
store = env.open_single(str::from_utf8(&self.name)?, StoreOptions::create())?;
|
||||
}
|
||||
Ok((rkv, store))
|
||||
}()));
|
||||
}
|
||||
|
|
@ -167,13 +243,40 @@ impl Task for PutTask {
|
|||
// We do the work within a closure that returns a Result so we can
|
||||
// use the ? operator to simplify the implementation.
|
||||
self.result.store(Some(|| -> Result<(), KeyValueError> {
|
||||
let key = str::from_utf8(&self.key)?;
|
||||
let env = self.rkv.read()?;
|
||||
let mut writer = env.write()?;
|
||||
let key = str::from_utf8(&self.key)?;
|
||||
let v = Value::from(&self.value);
|
||||
let mut resized = false;
|
||||
|
||||
self.store
|
||||
.put(&mut writer, key, &Value::from(&self.value))?;
|
||||
writer.commit()?;
|
||||
// Use a loop here in case we want to retry from a recoverable
|
||||
// error such as `lmdb::Error::MapFull`.
|
||||
loop {
|
||||
let mut writer = env.write()?;
|
||||
|
||||
match self.store.put(&mut writer, key, &v) {
|
||||
Ok(_) => (),
|
||||
|
||||
// Only handle the first MapFull error via passive resizing.
|
||||
// Propogate the subsequent MapFull error.
|
||||
Err(StoreError::LmdbError(lmdb::Error::MapFull)) if !resized => {
|
||||
// abort the failed transaction for resizing.
|
||||
writer.abort();
|
||||
|
||||
// calculate the size of pairs and resize the store accordingly.
|
||||
let pair_size = key.len()
|
||||
+ v.serialized_size().map_err(StoreError::from)? as usize;
|
||||
let wanted = round_to_pagesize(pair_size);
|
||||
passive_resize(&env, wanted)?;
|
||||
resized = true;
|
||||
continue;
|
||||
},
|
||||
|
||||
Err(err) => return Err(KeyValueError::StoreError(err)),
|
||||
}
|
||||
|
||||
writer.commit()?;
|
||||
break;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}()));
|
||||
|
|
@ -205,6 +308,21 @@ impl WriteManyTask {
|
|||
result: AtomicCell::default(),
|
||||
}
|
||||
}
|
||||
|
||||
fn calc_pair_size(&self) -> Result<usize, StoreError> {
|
||||
let mut total = 0;
|
||||
|
||||
for (key, value) in self.pairs.iter() {
|
||||
if let Some(val) = value {
|
||||
total += key.len();
|
||||
total += Value::from(val)
|
||||
.serialized_size()
|
||||
.map_err(StoreError::from)? as usize;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(total)
|
||||
}
|
||||
}
|
||||
|
||||
impl Task for WriteManyTask {
|
||||
|
|
@ -213,27 +331,57 @@ impl Task for WriteManyTask {
|
|||
// use the ? operator to simplify the implementation.
|
||||
self.result.store(Some(|| -> Result<(), KeyValueError> {
|
||||
let env = self.rkv.read()?;
|
||||
let mut writer = env.write()?;
|
||||
let mut resized = false;
|
||||
|
||||
for (key, value) in self.pairs.iter() {
|
||||
let key = str::from_utf8(key)?;
|
||||
match value {
|
||||
Some(val) => self.store.put(&mut writer, key, &Value::from(val))?,
|
||||
None => {
|
||||
match self.store.delete(&mut writer, key) {
|
||||
Ok(_) => (),
|
||||
// Use a loop here in case we want to retry from a recoverable
|
||||
// error such as `lmdb::Error::MapFull`.
|
||||
'outer: loop {
|
||||
let mut writer = env.write()?;
|
||||
|
||||
// LMDB fails with an error if the key to delete wasn't found,
|
||||
// and Rkv returns that error, but we ignore it, as we expect most
|
||||
// of our consumers to want this behavior.
|
||||
Err(StoreError::LmdbError(lmdb::Error::NotFound)) => (),
|
||||
for (key, value) in self.pairs.iter() {
|
||||
let key = str::from_utf8(key)?;
|
||||
match value {
|
||||
// To put.
|
||||
Some(val) => {
|
||||
match self.store.put(&mut writer, key, &Value::from(val)) {
|
||||
Ok(_) => (),
|
||||
|
||||
Err(err) => return Err(KeyValueError::StoreError(err)),
|
||||
};
|
||||
// Only handle the first MapFull error via passive resizing.
|
||||
// Propogate the subsequent MapFull error.
|
||||
Err(StoreError::LmdbError(lmdb::Error::MapFull)) if !resized => {
|
||||
// Abort the failed transaction for resizing.
|
||||
writer.abort();
|
||||
|
||||
// Calculate the size of pairs and resize accordingly.
|
||||
let pair_size = self.calc_pair_size()?;
|
||||
let wanted = round_to_pagesize(pair_size);
|
||||
passive_resize(&env, wanted)?;
|
||||
resized = true;
|
||||
continue 'outer;
|
||||
},
|
||||
|
||||
Err(err) => return Err(KeyValueError::StoreError(err)),
|
||||
}
|
||||
},
|
||||
// To delete.
|
||||
None => {
|
||||
match self.store.delete(&mut writer, key) {
|
||||
Ok(_) => (),
|
||||
|
||||
// LMDB fails with an error if the key to delete wasn't found,
|
||||
// and Rkv returns that error, but we ignore it, as we expect most
|
||||
// of our consumers to want this behavior.
|
||||
Err(StoreError::LmdbError(lmdb::Error::NotFound)) => (),
|
||||
|
||||
Err(err) => return Err(KeyValueError::StoreError(err)),
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
writer.commit()?;
|
||||
break; // 'outer: loop
|
||||
}
|
||||
writer.commit()?;
|
||||
|
||||
Ok(())
|
||||
}()));
|
||||
|
|
|
|||
|
|
@ -100,6 +100,24 @@ add_task(async function putGetHasDelete() {
|
|||
Assert.strictEqual(await database.get("bool-key"), null);
|
||||
});
|
||||
|
||||
add_task(async function putWithResizing() {
|
||||
const databaseDir = await makeDatabaseDir("putWithResizing");
|
||||
const database = await KeyValueService.getOrCreate(databaseDir, "db");
|
||||
|
||||
// The default store size is 1MB, putting key/value pairs bigger than that
|
||||
// would trigger auto resizing.
|
||||
const base = "A humongous string in 32 bytes!!";
|
||||
const val1M = base.repeat(32768);
|
||||
const val2M = val1M.repeat(2);
|
||||
Assert.strictEqual(await database.put("A-1M-value", val1M), undefined);
|
||||
Assert.strictEqual(await database.put("A-2M-value", val2M), undefined);
|
||||
Assert.strictEqual(await database.put("A-32B-value", base), undefined);
|
||||
|
||||
Assert.strictEqual(await database.get("A-1M-value"), val1M);
|
||||
Assert.strictEqual(await database.get("A-2M-value"), val2M);
|
||||
Assert.strictEqual(await database.get("A-32B-value"), base);
|
||||
});
|
||||
|
||||
add_task(async function largeNumbers() {
|
||||
const databaseDir = await makeDatabaseDir("largeNumbers");
|
||||
const database = await KeyValueService.getOrCreate(databaseDir, "db");
|
||||
|
|
@ -212,6 +230,51 @@ add_task(async function writeManyPutOnly() {
|
|||
await test_helper(mapPairs);
|
||||
});
|
||||
|
||||
add_task(async function writeManyLargePairsWithResizing() {
|
||||
const databaseDir = await makeDatabaseDir("writeManyWithResizing");
|
||||
const database = await KeyValueService.getOrCreate(databaseDir, "db");
|
||||
|
||||
// The default store size is 1MB, putting key/value pairs bigger than that
|
||||
// would trigger auto resizing.
|
||||
const base = "A humongous string in 32 bytes!!";
|
||||
const val1M = base.repeat(32768);
|
||||
const val2M = val1M.repeat(2);
|
||||
|
||||
// writeMany with an object
|
||||
const pairs = {
|
||||
"A-1M-value": val1M,
|
||||
"A-32B-value": base,
|
||||
"A-2M-value": val2M,
|
||||
};
|
||||
|
||||
Assert.strictEqual(await database.writeMany(pairs), undefined);
|
||||
|
||||
Assert.strictEqual(await database.get("A-1M-value"), val1M);
|
||||
Assert.strictEqual(await database.get("A-2M-value"), val2M);
|
||||
Assert.strictEqual(await database.get("A-32B-value"), base);
|
||||
});
|
||||
|
||||
add_task(async function writeManySmallPairsWithResizing() {
|
||||
const databaseDir = await makeDatabaseDir("writeManyWithResizing");
|
||||
const database = await KeyValueService.getOrCreate(databaseDir, "db");
|
||||
|
||||
// The default store size is 1MB, putting key/value pairs bigger than that
|
||||
// would trigger auto resizing.
|
||||
const base = "A humongous string in 32 bytes!!";
|
||||
const val1K = base.repeat(32);
|
||||
// writeMany with a key/value generator
|
||||
function* pairMaker() {
|
||||
for (let i = 0; i < 1024; i++) {
|
||||
yield [`key-${i}`, val1K];
|
||||
}
|
||||
}
|
||||
|
||||
Assert.strictEqual(await database.writeMany(pairMaker()), undefined);
|
||||
for (let i = 0; i < 1024; i++) {
|
||||
Assert.ok(await database.has(`key-${i}`));
|
||||
}
|
||||
});
|
||||
|
||||
add_task(async function writeManyDeleteOnly() {
|
||||
const databaseDir = await makeDatabaseDir("writeManyDeletesOnly");
|
||||
const database = await KeyValueService.getOrCreate(databaseDir, "db");
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче