diff --git a/Cargo.lock b/Cargo.lock index 80de257de716..1b485a686538 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -461,12 +461,12 @@ dependencies = [ "base64 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", "byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam-utils 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)", - "lmdb-rkv 0.11.2 (registry+https://github.com/rust-lang/crates.io-index)", + "lmdb-rkv 0.11.4 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "moz_task 0.1.0", "nserror 0.1.0", "nsstring 0.1.0", - "rkv 0.9.4 (registry+https://github.com/rust-lang/crates.io-index)", + "rkv 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)", "rust_cascade 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "sha2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "storage_variant 0.1.0", @@ -1542,12 +1542,12 @@ dependencies = [ "failure 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.51 (registry+https://github.com/rust-lang/crates.io-index)", - "lmdb-rkv 0.11.2 (registry+https://github.com/rust-lang/crates.io-index)", + "lmdb-rkv 0.11.4 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "moz_task 0.1.0", "nserror 0.1.0", "nsstring 0.1.0", - "rkv 0.9.4 (registry+https://github.com/rust-lang/crates.io-index)", + "rkv 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)", "storage_variant 0.1.0", "thin-vec 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "xpcom 0.1.0", @@ -1646,10 +1646,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "lmdb-rkv" -version = "0.11.2" +version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.51 (registry+https://github.com/rust-lang/crates.io-index)", "lmdb-rkv-sys 0.8.3 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2529,14 +2530,14 @@ dependencies = [ [[package]] name = "rkv" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "arrayref 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "bincode 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "failure 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "lmdb-rkv 0.11.2 (registry+https://github.com/rust-lang/crates.io-index)", + "lmdb-rkv 0.11.4 (registry+https://github.com/rust-lang/crates.io-index)", "ordered-float 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.88 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.88 (git+https://github.com/servo/serde?branch=deserialize_from_enums10)", @@ -3679,12 +3680,12 @@ dependencies = [ "failure 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.51 (registry+https://github.com/rust-lang/crates.io-index)", - "lmdb-rkv 0.11.2 (registry+https://github.com/rust-lang/crates.io-index)", + "lmdb-rkv 0.11.4 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "moz_task 0.1.0", "nserror 0.1.0", "nsstring 0.1.0", - "rkv 0.9.4 (registry+https://github.com/rust-lang/crates.io-index)", + "rkv 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.26 (registry+https://github.com/rust-lang/crates.io-index)", "xpcom 0.1.0", ] @@ -3856,7 +3857,7 @@ dependencies = [ "checksum libudev 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ea626d3bdf40a1c5aee3bcd4f40826970cae8d80a8fec934c82a63840094dcfe" "checksum libz-sys 1.0.16 (registry+https://github.com/rust-lang/crates.io-index)" = "3fdd64ef8ee652185674455c1d450b83cbc8ad895625d543b5324d923f82e4d8" "checksum linked-hash-map 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "70fb39025bc7cdd76305867c4eccf2f2dcf6e9a57f5b21a93e1c2d86cd03ec9e" -"checksum lmdb-rkv 0.11.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1452294309db7977dc75e1e8135a8c654d9e52e04ff0c0bd06c880897a91defd" +"checksum lmdb-rkv 0.11.4 (registry+https://github.com/rust-lang/crates.io-index)" = "e25b4069789bf7ac069d6fd58229f18aec20c6f7cc9173cb731d11c10dbb6b6e" "checksum lmdb-rkv-sys 0.8.3 (registry+https://github.com/rust-lang/crates.io-index)" = "1470e0168f1832e35afd6d0931ae60db625685332837b97aa156773ec9c5e393" "checksum lock_api 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ed946d4529956a20f2d63ebe1b69996d5a2137c91913fe3ebbeff957f5bca7ff" "checksum log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b" @@ -3940,7 +3941,7 @@ dependencies = [ "checksum regex-syntax 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ad890a5eef7953f55427c50575c680c42841653abd2b028b68cd223d157f62db" "checksum regex-syntax 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8f1ac0f60d675cc6cf13a20ec076568254472551051ad5dd050364d70671bf6b" "checksum remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3488ba1b9a2084d38645c4c08276a1752dcbf2c7130d74f1569681ad5d2799c5" -"checksum rkv 0.9.4 (registry+https://github.com/rust-lang/crates.io-index)" = "238764bd8750927754d91e4a27155ac672ba88934a2bf698c992d55e5ae25e5b" +"checksum rkv 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)" = "2c1b8d667bf149bfac7c47bb728dfb7246f35fdf61c2f16f9f588194f087d23c" "checksum ron 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "da06feaa07f69125ab9ddc769b11de29090122170b402547f64b86fe16ebc399" "checksum runloop 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5d79b4b604167921892e84afbbaad9d5ad74e091bf6c511d9dbfb0593f09fabd" "checksum rust-ini 0.10.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8a654c5bda722c699be6b0fe4c0d90de218928da5b724c3e467fc48865c37263" diff --git a/third_party/rust/lmdb-rkv/.cargo-checksum.json b/third_party/rust/lmdb-rkv/.cargo-checksum.json index 3d893876d6c4..c5bca388ac6e 100644 --- a/third_party/rust/lmdb-rkv/.cargo-checksum.json +++ b/third_party/rust/lmdb-rkv/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"ba319c56424eed8ac65e6add74a59e9610c2a1e30701801b0ede6313020cd596","LICENSE":"db6d163be642e3b568c5fb2104013da632316ecd4e75935df1613af8b0b37933","README.md":"97b61d73ff27afb03bde9ae960f12651093558214851303c8ae82f567abfe992","azure-pipelines-template.yml":"a75d2421df0feefcb2ea17072c12795e58f9ac6322bacfbea90890fbb5e09cb7","azure-pipelines.yml":"c5a206822c22921bd07ea0a469db734875361457400427991319808cfd13ffef","src/cursor.rs":"f51184cbf015d1aef1a45f0cc1a950524114e1d4aadc50bde7bdb712030a839a","src/database.rs":"003a214f53acd632bc70f2f02d01dcb0bc5bf7e777e1781ef1ff19246f0157d0","src/environment.rs":"774241ef807892b0ebede3392113560563dc8b197c87d97756c47811716a8b4c","src/error.rs":"0ea99c8bc1619f3789eff7734341efa7f48fcd8732dc9f3141804e0a802f5d71","src/flags.rs":"40fd3d4d72c8db8f9ecb893420300a3585e2ca4c49073065ec9ebf24fe23c064","src/lib.rs":"e8c93f6501c58f2d325998730e7991c156a8f53459130181005569ede9d2d871","src/transaction.rs":"09ac6501e3dd1a714c635acb46625c4a4d96d9296cd30bcda31d50224e27fd05"},"package":"1452294309db7977dc75e1e8135a8c654d9e52e04ff0c0bd06c880897a91defd"} \ No newline at end of file +{"files":{"Cargo.toml":"aafdb8200f4d1db63ef096c47e1a43a156f4e1690133508e43e9e21b90b15f93","LICENSE":"db6d163be642e3b568c5fb2104013da632316ecd4e75935df1613af8b0b37933","README.md":"97b61d73ff27afb03bde9ae960f12651093558214851303c8ae82f567abfe992","src/cursor.rs":"f51184cbf015d1aef1a45f0cc1a950524114e1d4aadc50bde7bdb712030a839a","src/database.rs":"f0a750497ff32cf1fd4ea21ca0da9234630a6338bb5b4887f72f0abfc6316135","src/environment.rs":"b07df0fe38186e239980904f03db25ca0b1c14f2593dc1bfdefc418b0f26c192","src/error.rs":"0ea99c8bc1619f3789eff7734341efa7f48fcd8732dc9f3141804e0a802f5d71","src/flags.rs":"40fd3d4d72c8db8f9ecb893420300a3585e2ca4c49073065ec9ebf24fe23c064","src/lib.rs":"858e28ebbd9613e0b49b0e8a0ec48a837bc9b2401c0517b5ceabe66b2f0f2a85","src/transaction.rs":"23394768fa7b8603e1f9c3312ba10b8be9fbd74fc26b0300ab93424c3277a400"},"package":"e25b4069789bf7ac069d6fd58229f18aec20c6f7cc9173cb731d11c10dbb6b6e"} \ No newline at end of file diff --git a/third_party/rust/lmdb-rkv/Cargo.toml b/third_party/rust/lmdb-rkv/Cargo.toml index 7f958f600a9e..bb3a718bc1d3 100644 --- a/third_party/rust/lmdb-rkv/Cargo.toml +++ b/third_party/rust/lmdb-rkv/Cargo.toml @@ -3,7 +3,7 @@ # When uploading crates to the registry Cargo will automatically # "normalize" Cargo.toml files for maximal compatibility # with all versions of Cargo and also rewrite `path` dependencies -# to registry (e.g. crates.io) dependencies +# to registry (e.g., crates.io) dependencies # # If you believe there's an error in this file please file an # issue against the rust-lang/cargo repository. If you're @@ -12,8 +12,9 @@ [package] name = "lmdb-rkv" -version = "0.11.2" +version = "0.11.4" authors = ["Dan Burkert "] +exclude = ["/.appveyor.yml", "/.travis.yml", "/azure-pipelines-template.yml", "/azure-pipelines.yml"] description = "Idiomatic and safe LMDB wrapper." documentation = "https://docs.rs/lmdb-rkv" readme = "README.md" @@ -27,14 +28,14 @@ name = "lmdb" [dependencies.bitflags] version = "1" +[dependencies.byteorder] +version = "1.0" + [dependencies.libc] version = "0.2" [dependencies.lmdb-rkv-sys] -version = "0.8.2" -[dev-dependencies.byteorder] -version = "1.0" - +version = "0.8.3" [dev-dependencies.rand] version = "0.4" diff --git a/third_party/rust/lmdb-rkv/azure-pipelines-template.yml b/third_party/rust/lmdb-rkv/azure-pipelines-template.yml deleted file mode 100644 index a125d8dde122..000000000000 --- a/third_party/rust/lmdb-rkv/azure-pipelines-template.yml +++ /dev/null @@ -1,49 +0,0 @@ -jobs: -- job: ${{ parameters.name }} - pool: - vmImage: ${{ parameters.vmImage }} - strategy: - matrix: - stable: - rustup_toolchain: stable - beta: - rustup_toolchain: beta - nightly: - rustup_toolchain: nightly - steps: - # Linux and macOS. - - ${{ if ne(parameters.name, 'Windows') }}: - - script: | - curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain $RUSTUP_TOOLCHAIN - echo "##vso[task.setvariable variable=PATH;]$PATH:$HOME/.cargo/bin" - displayName: Install rust - # Windows. - - ${{ if eq(parameters.name, 'Windows') }}: - - script: | - curl -sSf -o rustup-init.exe https://win.rustup.rs - rustup-init.exe -y --default-toolchain %RUSTUP_TOOLCHAIN% - set PATH=%PATH%;%USERPROFILE%\.cargo\bin - echo "##vso[task.setvariable variable=PATH;]%PATH%;%USERPROFILE%\.cargo\bin" - displayName: Install rust (windows) - # All platforms. - - script: | - rustc -Vv - cargo -V - displayName: Query rust and cargo versions - - script: cargo build - displayName: Build - # Linux and macOS w/nightly toolchain. - # Ideally we'd only run the script for the nightly toolchain, but I can't - # figure out how to determine that within the Azure Pipelines conditional. - - ${{ if ne(parameters.name, 'Windows') }}: - - script: | - if [ "$RUSTUP_TOOLCHAIN" = 'nightly' ] - then cargo test - fi - displayName: Test - # Windows w/nightly toolchain. - # Ideally we'd only run the script for the nightly toolchain, but I can't - # figure out how to determine that within the Azure Pipelines conditional. - - ${{ if eq(parameters.name, 'Windows') }}: - - script: if "%RUSTUP_TOOLCHAIN%" == "nightly" cargo test - displayName: Test diff --git a/third_party/rust/lmdb-rkv/azure-pipelines.yml b/third_party/rust/lmdb-rkv/azure-pipelines.yml deleted file mode 100644 index 5666af58f91e..000000000000 --- a/third_party/rust/lmdb-rkv/azure-pipelines.yml +++ /dev/null @@ -1,15 +0,0 @@ -jobs: -- template: azure-pipelines-template.yml - parameters: - name: macOS - vmImage: macOS-10.13 - -- template: azure-pipelines-template.yml - parameters: - name: Linux - vmImage: ubuntu-16.04 - -- template: azure-pipelines-template.yml - parameters: - name: Windows - vmImage: vs2017-win2016 diff --git a/third_party/rust/lmdb-rkv/src/database.rs b/third_party/rust/lmdb-rkv/src/database.rs index 61656dcdfb24..e7973ba45bf1 100644 --- a/third_party/rust/lmdb-rkv/src/database.rs +++ b/third_party/rust/lmdb-rkv/src/database.rs @@ -31,6 +31,12 @@ impl Database { Ok(Database { dbi: dbi }) } + pub(crate) fn freelist_db() -> Database { + Database { + dbi: 0, + } + } + /// Returns the underlying LMDB database handle. /// /// The caller **must** ensure that the handle is not used after the lifetime of the diff --git a/third_party/rust/lmdb-rkv/src/environment.rs b/third_party/rust/lmdb-rkv/src/environment.rs index 322014e4f702..e31d9f1dec91 100644 --- a/third_party/rust/lmdb-rkv/src/environment.rs +++ b/third_party/rust/lmdb-rkv/src/environment.rs @@ -10,7 +10,10 @@ use std::sync::Mutex; use ffi; -use error::{Result, lmdb_result}; +use byteorder::{ByteOrder, NativeEndian}; + +use cursor::Cursor; +use error::{Error, Result, lmdb_result}; use database::Database; use transaction::{RoTransaction, RwTransaction, Transaction}; use flags::{DatabaseFlags, EnvironmentFlags}; @@ -158,18 +161,109 @@ impl Environment { /// Retrieves statistics about this environment. pub fn stat(&self) -> Result { unsafe { - let mut stat = Stat(mem::zeroed()); - lmdb_try!(ffi::mdb_env_stat(self.env(), &mut stat.0)); + let mut stat = Stat::new(); + lmdb_try!(ffi::mdb_env_stat(self.env(), stat.mdb_stat())); Ok(stat) } } + + /// Retrieves info about this environment. + pub fn info(&self) -> Result { + unsafe { + let mut info = Info(mem::zeroed()); + lmdb_try!(ffi::mdb_env_info(self.env(), &mut info.0)); + Ok(info) + } + } + + /// Retrieves the total number of pages on the freelist. + /// + /// Along with `Environment::info()`, this can be used to calculate the exact number + /// of used pages as well as free pages in this environment. + /// + /// ```ignore + /// let env = Environment::new().open("/tmp/test").unwrap(); + /// let info = env.info().unwrap(); + /// let stat = env.stat().unwrap(); + /// let freelist = env.freelist().unwrap(); + /// let last_pgno = info.last_pgno() + 1; // pgno is 0 based. + /// let total_pgs = info.map_size() / stat.page_size() as usize; + /// let pgs_in_use = last_pgno - freelist; + /// let pgs_free = total_pgs - pgs_in_use; + /// ``` + /// + /// Note: + /// + /// * LMDB stores all the freelists in the designated database 0 in each environment, + /// and the freelist count is stored at the beginning of the value as `libc::size_t` + /// in the native byte order. + /// + /// * It will create a read transaction to traverse the freelist database. + pub fn freelist(&self) -> Result { + let mut freelist: size_t = 0; + let db = Database::freelist_db(); + let txn = self.begin_ro_txn()?; + let mut cursor = txn.open_ro_cursor(db)?; + + for result in cursor.iter() { + let (_key, value) = result?; + if value.len() < mem::size_of::() { + return Err(Error::Corrupted); + } + + let s = &value[..mem::size_of::()]; + if cfg!(target_pointer_width = "64") { + freelist += NativeEndian::read_u64(s) as size_t; + } else { + freelist += NativeEndian::read_u32(s) as size_t; + } + } + + Ok(freelist) + } + + /// Sets the size of the memory map to use for the environment. + /// + /// This could be used to resize the map when the environment is already open. + /// + /// Note: + /// + /// * No active transactions allowed when performing resizing in this process. + /// + /// * The size should be a multiple of the OS page size. Any attempt to set + /// a size smaller than the space already consumed by the environment will + /// be silently changed to the current size of the used space. + /// + /// * In the multi-process case, once a process resizes the map, other + /// processes need to either re-open the environment, or call set_map_size + /// with size 0 to update the environment. Otherwise, new transaction creation + /// will fail with `Error::MapResized`. + pub fn set_map_size(&self, size: size_t) -> Result<()> { + unsafe { + lmdb_result(ffi::mdb_env_set_mapsize(self.env(), size)) + } + } } /// Environment statistics. /// -/// Contains information about the size and layout of an LMDB environment. +/// Contains information about the size and layout of an LMDB environment or database. pub struct Stat(ffi::MDB_stat); +impl Stat { + /// Create a new Stat with zero'd inner struct `ffi::MDB_stat`. + pub(crate) fn new() -> Stat { + unsafe { + Stat(mem::zeroed()) + } + } + + /// Returns a mut pointer to `ffi::MDB_stat`. + pub(crate) fn mdb_stat(&mut self) -> *mut ffi::MDB_stat { + &mut self.0 + } +} + impl Stat { /// Size of a database page. This is the same for all databases in the environment. #[inline] @@ -208,6 +302,43 @@ impl Stat { } } +/// Environment information. +/// +/// Contains environment information about the map size, readers, last txn id etc. +pub struct Info(ffi::MDB_envinfo); + +impl Info { + /// Size of memory map. + #[inline] + pub fn map_size(&self) -> usize { + self.0.me_mapsize + } + + /// Last used page number + #[inline] + pub fn last_pgno(&self) -> usize { + self.0.me_last_pgno + } + + /// Last transaction ID + #[inline] + pub fn last_txnid(&self) -> usize { + self.0.me_last_txnid + } + + /// Max reader slots in the environment + #[inline] + pub fn max_readers(&self) -> u32 { + self.0.me_maxreaders + } + + /// Max reader slots used in the environment + #[inline] + pub fn num_readers(&self) -> u32 { + self.0.me_numreaders + } +} + unsafe impl Send for Environment {} unsafe impl Sync for Environment {} @@ -462,4 +593,75 @@ mod test { assert_eq!(stat.overflow_pages(), 0); assert_eq!(stat.entries(), 64); } + + #[test] + fn test_info() { + let map_size = 1024 * 1024; + let dir = TempDir::new("test").unwrap(); + let env = Environment::new() + .set_map_size(map_size) + .open(dir.path()) + .unwrap(); + + let info = env.info().unwrap(); + assert_eq!(info.map_size(), map_size); + assert_eq!(info.last_pgno(), 1); + assert_eq!(info.last_txnid(), 0); + // The default max readers is 126. + assert_eq!(info.max_readers(), 126); + assert_eq!(info.num_readers(), 0); + } + + #[test] + fn test_freelist() { + let dir = TempDir::new("test").unwrap(); + let env = Environment::new().open(dir.path()).unwrap(); + + let db = env.open_db(None).unwrap(); + let mut freelist = env.freelist().unwrap(); + assert_eq!(freelist, 0); + + // Write a few small values. + for i in 0..64 { + let mut value = [0u8; 8]; + LittleEndian::write_u64(&mut value, i); + let mut tx = env.begin_rw_txn().expect("begin_rw_txn"); + tx.put(db, &value, &value, WriteFlags::default()).expect("tx.put"); + tx.commit().expect("tx.commit") + } + let mut tx = env.begin_rw_txn().expect("begin_rw_txn"); + tx.clear_db(db).expect("clear"); + tx.commit().expect("tx.commit"); + + // Freelist should not be empty after clear_db. + freelist = env.freelist().unwrap(); + assert!(freelist > 0); + } + + #[test] + fn test_set_map_size() { + let dir = TempDir::new("test").unwrap(); + let env = Environment::new().open(dir.path()).unwrap(); + + let mut info = env.info().unwrap(); + let default_size = info.map_size(); + + // Resizing to 0 merely reloads the map size + env.set_map_size(0).unwrap(); + info = env.info().unwrap(); + assert_eq!(info.map_size(), default_size); + + env.set_map_size(2 * default_size).unwrap(); + info = env.info().unwrap(); + assert_eq!(info.map_size(), 2 * default_size); + + env.set_map_size(4 * default_size).unwrap(); + info = env.info().unwrap(); + assert_eq!(info.map_size(), 4 * default_size); + + // Decreasing is also fine if the space hasn't been consumed. + env.set_map_size(2 * default_size).unwrap(); + info = env.info().unwrap(); + assert_eq!(info.map_size(), 2 * default_size); + } } diff --git a/third_party/rust/lmdb-rkv/src/lib.rs b/third_party/rust/lmdb-rkv/src/lib.rs index 38742fb9aad5..161cdb31f193 100644 --- a/third_party/rust/lmdb-rkv/src/lib.rs +++ b/third_party/rust/lmdb-rkv/src/lib.rs @@ -3,10 +3,11 @@ #![cfg_attr(test, feature(test))] #![deny(missing_docs)] -#![doc(html_root_url = "https://docs.rs/lmdb-rkv/0.11.2")] +#![doc(html_root_url = "https://docs.rs/lmdb-rkv/0.11.4")] extern crate libc; extern crate lmdb_rkv_sys as ffi; +extern crate byteorder; #[cfg(test)] extern crate rand; #[cfg(test)] extern crate tempdir; @@ -21,7 +22,12 @@ pub use cursor::{ IterDup, }; pub use database::Database; -pub use environment::{Environment, Stat, EnvironmentBuilder}; +pub use environment::{ + Environment, + EnvironmentBuilder, + Info, + Stat, +}; pub use error::{Error, Result}; pub use flags::*; pub use transaction::{ @@ -62,9 +68,7 @@ mod transaction; #[cfg(test)] mod test_utils { - extern crate byteorder; - - use self::byteorder::{ByteOrder, LittleEndian}; + use byteorder::{ByteOrder, LittleEndian}; use tempdir::TempDir; use super::*; diff --git a/third_party/rust/lmdb-rkv/src/transaction.rs b/third_party/rust/lmdb-rkv/src/transaction.rs index 16804d36d4dd..78175b1ae274 100644 --- a/third_party/rust/lmdb-rkv/src/transaction.rs +++ b/third_party/rust/lmdb-rkv/src/transaction.rs @@ -5,7 +5,7 @@ use std::marker::PhantomData ; use ffi; use cursor::{RoCursor, RwCursor}; -use environment::Environment; +use environment::{Environment, Stat}; use database::Database; use error::{Error, Result, lmdb_result}; use flags::{DatabaseFlags, EnvironmentFlags, WriteFlags}; @@ -102,6 +102,15 @@ pub trait Transaction : Sized { } Ok(DatabaseFlags::from_bits_truncate(flags)) } + + /// Retrieves database statistics. + fn stat(&self, db: Database) -> Result { + unsafe { + let mut stat = Stat::new(); + lmdb_try!(ffi::mdb_stat(self.txn(), db.dbi(), stat.mdb_stat())); + Ok(stat) + } + } } /// An LMDB read-only transaction. @@ -654,6 +663,96 @@ mod test { } } + #[test] + fn test_stat() { + let dir = TempDir::new("test").unwrap(); + let env = Environment::new().open(dir.path()).unwrap(); + let db = env.create_db(None, DatabaseFlags::empty()).unwrap(); + + let mut txn = env.begin_rw_txn().unwrap(); + txn.put(db, b"key1", b"val1", WriteFlags::empty()).unwrap(); + txn.put(db, b"key2", b"val2", WriteFlags::empty()).unwrap(); + txn.put(db, b"key3", b"val3", WriteFlags::empty()).unwrap(); + txn.commit().unwrap(); + + { + let txn = env.begin_ro_txn().unwrap(); + let stat = txn.stat(db).unwrap(); + assert_eq!(stat.entries(), 3); + } + + let mut txn = env.begin_rw_txn().unwrap(); + txn.del(db, b"key1", None).unwrap(); + txn.del(db, b"key2", None).unwrap(); + txn.commit().unwrap(); + + { + let txn = env.begin_ro_txn().unwrap(); + let stat = txn.stat(db).unwrap(); + assert_eq!(stat.entries(), 1); + } + + let mut txn = env.begin_rw_txn().unwrap(); + txn.put(db, b"key4", b"val4", WriteFlags::empty()).unwrap(); + txn.put(db, b"key5", b"val5", WriteFlags::empty()).unwrap(); + txn.put(db, b"key6", b"val6", WriteFlags::empty()).unwrap(); + txn.commit().unwrap(); + + { + let txn = env.begin_ro_txn().unwrap(); + let stat = txn.stat(db).unwrap(); + assert_eq!(stat.entries(), 4); + } + } + + #[test] + fn test_stat_dupsort() { + let dir = TempDir::new("test").unwrap(); + let env = Environment::new().open(dir.path()).unwrap(); + let db = env.create_db(None, DatabaseFlags::DUP_SORT).unwrap(); + + let mut txn = env.begin_rw_txn().unwrap(); + txn.put(db, b"key1", b"val1", WriteFlags::empty()).unwrap(); + txn.put(db, b"key1", b"val2", WriteFlags::empty()).unwrap(); + txn.put(db, b"key1", b"val3", WriteFlags::empty()).unwrap(); + txn.put(db, b"key2", b"val1", WriteFlags::empty()).unwrap(); + txn.put(db, b"key2", b"val2", WriteFlags::empty()).unwrap(); + txn.put(db, b"key2", b"val3", WriteFlags::empty()).unwrap(); + txn.put(db, b"key3", b"val1", WriteFlags::empty()).unwrap(); + txn.put(db, b"key3", b"val2", WriteFlags::empty()).unwrap(); + txn.put(db, b"key3", b"val3", WriteFlags::empty()).unwrap(); + txn.commit().unwrap(); + + { + let txn = env.begin_ro_txn().unwrap(); + let stat = txn.stat(db).unwrap(); + assert_eq!(stat.entries(), 9); + } + + let mut txn = env.begin_rw_txn().unwrap(); + txn.del(db, b"key1", Some(b"val2")).unwrap(); + txn.del(db, b"key2", None).unwrap(); + txn.commit().unwrap(); + + { + let txn = env.begin_ro_txn().unwrap(); + let stat = txn.stat(db).unwrap(); + assert_eq!(stat.entries(), 5); + } + + let mut txn = env.begin_rw_txn().unwrap(); + txn.put(db, b"key4", b"val1", WriteFlags::empty()).unwrap(); + txn.put(db, b"key4", b"val2", WriteFlags::empty()).unwrap(); + txn.put(db, b"key4", b"val3", WriteFlags::empty()).unwrap(); + txn.commit().unwrap(); + + { + let txn = env.begin_ro_txn().unwrap(); + let stat = txn.stat(db).unwrap(); + assert_eq!(stat.entries(), 8); + } + } + #[bench] fn bench_get_rand(b: &mut Bencher) { let n = 100u32; diff --git a/third_party/rust/rkv/.cargo-checksum.json b/third_party/rust/rkv/.cargo-checksum.json index d4308736638a..be08548b26d2 100644 --- a/third_party/rust/rkv/.cargo-checksum.json +++ b/third_party/rust/rkv/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"bb25bb1f8a98037fac1f33ffef7244b6363396e6220af4b856b9fe0616c71b81","LICENSE":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30","README.md":"9dc24375b49fef42f35dec42e316e21827d7337622f9e7cf36243cd28808797a","examples/README.md":"143767fc145bf167ce269a65138cb3f7086cb715b8bc4f73626da82966e646f4","examples/iterator.rs":"ddc3997e394a30ad82d78d2675a48c4617353f88b89bb9a3df5a3804d59b8ef9","examples/simple-store.rs":"cae63e39f2f98ee6ac2f387dcb02d6b929828a74f32f7d18d69c7fc9c3cce765","run-all-examples.sh":"7f9d11d01017f77e1c9d26e3e82dfca8c6930deaec85e864458e33a7fa267de0","src/env.rs":"f886c42b8ea0633ed001d24fee2edcc246b7b0dd2b56dcfbdebf4aef4a36f7a0","src/error.rs":"46632b8fcb1070a1860247e09a59d39772079ebfba5d3d1bbee03d08e1252275","src/lib.rs":"67a1970626fcecf35c0a9ccb0305afbfb12b8a85e3d5060bff4c6617a3d1de78","src/manager.rs":"f06b14ee64f2e58d890a3b37677790b707a02d328242c1af0ce3c74e9028edd8","src/readwrite.rs":"fde695333e4845f4f53d63da6281f585919e2a3ac5cfe00d173cc139bc822763","src/store.rs":"409d13b1ea0d1254dae947ecbce50e741fb71c3ca118a78803b734336dce6a8f","src/store/integer.rs":"f386474c971f671c9b316a16ebff5b586be6837c886f443753ae13277a7e0070","src/store/integermulti.rs":"1a0912f97619297da31cc8c146e38941b88539d2857df81191a49c8dbd18625d","src/store/multi.rs":"2dec01c2202a2c9069cced4e1e42906b01d0b85df25d17e0ea810c05fa8395d0","src/store/single.rs":"c55c3600714f5ed9e820b16c2335ae00a0071174e0a32b9df89a34182a4b908c","src/value.rs":"ad74ba4c9ab0a77f1c4f8ee2650ceeb148e4036b017d804affc35085e97944fb","tests/integer-store.rs":"f7e06c71b0dead2323c7c61fc8bcbffbdd3a4796eebf6138db9cce3dbba716a3","tests/manager.rs":"97ec61145dc227f4f5fbcb6449c096bbe5b9a09db4e61ff4491c0443fe9adf26","tests/multi-integer-store.rs":"83295b0135c502321304aa06b05d5a9eeab41b1438ed7ddf2cb1a3613dfef4d9"},"package":"238764bd8750927754d91e4a27155ac672ba88934a2bf698c992d55e5ae25e5b"} \ No newline at end of file +{"files":{"CODE_OF_CONDUCT.md":"902d5357af363426631d907e641e220b3ec89039164743f8442b3f120479b7cf","Cargo.toml":"27ac84d5a81e9b054e79f7615086b5625071aa88803202f7c8de4a1e53e32e83","LICENSE":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30","README.md":"9dc24375b49fef42f35dec42e316e21827d7337622f9e7cf36243cd28808797a","examples/README.md":"143767fc145bf167ce269a65138cb3f7086cb715b8bc4f73626da82966e646f4","examples/iterator.rs":"ddc3997e394a30ad82d78d2675a48c4617353f88b89bb9a3df5a3804d59b8ef9","examples/simple-store.rs":"cae63e39f2f98ee6ac2f387dcb02d6b929828a74f32f7d18d69c7fc9c3cce765","run-all-examples.sh":"7f9d11d01017f77e1c9d26e3e82dfca8c6930deaec85e864458e33a7fa267de0","src/env.rs":"67b4a98d9631bb61ca6181afc557b7fda0876de2959cfb78bd7049da3cd0a623","src/error.rs":"46632b8fcb1070a1860247e09a59d39772079ebfba5d3d1bbee03d08e1252275","src/lib.rs":"6b21c99c067f550124986b3c15d018b1a090351c339758c0a76ba4f58f818f5d","src/manager.rs":"6142e31ec0d31fa18ed0fe29b123de020ad81ecaec21263c374742548f842ee8","src/readwrite.rs":"fde695333e4845f4f53d63da6281f585919e2a3ac5cfe00d173cc139bc822763","src/store.rs":"409d13b1ea0d1254dae947ecbce50e741fb71c3ca118a78803b734336dce6a8f","src/store/integer.rs":"f386474c971f671c9b316a16ebff5b586be6837c886f443753ae13277a7e0070","src/store/integermulti.rs":"1a0912f97619297da31cc8c146e38941b88539d2857df81191a49c8dbd18625d","src/store/multi.rs":"2dec01c2202a2c9069cced4e1e42906b01d0b85df25d17e0ea810c05fa8395d0","src/store/single.rs":"c55c3600714f5ed9e820b16c2335ae00a0071174e0a32b9df89a34182a4b908c","src/value.rs":"7fae77a8291b951591e557ec694bfdadc9eb78557dad36a970cfcdcfb83fd238","tests/integer-store.rs":"f7e06c71b0dead2323c7c61fc8bcbffbdd3a4796eebf6138db9cce3dbba716a3","tests/manager.rs":"97ec61145dc227f4f5fbcb6449c096bbe5b9a09db4e61ff4491c0443fe9adf26","tests/multi-integer-store.rs":"83295b0135c502321304aa06b05d5a9eeab41b1438ed7ddf2cb1a3613dfef4d9"},"package":"2c1b8d667bf149bfac7c47bb728dfb7246f35fdf61c2f16f9f588194f087d23c"} \ No newline at end of file diff --git a/third_party/rust/rkv/CODE_OF_CONDUCT.md b/third_party/rust/rkv/CODE_OF_CONDUCT.md new file mode 100644 index 000000000000..498baa3fb0f0 --- /dev/null +++ b/third_party/rust/rkv/CODE_OF_CONDUCT.md @@ -0,0 +1,15 @@ +# Community Participation Guidelines + +This repository is governed by Mozilla's code of conduct and etiquette guidelines. +For more details, please read the +[Mozilla Community Participation Guidelines](https://www.mozilla.org/about/governance/policies/participation/). + +## How to Report +For more information on how to report violations of the Community Participation Guidelines, please read our '[How to Report](https://www.mozilla.org/about/governance/policies/participation/reporting/)' page. + + diff --git a/third_party/rust/rkv/Cargo.toml b/third_party/rust/rkv/Cargo.toml index 8ae7f4ba6a04..94afee2ecd11 100644 --- a/third_party/rust/rkv/Cargo.toml +++ b/third_party/rust/rkv/Cargo.toml @@ -3,7 +3,7 @@ # When uploading crates to the registry Cargo will automatically # "normalize" Cargo.toml files for maximal compatibility # with all versions of Cargo and also rewrite `path` dependencies -# to registry (e.g. crates.io) dependencies +# to registry (e.g., crates.io) dependencies # # If you believe there's an error in this file please file an # issue against the rust-lang/cargo repository. If you're @@ -13,7 +13,7 @@ [package] edition = "2018" name = "rkv" -version = "0.9.4" +version = "0.9.5" authors = ["Richard Newman ", "Nan Jiang ", "Myk Melez "] description = "a simple, humane, typed Rust interface to LMDB" homepage = "https://github.com/mozilla/rkv" @@ -37,7 +37,7 @@ default_features = false version = "1.0.2" [dependencies.lmdb-rkv] -version = "0.11.2" +version = "0.11.4" [dependencies.ordered-float] version = "1.0" diff --git a/third_party/rust/rkv/src/env.rs b/third_party/rust/rkv/src/env.rs index defa6d4d1ab6..02936a96755a 100644 --- a/third_party/rust/rkv/src/env.rs +++ b/third_party/rust/rkv/src/env.rs @@ -22,6 +22,8 @@ use lmdb::{ DatabaseFlags, Environment, EnvironmentBuilder, + Error, + Info, Stat, }; @@ -197,13 +199,75 @@ impl Rkv { self.env.sync(force).map_err(Into::into) } - /// Retrieves statistics about this environment. + /// Retrieve statistics about this environment. + /// + /// It includes: + /// * Page size in bytes + /// * B-tree depth + /// * Number of internal (non-leaf) pages + /// * Number of leaf pages + /// * Number of overflow pages + /// * Number of data entries pub fn stat(&self) -> Result { self.env.stat().map_err(Into::into) } + + /// Retrieve information about this environment. + /// + /// It includes: + /// * Map size in bytes + /// * The last used page number + /// * The last transaction ID + /// * Max number of readers allowed + /// * Number of readers in use + pub fn info(&self) -> Result { + self.env.info().map_err(Into::into) + } + + /// Retrieve the load ratio (# of used pages / total pages) about this environment. + /// + /// With the formular: (last_page_no - freelist_pages) / total_pages + pub fn load_ratio(&self) -> Result { + let stat = self.stat()?; + let info = self.info()?; + let freelist = self.env.freelist()?; + + let last_pgno = info.last_pgno() + 1; // pgno is 0 based. + let total_pgs = info.map_size() / stat.page_size() as usize; + if freelist > last_pgno { + return Err(StoreError::LmdbError(Error::Corrupted)); + } + let used_pgs = last_pgno - freelist; + Ok(used_pgs as f32 / total_pgs as f32) + } + + /// Sets the size of the memory map to use for the environment. + /// + /// This can be used to resize the map when the environment is already open. + /// You can also use `Rkv::environment_builder()` to set the map size during + /// the `Rkv` initialization. + /// + /// Note: + /// + /// * No active transactions allowed when performing resizing in this process. + /// It's up to the consumer to enforce that. + /// + /// * The size should be a multiple of the OS page size. Any attempt to set + /// a size smaller than the space already consumed by the environment will + /// be silently changed to the current size of the used space. + /// + /// * In the multi-process case, once a process resizes the map, other + /// processes need to either re-open the environment, or call set_map_size + /// with size 0 to update the environment. Otherwise, new transaction creation + /// will fail with `LmdbError::MapResized`. + pub fn set_map_size(&self, size: usize) -> Result<(), StoreError> { + self.env.set_map_size(size).map_err(Into::into) + } } -#[allow(clippy::cyclomatic_complexity)] +// TODO: change this back to `clippy::cognitive_complexity` when Clippy stable +// deprecates `clippy::cyclomatic_complexity`. +#[allow(clippy::complexity)] #[cfg(test)] mod tests { use byteorder::{ @@ -224,6 +288,9 @@ mod tests { use super::*; use crate::*; + // The default size is 1MB. + const DEFAULT_SIZE: usize = 1024 * 1024; + /// We can't open a directory that doesn't exist. #[test] fn test_open_fails() { @@ -308,7 +375,7 @@ mod tests { // https://github.com/LMDB/lmdb/blob/26c7df88e44e31623d0802a564f24781acdefde3/libraries/liblmdb/mdb.c#L729 // sets the default map size to 1,048,576 bytes, i.e. 1MiB. // - 1024 * 1024 + 1 /* 1,048,576 + 1 bytes, i.e. 1MiB + 1 byte */ + DEFAULT_SIZE + 1 /* 1,048,576 + 1 bytes, i.e. 1MiB + 1 byte */ } #[test] @@ -773,7 +840,7 @@ mod tests { #[test] fn test_stat() { - let root = Builder::new().prefix("test_sync").tempdir().expect("tempdir"); + let root = Builder::new().prefix("test_stat").tempdir().expect("tempdir"); fs::create_dir_all(root.path()).expect("dir created"); let k = Rkv::new(root.path()).expect("new succeeded"); for i in 0..5 { @@ -791,6 +858,84 @@ mod tests { assert_eq!(k.stat().expect("stat").leaf_pages(), 1); } + #[test] + fn test_info() { + let root = Builder::new().prefix("test_info").tempdir().expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + let k = Rkv::new(root.path()).expect("new succeeded"); + let sk: SingleStore = k.open_single("sk", StoreOptions::create()).expect("opened"); + let mut writer = k.write().expect("writer"); + + sk.put(&mut writer, "foo", &Value::Str("bar")).expect("wrote"); + writer.commit().expect("commited"); + + let info = k.info().expect("info"); + + // The default size is 1MB. + assert_eq!(info.map_size(), DEFAULT_SIZE); + // Should greater than 0 after the write txn. + assert!(info.last_pgno() > 0); + // A txn to open_single + a txn to write. + assert_eq!(info.last_txnid(), 2); + // The default max readers is 126. + assert_eq!(info.max_readers(), 126); + assert_eq!(info.num_readers(), 0); + + // A new reader should increment the reader counter. + let _reader = k.read().expect("reader"); + let info = k.info().expect("info"); + + assert_eq!(info.num_readers(), 1); + } + + #[test] + fn test_load_ratio() { + let root = Builder::new().prefix("test_load_ratio").tempdir().expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + let k = Rkv::new(root.path()).expect("new succeeded"); + let sk: SingleStore = k.open_single("sk", StoreOptions::create()).expect("opened"); + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo", &Value::Str("bar")).expect("wrote"); + writer.commit().expect("commited"); + + let ratio = k.load_ratio().expect("ratio"); + assert!(ratio > 0.0_f32 && ratio < 1.0_f32); + + // Put data to database should increase the load ratio. + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "bar", &Value::Str(&"more-than-4KB".repeat(1000))).expect("wrote"); + writer.commit().expect("commited"); + let new_ratio = k.load_ratio().expect("ratio"); + assert!(new_ratio > ratio); + + // Clear the database so that all the used pages should go to freelist, hence the ratio + // should decrease. + let mut writer = k.write().expect("writer"); + sk.clear(&mut writer).expect("clear"); + writer.commit().expect("commited"); + let after_clear_ratio = k.load_ratio().expect("ratio"); + assert!(after_clear_ratio < new_ratio); + } + + #[test] + fn test_set_map_size() { + let root = Builder::new().prefix("test_size_map_size").tempdir().expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + let k = Rkv::new(root.path()).expect("new succeeded"); + let sk: SingleStore = k.open_single("sk", StoreOptions::create()).expect("opened"); + + assert_eq!(k.info().expect("info").map_size(), DEFAULT_SIZE); + + k.set_map_size(2 * DEFAULT_SIZE).expect("resized"); + + // Should be able to write. + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo", &Value::Str("bar")).expect("wrote"); + writer.commit().expect("commited"); + + assert_eq!(k.info().expect("info").map_size(), 2 * DEFAULT_SIZE); + } + #[test] fn test_iter() { let root = Builder::new().prefix("test_iter").tempdir().expect("tempdir"); diff --git a/third_party/rust/rkv/src/lib.rs b/third_party/rust/rkv/src/lib.rs index 7a675442af70..c6df2fba9265 100644 --- a/third_party/rust/rkv/src/lib.rs +++ b/third_party/rust/rkv/src/lib.rs @@ -216,6 +216,7 @@ pub mod value; pub use lmdb::{ Cursor, Database, + Info, Iter as LmdbIter, RoCursor, Stat, diff --git a/third_party/rust/rkv/src/manager.rs b/third_party/rust/rkv/src/manager.rs index c51564c8ec3f..7e9c8e1551a2 100644 --- a/third_party/rust/rkv/src/manager.rs +++ b/third_party/rust/rkv/src/manager.rs @@ -37,8 +37,6 @@ use crate::error::StoreError; use crate::Rkv; -/// A process is only permitted to have one open handle to each Rkv environment. -/// This manager exists to enforce that constraint: don't open environments directly. lazy_static! { static ref MANAGER: RwLock = RwLock::new(Manager::new()); } @@ -57,6 +55,8 @@ where Ok(canonical) } +/// A process is only permitted to have one open handle to each Rkv environment. +/// This manager exists to enforce that constraint: don't open environments directly. pub struct Manager { environments: BTreeMap>>, } diff --git a/third_party/rust/rkv/src/value.rs b/third_party/rust/rkv/src/value.rs index ffd93784f6d2..0f02250a0ea6 100644 --- a/third_party/rust/rkv/src/value.rs +++ b/third_party/rust/rkv/src/value.rs @@ -12,6 +12,7 @@ use arrayref::array_ref; use bincode::{ deserialize, serialize, + serialized_size, }; use ordered_float::OrderedFloat; @@ -176,10 +177,22 @@ impl<'s> Value<'s> { Value::Str(v) => serialize(&(Type::Str.to_tag(), v)), Value::Json(v) => serialize(&(Type::Json.to_tag(), v)), Value::Blob(v) => serialize(&(Type::Blob.to_tag(), v)), - Value::Uuid(v) => { - // Processed above to avoid verbose duplication of error transforms. - serialize(&(Type::Uuid.to_tag(), v)) - }, + Value::Uuid(v) => serialize(&(Type::Uuid.to_tag(), v)), + } + .map_err(DataError::EncodingError) + } + + pub fn serialized_size(&self) -> Result { + match self { + Value::Bool(v) => serialized_size(&(Type::Bool.to_tag(), *v)), + Value::U64(v) => serialized_size(&(Type::U64.to_tag(), *v)), + Value::I64(v) => serialized_size(&(Type::I64.to_tag(), *v)), + Value::F64(v) => serialized_size(&(Type::F64.to_tag(), v.0)), + Value::Instant(v) => serialized_size(&(Type::Instant.to_tag(), *v)), + Value::Str(v) => serialized_size(&(Type::Str.to_tag(), v)), + Value::Json(v) => serialized_size(&(Type::Json.to_tag(), v)), + Value::Blob(v) => serialized_size(&(Type::Blob.to_tag(), v)), + Value::Uuid(v) => serialized_size(&(Type::Uuid.to_tag(), v)), } .map_err(DataError::EncodingError) } @@ -216,3 +229,37 @@ impl<'s> From<&'s OwnedValue> for Value<'s> { } } } + +#[cfg(test)] +mod tests { + use super::*; + use ordered_float::OrderedFloat; + + #[test] + fn test_value_serialized_size() { + // | Value enum | tag: 1 byte | value_payload | + // |----------------------------------------------------------| + // | I64 | 1 | 8 | + // | U64 | 1 | 8 | + // | Bool | 1 | 1 | + // | Instant | 1 | 8 | + // | F64 | 1 | 8 | + // | Uuid | 1 | 16 | + // | Str/Blob/Json | 1 |(8: len + sizeof(payload))| + assert_eq!(Value::I64(-1000).serialized_size().unwrap(), 9); + assert_eq!(Value::U64(1000u64).serialized_size().unwrap(), 9); + assert_eq!(Value::Bool(true).serialized_size().unwrap(), 2); + assert_eq!(Value::Instant(1_558_020_865_224).serialized_size().unwrap(), 9); + assert_eq!(Value::F64(OrderedFloat(10000.1)).serialized_size().unwrap(), 9); + assert_eq!(Value::Str("hello!").serialized_size().unwrap(), 15); + assert_eq!(Value::Str("¡Hola").serialized_size().unwrap(), 15); + assert_eq!(Value::Blob(b"hello!").serialized_size().unwrap(), 15); + assert_eq!( + uuid(b"\x9f\xe2\xc4\xe9\x3f\x65\x4f\xdb\xb2\x4c\x02\xb1\x52\x59\x71\x6c") + .unwrap() + .serialized_size() + .unwrap(), + 17 + ); + } +} diff --git a/toolkit/components/kvstore/src/task.rs b/toolkit/components/kvstore/src/task.rs index e7a6ae1adda3..a4c48273e8bb 100644 --- a/toolkit/components/kvstore/src/task.rs +++ b/toolkit/components/kvstore/src/task.rs @@ -86,6 +86,76 @@ macro_rules! task_done { /// declare the type of `GetOrCreateTask::result`. type RkvStoreTuple = (Arc>, SingleStore); +// The threshold for active resizing. +const RESIZE_RATIO: f32 = 0.85; + +/// The threshold (50 MB) to switch the resizing policy from the double size to +/// the constant increment for active resizing. +const INCREMENTAL_RESIZE_THRESHOLD: usize = 52_428_800; + +/// The incremental resize step (5 MB) +const INCREMENTAL_RESIZE_STEP: usize = 5_242_880; + +/// The LMDB disk page size and mask. +const PAGE_SIZE: usize = 4096; +const PAGE_SIZE_MASK: usize = 0b_1111_1111_1111; + +/// Round the non-zero size to the multiple of page size greater or equal. +/// +/// It does not handle the special cases such as size zero and overflow, +/// because even if that happens (extremely unlikely though), LMDB will +/// ignore the new size if it's smaller than the current size. +/// +/// E.g: +/// [ 1 - 4096] -> 4096, +/// [4097 - 8192] -> 8192, +/// [8193 - 12286] -> 12286, +fn round_to_pagesize(size: usize) -> usize { + if size & PAGE_SIZE_MASK == 0 { + size + } else { + (size & !PAGE_SIZE_MASK) + PAGE_SIZE + } +} + +/// Kvstore employes two auto resizing strategies: active and passive resizing. +/// They work together to liberate consumers from having to guess the "proper" +/// size of the store upfront. See more detail about this in Bug 1543861. +/// +/// Active resizing that is performed at the store startup. +/// +/// It either increases the size in double, or by a constant size if its size +/// reaches INCREMENTAL_RESIZE_THRESHOLD. +/// +/// Note that on Linux / MAC OSX, the increased size would only take effect if +/// there is a write transaction committed afterwards. +fn active_resize(env: &Rkv) -> Result<(), StoreError> { + let info = env.info()?; + let current_size = info.map_size(); + let size; + + if current_size < INCREMENTAL_RESIZE_THRESHOLD { + size = current_size << 1; + } else { + size = current_size + INCREMENTAL_RESIZE_STEP; + } + + env.set_map_size(size)?; + Ok(()) +} + +/// Passive resizing that is performed when the MAP_FULL error occurs. It +/// increases the store with a `wanted` size. +/// +/// Note that the `wanted` size must be rounded to a multiple of page size +/// by using `round_to_pagesize`. +fn passive_resize(env: &Rkv, wanted: usize) -> Result<(), StoreError> { + let info = env.info()?; + let current_size = info.map_size(); + env.set_map_size(current_size + wanted)?; + Ok(()) +} + pub struct GetOrCreateTask { callback: AtomicCell>>, thread: AtomicCell>>, @@ -122,11 +192,17 @@ impl Task for GetOrCreateTask { // use the ? operator to simplify the implementation. self.result .store(Some(|| -> Result { + let store; let mut writer = Manager::singleton().write()?; let rkv = writer.get_or_create(Path::new(str::from_utf8(&self.path)?), Rkv::new)?; - let store = rkv - .write()? - .open_single(str::from_utf8(&self.name)?, StoreOptions::create())?; + { + let env = rkv.read()?; + let load_ratio = env.load_ratio()?; + if load_ratio > RESIZE_RATIO { + active_resize(&env)?; + } + store = env.open_single(str::from_utf8(&self.name)?, StoreOptions::create())?; + } Ok((rkv, store)) }())); } @@ -167,13 +243,40 @@ impl Task for PutTask { // We do the work within a closure that returns a Result so we can // use the ? operator to simplify the implementation. self.result.store(Some(|| -> Result<(), KeyValueError> { - let key = str::from_utf8(&self.key)?; let env = self.rkv.read()?; - let mut writer = env.write()?; + let key = str::from_utf8(&self.key)?; + let v = Value::from(&self.value); + let mut resized = false; - self.store - .put(&mut writer, key, &Value::from(&self.value))?; - writer.commit()?; + // Use a loop here in case we want to retry from a recoverable + // error such as `lmdb::Error::MapFull`. + loop { + let mut writer = env.write()?; + + match self.store.put(&mut writer, key, &v) { + Ok(_) => (), + + // Only handle the first MapFull error via passive resizing. + // Propogate the subsequent MapFull error. + Err(StoreError::LmdbError(lmdb::Error::MapFull)) if !resized => { + // abort the failed transaction for resizing. + writer.abort(); + + // calculate the size of pairs and resize the store accordingly. + let pair_size = key.len() + + v.serialized_size().map_err(StoreError::from)? as usize; + let wanted = round_to_pagesize(pair_size); + passive_resize(&env, wanted)?; + resized = true; + continue; + }, + + Err(err) => return Err(KeyValueError::StoreError(err)), + } + + writer.commit()?; + break; + } Ok(()) }())); @@ -205,6 +308,21 @@ impl WriteManyTask { result: AtomicCell::default(), } } + + fn calc_pair_size(&self) -> Result { + let mut total = 0; + + for (key, value) in self.pairs.iter() { + if let Some(val) = value { + total += key.len(); + total += Value::from(val) + .serialized_size() + .map_err(StoreError::from)? as usize; + } + } + + Ok(total) + } } impl Task for WriteManyTask { @@ -213,27 +331,57 @@ impl Task for WriteManyTask { // use the ? operator to simplify the implementation. self.result.store(Some(|| -> Result<(), KeyValueError> { let env = self.rkv.read()?; - let mut writer = env.write()?; + let mut resized = false; - for (key, value) in self.pairs.iter() { - let key = str::from_utf8(key)?; - match value { - Some(val) => self.store.put(&mut writer, key, &Value::from(val))?, - None => { - match self.store.delete(&mut writer, key) { - Ok(_) => (), + // Use a loop here in case we want to retry from a recoverable + // error such as `lmdb::Error::MapFull`. + 'outer: loop { + let mut writer = env.write()?; - // LMDB fails with an error if the key to delete wasn't found, - // and Rkv returns that error, but we ignore it, as we expect most - // of our consumers to want this behavior. - Err(StoreError::LmdbError(lmdb::Error::NotFound)) => (), + for (key, value) in self.pairs.iter() { + let key = str::from_utf8(key)?; + match value { + // To put. + Some(val) => { + match self.store.put(&mut writer, key, &Value::from(val)) { + Ok(_) => (), - Err(err) => return Err(KeyValueError::StoreError(err)), - }; + // Only handle the first MapFull error via passive resizing. + // Propogate the subsequent MapFull error. + Err(StoreError::LmdbError(lmdb::Error::MapFull)) if !resized => { + // Abort the failed transaction for resizing. + writer.abort(); + + // Calculate the size of pairs and resize accordingly. + let pair_size = self.calc_pair_size()?; + let wanted = round_to_pagesize(pair_size); + passive_resize(&env, wanted)?; + resized = true; + continue 'outer; + }, + + Err(err) => return Err(KeyValueError::StoreError(err)), + } + }, + // To delete. + None => { + match self.store.delete(&mut writer, key) { + Ok(_) => (), + + // LMDB fails with an error if the key to delete wasn't found, + // and Rkv returns that error, but we ignore it, as we expect most + // of our consumers to want this behavior. + Err(StoreError::LmdbError(lmdb::Error::NotFound)) => (), + + Err(err) => return Err(KeyValueError::StoreError(err)), + }; + } } } + + writer.commit()?; + break; // 'outer: loop } - writer.commit()?; Ok(()) }())); diff --git a/toolkit/components/kvstore/test/xpcshell/test_kvstore.js b/toolkit/components/kvstore/test/xpcshell/test_kvstore.js index 52f493804ddb..7c724e02ad33 100644 --- a/toolkit/components/kvstore/test/xpcshell/test_kvstore.js +++ b/toolkit/components/kvstore/test/xpcshell/test_kvstore.js @@ -100,6 +100,24 @@ add_task(async function putGetHasDelete() { Assert.strictEqual(await database.get("bool-key"), null); }); +add_task(async function putWithResizing() { + const databaseDir = await makeDatabaseDir("putWithResizing"); + const database = await KeyValueService.getOrCreate(databaseDir, "db"); + + // The default store size is 1MB, putting key/value pairs bigger than that + // would trigger auto resizing. + const base = "A humongous string in 32 bytes!!"; + const val1M = base.repeat(32768); + const val2M = val1M.repeat(2); + Assert.strictEqual(await database.put("A-1M-value", val1M), undefined); + Assert.strictEqual(await database.put("A-2M-value", val2M), undefined); + Assert.strictEqual(await database.put("A-32B-value", base), undefined); + + Assert.strictEqual(await database.get("A-1M-value"), val1M); + Assert.strictEqual(await database.get("A-2M-value"), val2M); + Assert.strictEqual(await database.get("A-32B-value"), base); +}); + add_task(async function largeNumbers() { const databaseDir = await makeDatabaseDir("largeNumbers"); const database = await KeyValueService.getOrCreate(databaseDir, "db"); @@ -212,6 +230,51 @@ add_task(async function writeManyPutOnly() { await test_helper(mapPairs); }); +add_task(async function writeManyLargePairsWithResizing() { + const databaseDir = await makeDatabaseDir("writeManyWithResizing"); + const database = await KeyValueService.getOrCreate(databaseDir, "db"); + + // The default store size is 1MB, putting key/value pairs bigger than that + // would trigger auto resizing. + const base = "A humongous string in 32 bytes!!"; + const val1M = base.repeat(32768); + const val2M = val1M.repeat(2); + + // writeMany with an object + const pairs = { + "A-1M-value": val1M, + "A-32B-value": base, + "A-2M-value": val2M, + }; + + Assert.strictEqual(await database.writeMany(pairs), undefined); + + Assert.strictEqual(await database.get("A-1M-value"), val1M); + Assert.strictEqual(await database.get("A-2M-value"), val2M); + Assert.strictEqual(await database.get("A-32B-value"), base); +}); + +add_task(async function writeManySmallPairsWithResizing() { + const databaseDir = await makeDatabaseDir("writeManyWithResizing"); + const database = await KeyValueService.getOrCreate(databaseDir, "db"); + + // The default store size is 1MB, putting key/value pairs bigger than that + // would trigger auto resizing. + const base = "A humongous string in 32 bytes!!"; + const val1K = base.repeat(32); + // writeMany with a key/value generator + function* pairMaker() { + for (let i = 0; i < 1024; i++) { + yield [`key-${i}`, val1K]; + } + } + + Assert.strictEqual(await database.writeMany(pairMaker()), undefined); + for (let i = 0; i < 1024; i++) { + Assert.ok(await database.has(`key-${i}`)); + } +}); + add_task(async function writeManyDeleteOnly() { const databaseDir = await makeDatabaseDir("writeManyDeletesOnly"); const database = await KeyValueService.getOrCreate(databaseDir, "db");