From 00243192ab001c6a44237a4cc83e6812b822970c Mon Sep 17 00:00:00 2001 From: Chris Date: Mon, 27 Jul 2026 22:36:57 +0100 Subject: [PATCH 1/3] fix(storage): cap LMDB map head-room on Windows to bound page-table memory The LMDB map is auto-sized to the whole disk (current data + all free space). On Windows the kernel eagerly builds page tables for the mapped region -- ~8 bytes per 4 KiB page (~1/512 of the map) as resident, non-pageable memory attributed to no process. A disk-sized map therefore costs ~0.2% of free disk in commit/RAM per node: a 3.7 TB map ~= 7.4 GB, a 10 TB map ~= 20 GB, multiplied by every node sharing the host. Linux populates map page tables lazily, so the sparse disk-sized map is nearly free there -- this overhead is Windows-specific. Cap the disk-headroom term at 32 GiB on Windows and rely on the existing try_resize() to extend the map on demand as data accumulates, keeping page-table overhead proportional to stored data rather than disk capacity. Existing data is always covered, so a capped map can never truncate the DB. Verified on a live node holding 22.6 GiB of chunks: map size 3,719 GiB -> 54.58 GiB commit 7,537 MB -> 181 MB virtual 3,719 GB -> 60 GB Linux behaviour is unchanged (#[cfg(windows)]); unit tests assert both the Windows head-room cap and the disk-sized default on other platforms. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/storage/lmdb.rs | 92 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 87 insertions(+), 5 deletions(-) diff --git a/src/storage/lmdb.rs b/src/storage/lmdb.rs index acda5ea9..55b7ca63 100644 --- a/src/storage/lmdb.rs +++ b/src/storage/lmdb.rs @@ -40,6 +40,27 @@ fn bytes_to_gib(bytes: u64) -> f64 { /// Set to 256 MiB — enough for millions of LMDB pages. const MIN_MAP_SIZE: usize = 256 * 1024 * 1024; +/// Maximum head-room (beyond the current data footprint) to reserve for the +/// LMDB map **on Windows**. +/// +/// The map is memory-mapped, and on Windows the kernel eagerly builds page +/// tables proportional to the *mapped* size — ~1 byte of page-table per 512 +/// bytes mapped (an 8-byte PTE per 4 KiB page). Those page-table pages are +/// resident, non-pageable kernel memory attributed to *no* process, so sizing +/// the map to the whole disk (as [`compute_map_size`] otherwise does) costs +/// ~0.2% of free disk in RAM/commit *per node*: a 10 TiB partition ≈ 20 GiB of +/// page tables, multiplied by every node sharing the host. Linux populates map +/// page tables lazily on access, so a sparse disk-sized map is nearly free +/// there — this overhead is Windows-specific. +/// +/// We therefore cap the head-room on Windows and lean on +/// [`LmdbStorage::try_resize`] to extend the map on demand as data actually +/// accumulates, keeping page-table overhead proportional to *stored data* +/// rather than to *disk capacity*. At 32 GiB the resident page-table cost is +/// ~64 MiB per node, and a resize happens at most once per 32 GiB written. +#[cfg(windows)] +const WINDOWS_MAP_HEADROOM: u64 = 32 * GIB; + /// How often to re-query available disk space (in seconds). /// /// Between checks the cached result is trusted. Disk space changes slowly @@ -164,7 +185,8 @@ impl LmdbStorage { // Auto-scale: current DB footprint + available space − reserve. let computed = compute_map_size(&env_dir, config.disk_reserve)?; info!( - "Auto-computed LMDB map size: {:.2} GiB (available disk minus {:.2} GiB reserve)", + "Auto-computed LMDB map size: {:.2} GiB (data + available disk minus {:.2} GiB \ + reserve, head-room capped on Windows to bound page-table overhead)", bytes_to_gib(computed as u64), bytes_to_gib(config.disk_reserve), ); @@ -696,6 +718,10 @@ enum PutOutcome { /// it back ensures the map is always large enough for the data already /// stored. /// +/// On Windows the disk-headroom term is additionally capped at +/// [`WINDOWS_MAP_HEADROOM`] to bound page-table overhead (see that constant); +/// [`LmdbStorage::try_resize`] extends the map on demand as data grows. +/// /// The result is page-aligned and never falls below [`MIN_MAP_SIZE`]. fn compute_map_size(db_dir: &Path, reserve: u64) -> Result { let available = fs2::available_space(db_dir) @@ -705,10 +731,7 @@ fn compute_map_size(db_dir: &Path, reserve: u64) -> Result { let mdb_file = db_dir.join("data.mdb"); let current_db_bytes = std::fs::metadata(&mdb_file).map_or(0, |m| m.len()); - // available_space excludes the DB file, so we add it back to get the - // total space the DB could occupy while still leaving `reserve` free. - let growth_room = available.saturating_sub(reserve); - let target = current_db_bytes.saturating_add(growth_room); + let target = map_target_bytes(current_db_bytes, available, reserve); // Align up to system page size (required by heed's resize). let page = page_size::get() as u64; @@ -718,6 +741,27 @@ fn compute_map_size(db_dir: &Path, reserve: u64) -> Result { Ok(result.max(MIN_MAP_SIZE)) } +/// Head-room policy for the LMDB map, split out from [`compute_map_size`] so it +/// is unit-testable without touching the real filesystem. +/// +/// `map = current_db_bytes + max(0, available − reserve)`, with the head-room +/// term capped at [`WINDOWS_MAP_HEADROOM`] on Windows. Existing data +/// (`current_db_bytes`) is always covered so a resize can never truncate the +/// database, even when the head-room cap or a nearly-full disk drives the +/// growth term to zero. +fn map_target_bytes(current_db_bytes: u64, available: u64, reserve: u64) -> u64 { + // available_space excludes the DB file, so we add it back to get the + // total space the DB could occupy while still leaving `reserve` free. + let growth_room = available.saturating_sub(reserve); + + // On Windows, bound the head-room so page tables stay proportional to + // stored data rather than disk capacity. Elsewhere, use all reachable space. + #[cfg(windows)] + let growth_room = growth_room.min(WINDOWS_MAP_HEADROOM); + + current_db_bytes.saturating_add(growth_room) +} + /// Reject the write early if available disk space is below `reserve`. fn check_disk_space(db_dir: &Path, reserve: u64) -> Result<()> { let available = fs2::available_space(db_dir) @@ -741,6 +785,44 @@ mod tests { use super::*; use tempfile::TempDir; + #[test] + fn map_target_covers_existing_data_and_headroom() { + // A partition with far more free space than any node needs. + let huge_free = 100 * 1024 * GIB; // 100 TiB + let reserve = 500 * MIB; + + // Fresh node, no data yet. + let fresh = map_target_bytes(0, huge_free, reserve); + // Node already holding 16 GiB of chunks. + let with_data = map_target_bytes(16 * GIB, huge_free, reserve); + + #[cfg(windows)] + { + // Windows: head-room is capped, so the map (and thus page tables) + // stay bounded regardless of disk size. Existing data always sits + // on top of the capped head-room. + assert_eq!(fresh, WINDOWS_MAP_HEADROOM); + assert_eq!(with_data, 16 * GIB + WINDOWS_MAP_HEADROOM); + // Sanity: page-table cost (~map/512) is tens of MiB, not tens of GiB. + assert!(with_data / 512 < 128 * MIB); + } + + #[cfg(not(windows))] + { + // Other platforms keep the disk-sized map (lazy page tables cost + // nothing), so head-room is the full free span minus reserve. + assert_eq!(fresh, huge_free - reserve); + assert_eq!(with_data, 16 * GIB + (huge_free - reserve)); + } + } + + #[test] + fn map_target_never_truncates_data_when_disk_nearly_full() { + // Free space below the reserve → head-room saturates to 0 on every + // platform, but the existing 4 GiB of data must still be covered. + assert_eq!(map_target_bytes(4 * GIB, 100 * MIB, 500 * MIB), 4 * GIB); + } + async fn create_test_storage() -> (LmdbStorage, TempDir) { let temp_dir = TempDir::new().expect("create temp dir"); let config = LmdbStorageConfig { From e44171fd8a0e26a68380b1406081568c39cf7ed8 Mon Sep 17 00:00:00 2001 From: Chris Date: Mon, 27 Jul 2026 23:30:49 +0100 Subject: [PATCH 2/3] fix(storage): guard all_keys/get_raw with env_lock; doc + wording fixes Address PR #187 review (V2-620): - Blocking (resize safety): all_keys() and get_raw() opened LMDB read transactions without taking the shared env_lock, so try_resize()'s unsafe Env::resize() (held under the exclusive lock) could unmap/remap the environment while a read txn or cursor was still live. This PR makes resize a routine ~per-32-GiB Windows event rather than a rare partition-expansion path, making that pre-existing race materially reachable. Both read paths now hold env_lock.read() across the transaction, matching every other txn site. Adds a regression test asserting both block while the exclusive lock is held. - Rustdoc: intra-doc links to WINDOWS_MAP_HEADROOM (a #[cfg(windows)] const) would be broken on non-Windows doc builds; switched to plain code spans. - Wording: describe the Windows overhead as the measured, size-proportional commit/private cost (~0.2% of mapped size) rather than asserting a specific hardware page-table mechanism that was inferred from the scaling, not directly measured. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/storage/lmdb.rs | 99 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 80 insertions(+), 19 deletions(-) diff --git a/src/storage/lmdb.rs b/src/storage/lmdb.rs index 55b7ca63..3e97bae1 100644 --- a/src/storage/lmdb.rs +++ b/src/storage/lmdb.rs @@ -43,21 +43,24 @@ const MIN_MAP_SIZE: usize = 256 * 1024 * 1024; /// Maximum head-room (beyond the current data footprint) to reserve for the /// LMDB map **on Windows**. /// -/// The map is memory-mapped, and on Windows the kernel eagerly builds page -/// tables proportional to the *mapped* size — ~1 byte of page-table per 512 -/// bytes mapped (an 8-byte PTE per 4 KiB page). Those page-table pages are -/// resident, non-pageable kernel memory attributed to *no* process, so sizing -/// the map to the whole disk (as [`compute_map_size`] otherwise does) costs -/// ~0.2% of free disk in RAM/commit *per node*: a 10 TiB partition ≈ 20 GiB of -/// page tables, multiplied by every node sharing the host. Linux populates map -/// page tables lazily on access, so a sparse disk-sized map is nearly free -/// there — this overhead is Windows-specific. +/// On Windows a node's committed / private memory scales with the *mapped* +/// size rather than with the data actually stored. Measured on a live node: +/// ~7.4 GB of extra commit for a ~3.7 TiB map, versus ~181 MB for a ~55 GiB +/// map — roughly 0.2% of the mapped size, resident and attributed to *no* +/// user-space allocation. That size-proportional cost is consistent with +/// kernel page tables / section metadata for the mapping; the exact kernel +/// structure was inferred from the scaling rather than measured directly, but +/// the size-proportional *effect* is what this cap targets. Linux keeps the +/// mapping sparse, so a disk-sized map is nearly free there — the overhead is +/// Windows-specific, which is why it only surfaced in Windows reports. /// -/// We therefore cap the head-room on Windows and lean on -/// [`LmdbStorage::try_resize`] to extend the map on demand as data actually -/// accumulates, keeping page-table overhead proportional to *stored data* -/// rather than to *disk capacity*. At 32 GiB the resident page-table cost is -/// ~64 MiB per node, and a resize happens at most once per 32 GiB written. +/// Sizing the map to the whole disk therefore costs ~0.2% of *free disk* per +/// node, multiplied by every node sharing the host (e.g. a 10 TiB partition ≈ +/// 20 GiB). We instead cap the head-room on Windows and lean on +/// `LmdbStorage::try_resize` to extend the map on demand as data accumulates, +/// keeping the overhead proportional to *stored data* rather than *disk +/// capacity*. At 32 GiB the extra commit is ~100 MB, and a resize happens at +/// most once per 32 GiB written. #[cfg(windows)] const WINDOWS_MAP_HEADROOM: u64 = 32 * GIB; @@ -546,8 +549,13 @@ impl LmdbStorage { pub async fn all_keys(&self) -> Result> { let env = self.env.clone(); let db = self.db; + let lock = Arc::clone(&self.env_lock); let keys = spawn_blocking(move || -> Result> { + // Hold the shared lock for the whole read so try_resize() (which + // takes the exclusive lock before the unsafe Env::resize()) cannot + // unmap the environment while this txn and its cursor are live. + let _guard = lock.read(); let rtxn = env .read_txn() .map_err(|e| Error::Storage(format!("Failed to create read txn: {e}")))?; @@ -589,8 +597,12 @@ impl LmdbStorage { let key = *address; let env = self.env.clone(); let db = self.db; + let lock = Arc::clone(&self.env_lock); let value = spawn_blocking(move || -> Result>> { + // Shared lock held until the bytes are copied out, so a concurrent + // try_resize() cannot unmap the environment mid-read. See all_keys. + let _guard = lock.read(); let rtxn = env .read_txn() .map_err(|e| Error::Storage(format!("Failed to create read txn: {e}")))?; @@ -719,8 +731,9 @@ enum PutOutcome { /// stored. /// /// On Windows the disk-headroom term is additionally capped at -/// [`WINDOWS_MAP_HEADROOM`] to bound page-table overhead (see that constant); -/// [`LmdbStorage::try_resize`] extends the map on demand as data grows. +/// `WINDOWS_MAP_HEADROOM` to bound the map-proportional commit overhead (see +/// that constant); [`LmdbStorage::try_resize`] extends the map on demand as +/// data grows. /// /// The result is page-aligned and never falls below [`MIN_MAP_SIZE`]. fn compute_map_size(db_dir: &Path, reserve: u64) -> Result { @@ -745,7 +758,7 @@ fn compute_map_size(db_dir: &Path, reserve: u64) -> Result { /// is unit-testable without touching the real filesystem. /// /// `map = current_db_bytes + max(0, available − reserve)`, with the head-room -/// term capped at [`WINDOWS_MAP_HEADROOM`] on Windows. Existing data +/// term capped at `WINDOWS_MAP_HEADROOM` on Windows. Existing data /// (`current_db_bytes`) is always covered so a resize can never truncate the /// database, even when the head-room cap or a nearly-full disk drives the /// growth term to zero. @@ -754,8 +767,9 @@ fn map_target_bytes(current_db_bytes: u64, available: u64, reserve: u64) -> u64 // total space the DB could occupy while still leaving `reserve` free. let growth_room = available.saturating_sub(reserve); - // On Windows, bound the head-room so page tables stay proportional to - // stored data rather than disk capacity. Elsewhere, use all reachable space. + // On Windows, bound the head-room so the mapped size (and its + // size-proportional commit overhead) stays proportional to stored data + // rather than disk capacity. Elsewhere, use all reachable space. #[cfg(windows)] let growth_room = growth_room.min(WINDOWS_MAP_HEADROOM); @@ -823,6 +837,53 @@ mod tests { assert_eq!(map_target_bytes(4 * GIB, 100 * MIB, 500 * MIB), 4 * GIB); } + /// Regression (V2-620 review): `all_keys` and `get_raw` must take the shared + /// `env_lock` so their LMDB read transaction can never run concurrently with + /// `try_resize()`'s unsafe `Env::resize()` — which this PR turns into a + /// routine ~per-32-GiB Windows event. We prove it by holding the exclusive + /// lock (as a resize does) and asserting both calls block until it is freed. + /// + /// Holding a `parking_lot` guard across `.await` is deliberate and safe here: + /// the guard stays on this current-thread test task while `all_keys`/`get_raw` + /// run their blocking work on the `spawn_blocking` pool (separate threads). + #[tokio::test] + #[allow(clippy::await_holding_lock)] + async fn read_paths_block_while_env_is_being_resized() { + use std::time::Duration; + let (storage, _temp) = create_test_storage().await; + + // Store one chunk so the read paths have real work to return. + let content = b"resize-safety"; + let address = LmdbStorage::compute_address(content); + storage.put(&address, content).await.expect("put"); + + // Simulate a resize in progress: hold the exclusive env lock. + let write_guard = storage.env_lock.write(); + + // Neither read path may complete while the exclusive lock is held — + // before the fix they took no lock and would return immediately. + assert!( + tokio::time::timeout(Duration::from_millis(250), storage.all_keys()) + .await + .is_err(), + "all_keys completed while env_lock was held exclusively — missing shared guard" + ); + assert!( + tokio::time::timeout(Duration::from_millis(250), storage.get_raw(&address)) + .await + .is_err(), + "get_raw completed while env_lock was held exclusively — missing shared guard" + ); + + // Once the exclusive lock is released, both proceed and see the data. + drop(write_guard); + assert_eq!(storage.all_keys().await.expect("all_keys").len(), 1); + assert_eq!( + storage.get_raw(&address).await.expect("get_raw").as_deref(), + Some(content.as_slice()) + ); + } + async fn create_test_storage() -> (LmdbStorage, TempDir) { let temp_dir = TempDir::new().expect("create temp dir"); let config = LmdbStorageConfig { From 43665bf35562000128d30e057d4448d342eceb94 Mon Sep 17 00:00:00 2001 From: Chris Date: Tue, 28 Jul 2026 11:59:29 +0100 Subject: [PATCH 3/3] =?UTF-8?q?docs(adr):=20ADR-0007=20=E2=80=94=20cap=20t?= =?UTF-8?q?he=20Windows=20LMDB=20map=20head-room=20(V2-620)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Records the decision behind PR #187: on Windows, cap the LMDB map head-room at 32 GiB and grow on demand via try_resize(), so committed/private memory is proportional to stored data rather than disk capacity. Non-Windows keeps the disk-sized map. Documents the resize-cadence trade-off and the env_lock discipline it depends on. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../ADR-0007-windows-lmdb-map-headroom-cap.md | 115 ++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 docs/adr/ADR-0007-windows-lmdb-map-headroom-cap.md diff --git a/docs/adr/ADR-0007-windows-lmdb-map-headroom-cap.md b/docs/adr/ADR-0007-windows-lmdb-map-headroom-cap.md new file mode 100644 index 00000000..79d0cc5c --- /dev/null +++ b/docs/adr/ADR-0007-windows-lmdb-map-headroom-cap.md @@ -0,0 +1,115 @@ +# ADR-0007: Cap the Windows LMDB map head-room + +- **Status:** Proposed +- **Date:** 2026-07-28 +- **Decision owners:** @jacderida +- **Reviewers:** @dirvine +- **Supersedes:** none +- **Superseded by:** none +- **Related:** PR #187, Linear V2-620 + +## Context + +The LMDB chunk store auto-sizes its memory map to the whole storage partition: +`map = current_db_bytes + max(0, available_disk − reserve)` (`compute_map_size`). +On Linux the mapping stays sparse and this is effectively free. + +On Windows, a node's committed / private memory scales with the *mapped* size +rather than with the data actually stored. Field reports of excessive +`ant-node.exe` memory were traced to this: a live single node showed ~7.4 GB of +extra commit for a ~3.7 TiB map, and a user running three nodes on a 64 GiB +machine reported ~60 GB "in use" that did not correspond to any per-process +working set. The overhead is size-proportional at roughly 0.2% of the mapped +size (measured: ~7.4 GB at ~3.7 TiB, ~181 MB at ~55 GiB). The exact kernel +structure — page tables / section metadata for the reserved file view — was +inferred from that scaling, not measured directly; the size-proportional effect +is what matters here. Because the map is sized to *free disk*, a larger disk +and/or multiple co-located nodes multiply the cost until it can exhaust a host's +commit limit or RAM. + +Per-process working set stays small (~250–750 MB), so the problem is invisible +in Task Manager's default column and appears only in commit / virtual / +system-total views — which is why it was historically hard to pin down. + +## Decision Drivers + +- Windows nodes (especially multi-node, large-disk hosts) must not consume + memory proportional to disk *capacity*. +- Nodes must retain existing data and still be able to grow to fill the disk. +- Linux behaviour, already correct and free, must not regress. +- The fix must be low-risk: no change to the wire protocol, stored-data format, + payments, or the upgrade mechanism. + +## Considered Options + +1. **Cap the map head-room on Windows and grow on demand.** Size the initial map + to `current_db_bytes + min(free − reserve, HEADROOM)` on Windows; rely on the + existing `try_resize()` (already used for operator disk-expansion) to extend + the map as data accumulates. +2. **Cap on all platforms.** Simpler cfg, but needlessly penalises Linux (more + frequent exclusive-lock resizes) for a problem it does not have. +3. **Expose only a manual `db_size_gb` cap.** Already exists but is off by + default; leaves every default Windows deployment broken and pushes the burden + onto operators. +4. **Change the LMDB mapping strategy / write mode on Windows.** Larger blast + radius, touches storage-engine internals and durability behaviour; + disproportionate to the problem. + +## Decision + +We will adopt Option 1: on Windows, cap the disk-headroom term at +`WINDOWS_MAP_HEADROOM = 32 GiB` (`map_target_bytes`, `#[cfg(windows)]`), always +covering `current_db_bytes` so a resize can never truncate the database, and +rely on `try_resize()` to extend the map on demand (~once per 32 GiB written). +Non-Windows platforms keep the disk-sized map unchanged. + +## Consequences + +### Positive + +- Windows commit/private overhead drops from ~0.2% of *disk capacity* to ~0.2% + of *stored data + 32 GiB* — e.g. ~7.4 GB → ~181 MB on the validated node; the + 3-node / 64 GiB report projects to well under 1 GB. +- Existing databases open unchanged (verified against a live 22.6 GiB store); no + migration. +- Linux is untouched. + +### Negative / Trade-offs + +- Resize becomes a routine Windows lifecycle event (~per 32 GiB) rather than a + rare partition-expansion path. `try_resize()` takes the exclusive `env_lock` + and calls the unsafe `Env::resize()`, so it briefly stalls that node's storage + transactions during the remap. Mitigated by the 32 GiB granularity (infrequent) + and by requiring *every* read/write transaction to hold the shared `env_lock` + for its full duration — this PR fixes `all_keys`/`get_raw`, which previously + did not, closing an unmap-during-read race. +- The head-room constant is a heuristic; a workload writing >32 GiB between + resizes would see back-to-back resizes. 32 GiB keeps the map-proportional + overhead ~100 MB while making resizes rare. + +### Neutral / Operational + +- Operators can still set an explicit `db_size_gb` cap to override the default. +- The exact Windows kernel accounting was inferred, not directly measured; a + follow-up can confirm via RAMMap's Page Table row if a definitive attribution + is wanted. + +## Validation + +- Unit tests: `map_target_bytes` cap binds on Windows and is a no-op elsewhere; + existing data is always covered even when the head-room term saturates to zero. +- Resize-safety regression test `read_paths_block_while_env_is_being_resized`: + proves `all_keys`/`get_raw` hold the shared lock and cannot run concurrently + with a resize. +- Live before/after on a Windows production node (22.6 GiB stored): map + 3,719 GiB → 54.58 GiB; commit 7,537 MB → 181 MB; virtual 3,719 GB → 60 GB; DB + reopened without data loss. +- Review trigger: revisit `WINDOWS_MAP_HEADROOM` if Windows resize-stall latency + is observed under concurrent replication/audit load, or if a direct + kernel-attribution measurement changes the cost model. + +## Notes for AI-assisted work + +AI tools may help draft this ADR, but **must not mark it Accepted without human +review**. Accepted ADRs are immutable: create a new superseding ADR rather than +editing an Accepted ADR.