diff --git a/Cargo.lock b/Cargo.lock index 8be9cff9..f90f3fa8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -813,6 +813,7 @@ version = "0.15.0" dependencies = [ "alloy", "ant-protocol", + "bao", "blake3", "bytes", "chrono", @@ -1322,6 +1323,16 @@ dependencies = [ "windows-link", ] +[[package]] +name = "bao" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9125f93587b894c196d58b0a752ce2213552d0d483c98ee41530e38202a511" +dependencies = [ + "arrayvec", + "blake3", +] + [[package]] name = "base16ct" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index 9add171f..8dd346a7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -111,6 +111,7 @@ page_size = "0.6" # Protocol serialization postcard = { version = "1.1.3", features = ["use-std"] } +bao = "0.13.1" [target.'cfg(unix)'.dependencies] libc = "0.2" diff --git a/docs/adr/ADR-0002-gossip-triggered-contiguous-subtree-audit.md b/docs/adr/ADR-0002-gossip-triggered-contiguous-subtree-audit.md index 24ab7413..b57b8857 100644 --- a/docs/adr/ADR-0002-gossip-triggered-contiguous-subtree-audit.md +++ b/docs/adr/ADR-0002-gossip-triggered-contiguous-subtree-audit.md @@ -1,12 +1,21 @@ # ADR-0002: Gossip-triggered contiguous-subtree storage audit - **Status:** Proposed -- **Date:** 2026-06-04 +- **Date:** 2026-06-04 (round-2 proof shape updated 2026-07 for the V2-685 verified-slice audit) - **Decision owners:** Anselme (@grumbach) - **Reviewers:** - **Supersedes:** none - **Superseded by:** none -- **Related:** none +- **Related:** ADR-0004 (audit grace / Amendment 1) + +> **Amendment (V2-685, 2026-07).** Round 2 no longer returns full chunk bytes: it +> returns a few-KB **BLAKE3/Bao verified slice** per opened block, checked against +> the chunk address (authenticity) and a keyed **nonced block-tree root** committed +> in round 1 (possession). Possession is now a round-1 cryptographic commitment +> rather than a deadline property; per-leaf "freshness hashes" become keyed nonced +> roots; retention is TTL-bounded (not "last two"); the subtree audit rides a +> dedicated protocol id; and round 1 gains responder rate/session/heavy-pool +> controls. The Decision and Consequences below reflect this. ## Context @@ -46,7 +55,7 @@ tree is a clean binary shape when N is not a power of two. - Ensure all nodes actually store the data they claim they are storing - Keep each proof small and keep steady-state audit traffic low. - Catch the three real cheating strategies: storing nothing and fetching on demand; deleting some fraction of data; and keeping only chunk *addresses* (which are public) while never holding the actual bytes, then fabricating proofs. -- Reuse the existing cryptographic building blocks (the Merkle tree, the signed commitment, the freshness hash) without inventing new ones. +- Reuse the existing cryptographic building blocks (the Merkle tree, the signed commitment, and a per-leaf keyed nonced block-tree root) without inventing new ones. - Never wrongly penalise honest nodes, even in extreme cases like on small or dense networks where every node legitimately holds almost all of the data. ## Considered Options @@ -81,8 +90,10 @@ tree is a clean binary shape when N is not a power of two. ## Decision We will make the audit **gossip-triggered** and replace its proof shape with a -**single contiguous-subtree storage proof**, reusing the existing tree, -commitment, and freshness-hash primitives. +**single contiguous-subtree storage proof**, reusing the existing tree and +signed commitment. Possession is proven by a keyed nonced block-tree opened by +Bao verified slices (per the V2-685 amendment above), not by per-leaf freshness +hashes. - **Trigger.** When a node ingests a neighbour's commitment during normal (steady-state) operation, it may start an audit of that neighbour — not every @@ -115,53 +126,67 @@ commitment, and freshness-hash primitives. branch is exactly the one the random value selects and that it contains at least the square root of the claimed held chunks in real leaves. -- **The proof.** The audited node returns every leaf of the selected subtree — - each given both as the plain content hash and as a freshness hash (the content - mixed with the auditor's random value) — plus one summary hash per level for the - unselected siblings along the path to the root. Everything outside the selected - branch costs a single hash; nothing there is touched. +- **The proof (round 1).** The audited node returns every leaf of the selected + subtree — each given as the plain content hash (the chunk address), its content + length, and a fresh **nonced block-tree root**: a Merkle root over the chunk's + 1 KiB blocks whose leaves are *keyed* BLAKE3 over each block, keyed by the fresh + audit nonce (plus peer and key). It also returns one summary hash per level for + the unselected siblings along the path to the root. Everything outside the + selected branch costs a single hash; nothing there is touched. (This replaces + the earlier flat per-leaf "freshness hash": keying every block by the nonce + leaves no nonce-independent chaining value to precompute, so building a correct + root requires *all* of a chunk's bytes under that nonce.) - **Verification, three independent checks.** - *Structure:* rebuild the root from the returned subtree and the sibling summaries; it must equal the freshly-published root the audit was started against. This proves the subtree genuinely belongs to the committed tree. - - *Real bytes:* after the full subtree proof is in hand, pick a small fixed - number of its leaves and demand the original chunk bytes for exactly those - keys from the audited node itself (a second-round surprise challenge), then - confirm both the plain hash (the chunk's content address) and the freshness - hash match the served bytes. The sample is drawn with **fresh randomness - chosen by the auditor after round 1 — NOT derived from the round-1 nonce**. - This is essential: the structural root check binds only `(key, bytes_hash)`, - both of which are public (the `bytes_hash` *is* the chunk's network address), - not the per-leaf freshness hash. If the sample were predictable at - proof-build time, a relay could fabricate the freshness hash on every leaf, - fetch only the few leaves it knew would be opened, and pass while holding - almost nothing. Drawing the sample after the proof commits turns this into a - cut-and-choose: the node must have produced a correct freshness hash — which - needs the real bytes — for essentially every leaf, or be caught. Possession - is non-delegable: the auditor needs to hold none of the node's chunks, and a - committed key the node cannot serve is a deterministic failure, never bad - luck. So a node that rebuilt the tree from public chunk addresses but never - held the bytes cannot serve content that hashes to the committed address; - faking a fraction of leaves survives only with probability (1 − fraction) - raised to the number of spot-checks. - - *Possession in time:* the whole response must arrive within a deadline sized - to hashing the subtree from local disk. A node that doesn't hold the data must - fetch it across the network first and misses the deadline. + - *Possession by verified slice (round 2):* after the full subtree proof is in + hand, pick a small fixed number of its leaves and, for each, a **freshly-random + 1 KiB block index** (plus the claimed final block, which pins the true content + length via Bao's EOF check). Demand a **verified slice** per opened block, not + the whole chunk: a Bao slice proving the block against the chunk address + (authenticity, ~KB not MB), plus a nonced-tree opening proving it against the + round-1 `nonced_root` (possession), both over the same served bytes. The sample + is drawn with **fresh randomness chosen by the auditor after round 1 — NOT + derived from the round-1 nonce**: the structural root binds only + `(key, bytes_hash)`, both public, so a predictable sample would let a relay + fabricate a `nonced_root` on every leaf and hold only the few it knew would be + opened. Drawing the sample after the proof commits makes this a cut-and-choose; + a node that never held the bytes cannot have committed a correct nonced root, + and faking a fraction of blocks survives only with probability (1 − fraction) + raised to the number of spot-checks. Possession is non-delegable and the + auditor holds none of the node's chunks. + - *Deadline (liveness, not the possession proof):* the response must still + arrive within a deadline, but — unlike the earlier full-byte round 2 — the + deadline is no longer load-bearing for possession (that is now the round-1 + `nonced_root` commitment). It bounds liveness and how long the auditor waits; + a missed deadline is graced (see Accounting). - **Retention — "you stay answerable for what you publish."** A node keeps the - chunk data behind its **last two published commitments**. Two, not one, absorbs - the normal race where an auditor is asking about the commitment a node published - just before its newest one. Because of this, an honest node can always answer an - audit about a commitment it published recently — so "I don't recognise that - commitment" about a recently-published root is now provably misbehaviour, not - lag. + chunk data behind its recently published commitments for a **bounded TTL + window** (with a slot backstop), so an honest node can always answer an audit + about a commitment it published recently — meaning "I don't recognise that + commitment" about a recently-published root is provably misbehaviour, not lag. + +- **Responder resource controls.** The heavy round 1 (which hashes the whole + `sqrt(N)` subtree, up to gigabytes for a maximal commitment) runs on a blocking + thread pool with its own tight admission (a small global + one per peer), + separate from the light responsible/slice audits, plus a per-peer rate cooldown; + a round-2 slice challenge is served only after a matching single-use round-1 + session. This keeps a burst of audit requests from starving honest audits or the + async runtime, without changing the audit's security properties. + +- **Rollout.** Only the subtree-audit messages changed wire format, so they ride a + dedicated protocol id while core replication stays on its existing id; a + mixed-version fleet keeps replicating, syncing, fetching and repairing across + versions, and only cross-version subtree audits pause during the upgrade window. - **Accounting and False Positives** "That chunk isn't in my commitment" can never occur, because the auditor only ever challenges leaves of the node's *own* committed tree, so every challenged leaf is in the commitment by construction. Failures that are deterministic and cannot be caused by bad luck — a - rebuilt root that doesn't match, a content or freshness hash that doesn't match, + rebuilt root that doesn't match, a content-address or nonced-root check that doesn't match, or repudiating a recently-published commitment — are acted on **the first time they occur**, because re-asking cannot turn a genuine failure into a pass. Failures that *can* be caused by transient bad luck — a missed response deadline @@ -202,19 +227,19 @@ commitment, and freshness-hash primitives. - The probabilistic approach to verification ensures that verification is cheap but over time efficient. - Each proof is small and contiguous (about the square root of N leaves plus a handful of summary hashes) instead of many scattered inclusion paths. - Audits are surprise exams pinned to the *freshly published* commitment, so there is no stale-data ambiguity unlike in the previous audit design -- Three independent defences cover the three cheating strategies: structure (belongs to the committed tree), real bytes (actually held, not fabricated from public addresses), and timeliness (held locally, not fetched on demand). +- Independent defences cover the cheating strategies: structure (belongs to the committed tree), possession (a keyed nonced-root commitment in round 1, opened by verified slices in round 2, so it cannot be fabricated from public addresses), with the response deadline bounding liveness rather than carrying the possession proof. - Acting on the first deterministic failure roughly cuts time-to-detection compared with requiring several strikes, with no added risk of false positives. ### Negative / Trade-offs - **Big-block deletion is caught only proportionally.** An attacker who deletes data in large contiguous blocks is caught, per audit, with probability roughly equal to the fraction deleted — independent of N and of subtree size. We accept this: there is no economic reason to delete a *small* fraction (you save almost nothing and are still eventually caught), and a node that deletes a large fraction to actually save resources is caught within one or two audits. If ever needed, the lever is auditing *more often*, not bigger subtrees. - **Inflating the claimed size is not fully prevented.** Only the selected subtree and the path summaries are verified each audit, so filler leaves elsewhere could inflate the claimed chunk count. Both the regular audits and the closeness check mitigates this over time. Fully auditing the entire claimed set would be too much effort. We accept this probabilistic approach in which over time cheaters are detected. -- **Retention has a storage cost.** A node must keep the chunk data behind its last two published commitments. This is an accepted cost. -- **The audit format change is breaking.** The whole network must upgrade before the new audit can be relied on and before eviction is enabled. +- **Retention has a storage cost.** A node must keep the chunk data behind its recently published commitments for a bounded TTL window (with a slot backstop). This is an accepted cost. +- **The audit format change is contained, not whole-network breaking.** Only the subtree-audit messages move to a dedicated protocol id; core replication stays on v2, so mixed-version nodes still interoperate for everything else. During rollout a cross-version subtree audit may go unanswered — a bounded, self-healing trust effect (see the `SUBTREE_AUDIT_PROTOCOL_ID` note), strictly milder than a global protocol bump. Eviction can be relied on once the fleet has upgraded. ### Neutral / Operational -- Introduces a few tunable settings: the per-gossip audit probability, the per-neighbour cooldown, the number of real-byte spot-checks, and the retention count (two). The grace allowance for missed deadlines reuses the existing strike threshold and applies to deadline misses only. +- Introduces a few tunable settings: the per-gossip audit probability, the per-neighbour cooldown, the number of block spot-checks per audit, the retention TTL window, and the heavy round-1 responder pool / rate cooldown / session TTL. The grace allowance for missed deadlines reuses the existing strike threshold and applies to deadline misses only. - The storage-commitment audit needs no periodic timer of its own — it is driven by gossip. (The separate responsible-chunk audit keeps its periodic tick; the two run side by side.) The related "node is capable but has no current commitment" special case is unnecessary on the gossip-triggered path, since that path always has a freshly-published commitment to pin. A silent node needs no special handling for this audit — it simply stops earning storage credit, so all nodes are naturally motivated to gossip. - At the chosen settings, steady-state audit load is on the order of a handful of small audits per node per hour. @@ -234,15 +259,16 @@ How we will know this decision remains correct: and identical on the auditor and the audited node; selection never lands on an all-padding branch across many awkward sizes (a regression test for the fixed-depth flaw this ADR fixes); the root rebuilds correctly from a single-branch - proof; possession verifies from the bytes the audited node itself serves in the - second-round byte challenge (the auditor holding none of them); a committed key - the node cannot serve is a deterministic failure; the real-byte spot-check catches a node that fabricated - freshness hashes, at the expected probability; deterministic failures are acted on + proof; possession verifies from the verified slices the audited node itself + serves in round 2 (the auditor holding none of the bytes); a committed key + the node cannot serve is a deterministic failure; the slice spot-check catches a + node that fabricated a `nonced_root` or under-stored, at the expected + probability; deterministic failures are acted on the first time while deadline misses honour the grace allowance; the adaptive timeout grace responds to widespread timeouts but never to deterministic failures; - repudiating a recently-published commitment fails; the last two published - commitments stay answerable; the response deadline is sized correctly; and a flood - of gossip does not multiply audits. + repudiating a recently-published commitment fails; recently published + commitments stay answerable within the TTL window; the response deadline is sized + correctly; and a flood of gossip does not multiply audits. - **Operational signals and re-open triggers.** Audits per node per hour stay within budget; false-positive penalties on a small, dense test network stay at zero diff --git a/src/replication/commitment_state.rs b/src/replication/commitment_state.rs index 027edc9a..3d8e4c29 100644 --- a/src/replication/commitment_state.rs +++ b/src/replication/commitment_state.rs @@ -3,8 +3,9 @@ //! Phase 2b of the v12 storage-bound audit design. Builds, signs, and //! caches a [`StorageCommitment`] over the responder's currently-stored //! key set; serves audit lookups by `expected_commitment_hash`; retains -//! the previous commitment across one rotation so an audit pinned to it -//! does not false-fail at the rotation boundary (v5/v12 §4 retention). +//! recently-gossiped commitments for a bounded TTL window (with a slot +//! backstop) so an audit pinned to a still-answerable commitment does not +//! false-fail across rotations. //! //! Rotation strategy: //! @@ -341,7 +342,7 @@ pub(crate) const GOSSIP_ANSWERABILITY_TTL: Duration = Duration::from_secs(3 * 36 /// be slightly early. Adding this margin on reload guarantees an honest node /// never *under*-retains across a restart (it may over-retain by the margin, /// which is harmless — it only makes the responder answer a little longer, and a -/// data-deleter still fails the round-2 byte challenge). Sized well above the +/// data-deleter still fails the round-2 slice challenge). Sized well above the /// persist interval + gossip cadence, far below the TTL. const RESTART_STAMP_GRACE: Duration = Duration::from_secs(5 * 60); @@ -588,11 +589,11 @@ impl ResponderCommitmentState { /// Whether `key` is committed under any retained slot (the current /// commitment plus any still-in-window gossiped ones) — i.e. whether a peer - /// could still pin a recently gossiped root and demand this key's bytes in a - /// round-2 byte challenge. + /// could still pin a recently gossiped root and open this key's blocks in a + /// round-2 slice challenge. /// /// This is the SAME predicate the round-2 responder uses to decide a key is - /// "committed" (`handle_subtree_byte_challenge` calls `built.proof_for(key)` + /// "committed" (`handle_subtree_slice_challenge` calls `built.proof_for(key)` /// on the pinned slot, which is committed iff `contains_key`), folded over /// every retained slot. The pruner consults it before deleting an /// out-of-range key, so "the pruner will not delete it" and "the responder diff --git a/src/replication/config.rs b/src/replication/config.rs index 2ce8f8c3..8eb4665f 100644 --- a/src/replication/config.rs +++ b/src/replication/config.rs @@ -14,7 +14,7 @@ use std::time::Duration; use rand::Rng; -use crate::ant_protocol::{CLOSE_GROUP_SIZE, MAX_CHUNK_SIZE}; +use crate::ant_protocol::CLOSE_GROUP_SIZE; // --------------------------------------------------------------------------- // Static constants (compile-time reference profile) @@ -131,19 +131,21 @@ pub const MAX_CONCURRENT_REPLICATION_SENDS: usize = 3; /// Maximum number of concurrent in-flight audit-responder tasks. /// -/// The responsible-chunk (audit #2), subtree (round 1), and byte (round 2) -/// challenge handlers are all spawned off the serial replication message loop so -/// their disk reads don't stall replication. This caps how many run at once +/// The LIGHT audit-responder handlers — responsible-chunk audits and subtree +/// slice (round 2) — are spawned off the serial replication message loop so their +/// disk reads don't stall replication. (The HEAVY subtree round 1 has its own +/// tighter pool, [`MAX_CONCURRENT_SUBTREE_ROUND1`].) This caps how many run at once /// across the engine, restoring backpressure: a peer flooding audit challenges -/// cannot fan out unbounded `get_raw` reads or multi-MiB byte serves. When the -/// cap is hit, the challenge is dropped and the caller's audit-specific timeout -/// policy applies. The cap must therefore stay high enough for honest audit -/// traffic while still throttling flooders. +/// cannot fan out unbounded `get_raw` reads. When the cap is hit, the challenge +/// is dropped and the caller's audit-specific timeout policy applies. The cap +/// must therefore stay high enough for honest audit traffic while still +/// throttling flooders. /// Sized to cover a handful of concurrent honest auditors (the per-peer /// gossip-audit cooldown is 30 min, so genuine concurrent audits are few) while -/// bounding the byte round's worst-case resident bytes -/// (`N × MAX_BYTE_CHALLENGE_KEYS × MAX_CHUNK_SIZE`). -pub const MAX_CONCURRENT_AUDIT_RESPONSES: usize = 32; +/// bounding the round-2 worst-case disk reads (each request reads at most +/// `BYTE_SPOTCHECK_MAX` distinct chunks — the openings are coalesced and the +/// distinct-key count is capped — to build its slice proofs). +pub const MAX_CONCURRENT_AUDIT_RESPONSES: usize = 16; /// Maximum concurrent in-flight audit-responder tasks from any SINGLE peer. /// @@ -157,6 +159,43 @@ pub const MAX_CONCURRENT_AUDIT_RESPONSES: usize = 32; /// headroom beyond the legitimate round-1 + round-2 overlap. pub const MAX_AUDIT_RESPONSES_PER_PEER: u32 = 4; +/// Dedicated global concurrency cap for the HEAVY subtree-audit round 1. +/// +/// Round 1 hashes every leaf of the selected `sqrt(key_count)` subtree (up to +/// ~1000 chunks × `MAX_CHUNK_SIZE` for a maximal commitment), far heavier than a +/// responsible-chunk or slice (round-2) response. Giving it its own tiny pool — +/// rather than sharing [`MAX_CONCURRENT_AUDIT_RESPONSES`] — keeps a burst of +/// round-1 proofs from starving the light audits, and bounds concurrent +/// multi-gigabyte hashing to this many at once. Two allows overlap without +/// admitting many simultaneous full-subtree hashes; there is little benefit in +/// more concurrent large LMDB scans against one disk. +pub const MAX_CONCURRENT_SUBTREE_ROUND1: usize = 2; + +/// Per-peer concurrency cap for the heavy subtree-audit round 1. One in-flight +/// round-1 proof per source at a time (an honest auditor never needs more). +pub const MAX_SUBTREE_ROUND1_PER_PEER: u32 = 1; + +/// Per-peer responder-side cooldown between heavy subtree round-1 proofs. +/// +/// An honest auditor already self-limits to one gossip-triggered subtree audit +/// per peer per 30 min, so matching that as a responder-side floor costs honest +/// traffic nothing while bounding the sustained round-1 work a single identity +/// can extract (a concurrency cap alone lets a peer refill its slot forever). +pub const SUBTREE_ROUND1_RESPONDER_COOLDOWN: Duration = Duration::from_secs(30 * 60); + +/// Lifetime of a single-use round-1 → round-2 session. +/// +/// A round-2 slice challenge is only served if the same peer completed a matching +/// round 1 within this window. Long enough for the auditor to verify round 1 and +/// send round 2, far shorter than commitment retention; ephemeral, so a loss +/// across a restart just +/// drops that round to the (graced) timeout lane. +pub const SUBTREE_SESSION_TTL: Duration = Duration::from_secs(2 * 60); + +/// Capacity backstop on the live round-1 session map (bounds memory if many +/// peers open sessions; oldest are evicted past this). +pub const MAX_SUBTREE_SESSIONS: usize = 4 * MAX_CONCURRENT_SUBTREE_ROUND1 * 256; + /// Concurrent fetches cap, derived from hardware thread count. /// /// Uses `std::thread::available_parallelism()` so the node scales to the @@ -191,18 +230,15 @@ pub const AUDIT_TICK_INTERVAL_MAX: Duration = Duration::from_secs(AUDIT_TICK_INT /// for the round-1 proof, whose payload is hashes (KB-scale). const AUDIT_RESPONSE_FLOOR_SECS: u64 = 4; -/// Floor on the round-2 BYTE-challenge deadline. +/// Floor on the round-2 SLICE-challenge deadline. /// -/// Unlike round 1 (KB of hashes), the byte challenge ships up to -/// `MAX_BYTE_CHALLENGE_KEYS` full chunks (2 × 4 MiB = 8 MiB) back over the -/// wire, so the envelope must also cover a cold QUIC handshake, the -/// multi-MiB upload back to the auditor, and a busy honest peer's disk read. -/// The round-1 4 s floor is still sized for a hashes-only reply; round 2 needs -/// a larger base for the §4 byte-serving envelope. 5 s matches the -/// cross-continent-RTT + handshake + 8 MiB transfer budget while keeping a relay -/// that must fetch the bytes over a residential link outside it (the scaled -/// term adds the per-byte estimate on top). Mirrors main's more generous -/// byte-round base. +/// The round-2 reply is only a few KB per opening (a 1 KiB block plus two short +/// hash chains), but an honest responder still reads each opened chunk's full +/// bytes from disk to build its Bao slice and nonced opening, so the floor must +/// cover a cold QUIC handshake plus a busy honest peer's full-chunk disk read. +/// The round-1 4 s floor is sized for a hashes-only reply; round 2 keeps a +/// slightly larger 5 s base for the disk-read envelope, with the per-byte scaled +/// term (one full chunk per opening) added on top. const BYTE_AUDIT_RESPONSE_FLOOR_SECS: u64 = 5; /// Conservative honest-responder read throughput, in bytes per second. @@ -245,53 +281,69 @@ const PRUNE_HYSTERESIS_DURATION_SECS: u64 = 3 * 24 * 60 * 60; // 3 days /// Minimum continuous out-of-range duration before pruning a key. pub const PRUNE_HYSTERESIS_DURATION: Duration = Duration::from_secs(PRUNE_HYSTERESIS_DURATION_SECS); -/// Protocol identifier for replication operations. +/// Protocol identifier for core replication operations (fresh replication, +/// neighbour sync, verification, fetch, repair, periodic possession/prune +/// audits, commitment fetch). /// -/// Bumped to `v2` for the v12 storage-bound audit. That change extends the -/// wire types (`NeighborSyncRequest`/`Response` carry an optional trailing -/// `StorageCommitment`, and the gossip-triggered storage-commitment audit adds -/// the `SubtreeAuditChallenge`/`SubtreeAuditResponse` and `SubtreeByteChallenge`/ -/// `SubtreeByteResponse` messages). The bump is for SEMANTIC interop, not -/// decode failure: postcard tolerates the appended optional field (an old -/// decoder reads the fields it knows and ignores the trailer — pinned by the -/// `old_decoder_tolerates_new_neighbor_sync_*` tests in `protocol.rs`), but -/// tolerating bytes is not interoperating. A v1 node cannot decode the NEW -/// message variants at all (unknown enum discriminant) and never acts on a -/// piggybacked commitment, so mixed-version replication would half-function — -/// audit challenges unanswered, commitments silently dropped — and a v2 node -/// could read that silence as misbehaviour. Rather than reason about each -/// such case, we route v12 replication on a distinct protocol id: a node only -/// delivers messages whose topic matches its own id (see the topic check in -/// `mod.rs`), so v1 and v2 nodes simply do not exchange replication traffic -/// during a mixed-version window. This is the rollout-safe behaviour: no -/// half-interpreted exchange, no spurious eviction. Replication between -/// matched-version peers is unaffected. (DHT routing/lookups are a separate -/// protocol and continue to span both versions.) +/// Kept at `v2`: none of these messages changed on the wire in the V2-685 slice +/// audit, so v2 and v3 nodes interoperate on all of them. Only the *subtree* +/// audit changed its wire format, and it rides a separate id +/// ([`SUBTREE_AUDIT_PROTOCOL_ID`]) so a version bump there cannot partition core +/// replication. A node filters inbound messages by exact topic match (see the +/// dispatch in `mod.rs`). pub const REPLICATION_PROTOCOL_ID: &str = "autonomi.ant.replication.v2"; +/// Protocol identifier for the subtree storage-commitment audit (ADR-0002 / +/// V2-685), both rounds: `SubtreeAuditChallenge`/`Response` (round 1) and +/// `SubtreeSliceChallenge`/`Response` (round 2). +/// +/// These are the only replication messages whose wire format changed for the +/// slice audit (round 1's `SubtreeLeaf` now carries `content_len` + `nonced_root` +/// instead of a flat `nonced_hash`; round 2 replaced full-byte responses with Bao +/// verified slices). Routing them on their own id — instead of bumping the whole +/// [`REPLICATION_PROTOCOL_ID`] — means a mixed-version fleet keeps doing fresh +/// replication, neighbour sync, fetch and repair across versions; only +/// cross-version subtree *audits* pause during the ~24 h auto-upgrade window. +/// +/// Rollout effect is bounded, not zero: `saorsa-core`'s `send_request` records a +/// unit trust failure on any unanswered request (before ant-node's graced-timeout +/// policy), so a v3 auditor's subtree challenge to a still-v2 peer (and the +/// reverse, where a v2 subtree audit is dropped by the id/body guard) each ding +/// that peer's EMA trust once per 30-minute audit cooldown. Trust decays back to +/// neutral (a worst-case dip recovers above the 0.35 routing-swap threshold in +/// ~1 online day, and a successful audit after upgrade adds a unit success), and +/// crossing 0.35 only makes a peer replaceable in a full bucket — it does not +/// delete data or ban the peer. This is strictly milder than bumping the shared +/// id, which would fail every cross-version request path (sync/quorum/prune/ +/// possession/repair/commitment-fetch) with no per-peer limiter. A truly +/// zero-penalty rollout needs an upstream `send_request` that does not +/// auto-report trust; tracked as a saorsa-core follow-up. +pub const SUBTREE_AUDIT_PROTOCOL_ID: &str = "autonomi.ant.replication.subtree-audit.v1"; + /// 10 MiB — maximum replication wire message size (accommodates hint batches). const REPLICATION_MESSAGE_SIZE_MIB: usize = 10; /// Maximum replication wire message size. pub const MAX_REPLICATION_MESSAGE_SIZE: usize = REPLICATION_MESSAGE_SIZE_MIB * 1024 * 1024; -/// Headroom reserved for the envelope (enum tags, ids, length prefixes) when -/// sizing a round-2 byte-challenge batch against the wire cap. -const BYTE_CHALLENGE_RESPONSE_HEADROOM: usize = 64 * 1024; - -/// Maximum keys per round-2 [`SubtreeByteChallenge`] (per-batch cap). +/// Maximum block openings per round-2 [`SubtreeSliceChallenge`]. /// -/// Sized so the WORST-CASE response (every requested chunk at -/// `MAX_CHUNK_SIZE`) still encodes under [`MAX_REPLICATION_MESSAGE_SIZE`]. -/// The auditor splits its spot-check sample into batches of this size (one -/// challenge per batch, same nonce/pin); the responder rejects any single -/// challenge requesting more. +/// Each opening is a Bao verified slice (a 1 KiB block plus O(log n) BLAKE3 +/// parent hashes) plus a nonced block-tree sibling chain — a few KB, so even +/// this many openings encode far under [`MAX_REPLICATION_MESSAGE_SIZE`] with no +/// batching. The auditor draws up to *two* openings per sampled leaf — one +/// fresh-random block (possession) and one at the claimed final block (a length +/// pin: opening the final block forces Bao's EOF validation, which authenticates +/// the true content length and defeats a forged-short `content_len` that would +/// otherwise shrink the challenge space). With at most `BYTE_SPOTCHECK_MAX` +/// leaves that is `2 × BYTE_SPOTCHECK_MAX` openings; this cap sits just above +/// that, and the responder rejects any challenge requesting more (a forged- +/// auditor guard: each opening forces one full chunk read to build its proof). /// -/// [`SubtreeByteChallenge`]: crate::replication::protocol::SubtreeByteChallenge -pub const MAX_BYTE_CHALLENGE_KEYS: usize = - (MAX_REPLICATION_MESSAGE_SIZE - BYTE_CHALLENGE_RESPONSE_HEADROOM) / MAX_CHUNK_SIZE; +/// [`SubtreeSliceChallenge`]: crate::replication::protocol::SubtreeSliceChallenge +pub const MAX_SLICE_OPENINGS: usize = 10; const _: () = assert!( - MAX_BYTE_CHALLENGE_KEYS >= 1, - "wire cap must fit at least one max-size chunk per byte-challenge response" + MAX_SLICE_OPENINGS >= 1, + "at least one block opening must be allowed per slice challenge" ); /// Rollout gate for ADR-0004 quote-arithmetic enforcement. @@ -571,6 +623,10 @@ pub struct ReplicationConfig { /// Upper bound of the possession-check delay window (ADR-0003). Defaults /// to [`POSSESSION_CHECK_DELAY_MAX`]. pub possession_check_delay_max: Duration, + /// Per-peer responder-side cooldown between heavy subtree round-1 proofs. + /// Defaults to [`SUBTREE_ROUND1_RESPONDER_COOLDOWN`]; tests set + /// it low so rapid back-to-back audits of one holder are not rate-dropped. + pub subtree_round1_responder_cooldown: Duration, } impl Default for ReplicationConfig { @@ -598,6 +654,7 @@ impl Default for ReplicationConfig { bootstrap_complete_timeout_secs: BOOTSTRAP_COMPLETE_TIMEOUT_SECS, possession_check_delay_min: POSSESSION_CHECK_DELAY_MIN, possession_check_delay_max: POSSESSION_CHECK_DELAY_MAX, + subtree_round1_responder_cooldown: SUBTREE_ROUND1_RESPONDER_COOLDOWN, } } } @@ -777,12 +834,11 @@ impl ReplicationConfig { // an honest HDD-backed peer at sqrt(N)=10 stored chunks could // miss the budget under load. let multiplied = total_bytes.saturating_mul(self.audit_response_honest_multiplier); - // Resolve the scaled term in MILLISECONDS, not seconds: at the - // byte-round sizes (MAX_BYTE_CHALLENGE_KEYS = 2 → 8 MiB) the per-second - // quotient `multiplied / bps` integer-truncates to 0, leaving only the - // floor. The §4 finding was that byte-serving challenges need the - // sub-second honest-read estimate (e.g. 8 MiB × 5 / 50 MB/s ≈ 840 ms) - // instead of dropping it. + // Resolve the scaled term in MILLISECONDS, not seconds: at small + // sample sizes (e.g. a 2-key challenge → 8 MiB) the per-second quotient + // `multiplied / bps` integer-truncates to 0, leaving only the floor. + // Small challenges still need the sub-second honest-read estimate + // (e.g. 8 MiB × 5 / 50 MB/s ≈ 840 ms) instead of dropping it. let scaled_ms = multiplied.saturating_mul(1000) / bps; // saturating_add avoids a panic if the floor plus the scaled term would // overflow `Duration::MAX`. @@ -790,20 +846,26 @@ impl ReplicationConfig { .saturating_add(Duration::from_millis(scaled_ms)) } - /// Deadline for the round-2 BYTE challenge serving `challenged_key_count` - /// full chunks back to the auditor. + /// Deadline for the round-2 SLICE challenge opening `openings` blocks. /// - /// Same per-byte scaling as [`Self::audit_response_timeout`] (so a relay - /// that must fetch the bytes over a residential link still blows it), but on - /// a higher floor (`BYTE_AUDIT_RESPONSE_FLOOR_SECS`) because the reply - /// carries up to - /// `MAX_BYTE_CHALLENGE_KEYS × MAX_CHUNK_SIZE` of chunk data — handshake + - /// multi-MiB upload + a busy honest disk read do not fit the hashes-only - /// round-1 floor (the §4 finding). + /// The reply itself is only a few KB per opening (a 1 KiB block plus two + /// short hash chains), but an honest responder still reads each opened + /// chunk's full bytes from disk to build its Bao slice and nonced opening. So + /// the deadline is sized to that honest full-chunk disk read (the same + /// per-byte scaling as [`Self::audit_response_timeout`], one full chunk per + /// opening), on the `BYTE_AUDIT_RESPONSE_FLOOR_SECS` floor to absorb the + /// handshake and a busy disk. + /// + /// Unlike the old full-byte round 2, security here does NOT rest on this + /// deadline being too tight for a relay to fetch bytes — the possession + /// guarantee is the round-1 `nonced_root` commitment (uncomputable without + /// all the bytes, under a fresh nonce), so the deadline can be generous + /// without weakening the audit. It exists only to bound how long the auditor + /// waits for an honest reply. #[must_use] - pub fn byte_audit_response_timeout(&self, challenged_key_count: usize) -> Duration { + pub fn slice_audit_response_timeout(&self, openings: usize) -> Duration { let scaled = self - .audit_response_timeout(challenged_key_count) + .audit_response_timeout(openings) .saturating_sub(self.audit_response_floor); Duration::from_secs(BYTE_AUDIT_RESPONSE_FLOOR_SECS).saturating_add(scaled) } @@ -897,13 +959,18 @@ mod tests { } #[test] - fn replication_protocol_id_is_v2() { - // The v12 storage-bound audit changes replication SEMANTICS. The - // protocol id MUST advance past v1 so v1 and v2 nodes never exchange - // replication traffic they can only half-interpret (rollout safety — - // see the const's doc). If this regresses to v1, mixed-version nodes - // would talk past each other and risk spurious penalties. + fn core_replication_id_stays_v2_audit_rides_own_id() { + // The V2-685 slice audit changes ONLY the subtree-audit wire format, so + // core replication stays on v2 (mixed-version fleets keep replicating), + // and the changed subtree audit rides its own id that can be bumped + // independently. Two distinct ids so a v2 subtree audit never lands on the + // core-id handler (and vice versa) — see the id/body guard in `mod.rs`. assert_eq!(REPLICATION_PROTOCOL_ID, "autonomi.ant.replication.v2"); + assert_eq!( + SUBTREE_AUDIT_PROTOCOL_ID, + "autonomi.ant.replication.subtree-audit.v1" + ); + assert_ne!(REPLICATION_PROTOCOL_ID, SUBTREE_AUDIT_PROTOCOL_ID); } #[test] diff --git a/src/replication/mod.rs b/src/replication/mod.rs index 98a44dc3..1bb1a4a4 100644 --- a/src/replication/mod.rs +++ b/src/replication/mod.rs @@ -29,6 +29,7 @@ pub mod pruning; pub mod quorum; pub mod recent_provers; pub mod scheduling; +pub mod slice; pub mod storage_commitment_audit; pub mod subtree; pub mod types; @@ -62,7 +63,9 @@ use crate::replication::commitment_state::{ }; use crate::replication::config::{ max_parallel_fetch, storage_admission_width, ReplicationConfig, MAX_AUDIT_RESPONSES_PER_PEER, - MAX_CONCURRENT_AUDIT_RESPONSES, MAX_CONCURRENT_REPLICATION_SENDS, REPLICATION_PROTOCOL_ID, + MAX_CONCURRENT_AUDIT_RESPONSES, MAX_CONCURRENT_REPLICATION_SENDS, + MAX_CONCURRENT_SUBTREE_ROUND1, MAX_SUBTREE_ROUND1_PER_PEER, MAX_SUBTREE_SESSIONS, + REPLICATION_PROTOCOL_ID, SUBTREE_AUDIT_PROTOCOL_ID, SUBTREE_SESSION_TTL, }; use crate::replication::paid_list::PaidList; use crate::replication::protocol::{ @@ -856,6 +859,38 @@ impl FirstAuditScheduler { /// Prefix used by saorsa-core's request-response mechanism. const RR_PREFIX: &str = "/rr/"; +/// Match an inbound topic against the replication protocol ids, in both the bare +/// gossip form and the `/rr/` request-response form. +/// +/// Returns the matched id (core [`REPLICATION_PROTOCOL_ID`] or +/// [`SUBTREE_AUDIT_PROTOCOL_ID`]) and whether it was the RR form. The matched id +/// is carried into the handler so it can enforce that subtree-audit bodies only +/// arrive on the audit id and core bodies only on the core id. +fn match_replication_protocol(topic: &str) -> Option<(&'static str, bool)> { + for id in [REPLICATION_PROTOCOL_ID, SUBTREE_AUDIT_PROTOCOL_ID] { + if topic == id { + return Some((id, false)); + } + if let Some(rest) = topic.strip_prefix(RR_PREFIX) { + if rest == id { + return Some((id, true)); + } + } + } + None +} + +/// Whether a decoded body belongs on the protocol id it arrived on: subtree-audit +/// bodies on [`SUBTREE_AUDIT_PROTOCOL_ID`], every other body on +/// [`REPLICATION_PROTOCOL_ID`]. +/// +/// The receive guard drops any mismatch (a cross-version or misrouted message); +/// sharing this one predicate between the guard and its regression test means a +/// change to the rule cannot pass the test unnoticed. +fn body_matches_protocol(body: &ReplicationMessageBody, protocol: &str) -> bool { + body.is_subtree_audit() == (protocol == SUBTREE_AUDIT_PROTOCOL_ID) +} + fn fresh_offer_payment_context() -> VerificationContext { VerificationContext::FreshReplication } @@ -1109,8 +1144,10 @@ pub struct ReplicationEngine { /// Limits concurrent outbound replication sends to prevent bandwidth /// saturation on home broadband connections. send_semaphore: Arc, - /// Bounds concurrent IN-FLIGHT audit-responder tasks (subtree round 1 + - /// byte round 2). Those are spawned off the serial message loop so disk + /// Bounds concurrent IN-FLIGHT LIGHT audit-responder tasks (responsible-chunk + /// audits + subtree slice round 2). The heavy subtree round 1 has its own + /// tighter pool ([`SubtreeRound1Limiter`]). Those are spawned off the serial + /// message loop so disk /// reads don't block replication; the semaphore restores a global /// backpressure ceiling so the node can't fan out unbounded `get_raw` reads /// / multi-MiB byte serves. @@ -1123,6 +1160,11 @@ pub struct ReplicationEngine { /// per-peer cap guarantees no single source can hold more than its share, /// so a flood self-throttles without denying service to everyone else. audit_responder_inflight: Arc>>, + /// Resource controls for the HEAVY subtree-audit round 1: its own + /// tight admission pool (so a burst of full-subtree hashing can't starve the + /// light audits), a per-peer rate cooldown, and single-use round-1 → round-2 + /// sessions binding a slice challenge to a matching round 1. + subtree_round1: SubtreeRound1Limiter, /// Receiver for fresh-write events from the chunk PUT handler. /// /// When present, `start()` spawns a drainer task that calls @@ -1219,6 +1261,7 @@ impl ReplicationEngine { send_semaphore: Arc::new(Semaphore::new(MAX_CONCURRENT_REPLICATION_SENDS)), audit_responder_semaphore: Arc::new(Semaphore::new(MAX_CONCURRENT_AUDIT_RESPONSES)), audit_responder_inflight: Arc::new(RwLock::new(HashMap::new())), + subtree_round1: SubtreeRound1Limiter::new(config.subtree_round1_responder_cooldown), fresh_write_rx: Some(fresh_write_rx), possession_check_tx, possession_check_rx: Some(possession_check_rx), @@ -1877,6 +1920,7 @@ impl ReplicationEngine { let sync_state = Arc::clone(&self.sync_state); let audit_responder_semaphore = Arc::clone(&self.audit_responder_semaphore); let audit_responder_inflight = Arc::clone(&self.audit_responder_inflight); + let subtree_round1 = self.subtree_round1.clone(); // ADR-0002 gossip-audit trigger: bundled state so an ingested *changed* // commitment can spawn a probabilistic, cooldown-gated subtree audit. @@ -1901,24 +1945,28 @@ impl ReplicationEngine { data, .. } = event { - // Determine if this is a replication message - // and whether it arrived via the /rr/ request-response - // path (which wraps payloads in RequestResponseEnvelope). - let rr_info = if topic == REPLICATION_PROTOCOL_ID { - Some((data.clone(), None)) - } else if topic.starts_with(RR_PREFIX) - && &topic[RR_PREFIX.len()..] == REPLICATION_PROTOCOL_ID - { - P2PNode::parse_request_envelope(&data) - .filter(|(_, is_resp, _)| !is_resp) - .map(|(msg_id, _, payload)| (payload, Some(msg_id))) - } else { - None - }; - if let Some((payload, rr_message_id)) = rr_info { + // Determine which replication protocol this message + // rode (core or subtree-audit) and whether it arrived + // via the /rr/ request-response path (which wraps + // payloads in a RequestResponseEnvelope). + let rr_info = match_replication_protocol(&topic).and_then( + |(matched_id, is_rr)| { + if is_rr { + P2PNode::parse_request_envelope(&data) + .filter(|(_, is_resp, _)| !is_resp) + .map(|(msg_id, _, payload)| { + (matched_id, payload, Some(msg_id)) + }) + } else { + Some((matched_id, data.clone(), None)) + } + }, + ); + if let Some((matched_id, payload, rr_message_id)) = rr_info { match handle_replication_message( &source, &payload, + matched_id, &p2p, &storage, &paid_list, @@ -1937,6 +1985,7 @@ impl ReplicationEngine { &gossip_audit, &audit_responder_semaphore, &audit_responder_inflight, + &subtree_round1, rr_message_id.as_deref(), ).await { Ok(()) => {} @@ -2867,6 +2916,127 @@ impl Drop for AuditResponderGuard { } } +/// A live round-1 → round-2 subtree-audit session: proof of a matching round 1. +struct SubtreeSession { + commitment_hash: [u8; 32], + nonce: [u8; 32], + inserted: Instant, +} + +/// Resource controls for the HEAVY subtree-audit round 1: a tight +/// admission pool separate from the light responsible/slice audits, a per-peer +/// rate cooldown, and single-use round-1 → round-2 sessions so a round-2 slice +/// challenge is only served after a matching round 1. +#[derive(Clone)] +struct SubtreeRound1Limiter { + semaphore: Arc, + inflight: Arc>>, + cooldown: Arc>>, + /// Per-peer minimum spacing between served round-1 proofs (config-driven; + /// [`SUBTREE_ROUND1_RESPONDER_COOLDOWN`] in production, near-zero in tests). + cooldown_interval: Duration, + sessions: Arc>>, +} + +impl SubtreeRound1Limiter { + fn new(cooldown_interval: Duration) -> Self { + Self { + semaphore: Arc::new(Semaphore::new(MAX_CONCURRENT_SUBTREE_ROUND1)), + inflight: Arc::new(RwLock::new(HashMap::new())), + cooldown: Arc::new(RwLock::new(HashMap::new())), + cooldown_interval, + sessions: Arc::new(RwLock::new(HashMap::new())), + } + } + + /// Admit one heavy round-1 proof for `source`: take a concurrency permit + /// FIRST (so a full pool never wastes the peer's cooldown allowance), then + /// enforce the per-peer rate cooldown. `None` drops the challenge (the remote + /// auditor applies its own graced-timeout policy). + async fn admit(&self, source: &PeerId) -> Option { + let guard = admit_audit_responder_with_limits( + &self.semaphore, + &self.inflight, + source, + MAX_CONCURRENT_SUBTREE_ROUND1, + MAX_SUBTREE_ROUND1_PER_PEER, + ) + .await + .ok()?; + let now = Instant::now(); + let mut cooldown = self.cooldown.write().await; + if let Some(&last) = cooldown.get(source) { + if now.duration_since(last) < self.cooldown_interval { + return None; // guard drops here, releasing the permit + slot + } + } + // Evict lapsed entries (their cooldown has expired, so they no longer + // limit) and cap capacity, so peer-id churn can't grow this map unbounded. + cooldown.retain(|_, &mut last| now.duration_since(last) < self.cooldown_interval); + if cooldown.len() >= MAX_SUBTREE_SESSIONS { + if let Some(oldest) = cooldown.iter().min_by_key(|(_, &t)| t).map(|(k, _)| *k) { + cooldown.remove(&oldest); + } + } + cooldown.insert(*source, now); + Some(guard) + } + + /// Record a single-use session once a round-1 proof is built and about to be + /// sent, so the matching round 2 is admitted exactly once. + async fn open_session( + &self, + source: PeerId, + challenge_id: u64, + commitment_hash: [u8; 32], + nonce: [u8; 32], + ) { + let now = Instant::now(); + let mut sessions = self.sessions.write().await; + sessions.retain(|_, e| now.duration_since(e.inserted) < SUBTREE_SESSION_TTL); + if sessions.len() >= MAX_SUBTREE_SESSIONS { + if let Some(oldest) = sessions + .iter() + .min_by_key(|(_, e)| e.inserted) + .map(|(k, _)| *k) + { + sessions.remove(&oldest); + } + } + sessions.insert( + (source, challenge_id), + SubtreeSession { + commitment_hash, + nonce, + inserted: now, + }, + ); + } + + /// Atomically consume the round-2 session for this exchange. `true` iff a + /// live session matching `(source, challenge_id, commitment_hash, nonce)` + /// existed (and is now removed); a miss silently drops round 2 to the graced + /// timeout lane (sessions are ephemeral and can be lost across a restart). + async fn consume_session( + &self, + source: &PeerId, + challenge_id: u64, + commitment_hash: &[u8; 32], + nonce: &[u8; 32], + ) -> bool { + let mut sessions = self.sessions.write().await; + let matches = sessions.get(&(*source, challenge_id)).is_some_and(|e| { + Instant::now().duration_since(e.inserted) < SUBTREE_SESSION_TTL + && &e.commitment_hash == commitment_hash + && &e.nonce == nonce + }); + if matches { + sessions.remove(&(*source, challenge_id)); + } + matches + } +} + /// Try to admit one audit-responder task for `source`: take a global permit AND /// a per-peer slot (both bounded). Returns `Err` with the binding ceiling and /// its decision-time counters (caller drops the challenge, leaving the remote @@ -2880,8 +3050,26 @@ async fn admit_audit_responder( inflight: &Arc>>, source: &PeerId, ) -> std::result::Result { - let global_limit = MAX_CONCURRENT_AUDIT_RESPONSES; - let peer_limit = MAX_AUDIT_RESPONSES_PER_PEER; + admit_audit_responder_with_limits( + semaphore, + inflight, + source, + MAX_CONCURRENT_AUDIT_RESPONSES, + MAX_AUDIT_RESPONSES_PER_PEER, + ) + .await +} + +/// Admission core shared by the light audit pool ([`admit_audit_responder`]) and +/// the tight heavy subtree round-1 pool: take a global permit AND a per-peer slot +/// under the given limits. +async fn admit_audit_responder_with_limits( + semaphore: &Arc, + inflight: &Arc>>, + source: &PeerId, + global_limit: usize, + peer_limit: u32, +) -> std::result::Result { // `available_permits()` is a cheap atomic load; `global_limit - available` // is the best-effort in-flight count at decision time. Not synchronized with // the per-peer lock, so it is a snapshot, not a single atomic view. @@ -2946,6 +3134,7 @@ async fn admit_audit_responder( async fn handle_replication_message( source: &PeerId, data: &[u8], + inbound_protocol: &str, p2p_node: &Arc, storage: &Arc, paid_list: &Arc, @@ -2964,11 +3153,28 @@ async fn handle_replication_message( gossip_audit: &GossipAuditTrigger, audit_responder_semaphore: &Arc, audit_responder_inflight: &Arc>>, + subtree_round1: &SubtreeRound1Limiter, rr_message_id: Option<&str>, ) -> Result<()> { let msg = ReplicationMessage::decode(data) .map_err(|e| Error::Protocol(format!("Failed to decode replication message: {e}")))?; + // Symmetric id/body guard: subtree-audit bodies are valid ONLY on the audit + // id, and core bodies ONLY on the core id. postcard::from_bytes ignores + // trailing bytes, so a mixed-version peer's message could otherwise decode + // into a valid-looking but wrong body (e.g. an old round-1 `Proof` on the + // core id misreading its bytes as the new `content_len`/`nonced_root`). The + // outer enum discriminants are unchanged across versions, so this drop by + // (id, is_subtree_audit) is exact. + if !body_matches_protocol(&msg.body, inbound_protocol) { + debug!( + "Dropping replication body (variant {}) on protocol {inbound_protocol}: \ + wrong id for its family (cross-version or misrouted)", + msg.body.variant_index() + ); + return Ok(()); + } + match msg.body { ReplicationMessageBody::FreshReplicationOffer(ref offer) => { handle_fresh_offer( @@ -3132,32 +3338,28 @@ async fn handle_replication_message( "Audit challenge received: kind=subtree source={source} request_response={}", rr_message_id.is_some(), ); - let guard = match admit_audit_responder( - audit_responder_semaphore, - audit_responder_inflight, - source, - ) - .await - { - Ok(guard) => guard, - Err(failure) => { - protocol::record_audit_drop(protocol::AuditDropKind::Subtree); - warn!( - "Audit challenge reply not sent: kind=subtree response=dropped \ - source={source} {failure}" - ); - return Ok(()); - } + // Round 1 is the HEAVY path (rebuilds + hashes the whole sqrt-subtree), + // so it uses its own tight admission pool + per-peer rate cooldown, + // separate from the light responsible/slice audits, and a miss silently + // drops (subtree auditors grace timeouts). + let Some(guard) = subtree_round1.admit(source).await else { + protocol::record_audit_drop(protocol::AuditDropKind::Subtree); + warn!( + "Audit challenge reply not sent: kind=subtree response=dropped \ + source={source} (heavy round-1 pool full or per-peer cooldown)" + ); + return Ok(()); }; let bootstrapping = *is_bootstrapping.read().await; let storage = Arc::clone(storage); let p2p_node = Arc::clone(p2p_node); let my_commitment_state = Arc::clone(my_commitment_state); + let subtree_round1 = subtree_round1.clone(); let source = *source; let request_id = msg.request_id; let rr_message_id = rr_message_id.map(ToOwned::to_owned); tokio::spawn(async move { - let _guard = guard; // global permit + per-peer slot, held until done + let _guard = guard; // heavy permit + per-peer slot, held until done let response = storage_commitment_audit::handle_subtree_challenge( &challenge, &storage, @@ -3166,6 +3368,22 @@ async fn handle_replication_message( Some(&my_commitment_state), ) .await; + // A round-1 proof authorizes exactly one matching round 2: open a + // single-use session so a slice challenge cannot be served without + // a live round-1 exchange. + if matches!( + &response, + crate::replication::protocol::SubtreeAuditResponse::Proof { .. } + ) { + subtree_round1 + .open_session( + source, + challenge.challenge_id, + challenge.expected_commitment_hash, + challenge.nonce, + ) + .await; + } let response_kind = subtree_audit_response_kind(&response); let sent = send_replication_response_checked( &source, @@ -3191,16 +3409,59 @@ async fn handle_replication_message( }); Ok(()) } - ReplicationMessageBody::SubtreeByteChallenge(challenge) => { - // Round 2 of the storage audit (ADR-0002): serve the original bytes - // for the auditor's spot-check keys, or signal `Absent` for a - // committed key we can no longer produce. Reads chunk bytes from - // disk, so likewise spawned off the serial loop (§5) under the same - // flood-fair admission (codex#1 + codex-r2 A). + ReplicationMessageBody::SubtreeSliceChallenge(challenge) => { + // Round 2 of the storage audit (ADR-0002 / V2-685): open one 1 KiB + // block of each of the auditor's spot-check keys with a Bao verified + // slice + nonced block-tree opening, or signal `Absent` for a + // committed key we can no longer produce. Reads chunk bytes from disk + // to build the proofs, so likewise spawned off the serial loop + // under the same flood-fair admission (a global ceiling plus a + // per-peer cap). info!( - "Audit challenge received: kind=byte source={source} request_response={}", + "Audit challenge received: kind=slice source={source} request_response={}", rr_message_id.is_some(), ); + // Round 2 must follow a live round-1 exchange from THIS peer: consume + // the single-use session (bound to challenge_id/commitment/nonce). On a + // miss, reply with a cheap `Transient` rejection rather than dropping + // silently. Sessions are ephemeral (an honest responder that restarts + // between rounds loses its session), and an unanswered `send_request` + // would make saorsa-core record a transport trust failure against that + // honest responder — an ongoing effect, not just a rollout-window one. + // A `Transient` reply routes the auditor to the graced timeout lane + // (no trust penalty; the responder re-earns pinned credit on the next + // audit) and does no chunk work, so it is not a DoS lever. + if !subtree_round1 + .consume_session( + source, + challenge.challenge_id, + &challenge.expected_commitment_hash, + &challenge.nonce, + ) + .await + { + protocol::record_audit_drop(protocol::AuditDropKind::Byte); + debug!( + "Slice challenge without a live round-1 session source={source} \ + challenge_id={} → Transient reject", + challenge.challenge_id + ); + send_replication_response_checked( + source, + p2p_node, + msg.request_id, + ReplicationMessageBody::SubtreeSliceResponse( + protocol::SubtreeSliceResponse::Rejected { + challenge_id: challenge.challenge_id, + kind: protocol::RejectKind::Transient, + reason: "no live round-1 session".to_string(), + }, + ), + rr_message_id, + ) + .await; + return Ok(()); + } let guard = match admit_audit_responder( audit_responder_semaphore, audit_responder_inflight, @@ -3227,7 +3488,7 @@ async fn handle_replication_message( let rr_message_id = rr_message_id.map(ToOwned::to_owned); tokio::spawn(async move { let _guard = guard; // global permit + per-peer slot, held until done - let response = storage_commitment_audit::handle_subtree_byte_challenge( + let response = storage_commitment_audit::handle_subtree_slice_challenge( &challenge, &storage, p2p_node.peer_id(), @@ -3235,24 +3496,24 @@ async fn handle_replication_message( Some(&my_commitment_state), ) .await; - let response_kind = subtree_byte_response_kind(&response); + let response_kind = subtree_slice_response_kind(&response); let sent = send_replication_response_checked( &source, &p2p_node, request_id, - ReplicationMessageBody::SubtreeByteResponse(response), + ReplicationMessageBody::SubtreeSliceResponse(response), rr_message_id.as_deref(), ) .await; if sent { info!( - "Audit challenge reply sent: kind=byte response={response_kind} \ + "Audit challenge reply sent: kind=slice response={response_kind} \ source={source} request_response={}", rr_message_id.is_some(), ); } else { warn!( - "Audit challenge reply not sent: kind=byte response={response_kind} \ + "Audit challenge reply not sent: kind=slice response={response_kind} \ source={source} request_response={}", rr_message_id.is_some(), ); @@ -3308,7 +3569,7 @@ async fn handle_replication_message( | ReplicationMessageBody::FetchResponse(_) | ReplicationMessageBody::AuditResponse(_) | ReplicationMessageBody::SubtreeAuditResponse(_) - | ReplicationMessageBody::SubtreeByteResponse(_) + | ReplicationMessageBody::SubtreeSliceResponse(_) | ReplicationMessageBody::GetCommitmentByPinResponse(_) => Ok(()), } } @@ -3907,11 +4168,11 @@ fn subtree_audit_response_kind(response: &protocol::SubtreeAuditResponse) -> &'s } } -fn subtree_byte_response_kind(response: &protocol::SubtreeByteResponse) -> &'static str { +fn subtree_slice_response_kind(response: &protocol::SubtreeSliceResponse) -> &'static str { match response { - protocol::SubtreeByteResponse::Items { .. } => "items", - protocol::SubtreeByteResponse::Bootstrapping { .. } => "bootstrapping", - protocol::SubtreeByteResponse::Rejected { .. } => "rejected", + protocol::SubtreeSliceResponse::Items { .. } => "items", + protocol::SubtreeSliceResponse::Bootstrapping { .. } => "bootstrapping", + protocol::SubtreeSliceResponse::Rejected { .. } => "rejected", } } @@ -3960,25 +4221,33 @@ async fn send_replication_response_checked( } }; // V2-684: per-peer served-bytes attribution for the heavy serve paths. - // `FetchResponse` + `SubtreeByteResponse` carry ~99% of served bytes; - // `NeighborSyncResponse` is included for completeness. Other response - // variants (verification/audit/commitment) are intentionally excluded. + // `FetchResponse` carries ~99% of served bytes; `NeighborSyncResponse` is + // included for completeness. Other response variants (verification/audit/ + // commitment) are intentionally excluded — the round-2 audit reply is now a + // few-KB verified slice (V2-685), not a full-chunk transfer, so it is light. if matches!( msg.body, - ReplicationMessageBody::FetchResponse(_) - | ReplicationMessageBody::SubtreeByteResponse(_) - | ReplicationMessageBody::NeighborSyncResponse(_) + ReplicationMessageBody::FetchResponse(_) | ReplicationMessageBody::NeighborSyncResponse(_) ) { protocol::record_served(peer, encoded.len()); } + // Reply on the id the request rode: subtree-audit responses go on the audit + // id, everything else on the core id. `is_subtree_audit()` is the same single + // predicate the receive guard uses, so the two can never disagree (an audit + // request only reaches a handler on the audit id, and it only produces an + // audit response). saorsa-core correlates RR responses by (peer, msg_id), not + // by protocol name, so a v3 auditor waiting on the audit id still matches. + let protocol = if msg.body.is_subtree_audit() { + SUBTREE_AUDIT_PROTOCOL_ID + } else { + REPLICATION_PROTOCOL_ID + }; let result = if let Some(msg_id) = rr_message_id { p2p_node - .send_response(peer, REPLICATION_PROTOCOL_ID, msg_id, encoded) + .send_response(peer, protocol, msg_id, encoded) .await } else { - p2p_node - .send_message(peer, REPLICATION_PROTOCOL_ID, encoded, &[]) - .await + p2p_node.send_message(peer, protocol, encoded, &[]).await }; if let Err(e) = result { debug!("Failed to send replication response to {peer}: {e}"); @@ -6081,14 +6350,16 @@ async fn rebuild_and_rotate_commitment( // the Merkle root commits to the SET OF KEYS, not to the bytes. The // commitment therefore binds "which keys I claim to hold"; it does NOT // by itself prove byte possession. Byte possession is enforced by the - // audit-verify path, which recomputes `bytes_hash == BLAKE3(local_bytes)` - // and the per-key digest against the AUDITOR'S OWN local copy of the - // bytes — so a responder that holds the key list but dropped the bytes - // still fails (`missing bytes for committed key` / digest mismatch). - // This is sound ONLY while keys are content addresses. If this module - // is ever reused for non-content-addressed records (`bytes_hash != key`), - // the `(k, k)` shortcut would let a byte-less node forge a valid root and - // MUST be replaced with `(key, BLAKE3(bytes))` computed from real bytes. + // round-2 slice audit: a Bao verified slice decoded against the chunk + // ADDRESS plus a keyed nonced block-tree opening under a fresh per-audit + // nonce, so a responder that holds the key list but dropped the bytes + // cannot answer. This is sound ONLY while keys are content addresses; + // the round-1 verifier enforces `bytes_hash == key` on every audited leaf + // (`evaluate_subtree_structure`), so a non-content-addressed + // `(key, bytes_hash)` leaf is rejected rather than letting a byte-less node + // earn credit for `key`. If this module is ever reused for + // non-content-addressed records, that `(k, k)` shortcut AND the verifier + // gate must be replaced with `(key, BLAKE3(bytes))` computed from real bytes. let entries: Vec<_> = keys.into_iter().take(cap).map(|k| (k, k)).collect(); // No-op-rotation guard: compute just the Merkle root from `entries` @@ -6175,12 +6446,109 @@ mod tests { use std::time::Instant; use std::time::SystemTime; + #[test] + fn match_replication_protocol_accepts_both_ids_bare_and_rr() { + // Core id, bare gossip form and /rr/ request-response form. + assert_eq!( + match_replication_protocol(REPLICATION_PROTOCOL_ID), + Some((REPLICATION_PROTOCOL_ID, false)) + ); + assert_eq!( + match_replication_protocol(&format!("{RR_PREFIX}{REPLICATION_PROTOCOL_ID}")), + Some((REPLICATION_PROTOCOL_ID, true)) + ); + // Subtree-audit id, both forms. + assert_eq!( + match_replication_protocol(SUBTREE_AUDIT_PROTOCOL_ID), + Some((SUBTREE_AUDIT_PROTOCOL_ID, false)) + ); + assert_eq!( + match_replication_protocol(&format!("{RR_PREFIX}{SUBTREE_AUDIT_PROTOCOL_ID}")), + Some((SUBTREE_AUDIT_PROTOCOL_ID, true)) + ); + // Foreign topics (incl. a bare /rr/ and an unrelated protocol) don't match. + assert_eq!(match_replication_protocol("autonomi.ant.dht.v1"), None); + assert_eq!(match_replication_protocol(RR_PREFIX), None); + assert_eq!( + match_replication_protocol("autonomi.ant.replication.v3"), + None + ); + } + + // The receive guard drops a body whose family disagrees with the id it rode: + // subtree-audit bodies only on the audit id, core bodies only on the core id. + // This is what stops a mixed-version peer's message from being honoured on the + // wrong handler after a postcard misdecode. The test drives the SAME + // `body_matches_protocol` the production guard uses, over real bodies, so a + // regression in the rule fails here. + #[test] + fn body_matches_protocol_is_symmetric_over_real_bodies() { + use crate::replication::protocol::{ + FreshReplicationOffer, ReplicationMessageBody, SubtreeSliceChallenge, + }; + // A subtree-audit body (models a v2 SubtreeByteChallenge, which decodes to + // this variant 13 under the new enum) and a core body. + let audit = ReplicationMessageBody::SubtreeSliceChallenge(SubtreeSliceChallenge { + challenge_id: 1, + nonce: [0u8; 32], + challenged_peer_id: [0u8; 32], + expected_commitment_hash: [0u8; 32], + openings: vec![], + }); + let core = ReplicationMessageBody::FreshReplicationOffer(FreshReplicationOffer { + key: [0u8; 32], + data: vec![], + proof_of_payment: vec![], + }); + // Correct routing is kept. + assert!(body_matches_protocol(&audit, SUBTREE_AUDIT_PROTOCOL_ID)); + assert!(body_matches_protocol(&core, REPLICATION_PROTOCOL_ID)); + // Cross-routing is dropped, both directions (a v2 subtree audit landing on + // the core id, and any core body on the audit id). + assert!(!body_matches_protocol(&audit, REPLICATION_PROTOCOL_ID)); + assert!(!body_matches_protocol(&core, SUBTREE_AUDIT_PROTOCOL_ID)); + } + fn test_peer(b: u8) -> PeerId { let mut bytes = [0u8; 32]; bytes[0] = b; PeerId::from_bytes(bytes) } + // The heavy round-1 limiter enforces the per-peer rate cooldown + // and single-use round-1 → round-2 sessions. + #[tokio::test] + async fn subtree_round1_limiter_cooldown_and_single_use_session() { + let limiter = SubtreeRound1Limiter::new(Duration::from_secs(3600)); + let peer = test_peer(1); + + // First round-1 is admitted; drop the guard so concurrency is free again. + let guard = limiter.admit(&peer).await; + assert!(guard.is_some(), "first round-1 admitted"); + drop(guard); + // A second round-1 within the cooldown is dropped even though the heavy + // pool now has a free slot — the rate cooldown, not concurrency, blocks it. + assert!( + limiter.admit(&peer).await.is_none(), + "second round-1 within cooldown is rate-dropped" + ); + // A different peer has its own cooldown. + assert!(limiter.admit(&test_peer(2)).await.is_some()); + + // Session: opened by round 1, consumed exactly once by the matching round 2. + let hash = [7u8; 32]; + let nonce = [9u8; 32]; + limiter.open_session(peer, 42, hash, nonce).await; + // Wrong nonce / commitment does not match. + assert!(!limiter.consume_session(&peer, 42, &hash, &[0u8; 32]).await); + assert!(!limiter.consume_session(&peer, 42, &[0u8; 32], &nonce).await); + // A round 2 with no prior round 1 (wrong challenge_id) misses. + assert!(!limiter.consume_session(&peer, 99, &hash, &nonce).await); + // The matching round 2 consumes it — and only once (single-use). + assert!(limiter.consume_session(&peer, 42, &hash, &nonce).await); + assert!(!limiter.consume_session(&peer, 42, &hash, &nonce).await); + } + fn test_key(b: u8) -> crate::ant_protocol::XorName { let mut k = [0u8; 32]; k[0] = b; diff --git a/src/replication/protocol.rs b/src/replication/protocol.rs index efc2366f..01204438 100644 --- a/src/replication/protocol.rs +++ b/src/replication/protocol.rs @@ -137,10 +137,10 @@ pub enum ReplicationMessageBody { SubtreeAuditChallenge(SubtreeAuditChallenge), /// Response to a contiguous-subtree storage audit challenge (round 1). SubtreeAuditResponse(SubtreeAuditResponse), - /// Surprise byte challenge for the spot-checked leaves (round 2). - SubtreeByteChallenge(SubtreeByteChallenge), - /// Response carrying the requested chunks' original bytes (round 2). - SubtreeByteResponse(SubtreeByteResponse), + /// Surprise slice challenge for the spot-checked leaves (round 2). + SubtreeSliceChallenge(SubtreeSliceChallenge), + /// Response carrying verified slices for the opened blocks (round 2). + SubtreeSliceResponse(SubtreeSliceResponse), // === Commitment fetch by pin (ADR-0004) === // APPENDED at the end so postcard variant discriminants of all the @@ -203,12 +203,32 @@ impl ReplicationMessageBody { Self::AuditResponse(_) => 10, Self::SubtreeAuditChallenge(_) => 11, Self::SubtreeAuditResponse(_) => 12, - Self::SubtreeByteChallenge(_) => 13, - Self::SubtreeByteResponse(_) => 14, + Self::SubtreeSliceChallenge(_) => 13, + Self::SubtreeSliceResponse(_) => 14, Self::GetCommitmentByPin(_) => 15, Self::GetCommitmentByPinResponse(_) => 16, } } + + /// Whether this body is a subtree storage-commitment audit message (both + /// rounds). These ride [`SUBTREE_AUDIT_PROTOCOL_ID`], not the core + /// [`REPLICATION_PROTOCOL_ID`]; the receive dispatch uses this to enforce + /// that audit bodies only arrive on the audit id and core bodies only on the + /// core id, so a mixed-version peer's message can never be honoured on the + /// wrong handler even if postcard misdecodes it into a valid-looking value. + /// + /// [`SUBTREE_AUDIT_PROTOCOL_ID`]: crate::replication::config::SUBTREE_AUDIT_PROTOCOL_ID + /// [`REPLICATION_PROTOCOL_ID`]: crate::replication::config::REPLICATION_PROTOCOL_ID + #[must_use] + pub fn is_subtree_audit(&self) -> bool { + matches!( + self, + Self::SubtreeAuditChallenge(_) + | Self::SubtreeAuditResponse(_) + | Self::SubtreeSliceChallenge(_) + | Self::SubtreeSliceResponse(_) + ) + } } // --------------------------------------------------------------------------- @@ -238,7 +258,7 @@ pub(crate) enum AuditDropKind { Responsible = 0, /// Subtree audit round-1 challenge. Subtree = 1, - /// Subtree audit round-2 byte challenge. + /// Subtree audit round-2 slice challenge. Byte = 2, } @@ -404,9 +424,9 @@ pub(crate) fn log_traffic_summary() { // // Follow-up to V2-623: the per-variant table above says how many bytes each // message type served, but not WHICH peers pulled them. This adds per-peer -// attribution for the heavy serve paths (`FetchResponse`, `SubtreeByteResponse`, -// `NeighborSyncResponse` — ~99% of served bytes), emitted as a top-10-by-bytes -// INFO line on the same cadence and target as `log_traffic_summary`. +// attribution for the heavy serve paths (`FetchResponse`, `NeighborSyncResponse` +// — ~99% of served bytes), emitted as a top-10-by-bytes INFO line on the same +// cadence and target as `log_traffic_summary`. // // Design (mirrors V2-623's process-global-static choice for the same reason — // the serve choke point is a free function with no engine handle): @@ -832,7 +852,8 @@ pub enum AuditResponse { /// [`SubtreeAuditResponse::Proof`] for that selected subtree against the pinned /// commitment, or a [`SubtreeAuditResponse::Rejected`] if it genuinely cannot /// (for a recently gossiped pinned commitment a rejection is a confirmed -/// failure, since the responder retains its last two gossiped commitments). +/// failure, since the responder retains its recently gossiped commitments for a +/// bounded TTL window). #[derive(Debug, Clone, Serialize, Deserialize)] pub struct SubtreeAuditChallenge { /// Unique challenge identifier. @@ -864,11 +885,12 @@ pub enum SubtreeAuditResponse { /// the root from the proof, and requires it to equal the commitment /// root (structure). /// - /// The leaves carry only hashes (`bytes_hash`, `nonced_hash`), so this round + /// The leaves carry only hashes (`bytes_hash`, `nonced_root`), so this round /// proves the tree SHAPE is committed — not that the bytes are still held. /// Real possession is proven in **round 2**: the auditor picks a few of the - /// just-verified leaves and sends a [`SubtreeByteChallenge`] requesting their - /// original chunk bytes FROM the responder (see that type). + /// just-verified leaves and sends a [`SubtreeSliceChallenge`] opening one + /// random 1 KiB block of each against both the chunk address and the round-1 + /// `nonced_root` (see that type). Proof { /// The challenge this response answers. challenge_id: u64, @@ -923,56 +945,79 @@ pub enum RejectKind { /// timeout lane (holder credit revoked, no trust penalty). Transient, /// Any other rejection (wrong target peer, no commitment state, malformed - /// proof plan, oversized byte challenge, …). CONFIRMED failure. + /// proof plan, oversized slice challenge, …). CONFIRMED failure. Protocol, } -/// Round 2 of the storage audit (ADR-0002): the **surprise byte challenge**. +/// A single block the round-2 slice challenge opens: which committed key, and +/// which 1 KiB block within it. +/// +/// The `block_index` is drawn with FRESH auditor randomness after round 1 (never +/// nonce-derived), so the responder cannot have prepared only the opened block. +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +pub struct SubtreeSliceOpening { + /// The committed key whose block is opened. + pub key: XorName, + /// The 1 KiB block index within the chunk, in `0..block_count(content_len)`. + pub block_index: u32, +} + +/// Round 2 of the storage audit (ADR-0002 / V2-685): the **surprise slice +/// challenge**. /// -/// After the auditor has structurally verified a [`SubtreeAuditResponse::Proof`] -/// it picks a small sample of that subtree's just-proven leaves with FRESH -/// randomness (chosen now, after the proof is committed — NOT derived from the -/// round-1 nonce, so the responder could not have predicted it at proof-build -/// time) and asks the responder to return the ORIGINAL chunk bytes for exactly -/// those keys. The auditor then checks each returned chunk against the committed -/// leaf: -/// - `BLAKE3(bytes) == leaf.bytes_hash` (the chunk's content address), AND -/// - `compute_audit_digest(nonce, peer, key, bytes) == leaf.nonced_hash`. +/// After structurally verifying a [`SubtreeAuditResponse::Proof`] the auditor +/// picks a small sample of the just-proven leaves with FRESH randomness (chosen +/// now, after the proof is committed — NOT derived from the round-1 nonce) and, +/// for each, a fresh-random 1 KiB block index. It asks the responder to open +/// exactly those blocks. For each opened block the responder returns a Bao +/// verified slice plus a nonced block-tree opening; the auditor checks, over the +/// same block bytes: +/// - the Bao slice against `leaf.bytes_hash` (the chunk's content address), AND +/// - the nonced opening against `leaf.nonced_root` (round-1 possession commit). /// -/// This makes possession non-delegable to the auditor: the auditor needs to -/// hold NONE of the responder's chunks. A responder that committed to a chunk it -/// no longer holds cannot fabricate bytes that hash to the committed address (a -/// preimage break), so it is caught regardless of who audits it. +/// This makes possession non-delegable *and* cheap to prove: the response is a +/// few KB, not up to two 4 MiB chunks. A responder that did not hold the bytes at +/// round-1 commit time cannot have committed a correct `nonced_root`, and cannot +/// fold an after-the-fact-fetched block to a foreign root without a preimage +/// break — so it is caught regardless of who audits it. #[derive(Debug, Clone, Serialize, Deserialize)] -pub struct SubtreeByteChallenge { +pub struct SubtreeSliceChallenge { /// The same `challenge_id` as the round-1 [`SubtreeAuditChallenge`], so the /// responder/auditor correlate the two rounds. pub challenge_id: u64, - /// The same nonce as round 1 — needed for the freshness (`nonced_hash`) - /// check and to bind these bytes to this audit. + /// The same nonce as round 1 — binds each nonced block opening to this audit. pub nonce: [u8; 32], - /// The challenged peer ID (bound into each leaf's possession hash). + /// The challenged peer ID (bound into every nonced block leaf). pub challenged_peer_id: [u8; 32], /// The pinned commitment hash from round 1, so the responder resolves the - /// SAME tree it just proved and serves bytes only for keys it committed to. + /// SAME tree it just proved and opens blocks only for keys it committed to. pub expected_commitment_hash: [u8; 32], - /// The exact keys whose original bytes the responder must return. These are - /// the auditor's freshly-randomised spot-check sample of the round-1 subtree - /// (chosen after the proof was received; not nonce-derived). - pub keys: Vec, + /// The exact blocks to open: the auditor's freshly-randomised spot-check + /// sample of the round-1 subtree (chosen after the proof was received; not + /// nonce-derived), up to two blocks per sampled leaf (a fresh-random block + /// plus the final block, for the content-length pin). + pub openings: Vec, } -/// One requested chunk in a [`SubtreeByteResponse`]. +/// One opened block in a [`SubtreeSliceResponse`]. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub enum SubtreeByteItem { - /// The responder holds this committed key and returns its original bytes. +pub enum SubtreeSliceItem { + /// The responder holds this committed key and opens the requested block. Present { /// The requested key. key: XorName, - /// The original chunk bytes (the auditor re-hashes to verify). - bytes: Vec, + /// The 1 KiB block index that was opened. + block_index: u32, + /// Bao verified slice: the block bytes plus the BLAKE3 parent hashes that + /// authenticate them against the chunk address. The auditor decodes this + /// to recover the verified block bytes. + bao_slice: Vec, + /// Sibling hashes on the path from this block up to the committed + /// `nonced_root`, bottom-up. The auditor folds the block leaf with these + /// to prove the block was committed under round 1's fresh nonce. + nonced_siblings: Vec<[u8; 32]>, }, - /// The responder committed to this key but cannot serve its bytes. This is a + /// The responder committed to this key but cannot serve its block. This is a /// PROVABLE cheat (it published a commitment over a chunk it does not hold), /// so the auditor counts it as a confirmed failure — NOT a graced timeout. /// Distinguishing this explicit signal from silence is what separates a @@ -983,29 +1028,43 @@ pub enum SubtreeByteItem { }, } -/// Response to a [`SubtreeByteChallenge`] (round 2). One item per requested key, -/// in the requested order. +/// Response to a [`SubtreeSliceChallenge`] (round 2). +/// +/// The contract is **coalesced and order-independent**: the responder groups the +/// requested openings by key (reading and hashing each chunk once), so +/// - a `Present` item is unique per distinct `(key, block_index)`; +/// - an `Absent` item is at most one per key and covers ALL of that key's +/// requested openings (it has no `block_index`); +/// - duplicate requested openings do not multiply work or response items; +/// - item order is unspecified — the auditor matches by identity, not position. +/// +/// The auditor rejects a response whose identities collide (a duplicate +/// `(key, block_index)`, or a key that is both `Present` and `Absent`) as +/// malformed, so first-match ambiguity cannot decide a verdict. /// -/// Sizing rule: a challenge carries at most -/// [`MAX_BYTE_CHALLENGE_KEYS`](super::config::MAX_BYTE_CHALLENGE_KEYS) keys — -/// the auditor batches its sample, the responder rejects larger requests — so -/// the WORST-CASE `Items` response (every chunk at `MAX_CHUNK_SIZE`) always -/// encodes under [`MAX_REPLICATION_MESSAGE_SIZE`]. +/// Each item is a few KB (a 1 KiB block plus O(log n) hashes on two short +/// chains), so even the worst-case sample fits far under +/// [`MAX_REPLICATION_MESSAGE_SIZE`] with no batching — the auditor bounds the +/// sample to [`MAX_SLICE_OPENINGS`](super::config::MAX_SLICE_OPENINGS) and the +/// responder rejects larger requests. #[derive(Debug, Clone, Serialize, Deserialize)] -pub enum SubtreeByteResponse { - /// The responder's per-key answers (bytes or an explicit absent signal). +pub enum SubtreeSliceResponse { + /// The responder's per-opening answers (a verified slice or an absent signal). Items { /// The challenge this response answers. challenge_id: u64, - /// One entry per requested key. - items: Vec, + /// One item per DISTINCT requested opening, coalesced and + /// order-independent (see the type-level contract above): duplicate + /// openings do not multiply items, and one `Absent` covers all of a + /// key's openings. + items: Vec, }, /// Peer is still bootstrapping (should not happen mid-audit, but handled). Bootstrapping { /// The challenge this response answers. challenge_id: u64, }, - /// The responder rejects the byte challenge outright. `kind` drives the + /// The responder rejects the slice challenge outright. `kind` drives the /// auditor's accounting (ADR-0004 A1: grace removed): [`RejectKind::Transient`] /// routes to the timeout lane (no trust penalty, holder credit revoked); every /// other kind is a confirmed failure, like round 1. @@ -1141,33 +1200,92 @@ mod tests { } } - // === Round-2 byte response sizing === + // === Round-2 slice response sizing === #[test] - fn max_batch_worst_case_byte_response_fits_wire_cap() { - // The auditor batches its round-2 sample to MAX_BYTE_CHALLENGE_KEYS per - // challenge precisely so this worst case — every requested chunk at - // MAX_CHUNK_SIZE — still encodes. If this fails, honest responders - // would hit encode errors and fail otherwise valid byte challenges. - let items: Vec = (0..crate::replication::config::MAX_BYTE_CHALLENGE_KEYS) - .map(|i| SubtreeByteItem::Present { + fn max_slice_response_is_tiny_relative_to_wire_cap() { + // A worst-case round-2 slice response is MAX_SLICE_OPENINGS openings, each + // a 1 KiB block plus a Bao proof and a nonced sibling chain. For a 4 MiB + // chunk that is ~4096 BLAKE3 chunks → ~12 parent hashes per chain. We + // overestimate generously (16 KiB slice + 24 sibling hashes per opening) + // and assert it encodes far under the wire cap — the whole point of the + // change is that round 2 is now KB-scale, not up to 8 MiB. + let items: Vec = (0..crate::replication::config::MAX_SLICE_OPENINGS) + .map(|i| SubtreeSliceItem::Present { key: [u8::try_from(i).unwrap_or(u8::MAX); 32], - bytes: vec![0xAB; crate::ant_protocol::MAX_CHUNK_SIZE], + block_index: u32::try_from(i).unwrap_or(u32::MAX), + bao_slice: vec![0xAB; 16 * 1024], + nonced_siblings: vec![[0x5A; 32]; 24], }) .collect(); let msg = ReplicationMessage { request_id: 7, - body: ReplicationMessageBody::SubtreeByteResponse(SubtreeByteResponse::Items { + body: ReplicationMessageBody::SubtreeSliceResponse(SubtreeSliceResponse::Items { challenge_id: 7, items, }), }; let encoded = msg .encode() - .expect("worst-case max-batch byte response must fit the wire cap"); + .expect("worst-case slice response must fit the wire cap"); + // Comfortably under 1 MiB, itself a fraction of the 10 MiB wire cap. + assert!(encoded.len() <= 1024 * 1024); assert!(encoded.len() <= MAX_REPLICATION_MESSAGE_SIZE); } + // `is_subtree_audit()` classifies exactly the four subtree-audit variants + // (both rounds), and NOTHING else — crucially not the periodic possession + // `AuditResponse`, which stays on the core id. The receive guard and the + // response-routing both key off this one predicate, so it must be exact. + #[test] + fn is_subtree_audit_covers_both_rounds_only() { + let z = [0u8; 32]; + let audit = [ + ReplicationMessageBody::SubtreeAuditChallenge(SubtreeAuditChallenge { + challenge_id: 1, + nonce: z, + challenged_peer_id: z, + expected_commitment_hash: z, + }), + ReplicationMessageBody::SubtreeAuditResponse(SubtreeAuditResponse::Bootstrapping { + challenge_id: 1, + }), + ReplicationMessageBody::SubtreeSliceChallenge(SubtreeSliceChallenge { + challenge_id: 1, + nonce: z, + challenged_peer_id: z, + expected_commitment_hash: z, + openings: vec![], + }), + ReplicationMessageBody::SubtreeSliceResponse(SubtreeSliceResponse::Bootstrapping { + challenge_id: 1, + }), + ]; + for body in &audit { + assert!( + body.is_subtree_audit(), + "variant {} must be subtree-audit", + body.variant_index() + ); + } + let core = [ + ReplicationMessageBody::FreshReplicationOffer(FreshReplicationOffer { + key: z, + data: vec![], + proof_of_payment: vec![], + }), + // Periodic possession audit — NOT the subtree audit; stays on core id. + ReplicationMessageBody::AuditResponse(AuditResponse::Bootstrapping { challenge_id: 1 }), + ]; + for body in &core { + assert!( + !body.is_subtree_audit(), + "variant {} must be core, not subtree-audit", + body.variant_index() + ); + } + } + // === Fresh Replication roundtrip === #[test] diff --git a/src/replication/recent_provers.rs b/src/replication/recent_provers.rs index b793c228..f49c99e4 100644 --- a/src/replication/recent_provers.rs +++ b/src/replication/recent_provers.rs @@ -312,6 +312,36 @@ mod tests { assert!(cache.is_credited_holder(&key(1), &peer(2), &hash(0xAB))); } + #[test] + fn forget_peer_drops_credit_across_all_commitments() { + // A confirmed audit failure (e.g. a `ResponsiveBootstrap` on a stale pin + // H1) revokes ALL of the peer's holder credit, INCLUDING credit earned + // under a newer commitment H2: the contradiction is identity-level, not + // scoped to one key set. Another peer's credit is untouched. + let mut cache = RecentProvers::new(); + let now = Instant::now(); + let h1 = hash(0x11); + let h2 = hash(0x22); + cache.record_proof(key(1), peer(1), h1, now); + cache.record_proof(key(2), peer(1), h2, now); + cache.record_proof(key(1), peer(2), h1, now); + + cache.forget_peer(&peer(1)); + + assert!( + !cache.is_credited_holder(&key(1), &peer(1), &h1), + "H1 credit dropped" + ); + assert!( + !cache.is_credited_holder(&key(2), &peer(1), &h2), + "H2 credit dropped too (peer-wide, not pin-scoped)" + ); + assert!( + cache.is_credited_holder(&key(1), &peer(2), &h1), + "another peer's credit is preserved" + ); + } + #[test] fn forget_commitment_drops_only_matching_entries() { let mut cache = RecentProvers::new(); diff --git a/src/replication/slice.rs b/src/replication/slice.rs new file mode 100644 index 00000000..cabe4dfa --- /dev/null +++ b/src/replication/slice.rs @@ -0,0 +1,840 @@ +//! BLAKE3 verified-slice possession proofs for the storage-commitment audit +//! (ADR-0002 round 2, V2-685). +//! +//! The audit's round 2 used to make a challenged peer return the **complete +//! original bytes** of the sampled chunks (up to two 4 MiB chunks per response), +//! which turned proof-of-storage into the fleet's second-largest bandwidth cost +//! (~7.5 TB/day). This module replaces that with a two-chain verified slice: the +//! response for one opened 1 KiB block is a few KB instead of a full chunk, a +//! ~1000× reduction, while proving *more* than the old flat check. +//! +//! ## Why two chains +//! +//! A chunk's address is `BLAKE3(content)` (its content hash), and BLAKE3 is +//! internally a Merkle tree over 1 KiB blocks, so a **Bao verified slice** proves +//! that a given block is the real content at a given offset *against the address +//! the auditor already knows* — content authenticity, for free, no full chunk. +//! But authenticity alone is checkable from purely public data (the address is +//! public), so a node that stores nothing could pass it by fetching the block on +//! demand from an honest holder. To bind **possession at commitment time** the +//! responder also commits, per leaf in round 1, a fresh **nonced block tree**: +//! a Merkle root over the same 1 KiB blocks whose leaves are **keyed** BLAKE3 +//! hashes, `keyed_BLAKE3(key=f(nonce ‖ peer ‖ key), block_index ‖ len ‖ block)`. +//! +//! The nonce is fed as the BLAKE3 **key**, not a message prefix, so it mixes +//! into every chunk compression of every block leaf: there is no +//! nonce-independent chaining value a responder can precompute across audits. +//! (A plain `BLAKE3(nonce ‖ … ‖ block)` prefix would leave the block's tail in a +//! second, nonce-free BLAKE3 chunk whose CV is precomputable, letting a node +//! store ~10.7% less than each block and still reconstruct the leaf for any +//! fresh nonce — a smaller version of the old flat-`nonced_hash` gap.) Building +//! the correct `nonced_root` therefore requires *all* of a chunk's bytes at +//! round-1 commit time, and the auditor picks which block to open with fresh +//! randomness *after* the roots are committed (cut-and-choose): a responder +//! cannot connect a real, after-the-fact-fetched block to a garbage committed +//! root without a preimage break, and cannot commit a correct root without +//! holding the bytes. +//! +//! In round 2 the auditor verifies both chains over the **same** block bytes: +//! the Bao chain against the address (authenticity) and the nonced chain against +//! the round-1 `nonced_root` (possession). Either failing is a confirmed cheat. + +use std::io::{Cursor, Read}; + +use crate::ant_protocol::XorName; + +/// Block size for slice audits: one BLAKE3 chunk (1 KiB). A block is the unit +/// both the Bao authenticity proof and the nonced possession tree open on. +/// +/// Matching BLAKE3's internal 1 KiB chunk means a single opened block maps to a +/// single BLAKE3 leaf, so the Bao proof for one block is minimal. +pub const AUDIT_BLOCK_SIZE: u64 = 1024; + +/// Domain tag for a nonced block-tree leaf. Distinct from every other hash in +/// the protocol so a leaf can never be reinterpreted as a node or a commitment. +const DOMAIN_BLOCK_LEAF: &[u8] = b"autonomi.ant.audit.slice.block-leaf.v1"; + +/// Domain tag for a nonced block-tree internal node. +const DOMAIN_BLOCK_NODE: &[u8] = b"autonomi.ant.audit.slice.block-node.v1"; + +/// Domain tag for deriving the per-audit BLAKE3 key of the nonced block tree. +const DOMAIN_BLOCK_KEY: &[u8] = b"autonomi.ant.audit.slice.block-key.v1"; + +/// Per-audit keying material for the nonced block tree, derived from the fresh +/// nonce, the challenged peer and the chunk key. Constant across a chunk's blocks. +/// +/// The nonce is fed in as a BLAKE3 **key**, not a message prefix. This is what +/// forces every BLAKE3 chunk of a block leaf to depend on the nonce: keyed +/// BLAKE3 mixes the key into the initial state of every chunk compression, so +/// there is no nonce-independent chaining value. Prefixing the nonce instead +/// leaves the block's tail in a second, nonce-free BLAKE3 chunk whose chaining +/// value is precomputable (leaf input is `domain ‖ nonce ‖ … ‖ block`, ~1166 +/// bytes, so the last ~142 block bytes fall in chunk 1 with no nonce), letting a +/// node store ~10.7% less than each block and still reconstruct the leaf for any +/// fresh nonce. +#[must_use] +fn nonced_block_key(nonce: &[u8; 32], peer: &[u8; 32], key: &XorName) -> [u8; 32] { + let mut h = blake3::Hasher::new(); + h.update(DOMAIN_BLOCK_KEY); + h.update(nonce); + h.update(peer); + h.update(key); + *h.finalize().as_bytes() +} + +/// Number of 1 KiB blocks covering `content_len` bytes. +/// +/// Always at least 1 (an empty chunk is one empty block) so every committed key +/// opens at least one block, and the block index the auditor draws is always in +/// range. +#[must_use] +pub fn block_count(content_len: u64) -> u32 { + if content_len == 0 { + return 1; + } + u32::try_from(content_len.div_ceil(AUDIT_BLOCK_SIZE)).unwrap_or(u32::MAX) +} + +/// Byte range `[start, end)` of block `index` within `content_len` bytes. +/// +/// The final block may be short; an out-of-range index clamps to an empty range +/// at `content_len` (callers never pass one — the auditor draws indices in +/// `0..block_count`). +#[must_use] +pub fn block_range(content_len: u64, index: u32) -> (u64, u64) { + let start = u64::from(index) + .saturating_mul(AUDIT_BLOCK_SIZE) + .min(content_len); + let end = start.saturating_add(AUDIT_BLOCK_SIZE).min(content_len); + (start, end) +} + +/// Slice a block's bytes out of a full chunk. Returns an empty slice for an +/// out-of-range index (never happens for auditor-drawn indices). +#[must_use] +fn block_bytes(content: &[u8], index: u32) -> &[u8] { + let (start, end) = block_range(content.len() as u64, index); + let start = usize::try_from(start).unwrap_or(usize::MAX); + let end = usize::try_from(end).unwrap_or(usize::MAX); + content.get(start..end).unwrap_or(&[]) +} + +/// Nonced block-leaf hash: keyed by the per-audit key (nonce/peer/key) so every +/// BLAKE3 chunk of the leaf depends on the nonce, binding the block index, block +/// length and block bytes with no nonce-independent state to precompute. +/// +/// See [`nonced_block_key`] for why the nonce is a key, not a prefix. +#[must_use] +fn nonced_block_leaf( + nonce: &[u8; 32], + peer: &[u8; 32], + key: &XorName, + index: u32, + block: &[u8], +) -> [u8; 32] { + let audit_key = nonced_block_key(nonce, peer, key); + let mut h = blake3::Hasher::new_keyed(&audit_key); + h.update(DOMAIN_BLOCK_LEAF); + h.update(&index.to_le_bytes()); + let block_len = u32::try_from(block.len()).unwrap_or(u32::MAX); + h.update(&block_len.to_le_bytes()); + h.update(block); + *h.finalize().as_bytes() +} + +/// Combine two child hashes into a nonced block-tree internal node. +#[must_use] +fn nonced_block_node(left: &[u8; 32], right: &[u8; 32]) -> [u8; 32] { + let mut h = blake3::Hasher::new(); + h.update(DOMAIN_BLOCK_NODE); + h.update(left); + h.update(right); + *h.finalize().as_bytes() +} + +/// Fold one level of a left-packed Merkle tree, self-pairing an unpaired last +/// node (`node(x, x)`) exactly like the commitment tree. +#[must_use] +fn fold_level(level: &[[u8; 32]]) -> Vec<[u8; 32]> { + let mut next = Vec::with_capacity(level.len().div_ceil(2)); + let mut i = 0; + while i < level.len() { + let left = level.get(i).copied().unwrap_or([0u8; 32]); + // Self-pair the last node when the level has an odd length. + let right = level.get(i + 1).copied().unwrap_or(left); + next.push(nonced_block_node(&left, &right)); + i += 2; + } + next +} + +/// The nonced block-tree leaves for a chunk's `content`, in block order. +#[must_use] +fn nonced_leaves( + nonce: &[u8; 32], + peer: &[u8; 32], + key: &XorName, + content: &[u8], +) -> Vec<[u8; 32]> { + let count = block_count(content.len() as u64); + (0..count) + .map(|i| nonced_block_leaf(nonce, peer, key, i, block_bytes(content, i))) + .collect() +} + +/// Compute the nonced block-tree root over a chunk's `content` (responder, round +/// 1). Requires every byte of the chunk, under the fresh nonce. +#[must_use] +pub fn nonced_block_root( + nonce: &[u8; 32], + peer: &[u8; 32], + key: &XorName, + content: &[u8], +) -> [u8; 32] { + let mut level = nonced_leaves(nonce, peer, key, content); + // A single leaf is its own root (matches the commitment tree's convention). + while level.len() > 1 { + level = fold_level(&level); + } + level.first().copied().unwrap_or([0u8; 32]) +} + +/// Sibling hashes on the path from block `index` up to the nonced root, +/// bottom-up (leaf level first). `None` if `index` is out of range for the tree. +/// +/// The verifier folds the recomputed leaf with these siblings using node-index +/// parity, so the sibling ordering is positional, not left/right-tagged. +#[must_use] +pub fn nonced_block_siblings( + nonce: &[u8; 32], + peer: &[u8; 32], + key: &XorName, + content: &[u8], + index: u32, +) -> Option> { + siblings_from_leaves(&nonced_leaves(nonce, peer, key, content), index) +} + +/// Sibling chain for `index` from prebuilt nonced leaves (no per-leaf rehash). +/// +/// Folding 32-byte CVs is cheap; the cost is computing the leaves, so callers +/// that open several blocks of one chunk build the leaves once and fold here per +/// block. `None` if `index` is out of range. +#[must_use] +fn siblings_from_leaves(leaves: &[[u8; 32]], index: u32) -> Option> { + if usize::try_from(index).ok()? >= leaves.len() { + return None; + } + let mut level = leaves.to_vec(); + let mut node_index = index as usize; + let mut siblings = Vec::new(); + while level.len() > 1 { + // Sibling is the other child of this node's parent; the last node of an + // odd level self-pairs, so its sibling is itself. + let sibling_index = node_index ^ 1; + let sibling = level + .get(sibling_index) + .or_else(|| level.get(node_index)) + .copied() + .unwrap_or([0u8; 32]); + siblings.push(sibling); + node_index /= 2; + level = fold_level(&level); + } + Some(siblings) +} + +/// Verify a nonced block opening (auditor, round 2): recompute the block leaf +/// from the served bytes and fold it with `siblings` to the committed +/// `nonced_root`. +/// +/// `block` must be the Bao-verified block bytes for `index`, so this proves the +/// responder committed a nonced root over the *real* content at round-1 time. +#[must_use] +#[allow(clippy::too_many_arguments)] +pub fn verify_nonced_block( + nonce: &[u8; 32], + peer: &[u8; 32], + key: &XorName, + index: u32, + block: &[u8], + siblings: &[[u8; 32]], + nonced_root: &[u8; 32], + block_count: u32, +) -> bool { + // Enforce the canonical tree geometry: the sibling chain MUST be exactly the + // depth of a `block_count`-leaf tree. `nonced_root` is responder-chosen in + // round 1, so without this a partial holder could set `nonced_root` to a + // single block's leaf and pass with zero siblings whenever the fresh draw + // happens to land on that block. Pinning the depth binds the claimed + // left-packed geometry and rejects such degraded proofs. + if siblings.len() != nonced_tree_depth(block_count) { + return false; + } + let mut node_index = index as usize; + let mut cur = nonced_block_leaf(nonce, peer, key, index, block); + for sibling in siblings { + cur = if node_index % 2 == 0 { + nonced_block_node(&cur, sibling) + } else { + nonced_block_node(sibling, &cur) + }; + node_index /= 2; + } + &cur == nonced_root +} + +/// Canonical sibling-chain length for a `block_count`-leaf nonced tree. +/// +/// The number of `div_ceil(2)` folds from the leaf level down to a single root +/// (0 for a single-block chunk). Matches [`nonced_block_siblings`]'s output. +#[must_use] +pub fn nonced_tree_depth(block_count: u32) -> usize { + let mut n = block_count.max(1) as usize; + let mut depth = 0; + while n > 1 { + n = n.div_ceil(2); + depth += 1; + } + depth +} + +/// Extract a Bao verified slice for block `index` of `content` (responder, round +/// 2). The slice carries the block bytes plus the O(log n) BLAKE3 parent hashes +/// that verify it against the chunk address. +/// +/// # Errors +/// +/// Returns the underlying IO error only if the in-memory Bao extraction fails, +/// which cannot happen for a well-formed in-memory chunk; surfaced as a +/// `Result` rather than a panic so the responder degrades to a rejection. +pub fn extract_block_slice(content: &[u8], index: u32) -> std::io::Result> { + // The outboard carries the BLAKE3 tree hashes separately from the content, so + // the extractor reads the real chunk bytes plus just the parent hashes on the + // block's path — no need to materialise a full Bao encoding of the chunk. + let (outboard, _hash) = bao::encode::outboard(content); + extract_block_slice_with_outboard(content, &outboard, index) +} + +/// Extract a Bao verified slice for block `index` reusing a prebuilt `outboard`. +/// +/// Building the outboard hashes the whole chunk, so when several blocks of the +/// same chunk are opened in one challenge (round + final, plus any dedup), the +/// caller builds the outboard once (see [`ChunkOpener`]) and calls this per +/// block instead of re-hashing. Borrows `content` directly (no copy). +/// +/// # Errors +/// +/// Surfaces an in-memory Bao extraction error as a `Result` rather than a panic; +/// it cannot happen for a well-formed in-memory chunk and outboard. +pub fn extract_block_slice_with_outboard( + content: &[u8], + outboard: &[u8], + index: u32, +) -> std::io::Result> { + let (start, end) = block_range(content.len() as u64, index); + let len = end - start; + let mut extractor = bao::encode::SliceExtractor::new_outboard( + Cursor::new(content), + Cursor::new(outboard), + start, + len, + ); + let mut slice = Vec::new(); + extractor.read_to_end(&mut slice)?; + Ok(slice) +} + +/// Reusable per-chunk responder state for building round-2 openings. +/// +/// Building the Bao outboard and the nonced block-tree leaves each hash the full +/// chunk. When one challenge opens several blocks of the same chunk (the normal +/// random + final pair, plus any duplicates), doing that work once and serving +/// every opening from it keeps a multi-opening challenge to a single hashing pass +/// over the bytes instead of one per opening (V2-685 round-2 amplification fix). +pub struct ChunkOpener<'a> { + content: &'a [u8], + outboard: Vec, + leaves: Vec<[u8; 32]>, +} + +impl<'a> ChunkOpener<'a> { + /// Build the chunk's reusable opening state once: its Bao outboard and its + /// nonced block-leaves (two hash passes over the resident chunk). Repeated + /// openings of the same chunk reuse this state instead of rehashing per opening. + #[must_use] + pub fn new(nonce: &[u8; 32], peer: &[u8; 32], key: &XorName, content: &'a [u8]) -> Self { + let (outboard, _hash) = bao::encode::outboard(content); + let leaves = nonced_leaves(nonce, peer, key, content); + Self { + content, + outboard, + leaves, + } + } + + /// Number of 1 KiB blocks in the chunk (always ≥ 1). + #[must_use] + pub fn block_count(&self) -> u32 { + u32::try_from(self.leaves.len()).unwrap_or(u32::MAX) + } + + /// Bao verified slice for `index`, reusing the prebuilt outboard. + /// + /// # Errors + /// Surfaces an in-memory Bao extraction error as a `Result` rather than a panic. + pub fn bao_slice(&self, index: u32) -> std::io::Result> { + extract_block_slice_with_outboard(self.content, &self.outboard, index) + } + + /// Nonced-tree sibling chain for `index`, reusing the prebuilt leaves. + #[must_use] + pub fn nonced_siblings(&self, index: u32) -> Option> { + siblings_from_leaves(&self.leaves, index) + } +} + +/// Size of the Bao slice's leading length header (a little-endian `u64` giving +/// the encoded content's total length, in bytes). This is the first field of +/// every Bao slice; the decoder authenticates it against the tree root, so it +/// cannot be forged to disagree with the `address` the slice verifies against. +const BAO_LENGTH_HEADER_SIZE: usize = 8; + +/// Verify a Bao slice for block `index` against the chunk `address` +/// (`BLAKE3(content)`), returning the verified block bytes (auditor, round 2). +/// +/// `content_len` is the responder's round-1 claim. It is NOT bound by the signed +/// commitment (the Merkle leaf hashes only `key ‖ bytes_hash`), so a malicious +/// responder can claim any length. Left unchecked, a **deflation** lie forges +/// possession: claiming `content_len = 1024` for a 4 MiB chunk collapses +/// `block_count` to 1, so the auditor can only ever open block 0, and a Bao +/// slice for the prefix range `[0, 1024)` verifies fine against the true +/// full-content address — a node storing ~1 KiB passes an audit for the whole +/// chunk. To close this, the slice's own authenticated length header (which the +/// decoder validates against `address`, so it equals the *true* content length) +/// must match the claimed `content_len`. A responder that lies about the length +/// then either fails the header check (claim ≠ true length) or is forced to +/// report the true length — in which case the block index is drawn from the full +/// range and possession of all blocks is required. +#[must_use] +pub fn verify_block_slice( + slice: &[u8], + address: &[u8; 32], + content_len: u64, + index: u32, +) -> Option> { + // Authenticate the claimed length against the slice's own header before + // trusting `content_len` for the block range. The header is the first 8 + // bytes of every Bao slice and is validated against `address` by the decode + // below (a forged header changes the tree shape and fails the root check), + // so requiring it to equal `content_len` forces the claim to be the true + // content length — defeating the deflation forgery described above. + let header = slice.get(..BAO_LENGTH_HEADER_SIZE)?; + let declared_len = u64::from_le_bytes(header.try_into().ok()?); + if declared_len != content_len { + return None; + } + + let (start, end) = block_range(content_len, index); + let len = end - start; + let hash = blake3::Hash::from_bytes(*address); + let mut decoder = bao::decode::SliceDecoder::new(Cursor::new(slice), &hash, start, len); + let mut verified = Vec::new(); + decoder.read_to_end(&mut verified).ok()?; + if verified.len() as u64 != len { + return None; + } + Some(verified) +} + +#[cfg(test)] +#[allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::cast_possible_truncation +)] +mod tests { + use super::*; + + const NONCE: [u8; 32] = [0x11; 32]; + const PEER: [u8; 32] = [0x22; 32]; + const KEY: XorName = [0x33; 32]; + + /// Deterministic pseudo-content of a given length (avoids RNG in tests). + fn content_of(len: usize) -> Vec { + (0..len).map(|i| (i * 31 + 7) as u8).collect() + } + + // -- block geometry ----------------------------------------------------- + + #[test] + fn block_count_is_ceil_with_empty_floor() { + assert_eq!(block_count(0), 1); + assert_eq!(block_count(1), 1); + assert_eq!(block_count(1024), 1); + assert_eq!(block_count(1025), 2); + assert_eq!(block_count(4096), 4); + assert_eq!(block_count(4 * 1024 * 1024), 4096); + assert_eq!(block_count(4 * 1024 * 1024 - 1), 4096); + } + + #[test] + fn block_range_covers_content_without_gaps_or_overlap() { + let len = 1024 * 3 + 500; + let count = block_count(len); + let mut expected_start = 0u64; + for i in 0..count { + let (s, e) = block_range(len, i); + assert_eq!(s, expected_start); + assert!(e <= len); + assert!(e > s || len == 0); + expected_start = e; + } + assert_eq!(expected_start, len); + } + + // -- bao slice == blake3 address --------------------------------------- + + #[test] + fn bao_root_equals_blake3_address_across_lengths() { + // The whole design rests on Bao's root being the chunk's BLAKE3 address. + for len in [0usize, 1, 1023, 1024, 1025, 2048, 4096, 10_000, 1 << 20] { + let content = content_of(len); + let (_outboard, bao_hash) = bao::encode::outboard(&content); + let blake = blake3::hash(&content); + assert_eq!( + bao_hash.as_bytes(), + blake.as_bytes(), + "bao root must equal blake3(content) at len {len}" + ); + } + } + + #[test] + fn slice_roundtrip_verifies_every_block() { + for len in [1usize, 1024, 1025, 4096, 5000, 1 << 16] { + let content = content_of(len); + let address = *blake3::hash(&content).as_bytes(); + let count = block_count(len as u64); + for i in 0..count { + let slice = extract_block_slice(&content, i).expect("extract"); + let verified = + verify_block_slice(&slice, &address, len as u64, i).expect("verify slice"); + let (s, e) = block_range(len as u64, i); + assert_eq!(verified.as_slice(), &content[s as usize..e as usize]); + } + } + } + + // Regression: content_len deflation must NOT forge possession. + // + // `content_len` is not bound by the signed round-1 commitment (the Merkle + // leaf hashes only key+bytes_hash), so a malicious responder can claim + // content_len=1024 for a 1 MiB chunk. That would collapse block_count to 1, + // let the auditor only ever open block 0, and — before the fix — pass a Bao + // slice for the prefix [0,1024) against the true full-content address while + // storing ~1 KiB. verify_block_slice now authenticates the slice's own length + // header against the claim, so the deflation is rejected at chain 1. + #[test] + fn content_len_deflation_is_rejected() { + let full = content_of(1 << 20); // 1 MiB chunk, truly 1024 blocks. + let address = *blake3::hash(&full).as_bytes(); + + // Attacker keeps only the first block's Bao slice (extracted honestly for + // range [0,1024) of the real content), discarding the rest of the 1 MiB. + let stored_slice = extract_block_slice(&full, 0).expect("extract block 0"); + + // Round 1: attacker claims content_len = 1024 (one block). + let claimed_len = 1024u64; + assert_eq!(block_count(claimed_len), 1, "deflated claim = one block"); + + // Round 2: only block 0 could be drawn. Chain 1 must reject because the + // slice's authenticated length header (1 MiB) disagrees with the claim. + assert!( + verify_block_slice(&stored_slice, &address, claimed_len, 0).is_none(), + "deflated content_len must fail the slice length-header check" + ); + + // An honest responder reporting the true length still verifies block 0. + let honest = verify_block_slice(&stored_slice, &address, (1 << 20) as u64, 0); + assert_eq!( + honest.expect("honest full-length slice must verify").len(), + 1024 + ); + } + + /// Rewrite a Bao slice's 8-byte little-endian length header in place. + fn rewrite_slice_len(slice: &mut [u8], forged_len: u64) { + slice[..BAO_LENGTH_HEADER_SIZE].copy_from_slice(&forged_len.to_le_bytes()); + } + + // Regression for the header-rewrite length forgery (the residual attack the + // plain header==content_len check does NOT catch on its own). + // + // Bao does not cryptographically bind the length header for a PREFIX slice + // that never touches EOF: the unopened right subtree is an opaque hash. So an + // attacker can claim `content_len` just past a subtree boundary (e.g. 2 KiB+1 + // for a 4 KiB / 4-block chunk), REWRITE each slice header to that forged + // length, supply the true right-subtree CV as the opaque root sibling, and + // pass any opening in the retained left half — storing only that half. The + // header check passes because header == forged content_len. + // + // The defence is to ALSO open the CLAIMED FINAL block: that decode reaches + // EOF, where Bao authenticates the encoded length against the address, so a + // forged-short length fails. This test shows both halves: a left-half opening + // slips past the header check, but the final-block opening rejects the forgery. + #[test] + fn header_rewrite_length_forgery_is_caught_at_the_final_block() { + // True chunk: 4 blocks. Root = parent(parent(b0,b1), parent(b2,b3)). + let full = content_of(4096); + let address = *blake3::hash(&full).as_bytes(); + + // Forged length: 2049 bytes → block_count 3 (b0, b1 full, b2' one byte). + // The claimed final block is index 2; blocks 0..2 are the retained half. + let forged_len = 2049u64; + assert_eq!(block_count(forged_len), 3); + let forged_final = block_count(forged_len) - 1; + + // Left-half opening (block 0): the attacker rewrites an honest block-0 + // slice's header to the forged length. The retained slice already carries + // the true right-subtree CV as the opaque root sibling, so it verifies + // against the real address AND passes the header==content_len check. + let mut left = extract_block_slice(&full, 0).expect("extract block 0"); + rewrite_slice_len(&mut left, forged_len); + assert!( + verify_block_slice(&left, &address, forged_len, 0).is_some(), + "header rewrite lets a left-half opening slip past the header check — \ + this is exactly why the final block must also be opened" + ); + + // Final-block opening (block 2 under the forged length): no slice the + // attacker can offer verifies, because reaching the forged EOF forces Bao + // to authenticate the length against the true address. The best attempt — + // an honest block-2 slice of the true content with a rewritten header — + // is rejected. + let mut fake_final = extract_block_slice(&full, 2).expect("extract block 2"); + rewrite_slice_len(&mut fake_final, forged_len); + assert!( + verify_block_slice(&fake_final, &address, forged_len, forged_final).is_none(), + "final-block opening under a forged short length must fail (EOF length auth)" + ); + } + + #[test] + fn slice_against_wrong_address_fails() { + let content = content_of(4096); + let mut wrong = *blake3::hash(&content).as_bytes(); + wrong[0] ^= 0x01; + let slice = extract_block_slice(&content, 2).expect("extract"); + assert!(verify_block_slice(&slice, &wrong, 4096, 2).is_none()); + } + + #[test] + fn tampered_slice_bytes_fail_verification() { + let content = content_of(4096); + let address = *blake3::hash(&content).as_bytes(); + let mut slice = extract_block_slice(&content, 1).expect("extract"); + // Corrupt the payload bytes near the end of the slice (a data byte). + if let Some(b) = slice.last_mut() { + *b ^= 0xFF; + } + assert!(verify_block_slice(&slice, &address, 4096, 1).is_none()); + } + + #[test] + fn slice_for_one_block_cannot_serve_a_different_block() { + let content = content_of(4096); + let address = *blake3::hash(&content).as_bytes(); + let slice = extract_block_slice(&content, 0).expect("extract"); + // A slice built for block 0 must not verify as block 2. + assert!(verify_block_slice(&slice, &address, 4096, 2).is_none()); + } + + // -- nonced block tree -------------------------------------------------- + + #[test] + fn nonced_openings_roundtrip_every_block() { + for len in [1usize, 1024, 1025, 3000, 4096, 9001] { + let content = content_of(len); + let root = nonced_block_root(&NONCE, &PEER, &KEY, &content); + let count = block_count(len as u64); + for i in 0..count { + let siblings = + nonced_block_siblings(&NONCE, &PEER, &KEY, &content, i).expect("siblings"); + let block = block_bytes(&content, i); + assert!( + verify_nonced_block(&NONCE, &PEER, &KEY, i, block, &siblings, &root, count), + "len {len} block {i} must verify" + ); + } + } + } + + #[test] + fn nonced_opening_rejects_wrong_block_bytes() { + let content = content_of(4096); + let root = nonced_block_root(&NONCE, &PEER, &KEY, &content); + let siblings = nonced_block_siblings(&NONCE, &PEER, &KEY, &content, 1).expect("siblings"); + let mut wrong = block_bytes(&content, 1).to_vec(); + wrong[0] ^= 0x01; + let bc = block_count(content.len() as u64); + assert!(!verify_nonced_block( + &NONCE, &PEER, &KEY, 1, &wrong, &siblings, &root, bc + )); + } + + #[test] + fn nonced_opening_binds_nonce_peer_and_key() { + let content = content_of(4096); + let root = nonced_block_root(&NONCE, &PEER, &KEY, &content); + let siblings = nonced_block_siblings(&NONCE, &PEER, &KEY, &content, 2).expect("siblings"); + let block = block_bytes(&content, 2); + let bc = block_count(content.len() as u64); + // Correct binding verifies. + assert!(verify_nonced_block( + &NONCE, &PEER, &KEY, 2, block, &siblings, &root, bc + )); + // A different nonce, peer, or key must not verify against the same root. + let other = [0xAB; 32]; + assert!(!verify_nonced_block( + &other, &PEER, &KEY, 2, block, &siblings, &root, bc + )); + assert!(!verify_nonced_block( + &NONCE, &other, &KEY, 2, block, &siblings, &root, bc + )); + assert!(!verify_nonced_block( + &NONCE, &PEER, &other, 2, block, &siblings, &root, bc + )); + } + + // Regression: the nonce must KEY every BLAKE3 chunk of a block + // leaf, not merely prefix the message. A prefixed leaf (`BLAKE3(nonce ‖ … ‖ + // block)`, ~1166 bytes) leaves the block's tail in a second, nonce-free + // BLAKE3 chunk whose chaining value is precomputable, letting a node store + // ~10.7% less than each block and rebuild the leaf for any fresh nonce. Lock + // in the keyed construction and prove it differs from the vulnerable one. + #[test] + fn leaf_is_keyed_by_the_nonce_not_prefixed() { + let block = content_of(1024); + let index = 3u32; + let block_len = block.len() as u32; + let got = nonced_block_leaf(&NONCE, &PEER, &KEY, index, &block); + + // Required construction: keyed BLAKE3, nonce in the key. + let audit_key = nonced_block_key(&NONCE, &PEER, &KEY); + let mut keyed = blake3::Hasher::new_keyed(&audit_key); + keyed.update(DOMAIN_BLOCK_LEAF); + keyed.update(&index.to_le_bytes()); + keyed.update(&block_len.to_le_bytes()); + keyed.update(&block); + assert_eq!( + got, + *keyed.finalize().as_bytes(), + "leaf must be keyed BLAKE3 with the nonce-derived key" + ); + + // The old, vulnerable prefixed construction must NOT match. + let mut prefixed = blake3::Hasher::new(); + prefixed.update(DOMAIN_BLOCK_LEAF); + prefixed.update(&NONCE); + prefixed.update(&PEER); + prefixed.update(&KEY); + prefixed.update(&index.to_le_bytes()); + prefixed.update(&block_len.to_le_bytes()); + prefixed.update(&block); + assert_ne!( + got, + *prefixed.finalize().as_bytes(), + "leaf must not use the precomputable prefixed-nonce construction" + ); + } + + #[test] + fn nonced_root_changes_with_any_block_edit() { + let content = content_of(5000); + let root = nonced_block_root(&NONCE, &PEER, &KEY, &content); + let mut edited = content; + // Flip a byte in the LAST block; the root must change (all blocks covered). + if let Some(b) = edited.last_mut() { + *b ^= 0x01; + } + let root2 = nonced_block_root(&NONCE, &PEER, &KEY, &edited); + assert_ne!(root, root2); + } + + #[test] + fn nonced_siblings_out_of_range_is_none() { + let content = content_of(2048); + let count = block_count(content.len() as u64); + assert!(nonced_block_siblings(&NONCE, &PEER, &KEY, &content, count).is_none()); + } + + // -- the combined possession property ---------------------------------- + + #[test] + fn relay_cannot_open_against_a_foreign_committed_root() { + // A responder that did NOT hold the bytes at round 1 commits a root over + // garbage (or a guess). Even if it later fetches the real block, folding + // the real leaf to that committed root would be a preimage break: model + // this by taking a root from DIFFERENT content and checking that the real + // block + honest siblings never fold to it. + let real = content_of(4096); + let garbage = content_of(4097); // different content => different tree + let foreign_root = nonced_block_root(&NONCE, &PEER, &KEY, &garbage); + let siblings = nonced_block_siblings(&NONCE, &PEER, &KEY, &real, 0).expect("siblings"); + let block = block_bytes(&real, 0); + assert!(!verify_nonced_block( + &NONCE, + &PEER, + &KEY, + 0, + block, + &siblings, + &foreign_root, + block_count(real.len() as u64) + )); + } + + // Canonical-depth regression: `nonced_root` is responder-chosen in round 1, + // so a partial holder could set it to a single block's leaf and pass with + // zero siblings whenever the fresh draw lands on that block. Pinning the + // sibling-chain length to the tree depth rejects that (and any wrong-depth + // chain) even when it would fold to the supplied root. + #[test] + fn canonical_depth_rejects_wrong_sibling_count() { + let content = content_of(4096); // 4 blocks → canonical depth 2 + let bc = block_count(content.len() as u64); + assert_eq!(nonced_tree_depth(bc), 2); + let block0 = block_bytes(&content, 0); + + // Degraded escape: claim root = block 0's own leaf, supply zero siblings. + // Folds to that root, but depth 0 != 2, so rejected. + let leaf0 = nonced_block_leaf(&NONCE, &PEER, &KEY, 0, block0); + assert!(!verify_nonced_block( + &NONCE, + &PEER, + &KEY, + 0, + block0, + &[], + &leaf0, + bc + )); + + // The honest, correct-depth opening still verifies. + let root = nonced_block_root(&NONCE, &PEER, &KEY, &content); + let siblings = nonced_block_siblings(&NONCE, &PEER, &KEY, &content, 0).expect("siblings"); + assert_eq!(siblings.len(), 2); + assert!(verify_nonced_block( + &NONCE, &PEER, &KEY, 0, block0, &siblings, &root, bc + )); + + // A too-long chain (extra sibling) is rejected on depth alone. + let mut too_long = siblings; + too_long.push([0u8; 32]); + assert!(!verify_nonced_block( + &NONCE, &PEER, &KEY, 0, block0, &too_long, &root, bc + )); + } +} diff --git a/src/replication/storage_commitment_audit.rs b/src/replication/storage_commitment_audit.rs index 14e5104f..951614b8 100644 --- a/src/replication/storage_commitment_audit.rs +++ b/src/replication/storage_commitment_audit.rs @@ -9,6 +9,7 @@ //! [`handle_subtree_challenge`]; the pure proof maths live in //! [`crate::replication::subtree`]. +use std::collections::{HashMap, HashSet}; use std::sync::Arc; use std::time::Duration; @@ -19,11 +20,12 @@ use crate::ant_protocol::XorName; use crate::replication::commitment::{commitment_hash, StorageCommitment}; use crate::replication::commitment_state::ResponderCommitmentState; use crate::replication::config::{ - ReplicationConfig, MAX_BYTE_CHALLENGE_KEYS, REPLICATION_PROTOCOL_ID, + ReplicationConfig, MAX_SLICE_OPENINGS, SUBTREE_AUDIT_PROTOCOL_ID, }; use crate::replication::protocol::{ RejectKind, ReplicationMessage, ReplicationMessageBody, SubtreeAuditChallenge, - SubtreeAuditResponse, SubtreeByteChallenge, SubtreeByteItem, SubtreeByteResponse, + SubtreeAuditResponse, SubtreeSliceChallenge, SubtreeSliceItem, SubtreeSliceOpening, + SubtreeSliceResponse, }; use crate::replication::recent_provers::RecentProvers; use crate::replication::subtree::{ @@ -44,14 +46,22 @@ use crate::replication::audit::AuditTickResult; // Auditor side // --------------------------------------------------------------------------- -/// ADR-0002 round-2 byte challenge samples a SMALL surprise set of the proven +/// ADR-0002 round-2 slice challenge samples a SMALL surprise set of the proven /// leaves (3..=5). Small enough that the responder's honest local-disk read of -/// the original chunks stays well inside the possession-in-time deadline, while +/// the original chunks stays well inside the response deadline (a liveness bound), while /// a relay forced to fetch them over the network blows it; large enough that /// faking a fraction `x` of leaves survives only `(1 - x)^k`. const BYTE_SPOTCHECK_MIN: u32 = 3; const BYTE_SPOTCHECK_MAX: u32 = 5; +// Each sampled leaf produces up to two openings (a fresh-random block plus the +// claimed final block, deduplicated when they coincide), so the opening cap must +// cover twice the leaf sample. +const _: () = assert!( + 2 * BYTE_SPOTCHECK_MAX as usize <= MAX_SLICE_OPENINGS, + "MAX_SLICE_OPENINGS must cover two openings per sampled leaf" +); + /// ADR-0004 A1: with grace removed, the responder retries a TRANSIENT chunk-read /// error a few times before rejecting `Transient` (which routes to the timeout /// lane). A momentary disk blip usually clears within these attempts; only a @@ -139,14 +149,16 @@ struct AuditCtx<'a> { /// /// 1. **Structure** (round 1) — the returned subtree rebuilds to the pinned /// root, within a size-scaled deadline. -/// 2. **Real bytes** (round 2) — the auditor demands the ORIGINAL chunk content -/// for a 3..=5 FRESHLY-RANDOM sample of the proven leaves (chosen after the -/// proof arrives, not nonce-derived — see `random_spotcheck_leaves`) FROM the -/// responder, and recomputes both the content-address hash and the nonce -/// freshness hash from that served content. The auditor holds none of the -/// peer's chunks. -/// 3. **Timing** — each round's deadline is sized to an honest local-disk read, -/// so a relay forced to fetch over the network blows it. +/// 2. **Verified slice** (round 2) — for a 3..=5 FRESHLY-RANDOM sample of the +/// proven leaves (chosen after the proof arrives, not nonce-derived — see +/// `random_spotcheck_leaves`), the auditor opens one random 1 KiB block per +/// leaf (plus its final block) and verifies a Bao slice against the chunk +/// address and a nonced-tree opening against the round-1 `nonced_root`, over +/// the bytes the responder serves. The auditor holds none of the peer's +/// chunks, and possession is the round-1 commitment, not a full-chunk transfer. +/// 3. **Timing** — a response deadline still bounds liveness, but (unlike the old +/// full-byte round 2) it is no longer the possession proof; that is the +/// round-1 `nonced_root` commitment. /// /// A timeout (either round) is reported as [`AuditFailureReason::Timeout`] (the /// caller applies the strike/grace policy). Any structural failure, served @@ -197,7 +209,7 @@ pub async fn run_subtree_audit( let timeout = config.audit_response_timeout(subtree_leaves); let response = match p2p_node - .send_request(challenged_peer, REPLICATION_PROTOCOL_ID, encoded, timeout) + .send_request(challenged_peer, SUBTREE_AUDIT_PROTOCOL_ID, encoded, timeout) .await { Ok(resp) => resp, @@ -231,11 +243,11 @@ pub async fn run_subtree_audit( dispatch_subtree_response(resp_msg.body, &ctx).await } -/// Outcome of the round-2 byte challenge round-trip (auditor side). -enum ByteRound { - /// The responder returned per-key items (verified by the caller). - Served(Vec), - /// The responder rejected the byte challenge (confirmed failure for a +/// Outcome of the round-2 slice challenge round-trip (auditor side). +enum SliceRound { + /// The responder returned per-opening items (verified by the caller). + Served(Vec), + /// The responder rejected the slice challenge (confirmed failure for a /// recently pinned commitment). Rejected, /// The responder rejected with `Transient` (a local read error): routed to @@ -244,50 +256,63 @@ enum ByteRound { /// must not keep stale credit. Distinct from a silent network `Timeout`, /// which keeps credit (a dropped packet is not evidence of loss). TransientReject, - /// No response within the byte deadline, or a transport error (graced + /// No response within the slice deadline, or a transport error (graced /// timeout). Keeps holder credit. Timeout, + /// The responder claimed `Bootstrapping` in round 2 after answering a valid + /// round-1 proof. A node that just produced a signed subtree proof is + /// provably not bootstrapping, so this responsive contradiction is a + /// confirmed failure (not a graced timeout): it revokes the peer's holder + /// credit and takes the trust penalty. + /// + /// This classification relies on `is_bootstrapping` being ONE-WAY (true → + /// false; see `mod.rs`): a round-1 proof implies the snapshot was already + /// `false`, and a restart drops the single-use round-1 session so round 2 is + /// answered `Transient` before this branch is ever reached. An honest running + /// node therefore cannot produce this response; only a malicious, incompatible, + /// or future broken-state peer can. If a "re-bootstrap" (false → true) + /// transition is ever added, it MUST clear live sessions or this policy must + /// be revisited. + ResponsiveBootstrap, /// Malformed / unexpected round-2 response body. Malformed, } -/// Round 2: ask the responder for the ORIGINAL chunk content of one BATCH of -/// auditor-selected spot-check `keys` (at most [`MAX_BYTE_CHALLENGE_KEYS`], so -/// the worst-case response of max-size chunks fits the wire cap), sized to a -/// possession-in-time deadline (honest local-disk read of `keys.len()` chunks). -/// The responder cannot have predicted which keys are sampled. -async fn request_byte_proof(ctx: &AuditCtx<'_>, keys: &[XorName]) -> ByteRound { - let challenge = SubtreeByteChallenge { +/// Round 2: ask the responder to open `openings` blocks (one 1 KiB block per +/// sampled leaf, at most [`MAX_SLICE_OPENINGS`]) with a Bao verified slice plus a +/// nonced block-tree opening each. The reply is a few KB total, so there is no +/// batching. The responder cannot have predicted which leaves — or which block +/// within each — are opened (fresh post-proof randomness). +async fn request_slice_proof(ctx: &AuditCtx<'_>, openings: &[SubtreeSliceOpening]) -> SliceRound { + let challenge = SubtreeSliceChallenge { challenge_id: ctx.challenge_id, nonce: ctx.nonce, challenged_peer_id: *ctx.challenged_peer.as_bytes(), expected_commitment_hash: ctx.expected_commitment_hash, - keys: keys.to_vec(), + openings: openings.to_vec(), }; let msg = ReplicationMessage { request_id: ctx.challenge_id, - body: ReplicationMessageBody::SubtreeByteChallenge(challenge), + body: ReplicationMessageBody::SubtreeSliceChallenge(challenge), }; let encoded = match msg.encode() { Ok(data) => data, Err(e) => { - warn!("Audit: failed to encode byte challenge: {e}"); - return ByteRound::Malformed; + warn!("Audit: failed to encode slice challenge: {e}"); + return SliceRound::Malformed; } }; - // Deadline sized to "honest responder reads `keys.len()` local chunks AND - // ships them back": a relay forced to fetch them over the network blows it - // (graced timeout, never a confirmed failure — same possession-in-time - // principle as round 1). Uses the byte-round floor, which is high enough for - // the multi-MiB reply (handshake + upload + busy disk) — the round-1 - // hashes-only floor would be too tight for 2 × 4 MiB (§4). - let timeout = ctx.config.byte_audit_response_timeout(keys.len()); + // Deadline sized to "honest responder reads `openings.len()` full local + // chunks to build their proofs": generous, because possession is now + // guaranteed by the round-1 nonced commitment, not by this deadline being + // too tight for a relay to fetch bytes. + let timeout = ctx.config.slice_audit_response_timeout(openings.len()); let response = match ctx .p2p_node .send_request( ctx.challenged_peer, - REPLICATION_PROTOCOL_ID, + SUBTREE_AUDIT_PROTOCOL_ID, encoded, timeout, ) @@ -296,27 +321,27 @@ async fn request_byte_proof(ctx: &AuditCtx<'_>, keys: &[XorName]) -> ByteRound { Ok(resp) => resp, Err(e) => { debug!( - "Audit: byte challenge to {} timed out / failed: {e}", + "Audit: slice challenge to {} timed out / failed: {e}", ctx.challenged_peer ); - return ByteRound::Timeout; + return SliceRound::Timeout; } }; let resp_msg = match ReplicationMessage::decode(&response.data) { Ok(m) => m, Err(e) => { - warn!("Audit: failed to decode byte response: {e}"); - return ByteRound::Malformed; + warn!("Audit: failed to decode slice response: {e}"); + return SliceRound::Malformed; } }; match resp_msg.body { - ReplicationMessageBody::SubtreeByteResponse(SubtreeByteResponse::Items { + ReplicationMessageBody::SubtreeSliceResponse(SubtreeSliceResponse::Items { challenge_id, items, - }) if challenge_id == ctx.challenge_id => ByteRound::Served(items), - ReplicationMessageBody::SubtreeByteResponse(SubtreeByteResponse::Rejected { + }) if challenge_id == ctx.challenge_id => SliceRound::Served(items), + ReplicationMessageBody::SubtreeSliceResponse(SubtreeSliceResponse::Rejected { challenge_id, kind, reason, @@ -329,28 +354,29 @@ async fn request_byte_proof(ctx: &AuditCtx<'_>, keys: &[XorName]) -> ByteRound { match grade_reject(kind) { RejectGrade::Confirmed => { warn!( - "Audit: {} rejected byte challenge ({kind:?}; confirmed): {reason}", + "Audit: {} rejected slice challenge ({kind:?}; confirmed): {reason}", ctx.challenged_peer ); - ByteRound::Rejected + SliceRound::Rejected } RejectGrade::TimeoutLane => { debug!( - "Audit: {} returned Transient for byte challenge (timeout lane): {reason}", + "Audit: {} returned Transient for slice challenge (timeout lane): {reason}", ctx.challenged_peer ); - ByteRound::TransientReject + SliceRound::TransientReject } } } - // A node claiming bootstrap MID-AUDIT (it answered round 1) is treated - // as a timeout: it didn't prove possession but the round-1 proof shows - // it isn't bootstrapping, so the bootstrap-claim-abuse detector (round 1) - // owns that lane; here we just don't credit it. - ReplicationMessageBody::SubtreeByteResponse(SubtreeByteResponse::Bootstrapping { + // A node claiming bootstrap MID-AUDIT (it just answered round 1 with a + // valid signed proof) is contradicting itself: a bootstrapping node has + // no committed data to prove. A graced timeout here would let it keep the + // holder credit it earned earlier while dodging every round-2 possession + // check, so this is a confirmed failure with credit revocation. + ReplicationMessageBody::SubtreeSliceResponse(SubtreeSliceResponse::Bootstrapping { challenge_id, - }) if challenge_id == ctx.challenge_id => ByteRound::Timeout, - _ => ByteRound::Malformed, + }) if challenge_id == ctx.challenge_id => SliceRound::ResponsiveBootstrap, + _ => SliceRound::Malformed, } } @@ -465,11 +491,12 @@ pub(crate) enum AuditVerdict { /// Runs the cheap gates in fail-fast order: pin / identity / signature → /// structure (the returned subtree rebuilds to the pinned root). It does **not** /// prove byte possession — the leaves carry only the public `bytes_hash` (the -/// chunk address) and a `nonced_hash` the responder computed itself. Possession -/// is proven in round 2 ([`verify_byte_response`]), where the auditor demands -/// the original chunk bytes for a freshly-random (post-proof) sample and -/// recomputes both hashes from the SERVED content. This removes any dependency -/// on the auditor holding the peer's chunks. +/// chunk address), a `content_len`, and a `nonced_root` the responder computed +/// itself. Possession is proven in round 2 ([`verify_slice_response`]), where the +/// auditor opens a freshly-random (post-proof) block per sampled leaf and +/// verifies a Bao slice against the address plus a nonced-tree opening against +/// the round-1 `nonced_root` — over the SERVED bytes, so the auditor never holds +/// the peer's chunks. /// /// Returns [`StructureVerdict::Valid`] (proceed to round 2) or a confirmed /// [`AuditFailureReason`] mapped from the failing gate. @@ -500,6 +527,22 @@ pub(crate) fn evaluate_subtree_structure( if let StructureVerdict::Invalid(_) = verify_subtree_proof(proof, nonce, commitment) { return Err(AuditFailureReason::DigestMismatch); } + + // -- Content-address binding (possession-forgery guard) -- + // The commitment is only sound for CONTENT-ADDRESSED chunks, where the key IS + // the content hash (`key == BLAKE3(content)`), so an honest leaf is always + // `(key, key)` (see the commitment builder's `(k, k)` shortcut). The signed + // Merkle leaf hashes `key ‖ bytes_hash` WITHOUT forcing them equal, yet round 2 + // authenticates the served slice against `bytes_hash` while holder credit is + // recorded for `key`. A leaf with `bytes_hash != key` would therefore let a peer + // earn credit for an expensive address `key` by proving possession of an + // unrelated (e.g. one-byte) chunk hashing to `bytes_hash`. Reject any such leaf: + // for honest content-addressed data the two are identical, so this never fails + // an honest holder, and it re-binds Chain 1's `bytes_hash` check to the credited + // `key`. + if proof.leaves.iter().any(|l| l.bytes_hash != l.key) { + return Err(AuditFailureReason::DigestMismatch); + } Ok(()) } @@ -510,17 +553,19 @@ pub(crate) fn evaluate_subtree_structure( /// CRITICAL (ADR-0002 soundness): the sample MUST NOT be derivable from /// anything the responder knew when it built the round-1 proof. The structural /// root check binds only `(key, bytes_hash)` (both public — `bytes_hash` is the -/// chunk's network address), NOT `nonced_hash`. So a relay holding only public -/// addresses can fabricate a structurally-valid proof with bogus `nonced_hash` +/// chunk's network address), NOT `nonced_root`. So a relay holding only public +/// addresses can fabricate a structurally-valid proof with a bogus `nonced_root` /// on every leaf and, if it could predict which leaves round 2 opens, fetch /// only those and pass — earning holder credit for leaves it never held. /// /// Picking the sample with fresh CSPRNG randomness AFTER the proof is received /// turns round 1 into a commitment and round 2 into an unpredictable challenge /// (cut-and-choose): to pass with probability above `(1 - faked_fraction)^count` -/// the responder must have produced a correct `nonced_hash` — which requires the -/// real bytes — for essentially every leaf at round-1 commit time. The auditor -/// still holds none of the peer's chunks. +/// the responder must have produced a correct `nonced_root` — which requires the +/// real bytes under the fresh nonce — for essentially every leaf at round-1 +/// commit time. The auditor still holds none of the peer's chunks. The block +/// index opened within each sampled leaf is likewise drawn fresh +/// ([`random_block_index`]), so no single block can be prepared in advance. fn random_spotcheck_leaves( proof: &SubtreeProof, count: u32, @@ -553,54 +598,150 @@ fn random_spotcheck_leaves( .collect() } -/// Round-2 verdict (ADR-0002): the responder served the original chunk content -/// for the auditor's spot-check sample; verify possession from THAT content. +/// Draw a fresh-random 1 KiB block index in `0..block_count(content_len)`. /// -/// `served(key)` returns what the responder returned for a requested key: -/// `Some(Some(bytes))` for [`SubtreeByteItem::Present`], `Some(None)` for an -/// explicit [`SubtreeByteItem::Absent`], and `None` if the responder omitted the -/// key entirely (treated like `Absent` — a committed key it would not serve). +/// Called AFTER the round-1 proof is in hand, per sampled leaf, so the responder +/// could not have prepared only the opened block (the cut-and-choose property, +/// now at block granularity). +fn random_block_index(content_len: u32) -> u32 { + let count = crate::replication::slice::block_count(u64::from(content_len)); + if count <= 1 { + return 0; + } + rand::thread_rng().gen_range(0..count) +} + +/// The two block indices opened for one sampled leaf: a fresh-random block (the +/// cut-and-choose possession check) and the claimed final block. +/// +/// The final block is opened so the auditor's Bao decode reaches EOF, which is +/// where Bao authenticates the encoded length against the address. Without it a +/// responder can forge a *shorter* `content_len` (which is not signed by the +/// round-1 commitment), collapse the challenge space, and pass while storing only +/// the prefix — the length header alone is not bound for a prefix slice that +/// never touches EOF. Opening the final block pins the true length: a forged +/// short length fails the final-block decode against the real address. /// -/// For each sampled leaf the auditor recomputes, from the SERVED content: -/// - `BLAKE3(content) == leaf.bytes_hash` (the chunk's content address), AND -/// - `BLAKE3(nonce ‖ peer ‖ key ‖ content) == leaf.nonced_hash` (freshness), -/// i.e. `compute_audit_digest(nonce, peer, key, content)`. +/// Returns one index when the random draw already lands on the final block (or +/// the chunk is a single block), two otherwise, never more than two. +fn block_indices_for_leaf(content_len: u32) -> Vec { + let count = crate::replication::slice::block_count(u64::from(content_len)); + let final_index = count.saturating_sub(1); + let random = random_block_index(content_len); + if random == final_index { + vec![final_index] + } else { + vec![random, final_index] + } +} + +/// Round-2 verdict (ADR-0002 / V2-685): the responder opened one 1 KiB block per +/// sampled leaf with a Bao verified slice plus a nonced block-tree opening; +/// verify possession from those. /// -/// The freshness inputs are byte-identical to what the responder used to BUILD -/// the leaf in round 1 (`subtree_leaf` → `nonced_leaf_hash`): the SAME four -/// inputs, so an honest holder's served content reproduces `nonced_hash` -/// exactly. Round 1 commits over the data (the `nonced_hash` is uncomputable -/// without the bytes); round 2 reveals a random subset to prove the commitment -/// was not fabricated. +/// `openings` pairs each sampled leaf with the block index the auditor drew for +/// it. `items` is what the responder returned. For each opening the auditor: +/// 1. finds the responder's `Present` item for exactly this `(key, block_index)` +/// (a missing / `Absent` / wrong-block item is a provable lie); +/// 2. **Chain 1** — decodes the Bao slice against `leaf.bytes_hash` (the chunk +/// address), recovering the verified block bytes. This proves the block is +/// the real content at that offset, without the auditor holding the chunk. +/// 3. **Chain 2** — folds the nonced block leaf (recomputed from those verified +/// bytes under the audit nonce/peer/key/index) with the returned siblings to +/// `leaf.nonced_root`. This proves the responder committed a nonced tree +/// over the real content at round-1 time, so it held the bytes then. /// -/// Both checks are over the content the responder sent, so the auditor needs to -/// hold none of the peer's chunks. Any `Absent`/omitted committed key, or any -/// served content that fails a hash, is a provable lie → confirmed -/// [`AuditFailureReason::DigestMismatch`]. All sampled leaves verifying → -/// `Pass { checked }`. -pub(crate) fn verify_byte_response( - leaves: &[&crate::replication::subtree::SubtreeLeaf], +/// Both chains are over the SAME block bytes and the auditor holds none of the +/// peer's chunks. Any missing/absent opening, a slice that fails to decode +/// against the address, or a nonced opening that does not fold to the committed +/// root, is a provable lie → confirmed [`AuditFailureReason::DigestMismatch`]. +/// All openings verifying → `Pass { checked }`. +pub(crate) fn verify_slice_response( + openings: &[(crate::replication::subtree::SubtreeLeaf, u32)], nonce: &[u8; 32], challenged_peer_bytes: &[u8; 32], - served: impl Fn(&XorName) -> Option>>, + items: &[SubtreeSliceItem], ) -> AuditVerdict { + // Validate the response shape before matching. A well-formed response has at + // most one item per SOLICITED identity; the per-opening `find_map` below + // returns the FIRST match, so a duplicate `(key, block_index)` `Present`, a + // key that is both `Present` and `Absent`, a repeated `Absent`, or an + // UNSOLICITED item would let the responder's item order (or padding) decide + // the verdict. Bounding `items.len()` to the request count also keeps this + // check (and the matching below) at O(openings) — a peer cannot pad the + // response with thousands of unique items to force quadratic validation work. + if items.len() > openings.len() { + return AuditVerdict::Fail(AuditFailureReason::MalformedResponse); + } + let requested_blocks: HashSet<(XorName, u32)> = + openings.iter().map(|(leaf, i)| (leaf.key, *i)).collect(); + let requested_keys: HashSet = openings.iter().map(|(leaf, _)| leaf.key).collect(); + let mut present: HashSet<(XorName, u32)> = HashSet::new(); + let mut absent: HashSet = HashSet::new(); + for it in items { + let ok = match it { + SubtreeSliceItem::Present { + key, block_index, .. + } => { + requested_blocks.contains(&(*key, *block_index)) + && present.insert((*key, *block_index)) + && !absent.contains(key) + } + SubtreeSliceItem::Absent { key } => { + requested_keys.contains(key) + && absent.insert(*key) + && !present.iter().any(|(k, _)| k == key) + } + }; + if !ok { + return AuditVerdict::Fail(AuditFailureReason::MalformedResponse); + } + } + let mut checked = 0usize; - for leaf in leaves { - // Present{bytes} -> Some(Some(bytes)); Absent -> Some(None); omitted -> None. - // A committed key the responder cannot / will not serve is a provable lie. - let Some(Some(content)) = served(&leaf.key) else { + for (leaf, block_index) in openings { + let block_index = *block_index; + // Match the responder's item for exactly this (key, block_index). A + // missing item, an explicit Absent, or a different block is a provable lie. + let served = items.iter().find_map(|it| match it { + SubtreeSliceItem::Present { + key, + block_index: served_index, + bao_slice, + nonced_siblings, + } if key == &leaf.key && *served_index == block_index => { + Some(Some((bao_slice.as_slice(), nonced_siblings.as_slice()))) + } + SubtreeSliceItem::Absent { key } if key == &leaf.key => Some(None), + _ => None, + }); + let Some(Some((bao_slice, nonced_siblings))) = served else { + return AuditVerdict::Fail(AuditFailureReason::DigestMismatch); + }; + + // Chain 1: authenticate the block against the chunk address. + let Some(block) = crate::replication::slice::verify_block_slice( + bao_slice, + &leaf.bytes_hash, + u64::from(leaf.content_len), + block_index, + ) else { return AuditVerdict::Fail(AuditFailureReason::DigestMismatch); }; - let plain = *blake3::hash(&content).as_bytes(); - let nonced = crate::replication::subtree::nonced_leaf_hash( + + // Chain 2: prove the block was committed under round 1's fresh nonce, + // with the sibling chain pinned to the canonical depth for the chunk's + // block count (bound the claimed tree geometry). + if !crate::replication::slice::verify_nonced_block( nonce, challenged_peer_bytes, &leaf.key, - &content, - ); - if leaf.bytes_hash != plain || leaf.nonced_hash != nonced { - // Served content does not hash to the committed address / freshness - // hash: cannot be the chunk it committed to. + block_index, + &block, + nonced_siblings, + &leaf.nonced_root, + crate::replication::slice::block_count(u64::from(leaf.content_len)), + ) { return AuditVerdict::Fail(AuditFailureReason::DigestMismatch); } checked += 1; @@ -613,13 +754,14 @@ pub(crate) fn verify_byte_response( /// **Round 1** (this proof): pin + identity + signature + structure. If the /// proof structurally rebuilds to the pinned root, the tree SHAPE is committed — /// but not yet that the bytes are held. **Round 2**: the auditor picks a small -/// freshly-random (post-proof) sample of the just-proven leaves and sends a -/// [`SubtreeByteChallenge`] demanding their original chunk content FROM the -/// responder, then verifies that content against the committed `bytes_hash` -/// (content address) and `nonced_hash` (freshness). A responder that committed -/// to a chunk it no longer holds cannot serve content that hashes to the -/// committed address, so it fails — regardless of what the auditor holds. On a -/// full pass, credits the peer as a proven holder. +/// freshly-random (post-proof) sample of the just-proven leaves, draws a fresh +/// block index for each, and sends a [`SubtreeSliceChallenge`] opening those +/// blocks. It verifies each opened block against the committed `bytes_hash` +/// (Bao slice → content address) and `nonced_root` (nonced block-tree opening → +/// round-1 possession commit). A responder that committed to a chunk it no +/// longer held cannot have committed a correct `nonced_root`, so it fails — +/// regardless of what the auditor holds. On a full pass, credits the peer as a +/// proven holder. async fn verify_subtree_response( ctx: &AuditCtx<'_>, commitment: &StorageCommitment, @@ -640,13 +782,14 @@ async fn verify_subtree_response( return failed(challenged_peer, challenge_id, reason); } - // -- Round 2: surprise byte challenge for a 3..=5 FRESHLY-RANDOM sample. -- + // -- Round 2: surprise slice challenge for a 3..=5 FRESHLY-RANDOM sample. -- // The sample is chosen now, with CSPRNG randomness, AFTER the round-1 proof // is in hand — NOT derived from the round-1 nonce. The responder committed - // every leaf's `nonced_hash` in round 1 without knowing which leaves we will - // open, so it cannot have fabricated the un-opened ones (cut-and-choose). - // We cap the sample at the ADR's 3..=5 band (clamped to the subtree size) so - // the round-2 message and the responder's disk read stay cheap. + // every leaf's `nonced_root` in round 1 without knowing which leaves — or + // which block within each — we will open, so it could not have prepared only + // the opened blocks (cut-and-choose). The reply is a few KB per opening (a + // 1 KiB block plus two short hash chains), so one round-2 message serves the + // whole sample; no batching. let sample_n = ctx .config .audit_spotcheck_count() @@ -662,85 +805,70 @@ async fn verify_subtree_response( AuditFailureReason::DigestMismatch, ); } - // The sample is challenged in batches of MAX_BYTE_CHALLENGE_KEYS so each - // response — worst case, every requested chunk at MAX_CHUNK_SIZE — still - // encodes under MAX_REPLICATION_MESSAGE_SIZE. Each batch carries its own - // possession-in-time deadline (sized to its own length), so splitting does - // not widen the per-chunk window a relay would need to fetch over the - // network. - // - // CRITICAL: verify each batch's served bytes AS IT ARRIVES, against that - // batch's own sampled leaves, and return a CONFIRMED failure immediately. - // Deferring all verification until every batch is collected would let a - // later batch's timeout-lane Timeout (`round_failure`) mask a deterministic - // failure already proven by an earlier batch (an absent committed key or a - // hash mismatch) — a confirmed cheat would be downgraded to a timeout. A - // Timeout/Rejected/Malformed only becomes the verdict if NO earlier batch - // already produced confirmed bad bytes. - let verdict = 'rounds: { - for batch in sampled.chunks(MAX_BYTE_CHALLENGE_KEYS) { - let batch_keys: Vec = batch.iter().map(|l| l.key).collect(); - match request_byte_proof(ctx, &batch_keys).await { - ByteRound::Served(items) => { - // Verify THIS batch now. A confirmed failure here is final — - // a later batch's timeout must not be able to overwrite it. - let v = verify_byte_response( - batch, - &ctx.nonce, - challenged_peer.as_bytes(), - |key| { - items.iter().find_map(|it| match it { - SubtreeByteItem::Present { key: k, bytes } if k == key => { - Some(Some(bytes.clone())) - } - SubtreeByteItem::Absent { key: k } if k == key => Some(None), - _ => None, - }) - }, - ); - if let AuditVerdict::Fail(reason) = v { - break 'rounds AuditVerdict::Fail(reason); - } - } - // The responder rejected the byte challenge for a recently - // pinned commitment → confirmed failure, same as round 1. - ByteRound::Rejected => { - break 'rounds AuditVerdict::Fail(AuditFailureReason::Rejected) - } - // Transient reject (a local read error): ADR-0004 A1 routes it to - // the timeout lane — no trust penalty, but revoke the holder - // credit for THIS pinned commitment (the peer answered and could - // not prove possession) before taking the Timeout verdict. Scoped - // to the commitment hash, not the whole peer, so it never erases - // credit the peer re-earned for a newer commitment. - ByteRound::TransientReject => { - if let Some(credit) = ctx.credit { - credit - .recent_provers - .write() - .await - .forget_commitment(&ctx.expected_commitment_hash); - } - break 'rounds AuditVerdict::Fail(AuditFailureReason::Timeout); - } - // No response within the byte deadline (or transport error) → - // timeout (graced by the caller's strike policy — could be - // honest slowness). Keeps credit (a dropped packet is not - // evidence of loss). Only reached when no earlier batch already - // confirmed bad bytes. - ByteRound::Timeout => { - break 'rounds AuditVerdict::Fail(AuditFailureReason::Timeout) - } - // Malformed/unexpected round-2 body. - ByteRound::Malformed => { - break 'rounds AuditVerdict::Fail(AuditFailureReason::MalformedResponse) - } - } + // Pair each sampled leaf with up to two block indices: a fresh-random block + // (possession) and the claimed final block (a length pin — see + // `block_indices_for_leaf`). Own the leaves so the borrow on `proof` ends + // before the await. The sample is <= BYTE_SPOTCHECK_MAX and each leaf yields + // <= 2 openings, so the total is <= 2 * BYTE_SPOTCHECK_MAX <= MAX_SLICE_OPENINGS + // (statically asserted); `take` is a defensive backstop. + let openings_with_leaves: Vec<(crate::replication::subtree::SubtreeLeaf, u32)> = sampled + .iter() + .flat_map(|leaf| { + let leaf = (*leaf).clone(); + block_indices_for_leaf(leaf.content_len) + .into_iter() + .map(move |block_index| (leaf.clone(), block_index)) + }) + .take(MAX_SLICE_OPENINGS) + .collect(); + let openings: Vec = openings_with_leaves + .iter() + .map(|(leaf, block_index)| SubtreeSliceOpening { + key: leaf.key, + block_index: *block_index, + }) + .collect(); + + let verdict = match request_slice_proof(ctx, &openings).await { + // The responder served openings: verify both chains for every one. Any + // failing chain is a confirmed cheat. + SliceRound::Served(items) => verify_slice_response( + &openings_with_leaves, + &ctx.nonce, + challenged_peer.as_bytes(), + &items, + ), + // Confirmed round-2 failures. `Rejected`: the responder repudiated the + // slice challenge for a recently pinned commitment. `ResponsiveBootstrap`: + // it claimed bootstrap AFTER producing a valid round-1 proof, which is a + // self-contradiction (a bootstrapping node has no committed data to + // prove). Both are classified as any other confirmed `Rejected`, which + // revokes the peer's holder credit and takes the trust penalty downstream + // (`apply_audit_failure_credit_revocation` → `forget_peer`). + SliceRound::Rejected | SliceRound::ResponsiveBootstrap => { + AuditVerdict::Fail(AuditFailureReason::Rejected) } - // Every batch served bytes that verified. - AuditVerdict::Pass { - checked: sampled.len(), + // Transient reject (a local read error): ADR-0004 A1 routes it to the + // timeout lane — no trust penalty, but revoke the holder credit for THIS + // pinned commitment (the peer answered and could not prove possession). + // Scoped to the commitment hash, so it never erases credit the peer + // re-earned for a newer commitment. + SliceRound::TransientReject => { + if let Some(credit) = ctx.credit { + credit + .recent_provers + .write() + .await + .forget_commitment(&ctx.expected_commitment_hash); + } + AuditVerdict::Fail(AuditFailureReason::Timeout) } + // No response within the slice deadline (or transport error) → timeout + // (graced by the caller's strike policy — could be honest slowness). + // Keeps credit (a dropped packet is not evidence of loss). + SliceRound::Timeout => AuditVerdict::Fail(AuditFailureReason::Timeout), + // Malformed/unexpected round-2 body. + SliceRound::Malformed => AuditVerdict::Fail(AuditFailureReason::MalformedResponse), }; match verdict { @@ -761,7 +889,7 @@ async fn verify_subtree_response( } info!( "Audit: peer {challenged_peer} passed subtree audit ({} leaves, {checked} \ - byte-checked)", + block openings verified)", proof.leaves.len() ); AuditTickResult::Passed { @@ -890,6 +1018,7 @@ fn subtree_failure_summary(reason: &AuditFailureReason) -> AuditFailureSummary { /// nonce-selected branch. If this node is bootstrapping it says so; if it /// genuinely does not retain the pinned commitment it rejects (which, with audit /// grace removed, the auditor treats as a confirmed failure for an in-window pin). +#[allow(clippy::too_many_lines)] pub async fn handle_subtree_challenge( challenge: &SubtreeAuditChallenge, storage: &LmdbStorage, @@ -989,13 +1118,33 @@ pub async fn handle_subtree_challenge( }; } }; - leaves.push(crate::replication::subtree::subtree_leaf( - &challenge.nonce, - &challenge.challenged_peer_id, - key, - &bytes, - )); - // bytes drops here. + // Hash the leaf (a full keyed-BLAKE3 pass over the chunk) on a blocking + // thread, not the async worker: a maximal subtree is ~sqrt(N) leaves of up + // to MAX_CHUNK_SIZE each, so doing this inline would tie up a Tokio worker + // for the whole proof and let a few round-1 requests starve the runtime. + // One chunk is resident at a time, so peak memory is bounded. + let nonce = challenge.nonce; + let peer = challenge.challenged_peer_id; + let leaf_key = *key; + let leaf = match tokio::task::spawn_blocking(move || { + crate::replication::subtree::subtree_leaf(&nonce, &peer, &leaf_key, &bytes) + }) + .await + { + Ok(leaf) => leaf, + Err(e) => { + warn!( + "Subtree audit: leaf hashing task failed for key {}: {e}", + hex::encode(key) + ); + return SubtreeAuditResponse::Rejected { + challenge_id: challenge.challenge_id, + kind: RejectKind::Transient, + reason: format!("leaf hashing task error: {e}"), + }; + } + }; + leaves.push(leaf); } SubtreeAuditResponse::Proof { @@ -1008,57 +1157,130 @@ pub async fn handle_subtree_challenge( } } -/// Handle a round-2 byte challenge (responder side), ADR-0002. +/// Build every requested opening for one committed key from a single hashing +/// pass over its bytes: a Bao verified slice (authenticity against the chunk +/// address) plus a nonced block-tree opening (possession against round 1's +/// `nonced_root`) per block index. +/// +/// The Bao outboard and nonced tree each hash the full chunk, so this builds +/// them once (via [`ChunkOpener`]) and serves all `indices` from them rather +/// than re-hashing per opening (V2-685 round-2 amplification fix). `indices` is +/// deduplicated by the caller. CPU-heavy, so callers run it on a blocking thread. +/// +/// Returns `Err((kind, reason))` for the terminal cases that abort the whole +/// response: a block index out of range (only a forged/buggy auditor sends one → +/// `Protocol`), or a surprise in-memory Bao extraction error (`Transient`, so an +/// honest holder is not branded a deleter). +fn build_slice_items_for_key( + nonce: [u8; 32], + peer: [u8; 32], + key: XorName, + bytes: &[u8], + indices: &[u32], +) -> Result, (RejectKind, String)> { + let opener = crate::replication::slice::ChunkOpener::new(&nonce, &peer, &key, bytes); + let count = opener.block_count(); + let mut items = Vec::with_capacity(indices.len()); + for &block_index in indices { + if block_index >= count { + return Err(( + RejectKind::Protocol, + format!( + "block index {block_index} out of range for key {}", + hex::encode(key) + ), + )); + } + let bao_slice = match opener.bao_slice(block_index) { + Ok(slice) => slice, + Err(e) => { + warn!( + "Subtree slice audit: bao extraction failed for key {}: {e}", + hex::encode(key) + ); + return Err((RejectKind::Transient, format!("bao extraction error: {e}"))); + } + }; + // `block_index < count` was just checked, so siblings must exist. A + // `None` here is an internal proof-building inconsistency, not an honest + // condition: abort the whole response as `Transient` (routes to the + // timeout lane) rather than emit an empty-siblings `Present` that would + // make an honest holder fail its own audit. + let Some(nonced_siblings) = opener.nonced_siblings(block_index) else { + warn!( + "Subtree slice audit: no nonced siblings for in-range block {block_index} of key {}", + hex::encode(key) + ); + return Err(( + RejectKind::Transient, + format!("nonced sibling build inconsistency for block {block_index}"), + )); + }; + items.push(SubtreeSliceItem::Present { + key, + block_index, + bao_slice, + nonced_siblings, + }); + } + Ok(items) +} + +/// Handle a round-2 slice challenge (responder side), ADR-0002 / V2-685. /// /// The auditor has already structurally verified this node's round-1 subtree -/// proof and now demands the ORIGINAL chunk bytes for a small freshly-random -/// sample of those leaves. For each requested key the responder either returns -/// the bytes ([`SubtreeByteItem::Present`]) or — if it committed to the key but -/// can no longer produce it — an explicit [`SubtreeByteItem::Absent`], which the -/// auditor counts as a provable failure (committing to bytes you don't hold). +/// proof and now opens up to two 1 KiB blocks (a fresh-random block plus the +/// final block) of a small freshly-random sample of those leaves. For each +/// opening the responder reads the committed chunk and builds a two-chain +/// opening (a Bao verified slice for authenticity against the chunk address, and +/// a nonced block-tree opening for possession against round 1's `nonced_root`), +/// returning [`SubtreeSliceItem::Present`]. If it committed to the key but can no +/// longer +/// produce the bytes it returns [`SubtreeSliceItem::Absent`], which the auditor +/// counts as a provable failure. /// /// A key the responder never committed to (not in the pinned tree) is also /// returned `Absent`: the auditor only ever samples keys it saw in round 1, so -/// in practice this guards against a malformed/forged byte challenge rather than -/// an honest mismatch. -pub async fn handle_subtree_byte_challenge( - challenge: &SubtreeByteChallenge, +/// in practice this guards against a malformed/forged challenge rather than an +/// honest mismatch. +pub async fn handle_subtree_slice_challenge( + challenge: &SubtreeSliceChallenge, storage: &LmdbStorage, self_peer_id: &PeerId, is_bootstrapping: bool, commitment_state: Option<&Arc>, -) -> SubtreeByteResponse { +) -> SubtreeSliceResponse { if is_bootstrapping { - return SubtreeByteResponse::Bootstrapping { + return SubtreeSliceResponse::Bootstrapping { challenge_id: challenge.challenge_id, }; } if challenge.challenged_peer_id != *self_peer_id.as_bytes() { - return SubtreeByteResponse::Rejected { + return SubtreeSliceResponse::Rejected { challenge_id: challenge.challenge_id, kind: RejectKind::Protocol, reason: "challenged_peer_id does not match this node".to_string(), }; } - // An honest auditor batches its sample to MAX_BYTE_CHALLENGE_KEYS per - // challenge so the worst-case response fits the wire cap. Reject larger - // requests up front: serving them could only produce an unencodable - // response (and invites disk-read amplification from a forged auditor). - if challenge.keys.len() > MAX_BYTE_CHALLENGE_KEYS { - let requested = challenge.keys.len(); - return SubtreeByteResponse::Rejected { + // An honest auditor opens at most MAX_SLICE_OPENINGS blocks per challenge. + // Reject larger requests up front: each opening forces a full chunk read to + // build its proof, so an oversized request is a disk-read amplification lever + // for a forged auditor. + if challenge.openings.len() > MAX_SLICE_OPENINGS { + let requested = challenge.openings.len(); + return SubtreeSliceResponse::Rejected { challenge_id: challenge.challenge_id, kind: RejectKind::Protocol, reason: format!( - "byte challenge requests {requested} keys; max {MAX_BYTE_CHALLENGE_KEYS} per challenge" + "slice challenge requests {requested} openings; max {MAX_SLICE_OPENINGS} per challenge" ), }; } let Some(state) = commitment_state else { - return SubtreeByteResponse::Rejected { + return SubtreeSliceResponse::Rejected { challenge_id: challenge.challenge_id, kind: RejectKind::Protocol, reason: "no commitment state".to_string(), @@ -1068,76 +1290,163 @@ pub async fn handle_subtree_byte_challenge( // retain it (rotated past it), reject as `UnknownCommitment`. With audit // grace removed (ADR-0004 A1) the auditor treats a responsive miss on an // in-window pin as a confirmed failure — answerability is restart-durable and - // pins are challenged only in-window. We serve bytes only for keys committed + // pins are challenged only in-window. We open blocks only for keys committed // under this pin. let Some(built) = state.lookup_by_hash(&challenge.expected_commitment_hash) else { - return SubtreeByteResponse::Rejected { + return SubtreeSliceResponse::Rejected { challenge_id: challenge.challenge_id, kind: RejectKind::UnknownCommitment, reason: "unknown commitment hash".to_string(), }; }; - let mut items = Vec::with_capacity(challenge.keys.len()); - for key in &challenge.keys { - // Serve ONLY keys committed under this pin. A key the auditor asks for - // that is not in the pinned tree is `Absent` — never served from local - // storage just because we happen to hold it (§15: serving an - // uncommitted-but-held key would let a forged challenge harvest bytes - // and muddy the possession proof, which must be about THIS commitment). - if built.proof_for(key).is_none() { - items.push(SubtreeByteItem::Absent { key: *key }); + // Coalesce openings by key, preserving first-seen order and deduplicating + // block indices per key, so each committed chunk is read from LMDB and hashed + // at most once even when the auditor opens several of its blocks (the normal + // random + final pair, or a forged duplicate). Without this a ten-opening + // request could re-read and re-hash the same chunk ten times. + let mut key_order: Vec = Vec::new(); + let mut indices_by_key: HashMap> = HashMap::new(); + for opening in &challenge.openings { + let entry = indices_by_key.entry(opening.key).or_default(); + if entry.is_empty() { + key_order.push(opening.key); + } + if !entry.contains(&opening.block_index) { + entry.push(opening.block_index); + } + } + + // Coalescing only saves work when keys REPEAT. A forged auditor could still + // spread its MAX_SLICE_OPENINGS across that many DISTINCT keys to force a + // full-chunk read each (up to ~40 MiB). An honest auditor samples at most + // BYTE_SPOTCHECK_MAX leaves, so cap distinct keys to that. + if key_order.len() > BYTE_SPOTCHECK_MAX as usize { + let distinct = key_order.len(); + return SubtreeSliceResponse::Rejected { + challenge_id: challenge.challenge_id, + kind: RejectKind::Protocol, + reason: format!( + "slice challenge spans {distinct} distinct keys; max {BYTE_SPOTCHECK_MAX}" + ), + }; + } + + let mut items = Vec::with_capacity(challenge.openings.len()); + for key in key_order { + let indices = indices_by_key.remove(&key).unwrap_or_default(); + // Open ONLY keys committed under this pin. A key not in the pinned tree + // is `Absent` — never served from local storage just because we happen to + // hold it (§15: serving an uncommitted-but-held key would let a forged + // challenge harvest data and muddy the possession proof for THIS commit). + if built.proof_for(&key).is_none() { + items.push(SubtreeSliceItem::Absent { key }); continue; } - match get_raw_retrying(storage, key).await { - // Committed key, bytes present → serve them. - Ok(Some(bytes)) => items.push(SubtreeByteItem::Present { key: *key, bytes }), - // Committed key, definitively absent → provable failure (§7: this is - // a real "I don't hold it" answer, distinct from a read error). - Ok(None) => { - warn!( - "Subtree byte audit: committed key {} requested but bytes absent", - hex::encode(key) - ); - items.push(SubtreeByteItem::Absent { key: *key }); - } - // Persistent transient read error after retries → do NOT brand the - // peer a deleter. Reject `Transient`; the auditor routes it to the - // timeout lane so a flaky LMDB read never manufactures a confirmed - // possession failure on an honest holder (which also gains no credit). - Err(e) => { - warn!( - "Subtree byte audit: storage read error for committed key {}: {e} \ - (rejecting as transient, not a confirmed failure)", - hex::encode(key) - ); - return SubtreeByteResponse::Rejected { - challenge_id: challenge.challenge_id, - kind: RejectKind::Transient, - reason: format!("transient storage read error: {e}"), - }; - } + match serve_committed_key_openings(challenge, storage, key, indices).await { + KeyServe::Items(mut built_items) => items.append(&mut built_items), + KeyServe::Absent => items.push(SubtreeSliceItem::Absent { key }), + KeyServe::Reject(reject) => return reject, } } - SubtreeByteResponse::Items { + SubtreeSliceResponse::Items { challenge_id: challenge.challenge_id, items, } } +/// Outcome of serving all requested openings for one committed key. +enum KeyServe { + /// Openings built for this key; append to the response. + Items(Vec), + /// Committed but the bytes are gone → provable `Absent`. + Absent, + /// A terminal condition (out-of-range index, read/build error) that aborts + /// the whole response. + Reject(SubtreeSliceResponse), +} + +/// Read a committed key's chunk once and build every requested opening from it. +/// +/// The Bao outboard + nonced tree hash the whole chunk, so the CPU-heavy build +/// runs on a blocking thread to keep an audit-responder flood off the Tokio pool. +/// `indices` is already deduplicated by the caller. +async fn serve_committed_key_openings( + challenge: &SubtreeSliceChallenge, + storage: &LmdbStorage, + key: XorName, + indices: Vec, +) -> KeyServe { + let reject = |kind, reason| { + KeyServe::Reject(SubtreeSliceResponse::Rejected { + challenge_id: challenge.challenge_id, + kind, + reason, + }) + }; + match get_raw_retrying(storage, &key).await { + Ok(Some(bytes)) => { + let nonce = challenge.nonce; + let peer = challenge.challenged_peer_id; + match tokio::task::spawn_blocking(move || { + build_slice_items_for_key(nonce, peer, key, &bytes, &indices) + }) + .await + { + Ok(Ok(built_items)) => KeyServe::Items(built_items), + Ok(Err((kind, reason))) => reject(kind, reason), + Err(e) => { + warn!( + "Subtree slice audit: proof build task failed for key {}: {e}", + hex::encode(key) + ); + reject( + RejectKind::Transient, + format!("proof build task error: {e}"), + ) + } + } + } + // Committed key, definitively absent → provable failure (§7: a real "I + // don't hold it" answer, distinct from a read error). + Ok(None) => { + warn!( + "Subtree slice audit: committed key {} requested but bytes absent", + hex::encode(key) + ); + KeyServe::Absent + } + // Persistent transient read error after retries → do NOT brand the peer a + // deleter. Reject `Transient`; the auditor routes it to the timeout lane + // so a flaky LMDB read never manufactures a confirmed possession failure + // on an honest holder (which also gains no credit). + Err(e) => { + warn!( + "Subtree slice audit: storage read error for committed key {}: {e} \ + (rejecting as transient, not a confirmed failure)", + hex::encode(key) + ); + reject( + RejectKind::Transient, + format!("transient storage read error: {e}"), + ) + } + } +} + #[cfg(test)] #[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] mod tests { use super::*; use crate::replication::commitment_state::BuiltCommitment; - use crate::replication::subtree::{build_subtree_proof, nonced_leaf_hash, SubtreeLeaf}; + use crate::replication::subtree::{build_subtree_proof, SubtreeLeaf}; use saorsa_pqc::api::sig::ml_dsa_65; /// ADR-0004 A1 grade flip (grace removed): a responsive `UnknownCommitment` /// or `Protocol` rejection is a CONFIRMED failure; only `Transient` routes to /// the timeout lane. This pure decision backs both audit rounds - /// (`Confirmed → AuditFailureReason::Rejected` / `ByteRound::Rejected`; + /// (`Confirmed → AuditFailureReason::Rejected` / `SliceRound::Rejected`; /// `TimeoutLane → AuditFailureReason::Timeout` + pinned-credit revocation). #[test] fn grade_reject_removes_grace_for_unknown_commitment() { @@ -1160,38 +1469,119 @@ mod tests { // structural root rebuild), // - sampling: `random_spotcheck_leaves` (3..=5 FRESHLY-RANDOM leaves chosen // after the proof is in hand — see its doc for the soundness argument), and - // - round 2: `verify_byte_response` (recompute content-address + freshness - // from the bytes the RESPONDER served — the auditor holds nothing). + // - round 2: `verify_slice_response` (decode the Bao slice against the chunk + // address + fold the nonced opening to the round-1 `nonced_root`, both from + // what the RESPONDER served — the auditor holds nothing). fn key(i: u32) -> XorName { let mut k = [0u8; 32]; k[..4].copy_from_slice(&i.to_be_bytes()); k } - /// The "chunk content" for a key in these fixtures. The committed tree's leaf - /// `bytes_hash` is `BLAKE3(chunk_bytes(key))`, mirroring the general - /// `(key, BLAKE3(content))` commitment; round 2 serves exactly this content. - fn chunk_bytes(k: &XorName) -> Vec { - let mut v = k.to_vec(); - v.extend_from_slice(b"chunk-body"); + + // The auditor's per-opening matcher returns the FIRST item with a matching + // identity, so an oversized, unsolicited, or colliding response must be + // rejected as malformed BEFORE matching — otherwise item order (or padding) + // could decide the verdict or force quadratic validation work. + #[test] + fn verify_slice_response_rejects_malformed_item_sets() { + let leaf = |k: XorName| SubtreeLeaf { + key: k, + bytes_hash: [0u8; 32], + content_len: 0, + nonced_root: [0u8; 32], + }; + let present = |k: XorName, i: u32| SubtreeSliceItem::Present { + key: k, + block_index: i, + bao_slice: vec![], + nonced_siblings: vec![], + }; + let malformed = |verdict| { + matches!( + verdict, + AuditVerdict::Fail(AuditFailureReason::MalformedResponse) + ) + }; + let nonce = [0u8; 32]; + let peer = [0u8; 32]; + // The auditor requested (key1, block 0), (key1, block 1), (key2, block 0). + let openings = vec![(leaf(key(1)), 0u32), (leaf(key(1)), 1), (leaf(key(2)), 0)]; + + // More items than requested openings. + let oversized = vec![ + present(key(1), 0), + present(key(1), 1), + present(key(2), 0), + present(key(2), 0), + ]; + assert!(malformed(verify_slice_response( + &openings, &nonce, &peer, &oversized + ))); + + // An item that was not requested. + let unsolicited = vec![present(key(9), 0)]; + assert!(malformed(verify_slice_response( + &openings, + &nonce, + &peer, + &unsolicited + ))); + + // Duplicate (key, block_index). + let dup = vec![present(key(1), 0), present(key(1), 0)]; + assert!(malformed(verify_slice_response( + &openings, &nonce, &peer, &dup + ))); + + // A key that is both Present and Absent. + let conflict = vec![present(key(1), 0), SubtreeSliceItem::Absent { key: key(1) }]; + assert!(malformed(verify_slice_response( + &openings, &nonce, &peer, &conflict + ))); + + // A well-formed, unique, solicited response passes the identity guard (the + // empty proofs then fail verification as DigestMismatch, NOT malformed). + let ok = vec![present(key(1), 0), present(key(1), 1), present(key(2), 0)]; + assert!(!malformed(verify_slice_response( + &openings, &nonce, &peer, &ok + ))); + } + /// Deterministic chunk content for fixture index `i`. Fixture keys are + /// CONTENT-ADDRESSED (`ckey(i) == BLAKE3(chunk_bytes(i))`), so a committed + /// leaf is `(key, key)` exactly as production, and round 2 serves this content. + fn chunk_bytes(i: u32) -> Vec { + let mut v = b"chunk-body".to_vec(); + v.extend_from_slice(&i.to_le_bytes()); v } + /// Content-addressed key for fixture index `i` (so `bytes_hash == key`). + fn ckey(i: u32) -> XorName { + *blake3::hash(&chunk_bytes(i)).as_bytes() + } + + /// The content behind a committed fixture key (reverse of `ckey`), so round-2 + /// fixtures can serve the real bytes for any sampled leaf. + fn content_for_key(k: &XorName) -> Vec { + (0..16_384u32) + .find(|&i| &ckey(i) == k) + .map(chunk_bytes) + .expect("fixture content for committed key") + } + /// Build an honest committed tree of `n` keys + a valid round-1 proof for /// `nonce`. Returns `(built, proof, peer_id)`. The auditor pins `built.hash()`. fn honest(n: u32, nonce: &[u8; 32]) -> (BuiltCommitment, SubtreeProof, [u8; 32]) { let (pk, sk) = ml_dsa_65().generate_keypair().unwrap(); let peer_id = *blake3::hash(&pk.to_bytes()).as_bytes(); let pk_b = pk.to_bytes(); - let entries: Vec<_> = (0..n) - .map(|i| { - let k = key(i); - (k, *blake3::hash(&chunk_bytes(&k)).as_bytes()) - }) - .collect(); + // Content-addressed: bytes_hash == key, exactly as production commits. + let entries: Vec<_> = (0..n).map(|i| (ckey(i), ckey(i))).collect(); let built = BuiltCommitment::build(entries, &peer_id, &sk, &pk_b).unwrap(); let proof = - build_subtree_proof(built.tree(), nonce, &peer_id, |k| Some(chunk_bytes(k))).unwrap(); + build_subtree_proof(built.tree(), nonce, &peer_id, |k| Some(content_for_key(k))) + .unwrap(); (built, proof, peer_id) } @@ -1205,8 +1595,8 @@ mod tests { evaluate_subtree_structure(built.commitment(), proof, nonce, &built.hash(), peer) } - /// The 3..=5 spot-check leaves the auditor would demand bytes for in round 2. - /// Now freshly-random (post-proof) rather than nonce-derived; the `_nonce`/ + /// The 3..=5 spot-check leaves the auditor would open in round 2. Now + /// freshly-random (post-proof) rather than nonce-derived; the `_nonce`/ /// `_key_count` params are kept so existing call sites read unchanged. fn sample<'a>( proof: &'a SubtreeProof, @@ -1216,12 +1606,43 @@ mod tests { random_spotcheck_leaves(proof, 8u32.clamp(BYTE_SPOTCHECK_MIN, BYTE_SPOTCHECK_MAX)) } - // A round-2 `served` closure that returns the HONEST content for every key. - // The nested-Option shape is the `verify_byte_response` callback contract: - // Present{bytes} -> Some(Some(bytes)); Absent -> Some(None); omitted -> None. - #[allow(clippy::option_option, clippy::unnecessary_wraps)] - fn served_honest(key: &XorName) -> Option>> { - Some(Some(chunk_bytes(key))) + /// Pair each sampled leaf with a block index for round 2. The fixtures use + /// short (single-block) chunks, so block 0 is the only block; the production + /// auditor draws a fresh random index via `random_block_index`. + fn openings_for(sample: &[&SubtreeLeaf]) -> Vec<(SubtreeLeaf, u32)> { + sample.iter().map(|l| ((*l).clone(), 0u32)).collect() + } + + /// Honest responder: for each opening build a real Bao slice + nonced opening + /// from the true chunk content, exactly as `handle_subtree_slice_challenge` + /// would. + fn served_honest_items( + openings: &[(SubtreeLeaf, u32)], + nonce: &[u8; 32], + peer: &[u8; 32], + ) -> Vec { + openings + .iter() + .map(|(leaf, block_index)| { + let content = content_for_key(&leaf.key); + let bao_slice = + crate::replication::slice::extract_block_slice(&content, *block_index).unwrap(); + let nonced_siblings = crate::replication::slice::nonced_block_siblings( + nonce, + peer, + &leaf.key, + &content, + *block_index, + ) + .unwrap(); + SubtreeSliceItem::Present { + key: leaf.key, + block_index: *block_index, + bao_slice, + nonced_siblings, + } + }) + .collect() } // ---- round 1: structure -------------------------------------------------- @@ -1232,15 +1653,44 @@ mod tests { let (built, proof, peer) = honest(400, &nonce); // Round 1. assert!(structure(&built, &proof, &nonce, &peer).is_ok()); - // Round 2: honest responder serves the real content for the sample. + // Round 2: honest responder opens real slices for the sample. let s = sample(&proof, &nonce, built.commitment().key_count); assert!(!s.is_empty()); - match verify_byte_response(&s, &nonce, &peer, served_honest) { + let openings = openings_for(&s); + let items = served_honest_items(&openings, &nonce, &peer); + match verify_slice_response(&openings, &nonce, &peer, &items) { AuditVerdict::Pass { checked } => assert!(checked >= 1, "must verify >=1 leaf"), other @ AuditVerdict::Fail(_) => panic!("expected Pass, got {other:?}"), } } + /// Possession-forgery guard: a peer that signs a commitment whose leaves + /// decouple the credited `key` from the authenticated content hash + /// (`bytes_hash != key`) is rejected at round 1 — even though the structural + /// root still rebuilds from `(key, bytes_hash)`. Without the guard such a peer + /// could earn holder credit for an expensive address `key` while only proving + /// possession of an unrelated (e.g. one-byte) chunk hashing to `bytes_hash`. + #[test] + fn leaf_with_bytes_hash_decoupled_from_key_is_rejected() { + let nonce = [5u8; 32]; + let (pk, sk) = ml_dsa_65().generate_keypair().unwrap(); + let peer_id = *blake3::hash(&pk.to_bytes()).as_bytes(); + let pk_b = pk.to_bytes(); + // Every leaf commits (key, bytes_hash) with bytes_hash != key. + let entries: Vec<_> = (0..64u32).map(|i| (ckey(i), ckey(i + 10_000))).collect(); + let built = BuiltCommitment::build(entries, &peer_id, &sk, &pk_b).unwrap(); + let proof = + build_subtree_proof(built.tree(), &nonce, &peer_id, |k| Some(content_for_key(k))) + .unwrap(); + // The structural root rebuilds (it is a genuinely signed tree), so the + // decoupled-address gate is what must reject it. + assert_eq!( + structure(&built, &proof, &nonce, &peer_id), + Err(AuditFailureReason::DigestMismatch), + "a leaf whose bytes_hash != key must be rejected at round 1" + ); + } + #[test] fn commitment_bound_to_another_peer_rejected() { let nonce = [3u8; 32]; @@ -1301,76 +1751,88 @@ mod tests { let (built, proof, peer) = honest(400, &nonce); assert!(structure(&built, &proof, &nonce, &peer).is_ok()); let s = sample(&proof, &nonce, built.commitment().key_count); - // Responder returns Absent for the FIRST sampled key, honest for the rest. - let victim = s.first().map(|l| l.key).unwrap(); - let v = verify_byte_response(&s, &nonce, &peer, |k| { - if *k == victim { - Some(None) // explicit Absent - } else { - Some(Some(chunk_bytes(k))) - } - }); + let openings = openings_for(&s); + // Responder returns Absent for the FIRST opening, honest for the rest. + let victim = openings.first().map(|(l, _)| l.key).unwrap(); + let mut items = served_honest_items(&openings, &nonce, &peer); + if let Some(slot) = items.first_mut() { + *slot = SubtreeSliceItem::Absent { key: victim }; + } + let v = verify_slice_response(&openings, &nonce, &peer, &items); assert_eq!(v, AuditVerdict::Fail(AuditFailureReason::DigestMismatch)); } #[test] fn omitted_committed_key_is_confirmed_failure() { - // A responder that simply omits a sampled committed key from its items + // A responder that simply omits a sampled committed opening from its items // (neither Present nor Absent) is treated identically to Absent: it // committed to the key and won't serve it → confirmed failure. let nonce = [9u8; 32]; let (built, proof, peer) = honest(400, &nonce); let s = sample(&proof, &nonce, built.commitment().key_count); - let victim = s.first().map(|l| l.key).unwrap(); - let v = verify_byte_response(&s, &nonce, &peer, |k| { - if *k == victim { - None // omitted entirely - } else { - Some(Some(chunk_bytes(k))) - } - }); + let openings = openings_for(&s); + let mut items = served_honest_items(&openings, &nonce, &peer); + items.remove(0); // omit the first opening entirely + let v = verify_slice_response(&openings, &nonce, &peer, &items); assert_eq!(v, AuditVerdict::Fail(AuditFailureReason::DigestMismatch)); } #[test] fn fake_storage_garbage_bytes_is_confirmed_failure() { - // A "fake-storage" responder claims possession but serves garbage. The - // garbage does not hash to the committed content address (`bytes_hash`), - // so the round-2 content-address check fails → confirmed failure. No - // auditor holdings involved. + // A "fake-storage" responder claims possession but opens a slice built + // from garbage content. The garbage slice does not decode against the + // committed content address (`bytes_hash`), so chain 1 fails → confirmed + // failure. No auditor holdings involved. let nonce = [9u8; 32]; let (built, proof, peer) = honest(400, &nonce); let s = sample(&proof, &nonce, built.commitment().key_count); - let v = verify_byte_response(&s, &nonce, &peer, |k| { - let mut garbage = blake3::hash(k).as_bytes().to_vec(); - garbage.extend_from_slice(b"adversary-fake-storage"); - Some(Some(garbage)) - }); + let openings = openings_for(&s); + let items: Vec = openings + .iter() + .map(|(leaf, bi)| { + let mut garbage = blake3::hash(&leaf.key).as_bytes().to_vec(); + garbage.extend_from_slice(b"adversary-fake-storage"); + let bao_slice = + crate::replication::slice::extract_block_slice(&garbage, *bi).unwrap(); + let nonced_siblings = crate::replication::slice::nonced_block_siblings( + &nonce, &peer, &leaf.key, &garbage, *bi, + ) + .unwrap(); + SubtreeSliceItem::Present { + key: leaf.key, + block_index: *bi, + bao_slice, + nonced_siblings, + } + }) + .collect(); + let v = verify_slice_response(&openings, &nonce, &peer, &items); assert_eq!(v, AuditVerdict::Fail(AuditFailureReason::DigestMismatch)); } #[test] - fn correct_content_address_but_stale_freshness_fails() { - // Suppose a responder could serve bytes that hash to the content address - // (it holds the chunk) — then BOTH checks pass; that is honest. But if - // it serves bytes whose freshness hash does not match (e.g. replaying a - // different nonce's digest is impossible since we recompute it here), the - // freshness check must catch any content that doesn't reproduce the - // committed `nonced_hash`. We model a leaf whose committed nonced_hash was - // built under a DIFFERENT nonce, so the audit nonce's recompute differs. + fn correct_content_address_but_stale_nonced_root_fails() { + // A responder can serve the real block (chain 1, the Bao slice against the + // address, passes), but if its committed `nonced_root` does not correspond + // to the audit's nonce over that content, the nonced opening (chain 2) + // cannot fold to it. We model a leaf whose committed `nonced_root` was + // built under a DIFFERENT nonce; the honest opening under the audit nonce + // then fails to match it. let nonce = [9u8; 32]; let (built, mut proof, peer) = honest(400, &nonce); - // Rewrite EVERY leaf's nonced_hash to one bound to a different nonce but - // keep its bytes_hash correct (so each leaf's content-address check is - // fine; only freshness is wrong). Tampering all leaves means the - // freshly-random sample is guaranteed to land on a stale-freshness leaf. let other_nonce = [0xEEu8; 32]; for leaf in &mut proof.leaves { - leaf.nonced_hash = - nonced_leaf_hash(&other_nonce, &peer, &leaf.key, &chunk_bytes(&leaf.key)); + leaf.nonced_root = crate::replication::slice::nonced_block_root( + &other_nonce, + &peer, + &leaf.key, + &content_for_key(&leaf.key), + ); } let s = sample(&proof, &nonce, built.commitment().key_count); - let v = verify_byte_response(&s, &nonce, &peer, served_honest); + let openings = openings_for(&s); + let items = served_honest_items(&openings, &nonce, &peer); + let v = verify_slice_response(&openings, &nonce, &peer, &items); assert_eq!(v, AuditVerdict::Fail(AuditFailureReason::DigestMismatch)); } @@ -1384,8 +1846,13 @@ mod tests { let (built, proof, peer) = honest(256, &nonce); assert!(structure(&built, &proof, &nonce, &peer).is_ok()); let s = sample(&proof, &nonce, built.commitment().key_count); - // Responder is a total deleter: Absent for everything. - let v = verify_byte_response(&s, &nonce, &peer, |_| Some(None)); + let openings = openings_for(&s); + // Responder is a total deleter: Absent for every opening. + let items: Vec = openings + .iter() + .map(|(l, _)| SubtreeSliceItem::Absent { key: l.key }) + .collect(); + let v = verify_slice_response(&openings, &nonce, &peer, &items); assert_eq!(v, AuditVerdict::Fail(AuditFailureReason::DigestMismatch)); } @@ -1410,8 +1877,10 @@ mod tests { let nonce = [11u8; 32]; let (built, proof, peer) = honest(400, &nonce); let s = sample(&proof, &nonce, built.commitment().key_count); - match verify_byte_response(&s, &nonce, &peer, served_honest) { - AuditVerdict::Pass { checked } => assert_eq!(checked, s.len()), + let openings = openings_for(&s); + let items = served_honest_items(&openings, &nonce, &peer); + match verify_slice_response(&openings, &nonce, &peer, &items) { + AuditVerdict::Pass { checked } => assert_eq!(checked, openings.len()), other @ AuditVerdict::Fail(_) => panic!("expected Pass, got {other:?}"), } } @@ -1420,9 +1889,9 @@ mod tests { #[test] fn structure_fail_short_circuits_before_round_2() { - // A structurally invalid proof is rejected in round 1; the byte challenge + // A structurally invalid proof is rejected in round 1; the slice challenge // is never issued. We assert the round-1 gate returns Err so the auditor - // (verify_subtree_response) never reaches request_byte_proof. + // (verify_subtree_response) never reaches request_slice_proof. let nonce = [5u8; 32]; let (built, mut proof, peer) = honest(300, &nonce); if let Some(first) = proof.leaves.first_mut() { @@ -1431,23 +1900,27 @@ mod tests { assert!(structure(&built, &proof, &nonce, &peer).is_err()); } - /// Build an honest committed tree whose keys are deliberately "FAR": their - /// addresses live at the high end of the XOR space (top bytes = 0xFF). On the - /// auditor side these are the leaves `observe_closeness` counts toward `far`. + /// Build an honest committed tree whose keys are content-addressed but biased + /// to the FAR half of the XOR space (top bit set), so `observe_closeness` + /// counts them toward `far`. Keys stay content-addressed (`bytes_hash == key`) + /// so round 2 serves real bytes. fn honest_far(n: u32, nonce: &[u8; 32]) -> (BuiltCommitment, SubtreeProof, [u8; 32]) { let (pk, sk) = ml_dsa_65().generate_keypair().unwrap(); let peer_id = *blake3::hash(&pk.to_bytes()).as_bytes(); let pk_b = pk.to_bytes(); - let entries: Vec<_> = (0..n) - .map(|i| { - let mut k = [0xFFu8; 32]; - k[28..].copy_from_slice(&i.to_be_bytes()); - (k, *blake3::hash(&chunk_bytes(&k)).as_bytes()) - }) - .collect(); + let mut entries: Vec<(XorName, [u8; 32])> = Vec::new(); + let mut i = 0u32; + while entries.len() < n as usize { + let k = ckey(i); + if k[0] >= 0x80 { + entries.push((k, k)); + } + i = i.saturating_add(1); + } let built = BuiltCommitment::build(entries, &peer_id, &sk, &pk_b).unwrap(); let proof = - build_subtree_proof(built.tree(), nonce, &peer_id, |k| Some(chunk_bytes(k))).unwrap(); + build_subtree_proof(built.tree(), nonce, &peer_id, |k| Some(content_for_key(k))) + .unwrap(); (built, proof, peer_id) } @@ -1461,12 +1934,24 @@ mod tests { let (built_far, proof_far, peer_far) = honest_far(400, &nonce); assert!(structure(&built_far, &proof_far, &nonce, &peer_far).is_ok()); let sf = sample(&proof_far, &nonce, built_far.commitment().key_count); - let v_far = verify_byte_response(&sf, &nonce, &peer_far, served_honest); + let of = openings_for(&sf); + let v_far = verify_slice_response( + &of, + &nonce, + &peer_far, + &served_honest_items(&of, &nonce, &peer_far), + ); let (built_near, proof_near, peer_near) = honest(400, &nonce); assert!(structure(&built_near, &proof_near, &nonce, &peer_near).is_ok()); let sn = sample(&proof_near, &nonce, built_near.commitment().key_count); - let v_near = verify_byte_response(&sn, &nonce, &peer_near, served_honest); + let on = openings_for(&sn); + let v_near = verify_slice_response( + &on, + &nonce, + &peer_near, + &served_honest_items(&on, &nonce, &peer_near), + ); match (&v_far, &v_near) { (AuditVerdict::Pass { checked: cf }, AuditVerdict::Pass { checked: cn }) => { @@ -1486,7 +1971,8 @@ mod tests { let _l = SubtreeLeaf { key: key(1), bytes_hash: [0u8; 32], - nonced_hash: [0u8; 32], + content_len: 0, + nonced_root: [0u8; 32], }; } } diff --git a/src/replication/subtree.rs b/src/replication/subtree.rs index 1aa73faf..2fc8e2ee 100644 --- a/src/replication/subtree.rs +++ b/src/replication/subtree.rs @@ -14,10 +14,11 @@ //! 1. **Structure** — [`verify_subtree_proof`] re-derives the selected branch //! from `(nonce, key_count)`, rebuilds the root from the returned leaves and //! cut-hashes, and requires it to equal the pinned root. -//! 2. **Real bytes** — [`select_spotcheck_indices`] picks a few leaves within -//! the subtree; the caller fetches their bytes and checks both the plain -//! content hash and the nonce freshness hash. Faking a fraction `x` of -//! leaves survives only `(1 - x)^k`. +//! 2. **Verified slice** — [`select_spotcheck_indices`] picks a few leaves +//! within the subtree; the caller opens one random block per leaf and checks +//! a Bao slice against the chunk address plus a nonced-tree opening against +//! the round-1 keyed `nonced_root`. Faking a fraction `x` of leaves survives +//! only `(1 - x)^k`. //! //! ## Tree geometry (must match [`super::commitment::MerkleTree`]) //! @@ -31,7 +32,6 @@ //! intersected with `0..N`. use super::commitment::{leaf_hash, node_hash, StorageCommitment, MAX_COMMITMENT_KEY_COUNT}; -use super::protocol::compute_audit_digest; use crate::ant_protocol::XorName; use serde::{Deserialize, Serialize}; @@ -44,14 +44,29 @@ pub const SMALL_TREE_FULL_AUDIT_FLOOR: u32 = 4; pub struct SubtreeLeaf { /// The committed key (chunk address) at this leaf position. pub key: XorName, - /// `BLAKE3(record_bytes)` — the plain content hash. This is also the - /// chunk's network address, so it is public; possessing it does NOT prove - /// possession of the bytes (that is what `nonced_hash` is for). + /// `BLAKE3(record_bytes)` — the plain content hash. For a content-addressed + /// chunk this EQUALS `key` (the address IS the content hash), and the round-1 + /// verifier enforces `bytes_hash == key` so credit for `key` cannot be earned + /// by proving possession of unrelated bytes. It is public; possessing it does + /// NOT prove possession of the bytes (that is what `nonced_root` is for). It is + /// also the BLAKE3/Bao root the round-2 slice verifies against. pub bytes_hash: [u8; 32], - /// `compute_audit_digest(nonce, peer_id, key, record_bytes)` — the - /// freshness hash. Only a holder of the actual bytes can produce it for a - /// fresh nonce, so a spot-check on it proves real possession. - pub nonced_hash: [u8; 32], + /// Length of the chunk's content, in bytes. Lets the auditor draw a random + /// 1 KiB block index in range and size the Bao slice for the final (short) + /// block. This field is NOT bound by the signed commitment (the Merkle leaf + /// hashes only `key ‖ bytes_hash`), so the round-2 verifier must not trust it + /// blind: `verify_block_slice` authenticates it against the Bao slice's own + /// length header (validated against the address), rejecting any claim that + /// disagrees with the true content length. Without that check a deflated + /// `content_len` would collapse the challenge to block 0 and forge possession + /// of a large chunk from a ~1 KiB prefix slice. + pub content_len: u32, + /// Root of the responder's fresh **nonced block tree** for this chunk (see + /// [`crate::replication::slice`]): a Merkle root over the chunk's 1 KiB + /// blocks whose leaves each bind the fresh nonce, peer, key and block bytes. + /// Building it requires every byte of the chunk under the fresh nonce, so it + /// commits real possession; round 2 opens one random block against it. + pub nonced_root: [u8; 32], } /// A responder's single-contiguous-subtree proof (ADR-0002 "The proof"). @@ -108,7 +123,8 @@ fn tree_depth(key_count: u32) -> Option { /// leaves. Pure function of geometry — identical on auditor and responder. /// /// `span = 2^(total_depth - depth)`; the node covers `[slot*span, (slot+1)*span)` -/// clamped to `0..key_count`. +/// clamped to `0..key_count`. Test-only geometry helper. +#[cfg(test)] #[must_use] fn real_leaves_under(depth: u32, slot: u64, key_count: u32, total_depth: u32) -> u32 { let levels_below = total_depth - depth; @@ -144,27 +160,44 @@ fn sqrt_floor(key_count: u32) -> u32 { u32::try_from(ceil.max(1)).unwrap_or(u32::MAX) } -/// Read bit `index` of the nonce (bit 0 = MSB of byte 0), `index` 0-based. +/// A stable `u64` drawn from the first 8 bytes of the nonce, used to pick the +/// subtree slot uniformly. The nonce is fresh CSPRNG per audit, so modulo bias +/// over the small slot count (≤ ~1000) is negligible. +#[must_use] +fn nonce_u64(nonce: &[u8; 32]) -> u64 { + let mut b = [0u8; 8]; + b.copy_from_slice(&nonce[..8]); + u64::from_le_bytes(b) +} + +/// The largest real-leaf count any selected subtree can have for `key_count`. /// -/// `1 → left child, 0 → right child` (ADR). With a 256-bit nonce and a tree -/// depth ≤ 20 we never run out of bits. +/// A full fixed-depth block of `next_power_of_two(ceil(sqrt(key_count)))` leaves. +/// Bounds a single round-1 request's read/hash work (≤ 1024 leaves at the +/// `MAX_COMMITMENT_KEY_COUNT` cap). Used both to size selection and as a +/// pre-read defense-in-depth ceiling in [`subtree_plan`]. #[must_use] -fn nonce_bit(nonce: &[u8; 32], index: u32) -> bool { - let byte = (index / 8) as usize; - let bit = 7 - (index % 8); - // byte < 32 because index < 256 for any reachable depth; guard anyway. - nonce.get(byte).is_some_and(|b| (b >> bit) & 1 == 1) +pub fn max_subtree_leaves(key_count: u32) -> u32 { + if key_count <= SMALL_TREE_FULL_AUDIT_FLOOR { + return key_count; + } + sqrt_floor(key_count).next_power_of_two() } /// Deterministically select one contiguous subtree from `(nonce, key_count)`. /// -/// Walks the nonce bits from the root, descending into the child the bit picks, -/// and **stops at the smallest branch whose real-leaf count is still ≥ -/// `ceil(sqrt(key_count))`**. Because an all-padding child has zero real leaves -/// (< the floor), the walk never descends into one — so the selection always -/// covers ≥ `sqrt` real leaves and is never empty (ADR dead-block fix). +/// Picks one **fixed-depth block** of `span = next_power_of_two(ceil(sqrt(N)))` +/// leaves, choosing the block uniformly by `nonce_u64 % slot_count`. Every block +/// (including the short tail) is selectable with roughly equal probability, so +/// the whole tree is covered over many audits, and a single request always reads +/// at most `span` (≤ ~sqrt(N), ≤ 1024 at the cap) leaves — regardless of the +/// nonce. This replaces the old nonce-bit descent, whose stop-at-parent rule let +/// an adversarial nonce select a subtree far larger than `sqrt(N)` (up to the +/// whole tree at `2^k + 1`), an unbounded round-1 work amplification. /// -/// For `key_count <= SMALL_TREE_FULL_AUDIT_FLOOR` the whole tree is selected. +/// Unpredictability is preserved: the responder cannot know the selected slot +/// until it receives the fresh nonce. For `key_count <= SMALL_TREE_FULL_AUDIT_FLOOR` +/// the whole tree is selected. /// /// Returns `None` only for an out-of-protocol `key_count` (caller rejects). #[must_use] @@ -181,36 +214,28 @@ pub fn select_subtree_path(nonce: &[u8; 32], key_count: u32) -> Option= sqrt(N), aligned to a tree level so the block IS a + // single subtree node at `depth = total_depth - log2(span)`. + let span = sqrt_floor(key_count).next_power_of_two(); + let span_log2 = span.trailing_zeros(); + // span <= next_power_of_two(key_count) == 2^total_depth, so span_log2 <= + // total_depth and depth is non-negative. + let depth = total_depth.saturating_sub(span_log2); + let slot_count = key_count.div_ceil(span); + let slot = u32::try_from(nonce_u64(nonce) % u64::from(slot_count)).unwrap_or(0); + + let span64 = u64::from(span); + let leaf_start = u32::try_from( + u64::from(slot) + .saturating_mul(span64) .min(u64::from(key_count)), ) .unwrap_or(key_count); + let leaf_end = leaf_start.saturating_add(span).min(key_count); Some(SubtreePath { depth, - slot: u32::try_from(slot).unwrap_or(u32::MAX), + slot, leaf_start, leaf_end, }) @@ -221,13 +246,14 @@ pub fn select_subtree_path(nonce: &[u8; 32], key_count: u32) -> Option, levels: u32) -> [u8; 32] { level.first().copied().unwrap_or([0u8; 32]) } -/// Build the per-leaf nonced freshness hash for a subtree leaf (responder -/// side), reusing the existing audit digest. -#[must_use] -pub fn nonced_leaf_hash( - nonce: &[u8; 32], - challenged_peer_id: &[u8; 32], - key: &XorName, - record_bytes: &[u8], -) -> [u8; 32] { - compute_audit_digest(nonce, challenged_peer_id, key, record_bytes) -} - /// Why a responder could not build a subtree proof for a challenge. #[derive(Debug, Clone, PartialEq, Eq)] pub enum BuildProofError { @@ -505,6 +519,14 @@ pub fn subtree_plan( let key_count = tree.key_count(); let path = select_subtree_path(nonce, key_count).ok_or(BuildProofError::BadKeyCount)?; + // Defense-in-depth: the fixed-depth selection already bounds this to one + // block of `max_subtree_leaves` leaves, but reject BEFORE any leaf read if a + // future selection change ever produced an oversized subtree, so one request + // can never force unbounded round-1 read/hash work. + if path.real_leaf_count() > max_subtree_leaves(key_count) { + return Err(BuildProofError::BadKeyCount); + } + let mut leaf_keys = Vec::with_capacity(path.real_leaf_count() as usize); for idx in path.leaf_start..path.leaf_end { let key = tree @@ -513,17 +535,16 @@ pub fn subtree_plan( leaf_keys.push(key); } - // Sibling cut-hashes, root-first. At descent step `d` (0-based from the - // root), the chosen child is on the side the nonce bit picks; the sibling - // is the other child at level `total_depth - (d + 1)` (counting up from - // leaves). On an odd-length level the missing sibling self-pairs, i.e. the - // sibling hash is the chosen node itself. + // Sibling cut-hashes, root-first. The fixed-depth slot selection no longer + // follows a per-level nonce walk, so derive the on-path node at each level + // from `path.slot`'s prefix bits: the top `d + 1` bits give the node index at + // level `d + 1` below the root. The sibling is the other child at level + // `total_depth - (d + 1)` (counting up from leaves); on an odd level the + // missing sibling self-pairs (the chosen node is its own sibling). let total_depth = u32::try_from(tree.levels_count().saturating_sub(1)).unwrap_or(0); let mut sibling_cut_hashes = Vec::with_capacity(path.depth as usize); - let mut slot = 0u64; for d in 0..path.depth { - let go_left = nonce_bit(nonce, d); - let child = slot * 2 + u64::from(!go_left); + let child = u64::from(path.slot) >> (path.depth - (d + 1)); let sibling = child ^ 1; let level_from_leaves = (total_depth - (d + 1)) as usize; let chosen_hash = tree.node_at(level_from_leaves, child); @@ -532,7 +553,6 @@ pub fn subtree_plan( .or(chosen_hash) .ok_or(BuildProofError::BadKeyCount)?; sibling_cut_hashes.push(sib_hash); - slot = child; } Ok(SubtreePlan { @@ -542,6 +562,10 @@ pub fn subtree_plan( } /// Build one subtree leaf from its key and the chunk bytes the responder holds. +/// +/// The plain `bytes_hash` is the chunk's content address; the `nonced_root` is +/// the fresh per-audit nonced block-tree root over the same bytes, committing +/// possession at round-1 time (see [`crate::replication::slice`]). #[must_use] pub fn subtree_leaf( nonce: &[u8; 32], @@ -552,7 +576,13 @@ pub fn subtree_leaf( SubtreeLeaf { key: *key, bytes_hash: *blake3::hash(bytes).as_bytes(), - nonced_hash: nonced_leaf_hash(nonce, challenged_peer_id, key, bytes), + content_len: u32::try_from(bytes.len()).unwrap_or(u32::MAX), + nonced_root: crate::replication::slice::nonced_block_root( + nonce, + challenged_peer_id, + key, + bytes, + ), } } @@ -609,28 +639,83 @@ mod tests { // ---- select_subtree_path: dead-block regression ----------------------- #[test] - fn selection_never_empty_across_many_sizes_and_nonces() { + fn selection_never_empty_and_within_ceiling_across_sizes_and_nonces() { for n in [ 5u32, 6, 7, 9, 13, 17, 33, 65, 100, 129, 333, 1000, 1024, 1025, ] { - let floor = sqrt_floor(n); + let ceiling = max_subtree_leaves(n); for seed in 0u8..=255 { let path = select_subtree_path(&nonce_of(seed), n).unwrap(); - assert!( - path.real_leaf_count() >= floor.min(n), - "n={n} seed={seed}: real={} < floor={floor}", - path.real_leaf_count() - ); + // Never empty (ADR dead-block fix). assert!( path.real_leaf_count() >= 1, "n={n} seed={seed}: empty selection" ); + // Bounded (DoS fix): one request reads at most one fixed-depth + // block, regardless of the (possibly adversarial) nonce. + assert!( + path.real_leaf_count() <= ceiling, + "n={n} seed={seed}: real={} > ceiling={ceiling}", + path.real_leaf_count() + ); assert!(path.leaf_end <= n); assert!(path.leaf_start < path.leaf_end); } } } + /// The denial-of-service the fixed-depth selection closes: under the OLD + /// nonce-bit descent, `key_count = 2^19 + 1` with a nonce steering to the + /// sparse side selected the WHOLE tree (~2 TiB of reads). Now every nonce + /// selects at most one `max_subtree_leaves`-sized block, and the tail slot is + /// a single leaf — not + /// the root. + #[test] + fn adversarial_nonce_cannot_amplify_round1_work() { + for &n in &[513u32, 1025, 524_289, 1_000_000, MAX_COMMITMENT_KEY_COUNT] { + let ceiling = max_subtree_leaves(n); + // A generous cap: sqrt-scale, never the whole tree for a large N. + assert!( + u64::from(ceiling) <= 1024.max(u64::from(n)), + "n={n}: ceiling {ceiling} not sqrt-scale" + ); + for nonce in [[0x00u8; 32], [0xFFu8; 32], [0xAAu8; 32], [0x55u8; 32]] { + let path = select_subtree_path(&nonce, n).unwrap(); + assert!( + path.real_leaf_count() <= ceiling && path.real_leaf_count() >= 1, + "n={n}: real={} outside 1..={ceiling}", + path.real_leaf_count() + ); + if n >= 512 { + assert!( + u64::from(path.real_leaf_count()) < u64::from(n), + "n={n}: a single request must not cover the whole tree" + ); + } + } + } + } + + /// Every slot is selectable and the slots partition `0..key_count` with no + /// gaps, so the tail leaf is never permanently unauditable (coverage is + /// uniform over many audits). + #[test] + fn slots_partition_all_leaves_with_no_gaps() { + for &n in &[5u32, 17, 100, 1025, 524_289] { + let span = max_subtree_leaves(n); + let slot_count = n.div_ceil(span); + let mut covered = 0u32; + for slot in 0..slot_count { + let start = slot.saturating_mul(span); + let end = start.saturating_add(span).min(n); + assert_eq!(start, covered, "n={n} slot={slot}: gap before this slot"); + assert!(end > start, "n={n} slot={slot}: empty slot"); + covered = end; + } + assert_eq!(covered, n, "n={n}: slots do not cover all leaves"); + } + } + #[test] fn small_trees_select_whole_tree() { for n in 1..=SMALL_TREE_FULL_AUDIT_FLOOR { @@ -980,25 +1065,27 @@ mod tests { } #[test] - fn fabricated_nonced_hash_caught_by_spotcheck_probability() { + fn fabricated_nonced_root_caught_by_spotcheck_probability() { // Simulate the realness check: a responder fabricates a fraction x of - // nonced hashes. The auditor spot-checks k leaves; probability all k - // land on honest leaves is (1-x)^k. Here we just assert the auditor - // *would* catch a fabricated leaf when it samples that position. + // nonced roots. The auditor opens k leaves; probability all k land on + // honest leaves is (1-x)^k. Here we just assert the auditor *would* catch + // a fabricated leaf: its committed root differs from the honest root + // recomputed from the real chunk bytes under the audit nonce. let peer = [1u8; 32]; let entries = entries_for(400); let nonce = nonce_of(9); let (mut proof, _commitment) = build_proof(&entries, &nonce, &peer); - // Fabricate the nonced hash on the first subtree leaf (wrong bytes). - proof.leaves[0].nonced_hash[0] ^= 0xFF; - // The realness check the caller runs: recompute from the real chunk - // bytes (the same fixture the honest tree was built from). - let leaf = &proof.leaves[0]; + // Fabricate the nonced root on the first subtree leaf. + if let Some(first) = proof.leaves.first_mut() { + first.nonced_root[0] ^= 0xFF; + } + let leaf = proof.leaves.first().expect("proof has leaves"); let real_bytes = chunk_bytes(&leaf.key); - let expected = nonced_leaf_hash(&nonce, &peer, &leaf.key, &real_bytes); + let expected = + crate::replication::slice::nonced_block_root(&nonce, &peer, &leaf.key, &real_bytes); assert_ne!( - leaf.nonced_hash, expected, - "fabricated nonced hash must differ from real" + leaf.nonced_root, expected, + "fabricated nonced root must differ from real" ); } diff --git a/tests/e2e/testnet.rs b/tests/e2e/testnet.rs index 551cbbbe..a281f5ea 100644 --- a/tests/e2e/testnet.rs +++ b/tests/e2e/testnet.rs @@ -292,6 +292,14 @@ impl TestNetworkConfig { node_count: SMALL_NODE_COUNT, bootstrap_count: DEFAULT_BOOTSTRAP_COUNT, stabilization_timeout: Duration::from_secs(SMALL_STABILIZATION_TIMEOUT_SECS), + // The heavy subtree round-1 responder cooldown defaults to 30 min for + // production rate-limiting; tests audit the same holder repeatedly in + // quick succession, so drop it to near-zero so audits are not + // rate-dropped (a dropped round-1 would surface as a timeout). + replication_config: Some(ReplicationConfig { + subtree_round1_responder_cooldown: Duration::from_millis(1), + ..ReplicationConfig::default() + }), ..Default::default() } } diff --git a/tests/poc_audit_handler_live.rs b/tests/poc_audit_handler_live.rs index 11c857d8..dc609af3 100644 --- a/tests/poc_audit_handler_live.rs +++ b/tests/poc_audit_handler_live.rs @@ -25,13 +25,14 @@ use std::sync::Arc; use ant_node::replication::commitment_state::{BuiltCommitment, ResponderCommitmentState}; -use ant_node::replication::config::MAX_BYTE_CHALLENGE_KEYS; +use ant_node::replication::config::MAX_SLICE_OPENINGS; use ant_node::replication::protocol::{ - SubtreeAuditChallenge, SubtreeAuditResponse, SubtreeByteChallenge, SubtreeByteItem, - SubtreeByteResponse, + SubtreeAuditChallenge, SubtreeAuditResponse, SubtreeSliceChallenge, SubtreeSliceItem, + SubtreeSliceOpening, SubtreeSliceResponse, }; +use ant_node::replication::slice::{nonced_block_root, verify_block_slice, verify_nonced_block}; use ant_node::replication::storage_commitment_audit::{ - handle_subtree_byte_challenge, handle_subtree_challenge, + handle_subtree_challenge, handle_subtree_slice_challenge, }; use ant_node::replication::subtree::{verify_subtree_proof, StructureVerdict}; use ant_node::storage::{LmdbStorage, LmdbStorageConfig}; @@ -335,48 +336,95 @@ async fn committed_key_with_missing_bytes_is_rejected() { } // --------------------------------------------------------------------------- -// 6. Round 2 (byte challenge): honest serve + oversize-request rejection +// 6. Round 2 (slice challenge): honest open + oversize-request rejection // --------------------------------------------------------------------------- -/// Round-2 happy path: a byte challenge pinned to the responder's retained -/// commitment, for keys it committed to and still stores, returns `Items` with -/// the ORIGINAL bytes (`Present`) for every requested key. +/// Round-2 happy path: a slice challenge pinned to the responder's retained +/// commitment, for keys it committed to and still stores, returns `Items` with a +/// `Present` verified opening for every requested key. The opening is fully +/// verified here — the Bao slice decodes against the chunk address to the real +/// block, and the nonced opening folds to the honest nonced root — so this FAILS +/// if the responder produces a malformed or non-possessing proof. /// -/// FLIPS IF: the responder stops serving original bytes for committed keys — -/// the auditor would then see byte-verification failures for honest nodes. +/// FLIPS IF: the responder stops opening valid slices for committed keys — the +/// auditor would then see verification failures for honest nodes. #[tokio::test] -async fn byte_challenge_serves_original_bytes_for_committed_keys() { +async fn slice_challenge_opens_valid_blocks_for_committed_keys() { let (storage, _t) = test_storage().await; let r = Responder::new(&storage, &[1, 2, 3, 4]).await; let pin = r.current_hash(); - - let keys = vec![Responder::address(1), Responder::address(2)]; - let challenge = SubtreeByteChallenge { + let nonce = [0x22u8; 32]; + + let openings = vec![ + SubtreeSliceOpening { + key: Responder::address(1), + block_index: 0, + }, + SubtreeSliceOpening { + key: Responder::address(2), + block_index: 0, + }, + ]; + let challenge = SubtreeSliceChallenge { challenge_id: 43, - nonce: [0x22u8; 32], + nonce, challenged_peer_id: r.peer_id_bytes, expected_commitment_hash: pin, - keys: keys.clone(), + openings: openings.clone(), }; let resp = - handle_subtree_byte_challenge(&challenge, &storage, &r.peer_id, false, Some(&r.state)) + handle_subtree_slice_challenge(&challenge, &storage, &r.peer_id, false, Some(&r.state)) .await; match resp { - SubtreeByteResponse::Items { + SubtreeSliceResponse::Items { challenge_id, items, } => { assert_eq!(challenge_id, 43); - assert_eq!(items.len(), keys.len(), "one item per requested key"); - for (item, (i, key)) in items.iter().zip([1u8, 2].into_iter().zip(keys)) { + assert_eq!( + items.len(), + openings.len(), + "one item per requested opening" + ); + for (item, (i, opening)) in items.iter().zip([1u8, 2].into_iter().zip(openings)) { match item { - SubtreeByteItem::Present { key: k, bytes } => { - assert_eq!(*k, key); - assert_eq!(*bytes, chunk_content(i), "must serve the ORIGINAL bytes"); + SubtreeSliceItem::Present { + key: k, + block_index, + bao_slice, + nonced_siblings, + } => { + assert_eq!(*k, opening.key); + assert_eq!(*block_index, 0); + let content = chunk_content(i); + let bytes_hash = *blake3::hash(&content).as_bytes(); + // Chain 1: the Bao slice decodes against the address to + // the real block (a 1024-byte chunk is a single block). + let block = + verify_block_slice(bao_slice, &bytes_hash, content.len() as u64, 0) + .expect("bao slice must verify against the chunk address"); + assert_eq!(block, content, "opened block must be the ORIGINAL bytes"); + // Chain 2: the nonced opening folds to the honest nonced + // root over the real content under this audit's nonce. + let root = + nonced_block_root(&nonce, &r.peer_id_bytes, &opening.key, &content); + assert!( + verify_nonced_block( + &nonce, + &r.peer_id_bytes, + &opening.key, + 0, + &block, + nonced_siblings, + &root, + ant_node::replication::slice::block_count(content.len() as u64), + ), + "nonced opening must fold to the honest nonced root" + ); } - other @ SubtreeByteItem::Absent { .. } => { + other @ SubtreeSliceItem::Absent { .. } => { panic!("expected Present for stored committed key, got {other:?}") } } @@ -386,43 +434,291 @@ async fn byte_challenge_serves_original_bytes_for_committed_keys() { } } -/// A byte challenge requesting more than `MAX_BYTE_CHALLENGE_KEYS` keys is -/// rejected up front: an honest auditor batches its sample to that cap so the -/// worst-case response (all chunks at max size) fits the replication wire cap; -/// anything larger is a forged/over-size request the responder must not try to -/// serve (the response could not encode, and reading the chunks first would be -/// disk-read amplification). +/// Coalescing at the live responder boundary: a challenge carrying DUPLICATE and +/// interleaved openings for the same `(key, block_index)` returns one `Present` +/// item per DISTINCT opening, not one per raw opening. This is the contract that +/// lets `serve_committed_key_openings` (via `ChunkOpener`) read and hash each +/// committed chunk once instead of re-reading it per repeated opening, so a +/// forged auditor cannot amplify responder disk/CPU work by repeating openings +/// while staying under the total-openings cap. /// -/// FLIPS IF: the per-challenge key cap is removed from the responder. +/// FLIPS IF: the responder stops deduplicating openings (item count would grow +/// with the raw request) or drops/duplicates a distinct identity. #[tokio::test] -async fn oversize_byte_challenge_is_rejected() { +async fn slice_challenge_coalesces_duplicate_and_interleaved_openings() { let (storage, _t) = test_storage().await; let r = Responder::new(&storage, &[1, 2, 3, 4]).await; let pin = r.current_hash(); + let nonce = [0x55u8; 32]; + + let k1 = Responder::address(1); + let k2 = Responder::address(2); + // Five raw openings, interleaved, over only TWO distinct (key, block) pairs. + let openings = vec![ + SubtreeSliceOpening { + key: k1, + block_index: 0, + }, + SubtreeSliceOpening { + key: k2, + block_index: 0, + }, + SubtreeSliceOpening { + key: k1, + block_index: 0, + }, + SubtreeSliceOpening { + key: k2, + block_index: 0, + }, + SubtreeSliceOpening { + key: k1, + block_index: 0, + }, + ]; + assert!(openings.len() <= MAX_SLICE_OPENINGS); + let challenge = SubtreeSliceChallenge { + challenge_id: 46, + nonce, + challenged_peer_id: r.peer_id_bytes, + expected_commitment_hash: pin, + openings, + }; + + let resp = + handle_subtree_slice_challenge(&challenge, &storage, &r.peer_id, false, Some(&r.state)) + .await; + + match resp { + SubtreeSliceResponse::Items { + challenge_id, + items, + } => { + assert_eq!(challenge_id, 46); + // Coalesced: one item per DISTINCT (key, block_index), not per raw opening. + assert_eq!( + items.len(), + 2, + "duplicate openings must not multiply response items" + ); + let mut seen: Vec<([u8; 32], u32)> = Vec::new(); + for item in &items { + match item { + SubtreeSliceItem::Present { + key, block_index, .. + } => { + assert!( + !seen.contains(&(*key, *block_index)), + "each (key, block) identity must appear at most once" + ); + seen.push((*key, *block_index)); + } + other @ SubtreeSliceItem::Absent { .. } => { + panic!("expected Present for a stored committed key, got {other:?}") + } + } + } + seen.sort_unstable(); + let mut want = vec![(k1, 0u32), (k2, 0u32)]; + want.sort_unstable(); + assert_eq!( + seen, want, + "both distinct openings must be served exactly once, order-independent" + ); + } + other => panic!("expected Items, got {other:?}"), + } +} + +/// Multi-block coverage: open a DEEP block of a genuinely large (multi-block) +/// chunk end-to-end through the live handler. The single-block happy-path test +/// above cannot exercise the Bao parent-hash chain or the multi-level nonced +/// tree; this one does — it stores a ~100 KB chunk (98 blocks), opens block 50, +/// and verifies both chains against that exact block. +/// +/// FLIPS IF: the Bao slice or nonced opening for a non-trivial tree is malformed +/// (e.g. wrong offset, wrong sibling ordering). +#[tokio::test] +async fn slice_challenge_opens_a_deep_block_of_a_large_chunk() { + use ant_node::replication::commitment::commitment_hash; + use ant_node::replication::slice::block_count; - let keys: Vec<[u8; 32]> = (0..=MAX_BYTE_CHALLENGE_KEYS) - .map(|i| [u8::try_from(i % 251).unwrap_or(0); 32]) + let (storage, _t) = test_storage().await; + + // One large chunk: ~100 KB => 98 one-KiB blocks. + let content: Vec = (0..100_000u32) + .map(|n| (n.wrapping_mul(2_654_435_761) >> 13) as u8) .collect(); - assert!(keys.len() > MAX_BYTE_CHALLENGE_KEYS); - let challenge = SubtreeByteChallenge { + let addr = LmdbStorage::compute_address(&content); + storage.put(&addr, &content).await.expect("put chunk"); + let bytes_hash = *blake3::hash(&content).as_bytes(); + + let (pk, sk) = keypair(); + let peer_id_bytes = *blake3::hash(&pk.to_bytes()).as_bytes(); + let peer_id = PeerId::from_bytes(peer_id_bytes); + let built = BuiltCommitment::build( + vec![(addr, bytes_hash)], + &peer_id_bytes, + &sk, + &pk.to_bytes(), + ) + .expect("build"); + let pin = commitment_hash(built.commitment()).unwrap(); + let state = Arc::new(ResponderCommitmentState::new()); + state.rotate(built); + + let count = block_count(content.len() as u64); + assert!( + count > 64, + "test needs a genuinely multi-block chunk, got {count} blocks" + ); + let block_index = 50u32; + let nonce = [0x5Au8; 32]; + let challenge = SubtreeSliceChallenge { + challenge_id: 77, + nonce, + challenged_peer_id: peer_id_bytes, + expected_commitment_hash: pin, + openings: vec![SubtreeSliceOpening { + key: addr, + block_index, + }], + }; + + let resp = + handle_subtree_slice_challenge(&challenge, &storage, &peer_id, false, Some(&state)).await; + + let SubtreeSliceResponse::Items { items, .. } = resp else { + panic!("expected Items, got {resp:?}"); + }; + let Some(SubtreeSliceItem::Present { + block_index: bi, + bao_slice, + nonced_siblings, + .. + }) = items.first() + else { + panic!("expected a single Present opening, got {items:?}"); + }; + assert_eq!(*bi, block_index); + + // Chain 1: the Bao slice decodes against the address to exactly block 50. + let block = verify_block_slice(bao_slice, &bytes_hash, content.len() as u64, block_index) + .expect("deep-block bao slice must verify"); + let start = block_index as usize * 1024; + assert_eq!( + block, + content[start..start + 1024], + "opened block must be block 50's bytes" + ); + + // Chain 2: the nonced opening folds up the multi-level tree to the honest root. + let root = nonced_block_root(&nonce, &peer_id_bytes, &addr, &content); + assert!( + verify_nonced_block( + &nonce, + &peer_id_bytes, + &addr, + block_index, + &block, + nonced_siblings, + &root, + ant_node::replication::slice::block_count(content.len() as u64) + ), + "deep-block nonced opening must fold to the honest root" + ); + // And the proof is genuinely small — a slice, not the whole 100 KB chunk. + assert!( + bao_slice.len() < content.len() / 4, + "the slice ({} bytes) must be far smaller than the chunk ({} bytes)", + bao_slice.len(), + content.len() + ); +} + +/// A slice challenge requesting more than `MAX_SLICE_OPENINGS` openings is +/// rejected up front: an honest auditor opens at most that many blocks; anything +/// larger is a forged/over-size request the responder must not try to serve +/// (each opening forces a full chunk read to build its proof — disk-read +/// amplification). +/// +/// FLIPS IF: the per-challenge openings cap is removed from the responder. +#[tokio::test] +async fn oversize_slice_challenge_is_rejected() { + let (storage, _t) = test_storage().await; + let r = Responder::new(&storage, &[1, 2, 3, 4]).await; + let pin = r.current_hash(); + + let openings: Vec = (0..=MAX_SLICE_OPENINGS) + .map(|i| SubtreeSliceOpening { + key: [u8::try_from(i % 251).unwrap_or(0); 32], + block_index: 0, + }) + .collect(); + assert!(openings.len() > MAX_SLICE_OPENINGS); + let challenge = SubtreeSliceChallenge { challenge_id: 44, nonce: [0x33u8; 32], challenged_peer_id: r.peer_id_bytes, expected_commitment_hash: pin, - keys, + openings, }; let resp = - handle_subtree_byte_challenge(&challenge, &storage, &r.peer_id, false, Some(&r.state)) + handle_subtree_slice_challenge(&challenge, &storage, &r.peer_id, false, Some(&r.state)) .await; match resp { - SubtreeByteResponse::Rejected { reason, .. } => { + SubtreeSliceResponse::Rejected { reason, .. } => { assert!( reason.contains("max"), - "expected per-challenge key-cap rejection, got: {reason}" + "expected per-challenge openings-cap rejection, got: {reason}" ); } other => panic!("expected Rejected(oversize), got {other:?}"), } } + +/// A slice challenge spanning more DISTINCT keys than the honest spot-check sample +/// is rejected even when it stays under the total-openings cap. Coalescing saves +/// work only when keys repeat, so a forged auditor could otherwise force a full +/// chunk read per distinct key (up to ~40 MiB) without exceeding `MAX_SLICE_OPENINGS`. +/// +/// FLIPS IF: the distinct-key cap is removed from the responder. +#[tokio::test] +async fn slice_challenge_with_too_many_distinct_keys_is_rejected() { + let (storage, _t) = test_storage().await; + let r = Responder::new(&storage, &[1, 2, 3, 4]).await; + let pin = r.current_hash(); + + // 6 distinct keys × 1 opening = 6 openings: under MAX_SLICE_OPENINGS (10) but + // over the honest BYTE_SPOTCHECK_MAX (5) distinct-key sample. + let openings: Vec = (0..6u8) + .map(|i| SubtreeSliceOpening { + key: [i + 1; 32], + block_index: 0, + }) + .collect(); + assert!(openings.len() <= MAX_SLICE_OPENINGS); + let challenge = SubtreeSliceChallenge { + challenge_id: 45, + nonce: [0x44u8; 32], + challenged_peer_id: r.peer_id_bytes, + expected_commitment_hash: pin, + openings, + }; + + let resp = + handle_subtree_slice_challenge(&challenge, &storage, &r.peer_id, false, Some(&r.state)) + .await; + + match resp { + SubtreeSliceResponse::Rejected { reason, .. } => { + assert!( + reason.contains("distinct keys"), + "expected distinct-key rejection, got: {reason}" + ); + } + other => panic!("expected Rejected(distinct keys), got {other:?}"), + } +} diff --git a/tests/poc_commitment_audit_attacks.rs b/tests/poc_commitment_audit_attacks.rs index bc67bc77..903af243 100644 --- a/tests/poc_commitment_audit_attacks.rs +++ b/tests/poc_commitment_audit_attacks.rs @@ -13,17 +13,17 @@ //! The production auditor's `verify_subtree_response` (in //! `src/replication/storage_commitment_audit.rs`) is private, so this file //! reproduces the exact ordered gates it runs — pin, peer-id binding, -//! signature, structural [`verify_subtree_proof`], then the **round-2 byte -//! challenge**: the auditor demands the ORIGINAL chunk bytes for a -//! nonce-selected sample of the just-proven leaves FROM THE RESPONDER and -//! verifies the served content against each leaf's committed `bytes_hash` -//! (content address) and `nonced_hash` (freshness). Possession is -//! non-delegable: the auditor needs to hold NONE of the responder's chunks, -//! and a committed key the responder cannot serve is a deterministic, -//! confirmed failure (`DigestMismatch` in production — never inconclusive, -//! never graced). The helper [`auditor_accepts`] runs these gates in the same -//! order with the same failure semantics, so a reviewer can see each attack -//! is caught at the same gate the network code would catch it. +//! signature, structural [`verify_subtree_proof`], then the **round-2 slice +//! challenge** (V2-685): the auditor opens one 1 KiB block of a fresh-random +//! sample of the just-proven leaves FROM THE RESPONDER and verifies each opened +//! block against the leaf's committed `bytes_hash` (a Bao verified slice against +//! the chunk address) and `nonced_root` (a nonced block-tree opening proving +//! round-1 possession). Possession is non-delegable: the auditor needs to hold +//! NONE of the responder's chunks, and a committed key the responder cannot +//! serve is a deterministic, confirmed failure (`DigestMismatch` in production — +//! never inconclusive, never graced). The helper [`auditor_accepts`] runs these +//! gates in the same order with the same failure semantics, so a reviewer can +//! see each attack is caught at the same gate the network code would catch it. //! //! ## What changed from the old per-key audit (and why) //! @@ -68,9 +68,13 @@ use ant_node::replication::commitment::{ }; use ant_node::replication::commitment_state::{BuiltCommitment, ResponderCommitmentState}; use ant_node::replication::config::AUDIT_SPOTCHECK_COUNT; +use ant_node::replication::slice::{ + extract_block_slice, nonced_block_root, nonced_block_siblings, verify_block_slice, + verify_nonced_block, +}; use ant_node::replication::subtree::{ - build_subtree_proof, nonced_leaf_hash, select_spotcheck_indices, select_subtree_path, - verify_subtree_proof, StructureVerdict, SubtreeProof, + build_subtree_proof, select_spotcheck_indices, select_subtree_path, verify_subtree_proof, + StructureVerdict, SubtreeLeaf, SubtreeProof, }; use rand::Rng; use saorsa_pqc::api::sig::{ml_dsa_65, MlDsaPublicKey, MlDsaSecretKey}; @@ -83,12 +87,12 @@ fn keypair() -> (MlDsaPublicKey, MlDsaSecretKey) { ml_dsa_65().generate_keypair().unwrap() } -/// Deterministic chunk bytes for key index `i`. The committed tree is built -/// from `BLAKE3(content(i))`, so an honest proof — which hashes the same bytes — -/// reconstructs the committed root and passes the real-bytes spot-check. +/// Deterministic chunk bytes for fixture index `i`. Keys are CONTENT-ADDRESSED +/// (`key(i) == BLAKE3(content(i)) == content_hash(i)`), so a committed leaf is +/// `(key, key)` exactly as production, and an honest proof — which hashes the +/// same bytes — reconstructs the committed root and passes the real-bytes check. fn content(i: u32) -> Vec { - let mut v = key(i).to_vec(); - v.extend_from_slice(b"subtree-audit-chunk-body"); + let mut v = b"subtree-audit-chunk-body".to_vec(); v.extend_from_slice(&i.to_le_bytes()); v } @@ -97,12 +101,12 @@ fn content_hash(i: u32) -> [u8; 32] { *blake3::hash(&content(i)).as_bytes() } -/// Big-endian key so numeric order matches the MerkleTree sort order; this lets -/// us reason about leaf positions when we tamper with them. +/// The content-addressed key for index `i`: `key(i) == content_hash(i)`, so +/// `bytes_hash == key` on every committed leaf. Leaf positions in the committed +/// tree therefore follow the hash order, not `i`; the attack tests below read the +/// actual key at each position via `key_at` rather than assuming `key(i)`. fn key(i: u32) -> [u8; 32] { - let mut k = [0u8; 32]; - k[..4].copy_from_slice(&i.to_be_bytes()); - k + content_hash(i) } /// A responder identity (real ML-DSA keypair) plus its retention state. Peer @@ -145,7 +149,7 @@ impl Responder { } /// Bytes source for an HONEST responder: it really holds every chunk it -/// committed to, so it can always produce a correct `nonced_hash`. +/// committed to, so it can always open a real slice + nonced block opening. fn honest_bytes(k: &[u8; 32]) -> Option> { for i in 0..4096u32 { if &key(i) == k { @@ -158,13 +162,15 @@ fn honest_bytes(k: &[u8; 32]) -> Option> { /// The auditor's full ordered verification, mirroring the production /// `verify_subtree_response` gates. Returns `Ok(byte_checked_count)` on accept. /// -/// `responder_serves(k)` models round 2 (`SubtreeByteChallenge`): what the -/// RESPONDER returns when the auditor demands the original bytes of sampled -/// leaf `k`. `Some(bytes)` is a `SubtreeByteItem::Present`; `None` is an -/// explicit `Absent` or an omitted key — a committed key the responder will -/// not serve, which production `verify_byte_response` counts as a confirmed -/// `DigestMismatch`. The auditor verifies the SERVED content, so it needs to -/// hold none of the responder's chunks and no inconclusive lane exists. +/// `responder_serves(k)` models round 2 (`SubtreeSliceChallenge`): the chunk +/// content the RESPONDER can produce for sampled leaf `k`, from which it builds a +/// Bao verified slice + nonced block opening exactly as +/// `handle_subtree_slice_challenge` does. `Some(bytes)` is a +/// `SubtreeSliceItem::Present`; `None` is an explicit `Absent` or an omitted key +/// — a committed key the responder will not serve, which production +/// `verify_slice_response` counts as a confirmed `DigestMismatch`. The auditor +/// verifies the served block against the committed address and nonced root, so it +/// needs to hold none of the responder's chunks and no inconclusive lane exists. fn auditor_accepts( challenged_peer_id: &[u8; 32], expected_commitment_hash: &[u8; 32], @@ -194,13 +200,21 @@ fn auditor_accepts( return Err(AuditError::StructureInvalid(why)); } - // -- Gate: round-2 byte challenge (responder-served possession) ---------- + // -- Gate: content-address binding (possession-forgery guard) ----------- + // Mirrors production `evaluate_subtree_structure`: a leaf whose `bytes_hash` + // is decoupled from its credited `key` is rejected at round 1 (content- + // addressed chunks always have `bytes_hash == key`). + if proof.leaves.iter().any(|l| l.bytes_hash != l.key) { + return Err(AuditError::StructureInvalid("leaf bytes_hash != key")); + } + + // -- Gate: round-2 slice challenge (responder-served possession) ---------- // Mirrors `verify_subtree_response` round 2: the sample is chosen with FRESH // randomness over the RECEIVED proof leaves (NOT nonce-derived), AFTER round - // 1, so the responder cannot predict which leaves will be opened (§1 - // cut-and-choose soundness). EVERY sampled leaf must verify from the bytes - // the responder serves. There is no skip and no inconclusive lane: a - // committed key the responder cannot serve is a provable lie. + // 1, so the responder cannot predict which leaves — or which block within — + // will be opened (§1 cut-and-choose soundness). EVERY sampled leaf must + // verify from the slice the responder serves. There is no skip and no + // inconclusive lane: a committed key the responder cannot serve is a lie. let spot = random_sample_indices( proof.leaves.len(), AUDIT_SPOTCHECK_COUNT.clamp(3, 5) as usize, @@ -221,9 +235,40 @@ fn auditor_accepts( // maps this to `DigestMismatch`), NOT a skip. return Err(AuditError::CommittedKeyUnserved); }; - let plain = *blake3::hash(&bytes).as_bytes(); - let nonced = nonced_leaf_hash(nonce, &commitment.sender_peer_id, &leaf.key, &bytes); - if leaf.bytes_hash != plain || leaf.nonced_hash != nonced { + // The fixtures use short (single-block) chunks, so the only block is 0; + // production draws a fresh random index. The responder builds its slice + // and nonced opening from the content it serves. + let block_index = 0u32; + let bao_slice = extract_block_slice(&bytes, block_index).unwrap(); + let nonced_siblings = nonced_block_siblings( + nonce, + &commitment.sender_peer_id, + &leaf.key, + &bytes, + block_index, + ) + .unwrap_or_default(); + + // Auditor chain 1: authenticate the block against the committed address. + let Some(block) = verify_block_slice( + &bao_slice, + &leaf.bytes_hash, + u64::from(leaf.content_len), + block_index, + ) else { + return Err(AuditError::RealBytesMismatch); + }; + // Auditor chain 2: fold the nonced opening to the committed nonced_root. + if !verify_nonced_block( + nonce, + &commitment.sender_peer_id, + &leaf.key, + block_index, + &block, + &nonced_siblings, + &leaf.nonced_root, + ant_node::replication::slice::block_count(u64::from(leaf.content_len)), + ) { return Err(AuditError::RealBytesMismatch); } checked += 1; @@ -255,8 +300,9 @@ enum AuditError { CommitmentHashMismatch, SignatureInvalid, StructureInvalid(&'static str), - /// Round 2: the responder served content that does not hash to the - /// committed address / freshness hash (production: `DigestMismatch`). + /// Round 2: the responder's opened block failed a chain — the Bao slice did + /// not decode against the committed address, or the nonced opening did not + /// fold to the committed `nonced_root` (production: `DigestMismatch`). RealBytesMismatch, /// Round 2: the responder would not serve a committed, sampled key /// (production: `DigestMismatch` — a deterministic, confirmed failure). @@ -310,15 +356,15 @@ fn honest_responder_passes_audit() { /// leaf's `bytes_hash` (that value IS the chunk's network address, which is /// public), but it DROPPED the actual bytes. It fabricates a proof: correct /// `key` and correct `bytes_hash` for every selected leaf (so the structural -/// root rebuild passes), but it cannot compute the `nonced_hash`, which requires -/// the real bytes under a fresh nonce. It fills in a forged `nonced_hash`. +/// root rebuild passes), but it cannot compute the `nonced_root`, which requires +/// the real bytes under a fresh nonce. It fills in a forged `nonced_root`. /// /// The structural gate PASSES (addresses alone rebuild the root), proving that /// structure is NOT sufficient — exactly the storage-binding hole. Round 2 is what -/// catches it: the auditor demands the original bytes FROM THE RELAY, and the -/// relay has nothing to serve. Refusing/omitting a sampled committed key is a -/// confirmed failure, and serving fabricated bytes cannot hash to the -/// committed content address (a preimage break) — both lanes are asserted. +/// catches it: the auditor opens a block FROM THE RELAY, and the relay has nothing +/// to serve. Refusing/omitting a sampled committed key is a confirmed failure, and +/// serving a fabricated block cannot decode against the committed content address +/// (a preimage break) — both lanes are asserted. #[test] fn relay_holding_only_addresses_caught_by_real_bytes_check() { let nonce = [0x77; 32]; @@ -328,18 +374,21 @@ fn relay_holding_only_addresses_caught_by_real_bytes_check() { // The lazy node fabricates the proof from PUBLIC data only: it knows each // leaf key and its bytes_hash (== address), but NOT the bytes, so it forges - // every nonced_hash. + // every nonced_root. let path = select_subtree_path(&nonce, built.commitment().key_count).unwrap(); let mut leaves = Vec::new(); for idx in path.leaf_start..path.leaf_end { let k = built.tree().key_at(idx as usize).unwrap(); - // bytes_hash is public (== the chunk address); the responder fakes the - // possession hash because it lacks the bytes. - let forged_nonced = *blake3::hash(b"i-do-not-have-the-bytes").as_bytes(); - leaves.push(ant_node::replication::subtree::SubtreeLeaf { + let c = honest_bytes(&k).unwrap(); + // For a content-addressed chunk bytes_hash == key (both public), so the + // relay can present a structurally valid leaf; it fakes the possession + // commitment because it lacks the bytes. + let forged_nonced_root = *blake3::hash(b"i-do-not-have-the-bytes").as_bytes(); + leaves.push(SubtreeLeaf { key: k, - bytes_hash: content_hash(idx), - nonced_hash: forged_nonced, + bytes_hash: k, + content_len: u32::try_from(c.len()).unwrap(), + nonced_root: forged_nonced_root, }); } // Real sibling cut-hashes from the committed tree (public, derivable). @@ -394,17 +443,17 @@ fn relay_holding_only_addresses_caught_by_real_bytes_check() { /// the relay attack, and the one the §1 review found: a relay holds only public /// addresses, but it knows the round-1 nonce, so under the OLD nonce-derived /// sampling it could compute EXACTLY which 3..=5 leaves round 2 would open, -/// fetch only those few chunks from neighbours, fill in correct `nonced_hash` -/// for them, and fabricate `nonced_hash` for every other leaf — passing the +/// fetch only those few chunks from neighbours, commit a correct `nonced_root` +/// for them, and fabricate `nonced_root` for every other leaf — passing the /// audit while holding almost nothing. /// /// With the fix, the auditor draws the sample with fresh randomness AFTER the /// proof is committed, so the relay's bet on the nonce-derived indices is -/// uncorrelated with what actually gets opened. We model the relay holding the -/// nonce-derived prediction set and nothing else: the random sample lands on a -/// leaf the relay did NOT fetch with overwhelming probability, and the audit -/// fails. Repeated across many nonces to make the probabilistic catch a -/// near-certainty in aggregate. +/// uncorrelated with what actually gets opened. We model the relay committing a +/// CORRECT `nonced_root` (and holding the bytes) only for the nonce-derived +/// prediction set and forging the rest: the random sample lands on a leaf the +/// relay did NOT prepare with overwhelming probability, and the audit fails. +/// Repeated across many nonces to make the probabilistic catch a near-certainty. #[test] fn predict_and_fetch_relay_is_caught_by_fresh_random_sample() { let r = Responder::new(); @@ -418,17 +467,34 @@ fn predict_and_fetch_relay_is_caught_by_fresh_random_sample() { let mut nonce = [0u8; 32]; nonce[..4].copy_from_slice(&t.to_le_bytes()); - // The relay builds a structurally-valid proof from PUBLIC data, forging - // every leaf's nonced_hash (it holds no bytes). let plan = ant_node::replication::subtree::subtree_plan(built.tree(), &nonce).unwrap(); let path = select_subtree_path(&nonce, n).unwrap(); + // The relay PREDICTS the old nonce-derived sample: those are the only + // leaves it fetches and can commit a correct nonced_root for. + let predicted_local: std::collections::HashSet = + select_spotcheck_indices(&nonce, &path, AUDIT_SPOTCHECK_COUNT.clamp(3, 5)) + .into_iter() + .collect(); + + // Build a structurally-valid proof: correct `nonced_root` for predicted + // leaves (it holds those bytes), forged for every other leaf. let mut leaves = Vec::new(); - for idx in path.leaf_start..path.leaf_end { + let mut predicted_keys = std::collections::HashSet::new(); + for (local, idx) in (path.leaf_start..path.leaf_end).enumerate() { let k = built.tree().key_at(idx as usize).unwrap(); - leaves.push(ant_node::replication::subtree::SubtreeLeaf { + let c = honest_bytes(&k).unwrap(); + let is_predicted = predicted_local.contains(&u32::try_from(local).unwrap()); + let nonced_root = if is_predicted { + predicted_keys.insert(k); + nonced_block_root(&nonce, &r.peer_id_bytes, &k, &c) + } else { + *blake3::hash(b"forged").as_bytes() + }; + leaves.push(SubtreeLeaf { key: k, - bytes_hash: content_hash(idx), - nonced_hash: *blake3::hash(b"forged").as_bytes(), + bytes_hash: k, + content_len: u32::try_from(c.len()).unwrap(), + nonced_root, }); } let forged = SubtreeProof { @@ -436,14 +502,6 @@ fn predict_and_fetch_relay_is_caught_by_fresh_random_sample() { sibling_cut_hashes: plan.sibling_cut_hashes, }; - // The relay PREDICTS the old nonce-derived sample and fetches exactly - // those chunks (correct bytes for them only). - let predicted: std::collections::HashSet<[u8; 32]> = - select_spotcheck_indices(&nonce, &path, AUDIT_SPOTCHECK_COUNT.clamp(3, 5)) - .into_iter() - .filter_map(|i| forged.leaves.get(i as usize).map(|l| l.key)) - .collect(); - // Responder serves real bytes ONLY for the predicted set; everything // else it cannot serve (it holds no other bytes). let res = auditor_accepts( @@ -453,9 +511,7 @@ fn predict_and_fetch_relay_is_caught_by_fresh_random_sample() { built.commitment(), &forged, |k| { - // The relay can only serve bytes for the chunks it fetched (the - // predicted set); for those it returns the real content. - if predicted.contains(k) { + if predicted_keys.contains(k) { (0..n).find(|&i| &key(i) == k).map(content) } else { None @@ -483,7 +539,7 @@ fn predict_and_fetch_relay_is_caught_by_fresh_random_sample() { /// the auditor now picks the sample with FRESH randomness after the proof is in /// hand (§1), the attacker cannot aim its forgeries away from the sample. We /// model the worst case for the attacker — every leaf's freshness forged — so -/// any random sample is fatal; round 2 re-derives the freshness hash from the +/// any random sample is fatal; round 2 re-derives the nonced opening from the /// served bytes and exposes it. #[test] fn fabricated_fraction_is_caught_when_a_forged_leaf_is_sampled() { @@ -492,14 +548,14 @@ fn fabricated_fraction_is_caught_when_a_forged_leaf_is_sampled() { let pin = r.commit_to_range(400); let (mut proof, commitment) = honest_proof_and_commitment(&r, &nonce); - // Forge every leaf's nonced hash. Under fresh-random sampling the auditor + // Forge every leaf's nonced root. Under fresh-random sampling the auditor // is guaranteed to open a forged leaf, so the audit must fail. for leaf in &mut proof.leaves { - leaf.nonced_hash[0] ^= 0xFF; + leaf.nonced_root[0] ^= 0xFF; } - // Even if the responder serves the REAL bytes in round 2, the freshness - // hash recomputed from that served content exposes the forgery. + // Even if the responder serves the REAL bytes in round 2, the nonced opening + // recomputed from that served content cannot fold to the forged root. let res = auditor_accepts( &r.peer_id_bytes, &pin, @@ -511,7 +567,7 @@ fn fabricated_fraction_is_caught_when_a_forged_leaf_is_sampled() { assert_eq!( res, Err(AuditError::RealBytesMismatch), - "a forged leaf landing under the byte challenge must fail, got {res:?}" + "a forged leaf landing under the slice challenge must fail, got {res:?}" ); } @@ -990,26 +1046,26 @@ fn signature_round_trips_correctly() { assert!(!verify_commitment_signature(&c2)); } -/// The per-leaf possession hash binds nonce, peer, key, and bytes — the -/// foundation of the real-bytes spot-check. Changing any input changes it, so a -/// responder cannot reuse a possession hash across nonces/peers/keys/chunks. +/// The per-leaf nonced block-tree root binds nonce, peer, key, and bytes — the +/// foundation of the possession opening. Changing any input changes it, so a +/// responder cannot reuse a nonced root across nonces/peers/keys/chunks. #[test] -fn nonced_leaf_hash_binds_all_inputs() { - let base = nonced_leaf_hash(&[1; 32], &[2; 32], &key(3), b"chunk"); +fn nonced_block_root_binds_all_inputs() { + let base = nonced_block_root(&[1; 32], &[2; 32], &key(3), b"chunk"); assert_ne!( base, - nonced_leaf_hash(&[9; 32], &[2; 32], &key(3), b"chunk") + nonced_block_root(&[9; 32], &[2; 32], &key(3), b"chunk") ); assert_ne!( base, - nonced_leaf_hash(&[1; 32], &[9; 32], &key(3), b"chunk") + nonced_block_root(&[1; 32], &[9; 32], &key(3), b"chunk") ); assert_ne!( base, - nonced_leaf_hash(&[1; 32], &[2; 32], &key(9), b"chunk") + nonced_block_root(&[1; 32], &[2; 32], &key(9), b"chunk") ); assert_ne!( base, - nonced_leaf_hash(&[1; 32], &[2; 32], &key(3), b"other") + nonced_block_root(&[1; 32], &[2; 32], &key(3), b"other") ); }