diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ab7f31f5..447f58df 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,6 +50,24 @@ jobs: version: ${{ env.FOUNDRY_VERSION }} - name: Run unit tests run: cargo test --lib --features test-utils + - name: Kill a node mid-write and check what survived + shell: bash + run: | + set -euo pipefail + cargo test --test chunk_store_crash_safety --features test-utils -- --test-threads=1 2>&1 | tee /tmp/crash_safety.log + grep -qE 'test result: ok\. [1-9]' /tmp/crash_safety.log \ + || { echo 'chunk_store_crash_safety ran no tests'; exit 1; } + - name: Startup scan, index memory and inode cost at scale + if: runner.os == 'Linux' + shell: bash + run: | + set -euo pipefail + cargo test --test storage_scale --features test-utils -- --nocapture --test-threads=1 2>&1 | tee /tmp/scale.log + # A target whose required features are not passed is skipped with a + # warning and a zero exit, so a harness can stop running without anyone + # noticing. This is what makes that loud. + grep -qE 'test result: ok\. [1-9]' /tmp/scale.log \ + || { echo 'scale ran no tests'; exit 1; } - name: Run e2e tests run: cargo test --test e2e --features test-utils -- --test-threads=1 - name: Run v12 storage-bound audit attack PoCs @@ -59,6 +77,54 @@ jobs: - name: Run bootstrap-stall PoC regression marker run: cargo test --test poc_bootstrap_stall --features test-utils + filesystems: + name: Storage on ${{ matrix.fs }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + fs: [ext4, xfs, btrfs] + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - name: Install the filesystem tools + run: sudo apt-get update && sudo apt-get install -y xfsprogs btrfs-progs + - name: Make a ${{ matrix.fs }} volume and mount it + shell: bash + run: | + set -euo pipefail + # A loopback image, so these run on a filesystem of the right kind rather than + # on whatever the runner happens to give us. ext4 is what most of the fleet is + # on; XFS and btrfs are the two the design reasons about separately, btrfs + # because it has been observed reordering writes around a rename. + # 3 GiB is ample: these tests use tens of MiB. + truncate -s 3G /tmp/${{ matrix.fs }}.img + mkfs.${{ matrix.fs }} -q /tmp/${{ matrix.fs }}.img + sudo mkdir -p /mnt/antfs + sudo mount -o loop /tmp/${{ matrix.fs }}.img /mnt/antfs + sudo chown "$USER" /mnt/antfs + df -hT /mnt/antfs + # TMPDIR is what `TempDir::new` uses, so this puts every temporary store these tests + # build on the mounted filesystem rather than on the runner's root. + # + # These used to be the migration harnesses. The migration is gone, and what is left + # worth asking of a filesystem is what the store itself does on it: publish a chunk + # through a temporary and a rename, flush the directory, unlink it again, and rebuild + # an index from the names afterwards. The storage tests do all of that, and running + # them here is what keeps ext4, XFS and btrfs covered now that the harnesses that used + # to cover them have been deleted. They do not measure free space before and after; + # the harness that did was about the migration and went with it. + - name: Storage behaviour on ${{ matrix.fs }} + env: + TMPDIR: /mnt/antfs + shell: bash + run: | + set -euo pipefail + cargo test --lib --features test-utils storage:: 2>&1 | tee /tmp/storage.log + grep -qE 'test result: ok\. [1-9]' /tmp/storage.log \ + || { echo 'the storage tests ran nothing'; exit 1; } + doc: name: Documentation runs-on: ubuntu-latest @@ -84,6 +150,29 @@ jobs: - uses: Swatinem/rust-cache@v2 - name: Build release (no logging) run: cargo build --release --no-default-features + # The crash harness drives the store through a failpoint that parks the process + # forever on an environment variable, and the store's own tests use the same one. It + # is compiled only under `test-utils`, which + # is not a default feature and is not passed by the release workflow, so a shipped + # binary does not contain it. This proves that rather than trusting it: the variable + # name is a string literal, so it survives into the binary whenever the code that + # reads it is compiled, and its absence is the absence of the failpoint. + - name: A shipped binary carries no failpoint + if: runner.os == 'Linux' + shell: bash + run: | + set -euo pipefail + cargo build --bin ant-node + found=$(strings -a target/debug/ant-node | grep -c 'ANT_HALT_' || true) + # With --features test-utils this count is not zero, which is what makes a zero + # here evidence rather than an accident of how the binary was stripped. The exact + # number is one per failpoint and is deliberately not asserted, so that adding a + # failpoint does not fail this check. + if [ "$found" != "0" ]; then + echo "the publish failpoint is compiled into a default-feature build" + exit 1 + fi + echo "no failpoint in a default-feature build" test-no-logging: name: Test (no logging) diff --git a/Cargo.lock b/Cargo.lock index bd8dd96d..c24c5b90 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -831,7 +831,6 @@ dependencies = [ "mimalloc", "objc2", "objc2-foundation", - "page_size", "parking_lot", "postcard", "proptest", diff --git a/Cargo.toml b/Cargo.toml index 2b99f0f1..9b32e35c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,7 +53,8 @@ xor_name = "5" lru = "0.16.3" parking_lot = "0.12" # Efficient mutex for cache -# Storage - LMDB via heed for content-addressed chunk store +# LMDB via heed for the paid-key list. The chunk store is one file per chunk and does +# not use it. heed = "0.22" blake3 = "1" @@ -106,8 +107,6 @@ sha2 = "0.10" # Cross-platform file locking for upgrade caches fs2 = "0.4" -# System page size (for LMDB map alignment during resize) -page_size = "0.6" # Protocol serialization postcard = { version = "1.1.3", features = ["use-std"] } @@ -129,6 +128,20 @@ proptest = "1" alloy = { version = "1", features = ["node-bindings"] } serial_test = "3" +# Startup scan time, index memory and inode cost at scale. Regression gates, not +# benchmarks; ANT_SCALE_KEYS raises the count for a deliberate larger run. +[[test]] +name = "storage_scale" +path = "tests/storage_scale.rs" +required-features = ["test-utils"] + +# A process killed part-way through a write: the store is whole or absent, never +# half-indexed, and what the interrupted write left behind is swept. +[[test]] +name = "chunk_store_crash_safety" +path = "tests/chunk_store_crash_safety.rs" +required-features = ["test-utils"] + # E2E test infrastructure (run with --features test-utils) [[test]] name = "e2e" @@ -145,7 +158,7 @@ path = "tests/poc_commitment_audit_attacks.rs" required-features = ["test-utils"] # Live responder-handler tests for the v12 audit. Use -# LmdbStorageConfig::test_default(), gated on test-utils. +# ChunkStoreConfig::test_default(), gated on test-utils. [[test]] name = "poc_audit_handler_live" path = "tests/poc_audit_handler_live.rs" @@ -166,14 +179,6 @@ name = "poc_price_floor_live" path = "tests/poc_price_floor_live.rs" required-features = ["test-utils"] -# Shutdown/LMDB-drain regression: `ReplicationEngine::shutdown()` must not -# return while a detached LMDB blocking op is still running. Uses the -# test-only storage put gate, so it requires the test-utils feature. -[[test]] -name = "poc_shutdown_lmdb_drain" -path = "tests/poc_shutdown_lmdb_drain.rs" -required-features = ["test-utils"] - [features] default = ["logging"] # Enable tracing/logging infrastructure. diff --git a/config/production.toml b/config/production.toml index ce44e017..22a86aa6 100644 --- a/config/production.toml +++ b/config/production.toml @@ -46,9 +46,6 @@ enabled = true # Verify content hash on read verify_on_read = true -# Maximum LMDB database size in GiB (0 = default 32 GiB) -db_size_gb = 0 - # --- Upgrade --- [upgrade] enabled = false diff --git a/docs/adr/ADR-0014-file-based-chunk-store-and-lmdb-retirement.md b/docs/adr/ADR-0014-file-based-chunk-store-and-lmdb-retirement.md new file mode 100644 index 00000000..c0631a2f --- /dev/null +++ b/docs/adr/ADR-0014-file-based-chunk-store-and-lmdb-retirement.md @@ -0,0 +1,558 @@ +# ADR-0014: One File Per Chunk, and Retiring LMDB Without Losing Data + +- **Status:** Proposed +- **Date:** 2026-08-25 +- **Decision owners:** Anselme Gaeremynck +- **Reviewers:** David Irvine, Chris O'Neil, Mick van der Most van Spijk +- **Supersedes:** none +- **Superseded by:** none +- **Related:** ADR-0002 (gossip-triggered subtree audit), ADR-0003 (possession checks), + ADR-0004 (commitment-bound quote pricing), ADR-0007 (Windows LMDB map headroom cap, + retired by this decision) + +## Context + +The node stores chunks in LMDB. LMDB returns a deleted page to its own free list and never +to the filesystem, so **deleting chunks does not free disk**. In one week the fleet deleted +2.29 million chunks and got back zero bytes. Operators read that as a bug and are tempted +to wipe node directories to reclaim space, which costs the network real replicas. + +There is no partial way out. Compaction needs free space equal to the live data, which is +exactly the condition a full node does not meet, and it does not get us off LMDB anyway. +Punching holes in `data.mdb` is Linux-only, needs LMDB internals to identify free pages, +and reads back as zeros. Disk comes back exactly once: when `chunks.mdb` is removed whole. + + peak disk during migration = allocated chunks.mdb (unchanged) + files written so far + +So a local migration is possible if and only if `free >= live payload`. On production +volumes today (55 volumes at 492 GiB, free median 25.8 GiB, p10 9.8 GiB, about 12 nodes +per volume, about 38 GiB of LMDB per node of which about 24 GiB is live) migrating one +node costs 24 GiB and returns 38 GiB. One at a time the host gains about 14 GiB per node +and the queue accelerates. All twelve at once need 288 GiB and all twelve stall. + +The chunk workload is the easiest possible case for a filesystem: content-addressed, +immutable, write once, read many, delete whole, and **4 MiB**, confirmed by the team +rather than assumed. That size is what makes one file per chunk the right shape; see the +Storj and borgbackup note under Validation for what would change the answer. + +## Decision Drivers + +- Deleting a chunk must return its blocks to the filesystem, on a full disk, with no free + space required and no compaction to schedule. +- No chunk may lose its last replica, including during a fleet rollback, a skipped + upgrade, or a crash halfway through the migration. +- Mass audit failures are as damaging as data loss. Nothing here may cause them. +- It has to work for every operator, not for our fleet. Most node operators are not us and + cannot be told to attach a second volume. +- No opt-in. Whatever we ship is what every node does by default. + +## Considered Options + +1. **Stay on LMDB and compact.** Needs free space equal to the live data, which is the + condition we are trying to escape, and leaves us on LMDB. +2. **Append-only packs** (borg segments, Storj hashstore). Reintroduces compaction, a free + list, and a cross-file index. That is LMDB's disease with a different allocator. +3. **Fixed-size slots** (Sia `hostd`, Swarm sharky). Cheaper than log packing, and Sia's + sector size is exactly our 4 MiB. But a freed slot returns space to the *store*, never + to the *filesystem*: the volume file never shrinks. It is the right design once a node + has a declared capacity, and the wrong one while our whole complaint is that disks stay + full as chunk counts drop. +4. **One file per chunk, sharded on the address prefix.** Broken for us, see below. +5. **One file per chunk, sharded on the address suffix.** Chosen. + +## Decision + +### The store + +One immutable file per chunk: + +```text +{root}/chunks/layout.json versioned layout marker +{root}/chunks//<64-hex> xy = the LAST two hex characters of the address +``` + +**Suffix, never prefix.** A node holds keys it is among the `CLOSE_GROUP_SIZE` closest to, +so its holdings share roughly `log2(N / 7)` leading bits with its own node ID, and that +shared prefix grows as the network grows. Distinct directories a single node would actually +use, sharding on the first hex characters: + +| nodes | shared bits | 2 hex | 3 hex | 4 hex | +|---:|---:|---:|---:|---:| +| 1,000 | 7.2 | 1.8 | 29 | 459 | +| 10,000 | 10.5 | 1 | 2.9 | 46 | +| 100,000 | 13.8 | 1 | 1 | 4.6 | +| 1,000,000 | 17.1 | 1 | 1 | 1 | + +At today's ~800 nodes a two-hex prefix is already down to about two directories. Prefix +sharding does not degrade, it fails, and it fails later for the nodes that grow into it. +Close-group membership constrains the leading bits and places no constraint at all on the +trailing ones, and the address is a BLAKE3 output, so the last byte is uniform by +construction at every network size. IPFS shipped the same fix for a different reason: its +prefixes were constant because of the CID encoding, not because of clustering, and the +flatfs `_README` still says *"Previously, we used prefixes, we now use the next-to-last two +characters."* The generalisation is the part worth keeping: **shard on bits you can prove +are uniform, not on bits that happen to be uniform today.** + +**256 shards, one level.** 23 files per directory at today's ~6,000 chunks per node, 977 at +a 1 TiB node, 39,000 at 10 TiB, for 1 MiB of directory inodes. 4,096 shards only starts to +pay past several million chunks and costs sixteen times the directory overhead for every +node that is not that large. + +**Lowercase hex filenames, full 64 characters.** NTFS and default APFS fold case, so under +base64url or base58 two distinct keys can share one case-folded filename, which is a silent +overwrite. No hex string can spell `CON`, `NUL`, `AUX`, `COM1` or `LPT1`, because none of +those letters is in `0-9a-f`. Keeping the whole key in the name means a `find` over the tree +recovers the store even if the directory layer is lost. + +**The scheme is recorded in `layout.json` at creation.** Nobody in this survey shipped an +in-place re-sharder and all of them paid for it: IPFS says export and re-import, Storj ran a +multi-year satellite-controlled backend migration, borg rewrites only on the next +compaction. One small file is the difference between changing the default later and never +being able to. + +### The index + +**The filesystem is the sole authority.** The key set is a `BTreeSet` rebuilt at +every open by reading directory entries, names only: no `stat`, no content read. A `stat` +per entry costs about ten times the enumeration on Linux and macOS and fifty to sixty times +on Windows, and buys nothing, because the filename is the key. + +No sidecar database, because a persistent index **cannot remove reconciliation**. Commit +the index first and a crash leaves a phantom key; rename the file first and a crash leaves +an unindexed file. Repairing either means looking at the filesystem anyway, so the +filesystem may as well be the authority, and then nothing can drift. Ceph FileStore's +tracker #17177 is the cautionary tale: a crash between `unlink` and the LevelDB flush +orphaned omap keys that were silently reattached to a different object later. + +`BTreeSet` rather than a hash set for three reasons: `all_keys()` must be sorted (the +commitment builder truncates the responsible subset with `take(cap)` *before* the Merkle +tree sorts it, so an unstable order would make the published commitment depend on iteration +luck), it never spikes memory while growing, and bulk-building it from a sorted vector packs +every node to capacity where repeated insertion converges on 68% fill for the same keys. + +**One process per data directory, enforced.** LMDB was genuinely multi-process safe. This +store is not: two of them keep independent in-memory indices, so both would report the same +write as newly stored and each would keep serving keys the other had deleted. A node whose +store is already held by another process refuses to start and says so. + +**Every in-memory mutation mirrors a filesystem operation that has already completed**, and +never anticipates one. Bitcask's issue #114 is what the opposite order looks like: an index +rebuilt at startup and then mutated in place drifted to 2,400 keys pointing at fewer than +100 files. + +### Durability + +Write, on Unix: reserve capacity, create a temp in the **destination** directory, write, +flush the file, rename, flush the shard directory, then admit the key. The publish is an +intra-directory rename, so it is atomic on every Unix filesystem we support and only that +one directory needs flushing. Off Unix there is no rename at all, for the reason the table +below gives; the file is created under its final name and flushed. Either way the final +name can never appear on partial content, because the name is the hash and a name that does +appear over the wrong bytes is caught on read. Delete: unlink, flush the shard directory, +then drop the key. + +Per platform, honestly: + +| | rename atomic | fsync(temp) + rename durable | directory fsync | +|---|---|---|---| +| ext4 | yes | **no**, `auto_da_alloc` only orders data before the rename's commit | yes, required | +| XFS | yes | not by that sequence | yes | +| btrfs | yes | **uncertain**, ALICE found reordering | yes | +| APFS | yes | `sync_all` already uses `F_FULLFSYNC` on Apple targets | returns 0, effect undocumented | +| NTFS | **not documented as atomic** | see below | **no documented way** | + +On Windows a node cannot make the rename durable through the standard library at all. Two +places in this design leaned on one, and neither leans on it now. + +Publishing a chunk off Unix does not rename: it creates the file under its final name and +flushes it, which Microsoft documents as flushing the creation metadata with it. The +content is content-addressed and re-replicable either way, so a file that does not survive +is detected on read and repaired from the network. + +Retirement still renames the environment aside before deleting it, and that rename is not +durable off Unix. What makes it safe is that **the mark goes inside the directory, not +beside it**. A power loss that reverts the rename brings the directory back under its live +name still carrying its mark, and a marked directory under the live name is never opened or +served from: its chunks are in the file store, which is what the mark records. A loss +before the mark leaves the directory unmarked under either name, and an unmarked directory +is always restored and reopened. Every one of those four states has a test. + +So the earlier position, that Windows should refuse to delete the legacy environment until +an operator overrode it, is not what ships. It has been replaced by a mechanism rather than +by a policy, which is the better answer: a switch nobody turns on is a migration that never +finishes. `ANT_MIGRATION_RETIRE_LEGACY=0` remains, per node, for an operator who wants to +hold retirement off one machine, and the forced power-loss run below is still an open fleet +gate on every platform including this one. What that run is now checking is directory +creation, which has no portable flush. + +### Retiring LMDB + +Three releases, because slashing is the *auditor's* decision. A node that has to give up +chunks cannot stop its auditors from penalising it, so the auditors have to stop first. + +| Release | Penalise a peer for not holding a close-group chunk? | Delete `chunks.mdb`? | +|---|---|---| +| **First**: stop one penalty | no | no | +| **Second**: migrate | no | yes | +| **Third**: restore it | yes | yes | + +What the first release withholds is deliberately narrow: only the penalty for **not holding a close-group +chunk you were supposed to be holding**. The commitment-bound subtree audit still +penalises, in every release. So does a responder whose own storage fails: a fetch answered +with an error means the read faulted or the bytes no longer hash to their address, which is +never what a node giving chunks up looks like, and a node that does not hold the chunk says +so with `NotFound` instead. That is not a compromise, it is what makes the rest work: a +node reduces its commitment precisely so its peers hold it to the smaller claim, and +suspending that enforcement would make the reduction meaningless. Audits of both kinds run +and record throughout. + +Both are **build constants with environment overrides, never serialised config**. A node +writes its effective configuration back to disk, so shipping them as ordinary fields would +bake the first release's values into every operator's file and the next would change nothing. + +Per node, in order: + +1. **Open both stores.** Reads are the union, writes go to files. New chunks are also + written to LMDB **first** while it exists: a chunk uploaded during the bridge to holders + that all revert to a pre-migration build would otherwise be gone from every one of them, + and that is client data, not a replica. +2. **Copy closest first**, throttled, stopping at a slack floor above the disk reserve. +3. **Settle.** The node commits only to its file-backed keys from here, while still serving + everything it ever committed to. Serving reads the union; the commitment reads the + file-backed set. A node is at worst over-honest. Nothing is deleted at this step: it + only narrows the claim, so the close group can learn the new one before anything goes. +4. **Verify.** Every chunk both stores hold is re-hashed and recopied from LMDB on + mismatch. A filename is not proof the bytes behind it are good, and the startup scan + reads names only. +5. **Retire.** Once the retirement delay has elapsed, at least two commitment rebuilds have + been published, and no key the node is giving up is still answerable under a retained + commitment slot: rename `chunks.mdb` aside, flush the parent, record the node as + file-only, and only then delete it. The rename is what makes the state change atomic, + because `remove_dir_all` is not: a failure partway through leaves a directory that can + no longer be opened as an environment, and recording completion on top of that would + have the node claim it had finished over a half-deleted store. **This is where the disk + comes back.** The gates that can change while nobody is looking are rechecked inside + the destructive step itself, in the same critical section that proves no other task + holds the store: the proof's health generation, the answerability veto, the announced + writes, and that every legacy-only key is in the approved set. The network gates, rank + and commitment delivery and possession, are rechecked immediately before that call and + outside the guard, so the window on those is the seconds it takes to take the guard + rather than the hours the verification pass can run for. Both matter, and they are not + the same claim. +6. **Refetch** the shortfall through ordinary replication, with the freed space to do it in. + +The delete gate is the pruner's existing retention contract +(`ResponderCommitmentState::is_held`, `GOSSIP_ANSWERABILITY_TTL` three hours). No new +protocol. + +**Nothing is given up without proof it exists elsewhere.** Only nodes that cannot fit +their payload give up anything at all, and such a node must clear three gates, in this +order, before a byte is deleted: + +1. **It is not near the front of the group for the chunk.** Only the last two positions of + the *admission group* (`storage_admission_width`, the close group plus its margin) are + eligible, which is the width the pruner treats as strictly in-range and refuses to + delete inside. A one-off migration must not be more willing to drop a chunk than the + thing that runs every day. +2. **Its close group has received the reduced commitment.** The node narrows what it claims + first, and only once peers have demonstrably received that narrower claim, proven by + them answering a neighbour sync that carried it, may anything be deleted. Until then + they audit it against the set it used to hold, and a wave of audit failures is as + damaging as losing the chunks. +3. **Other nodes have proven they hold the chunk, and are currently publishing a claim.** + All but one of its current close group must answer a cryptographic possession challenge + over a nonce they have never seen. This is the pruner's own evidence, reused + deliberately, and it is deliberately not the cheap `VerificationRequest`: that carries a + self-reported `present: bool`, and a node that has silently lost a chunk still answers + yes. A peer only counts if this node has also heard a commitment from it recently, which + excludes a peer sitting between a retired commitment and its next rotation. That gap is + exactly what a node in the middle of its own migration looks like, and counting it would + let two migrating nodes each conclude the other was covering the chunk. + +Rank alone would not do. Being far from a chunk says something about who *should* hold it, +not about who *does*, and in a fleet-wide migration the nodes that should hold it are +exactly the ones that may also be short of space. Without gate 3 the safety property is +merely statistical: every holder could be short at once and each drop the same chunk, and a +per-volume lock cannot see that, because it serialises one volume and this is a +network-wide question. + +A node that cannot clear these gates keeps both stores, does not free its disk, and tells +the operator to add storage. That is the correct answer, not a smaller replica count. + +Gates 2 and 3 are re-checked immediately before the environment is removed, not once when +the node settled hours earlier. The group moves, and two paths can put a key back into the +legacy-only set in between: a file that failed verification and is now being served from +the legacy copy, and a write whose file half failed. + +**Two of a close group at a time, not seven.** The gates above are per chunk, and they are +safe, but on their own they deadlock: if every holder migrates at once, none can prove to +the others that a copy survives and the whole group sits waiting. So each node derives a +migration wave from a hash of its own ID, and a group of seven is split into four waves. +Wave `w` opens `w * wave_hours` after the build first starts. It needs no coordination and +no protocol change, which matters because a node cannot usefully ask its close group "are +you migrating?" and would not trust the answer by the time it arrived. + +It is a stagger, not a guarantee: seven IDs hashed into four waves will not always land two, +two, two, one. What makes it safe rather than merely tidy is that it composes with the +possession gate. A node whose turn has come still cannot give a chunk up until its +neighbours prove they hold it, so an unlucky wave waits instead of over-shedding. Only nodes +that have to give something up wait for a wave; a node with room copies and retires +immediately, because it is never unable to serve. + +Separately, a host-wide advisory lock serialises migrations sharing a volume, held from the +first copy through retirement, so a node cannot release it and let eleven others start +before it has finished copying. It is released when the environment is unlinked and its +directory renamed aside, not when the last byte comes back: the deletion itself runs +detached so the node can serve while it happens, and it can take minutes on a large store. +So the next node in the queue can begin its copy while the previous one's tombstone is +still on the disk. That is deliberate, and it is worth stating rather than claiming a +tighter guarantee than there is. The two limits answer different questions: the lock is +about one machine's disk, the wave is about one chunk's replicas. + +Where the lock file lives is a deployment fact, and the wrong answer is silent: nodes that +cannot see each other's lock each take one and report success. A host whose nodes do not +share a `/tmp`, which is any host using `PrivateTmp=true`, has to be told where the lock +lives through `ANT_MIGRATION_LOCK_DIR`. The node logs the path it locked at so this can be +answered from a log rather than inferred from a unit file. + +## What the review added + +Five mechanisms are in the implementation that are not in the design above. Each exists +because adversarial review found a way for the destructive step to run on a belief that +was no longer true. They are recorded here because they are load-bearing, not incidental. + +**A directory that has been retired says so from the inside.** The rename that moves the +environment aside cannot be shown to be durable off Unix, so a power loss can bring it back +under its old name with its contents already deleted, and a node that opened that would +fail to start. A file written inside it after the rename and before any deletion travels +with the directory, so what it is never has to be inferred. A mark beside the environment +was tried first and was wrong: it would have to be cancelled when a retirement is abandoned, +cancellation can fail or be lost, and a stale one authorises deleting an environment that +has since taken a chunk. Deletion removes the mark last, so a failed deletion never leaves a +retired directory looking intact. + +**A chunk the node cannot serve is kept but not claimed.** Deleting it, or dropping it from +the index, puts the key in neither the file store's view nor the legacy one, and what +neither view protects is what retirement destroys. Claiming it puts the key in signed +commitments and answers presence probes with a yes for a chunk that cannot be served, which +the commitment-bound audit penalises. So the file stays and the answers stop. Two states, +not one: a chunk that could not be *read* is settled by a later read, and one whose bytes +were *proven wrong* is not, because reading them again says the same thing. + +**A verification proof expires.** The pre-retirement pass reads every chunk, and its result +is reused rather than re-read on every tick, because retirement is usually deferred by a +gate that has nothing to do with the files. A kept chunk that stops being servable in that +window is invisible: ordinary requests are still served from the legacy copy. The store +counts the times a chunk stops being servable, a proof records that count, and retirement +refuses a proof the store has outrun. + +**A write announces itself before it starts.** The work runs on a blocking thread that +outlives the future waiting for it, so a cancelled write can leave the environment holding a +chunk that nothing recorded. The announcement is deliberately not part of what the node +claims to hold: it vetoes retirement and is reconciled against the disk, but no commitment, +quote or presence answer sees it. A delete drains both halves of any announced write for its +key, so a publish cannot land afterwards and undo it. + +**The legacy environment never grows again.** Both stores sit on one disk, each measures +the same free space, and neither knows what the other is about to spend, so a chunk written +to both can be admitted twice against one lot of headroom and enough of them can cross the +reserve together and fill the volume this whole exercise exists to free. From the moment it +is adopted the environment is pinned to what it already occupies: it writes only from pages +it already has, and the file store's accounting becomes the only claim on free disk. The +cost is that the rollback copy is made only when the environment has room of its own, which +on a real node it usually does, because this migration exists precisely because deleting +millions of chunks filled the free list and returned nothing to the filesystem. + +The category underneath all five is the same: **a fact established at one moment being acted +on at another.** Copying, verifying and retiring are separated by hours by design, and every +gap between them is somewhere the store can move. The pattern that works is to make the +belief carry its own expiry — the directory carries its mark, the proof carries the count it +saw, the write carries its note — rather than to check again and hope the check is close +enough to the act. + +## Consequences + +### Positive + +- `unlink` returns blocks immediately. No free list, no compaction, no free space required + to reclaim space. This is the entire point. +- `exists()` and `current_chunks()` become in-memory lookups with no syscall, cheaper than + the LMDB reads they replace. +- `all_keys()` gains a stable ascending order, which the commitment builder needs and the + pruning cursor wants. +- A fresh node never opens a memory map at all. `storage.db_size_gb` and ADR-0007's Windows + map headroom cap die with LMDB. +- The store is self-describing: the filename is the hash, so an operator can verify a chunk + with `b3sum`, and a scrambled directory layer is recoverable with `find`. + +### Negative / Trade-offs + +- **There is no rollback once a node has deleted its LMDB.** The staged rollout is the only + control: a small leading batch, ours, and a wide window. +- **The window between the first and third releases is publicly known, and in it nobody is + penalised for failing to hold a close-group chunk.** The cheapest way to exploit it is + precise and worth writing down: a modified peer that never gossips a commitment at all is + credited as a legacy node, can answer `Present`, and can then return `NotFound` or fail a + possession check with no trust cost. It pays only for an identity and the traffic. One + such identity removes one of seven replicas; control of all seven positions removes the + chunk's availability. The commitment-bound audit is untouched, so this only works for a + peer that publishes no commitment at all, which is itself visible. The mitigation is not + a code change, it is not letting the third release slip. + It is bounded, because the third release evicts afterwards, and audits keep recording so we + can see it happening, but it is a real invitation for the duration. +- `exists()` is now an index lookup rather than a read of the backing store, so something + outside the node deleting files is not noticed until the next read of that key. The read + path self-heals, and a `stat` per call on the node's hottest path is not worth it. +- One inode and one directory entry per chunk. At 4 MiB per object that is 0.05% overhead + and block rounding for a full chunk is exactly zero, but it is real. +- Windows publishes chunks under their final name rather than by rename, so a crash + mid-write leaves a partial file that the write, read and pre-retirement paths each have + to detect rather than trust. +- The paid list is still LMDB. It is a fixed 256 MiB map that contributes nothing to the + disk problem, but it is why `heed` cannot be dropped yet. +- **Narrowing the commitment cuts the quoted price.** Price is quadratic in the committed + key count, so a node that has just proved it is short of disk advertises a cheaper quote + than its close-group peers and then refuses the store on capacity. A wasted round trip + rather than a mispayment. The fix belongs to the quote path and is a separate decision. +- **A cancelled awaiter drops the per-key lock while its blocking write runs on.** This was + accepted as bounded and is no longer accepted: review showed both consequences were worse + than they look. The file store now records what it is writing, per key, cleared by the + worker rather than the caller, and a delete waits out whatever is already writing its + key, so a publish cannot land afterwards and undo a prune. A write into the legacy + environment announces itself before it starts and is reconciled against the disk, so a + cancelled one cannot leave a chunk that neither view protects. + +### Neutral / Operational + +- Startup cost is the directory scan: 122 ms warm and 1.55 s cold at 250,000 files across + 256 shards on APFS, of which the index build is 2 to 11 ms. No fast-start snapshot in v1. + If one is ever added, validate it with the Merkle root of the sorted key set (which + ADR-0004 already computes) rather than a checksum, because a checksum passes for an + operator who restores yesterday's data directory and leaves yesterday's snapshot. +- APFS enumeration degrades with churn, not just size: a million files went from about 72 + to about 306 microseconds per entry over twenty cycles of 5% replacement. A long-lived + macOS node will get slower to start in a way a fresh benchmark never shows. +- NTFS 8.3 short-name generation is worse for us than for most, because a node's filenames + genuinely share a long prefix. Microsoft advises disabling it above 300,000 files per + directory. + +## Validation + +**Already proved, locally:** publish is exactly-once under sixteen concurrent writers of one +address; the index rebuilds from the filesystem across restarts with a stable order; a file +in the wrong shard, an uppercase name, and a non-hex name are all refused; an interrupted +write is swept; a corrupt file is removed and repaired from the legacy copy; a missing file +drops out of the index so replication repairs it; the copier is resumable and cannot +resurrect a pruned chunk; retirement is refused while any gate is unmet and removes the +environment when they are all met; the release switches never round-trip through a config +file. + +For the four mechanisms above: an environment carrying no mark is kept however badly it +reads, one carrying its own mark is removed whatever it is named, and the mark survives the +rename it exists to outlive; a chunk that cannot be read is kept on disk, not acknowledged, +not advertised, and answered for again once it can be read; a verification overtaken by a +file that stopped being readable does not authorise a deletion; a write in flight is not +claimed but does stop retirement; a delete outlasts a write nobody waited for; and a key the +environment holds that is in neither view refuses the proof and is put back where the gates +can see it. Each was verified by removing the fix and confirming the test fails. + +**Proved in CI, on every commit.** The three harnesses that touch durability run on Linux, +macOS and Windows, and again on ext4, XFS and btrfs loopback volumes. The fourth measures +what one file per chunk costs at scale, which is a fleet question on a fleet that is Linux, +so it runs there: + +- *The disk comes back.* Free space is sampled from the filesystem three times: before + anything is written, at the peak where both stores hold everything, and after the + environment is gone. Unlinking the environment while holding it open, which makes the + paths disappear and keeps every block, fails it. This is the claim the whole decision + rests on and the one the old store could not meet. +- *A crash loses nothing.* A child process is killed at a failpoint inside a publish, not + after a sleep, so the kill lands where a half-finished chunk exists. What the parent then + checks is that nothing is claimed that cannot be served, that a leftover is swept, and + that a chunk caught between the environment write and the file write is named on the + copier's list rather than lost between them. The same is done to a retirement: a child is + killed with the environment renamed aside and marked, nothing yet deleted, which is the + most destructive moment in the migration. The next start must finish that deletion and + never reopen the directory, because the node has already told the network it serves those + chunks from the file store. Refusing to believe the mark fails it. +- *Nodes sharing a disk take turns.* Two drivers on one volume, driving `migration::run` + rather than the copier, with the lock held first by an outsider so neither can be observed + making progress. Held through retirement as well as through copying, which is the heavier + half. Removing the lock from either branch of the driver fails these. +- *One file per chunk costs what was claimed.* 100,000 chunks, measured rather than + asserted: the startup scan takes about 100 ms, the index costs 52 bytes per chunk, opening + the store reads 125 bytes whatever the chunks contain, and `put` writes exactly one + directory entry per chunk. The claim underneath the scan's cost, that it reads names and + does not `stat` behind each one, is checked against the machine rather than against a + number: the same directory is walked twice in the same process, once reading names and + once calling `metadata` on every entry, and the scan has to land on the names-only side of + the two. A flat ceiling cannot settle that, because one `stat` per entry costs about three + times a bare walk and stays well inside any ceiling loose enough not to flake. Adding that + `stat` to the scan fails it. + +Each of these was checked by mutation: the fix removed, the test confirmed red, the fix +restored. + +**Fleet gates, which cannot be closed from a workstation:** + +- Forced power loss on ext4, XFS, btrfs, APFS and NTFS showing old-or-new, with antivirus + and 8.3 generation enabled on the NTFS run. The publish path off Unix does not rename at + all, precisely because a rename cannot be shown to be durable there: it creates the chunk + under its final name and flushes the file, which is documented to flush the creation + metadata with it. What that leaves unproven is directory creation, which has no portable + flush, so this run is what closes it. `ANT_MIGRATION_RETIRE_LEGACY=0` holds retirement off + a node until then, per node, without a separate build. + + The loopback jobs above do **not** close this and are not offered as doing so. Killing a + process and reopening the same mounted filesystem keeps the kernel page cache, so the + bytes written before the kill are still there to be read; removing every flush from the + publish path would leave those jobs green. What they do cover is the rest of what a + filesystem decides: rename behaviour, locking, deletion, and whether the space is actually + returned, which btrfs in particular accounts for differently from ext4. +- Startup scan, RSS and inode use at 1M and 10M keys, and on each filesystem. CI answers + 100,000 keys on ext4 and prints every number it measures, so drift is visible in the log + before it trips a gate; `ANT_SCALE_KEYS` raises the count for a deliberate larger run on a + machine with the disk for it. What CI cannot answer is where the curve stops being linear, + which is a question about a machine holding ten million files, not about the code. +- The first release gates on no audit-timeout regression on the quiet responsible lane and on + disk growth + matching prediction. +- The second gates on a soak of the first, plus a verified retirement returning the + predicted space. +- The third gates on migration-complete lines across the fleet, refetch backlogs drained, and the + recorded audit failure rate back to its pre-migration baseline. The first release's + observability is + what makes that decidable. +- **How often a short-of-disk node can actually clear the possession gate.** A node whose + close group is also short of space will not clear it, will not free its disk, and will + tell its operator to add storage. That is the intended answer, but the fleet needs to + show how large that population is before the second release, because it decides whether the + migration + completes on its own or needs operator action at scale. +- **Chunk size is 4 MiB, confirmed.** This was the open question that gated the whole + design and it is now answered. Storj and borgbackup both ran one file per object at scale + and reversed to packing, and both did so for *small* objects: Storj's pieces are *"often + smaller than a hard drive sector"* and over 60% of borg's chunks are under 8 KiB. Nobody + has reversed this decision for large objects. The tripwire remains: if the network ever + starts storing a large share of small records, this ADR should be revisited, and the + inode exposure below comes with it. + +**Review trigger:** if the network ever adopts a declared node capacity, fixed-slot packing +becomes the better store design and this decision should be reopened. + +## Implementation slices + +This ADR is landed by two pull requests, in this order: + +1. **Stop penalising a node for not holding a close-group chunk.** One switch, one helper, + six call sites. It must ship a release ahead of the migration, because the penalty is + the auditor's decision and a node cannot stop its peers applying it. The commitment-bound + subtree audit keeps penalising throughout. +2. **The file store and the migration.** Everything else in this document. + +A third release flips the switch from (1) back, gated on fleet evidence rather than a date, +which is why it is a release and not an expiry constant compiled into the first one. + +## Notes for AI-assisted work + +Drafted with AI assistance. Not to be marked Accepted without human review. diff --git a/docs/adr/ADR-0015-remove-the-lmdb-chunk-store.md b/docs/adr/ADR-0015-remove-the-lmdb-chunk-store.md new file mode 100644 index 00000000..f2695bff --- /dev/null +++ b/docs/adr/ADR-0015-remove-the-lmdb-chunk-store.md @@ -0,0 +1,190 @@ +# ADR-0015: Remove the LMDB Chunk Store and Restore the Close-Group Penalty + +- **Status:** Proposed +- **Date:** 2026-08-28 +- **Decision owners:** Anselme Gaeremynck +- **Reviewers:** David Irvine, Chris O'Neil, Mick van der Most van Spijk +- **Supersedes:** none +- **Superseded by:** none +- **Related:** ADR-0014 (one file per chunk, and retiring LMDB), which this completes + +## Context + +Moving chunks off LMDB shipped as three releases, because the penalty for not holding a +close-group chunk is the *auditor's* decision: a node that has to give chunks up cannot stop +its peers punishing it for that. So the peers stopped first. + +1. **First:** suspend that one penalty. +2. **Second:** copy every chunk into a file of its own, then delete `chunks.mdb`. ADR-0014. +3. **Third:** this one. + +ADR-0014 describes the third release as flipping the switch back and nothing more. What +actually has to happen is larger, and two parts of it are decisions rather than clean-up. + +## Decision + +**Restore the penalty, and keep its switch.** The constant goes back to `false`. The +process-wide atomic, the `ANT_SUSPEND_UNHELD_CHUNK_PENALTY` override and the startup +announcement all stay. They are not migration machinery: they are one release-level policy +that several audit paths have to obey identically, and the release that restores a penalty +is exactly the one most likely to need it undone in a hurry. Removing them would discard the +cheapest lever at the moment it is most useful. A test now pins the shipped value, because +the existing tests set the switch both ways on purpose and so could never notice which way +it was compiled. + +**Delete the LMDB chunk store and the migration, and keep the name `ChunkStore`.** There is +one store. It is one file per chunk, it lives in `src/storage/chunk_store.rs`, and it is +called `ChunkStore` because that is what it is and what every caller already called it. The +type that used to present two stores as one is gone with the second store. + +`heed` stays in the dependency list. The paid-key list has its own LMDB environment, which +this decision does not touch. + +**A node that still has an unretired `chunks.mdb` refuses to start.** This is the part worth +arguing. + +The tempting answer is to start anyway, serve what is in the file store, and warn. It is +wrong. The chunks in that environment are unreachable to this build, but the commitment this +node published before the upgrade *claimed* them, and a commitment stays answerable to its +neighbours for three hours (`GOSSIP_ANSWERABILITY_TTL`, which is `(RETAINED_GOSSIPED_ +COMMITMENTS + 1)` rotations). The accusation the first release suspended was "you did not have a chunk you +were supposed to hold". The commitment-bound subtree audit was never suspended in any +release, precisely because it rests on a signed claim. So a node that starts half-migrated +spends hours failing audits, at the full weight, on the one lane that always counted. It is +not a smaller node; it is a node being slashed for keys it cannot read. + +Refusing *everything* is also wrong, for a duller reason: a migration that finished and then +failed to delete the directory leaves one behind that is safe to ignore. A node whose only +fault is a failed `remove_dir_all` should not be held offline for it. + +So the question is not "is there an environment here" but "was it retired", and the evidence +is the mark the retirement wrote *inside* the directory before deleting anything: + +| what is on disk | what happens | +|---|---| +| nothing, or a root that does not exist yet | start | +| `chunks.mdb` or a tombstone carrying its `RETIRED` mark | start, warn that it is costing disk | +| either one with nothing in it | start, warn that it can be removed | +| either one, non-empty, without that mark | refuse, and name the directory | +| either one whose mark cannot be read | refuse, and say which | +| a root that cannot be read well enough to answer | refuse, and say so | + +The empty case is not a nicety. The previous release's cleanup emptied the directory, +removed the mark and then removed the directory, so a crash between the last two steps +leaves one that is empty, unmarked, and fully migrated. That release recognised the state +and tidied it. Refusing over a directory with nothing in it would be an outage for +bookkeeping. + +The check runs as soon as the root is known and before the transport is built. Asking later +lets a bind failure mask the answer, and charges a node for a transport it is about to throw +away. + +Three states rather than two, for the same reason the release that wrote those marks needed +three: reading one can fail for a reason that is neither yes nor no, and folding that into +"no mark" holds a node offline forever over an unreadable directory, while folding it into +"retired" waves through a live one. + +Not the migration marker file. The filesystem is authoritative, and ADR-0014's own recovery +resets a recorded `FilesOnly` phase back to bridging when it finds a live environment, so a +marker saying the migration finished is not evidence that it did. + +Tombstones are checked as well as the live name. A crash between the rename and the mark +leaves an intact environment wearing a retired-looking name; the previous release would have +restored and reopened it, and this one cannot, so it must not be waved through on the +strength of what it is called. + +Nothing of the old store is deleted. This build has no migration code, so it has no business +deciding that a directory it cannot read is safe to remove, and leaving it is what keeps a +rollback to the previous release possible. To be exact about what "nothing" covers: this +release does not migrate, delete, rename or rewrite `chunks.mdb`, does not rewrite the +migration marker, and does not change the file-store layout. It still does everything a node +normally does to its own chunks, and opening the store still creates the store's own files +and sweeps orphaned temporaries, as the previous release did. + +## Consequences + +### Positive + +- One store, one name, and about 5,600 lines of bridge and driver gone. +- The penalty means what it always meant again. +- A node cannot silently serve a fraction of what it is committed to. +- The per-volume migration lock and its deployment settings go with the migration. + +### Negative / Trade-offs + +- **A node that never finished migrating will not start.** That population is exactly the + short-of-disk nodes, and how large it is remains the open fleet question ADR-0014 records. + If a whole wave refuses at once, that is an availability incident, and the right response + is to stop the upgrade wave rather than to start those nodes blind. A rollout that cannot + halt on that signal is unsafe independently of this decision. +- Restoring the penalty and deleting the bridge in one release means the emergency lever for + the first is a switch, while the second can only be undone by rolling back the binary. + Shipping them as separate releases was considered and is a legitimate call for whoever + cuts the train; the two are separate commits so that remains possible. +- `ChunkStore` and its module were renamed from `FileStore` and `file_store.rs`. Callers did + not change, because they already used the facade's name. + +### Neutral / Operational + +- `ANT_SUSPEND_UNHELD_CHUNK_PENALTY` still works and still logs loudly when it disagrees + with the build. +- `storage.migration` and `storage.db_size_gb` are gone from the configuration. The second + capped a memory map that no longer exists, and a setting that silently does nothing is + worse than one that is absent. Nothing declares `deny_unknown_fields`, so a config file + written by the previous release still loads with both keys in it, which is what stops + every node on the fleet failing to start at once on upgrade. There is a test for that, + because adding that attribute later would look harmless. +- The `chunks.mdb` a refusing node names can be moved aside by hand once its contents are + known to be copied. There is no supported way to make this build read one. + +## Validation + +**Proved here.** A node with no leftovers starts; one with a retired leftover starts; one +with an empty leftover starts; one whose root does not exist yet starts. One with an +unretired environment refuses, and the test reads the message to check it names the +directory; one with an unretired tombstone refuses; one whose mark cannot be read refuses +with a message that says so; one whose root cannot be listed refuses. The warnings the +starting cases emit are not asserted, only the refusals' messages are. The unreadable case is staged with a +symbolic link pointing at itself, so looking for the mark returns a loop while everything +else about the directory keeps working, which is a state any user can reach and root cannot +skip. Restoring the penalty is pinned by a test that fails if the constant is flipped back. + +**Deleted, and what replaced it.** ADR-0014's validation section describes four harnesses. +Most of three of them existed to prove the bridge worked: that the disk came back when the +old store was deleted, that a node killed mid-copy lost nothing, and that several nodes on +one disk took turns. There is no bridge left for those to test. The fourth, which measures +what one file per chunk costs at scale, stays. + +Not all of it went, and saying it did was wrong. Two tests inside the crash harness were +never about the bridge: that a process killed mid-publish leaves no chunk the store cannot +serve, and that what an interrupted write leaves behind is swept. Those are about the store's +own publish path, which is now the only one there is, so they matter more after this release +rather than less. They are back as `tests/chunk_store_crash_safety.rs` and run in CI. A third +property, that engine shutdown waits for a detached store write, is named under the gaps +below. + +That leaves the loopback filesystem job with nothing to run, and deleting it would quietly +drop ext4, XFS and btrfs coverage of the store itself. It now runs the storage unit tests +against each mounted filesystem instead, which is what still has something to say there: +publishing through a temporary and a rename, flushing, deleting, and rebuilding an index +from the names. + +**Not proved here, and inherited from ADR-0014.** Forced power loss on the five filesystems. +Scale at one and ten million keys. How many short-of-disk nodes can clear the possession +gate, which this decision makes sharper: under ADR-0014 such a node kept serving from both +stores, and under this one it does not start. + +**Coverage this release drops, named rather than lost.** A harness proved that +`ReplicationEngine::shutdown()` waits for a store write whose awaiter was dropped before it +returns. It was written against the old store and went with it. The property is still +current and still claimed by that method's own documentation, and nothing tests it now. It +needs a live P2P node to stage, which is why it is called out here rather than quietly +rewritten in the same change that deleted it. + +**A fleet gate this decision adds.** Before this ships, the fleet has to show that nodes are +actually on the file store. The count that answers it is nodes reporting a completed +migration; the ones that cannot are the ones that will refuse to start. + +## Notes for AI-assisted work + +Drafted with AI assistance. Not to be marked Accepted without human review. diff --git a/scripts/adr-governance.py b/scripts/adr-governance.py index 7f56bde9..c8e6da44 100755 --- a/scripts/adr-governance.py +++ b/scripts/adr-governance.py @@ -53,6 +53,12 @@ def changed_files_against_base(base: str) -> list[str]: return [] +def base_adr_names(ref: str) -> list[str]: + """ADR filenames present on `ref`.""" + listing = run(["git", "ls-tree", "--name-only", ref, "docs/adr/"]) + return [Path(line).name for line in listing.splitlines() if line.startswith("docs/adr/ADR-")] + + def file_at(ref: str, path: str) -> str | None: try: return run(["git", "show", f"{ref}:{path}"]) @@ -83,6 +89,29 @@ def main() -> int: errors.append(f"{path}: duplicate ADR number also used by {seen_numbers[number]}") seen_numbers[number] = path + # And against the base branch, which is the check that actually catches this. A branch + # cut before another ADR merged does not contain it, so the loop above sees one file + # per number and passes, and the duplicate only exists once the two are merged + # together. That has happened here: a branch claimed a number main had already used and + # its governance run was green the whole time. + if base: + for path in sorted(changed_adr_paths): + if not path.exists() or file_at(base, str(path)) is not None: + # Not added by this PR: either gone, or already on the base under this + # exact name, in which case it is the same ADR rather than a clash. + continue + number = path.name.split("-", 2)[1] if "-" in path.name else path.name + for taken in base_adr_names(base): + if taken == path.name: + continue + taken_number = taken.split("-", 2)[1] if "-" in taken else taken + if taken_number == number: + errors.append( + f"{path}: ADR number {number} is already used on {base} by " + f"docs/adr/{taken}. Pick the next free number; merging both would " + f"leave two different ADRs wearing one number." + ) + for path in files_to_validate: if not FILENAME_RE.match(path.name): errors.append(f"{path}: filename must match ADR-NNNN-short-title.md") diff --git a/src/config.rs b/src/config.rs index 2319f96b..790bdf1c 100644 --- a/src/config.rs +++ b/src/config.rs @@ -409,7 +409,7 @@ const fn default_staged_rollout_hours() -> u64 { /// Controls how chunks are stored, including: /// - Whether storage is enabled /// - Content verification on read -/// - Database size limits (auto-scales with available disk by default) +/// - How much free disk to leave unused #[derive(Debug, Clone, Serialize, Deserialize)] pub struct StorageConfig { /// Enable chunk storage. @@ -422,14 +422,6 @@ pub struct StorageConfig { #[serde(default = "default_storage_verify_on_read")] pub verify_on_read: bool, - /// Explicit LMDB database size cap in GiB. - /// - /// When set to 0 (default), the map size is computed automatically from - /// available disk space at startup and grows on demand when the operator - /// adds storage. Set a non-zero value to impose a hard cap. - #[serde(default)] - pub db_size_gb: usize, - /// Minimum free disk space (in MiB) to preserve on the storage partition. /// /// Writes are refused when available space drops below this threshold, @@ -443,13 +435,12 @@ impl Default for StorageConfig { Self { enabled: default_storage_enabled(), verify_on_read: default_storage_verify_on_read(), - db_size_gb: 0, disk_reserve_mb: default_disk_reserve_mb(), } } } -/// Default: 500 MiB — matches `DEFAULT_DISK_RESERVE` in `storage::lmdb`. +/// Default: 500 MiB — matches `DEFAULT_DISK_RESERVE` in `storage`. const fn default_disk_reserve_mb() -> u64 { 500 } @@ -598,6 +589,57 @@ fn default_testnet_bootstrap() -> Vec { #[cfg(test)] #[allow(clippy::unwrap_used, clippy::expect_used)] mod tests { + + /// A config file written by the previous release still loads. + /// + /// The settings that drove the migration off the old chunk store are gone from this + /// build, and so is the database size cap, which configured a memory map that no longer + /// exists. Every node on the fleet has a config file on disk carrying them, written by + /// the release that did the migrating. If those keys made the file unparseable, every + /// one of those nodes would fail to start on upgrade, all at once. + /// + /// Nothing declares `deny_unknown_fields`, so serde ignores them. That is the behaviour + /// this depends on, which makes it worth a test rather than an assumption: adding that + /// attribute later would look harmless and would take down the fleet. + #[test] + fn a_config_file_from_the_previous_release_still_loads() { + let previous = r#" +[network] +port = 10000 + +[storage] +enabled = true +verify_on_read = true +db_size_gb = 32 +disk_reserve_mb = 500 + +[storage.migration] +shed_hold_hours = 72 +wave_hours = 24 +copier_throttle_mib_per_sec = 32 +copier_slack_mb = 2048 +retire_delay_hours = 4 + +[payment] +rewards_address = "0x0000000000000000000000000000000000000001" +"#; + let dir = tempfile::TempDir::new().expect("temp dir"); + let path = dir.path().join("config.toml"); + std::fs::write(&path, previous).expect("write the previous release's config"); + + // Through the loader a node actually uses, not a hand-picked table. The whole file + // has to parse, because that is what a node does with it on start. + let parsed = NodeConfig::from_file(&path) + .expect("a config file from the previous release must still load"); + + assert!(parsed.storage.enabled); + assert!(parsed.storage.verify_on_read); + assert_eq!( + parsed.storage.disk_reserve_mb, 500, + "the settings this build still uses must survive the ones it dropped" + ); + } + use super::*; use serial_test::serial; diff --git a/src/devnet.rs b/src/devnet.rs index d9e9de09..ececdbcd 100644 --- a/src/devnet.rs +++ b/src/devnet.rs @@ -11,7 +11,7 @@ use crate::payment::{ QuotingMetricsTracker, }; use crate::replication::config::ReplicationConfig; -use crate::storage::{AntProtocol, ChunkRequestContext, LmdbStorage, LmdbStorageConfig}; +use crate::storage::{AntProtocol, ChunkRequestContext, ChunkStore, ChunkStoreConfig}; use evmlib::Network as EvmNetwork; use evmlib::RewardsAddress; use rand::Rng; @@ -595,14 +595,14 @@ impl Devnet { identity: &NodeIdentity, config: &DevnetConfig, ) -> Result { - let storage_config = LmdbStorageConfig { + let storage_config = ChunkStoreConfig { root_dir: data_dir.to_path_buf(), verify_on_read: true, - ..LmdbStorageConfig::default() + ..ChunkStoreConfig::default() }; - let storage = LmdbStorage::new(storage_config) + let storage = ChunkStore::new(storage_config) .await - .map_err(|e| DevnetError::Core(format!("Failed to create LMDB storage: {e}")))?; + .map_err(|e| DevnetError::Core(format!("Failed to create the chunk store: {e}")))?; let evm_config = EvmVerifierConfig { network: config diff --git a/src/lib.rs b/src/lib.rs index 38cc9096..98819613 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -71,7 +71,7 @@ pub use event::{NodeEvent, NodeEventsChannel}; pub use node::{NodeBuilder, RunningNode}; pub use payment::{PaymentStatus, PaymentVerifier, PaymentVerifierConfig}; pub use replication::{config::ReplicationConfig, ReplicationEngine}; -pub use storage::{AntProtocol, LmdbStorage, LmdbStorageConfig}; +pub use storage::{AntProtocol, ChunkStore, ChunkStoreConfig}; /// Re-exports from `saorsa-core` so downstream crates (e.g. `ant-client`) /// can depend on `ant-node` alone without a direct `saorsa-core` dependency. diff --git a/src/node.rs b/src/node.rs index 65b66b4f..da0f74ec 100644 --- a/src/node.rs +++ b/src/node.rs @@ -13,9 +13,10 @@ use crate::payment::{ EvmVerifierConfig, PaymentVerifier, PaymentVerifierConfig, PriceFloorConfig, QuoteGenerator, }; use crate::replication::config::ReplicationConfig; +use crate::replication::fresh::FreshWriteEvent; use crate::replication::ReplicationEngine; -use crate::storage::lmdb::MIB; -use crate::storage::{AntProtocol, ChunkRequestContext, LmdbStorage, LmdbStorageConfig}; +use crate::storage::MIB; +use crate::storage::{AntProtocol, ChunkRequestContext, ChunkStore, ChunkStoreConfig}; use crate::upgrade::{ upgrade_cache_dir, AutoApplyUpgrader, BinaryCache, ReleaseCache, UpgradeMonitor, UpgradeResult, }; @@ -25,17 +26,25 @@ use saorsa_core::{ IPDiversityConfig as CoreDiversityConfig, MultiAddr, NodeConfig as CoreNodeConfig, P2PEvent, P2PNode, }; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicI32, Ordering}; use std::sync::Arc; use std::time::Instant; +use tokio::sync::mpsc::UnboundedReceiver; use tokio::sync::Semaphore; use tokio::task::JoinHandle; use tokio_util::sync::CancellationToken; +use tokio_util::task::TaskTracker; #[cfg(unix)] use tokio::signal::unix::{signal, SignalKind}; +/// How long shutdown waits for in-flight request handlers to finish. +/// +/// Short, because these are single request/response exchanges and the peer will retry. +/// The point is to stop new legacy reads starting, not to see every last one through. +const PROTOCOL_DRAIN_GRACE: std::time::Duration = std::time::Duration::from_secs(5); + /// Builder for constructing an Ant node. pub struct NodeBuilder { config: NodeConfig, @@ -97,6 +106,26 @@ impl NodeBuilder { // Ensure root directory exists std::fs::create_dir_all(&self.config.root_dir)?; + // As soon as the root is known, and before anything is built on top of it. The + // store's own constructor asks this too, but a node with `storage.enabled = false` + // never builds a store and would walk straight past it, and turning storage off is + // not consent to run beside chunks this build cannot read while the commitment that + // claims them is still live. + // + // Ahead of the P2P node specifically. That binds transports and spawns background + // tasks, so asking afterwards means a bind failure can mask this answer, and a + // caller that does see the refusal has already been charged for a transport it is + // about to throw away. + crate::storage::legacy_artifacts::refuse_if_unmigrated(&self.config.root_dir) + .map_err(|e| Error::Startup(e.to_string()))?; + + // One release-level decision, applied before anything can audit. It was suspended + // for two releases while the fleet moved off the old chunk store, because a node + // that has to give chunks up cannot stop its peers punishing it for that. This + // release restores it, so a peer is penalised again for failing to hold a chunk it + // was supposed to be holding. The commitment-bound audit penalised throughout. + crate::replication::config::apply_close_group_storage_penalty_policy(); + // Create shutdown token let shutdown = CancellationToken::new(); @@ -145,56 +174,20 @@ impl NodeBuilder { protocol.attach_p2p_node(Arc::clone(&p2p_arc)); } - // Initialize replication engine (if storage is enabled) - let replication_engine = if let (Some(ref protocol), Some(fresh_rx)) = - (&ant_protocol, fresh_write_rx) - { - let storage_arc = protocol.storage(); - let payment_verifier_arc = protocol.payment_verifier_arc(); - match ReplicationEngine::new( - repl_config, - Arc::clone(&p2p_arc), - storage_arc, - payment_verifier_arc, - Arc::clone(&identity), - &self.config.root_dir, - fresh_rx, - shutdown.clone(), - ) - .await - { - Ok(engine) => { - // ADR-0004: wire the engine's commitment state as the - // quote generator's commitment source so quotes force - // their price from the live storage commitment. Done - // here because the engine owns the commitment state and - // is built after the protocol. - if let Some(ref protocol) = ant_protocol { - let concrete = Arc::clone(engine.commitment_state()); - let source: Arc = concrete; - protocol.attach_commitment_source(source); - // ADR-0004: share the engine's gossip commitment - // cache with the verifier so the cross-check can - // resolve quote pins against neighbours' commitments. - protocol - .payment_verifier_arc() - .attach_commitment_cache(Arc::clone(engine.last_commitment_by_peer())); - // ADR-0004: give the verifier the monetized-pin sender so - // commitments that back a payment get a deterministic - // first audit from the engine's drainer. - protocol - .payment_verifier_arc() - .attach_monetized_pin_sender(engine.monetized_pin_sender()); - } - Some(engine) - } - Err(e) => { - warn!("Failed to initialize replication engine: {e}"); - None - } + let replication_engine = match (&ant_protocol, fresh_write_rx) { + (Some(protocol), Some(fresh_rx)) => { + Self::build_replication_engine( + protocol, + repl_config, + &p2p_arc, + &identity, + &self.config.root_dir, + fresh_rx, + &shutdown, + ) + .await? } - } else { - None + _ => None, }; let node = RunningNode { @@ -207,12 +200,69 @@ impl NodeBuilder { ant_protocol, replication_engine, protocol_task: None, + protocol_children: TaskTracker::new(), upgrade_exit_code: Arc::new(AtomicI32::new(-1)), }; Ok(node) } + /// Start the replication engine. + /// + /// # Errors + /// + /// Never, currently: an engine that fails to start is logged and the node runs without + /// one, as it always has. The signature keeps its `Result` because the caller's does, + /// and because the migration release did have a case that had to refuse. + async fn build_replication_engine( + protocol: &Arc, + repl_config: ReplicationConfig, + p2p: &Arc, + identity: &Arc, + root_dir: &Path, + fresh_rx: UnboundedReceiver, + shutdown: &CancellationToken, + ) -> Result> { + let engine = match ReplicationEngine::new( + repl_config, + Arc::clone(p2p), + protocol.storage(), + protocol.payment_verifier_arc(), + Arc::clone(identity), + root_dir, + fresh_rx, + shutdown.clone(), + ) + .await + { + Ok(engine) => engine, + Err(e) => { + warn!("Failed to initialize replication engine: {e}"); + return Ok(None); + } + }; + + // ADR-0004: wire the engine's commitment state as the quote generator's + // commitment source so quotes force their price from the live storage + // commitment. Done here because the engine owns the commitment state and is + // built after the protocol. + let concrete = Arc::clone(engine.commitment_state()); + let source: Arc = concrete; + protocol.attach_commitment_source(source); + // ADR-0004: share the engine's gossip commitment cache with the verifier so the + // cross-check can resolve quote pins against neighbours' commitments. + protocol + .payment_verifier_arc() + .attach_commitment_cache(Arc::clone(engine.last_commitment_by_peer())); + // ADR-0004: give the verifier the monetized-pin sender so commitments that back + // a payment get a deterministic first audit from the engine's drainer. + protocol + .payment_verifier_arc() + .attach_monetized_pin_sender(engine.monetized_pin_sender()); + + Ok(Some(engine)) + } + /// Build the saorsa-core `NodeConfig` from our config. fn build_core_config(config: &NodeConfig) -> Result { let local = matches!(config.network_mode, NetworkMode::Development); @@ -381,26 +431,23 @@ impl NodeBuilder { monitor } - /// Build the ANT protocol handler from config. /// - /// Initializes LMDB storage, payment verifier, and quote generator. + /// Initializes the chunk store, payment verifier, and quote generator. /// Wires ML-DSA-65 signing from the node's identity into the quote generator. async fn build_ant_protocol( config: &NodeConfig, identity: &NodeIdentity, close_group_size: usize, ) -> Result { - // Create LMDB storage - let storage_config = LmdbStorageConfig { + let storage_config = ChunkStoreConfig { root_dir: config.root_dir.clone(), verify_on_read: config.storage.verify_on_read, - max_map_size: config.storage.db_size_gb.saturating_mul(1024 * 1024 * 1024), disk_reserve: config.storage.disk_reserve_mb.saturating_mul(MIB), }; - let storage = LmdbStorage::new(storage_config) + let storage = ChunkStore::new(storage_config) .await - .map_err(|e| Error::Startup(format!("Failed to create LMDB storage: {e}")))?; + .map_err(|e| Error::Startup(format!("Failed to create the chunk store: {e}")))?; // Parse rewards address (required — node must know where to receive payments) let rewards_address = match config.payment.rewards_address { @@ -466,6 +513,13 @@ pub struct RunningNode { replication_engine: Option, /// Protocol message routing background task. protocol_task: Option>, + /// The per-message handler tasks the protocol loop spawns. + /// + /// Tracked rather than detached so shutdown can stop accepting work and then wait for + /// what is already in flight. Aborting only the loop leaves its children running, and + /// a chunk read that outlives the loop keeps working against a store the shutdown is + /// about to tear down. + protocol_children: TaskTracker, /// Exit code requested by a successful upgrade (-1 = no upgrade exit pending). upgrade_exit_code: Arc, } @@ -691,20 +745,48 @@ impl RunningNode { info!("Node running, waiting for shutdown signal"); - // Run the main event loop with signal handling + // The main event loop, with signal handling. Everything above this starts + // something; this is where the node waits. self.run_event_loop().await?; + // Protocol routing stops first, loop and children both. The routing loop waits on + // `events.recv()` and has no cancellation branch of its own, and it holds an `Arc` + // on the P2P node that keeps the sender it is waiting on alive, so nothing else + // here will ever wake it. Left running it holds the chunk store and its + // single-process lock open after the node has returned. Aborting the accept loop + // alone is not enough either: the requests already in flight run in their own + // tasks, which is what the drain below is for. + if let Some(handle) = self.protocol_task.take() { + handle.abort(); + // Awaited, not just asked to stop. `abort` schedules cancellation; it does not + // establish that the task is gone, and what matters here is that it has + // dropped its `Arc` on the protocol and with it the store's single-process + // lock before this function returns. The join resolves as cancelled. + let _ = handle.await; + } + // Cancelled first, so anything still queued behind the concurrency permits gives + // up rather than starting fresh storage work, then given a moment to finish what + // is genuinely in flight. + self.shutdown.cancel(); + self.protocol_children.close(); + if tokio::time::timeout(PROTOCOL_DRAIN_GRACE, self.protocol_children.wait()) + .await + .is_err() + { + warn!( + "{} request handler(s) had not finished after {}s; continuing shutdown \ + without them.", + self.protocol_children.len(), + PROTOCOL_DRAIN_GRACE.as_secs() + ); + } + // Shutdown replication engine before P2P so background tasks don't - // use a dead P2P layer, and Arc references are released. + // use a dead P2P layer, and Arc references are released. if let Some(ref mut engine) = self.replication_engine { engine.shutdown().await; } - // Stop protocol routing task - if let Some(handle) = self.protocol_task.take() { - handle.abort(); - } - // Shutdown P2P node info!("Shutting down P2P node..."); if let Err(e) = self.p2p_node.shutdown().await { @@ -777,6 +859,58 @@ impl RunningNode { Ok(()) } + /// Handle one inbound protocol message and send whatever it produced. + async fn answer_one_request( + protocol: &Arc, + p2p: &Arc, + source: &saorsa_core::identity::PeerId, + data: &[u8], + data_type: &str, + response_topic: &str, + received_at: Instant, + ) { + if data_type != "chunk" { + return; + } + let queue_wait = received_at.elapsed(); + let handled = protocol + .try_handle_request_with_context( + data, + Some(ChunkRequestContext::new( + source.to_string(), + received_at, + queue_wait, + )), + ) + .await; + let telemetry = handled.get_telemetry; + match handled.response { + Ok(Some(response)) => { + let send_started = Instant::now(); + let send_result = p2p + .send_message(source, response_topic, response.to_vec(), &[]) + .await; + if let Some(telemetry) = telemetry { + telemetry.finish_send(send_started.elapsed(), send_result.is_ok()); + } + if let Err(e) = send_result { + warn!("Failed to send {data_type} protocol response to {source}: {e}"); + } + } + Ok(None) => { + if let Some(telemetry) = telemetry { + telemetry.finish_without_send("no_response"); + } + } + Err(e) => { + if let Some(telemetry) = telemetry { + telemetry.finish_without_send("encode_error"); + } + warn!("{data_type} protocol handler error: {e}"); + } + } + } + /// Start the protocol message routing background task. /// /// Subscribes to P2P events and routes incoming chunk protocol messages @@ -790,6 +924,8 @@ impl RunningNode { let mut events = self.p2p_node.subscribe_events(); let p2p = Arc::clone(&self.p2p_node); let semaphore = Arc::new(Semaphore::new(64)); + let children = self.protocol_children.clone(); + let stopping = self.shutdown.clone(); self.protocol_task = Some(tokio::spawn(async move { while let Ok(event) = events.recv().await { @@ -812,60 +948,38 @@ impl RunningNode { let protocol = Arc::clone(&protocol); let p2p = Arc::clone(&p2p); let sem = semaphore.clone(); - tokio::spawn(async move { - let Ok(_permit) = sem.acquire().await else { - return; - }; - let queue_wait = received_at.elapsed(); - let handled = match data_type { - "chunk" => { - protocol - .try_handle_request_with_context( - &data, - Some(ChunkRequestContext::new( - source.to_string(), - received_at, - queue_wait, - )), - ) - .await + let stopping = stopping.clone(); + children.spawn(async move { + // A queued handler must not start work once shutdown has + // begun. With 64 permits and a busy node the queue behind them + // can be long, and every one of those would otherwise start + // fresh storage reads while the store beneath is being torn + // down. + let _permit = { + let acquired = tokio::select! { + biased; + () = stopping.cancelled() => return, + p = sem.acquire() => p, + }; + match acquired { + Ok(permit) => permit, + Err(_) => return, } - _ => return, }; - let telemetry = handled.get_telemetry; - match handled.response { - Ok(Some(response)) => { - let send_started = Instant::now(); - let send_result = p2p - .send_message( - &source, - response_topic, - response.to_vec(), - &[], - ) - .await; - if let Some(telemetry) = telemetry { - telemetry.finish_send( - send_started.elapsed(), - send_result.is_ok(), - ); - } - if let Err(e) = send_result { - warn!("Failed to send {data_type} protocol response to {source}: {e}"); - } - } - Ok(None) => { - if let Some(telemetry) = telemetry { - telemetry.finish_without_send("no_response"); - } - } - Err(e) => { - if let Some(telemetry) = telemetry { - telemetry.finish_without_send("encode_error"); - } - warn!("{data_type} protocol handler error: {e}"); - } + // Checked again: the wait for a permit may have been long. + if stopping.is_cancelled() { + return; } + Self::answer_one_request( + &protocol, + &p2p, + &source, + &data, + data_type, + response_topic, + received_at, + ) + .await; }); } } @@ -895,6 +1009,131 @@ fn jittered_interval(base: std::time::Duration) -> std::time::Duration { #[cfg(test)] #[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] mod tests { + use rand::Rng; + use tempfile::TempDir; + + /// The e2e port range, so a test bind never lands on a production or dev instance. + const TEST_PORT_RANGE: std::ops::Range = 20000..60000; + + /// How many times a bind is retried before the failure is treated as real. + const BIND_ATTEMPTS: u32 = 5; + + /// A well-formed address that receives nothing; no chain is contacted in these tests. + const TEST_REWARDS_ADDRESS: &str = "0x0000000000000000000000000000000000000001"; + + /// A node config that builds without touching a chain or a real network. + fn local_node_config(root: &std::path::Path, port: u16) -> NodeConfig { + NodeConfig { + root_dir: root.to_path_buf(), + port, + ipv4_only: true, + network_mode: NetworkMode::Development, + payment: crate::config::PaymentConfig { + rewards_address: Some(TEST_REWARDS_ADDRESS.to_string()), + ..crate::config::PaymentConfig::default() + }, + ..NodeConfig::default() + } + } + + /// A node builds on a root with nothing left over from the old store. + #[tokio::test] + async fn a_node_builds_on_a_clean_root() { + let dir = TempDir::new().expect("temp dir"); + let root = dir.path().join("node"); + std::fs::create_dir_all(&root).expect("mkdir"); + + let mut built = None; + let mut last_err = String::new(); + for _ in 0..BIND_ATTEMPTS { + let port = rand::thread_rng().gen_range(TEST_PORT_RANGE); + match NodeBuilder::new(local_node_config(&root, port)) + .build() + .await + { + Ok(node) => { + built = Some(node); + break; + } + Err(e) => { + last_err = e.to_string(); + tokio::time::sleep(std::time::Duration::from_millis(250)).await; + } + } + } + let Some(node) = built else { + panic!("could not build a node after {BIND_ATTEMPTS} attempts: {last_err}"); + }; + + node.shutdown.cancel(); + } + + /// A node with chunks in a store this build cannot read does not start, however it is + /// configured. + /// + /// Both ways, because they are different code paths and only one of them was covered. + /// The store's own constructor asks the question, but a node with `storage.enabled = + /// false` never builds a store and so never reaches it. Turning storage off is not + /// consent to run beside chunks that this node's own published commitment still claims + /// and that this build cannot read, so the question is asked before anything is built. + /// + /// Goes through `build()` rather than the check directly. The failure worth catching + /// here is the call site going missing, which is what happened: the check existed and + /// one of the two routes into the node walked straight past it. + #[tokio::test] + async fn a_node_with_an_unmigrated_store_refuses_to_build_however_it_is_configured() { + for storage_enabled in [true, false] { + let dir = TempDir::new().expect("temp dir"); + let root = dir.path().join("node"); + let env = root.join(crate::storage::LEGACY_ENV_DIR); + std::fs::create_dir_all(&env).expect("mkdir"); + std::fs::write(env.join("data.mdb"), b"chunks that were never copied out") + .expect("seed"); + + let port = rand::thread_rng().gen_range(TEST_PORT_RANGE); + let mut config = local_node_config(&root, port); + config.storage.enabled = storage_enabled; + + let err = NodeBuilder::new(config) + .build() + .await + .err() + .unwrap_or_else(|| { + panic!( + "a node with an unmigrated store built with storage.enabled = \ + {storage_enabled}" + ) + }); + let said = err.to_string(); + assert!( + said.contains("chunks.mdb"), + "the refusal must name the directory (storage.enabled = {storage_enabled}): \ + {said}" + ); + } + + // And it answers before the transport is built, not after. Asking afterwards means + // a bind failure masks this answer, and a caller that does see it has already been + // charged for a transport it is about to throw away. Staged with a privileged port, + // which an ordinary user cannot bind, so P2P construction would fail if it were + // reached: the refusal still has to be the one that comes back. + let dir = TempDir::new().expect("temp dir"); + let root = dir.path().join("node"); + let env = root.join(crate::storage::LEGACY_ENV_DIR); + std::fs::create_dir_all(&env).expect("mkdir"); + std::fs::write(env.join("data.mdb"), b"never copied out").expect("seed"); + + let said = NodeBuilder::new(local_node_config(&root, 1)) + .build() + .await + .err() + .map(|e| e.to_string()) + .unwrap_or_default(); + assert!( + said.contains("chunks.mdb"), + "the store answer must come back before the transport is built, got: {said}" + ); + } use super::*; use crate::config::NODES_SUBDIR; diff --git a/src/payment/metrics.rs b/src/payment/metrics.rs index b59c19f5..fe36002b 100644 --- a/src/payment/metrics.rs +++ b/src/payment/metrics.rs @@ -37,7 +37,7 @@ impl QuotingMetricsTracker { /// /// This is the deletion-aware path and the SINGLE source of truth for the /// priced record count: the handler calls it at quote time with the live - /// LMDB entry count (`current_chunks()`), so any record removed from + /// live chunk count (`current_chunks()`), so any record removed from /// storage — by delete, prune, or otherwise — is reflected on the next /// quote with no per-delete bookkeeping to keep in sync. `record_store` /// remains only an optimistic between-quote hint; the resync overwrites it. diff --git a/src/payment/verifier.rs b/src/payment/verifier.rs index fd550c77..23d7f5ae 100644 --- a/src/payment/verifier.rs +++ b/src/payment/verifier.rs @@ -13,7 +13,7 @@ use crate::payment::proof::{ }; use crate::replication::commitment::MAX_COMMITMENT_KEY_COUNT; use crate::replication::config::K_BUCKET_SIZE; -use crate::storage::lmdb::LmdbStorage; +use crate::storage::ChunkStore; use ant_protocol::payment::verify::{verify_quote_content, verify_quote_signature}; use evmlib::common::{Amount, QuoteHash}; use evmlib::contract::payment_vault; @@ -605,7 +605,7 @@ pub struct PaymentVerifier { /// midpoint in the live DHT. `None` in unit tests that don't exercise /// live-DHT checks; production startup MUST call [`attach_p2p_node`]. p2p_node: RwLock>>, - /// LMDB storage handle, attached post-construction. Retained for + /// Chunk store handle, attached post-construction. Retained for /// store-backed verifier checks that need the authoritative on-disk record /// count without depending on a side counter that may drift from /// replication/repair/prune paths. NOTE: the ADR-0006 price floor does NOT @@ -614,7 +614,7 @@ pub struct PaymentVerifier { /// compared unlike counts and false-rejected honest quotes). `None` in unit /// tests that don't exercise store-backed checks; production wires it via /// [`Self::attach_storage`]. - storage: RwLock>>, + storage: RwLock>>, /// Test-only override for the paid-quote issuer K-closest check. /// /// Production code derives closest peers from the attached [`P2PNode`]. @@ -878,7 +878,7 @@ impl PaymentVerifier { self.config.close_group_size } - /// Attach the node's [`LmdbStorage`] handle for store-backed verifier + /// Attach the node's [`ChunkStore`] handle for store-backed verifier /// checks that read the authoritative on-disk record count. /// /// NOTE: the ADR-0006 price floor does NOT depend on this handle — it is @@ -888,9 +888,9 @@ impl PaymentVerifier { /// attached still admits PUTs; this /// attachment only feeds any current/future store-count-backed checks. /// Idempotent: calling twice replaces the handle. - pub fn attach_storage(&self, storage: Arc) { + pub fn attach_storage(&self, storage: Arc) { *self.storage.write() = Some(storage); - debug!("PaymentVerifier: LmdbStorage attached for paid-quote price-floor checks"); + debug!("PaymentVerifier: ChunkStore attached for paid-quote price-floor checks"); } /// Attach the live commitment source for the price floor: the SAME diff --git a/src/replication/admission.rs b/src/replication/admission.rs index cd881625..23669ffa 100644 --- a/src/replication/admission.rs +++ b/src/replication/admission.rs @@ -17,7 +17,7 @@ use saorsa_core::P2PNode; use crate::ant_protocol::XorName; use crate::replication::config::ReplicationConfig; use crate::replication::paid_list::PaidList; -use crate::storage::LmdbStorage; +use crate::storage::ChunkStore; /// Result of admitting a set of hints from a neighbor sync. #[derive(Debug)] @@ -82,7 +82,7 @@ async fn is_relevant( key: &XorName, p2p_node: &Arc, config: &ReplicationConfig, - storage: &Arc, + storage: &Arc, paid_list: &Arc, pending_keys: &HashSet, ) -> bool { @@ -113,7 +113,7 @@ pub async fn admit_hints( paid_hints: &[XorName], p2p_node: &Arc, config: &ReplicationConfig, - storage: &Arc, + storage: &Arc, paid_list: &Arc, pending_keys: &HashSet, ) -> AdmissionResult { @@ -202,7 +202,7 @@ mod tests { // ----------------------------------------------------------------------- // AdmissionResult construction helpers for pure-logic tests // - // The full `admit_hints` function requires a live DHT + LMDB backend. + // The full `admit_hints` function requires a live DHT and chunk store. // For unit tests we directly exercise: // 1. Cross-set precedence logic // 2. Deduplication logic diff --git a/src/replication/audit.rs b/src/replication/audit.rs index 90dfd1b5..867b8368 100644 --- a/src/replication/audit.rs +++ b/src/replication/audit.rs @@ -21,7 +21,7 @@ use crate::replication::protocol::{ use crate::replication::types::{ AuditFailureReason, AuditFailureSummary, FailureEvidence, PeerSyncRecord, RepairProofs, }; -use crate::storage::LmdbStorage; +use crate::storage::ChunkStore; use saorsa_core::identity::PeerId; use saorsa_core::P2PNode; use tokio::sync::RwLock; @@ -34,7 +34,7 @@ use crate::replication::config::REPAIR_HINT_MIN_AGE; #[cfg(test)] use crate::replication::types::{BootstrapClaimObservation, NeighborSyncState}; #[cfg(test)] -use crate::storage::LmdbStorageConfig; +use crate::storage::ChunkStoreConfig; #[cfg(test)] use tempfile::TempDir; @@ -113,7 +113,7 @@ pub(crate) fn responsible_audit_response_timeout( )] pub async fn audit_tick_with_repair_proofs( p2p_node: &Arc, - storage: &Arc, + storage: &Arc, config: &ReplicationConfig, sync_history: &HashMap, repair_proofs: &Arc>, @@ -543,7 +543,7 @@ async fn verify_digests( nonce: &[u8; 32], keys: &[XorName], digests: &[[u8; 32]], - storage: &Arc, + storage: &Arc, p2p_node: &Arc, config: &ReplicationConfig, ) -> AuditTickResult { @@ -759,7 +759,7 @@ async fn handle_audit_timeout( /// attack where a malicious challenger forges digests for a different peer. pub async fn handle_audit_challenge( challenge: &AuditChallenge, - storage: &LmdbStorage, + storage: &ChunkStore, self_peer_id: &PeerId, is_bootstrapping: bool, stored_chunks: usize, @@ -890,16 +890,15 @@ mod tests { ); } - /// Create a test `LmdbStorage` backed by a temp directory. - async fn create_test_storage() -> (LmdbStorage, TempDir) { + /// Create a test `ChunkStore` backed by a temp directory. + async fn create_test_storage() -> (ChunkStore, TempDir) { let temp_dir = TempDir::new().expect("create temp dir"); - let config = LmdbStorageConfig { + let config = ChunkStoreConfig { root_dir: temp_dir.path().to_path_buf(), verify_on_read: false, - max_map_size: 0, disk_reserve: 0, }; - let storage = LmdbStorage::new(config).await.expect("create storage"); + let storage = ChunkStore::new(config).await.expect("create storage"); (storage, temp_dir) } @@ -931,11 +930,11 @@ mod tests { // Store two chunks. let content_a = b"chunk alpha"; - let addr_a = LmdbStorage::compute_address(content_a); + let addr_a = ChunkStore::compute_address(content_a); storage.put(&addr_a, content_a).await.expect("put a"); let content_b = b"chunk beta"; - let addr_b = LmdbStorage::compute_address(content_b); + let addr_b = ChunkStore::compute_address(content_b); storage.put(&addr_b, content_b).await.expect("put b"); let nonce = [0xAA; 32]; @@ -1011,7 +1010,7 @@ mod tests { let (storage, _temp) = create_test_storage().await; let content = b"present chunk"; - let addr_present = LmdbStorage::compute_address(content); + let addr_present = ChunkStore::compute_address(content); storage.put(&addr_present, content).await.expect("put"); let addr_absent = [0xDE; 32]; @@ -1199,7 +1198,7 @@ mod tests { let (storage, _temp) = create_test_storage().await; let content = b"stored but bootstrapping"; - let addr = LmdbStorage::compute_address(content); + let addr = ChunkStore::compute_address(content); storage.put(&addr, content).await.expect("put"); let challenge = make_challenge(200, [0xCC; 32], [0xDD; 32], vec![addr]); @@ -1230,11 +1229,11 @@ mod tests { // Store K1 and K2, but NOT K3 let content_k1 = b"key one data"; - let addr_k1 = LmdbStorage::compute_address(content_k1); + let addr_k1 = ChunkStore::compute_address(content_k1); storage.put(&addr_k1, content_k1).await.unwrap(); let content_k2 = b"key two data"; - let addr_k2 = LmdbStorage::compute_address(content_k2); + let addr_k2 = ChunkStore::compute_address(content_k2); storage.put(&addr_k2, content_k2).await.unwrap(); let addr_k3 = [0xFF; 32]; // Not stored @@ -1283,9 +1282,9 @@ mod tests { let c1 = b"chunk alpha"; let c2 = b"chunk beta"; let c3 = b"chunk gamma"; - let a1 = LmdbStorage::compute_address(c1); - let a2 = LmdbStorage::compute_address(c2); - let a3 = LmdbStorage::compute_address(c3); + let a1 = ChunkStore::compute_address(c1); + let a2 = ChunkStore::compute_address(c2); + let a3 = ChunkStore::compute_address(c3); storage.put(&a1, c1).await.unwrap(); storage.put(&a2, c2).await.unwrap(); storage.put(&a3, c3).await.unwrap(); @@ -1337,8 +1336,8 @@ mod tests { // Store K1 and K2 on the challenger (for expected digest computation). let c1 = b"scenario 55 key one"; let c2 = b"scenario 55 key two"; - let k1 = LmdbStorage::compute_address(c1); - let k2 = LmdbStorage::compute_address(c2); + let k1 = ChunkStore::compute_address(c1); + let k2 = ChunkStore::compute_address(c2); storage.put(&k1, c1).await.expect("put k1"); storage.put(&k2, c2).await.expect("put k2"); @@ -1622,7 +1621,7 @@ mod tests { // Store a single chunk let content = b"single chunk"; - let addr = LmdbStorage::compute_address(content); + let addr = ChunkStore::compute_address(content); storage.put(&addr, content).await.unwrap(); // Challenge with 1 stored + 4 absent = 5 keys total @@ -1682,7 +1681,7 @@ mod tests { // Store data so there *would* be work to audit. let content = b"should not be audited during bootstrap"; - let addr = LmdbStorage::compute_address(content); + let addr = ChunkStore::compute_address(content); storage.put(&addr, content).await.expect("put"); let challenge = make_challenge(2900, [0x29; 32], [0x29; 32], vec![addr]); @@ -1773,7 +1772,7 @@ mod tests { let mut addrs = Vec::new(); for i in 0u8..5 { let content = format!("dynamic challenge key {i}"); - let addr = LmdbStorage::compute_address(content.as_bytes()); + let addr = ChunkStore::compute_address(content.as_bytes()); storage.put(&addr, content.as_bytes()).await.expect("put"); addrs.push(addr); } @@ -1830,7 +1829,7 @@ mod tests { // Store data so there is an auditable key. let content = b"bootstrap grace test"; - let addr = LmdbStorage::compute_address(content); + let addr = ChunkStore::compute_address(content); storage.put(&addr, content).await.expect("put"); let challenge = make_challenge(4700, [0x47; 32], [0x47; 32], vec![addr]); @@ -1894,9 +1893,9 @@ mod tests { let c1 = b"scenario 53 key one"; let c2 = b"scenario 53 key two"; let c3 = b"scenario 53 key three"; - let k1 = LmdbStorage::compute_address(c1); - let k2 = LmdbStorage::compute_address(c2); - let k3 = LmdbStorage::compute_address(c3); + let k1 = ChunkStore::compute_address(c1); + let k2 = ChunkStore::compute_address(c2); + let k3 = ChunkStore::compute_address(c3); storage.put(&k1, c1).await.expect("put k1"); storage.put(&k2, c2).await.expect("put k2"); storage.put(&k3, c3).await.expect("put k3"); diff --git a/src/replication/audit_metrics.rs b/src/replication/audit_metrics.rs index 5646a889..e1f2cf85 100644 --- a/src/replication/audit_metrics.rs +++ b/src/replication/audit_metrics.rs @@ -407,9 +407,12 @@ static DIGEST_DISPATCH_LATENCY_COUNT: AtomicU64 = AtomicU64::new(0); static DIGEST_DISPATCH_LATENCY_TOTAL_MS: AtomicU64 = AtomicU64::new(0); static DIGEST_DISPATCH_LATENCY_MAX_MS: AtomicU64 = AtomicU64::new(0); -#[cfg(feature = "logging")] impl AuditType { /// Stable structured-log label. + /// + /// Not gated on the `logging` feature: it is passed as an ordinary argument to the + /// penalty helper, which evaluates its arguments whether or not the log macro that + /// consumes them compiles to anything. #[must_use] pub const fn as_str(self) -> &'static str { match self { diff --git a/src/replication/commitment_state.rs b/src/replication/commitment_state.rs index 8c7b2840..9daff439 100644 --- a/src/replication/commitment_state.rs +++ b/src/replication/commitment_state.rs @@ -24,9 +24,11 @@ //! its persisted key set — so an honest restarted node can answer every pin that //! is still inside its answerability window, and an unanswerable pin is provable //! misbehaviour rather than an honest crash-restart. Trees are otherwise rebuilt -//! from `LmdbStorage` at the next rotation tick. Memory cost is bounded by +//! from `ChunkStore` at the next rotation tick. Memory cost is bounded by //! `2 × (key_count × ~64 bytes + signature_size)` — for 10k keys, ~1.3 MB. +use saorsa_core::identity::PeerId; +use std::collections::HashSet; use std::sync::Arc; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; @@ -441,6 +443,18 @@ struct Inner { /// the answerability TTL, not a fixed count). A commitment is retained iff it /// is the live current one or its hash appears here with an unexpired stamp. recently_gossiped: Vec, + /// Peers that have demonstrably received the CURRENT commitment root. + /// + /// Distinct from `recently_gossiped`, which records that a root was put on the wire. + /// This records that a specific peer's node answered afterwards, so the request + /// carrying the root arrived. The storage migration needs that stronger statement: a + /// node must not start giving chunks up until its close group has actually seen the + /// reduced commitment, or those peers keep auditing it against the set it used to + /// hold. + current_recipients: HashSet, + /// The root `current_recipients` refers to. A rotation to a different root empties + /// the set, because nobody has seen the new one yet. + current_recipients_hash: Option<[u8; 32]>, } impl Default for ResponderCommitmentState { @@ -460,6 +474,8 @@ impl ResponderCommitmentState { slots: Vec::with_capacity(RETAINED_GOSSIPED_COMMITMENTS + 1), has_current: false, recently_gossiped: Vec::with_capacity(RETAINED_GOSSIPED_COMMITMENTS), + current_recipients: HashSet::new(), + current_recipients_hash: None, }), } } @@ -470,6 +486,12 @@ impl ResponderCommitmentState { pub fn rotate(&self, new_current: BuiltCommitment) { let new_current = Arc::new(new_current); let mut guard = self.inner.write(); + // Nobody has seen the new root yet, so nobody is a recipient of it. Clearing here + // rather than lazily on the next delivery is what makes the invariant true: the + // lazy version credited whichever peer happened to answer next, for a root that + // peer had never been sent. + guard.current_recipients.clear(); + guard.current_recipients_hash = None; guard.slots.insert(0, new_current); guard.has_current = true; prune_slots(&mut guard, Instant::now()); @@ -503,6 +525,64 @@ impl ResponderCommitmentState { /// `GOSSIP_ANSWERABILITY_TTL` after its last emission, which is what lets /// an out-of-range key age out even when the no-op guard freezes the /// committed key set. + /// Record that `peer` demonstrably received the commitment root `delivered`. + /// + /// Called when a peer answers a neighbour sync that carried that root, which is proof + /// of arrival rather than proof of emission. Ignored if the node has rotated since, + /// because the peer then saw a root that is no longer the one being attested. + pub fn note_commitment_delivered(&self, peer: PeerId, delivered: [u8; 32]) { + let mut guard = self.inner.write(); + if !guard.has_current { + return; + } + let Some(hash) = guard.slots.first().map(|c| c.cached_hash) else { + return; + }; + // The caller names the root it actually put on the wire. A rotation between the + // send and the reply means this peer saw the previous root, and crediting it to + // the current one would attest to something that did not happen. + if delivered != hash { + return; + } + if guard.current_recipients_hash != Some(hash) { + guard.current_recipients.clear(); + guard.current_recipients_hash = Some(hash); + } + guard.current_recipients.insert(peer); + } + + /// How many distinct peers have received the current commitment root. + /// + /// Zero once the root changes, because a rotation is a new claim that nobody has + /// seen yet. + #[must_use] + pub fn current_delivered_peer_count(&self) -> usize { + self.current_delivered_peers().len() + } + + /// Which peers have received the current commitment root. + /// + /// The caller intersects this with whoever is in the close group *now*. A peer that + /// has since left knowing the root is no evidence about the group that will audit + /// this node, and counting it would let a node give chunks up while its actual + /// neighbours still hold it to the larger key set. + #[must_use] + pub fn current_delivered_peers(&self) -> HashSet { + let guard = self.inner.read(); + if !guard.has_current { + return HashSet::new(); + } + let Some(hash) = guard.slots.first().map(|c| c.cached_hash) else { + return HashSet::new(); + }; + if guard.current_recipients_hash == Some(hash) { + guard.current_recipients.clone() + } else { + HashSet::new() + } + } + + /// Stamp `hash` as emitted on the wire, refreshing its answerability window. pub fn mark_gossiped(&self, hash: [u8; 32]) { let now = Instant::now(); let mut guard = self.inner.write(); @@ -875,6 +955,14 @@ mod tests { k } + fn peer(byte: u8) -> PeerId { + let mut bytes = [0u8; 32]; + if let Some(slot) = bytes.first_mut() { + *slot = byte; + } + PeerId::from_bytes(bytes) + } + fn bh(byte: u8) -> [u8; 32] { [byte ^ 0x5A; 32] } @@ -1259,6 +1347,54 @@ mod tests { /// Build a `BuiltCommitment` over the given keys for use in raw `prune_slots` /// tests (each key's `bytes_hash` is `bh(k[0])`). + #[test] + fn commitment_delivery_counts_per_root_and_a_rotation_resets_it() { + let state = ResponderCommitmentState::default(); + + // Nothing advertised, so nobody can have received anything. + state.note_commitment_delivered(peer(1), [0u8; 32]); + assert_eq!(state.current_delivered_peer_count(), 0); + + let first = built(&[1, 2, 3]); + let h_first = first.hash(); + state.rotate(first); + assert_eq!(state.current_delivered_peer_count(), 0); + + state.note_commitment_delivered(peer(1), h_first); + state.note_commitment_delivered(peer(2), h_first); + // The same peer twice is still one peer. + state.note_commitment_delivered(peer(2), h_first); + assert_eq!(state.current_delivered_peer_count(), 2); + + // A different key set is a different claim, and nobody has seen it yet. This is + // what stops a node treating "they knew my old commitment" as "they know my new + // smaller one", which is exactly the confusion the storage migration must avoid. + let second = built(&[1, 2]); + let h_second = second.hash(); + state.rotate(second); + assert_eq!( + state.current_delivered_peer_count(), + 0, + "a rotation must empty the set, not wait to be told" + ); + + // A reply to a sync that carried the OLD root arrives after the rotation. It is + // proof that peer saw the old root, and no evidence at all about the new one. + state.note_commitment_delivered(peer(3), h_first); + assert_eq!( + state.current_delivered_peer_count(), + 0, + "a late reply must not be credited to a root its peer never saw" + ); + + state.note_commitment_delivered(peer(1), h_second); + assert_eq!(state.current_delivered_peer_count(), 1); + + // Retiring the current root means there is nothing being advertised to know. + state.retire_current(); + assert_eq!(state.current_delivered_peer_count(), 0); + } + fn built(keys: &[u8]) -> BuiltCommitment { let (pk, sk) = keypair(); let entries: Vec<_> = keys.iter().map(|&b| (key(b), bh(b))).collect(); @@ -1288,6 +1424,8 @@ mod tests { let base = Instant::now(); let now = base + GOSSIP_ANSWERABILITY_TTL + Duration::from_secs(1); let mut inner = Inner { + current_recipients: HashSet::new(), + current_recipients_hash: None, slots: vec![Arc::clone(&c_current), Arc::clone(&c_stale)], has_current: true, recently_gossiped: vec![ @@ -1337,6 +1475,8 @@ mod tests { let base = Instant::now(); let now = base + GOSSIP_ANSWERABILITY_TTL / 2; let mut inner = Inner { + current_recipients: HashSet::new(), + current_recipients_hash: None, slots: vec![Arc::clone(&c_current), Arc::clone(&c_prev)], has_current: true, recently_gossiped: vec![ @@ -1405,6 +1545,8 @@ mod tests { let base = Instant::now(); let now = base + GOSSIP_ANSWERABILITY_TTL + Duration::from_secs(1); let mut inner = Inner { + current_recipients: HashSet::new(), + current_recipients_hash: None, slots: vec![Arc::clone(&c1)], has_current: false, // already retired recently_gossiped: vec![GossipedAt { @@ -1435,6 +1577,8 @@ mod tests { let base = Instant::now(); let now = base + GOSSIP_ANSWERABILITY_TTL / 2; let mut inner = Inner { + current_recipients: HashSet::new(), + current_recipients_hash: None, slots: vec![Arc::clone(&c1)], has_current: false, // retired recently_gossiped: vec![GossipedAt { diff --git a/src/replication/config.rs b/src/replication/config.rs index 66c8e0bd..2fe085c6 100644 --- a/src/replication/config.rs +++ b/src/replication/config.rs @@ -15,6 +15,11 @@ use std::time::Duration; use rand::Rng; use crate::ant_protocol::CLOSE_GROUP_SIZE; +use crate::logging::{debug, info, warn}; +use saorsa_core::identity::PeerId; +use saorsa_core::{P2PNode, TrustEvent}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; // --------------------------------------------------------------------------- // Static constants (compile-time reference profile) @@ -225,7 +230,7 @@ pub const AUDIT_RESPONDER_TOP_ORIGINS: usize = 10; /// round-1 proofs from starving the light audits, and bounds concurrent /// multi-gigabyte hashing to this many at once. Two allows overlap without /// admitting many simultaneous full-subtree hashes; there is little benefit in -/// more concurrent large LMDB scans against one disk. +/// more concurrent large store scans against one disk. pub const MAX_CONCURRENT_SUBTREE_ROUND1: usize = 2; /// Per-peer concurrency cap for the heavy subtree-audit round 1. One in-flight @@ -288,7 +293,7 @@ pub const SUBTREE_ROUND1_WORK_BURST_BYTES: i64 = 8 * 1024 * 1024 * 1024; /// Floor charged against the round-1 work budget per leaf attempted, in bytes. /// /// The budget counts content bytes, which is the right unit for the hashing but -/// misses what a leaf costs before its size is known: an LMDB point lookup with +/// misses what a leaf costs before its size is known: a point lookup with /// its retries, and a `spawn_blocking` dispatch and join. Nothing bounds a /// chunk from below, so a commitment made of a million tiny records would run a /// full subtree of reads and task round-trips per audit while charging almost @@ -606,7 +611,7 @@ pub const MAX_VERIFICATION_KEYS_PER_CYCLE: usize = 8_192; /// /// Senders aggregate all keys for a peer into one request. Matching this limit /// to the cycle bound lets an honest round use one request per peer while still -/// bounding the LMDB work performed on the responder's serial replication +/// bounding the storage work performed on the responder's serial replication /// message path. Oversized requests are rejected as an empty, wire-compatible /// verification response. pub const MAX_INCOMING_VERIFICATION_KEYS: usize = MAX_VERIFICATION_KEYS_PER_CYCLE; @@ -668,6 +673,153 @@ pub(crate) const CAPACITY_BLOCKED_RETRY: Duration = /// Trust event weight for confirmed audit failures. pub const AUDIT_FAILURE_TRUST_WEIGHT: f64 = 5.0; +/// Whether this build penalises a peer for not holding a chunk it was supposed to hold. +/// +/// **`false` again.** It was `true` for two releases while the fleet moved off the legacy +/// LMDB chunk store, because a node that has to give up chunks cannot stop its peers +/// penalising it for that, so the peers had to stop first. The fleet is on the file store +/// now, so the accusation means what it used to mean and is enforced again. +/// +/// Deliberately narrow. It covers exactly one accusation: "you did not have a chunk you +/// were supposed to be holding". It does **not** cover the commitment-bound subtree audit, +/// where a peer published a signed claim to hold specific keys and could not answer for +/// them. That contract stays enforced in every release. +/// +/// The reason it has to exist at all is that the penalty is the *auditor's* decision. A +/// node that has to give up chunks, because it cannot fit them while it moves them out of +/// a store that never returns disk, cannot stop its peers penalising it for that. So the +/// peers stop first, one release ahead, and the node moves in the next one. +/// +/// A build constant rather than a config field on purpose: a node writes its effective +/// configuration back to disk, so shipping this as an ordinary setting would bake this +/// release's value into every operator's file and the next release would change nothing. +pub const RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY: bool = false; + +/// Environment override for [`RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY`], for a canary. +/// +/// Kept after the flip rather than removed with the rest of the bridge. It is the cheapest +/// lever there is if restoring the penalty turns out to have been early, and the moment it +/// is most likely to be needed is the release that restores it. It suspends only the +/// penalties this node hands out, so an emergency suspension has to go to the fleet, not to +/// the node being penalised. +pub const SUSPEND_CLOSE_GROUP_STORAGE_PENALTY_ENV: &str = "ANT_SUSPEND_UNHELD_CHUNK_PENALTY"; + +/// The live switch. +/// +/// Initialised **from the release constant**, not to `false`. That matters: a code path +/// that never applies the policy then behaves like this release rather than the previous +/// one. Defaulting the other way meant any constructor that skipped the startup call would +/// keep penalising nodes for the very thing this release exists to stop penalising, and +/// `ReplicationEngine::new` is public and is constructed directly by test harnesses. +/// +/// Process-wide rather than threaded through a parameter because it is exactly that: one +/// release-level decision that every affected site has to obey identically, and those +/// sites are spread across call graphs that share no configuration object. +static CLOSE_GROUP_STORAGE_PENALTY_SUSPENDED: AtomicBool = + AtomicBool::new(RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY); + +/// Apply this release's decision. Called once, before anything can audit. +pub fn apply_close_group_storage_penalty_policy() { + let Ok(raw) = std::env::var(SUSPEND_CLOSE_GROUP_STORAGE_PENALTY_ENV) else { + apply_and_announce(RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY); + return; + }; + let suspended = match raw.trim().to_ascii_lowercase().as_str() { + "1" | "true" | "yes" | "on" => true, + "0" | "false" | "no" | "off" => false, + other => { + warn!( + "{SUSPEND_CLOSE_GROUP_STORAGE_PENALTY_ENV}={other} is not a boolean; \ + using the build default {RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY}" + ); + RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY + } + }; + apply_and_announce(suspended); +} + +/// Set the switch and say so, once, where an operator will see it. +/// +/// Both states are logged. An operator reading "penalties are suspended" and an operator +/// reading nothing at all cannot tell the second from a missing log line, and the state +/// that most needs to be visible is the one that disagrees with what the release intended. +fn apply_and_announce(suspended: bool) { + set_close_group_storage_penalty_suspended(suspended); + if suspended != RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY { + warn!( + close_group_storage_penalty_suspended = suspended, + "{SUSPEND_CLOSE_GROUP_STORAGE_PENALTY_ENV} overrides this build: the penalty \ + for not holding a close-group chunk is {}, where the release intends {}. \ + Clear that variable unless this node is a deliberate canary.", + if suspended { "SUSPENDED" } else { "APPLIED" }, + if RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY { + "SUSPENDED" + } else { + "APPLIED" + } + ); + } + if suspended { + info!( + close_group_storage_penalty_suspended = true, + "This release does NOT penalise a peer for failing to hold a close-group \ + chunk. Commitment-bound audits still penalise. Audits run and record \ + throughout." + ); + } else { + info!( + close_group_storage_penalty_suspended = false, + "This release penalises a peer for failing to hold a close-group chunk." + ); + } +} + +/// Set whether failing to hold a close-group chunk penalises. +/// +/// Startup applies the release policy through this. Tests that mean to exercise the +/// penalty itself set it explicitly, so what they assert is not an accident of whichever +/// release they happen to be compiled against. +pub fn set_close_group_storage_penalty_suspended(suspended: bool) { + CLOSE_GROUP_STORAGE_PENALTY_SUSPENDED.store(suspended, Ordering::Relaxed); +} + +/// Whether failing to hold a close-group chunk currently penalises. +#[must_use] +pub fn close_group_storage_penalty_suspended() -> bool { + CLOSE_GROUP_STORAGE_PENALTY_SUSPENDED.load(Ordering::Relaxed) +} + +/// Penalise `peer` at `weight` for not holding a chunk it was supposed to be holding, +/// unless this release withholds that particular penalty. +/// +/// Covers the responsible-chunk audit, the fresh-replication possession check, the prune +/// audit, and the fetch paths where a peer that answered `Present` could not then serve +/// the bytes. A node short of the disk to hold its chunks produces every one of those, so +/// leaving any of them out would stop some of its accusers and not others. +/// +/// Only the penalty is withheld. The caller has already logged the failure with its type, +/// class and key, and that record is what tells us when it is safe to switch the penalty +/// back on. +pub async fn penalise_unheld_close_group_chunk( + p2p_node: &Arc, + peer: &PeerId, + audit_type: &str, + weight: f64, +) { + if close_group_storage_penalty_suspended() { + debug!( + audit_type, + peer = %peer, + "Recorded but not penalised: this release withholds the penalty for not \ + holding a close-group chunk. Commitment-bound audits still penalise." + ); + return; + } + p2p_node + .report_trust_event(peer, TrustEvent::ApplicationFailure(weight)) + .await; +} + /// Probability of launching a subtree audit when a peer's *changed* commitment /// is ingested via gossip (ADR-0002). Keeps audits occasional surprise exams. pub const AUDIT_ON_GOSSIP_PROBABILITY: f64 = 0.2; @@ -1258,6 +1410,7 @@ fn random_duration_in_range(min: Duration, max: Duration) -> Duration { #[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] mod tests { use super::*; + use serial_test::serial; #[test] fn defaults_pass_validation() { @@ -1290,6 +1443,55 @@ mod tests { assert!((AUDIT_FAILURE_TRUST_WEIGHT - 5.0).abs() <= f64::EPSILON); } + /// One test rather than several, because the switch is process-wide: separate tests + /// would race each other under the default parallel runner. + #[test] + #[serial] + fn this_release_penalises_a_peer_for_not_holding_a_close_group_chunk() { + // The one assertion that names the value on purpose. The suspension existed for + // two releases so the fleet could move off a store that never returned disk, and + // leaving it on after that is a network that has quietly stopped enforcing the + // thing it suspended: nodes could drop close-group chunks and nobody would say so. + // + // A switch nobody notices is the failure this guards. Flipping it back is a + // legitimate emergency lever, and it should cost a deliberate edit to a test that + // says why, not a one-character change nothing reports. + // Asked of the live switch after applying this release's policy, rather than of + // the constant. Clippy rejects an assertion on a constant, and going through the + // switch is the better question anyway: what a node actually does. + apply_and_announce(RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY); + assert!( + !close_group_storage_penalty_suspended(), + "this release restores the penalty; suspending it again needs a reason" + ); + } + + #[test] + #[serial] + fn the_unheld_chunk_penalty_switch_follows_the_release_it_is_compiled_into() { + // A build that never applies the policy still behaves like THIS release, not the + // previous one. `ReplicationEngine::new` is public and is constructed directly by + // test harnesses, so defaulting the other way would leave those engines penalising + // exactly what the release exists to stop penalising. + assert_eq!( + close_group_storage_penalty_suspended(), + RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY + ); + + set_close_group_storage_penalty_suspended(true); + assert!(close_group_storage_penalty_suspended()); + set_close_group_storage_penalty_suspended(false); + assert!(!close_group_storage_penalty_suspended()); + + // And applying the release policy lands on whatever this build ships, without + // asserting the constant itself, which the follow-up release flips on purpose. + apply_and_announce(RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY); + assert_eq!( + close_group_storage_penalty_suspended(), + RELEASE_SUSPEND_CLOSE_GROUP_STORAGE_PENALTY + ); + } + #[test] fn core_replication_id_stays_v2_and_subtree_rides_its_own_id() { // Core replication, including all digest audit lanes, stays on v2. diff --git a/src/replication/mod.rs b/src/replication/mod.rs index 0b25e38c..8cf64664 100644 --- a/src/replication/mod.rs +++ b/src/replication/mod.rs @@ -97,7 +97,7 @@ use crate::replication::types::{ NeighborSyncState, PeerSyncRecord, PresenceEvidence, RepairProofs, VerificationEntry, VerificationState, }; -use crate::storage::{CapacityVerdict, LmdbStorage}; +use crate::storage::{CapacityVerdict, ChunkStore}; use saorsa_core::identity::{NodeIdentity, PeerId}; use saorsa_core::{DhtNetworkEvent, P2PEvent, P2PNode, TrustEvent}; use saorsa_pqc::api::sig::{MlDsaSecretKey, MlDsaVariant}; @@ -992,7 +992,7 @@ const INBOUND_REPLICATION_SERIAL_QUEUE_CAPACITY: usize = 64; /// Maximum fresh-replication offers processed concurrently, away from the /// serial non-audit loop. /// -/// Fresh offers can perform an on-chain payment verification and a 4 MiB LMDB +/// Fresh offers can perform an on-chain payment verification and a 4 MiB /// write. Four workers keep that latency off the responder dispatch path while /// keeping concurrent EVM/storage pressure small and predictable. const FRESH_OFFER_WORKER_LIMIT: usize = 4; @@ -1126,7 +1126,7 @@ const FETCH_RESPONDER_MAX_OUTSTANDING_PER_PEER: u32 = 2; /// Maximum verification batches served concurrently. /// -/// LMDB point lookups are fast, but a batch can contain 8,192 of them. Two +/// Point lookups are fast, but a batch can contain 8,192 of them. Two /// workers isolate that synchronous work from message dispatch without turning /// large batches into an I/O fan-out throughput contest. const VERIFICATION_RESPONDER_WORKER_LIMIT: usize = 2; @@ -1442,7 +1442,7 @@ impl Drop for FreshOfferEntryGuard { struct VerificationCycleContext<'a> { p2p_node: &'a Arc, paid_list: &'a Arc, - storage: &'a Arc, + storage: &'a Arc, queues: &'a Arc>, config: &'a ReplicationConfig, bootstrap_state: &'a Arc>, @@ -1484,13 +1484,13 @@ const BOOTSTRAP_DRAIN_CHECK_SECS: u64 = 5; /// observe the cancellation token and terminate before aborting it. /// /// Detached tasks are drained without a timeout because storage-capable work -/// may be awaiting a `spawn_blocking` LMDB operation, which continues running +/// may be awaiting a `spawn_blocking` storage operation, which continues running /// if its async waiter is dropped. const SHUTDOWN_TASK_DRAIN_TIMEOUT: Duration = Duration::from_secs(10); /// How often the responder rebuilds + rotates its storage commitment. /// -/// Each rebuild scans LMDB to compute leaf hashes; for ~10k keys this is +/// Each rebuild scans the store to compute leaf hashes; for ~10k keys this is /// sub-100ms (BLAKE3 + tree build). Retention is gossip-anchored, NOT /// rotation-anchored: the responder stays answerable for the current /// commitment plus every root it recently gossiped that is still in-window @@ -1655,7 +1655,7 @@ pub struct ReplicationEngine { /// P2P networking node. p2p_node: Arc, /// Local chunk storage. - storage: Arc, + storage: Arc, /// Persistent paid-for-list. paid_list: Arc, /// Payment verifier for `PoP` validation. @@ -1700,7 +1700,7 @@ pub struct ReplicationEngine { identity: Arc, /// Responder-side commitment state (two-slot atomic rotation). /// - /// Periodically rebuilt from the live LMDB key set; gossiped on + /// Periodically rebuilt from the live key set; gossiped on /// outbound `NeighborSyncRequest`/`Response`; consulted by the /// commitment-bound audit handler. commitment_state: Arc, @@ -1860,7 +1860,7 @@ impl ReplicationEngine { pub async fn new( config: ReplicationConfig, p2p_node: Arc, - storage: Arc, + storage: Arc, payment_verifier: Arc, identity: Arc, root_dir: &Path, @@ -1988,6 +1988,24 @@ impl ReplicationEngine { &self.commitment_state } + /// Neighbour-sync state, for the storage migration's possession challenges. + #[must_use] + pub fn sync_state(&self) -> &Arc> { + &self.sync_state + } + + /// The audit-challenge coordinator, for the storage migration's possession challenges. + #[must_use] + pub fn audit_challenge_coordinator(&self) -> &Arc { + &self.audit_challenge_coordinator + } + + /// Replication settings, for the storage migration's possession challenges. + #[must_use] + pub fn config(&self) -> &Arc { + &self.config + } + /// Get a reference to the auditor's last-commitment-by-peer table. #[must_use] pub fn last_commitment_by_peer(&self) -> &Arc>> { @@ -2262,15 +2280,15 @@ impl ReplicationEngine { /// Cancel all background tasks and wait for them to terminate. /// /// This must be awaited before dropping the engine when the caller needs - /// the `Arc` references held by background tasks to be - /// released (e.g. before reopening the same LMDB environment). + /// the `Arc` references held by background tasks to be + /// released (e.g. before reopening the same store). /// /// When this returns, no engine-spawned task still holds - /// `Arc` or `Arc`, and no LMDB blocking operation - /// (read or write, on either the chunk store or the paid-list + /// `Arc` or `Arc`, and no blocking storage operation + /// (read or write, against either the chunk store or the paid-list LMDB /// environment) is still running. Engine tasks race their work against /// the shutdown token; a dropped future may leave a `spawn_blocking` - /// LMDB transaction running detached, so this method additionally waits + /// operation running detached, so this method additionally waits /// for both storage layers to go quiescent before returning. pub async fn shutdown(&mut self) { self.shutdown.cancel(); @@ -2311,11 +2329,12 @@ impl ReplicationEngine { // All producers have stopped, so close and drain their detached work. // A started storage operation must run to completion: dropping an async // waiter does not cancel `spawn_blocking`, and would let shutdown return - // while an LMDB transaction still owns the environment. + // while a blocking storage operation is still running. // - // Deliberately unbounded: the LMDB contract requires every worker to - // release its `Arc` before the caller may reopen the - // environment, and a timeout here could return with one still held. + // Deliberately unbounded: every worker has to release its + // `Arc` before the caller may reopen the store, whose lock + // admits one process at a time, and a timeout here could return with one + // still held. // What makes that safe is that every detached task is now guaranteed to // finish — the pools above are closed, stale work is shed at dequeue, // and the one genuinely unbounded await (payment verification) races @@ -2324,7 +2343,7 @@ impl ReplicationEngine { self.detached_task_tracker.wait().await; // Every producer is gone, but a select! racing the shutdown token may - // have dropped a future while it awaited an LMDB `spawn_blocking` op + // have dropped a future while it awaited a storage `spawn_blocking` op // (fetch `storage.put`, prune `storage.delete` / // `paid_list.remove_batch`, verification `paid_list.insert`). The // detached blocking closure owns a cloned `Env`; wait for both @@ -2461,12 +2480,12 @@ impl ReplicationEngine { // so those waiters would drain only at the probe timeout // (roughly `queued / per-target-limit` probes deep) while // `detached_task_tracker.wait()` — deliberately unbounded - // for the LMDB contract — held shutdown open. + // for the storage contract — held shutdown open. // // Dropping this future mid-probe is safe and is the same // shape the neighbor-sync round uses: a parked coordinator // acquire releases its counted reference via - // `ReferenceGuard`, and a dropped LMDB `spawn_blocking` is + // `ReferenceGuard`, and a dropped storage `spawn_blocking` is // covered by the storage-quiescence wait in `shutdown`. tokio::select! { () = shutdown.cancelled() => {} @@ -3317,7 +3336,7 @@ impl ReplicationEngine { /// /// Phase 3 of the v12 storage-bound audit. Once per /// [`COMMITMENT_ROTATION_INTERVAL_SECS`], the responder reads the - /// current LMDB key set, builds a Merkle tree (for content-addressed + /// current key set, builds a Merkle tree (for content-addressed /// chunks `bytes_hash == key`, so no chunk re-read is needed), signs /// the root with the node's `MlDsaSecretKey`, and rotates the result /// into `commitment_state`. Old `previous` slot is dropped by the @@ -3603,7 +3622,7 @@ impl ReplicationEngine { in_flight.push(Box::pin(async move { // Tracked so shutdown() still awaits the task if // this awaiter is dropped (e.g. the worker is - // aborted): it holds Arc and must + // aborted): it holds Arc and must // not outlive the engine. let handle = tracker.spawn(async move { // Cancel-aware: abort when the engine shuts down. @@ -4252,7 +4271,7 @@ struct PeerResponderSlot { #[derive(Clone)] struct ReplicationMessageHandlerContext { p2p_node: Arc, - storage: Arc, + storage: Arc, paid_list: Arc, payment_verifier: Arc, queues: Arc>, @@ -4290,7 +4309,7 @@ struct ReplicationMessageHandlerContext { /// The engine's shutdown token, for detached responder work. /// /// Workers on [`Self::detached_task_tracker`] race this around their - /// *network* phase only — never around an LMDB `spawn_blocking` await, + /// *network* phase only — never around a storage `spawn_blocking` await, /// where dropping the awaiter would detach a live transaction. This is /// what lets `shutdown()` keep its unbounded `tracker.wait()` and still /// terminate: the wait stays safe because it is now guaranteed finite. @@ -5467,7 +5486,7 @@ async fn handle_replication_message( /// is guaranteed to end. /// /// Deliberately NOT applied to `storage.put`: that awaits `spawn_blocking`, so -/// dropping its awaiter would detach a live LMDB transaction and break the +/// dropping its awaiter would detach a live storage operation and break the /// very contract the unbounded wait exists to uphold. async fn verify_payment_until_shutdown( payment_verifier: &Arc, @@ -5697,7 +5716,7 @@ async fn refuse_stranded_fresh_offers( /// /// This runs on the serial non-audit message loop, so it must stay cheap: every /// path here is a set insert, a permit try, or a small response send. The offer -/// itself — an on-chain payment verification and a multi-MiB LMDB write — always +/// itself — an on-chain payment verification and a multi-MiB write — always /// runs on a tracked worker task, never inline, because stalling this loop backs /// up the inbound queue and ultimately drops replication messages wholesale. /// @@ -5815,7 +5834,10 @@ async fn dispatch_fresh_offer( responder_class = "fresh_offer", source = %source, key = %hex::encode(key), - "Fresh offer refused at admission — this node will be penalised for the resulting absence: {failure}" + penalty_suspended = config::close_group_storage_penalty_suspended(), + "Fresh offer refused at admission; the resulting absence is recorded \ + against this node, and penalised unless the release withholds it: \ + {failure}" ); // Release the key explicitly rather than on drop, so the next offer // opens a fresh entry rather than queueing behind a handler that was @@ -5839,7 +5861,7 @@ async fn dispatch_fresh_offer( let ctx = ctx.clone(); // Track the worker so `ReplicationEngine::shutdown()` can await it: it holds - // an `Arc` while writing, and the shutdown contract requires + // an `Arc` while writing, and the shutdown contract requires // those references be released before the caller reopens the environment. ctx.detached_task_tracker .clone() @@ -5851,7 +5873,7 @@ async fn dispatch_fresh_offer( /// /// Split out so `dispatch_fresh_offer` stays a readable admission decision. /// A started handler is never cancelled: `storage.put()` awaits -/// `spawn_blocking`, and dropping that awaiter would detach the live LMDB +/// `spawn_blocking`, and dropping that awaiter would detach the live storage /// transaction. Shutdown responsiveness comes from the closed worker semaphore /// and from `handle_fresh_offer` racing the token around payment verification. /// @@ -6487,7 +6509,7 @@ async fn handle_neighbor_sync_request( source: &PeerId, request: &protocol::NeighborSyncRequest, p2p_node: &Arc, - storage: &Arc, + storage: &Arc, paid_list: &Arc, queues: &Arc>, config: &ReplicationConfig, @@ -6678,7 +6700,7 @@ pub fn verification_requests_for_key_from_for_test(requester: &PeerId, key: &Xor async fn handle_verification_request( source: &PeerId, request: &protocol::VerificationRequest, - storage: &Arc, + storage: &Arc, paid_list: &Arc, p2p_node: &Arc, request_id: u64, @@ -6969,25 +6991,95 @@ fn request_is_stale(received_at: Instant, timeout: Duration) -> bool { received_at.elapsed() >= timeout } +/// How a fetch responder's answer is charged against its reputation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum FetchFault { + /// The peer does not hold a chunk it was expected to hold. + /// + /// This was the lane the migration releases withheld, because a node part-way through + /// moving off the old store answered exactly this way about chunks it had legitimately + /// given up. That is over, and it is penalised again. + UnheldChunk, + /// The peer's own storage failed, or served bytes that no longer hash to their + /// address. + /// + /// Never withheld. `FetchResponse::Error` has one producer, and it is the responder's + /// storage read returning an error: an I/O fault, an exhausted descriptor table, or a + /// failed integrity check. A peer that merely does not hold the chunk answers + /// `NotFound` instead, so nothing about the migration produces this. + ResponderFault, +} + +/// Classify a fetch response that did not carry the chunk. +/// +/// `Success` yields `None`. Every other answer is a fault of one kind or the other, and +/// which kind decides whether this release charges for it. +fn fetch_fault_for(response: &protocol::FetchResponse) -> Option { + match response { + protocol::FetchResponse::Success { .. } => None, + protocol::FetchResponse::NotFound { .. } => Some(FetchFault::UnheldChunk), + protocol::FetchResponse::Error { .. } => Some(FetchFault::ResponderFault), + } +} + +/// Charge a fetch fault to the responder. +/// +/// The only place the two kinds are treated differently. An unheld chunk goes through the +/// release switch, which is currently withholding it; a responder fault is charged +/// directly and is not affected by the switch at all. +async fn charge_fetch_fault( + p2p_node: &Arc, + source: &PeerId, + fault: FetchFault, + lane: &'static str, +) { + match fault { + FetchFault::UnheldChunk => { + config::penalise_unheld_close_group_chunk( + p2p_node, + source, + lane, + REPLICATION_TRUST_WEIGHT, + ) + .await; + } + FetchFault::ResponderFault => { + p2p_node + .report_trust_event( + source, + TrustEvent::ApplicationFailure(REPLICATION_TRUST_WEIGHT), + ) + .await; + } + } +} + +/// Turn the responder's storage read into the answer it sends back. +/// +/// The whole distinction the fetch lanes rest on is made here. A key this node does not +/// hold reads as `Ok(None)` and is answered `NotFound`. A read that fails, from an I/O +/// fault, an exhausted descriptor table, or a failed integrity check, is answered `Error`. +/// Nothing about a node giving chunks up produces the second. +fn fetch_response_for(key: XorName, read: Result>>) -> protocol::FetchResponse { + match read { + Ok(Some(data)) => protocol::FetchResponse::Success { key, data }, + Ok(None) => protocol::FetchResponse::NotFound { key }, + Err(e) => protocol::FetchResponse::Error { + key, + reason: format!("{e}"), + }, + } +} + async fn handle_fetch_request( source: &PeerId, request: &protocol::FetchRequest, - storage: &Arc, + storage: &Arc, p2p_node: &Arc, request_id: u64, rr_message_id: Option<&str>, ) -> Result<()> { - let response = match storage.get(&request.key).await { - Ok(Some(data)) => protocol::FetchResponse::Success { - key: request.key, - data, - }, - Ok(None) => protocol::FetchResponse::NotFound { key: request.key }, - Err(e) => protocol::FetchResponse::Error { - key: request.key, - reason: format!("{e}"), - }, - }; + let response = fetch_response_for(request.key, storage.get(&request.key).await); send_replication_response( source, @@ -7014,7 +7106,7 @@ struct AuditResponderCompletion { async fn handle_audit_challenge_msg( source: &PeerId, challenge: &protocol::AuditChallenge, - storage: &Arc, + storage: &Arc, p2p_node: &Arc, is_bootstrapping: bool, reply: ReplyRoute<'_>, @@ -7278,7 +7370,7 @@ async fn record_sent_replica_hints( #[allow(clippy::too_many_arguments, clippy::too_many_lines)] async fn run_neighbor_sync_round( p2p_node: &Arc, - storage: &Arc, + storage: &Arc, paid_list: &Arc, queues: &Arc>, config: &ReplicationConfig, @@ -7380,9 +7472,11 @@ async fn run_neighbor_sync_round( // same value across the batch is fine and reduces RwLock churn). Atomically // snapshot + mark-gossiped so we stay answerable for exactly what we emit // (ADR-0002 retention), with no TOCTOU vs a concurrent retire/rotate. - let my_commitment = commitment_state - .current_for_gossip() - .map(|b| b.commitment().clone()); + let gossiped = commitment_state.current_for_gossip(); + // The hash actually put on the wire, captured with the payload. A rotation later in + // the round must not let a reply be credited to a root the peer never saw. + let gossiped_hash = gossiped.as_ref().map(|b| b.hash()); + let my_commitment = gossiped.map(|b| b.commitment().clone()); let mut hints_by_peer = neighbor_sync::build_sync_hints_for_peers( &batch, @@ -7408,6 +7502,13 @@ async fn run_neighbor_sync_round( .await; if let Some(outcome) = outcome { + // The peer answered, so the request that carried our commitment root arrived. + // That is proof of delivery rather than proof of emission, and the storage + // migration will not let a node give anything up until its close group has + // actually seen the reduced root. + if let Some(hash) = gossiped_hash { + commitment_state.note_commitment_delivered(*peer, hash); + } handle_sync_response( &self_id, peer, @@ -7462,6 +7563,14 @@ async fn run_neighbor_sync_round( .await; if let Some(outcome) = replacement_outcome { + // Same payload, same round trip, same proof: a reply can only come + // back if the request carrying the root reached this peer. Omitting it + // here made the counter under-report on any node whose primary syncs + // often fall through to a replacement, which is exactly the node most + // likely to be short of disk, and stalled its migration indefinitely. + if let Some(hash) = gossiped_hash { + commitment_state.note_commitment_delivered(replacement_peer, hash); + } handle_sync_response( &self_id, &replacement_peer, @@ -7502,7 +7611,7 @@ async fn handle_sync_response( config: &ReplicationConfig, bootstrapping: bool, bootstrap_state: &Arc>, - storage: &Arc, + storage: &Arc, paid_list: &Arc, queues: &Arc>, sync_state: &Arc>, @@ -7698,7 +7807,7 @@ async fn admit_and_queue_hints( paid_hints: &[XorName], p2p_node: &Arc, config: &ReplicationConfig, - storage: &Arc, + storage: &Arc, paid_list: &Arc, queues: &Arc>, ) -> AdmissionOutcome { @@ -7726,7 +7835,7 @@ async fn admit_and_queue_hints( fn queue_admitted_hints( source_peer: &PeerId, admitted: admission::AdmissionResult, - storage: &LmdbStorage, + storage: &ChunkStore, q: &mut ReplicationQueues, ) -> AdmissionOutcome { let mut discovered = HashSet::new(); @@ -8171,7 +8280,7 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { } // Step 5: Update queues with the evaluated outcomes. - let mut bad_singleton_hints: HashMap = HashMap::new(); + let mut bad_singleton_hints: HashMap<(PeerId, SingletonHintFault), usize> = HashMap::new(); let mut q = queues.write().await; for (key, outcome) in evaluated { let replica_hint_sources = q @@ -8232,20 +8341,38 @@ async fn run_verification_cycle(ctx: VerificationCycleContext<'_>) { } drop(q); - for (peer, bad_hint_count) in bad_singleton_hints { + for ((peer, fault), bad_hint_count) in bad_singleton_hints { let reports = bad_hint_count.min(MAX_BAD_HINT_TRUST_REPORTS_PER_PEER_PER_CYCLE); warn!( "Peer {peer} submitted {bad_hint_count} rejected or self-contradicting \ - sole-source replica hints; \ + sole-source replica hints ({fault:?}); \ reporting {reports} bounded trust failure(s)" ); for _ in 0..reports { - p2p_node - .report_trust_event( - &peer, - TrustEvent::ApplicationFailure(REPLICATION_TRUST_WEIGHT), - ) - .await; + match fault { + // A claim about a key that does not exist. Punishable whatever the + // sender's disk is doing. + SingletonHintFault::RejectedByCloseGroup => { + p2p_node + .report_trust_event( + &peer, + TrustEvent::ApplicationFailure(REPLICATION_TRUST_WEIGHT), + ) + .await; + } + // "I advertised it and no longer have it." That is the one statement a + // node short of disk cannot avoid making while it moves its chunks, so + // it goes through the release switch. + SingletonHintFault::DeniedPossession => { + config::penalise_unheld_close_group_chunk( + p2p_node, + &peer, + "replica_hint_denied_possession", + REPLICATION_TRUST_WEIGHT, + ) + .await; + } + } } } } @@ -8297,25 +8424,43 @@ fn add_replica_hint_sources(sources: &mut Vec, replica_hint_sources: &Ha } } +/// Why a sole-source replica hint is punishable. +/// +/// The two cases look alike and are not. A hint the close group rejects outright is a +/// claim about a key that does not exist, which is a bad hint however the sender's disk is +/// doing. A sender that advertised a key and then answers `Absent` for it is making a +/// statement about its own storage, and that is the one thing a node short of disk cannot +/// avoid saying while it moves its chunks out of a store that will not give the space back. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +enum SingletonHintFault { + /// The close group says the key does not exist. + RejectedByCloseGroup, + /// The sender advertised the key and then denied holding it. + DeniedPossession, +} + /// Return the sole replica advertiser when either the close group definitively -/// rejects the key or the advertiser explicitly denies possessing it. +/// rejects the key or the advertiser explicitly denies possessing it, and say which. /// Paid-only advertisements, corroborated replica hints, and inconclusive /// rounds without that direct contradiction are deliberately non-penalizing. fn punishable_singleton_replica_hint_source( replica_hint_sources: &HashSet, outcome: &KeyVerificationOutcome, evidence: &crate::replication::types::KeyVerificationEvidence, -) -> Option { +) -> Option<(PeerId, SingletonHintFault)> { // A paid-only advertiser leaves this set empty, so the sole-source lane is // reserved for peers that actually claimed possession. if replica_hint_sources.len() != 1 { return None; } let source = *replica_hint_sources.iter().next()?; - let rejected_by_close_group = matches!(outcome, KeyVerificationOutcome::QuorumFailed); - let denied_possession = evidence.presence.get(&source) == Some(&PresenceEvidence::Absent); - - (rejected_by_close_group || denied_possession).then_some(source) + if matches!(outcome, KeyVerificationOutcome::QuorumFailed) { + return Some((source, SingletonHintFault::RejectedByCloseGroup)); + } + if evidence.presence.get(&source) == Some(&PresenceEvidence::Absent) { + return Some((source, SingletonHintFault::DeniedPossession)); + } + None } /// Post-verification bootstrap bookkeeping: remove terminal keys from the @@ -8449,7 +8594,7 @@ enum FetchResult { /// queue is deep enough for that window to be real. /// /// This check must also precede the capacity pre-check below, because - /// `LmdbStorage::put` tests `exists` *before* it tests disk space: without + /// `ChunkStore::put` tests `exists` *before* it tests disk space: without /// it, a full node would decline a key it already holds, which `put` would /// have accepted as a duplicate. AlreadyHeld, @@ -8573,7 +8718,7 @@ async fn is_storage_admitted( /// topology churn before the key is ever dequeued. async fn execute_single_fetch( p2p_node: Arc, - storage: Arc, + storage: Arc, config: Arc, key: XorName, source: PeerId, @@ -8592,7 +8737,7 @@ async fn execute_single_fetch( // Possession, then capacity — both before the dial, and in that order. // - // `LmdbStorage::put` tests `exists` before it tests disk space, so a full + // `ChunkStore::put` tests `exists` before it tests disk space, so a full // node still accepts a key it already holds. Checking possession first is // what keeps this pair of gates from declining work `put` would have // taken. @@ -8762,7 +8907,7 @@ async fn execute_single_fetch( if let Err(e) = storage.put(&resp_key, &data).await { // The bytes arrived and passed the content-address // check, so the source did its job; the failure is - // entirely local (disk-full, or an LMDB error). Any + // entirely local (disk-full, or a storage error). Any // valid source must serve identical content, so trying // the next one cannot cure a local error — it only // re-downloads the same chunk into the same store. @@ -8781,43 +8926,33 @@ async fn execute_single_fetch( result: FetchResult::Stored, } } - ReplicationMessageBody::FetchResponse(protocol::FetchResponse::NotFound { - .. - }) => { - // This peer was selected as a fetch source because it - // recently answered `Present` during verification. A - // subsequent NotFound is evidence of a stale/false claim - // or chunk wiping, so penalize lightly and try another - // verified source. - warn!( - "Fetch: verified source {source} returned NotFound for {}", - hex::encode(key) - ); - p2p_node - .report_trust_event( - &source, - TrustEvent::ApplicationFailure(REPLICATION_TRUST_WEIGHT), - ) - .await; - FetchOutcome { - key, - result: FetchResult::SourceFailed, + ReplicationMessageBody::FetchResponse( + ref response @ (protocol::FetchResponse::NotFound { .. } + | protocol::FetchResponse::Error { .. }), + ) => { + // This peer was selected as a fetch source because it recently + // answered `Present` during verification, so either answer is + // evidence of something. Which one decides what it is charged: a peer + // that does not hold the chunk is the lane this release withholds, a + // peer whose own read failed is not. + if let protocol::FetchResponse::Error { reason, .. } = response { + warn!( + "Fetch: peer {source} returned error for {}: {reason}", + hex::encode(key) + ); + } else { + warn!( + "Fetch: verified source {source} returned NotFound for {}", + hex::encode(key) + ); + } + if let Some(fault) = fetch_fault_for(response) { + let lane = match fault { + FetchFault::UnheldChunk => "fetch_not_found", + FetchFault::ResponderFault => "fetch_error", + }; + charge_fetch_fault(&p2p_node, &source, fault, lane).await; } - } - ReplicationMessageBody::FetchResponse(protocol::FetchResponse::Error { - reason, - .. - }) => { - warn!( - "Fetch: peer {source} returned error for {}: {reason}", - hex::encode(key) - ); - p2p_node - .report_trust_event( - &source, - TrustEvent::ApplicationFailure(REPLICATION_TRUST_WEIGHT), - ) - .await; FetchOutcome { key, result: FetchResult::SourceFailed, @@ -8905,6 +9040,11 @@ async fn handle_subtree_failed_audit( let mut provers_guard = recent_provers.write().await; apply_audit_failure_credit_revocation(&mut provers_guard, challenged_peer, reason); } + // Deliberately NOT routed through the release switch. This is the commitment-bound + // subtree audit: the peer published a signed claim to hold these keys and could not + // answer for them. That contract is enforced in every release, including the one that + // withholds the penalty for merely not holding a close-group chunk, because the whole + // migration depends on a node's reduced commitment still meaning something. p2p_node .report_trust_event( challenged_peer, @@ -9097,12 +9237,13 @@ async fn handle_audit_result( } else { debug!("Audit timeout for {challenged_peer}; retaining active bootstrap claim"); } - p2p_node - .report_trust_event( - challenged_peer, - TrustEvent::ApplicationFailure(config::AUDIT_FAILURE_TRUST_WEIGHT), - ) - .await; + config::penalise_unheld_close_group_chunk( + p2p_node, + challenged_peer, + crate::replication::audit_metrics::AuditType::ResponsibleChunk.as_str(), + config::AUDIT_FAILURE_TRUST_WEIGHT, + ) + .await; } } AuditTickResult::BootstrapClaim { peer } => { @@ -9737,7 +9878,7 @@ async fn write_retention_atomic(path: &Path, bytes: Vec) -> bool { } } -/// Read the current LMDB key set, build + sign a fresh +/// Read the current key set, build + sign a fresh /// `StorageCommitment`, and rotate it into `state` as the new `current`. /// The prior `current` is demoted to `previous`; the prior `previous` is /// dropped (per `ResponderCommitmentState::rotate`). @@ -9750,12 +9891,13 @@ async fn write_retention_atomic(path: &Path, bytes: Vec) -> bool { /// rotate. The auditor side handles "no commitment for this peer" by /// falling back to the legacy plain-digest audit path. async fn rebuild_and_rotate_commitment( - storage: &Arc, + storage: &Arc, identity: &Arc, state: &Arc, p2p: &Arc, config: &Arc, ) -> Result<()> { + // Not `all_keys()`. While the node is bridging off the legacy store these are the let stored_keys = storage .all_keys() .await @@ -9903,6 +10045,77 @@ async fn rebuild_and_rotate_commitment( #[cfg(test)] #[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] mod tests { + + /// The two fetch failures mean different things and must be charged differently. + /// + /// `NotFound` is a peer saying it does not hold the chunk, which is what a node + /// part-way through the migration says about chunks it has legitimately given up, so + /// it is the lane this release withholds. `Error` has a single producer, the + /// responder's own storage read failing, and that is never about the migration. + #[test] + fn a_missing_chunk_and_a_failed_read_are_different_faults() { + let key = [7u8; 32]; + assert_eq!( + fetch_fault_for(&protocol::FetchResponse::NotFound { key }), + Some(FetchFault::UnheldChunk) + ); + assert_eq!( + fetch_fault_for(&protocol::FetchResponse::Error { + key, + reason: "read failed".to_string(), + }), + Some(FetchFault::ResponderFault) + ); + assert_eq!( + fetch_fault_for(&protocol::FetchResponse::Success { + key, + data: vec![1, 2, 3], + }), + None + ); + } + + /// The responder's answer says which fault it is, so the mapping from a storage read + /// to a response is what the classification above rests on. + /// + /// A key the peer does not hold reads as `Ok(None)`. A read that fails, whether from + /// an I/O fault or a failed integrity check, reads as `Err`. Nothing turns the first + /// into the second. + #[tokio::test] + async fn a_missing_key_reads_as_a_plain_miss_and_a_failed_read_as_a_fault() { + let dir = tempfile::tempdir().expect("temp dir"); + let storage = crate::storage::ChunkStore::new(crate::storage::ChunkStoreConfig { + root_dir: dir.path().to_path_buf(), + verify_on_read: true, + disk_reserve: 0, + }) + .await + .expect("open store"); + + let absent = [9u8; 32]; + assert!( + matches!(storage.get(&absent).await, Ok(None)), + "a chunk this node does not hold must read as a plain miss, not a fault" + ); + + // And the answer each read produces. A miss is `NotFound`, which is the withheld + // lane; a failed read is `Error`, which is not. + assert!(matches!( + fetch_response_for(absent, Ok(None)), + protocol::FetchResponse::NotFound { .. } + )); + assert!(matches!( + fetch_response_for(absent, Ok(Some(vec![1, 2, 3]))), + protocol::FetchResponse::Success { .. } + )); + assert!(matches!( + fetch_response_for( + absent, + Err(crate::error::Error::Storage("read failed".into())) + ), + protocol::FetchResponse::Error { .. } + )); + } use super::*; use super::{ apply_audit_failure_credit_revocation, audit_failure_clears_bootstrap_claim, @@ -10364,7 +10577,9 @@ mod tests { assert_eq!( punishable_singleton_replica_hint_source(&HashSet::from([source]), &failed, &evidence), - Some(source) + Some((source, SingletonHintFault::RejectedByCloseGroup)), + "a close-group rejection outranks the denial: the key does not exist, which is \ + a bad hint however the sender's own disk is doing" ); assert_eq!( punishable_singleton_replica_hint_source( @@ -10387,7 +10602,7 @@ mod tests { .insert(source, PresenceEvidence::Unresolved); assert_eq!( punishable_singleton_replica_hint_source(&HashSet::from([source]), &failed, &evidence), - Some(source), + Some((source, SingletonHintFault::RejectedByCloseGroup)), "definitive close-group rejection is punishable without direct contradiction" ); assert_eq!( @@ -10409,8 +10624,10 @@ mod tests { }, &evidence, ), - Some(source), - "an explicit denial is punishable regardless of the overall outcome" + Some((source, SingletonHintFault::DeniedPossession)), + "an explicit denial is punishable regardless of the overall outcome, and is \ + classified separately because it is a statement about the sender's own \ + storage rather than about the key" ); } diff --git a/src/replication/neighbor_sync.rs b/src/replication/neighbor_sync.rs index 3ab9cab6..8b4e40bd 100644 --- a/src/replication/neighbor_sync.rs +++ b/src/replication/neighbor_sync.rs @@ -19,7 +19,7 @@ use crate::replication::protocol::{ NeighborSyncRequest, NeighborSyncResponse, ReplicationMessage, ReplicationMessageBody, }; use crate::replication::types::NeighborSyncState; -use crate::storage::LmdbStorage; +use crate::storage::ChunkStore; /// Hint-build duration that is worth surfacing at info level. const HINT_BUILD_SLOW_LOG_MS: u128 = 250; @@ -64,7 +64,7 @@ pub(crate) struct PeerSyncHints { /// this node is allowed to delete them. pub async fn build_replica_hints_for_peer( peer: &PeerId, - storage: &Arc, + storage: &Arc, p2p_node: &Arc, close_group_size: usize, ) -> Vec { @@ -77,7 +77,7 @@ pub async fn build_replica_hints_for_peer( pub(crate) async fn build_replica_hints_for_peer_with_close_groups( peer: &PeerId, - storage: &Arc, + storage: &Arc, p2p_node: &Arc, close_group_size: usize, ) -> Vec { @@ -107,7 +107,7 @@ pub(crate) async fn build_replica_hints_for_peer_with_close_groups( /// storage and one scan over the paid list. pub(crate) async fn build_sync_hints_for_peers( peers: &[PeerId], - storage: &Arc, + storage: &Arc, paid_list: &Arc, p2p_node: &Arc, close_group_size: usize, @@ -330,7 +330,7 @@ fn peer_on_cooldown( pub async fn sync_with_peer( peer: &PeerId, p2p_node: &Arc, - storage: &Arc, + storage: &Arc, paid_list: &Arc, config: &ReplicationConfig, is_bootstrapping: bool, @@ -355,7 +355,7 @@ pub async fn sync_with_peer( pub(crate) async fn sync_with_peer_with_outcome( peer: &PeerId, p2p_node: &Arc, - storage: &Arc, + storage: &Arc, paid_list: &Arc, config: &ReplicationConfig, is_bootstrapping: bool, @@ -485,7 +485,7 @@ pub async fn handle_sync_request( sender: &PeerId, request: &NeighborSyncRequest, p2p_node: &Arc, - storage: &Arc, + storage: &Arc, paid_list: &Arc, config: &ReplicationConfig, is_bootstrapping: bool, @@ -509,7 +509,7 @@ pub(crate) async fn handle_sync_request_with_proofs( sender: &PeerId, _request: &NeighborSyncRequest, p2p_node: &Arc, - storage: &Arc, + storage: &Arc, paid_list: &Arc, config: &ReplicationConfig, is_bootstrapping: bool, diff --git a/src/replication/paid_list.rs b/src/replication/paid_list.rs index f65172c1..62483028 100644 --- a/src/replication/paid_list.rs +++ b/src/replication/paid_list.rs @@ -60,7 +60,7 @@ pub struct PaidList { paid_prune_cursor: RwLock, /// Tracks every paid-list LMDB blocking task. /// - /// Same rationale as `LmdbStorage::blocking_tracker`: a `spawn_blocking` + /// Same rationale as `ChunkStore::blocking_tracker`: a `spawn_blocking` /// closure owns a cloned [`Env`] and keeps running when its async awaiter /// is dropped, so [`Self::wait_idle`] waits on the blocking tasks /// themselves before the environment may be reopened. diff --git a/src/replication/possession.rs b/src/replication/possession.rs index 72c4e969..cc552f2b 100644 --- a/src/replication/possession.rs +++ b/src/replication/possession.rs @@ -41,7 +41,7 @@ use crate::replication::protocol::{ ReplicationMessageBody, ABSENT_KEY_DIGEST, }; use crate::replication::types::{BootstrapClaimObservation, NeighborSyncState}; -use crate::storage::LmdbStorage; +use crate::storage::ChunkStore; use super::REPLICATION_TRUST_WEIGHT; @@ -137,7 +137,7 @@ pub(crate) async fn run_possession_check( key: XorName, peers: Vec, p2p_node: &Arc, - storage: &Arc, + storage: &Arc, config: &ReplicationConfig, sync_state: &Arc>, audit_challenge_coordinator: &Arc, @@ -225,15 +225,16 @@ async fn report_possession_confirmed_failure( peer = %peer, key = %key_hex, trust_weight = AUDIT_FAILURE_TRUST_WEIGHT, - "Possession check: {peer} failed to prove possession for {key_hex} ({}); penalising at audit severity", + "Possession check: {peer} failed to prove possession for {key_hex} ({}); recorded at audit severity", failure_reason.as_str() ); - p2p_node - .report_trust_event( - peer, - TrustEvent::ApplicationFailure(AUDIT_FAILURE_TRUST_WEIGHT), - ) - .await; + crate::replication::config::penalise_unheld_close_group_chunk( + p2p_node, + peer, + AuditType::Possession.as_str(), + AUDIT_FAILURE_TRUST_WEIGHT, + ) + .await; } async fn report_possession_audit_failure( @@ -248,15 +249,16 @@ async fn report_possession_audit_failure( peer = %peer, key = %key_hex, trust_weight = AUDIT_FAILURE_TRUST_WEIGHT, - "Possession check: {peer} {} for {key_hex}; penalising at audit severity", + "Possession check: {peer} {} for {key_hex}; recorded at audit severity", failure_class.as_str() ); - p2p_node - .report_trust_event( - peer, - TrustEvent::ApplicationFailure(AUDIT_FAILURE_TRUST_WEIGHT), - ) - .await; + crate::replication::config::penalise_unheld_close_group_chunk( + p2p_node, + peer, + AuditType::Possession.as_str(), + AUDIT_FAILURE_TRUST_WEIGHT, + ) + .await; } async fn handle_possession_bootstrap_claim( diff --git a/src/replication/pruning.rs b/src/replication/pruning.rs index 10acee66..aa503757 100644 --- a/src/replication/pruning.rs +++ b/src/replication/pruning.rs @@ -73,7 +73,7 @@ use crate::replication::types::{ BootstrapClaimObservation, KeyVerificationEvidence, NeighborSyncState, PaidListEvidence, RepairProofs, }; -use crate::storage::LmdbStorage; +use crate::storage::ChunkStore; // `RepairProofs` remains in the prune-pass context only so records deleted by // pruning also drop their (audit-path) repair-proof entries; it plays no part @@ -136,7 +136,7 @@ pub struct PrunePassContext<'a> { /// Local peer id. pub self_id: &'a PeerId, /// Local record storage. - pub storage: &'a Arc, + pub storage: &'a Arc, /// Persistent paid-list state. pub paid_list: &'a Arc, /// P2P node used for routing lookups and prune-confirmation audits. @@ -341,7 +341,7 @@ struct PruneAuditReportState { #[derive(Clone, Copy)] struct PruneAuditContext<'a> { - storage: &'a Arc, + storage: &'a Arc, p2p_node: &'a Arc, config: &'a ReplicationConfig, sync_state: &'a Arc>, @@ -1168,7 +1168,7 @@ async fn advance_prune_cursor( async fn delete_stored_records( keys_to_delete: &[XorName], - storage: &Arc, + storage: &Arc, paid_list: &Arc, repair_proofs: &Arc>, ) -> usize { @@ -1205,7 +1205,7 @@ async fn delete_stored_records( async fn collect_record_prune_proofs( candidates: &[RecordPruneCandidate], local_stored_key_count: usize, - storage: &Arc, + storage: &Arc, p2p_node: &Arc, config: &ReplicationConfig, sync_state: &Arc>, @@ -1289,7 +1289,7 @@ async fn revalidated_fast_prune_keys( (keys_to_delete, cleared) } -async fn stored_record_still_exists(key: &XorName, storage: &Arc) -> bool { +async fn stored_record_still_exists(key: &XorName, storage: &Arc) -> bool { match storage.get_raw(key).await { Ok(Some(_)) => true, Ok(None) => false, @@ -1501,7 +1501,7 @@ fn confirmed_keys_from_presence( /// from vetoing deletion forever without accepting under-replication. /// Groups of one or two peers require every proof: tolerating a miss there /// would allow deletion on a single attestation. -fn prune_proofs_needed(group_size: usize) -> usize { +pub(crate) fn prune_proofs_needed(group_size: usize) -> usize { if group_size <= 2 { group_size } else { @@ -1513,7 +1513,7 @@ fn prune_proofs_needed(group_size: usize) -> usize { /// /// `proofs_needed == 0` means confirmation is impossible (no targets), not /// trivially met. -fn target_peers_reported_present( +pub(crate) fn target_peers_reported_present( key: &XorName, target_peers: &[PeerId], present_by_key: &HashMap>, @@ -1915,14 +1915,14 @@ async fn local_record_digest( peer: &PeerId, key: &XorName, nonce: &[u8; 32], - storage: &Arc, + storage: &Arc, ) -> Option<[u8; 32]> { local_record_bytes(key, storage) .await .map(|bytes| compute_audit_digest(nonce, peer.as_bytes(), key, &bytes)) } -async fn local_record_bytes(key: &XorName, storage: &Arc) -> Option> { +async fn local_record_bytes(key: &XorName, storage: &Arc) -> Option> { match storage.get_raw(key).await { Ok(Some(bytes)) => Some(bytes), Ok(None) => { @@ -1980,12 +1980,13 @@ async fn report_prune_audit_failure_once( "Prune audit failure: peer={peer}, audit_failure_class={audit_failure_class}, key={}", hex::encode(key) ); - p2p_node - .report_trust_event( - peer, - saorsa_core::TrustEvent::ApplicationFailure(AUDIT_FAILURE_TRUST_WEIGHT), - ) - .await; + crate::replication::config::penalise_unheld_close_group_chunk( + p2p_node, + peer, + AuditType::Prune.as_str(), + AUDIT_FAILURE_TRUST_WEIGHT, + ) + .await; true } diff --git a/src/replication/storage_commitment_audit.rs b/src/replication/storage_commitment_audit.rs index 481272a0..12f86853 100644 --- a/src/replication/storage_commitment_audit.rs +++ b/src/replication/storage_commitment_audit.rs @@ -33,7 +33,7 @@ use crate::replication::subtree::{ select_subtree_path, subtree_plan, verify_subtree_proof, StructureVerdict, SubtreeProof, }; use crate::replication::types::{AuditFailureReason, AuditFailureSummary, FailureEvidence}; -use crate::storage::LmdbStorage; +use crate::storage::ChunkStore; use saorsa_core::identity::PeerId; use saorsa_core::P2PNode; use tokio::sync::RwLock; @@ -79,7 +79,7 @@ const AUDIT_READ_RETRY_BACKOFF: Duration = Duration::from_millis(200); /// an `Err` (transient IO) is. A persistent `Err` is returned so the caller emits /// `RejectKind::Transient` (timeout lane). async fn get_raw_retrying( - storage: &LmdbStorage, + storage: &ChunkStore, key: &XorName, ) -> crate::error::Result>> { let mut attempt = 1u32; @@ -1230,7 +1230,7 @@ fn subtree_failure_summary(reason: &AuditFailureReason) -> AuditFailureSummary { /// grace removed, the auditor treats as a confirmed failure for an in-window pin). pub async fn handle_subtree_challenge( challenge: &SubtreeAuditChallenge, - storage: &LmdbStorage, + storage: &ChunkStore, self_peer_id: &PeerId, is_bootstrapping: bool, commitment_state: Option<&Arc>, @@ -1250,7 +1250,7 @@ pub async fn handle_subtree_challenge( pub struct Round1Work { /// What to send back. pub response: SubtreeAuditResponse, - /// Chunk content read from LMDB and hashed BEFORE this response was + /// Chunk content read from the store and hashed BEFORE this response was /// produced. /// /// Counted on the rejecting paths too, which is the point. A subtree is read @@ -1267,7 +1267,7 @@ pub struct Round1Work { /// it performed so the caller can charge it on every exit path. pub async fn handle_subtree_challenge_measured( challenge: &SubtreeAuditChallenge, - storage: &LmdbStorage, + storage: &ChunkStore, self_peer_id: &PeerId, is_bootstrapping: bool, commitment_state: Option<&Arc>, @@ -1297,7 +1297,7 @@ pub async fn handle_subtree_challenge_measured( #[allow(clippy::too_many_lines)] async fn subtree_challenge_response( challenge: &SubtreeAuditChallenge, - storage: &LmdbStorage, + storage: &ChunkStore, self_peer_id: &PeerId, is_bootstrapping: bool, commitment_state: Option<&Arc>, @@ -1363,7 +1363,7 @@ async fn subtree_challenge_response( let mut leaves = Vec::with_capacity(plan.leaf_keys.len()); for key in &plan.leaf_keys { // Charge the fixed cost of ATTEMPTING a leaf before the read, because - // it is owed whether or not the read succeeds: the LMDB lookup and its + // it is owed whether or not the read succeeds: the lookup and its // retries, and the blocking-task round trip below. Charging only // content bytes left both a failing leaf and a tiny one nearly free, // and nothing bounds a chunk from below, so a commitment of a million @@ -1547,7 +1547,7 @@ fn build_slice_items_for_key( /// an answer against. pub async fn handle_subtree_slice_challenge( challenge: &SubtreeSliceChallenge, - storage: &LmdbStorage, + storage: &ChunkStore, self_peer_id: &PeerId, is_bootstrapping: bool, commitment_state: Option<&Arc>, @@ -1603,7 +1603,7 @@ pub async fn handle_subtree_slice_challenge( }; // Coalesce openings by key, preserving first-seen order and deduplicating - // block indices per key, so each committed chunk is read from LMDB and hashed + // block indices per key, so each committed chunk is read from the store and hashed // at most once even when the auditor opens several of its blocks (the normal // random + final pair, or a forged duplicate). Without this a ten-opening // request could re-read and re-hash the same chunk ten times. @@ -1713,7 +1713,7 @@ enum KeyServe { /// `indices` is already deduplicated by the caller. async fn serve_committed_key_openings( challenge: &SubtreeSliceChallenge, - storage: &LmdbStorage, + storage: &ChunkStore, key: XorName, indices: Vec, ) -> KeyServe { @@ -1758,7 +1758,7 @@ async fn serve_committed_key_openings( } // Persistent transient read error after retries → do NOT brand the peer a // deleter. Reject `Transient`; the auditor routes it to the timeout lane - // so a flaky LMDB read never manufactures a confirmed possession failure + // so a flaky read never manufactures a confirmed possession failure // on an honest holder (which also gains no credit). Err(e) => { warn!( diff --git a/src/storage/chunk_store.rs b/src/storage/chunk_store.rs new file mode 100644 index 00000000..cdaeba22 --- /dev/null +++ b/src/storage/chunk_store.rs @@ -0,0 +1,4036 @@ +//! One immutable file per chunk, content-addressed, with the filesystem as the +//! only authority. +//! +//! ```text +//! {root}/chunks/ store root +//! {root}/chunks/layout.json versioned layout marker +//! {root}/chunks/.lock advisory single-process guard +//! {root}/chunks//<64-hex> xy = the LAST two hex characters of the address +//! {root}/chunks//.tmp.. an in-flight write, in the destination directory +//! ``` +//! +//! # Why the *last* two hex characters +//! +//! A node holds keys for which it is among the [`CLOSE_GROUP_SIZE`] closest, so its +//! holdings share roughly `log2(N / CLOSE_GROUP_SIZE)` leading bits with its own node +//! ID, and that shared prefix grows as the network grows. Sharding on a prefix therefore +//! does not degrade, it collapses: at ~800 nodes a two-hex prefix already resolves to +//! about two distinct directories, and past a million nodes even a four-hex prefix +//! resolves to one. Close-group membership constrains the leading bits and places no +//! constraint at all on the trailing ones, and the address is a BLAKE3 output, so the +//! last byte is uniform by construction at every network size. +//! +//! 256 shards keeps a 24 GiB node at ~23 files per directory and a 1 TiB node at ~977, +//! for 1 MiB of directory inodes. The scheme and depth are recorded in `layout.json` at +//! creation so a future layout can be detected rather than silently misread. +//! +//! # Why lowercase hex names +//! +//! NTFS and default APFS fold case. Under an encoding with both cases (base64url, +//! base58) two distinct 32-byte keys can share one case-folded filename, which is a +//! silent overwrite. Hex has one case-folded form per key, and no hex string can ever +//! spell a reserved Windows device name (`CON`, `NUL`, `AUX`, `COM1`, ...) because none +//! of those letters is in `0-9a-f`. The full 64-character key stays in the filename, so +//! a `find` over the tree recovers the whole store even if the directory layer is lost. +//! +//! [`CLOSE_GROUP_SIZE`]: crate::ant_protocol::CLOSE_GROUP_SIZE + +use crate::ant_protocol::{XorName, MAX_CHUNK_SIZE, XORNAME_LEN}; +use crate::error::{Error, Result}; +use crate::logging::{debug, info, trace, warn}; +use crate::storage::StorageStats; +use fs2::FileExt; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeSet, HashMap, HashSet}; +use std::fs::{File, OpenOptions}; +use std::io::{ErrorKind, Read, Write}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::task::spawn_blocking; +use tokio_util::task::TaskTracker; + +/// Directory under the node root that holds the chunk files. +pub const CHUNKS_DIR_NAME: &str = "chunks"; + +/// Name of the layout marker written once at store creation. +pub const LAYOUT_FILE_NAME: &str = "layout.json"; + +/// Name of the advisory single-process lock file. +const LOCK_FILE_NAME: &str = ".lock"; + +/// Prefix that marks an in-flight write. Never a valid chunk name (chunk names are +/// exactly [`CHUNK_NAME_LEN`] lowercase hex characters, and `.` is not hex). +const TEMP_PREFIX: &str = ".tmp."; + +/// Number of shard directories. One level, `00` through `ff`. +const SHARD_COUNT: usize = 256; + +/// Length of a chunk filename: the full address in lowercase hex. +const CHUNK_NAME_LEN: usize = XORNAME_LEN * 2; + +/// How often to re-query available disk space, in seconds. +/// +/// Matches the LMDB store's cadence so the capacity predicate behaves identically +/// for callers that only ask "is there room at all". +const DISK_CHECK_INTERVAL_SECS: u64 = 5; + +/// Allocation granularity assumed when charging a pending write against free space. +/// +/// Every filesystem we support allocates in units of at least 4 KiB, so a write of +/// `n` bytes consumes at least `ceil(n / 4096) * 4096`. One extra unit covers the +/// directory entry and inode. +const ALLOC_UNIT: u64 = 4096; + +/// How many times a publish retries a transient Windows sharing violation. +const RENAME_RETRY_ATTEMPTS: u32 = 5; + +/// Base backoff between those retries; the wait grows linearly with the attempt. +const RENAME_RETRY_BACKOFF: Duration = Duration::from_millis(20); + +/// Longest absolute path a chunk file may need, checked once at open. +/// +/// Windows caps a non-verbatim path at `MAX_PATH` (260) including the terminating NUL. +/// Rust's standard library transparently switches to the `\\?\` verbatim form for long +/// absolute paths, so this is a warning rather than a hard failure, but an operator who +/// buries the node root ten directories deep should hear about it before the first write +/// fails rather than after. +#[cfg(windows)] +const WINDOWS_PATH_WARN_LEN: usize = 240; + +/// The on-disk layout marker. +/// +/// Written once when the store directory is created and read on every subsequent open. +/// Nothing in this survey of comparable stores (IPFS flatfs, Storj, borgbackup) shipped +/// an in-place re-sharder, and all three paid for it. Recording the scheme costs one +/// small file and is the difference between changing the default later and never being +/// able to. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct StoreLayout { + /// Marker schema version. A store written by a newer schema is refused. + pub schema: u32, + /// How a chunk address maps to a shard directory. + pub scheme: String, + /// How many hex characters of the address name the shard directory. + pub shard_chars: u8, + /// How many directory levels of sharding. + pub depth: u8, + /// How a chunk address maps to a filename. + pub name_encoding: String, +} + +/// Marker schema this build writes and understands. +const LAYOUT_SCHEMA: u32 = 1; +/// Shard scheme this build implements: the trailing hex characters of the address. +const LAYOUT_SCHEME_SUFFIX_HEX: &str = "suffix-hex"; +/// Filename encoding this build implements. +const LAYOUT_NAME_LOWER_HEX: &str = "lower-hex"; + +impl Default for StoreLayout { + fn default() -> Self { + Self { + schema: LAYOUT_SCHEMA, + scheme: LAYOUT_SCHEME_SUFFIX_HEX.to_string(), + shard_chars: 2, + depth: 1, + name_encoding: LAYOUT_NAME_LOWER_HEX.to_string(), + } + } +} + +impl StoreLayout { + /// Return an error unless this build can read a store written with this layout. + fn check_supported(&self) -> Result<()> { + if self.schema > LAYOUT_SCHEMA { + return Err(Error::Storage(format!( + "Chunk store layout schema {} is newer than this build understands ({LAYOUT_SCHEMA}). \ + Refusing to open rather than misread the store.", + self.schema + ))); + } + if self.scheme != LAYOUT_SCHEME_SUFFIX_HEX { + return Err(Error::Storage(format!( + "Chunk store uses shard scheme '{}', this build implements '{LAYOUT_SCHEME_SUFFIX_HEX}'", + self.scheme + ))); + } + if self.shard_chars != 2 || self.depth != 1 { + return Err(Error::Storage(format!( + "Chunk store uses {} shard characters at depth {}, this build implements 2 at depth 1", + self.shard_chars, self.depth + ))); + } + if self.name_encoding != LAYOUT_NAME_LOWER_HEX { + return Err(Error::Storage(format!( + "Chunk store names files with '{}', this build implements '{LAYOUT_NAME_LOWER_HEX}'", + self.name_encoding + ))); + } + Ok(()) + } +} + +/// What the store can say about free space right now. +/// +/// Three answers, because deciding how long to stand down needs the distinction: a full +/// disk is a standing condition worth waiting minutes on, while a failed query may have +/// cleared by the next attempt and must not be treated as one. +/// +/// Lived alongside the LMDB store until that was removed. It was never about LMDB. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CapacityVerdict { + /// Available space is at or above the configured reserve. That is what the query + /// establishes, and possibly from the TTL cache, not a promise the next write succeeds. + Writable, + /// Available space is below the configured reserve. + Full, + /// The query itself failed, so nothing is known about available space. + Unknown, +} + +/// Configuration for [`ChunkStore`]. +#[derive(Debug, Clone)] +pub struct ChunkStoreConfig { + /// Node root directory. The store lives at `{root_dir}/chunks/`. + pub root_dir: PathBuf, + /// Verify `BLAKE3(content) == address` on read. + pub verify_on_read: bool, + /// Free bytes to keep on the storage partition. Writes are refused below this. + pub disk_reserve: u64, +} + +impl Default for ChunkStoreConfig { + fn default() -> Self { + Self { + root_dir: PathBuf::from(".ant/chunks"), + verify_on_read: true, + disk_reserve: crate::storage::DEFAULT_DISK_RESERVE, + } + } +} + +impl ChunkStoreConfig { + /// The shipped defaults with the disk reserve removed, for tests on small volumes. + #[cfg(any(test, feature = "test-utils"))] + #[must_use] + pub fn test_default() -> Self { + Self { + disk_reserve: 0, + ..Self::default() + } + } +} + +/// Outcome of a single write attempt, used to keep the duplicate accounting honest. +enum PutOutcome { + /// The chunk was newly published. + New, + /// The chunk was already on disk. + Duplicate, +} + +/// Snapshot of free space, plus what has been written since it was taken. +#[derive(Debug)] +struct CapacitySnapshot { + /// When `available` was measured. `None` means never. + measured_at: Option, + /// Free bytes reported by the filesystem at `measured_at`. + available: u64, + /// Bytes published since `measured_at`, charged against `available`. + /// + /// Cleared by a fresh measurement, which already accounts for them. + written_since: u64, + /// Bytes reserved by writes that have not landed yet. + /// + /// Deliberately **not** cleared by a measurement: a `statvfs` taken while writes are + /// in flight reports space those writes are about to consume, so forgetting their + /// reservations at that moment would hand the same bytes out twice. That is precisely + /// the over-admission the reservation exists to prevent. + in_flight: u64, +} + +/// Size-aware free-space predicate with a short-lived cache. +/// +/// Free bytes alone stopped being a sufficient answer the moment chunks became files: +/// a caller wants to know whether *this* write fits, not whether the disk is non-empty. +/// The cache keeps the common case at one `statvfs` per interval while staying correct +/// under a burst, because bytes written since the measurement are charged against it. +#[derive(Debug)] +struct CapacityGuard { + /// Directory whose partition is measured. + dir: PathBuf, + /// Free bytes to keep unused. + reserve: u64, + /// The cached measurement. + snapshot: parking_lot::Mutex, +} + +impl CapacitySnapshot { + /// Free bytes, less everything written or promised since the measurement. + fn free_estimate(&self) -> u64 { + self.available + .saturating_sub(self.written_since) + .saturating_sub(self.in_flight) + } +} + +impl CapacityGuard { + /// Create a guard over the partition hosting `dir`. + fn new(dir: PathBuf, reserve: u64) -> Self { + Self { + dir, + reserve, + snapshot: parking_lot::Mutex::new(CapacitySnapshot { + measured_at: None, + available: 0, + written_since: 0, + in_flight: 0, + }), + } + } + + /// Bytes actually consumed on disk by a payload of `len` bytes. + fn charge(len: u64) -> u64 { + // Round the payload up to the allocation unit, then add one unit for the + // directory entry and inode. + len.div_ceil(ALLOC_UNIT) + .saturating_mul(ALLOC_UNIT) + .saturating_add(ALLOC_UNIT) + } + + /// Free bytes right now, or `None` if the question could not be answered. + /// + /// Deliberately separate from [`Self::measure`], which folds a failure into an error + /// the caller cannot tell from "below the reserve". + fn measure_available(&self) -> Option { + let mut snapshot = self.snapshot.lock(); + match self.measure(&mut snapshot) { + Ok(()) => Some(snapshot.free_estimate()), + Err(_) => None, + } + } + + /// Query the filesystem and refresh the snapshot. + fn measure(&self, snapshot: &mut CapacitySnapshot) -> Result<()> { + let available = fs2::available_space(&self.dir) + .map_err(|e| Error::Storage(format!("Failed to query available disk space: {e}")))?; + snapshot.available = available; + // Reservations survive: their bytes are not on the platter yet, so the fresh + // measurement does not include them. + snapshot.written_since = 0; + snapshot.measured_at = Some(Instant::now()); + Ok(()) + } + + /// Test `needed` against the snapshot, refreshing it if it is stale or short. + /// + /// Only *passing* results are cached, so a low-space condition is rechecked on every + /// call and freed space is noticed promptly. + fn admit(&self, snapshot: &mut CapacitySnapshot, needed: u64) -> Result<()> { + let want = self.reserve.saturating_add(Self::charge(needed)); + + let cache_fresh = snapshot + .measured_at + .is_some_and(|t| t.elapsed().as_secs() < DISK_CHECK_INTERVAL_SECS); + if cache_fresh && snapshot.free_estimate() >= want { + return Ok(()); + } + + self.measure(snapshot)?; + if snapshot.free_estimate() < want { + // Do not cache a failing result: `measured_at` is left set so the next call + // still re-measures, because the branch above only short-circuits a pass. + return Err(Error::Storage(format!( + "Insufficient disk space: {:.2} GiB available, {:.2} GiB reserve required. \ + Free disk space or increase the partition to continue storing chunks.", + bytes_to_gib(snapshot.free_estimate()), + bytes_to_gib(self.reserve), + ))); + } + Ok(()) + } + + /// Drop the cached measurement so the next question hits the filesystem. + fn invalidate(&self) { + let mut snapshot = self.snapshot.lock(); + snapshot.measured_at = None; + snapshot.written_since = 0; + } + + /// Return `Ok(())` if a write of `needed` bytes would fit. Charges nothing. + fn check(&self, needed: u64) -> Result<()> { + let mut snapshot = self.snapshot.lock(); + self.admit(&mut snapshot, needed) + } + + /// Admit a write of `needed` bytes and charge it in the same critical section. + /// + /// Checking and charging separately is the bug this exists to prevent: dozens of + /// protocol handlers can each pass against the same cached measurement before any of + /// them has written a byte, and collectively cross the reserve. + /// + /// The returned [`Reservation`] settles itself when dropped, so a caller whose future + /// is dropped mid-write cannot strand it. Nothing else ever decrements the in-flight + /// count, so a stranded reservation would be permanent, and enough of them would make + /// an empty disk look full until the process restarted. + fn reserve(self: &Arc, needed: u64) -> Result { + { + let mut snapshot = self.snapshot.lock(); + self.admit(&mut snapshot, needed)?; + snapshot.in_flight = snapshot.in_flight.saturating_add(Self::charge(needed)); + } + Ok(Reservation { + capacity: Arc::clone(self), + bytes: needed, + settled: false, + }) + } + + /// Give back a reservation whose write did not happen. + fn release(&self, needed: u64) { + let mut snapshot = self.snapshot.lock(); + snapshot.in_flight = snapshot.in_flight.saturating_sub(Self::charge(needed)); + } + + /// Turn a reservation into bytes that are now on disk. + fn commit_reservation(&self, needed: u64) { + let charge = Self::charge(needed); + let mut snapshot = self.snapshot.lock(); + snapshot.in_flight = snapshot.in_flight.saturating_sub(charge); + snapshot.written_since = snapshot.written_since.saturating_add(charge); + } + + /// Credit a completed delete back to the cached measurement. + fn record_removed(&self, len: u64) { + let mut snapshot = self.snapshot.lock(); + snapshot.written_since = snapshot.written_since.saturating_sub(Self::charge(len)); + } +} + +/// A charged, unsettled write. +/// +/// Held by whatever is actually doing the write, so the charge is released even if the +/// caller's future is dropped and only the blocking closure survives. +struct Reservation { + /// The guard this was taken from. + capacity: Arc, + /// Payload size, before rounding. + bytes: u64, + /// Whether it has already been accounted for. + settled: bool, +} + +impl Reservation { + /// The write landed: move the charge from in-flight to written. + fn commit(mut self) { + self.capacity.commit_reservation(self.bytes); + self.settled = true; + } +} + +impl Drop for Reservation { + fn drop(&mut self) { + if !self.settled { + self.capacity.release(self.bytes); + } + } +} + +/// Environment variable naming a failpoint: stop after the temp file, before the rename. +#[cfg(any(test, feature = "test-utils"))] +pub const HALT_BEFORE_PUBLISH: &str = "ANT_HALT_BEFORE_PUBLISH"; + +/// Park forever at a named failpoint, once a marker says the process has reached it. +/// +/// For crash tests, which need a process to die *inside* an operation rather than at +/// whatever point a sleep in another process happened to land. The variable holds a path: +/// this writes it, so the parent knows the child is exactly here, and then waits to be +/// killed. +/// +/// Costs one environment read per write when the feature is compiled in, and the feature +/// is not in a release build. +#[cfg(any(test, feature = "test-utils"))] +pub(crate) fn halt_here_if_asked(variable: &str, reached: &Path) { + let Ok(marker) = std::env::var(variable) else { + return; + }; + // Let the first few through. A test that stops the very first write leaves a store + // with nothing successfully in it, and an assertion over what it holds then passes by + // iterating nothing. Letting some land first means the crash happens to a store that + // has real chunks in it, which is the situation worth checking. + let skip: u64 = std::env::var(HALT_AFTER) + .ok() + .and_then(|raw| raw.parse().ok()) + .unwrap_or(0); + if HALTS_SEEN.fetch_add(1, std::sync::atomic::Ordering::AcqRel) < skip { + return; + } + if let Err(e) = std::fs::write(&marker, reached.as_os_str().as_encoded_bytes()) { + // The parent waits for this file. Saying so on the way past is the difference + // between a test that fails and one that hangs until the job times out. + eprintln!("failpoint could not write its marker {marker}: {e}"); + return; + } + loop { + std::thread::sleep(Duration::from_secs(3600)); + } +} + +/// How many writes to let through before the failpoint fires. +#[cfg(any(test, feature = "test-utils"))] +pub const HALT_AFTER: &str = "ANT_HALT_AFTER"; + +/// How many times the failpoint has been reached in this process. +#[cfg(any(test, feature = "test-utils"))] +static HALTS_SEEN: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + +/// Clears a write's registration when the work finishes, however it finishes. +/// +/// Held by the blocking closure rather than by the caller, so a dropped future cannot +/// leave an entry behind, and a panic in the work cannot either. +struct WriteInFlight { + writing: Arc>>, + finished: Arc, + address: XorName, +} + +impl Drop for WriteInFlight { + fn drop(&mut self) { + let was_last = { + let mut writing = self.writing.lock(); + match writing.get_mut(&self.address) { + Some(count) if *count > 1 => { + *count -= 1; + false + } + _ => { + writing.remove(&self.address); + true + } + } + }; + // Only when this was the last one. Waking a waiter while another write for the + // same key is still queued is exactly what the count exists to prevent. + if was_last { + self.finished.notify_waiters(); + } + } +} + +/// What is behind a chunk's name on disk. +/// +/// Four answers, not two, because "could not read it" must never be treated as "wrong": +/// replacing a chunk is destructive, and off Unix it truncates the file in place, so a +/// transient fault would turn a healthy sole copy into an empty one. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum StoredBytes { + /// The bytes are there and hash to the name. + Good, + /// The bytes are there and do not. + Wrong, + /// There is nothing behind the name. + Absent, + /// The question could not be answered this time. + Unreadable, +} + +/// Convert a byte count to GiB for human-readable log messages. +#[allow(clippy::cast_precision_loss)] // display only — sub-byte precision is irrelevant +fn bytes_to_gib(bytes: u64) -> f64 { + bytes as f64 / (1024.0 * 1024.0 * 1024.0) +} + +/// Content-addressed store holding one immutable file per chunk. +/// +/// The filesystem is the sole authority. The in-memory index is a cache of what the +/// directory tree already contains, rebuilt from directory entries at every open, and +/// every mutation of it mirrors a filesystem operation that has *already* completed. +/// Bitcask's issue #114 is the cautionary tale for the opposite order: an index that is +/// rebuilt at startup and then mutated in anticipation drifts, and the drift is silent. +#[derive(Debug)] +pub struct ChunkStore { + /// Store configuration. + config: ChunkStoreConfig, + /// `{root_dir}/chunks`. + chunks_dir: PathBuf, + /// Every address whose file is published, in ascending order. + /// + /// `BTreeSet` rather than a hash set because `all_keys()` must be sorted (the + /// commitment builder truncates with `take(cap)` *before* the Merkle tree sorts, so + /// an unstable order would make the node's published commitment depend on iteration + /// luck), and because it never spikes memory while growing. + index: Arc>>, + /// One mutex per shard, serialising writers of the same address. + /// + /// LMDB gave exactly-once `put` semantics for free: the duplicate test happened + /// inside the write transaction. Two threads publishing the same address here would + /// otherwise both see an absent file, both rename, and both report "newly stored", + /// double-counting the chunk. The lane is indexed by the address's LAST byte for the + /// same reason the shard is: a node's keys share their leading bytes, so lanes keyed + /// on the first byte would all collapse into one. + write_lanes: Arc>>, + + /// One lock per shard, held across a whole logical transition for a key. + /// + /// Not the same thing as the write lanes above, which are taken inside a blocking + /// closure and make one file write atomic. These are held across await points, which is + /// what the races that matter need: a delete has to exclude a read that is deciding + /// whether to accept an offered copy, and both span an await. Without it a prune can + /// remove the file between that read and its answer, and the caller is told the chunk + /// is already held while the copy that would have replaced it is discarded. + /// + /// Indexed by the address's LAST byte, for the reason the shard is: a node's keys share + /// their leading bytes, so lanes keyed on the first would collapse into one. + key_locks: Arc>>, + /// Operation counters, same shape as the LMDB store reported. + stats: parking_lot::RwLock, + /// Which of the 256 shard directories are known to exist, so a steady-state write + /// does not pay a `create_dir_all` syscall. + shards_present: Arc>, + /// Indexed chunks this store currently cannot read. + /// + /// Held back from everything the node says it has, while the files themselves are + /// left alone. See [`Self::mark_suspect`]. + suspect: Arc>>, + /// Indexed chunks a read has proven do not match their name. + /// + /// Separate from the above because they clear differently. Not being able to read a + /// file is a question a later read answers; bytes that are wrong stay wrong however + /// often they are read, and only a repair or a removal settles it. A raw read that + /// does not hash anything must not take a chunk out of this set. + known_wrong: Arc>>, + /// Addresses this store is part-way through writing. + /// + /// Every mutation registers here before it spawns its blocking work and clears the + /// entry *inside* that work, so a caller whose future is dropped cannot skip the + /// clearing while the write itself goes on to land. That is the difference that + /// matters: the blocking half is not cancelled with the future, so anything the + /// future was going to do afterwards is not a record of what happened. + /// + /// It lets a delete queue behind the exact write it would otherwise race, rather than + /// behind every write this store has in flight. + /// + /// Counted, not a set. Cancellation can release the facade's key lane while the + /// blocking half survives, so a second write for the same key can start behind the + /// first. With one entry between them, whichever finished first would remove it and a + /// waiter would be told the key is free while the other was still queued. + writing: Arc>>, + /// Woken when [`Self::writing`] loses its last entry for a key. + write_finished: Arc, + /// Bumped whenever a chunk stops being servable. + /// + /// A caller that reads every chunk and then reuses the result rather than re-reading + /// can tell from this that the store has not changed underneath it: the result carries + /// the value it saw, and a file that has since gone or stopped being readable makes it + /// stale. The verification pass before the old store was deleted worked this way; the + /// counter outlived it because the property is general. + health: Arc, + /// Size-aware free-space predicate. + capacity: Arc, + /// Monotonic counter that makes temp filenames unique within this store. + temp_seq: AtomicU64, + /// Random per-instance discriminator for temp filenames. + nonce: u32, + /// Held for the store's lifetime. Startup fails without it. + /// + /// Shared rather than owned so the blocking work that depends on it can hold a lease + /// of its own: that work outlives the future that spawned it, and a cancelled caller + /// releasing the lock would leave it writing into a directory another process had + /// just been let into. + lock: Arc, + /// Tracks every blocking task, so [`ChunkStore::wait_idle`] can wait for writes that + /// outlived their awaiting future. + blocking_tracker: TaskTracker, + /// Test-only gate read-acquired at the top of the put blocking closure. + /// + /// Tests hold the write half to park an in-flight write on the blocking pool, which + /// is the shape a `select!` losing to a shutdown token leaves behind. + #[cfg(test)] + test_put_gate: Arc>, + + /// Test-only: parks a put after it has taken the key's lane and before it registers + /// itself as in flight. + /// + /// Asynchronous, unlike the gate above. That one is taken inside a blocking closure on + /// its own thread; this one is taken on the runtime, so a synchronous lock here would + /// block the executor and the test would deadlock instead of observing anything. + /// + /// A separate gate from the one above, because that one sits inside the blocking + /// closure, which is after registration. The window this opens is the one the key lane + /// exists for: a put that a delete's wait cannot see yet, because there is nothing to + /// see. Without a hook here, a test cannot tell a delete blocked by the lane from a + /// delete blocked by the wait, and so cannot show the lane is doing anything. + #[cfg(test)] + test_pre_registration_gate: Arc>, + + /// Test-only: how many puts have reached that gate. + /// + /// So a test can wait for the put to be parked rather than sleeping and hoping. A sleep + /// makes the staging a guess, and a guess in a test that is meant to be deterministic + /// is a flake waiting for a loaded machine. + #[cfg(test)] + test_reached_pre_registration: Arc, +} + +impl ChunkStore { + /// Open (or create) the store at `{root_dir}/chunks/`. + /// + /// Sweeps orphaned temp files, then rebuilds the index from directory entries. + /// The scan reads names only: it never `stat`s an entry and never reads a chunk. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] if the directory cannot be created, the layout marker + /// is unreadable or describes a layout this build does not implement, or the scan + /// fails. + pub async fn new(config: ChunkStoreConfig) -> Result { + // Before anything is created. A node that still has chunks in the store this build + // cannot read must not start, and it must not leave a half-made file store behind + // when it declines to. + crate::storage::legacy_artifacts::refuse_if_unmigrated(&config.root_dir)?; + + let chunks_dir = config.root_dir.join(CHUNKS_DIR_NAME); + std::fs::create_dir_all(&chunks_dir).map_err(|e| { + Error::Storage(format!( + "Failed to create chunk store directory {}: {e}", + chunks_dir.display() + )) + })?; + + check_path_budget(&chunks_dir); + + let layout = read_or_write_layout(&chunks_dir)?; + layout.check_supported()?; + + // Startup fails without it, so from here this process is the only one using this + // directory and an interrupted write can only be its own. + let lock = acquire_store_lock(&chunks_dir)?; + + let scan_dir = chunks_dir.clone(); + // The scan holds the lease itself. It sweeps interrupted writes on the strength of + // being alone here, and it runs on a thread that outlives this future: a + // cancelled startup that released the lock would leave it sweeping a directory + // another process had just been let into. + let scan_lease = Arc::clone(&lock); + // The node root as well as the chunk tree. The scan sweeps interrupted writes + // under `chunks/`, which covers the layout marker's temporary because that lives + // there; the migration marker's lives in the root, where nothing looked. + let root = config.root_dir.clone(); + let scan = spawn_blocking(move || { + let _lease = scan_lease; + sweep_marker_temps(&root); + scan_store(&scan_dir) + }) + .await + .map_err(|e| Error::Storage(format!("Chunk store scan task failed: {e}")))??; + + let ScanResult { + keys, + shards_present, + swept_temps, + skipped, + } = scan; + + let key_count = keys.len(); + // Build from a sorted vector: bulk-building packs every B-tree node to its + // capacity, where repeated `insert` converges on ~68% fill for the same keys. + let index: BTreeSet = keys.into_iter().collect(); + + if swept_temps > 0 { + info!("Chunk store: removed {swept_temps} orphaned temporary file(s) from interrupted writes"); + } + if skipped > 0 { + warn!("Chunk store: ignored {skipped} directory entr(ies) that are not chunk files"); + } + info!( + "Chunk store open at {} ({key_count} chunks)", + chunks_dir.display() + ); + + let capacity = Arc::new(CapacityGuard::new(chunks_dir.clone(), config.disk_reserve)); + + Ok(Self { + config, + chunks_dir, + index: Arc::new(parking_lot::RwLock::new(index)), + key_locks: Arc::new( + std::iter::repeat_with(|| tokio::sync::Mutex::new(())) + .take(SHARD_COUNT) + .collect(), + ), + write_lanes: Arc::new( + std::iter::repeat_with(|| parking_lot::Mutex::new(())) + .take(SHARD_COUNT) + .collect(), + ), + stats: parking_lot::RwLock::new(StorageStats::default()), + shards_present: Arc::new(parking_lot::Mutex::new(shards_present)), + suspect: Arc::new(parking_lot::RwLock::new(HashSet::new())), + known_wrong: Arc::new(parking_lot::RwLock::new(HashSet::new())), + writing: Arc::new(parking_lot::Mutex::new(HashMap::new())), + write_finished: Arc::new(tokio::sync::Notify::new()), + health: Arc::new(std::sync::atomic::AtomicU64::new(0)), + capacity, + temp_seq: AtomicU64::new(0), + nonce: rand::random(), + lock, + blocking_tracker: TaskTracker::new(), + #[cfg(test)] + test_put_gate: Arc::new(parking_lot::RwLock::new(())), + #[cfg(test)] + test_pre_registration_gate: Arc::new(tokio::sync::RwLock::new(())), + #[cfg(test)] + test_reached_pre_registration: Arc::new(std::sync::atomic::AtomicU64::new(0)), + }) + } + + /// Store a chunk. + /// + /// On Unix, publishing is a rename within the destination directory, so the final name + /// can never appear on partial content: the name *is* the hash, and the content is + /// fully written and flushed before the name exists. Off Unix there is no rename, for + /// the reason `publish_in_place` gives (it is compiled only on those platforms, so this + /// is not a link), and a partial file can wear a real name; that + /// is why a duplicate is read and compared rather than trusted. + /// + /// # Returns + /// + /// `true` if the chunk was newly stored, `false` if it was already present. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] if the content does not hash to `address`, the disk + /// is too full, or the write fails. + pub async fn put(&self, address: &XorName, content: &[u8]) -> Result { + // The key's whole transition, not just the part that touches the disk. Registering + // the write is what a delete waits for, and everything before that registration + // happens outside it: validation, the duplicate read, the capacity reservation. A + // put that got that far before a delete arrived would otherwise be invisible to the + // delete's wait, register while the delete was already committed to going ahead, + // and publish afterwards. The node would then hold a chunk it had decided to prune. + let _lane = self.key_lock(address).await; + let computed = crate::client::compute_address(content); + if computed != *address { + return Err(Error::Storage(format!( + "Content address mismatch: expected {}, computed {}", + hex::encode(address), + hex::encode(computed) + ))); + } + // The read path refuses anything over the ceiling, so writing one would create a + // file the store could never read back and could never repair. + if content.len() > MAX_CHUNK_SIZE { + return Err(Error::Storage(format!( + "Chunk {} is {} bytes, over the {MAX_CHUNK_SIZE} byte maximum", + hex::encode(address), + content.len() + ))); + } + + // An indexed name is not proof of the bytes under it. The index is built from + // names, by the startup scan and by a completed publish, and a name can outlive + // what it points at: off Unix a chunk is created under its final name before its + // bytes are written, so a crash leaves a short file wearing a real name, and rot + // leaves a full-length one. Answering "already have it" to the copy that would fix + // either is how a node discards its own repair and is never offered another. + // + // So the bytes decide. Checked before the reservation below, so re-storing a chunk + // this node already holds stays a no-op on a full disk. + if self.index.read().contains(address) { + if let Some(answer) = self.settle_indexed_duplicate(address, content).await { + return answer; + } + } + + let len = content.len() as u64; + // Reserved after the duplicate test so re-storing an existing chunk stays a + // harmless no-op on a full disk, matching the LMDB store's ordering. + let reservation = self.capacity.reserve(len)?; + + let shard = self.chunks_dir.join(shard_name(address)); + let final_path = shard.join(hex::encode(address)); + let temp_path = shard.join(self.next_temp_name()); + let payload = content.to_vec(); + let lanes = Arc::clone(&self.write_lanes); + let index = Arc::clone(&self.index); + let shards_present = Arc::clone(&self.shards_present); + let chunks_dir = self.chunks_dir.clone(); + let lane = shard_index(address); + let key = *address; + #[cfg(test)] + let test_put_gate = Arc::clone(&self.test_put_gate); + // Registered before the work is spawned and cleared by the work itself, so a + // caller that goes away cannot leave a delete free to race this publish. + // Test-only: the window between taking the lane and being visible to a delete. + #[cfg(test)] + { + self.test_reached_pre_registration + .fetch_add(1, std::sync::atomic::Ordering::AcqRel); + drop(self.test_pre_registration_gate.read().await); + } + let in_flight = self.begin_write(address); + // And the lease, for the same reason the scan holds it: this thread writes into a + // directory whose exclusivity the lock is what establishes, and it can outlive + // the last owner of the store. + let lease = Arc::clone(&self.lock); + let known_wrong = Arc::clone(&self.known_wrong); + let suspect = Arc::clone(&self.suspect); + + let outcome = self + .blocking_tracker + .spawn_blocking(move || -> Result { + let _in_flight = in_flight; + let _lease = lease; + // Test-only: parks here while a test holds the write half. + #[cfg(test)] + let _test_put_gate = test_put_gate.read(); + let _lane = lanes.get(lane).map(parking_lot::Mutex::lock); + // `mkdir` plus a directory flush are syscalls, so they belong here and + // not on a runtime worker. + ensure_shard_dir(&chunks_dir, &shard, lane, &shards_present)?; + let outcome = match publish(&temp_path, &final_path, &payload, &shard) { + Ok(outcome) => outcome, + Err(PublishFailed { error, left_behind }) => { + // A publish that failed can still have left the bytes there: off + // Unix the chunk is created under its final name, and if the write + // or the flush then fails, the cleanup that removes it can fail + // too. Releasing the reservation would hand back a charge for a + // file that is on the disk. + // + // The publish says so rather than this deciding from a later + // `is_file`. Asking the filesystem afterwards infers ownership from + // a name being occupied, which is true under the store lock and the + // shard lane and not true against anything out of band, and this + // file spends a lot of its length arguing that a name is not + // evidence. A bit set by the code that created the file is. + if left_behind { + reservation.commit(); + } + return Err(error); + } + }; + // Placed, not yet durable. A failure from here on leaves the bytes on the + // disk: the chunk is rightly not reported as stored, because a copy that is + // not durable must not authorise deleting another, but the space is spent + // all the same. Dropping the reservation would hand that charge back and + // admit the next write against room that is already gone. + // + // Only for a chunk this call published. `Duplicate` means the file was + // already there and was charged by whoever wrote it, so charging it again + // here would count one file twice and shrink the store's idea of its own + // disk on every retry. + if let Err(e) = flush_publication(&final_path, &shard) { + if matches!(outcome, PutOutcome::New) { + reservation.commit(); + } + return Err(e); + } + // Index inside the lane, and only after the rename has returned. A + // concurrent delete of the same address therefore cannot interleave + // between publishing the file and admitting the key. + // + // Only for a chunk this call actually published. `Duplicate` says a file + // already wears the name, and a name is not evidence about the bytes under + // it: the four-way answer that decides whether they are good, wrong, absent + // or unreadable runs after the await below, and a caller whose future is + // dropped never reaches it. Admitting the key here would leave the node + // claiming, advertising and committing to bytes nothing has read, with no + // suspect or known-wrong mark to hold it back, and the sharpest case is a + // name the startup scan deliberately refused because what wears it is a + // fifo, a socket or a directory. The duplicate arm admits the key itself, + // once a read has proven the bytes. + if matches!(outcome, PutOutcome::New) { + index.write().insert(key); + // With the marks that would otherwise hold the key back. These bytes + // were hashed against their own name on the way in, so an older + // instance proven wrong or merely unreadable has just been replaced by + // a good one. Cleared here rather than after the await for the same + // reason the insert is here: a cancelled caller would leave the key + // indexed and suppressed at once, so a chunk this node really does hold + // would stay hidden from `exists` and `all_keys` until some later read + // happened to settle it. + known_wrong.write().remove(&key); + suspect.write().remove(&key); + // Settled here, inside the work, so a dropped awaiter cannot strand it. + reservation.commit(); + } + Ok(outcome) + }) + .await + .map_err(|e| Error::Storage(format!("Chunk store put task failed: {e}")))??; + + match outcome { + PutOutcome::Duplicate => self.settle_duplicate(address, content).await, + PutOutcome::New => { + // Freshly published bytes that were checked against their own name on the + // way in. The marks were already cleared inside the work, where a dropped + // caller cannot skip them; what is left here is only what a caller who is + // still waiting should see. + let mut stats = self.stats.write(); + stats.chunks_stored = stats.chunks_stored.saturating_add(1); + stats.bytes_stored = stats.bytes_stored.saturating_add(len); + drop(stats); + debug!("Stored chunk {} ({len} bytes)", hex::encode(address)); + Ok(true) + } + } + } + + /// Decide what a name that was already taken actually means. + /// + /// Split out of [`Self::put`] because it is a different question. `put` puts bytes on + /// a disk; this reads bytes back to find out whether the ones already there are the + /// ones the caller is offering, which is the only thing that makes a duplicate safe to + /// report as stored. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] when the existing file is absent or could not be read, + /// both of which mean this node must not report the chunk as held. + async fn settle_duplicate(&self, address: &XorName, content: &[u8]) -> Result { + // The file was already on disk, and its name is not evidence its contents + // are right. The startup scan indexes by name without reading anything, + // and on Windows a crash mid-write leaves a partial file under a real + // chunk name. Trusting the name here would acknowledge a chunk that was + // never stored, and then discard the good copy arriving to repair it. + // Every answer handled, because three of the four must not report the + // chunk as stored. A caller that hears success acts on it: a client drops + // its own copy, replication marks the key held, and the copier takes it + // out of the legacy-only set. + match self.stored_bytes_match(address).await { + StoredBytes::Good => { + // Admitted here, which is the first moment the bytes behind the + // name have been read and shown to hash to it. Idempotent: the + // ordinary case is a key the startup scan already indexed. + self.index.write().insert(*address); + { + let mut stats = self.stats.write(); + stats.duplicates = stats.duplicates.saturating_add(1); + } + Ok(false) + } + StoredBytes::Wrong => { + warn!( + "Chunk {} was already on disk but its contents are wrong; \ + replacing it with the copy just offered", + hex::encode(address) + ); + self.repair_holding_the_lane(address, content) + .await + .map(|()| true) + } + // The name was taken a moment ago and is not now, or was never a + // readable chunk file. Either way nothing holds these bytes, so say so + // rather than reporting a chunk that is not there. + StoredBytes::Absent => Err(Error::Storage(format!( + "Chunk {} was reported already on disk but nothing is there. Not \ + reporting it as stored.", + hex::encode(address) + ))), + // Replacing on an unanswered question would destroy a healthy copy, + // and reporting success would discard the offered one. The index entry + // stays: the file is still there, and dropping the entry would leave + // the chunk in neither this store's view nor the legacy one, which is + // what retirement destroys. Removing an entry is the quarantine path's + // job, and it removes the file with it, after a read that succeeded + // and proved the bytes wrong. + StoredBytes::Unreadable => Err(Error::Storage(format!( + "Chunk {} is on disk but could not be read to check it. Not \ + replacing it, and not reporting it as stored.", + hex::encode(address) + ))), + } + } + + /// Take the critical section for one key. + /// + /// Held across await points, unlike the write lanes, so a whole logical transition for + /// a key excludes another. `None` only if the table were empty, which it is not. + async fn key_lock(&self, address: &XorName) -> Option> { + let lane = address.last().copied().unwrap_or(0) as usize; + match self.key_locks.get(lane) { + Some(lock) => Some(lock.lock().await), + None => None, + } + } + + /// The address content hashes to. + /// + /// A convenience the old facade offered and callers still use, so it stays with the + /// store rather than making every one of them reach for the client module. + #[must_use] + pub fn compute_address(content: &[u8]) -> XorName { + crate::client::compute_address(content) + } + + /// Does this store already hold exactly these bytes under this address? + /// + /// Asked by the protocol handler before it accepts a client's PUT, so the answer has + /// to be about the bytes and not about the name. A name on disk is not evidence: the + /// startup scan indexes by name without reading anything, and a partial file can wear a + /// real chunk name. Answering yes on a name would acknowledge a chunk that was never + /// stored and then discard the good copy that had just arrived to replace it. + /// + /// A chunk that is on disk with the wrong bytes is repaired from what the caller + /// offered rather than turned away, because the caller has already checked those bytes + /// against the address. A chunk that cannot be read this time is not claimed as held + /// and not replaced either: the offer goes through the ordinary write path instead, + /// which never truncates a healthy file on the strength of an unanswered question. + pub async fn holds_verified(&self, address: &XorName, content: &[u8]) -> bool { + // The key's critical section for the whole check, so the answer is a linearizable + // statement about the store: at the moment this returns, the chunk was held and its + // bytes were these. + // + // It does not follow the answer out to the caller. The handler turns a `true` into + // an `AlreadyExists` and sends it afterwards, outside this lock, so a prune landing + // in between still means a peer is told to drop a copy of a chunk this node no + // longer has. Closing that would mean holding a per-key lock across a network + // response, which trades a narrow window for a much worse one. Replication finds + // the key missing and re-offers it. + let _lane = self.key_lock(address).await; + + if !self.is_indexed(address) { + return false; + } + // No cheap length pre-check. `metadata` failing is not the same as a length that + // does not match, and off Unix replacing a chunk truncates it in place, so acting + // on an unanswered question would empty a healthy sole copy. The read below + // distinguishes them. + match self.get_raw(address).await { + // Byte-for-byte what the caller has, and the caller checked those bytes against + // the address before getting here. Nothing is wrong with this file. + Ok(Some(stored)) if stored == content => { + self.note_bytes_proven_good(address); + true + } + Ok(_) => { + warn!( + "Chunk {} is on disk but its contents are wrong; replacing it with the \ + copy just offered", + hex::encode(address) + ); + // Recorded before the repair is attempted, not after it succeeds. A repair + // can fail for capacity or I/O, and a chunk proven wrong that goes on + // looking healthy is one the node keeps answering for. + self.note_known_wrong(address); + self.repair_holding_the_lane(address, content).await.is_ok() + } + // Unanswerable this time. Not claimed as held, so the offer goes through the + // ordinary path, which writes it rather than replacing anything. + Err(_) => false, + } + } + + /// Flush every directory a chunk can live in, so the names in them are durable. + /// + /// Byte integrity is not the whole of what a verification pass establishes. A chunk + /// whose contents are on the platter but whose *name* is not is still lost to a power + /// loss, and a publish whose rename landed and whose directory flush failed leaves + /// exactly that: the next attempt sees the name, the next verification reads the right + /// bytes, and nothing goes back to retry the flush. So the pass flushes + /// them itself rather than trusting that each publish did. + /// + /// Cheap: at most 257 directory flushes for a store of any size, and nothing off Unix, + /// where directories cannot be flushed and the retirement marker covers the same + /// ground instead. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] on the first directory that cannot be flushed. The + /// caller must treat that as a proof it did not get. + pub fn flush_namespace(&self) -> Result<()> { + fsync_dir(&self.chunks_dir).map_err(|e| { + Error::Storage(format!( + "Could not flush {}: {e}", + self.chunks_dir.display() + )) + })?; + let present = *self.shards_present.lock(); + for (shard, _) in present.iter().enumerate().filter(|(_, here)| **here) { + let dir = self.chunks_dir.join(format!("{shard:02x}")); + fsync_dir(&dir) + .map_err(|e| Error::Storage(format!("Could not flush {}: {e}", dir.display())))?; + } + Ok(()) + } + + /// Decide what to do about a write of a chunk the index already names. + /// + /// `None` means the index was wrong and there is nothing on disk, so the caller + /// publishes it as new. Everything else is the answer. + async fn settle_indexed_duplicate( + &self, + address: &XorName, + content: &[u8], + ) -> Option> { + match self.stored_bytes_match(address).await { + StoredBytes::Good => { + trace!("Chunk {} already exists", hex::encode(address)); + { + let mut stats = self.stats.write(); + stats.duplicates = stats.duplicates.saturating_add(1); + } + Some(Ok(false)) + } + StoredBytes::Wrong => { + warn!( + "Chunk {} is indexed but its bytes are wrong; replacing it with the \ + copy just offered", + hex::encode(address) + ); + Some( + self.repair_holding_the_lane(address, content) + .await + .map(|()| true), + ) + } + // Indexed but gone: publish it fresh rather than replacing something that is + // not there. + StoredBytes::Absent => None, + // Unanswerable this time. Do not touch what is there, and do not tell the + // caller the chunk is safely stored either: a client would take that as an + // acknowledgement and drop the only other copy. The index entry stays, for + // the reason given on the same case after publication. + StoredBytes::Unreadable => Some(Err(Error::Storage(format!( + "Chunk {} is indexed but could not be read to check it. Not replacing it, \ + and not reporting it as stored.", + hex::encode(address) + )))), + } + } + + /// The size of the file behind `address`, if there is one. + /// + /// One `metadata` call, no read. Used where an indexed name has to be checked against + /// what a caller is offering before that offer is turned away. + #[must_use] + pub fn stored_len(&self, address: &XorName) -> Option { + std::fs::metadata(self.chunk_path(address)) + .ok() + .filter(std::fs::Metadata::is_file) + .and_then(|m| usize::try_from(m.len()).ok()) + } + + /// Whether the file already stored under `address` really hashes to it. + async fn stored_bytes_match(&self, address: &XorName) -> StoredBytes { + match self.get_raw(address).await { + Ok(Some(bytes)) if crate::client::compute_address(&bytes) == *address => { + // A read that hashed. It settles both questions. + self.clear_suspect(address); + self.clear_known_wrong(address); + StoredBytes::Good + } + Ok(Some(_)) => { + self.mark_known_wrong(address); + StoredBytes::Wrong + } + Ok(None) => StoredBytes::Absent, + // NOT the same as wrong. A file that could not be read this once may be + // perfectly good, and off Unix replacing it means opening it with `truncate`, + // which would destroy a healthy sole copy on the strength of a transient + // fault. Say so and let the caller leave it alone. + Err(e) => { + debug!("Could not read {} to check it: {e}", hex::encode(address)); + self.mark_suspect(address); + StoredBytes::Unreadable + } + } + } + + /// Replace the file behind an address with bytes that hash to it. + /// + /// Unlike [`Self::put`], this deliberately publishes **over** an existing name, for + /// repairing a file whose bytes no longer hash to their own. Doing it as + /// delete-then-put would leave a window with no copy at all. + /// + /// **Only call this once a read has shown the existing bytes are wrong.** On Unix the + /// replacement is atomic and a failure leaves the old file untouched. Off Unix it is + /// not: there is no durable rename there, so the existing file is truncated and + /// rewritten in place, and a crash part-way leaves a mixture. That is tolerable when + /// the bytes being replaced were already known to be wrong, and is data loss when they + /// were not. Nothing enforces the precondition, which is why it is stated here. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] if `content` does not hash to `address`, or the write + /// fails. + pub async fn repair(&self, address: &XorName, content: &[u8]) -> Result<()> { + let _lane = self.key_lock(address).await; + self.repair_holding_the_lane(address, content).await + } + + /// The body of [`Self::repair`], for callers that already hold the key's lane. + /// + /// Split because the lane is a `tokio::sync::Mutex` and so is not reentrant: every + /// internal caller reaches this while holding it, and taking it again would deadlock + /// the task on itself. + async fn repair_holding_the_lane(&self, address: &XorName, content: &[u8]) -> Result<()> { + // The same ceiling `put` enforces. Without it a repair can install bytes the read + // path will refuse for ever, which is a chunk that verifies as present and can + // never be served. + if content.len() > MAX_CHUNK_SIZE { + return Err(Error::Storage(format!( + "Refusing to repair {} with {} bytes, over the {MAX_CHUNK_SIZE} byte \ + maximum", + hex::encode(address), + content.len() + ))); + } + let computed = crate::client::compute_address(content); + if computed != *address { + return Err(Error::Storage(format!( + "Refusing to repair {} with content that hashes to {}", + hex::encode(address), + hex::encode(computed) + ))); + } + // The replacement exists alongside the original until the rename, so the room for + // it has to be there first. Reserved rather than merely checked: a plain check + // passes against a cached measurement, so concurrent repairs and PUTs can each be + // admitted against the same headroom and cross the reserve together. + // Moved into the work below, so it is released when the write finishes rather + // than when its caller stops waiting. A caller that goes away otherwise frees + // room that the detached write is still about to consume. + let reservation = self.capacity.reserve(content.len() as u64)?; + + let shard = self.chunks_dir.join(shard_name(address)); + let final_path = shard.join(hex::encode(address)); + let temp_path = shard.join(self.next_temp_name()); + let payload = content.to_vec(); + let lanes = Arc::clone(&self.write_lanes); + let index = Arc::clone(&self.index); + let shards_present = Arc::clone(&self.shards_present); + let chunks_dir = self.chunks_dir.clone(); + let lane = shard_index(address); + let key = *address; + let in_flight = self.begin_write(address); + let lease = Arc::clone(&self.lock); + let capacity = Arc::clone(&self.capacity); + let suspect = Arc::clone(&self.suspect); + let known_wrong = Arc::clone(&self.known_wrong); + + self.blocking_tracker + .spawn_blocking(move || -> Result<()> { + let _in_flight = in_flight; + let _lease = lease; + let _reservation = reservation; + let _lane = lanes.get(lane).map(parking_lot::Mutex::lock); + ensure_shard_dir(&chunks_dir, &shard, lane, &shards_present)?; + write_and_replace(&temp_path, &final_path, &payload, &shard)?; + index.write().insert(key); + // Settled here rather than after the await. The replacement has landed + // and hashes to its own name, so nothing is wrong with this chunk any + // more; a caller that stopped waiting would otherwise leave a healthy + // file excluded from everything the node claims to hold, and the + // measurement believing the store is a chunk smaller than it is. + suspect.write().remove(&key); + known_wrong.write().remove(&key); + capacity.invalidate(); + Ok(()) + }) + .await + .map_err(|e| Error::Storage(format!("Chunk store repair task failed: {e}")))??; + + // Everything the success means was recorded by the work that succeeded: the + // reservation released, the marks cleared, the measurement thrown away. Released + // rather than committed because a repair is not a new chunk, and the measurement + // discarded rather than adjusted because the file it replaced may have been + // shorter, which is exactly the case a repair fixes. + debug!("Repaired chunk {}", hex::encode(address)); + Ok(()) + } + + /// Retrieve a chunk, verifying it against its address when configured to. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] on an I/O failure, or when verification fails. A + /// chunk whose bytes do not hash to its name is removed and dropped from the index + /// before the error is returned, so it leaves `all_keys()` and ordinary replication + /// repairs it. + pub async fn get(&self, address: &XorName) -> Result>> { + let Some(content) = self.read_file(address).await? else { + trace!("Chunk {} not found", hex::encode(address)); + return Ok(None); + }; + + if self.config.verify_on_read { + let computed = crate::client::compute_address(&content); + if computed != *address { + { + let mut stats = self.stats.write(); + stats.verification_failures = stats.verification_failures.saturating_add(1); + } + warn!( + "Chunk verification failed: expected {}, computed {}", + hex::encode(address), + hex::encode(computed) + ); + // Said before it is acted on. Removing the file can fail or be cancelled, + // and a chunk proven wrong that goes on looking healthy is one the node + // keeps committing to and keeps being audited for. + self.mark_known_wrong(address); + self.quarantine_corrupt(address).await; + return Err(Error::Storage(format!( + "Chunk verification failed for {}", + hex::encode(address) + ))); + } + } + + if self.config.verify_on_read { + // The bytes hashed to their name. Whatever this store thought was wrong with + // them is not wrong with them, and a mark that outlives the fault it + // describes means the node can serve a chunk it will not claim, commit or + // offer. + self.clear_known_wrong(address); + } + + let len = content.len() as u64; + { + let mut stats = self.stats.write(); + stats.chunks_retrieved = stats.chunks_retrieved.saturating_add(1); + stats.bytes_retrieved = stats.bytes_retrieved.saturating_add(len); + } + debug!("Retrieved chunk {} ({len} bytes)", hex::encode(address)); + Ok(Some(content)) + } + + /// Retrieve raw chunk bytes without content-address verification. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] on an I/O failure. + pub async fn get_raw(&self, address: &XorName) -> Result>> { + self.read_file(address).await + } + + /// Check whether a chunk is stored. + /// + /// An in-memory lookup: no syscall, no I/O. + /// + /// # Errors + /// + /// Never fails. The signature keeps the shape the LMDB store had, because callers + /// treat the error as "assume absent". + pub fn exists(&self, address: &XorName) -> Result { + if self.is_unservable(address) { + return Ok(false); + } + Ok(self.is_indexed(address)) + } + + /// Is this chunk one the node must not answer for? + #[must_use] + fn is_unservable(&self, address: &XorName) -> bool { + self.suspect.read().contains(address) || self.known_wrong.read().contains(address) + } + + /// Is this chunk in the index, whether or not it can currently be read? + /// + /// The physical question, as against [`Self::exists`]'s question about what the node + /// is willing to claim. The migration must ask this one: a suspect chunk is still a + /// file this store has, and treating it as absent would put the key in the legacy-only + /// set, from where the union view advertises it again — a key the node claims through + /// one view and cannot serve through either. + #[must_use] + pub fn is_indexed(&self, address: &XorName) -> bool { + self.index.read().contains(address) + } + + /// Delete a chunk, returning whether it was present. + /// + /// `unlink` returns the blocks to the filesystem immediately. That is the whole + /// point of this store: no free list, no compaction, no free space required to + /// reclaim space. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] if the file exists but cannot be removed. The index + /// keeps the key in that case, because the bytes are still on disk. + pub async fn delete(&self, address: &XorName) -> Result { + // The key's whole critical section, held across the wait below and the unlink. + let _lane = self.key_lock(address).await; + // Behind whatever is already writing this key, and only this key. A write's + // blocking half outlives the future that started it, so one landing after this + // would put back a chunk the node had decided to prune, and the next thing to look + // would find it in a store that no longer claims it. + self.wait_for_write(address).await; + + let path = self.chunk_path(address); + let lanes = Arc::clone(&self.write_lanes); + let index = Arc::clone(&self.index); + let lane = shard_index(address); + let key = *address; + // Carried into the closure for the reason `put`, `repair` and the startup scan + // carry it: this work outlives the future that started it, so a cancelled caller + // that drops the last `ChunkStore` would otherwise release the directory to another + // process while an unlink is still queued against it. Deleting is the operation + // where that matters most. + let lease = Arc::clone(&self.lock); + + let (existed, freed) = self + .blocking_tracker + .spawn_blocking(move || -> Result<(bool, u64)> { + let _lease = lease; + let _lane = lanes.get(lane).map(parking_lot::Mutex::lock); + let len = std::fs::metadata(&path).map_or(0, |m| m.len()); + let removed = match std::fs::remove_file(&path) { + Ok(()) => { + // Without this a crash can resurrect the entry on ext4, XFS, + // btrfs and APFS: the unlink is in the page cache, the directory + // is not. + if let Some(shard) = path.parent() { + fsync_dir_best_effort(shard); + } + true + } + // Already gone: the index was stale. Still a successful delete as + // far as the caller is concerned. + Err(e) if e.kind() == ErrorKind::NotFound => false, + Err(e) => { + return Err(Error::Storage(format!( + "Failed to delete chunk file {}: {e}", + path.display() + ))) + } + }; + // Index only after the filesystem operation has succeeded. On the error + // path above the entry stays, because the bytes are still on disk. + let was_indexed = index.write().remove(&key); + Ok((removed || was_indexed, if removed { len } else { 0 })) + }) + .await + .map_err(|e| Error::Storage(format!("Chunk store delete task failed: {e}")))??; + + if freed > 0 { + self.capacity.record_removed(freed); + debug!("Deleted chunk {}", hex::encode(address)); + } + Ok(existed) + } + + /// Return every stored key, in ascending order. + /// + /// The order is a correctness requirement, not a convenience: the commitment + /// builder truncates the responsible subset with `take(cap)` before the Merkle tree + /// sorts it, so an unstable order would make the node's published commitment depend + /// on iteration luck. + /// + /// # Errors + /// + /// Never fails. The signature matches the LMDB store's. + // Async without awaiting anything, deliberately: the whole point of this store is + // that the key set is already in memory. Callers are spread across the replication + // engine and cannot all be de-async'd in this change. + // + // Two lint names because they were renamed between toolchains, and `unknown_lints` + // so whichever one the compiler in use has never heard of stays quiet. + #[allow(unknown_lints)] + #[allow(clippy::unused_async, clippy::unused_async_trait_impl)] + pub async fn all_keys(&self) -> Result> { + // Copied out first so neither lock is held while the other is taken, and so the + // usual case, where nothing is suspect, costs one clone of an empty set. + let mut unservable: HashSet = self.suspect.read().clone(); + unservable.extend(self.known_wrong.read().iter().copied()); + let keys = self.index.read().clone(); + if unservable.is_empty() { + return Ok(keys.into_iter().collect()); + } + Ok(keys + .into_iter() + .filter(|key| !unservable.contains(key)) + .collect()) + } + + /// Stop answering for a chunk this store could not read. + /// + /// The file stays. It may be perfectly good and unreadable only for the moment, and + /// deleting it, or dropping it from the index, is how a chunk ends up in neither this + /// store's view nor the legacy one, which is what retirement destroys. + /// + /// What does change is what the node says about it. A chunk it cannot read is one it + /// cannot serve, and claiming it anyway puts the key in signed commitments, answers + /// presence probes with a yes, suppresses the replication that would repair it, and + /// earns a penalty at the next commitment-bound audit. Those penalties are not + /// suspended. + fn mark_suspect(&self, address: &XorName) { + if self.suspect.write().insert(*address) { + self.note_health_changed(); + warn!( + "Chunk {} is on disk but could not be read; this node stops answering for \ + it until a read succeeds", + hex::encode(address) + ); + } + } + + /// What the store's health looked like at this moment. + /// + /// Compare a value taken before a long-running check with one taken after, or after + /// taking a lock: different means a chunk stopped being servable in between and any + /// conclusion drawn from that check is out of date. + #[must_use] + pub fn health_generation(&self) -> u64 { + self.health.load(std::sync::atomic::Ordering::Acquire) + } + + /// Record that a chunk stopped being servable. + fn note_health_changed(&self) { + self.health + .fetch_add(1, std::sync::atomic::Ordering::AcqRel); + } + + /// Stop answering for a chunk a read has proven wrong. + /// + /// Unlike a chunk that merely could not be read, a later read does not clear this. + /// The bytes are wrong, and reading them again says the same thing; only replacing + /// them or removing them settles it. Bumping health matters as much as the suppression: a chunk that + /// has become unservable since the last pre-retirement pass must invalidate that pass, + /// or a repair that fails leaves the node deleting the copy it would have repaired + /// from. + /// + /// For callers outside this module that have proven it themselves. + pub fn note_known_wrong(&self, address: &XorName) { + self.mark_known_wrong(address); + } + + /// Stop answering for a chunk a read has proven wrong. + fn mark_known_wrong(&self, address: &XorName) { + if self.known_wrong.write().insert(*address) { + self.note_health_changed(); + warn!( + "Chunk {} does not match its name; this node stops answering for it until \ + it is repaired or removed", + hex::encode(address) + ); + } + } + + /// A caller outside this module has proven the stored bytes are right. + pub fn note_bytes_proven_good(&self, address: &XorName) { + self.clear_known_wrong(address); + self.clear_suspect(address); + } + + /// Answer for a chunk again, after it has been replaced or removed. + fn clear_known_wrong(&self, address: &XorName) { + self.known_wrong.write().remove(address); + } + + /// Answer for a chunk again, after a read that worked. + fn clear_suspect(&self, address: &XorName) { + if !self.suspect.read().contains(address) { + return; + } + if self.suspect.write().remove(address) { + info!( + "Chunk {} could be read again; this node answers for it once more", + hex::encode(address) + ); + } + } + + /// Number of chunks currently stored. + /// + /// The physical count: every name in the index, including chunks the node has stopped + /// answering for because a read found them wrong or could not read them at all. It is + /// deliberately not the same number as `all_keys().len()`, which is what the node is + /// willing to claim and so leaves those out. + /// + /// Anything asking "how much is on this disk" wants this one, and that is what its + /// callers ask: the migration's progress, the storage stats, and the size an audit is + /// built for. Anything asking "what will this node answer for" wants `all_keys`. + /// Quietly filtering this one would move all three of those without saying so, which + /// is why the difference is written down here rather than removed. + /// + /// # Errors + /// + /// Never fails. The signature matches the LMDB store's. + pub fn current_chunks(&self) -> Result { + Ok(self.index.read().len() as u64) + } + + /// Operation statistics, with the live chunk count filled in. + #[must_use] + pub fn stats(&self) -> StorageStats { + let mut stats = self.stats.read().clone(); + stats.current_chunks = self.index.read().len() as u64; + stats + } + + /// The node root directory this store was configured with. + #[must_use] + pub fn root_dir(&self) -> &Path { + &self.config.root_dir + } + + /// The directory holding the shard tree. + #[must_use] + pub fn chunks_dir(&self) -> &Path { + &self.chunks_dir + } + + /// Reject work early when the disk cannot take another chunk at all. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] when free space is below the configured reserve. + pub fn check_capacity(&self) -> Result<()> { + self.capacity.check(0) + } + + /// Three-way answer to "can this store take a write right now". + /// + /// Kept distinct from [`Self::check_capacity`] because a failed free-space query and a + /// genuinely full disk are not the same thing, and the replication verification cycle + /// depends on the difference: a full disk is a standing condition worth minutes of + /// backoff, while a `statvfs` that failed says nothing about available space and may + /// well succeed on the next pass. + #[must_use] + pub fn capacity_verdict(&self) -> crate::storage::CapacityVerdict { + match self.capacity.measure_available() { + Some(available) if available < self.capacity.reserve => { + crate::storage::CapacityVerdict::Full + } + Some(_) => crate::storage::CapacityVerdict::Writable, + None => crate::storage::CapacityVerdict::Unknown, + } + } + + /// Reject work early when the disk cannot take `bytes` more. + /// + /// # Errors + /// + /// Returns [`Error::Storage`] when the write would not fit above the reserve. + pub fn check_capacity_for(&self, bytes: u64) -> Result<()> { + self.capacity.check(bytes) + } + + /// Force the next capacity question to re-measure the filesystem. + /// + /// Called after the legacy environment is removed, because that is a step change in + /// free space that the short-lived cache would otherwise hide for a few seconds. + pub fn invalidate_capacity_cache(&self) { + self.capacity.invalidate(); + } + + /// Test-only handle to the put gate. + /// + /// Hold the write half to park the next write inside its blocking closure, for + /// example to prove that shutdown waits for a write whose awaiter was dropped. + #[cfg(test)] + fn test_put_gate(&self) -> Arc> { + Arc::clone(&self.test_put_gate) + } + + /// Test-only handle to the gate that parks a put before it registers itself. + #[cfg(test)] + fn test_pre_registration_gate(&self) -> Arc> { + Arc::clone(&self.test_pre_registration_gate) + } + + /// Test-only: how many puts have reached the pre-registration gate. + #[cfg(test)] + fn test_reached_pre_registration(&self) -> u64 { + self.test_reached_pre_registration + .load(std::sync::atomic::Ordering::Acquire) + } + + /// Register a write of `address` and hand back the token that clears it. + /// + /// The token must be moved into the blocking closure that does the work, so the entry + /// is cleared by the thread that finishes rather than by a caller that may be gone. + fn begin_write(&self, address: &XorName) -> WriteInFlight { + *self.writing.lock().entry(*address).or_insert(0) += 1; + WriteInFlight { + writing: Arc::clone(&self.writing), + finished: Arc::clone(&self.write_finished), + address: *address, + } + } + + /// Wait until nothing is part-way through writing `address`. + /// + /// For callers that must be last: a delete whose key still has a write in flight + /// would be undone by that write landing afterwards. + pub async fn wait_for_write(&self, address: &XorName) { + loop { + // Registered before the check, so a clear between the two is not missed. + let waiting = self.write_finished.notified(); + if !self.writing.lock().contains_key(address) { + return; + } + waiting.await; + } + } + + /// How many blocking tasks this store currently has in flight. Tests only. + /// + /// Lets a test wait for work to have actually started rather than guessing at a + /// delay, which is the difference between a test that proves something and one that + /// passes because the machine was quick. + #[cfg(test)] + #[must_use] + pub(crate) fn tasks_in_flight(&self) -> usize { + self.blocking_tracker.len() + } + + /// Wait until every blocking task this store spawned has finished. + /// + /// Dropping the awaiting future does not cancel a `spawn_blocking` closure, so + /// shutdown has to wait for the closure itself. + pub async fn wait_idle(&self) { + self.blocking_tracker.close(); + self.blocking_tracker.wait().await; + self.blocking_tracker.reopen(); + } + + /// Absolute path of a chunk file. + fn chunk_path(&self, address: &XorName) -> PathBuf { + self.chunks_dir + .join(shard_name(address)) + .join(hex::encode(address)) + } + + /// A temp name unique to this store instance, and distinguishable from a chunk name. + /// + /// The nonce matters: two `ChunkStore`s on one root in one process share a PID, and a + /// recycled PID collides with an age-gated leftover. Either way `create_new` would + /// fail and surface as a spurious write error. + fn next_temp_name(&self) -> String { + let seq = self.temp_seq.fetch_add(1, Ordering::Relaxed); + format!( + "{TEMP_PREFIX}{}.{:08x}.{seq}", + std::process::id(), + self.nonce + ) + } + + /// Read a chunk file, dropping the index entry if the file has vanished. + async fn read_file(&self, address: &XorName) -> Result>> { + let path = self.chunk_path(address); + let read = self + .blocking_tracker + .spawn_blocking(move || -> Result>> { + match open_regular(&path) { + Ok(Some(f)) => read_bounded(f, &path).map(Some), + Ok(None) => Ok(None), + Err(e) => Err(e), + } + }) + .await + .map_err(|e| Error::Storage(format!("Chunk store read task failed: {e}")))?; + + // Every read decides the question, not only the ones that were checking. A read + // that failed means this chunk cannot be served, whoever asked; a read that + // worked means it can be, whoever asked. Doing this anywhere else leaves a key + // stuck unadvertised after the fault has cleared, or advertised after it has not. + let read = match read { + Ok(read) => { + self.clear_suspect(address); + read + } + Err(e) => { + self.mark_suspect(address); + return Err(e); + } + }; + + if read.is_none() && self.forget_if_absent(address).await { + // The file went away underneath us. Stop advertising the key so the close + // group notices the shortfall and replication puts it back. + warn!( + "Chunk {} is indexed but its file is missing; dropped from the index so \ + replication can repair it", + hex::encode(address) + ); + } + Ok(read) + } + + /// Drop an index entry whose file is genuinely gone. + /// + /// Re-checks under the address's write lane, so a chunk republished between the + /// failing read and this call keeps its entry. + async fn forget_if_absent(&self, address: &XorName) -> bool { + // Not suspect any more: it is not unreadable, it is not there. + self.clear_suspect(address); + let path = self.chunk_path(address); + let lanes = Arc::clone(&self.write_lanes); + let index = Arc::clone(&self.index); + let lane = shard_index(address); + let key = *address; + // The bump happens inside the closure, with the mutation it describes. The + // closure runs to completion on its own thread whether or not anyone is still + // awaiting it, so bumping after the await is skipped entirely when a shutdown + // drops the caller — and the index change it was meant to announce still lands. + // A cached pre-retirement proof would then stay valid over a store that had + // quietly lost a chunk. + let health = Arc::clone(&self.health); + self.blocking_tracker + .spawn_blocking(move || { + let _lane = lanes.get(lane).map(parking_lot::Mutex::lock); + if path.exists() { + return false; + } + let forgotten = index.write().remove(&key); + if forgotten { + health.fetch_add(1, std::sync::atomic::Ordering::AcqRel); + } + forgotten + }) + .await + .unwrap_or(false) + } + + /// Remove a chunk whose bytes do not match its name, and stop advertising it. + /// + /// Re-reads and re-verifies under the address's write lane first. A read that failed + /// verification is rare enough that paying for one extra read is worth never + /// discarding a chunk that a concurrent write had already repaired. + async fn quarantine_corrupt(&self, address: &XorName) { + let path = self.chunk_path(address); + let lanes = Arc::clone(&self.write_lanes); + let index = Arc::clone(&self.index); + let lane = shard_index(address); + let key = *address; + // For the reason given on `forget_if_absent`: this closure outlives its awaiter, + // and the change it makes has to be announced by the same thread that makes it. + let health = Arc::clone(&self.health); + // And the store-lock lease, for the reason `put`, `repair`, `delete` and the + // startup scan carry it: this closure outlives its awaiter, so without it a + // cancelled verification whose caller dropped the last `ChunkStore` would unlink + // inside a directory a second process had already been handed. + let lease = Arc::clone(&self.lock); + let outcome = + self.blocking_tracker + .spawn_blocking(move || -> std::io::Result { + let _lease = lease; + let _lane = lanes.get(lane).map(parking_lot::Mutex::lock); + // Nothing is thrown away without proof. A re-read that fails says the + // question could not be answered this time, not that the bytes are wrong, + // and a repair may have published a good copy since the read that brought + // us here. Treating either as corruption deletes a chunk this node has. + let buf = match open_regular(&path) { + Ok(Some(f)) => read_bounded(f, &path) + .map_err(|e| std::io::Error::other(e.to_string()))?, + Ok(None) => { + index.write().remove(&key); + health.fetch_add(1, std::sync::atomic::Ordering::AcqRel); + return Ok(true); + } + Err(e) => return Err(std::io::Error::other(e.to_string())), + }; + if crate::client::compute_address(&buf) == key { + // Repaired between the failing read and now. Leave it alone. + return Ok(false); + } + std::fs::remove_file(&path)?; + // The same flush the ordinary delete does, for the same reason: an + // unlink that has not reached the directory can be undone by a power + // loss, and here the entry that comes back is one this node has proven + // wrong. The startup scan would re-index it by name, and the + // known-wrong mark that would otherwise hold it back lives only in + // memory and does not survive the restart, so the node would go back to + // claiming and committing to a chunk it already knows is bad. + if let Some(shard) = path.parent() { + fsync_dir_best_effort(shard); + } + index.write().remove(&key); + health.fetch_add(1, std::sync::atomic::Ordering::AcqRel); + Ok(true) + }) + .await; + match outcome { + Ok(Ok(true)) => { + self.clear_known_wrong(address); + self.clear_suspect(address); + warn!( + "Removed corrupt chunk file {}; replication will repair it", + hex::encode(address) + ); + } + Ok(Ok(false)) => { + // The re-read hashed and matched: a repair landed between the failing + // read and this one. + self.clear_known_wrong(address); + self.clear_suspect(address); + debug!( + "Chunk {} verified on re-read; leaving it in place", + hex::encode(address) + ); + } + // Still indexed, so it must not still be claimed: the read that brought us + // here proved the bytes wrong, and the node would otherwise go on committing + // to a chunk it knows it cannot serve. + Ok(Err(e)) => { + self.mark_suspect(address); + warn!( + "Corrupt chunk {} could not be removed: {e}. It stays on disk, and \ + this node stops answering for it.", + hex::encode(address) + ); + } + Err(e) => { + self.mark_suspect(address); + warn!("Corrupt-chunk removal task failed: {e}"); + } + } + } +} + +// ──────────────────────────────────────────────────────────────────────────── +// Free functions +// ──────────────────────────────────────────────────────────────────────────── + +/// Create the destination shard directory if this store has not seen it yet. +/// +/// A newly created directory entry is only durable once its parent is flushed; without +/// that a crash could take the directory and the chunk inside it together. +fn ensure_shard_dir( + chunks_dir: &Path, + dir: &Path, + shard: usize, + present: &parking_lot::Mutex<[bool; SHARD_COUNT]>, +) -> Result<()> { + if present.lock().get(shard).copied().unwrap_or(false) { + return Ok(()); + } + std::fs::create_dir_all(dir).map_err(|e| { + Error::Storage(format!( + "Failed to create shard directory {}: {e}", + dir.display() + )) + })?; + // Load-bearing, like the flush that publishes a chunk into this directory. Until the + // parent is flushed the shard's own entry can be lost, and losing it loses every chunk + // inside it. Reporting the shard present anyway would let the very first chunk written + // into it count as durably stored. + fsync_dir(chunks_dir).map_err(|e| { + Error::Storage(format!( + "Created shard directory {} but could not flush {}: {e}. Not marking the shard \ + usable, because a directory that is not durable cannot hold a chunk that is.", + dir.display(), + chunks_dir.display() + )) + })?; + if let Some(slot) = present.lock().get_mut(shard) { + *slot = true; + } + Ok(()) +} + +/// Shard directory index for an address: its last byte. +fn shard_index(address: &XorName) -> usize { + address.last().copied().unwrap_or(0) as usize +} + +/// Shard directory name for an address: the last two characters of its hex form. +fn shard_name(address: &XorName) -> String { + format!("{:02x}", shard_index(address)) +} + +/// True for a string of hex digits in either case. +fn is_hex_any_case(s: &str) -> bool { + !s.is_empty() && s.bytes().all(|b| b.is_ascii_hexdigit()) +} + +/// Move an entry aside under a name that can never be read as a chunk. +fn quarantine_entry(path: &Path) { + let aside = path.with_extension("not-a-chunk"); + match std::fs::rename(path, &aside) { + Ok(()) => warn!( + "Chunk store: moved {} aside to {}; a name that differs from a chunk name only \ + by case collides with it on Windows and macOS", + path.display(), + aside.display() + ), + Err(e) => warn!( + "Chunk store: {} collides with a chunk name by case folding and could not be \ + moved aside: {e}. Rename or delete it.", + path.display() + ), + } +} + +/// True for a string of lowercase hex digits only. +fn is_lower_hex(s: &str) -> bool { + !s.is_empty() && s.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')) +} + +/// Decode a filename back into the address it names, or `None` if it is not one. +/// +/// Rejects uppercase deliberately. On a case-folding filesystem (NTFS, default APFS) +/// accepting both cases would let one file answer to two index entries. +fn decode_chunk_name(name: &str) -> Option { + if name.len() != CHUNK_NAME_LEN || !is_lower_hex(name) { + return None; + } + let bytes = hex::decode(name).ok()?; + XorName::try_from(bytes.as_slice()).ok() +} + +/// Flush a directory so a rename or creation inside it survives power loss. +/// +/// Best effort by design. Linux and XFS require it, macOS accepts it with undocumented +/// effect, and Windows offers no way to do it at all through the standard library. The +/// content is content-addressed and re-replicable, so a lost directory entry costs a +/// refetch rather than data. Pretending otherwise in the code would be dishonest. +#[cfg(unix)] +fn fsync_dir_best_effort(path: &Path) { + if let Err(e) = fsync_dir(path) { + debug!("Directory flush of {} failed: {e}", path.display()); + } +} + +/// Flush a directory, reporting whether it worked. +/// +/// Used where the answer is load-bearing: a chunk copied out of the legacy store is only +/// durable once its directory entry is, and that copy is what permits the legacy store to +/// be deleted. +#[cfg(unix)] +fn fsync_dir(path: &Path) -> std::io::Result<()> { + File::open(path)?.sync_all() +} + +/// Off Unix there is no way to flush a directory through the standard library, so this +/// reports success without being able to promise anything. +/// +/// That is why the publish path off Unix does not use a rename at all: it creates the +/// chunk under its final name and flushes the file, which Microsoft documents as flushing +/// the creation metadata with it. Directory creation has no equivalent, so the guarantee +/// there rests on the pre-retirement pass, which re-reads every chunk before the legacy +/// store is deleted, and on the operator gate that keeps retirement off a platform until +/// forced power loss has been shown to hold old-or-new on it. +/// +/// Returns a `Result` so the callers that must handle a flush failure on Unix read the +/// same on every platform. +#[cfg(not(unix))] +#[allow(clippy::unnecessary_wraps)] +fn fsync_dir(_path: &Path) -> std::io::Result<()> { + Ok(()) +} + +/// No-op on platforms with no way to flush a directory handle. +#[cfg(not(unix))] +fn fsync_dir_best_effort(_path: &Path) {} + +/// Warn if the deepest chunk path this store can produce is close to `MAX_PATH`. +#[cfg(windows)] +fn check_path_budget(chunks_dir: &Path) { + // Measured absolute, because that is what the filesystem sees. A relative root is the + // case that still fails hard at MAX_PATH, since the standard library's long-path + // handling only applies to paths it resolves as absolute. + let absolute = if chunks_dir.is_absolute() { + chunks_dir.to_path_buf() + } else { + std::env::current_dir() + .map_or_else(|_| chunks_dir.to_path_buf(), |cwd| cwd.join(chunks_dir)) + }; + // `{chunks_dir}\{xy}\{64 hex}` — two separators, two shard characters, 64 name + // characters. + let deepest = absolute.as_os_str().len() + 1 + 2 + 1 + CHUNK_NAME_LEN; + if deepest > WINDOWS_PATH_WARN_LEN { + warn!( + "Chunk file paths will be {deepest} characters, close to the {} character \ + Windows limit. Move the node root closer to the drive letter if writes start \ + failing.", + WINDOWS_PATH_WARN_LEN + ); + } +} + +/// No-op where path length is not a practical constraint. +#[cfg(not(windows))] +fn check_path_budget(_chunks_dir: &Path) {} + +/// Write `bytes` to `path` so a reader sees either the old content or the new. +/// Is this the exact name [`write_file_atomic`] gives its temporaries? +/// +/// `.tmp..<8 hex>.marker`, with both middle parts checked. Matching on the prefix and +/// suffix alone would also take `.tmp.operator-notes.marker`, and this runs over a +/// directory holding a node's data, so what it removes is not a place to be approximate. +fn is_marker_temp_name(name: &str) -> bool { + let Some(rest) = name.strip_prefix(TEMP_PREFIX) else { + return false; + }; + let Some(rest) = rest.strip_suffix(".marker") else { + return false; + }; + let mut parts = rest.split('.'); + let (Some(pid), Some(nonce), None) = (parts.next(), parts.next(), parts.next()) else { + return false; + }; + !pid.is_empty() + && pid.bytes().all(|b| b.is_ascii_digit()) + && nonce.len() == 8 + && nonce.bytes().all(|b| b.is_ascii_hexdigit()) +} + +/// Remove marker temporaries a previous run left beside `path`. +/// +/// [`write_file_atomic`] writes its temporary next to its target. For the layout marker +/// that is inside `chunks/`, which the startup scan sweeps; for the migration marker it is +/// the node root, which nothing sweeps, so a crash between the write and the rename leaves +/// one there for the life of the node. Each is a few hundred bytes, so this is inodes +/// rather than capacity, but nothing else was ever going to remove them. +/// +/// Only the exact shape this module writes, and only files: a name has to carry the temp +/// prefix and the marker suffix. Anything broader would be this function deciding what +/// else in a node's root directory is rubbish, which is not its business. +/// +/// Best effort throughout. Failing to tidy up is not a reason to refuse to start, and the +/// caller takes the store lock before this runs, so there is no other process whose live +/// temporary this could take. +pub(crate) fn sweep_marker_temps(dir: &Path) { + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + for entry in entries.flatten() { + let name = entry.file_name(); + let Some(name) = name.to_str() else { + continue; + }; + if !is_marker_temp_name(name) { + continue; + } + if !entry.file_type().is_ok_and(|kind| kind.is_file()) { + continue; + } + match std::fs::remove_file(entry.path()) { + Ok(()) => debug!( + "Swept a leftover marker temporary {}", + entry.path().display() + ), + Err(e) => debug!("Could not sweep {}: {e}", entry.path().display()), + } + } +} + +fn write_file_atomic(path: &Path, bytes: &[u8]) -> Result<()> { + let Some(dir) = path.parent() else { + return Err(Error::Storage(format!( + "Refusing to write {} — it has no parent directory", + path.display() + ))); + }; + let temp = dir.join(format!( + "{TEMP_PREFIX}{}.{:08x}.marker", + std::process::id(), + rand::random::() + )); + write_temp(&temp, bytes)?; + // Through the retry, because these small files (the layout marker, the migration + // state) are rewritten while the node runs, and on Windows a scanner holding a handle + // for a few milliseconds turns an ordinary rewrite into a hard failure. + rename_with_retry(&temp, path).map_err(|e| { + let _ = std::fs::remove_file(&temp); + Error::Storage(format!("Failed to publish {}: {e}", path.display())) + })?; + fsync_dir_best_effort(dir); + Ok(()) +} + +/// Read the layout marker, writing the current one if the store is new. +fn read_or_write_layout(chunks_dir: &Path) -> Result { + let path = chunks_dir.join(LAYOUT_FILE_NAME); + match read_small_file(&path) { + Ok(bytes) => serde_json::from_slice(&bytes).map_err(|e| { + Error::Storage(format!( + "Chunk store layout marker {} is unreadable: {e}. Refusing to open rather \ + than guess the layout.", + path.display() + )) + }), + Err(e) if e.kind() == ErrorKind::NotFound => { + if store_has_entries(chunks_dir) { + warn!( + "Chunk store at {} has data but no layout marker. Adopting it under \ + the current scheme, which is the only one this build implements. If \ + it was written by a build with a different layout its chunks will \ + appear to be missing.", + chunks_dir.display() + ); + } + let layout = StoreLayout::default(); + let bytes = serde_json::to_vec_pretty(&layout) + .map_err(|e| Error::Storage(format!("Failed to encode chunk store layout: {e}")))?; + write_file_atomic(&path, &bytes)?; + debug!("Wrote chunk store layout marker to {}", path.display()); + Ok(layout) + } + Err(e) => Err(Error::Storage(format!( + "Failed to read chunk store layout marker {}: {e}", + path.display() + ))), + } +} + +/// Whether the store directory already holds at least one shard. +fn store_has_entries(chunks_dir: &Path) -> bool { + let Ok(entries) = std::fs::read_dir(chunks_dir) else { + return false; + }; + entries.filter_map(std::result::Result::ok).any(|e| { + e.file_name() + .to_str() + .is_some_and(|n| n.len() == 2 && is_lower_hex(n)) + }) +} + +/// Largest a metadata marker may be before it is treated as corrupt. +const MAX_MARKER_BYTES: u64 = 64 * 1024; + +/// Read a small metadata file, refusing an implausibly large one. +/// +/// The chunk path is bounded for exactly this reason; the markers live in the same data +/// directory and deserve the same ceiling. +/// +/// # Errors +/// +/// Returns an I/O error, including `NotFound`, so callers can distinguish "no marker yet". +pub fn read_small_file(path: &Path) -> std::io::Result> { + let file = File::open(path)?; + let mut bytes = Vec::new(); + let read = file.take(MAX_MARKER_BYTES + 1).read_to_end(&mut bytes)?; + if read as u64 > MAX_MARKER_BYTES { + return Err(std::io::Error::other(format!( + "{} is larger than the {MAX_MARKER_BYTES} byte limit for a marker file", + path.display() + ))); + } + Ok(bytes) +} + +/// Take the store lock, or refuse to open the store. +/// +/// Both failures are refusals, deliberately. Unlike LMDB, which was genuinely +/// multi-process safe, two of these stores on one directory keep independent in-memory +/// indices, independent views of what is in flight, and independent opinions about +/// whether the legacy environment may be deleted: both would report the same write as +/// new and each would keep serving keys the other had deleted. A node that cannot create +/// the lock file has no way to know it is alone, and this is the one migration where +/// being wrong about that destroys data. +/// +/// The lock is an [`Arc`] so the work that relies on it can hold a lease. The startup +/// scan sweeps interrupted writes on the strength of being alone in the directory, and it +/// runs on a thread that outlives the future that started it. +/// +/// # Errors +/// +/// Returns [`Error::Storage`] when another process owns the directory, or when the lock +/// file cannot be created. +fn acquire_store_lock(chunks_dir: &Path) -> Result> { + let path = chunks_dir.join(LOCK_FILE_NAME); + let file = match OpenOptions::new() + .write(true) + .create(true) + .truncate(false) + .open(&path) + { + Ok(f) => f, + // Not a warning and carry on. Without this lock two processes can open the same + // directory, each with its own index, its own view of what is in flight, and its + // own opinion about whether the legacy environment may be deleted. A node that + // cannot take it has no way to know it is alone, and this is the one migration + // where being wrong about that destroys data. + Err(e) => { + return Err(Error::Storage(format!( + "Could not create the chunk store lock {}: {e}. Refusing to start: \ + without it this node cannot tell whether another is using the same data \ + directory. Fix the permissions on that path, or remove a stale lock file \ + left by a different user.", + path.display() + ))) + } + }; + match file.try_lock_exclusive() { + Ok(()) => Ok(Arc::new(file)), + Err(e) => Err(Error::Storage(format!( + "Another process already has the chunk store at {} open ({e}). Two nodes \ + cannot share one data directory: each keeps its own index and they would \ + disagree about what is stored. Stop the other node first.", + chunks_dir.display() + ))), + } +} + +/// What a startup scan found. +struct ScanResult { + /// Every published address, ascending. + keys: Vec, + /// Which shard directories already exist. + shards_present: [bool; SHARD_COUNT], + /// Orphaned temp files removed. + swept_temps: usize, + /// Entries that were neither a chunk nor one of ours. + skipped: usize, +} + +/// Rebuild the key set from directory entries. +/// +/// Reads names only. A `stat` per entry costs about ten times the enumeration on Linux +/// and macOS and fifty to sixty times on Windows, and buys nothing: the filename is the +/// key, and the content is verified on read. +fn scan_store(chunks_dir: &Path) -> Result { + let mut result = ScanResult { + keys: Vec::new(), + shards_present: [false; SHARD_COUNT], + swept_temps: 0, + skipped: 0, + }; + + let top = std::fs::read_dir(chunks_dir).map_err(|e| { + Error::Storage(format!( + "Failed to enumerate chunk store {}: {e}", + chunks_dir.display() + )) + })?; + + for entry in top { + let entry = entry.map_err(|e| { + Error::Storage(format!( + "Failed to read an entry of {}: {e}", + chunks_dir.display() + )) + })?; + let name = entry.file_name(); + let Some(name) = name.to_str() else { + result.skipped = result.skipped.saturating_add(1); + continue; + }; + if name == LAYOUT_FILE_NAME || name == LOCK_FILE_NAME { + continue; + } + if name.starts_with(TEMP_PREFIX) { + if sweep_temp(&entry.path()) { + result.swept_temps = result.swept_temps.saturating_add(1); + } + continue; + } + if name.len() != 2 || !is_lower_hex(name) { + warn!( + "Chunk store: ignoring unexpected entry {name} in {}", + chunks_dir.display() + ); + result.skipped = result.skipped.saturating_add(1); + continue; + } + let Ok(shard) = u8::from_str_radix(name, 16) else { + result.skipped = result.skipped.saturating_add(1); + continue; + }; + // `shards_present` is set inside `scan_shard`, on success only. Setting it from + // the name alone would make a stray regular file called `ab` look like a shard + // that already exists, and every write to that shard would then fail with a + // misleading error until the node was restarted. + scan_shard(&entry.path(), shard, &mut result)?; + } + + result.keys.sort_unstable(); + result.keys.dedup(); + Ok(result) +} + +/// Scan one shard directory into `result`. +fn scan_shard(dir: &Path, shard: u8, result: &mut ScanResult) -> Result<()> { + let entries = match std::fs::read_dir(dir) { + Ok(e) => e, + // A stray file named like a shard, or a directory removed between the two reads. + // Neither is fatal, and neither marks the shard as present. + Err(e) if matches!(e.kind(), ErrorKind::NotFound | ErrorKind::NotADirectory) => { + warn!( + "Chunk store: {} is not a shard directory ({e}); ignoring it", + dir.display() + ); + result.skipped = result.skipped.saturating_add(1); + return Ok(()); + } + // Anything else is a real fault: a permission problem, exhausted descriptors, or + // failing hardware. Opening with a shard's worth of keys silently missing would + // make the node under-claim in its published commitment and stop serving chunks + // it still holds and is answerable for, so refuse to open at all. + Err(e) => { + return Err(Error::Storage(format!( + "Failed to enumerate shard {}: {e}. Refusing to open with an incomplete \ + key set.", + dir.display() + ))) + } + }; + if let Some(slot) = result.shards_present.get_mut(shard as usize) { + *slot = true; + } + + for entry in entries { + let entry = + entry.map_err(|e| Error::Storage(format!("Failed to read {}: {e}", dir.display())))?; + let name = entry.file_name(); + let Some(name) = name.to_str() else { + result.skipped = result.skipped.saturating_add(1); + continue; + }; + if name.starts_with(TEMP_PREFIX) { + if sweep_temp(&entry.path()) { + result.swept_temps = result.swept_temps.saturating_add(1); + } + continue; + } + let Some(key) = decode_chunk_name(name) else { + if name.len() == CHUNK_NAME_LEN && is_hex_any_case(name) { + // A case-folded twin of a real chunk name. On NTFS and default APFS the + // existence check in the write path folds onto it, so a paid write would + // be answered "already stored" and its bytes dropped. Move it aside. + quarantine_entry(&entry.path()); + } else { + warn!( + "Chunk store: ignoring non-chunk entry {name} in {}", + dir.display() + ); + } + result.skipped = result.skipped.saturating_add(1); + continue; + }; + // `file_type` comes from the directory entry itself on Linux and macOS and from + // the enumeration on Windows, so this is not the per-entry `stat` the scan + // deliberately avoids. A pipe, socket, device or directory wearing a chunk name + // must never enter the index: nothing downstream can read it, and it would sit in + // the published commitment forever. + match entry.file_type() { + Ok(kind) if kind.is_file() => {} + Ok(_) => { + warn!( + "Chunk store: {name} in {} is not a regular file; ignoring it", + dir.display() + ); + result.skipped = result.skipped.saturating_add(1); + continue; + } + // Not the same as knowing it is not a file. Treating an unanswered question + // as a no would drop a real chunk from the index and from the commitment + // while its bytes sit on disk, and the node would not serve it again until + // some later restart happened to succeed. Fail the scan instead: an index + // that is missing keys must never be published as this node's key set. + Err(e) => { + return Err(Error::Storage(format!( + "Could not tell what {name} in {} is: {e}. Refusing to publish an \ + index that may be missing chunks.", + dir.display() + ))); + } + } + // A file in the wrong shard is unreachable through `chunk_path`, so indexing it + // would make the index claim a key the read path cannot find. + if shard_index(&key) != shard as usize { + warn!( + "Chunk store: {name} is filed under shard {shard:02x} but belongs in {:02x}; \ + ignoring it. Move it or delete it.", + shard_index(&key) + ); + result.skipped = result.skipped.saturating_add(1); + continue; + } + result.keys.push(key); + } + Ok(()) +} + +/// Remove one orphaned temp file. Returns whether it went. +/// +/// Always removed. The scan that calls this runs only after the store lock has been taken, +/// so by then any temp file is an interrupted write of a previous run and there is no other +/// process that could be writing it. This used to describe a second, gentler mode for the +/// unlocked case; there was never any such branch and there is no caller that would need +/// one. +fn sweep_temp(path: &Path) -> bool { + match std::fs::remove_file(path) { + Ok(()) => { + debug!("Removed orphaned temporary file {}", path.display()); + true + } + Err(e) => { + debug!("Could not remove {}: {e}", path.display()); + false + } + } +} + +/// Open a chunk file, refusing anything that is not a regular file. +/// +/// `Ok(None)` means the file is not there. A named pipe wearing a valid chunk name would +/// otherwise block the opening thread forever: `open` on a FIFO with no writer does not +/// return, and enough of them would exhaust the blocking pool and stall every file and +/// database operation in the process. `O_NOFOLLOW` refuses a symlink for the same reason, +/// and both are checked on the handle rather than the path, so nothing can be swapped +/// underneath between the check and the open. +fn open_regular(path: &Path) -> Result> { + #[cfg(unix)] + let opened = { + use std::os::unix::fs::OpenOptionsExt; + OpenOptions::new() + .read(true) + .custom_flags(libc::O_NONBLOCK | libc::O_NOFOLLOW) + .open(path) + }; + #[cfg(not(unix))] + let opened = OpenOptions::new().read(true).open(path); + + let file = match opened { + Ok(f) => f, + Err(e) if e.kind() == ErrorKind::NotFound => return Ok(None), + Err(e) => { + return Err(Error::Storage(format!( + "Failed to open chunk file {}: {e}", + path.display() + ))) + } + }; + let is_regular = file.metadata().is_ok_and(|m| m.file_type().is_file()); + if !is_regular { + return Err(Error::Storage(format!( + "{} is not a regular file; refusing to read it as a chunk", + path.display() + ))); + } + Ok(Some(file)) +} + +/// Read a chunk file, refusing anything larger than a chunk can legitimately be. +/// +/// A corrupt, sparse, or locally planted file wearing a valid 64-hex name would +/// otherwise be read straight into memory, so a single bad entry could exhaust the node +/// during an ordinary GET or an audit response. +fn read_bounded(file: File, path: &Path) -> Result> { + let ceiling = MAX_CHUNK_SIZE as u64; + let mut buf = Vec::new(); + let read = file.take(ceiling + 1).read_to_end(&mut buf).map_err(|e| { + Error::Storage(format!("Failed to read chunk file {}: {e}", path.display())) + })?; + if read as u64 > ceiling { + return Err(Error::Storage(format!( + "Chunk file {} is larger than the {ceiling} byte maximum; refusing to read it", + path.display() + ))); + } + Ok(buf) +} + +/// Whether a Windows error is one a scanner or indexer holding a handle would produce. +/// +/// `ERROR_ACCESS_DENIED`, `ERROR_SHARING_VIOLATION`, `ERROR_LOCK_VIOLATION`. Every other +/// failure is deterministic and retrying it only burns a blocking thread. +fn is_windows_sharing_violation(e: &std::io::Error) -> bool { + matches!(e.raw_os_error(), Some(5 | 32 | 33)) +} + +/// Publish `temp_path` as `final_path`, retrying a transient sharing violation. +/// +/// On Windows an antivirus scanner or the search indexer can hold a handle to either +/// file for a few milliseconds after it is created, and `MoveFileEx` fails outright +/// rather than queueing. Retrying a bounded number of times turns that from a failed +/// write into a short pause. Every other error returns immediately. +fn rename_with_retry(temp_path: &Path, final_path: &Path) -> std::io::Result<()> { + let mut last = match std::fs::rename(temp_path, final_path) { + Ok(()) => return Ok(()), + Err(e) => e, + }; + if !cfg!(windows) || !is_windows_sharing_violation(&last) { + return Err(last); + } + for attempt in 1..=RENAME_RETRY_ATTEMPTS { + std::thread::sleep(RENAME_RETRY_BACKOFF * attempt); + match std::fs::rename(temp_path, final_path) { + Ok(()) => return Ok(()), + Err(e) => last = e, + } + } + Err(last) +} + +/// Write `payload` and publish it as `final_path`, replacing whatever is there. +/// +/// Success here means the bytes are durable, not merely written. The repair path this +/// serves runs during the pre-retirement pass, where a chunk that fails to match its +/// address is rewritten from the legacy store and the legacy store is then deleted. A +/// replacement that a power loss can undo would leave that chunk with the wrong bytes and +/// no other copy. +fn write_and_replace( + temp_path: &Path, + final_path: &Path, + payload: &[u8], + shard: &Path, +) -> Result<()> { + // Unix: an intra-directory rename is atomic, so a reader sees the old content or the + // new one and never an absence, and the directory flush is what makes it durable. + #[cfg(unix)] + { + write_temp(temp_path, payload)?; + if let Err(e) = rename_with_retry(temp_path, final_path) { + let _ = std::fs::remove_file(temp_path); + return Err(Error::Storage(format!( + "Failed to replace chunk {}: {e}", + final_path.display() + ))); + } + fsync_dir(shard).map_err(|e| { + Error::Storage(format!( + "Replaced {} but could not flush {}: {e}. Not reporting the repair as \ + done, because a rewrite that is not durable must not authorise deleting \ + the copy it was rewritten from.", + final_path.display(), + shard.display() + )) + })?; + Ok(()) + } + // Everywhere else, Windows included: there is no way to flush a directory through the + // standard library, so a rename cannot be shown to be durable at return. Overwriting + // the existing file changes no directory entry at all, and `sync_all` (FlushFileBuffers + // on Windows) is documented to flush the file's data, so a successful return is + // durable under a documented contract. + // + // The cost is that this is not atomic: a crash part-way leaves the file holding a mix + // of old and new bytes. + // + // That used to be justified by the legacy store still being there to repair from, which + // it no longer is. The argument now is narrower and does not depend on a second copy: + // every caller reaches this only after a read has proven the bytes under that name + // wrong. A crash part-way therefore leaves wrong bytes where wrong bytes already were, + // which is not a loss, and the next verified read finds them and repairs again. What it + // is NOT safe for is replacing bytes that were good, so this must not be reached on any + // path that has not established otherwise. In this crate that holds: the two + // `StoredBytes::Wrong` arms get there from a read that hashed and disagreed, and + // `holds_verified` gets there from a read that returned bytes which were not the + // caller's. The public entry point makes no such check and says so. + // + // A temporary and a rename would make it atomic, at the cost of a directory entry + // change that cannot be flushed here. That trade is worth revisiting on a platform + // where it can actually be tested; it is not worth making blind. + #[cfg(not(unix))] + { + let _ = temp_path; + let _ = shard; + let mut file = OpenOptions::new() + .write(true) + .truncate(true) + .open(final_path) + .map_err(|e| { + Error::Storage(format!( + "Failed to open {} for replacement: {e}", + final_path.display() + )) + })?; + file.write_all(payload).map_err(|e| { + Error::Storage(format!("Failed to rewrite {}: {e}", final_path.display())) + })?; + file.sync_all().map_err(|e| { + Error::Storage(format!( + "Rewrote {} but could not flush it: {e}. Not reporting the repair as \ + done, because a rewrite that is not durable must not authorise deleting \ + the copy it was rewritten from.", + final_path.display() + )) + })?; + Ok(()) + } +} + +/// Create `temp_path`, write `payload` into it, and flush it. +/// +/// Flushed before any rename. On ext4 `auto_da_alloc` only orders the data before the +/// rename's own commit; it does not make the data durable, and btrfs has been observed +/// reordering. A name must never become visible on bytes that are not on the platter. +fn write_temp(temp_path: &Path, payload: &[u8]) -> Result<()> { + let mut f = OpenOptions::new() + .write(true) + .create_new(true) + .open(temp_path) + .map_err(|e| { + Error::Storage(format!( + "Failed to create temporary file {}: {e}", + temp_path.display() + )) + })?; + if let Err(e) = f.write_all(payload) { + let _ = std::fs::remove_file(temp_path); + return Err(Error::Storage(format!( + "Failed to write {}: {e}", + temp_path.display() + ))); + } + if let Err(e) = f.sync_all() { + let _ = std::fs::remove_file(temp_path); + return Err(Error::Storage(format!( + "Failed to flush {}: {e}", + temp_path.display() + ))); + } + Ok(()) +} + +/// Write `payload` and publish it under `final_path`. +/// +/// The temp lives in the destination directory, so the publish is an intra-directory +/// rename: atomic on every filesystem we support, and needing only that one directory +/// Put `payload` on disk as `final_path`, durably. +/// +/// Returns [`PutOutcome::Duplicate`] when the name is already taken. The name is a hash +/// of the content, so that is not treated as proof the bytes are right: the caller +/// re-reads and verifies them. +#[cfg(unix)] +fn publish( + temp_path: &Path, + final_path: &Path, + payload: &[u8], + shard: &Path, +) -> std::result::Result { + // On Unix nothing is ever created under the final name by a failing path: the bytes go + // to a temporary and only a successful rename gives them the real name. So every + // failure here leaves the name as it found it. + publish_via_rename(temp_path, final_path, payload, shard) + .map_err(PublishFailed::nothing_written) +} + +/// Put `payload` on disk as `final_path`, durably. See [`publish_in_place`] for why this +/// takes a different route off Unix. +#[cfg(not(unix))] +fn publish( + temp_path: &Path, + final_path: &Path, + payload: &[u8], + shard: &Path, +) -> std::result::Result { + let _ = temp_path; + let _ = shard; + publish_in_place(final_path, payload) +} + +/// Create the chunk under its final name and flush it. Everywhere but Unix. +/// +/// There is no way to flush a directory through the standard library, and Microsoft does +/// not document `MoveFileEx` as durable at return unless it is called with +/// `MOVEFILE_WRITE_THROUGH`, which std does not use. So off Unix a rename cannot be +/// relied on to have reached the disk before the legacy store is deleted. +/// +/// Creating the file under its final name sidesteps the rename entirely. Microsoft +/// documents that creation metadata is cached and that `FlushFileBuffers`, which +/// `sync_all` calls on Windows, is the way to flush it. So a successful create, write and +/// flush is a durable publication under a documented contract, with no directory flush +/// and no rename involved. +/// +/// The cost is that a crash mid-write leaves a partial file wearing a real chunk name. +/// That is why a duplicate re-reads and verifies rather than trusting the name, and why +/// the pre-retirement pass re-hashes everything before anything is deleted. +#[cfg(not(unix))] +fn publish_in_place( + final_path: &Path, + payload: &[u8], +) -> std::result::Result { + // Test-only, and here rather than after the write so that it means the same thing on + // both platforms: the file half of a dual write has not happened yet. On Unix the + // equivalent point is the temporary file written and the rename not yet made, which is + // also before the chunk's name exists on disk. Stopping after the write instead would + // put the file under its real name already, so a crash there is not between the two + // halves at all, and it could not demonstrate anything about the missing flush either: + // killing a process does not empty the page cache, so the bytes are still there to be + // read. Only losing power loses them, which no test that kills a process can stage. + #[cfg(any(test, feature = "test-utils"))] + halt_here_if_asked(HALT_BEFORE_PUBLISH, final_path); + let mut file = match OpenOptions::new() + .write(true) + .create_new(true) + .open(final_path) + { + Ok(f) => f, + // Someone got there first. Immutable content under a content-addressed name, so + // the caller verifies what is already there rather than assuming it is right. + Err(e) if e.kind() == ErrorKind::AlreadyExists => return Ok(PutOutcome::Duplicate), + Err(e) => { + // Nothing was created, so nothing was spent. + return Err(PublishFailed::nothing_written(Error::Storage(format!( + "Failed to create chunk {}: {e}", + final_path.display() + )))); + } + }; + if let Err(e) = file.write_all(payload) { + drop(file); + // Taken back if it can be. Whether it could is what the caller needs: the file + // was created by this call, so if it is still there the space is spent. + let left_behind = std::fs::remove_file(final_path).is_err(); + return Err(PublishFailed { + error: Error::Storage(format!("Failed to write {}: {e}", final_path.display())), + left_behind, + }); + } + if let Err(e) = file.sync_all() { + drop(file); + // Taken back if it can be. Whether it could is what the caller needs: the file + // was created by this call, so if it is still there the space is spent. + let left_behind = std::fs::remove_file(final_path).is_err(); + return Err(PublishFailed { + error: Error::Storage(format!("Failed to flush {}: {e}", final_path.display())), + left_behind, + }); + } + Ok(PutOutcome::New) +} + +/// Write a temp beside the target and rename it into place. Unix only. +/// +/// Places the bytes and nothing more. Making the name durable is +/// [`flush_publication`]'s job, kept separate so a caller can tell a publish that spent no +/// space from one that spent it and could not be reported. +#[cfg(unix)] +fn publish_via_rename( + temp_path: &Path, + final_path: &Path, + payload: &[u8], + _shard: &Path, +) -> Result { + // Content is immutable and the name is its hash, so an existing file already holds + // exactly these bytes. Skipping the write is both cheaper and safer than replacing + // it: on Windows a rename over a file another thread has open fails outright. + // + // The caller flushes either way. A name that is already there is not proof it is + // durable: + // the write that put it there may have been this store's own previous attempt, whose + // rename landed and whose directory flush then failed. That attempt returned an + // error, so nothing was retired on the strength of it, but if this call reported a + // durable duplicate without flushing, the retry would silently launder an unflushed + // rename into a copy that authorises deleting the last other one. + let outcome = if final_path.exists() { + PutOutcome::Duplicate + } else { + write_temp(temp_path, payload)?; + // Test-only: the one moment a complete chunk exists on disk under a name nothing + // looks for. A crash test needs to die at a named point rather than wherever a + // sleep in another process happened to land. + #[cfg(any(test, feature = "test-utils"))] + halt_here_if_asked(HALT_BEFORE_PUBLISH, temp_path); + match rename_with_retry(temp_path, final_path) { + Ok(()) => PutOutcome::New, + Err(e) => { + let _ = std::fs::remove_file(temp_path); + // Another writer of the same address won the race, or the destination was + // open. Either way the bytes are already published. + if !final_path.exists() { + return Err(Error::Storage(format!( + "Failed to publish chunk {}: {e}", + final_path.display() + ))); + } + PutOutcome::Duplicate + } + } + }; + + Ok(outcome) +} + +/// A publish that failed, and whether it left its bytes on the disk. +/// +/// The second half is the point. A failure before anything was created has spent nothing; +/// one that created the file and then could not remove it again has spent the space, and +/// whoever is accounting for free space has to know which happened. Only the code that did +/// the creating can say. +struct PublishFailed { + error: Error, + left_behind: bool, +} + +impl PublishFailed { + /// A failure that created nothing. + fn nothing_written(error: Error) -> Self { + Self { + error, + left_behind: false, + } + } +} + +/// Make a publication durable by flushing the directory its name lives in. +/// +/// Separate from placing the bytes, because the caller has to tell the two failures apart. +/// A publish that fails before the bytes land has spent nothing; one that fails here has +/// spent the space and must not be reported as stored, so whoever is accounting for free +/// space has to charge it while whoever is accounting for chunks must not count it. +/// +/// NOT best effort. The directory flush is what makes the rename durable, and a copy +/// reported successful is what authorises deleting the only other copy. Swallowing the +/// failure would let a power loss discard the directory entry after the legacy store had +/// already been removed. +/// +/// # Errors +/// +/// Returns [`Error::Storage`] if the directory cannot be flushed. +#[cfg(unix)] +fn flush_publication(final_path: &Path, shard: &Path) -> Result<()> { + fsync_dir(shard).map_err(|e| { + Error::Storage(format!( + "Published {} but could not flush {}: {e}. Not reporting this chunk as stored, \ + because a copy that is not durable must not authorise deleting another.", + final_path.display(), + shard.display() + )) + }) +} + +/// Nothing to do off Unix, where the chunk is created under its final name and flushed +/// with `sync_all`, which is documented to carry its creation metadata with it, and where +/// there is no way to flush a directory at all. +#[cfg(not(unix))] +fn flush_publication(_final_path: &Path, _shard: &Path) -> Result<()> { + Ok(()) +} + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +mod tests { + use super::*; + use std::collections::HashSet; + + /// A directory flush that fails must say so. + /// + /// The quiet version of this function is only used where the answer does not change + /// what happens next. On the publish path it does. + #[cfg(unix)] + #[test] + fn flushing_a_directory_that_is_not_there_reports_the_failure() { + let dir = TempDir::new().expect("temp dir"); + assert!(fsync_dir(dir.path()).is_ok()); + assert!(fsync_dir(&dir.path().join("no-such-shard")).is_err()); + } + + /// A chunk whose directory entry was never flushed is not reported as stored. + /// + /// This is the whole safety argument for retirement: the legacy store is deleted + /// because every chunk was copied durably. A published file whose directory flush + /// failed can vanish on power loss, so counting it as copied would lose data. The + /// file staying on disk afterwards is fine, the next pass republishes it. + #[cfg(unix)] + #[test] + fn a_publish_whose_directory_flush_fails_is_not_reported_as_stored() { + let dir = TempDir::new().expect("temp dir"); + let temp_path = dir.path().join("chunk.tmp"); + let final_path = dir.path().join("chunk"); + let unflushable = dir.path().join("shard-that-does-not-exist"); + + // Asserted in two steps, not chained. Chaining them means a regression in placing + // the bytes also produces an error, and the test passes without the flush ever + // being reached: it would be checking that something went wrong rather than that + // this went wrong. + let placed = publish_via_rename(&temp_path, &final_path, b"payload", &unflushable); + assert!( + placed.is_ok(), + "the bytes must be placed before this can be about the flush: {:?}", + placed.err() + ); + let outcome = flush_publication(&final_path, &unflushable); + + assert!( + outcome.is_err(), + "an unflushed publication must not be reported as stored" + ); + assert!( + !temp_path.exists(), + "the temp file must not be left behind either way" + ); + } + + use tempfile::TempDir; + + /// Open a store on a fresh temp directory with the disk reserve disabled. + async fn test_store() -> (ChunkStore, TempDir) { + let dir = TempDir::new().expect("temp dir"); + let store = ChunkStore::new(ChunkStoreConfig { + root_dir: dir.path().to_path_buf(), + verify_on_read: true, + disk_reserve: 0, + }) + .await + .expect("open store"); + (store, dir) + } + + /// Open a store on an existing directory, as a restart would. + async fn reopen(dir: &TempDir) -> ChunkStore { + ChunkStore::new(ChunkStoreConfig { + root_dir: dir.path().to_path_buf(), + verify_on_read: true, + disk_reserve: 0, + }) + .await + .expect("reopen store") + } + + /// An ordinary read settles whether the node answers for a chunk. + /// + /// Not only the reads that were checking something. A read that failed means the + /// chunk cannot be served, whoever asked; a read that worked means it can be. Deciding + /// this anywhere else leaves a key stuck unadvertised after the fault has cleared, or + /// advertised after it has not. + #[cfg(unix)] + #[tokio::test] + async fn an_ordinary_read_decides_whether_the_node_answers_for_a_chunk() { + use std::os::unix::fs::PermissionsExt; + + let (store, dir) = test_store().await; + let (addr, content) = addressed("read-decides"); + store.put(&addr, &content).await.expect("put"); + let path = store.chunk_path(&addr); + + let mut perms = std::fs::metadata(&path).expect("meta").permissions(); + perms.set_mode(0o000); + std::fs::set_permissions(&path, perms).expect("chmod"); + + assert!(store.get(&addr).await.is_err(), "the read must fail"); + assert!( + !store.exists(&addr).expect("exists"), + "and a plain read that failed must stop the node answering for it" + ); + + let mut perms = std::fs::metadata(&path).expect("meta").permissions(); + perms.set_mode(0o600); + std::fs::set_permissions(&path, perms).expect("chmod back"); + + assert_eq!( + store.get(&addr).await.expect("get").expect("present"), + content + ); + assert!( + store.exists(&addr).expect("exists"), + "and a plain read that worked must start it answering again" + ); + drop(dir); + } + + /// Two writes for one key: waiting means waiting for both. + /// + /// Cancellation releases the caller's lane while the blocking half survives, so a + /// second write for the same key can start behind the first. If the registry only + /// recorded that *something* was writing, whichever finished first would clear it and + /// a delete would be told the key was free while the other was still queued, then be + /// undone by it. + #[tokio::test] + async fn waiting_for_a_key_waits_for_every_write_of_it() { + let (store, dir) = test_store().await; + let store = Arc::new(store); + let (addr, content) = addressed("two-writers"); + + // Two registrations, as two overlapping writes would make. + let first = store.begin_write(&addr); + let second = store.begin_write(&addr); + + let waiting = { + let store = Arc::clone(&store); + tokio::spawn(async move { store.wait_for_write(&addr).await }) + }; + tokio::time::sleep(Duration::from_millis(50)).await; + assert!(!waiting.is_finished()); + + // One finishes. The other has not, so the wait must continue. + drop(first); + tokio::time::sleep(Duration::from_millis(50)).await; + assert!( + !waiting.is_finished(), + "one write finishing does not mean the key is free" + ); + + drop(second); + waiting + .await + .expect("the wait ends once both have finished"); + + // And the store is still usable afterwards. + store.put(&addr, &content).await.expect("put"); + assert!(store.exists(&addr).expect("exists")); + drop(dir); + } + + /// A chunk this store cannot read is kept but not claimed. + /// + /// Both halves matter. Deleting it, or dropping it from the index, is how a chunk ends + /// up in neither this store's view nor the legacy one, which is what retirement + /// destroys. Claiming it anyway puts the key in signed commitments and answers + /// presence probes with a yes for a chunk the node cannot serve, and the audit that + /// catches that still penalises. + #[cfg(unix)] + #[tokio::test] + async fn a_chunk_that_cannot_be_read_is_kept_but_not_claimed() { + use std::os::unix::fs::PermissionsExt; + + let (store, dir) = test_store().await; + let (addr, content) = addressed("unreadable-for-now"); + store.put(&addr, &content).await.expect("put"); + assert!(store.exists(&addr).expect("exists")); + + let path = store.chunk_path(&addr); + let mut perms = std::fs::metadata(&path).expect("meta").permissions(); + perms.set_mode(0o000); + std::fs::set_permissions(&path, perms).expect("chmod"); + + // Offering the same bytes again must not be acknowledged, and must not replace + // what is there on the strength of a read that did not happen. + assert!( + store.put(&addr, &content).await.is_err(), + "an unreadable chunk must not be reported as stored" + ); + assert!(path.exists(), "and the file must be left alone"); + assert!( + !store.exists(&addr).expect("exists"), + "but the node must stop claiming it" + ); + assert!(!store.all_keys().await.expect("keys").contains(&addr)); + + // Readable again: the node answers for it once more. + let mut perms = std::fs::metadata(&path).expect("meta").permissions(); + perms.set_mode(0o600); + std::fs::set_permissions(&path, perms).expect("chmod back"); + assert!(!store.put(&addr, &content).await.expect("put again")); + assert!(store.exists(&addr).expect("exists")); + assert!(store.all_keys().await.expect("keys").contains(&addr)); + drop(dir); + } + + /// Content plus the address it hashes to. + fn addressed(seed: &str) -> (XorName, Vec) { + let content = format!("chunk-content-{seed}").into_bytes(); + (crate::client::compute_address(&content), content) + } + + #[tokio::test] + async fn put_then_get_returns_the_same_bytes() { + let (store, _dir) = test_store().await; + let (addr, content) = addressed("a"); + + assert!(store.put(&addr, &content).await.expect("put")); + let got = store.get(&addr).await.expect("get").expect("present"); + assert_eq!(got, content); + } + + #[tokio::test] + async fn a_second_put_of_the_same_chunk_reports_not_new() { + let (store, _dir) = test_store().await; + let (addr, content) = addressed("b"); + + assert!(store.put(&addr, &content).await.expect("first put")); + assert!(!store.put(&addr, &content).await.expect("second put")); + assert_eq!(store.current_chunks().expect("count"), 1); + assert_eq!(store.stats().duplicates, 1); + } + + #[tokio::test] + async fn get_of_an_unknown_address_is_none() { + let (store, _dir) = test_store().await; + let (addr, _) = addressed("missing"); + assert!(store.get(&addr).await.expect("get").is_none()); + } + + #[tokio::test] + async fn exists_tracks_the_store() { + let (store, _dir) = test_store().await; + let (addr, content) = addressed("c"); + + assert!(!store.exists(&addr).expect("exists")); + store.put(&addr, &content).await.expect("put"); + assert!(store.exists(&addr).expect("exists")); + store.delete(&addr).await.expect("delete"); + assert!(!store.exists(&addr).expect("exists")); + } + + #[tokio::test] + async fn delete_unlinks_the_file_and_returns_the_space() { + let (store, _dir) = test_store().await; + let (addr, content) = addressed("d"); + store.put(&addr, &content).await.expect("put"); + + let path = store.chunk_path(&addr); + assert!(path.exists(), "the chunk file should be on disk"); + + assert!(store.delete(&addr).await.expect("delete")); + assert!(!path.exists(), "delete must actually unlink the file"); + assert_eq!(store.current_chunks().expect("count"), 0); + + // Deleting again is a no-op that reports nothing was there. + assert!(!store.delete(&addr).await.expect("second delete")); + } + + #[tokio::test] + async fn content_that_does_not_hash_to_its_address_is_rejected() { + let (store, _dir) = test_store().await; + let (addr, _) = addressed("e"); + let err = store + .put(&addr, b"different content") + .await + .expect_err("must reject"); + assert!( + format!("{err}").contains("Content address mismatch"), + "unexpected error: {err}" + ); + assert_eq!(store.current_chunks().expect("count"), 0); + } + + #[tokio::test] + async fn a_chunk_is_filed_under_the_last_two_hex_characters_of_its_address() { + let (store, dir) = test_store().await; + let (addr, content) = addressed("f"); + store.put(&addr, &content).await.expect("put"); + + let name = hex::encode(addr); + let expected_shard = name + .get(name.len() - 2..) + .expect("64-character name") + .to_string(); + let path = dir + .path() + .join(CHUNKS_DIR_NAME) + .join(&expected_shard) + .join(&name); + assert!(path.exists(), "expected the chunk at {}", path.display()); + } + + #[tokio::test] + async fn the_index_is_rebuilt_from_the_filesystem_on_restart() { + let (store, dir) = test_store().await; + let mut written = Vec::new(); + for i in 0..64 { + let (addr, content) = addressed(&format!("restart-{i}")); + store.put(&addr, &content).await.expect("put"); + written.push(addr); + } + drop(store); + + let reopened = reopen(&dir).await; + assert_eq!(reopened.current_chunks().expect("count"), 64); + for addr in &written { + assert!(reopened.exists(addr).expect("exists"), "lost a key"); + } + } + + #[tokio::test] + async fn all_keys_is_sorted_ascending() { + let (store, dir) = test_store().await; + for i in 0..128 { + let (addr, content) = addressed(&format!("sorted-{i}")); + store.put(&addr, &content).await.expect("put"); + } + + let keys = store.all_keys().await.expect("all_keys"); + let mut sorted = keys.clone(); + sorted.sort_unstable(); + assert_eq!(keys, sorted, "all_keys() must be ordered"); + + // And the order has to survive a restart, because the commitment builder + // truncates the responsible subset before the Merkle tree sorts it. + drop(store); + let reopened = reopen(&dir).await; + assert_eq!(reopened.all_keys().await.expect("all_keys"), keys); + } + + #[tokio::test] + async fn get_raw_skips_verification() { + let (store, _dir) = test_store().await; + let (addr, content) = addressed("raw"); + store.put(&addr, &content).await.expect("put"); + + // Corrupt the file behind the store's back. + std::fs::write(store.chunk_path(&addr), b"tampered").expect("tamper"); + + let raw = store.get_raw(&addr).await.expect("get_raw").expect("bytes"); + assert_eq!(raw, b"tampered"); + } + + /// A delete waits for a put that is already under way for the same key. + /// + /// The narrower ordering, and the one waiting for registered writes does not cover. A + /// put does a lot before it registers itself: it checks the address, reads to see + /// whether the name is taken, and reserves capacity. A delete arriving in that window + /// would see nothing registered, wait for nothing, and go ahead; the put would register + /// and publish afterwards, and the node would keep a chunk it had decided to prune. + /// + /// Staged deterministically rather than by racing two tasks. A put parked inside its + /// own closure is holding the key's lane, so the delete must not be able to finish + /// while it is parked. An earlier version of this test started both and accepted either + /// ordering, which the bug also satisfies: it proved nothing. + #[tokio::test] + #[allow(clippy::await_holding_lock)] + async fn a_delete_waits_for_a_put_already_under_way() { + let dir = TempDir::new().expect("temp dir"); + let store = Arc::new(reopen(&dir).await); + let content = b"a chunk the pruner has decided to drop".to_vec(); + let addr = crate::client::compute_address(&content); + + // Park the put in the window that matters: it has taken the key's lane and has NOT + // yet registered itself, so a delete's wait for in-flight writes would see nothing. + // Parking it later, inside its closure, cannot show the lane doing anything: the + // delete would block on the wait instead and the test would pass either way. + let gate = store.test_pre_registration_gate(); + let held = gate.write().await; + let writing = { + let store = Arc::clone(&store); + let content = content.clone(); + tokio::spawn(async move { store.put(&addr, &content).await }) + }; + // Waited for, not slept at. A sleep makes the staging a guess, and on a loaded + // machine the guess is wrong and the test fails for the wrong reason. + let deadline = std::time::Instant::now() + Duration::from_secs(30); + while store.test_reached_pre_registration() == 0 { + assert!( + std::time::Instant::now() < deadline, + "the put never reached the gate" + ); + tokio::time::sleep(Duration::from_millis(5)).await; + } + + // The delete must not get past the lane while that put holds it. Without the lane + // on `put` this finishes immediately, which is the regression. + let deleting = { + let store = Arc::clone(&store); + tokio::spawn(async move { store.delete(&addr).await }) + }; + tokio::time::sleep(Duration::from_millis(200)).await; + assert!( + !deleting.is_finished(), + "the delete finished while a put for the same key was still under way" + ); + + // Released: the put completes, then the delete runs. The delete is the later + // decision, so the chunk must be gone. + drop(held); + let _ = writing.await.expect("the put task must not panic"); + let _ = deleting.await.expect("the delete task must not panic"); + store.wait_idle().await; + + assert!( + !store.is_indexed(&addr), + "the put landed after the delete and put {} back", + hex::encode(addr) + ); + assert!( + !store.chunk_path(&addr).exists(), + "and left its file behind" + ); + } + + /// A delete outlasts a write nobody waited for. + /// + /// A write's blocking half outlives the future that started it, deliberately, so the + /// work is never left half done. That means a cancelled put can still be queued when a + /// delete arrives, and if the delete does not wait for it the write lands afterwards + /// and puts back a chunk the node had decided to prune. The key is then in a store that + /// no longer claims it, which is what the next verification has to clean up. + /// + /// This ordering had a regression test before the migration facade was deleted, and the + /// test went with the facade even though the requirement did not. + #[tokio::test] + // The gate is held across awaits deliberately: holding it is what parks the put, which + // is the state the delete has to be ordered against. + #[allow(clippy::await_holding_lock)] + async fn a_delete_outlasts_a_write_nobody_waited_for() { + let dir = TempDir::new().expect("temp dir"); + let store = Arc::new(reopen(&dir).await); + let content = b"a chunk that is about to be pruned".to_vec(); + let addr = crate::client::compute_address(&content); + + // Park the put inside its closure, then drop the future waiting on it. + let gate = store.test_put_gate(); + let held = gate.write(); + let put = { + let store = Arc::clone(&store); + let content = content.clone(); + tokio::spawn(async move { store.put(&addr, &content).await }) + }; + let deadline = std::time::Instant::now() + Duration::from_secs(30); + while store.tasks_in_flight() == 0 { + assert!( + std::time::Instant::now() < deadline, + "the put never reached the closure" + ); + tokio::time::sleep(Duration::from_millis(5)).await; + } + put.abort(); + let _ = put.await; + + // The delete must not finish ahead of that write. Released after the delete has + // had time to be waiting, so if it does not wait, it wins the race and the test + // catches it. + let deleting = { + let store = Arc::clone(&store); + tokio::spawn(async move { store.delete(&addr).await }) + }; + tokio::time::sleep(Duration::from_millis(100)).await; + drop(held); + let _ = deleting + .await + .expect("the delete task itself must not fail"); + store.wait_idle().await; + + assert!( + !store.is_indexed(&addr), + "the write landed after the delete and put {} back", + hex::encode(addr) + ); + assert!( + !store.chunk_path(&addr).exists(), + "and left its file on disk" + ); + } + + /// A put whose caller goes away does not admit a key on bytes nothing has read. + /// + /// The blocking half of a put outlives the future that started it, deliberately, so + /// the work is never left half done. That makes anything it writes to memory a claim + /// the node keeps whether or not the caller is still there to finish checking it. + /// + /// For a chunk this call published the claim is earned: the bytes were hashed against + /// their own name on the way in. For a name that was already taken it is not. The + /// check that decides whether those bytes are good runs after the await, and a dropped + /// future skips it, so admitting the key in the closure claims a chunk nobody read. + /// + /// Staged with a fifo, which is the sharpest case and a real one: the startup scan + /// refuses non-regular entries by design, so this is a key the store has already + /// decided it must not claim, walked in through the back door. + #[cfg(unix)] + #[tokio::test] + // The gate is held across an await deliberately: holding it is what parks the put + // inside its closure, which is the state under test. Dropping it before awaiting would + // let the put finish and there would be nothing to cancel. + #[allow(clippy::await_holding_lock)] + async fn a_cancelled_put_does_not_admit_a_key_whose_bytes_were_never_read() { + let dir = TempDir::new().expect("temp dir"); + let store = Arc::new(reopen(&dir).await); + + // A name a real chunk would use, wearing something that is not a chunk. + let content = b"the bytes that belong under this name".to_vec(); + let addr = crate::client::compute_address(&content); + let shard = dir + .path() + .join(CHUNKS_DIR_NAME) + .join(format!("{:02x}", addr[31])); + std::fs::create_dir_all(&shard).expect("mkdir"); + let path = shard.join(hex::encode(addr)); + let name = std::ffi::CString::new(path.as_os_str().as_encoded_bytes()) + .expect("a path with no interior nul"); + // SAFETY: `name` is a valid NUL-terminated C string that outlives the call, and the + // mode is a constant. `mkfifo` reads the pointer and returns; nothing is retained. + #[allow(clippy::undocumented_unsafe_blocks, unsafe_code)] + let made = unsafe { libc::mkfifo(name.as_ptr(), 0o644) }; + assert_eq!(made, 0, "could not make the fifo this test needs"); + + // Hold the gate so the put parks inside the closure, then drop the future while it + // is parked. That is a caller going away mid-put, which is what a cancelled + // request, a client disconnect or a shutdown all look like from in here. + let gate = store.test_put_gate(); + let held = gate.write(); + let put = { + let store = Arc::clone(&store); + let content = content.clone(); + tokio::spawn(async move { store.put(&addr, &content).await }) + }; + // Waited for rather than slept at. A sleep proves nothing: if the put had not + // reached the gated closure yet, aborting would cancel it before it ever got + // there and the test would pass having staged nothing. + let deadline = std::time::Instant::now() + Duration::from_secs(30); + while store.tasks_in_flight() == 0 { + assert!( + std::time::Instant::now() < deadline, + "the put never reached the closure, so there was nothing to cancel" + ); + tokio::time::sleep(Duration::from_millis(5)).await; + } + put.abort(); + let _ = put.await; + drop(held); + store.wait_idle().await; + + assert!( + !store.is_indexed(&addr), + "a cancelled put admitted {} on bytes nothing read; the fifo under that name \ + would then be advertised, committed to, and audited against", + hex::encode(addr) + ); + assert!( + !store.exists(&addr).unwrap_or(true), + "and the node must not claim it either" + ); + } + + /// A marker temporary left in the node root is swept, and nothing else is. + /// + /// The migration marker is written next to itself in the root, which no sweep looked + /// at, so a crash between its write and its rename left one there for the life of the + /// node. Small, but nothing was ever going to remove it. + /// + /// The second half is the point: this runs over a directory holding a node's data, so + /// it has to take only the exact shape this module writes and leave everything else + /// where it is. + #[tokio::test] + async fn a_leftover_marker_temporary_is_swept_and_its_neighbours_are_not() { + let dir = TempDir::new().expect("temp dir"); + let root = dir.path(); + let leftover = root.join(format!("{TEMP_PREFIX}1234.abcdef01.marker")); + std::fs::write(&leftover, b"an interrupted marker write").expect("plant"); + + // Things that must survive: the marker itself, a chunk-shaped temp that belongs to + // the chunk tree's own sweep, and anything an operator put there. + let keep = [ + root.join("migration-state.json"), + root.join(format!("{TEMP_PREFIX}1234.abcdef01.chunk")), + root.join("notes.txt"), + // Prefix and suffix alone would take these. The pid and the nonce are checked + // because this runs over a directory holding a node's data. + root.join(format!("{TEMP_PREFIX}operator-notes.marker")), + root.join(format!("{TEMP_PREFIX}1234.nothex01.marker")), + root.join(format!("{TEMP_PREFIX}1234.abcdef0.marker")), + root.join(format!("{TEMP_PREFIX}1234.abcdef01.extra.marker")), + ]; + for path in &keep { + std::fs::write(path, b"keep me").expect("plant"); + } + + let store = reopen(&dir).await; + drop(store); + + assert!( + !leftover.exists(), + "the leftover marker temporary is still in the node root" + ); + for path in &keep { + assert!( + path.exists(), + "{} was swept and should not have been", + path.display() + ); + } + } + + #[tokio::test] + async fn a_corrupt_chunk_is_removed_so_replication_can_repair_it() { + let (store, _dir) = test_store().await; + let (addr, content) = addressed("corrupt"); + store.put(&addr, &content).await.expect("put"); + std::fs::write(store.chunk_path(&addr), b"tampered").expect("tamper"); + + let err = store.get(&addr).await.expect_err("verification must fail"); + assert!(format!("{err}").contains("verification failed"), "{err}"); + + assert!(!store.chunk_path(&addr).exists(), "corrupt file must go"); + assert!(!store.exists(&addr).expect("exists")); + assert!( + !store.all_keys().await.expect("all_keys").contains(&addr), + "a corrupt chunk must stop being advertised" + ); + assert_eq!(store.stats().verification_failures, 1); + } + + #[tokio::test] + async fn a_file_removed_underneath_the_store_drops_out_of_the_index() { + let (store, _dir) = test_store().await; + let (addr, content) = addressed("vanished"); + store.put(&addr, &content).await.expect("put"); + + std::fs::remove_file(store.chunk_path(&addr)).expect("remove behind our back"); + + assert!(store.get(&addr).await.expect("get").is_none()); + assert!(!store.exists(&addr).expect("exists")); + assert_eq!(store.current_chunks().expect("count"), 0); + } + + #[tokio::test] + async fn interrupted_writes_are_swept_at_startup() { + let (store, dir) = test_store().await; + let (addr, content) = addressed("sweep"); + store.put(&addr, &content).await.expect("put"); + let shard = store + .chunk_path(&addr) + .parent() + .expect("shard") + .to_path_buf(); + drop(store); + + let orphan = shard.join(format!("{TEMP_PREFIX}999.7")); + std::fs::write(&orphan, b"half a chunk").expect("write orphan"); + let stray_root = dir + .path() + .join(CHUNKS_DIR_NAME) + .join(format!("{TEMP_PREFIX}999.8")); + std::fs::write(&stray_root, b"half a marker").expect("write stray"); + + let reopened = reopen(&dir).await; + assert!(!orphan.exists(), "an interrupted write must not survive"); + assert!(!stray_root.exists(), "nor one at the store root"); + assert_eq!(reopened.current_chunks().expect("count"), 1); + } + + #[tokio::test] + async fn concurrent_writers_of_one_address_store_it_exactly_once() { + let (store, _dir) = test_store().await; + let store = Arc::new(store); + let (addr, content) = addressed("racing"); + + let mut tasks = Vec::new(); + for _ in 0..16 { + let store = Arc::clone(&store); + let content = content.clone(); + tasks.push(tokio::spawn( + async move { store.put(&addr, &content).await }, + )); + } + + let mut new_count = 0; + for task in tasks { + if task.await.expect("join").expect("put") { + new_count += 1; + } + } + assert_eq!(new_count, 1, "exactly one writer may report a new chunk"); + assert_eq!(store.current_chunks().expect("count"), 1); + assert_eq!( + store.get(&addr).await.expect("get").expect("present"), + content + ); + } + + #[tokio::test] + async fn names_that_are_not_lowercase_hex_are_ignored_by_the_scan() { + let (store, dir) = test_store().await; + let (addr, content) = addressed("scan"); + store.put(&addr, &content).await.expect("put"); + let shard = store + .chunk_path(&addr) + .parent() + .expect("shard") + .to_path_buf(); + drop(store); + + // Uppercase is deliberately rejected: on a case-folding filesystem accepting it + // would let one file answer to two index entries. + let upper = shard.join(hex::encode_upper(addressed("upper").0)); + std::fs::write(&upper, b"x").expect("write upper"); + std::fs::write(shard.join("not-a-chunk"), b"x").expect("write junk"); + std::fs::write(shard.join("deadbeef"), b"x").expect("write short"); + + let reopened = reopen(&dir).await; + assert_eq!(reopened.current_chunks().expect("count"), 1); + } + + #[tokio::test] + async fn a_chunk_filed_in_the_wrong_shard_is_not_indexed() { + let (store, dir) = test_store().await; + let (addr, content) = addressed("misfiled"); + store.put(&addr, &content).await.expect("put"); + drop(store); + + // Move it one shard over: the read path would never find it there, so indexing + // it would make the store advertise a key it cannot serve. + let correct = dir + .path() + .join(CHUNKS_DIR_NAME) + .join(shard_name(&addr)) + .join(hex::encode(addr)); + let wrong_shard_index = (shard_index(&addr) + 1) % SHARD_COUNT; + let wrong_dir = dir + .path() + .join(CHUNKS_DIR_NAME) + .join(format!("{wrong_shard_index:02x}")); + std::fs::create_dir_all(&wrong_dir).expect("mkdir"); + std::fs::rename(&correct, wrong_dir.join(hex::encode(addr))).expect("misfile"); + + let reopened = reopen(&dir).await; + assert_eq!(reopened.current_chunks().expect("count"), 0); + assert!(!reopened.exists(&addr).expect("exists")); + } + + #[tokio::test] + async fn the_layout_marker_is_written_once_and_checked_on_reopen() { + let (store, dir) = test_store().await; + drop(store); + + let marker = dir.path().join(CHUNKS_DIR_NAME).join(LAYOUT_FILE_NAME); + let layout: StoreLayout = + serde_json::from_slice(&std::fs::read(&marker).expect("read marker")) + .expect("parse marker"); + assert_eq!(layout, StoreLayout::default()); + + // A store written by a future build must be refused, not misread. + let future = StoreLayout { + schema: LAYOUT_SCHEMA + 1, + ..StoreLayout::default() + }; + std::fs::write(&marker, serde_json::to_vec(&future).expect("encode")).expect("write"); + let err = ChunkStore::new(ChunkStoreConfig { + root_dir: dir.path().to_path_buf(), + verify_on_read: true, + disk_reserve: 0, + }) + .await + .expect_err("must refuse a newer layout"); + assert!(format!("{err}").contains("newer than this build"), "{err}"); + } + + #[tokio::test] + async fn an_unknown_shard_scheme_is_refused() { + let (store, dir) = test_store().await; + drop(store); + let marker = dir.path().join(CHUNKS_DIR_NAME).join(LAYOUT_FILE_NAME); + let other = StoreLayout { + scheme: "prefix-hex".to_string(), + ..StoreLayout::default() + }; + std::fs::write(&marker, serde_json::to_vec(&other).expect("encode")).expect("write"); + let err = ChunkStore::new(ChunkStoreConfig { + root_dir: dir.path().to_path_buf(), + verify_on_read: true, + disk_reserve: 0, + }) + .await + .expect_err("must refuse an unknown scheme"); + assert!(format!("{err}").contains("shard scheme"), "{err}"); + } + + #[tokio::test] + async fn writes_are_refused_when_the_disk_reserve_cannot_be_met() { + let dir = TempDir::new().expect("temp dir"); + let store = ChunkStore::new(ChunkStoreConfig { + root_dir: dir.path().to_path_buf(), + verify_on_read: true, + disk_reserve: u64::MAX / 2, + }) + .await + .expect("open store"); + + let (addr, content) = addressed("full"); + let err = store.put(&addr, &content).await.expect_err("must refuse"); + assert!( + format!("{err}").contains("Insufficient disk space"), + "{err}" + ); + assert!(store.check_capacity().is_err()); + } + + #[tokio::test] + async fn capacity_is_size_aware() { + // Wide enough that a test running alongside this one cannot move the answer. + const MARGIN: u64 = 512 * 1024 * 1024; + + let dir = TempDir::new().expect("temp dir"); + let available = fs2::available_space(dir.path()).expect("free space"); + // A reserve that leaves room for a small write but not a huge one. This is the + // whole reason the predicate takes a size: free bytes alone stopped being a + // sufficient answer once chunks became files. + let store = ChunkStore::new(ChunkStoreConfig { + root_dir: dir.path().to_path_buf(), + verify_on_read: true, + disk_reserve: available.saturating_sub(MARGIN), + }) + .await + .expect("open store"); + + assert!(store.check_capacity_for(1024).is_ok()); + assert!(store.check_capacity_for(4 * MARGIN).is_err()); + } + + #[test] + fn suffix_shards_stay_uniform_for_a_close_group_of_keys() { + // The real distribution: a node holds keys it is closest to, so they share a + // long leading prefix with its own ID. Sharding on that prefix collapses to one + // directory. The trailing byte is untouched by close-group membership. + let mut prefix_dirs = HashSet::new(); + let mut suffix_dirs = HashSet::new(); + for i in 0u32..4096 { + let mut key = [0u8; XORNAME_LEN]; + // 20 shared leading bits, as a ~1M-node network would impose. + let tail = crate::client::compute_address(&i.to_le_bytes()); + key.copy_from_slice(&tail); + if let Some(b) = key.first_mut() { + *b = 0xab; + } + if let Some(b) = key.get_mut(1) { + *b = 0xcd; + } + if let Some(b) = key.get_mut(2) { + *b &= 0x0f; + } + prefix_dirs.insert(key.first().copied().unwrap_or(0)); + suffix_dirs.insert(shard_index(&key)); + } + assert_eq!( + prefix_dirs.len(), + 1, + "prefix sharding collapses for a node's own holdings" + ); + assert!( + suffix_dirs.len() > 250, + "suffix sharding must stay uniform, got {} of 256 directories", + suffix_dirs.len() + ); + } + + #[test] + fn no_chunk_filename_can_spell_a_reserved_windows_device_name() { + // Hex has no `n`, `u`, `x`, `p`, `r`, `l`, `t`, `o` or `s`, so `CON`, `NUL`, + // `AUX`, `PRN`, `COM1` and `LPT1` are all unspellable at any length. This is why + // the encoding is hex and not base32 or base64url. + for reserved in ["con", "prn", "aux", "nul", "com1", "com9", "lpt1", "lpt9"] { + assert!( + !is_lower_hex(reserved), + "{reserved} must not be a valid chunk or shard name" + ); + } + } + + #[test] + fn only_full_length_lowercase_hex_decodes_to_an_address() { + // 0xab so the hex form actually contains letters, which is where case matters. + assert!(decode_chunk_name(&hex::encode([0xabu8; XORNAME_LEN])).is_some()); + assert!(decode_chunk_name(&hex::encode_upper([0xabu8; XORNAME_LEN])).is_none()); + assert!(decode_chunk_name("deadbeef").is_none()); + assert!(decode_chunk_name("").is_none()); + assert!(decode_chunk_name(&"g".repeat(CHUNK_NAME_LEN)).is_none()); + } + + #[tokio::test] + async fn repair_replaces_bad_bytes_without_the_file_ever_being_absent() { + let (store, _dir) = test_store().await; + let (addr, content) = addressed("repairable"); + store.put(&addr, &content).await.expect("put"); + let path = store.chunk_path(&addr); + + std::fs::write(&path, b"rotted").expect("corrupt"); + store.repair(&addr, &content).await.expect("repair"); + + assert_eq!( + store.get(&addr).await.expect("get").expect("present"), + content + ); + assert!(store.exists(&addr).expect("exists")); + } + + #[tokio::test] + async fn a_repair_with_the_wrong_bytes_is_refused_and_changes_nothing() { + let (store, _dir) = test_store().await; + let (addr, content) = addressed("guarded"); + store.put(&addr, &content).await.expect("put"); + let path = store.chunk_path(&addr); + + // The whole point of repairing in place is that a failure must leave the old file + // where it was. Deleting first and writing after would open a window whose only + // surviving copy is the one the caller is about to destroy. + let err = store + .repair(&addr, b"not this chunk") + .await + .expect_err("must refuse"); + assert!(format!("{err}").contains("Refusing to repair"), "{err}"); + assert!( + path.exists(), + "the existing file must survive a refused repair" + ); + assert_eq!( + store.get(&addr).await.expect("get").expect("present"), + content + ); + } + + #[tokio::test] + async fn a_chunk_can_be_deleted_and_stored_again() { + let (store, _dir) = test_store().await; + let (addr, content) = addressed("cycle"); + + assert!(store.put(&addr, &content).await.expect("put")); + assert!(store.delete(&addr).await.expect("delete")); + assert!( + store.put(&addr, &content).await.expect("re-put"), + "a re-stored chunk is new again" + ); + assert_eq!( + store.get(&addr).await.expect("get").expect("present"), + content + ); + } + + /// Write a chunk file straight into its shard, the way an existing store already + /// contains thousands of them. Bypasses the write path deliberately: this exercises + /// the startup scan, not `put`. + fn plant(chunks_dir: &Path, key: &XorName) { + let dir = chunks_dir.join(shard_name(key)); + std::fs::create_dir_all(&dir).expect("mkdir"); + std::fs::write(dir.join(hex::encode(key)), key).expect("plant"); + } + + #[tokio::test] + async fn a_populated_and_churned_store_scans_correctly_at_scale() { + // Every shard populated, then aged the way a long-lived node ages: some keys + // deleted, others added in their place, so the directories carry holes rather + // than being freshly written. APFS enumeration is known to degrade with churn + // rather than with size, so a fresh corpus is not a realistic one. + const PLANTED: u32 = 20_000; + const CHURN: u32 = 1_000; + + let dir = TempDir::new().expect("temp dir"); + let chunks_dir = dir.path().join(CHUNKS_DIR_NAME); + std::fs::create_dir_all(&chunks_dir).expect("mkdir"); + + let mut expected: Vec = Vec::new(); + for i in 0..PLANTED { + let key = crate::client::compute_address(&i.to_le_bytes()); + plant(&chunks_dir, &key); + expected.push(key); + } + for i in 0..CHURN { + let key = crate::client::compute_address(&i.to_le_bytes()); + std::fs::remove_file(chunks_dir.join(shard_name(&key)).join(hex::encode(key))) + .expect("churn out"); + let replacement = crate::client::compute_address(&(PLANTED + i).to_le_bytes()); + plant(&chunks_dir, &replacement); + } + expected.retain(|k| chunks_dir.join(shard_name(k)).join(hex::encode(k)).exists()); + for i in 0..CHURN { + expected.push(crate::client::compute_address(&(PLANTED + i).to_le_bytes())); + } + expected.sort_unstable(); + expected.dedup(); + + let started = std::time::Instant::now(); + let store = reopen(&dir).await; + let scan = started.elapsed(); + + assert_eq!( + store.current_chunks().expect("count"), + expected.len() as u64 + ); + assert_eq!(store.all_keys().await.expect("all_keys"), expected); + + // Every shard should be in use at this size: 20,000 keys over 256 directories is + // about 78 each, and the last byte of a BLAKE3 output is uniform. + let occupied = std::fs::read_dir(&chunks_dir) + .expect("read store root") + .filter_map(std::result::Result::ok) + .filter(|e| e.file_name().to_str().is_some_and(|n| n.len() == 2)) + .count(); + assert_eq!(occupied, SHARD_COUNT, "the suffix must reach every shard"); + + println!( + "scan of {} keys across {SHARD_COUNT} shards took {scan:?}", + expected.len() + ); + } + + #[tokio::test] + async fn wait_idle_returns_once_writes_have_drained() { + let (store, _dir) = test_store().await; + let store = Arc::new(store); + for i in 0..32 { + let store = Arc::clone(&store); + let (addr, content) = addressed(&format!("drain-{i}")); + tokio::spawn(async move { store.put(&addr, &content).await }); + } + // Not a synchronisation point for tasks that have not been spawned yet, but it + // must not hang and it must leave the store usable. + store.wait_idle().await; + let (addr, content) = addressed("after-drain"); + assert!(store.put(&addr, &content).await.expect("put after drain")); + } +} diff --git a/src/storage/handler.rs b/src/storage/handler.rs index 31038a68..4f3f36fc 100644 --- a/src/storage/handler.rs +++ b/src/storage/handler.rs @@ -1,7 +1,7 @@ //! ANT protocol handler for autonomi protocol messages. //! //! This handler processes chunk PUT/GET requests with optional payment verification, -//! storing chunks to LMDB and using the DHT for network-wide retrieval. +//! storing chunks on disk and using the DHT for network-wide retrieval. //! //! # Architecture //! @@ -18,7 +18,7 @@ //! │ ChunkQuoteRequest ChunkPutRequest ChunkGetRequest //! │ │ │ │ │ //! │ ▼ ▼ ▼ │ -//! │ QuoteGenerator PaymentVerifier LmdbStorage│ +//! │ QuoteGenerator PaymentVerifier ChunkStore│ //! │ │ │ │ │ //! │ └─────────────────────────┴─────────────────┘ │ //! │ │ │ @@ -41,7 +41,7 @@ use crate::payment::{PaymentVerifier, QuoteGenerator, VerificationContext}; use crate::replication::admission; use crate::replication::config::K_BUCKET_SIZE; use crate::replication::fresh::FreshWriteEvent; -use crate::storage::lmdb::LmdbStorage; +use crate::storage::ChunkStore; use bytes::Bytes; use parking_lot::RwLock; use saorsa_core::P2PNode; @@ -210,11 +210,11 @@ impl Drop for GetRequestTelemetry { /// ANT protocol handler. /// -/// Handles chunk PUT/GET/Quote requests using LMDB storage for persistence +/// Handles chunk PUT/GET/Quote requests, persisting each chunk as its own file /// and optional payment verification. pub struct AntProtocol { - /// LMDB storage for chunk persistence. - storage: Arc, + /// The chunk store. + storage: Arc, /// Payment verifier for checking payments. payment_verifier: Arc, /// Quote generator for creating storage quotes. @@ -233,12 +233,12 @@ impl AntProtocol { /// /// # Arguments /// - /// * `storage` - LMDB storage for chunk persistence + /// * `storage` - the chunk store /// * `payment_verifier` - Payment verifier for validating payments /// * `quote_generator` - Quote generator for creating storage quotes #[must_use] pub fn new( - storage: Arc, + storage: Arc, payment_verifier: Arc, quote_generator: Arc, ) -> Self { @@ -291,9 +291,9 @@ impl AntProtocol { CHUNK_PROTOCOL_ID } - /// Get a reference to the underlying LMDB storage. + /// Get a reference to the underlying chunk store. #[must_use] - pub fn storage(&self) -> Arc { + pub fn storage(&self) -> Arc { Arc::clone(&self.storage) } @@ -520,17 +520,20 @@ impl AntProtocol { } // 3. Check if already exists (idempotent success) - match self.storage.exists(&address) { - Ok(true) => { - debug!("Chunk {addr_hex} already exists"); - return ChunkPutResponse::AlreadyExists { address }; - } - Err(e) => { - return ChunkPutResponse::Error(ProtocolError::Internal(format!( - "Storage read failed: {e}" - ))); - } - Ok(false) => {} + // + // Verified against the offered bytes, not answered from the name. A name can + // outlive the bytes under it, and acknowledging a good copy of a chunk this node + // holds only a damaged version of throws that copy away and does not get offered + // another. Reached only when this node already has the chunk, and the content + // address was checked in step 2, so a damaged copy is repaired from these bytes + // rather than the offer being refused. + if self + .storage + .holds_verified(&address, &request.content) + .await + { + debug!("Chunk {addr_hex} already exists"); + return ChunkPutResponse::AlreadyExists { address }; } // 4. Cheap disk-space pre-check — runs BEFORE the expensive payment @@ -679,13 +682,13 @@ impl AntProtocol { /// actually holds. /// /// The quote price is driven by `QuoteGenerator::records_stored()`. Reading - /// the live LMDB entry count (an O(1) B-tree page-header read) right before + /// the live chunk count right before /// pricing makes the metric deletion-aware: any chunk removed by - /// [`LmdbStorage::delete`] or by the replication prune pass is reflected + /// [`ChunkStore::delete`] or by the replication prune pass is reflected /// immediately, with no risk of missing a delete path. /// /// On a storage read error — or a count that does not fit `usize` — the - /// previous metric value is left untouched so a transient LMDB error never + /// previous metric value is left untouched so a transient read error never /// disrupts quote generation. fn resync_quote_metric(&self) { match self.storage.current_chunks() { @@ -895,7 +898,7 @@ mod tests { use super::*; use crate::payment::metrics::QuotingMetricsTracker; use crate::payment::{EvmVerifierConfig, PaymentVerifierConfig}; - use crate::storage::LmdbStorageConfig; + use crate::storage::ChunkStoreConfig; use evmlib::RewardsAddress; use saorsa_core::identity::NodeIdentity; use saorsa_core::MlDsa65; @@ -916,13 +919,13 @@ mod tests { async fn create_test_protocol_with_reserve(disk_reserve: u64) -> (AntProtocol, TempDir) { let temp_dir = TempDir::new().expect("create temp dir"); - let storage_config = LmdbStorageConfig { + let storage_config = ChunkStoreConfig { root_dir: temp_dir.path().to_path_buf(), disk_reserve, - ..LmdbStorageConfig::test_default() + ..ChunkStoreConfig::test_default() }; let storage = Arc::new( - LmdbStorage::new(storage_config) + ChunkStore::new(storage_config) .await .expect("create storage"), ); @@ -961,7 +964,7 @@ mod tests { let (protocol, _temp) = create_test_protocol().await; let content = b"hello world"; - let address = LmdbStorage::compute_address(content); + let address = ChunkStore::compute_address(content); // Pre-populate payment cache so EVM verification is bypassed protocol.payment_verifier().cache_insert(address); @@ -1153,7 +1156,7 @@ mod tests { // Create oversized content let content = vec![0u8; MAX_CHUNK_SIZE + 1]; - let address = LmdbStorage::compute_address(&content); + let address = ChunkStore::compute_address(&content); let put_request = ChunkPutRequest::new(address, Bytes::from(content)); let put_msg = ChunkMessage { @@ -1186,7 +1189,7 @@ mod tests { /// "Full" now means both halves of the predicate: the volume is below the /// reserve **and** the store has no reusable space. A freshly created store /// has no freed pages, so both hold and the pre-check short-circuits, as it - /// always did. The companion cases in `storage::lmdb::tests` cover the half + /// always did. The companion cases in `storage::chunk_store::tests` cover the half /// that changed, where pruning has left reusable pages and the node must be /// admitted rather than refused on `statvfs` alone. /// @@ -1201,7 +1204,7 @@ mod tests { let (protocol, _temp) = create_test_protocol_with_reserve(u64::MAX).await; let content = b"chunk for a disk-full node"; - let address = LmdbStorage::compute_address(content); + let address = ChunkStore::compute_address(content); let put_request = ChunkPutRequest::new(address, Bytes::copy_from_slice(content)); let put_msg = ChunkMessage { @@ -1241,7 +1244,7 @@ mod tests { let (protocol, _temp) = create_test_protocol().await; let content = b"duplicate content"; - let address = LmdbStorage::compute_address(content); + let address = ChunkStore::compute_address(content); // Pre-populate cache so EVM verification is bypassed protocol.payment_verifier().cache_insert(address); @@ -1287,7 +1290,7 @@ mod tests { let (protocol, _temp) = create_test_protocol().await; let content = b"local access test"; - let address = LmdbStorage::compute_address(content); + let address = ChunkStore::compute_address(content); assert!(!protocol.exists(&address).expect("exists check")); @@ -1307,7 +1310,7 @@ mod tests { let (protocol, _temp) = create_test_protocol().await; let content = b"cache test content"; - let address = LmdbStorage::compute_address(content); + let address = ChunkStore::compute_address(content); // Before insert: cache should be empty let stats_before = protocol.payment_cache_stats(); @@ -1346,7 +1349,7 @@ mod tests { let (protocol, _temp) = create_test_protocol().await; let content = b"duplicate cache test"; - let address = LmdbStorage::compute_address(content); + let address = ChunkStore::compute_address(content); // Pre-populate cache for first PUT protocol.payment_verifier().cache_insert(address); @@ -1390,7 +1393,7 @@ mod tests { // Pre-populate cache, then store a chunk to test stats let content = b"stats test"; - let address = LmdbStorage::compute_address(content); + let address = ChunkStore::compute_address(content); protocol.payment_verifier().cache_insert(address); let put_request = ChunkPutRequest::new(address, Bytes::copy_from_slice(content)); @@ -1499,7 +1502,7 @@ mod tests { let (protocol, _temp) = create_test_protocol().await; let content = b"already stored quote test"; - let address = LmdbStorage::compute_address(content); + let address = ChunkStore::compute_address(content); // Store the chunk first protocol.payment_verifier().cache_insert(address); @@ -1600,7 +1603,7 @@ mod tests { let contents: Vec> = (0u8..5).map(|i| vec![i; 64]).collect(); let mut addresses = Vec::new(); for content in &contents { - let addr = LmdbStorage::compute_address(content); + let addr = ChunkStore::compute_address(content); protocol.put_local(&addr, content).await.expect("put_local"); addresses.push(addr); } @@ -1635,7 +1638,7 @@ mod tests { let contents: Vec> = (0u8..10).map(|i| vec![i; 64]).collect(); let mut addresses = Vec::new(); for content in &contents { - let addr = LmdbStorage::compute_address(content); + let addr = ChunkStore::compute_address(content); protocol.put_local(&addr, content).await.expect("put_local"); addresses.push(addr); } diff --git a/src/storage/legacy_artifacts.rs b/src/storage/legacy_artifacts.rs new file mode 100644 index 00000000..d7163a44 --- /dev/null +++ b/src/storage/legacy_artifacts.rs @@ -0,0 +1,390 @@ +//! What a node does when it finds the old chunk store still on disk. +//! +//! Chunks used to live in an LMDB environment at `{root}/chunks.mdb`. One release copied +//! them into a file per chunk and deleted that environment; this build has no code that +//! can read it. So a node starting with one still there is a node whose migration did not +//! happen or did not finish, and something has to decide what that means. +//! +//! Starting anyway is the tempting answer and it is the wrong one. Those chunks are +//! unreachable, but the commitment the node published before the upgrade claimed them, and +//! a commitment stays answerable to its neighbours for three hours. The one accusation the migration +//! releases suspended was "you did not have a chunk you were supposed to hold"; the +//! commitment-bound audit was never suspended in any release, precisely because it rests +//! on a signed claim. So a node that starts half-migrated spends those hours failing audits +//! for keys it cannot read, on the lane that always counted. +//! +//! Refusing everything is also wrong, and for a duller reason: a migration that finished +//! and then failed to delete the directory leaves one behind that is safe to ignore. A node +//! whose only fault is a failed `remove_dir_all` should not be held offline for it. +//! +//! So the question is not "is there an environment here" but "was it retired". The +//! retirement wrote a mark inside the directory before deleting anything, and that mark is +//! the only durable evidence there is. Not the migration marker file: the filesystem is +//! authoritative, and a live environment beside a marker that says the migration finished +//! means the marker is wrong, which is a state the previous release explicitly handles by +//! believing the filesystem. + +use crate::error::{Error, Result}; +use crate::logging::warn; +use std::path::{Path, PathBuf}; + +/// The directory the old chunk store lived in. +pub const LEGACY_ENV_DIR: &str = "chunks.mdb"; + +/// What retirement renamed it to before deleting it. +pub const RETIRED_SUFFIX: &str = ".retired"; + +/// The file retirement wrote inside a directory to say it had finished with it. +const RETIRED_MARKER: &str = "RETIRED"; + +/// What a directory's own contents say about whether it was retired. +/// +/// Three answers, not two, for the reason the release that wrote these marks needed three: +/// reading one can fail for a reason that is neither yes nor no, and folding that into "no +/// mark" is what turns an unreadable directory into one this node refuses to start over +/// forever, while folding it into "retired" would let a live environment be ignored. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum RetirementMark { + /// It carries its mark. The migration got as far as deciding this was finished with. + Present, + /// It carries no mark, and that is known rather than assumed. + Absent, + /// Whether it carries one could not be determined. + Unknown, +} + +/// Read a directory's retirement mark. +fn retirement_mark(dir: &Path) -> RetirementMark { + match std::fs::symlink_metadata(dir) { + // A link is never treated as retired, whatever it points at: the mark would have + // been written through it into somebody else's directory. + Ok(meta) if meta.file_type().is_symlink() => return RetirementMark::Absent, + Ok(_) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return RetirementMark::Absent, + Err(_) => return RetirementMark::Unknown, + } + match dir.join(RETIRED_MARKER).try_exists() { + Ok(true) => RetirementMark::Present, + Ok(false) => RetirementMark::Absent, + Err(_) => RetirementMark::Unknown, + } +} + +/// Every leftover of the old chunk store under `root_dir`, live name and tombstones alike. +/// +/// The tombstones matter as much as the live name. Retirement renamed the directory aside +/// before deleting it, so a crash between the rename and the mark leaves an intact +/// environment wearing a retired-looking name. The previous release would have restored and +/// reopened it; this one cannot, so it must not be waved through on the strength of what it +/// is called. +fn legacy_directories(root_dir: &Path) -> Result> { + let mut found = Vec::new(); + + // `symlink_metadata`, not `try_exists`. The latter follows links, so a looping or + // dangling one at the live name reads as nothing being there. And an error is not an + // absence: a root that can be traversed but not queried would hide the environment + // this function exists to find, and the caller would start. + let live = root_dir.join(LEGACY_ENV_DIR); + match std::fs::symlink_metadata(&live) { + Ok(_) => found.push(live), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => { + return Err(Error::Storage(format!( + "Cannot tell whether {} is there ({e}), so this node cannot tell whether it \ + has chunks in a store this build cannot read. Refusing to start rather \ + than assume it does not.", + live.display() + ))) + } + } + + // The same for the tombstones. A directory that cannot be listed hides every one of + // them, and a single unreadable entry inside it hides that one. + let prefix = format!("{LEGACY_ENV_DIR}{RETIRED_SUFFIX}"); + let entries = match std::fs::read_dir(root_dir) { + Ok(entries) => entries, + // A root that is not there yet holds nothing, which is every node starting for the + // first time and every test that names a directory before creating it. That is an + // answer, not a failure to get one. + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(found), + Err(e) => { + return Err(Error::Storage(format!( + "Cannot list {} ({e}), so this node cannot tell what the storage migration \ + left behind. Refusing to start rather than assume it left nothing.", + root_dir.display() + ))) + } + }; + for entry in entries { + let entry = entry.map_err(|e| { + Error::Storage(format!( + "Cannot read an entry in {} ({e}), so this node cannot tell what the storage \ + migration left behind. Refusing to start rather than skip it.", + root_dir.display() + )) + })?; + if entry + .file_name() + .to_str() + .is_some_and(|name| name.starts_with(&prefix)) + { + found.push(entry.path()); + } + } + Ok(found) +} + +/// Is this directory empty? +/// +/// An error is not an emptiness. A directory that cannot be listed might hold anything, and +/// the caller uses this to decide whether it is safe to ignore. +fn is_empty(dir: &Path) -> Result { + let mut entries = std::fs::read_dir(dir).map_err(|e| { + Error::Storage(format!( + "Cannot list {} ({e}), so this node cannot tell whether it still has chunks in \ + it. Refusing to start rather than assume it is empty.", + dir.display() + )) + })?; + Ok(entries.next().is_none()) +} + +/// Refuse to start if this node still has chunks in a store this build cannot read. +/// +/// Called before the file store is opened, so a node that is going to refuse does not +/// create anything first. +/// +/// # Errors +/// +/// Returns [`Error::Storage`] naming the directory when one is present without a +/// retirement mark, when whether it carries one cannot be determined, or when the node +/// root cannot be read well enough to say whether one is there at all. +pub fn refuse_if_unmigrated(root_dir: &Path) -> Result<()> { + for dir in legacy_directories(root_dir)? { + match retirement_mark(&dir) { + // Retired before this build ever ran. Its chunks are in the file store and the + // deletion simply did not finish. Left exactly where it is: this build has no + // migration code, so it has no business deciding that a directory it cannot + // read is safe to delete. + RetirementMark::Present => warn!( + "{} is a leftover of the storage migration. It was already retired, so \ + its chunks are in the file store and nothing is missing. It is costing \ + disk until it is removed by hand.", + dir.display() + ), + // A directory with nothing in it holds no chunks, so it cannot be hiding any. + // The previous release's reaper emptied a tombstone, removed its mark and then + // removed the directory, so a crash between the last two steps leaves exactly + // this: empty, unmarked, and fully migrated. That release recognised the state + // and cleaned it up. Refusing to start over it would hold a node offline for a + // directory that has nothing in it. + RetirementMark::Absent if is_empty(&dir)? => warn!( + "{} is an empty leftover of the storage migration, which is what an \ + interrupted cleanup leaves. Nothing is in it. It can be removed.", + dir.display() + ), + RetirementMark::Absent => { + return Err(Error::Storage(format!( + "{} is still here and this build cannot read it. Chunks in there were \ + never copied into the file store, and starting without them would \ + leave this node failing audits for keys its own published commitment \ + still claims. Run a build with the storage migration, let it finish, \ + then upgrade again. Refusing to start rather than serve a fraction of \ + what this node is committed to.", + dir.display() + ))) + } + RetirementMark::Unknown => { + return Err(Error::Storage(format!( + "{} is still here and this node cannot tell whether its chunks were \ + ever copied out of it. Check that the directory and everything in it \ + can be read. Refusing to start rather than guess, in either \ + direction.", + dir.display() + ))) + } + } + } + Ok(()) +} + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +mod tests { + use super::*; + use tempfile::TempDir; + + fn retired(dir: &Path) { + std::fs::create_dir_all(dir).expect("mkdir"); + std::fs::write(dir.join(RETIRED_MARKER), b"retired").expect("mark"); + } + + /// A node with nothing left over starts, which is every node that migrated. + #[test] + fn a_node_with_no_leftovers_starts() { + let dir = TempDir::new().expect("temp dir"); + assert!(refuse_if_unmigrated(dir.path()).is_ok()); + } + + /// A node whose root does not exist yet starts. + /// + /// Every node starting for the first time, and every caller that names a directory + /// before creating it. Reading a missing root fails, and treating that failure the way + /// the unreadable cases are treated would refuse every fresh node on the network. A + /// root that is not there holds nothing, which is an answer. + #[test] + fn a_root_that_does_not_exist_yet_is_not_a_refusal() { + let dir = TempDir::new().expect("temp dir"); + let never_created = dir.path().join("node").join("deeper"); + assert!(refuse_if_unmigrated(&never_created).is_ok()); + } + + /// A directory that says it was retired is not a reason to stay down. + /// + /// The migration finished and the deletion did not. Its chunks are in the file store, + /// so the node has everything it is committed to and holding it offline would cost + /// availability for a directory that is only costing disk. + #[test] + fn a_retired_leftover_is_not_a_reason_to_refuse() { + let dir = TempDir::new().expect("temp dir"); + retired(&dir.path().join(LEGACY_ENV_DIR)); + assert!(refuse_if_unmigrated(dir.path()).is_ok()); + + // Under the name retirement renames to, as well. + let dir = TempDir::new().expect("temp dir"); + retired(&dir.path().join(format!("{LEGACY_ENV_DIR}{RETIRED_SUFFIX}"))); + assert!(refuse_if_unmigrated(dir.path()).is_ok()); + let dir = TempDir::new().expect("temp dir"); + retired( + &dir.path() + .join(format!("{LEGACY_ENV_DIR}{RETIRED_SUFFIX}.3")), + ); + assert!(refuse_if_unmigrated(dir.path()).is_ok()); + } + + /// An empty leftover is not a reason to stay down. + /// + /// The previous release's cleanup emptied the directory, removed its mark, and then + /// removed the directory. A crash between the last two steps leaves an empty, unmarked + /// one behind: fully migrated, nothing in it, and indistinguishable by name from a + /// store that was never copied. That release recognised the state and tidied it up. + /// Holding a node offline for a directory with nothing in it would be an outage for + /// bookkeeping. + #[test] + fn an_empty_leftover_is_not_a_reason_to_refuse() { + for name in [ + LEGACY_ENV_DIR.to_string(), + format!("{LEGACY_ENV_DIR}{RETIRED_SUFFIX}"), + format!("{LEGACY_ENV_DIR}{RETIRED_SUFFIX}.2"), + ] { + let dir = TempDir::new().expect("temp dir"); + std::fs::create_dir_all(dir.path().join(&name)).expect("mkdir"); + assert!( + refuse_if_unmigrated(dir.path()).is_ok(), + "{name} has nothing in it and must not stop the node" + ); + } + } + + /// An environment nobody retired stops the node. + /// + /// The chunks in it were never copied out, and the commitment this node last published + /// claims them. Serving the rest would be failing audits for the difference. + #[test] + fn an_unretired_environment_stops_the_node() { + let dir = TempDir::new().expect("temp dir"); + let env = dir.path().join(LEGACY_ENV_DIR); + std::fs::create_dir_all(&env).expect("mkdir"); + std::fs::write(env.join("data.mdb"), b"chunks that were never copied").expect("seed"); + + let err = refuse_if_unmigrated(dir.path()).expect_err("this node must not start"); + let said = format!("{err}"); + assert!(said.contains("chunks.mdb"), "{said}"); + assert!( + said.contains("migration"), + "the message must say what to do: {said}" + ); + } + + /// And so does one wearing a retired name with no mark inside it. + /// + /// A crash between the rename and the mark leaves an intact environment under the name + /// retirement uses. The name is not the evidence; the mark is. + #[test] + fn an_unmarked_tombstone_stops_the_node_too() { + let dir = TempDir::new().expect("temp dir"); + let tombstone = dir.path().join(format!("{LEGACY_ENV_DIR}{RETIRED_SUFFIX}")); + std::fs::create_dir_all(&tombstone).expect("mkdir"); + std::fs::write(tombstone.join("data.mdb"), b"still every chunk").expect("seed"); + + assert!( + refuse_if_unmigrated(dir.path()).is_err(), + "a directory that only looks retired is not retired" + ); + } + + /// A node root that cannot be listed stops the node. + /// + /// Finding the leftovers is as load-bearing as classifying them, and an error looking + /// for one is not the same as there being none. A root that can be traversed but not + /// listed would hide every tombstone, and the node would start believing it had + /// nothing left over. This is the same fail-open the classifier itself was written + /// three-state to avoid, one step earlier in the same function. + /// + /// Unix only: the state is staged by taking the permission to list away. + #[cfg(unix)] + #[test] + fn a_root_that_cannot_be_listed_stops_the_node() { + use std::os::unix::fs::PermissionsExt; + + let dir = TempDir::new().expect("temp dir"); + let root = dir.path().join("node"); + std::fs::create_dir_all(&root).expect("mkdir"); + std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o000)).expect("chmod"); + + let refused = refuse_if_unmigrated(&root); + std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o755)).expect("chmod"); + + let Err(e) = refused else { + println!("skipped: this user can list a 000 directory"); + return; + }; + assert!(format!("{e}").contains("Cannot"), "{e}"); + } + + /// A link where the environment should be is not an absence either. + /// + /// `try_exists` follows links, so a loop or a dangling target reads as nothing being + /// there. Whatever that is, it is not proof this node has no chunks in a store this + /// build cannot read. + #[cfg(unix)] + #[test] + fn a_link_wearing_the_environment_name_is_not_treated_as_absence() { + let dir = TempDir::new().expect("temp dir"); + let live = dir.path().join(LEGACY_ENV_DIR); + std::os::unix::fs::symlink(&live, &live).expect("a link to itself"); + + assert!( + refuse_if_unmigrated(dir.path()).is_err(), + "a link at the environment name must not read as nothing being there" + ); + } + + /// A directory nobody can classify stops the node rather than being guessed at. + /// + /// Unix only: the state is staged with a symbolic link, which makes looking for the + /// mark return a loop while leaving everything else about the directory alone. + #[cfg(unix)] + #[test] + fn an_unclassifiable_environment_stops_the_node() { + let dir = TempDir::new().expect("temp dir"); + let env = dir.path().join(LEGACY_ENV_DIR); + std::fs::create_dir_all(&env).expect("mkdir"); + let link = env.join(RETIRED_MARKER); + std::os::unix::fs::symlink(&link, &link).expect("a link to itself"); + assert_eq!(retirement_mark(&env), RetirementMark::Unknown); + + let err = refuse_if_unmigrated(dir.path()).expect_err("this node must not start"); + assert!(format!("{err}").contains("cannot tell"), "{err}"); + } +} diff --git a/src/storage/lmdb.rs b/src/storage/lmdb.rs deleted file mode 100644 index 52abb52f..00000000 --- a/src/storage/lmdb.rs +++ /dev/null @@ -1,2382 +0,0 @@ -//! Content-addressed LMDB storage for chunks. -//! -//! Provides persistent storage for chunks using LMDB (via heed) for -//! memory-mapped, zero-copy reads with ACID transactions. -//! -//! ```text -//! {root}/chunks.mdb/ -- LMDB environment directory -//! ``` - -use crate::ant_protocol::{XorName, MAX_CHUNK_SIZE}; -use crate::error::{Error, Result}; -use crate::logging::{debug, info, trace, warn}; -use heed::types::Bytes; -use heed::{Database, Env, EnvOpenOptions, MdbError}; -use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::sync::Arc; -use std::time::Instant; -use tokio::task::spawn_blocking; -use tokio_util::task::TaskTracker; - -use crate::ant_protocol::XORNAME_LEN; - -/// Bytes in one MiB. -pub const MIB: u64 = 1024 * 1024; - -/// Bytes in one GiB. -pub const GIB: u64 = 1024 * MIB; - -/// Default minimum free disk space to preserve on the storage partition. -const DEFAULT_DISK_RESERVE: u64 = 500 * MIB; - -/// Convert a byte count to GiB for human-readable log messages. -#[allow(clippy::cast_precision_loss)] // display only — sub-byte precision is irrelevant -fn bytes_to_gib(bytes: u64) -> f64 { - bytes as f64 / GIB as f64 -} - -/// Absolute minimum LMDB map size. -/// -/// Even on a nearly-full disk the database must be able to open. -/// Set to 256 MiB — enough for millions of LMDB pages. -const MIN_MAP_SIZE: usize = 256 * 1024 * 1024; - -/// Maximum head-room (beyond the current data footprint) to reserve for the -/// LMDB map **on Windows**. -/// -/// On Windows a node's committed / private memory scales with the *mapped* -/// size rather than with the data actually stored. Measured on a live node: -/// ~7.4 GB of extra commit for a ~3.7 TiB map, versus ~181 MB for a ~55 GiB -/// map — roughly 0.2% of the mapped size, resident and attributed to *no* -/// user-space allocation. That size-proportional cost is consistent with -/// kernel page tables / section metadata for the mapping; the exact kernel -/// structure was inferred from the scaling rather than measured directly, but -/// the size-proportional *effect* is what this cap targets. Linux keeps the -/// mapping sparse, so a disk-sized map is nearly free there — the overhead is -/// Windows-specific, which is why it only surfaced in Windows reports. -/// -/// Sizing the map to the whole disk therefore costs ~0.2% of *free disk* per -/// node, multiplied by every node sharing the host (e.g. a 10 TiB partition ≈ -/// 20 GiB). We instead cap the head-room on Windows and lean on -/// `LmdbStorage::try_resize` to extend the map on demand as data accumulates, -/// keeping the overhead proportional to *stored data* rather than *disk -/// capacity*. At 32 GiB the extra commit is ~100 MB, and a resize happens at -/// most once per 32 GiB written. -#[cfg(windows)] -const WINDOWS_MAP_HEADROOM: u64 = 32 * GIB; - -/// How often to re-query available disk space (in seconds). -/// -/// Between checks the cached result is trusted. Disk space changes slowly -/// relative to chunk-write throughput, so a multi-second window is safe. -const DISK_CHECK_INTERVAL_SECS: u64 = 5; - -/// Ceiling raise offered to a single *delete* that cannot copy-on-write inside -/// the pinned map. -/// -/// A delete is itself a write: LMDB copies the B-tree path before it frees the -/// leaf pages, and it may need a page for the free-list's own bookkeeping. On a -/// map pinned exactly to the file size a delete therefore has nowhere to go, -/// and the node could not prune its way back to health. -/// -/// Granted **only** on the delete retry path and taken away again inside the -/// same locked scope, so an ordinary store can never allocate from it. Leaving -/// it permanently in the ceiling would hand every node a little more of the -/// very reserve this mode exists to protect, multiplied by the nodes sharing -/// the volume. -const DELETE_COW_SLACK: u64 = 256 * 1024; - -/// Total permanent file growth deletes may cause per low-disk episode. -/// -/// What actually needs bounding is *growth*, not grants. Most slack-assisted -/// deletes reuse pages already inside `data.mdb` and grow it by nothing, and -/// those must stay free: a node has to be able to prune indefinitely, and page -/// reuse is not reliably available to the very next delete because LMDB cannot -/// hand back pages a still-recent transaction freed. Charging per grant instead -/// of per byte stops a node pruning after its first assisted delete. -/// -/// Only bytes the file actually gained are charged here. Reset when the store -/// leaves no-growth mode. A rounding error against [`DEFAULT_DISK_RESERVE`]. -const DELETE_COW_GROWTH_BUDGET: u64 = 1024 * 1024; - -/// Configuration for LMDB storage. -#[derive(Debug, Clone)] -pub struct LmdbStorageConfig { - /// Root directory for storage (LMDB env lives at `{root_dir}/chunks.mdb/`). - pub root_dir: PathBuf, - /// Whether to verify content on read (compares hash to address). - pub verify_on_read: bool, - /// Explicit LMDB map size cap in bytes. - /// - /// When 0 (default), the map size is computed automatically from available - /// disk space and grows on demand when more storage becomes available. - pub max_map_size: usize, - /// Minimum free disk space (in bytes) to preserve on the storage partition. - /// - /// Writes are refused when available space drops below this threshold. - pub disk_reserve: u64, -} - -impl Default for LmdbStorageConfig { - fn default() -> Self { - Self { - root_dir: PathBuf::from(".ant/chunks"), - verify_on_read: true, - max_map_size: 0, - disk_reserve: DEFAULT_DISK_RESERVE, - } - } -} - -impl LmdbStorageConfig { - /// A test-friendly default with `disk_reserve` set to 0 so unit tests - /// don't depend on the host having >= 1 GiB free disk space. - #[cfg(any(test, feature = "test-utils"))] - #[must_use] - pub fn test_default() -> Self { - Self { - disk_reserve: 0, - ..Self::default() - } - } -} - -/// Statistics about storage operations. -#[derive(Debug, Clone, Default)] -pub struct StorageStats { - /// Total number of chunks stored. - pub chunks_stored: u64, - /// Total number of chunks retrieved. - pub chunks_retrieved: u64, - /// Total bytes stored. - pub bytes_stored: u64, - /// Total bytes retrieved. - pub bytes_retrieved: u64, - /// Number of duplicate writes (already exists). - pub duplicates: u64, - /// Number of verification failures on read. - pub verification_failures: u64, - /// Number of chunks currently persisted. - pub current_chunks: u64, -} - -/// Content-addressed LMDB storage. -/// -/// Uses heed (LMDB wrapper) for memory-mapped, transactional chunk storage. -/// Keys are 32-byte `XorName` addresses, values are raw chunk bytes. -pub struct LmdbStorage { - /// LMDB environment. - env: Env, - /// The unnamed default database (key=XorName bytes, value=chunk bytes). - db: Database, - /// Storage configuration. - config: LmdbStorageConfig, - /// Path to the LMDB environment directory (for disk-space queries). - env_dir: PathBuf, - /// Operation statistics. - stats: parking_lot::RwLock, - /// Serialises access to the LMDB environment during a map resize. - /// - /// Normal read/write operations acquire a **shared** lock. The rare - /// resize path acquires an **exclusive** lock, ensuring no transactions - /// are active when `env.resize()` is called (an LMDB safety requirement). - env_lock: Arc>, - /// Timestamp of the last successful disk-space check. - /// - /// `None` means "never checked — check on next write". Updated only - /// after a passing check, so a low-space result is always rechecked. - last_disk_ok: parking_lot::Mutex>, - /// Whether the map is currently pinned to the file's high-water mark. - /// - /// Set once available disk drops below the reserve. While pinned, LMDB can - /// still serve a write from its own free list but cannot extend - /// `data.mdb`, so the reserve is preserved by the allocator itself rather - /// than by refusing every write up front. - no_growth: Arc, - /// Serialises entering and leaving no-growth mode. - /// - /// Setting `no_growth` and resizing the map is one compound transition - /// spanning an await. Without this, two callers straddling the threshold - /// can interleave so the flag ends up describing a map size that was never - /// applied, leaving the store unpinned while it believes it is pinned. - growth_mode_lock: tokio::sync::Mutex<()>, - /// Bytes `data.mdb` has permanently gained to slack-assisted deletes in - /// this low-disk episode. - /// - /// A delete's copy-on-write can extend the file, and LMDB never gives file - /// space back, so that growth is permanent. Bounding it stops repeated - /// fill-then-delete cycles walking the file into the reserve. Deletes that - /// find room inside the file cost nothing. Reset on leaving no-growth mode. - delete_growth_charged: Arc, - /// Tracks every LMDB blocking task spawned by this storage. - /// - /// A `spawn_blocking` closure owns a cloned [`Env`] and keeps running - /// even when its async awaiter is dropped (e.g. by a `select!` losing to - /// a shutdown token). Tracking the blocking task itself — not the async - /// wrapper — lets [`Self::wait_idle`] wait for true quiescence before - /// the environment may be reopened. - blocking_tracker: TaskTracker, - /// Test-only gate read-acquired at the top of the put blocking closure. - /// - /// Tests hold the write half to deterministically park an in-flight put - /// on the blocking pool (e.g. to prove [`Self::wait_idle`] waits for a - /// detached write). - #[cfg(any(test, feature = "test-utils"))] - test_put_gate: Arc>, - /// Test-only gate read-acquired inside the raw-read blocking closures, - /// immediately after the shared `env_lock` guard is taken. - /// - /// Tests hold the write half to deterministically park an in-flight raw - /// read while it still holds the shared environment lock (e.g. to prove - /// [`Self::try_resize`] waits for active raw reads before calling - /// `env.resize()`). - #[cfg(any(test, feature = "test-utils"))] - test_read_gate: Arc>, -} - -impl LmdbStorage { - /// Create a new LMDB storage instance. - /// - /// Opens (or creates) an LMDB environment at `{root_dir}/chunks.mdb/`. - /// - /// When `config.max_map_size` is 0 (the default) the map size is derived - /// from the available disk space on the partition that hosts the database, - /// minus `config.disk_reserve`. This allows a node to use all available - /// storage without a fixed cap. If the operator adds more storage later - /// the map is resized on demand (see [`Self::put`]). - /// - /// # Errors - /// - /// Returns an error if the LMDB environment cannot be opened. - #[allow(unsafe_code)] - pub async fn new(config: LmdbStorageConfig) -> Result { - let env_dir = config.root_dir.join("chunks.mdb"); - - // Create the directory synchronously before opening LMDB - std::fs::create_dir_all(&env_dir) - .map_err(|e| Error::Storage(format!("Failed to create LMDB directory: {e}")))?; - - let map_size = if config.max_map_size > 0 { - // Operator provided an explicit cap. - config.max_map_size - } else { - // Auto-scale: current DB footprint + available space − reserve. - let computed = compute_map_size(&env_dir, config.disk_reserve)?; - info!( - "Auto-computed LMDB map size: {:.2} GiB (data + available disk minus {:.2} GiB \ - reserve, head-room capped on Windows to bound page-table overhead)", - bytes_to_gib(computed as u64), - bytes_to_gib(config.disk_reserve), - ); - computed - }; - - let env_dir_clone = env_dir.clone(); - // Constructor-only blocking task: it runs before `self` (and its - // `blocking_tracker`) exists, so it is deliberately untracked. The - // constructor awaits it right here, so it cannot outlive this call. - let (env, db) = spawn_blocking(move || -> Result<(Env, Database)> { - // SAFETY: `EnvOpenOptions::open()` is unsafe because LMDB uses memory-mapped - // I/O and relies on OS file-locking to prevent corruption from concurrent - // access by multiple processes. We satisfy this by giving each node instance - // a unique `root_dir` (typically a directory named by its full 64-hex peer - // ID), ensuring no two processes open the same LMDB environment. Callers - // who manually configure `--root-dir` must not point multiple nodes at the - // same directory. - let env = unsafe { - EnvOpenOptions::new() - .map_size(map_size) - .max_dbs(1) - .open(&env_dir_clone) - .map_err(|e| Error::Storage(format!("Failed to open LMDB env: {e}")))? - }; - - let mut wtxn = env - .write_txn() - .map_err(|e| Error::Storage(format!("Failed to create write txn: {e}")))?; - let db: Database = env - .create_database(&mut wtxn, None) - .map_err(|e| Error::Storage(format!("Failed to create database: {e}")))?; - wtxn.commit() - .map_err(|e| Error::Storage(format!("Failed to commit db creation: {e}")))?; - - Ok((env, db)) - }) - .await - .map_err(|e| Error::Storage(format!("LMDB init task failed: {e}")))??; - - let storage = Self { - env, - db, - config, - env_dir, - stats: parking_lot::RwLock::new(StorageStats::default()), - env_lock: Arc::new(parking_lot::RwLock::new(())), - last_disk_ok: parking_lot::Mutex::new(None), - no_growth: Arc::new(AtomicBool::new(false)), - growth_mode_lock: tokio::sync::Mutex::new(()), - delete_growth_charged: Arc::new(AtomicU64::new(0)), - blocking_tracker: TaskTracker::new(), - #[cfg(any(test, feature = "test-utils"))] - test_put_gate: Arc::new(parking_lot::RwLock::new(())), - #[cfg(any(test, feature = "test-utils"))] - test_read_gate: Arc::new(parking_lot::RwLock::new(())), - }; - - debug!( - "Initialized LMDB storage at {:?} ({} existing chunks)", - storage.env_dir, - storage.current_chunks()? - ); - - Ok(storage) - } - - /// Store a chunk. - /// - /// Before writing, verifies that available disk space exceeds the - /// configured reserve. If the LMDB map is full but more disk space - /// exists (e.g. the operator added storage), the map is resized - /// automatically and the write is retried. - /// - /// # Returns - /// - /// Returns `true` if the chunk was newly stored, `false` if it already existed. - /// - /// # Errors - /// - /// Returns an error if the write fails, content doesn't match address, - /// or the disk is too full to accept new chunks. - pub async fn put(&self, address: &XorName, content: &[u8]) -> Result { - // Verify content address - let computed = Self::compute_address(content); - if computed != *address { - return Err(Error::Storage(format!( - "Content address mismatch: expected {}, computed {}", - hex::encode(address), - hex::encode(computed) - ))); - } - - // Fast-path duplicate check (read-only, no write lock needed). - // This is an optimistic hint — the authoritative check happens inside - // the write transaction below to prevent TOCTOU races. - if self.exists(address)? { - trace!("Chunk {} already exists", hex::encode(address)); - self.stats.write().duplicates += 1; - return Ok(false); - } - - // ── Capacity guard (cached — at most one syscall per interval) ── - // Placed after the duplicate check so that re-storing an existing - // chunk remains a harmless no-op even when disk space is low. - // - // Below the reserve this pins the map instead of refusing outright, so - // the write is still attempted and LMDB decides whether a freed page - // can take it. A node that has pruned heavily keeps serving the network - // from the space it already occupies. - let no_growth = self.sync_growth_mode().await?; - - // ── Write (with resize-on-demand) ─────────────────────────────── - match self.try_put(address, content).await? { - PutOutcome::New => {} - PutOutcome::Duplicate => { - trace!("Chunk {} already exists", hex::encode(address)); - self.stats.write().duplicates += 1; - return Ok(false); - } - PutOutcome::MapFull if no_growth => { - // Both halves are now true: the volume is below the reserve and - // no free page can take *this* value. Resizing would extend the - // file into the reserve, so refuse. - // - // The refusal is not remembered. `MapFull` is specific to the - // size just attempted — a smaller value may still fit a smaller - // run — so caching it would let one maximum-sized chunk lock out - // every subsequent write. `check_capacity` estimates instead. - return Err(Error::Storage(format!( - "Insufficient disk space: {:.2} GiB reserve required and no reusable page \ - in the local store fits this {} B value. \ - Free disk space or increase the partition to continue storing chunks.", - bytes_to_gib(self.config.disk_reserve), - content.len(), - ))); - } - PutOutcome::MapFull => { - // The map ceiling was reached but there may be more disk space - // available (e.g. operator expanded the partition). - // - // Guarded: `no_growth` was sampled before the write, so the - // store may have entered no-growth mode since. Growing the map - // outside the transition lock could undo a pin that a - // concurrent `sync_growth_mode` had just applied, handing the - // reserve back to ordinary writes. - self.try_resize_for_growth().await?; - // Retry once after resize. - match self.try_put(address, content).await? { - PutOutcome::New => {} - PutOutcome::Duplicate => { - self.stats.write().duplicates += 1; - return Ok(false); - } - PutOutcome::MapFull => { - return Err(Error::Storage( - "LMDB map full after resize — disk may be at capacity".into(), - )); - } - } - } - } - - { - let mut stats = self.stats.write(); - stats.chunks_stored += 1; - stats.bytes_stored += content.len() as u64; - } - - debug!( - "Stored chunk {} ({} bytes)", - hex::encode(address), - content.len() - ); - - Ok(true) - } - - /// Attempt a single put inside a write transaction. - /// - /// Returns [`PutOutcome::MapFull`] instead of an error when the LMDB map - /// ceiling is reached, so the caller can resize and retry. - async fn try_put(&self, address: &XorName, content: &[u8]) -> Result { - let key = *address; - let value = content.to_vec(); - let env = self.env.clone(); - let db = self.db; - let lock = Arc::clone(&self.env_lock); - #[cfg(any(test, feature = "test-utils"))] - let test_put_gate = Arc::clone(&self.test_put_gate); - - self.blocking_tracker - .spawn_blocking(move || -> Result { - // Test-only: parks here while a test holds the write half. - #[cfg(any(test, feature = "test-utils"))] - let _test_put_gate = test_put_gate.read(); - let _guard = lock.read(); - - let mut wtxn = env - .write_txn() - .map_err(|e| Error::Storage(format!("Failed to create write txn: {e}")))?; - - // Authoritative existence check inside the serialized write txn - if db - .get(&wtxn, &key) - .map_err(|e| Error::Storage(format!("Failed to check existence: {e}")))? - .is_some() - { - return Ok(PutOutcome::Duplicate); - } - - match db.put(&mut wtxn, &key, &value) { - Ok(()) => {} - Err(heed::Error::Mdb(MdbError::MapFull)) => return Ok(PutOutcome::MapFull), - Err(e) => { - return Err(Error::Storage(format!("Failed to put chunk: {e}"))); - } - } - - match wtxn.commit() { - Ok(()) => Ok(PutOutcome::New), - Err(heed::Error::Mdb(MdbError::MapFull)) => Ok(PutOutcome::MapFull), - Err(e) => Err(Error::Storage(format!("Failed to commit put: {e}"))), - } - }) - .await - .map_err(|e| Error::Storage(format!("LMDB put task failed: {e}")))? - } - - /// Retrieve a chunk. - /// - /// # Returns - /// - /// Returns `Some(content)` if found, `None` if not found. - /// - /// # Errors - /// - /// Returns an error if read fails or verification fails. - pub async fn get(&self, address: &XorName) -> Result>> { - let key = *address; - let env = self.env.clone(); - let db = self.db; - let lock = Arc::clone(&self.env_lock); - - let content = self - .blocking_tracker - .spawn_blocking(move || -> Result>> { - let _guard = lock.read(); - let rtxn = env - .read_txn() - .map_err(|e| Error::Storage(format!("Failed to create read txn: {e}")))?; - let value = db - .get(&rtxn, &key) - .map_err(|e| Error::Storage(format!("Failed to get chunk: {e}")))?; - Ok(value.map(Vec::from)) - }) - .await - .map_err(|e| Error::Storage(format!("LMDB get task failed: {e}")))??; - - let Some(content) = content else { - trace!("Chunk {} not found", hex::encode(address)); - return Ok(None); - }; - - // Verify content if configured - if self.config.verify_on_read { - let computed = Self::compute_address(&content); - if computed != *address { - self.stats.write().verification_failures += 1; - warn!( - "Chunk verification failed: expected {}, computed {}", - hex::encode(address), - hex::encode(computed) - ); - return Err(Error::Storage(format!( - "Chunk verification failed for {}", - hex::encode(address) - ))); - } - } - - { - let mut stats = self.stats.write(); - stats.chunks_retrieved += 1; - stats.bytes_retrieved += content.len() as u64; - } - - debug!( - "Retrieved chunk {} ({} bytes)", - hex::encode(address), - content.len() - ); - - Ok(Some(content)) - } - - /// Check if a chunk exists. - /// - /// # Errors - /// - /// Returns an error if the LMDB read transaction fails. - pub fn exists(&self, address: &XorName) -> Result { - let _guard = self.env_lock.read(); - let rtxn = self - .env - .read_txn() - .map_err(|e| Error::Storage(format!("Failed to create read txn: {e}")))?; - let found = self - .db - .get(&rtxn, address.as_ref()) - .map_err(|e| Error::Storage(format!("Failed to check existence: {e}")))? - .is_some(); - Ok(found) - } - - /// Delete a chunk. - /// - /// # Errors - /// - /// Returns an error if deletion fails. - pub async fn delete(&self, address: &XorName) -> Result { - let key = *address; - - // Establish growth mode first, exactly as `put` does. Otherwise a - // delete arriving while the volume is low but before any write has - // pinned the map would copy-on-write into whatever head-room the - // ceiling still had, growing `data.mdb` into the reserve without - // passing through the budgeted allowance below. - self.sync_growth_mode().await?; - - let deleted = match self.try_delete(&key).await? { - DeleteOutcome::Done(existed) => existed, - DeleteOutcome::MapFull => { - // A delete is a write: LMDB copies the B-tree path before it - // frees the leaf pages, so a store with no free page at all - // cannot delete inside a map pinned to the file size. Without a - // way through, a node that filled up before it ever pruned - // could never prune its way out. - // - // Serialised against `sync_growth_mode` so the two cannot - // interleave their resizes. - let _transition = self.growth_mode_lock.lock().await; - self.delete_with_slack(&key).await? - } - }; - - if deleted { - debug!("Deleted chunk {}", hex::encode(address)); - } - - Ok(deleted) - } - - /// Attempt one delete, reporting `MapFull` rather than raising it. - async fn try_delete(&self, key: &XorName) -> Result { - let key = *key; - let env = self.env.clone(); - let db = self.db; - let lock = Arc::clone(&self.env_lock); - - self.blocking_tracker - .spawn_blocking(move || -> Result { - let _guard = lock.read(); - delete_in_txn(&env, db, &key) - }) - .await - .map_err(|e| Error::Storage(format!("LMDB delete task failed: {e}")))? - } - - /// Get storage statistics. - #[must_use] - pub fn stats(&self) -> StorageStats { - let mut stats = self.stats.read().clone(); - match self.current_chunks() { - Ok(count) => stats.current_chunks = count, - Err(e) => { - warn!("Failed to read current_chunks for stats: {e}"); - stats.current_chunks = 0; - } - } - stats - } - - /// Return the number of chunks currently stored, queried from LMDB metadata. - /// - /// This is an O(1) read of the B-tree page header — not a full scan. - /// - /// # Errors - /// - /// Returns an error if the LMDB read transaction fails. - pub fn current_chunks(&self) -> Result { - let _guard = self.env_lock.read(); - let rtxn = self - .env - .read_txn() - .map_err(|e| Error::Storage(format!("Failed to create read txn: {e}")))?; - let entries = self - .db - .stat(&rtxn) - .map_err(|e| Error::Storage(format!("Failed to read db stats: {e}")))? - .entries; - Ok(entries as u64) - } - - /// Compute content address (BLAKE3 hash). - #[must_use] - pub fn compute_address(content: &[u8]) -> XorName { - crate::client::compute_address(content) - } - - /// Get the root directory. - #[must_use] - pub fn root_dir(&self) -> &Path { - &self.config.root_dir - } - - /// Return all stored record keys. - /// - /// Iterates the LMDB database in a read transaction. Used by the - /// replication subsystem for hint construction and audit sampling. - /// - /// # Errors - /// - /// Returns an error if the LMDB read transaction fails. - pub async fn all_keys(&self) -> Result> { - let env = self.env.clone(); - let db = self.db; - let lock = Arc::clone(&self.env_lock); - - let keys = self - .blocking_tracker - .spawn_blocking(move || -> Result> { - // Hold the shared lock for the whole read so try_resize() (which - // takes the exclusive lock before the unsafe Env::resize()) cannot - // unmap the environment while this txn and its cursor are live. - let _guard = lock.read(); - let rtxn = env - .read_txn() - .map_err(|e| Error::Storage(format!("Failed to create read txn: {e}")))?; - let mut keys = Vec::new(); - let iter = db - .iter(&rtxn) - .map_err(|e| Error::Storage(format!("Failed to iterate database: {e}")))?; - for result in iter { - let (key_bytes, _) = - result.map_err(|e| Error::Storage(format!("Failed to read entry: {e}")))?; - if key_bytes.len() == XORNAME_LEN { - let mut key = [0u8; XORNAME_LEN]; - key.copy_from_slice(key_bytes); - keys.push(key); - } else { - crate::logging::warn!( - "LmdbStorage: skipping entry with unexpected key length {} (expected {XORNAME_LEN})", - key_bytes.len() - ); - } - } - Ok(keys) - }) - .await - .map_err(|e| Error::Storage(format!("all_keys task failed: {e}")))?; - - keys - } - - /// Retrieve raw chunk bytes without content-address verification. - /// - /// Used by the audit subsystem to compute digests over stored bytes. - /// Unlike [`Self::get`], this does not verify `hash(content) == address`. - /// - /// # Errors - /// - /// Returns an error if the LMDB read transaction fails. - pub async fn get_raw(&self, address: &XorName) -> Result>> { - let key = *address; - let env = self.env.clone(); - let db = self.db; - let lock = Arc::clone(&self.env_lock); - #[cfg(any(test, feature = "test-utils"))] - let test_read_gate = Arc::clone(&self.test_read_gate); - - let value = self - .blocking_tracker - .spawn_blocking(move || -> Result>> { - // Shared lock held until the bytes are copied out, so a concurrent - // try_resize() cannot unmap the environment mid-read. See all_keys. - let _guard = lock.read(); - // Test-only: parks here, still holding the shared lock, while a - // test holds the write half — used to prove a resize waits. - #[cfg(any(test, feature = "test-utils"))] - let _test_read_gate = test_read_gate.read(); - let rtxn = env - .read_txn() - .map_err(|e| Error::Storage(format!("Failed to create read txn: {e}")))?; - let val = db - .get(&rtxn, key.as_ref()) - .map_err(|e| Error::Storage(format!("Failed to get chunk: {e}")))?; - Ok(val.map(Vec::from)) - }) - .await - .map_err(|e| Error::Storage(format!("get_raw task failed: {e}")))?; - - value - } - - /// Cheap capacity pre-check for callers that want to reject work *before* - /// doing expensive setup (e.g. the PUT handler skipping payment - /// verification on a full node — see `V2-411`). - /// - /// A node is full only when **both** halves are true: the volume is below - /// the reserve *and* the store has no reusable page left. Deleting a record - /// returns its pages to LMDB's free list and never to the filesystem, so a - /// node that has pruned heavily sits on reusable capacity while `statvfs` - /// still reports the volume as full. Refusing on the disk half alone stops - /// such a node from writing into space it already owns. - /// - /// This is a **hint**, deliberately biased towards admitting: it estimates - /// reusable bytes and only refuses when there is not even one chunk's worth. - /// The authority on whether a given write fits stays with LMDB's allocator - /// in [`Self::put`], because no page count can account for the - /// copy-on-write of the B-tree path, the contiguous run a multi-megabyte - /// value needs, or pages still pinned by an open read transaction. An - /// over-optimistic hint costs one refused write; an over-pessimistic one - /// would recreate the bug this exists to fix. - /// - /// # Errors - /// - /// Returns [`Error::Storage`] when the volume is below the reserve and the - /// store holds less than one chunk of reusable space, or when the - /// disk-space query itself fails. - pub(crate) fn check_capacity(&self) -> Result<()> { - let Some(available) = self.available_space_cached()? else { - return Ok(()); - }; - - let reusable = self.reusable_bytes()?; - if reusable >= MAX_CHUNK_SIZE as u64 { - return Ok(()); - } - - Err(Error::Storage(format!( - "Insufficient disk space: {:.2} GiB available, {:.2} GiB reserve required, \ - and only {reusable} B reusable inside the local store. \ - Free disk space or increase the partition to continue storing chunks.", - bytes_to_gib(available), - bytes_to_gib(self.config.disk_reserve), - ))) - } - - /// Capacity as a three-way verdict, distinguishing a full node from a - /// query that failed. - /// - /// Refuses on the same two-part predicate as [`Self::check_capacity`]: - /// `Full` only when the volume is below the reserve *and* the store holds - /// less than one chunk of reusable space. Deleted records return their - /// pages to LMDB's free list and never to the filesystem, so a pruned node - /// reads as full to `statvfs` while still able to store chunks — such a - /// node must keep discovering holders for the keys it owes. - /// - /// Shares the same TTL cache as [`Self::check_capacity`]: a passing disk - /// reading is cached, a failing one is always rechecked, so freed space is - /// noticed promptly. An admit that rests on reusable pages is deliberately - /// not cached, matching the pre-check, because the free list can drain a - /// chunk at a time. - pub(crate) fn capacity_verdict(&self) -> CapacityVerdict { - { - let last = self.last_disk_ok.lock(); - if let Some(t) = *last { - if t.elapsed().as_secs() < DISK_CHECK_INTERVAL_SECS { - return CapacityVerdict::Writable; - } - } - } - let disk = verdict_from_available_space( - fs2::available_space(&self.env_dir), - self.config.disk_reserve, - ); - match disk { - CapacityVerdict::Writable => { - *self.last_disk_ok.lock() = Some(Instant::now()); - CapacityVerdict::Writable - } - CapacityVerdict::Unknown => CapacityVerdict::Unknown, - // Below the reserve is only half the predicate: the store may - // still hold pages it can reuse without growing the file. - CapacityVerdict::Full => match self.reusable_bytes() { - Ok(reusable) if reusable >= MAX_CHUNK_SIZE as u64 => CapacityVerdict::Writable, - Ok(_) => CapacityVerdict::Full, - Err(e) => { - warn!("Could not query the store's reusable space: {e}"); - CapacityVerdict::Unknown - } - }, - } - } - - /// Estimated bytes inside `data.mdb` that LMDB could write without growing - /// the file: the file size minus the pages currently holding data. - /// - /// Deliberately an over-estimate. `stat()` counts only the branch, leaf and - /// overflow pages of the unnamed database, so the free-list's own pages and - /// the environment metadata fall on the "reusable" side. Erring high keeps - /// [`Self::check_capacity`] biased towards admitting the attempt. - /// - /// Uses `stat()` rather than heed's `non_free_pages_size()`, which walks the - /// unnamed database calling `String::from_utf8(key).unwrap()` on every key - /// without a zero byte. Our keys are 32 random bytes, so that call panics - /// almost immediately. A single unnamed database makes `stat()` equivalent. - fn reusable_bytes(&self) -> Result { - // Order matters. The two samples are not atomic, so read the live pages - // first and the file length second: a write committing in between then - // pairs an older (smaller) live count with a newer (larger) file, which - // over-estimates. Sampling the other way round pairs a stale file - // length with a fresh live count and can under-estimate, which would - // refuse a node that has room — the very bug this fixes. - let stat = self.env.stat(); - let live_pages = (stat.branch_pages as u64) - .saturating_add(stat.leaf_pages as u64) - .saturating_add(stat.overflow_pages as u64); - let live_bytes = live_pages.saturating_mul(u64::from(stat.page_size)); - - let file_bytes = self - .env - .real_disk_size() - .map_err(|e| Error::Storage(format!("Failed to query LMDB file size: {e}")))?; - - Ok(file_bytes.saturating_sub(live_bytes)) - } - - /// Available bytes on the storage volume, or `None` when a recent check - /// already showed it above the reserve. - /// - /// Only *passing* results are cached, so a low-space condition is always - /// re-measured and freed space is detected promptly. - fn available_space_cached(&self) -> Result> { - { - let last = self.last_disk_ok.lock(); - if let Some(t) = *last { - if t.elapsed().as_secs() < DISK_CHECK_INTERVAL_SECS { - return Ok(None); - } - } - } - - let available = fs2::available_space(&self.env_dir) - .map_err(|e| Error::Storage(format!("Failed to query available disk space: {e}")))?; - - if available >= self.config.disk_reserve { - *self.last_disk_ok.lock() = Some(Instant::now()); - return Ok(None); - } - - Ok(Some(available)) - } - - /// Align the map ceiling with the current disk state, returning whether the - /// store is in no-growth mode. - /// - /// Below the reserve the map is pinned to the file's high-water mark, so a - /// put succeeds exactly when LMDB can satisfy it from the free list and - /// returns `MapFull` the moment it would need to extend `data.mdb`. That - /// makes the allocator the authority on "can this write fit". - /// - /// The whole transition runs under `growth_mode_lock`. Setting the flag and - /// resizing the map is one compound change spanning an await, so without - /// serialisation two callers straddling the threshold can interleave and - /// leave the flag describing a map that was never applied. - async fn sync_growth_mode(&self) -> Result { - let _transition = self.growth_mode_lock.lock().await; - - // Re-measured inside the lock: a caller that queued behind a transition - // must act on the state that transition left behind, not the one it saw - // before waiting. - if self.available_space_cached()?.is_none() { - // At or above the reserve: restore normal head-room if we pinned it. - if self.no_growth.load(Ordering::Acquire) { - // Intent first, work second. A `spawn_blocking` body outlives a - // cancelled awaiter, so ordering between two resizes cannot be - // guaranteed by holding an async lock. Publishing the intent - // before the work lets each closure re-read it under the - // exclusive lock and decline if it has since been reversed. - self.no_growth.store(false, Ordering::Release); - self.try_resize().await?; - } - // Real disk again: the maintenance allowance is refreshed. Done on - // every healthy pass, not just the transition, so an allowance - // spent while the flag happened to be clear is still returned. - self.delete_growth_charged.store(0, Ordering::Release); - return Ok(false); - } - - // Called unconditionally, not just on the transition. A re-pin that - // failed, or a transition whose caller was cancelled while its detached - // resize was still in flight, can leave the flag set while the map is - // not actually pinned; re-asserting it here repairs that instead of - // trusting the flag. The call is a no-op when already pinned. - self.no_growth.store(true, Ordering::Release); - self.pin_map_to_high_water().await?; - - Ok(true) - } - - /// Pin the LMDB map to the size of `data.mdb` on disk. - /// - /// Every page already in the file stays usable, including free ones, but - /// the file cannot grow, so the configured reserve is preserved by LMDB - /// itself rather than by refusing writes it could have served. - /// - /// Deliberately leaves **no** head-room: any slack in the ceiling is - /// ordinary put capacity, so it would be spent on the next chunk rather - /// than kept for maintenance, and on a shared volume every node would take - /// its own slice out of the reserve. Deletes get their copy-on-write room - /// on demand instead, see [`Self::delete`]. - /// - /// Takes the **exclusive** `env_lock` for the same reason - /// [`Self::try_resize`] does: `mdb_env_set_mapsize` requires that no - /// transaction is active. Callers hold `growth_mode_lock`. - #[allow(unsafe_code)] - async fn pin_map_to_high_water(&self) -> Result<()> { - // The "is it already pinned?" test lives inside the exclusive lock - // below, not out here. An unlocked pre-check can observe "already - // pinned" moments before a detached resize from a cancelled transition - // lands, after which the flag would claim a pin that no longer holds. - // Callers invoke this on every low-disk write so the pinned state - // repairs itself; the locked section is a few reads when nothing is to - // be done. - let env = self.env.clone(); - let lock = Arc::clone(&self.env_lock); - let no_growth = Arc::clone(&self.no_growth); - - self.blocking_tracker - .spawn_blocking(move || -> Result<()> { - // Exclusive lock guarantees no concurrent transactions. - let _guard = lock.write(); - - // Re-read under the lock: this closure may have been queued - // behind others, or its awaiter cancelled, and the store may - // have left no-growth mode since it was spawned. - if !no_growth.load(Ordering::Acquire) { - return Ok(()); - } - - let current_map = env.info().map_size; - let file_bytes = env - .real_disk_size() - .map_err(|e| Error::Storage(format!("Failed to query LMDB file size: {e}")))?; - - let page = page_size::get() as u64; - let aligned = file_bytes.div_ceil(page) * page; - let target = usize::try_from(aligned).unwrap_or(usize::MAX); - - // Re-checked under the lock: the state may have moved between - // the cheap check and here. - if target >= current_map { - return Ok(()); - } - - // SAFETY: We hold an exclusive lock, so no transactions are active. - unsafe { - env.resize(target) - .map_err(|e| Error::Storage(format!("Failed to pin LMDB map: {e}")))?; - } - - info!( - "Disk below reserve: pinned LMDB map to {:.2} GiB (was {:.2} GiB). \ - Writes that fit in already-freed pages still succeed; \ - only writes that would grow the file are refused.", - bytes_to_gib(target as u64), - bytes_to_gib(current_map as u64), - ); - Ok(()) - }) - .await - .map_err(|e| Error::Storage(format!("LMDB map pin task failed: {e}")))? - } - - /// Grow the map for a write, unless the store is pinned below the reserve. - /// - /// Serialised against [`Self::sync_growth_mode`] so a resize cannot land - /// after a pin and quietly undo it. If the store entered no-growth mode - /// while the write was in flight, the caller's `MapFull` is final and no - /// growth happens. - async fn try_resize_for_growth(&self) -> Result<()> { - let _transition = self.growth_mode_lock.lock().await; - - if self.no_growth.load(Ordering::Acquire) { - return Ok(()); - } - - self.try_resize().await - } - - /// Delete `key` with [`DELETE_COW_SLACK`] of temporary map head-room, then - /// take the head-room straight back. - /// - /// The raise, the delete and the re-pin all happen inside **one** exclusive - /// `env_lock` scope. Doing them as three separate locked steps would leave - /// windows in which an ordinary put could allocate from the raised ceiling, - /// spending the reserve on a chunk instead of on the maintenance it was - /// granted for, and an error or cancellation between the steps would leave - /// the ceiling raised for good. - /// - /// What is budgeted is the *growth*, not the grant. If the copy-on-write - /// does extend `data.mdb` that growth is permanent, since LMDB never - /// returns file space, so repeated fill-then-delete cycles could otherwise - /// walk the file into the reserve a slice at a time. A delete that finds - /// room inside the file is charged nothing. - /// - /// Charging per grant instead would be wrong, and was: page reuse is not - /// reliably available to the very next delete, because LMDB will not hand - /// back pages a still-recent transaction freed. A one-grant budget - /// therefore stopped a node pruning after its first assisted delete, which - /// showed up as every delete failing on 4 KiB-page hosts while passing on - /// 16 KiB-page ones. The budget resets when the store leaves no-growth - /// mode, i.e. when there is real disk to work with again. - #[allow(unsafe_code)] - async fn delete_with_slack(&self, key: &XorName) -> Result { - let key = *key; - let env = self.env.clone(); - let db = self.db; - let lock = Arc::clone(&self.env_lock); - let budget = Arc::clone(&self.delete_growth_charged); - - let outcome = self - .blocking_tracker - .spawn_blocking(move || -> Result { - // Checked and charged entirely inside the closure. A - // `spawn_blocking` body keeps running when its awaiter is - // dropped, so accounting split across the await could be - // skipped, permanently costing the node its ability to prune. - if budget.load(Ordering::Acquire) >= DELETE_COW_GROWTH_BUDGET { - return Err(Error::Storage(format!( - "Cannot delete: the local store is full and deletes have already used \ - their {DELETE_COW_GROWTH_BUDGET} B growth allowance. \ - Free disk space to continue." - ))); - } - - // Exclusive for the whole sequence: no transaction may be - // active across either resize, and no put may observe the - // raised ceiling. - let _guard = lock.write(); - - let page = page_size::get() as u64; - let previous_map = env.info().map_size; - let file_before = env - .real_disk_size() - .map_err(|e| Error::Storage(format!("Failed to query LMDB file size: {e}")))?; - let raised = (previous_map as u64) - .saturating_add(DELETE_COW_SLACK) - .div_ceil(page) - .saturating_mul(page); - - // SAFETY: exclusive lock held, so no transactions are active. - let granted = unsafe { - env.resize(usize::try_from(raised).unwrap_or(usize::MAX)) - .map_err(|e| Error::Storage(format!("Failed to grant delete slack: {e}"))) - }; - granted?; - - // Armed across the delete so an unwind still restores the - // ceiling; disarmed once the explicit restore below succeeds. - let mut ceiling_guard = MapCeilingRestorer { - env: &env, - previous: previous_map, - armed: true, - }; - - let outcome = delete_in_txn(&env, db, &key); - - // Charge what the file actually gained, not the fact that slack - // was offered. A delete that found room inside `data.mdb` costs - // nothing and must not consume the allowance, otherwise a node - // stops being able to prune after its first assisted delete. - // Measured before the ceiling is restored, and before any error - // is propagated, so a committed delete is always accounted for. - let file_after = env.real_disk_size().unwrap_or(file_before); - let grew = file_after.saturating_sub(file_before); - if grew > 0 { - budget.fetch_add(grew, Ordering::AcqRel); - } - - // Undo the raise before releasing the lock, on every path and - // whatever the delete did. Restoring to the previous ceiling - // rather than to a freshly measured file size keeps this - // unconditional: it is exactly the inverse of the raise, needs - // no second syscall that could itself fail, and is correct - // whether or not the store was pinned. If the copy-on-write did - // extend the file, LMDB clamps a request below the space in use, - // so the map still covers the data. - // - // SAFETY: exclusive lock held, so no transactions are active. - let restored = unsafe { - env.resize(previous_map) - .map_err(|e| Error::Storage(format!("Failed to restore LMDB map: {e}"))) - }; - if restored.is_ok() { - ceiling_guard.armed = false; - } - - // A failed restore is reported ahead of a failed delete, so the - // failure is not lost behind the delete's own error. - match (outcome, restored) { - (Ok(outcome), Ok(())) => Ok(outcome), - (_, Err(e)) | (Err(e), Ok(())) => Err(e), - } - }) - .await - .map_err(|e| Error::Storage(format!("LMDB delete-slack task failed: {e}")))?; - - match outcome? { - DeleteOutcome::Done(existed) => Ok(existed), - DeleteOutcome::MapFull => Err(Error::Storage( - "LMDB map full during delete even with the maintenance allowance".into(), - )), - } - } - - /// Grow the LMDB map to match currently available disk space. - /// - /// The new size is the **larger** of: - /// 1. the current map size (so existing data is never truncated), and - /// 2. `current_db_file_size + available_space − reserve` - /// (so all reachable disk space can be used). - /// - /// Acquires an **exclusive** lock on `env_lock` so that no read or write - /// transactions are active when the underlying `mdb_env_set_mapsize` is - /// called (an LMDB safety requirement). - #[allow(unsafe_code)] - async fn try_resize(&self) -> Result<()> { - let env = self.env.clone(); - let lock = Arc::clone(&self.env_lock); - let no_growth = Arc::clone(&self.no_growth); - let env_dir = self.env_dir.clone(); - let reserve = self.config.disk_reserve; - - self.blocking_tracker - .spawn_blocking(move || -> Result<()> { - // Exclusive lock guarantees no concurrent transactions. - let _guard = lock.write(); - - // Re-read under the lock. A `spawn_blocking` body outlives a - // cancelled awaiter, so this closure may land after the store - // entered no-growth mode. Growing then would hand back the - // head-room a pin had just taken away, and with it the disk - // reserve. - if no_growth.load(Ordering::Acquire) { - return Ok(()); - } - - // Measured here rather than before the spawn, so a late closure - // sizes from the disk as it is now, not as it was when queued. - let from_disk = compute_map_size(&env_dir, reserve)?; - - // Never shrink below the current map — existing data must remain - // addressable regardless of what the disk-space calculation says. - let current_map = env.info().map_size; - let new_size = from_disk.max(current_map); - - if new_size <= current_map { - debug!("LMDB map resize skipped — no additional disk space available"); - return Ok(()); - } - - // SAFETY: We hold an exclusive lock, so no transactions are active. - unsafe { - env.resize(new_size) - .map_err(|e| Error::Storage(format!("Failed to resize LMDB map: {e}")))?; - } - - info!( - "Resized LMDB map to {:.2} GiB (was {:.2} GiB)", - bytes_to_gib(new_size as u64), - bytes_to_gib(current_map as u64), - ); - Ok(()) - }) - .await - .map_err(|e| Error::Storage(format!("LMDB resize task failed: {e}")))? - } - - /// Wait until every tracked LMDB blocking task has finished. - /// - /// Dropping an async caller (e.g. a `select!` losing to a shutdown token) - /// does not cancel an already-spawned blocking closure — the closure keeps - /// running on the blocking pool with a cloned [`Env`]. This method waits - /// for those detached closures too, so when it returns no blocking - /// operation still holds the environment. - /// - /// Quiescence is only meaningful once callers have stopped issuing new - /// operations; concurrent traffic can keep the tracker non-empty - /// indefinitely. The storage remains fully usable afterwards (the - /// internal tracker is reopened before returning). - pub async fn wait_idle(&self) { - self.blocking_tracker.close(); - self.blocking_tracker.wait().await; - self.blocking_tracker.reopen(); - } - - /// Test-only handle to the put gate. - /// - /// Hold the write half to deterministically park the next put inside its - /// blocking closure (e.g. to exercise [`Self::wait_idle`] with a write - /// still in flight). - #[cfg(any(test, feature = "test-utils"))] - #[must_use] - pub fn test_put_gate(&self) -> Arc> { - Arc::clone(&self.test_put_gate) - } - - /// Test-only handle to the raw-read gate. - /// - /// Hold the write half to deterministically park the next raw read - /// (`get_raw`) inside its blocking closure while it still holds the shared - /// environment lock (e.g. to prove `try_resize` waits for active - /// raw reads). - #[cfg(any(test, feature = "test-utils"))] - #[must_use] - pub fn test_read_gate(&self) -> Arc> { - Arc::clone(&self.test_read_gate) - } -} - -// ──────────────────────────────────────────────────────────────────────────── -// Helpers -// ──────────────────────────────────────────────────────────────────────────── - -/// Outcome of a single `try_put` attempt. -enum PutOutcome { - /// Chunk was newly stored. - New, - /// Chunk already existed (idempotent). - Duplicate, - /// The LMDB map ceiling was reached — caller should resize and retry. - MapFull, -} - -/// Restores an LMDB map ceiling when dropped, including while unwinding. -/// -/// The explicit restore in [`LmdbStorage::delete_with_slack`] is the normal -/// path, because it can report a failure to the caller. This exists so a panic -/// between the raise and that restore cannot leave the ceiling raised, which -/// would quietly hand ordinary writes the disk reserve. -struct MapCeilingRestorer<'a> { - env: &'a Env, - previous: usize, - armed: bool, -} - -impl Drop for MapCeilingRestorer<'_> { - #[allow(unsafe_code)] - fn drop(&mut self) { - if !self.armed { - return; - } - // SAFETY: the owner holds the exclusive `env_lock` for this whole - // scope, so no transaction is active. - unsafe { - if let Err(e) = self.env.resize(self.previous) { - warn!("Failed to restore the LMDB map ceiling while unwinding: {e}"); - } - } - } -} - -/// Run one delete in its own write transaction, reporting `MapFull` rather than -/// raising it. -/// -/// The caller owns the `env_lock` discipline: [`LmdbStorage::try_delete`] holds -/// the shared guard, [`LmdbStorage::delete_with_slack`] the exclusive one. -fn delete_in_txn(env: &Env, db: Database, key: &XorName) -> Result { - let mut wtxn = match env.write_txn() { - Ok(wtxn) => wtxn, - Err(heed::Error::Mdb(MdbError::MapFull)) => return Ok(DeleteOutcome::MapFull), - Err(e) => return Err(Error::Storage(format!("Failed to create write txn: {e}"))), - }; - let existed = match db.delete(&mut wtxn, key) { - Ok(existed) => existed, - Err(heed::Error::Mdb(MdbError::MapFull)) => return Ok(DeleteOutcome::MapFull), - Err(e) => return Err(Error::Storage(format!("Failed to delete chunk: {e}"))), - }; - match wtxn.commit() { - Ok(()) => Ok(DeleteOutcome::Done(existed)), - Err(heed::Error::Mdb(MdbError::MapFull)) => Ok(DeleteOutcome::MapFull), - Err(e) => Err(Error::Storage(format!("Failed to commit delete: {e}"))), - } -} - -/// Outcome of one delete attempt. -enum DeleteOutcome { - /// The delete committed; the flag is whether the key had existed. - Done(bool), - /// The map ceiling left no room for the delete's copy-on-write. - MapFull, -} - -/// Compute the LMDB map size from the disk hosting `db_dir`. -/// -/// The result covers **all existing data** plus all remaining usable disk -/// space: -/// -/// ```text -/// map_size = current_db_file_size + max(0, available_space − reserve) -/// ``` -/// -/// `available_space` (from `statvfs`) reports only the *free* bytes on the -/// partition — the DB file's own footprint is **not** included, so adding -/// it back ensures the map is always large enough for the data already -/// stored. -/// -/// On Windows the disk-headroom term is additionally capped at -/// `WINDOWS_MAP_HEADROOM` to bound the map-proportional commit overhead (see -/// that constant); [`LmdbStorage::try_resize`] extends the map on demand as -/// data grows. -/// -/// The result is page-aligned and never falls below [`MIN_MAP_SIZE`]. -fn compute_map_size(db_dir: &Path, reserve: u64) -> Result { - let available = fs2::available_space(db_dir) - .map_err(|e| Error::Storage(format!("Failed to query available disk space: {e}")))?; - - // The MDB data file may not exist yet on first run. - let mdb_file = db_dir.join("data.mdb"); - let current_db_bytes = std::fs::metadata(&mdb_file).map_or(0, |m| m.len()); - - let target = map_target_bytes(current_db_bytes, available, reserve); - - // Align up to system page size (required by heed's resize). - let page = page_size::get() as u64; - let aligned = target.div_ceil(page) * page; - - let result = usize::try_from(aligned).unwrap_or(usize::MAX); - Ok(result.max(MIN_MAP_SIZE)) -} - -/// Head-room policy for the LMDB map, split out from [`compute_map_size`] so it -/// is unit-testable without touching the real filesystem. -/// -/// `map = current_db_bytes + max(0, available − reserve)`, with the head-room -/// term capped at `WINDOWS_MAP_HEADROOM` on Windows. Existing data -/// (`current_db_bytes`) is always covered so a resize can never truncate the -/// database, even when the head-room cap or a nearly-full disk drives the -/// growth term to zero. -fn map_target_bytes(current_db_bytes: u64, available: u64, reserve: u64) -> u64 { - // available_space excludes the DB file, so we add it back to get the - // total space the DB could occupy while still leaving `reserve` free. - let growth_room = available.saturating_sub(reserve); - - // On Windows, bound the head-room so the mapped size (and its - // size-proportional commit overhead) stays proportional to stored data - // rather than disk capacity. Elsewhere, use all reachable space. - #[cfg(windows)] - let growth_room = growth_room.min(WINDOWS_MAP_HEADROOM); - - current_db_bytes.saturating_add(growth_room) -} - -/// Map the result of a space query onto the *disk half* of the capacity -/// verdict. -/// -/// `Full` here means "below the reserve", which since ant-node #210 is only -/// half the refusal predicate: [`LmdbStorage::capacity_verdict`] goes on to -/// consult the store's reusable pages before refusing. -/// -/// Split out from [`LmdbStorage::capacity_verdict`] because the three-way -/// mapping is the part worth proving, and proving it through the filesystem is -/// not portable: asking for the free space of a directory that does not exist -/// fails on Unix but succeeds on Windows, which resolves it to the volume. -fn verdict_from_available_space(available: std::io::Result, reserve: u64) -> CapacityVerdict { - match available { - Ok(available) if available < reserve => CapacityVerdict::Full, - Ok(_) => CapacityVerdict::Writable, - Err(e) => { - warn!("Could not query available disk space: {e}"); - CapacityVerdict::Unknown - } - } -} - -/// What a capacity check concluded, when the caller needs to tell "this node is -/// full" apart from "this node could not find out". -/// -/// [`LmdbStorage::check_capacity`] collapses both into `Err`, which is right for -/// a caller that only wants to know whether to attempt a write. A caller -/// deciding *how long to stand down* needs the distinction: a full disk is a -/// standing condition worth waiting minutes on, while a failed `statvfs` may -/// have cleared by the next attempt and must not be treated as one. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum CapacityVerdict { - /// Available space is at or above the configured reserve. That is what the - /// query establishes, and possibly from the TTL cache — not a promise the - /// next write succeeds. - Writable, - /// Available space is below the configured reserve. - Full, - /// The query itself failed, so nothing is known about available space. - Unknown, -} - -#[cfg(test)] -#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] -mod tests { - use super::*; - use crate::ant_protocol::MAX_CHUNK_SIZE; - - /// Short probe used to prove `wait_idle` is still blocked on a parked op. - const WAIT_IDLE_BLOCKED_PROBE: std::time::Duration = std::time::Duration::from_millis(200); - /// Generous ceiling for `wait_idle` to complete once the op is released. - const WAIT_IDLE_COMPLETE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); - /// Poll interval while waiting for a parked raw read to take the shared lock. - const RAW_READ_LOCK_POLL: std::time::Duration = std::time::Duration::from_millis(5); - /// Ceiling for a parked raw read to take the shared lock. - const RAW_READ_LOCK_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); - /// Short probe used to prove `try_resize` is still blocked on the shared lock. - const RESIZE_BLOCKED_PROBE: std::time::Duration = std::time::Duration::from_millis(200); - /// Generous ceiling for `try_resize` to complete once the raw read releases. - const RESIZE_COMPLETE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); - - #[test] - fn map_target_covers_existing_data_and_headroom() { - // A partition with far more free space than any node needs. - let huge_free = 100 * 1024 * GIB; // 100 TiB - let reserve = 500 * MIB; - - // Fresh node, no data yet. - let fresh = map_target_bytes(0, huge_free, reserve); - // Node already holding 16 GiB of chunks. - let with_data = map_target_bytes(16 * GIB, huge_free, reserve); - - #[cfg(windows)] - { - // Windows: head-room is capped, so the map (and thus page tables) - // stay bounded regardless of disk size. Existing data always sits - // on top of the capped head-room. - assert_eq!(fresh, WINDOWS_MAP_HEADROOM); - assert_eq!(with_data, 16 * GIB + WINDOWS_MAP_HEADROOM); - // Sanity: page-table cost (~map/512) is tens of MiB, not tens of GiB. - assert!(with_data / 512 < 128 * MIB); - } - - #[cfg(not(windows))] - { - // Other platforms keep the disk-sized map (lazy page tables cost - // nothing), so head-room is the full free span minus reserve. - assert_eq!(fresh, huge_free - reserve); - assert_eq!(with_data, 16 * GIB + (huge_free - reserve)); - } - } - - #[test] - fn map_target_never_truncates_data_when_disk_nearly_full() { - // Free space below the reserve → head-room saturates to 0 on every - // platform, but the existing 4 GiB of data must still be covered. - assert_eq!(map_target_bytes(4 * GIB, 100 * MIB, 500 * MIB), 4 * GIB); - } - - /// Regression (V2-620 review): `all_keys` and `get_raw` must take the shared - /// `env_lock` so their LMDB read transaction can never run concurrently with - /// `try_resize()`'s unsafe `Env::resize()` — which this PR turns into a - /// routine ~per-32-GiB Windows event. We prove it by holding the exclusive - /// lock (as a resize does) and asserting both calls block until it is freed. - /// - /// Holding a `parking_lot` guard across `.await` is deliberate and safe here: - /// the guard stays on this current-thread test task while `all_keys`/`get_raw` - /// run their blocking work on the `spawn_blocking` pool (separate threads). - #[tokio::test] - #[allow(clippy::await_holding_lock)] - async fn read_paths_block_while_env_is_being_resized() { - use std::time::Duration; - let (storage, _temp) = create_test_storage().await; - - // Store one chunk so the read paths have real work to return. - let content = b"resize-safety"; - let address = LmdbStorage::compute_address(content); - storage.put(&address, content).await.expect("put"); - - // Simulate a resize in progress: hold the exclusive env lock. - let write_guard = storage.env_lock.write(); - - // Neither read path may complete while the exclusive lock is held — - // before the fix they took no lock and would return immediately. - assert!( - tokio::time::timeout(Duration::from_millis(250), storage.all_keys()) - .await - .is_err(), - "all_keys completed while env_lock was held exclusively — missing shared guard" - ); - assert!( - tokio::time::timeout(Duration::from_millis(250), storage.get_raw(&address)) - .await - .is_err(), - "get_raw completed while env_lock was held exclusively — missing shared guard" - ); - - // Once the exclusive lock is released, both proceed and see the data. - drop(write_guard); - assert_eq!(storage.all_keys().await.expect("all_keys").len(), 1); - assert_eq!( - storage.get_raw(&address).await.expect("get_raw").as_deref(), - Some(content.as_slice()) - ); - } - - async fn create_test_storage() -> (LmdbStorage, tempfile::TempDir) { - let temp_dir = tempfile::TempDir::new().expect("create temp dir"); - let config = LmdbStorageConfig { - root_dir: temp_dir.path().to_path_buf(), - ..LmdbStorageConfig::test_default() - }; - let storage = LmdbStorage::new(config).await.expect("create storage"); - (storage, temp_dir) - } - - #[tokio::test] - async fn test_put_and_get() { - let (storage, _temp) = create_test_storage().await; - - let content = b"hello world"; - let address = LmdbStorage::compute_address(content); - - // Store chunk - let is_new = storage.put(&address, content).await.expect("put"); - assert!(is_new); - - // Retrieve chunk - let retrieved = storage.get(&address).await.expect("get"); - assert_eq!(retrieved, Some(content.to_vec())); - } - - #[tokio::test] - async fn test_put_duplicate() { - let (storage, _temp) = create_test_storage().await; - - let content = b"test data"; - let address = LmdbStorage::compute_address(content); - - // First store - let is_new1 = storage.put(&address, content).await.expect("put 1"); - assert!(is_new1); - - // Duplicate store - let is_new2 = storage.put(&address, content).await.expect("put 2"); - assert!(!is_new2); - - // Check stats - let stats = storage.stats(); - assert_eq!(stats.chunks_stored, 1); - assert_eq!(stats.duplicates, 1); - } - - #[tokio::test] - async fn test_get_not_found() { - let (storage, _temp) = create_test_storage().await; - - let address = [0xAB; 32]; - let result = storage.get(&address).await.expect("get"); - assert!(result.is_none()); - } - - #[tokio::test] - async fn test_exists() { - let (storage, _temp) = create_test_storage().await; - - let content = b"exists test"; - let address = LmdbStorage::compute_address(content); - - assert!(!storage.exists(&address).expect("exists")); - - storage.put(&address, content).await.expect("put"); - - assert!(storage.exists(&address).expect("exists")); - } - - #[tokio::test] - async fn test_delete() { - let (storage, _temp) = create_test_storage().await; - - let content = b"delete test"; - let address = LmdbStorage::compute_address(content); - - // Store - storage.put(&address, content).await.expect("put"); - assert!(storage.exists(&address).expect("exists")); - - // Delete - let deleted = storage.delete(&address).await.expect("delete"); - assert!(deleted); - assert!(!storage.exists(&address).expect("exists")); - - // Delete again (already deleted) - let deleted2 = storage.delete(&address).await.expect("delete 2"); - assert!(!deleted2); - } - - #[tokio::test] - async fn test_address_mismatch() { - let (storage, _temp) = create_test_storage().await; - - let content = b"some content"; - let wrong_address = [0xFF; 32]; // Wrong address - - let result = storage.put(&wrong_address, content).await; - assert!(result.is_err()); - assert!(result.unwrap_err().to_string().contains("mismatch")); - } - - #[test] - fn test_compute_address() { - // Known BLAKE3 hash of "hello world" - let content = b"hello world"; - let address = LmdbStorage::compute_address(content); - - let expected_hex = "d74981efa70a0c880b8d8c1985d075dbcbf679b99a5f9914e5aaf96b831a9e24"; - assert_eq!(hex::encode(address), expected_hex); - } - - #[tokio::test] - async fn test_stats() { - let (storage, _temp) = create_test_storage().await; - - let content1 = b"content 1"; - let content2 = b"content 2"; - let address1 = LmdbStorage::compute_address(content1); - let address2 = LmdbStorage::compute_address(content2); - - // Store two chunks - storage.put(&address1, content1).await.expect("put 1"); - storage.put(&address2, content2).await.expect("put 2"); - - // Retrieve one - storage.get(&address1).await.expect("get"); - - let stats = storage.stats(); - assert_eq!(stats.chunks_stored, 2); - assert_eq!(stats.chunks_retrieved, 1); - assert_eq!( - stats.bytes_stored, - content1.len() as u64 + content2.len() as u64 - ); - assert_eq!(stats.bytes_retrieved, content1.len() as u64); - assert_eq!(stats.current_chunks, 2); - } - - #[tokio::test] - async fn test_persistence_across_reopen() { - let temp_dir = tempfile::TempDir::new().expect("create temp dir"); - let content = b"persistent data"; - let address = LmdbStorage::compute_address(content); - - // Store a chunk - { - let config = LmdbStorageConfig { - root_dir: temp_dir.path().to_path_buf(), - ..LmdbStorageConfig::test_default() - }; - let storage = LmdbStorage::new(config).await.expect("create storage"); - storage.put(&address, content).await.expect("put"); - } - - // Re-open and verify it persisted - { - let config = LmdbStorageConfig { - root_dir: temp_dir.path().to_path_buf(), - ..LmdbStorageConfig::test_default() - }; - let storage = LmdbStorage::new(config).await.expect("reopen storage"); - assert_eq!(storage.current_chunks().expect("current_chunks"), 1); - let retrieved = storage.get(&address).await.expect("get"); - assert_eq!(retrieved, Some(content.to_vec())); - } - } - - #[tokio::test] - async fn test_all_keys() { - let (storage, _temp) = create_test_storage().await; - - // Empty storage - let keys = storage.all_keys().await.expect("all_keys empty"); - assert!(keys.is_empty()); - - // Store some chunks - let content1 = b"chunk one for keys"; - let content2 = b"chunk two for keys"; - let addr1 = LmdbStorage::compute_address(content1); - let addr2 = LmdbStorage::compute_address(content2); - storage.put(&addr1, content1).await.expect("put 1"); - storage.put(&addr2, content2).await.expect("put 2"); - - let mut keys = storage.all_keys().await.expect("all_keys"); - keys.sort_unstable(); - let mut expected = vec![addr1, addr2]; - expected.sort_unstable(); - assert_eq!(keys, expected); - } - - #[tokio::test] - async fn test_get_raw() { - let (storage, _temp) = create_test_storage().await; - - let content = b"raw test data"; - let address = LmdbStorage::compute_address(content); - storage.put(&address, content).await.expect("put"); - - // get_raw returns bytes without verification - let raw = storage.get_raw(&address).await.expect("get_raw"); - assert_eq!(raw, Some(content.to_vec())); - - // Non-existent key - let missing = storage.get_raw(&[0xFF; 32]).await.expect("get_raw missing"); - assert!(missing.is_none()); - } - - /// Dropping a put's awaiter does not cancel its `spawn_blocking` LMDB - /// transaction; `wait_idle` must wait for that detached write, and the - /// storage must remain usable afterwards. - // Holding the gate's write guard across awaits is the point of the test: - // it parks the blocking closure while we probe wait_idle. - #[allow(clippy::await_holding_lock)] - #[tokio::test] - async fn wait_idle_waits_for_detached_put_blocking_op() { - let (storage, _temp) = create_test_storage().await; - - let content = b"detached put survives its dropped awaiter"; - let address = LmdbStorage::compute_address(content); - - // Park the put's blocking closure on the test gate. - let gate = storage.test_put_gate(); - let parked = gate.write(); - - // Drop the awaiting future mid-flight: the biased select! polls the - // put once — far enough to spawn the blocking task, which parks on - // the gate — then completes on the ready branch, dropping the put. - tokio::select! { - biased; - res = storage.put(&address, content) => { - panic!("put must be parked on the test gate, got {res:?}") - } - () = std::future::ready(()) => {} - } - - // The blocking op is still running: wait_idle must not complete. - let blocked = tokio::time::timeout(WAIT_IDLE_BLOCKED_PROBE, storage.wait_idle()).await; - assert!( - blocked.is_err(), - "wait_idle returned while the blocking op was parked" - ); - - // Release the gate: the detached closure commits and exits. - drop(parked); - tokio::time::timeout(WAIT_IDLE_COMPLETE_TIMEOUT, storage.wait_idle()) - .await - .expect("wait_idle after release"); - - // The dropped awaiter did not lose the write: it committed. - assert!(storage.exists(&address).expect("exists after release")); - - // The storage remains usable after wait_idle (tracker reopened). - let more = b"storage still usable after wait_idle"; - let more_addr = LmdbStorage::compute_address(more); - assert!(storage - .put(&more_addr, more) - .await - .expect("put after wait_idle")); - } - - /// A map resize takes the environment's *exclusive* lock, so it must wait - /// for in-flight raw reads (which hold the *shared* lock) to finish before - /// calling `env.resize()`. This proves `get_raw` holds that shared lock for - /// the whole duration of its blocking closure; `all_keys` uses the same - /// guard. - // Holding the gate's write guard across awaits is the point of the test: - // it parks the raw read's blocking closure while we probe try_resize. - #[allow(clippy::await_holding_lock)] - #[tokio::test] - async fn resize_waits_for_in_flight_raw_read() { - let (storage, _temp) = create_test_storage().await; - - let content = b"raw read holds the shared env lock"; - let address = LmdbStorage::compute_address(content); - storage.put(&address, content).await.expect("put"); - - // Park the raw read's blocking closure on the test gate. It acquires - // the shared env_lock first, then parks here still holding it. - let gate = storage.test_read_gate(); - let parked = gate.write(); - - // Drop the awaiting future mid-flight: the biased select! polls get_raw - // once — far enough to spawn the blocking task, which takes the shared - // lock and parks on the gate — then completes on the ready branch, - // dropping the awaiter. The detached closure keeps holding the lock. - tokio::select! { - biased; - res = storage.get_raw(&address) => { - panic!("get_raw must be parked on the test gate, got {res:?}") - } - () = std::future::ready(()) => {} - } - - // Wait until the detached read has actually taken the shared lock, - // signalled by the exclusive half no longer being immediately available. - tokio::time::timeout(RAW_READ_LOCK_TIMEOUT, async { - loop { - let free = storage.env_lock.try_write().is_some(); - if !free { - break; - } - tokio::time::sleep(RAW_READ_LOCK_POLL).await; - } - }) - .await - .expect("raw read did not take the shared env lock"); - - // A resize needs the exclusive lock, so it must block while the raw - // read holds the shared lock. - let resize = storage.try_resize(); - tokio::pin!(resize); - let blocked = tokio::time::timeout(RESIZE_BLOCKED_PROBE, &mut resize).await; - assert!( - blocked.is_err(), - "try_resize completed while a raw read held the shared env lock" - ); - - // Release the read: it drops the shared lock, letting the resize take - // the exclusive lock and finish. - drop(parked); - tokio::time::timeout(RESIZE_COMPLETE_TIMEOUT, &mut resize) - .await - .expect("try_resize did not complete after the raw read released") - .expect("try_resize"); - } - - /// The gate fires on `Full` and only on `Full`, so the verdict has to tell a - /// disk below its reserve from one above it, and has to notice when that - /// stops being true. - /// - /// The third call is the one that matters for recovery: the below-reserve - /// condition clears, and the verdict has to follow it rather than stay stuck - /// on its earlier answer. A node that remembered a refusal would stop - /// fetching for good. - /// - /// What this does not show: that a changed free-space reading is re-read from - /// the filesystem. The condition is cleared by dropping the reserve, which is - /// the same comparison approached from the other side. - #[tokio::test] - async fn capacity_verdict_follows_the_reserve_and_notices_recovery() { - let (writable, _temp) = create_test_storage().await; - assert_eq!(writable.capacity_verdict(), CapacityVerdict::Writable); - - let temp_dir = tempfile::TempDir::new().expect("create temp dir"); - let config = LmdbStorageConfig { - root_dir: temp_dir.path().to_path_buf(), - // Far above any real free space, the same way the e2e builds a - // write-blocked node. - disk_reserve: u64::MAX / 2, - ..LmdbStorageConfig::test_default() - }; - let mut full = LmdbStorage::new(config).await.expect("create storage"); - assert_eq!(full.capacity_verdict(), CapacityVerdict::Full); - - // Still short of space, so still `Full`. A `Full` result that populated - // the passing-result cache would answer `Writable` here. - assert_eq!(full.capacity_verdict(), CapacityVerdict::Full); - - // Space is no longer short. Nothing cached a refusal, so the very next - // read has to see it. - full.config.disk_reserve = 0; - assert_eq!( - full.capacity_verdict(), - CapacityVerdict::Writable, - "a refusal must not be negatively cached: the next read has to see the \ - below-reserve condition clear" - ); - } - - /// A space query that fails says nothing about available space, so it must - /// not read as a full disk. The gate stands a key down for five minutes on - /// `Full` alone, and a failed `statvfs` is not a condition worth standing - /// down for: it may be gone by the next cycle. - /// - /// The two `Ok` cases pin the boundary the reserve names: equal to the - /// reserve is writable, one byte under it is not. - /// - /// What this does not show: that the gate leaves `Unknown` alone. The gate - /// tests `== Full` on a separate line inside the verification cycle, which - /// needs a network to reach, so this covers the classification only. - #[test] - fn a_failed_space_query_reads_as_unknown_not_full() { - const RESERVE: u64 = 1024; - - assert_eq!( - verdict_from_available_space(Err(std::io::Error::other("space query failed")), RESERVE), - CapacityVerdict::Unknown, - "a failed query must not be reported as a full disk" - ); - - assert_eq!( - verdict_from_available_space(Ok(RESERVE - 1), RESERVE), - CapacityVerdict::Full - ); - assert_eq!( - verdict_from_available_space(Ok(RESERVE), RESERVE), - CapacityVerdict::Writable, - "at the reserve is not below it" - ); - } - - /// The verdict and the pre-check have to refuse on the same condition. - /// - /// They are two readings of one question — can this node write? — and two - /// callers depend on them separately: the verification cycle gates the - /// close-group probe on the verdict, while `execute_single_fetch` gates the - /// dial on the pre-check. A verdict stricter than the pre-check is the - /// harmful direction. A node the pre-check would let write stops - /// discovering holders for keys it could have stored, which is - /// under-replication rather than a saved probe, and nothing else in the - /// change would notice. - /// - /// This is a tripwire, deliberately built on the state where the two are - /// about to part company rather than on a bare full disk. ant-node - /// \#210 makes the pre-check a two-part predicate — below the reserve *and* - /// out of reusable pages inside the store — and a store that has deleted - /// more than one chunk's worth of pages fails only the first half, because - /// LMDB returns those pages to its own free list and never to the - /// filesystem. On a bare full disk the two predicates still agree, so a - /// test built on one would pass straight through the divergence. Whichever - /// of the two changes merges second has to carry the second half into the - /// verdict, and this is what makes that a red test rather than a textual - /// conflict resolved without it. - /// - /// What this does not show: which of `Full` and `Unknown` a refusal is. - /// `a_failed_space_query_reads_as_unknown_not_full` pins that, and only - /// `Full` reaches the gate. - #[tokio::test] - async fn capacity_verdict_refuses_exactly_when_check_capacity_does() { - let (mut storage, _temp) = create_test_storage().await; - assert_eq!(storage.capacity_verdict(), CapacityVerdict::Writable); - assert!( - storage.check_capacity().is_ok(), - "a writable verdict has to mean the pre-check admits the write" - ); - - // Two chunks written and deleted, so the store sits on more than one - // chunk of space LMDB can reuse and the filesystem will never take - // back. This is the pruned node the two predicates disagree about. - for fill in [1u8, 2u8] { - let content = vec![fill; MAX_CHUNK_SIZE]; - let address = LmdbStorage::compute_address(&content); - assert!(storage.put(&address, &content).await.expect("put")); - assert!(storage.delete(&address).await.expect("delete")); - } - - // Established without either function under test, so the scenario does - // not rest on the thing being measured. - storage.config.disk_reserve = u64::MAX / 2; - *storage.last_disk_ok.lock() = None; - let available = fs2::available_space(&storage.env_dir).expect("query available space"); - assert!( - available < storage.config.disk_reserve, - "the volume has to read as below the reserve, or neither predicate is \ - being asked the interesting question" - ); - // Reusable bytes read the way the two-part predicate reads them: the - // file's size less the pages still holding data. `Env::stat` rather - // than `non_free_pages_size`, which calls `String::from_utf8(key)` and - // unwraps, so it panics on our 32 random bytes of key. - let stat = storage.env.stat(); - let live_pages = (stat.branch_pages as u64) - .saturating_add(stat.leaf_pages as u64) - .saturating_add(stat.overflow_pages as u64); - let live_bytes = live_pages.saturating_mul(u64::from(stat.page_size)); - let file_bytes = storage.env.real_disk_size().expect("query store file size"); - let reusable = file_bytes.saturating_sub(live_bytes); - assert!( - reusable > MAX_CHUNK_SIZE as u64, - "the store has to sit on more than one chunk of reusable space, or the \ - two predicates are not yet being asked to differ \ - (file_bytes={file_bytes}, live_bytes={live_bytes})" - ); - - // Neither call caches a refusal, so the order of the two does not - // decide either answer. - let pre_check_refuses = storage.check_capacity().is_err(); - let verdict = storage.capacity_verdict(); - assert_eq!( - pre_check_refuses, - verdict != CapacityVerdict::Writable, - "the dial pre-check and the verification gate disagree about whether \ - this node can write: check_capacity refuses={pre_check_refuses}, \ - verdict={verdict:?}. A verdict of Full under a pre-check that admits \ - the write leaves a node that can store chunks refusing to look for \ - them" - ); - } - - // ── Capacity below the disk reserve (LMDB reuse) ──────────────────── - - /// Value size for the reuse tests. A whole number of chunks' worth, so the - /// space freed by a few deletes is unambiguously enough for one more. - const REUSE_VALUE_LEN: usize = 1024 * 1024; - - /// Distinct filler of `REUSE_VALUE_LEN` bytes. - fn reuse_filler(seed: u32) -> Vec { - let mut content = seed.to_le_bytes().to_vec(); - content.resize(REUSE_VALUE_LEN, 0u8); - content - } - - /// A config for `dir` whose reserve exceeds any real disk, so the store - /// always sees itself as below the reserve. - fn below_reserve_config(dir: &Path) -> LmdbStorageConfig { - LmdbStorageConfig { - root_dir: dir.to_path_buf(), - disk_reserve: u64::MAX, - ..LmdbStorageConfig::test_default() - } - } - - /// Write `count` chunks with an unconstrained reserve, returning their - /// addresses in insertion order. - async fn seed_chunks(dir: &Path, count: u32) -> Vec { - let config = LmdbStorageConfig { - root_dir: dir.to_path_buf(), - ..LmdbStorageConfig::test_default() - }; - let storage = LmdbStorage::new(config).await.expect("create storage"); - - let mut addresses = Vec::new(); - for seed in 0..count { - let content = reuse_filler(seed); - let address = LmdbStorage::compute_address(&content); - storage.put(&address, &content).await.expect("seed put"); - addresses.push(address); - } - - storage.wait_idle().await; - addresses - } - - fn file_len(storage: &LmdbStorage) -> u64 { - storage.env.real_disk_size().expect("real_disk_size") - } - - /// The regression this change is about: a node whose volume is below the - /// reserve must still write into pages an earlier delete freed. Before the - /// fix the pre-check refused on the disk half alone, so a node that had - /// pruned sat on reusable space it could not use. - #[tokio::test] - async fn below_reserve_put_reuses_freed_pages() { - let temp_dir = tempfile::TempDir::new().expect("create temp dir"); - let seeded = seed_chunks(temp_dir.path(), 12).await; - - let storage = LmdbStorage::new(below_reserve_config(temp_dir.path())) - .await - .expect("reopen storage"); - - let content = reuse_filler(u32::MAX); - let address = LmdbStorage::compute_address(&content); - - // Nothing freed yet, so this write would have to grow the file. Below - // the reserve that is exactly what must be refused. - assert!( - storage.put(&address, &content).await.is_err(), - "a write that must grow the file was allowed below the reserve" - ); - - // Free several chunks. Their pages go on LMDB's free list, not back to - // the filesystem, so `statvfs` still reports the volume as full. - for seeded_address in seeded.iter().take(6) { - assert!(storage.delete(seeded_address).await.expect("delete")); - } - - let before = file_len(&storage); - let stored = storage - .put(&address, &content) - .await - .expect("put into freed pages was refused below the reserve"); - assert!(stored); - assert_eq!( - storage.get(&address).await.expect("get"), - Some(content), - "chunk written into reused pages did not read back" - ); - - // The whole point: it was served from inside the existing file. - assert_eq!( - file_len(&storage), - before, - "reusing freed pages grew data.mdb, consuming the reserve" - ); - } - - /// A refused write must not have grown the file on its way to failing, - /// which is what protects the reserve while the map is pinned. - #[tokio::test] - async fn below_reserve_refused_put_does_not_grow_the_file() { - let temp_dir = tempfile::TempDir::new().expect("create temp dir"); - let _ = seed_chunks(temp_dir.path(), 6).await; - - let storage = LmdbStorage::new(below_reserve_config(temp_dir.path())) - .await - .expect("reopen storage"); - - let content = reuse_filler(u32::MAX); - let address = LmdbStorage::compute_address(&content); - - // Take the baseline after the first attempt, so it includes the pin. - let refusal = storage - .put(&address, &content) - .await - .expect_err("a write that must grow the file was allowed"); - assert!( - refusal.to_string().contains("Insufficient disk space"), - "refused for the wrong reason: {refusal}" - ); - let before = file_len(&storage); - let pinned_map = storage.env.info().map_size; - - for seed in 0..4u32 { - let content = reuse_filler(u32::MAX - 1 - seed); - let address = LmdbStorage::compute_address(&content); - let refusal = storage - .put(&address, &content) - .await - .expect_err("a write that must grow the file was allowed"); - assert!( - refusal.to_string().contains("Insufficient disk space"), - "refused for the wrong reason: {refusal}" - ); - } - - assert_eq!( - storage.env.info().map_size, - pinned_map, - "the map ceiling drifted while writes were being refused" - ); - - assert_eq!( - file_len(&storage), - before, - "refused writes still extended data.mdb into the reserve" - ); - } - - /// One refused maximum-sized value must not lock out smaller ones. LMDB's - /// `MapFull` is specific to the allocation it was asked for, so remembering - /// it store-wide would let a single large chunk deny every later write. - #[tokio::test] - async fn large_refusal_does_not_block_a_smaller_put() { - let temp_dir = tempfile::TempDir::new().expect("create temp dir"); - let seeded = seed_chunks(temp_dir.path(), 10).await; - - let storage = LmdbStorage::new(below_reserve_config(temp_dir.path())) - .await - .expect("reopen storage"); - - // Free room for a small value, but not for a large one. - let Some(first) = seeded.first() else { - panic!("seed_chunks returned no addresses"); - }; - assert!(storage.delete(first).await.expect("delete")); - - // A value far larger than what was freed cannot fit. - let oversized = vec![3u8; 8 * REUSE_VALUE_LEN]; - let oversized_address = LmdbStorage::compute_address(&oversized); - assert!(storage.put(&oversized_address, &oversized).await.is_err()); - - // A small value still must, using the pages the delete released. - let small = b"small record that fits in a freed page".to_vec(); - let small_address = LmdbStorage::compute_address(&small); - let stored = storage - .put(&small_address, &small) - .await - .expect("a large refusal blocked a small put that had room"); - assert!(stored); - } - - /// A store with no reusable page must still be able to delete, or it can - /// never prune its way back to health. - #[tokio::test] - async fn full_store_below_reserve_can_still_delete() { - let temp_dir = tempfile::TempDir::new().expect("create temp dir"); - let seeded = seed_chunks(temp_dir.path(), 8).await; - - let storage = LmdbStorage::new(below_reserve_config(temp_dir.path())) - .await - .expect("reopen storage"); - - // Pin the map by attempting a write that cannot fit. - let content = reuse_filler(u32::MAX); - let address = LmdbStorage::compute_address(&content); - assert!(storage.put(&address, &content).await.is_err()); - assert!(storage.no_growth.load(Ordering::Acquire)); - - let pinned_map = storage.env.info().map_size; - - for seeded_address in &seeded { - assert!( - storage.delete(seeded_address).await.expect("delete"), - "a pinned store could not prune" - ); - } - assert_eq!(storage.current_chunks().expect("current_chunks"), 0); - - // The allowance bounds permanent file growth, not the number of - // assisted deletes. Pruning a pinned store must stay possible however - // many deletes it takes, so whatever was charged has to be growth the - // file really took, and has to stay inside the budget. - let charged = storage.delete_growth_charged.load(Ordering::Acquire); - assert!( - charged < DELETE_COW_GROWTH_BUDGET, - "deletes exhausted the growth allowance ({charged} B) while pruning a pinned store" - ); - - // Whether or not any delete needed the maintenance allowance, none of - // it may be left in the ceiling afterwards: a raised ceiling is - // ordinary put capacity, so leaking it hands away the reserve. - let file_bytes = file_len(&storage); - assert!( - storage.env.info().map_size as u64 <= file_bytes.max(pinned_map as u64), - "delete left maintenance slack in the map ceiling" - ); - - // And the store must still refuse a write it cannot fit, i.e. the pin - // is still doing its job after the prune. - let oversized = vec![9u8; 64 * REUSE_VALUE_LEN]; - let oversized_address = LmdbStorage::compute_address(&oversized); - assert!( - storage.put(&oversized_address, &oversized).await.is_err(), - "pinning stopped working after a delete used the allowance" - ); - } - - /// The pre-check must admit while reuse is plausible and refuse once it is - /// not. Refusing on `statvfs` alone is what blinded a node to its own free - /// pages, so being below the reserve cannot by itself be an error. - #[tokio::test] - async fn check_capacity_tracks_reusable_space_not_just_disk() { - let temp_dir = tempfile::TempDir::new().expect("create temp dir"); - let seeded = seed_chunks(temp_dir.path(), 16).await; - - let storage = LmdbStorage::new(below_reserve_config(temp_dir.path())) - .await - .expect("reopen storage"); - - // A freshly written store has almost no free page, so below the reserve - // the pre-check refuses and the caller skips its expensive setup. - assert!( - storage.check_capacity().is_err(), - "pre-check stayed open on a store with no reusable space" - ); - - // Pruning puts pages back on the free list. Nothing is returned to the - // filesystem, so `statvfs` is unchanged and only the reusable half of - // the predicate can reopen the node. - for seeded_address in seeded.iter().take(10) { - assert!(storage.delete(seeded_address).await.expect("delete")); - } - - storage - .check_capacity() - .expect("pre-check stayed closed after pruning freed pages"); - } - - /// Freeing disk must lift the pin, or a node would stay clamped to its - /// high-water mark after an operator grew the partition. - #[tokio::test] - async fn leaving_no_growth_restores_head_room() { - let temp_dir = tempfile::TempDir::new().expect("create temp dir"); - let _ = seed_chunks(temp_dir.path(), 6).await; - - let mut storage = LmdbStorage::new(below_reserve_config(temp_dir.path())) - .await - .expect("reopen storage"); - - let content = reuse_filler(u32::MAX); - let address = LmdbStorage::compute_address(&content); - assert!(storage.put(&address, &content).await.is_err()); - assert!(storage.no_growth.load(Ordering::Acquire)); - let pinned_map = storage.env.info().map_size; - - // Simulate the operator freeing space: the reserve is now satisfiable. - storage.config.disk_reserve = 0; - *storage.last_disk_ok.lock() = None; - - let stored = storage - .put(&address, &content) - .await - .expect("store stayed pinned after disk was freed"); - assert!(stored); - assert!(!storage.no_growth.load(Ordering::Acquire)); - assert!( - storage.env.info().map_size > pinned_map, - "map was not re-grown after leaving no-growth mode" - ); - } - - /// Above the reserve nothing changes: no pinning, and writes grow the file - /// on demand exactly as before. - #[tokio::test] - async fn above_reserve_behaviour_is_unchanged() { - let (storage, _temp) = create_test_storage().await; - - storage - .check_capacity() - .expect("pre-check on a healthy node"); - - let content = reuse_filler(1); - let address = LmdbStorage::compute_address(&content); - assert!(storage.put(&address, &content).await.expect("put")); - assert!(!storage.no_growth.load(Ordering::Acquire)); - storage - .check_capacity() - .expect("pre-check after a healthy put"); - } -} diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 64f9462c..7dee04e4 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -1,8 +1,8 @@ //! Storage subsystem for chunk persistence. //! -//! This module provides content-addressed LMDB storage for chunks, -//! along with a protocol handler that integrates with saorsa-core's -//! `Protocol` trait for automatic message routing. +//! This module provides content-addressed storage for chunks, one immutable file per +//! chunk, along with a protocol handler that integrates with saorsa-core's `Protocol` +//! trait for automatic message routing. //! //! # Architecture //! @@ -19,7 +19,7 @@ //! │ QuoteRequest ChunkPutRequest ChunkGetRequest //! │ │ │ │ │ //! │ ▼ ▼ ▼ │ -//! │ QuoteGenerator PaymentVerifier LmdbStorage│ +//! │ QuoteGenerator PaymentVerifier ChunkStore│ //! │ │ │ │ │ //! │ └─────────────────────────┴─────────────────┘ │ //! │ │ │ @@ -31,11 +31,11 @@ //! //! ```rust,ignore //! use std::sync::Arc; -//! use ant_node::storage::{AntProtocol, LmdbStorage, LmdbStorageConfig}; +//! use ant_node::storage::{AntProtocol, ChunkStore, ChunkStoreConfig}; //! //! // Create storage -//! let config = LmdbStorageConfig::default(); -//! let storage = Arc::new(LmdbStorage::new(config).await?); +//! let config = ChunkStoreConfig::default(); +//! let storage = Arc::new(ChunkStore::new(config).await?); //! //! // Create protocol handler //! let protocol = AntProtocol::new(storage, Arc::new(payment_verifier), Arc::new(quote_generator)); @@ -44,11 +44,46 @@ //! listener.register_protocol(protocol).await?; //! ``` +#[cfg(any(test, feature = "test-utils"))] +pub mod chunk_store; +#[cfg(not(any(test, feature = "test-utils")))] +pub(crate) mod chunk_store; mod handler; -pub(crate) mod lmdb; +pub mod legacy_artifacts; pub use crate::ant_protocol::XorName; +pub use chunk_store::{CapacityVerdict, ChunkStore, ChunkStoreConfig, StoreLayout}; pub use handler::AntProtocol; pub(crate) use handler::ChunkRequestContext; -pub(crate) use lmdb::CapacityVerdict; -pub use lmdb::{LmdbStorage, LmdbStorageConfig, StorageStats}; +pub use legacy_artifacts::LEGACY_ENV_DIR; + +/// Bytes in one MiB. +pub const MIB: u64 = 1024 * 1024; + +/// Bytes in one GiB. +pub const GIB: u64 = 1024 * MIB; + +/// Default free disk space to keep unused on the storage partition. +pub const DEFAULT_DISK_RESERVE: u64 = 500 * MIB; + +/// Statistics about storage operations. +/// +/// Counters other than `current_chunks` are cumulative for the lifetime of the +/// process; `current_chunks` is the live count. +#[derive(Debug, Clone, Default)] +pub struct StorageStats { + /// Total number of chunks stored. + pub chunks_stored: u64, + /// Total number of chunks retrieved. + pub chunks_retrieved: u64, + /// Total bytes stored. + pub bytes_stored: u64, + /// Total bytes retrieved. + pub bytes_retrieved: u64, + /// Number of duplicate writes (already exists). + pub duplicates: u64, + /// Number of verification failures on read. + pub verification_failures: u64, + /// Number of chunks currently persisted. + pub current_chunks: u64, +} diff --git a/tests/chunk_store_crash_safety.rs b/tests/chunk_store_crash_safety.rs new file mode 100644 index 00000000..60d0e489 --- /dev/null +++ b/tests/chunk_store_crash_safety.rs @@ -0,0 +1,247 @@ +//! What survives a process dying part-way through a write. +//! +//! The store rests on being able to stop at any moment and start again: every step is +//! idempotent and re-derived from the filesystem. That is easy to assert and hard to +//! believe without trying it, so these tests kill a real child process at a real point in +//! the work and then open the store in this one and check what is there. +//! +//! **What this does and does not prove.** A killed process loses nothing the kernel has +//! already accepted, so this covers ordering and bookkeeping: a chunk is whole or absent +//! and never half-indexed, and what an interrupted write leaves behind is swept. It does +//! not cover power loss, where the kernel loses what it accepted and never wrote. That is +//! still a fleet gate. +//! +//! These tests came from the harness that proved the migration off the old chunk store. +//! Most of that harness went with the migration; these two did not belong to it. They are +//! about the store's own publish path, which is now the only one there is. + +#![allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::missing_panics_doc, + clippy::cast_possible_truncation +)] + +use ant_node::storage::{ChunkStore, ChunkStoreConfig}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::time::Duration; +use tempfile::TempDir; + +/// Run this test binary again as a child in the mode named by `role`, wait until it has +/// reached the named point, and kill it there. +/// +/// A child process rather than a thread, because the point is to lose everything the +/// process was holding: buffers, in-memory index, locks, half-finished intentions. +/// +/// The wait is a handshake, not a sleep. An earlier version of this slept and hoped, and +/// on a quick machine the child had finished everything before the kill arrived, so the +/// test was checking a clean shutdown while claiming to check a crash. The child now stops +/// at a failpoint inside the write and says so by writing a marker; this waits for the +/// marker and then kills it, so the process always dies at the same point in the same +/// operation. +fn kill_child_at_failpoint(role: &str, root: &Path, failpoint: &str, let_through: u64) -> PathBuf { + let marker = root.join(format!("reached-{role}")); + let _ = std::fs::remove_file(&marker); + + let exe = std::env::current_exe().expect("this test binary"); + let mut child = Command::new(exe) + .arg("--exact") + .arg(role) + .arg("--nocapture") + .arg("--ignored") + .env("ANT_CRASH_TEST_ROOT", root) + .env(failpoint, &marker) + .env( + ant_node::storage::chunk_store::HALT_AFTER, + let_through.to_string(), + ) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()) + .spawn() + .expect("spawn the child"); + + // Generous, but not unbounded. Without a deadline a failpoint that stopped working + // would hang the job rather than fail it, and a hang says nothing about the code. + let deadline = std::time::Instant::now() + Duration::from_secs(120); + while !marker.exists() { + if let Ok(Some(status)) = child.try_wait() { + panic!("the child exited before reaching the failpoint: {status}"); + } + if std::time::Instant::now() > deadline { + let _ = child.kill(); + panic!("the child never reached the failpoint"); + } + std::thread::sleep(Duration::from_millis(10)); + } + + child.kill().expect("kill the child"); + let _ = child.wait(); + marker +} + +/// Where the child was told to work. +fn child_root() -> PathBuf { + PathBuf::from(std::env::var("ANT_CRASH_TEST_ROOT").expect("the child needs a root")) +} + +/// Child mode: write chunks into a file store until killed. +#[tokio::test] +#[ignore = "child process of a crash test, not run on its own"] +async fn child_writes_until_killed() { + let root = child_root(); + let store = ChunkStore::new(ChunkStoreConfig { + root_dir: root, + disk_reserve: 0, + ..ChunkStoreConfig::default() + }) + .await + .expect("open"); + + // Always a chunk it has not written before, so the kill lands in real work rather + // than in a re-offer of something already on disk. An earlier version cycled the same + // hundred keys and spent almost all its time confirming duplicates. + let mut n = 0usize; + loop { + let content = chunk_bytes(n); + let address = ant_node::client::compute_address(&content); + let _ = store.put(&address, &content).await; + n += 1; + } +} + +/// A process killed inside a publish leaves no chunk it cannot serve. +/// +/// The child is stopped at the last moment before the chunk's name exists on disk: on Unix +/// the bytes written to a temporary file with the rename not yet made, off Unix the point +/// before the file is created at all, since that platform writes under the final name +/// because a rename there carries no durability guarantee. The failure this guards against +/// is the same on both: a name outliving its bytes. The index is built from filenames at +/// startup, so a partial file wearing a real chunk name would be advertised, committed to, +/// and unservable. +#[tokio::test] +async fn a_process_killed_mid_publish_leaves_no_chunk_it_cannot_serve() { + let tmp = TempDir::new().expect("temp dir"); + let root = tmp.path().join("node"); + std::fs::create_dir_all(&root).expect("mkdir"); + + // Twenty chunks land before the crash, so the store this reopens has real content in + // it. Stopping the very first write would leave nothing indexed and the loop below + // would pass by iterating over nothing. + let marker = kill_child_at_failpoint( + "child_writes_until_killed", + &root, + ant_node::storage::chunk_store::HALT_BEFORE_PUBLISH, + 20, + ); + assert!( + marker.exists(), + "the child must have reached the failpoint before it was killed" + ); + + // Reopening is itself part of the assertion: a store that cannot start after a crash + // is a node that cannot start. + let store = reopen(&root).await; + // The child discards its put results and the failpoint counts arrivals, not successes, + // so every publish before the kill could in principle have failed. An empty store makes + // the loop below pass over nothing, which is the one outcome that would let this test + // report success having checked no chunk at all. + let held = store.all_keys().await.expect("all_keys"); + assert!( + !held.is_empty(), + "the child published nothing before it was killed, so there is nothing to check" + ); + for key in held { + let served = store.get(&key).await; + assert!( + matches!(served, Ok(Some(_))), + "chunk {} is claimed after a crash but cannot be served: {served:?}", + hex::encode(key) + ); + } +} + +/// The temporary file a killed publish left behind is swept, not indexed. +/// +/// It carries no chunk name, so it can never be served, and leaving it would cost disk +/// for the life of the node. +/// +/// Unix only, because the leftover only exists on Unix. Off Unix the store creates the +/// file under its final name and flushes it, deliberately, since a rename there is not +/// documented to be durable. So there is no temporary file to sweep and the equivalent +/// hazard is different: a real chunk name over bytes that are short or wrong. That one is +/// covered by the store's own tests, which run on every platform, and by the +/// re-hash-everything pass the retirement does before it deletes anything. +#[cfg(unix)] +#[tokio::test] +async fn the_leftovers_of_a_killed_publish_are_swept() { + let tmp = TempDir::new().expect("temp dir"); + let root = tmp.path().join("node"); + std::fs::create_dir_all(&root).expect("mkdir"); + + kill_child_at_failpoint( + "child_writes_until_killed", + &root, + ant_node::storage::chunk_store::HALT_BEFORE_PUBLISH, + 5, + ); + + let before = temp_files(&root.join("chunks")); + assert!( + before > 0, + "the child should have left a temporary file behind when it was killed" + ); + + let store = reopen(&root).await; + store.wait_idle().await; + assert_eq!( + temp_files(&root.join("chunks")), + 0, + "the store should sweep what an interrupted write left" + ); + drop(store); +} + +/// How many partly-written files are under `chunks_dir`. +#[cfg(unix)] +fn temp_files(chunks_dir: &Path) -> usize { + let Ok(shards) = std::fs::read_dir(chunks_dir) else { + return 0; + }; + shards + .flatten() + .filter_map(|shard| std::fs::read_dir(shard.path()).ok()) + .flat_map(std::iter::IntoIterator::into_iter) + .flatten() + .filter(|entry| { + entry + .file_name() + .to_str() + .is_some_and(|name| !name.chars().all(|c| c.is_ascii_hexdigit())) + }) + .count() +} + +/// Open the store the way a restart would. +async fn reopen(root: &Path) -> ChunkStore { + let config = ChunkStoreConfig { + root_dir: root.to_path_buf(), + disk_reserve: 0, + ..ChunkStoreConfig::default() + }; + ChunkStore::new(config) + .await + .expect("the store must open after a crash") +} + +/// Deterministic content for chunk `n`. `n` goes in verbatim so no two differ only by a +/// wrap and collapse into one chunk. +fn chunk_bytes(n: usize) -> Vec { + let mut content = vec![0u8; 4096]; + content[..8].copy_from_slice(&(n as u64).to_le_bytes()); + for (i, byte) in content.iter_mut().enumerate().skip(8) { + *byte = ((i.wrapping_mul(17)).wrapping_add(n) % 251) as u8; + } + content +} diff --git a/tests/e2e/data_types/chunk.rs b/tests/e2e/data_types/chunk.rs index 09729b93..650b8e08 100644 --- a/tests/e2e/data_types/chunk.rs +++ b/tests/e2e/data_types/chunk.rs @@ -67,7 +67,7 @@ mod tests { EvmVerifierConfig, PaymentVerifier, PaymentVerifierConfig, PriceFloorConfig, QuoteGenerator, QuotingMetricsTracker, }; - use ant_node::storage::{AntProtocol, LmdbStorage, LmdbStorageConfig}; + use ant_node::storage::{AntProtocol, ChunkStore, ChunkStoreConfig}; use ant_node::ReplicationConfig; use evmlib::testnet::Testnet; use evmlib::RewardsAddress; @@ -355,10 +355,10 @@ mod tests { // Shut down node 0 completely (simulates node restart): // 1. Shut down the replication engine and await its background tasks - // so all Arc clones are released. + // so all Arc clones are released. // 2. Abort the protocol task that holds an Arc. // 3. Drop the node's own Arc. - // This ensures the LMDB env is fully closed before reopening. + // This ensures the chunk store is fully closed before reopening. let data_dir = { let node = harness .network_mut() @@ -433,9 +433,9 @@ mod tests { let temp_dir = std::env::temp_dir().join(format!("{test_name}_{}", rand::random::())); tokio::fs::create_dir_all(&temp_dir).await?; - let storage = LmdbStorage::new(LmdbStorageConfig { + let storage = ChunkStore::new(ChunkStoreConfig { root_dir: temp_dir.clone(), - ..LmdbStorageConfig::test_default() + ..ChunkStoreConfig::test_default() }) .await?; diff --git a/tests/e2e/fetch_local_write_guard.rs b/tests/e2e/fetch_local_write_guard.rs index 8d433542..73025ac8 100644 --- a/tests/e2e/fetch_local_write_guard.rs +++ b/tests/e2e/fetch_local_write_guard.rs @@ -16,14 +16,14 @@ //! counter; the probe scenario is observed through a sender-side count of //! verification requests, since a request that was never sent leaves no trace on //! any receiver. -//! Only `LmdbStorage::get` increments it — the replication fetch responder and +//! Only `ChunkStore::get` increments it — the replication fetch responder and //! the client GET handler; audits read through `get_raw` and leave it alone. //! It is not keyed by chunk or requester, so it is an "it served something" //! signal rather than an exact per-key one; on a freshly built testnet with no //! other traffic to the holder, a delta means it served this fetch. //! //! Two gaps this file deliberately does not close, because neither is -//! constructible without adding test-only hooks to `LmdbStorage`: +//! constructible without adding test-only hooks to `ChunkStore`: //! //! - **Ordering.** Possession is checked before capacity so a full node still //! accepts a key it already holds, matching `put`. Proving it needs a node @@ -112,7 +112,7 @@ async fn ensure_pending_verify(engine: &ReplicationEngine, key: XorName, hinter: /// /// **Phase 1, the dial.** `execute_single_fetch` refuses before the dial, so no /// holder is conscripted. Observed through the holder's `chunks_retrieved` -/// counter: only `LmdbStorage::get` moves it — the replication fetch responder +/// counter: only `ChunkStore::get` moves it — the replication fetch responder /// and the client GET handler — while audits read through `get_raw` and leave it /// alone. It is not keyed by chunk or requester, so it is an "it served /// something" signal rather than an exact per-key one; on a freshly built diff --git a/tests/e2e/fresh_offer_capacity.rs b/tests/e2e/fresh_offer_capacity.rs index 5774c2a0..5b3fcecd 100644 --- a/tests/e2e/fresh_offer_capacity.rs +++ b/tests/e2e/fresh_offer_capacity.rs @@ -47,7 +47,7 @@ const UPLOAD_CHUNKS: usize = 48; /// one test process, so full-size chunks across every node would dominate the /// harness's memory. An admission slot is held per *offer* regardless of /// payload size, so the count above is what stresses the ceiling. The -/// trade-off is that the receiver's LMDB write is faster than production's, +/// trade-off is that the receiver's write is faster than production's, /// which is part of why this is a lower bound. const CHUNK_BYTES: usize = 64 * 1024; diff --git a/tests/e2e/replication.rs b/tests/e2e/replication.rs index 841da90a..adc13dda 100644 --- a/tests/e2e/replication.rs +++ b/tests/e2e/replication.rs @@ -266,7 +266,7 @@ async fn test_fresh_replication_propagates_to_close_group() { /// eviction acts on), via `P2PNode::peer_trust`. #[tokio::test] #[serial] -async fn possession_check_penalises_absent_peer_only() { +async fn possession_check_penalises_absent_peer_only_and_obeys_the_release_switch() { let harness = TestHarness::setup_small().await.expect("setup"); harness.warmup_dht().await.expect("warmup"); @@ -324,6 +324,10 @@ async fn possession_check_penalises_absent_peer_only() { "precondition: C must hold the chunk" ); + // Switched on explicitly, so this half keeps testing the possession mechanism rather + // than whichever release it happens to be compiled against. + ant_node::replication::config::set_close_group_storage_penalty_suspended(false); + let trust_b_before = p2p_a.peer_trust(&peer_b); let trust_c_before = p2p_a.peer_trust(&peer_c); @@ -345,6 +349,25 @@ async fn possession_check_penalises_absent_peer_only() { "present peer C must not be penalised: {trust_c_before} -> {trust_c_after}" ); + // And the other half of the contract, on the same harness. The release that moves + // nodes off the legacy chunk store withholds exactly this penalty: a node short of + // disk cannot avoid answering "absent" while it moves its chunks out of a store that + // never returns space, and it cannot stop its peers penalising it for that, because + // the penalty is the auditor's decision. So the auditors stop one release ahead. + ant_node::replication::config::set_close_group_storage_penalty_suspended(true); + let trust_b_suspended_before = p2p_a.peer_trust(&peer_b); + engine_a + .run_possession_check_now(address, vec![peer_b, peer_c]) + .await; + let trust_b_suspended_after = p2p_a.peer_trust(&peer_b); + ant_node::replication::config::set_close_group_storage_penalty_suspended(false); + + assert!( + trust_b_suspended_after >= trust_b_suspended_before - f64::EPSILON, + "an absent peer must not be penalised while the release withholds that penalty: \ + {trust_b_suspended_before} -> {trust_b_suspended_after}" + ); + harness.teardown().await.expect("teardown"); } @@ -391,6 +414,11 @@ async fn possession_scheduler_penalises_absent_close_peer_after_delay() { .collect(); assert!(!close_group.is_empty(), "expected a non-empty close group"); + // Switched on explicitly. The release that moves nodes off the legacy chunk store + // withholds this penalty by default, so a test that asserts it must say so, or it + // silently starts asserting whichever release it is compiled against. + ant_node::replication::config::set_close_group_storage_penalty_suspended(false); + let trust_before: Vec = close_group.iter().map(|p| p2p_a.peer_trust(p)).collect(); // The checker must hold the chunk it later probes for: the possession check @@ -560,6 +588,11 @@ async fn full_close_group_node_rejects_replica_and_is_penalised_as_absent() { .await .expect("put on checker"); + // Switched on explicitly. The release that moves nodes off the legacy chunk store + // withholds this penalty by default, so a test that asserts it must say so, or it + // silently starts asserting whichever release it is compiled against. + ant_node::replication::config::set_close_group_storage_penalty_suspended(false); + let trust_before = checker_p2p.peer_trust(&full_peer); checker_engine .replicate_fresh(&address, &content, &dummy_payment_proof) @@ -700,7 +733,7 @@ async fn test_paid_list_persistence() { dir }; - // Shut down the replication engine so the LMDB env is released + // Shut down the replication engine so the chunk store is released { let node = harness.network_mut().node_mut(3).expect("node"); if let Some(ref mut engine) = node.replication_engine { @@ -2135,6 +2168,11 @@ async fn scenario_11_repeated_failures_decrease_trust() { let peer_b = *p2p_b.peer_id(); // Get initial trust score for node B (should be neutral ~0.5) + // Switched on explicitly. The release that moves nodes off the legacy chunk store + // withholds this penalty by default, so a test that asserts it must say so, or it + // silently starts asserting whichever release it is compiled against. + ant_node::replication::config::set_close_group_storage_penalty_suspended(false); + let initial_trust = p2p_a.peer_trust(&peer_b); // Report multiple application failures @@ -2878,7 +2916,7 @@ async fn scenario_43_paid_list_persists_across_restart() { dir }; - // Shut down the replication engine so the LMDB env is released + // Shut down the replication engine so the chunk store is released { let node = harness.network_mut().node_mut(3).expect("node"); if let Some(ref mut engine) = node.replication_engine { diff --git a/tests/e2e/subtree_audit_testnet.rs b/tests/e2e/subtree_audit_testnet.rs index bb0b5f70..53a0e1ab 100644 --- a/tests/e2e/subtree_audit_testnet.rs +++ b/tests/e2e/subtree_audit_testnet.rs @@ -3,7 +3,7 @@ //! //! These spin a real multi-node testnet and drive the SHIPPED audit over the //! live wire (real `handle_subtree_challenge` responder + `run_subtree_audit` -//! auditor + real LMDB storage), via the test-only `audit_peer_now` / +//! auditor + a real chunk store), via the test-only `audit_peer_now` / //! `rebuild_commitment_now` engine hooks. They prove the two outcomes that //! matter for a testnet: //! diff --git a/tests/e2e/testnet.rs b/tests/e2e/testnet.rs index a281f5ea..fdd7a725 100644 --- a/tests/e2e/testnet.rs +++ b/tests/e2e/testnet.rs @@ -11,7 +11,7 @@ //! - Message encoding/decoding (postcard serialization) //! - Content address verification //! - Payment verification (when enabled) -//! - LMDB storage persistence +//! - chunk store persistence use ant_node::ant_protocol::{ ChunkGetRequest, ChunkGetResponse, ChunkMessage, ChunkMessageBody, ChunkPutRequest, @@ -23,7 +23,7 @@ use ant_node::payment::{ QuotingMetricsTracker, }; use ant_node::replication::config::MAX_REPLICATION_MESSAGE_SIZE; -use ant_node::storage::{AntProtocol, LmdbStorage, LmdbStorageConfig}; +use ant_node::storage::{AntProtocol, ChunkStore, ChunkStoreConfig}; use ant_node::{ReplicationConfig, ReplicationEngine}; use bytes::Bytes; use evmlib::Network as EvmNetwork; @@ -448,7 +448,7 @@ impl TestNode { info!("Shutting down test node {}", self.index); // Shut down replication engine and await its background tasks so all - // Arc clones are released before we drop the engine. + // Arc clones are released before we drop the engine. if let Some(ref mut engine) = self.replication_engine { engine.shutdown().await; } @@ -1037,7 +1037,7 @@ impl TestNetwork { /// Create a test node (but don't start it yet). /// /// Initializes the `AntProtocol` handler with: - /// - LMDB storage in the node's data directory + /// - the chunk store in the node's data directory /// - Payment verification configured per `TestNetworkConfig` /// - Quote generation with a test rewards address async fn create_node( @@ -1096,18 +1096,18 @@ impl TestNetwork { /// Create an `AntProtocol` handler for a test node. /// /// Configures: - /// - LMDB storage with verification enabled + /// - the chunk store with verification enabled /// - Payment verification (enabled/disabled based on `payment_enforcement`) /// - Quote generator with a test rewards address /// /// # Arguments /// - /// * `data_dir` - Directory for LMDB storage + /// * `data_dir` - Directory for the chunk store /// * `payment_enforcement` - Whether to enable EVM payment verification /// /// # Errors /// - /// Returns an error if LMDB storage initialisation fails. + /// Returns an error if the chunk store cannot be opened. pub async fn create_ant_protocol( data_dir: &std::path::Path, evm_network: Option, @@ -1120,22 +1120,22 @@ impl TestNetwork { /// /// # Errors /// - /// Returns an error if LMDB storage initialisation fails. + /// Returns an error if the chunk store cannot be opened. pub async fn create_ant_protocol_with_disk_reserve( data_dir: &std::path::Path, evm_network: Option, disk_reserve: u64, identity: &saorsa_core::identity::NodeIdentity, ) -> Result { - // Create LMDB storage - let storage_config = LmdbStorageConfig { + // Create the chunk store + let storage_config = ChunkStoreConfig { root_dir: data_dir.to_path_buf(), disk_reserve, - ..LmdbStorageConfig::test_default() + ..ChunkStoreConfig::test_default() }; - let storage = LmdbStorage::new(storage_config) + let storage = ChunkStore::new(storage_config) .await - .map_err(|e| TestnetError::Core(format!("Failed to create LMDB storage: {e}")))?; + .map_err(|e| TestnetError::Core(format!("Failed to create the chunk store: {e}")))?; // Create payment verifier (EVM is always on). // When an EVM network is provided (e.g. Anvil), use it for on-chain verification. diff --git a/tests/poc_audit_handler_live.rs b/tests/poc_audit_handler_live.rs index 03e865b7..a8f15e95 100644 --- a/tests/poc_audit_handler_live.rs +++ b/tests/poc_audit_handler_live.rs @@ -6,7 +6,7 @@ //! `poc_commitment_audit_attacks`. This file fills the remaining gap: the //! *live* responder control-flow branches in //! [`ant_node::replication::storage_commitment_audit::handle_subtree_challenge`] — the function the -//! network actually calls — driven against a real `LmdbStorage` and a real +//! network actually calls — driven against a real `ChunkStore` and a real //! `ResponderCommitmentState`, asserting on the exact `SubtreeAuditResponse` //! variant produced. //! @@ -40,7 +40,7 @@ use ant_node::replication::storage_commitment_audit::{ handle_subtree_challenge, handle_subtree_challenge_measured, handle_subtree_slice_challenge, }; use ant_node::replication::subtree::{verify_subtree_proof, StructureVerdict}; -use ant_node::storage::{LmdbStorage, LmdbStorageConfig}; +use ant_node::storage::{ChunkStore, ChunkStoreConfig}; use saorsa_core::identity::PeerId; use saorsa_pqc::api::sig::{ml_dsa_65, MlDsaPublicKey, MlDsaSecretKey}; use tempfile::TempDir; @@ -49,13 +49,13 @@ use tempfile::TempDir; // Fixtures // --------------------------------------------------------------------------- -async fn test_storage() -> (LmdbStorage, TempDir) { +async fn test_storage() -> (ChunkStore, TempDir) { let temp_dir = TempDir::new().expect("create temp dir"); - let config = LmdbStorageConfig { + let config = ChunkStoreConfig { root_dir: temp_dir.path().to_path_buf(), - ..LmdbStorageConfig::test_default() + ..ChunkStoreConfig::test_default() }; - let storage = LmdbStorage::new(config).await.expect("create storage"); + let storage = ChunkStore::new(config).await.expect("create storage"); (storage, temp_dir) } @@ -81,7 +81,7 @@ impl Responder { /// Build a responder that has stored `indices` and committed to them. /// The committed leaf binds `(address, BLAKE3(content))`; the responder /// reads bytes by address at audit time and rehashes them. - async fn new(storage: &LmdbStorage, indices: &[u8]) -> Self { + async fn new(storage: &ChunkStore, indices: &[u8]) -> Self { let (pk, sk) = keypair(); // Production identity derivation: peer_id == BLAKE3(pubkey_bytes). let peer_id_bytes = *blake3::hash(&pk.to_bytes()).as_bytes(); @@ -90,7 +90,7 @@ impl Responder { let mut entries = Vec::new(); for &i in indices { let content = chunk_content(i); - let addr = LmdbStorage::compute_address(&content); + let addr = ChunkStore::compute_address(&content); storage.put(&addr, &content).await.expect("put chunk"); let bytes_hash = *blake3::hash(&content).as_bytes(); entries.push((addr, bytes_hash)); @@ -112,7 +112,7 @@ impl Responder { } fn address(i: u8) -> [u8; 32] { - LmdbStorage::compute_address(&chunk_content(i)) + ChunkStore::compute_address(&chunk_content(i)) } } @@ -360,7 +360,7 @@ async fn committed_key_with_missing_bytes_is_rejected() { /// A successful proof reports what it read and hashed, at a floor per leaf. /// Anchors the rejection case below: it fixes what the measurement means. /// -/// A leaf costs more than its bytes — an LMDB lookup and a blocking-task round +/// A leaf costs more than its bytes — a store lookup and a blocking-task round /// trip are owed whatever its size — and nothing bounds a chunk from below, so /// the charge is `max(content, floor)` per leaf. These test records are 1 KiB, /// well under the floor, which is the case that used to be nearly free: the @@ -739,7 +739,7 @@ async fn slice_challenge_opens_a_deep_block_of_a_large_chunk() { let content: Vec = (0..100_000u32) .map(|n| (n.wrapping_mul(2_654_435_761) >> 13) as u8) .collect(); - let addr = LmdbStorage::compute_address(&content); + let addr = ChunkStore::compute_address(&content); storage.put(&addr, &content).await.expect("put chunk"); let bytes_hash = *blake3::hash(&content).as_bytes(); diff --git a/tests/poc_shutdown_lmdb_drain.rs b/tests/poc_shutdown_lmdb_drain.rs deleted file mode 100644 index 699765fb..00000000 --- a/tests/poc_shutdown_lmdb_drain.rs +++ /dev/null @@ -1,192 +0,0 @@ -//! Regression test for the LMDB drain guarantee of -//! [`ant_node::ReplicationEngine::shutdown`]. -//! -//! ## The vulnerability (pre-fix) -//! -//! Engine tasks race their work against the shutdown `CancellationToken` in -//! `select!`. Dropping the losing future does **not** cancel a -//! `tokio::task::spawn_blocking` LMDB transaction it was awaiting — the -//! closure keeps running on the blocking pool and owns a cloned heed `Env`. -//! `shutdown()` had nothing to wait on for those detached closures (fetch -//! `storage.put`, prune `storage.delete` / `paid_list.remove_batch`, -//! verification `paid_list.insert`), so it could return while the -//! environment was still open. Reopening the same LMDB file with the old -//! `Env` alive in-process is undefined behavior. -//! -//! ## The fix -//! -//! `LmdbStorage` and `PaidList` track their blocking tasks in a -//! `TaskTracker`; `shutdown()` awaits `wait_idle()` on both after draining -//! its own tasks. This test parks a chunk-store write inside its blocking -//! closure, drops the awaiter (the exact leak shape), and asserts that -//! `shutdown()` blocks until the write finishes — then proves both LMDB -//! environments reopen cleanly. - -#![allow( - clippy::unwrap_used, - clippy::expect_used, - clippy::panic, - clippy::missing_panics_doc -)] - -use ant_node::payment::{ - EvmVerifierConfig, PaymentVerifier, PaymentVerifierConfig, PriceFloorConfig, -}; -use ant_node::replication::paid_list::PaidList; -use ant_node::storage::{LmdbStorage, LmdbStorageConfig}; -use ant_node::{ReplicationConfig, ReplicationEngine}; -use evmlib::{Network as EvmNetwork, RewardsAddress}; -use rand::Rng; -use saorsa_core::identity::NodeIdentity; -use saorsa_core::{NodeConfig as CoreNodeConfig, P2PNode}; -use std::sync::Arc; -use std::time::Duration; -use tokio_util::sync::CancellationToken; - -/// E2E test port range (CLAUDE.md): tests must stay inside 20000-60000, -/// away from production ant-node's 10000-10999. -const TEST_PORT_RANGE_MIN: u16 = 20_000; -/// Upper bound (exclusive) of the E2E test port range. -const TEST_PORT_RANGE_MAX: u16 = 60_000; -/// Attempts to bind a random test port before giving up (mirrors the -/// transient port-bind retry in the e2e testnet harness). -const PORT_BIND_ATTEMPTS: usize = 4; -/// Short probe proving `shutdown()` is still waiting on the parked LMDB op. -const SHUTDOWN_BLOCKED_PROBE: Duration = Duration::from_millis(300); -/// Generous ceiling for `shutdown()` to finish once the op is released. -const SHUTDOWN_COMPLETE_TIMEOUT: Duration = Duration::from_secs(30); -/// Payment cache capacity for the test verifier. -const TEST_PAYMENT_CACHE_CAPACITY: usize = 1000; -/// Rewards address for the test verifier. -const TEST_REWARDS_ADDRESS: [u8; 20] = [0x01; 20]; - -/// Create and start a loopback P2P node on a random port in the test range. -async fn start_p2p_node(identity: &Arc) -> Arc { - let mut last_err = String::new(); - for _ in 0..PORT_BIND_ATTEMPTS { - let port = rand::thread_rng().gen_range(TEST_PORT_RANGE_MIN..TEST_PORT_RANGE_MAX); - let mut config = CoreNodeConfig::builder() - .port(port) - .ipv6(false) - .local(true) - .build() - .expect("build core config"); - config.node_identity = Some(Arc::clone(identity)); - match P2PNode::new(config).await { - Ok(node) => { - node.start().await.expect("start p2p node"); - return Arc::new(node); - } - Err(e) => last_err = e.to_string(), - } - } - panic!("failed to create P2P node after {PORT_BIND_ATTEMPTS} attempts: {last_err}"); -} - -/// A blocking LMDB write whose awaiter was dropped must delay `shutdown()` -/// until it commits, after which both LMDB environments reopen cleanly. -// Holding the gate's write guard across awaits is the point of the test: -// it parks the blocking closure while we probe shutdown(). -#[allow(clippy::await_holding_lock)] -#[tokio::test] -async fn shutdown_waits_for_detached_lmdb_op_and_envs_reopen() { - let temp_dir = tempfile::TempDir::new().expect("create temp dir"); - let root_dir = temp_dir.path().to_path_buf(); - - // The chunk store the engine will hold (and whose env we reopen below). - let storage = Arc::new( - LmdbStorage::new(LmdbStorageConfig { - root_dir: root_dir.clone(), - ..LmdbStorageConfig::test_default() - }) - .await - .expect("create storage"), - ); - - let identity = Arc::new(NodeIdentity::generate().expect("generate identity")); - let replication_config = ReplicationConfig::default(); - let payment_verifier = Arc::new(PaymentVerifier::new(PaymentVerifierConfig { - evm: EvmVerifierConfig { - network: EvmNetwork::ArbitrumSepoliaTest, - }, - cache_capacity: TEST_PAYMENT_CACHE_CAPACITY, - close_group_size: replication_config.close_group_size, - local_rewards_address: RewardsAddress::new(TEST_REWARDS_ADDRESS), - price_floor: PriceFloorConfig::default(), - })); - - let p2p = start_p2p_node(&identity).await; - - let (_fresh_tx, fresh_rx) = tokio::sync::mpsc::unbounded_channel(); - let mut engine = ReplicationEngine::new( - replication_config, - Arc::clone(&p2p), - Arc::clone(&storage), - payment_verifier, - identity, - &root_dir, - fresh_rx, - CancellationToken::new(), - ) - .await - .expect("create engine"); - engine.start(p2p.dht_manager().subscribe_events()); - - // Park a put's blocking closure on the test gate, then drop its awaiter - // mid-flight — the exact shape of a select! losing to the shutdown token - // while `storage.put()` awaits `spawn_blocking`. - let content = b"held-open write must block engine shutdown"; - let address = LmdbStorage::compute_address(content); - let gate = storage.test_put_gate(); - let parked = gate.write(); - tokio::select! { - biased; - res = storage.put(&address, content) => { - panic!("put must be parked on the test gate, got {res:?}") - } - () = std::future::ready(()) => {} - } - - { - let shutdown_fut = engine.shutdown(); - tokio::pin!(shutdown_fut); - - // shutdown() must not return while the blocking op is still running. - let blocked = tokio::time::timeout(SHUTDOWN_BLOCKED_PROBE, shutdown_fut.as_mut()).await; - assert!( - blocked.is_err(), - "shutdown() returned while an LMDB blocking op was in flight" - ); - - // Release the write; shutdown must now run to completion. - drop(parked); - tokio::time::timeout(SHUTDOWN_COMPLETE_TIMEOUT, shutdown_fut) - .await - .expect("shutdown after releasing the parked op"); - } - - // The detached write committed before shutdown returned. - assert!(storage.exists(&address).expect("exists after shutdown")); - - // Release every reference the test still holds. Per the shutdown - // contract, no engine-spawned work holds the storage or paid list any - // more, so these drops close both environments. - drop(engine); - p2p.shutdown().await.expect("p2p shutdown"); - drop(p2p); - drop(gate); - drop(storage); - - // Both LMDB environments reopen cleanly from the same directory. - let reopened = LmdbStorage::new(LmdbStorageConfig { - root_dir: root_dir.clone(), - ..LmdbStorageConfig::test_default() - }) - .await - .expect("reopen chunk store"); - let read_back = reopened.get(&address).await.expect("get after reopen"); - assert_eq!(read_back, Some(content.to_vec())); - - let paid_list = PaidList::new(&root_dir).await.expect("reopen paid list"); - assert_eq!(paid_list.count().expect("paid list count"), 0); -} diff --git a/tests/storage_scale.rs b/tests/storage_scale.rs new file mode 100644 index 00000000..d5ac2627 --- /dev/null +++ b/tests/storage_scale.rs @@ -0,0 +1,500 @@ +//! What one file per chunk costs at scale. +//! +//! The design accepted two costs on paper and never measured either: the startup scan +//! reads every filename in the store before the node serves anything, and every chunk +//! takes an inode and a directory entry. Both grow with the store, and a node that takes +//! minutes to start, or runs a filesystem out of inodes, is a node that is down. +//! +//! These are regression gates, not benchmarks. The ceilings are generous enough that a +//! loaded shared runner does not fail them and tight enough that an order-of-magnitude +//! regression does. What they measure precisely is printed, so a number that is drifting +//! is visible in the log before it ever trips the gate. +//! +//! `ANT_SCALE_KEYS` raises the count for a deliberate larger run. The default is what a +//! hosted runner can do in reasonable time; the fleet-scale figures the ADR wants still +//! need a machine with the disk for them. + +#![allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::missing_panics_doc, + // Test fixtures: every cast here is of a bounded loop counter into a byte, and the + // wrap is what makes the fill vary. + clippy::cast_possible_truncation +)] + +use ant_node::storage::{ChunkStore, ChunkStoreConfig}; +use std::path::Path; +use std::time::{Duration, Instant}; +use tempfile::TempDir; + +/// Keys to plant unless told otherwise. +const DEFAULT_KEYS: usize = 100_000; + +/// The longest a cold scan may take per chunk before this is a regression. +/// +/// Measured at 1,019 ns per key for 100,000 keys on a hosted CI runner. Fifty times that +/// leaves a slow, loaded, shared runner room to be slow while still catching a scan that +/// has gone from linear to something worse: the whole 100,000-key budget is five seconds +/// against a hundred milliseconds measured. +/// +/// Per key rather than a flat number, so that raising `ANT_SCALE_KEYS` for a larger run +/// raises the allowance with it instead of turning the gate into a coin toss. +const SCAN_CEILING_PER_KEY: Duration = Duration::from_micros(50); + +/// The most a startup scan may read, whatever the store holds. +/// +/// Fixed, deliberately, and not a fraction of the payload. A fraction grows with the +/// store, so it would keep permitting a per-chunk read as long as the chunks were big +/// enough: at the sizes below, a hundredth of the payload allowed 655 bytes per chunk, +/// which is a header read of every file in the store passing a test named for not doing +/// that. +/// +/// A scan that reads names reads the same handful of bytes whatever the store holds. +/// Measured at 125 bytes for 3,000 chunks on a hosted runner, which is the layout marker +/// and nothing else. 64 KiB is five hundred times that and still under 22 bytes per chunk +/// there, so any read that is per-chunk at all fails, and fails harder the larger the run. +#[cfg(target_os = "linux")] +const SCAN_READ_CEILING: u64 = 64 * 1024; + +/// How many keys this run should plant. +fn key_count() -> usize { + std::env::var("ANT_SCALE_KEYS") + .ok() + .and_then(|raw| raw.parse().ok()) + .unwrap_or(DEFAULT_KEYS) +} + +/// Plant `count` chunk files directly, without going through the store. +/// +/// Writing them by hand rather than through `put` is the point: this measures opening a +/// store that already holds them, which is what a restart does, not the cost of filling +/// one. +fn plant_chunks(chunks_dir: &Path, count: usize) { + for shard in 0u16..256 { + std::fs::create_dir_all(chunks_dir.join(format!("{shard:02x}"))).expect("mkdir"); + } + // One byte each. The scan reads names, never contents, so the payload would only cost + // the test disk it does not need. + for n in 0..count { + let mut address = [0u8; 32]; + address[..8].copy_from_slice(&(n as u64).to_le_bytes()); + // The shard is the last byte, so spread across all 256 rather than piling into one. + address[31] = (n % 256) as u8; + let path = chunks_dir + .join(format!("{:02x}", address[31])) + .join(hex::encode(address)); + std::fs::write(path, b"x").expect("plant a chunk"); + } +} + +/// Walk every shard under `chunks_dir`, optionally calling `metadata` on each entry. +/// +/// The control for the assertion above: the same directory, the same process, the same +/// moment, with and without the one syscall the design says the scan does not make. +fn walk(chunks_dir: &Path, stat_each: bool) -> Duration { + let started = Instant::now(); + let mut seen = 0usize; + if let Ok(shards) = std::fs::read_dir(chunks_dir) { + for shard in shards.flatten() { + let Ok(entries) = std::fs::read_dir(shard.path()) else { + continue; + }; + for entry in entries.flatten() { + // Touched so the name is not optimised away, exactly as the scan uses it. + seen += entry.file_name().as_encoded_bytes().len(); + if stat_each { + seen += usize::from(entry.metadata().is_ok()); + } + } + } + } + assert!(seen > 0, "the control walk found nothing to walk"); + started.elapsed() +} + +/// Resident memory of this process, in bytes, where the platform will say. +#[cfg(target_os = "linux")] +fn resident_bytes() -> Option { + let status = std::fs::read_to_string("/proc/self/status").ok()?; + status + .lines() + .find_map(|line| line.strip_prefix("VmRSS:")) + .and_then(|value| value.split_whitespace().next()?.parse::().ok()) + .map(|kb| kb * 1024) +} + +/// Not every platform makes this cheap to ask, and the gate below is the scan time. +#[cfg(not(target_os = "linux"))] +fn resident_bytes() -> Option { + None +} + +/// Opening a store that already holds a large number of chunks stays quick. +/// +/// This is the first thing a restarted node does and nothing is served until it finishes, +/// so it is the cost that decides whether a big node can be restarted at all. +#[tokio::test] +async fn opening_a_large_store_stays_quick() { + let keys = key_count(); + let tmp = TempDir::new().expect("temp dir"); + let root = tmp.path().join("node"); + let chunks_dir = root.join("chunks"); + std::fs::create_dir_all(&chunks_dir).expect("mkdir"); + + let planting = Instant::now(); + plant_chunks(&chunks_dir, keys); + let planted = planting.elapsed(); + + let before = resident_bytes(); + let opening = Instant::now(); + let store = ChunkStore::new(ChunkStoreConfig { + root_dir: root.clone(), + verify_on_read: true, + disk_reserve: 0, + }) + .await + .expect("open a store holding a large number of chunks"); + let scan = opening.elapsed(); + let after = resident_bytes(); + + let indexed = store.current_chunks().expect("count"); + assert_eq!( + indexed as usize, keys, + "the scan must find every planted chunk" + ); + + let per_key_ns = scan.as_nanos() / keys.max(1) as u128; + let growth = match (before, after) { + (Some(before), Some(after)) => format!("{} KiB", after.saturating_sub(before) / 1024), + _ => "not measured on this platform".to_string(), + }; + println!( + "scale: {keys} chunks planted in {planted:?}, scanned in {scan:?} \ + ({per_key_ns} ns/key), resident growth {growth}" + ); + + let ceiling = SCAN_CEILING_PER_KEY * u32::try_from(keys).unwrap_or(u32::MAX); + assert!( + scan < ceiling, + "scanning {keys} chunks took {scan:?} ({per_key_ns} ns/key), over the {ceiling:?} \ + ceiling" + ); + + drop(store); + + // And the scan reads names only, with no `stat` behind each one. That claim is what + // the cost above rests on, and a flat ceiling cannot settle it: one `stat` per entry + // costs about three times a bare walk, which is still far inside any ceiling loose + // enough not to flake on a shared runner. + // + // Measured against this machine instead of against a number. Two walks of the same + // directory, one reading names and one calling `metadata` on each, bracket what a scan + // of this store on this filesystem under this load costs. A scan that stats every entry + // lands at the far bracket. + // + // Three rounds, interleaved, and the median of each. One round of each would let a + // scheduling pause that happened to land on the scan and not on the walks decide the + // result: runner speed only cancels out when it moves all three together, and a + // preemption does not. Interleaving puts the three measurements next to each other in + // time and the median throws away the round that was interrupted. + let mut scans = Vec::new(); + let mut bare = Vec::new(); + let mut stats = Vec::new(); + for _ in 0..3 { + scans.push(time_a_scan(&root).await); + bare.push(walk(&chunks_dir, false)); + stats.push(walk(&chunks_dir, true)); + } + let scan = median(&mut scans); + let names_only = median(&mut bare); + let with_stat = median(&mut stats); + // Saturating, because a filesystem where a stat costs nothing would otherwise + // underflow here. On one of those the midpoint collapses onto the bare walk and this + // says little, which is the honest answer for such a filesystem. + let midpoint = names_only + with_stat.saturating_sub(names_only) / 2; + println!( + "scale: medians of three, bare walk {names_only:?} names only, {with_stat:?} with a \ + stat each, store scan {scan:?}, midpoint {midpoint:?}" + ); + assert!( + scan < midpoint, + "the scan took {scan:?}, past the {midpoint:?} midpoint between a names-only walk \ + ({names_only:?}) and one that stats every entry ({with_stat:?}), so it is doing \ + more per entry than reading a name" + ); +} + +/// Open a store at `root`, time the scan, and close it again. +async fn time_a_scan(root: &Path) -> Duration { + let started = Instant::now(); + let store = ChunkStore::new(ChunkStoreConfig { + root_dir: root.to_path_buf(), + verify_on_read: true, + disk_reserve: 0, + }) + .await + .expect("reopen the store"); + let elapsed = started.elapsed(); + drop(store); + elapsed +} + +/// The middle of three, so one interrupted round does not decide anything. +fn median(samples: &mut [Duration]) -> Duration { + samples.sort_unstable(); + samples.get(samples.len() / 2).copied().unwrap_or_default() +} + +/// The index costs a bounded amount of memory per chunk. +/// +/// One inode and one directory entry per chunk is the filesystem's share, and the ADR +/// accepts it. What it did not measure is the node's own share: an in-memory set of every +/// address, which is the part that could quietly make a large node unrunnable. +/// +/// Measured in a process of its own, which is the only way this measurement means +/// anything. `VmRSS` is process-wide and the allocator hands back what earlier work freed, +/// so opening a store in a process that has already opened and dropped one grows the +/// resident set by nothing at all. That is exactly what happened here: the test read zero +/// bytes per chunk and passed, having measured the allocator rather than the index. A child +/// that has done nothing else has no freed heap to reuse. +#[cfg(target_os = "linux")] +#[tokio::test] +async fn the_in_memory_index_costs_a_bounded_amount_per_chunk() { + let keys = key_count(); + let tmp = TempDir::new().expect("temp dir"); + let root = tmp.path().join("node"); + let chunks_dir = root.join("chunks"); + std::fs::create_dir_all(&chunks_dir).expect("mkdir"); + plant_chunks(&chunks_dir, keys); + + let per_key = index_cost_in_a_fresh_process(&root, keys); + println!("scale: index costs {per_key} bytes per chunk, measured in its own process"); + + // A 32-byte address in a sorted set, plus allocator and node overhead. Measured at 52 + // bytes per chunk on a hosted runner; 128 is comfortably above that and no longer five + // times it, which was loose enough to let an extra 128 bytes a key through unnoticed. + assert!( + per_key < 128, + "the index costs {per_key} bytes per chunk, which does not scale" + ); + // Zero is not a pass. It is what this test reported when it shared a process with one + // that had already opened and dropped a store of the same size, and it would report it + // again if the child ever stopped opening the store at all. + assert!( + per_key > 0, + "the index reported no cost at all, so nothing was measured" + ); +} + +/// Open a store of `keys` chunks in a child process and report its resident growth per key. +#[cfg(target_os = "linux")] +fn index_cost_in_a_fresh_process(root: &Path, keys: usize) -> u64 { + let exe = std::env::current_exe().expect("this test binary"); + let output = std::process::Command::new(exe) + .arg("--exact") + .arg("child_reports_index_memory") + .arg("--nocapture") + .arg("--ignored") + .env("ANT_SCALE_ROOT", root) + .env("ANT_SCALE_KEYS", keys.to_string()) + .output() + .expect("spawn the child"); + let said = String::from_utf8_lossy(&output.stdout); + assert!( + output.status.success(), + "the child failed: {said}{}", + String::from_utf8_lossy(&output.stderr) + ); + said.lines() + .find_map(|line| line.strip_prefix(INDEX_BYTES_PER_KEY)) + .and_then(|value| value.trim().parse().ok()) + .unwrap_or_else(|| panic!("the child reported no measurement: {said}")) +} + +/// What the child prints its answer behind. +#[cfg(target_os = "linux")] +const INDEX_BYTES_PER_KEY: &str = "INDEX_BYTES_PER_KEY="; + +/// Child mode: open the store named by the environment and report what it cost. +#[cfg(target_os = "linux")] +#[tokio::test] +#[ignore = "child process of the index memory measurement, not run on its own"] +async fn child_reports_index_memory() { + let root = std::path::PathBuf::from( + std::env::var("ANT_SCALE_ROOT").expect("the child needs a store to open"), + ); + let keys = key_count(); + + let before = resident_bytes().expect("linux reports this"); + let store = ChunkStore::new(ChunkStoreConfig { + root_dir: root, + verify_on_read: true, + disk_reserve: 0, + }) + .await + .expect("open"); + let after = resident_bytes().expect("linux reports this"); + + assert_eq!(store.current_chunks().expect("count") as usize, keys); + let grew = after.saturating_sub(before); + println!("{INDEX_BYTES_PER_KEY}{}", grew / keys.max(1) as u64); + // Held until after the measurement is printed, so the index is still resident when it + // is read rather than freed by an early drop. + drop(store); +} + +/// Every chunk the store writes takes exactly one directory entry. +/// +/// Through `put`, not through the fixture. An earlier version planted the files itself +/// and then counted them, which proves the test can count and nothing about the store: a +/// store that wrote a sidecar beside every chunk would have passed it. +/// +/// It matters because a filesystem runs out of inodes independently of bytes, and a node +/// that fills the inode table stops accepting writes while `df` still shows free space. +#[tokio::test] +async fn each_chunk_the_store_writes_costs_one_directory_entry() { + let keys = 2_000; + let tmp = TempDir::new().expect("temp dir"); + let root = tmp.path().join("node"); + std::fs::create_dir_all(&root).expect("mkdir"); + + let store = ChunkStore::new(ChunkStoreConfig { + root_dir: root.clone(), + verify_on_read: true, + disk_reserve: 0, + }) + .await + .expect("open"); + + for n in 0..keys { + // Real content through the real path, so anything `put` writes is counted. + let mut content = vec![0u8; 512]; + content[..8].copy_from_slice(&(n as u64).to_le_bytes()); + let address = ant_node::client::compute_address(&content); + store.put(&address, &content).await.expect("put"); + } + store.wait_idle().await; + + let entries = count_entries(&root.join("chunks")); + assert_eq!( + entries.files, keys, + "the store wrote {} files for {keys} chunks", + entries.files + ); + // 256 shards and the layout marker are the fixed overhead; nothing should be + // proportional to the chunk count but the chunks themselves. + assert!( + entries.dirs <= 256, + "the store made {} directories, which grows with the store", + entries.dirs + ); +} + +/// Files and directories under a path, counted rather than summed. +struct Entries { + files: usize, + dirs: usize, +} + +fn count_entries(path: &Path) -> Entries { + let mut counted = Entries { files: 0, dirs: 0 }; + let Ok(entries) = std::fs::read_dir(path) else { + return counted; + }; + for entry in entries.flatten() { + match entry.file_type() { + Ok(kind) if kind.is_dir() => { + counted.dirs += 1; + let nested = count_entries(&entry.path()); + counted.files += nested.files; + counted.dirs += nested.dirs; + } + // The store's own two files sit beside the shards and are not chunks: the + // layout marker, and the lock that keeps a second process out. Both are + // fixed, so neither grows with the store. + Ok(_) + if entry.file_name() == ant_node::storage::chunk_store::LAYOUT_FILE_NAME + || entry.file_name() == ".lock" => {} + Ok(_) => counted.files += 1, + Err(_) => {} + } + } + counted +} + +/// The startup scan does not read chunk contents. +/// +/// The claim the scan's cost rests on: a store of 4 MiB chunks would be unopenable if +/// starting meant reading them. +/// +/// Measured in bytes read, not in elapsed time. Timing cannot settle this: the files were +/// written moments earlier, so reading them back comes from the page cache and costs +/// almost nothing. A version of this test that compared durations passed with a +/// deliberate `read` of every file added to the scan. `rchar` counts what the process +/// asked the kernel for whether or not the answer was cached, which is the question. +/// +/// Linux only, for `/proc/self/io`. Nothing about the scan is platform-specific, and this +/// is the platform where the answer can be had exactly. +#[cfg(target_os = "linux")] +#[tokio::test] +async fn the_startup_scan_does_not_read_chunk_contents() { + let keys = 3_000; + let chunk = 64 * 1024; + let tmp = TempDir::new().expect("temp dir"); + let root = tmp.path().join("node"); + plant_sized(&root.join("chunks"), keys, chunk); + + let before = bytes_read().expect("linux reports this"); + let store = ChunkStore::new(ChunkStoreConfig { + root_dir: root.clone(), + verify_on_read: true, + disk_reserve: 0, + }) + .await + .expect("open"); + let read = bytes_read() + .expect("linux reports this") + .saturating_sub(before); + + let payload = (keys * chunk) as u64; + println!( + "scale: opening a store of {keys} chunks read {read} bytes, against {payload} \ + bytes of chunk" + ); + assert_eq!(store.current_chunks().expect("count") as usize, keys); + + assert!( + read < SCAN_READ_CEILING, + "the scan read {read} bytes of a {payload} byte store, over the \ + {SCAN_READ_CEILING} byte ceiling, so it is reading contents" + ); +} + +/// Bytes this process has asked the kernel to read, cached or not. +#[cfg(target_os = "linux")] +fn bytes_read() -> Option { + let io = std::fs::read_to_string("/proc/self/io").ok()?; + io.lines() + .find_map(|line| line.strip_prefix("rchar:")) + .and_then(|value| value.trim().parse().ok()) +} + +/// Plant `count` chunk files of `bytes` each. +#[cfg(target_os = "linux")] +fn plant_sized(chunks_dir: &Path, count: usize, bytes: usize) { + for shard in 0u16..256 { + std::fs::create_dir_all(chunks_dir.join(format!("{shard:02x}"))).expect("mkdir"); + } + let payload = vec![7u8; bytes]; + for n in 0..count { + let mut address = [0u8; 32]; + address[..8].copy_from_slice(&(n as u64).to_le_bytes()); + address[31] = (n % 256) as u8; + let path = chunks_dir + .join(format!("{:02x}", address[31])) + .join(hex::encode(address)); + std::fs::write(path, &payload).expect("plant a chunk"); + } +}