Record a storage schema version, and repair the history encoding

ledger/store/Cargo.toml

@@ -16,14 +16,19 @@ categories = [ "cryptography", "web-programming" ]
 license = "Apache-2.0"
 edition = "2024"
 
+[[bin]]
+name = "snarkvm-migrate-db"
+path = "src/bin/migrate_db.rs"
+required-features = [ "rocks" ]
+
 [features]
 default = [ "indexmap/rayon" ]
 history = [ ]
 history-staking-rewards = [ ]
 slipstream-plugins = [ "dep:snarkvm-slipstream-plugin-manager" ]
 locktick = [ "dep:locktick", "snarkvm-ledger-puzzle/locktick" ]
 metrics = [ "dep:snarkvm-metrics" ]
-rocks = [ "rocksdb", "smallvec" ]
+rocks = [ "rocksdb", "smallvec", "dep:tracing-subscriber" ]
 serial = [
   "snarkvm-console/serial",
   "snarkvm-ledger-block/serial",
@@ -124,6 +129,15 @@ optional = true
 [dependencies.tracing]
 workspace = true
 
+# Used only by the migration binary, for its progress output. Reviewers have asked for this
+# to be kept out of the library's dependency graph; deferred because the build command in
+# `snarkvm-migrate-db`'s documentation is already in a partner's hands, and a bin-scoped
+# feature would change it.
+[dependencies.tracing-subscriber]
+version = "0.3"
+features = [ "env-filter" ]
+optional = true
+
 [dev-dependencies.aleo-std]
 workspace = true
 

ledger/store/src/bin/migrate_db.rs

@@ -0,0 +1,212 @@
+// Copyright (c) 2019-2026 Provable Inc.
+// This file is part of the snarkVM library.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+
+// http://www.apache.org/licenses/LICENSE-2.0
+
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Brings a ledger's storage schema up to the version this build requires.
+//!
+//! Separate from the node deliberately. The work can rewrite billions of entries and take hours,
+//! which is not something a node should do as a side effect of starting: an operator wants to run
+//! it when they choose, against a copy first, watching it, able to stop and resume it. The node
+//! only checks the version and refuses to run against a schema it does not understand.
+//!
+//! ```text
+//! snarkvm-migrate-db --check ~/.aleo/storage/ledger-0   # read-only: what would this do?
+//! snarkvm-migrate-db         ~/.aleo/storage/ledger-0   # do it
+//! ```
+//!
+//! `--check` opens the database read-only and writes nothing, so it is safe against a copy of a
+//! production ledger. Without it the node must be stopped: RocksDB permits a single writer, so a
+//! migration will refuse to open a ledger a node still holds.
+
+use anyhow::{Result, bail};
+use snarkvm_ledger_store::helpers::rocksdb::{PREFIX_LEN, STORAGE_VERSION, is_resuming, migrate, plan, schema_version};
+
+/// Entries per second, measured migrating a 234-million-entry ledger end to end.
+///
+/// An earlier value of 300,000 came from synthetic benchmarks on a freshly built database and was
+/// roughly seven times optimistic: it advertised thirteen minutes for a run that took ninety. An
+/// operator plans a maintenance window around this number, so it is taken from a real migration
+/// (m7i.4xlarge, gp3) rather than from a microbenchmark, and rounded down.
+const ENTRIES_PER_SECOND: f64 = 40_000.0;
+
+/// The ledger to act on, and whether to only report.
+struct Args {
+    path: String,
+    network_id: u16,
+    check: bool,
+}
+
+const USAGE: &str = "\
+usage: snarkvm-migrate-db [--check] <ledger-dir> [network-id]
+
+    snarkvm-migrate-db --check ~/.aleo/storage/ledger-0    report what a migration would do
+    snarkvm-migrate-db         ~/.aleo/storage/ledger-0    perform it
+
+--check opens the ledger read-only and writes nothing. Without it, stop the node first.
+The network is inferred from a directory name ending in `-<id>` (0 = mainnet, 1 = testnet,
+2 = canary), or given as the final argument.";
+
+/// Returns the network id encoded in a ledger directory name, if it has one.
+///
+/// Production ledgers are `ledger-{network}`; development ones are `.ledger-{network}-{id}`, so a
+/// naive split from the right yields the dev id instead. Getting this wrong is quiet rather than
+/// loud -- the tool finds no history under the wrong prefix, reports nothing to do, and leaves the
+/// real history untouched -- so an id outside the known range is refused rather than assumed.
+fn network_from_name(path: &str) -> Option<u16> {
+    let name = std::path::Path::new(path).file_name()?.to_str()?;
+    let rest = name.strip_prefix('.').unwrap_or(name).strip_prefix("ledger-")?;
+    let network = rest.split('-').next()?.parse().ok()?;
+    // 0 = mainnet, 1 = testnet, 2 = canary. Anything else is a misread name.
+    (network <= 2).then_some(network)
+}
+
+fn parse() -> Result<Args> {
+    let mut check = false;
+    let mut positional = Vec::new();
+    for arg in std::env::args().skip(1) {
+        match arg.as_str() {
+            "--check" | "-c" => check = true,
+            // Asking for help is not an error.
+            "--help" | "-h" => {
+                println!("{USAGE}");
+                std::process::exit(0);
+            }
+            flag if flag.starts_with('-') => bail!("Unrecognized option {flag:?}\n\n{USAGE}"),
+            _ => positional.push(arg),
+        }
+    }
+
+    let Some(path) = positional.first().cloned() else { bail!("{USAGE}") };
+    let network_id = match positional.get(1) {
+        Some(explicit) => explicit.parse()?,
+        None => match network_from_name(&path) {
+            Some(id) => id,
+            None => bail!("Could not infer the network from {path:?}.\n\n{USAGE}"),
+        },
+    };
+    Ok(Args { path, network_id, check })
+}
+
+/// How many SST files RocksDB may hold open at once.
+///
+/// Deliberately far below the node's own limit. Left at RocksDB's default of unlimited, a tool
+/// scanning a multi-terabyte ledger opens every SST and keeps each one's index and filter blocks
+/// resident: on a 1.7 TiB ledger that exhausted the default 1024 file-descriptor limit outright,
+/// and reached 4.6 GB of resident memory where the work itself needs a few hundred megabytes.
+///
+/// A node wants a high limit because it serves random reads. This tool makes one sequential pass,
+/// so it gains almost nothing from keeping files open, and a small bound keeps it inside the
+/// default `ulimit -n` that an operator's shell will have.
+const MAX_OPEN_FILES: i32 = 256;
+
+/// The options a node opens the ledger with, save for the open-file bound.
+///
+/// The prefix extractor matters: it puts iteration into prefix-seek mode, so inspecting or
+/// migrating under different options would not be operating on the database as the node sees it.
+fn options() -> rocksdb::Options {
+    let mut options = rocksdb::Options::default();
+    options.set_compression_type(rocksdb::DBCompressionType::Lz4);
+    options.set_prefix_extractor(rocksdb::SliceTransform::create_fixed_prefix(PREFIX_LEN));
+    options.set_max_open_files(MAX_OPEN_FILES);
+    options
+}
+
+fn main() -> Result<()> {
+    let args = parse()?;
+
+    // Read-only for --check, so that inspecting a ledger cannot modify it. Note this may read a
+    // database whose write-ahead log has not been replayed, so on a ledger left by an unclean
+    // shutdown the very latest entries may not be visible.
+    let database = match args.check {
+        true => rocksdb::DB::open_for_read_only(&options(), &args.path, false)?,
+        false => rocksdb::DB::open(&options(), &args.path)?,
+    };
+
+    // Two point lookups before any scan: an already-migrated ledger should not be walked end to
+    // end just to print a status line, and neither should one being resumed.
+    // Compared against the version this build needs, not against zero: a tool that treats "any
+    // version at all" as done would refuse to run every future migration, telling an operator
+    // there is nothing to do about the very thing their node is refusing to start over.
+    if schema_version(&database, args.network_id)? >= STORAGE_VERSION {
+        println!("{} is already at storage schema v{STORAGE_VERSION}. Nothing to do.", args.path);
+        return Ok(());
+    }
+    if is_resuming(&database, args.network_id)? {
+        // A half-migrated map holds both encodings on the keys it had begun, so classifying it
+        // would report a ledger as unrepairable that would in fact finish perfectly well.
+        if args.check {
+            println!("A migration of {} was interrupted and has not finished.\n", args.path);
+            println!("Its state cannot be assessed while partly migrated -- the keys it had begun");
+            println!("hold both encodings. Re-run without --check to finish it:\n");
+            println!("    snarkvm-migrate-db {}", args.path);
+            return Ok(());
+        }
+        println!("Resuming an interrupted migration of {}\n", args.path);
+        tracing_subscriber::fmt().with_env_filter("info").with_target(false).init();
+        migrate(&database, args.network_id, None)?;
+        println!("\nMigration complete. The node can now be started.");
+        return Ok(());
+    }
+
+    let report = plan(&database, args.network_id)?;
+
+    let total = report.little_endian + report.big_endian;
+    println!("{} (network {})\n", args.path, args.network_id);
+    let span = |range: Option<(u32, u32)>| match range {
+        Some((low, high)) => format!("{low}..{high}"),
+        None => "none".to_string(),
+    };
+    println!("  chain tip        {}", report.tip);
+    println!("  mapping keys     {}", report.keys);
+    println!("  entries          {total}");
+    println!("    to migrate     {} over heights {}", report.little_endian, span(report.little_endian_range));
+    println!("    already done   {} over heights {}", report.big_endian, span(report.big_endian_range));
+    println!("  mixed keys       {} (hold both encodings; only these can be undecidable)", report.mixed_keys);
+    println!("  undecidable      {} across {} keys\n", report.undecidable.len(), report.undecidable_keys);
+
+    // Deliberately no inference from the big-endian span to "a build writing that format ran for N
+    // blocks". On a ledger written only by v4.8.0 that sentence reported a 2.3-million-block window
+    // for a build that had never touched it: entries whose two readings are identical are counted
+    // as already migrated, since moving them is a no-op, and they carry heights of their own. The
+    // spans above are facts; that reading of them was not.
+
+    if !report.undecidable.is_empty() {
+        println!("This ledger CANNOT be migrated and must be resynced from genesis.\n");
+        println!("{} entries cannot be attributed to a block height. Each reads as a", report.undecidable.len());
+        println!("plausible height under either encoding, on a key that holds both:");
+        for entry in report.undecidable.iter().take(10) {
+            println!("    height {} (little-endian) or {} (big-endian)", entry.little, entry.big);
+        }
+        if report.undecidable.len() > 10 {
+            println!("    ... and {} more", report.undecidable.len() - 10);
+        }
+        // A refusal is a refusal in either mode; nothing has been modified.
+        bail!("{} entries cannot be attributed to a block height", report.undecidable.len());
+    }
+
+    let minutes = report.little_endian as f64 / ENTRIES_PER_SECOND / 60.0;
+    if args.check {
+        println!("This ledger can be migrated in place. Estimated {minutes:.1} minutes.\n");
+        println!("    snarkvm-migrate-db {}", args.path);
+        return Ok(());
+    }
+
+    // The migration reports progress through `tracing`; without a subscriber a multi-hour
+    // run would print nothing at all.
+    tracing_subscriber::fmt().with_env_filter("info").with_target(false).init();
+    println!("Migrating. Estimated {minutes:.1} minutes; safe to interrupt and resume.\n");
+    migrate(&database, args.network_id, Some(&report))?;
+    println!("\nMigration complete. The node can now be started.");
+    Ok(())
+}

ledger/store/src/helpers/memory/program.rs

@@ -52,9 +52,6 @@ pub struct FinalizeMemory<N: Network> {
     /// The historical mapping value map (keyed by big-endian block height).
     #[cfg(feature = "history")]
     mapping_update_map: MemoryMap<(ProgramID<N>, Identifier<N>, Plaintext<N>, HeightBytes), Value<N>>,
-    /// The legacy heights index; present only for keys written before the BE schema change.
-    #[cfg(feature = "history")]
-    mapping_update_heights_map: MemoryMap<(ProgramID<N>, Identifier<N>, Plaintext<N>), Vec<u32>>,
     /// The current block height.
     #[cfg(feature = "history")]
     block_height: Arc<AtomicU32>,
@@ -73,8 +70,6 @@ impl<N: Network> FinalizeStorage<N> for FinalizeMemory<N> {
     type RejectedReasonMap = MemoryMap<Field<N>, RejectedReason<N>>;
     #[cfg(feature = "history")]
     type MappingUpdateMap = MemoryMap<(ProgramID<N>, Identifier<N>, Plaintext<N>, HeightBytes), Value<N>>;
-    #[cfg(feature = "history")]
-    type MappingUpdateHeightsMap = MemoryMap<(ProgramID<N>, Identifier<N>, Plaintext<N>), Vec<u32>>;
     #[cfg(feature = "history-staking-rewards")]
     type StakingRewardsMap = MemoryMap<(Address<N>, u32), (Address<N>, u64, u64)>;
 
@@ -96,8 +91,6 @@ impl<N: Network> FinalizeStorage<N> for FinalizeMemory<N> {
             #[cfg(feature = "history")]
             mapping_update_map: MemoryMap::default(),
             #[cfg(feature = "history")]
-            mapping_update_heights_map: MemoryMap::default(),
-            #[cfg(feature = "history")]
             block_height: Arc::new(AtomicU32::new(initial_height)),
             #[cfg(feature = "history-staking-rewards")]
             staking_rewards_map: MemoryMap::default(),
@@ -131,11 +124,6 @@ impl<N: Network> FinalizeStorage<N> for FinalizeMemory<N> {
         &self.mapping_update_map
     }
 
-    #[cfg(feature = "history")]
-    fn mapping_update_heights_map(&self) -> &Self::MappingUpdateHeightsMap {
-        &self.mapping_update_heights_map
-    }
-
     /// Returns the historical staking rewards map.
     #[cfg(feature = "history-staking-rewards")]
     fn staking_rewards_map(&self) -> &Self::StakingRewardsMap {