Add strict no-clobber restore

Cargo.toml

@@ -16,10 +16,12 @@ flate2 = "1.1.5"
 jiff = "0.2.34"
 lz4_flex = "0.14.0"
 postcard = { version = "1.1.3", features = ["use-std"] }
+rustix = { version = "1.1.4", features = ["fs"] }
 serde = { version = "1.0.229", features = ["derive"] }
 
 [dev-dependencies]
 criterion = "0.8.2"
+libc = "0.2.187"
 proptest = "1.11.0"
 rusqlite = { version = "0.40.1", features = ["bundled"] }
 serde_json = "1.0.149"

README.md

@@ -61,6 +61,7 @@ Verify or snapshot a directory store:
 ```sh
 cargo run --release -- check-store ./energy.ftwdb
 cargo run --release -- backup ./energy.ftwdb ./backups/energy-2026-07-21.ftwdb
+cargo run --release -- restore ./backups/energy-2026-07-21.ftwdb ./restored-energy.ftwdb
 ```
 
 ## Design documents
@@ -70,7 +71,7 @@ cargo run --release -- backup ./energy.ftwdb ./backups/energy-2026-07-21.ftwdb
 - [Storage format v1](docs/format.md)
 - [Immutable segment format](docs/segment-format.md)
 - [Persistent rollups and retention](docs/rollups.md)
-- [Integrity checks and backup](docs/operations.md)
+- [Integrity checks, backup, and restore](docs/operations.md)
 - [OSS database research](docs/research.md)
 - [Benchmark protocol](docs/benchmarking.md)
 - [Deterministic energy workload](docs/workload.md)

bench/sd-card-emulator/README.md

@@ -67,8 +67,12 @@ docker run --rm --privileged \
 
 The script installs `nbd-client` and `e2fsprogs` in the disposable container.
 It writes raw results under the ignored `bench-results/linux-nbd-smoke`
-directory. It uses the checked-in real-installation fixture and rejects a
-changed active-log checksum or recovered FTWDB watermark.
+directory, which must be absent or empty when the run starts. It uses the
+checked-in real-installation fixture and rejects a
+changed active-log checksum or recovered FTWDB watermark. After recovery, it
+backs up the checked store outside the emulated card, restores that backup to a
+new store on the card, and compares checked counts, snapshot CRCs, and active
+log SHA-256 values.
 
 ```sh
 mkdir -p bench-results/sd-emulator

bench/sd-card-emulator/linux-smoke.sh

@@ -4,6 +4,10 @@ set -euo pipefail
 source bench/sd-card-emulator/linux-nbd-common.sh
 
 out=${FTW_NBD_OUTPUT:-/work/bench-results/linux-nbd-smoke}
+if [[ -e "$out" ]] && find "$out" -mindepth 1 -print -quit | grep -q .; then
+  printf 'output directory must be absent or empty: %s\n' "$out" >&2
+  exit 1
+fi
 linux_nbd_prepare
 trap linux_nbd_cleanup EXIT
 linux_nbd_start bench/sd-card-emulator/profiles/healthy.json 42
@@ -58,12 +62,57 @@ test "$before_hash" = "$after_hash"
   --output "$out/verification.jsonl" \
   >"$out/verification.json"
 
+# Keep the backup off the emulated card, then prove that a new store restored
+# onto the recovered card has the same checked counts and selected bytes.
+"$ftw" backup "$mount_dir/database" "$out/restore-backup" \
+  >"$out/restore-backup.json"
+"$ftw" restore "$out/restore-backup" "$mount_dir/restored-database" \
+  >"$out/restore.json"
+"$ftw" check-store "$mount_dir/restored-database" \
+  >"$out/restore-check.json"
+sha256sum "$out/restore-backup/active.wlog" >"$out/restore-backup-sha.txt"
+sha256sum "$mount_dir/restored-database/active.wlog" >"$out/restore-target-sha.txt"
+
+json_u64() {
+  local file=$1
+  local field=$2
+  local value
+  value=$(sed -n "s/.*\"$field\":\([0-9][0-9]*\).*/\1/p" "$file")
+  test -n "$value"
+  printf '%s\n' "$value"
+}
+
+json_string() {
+  local file=$1
+  local field=$2
+  local value
+  value=$(sed -n "s/.*\"$field\":\"\([^\"]*\)\".*/\1/p" "$file")
+  test -n "$value"
+  printf '%s\n' "$value"
+}
+
+test "$(json_u64 "$out/check-after.json" raw_points)" = \
+  "$(json_u64 "$out/restore-check.json" raw_points)"
+test "$(json_u64 "$out/check-after.json" raw_commits)" = \
+  "$(json_u64 "$out/restore-check.json" raw_commits)"
+test "$(json_u64 "$out/check-after.json" raw_points)" = \
+  "$(json_u64 "$out/restore.json" raw_points)"
+test "$(json_u64 "$out/check-after.json" raw_commits)" = \
+  "$(json_u64 "$out/restore.json" raw_commits)"
+test "$(json_string "$out/restore.json" source_snapshot_crc32)" = \
+  "$(json_string "$out/restore.json" destination_snapshot_crc32)"
+test "$(cut -d' ' -f1 "$out/restore-backup-sha.txt")" = \
+  "$(cut -d' ' -f1 "$out/restore-target-sha.txt")"
+
 linux_nbd_finish
 trap - EXIT
 
 printf 'linux_nbd_smoke=passed\n'
+printf 'restore_drill=passed\n'
 printf 'fsck_exit=%d\n' "$fsck_exit"
 printf 'active_log_sha256=%s\n' "$after_hash"
 cat "$out/load.json"
 cat "$out/check-after.json"
 cat "$out/verification.json"
+cat "$out/restore.json"
+cat "$out/restore-check.json"

docs/operations.md

@@ -1,8 +1,8 @@
-# Integrity checks and backup
+# Integrity checks, backup, and restore
 
 ## Integrity check
 
-`ftwdb check-store <directory>` opens the store read-only: the active commit
+`ftw check-store <directory>` opens the store read-only: the active commit
 log is opened without write access under a shared lock, torn-tail recovery is
 simulated in memory, and no manifest generation is published, pruned, or swept.
 It loads the highest valid manifest generation and re-opens every active rollup
@@ -22,7 +22,7 @@ startup; a read-only open never deletes them.
 
 ## Snapshot backup
 
-`ftwdb backup <source> <absent-destination>` opens the source read-only, so a
+`ftw backup <source> <absent-destination>` opens the source read-only, so a
 backup can never alter (or accidentally create) the store it is copying, and
 uses this publication order:
 
@@ -32,9 +32,18 @@ uses this publication order:
 4. hard-link active immutable rollups and the selected manifest when source and
    destination share a filesystem, otherwise copy and sync them;
 5. sync both subdirectories and the temporary root;
-6. rename the temporary root to the requested absent destination;
+6. publish the temporary root with an atomic no-clobber rename;
 7. sync the parent directory;
-8. open and integrity-check the published backup.
+8. open the published backup read-only and integrity-check it.
+
+Backup and restore use Linux `renameat2(RENAME_NOREPLACE)` or the matching
+Apple exclusive rename through `rustix`. They do not use an `exists()` check
+followed by a replacing rename. On an unsupported Unix target, publication
+returns an unsupported-operation error instead of weakening this rule. If the
+final parent sync or post-check fails, the code verifies the directory identity
+and tries to move its own publication back to its hidden name, remove it, and
+sync the parent. If that rollback also fails, the error says that rollback
+failed; the caller must inspect the named destination before retrying.
 
 The active log is always copied, never hard-linked, because later appends to a
 shared inode would silently mutate the backup. A unit test writes to the source
@@ -49,8 +58,58 @@ the link error kind. If both fail, the returned I/O error keeps the copy error
 kind and its text and source also include the hard-link failure.
 
 This is a local consistent snapshot, not yet a remote backup policy. Encryption,
-incremental upload, retention, restore drills, and salvage of a corrupted source
-remain operational work.
+incremental upload, retention, and salvage of a corrupted source remain open.
+
+## Strict restore
+
+`ftw restore <backup> <absent-target>` restores only a fully valid snapshot.
+The command exits with code 2 for missing or extra arguments and code 1 for a
+store, file, or publication error. It has no replace option. An existing target,
+including an empty directory or a dangling symlink, causes `AlreadyExists` and
+stays unchanged.
+
+Restore opens the backup read-only under a shared lock, runs the full store
+check, and refuses any raw-log recovery. Both recovered tail bytes and a
+recovery reason must be zero. A short final header or payload, a full bad frame,
+a bad selected manifest, a bad active rollup, or an active rollup whose source
+watermark trails the raw log causes an error. Orphan rollups, older manifest
+generations, and other files outside the selected snapshot do not affect the
+restore and do not appear in the target.
+
+The selected snapshot contains:
+
+- `active.wlog`, always copied;
+- the selected manifest generation, when one exists;
+- each active immutable rollup named by that manifest.
+
+Every selected path must be a regular file according to `symlink_metadata` and
+the opened file identity. Read-only file opens use no-follow and nonblocking
+flags, and snapshot checksum traversal uses directory file descriptors, so
+restore rejects symlinks and special files without blocking or leaving the
+snapshot root. It copies the selected files to a unique hidden sibling
+directory, syncs each file and directory, and opens that stage read-only for a
+full check. The stage lock stays held through no-clobber publication and the
+target's read-only post-check, so a writer cannot append between publication
+and verification.
+
+The snapshot checksum uses the CRC32 domain `FTWDB snapshot CRC32 v1\0` and the
+selected relative paths sorted by UTF-8 bytes. Each file adds, in order, the
+path byte length as little-endian `u64`, the path bytes, the file length as
+little-endian `u64`, and the exact file bytes. The byte count is the sum of
+those file lengths. Permissions, times, directories, and unselected files do
+not enter the checksum. Restore compares source and stage before publication,
+then source and target after publication.
+
+On success, `ftwdb-restore-v1` reports `files`, `bytes`,
+`manifest_generation`, `raw_commits`, `raw_points`,
+`source_snapshot_crc32`, and `destination_snapshot_crc32`. The CRC values are
+eight lower-case hex digits. Equal CRCs are a deterministic corruption check,
+not proof that the selected bytes match, and they do not turn CRC32 into a
+cryptographic authenticity check.
+
+Restore does not repair a damaged backup and does not infer data past a corrupt
+frame. Use a separate target, keep the source backup, and run `ftw check-store`
+on the result. Salvage will use a separate command and safety review.
 
 ## Sync and full-disk checks
 
@@ -73,12 +132,14 @@ the M4 physical SD-card power-cut release gate.
 `tests/cli.rs` runs the built `ftw` binary as a subprocess. It fixes the usage
 contract at exit code 2 with usage text on standard error, while data, file,
 and store errors use exit code 1. The test covers the generated workload,
-sanitized real fixture, TSBS IoT, integrity check, log inspection, and backup
-paths. It parses each promised JSON record and checks its main counts and
-status fields.
+sanitized real fixture, TSBS IoT, integrity check, log inspection, backup, and
+strict restore paths. It parses each promised JSON record and checks its main
+counts and status fields.
 
 The test also snapshots every path and every file byte in a store before and
 after both `check-store` and `inspect`. Separate missing-path checks prove that
 neither command creates its input. The backup check covers linked and copied
 file counts plus the hard-link fallback count on a normal local filesystem.
-Restore drills, salvage, and physical SD-card tests remain M4 work.
+The restore check covers JSON, exact bytes, counts, checksum equality,
+corruption refusal, and no-clobber behavior. Salvage and physical SD-card tests
+remain M4 work.

docs/roadmap.md

@@ -80,12 +80,17 @@ workload and failure surface. A deterministic NBD SD-card emulator and one ARM64
 Linux/ext4 smoke run add repeatable media-cache and power-loss checks, but the
 recorded cut happened after the load and host sync. A repeatable 64 MiB
 Linux/NBD/ext4 run reached `ENOSPC` after 770,000 durable points in 77 commits;
-`check-store` and `inspect` passed on that prefix. Local snapshot backup plus
-post-publication integrity verification is implemented. Process-level CLI
-tests now cover usage errors, generated and sanitized input paths, integrity
-checks, inspection, backup JSON, and byte-for-byte read-only behavior. Together
-with the process-lock, sync-failure, `/dev/full`, and NBD tests, this closes the
+`check-store` and `inspect` passed on that prefix. Local snapshot backup and
+strict restore now use atomic no-clobber publication, read-only checks, selected
+file CRCs, and tested rollback after final sync or post-check errors.
+Process-level CLI tests cover usage errors, generated and sanitized input paths,
+integrity checks, inspection, backup and restore JSON, exact restored bytes,
+and byte-for-byte read-only behavior. The sanitized fixture restore fixes the
+expected result at 889,978 points, 89 commits, and snapshot CRC32 `9ff1a95b`.
+The Linux NBD power-loss run now follows recovery with an off-card backup and a
+verified restore to a new target on the emulated card. Together with the
+process-lock, sync-failure, `/dev/full`, and NBD tests, this closes the
 robustness gaps tracked in issue #17. Tests on the target board and SD cards,
-power cuts during commits, soak runs, remote backup policy, restore drills,
-corruption salvage, and the remaining result-verified adapters stay open and
-are required for the M4 exit.
+power cuts during commits, soak runs, remote backup policy, corruption salvage,
+and the remaining result-verified adapters stay open and are required for the
+M4 exit.

src/bin/ftw.rs

@@ -1,7 +1,8 @@
 use flate2::read::GzDecoder;
 use ftwdb::{
-    BackupReport, Config, Database, Durability, EnergyWorkload, Error, RollupResolution, Store,
-    Transaction, WorkloadConfig, gauge_bucket_checksum, load_real_fixture, load_tsbs_iot,
+    BackupReport, Config, Database, Durability, EnergyWorkload, Error, RestoreReport,
+    RollupResolution, Store, Transaction, WorkloadConfig, gauge_bucket_checksum, load_real_fixture,
+    load_tsbs_iot,
 };
 use std::env;
 use std::fs::File;
@@ -40,6 +41,7 @@ fn main() -> ExitCode {
         Some("inspect") => inspect(&arguments[2..]),
         Some("check-store") => check_store(&arguments[2..]),
         Some("backup") => backup(&arguments[2..]),
+        Some("restore") => restore(&arguments[2..]),
         Some("generate") => generate(&arguments[2..]),
         Some("bench-ftwdb") => bench_ftwdb(&arguments[2..]),
         Some("bench-real-fixture") => bench_real_fixture(&arguments[2..]),
@@ -292,6 +294,30 @@ fn backup(arguments: &[String]) -> CliResult<()> {
     Ok(())
 }
 
+fn restore(arguments: &[String]) -> CliResult<()> {
+    if arguments.len() != 2 {
+        return Err(usage_error(
+            "restore requires a backup and absent target directory",
+        ));
+    }
+    let report = Store::restore_from(&arguments[0], &arguments[1])?;
+    println!("{}", restore_report_json(&report));
+    Ok(())
+}
+
+fn restore_report_json(report: &RestoreReport) -> String {
+    format!(
+        "{{\"format\":\"ftwdb-restore-v1\",\"files\":{},\"bytes\":{},\"manifest_generation\":{},\"raw_commits\":{},\"raw_points\":{},\"source_snapshot_crc32\":\"{:08x}\",\"destination_snapshot_crc32\":\"{:08x}\"}}",
+        report.files,
+        report.bytes,
+        report.manifest_generation,
+        report.raw_commits,
+        report.raw_points,
+        report.source_snapshot_crc32,
+        report.destination_snapshot_crc32
+    )
+}
+
 fn backup_report_json(report: &BackupReport) -> String {
     let fallback_error_kinds = report
         .hard_link_fallback_error_kinds
@@ -548,14 +574,14 @@ fn runtime_invalid(reason: impl Into<String>) -> CliError {
 
 fn usage(program: &str) {
     eprintln!(
-        "usage:\n  {program} inspect <database-file>\n  {program} check-store <store-directory>\n  {program} backup <store-directory> <absent-destination>\n  {program} generate <output-directory> [--seed N] [--sites N] [--days N] [--cadence-seconds N] [--start-micros N]\n  {program} bench-ftwdb <workload-directory> <empty-database-directory> [--durability always|manual] [--batch-points N]\n  {program} bench-real-fixture <points.csv.gz> <empty-database-directory> [--durability always|manual|every-bytes:N] [--batch-points N]\n  {program} bench-tsbs-iot <influx-line-file|-> <empty-database-directory> [--durability always|manual|every-bytes:N] [--batch-rows N]"
+        "usage:\n  {program} inspect <database-file>\n  {program} check-store <store-directory>\n  {program} backup <store-directory> <absent-destination>\n  {program} restore <backup-directory> <absent-target>\n  {program} generate <output-directory> [--seed N] [--sites N] [--days N] [--cadence-seconds N] [--start-micros N]\n  {program} bench-ftwdb <workload-directory> <empty-database-directory> [--durability always|manual] [--batch-points N]\n  {program} bench-real-fixture <points.csv.gz> <empty-database-directory> [--durability always|manual|every-bytes:N] [--batch-points N]\n  {program} bench-tsbs-iot <influx-line-file|-> <empty-database-directory> [--durability always|manual|every-bytes:N] [--batch-rows N]"
     );
 }
 
 #[cfg(test)]
 mod tests {
-    use super::backup_report_json;
-    use ftwdb::BackupReport;
+    use super::{backup_report_json, restore_report_json};
+    use ftwdb::{BackupReport, RestoreReport};
 
     #[test]
     fn backup_json_uses_fixed_error_kind_names() {
@@ -574,4 +600,22 @@ mod tests {
             "{\"format\":\"ftwdb-backup-v1\",\"files\":4,\"bytes\":123,\"manifest_generation\":7,\"linked_files\":2,\"copied_files\":2,\"hard_link_fallbacks\":1,\"hard_link_fallback_error_kinds\":[\"crosses-devices\"]}"
         );
     }
+
+    #[test]
+    fn restore_json_has_versioned_checksums_and_counts() {
+        let report = RestoreReport {
+            files: 4,
+            bytes: 123,
+            manifest_generation: 7,
+            raw_commits: 8,
+            raw_points: 9,
+            source_snapshot_crc32: 0x0123_abcd,
+            destination_snapshot_crc32: 0x0123_abcd,
+        };
+
+        assert_eq!(
+            restore_report_json(&report),
+            "{\"format\":\"ftwdb-restore-v1\",\"files\":4,\"bytes\":123,\"manifest_generation\":7,\"raw_commits\":8,\"raw_points\":9,\"source_snapshot_crc32\":\"0123abcd\",\"destination_snapshot_crc32\":\"0123abcd\"}"
+        );
+    }
 }

src/error.rs

@@ -31,6 +31,12 @@ pub enum Error {
         path: PathBuf,
     },
     ReadOnly,
+    /// A no-clobber rename succeeded, but a later durability or verification
+    /// step failed and the process could not fully roll its own directory back.
+    SnapshotPublication {
+        path: PathBuf,
+        reason: String,
+    },
 }
 
 impl fmt::Display for Error {
@@ -64,6 +70,11 @@ impl fmt::Display for Error {
                 f,
                 "database is open read-only; reopen it writable to modify it"
             ),
+            Self::SnapshotPublication { path, reason } => write!(
+                f,
+                "snapshot publication at {} could not be rolled back cleanly: {reason}",
+                path.display()
+            ),
         }
     }
 }