Add remote verify script

README.md

@@ -27,8 +27,11 @@ Each local snapshot is named `move-YYYYMMDD-HHMMSS`.
 - Run the script as an account that can use `zfs` on the source server.
 - The source host needs Bash, ZFS utilities, and SSH.
 - The destination host needs ZFS utilities and the destination pool must exist.
+- Both hosts need `rsync` to run the verification script.
 - The SSH account on the destination must be able to execute `sudo -n zfs`
   without a password prompt.
+- For verification, that account must also be allowed to run `sudo -n rsync`
+  without a password prompt.
 - SSH must use a private key or SSH agent. Password authentication is disabled
   by the script so a migration cannot pause and prompt once per dataset.
 - SSH connectivity and host-key verification must already work.
@@ -48,6 +51,16 @@ Example:
 bash zfs-move-over-ssh.sh tank root@new-server backup/tank
 ```
 
+Verify the resulting filesystems after mounting them on both hosts:
+
+```bash
+bash zfs-verify-over-ssh.sh tank root@new-server backup/tank
+```
+
+The verifier maps datasets using the same relative paths as the move script.
+It uses a read-only `rsync --dry-run --checksum` comparison and exits with
+status `1` when it finds missing, extra, changed, or metadata-different files.
+
 Use a non-standard SSH key or port when needed:
 
 ```bash
@@ -139,10 +152,13 @@ Options:
   bash zfs-move-over-ssh.sh tank admin@new-server backup/tank
    ```
 
-5. Verify the received datasets and data before retiring the source:
+5. Mount the received datasets, then verify their data before retiring the
+  source. `zfs-move-over-ssh.sh` receives with `-u`, so the destination filesystems
+  are unmounted until you explicitly mount them:
 
    ```bash
-  ssh admin@new-server 'sudo zfs list -r backup/tank'
+  ssh admin@new-server 'sudo zfs mount -a'
+  bash zfs-verify-over-ssh.sh tank admin@new-server backup/tank
    ```
 
 ## Safety and Recovery

zfs-verify-over-ssh.sh

@@ -0,0 +1,160 @@
+#!/usr/bin/env bash
+# Verify each source ZFS filesystem against its matching remote filesystem.
+
+set -Eeuo pipefail
+
+usage() {
+    cat <<'EOF'
+Usage: zfs-verify-over-ssh.sh [options] SOURCE_POOL REMOTE_HOST DESTINATION_ROOT
+
+Compare every mounted filesystem in SOURCE_POOL with its corresponding mounted
+remote filesystem below DESTINATION_ROOT. The script is read-only and exits 1
+when it finds differences.
+
+Arguments:
+  SOURCE_POOL       Source ZFS pool, for example: tank
+  REMOTE_HOST       SSH host, optionally user@host
+  DESTINATION_ROOT  Remote destination ZFS filesystem, for example: backup/data
+
+Options:
+  -i, --identity-file FILE  SSH private key to use.
+  -p, --ssh-port PORT       SSH port to use.
+  -h, --help                Show this help.
+
+Both hosts need rsync. The remote SSH account must be able to run
+`sudo -n zfs` and `sudo -n rsync` without a password prompt.
+EOF
+}
+
+die() {
+    printf 'Error: %s\n' "$*" >&2
+    exit 2
+}
+
+SSH_OPTIONS=(-o BatchMode=yes)
+
+while (($#)); do
+    case "$1" in
+        -i|--identity-file)
+            (($# >= 2)) || die "$1 requires a file path"
+            SSH_OPTIONS+=(-i "$2")
+            shift
+            ;;
+        -p|--ssh-port)
+            (($# >= 2)) || die "$1 requires a port"
+            SSH_OPTIONS+=(-p "$2")
+            shift
+            ;;
+        -h|--help)
+            usage
+            exit 0
+            ;;
+        --)
+            shift
+            break
+            ;;
+        -*)
+            die "unknown option: $1"
+            ;;
+        *)
+            break
+            ;;
+    esac
+    shift
+done
+
+(($# == 3)) || {
+    usage >&2
+    exit 2
+}
+
+SOURCE_POOL="$1"
+REMOTE_HOST="$2"
+DESTINATION_ROOT="${3%/}"
+
+[[ "$REMOTE_HOST" != *@ ]] || die "REMOTE_HOST is missing a hostname; use user@server-name or server-name"
+
+command -v zfs >/dev/null || die "zfs command was not found"
+command -v zpool >/dev/null || die "zpool command was not found"
+command -v ssh >/dev/null || die "ssh command was not found"
+command -v rsync >/dev/null || die "rsync command was not found"
+zpool list -H -o name "$SOURCE_POOL" >/dev/null || die "source pool does not exist: $SOURCE_POOL"
+
+ssh "${SSH_OPTIONS[@]}" "$REMOTE_HOST" 'sudo -n zfs list -H -o name -t filesystem >/dev/null && sudo -n rsync --version >/dev/null' \
+    || die "$REMOTE_HOST must allow non-interactive sudo for zfs and rsync"
+
+map_destination() {
+    local filesystem="$1"
+    local relative_path="${filesystem#"$SOURCE_POOL"}"
+    printf '%s%s\n' "$DESTINATION_ROOT" "$relative_path"
+}
+
+get_remote_property() {
+    local property="$1"
+    local filesystem="$2"
+    local quoted_filesystem
+
+    printf -v quoted_filesystem '%q' "$filesystem"
+    ssh "${SSH_OPTIONS[@]}" "$REMOTE_HOST" \
+        "sudo -n zfs get -H -o value '$property' $quoted_filesystem"
+}
+
+require_mounted_filesystem() {
+    local host_label="$1"
+    local filesystem="$2"
+    local mountpoint="$3"
+    local mounted="$4"
+
+    [[ "$mountpoint" != none && "$mountpoint" != legacy ]] \
+        || die "$host_label filesystem $filesystem has mountpoint=$mountpoint"
+    [[ "$mounted" == yes ]] \
+        || die "$host_label filesystem $filesystem is not mounted at $mountpoint"
+
+    if [[ "$host_label" == local ]]; then
+        [[ -d "$mountpoint" ]] \
+            || die "local filesystem $filesystem is not mounted at $mountpoint"
+    fi
+}
+
+mapfile -t FILESYSTEMS < <(zfs list -H -o name -t filesystem -r "$SOURCE_POOL")
+((${#FILESYSTEMS[@]} > 0)) || die "no filesystems found in $SOURCE_POOL"
+
+SSH_COMMAND=(ssh "${SSH_OPTIONS[@]}")
+printf -v RSYNC_RSH '%q ' "${SSH_COMMAND[@]}"
+REPORT_FILE="$(mktemp)"
+trap 'rm -f "$REPORT_FILE"' EXIT
+DIFFERENCES=0
+
+for filesystem in "${FILESYSTEMS[@]}"; do
+    destination="$(map_destination "$filesystem")"
+    source_mountpoint="$(zfs get -H -o value mountpoint "$filesystem")"
+    source_mounted="$(zfs get -H -o value mounted "$filesystem")"
+    remote_mountpoint="$(get_remote_property mountpoint "$destination")"
+    remote_mounted="$(get_remote_property mounted "$destination")"
+
+    require_mounted_filesystem local "$filesystem" "$source_mountpoint" "$source_mounted"
+    require_mounted_filesystem remote "$destination" "$remote_mountpoint" "$remote_mounted"
+
+    printf 'Verifying %s against %s:%s\n' "$filesystem" "$REMOTE_HOST" "$destination"
+    : >"$REPORT_FILE"
+    if ! rsync --dry-run --archive --hard-links --acls --xattrs --checksum --delete \
+        --one-file-system --itemize-changes --out-format='%i %n%L' \
+        -e "$RSYNC_RSH" --rsync-path='sudo -n rsync' \
+        "$source_mountpoint/" "$REMOTE_HOST:$remote_mountpoint/" >"$REPORT_FILE"; then
+        printf 'Unable to compare %s.\n' "$filesystem" >&2
+        exit 2
+    fi
+
+    if [[ -s "$REPORT_FILE" ]]; then
+        printf 'Differences for %s:\n' "$filesystem" >&2
+        cat "$REPORT_FILE" >&2
+        DIFFERENCES=1
+    fi
+done
+
+if ((DIFFERENCES)); then
+    printf 'Verification failed: source and remote filesystems differ.\n' >&2
+    exit 1
+fi
+
+printf 'Verification succeeded: all checked filesystems are identical.\n'