zfs-remote-send/zfs-verify-over-ssh.sh

#!/usr/bin/env bash
# Verify each source ZFS filesystem against its matching remote filesystem.

set -Eeuo pipefail

usage() {
    cat <<'EOF'
Usage: zfs-verify-over-ssh.sh [options] SOURCE_POOL REMOTE_HOST DESTINATION_ROOT

Compare every mounted filesystem in SOURCE_POOL with its corresponding mounted
remote filesystem below DESTINATION_ROOT. The script is read-only and exits 1
when it finds differences.

Arguments:
  SOURCE_POOL       Source ZFS pool, for example: tank
  REMOTE_HOST       SSH host, optionally user@host
  DESTINATION_ROOT  Remote destination ZFS filesystem, for example: backup/data

Options:
  -i, --identity-file FILE  SSH private key to use.
  -p, --ssh-port PORT       SSH port to use.
  -h, --help                Show this help.

Both hosts need rsync. The remote SSH account must be able to run
`sudo -n zfs` and `sudo -n rsync` without a password prompt.
EOF
}

die() {
    printf 'Error: %s\n' "$*" >&2
    exit 2
}

SSH_OPTIONS=(-o BatchMode=yes)

while (($#)); do
    case "$1" in
        -i|--identity-file)
            (($# >= 2)) || die "$1 requires a file path"
            SSH_OPTIONS+=(-i "$2")
            shift
            ;;
        -p|--ssh-port)
            (($# >= 2)) || die "$1 requires a port"
            SSH_OPTIONS+=(-p "$2")
            shift
            ;;
        -h|--help)
            usage
            exit 0
            ;;
        --)
            shift
            break
            ;;
        -*)
            die "unknown option: $1"
            ;;
        *)
            break
            ;;
    esac
    shift
done

(($# == 3)) || {
    usage >&2
    exit 2
}

SOURCE_POOL="$1"
REMOTE_HOST="$2"
DESTINATION_ROOT="${3%/}"

[[ "$REMOTE_HOST" != *@ ]] || die "REMOTE_HOST is missing a hostname; use user@server-name or server-name"

command -v zfs >/dev/null || die "zfs command was not found"
command -v zpool >/dev/null || die "zpool command was not found"
command -v ssh >/dev/null || die "ssh command was not found"
command -v rsync >/dev/null || die "rsync command was not found"
zpool list -H -o name "$SOURCE_POOL" >/dev/null || die "source pool does not exist: $SOURCE_POOL"

ssh "${SSH_OPTIONS[@]}" "$REMOTE_HOST" 'sudo -n zfs list -H -o name -t filesystem >/dev/null && sudo -n rsync --version >/dev/null' \
    || die "$REMOTE_HOST must allow non-interactive sudo for zfs and rsync"

map_destination() {
    local filesystem="$1"
    local relative_path="${filesystem#"$SOURCE_POOL"}"
    printf '%s%s\n' "$DESTINATION_ROOT" "$relative_path"
}

get_remote_property() {
    local property="$1"
    local filesystem="$2"
    local quoted_filesystem

    printf -v quoted_filesystem '%q' "$filesystem"
    ssh "${SSH_OPTIONS[@]}" "$REMOTE_HOST" \
        "sudo -n zfs get -H -o value '$property' $quoted_filesystem"
}

require_mounted_filesystem() {
    local host_label="$1"
    local filesystem="$2"
    local mountpoint="$3"
    local mounted="$4"

    [[ "$mountpoint" != none && "$mountpoint" != legacy ]] \
        || die "$host_label filesystem $filesystem has mountpoint=$mountpoint"
    [[ "$mounted" == yes ]] \
        || die "$host_label filesystem $filesystem is not mounted at $mountpoint"

    if [[ "$host_label" == local ]]; then
        [[ -d "$mountpoint" ]] \
            || die "local filesystem $filesystem is not mounted at $mountpoint"
    fi
}

mapfile -t FILESYSTEMS < <(zfs list -H -o name -t filesystem -r "$SOURCE_POOL")
((${#FILESYSTEMS[@]} > 0)) || die "no filesystems found in $SOURCE_POOL"

SSH_COMMAND=(ssh "${SSH_OPTIONS[@]}")
printf -v RSYNC_RSH '%q ' "${SSH_COMMAND[@]}"
REPORT_FILE="$(mktemp)"
trap 'rm -f "$REPORT_FILE"' EXIT
DIFFERENCES=0

for filesystem in "${FILESYSTEMS[@]}"; do
    destination="$(map_destination "$filesystem")"
    source_mountpoint="$(zfs get -H -o value mountpoint "$filesystem")"
    source_mounted="$(zfs get -H -o value mounted "$filesystem")"
    remote_mountpoint="$(get_remote_property mountpoint "$destination")"
    remote_mounted="$(get_remote_property mounted "$destination")"

    require_mounted_filesystem local "$filesystem" "$source_mountpoint" "$source_mounted"
    require_mounted_filesystem remote "$destination" "$remote_mountpoint" "$remote_mounted"

    printf 'Verifying %s against %s:%s\n' "$filesystem" "$REMOTE_HOST" "$destination"
    : >"$REPORT_FILE"
    if ! rsync --dry-run --archive --hard-links --acls --xattrs --checksum --delete \
        --one-file-system --itemize-changes --out-format='%i %n%L' \
        -e "$RSYNC_RSH" --rsync-path='sudo -n rsync' \
        "$source_mountpoint/" "$REMOTE_HOST:$remote_mountpoint/" >"$REPORT_FILE"; then
        printf 'Unable to compare %s.\n' "$filesystem" >&2
        exit 2
    fi

    if [[ -s "$REPORT_FILE" ]]; then
        printf 'Differences for %s:\n' "$filesystem" >&2
        cat "$REPORT_FILE" >&2
        DIFFERENCES=1
    fi
done

if ((DIFFERENCES)); then
    printf 'Verification failed: source and remote filesystems differ.\n' >&2
    exit 1
fi

printf 'Verification succeeded: all checked filesystems are identical.\n'