202 lines
6.3 KiB
Bash
202 lines
6.3 KiB
Bash
#!/usr/bin/env bash
|
|
# Verify each source ZFS filesystem against its matching remote filesystem.
|
|
|
|
set -Eeuo pipefail
|
|
|
|
usage() {
|
|
cat <<'EOF'
|
|
Usage: zfs-verify-over-ssh.sh [options] SOURCE_POOL REMOTE_HOST DESTINATION_ROOT
|
|
|
|
Compare every mounted filesystem in SOURCE_POOL with its corresponding mounted
|
|
remote filesystem below DESTINATION_ROOT. The script is read-only, continues
|
|
after per-filesystem errors, and exits 1 when it finds differences or 2 when
|
|
one or more filesystems could not be checked.
|
|
|
|
Arguments:
|
|
SOURCE_POOL Source ZFS pool, for example: tank
|
|
REMOTE_HOST SSH host, optionally user@host
|
|
DESTINATION_ROOT Remote destination ZFS filesystem, for example: backup/data
|
|
|
|
Options:
|
|
-i, --identity-file FILE SSH private key to use.
|
|
-p, --ssh-port PORT SSH port to use.
|
|
-h, --help Show this help.
|
|
|
|
Both hosts need rsync. The remote SSH account must be able to run
|
|
`sudo -n zfs` and `sudo -n rsync` without a password prompt.
|
|
EOF
|
|
}
|
|
|
|
die() {
|
|
printf 'Error: %s\n' "$*" >&2
|
|
exit 2
|
|
}
|
|
|
|
SSH_OPTIONS=(-o BatchMode=yes)
|
|
|
|
while (($#)); do
|
|
case "$1" in
|
|
-i|--identity-file)
|
|
(($# >= 2)) || die "$1 requires a file path"
|
|
SSH_OPTIONS+=(-i "$2")
|
|
shift
|
|
;;
|
|
-p|--ssh-port)
|
|
(($# >= 2)) || die "$1 requires a port"
|
|
SSH_OPTIONS+=(-p "$2")
|
|
shift
|
|
;;
|
|
-h|--help)
|
|
usage
|
|
exit 0
|
|
;;
|
|
--)
|
|
shift
|
|
break
|
|
;;
|
|
-*)
|
|
die "unknown option: $1"
|
|
;;
|
|
*)
|
|
break
|
|
;;
|
|
esac
|
|
shift
|
|
done
|
|
|
|
(($# == 3)) || {
|
|
usage >&2
|
|
exit 2
|
|
}
|
|
|
|
SOURCE_POOL="$1"
|
|
REMOTE_HOST="$2"
|
|
DESTINATION_ROOT="${3%/}"
|
|
|
|
[[ "$REMOTE_HOST" != *@ ]] || die "REMOTE_HOST is missing a hostname; use user@server-name or server-name"
|
|
|
|
command -v zfs >/dev/null || die "zfs command was not found"
|
|
command -v zpool >/dev/null || die "zpool command was not found"
|
|
command -v ssh >/dev/null || die "ssh command was not found"
|
|
command -v rsync >/dev/null || die "rsync command was not found"
|
|
zpool list -H -o name "$SOURCE_POOL" >/dev/null || die "source pool does not exist: $SOURCE_POOL"
|
|
|
|
ssh "${SSH_OPTIONS[@]}" "$REMOTE_HOST" 'sudo -n zfs list -H -o name -t filesystem >/dev/null && sudo -n rsync --version >/dev/null' \
|
|
|| die "$REMOTE_HOST must allow non-interactive sudo for zfs and rsync"
|
|
|
|
map_destination() {
|
|
local filesystem="$1"
|
|
local relative_path="${filesystem#"$SOURCE_POOL"}"
|
|
printf '%s%s\n' "$DESTINATION_ROOT" "$relative_path"
|
|
}
|
|
|
|
get_remote_property() {
|
|
local property="$1"
|
|
local filesystem="$2"
|
|
local quoted_filesystem
|
|
|
|
printf -v quoted_filesystem '%q' "$filesystem"
|
|
ssh "${SSH_OPTIONS[@]}" "$REMOTE_HOST" \
|
|
"sudo -n zfs get -H -o value '$property' $quoted_filesystem"
|
|
}
|
|
|
|
require_mounted_filesystem() {
|
|
local host_label="$1"
|
|
local filesystem="$2"
|
|
local mountpoint="$3"
|
|
local mounted="$4"
|
|
|
|
if [[ "$mountpoint" == none || "$mountpoint" == legacy ]]; then
|
|
printf 'ERROR %s filesystem %s has mountpoint=%s\n' "$host_label" "$filesystem" "$mountpoint" >&2
|
|
return 1
|
|
fi
|
|
if [[ "$mounted" != yes ]]; then
|
|
printf 'ERROR %s filesystem %s is not mounted at %s\n' "$host_label" "$filesystem" "$mountpoint" >&2
|
|
return 1
|
|
fi
|
|
|
|
if [[ "$host_label" == local ]]; then
|
|
if [[ ! -d "$mountpoint" ]]; then
|
|
printf 'ERROR local filesystem %s is not mounted at %s\n' "$filesystem" "$mountpoint" >&2
|
|
return 1
|
|
fi
|
|
fi
|
|
}
|
|
|
|
mapfile -t FILESYSTEMS < <(zfs list -H -o name -t filesystem -r "$SOURCE_POOL")
|
|
((${#FILESYSTEMS[@]} > 0)) || die "no filesystems found in $SOURCE_POOL"
|
|
|
|
SSH_COMMAND=(ssh "${SSH_OPTIONS[@]}")
|
|
printf -v RSYNC_RSH '%q ' "${SSH_COMMAND[@]}"
|
|
REPORT_FILE="$(mktemp)"
|
|
trap 'rm -f "$REPORT_FILE"' EXIT
|
|
DIFFERENCES=0
|
|
ERRORS=0
|
|
|
|
verify_filesystem() {
|
|
local filesystem="$1"
|
|
destination="$(map_destination "$filesystem")"
|
|
local source_mountpoint
|
|
local source_mounted
|
|
local remote_mountpoint
|
|
local remote_mounted
|
|
|
|
if ! source_mountpoint="$(zfs get -H -o value mountpoint "$filesystem" 2>&1)"; then
|
|
printf 'ERROR %s: could not read local mountpoint: %s\n' "$filesystem" "$source_mountpoint" >&2
|
|
return 1
|
|
fi
|
|
if ! source_mounted="$(zfs get -H -o value mounted "$filesystem" 2>&1)"; then
|
|
printf 'ERROR %s: could not read local mounted state: %s\n' "$filesystem" "$source_mounted" >&2
|
|
return 1
|
|
fi
|
|
if ! remote_mountpoint="$(get_remote_property mountpoint "$destination" 2>&1)"; then
|
|
printf 'ERROR %s: could not read remote mountpoint for %s: %s\n' "$filesystem" "$destination" "$remote_mountpoint" >&2
|
|
return 1
|
|
fi
|
|
if ! remote_mounted="$(get_remote_property mounted "$destination" 2>&1)"; then
|
|
printf 'ERROR %s: could not read remote mounted state for %s: %s\n' "$filesystem" "$destination" "$remote_mounted" >&2
|
|
return 1
|
|
fi
|
|
|
|
if ! require_mounted_filesystem local "$filesystem" "$source_mountpoint" "$source_mounted"; then
|
|
return 1
|
|
fi
|
|
if ! require_mounted_filesystem remote "$destination" "$remote_mountpoint" "$remote_mounted"; then
|
|
return 1
|
|
fi
|
|
|
|
printf 'Verifying %s against %s:%s\n' "$filesystem" "$REMOTE_HOST" "$destination"
|
|
: >"$REPORT_FILE"
|
|
if ! rsync --dry-run --archive --hard-links --acls --xattrs --checksum --delete \
|
|
--one-file-system --itemize-changes --out-format='%i %n%L' \
|
|
-e "$RSYNC_RSH" --rsync-path='sudo -n rsync' \
|
|
"$source_mountpoint/" "$REMOTE_HOST:$remote_mountpoint/" >"$REPORT_FILE"; then
|
|
printf 'ERROR %s: rsync comparison failed.\n' "$filesystem" >&2
|
|
return 1
|
|
fi
|
|
|
|
if [[ -s "$REPORT_FILE" ]]; then
|
|
printf 'Differences for %s:\n' "$filesystem" >&2
|
|
cat "$REPORT_FILE" >&2
|
|
DIFFERENCES=1
|
|
fi
|
|
}
|
|
|
|
for filesystem in "${FILESYSTEMS[@]}"; do
|
|
if ! verify_filesystem "$filesystem"; then
|
|
((ERRORS += 1))
|
|
fi
|
|
done
|
|
|
|
if ((ERRORS)); then
|
|
printf 'Verification incomplete: %d filesystem(s) could not be checked.\n' "$ERRORS" >&2
|
|
printf 'Review the ERROR lines above; checked filesystem differences are also listed above.\n' >&2
|
|
exit 2
|
|
fi
|
|
|
|
if ((DIFFERENCES)); then
|
|
printf 'Verification failed: source and remote filesystems differ.\n' >&2
|
|
exit 1
|
|
fi
|
|
|
|
printf 'Verification succeeded: all checked filesystems are identical.\n' |