From 6e5f2ddde8ab2c79d4e984254e07bf474bd64438 Mon Sep 17 00:00:00 2001 From: Tero Date: Fri, 4 Sep 2026 09:10:14 +0300 Subject: [PATCH] Resume verify aven if error exsists --- README.md | 9 +++-- zfs-verify-over-ssh.sh | 76 ++++++++++++++++++++++++++++++++---------- 2 files changed, 66 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 2ae1bdd..20f0c1c 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,11 @@ checking, and `--one-file-system`. It does not modify either host. Any itemized `rsync` output identifies an extra, missing, changed, or metadata-different path. +The verifier continues with the next filesystem when one dataset cannot be +checked. It prints an `ERROR` line for each failed dataset and summarizes the +number of uncheckable filesystems at the end, so one run shows all failures +encountered along the way. + Verification options: | Option | Description | @@ -116,8 +121,8 @@ Exit statuses: | Status | Meaning | | --- | --- | | `0` | All checked filesystems are identical. | -| `1` | One or more filesystem pairs have differences. | -| `2` | Verification could not run, for example because of missing commands, SSH or sudo access, unavailable datasets, or unmounted filesystems. | +| `1` | All filesystem pairs were checked, but one or more have differences. | +| `2` | One or more filesystem pairs could not be checked, for example because a dataset is unavailable or unmounted, or its SSH, sudo, or `rsync` comparison failed. The script continues and lists these errors before exiting. | For a stable result, stop writes to the source and destination while the verifier runs, or verify mounted clones of the same snapshots. The verifier diff --git a/zfs-verify-over-ssh.sh b/zfs-verify-over-ssh.sh index 7e28ed0..64ea76f 100644 --- a/zfs-verify-over-ssh.sh +++ b/zfs-verify-over-ssh.sh @@ -8,8 +8,9 @@ usage() { Usage: zfs-verify-over-ssh.sh [options] SOURCE_POOL REMOTE_HOST DESTINATION_ROOT Compare every mounted filesystem in SOURCE_POOL with its corresponding mounted -remote filesystem below DESTINATION_ROOT. The script is read-only and exits 1 -when it finds differences. +remote filesystem below DESTINATION_ROOT. The script is read-only, continues +after per-filesystem errors, and exits 1 when it finds differences or 2 when +one or more filesystems could not be checked. Arguments: SOURCE_POOL Source ZFS pool, for example: tank @@ -105,14 +106,20 @@ require_mounted_filesystem() { local mountpoint="$3" local mounted="$4" - [[ "$mountpoint" != none && "$mountpoint" != legacy ]] \ - || die "$host_label filesystem $filesystem has mountpoint=$mountpoint" - [[ "$mounted" == yes ]] \ - || die "$host_label filesystem $filesystem is not mounted at $mountpoint" + if [[ "$mountpoint" == none || "$mountpoint" == legacy ]]; then + printf 'ERROR %s filesystem %s has mountpoint=%s\n' "$host_label" "$filesystem" "$mountpoint" >&2 + return 1 + fi + if [[ "$mounted" != yes ]]; then + printf 'ERROR %s filesystem %s is not mounted at %s\n' "$host_label" "$filesystem" "$mountpoint" >&2 + return 1 + fi if [[ "$host_label" == local ]]; then - [[ -d "$mountpoint" ]] \ - || die "local filesystem $filesystem is not mounted at $mountpoint" + if [[ ! -d "$mountpoint" ]]; then + printf 'ERROR local filesystem %s is not mounted at %s\n' "$filesystem" "$mountpoint" >&2 + return 1 + fi fi } @@ -124,16 +131,39 @@ printf -v RSYNC_RSH '%q ' "${SSH_COMMAND[@]}" REPORT_FILE="$(mktemp)" trap 'rm -f "$REPORT_FILE"' EXIT DIFFERENCES=0 +ERRORS=0 -for filesystem in "${FILESYSTEMS[@]}"; do +verify_filesystem() { + local filesystem="$1" destination="$(map_destination "$filesystem")" - source_mountpoint="$(zfs get -H -o value mountpoint "$filesystem")" - source_mounted="$(zfs get -H -o value mounted "$filesystem")" - remote_mountpoint="$(get_remote_property mountpoint "$destination")" - remote_mounted="$(get_remote_property mounted "$destination")" + local source_mountpoint + local source_mounted + local remote_mountpoint + local remote_mounted - require_mounted_filesystem local "$filesystem" "$source_mountpoint" "$source_mounted" - require_mounted_filesystem remote "$destination" "$remote_mountpoint" "$remote_mounted" + if ! source_mountpoint="$(zfs get -H -o value mountpoint "$filesystem" 2>&1)"; then + printf 'ERROR %s: could not read local mountpoint: %s\n' "$filesystem" "$source_mountpoint" >&2 + return 1 + fi + if ! source_mounted="$(zfs get -H -o value mounted "$filesystem" 2>&1)"; then + printf 'ERROR %s: could not read local mounted state: %s\n' "$filesystem" "$source_mounted" >&2 + return 1 + fi + if ! remote_mountpoint="$(get_remote_property mountpoint "$destination" 2>&1)"; then + printf 'ERROR %s: could not read remote mountpoint for %s: %s\n' "$filesystem" "$destination" "$remote_mountpoint" >&2 + return 1 + fi + if ! remote_mounted="$(get_remote_property mounted "$destination" 2>&1)"; then + printf 'ERROR %s: could not read remote mounted state for %s: %s\n' "$filesystem" "$destination" "$remote_mounted" >&2 + return 1 + fi + + if ! require_mounted_filesystem local "$filesystem" "$source_mountpoint" "$source_mounted"; then + return 1 + fi + if ! require_mounted_filesystem remote "$destination" "$remote_mountpoint" "$remote_mounted"; then + return 1 + fi printf 'Verifying %s against %s:%s\n' "$filesystem" "$REMOTE_HOST" "$destination" : >"$REPORT_FILE" @@ -141,8 +171,8 @@ for filesystem in "${FILESYSTEMS[@]}"; do --one-file-system --itemize-changes --out-format='%i %n%L' \ -e "$RSYNC_RSH" --rsync-path='sudo -n rsync' \ "$source_mountpoint/" "$REMOTE_HOST:$remote_mountpoint/" >"$REPORT_FILE"; then - printf 'Unable to compare %s.\n' "$filesystem" >&2 - exit 2 + printf 'ERROR %s: rsync comparison failed.\n' "$filesystem" >&2 + return 1 fi if [[ -s "$REPORT_FILE" ]]; then @@ -150,8 +180,20 @@ for filesystem in "${FILESYSTEMS[@]}"; do cat "$REPORT_FILE" >&2 DIFFERENCES=1 fi +} + +for filesystem in "${FILESYSTEMS[@]}"; do + if ! verify_filesystem "$filesystem"; then + ((ERRORS += 1)) + fi done +if ((ERRORS)); then + printf 'Verification incomplete: %d filesystem(s) could not be checked.\n' "$ERRORS" >&2 + printf 'Review the ERROR lines above; checked filesystem differences are also listed above.\n' >&2 + exit 2 +fi + if ((DIFFERENCES)); then printf 'Verification failed: source and remote filesystems differ.\n' >&2 exit 1