Resume verify aven if error exsists

README.md

@@ -94,6 +94,11 @@ checking, and `--one-file-system`. It does not modify either host. Any
 itemized `rsync` output identifies an extra, missing, changed, or
 metadata-different path.
 
+The verifier continues with the next filesystem when one dataset cannot be
+checked. It prints an `ERROR` line for each failed dataset and summarizes the
+number of uncheckable filesystems at the end, so one run shows all failures
+encountered along the way.
+
 Verification options:
 
 | Option | Description |
@@ -116,8 +121,8 @@ Exit statuses:
 | Status | Meaning |
 | --- | --- |
 | `0` | All checked filesystems are identical. |
-| `1` | One or more filesystem pairs have differences. |
+| `1` | All filesystem pairs were checked, but one or more have differences. |
-| `2` | Verification could not run, for example because of missing commands, SSH or sudo access, unavailable datasets, or unmounted filesystems. |
+| `2` | One or more filesystem pairs could not be checked, for example because a dataset is unavailable or unmounted, or its SSH, sudo, or `rsync` comparison failed. The script continues and lists these errors before exiting. |
 
 For a stable result, stop writes to the source and destination while the
 verifier runs, or verify mounted clones of the same snapshots. The verifier

zfs-verify-over-ssh.sh

@@ -8,8 +8,9 @@ usage() {
 Usage: zfs-verify-over-ssh.sh [options] SOURCE_POOL REMOTE_HOST DESTINATION_ROOT
 
 Compare every mounted filesystem in SOURCE_POOL with its corresponding mounted
-remote filesystem below DESTINATION_ROOT. The script is read-only and exits 1
+remote filesystem below DESTINATION_ROOT. The script is read-only, continues
-when it finds differences.
+after per-filesystem errors, and exits 1 when it finds differences or 2 when
+one or more filesystems could not be checked.
 
 Arguments:
   SOURCE_POOL       Source ZFS pool, for example: tank
@@ -105,14 +106,20 @@ require_mounted_filesystem() {
     local mountpoint="$3"
     local mounted="$4"
 
-    [[ "$mountpoint" != none && "$mountpoint" != legacy ]] \
+    if [[ "$mountpoint" == none || "$mountpoint" == legacy ]]; then
-        || die "$host_label filesystem $filesystem has mountpoint=$mountpoint"
+        printf 'ERROR %s filesystem %s has mountpoint=%s\n' "$host_label" "$filesystem" "$mountpoint" >&2
-    [[ "$mounted" == yes ]] \
+        return 1
-        || die "$host_label filesystem $filesystem is not mounted at $mountpoint"
+    fi
+    if [[ "$mounted" != yes ]]; then
+        printf 'ERROR %s filesystem %s is not mounted at %s\n' "$host_label" "$filesystem" "$mountpoint" >&2
+        return 1
+    fi
 
     if [[ "$host_label" == local ]]; then
-        [[ -d "$mountpoint" ]] \
+        if [[ ! -d "$mountpoint" ]]; then
-            || die "local filesystem $filesystem is not mounted at $mountpoint"
+            printf 'ERROR local filesystem %s is not mounted at %s\n' "$filesystem" "$mountpoint" >&2
+            return 1
+        fi
     fi
 }
 
@@ -124,16 +131,39 @@ printf -v RSYNC_RSH '%q ' "${SSH_COMMAND[@]}"
 REPORT_FILE="$(mktemp)"
 trap 'rm -f "$REPORT_FILE"' EXIT
 DIFFERENCES=0
+ERRORS=0
 
-for filesystem in "${FILESYSTEMS[@]}"; do
+verify_filesystem() {
+    local filesystem="$1"
     destination="$(map_destination "$filesystem")"
-    source_mountpoint="$(zfs get -H -o value mountpoint "$filesystem")"
+    local source_mountpoint
-    source_mounted="$(zfs get -H -o value mounted "$filesystem")"
+    local source_mounted
-    remote_mountpoint="$(get_remote_property mountpoint "$destination")"
+    local remote_mountpoint
-    remote_mounted="$(get_remote_property mounted "$destination")"
+    local remote_mounted
 
-    require_mounted_filesystem local "$filesystem" "$source_mountpoint" "$source_mounted"
+    if ! source_mountpoint="$(zfs get -H -o value mountpoint "$filesystem" 2>&1)"; then
-    require_mounted_filesystem remote "$destination" "$remote_mountpoint" "$remote_mounted"
+        printf 'ERROR %s: could not read local mountpoint: %s\n' "$filesystem" "$source_mountpoint" >&2
+        return 1
+    fi
+    if ! source_mounted="$(zfs get -H -o value mounted "$filesystem" 2>&1)"; then
+        printf 'ERROR %s: could not read local mounted state: %s\n' "$filesystem" "$source_mounted" >&2
+        return 1
+    fi
+    if ! remote_mountpoint="$(get_remote_property mountpoint "$destination" 2>&1)"; then
+        printf 'ERROR %s: could not read remote mountpoint for %s: %s\n' "$filesystem" "$destination" "$remote_mountpoint" >&2
+        return 1
+    fi
+    if ! remote_mounted="$(get_remote_property mounted "$destination" 2>&1)"; then
+        printf 'ERROR %s: could not read remote mounted state for %s: %s\n' "$filesystem" "$destination" "$remote_mounted" >&2
+        return 1
+    fi
+
+    if ! require_mounted_filesystem local "$filesystem" "$source_mountpoint" "$source_mounted"; then
+        return 1
+    fi
+    if ! require_mounted_filesystem remote "$destination" "$remote_mountpoint" "$remote_mounted"; then
+        return 1
+    fi
 
     printf 'Verifying %s against %s:%s\n' "$filesystem" "$REMOTE_HOST" "$destination"
     : >"$REPORT_FILE"
@@ -141,8 +171,8 @@ for filesystem in "${FILESYSTEMS[@]}"; do
         --one-file-system --itemize-changes --out-format='%i %n%L' \
         -e "$RSYNC_RSH" --rsync-path='sudo -n rsync' \
         "$source_mountpoint/" "$REMOTE_HOST:$remote_mountpoint/" >"$REPORT_FILE"; then
-        printf 'Unable to compare %s.\n' "$filesystem" >&2
+        printf 'ERROR %s: rsync comparison failed.\n' "$filesystem" >&2
-        exit 2
+        return 1
     fi
 
     if [[ -s "$REPORT_FILE" ]]; then
@@ -150,8 +180,20 @@ for filesystem in "${FILESYSTEMS[@]}"; do
         cat "$REPORT_FILE" >&2
         DIFFERENCES=1
     fi
+}
+
+for filesystem in "${FILESYSTEMS[@]}"; do
+    if ! verify_filesystem "$filesystem"; then
+        ((ERRORS += 1))
+    fi
 done
 
+if ((ERRORS)); then
+    printf 'Verification incomplete: %d filesystem(s) could not be checked.\n' "$ERRORS" >&2
+    printf 'Review the ERROR lines above; checked filesystem differences are also listed above.\n' >&2
+    exit 2
+fi
+
 if ((DIFFERENCES)); then
     printf 'Verification failed: source and remote filesystems differ.\n' >&2
     exit 1