Resume verify aven if error exsists

zfs-verify-over-ssh.sh

@@ -8,8 +8,9 @@ usage() {
 Usage: zfs-verify-over-ssh.sh [options] SOURCE_POOL REMOTE_HOST DESTINATION_ROOT
 
 Compare every mounted filesystem in SOURCE_POOL with its corresponding mounted
-remote filesystem below DESTINATION_ROOT. The script is read-only and exits 1
-when it finds differences.
+remote filesystem below DESTINATION_ROOT. The script is read-only, continues
+after per-filesystem errors, and exits 1 when it finds differences or 2 when
+one or more filesystems could not be checked.
 
 Arguments:
   SOURCE_POOL       Source ZFS pool, for example: tank
@@ -105,14 +106,20 @@ require_mounted_filesystem() {
     local mountpoint="$3"
     local mounted="$4"
 
-    [[ "$mountpoint" != none && "$mountpoint" != legacy ]] \
-        || die "$host_label filesystem $filesystem has mountpoint=$mountpoint"
-    [[ "$mounted" == yes ]] \
-        || die "$host_label filesystem $filesystem is not mounted at $mountpoint"
+    if [[ "$mountpoint" == none || "$mountpoint" == legacy ]]; then
+        printf 'ERROR %s filesystem %s has mountpoint=%s\n' "$host_label" "$filesystem" "$mountpoint" >&2
+        return 1
+    fi
+    if [[ "$mounted" != yes ]]; then
+        printf 'ERROR %s filesystem %s is not mounted at %s\n' "$host_label" "$filesystem" "$mountpoint" >&2
+        return 1
+    fi
 
     if [[ "$host_label" == local ]]; then
-        [[ -d "$mountpoint" ]] \
-            || die "local filesystem $filesystem is not mounted at $mountpoint"
+        if [[ ! -d "$mountpoint" ]]; then
+            printf 'ERROR local filesystem %s is not mounted at %s\n' "$filesystem" "$mountpoint" >&2
+            return 1
+        fi
     fi
 }
 
@@ -124,16 +131,39 @@ printf -v RSYNC_RSH '%q ' "${SSH_COMMAND[@]}"
 REPORT_FILE="$(mktemp)"
 trap 'rm -f "$REPORT_FILE"' EXIT
 DIFFERENCES=0
+ERRORS=0
 
-for filesystem in "${FILESYSTEMS[@]}"; do
+verify_filesystem() {
+    local filesystem="$1"
     destination="$(map_destination "$filesystem")"
-    source_mountpoint="$(zfs get -H -o value mountpoint "$filesystem")"
-    source_mounted="$(zfs get -H -o value mounted "$filesystem")"
-    remote_mountpoint="$(get_remote_property mountpoint "$destination")"
-    remote_mounted="$(get_remote_property mounted "$destination")"
+    local source_mountpoint
+    local source_mounted
+    local remote_mountpoint
+    local remote_mounted
 
-    require_mounted_filesystem local "$filesystem" "$source_mountpoint" "$source_mounted"
-    require_mounted_filesystem remote "$destination" "$remote_mountpoint" "$remote_mounted"
+    if ! source_mountpoint="$(zfs get -H -o value mountpoint "$filesystem" 2>&1)"; then
+        printf 'ERROR %s: could not read local mountpoint: %s\n' "$filesystem" "$source_mountpoint" >&2
+        return 1
+    fi
+    if ! source_mounted="$(zfs get -H -o value mounted "$filesystem" 2>&1)"; then
+        printf 'ERROR %s: could not read local mounted state: %s\n' "$filesystem" "$source_mounted" >&2
+        return 1
+    fi
+    if ! remote_mountpoint="$(get_remote_property mountpoint "$destination" 2>&1)"; then
+        printf 'ERROR %s: could not read remote mountpoint for %s: %s\n' "$filesystem" "$destination" "$remote_mountpoint" >&2
+        return 1
+    fi
+    if ! remote_mounted="$(get_remote_property mounted "$destination" 2>&1)"; then
+        printf 'ERROR %s: could not read remote mounted state for %s: %s\n' "$filesystem" "$destination" "$remote_mounted" >&2
+        return 1
+    fi
+
+    if ! require_mounted_filesystem local "$filesystem" "$source_mountpoint" "$source_mounted"; then
+        return 1
+    fi
+    if ! require_mounted_filesystem remote "$destination" "$remote_mountpoint" "$remote_mounted"; then
+        return 1
+    fi
 
     printf 'Verifying %s against %s:%s\n' "$filesystem" "$REMOTE_HOST" "$destination"
     : >"$REPORT_FILE"
@@ -141,8 +171,8 @@ for filesystem in "${FILESYSTEMS[@]}"; do
         --one-file-system --itemize-changes --out-format='%i %n%L' \
         -e "$RSYNC_RSH" --rsync-path='sudo -n rsync' \
         "$source_mountpoint/" "$REMOTE_HOST:$remote_mountpoint/" >"$REPORT_FILE"; then
-        printf 'Unable to compare %s.\n' "$filesystem" >&2
-        exit 2
+        printf 'ERROR %s: rsync comparison failed.\n' "$filesystem" >&2
+        return 1
     fi
 
     if [[ -s "$REPORT_FILE" ]]; then
@@ -150,8 +180,20 @@ for filesystem in "${FILESYSTEMS[@]}"; do
         cat "$REPORT_FILE" >&2
         DIFFERENCES=1
     fi
+}
+
+for filesystem in "${FILESYSTEMS[@]}"; do
+    if ! verify_filesystem "$filesystem"; then
+        ((ERRORS += 1))
+    fi
 done
 
+if ((ERRORS)); then
+    printf 'Verification incomplete: %d filesystem(s) could not be checked.\n' "$ERRORS" >&2
+    printf 'Review the ERROR lines above; checked filesystem differences are also listed above.\n' >&2
+    exit 2
+fi
+
 if ((DIFFERENCES)); then
     printf 'Verification failed: source and remote filesystems differ.\n' >&2
     exit 1