X-Git-Url: https://iankelling.org/git/?p=distro-setup;a=blobdiff_plain;f=btrbk-run;h=48550db1a4b2793418501cbfa32a1b234d538994;hp=2e839eefe586f60166df27c0de2a37ef5c8f83a2;hb=HEAD;hpb=7b47d6a266340223e78317cfe0570868f45a4cad diff --git a/btrbk-run b/btrbk-run index 2e839ee..8fc4c4f 100644 --- a/btrbk-run +++ b/btrbk-run @@ -1,17 +1,22 @@ #!/bin/bash -# Copyright (C) 2016 Ian Kelling -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Configure & run btrbk & related work on Ian's computers. +# Copyright (C) 2024 Ian Kelling -# http://www.apache.org/licenses/LICENSE-2.0 +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# SPDX-License-Identifier: GPL-3.0-or-later # todo: if we cancel in the middle of a btrfs send, then run again @@ -22,11 +27,12 @@ [[ $EUID == 0 ]] || exec sudo -E "${BASH_SOURCE[0]}" "$@" -source /usr/local/lib/err +set -e; . /usr/local/lib/bash-bear; set +e +shopt -s nullglob usage() { cat <<'EOF' -btrbk-run [OPTIONS] +btrbk-run [OPTIONS] [run|resume|archive] usually -t TARGET_HOST or -s SOURCE_HOST Note, at source location, intentionally not executable, run and read @@ -39,12 +45,108 @@ EOF } +pre=btrbk-run + +script_name="${BASH_SOURCE[0]}" +script_name="${script_name##*/}" +d() { + if $dry_run || $conf_only; then + printf "$pre dry-run: %s\n" "$*" + else + printf "$pre running: %s\n" "$*" + "$@" + fi +} +m() { if $verbose; then printf "$pre %s\n" "$*"; fi; "$@"; } +e() { printf "$pre %s\n" "$*"; } +die() { printf "$pre error: %s\n" "$*" >&2; echo "$pre exiting with status 1" >&2; exit 1; } +mexit() { echo "$pre exiting with status $1"; exit $1; } + +uninstalled-file-die() { + die "uninstalled file $1. run install-my-scripts or rerun with -f" +} + +set-location() { + case $HOSTNAME in + kw) + at_work=true + ;; + kd|frodo) + at_home=true + ;; + x2|x3|sy|so) + if [[ $(dig +short @10.2.0.1 -x 10.2.0.2 2>&1 ||:) == kd.b8.nz. ]] \ + && ip n show 10.2.0.1 | grep . &>/dev/null; then + # note: logic duplicated in 11-iank + at_home=true + elif ping -q -c1 -w1 hal.office.fsf.org &>/dev/null \ + && ip n show 192.168.0.26 | grep . &>/dev/null; then + at_work=true + fi + ;; + esac +} + +exit-if-no-default-targets() { + if ! $force && [[ $HOSTNAME != "$MAIL_HOST" ]]; then + echo "MAIL_HOST=$MAIL_HOST, nothing to do" + mexit 0 + fi + case $HOSTNAME in + kw|kd|frodo|x2|x3|sy|so) : ;; + *) + die "error: no default targets for this host, use -t" + ;; + esac +} + +add-x3-target() { + # main work machine + if ping -q -c1 -w1 x3.office.fsf.org &>/dev/null; then + targets+=(x3.office.fsf.org) + elif ping -q -c1 -w1 $h.b8.nz &>/dev/null; then + # in case we took it home + targets+=(x3.b8.nz) + elif ping -q -c1 -w1 ${h}w.b8.nz &>/dev/null; then + targets+=(x3w.b8.nz) + else + targets+=(x3wg.b8.nz) + fi +} + +add-wireless-target() { + local host + if [[ ! $1 ]]; then + set -- $h + fi + for host; do + # c = cabled, w = wireless + if ping -q -c1 -w1 ${host}c.b8.nz &>/dev/null; then + targets+=(${host}c.b8.nz) + elif ping -q -c1 -w1 $host.b8.nz &>/dev/null; then + targets+=($host.b8.nz) + elif ping -q -c1 -w1 ${host}w.b8.nz &>/dev/null; then + targets+=(${host}w.b8.nz) + fi + done +} + +qconf() { + case $sub in + q) + # q has sensitive data i dont want to backup for so long + cat >>/etc/btrbk$conf_suf.conf <&2; echo "exiting with status 1" >&2; exit 1; } -mexit() { echo "$pre: exiting with status $1"; exit $1; } # latest $MAIL_HOST if [[ -e /b/bash_unpublished/source-state ]]; then @@ -63,12 +165,20 @@ conf_only=false dry_run=false # mostly for testing rate_limit=no verbose=true; verbose_arg=-v -progress_arg="--progress" +force=false +if [[ $INVOCATION_ID ]]; then + # INVOCATION_ID means running as a systemd service. we cant show progress in this case, + # but if we pass the arg, it will insert mbuffer into the command. + progress_arg= +else + progress_arg="--progress" +fi incremental_strict=false pull_reexec=false default_args_file=/etc/btrbk-run.conf if [[ -s $default_args_file ]]; then + # shellcheck disable=SC2046 # we want word splitting set -- $(< $default_args_file) "$@" # i havent used this feature yet, so warn about it echo "$0: warning: default btrbk-run options set in $default_args_file (sleeping 5 seconds):" @@ -76,30 +186,61 @@ if [[ -s $default_args_file ]]; then sleep 5 fi -early=false # quit early, just btrbk, no extra remounting etc. -cron=false +once_args_file=/etc/btrbk-run-once.conf +if mv -f $once_args_file $once_args_file-tmp 2> >(sed '/No such file or directory/d'); then + # shellcheck disable=SC2046 # we want word splitting + set -- $(< $once_args_file-tmp) "$@" + # i havent used this feature yet, so warn about it + echo "$0: btrbk-run options set in $once_args_file:" + cat $once_args_file-tmp + rm -f $once_args_file-tmp +fi + + +targets=() +early=false +fast=false +kd_spread=false +check_installed=false orig_args=("$@") -temp=$(getopt -l cron,pull-reexec,help ceil:m:npqs:t:vh "$@") || usage 1 +temp=$(getopt -l check-installed,fast,pull-reexec,help 23cefikl:m:npqrs:t:vh "$@") || usage 1 eval set -- "$temp" while true; do case $1 in - --cron) - cron=true - pre= - shift - ;; + # for the rare case we want to run multiple instances at the same time + -2) conf_suf=2 ;; + -3) conf_suf=3 ;; # only creates the config file, does not run btrbk - -c) conf_only=true; shift ;; - -e) early=true; shift ;; - -i) incremental_strict=true; shift ;; + -c) conf_only=true ;; + --check-installed) + check_installed=true + ;; + # quit early, just btrbk, no extra remounting etc. + -e) early=true ;; + # avoids some default behaviors: + # - no skipping hosts where xprintidle haven't been idle recently + # - exit if we can't ssh to 1 or more hosts + # - still set default hosts despite MAIL_HOST status + -f) force=true ;; + # skip various checks. when we run twice in a row for + # switch mail-host, no need to repeat the same checks again. + --fast) fast=true ;; + -i) incremental_strict=true ;; + # note this implies resume and -p because it is just meant to make + # other hosts have the same snapshots, not do any expiry or new + # backups. + -k) kd_spread=true ;; # bytes per second, suffix k m g - -l) rate_limit=$2; shift 2 ;; + -l) rate_limit=$2; shift ;; # Comma separated mountpoints to backup. This has defaults set below. - -m) IFS=, mountpoints=($2); unset IFS; shift 2 ;; - -n) dry_run=true; dry_run_arg=-n; shift ;; - -p) progress_arg="--progress"; shift ;; - --pull-reexec) pull_reexec=true; shift ;; - -q) verbose=false; verbose_arg=; progress_arg=; shift ;; + -m) IFS=, mountpoints=($2); unset IFS; shift ;; + -n) dry_run=true ;; + # preserve existing snapshots and backups + -p) preserve_arg=-p ;; + # internal option for rerunning under newer SOURCE_HOST version. + --pull-reexec) pull_reexec=true;; + # quiet + -q) verbose=false; verbose_arg=; progress_arg= ;; # source host to receive a backup from -s) source=$2 @@ -107,23 +248,83 @@ while true; do if [[ $source == *:* ]]; then bbksource="[$source]" fi - shift 2 + shift ;; # target hosts to send to. empty is valid for just doing local # snapshot. we have default hosts we will populate. - -t) IFS=, targets=($2); unset IFS; shift 2 ;; - -v) verbose=true; verbose_arg=-v; shift ;; + -t) IFS=, targets=($2); unset IFS; shift ;; + # verbose. + -v) verbose=true; verbose_arg=-v ;; -h|--help) usage ;; --) shift; break ;; *) die "Internal error!" ;; esac + shift done -# only tested commands are resume and archive -cmd_arg=${1:-run} +cmd_arg="$1" + + + +if ! $force && { $check_installed || [[ ! $source ]]; } ; then + install_bin_files=( + mount-latest-subvol + check-subvol-stale + btrbk-run + ) + for f in ${install_bin_files[@]}; do + if ! diff -q /a/bin/ds/$f /usr/local/bin/$f; then + uninstalled-file-die $f + fi + done + if ! diff -q /a/bin/bash-bear-trap/bash-bear /usr/local/lib/bash-bear; then + uninstalled-file-die err + fi + if $check_installed; then + exit 0 + fi +fi + + +if $kd_spread; then + if [[ $cmd_arg && $cmd_arg != resume ]]; then + die "dont pass -k without resume or empty run arg" + fi + if [[ $HOSTNAME == "$MAIL_HOST" ]]; then + die "something went wrong, -k not meant to be run on MAIL_HOST" + fi + if [[ $HOSTNAME != kd ]]; then + die "something went wrong, -k only meant to run on kd" + fi + cmd_arg=resume + preserve_arg=-p + add-wireless-target sy so +fi + +if [[ ! $cmd_arg ]]; then + cmd_arg=run +fi + + +std_preserve="36h 14d 8w 24m" +q_preserve="18h 14d 8w" case $cmd_arg in - run|resume|archive) : ;; + run|resume) : ;; + + # This works better than the normal archive command. We have to + # specify the mount points, but that is what we are used to doing and + # we prefer it. Another difference is that archive works recursively + # and we don't care about that. Sometimes we may still want to run + # btrbk archive, but it doesn't even use the config file, so just + # run it directly, eg: + # time s btrbk -v archive /mnt/r7/amy/boot/btrbk ssh://bo/mnt/boot2/btrbk + archive) + cmd_arg=resume + std_preserve="999h 999d 999w 999m" + q_preserve="$std_preserve" + preserve_arg=-p + ;; *) die "untested command arg" ;; esac @@ -137,9 +338,6 @@ if [[ -v targets && $source ]]; then die "error: -t and -s are mutually exclusive" fi -if $verbose; then - printf "options: conf_only=%s\ndry_run=%s\nrate_limit=%s\nverbose=%s\ncmd_arg=%s" "$conf_only" "$dry_run" "$rate_limit" "$verbose" "$cmd_arg" -fi ### end options parsing # remove path from earlier version of btrbk @@ -152,82 +350,51 @@ if [[ /a/opt/btrbk/btrbk -nt /usr/bin/btrbk ]]; then fi cd /a/opt/btrbk m make install + cd / fi # TODO: i wonder if there should be an option to send to the default # targets, plus any given on the command line. -amy=false -# set default targets -if [[ ! -v targets && ! $source ]]; then - if [[ $HOSTNAME != "$MAIL_HOST" ]] && $cron ; then - echo "MAIL_HOST=$MAIL_HOST, nothing to do" - mexit 0 - else - amy=true - fi +at_work=false +at_home=false - at_work=false - targets=(frodo.b8.nz) - case $HOSTNAME in - kw) - at_work=true - ;;& - x2|x3|sy|bo) - if ping -q -c1 -w1 hal.office.fsf.org \ - && ip n show 192.168.0.26 | grep . &>/dev/null; then - at_work=true +# set default targets +if [[ ! -v targets && ! $source ]]; then + exit-if-no-default-targets + set-location + if $at_home; then + if ! $kd_spread && [[ $HOSTNAME != x3 ]]; then + add-x3-target + fi + if [[ $HOSTNAME != kd ]]; then + targets+=(kd.b8.nz) + fi + wireless_home_hosts=( + x2 + sy + so + ) + for h in ${wireless_home_hosts[@]}; do + if [[ $HOSTNAME != "$h" ]]; then + add-wireless-target fi - ;;& - kw|x2|x3|sy|bo) - if $at_work; then - if ping -q -c1 -w1 iank.vpn.office.fsf.org &>/dev/null; then - home=iank.vpn.office.fsf.org - else - home=i.b8.nz - fi - else - home=b8.nz + done + elif $at_work; then + targets+=(b8.nz) + for h in x2 x3 kw; do + if [[ $HOSTNAME == "$h" ]]; then + continue fi - ;;& - kw) - targets+=($home x3) - ;; - x2|x3|sy|bo) - targets+=($home) - if $at_work; then - targets+=(kw.office.fsf.org x2.b8.nz) - else - targets+=(kw.b8.nz) + if ping -q -c1 -w1 $h.office.fsf.org &>/dev/null; then + targets+=($h.office.fsf.org) fi - ;; - kd) - targets+=(x2.b8.nz kw.b8.nz) - # temporarily disabled while doing maint - # if ping -q -c1 -w1 bo.b8.nz &>/dev/null; then - # targets+=(bo.b8.nz) - # else - # targets+=(bow.b8.nz) - # fi - ;; - frodo) - # no targets - targets=() - ;; - *) - die "error: no default targets for this host, use -t" - ;; - esac -fi - -if [[ -v targets ]]; then - echo "targets: ${targets[*]}" -fi - -if [[ $source ]]; then - echo "source: $source" + done + else + targets+=(b8.nz) + fi fi if [[ ${mountpoints[0]} ]]; then @@ -242,28 +409,31 @@ else prospective_mps=(/a) else case $HOSTNAME in - frodo) - prospective_mps=(/i) - ;; *) - prospective_mps=(/a /q) + prospective_mps=() if [[ $source ]]; then - source_state="$(ssh $source cat /a/bin/bash_unpublished/source-state)" + source_state="$(ssh $source 'cat /a/bin/bash_unpublished/source-state; echo source_host=$HOSTNAME')" eval "$source_state" - source_host="$(ssh $source cat /etc/hostname)" + # shellcheck disable=SC2154 # assigned in the above eval. if [[ $source_host == "$MAIL_HOST" ]]; then prospective_mps+=(/o) fi + if [[ $source_host == "$HOST2" ]]; then + prospective_mps+=(/a /qr /qd /q) + fi else if [[ $HOSTNAME == "$MAIL_HOST" ]]; then - # HOST2 is really the mail host if it exists - if [[ $HOST2 && $HOST2 != "$HOSTNAME" ]]; then - echo "skipping /o because HOST2 is not us" - else - prospective_mps+=(/o) - fi + prospective_mps+=(/o) + fi + if [[ $HOSTNAME == "$HOST2" ]]; then + prospective_mps+=(/a /qr /qd /q) + fi + if $kd_spread; then + prospective_mps=(/a /o /qr /qd /q) fi fi + # note: put q last just in case its specific retention options were to + # affect other config sections. I havent tested if that is the case. ;; esac fi @@ -278,22 +448,88 @@ else done fi -echo "mountpoints: ${mountpoints[*]}" +tmp=$(( ${#mountpoints[@]} == 0 )) +if (( tmp )); then + die didnt get mountpoint arg and had no defaults +fi ##### end command line parsing ######## -if [[ $source ]]; then +#### begin pre-checks ##### + +# todo: this has a timing problem, since btrbk.timer could activate the service after this check. +if ! $fast && [[ $source ]]; then if [[ $(ssh $source ps --no-headers -o comm 1) == systemd ]]; then status=$(ssh $source systemctl is-active btrbk.service) || : # normally returns 3 case $status in inactive|failed) : ;; *) - echo "$0: error: cron btrbk is running on source. exiting out of caution" + echo "$0: error: btrbk is running on source. exiting out of caution" mexit 1 esac fi fi +if ! command -v btrbk &>/dev/null; then + die "error: no btrbk binary found" +fi + +if ! $pull_reexec && [[ $source ]] && $pulla && ! $force ; then + ssh root@$source btrbk-run --check-installed || exit 1 +fi + +#### end pre-checks ##### + + + +mkdir -p /var/log/btrbk +# The journal doesnt go back to my oldest backups, and I've found myself +# wanting older logs. Not going to bother expiring old logs, since it is +# fine if they go back years. +log_path=/var/log/btrbk/$(date +%F_%T%:::z).log +echo copying output to $log_path +exec &> >(pee cat 'ts "%F %T"|dd of='$log_path' status=none') + +# print some non-default opts +if $verbose; then + opts_show=() + if ! $conf_only; then + opts_show+=(conf_only=true) + fi + if ! $dry_run; then + opts_show+=(dry_run=true) + fi + if [[ $rate_limit != no ]]; then + opts_show+=("rate_limit=$rate_limit") + fi + if [[ $cmd_arg != run ]]; then + opts_show+=(cmd_arg=$cmd_arg) + fi + if (( ${#opts_show[@]} >= 1 )); then + first=true + for opt in ${opts_show[@]}; do + if $first; then + printf "%s" "$opt" + first=false + else + printf " %s" "$opt" + fi + done + echo + fi +fi + +if [[ -v targets ]]; then + echo "targets: ${targets[*]}" +fi + +if [[ $source ]]; then + echo "source: $source" +fi + +echo "mountpoints: ${mountpoints[*]}" + + # pull_reexec stops us from getting into an infinite loop if there is some # kind of weird problem pulla=false @@ -303,10 +539,11 @@ for m in "${mountpoints[@]}"; do break fi done + if ! $pull_reexec && [[ $source ]] && $pulla ; then tmpf=$(mktemp) m rsync -ra $source:/usr/local/bin/{mount-latest-subvol,check-subvol-stale} /usr/local/bin - m rsync -ra $source:/usr/local/lib/err /usr/local/lib + m rsync -ra $source:/usr/local/lib/bash-bear /usr/local/lib m scp $source:/a/bin/distro-setup/btrbk-run $tmpf if ! diff -q $tmpf ${BASH_SOURCE[0]}; then e "found different version on host $source. reexecing" @@ -316,74 +553,113 @@ if ! $pull_reexec && [[ $source ]] && $pulla ; then fi fi - -if ! which btrbk &>/dev/null; then - die "error: no btrbk binary found" +# todo: check if we have no snapshots yet, because I always want to run +# archive instead of run. Likely, I should give an error unless a cli +# override is passed. perhaps check-subvol-stale could give the error. +# see the error message "no snapshots found" in that file. +if ! $fast; then + # if our mountpoints are from stale snapshots, + # it doesn't make sense to do a backup. + m check-subvol-stale ${mountpoints[@]} || die "found stale mountpoints in ${mountpoints[*]}" + + # for an initial run, btrbk requires the dir to exist. + mkdir -p /mnt/{root,o}/btrbk fi -# if our mountpoints are from stale snapshots, -# it doesn't make sense to do a backup. -check-subvol-stale ${mountpoints[@]} || die "found stale mountpoints in ${mountpoints[*]}" - -# for an initial run, btrbk requires the dir to exist. -mkdir -p /mnt/root/btrbk local_zone=$(date +%z) if [[ $source ]]; then - if ! zone=$(ssh root@$source date +%z); then - die failed to ssh to root@$source - fi - if [[ $zone != "$local_zone" ]]; then - die "error: dont confuse yourself with multiple time zones. $h has different timezone than localhost" + if $fast; then + zone=$local_zone + else + if ! zone=$(ssh root@$source date +%z); then + if $conf_only; then + echo "$0: warning: failed to ssh to root@$source" + else + die failed to ssh to root@$source + fi + fi + if [[ $zone != "$local_zone" ]]; then + die "error: dont confuse yourself with multiple time zones. $h has different timezone than localhost" + fi fi - else sshable=() sshfail=() - min_idle_ms=$((1000 * 60 * 15)) - for h in ${targets[@]}; do - if remote_info=( $(timeout -s 9 6 ssh root@$h "mkdir -p /mnt/root/btrbk && date +%z && df --output=size,pcent / | tail -n1") ); then - zone=${remote_info[0]} - root_size=${remote_info[1]} - percent_used=${remote_info[2]%%%} + remote_str_cmd="mkdir -p /mnt/root/btrbk /mnt/o/btrbk && \ +date +%z && \ +df --output=size,pcent / | tail -n1" - if (( ${#remote_info[@]} != 3 )); then - die "error: didnt get 3 fields in test ssh to target $h. investigate" - fi - - - # we may be booted into a bootstrap fs or something - min_root_kb=$(( 1024 * 1024 * 200 )) # 200 gb - if (( root_size < min_root_kb )); then - continue + for h in ${targets[@]}; do + if $fast || $conf_only; then + # Use some typical values in this case + root_size=$(( 1024 * 1024 * 2000 )) #2tb + percent_used=10 + zone=$(date +%z) + elif remote_str=$(timeout -s 9 6 ssh root@$h "$remote_str_cmd"); then + mapfile -t tmp_array <<<"$remote_str" + zone="${tmp_array[0]}" + IFS=" " read -r root_size percent_used <<<"${tmp_array[1]}" + percent_used=${percent_used%%%} + + tmp=$(( ${#tmp_array[@]} != 2 )) + if (( tmp )); then + die "error: didnt get 2 lines in test ssh to target $h. investigate" fi + case $percent_used in + [0-9]|[1-9][0-9]) : ;; + *) + die "error: didnt get percent disk use in test ssh to target $h. investigate" + ;; + esac + else + sshfail+=($h) + continue + fi - if (( percent_used >= 98 )); then - die "error: filesystem on target $h is $percent_used % full" - fi + # we may be booted into a bootstrap fs or something + min_root_kb=$(( 1024 * 1024 * 200 )) # 200 gb + tmp=$(( root_size < min_root_kb )) + if (( tmp )); then + e "warning: $h: root_size=$root_size < 200gb, perhaps it is booted to bootstrap vol. skipping for now" + continue + fi - # This is a separate ssh because xprintidle can fail and thats ok. - if $cron && idle_ms=$(timeout -s 9 6 ssh $h DISPLAY=:0 xprintidle); then - if (( idle_ms < min_idle_ms )); then + tmp=$(( percent_used >= 98 )) + if (( tmp )); then + die "error: filesystem on target $h is $percent_used % full" + fi - # Ignore this host. i sometimes use a non-main machine for - # testing or web browsing, knowing that everything will be wiped - # by the next backup, but I dont want it to happen as Im using - # it from cronjob. - e "warning: $h: active X session in the last 15 minutes, skipping for now" - continue + # on sy, xprintidle is resetting every 12 seconds even when not + # idle, i dunno why, instead we are checking if the screen is locked, + # which is good enough. + # + # This is a separate ssh because the command can fail and thatis ok. + if ! $force; then + locked=false + if lock_info=$(timeout -s 9 6 ssh $h DISPLAY=:0 xscreensaver-command -time); then + if [[ $lock_info != *non-blanked* ]]; then + locked=true fi + else + locked=true fi - sshable+=($h) - if [[ $zone != "$local_zone" ]]; then - die "error: dont confuse yourself with multiple time zones. $h has different timezone than localhost" + if ! $locked; then + # Ignore this host. i sometimes use a non-main machine for + # testing or web browsing, knowing that everything will be wiped + # by the next backup, but I dont want it to happen as Im using + # it from cronjob. + e "warning: $h: seems to be actively in use, skipping for now" + continue fi - else - sshfail+=($h) + fi + sshable+=($h) + if [[ $zone != "$local_zone" ]]; then + die "error: dont confuse yourself with multiple time zones. $h has different timezone than localhost" fi done - if [[ ! ${sshable[*]} ]] || { ! $cron && [[ ${sshfail[*]} ]]; }; then - die "failed to ssh to hosts: ${sshfail[*]}" + if [[ ! ${sshable[*]} ]] || { $force && [[ ${sshfail[*]} ]]; }; then + die "see skipped host warning above or sshfail hosts: ${sshfail[*]}" else if [[ ${sshfail[*]} ]]; then ret=1 @@ -394,7 +670,7 @@ else fi -cat >/etc/btrbk.conf </etc/btrbk$conf_suf.conf <>/etc/btrbk.conf <>/etc/btrbk$conf_suf.conf <>/etc/btrbk.conf <>/etc/btrbk$conf_suf.conf <>/etc/btrbk$conf_suf.conf <>/etc/btrbk.conf <>/etc/btrbk$conf_suf.conf <>/etc/btrbk.conf <>/etc/btrbk$conf_suf.conf <>/etc/btrbk.conf <<'EOF' -# volume ssh://amy/mnt/root -# subvolume root_ubuntubionic -# target send-receive /mnt/root/btrbk -# EOF -# fi +# Delete any subvols on the receiving host that don't exist on the +# sending host. Otherwise, the receiving host could have snapshots that +# aren't on the sending side, and thus become odd leaf subvols, and then +# btrbk could try to use them when we sync back, creating a weird tree +# instead of linear parent/child relationship. Maybe this could lead to +# a missing source subvol error, so lets avoid it. + +get-orphan-tg-snaps() { + orphan_tg_snaps=() + for (( i=0; i < ${#mountpoints[@]}; i++ )); do + orphan_start_count=${#orphan_tg_snaps[@]} + tg_snap_count=0 + for tg_snap in ${tg_snaps[$i]}; do + tg_snap_count=$(( tg_snap_count + 1 )) + if [[ ! ${source_snaps[$tg_snap]} ]]; then + orphan_tg_snaps+=("$tg_snap") + fi + done + orphan_mp_count=$(( ${#orphan_tg_snaps[@]} - orphan_start_count )) + # sanity checking + tmp=$(( tg_snap_count > 1 && tg_snap_count == orphan_mp_count )) + if (( tmp )) ; then + die "something went wrong checking orphans on $tg: for mountpoint ${mountpoints[$i]}, $orphan_mp_count" + fi + done +} + +if [[ $source ]]; then + for snap in $(ssh root@$source "shopt -s nullglob; ${snap_list_cmds[*]}"); do + source_snaps[$snap]=t + done + get-orphan-tg-snaps + tmp=$(( ${#orphan_tg_snaps[*]} >= 1 )) + if (( tmp )); then + d btrfs sub del ${orphan_tg_snaps[*]} + fi +else # we have targets + for tg in ${targets[@]}; do + tmp_str=$(ssh root@$tg "shopt -s nullglob; ${snap_list_cmds[*]}") + mapfile -t tg_snaps <<<"$tmp_str" + get-orphan-tg-snaps + tmp=$(( ${#orphan_tg_snaps[*]} >= 1 )) + if (( tmp )); then + d ssh root@$tg "btrfs sub del ${orphan_tg_snaps[*]}" + fi + done +fi # todo: umount first to ensure we don't have any errors # todo: do some kill fuser stuff to make umount more reliable @@ -488,21 +843,16 @@ fi if $dry_run; then - m btrbk -v -n $cmd_arg - mexit 0 -elif [[ $cmd_arg == archive ]]; then - if [[ $source ]]; then - m btrbk $verbose_arg $progress_arg $cmd_arg ssh://$source$vol $vol - else - for tg in ${targets[@]}; do - m btrbk $verbose_arg $progress_arg $cmd_arg $vol ssh://$tg$vol - done - fi + m btrbk -c /etc/btrbk$conf_suf.conf -v -n $cmd_arg mexit 0 fi # -q and just using the syslog option seemed nice, # but it doesn't show when a send has a parent and when it doesn't. -m btrbk $verbose_arg $progress_arg $cmd_arg +m btrbk -c /etc/btrbk$conf_suf.conf $preserve_arg $verbose_arg $progress_arg $cmd_arg + +if $early; then + exit 0 +fi # todo: tp not valid anymore. # if we have it, sync to systems which don't @@ -522,17 +872,32 @@ if mountpoint $rsync_mountpoint >/dev/null; then done fi +subvols=() +for mp in "${mountpoints[@]}"; do + subvols+=("${mp##*/}") +done if [[ $source ]]; then - m mount-latest-subvol + m mount-latest-subvol "${subvols[@]}" else - m /a/exe/mount-latest-remote ${targets[@]} + for tg in ${targets[@]}; do + m /a/exe/mount-latest-remote "$tg" "${subvols[@]}" || ret=$? + done fi +# todo, we get hostnames earlier, reuse that. if [[ $ret == 0 ]]; then for tg in ${targets[@]}; do - : - #ssh root@$tg /a/exe/mail-backup-clean + h=$(ssh $tg hostname) + if [[ $h == kd && $HOSTNAME == x3 && $HOSTNAME == "$MAIL_HOST" ]]; then + m ssh root@$tg 'btrbk-spread-wrap &>/dev/null