X-Git-Url: https://iankelling.org/git/?a=blobdiff_plain;f=btrbk-run;h=dcca4b7dd036dca6fb662f40e9904a824160e6bd;hb=72c18f3a6a7f1ed0ca16af654a1f804ab96e1ff9;hp=8a310e61e931e495ef0700c64128e56dac1fab9e;hpb=ad09c51104f62d1da1782387025b44327a081872;p=distro-setup diff --git a/btrbk-run b/btrbk-run index 8a310e6..dcca4b7 100644 --- a/btrbk-run +++ b/btrbk-run @@ -26,7 +26,7 @@ source /usr/local/lib/err usage() { cat <<'EOF' -btrbk-run [OPTIONS] +btrbk-run [OPTIONS] [run|resume|archive] usually -t TARGET_HOST or -s SOURCE_HOST Note, at source location, intentionally not executable, run and read @@ -40,7 +40,9 @@ EOF -pre="${0##*/}: " +script_name="${BASH_SOURCE[0]}" +script_name="${script_name##*/}" +pre="${SSH_CLIENT:+$HOSTNAME} $script_name:" m() { if $verbose; then printf "$pre%s\n" "$*"; fi; "$@"; } e() { printf "$pre%s\n" "$*"; } die() { printf "$pre%s\n" "$*" >&2; echo "exiting with status 1" >&2; exit 1; } @@ -63,12 +65,19 @@ conf_only=false dry_run=false # mostly for testing rate_limit=no verbose=true; verbose_arg=-v -progress_arg="--progress" +if [[ $INVOCATION_ID ]]; then + # INVOCATION_ID means running as a systemd service. we cant show progress in this case, + # but if we pass the arg, it will insert mbuffer into the command. + progress_arg= +else + progress_arg="--progress" +fi incremental_strict=false pull_reexec=false default_args_file=/etc/btrbk-run.conf if [[ -s $default_args_file ]]; then + # shellcheck disable=SC2046 # we want word splitting set -- $(< $default_args_file) "$@" # i havent used this feature yet, so warn about it echo "$0: warning: default btrbk-run options set in $default_args_file (sleeping 5 seconds):" @@ -76,11 +85,12 @@ if [[ -s $default_args_file ]]; then sleep 5 fi -marchive=false +targets=() early=false cron=false +fast=false orig_args=("$@") -temp=$(getopt -l cron,pull-reexec,help ceil:m:npqrs:t:vh "$@") || usage 1 +temp=$(getopt -l cron,fast,pull-reexec,help 23ceil:m:npqrs:t:vh "$@") || usage 1 eval set -- "$temp" while true; do case $1 in @@ -91,24 +101,29 @@ while true; do --cron) cron=true pre= - shift ;; + # for the rare case we want to run multiple instances at the same time + -2) conf_suf=2 ;; + -3) conf_suf=3 ;; # only creates the config file, does not run btrbk - -c) conf_only=true; shift ;; + -c) conf_only=true ;; # quit early, just btrbk, no extra remounting etc. - -e) early=true; shift ;; - -i) incremental_strict=true; shift ;; + -e) early=true ;; + # skip various checks. when we run twice in a row for + # switch mail-host, no need to repeat the same checks again. + --fast) fast=true ;; + -i) incremental_strict=true ;; # bytes per second, suffix k m g - -l) rate_limit=$2; shift 2 ;; + -l) rate_limit=$2; shift ;; # Comma separated mountpoints to backup. This has defaults set below. - -m) IFS=, mountpoints=($2); unset IFS; shift 2 ;; - -n) dry_run=true; dry_run_arg=-n; shift ;; - # show progress - -p) progress_arg="--progress"; shift ;; + -m) IFS=, mountpoints=($2); unset IFS; shift ;; + -n) dry_run=true ;; + # hide progress + -p) progress_arg= ;; # internal option for rerunning under newer SOURCE_HOST version. - --pull-reexec) pull_reexec=true; shift ;; + --pull-reexec) pull_reexec=true;; # quiet - -q) verbose=false; verbose_arg=; progress_arg=; shift ;; + -q) verbose=false; verbose_arg=; progress_arg= ;; # source host to receive a backup from -s) source=$2 @@ -116,31 +131,38 @@ while true; do if [[ $source == *:* ]]; then bbksource="[$source]" fi - shift 2 + shift ;; # target hosts to send to. empty is valid for just doing local # snapshot. we have default hosts we will populate. - -t) IFS=, targets=($2); unset IFS; shift 2 ;; + -t) IFS=, targets=($2); unset IFS; shift ;; # verbose. - -v) verbose=true; verbose_arg=-v; shift ;; + -v) verbose=true; verbose_arg=-v ;; -h|--help) usage ;; --) shift; break ;; *) die "Internal error!" ;; esac + shift done -# only tested commands are resume and archive cmd_arg=${1:-run} -std_preserve="18h 14d 8w 24m" -q_preserve="18h 14d" +std_preserve="36h 14d 8w 24m" +q_preserve="18h 14d 8w" case $cmd_arg in - run|resume|archive) : ;; - marchive) - marchive=true - cmd=resume + run|resume) : ;; + + # This works better than the normal archive command. We have to + # specify the mount points, but that is what we are used to doing and + # we prefer it. Another difference is that archive works recursively + # and we don't care about that. Sometimes we may still want to run + # btrbk archive, but it doesn't even use the config file, so just + # run it directly, eg: + # time s btrbk -v archive /mnt/r7/amy/boot/btrbk ssh://bo/mnt/boot2/btrbk + archive) + cmd_arg=resume std_preserve="999h 999d 999w 999m" q_preserve="$std_preserve" preserve_arg=-p @@ -159,12 +181,10 @@ if [[ -v targets && $source ]]; then fi if $verbose; then - printf "options: conf_only=%s\ndry_run=%s\nrate_limit=%s\nverbose=%s\ncmd_arg=%s" "$conf_only" "$dry_run" "$rate_limit" "$verbose" "$cmd_arg" + printf "$pre options: conf_only=%s\ndry_run=%s\nrate_limit=%s\nverbose=%s\ncmd_arg=%s" "$conf_only" "$dry_run" "$rate_limit" "$verbose" "$cmd_arg" fi ### end options parsing -declare -A vols - # remove path from earlier version of btrbk rm -f /usr/sbin/btrbk # note, this still works as intended if there is no /usr/bin/btrbk @@ -181,66 +201,108 @@ fi # targets, plus any given on the command line. -amy=false + +kd_spread=false # set default targets if [[ ! -v targets && ! $source ]]; then - if [[ $HOSTNAME != "$MAIL_HOST" ]] && $cron ; then - echo "MAIL_HOST=$MAIL_HOST, nothing to do" - mexit 0 - else - amy=true + if $cron; then + if [[ $HOSTNAME != "$MAIL_HOST" ]]; then + if [[ $HOSTNAME == kd && $MAIL_HOST == x3 ]]; then + if ping -q -c1 -w1 x3.office.fsf.org &>/dev/null; then + work_host=x3.office.fsf.org + elif ping -q -c1 -w1 x3wg.b8.nz &>/dev/null; then + work_host=x3wg.b8.nz + fi + if [[ $work_host ]]; then + source_state="$(ssh $work_host cat /a/bin/bash_unpublished/source-state)" + eval "$source_state" + if [[ $MAIL_HOST == x3 ]]; then + kd_spread=true + else + echo "MAIL_HOST=$MAIL_HOST, nothing to do" + mexit 0 + fi + else + echo "MAIL_HOST=$MAIL_HOST, nothing to do" + mexit 0 + fi + else + echo "MAIL_HOST=$MAIL_HOST, nothing to do" + mexit 0 + fi + fi fi at_work=false + at_home=false + + case $HOSTNAME in + kw|kd|frodo|x2|x3|sy) : ;; + *) + die "error: no default targets for this host, use -t" + ;; + esac - targets=(frodo.b8.nz) case $HOSTNAME in - x2|kw) + kw) at_work=true ;;& - x2|x3|sy|bo) - if ping -q -c1 -w1 hal.office.fsf.org \ + kd|frodo) + at_home=true + ;;& + x2|x3|sy) + if [[ $(dig +short @10.2.0.1 -x 10.2.0.2 2>&1 ||:) == kd.b8.nz. ]] \ + && ip n show 10.2.0.1 | grep . &>/dev/null; then + at_home=true + elif ping -q -c1 -w1 hal.office.fsf.org &>/dev/null \ && ip n show 192.168.0.26 | grep . &>/dev/null; then at_work=true fi ;;& - kw|x2|x3|sy|bo) - if $at_work; then + *) + if $at_home; then + if ! $kd_spread; then + # main work machine + if ping -q -c1 -w1 x3.office.fsf.org &>/dev/null; then + targets+=(x3.office.fsf.org) + else + targets+=(x3wg.b8.nz) + fi + fi + for h in frodo kd; do + if [[ $HOSTNAME == "$h" ]]; then + continue + fi + targets+=($h.b8.nz) + done + for h in x2 x3 sy; do + if [[ $HOSTNAME == "$h" ]]; then + continue + fi + if ping -q -c1 -w1 $h.b8.nz &>/dev/null; then + targets+=($h.b8.nz) + elif ping -q -c1 -w1 ${h}w.b8.nz &>/dev/null; then + targets+=(${h}w.b8.nz) + fi + done + elif $at_work; then if ping -q -c1 -w1 iank.vpn.office.fsf.org &>/dev/null; then - home=iank.vpn.office.fsf.org + targets+=(iank.vpn.office.fsf.org) else - home=i.b8.nz + targets+=(i.b8.nz) fi + for h in x2 x3 kw; do + if [[ $HOSTNAME == "$h" ]]; then + continue + fi + if ping -q -c1 -w1 $h.office.fsf.org &>/dev/null; then + targets+=($h.office.fsf.org) + fi + done else - home=b8.nz - fi - ;;& - kw) - targets+=($home x2.office.fsf.org) - ;; - x2|x3|sy|bo) - targets+=($home) - if $at_work; then - targets+=(x2.office.fsf.org x2.b8.nz) - else - targets+=(x2wg.b8.nz) - fi - ;; - kd) - targets+=(x2wg.b8.nz x3.b8.nz) - if ping -q -c1 -w1 sy.b8.nz &>/dev/null; then - targets+=(sy.b8.nz) - else - targets+=(syw.b8.nz) + targets+=(i.b8.nz) fi ;; - frodo) - # no targets - targets=() - ;; - *) - die "error: no default targets for this host, use -t" - ;; esac fi @@ -264,9 +326,6 @@ else prospective_mps=(/a) else case $HOSTNAME in - frodo) - prospective_mps=(/i) - ;; *) prospective_mps=() if [[ $source ]]; then @@ -276,19 +335,19 @@ else if [[ $source_host == "$MAIL_HOST" ]]; then prospective_mps+=(/o) fi + if [[ $source_host == "$HOST2" ]]; then + prospective_mps+=(/a /ar /qr /q) + fi else if [[ $HOSTNAME == "$MAIL_HOST" ]]; then - # HOST2 is really the mail host if it exists - if [[ $HOST2 && $HOST2 != "$HOSTNAME" ]]; then - echo "skipping /o because HOST2 is not us" - else - prospective_mps+=(/o) - fi + prospective_mps+=(/o) + fi + if [[ $HOSTNAME == "$HOST2" ]]; then + prospective_mps+=(/a /ar /qr /q) fi fi # note: put q last just in case its specific retention options were to # affect other config sections. I havent tested if that is the case. - prospective_mps+=(/a /ar /qr /q) ;; esac fi @@ -307,7 +366,7 @@ echo "mountpoints: ${mountpoints[*]}" ##### end command line parsing ######## -if [[ $source ]]; then +if ! $fast && [[ $source ]]; then if [[ $(ssh $source ps --no-headers -o comm 1) == systemd ]]; then status=$(ssh $source systemctl is-active btrbk.service) || : # normally returns 3 case $status in @@ -345,29 +404,46 @@ fi if ! command -v btrbk &>/dev/null; then die "error: no btrbk binary found" fi -# if our mountpoints are from stale snapshots, -# it doesn't make sense to do a backup. -check-subvol-stale ${mountpoints[@]} || die "found stale mountpoints in ${mountpoints[*]}" -# for an initial run, btrbk requires the dir to exist. -mkdir -p /mnt/{root,o}/btrbk +if ! $fast; then + # if our mountpoints are from stale snapshots, + # it doesn't make sense to do a backup. + m check-subvol-stale ${mountpoints[@]} || die "found stale mountpoints in ${mountpoints[*]}" + + # for an initial run, btrbk requires the dir to exist. + mkdir -p /mnt/{root,o}/btrbk +fi local_zone=$(date +%z) if [[ $source ]]; then - if ! zone=$(ssh root@$source date +%z); then - die failed to ssh to root@$source - fi - if [[ $zone != "$local_zone" ]]; then - die "error: dont confuse yourself with multiple time zones. $h has different timezone than localhost" + if $fast; then + zone=$local_zone + else + if ! zone=$(ssh root@$source date +%z); then + if $conf_only; then + echo "$0: warning: failed to ssh to root@$source" + else + die failed to ssh to root@$source + fi + fi + if [[ $zone != "$local_zone" ]]; then + die "error: dont confuse yourself with multiple time zones. $h has different timezone than localhost" + fi fi - else sshable=() sshfail=() min_idle_ms=$((1000 * 60 * 15)) for h in ${targets[@]}; do - if remote_info=( $(timeout -s 9 6 ssh root@$h "mkdir -p /mnt/root/btrbk /mnt/o/btrbk && date +%z && df --output=size,pcent / | tail -n1") ); then + if $fast || $conf_only; then + # Use some typical values in this case + root_size=$(( 1024 * 1024 * 2000 )) #2tb + percent_used=10 + zone=$(date +%z) + elif tmpstr=$(timeout -s 9 6 ssh root@$h "mkdir -p /mnt/root/btrbk /mnt/o/btrbk && date +%z && df --output=size,pcent / | tail -n1"); then + IFS=" " read -r -a remote_info <<<"$tmpstr" + zone=${remote_info[0]} root_size=${remote_info[1]} percent_used=${remote_info[2]%%%} @@ -375,36 +451,36 @@ else if (( ${#remote_info[@]} != 3 )); then die "error: didnt get 3 fields in test ssh to target $h. investigate" fi + else + sshfail+=($h) + continue + fi + # we may be booted into a bootstrap fs or something + min_root_kb=$(( 1024 * 1024 * 200 )) # 200 gb + if (( root_size < min_root_kb )); then + continue + fi - # we may be booted into a bootstrap fs or something - min_root_kb=$(( 1024 * 1024 * 200 )) # 200 gb - if (( root_size < min_root_kb )); then - continue - fi - - if (( percent_used >= 98 )); then - die "error: filesystem on target $h is $percent_used % full" - fi + if (( percent_used >= 98 )); then + die "error: filesystem on target $h is $percent_used % full" + fi - # This is a separate ssh because xprintidle can fail and thats ok. - if $cron && idle_ms=$(timeout -s 9 6 ssh $h DISPLAY=:0 xprintidle); then - if (( idle_ms < min_idle_ms )); then + # This is a separate ssh because xprintidle can fail and thats ok. + if $cron && idle_ms=$(timeout -s 9 6 ssh $h DISPLAY=:0 xprintidle); then + if (( idle_ms < min_idle_ms )); then - # Ignore this host. i sometimes use a non-main machine for - # testing or web browsing, knowing that everything will be wiped - # by the next backup, but I dont want it to happen as Im using - # it from cronjob. - e "warning: $h: active X session in the last 15 minutes, skipping for now" - continue - fi - fi - sshable+=($h) - if [[ $zone != "$local_zone" ]]; then - die "error: dont confuse yourself with multiple time zones. $h has different timezone than localhost" + # Ignore this host. i sometimes use a non-main machine for + # testing or web browsing, knowing that everything will be wiped + # by the next backup, but I dont want it to happen as Im using + # it from cronjob. + e "warning: $h: active X session in the last 15 minutes, skipping for now" + continue fi - else - sshfail+=($h) + fi + sshable+=($h) + if [[ $zone != "$local_zone" ]]; then + die "error: dont confuse yourself with multiple time zones. $h has different timezone than localhost" fi done if [[ ! ${sshable[*]} ]] || { ! $cron && [[ ${sshfail[*]} ]]; }; then @@ -419,7 +495,7 @@ else fi -cat >/etc/btrbk.conf </etc/btrbk$conf_suf.conf <>/etc/btrbk.conf <>/etc/btrbk$conf_suf.conf <>/etc/btrbk$conf_suf.conf <>/etc/btrbk.conf <>/etc/btrbk$conf_suf.conf <>/etc/btrbk$conf_suf.conf <>/etc/btrbk.conf <>/etc/btrbk$conf_suf.conf <>/etc/btrbk.conf <>/etc/btrbk.conf <>/etc/btrbk$conf_suf.conf <>/etc/btrbk.conf <<'EOF' -# volume ssh://amy/mnt/root -# subvolume root_ubuntubionic -# target send-receive /mnt/root/btrbk -# EOF -# fi - # todo: umount first to ensure we don't have any errors # todo: do some kill fuser stuff to make umount more reliable @@ -534,25 +618,16 @@ fi if $dry_run; then - m btrbk -v -n $cmd_arg - mexit 0 -elif [[ $cmd_arg == archive ]]; then - if [[ $source ]]; then - for vol in ${!vols[@]}; do - m btrbk $verbose_arg $progress_arg $cmd_arg ssh://$source$vol $vol - done - else - for tg in ${targets[@]}; do - for vol in ${!vols[@]}; do - m btrbk $verbose_arg $progress_arg $cmd_arg $vol ssh://$tg$vol - done - done - fi + m btrbk -c /etc/btrbk$conf_suf.conf -v -n $cmd_arg mexit 0 fi # -q and just using the syslog option seemed nice, # but it doesn't show when a send has a parent and when it doesn't. -m btrbk $preserve_arg $verbose_arg $progress_arg $cmd_arg +m btrbk -c /etc/btrbk$conf_suf.conf $preserve_arg $verbose_arg $progress_arg $cmd_arg + +if $early; then + exit 0 +fi # todo: tp not valid anymore. # if we have it, sync to systems which don't @@ -572,8 +647,12 @@ if mountpoint $rsync_mountpoint >/dev/null; then done fi +subvols=() +for mp in "${mountpoints[@]}"; do + subvols+=("${mp##*/}") +done if [[ $source ]]; then - m mount-latest-subvol + m mount-latest-subvol "${subvols[@]}" else m /a/exe/mount-latest-remote ${targets[@]} fi