iankelling.org Git - distro-setup/blob - btrbk-run

   1 #!/bin/bash
   2 # Copyright (C) 2016 Ian Kelling
   3
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16
  17 # todo: if we cancel in the middle of a btrfs send, then run again
  18 # immediately, the received subvolume doesn't get a Received UUID:
  19 # field, and we won't mount it. Need to figure out a solution that will
  20 # fix this.
  21
  22
  23 [[ $EUID == 0 ]] || exec sudo -E "${BASH_SOURCE[0]}" "$@"
  24
  25 source /usr/local/lib/err
  26
  27 usage() {
  28   cat <<'EOF'
  29 btrbk-run [OPTIONS] [run|resume|archive]
  30 usually -t TARGET_HOST or -s SOURCE_HOST
  31
  32 Note, at source location, intentionally not executable, run and read
  33 install-my-scripts.
  34
  35 EOF
  36   echo "top of script file:"
  37   sed -n '1,/^[# ]*end command line/{p;b};q' "$0"
  38   exit $1
  39 }
  40
  41
  42
  43 script_name="${BASH_SOURCE[0]}"
  44 script_name="${script_name##*/}"
  45 m() { if $verbose; then printf "%s\n"  "$*"; fi;  "$@"; }
  46 e() { printf "%s\n"  "$*"; }
  47 die() { printf "%s\n" "$*" >&2; echo "exiting with status 1" >&2; exit 1; }
  48 mexit() { echo ": exiting with status $1"; exit $1; }
  49
  50 # latest $MAIL_HOST
  51 if [[ -e /b/bash_unpublished/source-state ]]; then
  52   source /b/bash_unpublished/source-state
  53 fi
  54
  55 # note q is owned by root:1000
  56
  57 mountpoints=()
  58
  59 rsync_mountpoint=/q
  60
  61 ret=0
  62 # default options
  63 conf_only=false
  64 dry_run=false # mostly for testing
  65 rate_limit=no
  66 verbose=true; verbose_arg=-v
  67 force=false
  68 if [[ $INVOCATION_ID ]]; then
  69   # INVOCATION_ID means running as a systemd service. we cant show progress in this case,
  70   # but if we pass the arg, it will insert mbuffer into the command.
  71   progress_arg=
  72 else
  73   progress_arg="--progress"
  74 fi
  75 incremental_strict=false
  76 pull_reexec=false
  77
  78 default_args_file=/etc/btrbk-run.conf
  79 if [[ -s $default_args_file ]]; then
  80   # shellcheck disable=SC2046 # we want word splitting
  81   set -- $(< $default_args_file) "$@"
  82   # i havent used this feature yet, so warn about it
  83   echo "$0: warning: default btrbk-run options set in $default_args_file (sleeping 5 seconds):"
  84   cat $default_args_file
  85   sleep 5
  86 fi
  87
  88 targets=()
  89 early=false
  90 fast=false
  91 kd_spread=false
  92 orig_args=("$@")
  93 temp=$(getopt -l fast,pull-reexec,help 23cefikl:m:npqrs:t:vh "$@") || usage 1
  94 eval set -- "$temp"
  95 while true; do
  96   case $1 in
  97     # for the rare case we want to run multiple instances at the same time
  98     -2) conf_suf=2 ;;
  99     -3) conf_suf=3 ;;
 100     # only creates the config file, does not run btrbk
 101     -c) conf_only=true ;;
 102     # quit early, just btrbk, no extra remounting etc.
 103     -e) early=true ;;
 104     # avoids some default behaviors:
 105     # - no skipping hosts where xprintidle haven't been idle recently
 106     # - exit if we can't ssh to 1 or more hosts
 107     # - still set default hosts despite MAIL_HOST status
 108     -f) force=true ;;
 109     # skip various checks. when we run twice in a row for
 110     # switch mail-host, no need to repeat the same checks again.
 111     --fast) fast=true ;;
 112     -i) incremental_strict=true ;;
 113     # note this implies resume and -p
 114     -k) kd_spread=true ;;
 115     # bytes per second, suffix k m g
 116     -l) rate_limit=$2; shift ;;
 117     # Comma separated mountpoints to backup. This has defaults set below.
 118     -m) IFS=, mountpoints=($2); unset IFS; shift ;;
 119     -n) dry_run=true ;;
 120     # preserve existing snapshots and backups
 121     -p) preserve_arg=-p ;;
 122     # internal option for rerunning under newer SOURCE_HOST version.
 123     --pull-reexec) pull_reexec=true;;
 124     # quiet
 125     -q) verbose=false; verbose_arg=; progress_arg= ;;
 126     # source host to receive a backup from
 127     -s)
 128       source=$2
 129       bbksource=$source
 130       if [[ $source == *:* ]]; then
 131         bbksource="[$source]"
 132       fi
 133       shift
 134       ;;
 135     # target hosts to send to. empty is valid for just doing local
 136     # snapshot. we have default hosts we will populate.
 137     -t) IFS=, targets=($2); unset IFS; shift ;;
 138     # verbose.
 139     -v) verbose=true; verbose_arg=-v ;;
 140     -h|--help) usage ;;
 141     --) shift; break ;;
 142     *) die "Internal error!" ;;
 143   esac
 144   shift
 145 done
 146
 147 cmd_arg="$1"
 148
 149 if $kd_spread; then
 150   if [[ $cmd_arg && $cmd_arg != resume ]]; then
 151     die "dont pass -k without resume or empty run arg"
 152   fi
 153   if [[ $HOSTNAME == "$MAIL_HOST" ]]; then
 154     die "something went wrong, -k not meant to be run on MAIL_HOST"
 155   fi
 156   if [[ $HOSTNAME != kd ]]; then
 157     die "something went wrong, -k only meant to run on kd"
 158   fi
 159   cmd_arg=resume
 160   preserve_arg=-p
 161   h=sy
 162   add-wireless-target-h
 163 fi
 164
 165 if [[ ! $cmd_arg ]]; then
 166   cmd_arg=run
 167 fi
 168
 169
 170 std_preserve="36h 14d 8w 24m"
 171 q_preserve="18h 14d 8w"
 172
 173 case $cmd_arg in
 174   run|resume) : ;;
 175
 176   # This works better than the normal archive command. We have to
 177   # specify the mount points, but that is what we are used to doing and
 178   # we prefer it. Another difference is that archive works recursively
 179   # and we don't care about that. Sometimes we may still want to run
 180   # btrbk archive, but it doesn't even use the config file, so just
 181   # run it directly, eg:
 182   # time s btrbk -v archive /mnt/r7/amy/boot/btrbk ssh://bo/mnt/boot2/btrbk
 183   archive)
 184     cmd_arg=resume
 185     std_preserve="999h 999d 999w 999m"
 186     q_preserve="$std_preserve"
 187     preserve_arg=-p
 188     ;;
 189   *) die "untested command arg" ;;
 190 esac
 191
 192 if (( $# > 1 )); then
 193   die: "only 1 nonoption arg is supported"
 194 fi
 195
 196 if [[ -v targets && $source ]]; then
 197   # note, this doesnt need to be the case, but
 198   # we would need to think about it.
 199   die "error: -t and -s are mutually exclusive"
 200 fi
 201
 202 ### end options parsing
 203
 204 # remove path from earlier version of btrbk
 205 rm -f /usr/sbin/btrbk
 206 # note, this still works as intended if there is no /usr/bin/btrbk
 207 if [[ /a/opt/btrbk/btrbk -nt /usr/bin/btrbk ]]; then
 208   if [[ -e /b/distro-functions/src/package-manager-abstractions ]]; then
 209     . /b/distro-functions/src/package-manager-abstractions
 210     pi asciidoctor
 211   fi
 212   cd /a/opt/btrbk
 213   m make install
 214 fi
 215
 216 # TODO: i wonder if there should be an option to send to the default
 217 # targets, plus any given on the command line.
 218
 219
 220 at_work=false
 221 at_home=false
 222
 223
 224 set-location() {
 225   case $HOSTNAME in
 226     kw)
 227       at_work=true
 228       ;;
 229     kd|frodo)
 230       at_home=true
 231       ;;
 232     x2|x3|sy)
 233       if [[ $(dig +short @10.2.0.1 -x 10.2.0.2 2>&1 ||:) == kd.b8.nz. ]] \
 234            && ip n show 10.2.0.1 | grep . &>/dev/null; then
 235         at_home=true
 236       elif ping -q -c1 -w1 hal.office.fsf.org &>/dev/null \
 237           && ip n show 192.168.0.26 | grep . &>/dev/null; then
 238         at_work=true
 239       fi
 240       ;;
 241   esac
 242 }
 243
 244 exit-if-no-default-targets() {
 245   if ! $force && [[ $HOSTNAME != "$MAIL_HOST" ]]; then
 246     echo "MAIL_HOST=$MAIL_HOST, nothing to do"
 247     mexit 0
 248   fi
 249   case $HOSTNAME in
 250     kw|kd|frodo|x2|x3|sy) : ;;
 251     *)
 252       die "error: no default targets for this host, use -t"
 253       ;;
 254   esac
 255 }
 256
 257 add-x3-target() {
 258   # main work machine
 259   if ping -q -c1 -w1 x3.office.fsf.org &>/dev/null; then
 260     targets+=(x3.office.fsf.org)
 261   elif ping -q -c1 -w1 $h.b8.nz &>/dev/null; then
 262     # in case we took it home
 263     targets+=(x3.b8.nz)
 264   else
 265     targets+=(x3wg.b8.nz)
 266   fi
 267 }
 268
 269 add-wireless-target-h() {
 270   if ping -q -c1 -w1 $h.b8.nz &>/dev/null; then
 271     targets+=($h.b8.nz)
 272   elif ping -q -c1 -w1 ${h}w.b8.nz &>/dev/null; then
 273     targets+=(${h}w.b8.nz)
 274   fi
 275 }
 276
 277
 278 # set default targets
 279 if [[ ! -v targets && ! $source ]]; then
 280   exit-if-no-default-targets
 281   set-location
 282   if $at_home; then
 283     if ! $kd_spread && [[ $HOSTNAME != x3 ]]; then
 284       add-x3-target
 285     fi
 286     if [[ $HOSTNAME != kd ]]; then
 287       targets+=(kd.b8.nz)
 288     fi
 289     wireless_home_hosts=(
 290       x2
 291       sy
 292     )
 293     for h in ${wireless_home_hosts[@]}; do
 294       if [[ $HOSTNAME != "$h" ]]; then
 295         add-wireless-target-h
 296       fi
 297     done
 298   elif $at_work; then
 299     targets+=(i.b8.nz)
 300     for h in x2 x3 kw; do
 301       if [[ $HOSTNAME == "$h" ]]; then
 302         continue
 303       fi
 304       if ping -q -c1 -w1 $h.office.fsf.org &>/dev/null; then
 305         targets+=($h.office.fsf.org)
 306       fi
 307     done
 308   else
 309     targets+=(i.b8.nz)
 310   fi
 311 fi
 312
 313 if [[ ${mountpoints[0]} ]]; then
 314   for mp in ${mountpoints[@]}; do
 315     if [[ -e /nocow/btrfs-stale/$mp ]]; then
 316       die "error: $mp is stale, mount-latest-subvol first"
 317     fi
 318   done
 319 else
 320   # set default mountpoints
 321   if [[ ${targets[0]} == tp ]]; then
 322     prospective_mps=(/a)
 323   else
 324     case $HOSTNAME in
 325       *)
 326         prospective_mps=()
 327         if [[ $source ]]; then
 328           source_state="$(ssh $source cat /a/bin/bash_unpublished/source-state)"
 329           eval "$source_state"
 330           source_host="$(ssh $source cat /etc/hostname)"
 331           if [[ $source_host == "$MAIL_HOST" ]]; then
 332             prospective_mps+=(/o)
 333           fi
 334           if [[ $source_host == "$HOST2" ]]; then
 335             prospective_mps+=(/a /ar /qr /qd /q)
 336           fi
 337         else
 338           if [[ $HOSTNAME == "$MAIL_HOST" ]]; then
 339             prospective_mps+=(/o)
 340           fi
 341           if [[ $HOSTNAME == "$HOST2" ]]; then
 342             prospective_mps+=(/a /ar /qr /qd /q)
 343           fi
 344           if $kd_spread; then
 345             prospective_mps=(/a /ar /o /qr /qd /q)
 346           fi
 347         fi
 348         # note: put q last just in case its specific retention options were to
 349         # affect other config sections. I havent tested if that is the case.
 350         ;;
 351     esac
 352   fi
 353   for mp in ${prospective_mps[@]}; do # default mountpoints to sync
 354     if [[ -e /nocow/btrfs-stale/$mp ]]; then
 355       e "warning: $mp stale, not adding to default mountpoints"
 356       continue
 357     fi
 358     if awk '{print $2}' /etc/fstab | grep -xF $mp &>/dev/null; then
 359       mountpoints+=($mp)
 360     fi
 361   done
 362 fi
 363
 364 if (( ! ${#mountpoints[@]} )); then
 365   die didnt get mountpoint arg and had no defaults
 366 fi
 367
 368 ##### end command line parsing ########
 369
 370 #### begin pre-checks #####
 371
 372 # todo: this has a timing problem, since btrbk.timer could activate the service after this check.
 373 if ! $fast && [[ $source ]]; then
 374   if [[ $(ssh $source ps --no-headers -o comm 1) == systemd ]]; then
 375     status=$(ssh $source systemctl is-active btrbk.service) || : # normally returns 3
 376     case $status in
 377       inactive|failed) : ;;
 378       *)
 379         echo "$0: error: btrbk is running on source. exiting out of caution"
 380         mexit 1
 381     esac
 382   fi
 383 fi
 384
 385 if ! command -v btrbk &>/dev/null; then
 386   die "error: no btrbk binary found"
 387 fi
 388
 389
 390 #### end pre-checks #####
 391
 392 mkdir -p /var/log/btrbk
 393 # The journal doesnt go back to my oldest backups, and I've found myself
 394 # wanting older logs. Not going to bother expiring old logs, since it is
 395 # fine if they go back years.
 396 log_path=/var/log/btrbk/$(date +%F_%T%:::z).log
 397 echo copying output to $log_path
 398 exec &> >(ts "%F %T" | tee -a $log_path)
 399
 400
 401 if $verbose; then
 402   printf " options: conf_only=%s\ndry_run=%s\nrate_limit=%s\nverbose=%s\ncmd_arg=%s" "$conf_only" "$dry_run" "$rate_limit" "$verbose" "$cmd_arg"
 403 fi
 404
 405 if [[ -v targets ]]; then
 406   echo "targets: ${targets[*]}"
 407 fi
 408
 409 if [[ $source ]]; then
 410   echo "source: $source"
 411 fi
 412
 413 echo "mountpoints: ${mountpoints[*]}"
 414
 415
 416 uninstalled-file-die() {
 417   err "uninstalled file $1. run install-my-scripts or rerun with -f"
 418   exit 1
 419 }
 420 if ! $force; then
 421   install_bin_files=(
 422     mount-latest-subvol
 423     check-subvol-stale
 424     btrbk-run
 425   )
 426   for f in ${install_bin_files[@]}; do
 427     if ! diff -q /a/bin/ds/$f /usr/local/bin/$f; then
 428       uninstalled-file-die $f
 429     fi
 430   done
 431   if ! diff -q /a/bin/errhandle/err /usr/local/lib/err; then
 432     uninstalled-file-die err
 433   fi
 434 fi
 435
 436
 437 # pull_reexec stops us from getting into an infinite loop if there is some
 438 # kind of weird problem
 439 pulla=false
 440 for m in "${mountpoints[@]}"; do
 441   if [[ $m == /a ]]; then
 442     pulla=true
 443     break
 444   fi
 445 done
 446 if ! $pull_reexec && [[ $source ]] && $pulla ; then
 447   tmpf=$(mktemp)
 448   m rsync -ra $source:/usr/local/bin/{mount-latest-subvol,check-subvol-stale} /usr/local/bin
 449   m rsync -ra $source:/usr/local/lib/err /usr/local/lib
 450   m scp $source:/a/bin/distro-setup/btrbk-run $tmpf
 451   if ! diff -q $tmpf ${BASH_SOURCE[0]}; then
 452     e "found different version on host $source. reexecing"
 453     install -T $tmpf /usr/local/bin/btrbk-run
 454     m /usr/local/bin/btrbk-run --pull-reexec "${orig_args[@]}"
 455     mexit 0
 456   fi
 457 fi
 458
 459
 460 if ! $fast; then
 461   # if our mountpoints are from stale snapshots,
 462   # it doesn't make sense to do a backup.
 463   m check-subvol-stale ${mountpoints[@]} || die "found stale mountpoints in ${mountpoints[*]}"
 464
 465   # for an initial run, btrbk requires the dir to exist.
 466   mkdir -p /mnt/{root,o}/btrbk
 467 fi
 468 local_zone=$(date +%z)
 469
 470 if [[ $source ]]; then
 471   if $fast; then
 472     zone=$local_zone
 473   else
 474     if ! zone=$(ssh root@$source date +%z); then
 475       if $conf_only; then
 476         echo "$0: warning: failed to ssh to root@$source"
 477       else
 478         die failed to ssh to root@$source
 479       fi
 480     fi
 481     if [[ $zone != "$local_zone" ]]; then
 482       die "error: dont confuse yourself with multiple time zones. $h has different timezone than localhost"
 483     fi
 484   fi
 485 else
 486
 487   sshable=()
 488   sshfail=()
 489   for h in ${targets[@]}; do
 490     if $fast || $conf_only; then
 491       # Use some typical values in this case
 492       root_size=$(( 1024 * 1024 * 2000 )) #2tb
 493       percent_used=10
 494       zone=$(date +%z)
 495     elif remote_str=$(timeout -s 9 6 ssh root@$h "mkdir -p /mnt/root/btrbk /mnt/o/btrbk && date +%z && df --output=size,pcent / | tail -n1"); then
 496       mapfile -t tmp_array <<<"$remote_str"
 497       zone="${tmp_array[0]}"
 498       IFS=" " read -r root_size percent_used <<<"${tmp_array[1]}"
 499       percent_used=${percent_used%%%}
 500
 501       if (( ${#tmp_array[@]} != 2 )); then
 502         die "error: didnt get 2 lines in test ssh to target $h. investigate"
 503       fi
 504       case $percent_used in
 505         [0-9]|[1-9][0-9]) : ;;
 506         *)
 507           die "error: didnt get percent disk use in test ssh to target $h. investigate"
 508           ;;
 509       esac
 510     else
 511       sshfail+=($h)
 512       continue
 513     fi
 514
 515     # we may be booted into a bootstrap fs or something
 516     min_root_kb=$(( 1024 * 1024 * 200 )) # 200 gb
 517     if (( root_size < min_root_kb )); then
 518       continue
 519     fi
 520
 521     if (( percent_used >= 98 )); then
 522       die "error: filesystem on target $h is $percent_used % full"
 523     fi
 524
 525     # on sy, xprintidle is resetting every 12 seconds even when not
 526     # idle, i dunno why, instead we are checking if the screen is locked,
 527     # which is good enough.
 528     #
 529     # This is a separate ssh because the command can fail and thatis ok.
 530     if ! $force; then
 531       locked=false
 532       if lock_info=$(timeout -s 9 6 ssh $h DISPLAY=:0 xscreensaver-command -time); then
 533         if [[ $lock_info != *non-blanked* ]]; then
 534           locked=true
 535         fi
 536       else
 537         locked=true
 538       fi
 539       if ! $locked; then
 540         # Ignore this host. i sometimes use a non-main machine for
 541         # testing or web browsing, knowing that everything will be wiped
 542         # by the next backup, but I dont want it to happen as Im using
 543         # it from cronjob.
 544         e "warning: $h: seems to be actively in use, skipping for now"
 545         continue
 546       fi
 547     fi
 548     sshable+=($h)
 549     if [[ $zone != "$local_zone" ]]; then
 550       die "error: dont confuse yourself with multiple time zones. $h has different timezone than localhost"
 551     fi
 552   done
 553   if [[ ! ${sshable[*]} ]] || { $force && [[ ${sshfail[*]} ]]; }; then
 554     die "failed to ssh to hosts: ${sshfail[*]}"
 555   else
 556     if [[ ${sshfail[*]} ]]; then
 557       ret=1
 558       e "error: failed to ssh to ${sshfail[*]} but continuing with other hosts"
 559     fi
 560     targets=(${sshable[@]})
 561   fi
 562 fi
 563
 564
 565 cat >/etc/btrbk$conf_suf.conf <<EOF
 566 ssh_identity /q/root/h
 567 #ssh_identity /root/.ssh/home
 568
 569 # Just a guess that local7 is a good facility to pick.
 570 # It's a bit odd that the transaction log has to be logged to
 571 # a file or syslog, while other output is sent to std out.
 572 # The man does not mention a way for them to be together, but
 573 # I dunno if setting a log level like warn might also output
 574 # transaction info.
 575 transaction_syslog local7
 576
 577 # trying this out
 578 #stream_compress zstd
 579
 580 # so we only run one at a time
 581 lockfile                   /var/lock/btrbk$conf_suf.lock
 582
 583 # default format of short does not accomidate hourly preservation setting
 584 timestamp_format long-iso
 585
 586 # only make a snapshot if things have changed
 587 snapshot_create onchange
 588 # I could make this different from target_preserve,
 589 # if one disk had less space.
 590 # for now, keeping them equal.
 591 snapshot_preserve $std_preserve
 592 snapshot_preserve_min 6h
 593 snapshot_dir btrbk
 594 # so, total backups = ~58
 595 target_preserve $std_preserve
 596 target_preserve_min 6h
 597
 598 # i tried this when investigating: clone no source subvolume found error
 599 #incremental_prefs sro:1 srn:1 sao san:1 aro:1 arn:1
 600
 601 # if something fails and it's not obvious, try doing
 602 # btrbk -l debug -v dryrun
 603
 604 rate_limit $rate_limit
 605 EOF
 606
 607 if $incremental_strict; then
 608   cat >>/etc/btrbk$conf_suf.conf <<EOF
 609 incremental strict
 610 EOF
 611 fi
 612
 613 qconf() {
 614   case $sub in
 615     q)
 616       # q has sensitive data i dont want to backup for so long
 617       cat >>/etc/btrbk$conf_suf.conf <<EOF
 618 snapshot_preserve $q_preserve
 619 snapshot_preserve_min 2h
 620 snapshot_dir btrbk
 621 target_preserve $q_preserve
 622 target_preserve_min 2h
 623 EOF
 624       ;;
 625   esac
 626
 627 }
 628
 629 # make /q be last
 630 mp_count=${#mountpoints[@]}
 631 for (( i=0; i < mp_count - 1 ; i++ )); do
 632   if [[ ${mountpoints[i]} == /q ]]; then
 633     unset "mountpoints[i]"
 634     mountpoints+=(/q)
 635   fi
 636 done
 637
 638 for m in ${mountpoints[@]}; do
 639   case $m in
 640     /o)
 641       vol=/mnt/o
 642       ;;
 643     *)
 644       vol=/mnt/root
 645       ;;
 646   esac
 647
 648   sub=${m#/}
 649   if [[ $source ]]; then
 650     cat >>/etc/btrbk$conf_suf.conf <<EOF
 651 volume ssh://$bbksource$vol
 652 subvolume $sub
 653 EOF
 654     qconf
 655     cat >>/etc/btrbk$conf_suf.conf <<EOF
 656 target send-receive $vol/btrbk
 657 EOF
 658   fi
 659   if (( ${#targets[@]} )); then
 660     cat >>/etc/btrbk$conf_suf.conf <<EOF
 661 volume $vol
 662 subvolume $sub
 663 EOF
 664     qconf
 665     for tg in ${targets[@]}; do
 666       # handle ipv6
 667       if [[ $tg == *:* ]]; then
 668         tg="[$tg]"
 669       fi
 670       cat >>/etc/btrbk$conf_suf.conf <<EOF
 671 target send-receive ssh://$tg$vol/btrbk
 672 EOF
 673     done
 674   fi
 675 done
 676
 677 # todo: umount first to ensure we don't have any errors
 678 # todo: do some kill fuser stuff to make umount more reliable
 679
 680
 681 if $conf_only; then
 682   mexit 0
 683 fi
 684
 685
 686
 687 if $dry_run; then
 688   m btrbk -c /etc/btrbk$conf_suf.conf -v -n $cmd_arg
 689   mexit 0
 690 fi
 691 # -q and just using the syslog option seemed nice,
 692 # but it doesn't show when a send has a parent and when it doesn't.
 693 m btrbk -c /etc/btrbk$conf_suf.conf $preserve_arg $verbose_arg $progress_arg $cmd_arg
 694
 695 if $early; then
 696   exit 0
 697 fi
 698
 699 # todo: tp not valid anymore.
 700 # if we have it, sync to systems which don't
 701 if mountpoint $rsync_mountpoint >/dev/null; then
 702   for tg in ${targets[@]}; do
 703     case $tg in
 704       tp)
 705         dirs=(/p/c/machine_specific/tp)
 706         for x in /p/c/machine_specific/*.hosts; do
 707           if grep -qxF $tg $x; then
 708             dirs+=(${x%.hosts})
 709           fi
 710         done
 711         m rsync -aSAXPH --specials --devices --delete --relative ${dirs[@]} root@$tg:/
 712         ;;
 713     esac
 714   done
 715 fi
 716
 717 subvols=()
 718 for mp in "${mountpoints[@]}"; do
 719   subvols+=("${mp##*/}")
 720 done
 721 if [[ $source ]]; then
 722   m mount-latest-subvol "${subvols[@]}"
 723 else
 724   for tg in ${targets[@]}; do
 725     m /a/exe/mount-latest-remote "$tg" "${subvols[@]}" || ret=$?
 726   done
 727 fi
 728
 729 if [[ $ret == 0 ]]; then
 730   for tg in ${targets[@]}; do
 731     h=$(ssh $tg hostname)
 732     if [[ $h == kd && $HOSTNAME == x3 && $HOSTNAME == "$MAIL_HOST" ]]; then
 733       ssh root$tg systemctl --no-block start btrbk-spread
 734     fi
 735     rsync -a -f"- */" -f"+ *" /var/log/btrbk/ root@$tg:/var/log/btrbk/$tg
 736     ssh root@$tg /usr/local/bin/mail-backup-clean
 737   done
 738   if [[ $source ]]; then
 739     rsync -a -f"- */" -f"+ *" $source:/var/log/btrbk/ /var/log/btrbk/$source
 740   fi
 741 fi
 742
 743 mexit $ret
 744
 745 # todo: move variable data we don't care about backing up
 746 # to /nocow and symlink it.
 747
 748
 749 # background on btrbk timezones. with short/long, timestamps use local time.
 750 # for long, if your local time moves backwards, by moving timezones or
 751 # for an hour when daylight savings changes it, you will temporarily get
 752 # a more aggressive retention policy for the overlapping period, and
 753 # vice versa for the opposite timezone move. The alternative is using
 754 # long-iso, which puts timezone info into the timestamp, which means
 755 # that instead of shifting time, you shift the start of day/week/month
 756 # which is used for retention to your new local time, which means for
 757 # example, if you moved forward by 8 hours, the daily/weekly/monthly
 758 # retention will be 8 hours more aggressive since midnight is at a new
 759 # time, unless you fake the timzeone using the TZ env variable.
 760 # However, in the short term, there will be no inconsistencies.
 761 # I don't see any problem with shifting when the day starts for
 762 # retention, so I'm using long-iso.
 763
 764 # note to create a long-iso timestamp: date +%Y%m%dT%H%M%S%z