iankelling.org Git - distro-setup/blob - btrbk-run

   1 #!/bin/bash
   2 # Copyright (C) 2016 Ian Kelling
   3
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16
  17 # todo: if we cancel in the middle of a btrfs send, then run again
  18 # immediately, the received subvolume doesn't get a Received UUID:
  19 # field, and we won't mount it. Need to figure out a solution that will
  20 # fix this.
  21
  22
  23 [[ $EUID == 0 ]] || exec sudo -E "${BASH_SOURCE[0]}" "$@"
  24
  25 source /usr/local/lib/err
  26
  27 usage() {
  28   cat <<'EOF'
  29 btrbk-run [OPTIONS] [run|resume|archive]
  30 usually -t TARGET_HOST or -s SOURCE_HOST
  31
  32 Note, at source location, intentionally not executable, run and read
  33 install-my-scripts.
  34
  35 EOF
  36   echo "top of script file:"
  37   sed -n '1,/^[# ]*end command line/{p;b};q' "$0"
  38   exit $1
  39 }
  40
  41
  42
  43 script_name="${BASH_SOURCE[0]}"
  44 script_name="${script_name##*/}"
  45 m() { if $verbose; then printf "%s\n"  "$*"; fi;  "$@"; }
  46 e() { printf "%s\n"  "$*"; }
  47 die() { printf "%s\n" "$*" >&2; echo "exiting with status 1" >&2; exit 1; }
  48 mexit() { echo ": exiting with status $1"; exit $1; }
  49
  50 # latest $MAIL_HOST
  51 if [[ -e /b/bash_unpublished/source-state ]]; then
  52   source /b/bash_unpublished/source-state
  53 fi
  54
  55 # note q is owned by root:1000
  56
  57 mountpoints=()
  58
  59 rsync_mountpoint=/q
  60
  61 ret=0
  62 # default options
  63 conf_only=false
  64 dry_run=false # mostly for testing
  65 rate_limit=no
  66 verbose=true; verbose_arg=-v
  67 force=false
  68 if [[ $INVOCATION_ID ]]; then
  69   # INVOCATION_ID means running as a systemd service. we cant show progress in this case,
  70   # but if we pass the arg, it will insert mbuffer into the command.
  71   progress_arg=
  72 else
  73   progress_arg="--progress"
  74 fi
  75 incremental_strict=false
  76 pull_reexec=false
  77
  78 default_args_file=/etc/btrbk-run.conf
  79 if [[ -s $default_args_file ]]; then
  80   # shellcheck disable=SC2046 # we want word splitting
  81   set -- $(< $default_args_file) "$@"
  82   # i havent used this feature yet, so warn about it
  83   echo "$0: warning: default btrbk-run options set in $default_args_file (sleeping 5 seconds):"
  84   cat $default_args_file
  85   sleep 5
  86 fi
  87
  88 targets=()
  89 early=false
  90 fast=false
  91 kd_spread=false
  92 orig_args=("$@")
  93 temp=$(getopt -l fast,pull-reexec,help 23cefikl:m:npqrs:t:vh "$@") || usage 1
  94 eval set -- "$temp"
  95 while true; do
  96   case $1 in
  97     # for the rare case we want to run multiple instances at the same time
  98     -2) conf_suf=2 ;;
  99     -3) conf_suf=3 ;;
 100     # only creates the config file, does not run btrbk
 101     -c) conf_only=true ;;
 102     # quit early, just btrbk, no extra remounting etc.
 103     -e) early=true ;;
 104     # avoids some default behaviors:
 105     # - no skipping hosts where xprintidle haven't been idle recently
 106     # - exit if we can't ssh to 1 or more hosts
 107     # - still set default hosts despite MAIL_HOST status
 108     -f) force=true ;;
 109     # skip various checks. when we run twice in a row for
 110     # switch mail-host, no need to repeat the same checks again.
 111     --fast) fast=true ;;
 112     -i) incremental_strict=true ;;
 113     # note this implies resume and -p
 114     -k) kd_spread=true ;;
 115     # bytes per second, suffix k m g
 116     -l) rate_limit=$2; shift ;;
 117     # Comma separated mountpoints to backup. This has defaults set below.
 118     -m) IFS=, mountpoints=($2); unset IFS; shift ;;
 119     -n) dry_run=true ;;
 120     # preserve existing snapshots and backups
 121     -p) preserve_arg=-p ;;
 122     # internal option for rerunning under newer SOURCE_HOST version.
 123     --pull-reexec) pull_reexec=true;;
 124     # quiet
 125     -q) verbose=false; verbose_arg=; progress_arg= ;;
 126     # source host to receive a backup from
 127     -s)
 128       source=$2
 129       bbksource=$source
 130       if [[ $source == *:* ]]; then
 131         bbksource="[$source]"
 132       fi
 133       shift
 134       ;;
 135     # target hosts to send to. empty is valid for just doing local
 136     # snapshot. we have default hosts we will populate.
 137     -t) IFS=, targets=($2); unset IFS; shift ;;
 138     # verbose.
 139     -v) verbose=true; verbose_arg=-v ;;
 140     -h|--help) usage ;;
 141     --) shift; break ;;
 142     *) die "Internal error!" ;;
 143   esac
 144   shift
 145 done
 146
 147 cmd_arg="$1"
 148
 149 if $kd_spread; then
 150   if [[ $cmd_arg && $cmd_arg != resume ]]; then
 151     die "dont pass -k without resume or empty run arg"
 152   fi
 153   if [[ $HOSTNAME == "$MAIL_HOST" ]]; then
 154     die "something went wrong, -k not meant to be run on MAIL_HOST"
 155   fi
 156   if [[ $HOSTNAME != kd ]]; then
 157     die "something went wrong, -k only meant to run on kd"
 158   fi
 159   cmd_arg=resume
 160   preserve_arg=-p
 161   h=sy
 162   add-wireless-target-h
 163 fi
 164
 165 if [[ ! $cmd_arg ]]; then
 166   cmd_arg=run
 167 fi
 168
 169
 170 std_preserve="36h 14d 8w 24m"
 171 q_preserve="18h 14d 8w"
 172
 173 case $cmd_arg in
 174   run|resume) : ;;
 175
 176   # This works better than the normal archive command. We have to
 177   # specify the mount points, but that is what we are used to doing and
 178   # we prefer it. Another difference is that archive works recursively
 179   # and we don't care about that. Sometimes we may still want to run
 180   # btrbk archive, but it doesn't even use the config file, so just
 181   # run it directly, eg:
 182   # time s btrbk -v archive /mnt/r7/amy/boot/btrbk ssh://bo/mnt/boot2/btrbk
 183   archive)
 184     cmd_arg=resume
 185     std_preserve="999h 999d 999w 999m"
 186     q_preserve="$std_preserve"
 187     preserve_arg=-p
 188     ;;
 189   *) die "untested command arg" ;;
 190 esac
 191
 192 if (( $# > 1 )); then
 193   die: "only 1 nonoption arg is supported"
 194 fi
 195
 196 if [[ -v targets && $source ]]; then
 197   # note, this doesnt need to be the case, but
 198   # we would need to think about it.
 199   die "error: -t and -s are mutually exclusive"
 200 fi
 201
 202 ### end options parsing
 203
 204 # remove path from earlier version of btrbk
 205 rm -f /usr/sbin/btrbk
 206 # note, this still works as intended if there is no /usr/bin/btrbk
 207 if [[ /a/opt/btrbk/btrbk -nt /usr/bin/btrbk ]]; then
 208   if [[ -e /b/distro-functions/src/package-manager-abstractions ]]; then
 209     . /b/distro-functions/src/package-manager-abstractions
 210     pi asciidoctor
 211   fi
 212   cd /a/opt/btrbk
 213   m make install
 214 fi
 215
 216 # TODO: i wonder if there should be an option to send to the default
 217 # targets, plus any given on the command line.
 218
 219
 220 at_work=false
 221 at_home=false
 222
 223
 224 set-location() {
 225   case $HOSTNAME in
 226     kw)
 227       at_work=true
 228       ;;
 229     kd|frodo)
 230       at_home=true
 231       ;;
 232     x2|x3|sy)
 233       if [[ $(dig +short @10.2.0.1 -x 10.2.0.2 2>&1 ||:) == kd.b8.nz. ]] \
 234            && ip n show 10.2.0.1 | grep . &>/dev/null; then
 235         at_home=true
 236       elif ping -q -c1 -w1 hal.office.fsf.org &>/dev/null \
 237           && ip n show 192.168.0.26 | grep . &>/dev/null; then
 238         at_work=true
 239       fi
 240       ;;
 241   esac
 242 }
 243
 244 exit-if-no-default-targets() {
 245   if ! $force && [[ $HOSTNAME != "$MAIL_HOST" ]]; then
 246     echo "MAIL_HOST=$MAIL_HOST, nothing to do"
 247     mexit 0
 248   fi
 249   case $HOSTNAME in
 250     kw|kd|frodo|x2|x3|sy) : ;;
 251     *)
 252       die "error: no default targets for this host, use -t"
 253       ;;
 254   esac
 255 }
 256
 257 add-x3-target() {
 258   # main work machine
 259   if ping -q -c1 -w1 x3.office.fsf.org &>/dev/null; then
 260     targets+=(x3.office.fsf.org)
 261   elif ping -q -c1 -w1 $h.b8.nz &>/dev/null; then
 262     # in case we took it home
 263     targets+=(x3.b8.nz)
 264   else
 265     targets+=(x3wg.b8.nz)
 266   fi
 267 }
 268
 269 add-wireless-target-h() {
 270   if ping -q -c1 -w1 $h.b8.nz &>/dev/null; then
 271     targets+=($h.b8.nz)
 272   elif ping -q -c1 -w1 ${h}w.b8.nz &>/dev/null; then
 273     targets+=(${h}w.b8.nz)
 274   fi
 275 }
 276
 277
 278 # set default targets
 279 if [[ ! -v targets && ! $source ]]; then
 280   exit-if-no-default-targets
 281   set-location
 282   if $at_home; then
 283     if ! $kd_spread && [[ $HOSTNAME != x3 ]]; then
 284       add-x3-target
 285     fi
 286     if [[ $HOSTNAME != kd ]]; then
 287       targets+=(kd.b8.nz)
 288     fi
 289     wireless_home_hosts=(
 290       x2
 291       sy
 292     )
 293     for h in ${wireless_home_hosts[@]}; do
 294       if [[ $HOSTNAME != "$h" ]]; then
 295         add-wireless-target-h
 296       fi
 297     done
 298   elif $at_work; then
 299     targets+=(i.b8.nz)
 300     for h in x2 x3 kw; do
 301       if [[ $HOSTNAME == "$h" ]]; then
 302         continue
 303       fi
 304       if ping -q -c1 -w1 $h.office.fsf.org &>/dev/null; then
 305         targets+=($h.office.fsf.org)
 306       fi
 307     done
 308   else
 309     targets+=(i.b8.nz)
 310   fi
 311 fi
 312
 313 if [[ ${mountpoints[0]} ]]; then
 314   for mp in ${mountpoints[@]}; do
 315     if [[ -e /nocow/btrfs-stale/$mp ]]; then
 316       die "error: $mp is stale, mount-latest-subvol first"
 317     fi
 318   done
 319 else
 320   # set default mountpoints
 321   if [[ ${targets[0]} == tp ]]; then
 322     prospective_mps=(/a)
 323   else
 324     case $HOSTNAME in
 325       *)
 326         prospective_mps=()
 327         if [[ $source ]]; then
 328           source_state="$(ssh $source cat /a/bin/bash_unpublished/source-state)"
 329           eval "$source_state"
 330           source_host="$(ssh $source cat /etc/hostname)"
 331           if [[ $source_host == "$MAIL_HOST" ]]; then
 332             prospective_mps+=(/o)
 333           fi
 334           if [[ $source_host == "$HOST2" ]]; then
 335             prospective_mps+=(/a /ar /qr /qd /q)
 336           fi
 337         else
 338           if [[ $HOSTNAME == "$MAIL_HOST" ]]; then
 339             prospective_mps+=(/o)
 340           fi
 341           if [[ $HOSTNAME == "$HOST2" ]]; then
 342             prospective_mps+=(/a /ar /qr /qd /q)
 343           fi
 344           if $kd_spread; then
 345             prospective_mps=(/a /ar /o /qr /qd /q)
 346           fi
 347         fi
 348         # note: put q last just in case its specific retention options were to
 349         # affect other config sections. I havent tested if that is the case.
 350         ;;
 351     esac
 352   fi
 353   for mp in ${prospective_mps[@]}; do # default mountpoints to sync
 354     if [[ -e /nocow/btrfs-stale/$mp ]]; then
 355       e "warning: $mp stale, not adding to default mountpoints"
 356       continue
 357     fi
 358     if awk '{print $2}' /etc/fstab | grep -xF $mp &>/dev/null; then
 359       mountpoints+=($mp)
 360     fi
 361   done
 362 fi
 363
 364 if (( ! ${#mountpoints[@]} )); then
 365   die didnt get mountpoint arg and had no defaults
 366 fi
 367
 368 ##### end command line parsing ########
 369
 370 #### begin pre-checks #####
 371
 372 # todo: this has a timing problem, since btrbk.timer could activate the service after this check.
 373 if ! $fast && [[ $source ]]; then
 374   if [[ $(ssh $source ps --no-headers -o comm 1) == systemd ]]; then
 375     status=$(ssh $source systemctl is-active btrbk.service) || : # normally returns 3
 376     case $status in
 377       inactive|failed) : ;;
 378       *)
 379         echo "$0: error: btrbk is running on source. exiting out of caution"
 380         mexit 1
 381     esac
 382   fi
 383 fi
 384
 385 if ! command -v btrbk &>/dev/null; then
 386   die "error: no btrbk binary found"
 387 fi
 388
 389
 390 #### end pre-checks #####
 391
 392 mkdir -p /var/log/btrbk
 393 # The journal doesnt go back to my oldest backups, and I've found myself
 394 # wanting older logs. Not going to bother expiring old logs, since it is
 395 # fine if they go back years.
 396 log_path=/var/log/btrbk/$(date +%F_%T%:::z).log
 397 echo copying output to $log_path
 398 exec &> >(ts "%F %T" | tee -a $log_path)
 399
 400
 401 if $verbose; then
 402   printf " options: conf_only=%s\ndry_run=%s\nrate_limit=%s\nverbose=%s\ncmd_arg=%s" "$conf_only" "$dry_run" "$rate_limit" "$verbose" "$cmd_arg"
 403 fi
 404
 405 if [[ -v targets ]]; then
 406   echo "targets: ${targets[*]}"
 407 fi
 408
 409 if [[ $source ]]; then
 410   echo "source: $source"
 411 fi
 412
 413 echo "mountpoints: ${mountpoints[*]}"
 414
 415
 416
 417 # pull_reexec stops us from getting into an infinite loop if there is some
 418 # kind of weird problem
 419 pulla=false
 420 for m in "${mountpoints[@]}"; do
 421   if [[ $m == /a ]]; then
 422     pulla=true
 423     break
 424   fi
 425 done
 426 if ! $pull_reexec && [[ $source ]] && $pulla ; then
 427   tmpf=$(mktemp)
 428   m rsync -ra $source:/usr/local/bin/{mount-latest-subvol,check-subvol-stale} /usr/local/bin
 429   m rsync -ra $source:/usr/local/lib/err /usr/local/lib
 430   m scp $source:/a/bin/distro-setup/btrbk-run $tmpf
 431   if ! diff -q $tmpf ${BASH_SOURCE[0]}; then
 432     e "found different version on host $source. reexecing"
 433     install -T $tmpf /usr/local/bin/btrbk-run
 434     m /usr/local/bin/btrbk-run --pull-reexec "${orig_args[@]}"
 435     mexit 0
 436   fi
 437 fi
 438
 439
 440 if ! $fast; then
 441   # if our mountpoints are from stale snapshots,
 442   # it doesn't make sense to do a backup.
 443   m check-subvol-stale ${mountpoints[@]} || die "found stale mountpoints in ${mountpoints[*]}"
 444
 445   # for an initial run, btrbk requires the dir to exist.
 446   mkdir -p /mnt/{root,o}/btrbk
 447 fi
 448 local_zone=$(date +%z)
 449
 450 if [[ $source ]]; then
 451   if $fast; then
 452     zone=$local_zone
 453   else
 454     if ! zone=$(ssh root@$source date +%z); then
 455       if $conf_only; then
 456         echo "$0: warning: failed to ssh to root@$source"
 457       else
 458         die failed to ssh to root@$source
 459       fi
 460     fi
 461     if [[ $zone != "$local_zone" ]]; then
 462       die "error: dont confuse yourself with multiple time zones. $h has different timezone than localhost"
 463     fi
 464   fi
 465 else
 466
 467   sshable=()
 468   sshfail=()
 469   for h in ${targets[@]}; do
 470     if $fast || $conf_only; then
 471       # Use some typical values in this case
 472       root_size=$(( 1024 * 1024 * 2000 )) #2tb
 473       percent_used=10
 474       zone=$(date +%z)
 475     elif remote_str=$(timeout -s 9 6 ssh root@$h "mkdir -p /mnt/root/btrbk /mnt/o/btrbk && date +%z && df --output=size,pcent / | tail -n1"); then
 476       mapfile -t tmp_array <<<"$remote_str"
 477       zone="${tmp_array[0]}"
 478       IFS=" " read -r root_size percent_used <<<"${tmp_array[1]}"
 479       percent_used=${percent_used%%%}
 480
 481       if (( ${#tmp_array[@]} != 2 )); then
 482         die "error: didnt get 2 lines in test ssh to target $h. investigate"
 483       fi
 484       case $percent_used in
 485         [0-9]|[1-9][0-9]) : ;;
 486         *)
 487           die "error: didnt get percent disk use in test ssh to target $h. investigate"
 488           ;;
 489       esac
 490     else
 491       sshfail+=($h)
 492       continue
 493     fi
 494
 495     # we may be booted into a bootstrap fs or something
 496     min_root_kb=$(( 1024 * 1024 * 200 )) # 200 gb
 497     if (( root_size < min_root_kb )); then
 498       continue
 499     fi
 500
 501     if (( percent_used >= 98 )); then
 502       die "error: filesystem on target $h is $percent_used % full"
 503     fi
 504
 505     # on sy, xprintidle is resetting every 12 seconds even when not
 506     # idle, i dunno why, instead we are checking if the screen is locked,
 507     # which is good enough.
 508     #
 509     # This is a separate ssh because the command can fail and thatis ok.
 510     if ! $force; then
 511       locked=false
 512       if lock_info=$(timeout -s 9 6 ssh $h DISPLAY=:0 xscreensaver-command -time); then
 513         if [[ $lock_info != *non-blanked* ]]; then
 514           locked=true
 515         fi
 516       else
 517         locked=true
 518       fi
 519       if ! $locked; then
 520         # Ignore this host. i sometimes use a non-main machine for
 521         # testing or web browsing, knowing that everything will be wiped
 522         # by the next backup, but I dont want it to happen as Im using
 523         # it from cronjob.
 524         e "warning: $h: seems to be actively in use, skipping for now"
 525         continue
 526       fi
 527     fi
 528     sshable+=($h)
 529     if [[ $zone != "$local_zone" ]]; then
 530       die "error: dont confuse yourself with multiple time zones. $h has different timezone than localhost"
 531     fi
 532   done
 533   if [[ ! ${sshable[*]} ]] || { $force && [[ ${sshfail[*]} ]]; }; then
 534     die "failed to ssh to hosts: ${sshfail[*]}"
 535   else
 536     if [[ ${sshfail[*]} ]]; then
 537       ret=1
 538       e "error: failed to ssh to ${sshfail[*]} but continuing with other hosts"
 539     fi
 540     targets=(${sshable[@]})
 541   fi
 542 fi
 543
 544
 545 cat >/etc/btrbk$conf_suf.conf <<EOF
 546 ssh_identity /q/root/h
 547 #ssh_identity /root/.ssh/home
 548
 549 # Just a guess that local7 is a good facility to pick.
 550 # It's a bit odd that the transaction log has to be logged to
 551 # a file or syslog, while other output is sent to std out.
 552 # The man does not mention a way for them to be together, but
 553 # I dunno if setting a log level like warn might also output
 554 # transaction info.
 555 transaction_syslog local7
 556
 557 # trying this out
 558 #stream_compress zstd
 559
 560 # so we only run one at a time
 561 lockfile                   /var/lock/btrbk$conf_suf.lock
 562
 563 # default format of short does not accomidate hourly preservation setting
 564 timestamp_format long-iso
 565
 566 # only make a snapshot if things have changed
 567 snapshot_create onchange
 568 # I could make this different from target_preserve,
 569 # if one disk had less space.
 570 # for now, keeping them equal.
 571 snapshot_preserve $std_preserve
 572 snapshot_preserve_min 6h
 573 snapshot_dir btrbk
 574 # so, total backups = ~58
 575 target_preserve $std_preserve
 576 target_preserve_min 6h
 577
 578 # i tried this when investigating: clone no source subvolume found error
 579 #incremental_prefs sro:1 srn:1 sao san:1 aro:1 arn:1
 580
 581 # if something fails and it's not obvious, try doing
 582 # btrbk -l debug -v dryrun
 583
 584 rate_limit $rate_limit
 585 EOF
 586
 587 if $incremental_strict; then
 588   cat >>/etc/btrbk$conf_suf.conf <<EOF
 589 incremental strict
 590 EOF
 591 fi
 592
 593 qconf() {
 594   case $sub in
 595     q)
 596       # q has sensitive data i dont want to backup for so long
 597       cat >>/etc/btrbk$conf_suf.conf <<EOF
 598 snapshot_preserve $q_preserve
 599 snapshot_preserve_min 2h
 600 snapshot_dir btrbk
 601 target_preserve $q_preserve
 602 target_preserve_min 2h
 603 EOF
 604       ;;
 605   esac
 606
 607 }
 608
 609 # make /q be last
 610 mp_count=${#mountpoints[@]}
 611 for (( i=0; i < mp_count - 1 ; i++ )); do
 612   if [[ ${mountpoints[i]} == /q ]]; then
 613     unset "mountpoints[i]"
 614     mountpoints+=(/q)
 615   fi
 616 done
 617
 618 for m in ${mountpoints[@]}; do
 619   case $m in
 620     /o)
 621       vol=/mnt/o
 622       ;;
 623     *)
 624       vol=/mnt/root
 625       ;;
 626   esac
 627
 628   sub=${m#/}
 629   if [[ $source ]]; then
 630     cat >>/etc/btrbk$conf_suf.conf <<EOF
 631 volume ssh://$bbksource$vol
 632 subvolume $sub
 633 EOF
 634     qconf
 635     cat >>/etc/btrbk$conf_suf.conf <<EOF
 636 target send-receive $vol/btrbk
 637 EOF
 638   fi
 639   if (( ${#targets[@]} )); then
 640     cat >>/etc/btrbk$conf_suf.conf <<EOF
 641 volume $vol
 642 subvolume $sub
 643 EOF
 644     qconf
 645     for tg in ${targets[@]}; do
 646       # handle ipv6
 647       if [[ $tg == *:* ]]; then
 648         tg="[$tg]"
 649       fi
 650       cat >>/etc/btrbk$conf_suf.conf <<EOF
 651 target send-receive ssh://$tg$vol/btrbk
 652 EOF
 653     done
 654   fi
 655 done
 656
 657 # todo: umount first to ensure we don't have any errors
 658 # todo: do some kill fuser stuff to make umount more reliable
 659
 660
 661 if $conf_only; then
 662   mexit 0
 663 fi
 664
 665
 666
 667 if $dry_run; then
 668   m btrbk -c /etc/btrbk$conf_suf.conf -v -n $cmd_arg
 669   mexit 0
 670 fi
 671 # -q and just using the syslog option seemed nice,
 672 # but it doesn't show when a send has a parent and when it doesn't.
 673 m btrbk -c /etc/btrbk$conf_suf.conf $preserve_arg $verbose_arg $progress_arg $cmd_arg
 674
 675 if $early; then
 676   exit 0
 677 fi
 678
 679 # todo: tp not valid anymore.
 680 # if we have it, sync to systems which don't
 681 if mountpoint $rsync_mountpoint >/dev/null; then
 682   for tg in ${targets[@]}; do
 683     case $tg in
 684       tp)
 685         dirs=(/p/c/machine_specific/tp)
 686         for x in /p/c/machine_specific/*.hosts; do
 687           if grep -qxF $tg $x; then
 688             dirs+=(${x%.hosts})
 689           fi
 690         done
 691         m rsync -aSAXPH --specials --devices --delete --relative ${dirs[@]} root@$tg:/
 692         ;;
 693     esac
 694   done
 695 fi
 696
 697 subvols=()
 698 for mp in "${mountpoints[@]}"; do
 699   subvols+=("${mp##*/}")
 700 done
 701 if [[ $source ]]; then
 702   m mount-latest-subvol "${subvols[@]}"
 703 else
 704   for tg in ${targets[@]}; do
 705     m /a/exe/mount-latest-remote "$tg" "${subvols[@]}" || ret=$?
 706   done
 707 fi
 708
 709 if [[ $ret == 0 ]]; then
 710   for tg in ${targets[@]}; do
 711     h=$(ssh $tg hostname)
 712     if [[ $h == kd && $HOSTNAME == x3 && $HOSTNAME == "$MAIL_HOST" ]]; then
 713       ssh root$tg systemctl --no-block start btrbk-spread
 714     fi
 715     rsync -a -f"- */" -f"+ *" /var/log/btrbk/ root@$tg:/var/log/btrbk/$tg
 716     ssh root@$tg /usr/local/bin/mail-backup-clean
 717   done
 718   if [[ $source ]]; then
 719     rsync -a -f"- */" -f"+ *" $source:/var/log/btrbk/ /var/log/btrbk/$source
 720   fi
 721 fi
 722
 723 mexit $ret
 724
 725 # todo: move variable data we don't care about backing up
 726 # to /nocow and symlink it.
 727
 728
 729 # background on btrbk timezones. with short/long, timestamps use local time.
 730 # for long, if your local time moves backwards, by moving timezones or
 731 # for an hour when daylight savings changes it, you will temporarily get
 732 # a more aggressive retention policy for the overlapping period, and
 733 # vice versa for the opposite timezone move. The alternative is using
 734 # long-iso, which puts timezone info into the timestamp, which means
 735 # that instead of shifting time, you shift the start of day/week/month
 736 # which is used for retention to your new local time, which means for
 737 # example, if you moved forward by 8 hours, the daily/weekly/monthly
 738 # retention will be 8 hours more aggressive since midnight is at a new
 739 # time, unless you fake the timzeone using the TZ env variable.
 740 # However, in the short term, there will be no inconsistencies.
 741 # I don't see any problem with shifting when the day starts for
 742 # retention, so I'm using long-iso.
 743
 744 # note to create a long-iso timestamp: date +%Y%m%dT%H%M%S%z