iankelling.org Git - distro-setup/blob - btrbk-run

   1 #!/bin/bash
   2 # Copyright (C) 2016 Ian Kelling
   3
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16
  17 # todo: if we cancel in the middle of a btrfs send, then run again
  18 # immediately, the received subvolume doesn't get a Received UUID:
  19 # field, and we won't mount it. Need to figure out a solution that will
  20 # fix this.
  21
  22
  23 [[ $EUID == 0 ]] || exec sudo -E "${BASH_SOURCE[0]}" "$@"
  24
  25 set -e; . /usr/local/lib/bash-bear; set +e
  26
  27 usage() {
  28   cat <<'EOF'
  29 btrbk-run [OPTIONS] [run|resume|archive]
  30 usually -t TARGET_HOST or -s SOURCE_HOST
  31
  32 Note, at source location, intentionally not executable, run and read
  33 install-my-scripts.
  34
  35 EOF
  36   echo "top of script file:"
  37   sed -n '1,/^[# ]*end command line/{p;b};q' "$0"
  38   exit $1
  39 }
  40
  41
  42 pre=btrbk-run
  43
  44 script_name="${BASH_SOURCE[0]}"
  45 script_name="${script_name##*/}"
  46 d() {
  47   if $dry_run || $conf_only; then
  48     printf "$pre dry-run: %s\n"  "$*"
  49   else
  50     printf "$pre running: %s\n"  "$*"
  51     "$@"
  52   fi
  53 }
  54 m() { if $verbose; then printf "$pre %s\n"  "$*"; fi;  "$@"; }
  55 e() { printf "$pre %s\n"  "$*"; }
  56 die() { printf "$pre error: %s\n" "$*" >&2; echo "$pre exiting with status 1" >&2; exit 1; }
  57 mexit() { echo "$pre exiting with status $1"; exit $1; }
  58
  59 uninstalled-file-die() {
  60   die "uninstalled file $1. run install-my-scripts or rerun with -f"
  61 }
  62
  63 set-location() {
  64   case $HOSTNAME in
  65     kw)
  66       at_work=true
  67       ;;
  68     kd|frodo)
  69       at_home=true
  70       ;;
  71     x2|x3|sy)
  72       if [[ $(dig +short @10.2.0.1 -x 10.2.0.2 2>&1 ||:) == kd.b8.nz. ]] \
  73            && ip n show 10.2.0.1 | grep . &>/dev/null; then
  74         at_home=true
  75       elif ping -q -c1 -w1 hal.office.fsf.org &>/dev/null \
  76           && ip n show 192.168.0.26 | grep . &>/dev/null; then
  77         at_work=true
  78       fi
  79       ;;
  80   esac
  81 }
  82
  83 exit-if-no-default-targets() {
  84   if ! $force && [[ $HOSTNAME != "$MAIL_HOST" ]]; then
  85     echo "MAIL_HOST=$MAIL_HOST, nothing to do"
  86     mexit 0
  87   fi
  88   case $HOSTNAME in
  89     kw|kd|frodo|x2|x3|sy) : ;;
  90     *)
  91       die "error: no default targets for this host, use -t"
  92       ;;
  93   esac
  94 }
  95
  96 add-x3-target() {
  97   # main work machine
  98   if ping -q -c1 -w1 x3.office.fsf.org &>/dev/null; then
  99     targets+=(x3.office.fsf.org)
 100   elif ping -q -c1 -w1 $h.b8.nz &>/dev/null; then
 101     # in case we took it home
 102     targets+=(x3.b8.nz)
 103   elif ping -q -c1 -w1 ${h}w.b8.nz &>/dev/null; then
 104     targets+=(x3w.b8.nz)
 105   else
 106     targets+=(x3wg.b8.nz)
 107   fi
 108 }
 109
 110 add-wireless-target-h() {
 111   if ping -q -c1 -w1 $h.b8.nz &>/dev/null; then
 112     targets+=($h.b8.nz)
 113   elif ping -q -c1 -w1 ${h}w.b8.nz &>/dev/null; then
 114     targets+=(${h}w.b8.nz)
 115   fi
 116 }
 117
 118 qconf() {
 119   case $sub in
 120     q)
 121       # q has sensitive data i dont want to backup for so long
 122       cat >>/etc/btrbk$conf_suf.conf <<EOF
 123 snapshot_preserve $q_preserve
 124 snapshot_preserve_min 2h
 125 snapshot_dir btrbk
 126 target_preserve $q_preserve
 127 target_preserve_min 2h
 128 EOF
 129       ;;
 130   esac
 131
 132 }
 133
 134
 135 # latest $MAIL_HOST
 136 if [[ -e /b/bash_unpublished/source-state ]]; then
 137   source /b/bash_unpublished/source-state
 138 fi
 139
 140 # note q is owned by root:1000
 141
 142 mountpoints=()
 143
 144 rsync_mountpoint=/q
 145
 146 ret=0
 147 # default options
 148 conf_only=false
 149 dry_run=false # mostly for testing
 150 rate_limit=no
 151 verbose=true; verbose_arg=-v
 152 force=false
 153 if [[ $INVOCATION_ID ]]; then
 154   # INVOCATION_ID means running as a systemd service. we cant show progress in this case,
 155   # but if we pass the arg, it will insert mbuffer into the command.
 156   progress_arg=
 157 else
 158   progress_arg="--progress"
 159 fi
 160 incremental_strict=false
 161 pull_reexec=false
 162
 163 default_args_file=/etc/btrbk-run.conf
 164 if [[ -s $default_args_file ]]; then
 165   # shellcheck disable=SC2046 # we want word splitting
 166   set -- $(< $default_args_file) "$@"
 167   # i havent used this feature yet, so warn about it
 168   echo "$0: warning: default btrbk-run options set in $default_args_file (sleeping 5 seconds):"
 169   cat $default_args_file
 170   sleep 5
 171 fi
 172
 173 once_args_file=/etc/btrbk-run-once.conf
 174 if mv -f $once_args_file $once_args_file-tmp 2> >(sed '/No such file or directory/d'); then
 175   # shellcheck disable=SC2046 # we want word splitting
 176   set -- $(< $once_args_file-tmp) "$@"
 177   # i havent used this feature yet, so warn about it
 178   echo "$0: btrbk-run options set in $once_args_file:"
 179   cat $once_args_file-tmp
 180   rm -f $once_args_file-tmp
 181 fi
 182
 183
 184 targets=()
 185 early=false
 186 fast=false
 187 kd_spread=false
 188 check_installed=false
 189 orig_args=("$@")
 190 temp=$(getopt -l check-installed,fast,pull-reexec,help 23cefikl:m:npqrs:t:vh "$@") || usage 1
 191 eval set -- "$temp"
 192 while true; do
 193   case $1 in
 194     # for the rare case we want to run multiple instances at the same time
 195     -2) conf_suf=2 ;;
 196     -3) conf_suf=3 ;;
 197     # only creates the config file, does not run btrbk
 198     -c) conf_only=true ;;
 199     --check-installed)
 200       check_installed=true
 201       ;;
 202     # quit early, just btrbk, no extra remounting etc.
 203     -e) early=true ;;
 204     # avoids some default behaviors:
 205     # - no skipping hosts where xprintidle haven't been idle recently
 206     # - exit if we can't ssh to 1 or more hosts
 207     # - still set default hosts despite MAIL_HOST status
 208     -f) force=true ;;
 209     # skip various checks. when we run twice in a row for
 210     # switch mail-host, no need to repeat the same checks again.
 211     --fast) fast=true ;;
 212     -i) incremental_strict=true ;;
 213     # note this implies resume and -p because it is just meant to make
 214     # other hosts have the same snapshots, not do any expiry or new
 215     # backups.
 216     -k) kd_spread=true ;;
 217     # bytes per second, suffix k m g
 218     -l) rate_limit=$2; shift ;;
 219     # Comma separated mountpoints to backup. This has defaults set below.
 220     -m) IFS=, mountpoints=($2); unset IFS; shift ;;
 221     -n) dry_run=true ;;
 222     # preserve existing snapshots and backups
 223     -p) preserve_arg=-p ;;
 224     # internal option for rerunning under newer SOURCE_HOST version.
 225     --pull-reexec) pull_reexec=true;;
 226     # quiet
 227     -q) verbose=false; verbose_arg=; progress_arg= ;;
 228     # source host to receive a backup from
 229     -s)
 230       source=$2
 231       bbksource=$source
 232       if [[ $source == *:* ]]; then
 233         bbksource="[$source]"
 234       fi
 235       shift
 236       ;;
 237     # target hosts to send to. empty is valid for just doing local
 238     # snapshot. we have default hosts we will populate.
 239     -t) IFS=, targets=($2); unset IFS; shift ;;
 240     # verbose.
 241     -v) verbose=true; verbose_arg=-v ;;
 242     -h|--help) usage ;;
 243     --) shift; break ;;
 244     *) die "Internal error!" ;;
 245   esac
 246   shift
 247 done
 248
 249 cmd_arg="$1"
 250
 251
 252
 253 if ! $force && { $check_installed || [[ ! $source ]]; } ; then
 254   install_bin_files=(
 255     mount-latest-subvol
 256     check-subvol-stale
 257     btrbk-run
 258   )
 259   for f in ${install_bin_files[@]}; do
 260     if ! diff -q /a/bin/ds/$f /usr/local/bin/$f; then
 261       uninstalled-file-die $f
 262     fi
 263   done
 264   if ! diff -q /a/bin/bash-bear-trap/bash-bear /usr/local/lib/bash-bear; then
 265     uninstalled-file-die err
 266   fi
 267   if $check_installed; then
 268     exit 0
 269   fi
 270 fi
 271
 272
 273 if $kd_spread; then
 274   if [[ $cmd_arg && $cmd_arg != resume ]]; then
 275     die "dont pass -k without resume or empty run arg"
 276   fi
 277   if [[ $HOSTNAME == "$MAIL_HOST" ]]; then
 278     die "something went wrong, -k not meant to be run on MAIL_HOST"
 279   fi
 280   if [[ $HOSTNAME != kd ]]; then
 281     die "something went wrong, -k only meant to run on kd"
 282   fi
 283   cmd_arg=resume
 284   preserve_arg=-p
 285   h=sy
 286   add-wireless-target-h
 287 fi
 288
 289 if [[ ! $cmd_arg ]]; then
 290   cmd_arg=run
 291 fi
 292
 293
 294 std_preserve="36h 14d 8w 24m"
 295 q_preserve="18h 14d 8w"
 296
 297 case $cmd_arg in
 298   run|resume) : ;;
 299
 300   # This works better than the normal archive command. We have to
 301   # specify the mount points, but that is what we are used to doing and
 302   # we prefer it. Another difference is that archive works recursively
 303   # and we don't care about that. Sometimes we may still want to run
 304   # btrbk archive, but it doesn't even use the config file, so just
 305   # run it directly, eg:
 306   # time s btrbk -v archive /mnt/r7/amy/boot/btrbk ssh://bo/mnt/boot2/btrbk
 307   archive)
 308     cmd_arg=resume
 309     std_preserve="999h 999d 999w 999m"
 310     q_preserve="$std_preserve"
 311     preserve_arg=-p
 312     ;;
 313   *) die "untested command arg" ;;
 314 esac
 315
 316 if (( $# > 1 )); then
 317   die: "only 1 nonoption arg is supported"
 318 fi
 319
 320 if [[ -v targets && $source ]]; then
 321   # note, this doesnt need to be the case, but
 322   # we would need to think about it.
 323   die "error: -t and -s are mutually exclusive"
 324 fi
 325
 326 ### end options parsing
 327
 328 # remove path from earlier version of btrbk
 329 rm -f /usr/sbin/btrbk
 330 # note, this still works as intended if there is no /usr/bin/btrbk
 331 if [[ /a/opt/btrbk/btrbk -nt /usr/bin/btrbk ]]; then
 332   if [[ -e /b/distro-functions/src/package-manager-abstractions ]]; then
 333     . /b/distro-functions/src/package-manager-abstractions
 334     pi asciidoctor
 335   fi
 336   cd /a/opt/btrbk
 337   m make install
 338   cd /
 339 fi
 340
 341 # TODO: i wonder if there should be an option to send to the default
 342 # targets, plus any given on the command line.
 343
 344
 345 at_work=false
 346 at_home=false
 347
 348
 349 # set default targets
 350 if [[ ! -v targets && ! $source ]]; then
 351   exit-if-no-default-targets
 352   set-location
 353   if $at_home; then
 354     if ! $kd_spread && [[ $HOSTNAME != x3 ]]; then
 355       add-x3-target
 356     fi
 357     if [[ $HOSTNAME != kd ]]; then
 358       targets+=(kd.b8.nz)
 359     fi
 360     wireless_home_hosts=(
 361       x2
 362       sy
 363     )
 364     for h in ${wireless_home_hosts[@]}; do
 365       if [[ $HOSTNAME != "$h" ]]; then
 366         add-wireless-target-h
 367       fi
 368     done
 369   elif $at_work; then
 370     targets+=(i.b8.nz)
 371     for h in x2 x3 kw; do
 372       if [[ $HOSTNAME == "$h" ]]; then
 373         continue
 374       fi
 375       if ping -q -c1 -w1 $h.office.fsf.org &>/dev/null; then
 376         targets+=($h.office.fsf.org)
 377       fi
 378     done
 379   else
 380     targets+=(i.b8.nz)
 381   fi
 382 fi
 383
 384 if [[ ${mountpoints[0]} ]]; then
 385   for mp in ${mountpoints[@]}; do
 386     if [[ -e /nocow/btrfs-stale/$mp ]]; then
 387       die "error: $mp is stale, mount-latest-subvol first"
 388     fi
 389   done
 390 else
 391   # set default mountpoints
 392   if [[ ${targets[0]} == tp ]]; then
 393     prospective_mps=(/a)
 394   else
 395     case $HOSTNAME in
 396       *)
 397         prospective_mps=()
 398         if [[ $source ]]; then
 399           source_state="$(ssh $source 'cat /a/bin/bash_unpublished/source-state; echo source_host=$HOSTNAME')"
 400           eval "$source_state"
 401           # shellcheck disable=SC2154 # assigned in the above eval.
 402           if [[ $source_host == "$MAIL_HOST" ]]; then
 403             prospective_mps+=(/o)
 404           fi
 405           if [[ $source_host == "$HOST2" ]]; then
 406             prospective_mps+=(/a /qr /qd /q)
 407           fi
 408         else
 409           if [[ $HOSTNAME == "$MAIL_HOST" ]]; then
 410             prospective_mps+=(/o)
 411           fi
 412           if [[ $HOSTNAME == "$HOST2" ]]; then
 413             prospective_mps+=(/a /qr /qd /q)
 414           fi
 415           if $kd_spread; then
 416             prospective_mps=(/a /o /qr /qd /q)
 417           fi
 418         fi
 419         # note: put q last just in case its specific retention options were to
 420         # affect other config sections. I havent tested if that is the case.
 421         ;;
 422     esac
 423   fi
 424   for mp in ${prospective_mps[@]}; do # default mountpoints to sync
 425     if [[ -e /nocow/btrfs-stale/$mp ]]; then
 426       e "warning: $mp stale, not adding to default mountpoints"
 427       continue
 428     fi
 429     if awk '{print $2}' /etc/fstab | grep -xF $mp &>/dev/null; then
 430       mountpoints+=($mp)
 431     fi
 432   done
 433 fi
 434
 435 tmp=$(( ${#mountpoints[@]} == 0 ))
 436 if (( tmp )); then
 437   die didnt get mountpoint arg and had no defaults
 438 fi
 439
 440 ##### end command line parsing ########
 441
 442 #### begin pre-checks #####
 443
 444 # todo: this has a timing problem, since btrbk.timer could activate the service after this check.
 445 if ! $fast && [[ $source ]]; then
 446   if [[ $(ssh $source ps --no-headers -o comm 1) == systemd ]]; then
 447     status=$(ssh $source systemctl is-active btrbk.service) || : # normally returns 3
 448     case $status in
 449       inactive|failed) : ;;
 450       *)
 451         echo "$0: error: btrbk is running on source. exiting out of caution"
 452         mexit 1
 453     esac
 454   fi
 455 fi
 456
 457 if ! command -v btrbk &>/dev/null; then
 458   die "error: no btrbk binary found"
 459 fi
 460
 461 if ! $pull_reexec && [[ $source ]] && $pulla && ! $force ; then
 462   ssh root@$source btrbk-run --check-installed || exit 1
 463 fi
 464
 465 #### end pre-checks #####
 466
 467
 468
 469 mkdir -p /var/log/btrbk
 470 # The journal doesnt go back to my oldest backups, and I've found myself
 471 # wanting older logs. Not going to bother expiring old logs, since it is
 472 # fine if they go back years.
 473 log_path=/var/log/btrbk/$(date +%F_%T%:::z).log
 474 echo copying output to $log_path
 475 exec &> >(pee cat 'ts "%F %T"|dd of='$log_path' status=none')
 476
 477
 478 if $verbose; then
 479   printf " options: conf_only=%s\ndry_run=%s\nrate_limit=%s\nverbose=%s\ncmd_arg=%s\n" "$conf_only" "$dry_run" "$rate_limit" "$verbose" "$cmd_arg"
 480 fi
 481
 482 if [[ -v targets ]]; then
 483   echo "targets: ${targets[*]}"
 484 fi
 485
 486 if [[ $source ]]; then
 487   echo "source: $source"
 488 fi
 489
 490 echo "mountpoints: ${mountpoints[*]}"
 491
 492
 493
 494
 495 # pull_reexec stops us from getting into an infinite loop if there is some
 496 # kind of weird problem
 497 pulla=false
 498 for m in "${mountpoints[@]}"; do
 499   if [[ $m == /a ]]; then
 500     pulla=true
 501     break
 502   fi
 503 done
 504
 505 if ! $pull_reexec && [[ $source ]] && $pulla ; then
 506   tmpf=$(mktemp)
 507   m rsync -ra $source:/usr/local/bin/{mount-latest-subvol,check-subvol-stale} /usr/local/bin
 508   m rsync -ra $source:/usr/local/lib/bash-bear /usr/local/lib
 509   m scp $source:/a/bin/distro-setup/btrbk-run $tmpf
 510   if ! diff -q $tmpf ${BASH_SOURCE[0]}; then
 511     e "found different version on host $source. reexecing"
 512     install -T $tmpf /usr/local/bin/btrbk-run
 513     m /usr/local/bin/btrbk-run --pull-reexec "${orig_args[@]}"
 514     mexit 0
 515   fi
 516 fi
 517
 518
 519 if ! $fast; then
 520   # if our mountpoints are from stale snapshots,
 521   # it doesn't make sense to do a backup.
 522   m check-subvol-stale ${mountpoints[@]} || die "found stale mountpoints in ${mountpoints[*]}"
 523
 524   # for an initial run, btrbk requires the dir to exist.
 525   mkdir -p /mnt/{root,o}/btrbk
 526 fi
 527 local_zone=$(date +%z)
 528
 529 if [[ $source ]]; then
 530   if $fast; then
 531     zone=$local_zone
 532   else
 533     if ! zone=$(ssh root@$source date +%z); then
 534       if $conf_only; then
 535         echo "$0: warning: failed to ssh to root@$source"
 536       else
 537         die failed to ssh to root@$source
 538       fi
 539     fi
 540     if [[ $zone != "$local_zone" ]]; then
 541       die "error: dont confuse yourself with multiple time zones. $h has different timezone than localhost"
 542     fi
 543   fi
 544 else
 545
 546   sshable=()
 547   sshfail=()
 548   for h in ${targets[@]}; do
 549     if $fast || $conf_only; then
 550       # Use some typical values in this case
 551       root_size=$(( 1024 * 1024 * 2000 )) #2tb
 552       percent_used=10
 553       zone=$(date +%z)
 554     elif remote_str=$(timeout -s 9 6 ssh root@$h "mkdir -p /mnt/root/btrbk /mnt/o/btrbk && date +%z && df --output=size,pcent / | tail -n1"); then
 555       mapfile -t tmp_array <<<"$remote_str"
 556       zone="${tmp_array[0]}"
 557       IFS=" " read -r root_size percent_used <<<"${tmp_array[1]}"
 558       percent_used=${percent_used%%%}
 559
 560       tmp=$(( ${#tmp_array[@]} != 2 ))
 561       if (( tmp )); then
 562         die "error: didnt get 2 lines in test ssh to target $h. investigate"
 563       fi
 564       case $percent_used in
 565         [0-9]|[1-9][0-9]) : ;;
 566         *)
 567           die "error: didnt get percent disk use in test ssh to target $h. investigate"
 568           ;;
 569       esac
 570     else
 571       sshfail+=($h)
 572       continue
 573     fi
 574
 575     # we may be booted into a bootstrap fs or something
 576     min_root_kb=$(( 1024 * 1024 * 200 )) # 200 gb
 577     tmp=$(( root_size < min_root_kb ))
 578     if (( tmp )); then
 579       continue
 580     fi
 581
 582     tmp=$(( percent_used >= 98 ))
 583     if (( tmp )); then
 584       die "error: filesystem on target $h is $percent_used % full"
 585     fi
 586
 587     # on sy, xprintidle is resetting every 12 seconds even when not
 588     # idle, i dunno why, instead we are checking if the screen is locked,
 589     # which is good enough.
 590     #
 591     # This is a separate ssh because the command can fail and thatis ok.
 592     if ! $force; then
 593       locked=false
 594       if lock_info=$(timeout -s 9 6 ssh $h DISPLAY=:0 xscreensaver-command -time); then
 595         if [[ $lock_info != *non-blanked* ]]; then
 596           locked=true
 597         fi
 598       else
 599         locked=true
 600       fi
 601       if ! $locked; then
 602         # Ignore this host. i sometimes use a non-main machine for
 603         # testing or web browsing, knowing that everything will be wiped
 604         # by the next backup, but I dont want it to happen as Im using
 605         # it from cronjob.
 606         e "warning: $h: seems to be actively in use, skipping for now"
 607         continue
 608       fi
 609     fi
 610     sshable+=($h)
 611     if [[ $zone != "$local_zone" ]]; then
 612       die "error: dont confuse yourself with multiple time zones. $h has different timezone than localhost"
 613     fi
 614   done
 615   if [[ ! ${sshable[*]} ]] || { $force && [[ ${sshfail[*]} ]]; }; then
 616     die "failed to ssh to hosts: ${sshfail[*]}"
 617   else
 618     if [[ ${sshfail[*]} ]]; then
 619       ret=1
 620       e "error: failed to ssh to ${sshfail[*]} but continuing with other hosts"
 621     fi
 622     targets=(${sshable[@]})
 623   fi
 624 fi
 625
 626
 627 cat >/etc/btrbk$conf_suf.conf <<EOF
 628 ssh_identity /q/root/h
 629 #ssh_identity /root/.ssh/home
 630
 631 # Just a guess that local7 is a good facility to pick.
 632 # It's a bit odd that the transaction log has to be logged to
 633 # a file or syslog, while other output is sent to std out.
 634 # The man does not mention a way for them to be together, but
 635 # I dunno if setting a log level like warn might also output
 636 # transaction info.
 637 transaction_syslog local7
 638
 639 # trying this out
 640 #stream_compress zstd
 641
 642 # so we only run one at a time
 643 lockfile                   /var/lock/btrbk$conf_suf.lock
 644
 645 # default format of short does not accomidate hourly preservation setting
 646 timestamp_format long-iso
 647
 648 # only make a snapshot if things have changed
 649 snapshot_create onchange
 650 # I could make this different from target_preserve,
 651 # if one disk had less space.
 652 # for now, keeping them equal.
 653 snapshot_preserve $std_preserve
 654 snapshot_preserve_min 6h
 655 snapshot_dir btrbk
 656 # so, total backups = ~58
 657 target_preserve $std_preserve
 658 target_preserve_min 6h
 659
 660 # it seems very likely that not doing this could result in clone source not found
 661 # errors, for example when expiry happens differently on different hosts,
 662 # also, as btrbk does by default, if a failed send happens, on the next run it
 663 # will warn about a stray subvolume, but then create a backup of a newer subvol
 664 # and use an older subvol as the parent.
 665 incremental_prefs sao:1
 666
 667 # if something fails and it's not obvious, try doing
 668 # btrbk -l debug -v dryrun
 669
 670 rate_limit $rate_limit
 671 EOF
 672
 673 if $incremental_strict; then
 674   cat >>/etc/btrbk$conf_suf.conf <<EOF
 675 incremental strict
 676 EOF
 677 fi
 678
 679
 680 # make /q be last
 681 mp_count=${#mountpoints[@]}
 682 for (( i=0; i < mp_count - 1 ; i++ )); do
 683   if [[ ${mountpoints[i]} == /q ]]; then
 684     unset "mountpoints[i]"
 685     mountpoints+=(/q)
 686   fi
 687 done
 688
 689
 690
 691 snap_list_cmds=()
 692 tg_snaps=()
 693 declare -A source_snaps
 694
 695 for m in ${mountpoints[@]}; do
 696   case $m in
 697     /o)
 698       vol=/mnt/o
 699       ;;
 700     *)
 701       vol=/mnt/root
 702       ;;
 703   esac
 704
 705   sub=${m#/}
 706   snap_list_cmds+=("echo $vol/btrbk/$sub.*;")
 707
 708   if [[ $source ]]; then
 709     tmp_a=($vol/btrbk/$sub.*)
 710     tg_snaps+=("${tmp_a[*]}")
 711     cat >>/etc/btrbk$conf_suf.conf <<EOF
 712 volume ssh://$bbksource$vol
 713 subvolume $sub
 714 EOF
 715     qconf
 716     cat >>/etc/btrbk$conf_suf.conf <<EOF
 717 target send-receive $vol/btrbk
 718 EOF
 719   else # we have targets
 720     for snap in "$vol/btrbk/$sub."*; do
 721       source_snaps[$snap]=t
 722     done
 723
 724     cat >>/etc/btrbk$conf_suf.conf <<EOF
 725 volume $vol
 726 subvolume $sub
 727 EOF
 728     qconf
 729     for tg in ${targets[@]}; do
 730       # handle ipv6
 731       if [[ $tg == *:* ]]; then
 732         tg="[$tg]"
 733       fi
 734       cat >>/etc/btrbk$conf_suf.conf <<EOF
 735 target send-receive ssh://$tg$vol/btrbk
 736 EOF
 737     done
 738   fi
 739 done
 740
 741 # Delete any subvols on the receiving host that don't exist on the
 742 # sending host.  Otherwise, the receiving host could have snapshots that
 743 # aren't on the sending side, and thus become odd leaf subvols, and then
 744 # btrbk could try to use them when we sync back, creating a weird tree
 745 # instead of linear parent/child relationship. Maybe this could lead to
 746 # a missing source subvol error, so lets avoid it.
 747
 748 get-orphan-tg-snaps() {
 749   orphan_tg_snaps=()
 750   for (( i=0; i < ${#mountpoints[@]}; i++ )); do
 751     orphan_start_count=${#orphan_tg_snaps[@]}
 752     tg_snap_count=0
 753     for tg_snap in ${tg_snaps[$i]}; do
 754       tg_snap_count=$(( tg_snap_count + 1 ))
 755       if [[ ! ${source_snaps[$tg_snap]} ]]; then
 756         orphan_tg_snaps+=("$tg_snap")
 757       fi
 758     done
 759     orphan_mp_count=$(( ${#orphan_tg_snaps[@]} - orphan_start_count ))
 760     # sanity checking
 761     tmp=$(( tg_snap_count  > 1 && tg_snap_count == orphan_mp_count ))
 762     if (( tmp )) ; then
 763       die "something went wrong checking orphans on $tg: for mountpoint ${mountpoints[$i]}, $orphan_mp_count"
 764     fi
 765   done
 766 }
 767
 768 if [[ $source ]]; then
 769   for snap in $(ssh root@$source "shopt -s nullglob; ${snap_list_cmds[*]}"); do
 770     source_snaps[$snap]=t
 771   done
 772   get-orphan-tg-snaps
 773   tmp=$(( ${#orphan_tg_snaps[*]} >= 1 ))
 774   if (( tmp )); then
 775     d btrfs sub del ${orphan_tg_snaps[*]}
 776   fi
 777 else # we have targets
 778   for tg in ${targets[@]}; do
 779     tmp_str=$(ssh root@$tg "shopt -s nullglob; ${snap_list_cmds[*]}")
 780     mapfile -t tg_snaps <<<"$tmp_str"
 781     get-orphan-tg-snaps
 782     tmp=$(( ${#orphan_tg_snaps[*]} >= 1 ))
 783     if (( tmp )); then
 784       d ssh root@$tg "btrfs sub del ${orphan_tg_snaps[*]}"
 785     fi
 786   done
 787 fi
 788
 789 # todo: umount first to ensure we don't have any errors
 790 # todo: do some kill fuser stuff to make umount more reliable
 791
 792
 793 if $conf_only; then
 794   mexit 0
 795 fi
 796
 797
 798
 799 if $dry_run; then
 800   m btrbk -c /etc/btrbk$conf_suf.conf -v -n $cmd_arg
 801   mexit 0
 802 fi
 803 # -q and just using the syslog option seemed nice,
 804 # but it doesn't show when a send has a parent and when it doesn't.
 805 m btrbk -c /etc/btrbk$conf_suf.conf $preserve_arg $verbose_arg $progress_arg $cmd_arg
 806
 807 if $early; then
 808   exit 0
 809 fi
 810
 811 # todo: tp not valid anymore.
 812 # if we have it, sync to systems which don't
 813 if mountpoint $rsync_mountpoint >/dev/null; then
 814   for tg in ${targets[@]}; do
 815     case $tg in
 816       tp)
 817         dirs=(/p/c/machine_specific/tp)
 818         for x in /p/c/machine_specific/*.hosts; do
 819           if grep -qxF $tg $x; then
 820             dirs+=(${x%.hosts})
 821           fi
 822         done
 823         m rsync -aSAXPH --specials --devices --delete --relative ${dirs[@]} root@$tg:/
 824         ;;
 825     esac
 826   done
 827 fi
 828
 829 subvols=()
 830 for mp in "${mountpoints[@]}"; do
 831   subvols+=("${mp##*/}")
 832 done
 833 if [[ $source ]]; then
 834   m mount-latest-subvol "${subvols[@]}"
 835 else
 836   for tg in ${targets[@]}; do
 837     m /a/exe/mount-latest-remote "$tg" "${subvols[@]}" || ret=$?
 838   done
 839 fi
 840
 841 # todo, we get hostnames earlier, reuse that.
 842 if [[ $ret == 0 ]]; then
 843   for tg in ${targets[@]}; do
 844     h=$(ssh $tg hostname)
 845     if [[ $h == kd && $HOSTNAME == x3 && $HOSTNAME == "$MAIL_HOST" ]]; then
 846       m ssh root@$tg 'btrbk-spread-wrap &>/dev/null </dev/null &'
 847     fi
 848     rsync --mkpath -a -f"- */" -f"+ *" /var/log/btrbk/ root@$tg:/var/log/btrbk/$tg
 849     ssh root@$tg /usr/local/bin/mail-backup-clean
 850   done
 851   if [[ $source ]]; then
 852     rsync --mkpath -a -f"- */" -f"+ *" $source:/var/log/btrbk/ /var/log/btrbk/$source
 853   fi
 854 fi
 855
 856 mexit $ret
 857
 858 # todo: move variable data we don't care about backing up
 859 # to /nocow and symlink it.
 860
 861
 862 # background on btrbk timezones. with short/long, timestamps use local time.
 863 # for long, if your local time moves backwards, by moving timezones or
 864 # for an hour when daylight savings changes it, you will temporarily get
 865 # a more aggressive retention policy for the overlapping period, and
 866 # vice versa for the opposite timezone move. The alternative is using
 867 # long-iso, which puts timezone info into the timestamp, which means
 868 # that instead of shifting time, you shift the start of day/week/month
 869 # which is used for retention to your new local time, which means for
 870 # example, if you moved forward by 8 hours, the daily/weekly/monthly
 871 # retention will be 8 hours more aggressive since midnight is at a new
 872 # time, unless you fake the timzeone using the TZ env variable.
 873 # However, in the short term, there will be no inconsistencies.
 874 # I don't see any problem with shifting when the day starts for
 875 # retention, so I'm using long-iso.
 876
 877 # note to create a long-iso timestamp: date +%Y%m%dT%H%M%S%z