iankelling.org Git - distro-setup/blob - btrbk-run

   1 #!/bin/bash
   2
   3 # Configure & run btrbk & related work on Ian's computers.
   4 # Copyright (C) 2024  Ian Kelling
   5
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation, either version 3 of the License, or
   9 # (at your option) any later version.
  10
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 # SPDX-License-Identifier: GPL-3.0-or-later
  20
  21
  22 # todo: if we cancel in the middle of a btrfs send, then run again
  23 # immediately, the received subvolume doesn't get a Received UUID:
  24 # field, and we won't mount it. Need to figure out a solution that will
  25 # fix this.
  26
  27
  28 [[ $EUID == 0 ]] || exec sudo -E "${BASH_SOURCE[0]}" "$@"
  29
  30 set -e; . /usr/local/lib/bash-bear; set +e
  31 shopt -s nullglob
  32
  33 usage() {
  34   cat <<'EOF'
  35 btrbk-run [OPTIONS] [run|resume|archive]
  36 usually -t TARGET_HOST or -s SOURCE_HOST
  37
  38 Note, at source location, intentionally not executable, run and read
  39 install-my-scripts.
  40
  41 EOF
  42   echo "top of script file:"
  43   sed -n '1,/^[# ]*end command line/{p;b};q' "$0"
  44   exit $1
  45 }
  46
  47
  48 pre=btrbk-run
  49
  50 script_name="${BASH_SOURCE[0]}"
  51 script_name="${script_name##*/}"
  52 d() {
  53   if $dry_run || $conf_only; then
  54     printf "$pre dry-run: %s\n"  "$*"
  55   else
  56     printf "$pre running: %s\n"  "$*"
  57     "$@"
  58   fi
  59 }
  60 m() { if $verbose; then printf "$pre %s\n"  "$*"; fi;  "$@"; }
  61 e() { printf "$pre %s\n"  "$*"; }
  62 die() { printf "$pre error: %s\n" "$*" >&2; echo "$pre exiting with status 1" >&2; exit 1; }
  63 mexit() { echo "$pre exiting with status $1"; exit $1; }
  64
  65 uninstalled-file-die() {
  66   die "uninstalled file $1. run install-my-scripts or rerun with -f"
  67 }
  68
  69 set-location() {
  70   case $HOSTNAME in
  71     kw)
  72       at_work=true
  73       ;;
  74     kd|frodo)
  75       at_home=true
  76       ;;
  77     x2|x3|sy|so)
  78       if [[ $(dig +short @10.2.0.1 -x 10.2.0.2 2>&1 ||:) == kd.b8.nz. ]] \
  79            && ip n show 10.2.0.1 | grep . &>/dev/null; then
  80         at_home=true
  81       elif ping -q -c1 -w1 hal.office.fsf.org &>/dev/null \
  82           && ip n show 192.168.0.26 | grep . &>/dev/null; then
  83         at_work=true
  84       fi
  85       ;;
  86   esac
  87 }
  88
  89 exit-if-no-default-targets() {
  90   if ! $force && [[ $HOSTNAME != "$MAIL_HOST" ]]; then
  91     echo "MAIL_HOST=$MAIL_HOST, nothing to do"
  92     mexit 0
  93   fi
  94   case $HOSTNAME in
  95     kw|kd|frodo|x2|x3|sy|so) : ;;
  96     *)
  97       die "error: no default targets for this host, use -t"
  98       ;;
  99   esac
 100 }
 101
 102 add-x3-target() {
 103   # main work machine
 104   if ping -q -c1 -w1 x3.office.fsf.org &>/dev/null; then
 105     targets+=(x3.office.fsf.org)
 106   elif ping -q -c1 -w1 $h.b8.nz &>/dev/null; then
 107     # in case we took it home
 108     targets+=(x3.b8.nz)
 109   elif ping -q -c1 -w1 ${h}w.b8.nz &>/dev/null; then
 110     targets+=(x3w.b8.nz)
 111   else
 112     targets+=(x3wg.b8.nz)
 113   fi
 114 }
 115
 116 add-wireless-target() {
 117   local host
 118   if [[ ! $1 ]]; then
 119     set -- $h
 120   fi
 121   for host; do
 122     # c = cabled, w = wireless
 123     if ping -q -c1 -w1 ${host}c.b8.nz &>/dev/null; then
 124       targets+=(${host}c.b8.nz)
 125     elif ping -q -c1 -w1 $host.b8.nz &>/dev/null; then
 126       targets+=($host.b8.nz)
 127     elif ping -q -c1 -w1 ${host}w.b8.nz &>/dev/null; then
 128       targets+=(${host}w.b8.nz)
 129     fi
 130   done
 131 }
 132
 133 qconf() {
 134   case $sub in
 135     q)
 136       # q has sensitive data i dont want to backup for so long
 137       cat >>/etc/btrbk$conf_suf.conf <<EOF
 138 snapshot_preserve $q_preserve
 139 snapshot_preserve_min 2h
 140 snapshot_dir btrbk
 141 target_preserve $q_preserve
 142 target_preserve_min 2h
 143 EOF
 144       ;;
 145   esac
 146
 147 }
 148
 149
 150 # latest $MAIL_HOST
 151 if [[ -e /b/bash_unpublished/source-state ]]; then
 152   source /b/bash_unpublished/source-state
 153 fi
 154
 155 # note q is owned by root:1000
 156
 157 mountpoints=()
 158
 159 rsync_mountpoint=/q
 160
 161 ret=0
 162 # default options
 163 conf_only=false
 164 dry_run=false # mostly for testing
 165 rate_limit=no
 166 verbose=true; verbose_arg=-v
 167 force=false
 168 if [[ $INVOCATION_ID ]]; then
 169   # INVOCATION_ID means running as a systemd service. we cant show progress in this case,
 170   # but if we pass the arg, it will insert mbuffer into the command.
 171   progress_arg=
 172 else
 173   progress_arg="--progress"
 174 fi
 175 incremental_strict=false
 176 pull_reexec=false
 177
 178 default_args_file=/etc/btrbk-run.conf
 179 if [[ -s $default_args_file ]]; then
 180   # shellcheck disable=SC2046 # we want word splitting
 181   set -- $(< $default_args_file) "$@"
 182   # i havent used this feature yet, so warn about it
 183   echo "$0: warning: default btrbk-run options set in $default_args_file (sleeping 5 seconds):"
 184   cat $default_args_file
 185   sleep 5
 186 fi
 187
 188 once_args_file=/etc/btrbk-run-once.conf
 189 if mv -f $once_args_file $once_args_file-tmp 2> >(sed '/No such file or directory/d'); then
 190   # shellcheck disable=SC2046 # we want word splitting
 191   set -- $(< $once_args_file-tmp) "$@"
 192   # i havent used this feature yet, so warn about it
 193   echo "$0: btrbk-run options set in $once_args_file:"
 194   cat $once_args_file-tmp
 195   rm -f $once_args_file-tmp
 196 fi
 197
 198
 199 targets=()
 200 early=false
 201 fast=false
 202 kd_spread=false
 203 check_installed=false
 204 orig_args=("$@")
 205 temp=$(getopt -l check-installed,fast,pull-reexec,help 23cefikl:m:npqrs:t:vh "$@") || usage 1
 206 eval set -- "$temp"
 207 while true; do
 208   case $1 in
 209     # for the rare case we want to run multiple instances at the same time
 210     -2) conf_suf=2 ;;
 211     -3) conf_suf=3 ;;
 212     # only creates the config file, does not run btrbk
 213     -c) conf_only=true ;;
 214     --check-installed)
 215       check_installed=true
 216       ;;
 217     # quit early, just btrbk, no extra remounting etc.
 218     -e) early=true ;;
 219     # avoids some default behaviors:
 220     # - no skipping hosts where xprintidle haven't been idle recently
 221     # - exit if we can't ssh to 1 or more hosts
 222     # - still set default hosts despite MAIL_HOST status
 223     -f) force=true ;;
 224     # skip various checks. when we run twice in a row for
 225     # switch mail-host, no need to repeat the same checks again.
 226     --fast) fast=true ;;
 227     -i) incremental_strict=true ;;
 228     # note this implies resume and -p because it is just meant to make
 229     # other hosts have the same snapshots, not do any expiry or new
 230     # backups.
 231     -k) kd_spread=true ;;
 232     # bytes per second, suffix k m g
 233     -l) rate_limit=$2; shift ;;
 234     # Comma separated mountpoints to backup. This has defaults set below.
 235     -m) IFS=, mountpoints=($2); unset IFS; shift ;;
 236     -n) dry_run=true ;;
 237     # preserve existing snapshots and backups
 238     -p) preserve_arg=-p ;;
 239     # internal option for rerunning under newer SOURCE_HOST version.
 240     --pull-reexec) pull_reexec=true;;
 241     # quiet
 242     -q) verbose=false; verbose_arg=; progress_arg= ;;
 243     # source host to receive a backup from
 244     -s)
 245       source=$2
 246       bbksource=$source
 247       if [[ $source == *:* ]]; then
 248         bbksource="[$source]"
 249       fi
 250       shift
 251       ;;
 252     # target hosts to send to. empty is valid for just doing local
 253     # snapshot. we have default hosts we will populate.
 254     -t) IFS=, targets=($2); unset IFS; shift ;;
 255     # verbose.
 256     -v) verbose=true; verbose_arg=-v ;;
 257     -h|--help) usage ;;
 258     --) shift; break ;;
 259     *) die "Internal error!" ;;
 260   esac
 261   shift
 262 done
 263
 264 cmd_arg="$1"
 265
 266
 267
 268 if ! $force && { $check_installed || [[ ! $source ]]; } ; then
 269   install_bin_files=(
 270     mount-latest-subvol
 271     check-subvol-stale
 272     btrbk-run
 273   )
 274   for f in ${install_bin_files[@]}; do
 275     if ! diff -q /a/bin/ds/$f /usr/local/bin/$f; then
 276       uninstalled-file-die $f
 277     fi
 278   done
 279   if ! diff -q /a/bin/bash-bear-trap/bash-bear /usr/local/lib/bash-bear; then
 280     uninstalled-file-die err
 281   fi
 282   if $check_installed; then
 283     exit 0
 284   fi
 285 fi
 286
 287
 288 if $kd_spread; then
 289   if [[ $cmd_arg && $cmd_arg != resume ]]; then
 290     die "dont pass -k without resume or empty run arg"
 291   fi
 292   if [[ $HOSTNAME == "$MAIL_HOST" ]]; then
 293     die "something went wrong, -k not meant to be run on MAIL_HOST"
 294   fi
 295   if [[ $HOSTNAME != kd ]]; then
 296     die "something went wrong, -k only meant to run on kd"
 297   fi
 298   cmd_arg=resume
 299   preserve_arg=-p
 300   add-wireless-target sy so
 301 fi
 302
 303 if [[ ! $cmd_arg ]]; then
 304   cmd_arg=run
 305 fi
 306
 307
 308 std_preserve="36h 14d 8w 24m"
 309 q_preserve="18h 14d 8w"
 310
 311 case $cmd_arg in
 312   run|resume) : ;;
 313
 314   # This works better than the normal archive command. We have to
 315   # specify the mount points, but that is what we are used to doing and
 316   # we prefer it. Another difference is that archive works recursively
 317   # and we don't care about that. Sometimes we may still want to run
 318   # btrbk archive, but it doesn't even use the config file, so just
 319   # run it directly, eg:
 320   # time s btrbk -v archive /mnt/r7/amy/boot/btrbk ssh://bo/mnt/boot2/btrbk
 321   archive)
 322     cmd_arg=resume
 323     std_preserve="999h 999d 999w 999m"
 324     q_preserve="$std_preserve"
 325     preserve_arg=-p
 326     ;;
 327   *) die "untested command arg" ;;
 328 esac
 329
 330 if (( $# > 1 )); then
 331   die: "only 1 nonoption arg is supported"
 332 fi
 333
 334 if [[ -v targets && $source ]]; then
 335   # note, this doesnt need to be the case, but
 336   # we would need to think about it.
 337   die "error: -t and -s are mutually exclusive"
 338 fi
 339
 340 ### end options parsing
 341
 342 # remove path from earlier version of btrbk
 343 rm -f /usr/sbin/btrbk
 344 # note, this still works as intended if there is no /usr/bin/btrbk
 345 if [[ /a/opt/btrbk/btrbk -nt /usr/bin/btrbk ]]; then
 346   if [[ -e /b/distro-functions/src/package-manager-abstractions ]]; then
 347     . /b/distro-functions/src/package-manager-abstractions
 348     pi asciidoctor
 349   fi
 350   cd /a/opt/btrbk
 351   m make install
 352   cd /
 353 fi
 354
 355 # TODO: i wonder if there should be an option to send to the default
 356 # targets, plus any given on the command line.
 357
 358
 359 at_work=false
 360 at_home=false
 361
 362
 363 # set default targets
 364 if [[ ! -v targets && ! $source ]]; then
 365   exit-if-no-default-targets
 366   set-location
 367   if $at_home; then
 368     if ! $kd_spread && [[ $HOSTNAME != x3 ]]; then
 369       add-x3-target
 370     fi
 371     if [[ $HOSTNAME != kd ]]; then
 372       targets+=(kd.b8.nz)
 373     fi
 374     wireless_home_hosts=(
 375       x2
 376       sy
 377       so
 378     )
 379     for h in ${wireless_home_hosts[@]}; do
 380       if [[ $HOSTNAME != "$h" ]]; then
 381         add-wireless-target
 382       fi
 383     done
 384   elif $at_work; then
 385     targets+=(b8.nz)
 386     for h in x2 x3 kw; do
 387       if [[ $HOSTNAME == "$h" ]]; then
 388         continue
 389       fi
 390       if ping -q -c1 -w1 $h.office.fsf.org &>/dev/null; then
 391         targets+=($h.office.fsf.org)
 392       fi
 393     done
 394   else
 395     targets+=(b8.nz)
 396   fi
 397 fi
 398
 399 if [[ ${mountpoints[0]} ]]; then
 400   for mp in ${mountpoints[@]}; do
 401     if [[ -e /nocow/btrfs-stale/$mp ]]; then
 402       die "error: $mp is stale, mount-latest-subvol first"
 403     fi
 404   done
 405 else
 406   # set default mountpoints
 407   if [[ ${targets[0]} == tp ]]; then
 408     prospective_mps=(/a)
 409   else
 410     case $HOSTNAME in
 411       *)
 412         prospective_mps=()
 413         if [[ $source ]]; then
 414           source_state="$(ssh $source 'cat /a/bin/bash_unpublished/source-state; echo source_host=$HOSTNAME')"
 415           eval "$source_state"
 416           # shellcheck disable=SC2154 # assigned in the above eval.
 417           if [[ $source_host == "$MAIL_HOST" ]]; then
 418             prospective_mps+=(/o)
 419           fi
 420           if [[ $source_host == "$HOST2" ]]; then
 421             prospective_mps+=(/a /qr /qd /q)
 422           fi
 423         else
 424           if [[ $HOSTNAME == "$MAIL_HOST" ]]; then
 425             prospective_mps+=(/o)
 426           fi
 427           if [[ $HOSTNAME == "$HOST2" ]]; then
 428             prospective_mps+=(/a /qr /qd /q)
 429           fi
 430           if $kd_spread; then
 431             prospective_mps=(/a /o /qr /qd /q)
 432           fi
 433         fi
 434         # note: put q last just in case its specific retention options were to
 435         # affect other config sections. I havent tested if that is the case.
 436         ;;
 437     esac
 438   fi
 439   for mp in ${prospective_mps[@]}; do # default mountpoints to sync
 440     if [[ -e /nocow/btrfs-stale/$mp ]]; then
 441       e "warning: $mp stale, not adding to default mountpoints"
 442       continue
 443     fi
 444     if awk '{print $2}' /etc/fstab | grep -xF $mp &>/dev/null; then
 445       mountpoints+=($mp)
 446     fi
 447   done
 448 fi
 449
 450 tmp=$(( ${#mountpoints[@]} == 0 ))
 451 if (( tmp )); then
 452   die didnt get mountpoint arg and had no defaults
 453 fi
 454
 455 ##### end command line parsing ########
 456
 457 #### begin pre-checks #####
 458
 459 # todo: this has a timing problem, since btrbk.timer could activate the service after this check.
 460 if ! $fast && [[ $source ]]; then
 461   if [[ $(ssh $source ps --no-headers -o comm 1) == systemd ]]; then
 462     status=$(ssh $source systemctl is-active btrbk.service) || : # normally returns 3
 463     case $status in
 464       inactive|failed) : ;;
 465       *)
 466         echo "$0: error: btrbk is running on source. exiting out of caution"
 467         mexit 1
 468     esac
 469   fi
 470 fi
 471
 472 if ! command -v btrbk &>/dev/null; then
 473   die "error: no btrbk binary found"
 474 fi
 475
 476 if ! $pull_reexec && [[ $source ]] && $pulla && ! $force ; then
 477   ssh root@$source btrbk-run --check-installed || exit 1
 478 fi
 479
 480 #### end pre-checks #####
 481
 482
 483
 484 mkdir -p /var/log/btrbk
 485 # The journal doesnt go back to my oldest backups, and I've found myself
 486 # wanting older logs. Not going to bother expiring old logs, since it is
 487 # fine if they go back years.
 488 log_path=/var/log/btrbk/$(date +%F_%T%:::z).log
 489 echo copying output to $log_path
 490 exec &> >(pee cat 'ts "%F %T"|dd of='$log_path' status=none')
 491
 492 # print some non-default opts
 493 if $verbose; then
 494   opts_show=()
 495   if ! $conf_only; then
 496     opts_show+=(conf_only=true)
 497   fi
 498   if ! $dry_run; then
 499     opts_show+=(dry_run=true)
 500   fi
 501   if [[ $rate_limit != no ]]; then
 502     opts_show+=("rate_limit=$rate_limit")
 503   fi
 504   if [[ $cmd_arg != run ]]; then
 505     opts_show+=(cmd_arg=$cmd_arg)
 506   fi
 507   if (( ${#opts_show[@]} >= 1 )); then
 508     first=true
 509     for opt in ${opts_show[@]}; do
 510       if $first; then
 511         printf "%s" "$opt"
 512         first=false
 513       else
 514         printf " %s" "$opt"
 515       fi
 516     done
 517     echo
 518   fi
 519 fi
 520
 521 if [[ -v targets ]]; then
 522   echo "targets: ${targets[*]}"
 523 fi
 524
 525 if [[ $source ]]; then
 526   echo "source: $source"
 527 fi
 528
 529 echo "mountpoints: ${mountpoints[*]}"
 530
 531
 532 # pull_reexec stops us from getting into an infinite loop if there is some
 533 # kind of weird problem
 534 pulla=false
 535 for m in "${mountpoints[@]}"; do
 536   if [[ $m == /a ]]; then
 537     pulla=true
 538     break
 539   fi
 540 done
 541
 542 if ! $pull_reexec && [[ $source ]] && $pulla ; then
 543   tmpf=$(mktemp)
 544   m rsync -ra $source:/usr/local/bin/{mount-latest-subvol,check-subvol-stale} /usr/local/bin
 545   m rsync -ra $source:/usr/local/lib/bash-bear /usr/local/lib
 546   m scp $source:/a/bin/distro-setup/btrbk-run $tmpf
 547   if ! diff -q $tmpf ${BASH_SOURCE[0]}; then
 548     e "found different version on host $source. reexecing"
 549     install -T $tmpf /usr/local/bin/btrbk-run
 550     m /usr/local/bin/btrbk-run --pull-reexec "${orig_args[@]}"
 551     mexit 0
 552   fi
 553 fi
 554
 555 # todo: check if we have no snapshots yet, because I always want to run
 556 # archive instead of run. Likely, I should give an error unless a cli
 557 # override is passed. perhaps check-subvol-stale could give the error.
 558 # see the error message "no snapshots found" in that file.
 559 if ! $fast; then
 560   # if our mountpoints are from stale snapshots,
 561   # it doesn't make sense to do a backup.
 562   m check-subvol-stale ${mountpoints[@]} || die "found stale mountpoints in ${mountpoints[*]}"
 563
 564   # for an initial run, btrbk requires the dir to exist.
 565   mkdir -p /mnt/{root,o}/btrbk
 566 fi
 567 local_zone=$(date +%z)
 568
 569 if [[ $source ]]; then
 570   if $fast; then
 571     zone=$local_zone
 572   else
 573     if ! zone=$(ssh root@$source date +%z); then
 574       if $conf_only; then
 575         echo "$0: warning: failed to ssh to root@$source"
 576       else
 577         die failed to ssh to root@$source
 578       fi
 579     fi
 580     if [[ $zone != "$local_zone" ]]; then
 581       die "error: dont confuse yourself with multiple time zones. $h has different timezone than localhost"
 582     fi
 583   fi
 584 else
 585
 586   sshable=()
 587   sshfail=()
 588   remote_str_cmd="mkdir -p /mnt/root/btrbk /mnt/o/btrbk && \
 589 date +%z && \
 590 df --output=size,pcent / | tail -n1"
 591
 592   for h in ${targets[@]}; do
 593     if $fast || $conf_only; then
 594       # Use some typical values in this case
 595       root_size=$(( 1024 * 1024 * 2000 )) #2tb
 596       percent_used=10
 597       zone=$(date +%z)
 598     elif remote_str=$(timeout -s 9 6 ssh root@$h "$remote_str_cmd"); then
 599       mapfile -t tmp_array <<<"$remote_str"
 600       zone="${tmp_array[0]}"
 601       IFS=" " read -r root_size percent_used <<<"${tmp_array[1]}"
 602       percent_used=${percent_used%%%}
 603
 604       tmp=$(( ${#tmp_array[@]} != 2 ))
 605       if (( tmp )); then
 606         die "error: didnt get 2 lines in test ssh to target $h. investigate"
 607       fi
 608       case $percent_used in
 609         [0-9]|[1-9][0-9]) : ;;
 610         *)
 611           die "error: didnt get percent disk use in test ssh to target $h. investigate"
 612           ;;
 613       esac
 614     else
 615       sshfail+=($h)
 616       continue
 617     fi
 618
 619     # we may be booted into a bootstrap fs or something
 620     min_root_kb=$(( 1024 * 1024 * 200 )) # 200 gb
 621     tmp=$(( root_size < min_root_kb ))
 622     if (( tmp )); then
 623       e "warning: $h: root_size=$root_size < 200gb, perhaps it is booted to bootstrap vol. skipping for now"
 624       continue
 625     fi
 626
 627     tmp=$(( percent_used >= 98 ))
 628     if (( tmp )); then
 629       die "error: filesystem on target $h is $percent_used % full"
 630     fi
 631
 632     # on sy, xprintidle is resetting every 12 seconds even when not
 633     # idle, i dunno why, instead we are checking if the screen is locked,
 634     # which is good enough.
 635     #
 636     # This is a separate ssh because the command can fail and thatis ok.
 637     if ! $force; then
 638       locked=false
 639       if lock_info=$(timeout -s 9 6 ssh $h DISPLAY=:0 xscreensaver-command -time); then
 640         if [[ $lock_info != *non-blanked* ]]; then
 641           locked=true
 642         fi
 643       else
 644         locked=true
 645       fi
 646       if ! $locked; then
 647         # Ignore this host. i sometimes use a non-main machine for
 648         # testing or web browsing, knowing that everything will be wiped
 649         # by the next backup, but I dont want it to happen as Im using
 650         # it from cronjob.
 651         e "warning: $h: seems to be actively in use, skipping for now"
 652         continue
 653       fi
 654     fi
 655     sshable+=($h)
 656     if [[ $zone != "$local_zone" ]]; then
 657       die "error: dont confuse yourself with multiple time zones. $h has different timezone than localhost"
 658     fi
 659   done
 660   if [[ ! ${sshable[*]} ]] || { $force && [[ ${sshfail[*]} ]]; }; then
 661     die "see skipped host warning above or sshfail hosts: ${sshfail[*]}"
 662   else
 663     if [[ ${sshfail[*]} ]]; then
 664       ret=1
 665       e "error: failed to ssh to ${sshfail[*]} but continuing with other hosts"
 666     fi
 667     targets=(${sshable[@]})
 668   fi
 669 fi
 670
 671
 672 cat >/etc/btrbk$conf_suf.conf <<EOF
 673 ssh_identity /q/root/h
 674 #ssh_identity /root/.ssh/home
 675
 676 # Just a guess that local7 is a good facility to pick.
 677 # It's a bit odd that the transaction log has to be logged to
 678 # a file or syslog, while other output is sent to std out.
 679 # The man does not mention a way for them to be together, but
 680 # I dunno if setting a log level like warn might also output
 681 # transaction info.
 682 transaction_syslog local7
 683
 684 # trying this out
 685 #stream_compress zstd
 686
 687 # so we only run one at a time
 688 lockfile                   /var/lock/btrbk$conf_suf.lock
 689
 690 # default format of short does not accomidate hourly preservation setting
 691 timestamp_format long-iso
 692
 693 # only make a snapshot if things have changed
 694 snapshot_create onchange
 695 # I could make this different from target_preserve,
 696 # if one disk had less space.
 697 # for now, keeping them equal.
 698 snapshot_preserve $std_preserve
 699 snapshot_preserve_min 6h
 700 snapshot_dir btrbk
 701 # so, total backups = ~58
 702 target_preserve $std_preserve
 703 target_preserve_min 6h
 704
 705 # it seems very likely that not doing this could result in clone source not found
 706 # errors, for example when expiry happens differently on different hosts,
 707 # also, as btrbk does by default, if a failed send happens, on the next run it
 708 # will warn about a stray subvolume, but then create a backup of a newer subvol
 709 # and use an older subvol as the parent.
 710 incremental_prefs sao:1
 711
 712 # if something fails and it's not obvious, try doing
 713 # btrbk -l debug -v dryrun
 714
 715 rate_limit $rate_limit
 716 EOF
 717
 718 if $incremental_strict; then
 719   cat >>/etc/btrbk$conf_suf.conf <<EOF
 720 incremental strict
 721 EOF
 722 fi
 723
 724
 725 # make /q be last
 726 mp_count=${#mountpoints[@]}
 727 for (( i=0; i < mp_count - 1 ; i++ )); do
 728   if [[ ${mountpoints[i]} == /q ]]; then
 729     unset "mountpoints[i]"
 730     mountpoints+=(/q)
 731   fi
 732 done
 733
 734
 735
 736 snap_list_cmds=()
 737 tg_snaps=()
 738 declare -A source_snaps
 739
 740 for m in ${mountpoints[@]}; do
 741   case $m in
 742     /o)
 743       vol=/mnt/o
 744       ;;
 745     *)
 746       vol=/mnt/root
 747       ;;
 748   esac
 749
 750   sub=${m#/}
 751   snap_list_cmds+=("echo $vol/btrbk/$sub.*;")
 752
 753   if [[ $source ]]; then
 754     tmp_a=($vol/btrbk/$sub.*)
 755     tg_snaps+=("${tmp_a[*]}")
 756     cat >>/etc/btrbk$conf_suf.conf <<EOF
 757 volume ssh://$bbksource$vol
 758 subvolume $sub
 759 EOF
 760     qconf
 761     cat >>/etc/btrbk$conf_suf.conf <<EOF
 762 target send-receive $vol/btrbk
 763 EOF
 764   else # we have targets
 765     for snap in "$vol/btrbk/$sub."*; do
 766       source_snaps[$snap]=t
 767     done
 768
 769     cat >>/etc/btrbk$conf_suf.conf <<EOF
 770 volume $vol
 771 subvolume $sub
 772 EOF
 773     qconf
 774     for tg in ${targets[@]}; do
 775       # handle ipv6
 776       if [[ $tg == *:* ]]; then
 777         tg="[$tg]"
 778       fi
 779       cat >>/etc/btrbk$conf_suf.conf <<EOF
 780 target send-receive ssh://$tg$vol/btrbk
 781 EOF
 782     done
 783   fi
 784 done
 785
 786 # Delete any subvols on the receiving host that don't exist on the
 787 # sending host.  Otherwise, the receiving host could have snapshots that
 788 # aren't on the sending side, and thus become odd leaf subvols, and then
 789 # btrbk could try to use them when we sync back, creating a weird tree
 790 # instead of linear parent/child relationship. Maybe this could lead to
 791 # a missing source subvol error, so lets avoid it.
 792
 793 get-orphan-tg-snaps() {
 794   orphan_tg_snaps=()
 795   for (( i=0; i < ${#mountpoints[@]}; i++ )); do
 796     orphan_start_count=${#orphan_tg_snaps[@]}
 797     tg_snap_count=0
 798     for tg_snap in ${tg_snaps[$i]}; do
 799       tg_snap_count=$(( tg_snap_count + 1 ))
 800       if [[ ! ${source_snaps[$tg_snap]} ]]; then
 801         orphan_tg_snaps+=("$tg_snap")
 802       fi
 803     done
 804     orphan_mp_count=$(( ${#orphan_tg_snaps[@]} - orphan_start_count ))
 805     # sanity checking
 806     tmp=$(( tg_snap_count  > 1 && tg_snap_count == orphan_mp_count ))
 807     if (( tmp )) ; then
 808       die "something went wrong checking orphans on $tg: for mountpoint ${mountpoints[$i]}, $orphan_mp_count"
 809     fi
 810   done
 811 }
 812
 813 if [[ $source ]]; then
 814   for snap in $(ssh root@$source "shopt -s nullglob; ${snap_list_cmds[*]}"); do
 815     source_snaps[$snap]=t
 816   done
 817   get-orphan-tg-snaps
 818   tmp=$(( ${#orphan_tg_snaps[*]} >= 1 ))
 819   if (( tmp )); then
 820     d btrfs sub del ${orphan_tg_snaps[*]}
 821   fi
 822 else # we have targets
 823   for tg in ${targets[@]}; do
 824     tmp_str=$(ssh root@$tg "shopt -s nullglob; ${snap_list_cmds[*]}")
 825     mapfile -t tg_snaps <<<"$tmp_str"
 826     get-orphan-tg-snaps
 827     tmp=$(( ${#orphan_tg_snaps[*]} >= 1 ))
 828     if (( tmp )); then
 829       d ssh root@$tg "btrfs sub del ${orphan_tg_snaps[*]}"
 830     fi
 831   done
 832 fi
 833
 834 # todo: umount first to ensure we don't have any errors
 835 # todo: do some kill fuser stuff to make umount more reliable
 836
 837
 838 if $conf_only; then
 839   mexit 0
 840 fi
 841
 842
 843
 844 if $dry_run; then
 845   m btrbk -c /etc/btrbk$conf_suf.conf -v -n $cmd_arg
 846   mexit 0
 847 fi
 848 # -q and just using the syslog option seemed nice,
 849 # but it doesn't show when a send has a parent and when it doesn't.
 850 m btrbk -c /etc/btrbk$conf_suf.conf $preserve_arg $verbose_arg $progress_arg $cmd_arg
 851
 852 if $early; then
 853   exit 0
 854 fi
 855
 856 # todo: tp not valid anymore.
 857 # if we have it, sync to systems which don't
 858 if mountpoint $rsync_mountpoint >/dev/null; then
 859   for tg in ${targets[@]}; do
 860     case $tg in
 861       tp)
 862         dirs=(/p/c/machine_specific/tp)
 863         for x in /p/c/machine_specific/*.hosts; do
 864           if grep -qxF $tg $x; then
 865             dirs+=(${x%.hosts})
 866           fi
 867         done
 868         m rsync -aSAXPH --specials --devices --delete --relative ${dirs[@]} root@$tg:/
 869         ;;
 870     esac
 871   done
 872 fi
 873
 874 subvols=()
 875 for mp in "${mountpoints[@]}"; do
 876   subvols+=("${mp##*/}")
 877 done
 878 if [[ $source ]]; then
 879   m mount-latest-subvol "${subvols[@]}"
 880 else
 881   for tg in ${targets[@]}; do
 882     m /a/exe/mount-latest-remote "$tg" "${subvols[@]}" || ret=$?
 883   done
 884 fi
 885
 886 # todo, we get hostnames earlier, reuse that.
 887 if [[ $ret == 0 ]]; then
 888   for tg in ${targets[@]}; do
 889     h=$(ssh $tg hostname)
 890     if [[ $h == kd && $HOSTNAME == x3 && $HOSTNAME == "$MAIL_HOST" ]]; then
 891       m ssh root@$tg 'btrbk-spread-wrap &>/dev/null </dev/null &'
 892     fi
 893     rsync --mkpath -a -f"- */" -f"+ *" /var/log/btrbk/ root@$tg:/var/log/btrbk/$tg
 894     cmd=/usr/local/bin/mail-backup-clean
 895     ssh root@$tg "if test -x $cmd; then $cmd; fi"
 896   done
 897   if [[ $source ]]; then
 898     rsync --mkpath -a -f"- */" -f"+ *" $source:/var/log/btrbk/ /var/log/btrbk/$source
 899   fi
 900 fi
 901
 902 mexit $ret
 903
 904 # todo: move variable data we don't care about backing up
 905 # to /nocow and symlink it.
 906
 907
 908 # background on btrbk timezones. with short/long, timestamps use local time.
 909 # for long, if your local time moves backwards, by moving timezones or
 910 # for an hour when daylight savings changes it, you will temporarily get
 911 # a more aggressive retention policy for the overlapping period, and
 912 # vice versa for the opposite timezone move. The alternative is using
 913 # long-iso, which puts timezone info into the timestamp, which means
 914 # that instead of shifting time, you shift the start of day/week/month
 915 # which is used for retention to your new local time, which means for
 916 # example, if you moved forward by 8 hours, the daily/weekly/monthly
 917 # retention will be 8 hours more aggressive since midnight is at a new
 918 # time, unless you fake the timzeone using the TZ env variable.
 919 # However, in the short term, there will be no inconsistencies.
 920 # I don't see any problem with shifting when the day starts for
 921 # retention, so I'm using long-iso.
 922
 923 # note to create a long-iso timestamp: date +%Y%m%dT%H%M%S%z