iankelling.org Git - distro-setup/blob - btrbk-run

   1 #!/bin/bash
   2 # Copyright (C) 2016 Ian Kelling
   3
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16
  17 # todo: if we cancel in the middle of a btrfs send, then run again
  18 # immediately, the received subvolume doesn't get a Received UUID:
  19 # field, and we won't mount it. Need to figure out a solution that will
  20 # fix this.
  21
  22
  23 [[ $EUID == 0 ]] || exec sudo -E "${BASH_SOURCE[0]}" "$@"
  24
  25 source /usr/local/lib/err
  26
  27 usage() {
  28   cat <<'EOF'
  29 btrbk-run [OPTIONS] [run|resume|archive]
  30 usually -t TARGET_HOST or -s SOURCE_HOST
  31
  32 Note, at source location, intentionally not executable, run and read
  33 install-my-scripts.
  34
  35 EOF
  36   echo "top of script file:"
  37   sed -n '1,/^[# ]*end command line/{p;b};q' "$0"
  38   exit $1
  39 }
  40
  41
  42
  43 pre="${0##*/}: "
  44 m() { if $verbose; then printf "$pre%s\n"  "$*"; fi;  "$@"; }
  45 e() { printf "$pre%s\n"  "$*"; }
  46 die() { printf "$pre%s\n" "$*" >&2; echo "exiting with status 1" >&2; exit 1; }
  47 mexit() { echo "$pre: exiting with status $1"; exit $1; }
  48
  49 # latest $MAIL_HOST
  50 if [[ -e /b/bash_unpublished/source-state ]]; then
  51   source /b/bash_unpublished/source-state
  52 fi
  53
  54 # note q is owned by root:1000
  55
  56 mountpoints=()
  57
  58 rsync_mountpoint=/q
  59
  60 ret=0
  61 # default options
  62 conf_only=false
  63 dry_run=false # mostly for testing
  64 rate_limit=no
  65 verbose=true; verbose_arg=-v
  66 if [[ $INVOCATION_ID ]]; then
  67   # INVOCATION_ID means running as a systemd service. we cant show progress in this case,
  68   # but if we pass the arg, it will insert mbuffer into the command.
  69   progress_arg=
  70 else
  71   progress_arg="--progress"
  72 fi
  73 incremental_strict=false
  74 pull_reexec=false
  75
  76 default_args_file=/etc/btrbk-run.conf
  77 if [[ -s $default_args_file ]]; then
  78   set -- $(< $default_args_file) "$@"
  79   # i havent used this feature yet, so warn about it
  80   echo "$0: warning: default btrbk-run options set in $default_args_file (sleeping 5 seconds):"
  81   cat $default_args_file
  82   sleep 5
  83 fi
  84
  85 early=false
  86 cron=false
  87 orig_args=("$@")
  88 temp=$(getopt -l cron,pull-reexec,help ceil:m:npqrs:t:vh "$@") || usage 1
  89 eval set -- "$temp"
  90 while true; do
  91   case $1 in
  92     # some behaviors specific to running under cron:
  93     # - skip hosts where xprintidle haven't been idle recently
  94     # - if we can't ssh to 1 or more hosts, still do the rest
  95     # - if we aren't MAIL_HOST and no -m or -s, just exit
  96     --cron)
  97       cron=true
  98       pre=
  99       shift
 100       ;;
 101     # only creates the config file, does not run btrbk
 102     -c) conf_only=true; shift ;;
 103     # quit early, just btrbk, no extra remounting etc.
 104     -e) early=true; shift ;;
 105     -i) incremental_strict=true; shift ;;
 106     # bytes per second, suffix k m g
 107     -l) rate_limit=$2; shift 2 ;;
 108     # Comma separated mountpoints to backup. This has defaults set below.
 109     -m) IFS=, mountpoints=($2); unset IFS; shift 2 ;;
 110     -n) dry_run=true; dry_run_arg=-n; shift ;;
 111     # hide progress
 112     -p) progress_arg=; shift ;;
 113     # internal option for rerunning under newer SOURCE_HOST version.
 114     --pull-reexec) pull_reexec=true; shift ;;
 115     # quiet
 116     -q) verbose=false; verbose_arg=; progress_arg=; shift ;;
 117     # source host to receive a backup from
 118     -s)
 119       source=$2
 120       bbksource=$source
 121       if [[ $source == *:* ]]; then
 122         bbksource="[$source]"
 123       fi
 124       shift 2
 125       ;;
 126     # target hosts to send to. empty is valid for just doing local
 127     # snapshot. we have default hosts we will populate.
 128     -t) IFS=, targets=($2); unset IFS; shift 2 ;;
 129     # verbose.
 130     -v) verbose=true; verbose_arg=-v; shift ;;
 131     -h|--help) usage ;;
 132     --) shift; break ;;
 133     *) die "Internal error!" ;;
 134   esac
 135 done
 136
 137 cmd_arg=${1:-run}
 138
 139
 140 std_preserve="36h 14d 8w 24m"
 141 q_preserve="18h 14d"
 142
 143 case $cmd_arg in
 144   run|resume) : ;;
 145
 146   # This works better than the normal archive command. We have to
 147   # specify the mount points, but that is what we are used to doing and
 148   # we prefer it. Another difference is that archive works recursively
 149   # and we don't care about that. Sometimes we may still want to run
 150   # btrbk archive, but it doesn't even use the config file, so just
 151   # run it directly, eg:
 152   # time s btrbk -v archive /mnt/r7/amy/boot/btrbk ssh://bo/mnt/boot2/btrbk
 153   archive)
 154     cmd_arg=resume
 155     std_preserve="999h 999d 999w 999m"
 156     q_preserve="$std_preserve"
 157     preserve_arg=-p
 158     ;;
 159   *) die "untested command arg" ;;
 160 esac
 161
 162 if (( $# > 1 )); then
 163   die: "only 1 nonoption arg is supported"
 164 fi
 165
 166 if [[ -v targets && $source ]]; then
 167   # note, this doesnt need to be the case, but
 168   # we would need to think about it.
 169   die "error: -t and -s are mutually exclusive"
 170 fi
 171
 172 if $verbose; then
 173   printf "options: conf_only=%s\ndry_run=%s\nrate_limit=%s\nverbose=%s\ncmd_arg=%s" "$conf_only" "$dry_run" "$rate_limit" "$verbose" "$cmd_arg"
 174 fi
 175 ### end options parsing
 176
 177 # remove path from earlier version of btrbk
 178 rm -f /usr/sbin/btrbk
 179 # note, this still works as intended if there is no /usr/bin/btrbk
 180 if [[ /a/opt/btrbk/btrbk -nt /usr/bin/btrbk ]]; then
 181   if [[ -e /b/distro-functions/src/package-manager-abstractions ]]; then
 182     . /b/distro-functions/src/package-manager-abstractions
 183     pi asciidoctor
 184   fi
 185   cd /a/opt/btrbk
 186   m make install
 187 fi
 188
 189 # TODO: i wonder if there should be an option to send to the default
 190 # targets, plus any given on the command line.
 191
 192
 193
 194 kd_spread=false
 195 # set default targets
 196 if [[ ! -v targets && ! $source ]]; then
 197   if $cron; then
 198     if [[ $HOSTNAME != "$MAIL_HOST" ]]; then
 199       if [[ $HOSTNAME == kd && $MAIL_HOST = x2 ]]; then
 200         kd_spread=true
 201       else
 202         echo "MAIL_HOST=$MAIL_HOST, nothing to do"
 203         mexit 0
 204       fi
 205     fi
 206   fi
 207
 208   # x2 at home atm
 209   kd_spread=false
 210
 211   at_work=false
 212
 213   # todo, fix this up once frodo is back
 214   # targets=(frodo.b8.nz)
 215   case $HOSTNAME in
 216     kw)
 217       at_work=true
 218       ;;&
 219     x2|x3|sy|bo)
 220       if ping -q -c1 -w1 hal.office.fsf.org \
 221           && ip n show 192.168.0.26 | grep . &>/dev/null; then
 222         at_work=true
 223       fi
 224       ;;&
 225     kw|x2|x3|sy|bo)
 226       if $at_work; then
 227         if ping -q -c1 -w1 iank.vpn.office.fsf.org &>/dev/null; then
 228           home=iank.vpn.office.fsf.org
 229         else
 230           home=i.b8.nz
 231         fi
 232       else
 233         if ping -q -c1 -w1 b8.nz &>/dev/null; then
 234           home=b8.nz
 235         else
 236           home=i.b8.nz
 237         fi
 238       fi
 239       ;;&
 240     x2)
 241       targets+=($home)
 242       ;;
 243     kw)
 244       targets+=($home x2.office.fsf.org)
 245       ;;
 246     x3|sy|bo)
 247       targets+=($home)
 248       if $at_work; then
 249         targets+=(x2.office.fsf.org x2.b8.nz)
 250       else
 251         targets+=(x2wg.b8.nz)
 252       fi
 253       ;;
 254     kd)
 255       if ! $kd_spread; then
 256         if ping -q -c1 -w1 x2.office.fsf.org &>/dev/null; then
 257           targets+=(x2.office.fsf.org)
 258         else
 259           targets+=(x2wg.b8.nz)
 260         fi
 261       fi
 262       if ping -q -c1 -w1 sy.b8.nz &>/dev/null; then
 263         targets+=(sy.b8.nz)
 264       else
 265         targets+=(syw.b8.nz)
 266       fi
 267       if ping -q -c1 -w1 x3.b8.nz &>/dev/null; then
 268         targets+=(x3.b8.nz)
 269       else
 270         targets+=(x3w.b8.nz)
 271       fi
 272       ;;
 273     frodo)
 274       # no targets
 275       targets=()
 276       ;;
 277     *)
 278       die "error: no default targets for this host, use -t"
 279       ;;
 280   esac
 281 fi
 282
 283 if [[ -v targets ]]; then
 284   echo "targets: ${targets[*]}"
 285 fi
 286
 287 if [[ $source ]]; then
 288   echo "source: $source"
 289 fi
 290
 291 if [[ ${mountpoints[0]} ]]; then
 292   for mp in ${mountpoints[@]}; do
 293     if [[ -e /nocow/btrfs-stale/$mp ]]; then
 294       die "error: $mp is stale, mount-latest-subvol first"
 295     fi
 296   done
 297 else
 298   # set default mountpoints
 299   if [[ ${targets[0]} == tp ]]; then
 300     prospective_mps=(/a)
 301   else
 302     case $HOSTNAME in
 303       frodo)
 304         prospective_mps=(/i)
 305         ;;
 306       *)
 307         prospective_mps=()
 308         if [[ $source ]]; then
 309           source_state="$(ssh $source cat /a/bin/bash_unpublished/source-state)"
 310           eval "$source_state"
 311           source_host="$(ssh $source cat /etc/hostname)"
 312           if [[ $source_host == "$MAIL_HOST" ]]; then
 313             prospective_mps+=(/o)
 314           fi
 315           if [[ $source_host == "$HOST2" ]]; then
 316             prospective_mps+=(/a /ar /qr /q)
 317           fi
 318         else
 319           if [[ $HOSTNAME == "$MAIL_HOST" ]]; then
 320             prospective_mps+=(/o)
 321           fi
 322           if [[ $HOSTNAME == "$HOST2" ]]; then
 323             prospective_mps+=(/a /ar /qr /q)
 324           fi
 325         fi
 326         # note: put q last just in case its specific retention options were to
 327         # affect other config sections. I havent tested if that is the case.
 328         ;;
 329     esac
 330   fi
 331   for mp in ${prospective_mps[@]}; do # default mountpoints to sync
 332     if [[ -e /nocow/btrfs-stale/$mp ]]; then
 333       e "warning: $mp stale, not adding to default mountpoints"
 334       continue
 335     fi
 336     if awk '{print $2}' /etc/fstab | grep -xF $mp &>/dev/null; then
 337       mountpoints+=($mp)
 338     fi
 339   done
 340 fi
 341
 342 echo "mountpoints: ${mountpoints[*]}"
 343
 344 ##### end command line parsing ########
 345
 346 if [[ $source ]]; then
 347   if [[ $(ssh $source ps --no-headers -o comm 1) == systemd ]]; then
 348     status=$(ssh $source systemctl is-active btrbk.service) || : # normally returns 3
 349     case $status in
 350       inactive|failed) : ;;
 351       *)
 352         echo "$0: error: cron btrbk is running on source. exiting out of caution"
 353         mexit 1
 354     esac
 355   fi
 356 fi
 357
 358 # pull_reexec stops us from getting into an infinite loop if there is some
 359 # kind of weird problem
 360 pulla=false
 361 for m in "${mountpoints[@]}"; do
 362   if [[ $m == /a ]]; then
 363     pulla=true
 364     break
 365   fi
 366 done
 367 if ! $pull_reexec && [[ $source ]] && $pulla ; then
 368   tmpf=$(mktemp)
 369   m rsync -ra $source:/usr/local/bin/{mount-latest-subvol,check-subvol-stale} /usr/local/bin
 370   m rsync -ra $source:/usr/local/lib/err /usr/local/lib
 371   m scp $source:/a/bin/distro-setup/btrbk-run $tmpf
 372   if ! diff -q $tmpf ${BASH_SOURCE[0]}; then
 373     e "found different version on host $source. reexecing"
 374     install -T $tmpf /usr/local/bin/btrbk-run
 375     m /usr/local/bin/btrbk-run --pull-reexec "${orig_args[@]}"
 376     mexit 0
 377   fi
 378 fi
 379
 380
 381 if ! command -v btrbk &>/dev/null; then
 382   die "error: no btrbk binary found"
 383 fi
 384 # if our mountpoints are from stale snapshots,
 385 # it doesn't make sense to do a backup.
 386 check-subvol-stale ${mountpoints[@]} || die "found stale mountpoints in ${mountpoints[*]}"
 387
 388 # for an initial run, btrbk requires the dir to exist.
 389 mkdir -p /mnt/{root,o}/btrbk
 390 local_zone=$(date +%z)
 391
 392 if [[ $source ]]; then
 393   if ! zone=$(ssh root@$source date +%z); then
 394     if $conf_only; then
 395       echo "$0: warning: failed to ssh to root@$source"
 396     else
 397       die failed to ssh to root@$source
 398     fi
 399   fi
 400   if [[ $zone != "$local_zone" ]]; then
 401     die "error: dont confuse yourself with multiple time zones. $h has different timezone than localhost"
 402   fi
 403
 404 else
 405
 406   sshable=()
 407   sshfail=()
 408   min_idle_ms=$((1000 * 60 * 15))
 409   for h in ${targets[@]}; do
 410     if remote_info=( $(timeout -s 9 6 ssh root@$h "mkdir -p /mnt/root/btrbk /mnt/o/btrbk && date +%z && df --output=size,pcent / | tail -n1") ); then
 411       zone=${remote_info[0]}
 412       root_size=${remote_info[1]}
 413       percent_used=${remote_info[2]%%%}
 414
 415       if (( ${#remote_info[@]} != 3 )); then
 416         die "error: didnt get 3 fields in test ssh to target $h. investigate"
 417       fi
 418     elif $conf_only; then
 419       # Use some typical values in this case
 420       root_size=$(( 1024 * 1024 * 2000 )) #2tb
 421       percent_used=10
 422       zone=$(date +%z)
 423     else
 424       sshfail+=($h)
 425       continue
 426     fi
 427
 428     # we may be booted into a bootstrap fs or something
 429     min_root_kb=$(( 1024 * 1024 * 200 )) # 200 gb
 430     if (( root_size < min_root_kb )); then
 431       continue
 432     fi
 433
 434     if (( percent_used >= 98 )); then
 435       die "error: filesystem on target $h is $percent_used % full"
 436     fi
 437
 438     # This is a separate ssh because xprintidle can fail and thats ok.
 439     if $cron && idle_ms=$(timeout -s 9 6 ssh $h DISPLAY=:0 xprintidle); then
 440       if (( idle_ms < min_idle_ms )); then
 441
 442         # Ignore this host. i sometimes use a non-main machine for
 443         # testing or web browsing, knowing that everything will be wiped
 444         # by the next backup, but I dont want it to happen as Im using
 445         # it from cronjob.
 446         e "warning: $h: active X session in the last 15 minutes, skipping for now"
 447         continue
 448       fi
 449     fi
 450     sshable+=($h)
 451     if [[ $zone != "$local_zone" ]]; then
 452       die "error: dont confuse yourself with multiple time zones. $h has different timezone than localhost"
 453     fi
 454   done
 455   if [[ ! ${sshable[*]} ]] || { ! $cron && [[ ${sshfail[*]} ]]; }; then
 456     die "failed to ssh to hosts: ${sshfail[*]}"
 457   else
 458     if [[ ${sshfail[*]} ]]; then
 459       ret=1
 460       e "error: failed to ssh to ${sshfail[*]} but continuing with other hosts"
 461     fi
 462     targets=(${sshable[@]})
 463   fi
 464 fi
 465
 466
 467 cat >/etc/btrbk.conf <<EOF
 468 ssh_identity /q/root/h
 469 #ssh_identity /root/.ssh/home
 470
 471 # Just a guess that local7 is a good facility to pick.
 472 # It's a bit odd that the transaction log has to be logged to
 473 # a file or syslog, while other output is sent to std out.
 474 # The man does not mention a way for them to be together, but
 475 # I dunno if setting a log level like warn might also output
 476 # transaction info.
 477 transaction_syslog local7
 478
 479 # trying this out
 480 #stream_compress zstd
 481
 482 # so we only run one at a time
 483 lockfile                   /var/lock/btrbk.lock
 484
 485 # default format of short does not accomidate hourly preservation setting
 486 timestamp_format long-iso
 487
 488 # only make a snapshot if things have changed
 489 snapshot_create onchange
 490 # I could make this different from target_preserve,
 491 # if one disk had less space.
 492 # for now, keeping them equal.
 493 snapshot_preserve $std_preserve
 494 snapshot_preserve_min 2h
 495 snapshot_dir btrbk
 496 # so, total backups = ~58
 497 target_preserve $std_preserve
 498 target_preserve_min 2h
 499
 500 # if something fails and it's not obvious, try doing
 501 # btrbk -l debug -v dryrun
 502
 503 rate_limit $rate_limit
 504 EOF
 505
 506 if $incremental_strict; then
 507   cat >>/etc/btrbk.conf <<EOF
 508 incremental strict
 509 EOF
 510 fi
 511
 512 for m in ${mountpoints[@]}; do
 513   case $m in
 514     /o)
 515       vol=/mnt/o
 516       ;;
 517     *)
 518       vol=/mnt/root
 519       ;;
 520   esac
 521
 522   sub=${m#/}
 523   if [[ $source ]]; then
 524     cat >>/etc/btrbk.conf <<EOF
 525 volume ssh://$bbksource$vol
 526 subvolume $sub
 527 target send-receive $vol/btrbk
 528 EOF
 529   fi
 530   if (( ${#targets[@]} )); then
 531     cat >>/etc/btrbk.conf <<EOF
 532 volume $vol
 533 subvolume $sub
 534 EOF
 535     case $sub in
 536       q)
 537         # q has sensitive data i dont want to backup for so long
 538         cat >>/etc/btrbk.conf <<EOF
 539 snapshot_preserve $std_preserve
 540 snapshot_preserve_min 2h
 541 snapshot_dir btrbk
 542 target_preserve $std_preserve
 543 target_preserve_min 2h
 544 EOF
 545         ;;
 546     esac
 547     for tg in ${targets[@]}; do
 548       # handle ipv6
 549       if [[ $tg == *:* ]]; then
 550         tg="[$tg]"
 551       fi
 552       cat >>/etc/btrbk.conf <<EOF
 553 target send-receive ssh://$tg$vol/btrbk
 554 EOF
 555     done
 556   fi
 557 done
 558
 559 # todo: umount first to ensure we don't have any errors
 560 # todo: do some kill fuser stuff to make umount more reliable
 561
 562
 563 if $conf_only; then
 564   mexit 0
 565 fi
 566
 567
 568
 569 if $dry_run; then
 570   m btrbk -v -n $cmd_arg
 571   mexit 0
 572 fi
 573 # -q and just using the syslog option seemed nice,
 574 # but it doesn't show when a send has a parent and when it doesn't.
 575 m btrbk $preserve_arg $verbose_arg $progress_arg $cmd_arg
 576
 577 # todo: tp not valid anymore.
 578 # if we have it, sync to systems which don't
 579 if mountpoint $rsync_mountpoint >/dev/null; then
 580   for tg in ${targets[@]}; do
 581     case $tg in
 582       tp)
 583         dirs=(/p/c/machine_specific/tp)
 584         for x in /p/c/machine_specific/*.hosts; do
 585           if grep -qxF $tg $x; then
 586             dirs+=(${x%.hosts})
 587           fi
 588         done
 589         m rsync -aSAXPH --specials --devices --delete --relative ${dirs[@]} root@$tg:/
 590         ;;
 591     esac
 592   done
 593 fi
 594
 595 if [[ $source ]]; then
 596   m mount-latest-subvol
 597 else
 598   m /a/exe/mount-latest-remote ${targets[@]}
 599 fi
 600
 601 if [[ $ret == 0 ]]; then
 602   for tg in ${targets[@]}; do
 603     :
 604     #ssh root@$tg /a/exe/mail-backup-clean
 605   done
 606 fi
 607
 608 mexit $ret
 609
 610 # todo: move variable data we don't care about backing up
 611 # to /nocow and symlink it.
 612
 613
 614 # background on btrbk timezones. with short/long, timestamps use local time.
 615 # for long, if your local time moves backwards, by moving timezones or
 616 # for an hour when daylight savings changes it, you will temporarily get
 617 # a more aggressive retention policy for the overlapping period, and
 618 # vice versa for the opposite timezone move. The alternative is using
 619 # long-iso, which puts timezone info into the timestamp, which means
 620 # that instead of shifting time, you shift the start of day/week/month
 621 # which is used for retention to your new local time, which means for
 622 # example, if you moved forward by 8 hours, the daily/weekly/monthly
 623 # retention will be 8 hours more aggressive since midnight is at a new
 624 # time, unless you fake the timzeone using the TZ env variable.
 625 # However, in the short term, there will be no inconsistencies.
 626 # I don't see any problem with shifting when the day starts for
 627 # retention, so I'm using long-iso.
 628
 629 # note to create a long-iso timestamp: date +%Y%m%dT%H%M%S%z