iankelling.org Git - distro-setup/blob - btrbk-run

   1 #!/bin/bash
   2 # Copyright (C) 2016 Ian Kelling
   3
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16
  17 # todo: if we cancel in the middle of a btrfs send, then run again
  18 # immediately, the received subvolume doesn't get a Received UUID:
  19 # field, and we won't mount it. Need to figure out a solution that will
  20 # fix this.
  21
  22
  23 [[ $EUID == 0 ]] || exec sudo -E "${BASH_SOURCE[0]}" "$@"
  24
  25 source /usr/local/lib/err
  26
  27 usage() {
  28   cat <<'EOF'
  29 btrbk-run [OPTIONS]
  30 usually -t TARGET_HOST or -s SOURCE_HOST
  31
  32 Note, at source location, intentionally not executable, run and read
  33 install-my-scripts.
  34
  35 EOF
  36   echo "top of script file:"
  37   sed -n '1,/^[# ]*end command line/{p;b};q' "$0"
  38   exit $1
  39 }
  40
  41
  42
  43 pre="${0##*/}: "
  44 m() { if $verbose; then printf "$pre%s\n"  "$*"; fi;  "$@"; }
  45 e() { printf "$pre%s\n"  "$*"; }
  46 die() { printf "$pre%s\n" "$*" >&2; echo "exiting with status 1" >&2; exit 1; }
  47 mexit() { echo "$pre: exiting with status $1"; exit $1; }
  48
  49 # latest $MAIL_HOST
  50 if [[ -e /b/bash_unpublished/source-state ]]; then
  51   source /b/bash_unpublished/source-state
  52 fi
  53
  54 # note q is owned by root:1000
  55
  56 mountpoints=()
  57
  58 rsync_mountpoint=/q
  59
  60 ret=0
  61 # default options
  62 conf_only=false
  63 dry_run=false # mostly for testing
  64 rate_limit=no
  65 verbose=true; verbose_arg=-v
  66 progress_arg="--progress"
  67 incremental_strict=false
  68 pull_reexec=false
  69
  70 default_args_file=/etc/btrbk-run.conf
  71 if [[ -s $default_args_file ]]; then
  72   set -- $(< $default_args_file) "$@"
  73   # i havent used this feature yet, so warn about it
  74   echo "$0: warning: default btrbk-run options set in $default_args_file (sleeping 5 seconds):"
  75   cat $default_args_file
  76   sleep 5
  77 fi
  78
  79 marchive=false
  80 early=false
  81 cron=false
  82 orig_args=("$@")
  83 temp=$(getopt -l cron,pull-reexec,help ceil:m:npqrs:t:vh "$@") || usage 1
  84 eval set -- "$temp"
  85 while true; do
  86   case $1 in
  87     # some behaviors specific to running under cron:
  88     # - skip hosts where xprintidle haven't been idle recently
  89     # - if we can't ssh to 1 or more hosts, still do the rest
  90     # - if we aren't MAIL_HOST and no -m or -s, just exit
  91     --cron)
  92       cron=true
  93       pre=
  94       shift
  95       ;;
  96     # only creates the config file, does not run btrbk
  97     -c) conf_only=true; shift ;;
  98     # quit early, just btrbk, no extra remounting etc.
  99     -e) early=true; shift ;;
 100     -i) incremental_strict=true; shift ;;
 101     # bytes per second, suffix k m g
 102     -l) rate_limit=$2; shift 2 ;;
 103     # Comma separated mountpoints to backup. This has defaults set below.
 104     -m) IFS=, mountpoints=($2); unset IFS; shift 2 ;;
 105     -n) dry_run=true; dry_run_arg=-n; shift ;;
 106     # show progress
 107     -p) progress_arg="--progress"; shift ;;
 108     # internal option for rerunning under newer SOURCE_HOST version.
 109     --pull-reexec) pull_reexec=true; shift ;;
 110     # quiet
 111     -q) verbose=false; verbose_arg=; progress_arg=; shift ;;
 112     # source host to receive a backup from
 113     -s)
 114       source=$2
 115       bbksource=$source
 116       if [[ $source == *:* ]]; then
 117         bbksource="[$source]"
 118       fi
 119       shift 2
 120       ;;
 121     # target hosts to send to. empty is valid for just doing local
 122     # snapshot. we have default hosts we will populate.
 123     -t) IFS=, targets=($2); unset IFS; shift 2 ;;
 124     # verbose.
 125     -v) verbose=true; verbose_arg=-v; shift ;;
 126     -h|--help) usage ;;
 127     --) shift; break ;;
 128     *) die "Internal error!" ;;
 129   esac
 130 done
 131
 132 # only tested commands are resume and archive
 133 cmd_arg=${1:-run}
 134
 135
 136 std_preserve="18h 14d 8w 24m"
 137 q_preserve="18h 14d"
 138
 139 case $cmd_arg in
 140   run|resume|archive) : ;;
 141   marchive)
 142     marchive=true
 143     cmd=resume
 144     std_preserve="999h 999d 999w 999m"
 145     q_preserve="$std_preserve"
 146     preserve_arg=-p
 147     ;;
 148   *) die "untested command arg" ;;
 149 esac
 150
 151 if (( $# > 1 )); then
 152   die: "only 1 nonoption arg is supported"
 153 fi
 154
 155 if [[ -v targets && $source ]]; then
 156   # note, this doesnt need to be the case, but
 157   # we would need to think about it.
 158   die "error: -t and -s are mutually exclusive"
 159 fi
 160
 161 if $verbose; then
 162   printf "options: conf_only=%s\ndry_run=%s\nrate_limit=%s\nverbose=%s\ncmd_arg=%s" "$conf_only" "$dry_run" "$rate_limit" "$verbose" "$cmd_arg"
 163 fi
 164 ### end options parsing
 165
 166 declare -A vols
 167
 168 # remove path from earlier version of btrbk
 169 rm -f /usr/sbin/btrbk
 170 # note, this still works as intended if there is no /usr/bin/btrbk
 171 if [[ /a/opt/btrbk/btrbk -nt /usr/bin/btrbk ]]; then
 172   if [[ -e /b/distro-functions/src/package-manager-abstractions ]]; then
 173     . /b/distro-functions/src/package-manager-abstractions
 174     pi asciidoctor
 175   fi
 176   cd /a/opt/btrbk
 177   m make install
 178 fi
 179
 180 # TODO: i wonder if there should be an option to send to the default
 181 # targets, plus any given on the command line.
 182
 183
 184 amy=false
 185 # set default targets
 186 if [[ ! -v targets && ! $source ]]; then
 187   if [[ $HOSTNAME != "$MAIL_HOST" ]] && $cron ; then
 188     echo "MAIL_HOST=$MAIL_HOST, nothing to do"
 189     mexit 0
 190   else
 191     amy=true
 192   fi
 193
 194   at_work=false
 195
 196   targets=(frodo.b8.nz)
 197   case $HOSTNAME in
 198     x2|kw)
 199       at_work=true
 200       ;;&
 201     x2|x3|sy|bo)
 202       if ping -q -c1 -w1 hal.office.fsf.org \
 203           && ip n show 192.168.0.26 | grep . &>/dev/null; then
 204         at_work=true
 205       fi
 206       ;;&
 207     kw|x2|x3|sy|bo)
 208       if $at_work; then
 209         if ping -q -c1 -w1 iank.vpn.office.fsf.org &>/dev/null; then
 210           home=iank.vpn.office.fsf.org
 211         else
 212           home=i.b8.nz
 213         fi
 214       else
 215         home=b8.nz
 216       fi
 217       ;;&
 218     kw)
 219       targets+=($home x2.office.fsf.org)
 220       ;;
 221     x2|x3|sy|bo)
 222       targets+=($home)
 223       if $at_work; then
 224         targets+=(x2.office.fsf.org x2.b8.nz)
 225       else
 226         targets+=(x2wg.b8.nz)
 227       fi
 228       ;;
 229     kd)
 230       targets+=(x2wg.b8.nz x3.b8.nz)
 231       if ping -q -c1 -w1 sy.b8.nz &>/dev/null; then
 232         targets+=(sy.b8.nz)
 233       else
 234         targets+=(syw.b8.nz)
 235       fi
 236       ;;
 237     frodo)
 238       # no targets
 239       targets=()
 240       ;;
 241     *)
 242       die "error: no default targets for this host, use -t"
 243       ;;
 244   esac
 245 fi
 246
 247 if [[ -v targets ]]; then
 248   echo "targets: ${targets[*]}"
 249 fi
 250
 251 if [[ $source ]]; then
 252   echo "source: $source"
 253 fi
 254
 255 if [[ ${mountpoints[0]} ]]; then
 256   for mp in ${mountpoints[@]}; do
 257     if [[ -e /nocow/btrfs-stale/$mp ]]; then
 258       die "error: $mp is stale, mount-latest-subvol first"
 259     fi
 260   done
 261 else
 262   # set default mountpoints
 263   if [[ ${targets[0]} == tp ]]; then
 264     prospective_mps=(/a)
 265   else
 266     case $HOSTNAME in
 267       frodo)
 268         prospective_mps=(/i)
 269         ;;
 270       *)
 271         prospective_mps=()
 272         if [[ $source ]]; then
 273           source_state="$(ssh $source cat /a/bin/bash_unpublished/source-state)"
 274           eval "$source_state"
 275           source_host="$(ssh $source cat /etc/hostname)"
 276           if [[ $source_host == "$MAIL_HOST" ]]; then
 277             prospective_mps+=(/o)
 278           fi
 279         else
 280           if [[ $HOSTNAME == "$MAIL_HOST" ]]; then
 281             # HOST2 is really the mail host if it exists
 282             if [[ $HOST2 && $HOST2 != "$HOSTNAME" ]]; then
 283               echo "skipping /o because HOST2 is not us"
 284             else
 285               prospective_mps+=(/o)
 286             fi
 287           fi
 288         fi
 289         # note: put q last just in case its specific retention options were to
 290         # affect other config sections. I havent tested if that is the case.
 291         prospective_mps+=(/a /ar /qr /q)
 292         ;;
 293     esac
 294   fi
 295   for mp in ${prospective_mps[@]}; do # default mountpoints to sync
 296     if [[ -e /nocow/btrfs-stale/$mp ]]; then
 297       e "warning: $mp stale, not adding to default mountpoints"
 298       continue
 299     fi
 300     if awk '{print $2}' /etc/fstab | grep -xF $mp &>/dev/null; then
 301       mountpoints+=($mp)
 302     fi
 303   done
 304 fi
 305
 306 echo "mountpoints: ${mountpoints[*]}"
 307
 308 ##### end command line parsing ########
 309
 310 if [[ $source ]]; then
 311   if [[ $(ssh $source ps --no-headers -o comm 1) == systemd ]]; then
 312     status=$(ssh $source systemctl is-active btrbk.service) || : # normally returns 3
 313     case $status in
 314       inactive|failed) : ;;
 315       *)
 316         echo "$0: error: cron btrbk is running on source. exiting out of caution"
 317         mexit 1
 318     esac
 319   fi
 320 fi
 321
 322 # pull_reexec stops us from getting into an infinite loop if there is some
 323 # kind of weird problem
 324 pulla=false
 325 for m in "${mountpoints[@]}"; do
 326   if [[ $m == /a ]]; then
 327     pulla=true
 328     break
 329   fi
 330 done
 331 if ! $pull_reexec && [[ $source ]] && $pulla ; then
 332   tmpf=$(mktemp)
 333   m rsync -ra $source:/usr/local/bin/{mount-latest-subvol,check-subvol-stale} /usr/local/bin
 334   m rsync -ra $source:/usr/local/lib/err /usr/local/lib
 335   m scp $source:/a/bin/distro-setup/btrbk-run $tmpf
 336   if ! diff -q $tmpf ${BASH_SOURCE[0]}; then
 337     e "found different version on host $source. reexecing"
 338     install -T $tmpf /usr/local/bin/btrbk-run
 339     m /usr/local/bin/btrbk-run --pull-reexec "${orig_args[@]}"
 340     mexit 0
 341   fi
 342 fi
 343
 344
 345 if ! command -v btrbk &>/dev/null; then
 346   die "error: no btrbk binary found"
 347 fi
 348 # if our mountpoints are from stale snapshots,
 349 # it doesn't make sense to do a backup.
 350 check-subvol-stale ${mountpoints[@]} || die "found stale mountpoints in ${mountpoints[*]}"
 351
 352 # for an initial run, btrbk requires the dir to exist.
 353 mkdir -p /mnt/{root,o}/btrbk
 354 local_zone=$(date +%z)
 355
 356 if [[ $source ]]; then
 357   if ! zone=$(ssh root@$source date +%z); then
 358     die failed to ssh to root@$source
 359   fi
 360   if [[ $zone != "$local_zone" ]]; then
 361     die "error: dont confuse yourself with multiple time zones. $h has different timezone than localhost"
 362   fi
 363
 364 else
 365
 366   sshable=()
 367   sshfail=()
 368   min_idle_ms=$((1000 * 60 * 15))
 369   for h in ${targets[@]}; do
 370     if remote_info=( $(timeout -s 9 6 ssh root@$h "mkdir -p /mnt/root/btrbk /mnt/o/btrbk && date +%z && df --output=size,pcent / | tail -n1") ); then
 371       zone=${remote_info[0]}
 372       root_size=${remote_info[1]}
 373       percent_used=${remote_info[2]%%%}
 374
 375       if (( ${#remote_info[@]} != 3 )); then
 376         die "error: didnt get 3 fields in test ssh to target $h. investigate"
 377       fi
 378
 379
 380       # we may be booted into a bootstrap fs or something
 381       min_root_kb=$(( 1024 * 1024 * 200 )) # 200 gb
 382       if (( root_size < min_root_kb )); then
 383         continue
 384       fi
 385
 386       if (( percent_used >= 98 )); then
 387         die "error: filesystem on target $h is $percent_used % full"
 388       fi
 389
 390       # This is a separate ssh because xprintidle can fail and thats ok.
 391       if $cron && idle_ms=$(timeout -s 9 6 ssh $h DISPLAY=:0 xprintidle); then
 392         if (( idle_ms < min_idle_ms )); then
 393
 394           # Ignore this host. i sometimes use a non-main machine for
 395           # testing or web browsing, knowing that everything will be wiped
 396           # by the next backup, but I dont want it to happen as Im using
 397           # it from cronjob.
 398           e "warning: $h: active X session in the last 15 minutes, skipping for now"
 399           continue
 400         fi
 401       fi
 402       sshable+=($h)
 403       if [[ $zone != "$local_zone" ]]; then
 404         die "error: dont confuse yourself with multiple time zones. $h has different timezone than localhost"
 405       fi
 406     else
 407       sshfail+=($h)
 408     fi
 409   done
 410   if [[ ! ${sshable[*]} ]] || { ! $cron && [[ ${sshfail[*]} ]]; }; then
 411     die "failed to ssh to hosts: ${sshfail[*]}"
 412   else
 413     if [[ ${sshfail[*]} ]]; then
 414       ret=1
 415       e "error: failed to ssh to ${sshfail[*]} but continuing with other hosts"
 416     fi
 417     targets=(${sshable[@]})
 418   fi
 419 fi
 420
 421
 422 cat >/etc/btrbk.conf <<EOF
 423 ssh_identity /q/root/h
 424 #ssh_identity /root/.ssh/home
 425
 426 # Just a guess that local7 is a good facility to pick.
 427 # It's a bit odd that the transaction log has to be logged to
 428 # a file or syslog, while other output is sent to std out.
 429 # The man does not mention a way for them to be together, but
 430 # I dunno if setting a log level like warn might also output
 431 # transaction info.
 432 transaction_syslog local7
 433
 434 # trying this out
 435 stream_compress zstd
 436
 437 # so we only run one at a time
 438 lockfile                   /var/lock/btrbk.lock
 439
 440 # default format of short does not accomidate hourly preservation setting
 441 timestamp_format long-iso
 442
 443 # only make a snapshot if things have changed
 444 snapshot_create onchange
 445 # I could make this different from target_preserve,
 446 # if one disk had less space.
 447 # for now, keeping them equal.
 448 snapshot_preserve $std_preserve
 449 snapshot_preserve_min 2h
 450 snapshot_dir btrbk
 451 # so, total backups = ~58
 452 target_preserve $std_preserve
 453 target_preserve_min 2h
 454
 455 # if something fails and it's not obvious, try doing
 456 # btrbk -l debug -v dryrun
 457
 458 rate_limit $rate_limit
 459 EOF
 460
 461 if $incremental_strict; then
 462   cat >>/etc/btrbk.conf <<EOF
 463 incremental strict
 464 EOF
 465 fi
 466
 467 for m in ${mountpoints[@]}; do
 468   case $m in
 469     /o)
 470       vol=/mnt/o
 471       vols[$vol]=t
 472       ;;
 473     *)
 474       vol=/mnt/root
 475       vols[$vol]=t
 476       ;;
 477   esac
 478
 479   sub=${m#/}
 480   if [[ $source ]]; then
 481     cat >>/etc/btrbk.conf <<EOF
 482 volume ssh://$bbksource$vol
 483 subvolume $sub
 484 target send-receive $vol/btrbk
 485 EOF
 486   fi
 487   if (( ${#targets[@]} )); then
 488     cat >>/etc/btrbk.conf <<EOF
 489 volume $vol
 490 subvolume $sub
 491 EOF
 492     case $sub in
 493       q)
 494         # q has sensitive data i dont want to backup for so long
 495         cat >>/etc/btrbk.conf <<EOF
 496 snapshot_preserve $std_preserve
 497 snapshot_preserve_min 2h
 498 snapshot_dir btrbk
 499 target_preserve $std_preserve
 500 target_preserve_min 2h
 501 EOF
 502         ;;
 503     esac
 504     for tg in ${targets[@]}; do
 505       # handle ipv6
 506       if [[ $tg == *:* ]]; then
 507         tg="[$tg]"
 508       fi
 509       cat >>/etc/btrbk.conf <<EOF
 510 target send-receive ssh://$tg$vol/btrbk
 511 EOF
 512     done
 513   fi
 514 done
 515
 516 # if $amy; then
 517 #   # to manually backup amy,
 518 #   # bbk -e -s amy -m root_ubuntubionic
 519 #   cat >>/etc/btrbk.conf <<'EOF'
 520 # volume ssh://amy/mnt/root
 521 # subvolume root_ubuntubionic
 522 # target send-receive /mnt/root/btrbk
 523 # EOF
 524 # fi
 525
 526 # todo: umount first to ensure we don't have any errors
 527 # todo: do some kill fuser stuff to make umount more reliable
 528
 529
 530 if $conf_only; then
 531   mexit 0
 532 fi
 533
 534
 535
 536 if $dry_run; then
 537   m btrbk -v -n $cmd_arg
 538   mexit 0
 539 elif [[ $cmd_arg == archive ]]; then
 540   if [[ $source ]]; then
 541     for vol in ${!vols[@]}; do
 542       m btrbk $verbose_arg $progress_arg $cmd_arg ssh://$source$vol $vol
 543     done
 544   else
 545     for tg in ${targets[@]}; do
 546       for vol in ${!vols[@]}; do
 547         m btrbk $verbose_arg $progress_arg $cmd_arg $vol ssh://$tg$vol
 548       done
 549     done
 550   fi
 551   mexit 0
 552 fi
 553 # -q and just using the syslog option seemed nice,
 554 # but it doesn't show when a send has a parent and when it doesn't.
 555 m btrbk $preserve_arg $verbose_arg $progress_arg $cmd_arg
 556
 557 # todo: tp not valid anymore.
 558 # if we have it, sync to systems which don't
 559 if mountpoint $rsync_mountpoint >/dev/null; then
 560   for tg in ${targets[@]}; do
 561     case $tg in
 562       tp)
 563         dirs=(/p/c/machine_specific/tp)
 564         for x in /p/c/machine_specific/*.hosts; do
 565           if grep -qxF $tg $x; then
 566             dirs+=(${x%.hosts})
 567           fi
 568         done
 569         m rsync -aSAXPH --specials --devices --delete --relative ${dirs[@]} root@$tg:/
 570         ;;
 571     esac
 572   done
 573 fi
 574
 575 if [[ $source ]]; then
 576   m mount-latest-subvol
 577 else
 578   m /a/exe/mount-latest-remote ${targets[@]}
 579 fi
 580
 581 if [[ $ret == 0 ]]; then
 582   for tg in ${targets[@]}; do
 583     :
 584     #ssh root@$tg /a/exe/mail-backup-clean
 585   done
 586 fi
 587
 588 mexit $ret
 589
 590 # todo: move variable data we don't care about backing up
 591 # to /nocow and symlink it.
 592
 593
 594 # background on btrbk timezones. with short/long, timestamps use local time.
 595 # for long, if your local time moves backwards, by moving timezones or
 596 # for an hour when daylight savings changes it, you will temporarily get
 597 # a more aggressive retention policy for the overlapping period, and
 598 # vice versa for the opposite timezone move. The alternative is using
 599 # long-iso, which puts timezone info into the timestamp, which means
 600 # that instead of shifting time, you shift the start of day/week/month
 601 # which is used for retention to your new local time, which means for
 602 # example, if you moved forward by 8 hours, the daily/weekly/monthly
 603 # retention will be 8 hours more aggressive since midnight is at a new
 604 # time, unless you fake the timzeone using the TZ env variable.
 605 # However, in the short term, there will be no inconsistencies.
 606 # I don't see any problem with shifting when the day starts for
 607 # retention, so I'm using long-iso.
 608
 609 # note to create a long-iso timestamp: date +%Y%m%dT%H%M%S%z