X-Git-Url: https://iankelling.org/git/?a=blobdiff_plain;f=btrbk-run;h=93e8dab79322bbbe8e4bea0fb14419718512e4e7;hb=HEAD;hp=7a22d42c414bfb84f771418721a0f2e94e3b02e1;hpb=c56f26063f141a0d13ccad28981e697da135163b;p=distro-setup

diff --git a/btrbk-run b/btrbk-run
index 7a22d42..9bd6e9b 100644
--- a/btrbk-run
+++ b/btrbk-run
@@ -45,25 +45,53 @@ EOF
 }
 
 
+
 pre=btrbk-run
 
+
+
 script_name="${BASH_SOURCE[0]}"
 script_name="${script_name##*/}"
+
+
+log-setup() {
+  if [[ ! $log_path ]]; then
+    mkdir -p /var/log/btrbk
+    log_path=/var/log/btrbk/$(date +%F_%T%:::z).log
+  fi
+}
 d() {
   if $dry_run || $conf_only; then
     printf "$pre dry-run: %s\n"  "$*"
   else
-    printf "$pre running: %s\n"  "$*"
-    "$@"
+    log-setup
+    printf "$pre running: %s\n"  "$*" |& pee cat 'ts "%F %T" >>'$log_path
+    "$@" |& pee cat 'ts "%F %T" >>'$log_path
   fi
 }
 m() { if $verbose; then printf "$pre %s\n"  "$*"; fi;  "$@"; }
 e() { printf "$pre %s\n"  "$*"; }
+
+logq() {
+  local exit_code
+  exit_code=0
+  log-setup
+  printf "$pre running: %s\n"  "$*" | pee cat 'ts "%F %T" >>'$log_path
+  e logging to $log_path
+  "$@" |& ts "%F %T" >>$log_path || exit_code=$?
+  printf "$pre exit code:%s of %s\n" "$exit_code"  "$*" | pee cat 'ts "%F %T" >>'$log_path
+  if (( exit_code > 0 )); then
+    e "error: command exit code: $exit_code. exiting after tail -n50 $log_path"
+    tail -n50 $log_path
+    exit $exit_code
+  fi
+}
+
 die() { printf "$pre error: %s\n" "$*" >&2; echo "$pre exiting with status 1" >&2; exit 1; }
 mexit() { echo "$pre exiting with status $1"; exit $1; }
 
 uninstalled-file-die() {
-  die "uninstalled file $1. run install-my-scripts or rerun with -f"
+  die "file $1 is not latest. run install-my-scripts or rerun with -f"
 }
 
 set-location() {
@@ -74,9 +102,10 @@ set-location() {
     kd|frodo)
       at_home=true
       ;;
-    x2|x3|sy)
+    x2|x3|sy|so)
       if [[ $(dig +short @10.2.0.1 -x 10.2.0.2 2>&1 ||:) == kd.b8.nz. ]] \
            && ip n show 10.2.0.1 | grep . &>/dev/null; then
+        # note: logic duplicated in 11-iank
         at_home=true
       elif ping -q -c1 -w1 hal.office.fsf.org &>/dev/null \
           && ip n show 192.168.0.26 | grep . &>/dev/null; then
@@ -92,7 +121,7 @@ exit-if-no-default-targets() {
     mexit 0
   fi
   case $HOSTNAME in
-    kw|kd|frodo|x2|x3|sy) : ;;
+    kw|kd|frodo|x2|x3|sy|so) : ;;
     *)
       die "error: no default targets for this host, use -t"
       ;;
@@ -103,22 +132,31 @@ add-x3-target() {
   # main work machine
   if ping -q -c1 -w1 x3.office.fsf.org &>/dev/null; then
     targets+=(x3.office.fsf.org)
-  elif ping -q -c1 -w1 $h.b8.nz &>/dev/null; then
+  elif ping -q -c1 -w1 x3.b8.nz &>/dev/null; then
     # in case we took it home
     targets+=(x3.b8.nz)
-  elif ping -q -c1 -w1 ${h}w.b8.nz &>/dev/null; then
+  elif ping -q -c1 -w1 x3w.b8.nz &>/dev/null; then
     targets+=(x3w.b8.nz)
   else
     targets+=(x3wg.b8.nz)
   fi
 }
 
-add-wireless-target-h() {
-  if ping -q -c1 -w1 $h.b8.nz &>/dev/null; then
-    targets+=($h.b8.nz)
-  elif ping -q -c1 -w1 ${h}w.b8.nz &>/dev/null; then
-    targets+=(${h}w.b8.nz)
+add-wireless-target() {
+  local host
+  if [[ ! $1 ]]; then
+    set -- $h
   fi
+  for host; do
+    # c = cabled, w = wireless
+    if ping -q -c1 -w1 ${host}c.b8.nz &>/dev/null; then
+      targets+=(${host}c.b8.nz)
+    elif ping -q -c1 -w1 $host.b8.nz &>/dev/null; then
+      targets+=($host.b8.nz)
+    elif ping -q -c1 -w1 ${host}w.b8.nz &>/dev/null; then
+      targets+=(${host}w.b8.nz)
+    fi
+  done
 }
 
 qconf() {
@@ -154,7 +192,7 @@ ret=0
 conf_only=false
 dry_run=false # mostly for testing
 rate_limit=no
-verbose=true; verbose_arg=-v
+verbose=true; verbose_arg="-l trace"
 force=false
 if [[ $INVOCATION_ID ]]; then
   # INVOCATION_ID means running as a systemd service. we cant show progress in this case,
@@ -193,13 +231,17 @@ fast=false
 kd_spread=false
 check_installed=false
 orig_args=("$@")
-temp=$(getopt -l check-installed,fast,pull-reexec,help 23cefikl:m:npqrs:t:vh "$@") || usage 1
+temp=$(getopt -l check-installed,fast,pull-reexec,help 23acefikl:m:npqrs:t:vh "$@") || usage 1
 eval set -- "$temp"
 while true; do
   case $1 in
     # for the rare case we want to run multiple instances at the same time
     -2) conf_suf=2 ;;
     -3) conf_suf=3 ;;
+    -a)
+      # all moiuntpoints
+      mountpoints=(/a /o /qr /qd /q)
+      ;;
     # only creates the config file, does not run btrbk
     -c) conf_only=true ;;
     --check-installed)
@@ -244,7 +286,9 @@ while true; do
     # snapshot. we have default hosts we will populate.
     -t) IFS=, targets=($2); unset IFS; shift ;;
     # verbose.
-    -v) verbose=true; verbose_arg=-v ;;
+    -v)
+      verbose=true; verbose_arg="-l trace"
+      ;;
     -h|--help) usage ;;
     --) shift; break ;;
     *) die "Internal error!" ;;
@@ -268,7 +312,7 @@ if ! $force && { $check_installed || [[ ! $source ]]; } ; then
     fi
   done
   if ! diff -q /a/bin/bash-bear-trap/bash-bear /usr/local/lib/bash-bear; then
-    uninstalled-file-die err
+    uninstalled-file-die bash-bear
   fi
   if $check_installed; then
     exit 0
@@ -288,8 +332,7 @@ if $kd_spread; then
   fi
   cmd_arg=resume
   preserve_arg=-p
-  h=sy
-  add-wireless-target-h
+  add-wireless-target sy so
 fi
 
 if [[ ! $cmd_arg ]]; then
@@ -366,14 +409,15 @@ if [[ ! -v targets && ! $source ]]; then
     wireless_home_hosts=(
       x2
       sy
+      so
     )
     for h in ${wireless_home_hosts[@]}; do
       if [[ $HOSTNAME != "$h" ]]; then
-        add-wireless-target-h
+        add-wireless-target
       fi
     done
   elif $at_work; then
-    targets+=(i.b8.nz)
+    targets+=(b8.nz)
     for h in x2 x3 kw; do
       if [[ $HOSTNAME == "$h" ]]; then
         continue
@@ -383,7 +427,7 @@ if [[ ! -v targets && ! $source ]]; then
       fi
     done
   else
-    targets+=(i.b8.nz)
+    targets+=(b8.nz)
   fi
 fi
 
@@ -464,21 +508,24 @@ if ! command -v btrbk &>/dev/null; then
   die "error: no btrbk binary found"
 fi
 
+# pull_reexec stops us from getting into an infinite loop if there is some
+# kind of weird problem
+pulla=false
+for m in "${mountpoints[@]}"; do
+  if [[ $m == /a ]]; then
+    pulla=true
+    break
+  fi
+done
+
 if ! $pull_reexec && [[ $source ]] && $pulla && ! $force ; then
-  ssh root@$source btrbk-run --check-installed || exit 1
+  ssh root@$source btrbk-run --check-installed
 fi
 
 #### end pre-checks #####
 
 
 
-mkdir -p /var/log/btrbk
-# The journal doesnt go back to my oldest backups, and I've found myself
-# wanting older logs. Not going to bother expiring old logs, since it is
-# fine if they go back years.
-log_path=/var/log/btrbk/$(date +%F_%T%:::z).log
-echo copying output to $log_path
-exec &> >(pee cat 'ts "%F %T"|dd of='$log_path' status=none')
 
 # print some non-default opts
 if $verbose; then
@@ -500,6 +547,7 @@ if $verbose; then
     for opt in ${opts_show[@]}; do
       if $first; then
         printf "%s" "$opt"
+        first=false
       else
         printf " %s" "$opt"
       fi
@@ -508,26 +556,6 @@ if $verbose; then
   fi
 fi
 
-if [[ -v targets ]]; then
-  echo "targets: ${targets[*]}"
-fi
-
-if [[ $source ]]; then
-  echo "source: $source"
-fi
-
-echo "mountpoints: ${mountpoints[*]}"
-
-
-# pull_reexec stops us from getting into an infinite loop if there is some
-# kind of weird problem
-pulla=false
-for m in "${mountpoints[@]}"; do
-  if [[ $m == /a ]]; then
-    pulla=true
-    break
-  fi
-done
 
 if ! $pull_reexec && [[ $source ]] && $pulla ; then
   tmpf=$(mktemp)
@@ -543,6 +571,19 @@ if ! $pull_reexec && [[ $source ]] && $pulla ; then
 fi
 
 
+if [[ -v targets ]]; then
+  echo "targets: ${targets[*]}"
+fi
+if [[ $source ]]; then
+  echo "source: $source"
+fi
+echo "mountpoints: ${mountpoints[*]}"
+
+
+# todo: check if we have no snapshots yet, because I always want to run
+# archive instead of run. Likely, I should give an error unless a cli
+# override is passed. perhaps check-subvol-stale could give the error.
+# see the error message "no snapshots found" in that file.
 if ! $fast; then
   # if our mountpoints are from stale snapshots,
   # it doesn't make sense to do a backup.
@@ -572,13 +613,17 @@ else
 
   sshable=()
   sshfail=()
+  remote_str_cmd="mkdir -p /mnt/root/btrbk /mnt/o/btrbk && \
+date +%z && \
+df --output=size,pcent / | tail -n1"
+
   for h in ${targets[@]}; do
     if $fast || $conf_only; then
       # Use some typical values in this case
       root_size=$(( 1024 * 1024 * 2000 )) #2tb
       percent_used=10
       zone=$(date +%z)
-    elif remote_str=$(timeout -s 9 6 ssh root@$h "mkdir -p /mnt/root/btrbk /mnt/o/btrbk && date +%z && df --output=size,pcent / | tail -n1"); then
+    elif remote_str=$(timeout -s 9 6 ssh root@$h "$remote_str_cmd"); then
       mapfile -t tmp_array <<<"$remote_str"
       zone="${tmp_array[0]}"
       IFS=" " read -r root_size percent_used <<<"${tmp_array[1]}"
@@ -603,6 +648,7 @@ else
     min_root_kb=$(( 1024 * 1024 * 200 )) # 200 gb
     tmp=$(( root_size < min_root_kb ))
     if (( tmp )); then
+      e "warning: $h: root_size=$root_size < 200gb, perhaps it is booted to bootstrap vol. skipping for now"
       continue
     fi
 
@@ -640,7 +686,7 @@ else
     fi
   done
   if [[ ! ${sshable[*]} ]] || { $force && [[ ${sshfail[*]} ]]; }; then
-    die "failed to ssh to hosts: ${sshfail[*]}"
+    die "see skipped host warning above or sshfail hosts: ${sshfail[*]}"
   else
     if [[ ${sshfail[*]} ]]; then
       ret=1
@@ -655,14 +701,6 @@ cat >/etc/btrbk$conf_suf.conf <<EOF
 ssh_identity /q/root/h
 #ssh_identity /root/.ssh/home
 
-# Just a guess that local7 is a good facility to pick.
-# It's a bit odd that the transaction log has to be logged to
-# a file or syslog, while other output is sent to std out.
-# The man does not mention a way for them to be together, but
-# I dunno if setting a log level like warn might also output
-# transaction info.
-transaction_syslog local7
-
 # trying this out
 #stream_compress zstd
 
@@ -692,7 +730,7 @@ target_preserve_min 6h
 incremental_prefs sao:1
 
 # if something fails and it's not obvious, try doing
-# btrbk -l debug -v dryrun
+# btrbk -l trace -v dryrun
 
 rate_limit $rate_limit
 EOF
@@ -829,7 +867,7 @@ if $dry_run; then
 fi
 # -q and just using the syslog option seemed nice,
 # but it doesn't show when a send has a parent and when it doesn't.
-m btrbk -c /etc/btrbk$conf_suf.conf $preserve_arg $verbose_arg $progress_arg $cmd_arg
+logq btrbk -c /etc/btrbk$conf_suf.conf $preserve_arg $verbose_arg $progress_arg $cmd_arg
 
 if $early; then
   exit 0
@@ -858,10 +896,10 @@ for mp in "${mountpoints[@]}"; do
   subvols+=("${mp##*/}")
 done
 if [[ $source ]]; then
-  m mount-latest-subvol "${subvols[@]}"
+  d mount-latest-subvol "${subvols[@]}"
 else
   for tg in ${targets[@]}; do
-    m /a/exe/mount-latest-remote "$tg" "${subvols[@]}" || ret=$?
+    d /a/exe/mount-latest-remote "$tg" "${subvols[@]}" || ret=$?
   done
 fi
 
@@ -870,10 +908,11 @@ if [[ $ret == 0 ]]; then
   for tg in ${targets[@]}; do
     h=$(ssh $tg hostname)
     if [[ $h == kd && $HOSTNAME == x3 && $HOSTNAME == "$MAIL_HOST" ]]; then
-      m ssh root@$tg 'btrbk-spread-wrap &>/dev/null </dev/null &'
+      d ssh root@$tg 'btrbk-spread-wrap &>/dev/null </dev/null &'
     fi
     rsync --mkpath -a -f"- */" -f"+ *" /var/log/btrbk/ root@$tg:/var/log/btrbk/$tg
-    ssh root@$tg /usr/local/bin/mail-backup-clean
+    cmd=/usr/local/bin/mail-backup-clean
+    ssh root@$tg "if test -x $cmd; then $cmd; fi"
   done
   if [[ $source ]]; then
     rsync --mkpath -a -f"- */" -f"+ *" $source:/var/log/btrbk/ /var/log/btrbk/$source