fix btrbk timing issue. improve email stuff

[distro-setup] / btrbk-run

diff --git a/btrbk-run b/btrbk-run
index 8d6eb285f20eecdff391d99101482325d1712fc3..fa20061128281857e82fcfd444225cb147e4158e 100644 (file)
--- a/btrbk-run
+++ b/btrbk-run
@@ -16,8 +16,50 @@
 
 # todo: if we cancel in the middle of a btrfs send, then run again immediately, the received subvolume doesn't get a Received UUID: field, and we won't mount it. Need to figure out a solution that will fix this.
 
-set -eE -o pipefail
-trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR
+err-bash-trace() {
+  local -i argc_index=0 frame i start=${1:-0} max_indent=8 indent
+  local source
+  local extdebug=false
+  if [[ $(shopt -p extdebug) == *-s* ]]; then
+    extdebug=true
+  fi
+  for ((frame=0; frame < ${#FUNCNAME[@]}-1; frame++)); do
+    argc=${BASH_ARGC[frame]}
+    argc_index+=$argc
+    ((frame < start)) && continue
+    if (( ${#BASH_SOURCE[@]} > 1 )); then
+      source="${BASH_SOURCE[frame+1]}:${BASH_LINENO[frame]}:"
+    fi
+    indent=$((frame-start + 1))
+    indent=$((indent < max_indent ? indent : max_indent))
+    printf "%${indent}s↳%sin \`%s" '' "$source" "${FUNCNAME[frame]}"
+    if $extdebug; then
+      for ((i=argc_index-1; i >= argc_index-argc; i--)); do
+        printf " %s" "${BASH_ARGV[i]}"
+      done
+    fi
+    echo \'
+  done
+  return 0
+}
+err-catch() {
+  set -E; shopt -s extdebug
+  _err-trap() {
+    err=$?
+    exec >&2
+    set +x
+    echo "${BASH_SOURCE[1]}:${BASH_LINENO[0]}: \`$BASH_COMMAND' returned $err"
+    err-bash-trace 2
+    set -e # err trap does not work within an error trap
+    "${_errcatch_cleanup[@]:-:}" # note :-: is to be compatible with set -u
+    echo "$0: exiting with code $err"
+    exit $err
+  }
+  trap _err-trap ERR
+  set -o pipefail
+}
+err-catch
+
 
 [[ $EUID == 0 ]] || exec sudo -E "$BASH_SOURCE" "$@"
 
@@ -47,8 +89,8 @@ m() { if $verbose; then printf "$pre %s\n"  "$*"; fi;  "$@"; }
 die() { printf "$pre %s\n" "$*" >&2;  exit 1; }
 
 # latest $MAIL_HOST
-if [[ -e /b/bash_unpublished/source-semi-priv ]]; then
-  source /b/bash_unpublished/source-semi-priv
+if [[ -e /b/bash_unpublished/source-state ]]; then
+  source /b/bash_unpublished/source-state
 fi
 
 # note q is owned by root:1000
@@ -77,7 +119,7 @@ fi
 pre="${0##*/}:"
 cron=false
 orig_args=("$@")
-temp=$(getopt -l cpull-reexec,help cl:m:npqs:t:vh "$@") || usage 1
+temp=$(getopt -l cron,pull-reexec,help cl:m:npqs:t:vh "$@") || usage 1
 eval set -- "$temp"
 while true; do
   case $1 in
@@ -120,15 +162,21 @@ if $verbose; then
 fi
 ### end options parsing
 
+# TODO: i wonder if there should be an option to send to the default
+# targets, plus any given on the command line.
 
 # set default targets
-if [[ ! -v targets && ! $source && $HOSTNAME == "$MAIL_HOST" ]]; then
+if [[ ! -v targets && ! $source ]]; then
+  if [[ $HOSTNAME != "$MAIL_HOST" ]] && $cron ; then
+    echo "MAIL_HOST=$MAIL_HOST, nothing to do"
+    exit 0
+  fi
   case $HOSTNAME in
     kw|x2)
       if ping -q -c1 -w1 iank.vpn.office.fsf.org &>/dev/null; then
         home=iank.vpn.office.fsf.org
       else
-        home=$HOME_DOMAIN
+        home=b8.nz
       fi
       ;;&
     kw)
@@ -199,7 +247,14 @@ fi
 
 echo "mountpoints: ${mountpoints[*]}"
 
+##### end command line parsing ########
 
+if [[ $source ]]; then
+  if [[ $(ssh $source systemctl is-active btrbk.service) != inactive ]]; then
+    echo "$0: error: cron btrbk is running on source. exiting out of caution"
+    exit 1
+  fi
+fi
 
 # pull_reexec stops us from getting into an infinite loop if there is some
 # kind of weird problem
@@ -210,7 +265,7 @@ for m in "${mountpoints[@]}"; do
     break
   fi
 done
-if ! $pull_reexec && [[ $source ]] && ! $pulla ; then
+if ! $pull_reexec && [[ $source ]] && $pulla ; then
   tmpf=$(mktemp)
   scp $source:/a/bin/distro-setup/btrbk-run $tmpf
   if ! diff -q $tmpf $BASH_SOURCE; then
@@ -222,7 +277,6 @@ if ! $pull_reexec && [[ $source ]] && ! $pulla ; then
 fi
 
 
-##### end command line parsing ########
 
 
 
@@ -238,7 +292,7 @@ mkdir -p /mnt/root/btrbk
 local_zone=$(date +%z)
 
 if [[ $source ]]; then
-  if ! zone=$(ssh root$source date +%z); then
+  if ! zone=$(ssh root@$source date +%z); then
     die failed to ssh to root@$source
   fi
   if [[ $zone != $local_zone ]]; then
@@ -249,15 +303,23 @@ else
 
   sshable=()
   sshfail=()
+  min_idle_ms=$((1000 * 60 * 15))
   for h in ${targets[@]}; do
     if zone=$(ssh root@$h "mkdir -p /mnt/root/btrbk && date +%z"); then
+      if $cron && DISPLAY=:0 xprintidle; then
+        # This is a separate ssh because xprintidle can fail and thats ok.
+        # Ignore this host. i sometimes use a non-main machine for testing or web browsing, knowing that
+        # everything will be wiped by the next backup, but I dont want it to happen as Im using
+        # it from cronjob.
+        continue
+      fi
       sshable+=($h)
+      if [[ $zone != $local_zone ]]; then
+        die "error: dont confuse yourself with multiple time zones. $h has different timezone than localhost"
+      fi
     else
       sshfail+=($h)
     fi
-    if [[ $zone != $local_zone ]]; then
-      die "error: dont confuse yourself with multiple time zones. $h has different timezone than localhost"
-    fi
   done
   if [[ ! $sshable ]] || { ! $cron && [[ $sshfail ]]; }; then
     die "failed to ssh to hosts: ${sshfail[*]}"