From: Ian Kelling <iank@fsf.org>
Date: Wed, 15 May 2019 19:20:15 +0000 (-0400)
Subject: btrbk improvements for cron
X-Git-Url: https://iankelling.org/git/?p=distro-setup;a=commitdiff_plain;h=93db0c16393fe75efef3167c1d2b332f50387c23

btrbk improvements for cron

initial dir for ansible
remove blank leaf volumes
enable btrbk cronjob
less verbose btrbk on cron
fix ssh host keys permission
---

diff --git a/.gitignore b/.gitignore
index c9344e7..0894806 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,3 +2,4 @@
 /old-unused
 /.emacs.d
 /Arduino
+/a/roles/prom
diff --git a/a/ansible.cfg b/a/ansible.cfg
new file mode 100644
index 0000000..946488b
--- /dev/null
+++ b/a/ansible.cfg
@@ -0,0 +1,9 @@
+[defaults]
+forks          = 200
+host_key_checking = False
+display_skipped_hosts = False
+retry_files_enabled = False
+
+[ssh_connection]
+pipelining = True
+retries = 2
diff --git a/a/site.yml b/a/site.yml
new file mode 100644
index 0000000..75da623
--- /dev/null
+++ b/a/site.yml
@@ -0,0 +1,4 @@
+---
+- hosts: tp.b8.nz
+  roles:
+    - prom
diff --git a/brc b/brc
index 487ee8f..e740a8d 100644
--- a/brc
+++ b/brc
@@ -353,19 +353,11 @@ bigclock() {
   xclock -digital -update 1 -face 'arial black-80:bold'
 }
 
-bpush() {
-  case $HOSTNAME in
-    tp) target=x2 ;;
-    x2) target=tp ;;
-  esac
-  btrbk-run -t $target && switch-mail-host $HOSTNAME $target
-}
 bpull() {
-  case $HOSTNAME in
-    tp) source=x2 ;;
-    x2) source=tp ;;
-  esac
-  btrbk-run -s $source && switch-mail-host $source $HOSTNAME
+  c /
+  # run latest
+  install-my-scripts
+  switch-mail-host $1 $HOSTNAMEs
 }
 
 btc() {
@@ -1100,6 +1092,7 @@ jtail() {
   journalctl -n 10000 -f "$@" | jfilter
 }
 jr() { journalctl "$@" | jfilter | less ; }
+jrf() { journalctl -f "$@" | jfilter; }
 
 kff() { # keyboardio firmware flash
   pushd /a/bin/distro-setup/Arduino/Model01-Firmware
diff --git a/btrbk-run b/btrbk-run
index 4a2816b..469b1b3 100644
--- a/btrbk-run
+++ b/btrbk-run
@@ -43,7 +43,8 @@ rsync-dirs() {
 }
 
 
-m() { printf "%s: %s\n" "${0##*/}" "$*";  "$@"; }
+m() { if $verbose; then printf "$pre%s\n"  "$*"; fi;  "$@"; }
+die() { printf "$pre%s\n" "$*" >&2;  exit 1; }
 
 # latest $MAIL_HOST
 if [[ -e /b/bash_unpublished/source-semi-priv ]]; then
@@ -72,11 +73,17 @@ if [[ -s $default_args_file ]]; then
   sleep 5
 fi
 
+cron=false
 orig_args=("$@")
-temp=$(getopt -l pull-reexec,help cl:m:nps:t:vh "$@") || usage 1
+temp=$(getopt -l cpull-reexec,help cl:m:npqs:t:vh "$@") || usage 1
 eval set -- "$temp"
 while true; do
   case $1 in
+    --cron)
+      cron=true
+      pre="${0##*/}: "
+      shift
+      ;;
     # only creates the config file, does not run btrbk
     -c) conf_only=true; shift ;;
     # bytes per second, suffix k m g
@@ -95,7 +102,7 @@ while true; do
     -v) verbose=true; verbose_arg=-v; shift ;;
     -h|--help) usage ;;
     --) shift; break ;;
-    *) echo "$0: Internal error!" ; exit 1 ;;
+    *) die "Internal error!" ;;
   esac
 done
 
@@ -103,37 +110,47 @@ done
 cmd_arg=${1:-run}
 
 if [[ -v targets && $source ]]; then
-  echo "$0: error: -t and -s are mutually exclusive" >&2
-  exit 1
+  die "error: -t and -s are mutually exclusive"
 fi
 
-
-echo -e "$0: options: conf_only=$conf_only\ndry_run=$dry_run\nrate_limit=$rate_limit\nverbose=$verbose\ncmd_arg=$cmd_arg"
+if $verbose; then
+  printf "$0: options: conf_only=%s\ndry_run=%s\nrate_limit=%s\nverbose=%s\ncmd_arg=%s" "$conf_only" "$dry_run" "$rate_limit" "$verbose" "$cmd_arg"
+fi
 ### end options parsing
 
 
 # set default targets
-if [[ ! -v targets && ! $source ]]; then
+if [[ ! -v targets && ! $source && $HOSTNAME == $MAIL_HOST ]]; then
   case $HOSTNAME in
-    x2|kw)
-      if [[ $HOSTNAME == "$MAIL_HOST" ]]; then
-        targets=($HOME_DOMAIN)
+    kw|x2)
+      if ping -q -c1 -w1 iank.vpn.office.fsf.org &>/dev/null; then
+        home=iank.vpn.office.fsf.org
+      else
+        home=$HOME_DOMAIN
       fi
+      ;;&
+    kw)
+      targets=($home x2)
+      ;;
+    x2)
+      targets=($home kw)
       ;;
     tp)
-      targets=(frodo)
-      if [[ $HOSTNAME == "$MAIL_HOST" ]]; then
-        if timeout -s 9 10 ssh x2 :; then
-          targets+=(x2)
-        fi
+      targets=(frodo kd)
+      # might not be connected to the vpn
+      if timeout -s 9 10 ssh kw :; then
+        targets+=(kw)
       fi
       ;;
-    frodo)
-      targets=()
+    kd)
+      targets=(frodo tp)
+      # might not be connected to the vpn
+      if timeout -s 9 10 ssh kw :; then
+        targets+=(kw)
+      fi
       ;;
     *)
-      echo "$0: error: no default targets for this host, use -t"
-      exit 1
+      die "error: no default targets for this host, use -t"
       ;;
   esac
 fi
@@ -150,8 +167,7 @@ fi
 if [[ $mountpoints ]]; then
   for mp in ${mountpoints[@]}; do # default mountpoints to sync
     if [[ -e /nocow/btrfs-stale/$mp ]]; then
-      echo "$0: error: $mp is stale, mount-latest-subvol first"
-      exit 1
+      die "error: $mp is stale, mount-latest-subvol first"
     fi
   done
 else
@@ -209,12 +225,11 @@ fi
 
 
 if ! which btrbk &>/dev/null; then
-  echo "$0: error: no btrbk binary found"
-  exit 1
+  die "error: no btrbk binary found"
 fi
 # if our mountpoints are from stale snapshots,
 # it doesn't make sense to do a backup.
-check-subvol-stale ${mountpoints[@]} || exit 1
+check-subvol-stale ${mountpoints[@]} || die "found stale mountpoints in ${mountpoints[*]}"
 
 
 cat >/etc/btrbk.conf <<EOF
@@ -262,10 +277,12 @@ EOF
 mkdir -p /mnt/root/btrbk
 local_zone=$(date +%z)
 for h in ${targets[@]} $source; do
-  zone=$(ssh root@$h "mkdir -p /mnt/root/btrbk; date +%z")
+  cmd="ssh root@$h mkdir -p /mnt/root/btrbk && date +%z"
+  if ! zone=$($cmd); then
+    die "error running: $cmd"
+  fi
   if [[ $zone != $local_zone ]]; then
-    echo "error: error. dont confuse yourself with multiple time zones. $h has different timezone than localhost" >&2
-    exit 1
+    die "error: dont confuse yourself with multiple time zones. $h has different timezone than localhost"
   fi
 done
 
@@ -337,9 +354,8 @@ if mountpoint $rsync_mountpoint >/dev/null; then
   done
 fi
 
-/a/bin/distro-setup/install-my-scripts
 if [[ $source ]]; then
-  m mount-latest-subvol
+  m mount-latest-subvol $verbose_arg
 else
   m /a/exe/mount-latest-remote ${targets[@]}
 fi
diff --git a/check-subvol-stale b/check-subvol-stale
index cc032d4..0870f0a 100644
--- a/check-subvol-stale
+++ b/check-subvol-stale
@@ -161,7 +161,7 @@ for d; do
     if [[ $svp_cgen -ge $last_received_cgen ]]; then
       stale=false
     else
-      echo "$svp stale: it's gen at creation, $svp_cgen, is earlier than the last received snapshot, $last_received's gen at creation: $last_received_cgen"
+      d "$svp stale: it's gen at creation, $svp_cgen, is earlier than the last received snapshot, $last_received's gen at creation: $last_received_cgen"
       freshest_snap=$last_received
       stale=true
     fi
diff --git a/conflink b/conflink
index 555d4d4..0010c9d 100755
--- a/conflink
+++ b/conflink
@@ -63,26 +63,30 @@ common-file-setup() {
 all_dirs=({/a/c,/p/c}{,/machine_specific/$HOSTNAME})
 # note, we assume a group of hosts does not have the
 # same name as a single host, which is no problem on our scale.
-for x in /p/c/machine_specific/*.hosts; do
+for x in /p/c/machine_specific/*.hosts /a/bin/ds/machine_specific/*.hosts; do
   if grep -qxF $HOSTNAME $x; then all_dirs+=( ${x%.hosts} ); fi
 done
 
 c_dirs=(/a/c{,/machine_specific/$HOSTNAME})
 case $USER in
-  ian|iank)
+  iank)
+    files=(/p/c/machine_specific/*/filesystem/etc/ssh/*_key)
+    if [[ -e $files ]]; then
+      chmod 600 ${files[@]}
+    fi
     # p needs to go first so .ssh link is created, then config link inside it
     m common-file-setup ${all_dirs[@]}
 
     #### begin special extra stuff ####
     install -d -m700 ~/gpg-agent-socket
 
-    files=(/var/lib/bind)
-    if [[ -e $files ]]; then
+    f=/var/lib/bind
+    if [[ -e $f ]]; then
       # reset to the original permissions.
-      m s chgrp -R bind ${files[@]}
-      m s chmod g+w ${files[@]}
+      m s chgrp -R bind $f
+      m s chmod g+w $f
     fi
-    sudo bash -c 'shopt -s nullglob; cd /etc/bind; for f in *.key *.private key.*; do chgrp bind $f; done'
+    sudo bash -c 'shopt -s nullglob; for f in /etc/bind/*.key /etc/bind/*.private /etc/bind/key.*; do chgrp bind $f; done'
     if [[ -e /etc/davpass ]] && getent group www-data &>/dev/null; then
       s chgrp www-data /etc/davpass
     fi
diff --git a/filesystem/etc/ansible/hosts b/filesystem/etc/ansible/hosts
new file mode 100644
index 0000000..4717dbc
--- /dev/null
+++ b/filesystem/etc/ansible/hosts
@@ -0,0 +1,7 @@
+[main]
+kd.b8.nz
+kw.office.fsf.org
+tp.b8.nz
+frodo.b8.nz
+x2.office.fsf.org
+x3.b8.nz
diff --git a/machine_specific/btrbk.hosts b/machine_specific/btrbk.hosts
index 1c0b74c..249ba95 100644
--- a/machine_specific/btrbk.hosts
+++ b/machine_specific/btrbk.hosts
@@ -1,2 +1,4 @@
 kd
-frodo
\ No newline at end of file
+kw
+x2
+tp
diff --git a/machine_specific/btrbk/filesystem/etc/systemd/system/btrbk.service b/machine_specific/btrbk/filesystem/etc/systemd/system/btrbk.service
index 4be721d..2669818 100644
--- a/machine_specific/btrbk/filesystem/etc/systemd/system/btrbk.service
+++ b/machine_specific/btrbk/filesystem/etc/systemd/system/btrbk.service
@@ -4,4 +4,4 @@ After=multi-user.target
 
 [Service]
 Type=oneshot
-ExecStart=/a/bin/log-quiet/sysd-mail-once btrbk /a/exe/btrbk-run
+ExecStart=/a/bin/log-quiet/sysd-mail-once btrbk /usr/local/bin/btrbk-run -q
diff --git a/machine_specific/kd/filesystem/etc/systemd/system/btrbk.service b/machine_specific/kd/filesystem/etc/systemd/system/btrbk.service
deleted file mode 100644
index 4be721d..0000000
--- a/machine_specific/kd/filesystem/etc/systemd/system/btrbk.service
+++ /dev/null
@@ -1,7 +0,0 @@
-[Unit]
-Description=Btrbk backup
-After=multi-user.target
-
-[Service]
-Type=oneshot
-ExecStart=/a/bin/log-quiet/sysd-mail-once btrbk /a/exe/btrbk-run
diff --git a/machine_specific/kd/filesystem/etc/systemd/system/btrbk.timer b/machine_specific/kd/filesystem/etc/systemd/system/btrbk.timer
deleted file mode 100644
index 27113f6..0000000
--- a/machine_specific/kd/filesystem/etc/systemd/system/btrbk.timer
+++ /dev/null
@@ -1,8 +0,0 @@
-[Unit]
-Description=Run btrbk-run once every 20 mins
-
-[Timer]
-OnCalendar=*:0/20
-
-[Install]
-WantedBy=timers.target
diff --git a/mount-latest-subvol b/mount-latest-subvol
index 359b534..aa35efa 100644
--- a/mount-latest-subvol
+++ b/mount-latest-subvol
@@ -13,13 +13,27 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# usage: mount-latest-subvol
-#
-# Note, at source location, intentionally not executable, run and read
-# install-my-scripts.
-
+script=$(readlink -f -- "$BASH_SOURCE")
 cd /
-[[ $EUID == 0 ]] || exec sudo -E "$BASH_SOURCE" "$@"
+[[ $EUID == 0 ]] || exec sudo -E "$script" "$@"
+
+
+usage() {
+  cat <<EOF
+Usage: ${0##*/} [OPTIONS]
+
+-h|--help  Print help and exit.
+
+
+Note, at source location, intentionally not executable, run and read
+install-my-scripts.
+
+Note: Uses util-linux getopt option parsing: spaces between args and
+options, short options can be combined, options before args.
+EOF
+  exit $1
+}
+
 
 errcatch() {
   set -E; shopt -s extdebug
@@ -69,12 +83,22 @@ tu() {
     grep -xFq "$line" "$file" || tee -a "$file"<<<"$line"
   done
 }
-e() { printf "%s\n" "$*"; "$@"; }
+m() {
+  if $verbose; then
+    printf "%s\n" "$*"
+  fi
+  "$@"
+}
+x() {
+  printf "%s\n" "$*"
+  "$@"
+}
+
 mnt() {
   dir=$1
-  if ! mountpoint $dir &>/dev/null; then
+  if ! mountpoint -q $dir; then
     mkdir -p $dir
-    e mount $dir
+    m mount $dir
   fi
 }
 fstab() {
@@ -121,10 +145,26 @@ kill-dir() {
   return 1
 }
 
+##### begin command line parsing ########
+
+# you can remove this if you do not have options which can have args with spaces or empty.
+
+verbose=false
 force=false
-if [[ $1 == -f ]]; then
-  force=true
-fi
+temp=$(getopt -l help,force,verbose hfv "$@") || usage 1
+eval set -- "$temp"
+while true; do
+  case $1 in
+    -f|--force) force=true ;;
+    -v|--verbose) verbose=true ;;
+    -h|--help) usage ;;
+    --) shift; break ;;
+    *) echo "$0: unexpected args: $*" >&2 ; usage 1 ;;
+  esac
+  shift
+done
+
+##### end command line parsing ########
 
 ret=0
 
@@ -132,7 +172,7 @@ ret=0
 root_dev=$(awk '$2 == "/" {print $1}' /etc/mtab)
 if [[ $root_dev == /dev/dm-* ]]; then
   for d in /dev/mapper/*; do
-    if [[ $(readlink -f $d) == $root_dev ]]; then
+    if [[ $(readlink -f $d) == "$root_dev" ]]; then
       root_dev=$d
       break
     fi
@@ -163,7 +203,7 @@ shopt -s nullglob
 # ownership, and ssh doesn\'t allow any group writable parent
 # directories, so we are forced to use a directory structure similar
 # to home directories
-f=(/mnt/root/btrbk/q.*)
+f=(/mnt/root/btrbk/q.*); f=${f[0]}
 if [[ -e $f ]]; then
   fstab <<EOF
 $crypt_dev  /q  btrfs  noatime,subvol=q,gid=1000$noauto  0 0
@@ -171,7 +211,7 @@ $crypt_dev  /q  btrfs  noatime,subvol=q,gid=1000$noauto  0 0
 EOF
 fi
 
-f=(/mnt/root/btrbk/o.*)
+f=(/mnt/root/btrbk/o.*); f=${f[0]}
 if [[ -e $f ]]; then
   fstab <<EOF
 $crypt_dev  /o  btrfs  noatime,subvol=o$noauto  0 0
@@ -226,7 +266,7 @@ for vol in q a o i; do
 
 
   # if latest is already mounted, make sure binds are mounted and move on
-  e check-subvol-stale $d
+  m check-subvol-stale $d
   # populated by check-subvol-stale if stale
   if ! fresh_snap=$(cat /nocow/btrfs-stale/$vol 2>/dev/null); then
     mnt $d
@@ -239,15 +279,15 @@ for vol in q a o i; do
   umount_ret=true
   unmounted=()
   for dir in $(echo $d ${binds[*]}\ |tac -s\ ); do
-    if mountpoint $dir; then
-      if e umount -R $dir; then
+    if mountpoint -q $dir; then
+      if m umount -R $dir; then
         unmounted+=($dir)
       else
         if ! kill-dir TERM TERM TERM INT INT HUP HUP; then
           if $force; then  kill-dir KILL; fi
         fi
 
-        if e umount -R $dir; then
+        if m umount -R $dir; then
           unmounted+=($dir)
         else
           echo "$0: failed to umount $dir"
@@ -269,21 +309,59 @@ for vol in q a o i; do
   # todo: decipher /mnt/root, like we do in check-subvol-stale
   cd /mnt/root
   if [[ -e $vol ]]; then
-    e mv $vol $vol.leaf.$(date +%Y-%m-%dT%H:%M:%S%z)
+    leaf=$vol.leaf.$(date +%Y-%m-%dT%H:%M:%S%z)
+    m mv $vol $leaf
+    m btrfs property set -ts $leaf ro true
+
+    ### begin check if leaf is different, delete it if not ###
+    if [[ -e /a/opt/btrfs-snapshots-diff/btrfs-snapshots-diff.py ]]; then
+      parentid=$(btrfs sub show $leaf | awk '$1 == "Parent" && $2 == "UUID:" {print $3}')
+      bsubs=(/mnt/root/btrbk/$vol.*)
+      bsub=
+      # go in reverse order as its more likely to be at the end
+      for ((i=${#bsubs[@]}-1; i>=0; i--)); do
+        if [[ $parentid == $(btrfs sub show ${bsubs[i]} | awk '$1 == "UUID:" {print $2}') ]]; then
+          bsub=${bsubs[i]}
+          break
+        fi
+      done
+      if [[ $bsub ]]; then
+        tmp=$(mktemp)
+        # in testing, same subvol is 136 bytes. allow some overhead
+        btrfs send --no-data -p $bsub $leaf | head -c 1000 > $tmp
+        if (( $(stat -c%s $tmp) < 1000)); then
+          # example output for an empty diff:
+          # Found a valid Btrfs stream header, version 1
+          # o.leaf.2019-05-15T14:00:50-0400;snapshot: uuid=ba045ea30737dd449003f1ee40ec12d0, ctrasid=109533, clone_uuid=3c7e3544e486834aa71d89e5b8f30056, clone_ctransid=109533
+          lines=$(/a/opt/btrfs-snapshots-diff/btrfs-snapshots-diff.py -s -f $tmp | \
+                    grep -vxF "Found a valid Btrfs stream header, version 1" | \
+                    grep -cv "^[^;]*;snapshot: ") ||:
+          if [[ $lines == 0 ]]; then
+            x btrfs sub del $leaf
+          fi
+        fi
+      fi
+    fi
+    ### end check if leaf is different, delete it if not ###
+
+    ## begin expire leaf vols ##
     leaf_vols=($vol.leaf.*)
+    count=1
     for leaf in ${leaf_vols[@]}; do
       leaf_secs=$(date -d ${leaf#$vol.leaf.} +%s)
-      if (( $(date +%s) - 60*60*24*60 > leaf_secs )); then # 60 days
-        e btrfs sub del $leaf
+      if (( $(date +%s) - 60*60*24*60 > leaf_secs || count > 200 )); then # 60 days
+        x btrfs sub del $leaf
       fi
+      count=$((count+1))
     done
+    ## end expire leaf vols ##
   fi
   # Note, we make a few assumptions in this script, like
   # $d was not a different subvol id than $vol, and
   # things otherwise didn't get mounted very strangely.
-  e btrfs sub snapshot $fresh_snap $vol
+  m btrfs sub snapshot $fresh_snap $vol
   for dir in $d ${binds[@]}; do
-    e mnt $dir
+    m mnt $dir
   done
   stale_dir=/nocow/btrfs-stale
   rm -f $stale_dir/$d
diff --git a/switch-mail-host b/switch-mail-host
index 0cc8c0b..c67f306 100755
--- a/switch-mail-host
+++ b/switch-mail-host
@@ -11,6 +11,7 @@ Adjust home network dns so NEW_HOST resolves locally if it is on the
 local network.  Turn off mail receiving on OLD_HOST, run btrbk to move
 mail to NEW_HOST, turn on mail receiving on NEW_HOST.
 
+-o         Only btrbk /o, instead of all filesystems.
 -w         Don't try to ssh to wrt. Should only be used in unusual network situation.
 -h|--help  Print help and exit.
 
@@ -23,9 +24,11 @@ EOF
 
 update_wrt=true # default
 temp=$(getopt -l help wh "$@") || usage 1
+mp_args=
 eval set -- "$temp"
 while true; do
   case $1 in
+    -o) mp_args="-m /o"; shift ;;
     -w) update_wrt=false; shift ;;
     -h|--help) usage ;;
     --) shift; break ;;
@@ -142,9 +145,9 @@ $old_shell primary-setup $new_hostname
 /a/bin/distro-setup/install-my-scripts
 
 if $localhost_new; then
-  btrbk-run -s $old_host -m /o
+  btrbk-run -s $old_host $mp_args
 else
-  btrbk-run -t $new_host -m /o
+  btrbk-run -t $new_host $mp_args
 fi
 
 $new_shell primary-setup $new_hostname