From 8796be333861fd2edc6c8b9b4c22307c11fce409 Mon Sep 17 00:00:00 2001 From: Ian Kelling Date: Wed, 9 Aug 2023 22:27:32 -0400 Subject: [PATCH] mostly a bunch of fixes --- brc | 28 +++-- brc2 | 9 +- btrbk-run | 261 +++++++++++++++++++++++++++++++--------------- btrbk-spread-wrap | 21 ++++ mail-setup | 33 +++++- mailtest-check | 4 +- rootsshsync | 4 +- script-files | 1 + 8 files changed, 258 insertions(+), 103 deletions(-) create mode 100755 btrbk-spread-wrap diff --git a/brc b/brc index afb73f1..ddb6e79 100644 --- a/brc +++ b/brc @@ -673,8 +673,20 @@ khcopy() { # ya, hacky hardcoded hostnames in 2023. we could do better hssh-update() { - local -a failed_hosts - for host in kd x3.office.fsf.org syw; do + local -a failed_hosts hosts + case $HOSTNAME in + sy|kd) + hosts=( + kd x3.office.fsf.org syw + ) + ;; + x3) + hosts=( + b8.nz sywg.b8.nz + ) + ;; + esac + for host in ${hosts[@]}; do e $host if ! scp /b/fai/fai/config/files/usr/local/bin/hssh/IANK root@$host:/usr/local/bin/hssh; then failed_hosts+=($host) @@ -1578,14 +1590,14 @@ pst() { pstree -apnA } -jtail() { - journalctl -n 10000 -f "$@" -} -jr() { journalctl "$@" ; } -jrf() { journalctl -f "$@" ; } +jr() { journalctl -e -n100000 "$@" ; } +jrf() { journalctl -n1000 -f "$@" ; } jru() { - journalctl -u exim4 _SYSTEMD_INVOCATION_ID="$(systemctl show -p InvocationID --value $1)" + # the invocation id is "assigned each time the unit changes from an inactive + # state into an activating or active state" man systemd.exec + journalctl -e --no-tail -u exim4 _SYSTEMD_INVOCATION_ID="$(systemctl show -p InvocationID --value $1)" } +ccomp journalctl jr jrf jru diff --git a/brc2 b/brc2 index 4d83fed..f291a8a 100644 --- a/brc2 +++ b/brc2 @@ -1283,7 +1283,7 @@ btrbk-date-sed() { done } jrbtrbk() { - jr -u btrbk-run -u btrbk -u switch-mail-host -u btrbk-spread "$@" + jr -u btrbk-run -u btrbk -u switch-mail-host "$@" } # internal function @@ -1327,7 +1327,7 @@ btrbk-host-debug() { ## this takes a while, we only want to do it on 1st run # if [[ -s /tmp/b/$host.log ]]; then continue; fi - # ssh $host journalctl -u btrbk-run -u btrbk -u switch-mail-host -u btrbk-spread >/tmp/b/$host.log + # ssh $host journalctl -u btrbk-run -u btrbk -u switch-mail-host >/tmp/b/$host.log done gr '\bsnapshot success' /var/log/btrbk/*.log >/tmp/b/local.log cd /tmp/b @@ -2292,11 +2292,6 @@ ccomp xdg-open o # jr() { journalctl "$@" | jfilter | less ; } # jrf() { journalctl -n 200 -f "$@" | jfilter; } -jr() { journalctl "$@" ; } -jrf() { journalctl -n 200 -f "$@" ; } - - -ccomp journalctl jtail jr jrf ## old version for model01. i need to get that firmware working again. # kff() { # keyboardio firmware flash. you must hold down the tilde key diff --git a/btrbk-run b/btrbk-run index 6726eb9..b5682ba 100644 --- a/btrbk-run +++ b/btrbk-run @@ -43,11 +43,93 @@ pre=btrbk-run script_name="${BASH_SOURCE[0]}" script_name="${script_name##*/}" +d() { + if $dry_run || $conf_only; then + printf "$pre dry-run: %s\n" "$*" + else + printf "$pre running: %s\n" "$*" + "$@" + fi +} m() { if $verbose; then printf "$pre %s\n" "$*"; fi; "$@"; } e() { printf "$pre %s\n" "$*"; } die() { printf "$pre error: %s\n" "$*" >&2; echo "$pre exiting with status 1" >&2; exit 1; } mexit() { echo "$pre exiting with status $1"; exit $1; } +uninstalled-file-die() { + die "uninstalled file $1. run install-my-scripts or rerun with -f" +} + +set-location() { + case $HOSTNAME in + kw) + at_work=true + ;; + kd|frodo) + at_home=true + ;; + x2|x3|sy) + if [[ $(dig +short @10.2.0.1 -x 10.2.0.2 2>&1 ||:) == kd.b8.nz. ]] \ + && ip n show 10.2.0.1 | grep . &>/dev/null; then + at_home=true + elif ping -q -c1 -w1 hal.office.fsf.org &>/dev/null \ + && ip n show 192.168.0.26 | grep . &>/dev/null; then + at_work=true + fi + ;; + esac +} + +exit-if-no-default-targets() { + if ! $force && [[ $HOSTNAME != "$MAIL_HOST" ]]; then + echo "MAIL_HOST=$MAIL_HOST, nothing to do" + mexit 0 + fi + case $HOSTNAME in + kw|kd|frodo|x2|x3|sy) : ;; + *) + die "error: no default targets for this host, use -t" + ;; + esac +} + +add-x3-target() { + # main work machine + if ping -q -c1 -w1 x3.office.fsf.org &>/dev/null; then + targets+=(x3.office.fsf.org) + elif ping -q -c1 -w1 $h.b8.nz &>/dev/null; then + # in case we took it home + targets+=(x3.b8.nz) + else + targets+=(x3wg.b8.nz) + fi +} + +add-wireless-target-h() { + if ping -q -c1 -w1 $h.b8.nz &>/dev/null; then + targets+=($h.b8.nz) + elif ping -q -c1 -w1 ${h}w.b8.nz &>/dev/null; then + targets+=(${h}w.b8.nz) + fi +} + +qconf() { + case $sub in + q) + # q has sensitive data i dont want to backup for so long + cat >>/etc/btrbk$conf_suf.conf < >(sed '/No such file or directory/d'); then + # shellcheck disable=SC2046 # we want word splitting + set -- $(< $once_args_file-tmp) "$@" + # i havent used this feature yet, so warn about it + echo "$0: btrbk-run options set in $once_args_file:" + cat $once_args_file-tmp + rm -f $once_args_file-tmp +fi + + targets=() early=false fast=false @@ -115,7 +208,9 @@ while true; do # switch mail-host, no need to repeat the same checks again. --fast) fast=true ;; -i) incremental_strict=true ;; - # note this implies resume and -p + # note this implies resume and -p because it is just meant to make + # other hosts have the same snapshots, not do any expiry or new + # backups. -k) kd_spread=true ;; # bytes per second, suffix k m g -l) rate_limit=$2; shift ;; @@ -152,9 +247,6 @@ done cmd_arg="$1" -uninstalled-file-die() { - die "uninstalled file $1. run install-my-scripts or rerun with -f" -} if ! $force && { $check_installed || [[ ! $source ]]; } ; then install_bin_files=( @@ -251,60 +343,6 @@ at_work=false at_home=false -set-location() { - case $HOSTNAME in - kw) - at_work=true - ;; - kd|frodo) - at_home=true - ;; - x2|x3|sy) - if [[ $(dig +short @10.2.0.1 -x 10.2.0.2 2>&1 ||:) == kd.b8.nz. ]] \ - && ip n show 10.2.0.1 | grep . &>/dev/null; then - at_home=true - elif ping -q -c1 -w1 hal.office.fsf.org &>/dev/null \ - && ip n show 192.168.0.26 | grep . &>/dev/null; then - at_work=true - fi - ;; - esac -} - -exit-if-no-default-targets() { - if ! $force && [[ $HOSTNAME != "$MAIL_HOST" ]]; then - echo "MAIL_HOST=$MAIL_HOST, nothing to do" - mexit 0 - fi - case $HOSTNAME in - kw|kd|frodo|x2|x3|sy) : ;; - *) - die "error: no default targets for this host, use -t" - ;; - esac -} - -add-x3-target() { - # main work machine - if ping -q -c1 -w1 x3.office.fsf.org &>/dev/null; then - targets+=(x3.office.fsf.org) - elif ping -q -c1 -w1 $h.b8.nz &>/dev/null; then - # in case we took it home - targets+=(x3.b8.nz) - else - targets+=(x3wg.b8.nz) - fi -} - -add-wireless-target-h() { - if ping -q -c1 -w1 $h.b8.nz &>/dev/null; then - targets+=($h.b8.nz) - elif ping -q -c1 -w1 ${h}w.b8.nz &>/dev/null; then - targets+=(${h}w.b8.nz) - fi -} - - # set default targets if [[ ! -v targets && ! $source ]]; then exit-if-no-default-targets @@ -355,9 +393,9 @@ else *) prospective_mps=() if [[ $source ]]; then - source_state="$(ssh $source cat /a/bin/bash_unpublished/source-state)" + source_state="$(ssh $source 'cat /a/bin/bash_unpublished/source-state; echo source_host=$HOSTNAME')" eval "$source_state" - source_host="$(ssh $source cat /etc/hostname)" + # shellcheck disable=SC2154 # assigned in the above eval. if [[ $source_host == "$MAIL_HOST" ]]; then prospective_mps+=(/o) fi @@ -391,7 +429,8 @@ else done fi -if (( ! ${#mountpoints[@]} )); then +tmp=$(( ${#mountpoints[@]} == 0 )) +if (( tmp )); then die didnt get mountpoint arg and had no defaults fi @@ -515,7 +554,8 @@ else IFS=" " read -r root_size percent_used <<<"${tmp_array[1]}" percent_used=${percent_used%%%} - if (( ${#tmp_array[@]} != 2 )); then + tmp=$(( ${#tmp_array[@]} != 2 )) + if (( tmp )); then die "error: didnt get 2 lines in test ssh to target $h. investigate" fi case $percent_used in @@ -531,11 +571,13 @@ else # we may be booted into a bootstrap fs or something min_root_kb=$(( 1024 * 1024 * 200 )) # 200 gb - if (( root_size < min_root_kb )); then + tmp=$(( root_size < min_root_kb )) + if (( tmp )); then continue fi - if (( percent_used >= 98 )); then + tmp=$(( percent_used >= 98 )) + if (( tmp )); then die "error: filesystem on target $h is $percent_used % full" fi @@ -612,8 +654,12 @@ snapshot_dir btrbk target_preserve $std_preserve target_preserve_min 6h -# i tried this when investigating: clone no source subvolume found error -#incremental_prefs sro:1 srn:1 sao san:1 aro:1 arn:1 +# it seems very likely that not doing this could result in clone source not found +# errors, for example when expiry happens differently on different hosts, +# also, as btrbk does by default, if a failed send happens, on the next run it +# will warn about a stray subvolume, but then create a backup of a newer subvol +# and use an older subvol as the parent. +incremental_prefs sao:1 # if something fails and it's not obvious, try doing # btrbk -l debug -v dryrun @@ -627,21 +673,6 @@ incremental strict EOF fi -qconf() { - case $sub in - q) - # q has sensitive data i dont want to backup for so long - cat >>/etc/btrbk$conf_suf.conf <>/etc/btrbk$conf_suf.conf <>/etc/btrbk$conf_suf.conf <>/etc/btrbk$conf_suf.conf < 1 && tg_snap_count == orphan_mp_count )) + if (( tmp )) ; then + die "something went wrong checking orphans on $tg: for mountpoint ${mountpoints[$i]}, $orphan_mp_count" + fi + done +} + +if [[ $source ]]; then + for snap in $(ssh root@$source "shopt -s nullglob; ${snap_list_cmds[*]}"); do + source_snaps[$snap]=t + done + get-orphan-tg-snaps + tmp=$(( ${#orphan_tg_snaps[*]} >= 1 )) + if (( tmp )); then + d btrfs sub del ${orphan_tg_snaps[*]} + fi +else # we have targets + for tg in ${targets[@]}; do + tmp_str=$(ssh root@$tg "shopt -s nullglob; ${snap_list_cmds[*]}") + mapfile -t tg_snaps <<<"$tmp_str" + get-orphan-tg-snaps + tmp=$(( ${#orphan_tg_snaps[*]} >= 1 )) + if (( tmp )); then + d ssh root@$tg "btrfs sub del ${orphan_tg_snaps[*]}" + fi + done +fi + # todo: umount first to ensure we don't have any errors # todo: do some kill fuser stuff to make umount more reliable @@ -743,11 +835,12 @@ else done fi +# todo, we get hostnames earlier, reuse that. if [[ $ret == 0 ]]; then for tg in ${targets[@]}; do h=$(ssh $tg hostname) if [[ $h == kd && $HOSTNAME == x3 && $HOSTNAME == "$MAIL_HOST" ]]; then - ssh root$tg systemctl --no-block start btrbk-spread + m ssh root@$tg 'btrbk-spread-wrap &>/dev/null &2; exit 1; fi +shopt -s inherit_errexit 2>/dev/null ||: # ignore fail in bash < 4.4 +set -eE -o pipefail +trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" exit status: $?, PIPESTATUS: ${PIPESTATUS[*]}" >&2' ERR + +/a/exe/install-my-scripts +printf "%s\n" -k >/etc/btrbk-run-once.conf +# running start if it is already starting causes it to just wait until +# it done starting. In that case, we'd need to run it twice, or we have +# another unit which has an After=, eg btrbk-spread, but that isn't as +# good because, we can't have btrbk have After= on btrbk-spread, and +# what if btrbk-spread is itself already running, we have the same +# problem. +systemctl start btrbk +if [[ -s /etc/btrbk-run-once.conf ]]; then + systemctl start btrbk +fi diff --git a/mail-setup b/mail-setup index dd87fba..65682c2 100755 --- a/mail-setup +++ b/mail-setup @@ -857,6 +857,7 @@ fi u /etc/spamassassin/mylocal.cf <<'EOF' # this is mylocal.cf because the normal local.cf has a bunch of upstream stuff i dont want to mess with + # /usr/share/doc/exim4-base/README.Debian.gz: # SpamAssassin's default report should not be used in a add_header # statement since it contains empty lines. (This triggers e.g. Amavis' @@ -881,6 +882,35 @@ PIDFILE="/var/run/spamd.pid" NICE="--nicelevel 15" CRON=1 EOF + +case $HOSTNAME in + bk) +u /etc/spamassassin/my_thishost.cf <<'EOF' +# note: these are duplicated in exim config +# veth0/1 # bk bk_ip6 +internal_networks 10.173.8.1/32 10.173.8.2/32 85.119.83.50/32 2001:ba8:1f1:f0c9::2 +trusted_networks 10.173.8.1/32 10.173.8.2/32 85.119.83.50/32 2001:ba8:1f1:f0c9::2 +EOF + + ;; + je) + u /etc/spamassassin/my_thishost.cf <<'EOF' +# note: these are duplicated in exim config +# veth0/1 # je je_ipv6 +internal_networks 10.173.8.1/32 10.173.8.2/32 85.119.82.128/32 2001:ba8:1f1:f09d::2/128 +trusted_networks 10.173.8.1/32 10.173.8.2/32 85.119.82.128/32 2001:ba8:1f1:f09d::2/128 +EOF + ;; + *) + u /etc/spamassassin/my_thishost.cf <<'EOF' +# note: these are duplicated in exim config +# veth0/1 # li li_ip6 +internal_networks 10.173.8.1/32 10.173.8.2/32 72.14.176.105/32 2600:3c00::f03c:91ff:fe6d:baf8/128 +trusted_networks 10.173.8.1/32 10.173.8.2/32 72.14.176.105/32 2600:3c00::f03c:91ff:fe6d:baf8/128 +EOF +;; + esac + ##### end spamassassin config @@ -1261,6 +1291,7 @@ DKIM_SIGN_HEADERS = mime-version:in-reply-to:references:from:date:subject:to domainlist local_hostnames = ! je.b8.nz : ! bk.b8.nz : *.b8.nz : b8.nz +# note: most of these are duplicated in spamassassin config hostlist iank_trusted = <; \ # veth0 10.173.8.1 ; \ @@ -2931,7 +2962,7 @@ gnusmarthost: debug_print = "R: smarthost for $local_part@$domain" driver = manualroute domains = ! +local_domains -# send most mail through eggs, helps fsfs sender reputation. +# comment senders to send most mail through eggs, helps fsfs sender reputation. # uncomment and optionally move to 188 file to send through my own servers again senders = *@gnu.org transport = smarthost_dkim diff --git a/mailtest-check b/mailtest-check index b2e02c3..c3c0bd2 100755 --- a/mailtest-check +++ b/mailtest-check @@ -1,10 +1,10 @@ #!/bin/bash -# Usage: mail-test-check [slow] [anything] +# Usage: mail-test-check [slow] [int|nonint] # # slow: do slow checks, like spamassassin # -# anything: consider non-interactive, dont print unless something went +# for non-interactive, dont print unless something went # wrong diff --git a/rootsshsync b/rootsshsync index 965bdbe..13100b6 100755 --- a/rootsshsync +++ b/rootsshsync @@ -59,7 +59,9 @@ if [[ -e $user_ssh_dir/config ]]; then fi chown -R root:root /root/.ssh -rsync -tp --chmod=755 --chown=root:root /a/bin/fai/fai/config/files/usr/local/bin/hssh/IANK /usr/local/bin/hssh +# --update, -u skip files that are newer on the receiver +# I often push out a new hssh +rsync -tpu --chmod=755 --chown=root:root /a/bin/fai/fai/config/files/usr/local/bin/hssh/IANK /usr/local/bin/hssh if [[ -e /a/opt/btrbk/ssh_filter_btrbk.sh ]]; then install /a/opt/btrbk/ssh_filter_btrbk.sh /usr/local/bin diff --git a/script-files b/script-files index 3c8b658..02ffa3c 100644 --- a/script-files +++ b/script-files @@ -16,6 +16,7 @@ my_bin_files=( exim-nn-iptables check-crypttab /a/bin/cedit/cedit + btrbk-spread-wrap ) for f in /b/log-quiet/*; do -- 2.30.2