From 95eb9558206f8287febab80dd3f51d168a3ca831 Mon Sep 17 00:00:00 2001 From: Ian Kelling Date: Wed, 11 May 2022 00:15:08 -0400 Subject: [PATCH] various fixes --- brc2 | 34 ++++++++++++-- btrbk-run | 18 +++++++- conflink | 59 ++++++++++++++++++------ distro-end | 27 ++++++----- filesystem/etc/prometheus/rules/iank.yml | 8 ++++ install-my-scripts | 26 +++++++++-- mailtest-check | 17 ++++++- mount-latest-subvol | 20 ++++---- pkgs | 2 +- 9 files changed, 164 insertions(+), 47 deletions(-) diff --git a/brc2 b/brc2 index 491dcad..5035ca2 100644 --- a/brc2 +++ b/brc2 @@ -1134,6 +1134,32 @@ EOF } +mns() { # mount namespace + ns=$1 + shift + s mkdir -p /root/mount_namespaces + if ! s mountpoint /root/mount_namespaces >/dev/null; then + m s mount --bind /root/mount_namespaces /root/mount_namespaces + fi + m s mount --make-private /root/mount_namespaces + if [[ ! -e /root/mount_namespaces/$ns ]]; then + s touch /root/mount_namespaces/$ns + fi + if ! s mountpoint /root/mount_namespaces/$ns >/dev/null; then + m unshare --propagation slave --mount=/root/mount_namespaces/$ns /bin/true + fi + m sudo -E /usr/bin/nsenter --mount=/root/mount_namespaces/$ns "$@" +} + +mnsnonet() { + ns=$1 + if ! s ip netns list | grep -Fx nonet &>/dev/null; then + s ip netns add nonet + fi + mns $ns --net=/var/run/netns/nonet sudo -E -u iank /bin/bash +} + + lom() { local l base if [[ $1 == /* ]]; then @@ -1153,12 +1179,12 @@ lom() { fi fi m sudo mkdir -p /mnt/$base - m sudo mount /dev/mapper/$base /mnt/$base - m sudo chown $USER:$USER /mnt/$base + m mns mount /dev/mapper/$base /mnt/$base + m mns chown $USER:$USER /mnt/$base else base=$1 - if mountpoint /mnt/$base &>/dev/null; then - m sudo umount /mnt/$base + if mns mountpoint /mnt/$base &>/dev/null; then + m mns umount /mnt/$base fi if sudo cryptsetup status /dev/mapper/$base &>/dev/null; then if ! m sudo cryptsetup luksClose /dev/mapper/$base; then diff --git a/btrbk-run b/btrbk-run index cd6b7e8..1137a72 100644 --- a/btrbk-run +++ b/btrbk-run @@ -245,7 +245,7 @@ else prospective_mps=(/i) ;; *) - prospective_mps=(/a /q) + prospective_mps=() if [[ $source ]]; then source_state="$(ssh $source cat /a/bin/bash_unpublished/source-state)" eval "$source_state" @@ -263,6 +263,9 @@ else fi fi fi + # note: put q last just in case its specific retention options were to + # affect other config sections. I havent tested if that is the case. + prospective_mps+=(/a /q) ;; esac fi @@ -422,7 +425,6 @@ snapshot_create onchange snapshot_preserve 18h 14d 8w 24m snapshot_preserve_min 2h snapshot_dir btrbk - # so, total backups = ~58 target_preserve 18h 14d 8w 24m target_preserve_min 2h @@ -454,6 +456,18 @@ EOF volume $vol subvolume $sub EOF + case $sub in + q) + # q has sensitive data i dont want to backup for so long + cat >>/etc/btrbk.conf <&2; exit 1; } + fast=false -if [[ $1 == -f ]]; then # f for fast - fast=true +verbose=false +temp=$(getopt -l help hvf "$@") || usage 1 +eval set -- "$temp" +while true; do + case $1 in + -v) verbose=true ;; + -f) fast=true ;; + -h|--help) usage ;; + --) shift; break ;; + *) echo "$0: unexpected args: $*" >&2 ; usage 1 ;; + esac shift -elif - [[ $1 ]]; then - echo "error: unrecognized arguments" >&2 - exit 0 -fi +done +readonly fast verbose + +##### end command line parsing ######## + if $fast; then lnf() { ln -sf "$@"; } fi +if $verbose; then + m() { + echo "$*" + "$@" + } +fi + shopt -s nullglob shopt -s extglob shopt -s dotglob @@ -97,7 +125,7 @@ common-file-setup() { case $HOSTNAME in kd) if systemctl is-active prometheus &>/dev/null; then - m s systemctl reload prometheus + v s systemctl reload prometheus fi ;; esac @@ -130,11 +158,11 @@ common-file-setup() { m lnf ${x[@]} ~ done if $reload_systemd; then - m s systemctl daemon-reload + v s systemctl daemon-reload fi for service in ${restart_services[@]}; do if systemctl is-active $service >/dev/null; then - m s systemctl restart $service + v s systemctl restart $service fi done } @@ -155,8 +183,8 @@ case $user in for t in systemstatus epanicclean btrfsmaintstop dynamicipupdate; do f=/etc/systemd/system/$t.timer if [[ -e $f ]]; then - s systemctl stop $t.timer - s systemctl disable $t.timer + v systemctl stop $t.timer + v systemctl disable $t.timer s rm -fv $f reload_systemd=true fi @@ -165,7 +193,10 @@ case $user in if [[ -e /etc/cron.daily/check-lets-encrypt-ssl-settings ]]; then m s rm -f /etc/cron.daily/check-lets-encrypt-ssl-settings fi - + # conversion from whole folder subdir to individual files. + if [[ -L /home/iank/.config/copyq ]]; then + rm -fv /home/iank/.config/copyq + fi /a/bin/ds/install-my-scripts files=(/p/c/machine_specific/*/filesystem/etc/ssh/*_key diff --git a/distro-end b/distro-end index 27cefea..4b73b8b 100755 --- a/distro-end +++ b/distro-end @@ -1309,18 +1309,21 @@ m reset-xscreensaver # cabal update # cabal install --upgrade-dependencies --force-reinstalls arbtt # also, i assume syncing this between machines somehow messed up the data. -if mountpoint /p &>/dev/null; then - case $codename in - etiona|nabia) - pi arbtt - # same as seru enable arbtt, but works over ssh when systemctl --user causes error: - # Failed to connect to bus: No such file or directory - lnf -T /a/bin/ds/subdir_files/.config/systemd/user/arbtt.service /home/iank/.config/systemd/user/default.target.wants/arbtt.service - # allow failure - seru start arbtt ||: - ;; - esac -fi + +## not using arbtt for now +# if mountpoint /p &>/dev/null; then +# case $codename in +# etiona|nabia) +# pi arbtt +# # same as seru enable arbtt, but works over ssh when systemctl --user causes error: +# # Failed to connect to bus: No such file or directory +# lnf -T /a/bin/ds/subdir_files/.config/systemd/user/arbtt.service /home/iank/.config/systemd/user/default.target.wants/arbtt.service +# # allow failure +# seru start arbtt ||: +# ;; +# esac +# fi +rm -fv /home/iank/.config/systemd/user/default.target.wants/arbtt.service m primary-setup diff --git a/filesystem/etc/prometheus/rules/iank.yml b/filesystem/etc/prometheus/rules/iank.yml index 14d4275..651eb00 100644 --- a/filesystem/etc/prometheus/rules/iank.yml +++ b/filesystem/etc/prometheus/rules/iank.yml @@ -184,6 +184,14 @@ groups: annotations: summary: 'jr -u mailtest-check -e' + - alert: mailtest_check_missing_dnswl + expr: |- + mailtest_check_missing_dnswl >= 1 + for: 30m + labels: + severity: day + annotations: + summary: 'jr -u mailtest-check -e' # We expect to be getting metrics, if we come up and notice we have # any missing in the past, and it wasn't from a reboot, and we haven't diff --git a/install-my-scripts b/install-my-scripts index c932cfe..dd52266 100755 --- a/install-my-scripts +++ b/install-my-scripts @@ -27,6 +27,18 @@ trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR x="$(readlink -f -- "${BASH_SOURCE[0]}")"; cd ${x%/*} # directory of this file +m() { + "$@" +} + +# be a bit more verbose if we are connected to a terminal +if test -t 0; then + #echo debug: in terminal + m() { + echo "$*" + "$@" + } +fi # scripts that would interfere with unmounting /a, put them elsewhere. # note: previously used the install command, but it had this habit of @@ -36,7 +48,7 @@ x="$(readlink -f -- "${BASH_SOURCE[0]}")"; cd ${x%/*} # directory of this file # changed, so that should fix it. /a/bin/log-quiet/setup rsync -t --chmod=755 --chown=root:root switch-mail-host btrbk-run mount-latest-subvol \ - check-subvol-stale myi3status mailtest-check \ + check-subvol-stale myi3status \ mailbindwatchdog \ check-mailq \ unsaved-buffers.el \ @@ -52,6 +64,7 @@ cmd=( rsync -aiSAX --chown=root:root --chmod=g-s epanic-clean system-status btrfsmaint + mailtest-check dynamic-ip-update /usr/local/bin ) @@ -60,19 +73,24 @@ sre() { if [[ $(systemctl is-active $1.service ||:) != inactive ]]; then # just fire and forget. sometimes a script restart can fail, but then # then auto restart mechanism makes it succeed. - systemctl restart $service ||: & + m systemctl restart $service ||: & fi } while read -r line; do file="${line:12}" + #echo debug: file: $file case $file in btrfsmaint) - sre btrfsmaintstop & + sre btrfsmaintstop + ;; + mailtest-check) + # we stopped removing the dashes in services recently. + sre $file ;; *) - sre ${file//-/} & + sre ${file//-/} ;; esac done < <("${cmd[@]}") diff --git a/mailtest-check b/mailtest-check index 89ed00c..67cca9c 100755 --- a/mailtest-check +++ b/mailtest-check @@ -120,6 +120,7 @@ EOF fi tmpfile=$(mktemp) declare -i unexpected=0 + declare -i missing_dnswl=0 for folder in ${folders[@]}; do for from in ${froms[@]}; do latest= @@ -221,7 +222,6 @@ EOF echo # ends our printf string buildup cat $resultfile echo mailtest-check: end of spam debug results - # lets just handle 1 failure at a time in interactive mode. if $int; then echo mailtest-check: from: $from, to: $to @@ -237,7 +237,19 @@ EOF #fi fi rm -f $resultfile - unexpected=$(( unexpected + ${#results[@]} + ${#missing[@]} )) + unexpected=$(( unexpected + ${#results[@]} )) + for miss in ${missing[@]}; do + # We expect dns reputation services to go down from time to time, so + # we count them separately and alert differently. + case $miss in + RCVD_IN_DNSWL_MED|DKIMWL_WL_HIGH) + missing_dnswl+=1 + ;; + *) + unexpected+=1 + ;; + esac + done fi # if spamdpid fi # if $slow @@ -253,6 +265,7 @@ EOF done # end for folder in ${folders[@]} if $slow; then pr <