From 56c55d8e02cdd6ec67d2fe53cc03785d8876442e Mon Sep 17 00:00:00 2001 From: Ian Kelling Date: Sun, 24 Apr 2022 17:10:23 -0400 Subject: [PATCH] fixes and alert improvements --- brc2 | 6 +++--- conflink | 27 ++++++++++++++---------- distro-begin | 8 +++++-- dynamic-ip-update | 2 +- filesystem/etc/prometheus/rules/iank.yml | 14 +++++++++--- mail-setup | 2 +- mailtest-check | 14 ++++++++++-- system-status | 10 --------- 8 files changed, 50 insertions(+), 33 deletions(-) diff --git a/brc2 b/brc2 index f3c4b24..d85511f 100644 --- a/brc2 +++ b/brc2 @@ -541,15 +541,15 @@ EOF dnsup() { conflink -f - m ser reload bind9 + m ser reload named } dnsb8() { local f=/var/lib/bind/db.b8.nz - m ser stop bind9 + m ser stop named m sleep 1 m sudo rm -fv $f.jnl m sudo install -m 644 -o bind -g bind /p/c/machine_specific/vps/bind-initial/db.b8.nz $f - m ser restart bind9 + m ser restart named } dnsecgen() { # keys generated like this diff --git a/conflink b/conflink index a3f6e35..e53605e 100755 --- a/conflink +++ b/conflink @@ -147,21 +147,26 @@ for x in /p/c/machine_specific/*.hosts /a/bin/ds/machine_specific/*.hosts; do if grep -qxF $HOSTNAME $x; then all_dirs+=( ${x%.hosts} ); fi done -# old files 2022-03 -for t in systemstatus epanicclean btrfsmaintstop dynamicipupdate; do - f=/etc/systemd/system/$t.timer - if [[ -e $f ]]; then - s systemctl stop $t.timer - s systemctl disable $t.timer - s rm -fv $f - reload_systemd=true - fi -done -rm -f /etc/cron.daily/check-lets-encrypt-ssl-settings c_dirs=(/a/c{,/machine_specific/$HOSTNAME}) case $user in iank) + # old files 2022-03 + for t in systemstatus epanicclean btrfsmaintstop dynamicipupdate; do + f=/etc/systemd/system/$t.timer + if [[ -e $f ]]; then + s systemctl stop $t.timer + s systemctl disable $t.timer + s rm -fv $f + reload_systemd=true + fi + done + # old 2022-04 + if [[ -e /etc/cron.daily/check-lets-encrypt-ssl-settings ]]; then + m s rm -f /etc/cron.daily/check-lets-encrypt-ssl-settings + fi + + /a/bin/ds/install-my-scripts files=(/p/c/machine_specific/*/filesystem/etc/ssh/*_key /p/c/machine_specific/*/filesystem/etc/openvpn/client/*.key diff --git a/distro-begin b/distro-begin index 84900fe..0fee0af 100755 --- a/distro-begin +++ b/distro-begin @@ -280,6 +280,9 @@ sudo /a/exe/lnf /a/c/.vim /a/c/.vimrc /a/c/.gvimrc /root if vps; then pi-nostart bind9 fi +if bitfolk; then + pi-nostart unbound +fi # this needs to be before installing pacserve so we have gpg conf. conflink rootsshsync @@ -319,8 +322,9 @@ case $HOSTNAME in bk) sudo sed -i --follow-symlinks 's/^ *hosts:.*/hosts: files dns myhostname/' /etc/nsswitch.conf soff systemd-resolved - ln -sf 127.0.0.1-resolv/stub-resolv.conf /etc/resolv.conf - sgo bind9 + sudo ln -sf 127.0.0.1-resolv/stub-resolv.conf /etc/resolv.conf + sgo unbound + sgo named # cautious measure to make sure bind is up sleep 1 ;; diff --git a/dynamic-ip-update b/dynamic-ip-update index f377293..1e6cabb 100755 --- a/dynamic-ip-update +++ b/dynamic-ip-update @@ -171,7 +171,7 @@ exit 0 # # push here? # #myunison -ob li # #ssh li conflink -# ssh li.b8.nz systemctl reload bind9 +# ssh li.b8.nz systemctl reload named # # b8.nz has address 65.96.178.16 diff --git a/filesystem/etc/prometheus/rules/iank.yml b/filesystem/etc/prometheus/rules/iank.yml index 971e392..4f3bb04 100644 --- a/filesystem/etc/prometheus/rules/iank.yml +++ b/filesystem/etc/prometheus/rules/iank.yml @@ -115,6 +115,14 @@ groups: annotations: summary: '12 minutes down' + - alert: mailtest_check_unexpected_spamd_vps + expr: |- + mailtest_check_unexpected_spamd_results >= 1 + labels: + severity: day + annotations: + summary: 'jr -u mailtest-check -e' + - alert: mailtest_check_mailhost expr: |- time() - max by (folder,from) (mailtest_check_last_usec{job="node"}) >= 60 * 12 @@ -123,14 +131,14 @@ groups: annotations: summary: '12 minutes down' - # 42 mins: enough for a 30 min queue run plus 12 + # 20 minutes. just allow for more due to prod alert. - alert: mailtest_check_gnu_mailhost expr: |- - time() - max by (folder,from) (mailtest_check_last_usec{folder="/m/md/l/testignore", from="iank@gnu.org"}) >= 60 * 42 + time() - max by (folder,from) (mailtest_check_last_usec{folder="/m/md/l/testignore", from="iank@gnu.org"}) >= 60 * 20 labels: severity: prod annotations: - summary: '42 minutes down' + summary: '20 minutes down' - alert: 1pmtest diff --git a/mail-setup b/mail-setup index 72af2a1..e12149f 100755 --- a/mail-setup +++ b/mail-setup @@ -3303,7 +3303,7 @@ EOFOUTER ;; *) soff mailtest-check.service - rm -fv /etc/cron.d/mailtest + rm -fv /etc/cron.d/mailtest /var/lib/prometheus/node-exporter/mailtest-check.prom* ;; esac diff --git a/mailtest-check b/mailtest-check index e28d5ae..03cf6ab 100755 --- a/mailtest-check +++ b/mailtest-check @@ -29,7 +29,7 @@ getspamdpid() { fi } pr() { - if [[ -e /var/lib/prometheus/node-exporter ]]; then + if $doprom && [[ -e /var/lib/prometheus/node-exporter ]]; then cat >>/var/lib/prometheus/node-exporter/mailtest-check.prom.$$ fi } @@ -66,7 +66,17 @@ fi # TODO, get je to deliver the local mailbox: /m/md/INBOX # dovecot appears to setup, i can t be sure. - +source /a/bin/bash_unpublished/source-state + +doprom=false +case $HOSTNAME in + $MAIL_HOST|bk|je) + doprom=true + ;; + *) + rm -f /var/lib/prometheus/node-exporter/mailtest-check.prom* + ;; +esac main() { diff --git a/system-status b/system-status index 12e0426..3ccde7c 100755 --- a/system-status +++ b/system-status @@ -219,16 +219,6 @@ write-status() { fi p $var_mail_msg | loday -1 var_mail - - tmp=(/var/local/cron-errors/mailtest-check*) - if (( ${#tmp[@]} )); then - chars+=(MAILPING) - fi - tmp=(/var/local/cron-errors/mailtest-slow*) - if (( ${#tmp[@]} )); then - chars+=(SPAMD) - fi - # early in install process, we dont have permission yet for exiqgrep. # 1100 helps allow for system restarts qlen=$(/usr/sbin/exiqgrep -o 1100 -c -b | awk '{print $1}') ||: -- 2.30.2