From: Ian Kelling <ian@iankelling.org>
Date: Sun, 24 Apr 2022 21:10:23 +0000 (-0400)
Subject: fixes and alert improvements
X-Git-Url: https://iankelling.org/git/?p=distro-setup;a=commitdiff_plain;h=56c55d8e02cdd6ec67d2fe53cc03785d8876442e

fixes and alert improvements
---

diff --git a/brc2 b/brc2
index f3c4b24..d85511f 100644
--- a/brc2
+++ b/brc2
@@ -541,15 +541,15 @@ EOF
 
 dnsup() {
   conflink -f
-  m ser reload bind9
+  m ser reload named
 }
 dnsb8() {
   local f=/var/lib/bind/db.b8.nz
-  m ser stop bind9
+  m ser stop named
   m sleep 1
   m sudo rm -fv $f.jnl
   m sudo install -m 644 -o bind -g bind /p/c/machine_specific/vps/bind-initial/db.b8.nz $f
-  m ser restart bind9
+  m ser restart named
 }
 dnsecgen() {
   # keys generated like this
diff --git a/conflink b/conflink
index a3f6e35..e53605e 100755
--- a/conflink
+++ b/conflink
@@ -147,21 +147,26 @@ for x in /p/c/machine_specific/*.hosts /a/bin/ds/machine_specific/*.hosts; do
   if grep -qxF $HOSTNAME $x; then all_dirs+=( ${x%.hosts} ); fi
 done
 
-# old files 2022-03
-for t in systemstatus epanicclean btrfsmaintstop dynamicipupdate; do
-  f=/etc/systemd/system/$t.timer
-  if [[ -e $f ]]; then
-    s systemctl stop $t.timer
-    s systemctl disable $t.timer
-    s rm -fv $f
-    reload_systemd=true
-  fi
-done
-rm -f /etc/cron.daily/check-lets-encrypt-ssl-settings
 
 c_dirs=(/a/c{,/machine_specific/$HOSTNAME})
 case $user in
   iank)
+    # old files 2022-03
+    for t in systemstatus epanicclean btrfsmaintstop dynamicipupdate; do
+      f=/etc/systemd/system/$t.timer
+      if [[ -e $f ]]; then
+        s systemctl stop $t.timer
+        s systemctl disable $t.timer
+        s rm -fv $f
+        reload_systemd=true
+      fi
+    done
+    # old 2022-04
+    if [[ -e /etc/cron.daily/check-lets-encrypt-ssl-settings ]]; then
+      m s rm -f /etc/cron.daily/check-lets-encrypt-ssl-settings
+    fi
+
+
     /a/bin/ds/install-my-scripts
     files=(/p/c/machine_specific/*/filesystem/etc/ssh/*_key
            /p/c/machine_specific/*/filesystem/etc/openvpn/client/*.key
diff --git a/distro-begin b/distro-begin
index 84900fe..0fee0af 100755
--- a/distro-begin
+++ b/distro-begin
@@ -280,6 +280,9 @@ sudo /a/exe/lnf /a/c/.vim /a/c/.vimrc /a/c/.gvimrc /root
 if vps; then
   pi-nostart bind9
 fi
+if bitfolk; then
+  pi-nostart unbound
+fi
 # this needs to be before installing pacserve so we have gpg conf.
 conflink
 rootsshsync
@@ -319,8 +322,9 @@ case $HOSTNAME in
   bk)
     sudo sed -i --follow-symlinks 's/^ *hosts:.*/hosts: files dns myhostname/' /etc/nsswitch.conf
     soff systemd-resolved
-    ln -sf 127.0.0.1-resolv/stub-resolv.conf /etc/resolv.conf
-    sgo bind9
+    sudo ln -sf 127.0.0.1-resolv/stub-resolv.conf /etc/resolv.conf
+    sgo unbound
+    sgo named
     # cautious measure to make sure bind is up
     sleep 1
     ;;
diff --git a/dynamic-ip-update b/dynamic-ip-update
index f377293..1e6cabb 100755
--- a/dynamic-ip-update
+++ b/dynamic-ip-update
@@ -171,7 +171,7 @@ exit 0
 #   # push here?
 #   #myunison -ob li
 #   #ssh li conflink
-#   ssh li.b8.nz systemctl reload bind9
+#   ssh li.b8.nz systemctl reload named
 
 
 #   # b8.nz has address 65.96.178.16
diff --git a/filesystem/etc/prometheus/rules/iank.yml b/filesystem/etc/prometheus/rules/iank.yml
index 971e392..4f3bb04 100644
--- a/filesystem/etc/prometheus/rules/iank.yml
+++ b/filesystem/etc/prometheus/rules/iank.yml
@@ -115,6 +115,14 @@ groups:
     annotations:
       summary: '12 minutes down'
 
+  - alert: mailtest_check_unexpected_spamd_vps
+    expr: |-
+      mailtest_check_unexpected_spamd_results >= 1
+    labels:
+      severity: day
+    annotations:
+      summary: 'jr -u mailtest-check -e'
+
   - alert: mailtest_check_mailhost
     expr: |-
       time() - max by (folder,from) (mailtest_check_last_usec{job="node"}) >= 60 * 12
@@ -123,14 +131,14 @@ groups:
     annotations:
       summary: '12 minutes down'
 
-  # 42 mins: enough for a 30 min queue run plus 12
+  # 20 minutes. just allow for more due to prod alert.
   - alert: mailtest_check_gnu_mailhost
     expr: |-
-      time() - max by (folder,from) (mailtest_check_last_usec{folder="/m/md/l/testignore", from="iank@gnu.org"}) >= 60 * 42
+      time() - max by (folder,from) (mailtest_check_last_usec{folder="/m/md/l/testignore", from="iank@gnu.org"}) >= 60 * 20
     labels:
       severity: prod
     annotations:
-      summary: '42 minutes down'
+      summary: '20 minutes down'
 
 
   - alert: 1pmtest
diff --git a/mail-setup b/mail-setup
index 72af2a1..e12149f 100755
--- a/mail-setup
+++ b/mail-setup
@@ -3303,7 +3303,7 @@ EOFOUTER
     ;;
   *)
     soff mailtest-check.service
-    rm -fv /etc/cron.d/mailtest
+    rm -fv /etc/cron.d/mailtest /var/lib/prometheus/node-exporter/mailtest-check.prom*
     ;;
 esac
 
diff --git a/mailtest-check b/mailtest-check
index e28d5ae..03cf6ab 100755
--- a/mailtest-check
+++ b/mailtest-check
@@ -29,7 +29,7 @@ getspamdpid() {
   fi
 }
 pr() {
-  if [[ -e /var/lib/prometheus/node-exporter ]]; then
+  if $doprom && [[ -e /var/lib/prometheus/node-exporter ]]; then
     cat >>/var/lib/prometheus/node-exporter/mailtest-check.prom.$$
   fi
 }
@@ -66,7 +66,17 @@ fi
 # TODO, get je to deliver the local mailbox: /m/md/INBOX
 # dovecot appears to setup, i can t be sure.
 
-
+source /a/bin/bash_unpublished/source-state
+
+doprom=false
+case $HOSTNAME in
+  $MAIL_HOST|bk|je)
+    doprom=true
+    ;;
+  *)
+    rm -f /var/lib/prometheus/node-exporter/mailtest-check.prom*
+    ;;
+esac
 
 main() {
 
diff --git a/system-status b/system-status
index 12e0426..3ccde7c 100755
--- a/system-status
+++ b/system-status
@@ -219,16 +219,6 @@ write-status() {
   fi
   p $var_mail_msg | loday -1 var_mail
 
-
-  tmp=(/var/local/cron-errors/mailtest-check*)
-  if (( ${#tmp[@]} )); then
-    chars+=(MAILPING)
-  fi
-  tmp=(/var/local/cron-errors/mailtest-slow*)
-  if (( ${#tmp[@]} )); then
-    chars+=(SPAMD)
-  fi
-
   # early in install process, we dont have permission yet for exiqgrep.
   # 1100 helps allow for system restarts
   qlen=$(/usr/sbin/exiqgrep -o 1100 -c -b | awk '{print $1}') ||: