big fix switch-mail-host plus minor improvements

[distro-setup] / mailtest-check

diff --git a/mailtest-check b/mailtest-check
index c7d40c17502e967b808e2e251a6b50b63cb1464c..b2e02c3112e06a32c7fd0864b2f30e621190155c 100755 (executable)
--- a/mailtest-check
+++ b/mailtest-check
@@ -28,11 +28,6 @@ getspamdpid() {
     done
   fi
 }
-pr() {
-  if $doprom && [[ -e /var/lib/prometheus/node-exporter ]]; then
-    cat >>/var/lib/prometheus/node-exporter/mailtest-check.prom.$$
-  fi
-}
 
 
 #### begin arg processing ####
@@ -57,10 +52,11 @@ if [[ $1 == nonint ]]; then
 fi
 #### end arg processing ####
 
-
-if ! $int; then
-  sleep 60
-fi
+# we put this in to avoid dns errors that happen on reboot,
+# but I want to debug them.
+# if ! $int; then
+#   sleep 60
+# fi
 
 
 # TODO, get je to deliver the local mailbox: /m/md/INBOX
@@ -80,6 +76,7 @@ esac
 
 main() {
 
+  local -a p_unexpected_spamd_results p_missing_dnswl p_last_usec
   case $HOSTNAME in
     bk)
       folders=(/m/md/{expertpathologyreview.com,amnimal.ninja}/testignore)
@@ -103,6 +100,7 @@ main() {
           try_left=$(( try_limit - ( EPOCHSECONDS - try_start_time) ))
           timeout=120 # somewhat arbitrary value
           if (( try_left < 0 )); then
+            echo "mailtest-check: failed to rsync fencepost > $try_limit seconds"
             break
           fi
           if (( try_left < timeout )); then
@@ -144,9 +142,11 @@ EOF
   fi
   tmpfile=$(mktemp)
   declare -i unexpected=0
-  declare -i missing_dnswl=0
   for folder in ${folders[@]}; do
     for from in ${froms[@]}; do
+      declare -i missing_dnswl=0
+      declare -i dnsfail=0
+      declare -i unexpected=0
       latest=
       last_sec=0
 
@@ -195,6 +195,10 @@ EOF
               # it seems like some versions of spamassassin do BODY_SINGLE_WORD, others dont, we dun care.
               # bayes_00 is a new one indicating ham, we dont care if its missing.
               BAYES_00|BODY_SINGLE_WORD|FROM_FMBLA_NEWDOM*|autolearn) : ;;
+
+              # These have somewhat randomly been added and removed, resulting in useless alerts, so ignore them.
+              RCVD_IN_DNSWL_MED|DKIMWL_WL_HIGH) : ;;
+
               SPF_HELO_NEUTRAL)
                 # some of my domains use neutral spf, treat them the same.
                 results[SPF_HELO_PASS]=t
@@ -211,12 +215,7 @@ EOF
           keys=(DKIM_SIGNED DKIM_VALID{,_AU,_EF} SPF_HELO_PASS SPF_PASS TVD_SPACE_RATIO)
           if [[ $to == *@gnu.org && $from == *@gnu.org ]]; then
             keys=(ALL_TRUSTED TVD_SPACE_RATIO)
-          elif [[ $to == *@gnu.org ]]; then
-            # eggs has RCVD_IN_DNSWL_MED
-            keys+=(RCVD_IN_DNSWL_MED)
-          elif [[ $from == *@gnu.org ]]; then
-            # eggs has these
-            keys+=(RCVD_IN_DNSWL_MED DKIMWL_WL_HIGH)
+            # from eggs had DKIMWL_WL_HIGH sometime in 2022, then DKIMWL_WL_MED unti march 2023
           fi
 
           for t in  ${keys[@]}; do
@@ -258,32 +257,36 @@ EOF
             #   echo mailtest-check: cat $latest:
             #   cat $latest
             #   echo mailtest-check: end of cat
-            #   echo "$(tput setaf 5 2>/dev/null ||:)█$(tput sgr0 2>/dev/null||:)%.0s" $(eval echo "{1..${COLUMNS:-60}}")
             #fi
           fi
           rm -f $resultfile
           for r in ${results[@]}; do
             case $r in
-              DKIM_INVALID|T_SPF_TEMPERROR|T_SPF_HELO_TEMPERROR)
-                missing_dnswl+=1
-                ;;
+              # iank: for when we want to handle dns errors differently
+              # DKIM_INVALID|T_SPF_TEMPERROR|T_SPF_HELO_TEMPERROR)
+              #   dnsfail+=1
+              #   ;;
               *)
                 unexpected=$(( unexpected + 1 ))
                 ;;
             esac
           done
           for miss in ${missing[@]}; do
-            # We expect dns failures from time to time, so
-            # we count them separately and alert differently.
+            # At some point we had annoying dns failures that we couldn't solve so we
+            # we counted dns fail related results separately and alert differently.
+            # DKIM_VALID|DKIM_VALID_AU|DKIM_VALID_EF|SPF_HELO_PASS|SPF_PASS|
             case $miss in
-              DKIM_VALID|DKIM_VALID_AU|DKIM_VALID_EF|SPF_HELO_PASS|SPF_PASS|RCVD_IN_DNSWL_MED|DKIMWL_WL_HIGH)
-                missing_dnswl+=1
-                ;;
               *)
                 unexpected+=1
                 ;;
             esac
           done
+          mapfile -O ${#p_missing_dnswl[@]} -t p_missing_dnswl <<EOF
+mailtest_check_missing_dnswl{folder="$folder",from="$from"} $missing_dnswl
+EOF
+          mapfile -O ${#p_unexpected_spamd_results[@]} -t p_unexpected_spamd_results <<EOF
+mailtest_check_unexpected_spamd_results{folder="$folder",from="$from"} $unexpected
+EOF
         fi # if spamdpid
       fi # if $slow
 
@@ -292,21 +295,25 @@ EOF
       e $((age_sec / 60)):$(( age_sec % 60 )) ago. to:$to from:$from $latest
 
       # usec = unix seconds
-      pr <<EOF
+      mapfile -O ${#p_last_usec[@]} -t p_last_usec <<EOF
 mailtest_check_last_usec{folder="$folder",from="$from"} $last_sec
 EOF
     done # end for from in ${froms[@]}
   done # end for folder in ${folders[@]}
-  if $slow; then
-    pr <<EOF
-mailtest_check_missing_dnswl $missing_dnswl
-mailtest_check_unexpected_spamd_results $unexpected
-EOF
-  fi
 
   dir=/var/lib/prometheus/node-exporter
-  if [[ -e $dir  ]]; then
-    mv $dir/mailtest-check.prom.$$ $dir/mailtest-check.prom
+  path=$dir/mailtest-check.prom.$$
+  if $doprom && [[ -e $dir  ]]; then
+    for l in "${p_unexpected_spamd_results[@]}"; do
+      printf "%s\n" "$l" >>$path
+      done
+    for l in "${p_missing_dnswl[@]}"; do
+      printf "%s\n" "$l" >>$path
+    done
+    for l in "${p_last_usec[@]}"; do
+      printf "%s\n" "$l" >>$path
+    done
+    mv $path $dir/mailtest-check.prom
     # note: node_textfile_mtime_seconds will tell us when this last happened. useful for debugging.
   fi
 }