mostly fixes, some improvements

[distro-setup] / mailtest-check

diff --git a/mailtest-check b/mailtest-check
index cce5908fb78267d53c06a8bb224df0248b40daca..42245111248bf7e68f2151345ac373e59132c5c7 100755 (executable)
--- a/mailtest-check
+++ b/mailtest-check
@@ -28,11 +28,6 @@ getspamdpid() {
     done
   fi
 }
-pr() {
-  if $doprom && [[ -e /var/lib/prometheus/node-exporter ]]; then
-    cat >>/var/lib/prometheus/node-exporter/mailtest-check.prom.$$
-  fi
-}
 
 
 #### begin arg processing ####
@@ -81,6 +76,7 @@ esac
 
 main() {
 
+  local -a p_unexpected_spamd_results p_missing_dnswl p_last_usec
   case $HOSTNAME in
     bk)
       folders=(/m/md/{expertpathologyreview.com,amnimal.ninja}/testignore)
@@ -104,6 +100,7 @@ main() {
           try_left=$(( try_limit - ( EPOCHSECONDS - try_start_time) ))
           timeout=120 # somewhat arbitrary value
           if (( try_left < 0 )); then
+            echo "mailtest-check: failed to rsync fencepost > $try_limit seconds"
             break
           fi
           if (( try_left < timeout )); then
@@ -145,10 +142,11 @@ EOF
   fi
   tmpfile=$(mktemp)
   declare -i unexpected=0
-  declare -i missing_dnswl=0
-  declare -i dnsfail=0
   for folder in ${folders[@]}; do
     for from in ${froms[@]}; do
+      declare -i missing_dnswl=0
+      declare -i dnsfail=0
+      declare -i unexpected=0
       latest=
       last_sec=0
 
@@ -197,6 +195,10 @@ EOF
               # it seems like some versions of spamassassin do BODY_SINGLE_WORD, others dont, we dun care.
               # bayes_00 is a new one indicating ham, we dont care if its missing.
               BAYES_00|BODY_SINGLE_WORD|FROM_FMBLA_NEWDOM*|autolearn) : ;;
+
+              # These have somewhat randomly been added and removed, resulting in useless alerts, so ignore them.
+              RCVD_IN_DNSWL_MED|DKIMWL_WL_HIGH) : ;;
+
               SPF_HELO_NEUTRAL)
                 # some of my domains use neutral spf, treat them the same.
                 results[SPF_HELO_PASS]=t
@@ -213,12 +215,7 @@ EOF
           keys=(DKIM_SIGNED DKIM_VALID{,_AU,_EF} SPF_HELO_PASS SPF_PASS TVD_SPACE_RATIO)
           if [[ $to == *@gnu.org && $from == *@gnu.org ]]; then
             keys=(ALL_TRUSTED TVD_SPACE_RATIO)
-          elif [[ $to == *@gnu.org ]]; then
-            # eggs has RCVD_IN_DNSWL_MED
-            keys+=(RCVD_IN_DNSWL_MED)
-          elif [[ $from == *@gnu.org ]]; then
-            # eggs has these
-            keys+=(RCVD_IN_DNSWL_MED DKIMWL_WL_HIGH)
+            # from eggs had DKIMWL_WL_HIGH sometime in 2022, then DKIMWL_WL_MED unti march 2023
           fi
 
           for t in  ${keys[@]}; do
@@ -276,19 +273,21 @@ EOF
             esac
           done
           for miss in ${missing[@]}; do
-            # We expect dns failures from time to time, so
-            # we count them separately and alert differently.
+            # At some point we had annoying dns failures that we couldn't solve so we
+            # we counted dns fail related results separately and alert differently.
+            # DKIM_VALID|DKIM_VALID_AU|DKIM_VALID_EF|SPF_HELO_PASS|SPF_PASS|
             case $miss in
-              # iank: dns fail
-              # DKIM_VALID|DKIM_VALID_AU|DKIM_VALID_EF|SPF_HELO_PASS|SPF_PASS|
-              RCVD_IN_DNSWL_MED|DKIMWL_WL_HIGH)
-                missing_dnswl+=1
-                ;;
               *)
                 unexpected+=1
                 ;;
             esac
           done
+          mapfile -O ${#p_missing_dnswl[@]} -t p_missing_dnswl <<EOF
+mailtest_check_missing_dnswl{folder="$folder",from="$from"} $missing_dnswl
+EOF
+          mapfile -O ${#p_unexpected_spamd_results[@]} -t p_unexpected_spamd_results <<EOF
+mailtest_check_unexpected_spamd_results{folder="$folder",from="$from"} $unexpected
+EOF
         fi # if spamdpid
       fi # if $slow
 
@@ -297,21 +296,25 @@ EOF
       e $((age_sec / 60)):$(( age_sec % 60 )) ago. to:$to from:$from $latest
 
       # usec = unix seconds
-      pr <<EOF
+      mapfile -O ${#p_last_usec[@]} -t p_last_usec <<EOF
 mailtest_check_last_usec{folder="$folder",from="$from"} $last_sec
 EOF
     done # end for from in ${froms[@]}
   done # end for folder in ${folders[@]}
-  if $slow; then
-    pr <<EOF
-mailtest_check_missing_dnswl $missing_dnswl
-mailtest_check_unexpected_spamd_results $unexpected
-EOF
-  fi
 
   dir=/var/lib/prometheus/node-exporter
-  if [[ -e $dir  ]]; then
-    mv $dir/mailtest-check.prom.$$ $dir/mailtest-check.prom
+  path=$dir/mailtest-check.prom.$$
+  if $doprom && [[ -e $dir  ]]; then
+    for l in "${p_unexpected_spamd_results[@]}"; do
+      printf "%s\n" "$l" >>$path
+      done
+    for l in "${p_missing_dnswl[@]}"; do
+      printf "%s\n" "$l" >>$path
+    done
+    for l in "${p_last_usec[@]}"; do
+      printf "%s\n" "$l" >>$path
+    done
+    mv $path $dir/mailtest-check.prom
     # note: node_textfile_mtime_seconds will tell us when this last happened. useful for debugging.
   fi
 }