X-Git-Url: https://iankelling.org/git/?a=blobdiff_plain;f=mailtest-check;h=599d4b6a4157ab0ddc31e74b58b0766e8cc61bc4;hb=55d37e930d1cd5b3b9800d33dc62f282050899e9;hp=67cca9ca84c9e347289306646f05c1aedb8601c8;hpb=95eb9558206f8287febab80dd3f51d168a3ca831;p=distro-setup

diff --git a/mailtest-check b/mailtest-check
index 67cca9c..599d4b6 100755
--- a/mailtest-check
+++ b/mailtest-check
@@ -28,11 +28,6 @@ getspamdpid() {
     done
   fi
 }
-pr() {
-  if $doprom && [[ -e /var/lib/prometheus/node-exporter ]]; then
-    cat >>/var/lib/prometheus/node-exporter/mailtest-check.prom.$$
-  fi
-}
 
 
 #### begin arg processing ####
@@ -57,10 +52,11 @@ if [[ $1 == nonint ]]; then
 fi
 #### end arg processing ####
 
-
-if ! $int; then
-  sleep 60
-fi
+# we put this in to avoid dns errors that happen on reboot,
+# but I want to debug them.
+# if ! $int; then
+#   sleep 60
+# fi
 
 
 # TODO, get je to deliver the local mailbox: /m/md/INBOX
@@ -80,6 +76,7 @@ esac
 
 main() {
 
+  local -a unexpected_spamd_results missing_dnswl last_usec
   case $HOSTNAME in
     bk)
       folders=(/m/md/{expertpathologyreview.com,amnimal.ninja}/testignore)
@@ -93,7 +90,32 @@ main() {
       folders=(/m/md/l/testignore)
       froms=(testignore@je.b8.nz testignore@expertpathologyreview.com testignore@amnimal.ninja ian@iankelling.org z@zroe.org iank@gnu.org)
       if ! $int; then
-        timeout 120 rsync --chown iank:iank -e "ssh -oIdentitiesOnly=yes -F /dev/null -i /root/.ssh/jtuttle" -t --inplace -r 'jtuttle@fencepost.gnu.org:/home/j/jtuttle/Maildir/new/' /m/md/l/testignore/new
+        ### begin rsyncing fencepost email ###
+        # We dont want to exit if rsync fails, that will get caught by
+        # our later test by virtue of not having the latest email.
+        did_rsync=false
+        try_start_time=$EPOCHSECONDS
+        try_limit=140 # somewhat arbitrary value
+        while ! $did_rsync; do
+          try_left=$(( try_limit - ( EPOCHSECONDS - try_start_time) ))
+          timeout=120 # somewhat arbitrary value
+          if (( try_left < 0 )); then
+            echo "mailtest-check: failed to rsync fencepost > $try_limit seconds"
+            break
+          fi
+          if (( try_left < timeout )); then
+            timeout=$try_left
+          fi
+          if timeout $timeout rsync --chown iank:iank -e "ssh -oIdentitiesOnly=yes -F /dev/null -i /root/.ssh/jtuttle" -t --inplace -r 'jtuttle@fencepost.gnu.org:/home/j/jtuttle/Maildir/new/' /m/md/l/testignore/new; then
+            did_rsync=true
+          else
+            sleep 4
+          fi
+        done
+        if ! $did_rsync; then
+          echo mailtest-check: warning: fencepost rsync failed
+        fi
+        ### end rsyncing fencepost email ###
       fi
       ;;
   esac
@@ -120,9 +142,11 @@ EOF
   fi
   tmpfile=$(mktemp)
   declare -i unexpected=0
-  declare -i missing_dnswl=0
   for folder in ${folders[@]}; do
     for from in ${froms[@]}; do
+      declare -i missing_dnswl=0
+      declare -i dnsfail=0
+      declare -i unexpected=0
       latest=
       last_sec=0
 
@@ -138,6 +162,7 @@ EOF
           last_sec="$file_sec"
         fi
       done <$tmpfile
+      rm -f $tmpfile
 
       to=$(awk '/^Envelope-to: / {print $2}' $latest)
 
@@ -190,8 +215,8 @@ EOF
             # eggs has RCVD_IN_DNSWL_MED
             keys+=(RCVD_IN_DNSWL_MED)
           elif [[ $from == *@gnu.org ]]; then
-            # eggs has these
-            keys+=(RCVD_IN_DNSWL_MED DKIMWL_WL_HIGH)
+            # eggs has this. it used to have DKIMWL_WL_HIGH sometime in 2022
+            keys+=(RCVD_IN_DNSWL_MED)
           fi
 
           for t in  ${keys[@]}; do
@@ -237,11 +262,23 @@ EOF
             #fi
           fi
           rm -f $resultfile
-          unexpected=$(( unexpected + ${#results[@]} ))
+          for r in ${results[@]}; do
+            case $r in
+              # iank: for when we want to handle dns errors differently
+              # DKIM_INVALID|T_SPF_TEMPERROR|T_SPF_HELO_TEMPERROR)
+              #   dnsfail+=1
+              #   ;;
+              *)
+                unexpected=$(( unexpected + 1 ))
+                ;;
+            esac
+          done
           for miss in ${missing[@]}; do
-            # We expect dns reputation services to go down from time to time, so
+            # We expect dns failures from time to time, so
             # we count them separately and alert differently.
             case $miss in
+              # iank: dns fail
+              # DKIM_VALID|DKIM_VALID_AU|DKIM_VALID_EF|SPF_HELO_PASS|SPF_PASS|
               RCVD_IN_DNSWL_MED|DKIMWL_WL_HIGH)
                 missing_dnswl+=1
                 ;;
@@ -250,6 +287,12 @@ EOF
                 ;;
             esac
           done
+          mapfile -O ${#missing_dnswl[@]} -t missing_dnswl <<EOF
+mailtest_check_missing_dnswl{folder="$folder",from="$from"} $missing_dnswl
+EOF
+          mapfile -O ${#unexpected_spamd_results[@]} -t unexpected_spamd_results <<EOF
+mailtest_check_unexpected_spamd_results{folder="$folder",from="$from"} $unexpected
+EOF
         fi # if spamdpid
       fi # if $slow
 
@@ -258,21 +301,25 @@ EOF
       e $((age_sec / 60)):$(( age_sec % 60 )) ago. to:$to from:$from $latest
 
       # usec = unix seconds
-      pr <<EOF
+      mapfile -O ${#last_usec[@]} -t last_usec <<EOF
 mailtest_check_last_usec{folder="$folder",from="$from"} $last_sec
 EOF
     done # end for from in ${froms[@]}
   done # end for folder in ${folders[@]}
-  if $slow; then
-    pr <<EOF
-mailtest_check_missing_dnswl $missing_dnswl
-mailtest_check_unexpected_spamd_results $unexpected
-EOF
-  fi
 
   dir=/var/lib/prometheus/node-exporter
-  if [[ -e $dir  ]]; then
-    mv $dir/mailtest-check.prom.$$ $dir/mailtest-check.prom
+  path=$dir/mailtest-check.prom.$$
+  if $doprom && [[ -e $dir  ]]; then
+    for l in "${unexpected_spamd_results missing_dnswl[@]}"; do
+      printf "%s\n" "$l" >>$path
+      done
+    for l in "${missing_dnswl[@]}"; do
+      printf "%s\n" "$l" >>$path
+    done
+    for l in "${last_usec[@]}"; do
+      printf "%s\n" "$l" >>$path
+    done
+    mv $path $dir/mailtest-check.prom
     # note: node_textfile_mtime_seconds will tell us when this last happened. useful for debugging.
   fi
 }
@@ -280,7 +327,7 @@ EOF
 loop-main() {
   # When running under systemd, the system just started. Ve nice and
   # give programs some time to finish their startup.
-  sleep 5
+  sleep 10
   while true; do
     premain_sec=$EPOCHSECONDS
     main