X-Git-Url: https://iankelling.org/git/?a=blobdiff_plain;f=mailtest-check;h=7454086099322db19eec59bee04471d555fc297d;hb=d7551546ac323c5d4b49370c885646bcf96e959f;hp=466908d30c55a92b32cac3b92f8f0bba1fb30f4a;hpb=0249e646cbed14fc191827f95a05044287d419ad;p=distro-setup

diff --git a/mailtest-check b/mailtest-check
index 466908d..7454086 100755
--- a/mailtest-check
+++ b/mailtest-check
@@ -21,7 +21,9 @@ e() { $int || return 0; printf "mailtest-check: %s\n" "$*"; }
 # We run this cronjob along with sending the test email every 5 minutes,
 # so give it 1 minute to arrive, then if the latest email is older than
 # 7 minutes, the last 2 haven't arrived in a reasonable amount of time.
-min_limit=7
+# However, when machines reboot things can get delayed, so add 10 mins,
+# not sure if that is a good number or not.
+min_limit=17
 
 
 # spamassassin checking takes about 8 seconds. only do that every
@@ -69,23 +71,39 @@ case $HOSTNAME in
     folders=(/m/md/l/testignore)
     froms=(testignore@je.b8.nz testignore@expertpathologyreview.com testignore@amnimal.ninja ian@iankelling.org z@zroe.org iank@gnu.org)
     if ! $int; then
-      timeout 120 rsync -e "ssh -oIdentitiesOnly=yes -F /dev/null -i /root/.ssh/jtuttle" -t --inplace -r 'jtuttle@fencepost.gnu.org:/home/j/jtuttle/Maildir/new/' /m/md/l/testignore/new
+      timeout 120 rsync --chown iank:iank -e "ssh -oIdentitiesOnly=yes -F /dev/null -i /root/.ssh/jtuttle" -t --inplace -r 'jtuttle@fencepost.gnu.org:/home/j/jtuttle/Maildir/new/' /m/md/l/testignore/new
     fi
     ;;
 esac
 
 getspamdpid() {
   if [[ ! $spamdpid || ! -d /proc/$spamdpid ]]; then
-    spamdpid=$(systemctl status spamassassin| sed -n '/^ *Main PID:/s/[^0-9]//gp' ||:)
+    # try twice in case we are restarting, it happens.
+    for i in 1 2; do
+      spamdpid=$(systemctl show --property MainPID --value spamassassin | sed 's/^[10]$//' ||:)
+      if [[ $spamdpid ]]; then
+        break
+      fi
+      sleep 30
+    done
   fi
 }
 getspamdpid
+pr() {
+  if [[ -e /var/lib/prometheus/node-exporter ]]; then
+    cat >>/var/lib/prometheus/node-exporter/mailtest-check.prom.$$
+  fi
+}
+pr <<EOF
+mailtest_check_found_spamd_pid_bool $(( ${spamdpid:-0} > 0 ))
+EOF
 e spamdpid: $spamdpid
 if [[ ! $spamdpid ]]; then
   echo $HOSTNAME mailtest spamd pid not found. systemctl status spamassassin:
   systemctl status spamassassin
 fi
 tmpfile=$(mktemp)
+declare -i unexpected=0
 for folder in ${folders[@]}; do
   for from in ${froms[@]}; do
     latest=
@@ -102,7 +120,10 @@ for folder in ${folders[@]}; do
       fi
     done <$tmpfile
 
-    if [[ $latest ]]; then
+    if [[ ! $latest ]]; then
+      # 10 is an arbitrary bad value
+      unexpected+=10
+    else
       to=$(awk '/^Envelope-to: / {print $2}' $latest)
       last_sec=$(awk '/^Subject: / {print $4}' $latest)
 
@@ -121,7 +142,8 @@ for folder in ${folders[@]}; do
           # servers.
           # example line that sed is parsing:
           # (-0.1 / 5.0 requ) DKIM_SIGNED=0.1,DKIM_VALID=-0.1,DKIM_VALID_AU=-0.1,SPF_HELO_PASS=-0.001,SPF_PASS=-0.001,TVD_SPACE_RATIO=0.001 autolearn=_AUTOLEARN
-          for r in $($spamcpre sudo -u Debian-exim spamassassin -t --cf='score PYZOR_CHECK 0' <"$latest" | tail -n2 | head -n1 | sed -r 's/^\([^)]*\) *//;s/=[^, ]*([, ]|$)/ /g'); do
+          raw_results="$($spamcpre sudo -u Debian-exim spamassassin -t --cf='score PYZOR_CHECK 0' <"$latest" | tail -n2 | head -n1 | sed -r 's/^\([^)]*\) *//;s/=[^, ]*([, ]|$)/ /g')"
+          for r in $raw_results; do
             case $r in
               # got this in an update 2022-01. dun care
               T_SCC_BODY_TEXT_LINE|SCC_BODY_SINGLE_WORD) : ;;
@@ -196,5 +218,19 @@ for folder in ${folders[@]}; do
     if (( last_sec <= limit )); then
       echo $HOSTNAME mailtest $folder $from $(date -d @$last_sec +'%a %m-%d %H:%M')
     fi
+    # usec = unix seconds
+    pr <<EOF
+mailtest_check_last_usec{folder="$folder",from="$from"} $last_sec
+EOF
   done
 done
+if $slow; then
+  pr <<EOF
+mailtest_check_unexpected_spamd_results $unexpected
+EOF
+fi
+
+dir=/var/lib/prometheus/node-exporter
+if [[ -e $dir  ]]; then
+  mv $dir/mailtest-check.prom.$$ $dir/mailtest-check.prom
+fi