iankelling.org Git - distro-setup/blob - mailtest-check

   1 #!/bin/bash
   2
   3 # Usage: mail-test-check [slow] [anything]
   4 #
   5 # slow: do slow checks, like spamassassin
   6 #
   7 # anything: consider non-interactive, dont print unless something went
   8 # wrong
   9
  10
  11 source /b/errhandle/err
  12
  13 [[ $EUID == 0 ]] || exec sudo -E "${BASH_SOURCE[0]}" "$@"
  14
  15 shopt -s nullglob
  16
  17 e() { $int || return 0; printf "mailtest-check: %s\n" "$*"; }
  18
  19 getspamdpid() {
  20   if [[ ! $spamdpid || ! -d /proc/$spamdpid ]]; then
  21     # try twice in case we are restarting, it happens.
  22     for i in 1 2; do
  23       spamdpid=$(systemctl show --property MainPID --value spamassassin | sed 's/^[10]$//' ||:)
  24       if [[ $spamdpid ]]; then
  25         break
  26       fi
  27       sleep 30
  28     done
  29   fi
  30 }
  31 pr() {
  32   if $doprom && [[ -e /var/lib/prometheus/node-exporter ]]; then
  33     cat >>/var/lib/prometheus/node-exporter/mailtest-check.prom.$$
  34   fi
  35 }
  36
  37
  38 #### begin arg processing ####
  39 # spamassassin checking takes about 8 seconds.
  40 slow=false
  41 if [[ $1 == slow ]]; then
  42   slow=true
  43   shift
  44 fi
  45
  46 int=false
  47 if [[ $SUDO_USER || $SSH_CONNECTION ]]; then
  48   int=true
  49 fi
  50
  51 if [[ $1 == int ]]; then
  52   int=true
  53 fi
  54
  55 if [[ $1 == nonint ]]; then
  56   int=false
  57 fi
  58 #### end arg processing ####
  59
  60 # we put this in to avoid dns errors that happen on reboot,
  61 # but I want to debug them.
  62 # if ! $int; then
  63 #   sleep 60
  64 # fi
  65
  66
  67 # TODO, get je to deliver the local mailbox: /m/md/INBOX
  68 # dovecot appears to setup, i can t be sure.
  69
  70 source /a/bin/bash_unpublished/source-state
  71
  72 doprom=false
  73 case $HOSTNAME in
  74   $MAIL_HOST|bk|je)
  75     doprom=true
  76     ;;
  77   *)
  78     rm -f /var/lib/prometheus/node-exporter/mailtest-check.prom*
  79     ;;
  80 esac
  81
  82 main() {
  83
  84   case $HOSTNAME in
  85     bk)
  86       folders=(/m/md/{expertpathologyreview.com,amnimal.ninja}/testignore)
  87       froms=(ian@iankelling.org z@zroe.org testignore@je.b8.nz iank@gnu.org)
  88       ;;
  89     je)
  90       froms=(ian@iankelling.org z@zroe.org iank@gnu.org testignore@amnimal.ninja)
  91       folders=(/m/md/je.b8.nz/testignore)
  92       ;;
  93     *)
  94       folders=(/m/md/l/testignore)
  95       froms=(testignore@je.b8.nz testignore@expertpathologyreview.com testignore@amnimal.ninja ian@iankelling.org z@zroe.org iank@gnu.org)
  96       if ! $int; then
  97         ### begin rsyncing fencepost email ###
  98         # We dont want to exit if rsync fails, that will get caught by
  99         # our later test by virtue of not having the latest email.
 100         did_rsync=false
 101         try_start_time=$EPOCHSECONDS
 102         try_limit=140 # somewhat arbitrary value
 103         while ! $did_rsync; do
 104           try_left=$(( try_limit - ( EPOCHSECONDS - try_start_time) ))
 105           timeout=120 # somewhat arbitrary value
 106           if (( try_left < 0 )); then
 107             echo "mailtest-check: failed to rsync fencepost > $try_limit seconds"
 108             break
 109           fi
 110           if (( try_left < timeout )); then
 111             timeout=$try_left
 112           fi
 113           if timeout $timeout rsync --chown iank:iank -e "ssh -oIdentitiesOnly=yes -F /dev/null -i /root/.ssh/jtuttle" -t --inplace -r 'jtuttle@fencepost.gnu.org:/home/j/jtuttle/Maildir/new/' /m/md/l/testignore/new; then
 114             did_rsync=true
 115           else
 116             sleep 4
 117           fi
 118         done
 119         if ! $did_rsync; then
 120           echo mailtest-check: warning: fencepost rsync failed
 121         fi
 122         ### end rsyncing fencepost email ###
 123       fi
 124       ;;
 125   esac
 126
 127
 128   # avoid errors like this:
 129   # Nov  8 08:16:05.439 [6080] warn: plugin: failed to parse plugin (from @INC): Can't locate Mail/SpamAssassin/Plugin/WLBLEval.pm: lib/Mail/SpamAssassin/Plugin/WLBLEval.pm: Permission denied at (eval 59) line 1.
 130   #Nov  8 08:16:05.439 [6080] warn: plugin: failed to parse plugin (from @INC): Can't locate Mail/SpamAssassin/Plugin/VBounce.pm: lib/Mail/SpamAssassin/Plugin/VBounce.pm: Permission denied at (eval 60) line 1.
 131   # i dont know why, i just found the solution online
 132   cd /m/md
 133
 134
 135   getspamdpid
 136   # first time we write, overwrite anything existing
 137   if [[ -e /var/lib/prometheus/node-exporter ]]; then
 138     cat >/var/lib/prometheus/node-exporter/mailtest-check.prom.$$ <<EOF
 139 mailtest_check_found_spamd_pid_bool $(( ${spamdpid:-0} > 0 ))
 140 EOF
 141   fi
 142   e spamdpid: $spamdpid
 143   if [[ ! $spamdpid ]]; then
 144     echo mailtest spamd pid not found. systemctl status spamassassin:
 145     systemctl status spamassassin
 146   fi
 147   tmpfile=$(mktemp)
 148   declare -i unexpected=0
 149   for folder in ${folders[@]}; do
 150     for from in ${froms[@]}; do
 151       declare -i missing_dnswl=0
 152       declare -i dnsfail=0
 153       declare -i unexpected=0
 154       latest=
 155       last_sec=0
 156
 157       if ! grep -rlFx "From: $from" $folder/{new,cur} >$tmpfile; then
 158         echo "no message found from: $from"
 159         continue
 160       fi
 161       # webmail sends them to cur it seems
 162       while read -r file; do
 163         file_sec=$(awk '/^Subject: / {print $4}' $file)
 164         if [[ $file_sec ]] && (( file_sec > last_sec )); then
 165           latest=$file
 166           last_sec="$file_sec"
 167         fi
 168       done <$tmpfile
 169       rm -f $tmpfile
 170
 171       to=$(awk '/^Envelope-to: / {print $2}' $latest)
 172
 173       if $slow; then
 174         if ! $int; then
 175           find $folder/new $folder/cur -type f -mmin +1080 -delete
 176         fi
 177         getspamdpid
 178         if [[ $spamdpid ]]; then
 179           if [[ $(readlink /proc/$$/ns/net) != "$(readlink /proc/$spamdpid/ns/net)" ]]; then
 180             spamcpre="nsenter -t $spamdpid -n -m"
 181           fi
 182           unset results
 183           declare -A results
 184           # pyzor fails for our test message, so dont put useless load on their
 185           # servers.
 186           # example line that sed is parsing:
 187           # (-0.1 / 5.0 requ) DKIM_SIGNED=0.1,DKIM_VALID=-0.1,DKIM_VALID_AU=-0.1,SPF_HELO_PASS=-0.001,SPF_PASS=-0.001,TVD_SPACE_RATIO=0.001 autolearn=_AUTOLEARN
 188           resultfile=$(mktemp)
 189           $spamcpre sudo -u Debian-exim spamassassin -D -t --cf='score PYZOR_CHECK 0' <"$latest" &>$resultfile
 190
 191           # note: on some mail, its 1 line after the send-test-forward, on others its 2 with a blank inbetween.
 192           # I use the sed -n to filter this.
 193           raw_results="$(tail $resultfile | grep -A2 -Fx /usr/local/bin/send-test-forward | tail -n+2 | sed -nr 's/^\([^)]*\) *//;s/=[^, ]*([, ]|$)/ /gp')"
 194           for r in $raw_results; do
 195             case $r in
 196               # got this in an update 2022-01. dun care
 197               T_SCC_BODY_TEXT_LINE|SCC_BODY_SINGLE_WORD) : ;;
 198               # we have a new domain, ignore this.
 199               # it seems like some versions of spamassassin do BODY_SINGLE_WORD, others dont, we dun care.
 200               # bayes_00 is a new one indicating ham, we dont care if its missing.
 201               BAYES_00|BODY_SINGLE_WORD|FROM_FMBLA_NEWDOM*|autolearn) : ;;
 202               SPF_HELO_NEUTRAL)
 203                 # some of my domains use neutral spf, treat them the same.
 204                 results[SPF_HELO_PASS]=t
 205                 ;;
 206               *)
 207                 results[$r]=t
 208                 ;;
 209             esac
 210           done
 211           # debugging
 212           # e results = ${!results[@]}
 213           missing=()
 214
 215           keys=(DKIM_SIGNED DKIM_VALID{,_AU,_EF} SPF_HELO_PASS SPF_PASS TVD_SPACE_RATIO)
 216           if [[ $to == *@gnu.org && $from == *@gnu.org ]]; then
 217             keys=(ALL_TRUSTED TVD_SPACE_RATIO)
 218           elif [[ $to == *@gnu.org ]]; then
 219             # eggs has RCVD_IN_DNSWL_MED
 220             keys+=(RCVD_IN_DNSWL_MED)
 221           elif [[ $from == *@gnu.org ]]; then
 222             # eggs has this. it used to have DKIMWL_WL_HIGH sometime in 2022
 223             keys+=(RCVD_IN_DNSWL_MED)
 224           fi
 225
 226           for t in  ${keys[@]}; do
 227             if [[ ${results[$t]} ]]; then
 228               unset "results[$t]"
 229             elif [[ $t == DKIM_VALID_EF && $from == *@[^.]*.[^.]*.[^.]* ]]; then
 230               :
 231               # third level domains dont hit this. its because
 232               # /usr/share/perl5/Mail/SpamAssassin/Plugin/DKIM.pm checks
 233               # if its signed with the registryboundaries domain. afaik:
 234               # we need the actual domain to sign it, this would result in
 235               # a second signature. I only use second level domains for
 236               # testing atm, fsf doesnt use them for anything but the
 237               # forum and I dont expect that to have any deliverability
 238               # problems.  So, not bothering atm.
 239             else
 240               missing+=($t)
 241             fi
 242           done
 243           if (( ${#results[@]} || ${#missing[@]} )); then
 244             printf "$HOSTNAME spamtest %s\n" "$latest"
 245             if (( ${#results[@]} )); then
 246               printf "unexpected %s" "${!results[*]} "
 247             fi
 248             if (( ${#missing[@]} )); then
 249               printf "missing %s" "${missing[*]}"
 250             fi
 251             echo # ends our printf string buildup
 252             cat $resultfile
 253             echo mailtest-check: end of spam debug results
 254             # lets just handle 1 failure at a time in interactive mode.
 255             if $int; then
 256               echo mailtest-check: from: $from, to: $to
 257               exit 0
 258             fi
 259
 260             # less verbose debug output, commented since I might want it another time.
 261             # if $int; then
 262             #   echo mailtest-check: cat $latest:
 263             #   cat $latest
 264             #   echo mailtest-check: end of cat
 265             #   echo "$(tput setaf 5 2>/dev/null ||:)█$(tput sgr0 2>/dev/null||:)%.0s" $(eval echo "{1..${COLUMNS:-60}}")
 266             #fi
 267           fi
 268           rm -f $resultfile
 269           for r in ${results[@]}; do
 270             case $r in
 271               # iank: for when we want to handle dns errors differently
 272               # DKIM_INVALID|T_SPF_TEMPERROR|T_SPF_HELO_TEMPERROR)
 273               #   dnsfail+=1
 274               #   ;;
 275               *)
 276                 unexpected=$(( unexpected + 1 ))
 277                 ;;
 278             esac
 279           done
 280           for miss in ${missing[@]}; do
 281             # We expect dns failures from time to time, so
 282             # we count them separately and alert differently.
 283             case $miss in
 284               # iank: dns fail
 285               # DKIM_VALID|DKIM_VALID_AU|DKIM_VALID_EF|SPF_HELO_PASS|SPF_PASS|
 286               RCVD_IN_DNSWL_MED|DKIMWL_WL_HIGH)
 287                 missing_dnswl+=1
 288                 ;;
 289               *)
 290                 unexpected+=1
 291                 ;;
 292             esac
 293           done
 294           pr <<EOF
 295 mailtest_check_missing_dnswl{folder="$folder",from="$from"} $missing_dnswl
 296 mailtest_check_unexpected_spamd_results{folder="$folder",from="$from"} $unexpected
 297 EOF
 298         fi # if spamdpid
 299       fi # if $slow
 300
 301       now=$EPOCHSECONDS
 302       age_sec=$(( now - last_sec ))
 303       e $((age_sec / 60)):$(( age_sec % 60 )) ago. to:$to from:$from $latest
 304
 305       # usec = unix seconds
 306       pr <<EOF
 307 mailtest_check_last_usec{folder="$folder",from="$from"} $last_sec
 308 EOF
 309     done # end for from in ${froms[@]}
 310   done # end for folder in ${folders[@]}
 311
 312   dir=/var/lib/prometheus/node-exporter
 313   if [[ -e $dir  ]]; then
 314     mv $dir/mailtest-check.prom.$$ $dir/mailtest-check.prom
 315     # note: node_textfile_mtime_seconds will tell us when this last happened. useful for debugging.
 316   fi
 317 }
 318
 319 loop-main() {
 320   # When running under systemd, the system just started. Ve nice and
 321   # give programs some time to finish their startup.
 322   sleep 10
 323   while true; do
 324     premain_sec=$EPOCHSECONDS
 325     main
 326     sleep $(( 300 - ( EPOCHSECONDS - premain_sec ) ))
 327   done
 328 }
 329
 330
 331 if [[ $INVOCATION_ID ]]; then
 332   loop-main
 333 else
 334   main
 335 fi
 336
 337 exit 0