iankelling.org Git - distro-setup/blob - mailtest-check

   1 #!/bin/bash
   2
   3 # Usage: mail-test-check [slow] [anything]
   4 #
   5 # slow: do slow checks, like spamassassin
   6 #
   7 # anything: consider non-interactive, dont print unless something went
   8 # wrong
   9
  10
  11 source /b/errhandle/err
  12
  13 [[ $EUID == 0 ]] || exec sudo -E "${BASH_SOURCE[0]}" "$@"
  14
  15 shopt -s nullglob
  16
  17 e() { $int || return 0; printf "mailtest-check: %s\n" "$*"; }
  18
  19 getspamdpid() {
  20   if [[ ! $spamdpid || ! -d /proc/$spamdpid ]]; then
  21     # try twice in case we are restarting, it happens.
  22     for i in 1 2; do
  23       spamdpid=$(systemctl show --property MainPID --value spamassassin | sed 's/^[10]$//' ||:)
  24       if [[ $spamdpid ]]; then
  25         break
  26       fi
  27       sleep 30
  28     done
  29   fi
  30 }
  31 pr() {
  32   if $doprom && [[ -e /var/lib/prometheus/node-exporter ]]; then
  33     cat >>/var/lib/prometheus/node-exporter/mailtest-check.prom.$$
  34   fi
  35 }
  36
  37
  38 #### begin arg processing ####
  39 # spamassassin checking takes about 8 seconds.
  40 slow=false
  41 if [[ $1 == slow ]]; then
  42   slow=true
  43   shift
  44 fi
  45
  46 int=false
  47 if [[ $SUDO_USER || $SSH_CONNECTION ]]; then
  48   int=true
  49 fi
  50
  51 if [[ $1 == int ]]; then
  52   int=true
  53 fi
  54
  55 if [[ $1 == nonint ]]; then
  56   int=false
  57 fi
  58 #### end arg processing ####
  59
  60 # we put this in to avoid dns errors that happen on reboot,
  61 # but I want to debug them.
  62 # if ! $int; then
  63 #   sleep 60
  64 # fi
  65
  66
  67 # TODO, get je to deliver the local mailbox: /m/md/INBOX
  68 # dovecot appears to setup, i can t be sure.
  69
  70 source /a/bin/bash_unpublished/source-state
  71
  72 doprom=false
  73 case $HOSTNAME in
  74   $MAIL_HOST|bk|je)
  75     doprom=true
  76     ;;
  77   *)
  78     rm -f /var/lib/prometheus/node-exporter/mailtest-check.prom*
  79     ;;
  80 esac
  81
  82 main() {
  83
  84   case $HOSTNAME in
  85     bk)
  86       folders=(/m/md/{expertpathologyreview.com,amnimal.ninja}/testignore)
  87       froms=(ian@iankelling.org z@zroe.org testignore@je.b8.nz iank@gnu.org)
  88       ;;
  89     je)
  90       froms=(ian@iankelling.org z@zroe.org iank@gnu.org testignore@amnimal.ninja)
  91       folders=(/m/md/je.b8.nz/testignore)
  92       ;;
  93     *)
  94       folders=(/m/md/l/testignore)
  95       froms=(testignore@je.b8.nz testignore@expertpathologyreview.com testignore@amnimal.ninja ian@iankelling.org z@zroe.org iank@gnu.org)
  96       if ! $int; then
  97         ### begin rsyncing fencepost email ###
  98         # We dont want to exit if rsync fails, that will get caught by
  99         # our later test by virtue of not having the latest email.
 100         did_rsync=false
 101         try_start_time=$EPOCHSECONDS
 102         try_limit=140 # somewhat arbitrary value
 103         while ! $did_rsync; do
 104           try_left=$(( try_limit - ( EPOCHSECONDS - try_start_time) ))
 105           timeout=120 # somewhat arbitrary value
 106           if (( try_left < 0 )); then
 107             break
 108           fi
 109           if (( try_left < timeout )); then
 110             timeout=$try_left
 111           fi
 112           if timeout $timeout rsync --chown iank:iank -e "ssh -oIdentitiesOnly=yes -F /dev/null -i /root/.ssh/jtuttle" -t --inplace -r 'jtuttle@fencepost.gnu.org:/home/j/jtuttle/Maildir/new/' /m/md/l/testignore/new; then
 113             did_rsync=true
 114           else
 115             sleep 4
 116           fi
 117         done
 118         if ! $did_rsync; then
 119           echo mailtest-check: warning: fencepost rsync failed
 120         fi
 121         ### end rsyncing fencepost email ###
 122       fi
 123       ;;
 124   esac
 125
 126
 127   # avoid errors like this:
 128   # Nov  8 08:16:05.439 [6080] warn: plugin: failed to parse plugin (from @INC): Can't locate Mail/SpamAssassin/Plugin/WLBLEval.pm: lib/Mail/SpamAssassin/Plugin/WLBLEval.pm: Permission denied at (eval 59) line 1.
 129   #Nov  8 08:16:05.439 [6080] warn: plugin: failed to parse plugin (from @INC): Can't locate Mail/SpamAssassin/Plugin/VBounce.pm: lib/Mail/SpamAssassin/Plugin/VBounce.pm: Permission denied at (eval 60) line 1.
 130   # i dont know why, i just found the solution online
 131   cd /m/md
 132
 133
 134   getspamdpid
 135   # first time we write, overwrite anything existing
 136   if [[ -e /var/lib/prometheus/node-exporter ]]; then
 137     cat >/var/lib/prometheus/node-exporter/mailtest-check.prom.$$ <<EOF
 138 mailtest_check_found_spamd_pid_bool $(( ${spamdpid:-0} > 0 ))
 139 EOF
 140   fi
 141   e spamdpid: $spamdpid
 142   if [[ ! $spamdpid ]]; then
 143     echo mailtest spamd pid not found. systemctl status spamassassin:
 144     systemctl status spamassassin
 145   fi
 146   tmpfile=$(mktemp)
 147   declare -i unexpected=0
 148   declare -i missing_dnswl=0
 149   declare -i dnsfail=0
 150   for folder in ${folders[@]}; do
 151     for from in ${froms[@]}; do
 152       latest=
 153       last_sec=0
 154
 155       if ! grep -rlFx "From: $from" $folder/{new,cur} >$tmpfile; then
 156         echo "no message found from: $from"
 157         continue
 158       fi
 159       # webmail sends them to cur it seems
 160       while read -r file; do
 161         file_sec=$(awk '/^Subject: / {print $4}' $file)
 162         if [[ $file_sec ]] && (( file_sec > last_sec )); then
 163           latest=$file
 164           last_sec="$file_sec"
 165         fi
 166       done <$tmpfile
 167       rm -f $tmpfile
 168
 169       to=$(awk '/^Envelope-to: / {print $2}' $latest)
 170
 171       if $slow; then
 172         if ! $int; then
 173           find $folder/new $folder/cur -type f -mmin +1080 -delete
 174         fi
 175         getspamdpid
 176         if [[ $spamdpid ]]; then
 177           if [[ $(readlink /proc/$$/ns/net) != "$(readlink /proc/$spamdpid/ns/net)" ]]; then
 178             spamcpre="nsenter -t $spamdpid -n -m"
 179           fi
 180           unset results
 181           declare -A results
 182           # pyzor fails for our test message, so dont put useless load on their
 183           # servers.
 184           # example line that sed is parsing:
 185           # (-0.1 / 5.0 requ) DKIM_SIGNED=0.1,DKIM_VALID=-0.1,DKIM_VALID_AU=-0.1,SPF_HELO_PASS=-0.001,SPF_PASS=-0.001,TVD_SPACE_RATIO=0.001 autolearn=_AUTOLEARN
 186           resultfile=$(mktemp)
 187           $spamcpre sudo -u Debian-exim spamassassin -D -t --cf='score PYZOR_CHECK 0' <"$latest" &>$resultfile
 188
 189           # note: on some mail, its 1 line after the send-test-forward, on others its 2 with a blank inbetween.
 190           # I use the sed -n to filter this.
 191           raw_results="$(tail $resultfile | grep -A2 -Fx /usr/local/bin/send-test-forward | tail -n+2 | sed -nr 's/^\([^)]*\) *//;s/=[^, ]*([, ]|$)/ /gp')"
 192           for r in $raw_results; do
 193             case $r in
 194               # got this in an update 2022-01. dun care
 195               T_SCC_BODY_TEXT_LINE|SCC_BODY_SINGLE_WORD) : ;;
 196               # we have a new domain, ignore this.
 197               # it seems like some versions of spamassassin do BODY_SINGLE_WORD, others dont, we dun care.
 198               # bayes_00 is a new one indicating ham, we dont care if its missing.
 199               BAYES_00|BODY_SINGLE_WORD|FROM_FMBLA_NEWDOM*|autolearn) : ;;
 200               SPF_HELO_NEUTRAL)
 201                 # some of my domains use neutral spf, treat them the same.
 202                 results[SPF_HELO_PASS]=t
 203                 ;;
 204               *)
 205                 results[$r]=t
 206                 ;;
 207             esac
 208           done
 209           # debugging
 210           # e results = ${!results[@]}
 211           missing=()
 212
 213           keys=(DKIM_SIGNED DKIM_VALID{,_AU,_EF} SPF_HELO_PASS SPF_PASS TVD_SPACE_RATIO)
 214           if [[ $to == *@gnu.org && $from == *@gnu.org ]]; then
 215             keys=(ALL_TRUSTED TVD_SPACE_RATIO)
 216           elif [[ $to == *@gnu.org ]]; then
 217             # eggs has RCVD_IN_DNSWL_MED
 218             keys+=(RCVD_IN_DNSWL_MED)
 219           elif [[ $from == *@gnu.org ]]; then
 220             # eggs has these
 221             keys+=(RCVD_IN_DNSWL_MED DKIMWL_WL_HIGH)
 222           fi
 223
 224           for t in  ${keys[@]}; do
 225             if [[ ${results[$t]} ]]; then
 226               unset "results[$t]"
 227             elif [[ $t == DKIM_VALID_EF && $from == *@[^.]*.[^.]*.[^.]* ]]; then
 228               :
 229               # third level domains dont hit this. its because
 230               # /usr/share/perl5/Mail/SpamAssassin/Plugin/DKIM.pm checks
 231               # if its signed with the registryboundaries domain. afaik:
 232               # we need the actual domain to sign it, this would result in
 233               # a second signature. I only use second level domains for
 234               # testing atm, fsf doesnt use them for anything but the
 235               # forum and I dont expect that to have any deliverability
 236               # problems.  So, not bothering atm.
 237             else
 238               missing+=($t)
 239             fi
 240           done
 241           if (( ${#results[@]} || ${#missing[@]} )); then
 242             printf "$HOSTNAME spamtest %s\n" "$latest"
 243             if (( ${#results[@]} )); then
 244               printf "unexpected %s" "${!results[*]} "
 245             fi
 246             if (( ${#missing[@]} )); then
 247               printf "missing %s" "${missing[*]}"
 248             fi
 249             echo # ends our printf string buildup
 250             cat $resultfile
 251             echo mailtest-check: end of spam debug results
 252             # lets just handle 1 failure at a time in interactive mode.
 253             if $int; then
 254               echo mailtest-check: from: $from, to: $to
 255               exit 0
 256             fi
 257
 258             # less verbose debug output, commented since I might want it another time.
 259             # if $int; then
 260             #   echo mailtest-check: cat $latest:
 261             #   cat $latest
 262             #   echo mailtest-check: end of cat
 263             #   echo "$(tput setaf 5 2>/dev/null ||:)█$(tput sgr0 2>/dev/null||:)%.0s" $(eval echo "{1..${COLUMNS:-60}}")
 264             #fi
 265           fi
 266           rm -f $resultfile
 267           for r in ${results[@]}; do
 268             case $r in
 269               # iank: for when we want to handle dns errors differently
 270               # DKIM_INVALID|T_SPF_TEMPERROR|T_SPF_HELO_TEMPERROR)
 271               #   dnsfail+=1
 272               #   ;;
 273               *)
 274                 unexpected=$(( unexpected + 1 ))
 275                 ;;
 276             esac
 277           done
 278           for miss in ${missing[@]}; do
 279             # We expect dns failures from time to time, so
 280             # we count them separately and alert differently.
 281             case $miss in
 282               # iank: dns fail
 283               # DKIM_VALID|DKIM_VALID_AU|DKIM_VALID_EF|SPF_HELO_PASS|SPF_PASS|
 284               RCVD_IN_DNSWL_MED|DKIMWL_WL_HIGH)
 285                 missing_dnswl+=1
 286                 ;;
 287               *)
 288                 unexpected+=1
 289                 ;;
 290             esac
 291           done
 292         fi # if spamdpid
 293       fi # if $slow
 294
 295       now=$EPOCHSECONDS
 296       age_sec=$(( now - last_sec ))
 297       e $((age_sec / 60)):$(( age_sec % 60 )) ago. to:$to from:$from $latest
 298
 299       # usec = unix seconds
 300       pr <<EOF
 301 mailtest_check_last_usec{folder="$folder",from="$from"} $last_sec
 302 EOF
 303     done # end for from in ${froms[@]}
 304   done # end for folder in ${folders[@]}
 305   if $slow; then
 306     pr <<EOF
 307 mailtest_check_missing_dnswl $missing_dnswl
 308 mailtest_check_unexpected_spamd_results $unexpected
 309 EOF
 310   fi
 311
 312   dir=/var/lib/prometheus/node-exporter
 313   if [[ -e $dir  ]]; then
 314     mv $dir/mailtest-check.prom.$$ $dir/mailtest-check.prom
 315     # note: node_textfile_mtime_seconds will tell us when this last happened. useful for debugging.
 316   fi
 317 }
 318
 319 loop-main() {
 320   # When running under systemd, the system just started. Ve nice and
 321   # give programs some time to finish their startup.
 322   sleep 10
 323   while true; do
 324     premain_sec=$EPOCHSECONDS
 325     main
 326     sleep $(( 300 - ( EPOCHSECONDS - premain_sec ) ))
 327   done
 328 }
 329
 330
 331 if [[ $INVOCATION_ID ]]; then
 332   loop-main
 333 else
 334   main
 335 fi
 336
 337 exit 0