iankelling.org Git - distro-setup/blob - mailtest-check

   1 #!/bin/bash
   2
   3 # Usage: mail-test-check [slow] [int|nonint]
   4 #
   5 # slow: do slow checks, like spamassassin
   6 #
   7 # for  non-interactive, dont print unless something went
   8 # wrong
   9
  10
  11 source /b/errhandle/err
  12
  13 [[ $EUID == 0 ]] || exec sudo -E "${BASH_SOURCE[0]}" "$@"
  14
  15 shopt -s nullglob
  16
  17 e() { $int || return 0; printf "mailtest-check: %s\n" "$*"; }
  18
  19 getspamdpid() {
  20   if [[ ! $spamdpid || ! -d /proc/$spamdpid ]]; then
  21     # try twice in case we are restarting, it happens.
  22     for i in 1 2; do
  23       spamdpid=$(systemctl show --property MainPID --value spamassassin | sed 's/^[10]$//' ||:)
  24       if [[ $spamdpid ]]; then
  25         break
  26       fi
  27       sleep 30
  28     done
  29   fi
  30 }
  31
  32
  33 #### begin arg processing ####
  34 # spamassassin checking takes about 8 seconds.
  35 slow=false
  36 if [[ $1 == slow ]]; then
  37   slow=true
  38   shift
  39 fi
  40
  41 int=false
  42 if [[ $SUDO_USER || $SSH_CONNECTION ]]; then
  43   int=true
  44 fi
  45
  46 if [[ $1 == int ]]; then
  47   int=true
  48 fi
  49
  50 if [[ $1 == nonint ]]; then
  51   int=false
  52 fi
  53 #### end arg processing ####
  54
  55 # we put this in to avoid dns errors that happen on reboot,
  56 # but I want to debug them.
  57 # if ! $int; then
  58 #   sleep 60
  59 # fi
  60
  61
  62 # TODO, get je to deliver the local mailbox: /m/md/INBOX
  63 # dovecot appears to setup, i can t be sure.
  64
  65 source /a/bin/bash_unpublished/source-state
  66
  67 doprom=false
  68 case $HOSTNAME in
  69   $MAIL_HOST|bk|je)
  70     doprom=true
  71     ;;
  72   *)
  73     rm -f /var/lib/prometheus/node-exporter/mailtest-check.prom*
  74     ;;
  75 esac
  76
  77 main() {
  78
  79   local -a p_unexpected_spamd_results p_missing_dnswl p_last_usec
  80   case $HOSTNAME in
  81     bk)
  82       folders=(/m/md/{expertpathologyreview.com,amnimal.ninja}/testignore)
  83       froms=(ian@iankelling.org z@zroe.org testignore@je.b8.nz iank@gnu.org)
  84       ;;
  85     je)
  86       froms=(ian@iankelling.org z@zroe.org iank@gnu.org testignore@amnimal.ninja)
  87       folders=(/m/md/je.b8.nz/testignore)
  88       ;;
  89     *)
  90       folders=(/m/md/l/testignore)
  91       froms=(testignore@je.b8.nz testignore@expertpathologyreview.com testignore@amnimal.ninja ian@iankelling.org z@zroe.org iank@gnu.org)
  92       if ! $int; then
  93         ### begin rsyncing fencepost email ###
  94         # We dont want to exit if rsync fails, that will get caught by
  95         # our later test by virtue of not having the latest email.
  96         did_rsync=false
  97         try_start_time=$EPOCHSECONDS
  98         try_limit=140 # somewhat arbitrary value
  99         while ! $did_rsync; do
 100           try_left=$(( try_limit - ( EPOCHSECONDS - try_start_time) ))
 101           timeout=120 # somewhat arbitrary value
 102           if (( try_left < 0 )); then
 103             echo "mailtest-check: failed to rsync fencepost > $try_limit seconds"
 104             break
 105           fi
 106           if (( try_left < timeout )); then
 107             timeout=$try_left
 108           fi
 109           if timeout $timeout rsync --chown iank:iank -e "ssh -oIdentitiesOnly=yes -F /dev/null -i /root/.ssh/jtuttle" -t --inplace -r 'jtuttle@fencepost.gnu.org:/home/j/jtuttle/Maildir/new/' /m/md/l/testignore/new; then
 110             did_rsync=true
 111           else
 112             sleep 4
 113           fi
 114         done
 115         if ! $did_rsync; then
 116           echo mailtest-check: warning: fencepost rsync failed
 117         fi
 118         ### end rsyncing fencepost email ###
 119       fi
 120       ;;
 121   esac
 122
 123
 124   # avoid errors like this:
 125   # Nov  8 08:16:05.439 [6080] warn: plugin: failed to parse plugin (from @INC): Can't locate Mail/SpamAssassin/Plugin/WLBLEval.pm: lib/Mail/SpamAssassin/Plugin/WLBLEval.pm: Permission denied at (eval 59) line 1.
 126   #Nov  8 08:16:05.439 [6080] warn: plugin: failed to parse plugin (from @INC): Can't locate Mail/SpamAssassin/Plugin/VBounce.pm: lib/Mail/SpamAssassin/Plugin/VBounce.pm: Permission denied at (eval 60) line 1.
 127   # i dont know why, i just found the solution online
 128   cd /m/md
 129
 130
 131   getspamdpid
 132   # first time we write, overwrite anything existing
 133   if [[ -e /var/lib/prometheus/node-exporter ]]; then
 134     cat >/var/lib/prometheus/node-exporter/mailtest-check.prom.$$ <<EOF
 135 mailtest_check_found_spamd_pid_bool $(( ${spamdpid:-0} > 0 ))
 136 EOF
 137   fi
 138   e spamdpid: $spamdpid
 139   if [[ ! $spamdpid ]]; then
 140     echo mailtest spamd pid not found. systemctl status spamassassin:
 141     systemctl status spamassassin
 142   fi
 143   tmpfile=$(mktemp)
 144   declare -i unexpected=0
 145   for folder in ${folders[@]}; do
 146     for from in ${froms[@]}; do
 147       declare -i missing_dnswl=0
 148       declare -i dnsfail=0
 149       declare -i unexpected=0
 150       latest=
 151       last_sec=0
 152
 153       if ! grep -rlFx "From: $from" $folder/{new,cur} >$tmpfile; then
 154         echo "no message found from: $from"
 155         continue
 156       fi
 157       # webmail sends them to cur it seems
 158       while read -r file; do
 159         file_sec=$(awk '/^Subject: / {print $4}' $file)
 160         if [[ $file_sec ]] && (( file_sec > last_sec )); then
 161           latest=$file
 162           last_sec="$file_sec"
 163         fi
 164       done <$tmpfile
 165       rm -f $tmpfile
 166
 167       to=$(awk '/^Envelope-to: / {print $2}' $latest)
 168
 169       if $slow; then
 170         if ! $int; then
 171           find $folder/new $folder/cur -type f -mmin +1080 -delete
 172         fi
 173         getspamdpid
 174         if [[ $spamdpid ]]; then
 175           if [[ $(readlink /proc/$$/ns/net) != "$(readlink /proc/$spamdpid/ns/net)" ]]; then
 176             spamcpre="nsenter -t $spamdpid -n -m"
 177           fi
 178           unset results
 179           declare -A results
 180           # pyzor fails for our test message, so dont put useless load on their
 181           # servers.
 182           # example line that sed is parsing:
 183           # (-0.1 / 5.0 requ) DKIM_SIGNED=0.1,DKIM_VALID=-0.1,DKIM_VALID_AU=-0.1,SPF_HELO_PASS=-0.001,SPF_PASS=-0.001,TVD_SPACE_RATIO=0.001 autolearn=_AUTOLEARN
 184           resultfile=$(mktemp)
 185           $spamcpre sudo -u Debian-exim spamassassin -D -t --cf='score PYZOR_CHECK 0' <"$latest" &>$resultfile
 186
 187           # note: on some mail, its 1 line after the send-test-forward, on others its 2 with a blank inbetween.
 188           # I use the sed -n to filter this.
 189           raw_results="$(tail $resultfile | grep -A2 -Fx /usr/local/bin/send-test-forward | tail -n+2 | sed -nr 's/^\([^)]*\) *//;s/=[^, ]*([, ]|$)/ /gp')"
 190           for r in $raw_results; do
 191             case $r in
 192               # got this in an update 2022-01. dun care
 193               T_SCC_BODY_TEXT_LINE|SCC_BODY_SINGLE_WORD) : ;;
 194               # we have a new domain, ignore this.
 195               # it seems like some versions of spamassassin do BODY_SINGLE_WORD, others dont, we dun care.
 196               # bayes_00 is a new one indicating ham, we dont care if its missing.
 197               BAYES_00|BODY_SINGLE_WORD|FROM_FMBLA_NEWDOM*|autolearn) : ;;
 198
 199               # These have somewhat randomly been added and removed, resulting in useless alerts, so ignore them.
 200               RCVD_IN_DNSWL_MED|DKIMWL_WL_HIGH) : ;;
 201
 202               SPF_HELO_NEUTRAL)
 203                 # some of my domains use neutral spf, treat them the same.
 204                 results[SPF_HELO_PASS]=t
 205                 ;;
 206               *)
 207                 results[$r]=t
 208                 ;;
 209             esac
 210           done
 211           # debugging
 212           # e results = ${!results[@]}
 213           missing=()
 214
 215           keys=(DKIM_SIGNED DKIM_VALID{,_AU,_EF} SPF_HELO_PASS SPF_PASS TVD_SPACE_RATIO)
 216           if [[ $to == *@gnu.org && $from == *@gnu.org ]]; then
 217             keys=(ALL_TRUSTED TVD_SPACE_RATIO)
 218             # from eggs had DKIMWL_WL_HIGH sometime in 2022, then DKIMWL_WL_MED unti march 2023
 219           fi
 220
 221           for t in  ${keys[@]}; do
 222             if [[ ${results[$t]} ]]; then
 223               unset "results[$t]"
 224             elif [[ $t == DKIM_VALID_EF && $from == *@[^.]*.[^.]*.[^.]* ]]; then
 225               :
 226               # third level domains dont hit this. its because
 227               # /usr/share/perl5/Mail/SpamAssassin/Plugin/DKIM.pm checks
 228               # if its signed with the registryboundaries domain. afaik:
 229               # we need the actual domain to sign it, this would result in
 230               # a second signature. I only use second level domains for
 231               # testing atm, fsf doesnt use them for anything but the
 232               # forum and I dont expect that to have any deliverability
 233               # problems.  So, not bothering atm.
 234             else
 235               missing+=($t)
 236             fi
 237           done
 238           if (( ${#results[@]} || ${#missing[@]} )); then
 239             printf "$HOSTNAME spamtest %s\n" "$latest"
 240             if (( ${#results[@]} )); then
 241               printf "unexpected %s" "${!results[*]} "
 242             fi
 243             if (( ${#missing[@]} )); then
 244               printf "missing %s" "${missing[*]}"
 245             fi
 246             echo # ends our printf string buildup
 247             cat $resultfile
 248             echo mailtest-check: end of spam debug results
 249             # lets just handle 1 failure at a time in interactive mode.
 250             if $int; then
 251               echo mailtest-check: from: $from, to: $to
 252               exit 0
 253             fi
 254
 255             # less verbose debug output, commented since I might want it another time.
 256             # if $int; then
 257             #   echo mailtest-check: cat $latest:
 258             #   cat $latest
 259             #   echo mailtest-check: end of cat
 260             #fi
 261           fi
 262           rm -f $resultfile
 263           for r in ${results[@]}; do
 264             case $r in
 265               # iank: for when we want to handle dns errors differently
 266               # DKIM_INVALID|T_SPF_TEMPERROR|T_SPF_HELO_TEMPERROR)
 267               #   dnsfail+=1
 268               #   ;;
 269               *)
 270                 unexpected=$(( unexpected + 1 ))
 271                 ;;
 272             esac
 273           done
 274           for miss in ${missing[@]}; do
 275             # At some point we had annoying dns failures that we couldn't solve so we
 276             # we counted dns fail related results separately and alert differently.
 277             # DKIM_VALID|DKIM_VALID_AU|DKIM_VALID_EF|SPF_HELO_PASS|SPF_PASS|
 278             case $miss in
 279               *)
 280                 unexpected+=1
 281                 ;;
 282             esac
 283           done
 284           mapfile -O ${#p_missing_dnswl[@]} -t p_missing_dnswl <<EOF
 285 mailtest_check_missing_dnswl{folder="$folder",from="$from"} $missing_dnswl
 286 EOF
 287           mapfile -O ${#p_unexpected_spamd_results[@]} -t p_unexpected_spamd_results <<EOF
 288 mailtest_check_unexpected_spamd_results{folder="$folder",from="$from"} $unexpected
 289 EOF
 290         fi # if spamdpid
 291       fi # if $slow
 292
 293       now=$EPOCHSECONDS
 294       age_sec=$(( now - last_sec ))
 295       e $((age_sec / 60)):$(( age_sec % 60 )) ago. to:$to from:$from $latest
 296
 297       # usec = unix seconds
 298       mapfile -O ${#p_last_usec[@]} -t p_last_usec <<EOF
 299 mailtest_check_last_usec{folder="$folder",from="$from"} $last_sec
 300 EOF
 301     done # end for from in ${froms[@]}
 302   done # end for folder in ${folders[@]}
 303
 304   dir=/var/lib/prometheus/node-exporter
 305   path=$dir/mailtest-check.prom.$$
 306   if $doprom && [[ -e $dir  ]]; then
 307     for l in "${p_unexpected_spamd_results[@]}"; do
 308       printf "%s\n" "$l" >>$path
 309       done
 310     for l in "${p_missing_dnswl[@]}"; do
 311       printf "%s\n" "$l" >>$path
 312     done
 313     for l in "${p_last_usec[@]}"; do
 314       printf "%s\n" "$l" >>$path
 315     done
 316     mv $path $dir/mailtest-check.prom
 317     # note: node_textfile_mtime_seconds will tell us when this last happened. useful for debugging.
 318   fi
 319 }
 320
 321 loop-main() {
 322   # When running under systemd, the system just started. Ve nice and
 323   # give programs some time to finish their startup.
 324   sleep 10
 325   while true; do
 326     premain_sec=$EPOCHSECONDS
 327     main
 328     sleep $(( 300 - ( EPOCHSECONDS - premain_sec ) ))
 329   done
 330 }
 331
 332
 333 if [[ $INVOCATION_ID ]]; then
 334   loop-main
 335 else
 336   main
 337 fi
 338
 339 exit 0