iankelling.org Git - distro-setup/blob - mailtest-check

   1 #!/bin/bash
   2 # I, Ian Kelling, follow the GNU license recommendations at
   3 # https://www.gnu.org/licenses/license-recommendations.en.html. They
   4 # recommend that small programs, < 300 lines, be licensed under the
   5 # Apache License 2.0. This file contains or is part of one or more small
   6 # programs. If a small program grows beyond 300 lines, I plan to switch
   7 # its license to GPL.
   8
   9 # Copyright 2024 Ian Kelling
  10
  11 # Licensed under the Apache License, Version 2.0 (the "License");
  12 # you may not use this file except in compliance with the License.
  13 # You may obtain a copy of the License at
  14
  15 #     http://www.apache.org/licenses/LICENSE-2.0
  16
  17 # Unless required by applicable law or agreed to in writing, software
  18 # distributed under the License is distributed on an "AS IS" BASIS,
  19 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  20 # See the License for the specific language governing permissions and
  21 # limitations under the License.
  22
  23
  24 # Usage: mailtest-check [slow] [int|nonint]
  25 #
  26 # slow: do slow checks, like spamassassin
  27 #
  28 # for  non-interactive, dont print unless something went
  29 # wrong
  30
  31 #set -x
  32
  33
  34 [[ $EUID == 0 ]] || exec sudo -E "${BASH_SOURCE[0]}" "$@"
  35
  36 source /b/bash-bear-trap/bash-bear
  37
  38 shopt -s nullglob
  39
  40 e() { $int || return 0; printf "mailtest-check: %s\n" "$*"; }
  41
  42 getspamdpid() {
  43   if [[ ! $spamdpid || ! -d /proc/$spamdpid ]]; then
  44     # try twice in case we are restarting, it happens.
  45     for (( i=0; i<2; i++ )); do
  46       spamdpid=$(systemctl show --property MainPID --value $spamd_ser | sed 's/^[10]$//' ||:)
  47       if [[ $spamdpid ]]; then
  48         break
  49       fi
  50       sleep 30
  51     done
  52   fi
  53 }
  54
  55
  56 #### begin arg processing ####
  57 # spamassassin checking takes about 8 seconds.
  58 slow=false
  59 if [[ $1 == slow ]]; then
  60   slow=true
  61   shift
  62 fi
  63
  64 int=false
  65 if [[ $SUDO_USER || $SSH_CONNECTION ]]; then
  66   int=true
  67 fi
  68
  69 if [[ $1 == int ]]; then
  70   int=true
  71 fi
  72
  73 if [[ $1 == nonint ]]; then
  74   int=false
  75 fi
  76 #### end arg processing ####
  77
  78 # we put this in to avoid dns errors that happen on reboot,
  79 # but I want to debug them.
  80 # if ! $int; then
  81 #   sleep 60
  82 # fi
  83
  84
  85 # TODO, get je to deliver the local mailbox: /m/md/INBOX
  86 # dovecot appears to setup, i can t be sure.
  87
  88 maini=0
  89
  90 spamd_ser=spamd
  91 if systemctl cat spamassassin &>/dev/null; then
  92   spamd_ser=spamassassin
  93 fi
  94
  95 source /a/bin/bash_unpublished/source-state
  96
  97 doprom=false
  98 case $HOSTNAME in
  99   $MAIL_HOST|bk|je)
 100     doprom=true
 101     ;;
 102   *)
 103     rm -f /var/lib/prometheus/node-exporter/mailtest-check.prom*
 104     ;;
 105 esac
 106
 107 main() {
 108
 109   local -a p_unexpected_spamd_results p_missing_dnswl p_last_usec
 110   case $HOSTNAME in
 111     bk)
 112       folders=(/m/md/{expertpathologyreview.com,amnimal.ninja}/testignore)
 113       froms=(ian@iankelling.org z@zroe.org testignore@je.b8.nz iank@gnu.org)
 114       ;;
 115     je)
 116       froms=(ian@iankelling.org z@zroe.org iank@gnu.org testignore@amnimal.ninja)
 117       folders=(/m/md/je.b8.nz/testignore)
 118       ;;
 119     *)
 120       folders=(/m/md/l/testignore)
 121       # save some cpu cycles
 122       froms=(testignore@je.b8.nz ian@iankelling.org)
 123       if (( maini % 10 == 0 )); then
 124         froms=(testignore@je.b8.nz testignore@expertpathologyreview.com testignore@amnimal.ninja ian@iankelling.org z@zroe.org)
 125       fi
 126       if ! $int; then
 127         ### begin rsyncing fencepost email ###
 128         # We dont want to exit if rsync fails, that will get caught by
 129         # our later test by virtue of not having the latest email.
 130         did_rsync=false
 131         try_start_time=$EPOCHSECONDS
 132         try_limit=140 # somewhat arbitrary value
 133         while ! $did_rsync; do
 134           try_left=$(( try_limit - ( EPOCHSECONDS - try_start_time) ))
 135           timeout=120 # somewhat arbitrary value
 136           if (( try_left < 0 )); then
 137             echo "mailtest-check: failed to rsync fencepost > $try_limit seconds"
 138             break
 139           fi
 140           if (( try_left < timeout )); then
 141             timeout=$try_left
 142           fi
 143           if timeout $timeout rsync --chown iank:iank -e "ssh -oIdentitiesOnly=yes -F /dev/null -i /root/.ssh/jtuttle" -t --inplace -r 'jtuttle@fencepost.gnu.org:/home/j/jtuttle/Maildir/new/' /m/md/l/testignore/new; then
 144             did_rsync=true
 145           else
 146             sleep 4
 147           fi
 148         done
 149         if ! $did_rsync; then
 150           echo mailtest-check: warning: fencepost rsync failed
 151         fi
 152         ### end rsyncing fencepost email ###
 153       fi
 154       ;;
 155   esac
 156
 157
 158   # avoid errors like this:
 159   # Nov  8 08:16:05.439 [6080] warn: plugin: failed to parse plugin (from @INC): Can't locate Mail/SpamAssassin/Plugin/WLBLEval.pm: lib/Mail/SpamAssassin/Plugin/WLBLEval.pm: Permission denied at (eval 59) line 1.
 160   #Nov  8 08:16:05.439 [6080] warn: plugin: failed to parse plugin (from @INC): Can't locate Mail/SpamAssassin/Plugin/VBounce.pm: lib/Mail/SpamAssassin/Plugin/VBounce.pm: Permission denied at (eval 60) line 1.
 161   # i dont know why, i just found the solution online
 162   cd /m/md
 163
 164
 165   getspamdpid
 166   # first time we write, overwrite anything existing
 167   if [[ -e /var/lib/prometheus/node-exporter ]]; then
 168     cat >/var/lib/prometheus/node-exporter/mailtest-check.prom.$$ <<EOF
 169 mailtest_check_found_spamd_pid_bool $(( ${spamdpid:-0} > 0 ))
 170 EOF
 171   fi
 172   e spamdpid: $spamdpid
 173   if [[ ! $spamdpid ]]; then
 174     echo mailtest spamd pid not found. systemctl status $spamd_ser:
 175     systemctl status $spamd_ser
 176   fi
 177   tmpfile=$(mktemp)
 178   declare -i unexpected=0
 179   for folder in ${folders[@]}; do
 180     for from in ${froms[@]}; do
 181       declare -i missing_dnswl=0
 182       #declare -i dnsfail=0
 183       declare -i unexpected=0
 184       latest=
 185       last_sec=0
 186
 187       if ! grep -rlFx "From: $from" $folder/{new,cur} >$tmpfile; then
 188         echo "no message found from: $from"
 189         continue
 190       fi
 191       # webmail sends them to cur it seems
 192       while read -r file; do
 193         file_sec=$(awk '/^Subject: / {print $4}' $file)
 194         if [[ $file_sec ]] && (( file_sec > last_sec )); then
 195           latest=$file
 196           last_sec="$file_sec"
 197         fi
 198       done <$tmpfile
 199       rm -f $tmpfile
 200
 201       to=$(awk '/^Envelope-to: / {print $2}' $latest)
 202
 203       if $slow; then
 204         if ! $int; then
 205           find $folder/new $folder/cur -type f -mmin +1080 -delete
 206         fi
 207         getspamdpid
 208         if [[ $spamdpid ]]; then
 209           if [[ $(readlink /proc/$$/ns/net) != "$(readlink /proc/$spamdpid/ns/net)" ]]; then
 210             spamcpre="nsenter -t $spamdpid -n -m"
 211           fi
 212           unset results
 213           declare -A results
 214           # pyzor fails for our test message, so dont put useless load on their
 215           # servers.
 216           # example line that sed is parsing:
 217           # (-0.1 / 5.0 requ) DKIM_SIGNED=0.1,DKIM_VALID=-0.1,DKIM_VALID_AU=-0.1,SPF_HELO_PASS=-0.001,SPF_PASS=-0.001,TVD_SPACE_RATIO=0.001 autolearn=_AUTOLEARN
 218           resultfile=$(mktemp)
 219           # add -D for debug info. usually it
 220           $spamcpre sudo -u Debian-exim spamassassin -t --cf='score PYZOR_CHECK 0' <"$latest" &>$resultfile
 221
 222           # note: on some mail, its 1 line after the send-test-forward, on others its 2 with a blank inbetween.
 223           # I use the sed -n to filter this.
 224           raw_results="$(tail $resultfile | grep -A2 -Fx /usr/local/bin/send-test-forward | tail -n+2 | sed -nr 's/^\([^)]*\) *//;s/=[^, ]*([, ]|$)/ /gp')"
 225           for r in $raw_results; do
 226             case $r in
 227               # This came in t12, but its just dkim + spf, and my
 228               # systems aren't all t12, so ignore it for now.
 229               DMARC_PASS) : ;;
 230               # got this in an update 2022-01. dun care
 231               T_SCC_BODY_TEXT_LINE|SCC_BODY_SINGLE_WORD) : ;;
 232               # we have a new domain, ignore this.
 233               # it seems like some versions of spamassassin do BODY_SINGLE_WORD, others dont, we dun care.
 234               # bayes_00 is a new one indicating ham, we dont care if its missing.
 235               BAYES_00|BODY_SINGLE_WORD|FROM_FMBLA_NEWDOM*|autolearn) : ;;
 236
 237               # These have somewhat randomly been added and removed, resulting in useless alerts, so ignore them.
 238               RCVD_IN_DNSWL_MED|DKIMWL_WL_HIGH) : ;;
 239
 240               SPF_HELO_NEUTRAL)
 241                 # some of my domains use neutral spf, treat them the same.
 242                 results[SPF_HELO_PASS]=t
 243                 ;;
 244               *)
 245                 results[$r]=t
 246                 ;;
 247             esac
 248           done
 249           # debugging
 250           # e results = ${!results[@]}
 251           missing=()
 252
 253           keys=(DKIM_SIGNED DKIM_VALID{,_AU,_EF} SPF_HELO_PASS SPF_PASS TVD_SPACE_RATIO)
 254           if [[ $to == *@gnu.org && $from == *@gnu.org ]]; then
 255             keys=(ALL_TRUSTED TVD_SPACE_RATIO)
 256             # from eggs had DKIMWL_WL_HIGH sometime in 2022, then DKIMWL_WL_MED unti march 2023
 257           fi
 258
 259           for t in  ${keys[@]}; do
 260             if [[ ${results[$t]} ]]; then
 261               unset "results[$t]"
 262             elif [[ $t == DKIM_VALID_EF && $from == *@[^.]*.[^.]*.[^.]* ]]; then
 263               :
 264               # third level domains dont hit this. its because
 265               # /usr/share/perl5/Mail/SpamAssassin/Plugin/DKIM.pm checks
 266               # if its signed with the registryboundaries domain. afaik:
 267               # we need the actual domain to sign it, this would result in
 268               # a second signature. I only use second level domains for
 269               # testing atm, fsf doesnt use them for anything but the
 270               # forum and I dont expect that to have any deliverability
 271               # problems.  So, not bothering atm.
 272             else
 273               missing+=($t)
 274             fi
 275           done
 276           if (( ${#results[@]} || ${#missing[@]} )); then
 277             printf "$HOSTNAME spamtest %s\n" "$latest"
 278             if (( ${#results[@]} )); then
 279               printf "unexpected %s" "${!results[*]} "
 280             fi
 281             if (( ${#missing[@]} )); then
 282               printf "missing %s" "${missing[*]}"
 283             fi
 284             echo # ends our printf string buildup
 285             cat $resultfile
 286             echo mailtest-check: end of spam debug results
 287             # lets just handle 1 failure at a time in interactive mode.
 288             if $int; then
 289               echo mailtest-check: from: $from, to: $to
 290               exit 0
 291             fi
 292
 293             # less verbose debug output, commented since I might want it another time.
 294             # if $int; then
 295             #   echo mailtest-check: cat $latest:
 296             #   cat $latest
 297             #   echo mailtest-check: end of cat
 298             #fi
 299           fi
 300           rm -f $resultfile
 301           for r in ${results[@]}; do
 302             case $r in
 303               # iank: for when we want to handle dns errors differently.
 304               # also uncomment declaration of dnsfail above.
 305               # DKIM_INVALID|T_SPF_TEMPERROR|T_SPF_HELO_TEMPERROR)
 306               #   dnsfail+=1
 307               #   ;;
 308               *)
 309                 unexpected=$(( unexpected + 1 ))
 310                 ;;
 311             esac
 312           done
 313           for miss in ${missing[@]}; do
 314             # At some point we had annoying dns failures that we couldn't solve so we
 315             # we counted dns fail related results separately and alert differently.
 316             # DKIM_VALID|DKIM_VALID_AU|DKIM_VALID_EF|SPF_HELO_PASS|SPF_PASS|
 317             case $miss in
 318               *)
 319                 unexpected+=1
 320                 ;;
 321             esac
 322           done
 323           mapfile -O ${#p_missing_dnswl[@]} -t p_missing_dnswl <<EOF
 324 mailtest_check_missing_dnswl{folder="$folder",from="$from"} $missing_dnswl
 325 EOF
 326           mapfile -O ${#p_unexpected_spamd_results[@]} -t p_unexpected_spamd_results <<EOF
 327 mailtest_check_unexpected_spamd_results{folder="$folder",from="$from"} $unexpected
 328 EOF
 329         fi # if spamdpid
 330       fi # if $slow
 331
 332       now=$EPOCHSECONDS
 333       age_sec=$(( now - last_sec ))
 334       e $((age_sec / 60)):$(( age_sec % 60 )) ago. to:$to from:$from $latest
 335
 336       # usec = unix seconds
 337       mapfile -O ${#p_last_usec[@]} -t p_last_usec <<EOF
 338 mailtest_check_last_usec{folder="$folder",from="$from"} $last_sec
 339 EOF
 340     done # end for from in ${froms[@]}
 341   done # end for folder in ${folders[@]}
 342
 343   dir=/var/lib/prometheus/node-exporter
 344   path=$dir/mailtest-check.prom.$$
 345   if $doprom && [[ -e $dir  ]]; then
 346     for l in "${p_unexpected_spamd_results[@]}"; do
 347       printf "%s\n" "$l" >>$path
 348     done
 349     for l in "${p_missing_dnswl[@]}"; do
 350       printf "%s\n" "$l" >>$path
 351     done
 352     for l in "${p_last_usec[@]}"; do
 353       printf "%s\n" "$l" >>$path
 354     done
 355     mv $path $dir/mailtest-check.prom
 356     # note: node_textfile_mtime_seconds will tell us when this last happened. useful for debugging.
 357   fi
 358 }
 359
 360 loop-main() {
 361   # When running under systemd, the system just started. Ve nice and
 362   # give programs some time to finish their startup.
 363   sleep 10
 364   while true; do
 365     premain_sec=$EPOCHSECONDS
 366     main
 367     maini=$((maini + 1))
 368     sleep $(( 300 - ( EPOCHSECONDS - premain_sec ) ))
 369   done
 370 }
 371
 372
 373 if [[ $PPID == 1 ]]; then
 374   loop-main
 375 else
 376   main
 377 fi
 378
 379 exit 0