iankelling.org Git - distro-setup/blob - mailtest-check

   1 #!/bin/bash
   2 # I, Ian Kelling, follow the GNU license recommendations at
   3 # https://www.gnu.org/licenses/license-recommendations.en.html. They
   4 # recommend that small programs, < 300 lines, be licensed under the
   5 # Apache License 2.0. This file contains or is part of one or more small
   6 # programs. If a small program grows beyond 300 lines, I plan to switch
   7 # its license to GPL.
   8
   9 # Copyright 2024 Ian Kelling
  10
  11 # Licensed under the Apache License, Version 2.0 (the "License");
  12 # you may not use this file except in compliance with the License.
  13 # You may obtain a copy of the License at
  14
  15 #     http://www.apache.org/licenses/LICENSE-2.0
  16
  17 # Unless required by applicable law or agreed to in writing, software
  18 # distributed under the License is distributed on an "AS IS" BASIS,
  19 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  20 # See the License for the specific language governing permissions and
  21 # limitations under the License.
  22
  23
  24 # Usage: mailtest-check [slow] [int|nonint]
  25 #
  26 # slow: do slow checks, like spamassassin
  27 #
  28 # for  non-interactive, dont print unless something went
  29 # wrong
  30
  31
  32 source /b/bash-bear-trap/bash-bear
  33
  34 [[ $EUID == 0 ]] || exec sudo -E "${BASH_SOURCE[0]}" "$@"
  35
  36 shopt -s nullglob
  37
  38 e() { $int || return 0; printf "mailtest-check: %s\n" "$*"; }
  39
  40 getspamdpid() {
  41   if [[ ! $spamdpid || ! -d /proc/$spamdpid ]]; then
  42     # try twice in case we are restarting, it happens.
  43     for (( i=0; i<2; i++ )); do
  44       spamdpid=$(systemctl show --property MainPID --value $spamd_ser | sed 's/^[10]$//' ||:)
  45       if [[ $spamdpid ]]; then
  46         break
  47       fi
  48       sleep 30
  49     done
  50   fi
  51 }
  52
  53
  54 #### begin arg processing ####
  55 # spamassassin checking takes about 8 seconds.
  56 slow=false
  57 if [[ $1 == slow ]]; then
  58   slow=true
  59   shift
  60 fi
  61
  62 int=false
  63 if [[ $SUDO_USER || $SSH_CONNECTION ]]; then
  64   int=true
  65 fi
  66
  67 if [[ $1 == int ]]; then
  68   int=true
  69 fi
  70
  71 if [[ $1 == nonint ]]; then
  72   int=false
  73 fi
  74 #### end arg processing ####
  75
  76 # we put this in to avoid dns errors that happen on reboot,
  77 # but I want to debug them.
  78 # if ! $int; then
  79 #   sleep 60
  80 # fi
  81
  82
  83 # TODO, get je to deliver the local mailbox: /m/md/INBOX
  84 # dovecot appears to setup, i can t be sure.
  85
  86 spamd_ser=spamd
  87 if systemctl cat spamassassin &>/dev/null; then
  88   spamd_ser=spamassassin
  89 fi
  90
  91 source /a/bin/bash_unpublished/source-state
  92
  93 doprom=false
  94 case $HOSTNAME in
  95   $MAIL_HOST|bk|je)
  96     doprom=true
  97     ;;
  98   *)
  99     rm -f /var/lib/prometheus/node-exporter/mailtest-check.prom*
 100     ;;
 101 esac
 102
 103 main() {
 104
 105   local -a p_unexpected_spamd_results p_missing_dnswl p_last_usec
 106   case $HOSTNAME in
 107     bk)
 108       folders=(/m/md/{expertpathologyreview.com,amnimal.ninja}/testignore)
 109       froms=(ian@iankelling.org z@zroe.org testignore@je.b8.nz iank@gnu.org)
 110       ;;
 111     je)
 112       froms=(ian@iankelling.org z@zroe.org iank@gnu.org testignore@amnimal.ninja)
 113       folders=(/m/md/je.b8.nz/testignore)
 114       ;;
 115     *)
 116       folders=(/m/md/l/testignore)
 117       froms=(testignore@je.b8.nz testignore@expertpathologyreview.com testignore@amnimal.ninja ian@iankelling.org z@zroe.org)
 118       if ! $int; then
 119         ### begin rsyncing fencepost email ###
 120         # We dont want to exit if rsync fails, that will get caught by
 121         # our later test by virtue of not having the latest email.
 122         did_rsync=false
 123         try_start_time=$EPOCHSECONDS
 124         try_limit=140 # somewhat arbitrary value
 125         while ! $did_rsync; do
 126           try_left=$(( try_limit - ( EPOCHSECONDS - try_start_time) ))
 127           timeout=120 # somewhat arbitrary value
 128           if (( try_left < 0 )); then
 129             echo "mailtest-check: failed to rsync fencepost > $try_limit seconds"
 130             break
 131           fi
 132           if (( try_left < timeout )); then
 133             timeout=$try_left
 134           fi
 135           if timeout $timeout rsync --chown iank:iank -e "ssh -oIdentitiesOnly=yes -F /dev/null -i /root/.ssh/jtuttle" -t --inplace -r 'jtuttle@fencepost.gnu.org:/home/j/jtuttle/Maildir/new/' /m/md/l/testignore/new; then
 136             did_rsync=true
 137           else
 138             sleep 4
 139           fi
 140         done
 141         if ! $did_rsync; then
 142           echo mailtest-check: warning: fencepost rsync failed
 143         fi
 144         ### end rsyncing fencepost email ###
 145       fi
 146       ;;
 147   esac
 148
 149
 150   # avoid errors like this:
 151   # Nov  8 08:16:05.439 [6080] warn: plugin: failed to parse plugin (from @INC): Can't locate Mail/SpamAssassin/Plugin/WLBLEval.pm: lib/Mail/SpamAssassin/Plugin/WLBLEval.pm: Permission denied at (eval 59) line 1.
 152   #Nov  8 08:16:05.439 [6080] warn: plugin: failed to parse plugin (from @INC): Can't locate Mail/SpamAssassin/Plugin/VBounce.pm: lib/Mail/SpamAssassin/Plugin/VBounce.pm: Permission denied at (eval 60) line 1.
 153   # i dont know why, i just found the solution online
 154   cd /m/md
 155
 156
 157   getspamdpid
 158   # first time we write, overwrite anything existing
 159   if [[ -e /var/lib/prometheus/node-exporter ]]; then
 160     cat >/var/lib/prometheus/node-exporter/mailtest-check.prom.$$ <<EOF
 161 mailtest_check_found_spamd_pid_bool $(( ${spamdpid:-0} > 0 ))
 162 EOF
 163   fi
 164   e spamdpid: $spamdpid
 165   if [[ ! $spamdpid ]]; then
 166     echo mailtest spamd pid not found. systemctl status $spamd_ser:
 167     systemctl status $spamd_ser
 168   fi
 169   tmpfile=$(mktemp)
 170   declare -i unexpected=0
 171   for folder in ${folders[@]}; do
 172     for from in ${froms[@]}; do
 173       declare -i missing_dnswl=0
 174       #declare -i dnsfail=0
 175       declare -i unexpected=0
 176       latest=
 177       last_sec=0
 178
 179       if ! grep -rlFx "From: $from" $folder/{new,cur} >$tmpfile; then
 180         echo "no message found from: $from"
 181         continue
 182       fi
 183       # webmail sends them to cur it seems
 184       while read -r file; do
 185         file_sec=$(awk '/^Subject: / {print $4}' $file)
 186         if [[ $file_sec ]] && (( file_sec > last_sec )); then
 187           latest=$file
 188           last_sec="$file_sec"
 189         fi
 190       done <$tmpfile
 191       rm -f $tmpfile
 192
 193       to=$(awk '/^Envelope-to: / {print $2}' $latest)
 194
 195       if $slow; then
 196         if ! $int; then
 197           find $folder/new $folder/cur -type f -mmin +1080 -delete
 198         fi
 199         getspamdpid
 200         if [[ $spamdpid ]]; then
 201           if [[ $(readlink /proc/$$/ns/net) != "$(readlink /proc/$spamdpid/ns/net)" ]]; then
 202             spamcpre="nsenter -t $spamdpid -n -m"
 203           fi
 204           unset results
 205           declare -A results
 206           # pyzor fails for our test message, so dont put useless load on their
 207           # servers.
 208           # example line that sed is parsing:
 209           # (-0.1 / 5.0 requ) DKIM_SIGNED=0.1,DKIM_VALID=-0.1,DKIM_VALID_AU=-0.1,SPF_HELO_PASS=-0.001,SPF_PASS=-0.001,TVD_SPACE_RATIO=0.001 autolearn=_AUTOLEARN
 210           resultfile=$(mktemp)
 211           $spamcpre sudo -u Debian-exim spamassassin -D -t --cf='score PYZOR_CHECK 0' <"$latest" &>$resultfile
 212
 213           # note: on some mail, its 1 line after the send-test-forward, on others its 2 with a blank inbetween.
 214           # I use the sed -n to filter this.
 215           raw_results="$(tail $resultfile | grep -A2 -Fx /usr/local/bin/send-test-forward | tail -n+2 | sed -nr 's/^\([^)]*\) *//;s/=[^, ]*([, ]|$)/ /gp')"
 216           for r in $raw_results; do
 217             case $r in
 218               # This came in t12, but its just dkim + spf, and my
 219               # systems aren't all t12, so ignore it for now.
 220               DMARC_PASS) : ;;
 221               # got this in an update 2022-01. dun care
 222               T_SCC_BODY_TEXT_LINE|SCC_BODY_SINGLE_WORD) : ;;
 223               # we have a new domain, ignore this.
 224               # it seems like some versions of spamassassin do BODY_SINGLE_WORD, others dont, we dun care.
 225               # bayes_00 is a new one indicating ham, we dont care if its missing.
 226               BAYES_00|BODY_SINGLE_WORD|FROM_FMBLA_NEWDOM*|autolearn) : ;;
 227
 228               # These have somewhat randomly been added and removed, resulting in useless alerts, so ignore them.
 229               RCVD_IN_DNSWL_MED|DKIMWL_WL_HIGH) : ;;
 230
 231               SPF_HELO_NEUTRAL)
 232                 # some of my domains use neutral spf, treat them the same.
 233                 results[SPF_HELO_PASS]=t
 234                 ;;
 235               *)
 236                 results[$r]=t
 237                 ;;
 238             esac
 239           done
 240           # debugging
 241           # e results = ${!results[@]}
 242           missing=()
 243
 244           keys=(DKIM_SIGNED DKIM_VALID{,_AU,_EF} SPF_HELO_PASS SPF_PASS TVD_SPACE_RATIO)
 245           if [[ $to == *@gnu.org && $from == *@gnu.org ]]; then
 246             keys=(ALL_TRUSTED TVD_SPACE_RATIO)
 247             # from eggs had DKIMWL_WL_HIGH sometime in 2022, then DKIMWL_WL_MED unti march 2023
 248           fi
 249
 250           for t in  ${keys[@]}; do
 251             if [[ ${results[$t]} ]]; then
 252               unset "results[$t]"
 253             elif [[ $t == DKIM_VALID_EF && $from == *@[^.]*.[^.]*.[^.]* ]]; then
 254               :
 255               # third level domains dont hit this. its because
 256               # /usr/share/perl5/Mail/SpamAssassin/Plugin/DKIM.pm checks
 257               # if its signed with the registryboundaries domain. afaik:
 258               # we need the actual domain to sign it, this would result in
 259               # a second signature. I only use second level domains for
 260               # testing atm, fsf doesnt use them for anything but the
 261               # forum and I dont expect that to have any deliverability
 262               # problems.  So, not bothering atm.
 263             else
 264               missing+=($t)
 265             fi
 266           done
 267           if (( ${#results[@]} || ${#missing[@]} )); then
 268             printf "$HOSTNAME spamtest %s\n" "$latest"
 269             if (( ${#results[@]} )); then
 270               printf "unexpected %s" "${!results[*]} "
 271             fi
 272             if (( ${#missing[@]} )); then
 273               printf "missing %s" "${missing[*]}"
 274             fi
 275             echo # ends our printf string buildup
 276             cat $resultfile
 277             echo mailtest-check: end of spam debug results
 278             # lets just handle 1 failure at a time in interactive mode.
 279             if $int; then
 280               echo mailtest-check: from: $from, to: $to
 281               exit 0
 282             fi
 283
 284             # less verbose debug output, commented since I might want it another time.
 285             # if $int; then
 286             #   echo mailtest-check: cat $latest:
 287             #   cat $latest
 288             #   echo mailtest-check: end of cat
 289             #fi
 290           fi
 291           rm -f $resultfile
 292           for r in ${results[@]}; do
 293             case $r in
 294               # iank: for when we want to handle dns errors differently.
 295               # also uncomment declaration of dnsfail above.
 296               # DKIM_INVALID|T_SPF_TEMPERROR|T_SPF_HELO_TEMPERROR)
 297               #   dnsfail+=1
 298               #   ;;
 299               *)
 300                 unexpected=$(( unexpected + 1 ))
 301                 ;;
 302             esac
 303           done
 304           for miss in ${missing[@]}; do
 305             # At some point we had annoying dns failures that we couldn't solve so we
 306             # we counted dns fail related results separately and alert differently.
 307             # DKIM_VALID|DKIM_VALID_AU|DKIM_VALID_EF|SPF_HELO_PASS|SPF_PASS|
 308             case $miss in
 309               *)
 310                 unexpected+=1
 311                 ;;
 312             esac
 313           done
 314           mapfile -O ${#p_missing_dnswl[@]} -t p_missing_dnswl <<EOF
 315 mailtest_check_missing_dnswl{folder="$folder",from="$from"} $missing_dnswl
 316 EOF
 317           mapfile -O ${#p_unexpected_spamd_results[@]} -t p_unexpected_spamd_results <<EOF
 318 mailtest_check_unexpected_spamd_results{folder="$folder",from="$from"} $unexpected
 319 EOF
 320         fi # if spamdpid
 321       fi # if $slow
 322
 323       now=$EPOCHSECONDS
 324       age_sec=$(( now - last_sec ))
 325       e $((age_sec / 60)):$(( age_sec % 60 )) ago. to:$to from:$from $latest
 326
 327       # usec = unix seconds
 328       mapfile -O ${#p_last_usec[@]} -t p_last_usec <<EOF
 329 mailtest_check_last_usec{folder="$folder",from="$from"} $last_sec
 330 EOF
 331     done # end for from in ${froms[@]}
 332   done # end for folder in ${folders[@]}
 333
 334   dir=/var/lib/prometheus/node-exporter
 335   path=$dir/mailtest-check.prom.$$
 336   if $doprom && [[ -e $dir  ]]; then
 337     for l in "${p_unexpected_spamd_results[@]}"; do
 338       printf "%s\n" "$l" >>$path
 339       done
 340     for l in "${p_missing_dnswl[@]}"; do
 341       printf "%s\n" "$l" >>$path
 342     done
 343     for l in "${p_last_usec[@]}"; do
 344       printf "%s\n" "$l" >>$path
 345     done
 346     mv $path $dir/mailtest-check.prom
 347     # note: node_textfile_mtime_seconds will tell us when this last happened. useful for debugging.
 348   fi
 349 }
 350
 351 loop-main() {
 352   # When running under systemd, the system just started. Ve nice and
 353   # give programs some time to finish their startup.
 354   sleep 10
 355   while true; do
 356     premain_sec=$EPOCHSECONDS
 357     main
 358     sleep $(( 300 - ( EPOCHSECONDS - premain_sec ) ))
 359   done
 360 }
 361
 362
 363 if [[ $INVOCATION_ID ]]; then
 364   loop-main
 365 else
 366   main
 367 fi
 368
 369 exit 0