From b70501b452c4c24dddbf6dda38917304f273f2c3 Mon Sep 17 00:00:00 2001 From: Ian Kelling Date: Tue, 10 Nov 2020 14:52:58 -0500 Subject: [PATCH] mail fixes, additional testing --- conflink | 1 + mail-setup | 132 +++++++++++++++++++++++++++++++++---------------- mailtest-check | 115 ++++++++++++++++++++++++++++++++++-------- system-status | 6 ++- 4 files changed, 191 insertions(+), 63 deletions(-) diff --git a/conflink b/conflink index 450edf7..b70e26f 100755 --- a/conflink +++ b/conflink @@ -128,6 +128,7 @@ common-file-setup() { } +install-my-scripts user=$(id -un) all_dirs=({/a/bin/ds,/p/c}{,/machine_specific/$HOSTNAME}) # note, we assume a group of hosts does not have the diff --git a/mail-setup b/mail-setup index d076600..c446bbb 100755 --- a/mail-setup +++ b/mail-setup @@ -3,14 +3,22 @@ # Copyright (C) 2019 Ian Kelling # SPDX-License-Identifier: AGPL-3.0-or-later +# todo: disable greylisting + +# todo: enable plus addressing and sort out mail filtering. +# consider maildrop, procmail, etc. + +# todo: test that bounces dont help create valid mailtest-check + # todo: move mail stuff in distro-end into this file +# todo: consider rotating dkim & publishing key so every past email I sent +# isnt necessarily signed + # todo: consider how to get clamav out of Debian-exim group # so it cant read/write the whole mail spool, for better # security. -# todo: fix ipv6 addr for li - # todo: create a cronjob to update or warn on expiring dnssec keys # todo: we should test failed mail daily or so @@ -25,10 +33,9 @@ # todo: mailtest-check failure on remote hosts is not going to alert me. # sort that out. -# todo: test that mail is dkim signing in our mail test. -# todo: send sending from all domains # todo: test mail failure as well as success. -# todo: we should monitor spamhaus etc to make sure we dont get blacklisted. +# +# todo: validate that mailtest-check is doing dnsbl checks. # background: I want to run exim in a network namespace so it can send # and receive through a vpn. This is needed so it can do ipv6, because @@ -191,7 +198,7 @@ fi # echo "bind txt record: remember to truncate $domain so its relative to the bind zone" # cat <$verf fi + # So, strangely, this worked in initial testing, but then + # on first run it wouldn't show the existing contacts until + # I went into the carddav settings and did "force immediate sync", + # which seemed to fix things. Note, some of these settings + # get initalized per/addressbook in the db, then need changing + # there or through the settings menu. + + # About categories, see https://www.davx5.com/tested-with/nextcloud + # https://github.com/blind-coder/rcmcarddav/blob/master/doc/GROUPS.md i $rcdir/plugins/carddav/config.inc.php < 'Main', 'username' => '%u', // login username 'password' => '%p', // login password - 'url' => 'https://$domain/nextcloud/remote.php/carddav/addressbooks/%u/contacts', + 'url' => 'https://$domain/nextcloud/remote.php/dav/addressbooks/users/%u/contacts', 'active' => true, 'readonly' => false, 'refresh_time' => '00:10:00', 'fixed' => array('username','password'), + 'use_categories' => false, 'hide' => false, - 'use_categories' => true, // https://www.davx5.com/tested-with/nextcloud ); ?> EOF @@ -1710,6 +1726,30 @@ fi # * exim host conditional config + +# ** exim certs + +all_dirs=(/p/c/filesystem) +for x in /p/c/machine_specific/*.hosts /a/bin/ds/machine_specific/*.hosts; do + if grep -qxF $HOSTNAME $x; then all_dirs+=( ${x%.hosts} ); fi +done +files=() +for d in ${all_dirs[@]}; do + f=$d/etc/exim4/passwd + if [[ -e $f ]]; then + files+=($f) + fi + tmp=($d/etc/exim4/*.pem) + if (( ${#tmp[@]} )); then + files+=(${tmp[@]}) + fi +done + +if (( ${#files[@]} )); then + sudo rsync -ahhi --chown=root:Debian-exim --chmod=0640 ${files[@]} /etc/exim4/ +fi + + # ** auth case $HOSTNAME in $MAIL_HOST) @@ -1802,9 +1842,7 @@ EOF MAIN_HARDCODE_PRIMARY_HOSTNAME = mail.iankelling.org EOF - /a/exe/cedit nn.b8.nz /etc/hosts <<'EOF' || [[ $? == 1 ]] -# note: i put this into bind for good measure -10.173.8.2 nn.b8.nz + /a/exe/cedit defaultnn /etc/hosts <<'EOF' || [[ $? == 1 ]] # this is just here to avoid mainlog errors, however, it doesnt seem to work # todo: look into it more. nsswitch.conf? cached result? i dunno # list matching forced to fail: failed to find host name for 10.173.8.1 @@ -1875,14 +1913,14 @@ EOF # This name won\'t appear on From: lines of outgoing messages if rewriting is enabled. echo iankelling.org > /etc/mailname - sudo rsync -ahhi --chown=root:Debian-exim --chmod=0640 \ - /p/c/filesystem/etc/exim4/passwd /p/c/filesystem/etc/exim4/*.pem /etc/exim4/ # mail.iankelling.org so local imap clients can connect with tls and # when they happen to not be local. - sed -ri -f - /etc/hosts <<'EOF' -/^127\.0\.1\.1.* mail\.iankelling\.org\b/{p;d} -/^127\.0\.1\.1 /s/ *$/ mail.iankelling.org/ + # todo: this should be 10.8.0.4 + + /a/exe/cedit nn /etc/hosts <<'EOF' || [[ $? == 1 ]] +# note: i put nn.b8.nz into bind for good measure +10.173.8.2 nn.b8.nz mail.iankelling.org EOF # note: systemd-resolved will consult /etc/hosts, dnsmasq wont. this assumes @@ -1924,6 +1962,11 @@ EOF bk) echo amnimal.ninja > /etc/mailname + /a/exe/cedit nn /etc/hosts <<'EOF' || [[ $? == 1 ]] +10.173.8.2 nn.b8.nz +EOF + + i /etc/myexim4/conf.d/router/180_vpnmanual <<'EOF' # copied from dnslookup, altered domains, added route_list, # changed driver, removed ignore_target_hosts since it @@ -1984,12 +2027,9 @@ EOF rm -fv $f done - # remove mail. uses 2 lines to properly remove whitespace - sed -ri -f - /etc/hosts <<'EOF' -s#^(127\.0\.1\.1 .*) +mail\.iankelling\.org$#\1# -s#^(127\.0\.1\.1 .*)mail\.iankelling\.org +(.*)#\1\2# -EOF - + # dont i dont care if defaultnn section gets left, it wont + # get used. + echo | /a/exe/cedit nn /etc/hosts || [[ $? == 1 ]] echo | /a/exe/cedit mail /etc/dnsmasq-servers.conf || [[ $? == 1 ]] cat >>/etc/exim4/update-exim4.conf.conf </dev/null; then + sstart clamav-daemon + # checking a log, clamav took 27 seconds to start. + # we get paniclog entries if its not available + m sleep 30 + fi + if $reload; then sre mailvpn unbound else @@ -2130,7 +2177,7 @@ case $HOSTNAME in ;;& $MAIL_HOST|bk|je) # start spamassassin/dovecot before exim. - sre dovecot spamassassin clamav-daemon + sre dovecot spamassassin sstart mailclean.timer ;;& $MAIL_HOST) @@ -2141,7 +2188,7 @@ esac case $HOSTNAME in $MAIL_HOST|bk|je) : ;; *) - soff radicale mailclean.timer dovecot spamassassin mailvpn mailnn + soff radicale mailclean.timer dovecot spamassassin mailvpn mailnn clamav-daemon ;; esac @@ -2164,7 +2211,8 @@ PATH=/usr/bin:/bin:/usr/local/bin MAILTO=alerts@iankelling.org */5 * * * * $u send-test-forward |& log-once send-test-forward */10 * * * * root chmod -R g+rw /m/md/bounces |& log-once -1 bounces-chmod -*/5 * * * * $u mailtest-check |& log-once -1 mailtest-check +5-59/5 * * * * root mailtest-check |& log-once -1 mailtest-check +0 * * * * root mailtest-check slow |& log-once -1 mailtest-slow EOF m sudo rsync -ahhi --chown=root:root --chmod=0755 \ /b/ds/mailtest-check /b/ds/check-remote-mailqs /usr/local/bin/ diff --git a/mailtest-check b/mailtest-check index 59b7dce..9e35a0a 100755 --- a/mailtest-check +++ b/mailtest-check @@ -17,44 +17,119 @@ e() { $int || return 0; printf "mailtest-check: %s\n" "$*"; } # 7 minutes, the last 2 haven't arrived in a reasonable amount of time. min_limit=7 -int=true -if [[ ! $1 && $- != *i* ]]; then + +# spamassassin checking takes about 8 seconds. only do that every +# once in a while. +slow=false +if [[ $1 == slow ]]; then + slow=true + shift +fi + +# add any argument to say we are running interactively +if [[ $SUDO_USER && ! $1 ]]; then + int=true +else int=false sleep 60 fi folders=(/m/md/l/testignore{,zroe}) +froms=(testignore@je.b8.nz testignore@expertpathologyreview.com testignore@amnimal.ninja) case $HOSTNAME in bk) folders=(/m/md/{expertpathologyreview.com,amnimal.ninja}/testignore) + froms=(ian@iankelling.org z@zroe.org testignore@je.b8.nz) ;; je) + froms=(ian@iankelling.org z@zroe.org testignore@expertpathologyreview.com testignore@amnimal.ninja) folders=(/m/md/je.b8.nz/testignore) ;; esac + for folder in ${folders[@]}; do + for from in ${froms[@]}; do + latest= + cd $folder + last_sec=0 + # webmail sends them to cur it seems + while read -r file; do + if [[ $file -nt $latest ]]; then + latest=$file + fi + done < <(grep -rlFx "From: $from" new cur) - find $folder/new $folder/cur -type f -mmin +300 -delete - cd $folder - last_sec=0 - # webmail sends them to cur it seems - for file in new/* cur/*; do - if [[ $file -nt $latest ]]; then - latest=$file - fi - done + if [[ $latest ]]; then + e latest = $folder/$latest + last_sec=$(awk '/^Subject: / {print $4}' $latest) + + if $slow; then + find $folder/new $folder/cur -type f -mmin +300 -delete + if [[ ! $spamdpid ]]; then + spamdpid=$(systemctl status spamassassin| sed -n '/^ *Main PID:/s/[^0-9]//gp') + fi + if [[ $spamdpid ]]; then + if [[ $(readlink /proc/$$/ns/net) != "$(readlink /proc/$spamdpid/ns/net)" ]]; then + spamcpre="nsenter -t $spamdpid -n -m" + fi - if [[ $latest ]]; then - e latest = $folder/$latest - last_sec=$(awk '/^Subject: / {print $4}' $latest) - fi + declare -A results + for r in $($spamcpre spamc -y <"$latest" |sed 's/,/ /g'); do + case $r in + # we have a new domain, ignore this. + FROM_FMBLA_NEWDOM*) : ;; + SPF_HELO_NEUTRAL) + # some of my domains use neutral spf, treat them the same. + results[SPF_HELO_PASS]=t + ;; + *) + results[$r]=t + ;; + esac + done + # debugging + # e results = ${!results[@]} + missing=() + for t in BODY_SINGLE_WORD DKIM_SIGNED DKIM_VALID{,_AU,_EF} SPF_HELO_PASS SPF_PASS TVD_SPACE_RATIO; do + if [[ ${results[$t]} ]]; then + unset "results[$t]" + elif [[ $t == DKIM_VALID_EF && $from == *@[^.]*.[^.]*.[^.]* ]]; then + : + # third level domains dont hit this. its because + # /usr/share/perl5/Mail/SpamAssassin/Plugin/DKIM.pm checks + # if its signed with the registryboundaries domain. afaik: + # we need the actual domain to sign it, this would result in + # a second signature. I only use second level domains for + # testing atm, fsf doesnt use them for anything but the + # forum and I dont expect that to have any deliverability + # problems. So, not bothering atm. + else + missing+=($t) + fi + done + if (( ${#results[@]} || ${#missing[@]} )); then + printf "$HOSTNAME spamtest %s %s" "$folder" "$from" + if (( ${#results[@]} )); then + printf " unexpected %s" "${!results[*]}" + fi + if (( ${#missing[@]} )); then + printf " missing %s" "${missing[*]}" + fi + echo + fi + else + echo $HOSTNAME mailtest spamd pid not found + fi + fi # if $slow + fi # if [[ $latest ]] - now=$(date +%s) - limit=$(( now - 60 * min_limit )) + now=$(date +%s) + limit=$(( now - 60 * min_limit )) - if (( last_sec <= limit )); then - echo $HOSTNAME mailtest $folder $(date -d @$last_sec +'%a %m-%d %H:%M') - fi + if (( last_sec <= limit )); then + echo $HOSTNAME mailtest $folder $from $(date -d @$last_sec +'%a %m-%d %H:%M') + fi + done done diff --git a/system-status b/system-status index b7f4433..93d31e4 100644 --- a/system-status +++ b/system-status @@ -67,10 +67,14 @@ write-status() { if [[ -e ${glob[0]} ]]; then chars+=("A") fi - tmp=(~/cron-errors/mailtest-check*) + tmp=(/var/local/cron-errors/mailtest-check*) if (( ${#tmp[@]} )); then chars+=("MAILPING") fi + tmp=(/var/local/cron-errors/mailtest-slow*) + if (( ${#tmp[@]} )); then + chars+=("SPAMD") + fi if ! qlen=$(/usr/sbin/exiqgrep -o 60 -c -b | awk '{print $1}'); then # early in install process, we dont have permission yet for exiqgrep -- 2.30.2