X-Git-Url: https://iankelling.org/git/?a=blobdiff_plain;ds=sidebyside;f=mail-setup;h=3ac3476c337be913ba03f3cee8a73bcafd51cef6;hb=HEAD;hp=050a716b2255ea78ce73fe5f646980ad659af6e1;hpb=c536de80536e71af6e69eb1a57575ca1a41101c6;p=distro-setup diff --git a/mail-setup b/mail-setup index 050a716..616d39b 100755 --- a/mail-setup +++ b/mail-setup @@ -41,15 +41,15 @@ # /usr/sbin/invoke-rc.d-diverted "$@" # fi -# Things I tend to forget. on MAIL_HOST, daemon runs with /etc/exim4/my.conf, +# Things I tend to forget. on MAIL_HOST, daemon runs with /etc/exim4/nn-mainlog.conf, # due to /etc/default/exim4 containing: -# COMMONOPTIONS='-C /etc/exim4/my.conf' -# UPEX4OPTS='-o /etc/exim4/my.conf' +# COMMONOPTIONS='-C /etc/exim4/nn-mainlog.conf' +# UPEX4OPTS='-o /etc/exim4/nn-mainlog.conf' # # The non-daemon config -# gets generated from this script calling update-exim4.conf -d /etc/myexim4 +# gets generated from this script calling update-exim4.conf -d /etc/nond-exim4 # which has log path -# log_file_path = /var/log/exim4/my%s +# log_file_path = /var/log/exim4/nond%s # # On non bk|MAIL_HOST, the config and log file are all standard. # @@ -107,7 +107,7 @@ # todo: run mailping test after running, or otherwise # clear out terminal alert -# todo: disable postgrey +# todo: disable postgrey. (why did we have it?) # todo: in testforward-check, we should also look @@ -361,28 +361,8 @@ reload=false if [[ -e /var/local/mail-setup-reload ]]; then reload=true fi -u() { # update file. note: duplicated in brc - local tmp tmpdir dest="$1" - local base="${dest##*/}" - local dir="${dest%/*}" - if [[ $dir != "$base" ]]; then - # dest has a directory component - mkdir -p "$dir" - fi - ur=false # u result - tmpdir=$(mktemp -d) - cat >$tmpdir/"$base" - tmp=$(rsync -ic $tmpdir/"$base" "$dest") - if [[ $tmp ]]; then - printf "%s\n" "$tmp" - ur=true - if [[ $dest == /etc/systemd/system/* ]]; then - touch /var/local/mail-setup-reload - reload=true - fi - fi - rm -rf $tmpdir -} + +source /a/bin/fai/fai/config/distro-install-common/bash-misc-funcs setini() { key="$1" value="$2" section="$3" file="/etc/radicale/config" @@ -415,7 +395,6 @@ sre() { mailhost() { [[ $HOSTNAME == "$MAIL_HOST" ]] } -e() { printf "%s\n" "$*"; } reifactive() { for service; do if systemctl is-active $service >/dev/null; then @@ -474,13 +453,42 @@ Pin-Priority: 500 EOF fi + +unit-exists() { + systemctl cat $1 &>/dev/null +} +spamd-timer-exists() { + unit-exists spamassassin-maintenance.timer +} + +# name change in t12, and now timer instead of cron option in /etc/default +first_spamd_run=false +if ! spamd-timer-exists; then + first_spamd_run=true +fi + + # light version of exim does not have sasl auth support. # note: for bitfolk hosts, unbound has important config with conflink. pi-nostart exim4 exim4-daemon-heavy spamassassin unbound clamav-daemon wireguard +spamd_ser=spamd +if systemctl cat spamassassin &>/dev/null; then + spamd_ser=spamassassin +elif $first_spamd_run; then + if spamd-timer-exists; then + systemctl start spamassassin-maintenance + fi +fi + +if spamd-timer-exists; then + systemctl enable --now spamassassin-maintenance.timer +fi + # note: pyzor debian readme says you need to run some initialization command # but its outdated. -pi spf-tools-perl p0f postgrey pyzor razor jq moreutils certbot fail2ban +pi spf-tools-perl p0f pyzor razor jq moreutils certbot fail2ban +pu postgrey case $HOSTNAME in je) : ;; # not included due to using wireguard: openvpn @@ -509,6 +517,77 @@ fi # our nostart pi fails to avoid enabling + +# * initial dns config & daemon setup +# +# use systemd-resolved for glibc resolutions, setup symlinks + +pi libnss-resolve + +# if this link gets replaced with a normal file we will get exim log +# errors on MAIL_HOST like so: +# +# R=fsfsmarthost defer (-36) DT=0s: host lookup for mail.fsf.org did not complete (DNS timeout?) + +if [[ ! -L /etc/nsswitch.conf ]]; then + sudo mkdir -p /etc/resolved-nsswitch + sudo mv /etc/nsswitch.conf /etc/resolved-nsswitch + sudo ln -sf /etc/resolved-nsswitch/nsswitch.conf /etc +fi + +f=/etc/basic-nsswitch/nsswitch.conf +if [[ ! -e $f ]]; then + sudo mkdir -p ${f%/*} + sudo cp /etc/nsswitch.conf $f + sudo sed -i --follow-symlinks 's/^ *hosts:.*/hosts: files dns myhostname/' $f +fi +case $HOSTNAME in + bk|je) + # je should be able to get along systemd-resolved, but ive had some odd + # very intermittent dns failures with spamassassin, it seems it might only + # be happening with systemd-resolved, so just use unbound + # to make it consistent with the other hosts. + sudo sed -i --follow-symlinks 's/^ *hosts:.*/hosts: files dns myhostname/' /etc/nsswitch.conf + soff systemd-resolved + sudo ln -sf 127.0.0.1-resolv/stub-resolv.conf /etc/resolv.conf + sgo unbound + # cautious measure to make sure resolution is working + sleep 1 + ;; + *) + # default is + # files mdns4_minimal [NOTFOUND=return] dns myhostname + # mdns4 is needed for my printer and for bbb webrtc, not sure exactly why. + # https://www.freedesktop.org/software/systemd/man/nss-resolve.html# + # seems more important than some potential use case. + # Interestingly, t9/t10 man page says use files before resolve, debian 10 says the opposite. + # removing files makes hostname -f not actually give the fully qualified domain name. + sudo sed -i --follow-symlinks 's/^ *hosts:.*/hosts: files resolve [!UNAVAIL=return] mdns4_minimal [NOTFOUND=return] myhostname/' /etc/resolved-nsswitch/nsswitch.conf + ;; +esac + +case $HOSTNAME in + bk) + sgo named + ;; +esac + + +u /etc/apparmor.d/abstractions/nameservice.d/iank <<'EOF' +/etc/resolved-nsswitch/nsswitch.conf r, +/etc/basic-nsswitch/nsswitch.conf r, +# Aug 06 23:09:11 kd audit[3995]: AVC apparmor="DENIED" operation="connect" profile="/usr/bin/freshclam" name="/run/systemd/resolve/io.systemd.Resolve" pid=3995 comm="freshclam" requested_mask="wr" denied_mask="wr" fsuid=109 ouid=101 +# I dont know if this is quite the right fix, but I saw other sockets +# in the nameservice files that were rw, so figured it was ok to add this and it worked. +/run/systemd/resolve/io.systemd.Resolve rw, +EOF + +if $ur && systemctl is-active apparmor; then + m systemctl reload apparmor +fi + + + # * Mail clean cronjob u /etc/systemd/system/mailclean.timer <<'EOF' @@ -775,25 +854,26 @@ EOF # this is just a bug fix for trisquel. f=/etc/apparmor.d/usr.sbin.unbound -line="/usr/sbin/unbound flags=(attach_disconnected) {" -if ! grep -qFx "$line" $f; then - badline="/usr/sbin/unbound {" - if ! grep -qFx "$badline" $f; then +good_string="/usr/sbin/unbound flags=(attach_disconnected) {" +if ! grep -qF "$good_string" $f; then + bad_string="/usr/sbin/unbound {" + if ! grep -qF "$bad_string" $f; then err expected line in $f not found fi - sed -i "s,^$badline$,$line," $f + sed -i "s,$bad_string$,$good_string," $f if systemctl is-active apparmor &>/dev/null; then m systemctl reload apparmor fi fi + # note: anything added to nn_progs needs corresponding rm # down below in the host switch nn_progs=(exim4) if mailhost; then # Note dovecots lmtp doesnt need to be in the same nn to accept delivery. # Its in the nn so remote clients can connect to it. - nn_progs+=(spamassassin dovecot) + nn_progs+=($spamd_ser dovecot) fi case $HOSTNAME in @@ -873,7 +953,7 @@ EOF done ;; *) - for unit in exim4 spamassassin dovecot unbound; do + for unit in exim4 $spamd_ser dovecot unbound; do f=/etc/systemd/system/$unit.service.d/nn.conf if [[ -s $f ]]; then rm -fv $f @@ -925,12 +1005,14 @@ EOF # 2020-10-19 remove old file. remove this when all hosts updated rm -fv /etc/systemd/system/spamddnsfix.{timer,service} -u /etc/default/spamassassin <<'EOF' +u /etc/default/$spamd_ser <<'EOF' # defaults plus debugging flags for an issue im having OPTIONS="--create-prefs --max-children 5 --helper-home-dir" -PIDFILE="/var/run/spamd.pid" +PIDFILE="/run/spamd.pid" # my additions NICE="--nicelevel 15" +# not used in t12+, that uses +# /usr/lib/systemd/system/spamassassin-maintenance.timer CRON=1 EOF @@ -2868,7 +2950,7 @@ debbugsconfig # ld for local debbugs -/a/exe/web-conf -t -a 127.0.1.1 -p 80 -r /var/lib/debbugs/www - apache2 ld <<'EOF' +/a/exe/web-conf -l -t -a 127.0.1.1 -p 80 -r /var/lib/debbugs/www - apache2 ld <<'EOF' # copied from debbugs upstream example Options Indexes SymLinksIfOwnerMatch MultiViews @@ -2981,10 +3063,11 @@ case $HOSTNAME in # which will overwrite any existing file u /etc/default/exim4 <<'EOF' QUEUERUNNER='combined' -# note: this is duplicated in brc2, 10m here is -q10m there. QUEUEINTERVAL='10m' -COMMONOPTIONS='-C /etc/exim4/my.conf' -UPEX4OPTS='-o /etc/exim4/my.conf' +COMMONOPTIONS='-C /etc/exim4/nn-mainlog.conf' +UPEX4OPTS='-o /etc/exim4/nn-mainlog.conf' +# in t12 exim, this replaces all the above options +EXIMSERVICE='-bdf -q10m -C /etc/exim4/nn-mainlog.conf' # i use epanic-clean for alerting if there are bad paniclog entries E4BCD_WATCH_PANICLOG='no' EOF @@ -2994,13 +3077,13 @@ EOF chmod g+s,u+s /usr/sbin/exim4 # need this to avoid error on service reload: # 2022-08-07 18:44:34.005 [892491] pid 892491: SIGHUP received: re-exec daemon - # 2022-08-07 18:44:34.036 [892491] cwd=/var/spool/exim4 5 args: /usr/sbin/exim4 -bd -q30m -C /etc/exim4/my.conf + # 2022-08-07 18:44:34.036 [892491] cwd=/var/spool/exim4 5 args: /usr/sbin/exim4 -bd -q30m -C /etc/exim4/nn-mainlog.conf # 2022-08-07 18:44:34.043 [892491] socket bind() to port 25 for address (any IPv6) failed: Permission denied: waiting 30s before trying again (9 more tries) # note: the daemon gives up and dies after retrying those 9 times. # I came upon this by guessing and trial and error. setcap CAP_NET_BIND_SERVICE+ei /usr/sbin/exim4 u /etc/exim4/trusted_configs <<'EOF' -/etc/exim4/my.conf +/etc/exim4/nn-mainlog.conf EOF ;; *) @@ -3181,6 +3264,11 @@ bounce_debbugs: domains = DEBBUGS_DOMAIN EOF + install -m=0775 -d -g Debian-exim -o iank /var/spool/exim4/gw + f=/var/spool/exim4/gw/.no-delay-eximids + if [[ ! -e $f ]]; then + install -g Debian-exim -o iank /dev/null $f + fi u /etc/exim4/conf.d/router/155_delay <<'EOF' # By default, delay sending email by 30-40 minutes in case I # change my mind. @@ -3200,14 +3288,14 @@ delay_iank: condition = ${if and { \ {< {$tod_epoch} {${eval10:$received_time + 60*30}}} \ {!def:h_i:} \ -{!bool{${lookup{$message_exim_id}lsearch{/etc/exim4/no-delay-eximids}{true}}}} \ -{!bool{${lookup{all}lsearch{/etc/exim4/no-delay-eximids}{true}}}} \ +{!bool{${lookup{$message_exim_id}lsearch{/var/spool/exim4/gw/.no-delay-eximids}{true}}}} \ +{!bool{${lookup{all}lsearch{/var/spool/exim4/gw/.no-delay-eximids}{true}}}} \ } {true}{false}} headers_remove = <; i: domains = ! +local_domains # uncomment for testing delays to jtuttle # local_parts = ! root : ! testignore : ! alerts : ! ian-pager : ! daylert - local_parts = ! root : ! testignore : ! alerts : ! jtuttle : ! ian-pager : ! daylert + local_parts = ! root : ! testignore : ! alerts : ! jtuttle : ! ian-pager : ! daylert : ! r2e ignore_target_hosts = ROUTER_DNSLOOKUP_IGNORE_TARGET_HOSTS EOF @@ -3462,7 +3550,7 @@ EOF # defaults but no queue runner and alternate config dir QUEUERUNNER='no' COMMONOPTIONS='-oP /run/exim4/eximin.pid' -UPEX4OPTS='-d /etc/myexim4' +UPEX4OPTS='-d /etc/nond-exim4' EOF echo bk.b8.nz > /etc/mailname @@ -3622,16 +3710,16 @@ case $HOSTNAME in m rsync -ra --delete --delete-excluded \ --exclude=/conf.d/router/161_backup_redir_nn \ --exclude=/conf.d/router/186_sentarchive_nn \ - --exclude=/conf.d/main/000_local-nn /etc/exim4/ /etc/myexim4 - cat >>/etc/myexim4/conf.d/main/000_local <<'EOF' + --exclude=/conf.d/main/000_local-nn /etc/exim4/ /etc/nond-exim4 + cat >>/etc/nond-exim4/conf.d/main/000_local <<'EOF' # this makes it easier to see which exim is doing what -log_file_path = /var/log/exim4/my%s +log_file_path = /var/log/exim4/nond%s EOF cat >/etc/logrotate.d/myexim <<'EOF' -/var/log/exim4/mymain /var/log/exim4/myreject { +/var/log/exim4/nondmain /var/log/exim4/nondreject { daily missingok rotate 1000 @@ -3639,7 +3727,7 @@ EOF notifempty nocreate } -/var/log/exim4/mypanic { +/var/log/exim4/nondpanic { size 10M missingok rotate 10 @@ -3653,9 +3741,9 @@ EOF # If we ever wanted to have a separate spool, # we could do it like this. # cat >>/etc/exim4/conf.d/main/000_local-nn <<'EOF' - # spool_directory = /var/spool/myexim4 + # spool_directory = /var/spool/nond-exim4 # EOF - cat >>/etc/myexim4/update-exim4.conf.conf <<'EOF' + cat >>/etc/nond-exim4/update-exim4.conf.conf <<'EOF' dc_eximconfig_configtype='smarthost' dc_smarthost='nn.b8.nz' EOF @@ -3663,14 +3751,14 @@ EOF bk) # config for the non-nn exim - cat >>/etc/myexim4/conf.d/main/000_local <<'EOF' + cat >>/etc/nond-exim4/conf.d/main/000_local <<'EOF' MAIN_HARDCODE_PRIMARY_HOSTNAME = mail2.iankelling.org EOF ;; $MAIL_HOST) - u /etc/myexim4/conf.d/router/185_sentarchive <<'EOF' + u /etc/nond-exim4/conf.d/router/185_sentarchive <<'EOF' sentarchive: driver = redirect domains = ! +local_domains @@ -3680,7 +3768,7 @@ sentarchive: unseen EOF - u /etc/myexim4/conf.d/router/160_backup_redir <<'EOF' + u /etc/nond-exim4/conf.d/router/160_backup_redir <<'EOF' backup_redir: driver = redirect # i dont email myself from my own machine much, so lets ignore that. @@ -3697,7 +3785,7 @@ backup_redir: EOF # for bk, we have a exim4in.service that will do this for us. - m update-exim4.conf -d /etc/myexim4 + m update-exim4.conf -d /etc/nond-exim4 ;; esac @@ -3784,6 +3872,131 @@ fi # err debbugs exist but is not uid 610: investigate # fi +# * mail monitoring / testing + +# note, to test clamav, send an email with body that only contains +# https://en.wikipedia.org/wiki/EICAR_test_file +# which set malware_name to Eicar-Signature +case $HOSTNAME in + $MAIL_HOST|bk|je) + # note: cronjob "ian" also does some important monitoring + # todo: this will sometimes cause an alert because mailtest-check will run + # before we have setup network namespace and spamassassin + u /etc/cron.d/mailtest <>/etc/cron.d/mailtest </usr/local/bin/send-test-forward <<'EOF' +#!/bin/bash +# we remove from the queue older than 4.3 minutes since we send every 5 minutes. +olds=( +$(/usr/sbin/exiqgrep -o 260 -i -r '^(testignore@(iankelling\.org|zroe\.org|expertpathologyreview\.com|amnimal\.ninja|je\.b8\.nz)|jtuttle@gnu\.org)$') +) +if (( ${#olds[@]} )); then + /usr/sbin/exim -Mrm "${olds[@]}" >/dev/null +fi +EOF + for test_from in ${test_froms[@]}; do + + test_to=${test_tos[0]} + for t in ${test_tos[@]:1}; do + if [[ $test_from == *@gnu.org && $t == *@gnu.org ]]; then + continue + fi + test_to+=", $t" + done + case $test_from in + testignore@expertpathologyreview.com) + test_to=testignore@zroe.org + ;; + esac + + cat >>/usr/local/bin/send-test-forward <>/etc/cron.d/mailtest </usr/local/bin/send-test-forward <<'EOF' -#!/bin/bash -# we remove from the queue older than 4.3 minutes since we send every 5 minutes. -olds=( -$(/usr/sbin/exiqgrep -o 260 -i -r '^(testignore@(iankelling\.org|zroe\.org|expertpathologyreview\.com|amnimal\.ninja|je\.b8\.nz)|jtuttle@gnu\.org)$') -) -if (( ${#olds[@]} )); then - /usr/sbin/exim -Mrm "${olds[@]}" >/dev/null -fi -EOF - for test_from in ${test_froms[@]}; do - - test_to=${test_tos[0]} - for t in ${test_tos[@]:1}; do - if [[ $test_from == *@gnu.org && $t == *@gnu.org ]]; then - continue - fi - test_to+=", $t" - done - case $test_from in - testignore@expertpathologyreview.com) - test_to=testignore@zroe.org - ;; - esac - - cat >>/usr/local/bin/send-test-forward <