X-Git-Url: https://iankelling.org/git/?a=blobdiff_plain;f=mail-setup;h=9b9a97c04589f8a356c65d0b0b80260df06dc010;hb=ad09c51104f62d1da1782387025b44327a081872;hp=7da0524bedc188bb98fe9d9783a824576a7db27b;hpb=fa5deaee2e0182ddfc7b39eea7ee2acedb259ddf;p=distro-setup diff --git a/mail-setup b/mail-setup index 7da0524..9b9a97c 100755 --- a/mail-setup +++ b/mail-setup @@ -4,16 +4,23 @@ # SPDX-License-Identifier: AGPL-3.0-or-later +# todo: setup an alert for bouncing test emails. + +# todo: bounces to my fsf mail can come from fsf@iankelling.org, +# think about making bounces go from the original address. + # todo: add a prometheus alert for dovecot. # todo: handle errors like this: # Mar 02 12:44:26 kw systemd[1]: exim4.service: Found left-over process 68210 (exim4) in control group while starting unit. Ignoring. # Mar 02 12:44:26 kw systemd[1]: This usually indicates unclean termination of a previous run, or service implementation deficiencies. +#eg: on eggs, on may 1st, ps grep for exim, 2 daemons running. 1 leftover from a month ago +#Debian-+ 1954 1 0 36231 11560 4 Apr02 ? 00:40:25 /usr/sbin/exim4 -bd -q30m +#Debian-+ 23058 1954 0 36821 10564 0 20:38 ? 00:00:00 /usr/sbin/exim4 -bd -q30m # todo: harden dovecot. need to do some research. one way is for it to only listen on a wireguard vpn interface, so only clients that are on the vpn can access it. # todo: consider hardening cups listening on 0.0.0.0 # todo: stop/disable local apache, and rpc.mountd, and kdeconnect when not in use. -# todo: check that spamd and unbound only listen locally. # todo: hosts should only allow external mail that is authed and # destined for backup route. it is a minor issue since traffic is @@ -25,10 +32,6 @@ # todo: run mailping test after running, or otherwise # clear out terminal alert -# todo: on bk, dont send email if mailvpn is not up - -# todo: mailtest-check should check on bk too - # todo: disable postgrey # todo: in testforward-check, we should also look @@ -287,7 +290,8 @@ i() { # install file local base="${dest##*/}" local dir="${dest%/*}" if [[ $dir != "$base" ]]; then - mkdir -p ${dest%/*} + # dest has a directory component + mkdir -p "$dir" fi ir=false # i result tmpdir=$(mktemp -d) @@ -343,7 +347,6 @@ stopifactive() { mxhost=mx.iankelling.org mxport=587 -forward=$u@$mxhost # old setup. left as comment for example # mxhost=mail.messagingengine.com @@ -386,6 +389,7 @@ EOF fi # light version of exim does not have sasl auth support. +# note: for bitfolk hosts, unbound has important config with conflink. pi-nostart exim4 exim4-daemon-heavy spamassassin unbound clamav-daemon wireguard # note: pyzor debian readme says you need to run some initialization command @@ -464,8 +468,48 @@ EOF # old. #vpnser=mailvpn.service -# todo: this hangs if it cant resolv the endpoint. we -# want it to just retry in the background. +# note: this hangs if it cant resolv the endpoint. we +# want it to just retry in the background. i just use a static ip instead. +# +# Note: at least on t10, on reboot, the service fails to come up according to systemd, but +# in reality it is up and working, then it tries to restart infinitely, and fails +# because it detects that the interface exists. +# +# failing output: +# +# Aug 02 21:59:27 sy wg-quick[2092]: [#] sysctl -q net.ipv4.conf.all.src_valid_mark=1 +# Aug 02 21:59:27 sy wg-quick[2248]: [#] iptables-restore -n +# Aug 02 21:59:27 sy wg-quick[2249]: Another app is currently holding the xtables lock. Perhaps you want to use the -w option? +# Aug 02 21:59:27 sy wg-quick[2259]: [#] iptables-restore -n +# Aug 02 21:59:27 sy wg-quick[2260]: Another app is currently holding the xtables lock. Perhaps you want to use the -w option? +# Aug 02 21:59:27 sy systemd[1]: wg-quick@wgmail.service: Main process exited, code=exited, status=4/NOPERMISSION + + +# successful output. +# Aug 03 14:12:47 sy wg-quick[711336]: [#] sysctl -q net.ipv4.conf.all.src_valid_mark=1 +# Aug 03 14:12:47 sy wg-quick[711384]: [#] iptables-restore -n +# Aug 03 14:12:47 sy wg-quick[711336]: [#] ping -w10 -c1 10.8.0.1 ||: +# Aug 03 14:12:47 sy wg-quick[711389]: PING 10.8.0.1 (10.8.0.1) 56(84) bytes of data. +# Aug 03 14:12:47 sy wg-quick[711389]: 64 bytes from 10.8.0.1: icmp_seq=1 ttl=64 time=73.0 ms +# Aug 03 14:12:47 sy wg-quick[711389]: --- 10.8.0.1 ping statistics --- +# Aug 03 14:12:47 sy wg-quick[711389]: 1 packets transmitted, 1 received, 0% packet loss, time 0ms +# Aug 03 14:12:47 sy wg-quick[711389]: rtt min/avg/max/mdev = 72.993/72.993/72.993/0.000 ms +# Aug 03 14:12:47 sy systemd[1]: Finished WireGuard via wg-quick(8) for wgmail. +# Aug 02 21:59:27 sy systemd[1]: wg-quick@wgmail.service: Failed with result 'exit-code'. +# Aug 02 21:59:27 sy systemd[1]: Failed to start WireGuard via wg-quick(8) for wgmail. +# Aug 02 21:59:47 sy systemd[1]: wg-quick@wgmail.service: Scheduled restart job, restart counter is at 1. +# Aug 02 21:59:47 sy systemd[1]: Stopped WireGuard via wg-quick(8) for wgmail. +# Aug 02 21:59:47 sy systemd[1]: Starting WireGuard via wg-quick(8) for wgmail... +# Aug 02 21:59:47 sy wg-quick[3424]: wg-quick: `wgmail' already exists +# Aug 02 21:59:47 sy systemd[1]: wg-quick@wgmail.service: Main process exited, code=exited, status=1/FAILURE +# Aug 02 21:59:47 sy systemd[1]: wg-quick@wgmail.service: Failed with result 'exit-code'. +# Aug 02 21:59:47 sy systemd[1]: Failed to start WireGuard via wg-quick(8) for wgmail. + + +# According to iptables -S and iptables -t nat -S, +# there are no modifications to iptables rules on a succsfull run, +# and + vpnser=wg-quick@wgmail.service case $HOSTNAME in @@ -954,6 +998,14 @@ enabled = true port = 25,587 filter = exim banaction = iptables-exim + +# 209.51.188.13 = mail.fsf.org +# 2001:470:142::13 = mail.fsf.org +# 209.51.188.92 = eggs.gnu.org +# 2001:470:142:3::10 = eggs.gnu.org +# 72.14.176.105 2600:3c00:e000:280::2 = mail.iankelling.org +# 10.173.8.1 = non-nn net +ignoreip = 209.51.188.13 2001:470:142::13 209.51.188.92 2001:470:142:3::10 72.14.176.105 2600:3c00:e000:280::2 10.173.8.1 EOF if $ir; then m systemctl restart fail2ban @@ -991,10 +1043,8 @@ awk 'BEGIN { FS = ":" } ; $6 ~ /^\/home/ && $7 !~ /\/nologin$/ { print $1 }' /et esac done -if ! grep -q "^ncsoft:" /etc/aliases; then - echo "ncsoft: graceq2323@gmail.com" |m tee -a /etc/aliases -fi +. /a/bin/bash_unpublished/priv-mail-setup m gpasswd -a iank adm #needed for reading logs @@ -1063,11 +1113,11 @@ rm -fv /etc/exim4/conf.d/retry/37_retry cat >/etc/exim4/conf.d/retry/17_retry <<'EOF' # Retry fast for my own domains -iankelling.org * F,1d,10m;F,14d,1h -amnimal.ninja * F,1d,10m;F,14d,1h -expertpathologyreview.com * F,1d,10m;F,14d,1h -je.b8.nz * F,1d,10m;F,14d,1h -zroe.org * F,1d,10m;F,14d,1h +iankelling.org * F,1d,4m;F,14d,1h +amnimal.ninja * F,1d,4m;F,14d,1h +expertpathologyreview.com * F,1d,4m;F,14d,1h +je.b8.nz * F,1d,4m;F,14d,1h +zroe.org * F,1d,4m;F,14d,1h eximbackup.b8.nz * F,1d,4m;F,14d,1h # The spec says the target domain will be used for temporary host errors, @@ -1078,6 +1128,13 @@ defaultnn.b8.nz * F,1d,4m;F,14d,1h mx.iankelling.org * F,1d,4m;F,14d,1h bk.b8.nz * F,1d,4m;F,14d,1h eggs.gnu.org * F,1d,4m;F,14d,1h +fencepost.gnu.org * F,1d,4m;F,14d,1h + +# afaik our retry doesnt need this, but just using everything +mx.amnimal.ninja * F,1d,4m;F,14d,1h +mx.expertpathologyreview.com * F,1d,4m;F,14d,1h + + mail.fsf.org * F,1d,15m;F,14d,1h EOF @@ -1146,17 +1203,17 @@ DKIM_SIGN_HEADERS = mime-version:in-reply-to:references:from:date:subject:to domainlist local_hostnames = ! je.b8.nz : ! bk.b8.nz : *.b8.nz : b8.nz -hostlist iank_trusted = <; \\ +hostlist iank_trusted = <; \ # veth0 -10.173.8.1 ; \\ +10.173.8.1 ; \ # li li_ip6 -72.14.176.105 ; 2600:3c00::f03c:91ff:fe6d:baf8 ; \\ +72.14.176.105 ; 2600:3c00::f03c:91ff:fe6d:baf8 ; \ # li_vpn_net li_vpn_net_ip6s -10.8.0.0/24; 2600:3c00:e000:280::/64 ; 2600:3c00:e002:3800::/56 ; \\ +10.8.0.0/24; 2600:3c00:e000:280::/64 ; 2600:3c00:e002:3800::/56 ; \ # bk bk_ip6 -85.119.83.50 ; 2001:ba8:1f1:f0c9::2 ; \\ +85.119.83.50 ; 2001:ba8:1f1:f0c9::2 ; \ # je je_ipv6 -85.119.82.128 ; 2001:ba8:1f1:f09d::2 ; \\ +85.119.82.128 ; 2001:ba8:1f1:f09d::2 ; \ # fsf_mit_net fsf_mit_net_ip6 fsf_net fsf_net_ip6 fsf_office_net 18.4.89.0/24 ; 2603:3005:71a:2e00::/64 ; 209.51.188.0/24 ; 2001:470:142::/48 ; 74.94.156.208/28 @@ -1172,6 +1229,17 @@ delay_warning_condition = ${if or {\ } {no}{yes}} +# enable 587 in addition to the default 25, so that +# i can send mail where port 25 is firewalled by isp +daemon_smtp_ports = 25 : 587 +# default of 25, can get stuck when catching up on mail +smtp_accept_max = 400 +smtp_accept_reserve = 100 +smtp_reserve_hosts = +iank_trusted + +# Rules that make receiving more liberal should be on backup hosts +# so that we dont reject mail accepted by MAIL_HOST +LOCAL_DENY_EXCEPTIONS_LOCAL_ACL_FILE = /etc/exim4/conf.d/local_deny_exceptions_acl EOF rm -fv /etc/exim4/rcpt_local_acl # old path @@ -1192,6 +1260,7 @@ accept EOF rm -fv /etc/exim4/data_local_acl # old path + i /etc/exim4/conf.d/data_local_acl <<'EOF' # Except for the "condition =", this was # a comment in the check_data acl. The comment about this not @@ -1210,6 +1279,8 @@ warn warn !hosts = +iank_trusted + # Smarthosts connect with residential ips and thus get flagged as spam if we do a spam check. + !authenticated = plain_server:login_server condition = ${if < {$message_size}{5000K}} spam = Debian-exim:true add_header = X-Spam_score_int: $spam_score_int @@ -1218,11 +1289,6 @@ warn add_header = X-Spam_report: $spam_report add_header = X-Spam_action: $spam_action -warn - condition = ${if def:malware_name} - remove_header = Subject: - add_header = Subject: [Clamav warning: $malware_name] $h_subject - log_message = heuristic malware warning: $malware_name #accept # spf = pass:fail:softfail:none:neutral:permerror:temperror @@ -1481,7 +1547,7 @@ EOF # disable power management feature, set to 240 min sync interval, # so it shouldn't be bad. - # davdroid from f-druid. + # davx^5 from f-droid # login with url and user name # url https://cal.iankelling.org/ian # username ian @@ -1588,7 +1654,7 @@ EOF cat <<'EOF' # https://ssl-config.mozilla.org ssl = required -# this is the same as the certbot list, in my cert cronjob, I check if that has changed upstream. +# this is the same as the certbot list, i check changes in /a/bin/ds/filesystem/usr/local/bin/check-lets-encrypt-ssl-settings ssl_cipher_list = ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384 ssl_protocols = TLSv1.2 ssl_prefer_server_ciphers = no @@ -1599,7 +1665,7 @@ protocol lmtp { mail_plugins = $mail_plugins sieve } EOF - if dpkg --compare-versions $(dpkg-query -f='${Version}\n' --show dovecot-core) ge 1:2.3; then + if dpkg --compare-versions "$(dpkg-query -f='${Version}\n' --show dovecot-core)" ge 1:2.3; then cat <config.php - m rm -f tmp.php + e running php tmp.php + php tmp.php >config.php + # leave in place for debugging + #m rm -f tmp.php m sudo -u www-data php $ncdir/occ maintenance:update:htaccess list=$(sudo -u www-data php $ncdir/occ --output=json_pretty app:list) # user_external not compaible with nc 23 @@ -2283,22 +2351,39 @@ EOF systemctl enable --now $ncbase.timer i /usr/local/bin/ncup <<'EOFOUTER' #!/bin/bash -if ! test "$BASH_VERSION"; then echo "error: shell is not bash" >&2; exit 1; fi -shopt -s inherit_errexit 2>/dev/null ||: # ignore fail in bash < 4.4 -set -eE -o pipefail -trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" exit status: $?, PIPESTATUS: ${PIPESTATUS[*]}" >&2' ERR -ncbase=$1 -if ! php /var/www/$ncbase/updater/updater.phar -n; then +source /usr/local/lib/err + +m() { printf "%s\n" "$*"; "$@"; } +err-cleanup() { echo failed nextcloud update for $ncbase >&2 - /sbin/exim -t <>/etc/exim4/conf.d/main/000_local <>/etc/exim4/conf.d/main/000_local <>/etc/myexim4/conf.d/main/000_local-nn <<'EOF' +# this makes it easier to see which exim is doing what +log_file_path = /var/log/exim4/my%s +EOF + + cat >/etc/logrotate.d/myexim <<'EOF' +/var/log/exim4/mymain /var/log/exim4/myreject { + daily + missingok + rotate 1000 + delaycompress + notifempty + nocreate +} +/var/log/exim4/mypanic { + size 10M + missingok + rotate 10 + compress + delaycompress + notifempty + nocreate +} +EOF + # If we ever wanted to have a separate spool, # we could do it like this. # cat >>/etc/exim4/conf.d/main/000_local-nn <<'EOF' @@ -3093,12 +3225,19 @@ case $HOSTNAME in $MAIL_HOST) # < 2.1 (eg: in t9), uses a different data format which required manual # migration. dont start if we are running an old version. - if dpkg --compare-versions $(dpkg -s radicale | awk '$1 == "Version:" { print $2 }') ge 2.1; then + if dpkg --compare-versions "$(dpkg -s radicale | awk '$1 == "Version:" { print $2 }')" ge 2.1; then m systemctl --now enable radicale fi ;;& esac +# for debugging dns issues +case $HOSTNAME in + je|bk) + systemctl enable --now logrotate-fast.timer + ;; +esac + # last use of $reload happens in previous block rm -f /var/local/mail-setup-reload @@ -3139,39 +3278,75 @@ case $HOSTNAME in # note: cronjob "ian" also does some important monitoring # todo: this will sometimes cause an alert because mailtest-check will run # before we have setup network namespace and spamassassin - cat >/etc/cron.d/mailtest <>/etc/cron.d/mailtest <> /etc/exim4/ignore-sent + done + cat >/usr/local/bin/send-test-forward <<'EOF' #!/bin/bash +# we remove from the queue older than 4.3 minutes since we send every 5 minutes. olds=( $(/usr/sbin/exiqgrep -o 260 -i -r '^(testignore@(iankelling\.org|zroe\.org|expertpathologyreview\.com|amnimal\.ninja|je\.b8\.nz)|jtuttle@gnu\.org)$') ) @@ -3180,8 +3355,19 @@ if (( ${#olds[@]} )); then fi EOF for test_from in ${test_froms[@]}; do + + test_to=${test_tos[0]} + for t in ${test_tos[@]:1}; do + test_to+=", $t" + done + case $test_from in + testignore@expertpathologyreview.com) + test_to=testignore@zroe.org + ;; + esac + cat >>/usr/local/bin/send-test-forward <