# Copyright (C) 2019 Ian Kelling
# SPDX-License-Identifier: AGPL-3.0-or-later
+
+# todo: setup an alert for bouncing test emails.
+
+# todo: bounces to my fsf mail can come from fsf@iankelling.org,
+# think about making bounces go from the original address.
+
+# todo: add a prometheus alert for dovecot.
+
# todo: handle errors like this:
# Mar 02 12:44:26 kw systemd[1]: exim4.service: Found left-over process 68210 (exim4) in control group while starting unit. Ignoring.
# Mar 02 12:44:26 kw systemd[1]: This usually indicates unclean termination of a previous run, or service implementation deficiencies.
+#eg: on eggs, on may 1st, ps grep for exim, 2 daemons running. 1 leftover from a month ago
+#Debian-+ 1954 1 0 36231 11560 4 Apr02 ? 00:40:25 /usr/sbin/exim4 -bd -q30m
+#Debian-+ 23058 1954 0 36821 10564 0 20:38 ? 00:00:00 /usr/sbin/exim4 -bd -q30m
# todo: harden dovecot. need to do some research. one way is for it to only listen on a wireguard vpn interface, so only clients that are on the vpn can access it.
# todo: consider hardening cups listening on 0.0.0.0
# todo: stop/disable local apache, and rpc.mountd, and kdeconnect when not in use.
-# todo: check that spamd and unbound only listen locally.
# todo: hosts should only allow external mail that is authed and
# destined for backup route. it is a minor issue since traffic is
# todo: run mailping test after running, or otherwise
# clear out terminal alert
-# todo: reinstall bk with bigger filesystem
-
-# todo: on bk, dont send email if mailvpn is not up
-
-# todo: mailtest-check should check on bk too
-
# todo: disable postgrey
# todo: in testforward-check, we should also look
local base="${dest##*/}"
local dir="${dest%/*}"
if [[ $dir != "$base" ]]; then
- mkdir -p ${dest%/*}
+ # dest has a directory component
+ mkdir -p "$dir"
fi
ir=false # i result
tmpdir=$(mktemp -d)
fi
# light version of exim does not have sasl auth support.
-pi-nostart exim4 exim4-daemon-heavy spamassassin openvpn unbound clamav-daemon wireguard
+# note: for bitfolk hosts, unbound has important config with conflink.
+pi-nostart exim4 exim4-daemon-heavy spamassassin unbound clamav-daemon wireguard
# note: pyzor debian readme says you need to run some initialization command
# but its outdated.
pi spf-tools-perl p0f postgrey pyzor razor jq moreutils certbot fail2ban
+case $HOSTNAME in
+ je) : ;;
+ # not included due to using wireguard: openvpn
+ *) pi wget git unzip iptables ;;
+esac
# bad packages that sometimes get automatically installed
pu openresolv resolvconf
i /etc/systemd/system/wg-quick@wgmail.service.d/override.conf <<EOF
[Unit]
Requires=mailnn.service
-After=network.target mailnn.service
JoinsNamespaceOf=mailnn.service
BindsTo=mailnn.service
StartLimitIntervalSec=0
for unit in ${nn_progs[@]}; do
i /etc/systemd/system/$unit.service.d/nn.conf <<EOF
[Unit]
-# commented for old openvpn
-Requires=$vpnser
+
+# Wants appears better than requires because with requires,
+# if the vpnser fails to start, this service won't get run at
+# all, even if the vpnser starts on an automatic restart.
+
+Wants=$vpnser
After=network.target mailnn.service $vpnser
JoinsNamespaceOf=mailnn.service
BindsTo=mailnn.service
# NOTE: Pulled options changed on restart, will need to close and reopen TUN/TAP device.
m /a/exe/vpn-mk-client-cert -b mailclient -n mail li.iankelling.org
fi
-case $HOSTNAME in
- bk)
- if [[ ! -e /etc/openvpn/client/mail.conf ]]; then
- echo "$0: error: first, on a system with /p/c/filesystem, run mail-setup, or the vpn-mk-client-cert line above this err" 2>&2
- exit 1
- fi
- ;;
-esac
+
+# With openvpn, I didn't get around to persisting the openvpn
+# cert/configs into /p/c/machine_specific/bk, so I had this case to
+# manually get the cert. However, we aren't using openvpn anymore, so it
+# is commented out.
+#
+# case $HOSTNAME in
+# bk)
+# if [[ ! -e /etc/openvpn/client/mail.conf ]]; then
+# echo "$0: error: first, on a system with /p/c/filesystem, run mail-setup, or the vpn-mk-client-cert line above this err" 2>&2
+# exit 1
+# fi
+# ;;
+# esac
m rsync -aiSAX --chown=root:root --chmod=g-s /a/bin/ds/mail-cert-cron /usr/local/bin
port = 25,587
filter = exim
banaction = iptables-exim
+
+# 209.51.188.13 = mail.fsf.org
+# 2001:470:142::13 = mail.fsf.org
+# 209.51.188.92 = eggs.gnu.org
+# 2001:470:142:3::10 = eggs.gnu.org
+# 72.14.176.105 2600:3c00:e000:280::2 = mail.iankelling.org
+# 10.173.8.1 = non-nn net
+ignoreip = 209.51.188.13 2001:470:142::13 209.51.188.92 2001:470:142:3::10 72.14.176.105 2600:3c00:e000:280::2 10.173.8.1
EOF
if $ir; then
m systemctl restart fail2ban
cat >/etc/exim4/conf.d/retry/17_retry <<'EOF'
# Retry fast for my own domains
-iankelling.org * F,1d,10m;F,14d,1h
-amnimal.ninja * F,1d,10m;F,14d,1h
-expertpathologyreview.com * F,1d,10m;F,14d,1h
-je.b8.nz * F,1d,10m;F,14d,1h
-zroe.org * F,1d,10m;F,14d,1h
+iankelling.org * F,1d,4m;F,14d,1h
+amnimal.ninja * F,1d,4m;F,14d,1h
+expertpathologyreview.com * F,1d,4m;F,14d,1h
+je.b8.nz * F,1d,4m;F,14d,1h
+zroe.org * F,1d,4m;F,14d,1h
eximbackup.b8.nz * F,1d,4m;F,14d,1h
# The spec says the target domain will be used for temporary host errors,
mx.iankelling.org * F,1d,4m;F,14d,1h
bk.b8.nz * F,1d,4m;F,14d,1h
eggs.gnu.org * F,1d,4m;F,14d,1h
+fencepost.gnu.org * F,1d,4m;F,14d,1h
+
+# afaik our retry doesnt need this, but just using everything
+mx.amnimal.ninja * F,1d,4m;F,14d,1h
+mx.expertpathologyreview.com * F,1d,4m;F,14d,1h
+
+
mail.fsf.org * F,1d,15m;F,14d,1h
EOF
domainlist local_hostnames = ! je.b8.nz : ! bk.b8.nz : *.b8.nz : b8.nz
-hostlist iank_trusted = <; \\
+hostlist iank_trusted = <; \
# veth0
-10.173.8.1 ; \\
+10.173.8.1 ; \
# li li_ip6
-72.14.176.105 ; 2600:3c00::f03c:91ff:fe6d:baf8 ; \\
+72.14.176.105 ; 2600:3c00::f03c:91ff:fe6d:baf8 ; \
# li_vpn_net li_vpn_net_ip6s
-10.8.0.0/24; 2600:3c00:e000:280::/64 ; 2600:3c00:e002:3800::/56 ; \\
+10.8.0.0/24; 2600:3c00:e000:280::/64 ; 2600:3c00:e002:3800::/56 ; \
# bk bk_ip6
-85.119.83.50 ; 2001:ba8:1f1:f0c9::2 ; \\
+85.119.83.50 ; 2001:ba8:1f1:f0c9::2 ; \
# je je_ipv6
-85.119.82.128 ; 2001:ba8:1f1:f09d::2 ; \\
+85.119.82.128 ; 2001:ba8:1f1:f09d::2 ; \
# fsf_mit_net fsf_mit_net_ip6 fsf_net fsf_net_ip6 fsf_office_net
18.4.89.0/24 ; 2603:3005:71a:2e00::/64 ; 209.51.188.0/24 ; 2001:470:142::/48 ; 74.94.156.208/28
{ match{$h_auto-submitted:}{(?i)auto-generated|auto-replied} }\
{ match_domain{$domain}{+local_domains} }\
} {no}{yes}}
-
-
EOF
rm -fv /etc/exim4/rcpt_local_acl # old path
EOF
rm -fv /etc/exim4/data_local_acl # old path
+
i /etc/exim4/conf.d/data_local_acl <<'EOF'
# Except for the "condition =", this was
# a comment in the check_data acl. The comment about this not
warn
!hosts = +iank_trusted
+ # Smarthosts connect with residential ips and thus get flagged as spam if we do a spam check.
+ !authenticated = plain_server:login_server
condition = ${if < {$message_size}{5000K}}
spam = Debian-exim:true
add_header = X-Spam_score_int: $spam_score_int
add_header = X-Spam_action: $spam_action
warn
+ !hosts = +iank_trusted
+ !authenticated = plain_server:login_server
condition = ${if def:malware_name}
remove_header = Subject:
add_header = Subject: [Clamav warning: $malware_name] $h_subject
i /etc/systemd/system/radicale.service.d/override.conf <<EOF
[Unit]
-# this unit is configured to start and stop whenever
-# $vpnser does
After=network.target network-online.target mailnn.service $vpnser
-BindsTo=$vpnser
-Wants=network-online.target
+Wants=$vpnser
JoinsNamespaceOf=mailnn.service
StartLimitIntervalSec=0
-----END DH PARAMETERS-----
EOF
{
+
if [[ $HOSTNAME == "$MAIL_HOST" ]]; then
cat <<'EOF'
ssl_cert = </etc/exim4/fullchain.pem
ssl_key = </etc/exim4/privkey.pem
EOF
else
+ # We have a lets encrypt hooks that puts things here.
+ # This is just for bk, which uses the vpn cert in exim
+ # for sending mail, but the local hostname cert for
+ # dovecot.
cat <<'EOF'
ssl_cert = </etc/exim4/exim.crt
ssl_key = </etc/exim4/exim.key
EOF
fi
+
cat <<'EOF'
# https://ssl-config.mozilla.org
ssl = required
i /etc/dovecot/dovecot-sql.conf.ext <<'EOF'
# from mailinabox
driver = sqlite
+# for je and bk, populated the testignore users for the relevant domains
connect = /m/rc/users.sqlite
default_pass_scheme = SHA512-CRYPT
password_query = SELECT email as user, password FROM users WHERE email='%u';
privileges TEXT NOT NULL DEFAULT '');
EOF
fi
+ # users.sqlite is saved into /p/c/machine_specific, so update it there!.
+ #
# example of adding a user:
# hash: doveadm pw -s SHA512-CRYPT -p passhere
# sqlite3 /m/rc/users.sqlite <<'EOF'
#### begin dl roundcube
# note, im r2e subbed to https://github.com/roundcube/roundcubemail/releases.atom
v=1.4.13; f=roundcubemail-$v-complete.tar.gz
- cd /a/opt
+ cd /root
if [[ -e $f ]]; then
timestamp=$(stat -c %Y $f)
else
fwrite(STDOUT, ";\n");
EOF
m php tmp.php >config.php
- m rm tmp.php
+ m rm -f tmp.php
m sudo -u www-data php $ncdir/occ maintenance:update:htaccess
list=$(sudo -u www-data php $ncdir/occ --output=json_pretty app:list)
# user_external not compaible with nc 23
EOF
fi
EOFOUTER
+ chmod +x /usr/local/bin/ncup
mkdir -p /var/www/cron-errors
chown www-data.www-data /var/www/cron-errors
i /etc/cron.d/$ncbase <<EOF
-PATH=/sbin:/usr/sbin:/usr/bin:/bin:/usr/local/bin
+PATH=/usr/sbin:/sbin:/usr/bin:/bin:/usr/local/bin
SHELL=/bin/bash
# https://docs.nextcloud.com/server/20/admin_manual/configuration_server/background_jobs_configuration.html
*/5 * * * * www-data php -f $ncdir/cron.php --define apc.enable_cli=1 |& log-once nccron
QUEUEINTERVAL='30m'
COMMONOPTIONS='-C /etc/exim4/my.conf'
UPEX4OPTS='-o /etc/exim4/my.conf'
-#E4BCD_PANICLOG_NOISE='exim user lost privilege for using -C option'
+# i use epanic-clean for alerting if there are bad paniclog entries
+E4BCD_WATCH_PANICLOG='no'
EOF
chown Debian-exim:Debian-exim /usr/sbin/exim4
# needs guid set in order to become Debian-exim
;;
*)
dirs=()
- for d in /d /m /media /mnt /nocow /o /p /q; do
+ for d in /a /d /m /media /mnt /nocow /o /p /q; do
if [[ -d $d ]]; then
dirs+=($d)
fi
EOF
echo|i /etc/exim4/conf.d/router/880_universal_forward
+
+ cat >>/etc/exim4/conf.d/main/000_local <<EOF
+MAILDIR_HOME_MAILDIR_LOCATION = /m/md/Sent
+EOF
+
# for iank@fsf.org, i have mail.fsf.org forward it to fsf@iankelling.org.
# and also have mail.iankelling.org whitelisted as a relay domain.
# I could avoid that if I changed this to submit to 587 with a
# password like a standard mua.
i /etc/exim4/conf.d/router/188_exim4-config_smarthost <<'EOF'
+# ian: save a copy of sent mail. i thought of other ways to
+# do this, for example, to only save sent mail that is not sent
+# from my mail client which saves a copy by default, but in the
+# end, it seems simplest to turn that off. We want to save
+# external mail sent by smarthosts.
+sentarchive:
+ driver = redirect
+ domains = ! +local_domains
+ condition = ${if !bool{${lookup{$local_part@$domain}lsearch{/etc/exim4/ignore-sent}{true}}}}
+ data = vojdedIdNejyebni@b8.nz
+ unseen
+
# ian: copied from /etc/exim4/conf.d/router/200_exim4-config_primary, and added senders = and
# replaced DCsmarthost with hostname
fsfsmarthost:
host_find_failed = ignore
same_domain_copy_routing = yes
no_more
-
EOF
# Greping /etc/exim4, unqualified mails this would end up as
$MAIL_HOST|bk)
# config for the non-nn exim
m rsync -ra --delete /etc/exim4/ /etc/myexim4
+ cat >>/etc/myexim4/conf.d/main/000_local-nn <<'EOF'
+# this makes it easier to see which exim is doing what
+log_file_path = /var/log/exim4/my%s
+EOF
+
+ cat >/etc/logrotate.d/myexim <<'EOF'
+/var/log/exim4/mymain /var/log/exim4/myreject {
+ daily
+ missingok
+ rotate 1000
+ delaycompress
+ notifempty
+ nocreate
+}
+/var/log/exim4/mypanic {
+ size 10M
+ missingok
+ rotate 10
+ compress
+ delaycompress
+ notifempty
+ nocreate
+}
+EOF
+
# If we ever wanted to have a separate spool,
# we could do it like this.
# cat >>/etc/exim4/conf.d/main/000_local-nn <<'EOF'
# without local-fs on exim, we get these kind of errors in paniclog on shutdown:
# Failed to create spool file /var/spool/exim4//input//1jCLxz-0008V4-V9-D: Permission denied
After=local-fs.target
+
+[Service]
+ExecStartPre=/usr/local/bin/exim-nn-iptables
EOF
if ! mountpoint -q $sdir; then
stopifactive exim4 exim4in
m systemctl daemon-reload
fi
+sysd-prom-fail-install epanicclean
m systemctl --now enable epanicclean
case $HOSTNAME in
sre mailcert.timer
case $HOSTNAME in
- bk)
- # todo, this should be done in distro-begin
- soff systemd-resolved
- ln -sf 127.0.0.1-resolv/stub-resolv.conf /etc/resolv.conf
- ;;&
$MAIL_HOST|bk)
m systemctl --now enable mailnn mailnnroute
;;&
case $HOSTNAME in
- $MAIL_HOST|bk|je) : ;;
+ $MAIL_HOST|bk|je|li)
+ # on li, these are never started, except $vpnser
+ :
+ ;;
*)
soff radicale mailclean.timer dovecot spamassassin $vpnser mailnn clamav-daemon
;;
# note: cronjob "ian" also does some important monitoring
# todo: this will sometimes cause an alert because mailtest-check will run
# before we have setup network namespace and spamassassin
- cat >/etc/cron.d/mailtest <<EOF
+ i /etc/cron.d/mailtest <<EOF
SHELL=/bin/bash
PATH=/usr/bin:/bin:/usr/local/bin
-MAILTO=daylerts@iankelling.org
+MAILTO=daylert@iankelling.org
*/5 * * * * $u send-test-forward |& log-once send-test-forward
*/10 * * * * root chmod -R g+rw /m/md/bounces |& log-once -1 bounces-chmod
-*/5 * * * * root timeout 290 mailtest-check slow |& log-once -4 mailtest-check
# if a bounce happened yesterday, dont let it slip through the cracks
-8 1 * * * root export MAILTO=alerts@iankelling.org; awk '\$5 == "**"' /var/log/exim4/mainlog.1
+8 1 * * * root export MAILTO=alerts@iankelling.org; [[ -s /var/log/exim4/mainlog.1 ]] && awk '\$5 == "**"' /var/log/exim4/mainlog.1
EOF
+
+
m sudo rsync -ahhi --chown=root:root --chmod=0755 \
/b/ds/mailtest-check /b/ds/check-remote-mailqs /usr/local/bin/
+ i /etc/systemd/system/mailtest-check.service <<'EOF'
+[Unit]
+Description=mailtest-check
+After=local-fs.target
+StartLimitIntervalSec=0
+
+[Service]
+Type=simple
+ExecStart=/usr/local/bin/mailtest-check slow
+Restart=always
+RestartSec=60
+
+[Install]
+WantedBy=graphical.target
+EOF
+ sysd-prom-fail-install mailtest-check
+ sre mailtest-check
;;&
$MAIL_HOST)
test_froms=(ian@iankelling.org z@zroe.org iank@gnu.org)
- test_to="testignore@expertpathologyreview.com, testignore@je.b8.nz, testignore@amnimal.ninja, jtuttle@gnu.org"
+ test_tos=(testignore@expertpathologyreview.com testignore@je.b8.nz testignore@amnimal.ninja jtuttle@gnu.org)
cat >>/etc/cron.d/mailtest <<EOF
0 13 * * * root echo "1pm alert. You are not in the matrix."
EOF
;;&
bk)
- test_froms=(testignore@expertpathologyreview.com testignore@amnimal.ninja)
- test_to="testignore@iankelling.org, testignore@zroe.org, testignore@je.b8.nz"
+ test_froms=(testignore@amnimal.ninja testignore@expertpathologyreview.com)
+ test_tos=(testignore@iankelling.org testignore@je.b8.nz)
+ # We dont need to send from different addresses to the same
+ # address. this breaks down our nice elegant logic of building up
+ # froms and tos , so I just handle expertpath in a special case
+ # below and set the to: to be testignore@zroe.org. If we did sent
+ # that way, it would also mess up our mailtest-check logic that
+ # finds which messages to check.
+ # for example: from testignore@amnimal.ninja to: testignore@iankelling.org testignore@zroe.org
+ # that would become 2 messages and we'd only check 1.
;;&
je)
test_froms=(testignore@je.b8.nz)
- test_to="testignore@iankelling.org, testignore@zroe.org, testignore@expertpathologyreview.com, testignore@amnimal.ninja"
+ test_tos=(testignore@iankelling.org testignore@zroe.org testignore@expertpathologyreview.com testignore@amnimal.ninja)
;;&
$MAIL_HOST|bk|je)
+
+ # Dont put these test messages into the sent folder or else it will
+ # overwhelm it, plus i dont want to save a copy at all.
+ rm -f /etc/exim4/ignore-sent
+ for t in ${test_tos[@]}; do
+ echo $t >> /etc/exim4/ignore-sent
+ done
+
cat >/usr/local/bin/send-test-forward <<'EOF'
#!/bin/bash
+# we remove from the queue older than 4.3 minutes since we send every 5 minutes.
olds=(
-$(/sbin/exiqgrep -o 260 -i -r '^(testignore@(iankelling\.org|zroe\.org|expertpathologyreview\.com|amnimal\.ninja|je\.b8\.nz)|jtuttle@gnu\.org)$')
+$(/usr/sbin/exiqgrep -o 260 -i -r '^(testignore@(iankelling\.org|zroe\.org|expertpathologyreview\.com|amnimal\.ninja|je\.b8\.nz)|jtuttle@gnu\.org)$')
)
if (( ${#olds[@]} )); then
- /sbin/exim -Mrm "${olds[@]}" >/dev/null
+ /usr/sbin/exim -Mrm "${olds[@]}" >/dev/null
fi
EOF
for test_from in ${test_froms[@]}; do
+
+ test_to=${test_tos[0]}
+ for t in ${test_tos[@]:1}; do
+ test_to+=", $t"
+ done
+ case $test_from in
+ testignore@expertpathologyreview.com)
+ test_to=testignore@zroe.org
+ ;;
+ esac
+
cat >>/usr/local/bin/send-test-forward <<EOFOUTER
/usr/sbin/exim -f $test_from -t <<EOF
From: $test_from
To: $test_to
-Subject: test \$(date +%Y-%m-%dT%H:%M:%S%z) \$(date +%s)
+Subject: test \$(date +%Y-%m-%dT%H:%M:%S%z) \$EPOCHSECONDS
/usr/local/bin/send-test-forward
EOF
m chmod +x /usr/local/bin/send-test-forward
;;
*)
- rm -fv /etc/cron.d/mailtest
+ soff mailtest-check.service
+ rm -fv /etc/cron.d/mailtest \
+ /var/lib/prometheus/node-exporter/mailtest-check.prom* \
+ /var/local/cron-errors/check-remote-mailqs*
;;
esac