#!/bin/bash
# * intro
# Program to install and configure Ian's email related programs
# Copyright (C) 2024 Ian Kelling
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
# SPDX-License-Identifier: GPL-3.0-or-later
# todo:
# on bk (and fsf servers that run multiple exim4 daemons, eg eximfsf2 and eximfsf3),
# make it so that when exim is restarted due to package upgrades,
# we also restart those daemons, which can be done like so, based on looking
# at the prerm and postinst scripts of exim4-daemon-heavy.
#
# if [[ ! -e /usr/sbin/invoke-rc.d-diverted ]]; then
# mv /usr/sbin/invoke-rc.d /usr/sbin/invoke-rc.d-diverted
# dpkg --divert /usr/sbin/invoke-rc.d-diverted --no-rename /usr/sbin/invoke-rc.d
# fi
# /usr/sbin/invoke-rc.d:
# #!/bin/bash
# if [[ DPKG_MAINTSCRIPT_PACKAGE == exim4* && $1 == exim4 ]]; then
# shift
# ret=0
# for daemon in exim4 eximfsf2 eximfsf3; do
# /usr/sbin/invoke-rc.d-diverted $daemon "$@" || ret=$?
# done
# else
# /usr/sbin/invoke-rc.d-diverted "$@"
# fi
# Things I tend to forget. on MAIL_HOST, daemon runs with /etc/exim4/my.conf,
# due to /etc/default/exim4 containing:
# COMMONOPTIONS='-C /etc/exim4/my.conf'
# UPEX4OPTS='-o /etc/exim4/my.conf'
#
# The non-daemon config
# gets generated from this script calling update-exim4.conf -d /etc/myexim4
# which has log path
# log_file_path = /var/log/exim4/my%s
#
# On non bk|MAIL_HOST, the config and log file are all standard.
#
# eximbackup folder is /bu/md
# it is cleaned up by mail-backup-clean, which is run by btrbk-run
# shellcheck disable=SC2254 # makes for a lot of unneeded quotes
# perusing through /el/mainlog without test messages:
# &!testignore|jtuttle|
#
#&! testignore|jtuttle|eximbackup|/usr/sbin/exim4 -bpu
# todo: this message seems to get dropped on the floor, it was due to a missing 2nd colon in
# condition = ${if def:h_fdate:}
# Figure out how to avoid this message being discarded.
# 2023-09-12 01:41:43 [722371] 1qfw9f-0031v9-0S <= ian@iankelling.org U=iank P=local S=483 id=87cyyogd7t.fsf@iankelling.org T="iank2" from for testignore@amnimal.ninja
# 2023-09-12 01:41:43 [722373] 1qfw9f-0031v9-0S H=nn.b8.nz [10.173.8.2]: SMTP error from remote mail server after pipelined end of data: 451 Temporary local problem - please try later
# 2023-09-12 01:41:43 [722372] 1qfw9f-0031v9-0S == testignore@amnimal.ninja R=smarthost T=remote_smtp_smarthost defer (-46) H=nn.b8.nz [10.173.8.2] DT=0s: SMTP error from remote mail server after pipelined end of data: 451 Temporary local problem - please try later
# todo: check new macro DKIM_TIMESTAMPS
# todo: check if REMOTE_SMTP_INTERFACE or REMOTE_SMTP_TRANSPORTS_HEADERS_REMOVE can simplify my or fsfs config
# todo: max line length macro changed in t11. look into it
# todo: check that all macros we use are still valid in t11
# todo: setup an alert for bouncing test emails.
# todo: bounces to my fsf mail can come from fsf@iankelling.org,
# think about making bounces go from the original address.
# todo: add a prometheus alert for dovecot.
# todo: handle errors like this:
# Mar 02 12:44:26 kw systemd[1]: exim4.service: Found left-over process 68210 (exim4) in control group while starting unit. Ignoring.
# Mar 02 12:44:26 kw systemd[1]: This usually indicates unclean termination of a previous run, or service implementation deficiencies.
#eg: on eggs, on may 1st, ps grep for exim, 2 daemons running. 1 leftover from a month ago
#Debian-+ 1954 1 0 36231 11560 4 Apr02 ? 00:40:25 /usr/sbin/exim4 -bd -q30m
#Debian-+ 23058 1954 0 36821 10564 0 20:38 ? 00:00:00 /usr/sbin/exim4 -bd -q30m
# todo: harden dovecot. need to do some research. one way is for it to only listen on a wireguard vpn interface, so only clients that are on the vpn can access it.
# todo: consider hardening cups listening on 0.0.0.0
# todo: stop/disable local apache, and rpc.mountd, and kdeconnect when not in use.
# todo: hosts should only allow external mail that is authed and
# destined for backup route. it is a minor issue since traffic is
# limited to the wghole network.
# todo: emailing info@amnimal.ninja produces a bounce, user doesn't exist
# instead of a simple rejection like it should.
# todo: run mailping test after running, or otherwise
# clear out terminal alert
# todo: disable postgrey. (why did we have it?)
# todo: in testforward-check, we should also look
# todo: test that bounces dont help create valid mailtest-check
# todo: move mail stuff in distro-end into this file
# todo: consider rotating dkim & publishing key so every past email I sent
# isnt necessarily signed
# todo: consider how to get clamav out of Debian-exim group
# so it cant read/write the whole mail spool, for better
# security.
# todo: create a cronjob to update or warn on expiring dnssec keys
# todo: we should test failed mail daily or so
# failed cronjob, failed sysd-log-once,
# a local bounce from a cronjob, a local bounce
# to a bad remote address, perhaps a local failure
# when the sending daemon is down.
# And send an alert email if no alerts have been sent
# in 2 or 3 days or something. todo, test cron mail on li.
# todo: look at mailinabox extra dns records, note these changelogs:
# - An MTA-STS policy for incoming mail is now published (in DNS and over HTTPS) when the primary hostname and email address domain both have a signed TLS certificate installed, allowing senders to know that an encrypted connection should be enforced.
# - The per-IP connection limit to the IMAP server has been doubled to allow more devices to connect at once, especially with multiple users behind a NAT.
#
# todo: mailtest-check failure on remote hosts is not going to alert me.
# sort that out.
# todo: test mail failure as well as success.
#
# todo: validate that mailtest-check is doing dnsbl checks.
# background: I want to run exim in a network namespace so it can send
# and receive through a vpn. This is needed so it can do ipv6, because
# outside the namespace if we dont have ipv6, to send ipv6 through the
# vpn, we have to send all our ipv6 through the vpn. I did this for a
# long time, it was fine, but it causes various pains, like increased
# latency, increased recaptcha because my ip is from a data center, just
# various issues I dont want on all the time. The problem with the
# namespace is that all kinds of programs want to invoke exim, but they
# wont be in the namespace. I could replace exim with a wrapper that
# jumps into the namespace, i tried that, it works fine. One remaining
# problem was that I would have needed to hook into exim upgrades to
# move exim and replace it with my wrapper script. Also, my script to
# join the namespace is not super reliable because it uses a pgrep.
# Instead, I should have created a systemd service for a process that
# will never die and just writes its pid somewhere convenient.
# That implementation
# is below here:
#
# sudoers:
# user ALL=(ALL) /usr/sbin/exim4
#
# move exim4 to eximian, use this script for exim4:
#
# #!/bin/bash
# if ip a show veth1-mail &>/dev/null; then
# /usr/sbin/eximian "$@"
# exit
# fi
# dosudo=false
# if [[ $USER && $USER != root ]]; then
# dosudo=true
# fi
# pid=$(pgrep -f "/usr/sbin/openvpn .* --config /etc/openvpn/.*mail.conf")
# if $dosudo; then
# sudo nsenter -t $pid -n -m sudo -u $USER /usr/sbin/eximian "$@"
# else
# nsenter -t $pid -n -m /usr/sbin/eximian "$@"
# fi
# ## end script
#
# an alternate solution: there is a small setguid program for
# network namespaces in my bookmarks.
#
# However, the solution I went with is: have 2 exim
# configs. A nonstandard location for the daemon that runs
# in the namespace. For all other invocations, it uses
# the default config location, which is altered to be
# in a smarthost config which sends mail to the deaemon.
#
# I have a bash function, enn to invoke exim like the daemon is running.
# and mailbash to just enter its network namespace.
if [ -z "$BASH_VERSION" ]; then echo "error: shell is not bash" >&2; exit 1; fi
shopt -s nullglob
if [[ -s /usr/local/lib/bash-bear ]]; then
source /usr/local/lib/bash-bear
elif [[ -s /a/bin/bash-bear-trap/bash-bear ]]; then
source /a/bin/bash-bear-trap/bash-bear
else
echo "no err tracing script found"
exit 1
fi
source /a/bin/distro-functions/src/identify-distros
source /a/bin/distro-functions/src/package-manager-abstractions
# has nextcloud_admin_pass in it
f=/p/c/machine_specific/$HOSTNAME/mail
if [[ -e $f ]]; then
# shellcheck source=/p/c/machine_specific/bk/mail
source $f
fi
[[ $EUID == 0 ]] || exec sudo -E "${BASH_SOURCE[0]}" "$@"
# note, this is hardcoded in /etc/exim4/conf.d/main/000_local
u=$(id -nu 1000)
usage() {
cat < preferences -> preferences -> advanced tab -> config editor button -> security.ssl.enable_ocsp_must_staple = false
# background: dovecot does not yet have ocsp stapling support
# reference: https://community.letsencrypt.org/t/simple-guide-using-lets-encrypt-ssl-certs-with-dovecot/2921
#
# for phone, k9mail, fdroid, same thing but username alerts, pass in ivy-pass.
# also, bk.b8.nz for secondary alerts, username is iank. same alerts pass.
# fetching mail settings: folder poll frequency 10 minutes.
# account settings, fetching mail, push folders: All. Then disable the persistent notification.
#######
# * perstent password instructions Note: for cert cron, we need to
# manually run first to accept known_hosts
# # exim passwords:
# # for hosts which have all private files I just use the same user
# # for other hosts, each one get\'s their own password.
# # for generating secure pass, and storing for server too:
# f=$(mktemp)
# host=tp
# apg -m 50 -x 70 -n 1 -a 1 -M CLN >$f
# s sed -i "/^$host:/d" /p/c/filesystem/etc/exim4/passwd
# echo "$host:$(mkpasswd -m sha-512 -s <$f)" >>/p/c/filesystem/etc/exim4/passwd
# #reference: exim4_passwd_client(5)
# dir=/p/c/machine_specific/$host/filesystem/etc/exim4
# mkdir -p $dir
# echo "mail.iankelling.org:$host:$(<$f)" > $dir/passwd.client
# # then run this script
# # dovecot password, i just need 1 as I\'m the only user
# mkdir /p/c/filesystem/etc/dovecot
# echo "iank:$(doveadm pw -s SHA512-CRYPT)::::::" >>/p/c/filesystem/etc/dovecot/users
####### end perstent password instructions ######
# * dkim dns
# # Remove 1 level of comments in this section, set the domain var
# # for the domain you are setting up, then run this and copy dns settings
# # into dns.
# domain=iankelling.org
# c /p/c/filesystem/etc/exim4
# # this has several bugs addressed in comments, but it was helpful
# # https://debian-administration.org/article/718/DKIM-signing_outgoing_mail_with_exim4
# openssl genrsa -out $domain-private.pem 2048
# # Then, to get the public key strings to put in bind:
# # selector is needed for having multiple keys for one domain.
# # I dun do that, so just use a static one: li
# # Debadmin page does not have v=, fastmail does, and this
# # says it\'s recommended in 3.6.1, default is DKIM1 anyways.
# # https://www.ietf.org/rfc/rfc6376.txt
# # Join and print all but first and last line.
# # last line: swap hold & pattern, remove newlines, print.
# # lines 2+: append to hold space
# echo "bind txt record: remember to truncate $domain so its relative to the bind zone"
# cat <&2; exit 1; }
reload=false
# This file is so if we fail in the middle and rerun, we dont lose state
if [[ -e /var/local/mail-setup-reload ]]; then
reload=true
fi
# update file.
# if the file changed, ur=true, else false.
# note: duplicated in brc
u() {
local tmp tmpdir dest="$1"
local base="${dest##*/}"
local dir="${dest%/*}"
if [[ $dir != "$base" ]]; then
# dest has a directory component
mkdir -p "$dir"
fi
ur=false # u result
tmpdir=$(mktemp -d)
cat >$tmpdir/"$base"
tmp=$(rsync -ic $tmpdir/"$base" "$dest")
if [[ $tmp ]]; then
printf "%s\n" "$tmp"
ur=true
if [[ $dest == /etc/systemd/system/* ]]; then
touch /var/local/mail-setup-reload
reload=true
fi
fi
rm -rf $tmpdir
}
setini() {
key="$1" value="$2" section="$3"
file="/etc/radicale/config"
sed -ri "/ *\[$section\]/,/^ *\[[^]]+\]/{/^\s*${key}[[:space:]=]/d};/ *\[$section\]/a $key = $value" "$file"
}
soff () {
for service; do
# ignore services that dont exist
if systemctl cat $service &>/dev/null; then
m systemctl disable --now $service
fi
done
}
sre() {
local enabled
for service; do
m systemctl restart $service
# Optimization for exim,
# is-enabled: 0m0.015s
# enable: 0m0.748s
# It is related to this message:
# exim4.service is not a native service, redirecting to systemd-sysv-install.
# Executing: /lib/systemd/systemd-sysv-install enable exim4
enabled=$(systemctl is-enabled $service 2>/dev/null ||:)
if [[ $enabled != enabled ]]; then
m systemctl enable $service
fi
done
}
mailhost() {
[[ $HOSTNAME == "$MAIL_HOST" ]]
}
reifactive() {
for service; do
if systemctl is-active $service >/dev/null; then
m systemctl restart $service
fi
done
}
stopifactive() {
for service; do
if systemctl is-active $service >/dev/null; then
m systemctl stop $service
fi
done
}
mxhost=mx.iankelling.org
mxport=587
# old setup. left as comment for example
# mxhost=mail.messagingengine.com
# mxport=587
# forward=ian@iankelling.org
smarthost="$mxhost::$mxport"
uhome=$(eval echo ~$u)
# Somehow on one machine, a file got written with 664 perms.
# just being defensive here.
umask 0022
source /a/bin/bash_unpublished/source-state
if [[ ! $MAIL_HOST ]]; then
err "\$MAIL_HOST not set"
fi
bhost_t=false
case $HOSTNAME in
$MAIL_HOST) : ;;
kd|x2|x3|kw|sy|bo|so)
bhost_t=true
;;
esac
# * Install universal packages
# installs epanicclean iptables-exim ip6tables-exim
/a/bin/ds/install-my-scripts
if [[ $(debian-codename-compat) == bionic ]]; then
cat >/etc/apt/preferences.d/spamassassin <<'EOF'
Package: spamassassin sa-compile spamc
Pin: release n=focal,o=Ubuntu
Pin-Priority: 500
EOF
fi
# name change in t12, and now timer instead of cron option in /etc/default
first_spamd_run=false
if ! systemctl cat spamassassin-maintenance.timer &>/dev/null; then
first_spamd_run=true
fi
# light version of exim does not have sasl auth support.
# note: for bitfolk hosts, unbound has important config with conflink.
pi-nostart exim4 exim4-daemon-heavy spamassassin unbound clamav-daemon wireguard
spamd_ser=spamd
if systemctl cat spamassassin &>/dev/null; then
spamd_ser=spamassassin
elif $first_spamd_run; then
systemctl start spamassassin-maintenance
fi
systemctl enable --now spamassassin-maintenance.timer
# note: pyzor debian readme says you need to run some initialization command
# but its outdated.
pi spf-tools-perl p0f pyzor razor jq moreutils certbot fail2ban
pu postgrey
case $HOSTNAME in
je) : ;;
# not included due to using wireguard: openvpn
*) pi wget git unzip iptables ;;
esac
# bad packages that sometimes get automatically installed
pu openresolv resolvconf
soff openvpn
if [[ $(debian-codename) == etiona ]]; then
# ip6tables stopped loading on boot. openvpn has reduced capability set,
# so running iptables as part of openvpn startup wont work. This should do it.
pi iptables-persistent
cat >/etc/iptables/rules.v6 <<'EOF'
*mangle
COMMIT
*nat
COMMIT
EOF
# load it now.
m ip6tables -S >/dev/null
fi
# our nostart pi fails to avoid enabling
# * initial dns config & daemon setup
#
# use systemd-resolved for glibc resolutions, setup symlinks
pi libnss-resolve
# if this link gets replaced with a normal file we will get exim log
# errors on MAIL_HOST like so:
#
# R=fsfsmarthost defer (-36) DT=0s: host lookup for mail.fsf.org did not complete (DNS timeout?)
if [[ ! -L /etc/nsswitch.conf ]]; then
sudo mkdir -p /etc/resolved-nsswitch
sudo mv /etc/nsswitch.conf /etc/resolved-nsswitch
sudo ln -sf /etc/resolved-nsswitch/nsswitch.conf /etc
fi
f=/etc/basic-nsswitch/nsswitch.conf
if [[ ! -e $f ]]; then
sudo mkdir -p ${f%/*}
sudo cp /etc/nsswitch.conf $f
sudo sed -i --follow-symlinks 's/^ *hosts:.*/hosts: files dns myhostname/' $f
fi
case $HOSTNAME in
bk|je)
# je should be able to get along systemd-resolved, but ive had some odd
# very intermittent dns failures with spamassassin, it seems it might only
# be happening with systemd-resolved, so just use unbound
# to make it consistent with the other hosts.
sudo sed -i --follow-symlinks 's/^ *hosts:.*/hosts: files dns myhostname/' /etc/nsswitch.conf
soff systemd-resolved
sudo ln -sf 127.0.0.1-resolv/stub-resolv.conf /etc/resolv.conf
sgo unbound
# cautious measure to make sure resolution is working
sleep 1
;;
*)
# default is
# files mdns4_minimal [NOTFOUND=return] dns myhostname
# mdns4 is needed for my printer and for bbb webrtc, not sure exactly why.
# https://www.freedesktop.org/software/systemd/man/nss-resolve.html#
# seems more important than some potential use case.
# Interestingly, t9/t10 man page says use files before resolve, debian 10 says the opposite.
# removing files makes hostname -f not actually give the fully qualified domain name.
sudo sed -i --follow-symlinks 's/^ *hosts:.*/hosts: files resolve [!UNAVAIL=return] mdns4_minimal [NOTFOUND=return] myhostname/' /etc/resolved-nsswitch/nsswitch.conf
;;
esac
case $HOSTNAME in
bk)
sgo named
;;
esac
u /etc/apparmor.d/abstractions/nameservice.d/iank <<'EOF'
/etc/resolved-nsswitch/nsswitch.conf r,
/etc/basic-nsswitch/nsswitch.conf r,
# Aug 06 23:09:11 kd audit[3995]: AVC apparmor="DENIED" operation="connect" profile="/usr/bin/freshclam" name="/run/systemd/resolve/io.systemd.Resolve" pid=3995 comm="freshclam" requested_mask="wr" denied_mask="wr" fsuid=109 ouid=101
# I dont know if this is quite the right fix, but I saw other sockets
# in the nameservice files that were rw, so figured it was ok to add this and it worked.
/run/systemd/resolve/io.systemd.Resolve rw,
EOF
if $ur && systemctl is-active apparmor; then
m systemctl reload apparmor
fi
# * Mail clean cronjob
u /etc/systemd/system/mailclean.timer <<'EOF'
[Unit]
Description=Run mailclean daily
[Timer]
OnCalendar=monthly
[Install]
WantedBy=timers.target
EOF
u /etc/systemd/system/mailclean.service </dev/null; then
m systemctl reload apparmor
fi
fi
# note: anything added to nn_progs needs corresponding rm
# down below in the host switch
nn_progs=(exim4)
if mailhost; then
# Note dovecots lmtp doesnt need to be in the same nn to accept delivery.
# Its in the nn so remote clients can connect to it.
nn_progs+=($spamd_ser dovecot)
fi
case $HOSTNAME in
$MAIL_HOST)
# todo, should this be after vpn service
u /etc/systemd/system/unbound.service.d/nn.conf < /sys/kernel/debug/dynamic_debug/control
# dmesg -w
;;&
$MAIL_HOST|bk)
for unit in ${nn_progs[@]}; do
u /etc/systemd/system/$unit.service.d/nn.conf <&2
# exit 1
# fi
# ;;
# esac
m rsync -aiSAX --chown=root:root --chmod=g-s /a/bin/ds/mail-cert-cron /usr/local/bin
u /etc/systemd/system/mailcert.service <<'EOF'
[Unit]
Description=Mail cert rsync
After=multi-user.target
[Service]
Type=oneshot
ExecStart=/usr/local/bin/sysd-mail-once mailcert /usr/local/bin/mail-cert-cron
EOF
u /etc/systemd/system/mailcert.timer <<'EOF'
[Unit]
Description=Run mail-cert once a day
[Timer]
OnCalendar=daily
[Install]
WantedBy=timers.target
EOF
wghost=${HOSTNAME}wg.b8.nz
if $bhost_t && [[ ! -e /etc/exim4/certs/$wghost/privkey.pem ]]; then
certbot -n --manual-public-ip-logging-ok --eff-email --agree-tos -m letsencrypt@iankelling.org \
certonly --manual --preferred-challenges=dns \
--manual-auth-hook /a/bin/ds/le-dns-challenge \
--manual-cleanup-hook /a/bin/ds/le-dns-challenge-cleanup \
--deploy-hook /a/bin/ds/le-exim-deploy -d $wghost
fi
# * fail2ban
# todo: test that these configs actually work, eg run
# s iptables-exim -S
# and see someone is banned.
sed 's/^ *before *= *iptables-common.conf/before = iptables-common-exim.conf/' \
/etc/fail2ban/action.d/iptables-multiport.conf| u /etc/fail2ban/action.d/iptables-exim.conf
u /etc/fail2ban/action.d/iptables-common-exim.conf <<'EOF'
# iank: same as iptables-common, except iptables is iptables-exim, ip6tables is ip6tables-exim
# Fail2Ban configuration file
#
# Author: Daniel Black
#
# This is a included configuration file and includes the definitions for the iptables
# used in all iptables based actions by default.
#
# The user can override the defaults in iptables-common.local
#
# Modified: Alexander Koeppe , Serg G. Brester
# made config file IPv6 capable (see new section Init?family=inet6)
[INCLUDES]
after = iptables-blocktype.local
iptables-common.local
# iptables-blocktype.local is obsolete
[Definition]
# Option: actionflush
# Notes.: command executed once to flush IPS, by shutdown (resp. by stop of the jail or this action)
# Values: CMD
#
actionflush = -F f2b-
[Init]
# Option: chain
# Notes specifies the iptables chain to which the Fail2Ban rules should be
# added
# Values: STRING Default: INPUT
chain = INPUT
# Default name of the chain
#
name = default
# Option: port
# Notes.: specifies port to monitor
# Values: [ NUM | STRING ] Default:
#
port = ssh
# Option: protocol
# Notes.: internally used by config reader for interpolations.
# Values: [ tcp | udp | icmp | all ] Default: tcp
#
protocol = tcp
# Option: blocktype
# Note: This is what the action does with rules. This can be any jump target
# as per the iptables man page (section 8). Common values are DROP
# REJECT, REJECT --reject-with icmp-port-unreachable
# Values: STRING
blocktype = REJECT --reject-with icmp-port-unreachable
# Option: returntype
# Note: This is the default rule on "actionstart". This should be RETURN
# in all (blocking) actions, except REJECT in allowing actions.
# Values: STRING
returntype = RETURN
# Option: lockingopt
# Notes.: Option was introduced to iptables to prevent multiple instances from
# running concurrently and causing irratic behavior. -w was introduced
# in iptables 1.4.20, so might be absent on older systems
# See https://github.com/fail2ban/fail2ban/issues/1122
# Values: STRING
lockingopt = -w
# Option: iptables
# Notes.: Actual command to be executed, including common to all calls options
# Values: STRING
iptables = /usr/local/bin/iptables-exim
[Init?family=inet6]
# Option: blocktype (ipv6)
# Note: This is what the action does with rules. This can be any jump target
# as per the iptables man page (section 8). Common values are DROP
# REJECT, REJECT --reject-with icmp6-port-unreachable
# Values: STRING
blocktype = REJECT --reject-with icmp6-port-unreachable
# Option: iptables (ipv6)
# Notes.: Actual command to be executed, including common to all calls options
# Values: STRING
iptables = /usr/local/bin/ip6tables-exim
EOF
u /etc/fail2ban/jail.d/exim.local <<'EOF'
[exim]
enabled = true
port = 25,587
filter = exim
banaction = iptables-exim
# 209.51.188.13 = mail.fsf.org
# 2001:470:142::13 = mail.fsf.org
# 209.51.188.92 = eggs.gnu.org
# 2001:470:142:3::10 = eggs.gnu.org
# 72.14.176.105 2600:3c00:e000:280::2 = mail.iankelling.org
# 10.173.8.1 = non-nn net
ignoreip = 209.51.188.13 2001:470:142::13 209.51.188.92 2001:470:142:3::10 72.14.176.105 2600:3c00:e000:280::2 10.173.8.1
EOF
if $ur; then
# Ensure the log file monitored by fail2ban exists, or else fail2ban can't start.
if [[ ! -e /var/log/exim4/mainlog ]]; then
install -m 640 -o Debian-exim -g adm /dev/null /var/log/exim4/mainlog
fi
m systemctl restart fail2ban
fi
# * common exim4 config
## old, not using forward files anymore
rm -fv $uhome/.forward /root/.forward
# Make all system users be aliases. preventative
# prevents things like cron mail for user without alias
awk 'BEGIN { FS = ":" } ; $6 !~ /^\/home/ || $7 ~ /\/nologin$/ { print $1 }' /etc/passwd| while read -r user; do
if [[ ! $user ]]; then
continue
fi
if ! grep -q "^$user:" /etc/aliases; then
echo "$user: root" |m tee -a /etc/aliases
fi
done
awk 'BEGIN { FS = ":" } ; $6 ~ /^\/home/ && $7 !~ /\/nologin$/ { print $1 }' /etc/passwd| while read -r user; do
case $HOSTNAME in
$MAIL_HOST)
sed -i "/^user:/d" /etc/aliases
;;
*)
if ! grep -q "^$user:" /etc/aliases; then
echo "$user: root" |m tee -a /etc/aliases
fi
;;
esac
done
. /a/bin/bash_unpublished/priv-mail-setup
m gpasswd -a iank adm #needed for reading logs
### make local bounces go to normal maildir
# local mail that bounces goes to /Maildir or /root/Maildir
dirs=(/m/md/bounces/{cur,tmp,new})
m mkdir -p ${dirs[@]}
m chown iank:iank /m /m/md
m ln -sfT /m/md /m/iank
m chmod 771 /m /m/md
m chown -R $u:Debian-exim /m/md/bounces
m chmod 775 ${dirs[@]}
m usermod -a -G Debian-exim $u
for d in /Maildir /root/Maildir; do
if [[ ! -L $d ]]; then
m rm -rf $d
fi
m ln -sf -T /m/md/bounces $d
done
# dkim, client passwd file
files=(/p/c/machine_specific/$HOSTNAME/filesystem/etc/exim4/*)
f=/p/c/filesystem/etc/exim4/passwd.client
if [[ -e $f ]]; then
files+=($f)
fi
if (( ${#files[@]} )); then
m rsync -ahhi --chown=root:Debian-exim --chmod=0640 \
${files[@]} /etc/exim4
fi
# By default, only 10 days of logs are kept. increase that.
# And dont compress, I look back at logs too often and
# dont need the annoyance of decompressing them all the time.
m sed -ri '/^\s*compress\s*$/d;s/^(\s*rotate\s).*/\11000/' /etc/logrotate.d/exim4-base
files=(/var/log/exim4/*.gz)
if (( ${#files[@]} )); then
gunzip ${files[@]}
fi
## disabled. not using .forward files, but this is still interesting
## for reference.
# ## https://blog.dhampir.no/content/make-exim4-on-debian-respect-forward-and-etcaliases-when-using-a-smarthost
# # i only need .forwards, so just doing that one.
# cd /etc/exim4/conf.d/router
# b=userforward_higher_priority
# # replace the router name so it is unique
# sed -r s/^\\S+:/$b:/ 600_exim4-config_userforward >175_$b
rm -fv /etc/exim4/conf.d/router/175_userforward_higher_priority
# todo, consider 'separate' in etc/exim4.conf, could it help on busy systems?
# alerts is basically the postmaster address
m sed -i --follow-symlinks -f - /etc/aliases </etc/exim4/conf.d/rewrite/34_iank_rewriting <<'EOF'
ncsoft@zroe.org graceq2323@gmail.com hE
EOF
# old name
rm -fv /etc/exim4/conf.d/retry/37_retry
cat >/etc/exim4/conf.d/retry/17_retry <<'EOF'
# Retry fast for my own domains
iankelling.org * F,1d,1m;F,14d,1h
amnimal.ninja * F,1d,1m;F,14d,1h
expertpathologyreview.com * F,1d,1m;F,14d,1h
je.b8.nz * F,1d,1m;F,14d,1h
zroe.org * F,1d,1m;F,14d,1h
eximbackup.b8.nz * F,1d,1m;F,14d,1h
# The spec says the target domain will be used for temporary host errors,
# but i've found that isn't correct, the hostname is required
# at least sometimes.
nn.b8.nz * F,1d,1m;F,14d,1h
defaultnn.b8.nz * F,1d,1m;F,14d,1h
mx.iankelling.org * F,1d,1m;F,14d,1h
bk.b8.nz * F,1d,1m;F,14d,1h
eggs.gnu.org * F,1d,1m;F,14d,1h
fencepost.gnu.org * F,1d,1m;F,14d,1h
# afaik our retry doesnt need this, but just using everything
mx.amnimal.ninja * F,1d,1m;F,14d,1h
mx.expertpathologyreview.com * F,1d,1m;F,14d,1h
mail.fsf.org * F,1d,15m;F,14d,1h
EOF
rm -vf /etc/exim4/conf.d/main/000_localmacros # old filename
# separate file so without quoted EOF for convenience
cat >/etc/exim4/conf.d/main/000_local2 </etc/exim4/conf.d/main/000_local <<'EOF'
MAIN_TLS_ENABLE = true
# require tls connections for all smarthosts
REMOTE_SMTP_SMARTHOST_HOSTS_REQUIRE_TLS = ! nn.b8.nz
REMOTE_SMTP_SMARTHOST_HOSTS_AVOID_TLS = nn.b8.nz
# debian exim config added this in 2016 or so?
# it's part of the smtp spec, to limit lines to 998 chars
# but a fair amount of legit mail does not adhere to it. I don't think
# this should be default, like it says in
# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=828801
# todo: the bug for introducing this was about headers, but
# the fix maybe is for all lines? one says gmail rejects, the
# other says gmail does not reject. figure out and open a new bug.
IGNORE_SMTP_LINE_LENGTH_LIMIT = true
# more verbose logs. used to use +all, but made it less for more efficiency.
MAIN_LOG_SELECTOR = -skip_delivery -tls_cipher -tls_certificate_verified +all_parents +address_rewrite +arguments +deliver_time +pid +queue_time +queue_time_overall +received_recipients +received_sender +return_path_on_delivery +sender_on_delivery +smtp_confirmation +subject
# Based on spec, seems like a good idea to be nice.
smtp_return_error_details = true
# default is 10. when exim has been down for a bit, fsf mailserver
# will do a big send in one connection, then exim decides to put
# the messages in the queue instead of delivering them, to avoid
# spawning too many delivery processes. This is the same as the
# fsfs value. And the corresponding one for how many messages
# to send out in 1 connection remote_max_parallel = 256
smtp_accept_queue_per_connection = 500
DKIM_CANON = relaxed
DKIM_SELECTOR = li
# The file is based on the outgoing domain-name in the from-header.
# sign if key exists
DKIM_PRIVATE_KEY = ${if exists{/etc/exim4/${dkim_domain}-private.pem} {/etc/exim4/${dkim_domain}-private.pem}}
# most of the ones that gmail seems to use.
# Exim has horrible default of signing unincluded
# list- headers since they got mentioned in an
# rfc, but this messes up mailing lists, like gnu/debian which want to
# keep your dkim signature intact but add list- headers.
DKIM_SIGN_HEADERS = mime-version:in-reply-to:references:from:date:subject:to
domainlist local_hostnames = ! je.b8.nz : ! bk.b8.nz : *.b8.nz : b8.nz
# note: most of these are duplicated in spamassassin config
hostlist iank_trusted = <; \
# veth0
10.173.8.1 ; \
# li li_ip6
72.14.176.105 ; 2600:3c00::f03c:91ff:fe6d:baf8 ; \
# li_vpn_net li_vpn_net_ip6s
10.8.0.0/24; 2600:3c00:e000:280::/64 ; 2600:3c00:e002:3800::/56 ; \
# bk bk_ip6
85.119.83.50 ; 2001:ba8:1f1:f0c9::2 ; \
# je je_ipv6
85.119.82.128 ; 2001:ba8:1f1:f09d::2 ; \
# fsf_mit_net fsf_mit_net_ip6 fsf_net fsf_net_ip6 fsf_office_net
18.4.89.0/24 ; 2603:3005:71a:2e00::/64 ; 209.51.188.0/24 ; 2001:470:142::/48 ; 74.94.156.208/28
# this is the default delay_warning_condition, plus matching on local_domains.
# If I have some problem with my local system that causes delayed delivery,
# I dont want to send warnings out to non-local domains.
delay_warning_condition = ${if or {\
{ !eq{$h_list-id:$h_list-post:$h_list-subscribe:}{} }\
{ match{$h_precedence:}{(?i)bulk|list|junk} }\
{ match{$h_auto-submitted:}{(?i)auto-generated|auto-replied} }\
{ match_domain{$domain}{+local_domains} }\
} {no}{yes}}
# enable 587 in addition to the default 25, so that
# i can send mail where port 25 is firewalled by isp
daemon_smtp_ports = 25 : 587 : 10025
# default of 25, can get stuck when catching up on mail
smtp_accept_max = 400
smtp_accept_reserve = 100
smtp_reserve_hosts = +iank_trusted
# Rules that make receiving more liberal should be on backup hosts
# so that we dont reject mail accepted by MAIL_HOST
LOCAL_DENY_EXCEPTIONS_LOCAL_ACL_FILE = /etc/exim4/conf.d/local_deny_exceptions_acl
acl_not_smtp = acl_check_not_smtp
DEBBUGS_DOMAIN = b.b8.nz
EOF
if dpkg --compare-versions "$(dpkg-query -f='${Version}\n' --show exim4)" ge 4.94; then
cat >>/etc/exim4/conf.d/main/000_local <<'EOF'
# In t11, we cant do the old anymore because this is tainted data used in a file lookup.
# /usr/share/doc/exim4/NEWS.Debian.gz suggests to use lookups to untaint data.
DKIM_DOMAIN = ${lookup {${domain:$rh_from:}}lsearch,ret=key{/etc/exim4/conf.d/my-dkim-domains}}
EOF
else
cat >>/etc/exim4/conf.d/main/000_local <<'EOF'
# From comments in
# https://debian-administration.org/article/718/DKIM-signing_outgoing_mail_with_exim4
# and its best for this to align https://tools.ietf.org/html/rfc7489#page-8
# There could be some circumstance when the
# from: isnt our domain, but the envelope sender is
# and so still want to sign, but I cant think of any case.
DKIM_DOMAIN = ${lc:${domain:$rh_from:}}
EOF
fi
cat >/etc/exim4/conf.d/main/30_local </etc/exim4/update-exim4.conf.conf <<'EOF'
# default stuff, i havent checked if its needed
dc_minimaldns='false'
CFILEMODE='644'
dc_use_split_config='true'
dc_mailname_in_oh='true'
EOF
# * radicale
if mailhost; then
if ! mountpoint /o; then
echo "error /o is not a mountpoint" >&2
exit 1
fi
# davx/davdroid setup instructions at the bottom
# main docs:
# http://radicale.org/user_documentation/
# https://davdroid.bitfire.at/configuration/
# note on debugging: if radicale can't bind to the address,
# in the log it just says "Starting Radicale". If you run
# it in the foreground, it will give more info. Background
# plus debug does not help.
# sudo -u radicale radicale -D
# created radicale password file with:
# htpasswd -c /p/c/machine_specific/li/filesystem/etc/caldav-htpasswd ian
# chmod 640 /p/c/machine_specific/li/filesystem/etc/caldav-htpasswd
# # setup chgrp www-data in ./conflink
pi-nostart radicale
m usermod -a -G radicale iank
u /etc/systemd/system/radicale.service.d/override.conf <: Fatal: master: service(lmtp): child 3839880 returned error 83 (Out of memory (service lmtp { vsz_limit=256 MB }, you may need to increase it) - set CORE_OUTOFMEM=1 environment to get core dump)
# exim would just queue mail until it eventually succeeded.
# Deciding what to increase it to, I found this
# https://dovecot.org/list/dovecot/2011-December/080056.html
# which suggests 3x the largest dovecot.index.cache file
# and then I found that
# md/l/testignore/dovecot.index.cache is 429M, my largest cache file,
# but that folder only has 2k messages.
# next biggest is md/l/qemu-devel/dovecot.index.cache 236M
# which lead to me a search https://doc.dovecot.org/admin_manual/known_issues/large_cache/
# which suggests 1.5x the maximum cache file size 1G, and
# that I can safely rm the index.
default_vsz_limit = 1500M
EOF
if dpkg --compare-versions "$(dpkg-query -f='${Version}\n' --show dovecot-core)" ge 1:2.3; then
cat </etc/dovecot/local.conf
;;&
# ** $MAIL_HOST)
$MAIL_HOST)
# If we changed 90-sieve.conf and removed the active part of the
# sieve option, we wouldn\'t need this, but I\'d rather not modify a
# default config if not needed. This won\'t work as a symlink in /a/c
# unfortunately.
m sudo -u $u /a/exe/lnf -T sieve/main.sieve $uhome/.dovecot.sieve
if [[ ! -e $uhome/sieve/personal.sieve ]]; then
m touch $uhome/sieve/personal{,end}{,test}.sieve
fi
rm -fv /etc/dovecot/conf.d/20-lmtp.conf # file from prev version
# Having backups of indexes is a waste of space. This also means we
# don't send them around with btrbk, I think it is probably
# preferable use a bit more cpu to recalculate indexes.
install -d -m 700 -o iank -g iank /var/dovecot-indexes
cat >>/etc/dovecot/local.conf <>/etc/dovecot/local.conf <
$domain
$domain Mail
$domain
mail2.iankelling.org
993
SSL
%EMAILADDRESS%
password-cleartext
mail2.iankelling.org
587
STARTTLS
%EMAILADDRESS%
password-cleartext
true
false
$domain website.
%EMAILADDRESS%
EOF
done
fi
# * roundcube setup
if [[ $HOSTNAME == bk ]]; then
# zip according to /installer
# which requires adding a line to /usr/local/lib/roundcubemail/config/config.inc.php
# $config['enable_installer'] = true;
pi roundcube roundcube-sqlite3 php-zip apache2 php-fpm
### begin composer install
# https://getcomposer.org/doc/faqs/how-to-install-composer-programmatically.md
cd /usr/local/bin
EXPECTED_CHECKSUM="$(php -r 'copy("https://composer.github.io/installer.sig", "php://stdout");')"
php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
ACTUAL_CHECKSUM="$(php -r "echo hash_file('sha384', 'composer-setup.php');")"
if [ "$EXPECTED_CHECKSUM" != "$ACTUAL_CHECKSUM" ]
then
>&2 echo 'ERROR: Invalid installer checksum'
rm composer-setup.php
exit 1
fi
php composer-setup.php --quiet
rm composer-setup.php
# based on error when running composer
mkdir -p /var/www/.composer
chown www-data:www-data /var/www/.composer
### end composer install
rcdirs=(/usr/local/lib/rcexpertpath /usr/local/lib/rcninja)
ncdirs=(/var/www/ncexpertpath /var/www/ncninja)
# point debian cronjob to our local install, preventing daily cron error
# debian's cronjob will fail, remove both paths it uses just to be sure
rm -fv /usr/share/roundcube/bin/cleandb.sh /etc/cron.d/roundcube-core
#### begin dl roundcube
# note, im r2e subbed to https://github.com/roundcube/roundcubemail/releases.atom
v=1.4.13; f=roundcubemail-$v-complete.tar.gz
cd /root
if [[ -e $f ]]; then
timestamp=$(stat -c %Y $f)
else
timestamp=0
fi
m wget -nv -N https://github.com/roundcube/roundcubemail/releases/download/$v/$f
new_timestamp=$(stat -c %Y $f)
for rcdir in ${rcdirs[@]}; do
if [[ $timestamp != "$new_timestamp" || ! -e "$rcdir/config/secret" ]]; then
m tar -C /usr/local/lib --no-same-owner -zxf $f
m rm -rf $rcdir
m mv /usr/local/lib/roundcubemail-$v $rcdir
fi
done
#### end dl roundcube
for ((i=0; i < ${#bkdomains[@]}; i++)); do
domain=${bkdomains[i]}
rcdir=${rcdirs[i]}
rcbase=${rcdir##*/}
ncdir=${ncdirs[i]}
# copied from debians cronjob
u /etc/cron.d/$rcbase </dev/null
EOF
m /a/exe/web-conf - apache2 $domain <
Options +FollowSymLinks
# This is needed to parse $rcdir/.htaccess.
AllowOverride All
Require all granted
# Protecting basic directories:
Options -FollowSymLinks
AllowOverride None
### end roundcube settings
### begin nextcloud settings
Alias /nextcloud "$ncdir/"
Require all granted
AllowOverride All
Options FollowSymLinks MultiViews
Dav off
# based on install checker, links to
# https://docs.nextcloud.com/server/19/admin_manual/issues/general_troubleshooting.html#service-discovery
# their example was a bit wrong, I figured it out by adding
# LogLevel warn rewrite:trace5
# then watching the apache logs
RewriteEngine on
RewriteRule ^/\.well-known/host-meta /nextcloud/public.php?service=host-meta [QSA,L]
RewriteRule ^/\.well-known/host-meta\.json /nextcloud/public.php?service=host-meta-json [QSA,L]
RewriteRule ^/\.well-known/webfinger /nextcloud/public.php?service=webfinger [QSA,L]
RewriteRule ^/\.well-known/carddav /nextcloud/remote.php/dav/ [R=301,L]
RewriteRule ^/\.well-known/caldav /nextcloud/remote.php/dav/ [R=301,L]
### end nextcloud settings
EOF
if [[ ! -e $rcdir/config/secret ]]; then
base64 $rcdir/config/secret || [[ $? == 141 || ${PIPESTATUS[0]} == 32 ]]
fi
secret=$(cat $rcdir/config/secret)
rclogdir=/var/log/$rcbase
rctmpdir=/var/tmp/$rcbase
rcdb=/m/rc/$rcbase.sqlite
# config from mailinabox
u $rcdir/config/config.inc.php < array(
'verify_peer' => false,
'verify_peer_name' => false,
),
);
\$config['imap_timeout'] = 15;
\$config['smtp_server'] = 'tls://127.0.0.1';
\$config['smtp_conn_options'] = array(
'ssl' => array(
'verify_peer' => false,
'verify_peer_name' => false,
),
);
\$config['product_name'] = 'webmail';
\$config['des_key'] = '$secret';
\$config['plugins'] = array('archive', 'zipdownload', 'password', 'managesieve', 'jqueryui', 'carddav', 'html5_notifier');
\$config['skin'] = 'elastic';
\$config['login_autocomplete'] = 2;
\$config['password_charset'] = 'UTF-8';
\$config['junk_mbox'] = 'Spam';
# disable builtin addressbook
\$config['address_book_type'] = '';
?>
EOF
m mkdir -p $rclogdir
m chmod 750 $rclogdir
m chown www-data:adm $rclogdir
# note: subscribed to updates:
# r2e add rcmcarddav https://github.com/blind-coder/rcmcarddav/commits/master.atom ian@iankelling.org
# r2e add roundcube https://github.com/roundcube/roundcubemail/releases.atom ian@iankelling.org
m mkdir -p $rctmpdir /m/rc
m chown -R www-data.www-data $rctmpdir /m/rc
m chmod 750 $rctmpdir
# todo: check for other mailinabox things
# Ensure the log file monitored by fail2ban exists, or else fail2ban can't start.
m sudo -u www-data touch $rclogdir/errors.log
#### begin carddav install
# This is the official roundcube carddav repo.
# Install doc suggests downloading with composer, but that
# didnt work, it said some ldap package for roundcube was missing,
# but I dont want to download some extra ldap thing.
# https://github.com/blind-coder/rcmcarddav/blob/master/doc/INSTALL.md
verf=$rcdir/plugins/carddav/myversion
upgrade=false
install=false
v=5.0.1
if [[ -e $verf ]]; then
if [[ $(cat $verf) != "$v" ]]; then
install=true
upgrade=true
fi
else
install=true
fi
if $install; then
m rm -rf $rcdir/plugins/carddav
tmpd=$(mktemp -d)
m wget -nv -O $tmpd/t.tgz https://github.com/blind-coder/rcmcarddav/releases/download/v$v/carddav-v$v.tar.gz
cd $rcdir/plugins
tar xzf $tmpd/t.tgz
rm -rf $tmpd
m chown -R www-data:www-data $rcdir/plugins/carddav
m cd $rcdir/plugins/carddav
if $upgrade; then
m sudo -u www-data composer.phar update --no-dev
else
m sudo -u www-data composer.phar install --no-dev
fi
m chown -R root:root $rcdir/plugins/carddav
echo $v >$verf
fi
# So, strangely, this worked in initial testing, but then
# on first run it wouldn't show the existing contacts until
# I went into the carddav settings and did "force immediate sync",
# which seemed to fix things. Note, some of these settings
# get initalized per/addressbook in the db, then need changing
# there or through the settings menu.
# About categories, see https://www.davx5.com/tested-with/nextcloud
# https://github.com/blind-coder/rcmcarddav/blob/master/doc/GROUPS.md
u $rcdir/plugins/carddav/config.inc.php < 'Main',
'username' => '%u', // login username
'password' => '%p', // login password
'url' => 'https://$domain/nextcloud/remote.php/dav/addressbooks/users/%u/contacts',
'active' => true,
'readonly' => false,
'refresh_time' => '00:10:00',
'fixed' => array('username','password'),
'use_categories' => false,
'hide' => false,
);
?>
EOF
#### end carddav install
cd $rcdir/plugins
if [[ ! -d html5_notifier ]]; then
m git clone https://github.com/stremlau/html5_notifier
fi
cd $rcdir/plugins/html5_notifier
m git pull --rebase
# todo: try out roundcube plugins: thunderbird labels
# Password changing plugin settings
cat $rcdir/plugins/password/config.inc.php.dist - >$rcdir/plugins/password/config.inc.php <<'EOF'
# following are from mailinabox
$config['password_minimum_length'] = 8;
$config['password_db_dsn'] = 'sqlite:////m/rc/users.sqlite';
$config['password_query'] = 'UPDATE users SET password=%D WHERE email=%u';
$config['password_dovecotpw'] = '/usr/bin/doveadm pw';
$config['password_dovecotpw_method'] = 'SHA512-CRYPT';
$config['password_dovecotpw_with_method'] = true;
EOF
# so PHP can use doveadm, for the password changing plugin
m usermod -a -G dovecot www-data
m usermod -a -G mail $u
# so php can update passwords
m chown www-data:dovecot /m/rc/users.sqlite
m chmod 664 /m/rc/users.sqlite
# Run Roundcube database migration script (database is created if it does not exist)
m $rcdir/bin/updatedb.sh --dir $rcdir/SQL --package roundcube
m chown www-data:www-data $rcdb
m chmod 664 $rcdb
done # end loop over domains and rcdirs
### begin php setup for rc ###
# Enable PHP modules.
m phpenmod -v php mcrypt imap
# dpkg says this is required.
# nextcloud needs these too
m a2enmod proxy_fcgi setenvif
fpm=$(dpkg-query -s php-fpm | sed -nr 's/^Depends:.* (php[^ ]*-fpm)( .*|$)/\1/p') # eg: php7.4-fpm
phpver=$(dpkg-query -s php-fpm | sed -nr 's/^Depends:.* php([^ ]*)-fpm( .*|$)/\1/p')
m a2enconf $fpm
# 3 useless guides on php fpm fcgi debian 10 later, i figure out from reading
# /etc/apache2/conf-enabled/php7.3-fpm.conf
m a2dismod php$phpver
# according to /install, we should set date.timezone,
# but that is dumb, the system already has the right zone in
# $rclogdir/errors.log
# todo: consider other settings in
# /a/opt/mailinabox/setup/nextcloud.sh
u /etc/php/$phpver/cli/conf.d/30-local.ini <<'EOF'
apc.enable_cli = 1
EOF
u /etc/php/$phpver/fpm/conf.d/30-local.ini <<'EOF'
date.timezone = "America/New_York"
# for nextcloud
upload_max_filesize = 2000M
post_max_size = 2000M
# install checker, nextcloud/settings/admin/overview
memory_limit = 512M
EOF
m systemctl restart $fpm
# dunno if reload/restart is needed
m systemctl reload apache2
# note bk backups are defined in crontab outside this file
### end php setup for rc ###
fi # end roundcube setup
# * nextcloud setup
if [[ $HOSTNAME == bk ]]; then
# from install checker, nextcloud/settings/admin/overview and
# https://docs.nextcloud.com/server/19/admin_manual/installation/source_installation.html
# curl from the web installer requirement, but i switched to cli
# it recommends php-file info, but that is part of php7.3-common, already got installed
# with roundcube.
m pi php-curl php-bz2 php-gmp php-bcmath php-imagick php-apcu
# https://docs.nextcloud.com/server/19/admin_manual/installation/source_installation.html
cat >/etc/php/$phpver/fpm/pool.d/localwww.conf <<'EOF'
[www]
clear_env = no
EOF
for ((i=0; i < ${#bkdomains[@]}; i++)); do
domain=${bkdomains[i]}
ncdir=${ncdirs[i]}
myncdir=/var/local/${ncdir##*/}
ncbase=${ncdir##*/}
mkdir -p $myncdir
m cd /var/www
if [[ ! -e $ncdir/index.php ]]; then
# if we wanted to only install a specific version, use something like
# file=latest-22.zip
file=latest.zip
m wget -nv -N https://download.nextcloud.com/server/releases/$file
m rm -rf nextcloud
m unzip -q $file
m rm -f $file
m chown -R www-data.www-data nextcloud
m mv nextcloud $ncdir
fi
if [[ ! -e $myncdir/done-install ]]; then
m cd $ncdir
m sudo -u www-data php occ maintenance:install --database sqlite --admin-user iank --admin-pass $nextcloud_admin_pass
m touch $myncdir/done-install
fi
m cd $ncdir/config
# if we did this more than once, it would revert the
# version number to the original.
if [[ ! -e $myncdir/config.php-orig || ! -s config.php ]]; then
if [[ -s config.php ]]; then
m cp -a config.php $myncdir/config.php-orig
# keep the file so it keeps the same permissions.
truncate -s0 config.php
fi
cat $myncdir/config.php-orig - >$myncdir/tmp.php < 'OC_User_IMAP','arguments' => array('127.0.0.1', 143, null),),);
# based on installer check
# https://docs.nextcloud.com/server/19/admin_manual/configuration_server/caching_configuration.html
\$CONFIG['memcache.local'] = '\OC\Memcache\APCu';
\$CONFIG['overwrite.cli.url'] = 'https://$domain/nextcloud';
\$CONFIG['htaccess.RewriteBase'] = '/nextcloud';
\$CONFIG['trusted_domains'] = array (
0 => '$domain',
);
#\$CONFIG[''] = '';
fwrite(STDOUT, "config.php
fi
cd $ncdir
m sudo -u www-data php occ maintenance:update:htaccess
list=$(sudo -u www-data php $ncdir/occ --output=json_pretty app:list)
# user_external not compaible with nc 23
for app in contacts calendar; do
if [[ $(printf "%s\n" "$list"| jq ".enabled.$app") == null ]]; then
cd $ncdir
m sudo -u www-data php occ app:install $app
fi
done
u /etc/systemd/system/$ncbase.service <&2
# -odf or else systemd will kill the background delivery process
# and the message will sit in the queue until the next queue run.
exim -odf -t <
Options Indexes SymLinksIfOwnerMatch MultiViews
DirectoryIndex index.html
Require all granted
ScriptAlias /cgi/ /var/lib/debbugs/www/cgi/
AllowOverride None
Options ExecCGI SymLinksIfOwnerMatch
Require all granted
RewriteEngine on
RewriteCond %{HTTP_USER_AGENT} .*apt-listbugs.*
RewriteRule .* /apt-listbugs.html [R,L]
# RewriteLog /org/bugs.debian.org/apache-rewrite.log
# RewriteLogLevel 0
#RewriteRule ^/$ http://www.debian.org/Bugs/
RewriteRule ^/(robots\.txt|release-critical|apt-listbugs\.html)$ - [L]
# The following two redirect to up-to-date pages
RewriteRule ^/[[:space:]]*#?([[:digit:]][[:digit:]][[:digit:]]+)([;&].+)?$ /cgi-bin/bugreport.cgi?bug=$1$2 [L,R,NE]
RewriteRule ^/([^/+]*)([+])([^/]*)$ "/$1%%{%}2B$3" [N]
RewriteRule ^/[Ff][Rr][Oo][Mm]:([^/]+\@.+)$ /cgi-bin/pkgreport.cgi?submitter=$1 [PT,NE]
# Commented out, 'cuz aj says it will crash master. (old master)
# RewriteRule ^/[Ss][Ee][Vv][Ee][Rr][Ii][Tt][Yy]:([^/]+\@.+)$ /cgi-bin/pkgreport.cgi?severity=$1 [L,R]
RewriteRule ^/([^/]+\@.+)$ /cgi-bin/pkgreport.cgi?maint=$1 [PT,NE]
RewriteRule ^/mbox:([[:digit:]][[:digit:]][[:digit:]]+)([;&].+)?$ /cgi-bin/bugreport.cgi?mbox=yes&bug=$1$2 [PT,NE]
RewriteRule ^/src:([^/]+)$ /cgi-bin/pkgreport.cgi?src=$1 [PT,NE]
RewriteRule ^/severity:([^/]+)$ /cgi-bin/pkgreport.cgi?severity=$1 [PT,NE]
RewriteRule ^/tag:([^/]+)$ /cgi-bin/pkgreport.cgi?tag=$1 [PT,NE]
# RewriteMap fix-chars int:noescape
RewriteCond %{REQUEST_URI} ^/(Access\.html|Developer\.html|Reporting\.html|server-request\.html|server-control\.html|server-refcard\.html).* [NC]
RewriteRule .* - [L]
# PT|passthrough to bugreport.cgi and pkgreport.cgi
RewriteRule ^/([0-9]+)$ /cgi-bin/bugreport.cgi?bug=$1 [PT,NE]
RewriteRule ^/([^/]+)$ /cgi-bin/pkgreport.cgi?pkg=$1 [PT,NE]
EOF
# * exim host conditional config
# ** exim certs
all_dirs=(/p/c/filesystem)
for x in /p/c/machine_specific/*.hosts /a/bin/ds/machine_specific/*.hosts; do
if grep -qxF $HOSTNAME $x; then all_dirs+=( ${x%.hosts} ); fi
done
files=()
for d in ${all_dirs[@]}; do
f=$d/etc/exim4/passwd
if [[ -e $f ]]; then
files+=($f)
fi
tmp=($d/etc/exim4/*.pem)
if (( ${#tmp[@]} )); then
files+=(${tmp[@]})
fi
done
if (( ${#files[@]} )); then
sudo rsync -ahhi --chown=root:Debian-exim --chmod=0640 ${files[@]} /etc/exim4/
fi
# ** exim: auth
case $HOSTNAME in
bk|je)
# avoid accepting mail for invalid users
# https://wiki.dovecot.org/LMTP/Exim
cat >>/etc/exim4/conf.d/rcpt_local_acl <<'EOF'
deny
message = invalid recipient
domains = +local_domains
!verify = recipient/callout=no_cache
EOF
u /etc/exim4/conf.d/auth/29_exim4-config_auth <<'EOF'
dovecot_plain:
driver = dovecot
public_name = PLAIN
server_socket = /var/run/dovecot/auth-client
server_set_id = $auth1
EOF
;;
esac
if $bhost_t; then
u /etc/exim4/conf.d/auth/29_exim4-config_auth <<'EOF'
# from 30_exim4-config_examples
plain_server:
driver = plaintext
public_name = PLAIN
server_condition = "${if crypteq{$auth3}{${extract{1}{:}{${lookup{$auth2}lsearch{CONFDIR/passwd}{$value}{*:*}}}}}{1}{0}}"
server_set_id = $auth2
server_prompts = :
.ifndef AUTH_SERVER_ALLOW_NOTLS_PASSWORDS
server_advertise_condition = ${if eq{$tls_in_cipher}{}{}{*}}
.endif
EOF
fi
# ** exim: main daemon use non-default config file
case $HOSTNAME in
bk|$MAIL_HOST)
# to see the default comments in /etc/default/exim4:
# s update-exim4defaults --force --init
# which will overwrite any existing file
u /etc/default/exim4 <<'EOF'
QUEUERUNNER='combined'
QUEUEINTERVAL='10m'
COMMONOPTIONS='-C /etc/exim4/my.conf'
UPEX4OPTS='-o /etc/exim4/my.conf'
# in t12 exim, this replaces all the above options
EXIMSERVICE='-bdf -q10m -C /etc/exim4/my.conf'
# i use epanic-clean for alerting if there are bad paniclog entries
E4BCD_WATCH_PANICLOG='no'
EOF
# make exim be a nonroot setuid program.
chown Debian-exim:Debian-exim /usr/sbin/exim4
# needs guid set in order to become Debian-exim
chmod g+s,u+s /usr/sbin/exim4
# need this to avoid error on service reload:
# 2022-08-07 18:44:34.005 [892491] pid 892491: SIGHUP received: re-exec daemon
# 2022-08-07 18:44:34.036 [892491] cwd=/var/spool/exim4 5 args: /usr/sbin/exim4 -bd -q30m -C /etc/exim4/my.conf
# 2022-08-07 18:44:34.043 [892491] socket bind() to port 25 for address (any IPv6) failed: Permission denied: waiting 30s before trying again (9 more tries)
# note: the daemon gives up and dies after retrying those 9 times.
# I came upon this by guessing and trial and error.
setcap CAP_NET_BIND_SERVICE+ei /usr/sbin/exim4
u /etc/exim4/trusted_configs <<'EOF'
/etc/exim4/my.conf
EOF
;;
*)
# default file
u /etc/default/exim4 <<'EOF'
QUEUERUNNER='combined'
QUEUEINTERVAL='30m'
EOF
;;
esac
# ** exim non-root
case $HOSTNAME in
bk|je|li)
# no reason to expect it to ever be there.
rm -fv /etc/systemd/system/exim4.service.d/nonroot.conf
;;
*)
dirs=()
for d in /a /d /m /media /mnt /nocow /o /p /q; do
if [[ -d $d ]]; then
dirs+=($d)
fi
done
u /etc/systemd/system/exim4.service.d/nonroot.conf <>/etc/exim4/update-exim4.conf.conf <>/etc/exim4/conf.d/main/000_local <>/etc/exim4/update-exim4.conf.conf </etc/exim4/conf.d/main/000_local-nn <>/etc/exim4/conf.d/main/000_local < /etc/mailname
# mail default domain.
u /etc/mailutils.conf <<'EOF'
address {
email-domain iankelling.org;
};
EOF
# mail.iankelling.org so local imap clients can connect with tls and
# when they happen to not be local.
# todo: this should be 10.8.0.4
/a/exe/cedit nn /etc/hosts <<'EOF' || [[ $? == 1 ]]
# note: i put nn.b8.nz into bind for good measure
10.173.8.2 nn.b8.nz mx.iankelling.org
EOF
# note: systemd-resolved will consult /etc/hosts, dnsmasq wont. this assumes
# weve configured this file in dnsmasq if we are using it.
/a/exe/cedit mail /etc/dnsmasq-servers.conf <<'EOF' || [[ $? == 1 ]]
server=/mx.iankelling.org/127.0.1.1
EOF
# I used to use debconf-set-selections + dpkg-reconfigure,
# which then updates this file
# but the process is slower than updating it directly and then I want to set other things in
# update-exim4.conf.conf, so there's no point.
# The file is documented in man update-exim4.conf,
# except the man page is not perfect, read the bash script to be sure about things.
# The debconf questions output is additional documentation that is not
# easily accessible, but super long, along with the initial default comment in this
# file, so I've saved that into ./mail-notes.conf.
#
# # TODO: remove mx.iankelling.org once systems get updated mail-setup from jan 2022
cat >>/etc/exim4/update-exim4.conf.conf <$f <<'EOF'
#!/bin/bash
cd /etc
wget -q -N https://publicsuffix.org/list/public_suffix_list.dat
EOF
m chmod 755 $f
;;
# ** bk
## we use this host to monitor MAIL_HOST and host a mail server for someone
bk)
# No clamav on je, it has 1.5g memory and clamav uses most of it.
#
# No clamav on MAIL_HOST because it is just a waste of useful cpu
# time and memory when I'm running on an x200, and it takes 30
# seconds to shut down.
cat >>/etc/exim4/conf.d/main/000_local <> /etc/exim4/conf.d/data_local_acl <<'EOF'
deny
malware = */defer_ok
!condition = ${if match {$malware_name}{\N^Heuristic\N}}
message = This message was detected as possible malware ($malware_name).
warn
!hosts = +iank_trusted
!authenticated = *
condition = ${if def:malware_name}
remove_header = Subject:
add_header = Subject: [Clamav warning: $malware_name] $h_subject
log_message = heuristic malware warning: $malware_name
warn
# fdate = future date. # tdate = temporary date.
condition = ${if def:h_fdate}
remove_header = fdate:
add_header = tdate:
control = freeze
EOF
/a/exe/cedit nn /etc/hosts <<'EOF' || [[ $? == 1 ]]
10.173.8.2 nn.b8.nz
EOF
sed -r -f - /etc/init.d/exim4 <<'EOF' |u /etc/init.d/exim4in
s,/etc/default/exim4,/etc/default/exim4in,g
s,/run/exim4/exim.pid,/run/exim4/eximin.pid,g
s,(^[ #]*Provides:).*,\1 exim4in,
s,(^[ #]*NAME=).*,\1"exim4in",
EOF
chmod +x /etc/init.d/exim4in
u /etc/systemd/system/exim4in.service.d/alwaysrestart.conf <<'EOF'
[Unit]
# needed to continually restart
StartLimitIntervalSec=0
[Service]
Restart=always
# time to sleep before restarting a service
RestartSec=20
EOF
u /etc/default/exim4in <<'EOF'
# defaults but no queue runner and alternate config dir
QUEUERUNNER='no'
COMMONOPTIONS='-oP /run/exim4/eximin.pid'
UPEX4OPTS='-d /etc/myexim4'
EOF
echo bk.b8.nz > /etc/mailname
cat >>/etc/exim4/update-exim4.conf.conf < /etc/mailname
cat >>/etc/exim4/update-exim4.conf.conf <>/etc/exim4/conf.d/main/000_local <>/etc/exim4/update-exim4.conf.conf <>/etc/exim4/update-exim4.conf.conf <>/etc/exim4/update-exim4.conf.conf <>/etc/exim4/update-exim4.conf.conf <>/etc/myexim4/conf.d/main/000_local <<'EOF'
# this makes it easier to see which exim is doing what
log_file_path = /var/log/exim4/my%s
EOF
cat >/etc/logrotate.d/myexim <<'EOF'
/var/log/exim4/mymain /var/log/exim4/myreject {
daily
missingok
rotate 1000
delaycompress
notifempty
nocreate
}
/var/log/exim4/mypanic {
size 10M
missingok
rotate 10
compress
delaycompress
notifempty
nocreate
}
EOF
# If we ever wanted to have a separate spool,
# we could do it like this.
# cat >>/etc/exim4/conf.d/main/000_local-nn <<'EOF'
# spool_directory = /var/spool/myexim4
# EOF
cat >>/etc/myexim4/update-exim4.conf.conf <<'EOF'
dc_eximconfig_configtype='smarthost'
dc_smarthost='nn.b8.nz'
EOF
;;&
bk)
# config for the non-nn exim
cat >>/etc/myexim4/conf.d/main/000_local <<'EOF'
MAIN_HARDCODE_PRIMARY_HOSTNAME = mail2.iankelling.org
EOF
;;
$MAIL_HOST)
u /etc/myexim4/conf.d/router/185_sentarchive <<'EOF'
sentarchive:
driver = redirect
domains = ! +local_domains
senders = <; *@fsf.org ; *@posteo.net
condition = ${if and {{!bool{${lookup{$local_part@$domain}lsearch{/etc/exim4/ignore-sent}{true}}}} {!match {$h_user-agent:}{emacs}}}}
data = vojdedIdNejyebni@b8.nz
unseen
EOF
u /etc/myexim4/conf.d/router/160_backup_redir <<'EOF'
backup_redir:
driver = redirect
# i dont email myself from my own machine much, so lets ignore that.
domains = ! +local_domains
senders = <; *@fsf.org ; *@posteo.net
condition = ${if and {{!bool{${lookup{$local_part@$domain}lsearch{/etc/exim4/ignore-sent}{true}}}} {!match {$h_user-agent:}{emacs}}}}
# b is just an arbirary short string
data = b@eximbackup.b8.nz
# note, to test this, i could temporarily allow testignore.
# alerts avoids potential mail loop.
local_parts = ! root : ! testignore : ! alerts : ! daylert
unseen = true
errors_to = alerts@iankelling.org
EOF
# for bk, we have a exim4in.service that will do this for us.
m update-exim4.conf -d /etc/myexim4
;;
esac
# * spool dir setup
# ** bind mount setup
# put spool dir in directory that spans multiple distros.
# based on http://www.postfix.org/qmgr.8.html and my notes in gnus
#
dir=/nocow/exim4
sdir=/var/spool/exim4
# we only do this if our system has $dir
# this used to do a symlink, but, in the boot logs, /nocow would get mounted succesfully,
# about 2 seconds later, exim starts, and immediately puts into paniclog:
# honVi-0000u3-82 Failed to create directory "/var/spool/exim4/input": No such file or directory
# so, im trying a bind mount to get rid of that.
if [[ -e /nocow ]]; then
if ! grep -Fx "/nocow/exim4 /var/spool/exim4 none bind 0 0" /etc/fstab; then
echo "/nocow/exim4 /var/spool/exim4 none bind 0 0" >>/etc/fstab
fi
u /etc/systemd/system/exim4.service.d/override.conf <<'EOF'
[Unit]
# without local-fs on exim, we get these kind of errors in paniclog on shutdown:
# Failed to create spool file /var/spool/exim4//input//1jCLxz-0008V4-V9-D: Permission denied
After=local-fs.target
[Service]
ExecStartPre=/usr/local/bin/exim-nn-iptables
EOF
if ! mountpoint -q $sdir; then
stopifactive exim4 exim4in
if [[ -L $sdir ]]; then
m rm $sdir
fi
if [[ ! -e $dir && -d $sdir ]]; then
m mv $sdir $dir
fi
if [[ ! -d $sdir ]]; then
m mkdir $sdir
m chmod 000 $sdir # only want it to be used when its mounted
fi
m mount $sdir
fi
fi
# ** exim/spool uid setup
# i have the spool directory be common to distro multi-boot, so
# we need the uid to be the same. 608 cuz it's kind of in the middle
# of the free system uids.
IFS=:; read -r _ _ uid _ < <(getent passwd Debian-exim ||:) ||:; unset IFS
IFS=:; read -r _ _ gid _ < <(getent group Debian-exim ||:) ||:; unset IFS
if [[ ! $uid ]]; then
# from /var/lib/dpkg/info/exim4-base.postinst, plus uid and gid options
m adduser --uid 608 --system --group --quiet --home /var/spool/exim4 \
--no-create-home --disabled-login --force-badname Debian-exim
elif [[ $uid != 608 ]]; then
stopifactive exim4 exim4in
m usermod -u 608 Debian-exim
m groupmod -g 608 Debian-exim
m usermod -g 608 Debian-exim
m find / /nocow -xdev -path ./var/tmp -prune -o -uid $uid -execdir chown -h 608 {} +
m find / /nocow -xdev -path ./var/tmp -prune -o -gid $gid -execdir chgrp -h 608 {} +
fi
# note: example config has a debbugs user,
# but my exim runs setuid as Debian-exim so it can't switch
# to another user. Anyways, I'm not exposing this to the
# internet at this time. If I do, the thing to do would
# be to use a sudo config (or sudo alternative). This
# would be how to setup
# IFS=:; read -r _ _ uid _ < <(getent passwd debbugs||:) ||:; unset IFS
# if [[ ! $uid ]]; then
# # /a/opt/debbugs/debian/README.mail
# adduser --uid 610 --system --group --home /o/debbugs \
# --no-create-home --disabled-login --force-badname debbugs
# m find /o/debbugs -xdev -path ./var/tmp -prune -o -uid $uid -execdir chown -h 610 {} +
# m find /o/debbugs -xdev -path ./var/tmp -prune -o -gid $gid -execdir chgrp -h 610 {} +
# elif [[ $uid != 610 ]]; then
# err debbugs exist but is not uid 610: investigate
# fi
# * start / stop services
reifactive dnsmasq nscd
if $reload; then
m systemctl daemon-reload
fi
# optimization, this only needs to run once.
if [[ ! -e /sys/class/net/wghole ]]; then
# checking bhost_t is redundant, but could help us catch errors.
if $bhost_t || [[ -e /etc/wireguard/wghole.conf ]]; then
# todo: in mail-setup, we have a static list of backup hosts, not *y
m systemctl --now enable wg-quick@wghole
fi
fi
# optimization, this only needs to be run once
if [[ ! -e /var/lib/prometheus/node-exporter/exim_paniclog.prom ]]; then
sysd-prom-fail-install epanicclean
m systemctl --now enable epanicclean
fi
case $HOSTNAME in
je)
/a/exe/web-conf apache2 je.b8.nz
;;
bk)
/a/exe/web-conf apache2 mail2.iankelling.org
;;
esac
# optimization, this only needs to run once. But, if we move to a
# computer we haven't used much, we need to fetch a fresh cert.
# Existence check is just to avoid ugly error message from openssl.
if [[ ! -e /etc/exim4/fullchain.pem ]] || ! openssl x509 -checkend $(( 60 * 60 * 24 * 3 )) -noout -in /etc/exim4/fullchain.pem; then
m /a/bin/ds/mail-cert-cron -1 -i
m systemctl --now enable mailcert.timer
fi
case $HOSTNAME in
$MAIL_HOST|bk)
m systemctl --now enable mailnn mailnnroute
;;&
$MAIL_HOST)
# we use dns to start wg
if $reload; then
sre unbound
else
m systemctl --now enable unbound
fi
;;&
$MAIL_HOST|bk)
# If these have changes, id rather manually restart it, id rather
# not restart and cause temporary errors
if $reload; then
sre $vpnser
else
m systemctl --now enable $vpnser
fi
;;&
bk)
if ! systemctl is-active clamav-daemon >/dev/null; then
m systemctl --now enable clamav-daemon
out=$(rsync -aiSAX --chown=root:root --chmod=g-s /a/bin/ds/filesystem/etc/systemd/system/epanicclean.service /etc/systemd/system)
if [[ $out ]]; then
reload=true
fi
# note, this will cause paniclog entries because it takes like 45
# seconds for clamav to start, i use ./epanic-clean to remove
# them.
fi
;;&
$MAIL_HOST|bk|je)
# start spamassassin/dovecot before exim.
sre dovecot $spamd_ser
# Wait a bit before restarting exim, else I get a paniclog entry
# like: spam acl condition: all spamd servers failed. But I'm tired
# of waiting. I'll deal with this some other way.
#
# sleep 3
m systemctl --now enable mailclean.timer
;;&
$MAIL_HOST)
# < 2.1 (eg: in t9), uses a different data format which required manual
# migration. dont start if we are running an old version.
if dpkg --compare-versions "$(dpkg -s radicale | awk '$1 == "Version:" { print $2 }')" ge 2.1; then
m systemctl --now enable radicale
fi
;;&
esac
# for debugging dns issues
case $HOSTNAME in
je|bk)
systemctl enable --now logrotate-fast.timer
;;
esac
# last use of $reload happens in previous block
rm -f /var/local/mail-setup-reload
case $HOSTNAME in
$MAIL_HOST|bk|je|li)
# on li, these are never started, except $vpnser
:
;;
*)
soff radicale mailclean.timer dovecot $spamd_ser $vpnser mailnn clamav-daemon
;;
esac
sre exim4
case $HOSTNAME in
$MAIL_HOST)
m systemctl --now enable mailbindwatchdog
;;
*)
soff mailbindwatchdog
;;
esac
case $HOSTNAME in
bk) sre exim4in ;;
esac
# * mail monitoring / testing
# note, to test clamav, send an email with body that only contains
# https://en.wikipedia.org/wiki/EICAR_test_file
# which set malware_name to Eicar-Signature
case $HOSTNAME in
$MAIL_HOST|bk|je)
# note: cronjob "ian" also does some important monitoring
# todo: this will sometimes cause an alert because mailtest-check will run
# before we have setup network namespace and spamassassin
u /etc/cron.d/mailtest <>/etc/cron.d/mailtest </usr/local/bin/send-test-forward <<'EOF'
#!/bin/bash
# we remove from the queue older than 4.3 minutes since we send every 5 minutes.
olds=(
$(/usr/sbin/exiqgrep -o 260 -i -r '^(testignore@(iankelling\.org|zroe\.org|expertpathologyreview\.com|amnimal\.ninja|je\.b8\.nz)|jtuttle@gnu\.org)$')
)
if (( ${#olds[@]} )); then
/usr/sbin/exim -Mrm "${olds[@]}" >/dev/null
fi
EOF
for test_from in ${test_froms[@]}; do
test_to=${test_tos[0]}
for t in ${test_tos[@]:1}; do
if [[ $test_from == *@gnu.org && $t == *@gnu.org ]]; then
continue
fi
test_to+=", $t"
done
case $test_from in
testignore@expertpathologyreview.com)
test_to=testignore@zroe.org
;;
esac
cat >>/usr/local/bin/send-test-forward <