X-Git-Url: https://iankelling.org/git/?a=blobdiff_plain;f=mail-setup;h=9ada9b6c5060c54ba2e8d1e3ca1df52c642634e9;hb=858993fb6c3e9351988b193e6c296e6ea7862501;hp=cfe5575d2090ebfcc120f39575c0aa40e5d8060c;hpb=65d1dda5c1e79aa9c85a2d16bdd1a12413f163af;p=distro-setup
diff --git a/mail-setup b/mail-setup
index cfe5575..9ada9b6 100755
--- a/mail-setup
+++ b/mail-setup
@@ -1,15 +1,60 @@
#!/bin/bash
# * intro
-# Copyright (C) 2019 Ian Kelling
-# SPDX-License-Identifier: AGPL-3.0-or-later
+
+# Program to install and configure Ian's email related programs
+# Copyright (C) 2024 Ian Kelling
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# todo:
+# on bk (and fsf servers that run multiple exim4 daemons, eg eximfsf2 and eximfsf3),
+# make it so that when exim is restarted due to package upgrades,
+# we also restart those daemons, which can be done like so, based on looking
+# at the prerm and postinst scripts of exim4-daemon-heavy.
+#
+# if [[ ! -e /usr/sbin/invoke-rc.d-diverted ]]; then
+# mv /usr/sbin/invoke-rc.d /usr/sbin/invoke-rc.d-diverted
+# dpkg --divert /usr/sbin/invoke-rc.d-diverted --no-rename /usr/sbin/invoke-rc.d
+# fi
+# /usr/sbin/invoke-rc.d:
+# #!/bin/bash
+# if [[ DPKG_MAINTSCRIPT_PACKAGE == exim4* && $1 == exim4 ]]; then
+# shift
+# ret=0
+# for daemon in exim4 eximfsf2 eximfsf3; do
+# /usr/sbin/invoke-rc.d-diverted $daemon "$@" || ret=$?
+# done
+# else
+# /usr/sbin/invoke-rc.d-diverted "$@"
+# fi
# Things I tend to forget. on MAIL_HOST, daemon runs with /etc/exim4/my.conf,
-# which gets generated from update-exim4.conf -d /etc/myexim4
+# due to /etc/default/exim4 containing:
+# COMMONOPTIONS='-C /etc/exim4/my.conf'
+# UPEX4OPTS='-o /etc/exim4/my.conf'
+#
+# The non-daemon config
+# gets generated from this script calling update-exim4.conf -d /etc/myexim4
# which has log path
# log_file_path = /var/log/exim4/my%s
#
-# so, normal mainlog is non-daemon, which delivers to the daemon.
-
+# On non bk|MAIL_HOST, the config and log file are all standard.
+#
+# eximbackup folder is /bu/md
+# it is cleaned up by mail-backup-clean, which is run by btrbk-run
# shellcheck disable=SC2254 # makes for a lot of unneeded quotes
@@ -19,6 +64,14 @@
#
#&! testignore|jtuttle|eximbackup|/usr/sbin/exim4 -bpu
+# todo: this message seems to get dropped on the floor, it was due to a missing 2nd colon in
+# condition = ${if def:h_fdate:}
+# Figure out how to avoid this message being discarded.
+
+# 2023-09-12 01:41:43 [722371] 1qfw9f-0031v9-0S <= ian@iankelling.org U=iank P=local S=483 id=87cyyogd7t.fsf@iankelling.org T="iank2" from for testignore@amnimal.ninja
+# 2023-09-12 01:41:43 [722373] 1qfw9f-0031v9-0S H=nn.b8.nz [10.173.8.2]: SMTP error from remote mail server after pipelined end of data: 451 Temporary local problem - please try later
+# 2023-09-12 01:41:43 [722372] 1qfw9f-0031v9-0S == testignore@amnimal.ninja R=smarthost T=remote_smtp_smarthost defer (-46) H=nn.b8.nz [10.173.8.2] DT=0s: SMTP error from remote mail server after pipelined end of data: 451 Temporary local problem - please try later
+
# todo: check new macro DKIM_TIMESTAMPS
# todo: check if REMOTE_SMTP_INTERFACE or REMOTE_SMTP_TRANSPORTS_HEADERS_REMOVE can simplify my or fsfs config
@@ -146,10 +199,10 @@ if [ -z "$BASH_VERSION" ]; then echo "error: shell is not bash" >&2; exit 1; fi
shopt -s nullglob
-if [[ -s /usr/local/lib/err ]]; then
- source /usr/local/lib/err
-elif [[ -s /a/bin/errhandle/err ]]; then
- source /a/bin/errhandle/err
+if [[ -s /usr/local/lib/bash-bear ]]; then
+ source /usr/local/lib/bash-bear
+elif [[ -s /a/bin/bash-bear-trap/bash-bear ]]; then
+ source /a/bin/bash-bear-trap/bash-bear
else
echo "no err tracing script found"
exit 1
@@ -203,8 +256,8 @@ fi
#######
-# * perstent password instructions
-# Note: for cert cron, we need to manually run first to accept known_hosts
+# * perstent password instructions Note: for cert cron, we need to
+# manually run first to accept known_hosts
# # exim passwords:
# # for hosts which have all private files I just use the same user
@@ -298,7 +351,7 @@ fi
# * functions & constants
-pre="${0##*/}:"
+pre="${0##*/}:${SSH_CLIENT:+ $HOSTNAME:}"
m() { printf "$pre %s\n" "$*"; "$@"; }
e() { printf "$pre %s\n" "$*"; }
err() { printf "$pre %s\n" "$*" >&2; exit 1; }
@@ -401,7 +454,7 @@ fi
bhost_t=false
case $HOSTNAME in
$MAIL_HOST) : ;;
- kd|frodo|x2|x3|kw|sy|bo)
+ kd|x2|x3|kw|sy|bo|so)
bhost_t=true
;;
esac
@@ -489,6 +542,9 @@ EOF
# * clamav
+# old file. remove when all hosts updated, 2023-09-11
+rm -fv /etc/exim4/conf.d/clamav_data_acl
+
m usermod -a -G Debian-exim clamav
u /etc/systemd/system/clamav-daemon.service.d/fix.conf </etc/exim4/conf.d/main/30_local <: Fatal: master: service(lmtp): child 3839880 returned error 83 (Out of memory (service lmtp { vsz_limit=256 MB }, you may need to increase it) - set CORE_OUTOFMEM=1 environment to get core dump)
+# exim would just queue mail until it eventually succeeded.
+# Deciding what to increase it to, I found this
+# https://dovecot.org/list/dovecot/2011-December/080056.html
+# which suggests 3x the largest dovecot.index.cache file
+# and then I found that
+# md/l/testignore/dovecot.index.cache is 429M, my largest cache file,
+# but that folder only has 2k messages.
+# next biggest is md/l/qemu-devel/dovecot.index.cache 236M
+# which lead to me a search https://doc.dovecot.org/admin_manual/known_issues/large_cache/
+# which suggests 1.5x the maximum cache file size 1G, and
+# that I can safely rm the index.
+default_vsz_limit = 1500M
+
EOF
if dpkg --compare-versions "$(dpkg-query -f='${Version}\n' --show dovecot-core)" ge 1:2.3; then
cat <>/etc/dovecot/local.conf <&2
- # rm -fv composer-setup.php
- # exit 1
- # fi
- # m php composer-setup.php --quiet
- # rm -fv composer-setup.php
- # m mv composer.phar /usr/local/bin
-
- # the above method gets composer2, carddav plugin at least doesnt work with that
- # yet, it was just released 10-24-2020.
- m cd /usr/local/bin
- m wget -nv -N https://getcomposer.org/composer-1.phar
- chmod +x composer-1.phar
+ cd /usr/local/bin
+ EXPECTED_CHECKSUM="$(php -r 'copy("https://composer.github.io/installer.sig", "php://stdout");')"
+ php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
+ ACTUAL_CHECKSUM="$(php -r "echo hash_file('sha384', 'composer-setup.php');")"
+
+ if [ "$EXPECTED_CHECKSUM" != "$ACTUAL_CHECKSUM" ]
+ then
+ >&2 echo 'ERROR: Invalid installer checksum'
+ rm composer-setup.php
+ exit 1
+ fi
+
+ php composer-setup.php --quiet
+ rm composer-setup.php
+
+ # based on error when running composer
+ mkdir -p /var/www/.composer
+ chown www-data:www-data /var/www/.composer
+
### end composer install
rcdirs=(/usr/local/lib/rcexpertpath /usr/local/lib/rcninja)
- ncdirs=(/var/www/ncninja)
ncdirs=(/var/www/ncexpertpath /var/www/ncninja)
# point debian cronjob to our local install, preventing daily cron error
@@ -2263,8 +2429,6 @@ if [[ $HOSTNAME == bk ]]; then
rcdir=${rcdirs[i]}
rcbase=${rcdir##*/}
ncdir=${ncdirs[i]}
- myncdir=/root/${ncdir##*/}
- mkdir -p $myncdir
# copied from debians cronjob
u /etc/cron.d/$rcbase <$verf
@@ -2531,7 +2695,9 @@ EOF
for ((i=0; i < ${#bkdomains[@]}; i++)); do
domain=${bkdomains[i]}
ncdir=${ncdirs[i]}
+ myncdir=/var/local/${ncdir##*/}
ncbase=${ncdir##*/}
+ mkdir -p $myncdir
m cd /var/www
if [[ ! -e $ncdir/index.php ]]; then
# if we wanted to only install a specific version, use something like
@@ -2551,13 +2717,16 @@ EOF
m touch $myncdir/done-install
fi
- # note, strange this happend where updater did not increment the version var,
- # mine was stuck on 20. I manually updated it.
m cd $ncdir/config
- if [[ ! -e $myncdir/config.php-orig ]]; then
- m cp -a config.php $myncdir/config.php-orig
- fi
- cat $myncdir/config.php-orig - >$myncdir/tmp.php <$myncdir/tmp.php <config.php
+ e running php $myncdir/tmp.php
+ # note: we leave it around place for debugging
+ # shellcheck disable=SC2024 # intended
+ sudo -u www-data php $myncdir/tmp.php >config.php
+ fi
cd $ncdir
m sudo -u www-data php occ maintenance:update:htaccess
list=$(sudo -u www-data php $ncdir/occ --output=json_pretty app:list)
@@ -2628,7 +2799,7 @@ EOF
u /usr/local/bin/ncup <<'EOFOUTER'
#!/bin/bash
-source /usr/local/lib/err
+source /usr/local/lib/bash-bear
m() { printf "%s\n" "$*"; "$@"; }
err-cleanup() {
@@ -2657,7 +2828,13 @@ fi
ncbase=$1
cd /var/www/$ncbase
# https://docs.nextcloud.com/server/22/admin_manual/maintenance/update.html?highlight=updater+phar
+# the docs claim this is all you need, which is not true.
+# You will go to the web ui and it will say that you need to click a button to update,
+# or that you can run occ upgrade
m php /var/www/$ncbase/updater/updater.phar -n
+# throw a sleep in just because who knows what else is undocumented
+sleep 5
+m php occ upgrade
EOFOUTER
chmod +x /usr/local/bin/ncup
@@ -2674,6 +2851,67 @@ EOF
fi
+# * debbugs
+
+pi debbugs
+# missing dependency. apache error log:
+# Can't locate List/AllUtils.pm in @INC (you may need to install the List::AllUtils module) (@INC contains: /etc/perl /usr/local/lib/x86_64-linux-gnu/perl/5.34.0 /usr/local/share/perl/5.34.0 /usr/lib/x86_64-linux-gnu/perl5/5.34 /usr/share/perl5 /usr/lib/x86_64-linux-gnu/perl-base /usr/lib/x86_64-linux-gnu/perl/5.34 /usr/share/perl/5.34 /usr/local/lib/site_perl) at /var/lib/debbugs/www/cgi/pkgreport.cgi line 23.
+pi liblist-allutils-perl lynx
+# workarounds for broken debbugsconfig which is
+# itself deprecated. this is temporary before I
+# figure out how to install from git
+if [[ -e /usr/share/doc/debbugs/examples/text.gz ]]; then
+ gunzip /usr/share/doc/debbugs/examples/text.gz
+fi
+mkdir -p /etc/debbugs/indices
+debbugsconfig
+
+
+# ld for local debbugs
+/a/exe/web-conf -l -t -a 127.0.1.1 -p 80 -r /var/lib/debbugs/www - apache2 ld <<'EOF'
+# copied from debbugs upstream example
+
+ Options Indexes SymLinksIfOwnerMatch MultiViews
+ DirectoryIndex index.html
+ Require all granted
+
+
+ScriptAlias /cgi/ /var/lib/debbugs/www/cgi/
+
+ AllowOverride None
+ Options ExecCGI SymLinksIfOwnerMatch
+ Require all granted
+
+
+RewriteEngine on
+RewriteCond %{HTTP_USER_AGENT} .*apt-listbugs.*
+RewriteRule .* /apt-listbugs.html [R,L]
+
+# RewriteLog /org/bugs.debian.org/apache-rewrite.log
+# RewriteLogLevel 0
+
+#RewriteRule ^/$ http://www.debian.org/Bugs/
+RewriteRule ^/(robots\.txt|release-critical|apt-listbugs\.html)$ - [L]
+# The following two redirect to up-to-date pages
+RewriteRule ^/[[:space:]]*#?([[:digit:]][[:digit:]][[:digit:]]+)([;&].+)?$ /cgi-bin/bugreport.cgi?bug=$1$2 [L,R,NE]
+RewriteRule ^/([^/+]*)([+])([^/]*)$ "/$1%%{%}2B$3" [N]
+RewriteRule ^/[Ff][Rr][Oo][Mm]:([^/]+\@.+)$ /cgi-bin/pkgreport.cgi?submitter=$1 [PT,NE]
+# Commented out, 'cuz aj says it will crash master. (old master)
+# RewriteRule ^/[Ss][Ee][Vv][Ee][Rr][Ii][Tt][Yy]:([^/]+\@.+)$ /cgi-bin/pkgreport.cgi?severity=$1 [L,R]
+RewriteRule ^/([^/]+\@.+)$ /cgi-bin/pkgreport.cgi?maint=$1 [PT,NE]
+RewriteRule ^/mbox:([[:digit:]][[:digit:]][[:digit:]]+)([;&].+)?$ /cgi-bin/bugreport.cgi?mbox=yes&bug=$1$2 [PT,NE]
+RewriteRule ^/src:([^/]+)$ /cgi-bin/pkgreport.cgi?src=$1 [PT,NE]
+RewriteRule ^/severity:([^/]+)$ /cgi-bin/pkgreport.cgi?severity=$1 [PT,NE]
+RewriteRule ^/tag:([^/]+)$ /cgi-bin/pkgreport.cgi?tag=$1 [PT,NE]
+# RewriteMap fix-chars int:noescape
+RewriteCond %{REQUEST_URI} ^/(Access\.html|Developer\.html|Reporting\.html|server-request\.html|server-control\.html|server-refcard\.html).* [NC]
+RewriteRule .* - [L]
+# PT|passthrough to bugreport.cgi and pkgreport.cgi
+RewriteRule ^/([0-9]+)$ /cgi-bin/bugreport.cgi?bug=$1 [PT,NE]
+RewriteRule ^/([^/]+)$ /cgi-bin/pkgreport.cgi?pkg=$1 [PT,NE]
+EOF
+
+
# * exim host conditional config
# ** exim certs
@@ -2743,7 +2981,8 @@ case $HOSTNAME in
# which will overwrite any existing file
u /etc/default/exim4 <<'EOF'
QUEUERUNNER='combined'
-QUEUEINTERVAL='30m'
+# note: this is duplicated in brc2, 10m here is -q10m there.
+QUEUEINTERVAL='10m'
COMMONOPTIONS='-C /etc/exim4/my.conf'
UPEX4OPTS='-o /etc/exim4/my.conf'
# i use epanic-clean for alerting if there are bad paniclog entries
@@ -2860,29 +3099,10 @@ EOF
# ** $MAIL_HOST|bk)
$MAIL_HOST|bk)
-
- # no clamav on je, it has 1.5g memory and clamav uses most of it
- u /etc/exim4/conf.d/clamav_data_acl <<'EOF'
-warn
-!hosts = +iank_trusted
-!authenticated = plain_server:login_server
-condition = ${if def:malware_name}
-remove_header = Subject:
-add_header = Subject: [Clamav warning: $malware_name] $h_subject
-log_message = heuristic malware warning: $malware_name
-EOF
-
- cat >>/etc/exim4/conf.d/main/000_local <>/etc/exim4/update-exim4.conf.conf <> /etc/exim4/conf.d/data_local_acl <<'EOF'
-deny
- malware = */defer_ok
- !condition = ${if match {$malware_name}{\N^Heuristic\N}}
- message = This message was detected as possible malware ($malware_name).
-EOF
cat >/etc/exim4/conf.d/main/000_local-nn < /etc/mailname
+ # mail default domain.
+ u /etc/mailutils.conf <<'EOF'
+address {
+ email-domain iankelling.org;
+};
+EOF
# mail.iankelling.org so local imap clients can connect with tls and
# when they happen to not be local.
@@ -3085,7 +3385,6 @@ EOF
# man page: is used to build the local_domains list, together with "localhost"
# this is duplicated in a later router.
dc_other_hostnames='iankelling.org;zroe.org;r2e.iankelling.org;mx.iankelling.org;!je.b8.nz;!bk.b8.nz;*.b8.nz;b8.nz'
-dc_relay_nets='defaultnn.b8.nz'
EOF
@@ -3103,6 +3402,39 @@ EOF
## we use this host to monitor MAIL_HOST and host a mail server for someone
bk)
+ # No clamav on je, it has 1.5g memory and clamav uses most of it.
+ #
+ # No clamav on MAIL_HOST because it is just a waste of useful cpu
+ # time and memory when I'm running on an x200, and it takes 30
+ # seconds to shut down.
+
+ cat >>/etc/exim4/conf.d/main/000_local <> /etc/exim4/conf.d/data_local_acl <<'EOF'
+deny
+ malware = */defer_ok
+ !condition = ${if match {$malware_name}{\N^Heuristic\N}}
+ message = This message was detected as possible malware ($malware_name).
+
+warn
+ !hosts = +iank_trusted
+ !authenticated = *
+ condition = ${if def:malware_name}
+ remove_header = Subject:
+ add_header = Subject: [Clamav warning: $malware_name] $h_subject
+ log_message = heuristic malware warning: $malware_name
+
+warn
+ # fdate = future date. # tdate = temporary date.
+ condition = ${if def:h_fdate}
+ remove_header = fdate:
+ add_header = tdate:
+ control = freeze
+EOF
+
/a/exe/cedit nn /etc/hosts <<'EOF' || [[ $? == 1 ]]
10.173.8.2 nn.b8.nz
@@ -3149,18 +3481,27 @@ EOF
;;
# ** not MAIL_HOST|bk|je
*)
- # this one should be removed for all non mail hosts, but
+ echo|u /etc/exim4/conf.d/transport/30_debbugs
+ echo|u /etc/exim4/conf.d/router/153_debbugs
+ echo|u /etc/exim4/conf.d/router/155_delay
+ # this one should be removed for all non mail_hosts. note
# bk and je never become mail_host
echo|u /etc/exim4/conf.d/router/195_dnslookup_vpn
echo|u /etc/exim4/conf.d/router/160_backup_redir
echo|u /etc/exim4/conf.d/router/161_backup_redir_nn
echo|u /etc/exim4/conf.d/router/185_sentarchive
echo|u /etc/exim4/conf.d/router/186_sentarchive_nn
+ # Note, in general we could submit to smarthosts on non MAIL_HOST.
+ # however, delayed mail makes this inconvenient, because I
+ # occasionally want to send an email from a non-MAIL_HOST and then
+ # turn off that computer or travel with it so it is disconnected.
+ # It is also probably easier to setup emacs to delay messages, but
+ # that would mean we need to keep emacs running, this is much
+ # nicer.
echo|u /etc/exim4/conf.d/router/188_exim4-config_smarthost
echo|u /etc/exim4/conf.d/router/190_exim4-config_fsfsmarthost
echo|u /etc/exim4/conf.d/rcpt_local_acl
echo|u /etc/exim4/conf.d/main/000_local-nn
- echo|u /etc/exim4/conf.d/clamav_data_acl
if $bhost_t; then
@@ -3227,11 +3568,13 @@ backup_local:
EOF
# Bind to wghole to receive mailbackup.
- wgholeip=$(sed -rn 's/^ *Address *= *([^/]+).*/\1/p' /etc/wireguard/wghole.conf)
- cat >>/etc/exim4/update-exim4.conf.conf <>/etc/exim4/update-exim4.conf.conf </dev/null; then
m systemctl --now enable clamav-daemon
out=$(rsync -aiSAX --chown=root:root --chmod=g-s /a/bin/ds/filesystem/etc/systemd/system/epanicclean.service /etc/systemd/system)
@@ -3643,6 +4006,9 @@ EOF
test_to=${test_tos[0]}
for t in ${test_tos[@]:1}; do
+ if [[ $test_from == *@gnu.org && $t == *@gnu.org ]]; then
+ continue
+ fi
test_to+=", $t"
done
case $test_from in