X-Git-Url: https://iankelling.org/git/?a=blobdiff_plain;ds=sidebyside;f=mail-setup;h=9ada9b6c5060c54ba2e8d1e3ca1df52c642634e9;hb=HEAD;hp=0ec838413a7f00e806e6f7fbb49fbc7d9e5b5252;hpb=ca2912304f8d0a727756763218c9c7f89768c76d;p=distro-setup diff --git a/mail-setup b/mail-setup index 0ec8384..616d39b 100755 --- a/mail-setup +++ b/mail-setup @@ -1,7 +1,76 @@ #!/bin/bash # * intro -# Copyright (C) 2019 Ian Kelling -# SPDX-License-Identifier: AGPL-3.0-or-later + +# Program to install and configure Ian's email related programs +# Copyright (C) 2024 Ian Kelling + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# SPDX-License-Identifier: GPL-3.0-or-later + +# todo: +# on bk (and fsf servers that run multiple exim4 daemons, eg eximfsf2 and eximfsf3), +# make it so that when exim is restarted due to package upgrades, +# we also restart those daemons, which can be done like so, based on looking +# at the prerm and postinst scripts of exim4-daemon-heavy. +# +# if [[ ! -e /usr/sbin/invoke-rc.d-diverted ]]; then +# mv /usr/sbin/invoke-rc.d /usr/sbin/invoke-rc.d-diverted +# dpkg --divert /usr/sbin/invoke-rc.d-diverted --no-rename /usr/sbin/invoke-rc.d +# fi +# /usr/sbin/invoke-rc.d: +# #!/bin/bash +# if [[ DPKG_MAINTSCRIPT_PACKAGE == exim4* && $1 == exim4 ]]; then +# shift +# ret=0 +# for daemon in exim4 eximfsf2 eximfsf3; do +# /usr/sbin/invoke-rc.d-diverted $daemon "$@" || ret=$? +# done +# else +# /usr/sbin/invoke-rc.d-diverted "$@" +# fi + +# Things I tend to forget. on MAIL_HOST, daemon runs with /etc/exim4/nn-mainlog.conf, +# due to /etc/default/exim4 containing: +# COMMONOPTIONS='-C /etc/exim4/nn-mainlog.conf' +# UPEX4OPTS='-o /etc/exim4/nn-mainlog.conf' +# +# The non-daemon config +# gets generated from this script calling update-exim4.conf -d /etc/nond-exim4 +# which has log path +# log_file_path = /var/log/exim4/nond%s +# +# On non bk|MAIL_HOST, the config and log file are all standard. +# +# eximbackup folder is /bu/md +# it is cleaned up by mail-backup-clean, which is run by btrbk-run + +# shellcheck disable=SC2254 # makes for a lot of unneeded quotes + + +# perusing through /el/mainlog without test messages: +# &!testignore|jtuttle| +# +#&! testignore|jtuttle|eximbackup|/usr/sbin/exim4 -bpu + +# todo: this message seems to get dropped on the floor, it was due to a missing 2nd colon in +# condition = ${if def:h_fdate:} +# Figure out how to avoid this message being discarded. + +# 2023-09-12 01:41:43 [722371] 1qfw9f-0031v9-0S <= ian@iankelling.org U=iank P=local S=483 id=87cyyogd7t.fsf@iankelling.org T="iank2" from for testignore@amnimal.ninja +# 2023-09-12 01:41:43 [722373] 1qfw9f-0031v9-0S H=nn.b8.nz [10.173.8.2]: SMTP error from remote mail server after pipelined end of data: 451 Temporary local problem - please try later +# 2023-09-12 01:41:43 [722372] 1qfw9f-0031v9-0S == testignore@amnimal.ninja R=smarthost T=remote_smtp_smarthost defer (-46) H=nn.b8.nz [10.173.8.2] DT=0s: SMTP error from remote mail server after pipelined end of data: 451 Temporary local problem - please try later # todo: check new macro DKIM_TIMESTAMPS @@ -38,7 +107,7 @@ # todo: run mailping test after running, or otherwise # clear out terminal alert -# todo: disable postgrey +# todo: disable postgrey. (why did we have it?) # todo: in testforward-check, we should also look @@ -130,10 +199,10 @@ if [ -z "$BASH_VERSION" ]; then echo "error: shell is not bash" >&2; exit 1; fi shopt -s nullglob -if [[ -s /usr/local/lib/err ]]; then - source /usr/local/lib/err -elif [[ -s /a/bin/errhandle/err ]]; then - source /a/bin/errhandle/err +if [[ -s /usr/local/lib/bash-bear ]]; then + source /usr/local/lib/bash-bear +elif [[ -s /a/bin/bash-bear-trap/bash-bear ]]; then + source /a/bin/bash-bear-trap/bash-bear else echo "no err tracing script found" exit 1 @@ -187,8 +256,8 @@ fi ####### -# * perstent password instructions -# Note: for cert cron, we need to manually run first to accept known_hosts +# * perstent password instructions Note: for cert cron, we need to +# manually run first to accept known_hosts # # exim passwords: # # for hosts which have all private files I just use the same user @@ -282,7 +351,7 @@ fi # * functions & constants -pre="${0##*/}:" +pre="${0##*/}:${SSH_CLIENT:+ $HOSTNAME:}" m() { printf "$pre %s\n" "$*"; "$@"; } e() { printf "$pre %s\n" "$*"; } err() { printf "$pre %s\n" "$*" >&2; exit 1; } @@ -292,28 +361,8 @@ reload=false if [[ -e /var/local/mail-setup-reload ]]; then reload=true fi -i() { # install file - local tmp tmpdir dest="$1" - local base="${dest##*/}" - local dir="${dest%/*}" - if [[ $dir != "$base" ]]; then - # dest has a directory component - mkdir -p "$dir" - fi - ir=false # i result - tmpdir=$(mktemp -d) - cat >$tmpdir/"$base" - tmp=$(rsync -ic $tmpdir/"$base" "$dest") - if [[ $tmp ]]; then - printf "%s\n" "$tmp" - ir=true - if [[ $dest == /etc/systemd/system/* ]]; then - touch /var/local/mail-setup-reload - reload=true - fi - fi - rm -rf $tmpdir -} + +source /a/bin/fai/fai/config/distro-install-common/bash-misc-funcs setini() { key="$1" value="$2" section="$3" file="/etc/radicale/config" @@ -328,15 +377,24 @@ soff () { done } sre() { + local enabled for service; do m systemctl restart $service - m systemctl enable $service; + # Optimization for exim, + # is-enabled: 0m0.015s + # enable: 0m0.748s + # It is related to this message: + # exim4.service is not a native service, redirecting to systemd-sysv-install. + # Executing: /lib/systemd/systemd-sysv-install enable exim4 + enabled=$(systemctl is-enabled $service 2>/dev/null ||:) + if [[ $enabled != enabled ]]; then + m systemctl enable $service + fi done } mailhost() { [[ $HOSTNAME == "$MAIL_HOST" ]] } -e() { printf "%s\n" "$*"; } reifactive() { for service; do if systemctl is-active $service >/dev/null; then @@ -375,7 +433,7 @@ fi bhost_t=false case $HOSTNAME in $MAIL_HOST) : ;; - kd|frodo|x2|x3|kw|sy|bo) + kd|x2|x3|kw|sy|bo|so) bhost_t=true ;; esac @@ -395,13 +453,42 @@ Pin-Priority: 500 EOF fi + +unit-exists() { + systemctl cat $1 &>/dev/null +} +spamd-timer-exists() { + unit-exists spamassassin-maintenance.timer +} + +# name change in t12, and now timer instead of cron option in /etc/default +first_spamd_run=false +if ! spamd-timer-exists; then + first_spamd_run=true +fi + + # light version of exim does not have sasl auth support. # note: for bitfolk hosts, unbound has important config with conflink. pi-nostart exim4 exim4-daemon-heavy spamassassin unbound clamav-daemon wireguard +spamd_ser=spamd +if systemctl cat spamassassin &>/dev/null; then + spamd_ser=spamassassin +elif $first_spamd_run; then + if spamd-timer-exists; then + systemctl start spamassassin-maintenance + fi +fi + +if spamd-timer-exists; then + systemctl enable --now spamassassin-maintenance.timer +fi + # note: pyzor debian readme says you need to run some initialization command # but its outdated. -pi spf-tools-perl p0f postgrey pyzor razor jq moreutils certbot fail2ban +pi spf-tools-perl p0f pyzor razor jq moreutils certbot fail2ban +pu postgrey case $HOSTNAME in je) : ;; # not included due to using wireguard: openvpn @@ -430,9 +517,80 @@ fi # our nostart pi fails to avoid enabling + +# * initial dns config & daemon setup +# +# use systemd-resolved for glibc resolutions, setup symlinks + +pi libnss-resolve + +# if this link gets replaced with a normal file we will get exim log +# errors on MAIL_HOST like so: +# +# R=fsfsmarthost defer (-36) DT=0s: host lookup for mail.fsf.org did not complete (DNS timeout?) + +if [[ ! -L /etc/nsswitch.conf ]]; then + sudo mkdir -p /etc/resolved-nsswitch + sudo mv /etc/nsswitch.conf /etc/resolved-nsswitch + sudo ln -sf /etc/resolved-nsswitch/nsswitch.conf /etc +fi + +f=/etc/basic-nsswitch/nsswitch.conf +if [[ ! -e $f ]]; then + sudo mkdir -p ${f%/*} + sudo cp /etc/nsswitch.conf $f + sudo sed -i --follow-symlinks 's/^ *hosts:.*/hosts: files dns myhostname/' $f +fi +case $HOSTNAME in + bk|je) + # je should be able to get along systemd-resolved, but ive had some odd + # very intermittent dns failures with spamassassin, it seems it might only + # be happening with systemd-resolved, so just use unbound + # to make it consistent with the other hosts. + sudo sed -i --follow-symlinks 's/^ *hosts:.*/hosts: files dns myhostname/' /etc/nsswitch.conf + soff systemd-resolved + sudo ln -sf 127.0.0.1-resolv/stub-resolv.conf /etc/resolv.conf + sgo unbound + # cautious measure to make sure resolution is working + sleep 1 + ;; + *) + # default is + # files mdns4_minimal [NOTFOUND=return] dns myhostname + # mdns4 is needed for my printer and for bbb webrtc, not sure exactly why. + # https://www.freedesktop.org/software/systemd/man/nss-resolve.html# + # seems more important than some potential use case. + # Interestingly, t9/t10 man page says use files before resolve, debian 10 says the opposite. + # removing files makes hostname -f not actually give the fully qualified domain name. + sudo sed -i --follow-symlinks 's/^ *hosts:.*/hosts: files resolve [!UNAVAIL=return] mdns4_minimal [NOTFOUND=return] myhostname/' /etc/resolved-nsswitch/nsswitch.conf + ;; +esac + +case $HOSTNAME in + bk) + sgo named + ;; +esac + + +u /etc/apparmor.d/abstractions/nameservice.d/iank <<'EOF' +/etc/resolved-nsswitch/nsswitch.conf r, +/etc/basic-nsswitch/nsswitch.conf r, +# Aug 06 23:09:11 kd audit[3995]: AVC apparmor="DENIED" operation="connect" profile="/usr/bin/freshclam" name="/run/systemd/resolve/io.systemd.Resolve" pid=3995 comm="freshclam" requested_mask="wr" denied_mask="wr" fsuid=109 ouid=101 +# I dont know if this is quite the right fix, but I saw other sockets +# in the nameservice files that were rw, so figured it was ok to add this and it worked. +/run/systemd/resolve/io.systemd.Resolve rw, +EOF + +if $ur && systemctl is-active apparmor; then + m systemctl reload apparmor +fi + + + # * Mail clean cronjob -i /etc/systemd/system/mailclean.timer <<'EOF' +u /etc/systemd/system/mailclean.timer <<'EOF' [Unit] Description=Run mailclean daily @@ -443,7 +601,7 @@ OnCalendar=monthly WantedBy=timers.target EOF -i /etc/systemd/system/mailclean.service </dev/null; then m systemctl reload apparmor fi fi + # note: anything added to nn_progs needs corresponding rm # down below in the host switch nn_progs=(exim4) if mailhost; then # Note dovecots lmtp doesnt need to be in the same nn to accept delivery. # Its in the nn so remote clients can connect to it. - nn_progs+=(spamassassin dovecot) + nn_progs+=($spamd_ser dovecot) fi case $HOSTNAME in $MAIL_HOST) # todo, should this be after vpn service - i /etc/systemd/system/unbound.service.d/nn.conf < EOF -i /etc/fail2ban/jail.d/exim.local <<'EOF' +u /etc/fail2ban/jail.d/exim.local <<'EOF' [exim] enabled = true port = 25,587 @@ -1030,7 +1224,11 @@ banaction = iptables-exim # 10.173.8.1 = non-nn net ignoreip = 209.51.188.13 2001:470:142::13 209.51.188.92 2001:470:142:3::10 72.14.176.105 2600:3c00:e000:280::2 10.173.8.1 EOF -if $ir; then +if $ur; then + # Ensure the log file monitored by fail2ban exists, or else fail2ban can't start. + if [[ ! -e /var/log/exim4/mainlog ]]; then + install -m 640 -o Debian-exim -g adm /dev/null /var/log/exim4/mainlog + fi m systemctl restart fail2ban fi @@ -1136,26 +1334,26 @@ rm -fv /etc/exim4/conf.d/retry/37_retry cat >/etc/exim4/conf.d/retry/17_retry <<'EOF' # Retry fast for my own domains -iankelling.org * F,1d,4m;F,14d,1h -amnimal.ninja * F,1d,4m;F,14d,1h -expertpathologyreview.com * F,1d,4m;F,14d,1h -je.b8.nz * F,1d,4m;F,14d,1h -zroe.org * F,1d,4m;F,14d,1h -eximbackup.b8.nz * F,1d,4m;F,14d,1h +iankelling.org * F,1d,1m;F,14d,1h +amnimal.ninja * F,1d,1m;F,14d,1h +expertpathologyreview.com * F,1d,1m;F,14d,1h +je.b8.nz * F,1d,1m;F,14d,1h +zroe.org * F,1d,1m;F,14d,1h +eximbackup.b8.nz * F,1d,1m;F,14d,1h # The spec says the target domain will be used for temporary host errors, # but i've found that isn't correct, the hostname is required # at least sometimes. -nn.b8.nz * F,1d,4m;F,14d,1h -defaultnn.b8.nz * F,1d,4m;F,14d,1h -mx.iankelling.org * F,1d,4m;F,14d,1h -bk.b8.nz * F,1d,4m;F,14d,1h -eggs.gnu.org * F,1d,4m;F,14d,1h -fencepost.gnu.org * F,1d,4m;F,14d,1h +nn.b8.nz * F,1d,1m;F,14d,1h +defaultnn.b8.nz * F,1d,1m;F,14d,1h +mx.iankelling.org * F,1d,1m;F,14d,1h +bk.b8.nz * F,1d,1m;F,14d,1h +eggs.gnu.org * F,1d,1m;F,14d,1h +fencepost.gnu.org * F,1d,1m;F,14d,1h # afaik our retry doesnt need this, but just using everything -mx.amnimal.ninja * F,1d,4m;F,14d,1h -mx.expertpathologyreview.com * F,1d,4m;F,14d,1h +mx.amnimal.ninja * F,1d,1m;F,14d,1h +mx.expertpathologyreview.com * F,1d,1m;F,14d,1h mail.fsf.org * F,1d,15m;F,14d,1h @@ -1177,27 +1375,16 @@ cd /etc/exim4 for f in *-private.pem; do echo ${f%-private.pem} done -} | i /etc/exim4/conf.d/my-dkim-domains - -if grep -Fq REMOTE_SMTP_SMARTHOST_TLS_VERIFY_HOSTS \ - /etc/exim4/conf.d/transport/10_exim4-config_transport-macros; then - cat >/etc/exim4/conf.d/transport/11_iank <<'EOF' -# This unsets the default macro defined in on t11 in -# /etc/exim4/conf.d/transport/10_exim4-config_transport-macros -# It seems like a very odd choice that this has become -# the default in t11. Normal smarthost clients use username/password -# auth. Oh well. -REMOTE_SMTP_SMARTHOST_TLS_VERIFY_HOSTS == -EOF -else - rm -f /etc/exim4/conf.d/transport/11_iank -fi +} | u /etc/exim4/conf.d/my-dkim-domains + +rm -f /etc/exim4/conf.d/transport/11_iank cat >/etc/exim4/conf.d/main/000_local <<'EOF' MAIN_TLS_ENABLE = true # require tls connections for all smarthosts -REMOTE_SMTP_SMARTHOST_HOSTS_REQUIRE_TLS = * +REMOTE_SMTP_SMARTHOST_HOSTS_REQUIRE_TLS = ! nn.b8.nz +REMOTE_SMTP_SMARTHOST_HOSTS_AVOID_TLS = nn.b8.nz # debian exim config added this in 2016 or so? # it's part of the smtp spec, to limit lines to 998 chars @@ -1209,8 +1396,8 @@ REMOTE_SMTP_SMARTHOST_HOSTS_REQUIRE_TLS = * # other says gmail does not reject. figure out and open a new bug. IGNORE_SMTP_LINE_LENGTH_LIMIT = true -# more verbose logs -MAIN_LOG_SELECTOR = +all +# more verbose logs. used to use +all, but made it less for more efficiency. +MAIN_LOG_SELECTOR = -skip_delivery -tls_cipher -tls_certificate_verified +all_parents +address_rewrite +arguments +deliver_time +pid +queue_time +queue_time_overall +received_recipients +received_sender +return_path_on_delivery +sender_on_delivery +smtp_confirmation +subject # Based on spec, seems like a good idea to be nice. smtp_return_error_details = true @@ -1241,6 +1428,7 @@ DKIM_SIGN_HEADERS = mime-version:in-reply-to:references:from:date:subject:to domainlist local_hostnames = ! je.b8.nz : ! bk.b8.nz : *.b8.nz : b8.nz +# note: most of these are duplicated in spamassassin config hostlist iank_trusted = <; \ # veth0 10.173.8.1 ; \ @@ -1269,7 +1457,7 @@ delay_warning_condition = ${if or {\ # enable 587 in addition to the default 25, so that # i can send mail where port 25 is firewalled by isp -daemon_smtp_ports = 25 : 587 +daemon_smtp_ports = 25 : 587 : 10025 # default of 25, can get stuck when catching up on mail smtp_accept_max = 400 smtp_accept_reserve = 100 @@ -1278,6 +1466,12 @@ smtp_reserve_hosts = +iank_trusted # Rules that make receiving more liberal should be on backup hosts # so that we dont reject mail accepted by MAIL_HOST LOCAL_DENY_EXCEPTIONS_LOCAL_ACL_FILE = /etc/exim4/conf.d/local_deny_exceptions_acl + +acl_not_smtp = acl_check_not_smtp + + +DEBBUGS_DOMAIN = b.b8.nz + EOF if dpkg --compare-versions "$(dpkg-query -f='${Version}\n' --show exim4)" ge 4.94; then @@ -1298,9 +1492,14 @@ DKIM_DOMAIN = ${lc:${domain:$rh_from:}} EOF fi +cat >/etc/exim4/conf.d/main/30_local <>/etc/dovecot/local.conf < @@ -2194,27 +2458,28 @@ if [[ $HOSTNAME == bk ]]; then ### begin composer install # https://getcomposer.org/doc/faqs/how-to-install-composer-programmatically.md - # cd $(mktemp -d) - # sum="$(wget -q -O - https://composer.github.io/installer.sig)" - # m php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" - # if [[ $sum != $(php -r "echo hash_file('sha384', 'composer-setup.php');") ]]; then - # echo 'ERROR: Invalid composer installer checksum' >&2 - # rm -fv composer-setup.php - # exit 1 - # fi - # m php composer-setup.php --quiet - # rm -fv composer-setup.php - # m mv composer.phar /usr/local/bin - - # the above method gets composer2, carddav plugin at least doesnt work with that - # yet, it was just released 10-24-2020. - m cd /usr/local/bin - m wget -nv -N https://getcomposer.org/composer-1.phar - chmod +x composer-1.phar + cd /usr/local/bin + EXPECTED_CHECKSUM="$(php -r 'copy("https://composer.github.io/installer.sig", "php://stdout");')" + php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" + ACTUAL_CHECKSUM="$(php -r "echo hash_file('sha384', 'composer-setup.php');")" + + if [ "$EXPECTED_CHECKSUM" != "$ACTUAL_CHECKSUM" ] + then + >&2 echo 'ERROR: Invalid installer checksum' + rm composer-setup.php + exit 1 + fi + + php composer-setup.php --quiet + rm composer-setup.php + + # based on error when running composer + mkdir -p /var/www/.composer + chown www-data:www-data /var/www/.composer + ### end composer install rcdirs=(/usr/local/lib/rcexpertpath /usr/local/lib/rcninja) - ncdirs=(/var/www/ncninja) ncdirs=(/var/www/ncexpertpath /var/www/ncninja) # point debian cronjob to our local install, preventing daily cron error @@ -2246,11 +2511,9 @@ if [[ $HOSTNAME == bk ]]; then rcdir=${rcdirs[i]} rcbase=${rcdir##*/} ncdir=${ncdirs[i]} - myncdir=/root/${ncdir##*/} - mkdir -p $myncdir # copied from debians cronjob - i /etc/cron.d/$rcbase </dev/null @@ -2310,7 +2573,7 @@ EOF rctmpdir=/var/tmp/$rcbase rcdb=/m/rc/$rcbase.sqlite # config from mailinabox - i $rcdir/config/config.inc.php <$verf @@ -2405,7 +2668,7 @@ EOF # About categories, see https://www.davx5.com/tested-with/nextcloud # https://github.com/blind-coder/rcmcarddav/blob/master/doc/GROUPS.md - i $rcdir/plugins/carddav/config.inc.php <$myncdir/tmp.php <$myncdir/tmp.php <config.php + e running php $myncdir/tmp.php + # note: we leave it around place for debugging + # shellcheck disable=SC2024 # intended + sudo -u www-data php $myncdir/tmp.php >config.php + fi cd $ncdir m sudo -u www-data php occ maintenance:update:htaccess list=$(sudo -u www-data php $ncdir/occ --output=json_pretty app:list) @@ -2585,7 +2855,7 @@ EOF m sudo -u www-data php occ app:install $app fi done - i /etc/systemd/system/$ncbase.service < + Options Indexes SymLinksIfOwnerMatch MultiViews + DirectoryIndex index.html + Require all granted + + +ScriptAlias /cgi/ /var/lib/debbugs/www/cgi/ + + AllowOverride None + Options ExecCGI SymLinksIfOwnerMatch + Require all granted + + +RewriteEngine on +RewriteCond %{HTTP_USER_AGENT} .*apt-listbugs.* +RewriteRule .* /apt-listbugs.html [R,L] + +# RewriteLog /org/bugs.debian.org/apache-rewrite.log +# RewriteLogLevel 0 + +#RewriteRule ^/$ http://www.debian.org/Bugs/ +RewriteRule ^/(robots\.txt|release-critical|apt-listbugs\.html)$ - [L] +# The following two redirect to up-to-date pages +RewriteRule ^/[[:space:]]*#?([[:digit:]][[:digit:]][[:digit:]]+)([;&].+)?$ /cgi-bin/bugreport.cgi?bug=$1$2 [L,R,NE] +RewriteRule ^/([^/+]*)([+])([^/]*)$ "/$1%%{%}2B$3" [N] +RewriteRule ^/[Ff][Rr][Oo][Mm]:([^/]+\@.+)$ /cgi-bin/pkgreport.cgi?submitter=$1 [PT,NE] +# Commented out, 'cuz aj says it will crash master. (old master) +# RewriteRule ^/[Ss][Ee][Vv][Ee][Rr][Ii][Tt][Yy]:([^/]+\@.+)$ /cgi-bin/pkgreport.cgi?severity=$1 [L,R] +RewriteRule ^/([^/]+\@.+)$ /cgi-bin/pkgreport.cgi?maint=$1 [PT,NE] +RewriteRule ^/mbox:([[:digit:]][[:digit:]][[:digit:]]+)([;&].+)?$ /cgi-bin/bugreport.cgi?mbox=yes&bug=$1$2 [PT,NE] +RewriteRule ^/src:([^/]+)$ /cgi-bin/pkgreport.cgi?src=$1 [PT,NE] +RewriteRule ^/severity:([^/]+)$ /cgi-bin/pkgreport.cgi?severity=$1 [PT,NE] +RewriteRule ^/tag:([^/]+)$ /cgi-bin/pkgreport.cgi?tag=$1 [PT,NE] +# RewriteMap fix-chars int:noescape +RewriteCond %{REQUEST_URI} ^/(Access\.html|Developer\.html|Reporting\.html|server-request\.html|server-control\.html|server-refcard\.html).* [NC] +RewriteRule .* - [L] +# PT|passthrough to bugreport.cgi and pkgreport.cgi +RewriteRule ^/([0-9]+)$ /cgi-bin/bugreport.cgi?bug=$1 [PT,NE] +RewriteRule ^/([^/]+)$ /cgi-bin/pkgreport.cgi?pkg=$1 [PT,NE] +EOF + + # * exim host conditional config # ** exim certs @@ -2694,7 +3031,7 @@ deny domains = +local_domains !verify = recipient/callout=no_cache EOF - i /etc/exim4/conf.d/auth/29_exim4-config_auth <<'EOF' + u /etc/exim4/conf.d/auth/29_exim4-config_auth <<'EOF' dovecot_plain: driver = dovecot public_name = PLAIN @@ -2704,7 +3041,7 @@ EOF ;; esac if $bhost_t; then - i /etc/exim4/conf.d/auth/29_exim4-config_auth <<'EOF' + u /etc/exim4/conf.d/auth/29_exim4-config_auth <<'EOF' # from 30_exim4-config_examples plain_server: driver = plaintext @@ -2724,11 +3061,13 @@ case $HOSTNAME in # to see the default comments in /etc/default/exim4: # s update-exim4defaults --force --init # which will overwrite any existing file - i /etc/default/exim4 <<'EOF' + u /etc/default/exim4 <<'EOF' QUEUERUNNER='combined' -QUEUEINTERVAL='30m' -COMMONOPTIONS='-C /etc/exim4/my.conf' -UPEX4OPTS='-o /etc/exim4/my.conf' +QUEUEINTERVAL='10m' +COMMONOPTIONS='-C /etc/exim4/nn-mainlog.conf' +UPEX4OPTS='-o /etc/exim4/nn-mainlog.conf' +# in t12 exim, this replaces all the above options +EXIMSERVICE='-bdf -q10m -C /etc/exim4/nn-mainlog.conf' # i use epanic-clean for alerting if there are bad paniclog entries E4BCD_WATCH_PANICLOG='no' EOF @@ -2738,18 +3077,18 @@ EOF chmod g+s,u+s /usr/sbin/exim4 # need this to avoid error on service reload: # 2022-08-07 18:44:34.005 [892491] pid 892491: SIGHUP received: re-exec daemon - # 2022-08-07 18:44:34.036 [892491] cwd=/var/spool/exim4 5 args: /usr/sbin/exim4 -bd -q30m -C /etc/exim4/my.conf + # 2022-08-07 18:44:34.036 [892491] cwd=/var/spool/exim4 5 args: /usr/sbin/exim4 -bd -q30m -C /etc/exim4/nn-mainlog.conf # 2022-08-07 18:44:34.043 [892491] socket bind() to port 25 for address (any IPv6) failed: Permission denied: waiting 30s before trying again (9 more tries) # note: the daemon gives up and dies after retrying those 9 times. # I came upon this by guessing and trial and error. setcap CAP_NET_BIND_SERVICE+ei /usr/sbin/exim4 - i /etc/exim4/trusted_configs <<'EOF' -/etc/exim4/my.conf + u /etc/exim4/trusted_configs <<'EOF' +/etc/exim4/nn-mainlog.conf EOF ;; *) # default file - i /etc/default/exim4 <<'EOF' + u /etc/default/exim4 <<'EOF' QUEUERUNNER='combined' QUEUEINTERVAL='30m' EOF @@ -2770,7 +3109,7 @@ case $HOSTNAME in dirs+=($d) fi done - i /etc/systemd/system/exim4.service.d/nonroot.conf <>/etc/exim4/update-exim4.conf.conf <>/etc/exim4/conf.d/main/000_local <>/etc/exim4/update-exim4.conf.conf <> /etc/exim4/conf.d/data_local_acl <<'EOF' -deny - malware = */defer_ok - !condition = ${if match {$malware_name}{\N^Heuristic\N}} - message = This message was detected as possible malware ($malware_name). -EOF cat >/etc/exim4/conf.d/main/000_local-nn <>/etc/exim4/conf.d/main/000_local < /etc/mailname + # mail default domain. + u /etc/mailutils.conf <<'EOF' +address { + email-domain iankelling.org; +}; +EOF # mail.iankelling.org so local imap clients can connect with tls and # when they happen to not be local. @@ -3080,19 +3490,52 @@ EOF ## we use this host to monitor MAIL_HOST and host a mail server for someone bk) + # No clamav on je, it has 1.5g memory and clamav uses most of it. + # + # No clamav on MAIL_HOST because it is just a waste of useful cpu + # time and memory when I'm running on an x200, and it takes 30 + # seconds to shut down. + + cat >>/etc/exim4/conf.d/main/000_local <> /etc/exim4/conf.d/data_local_acl <<'EOF' +deny + malware = */defer_ok + !condition = ${if match {$malware_name}{\N^Heuristic\N}} + message = This message was detected as possible malware ($malware_name). + +warn + !hosts = +iank_trusted + !authenticated = * + condition = ${if def:malware_name} + remove_header = Subject: + add_header = Subject: [Clamav warning: $malware_name] $h_subject + log_message = heuristic malware warning: $malware_name + +warn + # fdate = future date. # tdate = temporary date. + condition = ${if def:h_fdate} + remove_header = fdate: + add_header = tdate: + control = freeze +EOF + /a/exe/cedit nn /etc/hosts <<'EOF' || [[ $? == 1 ]] 10.173.8.2 nn.b8.nz EOF - sed -r -f - /etc/init.d/exim4 <<'EOF' | i /etc/init.d/exim4in + sed -r -f - /etc/init.d/exim4 <<'EOF' |u /etc/init.d/exim4in s,/etc/default/exim4,/etc/default/exim4in,g s,/run/exim4/exim.pid,/run/exim4/eximin.pid,g s,(^[ #]*Provides:).*,\1 exim4in, s,(^[ #]*NAME=).*,\1"exim4in", EOF chmod +x /etc/init.d/exim4in - i /etc/systemd/system/exim4in.service.d/alwaysrestart.conf <<'EOF' + u /etc/systemd/system/exim4in.service.d/alwaysrestart.conf <<'EOF' [Unit] # needed to continually restart StartLimitIntervalSec=0 @@ -3103,11 +3546,11 @@ Restart=always RestartSec=20 EOF - i /etc/default/exim4in <<'EOF' + u /etc/default/exim4in <<'EOF' # defaults but no queue runner and alternate config dir QUEUERUNNER='no' COMMONOPTIONS='-oP /run/exim4/eximin.pid' -UPEX4OPTS='-d /etc/myexim4' +UPEX4OPTS='-d /etc/nond-exim4' EOF echo bk.b8.nz > /etc/mailname @@ -3126,18 +3569,27 @@ EOF ;; # ** not MAIL_HOST|bk|je *) - # this one should be removed for all non mail hosts, but + echo|u /etc/exim4/conf.d/transport/30_debbugs + echo|u /etc/exim4/conf.d/router/153_debbugs + echo|u /etc/exim4/conf.d/router/155_delay + # this one should be removed for all non mail_hosts. note # bk and je never become mail_host - echo|i /etc/exim4/conf.d/router/195_dnslookup_vpn - echo|i /etc/exim4/conf.d/router/160_backup_redir - echo|i /etc/exim4/conf.d/router/161_backup_redir_nn - echo|i /etc/exim4/conf.d/router/185_sentarchive - echo|i /etc/exim4/conf.d/router/186_sentarchive_nn - echo|i /etc/exim4/conf.d/router/188_exim4-config_smarthost - echo|i /etc/exim4/conf.d/router/190_exim4-config_fsfsmarthost - echo|i /etc/exim4/conf.d/rcpt_local_acl - echo|i /etc/exim4/conf.d/main/000_local-nn - echo|i /etc/exim4/conf.d/clamav_data_acl + echo|u /etc/exim4/conf.d/router/195_dnslookup_vpn + echo|u /etc/exim4/conf.d/router/160_backup_redir + echo|u /etc/exim4/conf.d/router/161_backup_redir_nn + echo|u /etc/exim4/conf.d/router/185_sentarchive + echo|u /etc/exim4/conf.d/router/186_sentarchive_nn + # Note, in general we could submit to smarthosts on non MAIL_HOST. + # however, delayed mail makes this inconvenient, because I + # occasionally want to send an email from a non-MAIL_HOST and then + # turn off that computer or travel with it so it is disconnected. + # It is also probably easier to setup emacs to delay messages, but + # that would mean we need to keep emacs running, this is much + # nicer. + echo|u /etc/exim4/conf.d/router/188_exim4-config_smarthost + echo|u /etc/exim4/conf.d/router/190_exim4-config_fsfsmarthost + echo|u /etc/exim4/conf.d/rcpt_local_acl + echo|u /etc/exim4/conf.d/main/000_local-nn if $bhost_t; then @@ -3154,7 +3606,7 @@ EOF fi # catches things like cronjob email - i /etc/exim4/conf.d/router/880_universal_forward <<'EOF' + u /etc/exim4/conf.d/router/880_universal_forward <<'EOF' universal_forward: driver = redirect domains = +local_domains @@ -3179,7 +3631,7 @@ EOF if [[ -e /bu/md/cur && $(stat -c %u /bu/md/cur) == 1000 ]]; then chown -R Debian-exim:Debian-exim /bu/md fi - i /etc/exim4/conf.d/transport/30_backup_maildir <>/etc/exim4/update-exim4.conf.conf <>/etc/exim4/update-exim4.conf.conf <>/etc/exim4/update-exim4.conf.conf <>/etc/myexim4/conf.d/main/000_local <<'EOF' + --exclude=/conf.d/main/000_local-nn /etc/exim4/ /etc/nond-exim4 + cat >>/etc/nond-exim4/conf.d/main/000_local <<'EOF' # this makes it easier to see which exim is doing what -log_file_path = /var/log/exim4/my%s +log_file_path = /var/log/exim4/nond%s EOF cat >/etc/logrotate.d/myexim <<'EOF' -/var/log/exim4/mymain /var/log/exim4/myreject { +/var/log/exim4/nondmain /var/log/exim4/nondreject { daily missingok rotate 1000 @@ -3273,7 +3727,7 @@ EOF notifempty nocreate } -/var/log/exim4/mypanic { +/var/log/exim4/nondpanic { size 10M missingok rotate 10 @@ -3287,9 +3741,9 @@ EOF # If we ever wanted to have a separate spool, # we could do it like this. # cat >>/etc/exim4/conf.d/main/000_local-nn <<'EOF' - # spool_directory = /var/spool/myexim4 + # spool_directory = /var/spool/nond-exim4 # EOF - cat >>/etc/myexim4/update-exim4.conf.conf <<'EOF' + cat >>/etc/nond-exim4/update-exim4.conf.conf <<'EOF' dc_eximconfig_configtype='smarthost' dc_smarthost='nn.b8.nz' EOF @@ -3297,44 +3751,41 @@ EOF bk) # config for the non-nn exim - cat >>/etc/myexim4/conf.d/main/000_local <<'EOF' + cat >>/etc/nond-exim4/conf.d/main/000_local <<'EOF' MAIN_HARDCODE_PRIMARY_HOSTNAME = mail2.iankelling.org EOF ;; $MAIL_HOST) - i /etc/myexim4/conf.d/router/185_sentarchive <<'EOF' + u /etc/nond-exim4/conf.d/router/185_sentarchive <<'EOF' sentarchive: driver = redirect domains = ! +local_domains senders = <; *@fsf.org ; *@posteo.net - condition = ${if !bool{${lookup{$local_part@$domain}lsearch{/etc/exim4/ignore-sent}{true}}}} + condition = ${if and {{!bool{${lookup{$local_part@$domain}lsearch{/etc/exim4/ignore-sent}{true}}}} {!match {$h_user-agent:}{emacs}}}} data = vojdedIdNejyebni@b8.nz unseen EOF - - i /etc/myexim4/conf.d/router/160_backup_redir <<'EOF' + u /etc/nond-exim4/conf.d/router/160_backup_redir <<'EOF' backup_redir: -driver = redirect -# i dont email myself from my own machine much, so lets ignore that. -domains = ! +local_domains -senders = <; *@fsf.org ; *@posteo.net -condition = ${if !bool{${lookup{$local_part@$domain}lsearch{/etc/exim4/ignore-sent}{true}}}} -# b is just an arbirary short string -data = b@eximbackup.b8.nz -# note, to test this, i could temporarily allow testignore. -# alerts avoids potential mail loop. -local_parts = ! root : ! testignore : ! alerts : ! daylert -unseen = true -errors_to = alerts@iankelling.org + driver = redirect + # i dont email myself from my own machine much, so lets ignore that. + domains = ! +local_domains + senders = <; *@fsf.org ; *@posteo.net + condition = ${if and {{!bool{${lookup{$local_part@$domain}lsearch{/etc/exim4/ignore-sent}{true}}}} {!match {$h_user-agent:}{emacs}}}} + # b is just an arbirary short string + data = b@eximbackup.b8.nz + # note, to test this, i could temporarily allow testignore. + # alerts avoids potential mail loop. + local_parts = ! root : ! testignore : ! alerts : ! daylert + unseen = true + errors_to = alerts@iankelling.org EOF - - # for bk, we have a exim4in.service that will do this for us. - m update-exim4.conf -d /etc/myexim4 + m update-exim4.conf -d /etc/nond-exim4 ;; esac @@ -3356,7 +3807,7 @@ if [[ -e /nocow ]]; then if ! grep -Fx "/nocow/exim4 /var/spool/exim4 none bind 0 0" /etc/fstab; then echo "/nocow/exim4 /var/spool/exim4 none bind 0 0" >>/etc/fstab fi - i /etc/systemd/system/exim4.service.d/override.conf <<'EOF' + u /etc/systemd/system/exim4.service.d/override.conf <<'EOF' [Unit] # without local-fs on exim, we get these kind of errors in paniclog on shutdown: # Failed to create spool file /var/spool/exim4//input//1jCLxz-0008V4-V9-D: Permission denied @@ -3402,120 +3853,24 @@ elif [[ $uid != 608 ]]; then m find / /nocow -xdev -path ./var/tmp -prune -o -gid $gid -execdir chgrp -h 608 {} + fi -# * start / stop services - -reifactive dnsmasq nscd -if $reload; then - m systemctl daemon-reload -fi - -# checking bhost_t is redundant, but could help us catch errors. -if $bhost_t || [[ -e /etc/wireguard/wghole.conf ]]; then - # todo: in mail-setup, we have a static list of backup hosts, not *y - m systemctl --now enable wg-quick@wghole -fi - -sysd-prom-fail-install epanicclean -m systemctl --now enable epanicclean - -case $HOSTNAME in - je) - /a/exe/web-conf apache2 je.b8.nz - ;; - bk) - /a/exe/web-conf apache2 mail2.iankelling.org - ;; -esac - -m /a/bin/ds/mail-cert-cron -1 -sre mailcert.timer - -case $HOSTNAME in - $MAIL_HOST|bk) - m systemctl --now enable mailnn mailnnroute - ;;& - $MAIL_HOST) - # we use dns to start wg - if $reload; then - sre unbound - else - m systemctl --now enable unbound - fi - ;;& - $MAIL_HOST|bk) - # If these have changes, id rather manually restart it, id rather - # not restart and cause temporary errors - if $reload; then - sre $vpnser - else - m systemctl --now enable $vpnser - fi - if ! systemctl is-active clamav-daemon >/dev/null; then - m systemctl --now enable clamav-daemon - out=$(rsync -aiSAX --chown=root:root --chmod=g-s /a/bin/ds/filesystem/etc/systemd/system/epanicclean.service /etc/systemd/system) - if [[ $out ]]; then - reload=true - fi - - # note, this will cause paniclog entries because it takes like 45 - # seconds for clamav to start, i use ./epanic-clean to remove - # them. - fi - ;;& - $MAIL_HOST|bk|je) - # start spamassassin/dovecot before exim. - sre dovecot spamassassin - # need to wait a bit before restarting exim, else I - # get a paniclog entry like: spam acl condition: all spamd servers failed - sleep 3 - m systemctl --now enable mailclean.timer - ;;& - $MAIL_HOST) - # < 2.1 (eg: in t9), uses a different data format which required manual - # migration. dont start if we are running an old version. - if dpkg --compare-versions "$(dpkg -s radicale | awk '$1 == "Version:" { print $2 }')" ge 2.1; then - m systemctl --now enable radicale - fi - ;;& -esac - -# for debugging dns issues -case $HOSTNAME in - je|bk) - systemctl enable --now logrotate-fast.timer - ;; -esac - -# last use of $reload happens in previous block -rm -f /var/local/mail-setup-reload - - -case $HOSTNAME in - $MAIL_HOST|bk|je|li) - # on li, these are never started, except $vpnser - : - ;; - *) - soff radicale mailclean.timer dovecot spamassassin $vpnser mailnn clamav-daemon - ;; -esac - -sre exim4 - -case $HOSTNAME in - $MAIL_HOST) - m systemctl --now enable mailbindwatchdog - ;; - *) - soff mailbindwatchdog - ;; -esac - - -case $HOSTNAME in - bk) sre exim4in ;; -esac +# note: example config has a debbugs user, +# but my exim runs setuid as Debian-exim so it can't switch +# to another user. Anyways, I'm not exposing this to the +# internet at this time. If I do, the thing to do would +# be to use a sudo config (or sudo alternative). This +# would be how to setup + +# IFS=:; read -r _ _ uid _ < <(getent passwd debbugs||:) ||:; unset IFS +# if [[ ! $uid ]]; then +# # /a/opt/debbugs/debian/README.mail +# adduser --uid 610 --system --group --home /o/debbugs \ + # --no-create-home --disabled-login --force-badname debbugs +# m find /o/debbugs -xdev -path ./var/tmp -prune -o -uid $uid -execdir chown -h 610 {} + +# m find /o/debbugs -xdev -path ./var/tmp -prune -o -gid $gid -execdir chgrp -h 610 {} + +# elif [[ $uid != 610 ]]; then +# err debbugs exist but is not uid 610: investigate +# fi # * mail monitoring / testing @@ -3527,7 +3882,7 @@ case $HOSTNAME in # note: cronjob "ian" also does some important monitoring # todo: this will sometimes cause an alert because mailtest-check will run # before we have setup network namespace and spamassassin - i /etc/cron.d/mailtest </dev/null; then + m systemctl --now enable clamav-daemon + out=$(rsync -aiSAX --chown=root:root --chmod=g-s /a/bin/ds/filesystem/etc/systemd/system/epanicclean.service /etc/systemd/system) + if [[ $out ]]; then + reload=true + fi + + # note, this will cause paniclog entries because it takes like 45 + # seconds for clamav to start, i use ./epanic-clean to remove + # them. + fi + ;;& + $MAIL_HOST|bk|je) + # start spamassassin/dovecot before exim. + sre dovecot $spamd_ser mailtest-check + # Wait a bit before restarting exim, else I get a paniclog entry + # like: spam acl condition: all spamd servers failed. But I'm tired + # of waiting. I'll deal with this some other way. + # + # sleep 3 + m systemctl --now enable mailclean.timer + ;;& + $MAIL_HOST) + # < 2.1 (eg: in t9), uses a different data format which required manual + # migration. dont start if we are running an old version. + if dpkg --compare-versions "$(dpkg -s radicale | awk '$1 == "Version:" { print $2 }')" ge 2.1; then + m systemctl --now enable radicale + fi + ;;& +esac + +# for debugging dns issues +case $HOSTNAME in + je|bk) + systemctl enable --now logrotate-fast.timer + ;; +esac + +# last use of $reload happens in previous block +rm -f /var/local/mail-setup-reload + + +case $HOSTNAME in + $MAIL_HOST|bk|je|li) + # on li, these are never started, except $vpnser + : + ;; + *) + soff radicale mailclean.timer dovecot $spamd_ser $vpnser mailnn clamav-daemon + ;; +esac + +sre exim4 + +case $HOSTNAME in + $MAIL_HOST) + m systemctl --now enable mailbindwatchdog + ;; + *) + soff mailbindwatchdog + ;; +esac + + +case $HOSTNAME in + bk) sre exim4in ;; +esac # * misc m sudo -u $u mkdir -p /home/$u/.cache