X-Git-Url: https://iankelling.org/git/?p=distro-setup;a=blobdiff_plain;f=mail-setup;h=93d09c739402f2c4b8c78c7b79aa22641b228f5b;hp=22d1e3ddd3d4bdbfffa37142e4a6de720b50275d;hb=HEAD;hpb=f46ee5570766081a5a73ce0d2132c8a06ee966fb

diff --git a/mail-setup b/mail-setup
index 22d1e3d..9ada9b6 100755
--- a/mail-setup
+++ b/mail-setup
@@ -1,7 +1,76 @@
 #!/bin/bash
 # * intro
-# Copyright (C) 2019 Ian Kelling
-# SPDX-License-Identifier: AGPL-3.0-or-later
+
+# Program to install and configure Ian's email related programs
+# Copyright (C) 2024  Ian Kelling
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# todo:
+# on bk (and fsf servers that run multiple exim4 daemons, eg eximfsf2 and eximfsf3),
+# make it so that when exim is restarted due to package upgrades,
+# we also restart those daemons, which can be done like so, based on looking
+# at the prerm and postinst scripts of exim4-daemon-heavy.
+#
+# if [[ ! -e /usr/sbin/invoke-rc.d-diverted ]]; then
+#   mv /usr/sbin/invoke-rc.d /usr/sbin/invoke-rc.d-diverted
+#   dpkg --divert /usr/sbin/invoke-rc.d-diverted --no-rename /usr/sbin/invoke-rc.d
+# fi
+# /usr/sbin/invoke-rc.d:
+# #!/bin/bash
+# if [[ DPKG_MAINTSCRIPT_PACKAGE == exim4* && $1 == exim4 ]]; then
+#   shift
+#   ret=0
+#   for daemon in exim4 eximfsf2 eximfsf3; do
+#     /usr/sbin/invoke-rc.d-diverted $daemon "$@" || ret=$?
+#   done
+# else
+#   /usr/sbin/invoke-rc.d-diverted "$@"
+# fi
+
+# Things I tend to forget. on MAIL_HOST, daemon runs with /etc/exim4/my.conf,
+# due to /etc/default/exim4 containing:
+# COMMONOPTIONS='-C /etc/exim4/my.conf'
+# UPEX4OPTS='-o /etc/exim4/my.conf'
+#
+# The non-daemon config
+# gets generated from this script calling update-exim4.conf -d /etc/myexim4
+# which has log path
+# log_file_path = /var/log/exim4/my%s
+#
+# On non bk|MAIL_HOST, the config and log file are all standard.
+#
+# eximbackup folder is /bu/md
+# it is cleaned up by mail-backup-clean, which is run by btrbk-run
+
+# shellcheck disable=SC2254 # makes for a lot of unneeded quotes
+
+
+# perusing through /el/mainlog without test messages:
+# &!testignore|jtuttle|
+#
+#&! testignore|jtuttle|eximbackup|/usr/sbin/exim4 -bpu
+
+# todo: this message seems to get dropped on the floor, it was due to a missing 2nd colon in
+#  condition = ${if def:h_fdate:}
+# Figure out how to avoid this message being discarded.
+
+# 2023-09-12 01:41:43 [722371] 1qfw9f-0031v9-0S <= ian@iankelling.org U=iank P=local S=483 id=87cyyogd7t.fsf@iankelling.org T="iank2" from <ian@iankelling.org> for testignore@amnimal.ninja
+# 2023-09-12 01:41:43 [722373] 1qfw9f-0031v9-0S H=nn.b8.nz [10.173.8.2]: SMTP error from remote mail server after pipelined end of data: 451 Temporary local problem - please try later
+# 2023-09-12 01:41:43 [722372] 1qfw9f-0031v9-0S == testignore@amnimal.ninja R=smarthost T=remote_smtp_smarthost defer (-46) H=nn.b8.nz [10.173.8.2] DT=0s: SMTP error from remote mail server after pipelined end of data: 451 Temporary local problem - please try later
 
 # todo: check new macro DKIM_TIMESTAMPS
 
@@ -130,10 +199,10 @@ if [ -z "$BASH_VERSION" ]; then echo "error: shell is not bash" >&2; exit 1; fi
 
 shopt -s nullglob
 
-if [[ -s /usr/local/lib/err ]]; then
-  source /usr/local/lib/err
-elif [[ -s /a/bin/errhandle/err ]]; then
-  source /a/bin/errhandle/err
+if [[ -s /usr/local/lib/bash-bear ]]; then
+  source /usr/local/lib/bash-bear
+elif [[ -s /a/bin/bash-bear-trap/bash-bear ]]; then
+  source /a/bin/bash-bear-trap/bash-bear
 else
   echo "no err tracing script found"
   exit 1
@@ -187,8 +256,8 @@ fi
 #######
 
 
-# * perstent password instructions
-# Note: for cert cron, we need to manually run first to accept known_hosts
+# * perstent password instructions Note: for cert cron, we need to
+# manually run first to accept known_hosts
 
 # # exim passwords:
 # # for hosts which have all private files I just use the same user
@@ -282,7 +351,7 @@ fi
 
 # * functions & constants
 
-pre="${0##*/}:"
+pre="${0##*/}:${SSH_CLIENT:+ $HOSTNAME:}"
 m() { printf "$pre %s\n"  "$*"; "$@"; }
 e() { printf "$pre %s\n"  "$*"; }
 err() { printf "$pre %s\n"  "$*" >&2; exit 1; }
@@ -328,9 +397,19 @@ soff () {
   done
 }
 sre() {
+  local enabled
   for service; do
     m systemctl restart $service
-    m systemctl enable $service;
+    # Optimization for exim,
+    # is-enabled: 0m0.015s
+    # enable: 0m0.748s
+    # It is related to this message:
+    # exim4.service is not a native service, redirecting to systemd-sysv-install.
+    # Executing: /lib/systemd/systemd-sysv-install enable exim4
+    enabled=$(systemctl is-enabled $service 2>/dev/null ||:)
+    if [[ $enabled != enabled ]]; then
+      m systemctl enable $service
+    fi
   done
 }
 mailhost() {
@@ -375,7 +454,7 @@ fi
 bhost_t=false
 case $HOSTNAME in
   $MAIL_HOST) : ;;
-  kd|frodo|x2|x3|kw|sy|bo)
+  kd|x2|x3|kw|sy|bo|so)
     bhost_t=true
     ;;
 esac
@@ -463,6 +542,9 @@ EOF
 
 # * clamav
 
+# old file. remove when all hosts updated, 2023-09-11
+rm -fv /etc/exim4/conf.d/clamav_data_acl
+
 m usermod -a -G Debian-exim clamav
 
 u /etc/systemd/system/clamav-daemon.service.d/fix.conf <<EOF
@@ -505,7 +587,7 @@ EOF
 # Aug 02 21:59:27 sy systemd[1]: wg-quick@wgmail.service: Failed with result 'exit-code'.
 # Aug 02 21:59:27 sy systemd[1]: Failed to start WireGuard via wg-quick(8) for wgmail.
 # Aug 02 21:59:47 sy systemd[1]: wg-quick@wgmail.service: Scheduled restart job, restart counter is at 1.
-# Aug 02 21:59:47 sy systemd[1]: Stopped WireGuard via wg-quick(8) for wgmail.
+# Aug 02 21:95:47 sy systemd[1]: Stopped WireGuard via wg-quick(8) for wgmail.
 # Aug 02 21:59:47 sy systemd[1]: Starting WireGuard via wg-quick(8) for wgmail...
 # Aug 02 21:59:47 sy wg-quick[3424]: wg-quick: `wgmail' already exists
 # Aug 02 21:59:47 sy systemd[1]: wg-quick@wgmail.service: Main process exited, code=exited, status=1/FAILURE
@@ -826,6 +908,7 @@ fi
 u /etc/spamassassin/mylocal.cf <<'EOF'
 # this is mylocal.cf because the normal local.cf has a bunch of upstream stuff i dont want to mess with
 
+
 # /usr/share/doc/exim4-base/README.Debian.gz:
 #   SpamAssassin's default report should not be used in a add_header
 #   statement since it contains empty lines. (This triggers e.g. Amavis'
@@ -850,6 +933,35 @@ PIDFILE="/var/run/spamd.pid"
 NICE="--nicelevel 15"
 CRON=1
 EOF
+
+case $HOSTNAME in
+  bk)
+    u /etc/spamassassin/my_thishost.cf <<'EOF'
+# note: these are duplicated in exim config
+# veth0/1 # bk bk_ip6
+internal_networks 10.173.8.1/32 10.173.8.2/32 85.119.83.50/32 2001:ba8:1f1:f0c9::2
+trusted_networks 10.173.8.1/32 10.173.8.2/32 85.119.83.50/32 2001:ba8:1f1:f0c9::2
+EOF
+
+    ;;
+  je)
+    u /etc/spamassassin/my_thishost.cf <<'EOF'
+# note: these are duplicated in exim config
+# veth0/1 # je je_ipv6
+internal_networks 10.173.8.1/32 10.173.8.2/32 85.119.82.128/32 2001:ba8:1f1:f09d::2/128
+trusted_networks 10.173.8.1/32 10.173.8.2/32 85.119.82.128/32 2001:ba8:1f1:f09d::2/128
+EOF
+    ;;
+  *)
+    u /etc/spamassassin/my_thishost.cf <<'EOF'
+# note: these are duplicated in exim config
+# veth0/1 # li li_ip6
+internal_networks 10.173.8.1/32 10.173.8.2/32 72.14.176.105/32 2600:3c00::f03c:91ff:fe6d:baf8/128
+trusted_networks 10.173.8.1/32 10.173.8.2/32 72.14.176.105/32 2600:3c00::f03c:91ff:fe6d:baf8/128
+EOF
+    ;;
+esac
+
 #####   end spamassassin config
 
 
@@ -1031,6 +1143,10 @@ banaction = iptables-exim
 ignoreip = 209.51.188.13 2001:470:142::13 209.51.188.92 2001:470:142:3::10 72.14.176.105 2600:3c00:e000:280::2 10.173.8.1
 EOF
 if $ur; then
+  # Ensure the log file monitored by fail2ban exists, or else fail2ban can't start.
+  if [[ ! -e /var/log/exim4/mainlog ]]; then
+    install -m 640 -o Debian-exim -g adm /dev/null /var/log/exim4/mainlog
+  fi
   m systemctl restart fail2ban
 fi
 
@@ -1230,6 +1346,7 @@ DKIM_SIGN_HEADERS = mime-version:in-reply-to:references:from:date:subject:to
 
 domainlist local_hostnames = ! je.b8.nz : ! bk.b8.nz : *.b8.nz : b8.nz
 
+# note: most of these are duplicated in spamassassin config
 hostlist iank_trusted = <; \
 # veth0
 10.173.8.1 ; \
@@ -1258,7 +1375,7 @@ delay_warning_condition = ${if or {\
 
 # enable 587 in addition to the default 25, so that
 # i can send mail where port 25 is firewalled by isp
-daemon_smtp_ports = 25 : 587
+daemon_smtp_ports = 25 : 587 : 10025
 # default of 25, can get stuck when catching up on mail
 smtp_accept_max = 400
 smtp_accept_reserve = 100
@@ -1267,6 +1384,12 @@ smtp_reserve_hosts = +iank_trusted
 # Rules that make receiving more liberal should be on backup hosts
 # so that we dont reject mail accepted by MAIL_HOST
 LOCAL_DENY_EXCEPTIONS_LOCAL_ACL_FILE = /etc/exim4/conf.d/local_deny_exceptions_acl
+
+acl_not_smtp = acl_check_not_smtp
+
+
+DEBBUGS_DOMAIN = b.b8.nz
+
 EOF
 
 if dpkg --compare-versions "$(dpkg-query -f='${Version}\n' --show exim4)" ge 4.94; then
@@ -1287,6 +1410,11 @@ DKIM_DOMAIN = ${lc:${domain:$rh_from:}}
 EOF
 fi
 
+cat >/etc/exim4/conf.d/main/30_local <<EOF
+freeze_tell =
+EOF
+
+
 rm -fv /etc/exim4/rcpt_local_acl # old path
 
 u /etc/exim4/conf.d/local_deny_exceptions_acl <<'EOF'
@@ -1340,6 +1468,38 @@ warn
 #  dmarc_status = reject:quarantine
 #  add_header = Reply-to: dmarctest@iankelling.org
 
+# This allows us to delay sending an email until a specific time,
+# allowing us time to change our mind and also to appear to have
+# sent the message at a different time. In emacs copy the
+# automcatically date header add an f to make it fdate,
+# and then change the date to whenever you want to send it.
+# In the system-status script, I check once per minute
+# or more if it should be sent.
+
+warn
+  # fdate = future date.
+  condition = ${if def:h_fdate:}
+  remove_header = fdate:
+  remove_header = date:
+  add_header = date: $h_fdate
+  control = freeze
+EOF
+sed -i 's/^freeze_tell =.*/#\0/' /etc/exim4/conf.d/main/02_exim4-config_options
+
+u /etc/exim4/conf.d/acl/41_check_not_smtp <<'EOF'
+# todo: for non MAIL_HOST machines, i'd like
+# to send to the MAIL_HOST without freezing.
+# So, only do this if we are MAIL_HOST.
+
+acl_check_not_smtp:
+warn
+  # fdate = future date.
+  condition = ${if def:h_fdate:}
+  remove_header = fdate:
+  remove_header = date:
+  add_header = Date: $h_fdate
+  control = freeze
+accept
 EOF
 
 
@@ -1724,12 +1884,13 @@ if mailhost; then
   # plus debug does not help.
   # sudo -u radicale radicale -D
 
-  # created password file with:
-  # htpasswd -c /p/c/machine_specific/li/filesystem/etc/caldav-htpasswd
+  # created radicale password file with:
+  # htpasswd -c /p/c/machine_specific/li/filesystem/etc/caldav-htpasswd ian
   # chmod 640 /p/c/machine_specific/li/filesystem/etc/caldav-htpasswd
   # # setup chgrp www-data in ./conflink
 
   pi-nostart radicale
+  m usermod -a -G radicale iank
 
   u /etc/systemd/system/radicale.service.d/override.conf <<EOF
 [Unit]
@@ -1896,7 +2057,7 @@ EOF
 ssl = required
 # this is the same as the certbot list, i check changes in /a/bin/ds/filesystem/usr/local/bin/check-lets-encrypt-ssl-settings
 ssl_cipher_list = ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384
-ssl_protocols = TLSv1.2
+ssl_min_protocol = TLSv1.2
 ssl_prefer_server_ciphers = no
 
 protocol lmtp {
@@ -1904,6 +2065,23 @@ protocol lmtp {
 # default is just $mail_plugins
   mail_plugins = $mail_plugins sieve
 }
+
+# /etc/dovecot/conf.d/10-master.conf says the default is 256M.
+# but I started getting oom errors in the syslog
+# Mar 27 15:10:04 sy dovecot[330088]: lmtp(iank)<3839880><gO/BDwtvBGaIlzoA7AdaJQ>: Fatal: master: service(lmtp): child 3839880 returned error 83 (Out of memory (service lmtp { vsz_limit=256 MB }, you may need to increase it) - set CORE_OUTOFMEM=1 environment to get core dump)
+# exim would just queue mail until it eventually succeeded.
+# Deciding what to increase it to, I found this
+# https://dovecot.org/list/dovecot/2011-December/080056.html
+# which suggests 3x the largest dovecot.index.cache file
+# and then I found that
+# md/l/testignore/dovecot.index.cache is 429M, my largest cache file,
+# but that folder only has 2k messages.
+# next biggest is md/l/qemu-devel/dovecot.index.cache 236M
+# which lead to me a search https://doc.dovecot.org/admin_manual/known_issues/large_cache/
+# which suggests 1.5x the maximum cache file size 1G, and
+# that I can safely rm the index.
+default_vsz_limit = 1500M
+
 EOF
       if dpkg --compare-versions "$(dpkg-query -f='${Version}\n' --show dovecot-core)" ge 1:2.3; then
         cat <<EOF
@@ -1927,14 +2105,28 @@ EOF
     fi
 
     rm -fv /etc/dovecot/conf.d/20-lmtp.conf # file from prev version
+
+    # Having backups of indexes is a waste of space. This also means we
+    # don't send them around with btrbk, I think it is probably
+    # preferable use a bit more cpu to recalculate indexes.
+    install -d -m 700 -o iank -g iank /var/dovecot-indexes
     cat >>/etc/dovecot/local.conf <<EOF
+
+
+# This will decrease memory use, and seems likely to decrease cpu & disk
+# use since I rarely use dovecot for most folders.
+mail_cache_max_size = 50M
+
+
 # simple password file based login
 !include conf.d/auth-passwdfile.conf.ext
 
 # ian: %u is used for alerts user vs iank
-mail_location = maildir:/m/%u:LAYOUT=fs:INBOX=/m/%u/INBOX
-mail_uid = $u
-mail_gid = $u
+# https://doc.dovecot.org/configuration_manual/mail_location/Maildir/
+mail_location = maildir:/m/%u:LAYOUT=fs:INBOX=/m/%u/INBOX:INDEX=/var/dovecot-indexes/%u
+# note: i don't know if these need to be set, but this seems fine.
+mail_uid = iank
+mail_gid = iank
 
 protocol lmtp {
 # For a normal setup with exim, we need something like this, which
@@ -2184,27 +2376,28 @@ if [[ $HOSTNAME == bk ]]; then
 
   ### begin composer install
   # https://getcomposer.org/doc/faqs/how-to-install-composer-programmatically.md
-  # cd $(mktemp -d)
-  # sum="$(wget -q -O - https://composer.github.io/installer.sig)"
-  # m php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
-  # if [[ $sum != $(php -r "echo hash_file('sha384', 'composer-setup.php');") ]]; then
-  #   echo 'ERROR: Invalid composer installer checksum' >&2
-  #   rm -fv composer-setup.php
-  #   exit 1
-  # fi
-  # m php composer-setup.php --quiet
-  # rm -fv composer-setup.php
-  # m mv composer.phar /usr/local/bin
-
-  # the above method gets composer2, carddav plugin at least doesnt work with that
-  # yet, it was just released 10-24-2020.
-  m cd /usr/local/bin
-  m wget -nv -N https://getcomposer.org/composer-1.phar
-  chmod +x composer-1.phar
+  cd /usr/local/bin
+  EXPECTED_CHECKSUM="$(php -r 'copy("https://composer.github.io/installer.sig", "php://stdout");')"
+  php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
+  ACTUAL_CHECKSUM="$(php -r "echo hash_file('sha384', 'composer-setup.php');")"
+
+  if [ "$EXPECTED_CHECKSUM" != "$ACTUAL_CHECKSUM" ]
+  then
+    >&2 echo 'ERROR: Invalid installer checksum'
+    rm composer-setup.php
+    exit 1
+  fi
+
+  php composer-setup.php --quiet
+  rm composer-setup.php
+
+  # based on error when running composer
+  mkdir -p /var/www/.composer
+  chown www-data:www-data /var/www/.composer
+
   ### end composer install
 
   rcdirs=(/usr/local/lib/rcexpertpath /usr/local/lib/rcninja)
-  ncdirs=(/var/www/ncninja)
   ncdirs=(/var/www/ncexpertpath /var/www/ncninja)
   # point debian cronjob to our local install, preventing daily cron error
 
@@ -2236,8 +2429,6 @@ if [[ $HOSTNAME == bk ]]; then
     rcdir=${rcdirs[i]}
     rcbase=${rcdir##*/}
     ncdir=${ncdirs[i]}
-    myncdir=/root/${ncdir##*/}
-    mkdir -p $myncdir
 
     # copied from debians cronjob
     u /etc/cron.d/$rcbase <<EOF
@@ -2346,8 +2537,8 @@ EOF
     m mkdir -p $rctmpdir /m/rc
     m chown -R www-data.www-data $rctmpdir /m/rc
     m chmod 750 $rctmpdir
-    # Ensure the log file monitored by fail2ban exists, or else fail2ban can't start.
     # todo: check for other mailinabox things
+    # Ensure the log file monitored by fail2ban exists, or else fail2ban can't start.
     m sudo -u www-data touch $rclogdir/errors.log
 
     #### begin carddav install
@@ -2359,7 +2550,7 @@ EOF
     verf=$rcdir/plugins/carddav/myversion
     upgrade=false
     install=false
-    v=4.0.0
+    v=5.0.1
     if [[ -e $verf ]]; then
       if [[ $(cat $verf) != "$v" ]]; then
         install=true
@@ -2371,16 +2562,16 @@ EOF
     if $install; then
       m rm -rf $rcdir/plugins/carddav
       tmpd=$(mktemp -d)
-      m wget -nv -O $tmpd/t.tgz https://github.com/blind-coder/rcmcarddav/releases/download/v$v/carddav-v$v.tgz
+      m wget -nv -O $tmpd/t.tgz https://github.com/blind-coder/rcmcarddav/releases/download/v$v/carddav-v$v.tar.gz
       cd $rcdir/plugins
       tar xzf $tmpd/t.tgz
       rm -rf $tmpd
       m chown -R www-data:www-data $rcdir/plugins/carddav
       m cd $rcdir/plugins/carddav
       if $upgrade; then
-        m sudo -u www-data composer-1.phar update --no-dev
+        m sudo -u www-data composer.phar update --no-dev
       else
-        m sudo -u www-data composer-1.phar install --no-dev
+        m sudo -u www-data composer.phar install --no-dev
       fi
       m chown -R root:root $rcdir/plugins/carddav
       echo $v >$verf
@@ -2504,7 +2695,9 @@ EOF
   for ((i=0; i < ${#bkdomains[@]}; i++)); do
     domain=${bkdomains[i]}
     ncdir=${ncdirs[i]}
+    myncdir=/var/local/${ncdir##*/}
     ncbase=${ncdir##*/}
+    mkdir -p $myncdir
     m cd /var/www
     if [[ ! -e $ncdir/index.php ]]; then
       # if we wanted to only install a specific version, use something like
@@ -2524,13 +2717,16 @@ EOF
       m touch $myncdir/done-install
     fi
 
-    # note, strange this happend where updater did not increment the version var,
-    # mine was stuck on 20. I manually updated it.
     m cd $ncdir/config
-    if [[ ! -e $myncdir/config.php-orig ]]; then
-      m cp -a config.php $myncdir/config.php-orig
-    fi
-    cat $myncdir/config.php-orig - >$myncdir/tmp.php <<EOF
+    # if we did this more than once, it would revert the
+    # version number to the original.
+    if [[ ! -e $myncdir/config.php-orig || ! -s config.php ]]; then
+      if [[ -s config.php ]]; then
+        m cp -a config.php $myncdir/config.php-orig
+        # keep the file so it keeps the same permissions.
+        truncate -s0 config.php
+      fi
+      cat $myncdir/config.php-orig - >$myncdir/tmp.php <<EOF
 # https://docs.nextcloud.com/server/19/admin_manual/configuration_server/email_configuration.html
 \$CONFIG["mail_smtpmode"] = "sendmail";
 \$CONFIG["mail_smtphost"] = "127.0.0.1";
@@ -2562,9 +2758,11 @@ fwrite(STDOUT, "<?php\n\\\$CONFIG = ");
 var_export(\$CONFIG);
 fwrite(STDOUT, ";\n");
 EOF
-    e running php $myncdir/tmp.php
-    # note: we leave it around place for debugging
-    php $myncdir/tmp.php >config.php
+      e running php $myncdir/tmp.php
+      # note: we leave it around place for debugging
+      # shellcheck disable=SC2024 # intended
+      sudo -u www-data php $myncdir/tmp.php >config.php
+    fi
     cd $ncdir
     m sudo -u www-data php occ maintenance:update:htaccess
     list=$(sudo -u www-data php $ncdir/occ --output=json_pretty app:list)
@@ -2601,7 +2799,7 @@ EOF
     u /usr/local/bin/ncup <<'EOFOUTER'
 #!/bin/bash
 
-source /usr/local/lib/err
+source /usr/local/lib/bash-bear
 
 m() { printf "%s\n" "$*";  "$@"; }
 err-cleanup() {
@@ -2630,7 +2828,13 @@ fi
 ncbase=$1
 cd /var/www/$ncbase
 # https://docs.nextcloud.com/server/22/admin_manual/maintenance/update.html?highlight=updater+phar
+# the docs claim this is all you need, which is not true.
+# You will go to the web ui and it will say that you need to click a button to update,
+# or that you can run occ upgrade
 m php /var/www/$ncbase/updater/updater.phar -n
+# throw a sleep in just because who knows what else is undocumented
+sleep 5
+m php occ upgrade
 EOFOUTER
     chmod +x /usr/local/bin/ncup
 
@@ -2647,6 +2851,67 @@ EOF
 fi
 
 
+# * debbugs
+
+pi debbugs
+# missing dependency. apache error log:
+# Can't locate List/AllUtils.pm in @INC (you may need to install the List::AllUtils module) (@INC contains: /etc/perl /usr/local/lib/x86_64-linux-gnu/perl/5.34.0 /usr/local/share/perl/5.34.0 /usr/lib/x86_64-linux-gnu/perl5/5.34 /usr/share/perl5 /usr/lib/x86_64-linux-gnu/perl-base /usr/lib/x86_64-linux-gnu/perl/5.34 /usr/share/perl/5.34 /usr/local/lib/site_perl) at /var/lib/debbugs/www/cgi/pkgreport.cgi line 23.
+pi liblist-allutils-perl lynx
+# workarounds for broken debbugsconfig which is
+# itself deprecated. this is temporary before I
+# figure out how to install from git
+if [[ -e /usr/share/doc/debbugs/examples/text.gz ]]; then
+  gunzip /usr/share/doc/debbugs/examples/text.gz
+fi
+mkdir -p /etc/debbugs/indices
+debbugsconfig
+
+
+# ld for local debbugs
+/a/exe/web-conf -l -t -a 127.0.1.1 -p 80 -r /var/lib/debbugs/www - apache2 ld <<'EOF'
+# copied from debbugs upstream example
+<Directory /var/lib/debbugs/www>
+   Options Indexes SymLinksIfOwnerMatch MultiViews
+   DirectoryIndex index.html
+   Require all granted
+</Directory>
+
+ScriptAlias /cgi/ /var/lib/debbugs/www/cgi/
+<Directory "/var/lib/debbugs/www/cgi/">
+  AllowOverride None
+  Options ExecCGI SymLinksIfOwnerMatch
+  Require all granted
+</Directory>
+
+RewriteEngine on
+RewriteCond %{HTTP_USER_AGENT}	.*apt-listbugs.*
+RewriteRule .*			/apt-listbugs.html [R,L]
+
+#  RewriteLog /org/bugs.debian.org/apache-rewrite.log
+#  RewriteLogLevel 0
+
+#RewriteRule ^/$ http://www.debian.org/Bugs/
+RewriteRule ^/(robots\.txt|release-critical|apt-listbugs\.html)$ - [L]
+# The following two redirect to up-to-date pages
+RewriteRule ^/[[:space:]]*#?([[:digit:]][[:digit:]][[:digit:]]+)([;&].+)?$ /cgi-bin/bugreport.cgi?bug=$1$2 [L,R,NE]
+RewriteRule ^/([^/+]*)([+])([^/]*)$ "/$1%%{%}2B$3" [N]
+RewriteRule ^/[Ff][Rr][Oo][Mm]:([^/]+\@.+)$ /cgi-bin/pkgreport.cgi?submitter=$1 [PT,NE]
+# Commented out, 'cuz aj says it will crash master. (old master)
+# RewriteRule ^/[Ss][Ee][Vv][Ee][Rr][Ii][Tt][Yy]:([^/]+\@.+)$ /cgi-bin/pkgreport.cgi?severity=$1 [L,R]
+RewriteRule ^/([^/]+\@.+)$ /cgi-bin/pkgreport.cgi?maint=$1 [PT,NE]
+RewriteRule ^/mbox:([[:digit:]][[:digit:]][[:digit:]]+)([;&].+)?$ /cgi-bin/bugreport.cgi?mbox=yes&bug=$1$2 [PT,NE]
+RewriteRule ^/src:([^/]+)$ /cgi-bin/pkgreport.cgi?src=$1 [PT,NE]
+RewriteRule ^/severity:([^/]+)$ /cgi-bin/pkgreport.cgi?severity=$1 [PT,NE]
+RewriteRule ^/tag:([^/]+)$ /cgi-bin/pkgreport.cgi?tag=$1 [PT,NE]
+# RewriteMap fix-chars	int:noescape
+RewriteCond %{REQUEST_URI} ^/(Access\.html|Developer\.html|Reporting\.html|server-request\.html|server-control\.html|server-refcard\.html).* [NC]
+RewriteRule .* - [L]
+# PT|passthrough to bugreport.cgi and pkgreport.cgi
+RewriteRule ^/([0-9]+)$ /cgi-bin/bugreport.cgi?bug=$1 [PT,NE]
+RewriteRule ^/([^/]+)$ /cgi-bin/pkgreport.cgi?pkg=$1 [PT,NE]
+EOF
+
+
 # * exim host conditional config
 
 # ** exim certs
@@ -2716,7 +2981,8 @@ case $HOSTNAME in
     # which will overwrite any existing file
     u /etc/default/exim4 <<'EOF'
 QUEUERUNNER='combined'
-QUEUEINTERVAL='30m'
+# note: this is duplicated in brc2, 10m here is -q10m there.
+QUEUEINTERVAL='10m'
 COMMONOPTIONS='-C /etc/exim4/my.conf'
 UPEX4OPTS='-o /etc/exim4/my.conf'
 # i use epanic-clean for alerting if there are bad paniclog entries
@@ -2833,29 +3099,10 @@ EOF
   # ** $MAIL_HOST|bk)
   $MAIL_HOST|bk)
 
-
-    # no clamav on je, it has 1.5g memory and clamav uses most of it
-    u /etc/exim4/conf.d/clamav_data_acl <<'EOF'
-warn
-!hosts = +iank_trusted
-!authenticated = plain_server:login_server
-condition = ${if def:malware_name}
-remove_header = Subject:
-add_header = Subject: [Clamav warning: $malware_name] $h_subject
-log_message = heuristic malware warning: $malware_name
-EOF
-
-    cat >>/etc/exim4/conf.d/main/000_local <<EOF
-# je.b8.nz will run out of memory with freshclam
-av_scanner = clamd:/var/run/clamav/clamd.ctl
+    cat >>/etc/exim4/update-exim4.conf.conf <<EOF
+dc_relay_nets='defaultnn.b8.nz'
 EOF
 
-    cat >> /etc/exim4/conf.d/data_local_acl <<'EOF'
-deny
-  malware = */defer_ok
-  !condition = ${if match {$malware_name}{\N^Heuristic\N}}
-  message = This message was detected as possible malware ($malware_name).
-EOF
 
     cat >/etc/exim4/conf.d/main/000_local-nn <<EOF
 # MAIN_HARDCODE_PRIMARY_HOSTNAME might mess up the
@@ -2884,7 +3131,7 @@ gnusmarthost:
   debug_print = "R: smarthost for $local_part@$domain"
   driver = manualroute
   domains = ! +local_domains
-# send most mail through eggs, helps fsfs sender reputation.
+# comment senders to send most mail through eggs, helps fsfs sender reputation.
 # uncomment and optionally move to 188 file to send through my own servers again
   senders = *@gnu.org
   transport = smarthost_dkim
@@ -2901,6 +3148,69 @@ EOF
   # ** $MAIL_HOST)
   $MAIL_HOST)
 
+    if [[ ! -e /etc/exim4/no-delay-eximids ]]; then
+      install -o iank -g iank <(echo) /etc/exim4/no-delay-eximids
+    fi
+
+    u /etc/exim4/conf.d/transport/30_debbugs <<'EOF'
+debbugs_pipe:
+  debug_print = "T: debbugs_pipe for $local_part@$domain"
+  driver = pipe
+  command = /usr/lib/debbugs/receive
+  return_output
+EOF
+
+
+    # We dont want delays or backups for mail being stored locally.
+    # We could put domain exclusion on other routes, but going for
+    # higher priority instead.
+    u /etc/exim4/conf.d/router/153_debbugs <<'EOF'
+debbugs:
+  debug_print = "R: debbugs for $local_part@$domain"
+  driver = accept
+  transport = debbugs_pipe
+  local_parts = submit : bugs : maintonly : quiet : forwarded : \
+                done : close : request : submitter : control : ^\\d+
+  domains = DEBBUGS_DOMAIN
+
+bounce_debbugs:
+  debug_print = "R: bounce_debbugs for $local_part@$domain"
+  driver = redirect
+  allow_fail
+  data = :fail: Unknown user
+  domains = DEBBUGS_DOMAIN
+EOF
+
+    u /etc/exim4/conf.d/router/155_delay <<'EOF'
+# By default, delay sending email by 30-40 minutes in case I
+# change my mind.
+
+# Note, if we switch mail_host, the next queue run will
+# send the message to mail_host and the delay will be reset.
+# That is fine. I could probably set some header to track
+# the delay but it is not worth it.
+delay_iank:
+  driver = redirect
+  allow_defer
+  data = :defer:
+  # It hasnt been 30 minutes since we received the message.
+  # we can avoid delay by adding the header i: or putting the exim message id into a file,
+  # or pulling "all" into a file.
+  # note, true false at the end just for easier debugging when pasting into a exim -Mset ID -be.
+  condition = ${if and { \
+{< {$tod_epoch} {${eval10:$received_time + 60*30}}} \
+{!def:h_i:} \
+{!bool{${lookup{$message_exim_id}lsearch{/etc/exim4/no-delay-eximids}{true}}}} \
+{!bool{${lookup{all}lsearch{/etc/exim4/no-delay-eximids}{true}}}} \
+} {true}{false}}
+  headers_remove = <; i:
+  domains = ! +local_domains
+  # uncomment for testing delays to jtuttle
+  # local_parts = ! root : ! testignore : ! alerts : ! ian-pager : ! daylert
+  local_parts = ! root : ! testignore : ! alerts : ! jtuttle : ! ian-pager : ! daylert
+  ignore_target_hosts = ROUTER_DNSLOOKUP_IGNORE_TARGET_HOSTS
+EOF
+
     u /etc/exim4/conf.d/router/195_dnslookup_vpn <<'EOF'
 # copied from /etc/exim4/conf.d/router/200_exim4-config_primary, but
 # use vpn transport. lower priority so it overrides the default route.
@@ -2932,7 +3242,7 @@ condition = ${if !bool{${lookup{$local_part@$domain}lsearch{/etc/exim4/ignore-se
 # note, to test this, i could temporarily allow testignore.
 # alerts avoids potential mail loop. root is already
 # redirected earlier, so that is just being overly cautious.
-local_parts = ! root : ! testignore : ! alerts
+local_parts = ! root : ! testignore : ! alerts : ! jtuttle : ! ian-pager : ! daylert
 unseen = true
 errors_to = alerts@iankelling.org
 EOF
@@ -2970,16 +3280,27 @@ MAILDIR_HOME_MAILDIR_LOCATION = /m/md/Sent
 EOF
 
 
+    # ian: save a copy of sent mail. i thought of other ways to do this,
+    # for example, to only save sent mail that is not sent from my mail
+    # client which saves a copy by default, but in the end, it seems
+    # simplest to turn that off. We want to save external mail sent by
+    # smarthosts. However, there is one complication: encrypted
+    # mail. Saving it here just gets us an encrypted copy that we can't
+    # read. Soo, we could bcc ourselves: then we still have the
+    # annoyance that it is encrypted so we can't grep it. Or, we could
+    # hack emacs so that it sends us an unencrypted copy. Turns out that
+    # the emacs function which saves sent email can also send us a
+    # copy. But, then we have 3 copies: the encrypted copy exim saves,
+    # the unencrypted copy exim saves, and the copy emacs saves.  Soo,
+    # we can emacs send a copy directly to the sent alias but only when
+    # it is not mail_host, and have the exim condition for redirecting a
+    # copy to the sent alias avoid doing it if it has an emacs user
+    # agent header.
     u /etc/exim4/conf.d/router/186_sentarchive_nn <<'EOF'
-# ian: save a copy of sent mail. i thought of other ways to
-# do this, for example, to only save sent mail that is not sent
-# from my mail client which saves a copy by default, but in the
-# end, it seems simplest to turn that off. We want to save
-# external mail sent by smarthosts.
 sentarchive_nn:
   driver = redirect
   domains = ! +local_domains
-  condition = ${if !bool{${lookup{$local_part@$domain}lsearch{/etc/exim4/ignore-sent}{true}}}}
+  condition = ${if and {{!bool{${lookup{$local_part@$domain}lsearch{/etc/exim4/ignore-sent}{true}}}} {!match {$h_user-agent:}{emacs}}}}
   data    = vojdedIdNejyebni@b8.nz
   unseen
 EOF
@@ -3027,6 +3348,12 @@ EOF
     # This name won\'t appear on From: lines of outgoing messages if rewriting is enabled.
     echo iankelling.org > /etc/mailname
 
+    # mail default domain.
+    u /etc/mailutils.conf <<'EOF'
+address {
+  email-domain iankelling.org;
+};
+EOF
 
     # mail.iankelling.org so local imap clients can connect with tls and
     # when they happen to not be local.
@@ -3058,7 +3385,6 @@ EOF
 # man page: is used to build the local_domains list, together with "localhost"
 # this is duplicated in a later router.
 dc_other_hostnames='iankelling.org;zroe.org;r2e.iankelling.org;mx.iankelling.org;!je.b8.nz;!bk.b8.nz;*.b8.nz;b8.nz'
-dc_relay_nets='defaultnn.b8.nz'
 EOF
 
 
@@ -3076,6 +3402,39 @@ EOF
   ## we use this host to monitor MAIL_HOST and host a mail server for someone
   bk)
 
+    # No clamav on je, it has 1.5g memory and clamav uses most of it.
+    #
+    # No clamav on MAIL_HOST because it is just a waste of useful cpu
+    # time and memory when I'm running on an x200, and it takes 30
+    # seconds to shut down.
+
+    cat >>/etc/exim4/conf.d/main/000_local <<EOF
+# je.b8.nz will run out of memory with freshclam
+av_scanner = clamd:/var/run/clamav/clamd.ctl
+EOF
+
+    cat >> /etc/exim4/conf.d/data_local_acl <<'EOF'
+deny
+  malware = */defer_ok
+  !condition = ${if match {$malware_name}{\N^Heuristic\N}}
+  message = This message was detected as possible malware ($malware_name).
+
+warn
+  !hosts = +iank_trusted
+  !authenticated = *
+  condition = ${if def:malware_name}
+  remove_header = Subject:
+  add_header = Subject: [Clamav warning: $malware_name] $h_subject
+  log_message = heuristic malware warning: $malware_name
+
+warn
+  # fdate = future date. # tdate = temporary date.
+  condition = ${if def:h_fdate}
+  remove_header = fdate:
+  add_header = tdate:
+  control = freeze
+EOF
+
 
     /a/exe/cedit nn /etc/hosts <<'EOF' || [[ $? == 1 ]]
 10.173.8.2 nn.b8.nz
@@ -3122,18 +3481,27 @@ EOF
     ;;
   # ** not MAIL_HOST|bk|je
   *)
-    # this one should be removed for all non mail hosts, but
+    echo|u /etc/exim4/conf.d/transport/30_debbugs
+    echo|u /etc/exim4/conf.d/router/153_debbugs
+    echo|u /etc/exim4/conf.d/router/155_delay
+    # this one should be removed for all non mail_hosts. note
     # bk and je never become mail_host
     echo|u /etc/exim4/conf.d/router/195_dnslookup_vpn
     echo|u /etc/exim4/conf.d/router/160_backup_redir
     echo|u /etc/exim4/conf.d/router/161_backup_redir_nn
     echo|u /etc/exim4/conf.d/router/185_sentarchive
     echo|u /etc/exim4/conf.d/router/186_sentarchive_nn
+    # Note, in general we could submit to smarthosts on non MAIL_HOST.
+    # however, delayed mail makes this inconvenient, because I
+    # occasionally want to send an email from a non-MAIL_HOST and then
+    # turn off that computer or travel with it so it is disconnected.
+    # It is also probably easier to setup emacs to delay messages, but
+    # that would mean we need to keep emacs running, this is much
+    # nicer.
     echo|u /etc/exim4/conf.d/router/188_exim4-config_smarthost
     echo|u /etc/exim4/conf.d/router/190_exim4-config_fsfsmarthost
     echo|u /etc/exim4/conf.d/rcpt_local_acl
     echo|u /etc/exim4/conf.d/main/000_local-nn
-    echo|u /etc/exim4/conf.d/clamav_data_acl
 
 
     if $bhost_t; then
@@ -3200,11 +3568,13 @@ backup_local:
 EOF
 
       # Bind to wghole to receive mailbackup.
-      wgholeip=$(sed -rn 's/^ *Address *= *([^/]+).*/\1/p' /etc/wireguard/wghole.conf)
-      cat >>/etc/exim4/update-exim4.conf.conf <<EOF
+      if [[ -e /etc/wireguard/wghole.conf ]]; then
+        wgholeip=$(sed -rn 's/^ *Address *= *([^/]+).*/\1/p' /etc/wireguard/wghole.conf)
+        cat >>/etc/exim4/update-exim4.conf.conf <<EOF
 dc_other_hostnames='eximbackup.b8.nz'
 dc_local_interfaces='127.0.0.1;::1;$wgholeip'
 EOF
+      fi
 
       # wghole & thus exim will fail to start without internet connectivity.
       u /etc/systemd/system/exim4.service.d/backup.conf <<'EOF'
@@ -3305,30 +3675,27 @@ sentarchive:
   driver = redirect
   domains = ! +local_domains
   senders = <; *@fsf.org ; *@posteo.net
-  condition = ${if !bool{${lookup{$local_part@$domain}lsearch{/etc/exim4/ignore-sent}{true}}}}
+  condition = ${if and {{!bool{${lookup{$local_part@$domain}lsearch{/etc/exim4/ignore-sent}{true}}}} {!match {$h_user-agent:}{emacs}}}}
   data    = vojdedIdNejyebni@b8.nz
   unseen
 EOF
 
-
     u /etc/myexim4/conf.d/router/160_backup_redir <<'EOF'
 backup_redir:
-driver = redirect
-# i dont email myself from my own machine much, so lets ignore that.
-domains = ! +local_domains
-senders = <; *@fsf.org ; *@posteo.net
-condition = ${if !bool{${lookup{$local_part@$domain}lsearch{/etc/exim4/ignore-sent}{true}}}}
-# b is just an arbirary short string
-data = b@eximbackup.b8.nz
-# note, to test this, i could temporarily allow testignore.
-# alerts avoids potential mail loop.
-local_parts = ! root : ! testignore : ! alerts : ! daylert
-unseen = true
-errors_to = alerts@iankelling.org
+  driver = redirect
+  # i dont email myself from my own machine much, so lets ignore that.
+  domains = ! +local_domains
+  senders = <; *@fsf.org ; *@posteo.net
+  condition = ${if and {{!bool{${lookup{$local_part@$domain}lsearch{/etc/exim4/ignore-sent}{true}}}} {!match {$h_user-agent:}{emacs}}}}
+  # b is just an arbirary short string
+  data = b@eximbackup.b8.nz
+  # note, to test this, i could temporarily allow testignore.
+  # alerts avoids potential mail loop.
+  local_parts = ! root : ! testignore : ! alerts : ! daylert
+  unseen = true
+  errors_to = alerts@iankelling.org
 EOF
 
-
-
     # for bk, we have a exim4in.service that will do this for us.
     m update-exim4.conf -d /etc/myexim4
     ;;
@@ -3398,6 +3765,25 @@ elif [[ $uid != 608 ]]; then
   m find / /nocow -xdev -path ./var/tmp -prune -o -gid $gid -execdir chgrp -h 608 {} +
 fi
 
+
+# note: example config has a debbugs user,
+# but my exim runs setuid as Debian-exim so it can't switch
+# to another user. Anyways, I'm not exposing this to the
+# internet at this time. If I do, the thing to do would
+# be to use a sudo config (or sudo alternative). This
+# would be how to setup
+
+# IFS=:; read -r _ _ uid _ < <(getent passwd debbugs||:) ||:; unset IFS
+# if [[ ! $uid ]]; then
+#   # /a/opt/debbugs/debian/README.mail
+#   adduser --uid 610 --system --group --home /o/debbugs \
+  #           --no-create-home --disabled-login --force-badname debbugs
+# m find /o/debbugs -xdev -path ./var/tmp -prune -o -uid $uid -execdir chown -h 610 {} +
+# m find /o/debbugs -xdev -path ./var/tmp -prune -o -gid $gid -execdir chgrp -h 610 {} +
+# elif [[ $uid != 610 ]]; then
+#   err debbugs exist but is not uid 610: investigate
+# fi
+
 # * start / stop services
 
 reifactive dnsmasq nscd
@@ -3406,14 +3792,20 @@ if $reload; then
   m systemctl daemon-reload
 fi
 
-# checking bhost_t is redundant, but could help us catch errors.
-if $bhost_t || [[ -e /etc/wireguard/wghole.conf ]]; then
-  # todo: in mail-setup, we have a static list of backup hosts, not *y
-  m systemctl --now enable wg-quick@wghole
+# optimization, this only needs to run once.
+if [[ ! -e /sys/class/net/wghole ]]; then
+  # checking bhost_t is redundant, but could help us catch errors.
+  if $bhost_t || [[ -e /etc/wireguard/wghole.conf ]]; then
+    # todo: in mail-setup, we have a static list of backup hosts, not *y
+    m systemctl --now enable wg-quick@wghole
+  fi
 fi
 
-sysd-prom-fail-install epanicclean
-m systemctl --now enable epanicclean
+# optimization, this only needs to be run once
+if [[ ! -e /var/lib/prometheus/node-exporter/exim_paniclog.prom ]]; then
+  sysd-prom-fail-install epanicclean
+  m systemctl --now enable epanicclean
+fi
 
 case $HOSTNAME in
   je)
@@ -3424,8 +3816,13 @@ case $HOSTNAME in
     ;;
 esac
 
-m /a/bin/ds/mail-cert-cron -1
-sre mailcert.timer
+# optimization, this only needs to run once.  But, if we move to a
+# computer we haven't used much, we need to fetch a fresh cert.
+# Existence check is just to avoid ugly error message from openssl.
+if [[ ! -e /etc/exim4/fullchain.pem ]] ||  ! openssl x509 -checkend $(( 60 * 60 * 24 * 3 )) -noout -in /etc/exim4/fullchain.pem; then
+  m /a/bin/ds/mail-cert-cron -1 -i
+  m systemctl --now enable mailcert.timer
+fi
 
 case $HOSTNAME in
   $MAIL_HOST|bk)
@@ -3447,6 +3844,8 @@ case $HOSTNAME in
     else
       m systemctl --now enable $vpnser
     fi
+    ;;&
+  bk)
     if ! systemctl is-active clamav-daemon >/dev/null; then
       m systemctl --now enable clamav-daemon
       out=$(rsync -aiSAX --chown=root:root --chmod=g-s /a/bin/ds/filesystem/etc/systemd/system/epanicclean.service /etc/systemd/system)
@@ -3462,9 +3861,11 @@ case $HOSTNAME in
   $MAIL_HOST|bk|je)
     # start spamassassin/dovecot before exim.
     sre dovecot spamassassin
-    # need to wait a bit before restarting exim, else I
-    # get a paniclog entry like: spam acl condition: all spamd servers failed
-    sleep 3
+    # Wait a bit before restarting exim, else I get a paniclog entry
+    # like: spam acl condition: all spamd servers failed. But I'm tired
+    # of waiting. I'll deal with this some other way.
+    #
+    # sleep 3
     m systemctl --now enable mailclean.timer
     ;;&
   $MAIL_HOST)
@@ -3605,6 +4006,9 @@ EOF
 
       test_to=${test_tos[0]}
       for t in ${test_tos[@]:1}; do
+        if [[ $test_from == *@gnu.org && $t == *@gnu.org ]]; then
+          continue
+        fi
         test_to+=", $t"
       done
       case $test_from in