From: Ian Kelling <ian@iankelling.org>
Date: Tue, 23 Feb 2021 03:43:19 +0000 (-0500)
Subject: fixes and extra mail backup
X-Git-Url: https://iankelling.org/git/?a=commitdiff_plain;h=2cdeac6c56747f32dc7dc0b486d59197a72515f2;p=distro-setup

fixes and extra mail backup
---

diff --git a/bk-backup b/bk-backup
index 4050b5b..114a015 100755
--- a/bk-backup
+++ b/bk-backup
@@ -5,7 +5,7 @@ shopt -s inherit_errexit 2>/dev/null ||: # ignore fail in bash < 4.4
 set -eE -o pipefail
 trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?. PIPESTATUS: ${PIPESTATUS[*]}" >&2' ERR
 
-# need root for rsync  pull of file ownership/perms
+# need root for rsync pull of file ownership/perms
 [[ $EUID == 0 ]] || exec sudo -E "${BASH_SOURCE[0]}" "$@"
 
 host=bk.b8.nz
diff --git a/brc b/brc
index 685d6f2..9aeff8c 100644
--- a/brc
+++ b/brc
@@ -643,6 +643,9 @@ eqcat() {
     hlm exigrep $i /var/log/exim4/mainlog | cat ||:
   done
 }
+eqrmf() {
+  exiqgrep -i | xargs exim -Mrm
+  }
 
 
 # shellcheck disable=SC2032
diff --git a/brc2 b/brc2
index ef5fb0a..52e01d8 100644
--- a/brc2
+++ b/brc2
@@ -140,6 +140,18 @@ tback() {
   sqlite3 /p/.timetrap.db "update entries set end = NULL where id = (select max(id) from entries);"
 }
 
+bum() {
+  local host=$1
+  (( $# == 1 )) || return 1
+  sshfs $host:/bu/md /bu/mnt
+  ser start exim4
+}
+bu() {
+  fusermount -u /bu/mnt
+}
+eqgo() {
+  enn -M $(exiqgrep -i)
+}
 
 gnupload(){
   /a/f/gnulib/build-aux/gnupload "$@"
diff --git a/btrbk-run b/btrbk-run
index d6de66a..9aef6e0 100644
--- a/btrbk-run
+++ b/btrbk-run
@@ -189,7 +189,7 @@ if [[ ! -v targets && ! $source ]]; then
       targets+=($home kw.office.fsf.org)
       ;;
     kd)
-      targets+=(x2.b8.nz)
+      targets+=(x2.b8.nz sy.b8.nz)
       # might not be connected to the vpn
       if timeout -s 9 6 ssh kw.office.fsf.org :; then
         targets+=(kw.office.fsf.org)
@@ -379,13 +379,13 @@ snapshot_create onchange
 # I could make this different from target_preserve,
 # if one disk had less space.
 # for now, keeping them equal.
-snapshot_preserve 36h 14d 8w 24m
-snapshot_preserve_min 4h
+snapshot_preserve 18h 14d 8w 24m
+snapshot_preserve_min 2h
 snapshot_dir btrbk
 
-# so, total backups = ~75
-target_preserve 36h 14d 8w 24m
-target_preserve_min 4h
+# so, total backups = ~58
+target_preserve 18h 14d 8w 24m
+target_preserve_min 2h
 
 # if something fails and it's not obvious, try doing
 # btrbk -l debug -v dryrun
@@ -487,6 +487,10 @@ else
   m /a/exe/mount-latest-remote ${targets[@]}
 fi
 
+if [[ $ret == 0 ]]; then
+  /a/exe/mail-backup-clean
+fi
+
 mexit $ret
 
 # todo: move variable data we don't care about backing up
diff --git a/check-remote-mailqs b/check-remote-mailqs
index f7d00c0..8a2fc85 100755
--- a/check-remote-mailqs
+++ b/check-remote-mailqs
@@ -10,7 +10,7 @@ shopt -s nullglob
 shopt -s dotglob
 
 
-for h in tp.b8.nz vpn1 x2 x3.b8.nz frodo.b8.nz kd.b8.nz kw iankelling.org bk.b8.nz; do
+for h in bk.b8.nz je.b8.nz tp.b8.nz vpn1 x2 x3.b8.nz frodo.b8.nz kd.b8.nz kw iankelling.org bk.b8.nz; do
   if [[ $HOSTNAME == "${h%%.*}" ]]; then
     continue
   fi
diff --git a/distro-begin b/distro-begin
index 30ecb9b..1ad65f4 100755
--- a/distro-begin
+++ b/distro-begin
@@ -472,7 +472,7 @@ if encrypted; then
 fi
 
 ##### make extra dirs
-dirs=(/mnt/{1,2,3,4,5,6,7,8,9} /nocow/t)
+dirs=(/mnt/{1,2,3,4,5,6,7,8,9} /nocow/t /bu/md /bu/md/{cur,tmp,new} /bu/mnt)
 sudo mkdir -p "${dirs[@]}"
 # allow to fail because they could have read-only mounts on them
 sudo chown $USER:$USER  "${dirs[@]}" ||:
diff --git a/gitslink b/gitslink
index 99ef892..a2318e0 100755
--- a/gitslink
+++ b/gitslink
@@ -36,8 +36,11 @@ for x in *; do
 done
 
 cd /a/exe
+if (( ${#existing[@]} )); then
+  echo run manually:
+fi
 for f in ${!existing[@]}; do
-  echo want to do rm -fv $f
+  echo rm -fv $f
 done
 
 
diff --git a/mail-backup-clean b/mail-backup-clean
new file mode 100755
index 0000000..79ffe9b
--- /dev/null
+++ b/mail-backup-clean
@@ -0,0 +1,30 @@
+#!/bin/bash
+# Copyright (C) 2016 Ian Kelling
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if ! test "$BASH_VERSION"; then echo "error: shell is not bash" >&2; exit 1; fi
+shopt -s inherit_errexit 2>/dev/null ||: # ignore fail in bash < 4.4
+set -eE -o pipefail
+trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" exit status: $?, PIPESTATUS: ${PIPESTATUS[*]}" >&2' ERR
+
+if ! mountpoint /bu/mnt &>/dev/null; then
+  exit 0
+fi
+
+cd /mnt/root/btrbk
+tmp=(o*)
+last_snap_date=${tmp[-1]#o.}
+time=$(( $(date -d $(sed -r  's/(.{4})(..)(.{5})(..)(.*)/\1-\2-\3:\4:\5/' <<<$last_snap_date) +%s) -1 ))
+# 1 second granularity, so we could have a duplicate file, oh well, not worrying about that.
+find /bu/md /bu/mnt -type f \! -newermt @$time -delete
diff --git a/mail-setup b/mail-setup
index 9de76cf..901361c 100755
--- a/mail-setup
+++ b/mail-setup
@@ -3,6 +3,8 @@
 # Copyright (C) 2019 Ian Kelling
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
+# todo: auto restart of je on checkrestart
+
 # todo: remove old files from bk:/m/md/expertpathologyreview.com/testignore/cur
 
 # todo: run mailping test after running, or otherwise
@@ -578,7 +580,7 @@ if ! grep -qFx "$line" $f; then
     err expected line in $f not found
   fi
   sed -i "s,^$badline$,$line," $f
-  m ser reload apparmor
+  m systemctl reload apparmor
 fi
 
 # note: anything added to nn_progs needs corresponding rm
@@ -652,7 +654,7 @@ if $ir; then
 fi
 
 i /etc/spamassassin/mylocal.cf <<'EOF'
-# the normal local.cf has a bunch of upstream stuff i dont want to mess with
+# this is mylocal.cf because the normal local.cf has a bunch of upstream stuff i dont want to mess with
 
 # /usr/share/doc/exim4-base/README.Debian.gz:
 #   SpamAssassin's default report should not be used in a add_header
@@ -667,8 +669,6 @@ uridnsbl_skip_domain expertpathologyreview.com
 uridnsbl_skip_domain zroe.org
 EOF
 
-
-
 # 2020-10-19 remove old file. remove this when all hosts updated
 rm -fv /etc/systemd/system/spamddnsfix.{timer,service}
 
@@ -887,13 +887,6 @@ hostlist iank_trusted = <; \\
 18.4.89.0/24 ; 2603:3005:71a:2e00::/64 ; 209.51.188.0/24 ; 2001:470:142::/48 ; 74.94.156.208/28
 EOF
 
-# This file only exists in the nn config. for bk to accept mail
-# outside the nn, it needs a separate cert
-cat >/etc/exim4/conf.d/main/000_local-nn <<EOF
-MAIN_TLS_CERTIFICATE = /etc/exim4/fullchain.pem
-MAIN_TLS_PRIVATEKEY = /etc/exim4/privkey.pem
-EOF
-
 rm -fv /etc/exim4/rcpt_local_acl # old path
 
 i /etc/exim4/conf.d/local_deny_exceptions_acl <<'EOF'
@@ -957,15 +950,14 @@ warn
 
 EOF
 
-i /etc/exim4/conf.d/router/900_exim4-config_local_user <<'EOF'
+i /etc/exim4/conf.d/router/900_exim4-config_local_user <<EOF
 ### router/900_exim4-config_local_user
 #################################
 
 # This router matches local user mailboxes. If the router fails, the error
 # message is "Unknown user".
-
 local_user:
-  debug_print = "R: local_user for $local_part@$domain"
+  debug_print = "R: local_user for \$local_part@\$domain"
   driver = accept
   domains = +local_domains
 # ian: default file except where mentioned.
@@ -1161,11 +1153,21 @@ PLrwsYzXGGCdJsO2vsmmqqgLsZiapYJlUNjfiyWLt7E2H6WzkNB3VIhIPfLqFDPK
 xioE3sYKdjOt+p6mlg3l8+OLtODEFPHDqwIBAg==
 -----END DH PARAMETERS-----
 EOF
-    { cat <<EOF
-# https://ssl-config.mozilla.org
-ssl = required
+    {
+      if [[ $HOSTNAME == $MAIL_HOST ]]; then
+        cat <<'EOF'
 ssl_cert = </etc/exim4/fullchain.pem
 ssl_key = </etc/exim4/privkey.pem
+EOF
+      else
+        cat <<'EOF'
+ssl_cert = </etc/exim4/exim.crt
+ssl_key = </etc/exim4/exim.key
+EOF
+      fi
+      cat <<EOF
+# https://ssl-config.mozilla.org
+ssl = required
 # this is the same as the certbot list, in my cert cronjob, I check if that has changed upstream.
 ssl_cipher_list = ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384
 ssl_protocols = TLSv1.2
@@ -1484,7 +1486,7 @@ if [[ $HOSTNAME == bk ]]; then
 
   #### begin dl roundcube
   # note, im r2e subbed to https://github.com/roundcube/roundcubemail/releases.atom
-  v=1.4.8; f=roundcubemail-$v-complete.tar.gz
+  v=1.4.11; f=roundcubemail-$v-complete.tar.gz
   cd /a/opt
   if [[ -e $f ]]; then
     timestamp=$(stat -c %Y $f)
@@ -1941,8 +1943,7 @@ EOF
 
   # ** $MAIL_HOST|bk)
   $MAIL_HOST|bk)
-
-    cat >>/etc/exim4/conf.d/main/000_local-nn <<EOF
+    cat >/etc/exim4/conf.d/main/000_local-nn <<EOF
 # MAIN_HARDCODE_PRIMARY_HOSTNAME might mess up the
 # smarthost config type, not sure.
 # failing message on mail-tester.com:
@@ -1957,6 +1958,11 @@ MAIN_HARDCODE_PRIMARY_HOSTNAME = mail.iankelling.org
 # I used this to avoid sender verification, didnt work but it still
 # makes sense based on the spec.
 hosts_treat_as_local = defaultnn.b8.nz
+
+# Outside nn, we get the default cert location from a debian macro,
+# and the cert file is put in place by a certbot hook.
+MAIN_TLS_CERTIFICATE = /etc/exim4/fullchain.pem
+MAIN_TLS_PRIVATEKEY = /etc/exim4/privkey.pem
 EOF
 
     /a/exe/cedit defaultnn /etc/hosts <<'EOF' || [[ $? == 1 ]]
@@ -1966,6 +1972,42 @@ EOF
   # ** $MAIL_HOST)
   $MAIL_HOST)
 
+    i /etc/exim4/conf.d/transport/30_backup_maildir <<EOF
+# modified debian maildir transport
+backup_maildir:
+  driver = appendfile
+  directory = /bu/mnt
+  delivery_date_add
+  envelope_to_add
+  return_path_add
+  maildir_format
+  directory_mode = 0700
+  mode = 0644
+  mode_fail_narrower = false
+  # this makes it so if the directory is unmounted, messages stay in the queue
+  create_directory = false
+EOF
+
+    i /etc/exim4/conf.d/router/890_backup_copy <<EOF
+### router/900_exim4-config_local_user
+#################################
+
+# todo, it would be nice to save sent email too,
+# but its not so important, they still exist in my head.
+backup_copy:
+  driver = accept
+  domains = +local_domains
+  local_parts = ! root : ! testignore
+  # uncomment this when testing, comment the above line
+  #local_parts = ! root : testignore
+  transport = backup_maildir
+  cannot_route_message = Unknown user
+  local_part_suffix = +*
+  local_part_suffix_optional
+  unseen
+  user = $u
+EOF
+
     # this avoids some error. i cant remember what. todo:
     # test it out and document why/if its needed.
     i /etc/exim4/host_local_deny_exceptions <<'EOF'
@@ -2095,21 +2137,6 @@ EOF
 10.173.8.2 nn.b8.nz
 EOF
 
-
-    i /etc/myexim4/conf.d/router/180_vpnmanual <<'EOF'
-# copied from dnslookup, altered domains, added route_list,
-# changed driver, removed ignore_target_hosts since it
-# relies on a later defined macro
-vpnmanual:
-  debug_print = "R: dnslookup for $local_part@$domain"
-  driver = manualroute
-  domains = iankelling.org : zroe.org:r2e.iankelling.org
-  transport = remote_smtp
-  same_domain_copy_routing = yes
-  route_list = * 10.8.0.4
-  no_more
-EOF
-
     sed -r -f - /etc/init.d/exim4  <<'EOF' | i /etc/init.d/exim4in
 s,/etc/default/exim4,/etc/default/exim4in,g
 s,/run/exim4/exim.pid,/run/exim4/eximin.pid,g
@@ -2147,10 +2174,8 @@ EOF
     cat >>/etc/exim4/update-exim4.conf.conf <<EOF
 dc_other_hostnames='je.b8.nz'
 EOF
-
     echo|i /etc/exim4/conf.d/rcpt_local_acl
     echo|i /etc/exim4/conf.d/router/880_universal_forward
-
     ;;
   # ** not MAIL_HOST|bk|je
   *)
@@ -2210,7 +2235,7 @@ dc_smarthost='nn.b8.nz'
 EOF
     ;;&
   bk)
-    rm -f /etc/myexim4/conf.d/router/180_vpnmanual
+
     # config for the non-nn exim
     cat >/etc/myexim4/conf.d/main/000_local-nn <<'EOF'
 MAIN_HARDCODE_PRIMARY_HOSTNAME = mail2.iankelling.org
@@ -2337,6 +2362,9 @@ case $HOSTNAME in
   $MAIL_HOST|bk|je)
     # start spamassassin/dovecot before exim.
     sre dovecot spamassassin
+    # need to wait a bit before restarting exim, else I
+    # get a paniclog entry like: spam acl condition: all spamd servers failed
+    sleep 3
     sstart mailclean.timer
     ;;&
   $MAIL_HOST)
@@ -2351,7 +2379,16 @@ case $HOSTNAME in
     ;;
 esac
 
-sre exim4
+case $HOSTNAME in
+  $MAIL_HOST)
+    # we manually mount /bu/mnt before starting
+    m systemctl disable exim4
+    m systemctl restart exim4
+    ;;
+  *)
+    sre exim4
+    ;;
+esac
 case $HOSTNAME in
   bk) sre exim4in ;;
 esac
diff --git a/mailtest-check b/mailtest-check
index c8f4d4d..5b7b220 100755
--- a/mailtest-check
+++ b/mailtest-check
@@ -85,11 +85,15 @@ for folder in ${folders[@]}; do
           fi
 
           declare -A results
-          for r in $($spamcpre spamc -y <"$latest" |sed 's/,/ /g'); do
+          # pyzor fails for our test message, so dont put useless load on their
+          # servers.
+          # example line that sed is parsing:
+          # (-0.1 / 5.0 requ) DKIM_SIGNED=0.1,DKIM_VALID=-0.1,DKIM_VALID_AU=-0.1,SPF_HELO_PASS=-0.001,SPF_PASS=-0.001,TVD_SPACE_RATIO=0.001 autolearn=_AUTOLEARN
+          for r in $($spamcpre sudo -u Debian-exim spamassassin -t --cf='score PYZOR_CHECK 0' <"$latest" | tail -n2 | head -n1 | sed -r 's/^\([^)]*\) *//;s/=[^, ]*([, ]|$)/ /g'); do
             case $r in
               # we have a new domain, ignore this.
               # it seems like some versions of spamassassin do BODY_SINGLE_WORD, others dont, we dun care.
-              BODY_SINGLE_WORD|FROM_FMBLA_NEWDOM*) : ;;
+              BODY_SINGLE_WORD|FROM_FMBLA_NEWDOM*|autolearn) : ;;
               SPF_HELO_NEUTRAL)
                 # some of my domains use neutral spf, treat them the same.
                 results[SPF_HELO_PASS]=t
diff --git a/pkgs b/pkgs
index e059147..db4f7c2 100644
--- a/pkgs
+++ b/pkgs
@@ -91,6 +91,7 @@ p3=(
   elinks
   etckeeper
   evince
+  exim4-doc-html
   fakeroot
   fail2ban
   fdupes