From cedf5f4fab8b892c20acfa85c75d23cc32eaf3e9 Mon Sep 17 00:00:00 2001 From: Ian Kelling Date: Thu, 29 Oct 2020 10:46:39 -0400 Subject: [PATCH] mainly mail improvements, starting on new host --- bk-backup | 33 +- brc | 40 + brc2 | 89 +- check-stale-alerts | 2 +- distro-begin | 6 +- distro-end | 10 +- machine_specific/bitfolk.hosts | 2 + .../etc/systemd/resolved.conf.d/servers.conf | 3 + .../vps/filesystem/etc/bind/named.conf.local | 5 + mail-setup | 898 +++++++++--------- mailtest-check | 17 +- radicale-setup | 8 +- subdir_files/.config/mpv/mpv.conf | 2 +- system-status | 20 +- 14 files changed, 609 insertions(+), 526 deletions(-) create mode 100644 machine_specific/bitfolk.hosts create mode 100644 machine_specific/bitfolk/filesystem/etc/systemd/resolved.conf.d/servers.conf diff --git a/bk-backup b/bk-backup index 0069b37..0f54f75 100755 --- a/bk-backup +++ b/bk-backup @@ -5,10 +5,37 @@ shopt -s inherit_errexit 2>/dev/null ||: # ignore fail in bash < 4.4 set -eE -o pipefail trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?. PIPESTATUS: ${PIPESTATUS[*]}" >&2' ERR +host=bk.b8.nz +ret=0 source /a/bin/bash_unpublished/source-state + +restore=false +case $1 in + restore) + restore=true + ;; +esac + +if $restore; then + for ncdir in /var/www/ncexpertpath /var/www/ncninja; do + ssh root@$host sudo -u www-data php $ncdir/occ -q maintenance:mode --on ||: # might not be running + rsync -ra /p/bkbackup/$ncbase/ root@$host:$ncdir || ret=$? + done + rsync -ra /p/bkbackup/ root@$host:/m/rc +fi + if [[ $HOSTNAME == $MAIL_HOST ]]; then mkdir -p /p/bkbackup - ssh root@bk sudo -u www-data php /var/www/nextcloud/occ -q maintenance:mode --on - rsync -ra root@bk.b8.nz:/var/www/nextcloud/{config,data,themes} root@bk.b8.nz:/m/rc /p/bkbackup - ssh root@bk sudo -u www-data php /var/www/nextcloud/occ -q maintenance:mode --off + for ncdir in /var/www/ncexpertpath /var/www/ncninja; do + ncbase=${ncdir##*/} + mkdir -p /p/bkbackup/$ncbase + ssh root@$host sudo -u www-data php $ncdir/occ -q maintenance:mode --on + rsync -ra root@$host:$ncdir/{config,data,themes} /p/bkbackup/$ncbase || ret=$? + ssh root@$host sudo -u www-data php $ncdir/occ -q maintenance:mode --off + if (( $ret )) then + echo "$0: failed rsync $ncdir" + exit $ret + fi + done + rsync -ra root@$host:/m/rc /p/bkbackup fi diff --git a/brc b/brc index 4c3ee1a..8d05863 100644 --- a/brc +++ b/brc @@ -1107,6 +1107,46 @@ rsu() { # [OPTS] HOST PATH } ccomp rsync rsd rsa rst rsu +resolvcat() { + local f + if [[ $(systemctl is-active nscd ||:) != inactive ]]; then + m s nscd -i hosts + fi + f=/etc/resolv.conf + echo $f:; ccat $f + hr; s ss -lpn 'sport = 53' + if systemctl is-enabled dnsmasq &>/dev/null || [[ $(systemctl is-active dnsmasq ||:) != inactive ]]; then + # this will fail is dnsmasq is failed + hr; m ser status dnsmasq | cat || : + f=/etc/dnsmasq.conf + hr; echo $f:; ccat $f + hr; m grr '^ *(servers-file|server) *=|^ *no-resolv *$' /etc/dnsmasq.conf /etc/dnsmasq.d + f=/etc/dnsmasq-servers.conf + hr; echo $f:; ccat $f + fi + if systemctl is-enabled systemd-resolved &>/dev/null || [[ $(systemctl is-active systemd-resolved ||:) != inactive ]]; then + hr; m ser status systemd-resolved | cat || : + hr; m systemd-resolve --status | cat + fi + +} +rcat() { + resolvcat | less +} +reresolv() { + if [[ $(systemctl is-active nscd ||:) != inactive ]]; then + m ser stop nscd + sleep .5 + m ser start nscd + m sudo nscd -i hosts + fi + if [[ $(systemctl is-active dnsmasq ||:) != inactive ]]; then + m sudo systemctl restart dnsmasq + fi + if [[ $(systemctl is-active systemd-resolved ||:) != inactive ]]; then + m sudo systemctl restart systemd-resolved + fi +} rmstrips() { ssh fencepost head -n 300 /gd/gnuorg/EventAndTravelInfo/rms-current-trips.txt | less diff --git a/brc2 b/brc2 index 2e44a11..1029c6f 100644 --- a/brc2 +++ b/brc2 @@ -322,14 +322,38 @@ lipush() { p=(/a/bin /a/exe /a/h /a/c /p/c/machine_specific/vps{,.hosts} /a/opt/{emacs-debian10,mu}) a="-ahviSAXPH --specials --devices --delete --relative --exclude-from=/p/c/li-rsync-excludes" ret=0 - m rsync "$@" $a ${p[@]} /p/c/machine_specific/bk root@bk.b8.nz:/ || ret=$? - m rsync "$@" $a ${p[@]} /p/c/machine_specific/li root@li.b8.nz:/ || ret=$? + for h in je li bk; do + m rsync "$@" $a ${p[@]} /p/c/machine_specific/$h root@$h.b8.nz:/ || ret=$? + done m rsync "$@" -ahviSAXPH root@iankelling.org:/a/h/proposed-comments/ /a/h/proposed-comments || ret=$? return $ret } -lipushnoe() { # noe = noemacs. for running faster. - rsync "$@" --delete-excluded -ahviSAXPH --specials --devices --delete --relative \ - --exclude-from=/p/c/li-rsync-excludes /a/bin /a/exe /a/h /a/c /p/c/machine_specific/li root@li:/ +bkpush() { # no emacs. for running faster. + p=(/a/bin /a/exe /a/h /a/c /p/c/machine_specific/vps{,.hosts}) + a="-ahviSAXPH --specials --devices --delete --relative --exclude-from=/p/c/li-rsync-excludes" + ret=0 + m rsync "$@" $a ${p[@]} /p/c/machine_specific/bk root@bk.b8.nz:/ || ret=$? + return $ret +} +bkpush() { # no emacs. for running faster. + p=(/a/bin /a/exe /a/h /a/c /p/c/machine_specific/vps{,.hosts}) + a="-ahviSAXPH --specials --devices --delete --relative --exclude-from=/p/c/li-rsync-excludes" + ret=0 + m rsync "$@" $a ${p[@]} /p/c/machine_specific/je root@bk.b8.nz:/ || ret=$? + return $ret +} + + +dnsup() { + conflink + ser reload bind9 +} +dnsb8() { + local f=/var/lib/bind/db.b8.nz + ser stop bind9 + sudo rm -fv $f.jnl + sudo install -m 644 -o bind -g bind /p/c/machine_specific/vps/bind-initial/db.b8.nz $f + ser restart bind9 } @@ -445,13 +469,20 @@ dup() { ran_d=false system-status _ case $PS1 in - *DISTRO-BEGIN!*|*DISTRO!*) + *\]D\ *) pushd / /b/ds/distro-begin || return $? + /b/ds/distro-end || return $? popd ran_d=true ;;& - *DISTRO-END!*|*DISTRO!*) + *\]DB\ *) + pushd / + /b/ds/distro-begin || return $? + popd + ran_d=true + ;; + *\]DE\ *) pushd / /b/ds/distro-end || return $? popd @@ -990,44 +1021,6 @@ rebr() { sudo ifup br0 } -resolvcat() { - local f - m s nscd -i hosts - f=/etc/resolv.conf - echo $f:; ccat $f - hr; s ss -lpn 'sport = 53' - if systemctl is-enabled dnsmasq &>/dev/null || [[ $(systemctl is-active dnsmasq ||:) != inactive ]]; then - # this will fail is dnsmasq is failed - hr; m ser status dnsmasq | cat || : - f=/etc/dnsmasq.conf - hr; echo $f:; ccat $f - hr; m grr '^ *(servers-file|server) *=|^ *no-resolv *$' /etc/dnsmasq.conf /etc/dnsmasq.d - f=/etc/dnsmasq-servers.conf - hr; echo $f:; ccat $f - fi - if systemctl is-enabled systemd-resolved &>/dev/null || [[ $(systemctl is-active systemd-resolved ||:) != inactive ]]; then - hr; m ser status systemd-resolved | cat || : - hr; m systemd-resolve --status | cat - fi - -} -rcat() { - resolvcat | less -} -reresolv() { - if [[ $(systemctl is-active nscd ||:) != inactive ]]; then - m ser stop nscd - sleep .5 - m ser start nscd - m sudo nscd -i hosts - fi - if [[ $(systemctl is-active dnsmasq ||:) != inactive ]]; then - m sudo systemctl restart dnsmasq - fi - if [[ $(systemctl is-active systemd-resolved ||:) != inactive ]]; then - m sudo systemctl restart systemd-resolved - fi -} # only run on MAIL_HOST. simpler to keep this on one system. r2eadd() { # usage: name url @@ -1244,8 +1237,10 @@ enn() { fi } - -mailbash() { +mailnnbash() { + m sudo nsenter -t $(systemctl status mailnn| sed -n '/^ *Main PID:/s/[^0-9]//gp') -n -m sudo -u $USER -i bash +} +mailvpnbash() { m sudo nsenter -t $(pgrep -f "/usr/sbin/openvpn .* --config /etc/openvpn/.*mail.conf") -n -m sudo -u $USER -i bash } eximbash() { diff --git a/check-stale-alerts b/check-stale-alerts index 275a6ab..4dbdcc5 100755 --- a/check-stale-alerts +++ b/check-stale-alerts @@ -12,7 +12,7 @@ if [[ $out ]]; then echo HOSTNAME: $HOSTNAME printf "%s\n" "$out" fi -out=$(ssh bk.b8.nz find /m/md/INBOX/new /var/local/cron-errors /home/iank/cron-errors /sysd-mail-once-state -type f -mtime +1) +out=$(ssh bk.b8.nz find /m/md/bounces/new /var/local/cron-errors /home/iank/cron-errors /sysd-mail-once-state -type f -mtime +1) if [[ $out ]]; then echo bk.b8.nz: printf "%s\n" "$out" diff --git a/distro-begin b/distro-begin index 12fff61..8ccb574 100755 --- a/distro-begin +++ b/distro-begin @@ -97,19 +97,19 @@ source $script_dir/pkgs set +x source /a/bin/distro-functions/src/identify-distros $interactive || set -x -for f in kd x2 x3 frodo tp li bk demohost kw; do +for f in kd x2 x3 frodo tp li bk je demohost kw; do eval "$f() { [[ $HOSTNAME == $f ]]; }" done codename=$(debian-codename) has_wayland() { has_monitor && [[ $codename == buster ]]; } has_x() { has_monitor && [[ $codename != buster ]]; } has_monitor() { ! vps ; } -vps() { bk || li; } +vps() { je || bk || li; } # linode actually has btrfs now, but we dont do anything with it. has_btrfs() { ! vps; } home_network() { ! vps && ! tp; } has_p() { ! vps && ! tp; } -encrypted() { ! bk; } +encrypted() { ! bk && ! je; } shopt -s extglob export GLOBIGNORE="*/.:*/.." umask 022 diff --git a/distro-end b/distro-end index efbc3b3..ed5ccd4 100755 --- a/distro-end +++ b/distro-end @@ -486,14 +486,11 @@ EOF ###### begin website setup case $HOSTNAME in - li|bk) + li|bk|je) pi bind9 f=/var/lib/bind/db.b8.nz if [[ ! -e $f ]]; then - ser stop bind9 - sudo rm -fv $f.jnl - sudo install -m 644 -o bind -g bind /p/c/machine_specific/vps/bind-initial/db.b8.nz $f - ser restart bind9 + dnsb8 fi ;;& bk) @@ -553,12 +550,11 @@ EOF # https://community.openvpn.net/openvpn/wiki/IPv6 # and man openvpn - m vpn-server-setup -rd 2600:3c00:e000:280::1/64 2600:3c00::f03c:91ff:feb4:0bf3 -n mail + m vpn-server-setup -n mail -rd -6 2600:3c00:e000:280::/64 2600:3c00:e000:280::1/64 2600:3c00::f03c:91ff:feb4:0bf3 sudo tee /etc/openvpn/client-config-mail/mailclient <<'EOF' ifconfig-push 10.8.0.4 255.255.255.0 ifconfig-ipv6-push 2600:3c00:e000:280::2/64 EOF - sudo tee /etc/openvpn/client-config-mail/expertpath <<'EOF' ifconfig-push 10.8.0.5 255.255.255.0 ifconfig-ipv6-push 2600:3c00:e000:280::3/64 diff --git a/machine_specific/bitfolk.hosts b/machine_specific/bitfolk.hosts new file mode 100644 index 0000000..ca3fdae --- /dev/null +++ b/machine_specific/bitfolk.hosts @@ -0,0 +1,2 @@ +je +bk diff --git a/machine_specific/bitfolk/filesystem/etc/systemd/resolved.conf.d/servers.conf b/machine_specific/bitfolk/filesystem/etc/systemd/resolved.conf.d/servers.conf new file mode 100644 index 0000000..7a35e68 --- /dev/null +++ b/machine_specific/bitfolk/filesystem/etc/systemd/resolved.conf.d/servers.conf @@ -0,0 +1,3 @@ +[Resolve] +DNS=85.119.80.232 85.119.80.233 +Domains=~. diff --git a/machine_specific/vps/filesystem/etc/bind/named.conf.local b/machine_specific/vps/filesystem/etc/bind/named.conf.local index ead4c18..0655556 100644 --- a/machine_specific/vps/filesystem/etc/bind/named.conf.local +++ b/machine_specific/vps/filesystem/etc/bind/named.conf.local @@ -34,6 +34,11 @@ type master; file "/var/lib/bind/db.expertpathologyreview.com"; }; +zone "amnimal.ninja" { +type master; +file "/var/lib/bind/db.amnimal.ninja"; +}; + zone "9.c.0.f.1.f.1.0.8.a.b.0.1.0.0.2.ip6.arpa" { type master; file "/var/lib/bind/db.9.c.0.f.1.f.1.0.8.a.b.0.1.0.0.2.ip6.arpa"; diff --git a/mail-setup b/mail-setup index 4a498c5..a34d562 100755 --- a/mail-setup +++ b/mail-setup @@ -3,13 +3,6 @@ # Copyright (C) 2019 Ian Kelling # SPDX-License-Identifier: AGPL-3.0-or-later -# todo: for a locally generated message, if we delivery into the network -# namespace, it goes into the queue, but later fails again with -# something like "remote host is the localhost". Of course, we could -# have a separate queue runner and a separate spool, but Im thinking -# there might be another way. seeing debug output would probably be -# helpful. - # background: I want to run exim in a network namespace so it can send # and receive through a vpn. This is needed so it can do ipv6, because # outside the namespace if we dont have ipv6, to send ipv6 through the @@ -64,11 +57,6 @@ if [ -z "$BASH_VERSION" ]; then echo "error: shell is not bash" >&2; exit 1; fi -pre="${0##*/}:" -m() { printf "$pre %s\n" "$*"; "$@"; } -e() { printf "$pre %s\n" "$*"; } -err() { echo "[$(date +'%Y-%m-%d %H:%M:%S%z')]: $0: $*" >&2; } - shopt -s nullglob if [[ -s /usr/local/lib/err ]]; then @@ -82,7 +70,7 @@ fi source /a/bin/distro-functions/src/identify-distros # has nextcloud_admin_pass in it -f=/p/c/machine_specific/bk/mail +f=/p/c/machine_specific/je/mail if [[ -e $f ]]; then source $f fi @@ -220,6 +208,47 @@ fi # * functions & constants + +pre="${0##*/}:" +m() { printf "$pre %s\n" "$*"; "$@"; } +e() { printf "$pre %s\n" "$*"; } +err() { echo "[$(date +'%Y-%m-%d %H:%M:%S%z')]: $0: $*" >&2; } +reload=false +i() { # install file + local tmp tmpdir dest="$1" + local base="${dest##*/}" + mkdir -p ${dest%/*} + ir=false # i result + tmpdir=$(mktemp -d) + cat >$tmpdir/"$base" + tmp=$(rsync -ic $tmpdir/"$base" "$dest") + if [[ $tmp ]]; then + printf "%s\n" "$tmp" + ir=true + if [[ $dest == /etc/systemd/system/* ]]; then + reload=true + fi + fi + rm -rf $tmpdir +} +soff () { + for service; do + m systemctl stop $service; + m systemctl disable $service + done +} +sre () { + for service; do + m systemctl restart $service + m systemctl enable $service; + done +} +sstart() { + for service; do + m systemctl start $service + m systemctl enable $service; + done +} e() { printf "%s\n" "$*"; } pi() { # package install without starting daemons local f @@ -251,6 +280,13 @@ reifactive() { fi done } +stopifactive() { + for service; do + if systemctl is-active $service >/dev/null; then + m systemctl stop $service + fi + done +} mxhost=mail.iankelling.org mxport=587 @@ -286,27 +322,29 @@ pi exim4 exim4-daemon-heavy spamassassin spf-tools-perl openvpn p0f postgrey pyz # note: pyzor debian readme says you need to run some initialization command # but its outdated. +soff openvpn + + if [[ $(debian-codename) == etiona ]]; then # ip6tables stopped loading on boot. openvpn has reduced capability set, # so running iptables as part of openvpn startup wont work. This should do it. # Im sure there is a better way, but this works fine. yes no | pi iptables-persistent || [[ $? == 141 ]] cat >/etc/iptables/rules.v6 <<'EOF' -*nat *mangle -*filter +COMMIT +*nat COMMIT EOF # load it now. - ip6tables -S &>/dev/null + m ip6tables -S &>/dev/null fi # our nostart pi fails to avoid enabling -systemctl disable openvpn ### * user forward file case $HOSTNAME in - $MAIL_HOST|bk) + $MAIL_HOST) # afaik, these will get ignored on MAIL_HOST because they are routing to my own # machine, but rm them is safer rm -fv $uhome/.forward /root/.forward @@ -321,7 +359,7 @@ esac # * Mail clean cronjob -cat >/etc/systemd/system/mailclean.timer <<'EOF' +i /etc/systemd/system/mailclean.timer <<'EOF' [Unit] Description=Run mailclean daily @@ -332,7 +370,7 @@ OnCalendar=monthly WantedBy=timers.target EOF -cat >/etc/systemd/system/mailclean.service <> /etc/default/postgrey <<'EOF' +i /etc/default/postgrey <<'EOF' POSTGREY_OPTS="--exim --unix=/var/run/postgrey/postgrey.sock --retry-window=4 --max-age=60" EOF -# * spamassassin -## also has a bit of exim config +# * mail vpn config -cat >/etc/sysctl.d/80-iank-mail.conf <<'EOF' -# see exim spec -net.netfilter.nf_conntrack_tcp_timeout_close_wait = 120 -EOF -sysctl -p +# todo: figure out a reverse dns lookup for 10.173.8.1 in the nn. +# perhaps adding files in nsswitch should fix it? -cat >/etc/spamassassin/mylocal.cf <<'EOF' -# the normal local.cf has a bunch of upstream stuff i dont want to mess with +i /etc/systemd/system/mailnn.service <<'EOF' +[Unit] +Description=Network Namespace for mailvpn.service +After=syslog.target network-online.target +Wants=network-online.target -# /usr/share/doc/exim4-base/README.Debian.gz: -# SpamAssassin's default report should not be used in a add_header -# statement since it contains empty lines. (This triggers e.g. Amavis' -# warning "BAD HEADER SECTION, Improper folded header field made up -# entirely of whitespace".) This is a safe, terse alternative: -clear_report_template -report (_SCORE_ / _REQD_ requ) _TESTSSCORES(,)_ autolearn=_AUTOLEARN -internal_networks 85.119.83.50 10.173.8.1 -trusted_networks 72.14.176.105 2600:3c00:e000:280::2 85.119.83.50 18.4.89.0/24 209.51.188.0/24 74.94.156.208/28 2603:3005:71a:2e00::/64 2001:470:142::/48 10.173.8.1 -EOF +[Service] +Type=simple +PrivateNetwork=true +ExecStartPre=/usr/bin/flock -w 20 /tmp/newns.flock /a/bin/newns/newns -n 10.173.8 start mail +ExecStart=/bin/sleep infinity +ExecStopPost=/usr/bin/flock -w 20 /tmp/newns.flock /a/bin/newns/newns stop mail +[Install] +WantedBy=multi-user.target +EOF -# todo: figure out a reverse dns lookup for 10.173.8.1 in the nn. -# perhaps adding files in nsswitch should fix it? +# old service name +rm -fv /etc/systemd/system/openvpn-client-mail@.service # https://selivan.github.io/2017/12/30/systemd-serice-always-restart.html -tmpdir=$(mktemp -d) -cat >$tmpdir/openvpn-client-mail@.service <<'EOF' +i /etc/systemd/system/mailvpn.service <<'EOF' [Unit] -Description=OpenVPN tunnel for %I -After=syslog.target network-online.target +Description=OpenVPN tunnel for mail +After=syslog.target network-online.target mailnn.service Wants=network-online.target Documentation=man:openvpn(8) Documentation=https://community.openvpn.net/openvpn/wiki/Openvpn24ManPage Documentation=https://community.openvpn.net/openvpn/wiki/HOWTO # needed to continually restatr StartLimitIntervalSec=0 - +JoinsNamespaceOf=mailnn.service [Service] Type=notify RuntimeDirectory=openvpn-client RuntimeDirectoryMode=0710 WorkingDirectory=/etc/openvpn/client -ExecStart=/usr/sbin/openvpn --suppress-timestamps --nobind --config /etc/openvpn/client/%i.conf +ExecStart=/usr/sbin/openvpn --suppress-timestamps --nobind --config /etc/openvpn/client/mail.conf #CapabilityBoundingSet=CAP_IPC_LOCK CAP_NET_ADMIN CAP_NET_RAW CAP_SETGID CAP_SETUID CAP_SYS_CHROOT CAP_DAC_OVERRIDE LimitNPROC=10 # DeviceAllow=/dev/null rw # DeviceAllow=/dev/net/tun rw -ExecStartPre=/usr/bin/flock -w 20 /tmp/newns.flock /a/bin/newns/newns -n 10.173.8 start %i -ExecStopPost=/usr/bin/flock -w 20 /tmp/newns.flock /a/bin/newns/newns stop %i PrivateNetwork=true # in the network namespace, we cant connect to systemd-resolved on 127.0.0.53, # because of @@ -418,29 +449,17 @@ PrivateNetwork=true # I also like the idea of patching systemd-resolved so it # will listen on other interfaces, but its not worth my time. BindPaths=/etc/nn-resolv:/run/systemd/resolve:norbind - Restart=always # time to sleep before restarting a service RestartSec=1 - [Install] WantedBy=multi-user.target EOF -tmp=$(rsync -ic $tmpdir/* /etc/systemd/system) -rm -rf $tmpdir -if [[ $tmp ]]; then - m systemctl daemon-reload -fi -tmpdir=$(mktemp -d) -echo "nameserver 8.8.8.8" >$tmpdir/stub-resolv.conf -mkdir -p /etc/nn-resolv -rsync -ic $tmpdir/* /etc/nn-resolv -rm -rf $tmpdir +i /etc/nn-resolv/stub-resolv.conf <<<"nameserver 8.8.8.8" m chattr +i /etc/nn-resolv/stub-resolv.conf -### begin setup network namespace ### nn_progs=(exim4) if [[ $HOSTNAME == "$MAIL_HOST" ]]; then @@ -450,15 +469,20 @@ fi case $HOSTNAME in $MAIL_HOST|bk) - reload=false - tmpdir=$(mktemp -d) for unit in ${nn_progs[@]}; do - cat >$tmpdir/nn.conf <<'EOF' + i /etc/systemd/system/$unit.service.d/nn.conf <<'EOF' +[Unit] +JoinsNamespaceOf=mailnn.service + +[Service] +PrivateNetwork=true +EOF + i /etc/systemd/system/$unit.service.d/nn.conf <<'EOF' [Unit] After=network.target -Requires=openvpn-client-mail@mail.service -After=openvpn-client-mail@mail.service -JoinsNamespaceOf=openvpn-client-mail@mail.service +Requires=mailvpn.service +After=mailvpn.service +JoinsNamespaceOf=mailnn.service # needed to continually restart StartLimitIntervalSec=0 @@ -471,17 +495,7 @@ Restart=always # time to sleep before restarting a service RestartSec=1 EOF - mkdir -p /etc/systemd/system/$unit.service.d - tmp=$(rsync -ic $tmpdir/* /etc/systemd/system/$unit.service.d) - if [[ $tmp ]]; then - printf "rsync to /etc/systemd/system/$unit.service.d\n%s\n" "$tmp" - reload=true - fi done - rm -rf $tmpdir - if $reload; then - m systemctl daemon-reload - fi ;; *) reload=false @@ -492,28 +506,47 @@ EOF reload=true fi done - if $reload; then - m systemctl daemon-reload - fi - ;; esac -### end setup network namespace ### + +# * spamassassin config +i /etc/sysctl.d/80-iank-mail.conf <<'EOF' +# see exim spec +net.netfilter.nf_conntrack_tcp_timeout_close_wait = 120 +EOF +if $ir; then + m sysctl -p +fi + +i /etc/spamassassin/mylocal.cf <<'EOF' +# the normal local.cf has a bunch of upstream stuff i dont want to mess with + +# /usr/share/doc/exim4-base/README.Debian.gz: +# SpamAssassin's default report should not be used in a add_header +# statement since it contains empty lines. (This triggers e.g. Amavis' +# warning "BAD HEADER SECTION, Improper folded header field made up +# entirely of whitespace".) This is a safe, terse alternative: +clear_report_template +report (_SCORE_ / _REQD_ requ) _TESTSSCORES(,)_ autolearn=_AUTOLEARN +EOF + # 2020-10-19 remove old file. remove this when all hosts updated rm -fv /etc/systemd/system/spamddnsfix.{timer,service} # per readme.debian, allow nightly cronjob to run -sed -i '/^\s*CRON\s*=/d' /etc/default/spamassassin -e CRON=1 >>/etc/default/spamassassin +m sed -i '/^\s*CRON\s*=/d' /etc/default/spamassassin +e CRON=1 /etc/default/spamassassin case $HOSTNAME in $MAIL_HOST|bk) - # Just noticed this in the config file, seems like a good idea. - sed -i '/^\s*NICE\s*=/d' /etc/default/spamassassin - e 'NICE="--nicelevel 15"' >>/etc/default/spamassassin - + l='NICE="--nicelevel 15"' + if grep -qFx "$l" /etc/default/spamassassin; then + # Just noticed this in the config file, seems like a good idea. + m sed -i '/^\s*NICE\s*=/d' /etc/default/spamassassin + e "$l" | tee -a /etc/default/spamassassin + fi ;; esac ##### end spamassassin config @@ -531,9 +564,7 @@ fi m rsync -aiSAX --chown=root:root --chmod=g-s /a/bin/ds/mail-cert-cron /usr/local/bin -### begin install timer -tmpdir=$(mktemp -d) -cat >$tmpdir/mailcert.service <<'EOF' +i /etc/systemd/system/mailcert.service <<'EOF' [Unit] Description=Mail cert rsync After=multi-user.target @@ -542,7 +573,7 @@ After=multi-user.target Type=oneshot ExecStart=/a/bin/log-quiet/sysd-mail-once mailcert /usr/local/bin/mail-cert-cron EOF -cat >$tmpdir/mailcert.timer <<'EOF' +i /etc/systemd/system/mailcert.timer <<'EOF' [Unit] Description=Run mail-cert once a day @@ -552,17 +583,6 @@ OnCalendar=daily [Install] WantedBy=timers.target EOF -tmp=$(rsync -ic $tmpdir/* /etc/systemd/system) -if [[ $tmp ]]; then - m systemctl daemon-reload -fi -rm -rf $tmpdir -### end install timer - - -m systemctl start mailcert -m systemctl restart mailcert.timer -m systemctl enable mailcert.timer # * common exim4 config @@ -574,7 +594,7 @@ awk 'BEGIN { FS = ":" } ; $6 !~ /^\/home/ { print $1 }' /etc/passwd| while read continue fi if ! grep -q "^$user:" /etc/aliases; then - echo "$user: root" |tee -a /etc/aliases + echo "$user: root" |m tee -a /etc/aliases fi done @@ -625,25 +645,25 @@ sed -r s/^\\S+:/$b:/ 600_exim4-config_userforward >175_$b #in debian, config file used is first found of: #CONFIGURE_FILE=/etc/exim4/exim4.conf:/var/lib/exim4/config.autogenerated # but we can use this alternate for the daemon -update-exim4defaults -f --commonoptions '-C /etc/exim4/my.conf' +m update-exim4defaults -f --commonoptions '-C /etc/exim4/my.conf' l="UPEX4OPTS='-o /etc/exim4/my.conf'" if ! grep -Fxq "$l" /etc/default/exim4; then sed -i '/^ *UPEX4OPTS=/d' /etc/default/exim4 - echo "$l" >> /etc/default/exim4 + echo "$l" |m tee -a /etc/default/exim4 fi -cat >/etc/exim4/trusted_configs <<'EOF' +i /etc/exim4/trusted_configs <<'EOF' /etc/exim4/my.conf EOF #### end setup alternate config for main daemon # alerts is basically the postmaster address -sed -i --follow-symlinks -f - /etc/aliases </etc/exim4/conf.d/main/000_local </etc/exim4/conf.d/local_deny_exceptions_acl <<'EOF' +i /etc/exim4/conf.d/local_deny_exceptions_acl <<'EOF' accept authenticated = * EOF rm -fv /etc/exim4/data_local_acl # old path -cat >/etc/exim4/conf.d/data_local_acl <<'EOF' +i /etc/exim4/conf.d/data_local_acl <<'EOF' # Except for the "condition =", this was # a comment in the check_data acl. The comment about this not # being suitable has been changed in newer exim versions. The only thing @@ -733,12 +753,13 @@ cat >/etc/exim4/conf.d/data_local_acl <<'EOF' # pretty quickly looking through my spam folder. warn - # all internal ips. note this is duplicated in mylocal.cf, shouldnt have any effect there but leaving just in case - !hosts = <; 72.14.176.105 ; 2600:3c00:e000:280::2 ; 85.119.83.50 ; 18.4.89.0/24 ; 209.51.188.0/24 ; 74.94.156.208/28 ; 2603:3005:71a:2e00::/64 ; 2001:470:142::/48 ; 10.173.8.1 + # all internal ips. + # veth0 li li_ip6 li_vpn_net li_vpn_net_ip6 bk bk_ip6 fsf_mit_net fsf_mit_net_ip6 fsf_net fsf_net_ip6 fsf_office_net + !hosts = <; 10.173.8.1 ; 72.14.176.105 ; 2600:3c00::f03c:91ff:fe6d:baf8; 10.8.0.0/24; 2600:3c00:e000:280::/64 ; 85.119.83.50 ; 2001:ba8:1f1:f0c9::2 ; 18.4.89.0/24 ; 2603:3005:71a:2e00::/64 ; 209.51.188.0/24 ; 2001:470:142::/48 ; 74.94.156.208/28 remove_header = X-Spam_score: X-Spam_score_int : X-Spam_bar : X-Spam_report warn - !hosts = <; 72.14.176.105 ; 2600:3c00:e000:280::2 ; 85.119.83.50 ; 18.4.89.0/24 ; 209.51.188.0/24 ; 74.94.156.208/28 ; 2603:3005:71a:2e00::/64 ; 2001:470:142::/48 ; 10.173.8.1 + !hosts = <; 10.173.8.1 ; 72.14.176.105 ; 2600:3c00::f03c:91ff:fe6d:baf8; 10.8.0.0/24; 2600:3c00:e000:280::/64 ; 85.119.83.50 ; 2001:ba8:1f1:f0c9::2 ; 18.4.89.0/24 ; 2603:3005:71a:2e00::/64 ; 209.51.188.0/24 ; 2001:470:142::/48 ; 74.94.156.208/28 condition = ${if < {$message_size}{5000K}} spam = Debian-exim:true add_header = X-Spam_score_int: $spam_score_int @@ -755,7 +776,7 @@ warn EOF # see sender validation in /a/opt/mailinabox/setup/mail-users.sh -cat >/etc/exim4/conf.d/router/900_exim4-config_local_user <<'EOF' +i /etc/exim4/conf.d/router/900_exim4-config_local_user <<'EOF' ### router/900_exim4-config_local_user ################################# @@ -773,7 +794,7 @@ local_user: transport = LOCAL_DELIVERY cannot_route_message = Unknown user EOF -cat >/etc/exim4/conf.d/transport/30_exim4-config_dovecot_lmtp <<'EOF' +i /etc/exim4/conf.d/transport/30_exim4-config_dovecot_lmtp <<'EOF' dovecot_lmtp: driver = lmtp socket = /var/run/dovecot/lmtp @@ -793,32 +814,33 @@ dc_mailname_in_oh='true' EOF -# ** dovecot -dovecot-setup() { - # based on a little google and package search, just the dovecot - # packages we need instead of dovecot-common. - # - # dovecot-lmtpd is for exim to deliver to dovecot instead of maildir - # directly. The reason to do this is to use dovecot\'s sieve, which - # can generally do more than exims filters (a few things less) and - # sieve has the benefit of being supported in postfix and - # proprietary/weird environments, so there is more examples on the - # internet. - pi dovecot-core dovecot-imapd dovecot-sieve dovecot-lmtpd dovecot-sqlite +# * dovecot - for f in /p/c{/machine_specific/$HOSTNAME,}/filesystem/etc/dovecot/users; do - e $f - if [[ -e $f ]]; then - m rsync -ahhi --chown=root:dovecot --chmod=0640 $f /etc/dovecot/ - break - fi - done - for f in /p/c/subdir_files/sieve/*sieve /a/c/subdir_files/sieve/*sieve; do - m sudo -u $u /a/exe/lnf -T $f $uhome/sieve/${f##*/} - done +case $HOSTNAME in + $MAIL_HOST|bk) + # based on a little google and package search, just the dovecot + # packages we need instead of dovecot-common. + # + # dovecot-lmtpd is for exim to deliver to dovecot instead of maildir + # directly. The reason to do this is to use dovecot\'s sieve, which + # can generally do more than exims filters (a few things less) and + # sieve has the benefit of being supported in postfix and + # proprietary/weird environments, so there is more examples on the + # internet. + pi dovecot-core dovecot-imapd dovecot-sieve dovecot-lmtpd dovecot-sqlite + + for f in /p/c{/machine_specific/$HOSTNAME,}/filesystem/etc/dovecot/users; do + if [[ -e $f ]]; then + m rsync -ahhi --chown=root:dovecot --chmod=0640 $f /etc/dovecot/ + break + fi + done + for f in /p/c/subdir_files/sieve/*sieve /a/c/subdir_files/sieve/*sieve; do + m sudo -u $u /a/exe/lnf -T $f $uhome/sieve/${f##*/} + done - # https://wiki.dovecot.org/SSL/DovecotConfiguration - cat >/etc/dovecot/dhparam <<'EOF' + # https://wiki.dovecot.org/SSL/DovecotConfiguration + i /etc/dovecot/dhparam <<'EOF' -----BEGIN DH PARAMETERS----- MIIBCAKCAQEAoleil6SBxGqQKk7j0y2vV3Oklv6XupZKn7PkPv485QuFeFagifeS A+Jz6Wquqk5zhGyCu63Hp4wzGs4TyQqoLjkaWL6Ra/Bw3g3ofPEzMGEsV1Qdqde4 @@ -828,7 +850,7 @@ PLrwsYzXGGCdJsO2vsmmqqgLsZiapYJlUNjfiyWLt7E2H6WzkNB3VIhIPfLqFDPK xioE3sYKdjOt+p6mlg3l8+OLtODEFPHDqwIBAg== -----END DH PARAMETERS----- EOF - cat >/etc/dovecot/local.conf <>/etc/dovecot/local.conf </etc/dovecot/local.conf - case $HOSTNAME in - $MAIL_HOST) - # If we changed 90-sieve.conf and removed the active part of the - # sieve option, we wouldn\'t need this, but I\'d rather not modify a - # default config if not needed. This won\'t work as a symlink in /a/c - # unfortunately. - m sudo -u $u /a/exe/lnf -T sieve/main.sieve $uhome/.dovecot.sieve + ;;& - if [[ ! -e $uhome/sieve/personal.sieve ]]; then - touch $uhome/sieve/personal{,end}{,test}.sieve - fi + $MAIL_HOST) + # If we changed 90-sieve.conf and removed the active part of the + # sieve option, we wouldn\'t need this, but I\'d rather not modify a + # default config if not needed. This won\'t work as a symlink in /a/c + # unfortunately. + m sudo -u $u /a/exe/lnf -T sieve/main.sieve $uhome/.dovecot.sieve + + if [[ ! -e $uhome/sieve/personal.sieve ]]; then + m touch $uhome/sieve/personal{,end}{,test}.sieve + fi - rm -fv /etc/dovecot/conf.d/20-lmtp.conf # file from prev version - cat >>/etc/dovecot/local.conf <>/etc/dovecot/local.conf <>/etc/dovecot/local.conf <>/etc/dovecot/local.conf </etc/dovecot/sieve-spam.sieve <<'EOF' + i /etc/dovecot/sieve-spam.sieve <<'EOF' require ["regex", "fileinto", "imap4flags"]; if allof (header :regex "X-Spam-Status" "^Yes") { @@ -996,10 +1018,10 @@ if allof (header :regex "X-Spam-Status" "^Yes") { stop; } EOF - sievec /etc/dovecot/sieve-spam.sieve + m sievec /etc/dovecot/sieve-spam.sieve - cat >/etc/dovecot/local.conf.ext <<'EOF' + i /etc/dovecot/local.conf.ext <<'EOF' passdb { driver = sql args = /etc/dovecot/dovecot-sql.conf.ext @@ -1011,7 +1033,7 @@ userdb { EOF - cat >/etc/dovecot/dovecot-sql.conf.ext <<'EOF' + i /etc/dovecot/dovecot-sql.conf.ext <<'EOF' # from mailinabox driver = sqlite connect = /m/rc/users.sqlite @@ -1020,13 +1042,13 @@ password_query = SELECT email as user, password FROM users WHERE email='%u'; user_query = SELECT email AS user, "mail" as uid, "mail" as gid, "/m/md/%d/%n" as home FROM users WHERE email='%u'; iterate_query = SELECT email AS user FROM users; EOF - chmod 0600 /etc/dovecot/dovecot-sql.conf.ext # per Dovecot instructions + m chmod 0600 /etc/dovecot/dovecot-sql.conf.ext # per Dovecot instructions - # db needs to be in a www-data writable directory - db=/m/rc/users.sqlite - if [[ ! -s $db ]]; then - mkdir -p /m/rc - sqlite3 $db <<'EOF' + # db needs to be in a www-data writable directory + db=/m/rc/users.sqlite + if [[ ! -s $db ]]; then + m mkdir -p /m/rc + m sqlite3 $db <<'EOF' CREATE TABLE users ( id INTEGER PRIMARY KEY AUTOINCREMENT, email TEXT NOT NULL UNIQUE, @@ -1034,33 +1056,33 @@ password TEXT NOT NULL, extra, privileges TEXT NOT NULL DEFAULT ''); EOF - fi - # example of adding a user: - # hash: doveadm pw -s SHA512-CRYPT -p passhere - # sqlite3 /m/rc/users.sqlite <<'EOF' - #insert into users (email, password) values ('testignore@bk.b8.nz', 'hash'); - #EOF - # update users set password = 'hash' where email = 'testignore@bk.b8.nz'; - ;; - esac - ####### end dovecot-setup ######## -} + fi + # example of adding a user: + # hash: doveadm pw -s SHA512-CRYPT -p passhere + # sqlite3 /m/rc/users.sqlite <<'EOF' + #insert into users (email, password) values ('testignore@bk.b8.nz', 'hash'); + #EOF + # update users set password = 'hash' where email = 'testignore@bk.b8.nz'; + ;; +esac # * thunderbird autoconfig setup +bkdomains=(expertpathologyreview.com amnimal.ninja) if [[ $HOSTNAME == bk ]]; then - /a/exe/web-conf apache2 autoconfig.expertpathologyreview.com - dir=/var/www/autoconfig.expertpathologyreview.com/html/mail - mkdir -p $dir - # taken from mailinabox - cat >$dir/config-v1.1.xml <<'EOF' + for domain in ${bkdomains[@]}; do + m /a/exe/web-conf apache2 autoconfig.$domain + dir=/var/www/autoconfig.$domain/html/mail + m mkdir -p $dir + # taken from mailinabox + i $dir/config-v1.1.xml < - - expertpathologyreview.com + + $domain - expertpathologyreview.com Mail - expertpathologyreview.com + $domain Mail + $domain mail2.iankelling.org @@ -1080,23 +1102,24 @@ if [[ $HOSTNAME == bk ]]; then false - - expertpathologyreview.com website. + + $domain website. - - + + %EMAILADDRESS% - + EOF + done fi # * roundcube setup @@ -1107,36 +1130,29 @@ if [[ $HOSTNAME == bk ]]; then # https://getcomposer.org/doc/faqs/how-to-install-composer-programmatically.md cd $(mktemp -d) sum="$(wget -q -O - https://composer.github.io/installer.sig)" - php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" + m php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" if [[ $sum != $(php -r "echo hash_file('sha384', 'composer-setup.php');") ]]; then echo 'ERROR: Invalid composer installer checksum' >&2 - rm composer-setup.php + rm -fv composer-setup.php exit 1 fi - php composer-setup.php --quiet - rm composer-setup.php - mv composer.phar /usr/local/bin + m php composer-setup.php --quiet + rm -fv composer-setup.php + m mv composer.phar /usr/local/bin ### end composer install - - - # avoid prompt export DEBIAN_FRONTEND=noninteractive # zip according to /installer # which requires adding a line to /usr/local/lib/roundcubemail/config/config.inc.php # $config['enable_installer'] = true; pi roundcube roundcube-sqlite3 php-zip - rcdir=/usr/local/lib/roundcubemail + rcdirs=(/usr/local/lib/rcexpertpath /usr/local/lib/rcninja) + ncdirs=(/var/www/ncexpertpath /var/www/ncninja) # point debian cronjob to our local install, preventing daily cron error - f=/usr/share/roundcube/bin/cleandb.sh - if [[ ! -L $f ]]; then - if [[ -e $f ]]; then - m rm -f $f - fi - m ln -sfT $rcdir/bin/cleandb.sh /usr/share/roundcube/bin/cleandb.sh - fi + # debian's cronjob will fail, remove both paths it uses just to be sure + rm -fv /usr/share/roundcube/bin/cleandb.sh /etc/cron.d/roundcube-core #### begin dl roundcube # note, im r2e subbed to https://github.com/roundcube/roundcubemail/releases.atom @@ -1149,15 +1165,29 @@ if [[ $HOSTNAME == bk ]]; then fi m wget -nv -N https://github.com/roundcube/roundcubemail/releases/download/$v/$f new_timestamp=$(stat -c %Y $f) - if [[ $timestamp != "$new_timestamp" || ! -e "$rcdir/config/secret" ]]; then - m tar -C /usr/local/lib --no-same-owner -zxf $f - m rm -rf $rcdir - m mv $rcdir-$v $rcdir - fi - cd - + for rcdir in ${rcdirs[@]}; do + if [[ $timestamp != "$new_timestamp" || ! -e "$rcdir/config/secret" ]]; then + m tar -C /usr/local/lib --no-same-owner -zxf $f + m rm -rf $rcdir + m mv /usr/local/lib/roundcubemail-$v $rcdir + fi + done #### end dl roundcube - /a/exe/web-conf - apache2 expertpathologyreview.com </dev/null +EOF + + m /a/exe/web-conf - apache2 $domain < +Alias /nextcloud "$ncdir/" + Require all granted AllowOverride All Options FollowSymLinks MultiViews @@ -1202,22 +1232,24 @@ RewriteRule ^/\.well-known/carddav /nextcloud/remote.php/dav/ [R=301,L] RewriteRule ^/\.well-known/caldav /nextcloud/remote.php/dav/ [R=301,L] ### end nextcloud settings EOF + if [[ ! -e $rcdir/config/secret ]]; then + base64 $rcdir/config/secret || [[ $? == 141 ]] + fi + secret=$(cat $rcdir/config/secret) - if [[ ! -e $rcdir/config/secret ]]; then - base64 $rcdir/config/secret || [[ $? == 141 ]] - fi - secret=$(cat $rcdir/config/secret) - - # config from mailinabox - cat >$rcdir/config/config.inc.php < EOF - # todo rss subscribe to carddav plugin - m mkdir -p /var/tmp/roundcube /m/rc - m chown -R www-data.www-data /var/tmp/roundcube /m/rc - m chmod 750 /var/tmp/roundcube - # Ensure the log file monitored by fail2ban exists, or else fail2ban can't start. - # todo: setup fail2ban - # todo: setup dnssec. - # todo: check for other mailinabox things - m sudo -u www-data touch /var/log/roundcube/errors.log - - #### begin carddav install - # This is the official roundcube carddav repo. - # Install doc suggests downloading with composer, but that - # didnt work, it said some ldap package for roundcube was missing, - # but I dont want to download some extra ldap thing. - # https://github.com/blind-coder/rcmcarddav/blob/master/doc/INSTALL.md - verf=$rcdir/plugins/carddav/myversion - upgrade=false - install=false - v=4.0.0 - if [[ -e $verf ]]; then - if [[ $(cat $verf) != "$v" ]]; then + # todo, default charset an option? set to utf-8 + + m mkdir -p $rclogdir + m chmod 750 $rclogdir + m chown www-data:adm $rclogdir + # todo rss subscribe to carddav plugin + m mkdir -p $rctmpdir /m/rc + m chown -R www-data.www-data $rctmpdir /m/rc + m chmod 750 $rctmpdir + # Ensure the log file monitored by fail2ban exists, or else fail2ban can't start. + # todo: setup fail2ban + # todo: setup dnssec. + # todo: check for other mailinabox things + m sudo -u www-data touch $rclogdir/errors.log + + #### begin carddav install + # This is the official roundcube carddav repo. + # Install doc suggests downloading with composer, but that + # didnt work, it said some ldap package for roundcube was missing, + # but I dont want to download some extra ldap thing. + # https://github.com/blind-coder/rcmcarddav/blob/master/doc/INSTALL.md + verf=$rcdir/plugins/carddav/myversion + upgrade=false + install=false + v=4.0.0 + if [[ -e $verf ]]; then + if [[ $(cat $verf) != "$v" ]]; then + install=true + upgrade=true + fi + else install=true - upgrade=true fi - else - install=true - fi - if $install; then - rm -rf $rcdir/plugins/carddav - tmpd=$(mktemp -d) - m wget -nv -O $tmpd/t.tgz https://github.com/blind-coder/rcmcarddav/releases/download/v$v/carddav-v$v.tgz - cd $rcdir/plugins - tar xzf $tmpd/t.tgz - rm -rf $tmpd - chown -R www-data:www-data $rcdir/plugins/carddav - cd $rcdir/plugins/carddav - if $upgrade; then - sudo -u www-data composer.phar update --no-dev - else - sudo -u www-data composer.phar install --no-dev + if $install; then + m rm -rf $rcdir/plugins/carddav + tmpd=$(mktemp -d) + m wget -nv -O $tmpd/t.tgz https://github.com/blind-coder/rcmcarddav/releases/download/v$v/carddav-v$v.tgz + cd $rcdir/plugins + tar xzf $tmpd/t.tgz + rm -rf $tmpd + m chown -R www-data:www-data $rcdir/plugins/carddav + cd $rcdir/plugins/carddav + if $upgrade; then + m sudo -u www-data composer.phar update --no-dev + else + m sudo -u www-data composer.phar install --no-dev + fi + m chown -R root:root $rcdir/plugins/carddav + echo $v >$verf fi - chown -R root:root $rcdir/plugins/carddav - echo $v >$verf - fi - cat > $rcdir/plugins/carddav/config.inc.php <<'EOF'; + i $rcdir/plugins/carddav/config.inc.php < 'Main', 'username' => '%u', // login username 'password' => '%p', // login password - 'url' => 'https://expertpathologyreview.com/nextcloud/remote.php/carddav/addressbooks/%u/contacts', + 'url' => 'https://$domain/nextcloud/remote.php/carddav/addressbooks/%u/contacts', 'active' => true, 'readonly' => false, 'refresh_time' => '00:10:00', @@ -1310,12 +1347,19 @@ $prefs['davserver'] = array( ); ?> EOF - #### end carddav install + #### end carddav install + + cd $rcdir/plugins + if [[ ! -d html5_notifier ]]; then + m git clone https://github.com/stremlau/html5_notifier + fi + cd $rcdir/plugins/html5_notifier + m git pull --rebase - # todo: try out roundcube plugins: html5 notifier, nextcloud, thunderbird labels + # todo: try out roundcube plugins: html5 notifier, nextcloud, thunderbird labels - # Password changing plugin settings - cat $rcdir/plugins/password/config.inc.php.dist - >$rcdir/plugins/password/config.inc.php <<'EOF' + # Password changing plugin settings + cat $rcdir/plugins/password/config.inc.php.dist - >$rcdir/plugins/password/config.inc.php <<'EOF' # following are from mailinabox $config['password_minimum_length'] = 8; $config['password_db_dsn'] = 'sqlite:////m/rc/users.sqlite'; @@ -1324,22 +1368,23 @@ $config['password_dovecotpw'] = '/usr/bin/doveadm pw'; $config['password_dovecotpw_method'] = 'SHA512-CRYPT'; $config['password_dovecotpw_with_method'] = true; EOF - # so PHP can use doveadm, for the password changing plugin - m usermod -a -G dovecot www-data - m usermod -a -G mail $u + # so PHP can use doveadm, for the password changing plugin + m usermod -a -G dovecot www-data + m usermod -a -G mail $u - # so php can update passwords - m chown www-data:dovecot /m/rc/users.sqlite - m chmod 664 /m/rc/users.sqlite + # so php can update passwords + m chown www-data:dovecot /m/rc/users.sqlite + m chmod 664 /m/rc/users.sqlite - # Run Roundcube database migration script (database is created if it does not exist) - m $rcdir/bin/updatedb.sh --dir $rcdir/SQL --package roundcube - m chown www-data:www-data /m/rc/roundcube.sqlite - m chmod 664 /m/rc/roundcube.sqlite + # Run Roundcube database migration script (database is created if it does not exist) + m $rcdir/bin/updatedb.sh --dir $rcdir/SQL --package roundcube + m chown www-data:www-data $rcdb + m chmod 664 $rcdb + done # end loop over domains and rcdirs + ### begin php setup for rc ### # Enable PHP modules. m phpenmod -v php mcrypt imap - # dpkg says this is required m a2enmod proxy_fcgi setenvif fpm=$(dpkg-query -s php-fpm | sed -nr 's/^Depends:.* (php[^ ]*-fpm)( .*|$)/\1/p') # eg: php7.3-fpm @@ -1350,14 +1395,14 @@ EOF m a2dismod php$phpver # according to /install, we should set date.timezone, # but that is dumb, the system already has the right zone in - # /var/log/roundcubemail/errors.log + # $rclogdir/errors.log # todo: consider other settings in # /a/opt/mailinabox/setup/nextcloud.sh - cat >/etc/php/$phpver/cli/conf.d/30-local.ini <<'EOF' + i /etc/php/$phpver/cli/conf.d/30-local.ini <<'EOF' apc.enable_cli = 1 EOF - cat >/etc/php/$phpver/fpm/conf.d/30-local.ini <<'EOF' + i /etc/php/$phpver/fpm/conf.d/30-local.ini <<'EOF' date.timezone = "America/New_York" # for nextcloud upload_max_filesize = 2000M @@ -1365,91 +1410,103 @@ post_max_size = 2000M # install checker, nextcloud/settings/admin/overview memory_limit = 512M EOF - - # https://docs.nextcloud.com/server/19/admin_manual/configuration_server/background_jobs_configuration.html - cat >/etc/cron.d/nextcloud <<'EOF' -*/5 * * * * php -f /var/www/nextcloud/cron.php --define apc.enable_cli=1 -EOF - m systemctl restart $fpm # dunno if reload/restart is needed m systemctl reload apache2 # note bk backups are defined in crontab outside this file -fi # end nextcloud setup + ### end php setup for rc ### + +fi # end roundcube setup # * nextcloud setup if [[ $HOSTNAME == bk ]]; then + # from install checker, nextcloud/settings/admin/overview and # https://docs.nextcloud.com/server/19/admin_manual/installation/source_installation.html # curl from the web installer requirement, but i switched to cli - pi php-curl php-fileinfo php-bz2 - # install checker, nextcloud/settings/admin/overview - pi php-gmp php-bcmath php-imagick php-apcu - - cd /var/www - if [[ ! -e nextcloud/index.php ]]; then - wget https://download.nextcloud.com/server/releases/latest.zip - unzip -q latest.zip - rm -f latest.zip - chown -R www-data.www-data nextcloud - cd /var/www/nextcloud - sudo -u www-data php occ maintenance:install --database sqlite --admin-user iank --admin-pass $nextcloud_admin_pass - fi - cd /var/www/nextcloud/config + # it recommends php-file info, but that is part of php7.3-common, already got installed + # with roundcube. + m pi php-curl php-bz2 php-gmp php-bcmath php-imagick php-apcu + # https://docs.nextcloud.com/server/19/admin_manual/installation/source_installation.html cat >/etc/php/$phpver/fpm/pool.d/localwww.conf <<'EOF' [www] clear_env = no EOF - cat config.php - >tmp.php <<'EOF' + + for ((i=0; i < ${#bkdomains[@]}; i++)); do + domain=${bkdomains[i]} + ncdir=${ncdirs[i]} + ncbase=${ncdir##*/} + cd /var/www + if [[ ! -e $ncdir/index.php ]]; then + m wget -nv -N https://download.nextcloud.com/server/releases/latest.zip + m unzip -q latest.zip + m rm -f latest.zip + m chown -R www-data.www-data nextcloud + m mv nextcloud $ncdir + m cd $ncdir + m sudo -u www-data php occ maintenance:install --database sqlite --admin-user iank --admin-pass $nextcloud_admin_pass + fi + m cd $ncdir/config + if [[ ! -e config.php-orig ]]; then + m cp config.php config.php-orig + fi + cat config.php-orig - >tmp.php < 'OC_User_IMAP','arguments' => array('127.0.0.1', 143, null),),); +\$CONFIG['user_backends'] = array(array('class' => 'OC_User_IMAP','arguments' => array('127.0.0.1', 143, null),),); # based on installer check # https://docs.nextcloud.com/server/19/admin_manual/configuration_server/caching_configuration.html -$CONFIG['memcache.local'] = '\OC\Memcache\APCu'; +\$CONFIG['memcache.local'] = '\OC\Memcache\APCu'; -$CONFIG['overwrite.cli.url'] = 'https://expertpathologyreview.com/nextcloud'; -$CONFIG['htaccess.RewriteBase'] = '/nextcloud'; -$CONFIG['trusted_domains'] = array ( - 0 => 'expertpathologyreview.com', +\$CONFIG['overwrite.cli.url'] = 'https://$domain/nextcloud'; +\$CONFIG['htaccess.RewriteBase'] = '/nextcloud'; +\$CONFIG['trusted_domains'] = array ( + 0 => '$domain', ); -#$CONFIG[''] = ''; -fwrite(STDOUT, "config.php 2>/dev/null - rm tmp.php - sudo -u www-data php /var/www/nextcloud/occ maintenance:update:htaccess - list=$(sudo -u www-data php /var/www/nextcloud/occ --output=json_pretty app:list) - for app in contacts calendar user_external; do - if [[ $(printf "%s\n" "$list"| jq ".enabled.$app") == null ]]; then - m sudo -u www-data php /var/www/nextcloud/occ app:install $app - fi + m php tmp.php >config.php 2>/dev/null + m rm tmp.php + m sudo -u www-data php $ncdir/occ maintenance:update:htaccess + list=$(sudo -u www-data php $ncdir/occ --output=json_pretty app:list) + for app in contacts calendar user_external; do + if [[ $(printf "%s\n" "$list"| jq ".enabled.$app") == null ]]; then + m sudo -u www-data php $ncdir/occ app:install $app + fi + done + # https://docs.nextcloud.com/server/19/admin_manual/configuration_server/background_jobs_configuration.html + i /etc/cron.d/$ncbase <<'EOF' +*/5 * * * * php -f $ncdir/cron.php --define apc.enable_cli=1 +EOF + done fi -# * exim host conditional config +# * exim host conditional config # ** auth case $HOSTNAME in $MAIL_HOST) - cat >/etc/exim4/conf.d/auth/29_exim4-config_auth <<'EOF' + i /etc/exim4/conf.d/auth/29_exim4-config_auth <<'EOF' # from 30_exim4-config_examples plain_server: driver = plaintext @@ -1471,7 +1528,7 @@ deny domains = +local_domains !verify = recipient/callout=no_cache EOF - cat >/etc/exim4/conf.d/auth/29_exim4-config_auth <<'EOF' + i /etc/exim4/conf.d/auth/29_exim4-config_auth <<'EOF' dovecot_plain: driver = dovecot public_name = PLAIN @@ -1485,9 +1542,7 @@ esac case $HOSTNAME in # ** $MAIL_HOST|bk) $MAIL_HOST|bk) - dovecot-setup - m systemctl enable dovecot - m systemctl restart dovecot + cat >>/etc/exim4/update-exim4.conf.conf </etc/exim4/host_local_deny_exceptions <<'EOF' + i /etc/exim4/host_local_deny_exceptions <<'EOF' mail.fsf.org *.posteo.de EOF @@ -1567,7 +1615,7 @@ EOF # and also have mail.iankelling.org whitelisted as a relay domain. # I could avoid that if I changed this to submit to 587 with a # password like a standard mua. - cat >/etc/exim4/conf.d/router/190_exim4-config_fsfsmarthost <<'EOF' + i /etc/exim4/conf.d/router/190_exim4-config_fsfsmarthost <<'EOF' # smarthost for fsf mail # ian: copied from /etc/exim4/conf.d/router/200_exim4-config_primary, and added senders = and # replaced DCsmarthost with mail.fsf.org @@ -1632,8 +1680,6 @@ EOF /a/exe/cedit mail /etc/dnsmasq-servers.conf <<'EOF' || [[ $? == 1 ]] server=/mail.iankelling.org/127.0.1.1 EOF - reifactive dnsmasq nscd - # I used to use debconf-set-selections + dpkg-reconfigure, # which then updates this file # but the process is slower than updating it directly and then I want to set other things in @@ -1668,7 +1714,7 @@ EOF bk) echo bk.b8.nz > /etc/mailname - cat >/etc/myexim4/conf.d/router/180_vpnmanual <<'EOF' + i /etc/myexim4/conf.d/router/180_vpnmanual <<'EOF' # copied from dnslookup, altered domains, added route_list, # changed driver, removed ignore_target_hosts since it # relies on a later defined macro @@ -1682,17 +1728,14 @@ vpnmanual: no_more EOF - tmpdir=$(mktemp -d) - cp -a /etc/init.d/exim4 $tmpdir/exim4in - sed -i -f - $tmpdir/exim4in <<'EOF' + sed -r -f - /etc/init.d/exim4 <<'EOF' | i /etc/init.d/exim4in s,/etc/default/exim4,/etc/default/exim4in,g s,/run/exim4/exim.pid,/run/exim4/eximin.pid,g +s,(^[ #]*Provides:).*,\1 exim4in, +s,(^[ #]*NAME=).*,\1"exim4in", EOF - tmp=$(rsync -ic $tmpdir/* /etc/init.d) - rm -rf $tmpdir - - tmpdir=$(mktemp -d) - cat >$tmpdir/alwaysrestart.conf <<'EOF' + chmod +x /etc/init.d/exim4in + i /etc/systemd/system/exim4in.service.d/alwaysrestart.conf <<'EOF' [Unit] # needed to continually restart StartLimitIntervalSec=0 @@ -1702,14 +1745,8 @@ Restart=always # time to sleep before restarting a service RestartSec=1 EOF - mkdir -p /etc/systemd/system/exim4in.service.d - tmp+=$(rsync -ic $tmpdir/* /etc/systemd/system/exim4in.service.d) - if [[ $tmp ]]; then - printf "rsync to /etc/systemd/system/exim4in.service.d\n%s\n" "$tmp" - m systemctl daemon-reload - fi - cat >/etc/default/exim4in <<'EOF' + i /etc/default/exim4in <<'EOF' # defaults but no queue runner and alternate config dir QUEUERUNNER='no' COMMONOPTIONS='-oP /run/exim4/eximin.pid' @@ -1717,12 +1754,12 @@ UPEX4OPTS='-d /etc/myexim4' EOF # dkim, client passwd file - rsync -ahhi --chown=root:Debian-exim --chmod=0640 \ + m rsync -ahhi --chown=root:Debian-exim --chmod=0640 \ /p/c/machine_specific/bk/filesystem/etc/exim4/* /etc/exim4 cat >>/etc/exim4/update-exim4.conf.conf </dev/null ||: - m systemctl stop mailclean.timer &>/dev/null ||: - m systemctl disable openvpn-client-mail@mail - m systemctl stop openvpn-client-mail@mail cat >>/etc/exim4/update-exim4.conf.conf </etc/mailname + hostname -A|awk '{print $1}' |i /etc/mailname cat >>/etc/exim4/update-exim4.conf.conf <>/etc/myexim4/update-exim4.conf.conf <<'EOF' dc_eximconfig_configtype='smarthost' -dc_smarthost='nn.b8.nz' +dc_smarthost='10.173.8.2' EOF ;;& bk) @@ -1788,7 +1817,7 @@ EOF ;; $MAIL_HOST) # for bk, we have a exim4in.service that will do this for us. - update-exim4.conf -d /etc/myexim4 + m update-exim4.conf -d /etc/myexim4 ;; esac @@ -1810,24 +1839,15 @@ if [[ -e /nocow ]]; then if ! grep -Fx "/nocow/exim4 /var/spool/exim4 none bind 0 0" /etc/fstab; then echo "/nocow/exim4 /var/spool/exim4 none bind 0 0" >>/etc/fstab fi - reload=false - tmpdir=$(mktemp -d) - cat >$tmpdir/override.conf <<'EOF' + i /etc/systemd/system/exim4.service.d/override.conf <<'EOF' [Unit] # without this on exim, we get these kind of errors in paniclog on shutdown: # Failed to create spool file /var/spool/exim4//input//1jCLxz-0008V4-V9-D: Permission denied After=local-fs.target After=network.target EOF - mkdir -p /etc/systemd/system/exim4.service.d - tmp=$(rsync -ic $tmpdir/* /etc/systemd/system/exim4.service.d) - rm -rf $tmpdir - if [[ $tmp ]]; then - printf "rsync to /etc/systemd/system/exim4.service.d\n%s\n" "$tmp" - m systemctl daemon-reload - fi if ! mountpoint -q $sdir; then - m systemctl stop exim4 + stopifactive exim4 exim4in if [[ -L $sdir ]]; then m rm $sdir fi @@ -1855,7 +1875,7 @@ if [[ ! $uid ]]; then m adduser --uid 608 --system --group --quiet --home /var/spool/exim4 \ --no-create-home --disabled-login --force-badname Debian-exim elif [[ $uid != 608 ]]; then - m systemctl stop exim4 ||: + stopifactive exim4 exim4in m usermod -u 608 Debian-exim m groupmod -g 608 Debian-exim m usermod -g 608 Debian-exim @@ -1863,42 +1883,35 @@ elif [[ $uid != 608 ]]; then m find / /nocow -path ./var/tmp -prune -o -xdev -gid $gid -execdir chgrp -h 608 {} + fi +# * start / stop services +reifactive dnsmasq nscd +if $reload; then + m systemctl daemon-reload +fi +m systemctl start mailcert +sre mailcert.timer -# * start exim / spamassassin - -# start spamassassin before exim, stop it after so if we are -# transitioning from being mail_host to not, we dont have exim -# complaining about no spamassassin. case $HOSTNAME in $MAIL_HOST|bk) - m systemctl enable spamassassin - m systemctl start spamassassin + # If mailvpn has changes, id rather manually restart it, id rather + # not restart and lose connectivity. + sstart mailnn mailvpn + # start spamassassin/dovecot before exim. + sre dovecot spamassassin + sstart mailclean.timer + ;; + *) + soff mailclean.timer + soff mailclean.timer dovecot spamassassin mailvpn mailnn ;; esac -if systemctl is-active exim4 >/dev/null; then - m systemctl reload exim4 -else - m systemctl start exim4 -fi - +sre exim4 case $HOSTNAME in - $MAIL_HOST) : ;; - bk) - if systemctl is-active exim4in >/dev/null; then - m systemctl reload exim4in - else - m systemctl start exim4in - fi - m systemctl enable exim4in - ;; - *) - m systemctl stop spamassassin - m systemctl disable spamassassin - ;; + bk) sre exim4in ;; esac # * mail monitoring / testing @@ -1906,9 +1919,11 @@ esac case $HOSTNAME in $MAIL_HOST|bk) # note: cronjob "ian" also does some important monitoring + # todo: setup test mail cron for 3rd host cat >/etc/cron.d/mailtest </usr/local/bin/send-test-forward <&2; exit 1; fi +shopt -s inherit_errexit 2>/dev/null ||: # ignore fail in bash < 4.4 set -eE -o pipefail -trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR +trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?. PIPESTATUS: ${PIPESTATUS[*]}" >&2' ERR -if [[ $EUID != 1000 ]]; then - echo "$0: error run as normal user" >&2 - exit 1 -fi +[[ $EUID == 0 ]] || exec sudo -E "${BASH_SOURCE[0]}" "$@" shopt -s nullglob - # We run this cronjob along with sending the test email every 5 minutes, # so give it 1 minute to arrive, then if the latest email is older than # 7 minutes, the last 2 haven't arrived in a reasonable amount of time. @@ -22,9 +21,9 @@ fi folder=/m/md/l/testignore/new if [[ $HOSTNAME == bk ]]; then - /m/md/expertpathologyreview.com/testignore/new + folder=/m/md/expertpathologyreview.com/testignore/cur fi -find $folder -type f -mtime +1 -delete +find $folder -type f -mmin +300 -delete cd $folder @@ -43,5 +42,5 @@ now=$(date +%s) limit=$(( now - 60 * min_limit )) if (( last_sec <= limit )); then - echo $HOSTNAME mailtest failure + echo $HOSTNAME mailtest failure $(date -d @$last_sec +'%a %m-%d %H:%M') fi diff --git a/radicale-setup b/radicale-setup index 95391fa..0334a25 100755 --- a/radicale-setup +++ b/radicale-setup @@ -38,16 +38,16 @@ cat >$d/override.conf </dev/null) ]]; then v conflink newer filesystem files - chars+=("CONFLINK!") + chars+=("CONFLINK") break fi @@ -143,7 +143,7 @@ write-status() { fi if (( $(date -d "$(git log --diff-filter=ACR --format=%aD -1)" +%s) > fsec )); then v conflink: newer files checked in to git - chars+=("CONFLINK!") + chars+=("CONFLINK") break fi @@ -153,7 +153,7 @@ write-status() { done < <(git ls-files -o --exclude-standard) if [[ ${untracked[0]} && $(find "${untracked[@]}" -mmin $fminplus -type f -print -quit) ]]; then v conflink: untracked in $d - chars+=("CONFLINK!") + chars+=("CONFLINK") break fi done @@ -162,7 +162,7 @@ write-status() { fi if [[ ! -e $f || $(<$f) != 0 ]]; then v conflink: last run not found or failed - chars+=("CONFLINK!") + chars+=("CONFLINK") break fi done @@ -195,7 +195,7 @@ write-status() { if [[ $MAIL_HOST == "$HOSTNAME" ]]; then bbkmsg= if [[ $(systemctl is-active btrbk.timer) != active ]]; then - chars+=("BTRBK.TIMER!") + chars+=("BTRBK.TIMER") bbkmsg="btrbk.timer not enabled" fi lo -60 btrbk.timer $bbkmsg @@ -220,7 +220,7 @@ write-status() { fi done if (( maxtime < now - 60*60 )); then - chars+=("OLD-SNAP!") + chars+=("OLD-SNAP") snapshotmsg="/o snapshot older than 1 hour" fi lo -1 old-snapshot $snapshotmsg -- 2.30.2