X-Git-Url: https://iankelling.org/git/?a=blobdiff_plain;f=mail-setup;h=e753fbe9670ea27f7837b14be795d1ce5d7c95af;hb=c300392f92092b3a89281df462d42710f25d0351;hp=4619b2e2c8ed0db1977f0b84434f4c29837f91a8;hpb=110b15e4a7b19cc520d2aa3f2584f4213b94f5b7;p=distro-setup diff --git a/mail-setup b/mail-setup index 4619b2e..e753fbe 100755 --- a/mail-setup +++ b/mail-setup @@ -3,6 +3,27 @@ # Copyright (C) 2019 Ian Kelling # SPDX-License-Identifier: AGPL-3.0-or-later +# Things I tend to forget. on MAIL_HOST, daemon runs with /etc/exim4/my.conf, +# due to /etc/default/exim4 containing: +# COMMONOPTIONS='-C /etc/exim4/my.conf' +# UPEX4OPTS='-o /etc/exim4/my.conf' +# +# The non-daemon config +# gets generated from this script calling update-exim4.conf -d /etc/myexim4 +# which has log path +# log_file_path = /var/log/exim4/my%s +# +# eximbackup folder is /bu/md +# it is cleaned up by mail-backup-clean, which is run by btrbk-run + +# shellcheck disable=SC2254 # makes for a lot of unneeded quotes + + +# perusing through /el/mainlog without test messages: +# &!testignore|jtuttle| +# +#&! testignore|jtuttle|eximbackup|/usr/sbin/exim4 -bpu + # todo: check new macro DKIM_TIMESTAMPS # todo: check if REMOTE_SMTP_INTERFACE or REMOTE_SMTP_TRANSPORTS_HEADERS_REMOVE can simplify my or fsfs config @@ -282,7 +303,7 @@ fi # * functions & constants -pre="${0##*/}:" +pre="${0##*/}:${SSH_CLIENT:+ $HOSTNAME:}" m() { printf "$pre %s\n" "$*"; "$@"; } e() { printf "$pre %s\n" "$*"; } err() { printf "$pre %s\n" "$*" >&2; exit 1; } @@ -292,7 +313,7 @@ reload=false if [[ -e /var/local/mail-setup-reload ]]; then reload=true fi -i() { # install file +u() { # update file. note: duplicated in brc local tmp tmpdir dest="$1" local base="${dest##*/}" local dir="${dest%/*}" @@ -300,13 +321,13 @@ i() { # install file # dest has a directory component mkdir -p "$dir" fi - ir=false # i result + ur=false # u result tmpdir=$(mktemp -d) cat >$tmpdir/"$base" tmp=$(rsync -ic $tmpdir/"$base" "$dest") if [[ $tmp ]]; then printf "%s\n" "$tmp" - ir=true + ur=true if [[ $dest == /etc/systemd/system/* ]]; then touch /var/local/mail-setup-reload reload=true @@ -328,9 +349,19 @@ soff () { done } sre() { + local enabled for service; do m systemctl restart $service - m systemctl enable $service; + # Optimization for exim, + # is-enabled: 0m0.015s + # enable: 0m0.748s + # It is related to this message: + # exim4.service is not a native service, redirecting to systemd-sysv-install. + # Executing: /lib/systemd/systemd-sysv-install enable exim4 + enabled=$(systemctl is-enabled $service 2>/dev/null ||:) + if [[ $enabled != enabled ]]; then + m systemctl enable $service + fi done } mailhost() { @@ -432,7 +463,7 @@ fi # * Mail clean cronjob -i /etc/systemd/system/mailclean.timer <<'EOF' +u /etc/systemd/system/mailclean.timer <<'EOF' [Unit] Description=Run mailclean daily @@ -443,7 +474,7 @@ OnCalendar=monthly WantedBy=timers.target EOF -i /etc/systemd/system/mailclean.service < EOF -i /etc/fail2ban/jail.d/exim.local <<'EOF' +u /etc/fail2ban/jail.d/exim.local <<'EOF' [exim] enabled = true port = 25,587 @@ -1030,7 +1091,7 @@ banaction = iptables-exim # 10.173.8.1 = non-nn net ignoreip = 209.51.188.13 2001:470:142::13 209.51.188.92 2001:470:142:3::10 72.14.176.105 2600:3c00:e000:280::2 10.173.8.1 EOF -if $ir; then +if $ur; then m systemctl restart fail2ban fi @@ -1136,26 +1197,26 @@ rm -fv /etc/exim4/conf.d/retry/37_retry cat >/etc/exim4/conf.d/retry/17_retry <<'EOF' # Retry fast for my own domains -iankelling.org * F,1d,4m;F,14d,1h -amnimal.ninja * F,1d,4m;F,14d,1h -expertpathologyreview.com * F,1d,4m;F,14d,1h -je.b8.nz * F,1d,4m;F,14d,1h -zroe.org * F,1d,4m;F,14d,1h -eximbackup.b8.nz * F,1d,4m;F,14d,1h +iankelling.org * F,1d,1m;F,14d,1h +amnimal.ninja * F,1d,1m;F,14d,1h +expertpathologyreview.com * F,1d,1m;F,14d,1h +je.b8.nz * F,1d,1m;F,14d,1h +zroe.org * F,1d,1m;F,14d,1h +eximbackup.b8.nz * F,1d,1m;F,14d,1h # The spec says the target domain will be used for temporary host errors, # but i've found that isn't correct, the hostname is required # at least sometimes. -nn.b8.nz * F,1d,4m;F,14d,1h -defaultnn.b8.nz * F,1d,4m;F,14d,1h -mx.iankelling.org * F,1d,4m;F,14d,1h -bk.b8.nz * F,1d,4m;F,14d,1h -eggs.gnu.org * F,1d,4m;F,14d,1h -fencepost.gnu.org * F,1d,4m;F,14d,1h +nn.b8.nz * F,1d,1m;F,14d,1h +defaultnn.b8.nz * F,1d,1m;F,14d,1h +mx.iankelling.org * F,1d,1m;F,14d,1h +bk.b8.nz * F,1d,1m;F,14d,1h +eggs.gnu.org * F,1d,1m;F,14d,1h +fencepost.gnu.org * F,1d,1m;F,14d,1h # afaik our retry doesnt need this, but just using everything -mx.amnimal.ninja * F,1d,4m;F,14d,1h -mx.expertpathologyreview.com * F,1d,4m;F,14d,1h +mx.amnimal.ninja * F,1d,1m;F,14d,1h +mx.expertpathologyreview.com * F,1d,1m;F,14d,1h mail.fsf.org * F,1d,15m;F,14d,1h @@ -1177,27 +1238,16 @@ cd /etc/exim4 for f in *-private.pem; do echo ${f%-private.pem} done -} | i /etc/exim4/conf.d/my-dkim-domains - -if grep -Fq REMOTE_SMTP_SMARTHOST_TLS_VERIFY_HOSTS \ - /etc/exim4/conf.d/transport/10_exim4-config_transport-macros; then - cat >/etc/exim4/conf.d/transport/11_iank <<'EOF' -# This unsets the default macro defined in on t11 in -# /etc/exim4/conf.d/transport/10_exim4-config_transport-macros -# It seems like a very odd choice that this has become -# the default in t11. Normal smarthost clients use username/password -# auth. Oh well. -REMOTE_SMTP_SMARTHOST_TLS_VERIFY_HOSTS == -EOF -else - rm -f /etc/exim4/conf.d/transport/11_iank -fi +} | u /etc/exim4/conf.d/my-dkim-domains + +rm -f /etc/exim4/conf.d/transport/11_iank cat >/etc/exim4/conf.d/main/000_local <<'EOF' MAIN_TLS_ENABLE = true # require tls connections for all smarthosts -REMOTE_SMTP_SMARTHOST_HOSTS_REQUIRE_TLS = * +REMOTE_SMTP_SMARTHOST_HOSTS_REQUIRE_TLS = ! nn.b8.nz +REMOTE_SMTP_SMARTHOST_HOSTS_AVOID_TLS = nn.b8.nz # debian exim config added this in 2016 or so? # it's part of the smtp spec, to limit lines to 998 chars @@ -1210,7 +1260,7 @@ REMOTE_SMTP_SMARTHOST_HOSTS_REQUIRE_TLS = * IGNORE_SMTP_LINE_LENGTH_LIMIT = true # more verbose logs. used to use +all, but made it less for more efficiency. -MAIN_LOG_SELECTOR = -tls_cipher -tls_certificate_verified +pid +received_recipients +received_sender +sender_on_delivery +return_path_on_delivery +msg_id_created +subject +address_rewrite +smtp_confirmation +MAIN_LOG_SELECTOR = -skip_delivery -tls_cipher -tls_certificate_verified +all_parents +address_rewrite +arguments +deliver_time +pid +queue_time +queue_time_overall +received_recipients +received_sender +return_path_on_delivery +sender_on_delivery +smtp_confirmation +subject # Based on spec, seems like a good idea to be nice. smtp_return_error_details = true @@ -1241,6 +1291,7 @@ DKIM_SIGN_HEADERS = mime-version:in-reply-to:references:from:date:subject:to domainlist local_hostnames = ! je.b8.nz : ! bk.b8.nz : *.b8.nz : b8.nz +# note: most of these are duplicated in spamassassin config hostlist iank_trusted = <; \ # veth0 10.173.8.1 ; \ @@ -1269,7 +1320,7 @@ delay_warning_condition = ${if or {\ # enable 587 in addition to the default 25, so that # i can send mail where port 25 is firewalled by isp -daemon_smtp_ports = 25 : 587 +daemon_smtp_ports = 25 : 587 : 10025 # default of 25, can get stuck when catching up on mail smtp_accept_max = 400 smtp_accept_reserve = 100 @@ -1300,7 +1351,7 @@ fi rm -fv /etc/exim4/rcpt_local_acl # old path -i /etc/exim4/conf.d/local_deny_exceptions_acl <<'EOF' +u /etc/exim4/conf.d/local_deny_exceptions_acl <<'EOF' # This acl already exists in rcpt, this just makes it more widespread. # See the comment there for its rationale. The reason it needs to be # more widespread is that I've turned on sender verification, but cron @@ -1317,7 +1368,7 @@ EOF rm -fv /etc/exim4/data_local_acl # old path -i /etc/exim4/conf.d/data_local_acl <<'EOF' +u /etc/exim4/conf.d/data_local_acl <<'EOF' # Except for the "condition =", this was # a comment in the check_data acl. The comment about this not # being suitable has been changed in newer exim versions. The only thing @@ -1355,14 +1406,16 @@ EOF # old file -rm -fv /etc/exim4/conf.d/router/8{8,9}0_backup_copy /etc/exim4/conf.d/router/865_backup_redir +rm -fv /etc/exim4/conf.d/router/8{8,9}0_backup_copy \ + /etc/exim4/conf.d/router/865_backup_redir \ + /etc/exim4/conf.d/router/870_backup_local # It is important for this to exist everywhere except in MAIL_HOST # non-nn config. Previously, just had it in the nn-config on MAIL_HOST, # but that is a problem if we change mail host and still have something # in the queue which was destined for this router, but hosts were # unreachable, the routers will be reevaluated on the next retry. -i /etc/exim4/conf.d/router/170_backup_copy </etc/exim4/update-exim4.conf.conf <<'EOF' # default stuff, i havent checked if its needed dc_minimaldns='false' -dc_relay_nets='' CFILEMODE='644' dc_use_split_config='true' dc_mailname_in_oh='true' @@ -1740,8 +1792,9 @@ if mailhost; then # # setup chgrp www-data in ./conflink pi-nostart radicale + m usermod -a -G radicale iank - i /etc/systemd/system/radicale.service.d/override.conf < @@ -2194,27 +2247,28 @@ if [[ $HOSTNAME == bk ]]; then ### begin composer install # https://getcomposer.org/doc/faqs/how-to-install-composer-programmatically.md - # cd $(mktemp -d) - # sum="$(wget -q -O - https://composer.github.io/installer.sig)" - # m php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" - # if [[ $sum != $(php -r "echo hash_file('sha384', 'composer-setup.php');") ]]; then - # echo 'ERROR: Invalid composer installer checksum' >&2 - # rm -fv composer-setup.php - # exit 1 - # fi - # m php composer-setup.php --quiet - # rm -fv composer-setup.php - # m mv composer.phar /usr/local/bin - - # the above method gets composer2, carddav plugin at least doesnt work with that - # yet, it was just released 10-24-2020. - m cd /usr/local/bin - m wget -nv -N https://getcomposer.org/composer-1.phar - chmod +x composer-1.phar + cd /usr/local/bin + EXPECTED_CHECKSUM="$(php -r 'copy("https://composer.github.io/installer.sig", "php://stdout");')" + php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" + ACTUAL_CHECKSUM="$(php -r "echo hash_file('sha384', 'composer-setup.php');")" + + if [ "$EXPECTED_CHECKSUM" != "$ACTUAL_CHECKSUM" ] + then + >&2 echo 'ERROR: Invalid installer checksum' + rm composer-setup.php + exit 1 + fi + + php composer-setup.php --quiet + rm composer-setup.php + + # based on error when running composer + mkdir -p /var/www/.composer + chown www-data:www-data /var/www/.composer + ### end composer install rcdirs=(/usr/local/lib/rcexpertpath /usr/local/lib/rcninja) - ncdirs=(/var/www/ncninja) ncdirs=(/var/www/ncexpertpath /var/www/ncninja) # point debian cronjob to our local install, preventing daily cron error @@ -2246,11 +2300,9 @@ if [[ $HOSTNAME == bk ]]; then rcdir=${rcdirs[i]} rcbase=${rcdir##*/} ncdir=${ncdirs[i]} - myncdir=/root/${ncdir##*/} - mkdir -p $myncdir # copied from debians cronjob - i /etc/cron.d/$rcbase </dev/null @@ -2310,7 +2362,7 @@ EOF rctmpdir=/var/tmp/$rcbase rcdb=/m/rc/$rcbase.sqlite # config from mailinabox - i $rcdir/config/config.inc.php <$verf @@ -2405,7 +2457,7 @@ EOF # About categories, see https://www.davx5.com/tested-with/nextcloud # https://github.com/blind-coder/rcmcarddav/blob/master/doc/GROUPS.md - i $rcdir/plugins/carddav/config.inc.php <$myncdir/tmp.php <$myncdir/tmp.php <config.php + e running php $myncdir/tmp.php + # note: we leave it around place for debugging + # shellcheck disable=SC2024 # intended + sudo -u www-data php $myncdir/tmp.php >config.php + fi cd $ncdir m sudo -u www-data php occ maintenance:update:htaccess list=$(sudo -u www-data php $ncdir/occ --output=json_pretty app:list) @@ -2585,7 +2644,7 @@ EOF m sudo -u www-data php occ app:install $app fi done - i /etc/systemd/system/$ncbase.service <>/etc/exim4/update-exim4.conf.conf <>/etc/exim4/update-exim4.conf.conf <>/etc/exim4/conf.d/main/000_local <>/etc/exim4/update-exim4.conf.conf <>/etc/fstab fi - i /etc/systemd/system/exim4.service.d/override.conf <<'EOF' + u /etc/systemd/system/exim4.service.d/override.conf <<'EOF' [Unit] # without local-fs on exim, we get these kind of errors in paniclog on shutdown: # Failed to create spool file /var/spool/exim4//input//1jCLxz-0008V4-V9-D: Permission denied @@ -3410,14 +3480,20 @@ if $reload; then m systemctl daemon-reload fi -# checking bhost_t is redundant, but could help us catch errors. -if $bhost_t || [[ -e /etc/wireguard/wghole.conf ]]; then - # todo: in mail-setup, we have a static list of backup hosts, not *y - m systemctl --now enable wg-quick@wghole +# optimization, this only needs to run once. +if [[ ! -e /sys/class/net/wghole ]]; then + # checking bhost_t is redundant, but could help us catch errors. + if $bhost_t || [[ -e /etc/wireguard/wghole.conf ]]; then + # todo: in mail-setup, we have a static list of backup hosts, not *y + m systemctl --now enable wg-quick@wghole + fi fi -sysd-prom-fail-install epanicclean -m systemctl --now enable epanicclean +# optimization, this only needs to be run once +if [[ ! -e /var/lib/prometheus/node-exporter/exim_paniclog.prom ]]; then + sysd-prom-fail-install epanicclean + m systemctl --now enable epanicclean +fi case $HOSTNAME in je) @@ -3428,8 +3504,11 @@ case $HOSTNAME in ;; esac -m /a/bin/ds/mail-cert-cron -1 -sre mailcert.timer +# optimization, this only needs to run once. +if [[ ! -e /etc/exim4/fullchain.pem ]]; then + m /a/bin/ds/mail-cert-cron -1 + m systemctl --now enable mailcert.timer +fi case $HOSTNAME in $MAIL_HOST|bk) @@ -3466,9 +3545,11 @@ case $HOSTNAME in $MAIL_HOST|bk|je) # start spamassassin/dovecot before exim. sre dovecot spamassassin - # need to wait a bit before restarting exim, else I - # get a paniclog entry like: spam acl condition: all spamd servers failed - sleep 3 + # Wait a bit before restarting exim, else I get a paniclog entry + # like: spam acl condition: all spamd servers failed. But I'm tired + # of waiting. I'll deal with this some other way. + # + # sleep 3 m systemctl --now enable mailclean.timer ;;& $MAIL_HOST) @@ -3527,7 +3608,7 @@ case $HOSTNAME in # note: cronjob "ian" also does some important monitoring # todo: this will sometimes cause an alert because mailtest-check will run # before we have setup network namespace and spamassassin - i /etc/cron.d/mailtest <