X-Git-Url: https://iankelling.org/git/?a=blobdiff_plain;f=mail-setup;h=65682c2971f513a64b29c207633ca4888397725c;hb=8796be333861fd2edc6c8b9b4c22307c11fce409;hp=13f99478e6a7ffb2d85d59a79984522a7dbac57a;hpb=62dede3e7ad2c0ee566145f3efabf1fd23df46a7;p=distro-setup diff --git a/mail-setup b/mail-setup index 13f9947..65682c2 100755 --- a/mail-setup +++ b/mail-setup @@ -3,13 +3,51 @@ # Copyright (C) 2019 Ian Kelling # SPDX-License-Identifier: AGPL-3.0-or-later -# todo: sandbox / harden exim: -# restrict its filesystem access from within systemd +# Things I tend to forget. on MAIL_HOST, daemon runs with /etc/exim4/my.conf, +# due to /etc/default/exim4 containing: +# COMMONOPTIONS='-C /etc/exim4/my.conf' +# UPEX4OPTS='-o /etc/exim4/my.conf' +# +# The non-daemon config +# gets generated from this script calling update-exim4.conf -d /etc/myexim4 +# which has log path +# log_file_path = /var/log/exim4/my%s +# +# eximbackup folder is /bu/md +# it is cleaned up by mail-backup-clean, which is run by btrbk-run + +# shellcheck disable=SC2254 # makes for a lot of unneeded quotes + + +# perusing through /el/mainlog without test messages: +# &!testignore|jtuttle| +# +#&! testignore|jtuttle|eximbackup|/usr/sbin/exim4 -bpu + +# todo: check new macro DKIM_TIMESTAMPS + +# todo: check if REMOTE_SMTP_INTERFACE or REMOTE_SMTP_TRANSPORTS_HEADERS_REMOVE can simplify my or fsfs config + +# todo: max line length macro changed in t11. look into it +# todo: check that all macros we use are still valid in t11 + +# todo: setup an alert for bouncing test emails. + +# todo: bounces to my fsf mail can come from fsf@iankelling.org, +# think about making bounces go from the original address. + +# todo: add a prometheus alert for dovecot. + +# todo: handle errors like this: +# Mar 02 12:44:26 kw systemd[1]: exim4.service: Found left-over process 68210 (exim4) in control group while starting unit. Ignoring. +# Mar 02 12:44:26 kw systemd[1]: This usually indicates unclean termination of a previous run, or service implementation deficiencies. +#eg: on eggs, on may 1st, ps grep for exim, 2 daemons running. 1 leftover from a month ago +#Debian-+ 1954 1 0 36231 11560 4 Apr02 ? 00:40:25 /usr/sbin/exim4 -bd -q30m +#Debian-+ 23058 1954 0 36821 10564 0 20:38 ? 00:00:00 /usr/sbin/exim4 -bd -q30m # todo: harden dovecot. need to do some research. one way is for it to only listen on a wireguard vpn interface, so only clients that are on the vpn can access it. # todo: consider hardening cups listening on 0.0.0.0 # todo: stop/disable local apache, and rpc.mountd, and kdeconnect when not in use. -# todo: check that spamd and unbound only listen locally. # todo: hosts should only allow external mail that is authed and # destined for backup route. it is a minor issue since traffic is @@ -21,12 +59,6 @@ # todo: run mailping test after running, or otherwise # clear out terminal alert -# todo: reinstall bk with bigger filesystem - -# todo: on bk, dont send email if mailvpn is not up - -# todo: mailtest-check should check on bk too - # todo: disable postgrey # todo: in testforward-check, we should also look @@ -124,7 +156,8 @@ if [[ -s /usr/local/lib/err ]]; then elif [[ -s /a/bin/errhandle/err ]]; then source /a/bin/errhandle/err else - err "no err tracing script found" + echo "no err tracing script found" + exit 1 fi source /a/bin/distro-functions/src/identify-distros source /a/bin/distro-functions/src/package-manager-abstractions @@ -139,7 +172,7 @@ fi [[ $EUID == 0 ]] || exec sudo -E "${BASH_SOURCE[0]}" "$@" - +# note, this is hardcoded in /etc/exim4/conf.d/main/000_local u=$(id -nu 1000) @@ -168,9 +201,10 @@ fi # background: dovecot does not yet have ocsp stapling support # reference: https://community.letsencrypt.org/t/simple-guide-using-lets-encrypt-ssl-certs-with-dovecot/2921 # -# for phone, k9mail, same thing but username alerts, pass in ivy-pass. +# for phone, k9mail, fdroid, same thing but username alerts, pass in ivy-pass. # also, bk.b8.nz for secondary alerts, username is iank. same alerts pass. -# fetching mail settings: folder poll frequency 10 minutes +# fetching mail settings: folder poll frequency 10 minutes. +# account settings, fetching mail, push folders: All. Then disable the persistent notification. ####### @@ -269,7 +303,7 @@ fi # * functions & constants -pre="${0##*/}:" +pre="${0##*/}:${SSH_CLIENT:+ $HOSTNAME:}" m() { printf "$pre %s\n" "$*"; "$@"; } e() { printf "$pre %s\n" "$*"; } err() { printf "$pre %s\n" "$*" >&2; exit 1; } @@ -279,17 +313,21 @@ reload=false if [[ -e /var/local/mail-setup-reload ]]; then reload=true fi -i() { # install file +u() { # update file. note: duplicated in brc local tmp tmpdir dest="$1" local base="${dest##*/}" - mkdir -p ${dest%/*} - ir=false # i result + local dir="${dest%/*}" + if [[ $dir != "$base" ]]; then + # dest has a directory component + mkdir -p "$dir" + fi + ur=false # u result tmpdir=$(mktemp -d) cat >$tmpdir/"$base" tmp=$(rsync -ic $tmpdir/"$base" "$dest") if [[ $tmp ]]; then printf "%s\n" "$tmp" - ir=true + ur=true if [[ $dest == /etc/systemd/system/* ]]; then touch /var/local/mail-setup-reload reload=true @@ -311,9 +349,19 @@ soff () { done } sre() { + local enabled for service; do m systemctl restart $service - m systemctl enable $service; + # Optimization for exim, + # is-enabled: 0m0.015s + # enable: 0m0.748s + # It is related to this message: + # exim4.service is not a native service, redirecting to systemd-sysv-install. + # Executing: /lib/systemd/systemd-sysv-install enable exim4 + enabled=$(systemctl is-enabled $service 2>/dev/null ||:) + if [[ $enabled != enabled ]]; then + m systemctl enable $service + fi done } mailhost() { @@ -337,7 +385,6 @@ stopifactive() { mxhost=mx.iankelling.org mxport=587 -forward=$u@$mxhost # old setup. left as comment for example # mxhost=mail.messagingengine.com @@ -380,11 +427,17 @@ EOF fi # light version of exim does not have sasl auth support. -pi-nostart exim4 exim4-daemon-heavy spamassassin openvpn unbound clamav-daemon wireguard +# note: for bitfolk hosts, unbound has important config with conflink. +pi-nostart exim4 exim4-daemon-heavy spamassassin unbound clamav-daemon wireguard # note: pyzor debian readme says you need to run some initialization command # but its outdated. pi spf-tools-perl p0f postgrey pyzor razor jq moreutils certbot fail2ban +case $HOSTNAME in + je) : ;; + # not included due to using wireguard: openvpn + *) pi wget git unzip iptables ;; +esac # bad packages that sometimes get automatically installed pu openresolv resolvconf @@ -410,7 +463,7 @@ fi # * Mail clean cronjob -i /etc/systemd/system/mailclean.timer <<'EOF' +u /etc/systemd/system/mailclean.timer <<'EOF' [Unit] Description=Run mailclean daily @@ -421,7 +474,7 @@ OnCalendar=monthly WantedBy=timers.target EOF -i /etc/systemd/system/mailclean.service <&2 - exit 1 - fi +u /etc/spamassassin/my_thishost.cf <<'EOF' +# note: these are duplicated in exim config +# veth0/1 # bk bk_ip6 +internal_networks 10.173.8.1/32 10.173.8.2/32 85.119.83.50/32 2001:ba8:1f1:f0c9::2 +trusted_networks 10.173.8.1/32 10.173.8.2/32 85.119.83.50/32 2001:ba8:1f1:f0c9::2 +EOF + ;; -esac + je) + u /etc/spamassassin/my_thishost.cf <<'EOF' +# note: these are duplicated in exim config +# veth0/1 # je je_ipv6 +internal_networks 10.173.8.1/32 10.173.8.2/32 85.119.82.128/32 2001:ba8:1f1:f09d::2/128 +trusted_networks 10.173.8.1/32 10.173.8.2/32 85.119.82.128/32 2001:ba8:1f1:f09d::2/128 +EOF + ;; + *) + u /etc/spamassassin/my_thishost.cf <<'EOF' +# note: these are duplicated in exim config +# veth0/1 # li li_ip6 +internal_networks 10.173.8.1/32 10.173.8.2/32 72.14.176.105/32 2600:3c00::f03c:91ff:fe6d:baf8/128 +trusted_networks 10.173.8.1/32 10.173.8.2/32 72.14.176.105/32 2600:3c00::f03c:91ff:fe6d:baf8/128 +EOF +;; + esac + +##### end spamassassin config + + +# * Update mail cert + + +## needed only for openvpn mail vpn. +# if [[ -e /p/c/filesystem ]]; then +# # note, man openvpn implies we could just call mail-route on vpn startup/shutdown with +# # systemd, buuut it can remake the tun device unexpectedly, i got this in the log +# # after my internet was down for a bit: +# # NOTE: Pulled options changed on restart, will need to close and reopen TUN/TAP device. +# m /a/exe/vpn-mk-client-cert -b mailclient -n mail li.iankelling.org +# fi + +# With openvpn, I didn't get around to persisting the openvpn +# cert/configs into /p/c/machine_specific/bk, so I had this case to +# manually get the cert. However, we aren't using openvpn anymore, so it +# is commented out. +# +# case $HOSTNAME in +# bk) +# if [[ ! -e /etc/openvpn/client/mail.conf ]]; then +# echo "$0: error: first, on a system with /p/c/filesystem, run mail-setup, or the vpn-mk-client-cert line above this err" 2>&2 +# exit 1 +# fi +# ;; +# esac m rsync -aiSAX --chown=root:root --chmod=g-s /a/bin/ds/mail-cert-cron /usr/local/bin -i /etc/systemd/system/mailcert.service <<'EOF' +u /etc/systemd/system/mailcert.service <<'EOF' [Unit] Description=Mail cert rsync After=multi-user.target @@ -802,7 +951,7 @@ After=multi-user.target Type=oneshot ExecStart=/usr/local/bin/sysd-mail-once mailcert /usr/local/bin/mail-cert-cron EOF -i /etc/systemd/system/mailcert.timer <<'EOF' +u /etc/systemd/system/mailcert.timer <<'EOF' [Unit] Description=Run mail-cert once a day @@ -830,8 +979,8 @@ fi # and see someone is banned. sed 's/^ *before *= *iptables-common.conf/before = iptables-common-exim.conf/' \ - /etc/fail2ban/action.d/iptables-multiport.conf| i /etc/fail2ban/action.d/iptables-exim.conf -i /etc/fail2ban/action.d/iptables-common-exim.conf <<'EOF' + /etc/fail2ban/action.d/iptables-multiport.conf| u /etc/fail2ban/action.d/iptables-exim.conf +u /etc/fail2ban/action.d/iptables-common-exim.conf <<'EOF' # iank: same as iptables-common, except iptables is iptables-exim, ip6tables is ip6tables-exim # Fail2Ban configuration file @@ -927,14 +1076,22 @@ blocktype = REJECT --reject-with icmp6-port-unreachable iptables = /usr/local/bin/ip6tables-exim EOF -i /etc/fail2ban/jail.d/exim.local <<'EOF' +u /etc/fail2ban/jail.d/exim.local <<'EOF' [exim] enabled = true port = 25,587 filter = exim banaction = iptables-exim + +# 209.51.188.13 = mail.fsf.org +# 2001:470:142::13 = mail.fsf.org +# 209.51.188.92 = eggs.gnu.org +# 2001:470:142:3::10 = eggs.gnu.org +# 72.14.176.105 2600:3c00:e000:280::2 = mail.iankelling.org +# 10.173.8.1 = non-nn net +ignoreip = 209.51.188.13 2001:470:142::13 209.51.188.92 2001:470:142:3::10 72.14.176.105 2600:3c00:e000:280::2 10.173.8.1 EOF -if $ir; then +if $ur; then m systemctl restart fail2ban fi @@ -970,10 +1127,8 @@ awk 'BEGIN { FS = ":" } ; $6 ~ /^\/home/ && $7 !~ /\/nologin$/ { print $1 }' /et esac done -if ! grep -q "^ncsoft:" /etc/aliases; then - echo "ncsoft: graceq2323@gmail.com" |m tee -a /etc/aliases -fi +. /a/bin/bash_unpublished/priv-mail-setup m gpasswd -a iank adm #needed for reading logs @@ -1006,9 +1161,14 @@ if (( ${#files[@]} )); then ${files[@]} /etc/exim4 fi -# by default, only 10 days of logs are kept. increase that. -m sed -ri 's/^(\s*rotate\s).*/\11000/' /etc/logrotate.d/exim4-base - +# By default, only 10 days of logs are kept. increase that. +# And dont compress, I look back at logs too often and +# dont need the annoyance of decompressing them all the time. +m sed -ri '/^\s*compress\s*$/d;s/^(\s*rotate\s).*/\11000/' /etc/logrotate.d/exim4-base +files=(/var/log/exim4/*.gz) +if (( ${#files[@]} )); then + gunzip ${files[@]} +fi ## disabled. not using .forward files, but this is still interesting ## for reference. @@ -1018,6 +1178,7 @@ m sed -ri 's/^(\s*rotate\s).*/\11000/' /etc/logrotate.d/exim4-base # b=userforward_higher_priority # # replace the router name so it is unique # sed -r s/^\\S+:/$b:/ 600_exim4-config_userforward >175_$b +rm -fv /etc/exim4/conf.d/router/175_userforward_higher_priority # todo, consider 'separate' in etc/exim4.conf, could it help on busy systems? @@ -1036,21 +1197,57 @@ rm -fv /etc/exim4/conf.d/retry/37_retry cat >/etc/exim4/conf.d/retry/17_retry <<'EOF' # Retry fast for my own domains -iankelling.org * F,1d,10m;F,14d,1h -amnimal.ninja * F,1d,10m;F,14d,1h -expertpathologyreview.com * F,1d,10m;F,14d,1h -je.b8.nz * F,1d,10m;F,14d,1h -zroe.org * F,1d,10m;F,14d,1h -eximbackup.b8.nz * F,1d,4m;F,14d,1h +iankelling.org * F,1d,1m;F,14d,1h +amnimal.ninja * F,1d,1m;F,14d,1h +expertpathologyreview.com * F,1d,1m;F,14d,1h +je.b8.nz * F,1d,1m;F,14d,1h +zroe.org * F,1d,1m;F,14d,1h +eximbackup.b8.nz * F,1d,1m;F,14d,1h + +# The spec says the target domain will be used for temporary host errors, +# but i've found that isn't correct, the hostname is required +# at least sometimes. +nn.b8.nz * F,1d,1m;F,14d,1h +defaultnn.b8.nz * F,1d,1m;F,14d,1h +mx.iankelling.org * F,1d,1m;F,14d,1h +bk.b8.nz * F,1d,1m;F,14d,1h +eggs.gnu.org * F,1d,1m;F,14d,1h +fencepost.gnu.org * F,1d,1m;F,14d,1h + +# afaik our retry doesnt need this, but just using everything +mx.amnimal.ninja * F,1d,1m;F,14d,1h +mx.expertpathologyreview.com * F,1d,1m;F,14d,1h + + +mail.fsf.org * F,1d,15m;F,14d,1h EOF rm -vf /etc/exim4/conf.d/main/000_localmacros # old filename -cat >/etc/exim4/conf.d/main/000_local </etc/exim4/conf.d/main/000_local2 </etc/exim4/conf.d/main/000_local <<'EOF' MAIN_TLS_ENABLE = true # require tls connections for all smarthosts -REMOTE_SMTP_SMARTHOST_HOSTS_REQUIRE_TLS = * +REMOTE_SMTP_SMARTHOST_HOSTS_REQUIRE_TLS = ! nn.b8.nz +REMOTE_SMTP_SMARTHOST_HOSTS_AVOID_TLS = nn.b8.nz # debian exim config added this in 2016 or so? # it's part of the smtp spec, to limit lines to 998 chars @@ -1062,17 +1259,12 @@ REMOTE_SMTP_SMARTHOST_HOSTS_REQUIRE_TLS = * # other says gmail does not reject. figure out and open a new bug. IGNORE_SMTP_LINE_LENGTH_LIMIT = true -# more verbose logs -MAIN_LOG_SELECTOR = +all +# more verbose logs. used to use +all, but made it less for more efficiency. +MAIN_LOG_SELECTOR = -skip_delivery -tls_cipher -tls_certificate_verified +all_parents +address_rewrite +arguments +deliver_time +pid +queue_time +queue_time_overall +received_recipients +received_sender +return_path_on_delivery +sender_on_delivery +smtp_confirmation +subject # Based on spec, seems like a good idea to be nice. smtp_return_error_details = true -# normally empty, I set this so I can set the envelope address -# when doing mail redelivery to invoke filters. Also allows -# me exiqgrep and stuff. -MAIN_TRUSTED_GROUPS = $u - # default is 10. when exim has been down for a bit, fsf mailserver # will do a big send in one connection, then exim decides to put # the messages in the queue instead of delivering them, to avoid @@ -1085,16 +1277,10 @@ smtp_accept_queue_per_connection = 500 DKIM_CANON = relaxed DKIM_SELECTOR = li -# from comments in -# https://debian-administration.org/article/718/DKIM-signing_outgoing_mail_with_exim4 -# and its best for this to align https://tools.ietf.org/html/rfc7489#page-8 -# There could be some circumstance when the -# from: isnt our domain, but the envelope sender is -# and so still want to sign, but I cant think of any case. -DKIM_DOMAIN = \${lc:\${domain:\$rh_from:}} + # The file is based on the outgoing domain-name in the from-header. # sign if key exists -DKIM_PRIVATE_KEY = \${if exists{/etc/exim4/\${dkim_domain}-private.pem} {/etc/exim4/\${dkim_domain}-private.pem}} +DKIM_PRIVATE_KEY = ${if exists{/etc/exim4/${dkim_domain}-private.pem} {/etc/exim4/${dkim_domain}-private.pem}} # most of the ones that gmail seems to use. # Exim has horrible default of signing unincluded @@ -1105,24 +1291,67 @@ DKIM_SIGN_HEADERS = mime-version:in-reply-to:references:from:date:subject:to domainlist local_hostnames = ! je.b8.nz : ! bk.b8.nz : *.b8.nz : b8.nz -hostlist iank_trusted = <; \\ +# note: most of these are duplicated in spamassassin config +hostlist iank_trusted = <; \ # veth0 -10.173.8.1 ; \\ +10.173.8.1 ; \ # li li_ip6 -72.14.176.105 ; 2600:3c00::f03c:91ff:fe6d:baf8 ; \\ +72.14.176.105 ; 2600:3c00::f03c:91ff:fe6d:baf8 ; \ # li_vpn_net li_vpn_net_ip6s -10.8.0.0/24; 2600:3c00:e000:280::/64 ; 2600:3c00:e002:3800::/56 ; \\ +10.8.0.0/24; 2600:3c00:e000:280::/64 ; 2600:3c00:e002:3800::/56 ; \ # bk bk_ip6 -85.119.83.50 ; 2001:ba8:1f1:f0c9::2 ; \\ +85.119.83.50 ; 2001:ba8:1f1:f0c9::2 ; \ # je je_ipv6 -85.119.82.128 ; 2001:ba8:1f1:f09d::2 ; \\ +85.119.82.128 ; 2001:ba8:1f1:f09d::2 ; \ # fsf_mit_net fsf_mit_net_ip6 fsf_net fsf_net_ip6 fsf_office_net 18.4.89.0/24 ; 2603:3005:71a:2e00::/64 ; 209.51.188.0/24 ; 2001:470:142::/48 ; 74.94.156.208/28 + + +# this is the default delay_warning_condition, plus matching on local_domains. +# If I have some problem with my local system that causes delayed delivery, +# I dont want to send warnings out to non-local domains. +delay_warning_condition = ${if or {\ + { !eq{$h_list-id:$h_list-post:$h_list-subscribe:}{} }\ + { match{$h_precedence:}{(?i)bulk|list|junk} }\ + { match{$h_auto-submitted:}{(?i)auto-generated|auto-replied} }\ + { match_domain{$domain}{+local_domains} }\ + } {no}{yes}} + + +# enable 587 in addition to the default 25, so that +# i can send mail where port 25 is firewalled by isp +daemon_smtp_ports = 25 : 587 : 10025 +# default of 25, can get stuck when catching up on mail +smtp_accept_max = 400 +smtp_accept_reserve = 100 +smtp_reserve_hosts = +iank_trusted + +# Rules that make receiving more liberal should be on backup hosts +# so that we dont reject mail accepted by MAIL_HOST +LOCAL_DENY_EXCEPTIONS_LOCAL_ACL_FILE = /etc/exim4/conf.d/local_deny_exceptions_acl EOF +if dpkg --compare-versions "$(dpkg-query -f='${Version}\n' --show exim4)" ge 4.94; then + cat >>/etc/exim4/conf.d/main/000_local <<'EOF' +# In t11, we cant do the old anymore because this is tainted data used in a file lookup. +# /usr/share/doc/exim4/NEWS.Debian.gz suggests to use lookups to untaint data. +DKIM_DOMAIN = ${lookup {${domain:$rh_from:}}lsearch,ret=key{/etc/exim4/conf.d/my-dkim-domains}} +EOF +else + cat >>/etc/exim4/conf.d/main/000_local <<'EOF' +# From comments in +# https://debian-administration.org/article/718/DKIM-signing_outgoing_mail_with_exim4 +# and its best for this to align https://tools.ietf.org/html/rfc7489#page-8 +# There could be some circumstance when the +# from: isnt our domain, but the envelope sender is +# and so still want to sign, but I cant think of any case. +DKIM_DOMAIN = ${lc:${domain:$rh_from:}} +EOF +fi + rm -fv /etc/exim4/rcpt_local_acl # old path -i /etc/exim4/conf.d/local_deny_exceptions_acl <<'EOF' +u /etc/exim4/conf.d/local_deny_exceptions_acl <<'EOF' # This acl already exists in rcpt, this just makes it more widespread. # See the comment there for its rationale. The reason it needs to be # more widespread is that I've turned on sender verification, but cron @@ -1138,7 +1367,8 @@ accept EOF rm -fv /etc/exim4/data_local_acl # old path -i /etc/exim4/conf.d/data_local_acl <<'EOF' + +u /etc/exim4/conf.d/data_local_acl <<'EOF' # Except for the "condition =", this was # a comment in the check_data acl. The comment about this not # being suitable has been changed in newer exim versions. The only thing @@ -1156,6 +1386,8 @@ warn warn !hosts = +iank_trusted + # Smarthosts connect with residential ips and thus get flagged as spam if we do a spam check. + !authenticated = plain_server:login_server condition = ${if < {$message_size}{5000K}} spam = Debian-exim:true add_header = X-Spam_score_int: $spam_score_int @@ -1164,11 +1396,6 @@ warn add_header = X-Spam_report: $spam_report add_header = X-Spam_action: $spam_action -warn - condition = ${if def:malware_name} - remove_header = Subject: - add_header = Subject: [Clamav warning: $malware_name] $h_subject - log_message = heuristic malware warning: $malware_name #accept # spf = pass:fail:softfail:none:neutral:permerror:temperror @@ -1177,14 +1404,214 @@ warn EOF -i /etc/exim4/conf.d/router/900_exim4-config_local_user < {$max_received_linelength}{998} {1}{0}} -.endif + message_linelength_limit = 2097152 .ifdef REMOTE_SMTP_HOSTS_AVOID_TLS hosts_avoid_tls = REMOTE_SMTP_HOSTS_AVOID_TLS .endif @@ -1265,16 +1690,14 @@ tls_privatekey = REMOTE_SMTP_PRIVATEKEY interface = <; 10.8.0.4 ; 2600:3c00:e002:3800::4 EOF -i /etc/exim4/conf.d/transport/30_smarthost_dkim <<'EOF' +u /etc/exim4/conf.d/transport/30_smarthost_dkim <<'EOF' # ian: this is remote_smtp_smarthost plus the dkim parts from remote_smtp smarthost_dkim: debug_print = "T: remote_smtp_smarthost for $local_part@$domain" driver = smtp + message_linelength_limit = 2097152 multi_domain -.ifndef IGNORE_SMTP_LINE_LENGTH_LIMIT - message_size_limit = ${if > {$max_received_linelength}{998} {1}{0}} -.endif hosts_try_auth = <; ${if exists{CONFDIR/passwd.client} \ {\ ${lookup{$host}nwildlsearch{CONFDIR/passwd.client}{$host_address}}\ @@ -1338,7 +1761,6 @@ EOF cat >/etc/exim4/update-exim4.conf.conf <<'EOF' # default stuff, i havent checked if its needed dc_minimaldns='false' -dc_relay_nets='' CFILEMODE='644' dc_use_split_config='true' dc_mailname_in_oh='true' @@ -1362,7 +1784,7 @@ if mailhost; then # in the log it just says "Starting Radicale". If you run # it in the foreground, it will give more info. Background # plus debug does not help. - # sudo -u radicale radicale -D -f + # sudo -u radicale radicale -D # created password file with: # htpasswd -c /p/c/machine_specific/li/filesystem/etc/caldav-htpasswd @@ -1370,16 +1792,14 @@ if mailhost; then # # setup chgrp www-data in ./conflink pi-nostart radicale + m usermod -a -G radicale iank - i /etc/systemd/system/radicale.service.d/override.conf < @@ -1818,23 +2247,25 @@ if [[ $HOSTNAME == bk ]]; then ### begin composer install # https://getcomposer.org/doc/faqs/how-to-install-composer-programmatically.md - # cd $(mktemp -d) - # sum="$(wget -q -O - https://composer.github.io/installer.sig)" - # m php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" - # if [[ $sum != $(php -r "echo hash_file('sha384', 'composer-setup.php');") ]]; then - # echo 'ERROR: Invalid composer installer checksum' >&2 - # rm -fv composer-setup.php - # exit 1 - # fi - # m php composer-setup.php --quiet - # rm -fv composer-setup.php - # m mv composer.phar /usr/local/bin - - # the above method gets composer2, carddav plugin at least doesnt work with that - # yet, it was just released 10-24-2020. - m cd /usr/local/bin - m wget -nv -N https://getcomposer.org/composer-1.phar - chmod +x composer-1.phar + cd /usr/local/bin + EXPECTED_CHECKSUM="$(php -r 'copy("https://composer.github.io/installer.sig", "php://stdout");')" + php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" + ACTUAL_CHECKSUM="$(php -r "echo hash_file('sha384', 'composer-setup.php');")" + + if [ "$EXPECTED_CHECKSUM" != "$ACTUAL_CHECKSUM" ] + then + >&2 echo 'ERROR: Invalid installer checksum' + rm composer-setup.php + exit 1 + fi + + php composer-setup.php --quiet + rm composer-setup.php + + # based on error when running composer + mkdir -p /var/www/.composer + chown www-data:www-data /var/www/.composer + ### end composer install rcdirs=(/usr/local/lib/rcexpertpath /usr/local/lib/rcninja) @@ -1847,7 +2278,7 @@ if [[ $HOSTNAME == bk ]]; then #### begin dl roundcube # note, im r2e subbed to https://github.com/roundcube/roundcubemail/releases.atom v=1.4.13; f=roundcubemail-$v-complete.tar.gz - cd /a/opt + cd /root if [[ -e $f ]]; then timestamp=$(stat -c %Y $f) else @@ -1871,7 +2302,7 @@ if [[ $HOSTNAME == bk ]]; then ncdir=${ncdirs[i]} # copied from debians cronjob - i /etc/cron.d/$rcbase </dev/null @@ -1931,7 +2362,7 @@ EOF rctmpdir=/var/tmp/$rcbase rcdb=/m/rc/$rcbase.sqlite # config from mailinabox - i $rcdir/config/config.inc.php <$verf @@ -2026,7 +2457,7 @@ EOF # About categories, see https://www.davx5.com/tested-with/nextcloud # https://github.com/blind-coder/rcmcarddav/blob/master/doc/GROUPS.md - i $rcdir/plugins/carddav/config.inc.php <tmp.php <$myncdir/tmp.php <config.php - m rm tmp.php - m sudo -u www-data php $ncdir/occ maintenance:update:htaccess + e running php $myncdir/tmp.php + # note: we leave it around place for debugging + # shellcheck disable=SC2024 # intended + sudo -u www-data php $myncdir/tmp.php >config.php + fi + cd $ncdir + m sudo -u www-data php occ maintenance:update:htaccess list=$(sudo -u www-data php $ncdir/occ --output=json_pretty app:list) # user_external not compaible with nc 23 for app in contacts calendar; do if [[ $(printf "%s\n" "$list"| jq ".enabled.$app") == null ]]; then - m sudo -u www-data php $ncdir/occ app:install $app + cd $ncdir + m sudo -u www-data php occ app:install $app fi done - i /etc/systemd/system/$ncbase.service <&2; exit 1; fi -shopt -s inherit_errexit 2>/dev/null ||: # ignore fail in bash < 4.4 -set -eE -o pipefail -trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" exit status: $?, PIPESTATUS: ${PIPESTATUS[*]}" >&2' ERR -ncbase=$1 -if ! php /var/www/$ncbase/updater/updater.phar -n; then +source /usr/local/lib/err + +m() { printf "%s\n" "$*"; "$@"; } +err-cleanup() { echo failed nextcloud update for $ncbase >&2 - /sbin/exim -t <>/etc/exim4/update-exim4.conf.conf <>/etc/exim4/update-exim4.conf.conf <>/etc/exim4/conf.d/main/000_local < {$max_received_linelength}{998} {1}{0}} -.endif - hosts_require_auth = * - hosts_try_auth = * - envelope_to_add - # manual return path because we want it to be the envelope sender - # we got not the one we are using in this smtp transport - headers_add = "Return-path: $sender_address" -.ifdef REMOTE_SMTP_SMARTHOST_HOSTS_AVOID_TLS - hosts_avoid_tls = REMOTE_SMTP_SMARTHOST_HOSTS_AVOID_TLS -.endif -.ifdef REMOTE_SMTP_SMARTHOST_HOSTS_REQUIRE_TLS - hosts_require_tls = REMOTE_SMTP_SMARTHOST_HOSTS_REQUIRE_TLS -.endif -.ifdef REMOTE_SMTP_SMARTHOST_TLS_VERIFY_CERTIFICATES - tls_verify_certificates = REMOTE_SMTP_SMARTHOST_TLS_VERIFY_CERTIFICATES -.endif -.ifdef REMOTE_SMTP_SMARTHOST_TLS_VERIFY_HOSTS - tls_verify_hosts = REMOTE_SMTP_SMARTHOST_TLS_VERIFY_HOST -.endif -.ifdef REMOTE_SMTP_HEADERS_REWRITE - headers_rewrite = REMOTE_SMTP_HEADERS_REWRITE -.endif -.ifdef REMOTE_SMTP_HELO_DATA - helo_data=REMOTE_SMTP_HELO_DATA -.endif -.ifdef TLS_DH_MIN_BITS -tls_dh_min_bits = TLS_DH_MIN_BITS -.endif -.ifdef REMOTE_SMTP_SMARTHOST_TLS_CERTIFICATE -tls_certificate = REMOTE_SMTP_SMARTHOST_TLS_CERTIFICATE -.endif -.ifdef REMOTE_SMTP_SMARTHOST_PRIVATEKEY -tls_privatekey = REMOTE_SMTP_SMARTHOST_PRIVATEKEY -.endif -.ifdef REMOTE_SMTP_TRANSPORTS_HEADERS_REMOVE - headers_remove = REMOTE_SMTP_TRANSPORTS_HEADERS_REMOVE -.endif -EOF - - # this avoids some error. i cant remember what. todo: - # test it out and document why/if its needed. - # i /etc/exim4/host_local_deny_exceptions <<'EOF' - # mail.fsf.org - # *.posteo.de - # EOF + # This allows for forwarded mail to not get most rcpt checks, especially SPF, + # which would incorrectly get denied. + u /etc/exim4/host_local_deny_exceptions <<'EOF' +mail.fsf.org +*.posteo.de +EOF # cron email from smarthost hosts will automatically be to # USER@FQDN. I redirect that to alerts@, on the smarthosts, but in # case that doesn't work, we still want to accept that mail, but not # from any host except the smarthosts. local_hostnames and this rule # is for that purpose. - i /etc/exim4/conf.d/rcpt_local_acl <<'EOF' + u /etc/exim4/conf.d/rcpt_local_acl <<'EOF' deny !authenticated = * domains = +local_hostnames message = no relay + +# for testing bounce behavior +#deny +# senders = testlist-bounces+test=zroe.org@fsf.org +# message = iank-bounce +EOF + echo|u /etc/exim4/conf.d/router/880_universal_forward + + + cat >>/etc/exim4/conf.d/main/000_local < /etc/mailname /a/exe/cedit nn /etc/hosts <<'EOF' || [[ $? == 1 ]] 10.173.8.2 nn.b8.nz EOF - sed -r -f - /etc/init.d/exim4 <<'EOF' | i /etc/init.d/exim4in + sed -r -f - /etc/init.d/exim4 <<'EOF' |u /etc/init.d/exim4in s,/etc/default/exim4,/etc/default/exim4in,g s,/run/exim4/exim.pid,/run/exim4/eximin.pid,g s,(^[ #]*Provides:).*,\1 exim4in, s,(^[ #]*NAME=).*,\1"exim4in", EOF chmod +x /etc/init.d/exim4in - i /etc/systemd/system/exim4in.service.d/alwaysrestart.conf <<'EOF' + u /etc/systemd/system/exim4in.service.d/alwaysrestart.conf <<'EOF' [Unit] # needed to continually restart StartLimitIntervalSec=0 @@ -2703,16 +3176,17 @@ Restart=always RestartSec=20 EOF - i /etc/default/exim4in <<'EOF' + u /etc/default/exim4in <<'EOF' # defaults but no queue runner and alternate config dir QUEUERUNNER='no' COMMONOPTIONS='-oP /run/exim4/eximin.pid' UPEX4OPTS='-d /etc/myexim4' EOF + echo bk.b8.nz > /etc/mailname cat >>/etc/exim4/update-exim4.conf.conf <>/etc/exim4/update-exim4.conf.conf <>/etc/exim4/update-exim4.conf.conf <>/etc/exim4/update-exim4.conf.conf <>/etc/myexim4/conf.d/main/000_local <<'EOF' +# this makes it easier to see which exim is doing what +log_file_path = /var/log/exim4/my%s +EOF + + + + cat >/etc/logrotate.d/myexim <<'EOF' +/var/log/exim4/mymain /var/log/exim4/myreject { + daily + missingok + rotate 1000 + delaycompress + notifempty + nocreate +} +/var/log/exim4/mypanic { + size 10M + missingok + rotate 10 + compress + delaycompress + notifempty + nocreate +} +EOF + # If we ever wanted to have a separate spool, # we could do it like this. # cat >>/etc/exim4/conf.d/main/000_local-nn <<'EOF' @@ -2864,11 +3370,39 @@ EOF bk) # config for the non-nn exim - cat >/etc/myexim4/conf.d/main/000_local-nn <<'EOF' + cat >>/etc/myexim4/conf.d/main/000_local <<'EOF' MAIN_HARDCODE_PRIMARY_HOSTNAME = mail2.iankelling.org EOF ;; $MAIL_HOST) + + + u /etc/myexim4/conf.d/router/185_sentarchive <<'EOF' +sentarchive: + driver = redirect + domains = ! +local_domains + senders = <; *@fsf.org ; *@posteo.net + condition = ${if !bool{${lookup{$local_part@$domain}lsearch{/etc/exim4/ignore-sent}{true}}}} + data = vojdedIdNejyebni@b8.nz + unseen +EOF + + u /etc/myexim4/conf.d/router/160_backup_redir <<'EOF' +backup_redir: +driver = redirect +# i dont email myself from my own machine much, so lets ignore that. +domains = ! +local_domains +senders = <; *@fsf.org ; *@posteo.net +condition = ${if !bool{${lookup{$local_part@$domain}lsearch{/etc/exim4/ignore-sent}{true}}}} +# b is just an arbirary short string +data = b@eximbackup.b8.nz +# note, to test this, i could temporarily allow testignore. +# alerts avoids potential mail loop. +local_parts = ! root : ! testignore : ! alerts : ! daylert +unseen = true +errors_to = alerts@iankelling.org +EOF + # for bk, we have a exim4in.service that will do this for us. m update-exim4.conf -d /etc/myexim4 ;; @@ -2892,11 +3426,14 @@ if [[ -e /nocow ]]; then if ! grep -Fx "/nocow/exim4 /var/spool/exim4 none bind 0 0" /etc/fstab; then echo "/nocow/exim4 /var/spool/exim4 none bind 0 0" >>/etc/fstab fi - i /etc/systemd/system/exim4.service.d/override.conf <<'EOF' + u /etc/systemd/system/exim4.service.d/override.conf <<'EOF' [Unit] # without local-fs on exim, we get these kind of errors in paniclog on shutdown: # Failed to create spool file /var/spool/exim4//input//1jCLxz-0008V4-V9-D: Permission denied After=local-fs.target + +[Service] +ExecStartPre=/usr/local/bin/exim-nn-iptables EOF if ! mountpoint -q $sdir; then stopifactive exim4 exim4in @@ -2931,8 +3468,8 @@ elif [[ $uid != 608 ]]; then m usermod -u 608 Debian-exim m groupmod -g 608 Debian-exim m usermod -g 608 Debian-exim - m find / /nocow -path ./var/tmp -prune -o -xdev -uid $uid -execdir chown -h 608 {} + - m find / /nocow -path ./var/tmp -prune -o -xdev -gid $gid -execdir chgrp -h 608 {} + + m find / /nocow -xdev -path ./var/tmp -prune -o -uid $uid -execdir chown -h 608 {} + + m find / /nocow -xdev -path ./var/tmp -prune -o -gid $gid -execdir chgrp -h 608 {} + fi # * start / stop services @@ -2943,7 +3480,20 @@ if $reload; then m systemctl daemon-reload fi -m systemctl --now enable epanicclean.timer +# optimization, this only needs to run once. +if [[ ! -e /sys/class/net/wghole ]]; then + # checking bhost_t is redundant, but could help us catch errors. + if $bhost_t || [[ -e /etc/wireguard/wghole.conf ]]; then + # todo: in mail-setup, we have a static list of backup hosts, not *y + m systemctl --now enable wg-quick@wghole + fi +fi + +# optimization, this only needs to be run once +if [[ ! -e /var/lib/prometheus/node-exporter/exim_paniclog.prom ]]; then + sysd-prom-fail-install epanicclean + m systemctl --now enable epanicclean +fi case $HOSTNAME in je) @@ -2954,15 +3504,13 @@ case $HOSTNAME in ;; esac -m /a/bin/ds/mail-cert-cron -1 -sre mailcert.timer +# optimization, this only needs to run once. +if [[ ! -e /etc/exim4/fullchain.pem ]]; then + m /a/bin/ds/mail-cert-cron -1 + m systemctl --now enable mailcert.timer +fi case $HOSTNAME in - bk) - # todo, this should be done in distro-begin - soff systemd-resolved - ln -sf 127.0.0.1-resolv/stub-resolv.conf /etc/resolv.conf - ;;& $MAIL_HOST|bk) m systemctl --now enable mailnn mailnnroute ;;& @@ -2984,7 +3532,7 @@ case $HOSTNAME in fi if ! systemctl is-active clamav-daemon >/dev/null; then m systemctl --now enable clamav-daemon - out=$(rsync -aiSAX --chown=root:root --chmod=g-s /a/bin/ds/filesystem/etc/systemd/system/epanicclean.{timer,service} /etc/systemd/system) + out=$(rsync -aiSAX --chown=root:root --chmod=g-s /a/bin/ds/filesystem/etc/systemd/system/epanicclean.service /etc/systemd/system) if [[ $out ]]; then reload=true fi @@ -2997,26 +3545,38 @@ case $HOSTNAME in $MAIL_HOST|bk|je) # start spamassassin/dovecot before exim. sre dovecot spamassassin - # need to wait a bit before restarting exim, else I - # get a paniclog entry like: spam acl condition: all spamd servers failed - sleep 3 + # Wait a bit before restarting exim, else I get a paniclog entry + # like: spam acl condition: all spamd servers failed. But I'm tired + # of waiting. I'll deal with this some other way. + # + # sleep 3 m systemctl --now enable mailclean.timer ;;& $MAIL_HOST) # < 2.1 (eg: in t9), uses a different data format which required manual # migration. dont start if we are running an old version. - if dpkg --compare-versions $(dpkg -s radicale | awk '$1 == "Version:" { print $2 }') ge 2.1; then + if dpkg --compare-versions "$(dpkg -s radicale | awk '$1 == "Version:" { print $2 }')" ge 2.1; then m systemctl --now enable radicale fi ;;& esac +# for debugging dns issues +case $HOSTNAME in + je|bk) + systemctl enable --now logrotate-fast.timer + ;; +esac + # last use of $reload happens in previous block rm -f /var/local/mail-setup-reload case $HOSTNAME in - $MAIL_HOST|bk|je) : ;; + $MAIL_HOST|bk|je|li) + # on li, these are never started, except $vpnser + : + ;; *) soff radicale mailclean.timer dovecot spamassassin $vpnser mailnn clamav-daemon ;; @@ -3048,53 +3608,101 @@ case $HOSTNAME in # note: cronjob "ian" also does some important monitoring # todo: this will sometimes cause an alert because mailtest-check will run # before we have setup network namespace and spamassassin - cat >/etc/cron.d/mailtest <>/etc/cron.d/mailtest </usr/local/bin/send-test-forward <<'EOF' #!/bin/bash +# we remove from the queue older than 4.3 minutes since we send every 5 minutes. olds=( -$(/sbin/exiqgrep -o 260 -i -r '^(testignore@(iankelling\.org|zroe\.org|expertpathologyreview\.com|amnimal\.ninja|je\.b8\.nz)|jtuttle@gnu\.org)$') +$(/usr/sbin/exiqgrep -o 260 -i -r '^(testignore@(iankelling\.org|zroe\.org|expertpathologyreview\.com|amnimal\.ninja|je\.b8\.nz)|jtuttle@gnu\.org)$') ) if (( ${#olds[@]} )); then - /sbin/exim -Mrm "${olds[@]}" >/dev/null + /usr/sbin/exim -Mrm "${olds[@]}" >/dev/null fi EOF for test_from in ${test_froms[@]}; do + + test_to=${test_tos[0]} + for t in ${test_tos[@]:1}; do + test_to+=", $t" + done + case $test_from in + testignore@expertpathologyreview.com) + test_to=testignore@zroe.org + ;; + esac + cat >>/usr/local/bin/send-test-forward <