X-Git-Url: https://iankelling.org/git/?a=blobdiff_plain;f=mail-setup;h=e2f7b21991ced794b4081ed1f05fac6b198f7020;hb=51c8b40fd2aac71d29dc9298ca65425725ad1edd;hp=cfe5575d2090ebfcc120f39575c0aa40e5d8060c;hpb=65d1dda5c1e79aa9c85a2d16bdd1a12413f163af;p=distro-setup diff --git a/mail-setup b/mail-setup index cfe5575..e2f7b21 100755 --- a/mail-setup +++ b/mail-setup @@ -3,13 +3,42 @@ # Copyright (C) 2019 Ian Kelling # SPDX-License-Identifier: AGPL-3.0-or-later +# todo: +# on bk (and fsf servers that run multiple exim4 daemons, eg eximfsf2 and eximfsf3), +# make it so that when exim is restarted due to package upgrades, +# we also restart those daemons, which can be done like so, based on looking +# at the prerm and postinst scripts of exim4-daemon-heavy. +# +# if [[ ! -e /usr/sbin/invoke-rc.d-diverted ]]; then +# mv /usr/sbin/invoke-rc.d /usr/sbin/invoke-rc.d-diverted +# dpkg --divert /usr/sbin/invoke-rc.d-diverted --no-rename /usr/sbin/invoke-rc.d +# fi +# /usr/sbin/invoke-rc.d: +# #!/bin/bash +# if [[ DPKG_MAINTSCRIPT_PACKAGE == exim4* && $1 == exim4 ]]; then +# shift +# ret=0 +# for daemon in exim4 eximfsf2 eximfsf3; do +# /usr/sbin/invoke-rc.d-diverted $daemon "$@" || ret=$? +# done +# else +# /usr/sbin/invoke-rc.d-diverted "$@" +# fi + # Things I tend to forget. on MAIL_HOST, daemon runs with /etc/exim4/my.conf, -# which gets generated from update-exim4.conf -d /etc/myexim4 +# due to /etc/default/exim4 containing: +# COMMONOPTIONS='-C /etc/exim4/my.conf' +# UPEX4OPTS='-o /etc/exim4/my.conf' +# +# The non-daemon config +# gets generated from this script calling update-exim4.conf -d /etc/myexim4 # which has log path # log_file_path = /var/log/exim4/my%s # -# so, normal mainlog is non-daemon, which delivers to the daemon. - +# On non bk|MAIL_HOST, the config and log file are all standard. +# +# eximbackup folder is /bu/md +# it is cleaned up by mail-backup-clean, which is run by btrbk-run # shellcheck disable=SC2254 # makes for a lot of unneeded quotes @@ -19,6 +48,14 @@ # #&! testignore|jtuttle|eximbackup|/usr/sbin/exim4 -bpu +# todo: this message seems to get dropped on the floor, it was due to a missing 2nd colon in +# condition = ${if def:h_fdate:} +# Figure out how to avoid this message being discarded. + +# 2023-09-12 01:41:43 [722371] 1qfw9f-0031v9-0S <= ian@iankelling.org U=iank P=local S=483 id=87cyyogd7t.fsf@iankelling.org T="iank2" from for testignore@amnimal.ninja +# 2023-09-12 01:41:43 [722373] 1qfw9f-0031v9-0S H=nn.b8.nz [10.173.8.2]: SMTP error from remote mail server after pipelined end of data: 451 Temporary local problem - please try later +# 2023-09-12 01:41:43 [722372] 1qfw9f-0031v9-0S == testignore@amnimal.ninja R=smarthost T=remote_smtp_smarthost defer (-46) H=nn.b8.nz [10.173.8.2] DT=0s: SMTP error from remote mail server after pipelined end of data: 451 Temporary local problem - please try later + # todo: check new macro DKIM_TIMESTAMPS # todo: check if REMOTE_SMTP_INTERFACE or REMOTE_SMTP_TRANSPORTS_HEADERS_REMOVE can simplify my or fsfs config @@ -146,10 +183,10 @@ if [ -z "$BASH_VERSION" ]; then echo "error: shell is not bash" >&2; exit 1; fi shopt -s nullglob -if [[ -s /usr/local/lib/err ]]; then - source /usr/local/lib/err -elif [[ -s /a/bin/errhandle/err ]]; then - source /a/bin/errhandle/err +if [[ -s /usr/local/lib/bash-bear ]]; then + source /usr/local/lib/bash-bear +elif [[ -s /a/bin/bash-bear-trap/bash-bear ]]; then + source /a/bin/bash-bear-trap/bash-bear else echo "no err tracing script found" exit 1 @@ -203,8 +240,8 @@ fi ####### -# * perstent password instructions -# Note: for cert cron, we need to manually run first to accept known_hosts +# * perstent password instructions Note: for cert cron, we need to +# manually run first to accept known_hosts # # exim passwords: # # for hosts which have all private files I just use the same user @@ -298,7 +335,7 @@ fi # * functions & constants -pre="${0##*/}:" +pre="${0##*/}:${SSH_CLIENT:+ $HOSTNAME:}" m() { printf "$pre %s\n" "$*"; "$@"; } e() { printf "$pre %s\n" "$*"; } err() { printf "$pre %s\n" "$*" >&2; exit 1; } @@ -489,6 +526,9 @@ EOF # * clamav +# old file. remove when all hosts updated, 2023-09-11 +rm -fv /etc/exim4/conf.d/clamav_data_acl + m usermod -a -G Debian-exim clamav u /etc/systemd/system/clamav-daemon.service.d/fix.conf </etc/exim4/conf.d/main/30_local <&2 - # rm -fv composer-setup.php - # exit 1 - # fi - # m php composer-setup.php --quiet - # rm -fv composer-setup.php - # m mv composer.phar /usr/local/bin - - # the above method gets composer2, carddav plugin at least doesnt work with that - # yet, it was just released 10-24-2020. - m cd /usr/local/bin - m wget -nv -N https://getcomposer.org/composer-1.phar - chmod +x composer-1.phar + cd /usr/local/bin + EXPECTED_CHECKSUM="$(php -r 'copy("https://composer.github.io/installer.sig", "php://stdout");')" + php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" + ACTUAL_CHECKSUM="$(php -r "echo hash_file('sha384', 'composer-setup.php');")" + + if [ "$EXPECTED_CHECKSUM" != "$ACTUAL_CHECKSUM" ] + then + >&2 echo 'ERROR: Invalid installer checksum' + rm composer-setup.php + exit 1 + fi + + php composer-setup.php --quiet + rm composer-setup.php + + # based on error when running composer + mkdir -p /var/www/.composer + chown www-data:www-data /var/www/.composer + ### end composer install rcdirs=(/usr/local/lib/rcexpertpath /usr/local/lib/rcninja) - ncdirs=(/var/www/ncninja) ncdirs=(/var/www/ncexpertpath /var/www/ncninja) # point debian cronjob to our local install, preventing daily cron error @@ -2263,8 +2378,6 @@ if [[ $HOSTNAME == bk ]]; then rcdir=${rcdirs[i]} rcbase=${rcdir##*/} ncdir=${ncdirs[i]} - myncdir=/root/${ncdir##*/} - mkdir -p $myncdir # copied from debians cronjob u /etc/cron.d/$rcbase <$verf @@ -2531,7 +2644,9 @@ EOF for ((i=0; i < ${#bkdomains[@]}; i++)); do domain=${bkdomains[i]} ncdir=${ncdirs[i]} + myncdir=/var/local/${ncdir##*/} ncbase=${ncdir##*/} + mkdir -p $myncdir m cd /var/www if [[ ! -e $ncdir/index.php ]]; then # if we wanted to only install a specific version, use something like @@ -2551,13 +2666,16 @@ EOF m touch $myncdir/done-install fi - # note, strange this happend where updater did not increment the version var, - # mine was stuck on 20. I manually updated it. m cd $ncdir/config - if [[ ! -e $myncdir/config.php-orig ]]; then - m cp -a config.php $myncdir/config.php-orig - fi - cat $myncdir/config.php-orig - >$myncdir/tmp.php <$myncdir/tmp.php <config.php + e running php $myncdir/tmp.php + # note: we leave it around place for debugging + # shellcheck disable=SC2024 # intended + sudo -u www-data php $myncdir/tmp.php >config.php + fi cd $ncdir m sudo -u www-data php occ maintenance:update:htaccess list=$(sudo -u www-data php $ncdir/occ --output=json_pretty app:list) @@ -2628,7 +2748,7 @@ EOF u /usr/local/bin/ncup <<'EOFOUTER' #!/bin/bash -source /usr/local/lib/err +source /usr/local/lib/bash-bear m() { printf "%s\n" "$*"; "$@"; } err-cleanup() { @@ -2657,7 +2777,13 @@ fi ncbase=$1 cd /var/www/$ncbase # https://docs.nextcloud.com/server/22/admin_manual/maintenance/update.html?highlight=updater+phar +# the docs claim this is all you need, which is not true. +# You will go to the web ui and it will say that you need to click a button to update, +# or that you can run occ upgrade m php /var/www/$ncbase/updater/updater.phar -n +# throw a sleep in just because who knows what else is undocumented +sleep 5 +m php occ upgrade EOFOUTER chmod +x /usr/local/bin/ncup @@ -2674,6 +2800,67 @@ EOF fi +# * debbugs + +pi debbugs +# missing dependency. apache error log: +# Can't locate List/AllUtils.pm in @INC (you may need to install the List::AllUtils module) (@INC contains: /etc/perl /usr/local/lib/x86_64-linux-gnu/perl/5.34.0 /usr/local/share/perl/5.34.0 /usr/lib/x86_64-linux-gnu/perl5/5.34 /usr/share/perl5 /usr/lib/x86_64-linux-gnu/perl-base /usr/lib/x86_64-linux-gnu/perl/5.34 /usr/share/perl/5.34 /usr/local/lib/site_perl) at /var/lib/debbugs/www/cgi/pkgreport.cgi line 23. +pi liblist-allutils-perl lynx +# workarounds for broken debbugsconfig which is +# itself deprecated. this is temporary before I +# figure out how to install from git +if [[ -e /usr/share/doc/debbugs/examples/text.gz ]]; then + gunzip /usr/share/doc/debbugs/examples/text.gz +fi +mkdir -p /etc/debbugs/indices +debbugsconfig + + +# ld for local debbugs +/a/exe/web-conf -t -a 127.0.1.1 -p 80 -r /var/lib/debbugs/www - apache2 ld <<'EOF' +# copied from debbugs upstream example + + Options Indexes SymLinksIfOwnerMatch MultiViews + DirectoryIndex index.html + Require all granted + + +ScriptAlias /cgi/ /var/lib/debbugs/www/cgi/ + + AllowOverride None + Options ExecCGI SymLinksIfOwnerMatch + Require all granted + + +RewriteEngine on +RewriteCond %{HTTP_USER_AGENT} .*apt-listbugs.* +RewriteRule .* /apt-listbugs.html [R,L] + +# RewriteLog /org/bugs.debian.org/apache-rewrite.log +# RewriteLogLevel 0 + +#RewriteRule ^/$ http://www.debian.org/Bugs/ +RewriteRule ^/(robots\.txt|release-critical|apt-listbugs\.html)$ - [L] +# The following two redirect to up-to-date pages +RewriteRule ^/[[:space:]]*#?([[:digit:]][[:digit:]][[:digit:]]+)([;&].+)?$ /cgi-bin/bugreport.cgi?bug=$1$2 [L,R,NE] +RewriteRule ^/([^/+]*)([+])([^/]*)$ "/$1%%{%}2B$3" [N] +RewriteRule ^/[Ff][Rr][Oo][Mm]:([^/]+\@.+)$ /cgi-bin/pkgreport.cgi?submitter=$1 [PT,NE] +# Commented out, 'cuz aj says it will crash master. (old master) +# RewriteRule ^/[Ss][Ee][Vv][Ee][Rr][Ii][Tt][Yy]:([^/]+\@.+)$ /cgi-bin/pkgreport.cgi?severity=$1 [L,R] +RewriteRule ^/([^/]+\@.+)$ /cgi-bin/pkgreport.cgi?maint=$1 [PT,NE] +RewriteRule ^/mbox:([[:digit:]][[:digit:]][[:digit:]]+)([;&].+)?$ /cgi-bin/bugreport.cgi?mbox=yes&bug=$1$2 [PT,NE] +RewriteRule ^/src:([^/]+)$ /cgi-bin/pkgreport.cgi?src=$1 [PT,NE] +RewriteRule ^/severity:([^/]+)$ /cgi-bin/pkgreport.cgi?severity=$1 [PT,NE] +RewriteRule ^/tag:([^/]+)$ /cgi-bin/pkgreport.cgi?tag=$1 [PT,NE] +# RewriteMap fix-chars int:noescape +RewriteCond %{REQUEST_URI} ^/(Access\.html|Developer\.html|Reporting\.html|server-request\.html|server-control\.html|server-refcard\.html).* [NC] +RewriteRule .* - [L] +# PT|passthrough to bugreport.cgi and pkgreport.cgi +RewriteRule ^/([0-9]+)$ /cgi-bin/bugreport.cgi?bug=$1 [PT,NE] +RewriteRule ^/([^/]+)$ /cgi-bin/pkgreport.cgi?pkg=$1 [PT,NE] +EOF + + # * exim host conditional config # ** exim certs @@ -2743,7 +2930,8 @@ case $HOSTNAME in # which will overwrite any existing file u /etc/default/exim4 <<'EOF' QUEUERUNNER='combined' -QUEUEINTERVAL='30m' +# note: this is duplicated in brc2, 10m here is -q10m there. +QUEUEINTERVAL='10m' COMMONOPTIONS='-C /etc/exim4/my.conf' UPEX4OPTS='-o /etc/exim4/my.conf' # i use epanic-clean for alerting if there are bad paniclog entries @@ -2860,29 +3048,10 @@ EOF # ** $MAIL_HOST|bk) $MAIL_HOST|bk) - - # no clamav on je, it has 1.5g memory and clamav uses most of it - u /etc/exim4/conf.d/clamav_data_acl <<'EOF' -warn -!hosts = +iank_trusted -!authenticated = plain_server:login_server -condition = ${if def:malware_name} -remove_header = Subject: -add_header = Subject: [Clamav warning: $malware_name] $h_subject -log_message = heuristic malware warning: $malware_name -EOF - - cat >>/etc/exim4/conf.d/main/000_local <>/etc/exim4/update-exim4.conf.conf <> /etc/exim4/conf.d/data_local_acl <<'EOF' -deny - malware = */defer_ok - !condition = ${if match {$malware_name}{\N^Heuristic\N}} - message = This message was detected as possible malware ($malware_name). -EOF cat >/etc/exim4/conf.d/main/000_local-nn <>/etc/exim4/conf.d/main/000_local <> /etc/exim4/conf.d/data_local_acl <<'EOF' +deny + malware = */defer_ok + !condition = ${if match {$malware_name}{\N^Heuristic\N}} + message = This message was detected as possible malware ($malware_name). + +warn + !hosts = +iank_trusted + !authenticated = * + condition = ${if def:malware_name} + remove_header = Subject: + add_header = Subject: [Clamav warning: $malware_name] $h_subject + log_message = heuristic malware warning: $malware_name + +warn + # fdate = future date. # tdate = temporary date. + condition = ${if def:h_fdate} + remove_header = fdate: + add_header = tdate: + control = freeze +EOF + /a/exe/cedit nn /etc/hosts <<'EOF' || [[ $? == 1 ]] 10.173.8.2 nn.b8.nz @@ -3149,18 +3413,27 @@ EOF ;; # ** not MAIL_HOST|bk|je *) - # this one should be removed for all non mail hosts, but + echo|u /etc/exim4/conf.d/transport/30_debbugs + echo|u /etc/exim4/conf.d/router/153_debbugs + echo|u /etc/exim4/conf.d/router/155_delay + # this one should be removed for all non mail_hosts. note # bk and je never become mail_host echo|u /etc/exim4/conf.d/router/195_dnslookup_vpn echo|u /etc/exim4/conf.d/router/160_backup_redir echo|u /etc/exim4/conf.d/router/161_backup_redir_nn echo|u /etc/exim4/conf.d/router/185_sentarchive echo|u /etc/exim4/conf.d/router/186_sentarchive_nn + # Note, in general we could submit to smarthosts on non MAIL_HOST. + # however, delayed mail makes this inconvenient, because I + # occasionally want to send an email from a non-MAIL_HOST and then + # turn off that computer or travel with it so it is disconnected. + # It is also probably easier to setup emacs to delay messages, but + # that would mean we need to keep emacs running, this is much + # nicer. echo|u /etc/exim4/conf.d/router/188_exim4-config_smarthost echo|u /etc/exim4/conf.d/router/190_exim4-config_fsfsmarthost echo|u /etc/exim4/conf.d/rcpt_local_acl echo|u /etc/exim4/conf.d/main/000_local-nn - echo|u /etc/exim4/conf.d/clamav_data_acl if $bhost_t; then @@ -3337,7 +3610,6 @@ sentarchive: unseen EOF - u /etc/myexim4/conf.d/router/160_backup_redir <<'EOF' backup_redir: driver = redirect @@ -3354,8 +3626,6 @@ unseen = true errors_to = alerts@iankelling.org EOF - - # for bk, we have a exim4in.service that will do this for us. m update-exim4.conf -d /etc/myexim4 ;; @@ -3425,6 +3695,25 @@ elif [[ $uid != 608 ]]; then m find / /nocow -xdev -path ./var/tmp -prune -o -gid $gid -execdir chgrp -h 608 {} + fi + +# note: example config has a debbugs user, +# but my exim runs setuid as Debian-exim so it can't switch +# to another user. Anyways, I'm not exposing this to the +# internet at this time. If I do, the thing to do would +# be to use a sudo config (or sudo alternative). This +# would be how to setup + +# IFS=:; read -r _ _ uid _ < <(getent passwd debbugs||:) ||:; unset IFS +# if [[ ! $uid ]]; then +# # /a/opt/debbugs/debian/README.mail +# adduser --uid 610 --system --group --home /o/debbugs \ +# --no-create-home --disabled-login --force-badname debbugs +# m find /o/debbugs -xdev -path ./var/tmp -prune -o -uid $uid -execdir chown -h 610 {} + +# m find /o/debbugs -xdev -path ./var/tmp -prune -o -gid $gid -execdir chgrp -h 610 {} + +# elif [[ $uid != 610 ]]; then +# err debbugs exist but is not uid 610: investigate +# fi + # * start / stop services reifactive dnsmasq nscd @@ -3483,6 +3772,8 @@ case $HOSTNAME in else m systemctl --now enable $vpnser fi + ;;& + bk) if ! systemctl is-active clamav-daemon >/dev/null; then m systemctl --now enable clamav-daemon out=$(rsync -aiSAX --chown=root:root --chmod=g-s /a/bin/ds/filesystem/etc/systemd/system/epanicclean.service /etc/systemd/system) @@ -3643,6 +3934,9 @@ EOF test_to=${test_tos[0]} for t in ${test_tos[@]:1}; do + if [[ $test_from == *@gnu.org && $t == *@gnu.org ]]; then + continue + fi test_to+=", $t" done case $test_from in