From: Ian Kelling Date: Sun, 24 Apr 2022 19:54:19 +0000 (-0400) Subject: mainly alerting improvements X-Git-Url: https://iankelling.org/git/?p=distro-setup;a=commitdiff_plain;h=da085f71e5ae00a9f78ed903be6ae675f66eacfa mainly alerting improvements --- diff --git a/brc b/brc index 733a7c4..e0f2a88 100644 --- a/brc +++ b/brc @@ -1021,6 +1021,11 @@ ic() { git commit -am "$*" } +ipp() { + git pull + git push +} + ifn() { # insensitive find diff --git a/btrfsmaint b/btrfsmaint index eab8376..8e72141 100755 --- a/btrfsmaint +++ b/btrfsmaint @@ -100,6 +100,7 @@ main() { cat $tmp >$mnt/btrfs-dev-stats fi fi + rm -f $tmp if ! $idle; then if $dryrun; then diff --git a/conflink b/conflink index defb832..a3f6e35 100755 --- a/conflink +++ b/conflink @@ -157,6 +157,7 @@ for t in systemstatus epanicclean btrfsmaintstop dynamicipupdate; do reload_systemd=true fi done +rm -f /etc/cron.daily/check-lets-encrypt-ssl-settings c_dirs=(/a/c{,/machine_specific/$HOSTNAME}) case $user in diff --git a/filesystem/etc/cron.d/ian b/filesystem/etc/cron.d/ian index 8a0ac95..24fc23c 100644 --- a/filesystem/etc/cron.d/ian +++ b/filesystem/etc/cron.d/ian @@ -11,7 +11,5 @@ MAILTO=root 14 * * * * root /a/bin/ds/bk-backup |& log-once -24 bk-backup 0 7 * * * iank failmail myupgrade-iank 20 7 * * * root myupgrade |& log-once -1 myupgrade -# maybe try this again sometime. it needs updating to be like -# myupgrade. -#0 * * * * root mycheckrestart |& log-once -1 mycheckrestart 19 * * * * root check-crypttab +4 20 * * 5 iank /usr/local/bin/check-lets-encrypt-ssl-settings diff --git a/filesystem/etc/cron.daily/check-lets-encrypt-ssl-settings b/filesystem/usr/local/bin/check-lets-encrypt-ssl-settings similarity index 93% rename from filesystem/etc/cron.daily/check-lets-encrypt-ssl-settings rename to filesystem/usr/local/bin/check-lets-encrypt-ssl-settings index c108521..8fb11c9 100755 --- a/filesystem/etc/cron.daily/check-lets-encrypt-ssl-settings +++ b/filesystem/usr/local/bin/check-lets-encrypt-ssl-settings @@ -15,9 +15,6 @@ source ~/.bashrc -set -eE -o pipefail -trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR - if [[ ! -e /dev/shm/iank-status ]]; then exit 0 fi diff --git a/machine_specific/bitfolk/filesystem/etc/unbound/unbound.conf.d/ian.conf b/machine_specific/bitfolk/filesystem/etc/unbound/unbound.conf.d/ian.conf new file mode 100644 index 0000000..b31ed34 --- /dev/null +++ b/machine_specific/bitfolk/filesystem/etc/unbound/unbound.conf.d/ian.conf @@ -0,0 +1,22 @@ +server: + +# Based on reading the manual, it seems these log options would create +# duplicative logs to verbosity 4, but they may also be less verbose and +# have the info we want. dunno, maybe try it sometime. +# +#log-queries: yes +#log-replies: yes +#log-tag-queryreply: yes +#log-servfail: yes +#val-log-level: 1 +verbosity: 4 + +interface: 127.0.0.1 +interface: ::1 +interface: 10.173.8.1 +# from https://nlnetlabs.nl/documentation/unbound/unbound.conf/ +# it is unclear if we want both or one of these. +ip-transparent: yes +ip-freebind: yes + +access-control: 10.173.8.0/24 allow diff --git a/machine_specific/vps/filesystem/etc/bind/named.conf.options b/machine_specific/bk/filesystem/etc/bind/named.conf.options similarity index 79% rename from machine_specific/vps/filesystem/etc/bind/named.conf.options rename to machine_specific/bk/filesystem/etc/bind/named.conf.options index 157e295..42f1a3e 100644 --- a/machine_specific/vps/filesystem/etc/bind/named.conf.options +++ b/machine_specific/bk/filesystem/etc/bind/named.conf.options @@ -21,18 +21,11 @@ options { //======================================================================== dnssec-validation auto; - listen-on-v6 { any; }; - -// iank, allow queries from network namespace -// even if the interface doesnt exist when bind starts. -allow-recursion { - 10.173.8.0/24; - localhost; -}; -allow-query-cache { - 10.173.8.0/24; - localhost; -}; + // iank: only listen on our public addresses, lets use unbound for + // local process resolution, like we do at fsf, so debugging issues + // works for both. + listen-on { 85.119.83.50; }; + listen-on-v6 { 2001:ba8:1f1:f0c9::2; }; // end options }; diff --git a/machine_specific/je/filesystem/etc/bind/named.conf.options b/machine_specific/je/filesystem/etc/bind/named.conf.options new file mode 100644 index 0000000..5434fa5 --- /dev/null +++ b/machine_specific/je/filesystem/etc/bind/named.conf.options @@ -0,0 +1,31 @@ +// this is the default for t10, plus my commented additions +options { + directory "/var/cache/bind"; + + // If there is a firewall between you and nameservers you want + // to talk to, you may need to fix the firewall to allow multiple + // ports to talk. See http://www.kb.cert.org/vuls/id/800113 + + // If your ISP provided one or more IP addresses for stable + // nameservers, you probably want to use them as forwarders. + // Uncomment the following block, and insert the addresses replacing + // the all-0's placeholder. + + // forwarders { + // 0.0.0.0; + // }; + + //======================================================================== + // If BIND logs error messages about the root key being expired, + // you will need to update your keys. See https://www.isc.org/bind-keys + //======================================================================== + dnssec-validation auto; + + // iank: only listen on our public addresses, lets use unbound for + // local process resolution, like we do at fsf, so debugging issues + // works for both. + listen-on { 85.119.82.128; }; + listen-on-v6 { 2001:ba8:1f1:f09d::2; }; + +// end options +}; diff --git a/mail-setup b/mail-setup index ea2b1f2..72af2a1 100755 --- a/mail-setup +++ b/mail-setup @@ -4,10 +4,6 @@ # SPDX-License-Identifier: AGPL-3.0-or-later -# todo: install new alertmanager, like new prometheus - -# todo: setup a logrotate for /var/log/mymain and mypanic - # todo: setup an alert for bouncing test emails. # todo: bounces to my fsf mail can come from fsf@iankelling.org, @@ -22,7 +18,6 @@ # todo: harden dovecot. need to do some research. one way is for it to only listen on a wireguard vpn interface, so only clients that are on the vpn can access it. # todo: consider hardening cups listening on 0.0.0.0 # todo: stop/disable local apache, and rpc.mountd, and kdeconnect when not in use. -# todo: check that spamd and unbound only listen locally. # todo: hosts should only allow external mail that is authed and # destined for backup route. it is a minor issue since traffic is @@ -395,6 +390,7 @@ EOF fi # light version of exim does not have sasl auth support. +# note: for bitfolk hosts, unbound has important config with conflink. pi-nostart exim4 exim4-daemon-heavy spamassassin unbound clamav-daemon wireguard # note: pyzor debian readme says you need to run some initialization command @@ -1097,6 +1093,11 @@ bk.b8.nz * F,1d,4m;F,14d,1h eggs.gnu.org * F,1d,4m;F,14d,1h fencepost.gnu.org * F,1d,4m;F,14d,1h +# afaik our retry doesnt need this, but just using everything +mx.amnimal.ninja * F,1d,4m;F,14d,1h +mx.expertpathologyreview.com * F,1d,4m;F,14d,1h + + mail.fsf.org * F,1d,15m;F,14d,1h EOF @@ -2979,6 +2980,27 @@ case $HOSTNAME in # this makes it easier to see which exim is doing what log_file_path = /var/log/exim4/my%s EOF + + cat >/etc/logrotate.d/myexim <<'EOF' +/var/log/exim4/mymain /var/log/exim4/myreject { + daily + missingok + rotate 1000 + delaycompress + notifempty + nocreate +} +/var/log/exim4/mypanic { + size 10M + missingok + rotate 10 + compress + delaycompress + notifempty + nocreate +} +EOF + # If we ever wanted to have a separate spool, # we could do it like this. # cat >>/etc/exim4/conf.d/main/000_local-nn <<'EOF' @@ -3184,12 +3206,32 @@ PATH=/usr/bin:/bin:/usr/local/bin MAILTO=daylert@iankelling.org */5 * * * * $u send-test-forward |& log-once send-test-forward */10 * * * * root chmod -R g+rw /m/md/bounces |& log-once -1 bounces-chmod -*/5 * * * * root timeout 290 mailtest-check slow |& log-once -4 mailtest-check +# todo: delete, this is old +#*/5 * * * * root timeout 290 mailtest-check slow |& log-once -1 mailtest-check # if a bounce happened yesterday, dont let it slip through the cracks 8 1 * * * root export MAILTO=alerts@iankelling.org; [[ -s /var/log/exim4/mainlog.1 ]] && awk '\$5 == "**"' /var/log/exim4/mainlog.1 EOF + + m sudo rsync -ahhi --chown=root:root --chmod=0755 \ /b/ds/mailtest-check /b/ds/check-remote-mailqs /usr/local/bin/ + cat >/etc/systemd/system/mailtest-check.service <<'EOF' +[Unit] +Description=mailtest-check +After=local-fs.target +StartLimitIntervalSec=0 + +[Service] +Type=simple +ExecStart=/usr/local/bin/mailtest-check slow +Restart=always +RestartSec=60 + +[Install] +WantedBy=graphical.target +EOF + sysd-prom-fail-install mailtest-check + sre mailtest-check ;;& $MAIL_HOST) test_froms=(ian@iankelling.org z@zroe.org iank@gnu.org) @@ -3201,24 +3243,33 @@ EOF EOF ;;& bk) - test_froms=(testignore@expertpathologyreview.com testignore@amnimal.ninja) - test_tos=(testignore@iankelling.org testignore@zroe.org testignore@je.b8.nz) + test_froms=(testignore@amnimal.ninja testignore@expertpathologyreview.com) + test_tos=(testignore@iankelling.org testignore@je.b8.nz) + # We dont need to send from different addresses to the same + # address. this breaks down our nice elegant logic of building up + # froms and tos , so I just handle expertpath in a special case + # below and set the to: to be testignore@zroe.org. If we did sent + # that way, it would also mess up our mailtest-check logic that + # finds which messages to check. + # for example: from testignore@amnimal.ninja to: testignore@iankelling.org testignore@zroe.org + # that would become 2 messages and we'd only check 1. ;;& je) test_froms=(testignore@je.b8.nz) test_tos=(testignore@iankelling.org testignore@zroe.org testignore@expertpathologyreview.com testignore@amnimal.ninja) ;;& $MAIL_HOST|bk|je) - test_to=${test_tos[0]} - # dont put these test messages into the sent folder or else it will + + # Dont put these test messages into the sent folder or else it will # overwhelm it, plus i dont want to save a copy at all. - echo $test_to > /etc/exim4/ignore-sent - for t in ${test_tos[@]:1}; do - test_to+=", $t" + rm -f /etc/exim4/ignore-sent + for t in ${test_tos[@]}; do echo $t >> /etc/exim4/ignore-sent done + cat >/usr/local/bin/send-test-forward <<'EOF' #!/bin/bash +# we remove from the queue older than 4.3 minutes since we send every 5 minutes. olds=( $(/usr/sbin/exiqgrep -o 260 -i -r '^(testignore@(iankelling\.org|zroe\.org|expertpathologyreview\.com|amnimal\.ninja|je\.b8\.nz)|jtuttle@gnu\.org)$') ) @@ -3227,6 +3278,17 @@ if (( ${#olds[@]} )); then fi EOF for test_from in ${test_froms[@]}; do + + test_to=${test_tos[0]} + for t in ${test_tos[@]:1}; do + test_to+=", $t" + done + case $test_from in + testignore@expertpathologyreview.com) + test_to=testignore@zroe.org + ;; + esac + cat >>/usr/local/bin/send-test-forward <>/var/lib/prometheus/node-exporter/mailtest-check.prom.$$ + fi +} -# spamassassin checking takes about 8 seconds. only do that every -# once in a while. +#### begin arg processing #### +# spamassassin checking takes about 8 seconds. slow=false if [[ $1 == slow ]]; then slow=true @@ -46,89 +55,77 @@ fi if [[ $1 == nonint ]]; then int=false fi +#### end arg processing #### if ! $int; then sleep 60 fi -# avoid errors like this: -# Nov 8 08:16:05.439 [6080] warn: plugin: failed to parse plugin (from @INC): Can't locate Mail/SpamAssassin/Plugin/WLBLEval.pm: lib/Mail/SpamAssassin/Plugin/WLBLEval.pm: Permission denied at (eval 59) line 1. -#Nov 8 08:16:05.439 [6080] warn: plugin: failed to parse plugin (from @INC): Can't locate Mail/SpamAssassin/Plugin/VBounce.pm: lib/Mail/SpamAssassin/Plugin/VBounce.pm: Permission denied at (eval 60) line 1. -# i dont know why, i just found the solution online -cd /m/md + # TODO, get je to deliver the local mailbox: /m/md/INBOX # dovecot appears to setup, i can t be sure. -case $HOSTNAME in - bk) - folders=(/m/md/{expertpathologyreview.com,amnimal.ninja}/testignore) - froms=(ian@iankelling.org z@zroe.org testignore@je.b8.nz iank@gnu.org) - ;; - je) - froms=(ian@iankelling.org z@zroe.org testignore@expertpathologyreview.com testignore@amnimal.ninja) - folders=(/m/md/je.b8.nz/testignore) - ;; - *) - folders=(/m/md/l/testignore) - froms=(testignore@je.b8.nz testignore@expertpathologyreview.com testignore@amnimal.ninja ian@iankelling.org z@zroe.org iank@gnu.org) - if ! $int; then - timeout 120 rsync --chown iank:iank -e "ssh -oIdentitiesOnly=yes -F /dev/null -i /root/.ssh/jtuttle" -t --inplace -r 'jtuttle@fencepost.gnu.org:/home/j/jtuttle/Maildir/new/' /m/md/l/testignore/new - fi - ;; -esac -getspamdpid() { - if [[ ! $spamdpid || ! -d /proc/$spamdpid ]]; then - # try twice in case we are restarting, it happens. - for i in 1 2; do - spamdpid=$(systemctl show --property MainPID --value spamassassin | sed 's/^[10]$//' ||:) - if [[ $spamdpid ]]; then - break + +main() { + + case $HOSTNAME in + bk) + folders=(/m/md/{expertpathologyreview.com,amnimal.ninja}/testignore) + froms=(ian@iankelling.org z@zroe.org testignore@je.b8.nz iank@gnu.org) + ;; + je) + froms=(ian@iankelling.org z@zroe.org testignore@expertpathologyreview.com testignore@amnimal.ninja) + folders=(/m/md/je.b8.nz/testignore) + ;; + *) + folders=(/m/md/l/testignore) + froms=(testignore@je.b8.nz testignore@expertpathologyreview.com testignore@amnimal.ninja ian@iankelling.org z@zroe.org iank@gnu.org) + if ! $int; then + timeout 120 rsync --chown iank:iank -e "ssh -oIdentitiesOnly=yes -F /dev/null -i /root/.ssh/jtuttle" -t --inplace -r 'jtuttle@fencepost.gnu.org:/home/j/jtuttle/Maildir/new/' /m/md/l/testignore/new fi - sleep 30 - done - fi -} -getspamdpid -pr() { + ;; + esac + + + # avoid errors like this: + # Nov 8 08:16:05.439 [6080] warn: plugin: failed to parse plugin (from @INC): Can't locate Mail/SpamAssassin/Plugin/WLBLEval.pm: lib/Mail/SpamAssassin/Plugin/WLBLEval.pm: Permission denied at (eval 59) line 1. + #Nov 8 08:16:05.439 [6080] warn: plugin: failed to parse plugin (from @INC): Can't locate Mail/SpamAssassin/Plugin/VBounce.pm: lib/Mail/SpamAssassin/Plugin/VBounce.pm: Permission denied at (eval 60) line 1. + # i dont know why, i just found the solution online + cd /m/md + + + getspamdpid + # first time we write, overwrite anything existing if [[ -e /var/lib/prometheus/node-exporter ]]; then - cat >>/var/lib/prometheus/node-exporter/mailtest-check.prom.$$ - fi -} -# first time we write, overwrite anything existing -if [[ -e /var/lib/prometheus/node-exporter ]]; then - cat >/var/lib/prometheus/node-exporter/mailtest-check.prom.$$ </var/lib/prometheus/node-exporter/mailtest-check.prom.$$ < 0 )) EOF -fi -e spamdpid: $spamdpid -if [[ ! $spamdpid ]]; then - echo $HOSTNAME mailtest spamd pid not found. systemctl status spamassassin: - systemctl status spamassassin -fi -tmpfile=$(mktemp) -declare -i unexpected=0 -for folder in ${folders[@]}; do - for from in ${froms[@]}; do - latest= - last_sec=0 - - if ! grep -rlFx "From: $from" $folder/{new,cur} >$tmpfile; then - e "no message found from: $from" - continue - fi - # webmail sends them to cur it seems - while read -r file; do - if [[ $file -nt $latest ]]; then - latest=$file + fi + e spamdpid: $spamdpid + if [[ ! $spamdpid ]]; then + echo mailtest spamd pid not found. systemctl status spamassassin: + systemctl status spamassassin + fi + tmpfile=$(mktemp) + declare -i unexpected=0 + for folder in ${folders[@]}; do + for from in ${froms[@]}; do + latest= + last_sec=0 + + if ! grep -rlFx "From: $from" $folder/{new,cur} >$tmpfile; then + echo "no message found from: $from" + continue fi - done <$tmpfile + # webmail sends them to cur it seems + while read -r file; do + if [[ $file -nt $latest ]]; then + latest=$file + fi + done <$tmpfile - if [[ ! $latest ]]; then - # 10 is an arbitrary bad value - unexpected+=10 - else to=$(awk '/^Envelope-to: / {print $2}' $latest) last_sec=$(awk '/^Subject: / {print $4}' $latest) @@ -205,37 +202,55 @@ for folder in ${folders[@]}; do if (( ${#missing[@]} )); then printf "missing %s" "${missing[*]}" fi - echo - echo mailtest-check: cat $latest: - cat $latest - echo mailtest-check: end of cat - printf "$(tput setaf 5 2>/dev/null ||:)█$(tput sgr0 2>/dev/null||:)%.0s" $(eval echo "{1..${COLUMNS:-60}}") + echo # ends our printf string buildup + + if $int; then + echo mailtest-check: cat $latest: + cat $latest + echo mailtest-check: end of cat + echo "$(tput setaf 5 2>/dev/null ||:)█$(tput sgr0 2>/dev/null||:)%.0s" $(eval echo "{1..${COLUMNS:-60}}") + fi fi fi # if spamdpid fi # if $slow - fi # if [[ $latest ]] - now=$EPOCHSECONDS - limit=$(( now - 60 * min_limit )) - age_sec=$(( now - last_sec )) - e $((age_sec / 60)):$(( age_sec % 60 )) ago. to:$to from:$from $latest + now=$EPOCHSECONDS + age_sec=$(( now - last_sec )) + e $((age_sec / 60)):$(( age_sec % 60 )) ago. to:$to from:$from $latest - if (( last_sec <= limit )); then - echo $HOSTNAME mailtest $folder $from $(date -d @$last_sec +'%a %m-%d %H:%M') - fi - # usec = unix seconds - pr </dev/null; then + mount -o subvol=debianbullseye_bootstrap $dev /mnt/tmptimer +fi cd /mnt/tmptimer for d in dev proc sys dev/pts; do [[ -d $d ]] @@ -39,3 +41,4 @@ for d in dev/pts dev proc sys; do umount $d fi done +umount /mnt/tmptimer