From 602a1874cc11a7d371890cdae4c0dc982267ea89 Mon Sep 17 00:00:00 2001 From: Ian Kelling Date: Sun, 7 Aug 2022 20:04:03 -0400 Subject: [PATCH] mostly t11 fixes --- brc | 8 ++- brc2 | 58 ++++++++------- btrbk-run | 61 +++++++++------- distro-begin | 29 ++++++-- distro-end | 18 +---- filesystem/etc/profile.d/environment.sh | 12 ++++ .../etc/systemd/system/ssh-agent-iank.service | 7 +- .../etc/systemd/system/ssh-agent-root.service | 5 +- mail-setup | 70 ++++++++++++++++--- switch-mail-host | 1 - 10 files changed, 178 insertions(+), 91 deletions(-) diff --git a/brc b/brc index 6322eb9..7b01674 100644 --- a/brc +++ b/brc @@ -940,14 +940,18 @@ eless() { } ccomp less eless eqcat() { - exiqgrep -i -o 60 | while read -r i; do + exiqgrep -ir.\* -o 60 | while read -r i; do hlm exim -Mvc $i echo hlm exigrep $i /var/log/exim4/mainlog | cat ||: done } eqrmf() { - exiqgrep -i | xargs exim -Mrm + # other ways to get the list of message ids: + # exim -bp | awk 'NF == 4 {print $3}' + # # this is slower 160ms, vs 60. + # exipick -i + exiqgrep -ir.\* | xargs exim -Mrm } econfdevnew() { diff --git a/brc2 b/brc2 index 54fe47b..fdb2c06 100644 --- a/brc2 +++ b/brc2 @@ -87,6 +87,8 @@ multimic() { pactl load-module module-remap-source source_name=iancombine master=ianinput.monitor source_properties=device.description=iancombine } +# h ssh test +# For testing restrictive ssh. hstest() { install-my-scripts d=$(mktemp -d) @@ -94,13 +96,20 @@ hstest() { s command ssh -F $d/config -i /q/root/h "$@" } -hrtest() { +# h rsync test +# For testing restrictive rsync +hrtest() { # install-my-scripts d=$(mktemp -d) sed '/^ *IdentityFile/d' ~/.ssh/config >$d/config s rsync -e "ssh -F $d/config -i /q/root/h" "$@" } +# rsync as root and avoid the default restrictive h key & config. +rootrsync() { + s rsync -e "ssh -F /root/.ssh/confighome" "$@" +} + slemacs() { local arg rtime v @@ -264,10 +273,10 @@ tback() { # s sshfs bu@$host:/bu/home/md /bu/mnt -o reconnect,ServerAliveInterval=20,ServerAliveCountMax=30 -o allow_other eqgo() { - enn -M $(exiqgrep -i) + enn -M $(exiqgrep -i -r.\*) } eqgo1() { - enn -M $(exiqgrep -i|h1) + enn -M $(exipick -i -r.\*|h1) } @@ -1761,10 +1770,9 @@ testexim() { # # -t = get recipient from header exim -d -t <<'EOF' -From: i@dmarctest.b8.nz -To: mailman@dev.fsf.org +From: root@$(hostname-f) +To: root@$(hostname-f) Subject: test2 -Reply-to: rtest@iankelling.org This is a test message. EOF @@ -1837,16 +1845,23 @@ enn() { # get pid of systemd service servicepid() { - local pid unit + local pid unit dir unit="$1" pid=$(systemctl show --property MainPID --value "$unit") case $pid in [1-9]*) : ;; *) + + dir=/sys/fs/cgroup/system.slice + if [[ ! -d $dir ]]; then + # t10 and older directory. + dir=/sys/fs/cgroup/systemd/system.slice + fi + # 0 or empty. This file includes the MainPid, so I expect we # could just get this in the first place, but i don't know if that # is always the case. - pid=$(head -n1 /sys/fs/cgroup/systemd/system.slice/${unit%.service}.service/cgroup.procs) + pid=$(head -n1 $dir/${unit%.service}.service/cgroup.procs) ;; esac if [[ $pid ]]; then @@ -1904,33 +1919,22 @@ nmtc() { } mailnncheck() { - local p pid ns mailnn + local unit pid ns mailnn # mailvpn would belong on the list if using openvpn - for p in mailnn unbound dovecot spamassassin exim4 radicale; do - - - pid=$(systemctl show --property MainPID --value $unit) - case $pid in - [1-9]*) : ;; - *) - # 0 or empty. This file includes the MainPid, so I expect we - # could just get this in the first place, but i don't know if - # that is always the case. - pid=$(head -n1 /sys/fs/cgroup/systemd/system.slice/${unit%.service}.service/cgroup.procs) - ;; - esac - echo p=$p pid=$pid + for unit in mailnn unbound dovecot spamassassin exim4 radicale; do + pid=$(servicepid $unit) + echo debug: unit=$unit pid=$pid if [[ ! $pid ]]; then - echo failed to find pid for $p + echo failed to find pid for unit=$unit continue fi if ! ns=$(s readlink /proc/$pid/ns/net); then - echo failed to find ns for $p pid=$pid + echo failed to find ns for unit=$unit pid=$pid continue fi if [[ $mailnn ]]; then if [[ $ns != "$mailnn" ]]; then - echo "$p ns $ns != $mailnn" + echo "$unit ns $ns != $mailnn" fi else mailnn=$ns @@ -2212,7 +2216,7 @@ if type -P rg &>/dev/null; then # --no-messages because of annoying errors on broken symlinks # -z = search .gz etc files # -. = search dotfilesq - rg() { command rg -. -z --no-messages -L -i -M 900 --no-ignore-parent --no-ignore-vcs -g '!.git' "$@" || return $?; } + rg() { command rg -. -z --no-messages -L -i -M 900 --no-ignore-parent --no-ignore-vcs -g '!.git' -g '!auto-save-list' -g '!.savehist' "$@" || return $?; } #fails if not exist. ignore complete -r rg 2>/dev/null ||: else diff --git a/btrbk-run b/btrbk-run index 7f72a58..1b6cec5 100644 --- a/btrbk-run +++ b/btrbk-run @@ -358,7 +358,11 @@ local_zone=$(date +%z) if [[ $source ]]; then if ! zone=$(ssh root@$source date +%z); then - die failed to ssh to root@$source + if $conf_only; then + echo "$0: warning: failed to ssh to root@$source" + else + die failed to ssh to root@$source + fi fi if [[ $zone != "$local_zone" ]]; then die "error: dont confuse yourself with multiple time zones. $h has different timezone than localhost" @@ -378,36 +382,41 @@ else if (( ${#remote_info[@]} != 3 )); then die "error: didnt get 3 fields in test ssh to target $h. investigate" fi + elif $conf_only; then + # Use some typical values in this case + root_size=$(( 1024 * 1024 * 2000 )) #2tb + percent_used=10 + zone=$(date +%z) + else + sshfail+=($h) + continue + fi + # we may be booted into a bootstrap fs or something + min_root_kb=$(( 1024 * 1024 * 200 )) # 200 gb + if (( root_size < min_root_kb )); then + continue + fi - # we may be booted into a bootstrap fs or something - min_root_kb=$(( 1024 * 1024 * 200 )) # 200 gb - if (( root_size < min_root_kb )); then - continue - fi - - if (( percent_used >= 98 )); then - die "error: filesystem on target $h is $percent_used % full" - fi + if (( percent_used >= 98 )); then + die "error: filesystem on target $h is $percent_used % full" + fi - # This is a separate ssh because xprintidle can fail and thats ok. - if $cron && idle_ms=$(timeout -s 9 6 ssh $h DISPLAY=:0 xprintidle); then - if (( idle_ms < min_idle_ms )); then + # This is a separate ssh because xprintidle can fail and thats ok. + if $cron && idle_ms=$(timeout -s 9 6 ssh $h DISPLAY=:0 xprintidle); then + if (( idle_ms < min_idle_ms )); then - # Ignore this host. i sometimes use a non-main machine for - # testing or web browsing, knowing that everything will be wiped - # by the next backup, but I dont want it to happen as Im using - # it from cronjob. - e "warning: $h: active X session in the last 15 minutes, skipping for now" - continue - fi - fi - sshable+=($h) - if [[ $zone != "$local_zone" ]]; then - die "error: dont confuse yourself with multiple time zones. $h has different timezone than localhost" + # Ignore this host. i sometimes use a non-main machine for + # testing or web browsing, knowing that everything will be wiped + # by the next backup, but I dont want it to happen as Im using + # it from cronjob. + e "warning: $h: active X session in the last 15 minutes, skipping for now" + continue fi - else - sshfail+=($h) + fi + sshable+=($h) + if [[ $zone != "$local_zone" ]]; then + die "error: dont confuse yourself with multiple time zones. $h has different timezone than localhost" fi done if [[ ! ${sshable[*]} ]] || { ! $cron && [[ ${sshfail[*]} ]]; }; then diff --git a/distro-begin b/distro-begin index a0e9b07..d0b03f6 100755 --- a/distro-begin +++ b/distro-begin @@ -350,11 +350,28 @@ case $HOSTNAME in esac +lines=( + "/etc/resolved-nsswitch/nsswitch.conf r," + "/etc/basic-nsswitch/nsswitch.conf r," + # Aug 06 23:09:11 kd audit[3995]: AVC apparmor="DENIED" operation="connect" profile="/usr/bin/freshclam" name="/run/systemd/resolve/io.systemd.Resolve" pid=3995 comm="freshclam" requested_mask="wr" denied_mask="wr" fsuid=109 ouid=101 + # I dont know if this is quite the right fix, but I saw other sockets + # in the nameservice files that were rw, so figured it was ok to add this and it worked. + "/run/systemd/resolve/io.systemd.Resolve rw," +) f=/etc/apparmor.d/abstractions/nameservice -if [[ -e $f ]] && ! grep -q /etc/resolved-nsswitch/nsswitch.conf $f; then - sudo sed -i '/\/etc\/nsswitch.conf/a /etc/resolved-nsswitch/nsswitch.conf r,' $f - sudo sed -i '/\/etc\/nsswitch.conf/a /etc/basic-nsswitch/nsswitch.conf r,' $f - if sytemctl is-enabled apparmor; then +apparmor_reload=false +if [[ -e $f ]]; then + for l in "${lines[@]}"; do + if ! grep -qF "$l" $f; then + sudo sed -i "/\/nsswitch.conf/a $l" $f + apparmor_reload=true + if ! grep -qF "$l" $f; then + echo "$0: failed editing $f. investigate" + exit 1 + fi + fi + done + if $apparmor_reload && systemctl is-enabled apparmor; then m ser reload apparmor fi fi @@ -466,7 +483,7 @@ pi ${p1[@]} ######## fix evbug bug ###### case $(debian-codename-compat) in - xenial|bionic|focal) + xenial|bionic|focal|jammy) # noticed in flidas. dunno if it affects any others #https://bugs.launchpad.net/ubuntu/+source/module-init-tools/+bug/240553 #https://wiki.debian.org/KernelModuleBlacklisting @@ -731,8 +748,10 @@ if $emacs; then else if $recompile; then /a/bin/buildscripts/emacs + /a/bin/buildscripts/mu4e else /a/bin/buildscripts/emacs --no-r + /a/bin/buildscripts/mu4e --no-r fi fi # the first pup command can kill off our /etc/ mod, so rerun this diff --git a/distro-end b/distro-end index 6e04cc9..09ad1de 100755 --- a/distro-end +++ b/distro-end @@ -609,23 +609,6 @@ sudo rm -f /etc/cron.d/unattended-upgrade-reboot /usr/local/bin/zelous-unattende # Pin-Priority: 500 # EOF -if [[ -e /etc/wireguard/wghole.conf ]]; then - reload=false - if [[ ! -e /etc/systemd/system/wg-quick@wghole.service.d/override.conf ]]; then - reload=true - fi - sudo mkdir -p /etc/systemd/system/wg-quick@wghole.service.d - sd /etc/systemd/system/wg-quick@wghole.service.d/override.conf <<'EOF' -[Unit] -StartLimitIntervalSec=0 - -[Service] -Restart=on-failure -RestartSec=20 -EOF - if $reload; then ser daemon-reload; fi - sgo wg-quick@wghole -fi ###### begin website setup case $HOSTNAME in @@ -2004,6 +1987,7 @@ case $HOSTNAME in # either use iptables or, in # /etc/default/prometheus-node-exporter # listen on the wireguard interface + *) wgip=$(command sudo sed -rn 's,^ *Address *= *([^/]+).*,\1,p' /etc/wireguard/wghole.conf) # old filename. remove once all hosts are updated. diff --git a/filesystem/etc/profile.d/environment.sh b/filesystem/etc/profile.d/environment.sh index 80a1d5a..9e4feef 100644 --- a/filesystem/etc/profile.d/environment.sh +++ b/filesystem/etc/profile.d/environment.sh @@ -116,6 +116,18 @@ for _file in $(pgrep -a '^ssh-agent$' | sed -r 's/.*-a *([^ ]+).*/\1/'); do fi done +# in t11, user ssh-agent once again stopped working, it isn't started anymore +# by i3. So, enough is enough, I just made a service that will try to +# start it all the time for the user. If the var isn't set by the above, +# just set them to the values I know are in the service. +if ! test "$SSH_AUTH_SOCK"; then + if [ "$EUID" == 0 ]; then + export SSH_AUTH_SOCK=/run/openssh_agent + else + export SSH_AUTH_SOCK=/run/user/1000/openssh_agent + fi +fi + # background: # ubuntu has 002 for non-system users, debian has 022. 002 makes groups diff --git a/filesystem/etc/systemd/system/ssh-agent-iank.service b/filesystem/etc/systemd/system/ssh-agent-iank.service index 17d077c..8702043 100644 --- a/filesystem/etc/systemd/system/ssh-agent-iank.service +++ b/filesystem/etc/systemd/system/ssh-agent-iank.service @@ -1,11 +1,14 @@ [Unit] -Description=OpenSSH Agent for root +Description=OpenSSH Agent for iank Documentation=man:ssh-agent(1) +StartLimitIntervalSec=0 [Service] User=iank Environment=DISPLAY=:0 -ExecStart=/usr/bin/ssh-agent -a /run/user/1000/openssh_agent +ExecStart=/usr/bin/ssh-agent -D -a /run/user/1000/openssh_agent +Restart=always +RestartSec=30 [Install] WantedBy=multi-user.target diff --git a/filesystem/etc/systemd/system/ssh-agent-root.service b/filesystem/etc/systemd/system/ssh-agent-root.service index 1070cf6..ac0a965 100644 --- a/filesystem/etc/systemd/system/ssh-agent-root.service +++ b/filesystem/etc/systemd/system/ssh-agent-root.service @@ -3,9 +3,8 @@ Description=OpenSSH Agent for root Documentation=man:ssh-agent(1) [Service] -# using -d for debug info as i try this out -#ExecStart=/usr/bin/ssh-agent -D -a /run/openssh_agent -ExecStart=/usr/bin/ssh-agent -d -a /run/openssh_agent +# note: use -d instead of -D for debug info +ExecStart=/usr/bin/ssh-agent -D -a /run/openssh_agent [Install] WantedBy=multi-user.target diff --git a/mail-setup b/mail-setup index 9b9a97c..b3cb092 100755 --- a/mail-setup +++ b/mail-setup @@ -3,6 +3,8 @@ # Copyright (C) 2019 Ian Kelling # SPDX-License-Identifier: AGPL-3.0-or-later +# todo: max line length macro changed in t11. look into it +# todo: check that all macros we use are still valid in t11 # todo: setup an alert for bouncing test emails. @@ -460,7 +462,7 @@ m usermod -a -G Debian-exim clamav i /etc/systemd/system/clamav-daemon.service.d/fix.conf </etc/exim4/conf.d/main/000_local2 </etc/exim4/conf.d/transport/11_iank <<'EOF' +# This unsets the default macro defined in on t11 in +# /etc/exim4/conf.d/transport/10_exim4-config_transport-macros +# It seems like a very odd choice that this has become +# the default in t11. Normal smarthost clients use username/password +# auth. Oh well. +REMOTE_SMTP_SMARTHOST_TLS_VERIFY_HOSTS == +EOF + cat >/etc/exim4/conf.d/main/000_local <<'EOF' MAIN_TLS_ENABLE = true @@ -1183,13 +1214,17 @@ smtp_accept_queue_per_connection = 500 DKIM_CANON = relaxed DKIM_SELECTOR = li -# from comments in +# From comments in # https://debian-administration.org/article/718/DKIM-signing_outgoing_mail_with_exim4 # and its best for this to align https://tools.ietf.org/html/rfc7489#page-8 # There could be some circumstance when the # from: isnt our domain, but the envelope sender is # and so still want to sign, but I cant think of any case. -DKIM_DOMAIN = ${lc:${domain:$rh_from:}} +#DKIM_DOMAIN = ${lc:${domain:$rh_from:}} +# In t11, we cant do the above anymore because this is tainted data used in a file lookup. +# /usr/share/doc/exim4/NEWS.Debian.gz suggests to use lookups to untaint data. +DKIM_DOMAIN = ${lookup {${domain:$rh_from:}}lsearch,ret=key{/etc/exim4/conf.d/my-dkim-domains}} + # The file is based on the outgoing domain-name in the from-header. # sign if key exists DKIM_PRIVATE_KEY = ${if exists{/etc/exim4/${dkim_domain}-private.pem} {/etc/exim4/${dkim_domain}-private.pem}} @@ -2475,9 +2510,17 @@ UPEX4OPTS='-o /etc/exim4/my.conf' # i use epanic-clean for alerting if there are bad paniclog entries E4BCD_WATCH_PANICLOG='no' EOF + # make exim be a nonroot setuid program. chown Debian-exim:Debian-exim /usr/sbin/exim4 # needs guid set in order to become Debian-exim chmod g+s,u+s /usr/sbin/exim4 + # need this to avoid error on service reload: + # 2022-08-07 18:44:34.005 [892491] pid 892491: SIGHUP received: re-exec daemon + # 2022-08-07 18:44:34.036 [892491] cwd=/var/spool/exim4 5 args: /usr/sbin/exim4 -bd -q30m -C /etc/exim4/my.conf + # 2022-08-07 18:44:34.043 [892491] socket bind() to port 25 for address (any IPv6) failed: Permission denied: waiting 30s before trying again (9 more tries) + # note: the daemon gives up and dies after retrying those 9 times. + # I came upon this by guessing and trial and error. + setcap CAP_NET_BIND_SERVICE+ei /usr/sbin/exim4 i /etc/exim4/trusted_configs <<'EOF' /etc/exim4/my.conf EOF @@ -2517,7 +2560,10 @@ ProtectHome=yes # this whole setting doesnt work. tried it with a newer systemd 250 though # an nspawn, and it worked there. InaccessiblePaths=${dirs[@]} -NoNewPrivileges=yes +# this gives us the permission denied error: +# socket bind() to port 25 for address (any IPv6) failed: Permission denied +# but we also have to set the file capabilities to avoid the error. +#NoNewPrivileges=yes ProtectSystem=yes # when we get newer systemd @@ -2966,7 +3012,7 @@ EOF echo | /a/exe/cedit nn /etc/hosts || [[ $? == 1 ]] echo | /a/exe/cedit mail /etc/dnsmasq-servers.conf || [[ $? == 1 ]] - + # note: condition duplicated at else if $bhost_t; then install -d /bu install -d -g Debian-exim -o Debian-exim -m 771 /bu/md @@ -3014,7 +3060,7 @@ Restart=always RestartSec=20 EOF - else + else # if $bhost_t; then cat >>/etc/exim4/update-exim4.conf.conf <>/etc/myexim4/conf.d/main/000_local-nn <<'EOF' + m rsync -ra --delete --delete-excluded --exclude=/conf.d/main/000_local-nn /etc/exim4/ /etc/myexim4 + cat >>/etc/myexim4/conf.d/main/000_local <<'EOF' # this makes it easier to see which exim is doing what log_file_path = /var/log/exim4/my%s EOF + + cat >/etc/logrotate.d/myexim <<'EOF' /var/log/exim4/mymain /var/log/exim4/myreject { daily @@ -3167,6 +3215,12 @@ if $reload; then m systemctl daemon-reload fi +# checking bhost_t is redundant, but could help us catch errors. +if $bhost_t || [[ -e /etc/wireguard/wghole.conf ]]; then + # todo: in mail-setup, we have a static list of backup hosts, not *y + m systemctl --now enable wg-quick@wghole +fi + sysd-prom-fail-install epanicclean m systemctl --now enable epanicclean diff --git a/switch-mail-host b/switch-mail-host index b27cde9..65ca08d 100644 --- a/switch-mail-host +++ b/switch-mail-host @@ -149,7 +149,6 @@ while $active; do done # ensure these are unused before doing anything - e "On $new_host: umounting /m and /o, checking emacs" $new_shell bash -s <<'EOF' set -eE -- 2.30.2