From 0cff83aea2761109996e246123eed25985132e0a Mon Sep 17 00:00:00 2001 From: Ian Kelling Date: Sun, 22 Aug 2021 12:16:03 -0400 Subject: [PATCH] various improvements --- brc | 42 ++++++++++- brc2 | 8 +- distro-end | 5 +- filesystem/etc/cron.d/ian | 2 +- filesystem/usr/local/bin/myupgrade | 97 ++++++------------------- filesystem/usr/local/bin/myupgrade-iank | 44 +++++++++++ mail-setup | 24 ++---- mailtest-check | 5 +- primary-setup | 31 ++++---- rootsshsync | 5 +- subdir_files/sieve/lists.sieve | 1 + switch-mail-host | 16 ++-- system-status | 11 +-- untrusted-network | 28 +++++-- 14 files changed, 179 insertions(+), 140 deletions(-) create mode 100644 filesystem/usr/local/bin/myupgrade-iank diff --git a/brc b/brc index eb42029..f768e9e 100644 --- a/brc +++ b/brc @@ -965,17 +965,53 @@ lld() { ll -d "$@"; } ccomp ls l lg lt lld ll +# low recursively +lowr() { + local f dirs i a + local -a all + for dirs in false true; do + for f; do + if [[ -d $f ]]; then + all=("$f"/**) + # reverse the order to rename the nested dirs first. + # note: 0 element is the dir itself + for ((i=${#all[@]}-1; i>=1; i--)); do + a="${all[i]}" + if $dirs && [[ -d $a ]]; then + # e dirs low "$a" # debug + low "$a" + elif ! $dirs && [[ ! -d $a && -e $a ]]; then + # debug + # e not dirs low "$a" # debug + low "$a" + fi + done + fi + # just rename all the top level args on the second pass + if $dirs; then + # e final dirs low "$f" # debug + low "$f" + fi + done + done +} low() { # make filenames lowercase, remove bad chars - local f new - for f in "$@"; do + local arg new dir f + for arg; do + arg="${arg%%+(/)}" # remove trailing slashes. assumes we have extglob on. + dir="${arg%/*}" + if (( ${#dir} == ${#arg} )); then + dir=. + fi + f="${arg##*/}" new="${f,,}" # downcase new="${new//[^[:alnum:]._-]/_}" # sub bad chars new="${new#"${new%%[[:alnum:]]*}"}" # remove leading/trailing non-alnum new="${new%"${new##*[[:alnum:]]}"}" # remove bad underscores, like __ and _._ new=$(echo $new | sed -r 's/__+/_/g;s/_+([.-])|([.-])_+/\1/g') - safe_rename "$f" "$new" || return 1 + safe_rename "$dir/$f" "$dir/$new" || return 1 done return 0 } diff --git a/brc2 b/brc2 index ccc4968..06eb839 100644 --- a/brc2 +++ b/brc2 @@ -934,7 +934,7 @@ ilogs() { ilog() { chan=${1:-#fsfsys} # use * instead of -r since that does sorted order - ssh root@iankelling.org "cd /var/lib/znc/moddata/log/iank/libera/$chan && hr && for x in *; do echo \$x; sed \"s/^./\${x%log}/\" \$x; hr; done" | less +G + ssh root@iankelling.org "for n in freenode libera; do cd /var/lib/znc/moddata/log/iank/\$n/$chan && hr && for x in *; do echo \$x; sed \"s/^./\${x%log}/\" \$x; hr; done; done" | less +G } o() { @@ -1328,7 +1328,7 @@ testsieve() { } runsieve() { c ~/sieve; cp personal{test,}.sieve; cp lists{test,}.sieve; cp personalend{test,}.sieve - _dosieve ~/sieve/main.sieve -eW ${1:-INBOX} delete + _dosieve ~/sieve/main.sieve -eWv ${1:-INBOX} delete } # mail related @@ -1450,10 +1450,10 @@ mailnncheck() { vpncmd() { - m sudo env "PATH=$PATH" nsenter -t $(pgrep -f "/usr/sbin/openvpn .* --config /etc/openvpn/.*client.conf") -n -m "$@" + m sudo -E env "PATH=$PATH" nsenter -t $(pgrep -f "/usr/sbin/openvpn .* --config /etc/openvpn/.*client.conf") -n -m "$@" } vpnf() { - vpncmd sudo -u iank env "PATH=$PATH" abrowser -no-remote -P vpn &r + vpncmd sudo -E -u iank env "PATH=$PATH" abrowser -no-remote -P vpn &r } vpn2f() { vpncmd sudo -u iank env "PATH=$PATH" abrowser -no-remote -P vpn2 &r diff --git a/distro-end b/distro-end index 66d6758..f4afcb7 100755 --- a/distro-end +++ b/distro-end @@ -562,7 +562,10 @@ sd /etc/cron.d/myupgrade <<'EOF' SHELL=/bin/bash # default is /usr/bin:/bin PATH=/usr/bin:/bin:/usr/local/bin -#20 7 * * * iank myupgrade |& log-once -1 myupgrade +0 7 * * * iank myupgrade-iank |& log-once -1 myupgrade +20 7 * * * root myupgrade |& log-once -1 myupgrade +# maybe try this again sometime. it needs updating to be like +# myupgrade. #0 * * * * root mycheckrestart |& log-once -1 mycheckrestart EOF ##### end automatic upgrades #### diff --git a/filesystem/etc/cron.d/ian b/filesystem/etc/cron.d/ian index 53dc845..d1851b6 100644 --- a/filesystem/etc/cron.d/ian +++ b/filesystem/etc/cron.d/ian @@ -1,7 +1,7 @@ SHELL=/bin/bash PATH=/usr/bin:/bin:/usr/local/bin:/a/exe MAILTO=root -*/10 * * * * iank rootsshsync |& log-once -15 rootsshsync +*/10 * * * * root rootsshsync |& log-once -15 rootsshsync # If theres any logged errors we didnt handle in 4 days, maybe we accidentally missed them, # so report if we did 4 9 * * 5 root /a/bin/ds/check-stale-alerts diff --git a/filesystem/usr/local/bin/myupgrade b/filesystem/usr/local/bin/myupgrade index f439562..1e68c1e 100755 --- a/filesystem/usr/local/bin/myupgrade +++ b/filesystem/usr/local/bin/myupgrade @@ -10,10 +10,7 @@ m() { printf "$pre %s\n" "$*"; "$@"; } e() { printf "$pre %s\n" "$*"; } err() { echo "[$(date +'%Y-%m-%d %H:%M:%S%z')]: $pre: $*" >&2; } -if [[ $EUID == 0 ]]; then - err "run as nonroot" - exit 1 -fi +[[ $EUID == 0 ]] || exec sudo -E "${BASH_SOURCE[0]}" "$@" hn=$(hostname -f) source /a/bin/bash_unpublished/source-state @@ -21,80 +18,32 @@ source /a/bin/bash_unpublished/source-state l() { "$@" |& systemd-cat -t myupgrade } -l /a/bin/buildscripts/rust - -has_x=false -for pkg in xorg wayland; do - if dpkg -s -- $x |& grep -Fx "Status: install ok installed" &> /dev/null; then - has_x=true - break - fi -done - -if $has_x; then - l /a/bin/buildscripts/tor-browser - l /a/bin/buildscripts/misc -fi - - -source /a/bin/distro-setup/path-add-function -export GOPATH=$HOME/go -path-add $GOPATH/bin -path-add /usr/local/go/bin -l /a/bin/buildscripts/go -l go get -u mvdan.cc/fdroidcl -# i disabled gvfs-daemon.service. I ran systemd-analyze --user dot, -# which doesnt show why it started, so its getting started by some -# startup script, and I dont want to track it down. -# note, this is duplicated in /a/bin/ds/filesystem/usr/local/bin/mycheckrestart -ignore_lines=( - "Found 0 processes using old versions of upgraded files" - "lsof: WARNING: can't stat() fuse.gvfsd-fuse file system /run/user/1000/gvfs" - "Output information may be incomplete." - "Found 0 processes using old versions of upgraded files" -) -out= -while read -r line; do - ignore=false - for l in "${ignore_lines[@]}"; do - if [[ $line == "$l" ]]; then - ignore=true - break - fi - done - if $ignore; then continue; fi - out+="$line"$'\n' -done < <(sudo /usr/sbin/checkrestart -p 2>&1) -if (( ${#out} )); then - printf "%s" "$out" | pee cat wall -fi - -if (( ${#out} )); then -dorestart=true +if checkrestart -p -t &>/dev/null; then + exit 0 fi - -# no automatic reboot for these hosts -if [[ $HOSTNAME == "$MAIL_HOST" || $HOSTNAME == kw ]]; then +if [[ $HOSTNAME == "$MAIL_HOST" ]]; then + # no automatic reboot for this host, just make an alert + checkrestart -p 2>/dev/null ||: exit 0 +else + l checkrestart -p 2>/dev/null ||: fi -if [[ -s /var/log/checkrestart.log ]] || $dorestart; then - for x in {30..1}; do - echo "pid $PID. unattended upgrade, rebooting in $((x*10)) seconds" | sudo wall -n - sleep 10 - done - for x in {30..1}; do - if ! fuser /var/lib/dpkg/lock &> /dev/null; then - echo "pid $PID. unattended upgrade, rebooting now" | pee cat "sudo wall -n" - sudo /a/bin/ds/keyscript-on - sudo /sbin/reboot - exit 0 - fi - echo "pid $PID. unattended upgrade reboot waiting 10 seconds for dpkg lock" | sudo wall -n - sleep 10 - done - echo "pid $PID. dpkg locked for 5 minutes, automatic reboot failed" | pee cat "sudo wall -n" -fi +for x in {30..1}; do + echo "pid $PID. unattended upgrade, rebooting in $((x*10)) seconds" | wall -n + sleep 10 +done +for x in {30..1}; do + if ! fuser /var/lib/dpkg/lock &> /dev/null; then + echo "pid $PID. unattended upgrade, rebooting now" | pee cat "wall -n" + /a/bin/ds/keyscript-on + /sbin/reboot + exit 0 + fi + echo "pid $PID. unattended upgrade reboot waiting 10 seconds for dpkg lock" | wall -n + sleep 10 +done +echo "pid $PID. dpkg locked for 5 minutes, automatic reboot failed" | pee cat "wall -n" diff --git a/filesystem/usr/local/bin/myupgrade-iank b/filesystem/usr/local/bin/myupgrade-iank new file mode 100644 index 0000000..6f9abfb --- /dev/null +++ b/filesystem/usr/local/bin/myupgrade-iank @@ -0,0 +1,44 @@ +#!/bin/bash +# Copyright (C) 2019 Ian Kelling +# SPDX-License-Identifier: AGPL-3.0-or-later + +if [ -z "$BASH_VERSION" ]; then echo "error: shell is not bash" >&2; exit 1; fi + +source /usr/local/lib/err +pre="${0##*/}:" +m() { printf "$pre %s\n" "$*"; "$@"; } +e() { printf "$pre %s\n" "$*"; } +err() { echo "[$(date +'%Y-%m-%d %H:%M:%S%z')]: $pre: $*" >&2; } + +if [[ $EUID == 0 ]]; then + err "run as nonroot" + exit 1 +fi + +hn=$(hostname -f) + +l() { + "$@" |& systemd-cat -t myupgrade-iank +} +l /a/bin/buildscripts/rust + +has_x=false +for pkg in xorg wayland; do + if dpkg -s -- $x |& grep -Fx "Status: install ok installed" &> /dev/null; then + has_x=true + break + fi +done + +if $has_x; then + l /a/bin/buildscripts/tor-browser + l /a/bin/buildscripts/misc +fi + + +# source /a/bin/distro-setup/path-add-function +# export GOPATH=$HOME/go +# path-add $GOPATH/bin +# path-add /usr/local/go/bin +# l /a/bin/buildscripts/go +# l go get -u mvdan.cc/fdroidcl diff --git a/mail-setup b/mail-setup index 37bc9d9..60eea8b 100755 --- a/mail-setup +++ b/mail-setup @@ -3,6 +3,9 @@ # Copyright (C) 2019 Ian Kelling # SPDX-License-Identifier: AGPL-3.0-or-later +# todo: emailing info@amnimal.ninja produces a bounce, user doesn't exist +# instead of a simple rejection like it should. + # todo: auto restart of je on checkrestart # todo: run mailping test after running, or otherwise @@ -125,11 +128,8 @@ fi [[ $EUID == 0 ]] || exec sudo -E "${BASH_SOURCE[0]}" "$@" -if [[ ! $SUDO_USER ]]; then - echo "$0: error: requires running as nonroot or sudo" - exit 1 -fi -u=$SUDO_USER + +u=$(id -nu 1000) usage() { @@ -364,7 +364,7 @@ EOF fi # light version of exim does not have sasl auth support. -pi-nostart exim4 exim4-daemon-heavy spamassassin openvpn unbound clamav-daemon +pi-nostart exim4 exim4-daemon-heavy spamassassin openvpn unbound clamav-daemon wireguard # note: pyzor debian readme says you need to run some initialization command # but its outdated. @@ -2454,16 +2454,8 @@ case $HOSTNAME in ;; esac -case $HOSTNAME in - $MAIL_HOST) - # we manually mount /bu/mnt before starting - m systemctl disable exim4 - m systemctl restart exim4 - ;; - *) - sre exim4 - ;; -esac +sre exim4 + case $HOSTNAME in bk) sre exim4in ;; esac diff --git a/mailtest-check b/mailtest-check index a5e2d32..404fb65 100755 --- a/mailtest-check +++ b/mailtest-check @@ -69,7 +69,6 @@ for folder in ${folders[@]}; do done < <(grep -rlFx "From: $from" $folder/{new,cur} ) if [[ $latest ]]; then - e latest = $latest last_sec=$(awk '/^Subject: / {print $4}' $latest) if $slow; then @@ -89,7 +88,8 @@ for folder in ${folders[@]}; do case $r in # we have a new domain, ignore this. # it seems like some versions of spamassassin do BODY_SINGLE_WORD, others dont, we dun care. - BODY_SINGLE_WORD|FROM_FMBLA_NEWDOM*|autolearn) : ;; + # bayes_00 is a new one indicating ham, we dont care if its missing. + BAYES_00|BODY_SINGLE_WORD|FROM_FMBLA_NEWDOM*|autolearn) : ;; SPF_HELO_NEUTRAL) # some of my domains use neutral spf, treat them the same. results[SPF_HELO_PASS]=t @@ -138,6 +138,7 @@ for folder in ${folders[@]}; do now=$(date +%s) limit=$(( now - 60 * min_limit )) + e latest = $(( now - last_sec ))s $latest if (( last_sec <= limit )); then echo $HOSTNAME mailtest $folder $from $(date -d @$last_sec +'%a %m-%d %H:%M') diff --git a/primary-setup b/primary-setup index 5a780b5..61c4c32 100755 --- a/primary-setup +++ b/primary-setup @@ -11,10 +11,7 @@ e() { printf "$pre %s\n" "$*"; } err() { echo "[$(date +'%Y-%m-%d %H:%M:%S%z')]: $0: $*" >&2; } -if [[ $EUID == 0 && ! $SUDO_USER ]]; then - err "requires running as nonroot or sudo" - exit 1 -fi +[[ $EUID == 0 ]] || exec sudo -E "${BASH_SOURCE[0]}" "$@" if [[ $1 ]]; then new_host=$1 @@ -30,16 +27,16 @@ if dpkg -s rss2email &>/dev/null; then if [[ $HOSTNAME == "$MAIL_HOST" ]]; then # arbtt disabled for now #DISPLAY=:0 arbtt-capture --sample-rate=10 & - m sudo systemctl start rss2email.timer - m sudo systemctl enable rss2email.timer + m systemctl start rss2email.timer + m systemctl enable rss2email.timer # off is in mail-setup. no reason for this to be in the rss2email block. - m sudo systemctl start btrbk.timer - m sudo systemctl enable btrbk.timer + m systemctl start btrbk.timer + m systemctl enable btrbk.timer else - m sudo systemctl stop rss2email.timer - m sudo systemctl stop rss2email.service - m sudo systemctl disable rss2email.timer + m systemctl stop rss2email.timer + m systemctl stop rss2email.service + m systemctl disable rss2email.timer # arbtt disabled for now # for ((i=0; i<10; i++)); do # killall arbtt-capture || break @@ -55,17 +52,17 @@ fi # if dpkg -s radicale &>/dev/null; then # if [[ $HOSTNAME == "$MAIL_HOST" ]]; then -# m sudo systemctl restart radicale -# m sudo systemctl enable radicale +# m systemctl restart radicale +# m systemctl enable radicale # if [[ -e /etc/logrotate.d/radicale.disabled ]]; then -# m sudo mv /etc/logrotate.d/radicale{.disabled,} +# m mv /etc/logrotate.d/radicale{.disabled,} # fi # else -# m sudo systemctl stop radicale -# m sudo systemctl disable radicale +# m systemctl stop radicale +# m systemctl disable radicale # # weekly logrotate tries to restart radicale even if it's a disabled service in flidas. # if [[ -e /etc/logrotate.d/radicale ]]; then -# m sudo mv /etc/logrotate.d/radicale{,.disabled} +# m mv /etc/logrotate.d/radicale{,.disabled} # fi # fi # fi diff --git a/rootsshsync b/rootsshsync index 31bc939..3d1ee28 100755 --- a/rootsshsync +++ b/rootsshsync @@ -27,10 +27,7 @@ fi mkdir -p $dest chmod 700 $dest -user=$(id -un) -if [[ $SUDO_USER ]]; then - user=$SUDO_USER -fi +user=$(id -un 1000) user_ssh_dir=$(eval echo ~$user)/.ssh if [[ ! -s $user_ssh_dir/authorized_keys ]]; then diff --git a/subdir_files/sieve/lists.sieve b/subdir_files/sieve/lists.sieve index 1fda5cd..0e44307 100644 --- a/subdir_files/sieve/lists.sieve +++ b/subdir_files/sieve/lists.sieve @@ -32,6 +32,7 @@ if anyof ( if anyof ( header :contains "list-id" "", header :contains "list-id" "", + header :contains "list-id" "", header :contains "list-id" "", header :contains "list-id" "", header :contains "list-id" "", diff --git a/switch-mail-host b/switch-mail-host index 47f5fbe..98a2cdc 100644 --- a/switch-mail-host +++ b/switch-mail-host @@ -33,11 +33,11 @@ restore_old_btrbk=false err-cleanup() { if $restore_new_btrbk; then e WARNING: due to failure, btrbk.timer may need manual restoration: - e $new_shell sudo systemctl start btrbk.timer + e $new_shell systemctl start btrbk.timer fi if $restore_old_btrbk; then e WARNING: due to failure, btrbk.timer may need manual restoration: - e $old_shell sudo systemctl start btrbk.timer + e $old_shell systemctl start btrbk.timer fi } @@ -78,7 +78,7 @@ case $1 in old_hostname=$HOSTNAME new_host=$2 bbk_args="-t $new_host" - new_shell="ssh $new_host" + new_shell="ssh root@$new_host" new_hostname=$($new_shell hostname) ;; pull) @@ -86,7 +86,7 @@ case $1 in new_host=$HOSTNAME new_hostname=$HOSTNAME bbk_args="-s $old_host" - old_shell="ssh $old_host" + old_shell="ssh root@$old_host" # tests ssh connection if ! old_hostname=$($old_shell hostname); then echo "retrying failed $old_shell with -v" @@ -116,11 +116,11 @@ fi ########### end initial processing, begin actually modifying things ########## if $new_shell systemctl is-active btrbk.timer; then - m $new_shell sudo systemctl stop btrbk.timer + m $new_shell systemctl stop btrbk.timer restore_new_btrbk=true fi if $old_shell systemctl is-active btrbk.timer; then - m $old_shell sudo systemctl stop btrbk.timer + m $old_shell systemctl stop btrbk.timer restore_old_btrbk=true fi @@ -150,8 +150,8 @@ done e "umounting /m and /o via $new_shell" $new_shell bash -xs <<'EOF' set -eE -if mountpoint -q /m; then sudo umount /m; fi -if mountpoint -q /o; then sudo umount /o; fi +if mountpoint -q /m; then umount /m; fi +if mountpoint -q /o; then umount /o; fi EOF # previously, I was checking to see if the new mail host diff --git a/system-status b/system-status index a1f7b54..2c88d10 100644 --- a/system-status +++ b/system-status @@ -28,7 +28,10 @@ v() { printf "%s\n" "$*" fi } -lo() { /usr/local/bin/log-once "$@"; } +# log-once COUNT NAME [MESSAGE] +lo() { + /usr/local/bin/log-once "$@" | ifne mail -s "$HOSTNAME: system-status $2" root@localhost +} write-status() { chars=("${first_chars[@]}") @@ -79,10 +82,8 @@ write-status() { chars+=("SPAMD") fi - if ! qlen=$(/usr/sbin/exiqgrep -o 60 -c -b | awk '{print $1}'); then - # early in install process, we dont have permission yet for exiqgrep - qlen=$(sudo /usr/sbin/exiqgrep -o 60 -c -b | awk '{print $1}') - fi + # early in install process, we dont have permission yet for exiqgrep + qlen=$(/usr/sbin/exiqgrep -o 60 -c -b | awk '{print $1}') ||: if ((qlen)); then qmsg="queue length $qlen" chars+=("q $qlen") diff --git a/untrusted-network b/untrusted-network index 677aeb6..42af772 100755 --- a/untrusted-network +++ b/untrusted-network @@ -8,12 +8,30 @@ trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR # Usage: use when switching from a trusted network to an untrusted one, # like public wifi. + + +i() { # install file + local tmp tmpdir dest="$1" + local base="${dest##*/}" + mkdir -p ${dest%/*} + ir=false # i result + tmpdir=$(mktemp -d) + cat >$tmpdir/"$base" + tmp=$(rsync -ic $tmpdir/"$base" "$dest") + if [[ $tmp ]]; then + printf "%s\n" "$tmp" + ir=true + fi + rm -rf $tmpdir +} + + read -r _ ver _ < <(systemd-resolve --version) # removes malware and adult content servers=(1.1.1.3 1.0.0.3 2606:4700:4700::1113 2606:4700:4700::1003) -servers=(1.1.1.1 1.0.0.1 2606:4700:4700::1111 2606:4700:4700::1001s) +servers=(1.1.1.1 1.0.0.1 2606:4700:4700::1111 2606:4700:4700::1001) ## trying out google #servers=(8.8.8.8 8.8.4.4 2001:4860:4860::8888 2001:4860:4860::8844) @@ -27,15 +45,15 @@ Domains=b8.nz DNSOverTLS=yes EOF -mkdir -p /etc/NetworkManager/conf.d -cat >/etc/NetworkManager/conf.d/dns.conf <<'EOF' +i /etc/NetworkManager/conf.d/dns.conf <<'EOF' [main] dns=none systemd-resolved=false EOF -if [[ $(systemctl is-active NetworkManager) == active ]]; then - systemctl restart NetworkManager +if $ir && [[ $(systemctl is-active NetworkManager) == active ]]; then + m systemctl restart NetworkManager + m sleep 2 fi dhclient_restart=false -- 2.30.2