From: Ian Kelling Date: Wed, 27 Apr 2022 21:34:45 +0000 (-0400) Subject: mainly fixes X-Git-Url: https://iankelling.org/git/?a=commitdiff_plain;h=ae10fa08bb841b99b0df8e827735bef08c05f3ca;p=distro-setup mainly fixes --- diff --git a/brc b/brc index bd76bb4..ffb7964 100644 --- a/brc +++ b/brc @@ -1378,6 +1378,10 @@ rmstrips() { ssh fencepost head -n 300 /gd/gnuorg/EventAndTravelInfo/rms-current-trips.txt | less } +sudo () { + command sudo "$@" || return $? + DID_SUDO=true +} s() { # background # I use a function because otherwise we cant use in a script, diff --git a/brc2 b/brc2 index f18bfdb..d678668 100644 --- a/brc2 +++ b/brc2 @@ -1757,9 +1757,11 @@ vpncmd() { m sudo -E env "PATH=$PATH" nsenter -t $(pgrep -f "/usr/sbin/openvpn .* --config /etc/openvpn/.*client.conf") -n -m "$@" } vpnf() { + sudo -v vpncmd sudo -E -u iank env "PATH=$PATH" abrowser -no-remote -P vpn & r } vpn2f() { + sudo -v vpncmd sudo -u iank env "PATH=$PATH" abrowser -no-remote -P vpn2 & r } diff --git a/distro-begin b/distro-begin index f022ec3..ef18bbd 100755 --- a/distro-begin +++ b/distro-begin @@ -213,7 +213,7 @@ fi distro=$(distro-name) case $distro in ubuntu|debian|trisquel) - sudo bash -c ". /a/bin/fai/fai-wrapper && /a/bin/fai/fai/config/scripts/GRUB_PC/11-iank" + sudo bash -c ". /a/bin/fai/fai-wrapper && /a/bin/fai/fai/config/scripts/IANK/11-iank" ;; *) sudo bash -c ". /a/bin/fai/fai-wrapper && diff --git a/epanic-clean b/epanic-clean index bb9115e..f69182b 100755 --- a/epanic-clean +++ b/epanic-clean @@ -18,12 +18,18 @@ if [[ $1 ]]; then debug=true fi +verbose=true d() { if $debug; then printf "%s\n" "$*" fi } +v() { + if $verbose; then + printf "%s\n" "$*" + fi +} pl=/var/log/exim4/paniclog @@ -38,8 +44,10 @@ main() { # 2022-02-09 22:08:14.683 [59759] socket bind() to port 25 for address 10.8.0.28 failed: Cannot assign requested address: daemon abandoned if [[ -e /etc/systemd/system/exim4.service.d/backup.conf ]]; then regex="socket bind() to port 25 for address" - grep "$regex" $pl >> $pl-archive ||: - sed -i "/$regex/d" $pl + if grep "$regex" $pl |& tee -a $pl-archive; then + v "above is from grep $regex" + sed -i "/$regex/d" $pl + fi fi # this is a strange message due to running as nonroot @@ -49,27 +57,32 @@ main() { # seems to randomly be caused by # Starting exim4-base housekeeping, exim4-base.service regex="^[^ ]* 00:00:0.* Failed writing transport results to pipe: Broken pipe$" - grep "$regex" $pl >> $pl-archive ||: - sed -i "/$regex/d" $pl + if grep "$regex" $pl |& tee -a $pl-archive; then + v "above is from grep $regex" + sed -i "/$regex/d" $pl + fi + ### begin removing panic lines due to service restarts ### while read -r service regex; do found=false wipe=true d "$service $regex" while read -r d1 d2; do d "$d1 $d2" - found=true tmptime=$(date -d "$d1 $d2" +%s) # Checking the journal takes a second or two, so - # dont consider every matching line, just those > 60 seconds apart. We are + # dont consider every matching line, just those > 20 seconds apart. We are # testing the journal for 60 seconds after the message, so should be ok. + # It probably makes sense to even check for >59 seconds apart, using 20 + # seconds to be conservative. if [[ ! $logtime ]]; then logtime=$tmptime - elif (( tmptime > logtime + 60 )); then + elif (( tmptime > logtime + 20 )); then logtime=$tmptime else continue fi + found=true sec_min=$((logtime - 60)) sec_max=$((logtime + 60)) jmin="$(date -d @$sec_min "+%F %H:%M:%S")" @@ -81,26 +94,28 @@ main() { fi d "jrregex=$jrregex jmin=$jmin jmax=$jmax" # the sed clears out the initial time and process+pid - if ! journalctl -u $service -S "$jmin" -U "$jmax" \ + if journalctl -u $service -S "$jmin" -U "$jmax" \ | sed -r 's/^([^[:space:]]*[[:space:]]+){5}//' | grep "$jrregex" &>/dev/null; then + v "messages worth wiping in: journalctl -u $service -S '$jmin' -U '$jmax' | sed -r 's/^([^[:space:]]*[[:space:]]+){5}//' | grep '$jrregex':" + else + v "PANIC: message not found via: journalctl -u $service -S '$jmin' -U '$jmax' | sed -r 's/^([^[:space:]]*[[:space:]]+){5}//' | grep '$jrregex'" wipe=false break fi done < <(awk "/$regex/ "'{print $1,$2}' $pl) if $found && $wipe; then d "wiping $regex" - if [[ ! -w $pl-archive ]]; then - touch $pl-archive - chgrp adm $pl-archive - chmod 664 $pl-archive + if grep -E "$regex" $pl |& tee -a $pl-archive; then + v "above is from grep -E $regex" + sed -ri "/$regex/d" $pl fi - grep -E "$regex" $pl >> $pl-archive ||: - sed -ri "/$regex/d" $pl fi done <<'EOF' clamav-daemon malware acl condition spamassassin spam acl condition EOF + ### end removing panic lines due to service restarts ### + ## begin broken pipe & write lock & general alert ## regex="Failed to get write lock\|Failed writing transport results to pipe: Broken pipe$" @@ -122,18 +137,20 @@ EOF if (( count > 20 )); then pr_metric=1 elif ! $newlines; then - grep "$regex" $pl >>$pl-archive + grep "$regex" $pl |& tee -a $pl-archive + v "above is from grep $regex" sed -i "/$regex/d" $pl fi fi # I think we could alert on anything else older than 61 seconds, - # but lets just add some slack, make it 5 minutes. + # but lets just add some slack, make it 2 minutes. while read -r day time _; do log_s=$(date -d "$day $time" +%s) - if (( log_s < EPOCHSECONDS - 300 )); then + if (( EPOCHSECONDS - 120 > log_s )); then pr_metric=1 fi + # pr_metrix for $regex is handled above done < <(grep -v "$regex" $pl ||:) ## end broken pipe ## @@ -148,6 +165,13 @@ loop-main() { done } + +if [[ ! -w $pl-archive ]]; then + touch $pl-archive + chgrp adm $pl-archive + chmod 664 $pl-archive +fi + if [[ $INVOCATION_ID ]]; then loop-main else diff --git a/filesystem/etc/prometheus/rules/iank.yml b/filesystem/etc/prometheus/rules/iank.yml index 9838cac..bae264e 100644 --- a/filesystem/etc/prometheus/rules/iank.yml +++ b/filesystem/etc/prometheus/rules/iank.yml @@ -98,7 +98,7 @@ groups: expr: |- exim_paniclog > 0 labels: - severity: warn + severity: day - alert: check_crypttab expr: |- diff --git a/mail-setup b/mail-setup index 6a291c4..d3d1056 100755 --- a/mail-setup +++ b/mail-setup @@ -1225,8 +1225,7 @@ warn warn !hosts = +iank_trusted - # They dont send spam, but needed this because - # smarthosts connect with residential ips and thus get flagged as spam. + # Smarthosts connect with residential ips and thus get flagged as spam if we do a spam check. !authenticated = plain_server:login_server condition = ${if < {$message_size}{5000K}} spam = Debian-exim:true @@ -1237,6 +1236,7 @@ warn add_header = X-Spam_action: $spam_action warn + !hosts = +iank_trusted !authenticated = plain_server:login_server condition = ${if def:malware_name} remove_header = Subject: diff --git a/mailtest-check b/mailtest-check index 91e577c..02cdfcb 100755 --- a/mailtest-check +++ b/mailtest-check @@ -154,7 +154,10 @@ EOF # servers. # example line that sed is parsing: # (-0.1 / 5.0 requ) DKIM_SIGNED=0.1,DKIM_VALID=-0.1,DKIM_VALID_AU=-0.1,SPF_HELO_PASS=-0.001,SPF_PASS=-0.001,TVD_SPACE_RATIO=0.001 autolearn=_AUTOLEARN - raw_results="$($spamcpre sudo -u Debian-exim spamassassin -t --cf='score PYZOR_CHECK 0' <"$latest" | tail -n2 | head -n1 | sed -r 's/^\([^)]*\) *//;s/=[^, ]*([, ]|$)/ /g')" + resultfile=$(mktemp) + $spamcpre sudo -u Debian-exim spamassassin -t --cf='score PYZOR_CHECK 0' <"$latest" &>$resultfile + + raw_results="$(tail $resultfile | grep -A1 -Fx /usr/local/bin/send-test-forward | tail -n1 | sed -r 's/^\([^)]*\) *//;s/=[^, ]*([, ]|$)/ /g')" for r in $raw_results; do case $r in # got this in an update 2022-01. dun care @@ -205,7 +208,7 @@ EOF fi done if (( ${#results[@]} || ${#missing[@]} )); then - printf "$HOSTNAME spamtest %s/%s\n" "$latest" + printf "$HOSTNAME spamtest %s\n" "$latest" if (( ${#results[@]} )); then printf "unexpected %s" "${!results[*]} " fi @@ -213,14 +216,18 @@ EOF printf "missing %s" "${missing[*]}" fi echo # ends our printf string buildup - - if $int; then - echo mailtest-check: cat $latest: - cat $latest - echo mailtest-check: end of cat - echo "$(tput setaf 5 2>/dev/null ||:)█$(tput sgr0 2>/dev/null||:)%.0s" $(eval echo "{1..${COLUMNS:-60}}") - fi + cat $resultfile + echo mailtest-check: end of spam debug results + + # less verbose debug output, commented since I might want it another time. + # if $int; then + # echo mailtest-check: cat $latest: + # cat $latest + # echo mailtest-check: end of cat + # echo "$(tput setaf 5 2>/dev/null ||:)█$(tput sgr0 2>/dev/null||:)%.0s" $(eval echo "{1..${COLUMNS:-60}}") + #fi fi + rm -f $resultfile fi # if spamdpid fi # if $slow