#!/bin/bash
-# Copyright (C) 2019 Ian Kelling
-# SPDX-License-Identifier: AGPL-3.0-or-later
+# I, Ian Kelling, follow the GNU license recommendations at
+# https://www.gnu.org/licenses/license-recommendations.en.html. They
+# recommend that small programs, < 300 lines, be licensed under the
+# Apache License 2.0. This file contains or is part of one or more small
+# programs. If a small program grows beyond 300 lines, I plan to switch
+# its license to GPL.
+
+# Copyright 2024 Ian Kelling
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
# The panic log regularly gets some stuff in it we dont want to fix.
# Detect it and wipe it out.
debug=true
fi
+verbose=true
d() {
if $debug; then
printf "%s\n" "$*"
fi
}
+v() {
+ if $verbose; then
+ printf "%s\n" "$*"
+ fi
+}
pl=/var/log/exim4/paniclog
# 2022-02-09 22:08:14.683 [59759] socket bind() to port 25 for address 10.8.0.28 failed: Cannot assign requested address: daemon abandoned
if [[ -e /etc/systemd/system/exim4.service.d/backup.conf ]]; then
regex="socket bind() to port 25 for address"
- grep "$regex" $pl >> $pl-archive ||:
- sed -i "/$regex/d" $pl
+ if grep "$regex" $pl |& tee -a $pl-archive; then
+ v "above is from grep $regex"
+ sed -i "/$regex/d" $pl
+ fi
fi
# this is a strange message due to running as nonroot
# seems to randomly be caused by
# Starting exim4-base housekeeping, exim4-base.service
regex="^[^ ]* 00:00:0.* Failed writing transport results to pipe: Broken pipe$"
- grep "$regex" $pl >> $pl-archive ||:
- sed -i "/$regex/d" $pl
+ if grep "$regex" $pl |& tee -a $pl-archive; then
+ v "above is from grep $regex"
+ sed -i "/$regex/d" $pl
+ fi
+ ### begin removing panic lines due to service restarts ###
while read -r service regex; do
found=false
wipe=true
d "$service $regex"
while read -r d1 d2; do
d "$d1 $d2"
- found=true
tmptime=$(date -d "$d1 $d2" +%s)
# Checking the journal takes a second or two, so
- # dont consider every matching line, just those > 60 seconds apart. We are
+ # dont consider every matching line, just those > 20 seconds apart. We are
# testing the journal for 60 seconds after the message, so should be ok.
+ # It probably makes sense to even check for >59 seconds apart, using 20
+ # seconds to be conservative.
if [[ ! $logtime ]]; then
logtime=$tmptime
- elif (( tmptime > logtime + 60 )); then
+ elif (( tmptime > logtime + 20 )); then
logtime=$tmptime
else
continue
fi
+ found=true
sec_min=$((logtime - 60))
sec_max=$((logtime + 60))
jmin="$(date -d @$sec_min "+%F %H:%M:%S")"
fi
d "jrregex=$jrregex jmin=$jmin jmax=$jmax"
# the sed clears out the initial time and process+pid
- if ! journalctl -u $service -S "$jmin" -U "$jmax" \
+ if journalctl -u $service -S "$jmin" -U "$jmax" \
| sed -r 's/^([^[:space:]]*[[:space:]]+){5}//' | grep "$jrregex" &>/dev/null; then
+ v "messages worth wiping in: journalctl -u $service -S '$jmin' -U '$jmax' | sed -r 's/^([^[:space:]]*[[:space:]]+){5}//' | grep '$jrregex':"
+ else
+ v "PANIC: message not found via: journalctl -u $service -S '$jmin' -U '$jmax' | sed -r 's/^([^[:space:]]*[[:space:]]+){5}//' | grep '$jrregex'"
wipe=false
break
fi
done < <(awk "/$regex/ "'{print $1,$2}' $pl)
if $found && $wipe; then
d "wiping $regex"
- if [[ ! -w $pl-archive ]]; then
- touch $pl-archive
- chgrp adm $pl-archive
- chmod 664 $pl-archive
+ if grep -E "$regex" $pl |& tee -a $pl-archive; then
+ v "above is from grep -E $regex"
+ sed -ri "/$regex/d" $pl
fi
- grep -E "$regex" $pl >> $pl-archive ||:
- sed -ri "/$regex/d" $pl
fi
done <<'EOF'
clamav-daemon malware acl condition
spamassassin spam acl condition
EOF
+ ### end removing panic lines due to service restarts ###
+
## begin broken pipe & write lock & general alert ##
regex="Failed to get write lock\|Failed writing transport results to pipe: Broken pipe$"
if (( count > 20 )); then
pr_metric=1
elif ! $newlines; then
- grep "$regex" $pl >>$pl-archive
+ grep "$regex" $pl |& tee -a $pl-archive
+ v "above is from grep $regex"
sed -i "/$regex/d" $pl
fi
fi
# I think we could alert on anything else older than 61 seconds,
- # but lets just add some slack, make it 5 minutes.
+ # but lets just add some slack, make it 2 minutes.
while read -r day time _; do
+ # some lines dont have dates, just skip them
+ # 2022-09-16 15:21:06.250 [438097] Exim configuration error:
+ # can't redefine an undefined macro "REMOTE_SMTP_SMARTHOST_TLS_VERIFY_HOSTS"
+ if [[ $day != [2-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] ]]; then
+ continue
+ fi
log_s=$(date -d "$day $time" +%s)
- if (( log_s < EPOCHSECONDS - 300 )); then
+ if (( EPOCHSECONDS - 120 > log_s )); then
pr_metric=1
fi
+ # pr_metrix for $regex is handled above
done < <(grep -v "$regex" $pl ||:)
## end broken pipe ##
done
}
+
+if [[ ! -w $pl-archive ]]; then
+ touch $pl-archive
+ chgrp adm $pl-archive
+ chmod 664 $pl-archive
+fi
+
if [[ $INVOCATION_ID ]]; then
loop-main
else
iankelling.org / git / distro-setup / blobdiff