alerting fixes
[distro-setup] / system-status
1 #!/bin/bash
2 # Copyright (C) 2019 Ian Kelling
3 # SPDX-License-Identifier: AGPL-3.0-or-later
4
5 # usage: runs 4 times every 15 seconds unless any args are passed, then just runs once
6
7 if [ -z "$BASH_VERSION" ]; then echo "error: shell is not bash" >&2; exit 1; fi
8
9 set -eE -o pipefail
10 trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR
11
12 status_file=/dev/shm/iank-status
13
14 shopt -s nullglob
15 shopt -s dotglob
16
17 write-status() {
18 chars=()
19
20 glob=(/nocow/btrfs-stale/*)
21 if [[ -e $glob ]] ; then
22 chars+=("STALE!")
23 fi
24 glob=(/m/md/bounces/new/*)
25 if [[ -e $glob ]]; then
26 chars+=("BOUNCE!")
27 fi
28 glob=(/m/md/alerts/new/* /m/md/alerts/cur/*)
29 if [[ -e $glob ]]; then
30 chars+=("ALERT!")
31 fi
32 if [[ -e /nocow/user/mailtest-failure ]]; then
33 chars+=("MAILPING!")
34 fi
35 if [[ -s /var/log/exim4/paniclog ]]; then
36 chars+=("PANIC!")
37 fi
38
39 source /a/bin/bash_unpublished/source-state
40 if [[ $MAIL_HOST == $HOSTNAME ]]; then
41 if [[ $(systemctl is-active btrbk.timer) != active ]]; then
42 chars+=("BTRBK-TIMER!")
43 fi
44
45 ## check if last snapshot was within an hour
46 vol=o
47 snaps=(/mnt/root/btrbk/$vol.20*)
48 now=$(date +%s)
49 maxtime=0
50 for s in ${snaps[@]}; do
51 file=${s##*/}
52 t=$(date -d $(sed -r 's/(.{4})(..)(.{5})(..)(.*)/\1-\2-\3:\4:\5/' <<<${file#$vol.}) +%s)
53 if (( t > maxtime )); then
54 maxtime=$t
55 fi
56 done
57 if (( maxtime < now - 60*60 )); then
58 chars+=("OLD-SNAPSHOT!")
59 fi
60 fi
61
62 cat /a/bin/bash_unpublished/source-state >$status_file
63
64 if [[ $chars ]]; then
65 echo "ps_char=\"${chars[*]} \$ps_char\"" >>$status_file
66 fi
67
68 }
69 write-status
70 if [[ $@ ]]; then
71 cat $status_file
72 exit 0
73 fi
74 for ((i=1; i<=3; i++)); do
75 sleep 15
76 write-status
77 done