fix alerts for down hosts
[distro-setup] / check-remote-mailqs
1 #!/bin/bash
2 # Copyright (C) 2019 Ian Kelling
3 # SPDX-License-Identifier: AGPL-3.0-or-later
4
5 if [ -z "$BASH_VERSION" ]; then echo "error: shell is not bash" >&2; exit 1; fi
6
7 source /a/bin/errhandle/err
8
9 shopt -s nullglob
10 shopt -s dotglob
11
12 # temp disabled: x2wg kwwg
13 # this list duplicated in brc2 ralerts
14 for h in bk je li frodo x3wg kdwg sywg; do
15 statedir=/b/bash_unpublished/mailq-state
16 statefile=$statedir/$h
17 [[ -d $statedir ]] || continue
18 if [[ $HOSTNAME == "${h%wg}" ]]; then
19 rm -f $statefile
20 continue
21 fi
22 if c=$(timeout -v 10 ssh root@$h.b8.nz /usr/local/bin/check-mailq 2>&1 ); then
23 rm -f $statefile
24 if [[ $c ]]; then
25 echo q:$h=$c
26 fi
27 else
28 if [[ -s $statefile ]]; then
29 logsec=$(date +%s -d "$(head -n1 $statefile | awk '{print $1,$2}')")
30 case h in
31 frodo)
32 hours=200
33 ;;
34 *)
35 hours=20
36 ;;
37 esac
38 if (( logsec < EPOCHSECONDS - 60*60*hours )); then
39 echo $0: host $h ssh /usr/local/bin/check-mailq fail for over $hours hours
40 fi
41 fi
42 printf "%s\n" "$c" | ts "%F %T" >> $statefile
43 fi
44 done