iankelling.org Git - distro-setup/blob - system-status

   1 #!/bin/bash
   2 # Copyright (C) 2019 Ian Kelling
   3 # SPDX-License-Identifier: AGPL-3.0-or-later
   4
   5 # usage: runs once every 15 seconds unless any args are passed, or we
   6 # then just runs once. On battery power, run once per minute.
   7
   8 if [ -z "$BASH_VERSION" ]; then echo "error: shell is not bash" >&2; exit 1; fi
   9
  10 if [[ $EUID != 1000 ]]; then
  11   echo "$0: error, expected to be user 1000"
  12   exit 1
  13 fi
  14
  15 source /a/bin/errhandle/err
  16 status_file=/dev/shm/iank-status
  17
  18 shopt -s nullglob
  19 shopt -s dotglob
  20 shopt -s extglob
  21
  22 for p in ~/.gem/ruby/*/bin; do
  23   PATH="$PATH:$p"
  24 done
  25
  26
  27 verbose=false
  28 if [[ $1 ]]; then
  29   verbose=true
  30 fi
  31 v() {
  32   if $verbose; then
  33     printf "%s\n" "$*"
  34   fi
  35 }
  36 p() { printf "%s\n" "$*"; }
  37 # log-once COUNT NAME [MESSAGE]
  38 lo() {
  39   if type -p ifne &>/dev/null; then
  40     /usr/local/bin/log-once "$@" | ifne mail -s "$HOSTNAME: system-status $2" root@localhost
  41   fi
  42 }
  43
  44 loday() {
  45   if type -p ifne &>/dev/null; then
  46     /usr/local/bin/log-once "$@" | ifne mail -s "$HOSTNAME: system-status $2" daylert@iankelling.org
  47   fi
  48 }
  49 # rm glob
  50 rmg() {
  51   if (( $# )); then
  52     rm -f "$@"
  53   fi
  54 }
  55
  56 # todo, consider migrating some of these alerts into prometheus
  57 write-status() {
  58   chars=("${first_chars[@]}")
  59
  60   services=( epanicclean )
  61   case $HOSTNAME in
  62     bk|je|li) : ;;
  63     *)
  64       services+=(
  65         systemstatus
  66         btrfsmaintstop
  67         dynamicipupdate
  68       )
  69       bads=()
  70       if systemctl show -p SubState --value ${services[@]} | egrep -v '^(running|)$' &>/dev/null; then
  71         for s in ${services[@]}; do
  72           if [[ $(systemctl show -p SubState --value $s 2>&1) != running ]]; then
  73             bads+=($s)
  74           fi
  75         done
  76         chars+=(MYSERS)
  77       fi
  78       p ${bads[*]} | lo -240 mysers
  79       ;;
  80   esac
  81
  82   case $HOSTNAME in
  83     kd)
  84       services=(
  85         prometheus-node-exporter
  86         prometheus-alertmanager
  87         prometheus
  88       )
  89       bads=()
  90       if systemctl show -p SubState --value ${services[@]} | egrep -v '^(running|)$' &>/dev/null; then
  91         for s in ${services[@]}; do
  92           if [[ $(systemctl show -p SubState --value $s 2>&1) != running ]]; then
  93             bads+=($s)
  94           fi
  95         done
  96         chars+=(PROM)
  97       fi
  98       p ${bads[*]} | lo -240 prom
  99       ;;
 100   esac
 101
 102
 103   if [[ -e /a/bin/bash_unpublished/source-state ]]; then
 104     # /a gets remounted due to btrbk, ignore error code for file doesnt exist
 105     source /a/bin/bash_unpublished/source-state || [[ $? == 1 ]]
 106   fi
 107   if [[ $MAIL_HOST == "$HOSTNAME" ]]; then
 108
 109     bouncemsg=
 110     glob=(/m/md/bounces/new/*)
 111     if [[ -e ${glob[0]} ]]; then
 112       chars+=(BOUNCE)
 113       bouncemsg="message in /m/md/bounces/new"
 114     fi
 115     p $bouncemsg | loday -1 bounce
 116     # emails without the S (seen) flag. this only checks the last flag,
 117     # but its good enough for me.
 118     glob=(/m/md/alerts/{new,cur}/!(*,S))
 119     if [[ -e ${glob[0]} ]]; then
 120       chars+=(A)
 121     fi
 122
 123     glob=(/m/md/daylert/{new,cur}/!(*,S))
 124     if [[ -e ${glob[0]} ]]; then
 125       chars+=(DAY)
 126     fi
 127
 128     bbkmsg=
 129     if [[ $(systemctl is-active btrbk.timer) != active ]]; then
 130       chars+=(BTRBK.TIMER)
 131       bbkmsg="not enabled"
 132     fi
 133     p "$bbkmsg" | lo -480 btrbk.timer
 134
 135     ## check if last snapshot was within an hour
 136     vol=o
 137     # this section generally copied from btrbk scripts, but
 138     # this part modified to speed things up by about half a second.
 139     # I'm not sure if its quite as reliable, but it looks pretty safe.
 140     # Profiled it using time and also adding to the top of the file:
 141     # set -x
 142     # PS4='+ $(date "+%2N") '
 143     # allow failure in case there are no snapshots yet.
 144     # shellcheck disable=SC2012
 145     shopt -u nullglob
 146     files=(/mnt/root/btrbk/$vol.20*)
 147     shopt -s nullglob
 148     snaps=()
 149     if (( ${#files[@]} )); then
 150       snaps=($(ls -1avdr "${files[@]}" 2>/dev/null |head -n1 || : ))
 151     fi
 152     now=$EPOCHSECONDS
 153     maxtime=0
 154     for s in ${snaps[@]}; do
 155       file=${s##*/}
 156       t=$(date -d $(sed -r  's/(.{4})(..)(.{5})(..)(.*)/\1-\2-\3:\4:\5/' <<<${file#$vol.}) +%s)
 157       if (( t > maxtime )); then
 158         maxtime=$t
 159       fi
 160     done
 161     snapshotmsg=
 162     if (( maxtime < now - 4*60*60 )); then
 163       chars+=(OLD-SNAP)
 164       snapshotmsg="/o snapshot older than 4 hours"
 165     fi
 166     p "$snapshotmsg" | lo -1 old-snapshot
 167
 168
 169     # commented out, only using timetrap retrospectively.
 170     # # clock us out in timetrap if are idle too long
 171     # if [[ -e /p/.timetrap.db ]]; then
 172     #   export DISPLAY=:0
 173     #   if type -p xprintidle &>/dev/null && xidle=$(xprintidle 2>/dev/null); then
 174     #     if [[ $xidle == [0-9]* ]]; then
 175     #       sheet=$(sqlite3 /p/.timetrap.db "select sheet from entries where end is NULL;")
 176     #       idle=300000
 177     #       if [[ $sheet == w ]]; then
 178     #         idle=900000
 179     #       fi
 180     #       if [[ $sheet && $xidle -gt $idle ]]; then
 181     #         timetrap out
 182     #       fi
 183     #     fi
 184     #   fi
 185     # fi
 186   else # end if $MAIL_HOST
 187     rmg /home/iank/cron-errors/bounce* \
 188         /home/iank/cron-errors/btrbk.timer* \
 189         /home/iank/cron-errors/old-snapshot*
 190   fi
 191
 192   if ip l show tunfsf &>/dev/null; then
 193     # this is for tracking dns over tls issue, which
 194     # fixvpndns() in brc2 fixes.
 195     stat=$(resolvectl dnsovertls tunfsf 2>/dev/null ||: )
 196     read _ _ _ istls <<<"$stat"
 197     case $istls in
 198       no) : ;;
 199       *)
 200         printf "%s\n" "$istls" | ts >> /tmp/istls.log
 201         chars+=("T:$istls")
 202         ;;
 203     esac
 204   fi
 205
 206   # We do this once every 5 minutes, since this is not a grave problem.
 207   # For formatted elisp, see /b/ds/unsaved-buffers.el
 208   elisp='(format "%s" (-reduce-from (lambda (acc buf) (let ((bpath (buffer-file-name buf))) (if (and bpath (buffer-modified-p buf)) (cons bpath acc ) acc))) nil (buffer-list)))'
 209   if [[ ! $last_emacs_check || $emacsfiles ]] || (( last_emacs_check < EPOCHSECONDS - 300 )); then
 210     if pgrep -G iank -u iank -f 'emacs --daemon' &>/dev/null; then
 211       # i dun care if this fails
 212       emacsfiles="$(timeout 1 emacsclient -a /usr/bin/true --eval "$elisp" 2>/dev/null | sed '/^"nil"$/d;s/^"(/E: /;s/)"$//' ||:)"
 213       if [[ $emacsfiles ]]; then
 214         chars+=("$emacsfiles")
 215       fi
 216     fi
 217     last_emacs_check=$EPOCHSECONDS
 218   fi
 219
 220
 221   glob=(/nocow/btrfs-stale/*)
 222   if [[ -e ${glob[0]} ]]; then
 223     chars+=(STALE)
 224   fi
 225   var_mail_msg=
 226   if [[ $(find /var/mail -type f \! -empty -print -quit) ]]; then
 227     var_mail_msg="message in /var/mail"
 228   fi
 229   p $var_mail_msg | loday -1 var_mail
 230
 231   # early in install process, we dont have permission yet for exiqgrep.
 232   # 1100 helps allow for system restarts
 233   qlen=$(/usr/sbin/exiqgrep -o 1100 -c -b | awk '{print $1}') ||:
 234   qmsg=
 235   if ((qlen)); then
 236     qmsg="queue length $qlen"
 237     chars+=("q $qlen")
 238   fi
 239   case $HOSTNAME in
 240     # No point in emailing about the mailq on a host where we don't
 241     # check email.
 242     $MAIL_HOST)
 243       p $qmsg | loday -120 qlen
 244       ;;
 245     *)
 246       rmg /home/iank/cron-errors/qlen*
 247       ;;
 248   esac
 249
 250   begin=false
 251
 252   if ! make -C /b/ds -q ~/.local/distro-begin 2>/dev/null || [[ $(<~/.local/distro-begin) != 0 ]]; then
 253     begin=true
 254   fi
 255
 256   end=false
 257   if ! make -C /b/ds -q ~/.local/distro-end 2>/dev/null || [[ $(<~/.local/distro-end) != 0 ]]; then
 258     end=true
 259   fi
 260
 261   # these conditions are so we dont have an overly verbose prompt
 262   if $begin && $end; then
 263     chars+=(D)
 264   elif $begin; then
 265     chars+=(DB)
 266   elif $end; then
 267     chars+=(DE)
 268   else
 269     f=~/.local/conflink
 270     # shellcheck disable=SC2043
 271     for _ in 1; do
 272       if [[ -e $f ]]; then
 273         now=$EPOCHSECONDS
 274         fsec=$(stat -c%Y $f)
 275         # the / 60 makes it 0-59 seconds less strict, +1 to help make sure we
 276         # dont have any false positives.
 277         fmin=$(( (fsec - now + 1 ) / 60 ))
 278         fminplus=$(( fmin + 60*24 ))
 279         # Filesystem files get copied, so find any newer than the last run.
 280         # The rest are hueristics:
 281         # Given the last time we added a file in git, is that newer than the last conflink run.
 282         # Given new files not added to git, were they modified more recently than the last conflink? but,
 283         # push their modification time back by a day so we can develop them before needing to add them to git.
 284
 285         all_dirs=({/a/bin/ds,/p/c}{/filesystem,/machine_specific/$HOSTNAME/filesystem})
 286         # This part is copied from conflink
 287         for x in /p/c/machine_specific/*.hosts /a/bin/ds/machine_specific/*.hosts; do
 288           if grep -qxF $HOSTNAME $x; then all_dirs+=( ${x%.hosts} ); fi
 289         done
 290
 291         # Just because i forget a lot, -mmin -NUM means files modified <= NUM minutes ago
 292         if (( fmin < 0 )) && [[ $(find ${all_dirs[@]} -mmin $fmin -type f -print -quit 2>/dev/null) ]]; then
 293           v conflink newer filesystem files
 294           chars+=(CONFLINK)
 295           break
 296         fi
 297
 298         for d in /a/bin/distro-setup /p/c; do
 299           [[ -d $d ]] || continue
 300           cd $d
 301           if [[ ! -e .git ]]; then
 302             # some hosts i dont push all of /p/c
 303             continue
 304           fi
 305           if (( $(date -d "$(git log --diff-filter=ACR --format=%aD -1)" +%s) > fsec )); then
 306             v conflink: newer files checked in to git
 307             chars+=(CONFLINK)
 308             break
 309           fi
 310
 311           untracked=()
 312           while read -r l; do
 313             untracked+=("$l")
 314           done < <(git ls-files -o --exclude-standard)
 315           if [[ ${untracked[0]} && $(find "${untracked[@]}" -mmin $fminplus -type f -print -quit) ]]; then
 316             v conflink: untracked in $d
 317             chars+=(CONFLINK)
 318             break
 319           fi
 320         done
 321         cd /
 322
 323       fi
 324       if [[ ! -e $f || $(<$f) != 0 ]]; then
 325         v conflink: last run not found or failed
 326         chars+=(CONFLINK)
 327         break
 328       fi
 329     done
 330   fi
 331
 332   #  if [[ $(grep -v "exim user lost privilege for using -C option" /var/log/exim4/paniclog 2>/dev/null ||:) ]]; then
 333   if [[ -s /var/log/exim4/paniclog ]]; then
 334     chars+=("PANIC!")
 335     # leave it up to epanic-clean to send email notification
 336   fi
 337
 338   if [[ ! -e $status_file || -w $status_file ]]; then
 339     if [[ -e /a/bin/bash_unpublished/source-state ]]; then
 340       cat /a/bin/bash_unpublished/source-state >$status_file
 341     fi
 342
 343     if [[ ${chars[*]} ]]; then
 344       echo "ps_char=\"${chars[*]} \$ps_char\"" >>$status_file
 345     fi
 346   fi
 347 }
 348 # use this if we want to do something just once per minute
 349 first_chars=()
 350
 351
 352 write-status
 353 if [[ $1 ]]; then
 354   cat $status_file
 355   exit 0
 356 fi
 357
 358 main-loop() {
 359   while true; do
 360     power=true
 361     if [[ -e /sys/class/power_supply/AC/online && $(</sys/class/power_supply/AC/online) == 0 ]]; then
 362       power=false
 363     fi
 364     wait=15
 365     if ! $power; then
 366       wait=60
 367     fi
 368
 369     sleep $wait
 370     write-status
 371   done
 372 }
 373
 374 # ensure our long operations are one line so we are not prone errors
 375 # from this file being modified.
 376 main-loop