various fixes

[distro-setup] / system-status

diff --git a/system-status b/system-status
old mode 100644 (file)
new mode 100755 (executable)
index 28c0035..07c730d
--- a/system-status
+++ b/system-status
@@ -37,10 +37,48 @@ loday() {
   /usr/local/bin/log-once "$@" | ifne mail -s "$HOSTNAME: system-status $2" daylerts@iankelling.org
 }
 
-
+# todo, consider migrating some of these alerts into prometheus
 write-status() {
   chars=("${first_chars[@]}")
 
+  services=(
+    epanicclean
+    systemstatus
+    btrfsmaintstop
+    dynamicipupdate
+  )
+  bads=()
+  if systemctl show -p SubState --value ${services[@]} | egrep -v '^(running|)$'; then
+    for s in ${services[@]}; do
+      if [[ $(systemctl show -p SubState --value $s) != running ]]; then
+        bads+=($s)
+      fi
+    done
+    chars+=(MYSERS)
+
+  fi
+  lo -240 mysers ${bads[*]}
+
+  services=(
+    prometheus-node-exporter
+    prometheus-alertmanager
+    prometheus
+  )
+  case $HOSTNAME in
+    kd)
+      bads=()
+      if systemctl show -p SubState --value ${services[@]} | egrep -v '^(running|)$'; then
+        for s in ${services[@]}; do
+          if [[ $(systemctl show -p SubState --value $s) != running ]]; then
+            bads+=($s)
+          fi
+        done
+        chars+=(PROM)
+      fi
+      lo -240 prom ${bads[*]}
+      ;;
+  esac
+
   # clock us out in timetrap if are idle too long
   if [[ -e /p/.timetrap.db ]]; then
     export DISPLAY=:0
@@ -83,15 +121,18 @@ write-status() {
 
   glob=(/nocow/btrfs-stale/*)
   if [[ -e ${glob[0]} ]]; then
-    chars+=("STALE")
+    chars+=(STALE)
   fi
+  var_mail_msg=
   if [[ $(find /var/mail -type f \! -empty -print -quit) ]]; then
     var_mail_msg="message in /var/mail"
   fi
   loday -1 var_mail $var_mail_msg
+
+  bouncemsg=
   glob=(/m/md/bounces/new/*)
   if [[ -e ${glob[0]} ]]; then
-    chars+=("BOUNCE")
+    chars+=(BOUNCE)
     bouncemsg="message in /m/md/bounces/new"
   fi
   loday -1 bounce $bouncemsg
@@ -99,27 +140,28 @@ write-status() {
   # but its good enough for me.
   glob=(/m/md/alerts/{new,cur}/!(*,S))
   if [[ -e ${glob[0]} ]]; then
-    chars+=("A")
+    chars+=(A)
   fi
 
   glob=(/m/md/daylerts/{new,cur}/!(*,S))
   if [[ -e ${glob[0]} ]]; then
-    chars+=("L")
+    chars+=(DAY)
   fi
 
 
   tmp=(/var/local/cron-errors/mailtest-check*)
   if (( ${#tmp[@]} )); then
-    chars+=("MAILPING")
+    chars+=(MAILPING)
   fi
   tmp=(/var/local/cron-errors/mailtest-slow*)
   if (( ${#tmp[@]} )); then
-    chars+=("SPAMD")
+    chars+=(SPAMD)
   fi
 
   # early in install process, we dont have permission yet for exiqgrep.
   # 1100 helps allow for system restarts
   qlen=$(/usr/sbin/exiqgrep -o 1100 -c -b | awk '{print $1}') ||:
+  qmsg=
   if ((qlen)); then
     qmsg="queue length $qlen"
     chars+=("q $qlen")
@@ -144,11 +186,11 @@ write-status() {
 
   # these conditions are so we dont have an overly verbose prompt
   if $begin && $end; then
-    chars+=("D")
+    chars+=(D)
   elif $begin; then
-    chars+=("DB")
+    chars+=(DB)
   elif $end; then
-    chars+=("DE")
+    chars+=(DE)
   else
     f=~/.local/conflink
     # shellcheck disable=SC2043
@@ -175,7 +217,7 @@ write-status() {
         # Just because i forget a lot, -mmin -NUM means files modified <= NUM minutes ago
         if (( fmin < 0 )) && [[ $(find ${all_dirs[@]} -mmin $fmin -type f -print -quit 2>/dev/null) ]]; then
           v conflink newer filesystem files
-          chars+=("CONFLINK")
+          chars+=(CONFLINK)
           break
         fi
 
@@ -188,7 +230,7 @@ write-status() {
           fi
           if (( $(date -d "$(git log --diff-filter=ACR --format=%aD -1)" +%s) > fsec )); then
             v conflink: newer files checked in to git
-            chars+=("CONFLINK")
+            chars+=(CONFLINK)
             break
           fi
 
@@ -198,7 +240,7 @@ write-status() {
           done < <(git ls-files -o --exclude-standard)
           if [[ ${untracked[0]} && $(find "${untracked[@]}" -mmin $fminplus -type f -print -quit) ]]; then
             v conflink: untracked in $d
-            chars+=("CONFLINK")
+            chars+=(CONFLINK)
             break
           fi
         done
@@ -207,13 +249,13 @@ write-status() {
       fi
       if [[ ! -e $f || $(<$f) != 0 ]]; then
         v conflink: last run not found or failed
-        chars+=("CONFLINK")
+        chars+=(CONFLINK)
         break
       fi
     done
   fi
 
-#  if [[ $(grep -v "exim user lost privilege for using -C option" /var/log/exim4/paniclog 2>/dev/null ||:) ]]; then
+  #  if [[ $(grep -v "exim user lost privilege for using -C option" /var/log/exim4/paniclog 2>/dev/null ||:) ]]; then
   if [[ -s /var/log/exim4/paniclog ]]; then
     chars+=("PANIC!")
     # leave it up to epanic-clean to send email notification
@@ -223,10 +265,10 @@ write-status() {
   if [[ $MAIL_HOST == "$HOSTNAME" ]]; then
     bbkmsg=
     if [[ $(systemctl is-active btrbk.timer) != active ]]; then
-      chars+=("BTRBK.TIMER")
-      bbkmsg="btrbk.timer not enabled"
+      chars+=(BTRBK.TIMER)
+      bbkmsg="not enabled"
     fi
-    lo -48 btrbk.timer $bbkmsg
+    lo -480 btrbk.timer $bbkmsg
 
     ## check if last snapshot was within an hour
     vol=o
@@ -254,8 +296,9 @@ write-status() {
         maxtime=$t
       fi
     done
+    snapshotmsg=
     if (( maxtime < now - 4*60*60 )); then
-      chars+=("OLD-SNAP")
+      chars+=(OLD-SNAP)
       snapshotmsg="/o snapshot older than 4 hours"
     fi
     lo -1 old-snapshot $snapshotmsg
@@ -279,19 +322,19 @@ if [[ $1 ]]; then
 fi
 
 main-loop() {
-while true; do
-  power=true
-  if [[ -e /sys/class/power_supply/AC/online && $(</sys/class/power_supply/AC/online) == 0 ]]; then
-    power=false
-  fi
-  wait=15
-  if ! $power; then
-    wait=60
-  fi
+  while true; do
+    power=true
+    if [[ -e /sys/class/power_supply/AC/online && $(</sys/class/power_supply/AC/online) == 0 ]]; then
+      power=false
+    fi
+    wait=15
+    if ! $power; then
+      wait=60
+    fi
 
-  sleep $wait
-  write-status
-done
+    sleep $wait
+    write-status
+  done
 }
 
 # ensure our long operations are one line so we are not prone errors