fixes, prometheus, lots of stuff

[distro-setup] / system-status

diff --git a/system-status b/system-status
index 07c730d71a7338db5a9343d359a857829a481cad..f50d238b7d6523acc14c65765503894a52811916 100755 (executable)
--- a/system-status
+++ b/system-status
@@ -30,27 +30,38 @@ v() {
 }
 # log-once COUNT NAME [MESSAGE]
 lo() {
-  /usr/local/bin/log-once "$@" | ifne mail -s "$HOSTNAME: system-status $2" root@localhost
+  if type -p ifne &>/dev/null; then
+    /usr/local/bin/log-once "$@" | ifne mail -s "$HOSTNAME: system-status $2" root@localhost
+  fi
 }
 
 loday() {
-  /usr/local/bin/log-once "$@" | ifne mail -s "$HOSTNAME: system-status $2" daylerts@iankelling.org
+  if type -p ifne &>/dev/null; then
+    /usr/local/bin/log-once "$@" | ifne mail -s "$HOSTNAME: system-status $2" daylert@iankelling.org
+  fi
 }
 
 # todo, consider migrating some of these alerts into prometheus
 write-status() {
   chars=("${first_chars[@]}")
 
-  services=(
-    epanicclean
-    systemstatus
-    btrfsmaintstop
-    dynamicipupdate
-  )
+
+  services=( epanicclean )
+  case $HOSTNAME in
+    bk|je|li) : ;;
+    *)
+      services+=(
+        systemstatus
+        btrfsmaintstop
+        dynamicipupdate
+      )
+      ;;
+  esac
+
   bads=()
-  if systemctl show -p SubState --value ${services[@]} | egrep -v '^(running|)$'; then
+  if systemctl show -p SubState --value ${services[@]} | egrep -v '^(running|)$' &>/dev/null; then
     for s in ${services[@]}; do
-      if [[ $(systemctl show -p SubState --value $s) != running ]]; then
+      if [[ $(systemctl show -p SubState --value $s 2>&1) != running ]]; then
         bads+=($s)
       fi
     done
@@ -67,9 +78,9 @@ write-status() {
   case $HOSTNAME in
     kd)
       bads=()
-      if systemctl show -p SubState --value ${services[@]} | egrep -v '^(running|)$'; then
+      if systemctl show -p SubState --value ${services[@]} | egrep -v '^(running|)$' &>/dev/null; then
         for s in ${services[@]}; do
-          if [[ $(systemctl show -p SubState --value $s) != running ]]; then
+          if [[ $(systemctl show -p SubState --value $s 2>&1) != running ]]; then
             bads+=($s)
           fi
         done
@@ -143,7 +154,7 @@ write-status() {
     chars+=(A)
   fi
 
-  glob=(/m/md/daylerts/{new,cur}/!(*,S))
+  glob=(/m/md/daylert/{new,cur}/!(*,S))
   if [[ -e ${glob[0]} ]]; then
     chars+=(DAY)
   fi
@@ -175,12 +186,13 @@ write-status() {
   esac
 
   begin=false
-  if ! make -C /b/ds -q ~/.local/distro-begin || [[ $(<~/.local/distro-begin) != 0 ]]; then
+
+  if ! make -C /b/ds -q ~/.local/distro-begin 2>/dev/null || [[ $(<~/.local/distro-begin) != 0 ]]; then
     begin=true
   fi
 
   end=false
-  if ! make -C /b/ds -q ~/.local/distro-end || [[ $(<~/.local/distro-end) != 0 ]]; then
+  if ! make -C /b/ds -q ~/.local/distro-end 2>/dev/null || [[ $(<~/.local/distro-end) != 0 ]]; then
     end=true
   fi
 
@@ -261,7 +273,10 @@ write-status() {
     # leave it up to epanic-clean to send email notification
   fi
 
-  source /a/bin/bash_unpublished/source-state
+  if [[ -e /a/bin/bash_unpublished/source-state ]]; then
+    # /a gets remounted due to btrbk, ignore error code for file doesnt exist
+    source /a/bin/bash_unpublished/source-state || [[ $? == 1 ]]
+  fi
   if [[ $MAIL_HOST == "$HOSTNAME" ]]; then
     bbkmsg=
     if [[ $(systemctl is-active btrbk.timer) != active ]]; then
@@ -304,10 +319,14 @@ write-status() {
     lo -1 old-snapshot $snapshotmsg
   fi
 
-  cat /a/bin/bash_unpublished/source-state >$status_file
+  if [[ ! -e $status_file || -w $status_file ]]; then
+    if [[ -e /a/bin/bash_unpublished/source-state ]]; then
+      cat /a/bin/bash_unpublished/source-state >$status_file
+    fi
 
-  if [[ ${chars[*]} ]]; then
-    echo "ps_char=\"${chars[*]} \$ps_char\"" >>$status_file
+    if [[ ${chars[*]} ]]; then
+      echo "ps_char=\"${chars[*]} \$ps_char\"" >>$status_file
+    fi
   fi
 
 }