fix alerts for down hosts

diff --git a/brc2 b/brc2
index 1a695625be4d8ae5febc176194ae2c42e33785bb..fa84ca383c4ddbcfe90340caa4f8e10950b0412f 100644 (file)
--- a/brc2
+++ b/brc2
@@ -349,7 +349,7 @@ alerts() {
 ralerts() { # remote alerts
   local ret shell
   # this list is duplicated in check-remote-mailqs
-  for h in bk je li frodo kwwg x3wg x2wg kdwg sywg; do
+  for h in bk je li frodo x3wg kdwg sywg; do
     echo $h:
     shell="ssh $h"
     if [[ $HOSTNAME == "${h%wg}" ]]; then

diff --git a/check-remote-mailqs b/check-remote-mailqs
index df67a041544c7e049642462c4ac9bc39757c7d05..2e98521d01e1359d2d6284953a9e2992029639fd 100755 (executable)
--- a/check-remote-mailqs
+++ b/check-remote-mailqs
@@ -9,8 +9,9 @@ source /a/bin/errhandle/err
 shopt -s nullglob
 shopt -s dotglob
 
+# temp disabled: x2wg kwwg
 # this list duplicated in brc2 ralerts
-for h in bk je li frodo kwwg x3wg x2wg kdwg sywg; do
+for h in bk je li frodo x3wg kdwg sywg; do
   statedir=/b/bash_unpublished/mailq-state
   statefile=$statedir/$h
   [[ -d $statedir ]] || continue

diff --git a/distro-begin b/distro-begin
index 84ba7c4ac3c3f6ee68027842e254dbc2766c5cc0..c5d10cc015727b4e8ad09474864022f42af8bcff 100755 (executable)
--- a/distro-begin
+++ b/distro-begin
@@ -13,10 +13,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# for setting up a new machine
+#### for setting up a new machine
 # usage: $0 [-r] [HOSTNAME]
 # HOSTNAME changes the machine's hostname
 
+# Update target_down alerts in
+# /a/bin/ds/filesystem/etc/prometheus/rules/iank.yml
+#
+# Update hostnames in /b/ds/check-remote-mailqs
+
+### end new machine setup
+
 # tips:
 # run any sudo command first so your pass is cached
 # set the scrollback to unlimited in case something goes wrong

diff --git a/filesystem/etc/prometheus/rules/iank.yml b/filesystem/etc/prometheus/rules/iank.yml
index f64322b2c98ef4f48755daec9fe81f185b8c9488..0049743f24e0a2b349b643ff167c24d0dcf8abeb 100644 (file)
--- a/filesystem/etc/prometheus/rules/iank.yml
+++ b/filesystem/etc/prometheus/rules/iank.yml
@@ -274,7 +274,7 @@ groups:
       description: "A Prometheus job has disappeared\n  VALUE = {{ $value }}"
 
   - alert: lowpri_target_down
-    expr: up{instance!~"kdwg:9101|bkex.b8.nz:9101|liex.b8.nz:9101|10.2.0.1:9100"} == 0
+    expr: up{instance!~"kdwg:9101|bkex.b8.nz:9101|liex.b8.nz:9101|10.2.0.1:9100|kwwg:9101"} == 0
     for: 30m
     labels:
       severity: warn