add script for prometheus logging of systemd service failures

[log-quiet] / sysd-prom-fail-install

diff --git a/sysd-prom-fail-install b/sysd-prom-fail-install
new file mode 100755 (executable)
index 0000000..244e0a5
--- /dev/null
+++ b/sysd-prom-fail-install
@@ -0,0 +1,47 @@
+#!/bin/bash
+# Copyright (C) 2022 Ian Kelling
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+source /a/bin/errhandle/err
+[[ $EUID == 0 ]] || exec sudo -E "${BASH_SOURCE[0]}" "$@"
+
+ser_name=$1
+
+mkdir -p /var/lib/prometheus/node-exporter
+
+f=/var/local/sysd-prom-fail/${ser_name}-result-fail.prom
+
+
+if [[ ! -s /var/lib/prometheus/node-exporter/${ser_name}-result-fail.premerge ]]; then
+  SERVICE_RESULT=success /usr/local/bin/sysd-prom-fail $ser_name
+fi
+
+tmp=$(mktemp)
+
+cat >>$tmp <<EOF
+[Service]
+ExecStopPost=+/usr/local/bin/sysd-prom-fail $ser_name
+EOF
+
+dir=/etc/systemd/system/$ser_name.service.d
+mkdir -p $dir
+out=$(rsync -cipgo --chmod=644 --chown=root:root $tmp $dir/sysd-prom-fail.conf)
+if [[ $out ]]; then
+  echo $0: systemctl daemon-reload
+  systemctl daemon-reload
+  if systemctl is-active $ser_name &>/dev/null; then
+    echo $0: systemctl restart $ser_name
+    systemctl restart $ser_name ||:
+  fi
+fi