iankelling.org Git - log-quiet/blob - sysd-mail-once

   1 #!/bin/bash
   2 # Copyright (C) 2016 Ian Kelling
   3
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16
  17 set -eE -o pipefail
  18 trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR
  19
  20 errors=3
  21 cbase=$HOME/sysd-mail-once-state
  22 to=root
  23 dryrun=false
  24 while [[ $1 == -* ]]; do
  25   case "$1" in
  26     -h|--help)
  27       cat <<EOF
  28 Usage: sysd-mail-once [-t TO_ADDRESS] [-ERRORS] SERVICE COMMAND [COMMAND_ARGS...]
  29
  30 For use with systemd timers, to email (with exim) on repeated failure &
  31 success after failure.
  32
  33 In the service triggered by the timer, prepend this script to the ExecStart.
  34 The email will contain the service's logs for the last ERRORS runs.
  35
  36
  37 Stores error counts in $cbase
  38
  39 -ERRORS         ERRORS is the number of failurs to accumulate before mailing the error.
  40                 Default is 3.
  41
  42 -t TO_ADDRESS   Address to email about errors
  43 -n              Dry run. Execute command but only print out what we would do in response.
  44 EOF
  45       exit 0
  46       ;;
  47     -[0-9]*)
  48       errors=${1#-}
  49       ;;
  50     -t)
  51       to="$2"
  52       shift
  53       ;;
  54     -n)
  55       dryrun=true
  56       ;;
  57     *)
  58       echo "error: unexpected arg: $1"
  59       exit 1
  60       ;;
  61   esac
  62   shift
  63 done
  64
  65 if (( $# < 2 )); then
  66   echo "error: expected at least 2 args after options. args:"
  67   exit 1
  68 fi
  69
  70 service="$1"
  71 shift
  72
  73
  74 # maybe run, depending on $dryrun
  75 m() {
  76   if $dryrun; then
  77     printf "%s\n" "$*"
  78   else
  79     "$@"
  80   fi
  81 }
  82 # maybe run, with stdin
  83 mi() {
  84   if $dryrun; then
  85     printf "%s <<'EOF'\n" "$*"
  86     cat
  87     echo EOF
  88   else
  89     "$@"
  90   fi
  91 }
  92 e() {
  93   printf "dryrun: %s\n" "$*"
  94 }
  95
  96 c=$cbase/$service # c for command file path base
  97
  98 if $dryrun; then
  99   e "c=$c"
 100 fi
 101
 102 glob="${c}[0-9]*"
 103 arr=($glob); file="${arr[0]}"; [[ $glob != "$file" ]] || file=
 104 if [[ $dryrun && $file ]]; then
 105   e "file=$file"
 106 fi
 107
 108 if ! [[ -d $cbase ]]; then
 109   mkdir -p $cbase
 110 fi
 111
 112 if [[ ! $file ]]; then
 113   cursor=$(journalctl --show-cursor -qn0|sed 's/^\s*--\scursor:\s*//')
 114 fi
 115
 116 code=0
 117 "$@" || code=$?
 118 if (( code )); then
 119   send_mail=false
 120   if [[ $file ]]; then
 121     i=${file#"$c"}
 122     if (( i < errors )); then
 123       new_file=$c$((i+1))
 124       m mv $file $new_file
 125       file=$new_file
 126       if [[ $file == $c$errors ]]; then
 127         send_mail=true
 128       fi
 129     fi
 130   else
 131     file=${c}1
 132     if $dryrun; then
 133       printf "dryrun: creating $file, contents: %s\n" "$cursor"
 134     else
 135       printf "%s\n" "$cursor" >$file
 136     fi
 137     if (( errors == 1 )); then
 138       send_mail=true
 139     fi
 140   fi
 141   if $send_mail; then
 142     echo "sysd-mail-once: emailing on $errors errors. exit code: $code"
 143     mi exim -odf -t <<EOF
 144 To: $to
 145 From: $USER@$(hostname -f)
 146 Subject: $HOSTNAME: $service exit code: $code
 147
 148 $(journalctl -u $service.service --after-cursor="$(<$file)")
 149 EOF
 150   fi
 151 else
 152   if [[ $file ]]; then
 153     m rm -f $file
 154     if [[ $file == $c$errors ]]; then
 155       echo "sysd-mail-once: emailing success after >= $errors errors."
 156       mi exim -odf -t <<EOF
 157 To: $to
 158 From: $USER@$(hostname -f)
 159 Subject: $HOSTNAME: $service success
 160
 161 EOF
 162     fi
 163   fi
 164 fi