iankelling.org Git - log-quiet/blob - sysd-mail-once

   1 #!/bin/bash
   2 # Copyright (C) 2016 Ian Kelling
   3
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16
  17 set -eE -o pipefail
  18 trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR
  19
  20 errors=3
  21 tmp=(~)
  22 cbase="${tmp[0]}/sysd-mail-once-state"
  23 to=root
  24 dryrun=false
  25 print_all=false
  26 while [[ $1 == -* ]]; do
  27   case "$1" in
  28     -h|--help)
  29       cat <<EOF
  30 Usage: sysd-mail-once [-t TO_ADDRESS] [-ERRORS] SERVICE COMMAND [COMMAND_ARGS...]
  31
  32 For use with systemd timers, to email (with exim) on repeated failure &
  33 success after failure.
  34
  35 In the service triggered by the timer, prepend this script to the ExecStart.
  36 The email will contain the service's logs for the last ERRORS runs.
  37
  38
  39 Stores error counts in $cbase
  40
  41 -ERRORS         ERRORS is the number of failurs to accumulate before mailing the error.
  42                 Default is 3.
  43
  44 -a              Email the logs of all errors intead of just the last one.
  45 -t TO_ADDRESS   Address to email about errors
  46 -n              Dry run. Execute command but only print out what we would do in response.
  47 EOF
  48       exit 0
  49       ;;
  50     -[0-9]*)
  51       errors=${1#-}
  52       ;;
  53     -a)
  54       print_all=true
  55       ;;
  56     -t)
  57       to="$2"
  58       shift
  59       ;;
  60     -n)
  61       dryrun=true
  62       ;;
  63     *)
  64       echo "error: unexpected arg: $1"
  65       exit 1
  66       ;;
  67   esac
  68   shift
  69 done
  70
  71 if (( $# < 2 )); then
  72   echo "error: expected at least 2 args after options. args:"
  73   exit 1
  74 fi
  75
  76 service="$1"
  77 shift
  78
  79
  80 # maybe run, depending on $dryrun
  81 m() {
  82   if $dryrun; then
  83     printf "%s\n" "$*"
  84   else
  85     "$@"
  86   fi
  87 }
  88 # maybe run, with stdin
  89 mi() {
  90   if $dryrun; then
  91     printf "%s <<'EOF'\n" "$*"
  92     cat
  93     echo EOF
  94   else
  95     "$@"
  96   fi
  97 }
  98 e() {
  99   if $dryrun; then
 100     printf "dryrun: %s\n" "$*"
 101   fi
 102 }
 103
 104 c=$cbase/$service # c for command file path base
 105
 106 e "c=$c"
 107
 108 glob="${c}[0-9]*"
 109 arr=($glob); file="${arr[0]}"; [[ $glob != "$file" ]] || file=
 110 if [[ $file ]]; then
 111   e "file=$file"
 112 fi
 113
 114 if ! [[ -d $cbase ]]; then
 115   mkdir -p $cbase
 116 fi
 117
 118
 119 code=0
 120 "$@" || code=$?
 121 if (( code )); then
 122   send_mail=false
 123   cursor=$(journalctl --show-cursor -qn0|sed 's/^\s*--\scursor:\s*//')
 124   if [[ $file ]]; then
 125     i=${file#"$c"}
 126     if (( i < errors )); then
 127       new_file=$c$((i+1))
 128       m mv $file $new_file
 129       file=$new_file
 130       if [[ $file == $c$errors ]]; then
 131         send_mail=true
 132       else
 133         if $dryrun; then
 134           printf "dryrun: appending to file $file: %s\n" "$cursor"
 135         else
 136           printf "%s\n" "$cursor" >>$file
 137         fi
 138       fi
 139     fi
 140   else
 141     file=${c}1
 142     if $dryrun; then
 143       printf "dryrun: creating $file, contents: %s\n" "$cursor"
 144     else
 145       printf "%s\n" "$cursor" >$file
 146     fi
 147     if (( errors == 1 )); then
 148       send_mail=true
 149     fi
 150   fi
 151   if $send_mail; then
 152     if $print_all; then
 153       cursor=$(head -n1 $file)
 154     else
 155       cursor=$(tail -n1 $file)
 156     fi
 157     echo "sysd-mail-once: emailing on $errors errors. exit code: $code"
 158     mi exim -odf -t <<EOF
 159 To: $to
 160 From: $(id -u -n)@$(hostname -f)
 161 Subject: $HOSTNAME: $service exit code: $code
 162
 163 $(journalctl -u $service.service --after-cursor="$cursor")
 164 EOF
 165   fi
 166 else
 167   if [[ $file ]]; then
 168     m rm -f $file
 169     if [[ $file == $c$errors ]]; then
 170       echo "sysd-mail-once: emailing success after >= $errors errors."
 171       mi exim -odf -t <<EOF
 172 To: $to
 173 From: $(id -u -n)@$(hostname -f)
 174 Subject: $HOSTNAME: $service success
 175
 176 EOF
 177     fi
 178   fi
 179 fi