iankelling.org Git - log-quiet/blob - sysd-mail-once

   1 #!/bin/bash
   2 # Copyright (C) 2016 Ian Kelling
   3
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16
  17 set -eE -o pipefail
  18 trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR
  19
  20 errors=3
  21 tmp=(~)
  22 cbase="${tmp[0]}/sysd-mail-once-state"
  23 to=root
  24 dryrun=false
  25 while [[ $1 == -* ]]; do
  26   case "$1" in
  27     -h|--help)
  28       cat <<EOF
  29 Usage: sysd-mail-once [-t TO_ADDRESS] [-ERRORS] SERVICE COMMAND [COMMAND_ARGS...]
  30
  31 For use with systemd timers, to email (with exim) on repeated failure &
  32 success after failure.
  33
  34 In the service triggered by the timer, prepend this script to the ExecStart.
  35 The email will contain the service's logs for the last ERRORS runs.
  36
  37
  38 Stores error counts in $cbase
  39
  40 -ERRORS         ERRORS is the number of failurs to accumulate before mailing the error.
  41                 Default is 3.
  42
  43 -t TO_ADDRESS   Address to email about errors
  44 -n              Dry run. Execute command but only print out what we would do in response.
  45 EOF
  46       exit 0
  47       ;;
  48     -[0-9]*)
  49       errors=${1#-}
  50       ;;
  51     -t)
  52       to="$2"
  53       shift
  54       ;;
  55     -n)
  56       dryrun=true
  57       ;;
  58     *)
  59       echo "error: unexpected arg: $1"
  60       exit 1
  61       ;;
  62   esac
  63   shift
  64 done
  65
  66 if (( $# < 2 )); then
  67   echo "error: expected at least 2 args after options. args:"
  68   exit 1
  69 fi
  70
  71 service="$1"
  72 shift
  73
  74
  75 # maybe run, depending on $dryrun
  76 m() {
  77   if $dryrun; then
  78     printf "%s\n" "$*"
  79   else
  80     "$@"
  81   fi
  82 }
  83 # maybe run, with stdin
  84 mi() {
  85   if $dryrun; then
  86     printf "%s <<'EOF'\n" "$*"
  87     cat
  88     echo EOF
  89   else
  90     "$@"
  91   fi
  92 }
  93 e() {
  94   printf "dryrun: %s\n" "$*"
  95 }
  96
  97 c=$cbase/$service # c for command file path base
  98
  99 if $dryrun; then
 100   e "c=$c"
 101 fi
 102
 103 glob="${c}[0-9]*"
 104 arr=($glob); file="${arr[0]}"; [[ $glob != "$file" ]] || file=
 105 if [[ $dryrun && $file ]]; then
 106   e "file=$file"
 107 fi
 108
 109 if ! [[ -d $cbase ]]; then
 110   mkdir -p $cbase
 111 fi
 112
 113 if [[ ! $file ]]; then
 114   cursor=$(journalctl --show-cursor -qn0|sed 's/^\s*--\scursor:\s*//')
 115 fi
 116
 117 code=0
 118 "$@" || code=$?
 119 if (( code )); then
 120   send_mail=false
 121   if [[ $file ]]; then
 122     i=${file#"$c"}
 123     if (( i < errors )); then
 124       new_file=$c$((i+1))
 125       m mv $file $new_file
 126       file=$new_file
 127       if [[ $file == $c$errors ]]; then
 128         send_mail=true
 129       fi
 130     fi
 131   else
 132     file=${c}1
 133     if $dryrun; then
 134       printf "dryrun: creating $file, contents: %s\n" "$cursor"
 135     else
 136       printf "%s\n" "$cursor" >$file
 137     fi
 138     if (( errors == 1 )); then
 139       send_mail=true
 140     fi
 141   fi
 142   if $send_mail; then
 143     echo "sysd-mail-once: emailing on $errors errors. exit code: $code"
 144     mi exim -odf -t <<EOF
 145 To: $to
 146 From: $(id -u -n)@$(hostname -f)
 147 Subject: $HOSTNAME: $service exit code: $code
 148
 149 $(journalctl -u $service.service --after-cursor="$(<$file)")
 150 EOF
 151   fi
 152 else
 153   if [[ $file ]]; then
 154     m rm -f $file
 155     if [[ $file == $c$errors ]]; then
 156       echo "sysd-mail-once: emailing success after >= $errors errors."
 157       mi exim -odf -t <<EOF
 158 To: $to
 159 From: $(id -u -n)@$(hostname -f)
 160 Subject: $HOSTNAME: $service success
 161
 162 EOF
 163     fi
 164   fi
 165 fi