iankelling.org Git - log-quiet/blob - sysd-mail-once

   1 #!/bin/bash
   2 # I, Ian Kelling, follow the GNU license recommendations at
   3 # https://www.gnu.org/licenses/license-recommendations.en.html. They
   4 # recommend that small programs, < 300 lines, be licensed under the
   5 # Apache License 2.0. This file contains or is part of one or more small
   6 # programs. If a small program grows beyond 300 lines, I plan to switch
   7 # its license to GPL.
   8
   9 # Copyright 2024 Ian Kelling
  10
  11 # Licensed under the Apache License, Version 2.0 (the "License");
  12 # you may not use this file except in compliance with the License.
  13 # You may obtain a copy of the License at
  14
  15 #     http://www.apache.org/licenses/LICENSE-2.0
  16
  17 # Unless required by applicable law or agreed to in writing, software
  18 # distributed under the License is distributed on an "AS IS" BASIS,
  19 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  20 # See the License for the specific language governing permissions and
  21 # limitations under the License.
  22
  23
  24
  25 set -eE -o pipefail
  26 trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR
  27
  28 errors=3
  29 tmp=(~)
  30 cbase="${tmp[0]}/sysd-mail-once-state"
  31 to=root
  32 dryrun=false
  33 print_all=false
  34 while [[ $1 == -* ]]; do
  35   case "$1" in
  36     -h|--help)
  37       cat <<EOF
  38 Usage: sysd-mail-once [-t TO_ADDRESS] [-ERRORS] SERVICE COMMAND [COMMAND_ARGS...]
  39
  40 For use with systemd timers, to email (with exim) on repeated failure &
  41 success after failure.
  42
  43 In the service triggered by the timer, prepend this script to the ExecStart.
  44 The email will contain the service's logs for the last ERRORS runs.
  45
  46
  47 Stores error counts in $cbase
  48
  49 -ERRORS         ERRORS is the number of failurs to accumulate before mailing the error.
  50                 Default is 3.
  51
  52 -a              Email the logs of all errors intead of just the last one.
  53 -t TO_ADDRESS   Address to email about errors
  54 -n              Dry run. Execute command but only print out what we would do in response.
  55 EOF
  56       exit 0
  57       ;;
  58     -[0-9]*)
  59       errors=${1#-}
  60       ;;
  61     -a)
  62       print_all=true
  63       ;;
  64     -t)
  65       to="$2"
  66       shift
  67       ;;
  68     -n)
  69       dryrun=true
  70       ;;
  71     *)
  72       echo "error: unexpected arg: $1"
  73       exit 1
  74       ;;
  75   esac
  76   shift
  77 done
  78
  79 if (( $# < 2 )); then
  80   echo "error: expected at least 2 args after options. args:"
  81   exit 1
  82 fi
  83
  84 service="$1"
  85 shift
  86
  87
  88 # maybe run, depending on $dryrun
  89 m() {
  90   if $dryrun; then
  91     printf "%s\n" "$*"
  92   else
  93     "$@"
  94   fi
  95 }
  96 # maybe run, with stdin
  97 mi() {
  98   if $dryrun; then
  99     printf "%s <<'EOF'\n" "$*"
 100     cat
 101     echo EOF
 102   else
 103     "$@"
 104   fi
 105 }
 106 e() {
 107   if $dryrun; then
 108     printf "dryrun: %s\n" "$*"
 109   fi
 110 }
 111
 112 c=$cbase/$service # c for command file path base
 113
 114 e "c=$c"
 115
 116 glob="${c}[0-9]*"
 117 arr=($glob); file="${arr[0]}"; [[ $glob != "$file" ]] || file=
 118 if [[ $file ]]; then
 119   e "file=$file"
 120 fi
 121
 122 if ! [[ -d $cbase ]]; then
 123   mkdir -p $cbase
 124 fi
 125
 126
 127 code=0
 128 "$@" || code=$?
 129 if (( code )); then
 130   send_mail=false
 131   cursor=$(journalctl --show-cursor -qn0|sed 's/^\s*--\scursor:\s*//')
 132   if [[ $file ]]; then
 133     i=${file#"$c"}
 134     if (( i < errors )); then
 135       new_file=$c$((i+1))
 136       m mv $file $new_file
 137       file=$new_file
 138       if [[ $file == $c$errors ]]; then
 139         send_mail=true
 140       else
 141         if $dryrun; then
 142           printf "dryrun: appending to file $file: %s\n" "$cursor"
 143         else
 144           printf "%s\n" "$cursor" >>$file
 145         fi
 146       fi
 147     fi
 148   else
 149     file=${c}1
 150     if $dryrun; then
 151       printf "dryrun: creating $file, contents: %s\n" "$cursor"
 152     else
 153       printf "%s\n" "$cursor" >$file
 154     fi
 155     if (( errors == 1 )); then
 156       send_mail=true
 157     fi
 158   fi
 159   if $send_mail; then
 160     if $print_all; then
 161       cursor=$(head -n1 $file)
 162     else
 163       cursor=$(tail -n1 $file)
 164     fi
 165     echo "sysd-mail-once: emailing on $errors errors. exit code: $code"
 166     mi exim -odf -t <<EOF
 167 To: $to
 168 From: $(id -u -n)@$(hostname -f)
 169 Subject: $HOSTNAME: $service exit code: $code
 170
 171 $(journalctl -u $service.service --after-cursor="$cursor")
 172 EOF
 173   fi
 174 else
 175   if [[ $file ]]; then
 176     m rm -f $file
 177     if [[ $file == $c$errors ]]; then
 178       echo "sysd-mail-once: emailing success after >= $errors errors."
 179       mi exim -odf -t <<EOF
 180 To: $to
 181 From: $(id -u -n)@$(hostname -f)
 182 Subject: $HOSTNAME: $service success
 183
 184 EOF
 185     fi
 186   fi
 187 fi