iankelling.org Git - distro-setup/blob - filesystem/etc/prometheus/rules/iank.yml

   1
   2 groups:
   3 - name: standard alerts
   4   rules:
   5   - alert: mailtest-check
   6     expr: |-
   7       time() - mailtest_check_last_usec > 60 * 12
   8     labels:
   9       severity: day
  10     annotations:
  11       description: '{{ $labels.instance }} mailtest-check'
  12       summary: {{ $labels.instance }} mailtest-check
  13
  14   - alert: mailtest-check
  15     expr: |-
  16       # 42 mins: enough for a 30 min queue run plus 12
  17       time() - mailtest_check_last_usec > 60 * 42
  18     labels:
  19       severity: prod
  20     annotations:
  21       description: '{{ $labels.instance }} mailtest-check'
  22       summary: {{ $labels.instance }} mailtest-check
  23
  24
  25
  26   # - alert: NodeFilesystemAlmostOutOfSpace
  27   #   annotations:
  28   #     description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has
  29   #       only {{ printf "%.2f" $value }}% available space left.
  30   #     summary: Filesystem has less than 5% space left.
  31   #   expr: |-
  32   #     (
  33   #       node_filesystem_avail_bytes{job="node",fstype!=""} / node_filesystem_size_bytes{job="node",fstype!=""} * 100 < 5
  34   #     and
  35   #       node_filesystem_readonly{job="node",fstype!=""} == 0
  36   #     )
  37   #   for: 1h
  38   #   labels:
  39   #     severity: warning
  40   # - alert: NodeFilesystemAlmostOutOfSpace
  41   #   annotations:
  42   #     description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has
  43   #       only {{ printf "%.2f" $value }}% available space left.
  44   #     summary: Filesystem has less than 3% space left.
  45   #   expr: |-
  46   #     (
  47   #       node_filesystem_avail_bytes{job="node",fstype!=""} / node_filesystem_size_bytes{job="node",fstype!=""} * 100 < 3
  48   #     and
  49   #       node_filesystem_readonly{job="node",fstype!=""} == 0
  50   #     )
  51   #   for: 1h
  52   #   labels:
  53   #     severity: critical
  54   # - alert: NodeFilesystemFilesFillingUp
  55   #   annotations:
  56   #     description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has
  57   #       only {{ printf "%.2f" $value }}% available inodes left and is filling up.
  58   #     summary: Filesystem is predicted to run out of inodes within the next 24 hours.
  59   #   expr: |-
  60   #     (
  61   #       node_filesystem_files_free{job="node",fstype!=""} / node_filesystem_files{job="node",fstype!=""} * 100 < 40
  62   #     and
  63   #       predict_linear(node_filesystem_files_free{job="node",fstype!=""}[6h], 24*60*60) < 0
  64   #     and
  65   #       node_filesystem_readonly{job="node",fstype!=""} == 0
  66   #     )
  67   #   for: 1h
  68   #   labels:
  69   #     severity: warning
  70   # - alert: NodeFilesystemFilesFillingUp
  71   #   annotations:
  72   #     description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has
  73   #       only {{ printf "%.2f" $value }}% available inodes left and is filling up fast.
  74   #     summary: Filesystem is predicted to run out of inodes within the next 4 hours.
  75   #   expr: |-
  76   #     (
  77   #       node_filesystem_files_free{job="node",fstype!=""} / node_filesystem_files{job="node",fstype!=""} * 100 < 20
  78   #     and
  79   #       predict_linear(node_filesystem_files_free{job="node",fstype!=""}[6h], 4*60*60) < 0
  80   #     and
  81   #       node_filesystem_readonly{job="node",fstype!=""} == 0
  82   #     )
  83   #   for: 1h
  84   #   labels:
  85   #     severity: critical
  86   # - alert: NodeFilesystemAlmostOutOfFiles
  87   #   annotations:
  88   #     description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has
  89   #       only {{ printf "%.2f" $value }}% available inodes left.
  90   #     summary: Filesystem has less than 5% inodes left.
  91   #   expr: |-
  92   #     (
  93   #       node_filesystem_files_free{job="node",fstype!=""} / node_filesystem_files{job="node",fstype!=""} * 100 < 5
  94   #     and
  95   #       node_filesystem_readonly{job="node",fstype!=""} == 0
  96   #     )
  97   #   for: 1h
  98   #   labels:
  99   #     severity: warning
 100   # - alert: NodeFilesystemAlmostOutOfFiles
 101   #   annotations:
 102   #     description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has
 103   #       only {{ printf "%.2f" $value }}% available inodes left.
 104   #     summary: Filesystem has less than 3% inodes left.
 105   #   expr: |-
 106   #     (
 107   #       node_filesystem_files_free{job="node",fstype!=""} / node_filesystem_files{job="node",fstype!=""} * 100 < 3
 108   #     and
 109   #       node_filesystem_readonly{job="node",fstype!=""} == 0
 110   #     )
 111   #   for: 1h
 112   #   labels:
 113   #     severity: critical
 114   # - alert: NodeNetworkReceiveErrs
 115   #   annotations:
 116   #     description: '{{ $labels.instance }} interface {{ $labels.device }} has encountered
 117   #       {{ printf "%.0f" $value }} receive errors in the last two minutes.'
 118   #     summary: Network interface is reporting many receive errors.
 119   #   expr: |-
 120   #     increase(node_network_receive_errs_total[2m]) > 10
 121   #   for: 1h
 122   #   labels:
 123   #     severity: warning
 124   # - alert: NodeNetworkTransmitErrs
 125   #   annotations:
 126   #     description: '{{ $labels.instance }} interface {{ $labels.device }} has encountered
 127   #       {{ printf "%.0f" $value }} transmit errors in the last two minutes.'
 128   #     summary: Network interface is reporting many transmit errors.
 129   #   expr: |-
 130   #     increase(node_network_transmit_errs_total[2m]) > 10
 131   #   for: 1h
 132   #   labels:
 133   #     severity: warning
 134   # - alert: NodeHighNumberConntrackEntriesUsed
 135   #   annotations:
 136   #     description: '{{ $value | humanizePercentage }} of conntrack entries are used'
 137   #     summary: Number of conntrack are getting close to the limit
 138   #   expr: |-
 139   #     (node_nf_conntrack_entries / node_nf_conntrack_entries_limit) > 0.75
 140   #   labels:
 141   #     severity: warning
 142   # - alert: NodeClockSkewDetected
 143   #   annotations:
 144   #     message: Clock on {{ $labels.instance }} is out of sync by more than 300s. Ensure
 145   #       NTP is configured correctly on this host.
 146   #     summary: Clock skew detected.
 147   #   expr: |-
 148   #     (
 149   #       node_timex_offset_seconds > 0.05
 150   #     and
 151   #       deriv(node_timex_offset_seconds[5m]) >= 0
 152   #     )
 153   #     or
 154   #     (
 155   #       node_timex_offset_seconds < -0.05
 156   #     and
 157   #       deriv(node_timex_offset_seconds[5m]) <= 0
 158   #     )
 159   #   for: 10m
 160   #   labels:
 161   #     severity: warning
 162   # - alert: NodeClockNotSynchronising
 163   #   annotations:
 164   #     message: Clock on {{ $labels.instance }} is not synchronising. Ensure NTP is configured
 165   #       on this host.
 166   #     summary: Clock not synchronising.
 167   #   expr: |-
 168   #     min_over_time(node_timex_sync_status[5m]) == 0
 169   #   for: 10m
 170   #   labels:
 171   #     severity: warning
 172   # - alert: ianktest
 173   #   expr: node_systemd_version >= 300
 174   #   labels:
 175   #     severity: critical
 176   #   annotations:
 177   #     description: '{{ $labels.instance }} ianktest.'
 178   #     summary: Instance {{ $labels.instance }} - ianktest