fi
sudo chroot $d apt-get update
sudo DEBIAN_FRONTEND=noninteractive chroot $d apt-get -y dist-upgrade --purge --auto-remove
- sudo DEBIAN_FRONTEND=noninteractive schroot -c $n -- apt-get install --allow-unauthenticated -y ${apps[@]}
sudo cp -P {,$d}/etc/localtime
+ if (( ${#apps[@]} )); then
+ sudo DEBIAN_FRONTEND=noninteractive schroot -c $n -- apt-get install --allow-unauthenticated -y ${apps[@]}
+ fi
}
# always run this first, edit the test files, then run the following
testsieve() {
sieve-filter ~/sieve/maintest.sieve ${1:-INBOX} delete 2> >(head; tail) >/tmp/testsieve.log && sed -rn '/^Performed actions:/,/^[^ ]/{/^ /p}' /tmp/testsieve.log | sort | uniq -c
- _dosieve
}
runsieve() {
c ~/sieve; cp personal{test,}.sieve; cp lists{test,}.sieve; cp personalend{test,}.sieve
alertme() {
if [[ -t 0 ]]; then
exim -t <<EOF
-From: alertme@iankelling.org
+From: alertme@b8.nz
To: alerts@iankelling.org
Subject: $*
EOF
else
read sub
{ cat <<EOF
-From: alertme@iankelling.org
+From: alertme@b8.nz
To: alerts@iankelling.org
Subject: $sub
} | exim -t
fi
}
+daylertme() {
+ if [[ -t 0 ]]; then
+ exim -t <<EOF
+From: alertme@b8.nz
+To: daylerts@iankelling.org
+Subject: $*
+EOF
+ else
+ read sub
+ { cat <<EOF
+From: alertme@b8.nz
+To: daylerts@iankelling.org
+Subject: $sub
+
+EOF
+ cat
+ } | exim -t
+ fi
+}
+
+# alert when a page goes live. not urgent.
+alert200() {
+ url="$1"
+ tmpdir="$(mktemp -d)"
+ cd $tmpdir
+ while true; do
+ if torsocks wget -q "$url"; then
+ alertme $tmpdir
+ fi
+ sleep 600 + $(( RANDOM % 300 ))
+ done
+}
+
# mail related
testexim() {
done
}
-if $check; then
- # this is to prevent systemd from filling up the journal
- for (( runcount=0; runcount < 90; runcount++ )); do
+loop-main() {
+ while true; do
main
sleep 60
done
+}
+
+if $check; then
+ loop-main
else
main
fi
if grep -qxF $HOSTNAME $x; then all_dirs+=( ${x%.hosts} ); fi
done
+# old files 2022-03
+for t in systemstatus epanicclean btrfsmaintstop dynamicipupdate; do
+ f=/etc/systemd/system/$t.timer
+ if [[ -e $f ]]; then
+ s systemctl stop $t.timer
+ s systemctl disable $t.timer
+ s rm -fv $f
+ reload_systemd=true
+ fi
+done
+
c_dirs=(/a/c{,/machine_specific/$HOSTNAME})
case $user in
iank)
# Pin: release o=LP-PPA-system76-dev-stable
# Pin-Priority: 1001
# EOF
+ #
+ # TODO: I had to uninstall linux-image-generic-hwe-20.04 because of a conflict
+ # about linux-firmware. Should probably install it to begin with in fai if
+ # i'm going to use.
pi system76-driver system76-firmware
# if you get a notice about a firmware update, the notifier on i3
# is too dumb to do anything when you click it. so to see
EOF
s schroot -c flidas locale-gen
s schroot -c flidas update-locale LANG=en_US.UTF-8
+
+ m mkschroot -s /a/bin/fai/fai/config/files/etc/apt/sources.list.d/testing.list/TESTING_NONFREE debian unstable debootstrap
+ sudo cp -a /nocow/schroot/unstable/usr/share/debootstrap/scripts/* /usr/share/debootstrap/scripts
+
+ m mkschroot -s /a/bin/fai/fai/config/files/etc/apt/sources.list.d/impish.list/IMPISH ubuntu impish
+
;;
esac
;;
m /a/bin/distro-setup/mymimes
-sgo dynamicipupdate.timer
-
-
# stop autopoping windows when i plug in an android phone.
# dbus-launch makes this work within an ssh connection, otherwise you get this message,
# with still 0 exit code.
# btrfs maintenance
sgo btrfsmaint.timer
-sgo btrfsmaintstop.timer
-
-sgo systemstatus.timer
+sgo btrfsmaintstop
+sgo systemstatus
+sgo dynamicipupdate
if grep -xFq $HOSTNAME /a/bin/ds/machine_specific/btrbk.hosts; then
}
-if [[ $INVOCATION_ID ]]; then
- # this is to prevent systemd from filling up the journal
- for (( runcount=0; runcount < 100; runcount++ )); do
+loop-main() {
+ while true; do
main
sleep 30
done
+}
+
+
+if [[ $INVOCATION_ID ]]; then
+ loop-main
else
main
fi
-
-
-
-
exit 0
EOF
}
-if [[ $INVOCATION_ID ]]; then
- # this is to prevent systemd from filling up the journal
- for (( runcount=0; runcount < 100; runcount++ )); do
+loop-main() {
+ while true; do
main
sleep 30
done
+}
+
+if [[ $INVOCATION_ID ]]; then
+ loop-main
else
main
fi
+++ /dev/null
-# See https://prometheus.io/docs/alerting/configuration/ for documentation.
-
-global:
- resolve_timeout: 3m
- smtp_smarthost: mx.iankelling.org:587
- smtp_from: alerts@iankelling.org
- smtp_require_tls: False
- smtp_hello: defaultnn.b8.nz
-templates:
-- '/etc/prometheus/alertmanager_templates/*.tmpl'
-receivers:
-- email_configs:
- - html: null
- send_resolved: true
- text: '{{ template "opsgenie.default.description" . }}'
- to: alerts@iankelling.org
- name: defaultreceiver
-
-route:
- receiver: defaultreceiver
- repeat_interval: 5d
groups:
-- name: ansible managed alert rules
+- name: standard alerts
rules:
- - alert: NodeFilesystemAlmostOutOfSpace
- annotations:
- description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has
- only {{ printf "%.2f" $value }}% available space left.
- summary: Filesystem has less than 5% space left.
- expr: |-
- (
- node_filesystem_avail_bytes{job="node",fstype!=""} / node_filesystem_size_bytes{job="node",fstype!=""} * 100 < 5
- and
- node_filesystem_readonly{job="node",fstype!=""} == 0
- )
- for: 1h
- labels:
- severity: warning
- - alert: NodeFilesystemAlmostOutOfSpace
- annotations:
- description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has
- only {{ printf "%.2f" $value }}% available space left.
- summary: Filesystem has less than 3% space left.
- expr: |-
- (
- node_filesystem_avail_bytes{job="node",fstype!=""} / node_filesystem_size_bytes{job="node",fstype!=""} * 100 < 3
- and
- node_filesystem_readonly{job="node",fstype!=""} == 0
- )
- for: 1h
- labels:
- severity: critical
- - alert: NodeFilesystemFilesFillingUp
- annotations:
- description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has
- only {{ printf "%.2f" $value }}% available inodes left and is filling up.
- summary: Filesystem is predicted to run out of inodes within the next 24 hours.
- expr: |-
- (
- node_filesystem_files_free{job="node",fstype!=""} / node_filesystem_files{job="node",fstype!=""} * 100 < 40
- and
- predict_linear(node_filesystem_files_free{job="node",fstype!=""}[6h], 24*60*60) < 0
- and
- node_filesystem_readonly{job="node",fstype!=""} == 0
- )
- for: 1h
- labels:
- severity: warning
- - alert: NodeFilesystemFilesFillingUp
- annotations:
- description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has
- only {{ printf "%.2f" $value }}% available inodes left and is filling up fast.
- summary: Filesystem is predicted to run out of inodes within the next 4 hours.
- expr: |-
- (
- node_filesystem_files_free{job="node",fstype!=""} / node_filesystem_files{job="node",fstype!=""} * 100 < 20
- and
- predict_linear(node_filesystem_files_free{job="node",fstype!=""}[6h], 4*60*60) < 0
- and
- node_filesystem_readonly{job="node",fstype!=""} == 0
- )
- for: 1h
- labels:
- severity: critical
- - alert: NodeFilesystemAlmostOutOfFiles
- annotations:
- description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has
- only {{ printf "%.2f" $value }}% available inodes left.
- summary: Filesystem has less than 5% inodes left.
- expr: |-
- (
- node_filesystem_files_free{job="node",fstype!=""} / node_filesystem_files{job="node",fstype!=""} * 100 < 5
- and
- node_filesystem_readonly{job="node",fstype!=""} == 0
- )
- for: 1h
- labels:
- severity: warning
- - alert: NodeFilesystemAlmostOutOfFiles
- annotations:
- description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has
- only {{ printf "%.2f" $value }}% available inodes left.
- summary: Filesystem has less than 3% inodes left.
- expr: |-
- (
- node_filesystem_files_free{job="node",fstype!=""} / node_filesystem_files{job="node",fstype!=""} * 100 < 3
- and
- node_filesystem_readonly{job="node",fstype!=""} == 0
- )
- for: 1h
- labels:
- severity: critical
- - alert: NodeNetworkReceiveErrs
- annotations:
- description: '{{ $labels.instance }} interface {{ $labels.device }} has encountered
- {{ printf "%.0f" $value }} receive errors in the last two minutes.'
- summary: Network interface is reporting many receive errors.
+ - alert: mailtest-check
expr: |-
- increase(node_network_receive_errs_total[2m]) > 10
- for: 1h
+ time() - mailtest_check_last_usec > 60 * 12
labels:
- severity: warning
- - alert: NodeNetworkTransmitErrs
+ severity: day
annotations:
- description: '{{ $labels.instance }} interface {{ $labels.device }} has encountered
- {{ printf "%.0f" $value }} transmit errors in the last two minutes.'
- summary: Network interface is reporting many transmit errors.
- expr: |-
- increase(node_network_transmit_errs_total[2m]) > 10
- for: 1h
- labels:
- severity: warning
- - alert: NodeHighNumberConntrackEntriesUsed
- annotations:
- description: '{{ $value | humanizePercentage }} of conntrack entries are used'
- summary: Number of conntrack are getting close to the limit
- expr: |-
- (node_nf_conntrack_entries / node_nf_conntrack_entries_limit) > 0.75
- labels:
- severity: warning
- - alert: NodeClockSkewDetected
- annotations:
- message: Clock on {{ $labels.instance }} is out of sync by more than 300s. Ensure
- NTP is configured correctly on this host.
- summary: Clock skew detected.
- expr: |-
- (
- node_timex_offset_seconds > 0.05
- and
- deriv(node_timex_offset_seconds[5m]) >= 0
- )
- or
- (
- node_timex_offset_seconds < -0.05
- and
- deriv(node_timex_offset_seconds[5m]) <= 0
- )
- for: 10m
- labels:
- severity: warning
- - alert: NodeClockNotSynchronising
- annotations:
- message: Clock on {{ $labels.instance }} is not synchronising. Ensure NTP is configured
- on this host.
- summary: Clock not synchronising.
- expr: |-
- min_over_time(node_timex_sync_status[5m]) == 0
- for: 10m
- labels:
- severity: warning
- - alert: ianktest
- expr: node_systemd_version >= 300
- labels:
- severity: critical
- annotations:
- description: '{{ $labels.instance }} ianktest.'
- summary: Instance {{ $labels.instance }} - ianktest
- - alert: ianktest
- expr: node_systemd_version >= 300
- labels:
- severity: critical
- annotations:
- description: '{{ $labels.instance }} ianktest.'
- summary: Instance {{ $labels.instance }} - ianktest
+ description: '{{ $labels.instance }} mailtest-check'
+ summary: {{ $labels.instance }} mailtest-check
- - alert: ianktest
+ - alert: mailtest-check
expr: |-
- time() - mailtest_check_last_usec > 60 * 8
+ # 42 mins: enough for a 30 min queue run plus 12
+ time() - mailtest_check_last_usec > 60 * 42
labels:
- severity: critical
+ severity: prod
annotations:
- description: '{{ $labels.instance }} mailtest'
- summary: Instance {{ $labels.instance }} - ianktest
+ description: '{{ $labels.instance }} mailtest-check'
+ summary: {{ $labels.instance }} mailtest-check
+
+
+
+ # - alert: NodeFilesystemAlmostOutOfSpace
+ # annotations:
+ # description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has
+ # only {{ printf "%.2f" $value }}% available space left.
+ # summary: Filesystem has less than 5% space left.
+ # expr: |-
+ # (
+ # node_filesystem_avail_bytes{job="node",fstype!=""} / node_filesystem_size_bytes{job="node",fstype!=""} * 100 < 5
+ # and
+ # node_filesystem_readonly{job="node",fstype!=""} == 0
+ # )
+ # for: 1h
+ # labels:
+ # severity: warning
+ # - alert: NodeFilesystemAlmostOutOfSpace
+ # annotations:
+ # description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has
+ # only {{ printf "%.2f" $value }}% available space left.
+ # summary: Filesystem has less than 3% space left.
+ # expr: |-
+ # (
+ # node_filesystem_avail_bytes{job="node",fstype!=""} / node_filesystem_size_bytes{job="node",fstype!=""} * 100 < 3
+ # and
+ # node_filesystem_readonly{job="node",fstype!=""} == 0
+ # )
+ # for: 1h
+ # labels:
+ # severity: critical
+ # - alert: NodeFilesystemFilesFillingUp
+ # annotations:
+ # description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has
+ # only {{ printf "%.2f" $value }}% available inodes left and is filling up.
+ # summary: Filesystem is predicted to run out of inodes within the next 24 hours.
+ # expr: |-
+ # (
+ # node_filesystem_files_free{job="node",fstype!=""} / node_filesystem_files{job="node",fstype!=""} * 100 < 40
+ # and
+ # predict_linear(node_filesystem_files_free{job="node",fstype!=""}[6h], 24*60*60) < 0
+ # and
+ # node_filesystem_readonly{job="node",fstype!=""} == 0
+ # )
+ # for: 1h
+ # labels:
+ # severity: warning
+ # - alert: NodeFilesystemFilesFillingUp
+ # annotations:
+ # description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has
+ # only {{ printf "%.2f" $value }}% available inodes left and is filling up fast.
+ # summary: Filesystem is predicted to run out of inodes within the next 4 hours.
+ # expr: |-
+ # (
+ # node_filesystem_files_free{job="node",fstype!=""} / node_filesystem_files{job="node",fstype!=""} * 100 < 20
+ # and
+ # predict_linear(node_filesystem_files_free{job="node",fstype!=""}[6h], 4*60*60) < 0
+ # and
+ # node_filesystem_readonly{job="node",fstype!=""} == 0
+ # )
+ # for: 1h
+ # labels:
+ # severity: critical
+ # - alert: NodeFilesystemAlmostOutOfFiles
+ # annotations:
+ # description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has
+ # only {{ printf "%.2f" $value }}% available inodes left.
+ # summary: Filesystem has less than 5% inodes left.
+ # expr: |-
+ # (
+ # node_filesystem_files_free{job="node",fstype!=""} / node_filesystem_files{job="node",fstype!=""} * 100 < 5
+ # and
+ # node_filesystem_readonly{job="node",fstype!=""} == 0
+ # )
+ # for: 1h
+ # labels:
+ # severity: warning
+ # - alert: NodeFilesystemAlmostOutOfFiles
+ # annotations:
+ # description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has
+ # only {{ printf "%.2f" $value }}% available inodes left.
+ # summary: Filesystem has less than 3% inodes left.
+ # expr: |-
+ # (
+ # node_filesystem_files_free{job="node",fstype!=""} / node_filesystem_files{job="node",fstype!=""} * 100 < 3
+ # and
+ # node_filesystem_readonly{job="node",fstype!=""} == 0
+ # )
+ # for: 1h
+ # labels:
+ # severity: critical
+ # - alert: NodeNetworkReceiveErrs
+ # annotations:
+ # description: '{{ $labels.instance }} interface {{ $labels.device }} has encountered
+ # {{ printf "%.0f" $value }} receive errors in the last two minutes.'
+ # summary: Network interface is reporting many receive errors.
+ # expr: |-
+ # increase(node_network_receive_errs_total[2m]) > 10
+ # for: 1h
+ # labels:
+ # severity: warning
+ # - alert: NodeNetworkTransmitErrs
+ # annotations:
+ # description: '{{ $labels.instance }} interface {{ $labels.device }} has encountered
+ # {{ printf "%.0f" $value }} transmit errors in the last two minutes.'
+ # summary: Network interface is reporting many transmit errors.
+ # expr: |-
+ # increase(node_network_transmit_errs_total[2m]) > 10
+ # for: 1h
+ # labels:
+ # severity: warning
+ # - alert: NodeHighNumberConntrackEntriesUsed
+ # annotations:
+ # description: '{{ $value | humanizePercentage }} of conntrack entries are used'
+ # summary: Number of conntrack are getting close to the limit
+ # expr: |-
+ # (node_nf_conntrack_entries / node_nf_conntrack_entries_limit) > 0.75
+ # labels:
+ # severity: warning
+ # - alert: NodeClockSkewDetected
+ # annotations:
+ # message: Clock on {{ $labels.instance }} is out of sync by more than 300s. Ensure
+ # NTP is configured correctly on this host.
+ # summary: Clock skew detected.
+ # expr: |-
+ # (
+ # node_timex_offset_seconds > 0.05
+ # and
+ # deriv(node_timex_offset_seconds[5m]) >= 0
+ # )
+ # or
+ # (
+ # node_timex_offset_seconds < -0.05
+ # and
+ # deriv(node_timex_offset_seconds[5m]) <= 0
+ # )
+ # for: 10m
+ # labels:
+ # severity: warning
+ # - alert: NodeClockNotSynchronising
+ # annotations:
+ # message: Clock on {{ $labels.instance }} is not synchronising. Ensure NTP is configured
+ # on this host.
+ # summary: Clock not synchronising.
+ # expr: |-
+ # min_over_time(node_timex_sync_status[5m]) == 0
+ # for: 10m
+ # labels:
+ # severity: warning
+ # - alert: ianktest
+ # expr: node_systemd_version >= 300
+ # labels:
+ # severity: critical
+ # annotations:
+ # description: '{{ $labels.instance }} ianktest.'
+ # summary: Instance {{ $labels.instance }} - ianktest
--- /dev/null
+# Files to copy into the chroot from the host system.
+#
+# <source and destination>
+
+# ian comented out for this error
+#E: 20copyfiles: cp: '/etc/resolv.conf' and '/var/run/schroot/mount/impish-59f88d2b-b06e-4413-9f1e-33ad4d1af6e4/etc/resolv.conf' are the same file
+#E: impish-59f88d2b-b06e-4413-9f1e-33ad4d1af6e4: Chroot setup failed: stage=setup-start
+
+#/etc/resolv.conf
+++ /dev/null
-[Unit]
-Description=btrfsmaintstop
-
-[Timer]
-# for initial run. required.
-OnActiveSec=10
-# for subsequent runs.
-OnUnitInactiveSec=20
-
-[Install]
-WantedBy=timers.target
+++ /dev/null
-[Unit]
-Description=dynamicipupdate
-
-[Timer]
-# for initial run. required.
-OnActiveSec=10
-# for subsequent runs.
-OnUnitInactiveSec=30
-
-[Install]
-WantedBy=timers.target
[Unit]
Description=epanic-clean
-After=multi-user.target
+After=local-fs.target
+StartLimitIntervalSec=0
[Service]
-Type=oneshot
+Type=simple
ExecStart=/usr/local/bin/sysd-mail-once -3 epanic-clean /usr/local/bin/epanic-clean
+Restart=always
+RestartSec=600
+
+[Install]
+WantedBy=grahical.target
+++ /dev/null
-[Unit]
-Description=epanic-clean
-
-[Timer]
-# for initial run. required.
-OnActiveSec=10
-# for subsequent runs.
-OnUnitInactiveSec=30
-
-[Install]
-WantedBy=timers.target
[Unit]
Description=systemstatus
-After=multi-user.target
+StartLimitIntervalSec=0
+After=local-fs.target
[Service]
-Type=oneshot
+Type=simple
Environment=XDG_RUNTIME_DIR=/run/user/1000
ExecStart=/usr/local/bin/sysd-mail-once -3 systemstatus /usr/local/bin/system-status
IOSchedulingClass=idle
CPUSchedulingPolicy=idle
User=iank
Group=iank
+Restart=always
+RestartSec=600
+
+
+[Install]
+WantedBy=grahical.target
+++ /dev/null
-[Unit]
-Description=systemstatus
-
-[Timer]
-# for initial run. required.
-OnActiveSec=10
-# for subsequent runs.
-OnUnitInactiveSec=20
-
-[Install]
-WantedBy=timers.target
# changed, so that should fix it.
/a/bin/log-quiet/setup
rsync -t --chmod=755 --chown=root:root switch-mail-host btrbk-run mount-latest-subvol \
- check-subvol-stale system-status myi3status mailtest-check \
- epanic-clean mailbindwatchdog \
+ check-subvol-stale myi3status mailtest-check \
+ mailbindwatchdog \
/a/bin/log-quiet/sysd-mail-once hssh \
- btrfsmaint \
- dynamic-ip-update \
check-mailq \
unsaved-buffers.el \
mail-backup-clean \
ip6tables-exim \
/usr/local/bin
rsync -t --chmod=755 --chown=root:root /a/bin/errhandle/err /usr/local/lib
+
+cmd=( rsync -aiSAX --chown=root:root --chmod=g-s
+ epanic-clean
+ system-status
+ btrfsmaint
+ dynamic-ip-update /usr/local/bin
+ )
+
+sre() {
+ service=$1
+ if [[ $(systemctl is-active $1.service ||:) != inactive ]]; then
+ systemctl restart $service
+ fi
+
+}
+
+while read -r line; do
+ file="${line:12}"
+ echo $file
+ case $file in
+ btrfsmaint)
+ sre btrfsmaintstop
+ ;;
+ *)
+ sre ${file//-/}
+ ;;
+ esac
+done < <("${cmd[@]}")
$MAIL_HOST|bk)
local_mx=mail.iankelling.org
# ||: is to allow for temporary connection issues.
- rsync "${opt[@]}" -ogtL --chown=root:Debian-exim --chmod=640 \
+ rsync ${opt[@]} -ogtL --chown=root:Debian-exim --chmod=640 \
root@li.iankelling.org:/etc/letsencrypt/live/mail.iankelling.org/{fullchain.pem,privkey.pem} /etc/exim4 ||:
if ! openssl x509 -checkend $(( 60 * 60 * 24 * 3 )) -noout -in /etc/exim4/fullchain.pem; then
echo "$0: error!: cert rsync failed and it will expire in less than 3 days"
# background: dovecot does not yet have ocsp stapling support
# reference: https://community.letsencrypt.org/t/simple-guide-using-lets-encrypt-ssl-certs-with-dovecot/2921
#
-# for phone, k9mail, same thing but username alerts, pass in ivy-pass.
+# for phone, k9mail, fdroid, same thing but username alerts, pass in ivy-pass.
# also, bk.b8.nz for secondary alerts, username is iank. same alerts pass.
-# fetching mail settings: folder poll frequency 10 minutes
+# fetching mail settings: folder poll frequency 10 minutes.
+# account settings, fetching mail, push folders: All. Then disable the persistent notification.
#######
m systemctl daemon-reload
fi
-m systemctl --now enable epanicclean.timer
+m systemctl --now enable epanicclean
case $HOSTNAME in
je)
fi
if ! systemctl is-active clamav-daemon >/dev/null; then
m systemctl --now enable clamav-daemon
- out=$(rsync -aiSAX --chown=root:root --chmod=g-s /a/bin/ds/filesystem/etc/systemd/system/epanicclean.{timer,service} /etc/systemd/system)
+ out=$(rsync -aiSAX --chown=root:root --chmod=g-s /a/bin/ds/filesystem/etc/systemd/system/epanicclean.service /etc/systemd/system)
if [[ $out ]]; then
reload=true
fi
cat >/etc/cron.d/mailtest <<EOF
SHELL=/bin/bash
PATH=/usr/bin:/bin:/usr/local/bin
-MAILTO=alerts@iankelling.org
-*/5 * * * * $u send-test-forward |& log-once send-test-forward
+MAILTO=daylerts@iankelling.org
+*/5 * * * * $u send-test-forward |& log-once send-test-forward
*/10 * * * * root chmod -R g+rw /m/md/bounces |& log-once -1 bounces-chmod
-# im seeing some intermittent failures on the slow check, do it all the time
-# for now. It looks like a dns failure.
-#5-59/5 * * * * root mailtest-check |& log-once -1 mailtest-check
-#0 * * * * root mailtest-check slow |& log-once -1 mailtest-slow
-*/5 * * * * root timeout 290 mailtest-check slow |& log-once -12 mailtest-check
+*/5 * * * * root timeout 290 mailtest-check slow |& log-once -4 mailtest-check
# if a bounce happened yesterday, dont let it slip through the cracks
-8 1 * * * root awk '\$5 == "**"' /var/log/exim4/mainlog.1
+8 1 * * * root export MAILTO=alerts@iankelling.org; awk '\$5 == "**"' /var/log/exim4/mainlog.1
+0 13 * * * root echo "If the 1pm doesnt happen, you are in the matrix. Wake up."
EOF
m sudo rsync -ahhi --chown=root:root --chmod=0755 \
/b/ds/mailtest-check /b/ds/check-remote-mailqs /usr/local/bin/
# We run this cronjob along with sending the test email every 5 minutes,
# so give it 1 minute to arrive, then if the latest email is older than
# 7 minutes, the last 2 haven't arrived in a reasonable amount of time.
-min_limit=8
+# However, when machines reboot things can get delayed, so add 10 mins,
+# not sure if that is a good number or not.
+min_limit=17
# spamassassin checking takes about 8 seconds. only do that every
if [[ $1 ]]; then
new_host=$1
if [[ $new_host == localhost ]]; then
+ mailhost_p=1
new_host=$HOSTNAME
+ else
+ mailhost_p=0
+ fi
+
+ # https://www.robustperception.io/how-to-have-labels-for-machine-roles
+ dir=/var/lib/prometheus/node-exporter
+ if [[ -e $dir ]]; then
+ cat > $dir/mailhost.prom <<EOF
+host_role{role="mailhost"} $mailhost_p
+EOF
fi
+
m sed -ri "s/MAIL_HOST=.*/MAIL_HOST=$new_host/" /a/bin/bash_unpublished/source-state
fi
source /a/bin/bash_unpublished/source-state
[[ $EUID == 0 ]] || exec sudo -E "${BASH_SOURCE[0]}" "$@"
-for n in buster; do
+for n in bullseye; do
if [[ -e /etc/schroot/chroot.d/$n.conf ]]; then
cd /
+ schroot -c $n -- apt-get -y update
schroot -c $n -- apt-get -y dist-upgrade --purge --auto-remove
fi
done
# Copyright (C) 2019 Ian Kelling
# SPDX-License-Identifier: AGPL-3.0-or-later
-# usage: runs 4 times every 15 seconds unless any args are passed, or we
-# are on battery power, then just runs once.
+# usage: runs once every 15 seconds unless any args are passed, or we
+# then just runs once. On battery power, run once per minute.
if [ -z "$BASH_VERSION" ]; then echo "error: shell is not bash" >&2; exit 1; fi
/usr/local/bin/log-once "$@" | ifne mail -s "$HOSTNAME: system-status $2" root@localhost
}
+loday() {
+ /usr/local/bin/log-once "$@" | ifne mail -s "$HOSTNAME: system-status $2" daylerts@iankelling.org
+}
+
+
write-status() {
chars=("${first_chars[@]}")
if [[ $(find /var/mail -type f \! -empty -print -quit) ]]; then
var_mail_msg="message in /var/mail"
fi
- lo -1 var_mail $var_mail_msg
+ loday -1 var_mail $var_mail_msg
glob=(/m/md/bounces/new/*)
if [[ -e ${glob[0]} ]]; then
chars+=("BOUNCE")
bouncemsg="message in /m/md/bounces/new"
fi
- lo -1 bounce $bouncemsg
+ loday -1 bounce $bouncemsg
# emails without the S (seen) flag. this only checks the last flag,
# but its good enough for me.
glob=(/m/md/alerts/{new,cur}/!(*,S))
if [[ -e ${glob[0]} ]]; then
chars+=("A")
fi
+
+ glob=(/m/md/daylerts/{new,cur}/!(*,S))
+ if [[ -e ${glob[0]} ]]; then
+ chars+=("L")
+ fi
+
+
tmp=(/var/local/cron-errors/mailtest-check*)
if (( ${#tmp[@]} )); then
chars+=("MAILPING")
chars+=("SPAMD")
fi
- # early in install process, we dont have permission yet for exiqgrep
- qlen=$(/usr/sbin/exiqgrep -o 600 -c -b | awk '{print $1}') ||:
+ # early in install process, we dont have permission yet for exiqgrep.
+ # 1100 helps allow for system restarts
+ qlen=$(/usr/sbin/exiqgrep -o 1100 -c -b | awk '{print $1}') ||:
if ((qlen)); then
qmsg="queue length $qlen"
chars+=("q $qlen")
# No point in emailing about the mailq on a host where we don't
# check email.
$MAIL_HOST|bk)
- lo -120 qlen $qmsg
+ loday -120 qlen $qmsg
;;
esac
chars+=("BTRBK.TIMER")
bbkmsg="btrbk.timer not enabled"
fi
- lo -960 btrbk.timer $bbkmsg
+ lo -48 btrbk.timer $bbkmsg
## check if last snapshot was within an hour
vol=o
# use this if we want to do something just once per minute
first_chars=()
-power=true
-if [[ -e /sys/class/power_supply/AC/online && $(</sys/class/power_supply/AC/online) == 0 ]]; then
- power=false
-fi
write-status
if [[ $1 ]]; then
exit 0
fi
-if ! $power; then
- exit 0
-fi
+main-loop() {
+while true; do
+ power=true
+ if [[ -e /sys/class/power_supply/AC/online && $(</sys/class/power_supply/AC/online) == 0 ]]; then
+ power=false
+ fi
+ wait=15
+ if ! $power; then
+ wait=60
+ fi
-# about 15 minutes
-for ((i=1; i<=60; i++)); do
- sleep 15
+ sleep $wait
write-status
done
+}
+
+# ensure our long operations are one line so we are not prone errors
+# from this file being modified.
+main-loop
iankelling.org / git / distro-setup / commitdiff