+
+### begin prometheus ###
+
+case $HOSTNAME in
+ kd)
+ # Font awesome is needed for the alertmanager ui.
+ pi prometheus-alertmanager prometheus fonts-font-awesome
+ /c/roles/prom/files/simple/usr/local/bin/fsf-install-prometheus
+ # make it available for other machines
+ rsync -a /usr/local/bin/amtool /a/opt/bin
+ web-conf -p 9091 -f 9090 - apache2 i.b8.nz <<'EOF'
+<Location "/">
+AuthType Basic
+AuthName "basic_auth"
+# created with
+# htpasswd -c prometheus-htpasswd USERNAME
+AuthUserFile "/etc/prometheus-htpasswd"
+Require valid-user
+</Location>
+EOF
+
+ web-conf -p 9094 -f 9093 - apache2 i.b8.nz <<'EOF'
+<Location "/">
+AuthType Basic
+AuthName "basic_auth"
+# created with
+# htpasswd -c prometheus-htpasswd USERNAME
+AuthUserFile "/etc/prometheus-htpasswd"
+Require valid-user
+</Location>
+EOF
+
+ # by default, the alertmanager web ui is not enabled other than a page
+ # that suggests to use the amtool cli. that tool is good, but you cant
+ # silence things nearly as easily as with the gui.
+ if [[ ! -e /usr/share/prometheus/alertmanager/ui/index.html ]]; then
+ # default script didnt work, required some changes to get elm 19.1,
+ # which is a dependency of the latest alertmanager. I modified
+ # and copied it into /b/ds. In future, might need some other
+ # solution.
+ #sudo /usr/share/prometheus/alertmanager/generate-ui.sh
+ sudo /b/ds/generate-ui.sh
+ ser restart prometheus-alertmanager
+ fi
+
+ s /c/roles/prom_export/files/simple/usr/local/bin/fsf-install-node-exporter -l 127.0.0.1
+
+ for ser in prometheus-node-exporter prometheus-alertmanager prometheus; do
+ sysd-prom-fail-install $ser
+ done
+
+ ;;
+ *)
+ s /c/roles/prom_export/files/simple/usr/local/bin/fsf-install-node-exporter
+ ;;
+esac
+
+# cleanup old files. 2023-02
+x=(/var/lib/prometheus/node-exporter/*.premerge)
+if [[ -e ${x[0]} ]]; then
+ s rm /var/lib/prometheus/node-exporter/*
+fi
+
+
+case $HOSTNAME in
+ # todo, for limiting node exporter http,
+ # either use iptables or, in
+ # /etc/default/prometheus-node-exporter
+ # listen on the wireguard interface
+
+ *)
+ wgip=$(command sudo sed -rn 's,^ *Address *= *([^/]+).*,\1,p' /etc/wireguard/wghole.conf)
+ # old filename. remove once all hosts are updated.
+ s rm -fv /etc/apache2/sites-enabled/${HOSTNAME}wg.b8.nz.conf
+ web-conf -i -a $wgip -p 9101 -f 9100 - apache2 ${HOSTNAME}wg.b8.nz <<'EOF'
+<Location "/">
+AuthType Basic
+AuthName "basic_auth"
+# created with
+# htpasswd -c prometheus-export-htpasswd USERNAME
+AuthUserFile "/etc/prometheus-export-htpasswd"
+Require valid-user
+</Location>
+EOF
+ # For work, i think we will just use the firewall for hosts in the main data center, and
+ # vpn for hosts outside it.
+
+ # TODO: figure out how to detect the ping failure and try again.
+
+ # Binding to the wg interface, it might go down, so always restart, and wait for it on boot.
+ s mkdir /etc/systemd/system/apache2.service.d
+ sd /etc/systemd/system/apache2.service.d/restart.conf <<EOF
+[Unit]
+After=wg-quick@wghole.service
+StartLimitIntervalSec=0
+
+[Service]
+Restart=always
+RestartSec=30
+EOF
+
+ ;;
+esac
+
+### end prometheus ###
+
+### begin nagios ###
+
+pi nagios-nrpe-server
+