+ # Font awesome is needed for the alertmanager ui.
+ pi prometheus-alertmanager prometheus fonts-font-awesome
+ /c/roles/prom/files/simple/usr/local/bin/fsf-install-prometheus
+ # make it available for other machines
+ rsync -a /usr/local/bin/amtool /a/opt/bin
+ web-conf -p 9091 -f 9090 - apache2 i.b8.nz <<'EOF'
+<Location "/">
+AuthType Basic
+AuthName "basic_auth"
+# created with
+# htpasswd -c prometheus-htpasswd USERNAME
+AuthUserFile "/etc/prometheus-htpasswd"
+Require valid-user
+</Location>
+EOF
+
+ web-conf -p 9094 -f 9093 - apache2 i.b8.nz <<'EOF'
+<Location "/">
+AuthType Basic
+AuthName "basic_auth"
+# created with
+# htpasswd -c prometheus-htpasswd USERNAME
+AuthUserFile "/etc/prometheus-htpasswd"
+Require valid-user
+</Location>
+EOF
+
+ # by default, the alertmanager web ui is not enabled other than a page
+ # that suggests to use the amtool cli. that tool is good, but you cant
+ # silence things nearly as easily as with the gui.
+ if [[ ! -e /usr/share/prometheus/alertmanager/ui/index.html ]]; then
+ # default script didnt work, required some changes to get elm 19.1,
+ # which is a dependency of the latest alertmanager. I modified
+ # and copied it into /b/ds. In future, might need some other
+ # solution.
+ #sudo /usr/share/prometheus/alertmanager/generate-ui.sh
+ sudo /b/ds/generate-ui.sh
+ ser restart prometheus-alertmanager
+ fi
+
+ s /c/roles/prom_export/files/simple/usr/local/bin/fsf-install-node-exporter -l 127.0.0.1
+
+ for ser in prometheus-node-exporter prometheus-alertmanager prometheus; do
+ sysd-prom-fail-install $ser
+ done
+
+ ;;
+ *)
+ s /c/roles/prom_export/files/simple/usr/local/bin/fsf-install-node-exporter
+ ;;
+esac
+
+# cleanup old files. 2023-02
+x=(/var/lib/prometheus/node-exporter/*.premerge)
+if [[ -e ${x[0]} ]]; then
+ s rm /var/lib/prometheus/node-exporter/*
+fi
+
+
+case $HOSTNAME in
+ # todo, for limiting node exporter http,
+ # either use iptables or, in
+ # /etc/default/prometheus-node-exporter
+ # listen on the wireguard interface
+
+ *)
+ wgip=$(command sudo sed -rn 's,^ *Address *= *([^/]+).*,\1,p' /etc/wireguard/wghole.conf)
+ # old filename. remove once all hosts are updated.
+ s rm -fv /etc/apache2/sites-enabled/${HOSTNAME}wg.b8.nz.conf
+ web-conf -i -a $wgip -p 9101 -f 9100 - apache2 ${HOSTNAME}wg.b8.nz <<'EOF'
+<Location "/">
+AuthType Basic
+AuthName "basic_auth"
+# created with
+# htpasswd -c prometheus-export-htpasswd USERNAME
+AuthUserFile "/etc/prometheus-export-htpasswd"
+Require valid-user
+</Location>
+EOF
+ # For work, i think we will just use the firewall for hosts in the main data center, and
+ # vpn for hosts outside it.
+
+ # TODO: figure out how to detect the ping failure and try again.
+
+ # Binding to the wg interface, it might go down, so always restart, and wait for it on boot.
+ s mkdir /etc/systemd/system/apache2.service.d
+ sd /etc/systemd/system/apache2.service.d/restart.conf <<EOF
+[Unit]
+After=wg-quick@wghole.service
+StartLimitIntervalSec=0
+
+[Service]
+Restart=always
+RestartSec=30
+EOF
+
+ ;;
+esac
+
+### end prometheus ###
+
+### begin nagios ###
+
+pi nagios-nrpe-server
+
+case $HOSTNAME in
+ kd)
+ # the backport is for this bug: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=800345
+ pi nagios4 nagios-nrpe-plugin monitoring-plugins-basic/bullseye-backports
+ s rm -fv /etc/apache2/conf-enabled/nagios4-cgi.conf
+
+ # to add a password for admin:
+ # htdigest /etc/nagios4/htdigest.users Nagios4 iank
+ # now using the same pass as prometheus
+
+ # nagstamon auth settings, set to digest instead of basic.
+
+ web-conf -p 3005 - apache2 i.b8.nz <<'EOF'
+# adapted from /etc/apache2/conf-enabled/nagios4-cgi.conf
+
+ScriptAlias /cgi-bin/nagios4 /usr/lib/cgi-bin/nagios4
+ScriptAlias /nagios4/cgi-bin /usr/lib/cgi-bin/nagios4
+
+# Where the stylesheets (config files) reside
+Alias /nagios4/stylesheets /etc/nagios4/stylesheets
+
+# Where the HTML pages live
+Alias /nagios4 /usr/share/nagios4/htdocs
+
+<DirectoryMatch (/usr/share/nagios4/htdocs|/usr/lib/cgi-bin/nagios4|/etc/nagios4/stylesheets)>
+ Options FollowSymLinks
+ DirectoryIndex index.php index.html
+ AllowOverride AuthConfig
+ #
+ # The default Debian nagios4 install sets use_authentication=0 in
+ # /etc/nagios4/cgi.cfg, which turns off nagos's internal authentication.
+ # This is insecure. As a compromise this default apache2 configuration
+ # only allows private IP addresses access.
+ #
+ # The <Files>...</Files> below shows how you can secure the nagios4
+ # web site so anybody can view it, but only authenticated users can issue
+ # commands (such as silence notifications). To do that replace the
+ # "Require all granted" with "Require valid-user", and use htdigest
+ # program from the apache2-utils package to add users to
+ # /etc/nagios4/htdigest.users.
+ #
+ # A step up is to insist all users validate themselves by moving
+ # the stanza's in the <Files>..<Files> into the <DirectoryMatch>.
+ # Then by setting use_authentication=1 in /etc/nagios4/cgi.cfg you
+ # can configure which people get to see a particular service from
+ # within the nagios configuration.
+ #
+ AuthDigestDomain "Nagios4"
+ AuthDigestProvider file
+ AuthUserFile "/etc/nagios4-htdigest.users"
+ AuthGroupFile "/etc/group"
+ AuthName "Nagios4"
+ AuthType Digest
+ Require valid-user
+</DirectoryMatch>
+
+<Directory /usr/share/nagios4/htdocs>
+ Options +ExecCGI
+</Directory>
+EOF