+### begin prometheus ###
+
+
+
+pi prometheus-node-exporter-collectors
+case $HOSTNAME in
+ kd)
+ # Font awesome is needed for the alertmanager ui.
+ pi prometheus-alertmanager prometheus fonts-font-awesome
+ /a/bin/buildscripts/prometheus
+ web-conf -p 9091 -f 9090 - apache2 i.b8.nz <<'EOF'
+<Location "/">
+AuthType Basic
+AuthName "basic_auth"
+# created with
+# htpasswd -c prometheus-htpasswd USERNAME
+AuthUserFile "/etc/prometheus-htpasswd"
+Require valid-user
+</Location>
+EOF
+
+ web-conf -p 9094 -f 9093 - apache2 i.b8.nz <<'EOF'
+<Location "/">
+AuthType Basic
+AuthName "basic_auth"
+# created with
+# htpasswd -c prometheus-htpasswd USERNAME
+AuthUserFile "/etc/prometheus-htpasswd"
+Require valid-user
+</Location>
+EOF
+
+ # by default, the alertmanager web ui is not enabled other than a page
+ # that suggests to use the amtool cli. that tool is good, but you cant
+ # silence things nearly as easily as with the gui.
+ if [[ ! -e /usr/share/prometheus/alertmanager/ui/index.html ]]; then
+ # default script didnt work, required some changes to get elm 19.1,
+ # which is a dependency of the latest alertmanager. I modified
+ # and copied it into /b/ds. In future, might need some other
+ # solution.
+ #sudo /usr/share/prometheus/alertmanager/generate-ui.sh
+ sudo /b/ds/generate-ui.sh
+ ser restart prometheus-alertmanager
+ fi
+
+ /a/bin/buildscripts/prom-node-exporter -l
+
+ for ser in prometheus-node-exporter prometheus-alertmanager prometheus; do
+ sysd-prom-fail-install $ser
+ done
+
+ ;;
+ *)
+ /a/bin/buildscripts/prom-node-exporter
+ ;;
+esac
+
+# cleanup old files. 2023-02
+x=(/var/lib/prometheus/node-exporter/*.premerge)
+if [[ -e ${x[0]} ]]; then
+ s rm /var/lib/prometheus/node-exporter/*
+fi
+
+
+case $HOSTNAME in
+ # todo, for limiting node exporter http,
+ # either use iptables or, in
+ # /etc/default/prometheus-node-exporter
+ # listen on the wireguard interface
+
+ *)
+ wgip=$(command sudo sed -rn 's,^ *Address *= *([^/]+).*,\1,p' /etc/wireguard/wghole.conf)
+ # old filename. remove once all hosts are updated.
+ s rm -fv /etc/apache2/sites-enabled/${HOSTNAME}wg.b8.nz.conf
+ web-conf -i -a $wgip -p 9101 -f 9100 - apache2 ${HOSTNAME}wg.b8.nz <<'EOF'
+<Location "/">
+AuthType Basic
+AuthName "basic_auth"
+# created with
+# htpasswd -c prometheus-export-htpasswd USERNAME
+AuthUserFile "/etc/prometheus-export-htpasswd"
+Require valid-user
+</Location>
+EOF
+ # For work, i think we will just use the firewall for hosts in the main data center, and
+ # vpn for hosts outside it.
+
+ # TODO: figure out how to detect the ping failure and try again.
+
+ # Binding to the wg interface, it might go down, so always restart, and wait for it on boot.
+ s mkdir /etc/systemd/system/apache2.service.d
+ sd /etc/systemd/system/apache2.service.d/restart.conf <<EOF
+[Unit]
+After=wg-quick@wghole.service
+StartLimitIntervalSec=0
+
+[Service]
+Restart=always
+RestartSec=30
+EOF
+
+ ;;
+esac
+
+### end prometheus ###
+
+### begin nagios ###
+
+case $HOSTNAME in
+ kd)
+ pi nagios4
+ s rm -fv /etc/apache2/conf-enabled/nagios4-cgi.conf
+
+ # to add a password for admin:
+ # htdigest /etc/nagios4/htdigest.users Nagios4 iank
+ # now using the same pass as prometheus
+
+ # nagstamon auth settings, set to digest instead of basic.
+
+ web-conf -p 3005 - apache2 i.b8.nz <<'EOF'
+# adapted from /etc/apache2/conf-enabled/nagios4-cgi.conf
+
+ScriptAlias /cgi-bin/nagios4 /usr/lib/cgi-bin/nagios4
+ScriptAlias /nagios4/cgi-bin /usr/lib/cgi-bin/nagios4
+
+# Where the stylesheets (config files) reside
+Alias /nagios4/stylesheets /etc/nagios4/stylesheets
+
+# Where the HTML pages live
+Alias /nagios4 /usr/share/nagios4/htdocs
+
+<DirectoryMatch (/usr/share/nagios4/htdocs|/usr/lib/cgi-bin/nagios4|/etc/nagios4/stylesheets)>
+ Options FollowSymLinks
+ DirectoryIndex index.php index.html
+ AllowOverride AuthConfig
+ #
+ # The default Debian nagios4 install sets use_authentication=0 in
+ # /etc/nagios4/cgi.cfg, which turns off nagos's internal authentication.
+ # This is insecure. As a compromise this default apache2 configuration
+ # only allows private IP addresses access.
+ #
+ # The <Files>...</Files> below shows how you can secure the nagios4
+ # web site so anybody can view it, but only authenticated users can issue
+ # commands (such as silence notifications). To do that replace the
+ # "Require all granted" with "Require valid-user", and use htdigest
+ # program from the apache2-utils package to add users to
+ # /etc/nagios4/htdigest.users.
+ #
+ # A step up is to insist all users validate themselves by moving
+ # the stanza's in the <Files>..<Files> into the <DirectoryMatch>.
+ # Then by setting use_authentication=1 in /etc/nagios4/cgi.cfg you
+ # can configure which people get to see a particular service from
+ # within the nagios configuration.
+ #
+ AuthDigestDomain "Nagios4"
+ AuthDigestProvider file
+ AuthUserFile "/etc/nagios4/htdigest.users"
+ AuthGroupFile "/etc/group"
+ AuthName "Nagios4"
+ AuthType Digest
+ Require valid-user
+</DirectoryMatch>
+
+<Directory /usr/share/nagios4/htdocs>
+ Options +ExecCGI
+</Directory>
+EOF
+ ;;
+esac
+
+# when you alter a service through the web, it changes vars in /var/lib/nagios4/status.dat. for example:
+# notifications_enabled=1
+# note, the same variable exists in the correspdonding "define service {"
+
+# in the default config, we have these definitions
+
+# 11 define command {
+# 2 define contact {
+# 1 define contactgroup {
+# 9 define host {
+# 4 define hostgroup {
+# 23 define service {
+# 5 define timeperiod {
+
+
+# on klaxon
+
+# klaxon:/etc/nagios3 # grep -rho '^ *define [^{ ]*' | sort | uniq -c
+# 76 define command
+# 11 define contact
+# 6 define contactgroup
+# 162 define host
+# 1 define hostextinfo
+# 16 define hostgroup
+# 3040 define service
+# 2 define servicedependency
+# 6 define timeperiod
+
+
+### end nagios ###
+
+### begin bitcoin ###
+
+case $HOSTNAME in
+ sy)
+ f=$dir/bitcoin.conf
+ sudo install -m 0755 -o root -g root -t /usr/bin /a/opt/bitcoin-23.0/bin/*
+ sgo bitcoind
+ sudo usermod -a -G bitcoin iank
+ if [[ ! $(readlink -f /var/lib/bitcoind/wallets) == /q/wallets ]]; then
+ sudo lnf /q/wallets /var/lib/bitcoind
+ sudo chown -h bitcoin:bitcoin /var/lib/bitcoind/wallets
+ fi
+ # note, there exists
+ # /a/bin/ds/disabled/bitcoin
+ ;;
+esac
+
+### end bitcoin
+
+case $HOSTNAME in
+ kw|x3)
+ sd /etc/cups/client.conf <<'EOF'
+ServerName printserver0.office.fsf.org
+EOF
+ ;;
+esac
+
+