+ # Font awesome is needed for the alertmanager ui.
+ pi prometheus-alertmanager prometheus fonts-font-awesome
+ /c/roles/prom/files/simple/usr/local/bin/fsf-install-prometheus
+ # make it available for other machines
+ rsync -a /usr/local/bin/amtool /a/opt/bin
+ web-conf -p 9091 -f 9090 - apache2 b8.nz <<'EOF'
+<Location "/">
+AuthType Basic
+AuthName "basic_auth"
+# created with
+# htpasswd -c prometheus-htpasswd USERNAME
+AuthUserFile "/etc/prometheus-htpasswd"
+Require valid-user
+</Location>
+EOF
+
+ web-conf -p 9094 -f 9093 - apache2 b8.nz <<'EOF'
+<Location "/">
+AuthType Basic
+AuthName "basic_auth"
+# created with
+# htpasswd -c prometheus-htpasswd USERNAME
+AuthUserFile "/etc/prometheus-htpasswd"
+Require valid-user
+</Location>
+EOF
+
+ # by default, the alertmanager web ui is not enabled other than a page
+ # that suggests to use the amtool cli. that tool is good, but you cant
+ # silence things nearly as easily as with the gui.
+ if [[ ! -e /usr/share/prometheus/alertmanager/ui/index.html ]]; then
+ # default script didnt work, required some changes to get elm 19.1,
+ # which is a dependency of the latest alertmanager. I modified
+ # and copied it into /b/ds. In future, might need some other
+ # solution.
+ #sudo /usr/share/prometheus/alertmanager/generate-ui.sh
+ sudo /b/ds/generate-ui.sh
+ ser restart prometheus-alertmanager
+ fi
+
+ s /c/roles/prom_export/files/simple/usr/local/bin/fsf-install-node-exporter -l 127.0.0.1
+
+ for ser in prometheus-node-exporter prometheus-alertmanager prometheus; do
+ sysd-prom-fail-install $ser
+ done
+
+ ;;
+ *)
+ s /c/roles/prom_export/files/simple/usr/local/bin/fsf-install-node-exporter
+ ;;
+esac
+
+# cleanup old files. 2023-02
+x=(/var/lib/prometheus/node-exporter/*.premerge)
+if [[ -e ${x[0]} ]]; then
+ s rm /var/lib/prometheus/node-exporter/*
+fi
+
+
+case $HOSTNAME in
+ # todo, for limiting node exporter http,
+ # either use iptables or, in
+ # /etc/default/prometheus-node-exporter
+ # listen on the wireguard interface
+
+ *)
+ if [[ -e /etc/wireguard/wghole.conf ]]; then
+ wgip=$(command sudo sed -rn 's,^ *Address *= *([^/]+).*,\1,p' /etc/wireguard/wghole.conf)
+ # old filename. remove once all hosts are updated.
+ s rm -fv /etc/apache2/sites-enabled/${HOSTNAME}wg.b8.nz.conf
+ web-conf -i -a $wgip -p 9101 -f 9100 - apache2 ${HOSTNAME}wg.b8.nz <<'EOF'
+<Location "/">
+AuthType Basic
+AuthName "basic_auth"
+# created with
+# htpasswd -c prometheus-export-htpasswd USERNAME
+AuthUserFile "/etc/prometheus-export-htpasswd"
+Require valid-user
+</Location>
+EOF
+ # For work, i think we will just use the firewall for hosts in the main data center, and
+ # vpn for hosts outside it.
+
+ # TODO: figure out how to detect the ping failure and try again.
+
+ # Binding to the wg interface, it might go down, so always restart, and wait for it on boot.
+ s mkdir /etc/systemd/system/apache2.service.d
+ sd /etc/systemd/system/apache2.service.d/restart.conf <<EOF
+[Unit]
+After=wg-quick@wghole.service
+StartLimitIntervalSec=0
+
+[Service]
+Restart=always
+RestartSec=30
+EOF
+
+ fi
+ ;;
+esac
+
+### end prometheus ###
+
+### begin bitcoin ###
+
+case $HOSTNAME in
+ sy|kd|so)
+ sudo install -m 0755 -o root -g root -t /usr/bin /a/opt/bitcoin-27.0/bin/*
+ # Note: i leave it to system-status to start and stop bitcoin.
+ # note: the bitcoin user & group are setup in fai
+ sudo usermod -a -G bitcoin iank
+ # todo: make bitcoin have a stable uid/gid
+ sudo mkdir -p /var/lib/bitcoind
+ sudo chown bitcoin:bitcoin /var/lib/bitcoind
+ # 710 comes from the upstream bitcoin unit file
+ sudo chmod 710 /var/lib/bitcoind
+ # note, there exists
+ # /a/bin/ds/disabled/bitcoin
+ ;;
+esac
+
+### end bitcoin
+
+### begin live streaming ###
+
+# comparing nginx rtmp to icecast
+# `mpv --cache=no` had about 2.5 sec latency vs 4 seconds.
+# Then I discovered this command which had about .5 sec latency:
+#ffplay -f live_flv -fast -x 1280 -y 720 -fflags nobuffer -flags low_delay -strict experimental -vf "setpts=N/60/TB" -af "asetpts=N/60/TB" -noframedrop -i rtmp://url_here
+## a lot of those args arent needed, here is what I ended up with:
+# #ffplay -f live_flv -fflags nobuffer -flags low_delay -i rtmp://localhost/live
+#
+# A problem with rtmp is that it doesn't support vp8/vp9, requiring the partly patent encumbered h264.
+# Looking at alternative protocols: dash & hls are both high latency, I tested dash with the nginx-rtmp
+# module and got about 5 seconds of latency, web results imply that is normal.
+#
+# Webrtc is what jitsi & bbb use, but an annoying thing is that
+# generally requires a web browser with javascript, or some special
+# client, and afaik, it has a smaller limit on number of clients.
+#
+# Another option is to try rtp/rtsp, there are some servers here:
+# https://en.wikipedia.org/wiki/Real-Time_Streaming_Protocol
+
+
+## reference for setting up rtmp
+# pi nginx libnginx-mod-rtmp
+# cat >/etc/nginx/modules-enabled/rtmp.conf <<'EOF'
+## based on https://opensource.com/article/19/1/basic-live-video-streaming-server#comments
+## and https://github.com/arut/nginx-rtmp-module/wiki/Directives
+
+# rtmp {
+# allow publish 127.0.0.1;
+# deny publish all;
+# server {
+# listen 1935;
+# application live {
+# live on;
+# record off;
+# }
+# }
+# }
+# EOF
+
+### end live streaming ###
+
+### begin gh ####
+
+# from https://raw.githubusercontent.com/cli/cli/trunk/docs/install_linux.md
+# One time setup afterwards:
+# gh auth login
+#
+# When it gets to the page where it asks to authorize github, the button
+# is grayed out. You can just open browser dev tools, inspect the
+# button, remove disabled="", then click it and it works.
+#
+# Auth token gets saved into /p/c/subdir_files/.local/share/keyrings/
+#
+# initial config goes to /home/iank/.config/gh
+curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg \
+ && sudo chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg \
+ && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \
+ && sudo apt update \
+ && sudo apt install gh -y
+
+### end gh ####
+
+
+# remove trisquel banner. it is cool but takes up too much space.
+sudo rm -f /etc/update-motd.d/01-banner
+
+case $HOSTNAME in
+ kw|x3)
+ sd /etc/cups/client.conf <<'EOF'
+ServerName printserver1.office.fsf.org
+EOF