fixes, prometheus, lots of stuff

[distro-setup] / distro-end

diff --git a/distro-end b/distro-end
index 407987ad8c34852078ce352dd803d12fa589250f..a5ba321b9812b4e72771880f232a11de6a2d215a 100755 (executable)
--- a/distro-end
+++ b/distro-end
@@ -523,7 +523,13 @@ Package: chromium-*
 Pin: release n=bionic
 Pin-Priority: 500
 EOF
-
+    ;;
+  nabia)
+    sd /etc/apt/preferences.d/chromium-bullseye <<EOF
+Package: chromium chromium-* libicu67 libjpeg62-turbo libjsoncpp24 libre2-9 libwebpmux3
+Pin: release o=Debian*,n=bullseye*
+Pin-Priority: 500
+EOF
     ;;
 esac
 
@@ -598,7 +604,22 @@ case $HOSTNAME in
     if [[ ! -e $f ]]; then
       dnsb8
     fi
+
+    pi prometheus-node-exporter
+
+    # ex for exporter
+    web-conf -p 9101 -f 9100 - apache2 ${HOSTNAME}ex.b8.nz <<'EOF'
+<Location "/">
+AuthType Basic
+AuthName "basic_auth"
+# created with
+# htpasswd -c prometheus-export-htpasswd USERNAME
+AuthUserFile "/etc/prometheus-export-htpasswd"
+Require valid-user
+</Location>
+EOF
     ;;&
+
   bk)
     sgo wg-quick@wgmail
 
@@ -1656,7 +1677,7 @@ m /a/bin/buildscripts/rust
 m /a/bin/buildscripts/misc
 m /a/bin/buildscripts/pithosfly
 #m /a/bin/buildscripts/alacritty
-m /a/bin/buildscripts/kitty
+#m /a/bin/buildscripts/kitty
 
 pi-nostart virtinst virt-manager
 soff libvirtd
@@ -1832,7 +1853,10 @@ sudo debconf-set-selections <<EOF
 grub-pc grub-pc/install_devices multiselect ${devs[*]}
 EOF
 
-# btrfs maintenance
+
+sysd-prom-fail-install dynamicipupdate
+sysd-prom-fail-install systemstatus
+sysd-prom-fail-install btrfsmaintstop
 sgo btrfsmaint.timer
 sgo btrfsmaintstop
 sgo systemstatus
@@ -1857,7 +1881,8 @@ case $HOSTNAME in
   kd)
     # ive got these + a needed dependency pinned to bullseye, just to get
     # versions more in line with the main docs.
-    pi prometheus-alertmanager prometheus prometheus-node-exporter
+    # Font awesome is needed for the alertmanager ui.
+    pi prometheus-alertmanager prometheus prometheus-node-exporter fonts-font-awesome
     web-conf -p 9091 -f 9090 - apache2 i.b8.nz <<'EOF'
 <Location "/">
 AuthType Basic
@@ -1868,6 +1893,20 @@ AuthUserFile "/etc/prometheus-htpasswd"
 Require valid-user
 </Location>
 EOF
+    # by default, the alertmanager web ui is not enabled other than a page
+    # that suggests to use the amtool cli. that tool is good, but you cant
+    # silence things nearly as fast.
+    if [[ ! -e /usr/share/prometheus/alertmanager/ui/index.html ]]; then
+      sudo chroot /nocow/schroot/bullseye prometheus-alertmanager
+      sudo chroot /nocow/schroot/bullseye /usr/share/prometheus/alertmanager/generate-ui.sh
+      sudo rsync -avih /nocow/schroot/bullseye/usr/share/prometheus/alertmanager/ui/ /usr/share/prometheus/alertmanager/ui
+      ser restart prometheus-alertmanager
+    fi
+
+    for ser in prometheus-node-exporter prometheus-alertmanager prometheus; do
+      sysd-prom-fail-install $ser
+    done
+
     ;;
   *)
     pi prometheus-node-exporter
@@ -1881,21 +1920,10 @@ case $HOSTNAME in
   # either use iptables or, in
   # /etc/default/prometheus-node-exporter
   # listen on the wireguard interface
-  li|je|bk)
-    # ex for exporter
-    web-conf -p 9101 -f 9100 - apache2 ${HOSTNAME}ex.b8.nz <<'EOF'
-<Location "/">
-AuthType Basic
-AuthName "basic_auth"
-# created with
-# htpasswd -c prometheus-export-htpasswd USERNAME
-AuthUserFile "/etc/prometheus-export-htpasswd"
-Require valid-user
-</Location>
-EOF
-    ;;
   *)
     wgip=$(command sudo sed -rn 's,^ *Address *= *([^/]+).*,\1,p' /etc/wireguard/wghole.conf)
+    # old filename. remove once all hosts are updated.
+    s rm -fv /etc/apache2/sites-enabled/${HOSTNAME}wg.b8.nz.conf
     web-conf -i -a $wgip -p 9101 -f 9100 - apache2 ${HOSTNAME}wg.b8.nz <<'EOF'
 <Location "/">
 AuthType Basic
@@ -1906,7 +1934,25 @@ AuthUserFile "/etc/prometheus-export-htpasswd"
 Require valid-user
 </Location>
 EOF
-  ;;
+    # For work, i think we will just use the firewall for hosts in the main data center, and
+    # apache/nginx + tls + basic auth outside of it. or consider stunnel.
+
+
+    # TODO: figure out how to detect the ping failure and try again.
+
+    # Binding to the wg interface, it might go down, so always restart, and wait for it on boot.
+    s mkdir /etc/systemd/system/apache2.service.d
+    sd /etc/systemd/system/apache2.service.d/restart.conf <<EOF
+[Unit]
+After=wg-quick@wghole.service
+StartLimitIntervalSec=0
+
+[Service]
+Restart=always
+RestartSec=30
+EOF
+
+    ;;
 esac
 
 ### end prometheus ###