From f9ccde727d3471c99b909a528aa7168281368084 Mon Sep 17 00:00:00 2001
From: Ian Kelling <iank@fsf.org>
Date: Sat, 1 Jun 2019 18:02:30 -0400
Subject: [PATCH] add prometheus and friends

---
 .gitignore       |  4 ++++
 a/ansible.cfg    |  7 ++++++
 a/group_vars/all |  2 ++
 a/site.yml       | 62 +++++++++++++++++++++++++++++++++++++++++++++++-
 brc              | 13 ++++++----
 conflink         |  7 ++++++
 distro-end       | 16 +++++++++++++
 7 files changed, 106 insertions(+), 5 deletions(-)
 create mode 100644 a/group_vars/all

diff --git a/.gitignore b/.gitignore
index 0894806..d70eb01 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,4 +2,8 @@
 /old-unused
 /.emacs.d
 /Arduino
+# 3rd party git repos
 /a/roles/prom
+/a/roles/grafana
+/a/roles/node-exporter
+/a/roles/alertmanager
diff --git a/a/ansible.cfg b/a/ansible.cfg
index 946488b..83781c3 100644
--- a/a/ansible.cfg
+++ b/a/ansible.cfg
@@ -3,7 +3,14 @@ forks          = 200
 host_key_checking = False
 display_skipped_hosts = False
 retry_files_enabled = False
+# readable output
+stdout_callback = yaml
+callback_whitelist = timer, profile_tasks
 
 [ssh_connection]
 pipelining = True
 retries = 2
+
+[colors]
+# found in color.py. default blue is hard to read on a black background
+verbose = bright blue
diff --git a/a/group_vars/all b/a/group_vars/all
new file mode 100644
index 0000000..b74039d
--- /dev/null
+++ b/a/group_vars/all
@@ -0,0 +1,2 @@
+# iank: 1.1 because prometheus is configered elsewhere to use fqdn, which maps
+prometheus_web_listen_address: "127.0.1.1:9090"
diff --git a/a/site.yml b/a/site.yml
index 75da623..63b6cd1 100644
--- a/a/site.yml
+++ b/a/site.yml
@@ -1,4 +1,64 @@
 ---
 - hosts: tp.b8.nz
   roles:
-    - prom
+    - role: prom
+      prometheus_scrape_configs:
+        - job_name: "prometheus"
+          metrics_path: "{{ prometheus_metrics_path }}"
+          static_configs:
+            - targets:
+                - "{{ ansible_fqdn | default(ansible_host) | default('localhost') }}:9090"
+        - job_name: "node"
+          basic_auth:
+            username: prom
+            password_file: /etc/prometheus-pass
+          scheme: "https"
+          file_sd_configs:
+            - files:
+                - "{{ prometheus_config_dir }}/file_sd/node.yml"
+
+      prometheus_targets:
+        node:
+          - targets: "{{ groups.all|map('regex_replace','$',':9101')|list }}"
+
+    - role: node-exporter
+      node_exporter_web_listen_address: "127.0.0.1:9100"
+    - role: alertmanager
+      alertmanager_smtp:
+        smarthost: 'localhost:25'
+        from: "alertmanager@{{ ansible_fqdn | default(ansible_host) | default('localhost') }}"
+        require_tls: false
+      alertmanager_route:
+        receiver: defaultreceiver
+      alertmanager_receivers:
+        - name: defaultreceiver
+          email_configs:
+            - to: alerts@iankelling.org
+              html: "{% raw -%}{{ template \"email.default.html\" . }}{% endraw -%}"
+              text: "{% raw -%}{{ template \"email.default.text\" . }}{% endraw -%}"
+
+      alertmanager_listen_address: '127.0.0.1:9093'
+    - role: grafana
+      grafana_address: "127.0.0.1"
+      # iank: playbook will halt if no password is set. this is only
+      # available to localhost, so i dont really care, but might as well
+      # generate a pass isntead of putting in pw123 etc.
+      grafana_security: { admin_user: admin, admin_password: spheroid.recantation.shank }
+      grafana_datasources:
+        - name: prometheus
+          type: prometheus
+          access: proxy
+          url: 'http://{{ prometheus_web_listen_address }}'
+          basicAuth: false
+          isDefault: true
+      # This is based on looking at highly downloaded dashboards here
+      # https://grafana.com/dashboards?dataSource=prometheus&collector=nodeExporter
+      # Which is where you are lead to from
+      # https://prometheus.io/docs/visualization/grafana/
+      grafana_dashboards:
+        - dashboard_id: 1860
+          revision_id: 13
+          datasource: prometheus
+        - dashboard_id: 405
+          revision_id: 8
+          datasource: prometheus
diff --git a/brc b/brc
index ed1f51d..5ecfd81 100644
--- a/brc
+++ b/brc
@@ -331,14 +331,19 @@ anki() {
 
 ap() {
   # pushd in case current directory has an ansible.cfg file
-  pushd /a/xans
+  pushd /a/xans >/dev/null
   ansible-playbook -v -l ${1:- $(hostname -f)} site.yml
-  popd
+  popd >/dev/null
 }
 aw() {
-  pushd /a/work/ansible-configs
+  pushd /a/work/ansible-configs >/dev/null
   time ansible-playbook -v -i inventory adhoc.yml "$@"
-  popd
+  popd >/dev/null
+}
+ad() {
+  pushd /a/bin/distro-setup/a >/dev/null
+  ansible-playbook site.yml
+  popd >/dev/null
 }
 
 astudio() {
diff --git a/conflink b/conflink
index 0010c9d..b24427f 100755
--- a/conflink
+++ b/conflink
@@ -94,6 +94,13 @@ case $USER in
       s chown -R znc:znc /var/lib/znc
     fi
     /a/exe/lnf -T /p/arbtt-capture.log ~/.arbtt/capture.log
+    f=/etc/prometheus-htpasswd
+    if [[ -e $f ]]; then
+      s chmod 640 $f /etc/prometheus-pass
+      s chown root:www-data $f
+      s chown root:prometheus /etc/prometheus-pass
+    fi
+
     ##### end special extra stuff #####
 
     sudo bash -c 'shopt -s nullglob; cd /etc/openvpn; for f in client/* server/*; do ln -sf $f .; done'
diff --git a/distro-end b/distro-end
index 1b25a55..f2df46e 100755
--- a/distro-end
+++ b/distro-end
@@ -180,6 +180,22 @@ sgo certbotmail.timer
 
 pi ${p1[@]}
 
+
+## prometheus node exporter setup
+web-conf -f 9100 -p 9101 apache2 $(hostname -f) <<'EOF'
+#https://httpd.apache.org/docs/2.4/mod/mod_authn_core.html#authtype
+# https://stackoverflow.com/questions/5011102/apache-reverse-proxy-with-basic-authentication
+<Location />
+   AllowOverride None
+   AuthType basic
+   AuthName "Authentication Required"
+   # setup one time, with root:www-data, 640
+   AuthUserFile "/etc/prometheus-htpasswd"
+   Require valid-user
+</Location>
+EOF
+
+
 # website setup
 case $HOSTNAME in
   lj|li)
-- 
2.30.2