add prometheus and friends

diff --git a/.gitignore b/.gitignore
index 08948069ca451f14bcbf61447cbd9939cbed5d12..d70eb01db9aa033a4e2de68343d6d7ba51389d2d 100644 (file)
--- a/.gitignore
+++ b/.gitignore
@@ -2,4 +2,8 @@
 /old-unused
 /.emacs.d
 /Arduino
+# 3rd party git repos
 /a/roles/prom
+/a/roles/grafana
+/a/roles/node-exporter
+/a/roles/alertmanager

diff --git a/a/ansible.cfg b/a/ansible.cfg
index 946488b87043f770b967af0868b57f22ffaaab97..83781c3c0e95c04b208753febcb65fd66dd15312 100644 (file)
--- a/a/ansible.cfg
+++ b/a/ansible.cfg
@@ -3,7 +3,14 @@ forks          = 200
 host_key_checking = False
 display_skipped_hosts = False
 retry_files_enabled = False
+# readable output
+stdout_callback = yaml
+callback_whitelist = timer, profile_tasks
 
 [ssh_connection]
 pipelining = True
 retries = 2
+
+[colors]
+# found in color.py. default blue is hard to read on a black background
+verbose = bright blue

diff --git a/a/group_vars/all b/a/group_vars/all
new file mode 100644 (file)
index 0000000..b74039d
--- /dev/null
+++ b/a/group_vars/all
@@ -0,0 +1,2 @@
+# iank: 1.1 because prometheus is configered elsewhere to use fqdn, which maps
+prometheus_web_listen_address: "127.0.1.1:9090"

diff --git a/a/site.yml b/a/site.yml
index 75da6234c72cad9fc757a7f268dcbacd6bb52eab..63b6cd1d5841eea70b9431c1beae1a8db3bd5ee0 100644 (file)
--- a/a/site.yml
+++ b/a/site.yml
@@ -1,4 +1,64 @@
 ---
 - hosts: tp.b8.nz
   roles:
-    - prom
+    - role: prom
+      prometheus_scrape_configs:
+        - job_name: "prometheus"
+          metrics_path: "{{ prometheus_metrics_path }}"
+          static_configs:
+            - targets:
+                - "{{ ansible_fqdn | default(ansible_host) | default('localhost') }}:9090"
+        - job_name: "node"
+          basic_auth:
+            username: prom
+            password_file: /etc/prometheus-pass
+          scheme: "https"
+          file_sd_configs:
+            - files:
+                - "{{ prometheus_config_dir }}/file_sd/node.yml"
+
+      prometheus_targets:
+        node:
+          - targets: "{{ groups.all|map('regex_replace','$',':9101')|list }}"
+
+    - role: node-exporter
+      node_exporter_web_listen_address: "127.0.0.1:9100"
+    - role: alertmanager
+      alertmanager_smtp:
+        smarthost: 'localhost:25'
+        from: "alertmanager@{{ ansible_fqdn | default(ansible_host) | default('localhost') }}"
+        require_tls: false
+      alertmanager_route:
+        receiver: defaultreceiver
+      alertmanager_receivers:
+        - name: defaultreceiver
+          email_configs:
+            - to: alerts@iankelling.org
+              html: "{% raw -%}{{ template \"email.default.html\" . }}{% endraw -%}"
+              text: "{% raw -%}{{ template \"email.default.text\" . }}{% endraw -%}"
+
+      alertmanager_listen_address: '127.0.0.1:9093'
+    - role: grafana
+      grafana_address: "127.0.0.1"
+      # iank: playbook will halt if no password is set. this is only
+      # available to localhost, so i dont really care, but might as well
+      # generate a pass isntead of putting in pw123 etc.
+      grafana_security: { admin_user: admin, admin_password: spheroid.recantation.shank }
+      grafana_datasources:
+        - name: prometheus
+          type: prometheus
+          access: proxy
+          url: 'http://{{ prometheus_web_listen_address }}'
+          basicAuth: false
+          isDefault: true
+      # This is based on looking at highly downloaded dashboards here
+      # https://grafana.com/dashboards?dataSource=prometheus&collector=nodeExporter
+      # Which is where you are lead to from
+      # https://prometheus.io/docs/visualization/grafana/
+      grafana_dashboards:
+        - dashboard_id: 1860
+          revision_id: 13
+          datasource: prometheus
+        - dashboard_id: 405
+          revision_id: 8
+          datasource: prometheus

diff --git a/brc b/brc
index ed1f51d7e59f55d34d48f1d8a3218e2b4ea190bd..5ecfd817aa6d0a8072562a5e4b31f191d26a8e57 100644 (file)
--- a/brc
+++ b/brc
@@ -331,14 +331,19 @@ anki() {
 
 ap() {
   # pushd in case current directory has an ansible.cfg file
-  pushd /a/xans
+  pushd /a/xans >/dev/null
   ansible-playbook -v -l ${1:- $(hostname -f)} site.yml
-  popd
+  popd >/dev/null
 }
 aw() {
-  pushd /a/work/ansible-configs
+  pushd /a/work/ansible-configs >/dev/null
   time ansible-playbook -v -i inventory adhoc.yml "$@"
-  popd
+  popd >/dev/null
+}
+ad() {
+  pushd /a/bin/distro-setup/a >/dev/null
+  ansible-playbook site.yml
+  popd >/dev/null
 }
 
 astudio() {

diff --git a/conflink b/conflink
index 0010c9d6f347e04691120d02e2cdbfcf0f3a3ed9..b24427fe40737ef5cafff1f96895cc94e191873e 100755 (executable)
--- a/conflink
+++ b/conflink
@@ -94,6 +94,13 @@ case $USER in
       s chown -R znc:znc /var/lib/znc
     fi
     /a/exe/lnf -T /p/arbtt-capture.log ~/.arbtt/capture.log
+    f=/etc/prometheus-htpasswd
+    if [[ -e $f ]]; then
+      s chmod 640 $f /etc/prometheus-pass
+      s chown root:www-data $f
+      s chown root:prometheus /etc/prometheus-pass
+    fi
+
     ##### end special extra stuff #####
 
     sudo bash -c 'shopt -s nullglob; cd /etc/openvpn; for f in client/* server/*; do ln -sf $f .; done'

diff --git a/distro-end b/distro-end
index 1b25a55eced2a26e3dd6e5a74903af810ca14adf..f2df46e6770aa47f79cae78dd5903442f5aa26c0 100755 (executable)
--- a/distro-end
+++ b/distro-end
@@ -180,6 +180,22 @@ sgo certbotmail.timer
 
 pi ${p1[@]}
 
+
+## prometheus node exporter setup
+web-conf -f 9100 -p 9101 apache2 $(hostname -f) <<'EOF'
+#https://httpd.apache.org/docs/2.4/mod/mod_authn_core.html#authtype
+# https://stackoverflow.com/questions/5011102/apache-reverse-proxy-with-basic-authentication
+<Location />
+   AllowOverride None
+   AuthType basic
+   AuthName "Authentication Required"
+   # setup one time, with root:www-data, 640
+   AuthUserFile "/etc/prometheus-htpasswd"
+   Require valid-user
+</Location>
+EOF
+
+
 # website setup
 case $HOSTNAME in
   lj|li)