Prometheus Node Exporter

Basic system stats, to show stuff the synapse metrics
can't show such as resource usage by bridges, etc

Seems to work fine as well.

This too has only been tested on debian amd64 so far

group_vars/matrix_servers

@@ -1368,6 +1368,27 @@ matrix_synapse_admin_container_self_build: "{{ matrix_architecture != 'amd64' }}
 
 
 
+######################################################################
+#
+# matrix-prometheus-node-exporter
+#
+######################################################################
+
+matrix_prometheus_node_exporter_enabled: false
+
+# Normally, matrix-nginx-proxy is enabled and nginx can reach Prometheus Node Exporter over the container network.
+# If matrix-nginx-proxy is not enabled, or you otherwise have a need for it, you can expose
+# Prometheus' HTTP port to the local host.
+matrix_prometheus_node_exporter_container_http_host_bind_port: "{{ '' if matrix_nginx_proxy_enabled else '127.0.0.1:9100' }}"
+
+######################################################################
+#
+# /matrix-prometheus-node-exporter
+#
+######################################################################
+
+
+
 ######################################################################
 #
 # matrix-prometheus

roles/matrix-prometheus-node-exporter/defaults/main.yml

@@ -0,0 +1,21 @@
+# matrix-prometheus-node-exporter is an Prometheus exporter for machine metrics
+# See: https://prometheus.io/docs/guides/node-exporter/
+
+matrix_prometheus_node_exporter_enabled: false
+
+matrix_prometheus_node_exporter_docker_image: "docker.io/prom/node-exporter:v1.0.1"
+matrix_prometheus_node_exporter_docker_image_force_pull: "{{ matrix_prometheus_node_exporter_docker_image.endswith(':latest') }}"
+
+# A list of extra arguments to pass to the container
+matrix_prometheus_node_exporter_container_extra_arguments: []
+
+# List of systemd services that matrix-prometheus.service depends on
+matrix_prometheus_node_exporter_systemd_required_services_list: ['docker.service']
+
+# List of systemd services that matrix-prometheus.service wants
+matrix_prometheus_node_exporter_systemd_wanted_services_list: []
+
+# Controls whether the matrix-prometheus container exposes its HTTP port (tcp/9100 in the container).
+#
+# Takes an "<ip>:<port>" or "<port>" value (e.g. "127.0.0.1:9100"), or empty string to not expose.
+matrix_prometheus_node_exporter_container_http_host_bind_port: ''

roles/matrix-prometheus-node-exporter/tasks/init.yml

@@ -0,0 +1,5 @@
+- set_fact:
+    matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-prometheus-node-exporter.service'] }}"
+  when: matrix_prometheus_node_exporter_enabled|bool
+
+

roles/matrix-prometheus-node-exporter/tasks/main.yml

@@ -0,0 +1,8 @@
+- import_tasks: "{{ role_path }}/tasks/init.yml"
+  tags:
+    - always
+
+- import_tasks: "{{ role_path }}/tasks/setup.yml"
+  tags:
+    - setup-all
+    - setup-prometheus-node-exporter

roles/matrix-prometheus-node-exporter/tasks/setup.yml

@@ -0,0 +1,60 @@
+---
+
+#
+# Tasks related to setting up matrix-prometheus-node-exporter
+#
+
+- name: Ensure matrix-prometheus-node-exporter image is pulled
+  docker_image:
+    name: "{{ matrix_prometheus_node_exporter_docker_image }}"
+    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
+    force_source: "{{ matrix_prometheus_node_exporter_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
+    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_prometheus_node_exporter_docker_image_force_pull }}"
+  when: "matrix_prometheus_node_exporter_enabled|bool"
+
+- name: Ensure matrix-prometheus-node-exporter.service installed
+  template:
+    src: "{{ role_path }}/templates/systemd/matrix-prometheus-node-exporter.service.j2"
+    dest: "{{ matrix_systemd_path }}/matrix-prometheus-node-exporter.service"
+    mode: 0644
+  register: matrix_prometheus_node_exporter_systemd_service_result
+  when: matrix_prometheus_node_exporter_enabled|bool
+
+- name: Ensure systemd reloaded after matrix-prometheus.service installation
+  service:
+    daemon_reload: yes
+  when: "matrix_prometheus_node_exporter_enabled|bool and matrix_prometheus_node_exporter_systemd_service_result.changed"
+
+#
+# Tasks related to getting rid of matrix-prometheus-node-exporter (if it was previously enabled)
+#
+
+- name: Check existence of matrix-prometheus-node-exporter service
+  stat:
+    path: "{{ matrix_systemd_path }}/matrix-prometheus-node-exporter.service"
+  register: matrix_prometheus_node_exporter_service_stat
+
+- name: Ensure matrix-prometheus-node-exporter is stopped
+  service:
+    name: matrix-prometheus-node-exporter
+    state: stopped
+    daemon_reload: yes
+  register: stopping_result
+  when: "not matrix_prometheus_node_exporter_enabled|bool and matrix_prometheus_node_exporter_service_stat.stat.exists"
+
+- name: Ensure matrix-prometheus-node-exporter.service doesn't exist
+  file:
+    path: "{{ matrix_systemd_path }}/matrix-prometheus-node-exporter.service"
+    state: absent
+  when: "not matrix_prometheus_node_exporter_enabled|bool and matrix_prometheus_node_exporter_service_stat.stat.exists"
+
+- name: Ensure systemd reloaded after matrix-prometheus-node-exporter.service removal
+  service:
+    daemon_reload: yes
+  when: "not matrix_prometheus_node_exporter_enabled|bool and matrix_prometheus_node_exporter_service_stat.stat.exists"
+
+- name: Ensure matrix-prometheus-node-exporter Docker image doesn't exist
+  docker_image:
+    name: "{{ matrix_prometheus_node_exporter_docker_image }}"
+    state: absent
+  when: "not matrix_prometheus_node_exporter_enabled|bool"

roles/matrix-prometheus-node-exporter/tasks/validate_config.yml

@@ -0,0 +1,7 @@
+---
+
+- name: Fail if Synapse metrics or Prometheus Node Exporter not enabled
+  fail:
+    msg: >
+      You need to enable `matrix_synapse_metrics_enabled` and/or `matrix_prometheus_node_exporter_enabled` for Prometheus grab metrics.
+  when: "not matrix_synapse_metrics_enabled and not matrix_prometheus_node_exporter_enabled"

roles/matrix-prometheus-node-exporter/templates/systemd/matrix-prometheus-node-exporter.service.j2

@@ -0,0 +1,40 @@
+#jinja2: lstrip_blocks: "True"
+[Unit]
+Description=matrix-prometheus-node-exporter
+{% for service in matrix_prometheus_node_exporter_systemd_required_services_list %}
+Requires={{ service }}
+After={{ service }}
+{% endfor %}
+{% for service in matrix_prometheus_node_exporter_systemd_wanted_services_list %}
+Wants={{ service }}
+{% endfor %}
+DefaultDependencies=no
+
+[Service]
+Type=simple
+Environment="HOME={{ matrix_systemd_unit_home_path }}"
+ExecStartPre=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} kill matrix-prometheus-node-exporter 2>/dev/null'
+ExecStartPre=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} rm matrix-prometheus-node-exporter 2>/dev/null'
+
+
+ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-prometheus-node-exporter \
+			--log-driver=none \
+			--user={{ matrix_user_uid }}:{{ matrix_user_gid }} \
+			--cap-drop=ALL \
+			--network={{ matrix_docker_network }} \
+			{% if matrix_prometheus_node_exporter_container_http_host_bind_port %}
+			-p {{ matrix_prometheus_node_exporter_container_http_host_bind_port }}:9100 \
+			{% endif %}
+			{% for arg in matrix_prometheus_node_exporter_container_extra_arguments %}
+			{{ arg }} \
+			{% endfor %}
+			{{ matrix_prometheus_node_exporter_docker_image }}
+
+ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} kill matrix-prometheus-node-exporter 2>/dev/null'
+ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} rm matrix-prometheus-node-exporter 2>/dev/null'
+Restart=always
+RestartSec=30
+SyslogIdentifier=matrix-prometheus-node-exporter
+
+[Install]
+WantedBy=multi-user.target

roles/matrix-prometheus/tasks/validate_config.yml

@@ -1,7 +1,7 @@
 ---
 
-- name: Fail if Synapse metrics not enabled
+- name: Fail if Synapse metrics or Prometheus Node Exporter not enabled
   fail:
     msg: >
-      You need to enable `matrix_synapse_metrics_enabled` for Prometheus grab metrics.
+      You need to enable `matrix_synapse_metrics_enabled` and/or `matrix_prometheus_node_exporter_enabled` for Prometheus grab metrics.
-  when: "not matrix_synapse_metrics_enabled"
+  when: "not matrix_synapse_metrics_enabled and not matrix_prometheus_node_exporter_enabled"

roles/matrix-prometheus/templates/prometheus.yml.j2

@@ -33,3 +33,8 @@ scrape_configs:
     - targets: ['matrix-synapse:{{ matrix_synapse_metrics_port }}']
   {% endif %}
 
+  {% if matrix_prometheus_node_exporter_enabled %}
+  - job_name: node
+    static_configs:
+      - targets: ['matrix-prometheus-node-exporter:9100']
+  {% endif %}

setup.yml

@@ -28,6 +28,7 @@
     - matrix-bot-matrix-reminder-bot
     - matrix-synapse
     - matrix-synapse-admin
+    - matrix-prometheus-node-exporter
     - matrix-prometheus
     - matrix-registration
     - matrix-client-element