Merge pull request #838 from Peetz0r/stats

Prometheus and Grafana on stats.<domain>

roles/matrix-base/defaults/main.yml

@@ -21,6 +21,9 @@ matrix_server_fqn_dimension: "dimension.{{ matrix_domain }}"
 # This is where you access Jitsi.
 matrix_server_fqn_jitsi: "jitsi.{{ matrix_domain }}"
 
+# This is where you access Grafana.
+matrix_server_fqn_grafana: "stats.{{ matrix_domain }}"
+
 matrix_federation_public_port: 8448
 
 # The architecture that your server runs.

roles/matrix-grafana/defaults/main.yml

@@ -0,0 +1,47 @@
+# matrix-grafana is open source visualization and analytics software
+# See: https://github.com/matrix-org/synapse/blob/master/docs/metrics-howto.md
+
+matrix_grafana_enabled: false
+
+matrix_grafana_docker_image: "docker.io/grafana/grafana:7.4.0"
+matrix_grafana_docker_image_force_pull: "{{ matrix_grafana_docker_image.endswith(':latest') }}"
+
+# Not conditional, because when someone disables metrics
+# they might still want to look at the old existing data.
+# So it would be silly to delete the dashboard in such case.
+matrix_grafana_dashboard_download_urls:
+- "https://raw.githubusercontent.com/matrix-org/synapse/master/contrib/grafana/synapse.json"
+- "https://raw.githubusercontent.com/rfrail3/grafana-dashboards/master/prometheus/node-exporter-full.json"
+
+matrix_grafana_base_path: "{{ matrix_base_data_path }}/grafana"
+matrix_grafana_config_path: "{{ matrix_grafana_base_path }}/config"
+matrix_grafana_data_path: "{{ matrix_grafana_base_path }}/data"
+
+# Allow viewing Grafana without logging in
+matrix_grafana_anonymous_access: false
+
+# specify organization name that should be used for unauthenticated users
+# if you change this in the Grafana admin panel, this needs to be updated
+# to match to keep anonymous logins working
+matrix_grafana_anonymous_access_org_name: 'Main Org.'
+
+
+# default admin credentials, you are asked to change these on first login
+matrix_grafana_default_admin_user: admin
+matrix_grafana_default_admin_password: admin
+
+# A list of extra arguments to pass to the container
+matrix_grafana_container_extra_arguments: []
+
+# List of systemd services that matrix-grafana.service depends on
+matrix_grafana_systemd_required_services_list: ['docker.service']
+
+# List of systemd services that matrix-grafana.service wants
+matrix_grafana_systemd_wanted_services_list: []
+
+# Controls whether the matrix-grafana container exposes its HTTP port (tcp/3000 in the container).
+#
+# Takes an "<ip>:<port>" or "<port>" value (e.g. "127.0.0.1:3000"), or empty string to not expose.
+matrix_grafana_container_http_host_bind_port: ''
+
+

roles/matrix-grafana/tasks/init.yml

@@ -0,0 +1,5 @@
+- set_fact:
+    matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-grafana.service'] }}"
+  when: matrix_grafana_enabled|bool
+
+

roles/matrix-grafana/tasks/main.yml

@@ -0,0 +1,14 @@
+- import_tasks: "{{ role_path }}/tasks/init.yml"
+  tags:
+    - always
+
+- import_tasks: "{{ role_path }}/tasks/validate_config.yml"
+  when: run_setup|bool
+  tags:
+    - setup-all
+    - setup-grafana
+
+- import_tasks: "{{ role_path }}/tasks/setup.yml"
+  tags:
+    - setup-all
+    - setup-grafana

roles/matrix-grafana/tasks/setup.yml

@@ -0,0 +1,115 @@
+---
+
+#
+# Tasks related to setting up matrix-grafana
+#
+
+- name: Ensure matrix-grafana image is pulled
+  docker_image:
+    name: "{{ matrix_grafana_docker_image }}"
+    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
+    force_source: "{{ matrix_grafana_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
+    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_grafana_docker_image_force_pull }}"
+  when: "matrix_grafana_enabled|bool"
+
+- name: Ensure grafana paths exists
+  file:
+    path: "{{ item }}"
+    state: directory
+    mode: 0750
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_groupname }}"
+  with_items:
+    - "{{ matrix_grafana_base_path }}"
+    - "{{ matrix_grafana_config_path }}"
+    - "{{ matrix_grafana_config_path }}/provisioning"
+    - "{{ matrix_grafana_config_path }}/provisioning/datasources"
+    - "{{ matrix_grafana_config_path }}/provisioning/dashboards"
+    - "{{ matrix_grafana_config_path }}/dashboards"
+    - "{{ matrix_grafana_data_path }}"
+  when: matrix_grafana_enabled|bool
+  
+- name: Ensure grafana.ini present
+  template:
+    src: "{{ role_path }}/templates/grafana.ini.j2"
+    dest: "{{ matrix_grafana_config_path }}/grafana.ini"
+    mode: 0440
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_groupname }}"
+  when: matrix_grafana_enabled|bool
+  
+- name: Ensure provisioning/datasources/default.yaml present
+  template:
+    src: "{{ role_path }}/templates/datasources.yaml.j2"
+    dest: "{{ matrix_grafana_config_path }}/provisioning/datasources/default.yaml"
+    mode: 0440
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_groupname }}"
+  when: matrix_grafana_enabled|bool
+  
+- name: Ensure provisioning/dashboards/default.yaml present
+  template:
+    src: "{{ role_path }}/templates/dashboards.yaml.j2"
+    dest: "{{ matrix_grafana_config_path }}/provisioning/dashboards/default.yaml"
+    mode: 0440
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_groupname }}"
+  when: matrix_grafana_enabled|bool
+  
+- name: Ensure dashboard(s) downloaded
+  get_url:
+    url: "{{ item }}"
+    dest: "{{ matrix_grafana_config_path }}/dashboards/"
+    force: true
+    mode: 0440
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_groupname }}"
+  with_items: "{{ matrix_grafana_dashboard_download_urls }}"
+  when: matrix_grafana_enabled|bool
+
+- name: Ensure matrix-grafana.service installed
+  template:
+    src: "{{ role_path }}/templates/systemd/matrix-grafana.service.j2"
+    dest: "{{ matrix_systemd_path }}/matrix-grafana.service"
+    mode: 0644
+  register: matrix_grafana_systemd_service_result
+  when: matrix_grafana_enabled|bool
+
+- name: Ensure systemd reloaded after matrix-grafana.service installation
+  service:
+    daemon_reload: yes
+  when: "matrix_grafana_enabled|bool and matrix_grafana_systemd_service_result.changed"
+
+#
+# Tasks related to getting rid of matrix-grafana (if it was previously enabled)
+#
+
+- name: Check existence of matrix-grafana service
+  stat:
+    path: "{{ matrix_systemd_path }}/matrix-grafana.service"
+  register: matrix_grafana_service_stat
+
+- name: Ensure matrix-grafana is stopped
+  service:
+    name: matrix-grafana
+    state: stopped
+    daemon_reload: yes
+  register: stopping_result
+  when: "not matrix_grafana_enabled|bool and matrix_grafana_service_stat.stat.exists"
+
+- name: Ensure matrix-grafana.service doesn't exist
+  file:
+    path: "{{ matrix_systemd_path }}/matrix-grafana.service"
+    state: absent
+  when: "not matrix_grafana_enabled|bool and matrix_grafana_service_stat.stat.exists"
+
+- name: Ensure systemd reloaded after matrix-grafana.service removal
+  service:
+    daemon_reload: yes
+  when: "not matrix_grafana_enabled|bool and matrix_grafana_service_stat.stat.exists"
+
+- name: Ensure matrix-grafana Docker image doesn't exist
+  docker_image:
+    name: "{{ matrix_grafana_docker_image }}"
+    state: absent
+  when: "not matrix_grafana_enabled|bool"

roles/matrix-grafana/tasks/validate_config.yml

@@ -0,0 +1,7 @@
+---
+
+- name: Fail if Prometheus not enabled
+  fail:
+    msg: >
+      You need to enable `matrix_prometheus_enabled` to use Prometheus as data source for Grafana.
+  when: "not matrix_prometheus_enabled"

roles/matrix-grafana/templates/dashboards.yaml.j2

@@ -0,0 +1,9 @@
+apiVersion: 1
+
+providers:
+  - name: {{ matrix_server_fqn_matrix }} - Dashboards
+    folder: '' # The folder where to place the dashboards
+    type: file
+    allowUiUpdates: true
+    options:
+      path: /etc/grafana/dashboards

roles/matrix-grafana/templates/datasources.yaml.j2

@@ -0,0 +1,8 @@
+apiVersion: 1
+
+datasources:
+  - name: {{ matrix_server_fqn_matrix }} - Prometheus
+    type: prometheus
+    # Access mode - proxy (server in the UI) or direct (browser in the UI).
+    access: proxy
+    url: http://matrix-prometheus:9090

roles/matrix-grafana/templates/grafana.ini.j2

@@ -0,0 +1,20 @@
+[security]
+# default admin user, created on startup
+admin_user = {{ matrix_grafana_default_admin_user }}
+
+# default admin password, can be changed before first start of grafana,  or in profile settings
+admin_password = {{ matrix_grafana_default_admin_password }}
+
+[auth.anonymous]
+# enable anonymous access
+enabled = {{ matrix_grafana_anonymous_access }}
+
+# specify organization name that should be used for unauthenticated users
+org_name = {{ matrix_grafana_anonymous_access_org_name }}
+
+[dashboards]
+{% if matrix_synapse_metrics_enabled %}
+default_home_dashboard_path = /etc/grafana/dashboards/synapse.json
+{% else %}
+default_home_dashboard_path = /etc/grafana/dashboards/node-exporter-full.json
+{% endif %}

roles/matrix-grafana/templates/systemd/matrix-grafana.service.j2

@@ -0,0 +1,43 @@
+#jinja2: lstrip_blocks: "True"
+[Unit]
+Description=matrix-grafana
+{% for service in matrix_grafana_systemd_required_services_list %}
+Requires={{ service }}
+After={{ service }}
+{% endfor %}
+{% for service in matrix_grafana_systemd_wanted_services_list %}
+Wants={{ service }}
+{% endfor %}
+DefaultDependencies=no
+
+[Service]
+Type=simple
+Environment="HOME={{ matrix_systemd_unit_home_path }}"
+ExecStartPre=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} kill matrix-grafana 2>/dev/null'
+ExecStartPre=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} rm matrix-grafana 2>/dev/null'
+
+
+ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-grafana \
+			--log-driver=none \
+			--user={{ matrix_user_uid }}:{{ matrix_user_gid }} \
+			--cap-drop=ALL \
+			--read-only \
+			--network={{ matrix_docker_network }} \
+			{% if matrix_grafana_container_http_host_bind_port %}
+			-p {{ matrix_grafana_container_http_host_bind_port }}:3000 \
+			{% endif %}
+			-v {{ matrix_grafana_config_path }}:/etc/grafana:z \
+			-v {{ matrix_grafana_data_path }}:/var/lib/grafana:z \
+			{% for arg in matrix_grafana_container_extra_arguments %}
+			{{ arg }} \
+			{% endfor %}
+			{{ matrix_grafana_docker_image }}
+
+ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} kill matrix-grafana 2>/dev/null'
+ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} rm matrix-grafana 2>/dev/null'
+Restart=always
+RestartSec=30
+SyslogIdentifier=matrix-grafana
+
+[Install]
+WantedBy=multi-user.target

roles/matrix-nginx-proxy/defaults/main.yml

@@ -115,6 +115,10 @@ matrix_nginx_proxy_proxy_dimension_hostname: "{{ matrix_server_fqn_dimension }}"
 matrix_nginx_proxy_proxy_jitsi_enabled: false
 matrix_nginx_proxy_proxy_jitsi_hostname: "{{ matrix_server_fqn_jitsi }}"
 
+# Controls whether proxying the grafana domain should be done.
+matrix_nginx_proxy_proxy_grafana_enabled: false
+matrix_nginx_proxy_proxy_grafana_hostname: "{{ matrix_server_fqn_grafana }}"
+
 # Controls whether proxying for the matrix-corporal API (`/_matrix/corporal`) should be done (on the matrix domain)
 matrix_nginx_proxy_proxy_matrix_corporal_api_enabled: false
 matrix_nginx_proxy_proxy_matrix_corporal_api_addr_with_container: "matrix-corporal:41081"
@@ -212,6 +216,9 @@ matrix_nginx_proxy_proxy_dimension_additional_server_configuration_blocks: []
 # A list of strings containing additional configuration blocks to add to Jitsi's server configuration.
 matrix_nginx_proxy_proxy_jitsi_additional_server_configuration_blocks: []
 
+# A list of strings containing additional configuration blocks to add to Grafana's server configuration.
+matrix_nginx_proxy_proxy_grafana_additional_server_configuration_blocks: []
+
 # A list of strings containing additional configuration blocks to add to the base domain server configuration.
 matrix_nginx_proxy_proxy_domain_additional_server_configuration_blocks: []
 

roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml

@@ -80,6 +80,13 @@
     mode: 0644
   when: matrix_nginx_proxy_proxy_jitsi_enabled|bool
 
+- name: Ensure Matrix nginx-proxy configuration for grafana domain exists
+  template:
+    src: "{{ role_path }}/templates/nginx/conf.d/matrix-grafana.conf.j2"
+    dest: "{{ matrix_nginx_proxy_confd_path }}/matrix-grafana.conf"
+    mode: 0644
+  when: matrix_nginx_proxy_proxy_grafana_enabled|bool
+
 - name: Ensure Matrix nginx-proxy data directory for base domain exists
   file:
     path: "{{ matrix_nginx_proxy_data_path }}/matrix-domain"
@@ -183,6 +190,12 @@
     state: absent
   when: "not matrix_nginx_proxy_proxy_jitsi_enabled|bool"
 
+- name: Ensure Matrix nginx-proxy configuration for grafana domain deleted
+  file:
+    path: "{{ matrix_nginx_proxy_confd_path }}/matrix-grafana.conf"
+    state: absent
+  when: "not matrix_nginx_proxy_proxy_grafana_enabled|bool"
+
 - name: Ensure Matrix nginx-proxy homepage for base domain deleted
   file:
     path: "{{ matrix_nginx_proxy_data_path }}/matrix-domain/index.html"

roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-grafana.conf.j2

@@ -0,0 +1,79 @@
+#jinja2: lstrip_blocks: "True"
+
+{% macro render_vhost_directives() %}
+	gzip on;
+	gzip_types text/plain application/json application/javascript text/css image/x-icon font/ttf image/gif;
+	add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
+	add_header X-Content-Type-Options nosniff;
+	add_header X-Frame-Options SAMEORIGIN;
+	{% for configuration_block in matrix_nginx_proxy_proxy_grafana_additional_server_configuration_blocks %}
+		{{- configuration_block }}
+	{% endfor %}
+
+	location / {
+		{% if matrix_nginx_proxy_enabled %}
+			{# Use the embedded DNS resolver in Docker containers to discover the service #}
+			resolver 127.0.0.11 valid=5s;
+			set $backend "matrix-grafana:3000";
+			proxy_pass http://$backend;
+		{% else %}
+			{# Generic configuration for use outside of our container setup #}
+			proxy_pass http://127.0.0.1:3000;
+		{% endif %}
+
+		proxy_set_header Host $host;
+		proxy_set_header X-Forwarded-For $remote_addr;
+	}
+{% endmacro %}
+
+server {
+	listen {{ 8080 if matrix_nginx_proxy_enabled else 80 }};
+
+	server_name {{ matrix_nginx_proxy_proxy_grafana_hostname }};
+
+	server_tokens off;
+	root /dev/null;
+
+	{% if matrix_nginx_proxy_https_enabled %}
+		location /.well-known/acme-challenge {
+			{% if matrix_nginx_proxy_enabled %}
+				{# Use the embedded DNS resolver in Docker containers to discover the service #}
+				resolver 127.0.0.11 valid=5s;
+				set $backend "matrix-certbot:8080";
+				proxy_pass http://$backend;
+			{% else %}
+				{# Generic configuration for use outside of our container setup #}
+				proxy_pass http://127.0.0.1:{{ matrix_ssl_lets_encrypt_certbot_standalone_http_port }};
+			{% endif %}
+		}
+
+		location / {
+			return 301 https://$http_host$request_uri;
+		}
+	{% else %}
+		{{ render_vhost_directives() }}
+	{% endif %}
+}
+
+{% if matrix_nginx_proxy_https_enabled %}
+server {
+	listen {{ 8443 if matrix_nginx_proxy_enabled else 443 }} ssl http2;
+	listen [::]:{{ 8443 if matrix_nginx_proxy_enabled else 443 }} ssl http2;
+
+	server_name {{ matrix_nginx_proxy_proxy_grafana_hostname }};
+
+	server_tokens off;
+	root /dev/null;
+
+	ssl_certificate {{ matrix_ssl_config_dir_path }}/live/{{ matrix_nginx_proxy_proxy_grafana_hostname }}/fullchain.pem;
+	ssl_certificate_key {{ matrix_ssl_config_dir_path }}/live/{{ matrix_nginx_proxy_proxy_grafana_hostname }}/privkey.pem;
+
+	ssl_protocols {{ matrix_nginx_proxy_ssl_protocols }};
+	{% if matrix_nginx_proxy_ssl_ciphers != "" %}
+	ssl_ciphers {{ matrix_nginx_proxy_ssl_ciphers }};
+	{% endif %}
+	ssl_prefer_server_ciphers {{ matrix_nginx_proxy_ssl_prefer_server_ciphers }};
+
+	{{ render_vhost_directives() }}
+}
+{% endif %}

roles/matrix-prometheus-node-exporter/defaults/main.yml

@@ -0,0 +1,21 @@
+# matrix-prometheus-node-exporter is an Prometheus exporter for machine metrics
+# See: https://prometheus.io/docs/guides/node-exporter/
+
+matrix_prometheus_node_exporter_enabled: false
+
+matrix_prometheus_node_exporter_docker_image: "docker.io/prom/node-exporter:v1.1.0"
+matrix_prometheus_node_exporter_docker_image_force_pull: "{{ matrix_prometheus_node_exporter_docker_image.endswith(':latest') }}"
+
+# A list of extra arguments to pass to the container
+matrix_prometheus_node_exporter_container_extra_arguments: []
+
+# List of systemd services that matrix-prometheus.service depends on
+matrix_prometheus_node_exporter_systemd_required_services_list: ['docker.service']
+
+# List of systemd services that matrix-prometheus.service wants
+matrix_prometheus_node_exporter_systemd_wanted_services_list: []
+
+# Controls whether the matrix-prometheus container exposes its HTTP port (tcp/9100 in the container).
+#
+# Takes an "<ip>:<port>" or "<port>" value (e.g. "127.0.0.1:9100"), or empty string to not expose.
+matrix_prometheus_node_exporter_container_http_host_bind_port: ''

roles/matrix-prometheus-node-exporter/tasks/init.yml

@@ -0,0 +1,5 @@
+- set_fact:
+    matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-prometheus-node-exporter.service'] }}"
+  when: matrix_prometheus_node_exporter_enabled|bool
+
+

roles/matrix-prometheus-node-exporter/tasks/main.yml

@@ -0,0 +1,8 @@
+- import_tasks: "{{ role_path }}/tasks/init.yml"
+  tags:
+    - always
+
+- import_tasks: "{{ role_path }}/tasks/setup.yml"
+  tags:
+    - setup-all
+    - setup-prometheus-node-exporter

roles/matrix-prometheus-node-exporter/tasks/setup.yml

@@ -0,0 +1,60 @@
+---
+
+#
+# Tasks related to setting up matrix-prometheus-node-exporter
+#
+
+- name: Ensure matrix-prometheus-node-exporter image is pulled
+  docker_image:
+    name: "{{ matrix_prometheus_node_exporter_docker_image }}"
+    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
+    force_source: "{{ matrix_prometheus_node_exporter_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
+    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_prometheus_node_exporter_docker_image_force_pull }}"
+  when: "matrix_prometheus_node_exporter_enabled|bool"
+
+- name: Ensure matrix-prometheus-node-exporter.service installed
+  template:
+    src: "{{ role_path }}/templates/systemd/matrix-prometheus-node-exporter.service.j2"
+    dest: "{{ matrix_systemd_path }}/matrix-prometheus-node-exporter.service"
+    mode: 0644
+  register: matrix_prometheus_node_exporter_systemd_service_result
+  when: matrix_prometheus_node_exporter_enabled|bool
+
+- name: Ensure systemd reloaded after matrix-prometheus.service installation
+  service:
+    daemon_reload: yes
+  when: "matrix_prometheus_node_exporter_enabled|bool and matrix_prometheus_node_exporter_systemd_service_result.changed"
+
+#
+# Tasks related to getting rid of matrix-prometheus-node-exporter (if it was previously enabled)
+#
+
+- name: Check existence of matrix-prometheus-node-exporter service
+  stat:
+    path: "{{ matrix_systemd_path }}/matrix-prometheus-node-exporter.service"
+  register: matrix_prometheus_node_exporter_service_stat
+
+- name: Ensure matrix-prometheus-node-exporter is stopped
+  service:
+    name: matrix-prometheus-node-exporter
+    state: stopped
+    daemon_reload: yes
+  register: stopping_result
+  when: "not matrix_prometheus_node_exporter_enabled|bool and matrix_prometheus_node_exporter_service_stat.stat.exists"
+
+- name: Ensure matrix-prometheus-node-exporter.service doesn't exist
+  file:
+    path: "{{ matrix_systemd_path }}/matrix-prometheus-node-exporter.service"
+    state: absent
+  when: "not matrix_prometheus_node_exporter_enabled|bool and matrix_prometheus_node_exporter_service_stat.stat.exists"
+
+- name: Ensure systemd reloaded after matrix-prometheus-node-exporter.service removal
+  service:
+    daemon_reload: yes
+  when: "not matrix_prometheus_node_exporter_enabled|bool and matrix_prometheus_node_exporter_service_stat.stat.exists"
+
+- name: Ensure matrix-prometheus-node-exporter Docker image doesn't exist
+  docker_image:
+    name: "{{ matrix_prometheus_node_exporter_docker_image }}"
+    state: absent
+  when: "not matrix_prometheus_node_exporter_enabled|bool"

roles/matrix-prometheus-node-exporter/tasks/validate_config.yml

@@ -0,0 +1,7 @@
+---
+
+- name: Fail if Synapse metrics or Prometheus Node Exporter not enabled
+  fail:
+    msg: >
+      You need to enable `matrix_synapse_metrics_enabled` and/or `matrix_prometheus_node_exporter_enabled` for Prometheus grab metrics.
+  when: "not matrix_synapse_metrics_enabled and not matrix_prometheus_node_exporter_enabled"

roles/matrix-prometheus-node-exporter/templates/systemd/matrix-prometheus-node-exporter.service.j2

@@ -0,0 +1,44 @@
+#jinja2: lstrip_blocks: "True"
+[Unit]
+Description=matrix-prometheus-node-exporter
+{% for service in matrix_prometheus_node_exporter_systemd_required_services_list %}
+Requires={{ service }}
+After={{ service }}
+{% endfor %}
+{% for service in matrix_prometheus_node_exporter_systemd_wanted_services_list %}
+Wants={{ service }}
+{% endfor %}
+DefaultDependencies=no
+
+[Service]
+Type=simple
+Environment="HOME={{ matrix_systemd_unit_home_path }}"
+ExecStartPre=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} kill matrix-prometheus-node-exporter 2>/dev/null'
+ExecStartPre=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} rm matrix-prometheus-node-exporter 2>/dev/null'
+
+
+ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-prometheus-node-exporter \
+			--log-driver=none \
+			--user={{ matrix_user_uid }}:{{ matrix_user_gid }} \
+			--cap-drop=ALL \
+			--read-only \
+			{% if matrix_prometheus_node_exporter_container_http_host_bind_port %}
+			-p {{ matrix_prometheus_node_exporter_container_http_host_bind_port }}:9100 \
+			{% endif %}
+			{% for arg in matrix_prometheus_node_exporter_container_extra_arguments %}
+			{{ arg }} \
+			{% endfor %}
+			--net=host \
+			--pid=host \
+			--mount type=bind,src=/,dst=/host,ro,bind-propagation=rslave \
+			{{ matrix_prometheus_node_exporter_docker_image }} \
+			--path.rootfs=/host
+
+ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} kill matrix-prometheus-node-exporter 2>/dev/null'
+ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} rm matrix-prometheus-node-exporter 2>/dev/null'
+Restart=always
+RestartSec=30
+SyslogIdentifier=matrix-prometheus-node-exporter
+
+[Install]
+WantedBy=multi-user.target

roles/matrix-prometheus/defaults/main.yml

@@ -0,0 +1,66 @@
+# matrix-prometheus is an open-source systems monitoring and alerting toolkit
+# See: https://github.com/matrix-org/synapse/blob/master/docs/metrics-howto.md
+
+matrix_prometheus_enabled: false
+
+matrix_prometheus_docker_image: "docker.io/prom/prometheus:v2.24.1"
+matrix_prometheus_docker_image_force_pull: "{{ matrix_prometheus_docker_image.endswith(':latest') }}"
+
+matrix_prometheus_base_path: "{{ matrix_base_data_path }}/prometheus"
+matrix_prometheus_config_path: "{{ matrix_prometheus_base_path }}/config"
+matrix_prometheus_data_path: "{{ matrix_prometheus_base_path }}/data"
+
+# A list of extra arguments to pass to the container
+matrix_prometheus_container_extra_arguments: []
+
+# List of systemd services that matrix-prometheus.service depends on
+matrix_prometheus_systemd_required_services_list: ['docker.service']
+
+# List of systemd services that matrix-prometheus.service wants
+matrix_prometheus_systemd_wanted_services_list: []
+
+# Controls whether the matrix-prometheus container exposes its HTTP port (tcp/9090 in the container).
+#
+# Takes an "<ip>:<port>" or "<port>" value (e.g. "127.0.0.1:9090"), or empty string to not expose.
+matrix_prometheus_container_http_host_bind_port: ''
+
+# Tells whether the "synapse" scraper configuration is enabled.
+matrix_prometheus_scraper_synapse_enabled: false
+
+# Tells whether to download and load a Synapse rules file
+matrix_prometheus_scraper_synapse_rules_enabled: "{{ matrix_prometheus_scraper_synapse_enabled }}"
+matrix_prometheus_scraper_synapse_rules_synapse_tag: "master"
+matrix_prometheus_scraper_synapse_rules_download_url: "https://raw.githubusercontent.com/matrix-org/synapse/{{ matrix_prometheus_scraper_synapse_rules_synapse_tag }}/contrib/prometheus/synapse-v2.rules"
+
+matrix_prometheus_scraper_synapse_targets: []
+
+# Tells whether the "node" scraper configuration is enabled.
+# This configuration aims to scrape the current node (this server).
+matrix_prometheus_scraper_node_enabled: false
+
+# Target addresses for the "node" scraper configuration.
+# Unless you define this as a non-empty list, it gets populated at runtime with the IP address of `matrix-prometheus-node-exporter` and port 9100.
+matrix_prometheus_scraper_node_targets: []
+
+# Default prometheus configuration template which covers the generic use case.
+# You can customize it by controlling the various variables inside it.
+#
+# For a more advanced customization, you can extend the default (see `matrix_prometheus_configuration_extension_yaml`)
+# or completely replace this variable with your own template.
+matrix_prometheus_configuration_yaml: "{{ lookup('template', 'templates/prometheus.yml.j2') }}"
+
+matrix_prometheus_configuration_extension_yaml: |
+  # Your custom YAML configuration goes here.
+  # This configuration extends the default starting configuration (`matrix_prometheus_configuration_yaml`).
+  #
+  # You can override individual variables from the default configuration, or introduce new ones.
+  #
+  # If you need something more special, you can take full control by
+  # completely redefining `matrix_prometheus_configuration_yaml`.
+
+matrix_prometheus_configuration_extension: "{{ matrix_prometheus_configuration_extension_yaml|from_yaml if matrix_prometheus_configuration_extension_yaml|from_yaml is mapping else {} }}"
+
+# Holds the final configuration (a combination of the default and its extension).
+# You most likely don't need to touch this variable. Instead, see `matrix_prometheus_configuration_yaml`.
+matrix_prometheus_configuration: "{{ matrix_prometheus_configuration_yaml|from_yaml|combine(matrix_prometheus_configuration_extension, recursive=True) }}"
+

roles/matrix-prometheus/tasks/init.yml

@@ -0,0 +1,5 @@
+- set_fact:
+    matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-prometheus.service'] }}"
+  when: matrix_prometheus_enabled|bool
+
+

roles/matrix-prometheus/tasks/main.yml

@@ -0,0 +1,21 @@
+- import_tasks: "{{ role_path }}/tasks/init.yml"
+  tags:
+    - always
+
+- import_tasks: "{{ role_path }}/tasks/validate_config.yml"
+  when: run_setup|bool
+  tags:
+    - setup-all
+    - setup-prometheus
+
+- import_tasks: "{{ role_path }}/tasks/setup_install.yml"
+  when: "run_setup|bool and matrix_prometheus_enabled|bool"
+  tags:
+    - setup-all
+    - setup-prometheus
+
+- import_tasks: "{{ role_path }}/tasks/setup_uninstall.yml"
+  when: "run_setup|bool and not matrix_prometheus_enabled|bool"
+  tags:
+    - setup-all
+    - setup-prometheus

roles/matrix-prometheus/tasks/setup_install.yml

@@ -0,0 +1,66 @@
+---
+
+- name: Ensure matrix-prometheus image is pulled
+  docker_image:
+    name: "{{ matrix_prometheus_docker_image }}"
+    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
+    force_source: "{{ matrix_prometheus_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
+    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_prometheus_docker_image_force_pull }}"
+
+- name: Ensure Prometheus paths exists
+  file:
+    path: "{{ item }}"
+    state: directory
+    mode: 0750
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_groupname }}"
+  with_items:
+    - "{{ matrix_prometheus_base_path }}"
+    - "{{ matrix_prometheus_config_path }}"
+    - "{{ matrix_prometheus_data_path }}"
+
+- block:
+    # Well, this actually creates the network if it doesn't exist, but..
+    # The network should have been created by `matrix-base` already.
+    # We don't rely on that other call and its result, because it runs
+    # on `--tags=setup-all`, but will get skipped during `--tags=setup-prometheus`.
+    - name: Fetch Matrix Docker network details
+      docker_network:
+        name: "{{ matrix_docker_network }}"
+        driver: bridge
+      register: matrix_docker_network_info
+
+    - set_fact:
+        matrix_prometheus_scraper_node_targets: ["{{ matrix_docker_network_info.network.IPAM.Config[0].Gateway }}:9100"]
+  when: "matrix_prometheus_scraper_node_enabled|bool and matrix_prometheus_scraper_node_targets|length == 0"
+
+
+- name: Download synapse-v2.rules
+  get_url:
+    url: "{{ matrix_prometheus_scraper_synapse_rules_download_url }}"
+    dest: "{{ matrix_prometheus_config_path }}/synapse-v2.rules"
+    force: true
+    mode: 0440
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_groupname }}"
+  when: "matrix_prometheus_scraper_synapse_rules_enabled|bool"
+
+- name: Ensure prometheus.yml installed
+  copy:
+    content: "{{ matrix_prometheus_configuration|to_nice_yaml }}"
+    dest: "{{ matrix_prometheus_config_path }}/prometheus.yml"
+    mode: 0644
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_groupname }}"
+
+- name: Ensure matrix-prometheus.service installed
+  template:
+    src: "{{ role_path }}/templates/systemd/matrix-prometheus.service.j2"
+    dest: "{{ matrix_systemd_path }}/matrix-prometheus.service"
+    mode: 0644
+  register: matrix_prometheus_systemd_service_result
+
+- name: Ensure systemd reloaded after matrix-prometheus.service installation
+  service:
+    daemon_reload: yes
+  when: "matrix_prometheus_systemd_service_result.changed|bool"

roles/matrix-prometheus/tasks/setup_uninstall.yml

@@ -0,0 +1,31 @@
+---
+
+- name: Check existence of matrix-prometheus service
+  stat:
+    path: "{{ matrix_systemd_path }}/matrix-prometheus.service"
+  register: matrix_prometheus_service_stat
+
+- name: Ensure matrix-prometheus is stopped
+  service:
+    name: matrix-prometheus
+    state: stopped
+    daemon_reload: yes
+  register: stopping_result
+  when: "matrix_prometheus_service_stat.stat.exists|bool"
+
+- name: Ensure matrix-prometheus.service doesn't exist
+  file:
+    path: "{{ matrix_systemd_path }}/matrix-prometheus.service"
+    state: absent
+  when: "matrix_prometheus_service_stat.stat.exists|bool"
+
+- name: Ensure systemd reloaded after matrix-prometheus.service removal
+  service:
+    daemon_reload: yes
+  when: "matrix_prometheus_service_stat.stat.exists|bool"
+
+- name: Ensure matrix-prometheus Docker image doesn't exist
+  docker_image:
+    name: "{{ matrix_prometheus_docker_image }}"
+    state: absent
+  when: "not matrix_prometheus_enabled|bool"

roles/matrix-prometheus/tasks/validate_config.yml

@@ -0,0 +1,7 @@
+---
+
+- name: Fail if Synapse metrics or Prometheus Node Exporter not enabled
+  fail:
+    msg: >
+      You need to enable `matrix_prometheus_scraper_synapse_enabled` and/or `matrix_prometheus_scraper_node_enabled` for Prometheus grab metrics.
+  when: "not matrix_prometheus_scraper_synapse_enabled and not matrix_prometheus_scraper_node_enabled"

roles/matrix-prometheus/templates/prometheus.yml.j2

@@ -0,0 +1,40 @@
+#jinja2: lstrip_blocks: "True"
+global:
+  scrape_interval:     15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
+  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
+  # scrape_timeout is set to the global default (10s).
+
+# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
+rule_files:
+    {% if matrix_prometheus_scraper_synapse_rules_enabled %}
+    - 'synapse-v2.rules'
+    {% endif %}
+
+# A scrape configuration containing exactly one endpoint to scrape:
+# Here it's Prometheus itself.
+scrape_configs:
+  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
+  - job_name: 'prometheus'
+
+    # Override the global default and scrape targets from this job every 5 seconds.
+    scrape_interval: 5s
+    scrape_timeout: 5s
+
+    # metrics_path defaults to '/metrics'
+    # scheme defaults to 'http'.
+
+    static_configs:
+      - targets: ['localhost:9090']
+
+  {% if matrix_prometheus_scraper_synapse_enabled %}
+  - job_name: 'synapse'
+    metrics_path: '/_synapse/metrics'
+    static_configs:
+    - targets: {{ matrix_prometheus_scraper_synapse_targets|to_json }}
+  {% endif %}
+
+  {% if matrix_prometheus_scraper_node_enabled %}
+  - job_name: node
+    static_configs:
+      - targets: {{ matrix_prometheus_scraper_node_targets|to_json }}
+  {% endif %}

roles/matrix-prometheus/templates/systemd/matrix-prometheus.service.j2

@@ -0,0 +1,43 @@
+#jinja2: lstrip_blocks: "True"
+[Unit]
+Description=matrix-prometheus
+{% for service in matrix_prometheus_systemd_required_services_list %}
+Requires={{ service }}
+After={{ service }}
+{% endfor %}
+{% for service in matrix_prometheus_systemd_wanted_services_list %}
+Wants={{ service }}
+{% endfor %}
+DefaultDependencies=no
+
+[Service]
+Type=simple
+Environment="HOME={{ matrix_systemd_unit_home_path }}"
+ExecStartPre=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} kill matrix-prometheus 2>/dev/null'
+ExecStartPre=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} rm matrix-prometheus 2>/dev/null'
+
+
+ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-prometheus \
+			--log-driver=none \
+			--user={{ matrix_user_uid }}:{{ matrix_user_gid }} \
+			--cap-drop=ALL \
+			--read-only \
+			--network={{ matrix_docker_network }} \
+			{% if matrix_prometheus_container_http_host_bind_port %}
+			-p {{ matrix_prometheus_container_http_host_bind_port }}:9090 \
+			{% endif %}
+			-v {{ matrix_prometheus_config_path }}:/etc/prometheus:z \
+			-v {{ matrix_prometheus_data_path }}:/prometheus:z \
+			{% for arg in matrix_prometheus_container_extra_arguments %}
+			{{ arg }} \
+			{% endfor %}
+			{{ matrix_prometheus_docker_image }}
+
+ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} kill matrix-prometheus 2>/dev/null'
+ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} rm matrix-prometheus 2>/dev/null'
+Restart=always
+RestartSec=30
+SyslogIdentifier=matrix-prometheus
+
+[Install]
+WantedBy=multi-user.target