(BC Break) Redo how metrics are exposed to external Prometheus servers

roles/matrix-synapse/defaults/main.yml

@@ -334,6 +334,14 @@ matrix_url_preview_accept_language: ['en-US', 'en']
 matrix_synapse_metrics_enabled: false
 matrix_synapse_metrics_port: 9100
 
+# Controls whether Synapse metrics should be proxied (exposed) on:
+# - `matrix.DOMAIN/metrics/synapse/main-process` for the main process
+# - `matrix.DOMAIN/metrics/synapse/worker/{type}-{id}` for each worker process
+#
+# This will only work take effect if `matrix_nginx_proxy_proxy_matrix_metrics_enabled: true`.
+# See the `matrix-nginx-proxy` role for details about enabling `matrix_nginx_proxy_proxy_matrix_metrics_enabled`.
+matrix_synapse_metrics_proxying_enabled: false
+
 # Enable the Synapse manhole
 # See https://github.com/matrix-org/synapse/blob/master/docs/manhole.md
 matrix_synapse_manhole_enabled: false

roles/matrix-synapse/tasks/init.yml

@@ -25,3 +25,63 @@
 - set_fact:
     matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-goofys.service'] }}"
   when: matrix_s3_media_store_enabled|bool
+
+- block:
+    - name: Fail if matrix-nginx-proxy role already executed
+      fail:
+        msg: >-
+          Trying to append Synapse's reverse-proxying configuration to matrix-nginx-proxy,
+          but it's pointless since the matrix-nginx-proxy role had already executed.
+          To fix this, please change the order of roles in your playbook,
+          so that the matrix-nginx-proxy role would run after the matrix-synapse role.
+      when: matrix_nginx_proxy_role_executed|default(False)|bool
+
+    - name: Generate synapse metrics proxying configuration for matrix-nginx-proxy (matrix.DOMAIN/metrics/synapse/main-process)
+      set_fact:
+        matrix_synapse_nginx_metrics_configuration_block: |
+          location /metrics/synapse/main-process {
+            {% if matrix_nginx_proxy_enabled|default(False) %}
+              {# Use the embedded DNS resolver in Docker containers to discover the service #}
+              resolver 127.0.0.11 valid=5s;
+              set $backend "matrix-synapse:{{ matrix_synapse_metrics_port }}";
+              proxy_pass http://$backend/_synapse/metrics;
+            {% else %}
+              {# Generic configuration for use outside of our container setup #}
+              proxy_pass http://127.0.0.1:{{ matrix_synapse_metrics_port }}/_synapse/metrics;
+            {% endif %}
+          }
+
+    - name: Register synapse metrics proxying configuration with matrix-nginx-proxy (matrix.DOMAIN/metrics/synapse/main-process)
+      set_fact:
+        matrix_nginx_proxy_proxy_matrix_metrics_additional_system_location_configuration_blocks: |
+          {{
+            matrix_nginx_proxy_proxy_matrix_metrics_additional_system_location_configuration_blocks|default([])
+            +
+            [matrix_synapse_nginx_metrics_configuration_block]
+          }}
+
+    - name: Generate synapse worker metrics proxying configuration for matrix-nginx-proxy (matrix.DOMAIN/metrics/synapse/worker)
+      set_fact:
+        matrix_synapse_worker_nginx_metrics_configuration_block: |
+          {% for worker in matrix_synapse_workers_enabled_list %}
+          {% if worker.metrics_port != 0 %}
+          location /metrics/synapse/worker/{{ worker.type }}-{{ worker.instanceId }} {
+            resolver 127.0.0.11 valid=5s;
+            set $backend "matrix-synapse-worker-{{ worker.type }}-{{ worker.instanceId }}:{{ worker.metrics_port }}";
+            proxy_pass http://$backend/_synapse/metrics;
+            proxy_set_header Host $host;
+          }
+          {% endif %}
+          {% endfor %}
+      when: matrix_synapse_workers_enabled_list|length > 0
+
+    - name: Register synapse worker metrics proxying configuration with matrix-nginx-proxy (matrix.DOMAIN/metrics/synapse/worker)
+      set_fact:
+        matrix_nginx_proxy_proxy_matrix_metrics_additional_system_location_configuration_blocks: |
+          {{
+            matrix_nginx_proxy_proxy_matrix_metrics_additional_system_location_configuration_blocks|default([])
+            +
+            [matrix_synapse_worker_nginx_metrics_configuration_block]
+          }}
+      when: matrix_synapse_workers_enabled_list|length > 0
+  when: matrix_synapse_enabled|bool and matrix_synapse_metrics_proxying_enabled|bool

roles/matrix-synapse/tasks/synapse/setup_install.yml

@@ -119,3 +119,12 @@
     src: "{{ role_path }}/templates/synapse/usr-local-bin/matrix-synapse-register-user.j2"
     dest: "{{ matrix_local_bin_path }}/matrix-synapse-register-user"
     mode: 0755
+
+- name: Generate sample prometheus.yml for external scraping
+  template:
+    src: "{{ role_path }}/templates/synapse/prometheus/external_prometheus.yml.example.j2"
+    dest: "{{ matrix_synapse_base_path }}/external_prometheus.yml.example"
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_groupname }}"
+    mode: 0644
+  when: matrix_synapse_metrics_proxying_enabled|bool

roles/matrix-synapse/tasks/synapse/setup_uninstall.yml

@@ -29,3 +29,9 @@
   docker_image:
     name: "{{ matrix_synapse_docker_image }}"
     state: absent
+
+- name: Ensure sample prometheus.yml for external scraping is deleted
+  file:
+    path: "{{ matrix_synapse_base_path }}/external_prometheus.yml.example"
+    state: absent
+  when: "not matrix_synapse_metrics_proxying_enabled|bool"

roles/matrix-synapse/templates/synapse/prometheus/external_prometheus.yml.example.j2

@@ -0,0 +1,40 @@
+global:
+  scrape_interval: 5s
+
+  # Attach these labels to any time series or alerts when communicating with
+  # external systems (federation, remote storage, Alertmanager).
+  external_labels:
+    monitor: 'synapse-{{ matrix_domain }}'
+
+rule_files:
+  - /etc/prometheus/synapse-v2.rules
+
+scrape_configs:
+  - job_name: 'synapse'
+    metrics_path: /metrics/synapse/main-process
+    scheme: {{ 'https' if matrix_nginx_proxy_https_enabled|default(true) else 'http' }}
+{% if matrix_nginx_proxy_proxy_matrix_metrics_basic_auth_enabled|default(true) %}
+    basic_auth:
+      username: prometheus
+      password_file: /path/to/your/passwordfile.pwd
+{% endif %}
+    static_configs:
+      - targets: ['{{ matrix_server_fqn_matrix }}:{{ matrix_nginx_proxy_container_https_host_bind_port|default(443) if matrix_nginx_proxy_https_enabled|default(true) else matrix_nginx_proxy_container_http_host_bind_port|default(80) }}']
+        labels:
+          job: "master"
+          index: "0"
+{% for worker in matrix_synapse_workers_enabled_list %}
+  - job_name: 'synapse-{{ worker.type }}-{{ worker.instanceId }}'
+    metrics_path: /metrics/synapse/worker/{{ worker.type }}-{{ worker.instanceId }}
+    scheme: {{ 'https' if matrix_nginx_proxy_https_enabled|default(true) else 'http' }}
+{% if matrix_nginx_proxy_proxy_matrix_metrics_basic_auth_enabled|default(true) %}
+    basic_auth:
+      username: prometheus
+      password_file: /path/to/your/passwordfile.pwd
+{% endif %}
+    static_configs:
+      - targets: ['{{ matrix_server_fqn_matrix }}:{{ matrix_nginx_proxy_container_https_host_bind_port|default(443) if matrix_nginx_proxy_https_enabled|default(true) else matrix_nginx_proxy_container_http_host_bind_port|default(80) }}']
+        labels:
+          job: "{{ worker.type }}"
+          index: "{{ worker.instanceId }}"
+{% endfor %}