Merge pull request #1311 from HarHarLinks/master

add auto proxy synapse worker metrics

roles/matrix-nginx-proxy/defaults/main.yml

@@ -182,6 +182,7 @@ matrix_nginx_proxy_proxy_matrix_identity_api_addr_sans_container: "127.0.0.1:809
 
 # Controls whether proxying for metrics (`/_synapse/metrics`) should be done (on the matrix domain)
 matrix_nginx_proxy_proxy_synapse_metrics: false
+matrix_nginx_proxy_synapse_workers_enabled_list: []
 matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_enabled: false
 # The following value will be written verbatim to the htpasswd file that stores the password for nginx to check against and needs to be encoded appropriately.
 # Read the manpage at `man 1 htpasswd` to learn more, then encrypt your password, and paste the encrypted value here.
@@ -226,7 +227,7 @@ matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_prefix_regexes: |
     +
     (['/_synapse/admin'] if matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_synapse_admin_api_enabled else [])
     +
-    (['/_synapse/metrics'] if matrix_nginx_proxy_proxy_synapse_metrics else [])
+    (['/_synapse.*/metrics'] if matrix_nginx_proxy_proxy_synapse_metrics else [])
   }}
 
 # Specifies where requests for the root URI (`/`) on the `matrix.` domain should be redirected.

roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml

@@ -38,6 +38,15 @@
     mode: 0400
   when: "matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_enabled|bool and matrix_nginx_proxy_proxy_synapse_metrics|bool"
 
+- name: Generate sample prometheus.yml for external scraping
+  template:
+    src: "{{ role_path }}/templates/prometheus/external_prometheus.yml.example.j2"
+    dest: "{{ matrix_base_data_path }}/external_prometheus.yml.example"
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_groupname }}"
+    mode: 0644
+  when: matrix_nginx_proxy_proxy_synapse_metrics|bool
+
 - name: Ensure Matrix nginx-proxy configured (generic)
   template:
     src: "{{ role_path }}/templates/nginx/conf.d/nginx-http.conf.j2"
@@ -270,3 +279,9 @@
     path: "{{ matrix_nginx_proxy_data_path }}/matrix-synapse-metrics-htpasswd"
     state: absent
   when: "not matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_enabled|bool or not matrix_nginx_proxy_proxy_synapse_metrics|bool"
+
+- name: Ensure sample prometheus.yml for external scraping is deleted
+  file:
+    path: "{{ matrix_base_data_path }}/external_prometheus.yml.example"
+    state: absent
+  when: "not matrix_nginx_proxy_proxy_synapse_metrics|bool"

roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2

@@ -153,6 +153,24 @@ server {
 	}
 	{% endif %}
 
+	{% if matrix_nginx_proxy_enabled and matrix_nginx_proxy_proxy_synapse_metrics %}
+		{% for worker in matrix_nginx_proxy_proxy_synapse_workers_enabled_list %}
+			{% if worker.metrics_port != 0 %}
+				location /_synapse-worker-{{ worker.type }}-{{ worker.instanceId }}/metrics {
+					resolver 127.0.0.11 valid=5s;
+					set $backend "matrix-synapse-worker-{{ worker.type }}-{{ worker.instanceId }}:{{ worker.metrics_port }}";
+					proxy_pass http://$backend/_synapse/metrics;
+					proxy_set_header Host $host;
+
+					{% if matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_enabled %}
+						auth_basic "protected";
+						auth_basic_user_file /nginx-data/matrix-synapse-metrics-htpasswd;
+					{% endif %}
+				}
+			{% endif %}
+		{% endfor %}
+	{% endif %}
+
 	{# Everything else just goes to the API server ##}
 	location / {
 		{% if matrix_nginx_proxy_enabled %}

roles/matrix-nginx-proxy/templates/prometheus/external_prometheus.yml.example.j2

@@ -0,0 +1,40 @@
+global:
+  scrape_interval: 5s
+
+  # Attach these labels to any time series or alerts when communicating with
+  # external systems (federation, remote storage, Alertmanager).
+  external_labels:
+    monitor: 'synapse-{{ matrix_domain }}'
+
+rule_files:
+  - /etc/prometheus/synapse-v2.rules
+
+scrape_configs:
+  - job_name: 'synapse'
+    metrics_path: /_synapse/metrics
+    scheme: {{ 'https' if matrix_nginx_proxy_https_enabled else 'http' }}
+{% if matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_enabled %}
+    basic_auth:
+      username: prometheus
+      password_file: /path/to/your/passwordfile.pwd
+{% endif %}
+    static_configs:
+      - targets: ['{{ matrix_server_fqn_matrix }}:{{ matrix_nginx_proxy_container_https_host_bind_port if matrix_nginx_proxy_https_enabled else matrix_nginx_proxy_container_http_host_bind_port }}']
+        labels:
+          job: "master"
+          index: 1
+{% for worker in matrix_nginx_proxy_proxy_synapse_workers_enabled_list %}
+  - job_name: 'synapse-{{ worker.type }}-{{ worker.instanceId }}'
+    metrics_path: /_synapse-worker-{{ worker.type }}-{{ worker.instanceId }}/metrics
+    scheme: {{ 'https' if matrix_nginx_proxy_https_enabled else 'http' }}
+{% if matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_enabled %}
+    basic_auth:
+      username: prometheus
+      password_file: /path/to/your/passwordfile.pwd
+{% endif %}
+    static_configs:
+      - targets: ['{{ matrix_server_fqn_matrix }}:{{ matrix_nginx_proxy_container_https_host_bind_port if matrix_nginx_proxy_https_enabled else matrix_nginx_proxy_container_http_host_bind_port }}']
+        labels:
+          job: "{{ worker.type }}"
+          index: {{ worker.instanceId }}
+{% endfor %}