Merge pull request #1094 from coetzeer/master

Add support for a prometheus postgres exporter

docs/configuring-playbook-prometheus-postgres.md

@@ -0,0 +1,29 @@
+# Enabling metrics and graphs for Postgres (optional)
+
+Expanding on the metrics exposed by the [syapse exporter and the node exporter](configuring-playbook-prometheus-grafana.md), the playbook enables the [postgres exporter](https://github.com/prometheus-community/postgres_exporter) that exposes more detailed information about what's happening on your postgres database.
+
+You can enable this with the following settings in your configuration file (`inventory/host_vars/matrix.<your-domain>/vars.yml`):
+
+
+```yaml
+matrix_prometheus_postgres_exporter_enabled: true
+
+# the role creates a postgres user as credential. You can configure these if required:
+matrix_prometheus_postgres_exporter_database_username: 'matrix_prometheus_postgres_exporter'
+matrix_prometheus_postgres_exporter_database_password: 'some-password'
+
+```
+
+## What does it do?
+
+Name | Description
+-----|----------
+`matrix_prometheus_postgres_exporter_enabled`|Enable the postgres prometheus exporter. This sets up the docker container, connects it to the database and adds a 'job' to the prometheus config which tells prometheus about this new exporter. The default is 'false'
+`matrix_prometheus_postgres_exporter_database_username`| The 'username' for the user that the exporter uses to connect to the database. The default is 'matrix_prometheus_postgres_exporter'
+`matrix_prometheus_postgres_exporter_database_password`| The 'username' for the user that the exporter uses to connect to the database.
+
+
+## More information
+
+- [The PostgresSQL dashboard](https://grafana.com/grafana/dashboards/9628) (generic postgres dashboard)
+

group_vars/matrix_servers

@@ -1466,6 +1466,13 @@ matrix_postgres_additional_databases: |
       'username': matrix_sygnal_database_username,
       'password': matrix_sygnal_database_password,
     }] if (matrix_sygnal_enabled and matrix_sygnal_database_engine == 'postgres' and matrix_sygnal_database_hostname == 'matrix-postgres') else [])
+     +
+    ([{
+      'name': matrix_prometheus_postgres_exporter_database_name,
+      'username': matrix_prometheus_postgres_exporter_database_username,
+      'password': matrix_prometheus_postgres_exporter_database_password,
+    }] if (matrix_prometheus_postgres_exporter_enabled and matrix_prometheus_postgres_exporter_database_hostname == 'matrix-postgres') else [])
+   
    }}
 
 matrix_postgres_import_roles_to_ignore: |
@@ -1766,6 +1773,10 @@ matrix_prometheus_scraper_synapse_rules_synapse_tag: "{{ matrix_synapse_docker_i
 matrix_prometheus_scraper_node_enabled: "{{ matrix_prometheus_node_exporter_enabled }}"
 matrix_prometheus_scraper_node_targets: "{{ ['matrix-prometheus-node-exporter:9100'] if matrix_prometheus_node_exporter_enabled else [] }}"
 
+matrix_prometheus_scraper_postgres_enabled: "{{ matrix_prometheus_postgres_exporter_enabled }}"
+matrix_prometheus_scraper_postgres_targets: "{{ ['matrix-prometheus-postgres-exporter:'+ matrix_prometheus_postgres_exporter_port|string] if matrix_prometheus_scraper_postgres_enabled else [] }}"
+
+
 ######################################################################
 #
 # /matrix-prometheus
@@ -1773,6 +1784,27 @@ matrix_prometheus_scraper_node_targets: "{{ ['matrix-prometheus-node-exporter:91
 ######################################################################
 
 
+######################################################################
+#
+# matrix-prometheus-postgres-exporter
+#
+######################################################################
+
+matrix_prometheus_postgres_exporter_enabled: false
+matrix_prometheus_postgres_exporter_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'prometheus.pg.db') | to_uuid }}"
+
+matrix_prometheus_postgres_exporter_systemd_required_services_list: |
+  {{
+    ['docker.service']
+    +
+    (['matrix-postgres.service'] if matrix_postgres_enabled else [])
+  }}
+
+######################################################################
+#
+# /matrix-prometheus-postgres-exporter
+#
+######################################################################
 
 ######################################################################
 #
@@ -1787,6 +1819,14 @@ matrix_grafana_enabled: false
 # Grafana's HTTP port to the local host.
 matrix_grafana_container_http_host_bind_port: "{{ '' if matrix_nginx_proxy_enabled else '127.0.0.1:3000' }}"
 
+matrix_grafana_dashboard_download_urls_all: |
+  {{
+    matrix_grafana_dashboard_download_urls
+    +
+    (matrix_prometheus_postgres_exporter_dashboard_urls if matrix_prometheus_postgres_exporter_enabled else [])
+  }}
+
+
 ######################################################################
 #
 # /matrix-grafana

roles/matrix-grafana/tasks/setup.yml

@@ -64,7 +64,7 @@
     mode: 0440
     owner: "{{ matrix_user_username }}"
     group: "{{ matrix_user_groupname }}"
-  with_items: "{{ matrix_grafana_dashboard_download_urls }}"
+  with_items: "{{ matrix_grafana_dashboard_download_urls_all }}"
   when: matrix_grafana_enabled|bool
 
 - name: Ensure matrix-grafana.service installed

roles/matrix-prometheus-postgres-exporter/defaults/main.yml

@@ -0,0 +1,49 @@
+# matrix-prometheus-postgres-exporter is an Prometheus exporter for postgres metrics
+# See: https://github.com/prometheus-community/postgres_exporter
+
+matrix_prometheus_postgres_exporter_enabled: false
+
+matrix_prometheus_postgres_exporter_version: v0.9.0
+matrix_prometheus_postgres_exporter_port: 9187
+
+matrix_prometheus_postgres_exporter_docker_image: "quay.io/prometheuscommunity/postgres-exporter:{{ matrix_prometheus_postgres_exporter_version }}"
+matrix_prometheus_postgres_exporter_docker_image_force_pull: "{{ matrix_prometheus_postgres_exporter_docker_image.endswith(':latest') }}"
+
+# A list of extra arguments to pass to the container
+matrix_prometheus_postgres_exporter_container_extra_arguments: ["-e PG_EXPORTER_AUTO_DISCOVER_DATABASES=true",
+ "-e PG_EXPORTER_WEB_LISTEN_ADDRESS=\":{{matrix_prometheus_postgres_exporter_port}}\"",
+ "-e DATA_SOURCE_NAME=\"postgresql://{{matrix_prometheus_postgres_exporter_database_username}}:{{matrix_prometheus_postgres_exporter_database_password}}@{{matrix_prometheus_postgres_exporter_database_hostname}}:5432/{{matrix_prometheus_postgres_exporter_database_name}}?sslmode=disable\"" ]
+
+# List of systemd services that matrix-prometheus-postgres-exporter.service depends on
+matrix_prometheus_postgres_exporter_systemd_required_services_list: ['docker.service']
+
+# List of systemd services that matrix-prometheus-postgres-exporter.service wants
+matrix_prometheus_postgres_exporter_systemd_wanted_services_list: []
+
+# details for connecting to the database
+matrix_prometheus_postgres_exporter_database_username: 'matrix_prometheus_postgres_exporter'
+matrix_prometheus_postgres_exporter_database_password: 'some-password'
+matrix_prometheus_postgres_exporter_database_hostname: 'matrix-postgres'
+matrix_prometheus_postgres_exporter_database_port: 5432
+matrix_prometheus_postgres_exporter_database_name: 'matrix_prometheus_postgres_exporter'
+
+
+# Controls whether the matrix-prometheus container exposes its HTTP port (tcp/9100 in the container).
+#
+# Takes an "<ip>:<port>" value (e.g. "127.0.0.1:9100"), or empty string to not expose.
+#
+# Official recommendations are to run this container with `--net=host`,
+# but we don't do that, since it:
+# - likely exposes the metrics web server way too publicly (before applying https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/1008)
+# - or listens on a loopback interface only (--net=host and 127.0.0.1:9100), which is not reachable from another container (like `matrix-prometheus`)
+#
+# Using `--net=host` and binding to Docker's `matrix` bridge network may be a solution to both,
+# but that's trickier to accomplish and won't necessarily work (hasn't been tested).
+#
+# Not using `--net=host` means that our network statistic reports are likely broken (inaccurate),
+# because node-exporter can't see all interfaces, etc.
+# For now, we'll live with that, until someone develops a better solution.
+matrix_prometheus_postgres_exporter_container_http_host_bind_port: ''
+
+matrix_prometheus_postgres_exporter_dashboard_urls:
+- "https://grafana.com/api/dashboards/9628/revisions/7/download"

roles/matrix-prometheus-postgres-exporter/tasks/init.yml

@@ -0,0 +1,5 @@
+- set_fact:
+    matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-prometheus-postgres-exporter.service'] }}"
+  when: matrix_prometheus_postgres_exporter_enabled|bool
+
+

roles/matrix-prometheus-postgres-exporter/tasks/main.yml

@@ -0,0 +1,8 @@
+- import_tasks: "{{ role_path }}/tasks/init.yml"
+  tags:
+    - always
+
+- import_tasks: "{{ role_path }}/tasks/setup.yml"
+  tags:
+    - setup-all
+    - setup-prometheus-postgres-exporter

roles/matrix-prometheus-postgres-exporter/tasks/setup.yml

@@ -0,0 +1,54 @@
+---
+
+#
+# Tasks related to setting up matrix-prometheus-postgres-exporter
+#
+
+- name: Ensure matrix-prometheus-postgres-exporter image is pulled
+  docker_image:
+    name: "{{ matrix_prometheus_postgres_exporter_docker_image }}"
+    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
+    force_source: "{{ matrix_prometheus_postgres_exporter_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
+    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_prometheus_postgres_exporter_docker_image_force_pull }}"
+  when: "matrix_prometheus_postgres_exporter_enabled|bool"
+
+- name: Ensure matrix-prometheus-postgres-exporter.service installed
+  template:
+    src: "{{ role_path }}/templates/systemd/matrix-prometheus-postgres-exporter.service.j2"
+    dest: "{{ matrix_systemd_path }}/matrix-prometheus-postgres-exporter.service"
+    mode: 0644
+  register: matrix_prometheus_postgres_exporter_systemd_service_result
+  when: matrix_prometheus_postgres_exporter_enabled|bool
+
+- name: Ensure systemd reloaded after matrix-prometheus.service installation
+  service:
+    daemon_reload: yes
+  when: "matrix_prometheus_postgres_exporter_enabled|bool and matrix_prometheus_postgres_exporter_systemd_service_result.changed"
+
+#
+# Tasks related to getting rid of matrix-prometheus-postgres-exporter (if it was previously enabled)
+#
+
+- name: Check existence of matrix-prometheus-postgres-exporter service
+  stat:
+    path: "{{ matrix_systemd_path }}/matrix-prometheus-postgres-exporter.service"
+  register: matrix_prometheus_postgres_exporter_service_stat
+
+- name: Ensure matrix-prometheus-postgres-exporter is stopped
+  service:
+    name: matrix-prometheus-postgres-exporter
+    state: stopped
+    daemon_reload: yes
+  register: stopping_result
+  when: "not matrix_prometheus_postgres_exporter_enabled|bool and matrix_prometheus_postgres_exporter_service_stat.stat.exists"
+
+- name: Ensure matrix-prometheus-postgres-exporter.service doesn't exist
+  file:
+    path: "{{ matrix_systemd_path }}/matrix-prometheus-postgres-exporter.service"
+    state: absent
+  when: "not matrix_prometheus_postgres_exporter_enabled|bool and matrix_prometheus_postgres_exporter_service_stat.stat.exists"
+
+- name: Ensure systemd reloaded after matrix-prometheus-postgres-exporter.service removal
+  service:
+    daemon_reload: yes
+  when: "not matrix_prometheus_postgres_exporter_enabled|bool and matrix_prometheus_postgres_exporter_service_stat.stat.exists"

roles/matrix-prometheus-postgres-exporter/templates/systemd/matrix-prometheus-postgres-exporter.service.j2

@@ -0,0 +1,42 @@
+#jinja2: lstrip_blocks: "True"
+[Unit]
+Description=matrix-prometheus-postgres-exporter
+{% for service in matrix_prometheus_postgres_exporter_systemd_required_services_list %}
+Requires={{ service }}
+After={{ service }}
+{% endfor %}
+{% for service in matrix_prometheus_postgres_exporter_systemd_wanted_services_list %}
+Wants={{ service }}
+{% endfor %}
+DefaultDependencies=no
+
+[Service]
+Type=simple
+Environment="HOME={{ matrix_systemd_unit_home_path }}"
+ExecStartPre=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} kill matrix-prometheus-postgres-exporter 2>/dev/null'
+ExecStartPre=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} rm matrix-prometheus-postgres-exporter 2>/dev/null'
+
+
+ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-prometheus-postgres-exporter \
+			--log-driver=none \
+			--user={{ matrix_user_uid }}:{{ matrix_user_gid }} \
+			--cap-drop=ALL \
+			--read-only \
+			{% for arg in matrix_prometheus_postgres_exporter_container_extra_arguments %}
+			{{ arg }} \
+			{% endfor %}
+			--network={{ matrix_docker_network }} \
+			{% if matrix_prometheus_postgres_exporter_container_http_host_bind_port %}
+			-p {{ matrix_prometheus_postgres_exporter_container_http_host_bind_port }}:{{matrix_prometheus_postgres_exporter_port}} \
+			{% endif %}
+			--pid=host \
+			{{ matrix_prometheus_postgres_exporter_docker_image }} 
+
+ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} kill matrix-prometheus-postgres-exporter 2>/dev/null'
+ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} rm matrix-prometheus-postgres-exporter 2>/dev/null'
+Restart=always
+RestartSec=30
+SyslogIdentifier=matrix-prometheus-postgres-exporter
+
+[Install]
+WantedBy=multi-user.target

roles/matrix-prometheus/templates/prometheus.yml.j2

@@ -38,3 +38,9 @@ scrape_configs:
     static_configs:
       - targets: {{ matrix_prometheus_scraper_node_targets|to_json }}
   {% endif %}
+
+  {% if matrix_prometheus_scraper_postgres_enabled %}
+  - job_name: postgres
+    static_configs:
+      - targets: {{ matrix_prometheus_scraper_postgres_targets|to_json }}
+  {% endif %}

setup.yml

@@ -54,5 +54,5 @@
     - matrix-coturn
     - matrix-aux
     - matrix-postgres-backup
-    - matrix-common-after
+    - matrix-prometheus-postgres-exporter
-
+    - matrix-common-after