From 4e2780ff8858198704f73ca4bbd4a0325376bb2a Mon Sep 17 00:00:00 2001 From: Raymond Coetzee Date: Thu, 27 May 2021 20:13:29 +0100 Subject: [PATCH] Add support for a prometheus postgres exporter This commit introduces a new role that downloads and installs the prometheus community postgres exporter https://github.com/prometheus-community/postgres_exporter. A new credential is added to matrix_postgres_additional_databases that allows the exporter access to the database to gather statistics. A new dashboard was added to the grafana role, with some refactoring to enable the dashboard only if the new role is enabled. I've included some basic instructions for how to enable the role in the Docs section. In terms of testing, I've tested enabling the role, and disabling it to make sure it cleans up the container and systemd role. --- ...onfiguring-playbook-prometheus-postgres.md | 29 ++++++++++ group_vars/matrix_servers | 40 ++++++++++++++ roles/matrix-grafana/tasks/setup.yml | 2 +- .../defaults/main.yml | 49 +++++++++++++++++ .../tasks/init.yml | 5 ++ .../tasks/main.yml | 8 +++ .../tasks/setup.yml | 54 +++++++++++++++++++ ...ix-prometheus-postgres-exporter.service.j2 | 42 +++++++++++++++ .../templates/prometheus.yml.j2 | 6 +++ setup.yml | 4 +- 10 files changed, 236 insertions(+), 3 deletions(-) create mode 100644 docs/configuring-playbook-prometheus-postgres.md create mode 100644 roles/matrix-prometheus-postgres-exporter/defaults/main.yml create mode 100644 roles/matrix-prometheus-postgres-exporter/tasks/init.yml create mode 100644 roles/matrix-prometheus-postgres-exporter/tasks/main.yml create mode 100644 roles/matrix-prometheus-postgres-exporter/tasks/setup.yml create mode 100644 roles/matrix-prometheus-postgres-exporter/templates/systemd/matrix-prometheus-postgres-exporter.service.j2 diff --git a/docs/configuring-playbook-prometheus-postgres.md b/docs/configuring-playbook-prometheus-postgres.md new file mode 100644 index 000000000..9a6312968 --- /dev/null +++ b/docs/configuring-playbook-prometheus-postgres.md @@ -0,0 +1,29 @@ +# Enabling metrics and graphs for Postgres (optional) + +Expanding on the metrics exposed by the [syapse exporter and the node exporter](configuring-playbook-prometheus-grafana.md), the playbook enables the [postgres exporter](https://github.com/prometheus-community/postgres_exporter) that exposes more detailed information about what's happening on your postgres database. + +You can enable this with the following settings in your configuration file (`inventory/host_vars/matrix./vars.yml`): + + +```yaml +matrix_prometheus_postgres_exporter_enabled: true + +# the role creates a postgres user as credential. You can configure these if required: +matrix_prometheus_postgres_exporter_database_username: 'matrix_prometheus_postgres_exporter' +matrix_prometheus_postgres_exporter_database_password: 'some-password' + +``` + +## What does it do? + +Name | Description +-----|---------- +`matrix_prometheus_postgres_exporter_enabled`|Enable the postgres prometheus exporter. This sets up the docker container, connects it to the database and adds a 'job' to the prometheus config which tells prometheus about this new exporter. The default is 'false' +`matrix_prometheus_postgres_exporter_database_username`| The 'username' for the user that the exporter uses to connect to the database. The default is 'matrix_prometheus_postgres_exporter' +`matrix_prometheus_postgres_exporter_database_password`| The 'username' for the user that the exporter uses to connect to the database. + + +## More information + +- [The PostgresSQL dashboard](https://grafana.com/grafana/dashboards/9628) (generic postgres dashboard) + diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 86f1d5a36..96b933f35 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -1466,6 +1466,13 @@ matrix_postgres_additional_databases: | 'username': matrix_sygnal_database_username, 'password': matrix_sygnal_database_password, }] if (matrix_sygnal_enabled and matrix_sygnal_database_engine == 'postgres' and matrix_sygnal_database_hostname == 'matrix-postgres') else []) + + + ([{ + 'name': matrix_prometheus_postgres_exporter_database_name, + 'username': matrix_prometheus_postgres_exporter_database_username, + 'password': matrix_prometheus_postgres_exporter_database_password, + }] if (matrix_prometheus_postgres_exporter_enabled and matrix_prometheus_postgres_exporter_database_hostname == 'matrix-postgres') else []) + }} matrix_postgres_import_roles_to_ignore: | @@ -1766,6 +1773,10 @@ matrix_prometheus_scraper_synapse_rules_synapse_tag: "{{ matrix_synapse_docker_i matrix_prometheus_scraper_node_enabled: "{{ matrix_prometheus_node_exporter_enabled }}" matrix_prometheus_scraper_node_targets: "{{ ['matrix-prometheus-node-exporter:9100'] if matrix_prometheus_node_exporter_enabled else [] }}" +matrix_prometheus_scraper_postgres_enabled: "{{ matrix_prometheus_postgres_exporter_enabled }}" +matrix_prometheus_scraper_postgres_targets: "{{ ['matrix-prometheus-postgres-exporter:'+ matrix_prometheus_postgres_exporter_port|string] if matrix_prometheus_scraper_postgres_enabled else [] }}" + + ###################################################################### # # /matrix-prometheus @@ -1773,6 +1784,27 @@ matrix_prometheus_scraper_node_targets: "{{ ['matrix-prometheus-node-exporter:91 ###################################################################### +###################################################################### +# +# matrix-prometheus-postgres-exporter +# +###################################################################### + +matrix_prometheus_postgres_exporter_enabled: false +matrix_prometheus_postgres_exporter_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'prometheus.pg.db') | to_uuid }}" + +matrix_prometheus_postgres_exporter_systemd_required_services_list: | + {{ + ['docker.service'] + + + (['matrix-postgres.service'] if matrix_postgres_enabled else []) + }} + +###################################################################### +# +# /matrix-prometheus-postgres-exporter +# +###################################################################### ###################################################################### # @@ -1787,6 +1819,14 @@ matrix_grafana_enabled: false # Grafana's HTTP port to the local host. matrix_grafana_container_http_host_bind_port: "{{ '' if matrix_nginx_proxy_enabled else '127.0.0.1:3000' }}" +matrix_grafana_dashboard_download_urls_all: | + {{ + matrix_grafana_dashboard_download_urls + + + (matrix_prometheus_postgres_exporter_dashboard_urls if matrix_prometheus_postgres_exporter_enabled else []) + }} + + ###################################################################### # # /matrix-grafana diff --git a/roles/matrix-grafana/tasks/setup.yml b/roles/matrix-grafana/tasks/setup.yml index c2eea3485..00d2e230d 100644 --- a/roles/matrix-grafana/tasks/setup.yml +++ b/roles/matrix-grafana/tasks/setup.yml @@ -64,7 +64,7 @@ mode: 0440 owner: "{{ matrix_user_username }}" group: "{{ matrix_user_groupname }}" - with_items: "{{ matrix_grafana_dashboard_download_urls }}" + with_items: "{{ matrix_grafana_dashboard_download_urls_all }}" when: matrix_grafana_enabled|bool - name: Ensure matrix-grafana.service installed diff --git a/roles/matrix-prometheus-postgres-exporter/defaults/main.yml b/roles/matrix-prometheus-postgres-exporter/defaults/main.yml new file mode 100644 index 000000000..8aca45762 --- /dev/null +++ b/roles/matrix-prometheus-postgres-exporter/defaults/main.yml @@ -0,0 +1,49 @@ +# matrix-prometheus-postgres-exporter is an Prometheus exporter for postgres metrics +# See: https://github.com/prometheus-community/postgres_exporter + +matrix_prometheus_postgres_exporter_enabled: false + +matrix_prometheus_postgres_exporter_version: v0.9.0 +matrix_prometheus_postgres_exporter_port: 9187 + +matrix_prometheus_postgres_exporter_docker_image: "quay.io/prometheuscommunity/postgres-exporter:{{ matrix_prometheus_postgres_exporter_version }}" +matrix_prometheus_postgres_exporter_docker_image_force_pull: "{{ matrix_prometheus_postgres_exporter_docker_image.endswith(':latest') }}" + +# A list of extra arguments to pass to the container +matrix_prometheus_postgres_exporter_container_extra_arguments: ["-e PG_EXPORTER_AUTO_DISCOVER_DATABASES=true", + "-e PG_EXPORTER_WEB_LISTEN_ADDRESS=\":{{matrix_prometheus_postgres_exporter_port}}\"", + "-e DATA_SOURCE_NAME=\"postgresql://{{matrix_prometheus_postgres_exporter_database_username}}:{{matrix_prometheus_postgres_exporter_database_password}}@{{matrix_prometheus_postgres_exporter_database_hostname}}:5432/{{matrix_prometheus_postgres_exporter_database_name}}?sslmode=disable\"" ] + +# List of systemd services that matrix-prometheus-postgres-exporter.service depends on +matrix_prometheus_postgres_exporter_systemd_required_services_list: ['docker.service'] + +# List of systemd services that matrix-prometheus-postgres-exporter.service wants +matrix_prometheus_postgres_exporter_systemd_wanted_services_list: [] + +# details for connecting to the database +matrix_prometheus_postgres_exporter_database_username: 'matrix_prometheus_postgres_exporter' +matrix_prometheus_postgres_exporter_database_password: 'some-password' +matrix_prometheus_postgres_exporter_database_hostname: 'matrix-postgres' +matrix_prometheus_postgres_exporter_database_port: 5432 +matrix_prometheus_postgres_exporter_database_name: 'matrix_prometheus_postgres_exporter' + + +# Controls whether the matrix-prometheus container exposes its HTTP port (tcp/9100 in the container). +# +# Takes an ":" value (e.g. "127.0.0.1:9100"), or empty string to not expose. +# +# Official recommendations are to run this container with `--net=host`, +# but we don't do that, since it: +# - likely exposes the metrics web server way too publicly (before applying https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/1008) +# - or listens on a loopback interface only (--net=host and 127.0.0.1:9100), which is not reachable from another container (like `matrix-prometheus`) +# +# Using `--net=host` and binding to Docker's `matrix` bridge network may be a solution to both, +# but that's trickier to accomplish and won't necessarily work (hasn't been tested). +# +# Not using `--net=host` means that our network statistic reports are likely broken (inaccurate), +# because node-exporter can't see all interfaces, etc. +# For now, we'll live with that, until someone develops a better solution. +matrix_prometheus_postgres_exporter_container_http_host_bind_port: '' + +matrix_prometheus_postgres_exporter_dashboard_urls: +- "https://grafana.com/api/dashboards/9628/revisions/7/download" \ No newline at end of file diff --git a/roles/matrix-prometheus-postgres-exporter/tasks/init.yml b/roles/matrix-prometheus-postgres-exporter/tasks/init.yml new file mode 100644 index 000000000..2bd6904ec --- /dev/null +++ b/roles/matrix-prometheus-postgres-exporter/tasks/init.yml @@ -0,0 +1,5 @@ +- set_fact: + matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-prometheus-postgres-exporter.service'] }}" + when: matrix_prometheus_postgres_exporter_enabled|bool + + diff --git a/roles/matrix-prometheus-postgres-exporter/tasks/main.yml b/roles/matrix-prometheus-postgres-exporter/tasks/main.yml new file mode 100644 index 000000000..e3c364fa9 --- /dev/null +++ b/roles/matrix-prometheus-postgres-exporter/tasks/main.yml @@ -0,0 +1,8 @@ +- import_tasks: "{{ role_path }}/tasks/init.yml" + tags: + - always + +- import_tasks: "{{ role_path }}/tasks/setup.yml" + tags: + - setup-all + - setup-prometheus-postgres-exporter diff --git a/roles/matrix-prometheus-postgres-exporter/tasks/setup.yml b/roles/matrix-prometheus-postgres-exporter/tasks/setup.yml new file mode 100644 index 000000000..076ece1a8 --- /dev/null +++ b/roles/matrix-prometheus-postgres-exporter/tasks/setup.yml @@ -0,0 +1,54 @@ +--- + +# +# Tasks related to setting up matrix-prometheus-postgres-exporter +# + +- name: Ensure matrix-prometheus-postgres-exporter image is pulled + docker_image: + name: "{{ matrix_prometheus_postgres_exporter_docker_image }}" + source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_prometheus_postgres_exporter_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_prometheus_postgres_exporter_docker_image_force_pull }}" + when: "matrix_prometheus_postgres_exporter_enabled|bool" + +- name: Ensure matrix-prometheus-postgres-exporter.service installed + template: + src: "{{ role_path }}/templates/systemd/matrix-prometheus-postgres-exporter.service.j2" + dest: "{{ matrix_systemd_path }}/matrix-prometheus-postgres-exporter.service" + mode: 0644 + register: matrix_prometheus_postgres_exporter_systemd_service_result + when: matrix_prometheus_postgres_exporter_enabled|bool + +- name: Ensure systemd reloaded after matrix-prometheus.service installation + service: + daemon_reload: yes + when: "matrix_prometheus_postgres_exporter_enabled|bool and matrix_prometheus_postgres_exporter_systemd_service_result.changed" + +# +# Tasks related to getting rid of matrix-prometheus-postgres-exporter (if it was previously enabled) +# + +- name: Check existence of matrix-prometheus-postgres-exporter service + stat: + path: "{{ matrix_systemd_path }}/matrix-prometheus-postgres-exporter.service" + register: matrix_prometheus_postgres_exporter_service_stat + +- name: Ensure matrix-prometheus-postgres-exporter is stopped + service: + name: matrix-prometheus-postgres-exporter + state: stopped + daemon_reload: yes + register: stopping_result + when: "not matrix_prometheus_postgres_exporter_enabled|bool and matrix_prometheus_postgres_exporter_service_stat.stat.exists" + +- name: Ensure matrix-prometheus-postgres-exporter.service doesn't exist + file: + path: "{{ matrix_systemd_path }}/matrix-prometheus-postgres-exporter.service" + state: absent + when: "not matrix_prometheus_postgres_exporter_enabled|bool and matrix_prometheus_postgres_exporter_service_stat.stat.exists" + +- name: Ensure systemd reloaded after matrix-prometheus-postgres-exporter.service removal + service: + daemon_reload: yes + when: "not matrix_prometheus_postgres_exporter_enabled|bool and matrix_prometheus_postgres_exporter_service_stat.stat.exists" diff --git a/roles/matrix-prometheus-postgres-exporter/templates/systemd/matrix-prometheus-postgres-exporter.service.j2 b/roles/matrix-prometheus-postgres-exporter/templates/systemd/matrix-prometheus-postgres-exporter.service.j2 new file mode 100644 index 000000000..b25cb5ded --- /dev/null +++ b/roles/matrix-prometheus-postgres-exporter/templates/systemd/matrix-prometheus-postgres-exporter.service.j2 @@ -0,0 +1,42 @@ +#jinja2: lstrip_blocks: "True" +[Unit] +Description=matrix-prometheus-postgres-exporter +{% for service in matrix_prometheus_postgres_exporter_systemd_required_services_list %} +Requires={{ service }} +After={{ service }} +{% endfor %} +{% for service in matrix_prometheus_postgres_exporter_systemd_wanted_services_list %} +Wants={{ service }} +{% endfor %} +DefaultDependencies=no + +[Service] +Type=simple +Environment="HOME={{ matrix_systemd_unit_home_path }}" +ExecStartPre=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} kill matrix-prometheus-postgres-exporter 2>/dev/null' +ExecStartPre=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} rm matrix-prometheus-postgres-exporter 2>/dev/null' + + +ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-prometheus-postgres-exporter \ + --log-driver=none \ + --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ + --cap-drop=ALL \ + --read-only \ + {% for arg in matrix_prometheus_postgres_exporter_container_extra_arguments %} + {{ arg }} \ + {% endfor %} + --network={{ matrix_docker_network }} \ + {% if matrix_prometheus_postgres_exporter_container_http_host_bind_port %} + -p {{ matrix_prometheus_postgres_exporter_container_http_host_bind_port }}:{{matrix_prometheus_postgres_exporter_port}} \ + {% endif %} + --pid=host \ + {{ matrix_prometheus_postgres_exporter_docker_image }} + +ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} kill matrix-prometheus-postgres-exporter 2>/dev/null' +ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} rm matrix-prometheus-postgres-exporter 2>/dev/null' +Restart=always +RestartSec=30 +SyslogIdentifier=matrix-prometheus-postgres-exporter + +[Install] +WantedBy=multi-user.target diff --git a/roles/matrix-prometheus/templates/prometheus.yml.j2 b/roles/matrix-prometheus/templates/prometheus.yml.j2 index 9502a08b6..b3ee3b86e 100644 --- a/roles/matrix-prometheus/templates/prometheus.yml.j2 +++ b/roles/matrix-prometheus/templates/prometheus.yml.j2 @@ -38,3 +38,9 @@ scrape_configs: static_configs: - targets: {{ matrix_prometheus_scraper_node_targets|to_json }} {% endif %} + + {% if matrix_prometheus_scraper_postgres_enabled %} + - job_name: postgres + static_configs: + - targets: {{ matrix_prometheus_scraper_postgres_targets|to_json }} + {% endif %} \ No newline at end of file diff --git a/setup.yml b/setup.yml index 9e3cd6254..142364c46 100755 --- a/setup.yml +++ b/setup.yml @@ -54,5 +54,5 @@ - matrix-coturn - matrix-aux - matrix-postgres-backup - - matrix-common-after - + - matrix-prometheus-postgres-exporter + - matrix-common-after \ No newline at end of file