Created role for synapse-usage-exporter (#3442)

* Created role for synapse-usage-exporter

* Apply suggestions from code review

Co-authored-by: Slavi Pantaleev <slavi@devture.com>

* Renaming docker variables and moving synapse stats config location

* Respect devture_systemd_docker_base_docker_service_name

---------

Co-authored-by: Slavi Pantaleev <slavi@devture.com>

roles/custom/matrix-synapse-usage-exporter/defaults/main.yml

@@ -0,0 +1,76 @@
+---
+
+# Synapse Usage Exporter
+# Project source code URL: https://github.com/loelkes/synapse-usage-exporter
+
+matrix_synapse_usage_exporter_enabled: false
+
+# matrix_synapse_usage_exporter_identifier controls the identifier of this synapse-usage-exporter instance, which influences:
+# - the default storage path
+# - the names of systemd services and containers
+matrix_synapse_usage_exporter_identifier: matrix-synapse-usage-exporter
+matrix_synapse_usage_exporter_container_port: 5000
+
+# No docker images are currently hosted for the repo, so defaulting to true
+matrix_synapse_usage_exporter_container_image_self_build: true
+matrix_synapse_usage_exporter_container_image_self_build_repo: "https://github.com/loelkes/synapse-usage-exporter.git"
+
+matrix_synapse_usage_exporter_container_image_path: "loelkes/synapse-usage-exporter"
+matrix_synapse_usage_exporter_container_image: "{{ matrix_synapse_usage_exporter_container_image_name_prefix }}{{ matrix_synapse_usage_exporter_container_image_path }}:{{ matrix_synapse_usage_exporter_container_image_tag }}"
+matrix_synapse_usage_exporter_container_image_name_prefix: "{{ 'localhost/' if matrix_synapse_usage_exporter_container_image_self_build else matrix_container_global_registry_prefix }}"
+matrix_synapse_usage_exporter_container_image_tag: "{{ 'main' if matrix_synapse_usage_exporter_container_image_self_build else 'latest' }}"
+matrix_synapse_usage_exporter_container_image_force_pull: "{{ matrix_synapse_usage_exporter_container_image.endswith(':latest') }}"
+
+matrix_synapse_usage_exporter_base_path: "{{ matrix_base_data_path }}/{{ matrix_synapse_usage_exporter_identifier }}"
+matrix_synapse_usage_exporter_docker_src_files_path: "{{ matrix_synapse_usage_exporter_base_path }}/docker-src"
+
+# List of systemd services that synapse-usage-exporter.service depends on
+matrix_synapse_usage_exporter_systemd_required_services_list: "{{ matrix_synapse_usage_exporter_systemd_required_services_list_default + matrix_synapse_usage_exporter_systemd_required_services_list_auto + matrix_synapse_usage_exporter_systemd_required_services_list_custom }}"
+matrix_synapse_usage_exporter_systemd_required_services_list_default: "{{ [devture_systemd_docker_base_docker_service_name] if devture_systemd_docker_base_docker_service_name else [] }}"
+matrix_synapse_usage_exporter_systemd_required_services_list_auto: []
+matrix_synapse_usage_exporter_systemd_required_services_list_custom: []
+
+# List of systemd services that synapse-usage-exporter.service wants
+matrix_synapse_usage_exporter_systemd_wanted_services_list: []
+
+# The base container network. It will be auto-created by this role if it doesn't exist already.
+matrix_synapse_usage_exporter_container_network: "{{ matrix_synapse_usage_exporter_identifier }}"
+
+# A list of additional container networks that the container would be connected to.
+# The role does not create these networks, so make sure they already exist.
+# Use this to expose this container to another reverse proxy, which runs in a different container network.
+matrix_synapse_usage_exporter_container_additional_networks: []
+
+# Extra arguments for the Docker container
+matrix_synapse_usage_exporter_container_extra_arguments: []
+
+# Controls whether the synapse usage exporter should be proxied (exposed) on `matrix.DOMAIN/report-usage-stats/push`
+matrix_synapse_usage_exporter_proxying_enabled: false
+
+# matrix_synapse_usage_exporter_container_labels_traefik_enabled controls whether labels to assist a Traefik reverse-proxy will be attached to the container.
+# See `../templates/labels.j2` for details.
+#
+# To inject your own other container labels, see `matrix_synapse_usage_exporter_container_labels_additional_labels`.
+matrix_synapse_usage_exporter_container_labels_traefik_enabled: "{{ matrix_synapse_usage_exporter_proxying_enabled }}"
+matrix_synapse_usage_exporter_container_labels_traefik_docker_network: "{{ matrix_synapse_usage_exporter_container_network }}"
+
+matrix_synapse_usage_exporter_container_labels_traefik_path_prefix: "/report-usage-stats/push"
+matrix_synapse_usage_exporter_container_labels_traefik_rule: "Host(`{{ matrix_server_fqn_matrix }}`) && PathPrefix(`{{ matrix_synapse_usage_exporter_container_labels_traefik_path_prefix | quote }}`)"
+matrix_synapse_usage_exporter_container_labels_traefik_priority: 0
+matrix_synapse_usage_exporter_container_labels_traefik_entrypoints: "web-secure"
+matrix_synapse_usage_exporter_container_labels_traefik_tls: "{{ matrix_synapse_usage_exporter_container_labels_traefik_entrypoints != 'web' }}"
+matrix_synapse_usage_exporter_container_labels_traefik_tls_certResolver: default  # noqa var-naming
+
+# matrix_synapse_usage_exporter_container_labels_additional_labels contains a multiline string with additional labels to add to the container label file.
+# See `../templates/labels.j2` for details.
+#
+# Example:
+# matrix_synapse_usage_exporter_container_labels_additional_labels: |
+#   my.label=1
+#   another.label="here"
+matrix_synapse_usage_exporter_container_labels_additional_labels: ''
+
+# matrix_synapse_usage_exporter_dashboard_urls contains a list of URLs with Grafana dashboard definitions.
+# If the Grafana role is enabled, these dashboards will be downloaded.
+matrix_synapse_usage_exporter_dashboard_urls:
+  - https://raw.githubusercontent.com/spantaleev/matrix-docker-ansible-deploy/master/roles/custom/matrix-synapse-usage-exporter/templates/grafana/synapse-usage-exporter.json

roles/custom/matrix-synapse-usage-exporter/tasks/main.yml

@@ -0,0 +1,17 @@
+---
+
+- tags:
+    - setup-all
+    - setup-synapse-usage-exporter
+    - install-all
+    - install-synapse-usage-exporter
+  block:
+    - when: matrix_synapse_usage_exporter_enabled | bool
+      ansible.builtin.include_tasks: "{{ role_path }}/tasks/setup_install.yml"
+
+- tags:
+    - setup-all
+    - setup-synapse-usage-exporter
+  block:
+    - when: not matrix_synapse_usage_exporter_enabled | bool
+      ansible.builtin.include_tasks: "{{ role_path }}/tasks/setup_uninstall.yml"

roles/custom/matrix-synapse-usage-exporter/tasks/setup_install.yml

@@ -0,0 +1,77 @@
+---
+
+- name: Ensure synapse-usage-exporter paths exist
+  ansible.builtin.file:
+    path: "{{ item.path }}"
+    state: directory
+    mode: 0750
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_groupname }}"
+  with_items:
+    - path: "{{ matrix_synapse_usage_exporter_base_path }}"
+      when: true
+    - path: "{{ matrix_synapse_usage_exporter_docker_src_files_path }}"
+      when: "{{ matrix_synapse_usage_exporter_container_image_self_build }}"
+  when: "item.when | bool"
+
+- name: Ensure synapse-usage-exporter support files installed
+  ansible.builtin.template:
+    src: "{{ role_path }}/templates/{{ item }}.j2"
+    dest: "{{ matrix_synapse_usage_exporter_base_path }}/{{ item }}"
+    mode: 0640
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_groupname }}"
+  with_items:
+    - env
+    - labels
+
+- name: Ensure synapse-usage-exporter Docker image is pulled
+  community.docker.docker_image:
+    name: "{{ matrix_synapse_usage_exporter_container_image }}"
+    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
+    force_source: "{{ matrix_synapse_usage_exporter_container_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
+    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_synapse_usage_exporter_container_image_force_pull }}"
+  when: "not matrix_synapse_usage_exporter_container_image_self_build | bool"
+  register: result
+  retries: "{{ devture_playbook_help_container_retries_count }}"
+  delay: "{{ devture_playbook_help_container_retries_delay }}"
+  until: result is not failed
+
+- when: "matrix_synapse_usage_exporter_container_image_self_build | bool"
+  block:
+    - name: Ensure synapse-usage-exporter repository is present on self-build
+      ansible.builtin.git:
+        repo: "{{ matrix_synapse_usage_exporter_container_image_self_build_repo }}"
+        dest: "{{ matrix_synapse_usage_exporter_docker_src_files_path }}"
+        version: "{{ matrix_synapse_usage_exporter_container_image.split(':')[1] }}"
+        force: "yes"
+      become: true
+      become_user: "{{ matrix_user_username }}"
+      register: matrix_synapse_usage_exporter_git_pull_results
+
+    - name: Check if synapse-usage-exporter Docker image exists
+      ansible.builtin.command: "{{ devture_systemd_docker_base_host_command_docker }} images --quiet --filter 'reference={{ matrix_synapse_usage_exporter_container_image }}'"
+      register: matrix_synapse_usage_exporter_container_image_check_result
+      changed_when: false
+
+    # Invoking the `docker build` command here, instead of calling the `docker_image` Ansible module,
+    # because the latter does not support BuildKit.
+    # See: https://github.com/ansible-collections/community.general/issues/514
+    - name: Ensure synapse-usage-exporter Docker image is built
+      ansible.builtin.command:
+        cmd: "{{ devture_systemd_docker_base_host_command_docker }} build -t {{ matrix_synapse_usage_exporter_container_image }} -f {{ matrix_synapse_usage_exporter_docker_src_files_path }}/docker/Dockerfile {{ matrix_synapse_usage_exporter_docker_src_files_path }}"
+      environment:
+        DOCKER_BUILDKIT: 1
+      changed_when: true
+      when: "matrix_synapse_usage_exporter_git_pull_results.changed | bool or matrix_synapse_usage_exporter_container_image_check_result.stdout == ''"
+
+- name: Ensure synapse-usage-exporter container network is created
+  community.general.docker_network:
+    name: "{{ matrix_synapse_usage_exporter_container_network }}"
+    driver: bridge
+
+- name: Ensure synapse-usage-exporter service installed
+  ansible.builtin.template:
+    src: "{{ role_path }}/templates/systemd/matrix-synapse-usage-exporter.service.j2"
+    dest: "{{ devture_systemd_docker_base_systemd_path }}/{{ matrix_synapse_usage_exporter_identifier }}.service"
+    mode: 0640

roles/custom/matrix-synapse-usage-exporter/tasks/setup_uninstall.yml

@@ -0,0 +1,24 @@
+---
+
+- name: Check existence of synapse-usage-exporter service
+  ansible.builtin.stat:
+    path: "{{ devture_systemd_docker_base_systemd_path }}/{{ matrix_synapse_usage_exporter_identifier }}.service"
+  register: matrix_synapse_usage_exporter_service_stat
+
+- when: matrix_synapse_usage_exporter_service_stat.stat.exists | bool
+  block:
+    - name: Ensure synapse-usage-exporter is stopped
+      ansible.builtin.systemd:
+        name: "{{ matrix_synapse_usage_exporter_identifier }}"
+        state: stopped
+        daemon_reload: true
+
+    - name: Ensure synapse-usage-exporter service doesn't exist
+      ansible.builtin.file:
+        path: "{{ devture_systemd_docker_base_systemd_path }}/{{ matrix_synapse_usage_exporter_identifier }}.service"
+        state: absent
+
+    - name: Ensure synapse-usage-exporter files deleted
+      ansible.builtin.file:
+        path: "{{ matrix_synapse_usage_exporter_base_path }}"
+        state: absent

roles/custom/matrix-synapse-usage-exporter/templates/env.j2

@@ -0,0 +1,3 @@
+PROMETHEUS_MULTIPROC_DIR=/tmp/prometheus
+WERKZEUG_LOG_LEVEL=INFO
+APP_LOG_LEVEL=INFO

roles/custom/matrix-synapse-usage-exporter/templates/labels.j2

@@ -0,0 +1,38 @@
+{% if matrix_synapse_usage_exporter_container_labels_traefik_enabled %}
+traefik.enable=true
+
+{% if matrix_synapse_usage_exporter_container_labels_traefik_docker_network %}
+traefik.docker.network={{ matrix_synapse_usage_exporter_container_labels_traefik_docker_network }}
+{% endif %}
+
+traefik.http.services.matrix-synapse-usage-exporter.loadbalancer.server.port={{ matrix_synapse_usage_exporter_container_port }}
+
+############################################################
+#                                                          #
+# Report Usage Stats (/report-usage-stats/push)            #
+#                                                          #
+############################################################
+
+traefik.http.routers.matrix-synapse-usage-exporter.rule={{ matrix_synapse_usage_exporter_container_labels_traefik_rule }}
+
+{% if matrix_synapse_usage_exporter_container_labels_traefik_priority | int > 0 %}
+traefik.http.routers.matrix-synapse-usage-exporter.priority={{ matrix_synapse_usage_exporter_container_labels_traefik_priority }}
+{% endif %}
+
+traefik.http.routers.matrix-synapse-usage-exporter.service=matrix-synapse-usage-exporter
+traefik.http.routers.matrix-synapse-usage-exporter.entrypoints={{ matrix_synapse_usage_exporter_container_labels_traefik_entrypoints }}
+
+traefik.http.routers.matrix-synapse-usage-exporter.tls={{ matrix_synapse_usage_exporter_container_labels_traefik_tls | to_json }}
+{% if matrix_synapse_usage_exporter_container_labels_traefik_tls %}
+traefik.http.routers.matrix-synapse-usage-exporter.tls.certResolver={{ matrix_synapse_usage_exporter_container_labels_traefik_tls_certResolver }}
+{% endif %}
+
+############################################################
+#                                                          #
+# /Report Usage Stats (/report-usage-stats/push)           #
+#                                                          #
+############################################################
+
+{% endif %}
+
+{{ matrix_synapse_usage_exporter_container_labels_additional_labels }}

roles/custom/matrix-synapse-usage-exporter/templates/systemd/matrix-synapse-usage-exporter.service.j2

@@ -0,0 +1,47 @@
+#jinja2: lstrip_blocks: "True"
+[Unit]
+Description=Matrix synapse-usage-exporter
+{% for service in matrix_synapse_usage_exporter_systemd_required_services_list %}
+Requires={{ service }}
+After={{ service }}
+{% endfor %}
+{% for service in matrix_synapse_usage_exporter_systemd_wanted_services_list %}
+Wants={{ service }}
+{% endfor %}
+DefaultDependencies=no
+
+[Service]
+Type=simple
+Environment="HOME={{ devture_systemd_docker_base_systemd_unit_home_path }}"
+ExecStartPre=-{{ devture_systemd_docker_base_host_command_sh }} -c '{{ devture_systemd_docker_base_host_command_docker }} stop --time={{ devture_systemd_docker_base_container_stop_grace_time_seconds }} {{ matrix_synapse_usage_exporter_identifier }} 2>/dev/null || true'
+ExecStartPre=-{{ devture_systemd_docker_base_host_command_sh }} -c '{{ devture_systemd_docker_base_host_command_docker }} rm {{ matrix_synapse_usage_exporter_identifier }} 2>/dev/null || true'
+
+ExecStartPre={{ devture_systemd_docker_base_host_command_docker }} create \
+			--rm \
+			--name={{ matrix_synapse_usage_exporter_identifier }} \
+			--log-driver=none \
+			--user={{ matrix_user_uid }}:{{ matrix_user_gid }} \
+			--cap-drop=ALL \
+			--network={{ matrix_synapse_usage_exporter_container_network }} \
+			--env-file={{ matrix_synapse_usage_exporter_base_path }}/env \
+			--label-file={{ matrix_synapse_usage_exporter_base_path }}/labels \
+			{% for arg in matrix_synapse_usage_exporter_container_extra_arguments %}
+			{{ arg }} \
+			{% endfor %}
+			{{ matrix_synapse_usage_exporter_container_image }}
+
+{% for network in matrix_synapse_usage_exporter_container_additional_networks %}
+ExecStartPre={{ devture_systemd_docker_base_host_command_docker }} network connect {{ network }} {{ matrix_synapse_usage_exporter_identifier }}
+{% endfor %}
+
+ExecStart={{ devture_systemd_docker_base_host_command_docker }} start --attach {{ matrix_synapse_usage_exporter_identifier }}
+
+ExecStop=-{{ devture_systemd_docker_base_host_command_sh }} -c '{{ devture_systemd_docker_base_host_command_docker }} stop --time={{ devture_systemd_docker_base_container_stop_grace_time_seconds }} {{ matrix_synapse_usage_exporter_identifier }} 2>/dev/null || true'
+ExecStop=-{{ devture_systemd_docker_base_host_command_sh }} -c '{{ devture_systemd_docker_base_host_command_docker }} rm {{ matrix_synapse_usage_exporter_identifier }} 2>/dev/null || true'
+ExecReload={{ devture_systemd_docker_base_host_command_docker }} exec {{ matrix_synapse_usage_exporter_identifier }} /bin/sh -c 'kill -HUP 1'
+Restart=always
+RestartSec=30
+SyslogIdentifier={{ matrix_synapse_usage_exporter_identifier }}
+
+[Install]
+WantedBy=multi-user.target