From 70cbf3d5aea6edcf7ec5e73222a1a1f1f3299d97 Mon Sep 17 00:00:00 2001 From: Aine <97398200+aine-etke@users.noreply.github.com> Date: Thu, 15 Aug 2024 11:36:48 +0300 Subject: [PATCH] add synapse-auto-compressor workaround, fixes #3397 (#3473) * add synapse-auto-compressor workaround, fixes #3397 * Clarify what the PG-prefixed variables are for --------- Co-authored-by: Slavi Pantaleev --- group_vars/matrix_servers | 5 ++ .../defaults/main.yml | 7 +++ .../tasks/install.yml | 10 +++- .../templates/env.j2 | 6 +++ .../matrix-synapse-auto-compressor-fix.sh.j2 | 47 +++++++++++++++++++ .../matrix-synapse-auto-compressor.service.j2 | 3 +- 6 files changed, 76 insertions(+), 2 deletions(-) create mode 100644 roles/custom/matrix-synapse-auto-compressor/templates/matrix-synapse-auto-compressor-fix.sh.j2 diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index f25eed738..8a5fb1b6b 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -4514,6 +4514,11 @@ matrix_synapse_report_stats_endpoint: "http://{{ matrix_synapse_usage_exporter_i # Not enabled by default, for now matrix_synapse_auto_compressor_enabled: false +matrix_synapse_auto_compressor_uid: "{{ matrix_user_uid }}" +matrix_synapse_auto_compressor_gid: "{{ matrix_user_gid }}" + +matrix_synapse_auto_compressor_postgres_image: "{{ devture_postgres_container_image_to_use }}" + matrix_synapse_auto_compressor_container_image_self_build: "{{ matrix_architecture not in ['amd64'] }}" matrix_synapse_auto_compressor_container_network: "{{ (devture_postgres_container_network if (devture_postgres_enabled and matrix_synapse_auto_compressor_database_hostname == matrix_synapse_database_host and matrix_synapse_database_host == devture_postgres_connection_hostname) else 'matrix-synapse-auto-compressor') }}" diff --git a/roles/custom/matrix-synapse-auto-compressor/defaults/main.yml b/roles/custom/matrix-synapse-auto-compressor/defaults/main.yml index a321d888f..e80ce35f0 100644 --- a/roles/custom/matrix-synapse-auto-compressor/defaults/main.yml +++ b/roles/custom/matrix-synapse-auto-compressor/defaults/main.yml @@ -8,6 +8,13 @@ matrix_synapse_auto_compressor_enabled: true # renovate: datasource=docker depName=registry.gitlab.com/etke.cc/rust-synapse-compress-state matrix_synapse_auto_compressor_version: v0.1.4 +# note: UID/GID better to match the UID/GID of the postgres container, but it doesn't really matter, as volumes are not used here +matrix_synapse_auto_compressor_uid: '' +matrix_synapse_auto_compressor_gid: '' + +# note: the postgres image version doesn't matter, as it is used for a SQL script that doesn't use any specific features +matrix_synapse_auto_compressor_postgres_image: "postgres" + matrix_synapse_auto_compressor_base_path: "{{ matrix_base_data_path }}/synapse-auto-compressor" matrix_synapse_auto_compressor_container_src_files_path: "{{ matrix_synapse_auto_compressor_base_path }}/container-src" diff --git a/roles/custom/matrix-synapse-auto-compressor/tasks/install.yml b/roles/custom/matrix-synapse-auto-compressor/tasks/install.yml index bb26c0acf..4baeb3fb4 100644 --- a/roles/custom/matrix-synapse-auto-compressor/tasks/install.yml +++ b/roles/custom/matrix-synapse-auto-compressor/tasks/install.yml @@ -14,7 +14,7 @@ - path: "{{ matrix_synapse_auto_compressor_container_src_files_path }}" when: "{{ matrix_synapse_auto_compressor_container_image_self_build }}" -- name: Ensure synapse-auto-compressor labels installed +- name: Ensure synapse-auto-compressor env file is installed ansible.builtin.template: src: "{{ role_path }}/templates/env.j2" dest: "{{ matrix_synapse_auto_compressor_base_path }}/env" @@ -22,6 +22,14 @@ owner: "{{ matrix_user_username }}" group: "{{ matrix_user_groupname }}" +- name: Ensure synapse-auto-compressor workaround script is installed + ansible.builtin.template: + src: "{{ role_path }}/templates/matrix-synapse-auto-compressor-fix.sh.j2" + dest: "{{ matrix_synapse_auto_compressor_base_path }}/matrix-synapse-auto-compressor-fix.sh" + mode: 0750 + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_groupname }}" + - name: Ensure synapse-auto-compressor image is pulled community.docker.docker_image: name: "{{ matrix_synapse_auto_compressor_container_image }}" diff --git a/roles/custom/matrix-synapse-auto-compressor/templates/env.j2 b/roles/custom/matrix-synapse-auto-compressor/templates/env.j2 index 27fb1dd88..85f18066d 100644 --- a/roles/custom/matrix-synapse-auto-compressor/templates/env.j2 +++ b/roles/custom/matrix-synapse-auto-compressor/templates/env.j2 @@ -1 +1,7 @@ POSTGRES_LOCATION={{ matrix_synapse_auto_compressor_environment_variable_postgres_location }} + +# PG-prefixed variables below are for the matrix-synapse-auto-compressor-fix.sh script +PGHOST={{ matrix_synapse_auto_compressor_database_hostname }} +PGDATABASE={{ matrix_synapse_auto_compressor_database_name }} +PGUSER={{ matrix_synapse_auto_compressor_database_username }} +PGPASSWORD={{ matrix_synapse_auto_compressor_database_password }} diff --git a/roles/custom/matrix-synapse-auto-compressor/templates/matrix-synapse-auto-compressor-fix.sh.j2 b/roles/custom/matrix-synapse-auto-compressor/templates/matrix-synapse-auto-compressor-fix.sh.j2 new file mode 100644 index 000000000..87609dad6 --- /dev/null +++ b/roles/custom/matrix-synapse-auto-compressor/templates/matrix-synapse-auto-compressor-fix.sh.j2 @@ -0,0 +1,47 @@ +#jinja2: lstrip_blocks: "True" +#!/bin/bash + +# this script is used to workaround the https://github.com/matrix-org/rust-synapse-compress-state/issues/78, +# and it is based on postgres' cli-non-interactive and https://github.com/matrix-org/rust-synapse-compress-state/issues/78#issuecomment-1409932869 + +docker run \ + --rm \ + --user={{ matrix_synapse_auto_compressor_uid }}:{{ matrix_synapse_auto_compressor_gid }} \ + --cap-drop=ALL \ + --interactive \ + --network={{ matrix_synapse_auto_compressor_container_network }} \ + --env-file={{ matrix_synapse_auto_compressor_base_path }}/env \ + {{ matrix_synapse_auto_compressor_postgres_image }} \ + psql -h {{ matrix_synapse_auto_compressor_database_hostname }} \ + <<_EOF +BEGIN; + +DELETE +FROM state_compressor_state AS scs +WHERE NOT EXISTS + (SELECT * + FROM rooms AS r + WHERE r.room_id = scs.room_id); + +DELETE +FROM state_compressor_state AS scs +WHERE scs.room_id in + (SELECT DISTINCT room_id + FROM state_compressor_state AS scs2 + WHERE scs2.current_head IS NOT NULL + AND NOT EXISTS + (SELECT * + FROM state_groups AS sg + WHERE sg.id = scs2.current_head)); + +DELETE +FROM state_compressor_progress AS scp +WHERE NOT EXISTS + (SELECT * + FROM state_compressor_state AS scs + WHERE scs.room_id = scp.room_id); + +COMMIT; +_EOF + +# vim: ft=sh diff --git a/roles/custom/matrix-synapse-auto-compressor/templates/matrix-synapse-auto-compressor.service.j2 b/roles/custom/matrix-synapse-auto-compressor/templates/matrix-synapse-auto-compressor.service.j2 index d2f08a2d3..842f33670 100644 --- a/roles/custom/matrix-synapse-auto-compressor/templates/matrix-synapse-auto-compressor.service.j2 +++ b/roles/custom/matrix-synapse-auto-compressor/templates/matrix-synapse-auto-compressor.service.j2 @@ -15,6 +15,7 @@ Type=oneshot Environment="HOME={{ devture_systemd_docker_base_systemd_unit_home_path }}" ExecStartPre=-{{ devture_systemd_docker_base_host_command_sh }} -c '{{ devture_systemd_docker_base_host_command_docker }} stop --time={{ devture_systemd_docker_base_container_stop_grace_time_seconds }} matrix-synapse-auto-compressor 2>/dev/null || true' ExecStartPre=-{{ devture_systemd_docker_base_host_command_sh }} -c '{{ devture_systemd_docker_base_host_command_docker }} rm matrix-synapse-auto-compressor 2>/dev/null || true' +ExecStartPre=-{{ devture_systemd_docker_base_host_command_sh }} {{ matrix_synapse_auto_compressor_base_path }}/matrix-synapse-auto-compressor-fix.sh ExecStartPre={{ devture_systemd_docker_base_host_command_docker }} create \ --rm \ @@ -22,7 +23,7 @@ ExecStartPre={{ devture_systemd_docker_base_host_command_docker }} create \ --log-driver=none \ --cap-drop=ALL \ --read-only \ - --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ + --user={{ matrix_synapse_auto_compressor_uid }}:{{ matrix_synapse_auto_compressor_gid }} \ --network={{ matrix_synapse_auto_compressor_container_network }} \ --env-file={{ matrix_synapse_auto_compressor_base_path }}/env \ --entrypoint=/bin/sh \