From db70230ae16ff8351450b21d007f336003ff873e Mon Sep 17 00:00:00 2001 From: Charles Wright Date: Tue, 16 Jan 2024 09:17:24 -0600 Subject: [PATCH] Add room-workers as a new preset, with new room workers, sync workers, client readers, and federation readers. Based on https://tcpipuk.github.io/synapse/index.html --- group_vars/matrix_servers | 5 ++ .../defaults/main.yml | 5 ++ ...ix-synapse-reverse-proxy-companion.conf.j2 | 81 ++++++++++++++----- roles/custom/matrix-synapse/defaults/main.yml | 47 +++++++++++ .../tasks/synapse/workers/init.yml | 52 ++++++++++++ roles/custom/matrix-synapse/vars/main.yml | 19 +++++ 6 files changed, 190 insertions(+), 19 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 3ebe649be..d4f4ea1bc 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -4081,6 +4081,11 @@ matrix_synapse_reverse_proxy_companion_container_labels_internal_client_api_trae matrix_synapse_reverse_proxy_companion_synapse_workers_enabled: "{{ matrix_synapse_workers_enabled }}" matrix_synapse_reverse_proxy_companion_synapse_workers_list: "{{ matrix_synapse_workers_enabled_list }}" +matrix_synapse_reverse_proxy_companion_synapse_room_worker_client_server_locations: "{{ matrix_synapse_workers_room_worker_client_server_endpoints }}" +matrix_synapse_reverse_proxy_companion_synapse_room_worker_federation_locations: "{{ matrix_synapse_workers_room_worker_federation_endpoints }}" +matrix_synapse_reverse_proxy_companion_synapse_sync_worker_client_server_locations: "{{ matrix_synapse_workers_sync_worker_client_server_endpoints }}" +matrix_synapse_reverse_proxy_companion_synapse_client_reader_client_server_locations: "{{ matrix_synapse_workers_client_reader_client_server_endpoints }}" +matrix_synapse_reverse_proxy_companion_synapse_federation_reader_federation_locations: "{{ matrix_synapse_workers_federation_reader_federation_endpoints }}" matrix_synapse_reverse_proxy_companion_synapse_generic_worker_client_server_locations: "{{ matrix_synapse_workers_generic_worker_client_server_endpoints }}" matrix_synapse_reverse_proxy_companion_synapse_generic_worker_federation_locations: "{{ matrix_synapse_workers_generic_worker_federation_endpoints }}" matrix_synapse_reverse_proxy_companion_synapse_stream_writer_typing_stream_worker_client_server_locations: "{{ matrix_synapse_workers_stream_writer_typing_stream_worker_client_server_endpoints }}" diff --git a/roles/custom/matrix-synapse-reverse-proxy-companion/defaults/main.yml b/roles/custom/matrix-synapse-reverse-proxy-companion/defaults/main.yml index 7a3f4f709..3f19e3a2b 100644 --- a/roles/custom/matrix-synapse-reverse-proxy-companion/defaults/main.yml +++ b/roles/custom/matrix-synapse-reverse-proxy-companion/defaults/main.yml @@ -206,6 +206,11 @@ matrix_synapse_reverse_proxy_companion_synapse_federation_api_additional_server_ # synapse worker activation and endpoint mappings matrix_synapse_reverse_proxy_companion_synapse_workers_enabled: false matrix_synapse_reverse_proxy_companion_synapse_workers_list: [] +matrix_synapse_reverse_proxy_companion_synapse_room_worker_client_server_locations: [] +matrix_synapse_reverse_proxy_companion_synapse_room_worker_federation_locations: [] +matrix_synapse_reverse_proxy_companion_synapse_sync_worker_client_server_locations: [] +matrix_synapse_reverse_proxy_companion_synapse_client_reader_client_server_locations: [] +matrix_synapse_reverse_proxy_companion_synapse_federation_reader_federation_locations: [] matrix_synapse_reverse_proxy_companion_synapse_generic_worker_client_server_locations: [] matrix_synapse_reverse_proxy_companion_synapse_generic_worker_federation_locations: [] matrix_synapse_reverse_proxy_companion_synapse_stream_writer_typing_stream_worker_client_server_locations: [] diff --git a/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2 b/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2 index c94855d59..c3164e963 100644 --- a/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2 +++ b/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2 @@ -1,5 +1,9 @@ #jinja2: lstrip_blocks: "True" +{% set room_workers = matrix_synapse_reverse_proxy_companion_synapse_workers_list | selectattr('type', 'equalto', 'room_worker') | list %} +{% set sync_workers = matrix_synapse_reverse_proxy_companion_synapse_workers_list | selectattr('type', 'equalto', 'sync_worker') | list %} +{% set client_reader_workers = matrix_synapse_reverse_proxy_companion_synapse_workers_list | selectattr('type', 'equalto', 'client_reader') | list %} +{% set federation_reader_workers = matrix_synapse_reverse_proxy_companion_synapse_workers_list | selectattr('type', 'equalto', 'federation_reader') | list %} {% set generic_workers = matrix_synapse_reverse_proxy_companion_synapse_workers_list | selectattr('type', 'equalto', 'generic_worker') | list %} {% set stream_writer_typing_stream_workers = matrix_synapse_reverse_proxy_companion_synapse_workers_list | selectattr('type', 'equalto', 'stream_writer') | selectattr('stream_writer_stream', 'equalto', 'typing') | list %} {% set stream_writer_to_device_stream_workers = matrix_synapse_reverse_proxy_companion_synapse_workers_list | selectattr('type', 'equalto', 'stream_writer') | selectattr('stream_writer_stream', 'equalto', 'to_device') | list %} @@ -9,9 +13,31 @@ {% set media_repository_workers = matrix_synapse_reverse_proxy_companion_synapse_workers_list | selectattr('type', 'equalto', 'media_repository') | list %} {% set user_dir_workers = matrix_synapse_reverse_proxy_companion_synapse_workers_list | selectattr('type', 'equalto', 'user_dir') | list %} -{% macro render_worker_upstream(name, workers) %} + +# Maps from https://tcpipuk.github.io/synapse/deployment/nginx.html#mapsconf +# Client username from MXID +map $http_authorization $mxid_localpart { + default $http_authorization; + "~Bearer syt_(?.*?)_.*" $username; + "" $accesstoken_from_urlparam; +} + +# Whether to upgrade HTTP connection +map $http_upgrade $connection_upgrade { + default upgrade; + '' close; +} + +#Extract room name from URI +map $request_uri $room_name { + ~^/_matrix/(client|federation)/.*?(?:%21|!)(?[A-Za-z0-9._=\-\/]+)(?::|%3A)[A-Za-z0-9._=\-\/]+ $room; +} +# End maps + +{% macro render_worker_upstream(name, workers, load_balance) %} {% if workers | length > 0 %} upstream {{ name }} { + {{ load_balance }} keepalive {{ workers | length * 2 }}; {% for worker in workers %} server "{{ worker.name }}:{{ worker.port }}"; @@ -37,28 +63,22 @@ {% endif %} # Round Robin "upstream" pools for workers - {% if generic_workers |length > 0 %} - upstream generic_workers_upstream { - # ensures that requests from the same client will always be passed - # to the same server (except when this server is unavailable) - hash $http_x_forwarded_for; - keepalive {{ generic_workers | length * 2 }}; + {{ render_worker_upstream('room_workers_upstream', room_workers, 'hash $room_name consistent;') + {{ render_worker_upstream('sync_workers_upstream', room_workers, 'hash $mxid_localpart consistent;') + {{ render_worker_upstream('client_reader_workers_upstream', client_reader_workers, 'least_conn;') + {{ render_worker_upstream('federation_reader_workers_upstream', federation_reader_workers, 'hash $http_x_forwarded_for;') }} - {% for worker in generic_workers %} - server "{{ worker.name }}:{{ worker.port }}"; - {% endfor %} - } - {% endif %} + {{ render_worker_upstream('generic_workers_upstream', generic_workers, 'hash $http_x_forwarded_for;') }} - {{ render_worker_upstream('stream_writer_typing_stream_workers_upstream', stream_writer_typing_stream_workers) }} - {{ render_worker_upstream('stream_writer_to_device_stream_workers_upstream', stream_writer_to_device_stream_workers) }} - {{ render_worker_upstream('stream_writer_account_data_stream_workers_upstream', stream_writer_account_data_stream_workers) }} - {{ render_worker_upstream('stream_writer_receipts_stream_workers_upstream', stream_writer_receipts_stream_workers) }} - {{ render_worker_upstream('stream_writer_presence_stream_workers_upstream', stream_writer_presence_stream_workers) }} + {{ render_worker_upstream('stream_writer_typing_stream_workers_upstream', stream_writer_typing_stream_workers, '') }} + {{ render_worker_upstream('stream_writer_to_device_stream_workers_upstream', stream_writer_to_device_stream_workers, '') }} + {{ render_worker_upstream('stream_writer_account_data_stream_workers_upstream', stream_writer_account_data_stream_workers, '') }} + {{ render_worker_upstream('stream_writer_receipts_stream_workers_upstream', stream_writer_receipts_stream_workers, '') }} + {{ render_worker_upstream('stream_writer_presence_stream_workers_upstream', stream_writer_presence_stream_workers, '') }} - {{ render_worker_upstream('media_repository_workers_upstream', media_repository_workers) }} + {{ render_worker_upstream('media_repository_workers_upstream', media_repository_workers, 'least_conn;') }} - {{ render_worker_upstream('user_dir_workers_upstream', user_dir_workers) }} + {{ render_worker_upstream('user_dir_workers_upstream', user_dir_workers, '') }} {% endif %} server { @@ -74,6 +94,21 @@ server { {% if matrix_synapse_reverse_proxy_companion_synapse_workers_enabled %} {# Workers redirects BEGIN #} + {% if room_workers | length > 0 %} + # https://tcpipuk.github.io/synapse/deployment/workers.html + {{ render_locations_to_upstream(matrix_synapse_reverse_proxy_companion_synapse_room_worker_client_server_locations, 'room_workers_upstream') }} + {% endif %} + + {% if sync_workers | length > 0 %} + # https://tcpipuk.github.io/synapse/deployment/workers.html + {{ render_locations_to_upstream(matrix_synapse_reverse_proxy_companion_synapse_sync_worker_client_server_locations, 'sync_workers_upstream') }} + {% endif %} + + {% if client_reader_workers | length > 0 %} + # https://tcpipuk.github.io/synapse/deployment/workers.html + {{ render_locations_to_upstream(matrix_synapse_reverse_proxy_companion_synapse_client_reader_client_server_locations, 'client_reader_workers_upstream') }} + {% endif %} + {% if generic_workers | length > 0 %} # https://matrix-org.github.io/synapse/latest/workers.html#synapseappgeneric_worker {{ render_locations_to_upstream(matrix_synapse_reverse_proxy_companion_synapse_generic_worker_client_server_locations, 'generic_workers_upstream') }} @@ -165,6 +200,14 @@ server { gzip_types text/plain application/json; {% if matrix_synapse_reverse_proxy_companion_synapse_workers_enabled %} + {% if room_workers | length > 0 %} + # https://tcpipuk.github.io/synapse/deployment/workers.html + {{ render_locations_to_upstream(matrix_synapse_reverse_proxy_companion_synapse_room_worker_federation_locations, 'room_workers_upstream') }} + {% endif %} + {% if federation_reader_workers | length > 0 %} + # https://tcpipuk.github.io/synapse/deployment/workers.html + {{ render_locations_to_upstream(matrix_synapse_reverse_proxy_companion_synapse_federation_reader_federation_locations, 'federation_reader_workers_upstream') }} + {% endif %} {% if generic_workers | length > 0 %} # https://matrix-org.github.io/synapse/latest/workers.html#synapseappgeneric_worker {{ render_locations_to_upstream(matrix_synapse_reverse_proxy_companion_synapse_generic_worker_federation_locations, 'generic_workers_upstream') }} diff --git a/roles/custom/matrix-synapse/defaults/main.yml b/roles/custom/matrix-synapse/defaults/main.yml index 4b1513fa5..7dd7ec227 100644 --- a/roles/custom/matrix-synapse/defaults/main.yml +++ b/roles/custom/matrix-synapse/defaults/main.yml @@ -666,6 +666,10 @@ matrix_synapse_workers_preset: one-of-each matrix_synapse_workers_presets: little-federation-helper: + room_workers_count: 0 + sync_workers_count: 0 + client_reader_workers_count: 0 + federation_reader_workers_count: 0 generic_workers_count: 0 pusher_workers_count: 0 federation_sender_workers_count: 1 @@ -680,6 +684,10 @@ matrix_synapse_workers_presets: stream_writer_receipts_stream_workers_count: 0 stream_writer_presence_stream_workers_count: 0 one-of-each: + room_workers_count: 0 + sync_workers_count: 0 + client_reader_workers_count: 0 + federation_reader_workers_count: 0 generic_workers_count: 1 pusher_workers_count: 1 federation_sender_workers_count: 1 @@ -693,6 +701,24 @@ matrix_synapse_workers_presets: stream_writer_account_data_stream_workers_count: 1 stream_writer_receipts_stream_workers_count: 1 stream_writer_presence_stream_workers_count: 1 + room-workers: + room_workers_count: 1 + sync_workers_count: 1 + client_reader_workers_count: 1 + federation_reader_workers_count: 1 + generic_workers_count: 0 + pusher_workers_count: 1 + federation_sender_workers_count: 1 + media_repository_workers_count: 1 + appservice_workers_count: 1 + user_dir_workers_count: 1 + background_workers_count: 1 + stream_writer_events_stream_workers_count: 1 + stream_writer_typing_stream_workers_count: 1 + stream_writer_to_device_stream_workers_count: 1 + stream_writer_account_data_stream_workers_count: 1 + stream_writer_receipts_stream_workers_count: 1 + stream_writer_presence_stream_workers_count: 1 # Controls whether the matrix-synapse container exposes the various worker ports # (see `port` and `metrics_port` in `matrix_synapse_workers_enabled_list`) outside of the container. @@ -733,6 +759,27 @@ matrix_synapse_worker_container_labels_public_metrics_middleware_basic_auth_user # another.label="here" matrix_synapse_worker_container_labels_additional_labels: '' +# Room workers +matrix_synapse_workers_room_workers_count: "{{ matrix_synapse_workers_presets[matrix_synapse_workers_preset]['room_workers_count'] }}" +matrix_synapse_workers_room_workers_port_range_start: 28111 +matrix_synapse_workers_room_workers_metrics_range_start: 29111 + +# Sync workers +matrix_synapse_workers_sync_workers_count: "{{ matrix_synapse_workers_presets[matrix_synapse_workers_preset]['sync_workers_count'] }}" +matrix_synapse_workers_sync_workers_port_range_start: 28211 +matrix_synapse_workers_sync_workers_metrics_range_start: 29211 + +# Client reader workers +matrix_synapse_workers_client_reader_workers_count: "{{ matrix_synapse_workers_presets[matrix_synapse_workers_preset]['client_reader_workers_count'] }}" +matrix_synapse_workers_client_reader_workers_port_range_start: 28311 +matrix_synapse_workers_client_reader_workers_metrics_range_start: 29311 + +# Federation reader workers +matrix_synapse_workers_federation_reader_workers_count: "{{ matrix_synapse_workers_presets[matrix_synapse_workers_preset]['federation_reader_workers_count'] }}" +matrix_synapse_workers_federation_reader_workers_port_range_start: 28411 +matrix_synapse_workers_federation_reader_workers_metrics_range_start: 29411 + +# Generic workers matrix_synapse_workers_generic_workers_count: "{{ matrix_synapse_workers_presets[matrix_synapse_workers_preset]['generic_workers_count'] }}" matrix_synapse_workers_generic_workers_port_range_start: 18111 matrix_synapse_workers_generic_workers_metrics_range_start: 19111 diff --git a/roles/custom/matrix-synapse/tasks/synapse/workers/init.yml b/roles/custom/matrix-synapse/tasks/synapse/workers/init.yml index 68b8f37e3..a1bc4cc72 100644 --- a/roles/custom/matrix-synapse/tasks/synapse/workers/init.yml +++ b/roles/custom/matrix-synapse/tasks/synapse/workers/init.yml @@ -4,6 +4,58 @@ # set_fact within a loop does not work reliably in Ansible (it only executes on the first iteration for some reason), # so we're forced to do something much uglier. +- name: Build room workers + ansible.builtin.set_fact: + worker: + id: "room-worker-{{ item }}" + name: "matrix-synapse-worker-room-{{ item }}" + type: 'room_worker' + app: 'generic_worker' + webserving: true + port: "{{ matrix_synapse_workers_room_workers_port_range_start + item }}" + metrics_port: "{{ matrix_synapse_workers_room_workers_metrics_range_start + item }}" + register: "matrix_synapse_workers_list_results_room_workers" + loop: "{{ range(0, matrix_synapse_workers_room_workers_count | int) | list }}" + +- name: Build client sync workers + ansible.builtin.set_fact: + worker: + id: "sync-worker-{{ item }}" + name: "matrix-synapse-worker-sync-{{ item }}" + type: 'sync_worker' + app: 'generic_worker' + webserving: true + port: "{{ matrix_synapse_workers_sync_workers_port_range_start + item }}" + metrics_port: "{{ matrix_synapse_workers_sync_workers_metrics_range_start + item }}" + register: "matrix_synapse_workers_list_results_sync_workers" + loop: "{{ range(0, matrix_synapse_workers_sync_workers_count | int) | list }}" + +- name: Build client reader workers + ansible.builtin.set_fact: + worker: + id: "client-reader-{{ item }}" + name: "matrix-synapse-worker-client-reader-{{ item }}" + type: 'client_reader' + app: 'generic_worker' + webserving: true + port: "{{ matrix_synapse_workers_client_reader_workers_port_range_start + item }}" + metrics_port: "{{ matrix_synapse_workers_client_reader_workers_metrics_range_start + item }}" + register: "matrix_synapse_workers_list_results_client_reader_workers" + loop: "{{ range(0, matrix_synapse_workers_client_reader_workers_count | int) | list }}" + +- name: Build federation reader workers + ansible.builtin.set_fact: + worker: + id: "federation-reader-{{ item }}" + name: "matrix-synapse-worker-federation-reader-{{ item }}" + type: 'federation_reader' + app: 'generic_worker' + webserving: true + port: "{{ matrix_synapse_workers_federation_reader_workers_port_range_start + item }}" + metrics_port: "{{ matrix_synapse_workers_federation_reader_workers_metrics_range_start + item }}" + register: "matrix_synapse_workers_list_results_federation_reader_workers" + loop: "{{ range(0, matrix_synapse_workers_federation_reader_workers_count | int) | list }}" + - name: Build generic workers ansible.builtin.set_fact: worker: diff --git a/roles/custom/matrix-synapse/vars/main.yml b/roles/custom/matrix-synapse/vars/main.yml index 137a59737..63ad48550 100644 --- a/roles/custom/matrix-synapse/vars/main.yml +++ b/roles/custom/matrix-synapse/vars/main.yml @@ -8,6 +8,25 @@ matrix_synapse_media_store_directory_name: "{{ matrix_synapse_media_store_path | # Optionally: `false` to fully disable tls on outbound smtp matrix_synapse_email_smtp_enable_tls: true +# Room workers handle any URL that contains a room id, either through the client-server API or the federation API +# - see https://tcpipuk.github.io/synapse/deployment/nginx.html#locationsconf +matrix_synapse_workers_room_worker_client_server_endpoints: + - ^/_matrix/client/.*?!(?[A-Za-z0-9._=\-\/]+):[A-Za-z0-9.\-]+ +matrix_synapse_workers_room_worker_federation_endpoints: + - ^/_matrix/federation/v[12]/(?:state_ids|get_missing_events)/(?:%21|!)(?[A-Za-z0-9._=\-\/]+)(:|%3A)[A-Za-z0-9.\-]+ + +# Sync workers handle /sync and the (now deprecated) related endpoints +matrix_synapse_workers_sync_worker_client_server_endpoints: + - ^/_matrix/client/(api/v1|r0|v3|unstable)/(sync|events|initialSync|rooms/[^/]+/initialSync)$ + +# Client reader workers handle generic client-server endpoints that don't contain a roomid or sync +matrix_synapse_workers_client_reader_client_server_endpoints: + - ^/_matrix/client/(api/v1|r0|v3|unstable)/(room_keys/|keys/(query|changes|claim|upload/|room_keys/)|login|register(/available|/m.login.registration_token/validity|)|password_policy|profile|rooms/.*/(joined_members|context/.*|members|state|hierarchy|relations/|event/|aliases|timestamp_to_event|redact|send|state/|(join|invite|leave|ban|unban|kick))|createRoom|publicRooms|account/(3pid|whoami|devices)|versions|voip/turnServer|joined_rooms|search|user/.*/filter(/|$)|directory/room/.*|capabilities) + +# Federation reader workers handle generic federation endpoints that don't contain a roomid +matrix_synapse_workers_federation_reader_federation_endpoints: + - ^/_matrix/(federation/(v1|v2)|key/v2)/ + # A Synapse generic worker can handle both federation and client-server API endpoints. # We wish to split these, as we normally serve federation separately and don't want them mixed up. #