From db70230ae16ff8351450b21d007f336003ff873e Mon Sep 17 00:00:00 2001
From: Charles Wright <cvwright@futo.org>
Date: Tue, 16 Jan 2024 09:17:24 -0600
Subject: [PATCH 01/10] Add room-workers as a new preset, with new room
 workers, sync workers, client readers, and federation readers.  Based on
 https://tcpipuk.github.io/synapse/index.html

---
 group_vars/matrix_servers                     |  5 ++
 .../defaults/main.yml                         |  5 ++
 ...ix-synapse-reverse-proxy-companion.conf.j2 | 81 ++++++++++++++-----
 roles/custom/matrix-synapse/defaults/main.yml | 47 +++++++++++
 .../tasks/synapse/workers/init.yml            | 52 ++++++++++++
 roles/custom/matrix-synapse/vars/main.yml     | 19 +++++
 6 files changed, 190 insertions(+), 19 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index 3ebe649be..d4f4ea1bc 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -4081,6 +4081,11 @@ matrix_synapse_reverse_proxy_companion_container_labels_internal_client_api_trae
 
 matrix_synapse_reverse_proxy_companion_synapse_workers_enabled: "{{ matrix_synapse_workers_enabled }}"
 matrix_synapse_reverse_proxy_companion_synapse_workers_list: "{{ matrix_synapse_workers_enabled_list }}"
+matrix_synapse_reverse_proxy_companion_synapse_room_worker_client_server_locations: "{{ matrix_synapse_workers_room_worker_client_server_endpoints }}"
+matrix_synapse_reverse_proxy_companion_synapse_room_worker_federation_locations: "{{ matrix_synapse_workers_room_worker_federation_endpoints }}"
+matrix_synapse_reverse_proxy_companion_synapse_sync_worker_client_server_locations: "{{ matrix_synapse_workers_sync_worker_client_server_endpoints }}"
+matrix_synapse_reverse_proxy_companion_synapse_client_reader_client_server_locations: "{{ matrix_synapse_workers_client_reader_client_server_endpoints }}"
+matrix_synapse_reverse_proxy_companion_synapse_federation_reader_federation_locations: "{{ matrix_synapse_workers_federation_reader_federation_endpoints }}"
 matrix_synapse_reverse_proxy_companion_synapse_generic_worker_client_server_locations: "{{ matrix_synapse_workers_generic_worker_client_server_endpoints }}"
 matrix_synapse_reverse_proxy_companion_synapse_generic_worker_federation_locations: "{{ matrix_synapse_workers_generic_worker_federation_endpoints }}"
 matrix_synapse_reverse_proxy_companion_synapse_stream_writer_typing_stream_worker_client_server_locations: "{{ matrix_synapse_workers_stream_writer_typing_stream_worker_client_server_endpoints }}"
diff --git a/roles/custom/matrix-synapse-reverse-proxy-companion/defaults/main.yml b/roles/custom/matrix-synapse-reverse-proxy-companion/defaults/main.yml
index 7a3f4f709..3f19e3a2b 100644
--- a/roles/custom/matrix-synapse-reverse-proxy-companion/defaults/main.yml
+++ b/roles/custom/matrix-synapse-reverse-proxy-companion/defaults/main.yml
@@ -206,6 +206,11 @@ matrix_synapse_reverse_proxy_companion_synapse_federation_api_additional_server_
 # synapse worker activation and endpoint mappings
 matrix_synapse_reverse_proxy_companion_synapse_workers_enabled: false
 matrix_synapse_reverse_proxy_companion_synapse_workers_list: []
+matrix_synapse_reverse_proxy_companion_synapse_room_worker_client_server_locations: []
+matrix_synapse_reverse_proxy_companion_synapse_room_worker_federation_locations: []
+matrix_synapse_reverse_proxy_companion_synapse_sync_worker_client_server_locations: []
+matrix_synapse_reverse_proxy_companion_synapse_client_reader_client_server_locations: []
+matrix_synapse_reverse_proxy_companion_synapse_federation_reader_federation_locations: []
 matrix_synapse_reverse_proxy_companion_synapse_generic_worker_client_server_locations: []
 matrix_synapse_reverse_proxy_companion_synapse_generic_worker_federation_locations: []
 matrix_synapse_reverse_proxy_companion_synapse_stream_writer_typing_stream_worker_client_server_locations: []
diff --git a/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2 b/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2
index c94855d59..c3164e963 100644
--- a/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2
+++ b/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2
@@ -1,5 +1,9 @@
 #jinja2: lstrip_blocks: "True"
 
+{% set room_workers = matrix_synapse_reverse_proxy_companion_synapse_workers_list | selectattr('type', 'equalto', 'room_worker') | list %}
+{% set sync_workers = matrix_synapse_reverse_proxy_companion_synapse_workers_list | selectattr('type', 'equalto', 'sync_worker') | list %}
+{% set client_reader_workers = matrix_synapse_reverse_proxy_companion_synapse_workers_list | selectattr('type', 'equalto', 'client_reader') | list %}
+{% set federation_reader_workers = matrix_synapse_reverse_proxy_companion_synapse_workers_list | selectattr('type', 'equalto', 'federation_reader') | list %}
 {% set generic_workers = matrix_synapse_reverse_proxy_companion_synapse_workers_list | selectattr('type', 'equalto', 'generic_worker') | list %}
 {% set stream_writer_typing_stream_workers = matrix_synapse_reverse_proxy_companion_synapse_workers_list | selectattr('type', 'equalto', 'stream_writer') | selectattr('stream_writer_stream', 'equalto', 'typing') | list %}
 {% set stream_writer_to_device_stream_workers = matrix_synapse_reverse_proxy_companion_synapse_workers_list | selectattr('type', 'equalto', 'stream_writer') | selectattr('stream_writer_stream', 'equalto', 'to_device') | list %}
@@ -9,9 +13,31 @@
 {% set media_repository_workers = matrix_synapse_reverse_proxy_companion_synapse_workers_list | selectattr('type', 'equalto', 'media_repository') | list %}
 {% set user_dir_workers = matrix_synapse_reverse_proxy_companion_synapse_workers_list | selectattr('type', 'equalto', 'user_dir') | list %}
 
-{% macro render_worker_upstream(name, workers) %}
+
+# Maps from https://tcpipuk.github.io/synapse/deployment/nginx.html#mapsconf
+# Client username from MXID
+map $http_authorization $mxid_localpart {
+  default                           $http_authorization;
+  "~Bearer syt_(?<username>.*?)_.*" $username;
+  ""                                $accesstoken_from_urlparam;
+}
+
+# Whether to upgrade HTTP connection
+map $http_upgrade $connection_upgrade {
+  default upgrade;
+  '' close;
+}
+
+#Extract room name from URI
+map $request_uri $room_name {
+  ~^/_matrix/(client|federation)/.*?(?:%21|!)(?<room>[A-Za-z0-9._=\-\/]+)(?::|%3A)[A-Za-z0-9._=\-\/]+ $room;
+}
+# End maps
+
+{% macro render_worker_upstream(name, workers, load_balance) %}
 {% if workers | length > 0 %}
 	upstream {{ name }} {
+		{{ load_balance }}
 		keepalive {{ workers | length * 2 }};
 		{% for worker in workers %}
 			server "{{ worker.name }}:{{ worker.port }}";
@@ -37,28 +63,22 @@
 	{% endif %}
 	# Round Robin "upstream" pools for workers
 
-	{% if generic_workers |length > 0 %}
-	upstream generic_workers_upstream {
-		# ensures that requests from the same client will always be passed
-		# to the same server (except when this server is unavailable)
-		hash $http_x_forwarded_for;
-		keepalive {{ generic_workers | length * 2 }};
+	{{ render_worker_upstream('room_workers_upstream', room_workers, 'hash $room_name consistent;')
+	{{ render_worker_upstream('sync_workers_upstream', room_workers, 'hash $mxid_localpart consistent;')
+	{{ render_worker_upstream('client_reader_workers_upstream', client_reader_workers, 'least_conn;')
+	{{ render_worker_upstream('federation_reader_workers_upstream', federation_reader_workers, 'hash $http_x_forwarded_for;') }}
 
-		{% for worker in generic_workers %}
-			server "{{ worker.name }}:{{ worker.port }}";
-		{% endfor %}
-	}
-	{% endif %}
+	{{ render_worker_upstream('generic_workers_upstream', generic_workers, 'hash $http_x_forwarded_for;') }}
 
-	{{ render_worker_upstream('stream_writer_typing_stream_workers_upstream', stream_writer_typing_stream_workers) }}
-	{{ render_worker_upstream('stream_writer_to_device_stream_workers_upstream', stream_writer_to_device_stream_workers) }}
-	{{ render_worker_upstream('stream_writer_account_data_stream_workers_upstream', stream_writer_account_data_stream_workers) }}
-	{{ render_worker_upstream('stream_writer_receipts_stream_workers_upstream', stream_writer_receipts_stream_workers) }}
-	{{ render_worker_upstream('stream_writer_presence_stream_workers_upstream', stream_writer_presence_stream_workers) }}
+	{{ render_worker_upstream('stream_writer_typing_stream_workers_upstream', stream_writer_typing_stream_workers, '') }}
+	{{ render_worker_upstream('stream_writer_to_device_stream_workers_upstream', stream_writer_to_device_stream_workers, '') }}
+	{{ render_worker_upstream('stream_writer_account_data_stream_workers_upstream', stream_writer_account_data_stream_workers, '') }}
+	{{ render_worker_upstream('stream_writer_receipts_stream_workers_upstream', stream_writer_receipts_stream_workers, '') }}
+	{{ render_worker_upstream('stream_writer_presence_stream_workers_upstream', stream_writer_presence_stream_workers, '') }}
 
-	{{ render_worker_upstream('media_repository_workers_upstream', media_repository_workers) }}
+	{{ render_worker_upstream('media_repository_workers_upstream', media_repository_workers, 'least_conn;') }}
 
-	{{ render_worker_upstream('user_dir_workers_upstream', user_dir_workers) }}
+	{{ render_worker_upstream('user_dir_workers_upstream', user_dir_workers, '') }}
 {% endif %}
 
 server {
@@ -74,6 +94,21 @@ server {
 	{% if matrix_synapse_reverse_proxy_companion_synapse_workers_enabled %}
 		{# Workers redirects BEGIN #}
 
+		{% if room_workers | length > 0 %}
+			# https://tcpipuk.github.io/synapse/deployment/workers.html
+			{{ render_locations_to_upstream(matrix_synapse_reverse_proxy_companion_synapse_room_worker_client_server_locations, 'room_workers_upstream') }}
+		{% endif %}
+
+		{% if sync_workers | length > 0 %}
+			# https://tcpipuk.github.io/synapse/deployment/workers.html
+			{{ render_locations_to_upstream(matrix_synapse_reverse_proxy_companion_synapse_sync_worker_client_server_locations, 'sync_workers_upstream') }}
+		{% endif %}
+
+		{% if client_reader_workers | length > 0 %}
+			# https://tcpipuk.github.io/synapse/deployment/workers.html
+			{{ render_locations_to_upstream(matrix_synapse_reverse_proxy_companion_synapse_client_reader_client_server_locations, 'client_reader_workers_upstream') }}
+		{% endif %}
+
 		{% if generic_workers | length > 0 %}
 			# https://matrix-org.github.io/synapse/latest/workers.html#synapseappgeneric_worker
 			{{ render_locations_to_upstream(matrix_synapse_reverse_proxy_companion_synapse_generic_worker_client_server_locations, 'generic_workers_upstream') }}
@@ -165,6 +200,14 @@ server {
 	gzip_types text/plain application/json;
 
 	{% if matrix_synapse_reverse_proxy_companion_synapse_workers_enabled %}
+		{% if room_workers | length > 0 %}
+			# https://tcpipuk.github.io/synapse/deployment/workers.html
+			{{ render_locations_to_upstream(matrix_synapse_reverse_proxy_companion_synapse_room_worker_federation_locations, 'room_workers_upstream') }}
+		{% endif %}
+		{% if federation_reader_workers | length > 0 %}
+			# https://tcpipuk.github.io/synapse/deployment/workers.html
+			{{ render_locations_to_upstream(matrix_synapse_reverse_proxy_companion_synapse_federation_reader_federation_locations, 'federation_reader_workers_upstream') }}
+		{% endif %}
 		{% if generic_workers | length > 0 %}
 			# https://matrix-org.github.io/synapse/latest/workers.html#synapseappgeneric_worker
 			{{ render_locations_to_upstream(matrix_synapse_reverse_proxy_companion_synapse_generic_worker_federation_locations, 'generic_workers_upstream') }}
diff --git a/roles/custom/matrix-synapse/defaults/main.yml b/roles/custom/matrix-synapse/defaults/main.yml
index 4b1513fa5..7dd7ec227 100644
--- a/roles/custom/matrix-synapse/defaults/main.yml
+++ b/roles/custom/matrix-synapse/defaults/main.yml
@@ -666,6 +666,10 @@ matrix_synapse_workers_preset: one-of-each
 
 matrix_synapse_workers_presets:
   little-federation-helper:
+    room_workers_count: 0
+    sync_workers_count: 0
+    client_reader_workers_count: 0
+    federation_reader_workers_count: 0
     generic_workers_count: 0
     pusher_workers_count: 0
     federation_sender_workers_count: 1
@@ -680,6 +684,10 @@ matrix_synapse_workers_presets:
     stream_writer_receipts_stream_workers_count: 0
     stream_writer_presence_stream_workers_count: 0
   one-of-each:
+    room_workers_count: 0
+    sync_workers_count: 0
+    client_reader_workers_count: 0
+    federation_reader_workers_count: 0
     generic_workers_count: 1
     pusher_workers_count: 1
     federation_sender_workers_count: 1
@@ -693,6 +701,24 @@ matrix_synapse_workers_presets:
     stream_writer_account_data_stream_workers_count: 1
     stream_writer_receipts_stream_workers_count: 1
     stream_writer_presence_stream_workers_count: 1
+  room-workers:
+    room_workers_count: 1
+    sync_workers_count: 1
+    client_reader_workers_count: 1
+    federation_reader_workers_count: 1
+    generic_workers_count: 0
+    pusher_workers_count: 1
+    federation_sender_workers_count: 1
+    media_repository_workers_count: 1
+    appservice_workers_count: 1
+    user_dir_workers_count: 1
+    background_workers_count: 1
+    stream_writer_events_stream_workers_count: 1
+    stream_writer_typing_stream_workers_count: 1
+    stream_writer_to_device_stream_workers_count: 1
+    stream_writer_account_data_stream_workers_count: 1
+    stream_writer_receipts_stream_workers_count: 1
+    stream_writer_presence_stream_workers_count: 1
 
 # Controls whether the matrix-synapse container exposes the various worker ports
 # (see `port` and `metrics_port` in `matrix_synapse_workers_enabled_list`) outside of the container.
@@ -733,6 +759,27 @@ matrix_synapse_worker_container_labels_public_metrics_middleware_basic_auth_user
 #   another.label="here"
 matrix_synapse_worker_container_labels_additional_labels: ''
 
+# Room workers
+matrix_synapse_workers_room_workers_count: "{{ matrix_synapse_workers_presets[matrix_synapse_workers_preset]['room_workers_count'] }}"
+matrix_synapse_workers_room_workers_port_range_start: 28111
+matrix_synapse_workers_room_workers_metrics_range_start: 29111
+
+# Sync workers
+matrix_synapse_workers_sync_workers_count: "{{ matrix_synapse_workers_presets[matrix_synapse_workers_preset]['sync_workers_count'] }}"
+matrix_synapse_workers_sync_workers_port_range_start: 28211
+matrix_synapse_workers_sync_workers_metrics_range_start: 29211
+
+# Client reader workers
+matrix_synapse_workers_client_reader_workers_count: "{{ matrix_synapse_workers_presets[matrix_synapse_workers_preset]['client_reader_workers_count'] }}"
+matrix_synapse_workers_client_reader_workers_port_range_start: 28311
+matrix_synapse_workers_client_reader_workers_metrics_range_start: 29311
+
+# Federation reader workers
+matrix_synapse_workers_federation_reader_workers_count: "{{ matrix_synapse_workers_presets[matrix_synapse_workers_preset]['federation_reader_workers_count'] }}"
+matrix_synapse_workers_federation_reader_workers_port_range_start: 28411
+matrix_synapse_workers_federation_reader_workers_metrics_range_start: 29411
+
+# Generic workers
 matrix_synapse_workers_generic_workers_count: "{{ matrix_synapse_workers_presets[matrix_synapse_workers_preset]['generic_workers_count'] }}"
 matrix_synapse_workers_generic_workers_port_range_start: 18111
 matrix_synapse_workers_generic_workers_metrics_range_start: 19111
diff --git a/roles/custom/matrix-synapse/tasks/synapse/workers/init.yml b/roles/custom/matrix-synapse/tasks/synapse/workers/init.yml
index 68b8f37e3..a1bc4cc72 100644
--- a/roles/custom/matrix-synapse/tasks/synapse/workers/init.yml
+++ b/roles/custom/matrix-synapse/tasks/synapse/workers/init.yml
@@ -4,6 +4,58 @@
 # set_fact within a loop does not work reliably in Ansible (it only executes on the first iteration for some reason),
 # so we're forced to do something much uglier.
 
+- name: Build room workers
+  ansible.builtin.set_fact:
+    worker:
+      id: "room-worker-{{ item }}"
+      name: "matrix-synapse-worker-room-{{ item }}"
+      type: 'room_worker'
+      app: 'generic_worker'
+      webserving: true
+      port: "{{ matrix_synapse_workers_room_workers_port_range_start + item }}"
+      metrics_port: "{{ matrix_synapse_workers_room_workers_metrics_range_start + item }}"
+  register: "matrix_synapse_workers_list_results_room_workers"
+  loop: "{{ range(0, matrix_synapse_workers_room_workers_count | int) | list }}"
+
+- name: Build client sync workers
+  ansible.builtin.set_fact:
+    worker:
+      id: "sync-worker-{{ item }}"
+      name: "matrix-synapse-worker-sync-{{ item }}"
+      type: 'sync_worker'
+      app: 'generic_worker'
+      webserving: true
+      port: "{{ matrix_synapse_workers_sync_workers_port_range_start + item }}"
+      metrics_port: "{{ matrix_synapse_workers_sync_workers_metrics_range_start + item }}"
+  register: "matrix_synapse_workers_list_results_sync_workers"
+  loop: "{{ range(0, matrix_synapse_workers_sync_workers_count | int) | list }}"
+
+- name: Build client reader workers
+  ansible.builtin.set_fact:
+    worker:
+      id: "client-reader-{{ item }}"
+      name: "matrix-synapse-worker-client-reader-{{ item }}"
+      type: 'client_reader'
+      app: 'generic_worker'
+      webserving: true
+      port: "{{ matrix_synapse_workers_client_reader_workers_port_range_start + item }}"
+      metrics_port: "{{ matrix_synapse_workers_client_reader_workers_metrics_range_start + item }}"
+  register: "matrix_synapse_workers_list_results_client_reader_workers"
+  loop: "{{ range(0, matrix_synapse_workers_client_reader_workers_count | int) | list }}"
+
+- name: Build federation reader workers
+  ansible.builtin.set_fact:
+    worker:
+      id: "federation-reader-{{ item }}"
+      name: "matrix-synapse-worker-federation-reader-{{ item }}"
+      type: 'federation_reader'
+      app: 'generic_worker'
+      webserving: true
+      port: "{{ matrix_synapse_workers_federation_reader_workers_port_range_start + item }}"
+      metrics_port: "{{ matrix_synapse_workers_federation_reader_workers_metrics_range_start + item }}"
+  register: "matrix_synapse_workers_list_results_federation_reader_workers"
+  loop: "{{ range(0, matrix_synapse_workers_federation_reader_workers_count | int) | list }}"
+
 - name: Build generic workers
   ansible.builtin.set_fact:
     worker:
diff --git a/roles/custom/matrix-synapse/vars/main.yml b/roles/custom/matrix-synapse/vars/main.yml
index 137a59737..63ad48550 100644
--- a/roles/custom/matrix-synapse/vars/main.yml
+++ b/roles/custom/matrix-synapse/vars/main.yml
@@ -8,6 +8,25 @@ matrix_synapse_media_store_directory_name: "{{ matrix_synapse_media_store_path |
 # Optionally: `false` to fully disable tls on outbound smtp
 matrix_synapse_email_smtp_enable_tls: true
 
+# Room workers handle any URL that contains a room id, either through the client-server API or the federation API
+# - see https://tcpipuk.github.io/synapse/deployment/nginx.html#locationsconf
+matrix_synapse_workers_room_worker_client_server_endpoints:
+  - ^/_matrix/client/.*?!(?<room>[A-Za-z0-9._=\-\/]+):[A-Za-z0-9.\-]+
+matrix_synapse_workers_room_worker_federation_endpoints:
+  - ^/_matrix/federation/v[12]/(?:state_ids|get_missing_events)/(?:%21|!)(?<room>[A-Za-z0-9._=\-\/]+)(:|%3A)[A-Za-z0-9.\-]+
+
+# Sync workers handle /sync and the (now deprecated) related endpoints
+matrix_synapse_workers_sync_worker_client_server_endpoints:
+  - ^/_matrix/client/(api/v1|r0|v3|unstable)/(sync|events|initialSync|rooms/[^/]+/initialSync)$
+
+# Client reader workers handle generic client-server endpoints that don't contain a roomid or sync
+matrix_synapse_workers_client_reader_client_server_endpoints:
+  - ^/_matrix/client/(api/v1|r0|v3|unstable)/(room_keys/|keys/(query|changes|claim|upload/|room_keys/)|login|register(/available|/m.login.registration_token/validity|)|password_policy|profile|rooms/.*/(joined_members|context/.*|members|state|hierarchy|relations/|event/|aliases|timestamp_to_event|redact|send|state/|(join|invite|leave|ban|unban|kick))|createRoom|publicRooms|account/(3pid|whoami|devices)|versions|voip/turnServer|joined_rooms|search|user/.*/filter(/|$)|directory/room/.*|capabilities)
+
+# Federation reader workers handle generic federation endpoints that don't contain a roomid
+matrix_synapse_workers_federation_reader_federation_endpoints:
+  - ^/_matrix/(federation/(v1|v2)|key/v2)/
+
 # A Synapse generic worker can handle both federation and client-server API endpoints.
 # We wish to split these, as we normally serve federation separately and don't want them mixed up.
 #

From 0175a472d73556eb25ab0cededf25c5e891ac948 Mon Sep 17 00:00:00 2001
From: Charles Wright <cvwright@futo.org>
Date: Tue, 16 Jan 2024 10:02:36 -0600
Subject: [PATCH 02/10] Typo: forgot closing }}'s

---
 .../conf.d/matrix-synapse-reverse-proxy-companion.conf.j2   | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2 b/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2
index c3164e963..f44823aa8 100644
--- a/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2
+++ b/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2
@@ -63,9 +63,9 @@ map $request_uri $room_name {
 	{% endif %}
 	# Round Robin "upstream" pools for workers
 
-	{{ render_worker_upstream('room_workers_upstream', room_workers, 'hash $room_name consistent;')
-	{{ render_worker_upstream('sync_workers_upstream', room_workers, 'hash $mxid_localpart consistent;')
-	{{ render_worker_upstream('client_reader_workers_upstream', client_reader_workers, 'least_conn;')
+	{{ render_worker_upstream('room_workers_upstream', room_workers, 'hash $room_name consistent;') }}
+	{{ render_worker_upstream('sync_workers_upstream', room_workers, 'hash $mxid_localpart consistent;') }}
+	{{ render_worker_upstream('client_reader_workers_upstream', client_reader_workers, 'least_conn;') }}
 	{{ render_worker_upstream('federation_reader_workers_upstream', federation_reader_workers, 'hash $http_x_forwarded_for;') }}
 
 	{{ render_worker_upstream('generic_workers_upstream', generic_workers, 'hash $http_x_forwarded_for;') }}

From 12a8d535e81ad95fef232aa997c1afc5aa7dc624 Mon Sep 17 00:00:00 2001
From: Charles Wright <cvwright@futo.org>
Date: Tue, 16 Jan 2024 10:53:20 -0600
Subject: [PATCH 03/10] Move maps inside the if-workers block; Add Tom's map to
 extract access token from the URI arg

---
 ...ix-synapse-reverse-proxy-companion.conf.j2 | 45 ++++++++++---------
 1 file changed, 24 insertions(+), 21 deletions(-)

diff --git a/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2 b/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2
index f44823aa8..11619f204 100644
--- a/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2
+++ b/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2
@@ -13,27 +13,6 @@
 {% set media_repository_workers = matrix_synapse_reverse_proxy_companion_synapse_workers_list | selectattr('type', 'equalto', 'media_repository') | list %}
 {% set user_dir_workers = matrix_synapse_reverse_proxy_companion_synapse_workers_list | selectattr('type', 'equalto', 'user_dir') | list %}
 
-
-# Maps from https://tcpipuk.github.io/synapse/deployment/nginx.html#mapsconf
-# Client username from MXID
-map $http_authorization $mxid_localpart {
-  default                           $http_authorization;
-  "~Bearer syt_(?<username>.*?)_.*" $username;
-  ""                                $accesstoken_from_urlparam;
-}
-
-# Whether to upgrade HTTP connection
-map $http_upgrade $connection_upgrade {
-  default upgrade;
-  '' close;
-}
-
-#Extract room name from URI
-map $request_uri $room_name {
-  ~^/_matrix/(client|federation)/.*?(?:%21|!)(?<room>[A-Za-z0-9._=\-\/]+)(?::|%3A)[A-Za-z0-9._=\-\/]+ $room;
-}
-# End maps
-
 {% macro render_worker_upstream(name, workers, load_balance) %}
 {% if workers | length > 0 %}
 	upstream {{ name }} {
@@ -58,6 +37,30 @@ map $request_uri $room_name {
 {% endmacro %}
 
 {% if matrix_synapse_reverse_proxy_companion_synapse_workers_enabled %}
+
+# Maps from https://tcpipuk.github.io/synapse/deployment/nginx.html#mapsconf
+# Client username from access token
+map $arg_access_token $accesstoken_from_urlparam {
+  default                    $arg_access_token;
+  "~syt_(?<username>.*?)_.*" $username;
+}
+# Client username from MXID
+map $http_authorization $mxid_localpart {
+  default                           $http_authorization;
+  "~Bearer syt_(?<username>.*?)_.*" $username;
+  ""                                $accesstoken_from_urlparam;
+}
+# Whether to upgrade HTTP connection
+map $http_upgrade $connection_upgrade {
+  default upgrade;
+  '' close;
+}
+#Extract room name from URI
+map $request_uri $room_name {
+  ~^/_matrix/(client|federation)/.*?(?:%21|!)(?<room>[A-Za-z0-9._=\-\/]+)(?::|%3A)[A-Za-z0-9._=\-\/]+ $room;
+}
+# End maps
+
 	{% if matrix_synapse_reverse_proxy_companion_synapse_cache_enabled %}
     	proxy_cache_path  {{ matrix_synapse_reverse_proxy_companion_synapse_cache_path }} levels=1:2 keys_zone={{ matrix_synapse_reverse_proxy_companion_synapse_cache_keys_zone_name }}:{{ matrix_synapse_reverse_proxy_companion_synapse_cache_keys_zone_size }} inactive={{ matrix_synapse_reverse_proxy_companion_synapse_cache_inactive_time }} max_size={{ matrix_synapse_reverse_proxy_companion_synapse_cache_max_size_mb }}m;
 	{% endif %}

From 5ca9a7269a3ad6aee0c0c654ca348161cfdc2b2c Mon Sep 17 00:00:00 2001
From: Charles Wright <cvwright@futo.org>
Date: Tue, 16 Jan 2024 10:58:46 -0600
Subject: [PATCH 04/10] Add the new worker types to the list of available
 worker types

---
 roles/custom/matrix-synapse/vars/main.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/roles/custom/matrix-synapse/vars/main.yml b/roles/custom/matrix-synapse/vars/main.yml
index 63ad48550..828dcbeab 100644
--- a/roles/custom/matrix-synapse/vars/main.yml
+++ b/roles/custom/matrix-synapse/vars/main.yml
@@ -261,9 +261,13 @@ matrix_synapse_workers_user_dir_endpoints:
 
 matrix_synapse_workers_avail_list:
   - appservice
+  - client_reader
+  - federation_reader
   - federation_sender
   - generic_worker
   - media_repository
   - pusher
+  - room_worker
+  - sync_worker
   - user_dir
 ### workers:end

From 1379200e9dc8a8064d91a3a7ab14baba3e4ec5d3 Mon Sep 17 00:00:00 2001
From: Charles Wright <cvwright@futo.org>
Date: Tue, 16 Jan 2024 11:13:51 -0600
Subject: [PATCH 05/10] Add new worker types to the dynamic workers list

---
 .../custom/matrix-synapse/tasks/synapse/workers/init.yml  | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/roles/custom/matrix-synapse/tasks/synapse/workers/init.yml b/roles/custom/matrix-synapse/tasks/synapse/workers/init.yml
index a1bc4cc72..8bf8201a0 100644
--- a/roles/custom/matrix-synapse/tasks/synapse/workers/init.yml
+++ b/roles/custom/matrix-synapse/tasks/synapse/workers/init.yml
@@ -182,6 +182,14 @@
     matrix_synapse_dynamic_workers_list: "{{ matrix_synapse_dynamic_workers_list | default([]) + [item.ansible_facts.worker] }}"
   with_items: |
     {{
+      matrix_synapse_workers_list_results_room_workers.results
+      +
+      matrix_synapse_workers_list_results_sync_workers.results
+      +
+      matrix_synapse_workers_list_results_client_reader_workers.results
+      +
+      matrix_synapse_workers_list_results_federation_reader_workers.results
+      +
       matrix_synapse_workers_list_results_generic_workers.results
       +
       matrix_synapse_workers_list_results_stream_writer_workers.results

From 124524ea1fa28f3a2c9573ace834d9f1aa190f26 Mon Sep 17 00:00:00 2001
From: Charles Wright <cvwright@futo.org>
Date: Tue, 16 Jan 2024 11:22:46 -0600
Subject: [PATCH 06/10] Typo: Send sync endpoints to sync workers, not room
 workers

---
 .../nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2 b/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2
index 11619f204..781b2edb3 100644
--- a/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2
+++ b/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2
@@ -67,7 +67,7 @@ map $request_uri $room_name {
 	# Round Robin "upstream" pools for workers
 
 	{{ render_worker_upstream('room_workers_upstream', room_workers, 'hash $room_name consistent;') }}
-	{{ render_worker_upstream('sync_workers_upstream', room_workers, 'hash $mxid_localpart consistent;') }}
+	{{ render_worker_upstream('sync_workers_upstream', sync_workers, 'hash $mxid_localpart consistent;') }}
 	{{ render_worker_upstream('client_reader_workers_upstream', client_reader_workers, 'least_conn;') }}
 	{{ render_worker_upstream('federation_reader_workers_upstream', federation_reader_workers, 'hash $http_x_forwarded_for;') }}
 

From a1cbe7f39b43b6016543db02b8d6e102e62f2ff9 Mon Sep 17 00:00:00 2001
From: Charles Wright <cvwright@futo.org>
Date: Tue, 16 Jan 2024 16:32:32 -0600
Subject: [PATCH 07/10] Add overrides for locations that must go to the main
 Synapse process

---
 .../defaults/main.yml                         |  4 +-
 ...ix-synapse-reverse-proxy-companion.conf.j2 | 45 +++++++++++++++++++
 2 files changed, 48 insertions(+), 1 deletion(-)

diff --git a/roles/custom/matrix-synapse-reverse-proxy-companion/defaults/main.yml b/roles/custom/matrix-synapse-reverse-proxy-companion/defaults/main.yml
index 3f19e3a2b..4a330e7a8 100644
--- a/roles/custom/matrix-synapse-reverse-proxy-companion/defaults/main.yml
+++ b/roles/custom/matrix-synapse-reverse-proxy-companion/defaults/main.yml
@@ -220,7 +220,9 @@ matrix_synapse_reverse_proxy_companion_synapse_stream_writer_receipts_stream_wor
 matrix_synapse_reverse_proxy_companion_synapse_stream_writer_presence_stream_worker_client_server_locations: []
 matrix_synapse_reverse_proxy_companion_synapse_media_repository_locations: []
 matrix_synapse_reverse_proxy_companion_synapse_user_dir_locations: []
-
+matrix_synapse_reverse_proxy_companion_client_server_main_override_locations_regex: ^/_matrix/client/(api/v1|r0|v3|unstable)/(account/3pid/|directory/list/room/|pushrules/|rooms/[^/]+/(forget|upgrade)|login/sso/redirect/|register)
+matrix_synapse_reverse_proxy_companion_client_server_sso_override_locations_regex: ^(/_matrix/client/(api/v1|r0|v3|unstable)/login/sso/redirect|/_synapse/client/(pick_username|(new_user_consent|oidc/callback|pick_idp|sso_register)$))
+matrix_synapse_reverse_proxy_companion_federation_override_locations_regex: ^/_matrix/federation/v1/openid/userinfo$
 
 # synapse content caching
 matrix_synapse_reverse_proxy_companion_synapse_cache_enabled: false
diff --git a/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2 b/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2
index 781b2edb3..81c9ced20 100644
--- a/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2
+++ b/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2
@@ -95,6 +95,36 @@ server {
 	gzip_types text/plain application/json;
 
 	{% if matrix_synapse_reverse_proxy_companion_synapse_workers_enabled %}
+		# Client-server overrides -- These locations must go to the main Synapse process
+		location ~ {{ matrix_synapse_reverse_proxy_companion_client_server_main_override_locations_regex }} {
+			{# FIXME: This block was copied from the main Synapse fallback below.  It would be better to have it in one place and avoid duplication. #}
+			{# Use the embedded DNS resolver in Docker containers to discover the service #}
+			resolver {{ matrix_synapse_reverse_proxy_companion_http_level_resolver }} valid=5s;
+			set $backend "{{ matrix_synapse_reverse_proxy_companion_client_api_addr }}";
+			proxy_pass http://$backend;
+
+			proxy_set_header Host $host;
+
+			client_body_buffer_size 25M;
+			client_max_body_size {{ matrix_synapse_reverse_proxy_companion_client_api_client_max_body_size_mb }}M;
+			proxy_max_temp_file_size 0;
+		}
+
+		# Client-server SSO overrides -- These locations must go to the main Synapse process
+		location ~ {{ matrix_synapse_reverse_proxy_companion_client_server_sso_override_locations_regex }} {
+			{# FIXME: This block was copied from the main Synapse fallback below.  It would be better to have it in one place and avoid duplication. #}
+			{# Use the embedded DNS resolver in Docker containers to discover the service #}
+			resolver {{ matrix_synapse_reverse_proxy_companion_http_level_resolver }} valid=5s;
+			set $backend "{{ matrix_synapse_reverse_proxy_companion_client_api_addr }}";
+			proxy_pass http://$backend;
+
+			proxy_set_header Host $host;
+
+			client_body_buffer_size 25M;
+			client_max_body_size {{ matrix_synapse_reverse_proxy_companion_client_api_client_max_body_size_mb }}M;
+			proxy_max_temp_file_size 0;
+		}
+
 		{# Workers redirects BEGIN #}
 
 		{% if room_workers | length > 0 %}
@@ -203,6 +233,21 @@ server {
 	gzip_types text/plain application/json;
 
 	{% if matrix_synapse_reverse_proxy_companion_synapse_workers_enabled %}
+		# Federation overrides -- These locations must go to the main Synapse process
+		location ~ {{ matrix_synapse_reverse_proxy_companion_federation_override_locations_regex }} {
+			{# FIXME: This block was copied from the fallback location below.  It would be better to have it in one place and avoid duplication. #}
+			{# Use the embedded DNS resolver in Docker containers to discover the service #}
+			resolver {{ matrix_synapse_reverse_proxy_companion_http_level_resolver }} valid=5s;
+			set $backend "{{ matrix_synapse_reverse_proxy_companion_federation_api_addr }}";
+			proxy_pass http://$backend;
+
+			proxy_set_header Host $host;
+
+			client_body_buffer_size 25M;
+			client_max_body_size {{ matrix_synapse_reverse_proxy_companion_federation_api_client_max_body_size_mb }}M;
+			proxy_max_temp_file_size 0;
+		}
+
 		{% if room_workers | length > 0 %}
 			# https://tcpipuk.github.io/synapse/deployment/workers.html
 			{{ render_locations_to_upstream(matrix_synapse_reverse_proxy_companion_synapse_room_worker_federation_locations, 'room_workers_upstream') }}

From 0dbdaf5b9f2e339684e8c916787cf623b20a2cab Mon Sep 17 00:00:00 2001
From: Charles Wright <cvwright@futo.org>
Date: Tue, 16 Jan 2024 16:51:23 -0600
Subject: [PATCH 08/10] Enable HTTP resources for new worker types

---
 .../matrix-synapse/templates/synapse/worker.yaml.j2 | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/roles/custom/matrix-synapse/templates/synapse/worker.yaml.j2 b/roles/custom/matrix-synapse/templates/synapse/worker.yaml.j2
index 18b96a55f..738f0fa33 100644
--- a/roles/custom/matrix-synapse/templates/synapse/worker.yaml.j2
+++ b/roles/custom/matrix-synapse/templates/synapse/worker.yaml.j2
@@ -7,10 +7,17 @@ worker_log_config: /data/{{ matrix_server_fqn_matrix }}.log.config
 
 {% set http_resources = [] %}
 
-{% if matrix_synapse_worker_details.type == 'user_dir' %}
+{% if matrix_synapse_worker_details.type == 'room_worker' %}
+  {% set http_resources = http_resources + ['client', 'federation'] %}
+{% elif matrix_synapse_worker_details.type == 'sync_worker' %}
   {% set http_resources = http_resources + ['client'] %}
-{% endif %}
-{% if matrix_synapse_worker_details.type == 'generic_worker' %}
+{% elif matrix_synapse_worker_details.type == 'client_reader' %}
+  {% set http_resources = http_resources + ['client'] %}
+{% elif matrix_synapse_worker_details.type == 'federation_reader' %}
+  {% set http_resources = http_resources + ['federation'] %}
+{% elif matrix_synapse_worker_details.type == 'user_dir' %}
+  {% set http_resources = http_resources + ['client'] %}
+{% elif matrix_synapse_worker_details.type == 'generic_worker' %}
   {% set http_resources = http_resources + ['client', 'federation'] %}
 {% endif %}
 {#

From 55604f73c5d0726e006a936af8d302bdcafa0cff Mon Sep 17 00:00:00 2001
From: Charles Wright <cvwright@futo.org>
Date: Tue, 16 Jan 2024 17:24:13 -0600
Subject: [PATCH 09/10] Bugfix: Locations for new workers must go *after* the
 stream writers

---
 ...ix-synapse-reverse-proxy-companion.conf.j2 | 30 +++++++++----------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2 b/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2
index 81c9ced20..dcb6ebeea 100644
--- a/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2
+++ b/roles/custom/matrix-synapse-reverse-proxy-companion/templates/nginx/conf.d/matrix-synapse-reverse-proxy-companion.conf.j2
@@ -127,21 +127,6 @@ server {
 
 		{# Workers redirects BEGIN #}
 
-		{% if room_workers | length > 0 %}
-			# https://tcpipuk.github.io/synapse/deployment/workers.html
-			{{ render_locations_to_upstream(matrix_synapse_reverse_proxy_companion_synapse_room_worker_client_server_locations, 'room_workers_upstream') }}
-		{% endif %}
-
-		{% if sync_workers | length > 0 %}
-			# https://tcpipuk.github.io/synapse/deployment/workers.html
-			{{ render_locations_to_upstream(matrix_synapse_reverse_proxy_companion_synapse_sync_worker_client_server_locations, 'sync_workers_upstream') }}
-		{% endif %}
-
-		{% if client_reader_workers | length > 0 %}
-			# https://tcpipuk.github.io/synapse/deployment/workers.html
-			{{ render_locations_to_upstream(matrix_synapse_reverse_proxy_companion_synapse_client_reader_client_server_locations, 'client_reader_workers_upstream') }}
-		{% endif %}
-
 		{% if generic_workers | length > 0 %}
 			# https://matrix-org.github.io/synapse/latest/workers.html#synapseappgeneric_worker
 			{{ render_locations_to_upstream(matrix_synapse_reverse_proxy_companion_synapse_generic_worker_client_server_locations, 'generic_workers_upstream') }}
@@ -172,6 +157,21 @@ server {
 			{{ render_locations_to_upstream(matrix_synapse_reverse_proxy_companion_synapse_stream_writer_presence_stream_worker_client_server_locations, 'stream_writer_presence_stream_workers_upstream') }}
 		{% endif %}
 
+		{% if room_workers | length > 0 %}
+			# https://tcpipuk.github.io/synapse/deployment/workers.html
+			{{ render_locations_to_upstream(matrix_synapse_reverse_proxy_companion_synapse_room_worker_client_server_locations, 'room_workers_upstream') }}
+		{% endif %}
+
+		{% if sync_workers | length > 0 %}
+			# https://tcpipuk.github.io/synapse/deployment/workers.html
+			{{ render_locations_to_upstream(matrix_synapse_reverse_proxy_companion_synapse_sync_worker_client_server_locations, 'sync_workers_upstream') }}
+		{% endif %}
+
+		{% if client_reader_workers | length > 0 %}
+			# https://tcpipuk.github.io/synapse/deployment/workers.html
+			{{ render_locations_to_upstream(matrix_synapse_reverse_proxy_companion_synapse_client_reader_client_server_locations, 'client_reader_workers_upstream') }}
+		{% endif %}
+
 		{% if media_repository_workers | length > 0 %}
 			# https://matrix-org.github.io/synapse/latest/workers.html#synapseappmedia_repository
 			{% for location in matrix_synapse_reverse_proxy_companion_synapse_media_repository_locations %}

From 7d9eb561644811c912d9e074d4b7f70198060f10 Mon Sep 17 00:00:00 2001
From: Charles Wright <cvwright@futo.org>
Date: Wed, 17 Jan 2024 15:22:27 -0600
Subject: [PATCH 10/10] Add a validation step to fail when the user allocates
 generic workers together with all of the new worker types

---
 roles/custom/matrix-synapse/tasks/validate_config.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/roles/custom/matrix-synapse/tasks/validate_config.yml b/roles/custom/matrix-synapse/tasks/validate_config.yml
index ab389a432..fdf208e8d 100644
--- a/roles/custom/matrix-synapse/tasks/validate_config.yml
+++ b/roles/custom/matrix-synapse/tasks/validate_config.yml
@@ -47,6 +47,12 @@
     - "matrix_synapse_workers_stream_writer_receipts_stream_workers_count"
     - "matrix_synapse_workers_stream_writer_presence_stream_workers_count"
 
+- name: Fail when mixing generic workers with new specialized workers
+  ansible.builtin.fail:
+    msg: >-
+      Generic workers should not be mixed with the new specialized worker types (room workers, sync workers, client readers, and federation readers)
+  when: matrix_synapse_workers_generic_workers_count | int > 0 and matrix_synapse_workers_room_workers_count | int > 0 and matrix_synapse_workers_sync_workers_count | int > 0 and matrix_synapse_workers_client_reader_workers_count | int > 0 and matrix_synapse_workers_federation_reader_workers_count | int > 0
+
 - name: (Deprecation) Catch and report renamed settings
   ansible.builtin.fail:
     msg: >-