Add support for stream writer Synapse workers

As stream writer workers are also powered by the `generic_worker`
Synapse app, this necessitated that we provide means for distinguishing
between them and regular `generic_workers`.

I've also taken the time to optimize nginx configuration generation
(more Jinja2 macro usage, less duplication).

Worker names have also changed.
Workers are now named sequentially like this:
- `matrix-synapse-worker-0-generic`
- `matrix-synapse-worker-1-stream-writer-typing`
- `matrix-synapse-worker-2-pusher`

instead of `matrix-synapse-worker_generic_worker-18111` (indexed with a
port number).

People who modify `matrix_synapse_workers_enabled_list` directly will
need to adjust their configuration.

roles/matrix-synapse/tasks/init.yml

@@ -8,15 +8,15 @@
 
 # Unless `matrix_synapse_workers_enabled_list` is explicitly defined,
 # we'll generate it dynamically.
-- ansible.builtin.import_tasks: "{{ role_path }}/tasks/synapse/workers/init.yml"
+- ansible.builtin.include_tasks: "{{ role_path }}/tasks/synapse/workers/init.yml"
   when: "matrix_synapse_enabled and matrix_synapse_workers_enabled and matrix_synapse_workers_enabled_list|length == 0"
 
 - ansible.builtin.set_fact:
     matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-synapse.service'] }}"
   when: matrix_synapse_enabled | bool
 
-- name: Ensure systemd services for workers are injected
-  ansible.builtin.include_tasks: "{{ role_path }}/tasks/synapse/workers/util/inject_systemd_services_for_worker.yml"
+- name: Ensure workers are injected into various places
+  ansible.builtin.include_tasks: "{{ role_path }}/tasks/synapse/workers/util/inject_worker.yml"
   with_items: "{{ matrix_synapse_workers_enabled_list }}"
   loop_control:
     loop_var: matrix_synapse_worker_details
@@ -65,9 +65,9 @@
         matrix_synapse_worker_nginx_metrics_configuration_block: |
           {% for worker in matrix_synapse_workers_enabled_list %}
           {% if worker.metrics_port != 0 %}
-          location /metrics/synapse/worker/{{ worker.type }}-{{ worker.instanceId }} {
+          location /metrics/synapse/worker/{{ worker.id }} {
             resolver 127.0.0.11 valid=5s;
-            set $backend "matrix-synapse-worker-{{ worker.type }}-{{ worker.instanceId }}:{{ worker.metrics_port }}";
+            set $backend "{{ worker.name }}:{{ worker.metrics_port }}";
             proxy_pass http://$backend/_synapse/metrics;
             proxy_set_header Host $host;
           }

roles/matrix-synapse/tasks/synapse/workers/init.yml

@@ -7,18 +7,47 @@
 - name: Build generic workers
   ansible.builtin.set_fact:
     worker:
+      id: "generic-worker-{{ item }}"
+      name: "matrix-synapse-worker-generic-{{ item }}"
       type: 'generic_worker'
-      instanceId: "{{ matrix_synapse_workers_generic_workers_port_range_start + item }}"
+      app: 'generic_worker'
+      webserving: true
       port: "{{ matrix_synapse_workers_generic_workers_port_range_start + item }}"
       metrics_port: "{{ matrix_synapse_workers_generic_workers_metrics_range_start + item }}"
   register: "matrix_synapse_workers_list_results_generic_workers"
   loop: "{{ range(0, matrix_synapse_workers_generic_workers_count | int) | list }}"
 
+- name: Build stream writer workers
+  ansible.builtin.set_fact:
+    worker:
+      id: "stream-writer-{{ item }}-{{ item.stream }}"
+      # Names must not include understores. Certain stream writer streams (to_device, account_data, ..) do, so we fix them up.
+      name: "matrix-synapse-worker-stream-writer-{{ idx }}-{{ item.stream | replace('_', '-') }}"
+      type: 'stream_writer'
+      app: "generic_worker"
+      webserving: "{{ item.stream in matrix_synapse_workers_webserving_stream_writer_types }}"
+      stream_writer_stream: "{{ item.stream }}"
+      port: "{{ matrix_synapse_workers_stream_writer_workers_http_port_range_start + idx }}"
+      replication_port: "{{ matrix_synapse_workers_stream_writer_workers_replication_port_range_start + idx }}"
+      metrics_port: "{{ matrix_synapse_workers_stream_writer_workers_metrics_range_start + idx }}"
+  register: "matrix_synapse_workers_list_results_stream_writer_workers"
+  loop: "{{ matrix_synapse_workers_stream_writers }}"
+  loop_control:
+    index_var: idx
+
+- name: Populate matrix_synapse_stream_writers from enabled stream writer workers list
+  ansible.builtin.set_fact:
+    matrix_synapse_stream_writers: "{{ matrix_synapse_stream_writers | combine ({item.ansible_facts.worker.stream_writer_stream: [item.ansible_facts.worker.name]}) }}"
+  with_items: "{{ matrix_synapse_workers_list_results_stream_writer_workers.results }}"
+
 - name: Build federation sender workers
   ansible.builtin.set_fact:
     worker:
+      id: "federation-sender-{{ item }}"
+      name: "matrix-synapse-worker-federation-sender-{{ item }}"
       type: 'federation_sender'
-      instanceId: "{{ item }}"
+      app: 'federation_sender'
+      webserving: false
       port: 0
       metrics_port: "{{ matrix_synapse_workers_federation_sender_workers_metrics_range_start + item }}"
   register: "matrix_synapse_workers_list_results_federation_sender_workers"
@@ -28,8 +57,11 @@
 - name: Build pusher workers
   ansible.builtin.set_fact:
     worker:
+      id: "pusher-{{ item }}"
+      name: "matrix-synapse-worker-pusher-{{ item }}"
       type: 'pusher'
-      instanceId: "{{ item }}"
+      app: 'pusher'
+      webserving: false
       port: 0
       metrics_port: "{{ matrix_synapse_workers_pusher_workers_metrics_range_start + item }}"
   register: "matrix_synapse_workers_list_results_pusher_workers"
@@ -39,8 +71,11 @@
 - name: Build appservice workers
   ansible.builtin.set_fact:
     worker:
+      id: "appservice-{{ item }}"
+      name: "matrix-synapse-worker-appservice-{{ item }}"
       type: 'appservice'
-      instanceId: "{{ item }}"
+      app: 'appservice'
+      webserving: false
       port: 0
       metrics_port: "{{ matrix_synapse_workers_appservice_workers_metrics_range_start + item }}"
   register: "matrix_synapse_workers_list_results_appservice_workers"
@@ -49,8 +84,11 @@
 - name: Build media_repository workers
   ansible.builtin.set_fact:
     worker:
+      id: "media-repository-{{ item }}"
+      name: "matrix-synapse-worker-media-repository-{{ item }}"
       type: 'media_repository'
-      instanceId: "{{ matrix_synapse_workers_media_repository_workers_port_range_start + item }}"
+      app: 'media_repository'
+      webserving: true
       port: "{{ matrix_synapse_workers_media_repository_workers_port_range_start + item }}"
       metrics_port: "{{ matrix_synapse_workers_media_repository_workers_metrics_range_start + item }}"
   register: "matrix_synapse_workers_list_results_media_repository_workers"
@@ -62,6 +100,8 @@
     {{
       matrix_synapse_workers_list_results_generic_workers.results
       +
+      matrix_synapse_workers_list_results_stream_writer_workers.results
+      +
       matrix_synapse_workers_list_results_federation_sender_workers.results
       +
       matrix_synapse_workers_list_results_pusher_workers.results

roles/matrix-synapse/tasks/synapse/workers/util/inject_systemd_services_for_worker.yml

@@ -1,19 +0,0 @@
----
-# The tasks below run before `validate_config.yml`.
-# To avoid failing with a cryptic error message, we'll do validation here.
-#
-# This check is mostly relevant to people who explicitly define `matrix_synapse_workers_enabled_list`
-# (Synapse Workers users from the earlier days of this PR - https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/456).
-#
-# In the future, it should be possible to remove this check.
-# Our own code which dynamically builds `matrix_synapse_workers_enabled_list` does things right.
-- name: Fail if instanceId not defined for worker
-  ansible.builtin.fail:
-    msg: "Synapse workers (like {{ matrix_synapse_worker_details | to_json }}) need to define an instanceId property (type + instanceId must be unique)"
-  when: "'instanceId' not in matrix_synapse_worker_details"
-
-- ansible.builtin.set_fact:
-    matrix_synapse_worker_systemd_service_name: "matrix-synapse-worker-{{ matrix_synapse_worker_details.type }}-{{ matrix_synapse_worker_details.instanceId }}.service"
-
-- ansible.builtin.set_fact:
-    matrix_systemd_services_list: "{{ matrix_systemd_services_list + [matrix_synapse_worker_systemd_service_name] }}"

roles/matrix-synapse/tasks/synapse/workers/util/inject_worker.yml

@@ -0,0 +1,65 @@
+---
+# The tasks below run before `validate_config.yml`.
+# To avoid failing with a cryptic error message, we'll do validation here.
+#
+# This check is mostly relevant to people who explicitly define `matrix_synapse_workers_enabled_list`
+# (Synapse Workers users from the earlier days of this PR - https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/456).
+#
+# In the future, it should be possible to remove this check.
+# Our own code which dynamically builds `matrix_synapse_workers_enabled_list` does things right.
+- name: Fail if required property not defined for worker
+  ansible.builtin.fail:
+    msg: "Synapse workers (like {{ matrix_synapse_worker_details | to_json }}) need to define a `{{ item }}` property"
+  with_items:
+    - id
+    - name
+    - type
+    - app
+    - port
+    - webserving
+  when: "item not in matrix_synapse_worker_details"
+
+# Names are used for container names and systemd services.
+# Routing happens based on container names, so Synapse processes that try to route to workers with underscores in the name will complain. Example:
+# > InvalidCodepoint Codepoint U+005F at position 46 of 'matrix-synapse-worker-stream-writer-3-account_data' not allowed
+- name: Fail if worker name includes underscore
+  ansible.builtin.fail:
+    msg: "Unrecognized Synapse worker `name`: `{{ matrix_synapse_worker_details.name }}`. It must not include underscores"
+  when: "'_' in matrix_synapse_worker_details.name"
+
+- name: Fail if worker type unknown
+  ansible.builtin.fail:
+    msg: "Unrecognized Synapse worker `type`: `{{ matrix_synapse_worker_details.type }}`. Supported types are: {{ matrix_synapse_known_worker_types | join(', ') }}"
+  when: "matrix_synapse_worker_details.type not in matrix_synapse_known_worker_types"
+
+- name: Fail if worker app unknown
+  ansible.builtin.fail:
+    msg: "Unrecognized Synapse worker `app`: `{{ matrix_synapse_worker_details.app }}`. Supported types are: {{ matrix_synapse_workers_avail_list | join(', ') }}"
+  when: "matrix_synapse_worker_details.app not in matrix_synapse_workers_avail_list"
+
+- block:
+    - name: Fail if stream_writer_stream not defined for stream_writer worker
+      ansible.builtin.fail:
+        msg: >-
+          Synapse stream_writer workers (such as {{ item }}) need to define a valid `stream_writer_stream` property
+          (not `{{ matrix_synapse_worker_details.stream_writer_stream|default('undefined') }}`).
+          Supported types are: {{ matrix_synapse_workers_known_stream_writer_stream_types | join(', ') }}
+      when: "'stream_writer_stream' not in matrix_synapse_worker_details or matrix_synapse_worker_details.stream_writer_stream not in matrix_synapse_workers_known_stream_writer_stream_types"
+
+    - name: Fail if replication_port not defined for stream_writer worker
+      ansible.builtin.fail:
+        msg: "Synapse background workers of type stream_writer (such as {{ item }}) need to define a valid `replication_port` property"
+      when: "'replication_port' not in matrix_synapse_worker_details"
+  when: "matrix_synapse_worker_details.type == 'stream_writer'"
+
+- ansible.builtin.set_fact:
+    matrix_systemd_services_list: "{{ matrix_systemd_services_list + [matrix_synapse_worker_details.name + '.service'] }}"
+
+- ansible.builtin.set_fact:
+    matrix_synapse_webserving_workers_systemd_services_list: "{{ matrix_synapse_webserving_workers_systemd_services_list + [matrix_synapse_worker_details.name + '.service'] }}"
+  when: matrix_synapse_worker_details.webserving | bool
+
+# Inject stream writers and various other background workers into the instance map.
+- ansible.builtin.set_fact:
+    matrix_synapse_instance_map: "{{ matrix_synapse_instance_map | combine({matrix_synapse_worker_details.name: {'host': matrix_synapse_worker_details.name, 'port': matrix_synapse_worker_details.replication_port}}) }}"
+  when: matrix_synapse_worker_details.type in matrix_synapse_known_instance_map_eligible_worker_types

roles/matrix-synapse/tasks/synapse/workers/util/setup_files_for_worker.yml

@@ -1,13 +1,9 @@
 ---
 
 - ansible.builtin.set_fact:
-    matrix_synapse_worker_systemd_service_name: "matrix-synapse-worker-{{ matrix_synapse_worker_details.type }}-{{ matrix_synapse_worker_details.instanceId }}"
-
-- ansible.builtin.set_fact:
-    matrix_synapse_worker_container_name: "{{ matrix_synapse_worker_systemd_service_name }}"
-
-- ansible.builtin.set_fact:
-    matrix_synapse_worker_config_file_name: "worker.{{ matrix_synapse_worker_details.type }}_{{ matrix_synapse_worker_details.instanceId }}.yaml"
+    matrix_synapse_worker_systemd_service_name: "{{ matrix_synapse_worker_details.name }}"
+    matrix_synapse_worker_container_name: "{{ matrix_synapse_worker_details.name }}"
+    matrix_synapse_worker_config_file_name: "worker.{{ matrix_synapse_worker_details.name }}.yaml"
 
 - name: Ensure configuration exists for {{ matrix_synapse_worker_systemd_service_name }}
   ansible.builtin.template:

roles/matrix-synapse/tasks/validate_config.yml

@@ -32,6 +32,11 @@
   with_items:
     - "matrix_synapse_workers_pusher_workers_count"
     - "matrix_synapse_workers_federation_sender_workers_count"
+    - "matrix_synapse_workers_stream_writer_typing_stream_workers_count"
+    - "matrix_synapse_workers_stream_writer_to_device_stream_workers_count"
+    - "matrix_synapse_workers_stream_writer_account_data_stream_workers_count"
+    - "matrix_synapse_workers_stream_writer_receipts_stream_workers_count"
+    - "matrix_synapse_workers_stream_writer_presence_stream_workers_count"
 
 - name: (Deprecation) Catch and report renamed settings
   ansible.builtin.fail: