Define instanceId property on workers

This give us the possibility to run multiple instances of
workers that that don't expose a port.

Right now, we don't support that, but in the future we could
run multiple `federation_sender` or `pusher` workers, without
them fighting over naming (previously, they'd all be named
something like `matrix-synapse-worker-pusher-0`, because
they'd all define `port` as `0`).

roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2

@@ -14,7 +14,7 @@
 		ip_hash;
 
 		{% for worker in generic_workers %}
-		server "matrix-synapse-worker-{{ worker.type }}-{{ worker.port }}:{{ worker.port }}";
+		server "matrix-synapse-worker-{{ worker.type }}-{{ worker.instanceId }}:{{ worker.port }}";
 		{% endfor %}
 	}
 	{% endif %}
@@ -22,7 +22,7 @@
 	{% if frontend_proxy_workers %}
 	upstream frontend_proxy_upstream {
 		{% for worker in frontend_proxy_workers %}
-		server "matrix-synapse-worker-{{ worker.type }}-{{ worker.port }}:{{ worker.port }}";
+		server "matrix-synapse-worker-{{ worker.type }}-{{ worker.instanceId }}:{{ worker.port }}";
 		{% endfor %}
 	}
 	{% endif %}
@@ -30,7 +30,7 @@
 	{% if media_repository_workers %}
 	upstream media_repository_upstream {
 		{% for worker in media_repository_workers %}
-		server "matrix-synapse-worker-{{ worker.type }}-{{ worker.port }}:{{ worker.port }}";
+		server "matrix-synapse-worker-{{ worker.type }}-{{ worker.instanceId }}:{{ worker.port }}";
 		{% endfor %}
 	}
 	{% endif %}
@@ -38,7 +38,7 @@
 	{% if user_dir_workers %}
 	upstream user_dir_upstream {
 		{% for worker in user_dir_workers %}
-		server "matrix-synapse-worker-{{ worker.type }}-{{ worker.port }}:{{ worker.port }}";
+		server "matrix-synapse-worker-{{ worker.type }}-{{ worker.instanceId }}:{{ worker.port }}";
 		{% endfor %}
 	}
 	{% endif %}

roles/matrix-synapse/defaults/main.yml

@@ -353,18 +353,24 @@ matrix_synapse_workers_frontend_proxy_workers_metrics_range_start: 19771
 # Doing that is simpler and also protects you from shooting yourself in the foot,
 # as certain workers can only be spawned just once.
 #
-# Example of what this needs to look like:
+# Each worker instance in the list defines the following fields:
+# - `type` - the type of worker (`generic_worker`, etc.)
+# - `instanceId` - a string that identifies the worker. The combination of (`type` + `instanceId`) represents the name of the worker and must be unique.
+# - `port` - an HTTP port where the worker listens for requests (can be `0` for workers that don't do HTTP request processing)
+# - `metrics_port` - an HTTP port where the worker exports Prometheus metrics
+#
+# Example of what this needs to look like, if you're defining it manually:
 # matrix_synapse_workers_enabled_list:
-#   - { type: generic_worker, port: 18111, metrics_port: 19111 }
+#   - { type: generic_worker, instanceId: '18111', port: 18111, metrics_port: 19111 }
-#   - { type: generic_worker, port: 18112, metrics_port: 19112 }
+#   - { type: generic_worker, instanceId: '18112', port: 18112, metrics_port: 19112 }
-#   - { type: generic_worker, port: 18113, metrics_port: 19113 }
+#   - { type: generic_worker, instanceId: '18113', port: 18113, metrics_port: 19113 }
-#   - { type: generic_worker, port: 18114, metrics_port: 19114 }
+#   - { type: generic_worker, instanceId: '18114', port: 18114, metrics_port: 19114 }
-#   - { type: generic_worker, port: 18115, metrics_port: 19115 }
+#   - { type: generic_worker, instanceId: '18115', port: 18115, metrics_port: 19115 }
-#   - { type: generic_worker, port: 18116, metrics_port: 19116 }
+#   - { type: generic_worker, instanceId: '18116', port: 18116, metrics_port: 19116 }
-#   - { type: pusher, port: 0, metrics_port: 19200 }
+#   - { type: pusher, instanceId: '0', port: 0, metrics_port: 19200 }
-#   - { type: appservice, port: 0, metrics_port: 19300 }
+#   - { type: appservice, instanceId: '0', port: 0, metrics_port: 19300 }
-#   - { type: federation_sender, port: 0, metrics_port: 19400 }
+#   - { type: federation_sender, instanceId: '0', port: 0, metrics_port: 19400 }
-#   - { type: media_repository, port: 18551, metrics_port: 19551 }
+#   - { type: media_repository, instanceId: '18551', port: 18551, metrics_port: 19551 }
 matrix_synapse_workers_enabled_list: []
 
 # Redis information

roles/matrix-synapse/tasks/synapse/workers/init.yml

@@ -7,6 +7,7 @@
   set_fact:
     worker:
       type: 'generic_worker'
+      instanceId: "{{ matrix_synapse_workers_generic_workers_port_range_start + item }}"
       port: "{{ matrix_synapse_workers_generic_workers_port_range_start + item }}"
       metrics_port: "{{ matrix_synapse_workers_generic_workers_metrics_range_start + item }}"
   register: "matrix_synapse_workers_list_results_generic_workers"
@@ -16,6 +17,7 @@
   set_fact:
     worker:
       type: 'federation_sender'
+      instanceId: "{{ item }}"
       port: 0
       metrics_port: "{{ matrix_synapse_workers_federation_sender_workers_metrics_range_start + item }}"
   register: "matrix_synapse_workers_list_results_federation_sender_workers"
@@ -26,6 +28,7 @@
   set_fact:
     worker:
       type: 'pusher'
+      instanceId: "{{ item }}"
       port: 0
       metrics_port: "{{ matrix_synapse_workers_pusher_workers_metrics_range_start + item }}"
   register: "matrix_synapse_workers_list_results_pusher_workers"
@@ -36,6 +39,7 @@
   set_fact:
     worker:
       type: 'appservice'
+      instanceId: "{{ item }}"
       port: 0
       metrics_port: "{{ matrix_synapse_workers_appservice_workers_metrics_range_start + item }}"
   register: "matrix_synapse_workers_list_results_appservice_workers"
@@ -45,6 +49,7 @@
   set_fact:
     worker:
       type: 'media_repository'
+      instanceId: "{{ matrix_synapse_workers_media_repository_workers_port_range_start + item }}"
       port: "{{ matrix_synapse_workers_media_repository_workers_port_range_start + item }}"
       metrics_port: "{{ matrix_synapse_workers_media_repository_workers_metrics_range_start + item }}"
   register: "matrix_synapse_workers_list_results_media_repository_workers"
@@ -54,6 +59,7 @@
   set_fact:
     worker:
       type: 'frontend_proxy'
+      instanceId: "{{ matrix_synapse_workers_frontend_proxy_workers_port_range_start + item }}"
       port: "{{ matrix_synapse_workers_frontend_proxy_workers_port_range_start + item }}"
       metrics_port: "{{ matrix_synapse_workers_frontend_proxy_workers_metrics_range_start + item }}"
   register: "matrix_synapse_workers_list_results_frontend_proxy_workers"

roles/matrix-synapse/tasks/synapse/workers/util/inject_systemd_services_for_worker.yml

@@ -1,6 +1,18 @@
+# The tasks below run before `validate_config.yml`.
+# To avoid failing with a cryptic error message, we'll do validation here.
+#
+# This check is mostly relevant to people who explicitly define `matrix_synapse_workers_enabled_list`
+# (Synapse Workers users from the earlier days of this PR - https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/456).
+#
+# In the future, it should be possible to remove this check.
+# Our own code which dynamically builds `matrix_synapse_workers_enabled_list` does things right.
+- name: Fail if instanceId not defined for worker
+  fail:
+    msg: "Synapse workers (like {{ matrix_synapse_worker_details|to_json }}) need to define an instanceId property (type + instanceId must be unique)"
+  when: "'instanceId' not in matrix_synapse_worker_details"
 
 - set_fact:
-    matrix_synapse_worker_systemd_service_name: "matrix-synapse-worker-{{ matrix_synapse_worker_details.type }}-{{ matrix_synapse_worker_details.port }}.service"
+    matrix_synapse_worker_systemd_service_name: "matrix-synapse-worker-{{ matrix_synapse_worker_details.type }}-{{ matrix_synapse_worker_details.instanceId }}.service"
 
 - set_fact:
     matrix_systemd_services_list: "{{ matrix_systemd_services_list + [matrix_synapse_worker_systemd_service_name] }}"

roles/matrix-synapse/tasks/synapse/workers/util/setup_files_for_worker.yml

@@ -1,12 +1,11 @@
-
 - set_fact:
-    matrix_synapse_worker_systemd_service_name: "matrix-synapse-worker-{{ matrix_synapse_worker_details.type }}-{{ matrix_synapse_worker_details.port }}"
+    matrix_synapse_worker_systemd_service_name: "matrix-synapse-worker-{{ matrix_synapse_worker_details.type }}-{{ matrix_synapse_worker_details.instanceId }}"
 
 - set_fact:
     matrix_synapse_worker_container_name: "{{ matrix_synapse_worker_systemd_service_name }}"
 
 - set_fact:
-    matrix_synapse_worker_config_file_name: "worker.{{ matrix_synapse_worker_details.type }}_{{ matrix_synapse_worker_details.port }}.yaml"
+    matrix_synapse_worker_config_file_name: "worker.{{ matrix_synapse_worker_details.type }}_{{ matrix_synapse_worker_details.instanceId }}.yaml"
 
 - name: Ensure configuration exists for {{ matrix_synapse_worker_systemd_service_name }}
   template: