Run Synapse workers in their own containers

This switches the `docker exec` method of spawning
Synapse workers inside the `matrix-synapse` container with
dedicated containers for each worker.

We also have dedicated systemd services for each worker,
so this are now:
- more consistent with everything else (we don't use systemd
instantiated services anywhere)
- we don't need the "parse systemd instance name into worker name +
port" part
- we don't need to keep track of PIDs manually
- we don't need jq (less depenendencies)
- workers dying would be restarted by systemd correctly, like any other
service
- `docker ps` shows each worker separately and we can observe resource
usage

roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2

@@ -14,7 +14,7 @@
 		ip_hash;
 
 		{% for worker in generic_workers %}
-		server "matrix-synapse:{{ worker.port }}";
+		server "matrix-synapse-worker-{{ worker.type }}-{{ worker.port }}:{{ worker.port }}";
 		{% endfor %}
 	}
 	{% endif %}
@@ -22,7 +22,7 @@
 	{% if frontend_proxy_workers %}
 	upstream frontend_proxy_upstream {
 		{% for worker in frontend_proxy_workers %}
-		server "matrix-synapse:{{ worker.port }}";
+		server "matrix-synapse-worker-{{ worker.type }}-{{ worker.port }}:{{ worker.port }}";
 		{% endfor %}
 	}
 	{% endif %}
@@ -30,7 +30,7 @@
 	{% if media_repository_workers %}
 	upstream media_repository_upstream {
 		{% for worker in media_repository_workers %}
-		server "matrix-synapse:{{ worker.port }}";
+		server "matrix-synapse-worker-{{ worker.type }}-{{ worker.port }}:{{ worker.port }}";
 		{% endfor %}
 	}
 	{% endif %}
@@ -38,7 +38,7 @@
 	{% if user_dir_workers %}
 	upstream user_dir_upstream {
 		{% for worker in user_dir_workers %}
-		server "matrix-synapse:{{ worker.port }}";
+		server "matrix-synapse-worker-{{ worker.type }}-{{ worker.port }}:{{ worker.port }}";
 		{% endfor %}
 	}
 	{% endif %}

roles/matrix-synapse/files/usr-local-bin/matrix-synapse-worker-write-pid

@@ -1,30 +0,0 @@
-#!/bin/bash
-# Find a synapse worker's PID and write it to a file so systemd can manage it as a service
-
-# example invocation:
-# matrix-synapse-worker-write-pid user_dir:18700 /run/matrix-synapse-worker.user_dir:18700.pid
-
-docker_api_call() { curl --silent --unix-socket /var/run/docker.sock ${@}; }
-
-TARGETCONTAINER=matrix-synapse
-TARGETWORKER=${1}
-PIDFILE=${2}
-
-# get ID list of subprocesses executed in $TARGETCONTAINER, and for each..
-for EXECID in $(docker_api_call http://localhost/containers/${TARGETCONTAINER}/json | jq --raw-output '.ExecIDs[]')
-do
-  # fetch detailed process info
-  EXECINFO=$(docker_api_call http://localhost/exec/${EXECID}/json)
-
-  # extract config file path from last command argument
-  WORKERCONFIGFILE=$(echo ${EXECINFO} | jq --raw-output .ProcessConfig.arguments[-1])
-
-  # reconstruct worker name
-  WORKERNAME=${WORKERCONFIGFILE#*/worker.}
-  WORKERNAME=${WORKERNAME%.yaml}
-
-  # if name matches the target worker: write out most recent PID & quit
-  [ "${WORKERNAME}" = "${TARGETWORKER}" ] \
-    && echo ${EXECINFO} | jq --raw-output .Pid > ${PIDFILE} \
-    && exit 0
-done

roles/matrix-synapse/tasks/init.yml

@@ -2,6 +2,13 @@
     matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-synapse.service'] }}"
   when: matrix_synapse_enabled|bool
 
+- name: Ensure systemd services for workers are injected
+  include_tasks: "{{ role_path }}/tasks/workers/util/inject_systemd_services_for_worker.yml"
+  with_items: "{{ matrix_synapse_workers_enabled_list }}"
+  loop_control:
+    loop_var: matrix_synapse_worker_details
+  when: matrix_synapse_enabled|bool and matrix_synapse_workers_enabled|bool
+
 - set_fact:
     matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-goofys.service'] }}"
   when: matrix_s3_media_store_enabled|bool

roles/matrix-synapse/tasks/workers/setup.yml

@@ -1,5 +1,19 @@
 ---
 
+# A previous version of the worker setup used this.
+# This is a temporary cleanup for people who ran that version.
+- name: Ensure old matrix-synapse.service.wants directory is gone
+  file:
+    path: "{{ matrix_systemd_path }}/matrix-synapse.service.wants"
+    state: absent
+
+# Same. This was part of a previous version of the worker setup.
+# No longer necessary.
+- name: Ensure matrix-synapse-worker-write-pid script is removed
+  file:
+    path: "{{ matrix_local_bin_path }}/matrix-synapse-worker-write-pid"
+    state: absent
+
 - include_tasks: "{{ role_path }}/tasks/workers/setup_install.yml"
   when: "matrix_synapse_enabled|bool and matrix_synapse_workers_enabled|bool"
 

roles/matrix-synapse/tasks/workers/setup_install.yml

@@ -1,81 +1,33 @@
 ---
 
-- name: Ensure synapse worker base service file installed
-  template:
-    src: "{{ role_path }}/templates/synapse/systemd/matrix-synapse-worker@.service.j2"
-    dest: "{{ matrix_systemd_path }}/matrix-synapse-worker@.service"
-    mode: 0644
-  register: matrix_synapse_worker_systemd_service_result
-
-- name: Ensure previous worker service symlinks are cleaned
-  file:
-    path: "{{ item.root + '/' + item.path }}"
-    state: absent
-  when:
-    - item.state == 'link'
-    - item.path is match('matrix-synapse-worker@.*\\.service')
-  with_filetree:
-    - "{{ matrix_systemd_path }}/matrix-synapse.service.wants"
-
-- name: Ensure systemd reloaded the worker service unit
-  service:
-    daemon_reload: yes
-
-- name: Ensure individual worker service symlinks exist
-  service:
-    name: "matrix-synapse-worker@{{ item.type }}:{{ item.port }}.service"
-    enabled: true
-  with_items: "{{ matrix_synapse_workers_enabled_list }}"
-
 - name: Find worker configs to be cleaned
   find:
     path: "{{ matrix_synapse_config_dir_path }}"
     patterns: "worker.*.yaml"
     use_regex: true
-  register: worker_config_files
+  register: matrix_synapse_workers_current_config_files
 
 - name: Ensure previous worker configs are cleaned
   file:
     path: "{{ item.path }}"
     state: absent
-  with_items: "{{ worker_config_files.files }}"
+  with_items: "{{ matrix_synapse_workers_current_config_files.files }}"
 
-- name: Ensure creation of specific worker configs
-  template:
-    src: "{{ role_path }}/templates/synapse/worker.yaml.j2"
-    dest: "{{ matrix_synapse_config_dir_path }}/worker.{{ item.type }}:{{ item.port }}.yaml"
-  with_list: "{{ matrix_synapse_workers_enabled_list }}"
+- name: Find worker systemd services to be cleaned
+  find:
+    path: "{{ matrix_systemd_path }}"
+    patterns: "matrix-synapse-worker.*.service"
+    use_regex: true
+  register: matrix_synapse_workers_current_systemd_services
 
-- name: Add workers to synapse.wants list
-  set_fact:
-    matrix_synapse_systemd_wanted_services_list: >
-      {{ matrix_synapse_systemd_wanted_services_list +
-      ['matrix-synapse-worker@' + item.type + ':' + item.port|string + '.service'] }}
+- name: Ensure previous worker systemd services are cleaned
+  file:
+    path: "{{ item.path }}"
+    state: absent
+  with_items: "{{ matrix_synapse_workers_current_systemd_services.files }}"
+
+- name: Ensure creation of worker systemd service files and configuration files
+  include_tasks: "{{ role_path }}/tasks/workers/util/setup_files_for_worker.yml"
   with_items: "{{ matrix_synapse_workers_enabled_list }}"
-
-- name: Ensure matrix-synapse-worker-write-pid script is created
-  copy:
-    src: "{{ role_path }}/files/usr-local-bin/matrix-synapse-worker-write-pid"
-    dest: "{{ matrix_local_bin_path }}/matrix-synapse-worker-write-pid"
-    mode: 0750
-
-- name: Ensure jq is installed (Archlinux)
-  pacman:
-    name:
-      - jq
-    state: present
-  when: (ansible_distribution == 'Archlinux')
-
-- name: Ensure jq is installed (CentOS)
-  yum:
-    name:
-      - jq
-    state: present
-  when: (ansible_distribution == 'CentOS')
-
-- name: Ensure jq is installed (Debian)
-  apt:
-    name:
-      - jq
-    state: present
-  when: (ansible_os_family == 'Debian')
+  loop_control:
+    loop_var: matrix_synapse_worker_details

roles/matrix-synapse/tasks/workers/setup_uninstall.yml

@@ -7,46 +7,30 @@
   service:
     name: "{{ item.key }}"
     state: stopped
-  with_dict: "{{ ansible_facts.services|default({})|dict2items|selectattr('key', 'match', 'matrix-synapse-worker@.+\\.service')|list|items2dict }}"
-
-- name: Ensure worker service symlinks are cleaned
-  file:
-    path: "{{ item.root + '/' + item.path }}"
-    state: absent
-  when:
-    - item.state == 'link'
-    - item.path is match('matrix-synapse-worker@.*\\.service')
-  with_filetree:
-    - "{{ matrix_systemd_path }}/matrix-synapse.service.wants"
-
-- name: Ensure synapse worker base service file gets removed
-  file:
-    path: "{{ matrix_systemd_path }}/matrix-synapse-worker@.service"
-    state: absent
-  register: matrix_synapse_worker_systemd_service_result
+  with_dict: "{{ ansible_facts.services|default({})|dict2items|selectattr('key', 'match', 'matrix-synapse-worker-.+\\.service')|list|items2dict }}"
 
 - name: Find worker configs to be cleaned
   find:
     path: "{{ matrix_synapse_config_dir_path }}"
     patterns: "worker.*.yaml"
     use_regex: true
-  register: worker_config_files
+  register: matrix_synapse_workers_current_config_files
 
-- name: Ensure worker configs are cleaned
+- name: Ensure previous worker configs are cleaned
   file:
     path: "{{ item.path }}"
     state: absent
-  with_items: "{{ worker_config_files.files }}"
+  with_items: "{{ matrix_synapse_workers_current_config_files.files }}"
 
-- name: Remove workers from synapse.wants list
-  set_fact:
-    matrix_synapse_systemd_wanted_services_list: "{{ matrix_synapse_systemd_wanted_services_list | reject('search', '^matrix-synapse-worker@') | list }}"
+- name: Find worker systemd services to be cleaned
+  find:
+    path: "{{ matrix_systemd_path }}"
+    patterns: "matrix-synapse-worker.*.service"
+    use_regex: true
+  register: matrix_synapse_workers_current_systemd_services
 
-- name: Ensure systemd noticed removal of worker service units
-  service:
-    daemon_reload: yes
-
-- name: Ensure matrix-synapse-worker-write-pid script is removed
+- name: Ensure previous worker systemd services are cleaned
   file:
-    path: "{{ matrix_local_bin_path }}/matrix-synapse-worker-write-pid"
+    path: "{{ item.path }}"
     state: absent
+  with_items: "{{ matrix_synapse_workers_current_systemd_services.files }}"

roles/matrix-synapse/tasks/workers/util/inject_systemd_services_for_worker.yml

@@ -0,0 +1,6 @@
+
+- set_fact:
+    matrix_synapse_worker_systemd_service_name: "matrix-synapse-worker-{{ matrix_synapse_worker_details.type }}-{{ matrix_synapse_worker_details.port }}.service"
+
+- set_fact:
+    matrix_systemd_services_list: "{{ matrix_systemd_services_list + [matrix_synapse_worker_systemd_service_name] }}"

roles/matrix-synapse/tasks/workers/util/setup_files_for_worker.yml

@@ -0,0 +1,20 @@
+
+- set_fact:
+    matrix_synapse_worker_systemd_service_name: "matrix-synapse-worker-{{ matrix_synapse_worker_details.type }}-{{ matrix_synapse_worker_details.port }}"
+
+- set_fact:
+    matrix_synapse_worker_container_name: "{{ matrix_synapse_worker_systemd_service_name }}"
+
+- set_fact:
+    matrix_synapse_worker_config_file_name: "worker.{{ matrix_synapse_worker_details.type }}_{{ matrix_synapse_worker_details.port }}.yaml"
+
+- name: Ensure configuration exists for {{ matrix_synapse_worker_systemd_service_name }}
+  template:
+    src: "{{ role_path }}/templates/synapse/worker.yaml.j2"
+    dest: "{{ matrix_synapse_config_dir_path }}/{{ matrix_synapse_worker_config_file_name }}"
+
+- name: Ensure systemd service exists for {{ matrix_synapse_worker_systemd_service_name }}
+  template:
+    src: "{{ role_path }}/templates/synapse/systemd/matrix-synapse-worker.service.j2"
+    dest: "{{ matrix_systemd_path }}/{{ matrix_synapse_worker_systemd_service_name }}.service"
+    mode: 0644

roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker.service.j2

@@ -0,0 +1,58 @@
+#jinja2: lstrip_blocks: "True"
+[Unit]
+Description=Synapse worker ({{ matrix_synapse_worker_container_name }})
+AssertPathExists={{ matrix_synapse_config_dir_path }}/{{ matrix_synapse_worker_config_file_name }}
+After=matrix-synapse.service
+
+[Service]
+Type=simple
+Environment="HOME={{ matrix_systemd_unit_home_path }}"
+
+ExecStartPre=-{{ matrix_host_command_docker }} kill {{ matrix_synapse_worker_container_name }}
+ExecStartPre=-{{ matrix_host_command_docker }} rm {{ matrix_synapse_worker_container_name }}
+
+# Intentional delay, so that the homeserver can manage to start.
+ExecStartPre={{ matrix_host_command_sleep }} 5
+
+ExecStart={{ matrix_host_command_docker }} run --rm --name {{ matrix_synapse_worker_container_name }} \
+			--log-driver=none \
+			--user={{ matrix_user_uid }}:{{ matrix_user_gid }} \
+			--cap-drop=ALL \
+			--entrypoint=python \
+			--read-only \
+			--tmpfs=/tmp:rw,noexec,nosuid,size={{ matrix_synapse_tmp_directory_size_mb }}m \
+			--network={{ matrix_docker_network }} \
+			{% if matrix_synapse_workers_enabled and matrix_synapse_workers_container_host_bind_address %}
+			{% if matrix_synapse_worker_details.port != 0 %}
+			-p {{ '' if matrix_synapse_workers_container_host_bind_address == '*' else (matrix_synapse_workers_container_host_bind_address + ':') }}{{ matrix_synapse_worker_details.port }}:{{ matrix_synapse_worker_details.port }} \
+			{% endif %}
+			{% if matrix_synapse_worker_details.metrics_port != 0 %}
+			-p {{ '' if matrix_synapse_workers_container_host_bind_address == '*' else (matrix_synapse_workers_container_host_bind_address + ':') }}{{ matrix_synapse_worker_details.metrics_port }}:{{ matrix_synapse_worker_details.metrics_port }} \
+			{% endif %}
+			{% endif %}
+			--mount type=bind,src={{ matrix_synapse_config_dir_path }},dst=/data,ro \
+			--mount type=bind,src={{ matrix_synapse_storage_path }},dst=/matrix-media-store-parent,bind-propagation=slave \
+			{% for volume in matrix_synapse_container_additional_volumes %}
+			-v {{ volume.src }}:{{ volume.dst }}:{{ volume.options }} \
+			{% endfor %}
+			{% for arg in matrix_synapse_container_extra_arguments %}
+			{{ arg }} \
+			{% endfor %}
+			{{ matrix_synapse_docker_image }} \
+			-m synapse.app.{{ matrix_synapse_worker_details.type }} -c /data/homeserver.yaml -c /data/{{ matrix_synapse_worker_config_file_name }}
+
+
+ExecStop=-{{ matrix_host_command_docker }} kill {{ matrix_synapse_worker_container_name }}
+ExecStop=-{{ matrix_host_command_docker }} rm {{ matrix_synapse_worker_container_name }}
+
+ExecReload={{ matrix_host_command_docker }} exec {{ matrix_synapse_worker_container_name }} /bin/sh -c 'kill -HUP 1'
+Restart=always
+RestartSec=30
+SyslogIdentifier={{ matrix_synapse_worker_container_name }}
+
+# Intentionally not making this WantedBy=matrix-synapse.service,
+# as matrix.synapse.service already has `Wants=` lines.
+# Also, WantedBy will trigger the creation of some `matrix-synapse.service.wants/` directory,
+# which we'd have to clean, etc. Better not.
+[Install]
+WantedBy=multi-user.target

roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker@.service.j2

@@ -1,39 +0,0 @@
-#jinja2: lstrip_blocks: "True"
-# Instantiable worker service, running inside the synapse container
-# alongside the homeserver main process.
-# c.f. https://github.com/matrix-org/synapse/pull/4662
-[Unit]
-Description=Matrix worker synapse.app.%i
-AssertPathExists={{ matrix_synapse_config_dir_path }}/worker.%i.yaml
-After=matrix-synapse.service
-BindsTo=matrix-synapse.service
-
-[Service]
-Type=simple
-
-# Intentional delay, so that the homeserver can manage to start.
-ExecStartPre={{ matrix_host_command_sleep }} 5
-
-# no sane way of instancing more than one variable (systemd "cant-fix" 🤦)
-# c.f. https://github.com/systemd/systemd/issues/14895#issuecomment-594123923
-# So use good ol' shell parameter expansion to get the worker type..
-ExecStart=/bin/sh -c "WORKER=%i; WORKER=$${WORKER%%:*}; \
-			exec {{ matrix_host_command_docker }} exec \
-			--user={{ matrix_user_uid }}:{{ matrix_user_gid }} \
-			matrix-synapse \
-			python -m synapse.app.$${WORKER} -c /data/homeserver.yaml -c /data/worker.%i.yaml"
-
-# wait for worker startup & write out PID of actual worker process so systemd can handle it
-ExecStartPost={{ matrix_host_command_sleep }} 5
-ExecStartPost={{ matrix_local_bin_path }}/matrix-synapse-worker-write-pid %i /run/matrix-synapse-worker.%i.pid
-
-ExecReload=/bin/kill -HUP $MAINPID
-ExecStop=/bin/kill $MAINPID
-PIDFile=/run/matrix-synapse-worker.%i.pid
-KillMode=process
-Restart=always
-RestartSec=10
-SyslogIdentifier=matrix-synapse-%i
-
-[Install]
-WantedBy=matrix-synapse.service

roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2

@@ -4,10 +4,18 @@ Description=Synapse server
 {% for service in matrix_synapse_systemd_required_services_list %}
 Requires={{ service }}
 After={{ service }}
+
 {% endfor %}
 {% for service in matrix_synapse_systemd_wanted_services_list %}
 Wants={{ service }}
 {% endfor %}
+
+{% if matrix_synapse_workers_enabled %}
+{% for matrix_synapse_worker_details in matrix_synapse_workers_enabled_list %}
+Wants=matrix-synapse-worker-{{ matrix_synapse_worker_details.type }}-{{ matrix_synapse_worker_details.port }}.service
+{% endfor %}
+{% endif %}
+
 DefaultDependencies=no
 
 [Service]
@@ -45,16 +53,6 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-synapse \
 			{% if matrix_synapse_manhole_enabled and matrix_synapse_container_manhole_api_host_bind_port %}
 			-p {{ matrix_synapse_container_manhole_api_host_bind_port }}:9000 \
 			{% endif %}
-			{% if matrix_synapse_workers_enabled and matrix_synapse_workers_container_host_bind_address %}
-			{% for worker in matrix_synapse_workers_enabled_list %}
-			{% if worker.port != 0 %}
-			-p {{ '' if matrix_synapse_workers_container_host_bind_address == '*' else (matrix_synapse_workers_container_host_bind_address + ':') }}{{ worker.port }}:{{ worker.port }} \
-			{% endif %}
-			{% if worker.metrics_port != 0 %}
-			-p {{ '' if matrix_synapse_workers_container_host_bind_address == '*' else (matrix_synapse_workers_container_host_bind_address + ':') }}{{ worker.metrics_port }}:{{ worker.metrics_port }} \
-			{% endif %}
-			{% endfor %}
-			{% endif %}
 			--mount type=bind,src={{ matrix_synapse_config_dir_path }},dst=/data,ro \
 			--mount type=bind,src={{ matrix_synapse_storage_path }},dst=/matrix-media-store-parent,bind-propagation=slave \
 			{% for volume in matrix_synapse_container_additional_volumes %}
@@ -68,7 +66,7 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-synapse \
 
 ExecStop=-{{ matrix_host_command_docker }} kill matrix-synapse
 ExecStop=-{{ matrix_host_command_docker }} rm matrix-synapse
-ExecReload={{ matrix_host_command_docker }} exec matrix-synapse kill -HUP 1
+ExecReload={{ matrix_host_command_docker }} exec matrix-synapse /bin/sh -c 'kill -HUP 1'
 Restart=always
 RestartSec=30
 SyslogIdentifier=matrix-synapse

roles/matrix-synapse/templates/synapse/worker.yaml.j2

@@ -1,32 +1,32 @@
 #jinja2: lstrip_blocks: "True"
-worker_app: synapse.app.{{ item.type }}
-worker_name: {{ item.type ~ ':' ~ item.port }}
+worker_app: synapse.app.{{ matrix_synapse_worker_details.type }}
+worker_name: {{ matrix_synapse_worker_details.type ~ ':' ~ matrix_synapse_worker_details.port }}
 
-worker_replication_host: 127.0.0.1
+worker_replication_host: matrix-synapse
 worker_replication_http_port: {{ matrix_synapse_replication_http_port }}
 
 worker_listeners:
-{% if item.type not in [ 'appservice', 'federation_sender', 'pusher' ] %}
+{% if matrix_synapse_worker_details.type not in [ 'appservice', 'federation_sender', 'pusher' ] %}
   - type: http
-    port: {{ item.port }}
+    port: {{ matrix_synapse_worker_details.port }}
     resources:
       - names:
-{% if item.type in [ 'generic_worker', 'frontend_proxy', 'user_dir' ] %}
+{% if matrix_synapse_worker_details.type in [ 'generic_worker', 'frontend_proxy', 'user_dir' ] %}
         - client
 {% endif %}
-{% if item.type in [ 'generic_worker' ] %}
+{% if matrix_synapse_worker_details.type in [ 'generic_worker' ] %}
         - federation
-{% elif item.type in [ 'media_repository' ] %}
+{% elif matrix_synapse_worker_details.type in [ 'media_repository' ] %}
         - media
 {% endif %}
 
 {% endif %}
   - type: metrics
     bind_address: ['127.0.0.1']
-    port: {{ item.metrics_port }}
+    port: {{ matrix_synapse_worker_details.metrics_port }}
 
-{% if item.type == 'frontend_proxy' %}
-worker_main_http_uri: http://127.0.0.1:8008
+{% if matrix_synapse_worker_details.type == 'frontend_proxy' %}
+worker_main_http_uri: http://matrix-synapse:8008
 {% endif %}
 
 worker_daemonize: false