Run Synapse workers in their own containers

This switches the `docker exec` method of spawning
Synapse workers inside the `matrix-synapse` container with
dedicated containers for each worker.

We also have dedicated systemd services for each worker,
so this are now:
- more consistent with everything else (we don't use systemd
instantiated services anywhere)
- we don't need the "parse systemd instance name into worker name +
port" part
- we don't need to keep track of PIDs manually
- we don't need jq (less depenendencies)
- workers dying would be restarted by systemd correctly, like any other
service
- `docker ps` shows each worker separately and we can observe resource
usage

roles/matrix-synapse/tasks/init.yml

@@ -2,6 +2,13 @@
     matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-synapse.service'] }}"
   when: matrix_synapse_enabled|bool
 
+- name: Ensure systemd services for workers are injected
+  include_tasks: "{{ role_path }}/tasks/workers/util/inject_systemd_services_for_worker.yml"
+  with_items: "{{ matrix_synapse_workers_enabled_list }}"
+  loop_control:
+    loop_var: matrix_synapse_worker_details
+  when: matrix_synapse_enabled|bool and matrix_synapse_workers_enabled|bool
+
 - set_fact:
     matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-goofys.service'] }}"
   when: matrix_s3_media_store_enabled|bool

roles/matrix-synapse/tasks/workers/setup.yml

@@ -1,5 +1,19 @@
 ---
 
+# A previous version of the worker setup used this.
+# This is a temporary cleanup for people who ran that version.
+- name: Ensure old matrix-synapse.service.wants directory is gone
+  file:
+    path: "{{ matrix_systemd_path }}/matrix-synapse.service.wants"
+    state: absent
+
+# Same. This was part of a previous version of the worker setup.
+# No longer necessary.
+- name: Ensure matrix-synapse-worker-write-pid script is removed
+  file:
+    path: "{{ matrix_local_bin_path }}/matrix-synapse-worker-write-pid"
+    state: absent
+
 - include_tasks: "{{ role_path }}/tasks/workers/setup_install.yml"
   when: "matrix_synapse_enabled|bool and matrix_synapse_workers_enabled|bool"
 

roles/matrix-synapse/tasks/workers/setup_install.yml

@@ -1,81 +1,33 @@
 ---
 
-- name: Ensure synapse worker base service file installed
-  template:
-    src: "{{ role_path }}/templates/synapse/systemd/matrix-synapse-worker@.service.j2"
-    dest: "{{ matrix_systemd_path }}/matrix-synapse-worker@.service"
-    mode: 0644
-  register: matrix_synapse_worker_systemd_service_result
-
-- name: Ensure previous worker service symlinks are cleaned
-  file:
-    path: "{{ item.root + '/' + item.path }}"
-    state: absent
-  when:
-    - item.state == 'link'
-    - item.path is match('matrix-synapse-worker@.*\\.service')
-  with_filetree:
-    - "{{ matrix_systemd_path }}/matrix-synapse.service.wants"
-
-- name: Ensure systemd reloaded the worker service unit
-  service:
-    daemon_reload: yes
-
-- name: Ensure individual worker service symlinks exist
-  service:
-    name: "matrix-synapse-worker@{{ item.type }}:{{ item.port }}.service"
-    enabled: true
-  with_items: "{{ matrix_synapse_workers_enabled_list }}"
-
 - name: Find worker configs to be cleaned
   find:
     path: "{{ matrix_synapse_config_dir_path }}"
     patterns: "worker.*.yaml"
     use_regex: true
-  register: worker_config_files
+  register: matrix_synapse_workers_current_config_files
 
 - name: Ensure previous worker configs are cleaned
   file:
     path: "{{ item.path }}"
     state: absent
-  with_items: "{{ worker_config_files.files }}"
+  with_items: "{{ matrix_synapse_workers_current_config_files.files }}"
 
-- name: Ensure creation of specific worker configs
-  template:
-    src: "{{ role_path }}/templates/synapse/worker.yaml.j2"
-    dest: "{{ matrix_synapse_config_dir_path }}/worker.{{ item.type }}:{{ item.port }}.yaml"
-  with_list: "{{ matrix_synapse_workers_enabled_list }}"
+- name: Find worker systemd services to be cleaned
+  find:
+    path: "{{ matrix_systemd_path }}"
+    patterns: "matrix-synapse-worker.*.service"
+    use_regex: true
+  register: matrix_synapse_workers_current_systemd_services
 
-- name: Add workers to synapse.wants list
-  set_fact:
-    matrix_synapse_systemd_wanted_services_list: >
-      {{ matrix_synapse_systemd_wanted_services_list +
-      ['matrix-synapse-worker@' + item.type + ':' + item.port|string + '.service'] }}
+- name: Ensure previous worker systemd services are cleaned
+  file:
+    path: "{{ item.path }}"
+    state: absent
+  with_items: "{{ matrix_synapse_workers_current_systemd_services.files }}"
+
+- name: Ensure creation of worker systemd service files and configuration files
+  include_tasks: "{{ role_path }}/tasks/workers/util/setup_files_for_worker.yml"
   with_items: "{{ matrix_synapse_workers_enabled_list }}"
-
-- name: Ensure matrix-synapse-worker-write-pid script is created
-  copy:
-    src: "{{ role_path }}/files/usr-local-bin/matrix-synapse-worker-write-pid"
-    dest: "{{ matrix_local_bin_path }}/matrix-synapse-worker-write-pid"
-    mode: 0750
-
-- name: Ensure jq is installed (Archlinux)
-  pacman:
-    name:
-      - jq
-    state: present
-  when: (ansible_distribution == 'Archlinux')
-
-- name: Ensure jq is installed (CentOS)
-  yum:
-    name:
-      - jq
-    state: present
-  when: (ansible_distribution == 'CentOS')
-
-- name: Ensure jq is installed (Debian)
-  apt:
-    name:
-      - jq
-    state: present
-  when: (ansible_os_family == 'Debian')
+  loop_control:
+    loop_var: matrix_synapse_worker_details

roles/matrix-synapse/tasks/workers/setup_uninstall.yml

@@ -7,46 +7,30 @@
   service:
     name: "{{ item.key }}"
     state: stopped
-  with_dict: "{{ ansible_facts.services|default({})|dict2items|selectattr('key', 'match', 'matrix-synapse-worker@.+\\.service')|list|items2dict }}"
-
-- name: Ensure worker service symlinks are cleaned
-  file:
-    path: "{{ item.root + '/' + item.path }}"
-    state: absent
-  when:
-    - item.state == 'link'
-    - item.path is match('matrix-synapse-worker@.*\\.service')
-  with_filetree:
-    - "{{ matrix_systemd_path }}/matrix-synapse.service.wants"
-
-- name: Ensure synapse worker base service file gets removed
-  file:
-    path: "{{ matrix_systemd_path }}/matrix-synapse-worker@.service"
-    state: absent
-  register: matrix_synapse_worker_systemd_service_result
+  with_dict: "{{ ansible_facts.services|default({})|dict2items|selectattr('key', 'match', 'matrix-synapse-worker-.+\\.service')|list|items2dict }}"
 
 - name: Find worker configs to be cleaned
   find:
     path: "{{ matrix_synapse_config_dir_path }}"
     patterns: "worker.*.yaml"
     use_regex: true
-  register: worker_config_files
+  register: matrix_synapse_workers_current_config_files
 
-- name: Ensure worker configs are cleaned
+- name: Ensure previous worker configs are cleaned
   file:
     path: "{{ item.path }}"
     state: absent
-  with_items: "{{ worker_config_files.files }}"
+  with_items: "{{ matrix_synapse_workers_current_config_files.files }}"
 
-- name: Remove workers from synapse.wants list
-  set_fact:
-    matrix_synapse_systemd_wanted_services_list: "{{ matrix_synapse_systemd_wanted_services_list | reject('search', '^matrix-synapse-worker@') | list }}"
+- name: Find worker systemd services to be cleaned
+  find:
+    path: "{{ matrix_systemd_path }}"
+    patterns: "matrix-synapse-worker.*.service"
+    use_regex: true
+  register: matrix_synapse_workers_current_systemd_services
 
-- name: Ensure systemd noticed removal of worker service units
-  service:
-    daemon_reload: yes
-
-- name: Ensure matrix-synapse-worker-write-pid script is removed
+- name: Ensure previous worker systemd services are cleaned
   file:
-    path: "{{ matrix_local_bin_path }}/matrix-synapse-worker-write-pid"
+    path: "{{ item.path }}"
     state: absent
+  with_items: "{{ matrix_synapse_workers_current_systemd_services.files }}"

roles/matrix-synapse/tasks/workers/util/inject_systemd_services_for_worker.yml

@@ -0,0 +1,6 @@
+
+- set_fact:
+    matrix_synapse_worker_systemd_service_name: "matrix-synapse-worker-{{ matrix_synapse_worker_details.type }}-{{ matrix_synapse_worker_details.port }}.service"
+
+- set_fact:
+    matrix_systemd_services_list: "{{ matrix_systemd_services_list + [matrix_synapse_worker_systemd_service_name] }}"

roles/matrix-synapse/tasks/workers/util/setup_files_for_worker.yml

@@ -0,0 +1,20 @@
+
+- set_fact:
+    matrix_synapse_worker_systemd_service_name: "matrix-synapse-worker-{{ matrix_synapse_worker_details.type }}-{{ matrix_synapse_worker_details.port }}"
+
+- set_fact:
+    matrix_synapse_worker_container_name: "{{ matrix_synapse_worker_systemd_service_name }}"
+
+- set_fact:
+    matrix_synapse_worker_config_file_name: "worker.{{ matrix_synapse_worker_details.type }}_{{ matrix_synapse_worker_details.port }}.yaml"
+
+- name: Ensure configuration exists for {{ matrix_synapse_worker_systemd_service_name }}
+  template:
+    src: "{{ role_path }}/templates/synapse/worker.yaml.j2"
+    dest: "{{ matrix_synapse_config_dir_path }}/{{ matrix_synapse_worker_config_file_name }}"
+
+- name: Ensure systemd service exists for {{ matrix_synapse_worker_systemd_service_name }}
+  template:
+    src: "{{ role_path }}/templates/synapse/systemd/matrix-synapse-worker.service.j2"
+    dest: "{{ matrix_systemd_path }}/{{ matrix_synapse_worker_systemd_service_name }}.service"
+    mode: 0644