Move roles/matrix* to roles/custom/matrix*

This paves the way for installing other roles into `roles/galaxy` using `ansible-galaxy`,
similar to how it's done in:

- https://github.com/spantaleev/gitea-docker-ansible-deploy
- https://github.com/spantaleev/nextcloud-docker-ansible-deploy

In the near future, we'll be removing a lot of the shared role code from here
and using upstream roles for it. Some of the core `matrix-*` roles have
already been extracted out into other reusable roles:

- https://github.com/devture/com.devture.ansible.role.postgres
- https://github.com/devture/com.devture.ansible.role.systemd_docker_base
- https://github.com/devture/com.devture.ansible.role.timesync
- https://github.com/devture/com.devture.ansible.role.vars_preserver
- https://github.com/devture/com.devture.ansible.role.playbook_runtime_messages
- https://github.com/devture/com.devture.ansible.role.playbook_help

We just need to migrate to those.

roles/custom/matrix-synapse/tasks/synapse/workers/init.yml

@@ -0,0 +1,152 @@
+---
+# Below is a huge hack for dynamically building a list of workers and finally assigning it to `matrix_synapse_workers_enabled_list`.
+#
+# set_fact within a loop does not work reliably in Ansible (it only executes on the first iteration for some reason),
+# so we're forced to do something much uglier.
+
+- name: Build generic workers
+  ansible.builtin.set_fact:
+    worker:
+      id: "generic-worker-{{ item }}"
+      name: "matrix-synapse-worker-generic-{{ item }}"
+      type: 'generic_worker'
+      app: 'generic_worker'
+      webserving: true
+      port: "{{ matrix_synapse_workers_generic_workers_port_range_start + item }}"
+      metrics_port: "{{ matrix_synapse_workers_generic_workers_metrics_range_start + item }}"
+  register: "matrix_synapse_workers_list_results_generic_workers"
+  loop: "{{ range(0, matrix_synapse_workers_generic_workers_count | int) | list }}"
+
+- name: Build stream writer workers
+  ansible.builtin.set_fact:
+    worker:
+      id: "stream-writer-{{ idx }}-{{ item.stream }}"
+      # Names must not include understores. Certain stream writer streams (to_device, account_data, ..) do, so we fix them up.
+      name: "matrix-synapse-worker-stream-writer-{{ idx }}-{{ item.stream | replace('_', '-') }}"
+      type: 'stream_writer'
+      app: "generic_worker"
+      webserving: "{{ item.stream in matrix_synapse_workers_webserving_stream_writer_types }}"
+      stream_writer_stream: "{{ item.stream }}"
+      port: "{{ matrix_synapse_workers_stream_writer_workers_http_port_range_start + idx }}"
+      replication_port: "{{ matrix_synapse_workers_stream_writer_workers_replication_port_range_start + idx }}"
+      metrics_port: "{{ matrix_synapse_workers_stream_writer_workers_metrics_range_start + idx }}"
+  register: "matrix_synapse_workers_list_results_stream_writer_workers"
+  loop: "{{ matrix_synapse_workers_stream_writers }}"
+  loop_control:
+    index_var: idx
+
+- name: Populate matrix_synapse_stream_writers from enabled stream writer workers list
+  ansible.builtin.set_fact:
+    matrix_synapse_stream_writers: "{{ matrix_synapse_stream_writers | combine({item.ansible_facts.worker.stream_writer_stream: [item.ansible_facts.worker.name]}, list_merge='append') }}"
+  with_items: "{{ matrix_synapse_workers_list_results_stream_writer_workers.results }}"
+
+- name: Build federation sender workers
+  ansible.builtin.set_fact:
+    worker:
+      id: "federation-sender-{{ item }}"
+      name: "matrix-synapse-worker-federation-sender-{{ item }}"
+      type: 'federation_sender'
+      app: 'federation_sender'
+      webserving: false
+      port: 0
+      metrics_port: "{{ matrix_synapse_workers_federation_sender_workers_metrics_range_start + item }}"
+  register: "matrix_synapse_workers_list_results_federation_sender_workers"
+  loop: "{{ range(0, matrix_synapse_workers_federation_sender_workers_count | int) | list }}"
+
+- name: Populate matrix_synapse_federation_sender_instances from enabled federation sender workers list
+  ansible.builtin.set_fact:
+    matrix_synapse_federation_sender_instances: "{{ matrix_synapse_federation_sender_instances + [item.ansible_facts.worker.name] }}"
+  with_items: "{{ matrix_synapse_workers_list_results_federation_sender_workers.results }}"
+
+# This type of worker can only have a count of 1, at most
+- name: Build pusher workers
+  ansible.builtin.set_fact:
+    worker:
+      id: "pusher-{{ item }}"
+      name: "matrix-synapse-worker-pusher-{{ item }}"
+      type: 'pusher'
+      app: 'pusher'
+      webserving: false
+      port: 0
+      metrics_port: "{{ matrix_synapse_workers_pusher_workers_metrics_range_start + item }}"
+  register: "matrix_synapse_workers_list_results_pusher_workers"
+  loop: "{{ range(0, matrix_synapse_workers_pusher_workers_count | int) | list }}"
+
+# This type of worker can only have a count of 1, at most
+- name: Build appservice workers
+  ansible.builtin.set_fact:
+    worker:
+      id: "appservice-{{ item }}"
+      name: "matrix-synapse-worker-appservice-{{ item }}"
+      type: 'appservice'
+      app: 'generic_worker'
+      webserving: false
+      port: 0
+      metrics_port: "{{ matrix_synapse_workers_appservice_workers_metrics_range_start + item }}"
+  register: "matrix_synapse_workers_list_results_appservice_workers"
+  loop: "{{ range(0, matrix_synapse_workers_appservice_workers_count | int) | list }}"
+
+# This type of worker can only have a count of 1, at most
+- name: Build user_dir workers
+  ansible.builtin.set_fact:
+    worker:
+      id: "user-dir-{{ item }}"
+      name: "matrix-synapse-worker-user-dir-{{ item }}"
+      type: 'user_dir'
+      app: 'generic_worker'
+      webserving: true
+      port: "{{ matrix_synapse_workers_user_dir_workers_port_range_start + item }}"
+      metrics_port: "{{ matrix_synapse_workers_user_dir_workers_metrics_range_start + item }}"
+  register: "matrix_synapse_workers_list_results_user_dir_workers"
+  loop: "{{ range(0, matrix_synapse_workers_user_dir_workers_count | int) | list }}"
+
+# This type of worker can only have a count of 1, at most
+- name: Build background workers
+  ansible.builtin.set_fact:
+    worker:
+      id: "background-{{ item }}"
+      name: "matrix-synapse-worker-background-{{ item }}"
+      type: 'background'
+      app: 'generic_worker'
+      webserving: false
+      port: 0
+      metrics_port: "{{ matrix_synapse_workers_background_workers_metrics_range_start + item }}"
+  register: "matrix_synapse_workers_list_results_background_workers"
+  loop: "{{ range(0, matrix_synapse_workers_background_workers_count | int) | list }}"
+
+- name: Build media_repository workers
+  ansible.builtin.set_fact:
+    worker:
+      id: "media-repository-{{ item }}"
+      name: "matrix-synapse-worker-media-repository-{{ item }}"
+      type: 'media_repository'
+      app: 'media_repository'
+      webserving: true
+      port: "{{ matrix_synapse_workers_media_repository_workers_port_range_start + item }}"
+      metrics_port: "{{ matrix_synapse_workers_media_repository_workers_metrics_range_start + item }}"
+  register: "matrix_synapse_workers_list_results_media_repository_workers"
+  loop: "{{ range(0, matrix_synapse_workers_media_repository_workers_count | int) | list }}"
+
+- ansible.builtin.set_fact:
+    matrix_synapse_dynamic_workers_list: "{{ matrix_synapse_dynamic_workers_list | default([]) + [item.ansible_facts.worker] }}"
+  with_items: |
+    {{
+      matrix_synapse_workers_list_results_generic_workers.results
+      +
+      matrix_synapse_workers_list_results_stream_writer_workers.results
+      +
+      matrix_synapse_workers_list_results_federation_sender_workers.results
+      +
+      matrix_synapse_workers_list_results_pusher_workers.results
+      +
+      matrix_synapse_workers_list_results_appservice_workers.results
+      +
+      matrix_synapse_workers_list_results_user_dir_workers.results
+      +
+      matrix_synapse_workers_list_results_media_repository_workers.results
+      +
+      matrix_synapse_workers_list_results_background_workers.results
+    }}
+
+- ansible.builtin.set_fact:
+    matrix_synapse_workers_enabled_list: "{{ matrix_synapse_dynamic_workers_list }}"

roles/custom/matrix-synapse/tasks/synapse/workers/setup.yml

@@ -0,0 +1,21 @@
+---
+
+# A previous version of the worker setup used this.
+# This is a temporary cleanup for people who ran that version.
+- name: Ensure old matrix-synapse.service.wants directory is gone
+  ansible.builtin.file:
+    path: "{{ matrix_systemd_path }}/matrix-synapse.service.wants"
+    state: absent
+
+# Same. This was part of a previous version of the worker setup.
+# No longer necessary.
+- name: Ensure matrix-synapse-worker-write-pid script is removed
+  ansible.builtin.file:
+    path: "{{ matrix_local_bin_path }}/matrix-synapse-worker-write-pid"
+    state: absent
+
+- ansible.builtin.include_tasks: "{{ role_path }}/tasks/synapse/workers/setup_install.yml"
+  when: "matrix_synapse_enabled | bool and matrix_synapse_workers_enabled | bool"
+
+- ansible.builtin.include_tasks: "{{ role_path }}/tasks/synapse/workers/setup_uninstall.yml"
+  when: "not matrix_synapse_workers_enabled | bool"

roles/custom/matrix-synapse/tasks/synapse/workers/setup_install.yml

@@ -0,0 +1,42 @@
+---
+
+- name: Determine current worker configs
+  ansible.builtin.find:
+    path: "{{ matrix_synapse_config_dir_path }}"
+    patterns: "worker.*.yaml"
+    use_regex: true
+  register: matrix_synapse_workers_current_config_files
+
+# This also deletes some things which we need. They will be recreated below.
+- name: Ensure previous worker configs are cleaned
+  ansible.builtin.file:
+    path: "{{ item.path }}"
+    state: absent
+  with_items: "{{ matrix_synapse_workers_current_config_files.files }}"
+
+- name: Determine current worker systemd services
+  ansible.builtin.find:
+    path: "{{ matrix_systemd_path }}"
+    patterns: "matrix-synapse-worker.*.service"
+    use_regex: true
+  register: matrix_synapse_workers_current_systemd_services
+
+- name: Ensure unnecessary worker systemd services are stopped and disabled
+  ansible.builtin.service:
+    name: "{{ item.path | basename }}"
+    state: stopped
+    enabled: false
+  with_items: "{{ matrix_synapse_workers_current_systemd_services.files }}"
+  when: "not ansible_check_mode and item.path | basename not in matrix_systemd_services_list"
+
+- name: Ensure unnecessary worker systemd services are cleaned
+  ansible.builtin.file:
+    path: "{{ item.path }}"
+    state: absent
+  with_items: "{{ matrix_synapse_workers_current_systemd_services.files }}"
+
+- name: Ensure creation of worker systemd service files and configuration files
+  ansible.builtin.include_tasks: "{{ role_path }}/tasks/synapse/workers/util/setup_files_for_worker.yml"
+  with_items: "{{ matrix_synapse_workers_enabled_list }}"
+  loop_control:
+    loop_var: matrix_synapse_worker_details

roles/custom/matrix-synapse/tasks/synapse/workers/setup_uninstall.yml

@@ -0,0 +1,37 @@
+---
+
+- name: Populate service facts
+  ansible.builtin.service_facts:
+
+- name: Ensure any worker services are stopped
+  ansible.builtin.service:
+    name: "{{ item.key }}"
+    state: stopped
+  with_dict: "{{ ansible_facts.services | default({}) | dict2items | selectattr('key', 'match', 'matrix-synapse-worker-.+\\.service') | list | items2dict }}"
+  when: "item.value['status'] != 'not-found'"  # see https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/1461
+
+- name: Find worker configs to be cleaned
+  ansible.builtin.find:
+    path: "{{ matrix_synapse_config_dir_path }}"
+    patterns: "worker.*.yaml"
+    use_regex: true
+  register: matrix_synapse_workers_current_config_files
+
+- name: Ensure previous worker configs are cleaned
+  ansible.builtin.file:
+    path: "{{ item.path }}"
+    state: absent
+  with_items: "{{ matrix_synapse_workers_current_config_files.files }}"
+
+- name: Find worker systemd services to be cleaned
+  ansible.builtin.find:
+    path: "{{ matrix_systemd_path }}"
+    patterns: "matrix-synapse-worker.*.service"
+    use_regex: true
+  register: matrix_synapse_workers_current_systemd_services
+
+- name: Ensure previous worker systemd services are cleaned
+  ansible.builtin.file:
+    path: "{{ item.path }}"
+    state: absent
+  with_items: "{{ matrix_synapse_workers_current_systemd_services.files }}"

roles/custom/matrix-synapse/tasks/synapse/workers/util/inject_worker.yml

@@ -0,0 +1,65 @@
+---
+# The tasks below run before `validate_config.yml`.
+# To avoid failing with a cryptic error message, we'll do validation here.
+#
+# This check is mostly relevant to people who explicitly define `matrix_synapse_workers_enabled_list`
+# (Synapse Workers users from the earlier days of this PR - https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/456).
+#
+# In the future, it should be possible to remove this check.
+# Our own code which dynamically builds `matrix_synapse_workers_enabled_list` does things right.
+- name: Fail if required property not defined for worker
+  ansible.builtin.fail:
+    msg: "Synapse workers (like {{ matrix_synapse_worker_details | to_json }}) need to define a `{{ item }}` property"
+  with_items:
+    - id
+    - name
+    - type
+    - app
+    - port
+    - webserving
+  when: "item not in matrix_synapse_worker_details"
+
+# Names are used for container names and systemd services.
+# Routing happens based on container names, so Synapse processes that try to route to workers with underscores in the name will complain. Example:
+# > InvalidCodepoint Codepoint U+005F at position 46 of 'matrix-synapse-worker-stream-writer-3-account_data' not allowed
+- name: Fail if worker name includes underscore
+  ansible.builtin.fail:
+    msg: "Unrecognized Synapse worker `name`: `{{ matrix_synapse_worker_details.name }}`. It must not include underscores"
+  when: "'_' in matrix_synapse_worker_details.name"
+
+- name: Fail if worker type unknown
+  ansible.builtin.fail:
+    msg: "Unrecognized Synapse worker `type`: `{{ matrix_synapse_worker_details.type }}`. Supported types are: {{ matrix_synapse_known_worker_types | join(', ') }}"
+  when: "matrix_synapse_worker_details.type not in matrix_synapse_known_worker_types"
+
+- name: Fail if worker app unknown
+  ansible.builtin.fail:
+    msg: "Unrecognized Synapse worker `app`: `{{ matrix_synapse_worker_details.app }}`. Supported types are: {{ matrix_synapse_workers_avail_list | join(', ') }}"
+  when: "matrix_synapse_worker_details.app not in matrix_synapse_workers_avail_list"
+
+- when: "matrix_synapse_worker_details.type == 'stream_writer'"
+  block:
+    - name: Fail if stream_writer_stream not defined for stream_writer worker
+      ansible.builtin.fail:
+        msg: >-
+          Synapse stream_writer workers (such as {{ item }}) need to define a valid `stream_writer_stream` property
+          (not `{{ matrix_synapse_worker_details.stream_writer_stream | default('undefined') }}`).
+          Supported types are: {{ matrix_synapse_workers_known_stream_writer_stream_types | join(', ') }}
+      when: "'stream_writer_stream' not in matrix_synapse_worker_details or matrix_synapse_worker_details.stream_writer_stream not in matrix_synapse_workers_known_stream_writer_stream_types"
+
+    - name: Fail if replication_port not defined for stream_writer worker
+      ansible.builtin.fail:
+        msg: "Synapse background workers of type stream_writer (such as {{ item }}) need to define a valid `replication_port` property"
+      when: "'replication_port' not in matrix_synapse_worker_details"
+
+- ansible.builtin.set_fact:
+    matrix_systemd_services_list: "{{ matrix_systemd_services_list + [matrix_synapse_worker_details.name + '.service'] }}"
+
+- ansible.builtin.set_fact:
+    matrix_synapse_webserving_workers_systemd_services_list: "{{ matrix_synapse_webserving_workers_systemd_services_list + [matrix_synapse_worker_details.name + '.service'] }}"
+  when: matrix_synapse_worker_details.webserving | bool
+
+# Inject stream writers into the instance map.
+- ansible.builtin.set_fact:
+    matrix_synapse_instance_map: "{{ matrix_synapse_instance_map | combine({matrix_synapse_worker_details.name: {'host': matrix_synapse_worker_details.name, 'port': matrix_synapse_worker_details.replication_port}}) }}"
+  when: matrix_synapse_worker_details.type in matrix_synapse_known_instance_map_eligible_worker_types

roles/custom/matrix-synapse/tasks/synapse/workers/util/setup_files_for_worker.yml

@@ -0,0 +1,20 @@
+---
+
+- ansible.builtin.set_fact:
+    matrix_synapse_worker_systemd_service_name: "{{ matrix_synapse_worker_details.name }}"
+    matrix_synapse_worker_container_name: "{{ matrix_synapse_worker_details.name }}"
+    matrix_synapse_worker_config_file_name: "worker.{{ matrix_synapse_worker_details.name }}.yaml"
+
+- name: Ensure configuration exists for {{ matrix_synapse_worker_systemd_service_name }}
+  ansible.builtin.template:
+    src: "{{ role_path }}/templates/synapse/worker.yaml.j2"
+    dest: "{{ matrix_synapse_config_dir_path }}/{{ matrix_synapse_worker_config_file_name }}"
+    mode: 0644
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_groupname }}"
+
+- name: Ensure systemd service exists for {{ matrix_synapse_worker_systemd_service_name }}
+  ansible.builtin.template:
+    src: "{{ role_path }}/templates/synapse/systemd/matrix-synapse-worker.service.j2"
+    dest: "{{ matrix_systemd_path }}/{{ matrix_synapse_worker_systemd_service_name }}.service"
+    mode: 0644