Run Synapse workers in their own containers

This switches the `docker exec` method of spawning
Synapse workers inside the `matrix-synapse` container with
dedicated containers for each worker.

We also have dedicated systemd services for each worker,
so this are now:
- more consistent with everything else (we don't use systemd
instantiated services anywhere)
- we don't need the "parse systemd instance name into worker name +
port" part
- we don't need to keep track of PIDs manually
- we don't need jq (less depenendencies)
- workers dying would be restarted by systemd correctly, like any other
service
- `docker ps` shows each worker separately and we can observe resource
usage
This commit is contained in:
Slavi Pantaleev 2021-01-25 12:14:46 +02:00
parent da50fb27a0
commit 70796703d3
12 changed files with 160 additions and 190 deletions

View File

@ -14,7 +14,7 @@
ip_hash;
{% for worker in generic_workers %}
server "matrix-synapse:{{ worker.port }}";
server "matrix-synapse-worker-{{ worker.type }}-{{ worker.port }}:{{ worker.port }}";
{% endfor %}
}
{% endif %}
@ -22,7 +22,7 @@
{% if frontend_proxy_workers %}
upstream frontend_proxy_upstream {
{% for worker in frontend_proxy_workers %}
server "matrix-synapse:{{ worker.port }}";
server "matrix-synapse-worker-{{ worker.type }}-{{ worker.port }}:{{ worker.port }}";
{% endfor %}
}
{% endif %}
@ -30,7 +30,7 @@
{% if media_repository_workers %}
upstream media_repository_upstream {
{% for worker in media_repository_workers %}
server "matrix-synapse:{{ worker.port }}";
server "matrix-synapse-worker-{{ worker.type }}-{{ worker.port }}:{{ worker.port }}";
{% endfor %}
}
{% endif %}
@ -38,7 +38,7 @@
{% if user_dir_workers %}
upstream user_dir_upstream {
{% for worker in user_dir_workers %}
server "matrix-synapse:{{ worker.port }}";
server "matrix-synapse-worker-{{ worker.type }}-{{ worker.port }}:{{ worker.port }}";
{% endfor %}
}
{% endif %}

View File

@ -1,30 +0,0 @@
#!/bin/bash
# Find a synapse worker's PID and write it to a file so systemd can manage it as a service
# example invocation:
# matrix-synapse-worker-write-pid user_dir:18700 /run/matrix-synapse-worker.user_dir:18700.pid
docker_api_call() { curl --silent --unix-socket /var/run/docker.sock ${@}; }
TARGETCONTAINER=matrix-synapse
TARGETWORKER=${1}
PIDFILE=${2}
# get ID list of subprocesses executed in $TARGETCONTAINER, and for each..
for EXECID in $(docker_api_call http://localhost/containers/${TARGETCONTAINER}/json | jq --raw-output '.ExecIDs[]')
do
# fetch detailed process info
EXECINFO=$(docker_api_call http://localhost/exec/${EXECID}/json)
# extract config file path from last command argument
WORKERCONFIGFILE=$(echo ${EXECINFO} | jq --raw-output .ProcessConfig.arguments[-1])
# reconstruct worker name
WORKERNAME=${WORKERCONFIGFILE#*/worker.}
WORKERNAME=${WORKERNAME%.yaml}
# if name matches the target worker: write out most recent PID & quit
[ "${WORKERNAME}" = "${TARGETWORKER}" ] \
&& echo ${EXECINFO} | jq --raw-output .Pid > ${PIDFILE} \
&& exit 0
done

View File

@ -2,6 +2,13 @@
matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-synapse.service'] }}"
when: matrix_synapse_enabled|bool
- name: Ensure systemd services for workers are injected
include_tasks: "{{ role_path }}/tasks/workers/util/inject_systemd_services_for_worker.yml"
with_items: "{{ matrix_synapse_workers_enabled_list }}"
loop_control:
loop_var: matrix_synapse_worker_details
when: matrix_synapse_enabled|bool and matrix_synapse_workers_enabled|bool
- set_fact:
matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-goofys.service'] }}"
when: matrix_s3_media_store_enabled|bool

View File

@ -1,5 +1,19 @@
---
# A previous version of the worker setup used this.
# This is a temporary cleanup for people who ran that version.
- name: Ensure old matrix-synapse.service.wants directory is gone
file:
path: "{{ matrix_systemd_path }}/matrix-synapse.service.wants"
state: absent
# Same. This was part of a previous version of the worker setup.
# No longer necessary.
- name: Ensure matrix-synapse-worker-write-pid script is removed
file:
path: "{{ matrix_local_bin_path }}/matrix-synapse-worker-write-pid"
state: absent
- include_tasks: "{{ role_path }}/tasks/workers/setup_install.yml"
when: "matrix_synapse_enabled|bool and matrix_synapse_workers_enabled|bool"

View File

@ -1,81 +1,33 @@
---
- name: Ensure synapse worker base service file installed
template:
src: "{{ role_path }}/templates/synapse/systemd/matrix-synapse-worker@.service.j2"
dest: "{{ matrix_systemd_path }}/matrix-synapse-worker@.service"
mode: 0644
register: matrix_synapse_worker_systemd_service_result
- name: Ensure previous worker service symlinks are cleaned
file:
path: "{{ item.root + '/' + item.path }}"
state: absent
when:
- item.state == 'link'
- item.path is match('matrix-synapse-worker@.*\\.service')
with_filetree:
- "{{ matrix_systemd_path }}/matrix-synapse.service.wants"
- name: Ensure systemd reloaded the worker service unit
service:
daemon_reload: yes
- name: Ensure individual worker service symlinks exist
service:
name: "matrix-synapse-worker@{{ item.type }}:{{ item.port }}.service"
enabled: true
with_items: "{{ matrix_synapse_workers_enabled_list }}"
- name: Find worker configs to be cleaned
find:
path: "{{ matrix_synapse_config_dir_path }}"
patterns: "worker.*.yaml"
use_regex: true
register: worker_config_files
register: matrix_synapse_workers_current_config_files
- name: Ensure previous worker configs are cleaned
file:
path: "{{ item.path }}"
state: absent
with_items: "{{ worker_config_files.files }}"
with_items: "{{ matrix_synapse_workers_current_config_files.files }}"
- name: Ensure creation of specific worker configs
template:
src: "{{ role_path }}/templates/synapse/worker.yaml.j2"
dest: "{{ matrix_synapse_config_dir_path }}/worker.{{ item.type }}:{{ item.port }}.yaml"
with_list: "{{ matrix_synapse_workers_enabled_list }}"
- name: Find worker systemd services to be cleaned
find:
path: "{{ matrix_systemd_path }}"
patterns: "matrix-synapse-worker.*.service"
use_regex: true
register: matrix_synapse_workers_current_systemd_services
- name: Add workers to synapse.wants list
set_fact:
matrix_synapse_systemd_wanted_services_list: >
{{ matrix_synapse_systemd_wanted_services_list +
['matrix-synapse-worker@' + item.type + ':' + item.port|string + '.service'] }}
- name: Ensure previous worker systemd services are cleaned
file:
path: "{{ item.path }}"
state: absent
with_items: "{{ matrix_synapse_workers_current_systemd_services.files }}"
- name: Ensure creation of worker systemd service files and configuration files
include_tasks: "{{ role_path }}/tasks/workers/util/setup_files_for_worker.yml"
with_items: "{{ matrix_synapse_workers_enabled_list }}"
- name: Ensure matrix-synapse-worker-write-pid script is created
copy:
src: "{{ role_path }}/files/usr-local-bin/matrix-synapse-worker-write-pid"
dest: "{{ matrix_local_bin_path }}/matrix-synapse-worker-write-pid"
mode: 0750
- name: Ensure jq is installed (Archlinux)
pacman:
name:
- jq
state: present
when: (ansible_distribution == 'Archlinux')
- name: Ensure jq is installed (CentOS)
yum:
name:
- jq
state: present
when: (ansible_distribution == 'CentOS')
- name: Ensure jq is installed (Debian)
apt:
name:
- jq
state: present
when: (ansible_os_family == 'Debian')
loop_control:
loop_var: matrix_synapse_worker_details

View File

@ -7,46 +7,30 @@
service:
name: "{{ item.key }}"
state: stopped
with_dict: "{{ ansible_facts.services|default({})|dict2items|selectattr('key', 'match', 'matrix-synapse-worker@.+\\.service')|list|items2dict }}"
- name: Ensure worker service symlinks are cleaned
file:
path: "{{ item.root + '/' + item.path }}"
state: absent
when:
- item.state == 'link'
- item.path is match('matrix-synapse-worker@.*\\.service')
with_filetree:
- "{{ matrix_systemd_path }}/matrix-synapse.service.wants"
- name: Ensure synapse worker base service file gets removed
file:
path: "{{ matrix_systemd_path }}/matrix-synapse-worker@.service"
state: absent
register: matrix_synapse_worker_systemd_service_result
with_dict: "{{ ansible_facts.services|default({})|dict2items|selectattr('key', 'match', 'matrix-synapse-worker-.+\\.service')|list|items2dict }}"
- name: Find worker configs to be cleaned
find:
path: "{{ matrix_synapse_config_dir_path }}"
patterns: "worker.*.yaml"
use_regex: true
register: worker_config_files
register: matrix_synapse_workers_current_config_files
- name: Ensure worker configs are cleaned
- name: Ensure previous worker configs are cleaned
file:
path: "{{ item.path }}"
state: absent
with_items: "{{ worker_config_files.files }}"
with_items: "{{ matrix_synapse_workers_current_config_files.files }}"
- name: Remove workers from synapse.wants list
set_fact:
matrix_synapse_systemd_wanted_services_list: "{{ matrix_synapse_systemd_wanted_services_list | reject('search', '^matrix-synapse-worker@') | list }}"
- name: Find worker systemd services to be cleaned
find:
path: "{{ matrix_systemd_path }}"
patterns: "matrix-synapse-worker.*.service"
use_regex: true
register: matrix_synapse_workers_current_systemd_services
- name: Ensure systemd noticed removal of worker service units
service:
daemon_reload: yes
- name: Ensure matrix-synapse-worker-write-pid script is removed
- name: Ensure previous worker systemd services are cleaned
file:
path: "{{ matrix_local_bin_path }}/matrix-synapse-worker-write-pid"
path: "{{ item.path }}"
state: absent
with_items: "{{ matrix_synapse_workers_current_systemd_services.files }}"

View File

@ -0,0 +1,6 @@
- set_fact:
matrix_synapse_worker_systemd_service_name: "matrix-synapse-worker-{{ matrix_synapse_worker_details.type }}-{{ matrix_synapse_worker_details.port }}.service"
- set_fact:
matrix_systemd_services_list: "{{ matrix_systemd_services_list + [matrix_synapse_worker_systemd_service_name] }}"

View File

@ -0,0 +1,20 @@
- set_fact:
matrix_synapse_worker_systemd_service_name: "matrix-synapse-worker-{{ matrix_synapse_worker_details.type }}-{{ matrix_synapse_worker_details.port }}"
- set_fact:
matrix_synapse_worker_container_name: "{{ matrix_synapse_worker_systemd_service_name }}"
- set_fact:
matrix_synapse_worker_config_file_name: "worker.{{ matrix_synapse_worker_details.type }}_{{ matrix_synapse_worker_details.port }}.yaml"
- name: Ensure configuration exists for {{ matrix_synapse_worker_systemd_service_name }}
template:
src: "{{ role_path }}/templates/synapse/worker.yaml.j2"
dest: "{{ matrix_synapse_config_dir_path }}/{{ matrix_synapse_worker_config_file_name }}"
- name: Ensure systemd service exists for {{ matrix_synapse_worker_systemd_service_name }}
template:
src: "{{ role_path }}/templates/synapse/systemd/matrix-synapse-worker.service.j2"
dest: "{{ matrix_systemd_path }}/{{ matrix_synapse_worker_systemd_service_name }}.service"
mode: 0644

View File

@ -0,0 +1,58 @@
#jinja2: lstrip_blocks: "True"
[Unit]
Description=Synapse worker ({{ matrix_synapse_worker_container_name }})
AssertPathExists={{ matrix_synapse_config_dir_path }}/{{ matrix_synapse_worker_config_file_name }}
After=matrix-synapse.service
[Service]
Type=simple
Environment="HOME={{ matrix_systemd_unit_home_path }}"
ExecStartPre=-{{ matrix_host_command_docker }} kill {{ matrix_synapse_worker_container_name }}
ExecStartPre=-{{ matrix_host_command_docker }} rm {{ matrix_synapse_worker_container_name }}
# Intentional delay, so that the homeserver can manage to start.
ExecStartPre={{ matrix_host_command_sleep }} 5
ExecStart={{ matrix_host_command_docker }} run --rm --name {{ matrix_synapse_worker_container_name }} \
--log-driver=none \
--user={{ matrix_user_uid }}:{{ matrix_user_gid }} \
--cap-drop=ALL \
--entrypoint=python \
--read-only \
--tmpfs=/tmp:rw,noexec,nosuid,size={{ matrix_synapse_tmp_directory_size_mb }}m \
--network={{ matrix_docker_network }} \
{% if matrix_synapse_workers_enabled and matrix_synapse_workers_container_host_bind_address %}
{% if matrix_synapse_worker_details.port != 0 %}
-p {{ '' if matrix_synapse_workers_container_host_bind_address == '*' else (matrix_synapse_workers_container_host_bind_address + ':') }}{{ matrix_synapse_worker_details.port }}:{{ matrix_synapse_worker_details.port }} \
{% endif %}
{% if matrix_synapse_worker_details.metrics_port != 0 %}
-p {{ '' if matrix_synapse_workers_container_host_bind_address == '*' else (matrix_synapse_workers_container_host_bind_address + ':') }}{{ matrix_synapse_worker_details.metrics_port }}:{{ matrix_synapse_worker_details.metrics_port }} \
{% endif %}
{% endif %}
--mount type=bind,src={{ matrix_synapse_config_dir_path }},dst=/data,ro \
--mount type=bind,src={{ matrix_synapse_storage_path }},dst=/matrix-media-store-parent,bind-propagation=slave \
{% for volume in matrix_synapse_container_additional_volumes %}
-v {{ volume.src }}:{{ volume.dst }}:{{ volume.options }} \
{% endfor %}
{% for arg in matrix_synapse_container_extra_arguments %}
{{ arg }} \
{% endfor %}
{{ matrix_synapse_docker_image }} \
-m synapse.app.{{ matrix_synapse_worker_details.type }} -c /data/homeserver.yaml -c /data/{{ matrix_synapse_worker_config_file_name }}
ExecStop=-{{ matrix_host_command_docker }} kill {{ matrix_synapse_worker_container_name }}
ExecStop=-{{ matrix_host_command_docker }} rm {{ matrix_synapse_worker_container_name }}
ExecReload={{ matrix_host_command_docker }} exec {{ matrix_synapse_worker_container_name }} /bin/sh -c 'kill -HUP 1'
Restart=always
RestartSec=30
SyslogIdentifier={{ matrix_synapse_worker_container_name }}
# Intentionally not making this WantedBy=matrix-synapse.service,
# as matrix.synapse.service already has `Wants=` lines.
# Also, WantedBy will trigger the creation of some `matrix-synapse.service.wants/` directory,
# which we'd have to clean, etc. Better not.
[Install]
WantedBy=multi-user.target

View File

@ -1,39 +0,0 @@
#jinja2: lstrip_blocks: "True"
# Instantiable worker service, running inside the synapse container
# alongside the homeserver main process.
# c.f. https://github.com/matrix-org/synapse/pull/4662
[Unit]
Description=Matrix worker synapse.app.%i
AssertPathExists={{ matrix_synapse_config_dir_path }}/worker.%i.yaml
After=matrix-synapse.service
BindsTo=matrix-synapse.service
[Service]
Type=simple
# Intentional delay, so that the homeserver can manage to start.
ExecStartPre={{ matrix_host_command_sleep }} 5
# no sane way of instancing more than one variable (systemd "cant-fix" 🤦)
# c.f. https://github.com/systemd/systemd/issues/14895#issuecomment-594123923
# So use good ol' shell parameter expansion to get the worker type..
ExecStart=/bin/sh -c "WORKER=%i; WORKER=$${WORKER%%:*}; \
exec {{ matrix_host_command_docker }} exec \
--user={{ matrix_user_uid }}:{{ matrix_user_gid }} \
matrix-synapse \
python -m synapse.app.$${WORKER} -c /data/homeserver.yaml -c /data/worker.%i.yaml"
# wait for worker startup & write out PID of actual worker process so systemd can handle it
ExecStartPost={{ matrix_host_command_sleep }} 5
ExecStartPost={{ matrix_local_bin_path }}/matrix-synapse-worker-write-pid %i /run/matrix-synapse-worker.%i.pid
ExecReload=/bin/kill -HUP $MAINPID
ExecStop=/bin/kill $MAINPID
PIDFile=/run/matrix-synapse-worker.%i.pid
KillMode=process
Restart=always
RestartSec=10
SyslogIdentifier=matrix-synapse-%i
[Install]
WantedBy=matrix-synapse.service

View File

@ -4,10 +4,18 @@ Description=Synapse server
{% for service in matrix_synapse_systemd_required_services_list %}
Requires={{ service }}
After={{ service }}
{% endfor %}
{% for service in matrix_synapse_systemd_wanted_services_list %}
Wants={{ service }}
{% endfor %}
{% if matrix_synapse_workers_enabled %}
{% for matrix_synapse_worker_details in matrix_synapse_workers_enabled_list %}
Wants=matrix-synapse-worker-{{ matrix_synapse_worker_details.type }}-{{ matrix_synapse_worker_details.port }}.service
{% endfor %}
{% endif %}
DefaultDependencies=no
[Service]
@ -45,16 +53,6 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-synapse \
{% if matrix_synapse_manhole_enabled and matrix_synapse_container_manhole_api_host_bind_port %}
-p {{ matrix_synapse_container_manhole_api_host_bind_port }}:9000 \
{% endif %}
{% if matrix_synapse_workers_enabled and matrix_synapse_workers_container_host_bind_address %}
{% for worker in matrix_synapse_workers_enabled_list %}
{% if worker.port != 0 %}
-p {{ '' if matrix_synapse_workers_container_host_bind_address == '*' else (matrix_synapse_workers_container_host_bind_address + ':') }}{{ worker.port }}:{{ worker.port }} \
{% endif %}
{% if worker.metrics_port != 0 %}
-p {{ '' if matrix_synapse_workers_container_host_bind_address == '*' else (matrix_synapse_workers_container_host_bind_address + ':') }}{{ worker.metrics_port }}:{{ worker.metrics_port }} \
{% endif %}
{% endfor %}
{% endif %}
--mount type=bind,src={{ matrix_synapse_config_dir_path }},dst=/data,ro \
--mount type=bind,src={{ matrix_synapse_storage_path }},dst=/matrix-media-store-parent,bind-propagation=slave \
{% for volume in matrix_synapse_container_additional_volumes %}
@ -68,7 +66,7 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-synapse \
ExecStop=-{{ matrix_host_command_docker }} kill matrix-synapse
ExecStop=-{{ matrix_host_command_docker }} rm matrix-synapse
ExecReload={{ matrix_host_command_docker }} exec matrix-synapse kill -HUP 1
ExecReload={{ matrix_host_command_docker }} exec matrix-synapse /bin/sh -c 'kill -HUP 1'
Restart=always
RestartSec=30
SyslogIdentifier=matrix-synapse

View File

@ -1,32 +1,32 @@
#jinja2: lstrip_blocks: "True"
worker_app: synapse.app.{{ item.type }}
worker_name: {{ item.type ~ ':' ~ item.port }}
worker_app: synapse.app.{{ matrix_synapse_worker_details.type }}
worker_name: {{ matrix_synapse_worker_details.type ~ ':' ~ matrix_synapse_worker_details.port }}
worker_replication_host: 127.0.0.1
worker_replication_host: matrix-synapse
worker_replication_http_port: {{ matrix_synapse_replication_http_port }}
worker_listeners:
{% if item.type not in [ 'appservice', 'federation_sender', 'pusher' ] %}
{% if matrix_synapse_worker_details.type not in [ 'appservice', 'federation_sender', 'pusher' ] %}
- type: http
port: {{ item.port }}
port: {{ matrix_synapse_worker_details.port }}
resources:
- names:
{% if item.type in [ 'generic_worker', 'frontend_proxy', 'user_dir' ] %}
{% if matrix_synapse_worker_details.type in [ 'generic_worker', 'frontend_proxy', 'user_dir' ] %}
- client
{% endif %}
{% if item.type in [ 'generic_worker' ] %}
{% if matrix_synapse_worker_details.type in [ 'generic_worker' ] %}
- federation
{% elif item.type in [ 'media_repository' ] %}
{% elif matrix_synapse_worker_details.type in [ 'media_repository' ] %}
- media
{% endif %}
{% endif %}
- type: metrics
bind_address: ['127.0.0.1']
port: {{ item.metrics_port }}
port: {{ matrix_synapse_worker_details.metrics_port }}
{% if item.type == 'frontend_proxy' %}
worker_main_http_uri: http://127.0.0.1:8008
{% if matrix_synapse_worker_details.type == 'frontend_proxy' %}
worker_main_http_uri: http://matrix-synapse:8008
{% endif %}
worker_daemonize: false