Run Synapse workers in their own containers
This switches the `docker exec` method of spawning Synapse workers inside the `matrix-synapse` container with dedicated containers for each worker. We also have dedicated systemd services for each worker, so this are now: - more consistent with everything else (we don't use systemd instantiated services anywhere) - we don't need the "parse systemd instance name into worker name + port" part - we don't need to keep track of PIDs manually - we don't need jq (less depenendencies) - workers dying would be restarted by systemd correctly, like any other service - `docker ps` shows each worker separately and we can observe resource usage
This commit is contained in:
Slavi Pantaleev
@ -0,0 +1,58 @@
|
||||
#jinja2: lstrip_blocks: "True"
|
||||
[Unit]
|
||||
Description=Synapse worker ({{ matrix_synapse_worker_container_name }})
|
||||
AssertPathExists={{ matrix_synapse_config_dir_path }}/{{ matrix_synapse_worker_config_file_name }}
|
||||
After=matrix-synapse.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
Environment="HOME={{ matrix_systemd_unit_home_path }}"
|
||||
|
||||
ExecStartPre=-{{ matrix_host_command_docker }} kill {{ matrix_synapse_worker_container_name }}
|
||||
ExecStartPre=-{{ matrix_host_command_docker }} rm {{ matrix_synapse_worker_container_name }}
|
||||
|
||||
# Intentional delay, so that the homeserver can manage to start.
|
||||
ExecStartPre={{ matrix_host_command_sleep }} 5
|
||||
|
||||
ExecStart={{ matrix_host_command_docker }} run --rm --name {{ matrix_synapse_worker_container_name }} \
|
||||
--log-driver=none \
|
||||
--user={{ matrix_user_uid }}:{{ matrix_user_gid }} \
|
||||
--cap-drop=ALL \
|
||||
--entrypoint=python \
|
||||
--read-only \
|
||||
--tmpfs=/tmp:rw,noexec,nosuid,size={{ matrix_synapse_tmp_directory_size_mb }}m \
|
||||
--network={{ matrix_docker_network }} \
|
||||
{% if matrix_synapse_workers_enabled and matrix_synapse_workers_container_host_bind_address %}
|
||||
{% if matrix_synapse_worker_details.port != 0 %}
|
||||
-p {{ '' if matrix_synapse_workers_container_host_bind_address == '*' else (matrix_synapse_workers_container_host_bind_address + ':') }}{{ matrix_synapse_worker_details.port }}:{{ matrix_synapse_worker_details.port }} \
|
||||
{% endif %}
|
||||
{% if matrix_synapse_worker_details.metrics_port != 0 %}
|
||||
-p {{ '' if matrix_synapse_workers_container_host_bind_address == '*' else (matrix_synapse_workers_container_host_bind_address + ':') }}{{ matrix_synapse_worker_details.metrics_port }}:{{ matrix_synapse_worker_details.metrics_port }} \
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
--mount type=bind,src={{ matrix_synapse_config_dir_path }},dst=/data,ro \
|
||||
--mount type=bind,src={{ matrix_synapse_storage_path }},dst=/matrix-media-store-parent,bind-propagation=slave \
|
||||
{% for volume in matrix_synapse_container_additional_volumes %}
|
||||
-v {{ volume.src }}:{{ volume.dst }}:{{ volume.options }} \
|
||||
{% endfor %}
|
||||
{% for arg in matrix_synapse_container_extra_arguments %}
|
||||
{{ arg }} \
|
||||
{% endfor %}
|
||||
{{ matrix_synapse_docker_image }} \
|
||||
-m synapse.app.{{ matrix_synapse_worker_details.type }} -c /data/homeserver.yaml -c /data/{{ matrix_synapse_worker_config_file_name }}
|
||||
|
||||
|
||||
ExecStop=-{{ matrix_host_command_docker }} kill {{ matrix_synapse_worker_container_name }}
|
||||
ExecStop=-{{ matrix_host_command_docker }} rm {{ matrix_synapse_worker_container_name }}
|
||||
|
||||
ExecReload={{ matrix_host_command_docker }} exec {{ matrix_synapse_worker_container_name }} /bin/sh -c 'kill -HUP 1'
|
||||
Restart=always
|
||||
RestartSec=30
|
||||
SyslogIdentifier={{ matrix_synapse_worker_container_name }}
|
||||
|
||||
# Intentionally not making this WantedBy=matrix-synapse.service,
|
||||
# as matrix.synapse.service already has `Wants=` lines.
|
||||
# Also, WantedBy will trigger the creation of some `matrix-synapse.service.wants/` directory,
|
||||
# which we'd have to clean, etc. Better not.
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@ -1,39 +0,0 @@
|
||||
#jinja2: lstrip_blocks: "True"
|
||||
# Instantiable worker service, running inside the synapse container
|
||||
# alongside the homeserver main process.
|
||||
# c.f. https://github.com/matrix-org/synapse/pull/4662
|
||||
[Unit]
|
||||
Description=Matrix worker synapse.app.%i
|
||||
AssertPathExists={{ matrix_synapse_config_dir_path }}/worker.%i.yaml
|
||||
After=matrix-synapse.service
|
||||
BindsTo=matrix-synapse.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
|
||||
# Intentional delay, so that the homeserver can manage to start.
|
||||
ExecStartPre={{ matrix_host_command_sleep }} 5
|
||||
|
||||
# no sane way of instancing more than one variable (systemd "cant-fix" 🤦)
|
||||
# c.f. https://github.com/systemd/systemd/issues/14895#issuecomment-594123923
|
||||
# So use good ol' shell parameter expansion to get the worker type..
|
||||
ExecStart=/bin/sh -c "WORKER=%i; WORKER=$${WORKER%%:*}; \
|
||||
exec {{ matrix_host_command_docker }} exec \
|
||||
--user={{ matrix_user_uid }}:{{ matrix_user_gid }} \
|
||||
matrix-synapse \
|
||||
python -m synapse.app.$${WORKER} -c /data/homeserver.yaml -c /data/worker.%i.yaml"
|
||||
|
||||
# wait for worker startup & write out PID of actual worker process so systemd can handle it
|
||||
ExecStartPost={{ matrix_host_command_sleep }} 5
|
||||
ExecStartPost={{ matrix_local_bin_path }}/matrix-synapse-worker-write-pid %i /run/matrix-synapse-worker.%i.pid
|
||||
|
||||
ExecReload=/bin/kill -HUP $MAINPID
|
||||
ExecStop=/bin/kill $MAINPID
|
||||
PIDFile=/run/matrix-synapse-worker.%i.pid
|
||||
KillMode=process
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
SyslogIdentifier=matrix-synapse-%i
|
||||
|
||||
[Install]
|
||||
WantedBy=matrix-synapse.service
|
||||
@ -4,10 +4,18 @@ Description=Synapse server
|
||||
{% for service in matrix_synapse_systemd_required_services_list %}
|
||||
Requires={{ service }}
|
||||
After={{ service }}
|
||||
|
||||
{% endfor %}
|
||||
{% for service in matrix_synapse_systemd_wanted_services_list %}
|
||||
Wants={{ service }}
|
||||
{% endfor %}
|
||||
|
||||
{% if matrix_synapse_workers_enabled %}
|
||||
{% for matrix_synapse_worker_details in matrix_synapse_workers_enabled_list %}
|
||||
Wants=matrix-synapse-worker-{{ matrix_synapse_worker_details.type }}-{{ matrix_synapse_worker_details.port }}.service
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
DefaultDependencies=no
|
||||
|
||||
[Service]
|
||||
@ -45,16 +53,6 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-synapse \
|
||||
{% if matrix_synapse_manhole_enabled and matrix_synapse_container_manhole_api_host_bind_port %}
|
||||
-p {{ matrix_synapse_container_manhole_api_host_bind_port }}:9000 \
|
||||
{% endif %}
|
||||
{% if matrix_synapse_workers_enabled and matrix_synapse_workers_container_host_bind_address %}
|
||||
{% for worker in matrix_synapse_workers_enabled_list %}
|
||||
{% if worker.port != 0 %}
|
||||
-p {{ '' if matrix_synapse_workers_container_host_bind_address == '*' else (matrix_synapse_workers_container_host_bind_address + ':') }}{{ worker.port }}:{{ worker.port }} \
|
||||
{% endif %}
|
||||
{% if worker.metrics_port != 0 %}
|
||||
-p {{ '' if matrix_synapse_workers_container_host_bind_address == '*' else (matrix_synapse_workers_container_host_bind_address + ':') }}{{ worker.metrics_port }}:{{ worker.metrics_port }} \
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
--mount type=bind,src={{ matrix_synapse_config_dir_path }},dst=/data,ro \
|
||||
--mount type=bind,src={{ matrix_synapse_storage_path }},dst=/matrix-media-store-parent,bind-propagation=slave \
|
||||
{% for volume in matrix_synapse_container_additional_volumes %}
|
||||
@ -68,7 +66,7 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-synapse \
|
||||
|
||||
ExecStop=-{{ matrix_host_command_docker }} kill matrix-synapse
|
||||
ExecStop=-{{ matrix_host_command_docker }} rm matrix-synapse
|
||||
ExecReload={{ matrix_host_command_docker }} exec matrix-synapse kill -HUP 1
|
||||
ExecReload={{ matrix_host_command_docker }} exec matrix-synapse /bin/sh -c 'kill -HUP 1'
|
||||
Restart=always
|
||||
RestartSec=30
|
||||
SyslogIdentifier=matrix-synapse
|
||||
|
||||
@ -1,32 +1,32 @@
|
||||
#jinja2: lstrip_blocks: "True"
|
||||
worker_app: synapse.app.{{ item.type }}
|
||||
worker_name: {{ item.type ~ ':' ~ item.port }}
|
||||
worker_app: synapse.app.{{ matrix_synapse_worker_details.type }}
|
||||
worker_name: {{ matrix_synapse_worker_details.type ~ ':' ~ matrix_synapse_worker_details.port }}
|
||||
|
||||
worker_replication_host: 127.0.0.1
|
||||
worker_replication_host: matrix-synapse
|
||||
worker_replication_http_port: {{ matrix_synapse_replication_http_port }}
|
||||
|
||||
worker_listeners:
|
||||
{% if item.type not in [ 'appservice', 'federation_sender', 'pusher' ] %}
|
||||
{% if matrix_synapse_worker_details.type not in [ 'appservice', 'federation_sender', 'pusher' ] %}
|
||||
- type: http
|
||||
port: {{ item.port }}
|
||||
port: {{ matrix_synapse_worker_details.port }}
|
||||
resources:
|
||||
- names:
|
||||
{% if item.type in [ 'generic_worker', 'frontend_proxy', 'user_dir' ] %}
|
||||
{% if matrix_synapse_worker_details.type in [ 'generic_worker', 'frontend_proxy', 'user_dir' ] %}
|
||||
- client
|
||||
{% endif %}
|
||||
{% if item.type in [ 'generic_worker' ] %}
|
||||
{% if matrix_synapse_worker_details.type in [ 'generic_worker' ] %}
|
||||
- federation
|
||||
{% elif item.type in [ 'media_repository' ] %}
|
||||
{% elif matrix_synapse_worker_details.type in [ 'media_repository' ] %}
|
||||
- media
|
||||
{% endif %}
|
||||
|
||||
{% endif %}
|
||||
- type: metrics
|
||||
bind_address: ['127.0.0.1']
|
||||
port: {{ item.metrics_port }}
|
||||
port: {{ matrix_synapse_worker_details.metrics_port }}
|
||||
|
||||
{% if item.type == 'frontend_proxy' %}
|
||||
worker_main_http_uri: http://127.0.0.1:8008
|
||||
{% if matrix_synapse_worker_details.type == 'frontend_proxy' %}
|
||||
worker_main_http_uri: http://matrix-synapse:8008
|
||||
{% endif %}
|
||||
|
||||
worker_daemonize: false
|
||||
|
||||
Reference in New Issue
Block a user