Merge branch 'synapse-workers' into feature/add-worker-support
This commit is contained in:
Max Klenk
commit
567d0318b0
@ -101,6 +101,114 @@
|
||||
}
|
||||
{% endif %}
|
||||
|
||||
{% if synchrotron_workers %}
|
||||
{# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L134 #}
|
||||
location /_matrix/client/r0/sync {
|
||||
proxy_pass http://synchrotron$request_uri;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $remote_addr;
|
||||
}
|
||||
location /_matrix/client/r0/events {
|
||||
proxy_pass http://synchrotron$request_uri;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $remote_addr;
|
||||
}
|
||||
location /_matrix/client/r0/initialSync {
|
||||
proxy_pass http://synchrotron$request_uri;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $remote_addr;
|
||||
}
|
||||
location ~ ^/_matrix/client/r0/rooms/[^/]+/initialSync$ {
|
||||
proxy_pass http://synchrotron$request_uri;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $remote_addr;
|
||||
}
|
||||
{% endif %}
|
||||
|
||||
{% set client_reader_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'client_reader')|first %}
|
||||
{% if client_reader_worker %}
|
||||
{# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L252 #}
|
||||
location ^/_matrix/client/(versions$|(api/v1|r0|unstable)/(publicRooms$|rooms/.*/joined_me|rooms/.*/context/.|rooms/.*/members$|rooms/.*/messages$|rooms/.*/state$|login$|account/3pid$|keys/query$|keys/changes$|voip/turnServer$|joined_groups$|publicised_groups$|publicised_groups/|pushrules/.*$|groups/.*$|register$|auth/.*/fallback/web$)) {
|
||||
{# Use the embedded DNS resolver in Docker containers to discover the service #}
|
||||
resolver 127.0.0.11 valid=5s;
|
||||
set $backend "matrix-synapse:{{ client_reader_worker.port }}";
|
||||
proxy_pass http://$backend$request_uri;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $remote_addr;
|
||||
}
|
||||
{% endif %}
|
||||
|
||||
{% set media_repository_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'media_repository')|first %}
|
||||
{% if media_repository_worker %}
|
||||
{# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L219 #}
|
||||
location /_matrix/media/ {
|
||||
{# Use the embedded DNS resolver in Docker containers to discover the service #}
|
||||
resolver 127.0.0.11 valid=5s;
|
||||
set $backend "matrix-synapse:{{ media_repository_worker.port }}";
|
||||
proxy_pass http://$backend$request_uri;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $remote_addr;
|
||||
}
|
||||
{# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L223 #}
|
||||
location ~ ^/_synapse/admin/v1/(purge_media_cache|room/.*/media.*|user/.*/media.*|media/.*|quarantine_media/.*)$ {
|
||||
{# Use the embedded DNS resolver in Docker containers to discover the service #}
|
||||
resolver 127.0.0.11 valid=5s;
|
||||
set $backend "matrix-synapse:{{ media_repository_worker.port }}";
|
||||
proxy_pass http://$backend$request_uri;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $remote_addr;
|
||||
}
|
||||
{% endif %}
|
||||
|
||||
{% set event_creator_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'event_creator')|first %}
|
||||
{% if event_creator_worker %}
|
||||
{# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L323 #}
|
||||
location ~ ^/_matrix/client/(api/v1|r0|unstable)/(rooms/.*/send|rooms/.*/state/|rooms/.*/(join|invite|leave|ban|unban|kick)$|join/|profile/) {
|
||||
{# Use the embedded DNS resolver in Docker containers to discover the service #}
|
||||
resolver 127.0.0.11 valid=5s;
|
||||
set $backend "matrix-synapse:{{ event_creator_worker.port }}";
|
||||
proxy_pass http://$backend$request_uri;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $remote_addr;
|
||||
}
|
||||
{% endif %}
|
||||
|
||||
{% set frontend_proxy_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'frontend_proxy')|first %}
|
||||
{% if frontend_proxy_worker %}
|
||||
{# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L302 #}
|
||||
location ~ ^/_matrix/client/(api/v1|r0|unstable)/keys/upload {
|
||||
{# Use the embedded DNS resolver in Docker containers to discover the service #}
|
||||
resolver 127.0.0.11 valid=5s;
|
||||
set $backend "matrix-synapse:{{ frontend_proxy_worker.port }}";
|
||||
proxy_pass http://$backend$request_uri;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $remote_addr;
|
||||
}
|
||||
{% if not matrix_synapse_use_presence %}
|
||||
location ~ ^/_matrix/client/(api/v1|r0|unstable)/presence/[^/]+/status {
|
||||
{# Use the embedded DNS resolver in Docker containers to discover the service #}
|
||||
resolver 127.0.0.11 valid=5s;
|
||||
set $backend "matrix-synapse:{{ frontend_proxy_worker.port }}";
|
||||
proxy_pass http://$backend$request_uri;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $remote_addr;
|
||||
}
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
|
||||
{% set user_dir_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'user_dir')|first %}
|
||||
{% if user_dir_worker %}
|
||||
{# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L290 #}
|
||||
location ~ ^/_matrix/client/(api/v1|r0|unstable)/user_directory/search$ {
|
||||
{# Use the embedded DNS resolver in Docker containers to discover the service #}
|
||||
resolver 127.0.0.11 valid=5s;
|
||||
set $backend "matrix-synapse:{{ user_dir_worker.port }}";
|
||||
proxy_pass http://$backend$request_uri;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $remote_addr;
|
||||
}
|
||||
{% endif %}
|
||||
|
||||
{% for configuration_block in matrix_nginx_proxy_proxy_matrix_additional_server_configuration_blocks %}
|
||||
{{- configuration_block }}
|
||||
{% endfor %}
|
||||
@ -174,6 +282,19 @@
|
||||
}
|
||||
{% endmacro %}
|
||||
|
||||
{% set synchrotron_workers = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'synchrotron')|list %}
|
||||
{% if synchrotron_workers %}
|
||||
upstream synchrotron {
|
||||
# ensures that requests from the same client will always be passed
|
||||
# to the same server (except when this server is unavailable)
|
||||
ip_hash;
|
||||
|
||||
{% for synchrotron_worker in synchrotron_workers %}
|
||||
server "matrix-synapse:{{ synchrotron_worker.port }}";
|
||||
{% endfor %}
|
||||
}
|
||||
{% endif %}
|
||||
|
||||
server {
|
||||
listen {{ 8080 if matrix_nginx_proxy_enabled else 80 }};
|
||||
server_name {{ matrix_nginx_proxy_proxy_matrix_hostname }};
|
||||
@ -255,6 +376,19 @@ server {
|
||||
ssl_ciphers "EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH";
|
||||
{% endif %}
|
||||
|
||||
{% set federation_reader_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'federation_reader')|first %}
|
||||
{% if federation_reader_worker %}
|
||||
{# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L160 #}
|
||||
location ~ ^(/_matrix/federation/v1/event/|/_matrix/federation/v1/state/|/_matrix/federation/v1/state_ids/|/_matrix/federation/v1/backfill/|/_matrix/federation/v1/get_missing_events/|/_matrix/federation/v1/publicRooms|/_matrix/federation/v1/query/|/_matrix/federation/v1/make_join/|/_matrix/federation/v1/make_leave/|/_matrix/federation/v1/send_join/|/_matrix/federation/v2/send_join/|/_matrix/federation/v1/send_leave/|/_matrix/federation/v2/send_leave/|/_matrix/federation/v1/invite/|/_matrix/federation/v2/invite/|/_matrix/federation/v1/query_auth/|/_matrix/federation/v1/event_auth/|/_matrix/federation/v1/exchange_third_party_invite/|/_matrix/federation/v1/user/devices/|/_matrix/federation/v1/send/|/_matrix/federation/v1/get_groups_publicised$|/_matrix/key/v2/query|/_matrix/federation/v1/groups/) {
|
||||
{# Use the embedded DNS resolver in Docker containers to discover the service #}
|
||||
resolver 127.0.0.11 valid=5s;
|
||||
set $backend "matrix-synapse:{{ federation_reader_worker.port }}";
|
||||
proxy_pass http://$backend$request_uri;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $remote_addr;
|
||||
}
|
||||
{% endif %}
|
||||
|
||||
location / {
|
||||
{% if matrix_nginx_proxy_enabled %}
|
||||
{# Use the embedded DNS resolver in Docker containers to discover the service #}
|
||||
|
||||
@ -260,6 +260,43 @@ matrix_synapse_metrics_port: 9100
|
||||
# See https://github.com/matrix-org/synapse/blob/master/docs/manhole.md
|
||||
matrix_synapse_manhole_enabled: false
|
||||
|
||||
# Enable support for Synapse workers
|
||||
matrix_synapse_workers_enabled: false
|
||||
|
||||
# List of workers to spawn
|
||||
matrix_synapse_workers_enabled_list: []
|
||||
|
||||
# Default list of workers to spawn
|
||||
matrix_synapse_workers_enabled_list:
|
||||
- { worker: synchrotron, port: 18082 }
|
||||
- { worker: synchrotron, port: 18083 }
|
||||
- { worker: synchrotron, port: 18084 }
|
||||
- { worker: appservice, port: 18085 }
|
||||
- { worker: client_reader, port: 18086 }
|
||||
- { worker: event_creator, port: 18087 }
|
||||
- { worker: federation_reader, port: 18088 }
|
||||
- { worker: federation_sender, port: 18089 }
|
||||
- { worker: frontend_proxy, port: 18090 }
|
||||
- { worker: media_repository, port: 18091 }
|
||||
- { worker: pusher, port: 18092 }
|
||||
- { worker: user_dir, port: 18093 }
|
||||
|
||||
# The list of available workers (2020-04-14)
|
||||
matrix_synapse_workers_avail_list:
|
||||
- appservice
|
||||
- client_reader
|
||||
- event_creator
|
||||
- federation_reader
|
||||
- federation_sender
|
||||
- frontend_proxy
|
||||
- media_repository
|
||||
- pusher
|
||||
- synchrotron
|
||||
- user_dir
|
||||
|
||||
# Ports used for communication between main synapse process and workers
|
||||
matrix_synapse_replication_tcp_port: 9092
|
||||
matrix_synapse_replication_http_port: 9093
|
||||
|
||||
# Send ERROR logs to sentry.io for easier tracking
|
||||
# To set this up: go to sentry.io, create a python project, and set
|
||||
|
||||
@ -18,6 +18,8 @@
|
||||
|
||||
- import_tasks: "{{ role_path }}/tasks/ext/setup.yml"
|
||||
|
||||
- import_tasks: "{{ role_path }}/tasks/workers/setup.yml"
|
||||
|
||||
- import_tasks: "{{ role_path }}/tasks/synapse/setup.yml"
|
||||
|
||||
- import_tasks: "{{ role_path }}/tasks/goofys/setup.yml"
|
||||
|
||||
8
roles/matrix-synapse/tasks/workers/setup.yml
Normal file
8
roles/matrix-synapse/tasks/workers/setup.yml
Normal file
@ -0,0 +1,8 @@
|
||||
---
|
||||
# a negative when condition will not actually prevent ansible from executing loops in imported tasks!
|
||||
|
||||
- import_tasks: "{{ role_path }}/tasks/workers/setup_install.yml"
|
||||
when: "matrix_synapse_enabled|bool and matrix_synapse_workers_enabled|bool"
|
||||
|
||||
- import_tasks: "{{ role_path }}/tasks/workers/setup_uninstall.yml"
|
||||
when: "not matrix_synapse_workers_enabled|bool"
|
||||
42
roles/matrix-synapse/tasks/workers/setup_install.yml
Normal file
42
roles/matrix-synapse/tasks/workers/setup_install.yml
Normal file
@ -0,0 +1,42 @@
|
||||
---
|
||||
|
||||
- name: Ensure synapse worker base service file installed
|
||||
template:
|
||||
src: "{{ role_path }}/templates/synapse/systemd/matrix-synapse-worker@.service.j2"
|
||||
dest: "{{ matrix_systemd_path }}/matrix-synapse-worker@.service"
|
||||
mode: 0644
|
||||
register: matrix_synapse_worker_systemd_service_result
|
||||
|
||||
- name: Ensure previous worker service symlinks are cleaned (FIXME)
|
||||
file:
|
||||
path: "{{ item.root + '/' + item.path }}"
|
||||
state: absent
|
||||
when:
|
||||
- matrix_synapse_workers_enabled|bool
|
||||
- item.state == 'link'
|
||||
- item.path is match('matrix-synapse-worker@.*\\.service')
|
||||
with_filetree:
|
||||
- "{{ matrix_systemd_path }}/matrix-synapse.service.wants"
|
||||
|
||||
- name: Ensure systemd reloaded the worker service unit
|
||||
service:
|
||||
daemon_reload: yes
|
||||
|
||||
- name: Ensure individual worker service symlinks exist
|
||||
service:
|
||||
name: "matrix-synapse-worker@{{ item.worker }}:{{ item.port }}.service"
|
||||
enabled: true
|
||||
with_items: "{{ matrix_synapse_workers_enabled_list }}"
|
||||
|
||||
- name: Ensure creation of specific worker configs
|
||||
template:
|
||||
src: "{{ role_path }}/templates/synapse/worker.yaml.j2"
|
||||
dest: "{{ matrix_synapse_config_dir_path }}/worker.{{ item.worker }}:{{ item.port }}.yaml"
|
||||
with_list: "{{ matrix_synapse_workers_enabled_list }}"
|
||||
|
||||
- name: Add workers to synapse.wants list
|
||||
set_fact:
|
||||
matrix_synapse_systemd_wanted_services_list: >
|
||||
{{ matrix_synapse_systemd_wanted_services_list +
|
||||
['matrix-synapse-worker@' + item.worker + ':' + item.port|string + '.service'] }}
|
||||
with_items: "{{ matrix_synapse_workers_enabled_list }}"
|
||||
38
roles/matrix-synapse/tasks/workers/setup_uninstall.yml
Normal file
38
roles/matrix-synapse/tasks/workers/setup_uninstall.yml
Normal file
@ -0,0 +1,38 @@
|
||||
---
|
||||
|
||||
- name: Populate service facts
|
||||
service_facts:
|
||||
|
||||
- name: Ensure any worker services are stopped
|
||||
service:
|
||||
name: "{{ item.key }}"
|
||||
state: stopped
|
||||
with_dict: "{{ ansible_facts.services|default({})|dict2items|selectattr('key', 'match', 'matrix-synapse-worker@.+\\.service')|list|items2dict }}"
|
||||
|
||||
# As we cannot know the ports of workers removed from the enabled_list..
|
||||
# => .. just kill them all (FIXME?)
|
||||
- name: Ensure previous worker service symlinks are cleaned
|
||||
file:
|
||||
path: "{{ item.root + '/' + item.path }}"
|
||||
state: absent
|
||||
when:
|
||||
- not matrix_synapse_workers_enabled|bool
|
||||
- item.state == 'link'
|
||||
- item.path is match('matrix-synapse-worker@.*\\.service')
|
||||
with_filetree:
|
||||
- "{{ matrix_systemd_path }}/matrix-synapse.service.wants"
|
||||
|
||||
- name: Ensure synapse worker base service file gets removed
|
||||
file:
|
||||
path: "{{ matrix_systemd_path }}/matrix-synapse-worker@.service"
|
||||
state: absent
|
||||
register: matrix_synapse_worker_systemd_service_result
|
||||
|
||||
- name: Remove workers from synapse.wants list
|
||||
set_fact:
|
||||
matrix_synapse_systemd_wanted_services_list: "{{ matrix_synapse_systemd_wanted_services_list | reject('search', item) | list }}"
|
||||
with_items: "{{ matrix_synapse_workers_avail_list }}"
|
||||
|
||||
- name: Ensure systemd noticed removal of worker service units
|
||||
service:
|
||||
daemon_reload: yes
|
||||
@ -223,6 +223,44 @@ listeners:
|
||||
type: manhole
|
||||
{% endif %}
|
||||
|
||||
{% if matrix_synapse_workers_enabled %}
|
||||
# c.f. https://github.com/matrix-org/synapse/tree/master/docs/workers.md
|
||||
# TCP replication: streaming data from the master to the workers
|
||||
- port: {{ matrix_synapse_replication_tcp_port }}
|
||||
bind_addresses: ['0.0.0.0']
|
||||
type: replication
|
||||
|
||||
# HTTP replication: for the workers to send data to the main synapse process
|
||||
- port: {{ matrix_synapse_replication_http_port }}
|
||||
bind_addresses: ['0.0.0.0']
|
||||
type: http
|
||||
resources:
|
||||
- names: [replication]
|
||||
|
||||
# c.f. https://github.com/matrix-org/synapse/tree/master/contrib/systemd-with-workers/README.md
|
||||
worker_app: synapse.app.homeserver
|
||||
|
||||
# thx https://oznetnerd.com/2017/04/18/jinja2-selectattr-filter/
|
||||
# reduce the main worker's offerings to core homeserver business
|
||||
{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'appservice')|list %}
|
||||
notify_appservices: false
|
||||
{% endif %}
|
||||
{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'federation_sender')|list %}
|
||||
send_federation: false
|
||||
{% endif %}
|
||||
{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'media_repository')|list %}
|
||||
enable_media_repo: false
|
||||
{% endif %}
|
||||
{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'pusher')|list %}
|
||||
start_pushers: false
|
||||
{% endif %}
|
||||
{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'user_dir')|list %}
|
||||
update_user_directory: false
|
||||
{% endif %}
|
||||
|
||||
# rather let systemd handle the forking
|
||||
daemonize: false
|
||||
{% endif %}
|
||||
|
||||
# Forward extremities can build up in a room due to networking delays between
|
||||
# homeservers. Once this happens in a large room, calculation of the state of
|
||||
|
||||
@ -0,0 +1,29 @@
|
||||
#jinja2: lstrip_blocks: "True"
|
||||
# c.f. https://github.com/matrix-org/synapse/pull/4662
|
||||
[Unit]
|
||||
Description=Synapse Matrix Worker
|
||||
After=matrix-synapse.service
|
||||
BindsTo=matrix-synapse.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
|
||||
# Intentional delay, so that the homeserver (we likely depend on) can manage to start.
|
||||
ExecStartPre=/bin/sleep 5
|
||||
|
||||
# systemd ftw 🤦♂️
|
||||
# https://github.com/systemd/systemd/issues/14895#issuecomment-594123923
|
||||
ExecStart=/bin/sh -c "WORKER=%i; WORKER=$${WORKER%%:*}; \
|
||||
exec /usr/bin/docker exec \
|
||||
--user={{ matrix_user_uid }}:{{ matrix_user_gid }} \
|
||||
matrix-synapse \
|
||||
python -m synapse.app.$${WORKER} -c /data/homeserver.yaml -c /data/worker.%i.yaml"
|
||||
|
||||
ExecStop=/usr/bin/docker exec matrix-synapse pkill -f %i
|
||||
KillMode=process
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
SyslogIdentifier=matrix-synapse-%i
|
||||
|
||||
[Install]
|
||||
WantedBy=matrix-synapse.service
|
||||
@ -43,6 +43,11 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-synapse \
|
||||
{% if matrix_synapse_manhole_enabled and matrix_synapse_container_manhole_api_host_bind_port %}
|
||||
-p {{ matrix_synapse_container_manhole_api_host_bind_port }}:9000 \
|
||||
{% endif %}
|
||||
{% if matrix_synapse_workers_enabled %}
|
||||
{% for worker in matrix_synapse_workers_enabled_list %}
|
||||
-p {{ worker.port }}:{{ worker.port }} \
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
-v {{ matrix_synapse_config_dir_path }}:/data:ro \
|
||||
-v {{ matrix_synapse_storage_path }}:/matrix-media-store-parent:slave \
|
||||
{% for volume in matrix_synapse_container_additional_volumes %}
|
||||
|
||||
29
roles/matrix-synapse/templates/synapse/worker.yaml.j2
Normal file
29
roles/matrix-synapse/templates/synapse/worker.yaml.j2
Normal file
@ -0,0 +1,29 @@
|
||||
#jinja2: lstrip_blocks: "True"
|
||||
worker_app: synapse.app.{{ item.worker }}
|
||||
|
||||
worker_replication_host: 127.0.0.1
|
||||
worker_replication_port: {{ matrix_synapse_replication_tcp_port }}
|
||||
worker_replication_http_port: {{ matrix_synapse_replication_http_port }}
|
||||
|
||||
{% if item.worker not in [ 'appservice', 'federation_sender', 'pusher' ] %}
|
||||
worker_listeners:
|
||||
- type: http
|
||||
port: {{ item.port }}
|
||||
resources:
|
||||
- names:
|
||||
{% if item.worker in [ 'synchrotron', 'client_reader', 'event_creator', 'frontend_proxy', 'user_dir' ] %}
|
||||
- client
|
||||
{% elif item.worker in [ 'federation_reader' ] %}
|
||||
- federation
|
||||
{% elif item.worker in [ 'media_repository' ] %}
|
||||
- media
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
|
||||
{% if item.worker == 'frontend_proxy' %}
|
||||
worker_main_http_uri: http://127.0.0.1:8008
|
||||
{% endif %}
|
||||
|
||||
worker_daemonize: false
|
||||
worker_pid_file: /matrix-run/{{ item.worker }}.port{{ item.port }}.pid
|
||||
worker_log_config: /data/{{ matrix_server_fqn_matrix }}.log.config
|
||||
Loading…
Reference in New Issue
Block a user