From 353bc7c3624f6b7186d65757e7b66bc8e6e56e2c Mon Sep 17 00:00:00 2001 From: Marcel Partap Date: Fri, 10 Apr 2020 23:44:53 +0200 Subject: [PATCH 1/8] Add initial support for synapse workers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit · needs documentation; no checks yet for port clashes or typos in worker name · according to https://github.com/matrix-org/synapse/wiki/Workers-setup-with-nginx#results about 90% of requests go to the synchrotron endpoint · thus, the synchrotron worker is especially suited to be load-balanced · most of the other workers are documented to support only a single instance · https://github.com/matrix-org/synapse/blob/master/docs/workers.md --- roles/matrix-synapse/defaults/main.yml | 37 +++++++++++++++++ roles/matrix-synapse/tasks/setup_synapse.yml | 2 + roles/matrix-synapse/tasks/workers/setup.yml | 7 ++++ .../tasks/workers/setup_install.yml | 41 +++++++++++++++++++ .../tasks/workers/setup_uninstall.yml | 34 +++++++++++++++ .../templates/synapse/homeserver.yaml.j2 | 38 +++++++++++++++++ .../systemd/matrix-synapse-worker@.service.j2 | 29 +++++++++++++ .../templates/synapse/worker.yaml.j2 | 29 +++++++++++++ 8 files changed, 217 insertions(+) create mode 100644 roles/matrix-synapse/tasks/workers/setup.yml create mode 100644 roles/matrix-synapse/tasks/workers/setup_install.yml create mode 100644 roles/matrix-synapse/tasks/workers/setup_uninstall.yml create mode 100644 roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker@.service.j2 create mode 100644 roles/matrix-synapse/templates/synapse/worker.yaml.j2 diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index f117fe93d..dcd42ab83 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -258,6 +258,43 @@ matrix_synapse_metrics_port: 9100 # See https://github.com/matrix-org/synapse/blob/master/docs/manhole.md matrix_synapse_manhole_enabled: false +# Enable support for Synapse workers +matrix_synapse_workers_enabled: false + +# List of workers to spawn +matrix_synapse_workers_enabled_list: [] + +# Default list of workers to spawn +matrix_synapse_workers_enabled_list: + - { worker: synchrotron, port: 8082 } + - { worker: synchrotron, port: 8083 } + - { worker: synchrotron, port: 8084 } + - { worker: appservice, port: 8085 } + - { worker: client_reader, port: 8086 } + - { worker: event_creator, port: 8087 } + - { worker: federation_reader, port: 8088 } + - { worker: federation_sender, port: 8089 } + - { worker: frontend_proxy, port: 8090 } + - { worker: media_repository, port: 8091 } + - { worker: pusher, port: 8092 } + - { worker: user_dir, port: 8093 } + +# The list of available workers (2020-04-14) +matrix_synapse_workers_avail_list: + - appservice + - client_reader + - event_creator + - federation_reader + - federation_sender + - frontend_proxy + - media_repository + - pusher + - synchrotron + - user_dir + +# Ports used for communication between main synapse process and workers +matrix_synapse_replication_tcp_port: 9092 +matrix_synapse_replication_http_port: 9093 # Send ERROR logs to sentry.io for easier tracking # To set this up: go to sentry.io, create a python project, and set diff --git a/roles/matrix-synapse/tasks/setup_synapse.yml b/roles/matrix-synapse/tasks/setup_synapse.yml index c40ae170b..ad1b24e6b 100644 --- a/roles/matrix-synapse/tasks/setup_synapse.yml +++ b/roles/matrix-synapse/tasks/setup_synapse.yml @@ -19,6 +19,8 @@ - import_tasks: "{{ role_path }}/tasks/ext/setup.yml" +- import_tasks: "{{ role_path }}/tasks/workers/setup.yml" + - import_tasks: "{{ role_path }}/tasks/synapse/setup.yml" - import_tasks: "{{ role_path }}/tasks/goofys/setup.yml" diff --git a/roles/matrix-synapse/tasks/workers/setup.yml b/roles/matrix-synapse/tasks/workers/setup.yml new file mode 100644 index 000000000..faf2899e2 --- /dev/null +++ b/roles/matrix-synapse/tasks/workers/setup.yml @@ -0,0 +1,7 @@ +--- + +- import_tasks: "{{ role_path }}/tasks/workers/setup_install.yml" + when: "matrix_synapse_enabled|bool and matrix_synapse_workers_enabled|bool" + +- import_tasks: "{{ role_path }}/tasks/workers/setup_uninstall.yml" + when: "not matrix_synapse_workers_enabled|bool" diff --git a/roles/matrix-synapse/tasks/workers/setup_install.yml b/roles/matrix-synapse/tasks/workers/setup_install.yml new file mode 100644 index 000000000..1f23d0c5a --- /dev/null +++ b/roles/matrix-synapse/tasks/workers/setup_install.yml @@ -0,0 +1,41 @@ +--- + +- name: Ensure synapse worker base service file installed + template: + src: "{{ role_path }}/templates/synapse/systemd/matrix-synapse-worker@.service.j2" + dest: "{{ matrix_systemd_path }}/matrix-synapse-worker@.service" + mode: 0644 + register: matrix_synapse_worker_systemd_service_result + +- name: Ensure previous worker service symlinks are cleaned (FIXME) + file: + path: "{{ item.root + '/' + item.path }}" + state: absent + when: + - item.state == 'link' + - item.path is match('matrix-synapse-worker@*.service') + with_filetree: + - "{{ matrix_systemd_path }}/matrix-synapse.service.wants" + +- name: Ensure systemd reloaded the worker service unit + service: + daemon_reload: yes + +- name: Ensure individual worker service symlinks exist + service: + name: "matrix-synapse-worker@{{ item.worker }}:{{ item.port }}.service" + enabled: true + with_items: "{{ matrix_synapse_workers_enabled_list }}" + +- name: Ensure creation of specific worker configs + template: + src: "{{ role_path }}/templates/synapse/worker.yaml.j2" + dest: "{{ matrix_synapse_config_dir_path }}/worker.{{ item.worker }}:{{ item.port }}.yaml" + with_list: "{{ matrix_synapse_workers_enabled_list }}" + +- name: Add workers to synapse.wants list + set_fact: + matrix_synapse_systemd_wanted_services_list: > + {{ matrix_synapse_systemd_wanted_services_list + + ['matrix-synapse-worker@' + item.worker + ':' + item.port|string + '.service'] }} + with_items: "{{ matrix_synapse_workers_enabled_list }}" diff --git a/roles/matrix-synapse/tasks/workers/setup_uninstall.yml b/roles/matrix-synapse/tasks/workers/setup_uninstall.yml new file mode 100644 index 000000000..864308794 --- /dev/null +++ b/roles/matrix-synapse/tasks/workers/setup_uninstall.yml @@ -0,0 +1,34 @@ +--- + +- name: Ensure individual worker services are stopped + service: + name: "matrix-synapse-worker@{{ item.worker }}:{{ item.port }}.service" + state: stopped + with_items: "{{ matrix_synapse_workers_enabled_list }}" + +# As we cannot know the ports of workers removed from the enabled_list.. +# => .. just kill them all (FIXME?) +- name: Ensure previous worker service symlinks are cleaned + file: + path: "{{ item.root + '/' + item.path }}" + state: absent + when: + - item.state == 'link' + - item.path is match('matrix-synapse-worker@*.service') + with_filetree: + - "{{ matrix_systemd_path }}/matrix-synapse.service.wants" + +- name: Ensure synapse worker base service file gets removed + file: + path: "{{ matrix_systemd_path }}/matrix-synapse-worker@.service" + state: absent + register: matrix_synapse_worker_systemd_service_result + +- name: Remove workers from synapse.wants list + set_fact: + matrix_synapse_systemd_wanted_services_list: "{{ matrix_synapse_systemd_wanted_services_list | reject('search', item) | list }}" + with_items: "{{ matrix_synapse_workers_avail_list }}" + +- name: Ensure systemd noticed removal of worker service units + service: + daemon_reload: yes diff --git a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 index d41313e39..fde097f0c 100644 --- a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 +++ b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 @@ -251,6 +251,44 @@ listeners: type: manhole {% endif %} +{% if matrix_synapse_workers_enabled %} + # c.f. https://github.com/matrix-org/synapse/tree/master/docs/workers.md + # TCP replication: streaming data from the master to the workers + - port: {{ matrix_synapse_replication_tcp_port }} + bind_addresses: ['0.0.0.0'] + type: replication + + # HTTP replication: for the workers to send data to the main synapse process + - port: {{ matrix_synapse_replication_http_port }} + bind_addresses: ['0.0.0.0'] + type: http + resources: + - names: [replication] + +# c.f. https://github.com/matrix-org/synapse/tree/master/contrib/systemd-with-workers/README.md +worker_app: synapse.app.homeserver + +# thx https://oznetnerd.com/2017/04/18/jinja2-selectattr-filter/ +# reduce the main worker's offerings to core homeserver business +{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'appservice')|list %} +notify_appservices: false +{% endif %} +{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'federation_sender')|list %} +send_federation: false +{% endif %} +{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'media_repository')|list %} +enable_media_repo: false +{% endif %} +{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'pusher')|list %} +start_pushers: false +{% endif %} +{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'user_dir')|list %} +update_user_directory: false +{% endif %} + +# rather let systemd handle the forking +daemonize: false +{% endif %} ## Homeserver blocking ## diff --git a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker@.service.j2 b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker@.service.j2 new file mode 100644 index 000000000..a46517c4e --- /dev/null +++ b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker@.service.j2 @@ -0,0 +1,29 @@ +#jinja2: lstrip_blocks: "True" +# c.f. https://github.com/matrix-org/synapse/pull/4662 +[Unit] +Description=Synapse Matrix Worker +After=matrix-synapse.service +BindsTo=matrix-synapse.service + +[Service] +Type=simple + +# Intentional delay, so that the homeserver (we likely depend on) can manage to start. +ExecStartPre=/bin/sleep 5 + +# systemd ftw 🤦‍♂️ +# https://github.com/systemd/systemd/issues/14895#issuecomment-594123923 +ExecStart=/bin/sh -c "WORKER=%i; WORKER=$${WORKER%%:*}; \ + exec /usr/bin/docker exec \ + --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ + matrix-synapse \ + python -m synapse.app.$${WORKER} -c /data/homeserver.yaml -c /data/worker.%i.yaml" + +ExecStop=/usr/bin/docker exec matrix-synapse pkill -f %i +KillMode=process +Restart=always +RestartSec=10 +SyslogIdentifier=matrix-synapse-%i + +[Install] +WantedBy=matrix-synapse.service diff --git a/roles/matrix-synapse/templates/synapse/worker.yaml.j2 b/roles/matrix-synapse/templates/synapse/worker.yaml.j2 new file mode 100644 index 000000000..37a5f87b2 --- /dev/null +++ b/roles/matrix-synapse/templates/synapse/worker.yaml.j2 @@ -0,0 +1,29 @@ +#jinja2: lstrip_blocks: "True" +worker_app: synapse.app.{{ item.worker }} + +worker_replication_host: 127.0.0.1 +worker_replication_port: {{ matrix_synapse_replication_tcp_port }} +worker_replication_http_port: {{ matrix_synapse_replication_http_port }} + +{% if item.worker not in [ 'appservice', 'federation_sender', 'pusher' ] %} +worker_listeners: + - type: http + port: {{ item.port }} + resources: + - names: +{% if item.worker in [ 'synchrotron', 'client_reader', 'event_creator', 'frontend_proxy', 'user_dir' ] %} + - client +{% elif item.worker in [ 'federation_reader' ] %} + - federation +{% elif item.worker in [ 'media_repository' ] %} + - media +{% endif %} +{% endif %} + +{% if item.worker == 'frontend_proxy' %} +worker_main_http_uri: http://127.0.0.1:8008 +{% endif %} + +worker_daemonize: false +worker_pid_file: /matrix-run/{{ item.worker }}.port{{ item.port }}.pid +worker_log_config: /data/{{ matrix_server_fqn_matrix }}.log.config From a14b9c09adee37282b2deb7d0ff63594b5174107 Mon Sep 17 00:00:00 2001 From: Marcel Partap Date: Sat, 11 Apr 2020 03:08:59 +0200 Subject: [PATCH 2/8] Add to synapse nginx template conditional URL rewrites for workers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit · 😅 How to keep this in sync with the matrix-synapse documentation? · regex location matching is expensive · nginx syntax limit: one location only per block / statement · thus, lots of duplicate statements in this file --- .../nginx/conf.d/matrix-synapse.conf.j2 | 110 ++++++++++++++++++ 1 file changed, 110 insertions(+) diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 index 996625156..0f4982cc5 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 @@ -101,6 +101,93 @@ } {% endif %} + {% if synchrotron_workers %} + {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L134 #} + location /_matrix/client/r0/sync { + proxy_pass http://synchrotron$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + location /_matrix/client/r0/events { + proxy_pass http://synchrotron$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + location /_matrix/client/r0/initialSync { + proxy_pass http://synchrotron$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + location ~ ^/_matrix/client/r0/rooms/[^/]+/initialSync$ { + proxy_pass http://synchrotron$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {% endif %} + + {% set client_reader_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'client_reader')|first %} + {% if client_reader_worker %} + {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L252 #} + location ^/_matrix/client/(versions$|(api/v1|r0|unstable)/(publicRooms$|rooms/.*/joined_me|rooms/.*/context/.|rooms/.*/members$|rooms/.*/messages$|rooms/.*/state$|login$|account/3pid$|keys/query$|keys/changes$|voip/turnServer$|joined_groups$|publicised_groups$|publicised_groups/|pushrules/.*$|groups/.*$|register$|auth/.*/fallback/web$)) { + proxy_pass http://127.0.0.1:{{ client_reader_worker.port }}$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {% endif %} + + {% set media_repository_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'media_repository')|first %} + {% if media_repository_worker %} + {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L219 #} + location /_matrix/media/ { + proxy_pass http://127.0.0.1:{{ media_repository_worker.port }}$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L223 #} + location ~ ^/_synapse/admin/v1/(purge_media_cache|room/.*/media.*|user/.*/media.*|media/.*|quarantine_media/.*)$ { + proxy_pass http://127.0.0.1:{{ media_repository_worker.port }}$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {% endif %} + + {% set event_creator_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'event_creator')|first %} + {% if event_creator_worker %} + {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L323 #} + location ~ ^/_matrix/client/(api/v1|r0|unstable)/(rooms/.*/send|rooms/.*/state/|rooms/.*/(join|invite|leave|ban|unban|kick)$|join/|profile/) { + proxy_pass http://127.0.0.1:{{ event_creator_worker.port }}$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {% endif %} + + {% set frontend_proxy_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'frontend_proxy')|first %} + {% if frontend_proxy_worker %} + {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L302 #} + location ~ ^/_matrix/client/(api/v1|r0|unstable)/keys/upload { + proxy_pass http://127.0.0.1:{{ frontend_proxy_worker.port }}$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {% if not matrix_synapse_use_presence %} + location ~ ^/_matrix/client/(api/v1|r0|unstable)/presence/[^/]+/status { + proxy_pass http://127.0.0.1:{{ frontend_proxy_worker.port }}$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {% endif %} + {% endif %} + + {% set user_dir_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'user_dir')|first %} + {% if user_dir_worker %} + {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L290 #} + location ~ ^/_matrix/client/(api/v1|r0|unstable)/user_directory/search$ { + proxy_pass http://127.0.0.1:{{ user_dir_worker.port }}$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {% endif %} + {% for configuration_block in matrix_nginx_proxy_proxy_matrix_additional_server_configuration_blocks %} {{- configuration_block }} {% endfor %} @@ -174,6 +261,19 @@ } {% endmacro %} +{% set synchrotron_workers = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'synchrotron')|list %} +{% if synchrotron_workers %} +upstream synchrotron { + # ensures that requests from the same client will always be passed + # to the same server (except when this server is unavailable) + ip_hash; + + {% for worker in synchrotron_workers %} + server 127.0.0.1:{{ worker.port }}; + {% endfor %} +} +{% endif %} + server { listen {{ 8080 if matrix_nginx_proxy_enabled else 80 }}; server_name {{ matrix_nginx_proxy_proxy_matrix_hostname }}; @@ -255,6 +355,16 @@ server { ssl_ciphers "EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH"; {% endif %} + {% set federation_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'federation_reader')|first %} + {% if federation_worker %} + {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L160 #} + location ~ ^(/_matrix/federation/v1/event/|/_matrix/federation/v1/state/|/_matrix/federation/v1/state_ids/|/_matrix/federation/v1/backfill/|/_matrix/federation/v1/get_missing_events/|/_matrix/federation/v1/publicRooms|/_matrix/federation/v1/query/|/_matrix/federation/v1/make_join/|/_matrix/federation/v1/make_leave/|/_matrix/federation/v1/send_join/|/_matrix/federation/v2/send_join/|/_matrix/federation/v1/send_leave/|/_matrix/federation/v2/send_leave/|/_matrix/federation/v1/invite/|/_matrix/federation/v2/invite/|/_matrix/federation/v1/query_auth/|/_matrix/federation/v1/event_auth/|/_matrix/federation/v1/exchange_third_party_invite/|/_matrix/federation/v1/user/devices/|/_matrix/federation/v1/send/|/_matrix/federation/v1/get_groups_publicised$|/_matrix/key/v2/query|/_matrix/federation/v1/groups/) { + proxy_pass http://127.0.0.1:{{ federation_worker.port }}$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {% endif %} + location / { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} From cf452fdf0a1ece84ae4c54e6a73a8618114c5132 Mon Sep 17 00:00:00 2001 From: Marcel Partap Date: Sun, 19 Apr 2020 10:16:01 +0200 Subject: [PATCH 3/8] Fix corner-cases found through testing (aka ansible is nuts) --- roles/matrix-synapse/tasks/workers/setup.yml | 1 + roles/matrix-synapse/tasks/workers/setup_install.yml | 3 ++- .../matrix-synapse/tasks/workers/setup_uninstall.yml | 12 ++++++++---- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/roles/matrix-synapse/tasks/workers/setup.yml b/roles/matrix-synapse/tasks/workers/setup.yml index faf2899e2..4951ac2d7 100644 --- a/roles/matrix-synapse/tasks/workers/setup.yml +++ b/roles/matrix-synapse/tasks/workers/setup.yml @@ -1,4 +1,5 @@ --- +# a negative when condition will not actually prevent ansible from executing loops in imported tasks! - import_tasks: "{{ role_path }}/tasks/workers/setup_install.yml" when: "matrix_synapse_enabled|bool and matrix_synapse_workers_enabled|bool" diff --git a/roles/matrix-synapse/tasks/workers/setup_install.yml b/roles/matrix-synapse/tasks/workers/setup_install.yml index 1f23d0c5a..0031c236c 100644 --- a/roles/matrix-synapse/tasks/workers/setup_install.yml +++ b/roles/matrix-synapse/tasks/workers/setup_install.yml @@ -12,8 +12,9 @@ path: "{{ item.root + '/' + item.path }}" state: absent when: + - matrix_synapse_workers_enabled|bool - item.state == 'link' - - item.path is match('matrix-synapse-worker@*.service') + - item.path is match('matrix-synapse-worker@.*\\.service') with_filetree: - "{{ matrix_systemd_path }}/matrix-synapse.service.wants" diff --git a/roles/matrix-synapse/tasks/workers/setup_uninstall.yml b/roles/matrix-synapse/tasks/workers/setup_uninstall.yml index 864308794..d1e7e3b56 100644 --- a/roles/matrix-synapse/tasks/workers/setup_uninstall.yml +++ b/roles/matrix-synapse/tasks/workers/setup_uninstall.yml @@ -1,10 +1,13 @@ --- -- name: Ensure individual worker services are stopped +- name: Populate service facts + service_facts: + +- name: Ensure any worker services are stopped service: - name: "matrix-synapse-worker@{{ item.worker }}:{{ item.port }}.service" + name: "{{ item.key }}" state: stopped - with_items: "{{ matrix_synapse_workers_enabled_list }}" + with_dict: "{{ ansible_facts.services|default({})|dict2items|selectattr('key', 'match', 'matrix-synapse-worker@.+\\.service')|list|items2dict }}" # As we cannot know the ports of workers removed from the enabled_list.. # => .. just kill them all (FIXME?) @@ -13,8 +16,9 @@ path: "{{ item.root + '/' + item.path }}" state: absent when: + - not matrix_synapse_workers_enabled|bool - item.state == 'link' - - item.path is match('matrix-synapse-worker@*.service') + - item.path is match('matrix-synapse-worker@.*\\.service') with_filetree: - "{{ matrix_systemd_path }}/matrix-synapse.service.wants" From 66a40735126c361e396dde47e9f96c9fe9664ee4 Mon Sep 17 00:00:00 2001 From: Marcel Partap Date: Sun, 19 Apr 2020 17:42:51 +0200 Subject: [PATCH 4/8] Publish synapse worker ports, need to be accessible to nginx --- .../templates/synapse/systemd/matrix-synapse.service.j2 | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 index 0bd2c25d5..700e41349 100644 --- a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 +++ b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 @@ -44,6 +44,11 @@ ExecStart=/usr/bin/docker run --rm --name matrix-synapse \ {% if matrix_synapse_manhole_enabled and matrix_synapse_container_manhole_api_host_bind_port %} -p {{ matrix_synapse_container_manhole_api_host_bind_port }}:9000 \ {% endif %} + {% if matrix_synapse_workers_enabled %} + {% for worker in matrix_synapse_workers_enabled_list %} + -p {{ worker.port }}:{{ worker.port }} \ + {% endfor %} + {% endif %} -v {{ matrix_synapse_config_dir_path }}:/data:ro \ -v {{ matrix_synapse_run_path }}:/matrix-run:rw \ -v {{ matrix_synapse_storage_path }}:/matrix-media-store-parent:slave \ From e4763c21bc4a2e3a053629618d7ba7c78e120f92 Mon Sep 17 00:00:00 2001 From: Marcel Partap Date: Sun, 19 Apr 2020 18:41:27 +0200 Subject: [PATCH 5/8] nginx config: route traffic to workers on matrix-synapse FIXME: horrid duplication in template file --- .../nginx/conf.d/matrix-synapse.conf.j2 | 133 +++++++++++------- 1 file changed, 80 insertions(+), 53 deletions(-) diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 index 0f4982cc5..d7a1e6b3a 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 @@ -127,65 +127,86 @@ {% set client_reader_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'client_reader')|first %} {% if client_reader_worker %} - {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L252 #} - location ^/_matrix/client/(versions$|(api/v1|r0|unstable)/(publicRooms$|rooms/.*/joined_me|rooms/.*/context/.|rooms/.*/members$|rooms/.*/messages$|rooms/.*/state$|login$|account/3pid$|keys/query$|keys/changes$|voip/turnServer$|joined_groups$|publicised_groups$|publicised_groups/|pushrules/.*$|groups/.*$|register$|auth/.*/fallback/web$)) { - proxy_pass http://127.0.0.1:{{ client_reader_worker.port }}$request_uri; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; - } + {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L252 #} + location ^/_matrix/client/(versions$|(api/v1|r0|unstable)/(publicRooms$|rooms/.*/joined_me|rooms/.*/context/.|rooms/.*/members$|rooms/.*/messages$|rooms/.*/state$|login$|account/3pid$|keys/query$|keys/changes$|voip/turnServer$|joined_groups$|publicised_groups$|publicised_groups/|pushrules/.*$|groups/.*$|register$|auth/.*/fallback/web$)) { + {# Use the embedded DNS resolver in Docker containers to discover the service #} + resolver 127.0.0.11 valid=5s; + set $backend "matrix-synapse:{{ client_reader_worker.port }}" + proxy_pass http://$backend$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } {% endif %} {% set media_repository_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'media_repository')|first %} {% if media_repository_worker %} - {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L219 #} - location /_matrix/media/ { - proxy_pass http://127.0.0.1:{{ media_repository_worker.port }}$request_uri; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; - } - {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L223 #} - location ~ ^/_synapse/admin/v1/(purge_media_cache|room/.*/media.*|user/.*/media.*|media/.*|quarantine_media/.*)$ { - proxy_pass http://127.0.0.1:{{ media_repository_worker.port }}$request_uri; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; - } + {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L219 #} + location /_matrix/media/ { + {# Use the embedded DNS resolver in Docker containers to discover the service #} + resolver 127.0.0.11 valid=5s; + set $backend "matrix-synapse:{{ media_repository_worker.port }}" + proxy_pass http://$backend$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L223 #} + location ~ ^/_synapse/admin/v1/(purge_media_cache|room/.*/media.*|user/.*/media.*|media/.*|quarantine_media/.*)$ { + {# Use the embedded DNS resolver in Docker containers to discover the service #} + resolver 127.0.0.11 valid=5s; + set $backend "matrix-synapse:{{ media_repository_worker.port }}" + proxy_pass http://$backend$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } {% endif %} {% set event_creator_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'event_creator')|first %} {% if event_creator_worker %} - {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L323 #} - location ~ ^/_matrix/client/(api/v1|r0|unstable)/(rooms/.*/send|rooms/.*/state/|rooms/.*/(join|invite|leave|ban|unban|kick)$|join/|profile/) { - proxy_pass http://127.0.0.1:{{ event_creator_worker.port }}$request_uri; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; - } + {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L323 #} + location ~ ^/_matrix/client/(api/v1|r0|unstable)/(rooms/.*/send|rooms/.*/state/|rooms/.*/(join|invite|leave|ban|unban|kick)$|join/|profile/) { + {# Use the embedded DNS resolver in Docker containers to discover the service #} + resolver 127.0.0.11 valid=5s; + set $backend "matrix-synapse:{{ event_creator_worker.port }}" + proxy_pass http://$backend$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } {% endif %} {% set frontend_proxy_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'frontend_proxy')|first %} {% if frontend_proxy_worker %} - {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L302 #} - location ~ ^/_matrix/client/(api/v1|r0|unstable)/keys/upload { - proxy_pass http://127.0.0.1:{{ frontend_proxy_worker.port }}$request_uri; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; - } - {% if not matrix_synapse_use_presence %} - location ~ ^/_matrix/client/(api/v1|r0|unstable)/presence/[^/]+/status { - proxy_pass http://127.0.0.1:{{ frontend_proxy_worker.port }}$request_uri; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; - } - {% endif %} + {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L302 #} + location ~ ^/_matrix/client/(api/v1|r0|unstable)/keys/upload { + {# Use the embedded DNS resolver in Docker containers to discover the service #} + resolver 127.0.0.11 valid=5s; + set $backend "matrix-synapse:{{ frontend_proxy_worker.port }}" + proxy_pass http://$backend$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {% if not matrix_synapse_use_presence %} + location ~ ^/_matrix/client/(api/v1|r0|unstable)/presence/[^/]+/status { + {# Use the embedded DNS resolver in Docker containers to discover the service #} + resolver 127.0.0.11 valid=5s; + set $backend "matrix-synapse:{{ frontend_proxy_worker.port }}" + proxy_pass http://$backend$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {% endif %} {% endif %} {% set user_dir_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'user_dir')|first %} {% if user_dir_worker %} - {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L290 #} - location ~ ^/_matrix/client/(api/v1|r0|unstable)/user_directory/search$ { - proxy_pass http://127.0.0.1:{{ user_dir_worker.port }}$request_uri; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; - } + {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L290 #} + location ~ ^/_matrix/client/(api/v1|r0|unstable)/user_directory/search$ { + {# Use the embedded DNS resolver in Docker containers to discover the service #} + resolver 127.0.0.11 valid=5s; + set $backend "matrix-synapse:{{ user_dir_worker.port }}" + proxy_pass http://$backend$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } {% endif %} {% for configuration_block in matrix_nginx_proxy_proxy_matrix_additional_server_configuration_blocks %} @@ -268,8 +289,11 @@ upstream synchrotron { # to the same server (except when this server is unavailable) ip_hash; - {% for worker in synchrotron_workers %} - server 127.0.0.1:{{ worker.port }}; + {% for synchrotron_worker in synchrotron_workers %} + {# Use the embedded DNS resolver in Docker containers to discover the service #} + resolver 127.0.0.11 valid=5s; + set $backend "matrix-synapse:{{ synchrotron_worker.port }}" + server $backend:{{ synchrotron_worker.port }}; {% endfor %} } {% endif %} @@ -355,14 +379,17 @@ server { ssl_ciphers "EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH"; {% endif %} - {% set federation_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'federation_reader')|first %} - {% if federation_worker %} - {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L160 #} - location ~ ^(/_matrix/federation/v1/event/|/_matrix/federation/v1/state/|/_matrix/federation/v1/state_ids/|/_matrix/federation/v1/backfill/|/_matrix/federation/v1/get_missing_events/|/_matrix/federation/v1/publicRooms|/_matrix/federation/v1/query/|/_matrix/federation/v1/make_join/|/_matrix/federation/v1/make_leave/|/_matrix/federation/v1/send_join/|/_matrix/federation/v2/send_join/|/_matrix/federation/v1/send_leave/|/_matrix/federation/v2/send_leave/|/_matrix/federation/v1/invite/|/_matrix/federation/v2/invite/|/_matrix/federation/v1/query_auth/|/_matrix/federation/v1/event_auth/|/_matrix/federation/v1/exchange_third_party_invite/|/_matrix/federation/v1/user/devices/|/_matrix/federation/v1/send/|/_matrix/federation/v1/get_groups_publicised$|/_matrix/key/v2/query|/_matrix/federation/v1/groups/) { - proxy_pass http://127.0.0.1:{{ federation_worker.port }}$request_uri; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; - } + {% set federation_reader_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'federation_reader')|first %} + {% if federation_reader_worker %} + {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L160 #} + location ~ ^(/_matrix/federation/v1/event/|/_matrix/federation/v1/state/|/_matrix/federation/v1/state_ids/|/_matrix/federation/v1/backfill/|/_matrix/federation/v1/get_missing_events/|/_matrix/federation/v1/publicRooms|/_matrix/federation/v1/query/|/_matrix/federation/v1/make_join/|/_matrix/federation/v1/make_leave/|/_matrix/federation/v1/send_join/|/_matrix/federation/v2/send_join/|/_matrix/federation/v1/send_leave/|/_matrix/federation/v2/send_leave/|/_matrix/federation/v1/invite/|/_matrix/federation/v2/invite/|/_matrix/federation/v1/query_auth/|/_matrix/federation/v1/event_auth/|/_matrix/federation/v1/exchange_third_party_invite/|/_matrix/federation/v1/user/devices/|/_matrix/federation/v1/send/|/_matrix/federation/v1/get_groups_publicised$|/_matrix/key/v2/query|/_matrix/federation/v1/groups/) { + {# Use the embedded DNS resolver in Docker containers to discover the service #} + resolver 127.0.0.11 valid=5s; + set $backend "matrix-synapse:{{ federation_reader_worker.port }}" + proxy_pass http://$backend$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } {% endif %} location / { From 5f63d287b7dba70830c0218f18a873e02b9bd79c Mon Sep 17 00:00:00 2001 From: Marcel Partap Date: Sun, 19 Apr 2020 18:57:00 +0200 Subject: [PATCH 6/8] Move synapse worker ports up 10k --- roles/matrix-synapse/defaults/main.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index dcd42ab83..9e65d4218 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -266,18 +266,18 @@ matrix_synapse_workers_enabled_list: [] # Default list of workers to spawn matrix_synapse_workers_enabled_list: - - { worker: synchrotron, port: 8082 } - - { worker: synchrotron, port: 8083 } - - { worker: synchrotron, port: 8084 } - - { worker: appservice, port: 8085 } - - { worker: client_reader, port: 8086 } - - { worker: event_creator, port: 8087 } - - { worker: federation_reader, port: 8088 } - - { worker: federation_sender, port: 8089 } - - { worker: frontend_proxy, port: 8090 } - - { worker: media_repository, port: 8091 } - - { worker: pusher, port: 8092 } - - { worker: user_dir, port: 8093 } + - { worker: synchrotron, port: 18082 } + - { worker: synchrotron, port: 18083 } + - { worker: synchrotron, port: 18084 } + - { worker: appservice, port: 18085 } + - { worker: client_reader, port: 18086 } + - { worker: event_creator, port: 18087 } + - { worker: federation_reader, port: 18088 } + - { worker: federation_sender, port: 18089 } + - { worker: frontend_proxy, port: 18090 } + - { worker: media_repository, port: 18091 } + - { worker: pusher, port: 18092 } + - { worker: user_dir, port: 18093 } # The list of available workers (2020-04-14) matrix_synapse_workers_avail_list: From 765c046bebcd071cde1e24d5852f8235cfe17642 Mon Sep 17 00:00:00 2001 From: Christoph Johannes Kleine Date: Sun, 19 Apr 2020 19:50:42 +0200 Subject: [PATCH 7/8] add missing ; to matrix-synapse.conf.j2 --- .../nginx/conf.d/matrix-synapse.conf.j2 | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 index d7a1e6b3a..71605105d 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 @@ -131,7 +131,7 @@ location ^/_matrix/client/(versions$|(api/v1|r0|unstable)/(publicRooms$|rooms/.*/joined_me|rooms/.*/context/.|rooms/.*/members$|rooms/.*/messages$|rooms/.*/state$|login$|account/3pid$|keys/query$|keys/changes$|voip/turnServer$|joined_groups$|publicised_groups$|publicised_groups/|pushrules/.*$|groups/.*$|register$|auth/.*/fallback/web$)) { {# Use the embedded DNS resolver in Docker containers to discover the service #} resolver 127.0.0.11 valid=5s; - set $backend "matrix-synapse:{{ client_reader_worker.port }}" + set $backend "matrix-synapse:{{ client_reader_worker.port }}"; proxy_pass http://$backend$request_uri; proxy_set_header Host $host; proxy_set_header X-Forwarded-For $remote_addr; @@ -144,7 +144,7 @@ location /_matrix/media/ { {# Use the embedded DNS resolver in Docker containers to discover the service #} resolver 127.0.0.11 valid=5s; - set $backend "matrix-synapse:{{ media_repository_worker.port }}" + set $backend "matrix-synapse:{{ media_repository_worker.port }}"; proxy_pass http://$backend$request_uri; proxy_set_header Host $host; proxy_set_header X-Forwarded-For $remote_addr; @@ -153,7 +153,7 @@ location ~ ^/_synapse/admin/v1/(purge_media_cache|room/.*/media.*|user/.*/media.*|media/.*|quarantine_media/.*)$ { {# Use the embedded DNS resolver in Docker containers to discover the service #} resolver 127.0.0.11 valid=5s; - set $backend "matrix-synapse:{{ media_repository_worker.port }}" + set $backend "matrix-synapse:{{ media_repository_worker.port }}"; proxy_pass http://$backend$request_uri; proxy_set_header Host $host; proxy_set_header X-Forwarded-For $remote_addr; @@ -166,7 +166,7 @@ location ~ ^/_matrix/client/(api/v1|r0|unstable)/(rooms/.*/send|rooms/.*/state/|rooms/.*/(join|invite|leave|ban|unban|kick)$|join/|profile/) { {# Use the embedded DNS resolver in Docker containers to discover the service #} resolver 127.0.0.11 valid=5s; - set $backend "matrix-synapse:{{ event_creator_worker.port }}" + set $backend "matrix-synapse:{{ event_creator_worker.port }}"; proxy_pass http://$backend$request_uri; proxy_set_header Host $host; proxy_set_header X-Forwarded-For $remote_addr; @@ -179,7 +179,7 @@ location ~ ^/_matrix/client/(api/v1|r0|unstable)/keys/upload { {# Use the embedded DNS resolver in Docker containers to discover the service #} resolver 127.0.0.11 valid=5s; - set $backend "matrix-synapse:{{ frontend_proxy_worker.port }}" + set $backend "matrix-synapse:{{ frontend_proxy_worker.port }}"; proxy_pass http://$backend$request_uri; proxy_set_header Host $host; proxy_set_header X-Forwarded-For $remote_addr; @@ -188,7 +188,7 @@ location ~ ^/_matrix/client/(api/v1|r0|unstable)/presence/[^/]+/status { {# Use the embedded DNS resolver in Docker containers to discover the service #} resolver 127.0.0.11 valid=5s; - set $backend "matrix-synapse:{{ frontend_proxy_worker.port }}" + set $backend "matrix-synapse:{{ frontend_proxy_worker.port }}"; proxy_pass http://$backend$request_uri; proxy_set_header Host $host; proxy_set_header X-Forwarded-For $remote_addr; @@ -202,7 +202,7 @@ location ~ ^/_matrix/client/(api/v1|r0|unstable)/user_directory/search$ { {# Use the embedded DNS resolver in Docker containers to discover the service #} resolver 127.0.0.11 valid=5s; - set $backend "matrix-synapse:{{ user_dir_worker.port }}" + set $backend "matrix-synapse:{{ user_dir_worker.port }}"; proxy_pass http://$backend$request_uri; proxy_set_header Host $host; proxy_set_header X-Forwarded-For $remote_addr; @@ -292,7 +292,7 @@ upstream synchrotron { {% for synchrotron_worker in synchrotron_workers %} {# Use the embedded DNS resolver in Docker containers to discover the service #} resolver 127.0.0.11 valid=5s; - set $backend "matrix-synapse:{{ synchrotron_worker.port }}" + set $backend "matrix-synapse:{{ synchrotron_worker.port }}"; server $backend:{{ synchrotron_worker.port }}; {% endfor %} } @@ -385,7 +385,7 @@ server { location ~ ^(/_matrix/federation/v1/event/|/_matrix/federation/v1/state/|/_matrix/federation/v1/state_ids/|/_matrix/federation/v1/backfill/|/_matrix/federation/v1/get_missing_events/|/_matrix/federation/v1/publicRooms|/_matrix/federation/v1/query/|/_matrix/federation/v1/make_join/|/_matrix/federation/v1/make_leave/|/_matrix/federation/v1/send_join/|/_matrix/federation/v2/send_join/|/_matrix/federation/v1/send_leave/|/_matrix/federation/v2/send_leave/|/_matrix/federation/v1/invite/|/_matrix/federation/v2/invite/|/_matrix/federation/v1/query_auth/|/_matrix/federation/v1/event_auth/|/_matrix/federation/v1/exchange_third_party_invite/|/_matrix/federation/v1/user/devices/|/_matrix/federation/v1/send/|/_matrix/federation/v1/get_groups_publicised$|/_matrix/key/v2/query|/_matrix/federation/v1/groups/) { {# Use the embedded DNS resolver in Docker containers to discover the service #} resolver 127.0.0.11 valid=5s; - set $backend "matrix-synapse:{{ federation_reader_worker.port }}" + set $backend "matrix-synapse:{{ federation_reader_worker.port }}"; proxy_pass http://$backend$request_uri; proxy_set_header Host $host; proxy_set_header X-Forwarded-For $remote_addr; From 46984a4f99f1b3aece5fbce171feb43bf8e4a726 Mon Sep 17 00:00:00 2001 From: Marcel Partap Date: Sun, 19 Apr 2020 19:55:43 +0200 Subject: [PATCH 8/8] Nginx conf: more testing less b0rk --- .../templates/nginx/conf.d/matrix-synapse.conf.j2 | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 index 71605105d..68edab1da 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 @@ -290,10 +290,7 @@ upstream synchrotron { ip_hash; {% for synchrotron_worker in synchrotron_workers %} - {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; - set $backend "matrix-synapse:{{ synchrotron_worker.port }}"; - server $backend:{{ synchrotron_worker.port }}; + server "matrix-synapse:{{ synchrotron_worker.port }}"; {% endfor %} } {% endif %}