finallycoffee/matrix-docker-ansible-deploy
finallycoffee/matrix-docker-ansible-deploy
5
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

refactor to use new workers and routes they serve

Browse Source
This commit is contained in:
Max Klenk 2020-08-28 13:53:39 +02:00
parent 53ccc783b7
commit 06bc430c7c
No known key found for this signature in database
GPG Key ID: 26EFB2972F02F380
4 changed files with 206 additions and 156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

78
roles/matrix-nginx-proxy/defaults/main.yml
View File

@ -247,3 +247,81 @@ matrix_ssl_log_dir_path: "{{ matrix_ssl_base_path }}/log"
# nginx status page configurations. # nginx status page configurations.
matrix_nginx_proxy_proxy_matrix_nginx_status_enabled: false matrix_nginx_proxy_proxy_matrix_nginx_status_enabled: false
matrix_nginx_proxy_proxy_matrix_nginx_status_allowed_addresses: ['{{ ansible_default_ipv4.address }}'] matrix_nginx_proxy_proxy_matrix_nginx_status_allowed_addresses: ['{{ ansible_default_ipv4.address }}']
# worker
matrix_nginx_proxy_synapse_workers_enabled: "{{ matrix_synapse_workers_enabled }}"
matrix_nginx_proxy_synapse_workers_enabled_list: "{{ matrix_synapse_workers_enabled_list }}"
matrix_nginx_proxy_synapse_generic_worker_locations: [
# Sync requests
'^/_matrix/client/(v2_alpha|r0)/sync$',
'^/_matrix/client/(api/v1|v2_alpha|r0)/events$',
'^/_matrix/client/(api/v1|r0)/initialSync$',
'^/_matrix/client/(api/v1|r0)/rooms/[^/]+/initialSync$',
# Federation requests
'^/_matrix/federation/v1/event/',
'^/_matrix/federation/v1/state/',
'^/_matrix/federation/v1/state_ids/',
'^/_matrix/federation/v1/backfill/',
'^/_matrix/federation/v1/get_missing_events/',
'^/_matrix/federation/v1/publicRooms',
'^/_matrix/federation/v1/query/',
'^/_matrix/federation/v1/make_join/',
'^/_matrix/federation/v1/make_leave/',
'^/_matrix/federation/v1/send_join/',
'^/_matrix/federation/v2/send_join/',
'^/_matrix/federation/v1/send_leave/',
'^/_matrix/federation/v2/send_leave/',
'^/_matrix/federation/v1/invite/',
'^/_matrix/federation/v2/invite/',
'^/_matrix/federation/v1/query_auth/',
'^/_matrix/federation/v1/event_auth/',
'^/_matrix/federation/v1/exchange_third_party_invite/',
'^/_matrix/federation/v1/user/devices/',
'^/_matrix/federation/v1/get_groups_publicised$',
'^/_matrix/key/v2/query',
# Inbound federation transaction request
'^/_matrix/federation/v1/send/',
# Client API requests
'^/_matrix/client/(api/v1|r0|unstable)/publicRooms$',
'^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/joined_members$',
'^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/context/.*$',
'^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/members$',
'^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/state$',
'^/_matrix/client/(api/v1|r0|unstable)/account/3pid$',
'^/_matrix/client/(api/v1|r0|unstable)/keys/query$',
'^/_matrix/client/(api/v1|r0|unstable)/keys/changes$',
'^/_matrix/client/versions$',
'^/_matrix/client/(api/v1|r0|unstable)/voip/turnServer$',
'^/_matrix/client/(api/v1|r0|unstable)/joined_groups$',
'^/_matrix/client/(api/v1|r0|unstable)/publicised_groups$',
'^/_matrix/client/(api/v1|r0|unstable)/publicised_groups/',
# Registration/login requests
'^/_matrix/client/(api/v1|r0|unstable)/login$',
'^/_matrix/client/(r0|unstable)/register$',
'^/_matrix/client/(r0|unstable)/auth/.*/fallback/web$',
# Event sending requests
'^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/send',
'^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/state/',
'^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/(join|invite|leave|ban|unban|kick)$',
'^/_matrix/client/(api/v1|r0|unstable)/join/',
'^/_matrix/client/(api/v1|r0|unstable)/profile/',
]
matrix_nginx_proxy_synapse_media_repository_locations: [
'^/_matrix/media/*$',
'^/_synapse/admin/v1/purge_media_cache$',
'^/_synapse/admin/v1/room/.*/media.*$',
'^/_synapse/admin/v1/user/.*/media.*$',
'^/_synapse/admin/v1/media/.*$',
'^/_synapse/admin/v1/quarantine_media/.*$',
]
matrix_nginx_proxy_synapse_user_dir_locations: [
'matrix_nginx_proxy_synapse_media_workers_endpoints',
]

240
roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2
View File

@ -101,115 +101,60 @@
} }
{% endif %} {% endif %}
{% if matrix_synapse_workers_enabled %} {% if matrix_nginx_proxy_synapse_workers_enabled %}
{% if synchrotron_workers %} {# Synapse Workers #}
{# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L134 #}
location /_matrix/client/r0/sync { {% if generic_worker_workers %}
proxy_pass http://synchrotron$request_uri; {# https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappgeneric_worker #}
proxy_set_header Host $host; {% for location in matrix_nginx_proxy_synapse_generic_worker_locations %}
proxy_set_header X-Forwarded-For $remote_addr; location ~ {{ location }} {
} proxy_pass http://generic_worker_upstream$request_uri;
location /_matrix/client/r0/events { proxy_set_header Host $host;
proxy_pass http://synchrotron$request_uri; proxy_set_header X-Forwarded-For $remote_addr;
proxy_set_header Host $host; }
proxy_set_header X-Forwarded-For $remote_addr; {% endfor %}
} {# ToDo: add GET ^/_matrix/federation/v1/groups/ #}
location /_matrix/client/r0/initialSync { {% endif %}
proxy_pass http://synchrotron$request_uri;
proxy_set_header Host $host; {% if media_repository_workers %}
proxy_set_header X-Forwarded-For $remote_addr; {# https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappmedia_repository #}
} {% for location in matrix_nginx_proxy_synapse_media_repository_locations %}
location ~ ^/_matrix/client/r0/rooms/[^/]+/initialSync$ { location ~ {{ location }} {
proxy_pass http://synchrotron$request_uri; proxy_pass http://media_repository_upstream$request_uri;
proxy_set_header Host $host; proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $remote_addr; proxy_set_header X-Forwarded-For $remote_addr;
} }
{% endfor %}
{% endif %}
{% if user_dir_workers %}
{# https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappuser_dir #}
{% for location in matrix_nginx_proxy_synapse_user_dir_locations %}
location ~ {{ location }} {
proxy_pass http://user_dir_upstream$request_uri;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $remote_addr;
}
{% endfor %}
{% endif %}
{% if frontend_proxy_workers %}
{# https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappfrontend_proxy #}
location ~ ^/_matrix/client/(api/v1|r0|unstable)/keys/upload {
proxy_pass http://frontend_proxy_upstream$request_uri;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $remote_addr;
}
{% if not matrix_synapse_use_presence %}
location ~ ^/_matrix/client/(api/v1|r0|unstable)/presence/[^/]+/status {
proxy_pass http://frontend_proxy_upstream$request_uri;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $remote_addr;
}
{% endif %}
{% endif %}
{% endif %} {% endif %}
{% set client_reader_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'client_reader')|first %}
{% if client_reader_worker %}
{# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L252 #}
location ^/_matrix/client/(versions$|(api/v1|r0|unstable)/(publicRooms$|rooms/.*/joined_me|rooms/.*/context/.|rooms/.*/members$|rooms/.*/messages$|rooms/.*/state$|login$|account/3pid$|keys/query$|keys/changes$|voip/turnServer$|joined_groups$|publicised_groups$|publicised_groups/|pushrules/.*$|groups/.*$|register$|auth/.*/fallback/web$)) {
{# Use the embedded DNS resolver in Docker containers to discover the service #}
resolver 127.0.0.11 valid=5s;
set $backend "matrix-synapse:{{ client_reader_worker.port }}";
proxy_pass http://$backend$request_uri;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $remote_addr;
}
{% endif %}
{% set media_repository_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'media_repository')|first %}
{% if media_repository_worker %}
{# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L219 #}
location /_matrix/media/ {
{# Use the embedded DNS resolver in Docker containers to discover the service #}
resolver 127.0.0.11 valid=5s;
set $backend "matrix-synapse:{{ media_repository_worker.port }}";
proxy_pass http://$backend$request_uri;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $remote_addr;
}
{# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L223 #}
location ~ ^/_synapse/admin/v1/(purge_media_cache|room/.*/media.*|user/.*/media.*|media/.*|quarantine_media/.*)$ {
{# Use the embedded DNS resolver in Docker containers to discover the service #}
resolver 127.0.0.11 valid=5s;
set $backend "matrix-synapse:{{ media_repository_worker.port }}";
proxy_pass http://$backend$request_uri;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $remote_addr;
}
{% endif %}
{% set event_creator_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'event_creator')|first %}
{% if event_creator_worker %}
{# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L323 #}
location ~ ^/_matrix/client/(api/v1|r0|unstable)/(rooms/.*/send|rooms/.*/state/|rooms/.*/(join|invite|leave|ban|unban|kick)$|join/|profile/) {
{# Use the embedded DNS resolver in Docker containers to discover the service #}
resolver 127.0.0.11 valid=5s;
set $backend "matrix-synapse:{{ event_creator_worker.port }}";
proxy_pass http://$backend$request_uri;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $remote_addr;
}
{% endif %}
{% set frontend_proxy_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'frontend_proxy')|first %}
{% if frontend_proxy_worker %}
{# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L302 #}
location ~ ^/_matrix/client/(api/v1|r0|unstable)/keys/upload {
{# Use the embedded DNS resolver in Docker containers to discover the service #}
resolver 127.0.0.11 valid=5s;
set $backend "matrix-synapse:{{ frontend_proxy_worker.port }}";
proxy_pass http://$backend$request_uri;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $remote_addr;
}
{% if not matrix_synapse_use_presence %}
location ~ ^/_matrix/client/(api/v1|r0|unstable)/presence/[^/]+/status {
{# Use the embedded DNS resolver in Docker containers to discover the service #}
resolver 127.0.0.11 valid=5s;
set $backend "matrix-synapse:{{ frontend_proxy_worker.port }}";
proxy_pass http://$backend$request_uri;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $remote_addr;
}
{% endif %}
{% endif %}
{% set user_dir_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'user_dir')|first %}
{% if user_dir_worker %}
{# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L290 #}
location ~ ^/_matrix/client/(api/v1|r0|unstable)/user_directory/search$ {
{# Use the embedded DNS resolver in Docker containers to discover the service #}
resolver 127.0.0.11 valid=5s;
set $backend "matrix-synapse:{{ user_dir_worker.port }}";
proxy_pass http://$backend$request_uri;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $remote_addr;
}
{% endif %}
{% endif %}
{% for configuration_block in matrix_nginx_proxy_proxy_matrix_additional_server_configuration_blocks %} {% for configuration_block in matrix_nginx_proxy_proxy_matrix_additional_server_configuration_blocks %}
{{- configuration_block }} {{- configuration_block }}
@ -284,19 +229,51 @@
} }
{% endmacro %} {% endmacro %}
{% if matrix_synapse_workers_enabled %} {% set generic_worker_workers = matrix_nginx_proxy_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'generic_worker')|list %}
{% set synchrotron_workers = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'synchrotron')|list %} {% set media_repository_workers = matrix_nginx_proxy_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'media_repository')|list %}
{% if synchrotron_workers %} {% set user_dir_workers = matrix_nginx_proxy_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'user_dir')|list %}
upstream synchrotron { {% set frontend_proxy_workers = matrix_nginx_proxy_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'frontend_proxy')|list %}
# ensures that requests from the same client will always be passed {% if matrix_nginx_proxy_synapse_workers_enabled %}
# to the same server (except when this server is unavailable) {# Setup upstream for groups of workers #}
ip_hash;
{% for synchrotron_worker in synchrotron_workers %} {% if generic_worker_workers %}
server "matrix-synapse:{{ synchrotron_worker.port }}"; upstream generic_worker_upstream {
{% endfor %} # ensures that requests from the same client will always be passed
} # to the same server (except when this server is unavailable)
{% endif %} ip_hash;
{% for worker in generic_worker_workers %}
server "matrix-synapse:{{ worker.port }}";
{% endfor %}
}
{% endif %}
{% if frontend_proxy_workers %}
upstream frontend_proxy_upstream {
# Round Robin
{% for worker in frontend_proxy_workers %}
server "matrix-synapse:{{ worker.port }}";
{% endfor %}
}
{% endif %}
{% if media_repository_workers %}
upstream media_repository_upstream {
# Round Robin
{% for worker in media_repository_workers %}
server "matrix-synapse:{{ worker.port }}";
{% endfor %}
}
{% endif %}
{% if user_dir_workers %}
upstream user_dir_upstream {
# Round Robin
{% for worker in user_dir_workers %}
server "matrix-synapse:{{ worker.port }}";
{% endfor %}
}
{% endif %}
{% endif %} {% endif %}
server { server {
@ -380,19 +357,18 @@ server {
ssl_ciphers "EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH"; ssl_ciphers "EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH";
{% endif %} {% endif %}
{% if matrix_synapse_workers_enabled } {% if matrix_nginx_proxy_synapse_workers_enabled %}
{% set federation_reader_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'federation_reader')|first %} {% if generic_worker_workers %}
{% if federation_reader_worker %} {# https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappgeneric_worker #}
{# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L160 #} {% for location in matrix_nginx_proxy_synapse_generic_worker_locations %}
location ~ ^(/_matrix/federation/v1/event/|/_matrix/federation/v1/state/|/_matrix/federation/v1/state_ids/|/_matrix/federation/v1/backfill/|/_matrix/federation/v1/get_missing_events/|/_matrix/federation/v1/publicRooms|/_matrix/federation/v1/query/|/_matrix/federation/v1/make_join/|/_matrix/federation/v1/make_leave/|/_matrix/federation/v1/send_join/|/_matrix/federation/v2/send_join/|/_matrix/federation/v1/send_leave/|/_matrix/federation/v2/send_leave/|/_matrix/federation/v1/invite/|/_matrix/federation/v2/invite/|/_matrix/federation/v1/query_auth/|/_matrix/federation/v1/event_auth/|/_matrix/federation/v1/exchange_third_party_invite/|/_matrix/federation/v1/user/devices/|/_matrix/federation/v1/send/|/_matrix/federation/v1/get_groups_publicised$|/_matrix/key/v2/query|/_matrix/federation/v1/groups/) { location ~ {{ location }} {
{# Use the embedded DNS resolver in Docker containers to discover the service #} proxy_pass http://generic_worker_upstream$request_uri;
resolver 127.0.0.11 valid=5s; proxy_set_header Host $host;
set $backend "matrix-synapse:{{ federation_reader_worker.port }}"; proxy_set_header X-Forwarded-For $remote_addr;
proxy_pass http://$backend$request_uri; }
proxy_set_header Host $host; {% endfor %}
proxy_set_header X-Forwarded-For $remote_addr; {# ToDo: add GET ^/_matrix/federation/v1/groups/ #}
} {% endif %}
{% endif %}
{% endif %} {% endif %}
location / { location / {

39
roles/matrix-synapse/defaults/main.yml
View File

@ -265,31 +265,28 @@ matrix_synapse_workers_enabled: false
# List of workers to spawn # List of workers to spawn
matrix_synapse_workers_enabled_list: matrix_synapse_workers_enabled_list:
- { worker: synchrotron, port: 18082 } - { worker: generic_worker, port: 18101 }
- { worker: synchrotron, port: 18083 } - { worker: generic_worker, port: 18102 }
- { worker: synchrotron, port: 18084 } - { worker: generic_worker, port: 18103 }
- { worker: appservice, port: 18085 } - { worker: generic_worker, port: 18104 }
- { worker: client_reader, port: 18086 } - { worker: generic_worker, port: 18105 }
- { worker: event_creator, port: 18087 } - { worker: generic_worker, port: 18106 }
- { worker: federation_reader, port: 18088 } - { worker: pusher, port: 18201 }
- { worker: federation_sender, port: 18089 } - { worker: appservice, port: 18301 }
- { worker: frontend_proxy, port: 18090 } - { worker: federation_sender, port: 18401 }
- { worker: media_repository, port: 18091 } - { worker: media_repository, port: 18501 }
- { worker: pusher, port: 18092 } - { worker: user_dir, port: 18502 }
- { worker: user_dir, port: 18093 } - { worker: frontend_proxy, port: 18503 }
# The list of available workers (2020-04-14) # The list of available workers (2020-08-28)
matrix_synapse_workers_avail_list: matrix_synapse_workers_avail_list:
- appservice - generic_worker
- client_reader
- event_creator
- federation_reader
- federation_sender
- frontend_proxy
- media_repository
- pusher - pusher
- synchrotron - appservice
- federation_sender
- media_repository
- user_dir - user_dir
- frontend_proxy
# Ports used for communication between main synapse process and workers # Ports used for communication between main synapse process and workers
matrix_synapse_replication_tcp_port: 9092 matrix_synapse_replication_tcp_port: 9092

5
roles/matrix-synapse/templates/synapse/worker.yaml.j2
View File

@ -11,9 +11,8 @@ worker_listeners:
port: {{ item.port }} port: {{ item.port }}
resources: resources:
- names: - names:
{% if item.worker in [ 'synchrotron', 'client_reader', 'event_creator', 'frontend_proxy', 'user_dir' ] %} {% if item.worker in [ 'generic_worker', 'frontend_proxy', 'user_dir' ] %}
- client - client
{% elif item.worker in [ 'federation_reader' ] %}
- federation - federation
{% elif item.worker in [ 'media_repository' ] %} {% elif item.worker in [ 'media_repository' ] %}
- media - media
@ -23,7 +22,7 @@ worker_listeners:
{% if item.worker == 'frontend_proxy' %} {% if item.worker == 'frontend_proxy' %}
worker_main_http_uri: http://127.0.0.1:8008 worker_main_http_uri: http://127.0.0.1:8008
{% endif %} {% endif %}
worker_daemonize: false worker_daemonize: false
worker_pid_file: /matrix-run/{{ item.worker }}.port{{ item.port }}.pid worker_pid_file: /matrix-run/{{ item.worker }}.port{{ item.port }}.pid
worker_log_config: /data/{{ matrix_server_fqn_matrix }}.log.config worker_log_config: /data/{{ matrix_server_fqn_matrix }}.log.config