Initial work on matrix-homeserver-proxy role and eliminating matrix-nginx-proxy

This is still very far from usable.

Various bridges and bots are still talking to
`matrix-nginx-proxy` instead of the new `matrix-homeserver-proxy` role.
These services need to be reworked. While reworking them,
various cleanups are being done as well as adding Traefik-labels to
those that need them.
This commit is contained in:
Slavi Pantaleev
2024-01-02 15:56:56 +02:00
parent c744d29567
commit 4a6287c528
35 changed files with 968 additions and 166 deletions

View File

@ -0,0 +1,179 @@
---
# matrix-homeserver-proxy is a role which brings up a containerized nginx webserver which helps with reverse-proxying to the Matrix homeserver (Synapse, etc.).
#
# Certain services (like matrix-media-repo, matrix-corporal, identity servers, etc.) may need to capture some requests destined for the homeserver
# and handle them instead of it.
#
# This role helps other services (bots, bridges, etc.) reach the homeserver in a way that:
# - is not very direct, so as to allow for some routes (media repo, etc.) to actually go elsewhere
# - is not via the public network and/or via HTTPS, which introduces major performance penalties
#
# Performance-wise, benchmarks show that:
# - each local (container) nginx hop adds about a 200 rps penalty
# - SSL termination (on the Traefik side) adds a 350 rps penalty
# - going over the public network adds another 70 rps penalty
#
# It's something like this for an existing flow (which will be gone soon):
# 1. public network, Traefik + SSL: 70 rps
# 2. `matrix-nginx-proxy:8008`: 600 rps
# 3. `matrix-nginx-proxy:12080` 850 rps
# 4. `matrix-synapse-reverse-proxy-companion:8008`: 1000 rps
# 5. `matrix-synapse:8008`: 1200 rps
#
# Traefik was additionally benchmarked to see where the slowness comes from. Results are like this:
# 1. public network, Traefik + SSL: 70 rps
# 2. local (container) network, Traefik + SSL: 150 rps
# 3. local (container) network, Traefik without SSL: 500 rps
# 4. `matrix-nginx-proxy:8008`: 600 rps
#
# It's obvious that minimizing the number of hops helps a lot and that not using SSL and/or the public network is important.
matrix_homeserver_proxy_enabled: true
matrix_homeserver_proxy_ident: matrix-homeserver-proxy
# renovate: datasource=docker depName=nginx
matrix_homeserver_proxy_version: 1.25.3-alpine
matrix_homeserver_proxy_base_path: "{{ matrix_base_data_path }}/homeserver-proxy"
matrix_homeserver_proxy_confd_path: "{{ matrix_homeserver_proxy_base_path }}/conf.d"
# List of systemd services that matrix-homeserver-proxy.service depends on
matrix_homeserver_proxy_systemd_required_services_list: ['docker.service']
# List of systemd services that matrix-homeserver-proxy.service wants
matrix_homeserver_proxy_systemd_wanted_services_list: "{{ matrix_homeserver_proxy_systemd_wanted_services_list_auto + matrix_homeserver_proxy_systemd_wanted_services_list_custom }}"
matrix_homeserver_proxy_systemd_wanted_services_list_auto: []
matrix_homeserver_proxy_systemd_wanted_services_list_custom: []
# We use an official nginx image, which we fix-up to run unprivileged.
# An alternative would be an `nginxinc/nginx-unprivileged` image, but that is frequently out of date.
matrix_homeserver_proxy_container_image: "{{ matrix_container_global_registry_prefix }}nginx:{{ matrix_homeserver_proxy_version }}"
matrix_homeserver_proxy_container_image_force_pull: "{{ matrix_homeserver_proxy_container_image.endswith(':latest') }}"
matrix_homeserver_proxy_container_network: matrix-homeserver-proxy
# A list of additional container networks that matrix-homeserver-proxy would be connected to.
# The playbook does not create these networks, so make sure they already exist.
matrix_homeserver_proxy_container_additional_networks: []
# Controls whether the matrix-homeserver-proxy container exposes its HTTP Client-Server API port (tcp/8008 in the container).
#
# Takes an "<ip>:<port>" or "<port>" value (e.g. "127.0.0.1:8008"), or empty string to not expose.
matrix_homeserver_proxy_container_client_api_host_bind_port: ''
# Controls whether the matrix-homeserver-proxy container exposes its HTTP Federation (Server-Server) API port (tcp/8048 in the container).
#
# Takes an "<ip>:<port>" or "<port>" value (e.g. "127.0.0.1:8048"), or empty string to not expose.
matrix_homeserver_proxy_container_federation_api_host_bind_port: ''
# Option to disable the access log
matrix_homeserver_proxy_access_log_enabled: true
# The tmpfs at /tmp needs to be large enough to handle multiple concurrent file uploads.
matrix_homeserver_proxy_tmp_directory_size_mb: "{{ (matrix_homeserver_proxy_federation_api_client_max_body_size_mb | int) * 50 }}"
matrix_homeserver_proxy_tmp_cache_directory_size_mb: "{{ (matrix_homeserver_proxy_cache_max_size_mb | int) * 2 }}"
# A list of strings containing additional configuration blocks to add to the nginx server configuration (nginx.conf).
# for big matrixservers to enlarge the number of open files to prevent timeouts
# matrix_homeserver_proxy_additional_configuration_blocks:
# - 'worker_rlimit_nofile 30000;'
matrix_homeserver_proxy_additional_configuration_blocks: []
# A list of strings containing additional configuration blocks to add to the nginx event server configuration (nginx.conf).
matrix_homeserver_proxy_event_additional_configuration_blocks: []
# A list of strings containing additional configuration blocks to add to the nginx http's server configuration (nginx-http.conf).
matrix_homeserver_proxy_http_additional_server_configuration_blocks: []
# To increase request timeout in NGINX using proxy_read_timeout, proxy_connect_timeout, proxy_send_timeout, send_timeout directives
# Nginx Default: proxy_connect_timeout 60s; #Defines a timeout for establishing a connection with a proxied server
# Nginx Default: proxy_send_timeout 60s; #Sets a timeout for transmitting a request to the proxied server.
# Nginx Default: proxy_read_timeout 60s; #Defines a timeout for reading a response from the proxied server.
# Nginx Default: send_timeout 60s; #Sets a timeout for transmitting a response to the client.
#
# For more information visit:
# http://nginx.org/en/docs/http/ngx_http_proxy_module.html
# http://nginx.org/en/docs/http/ngx_http_core_module.html#send_timeout
# https://www.nginx.com/resources/wiki/start/topics/examples/fullexample2/
#
# Here we are sticking with nginx default values change this value carefully.
matrix_homeserver_proxy_proxy_connect_timeout: 60
matrix_homeserver_proxy_proxy_send_timeout: 60
matrix_homeserver_proxy_proxy_read_timeout: 60
matrix_homeserver_proxy_send_timeout: 60
# For OCSP purposes, we need to define a resolver at the `server{}` level or `http{}` level (we do the latter).
#
# Otherwise, we get warnings like this:
# > [warn] 22#22: no resolver defined to resolve r3.o.lencr.org while requesting certificate status, responder: r3.o.lencr.org, certificate: "/matrix/ssl/config/live/.../fullchain.pem"
#
# We point it to the internal Docker resolver, which likely delegates to nameservers defined in `/etc/resolv.conf`.
matrix_homeserver_proxy_http_level_resolver: 127.0.0.11
matrix_homeserver_proxy_hostname: "{{ matrix_homeserver_proxy_ident }}"
# matrix_homeserver_proxy_client_api_addr specifies the address where the Client-Server API is
matrix_homeserver_proxy_client_api_addr: ''
# This needs to be equal or higher than the maximum upload size accepted by the homeserver.
matrix_homeserver_proxy_client_api_client_max_body_size_mb: 50
# Tells whether `/_synapse/client` is forwarded to the Matrix Client API server.
matrix_homeserver_proxy_client_api_forwarded_location_synapse_client_api_enabled: true
# Tells whether `/_synapse/oidc` is forwarded to the Matrix Client API server.
# Enable this if you need OpenID Connect authentication support.
matrix_homeserver_proxy_client_api_forwarded_location_synapse_oidc_api_enabled: false
# Tells whether `/_synapse/admin` is forwarded to the Matrix Client API server.
# Following these recommendations (https://github.com/matrix-org/synapse/blob/master/docs/reverse_proxy.md), by default, we don't.
matrix_homeserver_proxy_client_api_forwarded_location_synapse_admin_api_enabled: false
# `matrix_homeserver_proxy_client_api_forwarded_location_prefix_regexes` holds
# the location prefixes that get forwarded to the Matrix Client API server.
# These locations get combined into a regex like this `^(/_matrix|/_synapse/client)`.
matrix_homeserver_proxy_client_api_forwarded_location_prefix_regexes: |
{{
(['/_matrix'])
+
(['/_synapse/client'] if matrix_homeserver_proxy_client_api_forwarded_location_synapse_client_api_enabled else [])
+
(['/_synapse/oidc'] if matrix_homeserver_proxy_client_api_forwarded_location_synapse_oidc_api_enabled else [])
+
(['/_synapse/admin'] if matrix_homeserver_proxy_client_api_forwarded_location_synapse_admin_api_enabled else [])
}}
# Specifies where requests for the root URI (`/`) on the `matrix.` domain should be redirected.
# If this has an empty value, they're just passed to the homeserver, which serves a static page.
# If you'd like to make `https://matrix.DOMAIN` redirect to `https://element.DOMAIN` (or something of that sort), specify the domain name here.
# Example value: `element.DOMAIN` (or `{{ matrix_server_fqn_element }}`).
matrix_homeserver_proxy_client_redirect_root_uri_to_domain: ""
# A list of strings containing additional configuration blocks to add to the nginx vhost handling the Client-Server API
matrix_homeserver_proxy_client_api_additional_server_configuration_blocks: "{{ matrix_homeserver_proxy_client_api_additional_server_configuration_blocks_auto + matrix_homeserver_proxy_client_api_additional_server_configuration_blocks_custom }}"
matrix_homeserver_proxy_client_api_additional_server_configuration_blocks_auto: []
matrix_homeserver_proxy_client_api_additional_server_configuration_blocks_custom: []
# matrix_homeserver_proxy_federation_api_enabled specifies whether reverse proxying for the Federation (Server-Server) API should be done
matrix_homeserver_proxy_federation_api_enabled: true
# matrix_homeserver_proxy_federation_api_addr specifies the address where the Federation (Server-Server) API is
matrix_homeserver_proxy_federation_api_addr: ''
matrix_homeserver_proxy_federation_api_client_max_body_size_mb: "{{ (matrix_homeserver_proxy_client_api_client_max_body_size_mb | int) * 3 }}"
# A list of strings containing additional configuration blocks to add to the nginx vhost handling the Federation (Server-Server) API
matrix_homeserver_proxy_federation_api_additional_server_configuration_blocks: "{{ matrix_homeserver_proxy_federation_api_additional_server_configuration_blocks_auto + matrix_homeserver_proxy_federation_api_additional_server_configuration_blocks_custom }}"
matrix_homeserver_proxy_federation_api_additional_server_configuration_blocks_auto: []
matrix_homeserver_proxy_federation_api_additional_server_configuration_blocks_custom: []
# Controls whether matrix-homeserver-proxy trusts an upstream server's X-Forwarded-Proto header.
# The `matrix-homeserver-proxy` does not terminate SSL and always expects to be fronted by another reverse-proxy server.
# As such, it trusts the protocol scheme forwarded by the upstream proxy.
matrix_homeserver_proxy_trust_forwarded_proto: true
matrix_homeserver_proxy_x_forwarded_proto_value: "{{ '$http_x_forwarded_proto' if matrix_homeserver_proxy_trust_forwarded_proto else '$scheme' }}"
# The amount of worker processes and connections
# Consider increasing these when you are expecting high amounts of traffic
# http://nginx.org/en/docs/ngx_core_module.html#worker_connections
matrix_homeserver_proxy_worker_processes: auto
matrix_homeserver_proxy_worker_connections: 1024

View File

@ -0,0 +1,49 @@
---
- name: Ensure Matrix Homeserver Proxy paths exist
ansible.builtin.file:
path: "{{ item }}"
state: directory
mode: 0750
owner: "{{ matrix_user_username }}"
group: "{{ matrix_user_groupname }}"
with_items:
- "{{ matrix_homeserver_proxy_base_path }}"
- "{{ matrix_homeserver_proxy_confd_path }}"
- name: Ensure Matrix Homeserver Proxy is configured
ansible.builtin.template:
src: "{{ item.src }}"
dest: "{{ item.dest }}"
owner: "{{ matrix_user_username }}"
group: "{{ matrix_user_groupname }}"
mode: 0644
with_items:
- src: "{{ role_path }}/templates/nginx/nginx.conf.j2"
dest: "{{ matrix_homeserver_proxy_base_path }}/nginx.conf"
- src: "{{ role_path }}/templates/nginx/conf.d/nginx-http.conf.j2"
dest: "{{ matrix_homeserver_proxy_confd_path }}/nginx-http.conf"
- src: "{{ role_path }}/templates/nginx/conf.d/matrix-homeserver-proxy.conf.j2"
dest: "{{ matrix_homeserver_proxy_confd_path }}/matrix-homeserver-proxy.conf"
- name: Ensure Matrix Homeserver Proxy nginx container image is pulled
community.docker.docker_image:
name: "{{ matrix_homeserver_proxy_container_image }}"
source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
force_source: "{{ matrix_homeserver_proxy_container_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_homeserver_proxy_container_image_force_pull }}"
register: result
retries: "{{ devture_playbook_help_container_retries_count }}"
delay: "{{ devture_playbook_help_container_retries_delay }}"
until: result is not failed
- name: Ensure Matrix Homeserver Proxy container network is created
community.general.docker_network:
name: "{{ matrix_homeserver_proxy_container_network }}"
driver: bridge
- name: Ensure Matrix Homeserver Proxy systemd service is installed
ansible.builtin.template:
src: "{{ role_path }}/templates/systemd/matrix-homeserver-proxy.service.j2"
dest: "{{ devture_systemd_docker_base_systemd_path }}/{{ matrix_homeserver_proxy_ident }}.service"
mode: 0644

View File

@ -0,0 +1,20 @@
---
- tags:
- setup-all
- setup-homeserver-proxy
- setup-synapse
- install-all
- install-homeserver-proxy
- install-synapse
block:
- when: matrix_homeserver_proxy_enabled | bool
ansible.builtin.include_tasks: "{{ role_path }}/tasks/install.yml"
- tags:
- setup-all
- setup-homeserver-proxy
- setup-synapse
block:
- when: not matrix_homeserver_proxy_enabled | bool
ansible.builtin.include_tasks: "{{ role_path }}/tasks/uninstall.yml"

View File

@ -0,0 +1,25 @@
---
- name: Check existence of Matrix Homeserver Proxy systemd service
ansible.builtin.stat:
path: "{{ devture_systemd_docker_base_systemd_path }}/{{ matrix_homeserver_proxy_ident }}.service"
register: matrix_homeserver_proxy_service_stat
- when: matrix_homeserver_proxy_service_stat.stat.exists | bool
block:
- name: Ensure Matrix Homeserver Proxy systemd service is stopped
ansible.builtin.service:
name: "{{ matrix_homeserver_proxy_ident }}"
state: stopped
enabled: false
daemon_reload: true
- name: Ensure Matrix Homeserver Proxy systemd service doesn't exist
ansible.builtin.file:
path: "{{ devture_systemd_docker_base_systemd_path }}/{{ matrix_homeserver_proxy_ident }}.service"
state: absent
- name: Ensure Matrix Homeserver Proxy directory is deleted
ansible.builtin.file:
path: "{{ matrix_homeserver_proxy_base_path }}"
state: absent

View File

@ -0,0 +1,62 @@
#jinja2: lstrip_blocks: "True"
server {
listen 8008;
server_name {{ matrix_homeserver_proxy_hostname }};
server_tokens off;
root /dev/null;
gzip on;
gzip_types text/plain application/json;
{% for configuration_block in matrix_homeserver_proxy_client_api_additional_server_configuration_blocks %}
{{- configuration_block }}
{% endfor %}
{# Everything else just goes to the API server ##}
location / {
{# Use the embedded DNS resolver in Docker containers to discover the service #}
resolver {{ matrix_homeserver_proxy_http_level_resolver }} valid=5s;
set $backend "{{ matrix_homeserver_proxy_client_api_addr }}";
proxy_pass http://$backend;
proxy_set_header Host $host;
client_body_buffer_size 25M;
client_max_body_size {{ matrix_homeserver_proxy_client_api_client_max_body_size_mb }}M;
proxy_max_temp_file_size 0;
}
}
{% if matrix_homeserver_proxy_federation_api_enabled %}
server {
listen 8048;
server_name {{ matrix_homeserver_proxy_hostname }};
server_tokens off;
root /dev/null;
gzip on;
gzip_types text/plain application/json;
{% for configuration_block in matrix_homeserver_proxy_federation_api_additional_server_configuration_blocks %}
{{- configuration_block }}
{% endfor %}
{# Everything else just goes to the API server ##}
location / {
{# Use the embedded DNS resolver in Docker containers to discover the service #}
resolver {{ matrix_homeserver_proxy_http_level_resolver }} valid=5s;
set $backend "{{ matrix_homeserver_proxy_federation_api_addr }}";
proxy_pass http://$backend;
proxy_set_header Host $host;
client_body_buffer_size 25M;
client_max_body_size {{ matrix_homeserver_proxy_federation_api_client_max_body_size_mb }}M;
proxy_max_temp_file_size 0;
}
}
{% endif %}

View File

@ -0,0 +1,13 @@
#jinja2: lstrip_blocks: "True"
# The default is aligned to the CPU's cache size,
# which can sometimes be too low.
# Thus, we ensure a larger bucket size value is used.
server_names_hash_bucket_size 64;
{% if matrix_homeserver_proxy_http_level_resolver %}
resolver {{ matrix_homeserver_proxy_http_level_resolver }};
{% endif %}
{% for configuration_block in matrix_homeserver_proxy_http_additional_server_configuration_blocks %}
{{- configuration_block }}
{% endfor %}

View File

@ -0,0 +1,77 @@
#jinja2: lstrip_blocks: "True"
# This is a custom nginx configuration file that we use in the container (instead of the default one),
# because it allows us to run nginx with a non-root user.
#
# For this to work, the default vhost file (`/etc/nginx/conf.d/default.conf`) also needs to be removed.
#
# The following changes have been done compared to a default nginx configuration file:
# - various temp paths are changed to `/tmp`, so that a non-root user can write to them
# - the `user` directive was removed, as we don't want nginx to switch users
worker_processes {{ matrix_homeserver_proxy_worker_processes }};
error_log /var/log/nginx/error.log warn;
pid /tmp/nginx.pid;
{% for configuration_block in matrix_homeserver_proxy_additional_configuration_blocks %}
{{- configuration_block }}
{% endfor %}
events {
worker_connections {{ matrix_homeserver_proxy_worker_connections }};
{% for configuration_block in matrix_homeserver_proxy_event_additional_configuration_blocks %}
{{- configuration_block }}
{% endfor %}
}
http {
proxy_temp_path /tmp/proxy_temp;
client_body_temp_path /tmp/client_temp;
fastcgi_temp_path /tmp/fastcgi_temp;
uwsgi_temp_path /tmp/uwsgi_temp;
scgi_temp_path /tmp/scgi_temp;
include /etc/nginx/mime.types;
default_type application/octet-stream;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
{% if matrix_homeserver_proxy_access_log_enabled %}
access_log /var/log/nginx/access.log main;
{% endif %}
{% if matrix_homeserver_proxy_access_log_syslog_integration_enabled %}
log_format prometheus_fmt 'matrix-homeserver-proxy $server_name - $upstream_addr - $remote_addr - $remote_user [$time_local] '
'$host "$request" '
'$status "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log syslog:server={{ matrix_homeserver_proxy_access_log_syslog_integration_server_port }},tag=matrix_nginx_proxy_companion prometheus_fmt;
{% endif %}
{% if not matrix_homeserver_proxy_access_log_enabled and not matrix_homeserver_proxy_access_log_syslog_integration_enabled %}
access_log off;
{% endif %}
proxy_connect_timeout {{ matrix_homeserver_proxy_proxy_connect_timeout }};
proxy_send_timeout {{ matrix_homeserver_proxy_proxy_send_timeout }};
proxy_read_timeout {{ matrix_homeserver_proxy_proxy_read_timeout }};
send_timeout {{ matrix_homeserver_proxy_send_timeout }};
sendfile on;
#tcp_nopush on;
keepalive_timeout 65;
server_tokens off;
#gzip on;
{# Map directive needed for proxied WebSocket upgrades #}
map $http_upgrade $connection_upgrade {
default upgrade;
'' close;
}
include /etc/nginx/conf.d/*.conf;
}

View File

@ -0,0 +1,52 @@
#jinja2: lstrip_blocks: "True"
[Unit]
Description=Matrix Homeserver Proxy
{% for service in matrix_homeserver_proxy_systemd_required_services_list %}
Requires={{ service }}
After={{ service }}
{% endfor %}
{% for service in matrix_homeserver_proxy_systemd_wanted_services_list %}
Wants={{ service }}
{% endfor %}
DefaultDependencies=no
[Service]
Type=simple
Environment="HOME={{ devture_systemd_docker_base_systemd_unit_home_path }}"
ExecStartPre=-{{ devture_systemd_docker_base_host_command_sh }} -c '{{ devture_systemd_docker_base_host_command_docker }} stop --time={{ devture_systemd_docker_base_container_stop_grace_time_seconds }} matrix-homeserver-proxy 2>/dev/null || true'
ExecStartPre=-{{ devture_systemd_docker_base_host_command_sh }} -c '{{ devture_systemd_docker_base_host_command_docker }} rm matrix-homeserver-proxy 2>/dev/null || true'
ExecStartPre={{ devture_systemd_docker_base_host_command_docker }} create \
--rm \
--name=matrix-homeserver-proxy \
--log-driver=none \
--user={{ matrix_user_uid }}:{{ matrix_user_gid }} \
--cap-drop=ALL \
--read-only \
--tmpfs=/tmp:rw,noexec,nosuid,size={{ matrix_homeserver_proxy_tmp_directory_size_mb }}m \
--network={{ matrix_homeserver_proxy_container_network }} \
{% if matrix_homeserver_proxy_container_client_api_host_bind_port %}
-p {{ matrix_homeserver_proxy_container_client_api_host_bind_port }}:8008 \
{% endif %}
{% if matrix_homeserver_proxy_container_federation_api_host_bind_port %}
-p {{ matrix_homeserver_proxy_container_federation_api_host_bind_port }}:8048 \
{% endif %}
--mount type=bind,src={{ matrix_homeserver_proxy_base_path }}/nginx.conf,dst=/etc/nginx/nginx.conf,ro \
--mount type=bind,src={{ matrix_homeserver_proxy_confd_path }},dst=/etc/nginx/conf.d,ro \
{{ matrix_homeserver_proxy_container_image }}
{% for network in matrix_homeserver_proxy_container_additional_networks %}
ExecStartPre={{ devture_systemd_docker_base_host_command_docker }} network connect {{ network }} matrix-homeserver-proxy
{% endfor %}
ExecStart={{ devture_systemd_docker_base_host_command_docker }} start --attach matrix-homeserver-proxy
ExecStop=-{{ devture_systemd_docker_base_host_command_sh }} -c '{{ devture_systemd_docker_base_host_command_docker }} stop --time={{ devture_systemd_docker_base_container_stop_grace_time_seconds }} matrix-homeserver-proxy 2>/dev/null || true'
ExecStop=-{{ devture_systemd_docker_base_host_command_sh }} -c '{{ devture_systemd_docker_base_host_command_docker }} rm matrix-homeserver-proxy 2>/dev/null || true'
ExecReload={{ devture_systemd_docker_base_host_command_docker }} exec matrix-homeserver-proxy /usr/sbin/nginx -s reload
Restart=always
RestartSec=30
SyslogIdentifier=matrix-homeserver-proxy
[Install]
WantedBy=multi-user.target

View File

@ -0,0 +1,4 @@
---
matrix_homeserver_proxy_client_api_url: "http://{{ matrix_homeserver_proxy_ident }}:8008"
matrix_homeserver_proxy_federation_api_url: "http://{{ matrix_homeserver_proxy_ident }}:8048"