Merge remote-tracking branch 'origin/master' into synapse-workers

docs/ansible.md

@@ -11,6 +11,8 @@ If your local computer cannot run Ansible, you can also run Ansible on some serv
 
 Ansible 2.7.0 or newer is required.
 
+Ubuntu (at least 20.04) ships with a buggy version (see this [bug](https://bugs.launchpad.net/ubuntu/+source/ansible/+bug/1880359)), which can't be used in combination with a host running new systemd (more detaisl in [#517](https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/517), [#669]([669](https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/669))). If this problem affects you, you can: avoid running Ubuntu 20.04 on your host; run Ansible from another machine targeting your host; or try to upgrade to a newer Ansible version (see below).
+
 
 ## Checking your Ansible version
 

docs/maintenance-and-troubleshooting.md

@@ -33,6 +33,12 @@ matrix_synapse_root_log_level: "INFO"
 
 Re-run the playbook after making these configuration changes.
 
+## Remove unused Docker data
+
+You can free some disk space from Docker, see [docker system prune](https://docs.docker.com/engine/reference/commandline/system_prune/) for more information.
+```bash
+ansible-playbook -i inventory/hosts setup.yml --tags=run-docker-prune
+```
 
 ## Postgres
 

docs/prerequisites.md

@@ -1,6 +1,12 @@
 # Prerequisites
 
-- An x86 server running **CentOS** (7 only for now; [8 is not yet supported](https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/300)), **Debian** (9/Stretch+), **Ubuntu** (16.04+), or **Archlinux**. This playbook doesn't support running on ARM ([see](https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/299)), however a minimal subset of the tools can be built on the host, which may result in a working configuration, even on a Raspberry pi (see [Alternative Architectures](alternative-architectures.md)). We only strive to support released stable versions of distributions, not betas or pre-releases. This playbook can take over your whole server or co-exist with other services that you have there.
+- An **x86** server running one of these operating systems:
+  - **CentOS** (7 only for now; [8 is not yet supported](https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/300))
+  - **Debian** (9/Stretch+)
+  - **Ubuntu** (16.04+, although [20.04 may be problematic](ansible.md#supported-ansible-versions))
+  - **Archlinux**
+
+This playbook doesn't support running on ARM (see [this issue](https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/299)), however a minimal subset of the tools can be built on the host, which may result in a working configuration, even on a Raspberry pi (see [Alternative Architectures](alternative-architectures.md)). We only strive to support released stable versions of distributions, not betas or pre-releases. This playbook can take over your whole server or co-exist with other services that you have there.
 
 - `root` access to your server (or a user capable of elevating to `root` via `sudo`).
 

roles/matrix-base/defaults/main.yml

@@ -67,6 +67,16 @@ matrix_client_element_jitsi_preferredDomain: ''
 # See: https://github.com/vector-im/element-web/blob/develop/docs/e2ee.md
 matrix_client_element_e2ee_default: true
 
+# Controls whether Element should require a secure backup set up before Element can be used.
+# Setting this to true will update `/.well-known/matrix/client` and tell Element require a secure backup.
+# See: https://github.com/vector-im/element-web/blob/develop/docs/e2ee.md
+matrix_client_element_e2ee_secure_backup_required: false
+
+# Controls which backup methods from ["key", "passphrase"] should be used, both is the default.
+# Setting this to other then empty will update `/.well-known/matrix/client` and tell Element which method to use
+# See: https://github.com/vector-im/element-web/blob/develop/docs/e2ee.md
+matrix_client_element_e2ee_secure_backup_setup_methods: []
+
 # The Docker network that all services would be put into
 matrix_docker_network: "matrix"
 

roles/matrix-base/templates/static-files/well-known/matrix-client.j2

@@ -26,12 +26,13 @@
 		"preferredDomain": {{ matrix_client_element_jitsi_preferredDomain|to_json }}
 	}
 	{% endif %}
-	{% if not matrix_client_element_e2ee_default %},
+	,
 	"io.element.e2ee": {
-		"default": false
+		"default": {{ matrix_client_element_e2ee_default|to_json }},
+		"secure_backup_required": {{ matrix_client_element_e2ee_secure_backup_required|to_json }},
+		"secure_backup_setup_methods": {{ matrix_client_element_e2ee_secure_backup_setup_methods|to_json }}
 	},
 	"im.vector.riot.e2ee": {
-		"default": false
+		"default": {{ matrix_client_element_e2ee_default|to_json }}
 	}
-	{% endif %}
 }

roles/matrix-bot-matrix-reminder-bot/templates/systemd/matrix-bot-matrix-reminder-bot.service.j2

@@ -1,6 +1,6 @@
 #jinja2: lstrip_blocks: "True"
 [Unit]
-Description=matrix-reminder-bot
+Description=Matrix reminder bot
 {% for service in matrix_bot_matrix_reminder_bot_systemd_required_services_list %}
 Requires={{ service }}
 After={{ service }}

roles/matrix-bridge-appservice-discord/templates/systemd/matrix-appservice-discord.service.j2

@@ -1,6 +1,6 @@
 #jinja2: lstrip_blocks: "True"
 [Unit]
-Description=Matrix Appservice Discord server
+Description=Matrix Appservice Discord bridge
 {% for service in matrix_appservice_discord_systemd_required_services_list %}
 Requires={{ service }}
 After={{ service }}

roles/matrix-bridge-appservice-irc/templates/systemd/matrix-appservice-irc.service.j2

@@ -1,6 +1,6 @@
 #jinja2: lstrip_blocks: "True"
 [Unit]
-Description=Matrix Appservice IRC server
+Description=Matrix Appservice IRC bridge
 {% for service in matrix_appservice_irc_systemd_required_services_list %}
 Requires={{ service }}
 After={{ service }}

roles/matrix-bridge-appservice-slack/templates/systemd/matrix-appservice-slack.service.j2

@@ -1,6 +1,6 @@
 #jinja2: lstrip_blocks: "True"
 [Unit]
-Description=Matrix Appservice Slack server
+Description=Matrix Appservice Slack bridge
 {% for service in matrix_appservice_slack_systemd_required_services_list %}
 Requires={{ service }}
 After={{ service }}

roles/matrix-bridge-appservice-webhooks/templates/systemd/matrix-appservice-webhooks.service.j2

@@ -1,6 +1,6 @@
 #jinja2: lstrip_blocks: "True"
 [Unit]
-Description=Matrix Appservice webhooks server
+Description=Matrix Appservice webhooks bridge
 {% for service in matrix_appservice_webhooks_systemd_required_services_list %}
 Requires={{ service }}
 After={{ service }}

roles/matrix-bridge-mautrix-facebook/templates/systemd/matrix-mautrix-facebook.service.j2

@@ -1,6 +1,6 @@
 #jinja2: lstrip_blocks: "True"
 [Unit]
-Description=Matrix Mautrix Facebook server
+Description=Matrix Mautrix Facebook bridge
 {% for service in matrix_mautrix_facebook_systemd_required_services_list %}
 Requires={{ service }}
 After={{ service }}

roles/matrix-bridge-mautrix-hangouts/templates/systemd/matrix-mautrix-hangouts.service.j2

@@ -1,6 +1,6 @@
 #jinja2: lstrip_blocks: "True"
 [Unit]
-Description=Matrix Mautrix Hangouts server
+Description=Matrix Mautrix Hangouts bridge
 {% for service in matrix_mautrix_hangouts_systemd_required_services_list %}
 Requires={{ service }}
 After={{ service }}

roles/matrix-bridge-mautrix-telegram/templates/systemd/matrix-mautrix-telegram.service.j2

@@ -1,6 +1,6 @@
 #jinja2: lstrip_blocks: "True"
 [Unit]
-Description=Matrix Mautrix Telegram server
+Description=Matrix Mautrix Telegram bridge
 {% for service in matrix_mautrix_telegram_systemd_required_services_list %}
 Requires={{ service }}
 After={{ service }}

roles/matrix-bridge-mautrix-whatsapp/templates/systemd/matrix-mautrix-whatsapp.service.j2

@@ -1,6 +1,6 @@
 #jinja2: lstrip_blocks: "True"
 [Unit]
-Description=Matrix Mautrix Whatsapp server
+Description=Matrix Mautrix Whatsapp bridge
 {% for service in matrix_mautrix_whatsapp_systemd_required_services_list %}
 Requires={{ service }}
 After={{ service }}

roles/matrix-bridge-mx-puppet-discord/templates/systemd/matrix-mx-puppet-discord.service.j2

@@ -1,6 +1,6 @@
 #jinja2: lstrip_blocks: "True"
 [Unit]
-Description=Matrix Mx Puppet Discord server
+Description=Matrix Mx Puppet Discord bridge
 {% for service in matrix_mx_puppet_discord_systemd_required_services_list %}
 Requires={{ service }}
 After={{ service }}

roles/matrix-bridge-mx-puppet-instagram/templates/systemd/matrix-mx-puppet-instagram.service.j2

@@ -1,6 +1,6 @@
 #jinja2: lstrip_blocks: "True"
 [Unit]
-Description=Matrix Mx Puppet Instagram server
+Description=Matrix Mx Puppet Instagram bridge
 {% for service in matrix_mx_puppet_instagram_systemd_required_services_list %}
 Requires={{ service }}
 After={{ service }}

roles/matrix-bridge-mx-puppet-skype/templates/systemd/matrix-mx-puppet-skype.service.j2

@@ -1,6 +1,6 @@
 #jinja2: lstrip_blocks: "True"
 [Unit]
-Description=Matrix Mx Puppet Skype server
+Description=Matrix Mx Puppet Skype bridge
 {% for service in matrix_mx_puppet_skype_systemd_required_services_list %}
 Requires={{ service }}
 After={{ service }}

roles/matrix-bridge-mx-puppet-slack/templates/systemd/matrix-mx-puppet-slack.service.j2

@@ -1,6 +1,6 @@
 #jinja2: lstrip_blocks: "True"
 [Unit]
-Description=Matrix Mx Puppet Slack server
+Description=Matrix Mx Puppet Slack bridge
 {% for service in matrix_mx_puppet_slack_systemd_required_services_list %}
 Requires={{ service }}
 After={{ service }}

roles/matrix-bridge-mx-puppet-steam/templates/systemd/matrix-mx-puppet-steam.service.j2

@@ -1,6 +1,6 @@
 #jinja2: lstrip_blocks: "True"
 [Unit]
-Description=Matrix Mx Puppet Steam server
+Description=Matrix Mx Puppet Steam bridge
 {% for service in matrix_mx_puppet_steam_systemd_required_services_list %}
 Requires={{ service }}
 After={{ service }}

roles/matrix-bridge-mx-puppet-twitter/templates/systemd/matrix-mx-puppet-twitter.service.j2

@@ -1,6 +1,6 @@
 #jinja2: lstrip_blocks: "True"
 [Unit]
-Description=Matrix Mx Puppet Twitter server
+Description=Matrix Mx Puppet Twitter bridge
 {% for service in matrix_mx_puppet_twitter_systemd_required_services_list %}
 Requires={{ service }}
 After={{ service }}

roles/matrix-bridge-sms/templates/systemd/matrix-sms-bridge-database.service.j2

@@ -1,6 +1,6 @@
 #jinja2: lstrip_blocks: "True"
 [Unit]
-Description=matrix-sms-bridge-database server
+Description=Matrix sms bridge database
 {% for service in matrix_sms_bridge_database_systemd_required_services_list %}
 Requires={{ service }}
 After={{ service }}

roles/matrix-bridge-sms/templates/systemd/matrix-sms-bridge.service.j2

@@ -1,6 +1,6 @@
 #jinja2: lstrip_blocks: "True"
 [Unit]
-Description=matrix-sms-bridge server
+Description=Matrix sms bridge
 {% for service in matrix_sms_bridge_systemd_required_services_list %}
 Requires={{ service }}
 After={{ service }}

roles/matrix-client-element/defaults/main.yml

@@ -2,7 +2,7 @@ matrix_client_element_enabled: true
 
 matrix_client_element_container_image_self_build: false
 
-matrix_client_element_docker_image: "vectorim/riot-web:v1.7.8"
+matrix_client_element_docker_image: "vectorim/riot-web:v1.7.10"
 matrix_client_element_docker_image_force_pull: "{{ matrix_client_element_docker_image.endswith(':latest') }}"
 
 matrix_client_element_data_path: "{{ matrix_base_data_path }}/client-element"

roles/matrix-common-after/tasks/main.yml

@@ -11,3 +11,7 @@
 - import_tasks: "{{ role_path }}/tasks/dump_runtime_results.yml"
   tags:
     - always
+
+- import_tasks: "{{ role_path }}/tasks/run_docker_prune.yml"
+  tags:
+    - run-docker-prune

roles/matrix-common-after/tasks/run_docker_prune.yml

@@ -0,0 +1,4 @@
+---
+
+- name: Run Docker System Prune
+  command: "{{ matrix_host_command_docker }} system prune -a -f"

roles/matrix-nginx-proxy/defaults/main.yml

@@ -3,7 +3,7 @@ matrix_nginx_proxy_enabled: true
 # We use an official nginx image, which we fix-up to run unprivileged.
 # An alternative would be an `nginxinc/nginx-unprivileged` image, but
 # that is frequently out of date.
-matrix_nginx_proxy_docker_image: "nginx:1.19.2-alpine"
+matrix_nginx_proxy_docker_image: "nginx:1.19.3-alpine"
 matrix_nginx_proxy_docker_image_force_pull: "{{ matrix_nginx_proxy_docker_image.endswith(':latest') }}"
 
 matrix_nginx_proxy_base_path: "{{ matrix_base_data_path }}/nginx-proxy"
@@ -227,7 +227,7 @@ matrix_ssl_domains_to_obtain_certificates_for: []
 
 # Controls whether to obtain production or staging certificates from Let's Encrypt.
 matrix_ssl_lets_encrypt_staging: false
-matrix_ssl_lets_encrypt_certbot_docker_image: "certbot/certbot:{{ matrix_ssl_architecture }}-v1.7.0"
+matrix_ssl_lets_encrypt_certbot_docker_image: "certbot/certbot:{{ matrix_ssl_architecture }}-v1.9.0"
 matrix_ssl_lets_encrypt_certbot_docker_image_force_pull: "{{ matrix_ssl_lets_encrypt_certbot_docker_image.endswith(':latest') }}"
 matrix_ssl_lets_encrypt_certbot_standalone_http_port: 2402
 matrix_ssl_lets_encrypt_support_email: ~

roles/matrix-synapse/defaults/main.yml

@@ -5,7 +5,7 @@ matrix_synapse_enabled: true
 
 matrix_synapse_container_image_self_build: false
 
-matrix_synapse_docker_image: "matrixdotorg/synapse:v1.20.1"
+matrix_synapse_docker_image: "matrixdotorg/synapse:v1.21.2"
 matrix_synapse_docker_image_force_pull: "{{ matrix_synapse_docker_image.endswith(':latest') }}"
 
 matrix_synapse_base_path: "{{ matrix_base_data_path }}/synapse"

roles/matrix-synapse/templates/synapse/homeserver.yaml.j2

@@ -9,10 +9,23 @@
 
 ## Server ##
 
-# The domain name of the server, with optional explicit port.
+# The public-facing domain of the server
-# This is used by remote servers to connect to this server,
+#
-# e.g. matrix.org, localhost:8080, etc.
+# The server_name name will appear at the end of usernames and room addresses
-# This is also the last part of your UserID.
+# created on this server. For example if the server_name was example.com,
+# usernames on this server would be in the format @user:example.com
+#
+# In most cases you should avoid using a matrix specific subdomain such as
+# matrix.example.com or synapse.example.com as the server_name for the same
+# reasons you wouldn't use user@email.example.com as your email address.
+# See https://github.com/matrix-org/synapse/blob/master/docs/delegate.md
+# for information on how to host Synapse on a subdomain while preserving
+# a clean server_name.
+#
+# The server_name cannot be changed later so it is important to
+# configure this correctly before you start Synapse. It should be all
+# lowercase and may contain an explicit port.
+# Examples: matrix.org, localhost:8080
 #
 server_name: "{{ matrix_domain }}"
 
@@ -107,7 +120,6 @@ default_room_version: {{ matrix_synapse_default_room_version|to_json }}
 #
 #enable_search: false
 
-
 # List of ports that Synapse should listen on, their purpose and their
 # configuration.
 #
@@ -422,6 +434,12 @@ retention:
   # 'longest_max_lifetime' of '3d' will handle every room with a retention policy
   # which 'max_lifetime' is lower than or equal to three days.
   #
+  # The rationale for this per-job configuration is that some rooms might have a
+  # retention policy with a low 'max_lifetime', where history needs to be purged
+  # of outdated messages on a more frequent basis than for the rest of the rooms
+  # (e.g. every 12h), but not want that purge to be performed by a job that's
+  # iterating over every room it knows, which could be heavy on the server.
+  #
   # If any purge job is configured, it is strongly recommended to have at least
   # a single job with neither 'shortest_max_lifetime' nor 'longest_max_lifetime'
   # set, or one job without 'shortest_max_lifetime' and one job without
@@ -447,6 +465,24 @@ retention:
 #
 #request_token_inhibit_3pid_errors: true
 
+# A list of domains that the domain portion of 'next_link' parameters
+# must match.
+#
+# This parameter is optionally provided by clients while requesting
+# validation of an email or phone number, and maps to a link that
+# users will be automatically redirected to after validation
+# succeeds. Clients can make use this parameter to aid the validation
+# process.
+#
+# The whitelist is applied whether the homeserver or an
+# identity server is handling validation.
+#
+# The default value is no whitelist functionality; all domains are
+# allowed. Setting this value to an empty list will instead disallow
+# all domains.
+#
+#next_link_domain_whitelist: ["matrix.org"]
+
 
 ## TLS ##
 
@@ -613,6 +649,7 @@ acme:
 #tls_fingerprints: [{"sha256": "<base64_encoded_sha256_fingerprint>"}]
 
 
+## Federation ##
 
 # Restrict federation to the following whitelist of domains.
 # N.B. we recommend also firewalling your federation listener to limit
@@ -650,6 +687,17 @@ federation_ip_range_blacklist:
   - 'fe80::/64'
   - 'fc00::/7'
 
+# Report prometheus metrics on the age of PDUs being sent to and received from
+# the following domains. This can be used to give an idea of "delay" on inbound
+# and outbound federation, though be aware that any delay can be due to problems
+# at either end or with the intermediate network.
+#
+# By default, no domains are monitored in this way.
+#
+#federation_metrics_domains:
+#  - matrix.org
+#  - example.com
+
 
 ## Caching ##
 
@@ -695,6 +743,7 @@ caches:
    per_cache_factors:
      #get_users_who_share_room_with_user: 2.0
 
+
 ## Database ##
 
 database:
@@ -1108,6 +1157,17 @@ account_validity:
   #
   #invalid_token_html_path: "invalid_token.html"
 
+# Time that a user's session remains valid for, after they log in.
+#
+# Note that this is not currently compatible with guest logins.
+#
+# Note also that this is calculated at login time: changes are not applied
+# retrospectively to users who have already logged in.
+#
+# By default, this is infinite.
+#
+#session_lifetime: 24h
+
 # The user must provide all of the below types of 3PID when registering.
 #
 #registrations_require_3pid:
@@ -1461,11 +1521,14 @@ trusted_key_servers: {{ matrix_synapse_trusted_key_servers|to_json }}
 # At least one of `sp_config` or `config_path` must be set in this section to
 # enable SAML login.
 #
-# (You will probably also want to set the following options to `false` to
+# You will probably also want to set the following options to `false` to
 # disable the regular login/registration flows:
 #   * enable_registration
 #   * password_config.enabled
 #
+# You will also want to investigate the settings under the "sso" configuration
+# section below.
+#
 # Once SAML support is enabled, a metadata file will be exposed at
 # https://<server>:<port>/_matrix/saml2/metadata.xml, which you may be able to
 # use to configure your SAML IdP with. Alternatively, you can manually configure
@@ -1687,6 +1750,19 @@ oidc_config:
   #
   #skip_verification: true
 
+  # Whether to fetch the user profile from the userinfo endpoint. Valid
+  # values are: "auto" or "userinfo_endpoint".
+  #
+  # Defaults to "auto", which fetches the userinfo endpoint if "openid" is included
+  # in `scopes`. Uncomment the following to always fetch the userinfo endpoint.
+  #
+  #user_profile_method: "userinfo_endpoint"
+
+  # Uncomment to allow a user logging in via OIDC to match a pre-existing account instead
+  # of failing. This could be used if switching from password logins to OIDC. Defaults to false.
+  #
+  #allow_existing_users: true
+
   # An external module can be provided here as a custom solution to mapping
   # attributes returned from a OIDC provider onto a matrix user.
   #
@@ -1864,6 +1940,24 @@ sso:
     #
     #algorithm: "provided-by-your-issuer"
 
+    # The issuer to validate the "iss" claim against.
+    #
+    # Optional, if provided the "iss" claim will be required and
+    # validated for all JSON web tokens.
+    #
+    #issuer: "provided-by-your-issuer"
+
+    # A list of audiences to validate the "aud" claim against.
+    #
+    # Optional, if provided the "aud" claim will be required and
+    # validated for all JSON web tokens.
+    #
+    # Note that if the "aud" claim is included in a JSON web token then
+    # validation will fail without configuring audiences.
+    #
+    #audiences:
+    #    - "provided-by-your-issuer"
+
 
 password_config:
    # Uncomment to disable password login
@@ -1970,9 +2064,13 @@ email:
   # * The contents of password reset emails sent by the homeserver:
   #   'password_reset.html' and 'password_reset.txt'
   #
-  # * HTML pages for success and failure that a user will see when they follow
+  # * An HTML page that a user will see when they follow the link in the password
-  #   the link in the password reset email: 'password_reset_success.html' and
+  #   reset email. The user will be asked to confirm the action before their
-  #   'password_reset_failure.html'
+  #   password is reset: 'password_reset_confirmation.html'
+  #
+  # * HTML pages for success and failure that a user will see when they confirm
+  #   the password reset flow using the page above: 'password_reset_success.html'
+  #   and 'password_reset_failure.html'
   #
   # * The contents of address verification emails sent during registration:
   #   'registration.html' and 'registration.txt'
@@ -2451,6 +2549,11 @@ opentracing:
 #  events: worker1
 #  typing: worker1
 
+# The worker that is used to run background tasks (e.g. cleaning up expired
+# data). If not provided this defaults to the main process.
+#
+#run_background_tasks_on: worker1
+
 
 # Configuration for Redis when using workers. This *must* be enabled when
 # using workers (unless using old style direct TCP configuration).