Move roles/matrix* to roles/custom/matrix*

This paves the way for installing other roles into `roles/galaxy` using `ansible-galaxy`,
similar to how it's done in:

- https://github.com/spantaleev/gitea-docker-ansible-deploy
- https://github.com/spantaleev/nextcloud-docker-ansible-deploy

In the near future, we'll be removing a lot of the shared role code from here
and using upstream roles for it. Some of the core `matrix-*` roles have
already been extracted out into other reusable roles:

- https://github.com/devture/com.devture.ansible.role.postgres
- https://github.com/devture/com.devture.ansible.role.systemd_docker_base
- https://github.com/devture/com.devture.ansible.role.timesync
- https://github.com/devture/com.devture.ansible.role.vars_preserver
- https://github.com/devture/com.devture.ansible.role.playbook_runtime_messages
- https://github.com/devture/com.devture.ansible.role.playbook_help

We just need to migrate to those.

roles/custom/matrix-synapse/templates/synapse/customizations/Dockerfile.j2

@@ -0,0 +1,7 @@
+FROM {{ matrix_synapse_docker_image }}
+
+{% if matrix_synapse_container_image_customizations_s3_storage_provider_installation_enabled %}
+RUN pip install synapse-s3-storage-provider=={{ matrix_synapse_ext_synapse_s3_storage_provider_version }}
+{% endif %}
+
+{{ matrix_synapse_container_image_customizations_dockerfile_body_custom }}

roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/database.yaml.j2

@@ -0,0 +1,5 @@
+user: {{ matrix_synapse_database_user | to_json }}
+password: {{ matrix_synapse_database_password | to_json }}
+database: {{ matrix_synapse_database_database | to_json }}
+host: {{ matrix_synapse_database_host | to_json }}
+port: {{ matrix_synapse_database_port | to_json }}

roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2

@@ -0,0 +1,11 @@
+AWS_ACCESS_KEY_ID={{ matrix_synapse_ext_synapse_s3_storage_provider_config_access_key_id }}
+AWS_SECRET_ACCESS_KEY={{ matrix_synapse_ext_synapse_s3_storage_provider_config_secret_access_key }}
+AWS_DEFAULT_REGION={{ matrix_synapse_ext_synapse_s3_storage_provider_config_region_name }}
+
+ENDPOINT={{ matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url }}
+BUCKET={{ matrix_synapse_ext_synapse_s3_storage_provider_config_bucket }}
+STORAGE_CLASS={{ matrix_synapse_ext_synapse_s3_storage_provider_config_storage_class }}
+
+MEDIA_PATH=/matrix-media-store-parent/{{ matrix_synapse_media_store_directory_name }}
+
+UPDATE_DB_DURATION={{ matrix_synapse_ext_synapse_s3_storage_provider_update_db_day_count }}d

roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/media_storage_provider.yaml.j2

@@ -0,0 +1,14 @@
+module: s3_storage_provider.S3StorageProviderBackend
+store_local: {{ matrix_synapse_ext_synapse_s3_storage_provider_store_local | to_json }}
+store_remote: {{ matrix_synapse_ext_synapse_s3_storage_provider_store_remote | to_json }}
+store_synchronous: {{ matrix_synapse_ext_synapse_s3_storage_provider_store_synchronous | to_json }}
+config:
+  bucket: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_bucket | to_json }}
+  region_name: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_region_name | to_json }}
+  endpoint_url: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url | to_json }}
+  access_key_id: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_access_key_id | to_json }}
+  secret_access_key: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_secret_access_key | to_json }}
+
+  storage_class: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_storage_class | to_json }}
+
+  threadpool_size: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_threadpool_size | to_json }}

roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/systemd/matrix-synapse-s3-storage-provider-migrate.service.j2

@@ -0,0 +1,7 @@
+[Unit]
+Description=Migrates locally-stored Synapse media store files to S3
+
+[Service]
+Type=oneshot
+Environment="HOME={{ matrix_systemd_unit_home_path }}"
+ExecStart={{ matrix_local_bin_path }}/matrix-synapse-s3-storage-provider-migrate

roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/systemd/matrix-synapse-s3-storage-provider-migrate.timer.j2

@@ -0,0 +1,9 @@
+[Unit]
+Description=Migrates locally-stored Synapse media store files to S3
+
+[Timer]
+Unit=matrix-synapse-s3-storage-provider-migrate.service
+OnCalendar=*-*-* 05:00:00
+
+[Install]
+WantedBy=timers.target

roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-migrate.j2

@@ -0,0 +1,13 @@
+#jinja2: lstrip_blocks: "True"
+#!/bin/bash
+
+{{ matrix_host_command_docker }} run \
+	--rm \
+	--env-file={{ matrix_synapse_ext_s3_storage_provider_path }}/env \
+	--mount type=bind,src={{ matrix_synapse_storage_path }},dst=/matrix-media-store-parent,bind-propagation=slave \
+	--mount type=bind,src={{ matrix_synapse_ext_s3_storage_provider_path }}/data,dst=/data \
+	--workdir=/data \
+	--network={{ matrix_docker_network }} \
+	--entrypoint=/bin/bash \
+	{{ matrix_synapse_docker_image_final }} \
+	-c 's3_media_upload update-db $UPDATE_DB_DURATION && s3_media_upload --no-progress check-deleted $MEDIA_PATH && s3_media_upload --no-progress upload $MEDIA_PATH $BUCKET --delete --storage-class $STORAGE_CLASS --endpoint-url $ENDPOINT'

roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-shell.j2

@@ -0,0 +1,13 @@
+#jinja2: lstrip_blocks: "True"
+#!/bin/bash
+
+{{ matrix_host_command_docker }} run \
+	-it \
+	--rm \
+	--env-file={{ matrix_synapse_ext_s3_storage_provider_path }}/env \
+	--mount type=bind,src={{ matrix_synapse_storage_path }},dst=/matrix-media-store-parent,bind-propagation=slave \
+	--mount type=bind,src={{ matrix_synapse_ext_s3_storage_provider_path }}/data,dst=/data \
+	--workdir=/data \
+	--network={{ matrix_docker_network }} \
+	--entrypoint=/bin/bash \
+	{{ matrix_synapse_docker_image_final }}

roles/custom/matrix-synapse/templates/synapse/prometheus/external_prometheus.yml.example.j2

@@ -0,0 +1,41 @@
+global:
+  scrape_interval: 5s
+
+  # Attach these labels to any time series or alerts when communicating with
+  # external systems (federation, remote storage, Alertmanager).
+  external_labels:
+    monitor: 'synapse-{{ matrix_domain }}'
+
+rule_files:
+  - /etc/prometheus/synapse-v2.rules
+
+scrape_configs:
+  - job_name: 'synapse'
+    metrics_path: /metrics/synapse/main-process
+    scheme: {{ 'https' if matrix_nginx_proxy_https_enabled|default(true) else 'http' }}
+{% if matrix_nginx_proxy_proxy_matrix_metrics_basic_auth_enabled|default(true) %}
+    basic_auth:
+      username: prometheus
+      password_file: /path/to/your/passwordfile.pwd
+{% endif %}
+    static_configs:
+      - targets: ['{{ matrix_server_fqn_matrix }}:{{ matrix_nginx_proxy_container_https_host_bind_port|default(443) if matrix_nginx_proxy_https_enabled|default(true) else matrix_nginx_proxy_container_http_host_bind_port|default(80) }}']
+        labels:
+          job: "master"
+          index: "0"
+{% for worker in matrix_synapse_workers_enabled_list %}
+  - job_name: '{{ worker.name }}'
+    metrics_path: /metrics/synapse/worker/{{ worker.id }}
+    scheme: {{ 'https' if matrix_nginx_proxy_https_enabled|default(true) else 'http' }}
+{% if matrix_nginx_proxy_proxy_matrix_metrics_basic_auth_enabled|default(true) %}
+    basic_auth:
+      username: prometheus
+      password_file: /path/to/your/passwordfile.pwd
+{% endif %}
+    static_configs:
+      - targets: ['{{ matrix_server_fqn_matrix }}:{{ matrix_nginx_proxy_container_https_host_bind_port|default(443) if matrix_nginx_proxy_https_enabled|default(true) else matrix_nginx_proxy_container_http_host_bind_port|default(80) }}']
+        labels:
+          worker_id: {{ worker.id }}
+          job: "{{ worker.type }}"
+          app: {{ worker.app }}
+{% endfor %}

roles/custom/matrix-synapse/templates/synapse/synapse.log.config.j2

@@ -0,0 +1,36 @@
+#jinja2: lstrip_blocks: "True"
+
+version: 1
+
+formatters:
+    precise:
+        format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
+
+filters:
+    context:
+        (): synapse.util.logcontext.LoggingContextFilter
+        request: ""
+
+handlers:
+    console:
+        class: logging.StreamHandler
+        formatter: precise
+        filters: [context]
+
+loggers:
+    synapse:
+        level: {{ matrix_synapse_log_level }}
+
+    synapse.storage.SQL:
+        # beware: increasing this to DEBUG will make synapse log sensitive
+        # information such as access tokens.
+        level: {{ matrix_synapse_storage_sql_log_level }}
+
+{% for logger in matrix_synapse_additional_loggers %}
+    {{ logger.name }}:
+        level: {{ logger.level }}
+{% endfor %}
+
+root:
+    level: {{ matrix_synapse_root_log_level }}
+    handlers: [console]

roles/custom/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker.service.j2

@@ -0,0 +1,62 @@
+#jinja2: lstrip_blocks: "True"
+[Unit]
+Description=Synapse worker ({{ matrix_synapse_worker_container_name }})
+AssertPathExists={{ matrix_synapse_config_dir_path }}/{{ matrix_synapse_worker_config_file_name }}
+After=matrix-synapse.service
+
+[Service]
+Type=simple
+Environment="HOME={{ matrix_systemd_unit_home_path }}"
+
+ExecStartPre=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} kill {{ matrix_synapse_worker_container_name }} 2>/dev/null || true'
+ExecStartPre=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} rm {{ matrix_synapse_worker_container_name }} 2>/dev/null || true'
+
+# Intentional delay, so that the homeserver can manage to start.
+ExecStartPre={{ matrix_host_command_sleep }} 5
+
+ExecStart={{ matrix_host_command_docker }} run --rm --name {{ matrix_synapse_worker_container_name }} \
+			--log-driver=none \
+			--user={{ matrix_user_uid }}:{{ matrix_user_gid }} \
+			--cap-drop=ALL \
+			--read-only \
+			--tmpfs=/tmp:rw,noexec,nosuid,size={{ matrix_synapse_tmp_directory_size_mb }}m \
+			--network={{ matrix_docker_network }} \
+			{% if matrix_synapse_worker_details.port != 0 %}
+			--health-cmd 'curl -fSs http://localhost:{{ matrix_synapse_worker_details.port }}/health || exit 1' \
+			{% else %}
+			--no-healthcheck \
+			{% endif %}
+			{% if matrix_synapse_workers_enabled and matrix_synapse_workers_container_host_bind_address %}
+			{% if matrix_synapse_worker_details.port != 0 %}
+			-p {{ '' if matrix_synapse_workers_container_host_bind_address == '*' else (matrix_synapse_workers_container_host_bind_address + ':') }}{{ matrix_synapse_worker_details.port }}:{{ matrix_synapse_worker_details.port }} \
+			{% endif %}
+			{% if matrix_synapse_worker_details.metrics_port != 0 %}
+			-p {{ '' if matrix_synapse_workers_container_host_bind_address == '*' else (matrix_synapse_workers_container_host_bind_address + ':') }}{{ matrix_synapse_worker_details.metrics_port }}:{{ matrix_synapse_worker_details.metrics_port }} \
+			{% endif %}
+			{% endif %}
+			--mount type=bind,src={{ matrix_synapse_config_dir_path }},dst=/data,ro \
+			--mount type=bind,src={{ matrix_synapse_storage_path }},dst=/matrix-media-store-parent,bind-propagation=slave \
+			{% for volume in matrix_synapse_container_additional_volumes %}
+			-v {{ volume.src }}:{{ volume.dst }}:{{ volume.options }} \
+			{% endfor %}
+			{% for arg in matrix_synapse_container_arguments %}
+			{{ arg }} \
+			{% endfor %}
+			{{ matrix_synapse_docker_image_final }} \
+			run -m synapse.app.{{ matrix_synapse_worker_details.app }} -c /data/homeserver.yaml -c /data/{{ matrix_synapse_worker_config_file_name }}
+
+
+ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} kill {{ matrix_synapse_worker_container_name }} 2>/dev/null || true'
+ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} rm {{ matrix_synapse_worker_container_name }} 2>/dev/null || true'
+
+ExecReload={{ matrix_host_command_docker }} exec {{ matrix_synapse_worker_container_name }} /bin/sh -c 'kill -HUP 1'
+Restart=always
+RestartSec=30
+SyslogIdentifier={{ matrix_synapse_worker_container_name }}
+
+# Intentionally not making this WantedBy=matrix-synapse.service,
+# as matrix.synapse.service already has `Wants=` lines.
+# Also, WantedBy will trigger the creation of some `matrix-synapse.service.wants/` directory,
+# which we'd have to clean, etc. Better not.
+[Install]
+WantedBy=multi-user.target

roles/custom/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2

@@ -0,0 +1,74 @@
+#jinja2: lstrip_blocks: "True"
+[Unit]
+Description=Synapse server
+{% for service in matrix_synapse_systemd_required_services_list %}
+Requires={{ service }}
+After={{ service }}
+
+{% endfor %}
+{% for service in matrix_synapse_systemd_wanted_services_list %}
+Wants={{ service }}
+{% endfor %}
+
+{% if matrix_synapse_workers_enabled %}
+{% for matrix_synapse_worker_details in matrix_synapse_workers_enabled_list %}
+Wants={{ matrix_synapse_worker_details.name }}.service
+{% endfor %}
+{% endif %}
+
+DefaultDependencies=no
+
+[Service]
+Type=simple
+Environment="HOME={{ matrix_systemd_unit_home_path }}"
+ExecStartPre=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} kill matrix-synapse 2>/dev/null || true'
+ExecStartPre=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} rm matrix-synapse 2>/dev/null || true'
+{% if matrix_s3_media_store_enabled %}
+# Allow for some time before starting, so that media store can mount.
+# Mounting can happen later too, but if we start writing,
+# we'd write files to the local filesystem and fusermount will complain.
+ExecStartPre={{ matrix_host_command_sleep }} 3
+{% endif %}
+
+ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-synapse \
+			--log-driver=none \
+			--user={{ matrix_user_uid }}:{{ matrix_user_gid }} \
+			--cap-drop=ALL \
+			--read-only \
+			--tmpfs=/tmp:rw,noexec,nosuid,size={{ matrix_synapse_tmp_directory_size_mb }}m \
+			--network={{ matrix_docker_network }} \
+			{% if matrix_synapse_container_client_api_host_bind_port %}
+			-p {{ matrix_synapse_container_client_api_host_bind_port }}:{{ matrix_synapse_container_client_api_port }} \
+			{% endif %}
+			{% if matrix_synapse_federation_enabled and matrix_synapse_tls_federation_listener_enabled and matrix_synapse_container_federation_api_tls_host_bind_port %}
+			-p {{ matrix_synapse_container_federation_api_tls_host_bind_port }}:{{ matrix_synapse_container_federation_api_tls_port }} \
+			{% endif %}
+			{% if matrix_synapse_federation_enabled and matrix_synapse_federation_port_enabled and matrix_synapse_container_federation_api_plain_host_bind_port %}
+			-p {{ matrix_synapse_container_federation_api_plain_host_bind_port }}:{{ matrix_synapse_container_federation_api_plain_port }} \
+			{% endif %}
+			{% if matrix_synapse_metrics_enabled and matrix_synapse_container_metrics_api_host_bind_port %}
+			-p {{ matrix_synapse_container_metrics_api_host_bind_port }}:{{ matrix_synapse_metrics_port }} \
+			{% endif %}
+			{% if matrix_synapse_manhole_enabled and matrix_synapse_container_manhole_api_host_bind_port %}
+			-p {{ matrix_synapse_container_manhole_api_host_bind_port }}:9000 \
+			{% endif %}
+			--mount type=bind,src={{ matrix_synapse_config_dir_path }},dst=/data,ro \
+			--mount type=bind,src={{ matrix_synapse_storage_path }},dst=/matrix-media-store-parent,bind-propagation=slave \
+			{% for volume in matrix_synapse_container_additional_volumes %}
+			-v {{ volume.src }}:{{ volume.dst }}:{{ volume.options }} \
+			{% endfor %}
+			{% for arg in matrix_synapse_container_arguments %}
+			{{ arg }} \
+			{% endfor %}
+			{{ matrix_synapse_docker_image_final }} \
+			run -m synapse.app.homeserver -c /data/homeserver.yaml
+
+ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} kill matrix-synapse 2>/dev/null || true'
+ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} rm matrix-synapse 2>/dev/null || true'
+ExecReload={{ matrix_host_command_docker }} exec matrix-synapse /bin/sh -c 'kill -HUP 1'
+Restart=always
+RestartSec=30
+SyslogIdentifier=matrix-synapse
+
+[Install]
+WantedBy=multi-user.target

roles/custom/matrix-synapse/templates/synapse/usr-local-bin/matrix-synapse-register-user.j2

@@ -0,0 +1,17 @@
+#jinja2: lstrip_blocks: "True"
+#!/bin/bash
+
+if [ $# -ne 3 ]; then
+	echo "Usage: "$0" <username> <password> <admin access: 0 or 1>"
+	exit 1
+fi
+
+user=$1
+password=$2
+admin=$3
+
+if [ "$admin" -eq "1" ]; then
+	docker exec matrix-synapse register_new_matrix_user -u "$user" -p "$password" -c /data/homeserver.yaml --admin http://localhost:{{ matrix_synapse_container_client_api_port }}
+else
+	docker exec matrix-synapse register_new_matrix_user -u "$user" -p "$password" -c /data/homeserver.yaml --no-admin http://localhost:{{ matrix_synapse_container_client_api_port }}
+fi

roles/custom/matrix-synapse/templates/synapse/worker.yaml.j2

@@ -0,0 +1,64 @@
+#jinja2: lstrip_blocks: "True"
+worker_app: synapse.app.{{ matrix_synapse_worker_details.app }}
+worker_name: {{ matrix_synapse_worker_details.name }}
+
+worker_daemonize: false
+worker_log_config: /data/{{ matrix_server_fqn_matrix }}.log.config
+
+{% if matrix_synapse_replication_listener_enabled %}
+worker_replication_host: matrix-synapse
+worker_replication_http_port: {{ matrix_synapse_replication_http_port }}
+{% endif %}
+
+{% if matrix_synapse_worker_details.type == 'generic_worker' %}
+worker_main_http_uri: http://matrix-synapse:{{ matrix_synapse_container_client_api_port }}
+{% endif %}
+
+{% set http_resources = [] %}
+
+{% if matrix_synapse_worker_details.type == 'user_dir' %}
+  {% set http_resources = http_resources + ['client'] %}
+{% endif %}
+{% if matrix_synapse_worker_details.type == 'generic_worker' %}
+  {% set http_resources = http_resources + ['client', 'federation'] %}
+{% endif %}
+{#
+  None of the background workers need to handle federation traffic.
+  Only some of the stream writers need to handle client traffic.
+#}
+{% if matrix_synapse_worker_details.type == 'stream_writer' and matrix_synapse_worker_details.webserving %}
+  {% set http_resources = http_resources + ['client'] %}
+{% endif %}
+{% if matrix_synapse_worker_details.type == 'media_repository' %}
+  {% set http_resources = http_resources + ['media'] %}
+{% endif %}
+
+{% set replication_http_resources = [] %}
+{% if matrix_synapse_worker_details.type == 'stream_writer' %}
+  {# All background workers need to handle replication traffic. #}
+  {% set replication_http_resources = replication_http_resources + ['replication'] %}
+{% endif %}
+
+{% if http_resources|length > 0 or matrix_synapse_metrics_enabled or replication_http_resources|length > 0 %}
+worker_listeners:
+{% if http_resources|length > 0 %}
+  - type: http
+    bind_addresses: ['::']
+    x_forwarded: true
+    port: {{ matrix_synapse_worker_details.port }}
+    resources:
+      - names: {{ http_resources|to_json }}
+{% endif %}
+{% if matrix_synapse_metrics_enabled %}
+  - type: metrics
+    bind_addresses: ['0.0.0.0']
+    port: {{ matrix_synapse_worker_details.metrics_port }}
+{% endif %}
+{% if replication_http_resources|length > 0 %}
+  - type: http
+    bind_addresses: ['::']
+    port: {{ matrix_synapse_worker_details.replication_port }}
+    resources:
+      - names: {{ replication_http_resources|to_json }}
+{% endif %}
+{% endif %}