Add support for synapse-s3-storage-provider

roles/matrix-synapse/defaults/main.yml

@@ -19,6 +19,10 @@ matrix_synapse_container_image_self_build_repo: "https://github.com/matrix-org/s
 # - `matrix_synapse_docker_image_final`
 matrix_synapse_container_image_customizations_enabled: "{{ matrix_synapse_ext_synapse_s3_storage_provider_enabled }}"
 
+# Controls whether custom build steps will be added to the Dockerfile for installing s3-storage-provider.
+# The version that will be installed is specified in `matrix_synapse_ext_synapse_s3_storage_provider_version`.
+matrix_synapse_container_image_customizations_s3_storage_provider_installation_enabled: "{{ matrix_synapse_ext_synapse_s3_storage_provider_enabled }}"
+
 # matrix_synapse_container_image_customizations_dockerfile_body contains your custom Dockerfile steps
 # for building your customized Synapse image based on the original (upstream) image (`matrix_synapse_docker_image`).
 # A `FROM ...` clause is included automatically so you don't have to.
@@ -52,6 +56,7 @@ matrix_synapse_config_dir_path: "{{ matrix_synapse_base_path }}/config"
 matrix_synapse_storage_path: "{{ matrix_synapse_base_path }}/storage"
 matrix_synapse_media_store_path: "{{ matrix_synapse_storage_path }}/media-store"
 matrix_synapse_ext_path: "{{ matrix_synapse_base_path }}/ext"
+matrix_synapse_ext_s3_storage_provider_path: "{{ matrix_synapse_base_path }}/ext/s3-storage-provider"
 
 matrix_synapse_container_client_api_port: 8008
 
@@ -787,6 +792,32 @@ matrix_synapse_ext_encryption_config_yaml: |
   patch_power_levels: {{ matrix_synapse_ext_encryption_disabler_patch_power_levels | to_json }}
 
 
+# matrix_synapse_ext_synapse_s3_storage_provider_enabled controls whether to enable https://github.com/matrix-org/synapse-s3-storage-provider
+# Installing it requires building a customized Docker image for Synapse (see `matrix_synapse_container_image_customizations_enabled`).
+# Enabling this will enable customizations and inject the appropriate Dockerfile clauses for installing synapse-s3-storage-provider.
+matrix_synapse_ext_synapse_s3_storage_provider_enabled: false
+matrix_synapse_ext_synapse_s3_storage_provider_version: 1.1.2
+# Controls whether media from this (local) server is stored in s3-storage-provider
+matrix_synapse_ext_synapse_s3_storage_provider_store_local: true
+# Controls whether media from remote servers is stored in s3-storage-provider
+matrix_synapse_ext_synapse_s3_storage_provider_store_remote: true
+# Controls whether files are stored to S3 at the same time they are stored on the local filesystem.
+# For slightly improved reliability, consider setting this to `true`.
+# Even with asynchronous uploading to S3 (`false` value), data loss shouldn't be possible,
+# because the local filesystem is a reliable data store anyway.
+matrix_synapse_ext_synapse_s3_storage_provider_store_synchronous: false
+matrix_synapse_ext_synapse_s3_storage_provider_config_bucket: ''
+matrix_synapse_ext_synapse_s3_storage_provider_config_region_name: ''
+matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url: ''
+matrix_synapse_ext_synapse_s3_storage_provider_config_access_key_id: ''
+matrix_synapse_ext_synapse_s3_storage_provider_config_secret_access_key: ''
+matrix_synapse_ext_synapse_s3_storage_provider_config_storage_class: STANDARD
+matrix_synapse_ext_synapse_s3_storage_provider_config_threadpool_size: 40
+# matrix_synapse_ext_synapse_s3_storage_provider_update_db_day_count is a day value (number) for the `s3_media_upload update-db` command.
+# It specifies how old files need to have been inactive to be eligible for migration from the local filesystem to the S3 data store.
+# By default, we use `0` which says "all files are eligible for migration".
+matrix_synapse_ext_synapse_s3_storage_provider_update_db_day_count: 0
+
 matrix_s3_media_store_enabled: false
 matrix_s3_media_store_custom_endpoint_enabled: false
 matrix_s3_goofys_docker_image: "ewoutp/goofys:latest"
@@ -839,6 +870,10 @@ matrix_synapse_media_storage_providers: "{{ matrix_synapse_media_storage_provide
 matrix_synapse_media_storage_providers_auto: |
   {{
     []
+    +
+    [
+      lookup('ansible.builtin.template', role_path + '/templates/synapse/ext/s3-storage-provider/media_storage_provider.yaml.j2') | from_yaml
+    ] if matrix_synapse_ext_synapse_s3_storage_provider_enabled else []
   }}
 
 # matrix_synapse_media_storage_providers_custom contains your own custom list of storage providers.

roles/matrix-synapse/tasks/ext/s3-storage-provider/init.yml

@@ -0,0 +1,5 @@
+---
+
+- ansible.builtin.set_fact:
+    matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-synapse-s3-storage-provider-migrate.timer'] }}"
+  when: matrix_synapse_ext_synapse_s3_storage_provider_enabled | bool

roles/matrix-synapse/tasks/ext/s3-storage-provider/setup.yml

@@ -0,0 +1,10 @@
+---
+
+- ansible.builtin.import_tasks: "{{ role_path }}/tasks/ext/s3-storage-provider/validate_config.yml"
+  when: matrix_synapse_ext_synapse_s3_storage_provider_enabled | bool
+
+- ansible.builtin.import_tasks: "{{ role_path }}/tasks/ext/s3-storage-provider/setup_install.yml"
+  when: matrix_synapse_ext_synapse_s3_storage_provider_enabled | bool
+
+- ansible.builtin.import_tasks: "{{ role_path }}/tasks/ext/s3-storage-provider/setup_uninstall.yml"
+  when: not matrix_synapse_ext_synapse_s3_storage_provider_enabled | bool

roles/matrix-synapse/tasks/ext/s3-storage-provider/setup_install.yml

@@ -0,0 +1,54 @@
+---
+
+# We install this into Synapse by making `matrix_synapse_ext_synapse_s3_storage_provider_enabled` influence other variables:
+# - `matrix_synapse_media_storage_providers` (via `matrix_synapse_media_storage_providers_auto`)
+# - `matrix_synapse_container_image_customizations_enabled`
+# - `matrix_synapse_container_image_customizations_s3_storage_provider_installation_enabled`
+#
+# Below are additional tasks for setting up various helper scripts, etc.
+
+- name: Ensure s3-storage-provider env file installed
+  ansible.builtin.template:
+    src: "{{ role_path }}/templates/synapse/ext/s3-storage-provider/env.j2"
+    dest: "{{ matrix_synapse_ext_s3_storage_provider_path }}/env"
+    mode: 0640
+
+- name: Ensure s3-storage-provider data path exists
+  ansible.builtin.file:
+    path: "{{ matrix_synapse_ext_s3_storage_provider_path }}/data"
+    state: directory
+    mode: 0750
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_groupname }}"
+
+- name: Ensure s3-storage-provider database.yaml file installed
+  ansible.builtin.template:
+    src: "{{ role_path }}/templates/synapse/ext/s3-storage-provider/database.yaml.j2"
+    dest: "{{ matrix_synapse_ext_s3_storage_provider_path }}/data/database.yaml"
+    mode: 0640
+
+- name: Ensure s3-storage-provider scripts installed
+  ansible.builtin.template:
+    src: "{{ role_path }}/templates/synapse/ext/s3-storage-provider/usr-local-bin/{{ item }}.j2"
+    dest: "{{ matrix_local_bin_path }}/{{ item }}"
+    mode: 0750
+  with_items:
+    - matrix-synapse-s3-storage-provider-shell
+    - matrix-synapse-s3-storage-provider-migrate
+
+- name: Ensure matrix-synapse-s3-storage-provider-migrate.service and timer are installed
+  ansible.builtin.template:
+    src: "{{ role_path }}/templates/systemd/.j2"
+    src: "{{ role_path }}/templates/synapse/ext/s3-storage-provider/systemd/{{ item }}.j2"
+    dest: "{{ matrix_systemd_path }}/{{ item }}"
+    mode: 0640
+  with_items:
+    - matrix-synapse-s3-storage-provider-migrate.service
+    - matrix-synapse-s3-storage-provider-migrate.timer
+  register: matrix_synapse_s3_storage_provider_systemd_service_result
+
+- name: Ensure systemd reloaded after matrix-synapse-s3-storage-provider-migrate.service installation
+  ansible.builtin.service:
+    daemon_reload: true
+  when: matrix_synapse_s3_storage_provider_systemd_service_result.changed | bool
+

roles/matrix-synapse/tasks/ext/s3-storage-provider/setup_uninstall.yml

@@ -0,0 +1,24 @@
+---
+
+- name: Ensure matrix-synapse-s3-storage-provider-migrate.service and timer don't exist
+  ansible.builtin.file:
+    path: "{{ matrix_systemd_path }}/{{ item }}"
+    state: absent
+  with_items:
+    - matrix-synapse-s3-storage-provider-migrate.timer
+    - matrix-synapse-s3-storage-provider-migrate.service
+  register: matrix_synapse_s3_storage_provider_migrate_sevice_removal
+
+- name: Ensure systemd reloaded after matrix-synapse-s3-storage-provider-migrate.service removal
+  ansible.builtin.service:
+    daemon_reload: true
+  when: matrix_synapse_s3_storage_provider_migrate_sevice_removal.changed | bool
+
+- name: Ensure s3-storage-provider files don't exist
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: absent
+  with_items:
+    - "{{ matrix_local_bin_path }}/matrix-synapse-s3-storage-provider-shell"
+    - "{{ matrix_local_bin_path }}/matrix-synapse-s3-storage-provider-migrate"
+    - "{{ matrix_synapse_ext_s3_storage_provider_path }}"

roles/matrix-synapse/tasks/ext/s3-storage-provider/validate_config.yml

@@ -0,0 +1,18 @@
+---
+
+- name: Fail if required s3-storage-provider settings not defined
+  ansible.builtin.fail:
+    msg: >-
+      You need to define a required configuration setting (`{{ item }}`) for using s3-storage-provider.
+  when: "vars[item] == ''"
+  with_items:
+    - "matrix_synapse_ext_synapse_s3_storage_provider_config_bucket"
+    - "matrix_synapse_ext_synapse_s3_storage_provider_config_region_name"
+    - "matrix_synapse_ext_synapse_s3_storage_provider_config_access_key_id"
+    - "matrix_synapse_ext_synapse_s3_storage_provider_config_secret_access_key"
+
+- name: Fail if required matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url looks invalid
+  ansible.builtin.fail:
+    msg: >-
+      `matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url` needs to look like a URL (`http://` or `https://` prefix).
+  when: "matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url != '' and not matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url.startswith('http')"

roles/matrix-synapse/tasks/ext/setup.yml

@@ -11,3 +11,5 @@
 - ansible.builtin.import_tasks: "{{ role_path }}/tasks/ext/synapse-simple-antispam/setup.yml"
 
 - ansible.builtin.import_tasks: "{{ role_path }}/tasks/ext/mjolnir-antispam/setup.yml"
+
+- ansible.builtin.import_tasks: "{{ role_path }}/tasks/ext/s3-storage-provider/setup.yml"

roles/matrix-synapse/tasks/init.yml

@@ -26,6 +26,9 @@
     matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-goofys.service'] }}"
   when: matrix_s3_media_store_enabled | bool
 
+- ansible.builtin.include_tasks: "{{ role_path }}/tasks/ext/s3-storage-provider/init.yml"
+  when: matrix_synapse_ext_synapse_s3_storage_provider_enabled | bool
+
 - when: matrix_synapse_enabled | bool and matrix_synapse_metrics_proxying_enabled | bool
   block:
     - name: Fail if matrix-nginx-proxy role already executed

roles/matrix-synapse/tasks/setup_synapse.yml

@@ -12,6 +12,7 @@
     - {path: "{{ matrix_synapse_ext_path }}", when: true}
     - {path: "{{ matrix_synapse_docker_src_files_path }}", when: "{{ matrix_synapse_container_image_self_build }}"}
     - {path: "{{ matrix_synapse_customized_docker_src_files_path }}", when: "{{ matrix_synapse_container_image_customizations_enabled }}"}
+    - {path: "{{ matrix_synapse_ext_s3_storage_provider_path }}", when: "{{ matrix_synapse_ext_synapse_s3_storage_provider_enabled }}"}
     # We handle matrix_synapse_media_store_path elsewhere (in ./synapse/setup_install.yml),
     # because if it's using Goofys and it's already mounted (from before),
     # trying to chown/chmod it here will cause trouble.

roles/matrix-synapse/templates/synapse/customizations/Dockerfile.j2

@@ -1,3 +1,7 @@
 FROM {{ matrix_synapse_docker_image }}
 
+{% if matrix_synapse_container_image_customizations_s3_storage_provider_installation_enabled %}
+RUN pip install synapse-s3-storage-provider=={{ matrix_synapse_ext_synapse_s3_storage_provider_version }}
+{% endif %}
+
 {{ matrix_synapse_container_image_customizations_dockerfile_body_custom }}

roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/database.yaml.j2

@@ -0,0 +1,5 @@
+user: {{ matrix_synapse_database_user | to_json }}
+password: {{ matrix_synapse_database_password | to_json }}
+database: {{ matrix_synapse_database_database | to_json }}
+host: {{ matrix_synapse_database_host | to_json }}
+port: {{ matrix_synapse_database_port | to_json }}

roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2

@@ -0,0 +1,16 @@
+AWS_ACCESS_KEY_ID={{ matrix_synapse_ext_synapse_s3_storage_provider_config_access_key_id }}
+AWS_SECRET_ACCESS_KEY={{ matrix_synapse_ext_synapse_s3_storage_provider_config_secret_access_key }}
+AWS_DEFAULT_REGION={{ matrix_synapse_ext_synapse_s3_storage_provider_config_region_name }}
+
+ENDPOINT={{ matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url }}
+BUCKET={{ matrix_synapse_ext_synapse_s3_storage_provider_config_bucket }}
+
+PG_USER={{ matrix_synapse_database_user }}
+PG_PASS={{ matrix_synapse_database_password }}
+PG_DB={{ matrix_synapse_database_database }}
+PG_HOST={{ matrix_synapse_database_host }}
+PG_PORT={{ matrix_synapse_database_port }}
+
+MEDIA_PATH=/matrix-media-store-parent/{{ matrix_synapse_media_store_directory_name }}
+
+UPDATE_DB_DURATION={{ matrix_synapse_ext_synapse_s3_storage_provider_update_db_day_count }}d

roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/media_storage_provider.yaml.j2

@@ -0,0 +1,14 @@
+module: s3_storage_provider.S3StorageProviderBackend
+store_local: {{ matrix_synapse_ext_synapse_s3_storage_provider_store_local | to_json }}
+store_remote: {{ matrix_synapse_ext_synapse_s3_storage_provider_store_remote | to_json }}
+store_synchronous: {{ matrix_synapse_ext_synapse_s3_storage_provider_store_synchronous | to_json }}
+config:
+  bucket: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_bucket | to_json }}
+  region_name: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_region_name | to_json }}
+  endpoint_url: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url | to_json }}
+  access_key_id: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_access_key_id | to_json }}
+  secret_access_key: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_secret_access_key | to_json }}
+
+  storage_class: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_storage_class | to_json }}
+
+  threadpool_size: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_threadpool_size | to_json }}

roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/systemd/matrix-synapse-s3-storage-provider-migrate.service.j2

@@ -0,0 +1,7 @@
+[Unit]
+Description=Migrates locally-stored Synapse media store files to S3
+
+[Service]
+Type=oneshot
+Environment="HOME={{ matrix_systemd_unit_home_path }}"
+ExecStart={{ matrix_local_bin_path }}/matrix-synapse-s3-storage-provider-migrate

roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/systemd/matrix-synapse-s3-storage-provider-migrate.timer.j2

@@ -0,0 +1,10 @@
+[Unit]
+Description=Migrates locally-stored Synapse media store files to S3
+
+[Timer]
+Unit=matrix-synapse-s3-storage-provider-migrate.service
+OnCalendar=*-*-* 05:00:00
+RandomizedDelaySec=2h
+
+[Install]
+WantedBy=timers.target

roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-migrate.j2

@@ -0,0 +1,13 @@
+#jinja2: lstrip_blocks: "True"
+#!/bin/bash
+
+{{ matrix_host_command_docker }} run \
+	--rm \
+	--env-file={{ matrix_synapse_ext_s3_storage_provider_path }}/env \
+	--mount type=bind,src={{ matrix_synapse_storage_path }},dst=/matrix-media-store-parent,bind-propagation=slave \
+	--mount type=bind,src={{ matrix_synapse_ext_s3_storage_provider_path }}/data,dst=/data \
+	--workdir=/data \
+	--network={{ matrix_docker_network }} \
+	--entrypoint=/bin/bash \
+	{{ matrix_synapse_docker_image_final }} \
+	-c 's3_media_upload update-db $UPDATE_DB_DURATION && s3_media_upload --no-progress check-deleted $MEDIA_PATH && s3_media_upload --no-progress upload $MEDIA_PATH $BUCKET --delete --endpoint-url $ENDPOINT'

roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-shell.j2

@@ -0,0 +1,13 @@
+#jinja2: lstrip_blocks: "True"
+#!/bin/bash
+
+{{ matrix_host_command_docker }} run \
+	-it \
+	--rm \
+	--env-file={{ matrix_synapse_ext_s3_storage_provider_path }}/env \
+	--mount type=bind,src={{ matrix_synapse_storage_path }},dst=/matrix-media-store-parent,bind-propagation=slave \
+	--mount type=bind,src={{ matrix_synapse_ext_s3_storage_provider_path }}/data,dst=/data \
+	--workdir=/data \
+	--network={{ matrix_docker_network }} \
+	--entrypoint=/bin/bash \
+	{{ matrix_synapse_docker_image_final }}