From 881fdd28f0bf2e10635ac7a6c3941fa8636cdbff Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 14 Oct 2022 16:33:19 +0300 Subject: [PATCH] Add support for Synapse container image customization This allows people to augment the Synapse image with custom tools and addons without having to rebuild it from scratch. If customizations are enabled, the playbook will build a new `localhost/matrixdotorg/synapse:VERSION-customized` image on top of the default one (`FROM matrixdotorg/synapse:VERSION`) and with custom Dockerfile build steps. For servers that self-build the Synapse image, the Synapse image will be built first, before proceding to extend it the same way. In the future, we'll also have easy to enable Dockerfile build steps for modules that the playbook supports. --- .../tasks/import_synapse_sqlite_db.yml | 2 +- roles/matrix-synapse/defaults/main.yml | 33 +++++++++++++++++++ roles/matrix-synapse/tasks/setup_synapse.yml | 1 + .../tasks/synapse/setup_install.yml | 19 +++++++++++ .../tasks/synapse/setup_uninstall.yml | 5 ++- .../synapse/customizations/Dockerfile.j2 | 3 ++ .../systemd/matrix-synapse-worker.service.j2 | 2 +- .../synapse/systemd/matrix-synapse.service.j2 | 2 +- 8 files changed, 63 insertions(+), 4 deletions(-) create mode 100644 roles/matrix-synapse/templates/synapse/customizations/Dockerfile.j2 diff --git a/roles/matrix-postgres/tasks/import_synapse_sqlite_db.yml b/roles/matrix-postgres/tasks/import_synapse_sqlite_db.yml index a459b6e1d..b885ea975 100644 --- a/roles/matrix-postgres/tasks/import_synapse_sqlite_db.yml +++ b/roles/matrix-postgres/tasks/import_synapse_sqlite_db.yml @@ -83,7 +83,7 @@ --mount type=bind,src={{ matrix_synapse_config_dir_path }},dst=/data --mount type=bind,src={{ matrix_synapse_config_dir_path }},dst=/matrix-media-store-parent/media-store --mount type=bind,src={{ server_path_homeserver_db }},dst=/{{ server_path_homeserver_db | basename }} - {{ matrix_synapse_docker_image }} + {{ matrix_synapse_docker_image_final }} /usr/local/bin/synapse_port_db --sqlite-database /{{ server_path_homeserver_db | basename }} --postgres-config /data/homeserver.yaml register: matrix_postgres_import_synapse_sqlite_db_result changed_when: matrix_postgres_import_synapse_sqlite_db_result.rc == 0 diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index d117eaafa..40e05be72 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -7,14 +7,47 @@ matrix_synapse_enabled: true matrix_synapse_container_image_self_build: false matrix_synapse_container_image_self_build_repo: "https://github.com/matrix-org/synapse.git" +# matrix_synapse_container_image_customizations_enabled controls whether a customized Synapse image will be built. +# +# We toggle this variable to `true` when certain features which require a custom build are enabled. +# Feel free to toggle this to `true` yourself and specify build steps in `matrix_synapse_container_image_customizations_dockerfile_body_custom`. +# +# See: +# - `roles/matrix-synapse/templates/synapse/customizations/Dockerfile.j2` +# - `matrix_synapse_container_image_customizations_dockerfile_body_custom` +# - `matrix_synapse_docker_image_customized` +# - `matrix_synapse_docker_image_final` +matrix_synapse_container_image_customizations_enabled: "{{ matrix_synapse_ext_synapse_s3_storage_provider_enabled }}" + +# matrix_synapse_container_image_customizations_dockerfile_body contains your custom Dockerfile steps +# for building your customized Synapse image based on the original (upstream) image (`matrix_synapse_docker_image`). +# A `FROM ...` clause is included automatically so you don't have to. +# +# Example: +# matrix_synapse_container_image_customizations_dockerfile_body_custom: | +# RUN echo 'This is a custom step for building the customized Docker image for Synapse.' +# RUN echo 'You can override matrix_synapse_container_image_customizations_dockerfile_body_custom to add your own steps.' +# RUN echo 'Note that matrix_synapse_container_image_customizations_dockerfile_body_auto injects steps before this.' +# RUN echo 'You do NOT need to include a FROM clause yourself.' +matrix_synapse_container_image_customizations_dockerfile_body_custom: '' + matrix_synapse_docker_image: "{{ matrix_synapse_docker_image_name_prefix }}matrixdotorg/synapse:{{ matrix_synapse_docker_image_tag }}" matrix_synapse_docker_image_name_prefix: "{{ 'localhost/' if matrix_synapse_container_image_self_build else matrix_container_global_registry_prefix }}" matrix_synapse_version: v1.68.0 matrix_synapse_docker_image_tag: "{{ matrix_synapse_version }}" matrix_synapse_docker_image_force_pull: "{{ matrix_synapse_docker_image.endswith(':latest') }}" +# matrix_synapse_docker_image_customized is the name of the locally built Synapse image +# which adds various customizations on top of the original (upstream) Synapse image. +# This image will be based on the upstream `matrix_synapse_docker_image` image, only if `matrix_synapse_container_image_customizations_enabled: true`. +matrix_synapse_docker_image_customized: "localhost/matrixdotorg/synapse:{{ matrix_synapse_docker_image_tag }}-customized" + +# matrix_synapse_docker_image_final holds the name of the Synapse image to run depending on whether or not customizations are enabled. +matrix_synapse_docker_image_final: "{{ matrix_synapse_docker_image_customized if matrix_synapse_container_image_customizations_enabled else matrix_synapse_docker_image }} " + matrix_synapse_base_path: "{{ matrix_base_data_path }}/synapse" matrix_synapse_docker_src_files_path: "{{ matrix_synapse_base_path }}/docker-src" +matrix_synapse_customized_docker_src_files_path: "{{ matrix_synapse_base_path }}/customized-docker-src" matrix_synapse_config_dir_path: "{{ matrix_synapse_base_path }}/config" matrix_synapse_storage_path: "{{ matrix_synapse_base_path }}/storage" matrix_synapse_media_store_path: "{{ matrix_synapse_storage_path }}/media-store" diff --git a/roles/matrix-synapse/tasks/setup_synapse.yml b/roles/matrix-synapse/tasks/setup_synapse.yml index d4e6ae95a..7b887f30f 100644 --- a/roles/matrix-synapse/tasks/setup_synapse.yml +++ b/roles/matrix-synapse/tasks/setup_synapse.yml @@ -11,6 +11,7 @@ - {path: "{{ matrix_synapse_config_dir_path }}", when: true} - {path: "{{ matrix_synapse_ext_path }}", when: true} - {path: "{{ matrix_synapse_docker_src_files_path }}", when: "{{ matrix_synapse_container_image_self_build }}"} + - {path: "{{ matrix_synapse_customized_docker_src_files_path }}", when: "{{ matrix_synapse_container_image_customizations_enabled }}"} # We handle matrix_synapse_media_store_path elsewhere (in ./synapse/setup_install.yml), # because if it's using Goofys and it's already mounted (from before), # trying to chown/chmod it here will cause trouble. diff --git a/roles/matrix-synapse/tasks/synapse/setup_install.yml b/roles/matrix-synapse/tasks/synapse/setup_install.yml index aea03f059..4d0e749f9 100644 --- a/roles/matrix-synapse/tasks/synapse/setup_install.yml +++ b/roles/matrix-synapse/tasks/synapse/setup_install.yml @@ -62,6 +62,25 @@ delay: "{{ matrix_container_retries_delay }}" until: result is not failed +- when: "matrix_synapse_container_image_customizations_enabled | bool" + block: + - name: Ensure customizations Dockerfile is created + ansible.builtin.template: + src: "{{ role_path }}/templates/synapse/customizations/Dockerfile.j2" + dest: "{{ matrix_synapse_customized_docker_src_files_path }}/Dockerfile" + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_groupname }}" + mode: 0640 + + - name: Ensure customized Docker image for Synapse is built + docker_image: + name: "{{ matrix_synapse_docker_image_customized }}" + source: build + build: + dockerfile: Dockerfile + path: "{{ matrix_synapse_customized_docker_src_files_path }}" + pull: true + - name: Check if a Synapse signing key exists ansible.builtin.stat: path: "{{ matrix_synapse_config_dir_path }}/{{ matrix_server_fqn_matrix }}.signing.key" diff --git a/roles/matrix-synapse/tasks/synapse/setup_uninstall.yml b/roles/matrix-synapse/tasks/synapse/setup_uninstall.yml index 17b1b8c45..06e550145 100644 --- a/roles/matrix-synapse/tasks/synapse/setup_uninstall.yml +++ b/roles/matrix-synapse/tasks/synapse/setup_uninstall.yml @@ -27,8 +27,11 @@ - name: Ensure Synapse Docker image doesn't exist docker_image: - name: "{{ matrix_synapse_docker_image }}" + name: "{{ item }}" state: absent + with_items: + - "{{ matrix_synapse_docker_image_final }}" + - "{{ matrix_synapse_docker_image }}" - name: Ensure sample prometheus.yml for external scraping is deleted ansible.builtin.file: diff --git a/roles/matrix-synapse/templates/synapse/customizations/Dockerfile.j2 b/roles/matrix-synapse/templates/synapse/customizations/Dockerfile.j2 new file mode 100644 index 000000000..7cce2086d --- /dev/null +++ b/roles/matrix-synapse/templates/synapse/customizations/Dockerfile.j2 @@ -0,0 +1,3 @@ +FROM {{ matrix_synapse_docker_image }} + +{{ matrix_synapse_container_image_customizations_dockerfile_body_custom }} diff --git a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker.service.j2 b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker.service.j2 index 96b65a0ae..3855b850c 100644 --- a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker.service.j2 +++ b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker.service.j2 @@ -42,7 +42,7 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name {{ matrix_synapse_wor {% for arg in matrix_synapse_container_arguments %} {{ arg }} \ {% endfor %} - {{ matrix_synapse_docker_image }} \ + {{ matrix_synapse_docker_image_final }} \ run -m synapse.app.{{ matrix_synapse_worker_details.app }} -c /data/homeserver.yaml -c /data/{{ matrix_synapse_worker_config_file_name }} diff --git a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 index eed50ad1e..f41cc54c8 100644 --- a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 +++ b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 @@ -60,7 +60,7 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-synapse \ {% for arg in matrix_synapse_container_arguments %} {{ arg }} \ {% endfor %} - {{ matrix_synapse_docker_image }} \ + {{ matrix_synapse_docker_image_final }} \ run -m synapse.app.homeserver -c /data/homeserver.yaml ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} kill matrix-synapse 2>/dev/null || true'