From da039614fdb0b7d6d32f8f05ba1bdc532d54de59 Mon Sep 17 00:00:00 2001 From: axiopaladin Date: Wed, 13 Dec 2023 22:36:43 -0500 Subject: [PATCH 1/4] updated MMR yaml defaults and docs for basic dendrite support --- docs/configuring-playbook-matrix-media-repo.md | 4 ++-- roles/custom/matrix-media-repo/defaults/main.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/configuring-playbook-matrix-media-repo.md b/docs/configuring-playbook-matrix-media-repo.md index d5d6eda37..82fab6402 100644 --- a/docs/configuring-playbook-matrix-media-repo.md +++ b/docs/configuring-playbook-matrix-media-repo.md @@ -1,6 +1,6 @@ # Setting up matrix-media-repo (optional) -[matrix-media-repo](https://docs.t2bot.io/matrix-media-repo/) is a highly customizable multi-domain media repository for Matrix. Intended for medium to large environments consisting of several homeservers, this media repo de-duplicates media (including remote media) while being fully compliant with the specification. +[matrix-media-repo](https://docs.t2bot.io/matrix-media-repo/) (often abbreviated "MMR") is a highly customizable multi-domain media repository for Matrix. Intended for medium to large environments consisting of several homeservers, this media repo de-duplicates media (including remote media) while being fully compliant with the specification. Smaller/individual homeservers can still make use of this project's features, though it may be difficult to set up or have higher than expected resource consumption. Please do your research before deploying this as this project may not be useful for your environment. @@ -102,5 +102,5 @@ matrix_media_repo_datastores: ``` -Full list of configuration options with documentation can be found in `roles/custom/matrix-media-repo/templates/defaults/main.yml` +Full list of configuration options with documentation can be found in [`roles/custom/matrix-media-repo/defaults/main.yml`](https://github.com/spantaleev/matrix-docker-ansible-deploy/blob/master/roles/custom/matrix-media-repo/defaults/main.yml) diff --git a/roles/custom/matrix-media-repo/defaults/main.yml b/roles/custom/matrix-media-repo/defaults/main.yml index 026de1501..c36336e58 100644 --- a/roles/custom/matrix-media-repo/defaults/main.yml +++ b/roles/custom/matrix-media-repo/defaults/main.yml @@ -166,7 +166,7 @@ matrix_media_repo_homeservers_auto: # during certain operations, like attempting to purge media from a room or validating server # admin status. This should be set to one of "synapse", "dendrite", or "matrix". When set # to "matrix", most functionality requiring the admin API will not work. - adminApiKind: "{{ 'synapse' if matrix_homeserver_implementation == 'synapse' else 'matrix' }}" + adminApiKind: "{{ 'synapse' if matrix_homeserver_implementation == 'synapse' else 'dendrite' if matrix_homeserver_implementation == 'dendrite' else 'matrix' }}" # Additional servers to be managed by MMR matrix_media_repo_homeservers_additional: [] From 45bcb026dd87539d71353e9c2c6098fd78e4153a Mon Sep 17 00:00:00 2001 From: axiopaladin Date: Wed, 13 Dec 2023 22:48:19 -0500 Subject: [PATCH 2/4] updated MMR docs with instructions for importing existing media --- .../configuring-playbook-matrix-media-repo.md | 32 +++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/docs/configuring-playbook-matrix-media-repo.md b/docs/configuring-playbook-matrix-media-repo.md index 82fab6402..12decd960 100644 --- a/docs/configuring-playbook-matrix-media-repo.md +++ b/docs/configuring-playbook-matrix-media-repo.md @@ -6,6 +6,12 @@ Smaller/individual homeservers can still make use of this project's features, th For a simpler alternative (which allows you to offload your media repository storage to S3, etc.), you can [configure S3 storage](configuring-playbook-s3.md) instead of setting up matrix-media-repo. +| **Table of Contents** | +| :------------------------------------------------------------------------------------------ | +| [Quickstart](#quickstart) | +| [Additional configuration options](#configuring-the-media-repo) | +| [Importing data from an existing media store](#importing-data-from-an-existing-media-store) | + ## Quickstart Add the following configuration to your `inventory/host_vars/matrix.DOMAIN/vars.yml` file: @@ -37,8 +43,9 @@ matrix_media_repo_database_max_connections: 25 matrix_media_repo_database_max_idle_connections: 5 # These users have full access to the administrative functions of the media repository. -# See https://github.com/turt2live/matrix-media-repo/blob/release-v1.2.8/docs/admin.md for information on what these people can do. They must belong to one of the -# configured homeservers above. +# See https://github.com/turt2live/matrix-media-repo/blob/release-v1.2.8/docs/admin.md for +# information on what these people can do. They must belong to one of the configured +# homeservers above. matrix_media_repo_admins: admins: [] # admins: @@ -104,3 +111,24 @@ matrix_media_repo_datastores: Full list of configuration options with documentation can be found in [`roles/custom/matrix-media-repo/defaults/main.yml`](https://github.com/spantaleev/matrix-docker-ansible-deploy/blob/master/roles/custom/matrix-media-repo/defaults/main.yml) +## Importing data from an existing media store + +If you want to add this repo to an existing homeserver managed by the playbook, you will need to import existing media into MMR's database or you will lose access to older media while it is active. MMR versions up to `v1.3.3` only support importing from Synapse, but newer versions (at time of writing: only `latest`) also support importing from Dendrite. + +1. Edit your vars.yml like in [quickstart](#quickstart) above and deploy with `just setup-all` +2. SSH into the homeserver and enter the Postgres command line interface: `/matrix/postgres/bin/cli`. +3. At that psql prompt, use `\l` to list all databases. On Dendrite, the one you're looking for is called `dendrite_mediaapi`; other homeserver software will have similar but slightly different names. Note down the correct name for your system and exit the prompt (use `\q`). +4. The MMR docker container includes an import utility explained [in its own docs here](https://github.com/turt2live/matrix-media-repo#importing-media-from-synapse). To invoke this tool, use the following command: + +```bash +docker exec -it matrix-media-repo /usr/local/bin/import_dendrite `# Synapse: import_synapse` \ + -dbName dendrite_mediaapi `# This is the database found in psql above` \ + -dbHost matrix-postgres \ + -dbPort 5432 \ + -dbUsername matrix \ + -dbPassword devture_postgres_connection_password `# Replace with the value from your vars.yml` +``` + +Enter `1` for the Machine ID when prompted (you are not doing any horizontal scaling) unless you know what you're doing. + +This should output a `msg="Import completed"` when finished successfully! From abd0823554bc652fb2900d8b37a36512d26f3487 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 14 Dec 2023 09:09:10 +0200 Subject: [PATCH 3/4] Add some parenthesis in if statement --- roles/custom/matrix-media-repo/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/custom/matrix-media-repo/defaults/main.yml b/roles/custom/matrix-media-repo/defaults/main.yml index c36336e58..8aada0896 100644 --- a/roles/custom/matrix-media-repo/defaults/main.yml +++ b/roles/custom/matrix-media-repo/defaults/main.yml @@ -166,7 +166,7 @@ matrix_media_repo_homeservers_auto: # during certain operations, like attempting to purge media from a room or validating server # admin status. This should be set to one of "synapse", "dendrite", or "matrix". When set # to "matrix", most functionality requiring the admin API will not work. - adminApiKind: "{{ 'synapse' if matrix_homeserver_implementation == 'synapse' else 'dendrite' if matrix_homeserver_implementation == 'dendrite' else 'matrix' }}" + adminApiKind: "{{ 'synapse' if matrix_homeserver_implementation == 'synapse' else ('dendrite' if matrix_homeserver_implementation == 'dendrite' else 'matrix') }}" # Additional servers to be managed by MMR matrix_media_repo_homeservers_additional: [] From 03fb357c81465db6dc0af9545cdb238d784c88cc Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 14 Dec 2023 09:26:14 +0200 Subject: [PATCH 4/4] Split Synapse and Dendrite importing instructions for MMR --- .../configuring-playbook-matrix-media-repo.md | 54 ++++++++++++++----- 1 file changed, 42 insertions(+), 12 deletions(-) diff --git a/docs/configuring-playbook-matrix-media-repo.md b/docs/configuring-playbook-matrix-media-repo.md index 12decd960..7dc011bb8 100644 --- a/docs/configuring-playbook-matrix-media-repo.md +++ b/docs/configuring-playbook-matrix-media-repo.md @@ -14,7 +14,7 @@ For a simpler alternative (which allows you to offload your media repository sto ## Quickstart -Add the following configuration to your `inventory/host_vars/matrix.DOMAIN/vars.yml` file: +Add the following configuration to your `inventory/host_vars/matrix.DOMAIN/vars.yml` file and [re-run the installation process](./installing.md) for the playbook: ```yaml matrix_media_repo_enabled: true @@ -115,18 +115,48 @@ Full list of configuration options with documentation can be found in [`roles/cu If you want to add this repo to an existing homeserver managed by the playbook, you will need to import existing media into MMR's database or you will lose access to older media while it is active. MMR versions up to `v1.3.3` only support importing from Synapse, but newer versions (at time of writing: only `latest`) also support importing from Dendrite. -1. Edit your vars.yml like in [quickstart](#quickstart) above and deploy with `just setup-all` -2. SSH into the homeserver and enter the Postgres command line interface: `/matrix/postgres/bin/cli`. -3. At that psql prompt, use `\l` to list all databases. On Dendrite, the one you're looking for is called `dendrite_mediaapi`; other homeserver software will have similar but slightly different names. Note down the correct name for your system and exit the prompt (use `\q`). -4. The MMR docker container includes an import utility explained [in its own docs here](https://github.com/turt2live/matrix-media-repo#importing-media-from-synapse). To invoke this tool, use the following command: +**Before importing**: ensure you have an initial matrix-media-repo deployment by following the [quickstart](#quickstart) guide above -```bash -docker exec -it matrix-media-repo /usr/local/bin/import_dendrite `# Synapse: import_synapse` \ - -dbName dendrite_mediaapi `# This is the database found in psql above` \ - -dbHost matrix-postgres \ - -dbPort 5432 \ - -dbUsername matrix \ - -dbPassword devture_postgres_connection_password `# Replace with the value from your vars.yml` +Depending on the homeserver implementation yu're using (Synapse, Dendrite), you'll need to use a different import tool (part of matrix-media-repo) and point it to the homeserver's database. + +### Importing data from the Synapse media store + +To import the Synapse media store, you're supposed to invoke the `import_synapse` tool which is part of the matrix-media-repo container image. Your Synapse database is called `synapse` by default, unless you've changed it by modifying `matrix_synapse_database_database`. + +This guide here is adapted from the [upstream documentation about the import_synapse script](https://github.com/turt2live/matrix-media-repo#importing-media-from-synapse). + +Run the following command on the server (after replacing `devture_postgres_connection_password` in it with the value found in your `vars.yml` file): + +```sh +docker exec -it matrix-media-repo \ + /usr/local/bin/import_synapse \ + -dbName synapse \ + -dbHost matrix-postgres \ + -dbPort 5432 \ + -dbUsername matrix \ + -dbPassword devture_postgres_connection_password +``` + +Enter `1` for the Machine ID when prompted (you are not doing any horizontal scaling) unless you know what you're doing. + +This should output a `msg="Import completed"` when finished successfully! + +### Importing data from the Dendrite media store + +If you're using the [Dendrite](configuring-playbook-dendrite.md) homeserver instead of the default for this playbook (Synapse), follow this importing guide here. + +To import the Dendrite media store, you're supposed to invoke the `import_dendrite` tool which is part of the matrix-media-repo container image. Your Dendrite database is called `dendrite_mediaapi` by default, unless you've changed it by modifying `matrix_dendrite_media_api_database`. + +Run the following command on the server (after replacing `devture_postgres_connection_password` in it with the value found in your `vars.yml` file): + +```sh +docker exec -it matrix-media-repo \ + /usr/local/bin/import_dendrite \ + -dbName dendrite_mediaapi \ + -dbHost matrix-postgres \ + -dbPort 5432 \ + -dbUsername matrix \ + -dbPassword devture_postgres_connection_password ``` Enter `1` for the Machine ID when prompted (you are not doing any horizontal scaling) unless you know what you're doing.