Add (SQLite + Postgres) support and automatic migration to matrix-ma1sd

group_vars/matrix_servers

@@ -835,6 +835,10 @@ matrix_ma1sd_systemd_wanted_services_list: |
     (['matrix-mailer.service'] if matrix_mailer_enabled else [])
   }}
 
+# Postgres is the default, except if not using `matrix_postgres` (internal postgres)
+matrix_ma1sd_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
+matrix_ma1sd_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'ma1sd.db') | to_uuid }}"
+
 ######################################################################
 #
 # /matrix-ma1sd
@@ -953,6 +957,12 @@ matrix_postgres_db_name: "homeserver"
 
 matrix_postgres_additional_databases: |
   {{
+    ([{
+      'name': matrix_ma1sd_database_db_name,
+      'username': matrix_ma1sd_database_username,
+      'password': matrix_ma1sd_database_password,
+    }] if (matrix_ma1sd_enabled and matrix_ma1sd_database_engine == 'postgres' and matrix_ma1sd_database_hostname == 'matrix-postgres') else [])
+    +
     ([{
       'name': matrix_bot_matrix_reminder_bot_database_db_name,
       'username': matrix_bot_matrix_reminder_bot_database_username,

roles/matrix-ma1sd/defaults/main.yml

@@ -39,6 +39,28 @@ matrix_ma1sd_systemd_wanted_services_list: []
 # Enabling this is discouraged. Learn more here: https://github.com/ma1uta/ma1sd/blob/master/docs/features/identity.md#lookups
 matrix_ma1sd_matrixorg_forwarding_enabled: false
 
+
+# Database-related configuration fields.
+#
+# To use SQLite, stick to these defaults.
+#
+# To use Postgres:
+# - change the engine (`matrix_ma1sd_database_engine: 'postgres'`)
+# - adjust your database credentials via the `matrix_ma1sd_postgres_*` variables
+matrix_ma1sd_database_engine: 'sqlite'
+
+matrix_ma1sd_sqlite_database_path_local: "{{ matrix_ma1sd_data_path }}/ma1sd.db"
+matrix_ma1sd_sqlite_database_path_in_container: "/var/ma1sd/ma1sd.db"
+
+matrix_ma1sd_database_username: 'matrix_ma1sd'
+matrix_ma1sd_database_password: 'some-password'
+matrix_ma1sd_database_hostname: 'matrix-postgres'
+matrix_ma1sd_database_port: 5432
+matrix_ma1sd_database_db_name: 'matrix_ma1sd'
+
+matrix_ma1sd_database_connection_string: 'postgresql://{{ matrix_ma1sd_database_username }}:{{ matrix_ma1sd_database_password }}@{{ matrix_ma1sd_database_hostname }}:{{ matrix_ma1sd_database_port }}/{{ matrix_ma1sd_database_db_name }}'
+
+
 # ma1sd has serveral supported identity stores.
 # One of them is storing identities directly in Synapse's database.
 # Learn more here: https://github.com/ma1uta/ma1sd/blob/master/docs/stores/synapse.md

roles/matrix-ma1sd/tasks/setup_install.yml

@@ -15,6 +15,36 @@
 
 - import_tasks: "{{ role_path }}/tasks/migrate_mxisd.yml"
 
+
+# These (SQLite -> Postgres) migration tasks are usually at the top,
+# but we'd like to run them after `migrate_mxisd.yml`, which requires the ma1sd paths to exist.
+- set_fact:
+    matrix_ma1sd_requires_restart: false
+
+- block:
+    - name: Check if an SQLite database already exists
+      stat:
+        path: "{{ matrix_ma1sd_sqlite_database_path_local }}"
+      register: matrix_ma1sd_sqlite_database_path_local_stat_result
+
+    - block:
+        - set_fact:
+            matrix_postgres_db_migration_request:
+              src: "{{ matrix_ma1sd_sqlite_database_path_local }}"
+              dst: "{{ matrix_ma1sd_database_connection_string }}"
+              caller: "{{ role_path|basename }}"
+              engine_variable_name: 'matrix_ma1sd_database_engine'
+              engine_old: 'sqlite'
+              systemd_services_to_stop: ['matrix-ma1sd.service']
+              pgloader_options: ['--with "quote identifiers"']
+
+        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+
+        - set_fact:
+            matrix_ma1sd_requires_restart: true
+      when: "matrix_ma1sd_sqlite_database_path_local_stat_result.stat.exists|bool"
+  when: "matrix_ma1sd_database_engine == 'postgres'"
+
 - name: Ensure ma1sd image is pulled
   docker_image:
     name: "{{ matrix_ma1sd_docker_image }}"
@@ -102,3 +132,9 @@
   service:
     daemon_reload: yes
   when: "matrix_ma1sd_systemd_service_result.changed|bool"
+
+- name: Ensure matrix-ma1sd.service restarted, if necessary
+  service:
+    name: "matrix-ma1sd.service"
+    state: restarted
+  when: "matrix_ma1sd_requires_restart|bool"

roles/matrix-ma1sd/templates/ma1sd.yaml.j2

@@ -11,9 +11,19 @@ key:
   path: /var/ma1sd/sign.key
 
 storage:
-  provider:
+  {% if matrix_ma1sd_database_engine == 'sqlite' %}
-    sqlite:
+    backend: sqlite
-      database: /var/ma1sd/ma1sd.db
+    provider:
+      sqlite:
+        database: {{ matrix_ma1sd_sqlite_database_path_in_container|to_json }}
+  {% elif matrix_ma1sd_database_engine == 'postgres' %}
+    backend: postgresql
+    provider:
+      postgresql:
+        database: //{{ matrix_ma1sd_database_hostname }}:{{ matrix_ma1sd_database_port }}/{{ matrix_ma1sd_database_db_name }}
+        username: {{ matrix_ma1sd_database_username|to_json }}
+        password: {{ matrix_ma1sd_database_password|to_json }}
+  {% endif %}
 
 {% if matrix_ma1sd_dns_overwrite_enabled %}
 dns:

roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml

@@ -93,7 +93,7 @@
       --entrypoint=/bin/sh
       {{ matrix_postgres_pgloader_docker_image }}
       -c
-      'pgloader /in.db {{ matrix_postgres_db_migration_request.dst }}'
+      'pgloader {{ matrix_postgres_db_migration_request.pgloader_options|default([])|join(' ') }} /in.db {{ matrix_postgres_db_migration_request.dst }}'
 
 - name: Archive {{ matrix_postgres_db_migration_request.engine_old }} database ({{ matrix_postgres_db_migration_request.src }} -> {{ matrix_postgres_db_migration_request.src }}.backup)
   command: