Add support for automatic (nedb -> Postgres) migration to matrix-appservice-slack

roles/matrix-base/defaults/main.yml

@@ -106,7 +106,6 @@ matrix_docker_package_name: docker-ce
 run_postgres_import: true
 run_postgres_upgrade: true
 run_postgres_import_sqlite_db: true
-run_postgres_import_nedb: true
 run_postgres_vacuum: true
 run_synapse_register_user: true
 run_synapse_update_user_password: true

roles/matrix-bridge-appservice-irc/tasks/migrate_nedb_to_postgres.yml

@@ -51,3 +51,14 @@
   with_items:
     - rooms.db
     - users.db
+
+- name: Inject result
+  set_fact:
+    matrix_playbook_runtime_results: |
+      {{
+        matrix_playbook_runtime_results|default([])
+        +
+        [
+          "NOTE: Your appservice-irc database files have been imported into Postgres. The original database files have been moved from `{{ matrix_appservice_irc_data_path }}/*.db` to `{{ matrix_appservice_irc_data_path }}/*.db.backup`. When you've confirmed that the import went well and everything works, you should be able to safely delete these files."
+        ]
+      }}

roles/matrix-bridge-appservice-slack/defaults/main.yml

@@ -51,13 +51,18 @@ matrix_appservice_slack_database_password: ~
 matrix_appservice_slack_database_hostname: 'matrix-postgres'
 matrix_appservice_slack_database_port: 5432
 matrix_appservice_slack_database_name: matrix_appservice_slack
-matrix_appservice_slack_database_file: /data
+
-matrix_appservice_slack_database_connString: >-2
+# This is just the Postgres connection string, if Postgres is used.
-  {%- if matrix_appservice_slack_database_engine == 'postgres' -%}
+# Naming clashes with `matrix_appservice_slack_database_connectionString` somewhat.
-  postgresql://{{ matrix_appservice_slack_database_username }}:{{ matrix_appservice_slack_database_password }}@{{ matrix_appservice_slack_database_hostname }}:{{ matrix_appservice_slack_database_port }}/{{ matrix_appservice_slack_database_name }}?sslmode=disable
+matrix_appservice_slack_database_connection_string: 'postgresql://{{ matrix_appservice_slack_database_username }}:{{ matrix_appservice_slack_database_password }}@{{ matrix_appservice_slack_database_hostname }}:{{ matrix_appservice_slack_database_port }}/{{ matrix_appservice_slack_database_name }}?sslmode=disable'
-  {%- elif matrix_appservice_slack_database_engine == 'nedb' -%}
+
-  {{ matrix_appservice_slack_database_engine }}://{{ matrix_appservice_slack_database_file }}
+# This is what actually goes into `database.connectionString` for the bridge.
-  {%- endif -%}
+matrix_appservice_slack_database_connectionString: "{{
+	{
+		'nedb': 'nedb:///data',
+		'postgres': matrix_appservice_slack_database_connection_string,
+	}[matrix_appservice_slack_database_engine]
+}}"
 
 
 matrix_appservice_slack_configuration_yaml: "{{ lookup('template', 'templates/config.yaml.j2') }}"

roles/matrix-bridge-appservice-slack/tasks/migrate_nedb_to_postgres.yml

@@ -0,0 +1,66 @@
+- name: Fail if Postgres not enabled
+  fail:
+    msg: "Postgres via the matrix-postgres role is not enabled (`matrix_postgres_enabled`). Cannot migrate."
+  when: "not matrix_postgres_enabled|bool"
+
+# Defaults
+
+- name: Set postgres_start_wait_time, if not provided
+  set_fact:
+    postgres_start_wait_time: 15
+  when: "postgres_start_wait_time|default('') == ''"
+
+# Actual import work
+
+- name: Ensure matrix-postgres is started
+  service:
+    name: matrix-postgres
+    state: started
+    daemon_reload: yes
+  register: matrix_postgres_service_start_result
+
+- name: Wait a bit, so that Postgres can start
+  wait_for:
+    timeout: "{{ postgres_start_wait_time }}"
+  delegate_to: 127.0.0.1
+  become: false
+  when: "matrix_postgres_service_start_result.changed|bool"
+
+- name: Ensure matrix-appservice-slack is stopped
+  service:
+    name: matrix-appservice-slack
+    state: stopped
+
+- name: Import appservice-slack NeDB database into Postgres
+  command:
+    cmd: >-
+      {{ matrix_host_command_docker }} run
+      --rm
+      --user={{ matrix_user_uid }}:{{ matrix_user_gid }}
+      --cap-drop=ALL
+      --network={{ matrix_docker_network }}
+      --mount type=bind,src={{ matrix_appservice_slack_data_path }},dst=/data
+      --entrypoint=/bin/sh
+      {{ matrix_appservice_slack_docker_image }}
+      -c
+      '/usr/local/bin/node /usr/src/app/lib/scripts/migrateToPostgres.js --dbdir /data --connectionString {{ matrix_appservice_slack_database_connection_string }}'
+
+- name: Archive NeDB database files
+  command:
+    cmd: "mv {{ matrix_appservice_slack_data_path }}/{{ item }} {{ matrix_appservice_slack_data_path }}/{{ item }}.backup"
+  with_items:
+    - teams.db
+    - room-store.db
+    - user-store.db
+    - event-store.db
+
+- name: Inject result
+  set_fact:
+    matrix_playbook_runtime_results: |
+      {{
+        matrix_playbook_runtime_results|default([])
+        +
+        [
+          "NOTE: Your appservice-slack database files have been imported into Postgres. The original database files have been moved from `{{ matrix_appservice_slack_data_path }}/*.db` to `{{ matrix_appservice_slack_data_path }}/*.db.backup`. When you've confirmed that the import went well and everything works, you should be able to safely delete these files."
+        ]
+      }}

roles/matrix-bridge-appservice-slack/tasks/setup_install.yml

@@ -1,12 +1,5 @@
 ---
 
-- name: Ensure Appservice Slack image is pulled
-  docker_image:
-    name: "{{ matrix_appservice_slack_docker_image }}"
-    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
-    force_source: "{{ matrix_appservice_slack_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
-    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_appservice_slack_docker_image_force_pull }}"
-
 - name: Ensure AppService Slack paths exist
   file:
     path: "{{ item }}"
@@ -19,6 +12,30 @@
     - "{{ matrix_appservice_slack_config_path }}"
     - "{{ matrix_appservice_slack_data_path }}"
 
+- set_fact:
+    matrix_appservice_slack_requires_restart: false
+
+- block:
+    - name: Check if a nedb database already exists
+      stat:
+        path: "{{ matrix_appservice_slack_data_path }}/teams.db"
+      register: matrix_appservice_slack_nedb_database_path_local_stat_result
+
+    - block:
+        - import_tasks: "{{ role_path }}/tasks/migrate_nedb_to_postgres.yml"
+
+        - set_fact:
+            matrix_appservice_slack_requires_restart: true
+      when: "matrix_appservice_slack_nedb_database_path_local_stat_result.stat.exists|bool"
+  when: "matrix_appservice_slack_database_engine == 'postgres'"
+
+- name: Ensure Appservice Slack image is pulled
+  docker_image:
+    name: "{{ matrix_appservice_slack_docker_image }}"
+    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
+    force_source: "{{ matrix_appservice_slack_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
+    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_appservice_slack_docker_image_force_pull }}"
+
 - name: Ensure Matrix Appservice Slack config installed
   copy:
     content: "{{ matrix_appservice_slack_configuration|to_nice_yaml }}"
@@ -46,3 +63,9 @@
   service:
     daemon_reload: yes
   when: "matrix_appservice_slack_systemd_service_result.changed"
+
+- name: Ensure matrix-appservice-slack.service restarted, if necessary
+  service:
+    name: "matrix-appservice-slack.service"
+    state: restarted
+  when: "matrix_appservice_slack_requires_restart|bool"

roles/matrix-bridge-appservice-slack/tasks/validate_config.yml

@@ -10,23 +10,3 @@
     - "matrix_appservice_slack_appservice_token"
     - "matrix_appservice_slack_homeserver_token"
     - "matrix_appservice_slack_id_token"
-
-- block:
-  - name: Check if a neDB database already exists
-    stat:
-      path: "{{ matrix_appservice_slack_data_path }}/"
-    register: matrix_appservice_slack_nedb_stat_result
-
-  - name: Fail if an neDB database already exists when using Postgres
-    fail:
-      msg: >-2
-        matrix_appservice_slack_database_engine has been set to `postgres` (which is our new default now).
-        However, we've discovered an existing neDB database in {{ matrix_appservice_slack_data_path }}/.
-        It appears that you've been using this bridge with the neDB engine until now.
-        To continue using neDB, opt into it explicitly: add `matrix_appservice_slack_database_engine: nedb` to your vars.yml file and re-run this same command.
-        Alternatively, to migrate your existing neDB database to Postgres:
-        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
-        2. Import the neDB database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_appservice_slack_data_path }} postgres_connection_string_variable_name=matrix_appservice_slack_database_connString'`)
-        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
-    when: "matrix_appservice_slack_nedb_stat_result.stat.exists"
-  when: "matrix_appservice_slack_database_engine == 'postgres'"

roles/matrix-bridge-appservice-slack/templates/config.yaml.j2

@@ -11,9 +11,10 @@ homeserver:
 
 {% if matrix_appservice_slack_database_engine == 'nedb' %}
 dbdir: "/data"
-{% endif %}
+{% else %}
 db:
-    engine: "{{ matrix_appservice_slack_database_engine }}"
+    engine: {{ matrix_appservice_slack_database_engine|to_json }}
-    connectionString: {{ matrix_appservice_slack_database_connString | to_json }}
+    connectionString: {{ matrix_appservice_slack_database_connectionString|to_json }}
+{% endif %}
 
 matrix_admin_room: "{{ matrix_appservice_slack_control_room_id }}"

roles/matrix-postgres/tasks/import_nedb.yml

@@ -1,98 +0,0 @@
----
-
-# Pre-checks
-
-- name: Fail if Postgres not enabled
-  fail:
-    msg: "Postgres via the matrix-postgres role is not enabled (`matrix_postgres_enabled`). Cannot import."
-  when: "not matrix_postgres_enabled|bool"
-
-- name: Fail if playbook called incorrectly
-  fail:
-    msg: "The `nedb_database_path` variable needs to be provided to this playbook, via --extra-vars"
-  when: "nedb_database_path is not defined or nedb_database_path.startswith('<')"
-
-- name: Check if the provided nedb database file exists
-  stat:
-    path: "{{ nedb_database_path }}"
-  register: nedb_database_path_stat_result
-
-- name: Fail if provided SQLite database file doesn't exist
-  fail:
-    msg: "File cannot be found on the server at {{ nedb_database_path }}"
-  when: "not nedb_database_path_stat_result.stat.exists"
-
-# We either expect `postgres_db_connection_string` specifying a full Postgres database connection string,
-# or `postgres_connection_string_variable_name`, specifying a name of a variable, which contains a valid connection string.
-
-- block:
-    - name: Fail if postgres_connection_string_variable_name points to an undefined variable
-      fail: msg="postgres_connection_string_variable_name is defined, but there is no variable with the name `{{ postgres_connection_string_variable_name }}`"
-      when: "postgres_connection_string_variable_name not in vars"
-
-    - name: Get Postgres connection string from variable
-      set_fact:
-        postgres_db_connection_string: "{{ lookup('vars', postgres_connection_string_variable_name) }}"
-  when: 'postgres_connection_string_variable_name is defined'
-
-- name: Fail if playbook called incorrectly
-  fail:
-    msg: >-
-      Either a `postgres_db_connection_string` variable or a `postgres_connection_string_variable_name` needs to be provided to this playbook, via `--extra-vars`.
-      Example: `--extra-vars="postgres_db_connection_string=postgresql://username:password@localhost:<port>/database_name"` or `--extra-vars="postgres_connection_string_variable_name=matrix_appservice_discord_database_connString"`
-  when: "postgres_db_connection_string is not defined or not postgres_db_connection_string.startswith('postgresql://')"
-
-
-# Defaults
-
-- name: Set postgres_start_wait_time, if not provided
-  set_fact:
-    postgres_start_wait_time: 15
-  when: "postgres_start_wait_time|default('') == ''"
-
-
-# Actual import work
-
-- name: Ensure matrix-postgres is started
-  service:
-    name: matrix-postgres
-    state: started
-    daemon_reload: yes
-  register: matrix_postgres_service_start_result
-
-- name: Wait a bit, so that Postgres can start
-  wait_for:
-    timeout: "{{ postgres_start_wait_time }}"
-  delegate_to: 127.0.0.1
-  become: false
-  when: "matrix_postgres_service_start_result.changed|bool"
-
-# No migration.sh available, but found this:
-# https://github.com/matrix-org/matrix-appservice-slack/blob/develop/src/scripts/migrateToPostgres.ts
-# Usage should be similar to appservice_irc
-- name: Import appservice_slack NeDB database from {{ sqlite_database_path }} into Postgres
-  when: database == 'appservice_slack'
-  command:
-    cmd: >-
-      {{ matrix_host_command_docker }} run
-      --rm
-      --user={{ matrix_user_uid }}:{{ matrix_user_gid }}
-      --cap-drop=ALL
-      --network={{ matrix_docker_network }}
-      --mount type=bind,src={{ matrix_appservice_irc_data_path }},dst=/data,ro
-      --entrypoint=/bin/sh
-      {{ matrix_appservice_slack_docker_image }}
-      -c
-      'node /lib/scripts/migrate-db-to-pgres.js -d /data -p passkey.pem -c {{ postgres_db_connection_string }}'
-
-
-- name: Inject result
-  set_fact:
-    matrix_playbook_runtime_results: |
-      {{
-        matrix_playbook_runtime_results|default([])
-        +
-        [
-          "NOTE: Your NeDB database file has been imported into Postgres. The original directory has been moved from `{{ nedb_database_path }}` to `{{ nedb_database_path }}.backup`. When you've confirmed that the import went well and everything works, you should be able to safely delete this file."
-        ]
-      }}

roles/matrix-postgres/tasks/main.yml

@@ -32,14 +32,6 @@
   tags:
     - import-generic-sqlite-db
 
-# Imports slacks neDB to postgres.
-- import_tasks: "{{ role_path }}/tasks/import_nedb.yml"
-  vars:
-    database: appservice_slack
-  when: run_postgres_import_nedb|bool
-  tags:
-    - import-slack-nedb
-
 - import_tasks: "{{ role_path }}/tasks/upgrade_postgres.yml"
   when: run_postgres_upgrade|bool
   tags: