From 69cc2145d2c710d3874fe2ec12315266561e10a1 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 22 Dec 2020 16:51:59 +0200 Subject: [PATCH] Add support for automatic (Postgres -> SQLite) migration to mx-puppet-steam --- group_vars/matrix_servers | 2 +- .../defaults/main.yml | 15 ++-- .../tasks/setup_install.yml | 82 +++++++++++++------ .../tasks/validate_config.yml | 22 ----- .../templates/config.yaml.j2 | 4 +- 5 files changed, 67 insertions(+), 58 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index fda40efdf..283d5b6e3 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -583,7 +583,7 @@ matrix_mx_puppet_steam_homeserver_token: "{{ matrix_synapse_macaroon_secret_key matrix_mx_puppet_steam_login_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret if matrix_synapse_ext_password_provider_shared_secret_auth_enabled else '' }}" # Postgres is the default, except if not using `matrix_postgres` (internal postgres) -matrix_mx_puppet_stream_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" +matrix_mx_puppet_steam_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" matrix_mx_puppet_steam_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_steam.db') | to_uuid }}" ###################################################################### diff --git a/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml b/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml index 77f7804bc..c1f6a3855 100644 --- a/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml @@ -56,18 +56,17 @@ matrix_mx_puppet_steam_homeserver_token: '' matrix_mx_puppet_steam_login_shared_secret: '' matrix_mx_puppet_steam_database_engine: sqlite + +matrix_mx_puppet_steam_sqlite_database_path_local: "{{ matrix_mx_puppet_steam_data_path }}/database.db" +matrix_mx_puppet_steam_sqlite_database_path_in_container: "/data/database.db" + matrix_mx_puppet_steam_database_username: matrix_mx_puppet_steam matrix_mx_puppet_steam_database_password: ~ matrix_mx_puppet_steam_database_hostname: 'matrix-postgres' matrix_mx_puppet_steam_database_port: 5432 -matrix_mx_puppet_steam_database_name: matrix_mx_puppet_steam -matrix_mx_puppet_steam_database_file: /data/database.db -matrix_mx_puppet_steam_database_connString: >-2 - {%- if matrix_mx_puppet_steam_database_engine == 'postgres' -%} - postgresql://{{ matrix_mx_puppet_steam_database_username }}:{{ matrix_mx_puppet_steam_database_password }}@{{ matrix_mx_puppet_steam_database_hostname }}:{{ matrix_mx_puppet_steam_database_port }}/{{ matrix_mx_puppet_steam_database_name }}?sslmode=disable - {%- elif matrix_mx_puppet_steam_database_engine == 'sqlite' -%} - {{ matrix_mx_puppet_steam_database_engine }}://{{ matrix_mx_puppet_steam_database_file }} - {%- endif -%} +matrix_mx_puppet_steam_database_db_name: matrix_mx_puppet_steam + +matrix_mx_puppet_steam_database_connection_string: 'postgresql://{{ matrix_mx_puppet_steam_database_username }}:{{ matrix_mx_puppet_steam_database_password }}@{{ matrix_mx_puppet_steam_database_hostname }}:{{ matrix_mx_puppet_steam_database_port }}/{{ matrix_mx_puppet_steam_database_db_name }}?sslmode=disable' # Default configuration template which covers the generic use case. # You can customize it by controlling the various variables inside it. diff --git a/roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml index 7b7f82115..71f6d8897 100644 --- a/roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml +++ b/roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml @@ -8,14 +8,6 @@ The matrix-bridge-mx-puppet-steam role needs to execute before the matrix-synapse role. when: "matrix_synapse_role_executed|default(False)" -- name: Ensure MX Puppet Steam image is pulled - docker_image: - name: "{{ matrix_mx_puppet_steam_docker_image }}" - source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" - force_source: "{{ matrix_mx_puppet_steam_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" - force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mx_puppet_steam_docker_image_force_pull }}" - when: matrix_mx_puppet_steam_enabled|bool and not matrix_mx_puppet_steam_container_image_self_build - - name: Ensure MX Puppet Steam paths exist file: path: "{{ item.path }}" @@ -30,6 +22,57 @@ - { path: "{{ matrix_mx_puppet_steam_docker_src_files_path }}", when: "{{ matrix_mx_puppet_steam_container_image_self_build }}" } when: matrix_mx_puppet_steam_enabled|bool and item.when|bool +- name: Check if an old database file already exists + stat: + path: "{{ matrix_mx_puppet_steam_base_path }}/database.db" + register: matrix_mx_puppet_steam_stat_database + +- name: (Data relocation) Ensure matrix-mx-puppet-steam.service is stopped + service: + name: matrix-mx-puppet-steam + state: stopped + daemon_reload: yes + failed_when: false + when: "matrix_mx_puppet_steam_stat_database.stat.exists" + +- name: (Data relocation) Move mx-puppet-steam database file to ./data directory + command: "mv {{ matrix_mx_puppet_steam_base_path }}/database.db {{ matrix_mx_puppet_steam_data_path }}/database.db" + when: "matrix_mx_puppet_steam_stat_database.stat.exists" + +- set_fact: + matrix_mx_puppet_steam_requires_restart: false + +- block: + - name: Check if an SQLite database already exists + stat: + path: "{{ matrix_mx_puppet_steam_sqlite_database_path_local }}" + register: matrix_mx_puppet_steam_sqlite_database_path_local_stat_result + + - block: + - set_fact: + matrix_postgres_db_migration_request: + src: "{{ matrix_mx_puppet_steam_sqlite_database_path_local }}" + dst: "{{ matrix_mx_puppet_steam_database_connection_string }}" + caller: "{{ role_path|basename }}" + engine_variable_name: 'matrix_mx_puppet_steam_database_engine' + engine_old: 'sqlite' + systemd_services_to_stop: ['matrix-mx-puppet-steam.service'] + + - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + + - set_fact: + matrix_mx_puppet_steam_requires_restart: true + when: "matrix_mx_puppet_steam_sqlite_database_path_local_stat_result.stat.exists|bool" + when: "matrix_mx_puppet_steam_database_engine == 'postgres'" + +- name: Ensure MX Puppet Steam image is pulled + docker_image: + name: "{{ matrix_mx_puppet_steam_docker_image }}" + source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_mx_puppet_steam_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mx_puppet_steam_docker_image_force_pull }}" + when: matrix_mx_puppet_steam_enabled|bool and not matrix_mx_puppet_steam_container_image_self_build + - name: Ensure MX Puppet Steam repository is present on self build git: repo: "{{ matrix_mx_puppet_steam_container_image_self_build_repo }}" @@ -49,23 +92,6 @@ pull: yes when: "matrix_mx_puppet_steam_enabled|bool and matrix_mx_puppet_steam_container_image_self_build" -- name: Check if an old database file already exists - stat: - path: "{{ matrix_mx_puppet_steam_base_path }}/database.db" - register: matrix_mx_puppet_steam_stat_database - -- name: (Data relocation) Ensure matrix-mx-puppet-steam.service is stopped - service: - name: matrix-mx-puppet-steam - state: stopped - daemon_reload: yes - failed_when: false - when: "matrix_mx_puppet_steam_stat_database.stat.exists" - -- name: (Data relocation) Move mx-puppet-steam database file to ./data directory - command: "mv {{ matrix_mx_puppet_steam_base_path }}/database.db {{ matrix_mx_puppet_steam_data_path }}/database.db" - when: "matrix_mx_puppet_steam_stat_database.stat.exists" - - name: Ensure mx-puppet-steam config.yaml installed copy: content: "{{ matrix_mx_puppet_steam_configuration|to_nice_yaml }}" @@ -93,3 +119,9 @@ service: daemon_reload: yes when: "matrix_mx_puppet_steam_systemd_service_result.changed" + +- name: Ensure matrix-mx-puppet-steam.service restarted, if necessary + service: + name: "matrix-mx-puppet-steam.service" + state: restarted + when: "matrix_mx_puppet_steam_requires_restart|bool" diff --git a/roles/matrix-bridge-mx-puppet-steam/tasks/validate_config.yml b/roles/matrix-bridge-mx-puppet-steam/tasks/validate_config.yml index 50ce15d9d..a8bc6a420 100644 --- a/roles/matrix-bridge-mx-puppet-steam/tasks/validate_config.yml +++ b/roles/matrix-bridge-mx-puppet-steam/tasks/validate_config.yml @@ -8,25 +8,3 @@ with_items: - "matrix_mx_puppet_steam_appservice_token" - "matrix_mx_puppet_steam_homeserver_token" - -- block: - - name: Check if an SQLite database already exists - stat: - path: "{{ matrix_mx_puppet_steam_data_path }}" - register: matrix_mx_puppet_steam_sqlite_database_path_local_stat_result - - - name: Fail if an SQLite database already exists when using Postgres - fail: - msg: >- - matrix_mx_puppet_steam_database_engine has been set to `postgres` (which is our new default now). - However, we've discovered an existing SQLite database in {{ matrix_mx_puppet_steam_data_path }}/database.db. - It appears that you've been using this bridge with the SQLite engine until now. - To continue using SQLite, opt into it explicitly: add `matrix_mx_puppet_steam_database_engine: sqlite` to your vars.yml file and re-run this same command. - Alternatively, to migrate your existing SQLite database to Postgres: - 1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) - 2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mx_puppet_steam_data_path }}/database.db postgres_connection_string_variable_name=matrix_mx_puppet_steam_database_connString'`) - 3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`) - when: "matrix_mx_puppet_steam_sqlite_database_path_local_stat_result.stat.exists" - when: "matrix_mx_puppet_steam_database_engine == 'postgres'" - - diff --git a/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2 index 8eda278dd..149e08b69 100644 --- a/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2 @@ -67,11 +67,11 @@ database: # Connection string to connect to the Postgres instance # with username "user", password "pass", host "localhost" and database name "dbname". # Modify each value as necessary - connString: {{ matrix_mx_puppet_steam_database_connString | to_json }} + connString: {{ matrix_mx_puppet_steam_database_connection_string|to_json }} {% else %} # Use SQLite3 as a database backend # The name of the database file - filename: {{ matrix_mx_puppet_steam_database_file }} + filename: {{ matrix_mx_puppet_steam_sqlite_database_path_in_container|to_json }} {% endif %} logging: