Split playbook into multiple roles

As suggested in #63 (Github issue), splitting the
playbook's logic into multiple roles will be beneficial for
maintainability.

This patch realizes this split. Still, some components
affect others, so the roles are not really independent of one
another. For example:
- disabling mxisd (`matrix_mxisd_enabled: false`), causes Synapse
and riot-web to reconfigure themselves with other (public)
Identity servers.

- enabling matrix-corporal (`matrix_corporal_enabled: true`) affects
how reverse-proxying (by `matrix-nginx-proxy`) is done, in order to
put matrix-corporal's gateway server in front of Synapse

We may be able to move away from such dependencies in the future,
at the expense of a more complicated manual configuration, but
it's probably not worth sacrificing the convenience we have now.

As part of this work, the way we do "start components" has been
redone now to use a loop, as suggested in #65 (Github issue).
This should make restarting faster and more reliable.

roles/matrix-postgres/tasks/import_postgres.yml

@@ -0,0 +1,60 @@
+---
+
+# Pre-checks
+
+- name: Fail if playbook called incorrectly
+  fail:
+    msg: "The `server_path_postgres_dump` variable needs to be provided to this playbook, via --extra-vars"
+  when: "server_path_postgres_dump is not defined or server_path_postgres_dump.startswith('<')"
+
+- name: Check if the provided Postgres dump file exists
+  stat:
+    path: "{{ server_path_postgres_dump }}"
+  register: result_server_path_postgres_dump_stat
+
+- name: Fail if provided Postgres dump file doesn't exists
+  fail:
+    msg: "File cannot be found on the server at {{ server_path_postgres_dump }}"
+  when: not result_server_path_postgres_dump_stat.stat.exists
+
+- import_tasks: tasks/util/detect_existing_postgres_version.yml
+
+- name: Abort, if no existing Postgres version detected
+  fail:
+    msg: "Could not find existing Postgres installation"
+  when: "not matrix_postgres_detected_existing"
+
+
+# Defaults
+
+- name: Set postgres_start_wait_time, if not provided
+  set_fact:
+    postgres_start_wait_time: 15
+  when: "postgres_start_wait_time|default('') == ''"
+
+
+# Actual import work
+
+- name: Ensure matrix-postgres is started
+  service:
+    name: matrix-postgres
+    state: started
+    daemon_reload: yes
+
+- name: Wait a bit, so that Postgres can start
+  wait_for:
+    timeout: "{{ postgres_start_wait_time }}"
+  delegate_to: 127.0.0.1
+  become: false
+
+- name: Perform Postgres database import
+  command: |
+    /usr/bin/docker run --rm --name matrix-postgres-import \
+    --network={{ matrix_docker_network }} \
+    --env-file={{ matrix_postgres_base_path }}/env-postgres-psql \
+    -v {{ server_path_postgres_dump }}:{{ server_path_postgres_dump }}:ro \
+    --entrypoint=/bin/sh
+    {{ matrix_postgres_docker_image_latest }}
+    -c 'cat {{ server_path_postgres_dump }} | \
+    {{ 'gunzip |' if server_path_postgres_dump.endswith('.gz') else '' }}
+    psql -v ON_ERROR_STOP=1 -h matrix-postgres'

roles/matrix-postgres/tasks/import_sqlite_db.yml

@@ -0,0 +1,84 @@
+---
+
+# Pre-checks
+
+- name: Fail if playbook called incorrectly
+  fail:
+    msg: "The `server_path_homeserver_db` variable needs to be provided to this playbook, via --extra-vars"
+  when: "server_path_homeserver_db is not defined or server_path_homeserver_db.startswith('<')"
+
+- name: Check if the provided SQLite homeserver.db file exists
+  stat:
+    path: "{{ server_path_homeserver_db }}"
+  register: result_server_path_homeserver_db_stat
+
+- name: Fail if provided SQLite homeserver.db file doesn't exist
+  fail:
+    msg: "File cannot be found on the server at {{ server_path_homeserver_db }}"
+  when: not result_server_path_homeserver_db_stat.stat.exists
+
+
+# Defaults
+
+- name: Set postgres_start_wait_time, if not provided
+  set_fact:
+    postgres_start_wait_time: 15
+  when: "postgres_start_wait_time|default('') == ''"
+
+
+# Actual import work
+
+- name: Ensure matrix-postgres is stopped
+  service:
+    name: matrix-postgres
+    state: stopped
+    daemon_reload: yes
+
+- name: Ensure postgres data is wiped out
+  file:
+    path: "{{ matrix_postgres_data_path }}"
+    state: absent
+
+- name: Ensure postgres data path exists
+  file:
+    path: "{{ matrix_postgres_data_path }}"
+    state: directory
+    mode: 0700
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_username }}"
+
+- name: Ensure matrix-postgres is started
+  service:
+    name: matrix-postgres
+    state: restarted
+    daemon_reload: yes
+
+- name: Wait a bit, so that Postgres can start
+  wait_for:
+    timeout: "{{ postgres_start_wait_time }}"
+  delegate_to: 127.0.0.1
+  become: false
+
+# If the actual migration command (below) fails, it will leave a container behind.
+# Starting it again later will relaunch that one, which may or may not work.
+# To ensure we're starting from a clean state, ensure any such leftovers are removed.
+- name: Cleanup any old leftover migration container
+  docker_container:
+    name: matrix-synapse-migrate
+    state: absent
+
+- name: Importing SQLite database into Postgres
+  docker_container:
+    name: matrix-synapse-migrate
+    image: "{{ matrix_synapse_docker_image }}"
+    detach: no
+    cleanup: yes
+    entrypoint: /usr/local/bin/python
+    command: "/usr/local/bin/synapse_port_db --sqlite-database {{ server_path_homeserver_db }} --postgres-config /data/homeserver.yaml"
+    user: "{{ matrix_user_uid }}:{{ matrix_user_gid }}"
+    volumes:
+      - "{{ matrix_synapse_config_dir_path }}:/data"
+      - "{{ matrix_synapse_run_path }}:/matrix-run"
+      - "{{ server_path_homeserver_db }}:/{{ server_path_homeserver_db }}:ro"
+    networks:
+      - name: "{{ matrix_docker_network }}"

roles/matrix-postgres/tasks/init.yml

@@ -0,0 +1,3 @@
+- set_fact:
+    matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-postgres'] }}"
+  when: "not matrix_postgres_use_external"

roles/matrix-postgres/tasks/main.yml

@@ -0,0 +1,24 @@
+- import_tasks: "{{ role_path }}/tasks/init.yml"
+  tags:
+    - always
+
+- import_tasks: "{{ role_path }}/tasks/setup_postgres.yml"
+  when: run_setup
+  tags:
+    - setup-postgres
+    - setup-all
+
+- import_tasks: "{{ role_path }}/tasks/import_postgres.yml"
+  when: run_import_postgres
+  tags:
+    - import-postgres
+
+- import_tasks: "{{ role_path }}/tasks/import_sqlite_db.yml"
+  when: run_import_sqlite_db
+  tags:
+    - import-sqlite-db
+
+- import_tasks: "{{ role_path }}/tasks/upgrade_postgres.yml"
+  when: run_upgrade_postgres
+  tags:
+    - upgrade-postgres

roles/matrix-postgres/tasks/migrate_postgres_data_directory.yml

@@ -0,0 +1,70 @@
+---
+
+# We used to store Postgres data directly under `/matrix/postgres` (what is now considered `matrix_postgres_base_path`).
+#
+# From now on, we expect to store Postgres data one directory below now (`/matrix/postgres/data` - `matrix_postgres_data_path`).
+# We wish to use the base directory for other purposes (storing environment variable files, etc.).
+# Mixing those with the Postgres data is no good and it leads to Postgres's `initdb` complaining to initialize
+# a database in a non-empty directory.
+#
+# For this reason, we store the Postgres data in `/matrix/postgres/data` and need to relocate any installations
+# which still store it in the parent directory (`/matrix/postgres`).
+
+- name: Check if old Postgres data directory is used
+  stat:
+    path: "{{ matrix_postgres_base_path }}/PG_VERSION"
+  register: result_pg_old_data_dir_stat
+
+- name: Warn if old Postgres data directory detected
+  debug:
+    msg: >
+      Found that you have Postgres data in `{{ matrix_postgres_base_path }}`.
+      From now on, Postgres data is supposed to be stored in `{{ matrix_postgres_data_path }}` instead.
+      We'll stop Postgres and relocate the files there for you.
+  when: "result_pg_old_data_dir_stat.stat.exists"
+
+- name: Find files and directories in old Postgres data path
+  find:
+    paths: "{{ matrix_postgres_base_path }}"
+    file_type: any
+    excludes: ["data"]
+  register: "result_pg_old_data_dir_find"
+  when: "result_pg_old_data_dir_stat.stat.exists"
+
+- name: Ensure new Postgres data path exists
+  file:
+    path: "{{ matrix_postgres_data_path }}"
+    state: directory
+    mode: 0700
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_username }}"
+  when: "result_pg_old_data_dir_stat.stat.exists"
+
+- name: Ensure matrix-postgres is stopped
+  service:
+    name: matrix-postgres
+    state: stopped
+    daemon_reload: yes
+  when: "result_pg_old_data_dir_stat.stat.exists"
+
+- block:
+    - name: Relocate Postgres data files from old directory to new
+      command: "mv {{ item.path }} {{ matrix_postgres_data_path }}/{{ item.path|basename }}"
+      with_items: "{{ result_pg_old_data_dir_find.files }}"
+  when: "result_pg_old_data_dir_stat.stat.exists"
+
+# Intentionally not starting matrix-postgres here.
+# It likely needs to be updated to point to the new directory.
+# In fact, let's even get rid of the outdated service, to ensure no one will start it
+# and have it initialize a new database.
+
+- name: Ensure outdated matrix-postgres.service doesn't exist
+  file:
+    path: "/etc/systemd/system/matrix-postgres.service"
+    state: absent
+  when: "result_pg_old_data_dir_stat.stat.exists"
+
+- name: Ensure systemd reloaded after getting rid of outdated matrix-postgres.service
+  service:
+    daemon_reload: yes
+  when: "result_pg_old_data_dir_stat.stat.exists"

roles/matrix-postgres/tasks/setup_postgres.yml

@@ -0,0 +1,106 @@
+---
+
+#
+# Generic tasks, no matter what kind of server we're using (internal/external)
+#
+
+- import_tasks: "{{ role_path }}/tasks/migrate_postgres_data_directory.yml"
+
+- import_tasks: "{{ role_path }}/tasks/util/detect_existing_postgres_version.yml"
+
+# If we have found an existing version (installed from before), we use its corresponding Docker image.
+# If not, we install using the latest Postgres.
+#
+# Upgrading is supposed to be performed separately and explicitly (see `upgrade_postgres.yml`).
+- set_fact:
+    matrix_postgres_docker_image_to_use: "{{ matrix_postgres_docker_image_latest if matrix_postgres_detected_version_corresponding_docker_image == '' else matrix_postgres_detected_version_corresponding_docker_image }}"
+
+- name: Warn if on an old version of Postgres
+  debug:
+    msg: "NOTE: Your setup is on an old Postgres version ({{ matrix_postgres_docker_image_to_use }}), while {{ matrix_postgres_docker_image_latest }} is supported. You can upgrade using --tags=upgrade-postgres"
+  when: "matrix_postgres_docker_image_to_use != matrix_postgres_docker_image_latest"
+
+# Even if we don't run the internal server, we still need this for running the CLI
+- name: Ensure postgres Docker image is pulled
+  docker_image:
+    name: "{{ matrix_postgres_docker_image_to_use }}"
+
+# We always create these directories, even if an external Postgres is used,
+# because we store environment variable files there.
+- name: Ensure Postgres paths exist
+  file:
+    path: "{{ item }}"
+    state: directory
+    mode: 0700
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_username }}"
+  with_items:
+    - "{{ matrix_postgres_base_path }}"
+    - "{{ matrix_postgres_data_path }}"
+
+- name: Ensure Postgres environment variables file created
+  template:
+    src: "{{ role_path }}/templates/{{ item }}.j2"
+    dest: "{{ matrix_postgres_base_path }}/{{ item }}"
+    mode: 0640
+  with_items:
+    - "env-postgres-psql"
+    - "env-postgres-server"
+
+- name: Ensure matrix-postgres-cli script created
+  template:
+    src: "{{ role_path }}/templates/usr-local-bin/matrix-postgres-cli.j2"
+    dest: "/usr/local/bin/matrix-postgres-cli"
+    mode: 0750
+
+- name: Ensure matrix-make-user-admin script created
+  template:
+    src: "{{ role_path }}/templates/usr-local-bin/matrix-make-user-admin.j2"
+    dest: "/usr/local/bin/matrix-make-user-admin"
+    mode: 0750
+
+#
+# Tasks related to setting up an internal postgres server
+#
+
+- name: Ensure matrix-postgres.service installed
+  template:
+    src: "{{ role_path }}/templates/systemd/matrix-postgres.service.j2"
+    dest: "/etc/systemd/system/matrix-postgres.service"
+    mode: 0644
+  when: "not matrix_postgres_use_external"
+
+#
+# Tasks related to getting rid of the internal postgres server (if it was previously enabled)
+#
+
+- name: Check existence of matrix-postgres service
+  stat:
+    path: "/etc/systemd/system/matrix-postgres.service"
+  register: matrix_postgres_service_stat
+  when: matrix_postgres_use_external
+
+- name: Ensure matrix-postgres is stopped
+  service:
+    name: matrix-postgres
+    state: stopped
+    daemon_reload: yes
+  when: "matrix_postgres_use_external and matrix_postgres_service_stat.stat.exists"
+
+- name: Ensure matrix-postgres.service doesn't exist
+  file:
+    path: "/etc/systemd/system/matrix-postgres.service"
+    state: absent
+  when: "matrix_postgres_use_external and matrix_postgres_service_stat.stat.exists"
+
+- name: Check existence of matrix-postgres local data path
+  stat:
+    path: "{{ matrix_postgres_data_path }}"
+  register: matrix_postgres_data_path_stat
+  when: matrix_postgres_use_external
+
+# We just want to notify the user. Deleting data is too destructive.
+- name: Notify if matrix-postgres local data remains
+  debug:
+    msg: "Note: You are not using a local PostgreSQL database, but some old data remains from before in {{ matrix_postgres_data_path }}. Feel free to delete it."
+  when: "matrix_postgres_use_external and matrix_postgres_data_path_stat.stat.exists"

roles/matrix-postgres/tasks/upgrade_postgres.yml

@@ -0,0 +1,124 @@
+---
+
+- name: Set default postgres_dump_dir, if not provided
+  set_fact:
+    postgres_dump_dir: "/tmp"
+  when: "postgres_dump_dir|default('') == ''"
+
+- name: Set postgres_dump_name, if not provided
+  set_fact:
+    postgres_dump_name: "matrix-postgres.out"
+  when: "postgres_dump_name|default('') == ''"
+
+- name: Set postgres_auto_upgrade_backup_data_path, if not provided
+  set_fact:
+    postgres_auto_upgrade_backup_data_path: "{{ matrix_postgres_data_path }}-auto-upgrade-backup"
+  when: "postgres_auto_upgrade_backup_data_path|default('') == ''"
+
+- name: Set postgres_start_wait_time, if not provided
+  set_fact:
+    postgres_start_wait_time: 15
+  when: "postgres_start_wait_time|default('') == ''"
+
+- name: Fail, if trying to upgrade external Postgres database
+  fail:
+    msg: "Your configuration indicates that you're using an external Postgres database. Refusing to try and upgrade that."
+  when: "matrix_postgres_use_external"
+
+- name: Check Postgres auto-upgrade backup data directory
+  stat:
+    path: "{{ postgres_auto_upgrade_backup_data_path }}"
+  register: result_auto_upgrade_path
+
+- name: Abort, if existing Postgres auto-upgrade data path detected
+  fail:
+    msg: "Detected that a left-over {{ postgres_auto_upgrade_backup_data_path }} exists. You should rename it to {{ matrix_postgres_data_path }} if the previous upgrade went wrong, or delete it if it went well."
+  when: "result_auto_upgrade_path.stat.exists"
+
+- import_tasks: tasks/util/detect_existing_postgres_version.yml
+
+- name: Abort, if no existing Postgres version detected
+  fail:
+    msg: "Could not find existing Postgres installation"
+  when: "not matrix_postgres_detected_existing"
+
+- name: Abort, if already at latest Postgres version
+  fail:
+    msg: "You are already running the latest Postgres version supported ({{ matrix_postgres_docker_image_latest }}). Nothing to do"
+  when: "matrix_postgres_detected_version_corresponding_docker_image == matrix_postgres_docker_image_latest"
+
+- debug:
+    msg: "Upgrading database from {{ matrix_postgres_detected_version_corresponding_docker_image }} to {{ matrix_postgres_docker_image_latest }}"
+
+- name: Ensure matrix-synapse is stopped
+  service:
+    name: matrix-synapse
+    state: stopped
+
+- name: Ensure matrix-postgres is started
+  service:
+    name: matrix-postgres
+    state: started
+    daemon_reload: yes
+
+- name: Wait a bit, so that Postgres can start
+  wait_for:
+    timeout: "{{ postgres_start_wait_time }}"
+  delegate_to: 127.0.0.1
+  become: false
+
+- name: Perform Postgres database dump
+  command: |
+    /usr/bin/docker run --rm --name matrix-postgres-dump \
+    --network={{ matrix_docker_network }} \
+    --env-file={{ matrix_postgres_base_path }}/env-postgres-psql \
+    -v {{ postgres_dump_dir }}:/out \
+    {{ matrix_postgres_detected_version_corresponding_docker_image }} pg_dump -h matrix-postgres {{ matrix_postgres_db_name }} -f /out/{{ postgres_dump_name }}
+
+- name: Ensure matrix-postgres is stopped
+  service:
+    name: matrix-postgres
+    state: stopped
+
+- name: Rename existing Postgres data directory
+  command: "mv {{ matrix_postgres_data_path }} {{ postgres_auto_upgrade_backup_data_path }}"
+
+- debug:
+    msg: "NOTE: Your Postgres data directory has been moved from `{{ matrix_postgres_data_path }}` to `{{ postgres_auto_upgrade_backup_data_path }}`. In the event of failure, you can move it back and run the playbook with --tags=setup-postgres to restore operation."
+
+- import_tasks: tasks/setup_postgres.yml
+
+- name: Ensure matrix-postgres autoruns and is restarted
+  service:
+    name: matrix-postgres
+    enabled: yes
+    state: restarted
+    daemon_reload: yes
+
+- name: Wait a bit, so that Postgres can start
+  wait_for:
+    timeout: "{{ postgres_start_wait_time }}"
+  delegate_to: 127.0.0.1
+  become: false
+
+- name: Perform Postgres database import
+  command: |
+    /usr/bin/docker run --rm --name matrix-postgres-import \
+    --network={{ matrix_docker_network }} \
+    --env-file={{ matrix_postgres_base_path }}/env-postgres-psql \
+    -v {{ postgres_dump_dir }}:/in:ro \
+    {{ matrix_postgres_docker_image_latest }} psql -v ON_ERROR_STOP=1 -h matrix-postgres -f /in/{{ postgres_dump_name }}
+
+- name: Delete Postgres database dump file
+  file:
+    path: "{{ postgres_dump_dir }}/{{ postgres_dump_name }}"
+    state: absent
+
+- name: Ensure matrix-synapse is started
+  service:
+    name: matrix-synapse
+    state: started
+    daemon_reload: yes
+
+- debug:
+    msg: "NOTE: Your old Postgres data directory is preserved at `{{ postgres_auto_upgrade_backup_data_path }}`. You might want to get rid of it once you've confirmed that all is well."

roles/matrix-postgres/tasks/util/detect_existing_postgres_version.yml

@@ -0,0 +1,46 @@
+---
+
+# This utility aims to determine if there is some existing Postgres version in use or not.
+# If there is, it also tries to detect the Docker image that corresponds to that version.
+
+- name: Initialize Postgres version determination variables (default to empty)
+  set_fact:
+    matrix_postgres_detection_pg_version_path: "{{ matrix_postgres_data_path }}/PG_VERSION"
+    matrix_postgres_detected_existing: false
+    matrix_postgres_detected_version: ""
+    matrix_postgres_detected_version_corresponding_docker_image: ""
+
+- name: Determine existing Postgres version (check PG_VERSION file)
+  stat:
+    path: "{{ matrix_postgres_detection_pg_version_path }}"
+  register: result_pg_version_stat
+
+- set_fact:
+    matrix_postgres_detected_existing: true
+  when: "result_pg_version_stat.stat.exists"
+
+- name: Determine existing Postgres version (read PG_VERSION file)
+  slurp:
+    src: "{{ matrix_postgres_detection_pg_version_path }}"
+  register: result_pg_version
+  when: "matrix_postgres_detected_existing"
+
+- name: Determine existing Postgres version (make sense of PG_VERSION file)
+  set_fact:
+    matrix_postgres_detected_version: "{{ result_pg_version['content']|b64decode|replace('\n', '') }}"
+  when: "matrix_postgres_detected_existing"
+
+- name: Determine corresponding Docker image to detected version (assume default of latest)
+  set_fact:
+    matrix_postgres_detected_version_corresponding_docker_image: "{{ matrix_postgres_docker_image_latest }}"
+  when: "matrix_postgres_detected_version != ''"
+
+- name: Determine corresponding Docker image to detected version (use 9.x, if detected)
+  set_fact:
+    matrix_postgres_detected_version_corresponding_docker_image: "{{ matrix_postgres_docker_image_v9 }}"
+  when: "matrix_postgres_detected_version.startswith('9.')"
+
+- name: Determine corresponding Docker image to detected version (use 10.x, if detected)
+  set_fact:
+    matrix_postgres_detected_version_corresponding_docker_image: "{{ matrix_postgres_docker_image_v10 }}"
+  when: "matrix_postgres_detected_version == '10' or matrix_postgres_detected_version.startswith('10.')"