Split playbook into multiple roles

As suggested in #63 (Github issue), splitting the
playbook's logic into multiple roles will be beneficial for
maintainability.

This patch realizes this split. Still, some components
affect others, so the roles are not really independent of one
another. For example:
- disabling mxisd (`matrix_mxisd_enabled: false`), causes Synapse
and riot-web to reconfigure themselves with other (public)
Identity servers.

- enabling matrix-corporal (`matrix_corporal_enabled: true`) affects
how reverse-proxying (by `matrix-nginx-proxy`) is done, in order to
put matrix-corporal's gateway server in front of Synapse

We may be able to move away from such dependencies in the future,
at the expense of a more complicated manual configuration, but
it's probably not worth sacrificing the convenience we have now.

As part of this work, the way we do "start components" has been
redone now to use a loop, as suggested in #65 (Github issue).
This should make restarting faster and more reliable.

roles/matrix-base/defaults/main.yml

@@ -0,0 +1,37 @@
+# The bare hostname which represents your identity.
+# This is something like "example.com".
+# Note: this playbook does not touch the server referenced here.
+hostname_identity: "{{ host_specific_hostname_identity|lower }}"
+
+# This is where your data lives and what we set up here.
+# This and the Riot hostname (see below) are expected to be on the same server.
+hostname_matrix: "matrix.{{ hostname_identity }}"
+
+# This is where you access the web UI from and what we set up here.
+# This and the Matrix hostname (see above) are expected to be on the same server.
+hostname_riot: "riot.{{ hostname_identity }}"
+
+
+matrix_user_username: "matrix"
+matrix_user_uid: 991
+matrix_user_gid: 991
+
+matrix_base_data_path: "/matrix"
+
+matrix_static_files_base_path: "{{ matrix_base_data_path }}/static-files"
+
+matrix_homeserver_url: "https://{{ hostname_matrix }}"
+matrix_identity_server_url: "https://{{ matrix_synapse_trusted_third_party_id_servers[0] }}"
+
+# The Docker network that all services would be put into
+matrix_docker_network: "matrix"
+
+# Variables to Control which parts of our roles run.
+run_setup: true
+run_import_postgres: true
+run_upgrade_postgres: true
+run_start: true
+run_register_user: true
+run_import_sqlite_db: true
+run_import_media_store: true
+run_self_check: true

roles/matrix-base/files/yum.repos.d/docker-ce.repo

@@ -0,0 +1,62 @@
+[docker-ce-stable]
+name=Docker CE Stable - $basearch
+baseurl=https://download.docker.com/linux/centos/7/$basearch/stable
+enabled=1
+gpgcheck=1
+gpgkey=https://download.docker.com/linux/centos/gpg
+
+[docker-ce-stable-debuginfo]
+name=Docker CE Stable - Debuginfo $basearch
+baseurl=https://download.docker.com/linux/centos/7/debug-$basearch/stable
+enabled=0
+gpgcheck=1
+gpgkey=https://download.docker.com/linux/centos/gpg
+
+[docker-ce-stable-source]
+name=Docker CE Stable - Sources
+baseurl=https://download.docker.com/linux/centos/7/source/stable
+enabled=0
+gpgcheck=1
+gpgkey=https://download.docker.com/linux/centos/gpg
+
+[docker-ce-edge]
+name=Docker CE Edge - $basearch
+baseurl=https://download.docker.com/linux/centos/7/$basearch/edge
+enabled=0
+gpgcheck=1
+gpgkey=https://download.docker.com/linux/centos/gpg
+
+[docker-ce-edge-debuginfo]
+name=Docker CE Edge - Debuginfo $basearch
+baseurl=https://download.docker.com/linux/centos/7/debug-$basearch/edge
+enabled=0
+gpgcheck=1
+gpgkey=https://download.docker.com/linux/centos/gpg
+
+[docker-ce-edge-source]
+name=Docker CE Edge - Sources
+baseurl=https://download.docker.com/linux/centos/7/source/edge
+enabled=0
+gpgcheck=1
+gpgkey=https://download.docker.com/linux/centos/gpg
+
+[docker-ce-test]
+name=Docker CE Test - $basearch
+baseurl=https://download.docker.com/linux/centos/7/$basearch/test
+enabled=0
+gpgcheck=1
+gpgkey=https://download.docker.com/linux/centos/gpg
+
+[docker-ce-test-debuginfo]
+name=Docker CE Test - Debuginfo $basearch
+baseurl=https://download.docker.com/linux/centos/7/debug-$basearch/test
+enabled=0
+gpgcheck=1
+gpgkey=https://download.docker.com/linux/centos/gpg
+
+[docker-ce-test-source]
+name=Docker CE Test - Sources
+baseurl=https://download.docker.com/linux/centos/7/source/test
+enabled=0
+gpgcheck=1
+gpgkey=https://download.docker.com/linux/centos/gpg

roles/matrix-base/tasks/clean_up_old_files.yml

@@ -0,0 +1,9 @@
+---
+
+- name: Get rid of old files and directories
+  file:
+    path: "{{ item }}"
+    state: absent
+  with_items:
+    - "{{ matrix_base_data_path }}/environment-variables"
+    - "{{ matrix_base_data_path }}/scratchpad"

roles/matrix-base/tasks/main.yml

@@ -0,0 +1,33 @@
+- import_tasks: "{{ role_path }}/tasks/clean_up_old_files.yml"
+  when: run_setup
+  tags:
+    - setup-all
+
+- import_tasks: "{{ role_path }}/tasks/setup_server_base.yml"
+  when: run_setup
+  tags:
+    - setup-all
+
+- import_tasks: "{{ role_path }}/tasks/setup_matrix_base.yml"
+  when: run_setup
+  tags:
+    - setup-all
+
+- import_tasks: "{{ role_path }}/tasks/setup_well_known.yml"
+  when: run_setup
+  tags:
+    - setup-all
+    - setup-mxisd
+    - setup-synapse
+    - setup-nginx-proxy
+
+- import_tasks: "{{ role_path }}/tasks/sanity_check.yml"
+  tags:
+    - always
+
+- import_tasks: "{{ role_path }}/tasks/self_check_dns.yml"
+  delegate_to: 127.0.0.1
+  become: false
+  when: run_self_check
+  tags:
+    - self-check

roles/matrix-base/tasks/sanity_check.yml

@@ -0,0 +1,39 @@
+---
+
+- set_fact:
+    matrix_ansible_outdated_fail_msg: "You are running on Ansible {{ ansible_version.string }}, which is not supported. See our guide about Ansible: https://github.com/spantaleev/matrix-docker-ansible-deploy/blob/master/docs/ansible.md"
+
+- name: Fail if running on Ansible < 2.4
+  fail:
+    msg: "{{ matrix_ansible_outdated_fail_msg }}"
+  when: "ansible_version.major <= 2 and ansible_version.minor < 4"
+
+# Ansible 2.5.0 and 2.5.1 are known to have a bug with `include_tasks` + `with_items`.
+# The bug has been fixed in Ansible 2.5.2.
+- name: Fail if running on Ansible 2.5.x (lower than 2.5.2)
+  fail:
+    msg: "{{ matrix_ansible_outdated_fail_msg }}"
+  when: "ansible_version.major == 2 and ansible_version.minor == 5 and ansible_version.revision < 2"
+
+- name: Fail if Macaroon key is missing
+  fail:
+    msg: "You need to set a secret in the matrix_synapse_macaroon_secret_key variable"
+  when: "matrix_synapse_macaroon_secret_key == ''"
+
+- name: Fail if Coturn Auth secret is missing
+  fail:
+    msg: "You need to set a secret in the matrix_coturn_turn_static_auth_secret variable"
+  when: "matrix_coturn_turn_static_auth_secret == ''"
+
+# This sanity check is only used to detect uppercase when people override these specific variables.
+#
+# If people set `host_specific_hostname_identity` without overriding other variables (the general use-case),
+# we take care to lower-case it automatically and it won't cause trouble anyway.
+- name: Fail if uppercase domain used
+  fail:
+    msg: "Detected that you're using an uppercase domain name - `{{ item }}`. This will cause trouble. Please use all-lowercase!"
+  when: "item != item|lower"
+  with_items:
+    - "{{ hostname_identity }}"
+    - "{{ hostname_matrix }}"
+    - "{{ hostname_riot }}"

roles/matrix-base/tasks/self_check_dns.yml

@@ -0,0 +1,28 @@
+---
+
+- name: Determine DNS SRV records to check (Matrix)
+  set_fact:
+    dns_srv_record_checks:
+      - service_and_protocol: "_matrix._tcp"
+        domain: "{{ (hostname_identity + '.') }}"
+        expected_target: "{{ (hostname_matrix + '.') }}"
+        expected_port: 8448
+
+- block:
+    - set_fact:
+        dns_srv_record_check_mxisd:
+          service_and_protocol: "_matrix-identity._tcp"
+          domain: "{{ (hostname_identity + '.') }}"
+          expected_target: "{{ (hostname_matrix + '.') }}"
+          expected_port: 443
+
+    - name: Determine domains that we require certificates for (mxisd)
+      set_fact:
+        dns_srv_record_checks: "{{ dns_srv_record_checks + [dns_srv_record_check_mxisd] }}"
+  when: "matrix_mxisd_enabled"
+
+- name: Perform DNS SRV checks
+  include_tasks: "{{ role_path }}/tasks/self_check_dns_srv.yml"
+  with_items: "{{ dns_srv_record_checks }}"
+  loop_control:
+    loop_var: dns_srv_record_check

roles/matrix-base/tasks/self_check_dns_srv.yml

@@ -0,0 +1,26 @@
+---
+
+# This requires the dnspython library and will fail with a friendly error when unavailable.
+- name: Check DNS SRV record for {{ dns_srv_record_check.service_and_protocol }} on {{ dns_srv_record_check.domain }}
+  set_fact:
+    result_dig_srv: "{{ lookup('dig', (dns_srv_record_check.service_and_protocol + '.' + dns_srv_record_check.domain + '/SRV'), 'flat=0', wantlist=False) }}"
+
+- name: Fail if DNS SRV record missing
+  fail:
+    msg: "It appears the DNS SRV record for {{ dns_srv_record_check.service_and_protocol }} on {{ dns_srv_record_check.domain }} is not set up correctly (the record is missing). See the 'Configuring DNS' documentation for this playbook."
+  when: "result_dig_srv == 'NXDOMAIN'"
+
+- name: Fail if DNS SRV record incorrect
+  fail:
+    msg: >
+      It appears the DNS SRV record for {{ dns_srv_record_check.service_and_protocol }} on {{ dns_srv_record_check.domain }} is not set up correctly.
+      Expected it to point to `{{ dns_srv_record_check.expected_target }}` (port {{ dns_srv_record_check.expected_port }}).
+      Found it pointing to `{{ result_dig_srv.target }}` (port {{ result_dig_srv.port }}).
+      See the 'Configuring DNS' documentation for this playbook.
+  when: "result_dig_srv.target != dns_srv_record_check.expected_target or result_dig_srv.port != dns_srv_record_check.expected_port"
+
+- name: Report correct DNS SRV record
+  debug:
+    msg: >
+      The DNS SRV record for `{{ dns_srv_record_check.service_and_protocol }}` on `{{ dns_srv_record_check.domain }}`
+      points to `{{ result_dig_srv.target }}` (port {{ dns_srv_record_check.expected_port }}), as expected

roles/matrix-base/tasks/setup_matrix_base.yml

@@ -0,0 +1,56 @@
+---
+
+- name: Ensure Matrix group is created
+  group:
+    name: "{{ matrix_user_username }}"
+    gid: "{{ matrix_user_gid }}"
+    state: present
+
+- name: Ensure Matrix user is created
+  user:
+    name: "{{ matrix_user_username }}"
+    uid: "{{ matrix_user_uid }}"
+    state: present
+    group: "{{ matrix_user_username }}"
+
+- name: Ensure Matrix base path exists
+  file:
+    path: "{{ item }}"
+    state: directory
+    mode: 0750
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_username }}"
+  with_items:
+    - "{{ matrix_base_data_path }}"
+
+# `docker_network` doesn't work as expected when the given network
+# is a substring of a network that already exists.
+#
+# See:
+# - https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/12
+# - https://github.com/ansible/ansible/issues/32926
+#
+# Due to that, we employ a workaround below.
+#
+# - name: Ensure Matrix network is created in Docker
+#   docker_network:
+#     name: "{{ matrix_docker_network }}"
+#     driver: bridge
+
+- name: Check existence of Matrix network in Docker
+  shell:
+    cmd: "docker network ls -q --filter='name=^{{ matrix_docker_network }}$'"
+  register: result_check_docker_network
+  changed_when: false
+
+- name: Create Matrix network in Docker
+  shell:
+    cmd: "docker network create --driver=bridge {{ matrix_docker_network }}"
+  when: "result_check_docker_network.stdout == ''"
+
+- name: Ensure matrix-remove-all script created
+  template:
+    src: "{{ role_path }}/templates/usr-local-bin/matrix-remove-all.j2"
+    dest: "/usr/local/bin/matrix-remove-all"
+    mode: 0750
+

roles/matrix-base/tasks/setup_server_base.yml

@@ -0,0 +1,87 @@
+---
+
+- name: Ensure Docker repository is enabled (CentOS)
+  template:
+    src: "{{ role_path }}/files/yum.repos.d/{{ item }}"
+    dest: "/etc/yum.repos.d/{{ item }}"
+    owner: "root"
+    group: "root"
+    mode: 0644
+  with_items:
+    - docker-ce.repo
+  when: ansible_distribution == 'CentOS'
+
+- name: Ensure Docker's RPM key is trusted
+  rpm_key:
+    state: present
+    key: https://download.docker.com/linux/centos/gpg
+  when: ansible_distribution == 'CentOS'
+
+- name: Ensure yum packages are installed (CentOS)
+  yum:
+    name:
+      - bash-completion
+      - docker-ce
+      - docker-python
+      - firewalld
+      - ntp
+      - fuse
+    state: latest
+    update_cache: yes
+  when: ansible_distribution == 'CentOS'
+
+- name: Ensure APT usage dependencies are installed (Debian)
+  apt:
+    name:
+      - apt-transport-https
+      - ca-certificates
+    state: present
+    update_cache: yes
+  when: ansible_os_family == 'Debian'
+
+- name: Ensure Docker's APT key is trusted (Debian)
+  apt_key:
+    url: https://download.docker.com/linux/ubuntu/gpg
+    id: 9DC858229FC7DD38854AE2D88D81803C0EBFCD88
+    state: present
+  register: add_repository_key
+  ignore_errors: true
+  when: ansible_os_family == 'Debian'
+
+- name: Ensure Docker repository is enabled (Debian)
+  apt_repository:
+    repo: "deb https://download.docker.com/linux/{{ ansible_distribution|lower }} {{ ansible_distribution_release }} stable"
+    state: present
+    update_cache: yes
+  when: ansible_os_family == 'Debian'
+
+- name: Ensure APT packages are installed (Debian)
+  apt:
+    name:
+      - bash-completion
+      - docker-ce
+      - python-docker
+      - ntp
+      - fuse
+    state: latest
+    update_cache: yes
+  when: ansible_os_family == 'Debian'
+
+- name: Ensure firewalld is started and autoruns
+  service:
+    name: firewalld
+    state: started
+    enabled: yes
+  when: ansible_os_family == 'RedHat'
+
+- name: Ensure Docker is started and autoruns
+  service:
+    name: docker 
+    state: started
+    enabled: yes
+
+- name: Ensure ntpd is started and autoruns
+  service:
+    name: "{{ 'ntpd' if ansible_os_family == 'RedHat' else 'ntp' }}"
+    state: started
+    enabled: yes

roles/matrix-base/tasks/setup_well_known.yml

@@ -0,0 +1,21 @@
+# We need others to be able to read these directories too,
+# so that matrix-nginx-proxy's nginx user can access the files.
+#
+# For running with another webserver, we recommend being part of the `matrix` group.
+- name: Ensure Matrix static-files path exists
+  file:
+    path: "{{ item }}"
+    state: directory
+    mode: 0755
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_username }}"
+  with_items:
+    - "{{ matrix_static_files_base_path }}/.well-known/matrix"
+
+- name: Ensure Matrix /.well-known/matrix/client configured
+  template:
+    src: "{{ role_path }}/templates/static-files/well-known/matrix-client.j2"
+    dest: "{{ matrix_static_files_base_path }}/.well-known/matrix"
+    mode: 0644
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_username }}"

roles/matrix-base/templates/static-files/well-known/matrix-client.j2

@@ -0,0 +1,8 @@
+{
+	"m.homeserver": {
+		"base_url": "{{ matrix_homeserver_url }}"
+	},
+	"m.identity_server": {
+		"base_url": "{{ matrix_identity_server_url }}"
+	}
+}

roles/matrix-base/templates/usr-local-bin/matrix-remove-all.j2

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+if [ "$(id -u)" != "0" ]; then
+        echo "This script must be executed as root! Aborting."
+        exit 1
+fi
+
+echo "WARNING! You are about to remove everything the playbook installs for matrix.{{ host_specific_hostname_identity }}: matrix, docker images,..."
+echo -n "If you're sure you want to do this, type: 'Yes, I really want to remove everything!'"
+read sure
+
+if [ "$sure" != "Yes, I really want to remove everything!" ]; then
+        echo "Good thing I asked, exiting"
+        exit 0
+else
+        echo "Stop and remove matrix services"
+        for s in $(find /etc/systemd/system/ -name "matrix-*" -printf "%f\n"); do
+                systemctl stop $s
+                rm -f /etc/systemd/system/$s
+        done
+        systemctl daemon-reload
+        echo "Remove matrix cronjobs"
+        find /etc/cron.d/ -name "matrix-*" -delete
+        echo "Remove matrix scripts"
+        find /usr/local/bin/ -name "matrix-*" -delete
+        echo "Remove every docker images"
+        docker rmi $(docker images -aq)
+        echo "Remove docker matrix network"
+        docker network rm matrix
+        echo "Remove /matrix directory"
+        rm -fr /matrix
+        exit 0
+fi
+

roles/matrix-base/vars/main.yml

@@ -0,0 +1,3 @@
+# This will contain a list of enabled services that the playbook is managing.
+# Each component is expected to append its service name to this list.
+matrix_systemd_services_list: []