From c3a7237de702b4f4ec26509cb1000f99051a8f83 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 4 Nov 2022 14:58:28 +0200 Subject: [PATCH] Initial work on using externally defined roles --- CHANGELOG.md | 21 +++++++ Makefile | 1 + docs/installing.md | 3 + docs/maintenance-upgrading-services.md | 4 +- group_vars/matrix_servers | 25 +++++++++ requirements.yml | 13 +++++ roles/custom/matrix-base/defaults/main.yml | 11 ---- .../matrix-base/tasks/setup_matrix_base.yml | 55 ------------------- .../matrix-bridge-hookshot/defaults/main.yml | 4 +- .../matrix_playbook_migration/tasks/main.yml | 5 ++ .../tasks/validate_config.yml | 12 ++++ setup.yml | 17 ++++++ 12 files changed, 101 insertions(+), 70 deletions(-) create mode 100644 requirements.yml create mode 100644 roles/custom/matrix_playbook_migration/tasks/main.yml create mode 100644 roles/custom/matrix_playbook_migration/tasks/validate_config.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d8b9b219..9bd06a8c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,24 @@ +# 2022-11-04 + +## The playbook now uses external roles for some things + +**TLDR**: when updating the playbook and before running it, you'll need to run `make roles` to make [ansible-galaxy](https://docs.ansible.com/ansible/latest/cli/ansible-galaxy.html) download dependency roles (see the [`requirements.yml` file](requirements.yml)) to the `roles/galaxy` directory. Without this, the playbook won't work. + +We're in the process of trimming the playbook and making it reuse Ansible roles. + +Starting now, the playbook is composed of 2 types of Ansible roles: + +- those that live within the playbook itself (`roles/custom/*`) + +- those downloaded from other sources (using [ansible-galaxy](https://docs.ansible.com/ansible/latest/cli/ansible-galaxy.html) to `roles/galaxy`, based on the [`requirements.yml` file](requirements.yml)). These roles are maintained by us or by other people from the Ansible community. + +We're doing this for greater code-reuse (across Ansible playbooks, including our own related playbooks [gitea-docker-ansible-deploy](https://github.com/spantaleev/gitea-docker-ansible-deploy) and [nextcloud-docker-ansible-deploy](https://github.com/spantaleev/nextcloud-docker-ansible-deploy)) and decreased maintenance burden. Until now, certain features were copy-pasted across playbooks or were maintained separately in each one, with improvements often falling behind. We've also tended to do too much by ourselves - installing Docker on the server from our `matrix-base` role, etc. - something that we'd rather not do anymore by switching to the [geerlingguy.docker](https://galaxy.ansible.com/geerlingguy/docker) role. + +Some variable names will change during the transition to having more and more external (galaxy) roles. There's a new `custom/matrix_playbook_migration` role added to the playbook which will tell you about these changes each time you run the playbook. + +From now on, every time you update the playbook (well, every time the `requirements.yml` file changes), it's best to run `make roles` to update the roles downloaded from other sources. + + # 2022-10-14 ## synapse-s3-storage-provider support diff --git a/Makefile b/Makefile index 62419150c..b7fc41bd7 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,7 @@ help: ## Show this help. @grep -F -h "##" $(MAKEFILE_LIST) | grep -v grep | sed -e 's/\\$$//' | sed -e 's/##//' roles: ## Pull roles + rm -rf roles/galaxy ansible-galaxy install -r requirements.yml -p roles/galaxy/ --force lint: ## Runs ansible-lint against all roles in the playbook diff --git a/docs/installing.md b/docs/installing.md index 308e6fe66..7c62cc393 100644 --- a/docs/installing.md +++ b/docs/installing.md @@ -2,6 +2,9 @@ If you've [configured your DNS](configuring-dns.md) and have [configured the playbook](configuring-playbook.md), you can start the installation procedure. +**Before installing** and each time you update the playbook in the future, you will need to update the Ansible roles in this playbook by running `make roles`. + + ## Playbook tags introduction The Ansible playbook's tasks are tagged, so that certain parts of the Ansible playbook can be run without running all other tasks. diff --git a/docs/maintenance-upgrading-services.md b/docs/maintenance-upgrading-services.md index fe289a86f..d1c707fd6 100644 --- a/docs/maintenance-upgrading-services.md +++ b/docs/maintenance-upgrading-services.md @@ -10,8 +10,8 @@ To upgrade services: - take a look at [the changelog](../CHANGELOG.md) to see if there have been any backward-incompatible changes that you need to take care of -- re-run the [playbook setup](installing.md): `ansible-playbook -i inventory/hosts setup.yml --tags=setup-all` +- download the upstream Ansible roles used by the playbook by running `make roles` -- restart the services: `ansible-playbook -i inventory/hosts setup.yml --tags=start` +- re-run the [playbook setup](installing.md) and restart all serivces: `ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,ensure-matrix-users-created,start` **Note**: major version upgrades to the internal PostgreSQL database are not done automatically. To upgrade it, refer to the [upgrading PostgreSQL guide](maintenance-postgres.md#upgrading-postgresql). diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 2e3a217c0..a75be04a7 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -9,6 +9,31 @@ # You can also override ANY variable (seen here or in any given role), # by re-defining it in your own configuration file (`inventory/host_vars/matrix.`). + + +###################################################################### +# +# com.devture.ansible.role.playbook_state_preserver +# +###################################################################### + +# To completely disable this feature, use `devture_playbook_state_preserver_enabled: false`. + +devture_playbook_state_preserver_uid: "{{ matrix_user_uid }}" +devture_playbook_state_preserver_gid: "{{ matrix_user_gid }}" + +devture_playbook_state_preserver_vars_preservation_dst: "{{ matrix_base_data_path }}/vars.yml" + +devture_playbook_state_preserver_commit_hash_preservation_dst: "{{ matrix_base_data_path }}/git_hash.yml" + +###################################################################### +# +# /com.devture.ansible.role.playbook_state_preserver +# +###################################################################### + + + ###################################################################### # # matrix-base diff --git a/requirements.yml b/requirements.yml new file mode 100644 index 000000000..9d6669af8 --- /dev/null +++ b/requirements.yml @@ -0,0 +1,13 @@ +--- + +- src: git+https://github.com/devture/com.devture.ansible.role.playbook_help.git + version: c1f40e82b4d6b072b6f0e885239322bdaaaf554f + +- src: git+https://github.com/devture/com.devture.ansible.role.systemd_docker_base.git + version: 327d2e17f5189ac2480d6012f58cf64a2b46efba + +- src: git+https://github.com/devture/com.devture.ansible.role.playbook_state_preserver.git + version: 0857450721d525238ca230c9e6f8f8ad3a248564 + +- src: git+https://github.com/devture/com.devture.ansible.role.playbook_runtime_messages.git + version: f1c78d4e85e875129790c58335d0e44385683f6b diff --git a/roles/custom/matrix-base/defaults/main.yml b/roles/custom/matrix-base/defaults/main.yml index 52049ed51..a18ba6b6b 100644 --- a/roles/custom/matrix-base/defaults/main.yml +++ b/roles/custom/matrix-base/defaults/main.yml @@ -255,12 +255,6 @@ matrix_well_known_matrix_support_configuration: "{{ matrix_well_known_matrix_sup # The Docker network that all services would be put into matrix_docker_network: "matrix" -# Controls whether we'll preserve the vars.yml file on the Matrix server. -# If you have a differently organized inventory, you may wish to disable this feature, -# or to repoint `matrix_vars_yml_snapshotting_src` to the file you'd like to preserve. -matrix_vars_yml_snapshotting_enabled: true -matrix_vars_yml_snapshotting_src: "{{ inventory_dir }}/host_vars/{{ inventory_hostname }}/vars.yml" - # Controls whether a `/.well-known/matrix/server` file is generated and used at all. # # If you wish to rely on DNS SRV records only, you can disable this. @@ -284,11 +278,6 @@ matrix_docker_installation_enabled: true # Possible values are "docker-ce" (default) and "docker.io" (Debian). matrix_docker_package_name: docker-ce -# Controls whether the current playbook's commit hash is saved in `git_hash.yml` on the target -# Set this to false if GIT is not installed on the local system (the system where the ansible command is run on) -# to suppress the warning message. -matrix_playbook_commit_hash_preservation_enabled: true - # Variables to Control which parts of our roles run. run_postgres_import: true run_postgres_upgrade: true diff --git a/roles/custom/matrix-base/tasks/setup_matrix_base.yml b/roles/custom/matrix-base/tasks/setup_matrix_base.yml index 2439fdea3..f954bd796 100644 --- a/roles/custom/matrix-base/tasks/setup_matrix_base.yml +++ b/roles/custom/matrix-base/tasks/setup_matrix_base.yml @@ -10,61 +10,6 @@ with_items: - "{{ matrix_base_data_path }}" -- name: Preserve vars.yml on the server for easily restoring if it gets lost later on - ansible.builtin.copy: - src: "{{ matrix_vars_yml_snapshotting_src }}" - dest: "{{ matrix_base_data_path }}/vars.yml" - owner: "{{ matrix_user_username }}" - group: "{{ matrix_user_groupname }}" - mode: '0660' - when: "matrix_vars_yml_snapshotting_enabled | bool" - -- name: Save current git-repo status on the target to aid with restoring in case of problems - when: "matrix_playbook_commit_hash_preservation_enabled|bool" - block: - - name: Get local git hash # noqa command-instead-of-module - delegate_to: 127.0.0.1 - become: false - register: git_describe - changed_when: false - ansible.builtin.shell: - git describe - --always - --tags - --dirty - --long - --all - - - ansible.builtin.set_fact: - git_hash: "{{ git_describe.stdout }}" - - - name: Git hash - ansible.builtin.debug: - msg: "Git hash: {{ git_hash }}" - - - name: Save git_hash.yml on target - ansible.builtin.copy: - content: "{{ git_hash }}" - dest: "{{ matrix_base_data_path }}/git_hash.yml" - owner: "{{ matrix_user_username }}" - group: "{{ matrix_user_groupname }}" - mode: '0660' - - rescue: - - name: GIT not found error - ansible.builtin.debug: - msg: >- - Couldn't find GIT on the local machine. Continuing without saving the GIT hash. - You can disable saving the GIT hash by setting 'matrix_playbook_commit_hash_preservation_enabled: false' in vars.yml - when: "git_describe.stderr.find('git: not found') != -1" - - - name: Get GIT hash error - ansible.builtin.fail: - msg: >- - Error when trying to get the GIT hash. Please consult the error message above. - You can disable saving the GIT hash by setting 'matrix_playbook_commit_hash_preservation_enabled: false' in vars.yml - when: "git_describe.stderr.find('git: not found') == -1" - - name: Ensure Matrix network is created in Docker community.docker.docker_network: name: "{{ matrix_docker_network }}" diff --git a/roles/custom/matrix-bridge-hookshot/defaults/main.yml b/roles/custom/matrix-bridge-hookshot/defaults/main.yml index 4c39876a2..4e6965847 100644 --- a/roles/custom/matrix-bridge-hookshot/defaults/main.yml +++ b/roles/custom/matrix-bridge-hookshot/defaults/main.yml @@ -129,8 +129,8 @@ matrix_hookshot_generic_user_id_prefix: '_webhooks_' matrix_hookshot_feeds_enabled: true -matrix_hookshot_feeds_pollIntervalSeconds: 600 # no-qa var-naming -matrix_hookshot_feeds_pollTimeoutSeconds: 10 # no-qa var-naming +matrix_hookshot_feeds_pollIntervalSeconds: 600 # noqa var-naming +matrix_hookshot_feeds_pollTimeoutSeconds: 10 # noqa var-naming # There is no need to edit ports. use matrix_hookshot_container_http_host_bind_ports below to expose ports instead. diff --git a/roles/custom/matrix_playbook_migration/tasks/main.yml b/roles/custom/matrix_playbook_migration/tasks/main.yml new file mode 100644 index 000000000..c346a759b --- /dev/null +++ b/roles/custom/matrix_playbook_migration/tasks/main.yml @@ -0,0 +1,5 @@ +--- + +- ansible.builtin.import_tasks: "{{ role_path }}/tasks/validate_config.yml" + tags: + - setup-all diff --git a/roles/custom/matrix_playbook_migration/tasks/validate_config.yml b/roles/custom/matrix_playbook_migration/tasks/validate_config.yml new file mode 100644 index 000000000..111016c9c --- /dev/null +++ b/roles/custom/matrix_playbook_migration/tasks/validate_config.yml @@ -0,0 +1,12 @@ +--- + +- name: (Deprecation) Catch and report renamed Matrix playbook settings + ansible.builtin.fail: + msg: >- + Your configuration contains a variable, which now has a different name. + Please change your configuration to rename the variable (`{{ item.old }}` -> `{{ item.new }}`). + when: "item.old in vars" + with_items: + - {'old': 'matrix_vars_yml_snapshotting_enabled', 'new': 'devture_playbook_state_preserver_vars_preservation_enabled'} + - {'old': 'matrix_vars_yml_snapshotting_src', 'new': 'devture_playbook_state_preserver_vars_preservation_src'} + - {'old': 'matrix_playbook_commit_hash_preservation_enabled', 'new': 'devture_playbook_state_preserver_commit_hash_preservation_enabled'} diff --git a/setup.yml b/setup.yml index 906c3bdf4..24f5754bb 100755 --- a/setup.yml +++ b/setup.yml @@ -7,6 +7,14 @@ - roles/custom/matrix-synapse/vars/workers.yml roles: + # This role has no tasks at all + - role: galaxy/com.devture.ansible.role.playbook_help + + # This role has no tasks at all + - role: galaxy/com.devture.ansible.role.systemd_docker_base + + - role: custom/matrix_playbook_migration + - custom/matrix-base - custom/matrix-dynamic-dns - custom/matrix-mailer @@ -74,3 +82,12 @@ - custom/matrix-backup-borg - custom/matrix-user-creator - custom/matrix-common-after + + # This is pretty much last, because we want it to better serve as a "last known good configuration". + # See: https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/2217#issuecomment-1301487601 + - when: devture_playbook_state_preserver_enabled | bool + role: galaxy/com.devture.ansible.role.playbook_state_preserver + tags: + - setup-all + + - role: galaxy/com.devture.ansible.role.playbook_runtime_messages