diff --git a/docs/configuring-awx-system.md b/docs/configuring-awx-system.md index cc90fb98f..1ab6950a3 100644 --- a/docs/configuring-awx-system.md +++ b/docs/configuring-awx-system.md @@ -8,9 +8,7 @@ Members can be assigned a server from Digitalocean, or they can connect their ow The AWX system is arranged into 'members' each with their own 'subscriptions'. After creating a subscription the user enters the 'provision stage' where they defined the URLs they will use, the servers location and whether or not there's already a website at the base domain. They then proceed onto the 'deploy stage' where they can configure their Matrix server. -Ideally this system can manage the updates, configuration, backups and monitoring on it's own. It is an extension of the popular deploy script [spantaleev/matrix-docker-ansible-deploy](https://github.com/spantaleev/matrix-docker-ansible-deploy). - -Warning: This project is currently alpha quality and should only be run by the brave. +This system can manage the updates, configuration, import and export, backups and monitoring on its own. It is an extension of the popular deploy script [spantaleev/matrix-docker-ansible-deploy](https://github.com/spantaleev/matrix-docker-ansible-deploy). ## Other Required Playbooks @@ -23,6 +21,7 @@ The following repositories allow you to copy and use this setup: [Ansible Provision Server](https://gitlab.com/GoMatrixHosting/ansible-provision-server) - Used by AWX members to perform initial configuration of their DigitalOcean or On-Premises server. + ## Testing Fork For This Playbook Updates to this section are trailed here: diff --git a/roles/matrix-awx/scripts/matrix_build_room_list.py b/roles/matrix-awx/scripts/matrix_build_room_list.py new file mode 100644 index 000000000..3abfcd8c9 --- /dev/null +++ b/roles/matrix-awx/scripts/matrix_build_room_list.py @@ -0,0 +1,28 @@ + +import sys +import requests +import json + +janitor_token = sys.argv[1] +synapse_container_ip = sys.argv[2] + +# collect total amount of rooms + +rooms_raw_url = 'http://' + synapse_container_ip + ':8008/_synapse/admin/v1/rooms' +rooms_raw_header = {'Authorization': 'Bearer ' + janitor_token} +rooms_raw = requests.get(rooms_raw_url, headers=rooms_raw_header) +rooms_raw_python = json.loads(rooms_raw.text) +total_rooms = rooms_raw_python["total_rooms"] + +# build complete room list file + +room_list_file = open("/tmp/room_list_complete.json", "w") + +for i in range(0, total_rooms, 100): + rooms_inc_url = 'http://' + synapse_container_ip + ':8008/_synapse/admin/v1/rooms?from=' + str(i) + rooms_inc = requests.get(rooms_inc_url, headers=rooms_raw_header) + room_list_file.write(rooms_inc.text) + +room_list_file.close() + +print(total_rooms) diff --git a/roles/matrix-awx/tasks/purge_database_main.yml b/roles/matrix-awx/tasks/purge_database_main.yml index ccd46c81a..76a437e14 100644 --- a/roles/matrix-awx/tasks/purge_database_main.yml +++ b/roles/matrix-awx/tasks/purge_database_main.yml @@ -17,136 +17,132 @@ file: '/var/lib/awx/projects/clients/{{ member_id }}/{{ subscription_id }}/matrix_vars.yml' no_log: True -- name: Collect size of Synapse database +- name: Collect before shrink size of Synapse database shell: du -sh /matrix/postgres/data register: db_size_before_stat + when: (purge_mode.find("Perform final shrink") != -1) no_log: True -- name: Print before size of Synapse database - debug: - msg: "{{ db_size_before_stat.stdout.split('\n') }}" - when: db_size_before_stat is defined - - name: Collect the internal IP of the matrix-synapse container shell: "/usr/bin/docker inspect --format '{''{range.NetworkSettings.Networks}''}{''{.IPAddress}''}{''{end}''}' matrix-synapse" + when: (purge_mode.find("No local users [recommended]") != -1) or (purge_mode.find("Number of users [slower]") != -1) or (purge_mode.find("Number of events [slower]") != -1) register: synapse_container_ip - name: Collect access token for janitor user shell: | curl -X POST -d '{"type":"m.login.password", "user":"janitor", "password":"{{ matrix_awx_janitor_user_password }}"}' "{{ synapse_container_ip.stdout }}:8008/_matrix/client/r0/login" | jq '.access_token' + when: (purge_mode.find("No local users [recommended]") != -1) or (purge_mode.find("Number of users [slower]") != -1) or (purge_mode.find("Number of events [slower]") != -1) register: janitors_token no_log: True -- name: Collect total number of rooms +- name: Copy build_room_list.py script to target machine + copy: + src: ./roles/matrix-awx/scripts/matrix_build_room_list.py + dest: /usr/local/bin/matrix_build_room_list.py + owner: matrix + group: matrix + mode: '0755' + when: (purge_mode.find("No local users [recommended]") != -1) or (purge_mode.find("Number of users [slower]") != -1) or (purge_mode.find("Number of events [slower]") != -1) + +- name: Run build_room_list.py script shell: | - curl -X GET --header "Authorization: Bearer {{ janitors_token.stdout[1:-1] }}" '{{ synapse_container_ip.stdout }}:8008/_synapse/admin/v1/rooms' | jq '.total_rooms' - when: purge_rooms|bool + runuser -u matrix -- python3 /usr/local/bin/matrix_build_room_list.py {{ janitors_token.stdout[1:-1] }} {{ synapse_container_ip.stdout }} register: rooms_total + when: (purge_mode.find("No local users [recommended]") != -1) or (purge_mode.find("Number of users [slower]") != -1) or (purge_mode.find("Number of events [slower]") != -1) -- name: Print total number of rooms - debug: - msg: '{{ rooms_total.stdout }}' - when: purge_rooms|bool +- name: Fetch complete room list from target machine + fetch: + src: /tmp/room_list_complete.json + dest: "/tmp/{{ subscription_id }}_room_list_complete.json" + flat: yes + when: (purge_mode.find("No local users [recommended]") != -1) or (purge_mode.find("Number of users [slower]") != -1) or (purge_mode.find("Number of events [slower]") != -1) -- name: Calculate every 100 values for total number of rooms - delegate_to: 127.0.0.1 - shell: | - seq 0 100 {{ rooms_total.stdout }} - when: purge_rooms|bool - register: every_100_rooms - -- name: Ensure room_list_complete.json file exists - delegate_to: 127.0.0.1 +- name: Remove complete room list from target machine file: - path: /tmp/{{ subscription_id }}_room_list_complete.json - state: touch - when: purge_rooms|bool - -- name: Build file with total room list - include_tasks: purge_database_build_list.yml - loop: "{{ every_100_rooms.stdout_lines | flatten(levels=1) }}" - when: purge_rooms|bool + path: /tmp/room_list_complete.json + state: absent + when: (purge_mode.find("No local users [recommended]") != -1) or (purge_mode.find("Number of users [slower]") != -1) or (purge_mode.find("Number of events [slower]") != -1) - name: Generate list of rooms with no local users delegate_to: 127.0.0.1 shell: | jq 'try .rooms[] | select(.joined_local_members == 0) | .room_id' < /tmp/{{ subscription_id }}_room_list_complete.json > /tmp/{{ subscription_id }}_room_list_no_local_users.txt - when: purge_rooms|bool + when: (purge_mode.find("No local users [recommended]") != -1) or (purge_mode.find("Number of users [slower]") != -1) or (purge_mode.find("Number of events [slower]") != -1) - name: Count number of rooms with no local users delegate_to: 127.0.0.1 shell: | wc -l /tmp/{{ subscription_id }}_room_list_no_local_users.txt | awk '{ print $1 }' register: rooms_no_local_total - when: purge_rooms|bool + when: (purge_mode.find("No local users [recommended]") != -1) or (purge_mode.find("Number of users [slower]") != -1) or (purge_mode.find("Number of events [slower]") != -1) - name: Setting host fact room_list_no_local_users set_fact: room_list_no_local_users: "{{ lookup('file', '/tmp/{{ subscription_id }}_room_list_no_local_users.txt') }}" no_log: True - when: purge_rooms|bool + when: (purge_mode.find("No local users [recommended]") != -1) or (purge_mode.find("Number of users [slower]") != -1) or (purge_mode.find("Number of events [slower]") != -1) - name: Purge all rooms with no local users include_tasks: purge_database_no_local.yml loop: "{{ room_list_no_local_users.splitlines() | flatten(levels=1) }}" - when: purge_rooms|bool + when: (purge_mode.find("No local users [recommended]") != -1) or (purge_mode.find("Number of users [slower]") != -1) or (purge_mode.find("Number of events [slower]") != -1) - name: Collect epoche time from date delegate_to: 127.0.0.1 shell: | date -d '{{ purge_date }}' +"%s" - when: purge_rooms|bool + when: (purge_mode.find("Number of users [slower]") != -1) or (purge_mode.find("Number of events [slower]") != -1) register: purge_epoche_time - name: Generate list of rooms with more then N users delegate_to: 127.0.0.1 shell: | jq 'try .rooms[] | select(.joined_members > {{ purge_metric_value }}) | .room_id' < /tmp/{{ subscription_id }}_room_list_complete.json > /tmp/{{ subscription_id }}_room_list_joined_members.txt - when: (purge_metric.find("Number of users") != -1) and (purge_rooms|bool) + when: purge_mode.find("Number of users [slower]") != -1 - name: Count number of rooms with more then N users delegate_to: 127.0.0.1 shell: | wc -l /tmp/{{ subscription_id }}_room_list_joined_members.txt | awk '{ print $1 }' register: rooms_join_members_total - when: (purge_metric.find("Number of users") != -1) and (purge_rooms|bool) + when: purge_mode.find("Number of users [slower]") != -1 - name: Setting host fact room_list_joined_members delegate_to: 127.0.0.1 set_fact: room_list_joined_members: "{{ lookup('file', '/tmp/{{ subscription_id }}_room_list_joined_members.txt') }}" - when: (purge_metric.find("Number of users") != -1) and (purge_rooms|bool) + when: purge_mode.find("Number of users [slower]") != -1 no_log: True - name: Purge all rooms with more then N users include_tasks: purge_database_users.yml loop: "{{ room_list_joined_members.splitlines() | flatten(levels=1) }}" - when: (purge_metric.find("Number of users") != -1) and (purge_rooms|bool) + when: purge_mode.find("Number of users [slower]") != -1 - name: Generate list of rooms with more then N events delegate_to: 127.0.0.1 shell: | jq 'try .rooms[] | select(.state_events > {{ purge_metric_value }}) | .room_id' < /tmp/{{ subscription_id }}_room_list_complete.json > /tmp/{{ subscription_id }}_room_list_state_events.txt - when: (purge_metric.find("Number of events") != -1) and (purge_rooms|bool) + when: purge_mode.find("Number of events [slower]") != -1 -- name: Count number of rooms with more then N users +- name: Count number of rooms with more then N events delegate_to: 127.0.0.1 shell: | wc -l /tmp/{{ subscription_id }}_room_list_state_events.txt | awk '{ print $1 }' register: rooms_state_events_total - when: (purge_metric.find("Number of events") != -1) and (purge_rooms|bool) + when: purge_mode.find("Number of events [slower]") != -1 - name: Setting host fact room_list_state_events delegate_to: 127.0.0.1 set_fact: room_list_state_events: "{{ lookup('file', '/tmp/{{ subscription_id }}_room_list_state_events.txt') }}" - when: (purge_metric.find("Number of events") != -1) and (purge_rooms|bool) + when: purge_mode.find("Number of events [slower]") != -1 no_log: True - name: Purge all rooms with more then N events include_tasks: purge_database_events.yml loop: "{{ room_list_state_events.splitlines() | flatten(levels=1) }}" - when: (purge_metric.find("Number of events") != -1) and (purge_rooms|bool) + when: purge_mode.find("Number of events [slower]") != -1 - name: Collect AWX admin token the hard way! delegate_to: 127.0.0.1 @@ -155,75 +151,162 @@ register: tower_token no_log: True +- name: Adjust 'Deploy/Update a Server' job template + delegate_to: 127.0.0.1 + awx.awx.tower_job_template: + name: "{{ matrix_domain }} - 0 - Deploy/Update a Server" + description: "Creates a new matrix service with Spantaleev's playbooks" + extra_vars: "{{ lookup('file', '/var/lib/awx/projects/clients/{{ member_id }}/{{ subscription_id }}/extra_vars.json') }}" + job_type: run + job_tags: "rust-synapse-compress-state" + inventory: "{{ member_id }}" + project: "{{ member_id }} - Matrix Docker Ansible Deploy" + playbook: setup.yml + credential: "{{ member_id }} - AWX SSH Key" + state: present + verbosity: 1 + tower_host: "https://{{ tower_host }}" + tower_oauthtoken: "{{ tower_token.stdout }}" + validate_certs: yes + when: (purge_mode.find("No local users [recommended]") != -1) or (purge_mode.find("Number of users [slower]") != -1) or (purge_mode.find("Number of events [slower]") != -1) or (purge_mode.find("Skip purging rooms [faster]") != -1) + - name: Execute rust-synapse-compress-state job template delegate_to: 127.0.0.1 awx.awx.tower_job_launch: job_template: "{{ matrix_domain }} - 0 - Deploy/Update a Server" - tags: "rust-synapse-compress-state" wait: yes tower_host: "https://{{ tower_host }}" tower_oauthtoken: "{{ tower_token.stdout }}" validate_certs: yes - register: job + when: (purge_mode.find("No local users [recommended]") != -1) or (purge_mode.find("Number of users [slower]") != -1) or (purge_mode.find("Number of events [slower]") != -1) or (purge_mode.find("Skip purging rooms [faster]") != -1) -- name: Stop Synapse service - shell: systemctl stop matrix-synapse.service +- name: Revert 'Deploy/Update a Server' job template + delegate_to: 127.0.0.1 + awx.awx.tower_job_template: + name: "{{ matrix_domain }} - 0 - Deploy/Update a Server" + description: "Creates a new matrix service with Spantaleev's playbooks" + extra_vars: "{{ lookup('file', '/var/lib/awx/projects/clients/{{ member_id }}/{{ subscription_id }}/extra_vars.json') }}" + job_type: run + job_tags: "setup-all,start" + inventory: "{{ member_id }}" + project: "{{ member_id }} - Matrix Docker Ansible Deploy" + playbook: setup.yml + credential: "{{ member_id }} - AWX SSH Key" + state: present + verbosity: 1 + tower_host: "https://{{ tower_host }}" + tower_oauthtoken: "{{ tower_token.stdout }}" + validate_certs: yes + when: (purge_mode.find("No local users [recommended]") != -1) or (purge_mode.find("Number of users [slower]") != -1) or (purge_mode.find("Number of events [slower]") != -1) or (purge_mode.find("Skip purging rooms [faster]") != -1) + +- name: Ensure matrix-synapse is stopped + service: + name: matrix-synapse + state: stopped + daemon_reload: yes + when: (purge_mode.find("Perform final shrink") != -1) - name: Re-index Synapse database shell: docker exec -i matrix-postgres psql "host=127.0.0.1 port=5432 dbname=synapse user=synapse password={{ matrix_synapse_connection_password }}" -c 'REINDEX (VERBOSE) DATABASE synapse' + when: (purge_mode.find("Perform final shrink") != -1) + +- name: Ensure matrix-synapse is started + service: + name: matrix-synapse + state: started + daemon_reload: yes + when: (purge_mode.find("Perform final shrink") != -1) + +- name: Adjust 'Deploy/Update a Server' job template + delegate_to: 127.0.0.1 + awx.awx.tower_job_template: + name: "{{ matrix_domain }} - 0 - Deploy/Update a Server" + description: "Creates a new matrix service with Spantaleev's playbooks" + extra_vars: "{{ lookup('file', '/var/lib/awx/projects/clients/{{ member_id }}/{{ subscription_id }}/extra_vars.json') }}" + job_type: run + job_tags: "run-postgres-vacuum,start" + inventory: "{{ member_id }}" + project: "{{ member_id }} - Matrix Docker Ansible Deploy" + playbook: setup.yml + credential: "{{ member_id }} - AWX SSH Key" + state: present + verbosity: 1 + tower_host: "https://{{ tower_host }}" + tower_oauthtoken: "{{ tower_token.stdout }}" + validate_certs: yes + when: (purge_mode.find("Perform final shrink") != -1) - name: Execute run-postgres-vacuum job template delegate_to: 127.0.0.1 awx.awx.tower_job_launch: job_template: "{{ matrix_domain }} - 0 - Deploy/Update a Server" - tags: "run-postgres-vacuum,start" wait: yes tower_host: "https://{{ tower_host }}" tower_oauthtoken: "{{ tower_token.stdout }}" validate_certs: yes - register: job + when: (purge_mode.find("Perform final shrink") != -1) + +- name: Revert 'Deploy/Update a Server' job template + delegate_to: 127.0.0.1 + awx.awx.tower_job_template: + name: "{{ matrix_domain }} - 0 - Deploy/Update a Server" + description: "Creates a new matrix service with Spantaleev's playbooks" + extra_vars: "{{ lookup('file', '/var/lib/awx/projects/clients/{{ member_id }}/{{ subscription_id }}/extra_vars.json') }}" + job_type: run + job_tags: "setup-all,start" + inventory: "{{ member_id }}" + project: "{{ member_id }} - Matrix Docker Ansible Deploy" + playbook: setup.yml + credential: "{{ member_id }} - AWX SSH Key" + state: present + verbosity: 1 + tower_host: "https://{{ tower_host }}" + tower_oauthtoken: "{{ tower_token.stdout }}" + validate_certs: yes + when: (purge_mode.find("Perform final shrink") != -1) - name: Cleanup room_list files delegate_to: 127.0.0.1 shell: | rm /tmp/{{ subscription_id }}_room_list* - when: purge_rooms|bool + when: (purge_mode.find("No local users [recommended]") != -1) or (purge_mode.find("Number of users [slower]") != -1) or (purge_mode.find("Number of events [slower]") != -1) ignore_errors: yes -- name: Collect size of Synapse database +- name: Collect after shrink size of Synapse database shell: du -sh /matrix/postgres/data register: db_size_after_stat + when: (purge_mode.find("Perform final shrink") != -1) no_log: True - name: Print total number of rooms processed debug: msg: '{{ rooms_total.stdout }}' - when: purge_rooms|bool + when: (purge_mode.find("No local users [recommended]") != -1) or (purge_mode.find("Number of users [slower]") != -1) or (purge_mode.find("Number of events [slower]") != -1) - name: Print the number of rooms purged with no local users debug: msg: '{{ rooms_no_local_total.stdout }}' - when: purge_rooms|bool + when: (purge_mode.find("No local users [recommended]") != -1) or (purge_mode.find("Number of users [slower]") != -1) or (purge_mode.find("Number of events [slower]") != -1) - name: Print the number of rooms purged with more then N users debug: msg: '{{ rooms_join_members_total.stdout }}' - when: (purge_metric.find("Number of users") != -1) and (purge_rooms|bool) + when: purge_mode.find("Number of users") != -1 - name: Print the number of rooms purged with more then N events debug: msg: '{{ rooms_state_events_total.stdout }}' - when: (purge_metric.find("Number of events") != -1) and (purge_rooms|bool) + when: purge_mode.find("Number of events") != -1 - name: Print before purge size of Synapse database debug: msg: "{{ db_size_before_stat.stdout.split('\n') }}" - when: db_size_before_stat is defined + when: (db_size_before_stat is defined) and (purge_mode.find("Perform final shrink") != -1) - name: Print after purge size of Synapse database debug: msg: "{{ db_size_after_stat.stdout.split('\n') }}" - when: db_size_after_stat is defined + when: (db_size_after_stat is defined) and (purge_mode.find("Perform final shrink") != -1) - name: Set boolean value to exit playbook set_fact: diff --git a/roles/matrix-synapse/tasks/rust-synapse-compress-state/main.yml b/roles/matrix-synapse/tasks/rust-synapse-compress-state/main.yml index eef46cb37..106c59d5b 100644 --- a/roles/matrix-synapse/tasks/rust-synapse-compress-state/main.yml +++ b/roles/matrix-synapse/tasks/rust-synapse-compress-state/main.yml @@ -10,7 +10,7 @@ - name: Set matrix_synapse_rust_synapse_compress_state_find_rooms_command_wait_time, if not provided set_fact: - matrix_synapse_rust_synapse_compress_state_find_rooms_command_wait_time: 180 + matrix_synapse_rust_synapse_compress_state_find_rooms_command_wait_time: 300 when: "matrix_synapse_rust_synapse_compress_state_find_rooms_command_wait_time|default('') == ''" - name: Set matrix_synapse_rust_synapse_compress_state_compress_room_time, if not provided