Merge branch 'spantaleev:master' into master

This commit is contained in:
sakkiii 2021-05-14 18:54:47 +05:30
commit 322b750aad
16 changed files with 327 additions and 10 deletions

View File

@ -129,3 +129,8 @@ When updating the playbook, refer to [the changelog](CHANGELOG.md) to catch up w
- IRC channel: `#matrix-docker-ansible-deploy` on the [Freenode](https://freenode.net/) IRC network (irc.freenode.net) - IRC channel: `#matrix-docker-ansible-deploy` on the [Freenode](https://freenode.net/) IRC network (irc.freenode.net)
- Github issues: [spantaleev/matrix-docker-ansible-deploy/issues](https://github.com/spantaleev/matrix-docker-ansible-deploy/issues) - Github issues: [spantaleev/matrix-docker-ansible-deploy/issues](https://github.com/spantaleev/matrix-docker-ansible-deploy/issues)
## Services by the community
- [etke.cc](https://etke.cc) - matrix-docker-ansible-deploy and system stuff "as a service". That service will create your matrix homeserver on your domain and server (doesn't matter if it's cloud provider or on old laptop in the corner of your room), (optional) maintains it (server's system updates, cleanup, security adjustments, tuning, etc.; matrix homeserver updates & maintainance) and (optional) provide full-featured email service for your domain
- [GoMatrixHosting](https://gomatrixhosting.com) - matrix-docker-ansible-deploy "as a service" with [Ansible AWX](https://github.com/ansible/awx). Members can be assigned a server from Digitalocean, or they can connect their own on-premises server. This AWX system can manage the updates, configuration, import and export, backups and monitoring on its own. For more information [see our GitLab group](https://gitlab.com/GoMatrixHosting) or come [visit us on Matrix](https://matrix.to/#/#general:gomatrixhosting.com).

View File

@ -32,9 +32,9 @@ Updates to this section are trailed here:
## Does I need an AWX setup to use this? How do I configure it? ## Does I need an AWX setup to use this? How do I configure it?
Yes, you'll need to configure an AWX instance, the [Create AWX System](https://gitlab.com/GoMatrixHosting/create-awx-system) repository makes it easy to do. Just follow the steps listed in '/docs/Installation.md' of that repository. Yes, you'll need to configure an AWX instance, the [Create AWX System](https://gitlab.com/GoMatrixHosting/create-awx-system) repository makes it easy to do. Just follow the steps listed in ['/docs/Installation.md' of that repository](https://gitlab.com/GoMatrixHosting/create-awx-system/-/blob/master/docs/Installation.md).
For simpler installation steps you can use to get started with this system, check out our minimal installation guide at '/doc/Installation_Minimal.md'. For simpler installation steps you can use to get started with this system, check out our minimal installation guide at ['/doc/Installation_Minimal.md of that repository'](https://gitlab.com/GoMatrixHosting/create-awx-system/-/blob/master/docs/Installation_Minimal.md).
## Does I need a front-end WordPress site? And a DigitalOcean account? ## Does I need a front-end WordPress site? And a DigitalOcean account?

View File

@ -1113,7 +1113,9 @@ matrix_ma1sd_synapsesql_connection: //{{ matrix_synapse_database_host }}/{{ matr
matrix_ma1sd_dns_overwrite_enabled: true matrix_ma1sd_dns_overwrite_enabled: true
matrix_ma1sd_dns_overwrite_homeserver_client_name: "{{ matrix_server_fqn_matrix }}" matrix_ma1sd_dns_overwrite_homeserver_client_name: "{{ matrix_server_fqn_matrix }}"
matrix_ma1sd_dns_overwrite_homeserver_client_value: "http://{{ matrix_nginx_proxy_proxy_matrix_client_api_addr_with_container }}" # The `matrix_ma1sd_dns_overwrite_homeserver_client_value` value when matrix_nginx_proxy_enabled is false covers the general case,
# but may be inaccurate if matrix-corporal is enabled.
matrix_ma1sd_dns_overwrite_homeserver_client_value: "{{ ('http://' + matrix_nginx_proxy_proxy_matrix_client_api_addr_with_container) if matrix_nginx_proxy_enabled else matrix_homeserver_container_url }}"
# By default, we send mail through the `matrix-mailer` service. # By default, we send mail through the `matrix-mailer` service.
matrix_ma1sd_threepid_medium_email_identity_from: "{{ matrix_mailer_sender_address }}" matrix_ma1sd_threepid_medium_email_identity_from: "{{ matrix_mailer_sender_address }}"

View File

@ -44,6 +44,15 @@
tags: tags:
- purge-media - purge-media
# Purge Synapse database if called
- include_tasks:
file: "purge_database_main.yml"
apply:
tags: purge-database
when: run_setup|bool and matrix_awx_enabled|bool
tags:
- purge-database
# Import configs, media repo from /chroot/backup import # Import configs, media repo from /chroot/backup import
- include_tasks: - include_tasks:
file: "import_awx.yml" file: "import_awx.yml"

View File

@ -0,0 +1,10 @@
- name: Collect entire room list into stdout
shell: |
curl -X GET --header "Authorization: Bearer {{ janitors_token.stdout[1:-1] }}" '{{ synapse_container_ip.stdout }}:8008/_synapse/admin/v1/rooms?from={{ item }}'
register: rooms_output
- name: Print stdout to file
delegate_to: 127.0.0.1
shell: |
echo '{{ rooms_output.stdout }}' >> /tmp/{{ subscription_id }}_room_list_complete.json

View File

@ -0,0 +1,13 @@
- name: Purge all rooms with more then N events
shell: |
curl --header "Authorization: Bearer {{ janitors_token.stdout[1:-1] }}" -X POST -H "Content-Type: application/json" -d '{ "delete_local_events": false, "purge_up_to_ts": {{ purge_epoche_time.stdout }}000 }' "{{ synapse_container_ip.stdout }}:8008/_synapse/admin/v1/purge_history/{{ item[1:-1] }}"
register: purge_command
- name: Print output of purge command
debug:
msg: "{{ purge_command.stdout }}"
- name: Pause for 5 seconds to let Synapse breathe
pause:
seconds: 5

View File

@ -0,0 +1,234 @@
- name: Ensure dateutils and curl is installed in AWX
delegate_to: 127.0.0.1
yum:
name: dateutils
state: latest
- name: Ensure dateutils, curl and jq intalled on target machine
apt:
pkg:
- curl
- jq
state: present
- name: Include vars in matrix_vars.yml
include_vars:
file: '/var/lib/awx/projects/clients/{{ member_id }}/{{ subscription_id }}/matrix_vars.yml'
no_log: True
- name: Collect size of Synapse database
shell: du -sh /matrix/postgres/data
register: db_size_before_stat
no_log: True
- name: Print before size of Synapse database
debug:
msg: "{{ db_size_before_stat.stdout.split('\n') }}"
when: db_size_before_stat is defined
- name: Collect the internal IP of the matrix-synapse container
shell: "/usr/bin/docker inspect --format '{''{range.NetworkSettings.Networks}''}{''{.IPAddress}''}{''{end}''}' matrix-synapse"
register: synapse_container_ip
- name: Collect access token for janitor user
shell: |
curl -X POST -d '{"type":"m.login.password", "user":"janitor", "password":"{{ matrix_awx_janitor_user_password }}"}' "{{ synapse_container_ip.stdout }}:8008/_matrix/client/r0/login" | jq '.access_token'
register: janitors_token
no_log: True
- name: Collect total number of rooms
shell: |
curl -X GET --header "Authorization: Bearer {{ janitors_token.stdout[1:-1] }}" '{{ synapse_container_ip.stdout }}:8008/_synapse/admin/v1/rooms' | jq '.total_rooms'
when: purge_rooms|bool
register: rooms_total
- name: Print total number of rooms
debug:
msg: '{{ rooms_total.stdout }}'
when: purge_rooms|bool
- name: Calculate every 100 values for total number of rooms
delegate_to: 127.0.0.1
shell: |
seq 0 100 {{ rooms_total.stdout }}
when: purge_rooms|bool
register: every_100_rooms
- name: Ensure room_list_complete.json file exists
delegate_to: 127.0.0.1
file:
path: /tmp/{{ subscription_id }}_room_list_complete.json
state: touch
when: purge_rooms|bool
- name: Build file with total room list
include_tasks: purge_database_build_list.yml
loop: "{{ every_100_rooms.stdout_lines | flatten(levels=1) }}"
when: purge_rooms|bool
- name: Generate list of rooms with no local users
delegate_to: 127.0.0.1
shell: |
jq 'try .rooms[] | select(.joined_local_members == 0) | .room_id' < /tmp/{{ subscription_id }}_room_list_complete.json > /tmp/{{ subscription_id }}_room_list_no_local_users.txt
when: purge_rooms|bool
- name: Count number of rooms with no local users
delegate_to: 127.0.0.1
shell: |
wc -l /tmp/{{ subscription_id }}_room_list_no_local_users.txt | awk '{ print $1 }'
register: rooms_no_local_total
when: purge_rooms|bool
- name: Setting host fact room_list_no_local_users
set_fact:
room_list_no_local_users: "{{ lookup('file', '/tmp/{{ subscription_id }}_room_list_no_local_users.txt') }}"
no_log: True
when: purge_rooms|bool
- name: Purge all rooms with no local users
include_tasks: purge_database_no_local.yml
loop: "{{ room_list_no_local_users.splitlines() | flatten(levels=1) }}"
when: purge_rooms|bool
- name: Collect epoche time from date
delegate_to: 127.0.0.1
shell: |
date -d '{{ purge_date }}' +"%s"
when: purge_rooms|bool
register: purge_epoche_time
- name: Generate list of rooms with more then N users
delegate_to: 127.0.0.1
shell: |
jq 'try .rooms[] | select(.joined_members > {{ purge_metric_value }}) | .room_id' < /tmp/{{ subscription_id }}_room_list_complete.json > /tmp/{{ subscription_id }}_room_list_joined_members.txt
when: (purge_metric.find("Number of users") != -1) and (purge_rooms|bool)
- name: Count number of rooms with more then N users
delegate_to: 127.0.0.1
shell: |
wc -l /tmp/{{ subscription_id }}_room_list_joined_members.txt | awk '{ print $1 }'
register: rooms_join_members_total
when: (purge_metric.find("Number of users") != -1) and (purge_rooms|bool)
- name: Setting host fact room_list_joined_members
delegate_to: 127.0.0.1
set_fact:
room_list_joined_members: "{{ lookup('file', '/tmp/{{ subscription_id }}_room_list_joined_members.txt') }}"
when: (purge_metric.find("Number of users") != -1) and (purge_rooms|bool)
no_log: True
- name: Purge all rooms with more then N users
include_tasks: purge_database_users.yml
loop: "{{ room_list_joined_members.splitlines() | flatten(levels=1) }}"
when: (purge_metric.find("Number of users") != -1) and (purge_rooms|bool)
- name: Generate list of rooms with more then N events
delegate_to: 127.0.0.1
shell: |
jq 'try .rooms[] | select(.state_events > {{ purge_metric_value }}) | .room_id' < /tmp/{{ subscription_id }}_room_list_complete.json > /tmp/{{ subscription_id }}_room_list_state_events.txt
when: (purge_metric.find("Number of events") != -1) and (purge_rooms|bool)
- name: Count number of rooms with more then N users
delegate_to: 127.0.0.1
shell: |
wc -l /tmp/{{ subscription_id }}_room_list_state_events.txt | awk '{ print $1 }'
register: rooms_state_events_total
when: (purge_metric.find("Number of events") != -1) and (purge_rooms|bool)
- name: Setting host fact room_list_state_events
delegate_to: 127.0.0.1
set_fact:
room_list_state_events: "{{ lookup('file', '/tmp/{{ subscription_id }}_room_list_state_events.txt') }}"
when: (purge_metric.find("Number of events") != -1) and (purge_rooms|bool)
no_log: True
- name: Purge all rooms with more then N events
include_tasks: purge_database_events.yml
loop: "{{ room_list_state_events.splitlines() | flatten(levels=1) }}"
when: (purge_metric.find("Number of events") != -1) and (purge_rooms|bool)
- name: Collect AWX admin token the hard way!
delegate_to: 127.0.0.1
shell: |
curl -sku {{ tower_username }}:{{ tower_password }} -H "Content-Type: application/json" -X POST -d '{"description":"Tower CLI", "application":null, "scope":"write"}' https://{{ tower_host }}/api/v2/users/1/personal_tokens/ | jq '.token' | sed -r 's/\"//g'
register: tower_token
no_log: True
- name: Execute rust-synapse-compress-state job template
delegate_to: 127.0.0.1
awx.awx.tower_job_launch:
job_template: "{{ matrix_domain }} - 0 - Deploy/Update a Server"
tags: "rust-synapse-compress-state"
wait: yes
tower_host: "https://{{ tower_host }}"
tower_oauthtoken: "{{ tower_token.stdout }}"
validate_certs: yes
register: job
- name: Stop Synapse service
shell: systemctl stop matrix-synapse.service
- name: Re-index Synapse database
shell: docker exec -i matrix-postgres psql "host=127.0.0.1 port=5432 dbname=synapse user=synapse password={{ matrix_synapse_connection_password }}" -c 'REINDEX (VERBOSE) DATABASE synapse'
- name: Execute run-postgres-vacuum job template
delegate_to: 127.0.0.1
awx.awx.tower_job_launch:
job_template: "{{ matrix_domain }} - 0 - Deploy/Update a Server"
tags: "run-postgres-vacuum,start"
wait: yes
tower_host: "https://{{ tower_host }}"
tower_oauthtoken: "{{ tower_token.stdout }}"
validate_certs: yes
register: job
- name: Cleanup room_list files
delegate_to: 127.0.0.1
shell: |
rm /tmp/{{ subscription_id }}_room_list*
when: purge_rooms|bool
ignore_errors: yes
- name: Collect size of Synapse database
shell: du -sh /matrix/postgres/data
register: db_size_after_stat
no_log: True
- name: Print total number of rooms processed
debug:
msg: '{{ rooms_total.stdout }}'
when: purge_rooms|bool
- name: Print the number of rooms purged with no local users
debug:
msg: '{{ rooms_no_local_total.stdout }}'
when: purge_rooms|bool
- name: Print the number of rooms purged with more then N users
debug:
msg: '{{ rooms_join_members_total.stdout }}'
when: (purge_metric.find("Number of users") != -1) and (purge_rooms|bool)
- name: Print the number of rooms purged with more then N events
debug:
msg: '{{ rooms_state_events_total.stdout }}'
when: (purge_metric.find("Number of events") != -1) and (purge_rooms|bool)
- name: Print before purge size of Synapse database
debug:
msg: "{{ db_size_before_stat.stdout.split('\n') }}"
when: db_size_before_stat is defined
- name: Print after purge size of Synapse database
debug:
msg: "{{ db_size_after_stat.stdout.split('\n') }}"
when: db_size_after_stat is defined
- name: Set boolean value to exit playbook
set_fact:
end_playbook: true
- name: End playbook early if this task is called.
meta: end_play
when: end_playbook is defined and end_playbook|bool

View File

@ -0,0 +1,13 @@
- name: Purge all rooms with no local users
shell: |
curl --header "Authorization: Bearer {{ janitors_token.stdout[1:-1] }}" -X POST -H "Content-Type: application/json" -d '{ "room_id": {{ item }} }' '{{ synapse_container_ip.stdout }}:8008/_synapse/admin/v1/purge_room'
register: purge_command
- name: Print output of purge command
debug:
msg: "{{ purge_command.stdout }}"
- name: Pause for 5 seconds to let Synapse breathe
pause:
seconds: 5

View File

@ -0,0 +1,13 @@
- name: Purge all rooms with more then N users
shell: |
curl --header "Authorization: Bearer {{ janitors_token.stdout[1:-1] }}" -X POST -H "Content-Type: application/json" -d '{ "delete_local_events": false, "purge_up_to_ts": {{ purge_epoche_time.stdout }}000 }' "{{ synapse_container_ip.stdout }}:8008/_synapse/admin/v1/purge_history/{{ item[1:-1] }}"
register: purge_command
- name: Print output of purge command
debug:
msg: "{{ purge_command.stdout }}"
- name: Pause for 5 seconds to let Synapse breathe
pause:
seconds: 5

View File

@ -4,7 +4,7 @@
date -d '{{ item }}' +"%s" date -d '{{ item }}' +"%s"
register: epoche_time register: epoche_time
- name: Purge local media to specific date - name: Purge remote media to specific date
shell: | shell: |
curl -X POST --header "Authorization: Bearer {{ janitors_token.stdout[1:-1] }}" '{{ synapse_container_ip.stdout }}:8008/_synapse/admin/v1/purge_media_cache?before_ts={{ epoche_time.stdout }}' curl -X POST --header "Authorization: Bearer {{ janitors_token.stdout[1:-1] }}" '{{ synapse_container_ip.stdout }}:8008/_synapse/admin/v1/purge_media_cache?before_ts={{ epoche_time.stdout }}'
register: purge_command register: purge_command

View File

@ -3,7 +3,7 @@ matrix_client_element_enabled: true
matrix_client_element_container_image_self_build: false matrix_client_element_container_image_self_build: false
matrix_client_element_container_image_self_build_repo: "https://github.com/vector-im/riot-web.git" matrix_client_element_container_image_self_build_repo: "https://github.com/vector-im/riot-web.git"
matrix_client_element_version: v1.7.26 matrix_client_element_version: v1.7.27
matrix_client_element_docker_image: "{{ matrix_client_element_docker_image_name_prefix }}vectorim/element-web:{{ matrix_client_element_version }}" matrix_client_element_docker_image: "{{ matrix_client_element_docker_image_name_prefix }}vectorim/element-web:{{ matrix_client_element_version }}"
matrix_client_element_docker_image_name_prefix: "{{ 'localhost/' if matrix_client_element_container_image_self_build else matrix_container_global_registry_prefix }}" matrix_client_element_docker_image_name_prefix: "{{ 'localhost/' if matrix_client_element_container_image_self_build else matrix_container_global_registry_prefix }}"
matrix_client_element_docker_image_force_pull: "{{ matrix_client_element_docker_image.endswith(':latest') }}" matrix_client_element_docker_image_force_pull: "{{ matrix_client_element_docker_image.endswith(':latest') }}"

View File

@ -35,7 +35,25 @@
with_dict: with_dict:
'matrix_awx_dimension_user_created': 'true' 'matrix_awx_dimension_user_created': 'true'
when: not matrix_awx_dimension_user_created|bool when: not matrix_awx_dimension_user_created|bool
- name: Create user account @mjolnir
command: |
/usr/local/bin/matrix-synapse-register-user mjolnir {{ matrix_awx_mjolnir_user_password | quote }} 0
register: cmd
when: not matrix_awx_mjolnir_user_created|bool
no_log: True
- name: Update AWX dimension user created variable
delegate_to: 127.0.0.1
lineinfile:
path: '/var/lib/awx/projects/clients/{{ member_id }}/{{ subscription_id }}/matrix_vars.yml'
regexp: "^#? *{{ item.key | regex_escape() }}:"
line: "{{ item.key }}: {{ item.value }}"
insertafter: 'AWX Settings'
with_dict:
'matrix_awx_mjolnir_user_created': 'true'
when: not matrix_awx_mjolnir_user_created|bool
- name: Ensure /chroot/website location has correct permissions - name: Ensure /chroot/website location has correct permissions
file: file:
path: /chroot/website path: /chroot/website

View File

@ -7,7 +7,7 @@ matrix_mailer_container_image_self_build_repository_url: "https://github.com/dev
matrix_mailer_container_image_self_build_src_files_path: "{{ matrix_mailer_base_path }}/docker-src" matrix_mailer_container_image_self_build_src_files_path: "{{ matrix_mailer_base_path }}/docker-src"
matrix_mailer_container_image_self_build_version: "{{ matrix_mailer_docker_image.split(':')[1] }}" matrix_mailer_container_image_self_build_version: "{{ matrix_mailer_docker_image.split(':')[1] }}"
matrix_mailer_version: 4.94.2-r0 matrix_mailer_version: 4.94.2-r0-1
matrix_mailer_docker_image: "{{ matrix_mailer_docker_image_name_prefix }}devture/exim-relay:{{ matrix_mailer_version }}" matrix_mailer_docker_image: "{{ matrix_mailer_docker_image_name_prefix }}devture/exim-relay:{{ matrix_mailer_version }}"
matrix_mailer_docker_image_name_prefix: "{{ 'localhost/' if matrix_mailer_container_image_self_build else matrix_container_global_registry_prefix }}" matrix_mailer_docker_image_name_prefix: "{{ 'localhost/' if matrix_mailer_container_image_self_build else matrix_container_global_registry_prefix }}"
matrix_mailer_docker_image_force_pull: "{{ matrix_mailer_docker_image.endswith(':latest') }}" matrix_mailer_docker_image_force_pull: "{{ matrix_mailer_docker_image.endswith(':latest') }}"

View File

@ -8,7 +8,7 @@ matrix_synapse_admin_container_self_build_repo: "https://github.com/Awesome-Tech
matrix_synapse_admin_docker_src_files_path: "{{ matrix_base_data_path }}/synapse-admin/docker-src" matrix_synapse_admin_docker_src_files_path: "{{ matrix_base_data_path }}/synapse-admin/docker-src"
matrix_synapse_admin_version: 0.8.0 matrix_synapse_admin_version: latest
matrix_synapse_admin_docker_image: "{{ matrix_synapse_admin_docker_image_name_prefix }}awesometechnologies/synapse-admin:{{ matrix_synapse_admin_version }}" matrix_synapse_admin_docker_image: "{{ matrix_synapse_admin_docker_image_name_prefix }}awesometechnologies/synapse-admin:{{ matrix_synapse_admin_version }}"
matrix_synapse_admin_docker_image_name_prefix: "{{ 'localhost/' if matrix_synapse_admin_container_self_build else matrix_container_global_registry_prefix }}" matrix_synapse_admin_docker_image_name_prefix: "{{ 'localhost/' if matrix_synapse_admin_container_self_build else matrix_container_global_registry_prefix }}"
matrix_synapse_admin_docker_image_force_pull: "{{ matrix_synapse_admin_docker_image.endswith(':latest') }}" matrix_synapse_admin_docker_image_force_pull: "{{ matrix_synapse_admin_docker_image.endswith(':latest') }}"

View File

@ -15,8 +15,8 @@ matrix_synapse_docker_image_name_prefix: "{{ 'localhost/' if matrix_synapse_cont
# amd64 gets released first. # amd64 gets released first.
# arm32 relies on self-building, so the same version can be built immediately. # arm32 relies on self-building, so the same version can be built immediately.
# arm64 users need to wait for a prebuilt image to become available. # arm64 users need to wait for a prebuilt image to become available.
matrix_synapse_version: v1.33.1 matrix_synapse_version: v1.33.2
matrix_synapse_version_arm64: v1.33.1 matrix_synapse_version_arm64: v1.33.2
matrix_synapse_docker_image_tag: "{{ matrix_synapse_version if matrix_architecture in ['arm32', 'amd64'] else matrix_synapse_version_arm64 }}" matrix_synapse_docker_image_tag: "{{ matrix_synapse_version if matrix_architecture in ['arm32', 'amd64'] else matrix_synapse_version_arm64 }}"
matrix_synapse_docker_image_force_pull: "{{ matrix_synapse_docker_image.endswith(':latest') }}" matrix_synapse_docker_image_force_pull: "{{ matrix_synapse_docker_image.endswith(':latest') }}"

View File

@ -10,7 +10,7 @@
- name: Set matrix_synapse_rust_synapse_compress_state_find_rooms_command_wait_time, if not provided - name: Set matrix_synapse_rust_synapse_compress_state_find_rooms_command_wait_time, if not provided
set_fact: set_fact:
matrix_synapse_rust_synapse_compress_state_find_rooms_command_wait_time: 15 matrix_synapse_rust_synapse_compress_state_find_rooms_command_wait_time: 180
when: "matrix_synapse_rust_synapse_compress_state_find_rooms_command_wait_time|default('') == ''" when: "matrix_synapse_rust_synapse_compress_state_find_rooms_command_wait_time|default('') == ''"
- name: Set matrix_synapse_rust_synapse_compress_state_compress_room_time, if not provided - name: Set matrix_synapse_rust_synapse_compress_state_compress_room_time, if not provided