Merge branch 'master' of https://github.com/spantaleev/matrix-docker-ansible-deploy

roles/matrix-base/defaults/main.yml

@@ -43,13 +43,15 @@ matrix_docker_network: "matrix"
 matrix_well_known_matrix_server_enabled: true
 
 # Variables to Control which parts of our roles run.
+run_postgres_import: true
+run_postgres_upgrade: true
+run_postgres_import_sqlite_db: true
+run_postgres_synapse_janitor: true
+run_postgres_vacuum: true
+run_synapse_register_user: true
+run_synapse_update_user_password: true
+run_synapse_import_media_store: true
 run_setup: true
-run_import_postgres: true
-run_upgrade_postgres: true
+run_self_check: true
 run_start: true
 run_stop: true
-run_register_user: true
-run_update_user_password: true
-run_import_sqlite_db: true
-run_import_media_store: true
-run_self_check: true

roles/matrix-bridge-appservice-discord/defaults/main.yml

@@ -69,7 +69,7 @@ matrix_appservice_discord_configuration_yaml: |
     disableJoinLeaveNotifications: false
   # Authentication configuration for the discord bot.
   auth:
-    clientID: {{ matrix_appservice_discord_client_id }}
+    clientID: {{ matrix_appservice_discord_client_id|string }}
     botToken: {{ matrix_appservice_discord_bot_token }}
   logging:
     # What level should the logger output to the console at.

roles/matrix-bridge-appservice-irc/tasks/setup_install.yml

@@ -59,8 +59,19 @@
 
 - name: Generate Appservice IRC passkey if it doesn't exist
   shell: /usr/bin/openssl genpkey -out {{ matrix_appservice_irc_data_path }}/passkey.pem -outform PEM -algorithm RSA -pkeyopt rsa_keygen_bits:2048
+  become: true
+  become_user: "{{ matrix_user_username }}"
   when: "not irc_passkey_file.stat.exists"
 
+# In the past, we used to generate the passkey.pem file with root, so permissions may not be okay.
+# Fix it.
+- name: (Migration) Ensure Appservice IRC passkey permissions are okay
+  file:
+    path: "{{ matrix_appservice_irc_data_path }}/passkey.pem"
+    mode: 0644
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_username }}"
+
 # Ideally, we'd like to generate the final registration.yaml file by ourselves.
 #
 # However, the IRC bridge supports multiple servers, which leads to multiple

roles/matrix-bridge-mautrix-telegram/defaults/main.yml

@@ -3,7 +3,7 @@
 
 matrix_mautrix_telegram_enabled: true
 
-matrix_mautrix_telegram_docker_image: "tulir/mautrix-telegram:v0.5.2"
+matrix_mautrix_telegram_docker_image: "tulir/mautrix-telegram:v0.6.0"
 matrix_mautrix_telegram_docker_image_force_pull: "{{ matrix_mautrix_telegram_docker_image.endswith(':latest') }}"
 
 matrix_mautrix_telegram_base_path: "{{ matrix_base_data_path }}/mautrix-telegram"
@@ -196,6 +196,8 @@ matrix_mautrix_telegram_configuration_yaml: |
       inline_images: false
       # Maximum size of image in megabytes before sending to Telegram as a document.
       image_as_file_size: 10
+      # Maximum size of Telegram documents in megabytes to bridge.
+      max_document_size: 100
 
       # Whether to bridge Telegram bot messages as m.notices or m.texts.
       bot_messages_as_notices: true
@@ -295,6 +297,40 @@ matrix_mautrix_telegram_configuration_yaml: |
       api_hash: {{ matrix_mautrix_telegram_api_hash }}
       # (Optional) Create your own bot at https://t.me/BotFather
       bot_token: disabled
+
+      # Telethon connection options.
+      connection:
+          # The timeout in seconds to be used when connecting.
+          timeout: 120
+          # How many times the reconnection should retry, either on the initial connection or when
+          # Telegram disconnects us. May be set to a negative or null value for infinite retries, but
+          # this is not recommended, since the program can get stuck in an infinite loop.
+          retries: 5
+          # The delay in seconds to sleep between automatic reconnections.
+          retry_delay: 1
+          # The threshold below which the library should automatically sleep on flood wait errors
+          # (inclusive). For instance, if a FloodWaitError for 17s occurs and flood_sleep_threshold
+          # is 20s, the library will sleep automatically. If the error was for 21s, it would raise
+          # the error instead. Values larger than a day (86400) will be changed to a day.
+          flood_sleep_threshold: 60
+          # How many times a request should be retried. Request are retried when Telegram is having
+          # internal issues, when there is a FloodWaitError less than flood_sleep_threshold, or when
+          # there's a migrate error. May take a negative or null value for infinite retries, but this
+          # is not recommended, since some requests can always trigger a call fail (such as searching
+          # for messages).
+          request_retries: 5
+
+      # Device info sent to Telegram.
+      device_info:
+          # "auto" = OS name+version.
+          device_model: auto
+          # "auto" = Telethon version.
+          system_version: auto
+          # "auto" = mautrix-telegram version.
+          app_version: auto
+          lang_code: en
+          system_lang_code: en
+
       # Custom server to connect to.
       server:
           # Set to true to use these server settings. If false, will automatically
@@ -306,6 +342,7 @@ matrix_mautrix_telegram_configuration_yaml: |
           ip: 149.154.167.40
           # The port to connect to. 443 may not work, 80 is better and both are equally secure.
           port: 80
+
       # Telethon proxy configuration.
       # You must install PySocks from pip for proxies to work.
       proxy:

roles/matrix-nginx-proxy/tasks/ssl/setup_ssl_lets_encrypt.yml

@@ -103,11 +103,11 @@
     state: absent
   when: "not matrix_nginx_proxy_enabled|bool"
 
-# When Let's Encrypt is not used at all, remove all cronjobs in that cron file.
 - name: Ensure matrix-ssl-lets-encrypt-renew cronjob removed
   cron:
     user: root
     cron_file: matrix-ssl-lets-encrypt
+    name: matrix-ssl-lets-encrypt-certificates-renew
     state: absent
   when: "matrix_ssl_retrieval_method != 'lets-encrypt'"
 

roles/matrix-postgres/defaults/main.yml

@@ -28,3 +28,5 @@ matrix_postgres_container_extra_arguments: []
 #
 # Takes an "<ip>:<port>" or "<port>" value (e.g. "127.0.0.1:5432"), or empty string to not expose.
 matrix_postgres_container_postgres_bind_port: ""
+
+matrix_postgres_tool_synapse_janitor: "https://raw.githubusercontent.com/xwiki-labs/synapse_scripts/0b3f035951932ceb396631de3fc701043b9723bc/synapse_janitor.sql"

roles/matrix-postgres/tasks/main.yml

@@ -15,16 +15,26 @@
     - setup-postgres
 
 - import_tasks: "{{ role_path }}/tasks/import_postgres.yml"
-  when: run_import_postgres|bool
+  when: run_postgres_import|bool
   tags:
     - import-postgres
 
 - import_tasks: "{{ role_path }}/tasks/import_sqlite_db.yml"
-  when: run_import_sqlite_db|bool
+  when: run_postgres_import_sqlite_db|bool
   tags:
     - import-sqlite-db
 
 - import_tasks: "{{ role_path }}/tasks/upgrade_postgres.yml"
-  when: run_upgrade_postgres|bool
+  when: run_postgres_upgrade|bool
   tags:
     - upgrade-postgres
+
+- import_tasks: "{{ role_path }}/tasks/run_synapse_janitor.yml"
+  when: run_postgres_synapse_janitor|bool
+  tags:
+    - run-postgres-synapse-janitor
+
+- import_tasks: "{{ role_path }}/tasks/run_vacuum.yml"
+  when: run_postgres_vacuum|bool
+  tags:
+    - run-postgres-vacuum

roles/matrix-postgres/tasks/run_synapse_janitor.yml

@@ -0,0 +1,110 @@
+---
+
+# Pre-checks
+
+- name: Fail if Postgres not enabled
+  fail:
+    msg: "Postgres via the matrix-postgres role is not enabled (`matrix_postgres_enabled`). Cannot run synapse-janitor."
+  when: "not matrix_postgres_enabled|bool"
+
+
+# Defaults
+
+- name: Set postgres_start_wait_time, if not provided
+  set_fact:
+    postgres_start_wait_time: 15
+  when: "postgres_start_wait_time|default('') == ''"
+
+- name: Set postgres_synapse_janitor_wait_time, if not provided
+  set_fact:
+    postgres_synapse_janitor_wait_time: "{{ 7 * 86400 }}"
+  when: "postgres_synapse_janitor_wait_time|default('') == ''"
+
+- name: Set postgres_synapse_janitor_tool_path, if not provided
+  set_fact:
+    postgres_synapse_janitor_tool_path: "{{ matrix_postgres_base_path }}/synapse_janitor.sql"
+  when: "postgres_synapse_janitor_tool_path|default('') == ''"
+
+
+# Actual janitor work
+
+- name: Download synapse-janitor tool
+  get_url:
+    url: "{{ matrix_postgres_tool_synapse_janitor }}"
+    dest: "{{ postgres_synapse_janitor_tool_path }}"
+    force: true
+    mode: 0550
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_username }}"
+
+- name: Ensure matrix-postgres is started
+  service:
+    name: matrix-postgres
+    state: started
+    daemon_reload: yes
+
+- name: Wait a bit, so that Postgres can start
+  wait_for:
+    timeout: "{{ postgres_start_wait_time }}"
+  delegate_to: 127.0.0.1
+  become: false
+
+- import_tasks: tasks/util/detect_existing_postgres_version.yml
+
+- name: Abort, if no existing Postgres version detected
+  fail:
+    msg: "Could not find existing Postgres installation"
+  when: "not matrix_postgres_detected_existing|bool"
+
+- name: Generate Postgres database synapse-janitor command
+  set_fact:
+    matrix_postgres_synapse_janitor_command: >-
+      /usr/bin/docker run --rm --name matrix-postgres-synapse-janitor
+      --user={{ matrix_user_uid }}:{{ matrix_user_gid }}
+      --cap-drop=ALL
+      --network={{ matrix_docker_network }}
+      --env-file={{ matrix_postgres_base_path }}/env-postgres-psql
+      --mount type=bind,src={{ postgres_synapse_janitor_tool_path }},dst=/synapse_janitor.sql,ro=true
+      {{ matrix_postgres_docker_image_latest }}
+      psql -v ON_ERROR_STOP=1 -h matrix-postgres {{ matrix_synapse_database_database }} -f /synapse_janitor.sql
+
+- name: Note about Postgres purging alternative
+  debug:
+    msg: >-
+      Running synapse-janitor with the following Postgres command: `{{ matrix_postgres_synapse_janitor_command }}`.
+      If this crashes, you can stop all processes (`systemctl stop matrix-*`),
+      start Postgres only (`systemctl start matrix-postgres`)
+      and manually run the above command directly on the server.
+
+- name: Populate service facts
+  service_facts:
+
+- set_fact:
+    matrix_postgres_synapse_was_running: "{{ ansible_facts.services['matrix-synapse.service']|default(none) is not none and ansible_facts.services['matrix-synapse.service'].state == 'running' }}"
+
+- name: Ensure matrix-synapse is stopped
+  service:
+    name: matrix-synapse
+    state: stopped
+    daemon_reload: yes
+
+- name: Run synapse-janitor
+  command: "{{ matrix_postgres_synapse_janitor_command }}"
+  async: "{{ postgres_synapse_janitor_wait_time }}"
+  poll: 10
+  register: matrix_postgres_synapse_janitor_result
+
+# Intentionally show the results
+- debug: var="matrix_postgres_synapse_janitor_result"
+
+- name: Ensure matrix-synapse is started, if it previously was
+  service:
+    name: matrix-synapse
+    state: started
+    daemon_reload: yes
+  when: "matrix_postgres_synapse_was_running|bool"
+
+- name: Delete synapse-janitor tool
+  file:
+    path: "{{ postgres_synapse_janitor_tool_path }}"
+    state: absent

roles/matrix-postgres/tasks/run_vacuum.yml

@@ -0,0 +1,90 @@
+---
+
+# Pre-checks
+
+- name: Fail if Postgres not enabled
+  fail:
+    msg: "Postgres via the matrix-postgres role is not enabled (`matrix_postgres_enabled`). Cannot run vacuum."
+  when: "not matrix_postgres_enabled|bool"
+
+
+# Defaults
+
+- name: Set postgres_start_wait_time, if not provided
+  set_fact:
+    postgres_start_wait_time: 15
+  when: "postgres_start_wait_time|default('') == ''"
+
+- name: Set postgres_vacuum_wait_time, if not provided
+  set_fact:
+    postgres_vacuum_wait_time: "{{ 7 * 86400 }}"
+  when: "postgres_vacuum_wait_time|default('') == ''"
+
+
+# Actual vacuuming work
+
+- name: Ensure matrix-postgres is started
+  service:
+    name: matrix-postgres
+    state: started
+    daemon_reload: yes
+
+- name: Wait a bit, so that Postgres can start
+  wait_for:
+    timeout: "{{ postgres_start_wait_time }}"
+  delegate_to: 127.0.0.1
+  become: false
+
+- import_tasks: tasks/util/detect_existing_postgres_version.yml
+
+- name: Abort, if no existing Postgres version detected
+  fail:
+    msg: "Could not find existing Postgres installation"
+  when: "not matrix_postgres_detected_existing|bool"
+
+- name: Generate Postgres database vacuum command
+  set_fact:
+    matrix_postgres_vacuum_command: >-
+      /usr/bin/docker run --rm --name matrix-postgres-synapse-vacuum
+      --user={{ matrix_user_uid }}:{{ matrix_user_gid }}
+      --cap-drop=ALL
+      --network={{ matrix_docker_network }}
+      --env-file={{ matrix_postgres_base_path }}/env-postgres-psql
+      {{ matrix_postgres_docker_image_latest }}
+      psql -v ON_ERROR_STOP=1 -h matrix-postgres {{ matrix_synapse_database_database }} -c 'VACUUM FULL VERBOSE'
+
+- name: Note about Postgres vacuum alternative
+  debug:
+    msg: >-
+      Running vacuum with the following Postgres command: `{{ matrix_postgres_vacuum_command }}`.
+      If this crashes, you can stop all processes (`systemctl stop matrix-*`),
+      start Postgres only (`systemctl start matrix-postgres`)
+      and manually run the above command directly on the server.
+
+- name: Populate service facts
+  service_facts:
+
+- set_fact:
+    matrix_postgres_synapse_was_running: "{{ ansible_facts.services['matrix-synapse.service']|default(none) is not none and ansible_facts.services['matrix-synapse.service'].state == 'running' }}"
+
+- name: Ensure matrix-synapse is stopped
+  service:
+    name: matrix-synapse
+    state: stopped
+    daemon_reload: yes
+
+- name: Run Postgres vacuum command
+  command: "{{ matrix_postgres_vacuum_command }}"
+  async: "{{ postgres_vacuum_wait_time }}"
+  poll: 10
+  register: matrix_postgres_synapse_vacuum_result
+
+# Intentionally show the results
+- debug: var="matrix_postgres_synapse_vacuum_result"
+
+- name: Ensure matrix-synapse is started, if it previously was
+  service:
+    name: matrix-synapse
+    state: started
+    daemon_reload: yes
+  when: "matrix_postgres_synapse_was_running|bool"

roles/matrix-postgres/tasks/setup_postgres.yml

@@ -1,7 +1,7 @@
 ---
 
 #
-# Generic tasks, no matter what kind of server we're using (internal/external)
+# Tasks related to setting up an internal postgres server
 #
 
 - import_tasks: "{{ role_path }}/tasks/migrate_postgres_data_directory.yml"
@@ -32,8 +32,6 @@
     force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_postgres_docker_image_force_pull }}"
   when: matrix_postgres_enabled|bool
 
-# We always create these directories, even if an external Postgres is used,
-# because we store environment variable files there.
 - name: Ensure Postgres paths exist
   file:
     path: "{{ item }}"
@@ -70,9 +68,12 @@
     mode: 0750
   when: matrix_postgres_enabled|bool
 
-#
-# Tasks related to setting up an internal postgres server
-#
+- name: Ensure matrix-postgres-update-user-password-hash script created
+  template:
+    src: "{{ role_path }}/templates/usr-local-bin/matrix-postgres-update-user-password-hash.j2"
+    dest: "/usr/local/bin/matrix-postgres-update-user-password-hash"
+    mode: 0750
+  when: matrix_postgres_enabled|bool
 
 - name: Ensure matrix-postgres.service installed
   template:
@@ -127,9 +128,12 @@
     msg: "Note: You are not using a local PostgreSQL database, but some old data remains from before in `{{ matrix_postgres_data_path }}`. Feel free to delete it."
   when: "not matrix_postgres_enabled|bool and matrix_postgres_data_path_stat.stat.exists"
 
-- name: Ensure matrix-postgres-update-user-password-hash script created
-  template:
-    src: "{{ role_path }}/templates/usr-local-bin/matrix-postgres-update-user-password-hash.j2"
-    dest: "/usr/local/bin/matrix-postgres-update-user-password-hash"
-    mode: 0750
-  when: matrix_postgres_enabled|bool
+- name: Remove Postgres scripts
+  file:
+    path: "/usr/local/bin/{{ item }}"
+    state: absent
+  with_items:
+    - matrix-postgres-cli
+    - matrix-make-user-admin
+    - matrix-postgres-update-user-password-hash
+  when: "not matrix_postgres_enabled|bool"

roles/matrix-riot-web/defaults/main.yml

@@ -1,6 +1,6 @@
 matrix_riot_web_enabled: true
 
-matrix_riot_web_docker_image: "bubuntux/riot-web:v1.2.2"
+matrix_riot_web_docker_image: "bubuntux/riot-web:v1.2.4"
 matrix_riot_web_docker_image_force_pull: "{{ matrix_riot_web_docker_image.endswith(':latest') }}"
 
 matrix_riot_web_data_path: "{{ matrix_base_data_path }}/riot-web"

roles/matrix-synapse/tasks/main.yml

@@ -15,12 +15,12 @@
     - setup-synapse
 
 - import_tasks: "{{ role_path }}/tasks/import_media_store.yml"
-  when: run_import_media_store|bool
+  when: run_synapse_import_media_store|bool
   tags:
     - import-media-store
 
 - import_tasks: "{{ role_path }}/tasks/register_user.yml"
-  when: run_register_user|bool
+  when: run_synapse_register_user|bool
   tags:
     - register-user
 
@@ -39,7 +39,7 @@
     - self-check
 
 - import_tasks: "{{ role_path }}/tasks/update_user_password.yml"
-  when: run_update_user_password|bool
+  when: run_synapse_update_user_password|bool
   tags:
     - update-user-password