From d9f4914e0d78fa45c7a76526a660faf94213b3df Mon Sep 17 00:00:00 2001 From: transcaffeine Date: Sun, 29 Nov 2020 15:20:22 +0100 Subject: [PATCH] WIP: postgres: create databases for all services If a service is enabled, a database for it is created in postgres with a uniqque password. The service can then use this database for data storage instead of relying on sqlite. --- group_vars/matrix_servers | 64 +++++++++++++++++++ .../templates/config.yaml.j2 | 4 +- .../templates/config.yaml.j2 | 4 +- .../templates/config.yaml.j2 | 3 + .../templates/config.yaml.j2 | 2 +- .../templates/config.yaml.j2 | 2 +- .../templates/config.yaml.j2 | 2 +- .../templates/config.yaml.j2 | 2 +- .../templates/config.yaml.j2 | 4 +- .../templates/config.yaml.j2 | 4 +- .../templates/config.yaml.j2 | 4 +- .../templates/config.yaml.j2 | 4 +- .../templates/config.yaml.j2 | 4 +- .../templates/config.yaml.j2 | 4 +- .../matrix-dimension/templates/config.yaml.j2 | 2 +- .../matrix-postgres/tasks/setup_postgres.yml | 30 +++++++++ 16 files changed, 118 insertions(+), 21 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 78a038517..8a099275d 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -889,6 +889,70 @@ matrix_postgres_connection_username: "synapse" matrix_postgres_connection_password: "synapse-password" matrix_postgres_db_name: "homeserver" +matrix_postgres_additional_databases: | + {{ + ([{ + name: 'matrix_appservice_discord', + pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_discord.db.secret') | string + }] if matrix_appservice_discord_enabled else []) + + ([{ + name: 'matrix_appservice_slack' + pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_slack.db.secret') | string + }] if matrix_appservice_slack_enabled else []) + + ([{ + name: 'matrix_appservice_irc' + pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_irc.db.secret') | string + }] if matrix_appservice_irc_enabled else []) + + ([{ + name: 'mautrix-bridge-facebook' + pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_facebook.db.secret') | string + }] if matrix_mautrix_facebook_enabled else []) + + ([{ + name: 'mautrix_bridge_hangouts' + pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_hangouts.db.secret') | string + }] if matrix_mautrix_hangouts_enabled else []) + + ([{ + name: 'mautrix_bridge_telegram' + pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_telegram.db.secret') | string + }] if matrix_mautrix_telegram_enabled else []) + + ([{ + name: 'mautrix_bridge_whatsapp' + pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_whatsapp.db.secret') | string + }] if matrix_mautrix_whatsapp_enabled else []) + + ([{ + name: 'matrix_bridge_sms' + pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | string + }] if matrix_sms_bridge_enabled else []) + + ([{ + name: 'matrix_puppet_skype' + pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_skype.db.secret') | string + }] if matrix_mx_puppet_skype_enabled else []) + + ([{ + name: 'matrix_puppet_slack' + pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_slack.db.secret') | string + }] if matrix_mx_puppet_slack_enabled else []) + + ([{ + name: 'matrix_puppet_twitter' + pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_twitter.db.secret') | string + }] if matrix_mx_puppet_twitter_enabled else []) + + ([{ + name: 'matrix_puppet_instagram' + pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_insta.db.secret') | string + ] if matrix_mx_puppet_instagram_enabled else []) + + ([{ + name: 'matrix_puppet_discord' + pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_puppet.db.secret') | string + }] if matrix_mx_puppet_discord_enabled else []) + + ([{ + name: 'matrix_puppet_steam' + pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_steam.db.secret') | string + }] if matrix_mx_puppet_steam_enabled else []) + + ([{ + name: 'matrix_dimension' + pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'dimension.db.secret') | string + }] if matrix_dimension_enabled else []) + }} + ###################################################################### # # /matrix-postgres diff --git a/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2 b/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2 index b99f522ab..de4182d9a 100644 --- a/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2 +++ b/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2 @@ -58,8 +58,8 @@ database: # If you are migrating, see https://github.com/Half-Shot/matrix-appservice-discord/blob/master/docs/howto.md#migrate-to-postgres-from-sqlite # WARNING: You will almost certainly be fine with sqlite unless your bridge # is in heavy demand and you suffer from IO slowness. - filename: "/data/discord.db" - # connString: "postgresql://user:password@localhost/database_name" + #filename: "/data/discord.db" + connString: "postgresql://matrix_appservice_discord:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_appservice_discord') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_appservice_discord" room: # Set the default visibility of alias rooms, defaults to "public". # One of: "public", "private" diff --git a/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2 b/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2 index 3daa18987..83b718358 100644 --- a/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2 +++ b/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2 @@ -127,8 +127,8 @@ advanced: # Use an external database to store bridge state. database: # database engine (must be 'postgres' or 'nedb'). Default: nedb - engine: "nedb" + engine: "postgres" # Either a PostgreSQL connection string, or a path to the NeDB storage directory. # For postgres, it must start with postgres:// # For NeDB, it must start with nedb://. The path is relative to the project directory. - connectionString: "nedb:///data" + connectionString: "postgres://matrix_appservice_irc:{{ matrix_addtional_databases | selectattr('name', 'equalto', 'matrix_appservice_irc') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_appservice_irc" diff --git a/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2 b/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2 index 8f48d3178..6c491134a 100644 --- a/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2 +++ b/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2 @@ -10,5 +10,8 @@ homeserver: server_name: "{{ matrix_domain }}" dbdir: "/data" +db: + engine: "postgres" + connectionString: "postgresql://matrix_appservice_slack:{{ matrix_addtional_databases | selectattr('name', 'equalto', 'matrix_appservice_slack') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_appservice_slack" matrix_admin_room: "{{ matrix_appservice_slack_control_room_id }}" diff --git a/roles/matrix-bridge-mautrix-facebook/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-facebook/templates/config.yaml.j2 index 9d8de2d5b..304571c13 100644 --- a/roles/matrix-bridge-mautrix-facebook/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mautrix-facebook/templates/config.yaml.j2 @@ -27,7 +27,7 @@ appservice: # Format examples: # SQLite: sqlite:///filename.db # Postgres: postgres://username:password@hostname/dbname - database: sqlite:////data/mautrix-facebook.db + database: sqlite://matrix_bridge_facebook:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_bridge_facebook') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_bridge_facebook # Public part of web server for out-of-Matrix interaction with the bridge. public: diff --git a/roles/matrix-bridge-mautrix-hangouts/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-hangouts/templates/config.yaml.j2 index f274b2034..e4e59ad26 100644 --- a/roles/matrix-bridge-mautrix-hangouts/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mautrix-hangouts/templates/config.yaml.j2 @@ -27,7 +27,7 @@ appservice: # Format examples: # SQLite: sqlite:///filename.db # Postgres: postgres://username:password@hostname/dbname - database: sqlite:////data/mautrix-hangouts.db + database: postgres://mautrix_bridge_hangouts:{{ matrix_additional_databases | selectattr('name', 'equalto', 'mautrix_bridge_hangouts') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/mautrix_bridge_hangouts # The unique ID of this appservice. id: hangouts diff --git a/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2 index 1a4ac43e9..65f17d338 100644 --- a/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2 @@ -27,7 +27,7 @@ appservice: # Format examples: # SQLite: sqlite:///filename.db # Postgres: postgres://username:password@hostname/dbname - database: sqlite:////data/mautrix-telegram.db + database: postgres://mautrix_bridge_telegram:{{ matrix_addtional_databases | selectattr('name', 'equalto', 'matrix_bridge_telegram') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/mautrix_bridge_telegram # Public part of web server for out-of-Matrix interaction with the bridge. # Used for things like login if the user wants to make sure the 2FA password isn't stored in diff --git a/roles/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2 index a527a1884..93956049c 100644 --- a/roles/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2 @@ -23,7 +23,7 @@ appservice: # The database URI. # SQLite: File name is enough. https://github.com/mattn/go-sqlite3#connection-string # Postgres: Connection string. For example, postgres://user:password@host/database - uri: mautrix-whatsapp.db + uri: postgres://matrix_bridge_whatsapp@{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_bridge_whatsapp') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_bridge_whatsapp # Maximum number of connections. Mostly relevant for Postgres. max_open_conns: 20 max_idle_conns: 2 diff --git a/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2 index 2c7037961..88c591e2f 100644 --- a/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2 @@ -105,10 +105,10 @@ database: # Connection string to connect to the Postgres instance # with username "user", password "pass", host "localhost" and database name "dbname". # Modify each value as necessary - #connString: "postgres://user:pass@localhost/dbname?sslmode=disable" + connString: "postgres://matrix_puppet_discord:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_discord') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_discord?sslmode=disable" # Use SQLite3 as a database backend # The name of the database file - filename: /data/database.db + #filename: /data/database.db logging: # Log level of console output diff --git a/roles/matrix-bridge-mx-puppet-instagram/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-instagram/templates/config.yaml.j2 index 634fbaecd..2d1891957 100644 --- a/roles/matrix-bridge-mx-puppet-instagram/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mx-puppet-instagram/templates/config.yaml.j2 @@ -49,10 +49,10 @@ database: # Connection string to connect to the Postgres instance # with username "user", password "pass", host "localhost" and database name "dbname". # Modify each value as necessary - #connString: "postgres://user:pass@localhost/dbname?sslmode=disable" + connString: "postgres://matrix_puppet_instagram:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_instagram') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_instagram?sslmode=disable" # Use SQLite3 as a database backend # The name of the database file - filename: /data/database.db + #filename: /data/database.db logging: # Log level of console output diff --git a/roles/matrix-bridge-mx-puppet-skype/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-skype/templates/config.yaml.j2 index c7b5c870e..a54ca758b 100644 --- a/roles/matrix-bridge-mx-puppet-skype/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mx-puppet-skype/templates/config.yaml.j2 @@ -73,10 +73,10 @@ database: # Connection string to connect to the Postgres instance # with username "user", password "pass", host "localhost" and database name "dbname". # Modify each value as necessary - #connString: "postgres://user:pass@localhost/dbname?sslmode=disable" + connString: "postgres://matrix_puppet_skype:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_skype') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_skype?sslmode=disable" # Use SQLite3 as a database backend # The name of the database file - filename: /data/database.db + #filename: /data/database.db provisioning: # Regex of Matrix IDs allowed to use the puppet bridge diff --git a/roles/matrix-bridge-mx-puppet-slack/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-slack/templates/config.yaml.j2 index b6e887848..7f1b21d78 100644 --- a/roles/matrix-bridge-mx-puppet-slack/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mx-puppet-slack/templates/config.yaml.j2 @@ -63,10 +63,10 @@ database: # Connection string to connect to the Postgres instance # with username "user", password "pass", host "localhost" and database name "dbname". # Modify each value as necessary - #connString: "postgres://user:pass@localhost/dbname?sslmode=disable" + connString: "postgres://matrix_puppet_slack:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_slack') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_slack?sslmode=disable" # Use SQLite3 as a database backend # The name of the database file - filename: /data/database.db + #filename: /data/database.db logging: # Log level of console output diff --git a/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2 index d08982ca9..14f863196 100644 --- a/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2 @@ -66,10 +66,10 @@ database: # Connection string to connect to the Postgres instance # with username "user", password "pass", host "localhost" and database name "dbname". # Modify each value as necessary - #connString: "postgres://user:pass@localhost/dbname?sslmode=disable" + connString: "postgres://matrix_puppet_steam:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_steam') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_steam?sslmode=disable" # Use SQLite3 as a database backend # The name of the database file - filename: /data/database.db + #filename: /data/database.db logging: # Log level of console output diff --git a/roles/matrix-bridge-mx-puppet-twitter/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-twitter/templates/config.yaml.j2 index 7d3033b39..853d23fc2 100644 --- a/roles/matrix-bridge-mx-puppet-twitter/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mx-puppet-twitter/templates/config.yaml.j2 @@ -59,10 +59,10 @@ database: # Connection string to connect to the Postgres instance # with username "user", password "pass", host "localhost" and database name "dbname". # Modify each value as necessary - #connString: "postgres://user:pass@localhost/dbname?sslmode=disable" + connString: "postgres://matrix_puppet_twitter:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_twitter') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_twitter?sslmode=disable" # Use SQLite3 as a database backend # The name of the database file - filename: /data/database.db + #filename: /data/database.db logging: # Log level of console output diff --git a/roles/matrix-dimension/templates/config.yaml.j2 b/roles/matrix-dimension/templates/config.yaml.j2 index a05b6c35a..b7ca1ee3b 100644 --- a/roles/matrix-dimension/templates/config.yaml.j2 +++ b/roles/matrix-dimension/templates/config.yaml.j2 @@ -44,7 +44,7 @@ widgetBlacklist: # Where the database for Dimension is database: - file: "dimension.db" + uri: "postgres://matrix_dimension:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_dimension') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_dimension" # Display settings that apply to self-hosted go-neb instances goneb: diff --git a/roles/matrix-postgres/tasks/setup_postgres.yml b/roles/matrix-postgres/tasks/setup_postgres.yml index f186bdca4..85a8604c6 100644 --- a/roles/matrix-postgres/tasks/setup_postgres.yml +++ b/roles/matrix-postgres/tasks/setup_postgres.yml @@ -162,3 +162,33 @@ - matrix-change-user-admin-status - matrix-postgres-update-user-password-hash when: "not matrix_postgres_enabled|bool" + +# Create additional databases +- name: Retrieve IP of postgres container + shell: "docker inspect matrix-postgres | jq -r '.[0].NetworkSettings.Networks.{{ matrix_docker_network }}.IPAddress'" + register: matirx_postgres_container_ip + +- name: Create additional users in postgres + postgresql_user: + name: "{{ item.name }}" + password: "{{ item.pass }}" + login_host: "{{ matrx_postgres_container_ip.stdout }}" + login_port: 5432 + login_user: "{{ matrix_postgres_connection_username }}" + login_password: "{{ matrix_postgres_connection_password }}" + login_db: "{{ matrix_postgres_db_name }}" + loop: matrix_postgres_additional_databases + when: matrix_postgres_enabed|bool + +- name: Create additional users in postgres + postgresql_db: + name: "{{ item.name }}" + owner: "{{ item.name }}" + lc_ctype: 'C' + lc_collate: 'C' + login_host: "{{ matrx_postgres_container_ip.stdout }}" + login_port: 5432 + login_user: "{{ matrix_postgres_connection_username }}" + login_password: "{{ matrix_postgres_connection_password }}" + loop: matrix_postgres_additional_databases + when: matrix_postgres_enabled|bool