From ceda4c41ccdd59f267bdb7eee8995d44a601e271 Mon Sep 17 00:00:00 2001 From: Zac Date: Tue, 13 Jun 2023 10:22:37 -0500 Subject: [PATCH 1/6] remove the offending lines --- .../tasks/ext/s3-storage-provider/validate_config.yml | 2 -- .../templates/synapse/ext/s3-storage-provider/env.j2 | 2 -- .../ext/s3-storage-provider/media_storage_provider.yaml.j2 | 2 -- 3 files changed, 6 deletions(-) diff --git a/roles/custom/matrix-synapse/tasks/ext/s3-storage-provider/validate_config.yml b/roles/custom/matrix-synapse/tasks/ext/s3-storage-provider/validate_config.yml index 317269b3c..dfa3d9e5a 100644 --- a/roles/custom/matrix-synapse/tasks/ext/s3-storage-provider/validate_config.yml +++ b/roles/custom/matrix-synapse/tasks/ext/s3-storage-provider/validate_config.yml @@ -8,8 +8,6 @@ with_items: - "matrix_synapse_ext_synapse_s3_storage_provider_config_bucket" - "matrix_synapse_ext_synapse_s3_storage_provider_config_region_name" - - "matrix_synapse_ext_synapse_s3_storage_provider_config_access_key_id" - - "matrix_synapse_ext_synapse_s3_storage_provider_config_secret_access_key" - "matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url" - name: Fail if required matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url looks invalid diff --git a/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 b/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 index 58d262558..d895b742d 100644 --- a/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 +++ b/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 @@ -1,5 +1,3 @@ -AWS_ACCESS_KEY_ID={{ matrix_synapse_ext_synapse_s3_storage_provider_config_access_key_id }} -AWS_SECRET_ACCESS_KEY={{ matrix_synapse_ext_synapse_s3_storage_provider_config_secret_access_key }} AWS_DEFAULT_REGION={{ matrix_synapse_ext_synapse_s3_storage_provider_config_region_name }} ENDPOINT={{ matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url }} diff --git a/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/media_storage_provider.yaml.j2 b/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/media_storage_provider.yaml.j2 index ac2b58dbb..71394acef 100644 --- a/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/media_storage_provider.yaml.j2 +++ b/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/media_storage_provider.yaml.j2 @@ -6,8 +6,6 @@ config: bucket: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_bucket | to_json }} region_name: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_region_name | to_json }} endpoint_url: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url | to_json }} - access_key_id: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_access_key_id | to_json }} - secret_access_key: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_secret_access_key | to_json }} {% if matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_enabled %} sse_customer_key: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_key | to_json }} From 0e701bbecec26df41dab8c65036252532d624f47 Mon Sep 17 00:00:00 2001 From: cbackas Date: Tue, 13 Jun 2023 11:08:14 -0500 Subject: [PATCH 2/6] add var to make the creds optional --- roles/custom/matrix-synapse/defaults/main.yml | 2 ++ .../s3-storage-provider/validate_config.yml | 21 +++++++++++++++---- .../synapse/ext/s3-storage-provider/env.j2 | 4 ++++ .../media_storage_provider.yaml.j2 | 4 ++++ 4 files changed, 27 insertions(+), 4 deletions(-) diff --git a/roles/custom/matrix-synapse/defaults/main.yml b/roles/custom/matrix-synapse/defaults/main.yml index 32d63df7d..fa09d83f7 100644 --- a/roles/custom/matrix-synapse/defaults/main.yml +++ b/roles/custom/matrix-synapse/defaults/main.yml @@ -885,6 +885,8 @@ matrix_synapse_ext_synapse_s3_storage_provider_config_region_name: '' matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url: '' matrix_synapse_ext_synapse_s3_storage_provider_config_access_key_id: '' matrix_synapse_ext_synapse_s3_storage_provider_config_secret_access_key: '' +# Enable this to use EC2 instance profile metadata to grab IAM credentials instead of passing credentials directly. +matrix_synapse_ext_synapse_s3_storage_provider_config_ec2_instance_profile: false matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_enabled: false matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_key: '' matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_algo: 'AES256' diff --git a/roles/custom/matrix-synapse/tasks/ext/s3-storage-provider/validate_config.yml b/roles/custom/matrix-synapse/tasks/ext/s3-storage-provider/validate_config.yml index dfa3d9e5a..c3034531d 100644 --- a/roles/custom/matrix-synapse/tasks/ext/s3-storage-provider/validate_config.yml +++ b/roles/custom/matrix-synapse/tasks/ext/s3-storage-provider/validate_config.yml @@ -1,14 +1,27 @@ --- +- name: Set base required s3-storage-provider settings + set_fact: + base_s3_storage_provider_config: + - "matrix_synapse_ext_synapse_s3_storage_provider_config_bucket" + - "matrix_synapse_ext_synapse_s3_storage_provider_config_region_name" + - "matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url" + +- name: Set optional required s3-storage-provider settings + set_fact: + optional_s3_storage_provider_config: + - "matrix_synapse_ext_synapse_s3_storage_provider_config_access_key_id" + - "matrix_synapse_ext_synapse_s3_storage_provider_config_secret_access_key" + +- name: Prepare a list of required s3-storage-provider settings + set_fact: + required_s3_settings: "{{ base_s3_storage_provider_config + (optional_s3_storage_provider_config if not matrix_synapse_ext_synapse_s3_storage_provider_config_ec2_instance_profile|default(false)|bool else []) }}" - name: Fail if required s3-storage-provider settings not defined ansible.builtin.fail: msg: >- You need to define a required configuration setting (`{{ item }}`) for using s3-storage-provider. when: "vars[item] == ''" - with_items: - - "matrix_synapse_ext_synapse_s3_storage_provider_config_bucket" - - "matrix_synapse_ext_synapse_s3_storage_provider_config_region_name" - - "matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url" + with_items: "{{ required_s3_settings }}" - name: Fail if required matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url looks invalid ansible.builtin.fail: diff --git a/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 b/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 index d895b742d..c5e896032 100644 --- a/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 +++ b/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 @@ -1,3 +1,7 @@ +{% if not matrix_synapse_ext_synapse_s3_storage_provider_config_ec2_instance_profile|default(false)|bool %} +AWS_ACCESS_KEY_ID={{ matrix_synapse_ext_synapse_s3_storage_provider_config_access_key_id }} +AWS_SECRET_ACCESS_KEY={{ matrix_synapse_ext_synapse_s3_storage_provider_config_secret_access_key }} +{% endif %} AWS_DEFAULT_REGION={{ matrix_synapse_ext_synapse_s3_storage_provider_config_region_name }} ENDPOINT={{ matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url }} diff --git a/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/media_storage_provider.yaml.j2 b/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/media_storage_provider.yaml.j2 index 71394acef..32c8a0d17 100644 --- a/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/media_storage_provider.yaml.j2 +++ b/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/media_storage_provider.yaml.j2 @@ -6,6 +6,10 @@ config: bucket: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_bucket | to_json }} region_name: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_region_name | to_json }} endpoint_url: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url | to_json }} +{% if not matrix_synapse_ext_synapse_s3_storage_provider_config_ec2_instance_profile|default(false)|bool %} + access_key_id: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_access_key_id | to_json }} + secret_access_key: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_secret_access_key | to_json }} +{% endif %} {% if matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_enabled %} sse_customer_key: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_key | to_json }} From f6d260dc0036a7049daef262b09dbe0f85b3ff7e Mon Sep 17 00:00:00 2001 From: cbackas Date: Tue, 13 Jun 2023 11:58:19 -0500 Subject: [PATCH 3/6] this is better --- .../s3-storage-provider/validate_config.yml | 31 ++++++++----------- 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/roles/custom/matrix-synapse/tasks/ext/s3-storage-provider/validate_config.yml b/roles/custom/matrix-synapse/tasks/ext/s3-storage-provider/validate_config.yml index c3034531d..78b02f387 100644 --- a/roles/custom/matrix-synapse/tasks/ext/s3-storage-provider/validate_config.yml +++ b/roles/custom/matrix-synapse/tasks/ext/s3-storage-provider/validate_config.yml @@ -1,27 +1,22 @@ --- -- name: Set base required s3-storage-provider settings - set_fact: - base_s3_storage_provider_config: - - "matrix_synapse_ext_synapse_s3_storage_provider_config_bucket" - - "matrix_synapse_ext_synapse_s3_storage_provider_config_region_name" - - "matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url" - -- name: Set optional required s3-storage-provider settings - set_fact: - optional_s3_storage_provider_config: - - "matrix_synapse_ext_synapse_s3_storage_provider_config_access_key_id" - - "matrix_synapse_ext_synapse_s3_storage_provider_config_secret_access_key" - -- name: Prepare a list of required s3-storage-provider settings - set_fact: - required_s3_settings: "{{ base_s3_storage_provider_config + (optional_s3_storage_provider_config if not matrix_synapse_ext_synapse_s3_storage_provider_config_ec2_instance_profile|default(false)|bool else []) }}" - - name: Fail if required s3-storage-provider settings not defined ansible.builtin.fail: msg: >- You need to define a required configuration setting (`{{ item }}`) for using s3-storage-provider. when: "vars[item] == ''" - with_items: "{{ required_s3_settings }}" + with_items: + - "matrix_synapse_ext_synapse_s3_storage_provider_config_bucket" + - "matrix_synapse_ext_synapse_s3_storage_provider_config_region_name" + - "matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url" + +- name: Fail if required s3-storage-provider auth settings not defined + ansible.builtin.fail: + msg: >- + You need to define a required configuration setting (`{{ item }}`) for using s3-storage-provider. + when: "not matrix_synapse_ext_synapse_s3_storage_provider_config_ec2_instance_profile|default(false)|bool and vars[item] == ''" + with_items: + - "matrix_synapse_ext_synapse_s3_storage_provider_config_access_key_id" + - "matrix_synapse_ext_synapse_s3_storage_provider_config_secret_access_key" - name: Fail if required matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url looks invalid ansible.builtin.fail: From f026c7cee1dff3fe577a0d37bd153ecf5c1f8295 Mon Sep 17 00:00:00 2001 From: Zac Date: Tue, 13 Jun 2023 14:48:44 -0500 Subject: [PATCH 4/6] Apply suggestions from code review Co-authored-by: Slavi Pantaleev --- roles/custom/matrix-synapse/defaults/main.yml | 3 ++- .../tasks/ext/s3-storage-provider/validate_config.yml | 4 ++-- .../templates/synapse/ext/s3-storage-provider/env.j2 | 2 +- .../ext/s3-storage-provider/media_storage_provider.yaml.j2 | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/roles/custom/matrix-synapse/defaults/main.yml b/roles/custom/matrix-synapse/defaults/main.yml index fa09d83f7..88ef768e9 100644 --- a/roles/custom/matrix-synapse/defaults/main.yml +++ b/roles/custom/matrix-synapse/defaults/main.yml @@ -885,7 +885,8 @@ matrix_synapse_ext_synapse_s3_storage_provider_config_region_name: '' matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url: '' matrix_synapse_ext_synapse_s3_storage_provider_config_access_key_id: '' matrix_synapse_ext_synapse_s3_storage_provider_config_secret_access_key: '' -# Enable this to use EC2 instance profile metadata to grab IAM credentials instead of passing credentials directly. +# Enable this to use EC2 instance profile metadata to grab IAM credentials instead of passing credentials directly +# via matrix_synapse_ext_synapse_s3_storage_provider_config_access_key_id and matrix_synapse_ext_synapse_s3_storage_provider_config_secret_access_key matrix_synapse_ext_synapse_s3_storage_provider_config_ec2_instance_profile: false matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_enabled: false matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_key: '' diff --git a/roles/custom/matrix-synapse/tasks/ext/s3-storage-provider/validate_config.yml b/roles/custom/matrix-synapse/tasks/ext/s3-storage-provider/validate_config.yml index 78b02f387..406f186dc 100644 --- a/roles/custom/matrix-synapse/tasks/ext/s3-storage-provider/validate_config.yml +++ b/roles/custom/matrix-synapse/tasks/ext/s3-storage-provider/validate_config.yml @@ -9,11 +9,11 @@ - "matrix_synapse_ext_synapse_s3_storage_provider_config_region_name" - "matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url" -- name: Fail if required s3-storage-provider auth settings not defined +- name: Fail if required s3-storage-provider auth settings not defined when not using an EC2 profile ansible.builtin.fail: msg: >- You need to define a required configuration setting (`{{ item }}`) for using s3-storage-provider. - when: "not matrix_synapse_ext_synapse_s3_storage_provider_config_ec2_instance_profile|default(false)|bool and vars[item] == ''" + when: "not matrix_synapse_ext_synapse_s3_storage_provider_config_ec2_instance_profile | bool and vars[item] == ''" with_items: - "matrix_synapse_ext_synapse_s3_storage_provider_config_access_key_id" - "matrix_synapse_ext_synapse_s3_storage_provider_config_secret_access_key" diff --git a/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 b/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 index c5e896032..227fd89e4 100644 --- a/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 +++ b/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 @@ -1,4 +1,4 @@ -{% if not matrix_synapse_ext_synapse_s3_storage_provider_config_ec2_instance_profile|default(false)|bool %} +{% if not matrix_synapse_ext_synapse_s3_storage_provider_config_ec2_instance_profile | bool %} AWS_ACCESS_KEY_ID={{ matrix_synapse_ext_synapse_s3_storage_provider_config_access_key_id }} AWS_SECRET_ACCESS_KEY={{ matrix_synapse_ext_synapse_s3_storage_provider_config_secret_access_key }} {% endif %} diff --git a/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/media_storage_provider.yaml.j2 b/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/media_storage_provider.yaml.j2 index 32c8a0d17..97387e55c 100644 --- a/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/media_storage_provider.yaml.j2 +++ b/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/media_storage_provider.yaml.j2 @@ -6,7 +6,7 @@ config: bucket: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_bucket | to_json }} region_name: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_region_name | to_json }} endpoint_url: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url | to_json }} -{% if not matrix_synapse_ext_synapse_s3_storage_provider_config_ec2_instance_profile|default(false)|bool %} +{% if not matrix_synapse_ext_synapse_s3_storage_provider_config_ec2_instance_profile | bool %} access_key_id: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_access_key_id | to_json }} secret_access_key: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_secret_access_key | to_json }} {% endif %} From d9f8ac0f8b81a026e0ecd194029c2268980b5e47 Mon Sep 17 00:00:00 2001 From: cbackas Date: Tue, 13 Jun 2023 14:55:07 -0500 Subject: [PATCH 5/6] add thing to docs --- docs/configuring-playbook-synapse-s3-storage-provider.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/configuring-playbook-synapse-s3-storage-provider.md b/docs/configuring-playbook-synapse-s3-storage-provider.md index d5d5ca5e2..549fe6010 100644 --- a/docs/configuring-playbook-synapse-s3-storage-provider.md +++ b/docs/configuring-playbook-synapse-s3-storage-provider.md @@ -37,6 +37,10 @@ matrix_synapse_ext_synapse_s3_storage_provider_config_access_key_id: access-key- matrix_synapse_ext_synapse_s3_storage_provider_config_secret_access_key: secret-key-goes-here matrix_synapse_ext_synapse_s3_storage_provider_config_storage_class: STANDARD # or STANDARD_IA, etc. +# If you're using an EC2 instance with an instance profile that grants it permissions to access S3, set the following variable to true +# Defaulted to false, when this is enabled you do not need to provide the access_key_id or secret_access_key. +matrix_synapse_ext_synapse_s3_storage_provider_config_ec2_instance_profile: true + # For additional advanced settings, take a look at `roles/custom/matrix-synapse/defaults/main.yml` ``` From ba3534903ac4c6167d11b4f743cfba39f540f7e1 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Wed, 14 Jun 2023 09:27:40 +0300 Subject: [PATCH 6/6] Make S3 authentication options clearer --- ...uring-playbook-synapse-s3-storage-provider.md | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/docs/configuring-playbook-synapse-s3-storage-provider.md b/docs/configuring-playbook-synapse-s3-storage-provider.md index 549fe6010..c704a747f 100644 --- a/docs/configuring-playbook-synapse-s3-storage-provider.md +++ b/docs/configuring-playbook-synapse-s3-storage-provider.md @@ -30,16 +30,22 @@ After [creating the S3 bucket and configuring it](configuring-playbook-s3.md#buc ```yaml matrix_synapse_ext_synapse_s3_storage_provider_enabled: true + matrix_synapse_ext_synapse_s3_storage_provider_config_bucket: your-bucket-name matrix_synapse_ext_synapse_s3_storage_provider_config_region_name: some-region-name # e.g. eu-central-1 matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url: https://s3.REGION_NAME.amazonaws.com # adjust this -matrix_synapse_ext_synapse_s3_storage_provider_config_access_key_id: access-key-goes-here -matrix_synapse_ext_synapse_s3_storage_provider_config_secret_access_key: secret-key-goes-here matrix_synapse_ext_synapse_s3_storage_provider_config_storage_class: STANDARD # or STANDARD_IA, etc. -# If you're using an EC2 instance with an instance profile that grants it permissions to access S3, set the following variable to true -# Defaulted to false, when this is enabled you do not need to provide the access_key_id or secret_access_key. -matrix_synapse_ext_synapse_s3_storage_provider_config_ec2_instance_profile: true +# Authentication Method 1 - (access key id + secret) +# This works on all providers (AWS and other compatible systems). +# Uncomment the variables below to use it. +# matrix_synapse_ext_synapse_s3_storage_provider_config_access_key_id: access-key-goes-here +# matrix_synapse_ext_synapse_s3_storage_provider_config_secret_access_key: secret-key-goes-here + +# Authentication Method 2 - EC2 instance profile which grants permission to access S3 +# This only works on AWS when your server is hosted on an EC2 instance with the correct instance profile set. +# Uncomment the variable below to use it. +# matrix_synapse_ext_synapse_s3_storage_provider_config_ec2_instance_profile: true # For additional advanced settings, take a look at `roles/custom/matrix-synapse/defaults/main.yml` ```