Switch from s3fs to Goofys

Improves performance of media store operations.

roles/matrix-server/defaults/main.yml

@@ -30,7 +30,8 @@ matrix_environment_variables_data_path: "{{ matrix_base_data_path }}/environment
 matrix_synapse_base_path: "{{ matrix_base_data_path }}/synapse"
 matrix_synapse_config_dir_path: "{{ matrix_synapse_base_path }}/config"
 matrix_synapse_run_path: "{{ matrix_synapse_base_path }}/run"
-matrix_synapse_media_store_path: "{{ matrix_synapse_base_path }}/media-store"
+matrix_synapse_storage_path: "{{ matrix_synapse_base_path }}/storage"
+matrix_synapse_media_store_path: "{{ matrix_synapse_storage_path }}/media-store"
 matrix_postgres_data_path: "{{ matrix_base_data_path }}/postgres"
 matrix_nginx_proxy_data_path: "{{ matrix_base_data_path }}/nginx-proxy"
 matrix_nginx_proxy_confd_path: "{{ matrix_nginx_proxy_data_path }}/conf.d"
@@ -42,6 +43,7 @@ docker_matrix_image: "avhost/docker-matrix:v0.26.0"
 docker_nginx_image: "nginx:1.13.8-alpine"
 docker_riot_image: "avhost/docker-matrix-riot:latest"
 docker_s3fs_image: "xueshanf/s3fs:latest"
+docker_goofys_image: "cloudproto/goofys:latest"
 
 # To avoid Synapse's macaroon secret key from changing every time
 # a new config is built from scratch, you can specify one here.
@@ -61,6 +63,7 @@ matrix_s3_media_store_enabled: false
 matrix_s3_media_store_bucket_name: "your-bucket-name"
 matrix_s3_media_store_aws_access_key: "your-aws-access-key"
 matrix_s3_media_store_aws_secret_key: "your-aws-secret-key"
+matrix_s3_media_store_region: "eu-central-1"
 
 # By default, this playbook installs the Riot.IM web UI on the `hostname_riot` domain.
 # If you wish to connect to your Matrix server by other means,

roles/matrix-server/tasks/import_media_store.yml

@@ -42,12 +42,11 @@
     # It's wasteful to preserve owner/group now. We chown below anyway.
     owner: no
     group: no
-    # The default of times=yes does not work when s3fs is used.
     times: "{{ False if matrix_s3_media_store_enabled else True }}"
     perms: "{{ False if matrix_s3_media_store_enabled else True }}"
 
-# This is for the generic case and fails for remote file systems,
-# because the base path (matrix_synapse_media_store_path) is a mount point.
+# This is for the generic case and fails in other cases (remote file systems),
+# because in such cases the base path (matrix_synapse_media_store_path) is a mount point.
 - name: Ensure media store permissions are correct (generic case)
   file:
     path: "{{ matrix_synapse_media_store_path }}"
@@ -56,23 +55,8 @@
     recurse: yes
   when: "not matrix_s3_media_store_enabled"
 
-- name: Determine media store subdirectories
-  find: paths="{{ local_path_media_store }}" file_type=directory
-  delegate_to: 127.0.0.1
-  become: false
-  register: media_store_directories_result
-  when: "matrix_s3_media_store_enabled"
-
-# This is the s3fs special case. We chown the subdirectories one by one,
-# without touching the base directory.
-- name: Ensure media store permissions are correct (s3fs)
-  file:
-    path: "{{ matrix_synapse_media_store_path }}/{{ item.path|basename }}"
-    owner: "{{ matrix_user_username }}"
-    group: "{{ matrix_user_username }}"
-    recurse: yes
-  with_items: "{{ media_store_directories_result.files }}"
-  when: "matrix_s3_media_store_enabled"
+# We don't chown for Goofys, because due to the way it's mounted,
+# all files become owned by whoever needs to own them.
 
 - name: Ensure Matrix Synapse is started (if it previously was)
   service: name="{{ item }}" state=started daemon_reload=yes

roles/matrix-server/tasks/main.yml

@@ -23,6 +23,11 @@
     - setup-main
     - setup-s3fs
 
+- include: tasks/setup_goofys.yml
+  tags:
+    - setup-main
+    - setup-goofys
+
 - include: tasks/setup_synapse.yml
   tags:
     - setup-main

roles/matrix-server/tasks/setup_base.yml

@@ -25,6 +25,7 @@
     - docker-python
     - firewalld
     - ntp
+    - fuse
   when: ansible_distribution == 'CentOS'
 
 - name: Ensure APT usage dependencies are installed (Debian)
@@ -60,6 +61,7 @@
     - docker-ce
     - python-docker
     - ntp
+    - fuse
   when: ansible_os_family == 'Debian'
 
 - name: Ensure firewalld is started and autoruns

roles/matrix-server/tasks/setup_goofys.yml

@@ -0,0 +1,70 @@
+#
+# Tasks related to setting up Goofys
+#
+
+- name: Ensure Goofys Docker image is pulled
+  docker_image:
+    name: "{{ docker_goofys_image }}"
+  when: matrix_s3_media_store_enabled
+
+# This will throw a Permission Denied error if already mounted
+- name: Check Matrix Goofys external storage mountpoint path
+  stat: path="{{ matrix_synapse_media_store_path }}"
+  register: local_path_matrix_synapse_media_store_path_stat
+  ignore_errors: yes
+  when: matrix_s3_media_store_enabled
+
+- name: Ensure Matrix Goofys external storage mountpoint exists
+  file:
+    path: "{{ matrix_synapse_media_store_path }}"
+    state: directory
+    mode: 0750
+    owner: "{{ matrix_user_uid }}"
+    group: "{{ matrix_user_gid }}"
+  when: "matrix_s3_media_store_enabled and not local_path_matrix_synapse_media_store_path_stat.failed and not local_path_matrix_synapse_media_store_path_stat.stat.exists"
+
+- name: Ensure goofys environment variables file created
+  template:
+    src: "{{ role_path }}/templates/env/env-goofys.j2"
+    dest: "{{ matrix_environment_variables_data_path }}/goofys"
+    owner: root
+    mode: 0600
+  when: matrix_s3_media_store_enabled
+
+- name: Ensure matrix-goofys.service installed
+  template:
+    src: "{{ role_path }}/templates/systemd/matrix-goofys.service.j2"
+    dest: "/etc/systemd/system/matrix-goofys.service"
+    mode: 0644
+  when: matrix_s3_media_store_enabled
+
+#
+# Tasks related to getting rid of goofys (if it was previously enabled)
+#
+
+- name: Check existence of matrix-goofys service
+  stat: path="/etc/systemd/system/matrix-goofys.service"
+  register: matrix_goofys_service_stat
+
+- name: Ensure matrix-goofys is stopped
+  service: name=matrix-goofys state=stopped daemon_reload=yes
+  register: stopping_result
+  when: "not matrix_s3_media_store_enabled and matrix_goofys_service_stat.stat.exists"
+
+- name: Ensure matrix-goofys.service doesn't exist
+  file:
+    path: "/etc/systemd/system/matrix-goofys.service"
+    state: absent
+  when: "not matrix_s3_media_store_enabled and matrix_goofys_service_stat.stat.exists"
+
+- name: Ensure goofys environment variables file doesn't exist
+  file:
+    path: "{{ matrix_environment_variables_data_path }}/goofys"
+    state: absent
+  when: "not matrix_s3_media_store_enabled"
+
+- name: Ensure Goofys Docker image doesn't exist
+  docker_image:
+    name: "{{ docker_goofys_image }}"
+    state: absent
+  when: "not matrix_s3_media_store_enabled"

roles/matrix-server/tasks/setup_s3fs.yml

@@ -1,29 +1,5 @@
 #
-# Tasks related to setting up s3fs
-#
-
-- name: Ensure S3fs Docker image is pulled
-  docker_image:
-    name: "{{ docker_s3fs_image }}"
-  when: matrix_s3_media_store_enabled
-
-- name: Ensure s3fs-credentials file created
-  template:
-    src: "{{ role_path }}/templates/s3fs-credentials.j2"
-    dest: "{{ matrix_base_data_path }}/s3fs-credentials"
-    owner: root
-    mode: 0600
-  when: matrix_s3_media_store_enabled
-
-- name: Ensure matrix-s3fs.service installed
-  template:
-    src: "{{ role_path }}/templates/systemd/matrix-s3fs.service.j2"
-    dest: "/etc/systemd/system/matrix-s3fs.service"
-    mode: 0644
-  when: matrix_s3_media_store_enabled
-
-#
-# Tasks related to getting rid of s3fs (if it was previously enabled)
+# Tasks related to getting rid of s3fs (if it was previously installed)
 #
 
 - name: Check existence of matrix-s3fs service
@@ -33,22 +9,19 @@
 - name: Ensure matrix-s3fs is stopped
   service: name=matrix-s3fs state=stopped daemon_reload=yes
   register: stopping_result
-  when: "not matrix_s3_media_store_enabled and matrix_s3fs_service_stat.stat.exists"
+  when: "matrix_s3fs_service_stat.stat.exists"
 
 - name: Ensure matrix-s3fs.service doesn't exist
   file:
     path: "/etc/systemd/system/matrix-s3fs.service"
     state: absent
-  when: "not matrix_s3_media_store_enabled and matrix_s3fs_service_stat.stat.exists"
 
 - name: Ensure s3fs-credentials doesn't exist
   file:
     path: "{{ matrix_base_data_path }}/s3fs-credentials"
     state: absent
-  when: "not matrix_s3_media_store_enabled"
 
 - name: Ensure S3fs Docker image doesn't exist
   docker_image:
     name: "{{ docker_s3fs_image }}"
     state: absent
-  when: "not matrix_s3_media_store_enabled"

roles/matrix-server/tasks/setup_synapse.yml

@@ -10,16 +10,19 @@
   with_items:
     - "{{ matrix_synapse_config_dir_path }}"
     - "{{ matrix_synapse_run_path }}"
+    - "{{ matrix_synapse_storage_path }}"
     # We handle matrix_synapse_media_store_path below, not here,
     # because if it's using S3fs and it's already mounted (from before),
     # trying to chown/chmod it here will cause trouble.
 
+# This will throw a Permission Denied error if already mounted using fuse
 - name: Check Matrix Synapse media store path
   stat: path="{{ matrix_synapse_media_store_path }}"
   register: local_path_media_store_stat
+  ignore_errors: yes
 
 # This is separate and conditional, to ensure we don't execute it
-# if the path already exists (and is likely used by an s3fs mount).
+# if the path already exists or we failed to check, because it's mounted using fuse.
 - name: Ensure Matrix media store path exists
   file:
     path: "{{ matrix_synapse_media_store_path }}"
@@ -27,7 +30,7 @@
     mode: 0750
     owner: "{{ matrix_user_username }}"
     group: "{{ matrix_user_username }}"
-  when: "not local_path_media_store_stat.stat.exists"
+  when: "not local_path_media_store_stat.failed and not local_path_media_store_stat.stat.exists"
 
 - name: Ensure Matrix Docker image is pulled
   docker_image:
@@ -74,7 +77,7 @@
     - {"regexp": "^turn_allow_guests:", "line": 'turn_allow_guests: False'}
     - {"regexp": "^url_preview_enabled:", "line": 'url_preview_enabled: True'}
     - {"regexp": "^max_upload_size:", "line": 'max_upload_size: "{{ matrix_max_upload_size_mb }}M"'}
-    - {"regexp": "^media_store_path:", "line": 'media_store_path: "/matrix-media-store"'}
+    - {"regexp": "^media_store_path:", "line": 'media_store_path: "/matrix-storage/media-store"'}
 
 - name: Augment Matrix config (configure Macaroon secret)
   lineinfile: "dest={{ matrix_synapse_config_dir_path }}/homeserver.yaml"

roles/matrix-server/tasks/start.yml

@@ -4,8 +4,8 @@
   service: name=matrix-postgres enabled=yes state=restarted daemon_reload=yes
   when: "not matrix_postgres_use_external"
 
-- name: Ensure matrix-s3fs autoruns and is restarted
-  service: name=matrix-s3fs enabled=yes state=restarted daemon_reload=yes
+- name: Ensure matrix-goofys autoruns and is restarted
+  service: name=matrix-goofys enabled=yes state=restarted daemon_reload=yes
   when: matrix_s3_media_store_enabled
 
 - name: Ensure matrix-synapse autoruns and is restarted

roles/matrix-server/templates/env/env-goofys.j2

@@ -0,0 +1,2 @@
+AWS_ACCESS_KEY={{ matrix_s3_media_store_aws_access_key }}
+AWS_SECRET_KEY={{ matrix_s3_media_store_aws_secret_key }}

roles/matrix-server/templates/systemd/matrix-goofys.service.j2

@@ -0,0 +1,32 @@
+[Unit]
+Description=Matrix Goofys media store
+After=docker.service
+Requires=docker.service
+
+[Service]
+Type=simple
+ExecStartPre=-/usr/bin/docker kill %n
+ExecStartPre=-/usr/bin/docker rm %n
+ExecStart=/usr/bin/docker run --rm --name %n \
+				--user={{ matrix_user_uid }}:{{ matrix_user_gid }} \
+				-v /etc/passwd:/etc/passwd:ro \
+				-v /etc/group:/etc/group:ro \
+				--security-opt apparmor:unconfined \
+				--cap-add mknod \
+				--cap-add sys_admin \
+				--device=/dev/fuse \
+				-v {{ matrix_synapse_media_store_path }}:/s3:shared \
+				--env-file={{ matrix_environment_variables_data_path }}/goofys \
+				--entrypoint /bin/sh \
+				{{ docker_goofys_image }} \
+				-c 'goofys -f --storage-class=STANDARD_IA --region {{ matrix_s3_media_store_region }} --stat-cache-ttl 60m0s --type-cache-ttl 60m0s --dir-mode 0700 --file-mode 0700 {{ matrix_s3_media_store_bucket_name }} /s3'
+TimeoutStartSec=5min
+ExecStop=-/usr/bin/docker stop %n
+ExecStop=-/usr/bin/docker kill %n
+ExecStop=-/usr/bin/docker rm %n
+ExecStop=-/bin/fusermount -u {{ matrix_synapse_media_store_path }}
+Restart=always
+RestartSec=5
+
+[Install]
+WantedBy=multi-user.target

roles/matrix-server/templates/systemd/matrix-s3fs.service.j2

@@ -1,35 +0,0 @@
-[Unit]
-Description=Matrix S3fs media store
-After=docker.service
-Requires=docker.service
-
-[Service]
-Type=simple
-ExecStartPre=-/usr/bin/docker kill %n
-ExecStartPre=-/usr/bin/docker rm %n
-ExecStartPre=-/usr/bin/mkdir /tmp/matrix-s3fs-cache
-ExecStart=/usr/bin/docker run --rm --name %n \
-				-v {{ matrix_base_data_path }}/s3fs-credentials:/s3fs-credentials \
-				--security-opt apparmor:unconfined \
-				--cap-add mknod \
-				--cap-add sys_admin \
-				--device=/dev/fuse \
-				-v {{ matrix_synapse_media_store_path }}:/media-store:shared \
-				-v /tmp/matrix-s3fs-cache:/s3fs-cache \
-				{{ docker_s3fs_image }} \
-				/usr/bin/s3fs -f \
-					-o allow_other \
-					-o use_cache=/s3fs-cache \
-					-o storage_class=standard_ia \
-					-o passwd_file=/s3fs-credentials \
-					{{ matrix_s3_media_store_bucket_name }} /media-store
-TimeoutStartSec=5min
-ExecStop=-/usr/bin/docker stop %n
-ExecStop=-/usr/bin/docker kill %n
-ExecStop=-/usr/bin/docker rm %n
-ExecStop=-/usr/bin/rm -rf /tmp/matrix-s3fs-cache
-Restart=always
-RestartSec=5
-
-[Install]
-WantedBy=multi-user.target

roles/matrix-server/templates/systemd/matrix-synapse.service.j2

@@ -7,14 +7,20 @@ Requires=matrix-postgres.service
 After=matrix-postgres.service
 {% endif %}
 {% if matrix_s3_media_store_enabled %}
-After=matrix-s3fs.service
-Requires=matrix-s3fs.service
+After=matrix-goofys.service
+Requires=matrix-goofys.service
 {% endif %}
 
 [Service]
 Type=simple
 ExecStartPre=-/usr/bin/docker kill matrix-synapse
 ExecStartPre=-/usr/bin/docker rm matrix-synapse
+{% if matrix_s3_media_store_enabled %}
+# Allow for some time before starting, so that media store can mount.
+# Mounting can happen later too, but if we start writing,
+# we'd write files to the local filesystem and fusermount will complain.
+ExecStartPre=/bin/sleep 5
+{% endif %}
 ExecStart=/usr/bin/docker run --rm --name matrix-synapse \
 			{% if not matrix_postgres_use_external %}
 			--link matrix-postgres:{{ matrix_postgres_connection_hostname }} \
@@ -28,7 +34,7 @@ ExecStart=/usr/bin/docker run --rm --name matrix-synapse \
 			-p {{ matrix_coturn_turn_udp_min_port }}-{{ matrix_coturn_turn_udp_max_port }}:{{ matrix_coturn_turn_udp_min_port }}-{{ matrix_coturn_turn_udp_max_port }}/udp \
 			-v {{ matrix_synapse_config_dir_path }}:/data \
 			-v {{ matrix_synapse_run_path }}:/matrix-run \
-			-v {{ matrix_synapse_media_store_path }}:/matrix-media-store \
+			-v {{ matrix_synapse_storage_path }}:/matrix-storage:slave \
 			{{ docker_matrix_image }}
 ExecStop=-/usr/bin/docker kill matrix-synapse
 ExecStop=-/usr/bin/docker rm matrix-synapse