From accd829e9130206b05fbaa1c4f1d98b23ca6eadb Mon Sep 17 00:00:00 2001 From: Johanna Dorothea Reichmann Date: Sat, 23 Apr 2022 18:02:30 +0200 Subject: [PATCH] feat(mastodon): add ansible role for deployment --- README.md | 3 + roles/mastodon/README.md | 65 +++++++++ roles/mastodon/defaults/main.yml | 106 +++++++++++++++ roles/mastodon/handlers/main.yml | 33 +++++ roles/mastodon/tasks/main.yml | 176 +++++++++++++++++++++++++ roles/mastodon/templates/env.j2 | 86 ++++++++++++ roles/mastodon/templates/nginx.conf.j2 | 94 +++++++++++++ roles/mastodon/vars/main.yml | 0 8 files changed, 563 insertions(+) create mode 100644 roles/mastodon/README.md create mode 100644 roles/mastodon/defaults/main.yml create mode 100644 roles/mastodon/handlers/main.yml create mode 100644 roles/mastodon/tasks/main.yml create mode 100644 roles/mastodon/templates/env.j2 create mode 100644 roles/mastodon/templates/nginx.conf.j2 create mode 100644 roles/mastodon/vars/main.yml diff --git a/README.md b/README.md index 4f342da..7ba4a03 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,9 @@ available. ## Roles +- [`mastodon`](roles/mastodon/README.md): deployment using a container based + setup, able to use webfinger delegation. + ## License [CNPLv7+](LICENSE.md): Cooperative Nonviolent Public License diff --git a/roles/mastodon/README.md b/roles/mastodon/README.md new file mode 100644 index 0000000..fe3dba4 --- /dev/null +++ b/roles/mastodon/README.md @@ -0,0 +1,65 @@ +# `finallycoffee.fediverse.mastodon` ansible role + +## Overview + +This role aims to automate as much as possible with running a docker container +based mastodon setup. It provides you with the streaming container, sidekiq and +web (api) as well an nginx routing the application traffic. + +You need to provide a postgresql database, the redis server, optionally an +elasticsearch instance and the mail server. Roles providing components are linked, +if applicable. + +### Usage + +The minimum configuration could be as follows: + +```yaml +mastodon_domain: finally.coffee +# Optional, if you want to host your frontend + api somewhere else +mastodon_web_domain: frontend.some.website + +# you need to provide and manage the following secrets +mastodon_secret_key: very_long_secret +mastodon_otp_secret: also_very_long_secret +mastodon_vapid_public_key: check_mastodon_docs_for_this +mastodon_vapid_private_key: see_above +``` + +#### Database + +The database configuration is as follows: + +```yaml +mastodon_database_host: postgres.local +mastodon_database_port: 5432 #optional, defaults to this +mastodon_database_user: mastodont +mastodon_database_pass: hopefully_secure +mastodon_database_name: mastodon +``` + +For seeding the database during initial deployment, you need to set +`mastodon_seed_database: true` exactly once (when it succeeds). + +### Redis + +As of writing this, it seems that atleast one component of mastodon can't +deal with a password for redis, leading to the need to run redis without +authentification for all components: + +```yaml +mastodon_redis_url: unix:///var/run/redis/mastodon.sock +``` + +#### Mail + +The mail server for verifications and notifications can be configured as followed: + +```yaml +mastodon_mail_server: mail.example.org +mastodon_mail_user: mailuser@mydomain.org +mastodon_mail_password: very_secure_password_for_mailing_account +``` + +For further Configuration, see [`defaults/main.yml`](defaults/main.yml) to +override further keys for configuration diff --git a/roles/mastodon/defaults/main.yml b/roles/mastodon/defaults/main.yml new file mode 100644 index 0000000..3296e0a --- /dev/null +++ b/roles/mastodon/defaults/main.yml @@ -0,0 +1,106 @@ +--- + +mastodon_user: mastodon +mastodon_base_path: /opt/mastodon +mastodon_domain: ~ +mastodon_web_domain: ~ +mastodon_version: 3.5.1 +mastodon_git_upstream_url: "https://github.com/mastodon/mastodon.git" + +mastodon_data_path: "{{ mastodon_base_path }}/data" +mastodon_repo_path: "{{ mastodon_base_path }}/src" +mastodon_config_path: "{{ mastodon_base_path }}/config" +mastodon_config_env_file: "{{ mastodon_config_path }}/env.production" +mastodon_nginx_config_path: "{{ mastodon_base_path }}/nginx-config" +mastodon_nginx_config_file: "{{ mastodon_nginx_config_path }}/nginx.conf" +mastodon_nginx_cache_path: "{{ mastodon_base_path }}/nginx-cache" + +mastodon_container_bind_ip: "127.0.0.1" +mastodon_streaming_backend: "{{ mastodon_container_bind_ip }}:4000" +mastodon_api_backend: "{{ mastodon_container_bind_ip }}:3000" +mastodon_backend: "{{ mastodon_container_bind_ip }}:5000" + +mastodon_container_name: mastodon +mastodon_container_name_sidekiq: "{{ mastodon_container_name }}_sidekiq" +mastodon_container_name_streaming: "{{ mastodon_container_name }}_streaming" +mastodon_container_image_name: "tootsuite/mastodon" +mastodon_container_image_tag: "v{{ mastodon_version }}" +mastodon_container_image_ref: "{{ mastodon_container_image_name }}:{{ mastodon_container_image_tag }}" +mastodon_container_networks: + - name: "{{ mastodon_container_network_name }}" + +mastodon_container_base_volumes_streaming: [] +mastodon_container_extra_volumes_streaming: "{{ mastodon_container_extra_volumes }}" +mastodon_container_volumes_streaming: >- + {{ mastodon_container_base_volumes_streaming + mastodon_container_extra_volumes_streaming }} + +mastodon_container_base_volumes_sidekiq: + - "{{ mastodon_repo_path }}/public/system:/mastodon/public/system:ro" +mastodon_container_extra_volumes_sidekiq: "{{ mastodon_container_extra_volumes }}" +mastodon_container_volumes_sidekiq: >- + {{ mastodon_container_base_volumes_sidekiq + mastodon_container_extra_volumes_sidekiq }} + +mastodon_container_base_volumes: + - "{{ mastodon_repo_path }}/public:/mastodon/public:z" +mastodon_container_extra_volumes: [] +mastodon_container_volumes: >- + {{ mastodon_container_base_volumes + mastodon_container_extra_volumes }} + +mastodon_container_ports_streaming: + - "{{ mastodon_streaming_backend }}:4000" +mastodon_container_ports: + - "{{ mastodon_api_backend }}:3000" +mastodon_container_restart_policy: unless-stopped + +mastodon_nginx_version: 1.21.6 +mastodon_nginx_server_name: "{{ mastodon_domain }}" +mastodon_container_nginx_name: "{{ mastodon_container_name }}_nginx" +mastodon_container_nginx_image_name: docker.io/library/nginx +mastodon_container_nginx_image_tag: ~ +mastodon_container_nginx_image_flavour: alpine +mastodon_container_nginx_image: >-2 + {{ mastodon_container_nginx_image_name }}:{{ mastodon_container_nginx_image_tag + | default(mastodon_nginx_version + ('-' + mastodon_container_nginx_image_flavour if mastodon_container_nginx_image_flavour else ''), True) }} +mastodon_container_nginx_working_directory: "/var/www/mastodon" +mastodon_container_nginx_cache_directory: "/var/cache/nginx" +mastodon_container_volumes_nginx: + - "{{ mastodon_nginx_config_file }}:/etc/nginx/conf.d/default.conf:ro" + - "{{ mastodon_repo_path }}/public:{{ mastodon_container_nginx_working_directory }}:ro" + - "{{ mastodon_nginx_cache_path }}:{{ mastodon_container_nginx_cache_directory }}:z" + +mastodon_container_network_name: mastodon + +mastodon_secret_key: ~ +mastodon_otp_secret: ~ +mastodon_vapid_public_key: ~ +mastodon_vapid_private_key: ~ + +mastodon_redis_host: ~ +mastodon_redis_port: ~ +mastodon_redis_url: ~ +mastodon_redis_password: ~ +mastodon_redis_db_index: ~ + +mastodon_database_host: localhost +mastodon_database_port: 5432 +mastodon_database_user: mastodon +mastodon_database_pass: ~ +mastodon_database_name: mastodon + +mastodon_mail_server: ~ +mastodon_mail_port: 587 +mastodon_mail_user: ~ +mastodon_mail_password: ~ +mastodon_mail_from_address: "notifications@{{ mastodon_domain }}" + +mastodon_elasticsearch_enabled: false +mastodon_elasticsearch_host: ~ +mastodon_elasticsearch_port: ~ +mastodon_elasticsearch_user: ~ +mastodon_elasticsearch_pass: ~ + +mastodon_s3_enabled: false +mastodon_s3_bucket: ~ +mastodon_s3_aws_access_key_id: ~ +mastodon_s3_aws_secret_access_key: ~ +mastodon_s3_alias_host: ~ diff --git a/roles/mastodon/handlers/main.yml b/roles/mastodon/handlers/main.yml new file mode 100644 index 0000000..f387ede --- /dev/null +++ b/roles/mastodon/handlers/main.yml @@ -0,0 +1,33 @@ +--- + +- name: Restart mastodon sidekiq + docker_container: + name: "{{ mastodon_container_name_sidekiq }}" + state: started + restart: true + listen: + - restart-mastodon + - restart-mastodon-sidekiq + +- name: Restart mastodon streaming + docker_container: + name: "{{ mastodon_container_name_streaming }}" + state: started + restart: true + listen: + - restart-mastodon + - restart-mastodon-streaming + +- name: Restart mastodon web + docker_container: + name: "{{ mastodon_container_name }}" + state: started + restart: true + listen: restart-mastodon + +- name: Restart mastodon nginx + docker_container: + name: "{{ mastodon_container_nginx_name }}" + state: started + restart: true + listen: restart-mastodon-nginx diff --git a/roles/mastodon/tasks/main.yml b/roles/mastodon/tasks/main.yml new file mode 100644 index 0000000..98caf9d --- /dev/null +++ b/roles/mastodon/tasks/main.yml @@ -0,0 +1,176 @@ +--- + +- name: Ensure mastodon user '{{ mastodon_user }}' exists + user: + name: "{{ mastodon_user }}" + state: present + system: true + register: mastodon_user_info + +- name: Ensure host directories are present + file: + path: "{{ item.path }}" + state: directory + owner: "{{ item.owner | default(mastodon_user) }}" + group: "{{ item.group | default(mastodon_user) }}" + mode: "{{ item.mode | default('0750') }}" + loop: + - path: "{{ mastodon_base_path }}" + mode: '0755' + - path: "{{ mastodon_config_path }}" + - path: "{{ mastodon_data_path }}" + - path: "{{ mastodon_repo_path }}" + mode: '0700' + - path: "{{ mastodon_nginx_config_path }}" + - path: "{{ mastodon_nginx_cache_path }}" + loop_control: { label: "{{ item.path }}" } + +- name: Ensure environment file is templated + template: + src: env.j2 + dest: "{{ mastodon_config_env_file }}" + owner: "{{ mastodon_user_info.uid | default(mastodon_user) }}" + group: "{{ mastodon_user_info.group | default(mastodon_user) }}" + mode: "0640" + notify: restart-mastodon + +- name: Ensure reverse proxy configuration is templated + template: + src: nginx.conf.j2 + dest: "{{ mastodon_nginx_config_file }}" + owner: "{{ mastodon_user_info.uid | default(mastodon_user) }}" + group: "{{ mastodon_user_info.group | default(mastodon_user) }}" + mode: "0640" + notify: restart-mastodon-nginx + +- name: Ensure mastodon git repository is present and up-to-date + git: + repo: "{{ mastodon_git_upstream_url }}" + dest: "{{ mastodon_repo_path }}" + refspec: "v{{ mastodon_version }}" + version: "v{{ mastodon_version }}" + force: no + recursive: yes + track_submodules: yes + register: git_repo_info + +- name: Ensure mastodon git repository and children belong to {{ mastodon_user }} + file: + path: "{{ mastodon_repo_path }}" + state: directory + recurse: yes + owner: "{{ mastodon_user }}" + group: "{{ mastodon_user }}" + +- name: Ensure docker network for backend communication is created + docker_network: + name: "{{ mastodon_container_network_name }}" + state: present + +- name: Ensure mastodon docker image is built + docker_image: + name: "{{ mastodon_container_image_name }}" + tag: "{{ mastodon_container_image_tag }}" + state: present + source: build + build: + path: "{{ mastodon_repo_path }}" + args: + UID: "{{ mastodon_user_info.uid }}" + GID: "{{ mastodon_user_info.group }}" + when: git_repo_info.before != git_repo_info.after + +- name: Ensure nginx reverse proxy image is present + docker_image: + name: "{{ mastodon_container_nginx_image }}" + state: present + source: pull + force_source: "{{ mastodon_container_nginx_image_tag|default(false, true) | bool }}" + register: masto_nginx_pull + until: masto_nginx_pull is succeeded + retries: 5 + delay: 3 + +- name: Ensure database is seeded + docker_container: + name: "{{ mastodon_container_name }}_setup_db" + image: "{{ mastodon_container_image_ref }}" + networks: "{{ mastodon_container_networks }}" + volumes: "{{ mastodon_container_volumes }}" + env_file: "{{ mastodon_config_env_file }}" + command: "bash -c \"bundle exec rails db:setup\"" + tty: yes + interactive: yes + detach: no + cleanup: yes + when: mastodon_seed_database|default(false, true) + +- name: Ensure mastodon sidekiq container '{{ mastodon_container_name_sidekiq }}' is running + docker_container: + name: "{{ mastodon_container_name_sidekiq }}" + image: "{{ mastodon_container_image_ref }}" + networks: "{{ mastodon_container_networks }}" + volumes: "{{ mastodon_container_volumes_sidekiq }}" + env_file: "{{ mastodon_config_env_file }}" + command: "bundle exec sidekiq" + restart_policy: "{{ mastodon_container_restart_policy }}" + healthcheck: + test: ["CMD-SHELL", "ps aux | grep '[s]idekiq\ 6' || false"] + interval: 5s + retries: 3 + start_period: 0s + timeout: 5s + +- name: Ensure mastodon streaming container '{{ mastodon_container_name_streaming }}' is running + docker_container: + name: "{{ mastodon_container_name_streaming }}" + image: "{{ mastodon_container_image_ref }}" + networks: "{{ mastodon_container_networks }}" + volumes: "{{ mastodon_container_volumes_streaming }}" + env_file: "{{ mastodon_config_env_file }}" + command: "node ./streaming" + restart_policy: "{{ mastodon_container_restart_policy }}" + ports: "{{ mastodon_container_ports_streaming }}" + healthcheck: + test: ["CMD-SHELL", "wget -q --spider --proxy=off localhost:4000/api/v1/streaming/health || exit 1"] + interval: 5s + retries: 3 + start_period: 0s + timeout: 5s + +- name: Ensure mastodon container '{{ mastodon_container_name }}' is running + docker_container: + name: "{{ mastodon_container_name }}" + image: "{{ mastodon_container_image_ref }}" + networks: "{{ mastodon_container_networks }}" + volumes: "{{ mastodon_container_volumes }}" + env_file: "{{ mastodon_config_env_file }}" + command: "bash -c \"rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 3000\"" + restart_policy: "{{ mastodon_container_restart_policy }}" + ports: "{{ mastodon_container_ports }}" + user: "{{ mastodon_user }}" + healthcheck: + test: ["CMD-SHELL", "wget -q --spider --proxy=off localhost:3000/health || exit 1"] + interval: 5s + retries: 3 + start_period: 0s + timeout: 5s + +- name: Ensure mastodon-nginx container '{{ mastodon_container_nginx_name }}' is running + docker_container: + name: "{{ mastodon_container_nginx_name }}" + image: "{{ mastodon_container_nginx_image }}" + network_mode: host + volumes: "{{ mastodon_container_volumes_nginx }}" + restart_policy: "{{ mastodon_container_restart_policy }}" + +- name: Ensure assets are precompiled + docker_container: + name: "{{ mastodon_container_name }}" + env_file: "{{ mastodon_config_env_file }}" + command: "bash -c \"bundle exec rails assets:precompile\"" + user: "{{ mastodon_user }}" + tty: yes + interactive: yes + detach: no + when: git_repo_info.before != git_repo_info.after diff --git a/roles/mastodon/templates/env.j2 b/roles/mastodon/templates/env.j2 new file mode 100644 index 0000000..1b9e949 --- /dev/null +++ b/roles/mastodon/templates/env.j2 @@ -0,0 +1,86 @@ +# This is a sample configuration file. You can generate your configuration +# with the `rake mastodon:setup` interactive setup wizard, but to customize +# your setup even further, you'll need to edit it manually. This sample does +# not demonstrate all available configuration options. Please look at +# https://docs.joinmastodon.org/admin/config/ for the full documentation. + +# Note that this file accepts slightly different syntax depending on whether +# you are using `docker-compose` or not. In particular, if you use +# `docker-compose`, the value of each declared variable will be taken verbatim, +# including surrounding quotes. +# See: https://github.com/mastodon/mastodon/issues/16895 + +# Federation +# ---------- +# This identifies your server and cannot be changed safely later +# ---------- +LOCAL_DOMAIN={{ mastodon_domain }} +{% if mastodon_web_domain|default(false, true) %} +WEB_DOMAIN={{ mastodon_web_domain }} +{% endif %} + +# Redis +# ----- +{% if mastodon_redis_host|default(false, true) %} +REDIS_HOST={{ mastodon_redis_host }} +{% endif %} +{% if mastodon_redis_port|default(false, true) %} +REDIS_PORT={{ mastodon_redis_port }} +{% endif %} +{% if mastodon_redis_url %} +REDIS_URL={{ mastodon_redis_url }} +{% endif %} +{% if mastodon_redis_password %} +REDIS_PASSWORD={{ mastodon_redis_password }} +{% endif %} +{% if mastodon_redis_db_index %} +REDIS_DB_INDEX={{ mastodon_redis_db_index }} +{% endif %} + +# PostgreSQL +# ---------- +DB_HOST={{ mastodon_database_host }} +DB_USER={{ mastodon_database_user }} +DB_NAME={{ mastodon_database_name }} +DB_PASS={{ mastodon_database_pass }} +DB_PORT={{ mastodon_database_port }} + +# Elasticsearch (optional) +# ------------------------ +ES_ENABLED={{ mastodon_elasticsearch_enabled }} +ES_HOST={{ mastodon_elasticsearch_host }} +ES_PORT={{ mastodon_elasticsearch_port }} +# Authentication for ES (optional) +ES_USER={{ mastodon_elasticsearch_user }} +ES_PASS={{ mastodon_elasticsearch_pass }} + +# Secrets +# ------- +# Make sure to use `rake secret` to generate secrets +# ------- +SECRET_KEY_BASE={{ mastodon_secret_key }} +OTP_SECRET={{ mastodon_otp_secret }} + +# Web Push +# -------- +# Generate with `rake mastodon:webpush:generate_vapid_key` +# -------- +VAPID_PRIVATE_KEY={{ mastodon_vapid_private_key }} +VAPID_PUBLIC_KEY={{ mastodon_vapid_public_key }} + +# Sending mail +# ------------ +SMTP_SERVER={{ mastodon_mail_server }} +SMTP_PORT={{ mastodon_mail_port }} +SMTP_LOGIN={{ mastodon_mail_user }} +SMTP_PASSWORD={{ mastodon_mail_password }} +SMTP_FROM_ADDRESS={{ mastodon_mail_from_address }} + +# File storage (optional) +# ----------------------- +S3_ENABLED={{ mastodon_s3_enabled }} +S3_BUCKET={{ mastodon_s3_bucket }} +AWS_ACCESS_KEY_ID={{ mastodon_s3_aws_access_key_id }} +AWS_SECRET_ACCESS_KEY={{ mastodon_s3_aws_secret_access_key }} +S3_ALIAS_HOST={{ mastodon_s3_alias_host }} + diff --git a/roles/mastodon/templates/nginx.conf.j2 b/roles/mastodon/templates/nginx.conf.j2 new file mode 100644 index 0000000..37d9b54 --- /dev/null +++ b/roles/mastodon/templates/nginx.conf.j2 @@ -0,0 +1,94 @@ +map $http_upgrade $connection_upgrade { + default upgrade; + '' close; +} + +upstream backend { + server {{ mastodon_api_backend }} fail_timeout=0; +} + +upstream streaming { + server {{ mastodon_streaming_backend }} fail_timeout=0; +} + +proxy_cache_path {{ mastodon_container_nginx_cache_directory }} levels=1:2 keys_zone=CACHE:10m inactive=7d max_size=2g; + +server { + listen {{ mastodon_backend }}; + server_name {{ mastodon_nginx_server_name }}; + + keepalive_timeout 70; + sendfile on; + client_max_body_size 200m; + + root {{ mastodon_container_nginx_working_directory }}; + + gzip on; + gzip_disable "msie6"; + gzip_vary on; + gzip_proxied any; + gzip_comp_level 6; + gzip_buffers 16 8k; + gzip_http_version 1.1; + gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript image/svg+xml image/x-icon; + + location / { + try_files $uri @proxy; + } + + location ~ ^/(emoji|packs|system/accounts/avatars|system/media_attachments/files) { + add_header Cache-Control "public, max-age=31536000, immutable"; + add_header Strict-Transport-Security "max-age=31536000" always; + try_files $uri @proxy; + } + + location /sw.js { + add_header Cache-Control "public, max-age=0"; + add_header Strict-Transport-Security "max-age=31536000" always; + try_files $uri @proxy; + } + + location @proxy { + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Proxy ""; + proxy_pass_header Server; + + proxy_pass http://backend; + proxy_buffering on; + proxy_redirect off; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + + proxy_cache CACHE; + proxy_cache_valid 200 7d; + proxy_cache_valid 410 24h; + proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504; + add_header X-Cached $upstream_cache_status; + add_header Strict-Transport-Security "max-age=31536000" always; + + tcp_nodelay on; + } + + location /api/v1/streaming { + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Proxy ""; + + proxy_pass http://streaming; + proxy_buffering off; + proxy_redirect off; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + + tcp_nodelay on; + } + + error_page 500 501 502 503 504 /500.html; +} diff --git a/roles/mastodon/vars/main.yml b/roles/mastodon/vars/main.yml new file mode 100644 index 0000000..e69de29