feat(mastodon): add ansible role for deployment

README.md

@@ -9,6 +9,9 @@ available.
 
 ## Roles
 
+- [`mastodon`](roles/mastodon/README.md): deployment using a container based
+  setup, able to use webfinger delegation.
+
 ## License
 
 [CNPLv7+](LICENSE.md): Cooperative Nonviolent Public License

roles/mastodon/README.md

@@ -0,0 +1,65 @@
+# `finallycoffee.fediverse.mastodon` ansible role
+
+## Overview
+
+This role aims to automate as much as possible with running a docker container
+based mastodon setup. It provides you with the streaming container, sidekiq and
+web (api) as well an nginx routing the application traffic.
+
+You need to provide a postgresql database, the redis server, optionally an
+elasticsearch instance and the mail server. Roles providing components are linked,
+if applicable.
+
+### Usage
+
+The minimum configuration could be as follows:
+
+```yaml
+mastodon_domain: finally.coffee
+# Optional, if you want to host your frontend + api somewhere else
+mastodon_web_domain: frontend.some.website
+
+# you need to provide and manage the following secrets
+mastodon_secret_key: very_long_secret
+mastodon_otp_secret: also_very_long_secret
+mastodon_vapid_public_key: check_mastodon_docs_for_this
+mastodon_vapid_private_key: see_above
+```
+
+#### Database
+
+The database configuration is as follows:
+
+```yaml
+mastodon_database_host: postgres.local
+mastodon_database_port: 5432 #optional, defaults to this
+mastodon_database_user: mastodont
+mastodon_database_pass: hopefully_secure
+mastodon_database_name: mastodon
+```
+
+For seeding the database during initial deployment, you need to set
+`mastodon_seed_database: true` exactly once (when it succeeds).
+
+### Redis
+
+As of writing this, it seems that atleast one component of mastodon can't
+deal with a password for redis, leading to the need to run redis without
+authentification for all components:
+
+```yaml
+mastodon_redis_url: unix:///var/run/redis/mastodon.sock
+```
+
+#### Mail
+
+The mail server for verifications and notifications can be configured as followed:
+
+```yaml
+mastodon_mail_server: mail.example.org
+mastodon_mail_user: mailuser@mydomain.org
+mastodon_mail_password: very_secure_password_for_mailing_account
+```
+
+For further Configuration, see [`defaults/main.yml`](defaults/main.yml) to
+override further keys for configuration

roles/mastodon/defaults/main.yml

@@ -0,0 +1,106 @@
+---
+
+mastodon_user: mastodon
+mastodon_base_path: /opt/mastodon
+mastodon_domain: ~
+mastodon_web_domain: ~
+mastodon_version: 3.5.1
+mastodon_git_upstream_url: "https://github.com/mastodon/mastodon.git"
+
+mastodon_data_path: "{{ mastodon_base_path }}/data"
+mastodon_repo_path: "{{ mastodon_base_path }}/src"
+mastodon_config_path: "{{ mastodon_base_path }}/config"
+mastodon_config_env_file: "{{ mastodon_config_path }}/env.production"
+mastodon_nginx_config_path: "{{ mastodon_base_path }}/nginx-config"
+mastodon_nginx_config_file: "{{ mastodon_nginx_config_path }}/nginx.conf"
+mastodon_nginx_cache_path: "{{ mastodon_base_path }}/nginx-cache"
+
+mastodon_container_bind_ip: "127.0.0.1"
+mastodon_streaming_backend: "{{ mastodon_container_bind_ip }}:4000"
+mastodon_api_backend: "{{ mastodon_container_bind_ip }}:3000"
+mastodon_backend: "{{ mastodon_container_bind_ip }}:5000"
+
+mastodon_container_name: mastodon
+mastodon_container_name_sidekiq: "{{ mastodon_container_name }}_sidekiq"
+mastodon_container_name_streaming: "{{ mastodon_container_name }}_streaming"
+mastodon_container_image_name: "tootsuite/mastodon"
+mastodon_container_image_tag: "v{{ mastodon_version }}"
+mastodon_container_image_ref: "{{ mastodon_container_image_name }}:{{ mastodon_container_image_tag }}"
+mastodon_container_networks:
+  - name: "{{ mastodon_container_network_name }}"
+
+mastodon_container_base_volumes_streaming: []
+mastodon_container_extra_volumes_streaming: "{{ mastodon_container_extra_volumes }}"
+mastodon_container_volumes_streaming: >-
+  {{ mastodon_container_base_volumes_streaming + mastodon_container_extra_volumes_streaming }}
+
+mastodon_container_base_volumes_sidekiq:
+  - "{{ mastodon_repo_path }}/public/system:/mastodon/public/system:ro"
+mastodon_container_extra_volumes_sidekiq: "{{ mastodon_container_extra_volumes }}"
+mastodon_container_volumes_sidekiq: >-
+  {{ mastodon_container_base_volumes_sidekiq + mastodon_container_extra_volumes_sidekiq }}
+
+mastodon_container_base_volumes:
+  - "{{ mastodon_repo_path }}/public:/mastodon/public:z"
+mastodon_container_extra_volumes: []
+mastodon_container_volumes: >-
+  {{ mastodon_container_base_volumes + mastodon_container_extra_volumes }}
+
+mastodon_container_ports_streaming:
+  - "{{ mastodon_streaming_backend }}:4000"
+mastodon_container_ports:
+  - "{{ mastodon_api_backend }}:3000"
+mastodon_container_restart_policy: unless-stopped
+
+mastodon_nginx_version: 1.21.6
+mastodon_nginx_server_name: "{{ mastodon_domain }}"
+mastodon_container_nginx_name: "{{ mastodon_container_name }}_nginx"
+mastodon_container_nginx_image_name: docker.io/library/nginx
+mastodon_container_nginx_image_tag: ~
+mastodon_container_nginx_image_flavour: alpine
+mastodon_container_nginx_image: >-2
+  {{ mastodon_container_nginx_image_name }}:{{ mastodon_container_nginx_image_tag
+  | default(mastodon_nginx_version + ('-' + mastodon_container_nginx_image_flavour if mastodon_container_nginx_image_flavour else ''), True) }}
+mastodon_container_nginx_working_directory: "/var/www/mastodon"
+mastodon_container_nginx_cache_directory: "/var/cache/nginx"
+mastodon_container_volumes_nginx:
+  - "{{ mastodon_nginx_config_file }}:/etc/nginx/conf.d/default.conf:ro"
+  - "{{ mastodon_repo_path }}/public:{{ mastodon_container_nginx_working_directory }}:ro"
+  - "{{ mastodon_nginx_cache_path }}:{{ mastodon_container_nginx_cache_directory }}:z"
+
+mastodon_container_network_name: mastodon
+
+mastodon_secret_key: ~
+mastodon_otp_secret: ~
+mastodon_vapid_public_key: ~
+mastodon_vapid_private_key: ~
+
+mastodon_redis_host: ~
+mastodon_redis_port: ~
+mastodon_redis_url: ~
+mastodon_redis_password: ~
+mastodon_redis_db_index: ~
+
+mastodon_database_host: localhost
+mastodon_database_port: 5432
+mastodon_database_user: mastodon
+mastodon_database_pass: ~
+mastodon_database_name: mastodon
+
+mastodon_mail_server: ~
+mastodon_mail_port: 587
+mastodon_mail_user: ~
+mastodon_mail_password: ~
+mastodon_mail_from_address: "notifications@{{ mastodon_domain }}"
+
+mastodon_elasticsearch_enabled: false
+mastodon_elasticsearch_host: ~
+mastodon_elasticsearch_port: ~
+mastodon_elasticsearch_user: ~
+mastodon_elasticsearch_pass: ~
+
+mastodon_s3_enabled: false
+mastodon_s3_bucket: ~
+mastodon_s3_aws_access_key_id: ~
+mastodon_s3_aws_secret_access_key: ~
+mastodon_s3_alias_host: ~

roles/mastodon/handlers/main.yml

@@ -0,0 +1,33 @@
+---
+
+- name: Restart mastodon sidekiq
+  docker_container:
+    name: "{{ mastodon_container_name_sidekiq }}"
+    state: started
+    restart: true
+  listen:
+    - restart-mastodon
+    - restart-mastodon-sidekiq
+
+- name: Restart mastodon streaming
+  docker_container:
+    name: "{{ mastodon_container_name_streaming }}"
+    state: started
+    restart: true
+  listen:
+    - restart-mastodon
+    - restart-mastodon-streaming
+
+- name: Restart mastodon web
+  docker_container:
+    name: "{{ mastodon_container_name }}"
+    state: started
+    restart: true
+  listen: restart-mastodon
+
+- name: Restart mastodon nginx
+  docker_container:
+    name: "{{ mastodon_container_nginx_name }}"
+    state: started
+    restart: true
+  listen: restart-mastodon-nginx

roles/mastodon/tasks/main.yml

@@ -0,0 +1,176 @@
+---
+
+- name: Ensure mastodon user '{{ mastodon_user }}' exists
+  user:
+    name: "{{ mastodon_user }}"
+    state: present
+    system: true
+  register: mastodon_user_info
+
+- name: Ensure host directories are present
+  file:
+    path: "{{ item.path }}"
+    state: directory
+    owner: "{{ item.owner | default(mastodon_user) }}"
+    group: "{{ item.group | default(mastodon_user) }}"
+    mode: "{{ item.mode | default('0750') }}"
+  loop:
+    - path: "{{ mastodon_base_path }}"
+      mode: '0755'
+    - path: "{{ mastodon_config_path }}"
+    - path: "{{ mastodon_data_path }}"
+    - path: "{{ mastodon_repo_path }}"
+      mode: '0700'
+    - path: "{{ mastodon_nginx_config_path }}"
+    - path: "{{ mastodon_nginx_cache_path }}"
+  loop_control: { label: "{{ item.path }}" }
+
+- name: Ensure environment file is templated
+  template:
+    src: env.j2
+    dest: "{{ mastodon_config_env_file }}"
+    owner: "{{ mastodon_user_info.uid | default(mastodon_user) }}"
+    group: "{{ mastodon_user_info.group | default(mastodon_user) }}"
+    mode: "0640"
+  notify: restart-mastodon
+
+- name: Ensure reverse proxy configuration is templated
+  template:
+    src: nginx.conf.j2
+    dest: "{{ mastodon_nginx_config_file }}"
+    owner: "{{ mastodon_user_info.uid | default(mastodon_user) }}"
+    group: "{{ mastodon_user_info.group | default(mastodon_user) }}"
+    mode: "0640"
+  notify: restart-mastodon-nginx
+
+- name: Ensure mastodon git repository is present and up-to-date
+  git:
+    repo: "{{ mastodon_git_upstream_url }}"
+    dest: "{{ mastodon_repo_path }}"
+    refspec: "v{{ mastodon_version }}"
+    version: "v{{ mastodon_version }}"
+    force: no
+    recursive: yes
+    track_submodules: yes
+  register: git_repo_info
+
+- name: Ensure mastodon git repository and children belong to {{ mastodon_user }}
+  file:
+    path: "{{ mastodon_repo_path }}"
+    state: directory
+    recurse: yes
+    owner: "{{ mastodon_user }}"
+    group: "{{ mastodon_user }}"
+
+- name: Ensure docker network for backend communication is created
+  docker_network:
+    name: "{{ mastodon_container_network_name }}"
+    state: present
+
+- name: Ensure mastodon docker image is built
+  docker_image:
+    name: "{{ mastodon_container_image_name }}"
+    tag: "{{ mastodon_container_image_tag }}"
+    state: present
+    source: build
+    build:
+      path: "{{ mastodon_repo_path }}"
+      args:
+        UID: "{{ mastodon_user_info.uid }}"
+        GID: "{{ mastodon_user_info.group }}"
+  when: git_repo_info.before != git_repo_info.after
+
+- name: Ensure nginx reverse proxy image is present
+  docker_image:
+    name: "{{ mastodon_container_nginx_image }}"
+    state: present
+    source: pull
+    force_source: "{{ mastodon_container_nginx_image_tag|default(false, true) | bool }}"
+  register: masto_nginx_pull
+  until: masto_nginx_pull is succeeded
+  retries: 5
+  delay: 3
+
+- name: Ensure database is seeded
+  docker_container:
+    name: "{{ mastodon_container_name }}_setup_db"
+    image: "{{ mastodon_container_image_ref }}"
+    networks: "{{ mastodon_container_networks }}"
+    volumes: "{{ mastodon_container_volumes }}"
+    env_file: "{{ mastodon_config_env_file }}" 
+    command: "bash -c \"bundle exec rails db:setup\""
+    tty: yes
+    interactive: yes
+    detach: no
+    cleanup: yes
+  when: mastodon_seed_database|default(false, true)
+
+- name: Ensure mastodon sidekiq container '{{ mastodon_container_name_sidekiq }}' is running
+  docker_container:
+    name: "{{ mastodon_container_name_sidekiq }}"
+    image: "{{ mastodon_container_image_ref }}"
+    networks: "{{ mastodon_container_networks }}"
+    volumes: "{{ mastodon_container_volumes_sidekiq }}"
+    env_file: "{{ mastodon_config_env_file }}" 
+    command: "bundle exec sidekiq"
+    restart_policy: "{{ mastodon_container_restart_policy }}"
+    healthcheck:
+      test: ["CMD-SHELL", "ps aux | grep '[s]idekiq\ 6' || false"]
+      interval: 5s
+      retries: 3
+      start_period: 0s
+      timeout: 5s
+
+- name: Ensure mastodon streaming container '{{ mastodon_container_name_streaming }}' is running
+  docker_container:
+    name: "{{ mastodon_container_name_streaming }}"
+    image: "{{ mastodon_container_image_ref }}"
+    networks: "{{ mastodon_container_networks }}"
+    volumes: "{{ mastodon_container_volumes_streaming }}"
+    env_file: "{{ mastodon_config_env_file }}" 
+    command: "node ./streaming"
+    restart_policy: "{{ mastodon_container_restart_policy }}"
+    ports: "{{ mastodon_container_ports_streaming }}"
+    healthcheck:
+      test: ["CMD-SHELL", "wget -q --spider --proxy=off localhost:4000/api/v1/streaming/health || exit 1"]
+      interval: 5s
+      retries: 3
+      start_period: 0s
+      timeout: 5s
+
+- name: Ensure mastodon container '{{ mastodon_container_name }}' is running
+  docker_container:
+    name: "{{ mastodon_container_name }}"
+    image: "{{ mastodon_container_image_ref }}"
+    networks: "{{ mastodon_container_networks }}"
+    volumes: "{{ mastodon_container_volumes }}"
+    env_file: "{{ mastodon_config_env_file }}" 
+    command: "bash -c \"rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 3000\""
+    restart_policy: "{{ mastodon_container_restart_policy }}"
+    ports: "{{ mastodon_container_ports }}"
+    user: "{{ mastodon_user }}"
+    healthcheck:
+      test: ["CMD-SHELL", "wget -q --spider --proxy=off localhost:3000/health || exit 1"]
+      interval: 5s
+      retries: 3
+      start_period: 0s
+      timeout: 5s
+
+- name: Ensure mastodon-nginx container '{{ mastodon_container_nginx_name }}' is running
+  docker_container:
+    name: "{{ mastodon_container_nginx_name }}"
+    image: "{{ mastodon_container_nginx_image }}"
+    network_mode: host
+    volumes: "{{ mastodon_container_volumes_nginx }}"
+    restart_policy: "{{ mastodon_container_restart_policy }}"
+
+- name: Ensure assets are precompiled
+  docker_container:
+    name: "{{ mastodon_container_name }}"
+    env_file: "{{ mastodon_config_env_file }}"
+    command: "bash -c \"bundle exec rails assets:precompile\""
+    user: "{{ mastodon_user }}"
+    tty: yes
+    interactive: yes
+    detach: no
+  when: git_repo_info.before != git_repo_info.after

roles/mastodon/templates/env.j2

@@ -0,0 +1,86 @@
+# This is a sample configuration file. You can generate your configuration
+# with the `rake mastodon:setup` interactive setup wizard, but to customize
+# your setup even further, you'll need to edit it manually. This sample does
+# not demonstrate all available configuration options. Please look at
+# https://docs.joinmastodon.org/admin/config/ for the full documentation.
+
+# Note that this file accepts slightly different syntax depending on whether
+# you are using `docker-compose` or not. In particular, if you use
+# `docker-compose`, the value of each declared variable will be taken verbatim,
+# including surrounding quotes.
+# See: https://github.com/mastodon/mastodon/issues/16895
+
+# Federation
+# ----------
+# This identifies your server and cannot be changed safely later
+# ----------
+LOCAL_DOMAIN={{ mastodon_domain }}
+{% if mastodon_web_domain|default(false, true) %}
+WEB_DOMAIN={{ mastodon_web_domain }}
+{% endif %}
+
+# Redis
+# -----
+{% if mastodon_redis_host|default(false, true) %}
+REDIS_HOST={{ mastodon_redis_host }}
+{% endif %}
+{% if mastodon_redis_port|default(false, true) %}
+REDIS_PORT={{ mastodon_redis_port }}
+{% endif %}
+{% if mastodon_redis_url %}
+REDIS_URL={{ mastodon_redis_url }}
+{% endif %}
+{% if mastodon_redis_password %}
+REDIS_PASSWORD={{ mastodon_redis_password }}
+{% endif %}
+{% if mastodon_redis_db_index %}
+REDIS_DB_INDEX={{ mastodon_redis_db_index }}
+{% endif %}
+
+# PostgreSQL
+# ----------
+DB_HOST={{ mastodon_database_host }}
+DB_USER={{ mastodon_database_user }}
+DB_NAME={{ mastodon_database_name }}
+DB_PASS={{ mastodon_database_pass }}
+DB_PORT={{ mastodon_database_port }}
+
+# Elasticsearch (optional)
+# ------------------------
+ES_ENABLED={{ mastodon_elasticsearch_enabled }}
+ES_HOST={{ mastodon_elasticsearch_host }}
+ES_PORT={{ mastodon_elasticsearch_port }}
+# Authentication for ES (optional)
+ES_USER={{ mastodon_elasticsearch_user }}
+ES_PASS={{ mastodon_elasticsearch_pass }}
+
+# Secrets
+# -------
+# Make sure to use `rake secret` to generate secrets
+# -------
+SECRET_KEY_BASE={{ mastodon_secret_key }}
+OTP_SECRET={{ mastodon_otp_secret }}
+
+# Web Push
+# --------
+# Generate with `rake mastodon:webpush:generate_vapid_key`
+# --------
+VAPID_PRIVATE_KEY={{ mastodon_vapid_private_key }}
+VAPID_PUBLIC_KEY={{ mastodon_vapid_public_key }}
+
+# Sending mail
+# ------------
+SMTP_SERVER={{ mastodon_mail_server }}
+SMTP_PORT={{ mastodon_mail_port }}
+SMTP_LOGIN={{ mastodon_mail_user }}
+SMTP_PASSWORD={{ mastodon_mail_password }}
+SMTP_FROM_ADDRESS={{ mastodon_mail_from_address }}
+
+# File storage (optional)
+# -----------------------
+S3_ENABLED={{ mastodon_s3_enabled }}
+S3_BUCKET={{ mastodon_s3_bucket }}
+AWS_ACCESS_KEY_ID={{ mastodon_s3_aws_access_key_id }}
+AWS_SECRET_ACCESS_KEY={{ mastodon_s3_aws_secret_access_key }}
+S3_ALIAS_HOST={{ mastodon_s3_alias_host }}
+

roles/mastodon/templates/nginx.conf.j2

@@ -0,0 +1,94 @@
+map $http_upgrade $connection_upgrade {
+  default upgrade;
+  ''      close;
+}
+
+upstream backend {
+    server {{ mastodon_api_backend }} fail_timeout=0;
+}
+
+upstream streaming {
+    server {{ mastodon_streaming_backend }} fail_timeout=0;
+}
+
+proxy_cache_path {{ mastodon_container_nginx_cache_directory }} levels=1:2 keys_zone=CACHE:10m inactive=7d max_size=2g;
+
+server {
+  listen {{ mastodon_backend }};
+  server_name {{ mastodon_nginx_server_name }};
+
+  keepalive_timeout    70;
+  sendfile             on;
+  client_max_body_size 200m;
+
+  root {{ mastodon_container_nginx_working_directory }};
+
+  gzip on;
+  gzip_disable "msie6";
+  gzip_vary on;
+  gzip_proxied any;
+  gzip_comp_level 6;
+  gzip_buffers 16 8k;
+  gzip_http_version 1.1;
+  gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript image/svg+xml image/x-icon;
+
+  location / {
+    try_files $uri @proxy;
+  }
+
+  location ~ ^/(emoji|packs|system/accounts/avatars|system/media_attachments/files) {
+    add_header Cache-Control "public, max-age=31536000, immutable";
+    add_header Strict-Transport-Security "max-age=31536000" always;
+    try_files $uri @proxy;
+  }
+
+  location /sw.js {
+    add_header Cache-Control "public, max-age=0";
+    add_header Strict-Transport-Security "max-age=31536000" always;
+    try_files $uri @proxy;
+  }
+
+  location @proxy {
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_set_header Proxy "";
+    proxy_pass_header Server;
+
+    proxy_pass http://backend;
+    proxy_buffering on;
+    proxy_redirect off;
+    proxy_http_version 1.1;
+    proxy_set_header Upgrade $http_upgrade;
+    proxy_set_header Connection $connection_upgrade;
+
+    proxy_cache CACHE;
+    proxy_cache_valid 200 7d;
+    proxy_cache_valid 410 24h;
+    proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504;
+    add_header X-Cached $upstream_cache_status;
+    add_header Strict-Transport-Security "max-age=31536000" always;
+
+    tcp_nodelay on;
+  }
+
+  location /api/v1/streaming {
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_set_header Proxy "";
+
+    proxy_pass http://streaming;
+    proxy_buffering off;
+    proxy_redirect off;
+    proxy_http_version 1.1;
+    proxy_set_header Upgrade $http_upgrade;
+    proxy_set_header Connection $connection_upgrade;
+
+    tcp_nodelay on;
+  }
+
+  error_page 500 501 502 503 504 /500.html;
+}