feat(restic): add migrated role from finallycoffee.services

README.md

@@ -5,12 +5,25 @@
 This ansible collection provides various roles for installing
 and configuring basic system utilities like gnupg, ssh etc
 
+- [`elasticsearch`](roles/elasticsearch/README.md): Deploy [elasticsearch](https://www.docker.elastic.co/r/elasticsearch/elasticsearch-oss),
+  a popular (distributed) search and analytics engine, mostly known by it's
+  letter "E" in the ELK-stack.
+
 - [`git`](roles/git/README.md): configures git on the target system
+
 - [`gnupg`](roles/gnupg/README.md): configures gnupg on the target system
+
 - [`mariadb`](roles/mariadb/README.md): runs [MariaDB Server](https://mariadb.org/), one of the world's most popular open source relational database
+
+- [`minio`](roles/minio/README.md): Deploy [min.io](https://min.io), an
+  s3-compatible object storage server, using docker containers.
+
 - [`nginx`](roles/nginx/README.md): [nginx](https://www.nginx.com/),
   an advanced load balancer, webserver and reverse proxy.
 
+- [`restic`](roles/restic/README.md): Manage backups using restic
+  and persist them to a configurable backend.
+
 ## License
 
 [CNPLv7+](LICENSE.md): Cooperative Nonviolent Public License

roles/elasticsearch/README.md

@@ -0,0 +1,22 @@
+# `finallycoffee.base.elastiscsearch`
+
+A simple ansible role which deploys a single-node elastic container to provide
+an easy way to do some indexing.
+
+## Usage
+
+Per default, `/opt/elasticsearch/data` is used to persist data, it is
+customizable by using either `elasticsearch_base_path` or `elasticsearch_data_path`.
+
+As elasticsearch be can be quite memory heavy, the maximum amount of allowed RAM
+can be configured using `elasticsearch_allocated_ram_mb`, defaulting to 512 (mb).
+
+The cluster name and discovery type can be overridden using
+`elasticsearch_config_cluster_name` (default: elastic) and
+`elasticsearch_config_discovery_type` (default: single-node), should one
+need a multi-node elasticsearch deployment.
+
+Per default, no ports or networks are mapped, and explizit mapping using
+either ports (`elasticsearch_container_ports`) or networks
+(`elasticsearch_container_networks`) is required in order for other services
+to use elastic.

roles/elasticsearch/defaults/main.yml

@@ -0,0 +1,35 @@
+---
+
+elasticsearch_version: 7.17.7
+
+elasticsearch_base_path: /opt/elasticsearch
+elasticsearch_data_path: "{{ elasticsearch_base_path }}/data"
+
+elasticsearch_config_cluster_name: elastic
+elasticsearch_config_discovery_type: single-node
+elasticsearch_config_boostrap_memory_lock: true
+elasticsearch_allocated_ram_mb: 512
+
+elasticsearch_container_image_name: docker.elastic.co/elasticsearch/elasticsearch-oss
+elasticsearch_container_image_tag: ~
+elasticsearch_container_image: >-
+  {{ elasticsearch_container_image_name }}:{{ elasticsearch_container_image_tag | default(elasticsearch_version, true) }}
+
+elasticsearch_container_name: elasticsearch
+elasticsearch_container_env:
+  "ES_JAVA_OPTS": "-Xms{{ elasticsearch_allocated_ram_mb }}m -Xmx{{ elasticsearch_allocated_ram_mb }}m"
+  "cluster.name": "{{ elasticsearch_config_cluster_name }}"
+  "discovery.type": "{{ elasticsearch_config_discovery_type }}"
+  "bootstrap.memory_lock": "{{ 'true' if elasticsearch_config_boostrap_memory_lock else 'false' }}"
+elasticsearch_container_user: ~
+elasticsearch_container_ports: ~
+elasticsearch_container_labels:
+  version: "{{ elasticsearch_version }}"
+elasticsearch_container_ulimits:
+#  - "memlock:{{ (1.5 * 1024 * elasticsearch_allocated_ram_mb) | int }}:{{ (1.5 * 1024 * elasticsearch_allocated_ram_mb) | int }}"
+  - "memlock:-1:-1"
+elasticsearch_container_volumes:
+  - "{{ elasticsearch_data_path }}:/usr/share/elasticsearch/data:z"
+elasticsearch_container_networks: ~
+elasticsearch_container_purge_networks: ~
+elasticsearch_container_restart_policy: unless-stopped

roles/elasticsearch/tasks/main.yml

@@ -0,0 +1,32 @@
+---
+
+- name: Ensure host directories are present
+  file:
+    path: "{{ item }}"
+    state: directory
+    mode: "0777"
+  loop:
+    - "{{ elasticsearch_base_path }}"
+    - "{{ elasticsearch_data_path }}"
+
+- name: Ensure elastic container image is present
+  docker_image:
+    name: "{{ elasticsearch_container_image }}"
+    state: present
+    source: pull
+    force_source: "{{ elasticsearch_container_image_tag|default(false, true)|bool }}"
+
+- name: Ensure elastic container is running
+  docker_container:
+    name: "{{ elasticsearch_container_name }}"
+    image: "{{ elasticsearch_container_image }}"
+    env: "{{ elasticsearch_container_env | default(omit, True) }}"
+    user: "{{ elasticsearch_container_user | default(omit, True) }}"
+    ports: "{{ elasticsearch_container_ports | default(omit, True) }}"
+    labels: "{{ elasticsearch_container_labels | default(omit, True) }}"
+    volumes: "{{ elasticsearch_container_volumes }}"
+    ulimits: "{{ elasticsearch_container_ulimits }}"
+    networks: "{{ elasticsearch_container_networks | default(omit, True) }}"
+    purge_networks: "{{ elasticsearch_container_purge_networks | default(omit, True) }}"
+    restart_policy: "{{ elasticsearch_container_restart_policy }}"
+    state: started

roles/minio/README.md

@@ -0,0 +1,29 @@
+# `finallycoffee.base.minio` ansible role
+
+## Overview
+
+This role deploys a [min.io](https://min.io) server (s3-compatible object storage server)
+using the official docker container image.
+
+## Configuration
+
+The role requires setting the password for the `root` user (name can be changed by
+setting `minio_root_username`) in `minio_root_password`. That user has full control
+over the minio-server instance.
+
+### Useful config hints
+
+Most configuration is done by setting environment variables in
+`minio_container_extra_env`, for example:
+
+```yaml
+minio_container_extra_env:
+  # disable the "console" web browser UI
+  MINIO_BROWSER: off
+  # enable public prometheus metrics on `/minio/v2/metrics/cluster`
+  MINIO_PROMETHEUS_AUTH_TYPE: public
+```
+
+When serving minio (or any s3-compatible server) on a "subfolder",
+see https://docs.aws.amazon.com/AmazonS3/latest/userguide/RESTRedirect.html
+and https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html

roles/minio/defaults/main.yml

@@ -0,0 +1,40 @@
+---
+
+minio_user: ~
+minio_data_path: /opt/minio
+
+minio_create_user: false
+minio_manage_host_filesystem: false
+
+minio_root_username: root
+minio_root_password: ~
+
+minio_container_name: minio
+minio_container_image_name: docker.io/minio/minio
+minio_container_image_tag: latest
+minio_container_image: "{{ minio_container_image_name }}:{{ minio_container_image_tag }}"
+minio_container_networks: []
+minio_container_ports: []
+
+minio_container_base_volumes:
+  - "{{ minio_data_path }}:{{ minio_container_data_path }}:z"
+minio_container_extra_volumes: []
+
+minio_container_base_env:
+  MINIO_ROOT_USER: "{{ minio_root_username }}"
+  MINIO_ROOT_PASSWORD: "{{ minio_root_password }}"
+minio_container_extra_env: {}
+
+minio_container_labels: {}
+
+minio_container_command:
+  - "server"
+  - "{{ minio_container_data_path }}"
+  - "--console-address \":{{ minio_container_listen_port_console }}\""
+minio_container_restart_policy: "unless-stopped"
+minio_container_image_force_source: "{{ (minio_container_image_tag == 'latest')|bool }}"
+
+minio_container_listen_port_api: 9000
+minio_container_listen_port_console: 8900
+
+minio_container_data_path: /storage

roles/minio/tasks/main.yml

@@ -0,0 +1,37 @@
+---
+
+- name: Ensure minio run user is present
+  user:
+    name: "{{ minio_user }}"
+    state: present
+    system: yes
+  when: minio_create_user
+
+- name: Ensure filesystem mounts ({{ minio_data_path }}) for container volumes are present
+  file:
+    path: "{{ minio_data_path }}"
+    state: directory
+    user: "{{ minio_user|default(omit, True) }}"
+    group: "{{ minio_user|default(omit, True) }}"
+  when: minio_manage_host_filesystem
+
+- name: Ensure container image for minio is present
+  community.docker.docker_image:
+    name: "{{ minio_container_image }}"
+    state: present
+    source: pull
+    force_source: "{{ minio_container_image_force_source }}"
+
+- name: Ensure container {{ minio_container_name }} is running
+  docker_container:
+    name: "{{ minio_container_name }}"
+    image: "{{ minio_container_image }}"
+    volumes: "{{ minio_container_volumes }}"
+    env: "{{ minio_container_env }}"
+    labels: "{{ minio_container_labels }}"
+    networks: "{{ minio_container_networks }}"
+    ports: "{{ minio_container_ports }}"
+    user: "{{ minio_user|default(omit, True) }}"
+    command: "{{ minio_container_command }}"
+    restart_policy: "{{ minio_container_restart_policy }}"
+    state: started

roles/minio/vars/main.yml

@@ -0,0 +1,5 @@
+---
+
+minio_container_volumes: "{{ minio_container_base_volumes + minio_container_extra_volumes }}"
+
+minio_container_env: "{{ minio_container_base_env | combine(minio_container_extra_env) }}"

roles/restic/README.md

@@ -0,0 +1,77 @@
+# `finallycoffee.base.restic`
+
+Ansible role for backup up data using `restic`, utilizing `systemd` timers for scheduling.
+
+## Overview
+
+As restic encrypts the data before storing it, the `restic_repo_password` needs
+to be populated with a strong key, and saved accordingly as only this key can
+be used to decrypt the data for a restore!
+
+### Backends
+
+#### S3 Backend
+
+To use a `s3`-compatible backend like AWS buckets or minio, both `restic_s3_key_id`
+and `restic_s3_access_key` need to be populated, and the `restic_repo_url` has the
+format `s3:https://my.s3.endpoint:port/bucket-name`.
+
+#### SFTP Backend
+
+Using the `sftp` backend requires the configured `restic_user` to be able to
+authenticate to the configured SFTP-Server using password-less methods like
+publickey-authentication. The `restic_repo_url` then follows the format
+`sftp:{user}@{server}:/my-restic-repository` (or without leading `/` for relative
+paths to the `{user}`s home directory.
+
+### Backing up data
+
+A job name like `$service-postgres` or similar needs to be set in `restic_job_name`,
+which is used for naming the `systemd` units, their syslog identifiers etc.
+
+If backing up filesystem locations, the paths need to be specified in
+`restic_backup_paths` as lists of strings representing absolute filesystem
+locations.
+
+If backing up f.ex. database or other data which is generating backups using
+a command like `pg_dump`, use `restic_backup_stdin_command` (which needs to output
+to `stdout`) in conjunction with `restic_backup_stdin_command_filename` to name
+the resulting output (required).
+
+### Policy
+
+The backup policy can be adjusted by overriding the `restic_policy_keep_*`
+variables, with the defaults being:
+
+```yaml
+restic_policy_keep_all_within: 1d
+restic_policy_keep_hourly: 6
+restic_policy_keep_daily: 2
+restic_policy_keep_weekly: 7
+restic_policy_keep_monthly: 4
+restic_policy_backup_frequency: hourly
+```
+
+**Note:** `restic_policy_backup_frequency` must conform to `systemd`s
+`OnCalendar` syntax, which can be checked using `systemd-analyze calender $x`.
+
+## Role behaviour
+
+Per default, when the systemd unit for a job changes, the job is not immediately
+started. This can be overridden using `restic_start_job_on_unit_change: true`,
+which will immediately start the backup job if it's configuration changed.
+
+The systemd unit runs with `restic_user`, which is root by default, guaranteeing
+that filesystem paths are always readable. The `restic_user` can be overridden,
+but care needs to be taken to ensure the user has permission to read all the
+provided filesystem paths / the backup command may be executed by the user.
+
+If ansible should create the user, set `restic_create_user` to `true`, which
+will attempt to create the `restic_user` as a system user.
+
+### Installing
+
+For Debian and RedHat, the role attempts to install restic using the default
+package manager's ansible module (apt/dnf). For other distributions, the generic
+`package` module tries to install `restic_package_name` (default: `restic`),
+which can be overridden if needed.

roles/restic/defaults/main.yml

@@ -0,0 +1,37 @@
+---
+
+restic_repo_url: ~
+restic_repo_password: ~
+restic_s3_key_id: ~
+restic_s3_access_key: ~
+
+restic_backup_paths: []
+restic_backup_stdin_command: ~
+restic_backup_stdin_command_filename: ~
+
+restic_policy_keep_all_within: 1d
+restic_policy_keep_hourly: 6
+restic_policy_keep_daily: 2
+restic_policy_keep_weekly: 7
+restic_policy_keep_monthly: 4
+restic_policy_backup_frequency: hourly
+
+restic_policy:
+  keep_within: "{{ restic_policy_keep_all_within }}"
+  hourly: "{{ restic_policy_keep_hourly }}"
+  daily: "{{ restic_policy_keep_daily }}"
+  weekly: "{{ restic_policy_keep_weekly }}"
+  monthly: "{{ restic_policy_keep_monthly }}"
+  frequency: "{{ restic_policy_backup_frequency }}"
+
+restic_user: root
+restic_create_user: false
+restic_start_job_on_unit_change: false
+
+restic_job_name: ~
+restic_job_description: "Restic backup job for {{ restic_job_name }}"
+restic_systemd_unit_naming_scheme: "restic.{{ restic_job_name }}"
+restic_systemd_working_directory: /tmp
+restic_systemd_syslog_identifier: "restic-{{ restic_job_name }}"
+
+restic_package_name: restic

roles/restic/handlers/main.yml

@@ -0,0 +1,13 @@
+---
+
+- name: Ensure system daemon is reloaded
+  listen: reload-systemd
+  systemd:
+    daemon_reload: true
+
+- name: Ensure systemd service for '{{ restic_job_name }}' is started immediately
+  listen: trigger-restic
+  systemd:
+    name: "{{ restic_systemd_unit_naming_scheme }}.service"
+    state: started
+  when: restic_start_job_on_unit_change

roles/restic/tasks/main.yml

@@ -0,0 +1,77 @@
+---
+
+- name: Ensure {{ restic_user }} system user exists
+  user:
+    name: "{{ restic_user }}"
+    state: present
+    system: true
+  when: restic_create_user
+
+- name: Ensure either backup_paths or backup_stdin_command is populated
+  when: restic_backup_paths|length > 0 and restic_backup_stdin_command
+  fail:
+    msg: "Setting both `restic_backup_paths` and `restic_backup_stdin_command` is not supported"
+
+- name: Ensure a filename for stdin_command backup is given
+  when: restic_backup_stdin_command and not restic_backup_stdin_command_filename
+  fail:
+    msg: "`restic_backup_stdin_command` was set but no filename for the resulting output was supplied in `restic_backup_stdin_command_filename`"
+
+- name: Ensure backup frequency adheres to systemd's OnCalender syntax
+  command:
+    cmd: "systemd-analyze calendar {{ restic_policy.frequency }}"
+  register: systemd_calender_parse_res
+  failed_when: systemd_calender_parse_res.rc != 0
+  changed_when: false
+
+- name: Ensure restic is installed
+  block:
+    - name: Ensure restic is installed via apt
+      apt:
+        package: restic
+        state: latest
+      when: ansible_os_family == 'Debian'
+    - name: Ensure restic is installed via dnf
+      dnf:
+        name: restic
+        state: latest
+      when: ansible_os_family == 'RedHat'
+    - name: Ensure restic is installed using the auto-detected package-manager
+      package:
+        name: "{{ restic_package_name }}"
+        state: present
+      when: ansible_os_family not in ['RedHat', 'Debian']
+
+- name: Ensure systemd service file for '{{ restic_job_name }}' is templated
+  template:
+    dest: "/etc/systemd/system/{{ restic_systemd_unit_naming_scheme }}.service"
+    src: restic.service.j2
+    owner: root
+    group: root
+    mode: 0640
+  notify:
+    - reload-systemd
+    - trigger-restic
+
+- name: Ensure systemd service file for '{{ restic_job_name }}' is templated
+  template:
+    dest: "/etc/systemd/system/{{ restic_systemd_unit_naming_scheme }}.timer"
+    src: restic.timer.j2
+    owner: root
+    group: root
+    mode: 0640
+  notify:
+    - reload-systemd
+
+- name: Flush handlers to ensure systemd knows about '{{ restic_job_name }}'
+  meta: flush_handlers
+
+- name: Ensure systemd timer for '{{ restic_job_name }}' is activated
+  systemd:
+    name: "{{ restic_systemd_unit_naming_scheme }}.timer"
+    enabled: true
+
+- name: Ensure systemd timer for '{{ restic_job_name }}' is started
+  systemd:
+    name: "{{ restic_systemd_unit_naming_scheme }}.timer"
+    state: started

roles/restic/templates/restic.service.j2

@@ -0,0 +1,28 @@
+[Unit]
+Description={{ restic_job_description }}
+
+[Service]
+Type=oneshot
+User={{ restic_user }}
+WorkingDirectory={{ restic_systemd_working_directory }}
+SyslogIdentifier={{ restic_systemd_syslog_identifier }}
+
+Environment=RESTIC_REPOSITORY={{ restic_repo_url }}
+Environment=RESTIC_PASSWORD={{ restic_repo_password }}
+{% if restic_s3_key_id and restic_s3_access_key %}
+Environment=AWS_ACCESS_KEY_ID={{ restic_s3_key_id }}
+Environment=AWS_SECRET_ACCESS_KEY={{ restic_s3_access_key }}
+{% endif %}
+
+ExecStartPre=-/bin/sh -c '/usr/bin/restic snapshots || /usr/bin/restic init'
+{% if restic_backup_stdin_command %}
+ExecStart=/bin/sh -c '{{ restic_backup_stdin_command }} | /usr/bin/restic backup --verbose --stdin --stdin-filename {{ restic_backup_stdin_command_filename }}'
+{% else %}
+ExecStart=/usr/bin/restic --verbose backup {{ restic_backup_paths | join(' ') }}
+{% endif %}
+ExecStartPost=/usr/bin/restic forget --prune --keep-within={{ restic_policy.keep_within }} --keep-hourly={{ restic_policy.hourly }} --keep-daily={{ restic_policy.daily }} --keep-weekly={{ restic_policy.weekly }} --keep-monthly={{ restic_policy.monthly }}
+ExecStartPost=-/usr/bin/restic snapshots
+ExecStartPost=/usr/bin/restic check
+
+[Install]
+WantedBy=multi-user.target

roles/restic/templates/restic.timer.j2

@@ -0,0 +1,10 @@
+[Unit]
+Description=Run {{ restic_job_name }}
+
+[Timer]
+OnCalendar={{ restic_policy.frequency }}
+Persistent=True
+Unit={{ restic_systemd_unit_naming_scheme }}.service
+
+[Install]
+WantedBy=timers.target