meta: bump collection version to 0.1.1

README.md

@@ -11,10 +11,20 @@ metrics or alerting.
   alertmanager for receiving alerts from prometheus and routing them
   to the correct configured receivers.
 
+- [`grafana`](roles/grafana/README.md): a popular visualization and
+  dashboard creation tool able to use various datasources.
+
 - [`matrix-alertmanager`](roles/matrix-alertmanager/README.md): An alert-
   manager receiver which posts alerts to a configured matrix channel
   using alertmanagers' webhooks.
 
+- [`vmagent`](roles/vmagent/README.md): VictoriaMetrics agent
+
+- [`vmtsdb`](roles/vmtsdb/README.md): VictoriaMetrics time series database.
+
+- [`vmalert`](roles/vmalert/README.md): VictoriaMetrics alerting and
+  ruling engine.
+
 - [`postgres_exporter`](roles/postgres_exporter/README.md): Prometheus
   exporter for postgres databases, in a docker container.
 

galaxy.yml

@@ -1,14 +1,20 @@
 namespace: finallycoffee
 name: observability
-version: 0.0.1
+version: 0.1.1
 readme: README.md
 authors:
 - transcaffeine <transcaffeine@finally.coffee>
 description: Various ansible roles useful for automating infrastructure
 dependencies:
-  "community.docker": "^1.10.0"
+  "community.docker": "^3.0.0"
 license_file: LICENSE.md
 build_ignore:
 - '*.tar.gz'
 repository: https://git.finally.coffee/finallycoffee/observability
-issues: https://git.finally.coffee/finallycoffee/observability/issues
+issues: https://codeberg.org/finallycoffee/ansible-collection-observability/issues
+tags:
+  - observability
+  - monitoring
+  - prometheus
+  - victoriametrics
+  - grafana

meta/runtime.yml

@@ -1,3 +1,2 @@
 ---
-
+requires_ansible: ">=2.15"
-requires_ansible: ">=2.12"

playbooks/alertmanager.yml

@@ -0,0 +1,6 @@
+---
+- name: Ensure alertmanager is configured and running
+  hosts: "{{ alertmanager_hosts | default('alertmanager') }}"
+  become: "{{ alertmanager_become | default(false, false) }}"
+  roles:
+    - role: finallycoffee.observability.alertmanager

playbooks/grafana.yml

@@ -0,0 +1,6 @@
+---
+- name: Install and configure grafana
+  hosts: "{{ grafana_hosts | default('grafana') }}"
+  become: "{{ grafana_become | default(false, true) }}"
+  roles:
+    - role: finallycoffee.observability.grafana

playbooks/vmagent.yml

@@ -0,0 +1,6 @@
+---
+- name: Install and configure vmagent
+  hosts: "{{ vmagent_hosts | default('vmagent') }}"
+  become: "{{ vmagent_become | default(false) }}"
+  roles:
+    - role: finallycoffee.observability.vmagent

playbooks/vmalert.yml

@@ -0,0 +1,6 @@
+---
+- name: Install vmalert using docker
+  hosts: "{{ vmalert_hosts | default('vmalert') }}"
+  become: "{{ vmalert_become | default(false) }}"
+  roles:
+    - role: finallycoffee.observability.vmalert

playbooks/vmtsdb.yml

@@ -0,0 +1,6 @@
+---
+- name: Install vmtsdb using docker
+  hosts: "{{ vmtsdb_hosts | default('vmtsdb') }}"
+  become: "{{ vmtsdb_become | default(false) }}"
+  roles:
+    - role: finallycoffee.observability.vmtsdb

roles/alertmanager/defaults/main.yml

@@ -1,7 +1,6 @@
 ---
-
 alertmanager_user: alertmanager
-alertmanager_version: 0.25.0
+alertmanager_version: 0.27.0
 alertmanager_base_path: /opt/alertmanager
 alertmanager_config_path: "{{ alertmanager_base_path }}/config"
 alertmanager_config_file: "{{ alertmanager_config_path }}/alertmanager.yml"

roles/cadvisor/defaults/main.yml

@@ -1,6 +1,5 @@
 ---
-
+cadvisor_version: 0.49.1
-cadvisor_version: 0.45.0
 
 cadvisor_container_name: cadvisor
 cadvisor_container_image_name: gcr.io/cadvisor/cadvisor

roles/grafana/README.md

@@ -0,0 +1,13 @@
+# `finallycoffee.observability.grafana` ansible role
+
+Ansible role to install and configure grafana, currently only supports docker. For docker, the python library `docker` must be installed on the target host.
+
+## Usage
+
+Ensure the following variables are populated:
+- `grafana_config_security_secret_key`
+- `grafana_config_security_admin_password`
+
+### Authentication via OAuth2
+
+Set `grafna_config_auth_generic_oauth_enabled` to `true` and populate variables according to the grafana docs, all generic oauth configuration values are available prefixed with `grafana_config_auth_generic_oauth_`.

roles/grafana/defaults/main.yml

@@ -0,0 +1,235 @@
+---
+grafana_user: grafana
+grafana_version: "10.1.1"
+grafana_base_path: "/opt/grafana"
+grafana_config_path: "{{ grafana_base_path }}/config"
+grafana_config_file: "{{ grafana_config_path }}/grafana.ini"
+grafana_ldap_config_file: "{{ grafana_config_path }}/ldap.toml"
+grafana_provisioning_path: "{{ grafana_config_path }}/provisioning"
+grafana_notifier_provisioning_path: "{{ grafana_provisioning_path }}/notifiers"
+grafana_dashboard_provisioning_path: "{{ grafana_provisioning_path }}/dashboards"
+grafana_datasource_provisioning_path: "{{ grafana_provisioning_path }}/datasources"
+grafana_plugin_provisioning_path: "{{ grafana_provisioning_path }}/plugins"
+grafana_data_path: "{{ grafana_base_path }}/data"
+grafana_logs_path: "{{ grafana_base_path }}/logs"
+grafana_state: present
+
+grafana_run_user: >-
+  {{ ('uid' in (grafana_user_info | default([]))) | ternary(grafana_user_info.uid, grafana_user) }}
+grafana_run_group: >-
+  {{ ('group' in (grafana_user_info | default([]))) | ternary(grafana_user_info.group, grafana_user) }}
+
+grafana_container_image_server: "docker.io"
+grafana_container_image_namespace: "grafana"
+grafana_container_image_container: "grafana"
+grafana_container_image_name: >-
+  {{
+    [
+      ((grafana_container_image_server is defined)
+        | ternary([ grafana_container_image_server ], [])),
+      ((grafana_container_image_namespace is defined)
+        | ternary([ grafana_container_image_namespace], [])),
+      grafana_container_image_container,
+    ] | ansible.builtin.flatten | join('/')
+  }}
+grafana_container_image: >-
+  {{ grafana_container_image_name }}:{{ grafana_container_image_tag | default(grafana_version, true) }}
+
+grafana_container_name: grafana
+grafana_container_base_volumes:
+  - "{{ grafana_config_path }}:{{ grafana_container_config_path }}:ro"
+  - "{{ grafana_data_path }}:{{ grafana_container_data_path }}:rw"
+  - "{{ grafana_logs_path }}:{{ grafana_container_logs_path }}:rw"
+grafana_container_volumes: []
+grafana_container_collected_volumes: >-
+  {{ grafana_container_base_volumes + grafana_container_volumes }}
+grafana_container_restart_policy: "unless-stopped"
+
+grafana_config_log_mode:
+  - console
+  - file
+grafana_config_auth_generic_oauth_scopes:
+  - openid
+  - profile
+  - email
+  - roles
+  - offline_access
+
+grafana_config_auth_generic_oauth_config:
+  enabled: "{{ grafana_config_auth_generic_oauth_enabled }}"
+  name: "{{ grafana_config_auth_generic_oauth_name }}"
+  client_id: "{{ grafana_config_auth_generic_oauth_client_id }}"
+  client_secret: "{{ grafana_config_auth_generic_oauth_client_secret }}"
+  scopes: "{{ grafana_config_auth_generic_oauth_scopes | join(' ') }}"
+  email_attribute_name: "{{ grafana_config_auth_generic_oauth_email_attribute_name | default('email') }}"
+  email_attribute_path: "{{ grafana_config_auth_generic_oauth_email_attribute_name | default('email') }}"
+  login_attribute_path: "{{ grafana_config_auth_generic_oauth_login_attribute_name | default('preferred_username') }}"
+  name_attribute_path: "{{ grafana_config_auth_generic_oauth_name_attribute_name | default('name') }}"
+  api_url: "{{ grafana_config_auth_generic_oauth_api_url }}"
+  auth_url: "{{ grafana_config_auth_generic_oauth_auth_url }}"
+  token_url: "{{ grafana_config_auth_generic_oauth_token_url }}"
+  role_attribute_path: "{{ grafana_config_auth_generic_oauth_role_attribute_path | default('') }}"
+
+grafana_default_config:
+  DEFAULT:
+    app_mode: "{{ grafana_config_app_mode | default('production') }}"
+    instance_name: "{{ grafana_config_instance_name | default('${HOSTNAME}') }}"
+  paths:
+    data: "{{ grafana_config_paths_data | default('/var/lib/grafana') }}"
+    temp_data_lifetime: "{{ grafana_config_paths_temp_data_lifetime | default('24h') }}"
+    logs: "{{ grafana_config_paths_logs | default('/var/log/grafana') }}"
+    plugins: "{{ grafana_config_paths_plugins | default('/var/lib/grafana/plugins') }}"
+    provisioning: "{{ grafana_config_paths_provisioning | default('conf/provisioning') }}"
+  server:
+    protocol: "{{ grafana_config_server_protocol | default('http') }}"
+    http_addr: "{{ grafana_config_server_http_addr | default('\"\"') }}"
+    http_port: "{{ grafana_config_server_http_port | default(3000) }}"
+    domain: "{{ grafana_config_server_domain }}"
+    enforce_domain: "{{ grafana_config_server_enforce_domain | default(true) }}"
+    root_url: "{{ grafana_config_server_root_url | default('%(protocol)s://%(domain)s:%(http_port)s/') }}"
+    serve_from_subpath: "{{ grafana_config_server_serve_from_subpath | default(false) }}"
+    router_logging: "{{ grafana_config_server_router_logging | default(false) }}"
+    static_root_path: "{{ grafana_config_server_static_root_path | default('public') }}"
+    enable_gzip: "{{ grafana_config_server_enable_gzip | default(false) }}"
+    cert_file: "{{ grafana_config_server_cert_file | default('\"\"') }}"
+    cert_key: "{{ grafana_config_server_cert_key | default('\"\"') }}"
+    socket: "{{ grafana_config_server_socket | default('\"\"') }}"
+  database:
+    type: "{{ grafana_config_database_type | default('sqlite3') }}"
+    host: "{{ grafana_config_database_host | default('127.0.0.1:3306') }}"
+    name: "{{ grafana_config_database_name | default('grafana') }}"
+    user: "{{ grafana_config_database_user | default('root') }}"
+    password: "{{ grafana_config_database_password | default('') }}"
+    url: "{{ grafana_config_database_url | default('') }}"
+    ssl_mode: "{{ grafana_config_database_ssl_mode | default('disable') }}"
+    ca_cert_path: "{{ grafana_config_database_ca_cert_path | default('') }}"
+    client_key_path: "{{ grafana_config_database_client_key_path | default('') }}"
+    client_cert_path: "{{ grafana_config_database_client_cert_path | default('') }}"
+    server_cert_name: "{{ grafana_config_database_srver_cert_name | default('') }}"
+    path: "{{ grafana_config_database_path | default('grafana.db') }}"
+    max_idle_conn: "{{ grafana_config_database_max_idle_conn | default(2) }}"
+    max_open_conn: "{{ grafana_config_database_max_open_conn | default(0) }}"
+    conn_max_lifetime: "{{ grafana_config_database_conn_max_lifetime | default(14400) }}"
+    log_queries: "{{ grafana_config_database_log_queries | default(false) }}"
+    cache_mode: "{{ grafana_config_database_cache_mode | default('private') }}"
+  remote_cache:
+    type: "{{ grafana_config_config_remote_cache_type | default('database') }}"
+    connstr: "{{ grafana_config_remote_cache_connstr | default('') }}"
+  dataproxy:
+    logging: "{{ grafana_config_dataproxy_logging | default(false) }}"
+    timeout: "{{ grafana_config_dataproxy_timeout | default(30) }}"
+    send_user_header: "{{ grafana_config_dataproxy_send_header | default(false) }}"
+  analytics:
+    reporting_enabled: "{{ grafana_config_analytics_reporting_enabled | default(true) }}"
+    check_for_updates: "{{ grafana_config_analytics_check_for_updates | default(true) }}"
+    google_analytics_ua_id: "{{ grafana_config_analytics_google_analytics_ua_id | default('') }}"
+    google_tag_manager_id: "{{ grafana_config_analytics_google_tag_manager_id | default('') }}"
+  security:
+    disable_initial_admin_create: "{{ grafana_config_security_disable_initial_admin_creation | default(false) }}"
+    admin_user: "{{ grafana_config_security_admin_user | default('admin') }}"
+    admin_password: "{{ grafana_config_security_admin_password }}"
+    secret_key: "{{ grafana_config_security_secret_key }}"
+    disable_gravatar: "{{ grafana_config_security_disable_gravatar | default(true) }}"
+    data_source_proxy_whitelist: "{{ grafana_config_security_data_source_proxy_whitelist | default([]) | join(' ') }}"
+    disable_brute_force_login_protection: "{{ grafana_config_security_disable_brute_force_login_protection | default(false) }}"
+    cookie_secure: "{{ grafana_config_security_cookie_secure | default(false) }}"
+    cookie_samesite: "{{ grafana_config_security_cookie_samesite | default('lax') }}"
+    allow_embedding: "{{ grafana_config_security_allow_embedding | default(false) }}"
+    strict_transport_security: "{{ grafana_config_security_strict_transport_security | default(false) }}"
+    strict_transport_security_max_age_seconds: "{{ grafana_config_security_strict_transport_security_max_age_seconds | default(86400) }}"
+    strict_transport_security_preload: "{{ grafana_config_security_strict_transport_security_preload | default(false) }}"
+    strict_transport_security_subdomains: "{{ grafana_config_security_strict_transport_security_subdomains | default(false) }}"
+    x_content_type_options: "{{ grafana_config_security_x_content_type_options | default(false) }}"
+    x_xss_protection: "{{ grafana_config_security_x_xss_protection | default(true) }}"
+  snapshots:
+    external_enabled: "{{ grafana_config_snapshots_external_enabled | default(false) }}"
+    external_snapshot_url: "{{ grafana_config_snapshots_external | default('') }}"
+    external_snapshot_name: "{{ grafana_config_snapshots_external | default('') }}"
+    public_mode_: "{{ grafana_config_snapshots_public_mode | default(false) }}"
+    snapshot_remove_expired: "{{ grafana_config_snapshots_snapshot_remove_expired | default(true) }}"
+  dashboards:
+    versions_to_keep: "{{ grafana_config_dashboards_versions_to_keep | default(20) }}"
+  users:
+    allow_sign_up: "{{ grafana_config_users_allow_sign_up | default(true) }}"
+    allow_org_create: "{{ grafana_config_users_allow_org_create | default(false) }}"
+    auto_assign_org: "{{ grafana_config_users_auto_assign_org | default(true) }}"
+    auto_assign_org_id: "{{ grafana_config_users_auto_assign_org_id | default(1) }}"
+    auto_assign_org_role: "{{ grafana_config_users_auto_assign_org_role | default('Viewer') }}"
+    verify_email_enabled: "{{ grafana_config_users_verify_email_enabled | default(false) }}"
+    login_hint: "{{ grafana_config_users_login_hint | default('email or username') }}"
+
+    password_hint: "{{ grafana_config_users_password_hint | default('password') }}"
+    viewers_can_edit: "{{ grafana_config_users_viewers_can_edit | default(true) }}"
+    editors_can_admin: "{{ grafana_config_users_editors_can_admin | default(false) }}"
+  auth:
+    login_cookie_name: "{{ grafana_config_auth_login_cookie_name | default('grafana_session') }}"
+    login_maximum_inactive_lifetime_days: "{{ grafana_config_auth_login_maximum_inactive_lifetime_days | default(7) }}"
+    login_maximum_lifetime_days: "{{ grafana_config_auth_login_maximum_lifetime_days | default(30) }}"
+    token_rotation_interval_minutes: "{{ grafana_config_auth_token_rotation_interval_minutes | default(10) }}"
+    disable_login_form: "{{ grafana_config_auth_disable_login_form | default(false) }}"
+    disable_signout_menu: "{{ grafana_config_auth_disable_signout_menu | default(false) }}"
+    signout_redirect_url: "{{ grafana_config_auth_signout_redirect_url | default('') }}"
+    api_key_max_seconds_to_live: "{{ grafana_config_api_key_max_seconds_to_live | default(-1) }}"
+    oauth_auto_login: "{{ grafana_config_auth_oauth_auto_login | default(false) }}"
+    oauth_allow_insecure_email_lookup: "{{ grafana_config_oauth_allow_insecure_email_lookup | default(false) }}"
+  smtp:
+    enabled: "{{ grafana_config_smtp_enabled | default(false) }}"
+    host: "{{ grafana_config_smtp_host | default('localhost:25') }}"
+    user: "{{ grafana_config_smtp_user | default('') }}"
+    password: "{{ grafana_config_smtp_password | default('') }}"
+    cert_file: "{{ grafana_config_smtp_cert_file | default('') }}"
+    key_file: "{{ grafana_config_smtp_key_file | default('') }}"
+    skip_verify: "{{ grafana_config_smtp_skip_verify | default('') }}"
+    ehlo_identity: "{{ grafana_config_smtp_ehlo_identity | default('') }}"
+    from_address: "{{ grafana_config_smtp_from_address | default('admin@grafana.localhost') }}"
+    from_name: "{{ grafana_config_smtp_from_name | default('Grafana') }}"
+  emails:
+    welcome_email_on_sign_up: "{{ grafana_config_emails_welcome_email_on_sign_up | default(false) }}"
+  log:
+    mode: "{{ grafana_config_log_mode | join(' ') }}"
+    level: "{{ grafana_config_log_level | default('info') }}"
+    filters: "{{ grafana_config_log_filters | default('') }}"
+  "log.syslog":
+    format: "{{ grafana_config_log_syslog_format | default('text') }}"
+  quota:
+    enabled: "{{ grafana_config_quota_enabled | default(false) }}"
+  explore:
+    enabled: "{{ grafana_config_explore_enabled | default(true) }}"
+  metrics:
+    enabled: "{{ grafana_config_metrics_enabled | default(false) }}"
+  grafana_com:
+    url: "{{ grafana_config_grafana_com_url | default('https://grafana.com') }}"
+
+grafana_merged_config: >-
+  {{
+    grafana_default_config
+    | combine({"auth.generic_oauth": grafana_config_auth_generic_oauth_config}
+      if grafana_config_auth_generic_oauth_enabled else {}, recursive=true)
+    | combine(grafana_config | default({}), recursive=true) }}
+
+grafana_ldap_config:
+  log:
+    filters: "{{ grafana_ldap_config_log_filters | default('ldap:trace') }}"
+  servers: "{{ grafana_ldap_config_default_servers }}"
+grafana_ldap_config_default_servers:
+  - host: "{{ grafana_ldap_config_servers_host }}"
+    port: "{{ grafana_ldap_config_servers_port }}"
+    use_ssl: "{{ grafana_ldap_config_servers_use_ssl | bool }}"
+    start_ssl: "{{ grafana_ldap_config_servers_start_ssl | bool }}"
+    ssl_skip_verify: "{{ grafana_ldap_config_servers_ssl_skip_verify | bool }}"
+    bind_dn: "{{ grafana_ldap_config_servers_bind_dn }}"
+    bind_passwort: "{{ grafana_ldap_config_servers_bind_passwort }}"
+    search_filter: "{{ grafana_ldap_config_servers_search_filter }}"
+    search_base_dns: "{{ grafana_ldap_config_servers_search_base_dns | to_json }}"
+    attributes:
+      name: "{{ grafana_ldap_config_servers_attributes_name | default('givenName') }}"
+      surname: "{{ grafana_ldap_config_servers_attributes_name | default('sn') }}"
+      username: "{{ grafana_ldap_config_servers_attributes_name | default('uid') }}"
+      member_of: "{{ grafana_ldap_config_servers_attributes_member_of | default('memberOf') }}"
+      email: "{{ grafana_ldap_config_servers_attributes_email | default('mail') }}"
+    group_mappings: "{{ grafana_ldap_config_default_group_mappings }}"
+grafana_ldap_config_default_group_mappings:
+  - group_dn: "{{ grafana_ldap_config_servers_group_mappings_group_dn }}"
+    org_role: "{{ grafana_ldap_config_servers_group_mappings_org_role }}"
+    org_id: "{{ grafana_ldap_config_servers_group_mappings_org_id }}"
+    grafana_admin: "{{ grafana_ldap_config_servers_group_mappings_grafana_admin }}"

roles/grafana/handlers/main.yml

@@ -0,0 +1,8 @@
+---
+- name: Ensure grafana is restarted
+  community.general.docker_container:
+    name: "{{ grafana_container_name }}"
+    state: "started"
+    restart: true
+  when: "grafana_state == 'present'"
+  listen: grafana-restart

roles/grafana/tasks/main.yml

@@ -0,0 +1,86 @@
+---
+- name: Ensure grafana_state is valid
+  ansible.builtin.assert:
+    that:
+      - "grafana_state in ['present', 'absent']"
+    fail_msg: >-
+      Only 'present' and 'absent' are allowed as values for grafana_state
+
+- name: Ensure user '{{ grafana_user }}' is {{ grafana_state }}
+  ansible.builtin.user:
+    name: "{{ grafana_user }}"
+    state: "{{ grafana_state }}"
+    system: true
+    create_home: false
+    groups: "{{ grafana_user_groups | default(omit) }}"
+    append: "{{ grafana_user_groups_append | default(omit) }}"
+  register: grafana_user_info
+
+- name: Ensure paths are {{ grafana_state }}
+  ansible.builtin.file:
+    path: "{{ item.path }}"
+    state: >-
+      {{ (grafana_state == 'present') | ternary('directory', 'absent') }}
+    owner: "{{ grafana_run_user }}"
+    group: "{{ grafana_run_group }}"
+    mode: "{{ item.mode | default('0755') }}"
+  loop:
+    - path: "{{ grafana_base_path }}"
+    - path: "{{ grafana_config_path }}"
+    - path: "{{ grafana_provisioning_path }}"
+    - path: "{{ grafana_notifier_provisioning_path }}"
+    - path: "{{ grafana_dashboard_provisioning_path }}"
+    - path: "{{ grafana_datasource_provisioning_path }}"
+    - path: "{{ grafana_plugin_provisioning_path }}"
+    - path: "{{ grafana_data_path }}"
+    - path: "{{ grafana_logs_path }}"
+
+- name: Ensure configuration file '{{ grafana_config_file }}' is templated
+  ansible.builtin.copy:
+    dest: "{{ grafana_config_file }}"
+    content: "{{ grafana_merged_config | community.general.to_ini }}"
+    owner: "{{ grafana_run_user }}"
+    group: "{{ grafana_run_group }}"
+    mode: "0640"
+  when: "grafana_state == 'present'"
+  tags:
+    - grafana-update-config
+  notify: grafana-restart
+
+- name: Ensure ldap configuration file '{{ grafana_ldap_config_file }}' is templated if required
+  ansible.builtin.copy:
+    dest: "{{ grafana_ldap_config_file }}"
+    content: "{{ grafana_ldap_config | ansible.builtin.to_toml }}"
+    owner: "{{ grafana_run_user }}"
+    group: "{{ grafana_run_group }}"
+    mode: "0640"
+  when:
+    - "grafana_state == 'present'"
+    - "grafana_config_auth_ldap_enabled | default(false) | bool"
+  tags:
+    - grafana-update-config
+  notify: grafana-restart
+
+- name: Ensure grafana container image '{{ grafana_container_image }}' is {{ grafana_state }}
+  community.docker.docker_image:
+    name: "{{ grafana_container_image }}"
+    state: "{{ grafana_state }}"
+    source: >-
+      {{ (grafana_state == 'present') | ternary('pull', omit) }}
+    force_source: >-
+      {{ (grafana_state == 'present') | ternary((grafana_container_image_tag is defined), omit) }}
+
+- name: Ensure grafana container '{{ grafana_container_name }}' is {{ grafana_state }}
+  community.docker.docker_container:
+    name: "{{ grafana_container_name }}"
+    env: "{{ grafana_container_env | default(omit) }}"
+    user: "{{ grafana_run_user }}"
+    ports: "{{ grafana_container_ports | default(omit) }}"
+    groups: "{{ grafana_run_group }}"
+    labels: "{{ grafana_container_labels | default(omit) }}"
+    volumes: "{{ grafana_container_collected_volumes }}"
+    networks: "{{ grafana_container_networks | default(omit, true) }}"
+    restart_policy: "{{ grafana_container_restart_policy }}"
+    state: "{{ (grafana_state == 'present') | ternary('started', 'absent') }}"
+    comparisons:
+      '*': strict

roles/grafana/vars/main.yml

@@ -0,0 +1,4 @@
+---
+grafana_container_config_path: "/etc/grafana"
+grafana_container_data_path: "/var/lib/grafana"
+grafana_container_logs_path: "/var/log/grafana"

roles/vmagent/README.md

@@ -0,0 +1,27 @@
+# `finallycoffee.observability.vmagent` ansible role
+
+Install and configure the
+[victoriametrics agent `vmagent`](https://docs.victoriametrics.com/vmagent/)
+using the [supported deployment types (see `vars/main.yml#L5`)](vars/main.yml#L5).
+
+## Configuration
+
+Set scrape job configuration as complex data in `vmagent_config_scrape_configs`.
+To tune the scrape interval, override `vmagent_config_global_scrape_interval`,
+or modify / extend `vmagent_config` directly.
+
+### Prometheus remote write api with basic auth
+
+One of the more common methods of sending the collected metrics to a
+central prometheus server. Set the following variables to archieve this:
+
+```yaml
+vmagent_flags:
+  remoteWrite_url: https://my.prometheus.instance.example.com/api/v1/write
+  remoteWrite_basicAuth_username: my_prom_user
+  remoteWrite_basicAuth_passwordFile: /path/to/password/file.key
+```
+
+For the full set of options, see either the
+[vmagents' "Advanced usage" documentation](https://docs.victoriametrics.com/vmagent/#advanced-usage)
+or run `vmagent -help` for the same output.

roles/vmagent/defaults/main/config.yml

@@ -0,0 +1,17 @@
+---
+vmagent_config_global_scrape_interval: "30s"
+vmagent_config_global_scrape_timeout: "10s"
+vmagent_config_global_external_labels: []
+vmagent_config_scrape_configs: []
+
+vmagent_config: ~
+vmagent_base_config:
+  global:
+    scrape_interval: "{{ vmagent_config_global_scrape_interval }}"
+    scrape_timeout: "{{ vmagent_config_global_scrape_timeout }}"
+    external_labels: "{{ vmagent_config_global_external_labels }}"
+  scrape_configs: "{{ vmagent_config_scrape_configs }}"
+
+vmagent_merged_config: >-2
+  {{ (vmagent_base_config | default({}, true))
+    | combine(vmagent_config | default({}, true), recursive=True) }}

roles/vmagent/defaults/main/container.yml

@@ -0,0 +1,54 @@
+---
+vmagent_container_image_registry: "docker.io"
+vmagent_container_image_namespace: "victoriametrics"
+vmagent_container_image_name: "vmagent"
+vmagent_container_image_tag: ~
+vmagent_container_image: >-2
+  {{
+    ([
+      vmagent_container_image_registry,
+      vmagent_container_image_namespace,
+      vmagent_container_image_name,
+    ] | join('/'))
+    + ':'
+    + (vmagent_container_image_tag
+      | default('v' + vmagent_version, true))
+  }}
+vmagent_container_image_source: pull
+vmagent_container_image_force_source: >-2
+  {{ vmagent_container_image_tag | default(false, true) | bool }}
+vmagent_container_image_network_retries: 3
+vmagent_container_image_network_delay: 5
+
+vmagent_container_name: vmagent
+vmagent_container_user: ~
+vmagent_container_ports: ~
+vmagent_container_labels: ~
+vmagent_container_command: >-2
+  {% for flag in vmagent_all_flags -%}
+  -{{ flag }}
+  {% endfor -%}
+vmagent_container_networks: ~
+vmagent_container_etc_hosts: ~
+vmagent_container_dns_servers: ~
+vmagent_container_restart_policy: >-2
+  {{ (vmagent_deployment_type == 'docker')
+    | ternary('unless-stopped', 'on-failure') }}
+vmagent_container_state: >-2
+  {{ (vmagent_state == 'present') | ternary('started', 'absent') }}
+
+vmagent_container_base_volumes:
+  - "{{ vmagent_scrape_config_file }}:{{ vmagent_scrape_config_file }}:ro"
+  - "{{ vmagent_cache_path }}:{{ vmagent_cache_path }}:z"
+vmagent_container_volumes: ~
+vmagent_container_all_volumes: >-2
+  {{ (vmagent_container_base_volumes | default([], true))
+    + (vmagent_container_volumes | default([], true)) }}
+
+vmagent_container_base_env:
+  remoteWrite_tmpDataPath: "{{ vmagent_cache_path }}"
+  promscrape_config: "{{ vmagent_scrape_config_file }}"
+vmagent_container_env: ~
+vmagent_container_merged_env: >-2
+  {{ (vmagent_container_base_env | default({}, true))
+    | combine(vmagent_container_env | default({})) }}

roles/vmagent/defaults/main/main.yml

@@ -0,0 +1,16 @@
+---
+vmagent_user: vmagent
+vmagent_version: "1.104.0"
+vmagent_state: present
+vmagent_deployment_method: "docker"
+
+vmagent_scrape_config_file: "/etc/vmagent/scrape_config.yml"
+vmagent_config_path: "{{ vmagent_scrape_config_file | dirname }}"
+vmagent_cache_path: "/var/cache/vmagent"
+
+vmagent_base_flags:
+  - "enableTCP6"
+  - "envflag.enable"
+vmagent_flags: ~
+vmagent_all_flags: >-2
+  {{ vmagent_base_flags + (vmagent_flags | default([], true)) }}

roles/vmagent/defaults/main/user.yml

@@ -0,0 +1,7 @@
+---
+vmagent_user_groups: ~
+
+vmagent_run_user_id: >-2
+  {{ vmagent_user_info.uid | default(vmagent_user) }}
+vmagent_run_group_id: >-2
+  {{ vmagent_user_info.group | default(vmagent_user) }}

roles/vmagent/handlers/main.yml

@@ -0,0 +1,9 @@
+---
+- name: Ensure vmagent container '{{ vmagent_container_name }}' is restarted
+  community.docker.docker_container:
+    name: "{{ vmagent_container_name }}"
+    state: "{{ vmagent_container_state }}"
+    restart: true
+  listen: "vmagent-reload"
+  ignore_errors: "{{ ansible_check_mode }}"
+  when: vmagent_deployment_method == 'docker'

roles/vmagent/meta/main.yml

@@ -0,0 +1,10 @@
+---
+allow_duplicates: true
+dependencies: []
+galaxy_info:
+  role_name: vmagent
+  description: Deploy and configure the victoriametrics agent `vmagent`
+  galaxy_tags:
+    - victoriametrics
+    - vmagent
+    - prometheus

roles/vmagent/tasks/deploy-docker.yml

@@ -0,0 +1,27 @@
+---
+- name: Ensure container image '{{ vmagent_container_image }}' is {{ vmagent_state }}
+  community.docker.docker_image:
+    name: "{{ vmagent_container_image }}"
+    state: "{{ vmagent_state }}"
+    source: "{{ vmagent_container_image_source }}"
+    force_source: "{{ vmagent_container_image_force_source }}"
+  register: vmagent_container_image_info
+  until: vmagent_container_image_info is success
+  retries: "{{ vmagent_container_image_network_retries }}"
+  delay: "{{ vmagent_container_image_network_delay }}"
+
+- name: Ensure container '{{ vmagent_container_name }}' is {{ vmagent_container_state }}
+  community.docker.docker_container:
+    name: "{{ vmagent_container_name }}"
+    image: "{{ vmagent_container_image }}"
+    env: "{{ vmagent_container_merged_env }}"
+    user: "{{ vmagent_container_user }}"
+    ports: "{{ vmagent_container_ports | default(omit, true) }}"
+    labels: "{{ vmagent_container_labels | default(omit, true) }}"
+    command: "{{ vmagent_container_command }}"
+    volumes: "{{ vmagent_container_all_volumes }}"
+    networks: "{{ vmagent_container_networks | default(omit, true) }}"
+    etc_hosts: "{{ vmagent_container_etc_hosts | default(omit, true) }}"
+    dns_servers: "{{ vmagent_container_dns_servers | default(omit, true) }}"
+    restart_policy: "{{ vmagent_container_restart_policy | default(omit, true) }}"
+    state: "{{ vmagent_container_state }}"

roles/vmagent/tasks/main.yml

@@ -0,0 +1,54 @@
+---
+- name: Check that `vmagent_state` is valid
+  ansible.builtin.fail:
+    msg: >-2
+      Unsupported state '{{ vmagent_state }}'! Supported states
+      are {{ vmagent_states | join(', ') }}.
+  when: vmagent_state not in vmagent_states
+
+- name: Check that `vmagent_deployment_method` is valid
+  ansible.builtin.fail:
+    msg: >-2
+      Unsupported deployment method '{{ vmagent_deployment_method }}'!
+      Supported are: {{ vmagent_deployment_methods | join(', ') }}.
+  when: vmagent_deployment_method not in vmagent_deployment_methods
+
+- name: Ensure vmagent user '{{ vmagent_user }}' is {{ vmagent_state }}
+  ansible.builtin.user:
+    name: "{{ vmagent_user }}"
+    state: "{{ vmagent_state }}"
+    system: "{{ vmagent_user_system | default(true, true) }}"
+    groups: "{{ vmagent_user_groups | default(omit, true) }}"
+    append: "{{ (vmagent_user_groups | default([], true)) | length > 0 }}"
+    create_home: "{{ vmagent_user_create_home | default(false, true) }}"
+  register: vmagent_user_info
+
+- name: Ensure configuration file '{{ vmagent_scrape_config_file }}' is {{ vmagent_state }}
+  ansible.builtin.file:
+    path: "{{ vmagent_scrape_config_file }}"
+    state: "{{ vmagent_state }}"
+  when: vmagent_state == 'absent'
+
+- name: Ensure config directory '{{ vmagent_config_path }}' is {{ vmagent_state }}
+  ansible.builtin.file:
+    path: "{{ vmagent_config_path }}"
+    state: >-2
+      {{ (vmagent_state == 'present') | ternary('directory', 'absent') }}
+    owner: "{{ vmagent_run_user_id }}"
+    group: "{{ vmagent_run_group_id }}"
+    mode: "0755"
+
+- name: Ensure configuration file '{{ vmagent_scrape_config_file }}' is {{ vmagent_state }}
+  ansible.builtin.copy:
+    dest: "{{ vmagent_scrape_config_file }}"
+    content: "{{ vmagent_merged_config | to_nice_yaml(indent=4, width=1000) }}"
+    owner: "{{ vmagent_run_user_id }}"
+    group: "{{ vmagent_run_group_id }}"
+    mode: "0644"
+  when: vmagent_state == 'present'
+  notify:
+    - vmagent-reload
+
+- name: Ensure vmagent is deployed using {{ vmagent_deployment_method }}
+  ansible.builtin.include_tasks:
+    file: "deploy-{{ vmagent_deployment_method }}.yml"

roles/vmagent/vars/main.yml

@@ -0,0 +1,6 @@
+---
+vmagent_states:
+  - present
+  - absent
+vmagent_deployment_methods:
+  - docker

roles/vmalert/README.md

@@ -0,0 +1,11 @@
+# `finallycoffee.observability.vmalert` ansible role
+
+## Description
+
+This role configures `vmalert` and runs it in the officially distributed docker container.
+
+The default configuration file for recording rules is `vmalert_recording_config` and the default file for alerts is `vmalert_alert_config`. To set rules in a prometheus-like syntax, supply them to the role using `vmalert_alerts` or `vmalert_records`.
+
+It is also possible to pass extra rule-files to load using `vmalert_rule_files`, though care must be taken to also mount them to the location in the container by populating `vmalert_container_volumes`.
+
+VM alert runs with the `envflag.enable` flag by default, so configuration to vmalert can be passed using `vmalert_container_env` with the syntax found on the official victoriametrics documentation.

roles/vmalert/defaults/main.yml

@@ -0,0 +1,57 @@
+---
+vmalert_state: present
+vmalert_user: vmalert
+vmalert_version: "1.104.0"
+vmalert_base_path: "/opt/vmalert"
+vmalert_config_path: "{{ vmalert_base_path }}/config"
+vmalert_alert_config: "{{ vmalert_config_path }}/alerts.yml"
+vmalert_recording_config: "{{ vmalert_config_path }}/records.yml"
+
+vmalert_alerts: {}
+vmalert_records: {}
+vmalert_rule_files: []
+vmalert_default_rule_files:
+  - "{{ vmalert_alert_config }}"
+  - "{{ vmalert_recording_config }}"
+vmalert_merged_rule_files: >-
+  {{ vmalert_default_rule_files + vmalert_rule_files }}
+
+vmalert_container_image_server: docker.io
+vmalert_container_image_namespace: "victoriametrics"
+vmalert_container_image_container: "vmalert"
+vmalert_container_image_name: >-2
+  {{
+    vmalert_container_image_server
+    + ((vmalert_container_image_namespace is defined)
+      | ternary('/' ~ vmalert_container_image_namespace, ''))
+    + '/' + vmalert_container_image_container
+  }}
+#vmalert_container_image_tag:
+vmalert_container_image: >-2
+  {{ vmalert_container_image_name }}:{{ vmalert_container_image_tag | default('v' + vmalert_version, false) }}
+
+vmalert_user_id: >-
+  {{ (vmalert_user_info is defined and 'uid' in vmalert_user_info) | ternary(vmalert_user_info.uid, vmalert_user) }}
+vmalert_group_id: >-
+  {{ (vmalert_user_info is defined and 'group' in vmalert_user_info) | ternary(vmalert_user_info.group, vmalert_user) }}
+vmalert_container_user: "{{ vmalert_user_id }}"
+vmalert_container_group: "{{ vmalert_group_id }}"
+vmalert_container_name: "vmalert"
+vmalert_container_command: []
+vmalert_container_default_command:
+  - "-enableTCP6"
+  - "-envflag.enable"
+vmalert_container_merged_command: >-
+  {{ vmalert_container_default_command + (vmalert_container_command | default([], false)) }}
+vmalert_container_env: {}
+vmalert_container_default_env:
+  PATH: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+  rule: "{{ vmalert_merged_rule_files | join(',') }}"
+vmalert_container_merged_env: >-
+  {{ vmalert_container_default_env | combine(vmalert_container_env) }} 
+vmalert_container_volumes: []
+vmalert_container_default_volumes:
+  - "{{ vmalert_config_path }}:{{ vmalert_config_path }}:z"
+vmalert_container_merged_volumes: >-
+  {{ vmalert_container_default_volumes | combine(vmalert_container_volumes) }}
+vmalert_container_restart_policy: "unless-stopped"

roles/vmalert/tasks/main.yml

@@ -0,0 +1,69 @@
+---
+- name: Ensure user {{ vmalert_user }} is {{ vmalert_state }}
+  ansible.builtin.user:
+    name: "{{ vmalert_user }}"
+    state: present
+    system: true
+    create_home: false
+  register: vmalert_user_info
+
+- name: Ensure directories for vmalert are {{ vmalert_state }}
+  ansible.builtin.file:
+    path: "{{ item.path }}"
+    state: "{{ (vmalert_state == 'present') | ternary('directory', 'absent') }}"
+    owner: "{{ item.owner | default(vmalert_user_id) }}"
+    group: "{{ item.group | default(vmalert_group_id) }}"
+    mode: "{{ item.mode | default('0775') }}"
+  loop:
+    - path: "{{ vmalert_base_path }}"
+    - path: "{{ vmalert_config_path }}"
+      mode: "0755"
+  loop_control:
+    label: "{{ item.path }}"
+
+- name: Ensure alert configuration is present
+  ansible.builtin.copy:
+    dest: "{{ vmalert_alert_config }}"
+    content: |
+      {{ ({ 'groups': vmalert_alerts})
+      | to_nice_yaml(indent=2, width=1024, default_style='"') }}
+    owner: "{{ item.owner | default(vmalert_user_id) }}"
+    group: "{{ item.group | default(vmalert_group_id) }}"
+    mode: "{{ item.mode | default('0775') }}"
+  when: vmalert_state == 'present'
+
+- name: Ensure recording rule configuration is present
+  ansible.builtin.copy:
+    dest: "{{ vmalert_recording_config }}"
+    content: |
+      {{ ({ 'groups': vmalert_records})
+      | to_nice_yaml(indent=2, width=1024, default_style='"') }}
+    owner: "{{ item.owner | default(vmalert_user_id) }}"
+    group: "{{ item.group | default(vmalert_group_id) }}"
+    mode: "{{ item.mode | default('0775') }}"
+  when: vmalert_state == 'present'
+
+- name: Ensure container image {{ vmalert_container_image }} is {{ vmalert_state }}
+  community.docker.docker_image:
+    name: "{{ vmalert_container_image }}"
+    state: "{{ vmalert_state }}"
+    source: "{{ (vmalert_state == 'present') | ternary('pull', omit) }}"
+    force_source: >-2
+      {{ (vmalert_container_image == 'present') | ternary(vmalert_container_image_tag, omit) }}
+
+- name: Ensure vmalert container is {{ vmalert_state }}
+  community.docker.docker_container:
+    name: "{{ vmalert_container_name}}"
+    image: "{{ vmalert_container_image }}"
+    env: "{{ vmalert_container_merged_env }}"
+    user: "{{ vmalert_container_user }}"
+    ports: "{{ vmalert_container_ports | default(omit) }}"
+    groups: "{{ vmalert_container_group }}"
+    labels: "{{ vmalert_container_labels | default(omit) }}"
+    volumes: "{{ vmalert_container_merged_volumes }}"
+    command: "{{ vmalert_container_merged_command }}"
+    networks: "{{ vmalert_container_networks | default(omit) }}"
+    etc_hosts: "{{ vmalert_container_etc_hosts | default(omit )}}"
+    purge_networks: "{{ vmalert_container_purge_networks | default(omit) }}"
+    restart_policy: "{{ vmalert_container_restart_policy | default(omit) }}"
+    state: "{{ (vmalert_state == 'present') | ternary('started', 'absent') }}"

roles/vmtsdb/README.md

@@ -0,0 +1,7 @@
+# `finallycoffee.observability.vmtsdb` ansible role
+
+## Description
+
+This role configures `vmtsdb`, the time-series database part of victoria metrics, run in a docker container.
+
+Per default `enableTCP6` and `envflag.enable` flags are passed to victoriametrics, enabling configuration using `vmtsdb_container_env`, using the syntax found on the official victoriametrics documentation.

roles/vmtsdb/defaults/main.yml

@@ -0,0 +1,45 @@
+---
+vmtsdb_state: present
+vmtsdb_user: vmtsdb
+vmtsdb_version: "1.104.0"
+vmtsdb_base_path: "/opt/vmtsdb"
+vmtsdb_data_path: "{{ vmtsdb_base_path }}/data"
+
+vmtsdb_container_image_server: docker.io
+vmtsdb_container_image_namespace: "victoriametrics"
+vmtsdb_container_image_container: "victoria-metrics"
+vmtsdb_container_image_name: >-2
+  {{
+    vmtsdb_container_image_server
+    + ((vmtsdb_container_image_namespace is defined)
+      | ternary('/' ~ vmtsdb_container_image_namespace, ''))
+    + '/' + vmtsdb_container_image_container
+  }}
+#vmtsdb_container_image_tag:
+vmtsdb_container_image: >-2
+  {{ vmtsdb_container_image_name }}:{{ vmtsdb_container_image_tag | default('v' + vmtsdb_version, false) }}
+
+vmtsdb_user_id: >-
+  {{ (vmtsdb_user_info is defined and 'uid' in vmtsdb_user_info) | ternary(vmtsdb_user_info.uid, vmtsdb_user) }}
+vmtsdb_group_id: >-
+  {{ (vmtsdb_user_info is defined and 'group' in vmtsdb_user_info) | ternary(vmtsdb_user_info.group, vmtsdb_user) }}
+vmtsdb_container_user: "{{ vmtsdb_user_id }}"
+vmtsdb_container_group: "{{ vmtsdb_group_id }}"
+vmtsdb_container_name: "vmtsdb"
+vmtsdb_container_command: []
+vmtsdb_container_default_command:
+  - "-enableTCP6"
+  - "-envflag.enable"
+vmtsdb_container_merged_command: >-
+  {{ vmtsdb_container_default_command + (vmtsdb_container_command | default([], false)) }}
+vmtsdb_container_env: {}
+vmtsdb_container_default_env:
+  PATH: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+vmtsdb_container_merged_env: >-
+  {{ vmtsdb_container_default_env | combine(vmtsdb_container_env) }} 
+vmtsdb_container_volumes: []
+vmtsdb_container_default_volumes:
+  - "{{ vmtsdb_data_path }}:/victoria-metrics-data:z"
+vmtsdb_container_merged_volumes: >-
+  {{ vmtsdb_container_default_volumes | combine(vmtsdb_container_volumes) }}
+vmtsdb_container_restart_policy: "unless-stopped"

roles/vmtsdb/tasks/main.yml

@@ -0,0 +1,50 @@
+---
+- name: Ensure user {{ vmtsdb_user }} is {{ vmtsdb_state }}
+  ansible.builtin.user:
+    name: "{{ vmtsdb_user }}"
+    state: present
+    system: true
+    create_home: false
+  register: vmtsdb_user_info
+
+- name: Ensure directories for vmtsdb are {{ vmtsdb_state }}
+  ansible.builtin.file:
+    path: "{{ item.path }}"
+    state: "{{ (vmtsdb_state == 'present') | ternary('directory', 'absent') }}"
+    owner: "{{ item.owner | default(vmtsdb_user_id) }}"
+    group: "{{ item.group | default(vmtsdb_group_id) }}"
+    mode: "{{ item.mode | default('0775') }}"
+  loop:
+    - path: "{{ vmtsdb_base_path }}"
+    - path: "{{ vmtsdb_data_path }}"
+      mode: "0755"
+  loop_control:
+    label: "{{ item.path }}"
+
+- name: Ensure container image {{ vmtsdb_container_image }} is {{ vmtsdb_state }}
+  community.docker.docker_image:
+    name: "{{ vmtsdb_container_image }}"
+    state: "{{ vmtsdb_state }}"
+    source: "{{ (vmtsdb_state == 'present') | ternary('pull', omit) }}"
+    force_source: >-2
+      {{ (vmtsdb_container_image == 'present') | ternary(vmtsdb_container_image_tag, omit) }}
+
+- name: Ensure vmtsdb container is {{ vmtsdb_state }}
+  community.docker.docker_container:
+    name: "{{ vmtsdb_container_name}}"
+    image: "{{ vmtsdb_container_image }}"
+    env: "{{ vmtsdb_container_merged_env }}"
+    user: "{{ vmtsdb_container_user }}"
+    ports: "{{ vmtsdb_container_ports | default(omit) }}"
+    groups: "{{ vmtsdb_container_group }}"
+    labels: "{{ vmtsdb_container_labels | default(omit) }}"
+    volumes: "{{ vmtsdb_container_merged_volumes }}"
+    command: "{{ vmtsdb_container_merged_command }}"
+    networks: "{{ vmtsdb_container_networks | default(omit) }}"
+    etc_hosts: "{{ vmtsdb_container_etc_hosts | default(omit )}}"
+    purge_networks: "{{ vmtsdb_container_purge_networks | default(omit) }}"
+    restart_policy: "{{ vmtsdb_container_restart_policy | default(omit) }}"
+    state: "{{ (vmtsdb_state == 'present') | ternary('started', 'absent') }}"
+    comparisons:
+      labels: strict
+      env: strict