synapse workers: reworkings + get endpoints from upstream docs via awk

(yes, a bit awkward and brittle… xD)

roles/matrix-synapse/defaults/main.yml

@@ -275,31 +275,22 @@ matrix_synapse_manhole_enabled: false
 # Enable support for Synapse workers
 matrix_synapse_workers_enabled: false
 
-# Default list of workers to spawn
-# (worker with ports ending on 00 are single-instance)
+# Default list of workers to spawn (order in accord to docs)
+# - no endpoints / doesn't need port mapping if port ends on 0
+# - single-instance-only if 2nd last digit of port number is 0
 matrix_synapse_workers_enabled_list:
-  - { worker: generic_worker, port: 18101 }
-  - { worker: generic_worker, port: 18102 }
-  - { worker: generic_worker, port: 18103 }
-  - { worker: generic_worker, port: 18104 }
-  - { worker: generic_worker, port: 18105 }
-  - { worker: generic_worker, port: 18106 }
-  - { worker: appservice, port: 18200 }
-  - { worker: federation_sender, port: 18301 }
-  - { worker: frontend_proxy, port: 18400 }
-  - { worker: media_repository, port: 18501 }
-  - { worker: pusher, port: 18600 }
-  - { worker: user_dir, port: 18700 }
-
-# The list of available workers (2020-08-28)
-matrix_synapse_workers_avail_list:
-  - generic_worker
-  - appservice
-  - federation_sender
-  - frontend_proxy
-  - media_repository
-  - pusher
-  - user_dir
+  - { type: generic_worker, port: 18111 }
+  - { type: generic_worker, port: 18112 }
+  - { type: generic_worker, port: 18113 }
+  - { type: generic_worker, port: 18114 }
+  - { type: generic_worker, port: 18115 }
+  - { type: generic_worker, port: 18116 }
+  - { type: pusher, port: 00 }
+  - { type: appservice, port: 00 }
+  - { type: federation_sender, port: 0 }
+  - { type: media_repository, port: 18221 }
+  - { type: user_dir, port: 18331 }
+  - { type: frontend_proxy, port: 18441 }
 
 # Redis information
 matrix_synapse_redis_enabled: false

roles/matrix-synapse/files/workers-doc-to-yaml.awk

@@ -0,0 +1,137 @@
+#!/usr/bin/awk
+# Hackish approach to get a machine-readable list of current matrix 
+# synapse REST API endpoints from the official documentation at
+# https://github.com/matrix-org/synapse/raw/master/docs/workers.md
+#
+# invoke in shell with:
+# URL=https://github.com/matrix-org/synapse/raw/master/docs/workers.md
+# curl -L ${URL} | awk -f parse-workers-docs.awk -
+
+function worker_stanza_append(string) {
+    worker_stanza = worker_stanza string
+}
+
+function line_is_endpoint_url(line) {
+    # probably API endpoint if it starts with white-space and ^ or /
+    return (line ~ /^ +[\^/].*\//)
+}
+
+# Put YAML marker at beginning of file.
+BEGIN {
+    print "---"
+    endpoint_conditional_comment = "  # FIXME: ADDITIONAL CONDITIONS REQUIRED: to be enabled manually\n"
+}
+
+# Enable further processing after the introductory text.
+# Read each synapse worker section as record and its lines as fields.
+/Available worker applications/ {
+    enable_parsing = 1
+    # set record separator to markdown section header
+    RS = "\n### "
+    # set field separator to newline
+    FS = "\n"
+}
+
+# Once parsing is active, this will process each section as record.
+enable_parsing {
+    # Each worker section starts with a synapse.app.X headline
+    if ($1 ~ /synapse\.app\./) {
+
+        # get rid of the backticks and extract worker type from headline
+        gsub("`", "", $1)
+        gsub("synapse.app.", "", $1)
+        worker_type = $1
+
+        # initialize empty worker stanza
+        worker_stanza = ""
+
+        # track if any endpoints are mentioned in a specific section
+        worker_has_urls = 0
+
+        # some endpoint descriptions contain flag terms
+        endpoints_seem_conditional = 0
+
+        # also, collect a list of available workers
+        workers = (workers ? workers "\n" : "") "  - " worker_type
+
+        # loop through the lines (2 - number of fields in record)
+        for (i = 1; i < NF + 1; i++) {
+            # copy line for gsub replacements
+            line = $i
+
+            # end all lines but the last with a linefeed
+            linefeed = (i < NF - 1) ? "\n" : ""
+
+            # line starts with white-space and a hash: endpoint block headline
+            if (line ~ /^ +#/) {
+
+                # copy to output verbatim, normalizing white-space
+                gsub(/^ +/, "", line)
+                worker_stanza_append("  " line linefeed)
+
+            } else if (line_is_endpoint_url(line)) {
+
+                # mark section for special output formatting
+                worker_has_urls = 1
+
+                # remove leading white-space
+                gsub(/^ +/, "", line)
+                api_endpoint_regex = line
+
+                # FIXME: https://github.com/matrix-org/synapse/issues/new
+                # munge inconsistent media_repository endpoint notation
+                if (api_endpoint_regex == "/_matrix/media/") {
+                    api_endpoint_regex = "^" line
+                }
+
+                # disable endpoints which specify complications
+                if (endpoints_seem_conditional) {
+                    # only add notice if previous line didn't match
+                    if (!line_is_endpoint_url($(i - 1))) {
+                        worker_stanza_append(endpoint_conditional_comment)
+                    }
+                    worker_stanza_append("  # " api_endpoint_regex linefeed)
+                } else {
+                    # output endpoint regex
+                    worker_stanza_append("  - " api_endpoint_regex linefeed)
+                }
+
+            # white-space only line?
+            } else if (line ~ /^\w*$/) {
+
+                if (i > 3 && i < NF) {
+                    # print white-space lines unless 1st or last line in section
+                    worker_stanza_append(line linefeed)
+                }
+
+            # nothing of the above: the line is regular documentation text
+            } else {
+
+                # include this text line as comment
+                worker_stanza_append("  # " line linefeed)
+
+                # and take note of words hinting at additional conditions to be met
+                if (line ~ /\<[Ii]f\>|\<[Ff]or\>/) {
+                    endpoints_seem_conditional = 1
+                }
+            }
+        }
+
+        if (worker_has_urls) {
+            print "\nmatrix_synapse_workers_" worker_type "_endpoints:"
+            print worker_stanza
+        } else {
+            # include workers without endpoints as well for reference
+            print "\n# " worker_type " worker (no API endpoints) ["
+            print worker_stanza
+            print "# ]"
+        }
+    }
+}
+
+END {
+    print "\nmatrix_synapse_workers_avail_list:"
+    print workers | "sort"
+}
+
+# vim: tabstop=4 shiftwidth=4 expandtab autoindent

roles/matrix-synapse/tasks/workers/setup_install.yml

@@ -1,5 +1,18 @@
 ---
 
+- name: Download synapse workers doc
+  get_url:
+    url: https://github.com/matrix-org/synapse/raw/master/docs/workers.md
+    dest: "{{ role_path }}/files/workers.upstream-documentation.md"
+
+- name: Download synapse workers doc and convert into YAML
+  shell:
+    cmd: "awk -f {{ role_path }}/files/workers-doc-to-yaml.awk -- {{ role_path }}/files/workers.upstream-documentation.md > {{ role_path }}/vars/workers.yml"
+    creates: "{{ role_path }}/vars/workers.yml"
+
+- name: Load list of available worker apps and endpoints
+  include_vars: "{{ role_path }}/vars/workers.yml"
+
 - name: Ensure synapse worker base service file installed
   template:
     src: "{{ role_path }}/templates/synapse/systemd/matrix-synapse-worker@.service.j2"
@@ -23,7 +36,7 @@
 
 - name: Ensure individual worker service symlinks exist
   service:
-    name: "matrix-synapse-worker@{{ item.worker }}:{{ item.port }}.service"
+    name: "matrix-synapse-worker@{{ item.type }}:{{ item.port }}.service"
     enabled: true
   with_items: "{{ matrix_synapse_workers_enabled_list }}"
 
@@ -37,14 +50,14 @@
 - name: Ensure creation of specific worker configs
   template:
     src: "{{ role_path }}/templates/synapse/worker.yaml.j2"
-    dest: "{{ matrix_synapse_config_dir_path }}/worker.{{ item.worker }}:{{ item.port }}.yaml"
+    dest: "{{ matrix_synapse_config_dir_path }}/worker.{{ item.type }}:{{ item.port }}.yaml"
   with_list: "{{ matrix_synapse_workers_enabled_list }}"
 
 - name: Add workers to synapse.wants list
   set_fact:
     matrix_synapse_systemd_wanted_services_list: >
       {{ matrix_synapse_systemd_wanted_services_list +
-      ['matrix-synapse-worker@' + item.worker + ':' + item.port|string + '.service'] }}
+      ['matrix-synapse-worker@' + item.type + ':' + item.port|string + '.service'] }}
   with_items: "{{ matrix_synapse_workers_enabled_list }}"
 
 - name: Ensure matrix-synapse-worker-write-pid script is created

roles/matrix-synapse/templates/synapse/homeserver.yaml.j2

@@ -249,19 +249,19 @@ worker_app: synapse.app.homeserver
 
 # thx https://oznetnerd.com/2017/04/18/jinja2-selectattr-filter/
 # reduce the main worker's offerings to core homeserver business
-{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'appservice')|list  %}
+{% if matrix_synapse_workers_enabled_list|selectattr('type', 'equalto', 'appservice')|list  %}
 notify_appservices: false
 {% endif %}
-{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'federation_sender')|list  %}
+{% if matrix_synapse_workers_enabled_list|selectattr('type', 'equalto', 'federation_sender')|list  %}
 send_federation: false
 {% endif %}
-{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'media_repository')|list  %}
+{% if matrix_synapse_workers_enabled_list|selectattr('type', 'equalto', 'media_repository')|list  %}
 enable_media_repo: false
 {% endif %}
-{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'pusher')|list  %}
+{% if matrix_synapse_workers_enabled_list|selectattr('type', 'equalto', 'pusher')|list  %}
 start_pushers: false
 {% endif %}
-{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'user_dir')|list  %}
+{% if matrix_synapse_workers_enabled_list|selectattr('type', 'equalto', 'user_dir')|list  %}
 update_user_directory: false
 {% endif %}
 

roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2

@@ -43,9 +43,12 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-synapse \
 			{% if matrix_synapse_manhole_enabled and matrix_synapse_container_manhole_api_host_bind_port %}
 			-p {{ matrix_synapse_container_manhole_api_host_bind_port }}:9000 \
 			{% endif %}
-			{% if matrix_synapse_workers_enabled %}
+			{% if matrix_synapse_workers_enabled and not matrix_nginx_proxy_enabled|default(False) %}
+			{# Expose worker (by default 18xxx range) ports on host if not using internal nginx proxy #}
 			{% for worker in matrix_synapse_workers_enabled_list %}
+			{% if worker.port != 0 %}
 			-p {{ worker.port }}:{{ worker.port }} \
+			{% endif %}
 			{% endfor %}
 			{% endif %}
 			-v {{ matrix_synapse_config_dir_path }}:/data:ro \

roles/matrix-synapse/templates/synapse/worker.yaml.j2

@@ -1,27 +1,27 @@
 #jinja2: lstrip_blocks: "True"
-worker_app: synapse.app.{{ item.worker }}
-worker_name: {{ item.worker ~ ':' ~ item.port }}
+worker_app: synapse.app.{{ item.type }}
+worker_name: {{ item.type ~ ':' ~ item.port }}
 
 worker_replication_host: 127.0.0.1
 worker_replication_http_port: {{ matrix_synapse_replication_http_port }}
 
-{% if item.worker not in [ 'appservice', 'federation_sender', 'pusher' ] %}
+{% if item.type not in [ 'appservice', 'federation_sender', 'pusher' ] %}
 worker_listeners:
   - type: http
     port: {{ item.port }}
     resources:
       - names:
-{% if item.worker in [ 'generic_worker', 'frontend_proxy', 'user_dir' ] %}
+{% if item.type in [ 'generic_worker', 'frontend_proxy', 'user_dir' ] %}
         - client
 {% endif %}
-{% if item.worker in [ 'generic_worker' ] %}
+{% if item.type in [ 'generic_worker' ] %}
         - federation
-{% elif item.worker in [ 'media_repository' ] %}
+{% elif item.type in [ 'media_repository' ] %}
         - media
 {% endif %}
 {% endif %}
 
-{% if item.worker == 'frontend_proxy' %}
+{% if item.type == 'frontend_proxy' %}
 worker_main_http_uri: http://127.0.0.1:8008
 {% endif %}