From 3fd6fd647f85ca5cb07550b3d1361b00ea321f32 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Wed, 8 Aug 2018 08:23:36 +0300 Subject: [PATCH] Put all containers in their own isolated Docker network (matrix) Moving away from using the default bridge network to using our own. This isolates our services from other Docker containers running on the default network on the same host. The benefits are that: - isolation is a little better - we no longer share a default bridge network with any other containers that might be running on the host - there are no longer hard dependencies - we do service discovery by DNS name, and not via explicit `--link` usage during container start, so containers can start out of order and fail without bringing down others with them (`matrix-nginx-proxy` can continue running, even if one of the other services dies) In the future, when other services get introduced, the increased resilience and simplicity will help as well. --- CHANGELOG.md | 6 ++++++ roles/matrix-server/defaults/main.yml | 4 +++- roles/matrix-server/tasks/setup_main.yml | 7 ++++++- .../templates/nginx-conf.d/matrix-riot-web.conf.j2 | 11 ++++++++++- .../templates/nginx-conf.d/matrix-synapse.conf.j2 | 11 ++++++++++- .../templates/systemd/matrix-nginx-proxy.service.j2 | 11 +++-------- .../templates/systemd/matrix-postgres.service.j2 | 1 + .../templates/systemd/matrix-riot-web.service.j2 | 1 + .../templates/systemd/matrix-synapse.service.j2 | 4 +--- .../templates/usr-local-bin/matrix-postgres-cli.j2 | 4 +--- 10 files changed, 42 insertions(+), 18 deletions(-) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 000000000..9820a6201 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,6 @@ +# 2018-08-08 + + +## Docker container linking + +Changed the way the Docker containers are linked together. The ones that need to communicate with others operate in a `matrix` network now and not in the default bridge network. \ No newline at end of file diff --git a/roles/matrix-server/defaults/main.yml b/roles/matrix-server/defaults/main.yml index d3a3a685d..170b1390a 100644 --- a/roles/matrix-server/defaults/main.yml +++ b/roles/matrix-server/defaults/main.yml @@ -18,7 +18,7 @@ matrix_user_gid: 991 # The defaults below cause a postgres server to be configured (running within a container). # Using an external server is possible by tweaking all of the parameters below. matrix_postgres_use_external: false -matrix_postgres_connection_hostname: "postgres" +matrix_postgres_connection_hostname: "matrix-postgres" matrix_postgres_connection_username: "synapse" matrix_postgres_connection_password: "synapse-password" matrix_postgres_db_name: "homeserver" @@ -70,6 +70,8 @@ docker_s3fs_image: "xueshanf/s3fs:latest" docker_goofys_image: "cloudproto/goofys:latest" docker_coturn_image: "instrumentisto/coturn:4.5.0.7" +# The Docker network that all services would be put into +matrix_docker_network: "matrix" # A shared secret (between Synapse and Coturn) used for authentication. # You can put any string here, but generating a strong one is preferred (e.g. `pwgen -s 64 1`). diff --git a/roles/matrix-server/tasks/setup_main.yml b/roles/matrix-server/tasks/setup_main.yml index e16e20db8..71a6b96c8 100644 --- a/roles/matrix-server/tasks/setup_main.yml +++ b/roles/matrix-server/tasks/setup_main.yml @@ -28,4 +28,9 @@ group: "{{ matrix_user_username }}" with_items: - "{{ matrix_base_data_path }}" - - "{{ matrix_synapse_base_path }}" \ No newline at end of file + - "{{ matrix_synapse_base_path }}" + +- name: Ensure Matrix network is created in Docker + docker_network: + name: "{{ matrix_docker_network }}" + driver: bridge diff --git a/roles/matrix-server/templates/nginx-conf.d/matrix-riot-web.conf.j2 b/roles/matrix-server/templates/nginx-conf.d/matrix-riot-web.conf.j2 index 24c149fb6..c24daedab 100644 --- a/roles/matrix-server/templates/nginx-conf.d/matrix-riot-web.conf.j2 +++ b/roles/matrix-server/templates/nginx-conf.d/matrix-riot-web.conf.j2 @@ -40,7 +40,16 @@ server { ssl_ciphers "EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH"; location / { - proxy_pass http://{{ 'riot' if matrix_nginx_proxy_enabled else 'localhost' }}:8765; + {% if matrix_nginx_proxy_enabled %} + {# Use the embedded DNS resolver in Docker containers to discover the service #} + resolver 127.0.0.11 valid=5s; + set $backend "matrix-riot-web:8765"; + proxy_pass http://$backend; + {% else %} + {# Generic configuration for people to use outside of our container setup #} + proxy_pass http://localhost:8765; + {% endif %} + proxy_set_header X-Forwarded-For $remote_addr; } } diff --git a/roles/matrix-server/templates/nginx-conf.d/matrix-synapse.conf.j2 b/roles/matrix-server/templates/nginx-conf.d/matrix-synapse.conf.j2 index aba7c319d..56c786571 100644 --- a/roles/matrix-server/templates/nginx-conf.d/matrix-synapse.conf.j2 +++ b/roles/matrix-server/templates/nginx-conf.d/matrix-synapse.conf.j2 @@ -40,7 +40,16 @@ server { ssl_ciphers "EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH"; location /_matrix { - proxy_pass http://{{ 'synapse' if matrix_nginx_proxy_enabled else 'localhost' }}:8008; + {% if matrix_nginx_proxy_enabled %} + {# Use the embedded DNS resolver in Docker containers to discover the service #} + resolver 127.0.0.11 valid=5s; + set $backend "matrix-synapse:8008"; + proxy_pass http://$backend; + {% else %} + {# Generic configuration for people to use outside of our container setup #} + proxy_pass http://localhost:8008; + {% endif %} + proxy_set_header X-Forwarded-For $remote_addr; client_body_buffer_size 25M; diff --git a/roles/matrix-server/templates/systemd/matrix-nginx-proxy.service.j2 b/roles/matrix-server/templates/systemd/matrix-nginx-proxy.service.j2 index 579a1881c..c37b209fb 100644 --- a/roles/matrix-server/templates/systemd/matrix-nginx-proxy.service.j2 +++ b/roles/matrix-server/templates/systemd/matrix-nginx-proxy.service.j2 @@ -2,11 +2,9 @@ Description=Matrix nginx proxy server After=docker.service Requires=docker.service -Requires=matrix-synapse.service -After=matrix-synapse.service +Wants=matrix-synapse.service {% if matrix_riot_web_enabled %} -Requires=matrix-riot-web.service -After=matrix-riot-web.service +Wants=matrix-riot-web.service {% endif %} [Service] @@ -14,12 +12,9 @@ Type=simple ExecStartPre=-/usr/bin/docker kill matrix-nginx-proxy ExecStartPre=-/usr/bin/docker rm matrix-nginx-proxy ExecStart=/usr/bin/docker run --rm --name matrix-nginx-proxy \ + --network {{ matrix_docker_network }} \ -p 80:80 \ -p 443:443 \ - --link matrix-synapse:synapse \ - {% if matrix_riot_web_enabled %} - --link matrix-riot-web:riot \ - {% endif %} -v {{ matrix_nginx_proxy_confd_path }}:/etc/nginx/conf.d:ro \ -v {{ matrix_ssl_certs_path }}:{{ matrix_ssl_certs_path }}:ro \ {{ docker_nginx_image }} diff --git a/roles/matrix-server/templates/systemd/matrix-postgres.service.j2 b/roles/matrix-server/templates/systemd/matrix-postgres.service.j2 index f0bc4bd4f..60edf39b9 100644 --- a/roles/matrix-server/templates/systemd/matrix-postgres.service.j2 +++ b/roles/matrix-server/templates/systemd/matrix-postgres.service.j2 @@ -11,6 +11,7 @@ ExecStartPre=-/usr/bin/mkdir {{ matrix_postgres_data_path }} ExecStartPre=-/usr/bin/chown {{ matrix_user_uid }}:{{ matrix_user_gid }} {{ matrix_postgres_data_path }} ExecStart=/usr/bin/docker run --rm --name matrix-postgres \ --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ + --network {{ matrix_docker_network }} \ --env-file={{ matrix_environment_variables_data_path }}/env-postgres-server-docker \ -v {{ matrix_postgres_data_path }}:/var/lib/postgresql/data \ -v /etc/passwd:/etc/passwd:ro \ diff --git a/roles/matrix-server/templates/systemd/matrix-riot-web.service.j2 b/roles/matrix-server/templates/systemd/matrix-riot-web.service.j2 index 075825565..ee2035c64 100644 --- a/roles/matrix-server/templates/systemd/matrix-riot-web.service.j2 +++ b/roles/matrix-server/templates/systemd/matrix-riot-web.service.j2 @@ -11,6 +11,7 @@ ExecStart=/usr/bin/docker run --rm --name matrix-riot-web \ --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ -v {{ matrix_nginx_riot_web_data_path }}/config.json:/riot-web/webapp/config.json:ro \ -v {{ matrix_nginx_riot_web_data_path }}/riot.im.conf:/data/riot.im.conf:ro \ + --network {{ matrix_docker_network }} \ {% if not matrix_nginx_proxy_enabled %} -p 127.0.0.1:8765:8765 \ {% endif %} diff --git a/roles/matrix-server/templates/systemd/matrix-synapse.service.j2 b/roles/matrix-server/templates/systemd/matrix-synapse.service.j2 index 19f4341c9..7eb70e364 100644 --- a/roles/matrix-server/templates/systemd/matrix-synapse.service.j2 +++ b/roles/matrix-server/templates/systemd/matrix-synapse.service.j2 @@ -23,9 +23,7 @@ ExecStartPre=-/usr/bin/docker rm matrix-synapse ExecStartPre=/bin/sleep 5 {% endif %} ExecStart=/usr/bin/docker run --rm --name matrix-synapse \ - {% if not matrix_postgres_use_external %} - --link matrix-postgres:{{ matrix_postgres_connection_hostname }} \ - {% endif %} + --network {{ matrix_docker_network }} \ -p 8448:8448 \ {% if not matrix_nginx_proxy_enabled %} -p 127.0.0.1:8008:8008 \ diff --git a/roles/matrix-server/templates/usr-local-bin/matrix-postgres-cli.j2 b/roles/matrix-server/templates/usr-local-bin/matrix-postgres-cli.j2 index f23792765..d821c4bb7 100644 --- a/roles/matrix-server/templates/usr-local-bin/matrix-postgres-cli.j2 +++ b/roles/matrix-server/templates/usr-local-bin/matrix-postgres-cli.j2 @@ -4,8 +4,6 @@ docker run \ -it \ --rm \ --env-file={{ matrix_environment_variables_data_path }}/env-postgres-pgsql-docker \ - {% if not matrix_postgres_use_external %} - --link=matrix-postgres:{{ matrix_postgres_connection_hostname }} \ - {% endif %} + --network {{ matrix_docker_network }} \ {{ docker_postgres_image_to_use }} \ psql -h {{ matrix_postgres_connection_hostname }} \ No newline at end of file