gateway

colony/cli.py

@@ -125,6 +125,10 @@ def role_states(payload):
     return states
 
 
+def address_of(key):
+    return f"{key[0]}:{key[1]}" if key else ""
+
+
 def render_status(payload):
     members = payload["members"]
 
@@ -142,7 +146,8 @@ def render_status(payload):
         title += f", {down} down"
 
     show_declined = any(member["declined"] for member in members)
-    node_headers = ["NODE", "ADDRESS", "STATE", "SEEN", "ROLES"] + (["DECLINED"] if show_declined else [])
+    show_route = any(member.get("via") for member in members)
+    node_headers = ["NODE", "ADDRESS", "STATE", "SEEN", "ROLES"] + (["DECLINED"] if show_declined else []) + (["ROUTE"] if show_route else [])
     node_rows = []
     for member in members:
         label = labels[address(member)] + (" (this node)" if member["self"] else "")
@@ -155,6 +160,12 @@ def render_status(payload):
         ]
         if show_declined:
             row.append(" ".join(member["declined"]) or "-")
+        if show_route:
+            via = member.get("via")
+            if via:
+                row.append(f"via {labels.get(address_of(via), address_of(via))}")
+            else:
+                row.append("direct" if member.get("direct") or member["self"] else "unknown")
         node_rows.append(row)
 
     alive_addresses = {address(member) for member in alive}

colony/core/context.py

@@ -11,7 +11,19 @@ class RoleContext:
         return self._network.execute(command, *args, **kwargs)
 
     def execute_to_unregistered_node(self, ip, port, command, *args, **kwargs):
-        return self._network.execute_to_unregistered_node(ip, port, command, *args, **kwargs)
+        return self._network.send_command(ip, port, command, *args, **kwargs)
+
+    def send_direct(self, ip, port, payload, timeout=None, connect_timeout=None):
+        return self._network.send_direct(ip, port, payload, timeout, connect_timeout)
+
+    def is_direct(self, key):
+        return self._network.is_direct(key)
+
+    def route_of(self, key):
+        return self._network.route_of(key)
+
+    def set_dial_alias(self, key, address):
+        self._network.set_dial_alias(key, address)
 
     def peers(self):
         return self._network.get_all_nodes()

colony/core/dispatcher.py

@@ -1,6 +1,28 @@
 from colony.core.logger.logging import get_logger
 
 
+def parse_command(payload):
+    command = payload.get("command") if isinstance(payload, dict) else None
+    if not isinstance(command, dict):
+        raise ValueError("Payload has no command")
+
+    role_name = command.get("role")
+    command_name = command.get("name")
+    if not isinstance(role_name, str) or not role_name or not isinstance(command_name, str) or not command_name:
+        raise ValueError("Command needs a role and a name")
+
+    args = command.get("args")
+    if args is None:
+        args = []
+    kwargs = command.get("kwargs")
+    if kwargs is None:
+        kwargs = {}
+    if not isinstance(args, (list, tuple)) or not isinstance(kwargs, dict):
+        raise ValueError("Command args must be a list and kwargs a dict")
+
+    return role_name, command_name, tuple(args), kwargs
+
+
 class Dispatcher:
     def __init__(self, network, role_registry):
         self.logger = get_logger(self.__class__.__name__)
@@ -8,30 +30,8 @@ class Dispatcher:
         self.network = network
         self.role_registry = role_registry
 
-    @staticmethod
-    def _parse(payload):
-        command = payload.get("command") if isinstance(payload, dict) else None
-        if not isinstance(command, dict):
-            raise ValueError("Payload has no command")
-
-        role_name = command.get("role")
-        command_name = command.get("name")
-        if not isinstance(role_name, str) or not role_name or not isinstance(command_name, str) or not command_name:
-            raise ValueError("Command needs a role and a name")
-
-        args = command.get("args")
-        if args is None:
-            args = []
-        kwargs = command.get("kwargs")
-        if kwargs is None:
-            kwargs = {}
-        if not isinstance(args, (list, tuple)) or not isinstance(kwargs, dict):
-            raise ValueError("Command args must be a list and kwargs a dict")
-
-        return role_name, command_name, tuple(args), kwargs
-
     def dispatch(self, payload):
-        role_name, command_name, args, kwargs = self._parse(payload)
+        role_name, command_name, args, kwargs = parse_command(payload)
         self.logger.debug(f"Dispatching payload for command {role_name}.{command_name}")
 
         role_entry = self.role_registry.get_role_class_by_name(role_name)

colony/core/network/core.py

@@ -4,15 +4,16 @@ from typing import Optional
 
 from colony.core.network.membership import Membership
 from colony.core.network.nodes import LocalNode, Node
+from colony.core.network.relay import GATEWAY_ROLE, RelayError, build_forward, read_reply
 from colony.core.network.requests import CommandRequest
 from colony.core.logger.logging import get_logger
-from colony.settings import configuration
+from colony.settings import configuration, setting
-from colony.transport.whisperer import Whisperer
+from colony.transport.whisperer import ConnectFailed, Whisperer
 from colony.utils.network import get_local_ip, get_hostname
 
 
 class Network:
-    def __init__(self, ip=None, port=None, hostname=None, membership=None):
+    def __init__(self, ip=None, port=None, hostname=None, membership=None, sender=None):
         self.logger = get_logger(self.__class__.__name__)
 
         self.self_node = LocalNode(
@@ -27,6 +28,19 @@ class Network:
             self.self_node.ip, self.self_node.port, self.self_node.hostname
         )
 
+        self._sender = sender or self._whisper
+        self._client_fallback = setting("gateway.client_fallback")
+        self._connect_timeout = setting("gateway.connect_timeout")
+        self._direct_retry = setting("gateway.direct_retry")
+        self._max_attempts = setting("gateway.max_attempts")
+        self._forward_timeout = setting("gateway.forward_timeout")
+        self._direct_failed = {}
+        self._gateway_bad = {}
+        self._no_route = {}
+        self._direct_ok = {}
+        self._via = {}
+        self._dial = {}
+
     @property
     def nodes(self):
         peers = [
@@ -59,10 +73,114 @@ class Network:
         if node is None:
             raise ValueError(f"No node found with role {request.role}")
 
-        result = node.execute(command, *args, **kwargs)
+        result = self.send_command(node.ip, node.port, command, *args, **kwargs)
         self.logger.debug(f"Command {request.id} executed")
         return result
 
+    @staticmethod
+    def _whisper(ip, port, payload, timeout, connect_timeout):
+        return Whisperer(ip, port).send(payload, timeout=timeout, connect_timeout=connect_timeout)
+
+    def set_dial_alias(self, key, address):
+        key, address = tuple(key), tuple(address)
+        members = {record.key for record in self.membership.alive_records()}
+        if key == address or key == self.membership.key or address in members:
+            return
+        self._dial[key] = (address, self.membership.now() + self.membership.forget_after)
+
+    def _dial_address(self, key):
+        entry = self._dial.get(key)
+        if entry is None:
+            return key
+        if entry[1] <= self.membership.now():
+            self._dial.pop(key, None)
+            return key
+        return entry[0]
+
+    def send_direct(self, ip, port, payload, timeout=None, connect_timeout=None):
+        ip, port = self._dial_address((ip, port))
+        return self._sender(ip, port, payload, timeout or Whisperer.DEFAULT_TIMEOUT, connect_timeout)
+
+    def _gateways(self, key, now):
+        if not self._client_fallback:
+            return []
+        candidates = [
+            record.key for record in self.membership.alive_records()
+            if GATEWAY_ROLE in record.roles and record.key not in (self.membership.key, key)
+            and self._gateway_bad.get(record.key, 0) <= now
+            and self._no_route.get((record.key, key), 0) <= now
+        ]
+        preferred = self._via.get(key)
+        return sorted(candidates, key=lambda gateway: (gateway != preferred, gateway))
+
+    def is_direct(self, key):
+        seen = self._direct_ok.get(key)
+        return seen is not None and seen + self.membership.member_timeout * 2 > self.membership.now()
+
+    def route_of(self, key):
+        if self._direct_failed.get(key, 0) > self.membership.now():
+            return self._via.get(key)
+        return None
+
+    def send_command(self, ip, port, command, *args, timeout=None, **kwargs):
+        payload = CommandRequest.form(command, *args, **kwargs).collapse()
+        key = (ip, port)
+        now = self.membership.now()
+        gateways = self._gateways(key, now)
+
+        if gateways and self._direct_failed.get(key, 0) > now:
+            return self._relay(key, payload, timeout, gateways, None)
+
+        try:
+            response = self.send_direct(ip, port, payload, timeout, self._connect_timeout if gateways else None)
+        except ConnectFailed as error:
+            if not gateways:
+                raise
+            self._direct_failed[key] = now + self._direct_retry
+            self._direct_ok.pop(key, None)
+            return self._relay(key, payload, timeout, gateways, error)
+
+        self._direct_failed.pop(key, None)
+        self._via.pop(key, None)
+        self._direct_ok[key] = now
+        return response
+
+    def _relay(self, key, payload, timeout, gateways, direct_error):
+        envelope = build_forward(key, payload["command"], min(timeout or Whisperer.DEFAULT_TIMEOUT, self._forward_timeout))
+        now = self.membership.now()
+
+        for gateway in gateways[:self._max_attempts]:
+            try:
+                reply = self.send_direct(gateway[0], gateway[1], envelope, timeout, self._connect_timeout)
+            except ConnectFailed:
+                self._gateway_bad[gateway] = now + self._direct_retry
+                continue
+
+            outcome, value = read_reply(reply)
+            if outcome == "ok":
+                self._via[key] = gateway
+                return value
+            if outcome == "failover":
+                if value == "unreachable":
+                    self._no_route[(gateway, key)] = now + self._direct_retry
+                else:
+                    self._gateway_bad[gateway] = now + self._direct_retry
+                continue
+            raise RelayError(value)
+
+        if direct_error is not None:
+            raise direct_error
+        return self._last_resort(key, payload, timeout)
+
+    def _last_resort(self, key, payload, timeout):
+        try:
+            response = self.send_direct(key[0], key[1], payload, timeout, self._connect_timeout)
+        except ConnectFailed:
+            raise ConnectFailed(f"Cannot reach {key[0]}:{key[1]} directly or through a gateway") from None
+        self._direct_failed.pop(key, None)
+        self._via.pop(key, None)
+        self._direct_ok[key] = self.membership.now()
+        return response
 
     @staticmethod
     def execute_to_unregistered_node(ip, port, command: Enum, *args, timeout=None, **kwargs):

colony/core/network/relay.py

@@ -0,0 +1,30 @@
+GATEWAY_ROLE = "Gateway"
+FAIL_OVER_CODES = {"busy", "unreachable"}
+
+
+class RelayError(OSError):
+    def __init__(self, code):
+        super().__init__(f"The gateway refused the request: {code}")
+        self.code = code
+
+
+def build_forward(target, inner_command, timeout):
+    return {"command": {
+        "role": GATEWAY_ROLE,
+        "name": "Forward",
+        "args": [{"target": list(target), "command": inner_command, "timeout": timeout}],
+        "kwargs": {},
+    }}
+
+
+def read_reply(envelope):
+    if not isinstance(envelope, dict) or envelope.get("status") != "ok" or not isinstance(envelope.get("result"), dict):
+        return "failover", "gateway_error"
+
+    result = envelope["result"]
+    if result.get("ok") is True and isinstance(result.get("response"), dict):
+        return "ok", result["response"]
+
+    code = result.get("code")
+    code = code if isinstance(code, str) else "error"
+    return ("failover" if code in FAIL_OVER_CODES else "refused"), code

colony/core/roles/gateway/role.py

@@ -0,0 +1,105 @@
+import ipaddress
+import json
+import math
+import threading
+from enum import Enum
+
+from colony.core.dispatcher import parse_command
+from colony.core.network.relay import GATEWAY_ROLE
+from colony.core.roles.role import Role, Command
+from colony.settings import setting
+from colony.transport.whisperer import ConnectFailed
+
+
+def failure(code):
+    return {"ok": False, "code": code}
+
+
+class Gateway(Role):
+    class COMMANDS(Enum):
+        FORWARD = 'Gateway.Forward'
+
+    def __init__(self, context):
+        commands = {
+            self.COMMANDS.FORWARD.value: Command(self.forward),
+        }
+
+        super().__init__(commands, context)
+
+        self.allow = set(setting("gateway.allow_commands"))
+        self.max_payload = setting("gateway.max_payload_bytes")
+        self.forward_timeout = setting("gateway.forward_timeout")
+        self.connect_timeout = setting("gateway.connect_timeout")
+        self._slots = threading.BoundedSemaphore(setting("gateway.max_concurrent"))
+        self.forwarded = 0
+
+    def forward(self, request):
+        result = self._forward(request)
+        if result["ok"]:
+            self.forwarded += 1
+        self.logger.info(f"Forward finished: {'ok' if result['ok'] else result['code']}")
+        return result
+
+    def _read(self, request):
+        if not isinstance(request, dict):
+            raise ValueError("Request must be an object")
+
+        target = request.get("target")
+        if not isinstance(target, list) or len(target) != 2:
+            raise ValueError("Target must be [ip, port]")
+        ip, port = target
+        if not isinstance(ip, str):
+            raise ValueError("Bad target ip")
+        ip = str(ipaddress.ip_address(ip))
+        if not isinstance(port, int) or isinstance(port, bool) or not 1 <= port <= 65535:
+            raise ValueError("Bad target port")
+
+        command = request.get("command")
+        parse_command({"command": command})
+
+        timeout = request.get("timeout", self.forward_timeout)
+        if isinstance(timeout, bool) or not isinstance(timeout, (int, float)) or not math.isfinite(timeout) or timeout <= 0:
+            raise ValueError("Bad timeout")
+        return (ip, port), command, min(timeout, self.forward_timeout)
+
+    def _allowed(self, command):
+        name = f"{command['role']}.{command['name']}"
+        return command["role"] != GATEWAY_ROLE and ("*" in self.allow or name in self.allow)
+
+    def _is_member(self, target):
+        membership = self.context.membership()
+        return any(record.key == target for record in membership.alive_records() if record.key != membership.key)
+
+    def _forward(self, request):
+        try:
+            target, command, timeout = self._read(request)
+        except (ValueError, TypeError):
+            return failure("bad_request")
+
+        if not self._allowed(command):
+            return failure("denied")
+        if not self._is_member(target):
+            return failure("unknown_target")
+
+        payload = {"command": command}
+        if len(json.dumps(payload)) > self.max_payload:
+            return failure("too_large")
+        if not self._slots.acquire(blocking=False):
+            return failure("busy")
+
+        try:
+            response = self.context.send_direct(
+                target[0], target[1], payload, timeout=timeout, connect_timeout=min(self.connect_timeout, timeout)
+            )
+        except ConnectFailed:
+            return failure("unreachable")
+        except TimeoutError:
+            return failure("timeout")
+        except Exception:
+            return failure("error")
+        finally:
+            self._slots.release()
+
+        if len(json.dumps(response)) > self.max_payload:
+            return failure("too_large")
+        return {"ok": True, "response": response}

colony/core/roles/gossip/role.py

@@ -73,7 +73,7 @@ class Gossip(Role):
 
         records = self._parse_records(known)
         self.membership.merge([record] + records)
-        return {"admitted": True, "members": self.membership.export()}
+        return {"admitted": True, "members": self.membership.export(), "responder": self.membership.self_record().to_dict()}
 
     def _parse_records(self, raw):
         if not isinstance(raw, list) or len(raw) > self.max_members:
@@ -152,8 +152,20 @@ class Gossip(Role):
         except ValueError as e:
             self.logger.warning(f"Ignoring bad member list from {ip}:{port}: {e}")
             return FAILED
+
+        self.remember_dial_address(result.get("responder"), (ip, port))
         return ADMITTED
 
+    def remember_dial_address(self, responder, address):
+        if not isinstance(responder, dict):
+            return
+        try:
+            key = MemberRecord.from_dict(responder).key
+        except ValueError:
+            return
+        if key != address:
+            self.context.set_dial_alias(key, address)
+
     def hardware(self):
         now = self.membership.now()
         if self._hardware_at is None or now - self._hardware_at >= self.HARDWARE_TTL:

colony/core/roles/unit/role.py

@@ -29,8 +29,14 @@ class Unit(Role):
 
     def show_cluster(self):
         roles = self.context.get_role_registry().roles
+        members = self.context.membership().overview()
+        for member in members:
+            key = (member["ip"], member["port"])
+            via = self.context.route_of(key)
+            member["via"] = list(via) if via else None
+            member["direct"] = self.context.is_direct(key)
         return {
-            "members": self.context.membership().overview(),
+            "members": members,
             "roles": [
                 {
                     "name": name,

colony/core/roles/webview/role.py

@@ -1,12 +1,19 @@
 import threading
+from concurrent.futures import ThreadPoolExecutor
 from enum import Enum
 
 from werkzeug.serving import make_server
 
 from colony.core.roles.role import Role
+from colony.core.roles.unit.role import Unit
 from colony.core.roles.webview.app import create_app
 from colony.settings import setting
 
+VIEW_INTERVAL = 5
+VIEW_TIMEOUT = 2
+VIEW_MAX_AGE = 20
+VIEW_WORKERS = 8
+
 
 class WebView(Role):
     class COMMANDS(Enum):
@@ -21,13 +28,75 @@ class WebView(Role):
             raise OSError(f"Web view cannot listen on {host}:{port}") from None
         self.port = self._server.server_port
 
+        self._views = {}
+        self._views_lock = threading.Lock()
+
         super().__init__({}, context)
 
-    def _status(self):
+    def _unit(self):
         unit = self.context.self().get_role_instances().get("Unit")
         if unit is None:
             raise RuntimeError("The Unit role is not available")
-        return unit.show_cluster()
+        return unit
+
+    def _status(self):
+        payload = self._unit().show_cluster()
+        payload["links"] = self._links(payload["members"])
+        return payload
+
+    def _links(self, own_rows):
+        own_key = next((member["ip"], member["port"]) for member in own_rows if member["self"])
+        views = self._fresh_views()
+        views[own_key] = own_rows
+
+        links = []
+        for source, rows in views.items():
+            for row in rows:
+                if row.get("self") or not row.get("alive") or "direct" not in row:
+                    continue
+                if row["direct"] or row.get("via"):
+                    links.append({"from": list(source), "to": [row["ip"], row["port"]], "via": None if row["direct"] else row["via"]})
+        return links
+
+    def _fresh_views(self):
+        now = self.context.membership().now()
+        with self._views_lock:
+            return {key: rows for key, (seen, rows) in self._views.items() if now - seen <= VIEW_MAX_AGE}
+
+    def _fetch_view(self, key):
+        try:
+            resp = self.context.execute_to_unregistered_node(key[0], key[1], Unit.COMMANDS.SHOW_CLUSTER, timeout=VIEW_TIMEOUT)
+        except Exception:
+            return None
+
+        result = resp.get("result") if isinstance(resp, dict) and resp.get("status") == "ok" else None
+        rows = result.get("members") if isinstance(result, dict) else None
+        if not isinstance(rows, list) or not all(isinstance(row, dict) for row in rows):
+            return None
+        return rows if any("direct" in row for row in rows) else None
+
+    def refresh_views(self):
+        targets = [
+            (member["ip"], member["port"]) for member in self._unit().show_cluster()["members"]
+            if member["alive"] and not member["self"]
+        ]
+        with ThreadPoolExecutor(max_workers=max(1, min(VIEW_WORKERS, len(targets)))) as pool:
+            results = list(pool.map(self._fetch_view, targets))
+
+        now = self.context.membership().now()
+        with self._views_lock:
+            for key, rows in zip(targets, results):
+                if rows is not None:
+                    self._views[key] = (now, rows)
+            self._views = {key: entry for key, entry in self._views.items() if key in targets}
+
+    def _refresh_loop(self):
+        while not self._stop_event.is_set():
+            try:
+                self.refresh_views()
+            except Exception:
+                self.logger.exception("Could not refresh the topology")
+            self._stop_event.wait(VIEW_INTERVAL)
 
     def start(self):
         with self._start_lock:
@@ -35,6 +104,7 @@ class WebView(Role):
                 return
             self._thread = threading.Thread(target=self._server.serve_forever, daemon=True, name=str(self))
             self._thread.start()
+            threading.Thread(target=self._refresh_loop, daemon=True, name="WebViewTopology").start()
 
         self.logger.info(f"Web view at http://{self.context.self().ip}:{self.port}")
 

colony/core/roles/webview/static/app.js

@@ -190,6 +190,51 @@ function chipLayout(labels, maxWidth, maxLines) {
   return chips;
 }
 
+function edgePoint(card, tx, ty) {
+  const dx = tx - card.cx;
+  const dy = ty - card.cy;
+  if (dx === 0 && dy === 0) return { x: card.cx, y: card.cy };
+  const scale = Math.min(dx !== 0 ? card.w / 2 / Math.abs(dx) : Infinity, dy !== 0 ? card.h / 2 / Math.abs(dy) : Infinity);
+  return { x: card.cx + dx * scale, y: card.cy + dy * scale };
+}
+
+function relayCounts(payload) {
+  const counts = new Map();
+  (payload.links || []).forEach((link) => {
+    if (link.via) {
+      const key = link.via[0] + ":" + link.via[1];
+      counts.set(key, (counts.get(key) || 0) + 1);
+    }
+  });
+  return counts;
+}
+
+function reachLinks(payload, cards) {
+  const direct = new Set();
+  payload.links.forEach((link) => {
+    if (!link.via) direct.add(link.from[0] + ":" + link.from[1] + ">" + link.to[0] + ":" + link.to[1]);
+  });
+
+  const links = [];
+  for (let i = 0; i < cards.length; i++) {
+    for (let j = i + 1; j < cards.length; j++) {
+      const a = cards[i];
+      const b = cards[j];
+      const forward = direct.has(a.key + ">" + b.key);
+      const backward = direct.has(b.key + ">" + a.key);
+      if (forward && backward) {
+        links.push({ x1: a.cx, y1: a.cy, x2: b.cx, y2: b.cy, kind: "link" });
+      } else if (forward || backward) {
+        const [from, to] = forward ? [a, b] : [b, a];
+        const start = edgePoint(from, to.cx, to.cy);
+        const end = edgePoint(to, from.cx, from.cy);
+        links.push({ x1: start.x, y1: start.y, x2: end.x, y2: end.y, kind: "oneway" });
+      }
+    }
+  }
+  return links;
+}
+
 function layout(payload, narrow) {
   const members = payload.members;
   const count = members.length;
@@ -242,9 +287,13 @@ function layout(payload, narrow) {
     const cy = -top;
 
     const alive = cards.filter((card) => card.member.alive);
-    for (let i = 0; i < alive.length; i++) {
+    if (Array.isArray(payload.links)) {
-      for (let j = i + 1; j < alive.length; j++) {
+      reachLinks(payload, alive).forEach((link) => links.push(link));
-        links.push({ x1: alive[i].cx, y1: alive[i].cy, x2: alive[j].cx, y2: alive[j].cy, kind: "link" });
+    } else {
+      for (let i = 0; i < alive.length; i++) {
+        for (let j = i + 1; j < alive.length; j++) {
+          links.push({ x1: alive[i].cx, y1: alive[i].cy, x2: alive[j].cx, y2: alive[j].cy, kind: "link" });
+        }
       }
     }
     const viewer = cards.find((card) => card.member.self);
@@ -259,7 +308,7 @@ function layout(payload, narrow) {
     }
   }
 
-  return { width: width, height: height, narrow: narrow, cards: cards, links: links, center: center };
+  return { width: width, height: height, narrow: narrow, cards: cards, links: links, center: center, relays: relayCounts(payload) };
 }
 
 if (typeof document !== "undefined") {
@@ -300,11 +349,13 @@ if (typeof document !== "undefined") {
     }
   };
 
-  const drawCard = (parent, card, payload) => {
+  const drawCard = (parent, card, payload, relays) => {
     const member = card.member;
     const stats = describeStats(member.stats);
+    const relayed = relays.get(card.key) || 0;
     const classes = ["node", member.alive ? "alive" : "down"];
     if (member.self) classes.push("self");
+    if (relayed) classes.push("bridge");
 
     const group = svg("g", { class: classes.join(" "), transform: "translate(" + card.x + "," + card.y + ")" });
     group.appendChild(svg("title", {}, member.hostname + " " + card.key + "\nroles: " + (member.roles.join(", ") || "none") +
@@ -324,7 +375,8 @@ if (typeof document !== "undefined") {
     drawBar(group, 104, card.w, "CPU", stats.cpu);
     drawBar(group, 120, card.w, "RAM", stats.ram);
     drawBar(group, 136, card.w, "DISK", stats.disk);
-    group.appendChild(svg("text", { class: "t-foot", x: 14, y: card.h - 8 }, shorten(stats.footer, 56)));
+    const foot = relayed ? stats.footer + " · relaying " + relayed : stats.footer;
+    group.appendChild(svg("text", { class: "t-foot", x: 14, y: card.h - 8 }, shorten(foot, 56)));
     parent.appendChild(group);
   };
 
@@ -346,11 +398,19 @@ if (typeof document !== "undefined") {
     graph.style.maxWidth = Math.round(geometry.width) + "px";
     graph.replaceChildren();
 
+    const defs = svg("defs");
+    const marker = svg("marker", { id: "arrow", viewBox: "0 0 10 10", refX: 9, refY: 5, markerWidth: 8, markerHeight: 8, orient: "auto" });
+    marker.appendChild(svg("path", { class: "arrowhead", d: "M 0 0 L 10 5 L 0 10 z" }));
+    defs.appendChild(marker);
+    graph.appendChild(defs);
+
     geometry.links.forEach((link) => {
-      graph.appendChild(svg("line", { class: "link " + link.kind, x1: link.x1, y1: link.y1, x2: link.x2, y2: link.y2 }));
+      const attributes = { class: "link " + link.kind, x1: link.x1, y1: link.y1, x2: link.x2, y2: link.y2 };
+      if (link.kind === "oneway") attributes["marker-end"] = "url(#arrow)";
+      graph.appendChild(svg("line", attributes));
     });
     if (geometry.center) drawCenter(graph, geometry.center);
-    geometry.cards.forEach((card) => drawCard(graph, card, payload));
+    geometry.cards.forEach((card) => drawCard(graph, card, payload, geometry.relays));
   };
 
   const drawHeader = (payload) => {
@@ -404,5 +464,5 @@ if (typeof document !== "undefined") {
 }
 
 if (typeof module !== "undefined") {
-  module.exports = { address, roleStates, layout, nodeRoles, chipLayout, chipWidth, shorten, formatAge, formatSize, formatUptime, describeStats, barLevel, alerts, summarize, onDemandHint };
+  module.exports = { address, roleStates, layout, nodeRoles, chipLayout, chipWidth, shorten, formatAge, formatSize, formatUptime, describeStats, barLevel, alerts, summarize, onDemandHint, edgePoint, relayCounts };
 }

colony/core/roles/webview/static/index.html

@@ -19,7 +19,9 @@
     <ul class="legend">
       <li><span class="swatch alive"></span>node alive</li>
       <li><span class="swatch down"></span>node down</li>
-      <li><span class="swatch link"></span>same network</li>
+      <li><span class="swatch link"></span>direct link</li>
+      <li><span class="swatch oneway"></span>one-way reach</li>
+      <li><span class="swatch bridge"></span>relays for others</li>
       <li><span class="swatch held"></span>role held</li>
       <li><span class="swatch unmet"></span>role unmet</li>
       <li><span class="swatch duplicate"></span>held twice</li>

colony/core/roles/webview/static/style.css

@@ -64,6 +64,9 @@ main { margin-top: 14px; }
 
 .link { stroke: var(--line); stroke-width: 2.5; }
 .link.down { stroke-dasharray: 6 6; opacity: 0.7; }
+.link.oneway { stroke: var(--muted); }
+.arrowhead { fill: var(--muted); }
+.node.bridge .card { stroke: var(--held); stroke-width: 2; }
 
 .card { fill: var(--card); stroke: var(--line); stroke-width: 1.5; }
 .node.self .card { stroke: var(--alive); stroke-width: 2; }
@@ -106,6 +109,8 @@ main { margin-top: 14px; }
 .swatch.down { background: var(--down); }
 .swatch.held, .swatch.unmet, .swatch.duplicate, .swatch.general { border-radius: 6px; background: transparent; border: 2px solid; }
 .swatch.general { border-color: var(--line); }
+.swatch.oneway { width: 18px; height: 0; border-radius: 0; border: 0; border-top: 2.5px solid var(--muted); vertical-align: 3px; }
+.swatch.bridge { border-radius: 4px; background: transparent; border: 2px solid var(--held); }
 .swatch.link { width: 18px; height: 0; border-radius: 0; border: 0; border-top: 2.5px solid var(--line); vertical-align: 3px; }
 .swatch.held { border-color: var(--held); }
 .swatch.unmet { border-color: var(--unmet); border-style: dashed; }

colony/settings.py

@@ -24,6 +24,21 @@ DEFAULTS = {
     "foreman.acquire_delay": 10,
     "foreman.proposal_stagger": 2,
     "foreman.proposal_cooldown": 30,
+    "gateway.client_fallback": True,
+    "gateway.connect_timeout": 2,
+    "gateway.direct_retry": 30,
+    "gateway.max_attempts": 4,
+    "gateway.forward_timeout": 8,
+    "gateway.max_concurrent": 16,
+    "gateway.max_payload_bytes": 4 * 1024 * 1024,
+    "gateway.allow_commands": [
+        "Gossip.Hello",
+        "Unit.ShowStats",
+        "Unit.ShowRoles",
+        "Unit.ShowHardware",
+        "Unit.ShowCluster",
+        "Unit.RequestRole",
+    ],
     "web.host": "0.0.0.0",
     "web.port": 8080,
     "role_policy.max_roles": None,

colony/transport/whisperer.py

@@ -6,6 +6,10 @@ from colony.settings import configuration
 from colony.transport.framing import send_frame, recv_frame
 
 
+class ConnectFailed(OSError):
+    pass
+
+
 class Whisperer:
     DEFAULT_TIMEOUT = 10
 
@@ -24,16 +28,22 @@ class Whisperer:
         context.load_cert_chain(certfile=self.cert_file, keyfile=self.keyfile)
         return context
 
-    def send(self, payload, timeout=DEFAULT_TIMEOUT):
+    def send(self, payload, timeout=DEFAULT_TIMEOUT, connect_timeout=None):
         data = json.dumps(payload).encode()
         context = self._build_ssl_context()
 
         try:
-            with socket.create_connection((self.host, self.port), timeout=timeout) as sock:
+            sock = socket.create_connection((self.host, self.port), timeout=connect_timeout or timeout)
+        except OSError as e:
+            raise ConnectFailed(f"Cannot connect to {self.host}:{self.port}: {e}") from e
+
+        with sock:
+            sock.settimeout(timeout)
+            try:
                 with context.wrap_socket(sock) as tls_sock:
                     send_frame(tls_sock, data)
                     response = recv_frame(tls_sock)
-        except socket.timeout:
+            except socket.timeout:
-            raise TimeoutError("No response before timeout. ")
+                raise TimeoutError("No response before timeout. ")
 
         return json.loads(response.decode())

config.json

@@ -38,6 +38,9 @@
     "host": "0.0.0.0",
     "port": 8080
   },
+  "gateway": {
+    "max_attempts": 4
+  },
   "role_policy": {
     "max_roles": null,
     "deny": [],

role_settings.py

@@ -2,6 +2,7 @@ from colony.core.roles.explorer.role import Explorer
 from colony.core.roles.unit.role import Unit
 from colony.role_registry import RoleStats, HardwarePreferences, RolePriority, HardwareType
 from colony.core.roles.foreman.role import Foreman
+from colony.core.roles.gateway.role import Gateway
 from colony.core.roles.gossip.role import Gossip
 from colony.core.roles.vaultwarden.role import VaultWarden
 from colony.core.roles.webview.role import WebView
@@ -43,4 +44,8 @@ available_roles = [
         ),
         on_demand=True,
     ),
+    RoleStats(
+        cls=Gateway,
+        priority=RolePriority.GENERAL,
+    ),
 ]