387 lines
17 KiB
JavaScript
387 lines
17 KiB
JavaScript
"use strict";
|
||
|
||
/* Pure facts about the cluster: who is in which network, who relays for whom, how
|
||
one node reaches the others. No DOM, no geometry. Loaded by the page as a classic
|
||
script (functions land on globalThis.Colony) and by the tests through require. */
|
||
(function () {
|
||
const OTHER_NETWORK = "other addresses";
|
||
const COLORS = 6;
|
||
|
||
const address = (member) => member.ip + ":" + member.port;
|
||
const keyOf = (pair) => pair[0] + ":" + pair[1];
|
||
|
||
function networkOf(ip) {
|
||
const parts = String(ip).split(".");
|
||
const octets = parts.every((part) => /^\d{1,3}$/.test(part) && Number(part) <= 255);
|
||
return parts.length === 4 && octets ? parts.slice(0, 3).join(".") + ".0/24" : OTHER_NETWORK;
|
||
}
|
||
|
||
function compareNetworks(a, b) {
|
||
if (a === OTHER_NETWORK || b === OTHER_NETWORK) return (a === OTHER_NETWORK) - (b === OTHER_NETWORK);
|
||
return a.localeCompare(b, undefined, { numeric: true });
|
||
}
|
||
|
||
function compareAddresses(a, b) {
|
||
return a.ip.localeCompare(b.ip, undefined, { numeric: true }) || a.port - b.port;
|
||
}
|
||
|
||
function shorten(text, limit) {
|
||
return text.length > limit ? text.slice(0, limit - 1) + "…" : text;
|
||
}
|
||
|
||
/* Fill up to maxLines lines of at most limit characters with the items, then cut. */
|
||
function wrapLabels(items, limit, maxLines) {
|
||
const lines = [];
|
||
let line = "";
|
||
items.forEach((item) => {
|
||
const next = line ? line + " · " + item : item;
|
||
if (line && next.length > limit) {
|
||
lines.push(line);
|
||
line = item;
|
||
} else {
|
||
line = next;
|
||
}
|
||
});
|
||
if (line) lines.push(line);
|
||
const shown = lines.slice(0, maxLines);
|
||
if (lines.length > maxLines) shown[maxLines - 1] = shorten(shown[maxLines - 1] + " · …", limit);
|
||
return shown.map((text) => shorten(text, limit));
|
||
}
|
||
|
||
function roleStates(members, roles) {
|
||
return roles.map((role) => {
|
||
const holders = members.filter((node) => node.alive && node.roles.includes(role.name)).length;
|
||
const limit = role.max === undefined ? role.required : role.max;
|
||
let state = "held";
|
||
if (role.on_demand) state = holders ? "held" : "off";
|
||
else if (role.required === null) state = "general";
|
||
else if (holders < role.required) state = "unmet";
|
||
else if (limit !== null && holders > limit) state = "duplicate";
|
||
return { name: role.name, state: state };
|
||
});
|
||
}
|
||
|
||
/* What one node reported about how it reaches the others.
|
||
Full payloads list every reached pair in `links`; compact payloads (`routes`)
|
||
list only the relayed and the unreached members, everything else is direct. */
|
||
function readRoutes(payload, nodes) {
|
||
const reporters = new Map((payload.views || []).map((pair) => [keyOf(pair), { via: new Map(), direct: null, none: null }]));
|
||
|
||
if (Array.isArray(payload.routes)) {
|
||
reporters.forEach((entry) => (entry.none = new Set()));
|
||
payload.routes.forEach((route) => {
|
||
const from = nodes[route.from];
|
||
const entry = from && reporters.get(from.key);
|
||
if (!entry) return;
|
||
(route.none || []).forEach((index) => nodes[index] && entry.none.add(nodes[index].key));
|
||
Object.keys(route.via || {}).forEach((gateway) => {
|
||
if (!nodes[gateway]) return;
|
||
route.via[gateway].forEach((index) => nodes[index] && entry.via.set(nodes[index].key, nodes[gateway].key));
|
||
});
|
||
});
|
||
} else {
|
||
reporters.forEach((entry) => (entry.direct = new Set()));
|
||
(payload.links || []).forEach((link) => {
|
||
const entry = reporters.get(keyOf(link.from));
|
||
if (!entry) return;
|
||
if (link.via) entry.via.set(keyOf(link.to), keyOf(link.via));
|
||
else entry.direct.add(keyOf(link.to));
|
||
});
|
||
}
|
||
return reporters;
|
||
}
|
||
|
||
const memo = new WeakMap();
|
||
|
||
/* Everything the page needs from a payload, computed once per payload. */
|
||
function analyze(payload) {
|
||
if (memo.has(payload)) return memo.get(payload);
|
||
|
||
const nodes = payload.members.map((member, index) => ({
|
||
index: index,
|
||
key: address(member),
|
||
net: networkOf(member.ip),
|
||
id: member.hostname + "@" + member.ip,
|
||
hostname: member.hostname,
|
||
ip: member.ip,
|
||
port: member.port,
|
||
alive: Boolean(member.alive),
|
||
self: Boolean(member.self),
|
||
roles: member.roles,
|
||
declined: member.declined || [],
|
||
stats: member.stats || {},
|
||
}));
|
||
const byKey = new Map(nodes.map((node) => [node.key, node]));
|
||
const states = roleStates(nodes, payload.roles);
|
||
const reporters = readRoutes(payload, nodes);
|
||
|
||
const route = (from, to) => {
|
||
const target = byKey.get(to);
|
||
if (!target || !target.alive) return null;
|
||
const entry = reporters.get(from);
|
||
if (!entry) return { kind: "unknown", via: null };
|
||
const gateway = entry.via.get(to);
|
||
if (gateway !== undefined) return byKey.has(gateway) && byKey.get(gateway).alive ? { kind: "via", via: gateway } : { kind: "none", via: null };
|
||
const direct = entry.direct ? entry.direct.has(to) : !entry.none.has(to);
|
||
return { kind: direct ? "direct" : "none", via: null };
|
||
};
|
||
|
||
const model = {
|
||
payload: payload,
|
||
nodes: nodes,
|
||
byKey: byKey,
|
||
selfKey: (nodes.find((node) => node.self) || {}).key || null,
|
||
states: states,
|
||
general: new Set(states.filter((role) => role.state === "general").map((role) => role.name)),
|
||
reporters: reporters,
|
||
route: route,
|
||
networks: Array.from(new Set(nodes.map((node) => node.net))).sort(compareNetworks),
|
||
bridges: [],
|
||
bridgeAt: new Map(),
|
||
signature: JSON.stringify([
|
||
nodes.map((node) => [node.key, node.hostname, node.alive, node.self, node.roles, node.declined]),
|
||
payload.roles,
|
||
payload.views,
|
||
payload.routes || payload.links,
|
||
]),
|
||
};
|
||
model.bridges = bridgesOf(model);
|
||
model.bridgeAt = new Map(model.bridges.map((bridge) => [bridge.key, bridge]));
|
||
memo.set(payload, model);
|
||
return model;
|
||
}
|
||
|
||
/* A gateway is a node that really relays something for somebody, never one that only
|
||
holds the role: every node holds it. */
|
||
function bridgesOf(model) {
|
||
const found = new Map();
|
||
const counted = new Set();
|
||
|
||
model.reporters.forEach((entry, from) => {
|
||
entry.via.forEach((gateway, to) => {
|
||
const via = model.byKey.get(gateway);
|
||
const target = model.byKey.get(to);
|
||
const source = model.byKey.get(from);
|
||
if (!via || !target || !source || !via.alive || !target.alive) return;
|
||
|
||
const pair = gateway + "|" + [from, to].sort().join(">");
|
||
if (counted.has(pair)) return;
|
||
counted.add(pair);
|
||
|
||
if (!found.has(gateway)) found.set(gateway, { key: gateway, hostname: via.hostname, home: via.net, pairs: 0, touches: new Set([via.net]) });
|
||
const bridge = found.get(gateway);
|
||
bridge.pairs += 1;
|
||
bridge.touches.add(source.net);
|
||
bridge.touches.add(target.net);
|
||
});
|
||
});
|
||
|
||
return Array.from(found.values()).sort((a, b) => a.key.localeCompare(b.key, undefined, { numeric: true }));
|
||
}
|
||
|
||
function resolveView(model, wanted) {
|
||
const alive = model.nodes.filter((node) => node.alive);
|
||
if (wanted && alive.some((node) => node.key === wanted)) return wanted;
|
||
return model.selfKey || (alive.length ? alive[0].key : null);
|
||
}
|
||
|
||
/* How one node reaches every other alive node. */
|
||
function routesFrom(model, viewKey) {
|
||
const byKey = new Map();
|
||
const counts = { direct: 0, none: 0 };
|
||
const relayed = new Map();
|
||
const reported = model.reporters.has(viewKey);
|
||
|
||
model.nodes.forEach((node) => {
|
||
if (node.key === viewKey || !node.alive) return;
|
||
const route = model.route(viewKey, node.key);
|
||
byKey.set(node.key, route);
|
||
if (route.kind === "direct") counts.direct += 1;
|
||
else if (route.kind === "none") counts.none += 1;
|
||
else if (route.kind === "via") relayed.set(route.via, (relayed.get(route.via) || 0) + 1);
|
||
});
|
||
|
||
return { viewKey: viewKey, reported: reported, byKey: byKey, counts: counts, relayed: relayed };
|
||
}
|
||
|
||
function routeSummary(model, routes) {
|
||
const viewer = model.byKey.get(routes.viewKey);
|
||
if (!routes.reported) return [{ kind: "unknown", text: "no report from " + (viewer ? viewer.hostname : "it") + " yet" }];
|
||
|
||
const parts = [];
|
||
if (routes.counts.direct) parts.push({ kind: "direct", text: routes.counts.direct + " direct" });
|
||
model.bridges.forEach((bridge) => {
|
||
if (routes.relayed.has(bridge.key)) parts.push({ kind: "via", network: bridge.home, text: routes.relayed.get(bridge.key) + " via " + bridge.hostname });
|
||
});
|
||
if (routes.counts.none) parts.push({ kind: "none", text: routes.counts.none + " no recent contact" });
|
||
return parts;
|
||
}
|
||
|
||
/* The roles worth naming on the map: everything except the ones every node runs. */
|
||
function specialRoles(model, node) {
|
||
return node.roles.filter((name) => !model.general.has(name)).sort();
|
||
}
|
||
|
||
function alerts(model) {
|
||
const named = (state) => model.states.filter((role) => role.state === state).map((role) => role.name);
|
||
const down = model.nodes.filter((node) => !node.alive).map((node) => node.hostname);
|
||
const out = [];
|
||
if (named("unmet").length) out.push({ level: "bad", text: "Unmet: " + named("unmet").join(", ") });
|
||
if (named("duplicate").length) out.push({ level: "warn", text: "Held twice: " + named("duplicate").join(", ") });
|
||
if (down.length) out.push({ level: "info", text: "Down: " + down.join(", ") });
|
||
return out;
|
||
}
|
||
|
||
function onDemandHint(model) {
|
||
const off = model.states.filter((role) => role.state === "off").map((role) => role.name);
|
||
return off.length ? "Not running (on demand): " + off.join(", ") + " · start one with: colonyctl role add " + off[0] : "";
|
||
}
|
||
|
||
function chatState(model, key) {
|
||
const peer = model.byKey.get(key);
|
||
if (!peer || peer.self) return null;
|
||
const own = model.byKey.get(model.selfKey);
|
||
if (!own || !own.roles.includes("Chatter")) {
|
||
return { read: false, send: false, text: "Chat needs Chatter on this node too. Start it with: colonyctl role add Chatter" };
|
||
}
|
||
if (!peer.roles.includes("Chatter")) return { read: true, send: false, text: peer.hostname + " is not running Chatter, so it cannot receive messages." };
|
||
if (!peer.alive) return { read: true, send: false, text: peer.hostname + " is down." };
|
||
return { read: true, send: true, text: "" };
|
||
}
|
||
|
||
function summarize(model) {
|
||
const alive = model.nodes.filter((node) => node.alive).length;
|
||
const down = model.nodes.length - alive;
|
||
const assigned = model.states.filter((role) => role.state !== "general" && role.state !== "off");
|
||
const held = assigned.filter((role) => role.state !== "unmet");
|
||
let text = alive + " node" + (alive === 1 ? "" : "s") + " alive";
|
||
if (model.networks.length > 1) text += " · " + model.networks.length + " networks";
|
||
if (down) text += " · " + down + " down";
|
||
return text + " · " + held.length + "/" + assigned.length + " roles held";
|
||
}
|
||
|
||
/* ------------------------------------------------------------------ hardware */
|
||
|
||
function formatSize(gb) {
|
||
if (gb >= 1000) return (gb / 1024).toFixed(1) + " TB";
|
||
if (gb >= 100) return Math.round(gb) + " GB";
|
||
return gb.toFixed(1) + " GB";
|
||
}
|
||
|
||
function formatUptime(hours) {
|
||
if (hours < 1) return Math.round(hours * 60) + "m";
|
||
if (hours < 48) return Math.round(hours) + "h";
|
||
return Math.floor(hours / 24) + "d " + Math.round(hours % 24) + "h";
|
||
}
|
||
|
||
const percent = (used, total) => (total > 0 ? Math.max(0, Math.min(100, Math.round((100 * used) / total))) : null);
|
||
const barLevel = (pct) => (pct >= 90 ? "crit" : pct >= 75 ? "warn" : "ok");
|
||
|
||
/* What one node reports about itself, ready to draw: three bars and a footer. */
|
||
function describeStats(stats) {
|
||
stats = stats || {};
|
||
const num = (key) => (typeof stats[key] === "number" ? stats[key] : null);
|
||
const usage = (usedKey, totalKey) => {
|
||
const used = num(usedKey);
|
||
const total = num(totalKey);
|
||
return used === null || total === null ? null : { pct: percent(used, total), text: formatSize(used) + " / " + formatSize(total) };
|
||
};
|
||
|
||
const load = num("cpu_percent");
|
||
const cores = num("cpu_cores");
|
||
const threads = num("cpu_threads");
|
||
let cpu = null;
|
||
if (load !== null) {
|
||
let text = Math.round(load) + "%";
|
||
if (cores && threads) text += " · " + cores + "C/" + threads + "T";
|
||
else if (threads) text += " · " + threads + "T";
|
||
cpu = { pct: Math.round(load), text: text };
|
||
}
|
||
const gpuCount = num("gpu_count");
|
||
const gpuUsage = usage("gpu_mem_used_gb", "gpu_mem_total_gb");
|
||
const uptime = num("uptime_hours");
|
||
|
||
const ram = usage("mem_used_gb", "mem_total_gb");
|
||
const disk = usage("disk_used_gb", "disk_total_gb");
|
||
const gpu = gpuCount && gpuUsage ? (gpuCount > 1 ? gpuCount + "× " : "") + "GPU " + gpuUsage.text : null;
|
||
const footer = [uptime === null ? null : "up " + formatUptime(uptime), gpu].filter(Boolean).join(" · ");
|
||
return { rows: [["CPU", cpu], ["RAM", ram], ["DISK", disk]], footer: cpu || ram || disk ? footer : "no stats reported" };
|
||
}
|
||
|
||
/* Per machine these are the same for every node on it; the "used" ones move and are averaged. */
|
||
const TOTAL_KEYS = ["cpu_cores", "cpu_threads", "mem_total_gb", "disk_total_gb", "gpu_count", "gpu_mem_total_gb"];
|
||
const USED_KEYS = ["mem_used_gb", "disk_used_gb", "gpu_mem_used_gb"];
|
||
|
||
const plural = (count, word) => count + " " + word + (count === 1 ? "" : "s");
|
||
|
||
/* "CPU 9% · 6C/12T, RAM ..." for the rows that have numbers. */
|
||
const describeRows = (rows) => rows.filter(([, usage]) => usage).map(([name, usage]) => name + " " + usage.text);
|
||
|
||
/* The hardware of one network added up. Nodes that report the same complete totals are one
|
||
machine (containers on a host all see the host), so a machine is counted once and its
|
||
numbers are the average of what its nodes report. Load is weighted by threads. */
|
||
function networkStats(model, net) {
|
||
const reporting = model.nodes.filter((node) => node.net === net && node.alive && Object.keys(node.stats).length);
|
||
const machines = new Map();
|
||
reporting.forEach((node) => {
|
||
const s = node.stats;
|
||
const complete = typeof s.mem_total_gb === "number" && typeof s.disk_total_gb === "number";
|
||
const fingerprint = complete ? TOTAL_KEYS.map((key) => s[key]).join("|") : node.key;
|
||
if (!machines.has(fingerprint)) machines.set(fingerprint, []);
|
||
machines.get(fingerprint).push(s);
|
||
});
|
||
|
||
const numbers = (list, key) => list.map((s) => s[key]).filter((value) => typeof value === "number");
|
||
const mean = (list, key) => {
|
||
const values = numbers(list, key);
|
||
return values.length ? values.reduce((a, b) => a + b, 0) / values.length : null;
|
||
};
|
||
const stats = {};
|
||
const add = (key, value) => {
|
||
if (typeof value === "number") stats[key] = (stats[key] || 0) + value;
|
||
};
|
||
let weighted = 0;
|
||
let weight = 0;
|
||
machines.forEach((list) => {
|
||
TOTAL_KEYS.forEach((key) => add(key, numbers(list, key)[0]));
|
||
USED_KEYS.forEach((key) => add(key, mean(list, key)));
|
||
const load = mean(list, "cpu_percent");
|
||
if (load !== null) {
|
||
const threads = numbers(list, "cpu_threads")[0] || 1;
|
||
weighted += load * threads;
|
||
weight += threads;
|
||
}
|
||
});
|
||
if (weight) stats.cpu_percent = weighted / weight;
|
||
return Object.assign(describeStats(stats), { machines: machines.size, nodes: reporting.length });
|
||
}
|
||
|
||
/* The full text of a node: tooltip and screen reader. */
|
||
function describeNode(model, node, routes) {
|
||
const lines = [node.hostname + " " + node.key + (node.self ? " (this node)" : "") + (node.alive ? "" : " (down)")];
|
||
const bridge = model.bridgeAt.get(node.key);
|
||
if (bridge) lines.push("gateway: relays " + plural(bridge.pairs, "node pair"));
|
||
lines.push("roles: " + (node.roles.join(", ") || "none"));
|
||
const hardware = describeStats(node.stats);
|
||
lines.push(describeRows(hardware.rows).concat(hardware.footer || []).join(" · "));
|
||
if (node.declined.length) lines.push("declined: " + node.declined.join(", "));
|
||
|
||
const route = routes && node.alive && node.key !== routes.viewKey ? routes.byKey.get(node.key) : null;
|
||
if (route) {
|
||
const from = (model.byKey.get(routes.viewKey) || {}).hostname || "the viewpoint";
|
||
const via = route.via && model.byKey.get(route.via);
|
||
lines.push("from " + from + ": " + {
|
||
direct: "reached directly",
|
||
via: "reached through " + (via ? via.hostname : route.via),
|
||
none: "no contact in the last 40 s (not necessarily unreachable)",
|
||
unknown: "no report from " + from + " yet",
|
||
}[route.kind]);
|
||
}
|
||
return lines.join("\n");
|
||
}
|
||
|
||
const api = { describeStats, describeRows, networkStats, plural, barLevel, OTHER_NETWORK, COLORS, address, keyOf, networkOf, compareNetworks, compareAddresses, shorten, wrapLabels, analyze, bridgesOf, resolveView, routesFrom, routeSummary, specialRoles, alerts, onDemandHint, chatState, summarize, describeNode };
|
||
if (typeof module !== "undefined") module.exports = api;
|
||
else globalThis.Colony = Object.assign(globalThis.Colony || {}, api);
|
||
})();
|