Stuff
This commit is contained in:
ciomek committed
1 parent
b954d9d448
commit
01f4c13954
22 files changed
+929
-170
No files matched your search
@@ -1,22 +1,14 @@
|
||||
from enum import Enum
|
||||
|
||||
from colony.core.network.hardware import HardwareStatistics, HardwareScorer
|
||||
from colony.core.network import advice
|
||||
from colony.core.roles.role import Role
|
||||
from colony.core.roles.unit.role import Unit
|
||||
from colony.role_registry import HardwareType, RolePriority
|
||||
|
||||
_HARDWARE_SCORE_KEY = {
|
||||
HardwareType.DISK_SPACE: "disk",
|
||||
HardwareType.CPU: "cpu",
|
||||
HardwareType.GPU: "gpu",
|
||||
HardwareType.RAM: "memory",
|
||||
HardwareType.LONGEVITY: "longevity",
|
||||
}
|
||||
from colony.settings import setting
|
||||
|
||||
|
||||
class Foreman(Role):
|
||||
CONTINUOUS = True
|
||||
LOOP_SECONDS_WAIT = 60
|
||||
LOOP_SECONDS_WAIT = 5
|
||||
|
||||
class COMMANDS(Enum):
|
||||
pass
|
||||
@@ -26,138 +18,64 @@ class Foreman(Role):
|
||||
|
||||
}
|
||||
|
||||
self.LOOP_SECONDS_WAIT = setting("foreman.interval")
|
||||
super().__init__(commands, context)
|
||||
|
||||
def _peers_for_assignment(self):
|
||||
self_node = self.context.self()
|
||||
by_key = {}
|
||||
for p in self.context.peers():
|
||||
by_key[(p.ip, p.port)] = p
|
||||
k = (self_node.ip, self_node.port)
|
||||
if k not in by_key:
|
||||
by_key[k] = self_node
|
||||
return [by_key[key] for key in sorted(by_key)]
|
||||
self.acquire_delay = setting("foreman.acquire_delay")
|
||||
self.proposal_stagger = setting("foreman.proposal_stagger")
|
||||
self.proposal_cooldown = setting("foreman.proposal_cooldown")
|
||||
|
||||
def _call(self, peer, command, *args):
|
||||
try:
|
||||
resp = peer.execute(command, *args)
|
||||
except Exception as e:
|
||||
self.logger.warning("%s failed for %s:%s: %s", command.value, peer.ip, peer.port, e)
|
||||
return None
|
||||
|
||||
if not isinstance(resp, dict) or resp.get("status") != "ok":
|
||||
self.logger.warning("%s failed for %s:%s", command.value, peer.ip, peer.port)
|
||||
return None
|
||||
return resp
|
||||
|
||||
def _fetch_live_roles(self, peers):
|
||||
live = {}
|
||||
for peer in peers:
|
||||
resp = self._call(peer, Unit.COMMANDS.SHOW_ROLES)
|
||||
if resp is None:
|
||||
continue
|
||||
if not isinstance(resp.get("result"), list):
|
||||
self.logger.warning("Bad role list from %s:%s, leaving it out of this round", peer.ip, peer.port)
|
||||
continue
|
||||
live[peer] = list(resp["result"])
|
||||
return live
|
||||
|
||||
def _assign_critical_holders(self, peers, critical_roles, live_roles):
|
||||
peer_list = list(peers)
|
||||
n = len(peer_list)
|
||||
holders = {}
|
||||
for i, critical_role in enumerate(critical_roles):
|
||||
current = next(
|
||||
(p for p in peer_list if critical_role in live_roles.get(p, ())),
|
||||
None,
|
||||
)
|
||||
if current is not None:
|
||||
holders[critical_role] = current
|
||||
else:
|
||||
holders[critical_role] = peer_list[i % n]
|
||||
return holders
|
||||
|
||||
def _fetch_hardware_scores(self, peers):
|
||||
scores = {}
|
||||
for peer in peers:
|
||||
resp = self._call(peer, Unit.COMMANDS.SHOW_HARDWARE)
|
||||
if resp is None:
|
||||
continue
|
||||
hw_obj = HardwareStatistics.from_dict(resp["result"])
|
||||
scores[peer] = HardwareScorer.from_object(hw_obj)
|
||||
return scores
|
||||
|
||||
def _best_peer_for_optional(self, peers, role_stat, hardware_scores):
|
||||
best_score = float("-inf")
|
||||
best_peer = None
|
||||
for peer in peers:
|
||||
hs = hardware_scores.get(peer)
|
||||
if hs is None:
|
||||
continue
|
||||
score = 0.0
|
||||
if role_stat.hardware_preferences:
|
||||
for hw_type, weight in role_stat.hardware_preferences.priorities.items():
|
||||
key = _HARDWARE_SCORE_KEY.get(hw_type)
|
||||
if key is not None:
|
||||
score += hs.get(key, 0) * weight
|
||||
if score > best_score:
|
||||
best_score = score
|
||||
best_peer = peer
|
||||
return best_peer
|
||||
|
||||
@staticmethod
|
||||
def _dedupe_preserve_order(role_names):
|
||||
seen = set()
|
||||
out = []
|
||||
for r in role_names:
|
||||
if r not in seen:
|
||||
seen.add(r)
|
||||
out.append(r)
|
||||
return out
|
||||
self._needed_since = {}
|
||||
self._last_proposal = {}
|
||||
|
||||
def loop(self):
|
||||
role_registry = self.context.get_role_registry()
|
||||
roles = role_registry.roles
|
||||
self.tick()
|
||||
|
||||
general_roles = sorted(
|
||||
name for name, stats in roles.items() if stats.priority == RolePriority.GENERAL
|
||||
)
|
||||
critical_roles = sorted(
|
||||
name for name, stats in roles.items() if stats.priority == RolePriority.CRITICAL
|
||||
)
|
||||
optional_roles = sorted(
|
||||
name
|
||||
for name, stats in roles.items()
|
||||
if stats.priority not in (RolePriority.GENERAL, RolePriority.CRITICAL)
|
||||
)
|
||||
def tick(self):
|
||||
membership = self.context.membership()
|
||||
if membership.denied:
|
||||
return None
|
||||
|
||||
peers = self._peers_for_assignment()
|
||||
if not peers:
|
||||
return
|
||||
now = membership.now()
|
||||
records = membership.alive_records()
|
||||
roles = self.context.get_role_registry().roles
|
||||
|
||||
live_roles = self._fetch_live_roles(peers)
|
||||
peers = [peer for peer in peers if peer in live_roles]
|
||||
if not peers:
|
||||
return
|
||||
unmet = advice.unmet_roles(records, roles)
|
||||
self._needed_since = {name: self._needed_since.get(name, now) for name in unmet}
|
||||
|
||||
critical_holders = self._assign_critical_holders(
|
||||
peers, critical_roles, live_roles
|
||||
)
|
||||
alone = len(records) == 1
|
||||
rank = advice.rank_of(membership.key, records)
|
||||
wait = 0 if alone else self.acquire_delay + min(rank, 3) * self.proposal_stagger
|
||||
|
||||
desired = {peer: list(general_roles) for peer in peers}
|
||||
for critical_role, holder in critical_holders.items():
|
||||
desired[holder].append(critical_role)
|
||||
for role_name in unmet:
|
||||
if now - self._needed_since[role_name] < wait:
|
||||
continue
|
||||
|
||||
hardware_scores = self._fetch_hardware_scores(peers) if optional_roles else {}
|
||||
for role_name in optional_roles:
|
||||
role_stat = roles[role_name]
|
||||
best_peer = self._best_peer_for_optional(peers, role_stat, hardware_scores)
|
||||
if best_peer is not None:
|
||||
desired[best_peer].append(role_name)
|
||||
target = advice.pick_candidate(role_name, roles[role_name], records)
|
||||
if target is None:
|
||||
self.logger.warning(f"No node is willing to take role {role_name}")
|
||||
continue
|
||||
|
||||
for peer in peers:
|
||||
self._call(
|
||||
peer,
|
||||
Unit.COMMANDS.SET_ROLES,
|
||||
self._dedupe_preserve_order(desired[peer]),
|
||||
last = self._last_proposal.get((role_name, target.key))
|
||||
if last is not None and now - last < self.proposal_cooldown:
|
||||
continue
|
||||
|
||||
self._last_proposal[(role_name, target.key)] = now
|
||||
return self.propose(role_name, target)
|
||||
|
||||
return None
|
||||
|
||||
def propose(self, role_name, target):
|
||||
self.logger.info(f"Asking {target.ip}:{target.port} to take role {role_name}")
|
||||
try:
|
||||
resp = self.context.execute_to_unregistered_node(
|
||||
target.ip, target.port, Unit.COMMANDS.REQUEST_ROLE, role_name
|
||||
)
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Could not ask {target.ip}:{target.port} for role {role_name}: {e}")
|
||||
return (role_name, target.key, None)
|
||||
|
||||
result = resp.get("result") if isinstance(resp, dict) and resp.get("status") == "ok" else None
|
||||
accepted = result.get("accepted") if isinstance(result, dict) else None
|
||||
self.logger.info(f"{target.ip}:{target.port} answered {role_name}: {result}")
|
||||
return (role_name, target.key, accepted)
|
||||
Reference in new issue
Block a user