web view

colony/core/roles/foreman/role.py

@@ -22,7 +22,7 @@ class Foreman(Role):
         self.proposal_cooldown = setting("foreman.proposal_cooldown")
 
         self._needed_since = {}
-        self._last_proposal = {}
+        self._proposals = {}
 
     def loop(self):
         self.tick()
@@ -39,6 +39,11 @@ class Foreman(Role):
         unmet = advice.unmet_roles(records, roles)
         self._needed_since = {name: self._needed_since.get(name, now) for name in unmet}
 
+        self._proposals = {
+            key: (at, accepted) for key, (at, accepted) in self._proposals.items()
+            if now - at < self.proposal_cooldown
+        }
+
         alone = len(records) == 1
         rank = advice.rank_of(membership.key, records)
         wait = 0 if alone else self.acquire_delay + min(rank, 3) * self.proposal_stagger
@@ -47,17 +52,19 @@ class Foreman(Role):
             if now - self._needed_since[role_name] < wait:
                 continue
 
-            target = advice.pick_candidate(role_name, roles[role_name], records)
+            recent = {key: accepted for (name, key), (_, accepted) in self._proposals.items() if name == role_name}
+            if any(recent.values()):
+                continue
+
+            target = advice.pick_candidate(role_name, roles[role_name], records, exclude=set(recent))
             if target is None:
-                self.logger.warning(f"No node is willing to take role {role_name}")
+                if not recent:
+                    self.logger.warning(f"No node is willing to take role {role_name}")
                 continue
 
-            last = self._last_proposal.get((role_name, target.key))
-            if last is not None and now - last < self.proposal_cooldown:
-                continue
-
-            self._last_proposal[(role_name, target.key)] = now
-            return self.propose(role_name, target)
+            result = self.propose(role_name, target)
+            self._proposals[(role_name, target.key)] = (now, result[2] is True)
+            return result
 
         return None