| 1 | package rollout |
| 2 | |
| 3 | import ( |
| 4 | "github.com/carlosframework/platform/internal/status" |
| 5 | ) |
| 6 | |
| 7 | // Status is the box's rollout report (spec §8.4, §9.1), for the private |
| 8 | // Metrics block. It reads the queue, the failure book and the config tracks |
| 9 | // and asks the activator only about a host's process (Owns, Current), with no |
| 10 | // Manager or scheduler lock held (the lock order on Manager): Manager.mu is |
| 11 | // released before any scheduler.mu is taken, the host names are gathered under |
| 12 | // each scheduler.mu and the activator is asked after it is released, and the |
| 13 | // book is read under a scheduler.mu as the order allows. The hosts' serving |
| 14 | // state is what the last agent pass and the last attempts recorded, except for |
| 15 | // a host whose process is starting or stopping (the activator owns it and has |
| 16 | // no current generation): that one is Transient, with no serving build, because |
| 17 | // the cached one says nothing settled (Astra R3 section 5 item 10; plan review |
| 18 | // finding 8). |
| 19 | func (m *Manager) Status() status.RolloutReport { |
| 20 | r := status.RolloutReport{Hosts: map[string]status.HostRollout{}, Apps: map[string]status.AppRollout{}} |
| 21 | for _, f := range m.d.Book.List() { |
| 22 | r.Failures = append(r.Failures, status.RolloutFailure{Account: f.Account, App: f.App, Build: f.Build, Host: f.Host, |
| 23 | Mode: f.Mode, Reason: reasonWord(f.Reason), PromotedAt: f.PromotedAt, At: f.At, HeldDown: f.HeldDown}) |
| 24 | } |
| 25 | m.mu.Lock() |
| 26 | scheds := make([]*scheduler, 0, len(m.apps)) |
| 27 | for _, s := range m.apps { |
| 28 | scheds = append(scheds, s) |
| 29 | } |
| 30 | observing := make(map[string]bool, len(m.observing)) |
| 31 | for h := range m.observing { // whichever wake's token: the host is settling |
| 32 | observing[h] = true |
| 33 | } |
| 34 | type cfgView struct { |
| 35 | pending bool |
| 36 | reason string |
| 37 | } |
| 38 | cfg := make(map[string]cfgView, len(m.cfg)) |
| 39 | for h, t := range m.cfg { |
| 40 | if t.delivered != "" && t.delivered != t.active { |
| 41 | v := cfgView{pending: true} |
| 42 | if t.lastErr != "" { |
| 43 | // A reason word, never the raw text (which can name paths). |
| 44 | v.reason = reasonWord(t.lastErr) |
| 45 | } |
| 46 | cfg[h] = v |
| 47 | } |
| 48 | } |
| 49 | m.mu.Unlock() |
| 50 | asked := m.askActivator(scheds) |
| 51 | for _, s := range scheds { |
| 52 | app := status.AppRollout{} |
| 53 | s.mu.Lock() |
| 54 | for host, h := range s.q.hosts { |
| 55 | hr := status.HostRollout{Account: s.key.acct, App: s.key.app, Installed: h.wantBuild, Serving: h.serving, |
| 56 | Good: h.good, Awake: h.serving != ""} |
| 57 | // A host the activator was not asked about joined between the name |
| 58 | // read and this loop: nothing says it is settled, so this report |
| 59 | // calls it transient and the next one asks. |
| 60 | if tr, ok := asked[host]; !ok || tr { |
| 61 | hr.Serving, hr.Awake, hr.Transient = "", false, true |
| 62 | } |
| 63 | if v, ok := cfg[host]; ok { |
| 64 | hr.ConfigPending, hr.ConfigError = true, v.reason |
| 65 | } |
| 66 | switch { |
| 67 | case h.running != nil && (h.running.Canary || observing[host]): |
| 68 | hr.Attempt, hr.AttemptBuild = status.AttemptSettling, h.running.ID.Build |
| 69 | app.Settling++ |
| 70 | case h.running != nil: |
| 71 | hr.Attempt, hr.AttemptBuild = status.AttemptRunning, h.running.ID.Build |
| 72 | app.Running++ |
| 73 | case observing[host]: |
| 74 | hr.Attempt, hr.AttemptBuild = status.AttemptSettling, h.wantBuild |
| 75 | app.Settling++ |
| 76 | default: |
| 77 | if d, ok := s.q.pending(h); ok { |
| 78 | hr.Attempt, hr.AttemptBuild, app.Queued = status.AttemptQueued, d.ID.Build, app.Queued+1 |
| 79 | } |
| 80 | } |
| 81 | // A build held for this instance, with nothing else going on: the |
| 82 | // failed words. A held-down instance is down. |
| 83 | if hr.Attempt == "" && h.wantBuild != "" && h.serving != h.wantBuild { |
| 84 | if rec, ok := m.d.Book.Lookup(s.key.acct, s.key.app, h.wantBuild); ok && |
| 85 | m.d.Book.Blocks(s.key.acct, s.key.app, h.wantBuild, host, h.wantBuild == h.good) { |
| 86 | hr.Attempt, hr.AttemptBuild, hr.HeldBy, hr.Reason = status.AttemptFailed, rec.Build, rec.Build, reasonWord(rec.Reason) |
| 87 | // The generation the console signs into its retry grant |
| 88 | // (plan review finding 13), 0 while the book owes a |
| 89 | // save (the crash-epoch hard limit, rule 2). |
| 90 | hr.HeldEpoch = m.d.Book.PublishedEpoch(s.key.acct, s.key.app, rec.Build, host) |
| 91 | hr.HeldBook = m.d.Book.ID() // final review round 1, finding 3 |
| 92 | hr.RetryV = status.HostRetryV // grants ride the host-retry log |
| 93 | if m.d.Book.HeldDown(s.key.acct, s.key.app, h.wantBuild, host) { |
| 94 | hr.Attempt = status.AttemptDown |
| 95 | } |
| 96 | } |
| 97 | } |
| 98 | // A hold build ran here without passing (final review wave 2, N2): |
| 99 | // the instance is held to it, in the same words. Asleep with a |
| 100 | // good build, its wakes are refused (decideHoldRan), so it is down |
| 101 | // when a record holds its installed build, the marked build |
| 102 | // included (Astra R3 N7), and when its installed build is the good |
| 103 | // build, until a signed rollback lifts the marker (ruling 205), |
| 104 | // unless the good build is the marked build itself, which its wake |
| 105 | // boots. Awake, it serves what it serves. |
| 106 | if mark, ok := holdRanFor(m.d.Book, s.key.acct, s.key.app, host); ok && h.serving == "" { |
| 107 | switch { |
| 108 | case hr.HeldBy != "" && h.good != "": |
| 109 | hr.Attempt = status.AttemptDown |
| 110 | case hr.Attempt == "": |
| 111 | hr.HeldBy = mark.Build |
| 112 | hr.HeldEpoch = m.d.Book.PublishedEpoch(s.key.acct, s.key.app, mark.Build, host) |
| 113 | hr.HeldBook = m.d.Book.ID() |
| 114 | hr.RetryV = status.HostRetryV |
| 115 | if rec, ok := m.d.Book.Lookup(s.key.acct, s.key.app, mark.Build); ok { |
| 116 | hr.Reason = reasonWord(rec.Reason) |
| 117 | } |
| 118 | // The marked build installed boots under its own rule, even |
| 119 | // when it is the good build (Task 3 fix round 1). |
| 120 | if h.good != "" && h.wantBuild == h.good && h.wantBuild != mark.Build { |
| 121 | hr.Attempt = status.AttemptDown |
| 122 | } |
| 123 | } |
| 124 | } |
| 125 | r.Hosts[host] = hr |
| 126 | } |
| 127 | s.mu.Unlock() |
| 128 | r.Apps[s.key.acct+"/"+s.key.app] = app |
| 129 | } |
| 130 | return r |
| 131 | } |
| 132 | |
| 133 | // askActivator answers, for each host among the schedulers' at the moment |
| 134 | // their names are read, whether its process is starting or stopping: the |
| 135 | // activator owns the host and has no current generation for it (a wake or an |
| 136 | // observation not yet current, a hibernation under way). A host the activator |
| 137 | // does not own is asleep, and one it has a current generation for is awake; |
| 138 | // both answer false. The names are read under each scheduler.mu and the |
| 139 | // activator is asked after it is released (scheduler.view's rule), so no |
| 140 | // Manager or scheduler lock is held across the ask. A host missing from the |
| 141 | // answer joined after the names were read, and Status reports it transient. |
| 142 | func (m *Manager) askActivator(scheds []*scheduler) map[string]bool { |
| 143 | var names []string |
| 144 | for _, s := range scheds { |
| 145 | s.mu.Lock() |
| 146 | for host := range s.q.hosts { |
| 147 | names = append(names, host) |
| 148 | } |
| 149 | s.mu.Unlock() |
| 150 | } |
| 151 | out := make(map[string]bool, len(names)) |
| 152 | for _, host := range names { |
| 153 | if g, ok := m.d.Act.Current(host); ok && g.Build != "" { |
| 154 | out[host] = false |
| 155 | continue |
| 156 | } |
| 157 | out[host] = m.d.Act.Owns(host) |
| 158 | } |
| 159 | return out |
| 160 | } |
| 161 | |
amadan