| 1 | package console |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "errors" |
| 7 | "os" |
| 8 | |
| 9 | "github.com/carlosframework/platform/internal/hostclaim" |
| 10 | "github.com/carlosframework/platform/internal/hostops" |
| 11 | "github.com/carlosframework/platform/internal/provision" |
| 12 | ) |
| 13 | |
| 14 | type ClaimPlacementCoordinator struct { |
| 15 | Server *Server |
| 16 | Controller hostops.Controller |
| 17 | Authority *hostops.ProductionAuthority |
| 18 | Owner hostops.RegistrationOwner |
| 19 | } |
| 20 | |
| 21 | type metadataPlacementHost struct { |
| 22 | Host string `json:"host"` |
| 23 | Incarnation string `json:"incarnation"` |
| 24 | Executor string `json:"executor"` |
| 25 | Stamp uint64 `json:"stamp"` |
| 26 | RecordETag string `json:"record_etag"` |
| 27 | ClaimETag string `json:"claim_etag"` |
| 28 | ClaimGen uint64 `json:"claim_gen"` |
| 29 | } |
| 30 | |
| 31 | func (p *ClaimPlacementCoordinator) ready() error { |
| 32 | if p.Authority == nil || p.Controller.Control == nil || p.Owner == (hostops.RegistrationOwner{}) { |
| 33 | return hostops.ErrContract |
| 34 | } |
| 35 | return nil |
| 36 | } |
| 37 | |
| 38 | func (p *ClaimPlacementCoordinator) Start(ctx context.Context, account, app, target, by string) (string, error) { |
| 39 | if err := p.ready(); err != nil { |
| 40 | return "", err |
| 41 | } |
| 42 | acct, err := p.Server.store.AccountBySqid(account) |
| 43 | if err != nil { |
| 44 | return "", err |
| 45 | } |
| 46 | if acct == nil { |
| 47 | return "", hostops.ErrIdentity |
| 48 | } |
| 49 | appID, err := p.Server.store.ClaimAppID(acct.ID, app) |
| 50 | if err != nil { |
| 51 | return "", err |
| 52 | } |
| 53 | a, err := hostops.ReadAdmission(ctx, p.Controller.Store, account, app, appID) |
| 54 | if err != nil { |
| 55 | return "", err |
| 56 | } |
| 57 | if a.Value.Op != nil { |
| 58 | if a.Value.Op.Kind != hostops.OpPlacing || a.Value.Op.Intent["target"] != target { |
| 59 | return "", hostops.ErrPending |
| 60 | } |
| 61 | return a.Value.Op.ID, p.Resume(ctx, account, app, appID, a.Value.Op.ID) |
| 62 | } |
| 63 | source, err := p.Server.store.AppPlacement(acct.ID, app) |
| 64 | if err != nil { |
| 65 | return "", err |
| 66 | } |
| 67 | id := p.Controller.NewID() |
| 68 | op := hostops.AppOperation{ID: id, Kind: hostops.OpPlacing, AppID: appID, Phase: "draining", Intent: map[string]string{"source": source, "target": target, "by": by}} |
| 69 | if _, err = p.Controller.Control.BeginExclusive(ctx, account, app, appID, op, p.Owner); err != nil { |
| 70 | return "", err |
| 71 | } |
| 72 | // Resume binds the SQL operation fence as its first effect. |
| 73 | return id, p.Resume(ctx, account, app, appID, id) |
| 74 | } |
| 75 | |
| 76 | // sqlEffect runs one app SQL write as an effect of the placing operation; |
| 77 | // the SQL gate admits it on that effect's permit. |
| 78 | func (p *ClaimPlacementCoordinator) sqlEffect(ctx context.Context, h *hostops.OperationHandle, accountID int64, action string, fn func(*Store) error) error { |
| 79 | id, err := claimEffectID() |
| 80 | if err != nil { |
| 81 | return err |
| 82 | } |
| 83 | scope := claimSQLScope{AccountID: accountID, App: h.App(), AppID: h.AppID()} |
| 84 | var localErr error |
| 85 | _, err = h.Runner(p.Controller.Now).Run(ctx, hostops.Effect{ID: id, Resource: "app-sql:" + action, Action: action}, func(ctx context.Context) (string, error) { |
| 86 | // SQL has joined before it returns. Preserve its error for the caller, |
| 87 | // but do not strand a remote effect for a local, replayable projection. |
| 88 | localErr = p.Server.store.WithAppMutation(ctx, []sqlGrant{permitGrant(ctx, p.Controller.Store, h.Account(), scope)}, fn) |
| 89 | return "joined", nil |
| 90 | }) |
| 91 | return errors.Join(localErr, err) |
| 92 | } |
| 93 | |
| 94 | func (p *ClaimPlacementCoordinator) Resume(ctx context.Context, account, app, appID, id string) error { |
| 95 | if err := p.ready(); err != nil { |
| 96 | return err |
| 97 | } |
| 98 | h, err := p.Controller.Control.ResumeExclusive(ctx, account, app, appID, id, hostops.OpPlacing, p.Owner) |
| 99 | if err != nil { |
| 100 | return err |
| 101 | } |
| 102 | ctx = h.Lifecycle(ctx) |
| 103 | a, err := hostops.ReadAdmission(ctx, p.Controller.Store, account, app, appID) |
| 104 | if err != nil { |
| 105 | return err |
| 106 | } |
| 107 | acct, err := p.Server.store.AccountBySqid(account) |
| 108 | if err != nil { |
| 109 | return err |
| 110 | } |
| 111 | if acct == nil { |
| 112 | return hostops.ErrIdentity |
| 113 | } |
| 114 | op := a.Value.Op |
| 115 | if op == nil || op.ID != id { |
| 116 | return hostops.ErrPending |
| 117 | } |
| 118 | if op.Phase != "releasing" { |
| 119 | revision := a.Value.Revision |
| 120 | if err = p.sqlEffect(ctx, h, acct.ID, "bind-operation", func(st *Store) error { |
| 121 | return st.BindClaimAppOperation(acct.ID, app, appID, id, revision) |
| 122 | }); err != nil { |
| 123 | return err |
| 124 | } |
| 125 | } |
| 126 | var hosts []metadataPlacementHost |
| 127 | switch op.Phase { |
| 128 | case "draining": |
| 129 | if err = h.Drain(ctx, p.Controller.Now(), p.Controller.RecoverCreate); err != nil { |
| 130 | return err |
| 131 | } |
| 132 | // The census is keyed by this operation's reservation: every roster |
| 133 | // box publishes its report for it. Until each one has, the owned |
| 134 | // operation is waiting, not failed (an empty app included: a box |
| 135 | // may still serve a stale installation of it). |
| 136 | census, err := p.Authority.Census(ctx, account, app, id) |
| 137 | if errors.Is(err, os.ErrNotExist) { |
| 138 | return errors.Join(hostops.ErrPending, err) |
| 139 | } |
| 140 | if err != nil { |
| 141 | return err |
| 142 | } |
| 143 | byHost := map[string]metadataPlacementHost{} |
| 144 | for _, record := range census.Records { |
| 145 | if record.Holder.Family != "instance" { |
| 146 | return p.cancelUnsupported(ctx, h, acct.ID) |
| 147 | } |
| 148 | wire, err := provision.DecodeRecord(record.Body) |
| 149 | if err != nil { |
| 150 | return err |
| 151 | } |
| 152 | if len(wire.Replicas) != 0 || wire.Region != op.Intent["target"] { |
| 153 | return p.cancelUnsupported(ctx, h, acct.ID) |
| 154 | } |
| 155 | obs, err := hostclaim.Read(ctx, p.Controller.Store, record.Holder.Host) |
| 156 | if err != nil { |
| 157 | return err |
| 158 | } |
| 159 | incarnation, err := record.Incarnation() |
| 160 | if err != nil { |
| 161 | return err |
| 162 | } |
| 163 | if obs.Absent || obs.Claim.Holder == nil || obs.Claim.Holder.Incarnation != incarnation || obs.Claim.Op.Kind != hostclaim.KindServing || !obs.Claim.Touched.Complete { |
| 164 | return hostops.ErrPending |
| 165 | } |
| 166 | if len(obs.Claim.Op.Participants) != 1 || obs.Claim.Op.Exec != obs.Claim.Op.Participants[0] { |
| 167 | return p.cancelUnsupported(ctx, h, acct.ID) |
| 168 | } |
| 169 | byHost[obs.Host] = metadataPlacementHost{Host: obs.Host, Incarnation: incarnation, Executor: obs.Claim.Op.Exec, Stamp: obs.Claim.Holder.Stamp, RecordETag: record.ETag, ClaimETag: obs.ETag, ClaimGen: obs.Claim.Gen} |
| 170 | } |
| 171 | seen := map[string]bool{} |
| 172 | for _, report := range census.Reports { |
| 173 | for _, installation := range report.Installations { |
| 174 | if installation.Account != account || installation.App != app { |
| 175 | continue |
| 176 | } |
| 177 | ph, ok := byHost[installation.Host] |
| 178 | if !ok || installation.Incarnation != ph.Incarnation || installation.Stamp != ph.Stamp || report.Box != ph.Executor || installation.Placement != op.Intent["target"] || seen[ph.Host] { |
| 179 | return p.cancelUnsupported(ctx, h, acct.ID) |
| 180 | } |
| 181 | seen[ph.Host] = true |
| 182 | } |
| 183 | } |
| 184 | for host, ph := range byHost { |
| 185 | if !seen[host] { |
| 186 | return hostops.ErrPending |
| 187 | } |
| 188 | hosts = append(hosts, ph) |
| 189 | } |
| 190 | raw, err := json.Marshal(hosts) |
| 191 | if err != nil { |
| 192 | return err |
| 193 | } |
| 194 | return h.SetPhase(ctx, "draining", "claimed", map[string]string{"hosts": string(raw)}) |
| 195 | case "claimed": |
| 196 | if err = json.Unmarshal([]byte(op.Intent["hosts"]), &hosts); err != nil { |
| 197 | return err |
| 198 | } |
| 199 | for _, ph := range hosts { |
| 200 | obs, err := hostclaim.Read(ctx, p.Controller.Store, ph.Host) |
| 201 | if err != nil { |
| 202 | return err |
| 203 | } |
| 204 | if obs.Absent || obs.Claim.Holder == nil || obs.Claim.Holder.Incarnation != ph.Incarnation || obs.Claim.Holder.Stamp != ph.Stamp || obs.Claim.Op.Exec != ph.Executor { |
| 205 | return hostops.ErrPending |
| 206 | } |
| 207 | if obs.Claim.Op.Kind == hostclaim.KindPlacing && obs.Claim.Op.ID == id { |
| 208 | continue |
| 209 | } |
| 210 | if obs.ETag != ph.ClaimETag || obs.Claim.Gen != ph.ClaimGen { |
| 211 | // A cancelled unused hold invalidated the persisted observation: |
| 212 | // re-save it for this same incarnation only. |
| 213 | if obs.Claim.Op.Kind != hostclaim.KindServing { |
| 214 | return hostops.ErrPending |
| 215 | } |
| 216 | for i := range hosts { |
| 217 | if hosts[i].Host == ph.Host { |
| 218 | hosts[i].ClaimETag, hosts[i].ClaimGen = obs.ETag, obs.Claim.Gen |
| 219 | } |
| 220 | } |
| 221 | raw, err := json.Marshal(hosts) |
| 222 | if err != nil { |
| 223 | return err |
| 224 | } |
| 225 | return h.SetPhase(ctx, "claimed", "claimed", map[string]string{"hosts": string(raw)}) |
| 226 | } |
| 227 | if _, err = p.Authority.StartPlacementClaim(ctx, obs, account, app, appID, id, op.Intent["by"], op.Intent["target"]); err != nil { |
| 228 | return err |
| 229 | } |
| 230 | } |
| 231 | for _, ph := range hosts { |
| 232 | obs, err := hostclaim.Read(ctx, p.Controller.Store, ph.Host) |
| 233 | if err != nil { |
| 234 | return err |
| 235 | } |
| 236 | if obs.Absent || obs.Claim.Op.Kind != hostclaim.KindPlacing || obs.Claim.Op.ID != id { |
| 237 | return hostops.ErrPending |
| 238 | } |
| 239 | if err = p.Authority.FirstPhase(ctx, obs); err != nil { |
| 240 | return err |
| 241 | } |
| 242 | record, err := hostops.ReadRecord(ctx, p.Controller.Store, obs.Claim.Holder.Key) |
| 243 | if err != nil { |
| 244 | return err |
| 245 | } |
| 246 | if record.ETag != ph.RecordETag { |
| 247 | return hostops.ErrPending |
| 248 | } |
| 249 | wire, err := provision.DecodeRecord(record.Body) |
| 250 | if err != nil { |
| 251 | return err |
| 252 | } |
| 253 | if wire.Region != op.Intent["target"] || len(wire.Replicas) != 0 { |
| 254 | return hostops.ErrPending |
| 255 | } |
| 256 | } |
| 257 | // The app-level projection correction is an app SQL write: it runs |
| 258 | // as an effect so the SQL gate admits it on that permit. |
| 259 | target := op.Intent["target"] |
| 260 | if err = p.sqlEffect(ctx, h, acct.ID, "placement", func(st *Store) error { |
| 261 | return st.SetClaimAppPlacement(acct.ID, app, appID, id, target) |
| 262 | }); err != nil { |
| 263 | return err |
| 264 | } |
| 265 | return h.SetPhase(ctx, "claimed", "releasing", nil) |
| 266 | case "releasing": |
| 267 | if err = json.Unmarshal([]byte(op.Intent["hosts"]), &hosts); err != nil { |
| 268 | return err |
| 269 | } |
| 270 | for _, ph := range hosts { |
| 271 | obs, err := hostclaim.Read(ctx, p.Controller.Store, ph.Host) |
| 272 | if err != nil { |
| 273 | return err |
| 274 | } |
| 275 | if obs.Absent || obs.Claim.Holder == nil || obs.Claim.Holder.Incarnation != ph.Incarnation || obs.Claim.Holder.Stamp != ph.Stamp || obs.Claim.Op.Exec != ph.Executor { |
| 276 | return hostops.ErrPending |
| 277 | } |
| 278 | if obs.Claim.Op.Kind == hostclaim.KindServing { |
| 279 | continue |
| 280 | } |
| 281 | if _, err = hostclaim.Update(ctx, p.Controller.Store, obs, hostops.CompleteMetadataPlacement(id, appID)); err != nil { |
| 282 | return err |
| 283 | } |
| 284 | } |
| 285 | return p.finish(ctx, h, acct.ID) |
| 286 | } |
| 287 | return hostops.ErrPending |
| 288 | } |
| 289 | |
| 290 | func (p *ClaimPlacementCoordinator) finish(ctx context.Context, h *hostops.OperationHandle, accountID int64) error { |
| 291 | a, err := hostops.ReadAdmission(ctx, p.Controller.Store, h.Account(), h.App(), h.AppID()) |
| 292 | if err != nil { |
| 293 | return err |
| 294 | } |
| 295 | if a.Value.Op == nil || a.Value.Op.ID != h.ID() { |
| 296 | return hostops.ErrPending |
| 297 | } |
| 298 | // Cancellation during draining has no host claims. Persist releasing |
| 299 | // before touching SQL, so a crash cannot re-enter the older bind. |
| 300 | if a.Value.Op.Phase == "draining" { |
| 301 | if err = h.SetPhase(ctx, "draining", "releasing", map[string]string{"hosts": "[]"}); err != nil { |
| 302 | return err |
| 303 | } |
| 304 | } |
| 305 | // Advance the SQL fence BEFORE reopening admission. FinishPlacement adds |
| 306 | // exactly one revision, so old workers cannot bind again afterwards. |
| 307 | // Advance the SQL fence BEFORE reopening. Inside this effect prepare and |
| 308 | // issue have already written; its settle and FinishPlacement add one CAS |
| 309 | // each, so the reopened revision is the current one plus two. Stale |
| 310 | // workers carry lower revisions and cannot bind again. |
| 311 | if err = p.sqlEffect(ctx, h, accountID, "release-fence", func(st *Store) error { |
| 312 | fresh, err := hostops.ReadAdmission(ctx, p.Controller.Store, h.Account(), h.App(), h.AppID()) |
| 313 | if err != nil { |
| 314 | return err |
| 315 | } |
| 316 | return st.BindClaimAppOperation(accountID, h.App(), h.AppID(), "", fresh.Value.Revision+2) |
| 317 | }); err != nil { |
| 318 | return err |
| 319 | } |
| 320 | return h.FinishPlacement(ctx) |
| 321 | } |
| 322 | |
| 323 | // cancelUnsupported is reached only during draining, before any host claim |
| 324 | // or projection changed. |
| 325 | func (p *ClaimPlacementCoordinator) cancelUnsupported(ctx context.Context, h *hostops.OperationHandle, accountID int64) error { |
| 326 | if err := p.finish(ctx, h, accountID); err != nil { |
| 327 | return err |
| 328 | } |
| 329 | return errors.Join(hostops.ErrContract, errors.New("console: placement shape unsupported")) |
| 330 | } |
| 331 | |
amadan