| 1 | package console |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "crypto/rand" |
| 6 | "database/sql" |
| 7 | "encoding/json" |
| 8 | "errors" |
| 9 | "fmt" |
| 10 | "github.com/carlosframework/platform/internal/park" |
| 11 | "log" |
| 12 | "net/http" |
| 13 | "os" |
| 14 | "slices" |
| 15 | "strings" |
| 16 | "time" |
| 17 | |
| 18 | "github.com/carlosframework/platform/internal/activator" |
| 19 | "github.com/carlosframework/platform/internal/appconfig" |
| 20 | "github.com/carlosframework/platform/internal/hostactors" |
| 21 | "github.com/carlosframework/platform/internal/hostclaim" |
| 22 | "github.com/carlosframework/platform/internal/hostfence" |
| 23 | "github.com/carlosframework/platform/internal/hostops" |
| 24 | "github.com/carlosframework/platform/internal/hostresolve" |
| 25 | "github.com/carlosframework/platform/internal/moves" |
| 26 | "github.com/carlosframework/platform/internal/objstore" |
| 27 | "github.com/carlosframework/platform/internal/provision" |
| 28 | "github.com/carlosframework/platform/internal/registry" |
| 29 | "github.com/carlosframework/platform/internal/release" |
| 30 | "github.com/carlosframework/platform/internal/svcrole" |
| 31 | ) |
| 32 | |
| 33 | // The residence-move doors (issue #331, instance-move spec): the console |
| 34 | // writes REQUESTS onto the record and bumps the serial; everything that |
| 35 | // touches data happens on the source box's move worker. Three verbs: |
| 36 | // |
| 37 | // POST .../instances/{host}/move {"region":"uk"} → preparing |
| 38 | // POST .../instances/{host}/move/cut → cutting |
| 39 | // DELETE .../instances/{host}/move → aborted |
| 40 | // |
| 41 | // State is read through the ordinary instance GET — the record carries |
| 42 | // `move` verbatim while one is in flight. |
| 43 | |
| 44 | // moveGate is the shared front half: the instance family's gate, the |
| 45 | // byte-identical 404 for a host that names no record, and the class rail — |
| 46 | // only an exec-backed instance route's state is the platform's to move |
| 47 | // (spec §Out of scope), and fleet placements keep their deliberate |
| 48 | // state-does-not-follow semantics. |
| 49 | func (s *Server) moveGate(w http.ResponseWriter, r *http.Request) (acct *Account, app, host string, rec *provision.Record, ok bool) { |
| 50 | c := mainMoveCopy() |
| 51 | if s.cfg.MovesEnabled { |
| 52 | c = phase1MoveCopy() |
| 53 | } |
| 54 | _, acct, app, gok := s.instanceGateSaying(w, r, "instance", c.gate) |
| 55 | if !gok { |
| 56 | return nil, "", "", nil, false |
| 57 | } |
| 58 | host = r.PathValue("host") |
| 59 | if !provision.ValidHostShape(host) { |
| 60 | http.NotFound(w, r) |
| 61 | return nil, "", "", nil, false |
| 62 | } |
| 63 | ctx := r.Context() |
| 64 | rec, err := provision.GetRecord(ctx, s.cfg.Releases, acct.Sqid, app, host) |
| 65 | if errors.Is(err, os.ErrNotExist) { |
| 66 | http.NotFound(w, r) |
| 67 | return nil, "", "", nil, false |
| 68 | } |
| 69 | if err != nil { |
| 70 | log.Printf("console: api instance move %s/%s/%s: %v", acct.Sqid, app, host, err) |
| 71 | jsonError(w, http.StatusBadGateway, "could not read the instance") |
| 72 | return nil, "", "", nil, false |
| 73 | } |
| 74 | if k := rec.Kind; k != "" && k != "instance" { |
| 75 | jsonError(w, http.StatusBadRequest, c.kind(k)) |
| 76 | return nil, "", "", nil, false |
| 77 | } |
| 78 | if rec.Addr != "" || len(rec.Addrs) > 0 { |
| 79 | jsonError(w, http.StatusBadRequest, c.addr) |
| 80 | return nil, "", "", nil, false |
| 81 | } |
| 82 | if _, fleet := registry.FleetPlacement(rec.Region); fleet { |
| 83 | jsonError(w, http.StatusBadRequest, c.fleet) |
| 84 | return nil, "", "", nil, false |
| 85 | } |
| 86 | if rec.Pool != "" { |
| 87 | jsonError(w, http.StatusBadRequest, c.pool) |
| 88 | return nil, "", "", nil, false |
| 89 | } |
| 90 | if !rec.HibernateOrDefault() { |
| 91 | jsonError(w, http.StatusBadRequest, c.hibernate) |
| 92 | return nil, "", "", nil, false |
| 93 | } |
| 94 | return acct, app, host, rec, true |
| 95 | } |
| 96 | |
| 97 | // moveCopy is what moveGate says: its class rails and the shared gates' |
| 98 | // sentences. With the switch off the move doors speak main's bytes |
| 99 | // (mainMoveCopy()); with it on, the Phase 1 sentences (phase1MoveCopy(), copy |
| 100 | // record MV-N47 to MV-N54), which no other door ever speaks. |
| 101 | type moveCopy struct { |
| 102 | gate gateCopy |
| 103 | kind func(kind string) string |
| 104 | addr, fleet, pool, hibernate string |
| 105 | } |
| 106 | |
| 107 | // mainMoveCopy returns main's sentences, byte for byte (the dashes are escaped |
| 108 | // only in the source). |
| 109 | func mainMoveCopy() moveCopy { |
| 110 | return moveCopy{ |
| 111 | gate: mainGateCopy(), |
| 112 | kind: func(k string) string { return "only exec-backed instances move \u2014 this route is kind " + k }, |
| 113 | addr: "an addr-backed route's process is not platform-managed \u2014 nothing to move", |
| 114 | fleet: "fleet-placed instances keep state-does-not-follow semantics \u2014 move is for platform regions", |
| 115 | pool: "pooled instances do not move in v1 \u2014 their data lives on the pool's own box", |
| 116 | hibernate: "a non-hibernating instance runs outside the activator \u2014 nothing manages its data, nothing can move it", |
| 117 | } |
| 118 | } |
| 119 | |
| 120 | func phase1MoveCopy() moveCopy { |
| 121 | return moveCopy{ |
| 122 | gate: gateCopy{ |
| 123 | ambiguous: func(seg, alts string) string { |
| 124 | return fmt.Sprintf("account %q is ambiguous. pass --account-id with one of: %s", seg, alts) |
| 125 | }, |
| 126 | // moveGate asks the family's gate for the noun "instance" and no |
| 127 | // other, so the recorded sentence (MV-N52) is written whole. |
| 128 | tooMany: func(string) string { return "too many instance requests. wait a minute and try again" }, |
| 129 | noBucket: "this console has no deployment bucket, so it cannot answer this", |
| 130 | }, |
| 131 | kind: func(k string) string { return "only exec-backed instances move. this route is kind " + k }, |
| 132 | addr: "an addr-backed route's process is not platform-managed. there is nothing to move", |
| 133 | fleet: "fleet-placed instances keep state-does-not-follow semantics. moves are for platform regions", |
| 134 | pool: "pooled instances do not move yet. their data lives on the pool's own box", |
| 135 | hibernate: "a non-hibernating instance runs outside the activator. nothing manages its data, so nothing can move it", |
| 136 | } |
| 137 | } |
| 138 | |
| 139 | // moveScopedHolder is MV-N167's sentence, the one the cut's admission |
| 140 | // records (provision.RefusedReplicaLayout); the door speaks it for a holder |
| 141 | // it will not move (ruling T5-lease). A literal, because the copy gate |
| 142 | // reads this file's sentences; TestMoveDoorRefusesAScopedHolder pins it |
| 143 | // equal to the recorded constant. |
| 144 | const moveScopedHolder = "the instance keeps its data under a per-incarnation path, which moves cannot cut yet" |
| 145 | |
| 146 | // moveView is the flat answer every move door speaks — titogo's consumer |
| 147 | // contract (teamtito/titogo#1848, region-doors design): state, target, one |
| 148 | // freshness stamp. "none" is a real state, not an error: a host with no |
| 149 | // move in flight answers 200 {"state":"none"} so a poller needs no 404 |
| 150 | // branch. Target carries the same NAME the caller used — the home code is |
| 151 | // un-normalized on the way out (see homeRegionOut). |
| 152 | // |
| 153 | // With CARLOS_MOVES_PHASE1 on (Plan 6 Task 2, preflight L2 and L4) the view |
| 154 | // also carries the in-flight move's id, the outcome the last close left on |
| 155 | // the record (D11's last), and the Phase 1 marker, so a CLI can bind its |
| 156 | // wait to the move it requested and can tell a Phase 1 console from a |
| 157 | // legacy one before it sends anything. None of the three reveals anything |
| 158 | // the state did not, so every caller of the flat view sees them, the |
| 159 | // Instance-role token included (M18). With the switch off they are absent |
| 160 | // and the bytes are main's. |
| 161 | type moveView struct { |
| 162 | State string `json:"state"` |
| 163 | Target string `json:"target,omitempty"` |
| 164 | UpdatedAt int64 `json:"updated_at,omitempty"` |
| 165 | ID string `json:"id,omitempty"` |
| 166 | Last *provision.MoveOutcome `json:"last,omitempty"` |
| 167 | Protocol string `json:"protocol,omitempty"` |
| 168 | // Phase is the step within the state (movePhase), so a poller reads |
| 169 | // detail=1 only when the state or the phase changes (preflight L19). |
| 170 | Phase string `json:"phase,omitempty"` |
| 171 | } |
| 172 | |
| 173 | // movePhase is a move's phase as the flat view shows it: the cut's phase |
| 174 | // while cutting, the landing's while landing, and the rollback's target and |
| 175 | // source phases while rolling back. Other states have none. |
| 176 | func movePhase(m *provision.MoveState) string { |
| 177 | switch m.State { |
| 178 | case provision.MoveCutting: |
| 179 | return m.CutPhase |
| 180 | case provision.MoveLanding: |
| 181 | return m.LandingPhase |
| 182 | case provision.MoveRollingBack: |
| 183 | if m.Rollback != nil { |
| 184 | return m.Rollback.TargetPhase + "/" + m.Rollback.SourcePhase |
| 185 | } |
| 186 | } |
| 187 | return "" |
| 188 | } |
| 189 | |
| 190 | // moveProtocolPhase1 is the Phase 1 marker every move answer carries with |
| 191 | // the switch on (preflight L4). The CLI refuses to send a mutation to a |
| 192 | // console whose answer lacks it. |
| 193 | const moveProtocolPhase1 = "phase1" |
| 194 | |
| 195 | func (s *Server) moveViewOf(rec *provision.Record) moveView { |
| 196 | v := moveView{State: "none"} |
| 197 | if rec != nil && rec.Move != nil { |
| 198 | v = moveView{ |
| 199 | State: rec.Move.State, |
| 200 | Target: s.homeRegionOut(rec.Move.To), |
| 201 | UpdatedAt: rec.Move.UpdatedAt.Unix(), |
| 202 | } |
| 203 | } |
| 204 | if s.cfg.MovesEnabled { |
| 205 | v.Protocol = moveProtocolPhase1 |
| 206 | if rec != nil { |
| 207 | if rec.Move != nil { |
| 208 | v.ID, v.Phase = rec.Move.ID, movePhase(rec.Move) |
| 209 | } |
| 210 | v.Last = rec.LastMove |
| 211 | } |
| 212 | } |
| 213 | return v |
| 214 | } |
| 215 | |
| 216 | // armedCodes renders the armed half of an armedPlacements answer as the |
| 217 | // sorted comma list every refusal names — one spelling for the create door |
| 218 | // and the move door alike. |
| 219 | func armedCodes(set map[string]bool) string { |
| 220 | codes := make([]string, 0, len(set)) |
| 221 | for c, armed := range set { |
| 222 | if armed { |
| 223 | codes = append(codes, c) |
| 224 | } |
| 225 | } |
| 226 | slices.Sort(codes) |
| 227 | return strings.Join(codes, ", ") |
| 228 | } |
| 229 | |
| 230 | // stagingCodes is armedCodes for the move door's stricter bar: the regions |
| 231 | // that publish a staging bucket, which is the set a move can actually name. |
| 232 | // A separate renderer rather than a flag on armedCodes, because a refusal |
| 233 | // that listed "armed" regions a move still could not use would send the |
| 234 | // caller round the same loop again. |
| 235 | func stagingCodes(caps map[string]placementCapability) string { |
| 236 | codes := make([]string, 0, len(caps)) |
| 237 | for c, cap := range caps { |
| 238 | if cap.Reachable && cap.Staging { |
| 239 | codes = append(codes, c) |
| 240 | } |
| 241 | } |
| 242 | slices.Sort(codes) |
| 243 | if len(codes) == 0 { |
| 244 | return "(none)" |
| 245 | } |
| 246 | return strings.Join(codes, ", ") |
| 247 | } |
| 248 | |
| 249 | // stagingNames is stagingCodes as the Phase 1 door says it (MV-N20): each |
| 250 | // code under the name a caller passes (homeRegionOut), so movePlacements' |
| 251 | // folded home is listed by its name. An unnamed home has no name a caller |
| 252 | // could pass and is left out, and a home that is both folded and published |
| 253 | // under its name is listed once. |
| 254 | func (s *Server) stagingNames(caps map[string]placementCapability) string { |
| 255 | names := make([]string, 0, len(caps)) |
| 256 | for c, cap := range caps { |
| 257 | if name := s.homeRegionOut(c); name != "" && cap.Reachable && cap.Staging { |
| 258 | names = append(names, name) |
| 259 | } |
| 260 | } |
| 261 | slices.Sort(names) |
| 262 | names = slices.Compact(names) |
| 263 | if len(names) == 0 { |
| 264 | return "(none)" |
| 265 | } |
| 266 | return strings.Join(names, ", ") |
| 267 | } |
| 268 | |
| 269 | // moveHomeName is the home region's configured name as the move door uses |
| 270 | // it: the claim's home (dbClaimHomeRegion) on a host-claim console, |
| 271 | // CONSOLE_HOME_REGION otherwise. It is the name homeRegionIn and |
| 272 | // homeRegionOut translate with (they spell the rule out inline, which keeps |
| 273 | // the copy gate's derivation of their sentences shallow), and box selection |
| 274 | // reads the home with it, so the door and the selection agree when the |
| 275 | // console's home is the claim-mode fallback. |
| 276 | func (s *Server) moveHomeName() string { |
| 277 | if s.cfg.HostClaim { |
| 278 | return s.cfg.dbClaimHomeRegion() |
| 279 | } |
| 280 | return s.cfg.HomeRegion |
| 281 | } |
| 282 | |
| 283 | // moveRegionAllowed is the manifest's allow-list for a move target. A |
| 284 | // named target must be listed as it is. The home target ("") is allowed |
| 285 | // when some entry names the home, read through homeRegionIn as the door |
| 286 | // reads the caller's region. An empty list allows any target. |
| 287 | func (s *Server) moveRegionAllowed(allowed []string, to string) bool { |
| 288 | if len(allowed) == 0 { |
| 289 | return true |
| 290 | } |
| 291 | if to != "" { |
| 292 | return slices.Contains(allowed, to) |
| 293 | } |
| 294 | for _, r := range allowed { |
| 295 | if s.homeRegionIn(r) == "" { |
| 296 | return true |
| 297 | } |
| 298 | } |
| 299 | return false |
| 300 | } |
| 301 | |
| 302 | // homeRegionIn/Out translate between the home region's NAME (Config's |
| 303 | // CONSOLE_HOME_REGION, what pickers and callers speak — issue #346) and the |
| 304 | // record's "" (the unconstrained value the fleet's reconcile has always |
| 305 | // scoped by). The record NEVER stores the code; the API never shows "". |
| 306 | func (s *Server) homeRegionIn(region string) string { |
| 307 | home := s.cfg.HomeRegion |
| 308 | if s.cfg.HostClaim { |
| 309 | home = s.cfg.dbClaimHomeRegion() |
| 310 | } |
| 311 | return homePlacementCode(home, region) |
| 312 | } |
| 313 | |
| 314 | // homePlacementCode is homeRegionIn's rule for a given home name: the |
| 315 | // home's name is the record's "", any other region is itself, and with no |
| 316 | // home named nothing is the home. claimMoveRosterPlaces reads roster |
| 317 | // regions through it too, so the door and the API translate home one way. |
| 318 | func homePlacementCode(home, region string) string { |
| 319 | return park.HomePlacementCode(home, region) |
| 320 | } |
| 321 | |
| 322 | func (s *Server) homeRegionOut(region string) string { |
| 323 | if region == "" { |
| 324 | if s.cfg.HostClaim { |
| 325 | return s.cfg.dbClaimHomeRegion() |
| 326 | } |
| 327 | return s.cfg.HomeRegion |
| 328 | } |
| 329 | return region |
| 330 | } |
| 331 | |
| 332 | // moveTransition applies one guarded transition and maps its outcomes: |
| 333 | // ErrMoveConflict → 409 with the transition's own sentence, success → the |
| 334 | // flat move view + a serial bump so the source box notices in seconds. |
| 335 | func (s *Server) moveTransition(w http.ResponseWriter, r *http.Request, acct *Account, app, host string, mut func(*provision.Record) error, okStatus int) { |
| 336 | s.moveTransitionSaying(w, r, acct, app, host, mut, okStatus, error.Error, "could not write the move \u2014 try again") // main's bytes |
| 337 | } |
| 338 | |
| 339 | // moveTransitionP1 is moveTransition for the Phase 1 door: a refusal is its |
| 340 | // recorded sentence (phase1Conflict), never err.Error(), and the failure |
| 341 | // sentence has no em dash. The legacy door keeps main's bytes. |
| 342 | func (s *Server) moveTransitionP1(w http.ResponseWriter, r *http.Request, acct *Account, app, host string, mut func(*provision.Record) error, okStatus int) { |
| 343 | s.moveTransitionSaying(w, r, acct, app, host, mut, okStatus, phase1Conflict, "could not write the move. try again") |
| 344 | } |
| 345 | |
| 346 | func (s *Server) moveTransitionSaying(w http.ResponseWriter, r *http.Request, acct *Account, app, host string, mut func(*provision.Record) error, okStatus int, conflict func(error) string, failed string) { |
| 347 | ctx := r.Context() |
| 348 | // Host-claim consoles: request, cut and abort all rewrite the record's |
| 349 | // move journal, which only the (unbuilt) Moves protocol may do under |
| 350 | // its own operation. Refused before any write. |
| 351 | if s.cfg.HostClaim { |
| 352 | jsonError(w, http.StatusConflict, claimMoveUnsupportedMsg) |
| 353 | return |
| 354 | } |
| 355 | rec, err := provision.SetRecordMove(ctx, s.cfg.Releases, acct.Sqid, app, host, mut) |
| 356 | if errors.Is(err, provision.ErrMoveConflict) { |
| 357 | jsonError(w, http.StatusConflict, conflict(err)) |
| 358 | return |
| 359 | } |
| 360 | if err != nil { |
| 361 | log.Printf("console: api instance move %s/%s/%s: %v", acct.Sqid, app, host, err) |
| 362 | jsonError(w, http.StatusBadGateway, failed) |
| 363 | return |
| 364 | } |
| 365 | s.bumpSerial(ctx) |
| 366 | writeJSON(w, okStatus, s.moveViewOf(rec)) |
| 367 | } |
| 368 | |
| 369 | // handleAPIInstanceMoveStatus answers the poll: the flat view, "none" when |
| 370 | // quiet — the tenant's freeze loop lives on this door. |
| 371 | // |
| 372 | // With the switch on (Plan 6 Task 2), an operator's GET with detail=1 gets |
| 373 | // the detail answer (moveDetail); anyone else's gets the flat view, with no |
| 374 | // error (M18). A committed landing whose claim has been converted to a |
| 375 | // relocation (Paul's J3) answers state relocating in both, so the flat view |
| 376 | // reads the claim while the record says committed, and only then. If that |
| 377 | // read fails the flat view says what the record says; the poll goes on. |
| 378 | func (s *Server) handleAPIInstanceMoveStatus(w http.ResponseWriter, r *http.Request) { |
| 379 | acct, app, host, rec, ok := s.moveGate(w, r) |
| 380 | if !ok { |
| 381 | return |
| 382 | } |
| 383 | view := s.moveViewOf(rec) |
| 384 | if !s.cfg.MovesEnabled { |
| 385 | writeJSON(w, http.StatusOK, view) |
| 386 | return |
| 387 | } |
| 388 | if r.URL.Query().Get("detail") == "1" { |
| 389 | if _, isOp := s.moveOperator(r); isOp { |
| 390 | s.writeMoveDetail(w, r, acct, app, host, rec, view) |
| 391 | return |
| 392 | } |
| 393 | } |
| 394 | if rec.Move != nil && rec.Move.State == provision.MoveCommitted { |
| 395 | obs, err := hostclaim.Read(r.Context(), s.cfg.Releases, host) |
| 396 | if err != nil { |
| 397 | log.Printf("console: api instance move %s/%s/%s: status: claim read: %v", acct.Sqid, app, host, err) |
| 398 | } else if moveConvertedAfterCommit(rec, obs) { |
| 399 | view.State = provision.MoveRelocating |
| 400 | } |
| 401 | } |
| 402 | writeJSON(w, http.StatusOK, view) |
| 403 | } |
| 404 | |
| 405 | func (s *Server) legacyHandleAPIInstanceMoveRequest(w http.ResponseWriter, r *http.Request) { |
| 406 | acct, app, host, rec, ok := s.moveGate(w, r) |
| 407 | if !ok { |
| 408 | return |
| 409 | } |
| 410 | if s.refuseSeedMove(w, rec) { |
| 411 | return |
| 412 | } |
| 413 | // A replica is refused at the request only (G14): moveGate also fronts |
| 414 | // the status and abort doors, and a replica that got a move before this |
| 415 | // rail existed must still be able to read and abort it. |
| 416 | if rec.Role != "" { |
| 417 | jsonError(w, http.StatusBadRequest, "a replica does not move. delete it and create one in the new region") |
| 418 | return |
| 419 | } |
| 420 | var req struct { |
| 421 | Region string `json:"region"` |
| 422 | } |
| 423 | if !readJSON(w, r, &req) { |
| 424 | return |
| 425 | } |
| 426 | region := strings.TrimSpace(req.Region) |
| 427 | to := s.homeRegionIn(region) |
| 428 | if region == "" || !provision.ValidRegion(to) { |
| 429 | jsonError(w, http.StatusBadRequest, "invalid region") |
| 430 | return |
| 431 | } |
| 432 | // A move to the HOME region ("" after normalization) has no armed-edges |
| 433 | // row to validate against and no staging lane wired for the home's |
| 434 | // bucket yet — refused plainly rather than wedging in preparing. |
| 435 | if to == "" { |
| 436 | jsonError(w, http.StatusBadRequest, "moves to the home region are not supported yet — the home publishes no staging lane") |
| 437 | return |
| 438 | } |
| 439 | // The armed gate is MANDATORY for moves — unlike create there is no |
| 440 | // dormant path, because a move needs a live target box to restore on; |
| 441 | // a deployment with no armed placements cannot be a move target at all. |
| 442 | // |
| 443 | // And the bar here is STAGING, not merely hosting: the requirement is |
| 444 | // unconditional and is the target's published data-bucket coordinates, |
| 445 | // because that is precisely what the worker on the other side demands. |
| 446 | // internal/host's moveTargets skips any target whose edges file carries |
| 447 | // no data_bucket, and runMoveRound then answers "target region %s |
| 448 | // publishes no data bucket yet — waiting" on every pass, forever. Before |
| 449 | // this the door accepted such a target on reachability alone and the |
| 450 | // move wedged in preparing with no way out but a manual cancel — the |
| 451 | // door and the worker disagreeing about the same fact, which is the |
| 452 | // failure this whole family of gates exists to prevent. Unknown is not |
| 453 | // an acceptable answer for a move the way it is for a create: a create |
| 454 | // into an unconfigured box strands one instance, while a move stakes a |
| 455 | // LIVE instance's residence on a lane that does not exist. |
| 456 | caps, published := placements(r.Context(), s.cfg.Releases) |
| 457 | if !published { |
| 458 | jsonError(w, http.StatusBadRequest, "this deployment publishes no armed regions — nothing to move to") |
| 459 | return |
| 460 | } |
| 461 | if !caps[to].Reachable || !caps[to].Staging { |
| 462 | jsonError(w, http.StatusBadRequest, "region "+to+" cannot take a move in this deployment — it publishes no staging bucket. Regions that can: "+stagingCodes(caps)) |
| 463 | return |
| 464 | } |
| 465 | s.moveTransition(w, r, acct, app, host, provision.MoveRequest(to, time.Now()), http.StatusAccepted) |
| 466 | } |
| 467 | |
| 468 | func (s *Server) legacyHandleAPIInstanceMoveCut(w http.ResponseWriter, r *http.Request) { |
| 469 | acct, app, host, _, ok := s.moveGate(w, r) |
| 470 | if !ok { |
| 471 | return |
| 472 | } |
| 473 | s.moveTransition(w, r, acct, app, host, provision.MoveMarkCutting(), http.StatusAccepted) |
| 474 | } |
| 475 | |
| 476 | // legacyHandleAPIInstanceMoveAbort: abort is two-phase (review round 1): |
| 477 | // this door marks MoveAborting and the |
| 478 | // SOURCE box deletes the staged copy before clearing the field — a bare |
| 479 | // clear would leave staging behind to win a later landing on the target |
| 480 | // (staging-wins). 202, like its siblings: the caller polls the record. |
| 481 | func (s *Server) legacyHandleAPIInstanceMoveAbort(w http.ResponseWriter, r *http.Request) { |
| 482 | acct, app, host, _, ok := s.moveGate(w, r) |
| 483 | if !ok { |
| 484 | return |
| 485 | } |
| 486 | s.moveTransition(w, r, acct, app, host, provision.MoveAbort(), http.StatusAccepted) |
| 487 | } |
| 488 | |
| 489 | // ---- Phase 1 (CARLOS_MOVES_PHASE1, index D13) ---- |
| 490 | // |
| 491 | // With the switch off each handler below hands the request to its legacy |
| 492 | // twin above, unchanged, and the resolve door answers the family's 404: |
| 493 | // the console behaves exactly as before Phase 1. With it on, every |
| 494 | // mutation is operator-only, the request door CASes the host claim from |
| 495 | // serving to moving before it journals the move (§7.1, K2), and cut and |
| 496 | // abort carry the journal's id and incarnation (MoveGuard). |
| 497 | |
| 498 | // moveOperatorGate is moveGate for the mutations (§12): the caller must be |
| 499 | // an operator presenting their own CLI token, else the family's |
| 500 | // byte-identical 404. A member's token and the Instance-role service token |
| 501 | // keep the status GET only (M18), so the member-minted instance token can |
| 502 | // no longer move anything (codex 11). The routes keep |
| 503 | // svcrole.Allow(svcrole.Instance) so a service token meets this 404 rather |
| 504 | // than the role gate's 403. |
| 505 | func (s *Server) moveOperatorGate(w http.ResponseWriter, r *http.Request) (acct *Account, app, host string, rec *provision.Record, addr string, ok bool) { |
| 506 | acct, app, host, rec, gok := s.moveGate(w, r) |
| 507 | if !gok { |
| 508 | return nil, "", "", nil, "", false |
| 509 | } |
| 510 | addr, ok = s.moveOperator(r) |
| 511 | if !ok { |
| 512 | http.NotFound(w, r) |
| 513 | return nil, "", "", nil, "", false |
| 514 | } |
| 515 | return acct, app, host, rec, addr, true |
| 516 | } |
| 517 | |
| 518 | // moveOperator reports whether the request presents an operator's own CLI |
| 519 | // token, and whose. A service token, a member's token and an error reading |
| 520 | // the operator flag are all "no" (the error is logged). |
| 521 | func (s *Server) moveOperator(r *http.Request) (addr string, ok bool) { |
| 522 | addr, ok = s.store.CLITokenAddress(bearer(r), time.Now()) |
| 523 | if !ok { |
| 524 | return "", false |
| 525 | } |
| 526 | isOp, err := s.store.IsOperator(addr) |
| 527 | if err != nil { |
| 528 | log.Printf("console: api instance move: is-operator %s: %v", addr, err) |
| 529 | } |
| 530 | if err != nil || !isOp { |
| 531 | return "", false |
| 532 | } |
| 533 | return addr, true |
| 534 | } |
| 535 | |
| 536 | // moveGuardOf is the guard a door derives from the record it read. A record |
| 537 | // with no move yields the zero guard, which every guarded transition |
| 538 | // refuses with its own 409 ("no journaled move in flight", MV-N12). |
| 539 | func moveGuardOf(rec *provision.Record) provision.MoveGuard { |
| 540 | if rec.Move == nil { |
| 541 | return provision.MoveGuard{} |
| 542 | } |
| 543 | return provision.MoveGuard{ID: rec.Move.ID, Incarnation: rec.Move.Incarnation} |
| 544 | } |
| 545 | |
| 546 | // handleAPIInstanceMoveRequest is the request door (§7.1). Its order: the |
| 547 | // class rails and the operator (moveOperatorGate), the record's own rails, |
| 548 | // the body, the region, the app's regions, the instance's storage binding, |
| 549 | // the armed placements, the fleet (selectMoveBoxes: one eligible box on |
| 550 | // each side, the floor), then the claim (present, this incarnation, |
| 551 | // serving, executed by the source's eligible box). Every refusal comes |
| 552 | // before the claim CAS, so a refused request writes nothing. |
| 553 | // |
| 554 | // If the record write fails after the claim CAS, the door answers 502 and |
| 555 | // logs: the claim is then moving with no journaled move, and the console |
| 556 | // pass of Plan 5 Task 7 releases it. |
| 557 | func (s *Server) handleAPIInstanceMoveRequest(w http.ResponseWriter, r *http.Request) { |
| 558 | if !s.cfg.MovesEnabled { |
| 559 | s.legacyHandleAPIInstanceMoveRequest(w, r) |
| 560 | return |
| 561 | } |
| 562 | acct, app, host, rec, addr, ok := s.moveOperatorGate(w, r) |
| 563 | if !ok { |
| 564 | return |
| 565 | } |
| 566 | if s.refuseSeedMove(w, rec) { |
| 567 | return |
| 568 | } |
| 569 | // Host-claim consoles start the move under the app's admission: the |
| 570 | // claim runtime's authority registers it (RegisterMove), and the start |
| 571 | // is hostops.BeginClaimMove then hostops.StartClaimMoveRecord, never a |
| 572 | // plain BeginMove (whose claim C7 cannot convert). A console with the |
| 573 | // flag on but no runtime cannot admit anything, so it refuses before any |
| 574 | // write, like cut and abort (moveTransitionSaying). |
| 575 | if s.cfg.HostClaim && s.cfg.ClaimAuthority == nil { |
| 576 | jsonError(w, http.StatusConflict, claimMoveUnsupportedMsg) |
| 577 | return |
| 578 | } |
| 579 | ctx := r.Context() |
| 580 | if rec.Role != "" { |
| 581 | jsonError(w, http.StatusBadRequest, "a replica does not move. delete it and create one in the new region") |
| 582 | return |
| 583 | } |
| 584 | if len(rec.Replicas) > 0 { |
| 585 | jsonError(w, http.StatusBadRequest, "a home with replicas does not move. remove its replicas first") |
| 586 | return |
| 587 | } |
| 588 | var req struct { |
| 589 | Region string `json:"region"` |
| 590 | Cut string `json:"cut"` |
| 591 | } |
| 592 | if !readJSON(w, r, &req) { |
| 593 | return |
| 594 | } |
| 595 | cut := req.Cut |
| 596 | if cut == "" { |
| 597 | cut = "auto" |
| 598 | } |
| 599 | if cut != "auto" && cut != "manual" { |
| 600 | jsonError(w, http.StatusBadRequest, `cut must be "auto" or "manual"`) |
| 601 | return |
| 602 | } |
| 603 | region := strings.TrimSpace(req.Region) |
| 604 | to := s.homeRegionIn(region) |
| 605 | if region == "" || !provision.ValidRegion(to) { |
| 606 | jsonError(w, http.StatusBadRequest, "invalid region") |
| 607 | return |
| 608 | } |
| 609 | // Ruling R1 (home part 2): a move into the home region is admitted only |
| 610 | // on a host-claim console, whose claim pins the home's engine region |
| 611 | // (@home) under the roster's designation. The plain path refuses it |
| 612 | // before any read or write. |
| 613 | if to == "" && !s.cfg.HostClaim { |
| 614 | jsonError(w, http.StatusBadRequest, "moves into the home region need host claims. this console runs without them") |
| 615 | return |
| 616 | } |
| 617 | // toName is the target as the caller named it, which is |
| 618 | // homeRegionOut(to): homeRegionIn maps only the home's own name to "", |
| 619 | // so the round trip gives region back (Ruling R3: the caller names the |
| 620 | // home by its configured name). Every sentence below names the home so, |
| 621 | // never by the record's "". |
| 622 | toName := region |
| 623 | if to == rec.Region { |
| 624 | jsonError(w, http.StatusConflict, "the instance is already in "+toName) |
| 625 | return |
| 626 | } |
| 627 | // Both policy reads fail closed (round-4 finding 13): an unreadable |
| 628 | // manifest or storage binding is a 503, never a pass. |
| 629 | m, err := provision.GetManifest(ctx, s.cfg.Releases, acct.Sqid, app) |
| 630 | if err != nil && !errors.Is(err, os.ErrNotExist) { |
| 631 | log.Printf("console: api instance move %s/%s/%s: manifest: %v", acct.Sqid, app, host, err) |
| 632 | jsonError(w, http.StatusServiceUnavailable, "could not read the app's instances manifest. try again") |
| 633 | return |
| 634 | } |
| 635 | if m != nil && !s.moveRegionAllowed(m.Regions, to) { |
| 636 | jsonError(w, http.StatusBadRequest, "region "+toName+" is not one of this app's regions: "+strings.Join(m.Regions, ", ")) |
| 637 | return |
| 638 | } |
| 639 | // Private storage is the instance's BINDING, not the app's create-time |
| 640 | // policy (which is mutable and says nothing about an instance that |
| 641 | // already exists): the create door stamps a store per instance under |
| 642 | // its environment, or its slug when it has none (InstanceStoreEnv), and |
| 643 | // turning the policy off later removes no store. A name that cannot be |
| 644 | // an environment can hold no store, so it is not read. |
| 645 | if env := provision.InstanceStoreEnv(rec); env == "" || appconfig.ValidEnvironmentName(env) { |
| 646 | store, _, err := objstore.GetIn(ctx, s.cfg.Releases, acct.Sqid, app, env) |
| 647 | if err != nil { |
| 648 | log.Printf("console: api instance move %s/%s/%s: storage binding: %v", acct.Sqid, app, host, err) |
| 649 | jsonError(w, http.StatusServiceUnavailable, "could not read this instance's storage binding. try again") |
| 650 | return |
| 651 | } |
| 652 | if store != nil { |
| 653 | jsonError(w, http.StatusBadRequest, host+" uses private storage. Instances with private storage cannot move yet.") |
| 654 | return |
| 655 | } |
| 656 | } |
| 657 | // Region HA (spec 2 §7, H27): a destination with no box that has room |
| 658 | // is the no-room refusal, before the placement checks below answer it |
| 659 | // as a 400. It only refuses: nothing is chosen. An incomplete read is |
| 660 | // the temporarily unavailable 503, never no room. |
| 661 | if err := s.roomIn(ctx, to, ""); err != nil { |
| 662 | writeClaimResponseError(w, err) |
| 663 | return |
| 664 | } |
| 665 | caps, homeNamed, published := movePlacements(ctx, s.cfg.Releases, s.moveHomeName()) |
| 666 | // Review Focus 5: a box that sets CARLOS_REGION to the home's name is |
| 667 | // home to the console but not to itself, so it cannot take a move into |
| 668 | // home. It is refused here, before the staging check: the home's fold |
| 669 | // takes placementless files only, so that check would answer "publishes |
| 670 | // no staging bucket", which hides what the operator has to fix. The |
| 671 | // files are not liveness-joined, so a stale file of a box that has gone |
| 672 | // refuses too (fail closed): the sentence names the file's box. |
| 673 | if to == "" && len(homeNamed) > 0 { |
| 674 | jsonError(w, http.StatusConflict, homeNamed[0]+" sets CARLOS_REGION, so it cannot take a move into the home region") |
| 675 | return |
| 676 | } |
| 677 | if !published { |
| 678 | jsonError(w, http.StatusBadRequest, "this deployment publishes no armed regions. nothing to move to") |
| 679 | return |
| 680 | } |
| 681 | if !caps[to].Reachable || !caps[to].Staging { |
| 682 | jsonError(w, http.StatusBadRequest, "region "+toName+" cannot take a move in this deployment. it publishes no staging bucket. Regions that can: "+s.stagingNames(caps)) |
| 683 | return |
| 684 | } |
| 685 | from := s.homeRegionOut(rec.Region) |
| 686 | sel, err := s.selectMoveBoxes(ctx, host, to, rec.Region, acct.Sqid+"/"+app) |
| 687 | // No box in the destination has room (region HA spec 2 §7, H27), or the |
| 688 | // read that would tell could not be completed: the claim doors' answers, |
| 689 | // not the protocol floor's below. |
| 690 | var ( |
| 691 | full *noCapacityError |
| 692 | check *roomCheckError |
| 693 | ) |
| 694 | if errors.As(err, &full) || errors.As(err, &check) { |
| 695 | writeClaimResponseError(w, err) |
| 696 | return |
| 697 | } |
| 698 | if errors.Is(err, errEdgesIncomplete) { |
| 699 | jsonError(w, http.StatusConflict, "the fleet's edges files could not all be read. try again") |
| 700 | return |
| 701 | } |
| 702 | if err != nil { |
| 703 | log.Printf("console: api instance move %s/%s/%s: protocol floor: %v", acct.Sqid, app, host, err) |
| 704 | jsonError(w, http.StatusServiceUnavailable, "could not read the moves protocol floor. try again") |
| 705 | return |
| 706 | } |
| 707 | // Phase 1 is for one-box regions (spec rev 7.20 §7.1, index D15): a |
| 708 | // region with more than one eligible box, on either side, is refused. |
| 709 | if sel.Many { |
| 710 | jsonError(w, http.StatusBadRequest, sel.ManyRegion+" has more than one box. Moves between regions with several boxes are not supported yet.") |
| 711 | return |
| 712 | } |
| 713 | if sel.Target == "" { |
| 714 | jsonError(w, http.StatusBadRequest, "no box in "+toName+" publishes an edges file yet") |
| 715 | return |
| 716 | } |
| 717 | if sel.Source == "" { |
| 718 | jsonError(w, http.StatusBadRequest, "no eligible box in "+from+" publishes an edges file") |
| 719 | return |
| 720 | } |
| 721 | if sel.ConsoleBelowFloor { |
| 722 | jsonError(w, http.StatusConflict, "this console is below the moves protocol floor. roll the console first") |
| 723 | return |
| 724 | } |
| 725 | if !sel.FloorOK { |
| 726 | jsonError(w, http.StatusConflict, "a box in "+from+" or "+toName+" is below the moves protocol floor. roll the platform first") |
| 727 | return |
| 728 | } |
| 729 | inc, err := hostclaim.Incarnation(acct.Sqid, app, host, rec.CreatedAt) |
| 730 | if err != nil { |
| 731 | jsonError(w, http.StatusBadRequest, "the record's identity is malformed") |
| 732 | return |
| 733 | } |
| 734 | obs, err := hostclaim.Read(ctx, s.cfg.Releases, host) |
| 735 | if err != nil { |
| 736 | log.Printf("console: api instance move %s/%s/%s: claim read: %v", acct.Sqid, app, host, err) |
| 737 | jsonError(w, http.StatusBadGateway, "could not read the host claim") |
| 738 | return |
| 739 | } |
| 740 | if obs.Absent || obs.Claim.Holder == nil { |
| 741 | jsonError(w, http.StatusConflict, "the instance has no host claim yet. it is adopted on its next reconcile pass") |
| 742 | return |
| 743 | } |
| 744 | if obs.Claim.Holder.Incarnation != inc { |
| 745 | jsonError(w, http.StatusConflict, "the host claim names another incarnation of this instance") |
| 746 | return |
| 747 | } |
| 748 | if obs.Claim.Op.Kind != hostclaim.KindServing { |
| 749 | jsonError(w, http.StatusConflict, "the instance is under a "+obs.Claim.Op.Kind+" operation. wait for it to finish") |
| 750 | return |
| 751 | } |
| 752 | // Ruling T5-lease (Task 5 fix round 2): Phase 1 cuts only a holder |
| 753 | // whose location is the activator's own lease prefix. A scoped holder |
| 754 | // (instances/<host>/<incarnation>) is refused here, before any claim |
| 755 | // write, with the sentence the cut's admission would record (MV-N167); |
| 756 | // when this console knows the activator's prefix, so is any other |
| 757 | // location than the activator's. Moving such a host waits for a later |
| 758 | // plan. |
| 759 | if loc := strings.TrimSuffix(obs.Claim.Holder.Location, "/"); loc == "instances/"+host+"/"+inc || |
| 760 | (s.cfg.HibernatePrefix != "" && loc != activator.HibernatePrefix(s.cfg.HibernatePrefix, host)) { |
| 761 | jsonError(w, http.StatusConflict, moveScopedHolder) |
| 762 | return |
| 763 | } |
| 764 | // §7.1 binds the source to the claim's executor (round-5 finding 7). |
| 765 | if obs.Claim.Op.Exec != sel.Source { |
| 766 | jsonError(w, http.StatusConflict, "the source box "+obs.Claim.Op.Exec+" is not the eligible box of "+from) |
| 767 | return |
| 768 | } |
| 769 | // Ruling E2, kept as the early refusal: from ONE roster observation, |
| 770 | // the source must be an active roster entry with an enrollment and must |
| 771 | // carry no termination fence. Anything else is not a box to move from |
| 772 | // (fail closed). A roster or fence that cannot be read or trusted is a |
| 773 | // 503, with the error logged and never sent. On a host-claim console |
| 774 | // BeginClaimMove checks both boxes again from the roster and records |
| 775 | // both enrollments on the op, which is what the claim's conversion (C7) |
| 776 | // is pinned to. |
| 777 | dir := hostactors.Directory{Store: s.cfg.Releases, PublicKey: s.cfg.ReleasePub, Bucket: s.cfg.Bucket} |
| 778 | roster, err := dir.Read(ctx) |
| 779 | if err != nil { |
| 780 | log.Printf("console: api instance move %s/%s/%s: host roster: %v", acct.Sqid, app, host, err) |
| 781 | jsonError(w, http.StatusServiceUnavailable, "could not read the host roster. try again") |
| 782 | return |
| 783 | } |
| 784 | if _, eligible := sourceEnrollment(roster.Roster, obs.Claim.Op.Exec); !eligible { |
| 785 | jsonError(w, http.StatusConflict, "the source box is not eligible to move from") |
| 786 | return |
| 787 | } |
| 788 | fenced, err := dir.TerminationFence(ctx, roster, obs.Claim.Op.Exec, time.Now()) |
| 789 | if err != nil { |
| 790 | log.Printf("console: api instance move %s/%s/%s: termination fence of %s: %v", acct.Sqid, app, host, obs.Claim.Op.Exec, err) |
| 791 | jsonError(w, http.StatusServiceUnavailable, "could not read the host roster. try again") |
| 792 | return |
| 793 | } |
| 794 | if fenced { |
| 795 | jsonError(w, http.StatusConflict, "the source box is not eligible to move from") |
| 796 | return |
| 797 | } |
| 798 | id, err := moves.NewMoveID(rand.Reader) |
| 799 | if err != nil { |
| 800 | log.Printf("console: api instance move %s/%s/%s: move id: %v", acct.Sqid, app, host, err) |
| 801 | jsonError(w, http.StatusBadGateway, "could not mint a move id") |
| 802 | return |
| 803 | } |
| 804 | now := time.Now() |
| 805 | ms := provision.MoveStart{ID: id, Incarnation: inc, From: rec.Region, To: to, Cut: cut, By: addr, Now: now} |
| 806 | start := provision.MoveStartTransition(ms) |
| 807 | // The record's own refusals (a move already journaled, a legacy move |
| 808 | // in flight) are tried on a copy first, so they too come before the |
| 809 | // claim CAS. The transition only replaces Move, so a shallow copy |
| 810 | // leaves rec as read. |
| 811 | probe := *rec |
| 812 | if err := start(&probe); err != nil { |
| 813 | jsonError(w, http.StatusConflict, phase1Conflict(err)) |
| 814 | return |
| 815 | } |
| 816 | exec := obs.Claim.Op.Exec |
| 817 | if s.cfg.HostClaim { |
| 818 | s.startClaimModeMove(w, r, acct, app, host, obs, ms, sel.Target, from, rec.Region, dir, roster) |
| 819 | return |
| 820 | } |
| 821 | moving, err := hostclaim.Update(ctx, s.cfg.Releases, obs, hostclaim.BeginMove(id, addr, exec, sel.Target, to, now)) |
| 822 | if err != nil { |
| 823 | moveClaimCASFailed(w, acct, app, host, toName, err) |
| 824 | return |
| 825 | } |
| 826 | got, err := provision.SetRecordMove(ctx, s.cfg.Releases, acct.Sqid, app, host, start) |
| 827 | if err != nil { |
| 828 | log.Printf("console: api instance move %s/%s/%s: the claim is moving under %s but the record write failed: %v", acct.Sqid, app, host, id, err) |
| 829 | got = s.settleFailedMoveWrite(w, r, ctx, acct, app, host, id, exec, moving, false) |
| 830 | if got == nil { |
| 831 | return |
| 832 | } |
| 833 | } |
| 834 | s.bumpSerial(ctx) |
| 835 | writeJSON(w, http.StatusAccepted, s.moveViewOf(got)) |
| 836 | } |
| 837 | |
| 838 | // moveClaimCASFailed answers a refused or failed start CAS, plain or |
| 839 | // admitted, with its recorded sentence. The error itself is only logged. |
| 840 | // toName is the target as the caller named it (homeRegionOut), never the |
| 841 | // record's "" for the home. |
| 842 | func moveClaimCASFailed(w http.ResponseWriter, acct *Account, app, host, toName string, err error) { |
| 843 | switch { |
| 844 | case errors.Is(err, hostclaim.ErrRelocationFinishing): |
| 845 | jsonError(w, http.StatusConflict, "the instance's last relocation is still finishing. try again in a few minutes") |
| 846 | case errors.Is(err, hostclaim.ErrPendingChore): |
| 847 | jsonError(w, http.StatusConflict, "the instance has unfinished cleanup in "+toName+". wait for it to finish") |
| 848 | case errors.Is(err, hostclaim.ErrConflict), errors.Is(err, hostclaim.ErrWrongState): |
| 849 | jsonError(w, http.StatusConflict, "the host claim changed under this request. try again") |
| 850 | default: |
| 851 | log.Printf("console: api instance move %s/%s/%s: claim CAS: %v", acct.Sqid, app, host, err) |
| 852 | jsonError(w, http.StatusBadGateway, "could not write the host claim") |
| 853 | } |
| 854 | } |
| 855 | |
| 856 | // startClaimModeMove is the request door's start on a host-claim console |
| 857 | // (claim Plan 5 Task 7): the claim runtime's authority registers the move |
| 858 | // under the app's admission (the registration id is the move id), then |
| 859 | // BeginClaimMove CASes the same claim observation from serving to moving |
| 860 | // with the app id, the registration and both roster enrollments on the op, |
| 861 | // and StartClaimMoveRecord journals the move on the record as one effect |
| 862 | // of that registration. The door never builds an authority of its own. |
| 863 | // |
| 864 | // Before anything is registered, the door's own roster observation (the |
| 865 | // one E2 read) must place the move as BeginClaimMove will require: both |
| 866 | // boxes active and enrolled, the source in the record's region, the target |
| 867 | // in to, and no termination fence on the target. A roster that does not is |
| 868 | // MV-N177. BeginClaimMove rereads the roster and the record; if it still |
| 869 | // refuses (ErrIdentity, ErrRoster), something changed under the request |
| 870 | // and the door does not guess what (MV-N178). |
| 871 | // |
| 872 | // Every refusal after RegisterMove and before the claim CAS writes nothing |
| 873 | // but the registration, which the admission sweep settles once its TTL |
| 874 | // passes. A failed record start after the CAS is settled as on the plain |
| 875 | // path, with the release written under the registration's lifecycle grant. |
| 876 | func (s *Server) startClaimModeMove(w http.ResponseWriter, r *http.Request, acct *Account, app, host string, obs hostclaim.Observed, ms provision.MoveStart, target, from, recRegion string, dir hostactors.Directory, roster hostactors.Observed) { |
| 877 | ctx := r.Context() |
| 878 | a := s.cfg.ClaimAuthority |
| 879 | // to is the record's code ("" for the home), which the claim calls |
| 880 | // receive; toName is what the sentences say (home part 2). |
| 881 | to := ms.To |
| 882 | toName := s.homeRegionOut(to) |
| 883 | if !roster.Roster.MatchesHome(s.cfg.dbClaimHomeRegion()) || !claimMoveRosterPlaces(roster.Roster, obs.Claim.Op.Exec, target, recRegion, to, s.cfg.dbClaimHomeRegion()) { |
| 884 | jsonError(w, http.StatusConflict, "the host roster does not allow this move. the source and target must be active boxes in "+from+" and "+toName) |
| 885 | return |
| 886 | } |
| 887 | fenced, err := dir.TerminationFence(ctx, roster, target, time.Now()) |
| 888 | if err != nil { |
| 889 | log.Printf("console: api instance move %s/%s/%s: termination fence of %s: %v", acct.Sqid, app, host, target, err) |
| 890 | jsonError(w, http.StatusServiceUnavailable, "could not read the host roster. try again") |
| 891 | return |
| 892 | } |
| 893 | if fenced { |
| 894 | jsonError(w, http.StatusConflict, "the host roster does not allow this move. the source and target must be active boxes in "+from+" and "+toName) |
| 895 | return |
| 896 | } |
| 897 | appID, err := s.store.ClaimAppID(acct.ID, app) |
| 898 | if errors.Is(err, sql.ErrNoRows) || errors.Is(err, hostops.ErrIdentity) { |
| 899 | jsonError(w, http.StatusConflict, "this app has no host-claim admission yet. try again shortly") |
| 900 | return |
| 901 | } |
| 902 | if err != nil { |
| 903 | log.Printf("console: api instance move %s/%s/%s: app id: %v", acct.Sqid, app, host, err) |
| 904 | jsonError(w, http.StatusServiceUnavailable, "could not read this app's admission. try again") |
| 905 | return |
| 906 | } |
| 907 | h, err := a.RegisterMove(ctx, obs, appID, ms.ID) |
| 908 | if err != nil { |
| 909 | switch { |
| 910 | case errors.Is(err, os.ErrNotExist): |
| 911 | jsonError(w, http.StatusConflict, "this app has no host-claim admission yet. try again shortly") |
| 912 | case errors.Is(err, hostops.ErrPending): |
| 913 | jsonError(w, http.StatusConflict, "another operation holds this app. try again when it finishes") |
| 914 | default: |
| 915 | log.Printf("console: api instance move %s/%s/%s: register move %s: %v", acct.Sqid, app, host, ms.ID, err) |
| 916 | jsonError(w, http.StatusBadGateway, "could not register the move. try again") |
| 917 | } |
| 918 | return |
| 919 | } |
| 920 | moving, err := hostops.BeginClaimMove(ctx, s.cfg.Releases, a.Directory, h, obs, ms.ID, ms.By, target, to, ms.Now, s.cfg.dbClaimHomeRegion()) |
| 921 | if err != nil { |
| 922 | switch { |
| 923 | case errors.Is(err, hostops.ErrIdentity), errors.Is(err, hostactors.ErrRoster): |
| 924 | log.Printf("console: api instance move %s/%s/%s: begin move %s: %v", acct.Sqid, app, host, ms.ID, err) |
| 925 | jsonError(w, http.StatusConflict, "the instance or its boxes changed under this request. read its state and try again") |
| 926 | case errors.Is(err, hostops.ErrPending): |
| 927 | jsonError(w, http.StatusConflict, "another operation holds this app. try again when it finishes") |
| 928 | default: |
| 929 | moveClaimCASFailed(w, acct, app, host, toName, err) |
| 930 | } |
| 931 | return |
| 932 | } |
| 933 | got, err := hostops.StartClaimMoveRecord(ctx, s.cfg.Releases, h, moving, ms) |
| 934 | if err != nil { |
| 935 | log.Printf("console: api instance move %s/%s/%s: the claim is moving under %s but the record start failed: %v", acct.Sqid, app, host, ms.ID, err) |
| 936 | // No retry is promised (MV-N179): the record-start effect is left |
| 937 | // uncertain and refuses the next start on this record until the |
| 938 | // claim code resolves it (task E-switch fix round 2). |
| 939 | got = s.settleFailedMoveWrite(w, r, h.Lifecycle(ctx), acct, app, host, ms.ID, obs.Claim.Op.Exec, moving, true) |
| 940 | if got == nil { |
| 941 | return |
| 942 | } |
| 943 | } |
| 944 | s.bumpSerial(ctx) |
| 945 | writeJSON(w, http.StatusAccepted, s.moveViewOf(got)) |
| 946 | } |
| 947 | |
| 948 | // settleFailedMoveWrite decides a record write that failed after the claim |
| 949 | // CAS succeeded. A failed write may still have landed, so the record is |
| 950 | // read again first: |
| 951 | // |
| 952 | // - it carries this move's id: the write landed and only its answer was |
| 953 | // lost. The move is returned and the door answers success; |
| 954 | // - it definitely does not: the claim goes back to serving under exec |
| 955 | // (ReleaseMove, guarded by RequireMoveOp so it can only undo this |
| 956 | // move, before its seal) and the caller is told to try again; |
| 957 | // - the read fails, or the release does: the outcome is unknown, so the |
| 958 | // claim is left as it is and the answer promises nothing. The host and |
| 959 | // move id are logged. Plan 5's recovery pass, which must land before |
| 960 | // the switch is turned on anywhere, finds a moving claim with no |
| 961 | // journaled move and releases it. |
| 962 | // |
| 963 | // claimCtx carries the authority the release's claim writes run under: the |
| 964 | // request's own context on the plain path, the move registration's |
| 965 | // lifecycle grant on a host-claim console (whose data boundary refuses an |
| 966 | // unbound claim CAS). held picks the 502's sentence once the claim is |
| 967 | // released: the plain path's invites a retry; the host-claim path's |
| 968 | // (MV-N179, held) does not, because its record-start effect stays uncertain |
| 969 | // and blocks the next one until the claim code can resolve it. |
| 970 | // |
| 971 | // It answers w itself and returns nil in every case but the first. |
| 972 | func (s *Server) settleFailedMoveWrite(w http.ResponseWriter, r *http.Request, claimCtx context.Context, acct *Account, app, host, id, exec string, moving hostclaim.Observed, held bool) *provision.Record { |
| 973 | ctx := r.Context() |
| 974 | unknown := func(why string, err error) { |
| 975 | log.Printf("console: api instance move %s/%s/%s: move %s: %s: %v; the claim is left moving", acct.Sqid, app, host, id, why, err) |
| 976 | jsonError(w, http.StatusBadGateway, "could not confirm whether the move was recorded. read its state before trying again") |
| 977 | } |
| 978 | rec, err := provision.GetRecord(ctx, s.cfg.Releases, acct.Sqid, app, host) |
| 979 | if err != nil { |
| 980 | unknown("record reread", err) |
| 981 | return nil |
| 982 | } |
| 983 | if rec.Move != nil && rec.Move.ID == id { |
| 984 | return rec |
| 985 | } |
| 986 | release := func(c *hostclaim.Claim) error { |
| 987 | if err := hostclaim.RequireMoveOp(id, hostclaim.KindMoving, hostclaim.PhaseMoving)(c); err != nil { |
| 988 | return err |
| 989 | } |
| 990 | return hostclaim.ReleaseMove(id, exec)(c) |
| 991 | } |
| 992 | _, err = hostclaim.Update(claimCtx, s.cfg.Releases, moving, release) |
| 993 | if errors.Is(err, hostclaim.ErrConflict) { |
| 994 | // Something else wrote the claim since the CAS: read it and try |
| 995 | // once more. The guard refuses unless it is still this move. |
| 996 | var cur hostclaim.Observed |
| 997 | if cur, err = hostclaim.Read(ctx, s.cfg.Releases, host); err == nil { |
| 998 | _, err = hostclaim.Update(claimCtx, s.cfg.Releases, cur, release) |
| 999 | } |
| 1000 | } |
| 1001 | if err != nil { |
| 1002 | unknown("claim release", err) |
| 1003 | return nil |
| 1004 | } |
| 1005 | if held { |
| 1006 | jsonError(w, http.StatusBadGateway, "the move could not be recorded and is held until an operator checks the record") |
| 1007 | return nil |
| 1008 | } |
| 1009 | jsonError(w, http.StatusBadGateway, "could not record the move. try again") |
| 1010 | return nil |
| 1011 | } |
| 1012 | |
| 1013 | // handleAPIInstanceMoveCut is ready → cutting/requested under the guard. A |
| 1014 | // move journaled before ids existed (Legacy) has no guard to carry; it can |
| 1015 | // only be aborted (MV-N7 says so). |
| 1016 | func (s *Server) handleAPIInstanceMoveCut(w http.ResponseWriter, r *http.Request) { |
| 1017 | if !s.cfg.MovesEnabled { |
| 1018 | s.legacyHandleAPIInstanceMoveCut(w, r) |
| 1019 | return |
| 1020 | } |
| 1021 | acct, app, host, rec, _, ok := s.moveOperatorGate(w, r) |
| 1022 | if !ok { |
| 1023 | return |
| 1024 | } |
| 1025 | if rec.Move.Legacy() { |
| 1026 | jsonError(w, http.StatusConflict, "this move was started by an older console and has no id. abort it and request it again") |
| 1027 | return |
| 1028 | } |
| 1029 | if s.cfg.HostClaim { |
| 1030 | s.claimConsoleMove(w, r, acct, app, host, rec, hostops.ConsoleMoveCut) |
| 1031 | return |
| 1032 | } |
| 1033 | s.moveTransitionP1(w, r, acct, app, host, provision.MoveGuardedCut(moveGuardOf(rec)), http.StatusAccepted) |
| 1034 | } |
| 1035 | |
| 1036 | // handleAPIInstanceMoveAbort marks the move aborting under the guard |
| 1037 | // (legal in preparing, ready and cutting/requested, §7.11). It has no |
| 1038 | // claim-side effect: the claim stays moving until the source's interim |
| 1039 | // abort deletes staging and releases it (hostclaim.ReleaseMove, Task 14). |
| 1040 | // A Legacy() record takes the legacy MoveAbort (F7(a)), the one transition |
| 1041 | // that accepts it. Once the cut has started the answer is M-C13 (preflight |
| 1042 | // L5) on either console: without the host claim abortBeforeTheCut decides |
| 1043 | // it under the record's CAS; a host-claim console decides it from the record |
| 1044 | // the operator gate read, before claim Plan 10's typed grant |
| 1045 | // (claimConsoleMove), which after the seal would refuse with "the move |
| 1046 | // changed" (final review I5). A stuck move keeps the grant's own answer. |
| 1047 | func (s *Server) handleAPIInstanceMoveAbort(w http.ResponseWriter, r *http.Request) { |
| 1048 | if !s.cfg.MovesEnabled { |
| 1049 | s.legacyHandleAPIInstanceMoveAbort(w, r) |
| 1050 | return |
| 1051 | } |
| 1052 | acct, app, host, rec, _, ok := s.moveOperatorGate(w, r) |
| 1053 | if !ok { |
| 1054 | return |
| 1055 | } |
| 1056 | if rec.Move.Legacy() { |
| 1057 | s.moveTransitionP1(w, r, acct, app, host, legacyAbortOf(*rec.Move), http.StatusAccepted) |
| 1058 | return |
| 1059 | } |
| 1060 | if s.cfg.HostClaim { |
| 1061 | // A console without its claim runtime keeps HC5-N1 for every move |
| 1062 | // (claimConsoleMove), and a record with no move keeps its refusal. |
| 1063 | if a := s.cfg.ClaimAuthority; a != nil && a.Control != nil && rec.Move != nil && moveCutStarted(rec.Move) { |
| 1064 | jsonError(w, http.StatusConflict, moveCutStartedSentence) |
| 1065 | return |
| 1066 | } |
| 1067 | s.claimConsoleMove(w, r, acct, app, host, rec, hostops.ConsoleMoveAbort) |
| 1068 | return |
| 1069 | } |
| 1070 | s.moveTransitionP1(w, r, acct, app, host, abortBeforeTheCut(moveGuardOf(rec)), http.StatusAccepted) |
| 1071 | } |
| 1072 | |
| 1073 | // moveCutStartedSentence is M-C13, the abort door's answer once the cut has |
| 1074 | // started (preflight L5): the move then finishes forward or moves back by |
| 1075 | // itself, and an abort could only confuse the operator about which. |
| 1076 | const moveCutStartedSentence = "This move has started its cut. It will finish or move back on its own." |
| 1077 | |
| 1078 | // moveCutStarted is "past cutting/requested and not stuck": a cut past its |
| 1079 | // request (MoveGuardedAbort's own bar), a landing, a committed move, a |
| 1080 | // rollback or a revert. A stuck move keeps provision's refusal, and |
| 1081 | // relocating and aborting keep theirs. |
| 1082 | func moveCutStarted(m *provision.MoveState) bool { |
| 1083 | switch m.State { |
| 1084 | case provision.MoveCutting: |
| 1085 | return m.CutPhase != provision.CutRequested |
| 1086 | case provision.MoveLanding, provision.MoveCommitted, provision.MoveRollingBack, provision.MoveReverting: |
| 1087 | return true |
| 1088 | } |
| 1089 | return false |
| 1090 | } |
| 1091 | |
| 1092 | // abortBeforeTheCut is MoveGuardedAbort that refuses with M-C13 when the |
| 1093 | // journal it is about to abort, this guard's move, has started its cut. It |
| 1094 | // decides on the record SetRecordMove reads under its CAS, not on the |
| 1095 | // door's earlier read, so a cut that advances in between is still M-C13. |
| 1096 | // Anything else is MoveGuardedAbort's own answer. |
| 1097 | func abortBeforeTheCut(g provision.MoveGuard) func(*provision.Record) error { |
| 1098 | abort := provision.MoveGuardedAbort(g) |
| 1099 | return func(rec *provision.Record) error { |
| 1100 | if m := rec.Move; m != nil && m.ID == g.ID && m.Incarnation == g.Incarnation && moveCutStarted(m) { |
| 1101 | return fmt.Errorf("%w: %s", provision.ErrMoveConflict, moveCutStartedSentence) |
| 1102 | } |
| 1103 | return abort(rec) |
| 1104 | } |
| 1105 | } |
| 1106 | |
| 1107 | // handleAPIInstanceMoveResolve is the operator's door out of a stuck move |
| 1108 | // (M-C11). Its body is {"retry": true} (Plan 4 Task 7, §7.8, §7.10): the |
| 1109 | // record write is provision.MoveRetry under the guard from the record, which |
| 1110 | // answers 409 unless the move is stuck. The retry clears the stuck reason |
| 1111 | // only; the box then resumes at the phase the stuck rollback recorded |
| 1112 | // (a verification runs again and is never bypassed, a probe gets its nine |
| 1113 | // attempts again, a revert resumes its rev_phase). The answer is the flat |
| 1114 | // view: the stuck reason is the CLI's to say (M-C11), read from the record. |
| 1115 | // Any other body is "nothing to resolve yet" until Plan 4 Task 10 and Plan 6 |
| 1116 | // Task 2 give the door the rest. With the switch off it is the 404 the |
| 1117 | // route answered before it existed. |
| 1118 | func (s *Server) handleAPIInstanceMoveResolve(w http.ResponseWriter, r *http.Request) { |
| 1119 | if !s.cfg.MovesEnabled { |
| 1120 | http.NotFound(w, r) |
| 1121 | return |
| 1122 | } |
| 1123 | acct, app, host, rec, addr, ok := s.moveOperatorGate(w, r) |
| 1124 | if !ok { |
| 1125 | return |
| 1126 | } |
| 1127 | if s.cfg.HostClaim { |
| 1128 | s.claimMoveResolve(w, r, acct, app, host, rec, addr) |
| 1129 | return |
| 1130 | } |
| 1131 | var req struct { |
| 1132 | Retry bool `json:"retry"` |
| 1133 | DeclareLost string `json:"declare_lost"` |
| 1134 | } |
| 1135 | if !readJSON(w, r, &req) { |
| 1136 | return |
| 1137 | } |
| 1138 | // A console without the host claim cannot run the claim's loss protocol |
| 1139 | // (claim Plan 10's inspect and declare): MV-N11, nothing written |
| 1140 | // (preflight L7). |
| 1141 | if req.DeclareLost != "" { |
| 1142 | jsonError(w, http.StatusConflict, "declare-lost is not enabled in this deployment yet") |
| 1143 | return |
| 1144 | } |
| 1145 | if !req.Retry { |
| 1146 | jsonError(w, http.StatusConflict, "nothing to resolve yet") |
| 1147 | return |
| 1148 | } |
| 1149 | s.moveTransitionP1(w, r, acct, app, host, provision.MoveRetry(moveGuardOf(rec)), http.StatusAccepted) |
| 1150 | } |
| 1151 | |
| 1152 | // claimMoveResolve is the resolve door on a host-claim console (claim Plan |
| 1153 | // 10, rulings J17 and J4). Its body is exactly one of |
| 1154 | // |
| 1155 | // {"retry": true} the typed retry (claimConsoleMove) |
| 1156 | // {"declare_lost": "<box>"} the signed challenge for the target's loss |
| 1157 | // {"declare_lost": "<box>", "submission": {…}} the declaration of that challenge |
| 1158 | // |
| 1159 | // decoded strictly: an unknown field, or retry beside a declaration, is |
| 1160 | // refused before anything is read. An empty body is still "nothing to |
| 1161 | // resolve yet". Without its claim runtime the console keeps HC5-N1. |
| 1162 | func (s *Server) claimMoveResolve(w http.ResponseWriter, r *http.Request, acct *Account, app, host string, rec *provision.Record, actor string) { |
| 1163 | var req struct { |
| 1164 | Retry bool `json:"retry"` |
| 1165 | DeclareLost string `json:"declare_lost"` |
| 1166 | Submission *hostfence.Submission `json:"submission"` |
| 1167 | } |
| 1168 | if !readJSONStrictLimit(w, r, &req, 1<<20) { |
| 1169 | return |
| 1170 | } |
| 1171 | switch { |
| 1172 | case req.Retry && (req.DeclareLost != "" || req.Submission != nil), req.DeclareLost == "" && req.Submission != nil: |
| 1173 | jsonError(w, http.StatusBadRequest, "invalid resolution request") |
| 1174 | case req.Retry: |
| 1175 | s.claimConsoleMove(w, r, acct, app, host, rec, hostops.ConsoleMoveRetry) |
| 1176 | case req.DeclareLost != "": |
| 1177 | s.claimMoveLoss(w, r, acct, app, host, actor, req.DeclareLost, req.Submission) |
| 1178 | default: |
| 1179 | jsonError(w, http.StatusConflict, "nothing to resolve yet") |
| 1180 | } |
| 1181 | } |
| 1182 | |
| 1183 | // claimMoveLoss inspects (no submission) or declares (a submission) the |
| 1184 | // loss of a move's target through the configured ClaimAuthority, never an |
| 1185 | // authority, writer or store of its own. A declaration answers the move's |
| 1186 | // flat view, like retry. |
| 1187 | func (s *Server) claimMoveLoss(w http.ResponseWriter, r *http.Request, acct *Account, app, host, actor, box string, sub *hostfence.Submission) { |
| 1188 | a := s.cfg.ClaimAuthority |
| 1189 | if a == nil || a.Control == nil { |
| 1190 | jsonError(w, http.StatusConflict, claimMoveUnsupportedMsg) |
| 1191 | return |
| 1192 | } |
| 1193 | ctx := r.Context() |
| 1194 | if sub == nil { |
| 1195 | in, err := a.InspectMoveLoss(ctx, actor, acct.Sqid, app, host, box) |
| 1196 | if err != nil { |
| 1197 | claimMoveLossFailed(w, acct, app, host, err) |
| 1198 | return |
| 1199 | } |
| 1200 | writeJSON(w, http.StatusOK, in) |
| 1201 | return |
| 1202 | } |
| 1203 | if err := s.declareMoveLossLogged(ctx, acct.Sqid, app, host, *sub, func() error { |
| 1204 | return a.DeclareMoveLoss(ctx, actor, acct.Sqid, app, host, box, *sub) |
| 1205 | }); err != nil { |
| 1206 | claimMoveLossFailed(w, acct, app, host, err) |
| 1207 | return |
| 1208 | } |
| 1209 | s.bumpSerial(ctx) |
| 1210 | got, err := provision.GetRecord(ctx, s.cfg.Releases, acct.Sqid, app, host) |
| 1211 | if err != nil { |
| 1212 | log.Printf("console: api instance move %s/%s/%s: declared loss: record reread: %v", acct.Sqid, app, host, err) |
| 1213 | jsonError(w, http.StatusBadGateway, "could not read the instance") |
| 1214 | return |
| 1215 | } |
| 1216 | writeJSON(w, http.StatusAccepted, s.moveViewOf(got)) |
| 1217 | } |
| 1218 | |
| 1219 | // claimMoveLossFailed answers a refused inspection or declaration: a |
| 1220 | // recorded refusal is its sentence, a stale or refused binding is |
| 1221 | // moveChangedSentence, anything else the fixed failure. Errors are logged. |
| 1222 | func claimMoveLossFailed(w http.ResponseWriter, acct *Account, app, host string, err error) { |
| 1223 | switch { |
| 1224 | case errors.Is(err, provision.ErrMoveConflict): |
| 1225 | jsonError(w, http.StatusConflict, phase1Conflict(err)) |
| 1226 | case errors.Is(err, hostresolve.ErrResolve), errors.Is(err, hostfence.ErrChallenge), errors.Is(err, hostfence.ErrObservation), errors.Is(err, hostactors.ErrRoster), |
| 1227 | errors.Is(err, hostclaim.ErrLossRecord), errors.Is(err, hostclaim.ErrStaleOp), errors.Is(err, hostclaim.ErrLossOwnerRequired): |
| 1228 | log.Printf("console: api instance move %s/%s/%s: declared loss refused: %v", acct.Sqid, app, host, err) |
| 1229 | jsonError(w, http.StatusConflict, moveChangedSentence) |
| 1230 | default: |
| 1231 | claimConsoleMoveFailed(w, acct, app, host, err) |
| 1232 | } |
| 1233 | } |
| 1234 | |
| 1235 | // claimConsoleMove is the cut, abort and retry doors on a host-claim console |
| 1236 | // (claim Plan 10, ruling J17): the configured ControlWriter's typed |
| 1237 | // transition, bound to the claim and record observed here. It never builds |
| 1238 | // an authority, a writer or a store. A console without its claim runtime, |
| 1239 | // and a move journal with no id, keep HC5-N1. Every refusal is a recorded |
| 1240 | // sentence; anything else is logged and answers the fixed failure. |
| 1241 | func (s *Server) claimConsoleMove(w http.ResponseWriter, r *http.Request, acct *Account, app, host string, rec *provision.Record, action hostops.ConsoleMoveAction) { |
| 1242 | a := s.cfg.ClaimAuthority |
| 1243 | if a == nil || a.Control == nil || rec.Move.Legacy() { |
| 1244 | jsonError(w, http.StatusConflict, claimMoveUnsupportedMsg) |
| 1245 | return |
| 1246 | } |
| 1247 | if rec.Move == nil { |
| 1248 | jsonError(w, http.StatusConflict, "no journaled move in flight") |
| 1249 | return |
| 1250 | } |
| 1251 | ctx := r.Context() |
| 1252 | o, err := hostops.ObserveConsoleMove(ctx, s.cfg.Releases, acct.Sqid, app, host) |
| 1253 | var got *provision.Record |
| 1254 | if err == nil && (o.OpID != rec.Move.ID || o.Incarnation != rec.Move.Incarnation || o.RecordSeq != rec.RecordSeq) { |
| 1255 | // The gate read a different move, incarnation or record than the |
| 1256 | // claim and record observed now: something replaced it in between. |
| 1257 | err = hostops.ErrIdentity |
| 1258 | } |
| 1259 | if err == nil { |
| 1260 | got, err = a.Control.ConsoleMove(ctx, o, action) |
| 1261 | } |
| 1262 | if err != nil { |
| 1263 | claimConsoleMoveFailed(w, acct, app, host, err) |
| 1264 | return |
| 1265 | } |
| 1266 | s.bumpSerial(ctx) |
| 1267 | writeJSON(w, http.StatusAccepted, s.moveViewOf(got)) |
| 1268 | } |
| 1269 | |
| 1270 | // claimConsoleMoveFailed answers a refused typed move write. A provision |
| 1271 | // refusal is its recorded sentence; a changed claim, record or registration |
| 1272 | // is moveChangedSentence; the error itself is only logged. |
| 1273 | func claimConsoleMoveFailed(w http.ResponseWriter, acct *Account, app, host string, err error) { |
| 1274 | switch { |
| 1275 | case errors.Is(err, provision.ErrMoveConflict): |
| 1276 | jsonError(w, http.StatusConflict, phase1Conflict(err)) |
| 1277 | case errors.Is(err, hostops.ErrIdentity), errors.Is(err, hostops.ErrPending), errors.Is(err, hostclaim.ErrConflict), |
| 1278 | errors.Is(err, hostclaim.ErrWrongState), errors.Is(err, release.ErrCASConflict), errors.Is(err, os.ErrNotExist): |
| 1279 | log.Printf("console: api instance move %s/%s/%s: typed move write refused: %v", acct.Sqid, app, host, err) |
| 1280 | jsonError(w, http.StatusConflict, moveChangedSentence) |
| 1281 | default: |
| 1282 | log.Printf("console: api instance move %s/%s/%s: typed move write: %v", acct.Sqid, app, host, err) |
| 1283 | jsonError(w, http.StatusBadGateway, "could not write the move. try again") |
| 1284 | } |
| 1285 | } |
| 1286 | |
| 1287 | // registerMovesPhase1Routes adds the routes that exist only with the switch |
| 1288 | // on. The Instance role stays on resolve only so a service token meets the |
| 1289 | // family's 404 rather than the role gate's 403. |
| 1290 | func (s *Server) registerMovesPhase1Routes() { |
| 1291 | s.cliRoute("POST /api/cli/apps/{account}/{app}/instances/{host}/move/resolve", svcrole.Allow(svcrole.Instance), s.handleAPIInstanceMoveResolve) |
| 1292 | } |
| 1293 | |
| 1294 | // errMoveChanged refuses a write whose move is no longer the one the door |
| 1295 | // read. It wraps ErrMoveConflict so moveTransition answers 409. |
| 1296 | var errMoveChanged = fmt.Errorf("%w: %s", provision.ErrMoveConflict, moveChangedSentence) |
| 1297 | |
| 1298 | const moveChangedSentence = "the move changed since this request read it. read its state and try again" |
| 1299 | |
| 1300 | // legacyAbortOf is the legacy MoveAbort bound to the legacy move the door |
| 1301 | // read (F7(a)). SetRecordMove reads the record again before it writes, and |
| 1302 | // MoveAbort alone would abort whatever preparing or ready move it found |
| 1303 | // there, so a request delayed past that move's end could abort its |
| 1304 | // replacement. A legacy journal has no id; its target and request time |
| 1305 | // identify it. |
| 1306 | func legacyAbortOf(seen provision.MoveState) func(*provision.Record) error { |
| 1307 | return func(rec *provision.Record) error { |
| 1308 | m := rec.Move |
| 1309 | if m == nil || !m.Legacy() || m.To != seen.To || !m.RequestedAt.Equal(seen.RequestedAt) { |
| 1310 | return errMoveChanged |
| 1311 | } |
| 1312 | return provision.MoveAbort()(rec) |
| 1313 | } |
| 1314 | } |
| 1315 | |
| 1316 | // phase1Conflict maps a provision transition refusal to its recorded |
| 1317 | // sentence (copy record MV-N12 to MV-N16, MV-N38 to MV-N46, and Plan 4's |
| 1318 | // MV-N300 to MV-N309: the rollback, retry and declare-lost refusals and the |
| 1319 | // rollback and stuck reasons): never the |
| 1320 | // "provision: move state conflict:" prefix, never an em dash. A refusal |
| 1321 | // this table does not know answers a fixed sentence, so a new provision |
| 1322 | // sentence cannot reach an operator unreviewed. |
| 1323 | func phase1Conflict(err error) string { |
| 1324 | msg := strings.TrimPrefix(err.Error(), provision.ErrMoveConflict.Error()+": ") |
| 1325 | switch { |
| 1326 | case msg == "past the point of no return \u2014 the cut completes forward": |
| 1327 | // The legacy MoveAbort's sentence, reworded. |
| 1328 | return "past the point of no return. the cut completes forward" |
| 1329 | case strings.Contains(msg, "\u2014"): |
| 1330 | case msg == "no journaled move in flight", |
| 1331 | msg == "malformed move start", |
| 1332 | msg == "past the point of no return. the cut completes forward or moves back on its own", |
| 1333 | msg == "cut is only legal from ready", |
| 1334 | msg == "already aborting", |
| 1335 | msg == "no move in flight", |
| 1336 | msg == moveChangedSentence, |
| 1337 | msg == moveCutStartedSentence, |
| 1338 | // provision.RefusedCommitted, RefusedNotStuck and |
| 1339 | // RefusedNothingToDeclare (TestPhase1ConflictPassesPlan4Refusals |
| 1340 | // holds the literals to the constants). |
| 1341 | msg == "the move has committed. it cannot roll back", |
| 1342 | msg == "the move is not stuck. there is nothing to retry", |
| 1343 | msg == "the target holds nothing the move still needs. there is nothing to declare lost", |
| 1344 | // The reasons Plan 4 records (ruling J9): provision.RollbackDeclaredLost, |
| 1345 | // StuckBaselineMismatch, StuckRecoveryProbeAttempts, StuckRevertProbeBudget, |
| 1346 | // and the shared installer's StuckReturnFilesOccupied, |
| 1347 | // StuckReturnInstallMismatch and StuckReturnUnknownStamp (Task 3), and |
| 1348 | // the target's StuckReturnTargetDatabase, StuckReturnTargetChanged |
| 1349 | // and StuckReturnTargetFiles (Task 4). |
| 1350 | msg == "the target box was declared lost", |
| 1351 | msg == "the source no longer matches what it recorded when it stopped", |
| 1352 | msg == "the recovery probe ran out of attempts", |
| 1353 | msg == "the source did not answer within 15 minutes of moving back", |
| 1354 | msg == "the files directory holds files the return did not write", |
| 1355 | msg == "the returned database does not match the build", |
| 1356 | msg == "the source holds files of an installation the return cannot account for", |
| 1357 | msg == "the target's stopped database is missing", |
| 1358 | msg == "the target changed after it stopped", |
| 1359 | msg == "the target's files cannot be archived", |
| 1360 | strings.HasPrefix(msg, "the record's move is ") && strings.Contains(msg, " for incarnation "), |
| 1361 | strings.HasPrefix(msg, "a move to ") && strings.Contains(msg, " is already "), |
| 1362 | strings.HasPrefix(msg, "the instance is already in "), |
| 1363 | strings.HasPrefix(msg, "a move in state ") && strings.HasSuffix(msg, " cannot be aborted"), |
| 1364 | strings.HasPrefix(msg, "a ") && strings.HasSuffix(msg, " does not move. delete it and create one in the new region"): |
| 1365 | return msg |
| 1366 | } |
| 1367 | return "this move's state does not allow that. read its state and try again" |
| 1368 | } |
| 1369 | |
| 1370 | // claimMoveRosterPlaces is BeginClaimMove's roster precondition, read from |
| 1371 | // the door's own observation: both boxes have active, enrolled entries |
| 1372 | // under their box references, they differ, the source is in the record's |
| 1373 | // region and the target in to, which is not the source's region. |
| 1374 | // |
| 1375 | // The roster and the record name regions differently for one region, the |
| 1376 | // home: the roster refuses an empty region, so a home box is enrolled under |
| 1377 | // the home's name (CONSOLE_HOME_REGION, as boxchoice reads it), while the |
| 1378 | // record keeps "". Both boxes' roster regions are read as record codes |
| 1379 | // through homePlacementCode with home, the deployment's home name (the rule |
| 1380 | // homeRegionIn applies), so a move out of home is placed and a named region |
| 1381 | // is compared as it is. With no home named nothing translates, and a home |
| 1382 | // source is refused as before (home-region parks, part 1). |
| 1383 | func claimMoveRosterPlaces(r hostactors.Roster, source, target, recRegion, to string, home ...string) bool { |
| 1384 | return hostops.ClaimMoveRosterPlaces(r, source, target, recRegion, to, home...) |
| 1385 | } |
| 1386 | |
| 1387 | // sourceEnrollment is the enrollment of the box that executes the claim, and |
| 1388 | // whether the roster lets a move start from it: an entry under that box |
| 1389 | // reference (never a label), in state "active", with a non-empty |
| 1390 | // enrollment. The termination fence is a separate check on the same |
| 1391 | // observation. |
| 1392 | func sourceEnrollment(r hostactors.Roster, box string) (string, bool) { |
| 1393 | b, ok := r.Boxes[box] |
| 1394 | if !ok || b.State != "active" || b.Enrollment == "" { |
| 1395 | return "", false |
| 1396 | } |
| 1397 | return b.Enrollment, true |
| 1398 | } |
| 1399 | |
| 1400 | // ---- GET …/move?detail=1 (Plan 6 Task 2, spec §12) ---- |
| 1401 | |
| 1402 | // moveDetail is the operator's detail answer: the flat view, unchanged, and |
| 1403 | // beside it the detail object. Everything in it is a record field, a claim |
| 1404 | // field, a report the boxes published for the CLI, or an object key: never |
| 1405 | // an evidence or log body, and never a launch environment value. The |
| 1406 | // console cannot tell an HMAC from a raw value of the same shape in a |
| 1407 | // malformed report, so neither report's env section passes any value: the |
| 1408 | // source's environment is names and kinds (moveSourceReport), and the |
| 1409 | // target's env input is blanked. |
| 1410 | type moveDetail struct { |
| 1411 | moveView |
| 1412 | Detail moveDetailBody `json:"detail"` |
| 1413 | } |
| 1414 | |
| 1415 | type moveDetailBody struct { |
| 1416 | // Move is the record's journal as read; nil when no move is in flight. |
| 1417 | Move *provision.MoveState `json:"move,omitempty"` |
| 1418 | // Claim is the host claim's operation; nil when the host has no claim. |
| 1419 | Claim *moveClaimView `json:"claim,omitempty"` |
| 1420 | // TargetReport and SourceReport are the reports under the move's report |
| 1421 | // prefix, nil when absent or unreadable as a report. |
| 1422 | TargetReport *moves.TargetReport `json:"target_report,omitempty"` |
| 1423 | SourceReport *moveSourceReport `json:"source_report,omitempty"` |
| 1424 | // Evidence names the stop evidence the record points at, by key. |
| 1425 | Evidence moveEvidenceKeys `json:"evidence"` |
| 1426 | // Log is every key under the move's log/ prefix, sorted (preflight L26: |
| 1427 | // a move writes a few dozen; the CLI reads bodies only on demand). |
| 1428 | Log []string `json:"log"` |
| 1429 | // Timings are a closed move's timings against their budgets (Plan 6 |
| 1430 | // Task 4, moves.Timings), in seconds: the timing fields of its done or |
| 1431 | // rolled_back log entry and nothing else of it, as the entry's writer |
| 1432 | // worked them out from the record's journal. Absent until that entry |
| 1433 | // is logged. |
| 1434 | Timings map[string]float64 `json:"timings,omitempty"` |
| 1435 | // WaitingForBox is the region a relocating move waits on for a live |
| 1436 | // enrolled box (M-C22), empty otherwise and when it cannot be told. |
| 1437 | WaitingForBox string `json:"waiting_for_box,omitempty"` |
| 1438 | // From is the move's origin and Region the record's region now, each |
| 1439 | // named as the caller names it (homeRegionOut), so the CLI's sentences |
| 1440 | // (M-C1, M-C7, M-C9) never print the record's "" for the home. |
| 1441 | From string `json:"from,omitempty"` |
| 1442 | Region string `json:"region"` |
| 1443 | } |
| 1444 | |
| 1445 | // moveSourceReport is the source report with its environment replaced by |
| 1446 | // one entry per variable: the name and its T3 kind (moves.Class). The |
| 1447 | // report's Env (per-name HMACs and their digest) is shadowed by this Env |
| 1448 | // and never encoded. |
| 1449 | type moveSourceReport struct { |
| 1450 | moves.SourceReport |
| 1451 | Env []moveEnvEntry `json:"env"` |
| 1452 | } |
| 1453 | |
| 1454 | // moveEnvEntry is a launch environment variable as the detail shows it. The |
| 1455 | // reports compare values only by HMAC and record no per-name result, so |
| 1456 | // the kind is all there is to say beside the name. |
| 1457 | type moveEnvEntry struct { |
| 1458 | Name string `json:"name"` |
| 1459 | Kind string `json:"kind"` |
| 1460 | } |
| 1461 | |
| 1462 | // sourceReportWithoutValues is the source report as the detail shows it. |
| 1463 | func sourceReportWithoutValues(r moves.SourceReport) *moveSourceReport { |
| 1464 | names := make([]string, 0, len(r.Env.Vars)) |
| 1465 | for name := range r.Env.Vars { |
| 1466 | names = append(names, name) |
| 1467 | } |
| 1468 | slices.Sort(names) |
| 1469 | out := &moveSourceReport{Env: make([]moveEnvEntry, 0, len(names))} |
| 1470 | for _, name := range names { |
| 1471 | out.Env = append(out.Env, moveEnvEntry{Name: name, Kind: moves.Class(name)}) |
| 1472 | } |
| 1473 | r.Env = moves.EnvReport{} |
| 1474 | out.SourceReport = r |
| 1475 | return out |
| 1476 | } |
| 1477 | |
| 1478 | // moveClaimView is the claim operation's shape and nothing else (no journal, |
| 1479 | // no holder). Target is the move's target box, Intent["target"], which M-C10 |
| 1480 | // names (preflight L28). |
| 1481 | type moveClaimView struct { |
| 1482 | Kind string `json:"kind"` |
| 1483 | Phase string `json:"phase,omitempty"` |
| 1484 | Exec string `json:"exec,omitempty"` |
| 1485 | Participants []string `json:"participants,omitempty"` |
| 1486 | Target string `json:"target,omitempty"` |
| 1487 | } |
| 1488 | |
| 1489 | // moveEvidenceKeys are the stop evidence keys the record carries: the |
| 1490 | // source's (Move.Source.Evidence, written at stopped) and the target's |
| 1491 | // (Move.Rollback.Evidence, written when a rollback released it). |
| 1492 | type moveEvidenceKeys struct { |
| 1493 | Source string `json:"source,omitempty"` |
| 1494 | Target string `json:"target,omitempty"` |
| 1495 | } |
| 1496 | |
| 1497 | // moveRelocationRegion is the region of a relocating claim operation that |
| 1498 | // belongs to this record's move (the same holder incarnation), and whether |
| 1499 | // there is one. |
| 1500 | func moveRelocationRegion(rec *provision.Record, obs hostclaim.Observed) (string, bool) { |
| 1501 | if rec == nil || rec.Move == nil || obs.Absent || obs.Claim == nil || obs.Claim.Holder == nil { |
| 1502 | return "", false |
| 1503 | } |
| 1504 | c := obs.Claim |
| 1505 | if c.Op.Kind != hostclaim.KindRelocating || c.Holder.Incarnation != rec.Move.Incarnation || c.Op.Intent["region"] == "" { |
| 1506 | return "", false |
| 1507 | } |
| 1508 | return c.Op.Intent["region"], true |
| 1509 | } |
| 1510 | |
| 1511 | // moveConvertedAfterCommit is Paul's J3 as the status door sees it: the |
| 1512 | // record says committed and the claim is relocating this incarnation in the |
| 1513 | // move's target region, because the target was terminated after the commit. |
| 1514 | // The door answers relocating for it. |
| 1515 | func moveConvertedAfterCommit(rec *provision.Record, obs hostclaim.Observed) bool { |
| 1516 | region, ok := moveRelocationRegion(rec, obs) |
| 1517 | return ok && rec.Move.State == provision.MoveCommitted && park.PlacementMatches(obs.Claim.Op.Intent["home_region"], rec.Move.To, region) |
| 1518 | } |
| 1519 | |
| 1520 | // writeMoveDetail answers an operator's detail=1. The claim read and the |
| 1521 | // report and log reads fail closed with a fixed sentence; a report that does |
| 1522 | // not parse is left out, as the boxes treat it (no report). The region |
| 1523 | // check behind waiting_for_box never fails the answer: when it cannot tell, |
| 1524 | // the field is left out and the state still says relocating. |
| 1525 | func (s *Server) writeMoveDetail(w http.ResponseWriter, r *http.Request, acct *Account, app, host string, rec *provision.Record, view moveView) { |
| 1526 | ctx := r.Context() |
| 1527 | obs, err := hostclaim.Read(ctx, s.cfg.Releases, host) |
| 1528 | if err != nil { |
| 1529 | log.Printf("console: api instance move %s/%s/%s: detail: claim read: %v", acct.Sqid, app, host, err) |
| 1530 | jsonError(w, http.StatusBadGateway, "could not read the host claim") |
| 1531 | return |
| 1532 | } |
| 1533 | out := moveDetail{moveView: view, Detail: moveDetailBody{Move: rec.Move, Log: []string{}, Region: s.homeRegionOut(rec.Region)}} |
| 1534 | if rec.Move != nil { |
| 1535 | out.Detail.From = s.homeRegionOut(rec.Move.From) |
| 1536 | } |
| 1537 | if !obs.Absent && obs.Claim != nil { |
| 1538 | op := obs.Claim.Op |
| 1539 | out.Detail.Claim = &moveClaimView{Kind: op.Kind, Phase: op.Phase, Exec: op.Exec, Participants: op.Participants, Target: op.Intent["target"]} |
| 1540 | } |
| 1541 | if m := rec.Move; m != nil { |
| 1542 | if m.Source != nil { |
| 1543 | out.Detail.Evidence.Source = m.Source.Evidence |
| 1544 | } |
| 1545 | if m.Rollback != nil { |
| 1546 | out.Detail.Evidence.Target = m.Rollback.Evidence |
| 1547 | } |
| 1548 | } |
| 1549 | // The reports and the log of the move in flight, or of the last one to |
| 1550 | // close. A legacy journal has no id and so no report prefix. |
| 1551 | opID := "" |
| 1552 | if rec.Move != nil { |
| 1553 | opID = rec.Move.ID |
| 1554 | } else if rec.LastMove != nil { |
| 1555 | opID = rec.LastMove.ID |
| 1556 | } |
| 1557 | if provision.ValidMoveID(opID) { |
| 1558 | prefix := moves.ReportPrefix(rec.Account, rec.App, rec.Host, opID) |
| 1559 | var target moves.TargetReport |
| 1560 | var source moves.SourceReport |
| 1561 | tok, terr := s.readMoveReport(ctx, moves.TargetReportKey(prefix), &target) |
| 1562 | sok, serr := s.readMoveReport(ctx, moves.SourceReportKey(prefix), &source) |
| 1563 | keys, lerr := s.cfg.Releases.List(ctx, prefix+"log/") |
| 1564 | if err := errors.Join(terr, serr, lerr); err != nil { |
| 1565 | log.Printf("console: api instance move %s/%s/%s: detail: reports of %s: %v", acct.Sqid, app, host, opID, err) |
| 1566 | jsonError(w, http.StatusBadGateway, "could not read the move's reports. try again") |
| 1567 | return |
| 1568 | } |
| 1569 | if tok { |
| 1570 | target.Inputs.EnvReport = "" |
| 1571 | out.Detail.TargetReport = &target |
| 1572 | } |
| 1573 | if sok { |
| 1574 | out.Detail.SourceReport = sourceReportWithoutValues(source) |
| 1575 | } |
| 1576 | slices.Sort(keys) |
| 1577 | out.Detail.Log = append(out.Detail.Log, keys...) |
| 1578 | if rec.Move == nil && rec.LastMove != nil { |
| 1579 | timings, err := s.readMoveTimings(ctx, prefix, rec.LastMove.Outcome) |
| 1580 | if err != nil { |
| 1581 | log.Printf("console: api instance move %s/%s/%s: detail: timings of %s: %v", acct.Sqid, app, host, opID, err) |
| 1582 | jsonError(w, http.StatusBadGateway, "could not read the move's reports. try again") |
| 1583 | return |
| 1584 | } |
| 1585 | out.Detail.Timings = timings |
| 1586 | } |
| 1587 | } |
| 1588 | if (rec.Move != nil && rec.Move.State == provision.MoveRelocating) || moveConvertedAfterCommit(rec, obs) { |
| 1589 | out.State = provision.MoveRelocating |
| 1590 | // Any phase that runs on no box (starting, releasing and pinning |
| 1591 | // today, hostclaim's validateRelocating) is waiting for one. |
| 1592 | if region, ok := moveRelocationRegion(rec, obs); ok && obs.Claim.Op.Exec == "none" { |
| 1593 | code, valid := park.EnginePlacement(obs.Claim.Op.Intent["home_region"], region) |
| 1594 | live, known := false, false |
| 1595 | if valid { |
| 1596 | live, known = s.regionHasLiveBox(ctx, code) |
| 1597 | } |
| 1598 | if known && !live { |
| 1599 | out.Detail.WaitingForBox = s.homeRegionOut(code) |
| 1600 | } |
| 1601 | } |
| 1602 | } |
| 1603 | writeJSON(w, http.StatusOK, out) |
| 1604 | } |
| 1605 | |
| 1606 | // readMoveTimings is the timings a closed move's terminal log entry carries |
| 1607 | // (moves.TimingsOf over its Fields): nil for an outcome that carries none |
| 1608 | // (aborted), for an entry not logged yet, and for one that does not parse |
| 1609 | // (logged); an error when the read failed. |
| 1610 | func (s *Server) readMoveTimings(ctx context.Context, prefix, outcome string) (map[string]float64, error) { |
| 1611 | if outcome != "done" && outcome != "rolled_back" { |
| 1612 | return nil, nil |
| 1613 | } |
| 1614 | var ev moves.LogEvent |
| 1615 | ok, err := s.readMoveReport(ctx, moves.LogKeyFor(prefix, outcome), &ev) |
| 1616 | if !ok || err != nil { |
| 1617 | return nil, err |
| 1618 | } |
| 1619 | if t := moves.TimingsOf(ev.Fields); len(t) > 0 { |
| 1620 | return t, nil |
| 1621 | } |
| 1622 | return nil, nil |
| 1623 | } |
| 1624 | |
| 1625 | // readMoveReport reads one report: (false, nil) when it is absent or does |
| 1626 | // not parse (logged), an error when the read failed. |
| 1627 | func (s *Server) readMoveReport(ctx context.Context, key string, into any) (bool, error) { |
| 1628 | body, _, err := s.cfg.Releases.Get(ctx, key) |
| 1629 | if errors.Is(err, os.ErrNotExist) { |
| 1630 | return false, nil |
| 1631 | } |
| 1632 | if err != nil { |
| 1633 | return false, fmt.Errorf("read %s: %w", key, err) |
| 1634 | } |
| 1635 | if err := json.Unmarshal(body, into); err != nil { |
| 1636 | log.Printf("console: move report %s does not parse: %v", key, err) |
| 1637 | return false, nil |
| 1638 | } |
| 1639 | return true, nil |
| 1640 | } |
| 1641 | |
| 1642 | // regionHasLiveBox reports whether the region (record vocabulary) has a live |
| 1643 | // enrolled box, through the region HA contracts (Task 2 rulings, L18): |
| 1644 | // boxchoice.Eligible (eligibleBoxes) over the live edges snapshot, with the |
| 1645 | // roster read through hostactors.Directory, never directly. A roster entry |
| 1646 | // counts when it is active, has an enrollment, is in the region, and has no |
| 1647 | // termination fence; its placement and cloud are the roster's. The AWS |
| 1648 | // region is the one the code's eligible boxes publish (regionCands), as the |
| 1649 | // move door selects. known is false when the snapshot is incomplete or the |
| 1650 | // roster or a fence cannot be read or trusted: then nothing is concluded. |
| 1651 | func (s *Server) regionHasLiveBox(ctx context.Context, region string) (live, known bool) { |
| 1652 | snap, err := s.liveEdges(ctx) |
| 1653 | if err != nil || !snap.EvidenceComplete() { |
| 1654 | log.Printf("console: move detail: the edges snapshot of %s is incomplete (%v)", region, err) |
| 1655 | return false, false |
| 1656 | } |
| 1657 | dir := hostactors.Directory{Store: s.cfg.Releases, PublicKey: s.cfg.ReleasePub, Bucket: s.cfg.Bucket} |
| 1658 | o, err := dir.Read(ctx) |
| 1659 | if err != nil { |
| 1660 | log.Printf("console: move detail: host roster: %v", err) |
| 1661 | return false, false |
| 1662 | } |
| 1663 | now := time.Now() |
| 1664 | roster := map[string]rosterBox{} |
| 1665 | for ref, b := range o.Roster.Boxes { |
| 1666 | if b.State != "active" || b.Enrollment == "" || s.homeRegionIn(b.Region) != region { |
| 1667 | continue |
| 1668 | } |
| 1669 | fenced, err := dir.TerminationFence(ctx, o, ref, now) |
| 1670 | if err != nil { |
| 1671 | log.Printf("console: move detail: termination fence of %s: %v", ref, err) |
| 1672 | return false, false |
| 1673 | } |
| 1674 | if !fenced { |
| 1675 | roster[ref] = rosterBox{Placement: b.Region, Cloud: b.Cloud} |
| 1676 | } |
| 1677 | } |
| 1678 | aws := "" |
| 1679 | for _, c := range s.regionCands(snap) { |
| 1680 | if c.Code == region { |
| 1681 | aws = c.AWSRegion |
| 1682 | } |
| 1683 | } |
| 1684 | return len(eligibleBoxes(snap, roster, region, aws, s.cfg.dbClaimHomeRegion(), s.expectedSecretsFP(), nil)) > 0, true |
| 1685 | } |
| 1686 | |
| 1687 | // ---- the delete door during a move (Plan 6 Task 2b, preflight L6) ---- |
| 1688 | |
| 1689 | // moveDeleteRefusal is M-C19, what the delete door says for a host that is |
| 1690 | // being moved. |
| 1691 | func moveDeleteRefusal(host string) string { |
| 1692 | return host + " is being moved. Delete it after the move finishes, or abort the move first." |
| 1693 | } |
| 1694 | |
| 1695 | // moveClaimKinds are the claim operations a move runs under. A relocating |
| 1696 | // operation is a move's only when the record still journals the move (a |
| 1697 | // region HA relocation of its own is not one). |
| 1698 | var moveClaimKinds = map[string]bool{hostclaim.KindMoving: true, hostclaim.KindLanding: true, hostclaim.KindReverting: true, hostclaim.KindRestoring: true} |
| 1699 | |
| 1700 | // refuseDeleteDuringMove is the delete door's move check, with the switch |
| 1701 | // on: before any retire effect, a host that is being moved answers 409 |
| 1702 | // M-C19 and nothing is written. On a console without the host claim that is |
| 1703 | // a record carrying a move (any state, legacy included); on a host-claim |
| 1704 | // console it is a claim whose operation is a move's. The claim would refuse |
| 1705 | // the retire anyway; this check only says why, before the retire registers |
| 1706 | // anything. A record that is not there is main's idempotent delete. A read |
| 1707 | // that fails is a 502 (HC5-N3), never a pass. It reports whether it |
| 1708 | // answered. |
| 1709 | func (s *Server) refuseDeleteDuringMove(ctx context.Context, w http.ResponseWriter, acct, app, host string) bool { |
| 1710 | if !s.cfg.MovesEnabled { |
| 1711 | return false |
| 1712 | } |
| 1713 | rec, err := provision.GetRecord(ctx, s.cfg.Releases, acct, app, host) |
| 1714 | if errors.Is(err, os.ErrNotExist) { |
| 1715 | return false |
| 1716 | } |
| 1717 | if err != nil { |
| 1718 | log.Printf("console: api instances delete %s/%s/%s: move check: record: %v", acct, app, host, err) |
| 1719 | jsonError(w, http.StatusBadGateway, "could not read the instance") |
| 1720 | return true |
| 1721 | } |
| 1722 | moving := rec.Move != nil |
| 1723 | if s.cfg.HostClaim { |
| 1724 | obs, err := hostclaim.Read(ctx, s.cfg.Releases, host) |
| 1725 | if err != nil { |
| 1726 | log.Printf("console: api instances delete %s/%s/%s: move check: claim: %v", acct, app, host, err) |
| 1727 | jsonError(w, http.StatusBadGateway, "could not read the instance") |
| 1728 | return true |
| 1729 | } |
| 1730 | kind := "" |
| 1731 | if !obs.Absent && obs.Claim != nil { |
| 1732 | kind = obs.Claim.Op.Kind |
| 1733 | } |
| 1734 | moving = moveClaimKinds[kind] || (kind == hostclaim.KindRelocating && rec.Move != nil) |
| 1735 | } |
| 1736 | if !moving { |
| 1737 | return false |
| 1738 | } |
| 1739 | jsonError(w, http.StatusConflict, moveDeleteRefusal(host)) |
| 1740 | return true |
| 1741 | } |
| 1742 | |
amadan