amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package provision
2
3import (
4 "errors"
5 "fmt"
6 "slices"
7 "time"
8)
9
10// The offline operator move (docs/superpowers/plans/2026-10-20-offline-
11// operator-move.md, Task 1). One hibernating instance moves between two
12// regions with planned downtime: the operator stops it, the move copies a
13// checked list of everything it depends on, and only then does the target
14// open it. The record carries the move as move.kind "offline" through the
15// states below, each advanced by exactly one transition in this file.
16//
17// Kinds never cross. Nothing here touches a legacy or Phase 1 move (every
18// transition refuses a move whose kind is not offline), and no legacy or
19// Phase 1 transition accepts one (the legacy transitions name the states
20// they admit, none of which is an offline state; MoveGuard.check refuses the
21// kind outright).
22//
23// No code in this file Puts a record. Each transition is a mut for
24// SetRecordMove, so the write is provision's own read-CAS-write, which
25// advances record_seq. Across a whole move the record changes only in
26// region, move, last_move, record_seq and move.updated_at.
27
28// MoveKindOffline is MoveState.Kind for an offline operator move. Every other
29// move leaves Kind empty.
30const MoveKindOffline = "offline"
31
32// The offline move's states, in the order a move takes them:
33//
34// [offline-staging ->] offline-stopping -> offline-sealing -> offline-sealed
35// -> offline-landing -> offline-landed -> offline-open -> offline-finishing
36// -> (cleared)
37//
38// offline-staging is optional: OfflineStage creates the move there instead
39// of in stopping, and the one source box that stages (moves.OfflineStager)
40// pre-copies the replica into the target's staging prefix while the
41// instance keeps serving. Nothing holds, fences or tombstones the instance
42// in staging. OfflineStartStaged then takes it to stopping, and the seal's
43// own round finds the pre-copied objects and skips them by their receipts.
44// offline-reverting is abort's state, reached from staging through landed.
45const (
46 OfflineStaging = "offline-staging"
47 OfflineStopping = "offline-stopping"
48 OfflineSealing = "offline-sealing"
49 OfflineSealed = "offline-sealed"
50 OfflineLanding = "offline-landing"
51 OfflineLanded = "offline-landed"
52 OfflineOpen = "offline-open"
53 OfflineFinishing = "offline-finishing"
54 OfflineReverting = "offline-reverting"
55)
56
57// The outcomes OfflineFinish leaves in LastMove.
58const (
59 OfflineOutcomeDone = "done"
60 OfflineOutcomeAborted = "aborted"
61)
62
63// The offline transitions' refusals that can reach an operator. Each is a
64// recorded sentence (copy rows MV-O34 to MV-O43); a console door maps an
65// offline ErrMoveConflict onto them or onto its own rows (MV-O24, MV-O25,
66// MV-O27) and never sends err.Error() through unchecked. The refusals that
67// name a value are written in place as one format.
68const (
69 // refusedOfflineKind is MoveGuard.check's refusal of an offline move
70 // (MV-O42).
71 refusedOfflineKind = "this is an offline move. use carlos move offline"
72 // refusedNotOffline: an offline transition met a legacy or Phase 1 move
73 // (MV-O34).
74 refusedNotOffline = "this is not an offline move"
75 // refusedOtherOfflineMove: the record's offline move has another id
76 // (MV-O35).
77 refusedOtherOfflineMove = "the record carries a different offline move"
78 // refusedOfflineSeal: a seal that names no sealer, no boxes, or a sealer
79 // that is not one of them (MV-O37).
80 refusedOfflineSeal = "a seal names the box that seals and the boxes that stopped the instance. the box that seals is one of them"
81)
82
83// errRecordUnchanged is what a transition returns to say "the record already
84// says this": updateRecord then writes nothing and returns the record as read
85// (no record_seq advance, no updated_at stamp). Only OfflineAbort from
86// reverting uses it. Applied outside updateRecord the mut returns it as an
87// error, so a transition that can return it is called only through
88// SetRecordMove.
89var errRecordUnchanged = errors.New("provision: the record already says this")
90
91// Offline reports whether this is an offline operator move. Nil-safe, so a
92// record at rest answers false.
93func (m *MoveState) Offline() bool { return m != nil && m.Kind == MoveKindOffline }
94
95// offlineAt checks what every offline transition checks first: the record
96// carries an offline move, it is the move the caller means, and it is in the
97// one state the transition leaves. Every failure is ErrMoveConflict.
98func offlineAt(rec *Record, id, from string) error {
99 if err := offlineIs(rec, id); err != nil {
100 return err
101 }
102 if rec.Move.State != from {
103 return fmt.Errorf("%w: %s", ErrMoveConflict, offlineWrongState(rec.Move.State, from))
104 }
105 return nil
106}
107
108// offlineIs is offlineAt without the state.
109func offlineIs(rec *Record, id string) error {
110 if rec.Move == nil {
111 return fmt.Errorf("%w: %s", ErrMoveConflict, refusedNoMove)
112 }
113 if !rec.Move.Offline() {
114 return fmt.Errorf("%w: %s", ErrMoveConflict, refusedNotOffline)
115 }
116 if rec.Move.ID != id {
117 return fmt.Errorf("%w: %s", ErrMoveConflict, refusedOtherOfflineMove)
118 }
119 return nil
120}
121
122// OfflineStart begins an offline move from region `from` to region `to`
123// (either may be "", the home placement). It carries MoveRequest's refusals:
124// a seeded record, an open run, a move already in flight, the same region and
125// a replica. It also refuses a `from` that is not the record's region, and a
126// malformed id. It stamps kind, id, from, to, state, requested_by and
127// requested_at; it does not touch the region.
128func OfflineStart(id, from, to, by string, now time.Time) func(*Record) error {
129 return offlineBegin(id, from, to, by, now, OfflineStopping)
130}
131
132// StageTarget is what a stage fixes for the whole move: the one box that
133// copies (Stager) and where the copy goes (Bucket, the target's data
134// bucket, in Region).
135type StageTarget struct{ Stager, Bucket, Region string }
136
137// OfflineStage is OfflineStart into offline-staging: the same refusals and
138// the same stamps, plus staged_at, the stager and the staging destination
139// (a stage that names any of them empty is malformed). The instance keeps
140// serving.
141func OfflineStage(id, from, to, by string, st StageTarget, now time.Time) func(*Record) error {
142 begin := offlineBegin(id, from, to, by, now, OfflineStaging)
143 return func(rec *Record) error {
144 if st.Stager == "" || st.Bucket == "" {
145 return fmt.Errorf("%w: %s", ErrMoveConflict, refusedMalformedStart)
146 }
147 if err := begin(rec); err != nil {
148 return err
149 }
150 rec.Move.Stager, rec.Move.StageBucket, rec.Move.StageRegion = st.Stager, st.Bucket, st.Region
151 return nil
152 }
153}
154
155func offlineBegin(id, from, to, by string, now time.Time, state string) func(*Record) error {
156 return func(rec *Record) error {
157 if err := seedMoveRefusal(rec); err != nil {
158 return err
159 }
160 if rec.ActiveRun.Open() {
161 return &MoveRunOpenError{Run: *rec.ActiveRun}
162 }
163 if rec.Move != nil {
164 return fmt.Errorf("%w: a move to %s is already %s", ErrMoveConflict, rec.Move.To, rec.Move.State)
165 }
166 if !moveIDShape.MatchString(id) || now.IsZero() {
167 return fmt.Errorf("%w: %s", ErrMoveConflict, refusedMalformedStart)
168 }
169 if rec.Region == to {
170 return fmt.Errorf("%w: the instance is already in %s", ErrMoveConflict, regionWord(rec.Region))
171 }
172 if from != rec.Region {
173 return fmt.Errorf("%w: %s", ErrMoveConflict, offlineNotFrom(rec.Region, from))
174 }
175 if rec.Role != "" {
176 return fmt.Errorf("%w: a %s does not move. delete it and create one in the new region", ErrMoveConflict, rec.Role)
177 }
178 rec.Move = &MoveState{
179 Kind: MoveKindOffline,
180 ID: id,
181 From: from,
182 To: to,
183 State: state,
184 RequestedBy: by,
185 RequestedAt: now.UTC(),
186 }
187 if state == OfflineStaging {
188 t := now.UTC()
189 rec.Move.StagedAt = &t
190 }
191 return nil
192 }
193}
194
195// OfflineStartStaged is staging to stopping: the operator takes a staged
196// instance offline. The record's region must still be the move's source.
197// It stamps requested_by and requested_at anew, since the instance is
198// offline from now, and requested_at is always after staged_at, which is
199// how OfflineFinish tells a move that started from one that never did.
200func OfflineStartStaged(id, by string, now time.Time) func(*Record) error {
201 return func(rec *Record) error {
202 if err := offlineAt(rec, id, OfflineStaging); err != nil {
203 return err
204 }
205 if rec.Region != rec.Move.From {
206 return fmt.Errorf("%w: %s", ErrMoveConflict, offlineRegionMoved(rec.Region))
207 }
208 if now.IsZero() {
209 return fmt.Errorf("%w: %s", ErrMoveConflict, refusedMalformedStart)
210 }
211 t := now.UTC()
212 if s := rec.Move.StagedAt; s != nil && !t.After(*s) {
213 t = s.Add(time.Nanosecond)
214 }
215 rec.Move.State, rec.Move.RequestedBy, rec.Move.RequestedAt = OfflineStopping, by, t
216 return nil
217 }
218}
219
220// OfflineRestage is staging to staging: the operator asked for another
221// pre-copy round. It moves staged_at to now (always forward), which is the
222// request the stager answers with exactly one round (the stager report's
223// stage_request names the staged_at it served). requested_at stays, so the
224// move still reads as never started.
225func OfflineRestage(id string, now time.Time) func(*Record) error {
226 return func(rec *Record) error {
227 if err := offlineAt(rec, id, OfflineStaging); err != nil {
228 return err
229 }
230 if rec.Region != rec.Move.From {
231 return fmt.Errorf("%w: %s", ErrMoveConflict, offlineRegionMoved(rec.Region))
232 }
233 if now.IsZero() {
234 return fmt.Errorf("%w: %s", ErrMoveConflict, refusedMalformedStart)
235 }
236 t := now.UTC()
237 if s := rec.Move.StagedAt; s != nil && !t.After(*s) {
238 t = s.Add(time.Nanosecond)
239 }
240 rec.Move.StagedAt = &t
241 return nil
242 }
243}
244
245// StartedOffline reports whether the move ever took its instance offline:
246// a move created in stopping did at once, a staged one at OfflineStartStaged.
247func (m *MoveState) StartedOffline() bool {
248 return m.StagedAt == nil || m.RequestedAt.After(*m.StagedAt)
249}
250
251// regionWord is a region as a sentence names it: the home placement's empty
252// code reads "home".
253func regionWord(r string) string {
254 if r == "" {
255 return "home"
256 }
257 return r
258}
259
260// OfflineSeal is stopping to sealing: every source box has confirmed the stop
261// and the operator asked for the copy. It names the sealer (the one box that
262// copies) and the source boxes whose reports the later doors wait for.
263func OfflineSeal(id, sealer string, boxes []string) func(*Record) error {
264 return func(rec *Record) error {
265 if err := offlineAt(rec, id, OfflineStopping); err != nil {
266 return err
267 }
268 if sealer == "" || len(boxes) == 0 || !slices.Contains(boxes, sealer) {
269 return fmt.Errorf("%w: %s", ErrMoveConflict, refusedOfflineSeal)
270 }
271 rec.Move.State = OfflineSealing
272 rec.Move.Sealer = sealer
273 rec.Move.Boxes = append([]string(nil), boxes...)
274 return nil
275 }
276}
277
278// OfflineSealedBy is sealing to sealed: the sealer recorded what it copied.
279// Only the box the seal named may write it. src is copied, so the journal
280// never aliases the caller's value.
281func OfflineSealedBy(id, sealer string, src MoveSource) func(*Record) error {
282 return func(rec *Record) error {
283 if err := offlineAt(rec, id, OfflineSealing); err != nil {
284 return err
285 }
286 if sealer == "" || rec.Move.Sealer != sealer {
287 return fmt.Errorf("%w: %s", ErrMoveConflict, offlineSealedBy(rec.Move.Sealer))
288 }
289 s := src
290 if src.Files != nil {
291 f := *src.Files
292 s.Files = &f
293 }
294 rec.Move.Source = &s
295 rec.Move.State = OfflineSealed
296 return nil
297 }
298}
299
300// OfflineFlip is sealed to landing, and the one record write that moves the
301// region: region = to. The record's region must still be the move's source,
302// so a record that was changed under the move is refused rather than flipped.
303func OfflineFlip(id string) func(*Record) error {
304 return func(rec *Record) error {
305 if err := offlineAt(rec, id, OfflineSealed); err != nil {
306 return err
307 }
308 if rec.Region != rec.Move.From {
309 return fmt.Errorf("%w: %s", ErrMoveConflict, offlineRegionMoved(rec.Region))
310 }
311 rec.Region = rec.Move.To
312 rec.Move.State = OfflineLanding
313 return nil
314 }
315}
316
317// OfflineLandedBy is landing to landed: the target box reports that every
318// item is in place. Guarded by the record's region, so a landing that raced an
319// abort (which put the region back) is refused.
320func OfflineLandedBy(id, to string) func(*Record) error {
321 return func(rec *Record) error {
322 if err := offlineAt(rec, id, OfflineLanding); err != nil {
323 return err
324 }
325 if rec.Region != to || rec.Move.To != to {
326 return fmt.Errorf("%w: %s", ErrMoveConflict, offlineRegionMoved(rec.Region))
327 }
328 rec.Move.State = OfflineLanded
329 return nil
330 }
331}
332
333// OfflineOpenTransition is landed to open: the operator opens the target. The
334// region must be the target's. It stamps opened_at from now, which is when the
335// instance came back and so what the finished move's offline time is counted
336// to.
337//
338// The brief named this OfflineOpen, which is also the name of the state
339// constant; Go has one namespace, so the two transitions that clash with a
340// state (this one and OfflineFinishingTransition) carry a suffix.
341func OfflineOpenTransition(id string, now time.Time) func(*Record) error {
342 return func(rec *Record) error {
343 if err := offlineAt(rec, id, OfflineLanded); err != nil {
344 return err
345 }
346 if rec.Region != rec.Move.To {
347 return fmt.Errorf("%w: %s", ErrMoveConflict, offlineRegionMoved(rec.Region))
348 }
349 t := now.UTC()
350 rec.Move.State, rec.Move.OpenedAt = OfflineOpen, &t
351 return nil
352 }
353}
354
355// OfflineFinishingTransition is open to finishing: the operator asked the
356// source to clean up. After open the move only goes forward.
357func OfflineFinishingTransition(id string) func(*Record) error {
358 return func(rec *Record) error {
359 if err := offlineAt(rec, id, OfflineOpen); err != nil {
360 return err
361 }
362 rec.Move.State = OfflineFinishing
363 return nil
364 }
365}
366
367// OfflineAbort puts the move on its way back: any of staging, stopping,
368// sealing, sealed, landing or landed becomes reverting. From landing and landed the
369// region goes back to the source in the same write (the flip is undone with
370// the state, never separately). From open and finishing it refuses, because
371// the target already serves. From reverting it changes nothing and writes
372// nothing (updateRecord sees errRecordUnchanged and returns the record as
373// read), so a second abort is a success that leaves record_seq and updated_at
374// alone. Call it through SetRecordMove.
375func OfflineAbort(id string) func(*Record) error {
376 return func(rec *Record) error {
377 if err := offlineIs(rec, id); err != nil {
378 return err
379 }
380 switch rec.Move.State {
381 case OfflineReverting:
382 return errRecordUnchanged
383 case OfflineStaging, OfflineStopping, OfflineSealing, OfflineSealed:
384 case OfflineLanding, OfflineLanded:
385 rec.Region = rec.Move.From
386 default:
387 return fmt.Errorf("%w: %s", ErrMoveConflict, offlineAfterOpen(rec.Move.State))
388 }
389 rec.Move.State = OfflineReverting
390 return nil
391 }
392}
393
394// OfflineFinish clears the move and leaves its outcome in last_move. It is
395// legal from finishing (outcome done, region must be the target's) and from
396// reverting (outcome aborted, region must be the source's); anything else is
397// refused. The outcome's offline time runs from the request to the open for a
398// done move and to now for an aborted one; a move aborted while it was still
399// staging was never offline, and records none.
400func OfflineFinish(id string, now time.Time) func(*Record) error {
401 return func(rec *Record) error {
402 if err := offlineIs(rec, id); err != nil {
403 return err
404 }
405 m := rec.Move
406 out := &MoveOutcome{ID: id, At: now.UTC()}
407 end := now
408 switch m.State {
409 case OfflineFinishing:
410 if rec.Region != m.To {
411 return fmt.Errorf("%w: %s", ErrMoveConflict, offlineRegionMoved(rec.Region))
412 }
413 out.Outcome = OfflineOutcomeDone
414 if m.OpenedAt != nil {
415 end = *m.OpenedAt
416 }
417 case OfflineReverting:
418 if rec.Region != m.From {
419 return fmt.Errorf("%w: %s", ErrMoveConflict, offlineRegionMoved(rec.Region))
420 }
421 out.Outcome = OfflineOutcomeAborted
422 default:
423 return fmt.Errorf("%w: %s", ErrMoveConflict, offlineFinishState(m.State))
424 }
425 if s := end.Sub(m.RequestedAt).Seconds(); s > 0 && m.StartedOffline() {
426 out.OfflineS = s
427 }
428 rec.LastMove, rec.Move = out, nil
429 return nil
430 }
431}
432
433// offlineWrongState is MV-O36.
434func offlineWrongState(state, from string) string {
435 return fmt.Sprintf("this offline move is %s. this step follows %s", state, from)
436}
437
438// offlineSealedBy is MV-O38.
439func offlineSealedBy(box string) string {
440 return fmt.Sprintf("this offline move is sealed by %s", box)
441}
442
443// offlineAfterOpen is MV-O40.
444func offlineAfterOpen(state string) string {
445 return fmt.Sprintf("this offline move is %s. it cannot be aborted once it is open", state)
446}
447
448// offlineFinishState is MV-O41.
449func offlineFinishState(state string) string {
450 return fmt.Sprintf("this offline move is %s. it finishes from finishing or reverting", state)
451}
452
453// offlineNotFrom is MV-O43.
454func offlineNotFrom(region, from string) string {
455 return fmt.Sprintf("the instance is in %s, not %s", regionWord(region), regionWord(from))
456}
457
458// offlineRegionMoved is the refusal for a record whose region is not where
459// the step expects it (MV-O39): something wrote the region under the move.
460func offlineRegionMoved(region string) string {
461 return fmt.Sprintf("the record's region is %s, which this step does not expect. the instance changed under the move", regionWord(region))
462}
463