| 1 | package hostops |
| 2 | |
| 3 | // restore_recover.go extends the control writer's restore transitions |
| 4 | // (control_writer.go: PrepareRestore, CompleteRestore, RecoverMutation) with |
| 5 | // the one case RecoverMutation leaves pending: a restore operation that never |
| 6 | // saved its receipt. |
| 7 | // |
| 8 | // A restore's only external effect is the exact-ETag delete of one trash |
| 9 | // marker, and the issuer saves the receipt (PrepareRestore) right after |
| 10 | // BeginExclusive, before it issues any effect and before any SQL. So an |
| 11 | // operation with no receipt has issued nothing, and reopening it is a fence, |
| 12 | // not a guess: after the reopen CAS (which advances the epoch) the stale |
| 13 | // issuer can neither save a receipt (OperationHandle.current refuses), nor |
| 14 | // issue an effect (EffectRunner.authorized refuses), nor pass the data |
| 15 | // boundary (its permit needs the closed operation, its epoch and an issued |
| 16 | // effect). An operation that does have an issued or uncertain effect is |
| 17 | // never abandoned, whatever its age or receipt. The age bound only avoids |
| 18 | // interrupting a live issuer between BeginExclusive and PrepareRestore; it |
| 19 | // is not the safety argument. |
| 20 | |
| 21 | import ( |
| 22 | "context" |
| 23 | "math" |
| 24 | "strconv" |
| 25 | "time" |
| 26 | ) |
| 27 | |
| 28 | // RestoreActionStarted is the intent key the restore issuer sets at |
| 29 | // BeginExclusive (unix seconds); AbandonRestore measures age from it. |
| 30 | const RestoreActionStarted = "started" |
| 31 | |
| 32 | // RestoreOutcome is what RecoverRestore did. |
| 33 | type RestoreOutcome string |
| 34 | |
| 35 | const ( |
| 36 | RestoreNone RestoreOutcome = "" // no restore operation owns the app |
| 37 | RestoreRestored RestoreOutcome = "restored" // a saved receipt was finished and admission reopened |
| 38 | RestoreAbandoned RestoreOutcome = "abandoned" // no receipt was saved; admission reopened |
| 39 | ) |
| 40 | |
| 41 | // IsRestoreOp reports whether op is a restore app-mutation. |
| 42 | func IsRestoreOp(op *AppOperation) bool { |
| 43 | return op != nil && op.Kind == OpAppMutation && op.Intent["action"] == "restore" |
| 44 | } |
| 45 | |
| 46 | // AbandonRestore reopens admission held by a receipt-less restore operation |
| 47 | // at least minAge old. It reopens only when nothing of the operation was |
| 48 | // issued: each of its effects is absent, already complete, or merely |
| 49 | // prepared (settled "superseded", as EffectRunner.Issue supersedes one). An |
| 50 | // issued or uncertain effect, a saved receipt, a registration or any other |
| 51 | // operation's unsettled effect refuses. The reopen advances the epoch, as |
| 52 | // every reopen does. |
| 53 | func (w *ControlWriter) AbandonRestore(ctx context.Context, account, app, appID, opID string, now time.Time, minAge time.Duration) error { |
| 54 | a, err := w.read(ctx, account, app, appID) |
| 55 | if err != nil { |
| 56 | return err |
| 57 | } |
| 58 | _, err = w.updateAdmission(ctx, a, func(v *AppAdmission) error { |
| 59 | if v.State != "closed" || v.Op == nil || v.Op.ID != opID || v.Op.AppID != appID || !IsRestoreOp(v.Op) || len(v.Registrations) != 0 { |
| 60 | return ErrPending |
| 61 | } |
| 62 | if r := v.LastRestore; r != nil && r.ID == opID { |
| 63 | return ErrPending // a saved receipt is finished by RecoverMutation, never abandoned |
| 64 | } |
| 65 | sec, perr := strconv.ParseInt(v.Op.Intent[RestoreActionStarted], 10, 64) |
| 66 | if perr != nil || sec <= 0 || now.Before(time.Unix(sec, 0).Add(minAge)) { |
| 67 | return ErrPending |
| 68 | } |
| 69 | for id, e := range v.Effects { |
| 70 | switch { |
| 71 | case e.State == "complete": |
| 72 | case e.OperationID == opID && e.AppID == appID && e.State == "prepared": |
| 73 | e.State, e.ResultReference = "complete", "superseded" |
| 74 | v.Effects[id] = e |
| 75 | default: |
| 76 | return ErrPending // issued, uncertain, or another operation's |
| 77 | } |
| 78 | } |
| 79 | if v.Epoch == math.MaxUint64 { |
| 80 | return ErrPending |
| 81 | } |
| 82 | v.Op, v.State = nil, "open" |
| 83 | v.Epoch++ |
| 84 | return nil |
| 85 | }) |
| 86 | return err |
| 87 | } |
| 88 | |
| 89 | // RecoverRestore finishes whatever restore operation owns the app: a saved |
| 90 | // receipt through RecoverMutation (the exact-ETag delete replay; a prepared |
| 91 | // receipt may have no effect yet, because the receipt precedes Issue), a |
| 92 | // receipt-less operation through AbandonRestore. Any other owner, or no |
| 93 | // owner, is RestoreNone. It never starts an operation and never reopens an |
| 94 | // admission a non-restore operation holds. |
| 95 | func (w *ControlWriter) RecoverRestore(ctx context.Context, account, app, appID string, now time.Time, minAge time.Duration) (RestoreOutcome, error) { |
| 96 | a, err := w.read(ctx, account, app, appID) |
| 97 | if err != nil { |
| 98 | return RestoreNone, err |
| 99 | } |
| 100 | op := a.Value.Op |
| 101 | if a.Value.State != "closed" || !IsRestoreOp(op) { |
| 102 | return RestoreNone, nil |
| 103 | } |
| 104 | if r := a.Value.LastRestore; r != nil && r.ID == op.ID { |
| 105 | if err := w.RecoverMutation(ctx, account, app, appID, op.ID); err != nil { |
| 106 | return RestoreNone, err |
| 107 | } |
| 108 | return RestoreRestored, nil |
| 109 | } |
| 110 | if err := w.AbandonRestore(ctx, account, app, appID, op.ID, now, minAge); err != nil { |
| 111 | return RestoreNone, err |
| 112 | } |
| 113 | return RestoreAbandoned, nil |
| 114 | } |
| 115 | |
amadan