| 1 | package hostops |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "crypto/ed25519" |
| 6 | "encoding/json" |
| 7 | "errors" |
| 8 | "os" |
| 9 | "strconv" |
| 10 | "testing" |
| 11 | "time" |
| 12 | |
| 13 | "github.com/carlosframework/platform/internal/release" |
| 14 | ) |
| 15 | |
| 16 | type recoverFixture struct { |
| 17 | ctx context.Context |
| 18 | raw *release.DirStore |
| 19 | w *ControlWriter |
| 20 | now time.Time |
| 21 | owner RegistrationOwner |
| 22 | op string |
| 23 | tag string |
| 24 | at int64 |
| 25 | } |
| 26 | |
| 27 | func newRecoverFixture(t *testing.T) *recoverFixture { |
| 28 | t.Helper() |
| 29 | f := &recoverFixture{ctx: t.Context(), raw: release.NewDirStore(t.TempDir()), now: time.Unix(1_800_000_000, 0), op: "restore-1"} |
| 30 | f.owner = RegistrationOwner{Console: "console-a", Enrollment: "enrollment-a", Controller: f.op} |
| 31 | var err error |
| 32 | if f.w, err = NewControlWriter(f.raw, make(ed25519.PublicKey, ed25519.PublicKeySize), func() time.Time { return f.now }); err != nil { |
| 33 | t.Fatal(err) |
| 34 | } |
| 35 | a := AppAdmission{V: 1, AppID: "inc-a", Revision: 1, Epoch: 1, State: "open", Registrations: map[string]Registration{}, Effects: map[string]Effect{}} |
| 36 | body, _ := json.Marshal(a) |
| 37 | if _, err := f.raw.Put(f.ctx, "apps/bab/demo/admission.json", body, ""); err != nil { |
| 38 | t.Fatal(err) |
| 39 | } |
| 40 | deleted := f.now.Add(-time.Hour).UTC() |
| 41 | tomb, _ := json.Marshal(release.AppTombstone{Schema: 1, Account: "bab", App: "demo", DeletedAt: deleted, Actor: "p"}) |
| 42 | if f.tag, err = f.raw.Put(f.ctx, release.AppTombstoneKey("bab", "demo"), tomb, ""); err != nil { |
| 43 | t.Fatal(err) |
| 44 | } |
| 45 | f.at = deleted.Unix() |
| 46 | return f |
| 47 | } |
| 48 | |
| 49 | // begin is the restore issuer's BeginExclusive, exactly as the console runs it. |
| 50 | func (f *recoverFixture) begin(t *testing.T) *OperationHandle { |
| 51 | t.Helper() |
| 52 | h, err := f.w.BeginExclusive(f.ctx, "bab", "demo", "inc-a", AppOperation{ID: f.op, Kind: OpAppMutation, AppID: "inc-a", Phase: "issued", |
| 53 | Intent: map[string]string{"action": "restore", RestoreActionStarted: strconv.FormatInt(f.now.Unix(), 10)}}, f.owner) |
| 54 | if err != nil { |
| 55 | t.Fatal(err) |
| 56 | } |
| 57 | return h |
| 58 | } |
| 59 | |
| 60 | // issue runs the restore issuer up to its permit in the console's order: |
| 61 | // BeginExclusive, PrepareRestore (the receipt), then Issue. |
| 62 | func (f *recoverFixture) issue(t *testing.T) (*OperationHandle, context.Context) { |
| 63 | t.Helper() |
| 64 | h := f.begin(t) |
| 65 | if err := h.PrepareRestore(f.ctx, f.tag, f.at); err != nil { |
| 66 | t.Fatal(err) |
| 67 | } |
| 68 | pctx, _, err := h.Runner(func() time.Time { return f.now }).Issue(f.ctx, Effect{ID: f.op, Resource: OpAppMutation, Action: "restore"}) |
| 69 | if err != nil { |
| 70 | t.Fatal(err) |
| 71 | } |
| 72 | return h, pctx |
| 73 | } |
| 74 | |
| 75 | // stageEffect writes one effect of the restore operation straight into the |
| 76 | // raw admission object: a crashed issuer's leftovers, staged as world state. |
| 77 | func (f *recoverFixture) stageEffect(t *testing.T, state string) { |
| 78 | t.Helper() |
| 79 | obs, err := ReadAdmission(f.ctx, f.raw, "bab", "demo", "inc-a") |
| 80 | if err != nil { |
| 81 | t.Fatal(err) |
| 82 | } |
| 83 | v := obs.Value |
| 84 | v.Effects[f.op] = Effect{ID: f.op, OperationID: f.op, AppID: "inc-a", Resource: OpAppMutation, Action: "restore", Owner: f.owner, State: state} |
| 85 | v.Revision++ |
| 86 | body, _ := json.Marshal(v) |
| 87 | if _, err := f.raw.Put(f.ctx, obs.Key, body, obs.ETag); err != nil { |
| 88 | t.Fatal(err) |
| 89 | } |
| 90 | } |
| 91 | |
| 92 | func (f *recoverFixture) admission(t *testing.T) AppAdmission { |
| 93 | t.Helper() |
| 94 | a, err := ReadAdmission(f.ctx, f.raw, "bab", "demo", "inc-a") |
| 95 | if err != nil { |
| 96 | t.Fatal(err) |
| 97 | } |
| 98 | return a.Value |
| 99 | } |
| 100 | |
| 101 | func (f *recoverFixture) marker(t *testing.T) string { |
| 102 | t.Helper() |
| 103 | _, tag, err := f.raw.Get(f.ctx, release.AppTombstoneKey("bab", "demo")) |
| 104 | if errors.Is(err, os.ErrNotExist) { |
| 105 | return "" |
| 106 | } |
| 107 | if err != nil { |
| 108 | t.Fatal(err) |
| 109 | } |
| 110 | return tag |
| 111 | } |
| 112 | |
| 113 | func requireRestored(t *testing.T, a AppAdmission, op, tag string) { |
| 114 | t.Helper() |
| 115 | if a.State != "open" || a.Op != nil || a.LastRestore == nil || a.LastRestore.ID != op || a.LastRestore.TombstoneETag != tag || a.LastRestore.State != "complete" || a.Effects[op].State != "complete" { |
| 116 | t.Fatalf("not restored: %+v %+v", a, a.LastRestore) |
| 117 | } |
| 118 | } |
| 119 | |
| 120 | func TestRecoverRestoreAfterReceiptBeforeDelete(t *testing.T) { |
| 121 | f := newRecoverFixture(t) |
| 122 | f.issue(t) |
| 123 | // The issuer crashed. Another console finishes the receipt at any age. |
| 124 | out, err := f.w.RecoverRestore(f.ctx, "bab", "demo", "inc-a", f.now, time.Hour) |
| 125 | if err != nil || out != RestoreRestored { |
| 126 | t.Fatalf("recover = %q %v", out, err) |
| 127 | } |
| 128 | if f.marker(t) != "" { |
| 129 | t.Fatal("receipt not replayed") |
| 130 | } |
| 131 | requireRestored(t, f.admission(t), f.op, f.tag) |
| 132 | } |
| 133 | |
| 134 | func TestRecoverRestoreAfterDeleteBeforeCompletion(t *testing.T) { |
| 135 | f := newRecoverFixture(t) |
| 136 | f.issue(t) |
| 137 | if err := release.DeleteIfMatch(f.ctx, f.raw, release.AppTombstoneKey("bab", "demo"), f.tag); err != nil { |
| 138 | t.Fatal(err) |
| 139 | } |
| 140 | out, err := f.w.RecoverRestore(f.ctx, "bab", "demo", "inc-a", f.now, time.Hour) |
| 141 | if err != nil || out != RestoreRestored { |
| 142 | t.Fatalf("recover = %q %v", out, err) |
| 143 | } |
| 144 | requireRestored(t, f.admission(t), f.op, f.tag) |
| 145 | } |
| 146 | |
| 147 | // The receipt precedes Issue, so a crash between them leaves a prepared |
| 148 | // receipt and no effect. It is finished (never abandoned), at any age. |
| 149 | // Needs ControlWriter.RecoverMutation to accept a prepared receipt with no |
| 150 | // effect (control-work adaptation; integration-notes §C). |
| 151 | func TestRecoverRestoreReceiptWithoutEffect(t *testing.T) { |
| 152 | f := newRecoverFixture(t) |
| 153 | h := f.begin(t) |
| 154 | if err := h.PrepareRestore(f.ctx, f.tag, f.at); err != nil { |
| 155 | t.Fatal(err) |
| 156 | } |
| 157 | if err := f.w.AbandonRestore(f.ctx, "bab", "demo", "inc-a", f.op, f.now.Add(time.Hour), time.Minute); !errors.Is(err, ErrPending) { |
| 158 | t.Fatalf("a saved receipt was abandoned: %v", err) |
| 159 | } |
| 160 | out, err := f.w.RecoverRestore(f.ctx, "bab", "demo", "inc-a", f.now, time.Hour) |
| 161 | if err != nil || out != RestoreRestored { |
| 162 | t.Fatalf("recover = %q %v", out, err) |
| 163 | } |
| 164 | if f.marker(t) != "" { |
| 165 | t.Fatal("receipt not replayed") |
| 166 | } |
| 167 | a := f.admission(t) |
| 168 | if a.State != "open" || a.Op != nil || a.LastRestore == nil || a.LastRestore.State != "complete" || a.Epoch != 2 { |
| 169 | t.Fatalf("not restored: %+v %+v", a, a.LastRestore) |
| 170 | } |
| 171 | } |
| 172 | |
| 173 | func TestRecoverRestoreWithoutReceiptAbandonsAndFencesTheIssuer(t *testing.T) { |
| 174 | f := newRecoverFixture(t) |
| 175 | h := f.begin(t) // crashed or slow before PrepareRestore: nothing issued |
| 176 | if _, err := f.w.RecoverRestore(f.ctx, "bab", "demo", "inc-a", f.now.Add(time.Minute), 2*time.Minute); !errors.Is(err, ErrPending) { |
| 177 | t.Fatalf("young restore abandoned: %v", err) |
| 178 | } |
| 179 | out, err := f.w.RecoverRestore(f.ctx, "bab", "demo", "inc-a", f.now.Add(3*time.Minute), 2*time.Minute) |
| 180 | if err != nil || out != RestoreAbandoned { |
| 181 | t.Fatalf("recover = %q %v", out, err) |
| 182 | } |
| 183 | a := f.admission(t) |
| 184 | if a.State != "open" || a.Op != nil || a.LastRestore != nil || len(a.Effects) != 0 || a.Epoch != 2 { |
| 185 | t.Fatalf("not abandoned with a new epoch: %+v", a) |
| 186 | } |
| 187 | // The stale issuer is fenced at every step it has left. |
| 188 | if err := h.PrepareRestore(f.ctx, f.tag, f.at); !errors.Is(err, ErrPending) { |
| 189 | t.Fatalf("stale receipt: %v", err) |
| 190 | } |
| 191 | if _, _, err := h.Runner(func() time.Time { return f.now }).Issue(f.ctx, Effect{ID: f.op, Resource: OpAppMutation, Action: "restore"}); err == nil { |
| 192 | t.Fatal("stale issuer issued an effect after the abandon") |
| 193 | } |
| 194 | if f.marker(t) != f.tag { |
| 195 | t.Fatal("marker changed") |
| 196 | } |
| 197 | } |
| 198 | |
| 199 | // A prepared effect never issued its call: abandoning supersedes it. An |
| 200 | // issued or uncertain effect may have deleted the marker: never abandoned, |
| 201 | // at any age, receipt or not. |
| 202 | func TestAbandonRestoreOnlyReopensUnissuedEffects(t *testing.T) { |
| 203 | for _, state := range []string{"prepared", "issued", "uncertain"} { |
| 204 | t.Run(state, func(t *testing.T) { |
| 205 | f := newRecoverFixture(t) |
| 206 | f.begin(t) |
| 207 | f.stageEffect(t, state) |
| 208 | err := f.w.AbandonRestore(f.ctx, "bab", "demo", "inc-a", f.op, f.now.Add(24*time.Hour), time.Minute) |
| 209 | a := f.admission(t) |
| 210 | if state == "prepared" { |
| 211 | if err != nil || a.State != "open" || a.Effects[f.op].State != "complete" || a.Effects[f.op].ResultReference != "superseded" { |
| 212 | t.Fatalf("prepared effect not superseded: %v %+v", err, a) |
| 213 | } |
| 214 | return |
| 215 | } |
| 216 | if !errors.Is(err, ErrPending) || a.State != "closed" || a.Op == nil || a.Effects[f.op].State != state { |
| 217 | t.Fatalf("%s effect abandoned: %v %+v", state, err, a) |
| 218 | } |
| 219 | if out, err := f.w.RecoverRestore(f.ctx, "bab", "demo", "inc-a", f.now.Add(24*time.Hour), time.Minute); !errors.Is(err, ErrPending) || out != RestoreNone { |
| 220 | t.Fatalf("recover reopened a %s effect: %q %v", state, out, err) |
| 221 | } |
| 222 | }) |
| 223 | } |
| 224 | } |
| 225 | |
| 226 | func TestRecoverRestoreLeavesOtherOperationsAlone(t *testing.T) { |
| 227 | f := newRecoverFixture(t) |
| 228 | if _, err := f.w.BeginExclusive(f.ctx, "bab", "demo", "inc-a", AppOperation{ID: "del", Kind: OpDeleting, AppID: "inc-a"}, f.owner); err != nil { |
| 229 | t.Fatal(err) |
| 230 | } |
| 231 | before, _, err := f.raw.Get(f.ctx, "apps/bab/demo/admission.json") |
| 232 | if err != nil { |
| 233 | t.Fatal(err) |
| 234 | } |
| 235 | out, err := f.w.RecoverRestore(f.ctx, "bab", "demo", "inc-a", f.now.Add(time.Hour), time.Minute) |
| 236 | if err != nil || out != RestoreNone { |
| 237 | t.Fatalf("recover = %q %v", out, err) |
| 238 | } |
| 239 | if err := f.w.AbandonRestore(f.ctx, "bab", "demo", "inc-a", "del", f.now.Add(time.Hour), time.Minute); !errors.Is(err, ErrPending) { |
| 240 | t.Fatalf("abandoned a delete: %v", err) |
| 241 | } |
| 242 | after, _, err := f.raw.Get(f.ctx, "apps/bab/demo/admission.json") |
| 243 | if err != nil || string(after) != string(before) { |
| 244 | t.Fatal("admission changed") |
| 245 | } |
| 246 | } |
| 247 | |
amadan