| 1 | package hostops |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "errors" |
| 7 | "strings" |
| 8 | "testing" |
| 9 | "time" |
| 10 | |
| 11 | "github.com/carlosframework/platform/internal/hostactors" |
| 12 | "github.com/carlosframework/platform/internal/hostclaim" |
| 13 | "github.com/carlosframework/platform/internal/release" |
| 14 | "github.com/carlosframework/platform/internal/termfence" |
| 15 | ) |
| 16 | |
| 17 | const completionHost = "shop.example" |
| 18 | |
| 19 | // completionWorld is ruling 86's schedule (the two-box model, seed 2224): |
| 20 | // a relocation completed on box-a, which is terminated before it |
| 21 | // discharges, so the serving op's completed relocation stays undischarged |
| 22 | // and box-a is not on lost. The signed roster holds box-a and the worker |
| 23 | // box-w that writes box-a's fence; the authority is production's |
| 24 | // (ProductionAuthority over that roster, ClaimFenceAuthority). |
| 25 | type completionWorld struct { |
| 26 | s release.Store |
| 27 | a ProductionAuthority |
| 28 | } |
| 29 | |
| 30 | func newCompletionWorld(t *testing.T, change func(*hostclaim.Claim)) *completionWorld { |
| 31 | t.Helper() |
| 32 | ctx := context.Background() |
| 33 | x, s, _, _ := chooserFixture(t, "box-a", "box-w") |
| 34 | w := &completionWorld{s: s, a: ProductionAuthority{Store: s, Directory: x.Directory, Namespace: "platform", Now: x.Now}} |
| 35 | obs, err := hostclaim.Read(ctx, s, completionHost) |
| 36 | if err != nil { |
| 37 | t.Fatal(err) |
| 38 | } |
| 39 | obs, err = hostclaim.Update(ctx, s, obs, hostclaim.Adopt(hostclaim.NewHolder{Family: "instance", Key: "apps/acct/app/instances/" + completionHost + ".json", Account: "acct", App: "app", Host: completionHost, Region: "ie"}, "box-a", nil, nil, "adopt", func() string { return "id" })) |
| 40 | if err != nil { |
| 41 | t.Fatal(err) |
| 42 | } |
| 43 | done, err := json.Marshal(hostclaim.CompletedRelocation{Op: "rel-1", Stop: hostclaim.ProbeStop{Box: "box-a", Session: 7, Receipt: "probe-fence-1", ReceiptSHA256: strings.Repeat("a", 64)}, Phase: hostclaim.PhaseRelocatingProbing}) |
| 44 | if err != nil { |
| 45 | t.Fatal(err) |
| 46 | } |
| 47 | if _, err = hostclaim.Update(ctx, s, obs, func(c *hostclaim.Claim) error { |
| 48 | c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: "box-a", Participants: []string{"box-a"}, Journal: map[string]string{hostclaim.JournalCompleted: string(done)}} |
| 49 | if change != nil { |
| 50 | change(c) |
| 51 | } |
| 52 | return nil |
| 53 | }); err != nil { |
| 54 | t.Fatal(err) |
| 55 | } |
| 56 | return w |
| 57 | } |
| 58 | |
| 59 | // fence writes raw at box-a's fence key (its roster enrollment). |
| 60 | func (w *completionWorld) fence(t *testing.T, raw []byte) { |
| 61 | t.Helper() |
| 62 | if _, err := w.s.Put(context.Background(), "control/box-fences/"+strings.Repeat("a", 32)+".json", raw, ""); err != nil { |
| 63 | t.Fatal(err) |
| 64 | } |
| 65 | } |
| 66 | |
| 67 | // terminated is box-w's termination fence of box-a, valid under the |
| 68 | // roster; edit changes it before it is written. |
| 69 | func (w *completionWorld) terminated(t *testing.T, edit func(*termfence.Record)) { |
| 70 | t.Helper() |
| 71 | f := termfence.Record{V: 1, Box: "box-a", Enrollment: strings.Repeat("a", 32), State: "terminated", |
| 72 | Cloud: hostactors.Cloud{Provider: "aws", Account: "123", Region: "eu-west-1", InstanceID: "i-box-a"}, |
| 73 | ObservedAt: "2026-10-07T08:00:00Z", Via: termfence.ViaDescribe, RequestID: "req", |
| 74 | WrittenBy: termfence.WrittenBy{Box: "box-w", Enrollment: strings.Repeat("b", 32)}} |
| 75 | if edit != nil { |
| 76 | edit(&f) |
| 77 | } |
| 78 | raw, err := json.Marshal(f) |
| 79 | if err != nil { |
| 80 | t.Fatal(err) |
| 81 | } |
| 82 | w.fence(t, raw) |
| 83 | } |
| 84 | |
| 85 | // purge is the purge start as startAppLifecycle makes it: the verified |
| 86 | // completion fence, then BeginPurge's CAS with it. delete is the same |
| 87 | // for StartDeleteClaim and BeginDelete. |
| 88 | func (w *completionWorld) purge(t *testing.T) (hostclaim.Observed, error) { |
| 89 | t.Helper() |
| 90 | ctx := context.Background() |
| 91 | obs, err := hostclaim.Read(ctx, w.s, completionHost) |
| 92 | if err != nil { |
| 93 | t.Fatal(err) |
| 94 | } |
| 95 | fenced, err := w.a.completionFence(ctx, obs.Claim) |
| 96 | if err != nil { |
| 97 | return obs, err |
| 98 | } |
| 99 | return hostclaim.Update(ctx, w.s, obs, BeginPurge("purge-1", "console", time.Unix(200, 0), []string{"ie"}, []string{"box-a", "box-w"}, fenced)) |
| 100 | } |
| 101 | |
| 102 | func (w *completionWorld) delete(t *testing.T) (hostclaim.Observed, error) { |
| 103 | t.Helper() |
| 104 | ctx := context.Background() |
| 105 | obs, err := hostclaim.Read(ctx, w.s, completionHost) |
| 106 | if err != nil { |
| 107 | t.Fatal(err) |
| 108 | } |
| 109 | fenced, err := w.a.completionFence(ctx, obs.Claim) |
| 110 | if err != nil { |
| 111 | return obs, err |
| 112 | } |
| 113 | return hostclaim.Update(ctx, w.s, obs, hostclaim.BeginDelete("delete-1", "console", "record-etag", time.Unix(200, 0), []string{"ie"}, []string{"box-a", "box-w"}, fenced)) |
| 114 | } |
| 115 | |
| 116 | func (w *completionWorld) unchanged(t *testing.T, what string) { |
| 117 | t.Helper() |
| 118 | obs, err := hostclaim.Read(context.Background(), w.s, completionHost) |
| 119 | if err != nil { |
| 120 | t.Fatal(err) |
| 121 | } |
| 122 | if obs.Claim.Op.Kind != hostclaim.KindServing || obs.Claim.Op.ID != "" { |
| 123 | t.Fatalf("%s: the claim was started: %+v", what, obs.Claim.Op) |
| 124 | } |
| 125 | } |
| 126 | |
| 127 | // Ruling 86, the deadlock schedule: the completion box is terminated before |
| 128 | // it discharged, and a purge (or a delete) starts. With box-a's termination |
| 129 | // fence verified under the roster, the start passes the completion barrier |
| 130 | // as a failover's would; without the exemption StartGuard's barrier |
| 131 | // refuses it, which is the deadlock the model found. |
| 132 | func TestPurgeStartsOverATerminatedCompletionBox(t *testing.T) { |
| 133 | w := newCompletionWorld(t, nil) |
| 134 | w.terminated(t, nil) |
| 135 | ctx := context.Background() |
| 136 | obs, err := hostclaim.Read(ctx, w.s, completionHost) |
| 137 | if err != nil { |
| 138 | t.Fatal(err) |
| 139 | } |
| 140 | if _, err := hostclaim.Update(ctx, w.s, obs, BeginPurge("purge-1", "console", time.Unix(200, 0), []string{"ie"}, []string{"box-a"}, "")); !errors.Is(err, hostclaim.ErrRelocationFinishing) { |
| 141 | t.Fatalf("the barrier without the exemption: %v (the deadlock's refusal)", err) |
| 142 | } |
| 143 | if fenced, err := w.a.completionFence(ctx, obs.Claim); err != nil || fenced != "box-a" { |
| 144 | t.Fatalf("the verified completion fence: %q %v", fenced, err) |
| 145 | } |
| 146 | started, err := w.purge(t) |
| 147 | if err != nil || started.Claim.Op.Kind != hostclaim.KindPurging || started.Claim.Op.ID != "purge-1" { |
| 148 | t.Fatalf("the purge over a terminated completion box: %v %+v", err, started.Claim) |
| 149 | } |
| 150 | d := newCompletionWorld(t, nil) |
| 151 | d.terminated(t, nil) |
| 152 | if started, err := d.delete(t); err != nil || started.Claim.Op.Kind != hostclaim.KindDeleting { |
| 153 | t.Fatalf("the delete over a terminated completion box: %v", err) |
| 154 | } |
| 155 | } |
| 156 | |
| 157 | // Ruling 86's guards: only a verified fence exempts. A live completion box |
| 158 | // (no fence), a fence that does not verify (another instance, written by |
| 159 | // the box itself, not terminated, another box's record at this key), an |
| 160 | // unreadable fence and a completion box the roster does not hold all leave |
| 161 | // the purge refused; an unreadable or unverifiable fence is its error. A |
| 162 | // completion already discharged, or whose box is on lost, needs no fence: |
| 163 | // the barrier passes it and an unreadable fence is never read. |
| 164 | func TestPurgeOverACompletionNeedsAVerifiedFence(t *testing.T) { |
| 165 | ctx := context.Background() |
| 166 | t.Run("a live completion box", func(t *testing.T) { |
| 167 | w := newCompletionWorld(t, nil) |
| 168 | if _, err := w.purge(t); !errors.Is(err, hostclaim.ErrRelocationFinishing) { |
| 169 | t.Fatalf("a live completion box: %v", err) |
| 170 | } |
| 171 | if _, err := w.delete(t); !errors.Is(err, hostclaim.ErrRelocationFinishing) { |
| 172 | t.Fatalf("a live completion box (delete): %v", err) |
| 173 | } |
| 174 | w.unchanged(t, "a live completion box") |
| 175 | }) |
| 176 | for name, edit := range map[string]func(*termfence.Record){ |
| 177 | "another instance": func(f *termfence.Record) { f.Cloud.InstanceID = "i-other" }, |
| 178 | "written by the box": func(f *termfence.Record) { |
| 179 | f.WrittenBy = termfence.WrittenBy{Box: "box-a", Enrollment: strings.Repeat("a", 32)} |
| 180 | }, |
| 181 | "not terminated": func(f *termfence.Record) { f.State = "running" }, |
| 182 | "another box's record": func(f *termfence.Record) { f.Box = "box-w" }, |
| 183 | "written by an unenrolled": func(f *termfence.Record) { f.WrittenBy.Box = "box-x" }, |
| 184 | } { |
| 185 | t.Run("a fence that does not verify: "+name, func(t *testing.T) { |
| 186 | w := newCompletionWorld(t, nil) |
| 187 | w.terminated(t, edit) |
| 188 | if _, err := w.purge(t); err == nil { |
| 189 | t.Fatalf("an unverified fence exempted the purge") |
| 190 | } |
| 191 | if _, err := w.delete(t); err == nil { |
| 192 | t.Fatalf("an unverified fence exempted the delete") |
| 193 | } |
| 194 | w.unchanged(t, name) |
| 195 | }) |
| 196 | } |
| 197 | t.Run("an unreadable fence", func(t *testing.T) { |
| 198 | w := newCompletionWorld(t, nil) |
| 199 | w.fence(t, []byte("{")) |
| 200 | if _, err := w.purge(t); err == nil || errors.Is(err, hostclaim.ErrRelocationFinishing) { |
| 201 | t.Fatalf("an unreadable fence: %v (want its error)", err) |
| 202 | } |
| 203 | w.unchanged(t, "an unreadable fence") |
| 204 | }) |
| 205 | t.Run("a fence read that fails", func(t *testing.T) { |
| 206 | w := newCompletionWorld(t, nil) |
| 207 | w.terminated(t, nil) |
| 208 | w.a.Store = &faultStore{Store: w.s, failGet: "control/box-fences/" + strings.Repeat("a", 32) + ".json"} |
| 209 | if fenced, err := w.a.completionFence(ctx, mustClaim(t, w)); fenced != "" || err == nil { |
| 210 | t.Fatalf("a failed fence read: %q %v (want its error)", fenced, err) |
| 211 | } |
| 212 | }) |
| 213 | t.Run("a completion box the roster does not hold", func(t *testing.T) { |
| 214 | w := newCompletionWorld(t, func(c *hostclaim.Claim) { |
| 215 | done, _, _ := hostclaim.ReadCompletedRelocation(c) |
| 216 | done.Stop.Box = "box-gone" |
| 217 | raw, _ := json.Marshal(done) |
| 218 | c.Op.Exec, c.Op.Participants, c.Op.Journal[hostclaim.JournalCompleted] = "box-gone", []string{"box-gone"}, string(raw) |
| 219 | }) |
| 220 | w.terminated(t, nil) |
| 221 | if fenced, err := w.a.completionFence(ctx, mustClaim(t, w)); fenced != "" || err != nil { |
| 222 | t.Fatalf("an unrostered box: %q %v", fenced, err) |
| 223 | } |
| 224 | if _, err := w.purge(t); !errors.Is(err, hostclaim.ErrRelocationFinishing) { |
| 225 | t.Fatalf("an unrostered box: %v", err) |
| 226 | } |
| 227 | }) |
| 228 | t.Run("a completion record that cannot be read", func(t *testing.T) { |
| 229 | w := newCompletionWorld(t, func(c *hostclaim.Claim) { c.Op.Journal[hostclaim.JournalCompleted] = "{" }) |
| 230 | w.terminated(t, nil) |
| 231 | if _, err := w.purge(t); !errors.Is(err, hostclaim.ErrRelocation) { |
| 232 | t.Fatalf("an unreadable completion record: %v", err) |
| 233 | } |
| 234 | w.unchanged(t, "an unreadable completion record") |
| 235 | }) |
| 236 | for name, change := range map[string]func(*hostclaim.Claim){ |
| 237 | "already discharged": func(c *hostclaim.Claim) { |
| 238 | done, _, _ := hostclaim.ReadCompletedRelocation(c) |
| 239 | done.Discharged = true |
| 240 | raw, _ := json.Marshal(done) |
| 241 | c.Op.Journal[hostclaim.JournalCompleted] = string(raw) |
| 242 | }, |
| 243 | "its box on lost": func(c *hostclaim.Claim) { c.Lost = append(c.Lost, "box-a") }, |
| 244 | "no completion": func(c *hostclaim.Claim) { c.Op.Journal = nil }, |
| 245 | } { |
| 246 | t.Run("nothing to exempt: "+name, func(t *testing.T) { |
| 247 | w := newCompletionWorld(t, change) |
| 248 | w.fence(t, []byte("{")) |
| 249 | if fenced, err := w.a.completionFence(ctx, mustClaim(t, w)); fenced != "" || err != nil { |
| 250 | t.Fatalf("%s: %q %v (the fence must not be read)", name, fenced, err) |
| 251 | } |
| 252 | if started, err := w.purge(t); err != nil || started.Claim.Op.Kind != hostclaim.KindPurging { |
| 253 | t.Fatalf("%s: %v", name, err) |
| 254 | } |
| 255 | }) |
| 256 | } |
| 257 | } |
| 258 | |
| 259 | func mustClaim(t *testing.T, w *completionWorld) *hostclaim.Claim { |
| 260 | t.Helper() |
| 261 | obs, err := hostclaim.Read(context.Background(), w.s, completionHost) |
| 262 | if err != nil { |
| 263 | t.Fatal(err) |
| 264 | } |
| 265 | return obs.Claim |
| 266 | } |
| 267 | |
amadan