| 1 | package hostclaim |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "errors" |
| 7 | "fmt" |
| 8 | "reflect" |
| 9 | "testing" |
| 10 | "time" |
| 11 | |
| 12 | "github.com/carlosframework/platform/internal/release" |
| 13 | ) |
| 14 | |
| 15 | func TestAdmitWriterWarmColdAdmissions(t *testing.T) { |
| 16 | ctx := context.Background() |
| 17 | s := release.NewDirStore(t.TempDir()) |
| 18 | o := servingClaim(t, ctx, s) |
| 19 | stamp := o.Claim.Holder.Stamp |
| 20 | if o.Claim.Holder.Session != 0 || o.Claim.Holder.WriterBox != "" { |
| 21 | t.Fatalf("legacy session = %d, want zero", o.Claim.Holder.Session) |
| 22 | } |
| 23 | raw, err := json.Marshal(o.Claim.Holder) |
| 24 | if err != nil { |
| 25 | t.Fatal(err) |
| 26 | } |
| 27 | var fields map[string]json.RawMessage |
| 28 | if err := json.Unmarshal(raw, &fields); err != nil { |
| 29 | t.Fatal(err) |
| 30 | } |
| 31 | for _, name := range []string{"session", "writer_box", "admissions"} { |
| 32 | if _, ok := fields[name]; ok { |
| 33 | t.Fatalf("zero %s must be omitted", name) |
| 34 | } |
| 35 | } |
| 36 | o, err = Update(ctx, s, o, AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", Box: "adi-knitting", Mode: "cold", NewGeneration: "gen_a", PredecessorCertified: true})) |
| 37 | if err != nil { |
| 38 | t.Fatal(err) |
| 39 | } |
| 40 | cold := o.Claim.Gen |
| 41 | if o.Claim.Holder.WriterBox != "adi-knitting" || o.Claim.Holder.Generation != "gen_a" || o.Claim.Holder.Stamp != cold || o.Claim.Holder.Session != cold || cold <= stamp { |
| 42 | t.Fatalf("cold holder = %+v, gen = %d", o.Claim.Holder, cold) |
| 43 | } |
| 44 | first := AdmissionRecord{Region: "ie", PredecessorRegion: "ie", Session: cold, Box: "adi-knitting", Generation: "gen_a", Mode: "cold"} |
| 45 | assertAdmissions(t, o.Claim, []AdmissionRecord{first}) |
| 46 | o, err = Update(ctx, s, o, func(c *Claim) error { |
| 47 | c.Op.Exec = "boxT" |
| 48 | return nil |
| 49 | }) |
| 50 | if err != nil { |
| 51 | t.Fatal(err) |
| 52 | } |
| 53 | o, err = Update(ctx, s, o, AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", ObservedSession: cold, ObservedGeneration: "gen_a", Box: "boxT", Mode: "warm", PredecessorCertified: true})) |
| 54 | if err != nil { |
| 55 | t.Fatal(err) |
| 56 | } |
| 57 | warm := o.Claim.Gen |
| 58 | if o.Claim.Holder.WriterBox != "boxT" || o.Claim.Holder.Session != warm || warm <= cold || o.Claim.Holder.Generation != "gen_a" || o.Claim.Holder.Stamp != cold { |
| 59 | t.Fatalf("warm holder = %+v, gen = %d", o.Claim.Holder, warm) |
| 60 | } |
| 61 | second := AdmissionRecord{Region: "ie", PredecessorRegion: "ie", Session: warm, Box: "boxT", Generation: "gen_a", Mode: "warm", Predecessor: "gen_a", Covered: true} |
| 62 | assertAdmissions(t, o.Claim, []AdmissionRecord{first, second}) |
| 63 | o, err = Update(ctx, s, o, AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", ObservedSession: warm, ObservedGeneration: "gen_a", Box: "boxT", Mode: "cold", NewGeneration: "gen_b", PredecessorCertified: true})) |
| 64 | if err != nil { |
| 65 | t.Fatal(err) |
| 66 | } |
| 67 | if o.Claim.Holder.Generation != "gen_b" || o.Claim.Holder.Stamp != o.Claim.Gen || o.Claim.Holder.Session != o.Claim.Gen { |
| 68 | t.Fatalf("second cold holder = %+v, gen = %d", o.Claim.Holder, o.Claim.Gen) |
| 69 | } |
| 70 | third := AdmissionRecord{Region: "ie", PredecessorRegion: "ie", Session: o.Claim.Gen, Box: "boxT", Generation: "gen_b", Mode: "cold", Predecessor: "gen_a"} |
| 71 | assertAdmissions(t, o.Claim, []AdmissionRecord{first, second, third}) |
| 72 | stored, err := Read(ctx, s, testHost) |
| 73 | if err != nil { |
| 74 | t.Fatal(err) |
| 75 | } |
| 76 | if !reflect.DeepEqual(stored, o) { |
| 77 | t.Fatalf("stored admission = %+v, want %+v", stored, o) |
| 78 | } |
| 79 | } |
| 80 | |
| 81 | func assertAdmissions(t *testing.T, c *Claim, want []AdmissionRecord) { |
| 82 | t.Helper() |
| 83 | if !reflect.DeepEqual(c.Holder.Admissions, want) { |
| 84 | t.Fatalf("admissions = %+v, want %+v", c.Holder.Admissions, want) |
| 85 | } |
| 86 | if _, ok := c.Op.Journal["writers"]; ok { |
| 87 | t.Fatal("admission wrote an operation writers journal") |
| 88 | } |
| 89 | } |
| 90 | |
| 91 | func TestAdmitWriterPreservesOperation(t *testing.T) { |
| 92 | for _, kind := range []string{KindServing, KindMoving, KindLanding, KindReverting, KindRestoring, KindPushing} { |
| 93 | for _, mode := range []string{"warm", "cold"} { |
| 94 | t.Run(kind+"/"+mode, func(t *testing.T) { |
| 95 | ctx := context.Background() |
| 96 | s := release.NewDirStore(t.TempDir()) |
| 97 | o := servingClaim(t, ctx, s) |
| 98 | o, err := Update(ctx, s, o, func(c *Claim) error { |
| 99 | c.Op = Op{Kind: kind, ID: "m1", Phase: "probe", Exec: "boxT", Participants: []string{"boxS", "boxT"}, |
| 100 | Intent: map[string]string{"target": "boxT"}, Journal: map[string]string{"source": "unchanged"}, |
| 101 | By: "operator", StartedAt: c.Holder.CreatedAt} |
| 102 | c.Obligations = []Obligation{{ID: "ob1", Op: "m1", Kind: "park-local", Box: "boxS", Owner: "box:boxS", Retirement: "rt1"}} |
| 103 | return nil |
| 104 | }) |
| 105 | if err != nil { |
| 106 | t.Fatal(err) |
| 107 | } |
| 108 | generation := "" |
| 109 | if mode == "cold" { |
| 110 | generation = "gen_a" |
| 111 | } |
| 112 | next, err := Update(ctx, s, o, AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", ObservedOpID: "m1", ObservedPhase: "probe", Box: "boxT", Mode: mode, NewGeneration: generation, PredecessorCertified: true})) |
| 113 | if err != nil { |
| 114 | t.Fatal(err) |
| 115 | } |
| 116 | want := *o.Claim |
| 117 | holder := *want.Holder |
| 118 | want.Holder = &holder |
| 119 | want.Gen++ |
| 120 | want.Holder.Session = want.Gen |
| 121 | want.Holder.WriterBox = "boxT" |
| 122 | if mode == "cold" { |
| 123 | want.Holder.Generation = generation |
| 124 | want.Holder.Stamp = want.Gen |
| 125 | } |
| 126 | want.Holder.Admissions = []AdmissionRecord{{Region: "ie", PredecessorRegion: "ie", Session: want.Gen, Box: "boxT", Generation: generation, Mode: mode, Covered: mode == "warm"}} |
| 127 | if !reflect.DeepEqual(next.Claim, &want) { |
| 128 | t.Fatalf("admission changed fields outside holder writer fields and admissions: got %+v, want %+v", next.Claim, &want) |
| 129 | } |
| 130 | }) |
| 131 | } |
| 132 | } |
| 133 | } |
| 134 | |
| 135 | func TestAdmitWriterRefusals(t *testing.T) { |
| 136 | for _, name := range []string{"no holder", "no executor", "non-executor", "stale session", "stale generation", "warm with generation", "warm with current generation", "cold without generation", "cold same generation", "unknown mode", "empty mode", "unknown coverage", "warm without certificate", "stale op", "stale phase", "creating", "deleting", "parking", "purging", "placing"} { |
| 137 | t.Run(name, func(t *testing.T) { |
| 138 | ctx := context.Background() |
| 139 | s := release.NewDirStore(t.TempDir()) |
| 140 | o := servingClaim(t, ctx, s) |
| 141 | o, err := Update(ctx, s, o, func(c *Claim) error { |
| 142 | c.Holder.Session = c.Gen + 1 |
| 143 | c.Holder.Generation = "gen_a" |
| 144 | switch name { |
| 145 | case "no holder": |
| 146 | c.Holder = nil |
| 147 | c.Op = Op{} |
| 148 | case "no executor": |
| 149 | c.Op.Exec = "" |
| 150 | case "creating", "deleting", "parking", "purging", "placing": |
| 151 | c.Op.Kind, c.Op.ID = name, "op1" |
| 152 | } |
| 153 | return nil |
| 154 | }) |
| 155 | if err != nil { |
| 156 | t.Fatal(err) |
| 157 | } |
| 158 | session, generation, box, mode, newGeneration := o.Claim.Gen, "gen_a", "adi-knitting", "cold", "gen_b" |
| 159 | switch name { |
| 160 | case "no executor": |
| 161 | box = "" |
| 162 | case "non-executor": |
| 163 | box = "boxT" |
| 164 | case "stale session": |
| 165 | session-- |
| 166 | case "stale generation": |
| 167 | generation = "old" |
| 168 | case "warm with generation": |
| 169 | mode = "warm" |
| 170 | case "warm with current generation": |
| 171 | mode, newGeneration = "warm", generation |
| 172 | case "cold without generation": |
| 173 | newGeneration = "" |
| 174 | case "cold same generation": |
| 175 | newGeneration = generation |
| 176 | case "unknown mode": |
| 177 | mode = "hot" |
| 178 | case "empty mode": |
| 179 | mode = "" |
| 180 | } |
| 181 | in := Admission{Region: "ie", PredecessorRegion: "ie", ObservedOpID: o.Claim.Op.ID, ObservedPhase: o.Claim.Op.Phase, ObservedSession: session, ObservedGeneration: generation, Box: box, Mode: mode, NewGeneration: newGeneration, PredecessorCertified: true} |
| 182 | switch name { |
| 183 | case "warm without certificate": |
| 184 | in.Mode, in.NewGeneration, in.PredecessorCertified = "warm", "", false |
| 185 | case "unknown coverage": |
| 186 | in.Coverage = "remote" |
| 187 | case "stale op": |
| 188 | in.ObservedOpID = "old" |
| 189 | case "stale phase": |
| 190 | in.ObservedPhase = "old" |
| 191 | } |
| 192 | if _, err := Update(ctx, s, o, AdmitWriter(in)); !errors.Is(err, ErrWrongState) { |
| 193 | t.Fatalf("admit writer = %v, want ErrWrongState", err) |
| 194 | } |
| 195 | stored, err := Read(ctx, s, testHost) |
| 196 | if err != nil { |
| 197 | t.Fatal(err) |
| 198 | } |
| 199 | if !reflect.DeepEqual(stored, o) { |
| 200 | t.Fatalf("refused admission changed stored claim or ETag: got %+v, want %+v", stored, o) |
| 201 | } |
| 202 | }) |
| 203 | } |
| 204 | } |
| 205 | |
| 206 | func TestAdmitWriterCompetingAdmissions(t *testing.T) { |
| 207 | ctx := context.Background() |
| 208 | s := release.NewDirStore(t.TempDir()) |
| 209 | before := servingClaim(t, ctx, s) |
| 210 | in := Admission{Region: "ie", PredecessorRegion: "ie", Box: "adi-knitting", Mode: "cold", NewGeneration: "gen_a"} |
| 211 | first := AdmitWriter(in) |
| 212 | in.NewGeneration = "gen_b" |
| 213 | second := AdmitWriter(in) |
| 214 | admitted, err := Update(ctx, s, before, first) |
| 215 | if err != nil { |
| 216 | t.Fatal(err) |
| 217 | } |
| 218 | if _, err := Update(ctx, s, before, second); !errors.Is(err, ErrConflict) { |
| 219 | t.Fatalf("competing admission = %v, want ErrConflict", err) |
| 220 | } |
| 221 | current := assertAdmissionStored(t, ctx, s, admitted) |
| 222 | if _, err := Update(ctx, s, current, second); !errors.Is(err, ErrWrongState) { |
| 223 | t.Fatalf("stale evidence after reread = %v, want ErrWrongState", err) |
| 224 | } |
| 225 | assertAdmissionStored(t, ctx, s, admitted) |
| 226 | } |
| 227 | |
| 228 | func TestAdmitWriterLostResponse(t *testing.T) { |
| 229 | ctx := context.Background() |
| 230 | s := release.NewDirStore(t.TempDir()) |
| 231 | before := servingClaim(t, ctx, s) |
| 232 | admit := AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", Box: "adi-knitting", Mode: "cold", NewGeneration: "gen_a"}) |
| 233 | admitted, err := Update(ctx, s, before, admit) |
| 234 | if err != nil { |
| 235 | t.Fatal(err) |
| 236 | } |
| 237 | if _, err := Update(ctx, s, before, admit); !errors.Is(err, ErrConflict) { |
| 238 | t.Fatalf("retry after lost response = %v, want ErrConflict", err) |
| 239 | } |
| 240 | assertAdmissionStored(t, ctx, s, admitted) |
| 241 | } |
| 242 | |
| 243 | func TestAdmitWriterBoundToOperationAndPhase(t *testing.T) { |
| 244 | for _, change := range []string{"operation", "phase", "delete"} { |
| 245 | t.Run(change, func(t *testing.T) { |
| 246 | ctx := context.Background() |
| 247 | s := release.NewDirStore(t.TempDir()) |
| 248 | o := servingClaim(t, ctx, s) |
| 249 | stale := AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", Box: "adi-knitting", Mode: "cold", NewGeneration: "gen_a"}) |
| 250 | var err error |
| 251 | if change == "delete" { |
| 252 | o, err = Update(ctx, s, o, BeginDelete("d1", "console", "etag", time.Now(), nil, nil, "")) |
| 253 | } else { |
| 254 | o, err = Update(ctx, s, o, func(c *Claim) error { |
| 255 | if change == "operation" { |
| 256 | c.Op.Kind, c.Op.ID = KindMoving, "m1" |
| 257 | } else { |
| 258 | c.Op.Phase = "probe" |
| 259 | } |
| 260 | return nil |
| 261 | }) |
| 262 | } |
| 263 | if err != nil { |
| 264 | t.Fatal(err) |
| 265 | } |
| 266 | if _, err := Update(ctx, s, o, stale); !errors.Is(err, ErrWrongState) { |
| 267 | t.Fatalf("admission after %s changed = %v, want ErrWrongState", change, err) |
| 268 | } |
| 269 | assertAdmissionStored(t, ctx, s, o) |
| 270 | if change == "delete" { |
| 271 | fresh := AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", ObservedOpID: "d1", Box: "adi-knitting", Mode: "cold", NewGeneration: "gen_a"}) |
| 272 | if _, err := Update(ctx, s, o, fresh); !errors.Is(err, ErrWrongState) { |
| 273 | t.Fatalf("deleting admission = %v", err) |
| 274 | } |
| 275 | assertAdmissionStored(t, ctx, s, o) |
| 276 | id := 0 |
| 277 | o, err = Update(ctx, s, o, ToParking("d1", nil, nil, func() string { id++; return fmt.Sprintf("o%d", id) })) |
| 278 | if err != nil { |
| 279 | t.Fatal(err) |
| 280 | } |
| 281 | if _, err := Update(ctx, s, o, fresh); !errors.Is(err, ErrWrongState) { |
| 282 | t.Fatalf("parking admission = %v", err) |
| 283 | } |
| 284 | assertAdmissionStored(t, ctx, s, o) |
| 285 | } |
| 286 | }) |
| 287 | } |
| 288 | } |
| 289 | |
| 290 | func TestAdmitWriterSurvivesOperationReplacement(t *testing.T) { |
| 291 | for _, transition := range []string{"delete", "move"} { |
| 292 | t.Run(transition, func(t *testing.T) { |
| 293 | ctx := context.Background() |
| 294 | s := release.NewDirStore(t.TempDir()) |
| 295 | o := servingClaim(t, ctx, s) |
| 296 | o, err := Update(ctx, s, o, AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", Box: "adi-knitting", Mode: "cold", NewGeneration: "gen_a"})) |
| 297 | if err != nil { |
| 298 | t.Fatal(err) |
| 299 | } |
| 300 | writer := *o.Claim.Holder |
| 301 | if writer.WriterBox != "adi-knitting" || writer.Session != o.Claim.Gen { |
| 302 | t.Fatalf("admitted holder = %+v", writer) |
| 303 | } |
| 304 | if transition == "delete" { |
| 305 | o, err = Update(ctx, s, o, BeginDelete("d1", "console", "etag", time.Now(), nil, nil, "")) |
| 306 | } else { |
| 307 | o, err = Update(ctx, s, o, func(c *Claim) error { |
| 308 | c.Op = Op{Kind: KindMoving, ID: "m1", Phase: "probe", Exec: "boxT"} |
| 309 | return nil |
| 310 | }) |
| 311 | } |
| 312 | if err != nil { |
| 313 | t.Fatal(err) |
| 314 | } |
| 315 | o = assertAdmissionStored(t, ctx, s, o) |
| 316 | if !reflect.DeepEqual(*o.Claim.Holder, writer) { |
| 317 | t.Fatalf("%s changed writer: %+v", transition, o.Claim.Holder) |
| 318 | } |
| 319 | if transition == "delete" { |
| 320 | return |
| 321 | } |
| 322 | o, err = Update(ctx, s, o, AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", ObservedOpID: "m1", ObservedPhase: "probe", ObservedSession: writer.Session, ObservedGeneration: writer.Generation, Box: "boxT", Mode: "cold", NewGeneration: "gen_b", RemoteMaxTXID: 42})) |
| 323 | if err != nil { |
| 324 | t.Fatal(err) |
| 325 | } |
| 326 | want := append(append([]AdmissionRecord(nil), writer.Admissions...), AdmissionRecord{Region: "ie", PredecessorRegion: "ie", |
| 327 | Session: o.Claim.Gen, Box: "boxT", Generation: "gen_b", Mode: "cold", Predecessor: "gen_a", |
| 328 | WritesLost: &WritesLost{Box: "adi-knitting", Generation: "gen_a", Session: writer.Session, RemoteMaxTXID: 42}, |
| 329 | }) |
| 330 | assertAdmissions(t, o.Claim, want) |
| 331 | if o.Claim.Holder.WriterBox != "boxT" { |
| 332 | t.Fatalf("new writer = %q", o.Claim.Holder.WriterBox) |
| 333 | } |
| 334 | assertAdmissionStored(t, ctx, s, o) |
| 335 | }) |
| 336 | } |
| 337 | } |
| 338 | |
| 339 | func assertAdmissionStored(t *testing.T, ctx context.Context, s release.Store, want Observed) Observed { |
| 340 | t.Helper() |
| 341 | got, err := Read(ctx, s, testHost) |
| 342 | if err != nil { |
| 343 | t.Fatal(err) |
| 344 | } |
| 345 | if !reflect.DeepEqual(got, want) { |
| 346 | t.Fatalf("stored claim or ETag changed: got %+v, want %+v", got, want) |
| 347 | } |
| 348 | return got |
| 349 | } |
| 350 | |
| 351 | func TestAdmitWriterKeepStamp(t *testing.T) { |
| 352 | for _, name := range []string{"landing", "equal session", "later session", "warm"} { |
| 353 | t.Run(name, func(t *testing.T) { |
| 354 | ctx := context.Background() |
| 355 | s := release.NewDirStore(t.TempDir()) |
| 356 | o := servingClaim(t, ctx, s) |
| 357 | o, err := Update(ctx, s, o, AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", Box: "adi-knitting", Mode: "cold", NewGeneration: "gen_a", PredecessorCertified: true})) |
| 358 | if err != nil { |
| 359 | t.Fatal(err) |
| 360 | } |
| 361 | if name == "landing" || name == "warm" { |
| 362 | o, err = Update(ctx, s, o, func(c *Claim) error { |
| 363 | c.Op = Op{Kind: KindLanding, ID: "m1", Phase: "probe", Exec: "boxT"} |
| 364 | c.Holder.Stamp = c.Gen + 1 |
| 365 | return nil |
| 366 | }) |
| 367 | } else if name == "later session" { |
| 368 | o, err = Update(ctx, s, o, AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", ObservedSession: o.Claim.Holder.Session, ObservedGeneration: "gen_a", Box: "adi-knitting", Mode: "warm", PredecessorCertified: true})) |
| 369 | } |
| 370 | if err != nil { |
| 371 | t.Fatal(err) |
| 372 | } |
| 373 | before := *o.Claim.Holder |
| 374 | in := Admission{Region: "ie", PredecessorRegion: "ie", ObservedOpID: o.Claim.Op.ID, ObservedPhase: o.Claim.Op.Phase, ObservedSession: before.Session, ObservedGeneration: before.Generation, Box: o.Claim.Op.Exec, Mode: "cold", NewGeneration: "gen_b", KeepStamp: true, PredecessorCertified: true} |
| 375 | if name == "warm" { |
| 376 | in.Mode, in.NewGeneration = "warm", "" |
| 377 | } |
| 378 | next, err := Update(ctx, s, o, AdmitWriter(in)) |
| 379 | if name != "landing" { |
| 380 | if !errors.Is(err, ErrWrongState) { |
| 381 | t.Fatalf("KeepStamp admission = %v, want ErrWrongState", err) |
| 382 | } |
| 383 | assertAdmissionStored(t, ctx, s, o) |
| 384 | return |
| 385 | } |
| 386 | if err != nil { |
| 387 | t.Fatal(err) |
| 388 | } |
| 389 | if next.Claim.Holder.Stamp != before.Stamp || next.Claim.Holder.Session != next.Claim.Gen || next.Claim.Holder.Generation != "gen_b" { |
| 390 | t.Fatalf("landing admission holder = %+v, prior stamp = %d", next.Claim.Holder, before.Stamp) |
| 391 | } |
| 392 | want := append(append([]AdmissionRecord(nil), before.Admissions...), AdmissionRecord{Region: "ie", PredecessorRegion: "ie", |
| 393 | Session: next.Claim.Gen, Box: "boxT", Generation: "gen_b", Mode: "cold", Predecessor: "gen_a", |
| 394 | }) |
| 395 | assertAdmissions(t, next.Claim, want) |
| 396 | assertAdmissionStored(t, ctx, s, next) |
| 397 | in.ObservedSession, in.ObservedGeneration, in.NewGeneration = next.Claim.Holder.Session, "gen_b", "gen_c" |
| 398 | if _, err := Update(ctx, s, next, AdmitWriter(in)); !errors.Is(err, ErrWrongState) { |
| 399 | t.Fatalf("restarted probe keeping stamp = %v, want ErrWrongState", err) |
| 400 | } |
| 401 | assertAdmissionStored(t, ctx, s, next) |
| 402 | }) |
| 403 | } |
| 404 | } |
| 405 | |
| 406 | func TestAdmitWriterCoverage(t *testing.T) { |
| 407 | for _, mode := range []string{"cold", "warm"} { |
| 408 | for _, certified := range []bool{false, true} { |
| 409 | for _, coverage := range []string{"", "sealed", "clean-return"} { |
| 410 | if mode == "warm" && !certified { |
| 411 | continue |
| 412 | } |
| 413 | t.Run(fmt.Sprintf("%s/certified=%t/coverage=%s", mode, certified, coverage), func(t *testing.T) { |
| 414 | ctx := context.Background() |
| 415 | s := release.NewDirStore(t.TempDir()) |
| 416 | o := servingClaim(t, ctx, s) |
| 417 | o = updateGenerationTest(t, ctx, s, o, AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", Box: "adi-knitting", Mode: "cold", NewGeneration: "gen_a", PredecessorCertified: true})) |
| 418 | before := *o.Claim.Holder |
| 419 | in := Admission{Region: "ie", PredecessorRegion: "ie", ObservedSession: before.Session, ObservedGeneration: before.Generation, Box: "adi-knitting", Mode: mode, PredecessorCertified: certified, Coverage: coverage, RemoteMaxTXID: 42} |
| 420 | generation := before.Generation |
| 421 | if mode == "cold" { |
| 422 | in.NewGeneration, generation = "gen_b", "gen_b" |
| 423 | } |
| 424 | o = updateGenerationTest(t, ctx, s, o, AdmitWriter(in)) |
| 425 | last := AdmissionRecord{Region: "ie", PredecessorRegion: "ie", Session: o.Claim.Gen, Box: "adi-knitting", Generation: generation, Predecessor: "gen_a", Mode: mode, Coverage: coverage, Covered: mode == "warm"} |
| 426 | if mode == "cold" && !certified && coverage == "" { |
| 427 | last.WritesLost = &WritesLost{Box: "adi-knitting", Generation: "gen_a", Session: before.Session, RemoteMaxTXID: 42} |
| 428 | } |
| 429 | assertAdmissions(t, o.Claim, append(append([]AdmissionRecord(nil), before.Admissions...), last)) |
| 430 | assertAdmissionStored(t, ctx, s, o) |
| 431 | }) |
| 432 | } |
| 433 | } |
| 434 | } |
| 435 | } |
| 436 | |
| 437 | func updateGenerationTest(t *testing.T, ctx context.Context, s release.Store, o Observed, tr func(*Claim) error) Observed { |
| 438 | t.Helper() |
| 439 | next, err := Update(ctx, s, o, tr) |
| 440 | if err != nil { |
| 441 | t.Fatal(err) |
| 442 | } |
| 443 | return assertAdmissionStored(t, ctx, s, next) |
| 444 | } |
| 445 | |
| 446 | func TestAdmitWriterCleanReturnPredecessor(t *testing.T) { |
| 447 | for _, coverage := range []string{"", "clean-return"} { |
| 448 | t.Run(coverage, func(t *testing.T) { |
| 449 | ctx := context.Background() |
| 450 | s := release.NewDirStore(t.TempDir()) |
| 451 | o := servingClaim(t, ctx, s) |
| 452 | o = updateGenerationTest(t, ctx, s, o, func(c *Claim) error { |
| 453 | c.Op = Op{Kind: KindLanding, ID: "move", Phase: "probe", Exec: "boxT"} |
| 454 | return nil |
| 455 | }) |
| 456 | o = updateGenerationTest(t, ctx, s, o, AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", ObservedOpID: "move", ObservedPhase: "probe", Box: "boxT", Mode: "cold", NewGeneration: "GT"})) |
| 457 | target := o.Claim.Holder.Admissions[0] |
| 458 | o = updateGenerationTest(t, ctx, s, o, func(c *Claim) error { |
| 459 | c.Holder.Generation = "GS" |
| 460 | c.Op = Op{Kind: KindServing, Exec: "boxS"} |
| 461 | return nil |
| 462 | }) |
| 463 | o = updateGenerationTest(t, ctx, s, o, AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", ObservedSession: target.Session, ObservedGeneration: "GS", Box: "boxS", Mode: "cold", NewGeneration: "GS2", Coverage: coverage, RemoteMaxTXID: 42})) |
| 464 | last := AdmissionRecord{Region: "ie", PredecessorRegion: "ie", Session: o.Claim.Gen, Box: "boxS", Generation: "GS2", Predecessor: "GT", Mode: "cold", Coverage: coverage} |
| 465 | if coverage == "" { |
| 466 | last.WritesLost = &WritesLost{Box: "boxT", Generation: "GT", Session: target.Session, RemoteMaxTXID: 42} |
| 467 | } |
| 468 | assertAdmissions(t, o.Claim, []AdmissionRecord{target, last}) |
| 469 | }) |
| 470 | } |
| 471 | } |
| 472 | |
| 473 | func TestAdoptAdmitPublishRootGeneration(t *testing.T) { |
| 474 | for _, replaceOp := range []bool{false, true} { |
| 475 | t.Run(fmt.Sprintf("replaceOp=%t", replaceOp), func(t *testing.T) { |
| 476 | ctx := context.Background() |
| 477 | s := release.NewDirStore(t.TempDir()) |
| 478 | o, err := Read(ctx, s, testHost) |
| 479 | if err != nil { |
| 480 | t.Fatal(err) |
| 481 | } |
| 482 | o = updateGenerationTest(t, ctx, s, o, Adopt(newHolder(), "boxS", nil, nil, "adopt", counter())) |
| 483 | if o.Claim.Holder.WriterBox != "boxS" || o.Claim.Holder.Generation != "" || RootGeneration != "@root" { |
| 484 | t.Fatalf("adopted holder = %+v", o.Claim.Holder) |
| 485 | } |
| 486 | for _, generation := range []string{"", RootGeneration} { |
| 487 | newID := func() string { t.Fatal("unadmitted publication allocated an ID"); return "" } |
| 488 | if _, err := Update(ctx, s, o, PublishGeneration("", "", 0, generation, "ie", newID)); !errors.Is(err, ErrWrongState) { |
| 489 | t.Fatalf("publication before admission = %v", err) |
| 490 | } |
| 491 | assertAdmissionStored(t, ctx, s, o) |
| 492 | } |
| 493 | if replaceOp { |
| 494 | o = updateGenerationTest(t, ctx, s, o, func(c *Claim) error { |
| 495 | c.Op = Op{Kind: KindLanding, ID: "move", Phase: "probe", Exec: "boxT"} |
| 496 | return nil |
| 497 | }) |
| 498 | o = updateGenerationTest(t, ctx, s, o, func(c *Claim) error { |
| 499 | c.Op = Op{Kind: KindServing, Exec: "boxT"} |
| 500 | return nil |
| 501 | }) |
| 502 | } |
| 503 | o = updateGenerationTest(t, ctx, s, o, AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", Box: o.Claim.Op.Exec, Mode: "cold", NewGeneration: "G1", RemoteMaxTXID: 42})) |
| 504 | assertAdmissions(t, o.Claim, []AdmissionRecord{{Region: "ie", PredecessorRegion: "ie", |
| 505 | Session: o.Claim.Gen, Box: o.Claim.Op.Exec, Generation: "G1", Mode: "cold", |
| 506 | WritesLost: &WritesLost{Box: "boxS", Session: 0, Generation: "", RemoteMaxTXID: 42}, |
| 507 | }}) |
| 508 | o = updateGenerationTest(t, ctx, s, o, PublishGeneration("", "", o.Claim.Holder.Session, "G1", "ie", counter())) |
| 509 | if !o.Claim.Holder.Admissions[0].Covered || len(o.Claim.Obligations) != 1 { |
| 510 | t.Fatalf("published claim = %+v", o.Claim) |
| 511 | } |
| 512 | ob := o.Claim.Obligations[0] |
| 513 | if ob.Generation != RootGeneration || ob.Kind != "park-generation" || ob.Region != "ie" || ob.Owner != "region:ie" || ob.Op != fmt.Sprintf("session:%d", o.Claim.Holder.Session) { |
| 514 | t.Fatalf("root obligation = %+v", ob) |
| 515 | } |
| 516 | holder := *o.Claim.Holder |
| 517 | o, err = Update(ctx, s, o, CompleteObligation(ob.Op, ob.ID)) |
| 518 | if err != nil { |
| 519 | t.Fatal(err) |
| 520 | } |
| 521 | stored, err := Read(ctx, s, testHost) |
| 522 | if err != nil || stored.ETag != o.ETag || len(stored.Claim.Obligations) != 0 || !reflect.DeepEqual(*stored.Claim.Holder, holder) || stored.Claim.Op.Kind != KindServing { |
| 523 | t.Fatalf("completed root retirement = %+v, %v", stored, err) |
| 524 | } |
| 525 | }) |
| 526 | } |
| 527 | } |
| 528 | |
| 529 | func TestPublishGenerationChain(t *testing.T) { |
| 530 | for _, kind := range []string{KindServing, KindMoving, KindLanding, KindReverting, KindRestoring, KindPushing} { |
| 531 | t.Run(kind, func(t *testing.T) { |
| 532 | ctx := context.Background() |
| 533 | s := release.NewDirStore(t.TempDir()) |
| 534 | o := servingClaim(t, ctx, s) |
| 535 | if kind != KindServing { |
| 536 | o = updateGenerationTest(t, ctx, s, o, func(c *Claim) error { |
| 537 | c.Op = Op{Kind: kind, ID: "move", Phase: "probe", Exec: "boxT", Journal: map[string]string{"source": "unchanged"}} |
| 538 | return nil |
| 539 | }) |
| 540 | } |
| 541 | for _, generation := range []string{"G1", "G2"} { |
| 542 | o = updateGenerationTest(t, ctx, s, o, AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", ObservedOpID: o.Claim.Op.ID, ObservedPhase: o.Claim.Op.Phase, |
| 543 | ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation, |
| 544 | Box: o.Claim.Op.Exec, Mode: "cold", NewGeneration: generation})) |
| 545 | } |
| 546 | before := o |
| 547 | calls := 0 |
| 548 | newID := func() string { calls++; return fmt.Sprintf("id%d", calls) } |
| 549 | publish := PublishGeneration(o.Claim.Op.ID, o.Claim.Op.Phase, o.Claim.Holder.Session, "G2", "ie", newID) |
| 550 | o = updateGenerationTest(t, ctx, s, o, publish) |
| 551 | op := before.Claim.Op.ID |
| 552 | if op == "" { |
| 553 | op = fmt.Sprintf("session:%d", before.Claim.Holder.Session) |
| 554 | } |
| 555 | want := *before.Claim |
| 556 | holder := *want.Holder |
| 557 | want.Holder = &holder |
| 558 | want.Gen++ |
| 559 | holder.Admissions = append([]AdmissionRecord(nil), holder.Admissions...) |
| 560 | for i := range holder.Admissions { |
| 561 | holder.Admissions[i].Covered = true |
| 562 | } |
| 563 | want.Obligations = []Obligation{ |
| 564 | {ID: "id1", Op: op, Kind: "park-generation", Region: "ie", Generation: RootGeneration, Retirement: "id2", Owner: "region:ie"}, |
| 565 | {ID: "id3", Op: op, Kind: "park-generation", Region: "ie", Generation: "G1", Retirement: "id4", Owner: "region:ie"}, |
| 566 | } |
| 567 | if calls != 4 || !reflect.DeepEqual(o.Claim, &want) { |
| 568 | t.Fatalf("publication = %+v, want %+v; ID calls %d", o.Claim, &want, calls) |
| 569 | } |
| 570 | o = updateGenerationTest(t, ctx, s, o, publish) |
| 571 | want.Gen++ |
| 572 | if calls != 4 || !reflect.DeepEqual(o.Claim, &want) { |
| 573 | t.Fatalf("republication changed coverage or chores: %+v; calls %d", o.Claim, calls) |
| 574 | } |
| 575 | }) |
| 576 | } |
| 577 | } |
| 578 | |
| 579 | func TestPublishGenerationRefusals(t *testing.T) { |
| 580 | for _, name := range []string{"no holder", "creating", "deleting", "parking", "purging", "placing", "stale op", "stale phase", "stale session", "wrong generation", "empty region", "no matching admission"} { |
| 581 | t.Run(name, func(t *testing.T) { |
| 582 | ctx := context.Background() |
| 583 | s := release.NewDirStore(t.TempDir()) |
| 584 | o := servingClaim(t, ctx, s) |
| 585 | o = updateGenerationTest(t, ctx, s, o, AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", Box: "adi-knitting", Mode: "cold", NewGeneration: "G1"})) |
| 586 | o = updateGenerationTest(t, ctx, s, o, func(c *Claim) error { |
| 587 | c.Op = Op{Kind: KindLanding, ID: "move", Phase: "probe", Exec: "boxT"} |
| 588 | if name == "no holder" { |
| 589 | c.Holder, c.Op = nil, Op{} |
| 590 | } else if knownKinds[name] { |
| 591 | c.Op.Kind = name |
| 592 | } else if name == "no matching admission" { |
| 593 | c.Holder.Generation = "unrecorded" |
| 594 | } |
| 595 | return nil |
| 596 | }) |
| 597 | op, phase, session, generation, region := o.Claim.Op.ID, o.Claim.Op.Phase, uint64(0), "G1", "ie" |
| 598 | if o.Claim.Holder != nil { |
| 599 | session, generation = o.Claim.Holder.Session, o.Claim.Holder.Generation |
| 600 | } |
| 601 | switch name { |
| 602 | case "stale op": |
| 603 | op = "" |
| 604 | case "stale phase": |
| 605 | phase = "" |
| 606 | case "stale session": |
| 607 | session-- |
| 608 | case "wrong generation": |
| 609 | generation = "old" |
| 610 | case "empty region": |
| 611 | region = "" |
| 612 | } |
| 613 | newID := func() string { t.Fatal("refusal allocated an ID"); return "" } |
| 614 | if _, err := Update(ctx, s, o, PublishGeneration(op, phase, session, generation, region, newID)); !errors.Is(err, ErrWrongState) { |
| 615 | t.Fatalf("publication = %v, want ErrWrongState", err) |
| 616 | } |
| 617 | assertAdmissionStored(t, ctx, s, o) |
| 618 | }) |
| 619 | } |
| 620 | } |
| 621 | |
| 622 | func TestPublishGenerationAllAdmissionsInOrder(t *testing.T) { |
| 623 | ctx := context.Background() |
| 624 | s := release.NewDirStore(t.TempDir()) |
| 625 | o := servingClaim(t, ctx, s) |
| 626 | o = updateGenerationTest(t, ctx, s, o, func(c *Claim) error { |
| 627 | c.Holder.Generation = "G2" |
| 628 | c.Holder.Admissions = []AdmissionRecord{ |
| 629 | {Region: "ie", PredecessorRegion: "ie", Generation: "old", Predecessor: "already-covered", Covered: true}, |
| 630 | {Region: "ie", PredecessorRegion: "ie", Generation: "G1", Predecessor: ""}, |
| 631 | {Region: "ie", PredecessorRegion: "ie", Generation: "G1", Predecessor: RootGeneration}, |
| 632 | {Region: "ie", PredecessorRegion: "ie", Generation: "G2", Predecessor: "G1"}, |
| 633 | {Region: "ie", PredecessorRegion: "ie", Generation: "G2", Predecessor: "G2", Mode: "warm", Covered: true}, |
| 634 | {Region: "ie", PredecessorRegion: "ie", Generation: "later", Predecessor: "G2", Mode: "cold"}, |
| 635 | {Region: "ie", PredecessorRegion: "ie", Generation: "after-last-match", Predecessor: "later", Mode: "cold"}, |
| 636 | {Region: "ie", PredecessorRegion: "uk", Generation: "foreign", Predecessor: "foreign-predecessor", Mode: "cold"}, |
| 637 | {Region: "ie", Generation: "empty", Predecessor: "no-objects", Mode: "cold"}, |
| 638 | {Region: "ie", PredecessorRegion: "ie", Generation: "duplicate", Predecessor: "later", Mode: "cold"}, |
| 639 | {Region: "ie", PredecessorRegion: "ie", Generation: "warm-only", Predecessor: "warm-only", Mode: "warm", Covered: true}, |
| 640 | {Region: "ie", PredecessorRegion: "ie", Generation: "last", Predecessor: "final-predecessor", Mode: "cold"}, |
| 641 | } |
| 642 | c.Obligations = []Obligation{ |
| 643 | {ID: "existing", Op: "old-op", Kind: "park-generation", Region: "ie", Generation: "G1", Retirement: "ret1", Owner: "region:ie"}, |
| 644 | {ID: "other-region", Op: "old-op", Kind: "park-generation", Region: "uk", Generation: RootGeneration, Retirement: "ret2", Owner: "region:uk"}, |
| 645 | } |
| 646 | return nil |
| 647 | }) |
| 648 | before := o |
| 649 | calls := 0 |
| 650 | newID := func() string { calls++; return fmt.Sprintf("id%d", calls) } |
| 651 | o = updateGenerationTest(t, ctx, s, o, PublishGeneration("", "", 0, "G2", "ie", newID)) |
| 652 | want := append(append([]Obligation(nil), before.Claim.Obligations...), |
| 653 | Obligation{ID: "id1", Op: "session:0", Kind: "park-generation", Region: "ie", Generation: RootGeneration, Retirement: "id2", Owner: "region:ie"}, |
| 654 | Obligation{ID: "id3", Op: "session:0", Kind: "park-generation", Region: "ie", Generation: "later", Retirement: "id4", Owner: "region:ie"}, |
| 655 | Obligation{ID: "id5", Op: "session:0", Kind: "park-generation", Region: "ie", Generation: "final-predecessor", Retirement: "id6", Owner: "region:ie"}, |
| 656 | ) |
| 657 | if calls != 6 || !reflect.DeepEqual(o.Claim.Obligations, want) { |
| 658 | t.Fatalf("deduplicated chores = %+v, want %+v; calls %d", o.Claim.Obligations, want, calls) |
| 659 | } |
| 660 | for i, record := range o.Claim.Holder.Admissions { |
| 661 | if record.Covered != (i != 5 && i != 7) { |
| 662 | t.Fatalf("record %d covered = %t", i, record.Covered) |
| 663 | } |
| 664 | } |
| 665 | } |
| 666 | |
| 667 | func TestPublishGenerationActiveRoot(t *testing.T) { |
| 668 | ctx := context.Background() |
| 669 | s := release.NewDirStore(t.TempDir()) |
| 670 | o := servingClaim(t, ctx, s) |
| 671 | o = updateGenerationTest(t, ctx, s, o, AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", Box: "adi-knitting", Mode: "warm", PredecessorCertified: true})) |
| 672 | o = updateGenerationTest(t, ctx, s, o, PublishGeneration("", "", o.Claim.Holder.Session, "", "ie", func() string { |
| 673 | t.Fatal("active root must not be retired") |
| 674 | return "" |
| 675 | })) |
| 676 | if !o.Claim.Holder.Admissions[0].Covered || len(o.Claim.Obligations) != 0 { |
| 677 | t.Fatalf("active root publication = %+v", o.Claim) |
| 678 | } |
| 679 | } |
| 680 | |
| 681 | func TestAdmitWriterTrimsCoveredPrefix(t *testing.T) { |
| 682 | for _, name := range []string{"all covered", "uncovered first", "uncovered middle"} { |
| 683 | t.Run(name, func(t *testing.T) { |
| 684 | ctx := context.Background() |
| 685 | s := release.NewDirStore(t.TempDir()) |
| 686 | o := servingClaim(t, ctx, s) |
| 687 | for i := 1; i <= 10; i++ { |
| 688 | o = updateGenerationTest(t, ctx, s, o, AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation, |
| 689 | Box: o.Claim.Op.Exec, Mode: "cold", NewGeneration: fmt.Sprintf("G%d", i)})) |
| 690 | } |
| 691 | if len(o.Claim.Holder.Admissions) != 10 { |
| 692 | t.Fatalf("uncovered records were trimmed: %+v", o.Claim.Holder.Admissions) |
| 693 | } |
| 694 | o = updateGenerationTest(t, ctx, s, o, PublishGeneration("", "", o.Claim.Holder.Session, "G10", "ie", counter())) |
| 695 | if name != "all covered" { |
| 696 | o = updateGenerationTest(t, ctx, s, o, func(c *Claim) error { |
| 697 | i := 0 |
| 698 | if name == "uncovered middle" { |
| 699 | i = 1 |
| 700 | } |
| 701 | c.Holder.Admissions[i].Covered = false |
| 702 | return nil |
| 703 | }) |
| 704 | } |
| 705 | before := o.Claim.Holder.Admissions |
| 706 | o = updateGenerationTest(t, ctx, s, o, AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", ObservedSession: o.Claim.Holder.Session, ObservedGeneration: "G10", |
| 707 | Box: o.Claim.Op.Exec, Mode: "cold", NewGeneration: "G11", PredecessorCertified: true})) |
| 708 | // Loss-bearing rows remain until archival, even when covered. |
| 709 | drop := 1 |
| 710 | if name == "uncovered first" { |
| 711 | drop = 0 |
| 712 | } |
| 713 | want := append(append([]AdmissionRecord(nil), before[drop:]...), AdmissionRecord{Region: "ie", PredecessorRegion: "ie", Session: o.Claim.Gen, Box: o.Claim.Op.Exec, Generation: "G11", Predecessor: "G10", Mode: "cold"}) |
| 714 | assertAdmissions(t, o.Claim, want) |
| 715 | }) |
| 716 | } |
| 717 | } |
| 718 | |
| 719 | func TestAdmissionRecordJSON(t *testing.T) { |
| 720 | ctx := context.Background() |
| 721 | s := release.NewDirStore(t.TempDir()) |
| 722 | o := servingClaim(t, ctx, s) |
| 723 | o = updateGenerationTest(t, ctx, s, o, AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", Box: "adi-knitting", Mode: "cold", NewGeneration: "G1", RemoteMaxTXID: 42})) |
| 724 | o = updateGenerationTest(t, ctx, s, o, AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", ObservedSession: o.Claim.Holder.Session, ObservedGeneration: "G1", Box: "adi-knitting", Mode: "cold", NewGeneration: "G2", Coverage: "sealed", PredecessorCertified: true})) |
| 725 | o = updateGenerationTest(t, ctx, s, o, PublishGeneration("", "", o.Claim.Holder.Session, "G2", "ie", counter())) |
| 726 | raw, err := json.Marshal(o.Claim.Holder) |
| 727 | if err != nil { |
| 728 | t.Fatal(err) |
| 729 | } |
| 730 | var holder map[string]json.RawMessage |
| 731 | if err := json.Unmarshal(raw, &holder); err != nil { |
| 732 | t.Fatal(err) |
| 733 | } |
| 734 | var got []map[string]any |
| 735 | if err := json.Unmarshal(holder["admissions"], &got); err != nil { |
| 736 | t.Fatal(err) |
| 737 | } |
| 738 | want := []map[string]any{ |
| 739 | {"region": "ie", "predecessor_region": "ie", "session": float64(o.Claim.Holder.Admissions[0].Session), "box": "adi-knitting", "generation": "G1", "mode": "cold", "covered": true}, |
| 740 | {"region": "ie", "predecessor_region": "ie", "session": float64(o.Claim.Holder.Session), "box": "adi-knitting", "generation": "G2", "predecessor": "G1", "mode": "cold", "coverage": "sealed", "covered": true}, |
| 741 | } |
| 742 | if !reflect.DeepEqual(got, want) { |
| 743 | t.Fatalf("admissions JSON = %s, want %+v", holder["admissions"], want) |
| 744 | } |
| 745 | } |
| 746 | |
| 747 | func TestAdmitWriterIgnoresLegacyWriterJournal(t *testing.T) { |
| 748 | ctx := context.Background() |
| 749 | s := release.NewDirStore(t.TempDir()) |
| 750 | o := servingClaim(t, ctx, s) |
| 751 | o = updateGenerationTest(t, ctx, s, o, func(c *Claim) error { |
| 752 | c.Op.Journal = map[string]string{"writers": "not-json", "source": "unchanged"} |
| 753 | return nil |
| 754 | }) |
| 755 | before := o.Claim.Op |
| 756 | o = updateGenerationTest(t, ctx, s, o, AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", Box: "adi-knitting", Mode: "cold", NewGeneration: "G1"})) |
| 757 | if !reflect.DeepEqual(o.Claim.Op, before) || len(o.Claim.Holder.Admissions) != 1 { |
| 758 | t.Fatalf("legacy journal affected admission: %+v", o.Claim) |
| 759 | } |
| 760 | } |
| 761 | |
| 762 | func TestPublishGenerationForeignRegion(t *testing.T) { |
| 763 | ctx := context.Background() |
| 764 | s := release.NewDirStore(t.TempDir()) |
| 765 | o := servingClaim(t, ctx, s) |
| 766 | o = updateGenerationTest(t, ctx, s, o, AdmitWriter(Admission{Region: "ie", PredecessorRegion: "ie", Box: o.Claim.Op.Exec, Mode: "cold", NewGeneration: "G1"})) |
| 767 | o = updateGenerationTest(t, ctx, s, o, AdmitWriter(Admission{Region: "uk", ObservedSession: o.Claim.Holder.Session, ObservedGeneration: "G1", Box: o.Claim.Op.Exec, Mode: "cold", NewGeneration: "G2"})) |
| 768 | newID := func() string { t.Fatal("foreign-region publication allocated an ID"); return "" } |
| 769 | publish := PublishGeneration("", "", o.Claim.Holder.Session, "G2", "uk", newID) |
| 770 | o = updateGenerationTest(t, ctx, s, o, publish) |
| 771 | for _, record := range o.Claim.Holder.Admissions { |
| 772 | if record.Covered || record.PredecessorRegion != "ie" { |
| 773 | t.Fatalf("foreign predecessor = %+v", record) |
| 774 | } |
| 775 | } |
| 776 | if len(o.Claim.Obligations) != 0 { |
| 777 | t.Fatalf("foreign publication queued chores: %+v", o.Claim.Obligations) |
| 778 | } |
| 779 | o = updateGenerationTest(t, ctx, s, o, CoverRegionPredecessors("", "", "ie")) |
| 780 | for _, record := range o.Claim.Holder.Admissions { |
| 781 | if !record.Covered { |
| 782 | t.Fatalf("source cleanup left predecessor uncovered: %+v", record) |
| 783 | } |
| 784 | } |
| 785 | before := *o.Claim |
| 786 | o = updateGenerationTest(t, ctx, s, o, publish) |
| 787 | before.Gen++ |
| 788 | if len(o.Claim.Obligations) != 0 || !reflect.DeepEqual(o.Claim, &before) { |
| 789 | t.Fatalf("publication after source coverage changed claim: %+v", o.Claim) |
| 790 | } |
| 791 | } |
| 792 | |
| 793 | func TestPublishGenerationHoldsNone(t *testing.T) { |
| 794 | ctx := context.Background() |
| 795 | s := release.NewDirStore(t.TempDir()) |
| 796 | o := servingClaim(t, ctx, s) |
| 797 | o = updateGenerationTest(t, ctx, s, o, AdmitWriter(Admission{Region: "ie", Box: o.Claim.Op.Exec, Mode: "cold", NewGeneration: "G1"})) |
| 798 | o = updateGenerationTest(t, ctx, s, o, PublishGeneration("", "", o.Claim.Holder.Session, "G1", "ie", func() string { |
| 799 | t.Fatal("predecessor with no objects allocated an ID") |
| 800 | return "" |
| 801 | })) |
| 802 | if !o.Claim.Holder.Admissions[0].Covered || len(o.Claim.Obligations) != 0 { |
| 803 | t.Fatalf("empty predecessor publication = %+v", o.Claim) |
| 804 | } |
| 805 | raw, err := json.Marshal(o.Claim.Holder.Admissions[0]) |
| 806 | if err != nil { |
| 807 | t.Fatal(err) |
| 808 | } |
| 809 | var fields map[string]json.RawMessage |
| 810 | if err := json.Unmarshal(raw, &fields); err != nil { |
| 811 | t.Fatal(err) |
| 812 | } |
| 813 | if string(fields["region"]) != `"ie"` || fields["predecessor_region"] != nil { |
| 814 | t.Fatalf("region JSON = %s", raw) |
| 815 | } |
| 816 | } |
| 817 | |
| 818 | func TestAdmitWriterRegionRules(t *testing.T) { |
| 819 | for _, name := range []string{"empty region", "warm other region", "warm same region", "recorded predecessor region", "first predecessor region", "first warm other region"} { |
| 820 | t.Run(name, func(t *testing.T) { |
| 821 | ctx := context.Background() |
| 822 | s := release.NewDirStore(t.TempDir()) |
| 823 | o := servingClaim(t, ctx, s) |
| 824 | if name != "first predecessor region" && name != "empty region" && name != "first warm other region" { |
| 825 | o = updateGenerationTest(t, ctx, s, o, AdmitWriter(Admission{Region: "ie", Box: o.Claim.Op.Exec, Mode: "cold", NewGeneration: "G1"})) |
| 826 | } |
| 827 | in := Admission{Region: "uk", PredecessorRegion: "fr", ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation, |
| 828 | Box: o.Claim.Op.Exec, Mode: "cold", NewGeneration: "G2", PredecessorCertified: true} |
| 829 | switch name { |
| 830 | case "empty region": |
| 831 | in.Region = "" |
| 832 | case "warm other region", "warm same region", "first warm other region": |
| 833 | in.Mode, in.NewGeneration = "warm", "" |
| 834 | if name == "warm same region" { |
| 835 | in.Region = "ie" |
| 836 | } |
| 837 | } |
| 838 | next, err := Update(ctx, s, o, AdmitWriter(in)) |
| 839 | if name == "empty region" || name == "warm other region" || name == "first warm other region" { |
| 840 | if !errors.Is(err, ErrWrongState) { |
| 841 | t.Fatalf("admission = %v, want ErrWrongState", err) |
| 842 | } |
| 843 | assertAdmissionStored(t, ctx, s, o) |
| 844 | return |
| 845 | } |
| 846 | if err != nil { |
| 847 | t.Fatal(err) |
| 848 | } |
| 849 | assertAdmissionStored(t, ctx, s, next) |
| 850 | record := next.Claim.Holder.Admissions[len(next.Claim.Holder.Admissions)-1] |
| 851 | wantPredecessorRegion := "ie" |
| 852 | if name == "first predecessor region" || name == "first warm other region" { |
| 853 | wantPredecessorRegion = "fr" |
| 854 | } |
| 855 | if record.Region != in.Region || record.PredecessorRegion != wantPredecessorRegion { |
| 856 | t.Fatalf("admission regions = %+v, want %q / %q", record, in.Region, wantPredecessorRegion) |
| 857 | } |
| 858 | }) |
| 859 | } |
| 860 | } |
| 861 | |
| 862 | func TestAdmitWriterNeverTrimsForeignUncovered(t *testing.T) { |
| 863 | ctx := context.Background() |
| 864 | s := release.NewDirStore(t.TempDir()) |
| 865 | o := servingClaim(t, ctx, s) |
| 866 | newID := counter() |
| 867 | for i := 1; i <= 12; i++ { |
| 868 | o = updateGenerationTest(t, ctx, s, o, AdmitWriter(Admission{Region: "uk", PredecessorRegion: "ie", |
| 869 | ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation, |
| 870 | Box: o.Claim.Op.Exec, Mode: "cold", NewGeneration: fmt.Sprintf("G%d", i)})) |
| 871 | o = updateGenerationTest(t, ctx, s, o, PublishGeneration("", "", o.Claim.Holder.Session, o.Claim.Holder.Generation, "uk", newID)) |
| 872 | } |
| 873 | if len(o.Claim.Holder.Admissions) != 12 || o.Claim.Holder.Admissions[0].Generation != "G1" || o.Claim.Holder.Admissions[0].Covered { |
| 874 | t.Fatalf("foreign uncovered head was trimmed or covered: %+v", o.Claim.Holder.Admissions) |
| 875 | } |
| 876 | for _, record := range o.Claim.Holder.Admissions[1:] { |
| 877 | if !record.Covered { |
| 878 | t.Fatalf("publication stopped at foreign predecessor: %+v", record) |
| 879 | } |
| 880 | } |
| 881 | } |
| 882 | |
| 883 | func TestCoverRegionPredecessors(t *testing.T) { |
| 884 | ctx := context.Background() |
| 885 | s := release.NewDirStore(t.TempDir()) |
| 886 | o := servingClaim(t, ctx, s) |
| 887 | o = updateGenerationTest(t, ctx, s, o, func(c *Claim) error { |
| 888 | c.Op = Op{Kind: KindLanding, ID: "move", Phase: "cleanup", Exec: c.Op.Exec} |
| 889 | c.Holder.Admissions = []AdmissionRecord{ |
| 890 | {Region: "ie", PredecessorRegion: "ie", Covered: true}, |
| 891 | {Region: "uk", PredecessorRegion: "uk"}, |
| 892 | {Region: "uk", PredecessorRegion: "ie"}, |
| 893 | {Region: "ie"}, |
| 894 | {Region: "ie", PredecessorRegion: "ie"}, |
| 895 | } |
| 896 | c.Obligations = []Obligation{{ID: "source", Op: "move", Kind: "park-replica", Region: "ie", Retirement: "ret", Owner: "region:ie"}} |
| 897 | return nil |
| 898 | }) |
| 899 | want := *o.Claim |
| 900 | holder := *want.Holder |
| 901 | want.Holder = &holder |
| 902 | holder.Admissions = append([]AdmissionRecord(nil), holder.Admissions...) |
| 903 | holder.Admissions[2].Covered, holder.Admissions[4].Covered = true, true |
| 904 | for i := 0; i < 2; i++ { |
| 905 | o = updateGenerationTest(t, ctx, s, o, CoverRegionPredecessors("move", "cleanup", "ie")) |
| 906 | want.Gen++ |
| 907 | if !reflect.DeepEqual(o.Claim, &want) { |
| 908 | t.Fatalf("region coverage = %+v, want %+v", o.Claim, &want) |
| 909 | } |
| 910 | } |
| 911 | } |
| 912 | |
| 913 | func TestCoverRegionPredecessorsRefusals(t *testing.T) { |
| 914 | for _, name := range []string{"stale op", "stale phase", "nil holder", "empty region"} { |
| 915 | t.Run(name, func(t *testing.T) { |
| 916 | ctx := context.Background() |
| 917 | s := release.NewDirStore(t.TempDir()) |
| 918 | o := servingClaim(t, ctx, s) |
| 919 | o = updateGenerationTest(t, ctx, s, o, func(c *Claim) error { |
| 920 | c.Op = Op{Kind: KindLanding, ID: "move", Phase: "cleanup", Exec: c.Op.Exec} |
| 921 | c.Holder.Admissions = []AdmissionRecord{{Region: "uk", PredecessorRegion: "ie"}} |
| 922 | if name == "nil holder" { |
| 923 | c.Holder, c.Op = nil, Op{} |
| 924 | } |
| 925 | return nil |
| 926 | }) |
| 927 | op, phase, region := o.Claim.Op.ID, o.Claim.Op.Phase, "ie" |
| 928 | switch name { |
| 929 | case "stale op": |
| 930 | op = "" |
| 931 | case "stale phase": |
| 932 | phase = "" |
| 933 | case "empty region": |
| 934 | region = "" |
| 935 | } |
| 936 | if _, err := Update(ctx, s, o, CoverRegionPredecessors(op, phase, region)); !errors.Is(err, ErrWrongState) { |
| 937 | t.Fatalf("region coverage = %v, want ErrWrongState", err) |
| 938 | } |
| 939 | assertAdmissionStored(t, ctx, s, o) |
| 940 | }) |
| 941 | } |
| 942 | } |
| 943 | |
| 944 | func TestValidateAdmissionRegion(t *testing.T) { |
| 945 | ctx := context.Background() |
| 946 | s := release.NewDirStore(t.TempDir()) |
| 947 | o := servingClaim(t, ctx, s) |
| 948 | for _, records := range [][]AdmissionRecord{ |
| 949 | {{Region: ""}}, |
| 950 | {{Region: "ie"}, {Region: ""}}, |
| 951 | {{Region: "", Covered: true}, {Region: "ie"}}, |
| 952 | } { |
| 953 | o.Claim.Holder.Admissions = records |
| 954 | if err := o.Claim.Validate(); !errors.Is(err, ErrMalformed) { |
| 955 | t.Fatalf("admissions %+v: Validate = %v, want ErrMalformed", records, err) |
| 956 | } |
| 957 | } |
| 958 | } |
| 959 | |