| 1 | //go:build linux |
| 2 | |
| 3 | package hostexec |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "errors" |
| 8 | "testing" |
| 9 | ) |
| 10 | |
| 11 | func TestPlan14ExecutionRecoverySettlesWithoutControllerToken(t *testing.T) { |
| 12 | for _, fail := range []string{"", "cleanup", "cc9"} { |
| 13 | t.Run(fail, func(t *testing.T) { |
| 14 | b, _, remote, _, q, peer := brokerFixture(t) |
| 15 | if _, err := b.Execute(t.Context(), peer, q); err != nil { |
| 16 | t.Fatal(err) |
| 17 | } |
| 18 | if err := b.Close(t.Context()); err != nil { |
| 19 | t.Fatal(err) |
| 20 | } |
| 21 | cfg := b.cfg |
| 22 | cfg.Identity = "replacement" |
| 23 | refused := errors.New("unresolved") |
| 24 | if fail == "cleanup" { |
| 25 | cfg.Cleanup = func(context.Context, ExecutionRecord) error { return refused } |
| 26 | } |
| 27 | if fail == "cc9" { |
| 28 | cfg.Settle = func(context.Context, ExecutionRecord) error { return refused } |
| 29 | } |
| 30 | next, err := NewRootBroker(cfg) |
| 31 | if err != nil { |
| 32 | t.Fatal(err) |
| 33 | } |
| 34 | defer next.Close(t.Context()) |
| 35 | err = next.Recover(t.Context()) |
| 36 | r, readErr := next.read(q.Host) |
| 37 | if readErr != nil { |
| 38 | t.Fatal(readErr) |
| 39 | } |
| 40 | if fail != "" { |
| 41 | if !errors.Is(err, refused) || r.Status == "settled" { |
| 42 | t.Fatalf("unproven recovery: %+v %v", r, err) |
| 43 | } |
| 44 | next.cfg.Cleanup = func(context.Context, ExecutionRecord) error { return nil } |
| 45 | next.cfg.Settle = func(context.Context, ExecutionRecord) error { return nil } |
| 46 | if err = next.Monitor(t.Context()); err != nil { |
| 47 | t.Fatal("retry", err) |
| 48 | } |
| 49 | r, err = next.read(q.Host) |
| 50 | } |
| 51 | if err != nil || r.Status != "settled" { |
| 52 | t.Fatalf("recovery left an ownerless obligation: %+v %v", r, err) |
| 53 | } |
| 54 | if remote.calls != 1 { |
| 55 | t.Fatal("recovery acquired a lease") |
| 56 | } |
| 57 | if _, err = next.Execute(t.Context(), peer, q); err == nil { |
| 58 | t.Fatal("recovered begin resurrected") |
| 59 | } |
| 60 | }) |
| 61 | } |
| 62 | } |
| 63 | |
| 64 | func TestPlan14ExecutionRecoveryDoesNotReleaseNewOwner(t *testing.T) { |
| 65 | b, _, remote, _, q, peer := brokerFixture(t) |
| 66 | if _, err := b.Execute(t.Context(), peer, q); err != nil { |
| 67 | t.Fatal(err) |
| 68 | } |
| 69 | if err := b.Close(t.Context()); err != nil { |
| 70 | t.Fatal(err) |
| 71 | } |
| 72 | key, err := Key(q.Host, "box") |
| 73 | if err != nil { |
| 74 | t.Fatal(err) |
| 75 | } |
| 76 | _, etag, err := remote.Get(t.Context(), key) |
| 77 | if err != nil { |
| 78 | t.Fatal(err) |
| 79 | } |
| 80 | newer := []byte(`{"op":"another-owner","expires":"2099-01-01T00:00:00Z"}`) |
| 81 | nextETag, err := remote.Put(t.Context(), key, newer, etag) |
| 82 | if err != nil { |
| 83 | t.Fatal(err) |
| 84 | } |
| 85 | cfg := b.cfg |
| 86 | cfg.Identity = "replacement" |
| 87 | next, err := NewRootBroker(cfg) |
| 88 | if err != nil { |
| 89 | t.Fatal(err) |
| 90 | } |
| 91 | defer next.Close(t.Context()) |
| 92 | if err = next.Recover(t.Context()); err == nil { |
| 93 | t.Fatal("replacement lease counted as our release") |
| 94 | } |
| 95 | body, got, err := remote.Get(t.Context(), key) |
| 96 | if err != nil || got != nextETag || string(body) != string(newer) { |
| 97 | t.Fatalf("recovery deleted a newer owner: %s %s %v", body, got, err) |
| 98 | } |
| 99 | r, err := next.read(q.Host) |
| 100 | if err != nil || r.Status == "settled" { |
| 101 | t.Fatalf("unresolved release settled: %+v %v", r, err) |
| 102 | } |
| 103 | } |
| 104 | |
| 105 | func TestPlan14ExecutionRecoveryJoinsRemoteWrite(t *testing.T) { |
| 106 | b, _, remote, clock, q, peer := brokerFixture(t) |
| 107 | out, err := b.Execute(t.Context(), peer, q) |
| 108 | if err != nil { |
| 109 | t.Fatal(err) |
| 110 | } |
| 111 | entered, resume := make(chan struct{}), make(chan struct{}) |
| 112 | remote.onPut = func(int) { close(entered); <-resume } |
| 113 | q.Operation, q.ID, q.Token = "executionRenew", "renew-in-flight", out.Token |
| 114 | done := make(chan error, 1) |
| 115 | go func() { _, err := b.Execute(t.Context(), peer, q); done <- err }() |
| 116 | <-entered |
| 117 | clock.advance(TTL) |
| 118 | _ = b.Monitor(t.Context()) |
| 119 | err = b.Monitor(t.Context()) |
| 120 | r, readErr := b.read(q.Host) |
| 121 | close(resume) |
| 122 | writeErr := <-done |
| 123 | if err != nil || readErr != nil || r.Status != "lost" || writeErr == nil { |
| 124 | t.Fatalf("settled before remote write joined: %+v monitor=%v read=%v write=%v", r, err, readErr, writeErr) |
| 125 | } |
| 126 | if err = b.Monitor(t.Context()); err != nil { |
| 127 | t.Fatal(err) |
| 128 | } |
| 129 | r, err = b.read(q.Host) |
| 130 | if err != nil || r.Status != "settled" { |
| 131 | t.Fatalf("joined write cannot settle: %+v %v", r, err) |
| 132 | } |
| 133 | } |
| 134 | |
amadan