| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "bufio" |
| 5 | "context" |
| 6 | "encoding/json" |
| 7 | "errors" |
| 8 | "fmt" |
| 9 | "net" |
| 10 | "net/http" |
| 11 | "path/filepath" |
| 12 | "strings" |
| 13 | "testing" |
| 14 | "time" |
| 15 | |
| 16 | "github.com/carlosframework/platform/internal/hostbox" |
| 17 | "github.com/carlosframework/platform/internal/hostclaim" |
| 18 | "github.com/carlosframework/platform/internal/hostguard" |
| 19 | "github.com/carlosframework/platform/internal/hostwrite" |
| 20 | "github.com/carlosframework/platform/internal/instancectl" |
| 21 | "github.com/carlosframework/platform/internal/registry" |
| 22 | ) |
| 23 | |
| 24 | func TestClaimCompletedProducersBoundStopAndRetry(t *testing.T) { |
| 25 | t.Parallel() |
| 26 | c, req, store := claimStopFixture(t) |
| 27 | nextID := 0 |
| 28 | c.NewID = func() string { nextID++; return fmt.Sprintf("stop-%d", nextID) } |
| 29 | target := hostguard.FenceTarget{Host: req.Host, Incarnation: req.Incarnation, Stamp: req.InstallationStamp, Session: req.WriterSession, Box: req.Box, Enrollment: req.Enrollment, BootID: req.BootID, ControllerID: req.ControllerID} |
| 30 | path := filepath.Join(t.TempDir(), "launch.db") |
| 31 | j, err := hostbox.OpenLaunchJournal(path) |
| 32 | if err != nil { |
| 33 | t.Fatal(err) |
| 34 | } |
| 35 | t.Cleanup(func() { j.Close() }) |
| 36 | if err := j.Prelaunch(t.Context(), hostbox.LaunchInventory{Target: target, ControllerPID: 42, ProcessStart: "start-42", NoState: true, Groups: []hostbox.LaunchGroup{{RelativePath: "parent", Device: 1, Inode: 2}}}); err != nil { |
| 37 | t.Fatal(err) |
| 38 | } |
| 39 | p, err := NewClaimProducers(target, "start-42", j.RecordProducer, j.CompleteProducer) |
| 40 | if err != nil { |
| 41 | t.Fatal(err) |
| 42 | } |
| 43 | const deliveries = 1024 |
| 44 | producerID := func(n int) string { return fmt.Sprintf("%01024d", n) } |
| 45 | set := &ClaimProducerSet{} |
| 46 | if err := set.Add(p); err != nil { |
| 47 | t.Fatal(err) |
| 48 | } |
| 49 | source := hostbox.WorkSource{Guard: c.Guard, Box: req.Box, Region: "ie", Now: time.Now, |
| 50 | Policy: func(context.Context, *hostclaim.Claim, hostbox.Selection, hostwrite.Identity) error { return nil }, |
| 51 | Local: func(context.Context, *hostclaim.Claim, hostbox.Selection) error { return nil }} |
| 52 | work, err := source.Select(t.Context(), req.Host, "timer") |
| 53 | if err != nil { |
| 54 | t.Fatal(err) |
| 55 | } |
| 56 | route := registry.Route{Host: req.Host, ClaimOwnership: registry.ClaimOwnership{ClaimIncarnation: req.Incarnation, ClaimStamp: req.InstallationStamp}} |
| 57 | timer := registry.Timer{Host: req.Host, ClaimOwnership: route.ClaimOwnership} |
| 58 | seq := 0 |
| 59 | deliver := ClaimTimerDelivery(set, func() string { id := producerID(seq); seq++; return id }) |
| 60 | oldIDs := make([]string, 0, deliveries) |
| 61 | for n := 0; n < deliveries; n++ { |
| 62 | id := "timer-" + producerID(n) |
| 63 | oldIDs = append(oldIDs, id) |
| 64 | _, finish, err := deliver(t.Context(), work, route, timer) |
| 65 | if err != nil { |
| 66 | t.Fatal(err) |
| 67 | } |
| 68 | finish() |
| 69 | finish() |
| 70 | if len(p.workers) != 0 { |
| 71 | t.Fatalf("completed delivery %d remains in memory", n) |
| 72 | } |
| 73 | } |
| 74 | legacy, _ := json.Marshal(DrainClaimAcknowledgement("fence", req.ControllerID, "start-42", oldIDs)) |
| 75 | if len(legacy) <= 1<<20 { |
| 76 | t.Fatal("fixture does not exceed the transport limit") |
| 77 | } |
| 78 | if err := j.Close(); err != nil { |
| 79 | t.Fatal(err) |
| 80 | } |
| 81 | j, err = hostbox.OpenLaunchJournal(path) |
| 82 | if err != nil { |
| 83 | t.Fatal(err) |
| 84 | } |
| 85 | in, err := j.Read(t.Context(), target) |
| 86 | if err != nil { |
| 87 | t.Fatal(err) |
| 88 | } |
| 89 | raw, _ := json.Marshal(in) |
| 90 | if len(in.ProducerIDs) != 0 || in.Completed.Count != deliveries || len(in.Completed.Digest) != 64 || len(raw) > 4096 { |
| 91 | t.Fatalf("unbounded or missing durable completion: %d bytes, %+v", len(raw), in.Completed) |
| 92 | } |
| 93 | checks := 0 |
| 94 | c.Kernel = stopKernelFunc(func(ctx context.Context, target hostguard.FenceTarget, _ StopInventory, ack DrainAcknowledgement) (StopKernelEvidence, error) { |
| 95 | checks++ |
| 96 | in, err := j.ReadStopKernelJournal(ctx, target) |
| 97 | if err != nil { |
| 98 | return StopKernelEvidence{}, err |
| 99 | } |
| 100 | if err := validateClaimDrainProducers(in.ProducerIDs, ack, in.Completed); err != nil { |
| 101 | return StopKernelEvidence{}, err |
| 102 | } |
| 103 | bad := ack |
| 104 | bad.Completed.Count++ |
| 105 | if validateClaimDrainProducers(in.ProducerIDs, bad, in.Completed) == nil { |
| 106 | t.Fatal("changed completion accepted") |
| 107 | } |
| 108 | return StopKernelEvidence{InventorySHA256: strings.Repeat("b", 64), LocalJournalSHA256: digestStop(in), CgroupIdentities: []string{"cg-1"}}, nil |
| 109 | }) |
| 110 | peer, _ := c.Peers.AuthenticatedPeer(t.Context()) |
| 111 | call := func() (StopResult, error) { |
| 112 | client, serverConn := net.Pipe() |
| 113 | ln := &stopPipeListener{conn: serverConn, closed: make(chan struct{})} |
| 114 | ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) |
| 115 | h := ClaimStopHTTP{Coordinator: c, Authenticate: func(context.Context, net.Conn) (StopPeer, error) { return peer, nil }} |
| 116 | s := instancectl.Server{PeerOK: func(net.Conn) error { return nil }, ClaimStop: h} |
| 117 | done := make(chan error, 1) |
| 118 | go func() { done <- s.Serve(ctx, ln) }() |
| 119 | result, err := collectClaimStop(ctx, "unused", req, p, func(context.Context, string, string) (net.Conn, error) { return client, nil }) |
| 120 | cancel() |
| 121 | if serveErr := <-done; serveErr != nil { |
| 122 | t.Fatal(serveErr) |
| 123 | } |
| 124 | return result, err |
| 125 | } |
| 126 | store.err = errors.New("publication failure") |
| 127 | if _, err := call(); err == nil { |
| 128 | t.Fatal("publication failure hidden") |
| 129 | } |
| 130 | first, err := call() |
| 131 | if err != nil { |
| 132 | t.Fatal(err) |
| 133 | } |
| 134 | again, err := call() |
| 135 | if err != nil || first.Digest != again.Digest || !first.EvidenceAt.Equal(again.EvidenceAt) || checks != 1 { |
| 136 | t.Fatalf("receipt retry changed evidence: %v checks=%d", err, checks) |
| 137 | } |
| 138 | } |
| 139 | |
| 140 | func TestClaimStopDisconnectedPeerCancelsPipe(t *testing.T) { |
| 141 | t.Parallel() |
| 142 | c, req, _ := claimStopFixture(t) |
| 143 | client, server := net.Pipe() |
| 144 | defer client.Close() |
| 145 | defer server.Close() |
| 146 | serverDone := make(chan struct{}) |
| 147 | go func() { |
| 148 | defer close(serverDone) |
| 149 | defer server.Close() |
| 150 | r, err := http.ReadRequest(bufio.NewReader(server)) |
| 151 | if err == nil { |
| 152 | _ = json.NewDecoder(r.Body).Decode(new(StopRequest)) |
| 153 | } |
| 154 | }() |
| 155 | ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond) |
| 156 | defer cancel() |
| 157 | done := make(chan error, 1) |
| 158 | go func() { |
| 159 | _, err := collectClaimStop(ctx, "unused", req, c.Drainer, func(context.Context, string, string) (net.Conn, error) { return client, nil }) |
| 160 | done <- err |
| 161 | }() |
| 162 | select { |
| 163 | case err := <-done: |
| 164 | if err == nil { |
| 165 | t.Fatal("disconnected peer accepted") |
| 166 | } |
| 167 | case <-time.After(2 * time.Second): |
| 168 | t.Fatal("request pipe ignored cancellation") |
| 169 | } |
| 170 | <-serverDone |
| 171 | } |
| 172 | |
| 173 | func TestClaimProducerCompactionFailureRetriesBeforeAdmission(t *testing.T) { |
| 174 | _, req, _ := claimStopFixture(t) |
| 175 | target := hostguard.FenceTarget{Host: req.Host, Incarnation: req.Incarnation, Stamp: req.InstallationStamp, Session: req.WriterSession, Box: req.Box, Enrollment: req.Enrollment, BootID: req.BootID, ControllerID: req.ControllerID} |
| 176 | j, err := hostbox.OpenLaunchJournal(filepath.Join(t.TempDir(), "launch.db")) |
| 177 | if err != nil { |
| 178 | t.Fatal(err) |
| 179 | } |
| 180 | defer j.Close() |
| 181 | if err := j.Prelaunch(t.Context(), hostbox.LaunchInventory{Target: target, ControllerPID: 42, ProcessStart: "start-42", NoState: true, Groups: []hostbox.LaunchGroup{{RelativePath: "parent", Device: 1, Inode: 2}}}); err != nil { |
| 182 | t.Fatal(err) |
| 183 | } |
| 184 | fail := true |
| 185 | p, err := NewClaimProducers(target, "start-42", j.RecordProducer, func(ctx context.Context, target hostguard.FenceTarget, id string, previous hostbox.ProducerCompletion) (hostbox.ProducerCompletion, error) { |
| 186 | if fail { |
| 187 | return previous, errors.New("disk failure") |
| 188 | } |
| 189 | return j.CompleteProducer(ctx, target, id, previous) |
| 190 | }) |
| 191 | if err != nil { |
| 192 | t.Fatal(err) |
| 193 | } |
| 194 | _, finish, err := p.Register(t.Context(), target, "timer-first") |
| 195 | if err != nil { |
| 196 | t.Fatal(err) |
| 197 | } |
| 198 | finish(nil) |
| 199 | if _, _, err := p.Register(t.Context(), target, "timer-next"); err == nil { |
| 200 | t.Fatal("admission ignored failed compaction") |
| 201 | } |
| 202 | if _, err := p.DrainClaim(t.Context(), req, "fence"); err == nil { |
| 203 | t.Fatal("drain ignored failed compaction") |
| 204 | } |
| 205 | fail = false |
| 206 | ack, err := p.DrainClaim(t.Context(), req, "fence") |
| 207 | if err != nil || ack.Completed.Count != 1 || len(p.workers) != 0 { |
| 208 | t.Fatalf("compaction retry failed: %+v %v", ack, err) |
| 209 | } |
| 210 | // A lost successful commit response must also be safe to replay. |
| 211 | completed, err := j.CompleteProducer(t.Context(), target, "timer-first", hostbox.ProducerCompletion{}) |
| 212 | if err != nil || completed != ack.Completed { |
| 213 | t.Fatalf("commit replay: %+v %v", completed, err) |
| 214 | } |
| 215 | } |
| 216 | |
amadan