| 1 | package hostbox |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "reflect" |
| 7 | "strings" |
| 8 | "testing" |
| 9 | "time" |
| 10 | ) |
| 11 | |
| 12 | func parkRequest(t *testing.T) JournalRequest { |
| 13 | t.Helper() |
| 14 | p := ServingParkRequest{Successor: ServingArtifactIdentity{Host: "app.example", Box: "box", Region: "ie", Incarnation: "inc", Location: "instances/app.example", Generation: "G2", Stamp: 2, Session: 2}} |
| 15 | p.Predecessor = p.Successor |
| 16 | p.Predecessor.Stamp, p.Predecessor.Session, p.Predecessor.Generation = 1, 1, "G" |
| 17 | return plan9Mutation(t, JournalRequest{Op: "serving-park", ServingPark: &p}) |
| 18 | } |
| 19 | |
| 20 | func TestServingParkJournalPendingBeforeEffectAndReplay(t *testing.T) { |
| 21 | s, path, _ := journalFixture(t) |
| 22 | q := parkRequest(t) |
| 23 | result := ServingParkResult{ID: strings.Repeat("a", 64), Successor: q.ServingPark.Successor} |
| 24 | calls := 0 |
| 25 | run := func(ctx context.Context) (ServingParkResult, error) { |
| 26 | calls++ |
| 27 | // This read would block if the callback held the write transaction. |
| 28 | out, found, err := s.Replay(ctx, "peer", q) |
| 29 | if err != nil || !found || out.Error != "uncertain" { |
| 30 | t.Fatalf("effect without durable pending receipt: %+v %t %v", out, found, err) |
| 31 | } |
| 32 | if calls == 1 { |
| 33 | return result, errors.New("lost retirement save reply") |
| 34 | } |
| 35 | return result, nil |
| 36 | } |
| 37 | if _, err := s.ExecuteServingPark(t.Context(), "peer", q, run); !errors.Is(err, ErrJournalUncertain) { |
| 38 | t.Fatal(err) |
| 39 | } |
| 40 | if err := s.Launches.Close(); err != nil { |
| 41 | t.Fatal(err) |
| 42 | } |
| 43 | j, err := OpenLaunchJournal(path) |
| 44 | if err != nil { |
| 45 | t.Fatal(err) |
| 46 | } |
| 47 | t.Cleanup(func() { j.Close() }) |
| 48 | s, err = NewJournalService(j) |
| 49 | if err != nil { |
| 50 | t.Fatal(err) |
| 51 | } |
| 52 | out, err := s.ExecuteServingPark(t.Context(), "peer", q, run) |
| 53 | if err != nil || out.ServingPark == nil || *out.ServingPark != result || calls != 2 { |
| 54 | t.Fatalf("pending replay: %+v %v calls=%d", out, err, calls) |
| 55 | } |
| 56 | s.Now = func() time.Time { return time.Now().Add(48 * time.Hour) } |
| 57 | replayed, err := s.ExecuteServingPark(t.Context(), "peer", q, run) |
| 58 | if err != nil || !reflect.DeepEqual(replayed, out) || calls != 2 { |
| 59 | t.Fatalf("completed replay executed again: %+v %v calls=%d", replayed, err, calls) |
| 60 | } |
| 61 | if _, err = s.ExecuteServingPark(t.Context(), "other-enrollment", q, run); !errors.Is(err, ErrChanged) || calls != 2 { |
| 62 | t.Fatalf("foreign owner replay: %v", err) |
| 63 | } |
| 64 | } |
| 65 | |
| 66 | func TestServingParkJournalAgeAndPrunedReplay(t *testing.T) { |
| 67 | for _, pending := range []bool{false, true} { |
| 68 | t.Run(map[bool]string{false: "new", true: "pending"}[pending], func(t *testing.T) { |
| 69 | s, _, _ := journalFixture(t) |
| 70 | q := parkRequest(t) |
| 71 | calls := 0 |
| 72 | run := func(context.Context) (ServingParkResult, error) { |
| 73 | calls++ |
| 74 | return ServingParkResult{}, errors.New("partial rename") |
| 75 | } |
| 76 | if pending { |
| 77 | _, _ = s.ExecuteServingPark(t.Context(), "peer", q, run) |
| 78 | } |
| 79 | want := calls |
| 80 | for _, shift := range []time.Duration{-16 * time.Minute, 16 * time.Minute} { |
| 81 | s.Now = func() time.Time { return time.Unix(0, q.IssuedAt).Add(shift) } |
| 82 | if _, err := s.ExecuteServingPark(t.Context(), "peer", q, run); !errors.Is(err, ErrJournalUncertain) || calls != want { |
| 83 | t.Fatalf("expired request ran: %v calls=%d", err, calls) |
| 84 | } |
| 85 | } |
| 86 | }) |
| 87 | } |
| 88 | s, _, _ := journalFixture(t) |
| 89 | q := parkRequest(t) |
| 90 | result := ServingParkResult{ID: strings.Repeat("a", 64), Successor: q.ServingPark.Successor} |
| 91 | run := func(context.Context) (ServingParkResult, error) { return result, nil } |
| 92 | if _, err := s.ExecuteServingPark(t.Context(), "peer", q, run); err != nil { |
| 93 | t.Fatal(err) |
| 94 | } |
| 95 | // A new request prunes the old receipt. Roll the clock back afterwards: |
| 96 | // the persisted rejection boundary still prevents executing the old ID. |
| 97 | now := time.Unix(0, q.IssuedAt).Add(48 * time.Hour) |
| 98 | s.Now = func() time.Time { return now } |
| 99 | newQ, err := journalPrepare(JournalRequest{Op: q.Op, ServingPark: q.ServingPark}, now) |
| 100 | if err != nil { |
| 101 | t.Fatal(err) |
| 102 | } |
| 103 | if _, err = s.ExecuteServingPark(t.Context(), "peer", newQ, run); err != nil { |
| 104 | t.Fatal(err) |
| 105 | } |
| 106 | s.Now = func() time.Time { return time.Unix(0, q.IssuedAt) } |
| 107 | if _, err = s.ExecuteServingPark(t.Context(), "peer", q, func(context.Context) (ServingParkResult, error) { t.Fatal("pruned request reran"); return result, nil }); !errors.Is(err, ErrJournalUncertain) { |
| 108 | t.Fatal(err) |
| 109 | } |
| 110 | } |
| 111 | |
| 112 | func TestServingParkJournalRejectsMixedAndChangedSelectors(t *testing.T) { |
| 113 | s, _, _ := journalFixture(t) |
| 114 | for name, change := range map[string]func(*JournalRequest){ |
| 115 | "version": func(q *JournalRequest) { q.V = 2 }, |
| 116 | "unknown": func(q *JournalRequest) { q.Op = "serving-park-delete" }, |
| 117 | "path-field": func(q *JournalRequest) { q.RecordKey = "/victim" }, |
| 118 | "launch": func(q *JournalRequest) { q.Launch.NoState = true }, |
| 119 | "target": func(q *JournalRequest) { q.Target.Host = "victim" }, |
| 120 | "successor": func(q *JournalRequest) { q.ServingPark.Successor.Generation = "forged" }, |
| 121 | "predecessor": func(q *JournalRequest) { q.ServingPark.Predecessor.Generation = "forged" }, |
| 122 | "legacy": func(q *JournalRequest) { q.ServingPark.Legacy.Empty = true }, |
| 123 | "retirement": func(q *JournalRequest) { q.ServingPark.Retirement = strings.Repeat("f", 64) }, |
| 124 | "pressure": func(q *JournalRequest) { q.Op = "serving-park-pressure" }, |
| 125 | } { |
| 126 | t.Run(name, func(t *testing.T) { |
| 127 | q := parkRequest(t) |
| 128 | change(&q) |
| 129 | if _, err := s.ExecuteServingPark(t.Context(), "peer", q, func(context.Context) (ServingParkResult, error) { |
| 130 | t.Fatal("invalid request reached effects") |
| 131 | return ServingParkResult{}, nil |
| 132 | }); err == nil { |
| 133 | t.Fatal("accepted invalid request") |
| 134 | } |
| 135 | }) |
| 136 | } |
| 137 | } |
| 138 | |
| 139 | func TestServingParkClientRetainsExactUncertainRequest(t *testing.T) { |
| 140 | var sent JournalRequest |
| 141 | client := ServingParkClient{Call: func(_ context.Context, q JournalRequest) (JournalResponse, error) { |
| 142 | sent = q |
| 143 | return JournalResponse{}, errors.New("lost reply") |
| 144 | }} |
| 145 | p := parkRequest(t).ServingPark |
| 146 | _, err := client.Park(t.Context(), p.Successor, p.Predecessor) |
| 147 | var uncertain *JournalUncertainError |
| 148 | if !errors.As(err, &uncertain) || !reflect.DeepEqual(uncertain.Request, sent) || sent.V != 4 { |
| 149 | t.Fatalf("lost request: %v", err) |
| 150 | } |
| 151 | client.Call = func(_ context.Context, q JournalRequest) (JournalResponse, error) { |
| 152 | if !reflect.DeepEqual(q, sent) { |
| 153 | t.Fatal("request changed on replay") |
| 154 | } |
| 155 | return JournalResponse{V: 1, ID: q.ID, Error: "refused"}, nil |
| 156 | } |
| 157 | if _, err = client.ReplayUncertain(t.Context(), uncertain); !errors.As(err, &uncertain) { |
| 158 | t.Fatalf("late refusal became definite: %v", err) |
| 159 | } |
| 160 | client.Call = func(_ context.Context, q JournalRequest) (JournalResponse, error) { |
| 161 | return JournalResponse{V: 1, ID: q.ID}, nil |
| 162 | } |
| 163 | if _, err = client.Park(t.Context(), p.Successor, p.Predecessor); !errors.As(err, &uncertain) { |
| 164 | t.Fatalf("missing result lost request: %v", err) |
| 165 | } |
| 166 | } |
| 167 | |
| 168 | func TestServingParkJournalReceiptFailureKeepsUncertainty(t *testing.T) { |
| 169 | for _, point := range []string{"before-effect", "after-effect"} { |
| 170 | t.Run(point, func(t *testing.T) { |
| 171 | s, _, _ := journalFixture(t) |
| 172 | q := parkRequest(t) |
| 173 | verb := "INSERT" |
| 174 | if point == "after-effect" { |
| 175 | verb = "UPDATE" |
| 176 | } |
| 177 | if _, err := s.Launches.db.Exec("CREATE TRIGGER fail_park_receipt BEFORE " + verb + " ON journal_requests BEGIN SELECT RAISE(ABORT, 'receipt unavailable'); END"); err != nil { |
| 178 | t.Fatal(err) |
| 179 | } |
| 180 | calls := 0 |
| 181 | run := func(context.Context) (ServingParkResult, error) { |
| 182 | calls++ |
| 183 | return ServingParkResult{ID: strings.Repeat("a", 64), Successor: q.ServingPark.Successor}, nil |
| 184 | } |
| 185 | _, err := s.ExecuteServingPark(t.Context(), "peer", q, run) |
| 186 | if err == nil { |
| 187 | t.Fatal("receipt failure succeeded") |
| 188 | } |
| 189 | if point == "before-effect" && calls != 0 { |
| 190 | t.Fatal("effect without pending receipt") |
| 191 | } |
| 192 | if point == "after-effect" && (calls != 1 || !errors.Is(err, ErrJournalUncertain)) { |
| 193 | t.Fatalf("lost outcome: %v calls=%d", err, calls) |
| 194 | } |
| 195 | if _, err = s.Launches.db.Exec("DROP TRIGGER fail_park_receipt"); err != nil { |
| 196 | t.Fatal(err) |
| 197 | } |
| 198 | if _, err = s.ExecuteServingPark(t.Context(), "peer", q, run); err != nil { |
| 199 | t.Fatal(err) |
| 200 | } |
| 201 | }) |
| 202 | } |
| 203 | } |
| 204 | |
amadan