| 1 | package console |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "fmt" |
| 7 | "net/http" |
| 8 | "strings" |
| 9 | "testing" |
| 10 | "time" |
| 11 | |
| 12 | "github.com/carlosframework/platform/internal/provision" |
| 13 | "github.com/carlosframework/platform/internal/registry" |
| 14 | "github.com/carlosframework/platform/internal/release" |
| 15 | "github.com/carlosframework/platform/internal/rolloutwords" |
| 16 | "github.com/carlosframework/platform/internal/status" |
| 17 | ) |
| 18 | |
| 19 | // writeFleetHeldBox registers fleet to owner and writes, under the fleet's |
| 20 | // own prefix, a box whose report holds host back on build for this |
| 21 | // fixture's account and app. It returns the box's label. |
| 22 | func writeFleetHeldBox(t *testing.T, fx *consoleFixture, fleet, owner, host, app, build string) string { |
| 23 | t.Helper() |
| 24 | reg, err := registry.Open(fx.regPath) |
| 25 | if err != nil { |
| 26 | t.Fatal(err) |
| 27 | } |
| 28 | defer reg.Close() |
| 29 | if err := reg.UpsertFleet(registry.Fleet{Name: fleet, Account: owner, Bucket: fleet + "-bucket"}); err != nil { |
| 30 | t.Fatal(err) |
| 31 | } |
| 32 | box := status.CollectBox(nil, status.Probes{MachineID: fleet + "-box"}, time.Now()) |
| 33 | box.Metrics = &status.BoxMetrics{Rollout: &status.RolloutReport{Hosts: map[string]status.HostRollout{ |
| 34 | host: {Account: fx.sqid, App: app, Installed: build, Serving: "v1", Good: "v1", Awake: true, HeldBy: build, Reason: status.ReasonHealth, |
| 35 | HeldEpoch: heldEpoch, HeldBook: heldBook, RetryV: status.HostRetryV}, |
| 36 | }}} |
| 37 | if err := status.WriteBox(context.Background(), release.NewPrefixStore(fx.rel, "fleets/"+fleet+"/"), box); err != nil { |
| 38 | t.Fatal(err) |
| 39 | } |
| 40 | return box.Label |
| 41 | } |
| 42 | |
| 43 | // listRollout is the list's answer for host: its rollout view, and whether |
| 44 | // the answer said rollout_incomplete. |
| 45 | func listRollout(t *testing.T, fx *consoleFixture, token, app, host string) (*rolloutView, bool) { |
| 46 | t.Helper() |
| 47 | code, _, out := rawReq(t, http.MethodGet, fx.ts.URL+"/api/cli/apps/"+fx.sqid+"/"+app+"/instances", token, "") |
| 48 | if code != http.StatusOK { |
| 49 | t.Fatalf("list = %d %s", code, out) |
| 50 | } |
| 51 | var list struct { |
| 52 | Instances []struct { |
| 53 | Host string `json:"host"` |
| 54 | Rollout *rolloutView `json:"rollout"` |
| 55 | } `json:"instances"` |
| 56 | RolloutIncomplete bool `json:"rollout_incomplete"` |
| 57 | } |
| 58 | if err := json.Unmarshal([]byte(out), &list); err != nil { |
| 59 | t.Fatal(err) |
| 60 | } |
| 61 | for _, r := range list.Instances { |
| 62 | if r.Host == host { |
| 63 | return r.Rollout, list.RolloutIncomplete |
| 64 | } |
| 65 | } |
| 66 | t.Fatalf("list lacks %s: %s", host, out) |
| 67 | return nil, false |
| 68 | } |
| 69 | |
| 70 | // placedRecord writes app's instance record for host, placed at region |
| 71 | // ("ie", or "fleet/<name>" for a customer fleet). |
| 72 | func placedRecord(t *testing.T, fx *consoleFixture, app, host, region string) { |
| 73 | t.Helper() |
| 74 | if err := provision.CreateRecord(context.Background(), fx.rel, provision.Record{ |
| 75 | Schema: 1, Host: host, Slug: strings.SplitN(host, ".", 2)[0], |
| 76 | App: app, Account: release.OpsAccount, Region: region, Channel: "stable", |
| 77 | CreatedAt: time.Now().UTC(), CreatedBy: "paul@keymail.dev", |
| 78 | }); err != nil { |
| 79 | t.Fatal(err) |
| 80 | } |
| 81 | } |
| 82 | |
| 83 | // Astra plan 6 R1 #6: a customer-fleet box's report counts. Its held host |
| 84 | // shows held on the list and on the instance page, and the retry door takes |
| 85 | // the retry, through the one loader all three share. The record is placed on |
| 86 | // the fleet, so an unreadable snapshot there makes the list incomplete. |
| 87 | func TestFleetBoxHoldShowsOnTheListPageAndRetryDoor(t *testing.T) { |
| 88 | fx := newConsoleFixture(t) |
| 89 | fx.addRoute(t, registry.Route{Host: heldHost, App: "kass", Account: fx.sqid, Kind: "instance", Status: "ready", Version: "v2", Channel: "stable"}) |
| 90 | placedRecord(t, fx, "kass", heldHost, registry.PlacementTarget("pi")) |
| 91 | label := writeFleetHeldBox(t, fx, "pi", fx.sqid, heldHost, "kass", "v2") |
| 92 | token := cliToken(t, fx, "paul@keymail.dev", "code-fleet-held") |
| 93 | |
| 94 | ro, incomplete := listRollout(t, fx, token, "kass", heldHost) |
| 95 | if ro == nil || ro.Box != label || ro.HeldBy != "v2" || incomplete { |
| 96 | t.Fatalf("list rollout = %+v, incomplete %v: want the fleet box's hold, complete", ro, incomplete) |
| 97 | } |
| 98 | c := signInPaul(t, fx) |
| 99 | if code, _, body := get(t, c, fx.u("/apps/kass/instances/"+heldHost)); code != http.StatusOK || |
| 100 | !strings.Contains(body, fmt.Sprintf(swapCopy.HoldText, "v2", rolloutwords.Reason(status.ReasonHealth), "v1")) { |
| 101 | t.Fatalf("page = %d, lacks the fleet box's hold", code) |
| 102 | } |
| 103 | code, _, body := rawReq(t, http.MethodPost, fx.ts.URL+"/api/cli/apps/"+fx.sqid+"/kass/instances/"+heldHost+"/retry", token, `{}`) |
| 104 | if code != http.StatusOK || !strings.Contains(body, `"version":"v2"`) { |
| 105 | t.Fatalf("retry = %d %s, want 200 for v2", code, body) |
| 106 | } |
| 107 | |
| 108 | // A snapshot that cannot be read on the fleet the record is placed on |
| 109 | // makes the join incomplete. |
| 110 | if _, err := fx.rel.Put(context.Background(), "fleets/pi/status/boxes/broken.json", []byte("{not json"), ""); err != nil { |
| 111 | t.Fatal(err) |
| 112 | } |
| 113 | if _, incomplete := listRollout(t, fx, token, "kass", heldHost); !incomplete { |
| 114 | t.Fatal("an unreadable fleet box snapshot left the join complete") |
| 115 | } |
| 116 | } |
| 117 | |
| 118 | // Astra plan 6 R1 #6: only this account's fleets are read. Another |
| 119 | // account's fleet box that names this account's host is never joined, |
| 120 | // whatever account its report claims: no hold on the list or the page, and |
| 121 | // the retry door refuses. |
| 122 | func TestAnotherAccountsFleetBoxIsNeverJoined(t *testing.T) { |
| 123 | fx := newConsoleFixture(t) |
| 124 | fx.addRoute(t, registry.Route{Host: heldHost, App: "kass", Account: fx.sqid, Kind: "instance", Status: "ready", Version: "v2", Channel: "stable"}) |
| 125 | sibling(t, fx, "kass", heldHost, "stable") |
| 126 | writeFleetHeldBox(t, fx, "stranger", "zzz", heldHost, "kass", "v2") |
| 127 | token := cliToken(t, fx, "paul@keymail.dev", "code-fleet-stranger") |
| 128 | |
| 129 | if ro, _ := listRollout(t, fx, token, "kass", heldHost); ro != nil { |
| 130 | t.Fatalf("list joined another account's fleet box: %+v", ro) |
| 131 | } |
| 132 | c := signInPaul(t, fx) |
| 133 | if _, _, body := get(t, c, fx.u("/apps/kass/instances/"+heldHost)); strings.Contains(body, swapCopy.Installed) { |
| 134 | t.Fatal("page joined another account's fleet box") |
| 135 | } |
| 136 | code, _, body := rawReq(t, http.MethodPost, fx.ts.URL+"/api/cli/apps/"+fx.sqid+"/kass/instances/"+heldHost+"/retry", token, `{}`) |
| 137 | if code != http.StatusConflict || !strings.Contains(body, fmt.Sprintf(apiRetryNotHeld, heldHost)) { |
| 138 | t.Fatalf("retry = %d %s, want 409", code, body) |
| 139 | } |
| 140 | } |
| 141 | |
| 142 | // A platform box's report wins over a fleet box's for the same host: the |
| 143 | // operator's boxes are the ones the console trusts most. |
| 144 | func TestJoinFleetRolloutsNeverOverrideAPlatformBox(t *testing.T) { |
| 145 | now := time.Now() |
| 146 | view := func(label, build string) status.Box { |
| 147 | return status.Box{GeneratedAt: now, Label: label, Metrics: &status.BoxMetrics{Rollout: &status.RolloutReport{ |
| 148 | Hosts: map[string]status.HostRollout{"h": {Account: "bab", App: "shop", Installed: build}}, |
| 149 | }}} |
| 150 | } |
| 151 | j := joinBoxes([]status.Box{view("platform", "v1")}, now) |
| 152 | j.joinFleetRollouts([]status.Box{view("fleet", "v9")}, now) |
| 153 | if got := j.rollout("h", "bab", "shop"); got == nil || got.Box != "platform" || got.Installed != "v1" { |
| 154 | t.Fatalf("h = %+v, want the platform box's report", got) |
| 155 | } |
| 156 | var empty boxJoin |
| 157 | empty.joinFleetRollouts([]status.Box{view("fleet", "v9")}, now) |
| 158 | if got := empty.rollout("h", "bab", "shop"); got == nil || got.Box != "fleet" { |
| 159 | t.Fatalf("h on an empty join = %+v, want the fleet box's report", got) |
| 160 | } |
| 161 | } |
| 162 | |
| 163 | // Astra plan 6 R2 (new Medium): completeness covers only the sources that |
| 164 | // could hold this app's reports. The account has a fleet whose snapshot is |
| 165 | // corrupt, but none of the app's instances is placed there; the app runs on |
| 166 | // healthy platform boxes with no reports (flag off). The list must not say |
| 167 | // rollout_incomplete, so the deploy watch falls back to today's URL watch. |
| 168 | func TestAnUnrelatedUnreadableFleetDoesNotMakeTheListIncomplete(t *testing.T) { |
| 169 | fx := newConsoleFixture(t) |
| 170 | placedRecord(t, fx, "kass", heldHost, "ie") |
| 171 | reg, err := registry.Open(fx.regPath) |
| 172 | if err != nil { |
| 173 | t.Fatal(err) |
| 174 | } |
| 175 | defer reg.Close() |
| 176 | if err := reg.UpsertFleet(registry.Fleet{Name: "spare", Account: fx.sqid, Bucket: "spare-bucket"}); err != nil { |
| 177 | t.Fatal(err) |
| 178 | } |
| 179 | if _, err := fx.rel.Put(context.Background(), "fleets/spare/status/boxes/broken.json", []byte("{not json"), ""); err != nil { |
| 180 | t.Fatal(err) |
| 181 | } |
| 182 | box := status.CollectBox(nil, status.Probes{MachineID: "box1"}, time.Now()) |
| 183 | box.Metrics = &status.BoxMetrics{RouteVersions: map[string]string{heldHost: "v1"}} |
| 184 | if err := status.WriteBox(context.Background(), fx.rel, box); err != nil { |
| 185 | t.Fatal(err) |
| 186 | } |
| 187 | token := cliToken(t, fx, "paul@keymail.dev", "code-fleet-unrelated") |
| 188 | ro, incomplete := listRollout(t, fx, token, "kass", heldHost) |
| 189 | if ro != nil || incomplete { |
| 190 | t.Fatalf("rollout %+v, incomplete %v: an unrelated fleet's unreadable snapshot blocked the legacy watch", ro, incomplete) |
| 191 | } |
| 192 | } |
| 193 | |
| 194 | // Astra plan 6 R3: an unreadable record must never drop its fleet from the |
| 195 | // completeness check. The record's host sits on fleet/pi, whose only |
| 196 | // snapshot is unreadable too. The readable record is on the platform, so |
| 197 | // without the fix no fleet is relevant and the list would claim to be |
| 198 | // complete, sending the deploy watch to poll URLs it must not poll. |
| 199 | func TestAnUnreadableRecordKeepsEveryFleetInTheCompletenessCheck(t *testing.T) { |
| 200 | fx := newConsoleFixture(t) |
| 201 | const piHost = "pi-host.example.test" |
| 202 | placedRecord(t, fx, "kass", heldHost, "ie") |
| 203 | placedRecord(t, fx, "kass", piHost, registry.PlacementTarget("pi")) |
| 204 | reg, err := registry.Open(fx.regPath) |
| 205 | if err != nil { |
| 206 | t.Fatal(err) |
| 207 | } |
| 208 | defer reg.Close() |
| 209 | if err := reg.UpsertFleet(registry.Fleet{Name: "pi", Account: fx.sqid, Bucket: "pi-bucket"}); err != nil { |
| 210 | t.Fatal(err) |
| 211 | } |
| 212 | box := status.CollectBox(nil, status.Probes{MachineID: "box1"}, time.Now()) |
| 213 | box.Metrics = &status.BoxMetrics{RouteVersions: map[string]string{heldHost: "v1"}} |
| 214 | if err := status.WriteBox(context.Background(), fx.rel, box); err != nil { |
| 215 | t.Fatal(err) |
| 216 | } |
| 217 | token := cliToken(t, fx, "paul@keymail.dev", "code-unreadable-record") |
| 218 | |
| 219 | // Both readable and complete first: the platform record's fleet set is |
| 220 | // empty, but the pi record is placed on pi, whose snapshot is fine. |
| 221 | if _, incomplete := listRollout(t, fx, token, "kass", heldHost); incomplete { |
| 222 | t.Fatal("a healthy app was reported incomplete before anything was broken") |
| 223 | } |
| 224 | |
| 225 | // Now the pi record and the pi snapshot both go unreadable. |
| 226 | key := provision.RecordKey(release.OpsAccount, "kass", piHost) |
| 227 | _, etag, err := fx.rel.Get(context.Background(), key) |
| 228 | if err != nil { |
| 229 | t.Fatal(err) |
| 230 | } |
| 231 | if _, err := fx.rel.Put(context.Background(), key, []byte("{not json"), etag); err != nil { |
| 232 | t.Fatal(err) |
| 233 | } |
| 234 | if _, err := fx.rel.Put(context.Background(), "fleets/pi/status/boxes/broken.json", []byte("{not json"), ""); err != nil { |
| 235 | t.Fatal(err) |
| 236 | } |
| 237 | if _, incomplete := listRollout(t, fx, token, "kass", heldHost); !incomplete { |
| 238 | t.Fatal("an unreadable record dropped its fleet from the completeness check") |
| 239 | } |
| 240 | } |
| 241 | |
| 242 | // The fleets that could hold an app's reports: where its records are placed, |
| 243 | // and both ends of a move in flight. A platform region names none. |
| 244 | func TestRecordFleetsNamesPlacementsAndMoveEnds(t *testing.T) { |
| 245 | got := recordFleets([]provision.Record{ |
| 246 | {Host: "a", Region: "ie"}, |
| 247 | {Host: "b", Region: registry.PlacementTarget("pi")}, |
| 248 | {Host: "c", Region: "ie", Move: &provision.MoveState{From: "ie", To: registry.PlacementTarget("zero")}}, |
| 249 | }) |
| 250 | if len(got) != 2 || !got["pi"] || !got["zero"] { |
| 251 | t.Fatalf("fleets = %v, want pi and zero", got) |
| 252 | } |
| 253 | if got := recordFleets(nil); got == nil || len(got) != 0 { |
| 254 | t.Fatalf("no records = %v, want an empty, non-nil set", got) |
| 255 | } |
| 256 | } |
| 257 | |
amadan