| 1 | package console |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "fmt" |
| 7 | "net/http" |
| 8 | "strings" |
| 9 | "testing" |
| 10 | "time" |
| 11 | |
| 12 | "github.com/carlosframework/platform/internal/release" |
| 13 | "github.com/carlosframework/platform/internal/status" |
| 14 | ) |
| 15 | |
| 16 | func TestAPIInstanceListCarriesTheRollout(t *testing.T) { |
| 17 | fx, token := instanceFixture(t, "code-inst-rollout") |
| 18 | enableInstances(t, fx, "titogo", titoManifest()) |
| 19 | u := instancesURL(fx, "bab", "titogo") |
| 20 | for _, h := range []string{"acme.go.tito.io", "beta.go.tito.io"} { |
| 21 | if code, _, out := rawReq(t, http.MethodPost, u, token, fmt.Sprintf(`{"host":%q,"region":"ie"}`, h)); code != http.StatusCreated { |
| 22 | t.Fatalf("create %s = %d %s", h, code, out) |
| 23 | } |
| 24 | } |
| 25 | box := status.CollectBox(nil, status.Probes{MachineID: "box1"}, time.Now()) |
| 26 | box.Metrics = &status.BoxMetrics{Rollout: &status.RolloutReport{ |
| 27 | Hosts: map[string]status.HostRollout{ |
| 28 | "acme.go.tito.io": {Account: "bab", App: "titogo", Installed: "v2", Serving: "v1", Good: "v1", |
| 29 | Awake: true, Attempt: status.AttemptRunning, AttemptBuild: "v2"}, |
| 30 | // A report naming another app under this host: never joined. |
| 31 | "beta.go.tito.io": {Account: "bab", App: "other", Installed: "v9", Serving: "v9"}, |
| 32 | }, |
| 33 | Apps: map[string]status.AppRollout{"bab/titogo": {Running: 1}}, |
| 34 | }} |
| 35 | if err := status.WriteBox(context.Background(), fx.rel, box); err != nil { |
| 36 | t.Fatal(err) |
| 37 | } |
| 38 | code, _, out := rawReq(t, http.MethodGet, u, token, "") |
| 39 | if code != http.StatusOK { |
| 40 | t.Fatalf("list = %d %s", code, out) |
| 41 | } |
| 42 | var list struct { |
| 43 | Instances []struct { |
| 44 | Host string `json:"host"` |
| 45 | Rollout *rolloutView `json:"rollout"` |
| 46 | } `json:"instances"` |
| 47 | } |
| 48 | if err := json.Unmarshal([]byte(out), &list); err != nil { |
| 49 | t.Fatal(err) |
| 50 | } |
| 51 | got := map[string]*rolloutView{} |
| 52 | for _, r := range list.Instances { |
| 53 | got[r.Host] = r.Rollout |
| 54 | } |
| 55 | a := got["acme.go.tito.io"] |
| 56 | if a == nil || a.Box != box.Label || a.Installed != "v2" || a.Serving != "v1" || a.Attempt != status.AttemptRunning || a.SchedulerIdle { |
| 57 | t.Fatalf("acme rollout = %+v", a) |
| 58 | } |
| 59 | if got["beta.go.tito.io"] != nil { |
| 60 | t.Fatalf("beta joined another app's report: %+v", got["beta.go.tito.io"]) |
| 61 | } |
| 62 | } |
| 63 | |
| 64 | func TestJoinBoxesIdleNeedsNothingQueuedRunningOrSettling(t *testing.T) { |
| 65 | idle := func(a status.AppRollout) bool { |
| 66 | b := status.Box{GeneratedAt: time.Now(), Label: "x", Metrics: &status.BoxMetrics{Rollout: &status.RolloutReport{ |
| 67 | Hosts: map[string]status.HostRollout{"h": {Account: "bab", App: "shop", Installed: "v1"}}, |
| 68 | Apps: map[string]status.AppRollout{"bab/shop": a}, |
| 69 | }}} |
| 70 | return joinBoxes([]status.Box{b}, time.Now()).rollout("h", "bab", "shop").SchedulerIdle |
| 71 | } |
| 72 | if !idle(status.AppRollout{}) { |
| 73 | t.Error("an empty scheduler is not idle") |
| 74 | } |
| 75 | for _, a := range []status.AppRollout{{Queued: 1}, {Running: 1}, {Settling: 1}} { |
| 76 | if idle(a) { |
| 77 | t.Errorf("%+v reads as idle", a) |
| 78 | } |
| 79 | } |
| 80 | } |
| 81 | |
| 82 | // Plan 5 reports every host a failure record holds as failed. Spec §9.1: |
| 83 | // the one the build failed on is failed, its siblings are held. |
| 84 | func TestJoinBoxesTellsFailedFromHeld(t *testing.T) { |
| 85 | failed := status.HostRollout{Account: "bab", App: "shop", Installed: "v2", Serving: "v1", Good: "v1", Awake: true, |
| 86 | Attempt: status.AttemptFailed, AttemptBuild: "v2", HeldBy: "v2", Reason: status.ReasonHealth} |
| 87 | b := status.Box{GeneratedAt: time.Now(), Label: "x", Metrics: &status.BoxMetrics{Rollout: &status.RolloutReport{ |
| 88 | Hosts: map[string]status.HostRollout{"a": failed, "b": failed}, |
| 89 | Apps: map[string]status.AppRollout{"bab/shop": {}}, |
| 90 | Failures: []status.RolloutFailure{{Account: "bab", App: "shop", Build: "v2", Host: "a", Reason: status.ReasonHealth}}, |
| 91 | }}} |
| 92 | j := joinBoxes([]status.Box{b}, time.Now()) |
| 93 | if a := j.rollout("a", "bab", "shop"); a.Attempt != status.AttemptFailed || a.AttemptBuild != "v2" || a.HeldBy != "v2" { |
| 94 | t.Errorf("a, where v2 failed = %+v", a) |
| 95 | } |
| 96 | if sib := j.rollout("b", "bab", "shop"); sib.Attempt != "" || sib.AttemptBuild != "" || sib.HeldBy != "v2" || sib.Reason != status.ReasonHealth { |
| 97 | t.Errorf("b, held by a's failure = %+v", sib) |
| 98 | } |
| 99 | } |
| 100 | |
| 101 | func TestAPIPromoteHeldSaysWhenItLifts(t *testing.T) { |
| 102 | fx := newConsoleFixture(t) |
| 103 | if _, err := fx.st.CreateAppIn(OpsAccountID, "titogo", "paul@keymail.dev", time.Now()); err != nil { |
| 104 | t.Fatal(err) |
| 105 | } |
| 106 | shipVersion(t, fx, "titogo", "abc1234") |
| 107 | seedChannel(t, fx, "titogo", "edge", "abc1234") |
| 108 | if err := release.SetProduction(context.Background(), fx.rel, fx.priv, release.OpsAccount, "titogo", true, "test"); err != nil { |
| 109 | t.Fatal(err) |
| 110 | } |
| 111 | token := cliToken(t, fx, "paul@keymail.dev", "code-held-until") |
| 112 | before := time.Now() |
| 113 | code, body := postRaw(t, fx.ts.URL+"/api/cli/apps/bab/titogo/promote", |
| 114 | map[string]string{"version": "abc1234", "channel": "beta"}, token) |
| 115 | if code != http.StatusOK || !held(t, body) { |
| 116 | t.Fatalf("beta promote = %d %s", code, body) |
| 117 | } |
| 118 | var out struct { |
| 119 | HeldUntil string `json:"held_until"` |
| 120 | } |
| 121 | if err := json.Unmarshal([]byte(body), &out); err != nil { |
| 122 | t.Fatal(err) |
| 123 | } |
| 124 | until, err := time.Parse(time.RFC3339, out.HeldUntil) |
| 125 | if err != nil { |
| 126 | t.Fatalf("held_until = %q: %v", out.HeldUntil, err) |
| 127 | } |
| 128 | window := release.HoldFor(release.DefaultHolds, "beta") |
| 129 | if until.Before(before.Add(window).Add(-time.Minute)) || until.After(time.Now().Add(window).Add(time.Minute)) { |
| 130 | t.Fatalf("held_until = %s, want about now + %s", until, window) |
| 131 | } |
| 132 | code, body = postRaw(t, fx.ts.URL+"/api/cli/apps/bab/titogo/promote", |
| 133 | map[string]string{"version": "abc1234", "channel": "canary/x"}, token) |
| 134 | var canary struct { |
| 135 | HeldUntil string `json:"held_until"` |
| 136 | } |
| 137 | if code != http.StatusOK || held(t, body) || json.Unmarshal([]byte(body), &canary) != nil || canary.HeldUntil != "" { |
| 138 | t.Fatalf("canary promote = %d %s", code, body) |
| 139 | } |
| 140 | } |
| 141 | |
| 142 | // A box that armed nothing says why (CARLOS_SWAP is on but the scheduler could |
| 143 | // not arm): its Hosts map is empty, so the join answers from the box's route |
| 144 | // versions with the reason code and nothing else. |
| 145 | func TestJoinBoxesCarriesTheRefusalOfAnUnavailableBox(t *testing.T) { |
| 146 | b := status.Box{GeneratedAt: time.Now(), Label: "refusing", Metrics: &status.BoxMetrics{ |
| 147 | RouteVersions: map[string]string{"h": "v3"}, |
| 148 | Rollout: &status.RolloutReport{Unavailable: "no-deploy-store"}, |
| 149 | }} |
| 150 | got := joinBoxes([]status.Box{b}, time.Now()).rollout("h", "bab", "shop") |
| 151 | if got == nil || got.Box != "refusing" || got.Installed != "v3" || got.Unavailable != "no-deploy-store" || |
| 152 | got.Serving != "" || got.Attempt != "" || got.SchedulerIdle { |
| 153 | t.Fatalf("refusing box rollout = %+v", got) |
| 154 | } |
| 155 | if joinBoxes([]status.Box{b}, time.Now()).rollout("other", "bab", "shop") != nil { |
| 156 | t.Error("a host the refusing box does not list joined a rollout") |
| 157 | } |
| 158 | raw, err := json.Marshal(got) |
| 159 | if err != nil || !strings.Contains(string(raw), `"unavailable":"no-deploy-store"`) { |
| 160 | t.Fatalf("wire = %s, %v", raw, err) |
| 161 | } |
| 162 | } |
| 163 | |
| 164 | // A box with no rollout report (CARLOS_SWAP off, or from before plan 5) joins |
| 165 | // nothing: the flag-off wire is unchanged. |
| 166 | func TestJoinBoxesNilReportJoinsNothing(t *testing.T) { |
| 167 | b := status.Box{GeneratedAt: time.Now(), Label: "old", Metrics: &status.BoxMetrics{ |
| 168 | RouteVersions: map[string]string{"h": "v3"}, |
| 169 | }} |
| 170 | if got := joinBoxes([]status.Box{b}, time.Now()).rollout("h", "bab", "shop"); got != nil { |
| 171 | t.Fatalf("nil report joined %+v", got) |
| 172 | } |
| 173 | } |
| 174 | |
| 175 | // Astra plan 6 R1 #4: the list says when its rollout join could not read |
| 176 | // every box's report (rollout_incomplete), so the deploy watch never reads a |
| 177 | // missing report as "this box does not report". The flag is additive: a |
| 178 | // complete join leaves the key off, and the wire is as before. |
| 179 | func TestAPIInstanceListSaysWhenTheRolloutJoinIsIncomplete(t *testing.T) { |
| 180 | fx, token := instanceFixture(t, "code-inst-incomplete") |
| 181 | enableInstances(t, fx, "titogo", titoManifest()) |
| 182 | u := instancesURL(fx, "bab", "titogo") |
| 183 | if code, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"acme.go.tito.io","region":"ie"}`); code != http.StatusCreated { |
| 184 | t.Fatalf("create = %d %s", code, out) |
| 185 | } |
| 186 | box := status.CollectBox(nil, status.Probes{MachineID: "box1"}, time.Now()) |
| 187 | box.Metrics = &status.BoxMetrics{Rollout: &status.RolloutReport{Apps: map[string]status.AppRollout{"bab/titogo": {}}}} |
| 188 | if err := status.WriteBox(context.Background(), fx.rel, box); err != nil { |
| 189 | t.Fatal(err) |
| 190 | } |
| 191 | incomplete := func() (bool, bool) { |
| 192 | t.Helper() |
| 193 | code, _, out := rawReq(t, http.MethodGet, u, token, "") |
| 194 | if code != http.StatusOK { |
| 195 | t.Fatalf("list = %d %s", code, out) |
| 196 | } |
| 197 | var raw map[string]json.RawMessage |
| 198 | if err := json.Unmarshal([]byte(out), &raw); err != nil { |
| 199 | t.Fatal(err) |
| 200 | } |
| 201 | v, ok := raw["rollout_incomplete"] |
| 202 | return ok, ok && string(v) == "true" |
| 203 | } |
| 204 | if present, _ := incomplete(); present { |
| 205 | t.Fatal("a complete join carries rollout_incomplete") |
| 206 | } |
| 207 | if _, err := fx.rel.Put(context.Background(), "status/boxes/broken.json", []byte("{not json"), ""); err != nil { |
| 208 | t.Fatal(err) |
| 209 | } |
| 210 | if _, set := incomplete(); !set { |
| 211 | t.Fatal("a join that skipped an unreadable box snapshot does not say rollout_incomplete") |
| 212 | } |
| 213 | } |
| 214 | |
| 215 | // A box's transient word reaches the member as "transient": true, and an |
| 216 | // ordinary host carries no such key. |
| 217 | func TestAPIInstanceListCarriesATransientHost(t *testing.T) { |
| 218 | fx, token := instanceFixture(t, "code-inst-transient") |
| 219 | enableInstances(t, fx, "titogo", titoManifest()) |
| 220 | u := instancesURL(fx, "bab", "titogo") |
| 221 | for _, h := range []string{"acme.go.tito.io", "beta.go.tito.io"} { |
| 222 | if code, _, out := rawReq(t, http.MethodPost, u, token, fmt.Sprintf(`{"host":%q,"region":"ie"}`, h)); code != http.StatusCreated { |
| 223 | t.Fatalf("create %s = %d %s", h, code, out) |
| 224 | } |
| 225 | } |
| 226 | box := status.CollectBox(nil, status.Probes{MachineID: "box1"}, time.Now()) |
| 227 | box.Metrics = &status.BoxMetrics{Rollout: &status.RolloutReport{ |
| 228 | Hosts: map[string]status.HostRollout{ |
| 229 | "acme.go.tito.io": {Account: "bab", App: "titogo", Installed: "v2", Good: "v1", Transient: true}, |
| 230 | "beta.go.tito.io": {Account: "bab", App: "titogo", Installed: "v2", Serving: "v2", Good: "v2", Awake: true}, |
| 231 | }, |
| 232 | Apps: map[string]status.AppRollout{"bab/titogo": {}}, |
| 233 | }} |
| 234 | if err := status.WriteBox(context.Background(), fx.rel, box); err != nil { |
| 235 | t.Fatal(err) |
| 236 | } |
| 237 | code, _, out := rawReq(t, http.MethodGet, u, token, "") |
| 238 | if code != http.StatusOK { |
| 239 | t.Fatalf("list = %d %s", code, out) |
| 240 | } |
| 241 | var list struct { |
| 242 | Instances []struct { |
| 243 | Host string `json:"host"` |
| 244 | Rollout json.RawMessage `json:"rollout"` |
| 245 | } `json:"instances"` |
| 246 | } |
| 247 | if err := json.Unmarshal([]byte(out), &list); err != nil { |
| 248 | t.Fatal(err) |
| 249 | } |
| 250 | got := map[string]map[string]any{} |
| 251 | for _, r := range list.Instances { |
| 252 | var m map[string]any |
| 253 | if err := json.Unmarshal(r.Rollout, &m); err != nil { |
| 254 | t.Fatalf("%s rollout: %v", r.Host, err) |
| 255 | } |
| 256 | got[r.Host] = m |
| 257 | } |
| 258 | if got["acme.go.tito.io"]["transient"] != true { |
| 259 | t.Fatalf("acme rollout = %v, want transient true", got["acme.go.tito.io"]) |
| 260 | } |
| 261 | if _, has := got["beta.go.tito.io"]["transient"]; has { |
| 262 | t.Fatalf("beta rollout = %v, want no transient key", got["beta.go.tito.io"]) |
| 263 | } |
| 264 | } |
| 265 | |
amadan