| 1 | package console |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "fmt" |
| 7 | "net/http" |
| 8 | "strings" |
| 9 | "testing" |
| 10 | "time" |
| 11 | |
| 12 | "github.com/carlosframework/platform/internal/release" |
| 13 | "github.com/carlosframework/platform/internal/status" |
| 14 | ) |
| 15 | |
| 16 | // lastScoped reads the newest request in the app's host-retry log, where |
| 17 | // a retry's grant rides (readiness round 2, Task 3), and checks the |
| 18 | // restart-scope log took none. |
| 19 | func lastScoped(t *testing.T, fx *consoleFixture, app string) release.RestartRequest { |
| 20 | t.Helper() |
| 21 | g, found, err := release.LoadHostRetries(context.Background(), fx.rel, fx.sqid, app) |
| 22 | if err != nil || !found || len(g.Requests) == 0 { |
| 23 | t.Fatalf("host retries = %+v, %v, %v", g, found, err) |
| 24 | } |
| 25 | if old, found, _ := release.LoadRestartScopes(context.Background(), fx.rel, fx.sqid, app); found && len(old.Requests) != 0 { |
| 26 | t.Fatalf("a retry wrote the restart-scope log: %+v", old.Requests) |
| 27 | } |
| 28 | return g.Requests[len(g.Requests)-1] |
| 29 | } |
| 30 | |
| 31 | // noRetryWritten checks neither log holds a request for app. |
| 32 | func noRetryWritten(t *testing.T, fx *consoleFixture, app string) { |
| 33 | t.Helper() |
| 34 | ctx := context.Background() |
| 35 | g, found, err := release.LoadHostRetries(ctx, fx.rel, fx.sqid, app) |
| 36 | if err != nil { |
| 37 | t.Fatalf("load the host retries: %v", err) |
| 38 | } |
| 39 | if found && len(g.Requests) != 0 { |
| 40 | t.Fatalf("a retry was written: %+v", g.Requests) |
| 41 | } |
| 42 | g, found, err = release.LoadRestartScopes(ctx, fx.rel, fx.sqid, app) |
| 43 | if err != nil { |
| 44 | t.Fatalf("load the restart-scope log: %v", err) |
| 45 | } |
| 46 | if found && len(g.Requests) != 0 { |
| 47 | t.Fatalf("a request was written to the restart-scope log: %+v", g.Requests) |
| 48 | } |
| 49 | } |
| 50 | |
| 51 | // heldEpoch is the restriction epoch writeHeldBox reports, and heldBook the |
| 52 | // failure book it counts in (final review round 1, finding 3). |
| 53 | const ( |
| 54 | heldEpoch = 17 |
| 55 | heldBook = "0123456789abcdef0123456789abcdef" |
| 56 | ) |
| 57 | |
| 58 | func writeHeldBox(t *testing.T, fx *consoleFixture, host, app, build string) { |
| 59 | t.Helper() |
| 60 | box := status.CollectBox(nil, status.Probes{MachineID: "box1"}, time.Now()) |
| 61 | box.Metrics = &status.BoxMetrics{Rollout: &status.RolloutReport{Hosts: map[string]status.HostRollout{ |
| 62 | host: {Account: fx.sqid, App: app, Installed: build, Serving: "v1", Good: "v1", Awake: true, HeldBy: build, HeldEpoch: heldEpoch, HeldBook: heldBook, RetryV: status.HostRetryV, Reason: status.ReasonHealth}, |
| 63 | }}} |
| 64 | if err := status.WriteBox(context.Background(), fx.rel, box); err != nil { |
| 65 | t.Fatal(err) |
| 66 | } |
| 67 | } |
| 68 | |
| 69 | func TestAPIInstanceRetryWritesARetryRequest(t *testing.T) { |
| 70 | fx := newConsoleFixture(t) |
| 71 | const host = "web.kass.carlosapp.dev" |
| 72 | sibling(t, fx, "kass", host, "stable") |
| 73 | token := cliToken(t, fx, "paul@keymail.dev", "code-host-retry") |
| 74 | u := fx.ts.URL + "/api/cli/apps/" + fx.sqid + "/kass/instances/" + host + "/retry" |
| 75 | |
| 76 | if code, _, body := rawReq(t, http.MethodPost, u, token, `{}`); code != http.StatusConflict || !strings.Contains(body, fmt.Sprintf(apiRetryNotHeld, host)) { |
| 77 | t.Fatalf("no report = %d %s, want 409 naming %s as not held", code, body, host) |
| 78 | } |
| 79 | writeHeldBox(t, fx, host, "kass", "v2") |
| 80 | code, _, body := rawReq(t, http.MethodPost, u, token, `{}`) |
| 81 | if code != http.StatusOK { |
| 82 | t.Fatalf("retry = %d %s", code, body) |
| 83 | } |
| 84 | var out struct{ ID, Version, Host string } |
| 85 | if err := json.Unmarshal([]byte(body), &out); err != nil || out.Version != "v2" || out.Host != host || out.ID == "" { |
| 86 | t.Fatalf("answer = %s", body) |
| 87 | } |
| 88 | r := lastScoped(t, fx, "kass") |
| 89 | if r.Scope.Kind != release.RestartScopeRetry || r.Scope.Value != host || r.ID != out.ID { |
| 90 | t.Fatalf("stored = %+v", r) |
| 91 | } |
| 92 | // The entry carries the root's grant for the held build at the box's |
| 93 | // epoch (spec §11, plan review finding 13). |
| 94 | g, err := release.VerifyHostRetry(fx.pub, r.Scope.Signed, fx.sqid, "kass", host, time.Now()) |
| 95 | if err != nil || g.Build != "v2" || g.Epoch != heldEpoch || g.Book != heldBook { |
| 96 | t.Fatalf("signed grant = %+v, %v; want v2 at epoch %d of book %s", g, err, heldEpoch, heldBook) |
| 97 | } |
| 98 | if n := serial(t, fx); n != 1 { |
| 99 | t.Fatalf("serial = %d", n) |
| 100 | } |
| 101 | code, _, body = rawReq(t, http.MethodPost, fx.ts.URL+"/api/cli/apps/"+fx.sqid+"/kass/instances/nope.kass.carlosapp.dev/retry", token, `{}`) |
| 102 | if code != http.StatusNotFound || body != "404 page not found\n" { |
| 103 | t.Fatalf("unknown host = %d %q", code, body) |
| 104 | } |
| 105 | } |
| 106 | |
| 107 | func TestAPIInstanceRetryIgnoresAnotherAppsReport(t *testing.T) { |
| 108 | fx := newConsoleFixture(t) |
| 109 | const host = "web.kass.carlosapp.dev" |
| 110 | sibling(t, fx, "kass", host, "stable") |
| 111 | writeHeldBox(t, fx, host, "otherapp", "z2") |
| 112 | token := cliToken(t, fx, "paul@keymail.dev", "code-host-retry-x") |
| 113 | u := fx.ts.URL + "/api/cli/apps/" + fx.sqid + "/kass/instances/" + host + "/retry" |
| 114 | if code, _, body := rawReq(t, http.MethodPost, u, token, `{}`); code != http.StatusConflict || !strings.Contains(body, fmt.Sprintf(apiRetryNotHeld, host)) { |
| 115 | t.Fatalf("another app's hold = %d %s, want 409 naming %s as not held", code, body, host) |
| 116 | } |
| 117 | } |
| 118 | |
| 119 | // A box that refuses every managed start reports no per-host state: its view |
| 120 | // is built with Unavailable and no HeldBy, so the host is not held. This pins |
| 121 | // that whole path to the 409. TestAPIInstanceRetryOfADeclinedBoxsHold covers |
| 122 | // a view carrying both. |
| 123 | func TestAPIInstanceRetryRefusesAnUnavailableBox(t *testing.T) { |
| 124 | fx := newConsoleFixture(t) |
| 125 | const host = "web.kass.carlosapp.dev" |
| 126 | sibling(t, fx, "kass", host, "stable") |
| 127 | box := status.CollectBox(nil, status.Probes{MachineID: "box1"}, time.Now()) |
| 128 | box.Metrics = &status.BoxMetrics{ |
| 129 | RouteVersions: map[string]string{host: "v2"}, |
| 130 | Rollout: &status.RolloutReport{Unavailable: "no-deploy-store"}, |
| 131 | } |
| 132 | if err := status.WriteBox(context.Background(), fx.rel, box); err != nil { |
| 133 | t.Fatal(err) |
| 134 | } |
| 135 | token := cliToken(t, fx, "paul@keymail.dev", "code-host-retry-u") |
| 136 | u := fx.ts.URL + "/api/cli/apps/" + fx.sqid + "/kass/instances/" + host + "/retry" |
| 137 | code, _, body := rawReq(t, http.MethodPost, u, token, `{}`) |
| 138 | if code != http.StatusConflict || !strings.Contains(body, fmt.Sprintf(apiRetryNotHeld, host)) { |
| 139 | t.Fatalf("unavailable box = %d %s, want 409 naming %s as not held", code, body, host) |
| 140 | } |
| 141 | } |
| 142 | |
| 143 | // Task 10 Correction 1 (plan review #5): a box that declined |
| 144 | // CARLOS_SWAP=overlap but honours the failure book an armed run left |
| 145 | // publishes its held hosts beside its decline code, and the door signs a |
| 146 | // retry for one from that report. Any other Unavailable code is still |
| 147 | // refused, whatever its report carries, and a declined box's host it does |
| 148 | // not report as held is not held. |
| 149 | func TestAPIInstanceRetryOfADeclinedBoxsHold(t *testing.T) { |
| 150 | const host = "web.kass.carlosapp.dev" |
| 151 | write := func(t *testing.T, fx *consoleFixture, code string, held bool) { |
| 152 | t.Helper() |
| 153 | ro := &status.RolloutReport{Unavailable: code, FailureBook: "honoured"} |
| 154 | if held { |
| 155 | ro.Hosts = map[string]status.HostRollout{host: {Account: fx.sqid, App: "kass", Installed: "v2", Good: "v1", |
| 156 | Attempt: status.AttemptDown, AttemptBuild: "v2", HeldBy: "v2", HeldEpoch: heldEpoch, HeldBook: heldBook, RetryV: status.HostRetryV}} |
| 157 | } |
| 158 | box := status.CollectBox(nil, status.Probes{MachineID: "box1"}, time.Now()) |
| 159 | box.Metrics = &status.BoxMetrics{RouteVersions: map[string]string{host: "v2"}, Rollout: ro} |
| 160 | if err := status.WriteBox(context.Background(), fx.rel, box); err != nil { |
| 161 | t.Fatal(err) |
| 162 | } |
| 163 | } |
| 164 | for _, tc := range []struct { |
| 165 | name string |
| 166 | code string |
| 167 | held bool |
| 168 | ok bool |
| 169 | }{ |
| 170 | {"declined, held", "no-cgroup-delegation", true, true}, |
| 171 | {"claims on, held", "host-claims-on", true, true}, |
| 172 | {"declined, not held", "generations-off", false, false}, |
| 173 | {"refusing, with hosts", "no-deploy-store", true, false}, |
| 174 | {"unknown code, with hosts", "a-newer-code", true, false}, |
| 175 | } { |
| 176 | t.Run(tc.name, func(t *testing.T) { |
| 177 | fx := newConsoleFixture(t) |
| 178 | sibling(t, fx, "kass", host, "stable") |
| 179 | write(t, fx, tc.code, tc.held) |
| 180 | token := cliToken(t, fx, "paul@keymail.dev", "code-host-retry-d") |
| 181 | u := fx.ts.URL + "/api/cli/apps/" + fx.sqid + "/kass/instances/" + host + "/retry" |
| 182 | code, _, body := rawReq(t, http.MethodPost, u, token, `{}`) |
| 183 | if !tc.ok { |
| 184 | if code != http.StatusConflict || !strings.Contains(body, fmt.Sprintf(apiRetryNotHeld, host)) { |
| 185 | t.Fatalf("retry = %d %s, want 409 naming %s as not held", code, body, host) |
| 186 | } |
| 187 | noRetryWritten(t, fx, "kass") |
| 188 | return |
| 189 | } |
| 190 | if code != http.StatusOK { |
| 191 | t.Fatalf("retry = %d %s, want the declined box's hold retried", code, body) |
| 192 | } |
| 193 | r := lastScoped(t, fx, "kass") |
| 194 | g, err := release.VerifyHostRetry(fx.pub, r.Scope.Signed, fx.sqid, "kass", host, time.Now()) |
| 195 | if err != nil || g.Build != "v2" || g.Epoch != heldEpoch || g.Book != heldBook { |
| 196 | t.Fatalf("signed grant = %+v, %v; want v2 at epoch %d of book %s", g, err, heldEpoch, heldBook) |
| 197 | } |
| 198 | }) |
| 199 | } |
| 200 | // Another app's report under the host's name is not this app's hold. |
| 201 | fx := newConsoleFixture(t) |
| 202 | sibling(t, fx, "kass", host, "stable") |
| 203 | box := status.CollectBox(nil, status.Probes{MachineID: "box1"}, time.Now()) |
| 204 | box.Metrics = &status.BoxMetrics{RouteVersions: map[string]string{host: "v2"}, Rollout: &status.RolloutReport{ |
| 205 | Unavailable: "no-cgroup-delegation", FailureBook: "honoured", |
| 206 | Hosts: map[string]status.HostRollout{host: {Account: fx.sqid, App: "otherapp", Installed: "v2", HeldBy: "v2", HeldEpoch: heldEpoch, HeldBook: heldBook}}}} |
| 207 | if err := status.WriteBox(context.Background(), fx.rel, box); err != nil { |
| 208 | t.Fatal(err) |
| 209 | } |
| 210 | token := cliToken(t, fx, "paul@keymail.dev", "code-host-retry-o") |
| 211 | u := fx.ts.URL + "/api/cli/apps/" + fx.sqid + "/kass/instances/" + host + "/retry" |
| 212 | if code, _, body := rawReq(t, http.MethodPost, u, token, `{}`); code != http.StatusConflict { |
| 213 | t.Fatalf("another app's declined hold = %d %s, want 409", code, body) |
| 214 | } |
| 215 | } |
| 216 | |
| 217 | // A box that reports the host and holds nothing back: there is nothing to |
| 218 | // retry, and no request may be written. |
| 219 | func TestAPIInstanceRetryRefusesAHostThatIsNotHeld(t *testing.T) { |
| 220 | fx := newConsoleFixture(t) |
| 221 | const host = "web.kass.carlosapp.dev" |
| 222 | sibling(t, fx, "kass", host, "stable") |
| 223 | box := status.CollectBox(nil, status.Probes{MachineID: "box1"}, time.Now()) |
| 224 | box.Metrics = &status.BoxMetrics{Rollout: &status.RolloutReport{Hosts: map[string]status.HostRollout{ |
| 225 | host: {Account: fx.sqid, App: "kass", Installed: "v2", Serving: "v2", Good: "v2", Awake: true}, |
| 226 | }}} |
| 227 | if err := status.WriteBox(context.Background(), fx.rel, box); err != nil { |
| 228 | t.Fatal(err) |
| 229 | } |
| 230 | token := cliToken(t, fx, "paul@keymail.dev", "code-host-retry-n") |
| 231 | u := fx.ts.URL + "/api/cli/apps/" + fx.sqid + "/kass/instances/" + host + "/retry" |
| 232 | code, _, body := rawReq(t, http.MethodPost, u, token, `{}`) |
| 233 | if code != http.StatusConflict || !strings.Contains(body, fmt.Sprintf(apiRetryNotHeld, host)) { |
| 234 | t.Fatalf("a healthy host = %d %s, want 409 naming %s as not held", code, body, host) |
| 235 | } |
| 236 | noRetryWritten(t, fx, "kass") |
| 237 | } |
| 238 | |
| 239 | // A console with no root key (read-only) cannot sign a retry: it answers |
| 240 | // 503 and writes nothing, so no box is handed an unsigned entry it would |
| 241 | // drop (spec §11). |
| 242 | func TestAPIInstanceRetryWithNoKeyWritesNothing(t *testing.T) { |
| 243 | fx := newConsoleFixture(t) |
| 244 | const host = "web.kass.carlosapp.dev" |
| 245 | sibling(t, fx, "kass", host, "stable") |
| 246 | writeHeldBox(t, fx, host, "kass", "v2") |
| 247 | fx.srv.cfg.Key = nil |
| 248 | token := cliToken(t, fx, "paul@keymail.dev", "code-host-retry-k") |
| 249 | u := fx.ts.URL + "/api/cli/apps/" + fx.sqid + "/kass/instances/" + host + "/retry" |
| 250 | code, _, body := rawReq(t, http.MethodPost, u, token, `{}`) |
| 251 | if code != http.StatusServiceUnavailable || !strings.Contains(body, apiRetryNoKey) { |
| 252 | t.Fatalf("no key = %d %s, want 503", code, body) |
| 253 | } |
| 254 | noRetryWritten(t, fx, "kass") |
| 255 | if n := serial(t, fx); n != 0 { |
| 256 | t.Fatalf("serial = %d, want no announcement", n) |
| 257 | } |
| 258 | } |
| 259 | |
| 260 | // writeRetryReport publishes host held by v2 with the given retry fields, on |
| 261 | // an armed box or, with code set, a declined one honouring its book. |
| 262 | func writeRetryReport(t *testing.T, fx *consoleFixture, host, code string, retryV int, epoch uint64, book string) { |
| 263 | t.Helper() |
| 264 | hr := status.HostRollout{Account: fx.sqid, App: "kass", Installed: "v2", Good: "v1", HeldBy: "v2", Reason: status.ReasonHealth, |
| 265 | HeldEpoch: epoch, HeldBook: book, RetryV: retryV} |
| 266 | ro := &status.RolloutReport{Hosts: map[string]status.HostRollout{host: hr}} |
| 267 | if code != "" { |
| 268 | ro.Unavailable, ro.FailureBook = code, "honoured" |
| 269 | hr.Attempt, hr.AttemptBuild, hr.Reason = status.AttemptDown, "v2", "" |
| 270 | ro.Hosts[host] = hr |
| 271 | } |
| 272 | box := status.CollectBox(nil, status.Probes{MachineID: "box1"}, time.Now()) |
| 273 | box.Metrics = &status.BoxMetrics{RouteVersions: map[string]string{host: "v2"}, Rollout: ro} |
| 274 | if err := status.WriteBox(context.Background(), fx.rel, box); err != nil { |
| 275 | t.Fatal(err) |
| 276 | } |
| 277 | } |
| 278 | |
| 279 | // Readiness round 2, Task 3 (plan 9's transitions): the door asks the |
| 280 | // box's report before it writes a retry. A box with no retry_v (built |
| 281 | // before the host-retry log) or no held_book cannot take one: 409, and |
| 282 | // nothing is written or announced. A box whose failure book is not saved |
| 283 | // publishes epoch 0, and no grant for it could clear anything: 409, try |
| 284 | // again. A box with retry_v, an epoch and a book gets the signed grant in |
| 285 | // host-retry.json, and nothing in restart-scope.json. Armed and declined |
| 286 | // boxes alike. |
| 287 | func TestAPIInstanceRetryAsksTheBoxsReportFirst(t *testing.T) { |
| 288 | const host = "web.kass.carlosapp.dev" |
| 289 | for _, box := range []string{"", "no-cgroup-delegation"} { |
| 290 | for _, tc := range []struct { |
| 291 | name string |
| 292 | retryV int |
| 293 | epoch uint64 |
| 294 | book string |
| 295 | want string // "" is the grant written |
| 296 | }{ |
| 297 | {"an older box", 0, heldEpoch, heldBook, apiRetryUnsupported}, |
| 298 | {"an older box with no epoch", 0, 0, heldBook, apiRetryUnsupported}, |
| 299 | {"no book", status.HostRetryV, heldEpoch, "", apiRetryUnsupported}, |
| 300 | {"book not saved", status.HostRetryV, 0, heldBook, apiRetryUnsaved}, |
| 301 | {"a box that takes it", status.HostRetryV, heldEpoch, heldBook, ""}, |
| 302 | } { |
| 303 | t.Run(box+" "+tc.name, func(t *testing.T) { |
| 304 | fx := newConsoleFixture(t) |
| 305 | sibling(t, fx, "kass", host, "stable") |
| 306 | writeRetryReport(t, fx, host, box, tc.retryV, tc.epoch, tc.book) |
| 307 | token := cliToken(t, fx, "paul@keymail.dev", "code-host-retry-v") |
| 308 | u := fx.ts.URL + "/api/cli/apps/" + fx.sqid + "/kass/instances/" + host + "/retry" |
| 309 | code, _, body := rawReq(t, http.MethodPost, u, token, `{}`) |
| 310 | if tc.want != "" { |
| 311 | if code != http.StatusConflict || !strings.Contains(body, tc.want) { |
| 312 | t.Fatalf("retry = %d %s, want 409 %q", code, body, tc.want) |
| 313 | } |
| 314 | noRetryWritten(t, fx, "kass") |
| 315 | if n := serial(t, fx); n != 0 { |
| 316 | t.Fatalf("serial = %d, want no announcement", n) |
| 317 | } |
| 318 | return |
| 319 | } |
| 320 | if code != http.StatusOK { |
| 321 | t.Fatalf("retry = %d %s", code, body) |
| 322 | } |
| 323 | r := lastScoped(t, fx, "kass") |
| 324 | g, err := release.VerifyHostRetry(fx.pub, r.Scope.Signed, fx.sqid, "kass", host, time.Now()) |
| 325 | if r.Scope.Kind != release.RestartScopeRetry || r.Scope.Value != host || err != nil || g.Build != "v2" || g.Epoch != heldEpoch || g.Book != heldBook { |
| 326 | t.Fatalf("stored %+v, grant %+v, %v", r, g, err) |
| 327 | } |
| 328 | if n := serial(t, fx); n != 1 { |
| 329 | t.Fatalf("serial = %d", n) |
| 330 | } |
| 331 | }) |
| 332 | } |
| 333 | } |
| 334 | } |
| 335 | |
amadan