| 1 | // End-to-end acceptance test for issue #66: `carlos instances |
| 2 | // enable/create/list/delete` working through the console API with `carlos |
| 3 | // auth login` alone — the last step between "promoted" and "reachable" |
| 4 | // without a box shell or bucket credentials. |
| 5 | // |
| 6 | // Follows TestE2ECLIThroughConsole's shape (auth_e2e_test.go): a REAL |
| 7 | // console.Server over a DirStore, a device-flow login landing a real bearer |
| 8 | // token in a temp credential file, then — with NO store env set — the real |
| 9 | // run* functions driven in-process. Ground truth is the DirStore's own |
| 10 | // manifest and record objects, plus the `stdout` seam's confirmation lines |
| 11 | // (these verbs print through cli.go's stdout var, so captureStdout reaches |
| 12 | // them). |
| 13 | package main |
| 14 | |
| 15 | import ( |
| 16 | "context" |
| 17 | "fmt" |
| 18 | "net/http" |
| 19 | "net/http/httptest" |
| 20 | "path/filepath" |
| 21 | "strings" |
| 22 | "testing" |
| 23 | "time" |
| 24 | |
| 25 | "github.com/carlosframework/platform/internal/appconfig" |
| 26 | "github.com/carlosframework/platform/internal/console" |
| 27 | "github.com/carlosframework/platform/internal/provision" |
| 28 | "github.com/carlosframework/platform/internal/release" |
| 29 | "github.com/carlosframework/platform/internal/status" |
| 30 | ) |
| 31 | |
| 32 | func TestE2EInstancesThroughConsole(t *testing.T) { |
| 33 | useTempCredentials(t) |
| 34 | clearDeploymentEnv(t) |
| 35 | checkNoRegistry := requireNoRegistryTouch(t) |
| 36 | |
| 37 | tmp := t.TempDir() |
| 38 | relStore := release.NewDirStore(filepath.Join(tmp, "store")) |
| 39 | seed, pubB64, err := release.GenerateKey() |
| 40 | if err != nil { |
| 41 | t.Fatal(err) |
| 42 | } |
| 43 | priv, err := release.ParseSigningKey(seed, "e2e") |
| 44 | if err != nil { |
| 45 | t.Fatal(err) |
| 46 | } |
| 47 | pub, err := release.ParsePublicKey(pubB64, "e2e") |
| 48 | if err != nil { |
| 49 | t.Fatal(err) |
| 50 | } |
| 51 | cst, err := console.Open(filepath.Join(tmp, "console.db")) |
| 52 | if err != nil { |
| 53 | t.Fatal(err) |
| 54 | } |
| 55 | t.Cleanup(func() { cst.Close() }) |
| 56 | // The -addr / set-upstreams leg below states upstreams, which only a |
| 57 | // platform operator may do. |
| 58 | if err := cst.SetOperator("paul@keymail.dev", true); err != nil { |
| 59 | t.Fatal(err) |
| 60 | } |
| 61 | |
| 62 | var srv *console.Server |
| 63 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 64 | srv.ServeHTTP(w, r) |
| 65 | })) |
| 66 | t.Cleanup(ts.Close) |
| 67 | srv, err = console.NewServer(console.Config{ |
| 68 | Origin: ts.URL, |
| 69 | Keymail: "http://keymail.invalid", // never dialled — see e2eDeviceLogin |
| 70 | Registry: filepath.Join(tmp, "registry.db"), |
| 71 | Releases: relStore, |
| 72 | ReleasePub: pub, |
| 73 | Key: priv, |
| 74 | AppScheme: "http", |
| 75 | AppsDomain: "oncarlos.test", |
| 76 | }, cst) |
| 77 | if err != nil { |
| 78 | t.Fatal(err) |
| 79 | } |
| 80 | |
| 81 | const addr = "paul@keymail.dev" |
| 82 | if err := cst.AddMember(console.OpsAccountID, addr, console.RoleOwner, time.Now()); err != nil { |
| 83 | t.Fatal(err) |
| 84 | } |
| 85 | if _, err := cst.CreateAppIn(console.OpsAccountID, "titogo", addr, time.Now()); err != nil { |
| 86 | t.Fatal(err) |
| 87 | } |
| 88 | |
| 89 | token, expiresAt := e2eDeviceLogin(t, ts.URL, cst, addr, "e2e@laptop") |
| 90 | credFile, err := loadCredentials() |
| 91 | if err != nil { |
| 92 | t.Fatal(err) |
| 93 | } |
| 94 | credFile.put(credential{Console: ts.URL, Address: addr, Token: token, ExpiresAt: expiresAt}) |
| 95 | if err := credFile.save(); err != nil { |
| 96 | t.Fatal(err) |
| 97 | } |
| 98 | |
| 99 | // --- enable: the console pins the domain to the account's platform |
| 100 | // subdomain; no -domain crosses the wire --------------------------------- |
| 101 | const domain = "bab.oncarlos.test" |
| 102 | buf := captureStdout(t) |
| 103 | runInstancesEnable([]string{"-app", "titogo", "-account", "ops", "-channel", "beta"}) |
| 104 | if want := "enabled instances for titogo under " + domain; !strings.Contains(buf.String(), want) { |
| 105 | t.Fatalf("enable stdout = %q, want %q", buf.String(), want) |
| 106 | } |
| 107 | m, err := provision.GetManifest(context.Background(), relStore, release.OpsAccount, "titogo") |
| 108 | if err != nil { |
| 109 | t.Fatalf("manifest after enable: %v", err) |
| 110 | } |
| 111 | if m.Domain != domain || m.Channel != "beta" || m.Pool != "" { |
| 112 | t.Fatalf("manifest = %+v, want %s on beta with the shared pool", m, domain) |
| 113 | } |
| 114 | |
| 115 | // --- create: the app's own platform host, one label under the pin ------ |
| 116 | host := "titogo." + domain |
| 117 | buf.Reset() |
| 118 | runInstancesCreate([]string{"-app", "titogo", "-account", "ops", "-host", host}) |
| 119 | if want := "created instance " + host + " (channel beta)"; !strings.Contains(buf.String(), want) { |
| 120 | t.Fatalf("create stdout = %q, want %q", buf.String(), want) |
| 121 | } |
| 122 | rec, err := provision.GetRecord(context.Background(), relStore, release.OpsAccount, "titogo", host) |
| 123 | if err != nil { |
| 124 | t.Fatalf("record after create: %v", err) |
| 125 | } |
| 126 | if rec.Channel != "beta" || rec.Slug != "titogo" || rec.CreatedBy != addr { |
| 127 | t.Fatalf("record = %+v, want beta/titogo created by %s", rec, addr) |
| 128 | } |
| 129 | |
| 130 | // --- create -kind static: the flag reaches the record ------------------ |
| 131 | // The shape `carlos deploy` mints for a static app's platform host. Here |
| 132 | // only to prove the FLAG is wired to the wire field the door has always |
| 133 | // read; the door's own acceptance of it is pinned console-side. |
| 134 | staticHost := "site." + domain |
| 135 | buf.Reset() |
| 136 | runInstancesCreate([]string{"-app", "titogo", "-account", "ops", "-host", staticHost, "-kind", "static"}) |
| 137 | staticRec, err := provision.GetRecord(context.Background(), relStore, release.OpsAccount, "titogo", staticHost) |
| 138 | if err != nil { |
| 139 | t.Fatalf("record after create --kind static: %v", err) |
| 140 | } |
| 141 | if staticRec.Kind != "static" { |
| 142 | t.Fatalf("record = %+v, want kind static", staticRec) |
| 143 | } |
| 144 | |
| 145 | // --- set-channel: the record is where a route's channel lives ---------- |
| 146 | buf.Reset() |
| 147 | runInstancesSetChannel([]string{"-app", "titogo", "-account", "ops", "-host", host, "-channel", "edge"}) |
| 148 | if want := "moved " + host + " to edge"; !strings.Contains(buf.String(), want) { |
| 149 | t.Fatalf("set-channel stdout = %q, want %q", buf.String(), want) |
| 150 | } |
| 151 | moved, err := provision.GetRecord(context.Background(), relStore, release.OpsAccount, "titogo", host) |
| 152 | if err != nil { |
| 153 | t.Fatalf("record after set-channel: %v", err) |
| 154 | } |
| 155 | if moved.Channel != "edge" { |
| 156 | t.Fatalf("record channel = %q, want edge", moved.Channel) |
| 157 | } |
| 158 | // And back, so the rest of this test still sees the channel it set up. |
| 159 | buf.Reset() |
| 160 | runInstancesSetChannel([]string{"-app", "titogo", "-account", "ops", "-host", host, "-channel", "beta"}) |
| 161 | |
| 162 | // --- list ---------------------------------------------------------------- |
| 163 | buf.Reset() |
| 164 | runInstancesList([]string{"-app", "titogo", "-account", "ops"}) |
| 165 | if out := buf.String(); !strings.Contains(out, host) || !strings.Contains(out, "beta") { |
| 166 | t.Fatalf("list stdout = %q, want the record's host and channel", out) |
| 167 | } |
| 168 | |
| 169 | // --- delete: idempotent record retirement ------------------------------- |
| 170 | buf.Reset() |
| 171 | runInstancesDelete([]string{"-app", "titogo", "-account", "ops", "-host", host}) |
| 172 | if want := "deleted instance " + host; !strings.Contains(buf.String(), want) { |
| 173 | t.Fatalf("delete stdout = %q, want %q", buf.String(), want) |
| 174 | } |
| 175 | if _, err := provision.GetRecord(context.Background(), relStore, release.OpsAccount, "titogo", host); err == nil { |
| 176 | t.Fatalf("record survived delete") |
| 177 | } |
| 178 | |
| 179 | // --- create with -addr/-health-*, then set-upstreams (Task 8) ---------- |
| 180 | haHost := "ha." + domain |
| 181 | buf.Reset() |
| 182 | runInstancesCreate([]string{"-app", "titogo", "-account", "ops", "-host", haHost, |
| 183 | "-addr", "10.0.0.1:8090,10.0.0.2:8090", "-health-path", "/healthz", "-health-status", "204"}) |
| 184 | if want := "created instance " + haHost; !strings.Contains(buf.String(), want) { |
| 185 | t.Fatalf("create --addr stdout = %q, want %q", buf.String(), want) |
| 186 | } |
| 187 | haRec, err := provision.GetRecord(context.Background(), relStore, release.OpsAccount, "titogo", haHost) |
| 188 | if err != nil { |
| 189 | t.Fatalf("record after create --addr: %v", err) |
| 190 | } |
| 191 | if len(haRec.Addrs) != 2 || haRec.Addrs[0] != "10.0.0.1:8090" || haRec.Addrs[1] != "10.0.0.2:8090" { |
| 192 | t.Fatalf("record.Addrs = %v, want the two --addr entries", haRec.Addrs) |
| 193 | } |
| 194 | if haRec.Health == nil || haRec.Health.Path != "/healthz" || haRec.Health.Status != 204 { |
| 195 | t.Fatalf("record.Health = %+v, want {Path: /healthz, Status: 204}", haRec.Health) |
| 196 | } |
| 197 | |
| 198 | // list must show the UPSTREAMS column, "-" here since no box has |
| 199 | // reported this fleet-fresh host through the status join yet. |
| 200 | buf.Reset() |
| 201 | runInstancesList([]string{"-app", "titogo", "-account", "ops"}) |
| 202 | listOut := buf.String() |
| 203 | if !strings.Contains(listOut, "UPSTREAMS") { |
| 204 | t.Fatalf("list stdout = %q, want an UPSTREAMS column header", listOut) |
| 205 | } |
| 206 | if !strings.Contains(listOut, haHost) { |
| 207 | t.Fatalf("list stdout = %q, want the multi-upstream host listed", listOut) |
| 208 | } |
| 209 | |
| 210 | // set-upstreams repoints the whole set and probe in one PATCH. |
| 211 | buf.Reset() |
| 212 | runInstancesSetUpstreams([]string{"-app", "titogo", "-account", "ops", "-host", haHost, |
| 213 | "-addrs", "10.0.0.3:8090", "-health-path", "/up", "-health-status", "200"}) |
| 214 | if want := "updated upstreams for " + haHost; !strings.Contains(buf.String(), want) { |
| 215 | t.Fatalf("set-upstreams stdout = %q, want %q", buf.String(), want) |
| 216 | } |
| 217 | haRec, err = provision.GetRecord(context.Background(), relStore, release.OpsAccount, "titogo", haHost) |
| 218 | if err != nil { |
| 219 | t.Fatalf("record after set-upstreams: %v", err) |
| 220 | } |
| 221 | if len(haRec.Addrs) != 1 || haRec.Addrs[0] != "10.0.0.3:8090" { |
| 222 | t.Fatalf("record.Addrs after set-upstreams = %v, want [10.0.0.3:8090]", haRec.Addrs) |
| 223 | } |
| 224 | if haRec.Health == nil || haRec.Health.Path != "/up" || haRec.Health.Status != 200 { |
| 225 | t.Fatalf("record.Health after set-upstreams = %+v, want {Path: /up, Status: 200}", haRec.Health) |
| 226 | } |
| 227 | |
| 228 | // set-upstreams naming NEITHER -addrs nor a -health-* flag reaches the |
| 229 | // console's own refusal, surfaced verbatim rather than a client-side |
| 230 | // message that would paraphrase it (self-review rule this task carries). |
| 231 | // That path is log.Fatal, same as every other malformed-invocation path |
| 232 | // in this file (instances_cli_test.go's own doc comment: no argv test |
| 233 | // harness exists here for os.Exit paths) — pinned instead at the client |
| 234 | // layer, where the request/response round trip is actually observable: |
| 235 | // TestInstanceSetUpstreamsSurfacesRefusalVerbatim (apimode_test.go) |
| 236 | // proves instanceSetUpstreams returns the door's error text unmodified. |
| 237 | |
| 238 | // --- steering (Task 3): a top-level verb, not an "instances" sub-verb -- |
| 239 | // haHost's record lives under this manifest's shared pool (Pool == "" |
| 240 | // per the assertion at line 103-105 above), so it may opt into latency |
| 241 | // steering and back off. |
| 242 | buf.Reset() |
| 243 | runSteering([]string{"-app", "titogo", "-account", "ops", "-host", haHost, "latency"}) |
| 244 | if want := "steering set to latency for " + haHost; !strings.Contains(buf.String(), want) { |
| 245 | t.Fatalf("steering stdout = %q, want %q", buf.String(), want) |
| 246 | } |
| 247 | haRec, err = provision.GetRecord(context.Background(), relStore, release.OpsAccount, "titogo", haHost) |
| 248 | if err != nil { |
| 249 | t.Fatalf("record after steering latency: %v", err) |
| 250 | } |
| 251 | if haRec.Steering != "latency" { |
| 252 | t.Fatalf("record.Steering after steering latency = %q, want latency", haRec.Steering) |
| 253 | } |
| 254 | // Everything set-upstreams already wrote onto this record is undisturbed |
| 255 | // by a steering-only patch. |
| 256 | if len(haRec.Addrs) != 1 || haRec.Addrs[0] != "10.0.0.3:8090" { |
| 257 | t.Fatalf("steering patch moved addrs: %v", haRec.Addrs) |
| 258 | } |
| 259 | |
| 260 | buf.Reset() |
| 261 | runSteering([]string{"-app", "titogo", "-account", "ops", "-host", haHost, "off"}) |
| 262 | if want := "steering turned off for " + haHost; !strings.Contains(buf.String(), want) { |
| 263 | t.Fatalf("steering off stdout = %q, want %q", buf.String(), want) |
| 264 | } |
| 265 | haRec, err = provision.GetRecord(context.Background(), relStore, release.OpsAccount, "titogo", haHost) |
| 266 | if err != nil { |
| 267 | t.Fatalf("record after steering off: %v", err) |
| 268 | } |
| 269 | if haRec.Steering != "" { |
| 270 | t.Fatalf("record.Steering after steering off = %q, want empty", haRec.Steering) |
| 271 | } |
| 272 | |
| 273 | // The pool-conflict 409 (a record whose Pool != "") is a log.Fatal path |
| 274 | // through this verb, same as set-upstreams' own refusal above — no argv |
| 275 | // test harness exists here for os.Exit paths. It is proven at the console |
| 276 | // door (TestAPIInstanceUpdateSteeringPoolConflict, internal/console) and |
| 277 | // at this client's own layer (TestInstanceSetSteeringSurfacesRefusalVerbatim, |
| 278 | // apimode_test.go), which is where the request/response round trip is |
| 279 | // actually observable. |
| 280 | |
| 281 | // --- apps set (zero-downtime plan 6, Task 5), in the spec's order -------- |
| 282 | buf.Reset() |
| 283 | runAppsSet([]string{"titogo", "--swap", "hold", "-account", "ops"}) |
| 284 | if want := fmt.Sprintf(appsSetDone, "titogo", "hold", 1); !strings.Contains(buf.String(), want) { |
| 285 | t.Fatalf("apps set stdout = %q, want %q", buf.String(), want) |
| 286 | } |
| 287 | policy, err := appconfig.GetSwapPolicy(context.Background(), relStore, release.OpsAccount, "titogo") |
| 288 | if err != nil || policy.Swap != appconfig.SwapHold { |
| 289 | t.Fatalf("stored policy = %+v, %v", policy, err) |
| 290 | } |
| 291 | |
| 292 | // --- instances retry (Task 11): host first, as the spec writes it -------- |
| 293 | box := status.CollectBox(nil, status.Probes{MachineID: "e2e-box"}, time.Now()) |
| 294 | box.Metrics = &status.BoxMetrics{Rollout: &status.RolloutReport{Hosts: map[string]status.HostRollout{ |
| 295 | haHost: {Account: release.OpsAccount, App: "titogo", Installed: "v2", Serving: "v1", Good: "v1", |
| 296 | Awake: true, HeldBy: "v2", HeldEpoch: 9, HeldBook: "e2e-book", Reason: status.ReasonHealth, |
| 297 | RetryV: status.HostRetryV}, // the retry protocol this box takes (readiness round 2, Task 3) |
| 298 | }}} |
| 299 | if err := status.WriteBox(context.Background(), relStore, box); err != nil { |
| 300 | t.Fatal(err) |
| 301 | } |
| 302 | buf.Reset() |
| 303 | runInstancesRetry([]string{haHost, "-app", "titogo", "-account", "ops"}) |
| 304 | if want := fmt.Sprintf(instancesRetryDone, haHost, "v2"); !strings.Contains(buf.String(), want) { |
| 305 | t.Fatalf("instances retry stdout = %q, want %q", buf.String(), want) |
| 306 | } |
| 307 | // Grants ride the app's host-retry log, not restart-scope.json (Task 3). |
| 308 | scopes, found, err := release.LoadHostRetries(context.Background(), relStore, release.OpsAccount, "titogo") |
| 309 | if err != nil || !found || len(scopes.Requests) == 0 { |
| 310 | t.Fatalf("host retries = %+v, %v, %v", scopes, found, err) |
| 311 | } |
| 312 | last := scopes.Requests[len(scopes.Requests)-1] |
| 313 | if last.Scope.Kind != release.RestartScopeRetry || last.Scope.Value != haHost { |
| 314 | t.Fatalf("last request = %+v", last) |
| 315 | } |
| 316 | // The entry carries the console's root-signed grant for the held build |
| 317 | // at the epoch and book the box reported (Task 8, spec §11; final review |
| 318 | // round 1, finding 3): the box clears nothing on an entry without one. |
| 319 | if g, err := release.VerifyHostRetry(pub, last.Scope.Signed, release.OpsAccount, "titogo", haHost, time.Now()); err != nil || g.Build != "v2" || g.Epoch != 9 || g.Book != "e2e-book" { |
| 320 | t.Fatalf("signed grant = %+v, %v; want v2 at epoch 9 of book e2e-book", g, err) |
| 321 | } |
| 322 | |
| 323 | // The whole flow ran through the console: nothing here may have touched |
| 324 | // a local registry. |
| 325 | checkNoRegistry() |
| 326 | } |
| 327 | |
amadan