| 1 | package console |
| 2 | |
| 3 | // The instance API (apiinstances.go): the console's provisioning verbs and |
| 4 | // the app-held token that may speak to them. The properties that matter, in |
| 5 | // the order they'd hurt if lost: every refusal before the principal is |
| 6 | // proven is the family's byte-identical 404 (a probe learns nothing — not |
| 7 | // whether an account exists, not whether an app does, not whether an app |
| 8 | // token's binding is close to right), an app token is contained to its own |
| 9 | // account+app and can mint nothing, no host reaches a bucket key without |
| 10 | // passing the label+suffix check, and the rate limiter is consulted only |
| 11 | // AFTER the gate so a 429 can never confirm what a 404 hides. |
| 12 | |
| 13 | import ( |
| 14 | "bytes" |
| 15 | "context" |
| 16 | "encoding/json" |
| 17 | "errors" |
| 18 | "fmt" |
| 19 | "io" |
| 20 | "maps" |
| 21 | "net/http" |
| 22 | "net/http/httptest" |
| 23 | "net/url" |
| 24 | "os" |
| 25 | "slices" |
| 26 | "sort" |
| 27 | "strings" |
| 28 | "testing" |
| 29 | "time" |
| 30 | |
| 31 | "github.com/carlosframework/platform/internal/activator" |
| 32 | "github.com/carlosframework/platform/internal/metrics" |
| 33 | "github.com/carlosframework/platform/internal/provision" |
| 34 | "github.com/carlosframework/platform/internal/registry" |
| 35 | "github.com/carlosframework/platform/internal/release" |
| 36 | "github.com/carlosframework/platform/internal/status" |
| 37 | ) |
| 38 | |
| 39 | // rawReq is the tests' one-shot request: any method, an optional bearer and |
| 40 | // an optional JSON body, and the response handed back as status + headers + |
| 41 | // bytes so a refusal can be compared to http.NotFound's own output field by |
| 42 | // field rather than merely "looks like a 404". |
| 43 | func rawReq(t *testing.T, method, u, bearer, body string) (int, http.Header, string) { |
| 44 | t.Helper() |
| 45 | var rdr io.Reader |
| 46 | if body != "" { |
| 47 | rdr = strings.NewReader(body) |
| 48 | } |
| 49 | req, err := http.NewRequest(method, u, rdr) |
| 50 | if err != nil { |
| 51 | t.Fatal(err) |
| 52 | } |
| 53 | if body != "" { |
| 54 | req.Header.Set("Content-Type", "application/json") |
| 55 | } |
| 56 | if bearer != "" { |
| 57 | req.Header.Set("Authorization", "Bearer "+bearer) |
| 58 | } |
| 59 | res, err := http.DefaultClient.Do(req) |
| 60 | if err != nil { |
| 61 | t.Fatal(err) |
| 62 | } |
| 63 | defer res.Body.Close() |
| 64 | out, err := io.ReadAll(res.Body) |
| 65 | if err != nil { |
| 66 | t.Fatal(err) |
| 67 | } |
| 68 | return res.StatusCode, res.Header, string(out) |
| 69 | } |
| 70 | |
| 71 | // notFound404 is one refusal reduced to the parts a prober could read: the |
| 72 | // status, the two headers http.Error sets, and the body. |
| 73 | type notFound404 struct { |
| 74 | Status int |
| 75 | Type string |
| 76 | NoSniff string |
| 77 | Body string |
| 78 | } |
| 79 | |
| 80 | // canonicalNotFound captures what the STDLIB writes, rather than typing the |
| 81 | // expected bytes out here: the property under test is "identical to |
| 82 | // http.NotFound", so http.NotFound is what the tests must compare against. |
| 83 | func canonicalNotFound(t *testing.T) notFound404 { |
| 84 | t.Helper() |
| 85 | rec := httptest.NewRecorder() |
| 86 | http.NotFound(rec, httptest.NewRequest(http.MethodGet, "/", nil)) |
| 87 | return notFound404{rec.Code, rec.Header().Get("Content-Type"), |
| 88 | rec.Header().Get("X-Content-Type-Options"), rec.Body.String()} |
| 89 | } |
| 90 | |
| 91 | func assertNotFound(t *testing.T, what string, want notFound404, status int, h http.Header, body string) { |
| 92 | t.Helper() |
| 93 | got := notFound404{status, h.Get("Content-Type"), h.Get("X-Content-Type-Options"), body} |
| 94 | if got != want { |
| 95 | t.Errorf("%s\n got: %+v\nwant: the byte-identical %+v", what, got, want) |
| 96 | } |
| 97 | } |
| 98 | |
| 99 | // instanceFixture is a console with titogo claimed in the ops account and a |
| 100 | // MEMBER (not owner) token for it — member-level is exactly the gate these |
| 101 | // verbs grant, the same one promote and the ledger API use. |
| 102 | func instanceFixture(t *testing.T, code string) (*consoleFixture, string) { |
| 103 | t.Helper() |
| 104 | fx := newConsoleFixture(t) |
| 105 | now := time.Now() |
| 106 | if _, err := fx.st.CreateAppIn(OpsAccountID, "titogo", "paul@keymail.dev", now); err != nil { |
| 107 | t.Fatal(err) |
| 108 | } |
| 109 | if err := fx.st.AddMember(OpsAccountID, "alice@keymail.dev", RoleMember, now); err != nil { |
| 110 | t.Fatal(err) |
| 111 | } |
| 112 | return fx, cliToken(t, fx, "alice@keymail.dev", code) |
| 113 | } |
| 114 | |
| 115 | // enableInstances writes the app's manifest — the operator act (`carlos |
| 116 | // instances enable`) whose ABSENCE is what makes the feature not exist. |
| 117 | func enableInstances(t *testing.T, fx *consoleFixture, app string, m provision.Manifest) { |
| 118 | t.Helper() |
| 119 | if err := provision.PutManifest(context.Background(), fx.rel, release.OpsAccount, app, m); err != nil { |
| 120 | t.Fatal(err) |
| 121 | } |
| 122 | } |
| 123 | |
| 124 | func titoManifest() provision.Manifest { |
| 125 | return provision.Manifest{Schema: 1, App: "titogo", Domain: "go.tito.io", |
| 126 | Pool: "tito", Channel: "stable", Regions: []string{"ie"}, Seed: []string{"seed"}} |
| 127 | } |
| 128 | |
| 129 | func instancesURL(fx *consoleFixture, acct, app string) string { |
| 130 | return fx.ts.URL + "/api/cli/apps/" + acct + "/" + app + "/instances" |
| 131 | } |
| 132 | |
| 133 | // TestAPIInstanceRefusalMatrix: every way of not being allowed here — a |
| 134 | // stranger's token, an app token pointed at a neighbouring account or a |
| 135 | // neighbouring app, an account that does not exist, an app that was never |
| 136 | // claimed — answers with the SAME bytes http.NotFound writes. Asserted |
| 137 | // against the stdlib's own output, on every verb. |
| 138 | func TestAPIInstanceRefusalMatrix(t *testing.T) { |
| 139 | fx, member := instanceFixture(t, "code-inst-refusal") |
| 140 | enableInstances(t, fx, "titogo", titoManifest()) |
| 141 | want := canonicalNotFound(t) |
| 142 | |
| 143 | // A second account, so the app token's cross-account probe names one |
| 144 | // that genuinely exists: the refusal must not depend on that. |
| 145 | now := time.Now() |
| 146 | if err := fx.st.CreateAccount(42, "cdef", "widgets", "bob@keymail.dev", now); err != nil { |
| 147 | t.Fatal(err) |
| 148 | } |
| 149 | if _, err := fx.st.CreateAppIn(42, "titogo", "bob@keymail.dev", now); err != nil { |
| 150 | t.Fatal(err) |
| 151 | } |
| 152 | // An app token bound to (bab, titogo) — the honest binding. |
| 153 | appTok, err := fx.st.NewAppToken(OpsAccountID, "titogo", "alice@keymail.dev", "signup", now) |
| 154 | if err != nil { |
| 155 | t.Fatal(err) |
| 156 | } |
| 157 | // A member of nothing at all. |
| 158 | eve := cliToken(t, fx, "eve@keymail.dev", "code-inst-eve") |
| 159 | |
| 160 | body := `{"host":"acme.go.tito.io","region":"ie"}` |
| 161 | cases := []struct { |
| 162 | what string |
| 163 | method string |
| 164 | url string |
| 165 | token string |
| 166 | body string |
| 167 | }{ |
| 168 | {"non-member list", http.MethodGet, instancesURL(fx, "bab", "titogo"), eve, ""}, |
| 169 | {"non-member create", http.MethodPost, instancesURL(fx, "bab", "titogo"), eve, body}, |
| 170 | {"non-member get", http.MethodGet, instancesURL(fx, "bab", "titogo") + "/acme.go.tito.io", eve, ""}, |
| 171 | {"non-member delete", http.MethodDelete, instancesURL(fx, "bab", "titogo") + "/acme.go.tito.io", eve, ""}, |
| 172 | {"app token, foreign account", http.MethodGet, instancesURL(fx, "cdef", "titogo"), appTok, ""}, |
| 173 | {"app token, foreign account create", http.MethodPost, instancesURL(fx, "cdef", "titogo"), appTok, body}, |
| 174 | {"app token, foreign app", http.MethodGet, instancesURL(fx, "bab", "other"), appTok, ""}, |
| 175 | {"app token, foreign app create", http.MethodPost, instancesURL(fx, "bab", "other"), appTok, body}, |
| 176 | // The account segment resolves by NAME for a user token; an app |
| 177 | // token holds no memberships to resolve one against, so its own |
| 178 | // account's name is not a way in either. |
| 179 | {"app token, own account by name", http.MethodGet, instancesURL(fx, "ops", "titogo"), appTok, ""}, |
| 180 | {"unknown account", http.MethodGet, instancesURL(fx, "zzzz", "titogo"), member, ""}, |
| 181 | {"unclaimed app", http.MethodGet, instancesURL(fx, "bab", "ghost-app"), member, ""}, |
| 182 | {"unclaimed app create", http.MethodPost, instancesURL(fx, "bab", "ghost-app"), member, body}, |
| 183 | } |
| 184 | for _, c := range cases { |
| 185 | status, h, out := rawReq(t, c.method, c.url, c.token, c.body) |
| 186 | assertNotFound(t, c.what, want, status, h, out) |
| 187 | } |
| 188 | |
| 189 | // None of the refused calls wrote anything. |
| 190 | if _, _, err := fx.rel.Get(context.Background(), |
| 191 | provision.RecordKey("bab", "titogo", "acme.go.tito.io")); err == nil { |
| 192 | t.Fatalf("a record exists after only refused creates") |
| 193 | } |
| 194 | if _, _, err := fx.rel.Get(context.Background(), |
| 195 | provision.RecordKey("cdef", "titogo", "acme.go.tito.io")); err == nil { |
| 196 | t.Fatalf("a record exists in the neighbouring account after only refused creates") |
| 197 | } |
| 198 | } |
| 199 | |
| 200 | // TestAPIInstanceNoToken: no credential at all is the 401 every CLI door |
| 201 | // gives, on all five routes. |
| 202 | func TestAPIInstanceNoToken(t *testing.T) { |
| 203 | fx, _ := instanceFixture(t, "code-inst-notoken") |
| 204 | for _, c := range []struct{ method, url, body string }{ |
| 205 | {http.MethodGet, instancesURL(fx, "bab", "titogo"), ""}, |
| 206 | {http.MethodPost, instancesURL(fx, "bab", "titogo"), `{"host":"a.go.tito.io"}`}, |
| 207 | {http.MethodGet, instancesURL(fx, "bab", "titogo") + "/a.go.tito.io", ""}, |
| 208 | {http.MethodDelete, instancesURL(fx, "bab", "titogo") + "/a.go.tito.io", ""}, |
| 209 | {http.MethodPost, fx.ts.URL + "/api/cli/apps/bab/titogo/instance-token", `{}`}, |
| 210 | } { |
| 211 | if status, _, out := rawReq(t, c.method, c.url, "", c.body); status != http.StatusUnauthorized { |
| 212 | t.Errorf("%s %s with no token = %d %s, want 401", c.method, c.url, status, out) |
| 213 | } |
| 214 | } |
| 215 | } |
| 216 | |
| 217 | // TestAPIInstanceAppTokenContainment: the app token is contained in both |
| 218 | // directions. It cannot speak to the verbs a person's token speaks to |
| 219 | // (promote), it cannot mint another token, and it CAN do exactly the four |
| 220 | // instance verbs on its own app. |
| 221 | func TestAPIInstanceAppTokenContainment(t *testing.T) { |
| 222 | fx, _ := instanceFixture(t, "code-inst-contain") |
| 223 | enableInstances(t, fx, "titogo", titoManifest()) |
| 224 | appTok, err := fx.st.NewAppToken(OpsAccountID, "titogo", "alice@keymail.dev", "signup", time.Now()) |
| 225 | if err != nil { |
| 226 | t.Fatal(err) |
| 227 | } |
| 228 | |
| 229 | // Promote: the containment is unchanged, the ANSWER moved from 401 to |
| 230 | // 403 when the role gate landed (service-principals spec §3b, §8). An |
| 231 | // app token is a service principal holding role `instance`, and promote |
| 232 | // names `publish` — so cliRoute refuses it out at the wrapper, before |
| 233 | // the handler and before the account segment is read. 403 discloses |
| 234 | // nothing the holder does not already know: its own role, and the roles |
| 235 | // this door wants. It is the same constant answer for every account |
| 236 | // segment, which is what keeps it from being a membership oracle |
| 237 | // (TestRoleRefusalIsConstantAcrossAccounts). |
| 238 | if status, _, out := rawReq(t, http.MethodPost, |
| 239 | fx.ts.URL+"/api/cli/apps/bab/titogo/promote", appTok, |
| 240 | `{"version":"1.0.0","channel":"canary/a"}`); status != http.StatusForbidden { |
| 241 | t.Fatalf("app token on promote = %d %s, want 403", status, out) |
| 242 | } |
| 243 | // Minting is a human act: the mint door is registered with the zero |
| 244 | // Set, so no role reaches it and admin is no escape hatch either |
| 245 | // (TestNoRoleReachesAMintFleetOrPolicyDoor). Behind it, apiPerson still |
| 246 | // refuses a non-person outright — the gate is the outer of two walls, |
| 247 | // not a replacement for the inner one. (apiAccountApp used to be that |
| 248 | // inner wall by accident, because it resolved through cliUser; since it |
| 249 | // became principal-aware the refusal is stated rather than inherited.) |
| 250 | if status, _, out := rawReq(t, http.MethodPost, |
| 251 | fx.ts.URL+"/api/cli/apps/bab/titogo/instance-token", appTok, `{}`); status != http.StatusForbidden { |
| 252 | t.Fatalf("app token minting a token = %d %s, want 403", status, out) |
| 253 | } |
| 254 | // Its own app's verbs work. |
| 255 | status, _, out := rawReq(t, http.MethodPost, instancesURL(fx, "bab", "titogo"), appTok, |
| 256 | `{"host":"acme.go.tito.io","region":"ie"}`) |
| 257 | if status != http.StatusCreated { |
| 258 | t.Fatalf("app token create on its own app = %d %s, want 201", status, out) |
| 259 | } |
| 260 | var rec provision.Record |
| 261 | if err := json.Unmarshal([]byte(out), &rec); err != nil { |
| 262 | t.Fatalf("create body %s: %v", out, err) |
| 263 | } |
| 264 | if rec.CreatedBy != "app:bab/titogo" { |
| 265 | t.Fatalf("created_by = %q, want the app principal app:bab/titogo", rec.CreatedBy) |
| 266 | } |
| 267 | if status, _, out := rawReq(t, http.MethodGet, instancesURL(fx, "bab", "titogo"), appTok, ""); status != http.StatusOK { |
| 268 | t.Fatalf("app token list = %d %s, want 200", status, out) |
| 269 | } |
| 270 | if status, _, out := rawReq(t, http.MethodDelete, |
| 271 | instancesURL(fx, "bab", "titogo")+"/acme.go.tito.io", appTok, ""); status != http.StatusOK { |
| 272 | t.Fatalf("app token delete = %d %s, want 200", status, out) |
| 273 | } |
| 274 | } |
| 275 | |
| 276 | // TestAPIInstanceCreateRefusalOrder walks spec §7's order with a member |
| 277 | // token: membership is already proven, so from here on refusals may name |
| 278 | // their cause — and each one must be the RIGHT cause, in the right order. |
| 279 | func TestAPIInstanceCreateRefusalOrder(t *testing.T) { |
| 280 | fx, token := instanceFixture(t, "code-inst-create") |
| 281 | u := instancesURL(fx, "bab", "titogo") |
| 282 | |
| 283 | // No manifest at all: the feature does not exist for this app. |
| 284 | status, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"acme.go.tito.io","region":"ie"}`) |
| 285 | if status != http.StatusConflict { |
| 286 | t.Fatalf("create with no manifest = %d %s, want 409", status, out) |
| 287 | } |
| 288 | if !strings.Contains(out, "carlos instances enable") { |
| 289 | t.Fatalf("no-manifest refusal = %s, want it to name `carlos instances enable`", out) |
| 290 | } |
| 291 | |
| 292 | enableInstances(t, fx, "titogo", titoManifest()) |
| 293 | |
| 294 | for _, c := range []struct{ what, body string }{ |
| 295 | {"bad host shape", `{"host":"-nope-.go.tito.io","region":"ie"}`}, |
| 296 | {"traversal host", `{"host":"../../etc/passwd","region":"ie"}`}, |
| 297 | {"wrong suffix", `{"host":"acme.evil.example","region":"ie"}`}, |
| 298 | {"bare domain, no label", `{"host":"go.tito.io","region":"ie"}`}, |
| 299 | {"region outside the allow-list", `{"host":"acme.go.tito.io","region":"us"}`}, |
| 300 | {"malformed region", `{"host":"acme.go.tito.io","region":"IE/../x"}`}, |
| 301 | {"bad channel", `{"host":"acme.go.tito.io","region":"ie","channel":"../etc"}`}, |
| 302 | } { |
| 303 | status, _, out := rawReq(t, http.MethodPost, u, token, c.body) |
| 304 | if status != http.StatusBadRequest { |
| 305 | t.Errorf("%s = %d %s, want 400", c.what, status, out) |
| 306 | } |
| 307 | } |
| 308 | // Nothing above reached the bucket. |
| 309 | keys, err := fx.rel.List(context.Background(), provision.RecordsPrefix("bab", "titogo")) |
| 310 | if err != nil { |
| 311 | t.Fatal(err) |
| 312 | } |
| 313 | if len(keys) != 0 { |
| 314 | t.Fatalf("refused creates left %v in the bucket", keys) |
| 315 | } |
| 316 | |
| 317 | // The happy path: channel defaults from the manifest, pool is stamped |
| 318 | // from it (never from the request), the record comes back whole. |
| 319 | status, _, out = rawReq(t, http.MethodPost, u, token, |
| 320 | `{"host":"acme.go.tito.io","region":"ie","pool":"attacker"}`) |
| 321 | if status != http.StatusCreated { |
| 322 | t.Fatalf("create = %d %s, want 201", status, out) |
| 323 | } |
| 324 | var rec provision.Record |
| 325 | if err := json.Unmarshal([]byte(out), &rec); err != nil { |
| 326 | t.Fatalf("create body %s: %v", out, err) |
| 327 | } |
| 328 | if rec.Schema != 1 || rec.Host != "acme.go.tito.io" || rec.Slug != "acme" || |
| 329 | rec.App != "titogo" || rec.Account != "bab" || rec.Pool != "tito" || |
| 330 | rec.Region != "ie" || rec.Channel != "stable" { |
| 331 | t.Fatalf("record = %+v", rec) |
| 332 | } |
| 333 | if rec.CreatedBy != "alice@keymail.dev" { |
| 334 | t.Fatalf("created_by = %q, want the member's address", rec.CreatedBy) |
| 335 | } |
| 336 | |
| 337 | // It is in the bucket at exactly the key the reconciler will look for. |
| 338 | raw, _, err := fx.rel.Get(context.Background(), provision.RecordKey("bab", "titogo", "acme.go.tito.io")) |
| 339 | if err != nil { |
| 340 | t.Fatalf("stored record: %v", err) |
| 341 | } |
| 342 | var stored provision.Record |
| 343 | if err := json.Unmarshal(raw, &stored); err != nil { |
| 344 | t.Fatal(err) |
| 345 | } |
| 346 | if stored.CreatedBy != "alice@keymail.dev" || stored.Host != "acme.go.tito.io" { |
| 347 | t.Fatalf("stored record = %+v", stored) |
| 348 | } |
| 349 | if stored.CreatedAt.IsZero() { |
| 350 | t.Fatalf("stored record has no created_at: %+v", stored) |
| 351 | } |
| 352 | |
| 353 | // The same host twice is a lost race, not a silent overwrite. |
| 354 | status, _, out = rawReq(t, http.MethodPost, u, token, `{"host":"acme.go.tito.io","region":"ie"}`) |
| 355 | if status != http.StatusConflict { |
| 356 | t.Fatalf("duplicate create = %d %s, want 409", status, out) |
| 357 | } |
| 358 | if !strings.Contains(out, "already exists") { |
| 359 | t.Fatalf("duplicate refusal = %s, want it to say the instance already exists", out) |
| 360 | } |
| 361 | |
| 362 | // An explicit channel is honoured over the manifest's default. |
| 363 | status, _, out = rawReq(t, http.MethodPost, u, token, |
| 364 | `{"host":"beta.go.tito.io","region":"ie","channel":"canary/beta"}`) |
| 365 | if status != http.StatusCreated { |
| 366 | t.Fatalf("create with channel = %d %s, want 201", status, out) |
| 367 | } |
| 368 | if err := json.Unmarshal([]byte(out), &rec); err != nil { |
| 369 | t.Fatal(err) |
| 370 | } |
| 371 | if rec.Channel != "canary/beta" { |
| 372 | t.Fatalf("channel = %q, want the requested canary/beta", rec.Channel) |
| 373 | } |
| 374 | } |
| 375 | |
| 376 | // TestAPIInstanceCreateInheritsFleetPlacement: a fleet-placed app's |
| 377 | // instances inherit the app's placement (customer-fleets plan, spec's "one |
| 378 | // flag on the app, all its instances follow") — the request body's region |
| 379 | // is ignored entirely, and even a region the manifest's allow-list would |
| 380 | // otherwise refuse succeeds, because it is never consulted once the app is |
| 381 | // placed. |
| 382 | // |
| 383 | // A route must already exist for "titogo" before place is called: |
| 384 | // appPlacement (fleets.go) derives an app's placement from its ALREADY |
| 385 | // STAMPED routes, so an app placed before it has any route/instance at all |
| 386 | // has nothing yet to derive from — a known limitation of the routes-derived |
| 387 | // approach, not exercised by this test (see fleets.go's appPlacement doc |
| 388 | // comment). Seeding one here is the realistic shape: an app already |
| 389 | // running (e.g. via `carlos add`) before being placed onto a fleet. |
| 390 | func TestAPIInstanceCreateInheritsFleetPlacement(t *testing.T) { |
| 391 | fx, token := instanceFixture(t, "code-inst-placed") |
| 392 | enableInstances(t, fx, "titogo", titoManifest()) // Regions: []string{"ie"} |
| 393 | |
| 394 | reg, err := registry.Open(fx.regPath) |
| 395 | if err != nil { |
| 396 | t.Fatal(err) |
| 397 | } |
| 398 | if err := reg.Upsert(registry.Route{Host: "www.go.tito.io", Kind: "instance", App: "titogo", Account: "bab"}); err != nil { |
| 399 | t.Fatal(err) |
| 400 | } |
| 401 | |
| 402 | if status, body := postJSON(t, fx.ts.URL+"/api/cli/fleets/bab/new", |
| 403 | map[string]string{"name": "pi", "bucket": "pi-fleet-data"}, token); status != http.StatusOK { |
| 404 | t.Fatalf("create fleet = %d %v, want 200", status, body) |
| 405 | } |
| 406 | if status, body := postJSON(t, fx.ts.URL+"/api/cli/apps/bab/titogo/place", |
| 407 | map[string]string{"target": "fleet/pi"}, token); status != http.StatusOK { |
| 408 | t.Fatalf("place = %d %v, want 200", status, body) |
| 409 | } |
| 410 | |
| 411 | u := instancesURL(fx, "bab", "titogo") |
| 412 | // "us" is outside titoManifest's ["ie"] allow-list — refused on an |
| 413 | // unplaced app (TestAPIInstanceCreateUnplacedHonorsBodyRegion below), |
| 414 | // but accepted (and overridden) here. |
| 415 | status, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"acme.go.tito.io","region":"us"}`) |
| 416 | if status != http.StatusCreated { |
| 417 | t.Fatalf("create on a placed app = %d %s, want 201", status, out) |
| 418 | } |
| 419 | var rec provision.Record |
| 420 | if err := json.Unmarshal([]byte(out), &rec); err != nil { |
| 421 | t.Fatalf("create body %s: %v", out, err) |
| 422 | } |
| 423 | if rec.Region != "fleet/pi" { |
| 424 | t.Fatalf("record Region = %q, want the inherited fleet/pi (body region ignored)", rec.Region) |
| 425 | } |
| 426 | |
| 427 | // The stored record agrees. |
| 428 | stored, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "acme.go.tito.io") |
| 429 | if err != nil { |
| 430 | t.Fatal(err) |
| 431 | } |
| 432 | if stored.Region != "fleet/pi" { |
| 433 | t.Fatalf("stored record Region = %q, want fleet/pi", stored.Region) |
| 434 | } |
| 435 | } |
| 436 | |
| 437 | // TestAPIInstanceCreateInheritsRouteLessPlacedApp is the fix for the |
| 438 | // limitation TestAPIInstanceCreateInheritsFleetPlacement's doc comment used |
| 439 | // to describe: an app placed at CREATE TIME (`carlos apps create -place |
| 440 | // fleet/<f>`, 2026-08-06 ruling) has ZERO routes at the instant of |
| 441 | // placement — nothing for the old route-derived appPlacement to find — yet |
| 442 | // its first instance must still inherit the target, because the store's |
| 443 | // placement column (not routes) is now the source of truth. Unlike |
| 444 | // TestAPIInstanceCreateInheritsFleetPlacement, no route is ever seeded here. |
| 445 | func TestAPIInstanceCreateInheritsRouteLessPlacedApp(t *testing.T) { |
| 446 | fx := newConsoleFixture(t) |
| 447 | token := cliToken(t, fx, "paul@keymail.dev", "code-inst-routeless-placed") |
| 448 | |
| 449 | if status, body := postJSON(t, fx.ts.URL+"/api/cli/fleets/"+fx.sqid+"/new", |
| 450 | map[string]string{"name": "pi", "bucket": "pi-fleet-data"}, token); status != http.StatusOK { |
| 451 | t.Fatalf("create fleet = %d %v, want 200", status, body) |
| 452 | } |
| 453 | // The app itself is created THROUGH the API, carrying place in the same |
| 454 | // call — the exact CLI surface this test exists for, and the only way an |
| 455 | // app can reach instance-create with a placement and genuinely zero |
| 456 | // routes (a pre-seeded fixture app would not prove the store path). |
| 457 | if status, body := postJSON(t, fx.ts.URL+"/api/cli/apps/"+fx.sqid+"/new", |
| 458 | map[string]string{"name": "titogo", "place": "fleet/pi"}, token); status != http.StatusOK { |
| 459 | t.Fatalf("create app with place = %d %v, want 200", status, body) |
| 460 | } |
| 461 | enableInstances(t, fx, "titogo", titoManifest()) // Regions: []string{"ie"} |
| 462 | |
| 463 | // Sanity: genuinely zero routes for this app — the case route-derivation |
| 464 | // alone cannot answer. |
| 465 | reg, err := registry.Open(fx.regPath) |
| 466 | if err != nil { |
| 467 | t.Fatal(err) |
| 468 | } |
| 469 | routes, err := reg.List() |
| 470 | if err != nil { |
| 471 | t.Fatal(err) |
| 472 | } |
| 473 | for _, rt := range routes { |
| 474 | if rt.Account == fx.sqid && rt.App == "titogo" { |
| 475 | t.Fatalf("titogo already has a route (%s) — this test needs zero to prove the store path", rt.Host) |
| 476 | } |
| 477 | } |
| 478 | |
| 479 | u := instancesURL(fx, fx.sqid, "titogo") |
| 480 | // "us" is outside titoManifest's ["ie"] allow-list — accepted (and |
| 481 | // overridden) here only because the store's placement inherits. |
| 482 | status, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"acme.go.tito.io","region":"us"}`) |
| 483 | if status != http.StatusCreated { |
| 484 | t.Fatalf("create on a route-less placed app = %d %s, want 201", status, out) |
| 485 | } |
| 486 | var rec provision.Record |
| 487 | if err := json.Unmarshal([]byte(out), &rec); err != nil { |
| 488 | t.Fatalf("create body %s: %v", out, err) |
| 489 | } |
| 490 | if rec.Region != "fleet/pi" { |
| 491 | t.Fatalf("record Region = %q, want the inherited fleet/pi (body region ignored)", rec.Region) |
| 492 | } |
| 493 | } |
| 494 | |
| 495 | // TestAPIInstanceCreateUnplacedHonorsBodyRegion is the regression guard for |
| 496 | // the inheritance check above: without a placement, the existing contract |
| 497 | // is unchanged — the body's region is validated against the manifest's |
| 498 | // allow-list and stored exactly as before. |
| 499 | func TestAPIInstanceCreateUnplacedHonorsBodyRegion(t *testing.T) { |
| 500 | fx, token := instanceFixture(t, "code-inst-unplaced") |
| 501 | enableInstances(t, fx, "titogo", titoManifest()) |
| 502 | u := instancesURL(fx, "bab", "titogo") |
| 503 | |
| 504 | status, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"acme.go.tito.io","region":"ie"}`) |
| 505 | if status != http.StatusCreated { |
| 506 | t.Fatalf("create = %d %s, want 201", status, out) |
| 507 | } |
| 508 | var rec provision.Record |
| 509 | if err := json.Unmarshal([]byte(out), &rec); err != nil { |
| 510 | t.Fatal(err) |
| 511 | } |
| 512 | if rec.Region != "ie" { |
| 513 | t.Fatalf("Region = %q, want the body's ie (no placement in play)", rec.Region) |
| 514 | } |
| 515 | |
| 516 | // A region outside the manifest's allow-list is still refused, exactly |
| 517 | // as it was before placement inheritance existed. |
| 518 | status, _, out = rawReq(t, http.MethodPost, u, token, `{"host":"beta.go.tito.io","region":"us"}`) |
| 519 | if status != http.StatusBadRequest { |
| 520 | t.Fatalf("create outside the allow-list on an unplaced app = %d %s, want 400", status, out) |
| 521 | } |
| 522 | } |
| 523 | |
| 524 | // TestAPIInstanceCreateEmptyRegionAllowList: an empty allow-list means any |
| 525 | // shape-valid region, per §3.1 — including the unconstrained empty one. |
| 526 | func TestAPIInstanceCreateEmptyRegionAllowList(t *testing.T) { |
| 527 | fx, token := instanceFixture(t, "code-inst-anyregion") |
| 528 | m := titoManifest() |
| 529 | m.Regions = nil |
| 530 | m.Channel = "" |
| 531 | enableInstances(t, fx, "titogo", m) |
| 532 | u := instancesURL(fx, "bab", "titogo") |
| 533 | |
| 534 | status, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"acme.go.tito.io","region":"us-east"}`) |
| 535 | if status != http.StatusCreated { |
| 536 | t.Fatalf("create with an open allow-list = %d %s, want 201", status, out) |
| 537 | } |
| 538 | var rec provision.Record |
| 539 | if err := json.Unmarshal([]byte(out), &rec); err != nil { |
| 540 | t.Fatal(err) |
| 541 | } |
| 542 | // No manifest channel either, and this app has no pipeline, so the |
| 543 | // record follows the entry channel every app's releases land on. |
| 544 | // apiinstanceschannel_test.go is where that resolution is pinned; here |
| 545 | // it is only asserted so an open allow-list cannot quietly change it. |
| 546 | if rec.Channel != "edge" { |
| 547 | t.Fatalf("channel = %q, want the entry-channel default", rec.Channel) |
| 548 | } |
| 549 | if status, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"beta.go.tito.io"}`); status != http.StatusCreated { |
| 550 | t.Fatalf("create with no region = %d %s, want 201", status, out) |
| 551 | } |
| 552 | } |
| 553 | |
| 554 | // TestAPIInstanceListAndGet: what was created lists and reads back, and a |
| 555 | // host that isn't there is the byte-identical 404 — a record's existence is |
| 556 | // not something to probe for either. |
| 557 | func TestAPIInstanceListAndGet(t *testing.T) { |
| 558 | fx, token := instanceFixture(t, "code-inst-list") |
| 559 | enableInstances(t, fx, "titogo", titoManifest()) |
| 560 | u := instancesURL(fx, "bab", "titogo") |
| 561 | for _, h := range []string{"acme.go.tito.io", "beta.go.tito.io"} { |
| 562 | if status, _, out := rawReq(t, http.MethodPost, u, token, |
| 563 | fmt.Sprintf(`{"host":%q,"region":"ie"}`, h)); status != http.StatusCreated { |
| 564 | t.Fatalf("create %s = %d %s", h, status, out) |
| 565 | } |
| 566 | } |
| 567 | |
| 568 | status, _, out := rawReq(t, http.MethodGet, u, token, "") |
| 569 | if status != http.StatusOK { |
| 570 | t.Fatalf("list = %d %s", status, out) |
| 571 | } |
| 572 | var list struct { |
| 573 | Instances []provision.Record `json:"instances"` |
| 574 | Truncated bool `json:"truncated"` |
| 575 | } |
| 576 | if err := json.Unmarshal([]byte(out), &list); err != nil { |
| 577 | t.Fatalf("list body %s: %v", out, err) |
| 578 | } |
| 579 | if len(list.Instances) != 2 || list.Truncated { |
| 580 | t.Fatalf("list = %+v, want both records and truncated false", list) |
| 581 | } |
| 582 | if list.Instances[0].Host != "acme.go.tito.io" || list.Instances[1].Host != "beta.go.tito.io" { |
| 583 | t.Fatalf("list order = %+v, want host order", list.Instances) |
| 584 | } |
| 585 | |
| 586 | status, _, out = rawReq(t, http.MethodGet, u+"/acme.go.tito.io", token, "") |
| 587 | if status != http.StatusOK { |
| 588 | t.Fatalf("get = %d %s", status, out) |
| 589 | } |
| 590 | var rec provision.Record |
| 591 | if err := json.Unmarshal([]byte(out), &rec); err != nil { |
| 592 | t.Fatal(err) |
| 593 | } |
| 594 | if rec.Host != "acme.go.tito.io" || rec.Slug != "acme" { |
| 595 | t.Fatalf("get = %+v", rec) |
| 596 | } |
| 597 | |
| 598 | want := canonicalNotFound(t) |
| 599 | status, h, out := rawReq(t, http.MethodGet, u+"/missing.go.tito.io", token, "") |
| 600 | assertNotFound(t, "get of a host with no record", want, status, h, out) |
| 601 | // A hostile host never reaches the key builder: same 404, nothing read. |
| 602 | status, h, out = rawReq(t, http.MethodGet, u+"/..%2f..%2fchannels", token, "") |
| 603 | assertNotFound(t, "get of a traversal host", want, status, h, out) |
| 604 | } |
| 605 | |
| 606 | // TestAPIInstanceListStagedVersion (issue #78): the list join. A record |
| 607 | // whose host was named in a FRESH box's private status/boxes/<label>.json |
| 608 | // RouteVersions carries that box's reported version as staged_version; a |
| 609 | // record no box has ever reported gets the empty string; a record whose |
| 610 | // only reporting box has gone STALE (status.BoxStale — the same rule the |
| 611 | // operator fleet page's HostRows votes with) is EXCLUDED from the join, not |
| 612 | // served that box's last-known reading as if it were current; and neither a |
| 613 | // garbage object under status/boxes/ nor an entirely absent one turns the |
| 614 | // list into a 500 — the join is best-effort observability layered onto the |
| 615 | // record at read time, never load-bearing for the list itself. |
| 616 | func TestAPIInstanceListStagedVersion(t *testing.T) { |
| 617 | fx, token := instanceFixture(t, "code-inst-staged") |
| 618 | enableInstances(t, fx, "titogo", titoManifest()) |
| 619 | u := instancesURL(fx, "bab", "titogo") |
| 620 | for _, h := range []string{"acme.go.tito.io", "beta.go.tito.io", "gamma.go.tito.io"} { |
| 621 | if code, _, out := rawReq(t, http.MethodPost, u, token, |
| 622 | fmt.Sprintf(`{"host":%q,"region":"ie"}`, h)); code != http.StatusCreated { |
| 623 | t.Fatalf("create %s = %d %s", h, code, out) |
| 624 | } |
| 625 | } |
| 626 | |
| 627 | // No box has reported at all yet: the list must still serve, every |
| 628 | // record's staged_version empty. |
| 629 | code, _, out := rawReq(t, http.MethodGet, u, token, "") |
| 630 | if code != http.StatusOK { |
| 631 | t.Fatalf("list with no status objects = %d %s, want 200", code, out) |
| 632 | } |
| 633 | var list struct { |
| 634 | Instances []struct { |
| 635 | Host string `json:"host"` |
| 636 | StagedVersion string `json:"staged_version"` |
| 637 | } `json:"instances"` |
| 638 | } |
| 639 | if err := json.Unmarshal([]byte(out), &list); err != nil { |
| 640 | t.Fatalf("list body %s: %v", out, err) |
| 641 | } |
| 642 | for _, r := range list.Instances { |
| 643 | if r.StagedVersion != "" { |
| 644 | t.Errorf("%s staged_version = %q before any box reported, want empty", r.Host, r.StagedVersion) |
| 645 | } |
| 646 | } |
| 647 | |
| 648 | // Seed a FRESH box snapshot naming acme's staged version, a STALE one |
| 649 | // (older than status.SnapshotPeriod x 3 — see status.BoxStale) naming |
| 650 | // gamma's, and a garbage object alongside them that must be skipped, not |
| 651 | // fatal. beta gets no box at all. |
| 652 | fresh := status.CollectBox(nil, status.Probes{MachineID: "box1"}, time.Now()) |
| 653 | fresh.Metrics = &status.BoxMetrics{RouteVersions: map[string]string{"acme.go.tito.io": "abc1234"}} |
| 654 | if err := status.WriteBox(context.Background(), fx.rel, fresh); err != nil { |
| 655 | t.Fatal(err) |
| 656 | } |
| 657 | stale := status.CollectBox(nil, status.Probes{MachineID: "box2"}, time.Now().Add(-time.Hour)) |
| 658 | stale.Metrics = &status.BoxMetrics{RouteVersions: map[string]string{"gamma.go.tito.io": "deadbeef"}} |
| 659 | if err := status.WriteBox(context.Background(), fx.rel, stale); err != nil { |
| 660 | t.Fatal(err) |
| 661 | } |
| 662 | if _, err := fx.rel.Put(context.Background(), "status/boxes/broken.json", []byte("{not json"), ""); err != nil { |
| 663 | t.Fatal(err) |
| 664 | } |
| 665 | |
| 666 | code, _, out = rawReq(t, http.MethodGet, u, token, "") |
| 667 | if code != http.StatusOK { |
| 668 | t.Fatalf("list with seeded boxes and a garbage object = %d %s, want 200", code, out) |
| 669 | } |
| 670 | if err := json.Unmarshal([]byte(out), &list); err != nil { |
| 671 | t.Fatalf("list body %s: %v", out, err) |
| 672 | } |
| 673 | got := map[string]string{} |
| 674 | for _, r := range list.Instances { |
| 675 | got[r.Host] = r.StagedVersion |
| 676 | } |
| 677 | if got["acme.go.tito.io"] != "abc1234" { |
| 678 | t.Errorf("acme staged_version = %q, want abc1234 (its box is fresh)", got["acme.go.tito.io"]) |
| 679 | } |
| 680 | if got["beta.go.tito.io"] != "" { |
| 681 | t.Errorf("beta staged_version = %q, want empty (no box has reported it)", got["beta.go.tito.io"]) |
| 682 | } |
| 683 | if got["gamma.go.tito.io"] != "" { |
| 684 | t.Errorf("gamma staged_version = %q, want empty — its only reporting box is stale and must not be joined", got["gamma.go.tito.io"]) |
| 685 | } |
| 686 | } |
| 687 | |
| 688 | // TestAPIInstanceListHealthAndFilters (Task 9): the list door grows the same |
| 689 | // controls the Running tab has — a health word per record, a substring |
| 690 | // filter, a health filter, and an opaque cursor — computed server-side so the |
| 691 | // CLI, the API and the page can never disagree about an instance's state. The |
| 692 | // growth is ADDITIVE: a client that knows only `instances` + `truncated` |
| 693 | // still parses this body, every pre-existing field byte-identical, and a |
| 694 | // request carrying none of the new params still returns every record it did |
| 695 | // before. |
| 696 | func TestAPIInstanceListHealthAndFilters(t *testing.T) { |
| 697 | fx, token := instanceFixture(t, "code-inst-health") |
| 698 | enableInstances(t, fx, "titogo", titoManifest()) |
| 699 | u := instancesURL(fx, "bab", "titogo") |
| 700 | // "a" and "ab" share a prefix so q=a proves a SUBSTRING match rather than |
| 701 | // an accidental equality, and "b" proves it excludes. |
| 702 | for _, h := range []string{"a.go.tito.io", "ab.go.tito.io", "b.go.tito.io", "run.go.tito.io"} { |
| 703 | if code, _, out := rawReq(t, http.MethodPost, u, token, |
| 704 | fmt.Sprintf(`{"host":%q,"region":"ie"}`, h)); code != http.StatusCreated { |
| 705 | t.Fatalf("create %s = %d %s", h, code, out) |
| 706 | } |
| 707 | } |
| 708 | // a's record has been materialized into a route: ready, instance-kind, |
| 709 | // not hibernating, no stamped unit and not provisioned — nothing the |
| 710 | // sweep was ever told how to sample, so its silence is honest ignorance |
| 711 | // ("No readings"), not an outage. |
| 712 | fx.addRoute(t, registry.Route{Host: "a.go.tito.io", Kind: "instance", Status: "ready", |
| 713 | App: "titogo", Account: fx.sqid, Channel: "stable"}) |
| 714 | // ab has no registry row at all: still provisioning, "Setting up". |
| 715 | // |
| 716 | // b's host carries a route stamped to ANOTHER account and app, with a |
| 717 | // bespoke unit that IS reporting in the sweep below. That row is not this |
| 718 | // record's route — the box's reconcile pass refuses to converge it too — |
| 719 | // so b must read "Setting up", and the neighbour's unit must never enter |
| 720 | // the metrics allowlist. If the tenant check were dropped, b would read |
| 721 | // "Running" off someone else's process. |
| 722 | fx.addRoute(t, registry.Route{Host: "b.go.tito.io", Kind: "instance", Status: "ready", |
| 723 | App: "neighbour", Account: "zzz", Unit: "neighbour-canary@b.service"}) |
| 724 | // run is the live case: provisioned, and its carlos-app@ unit is in a |
| 725 | // fresh sweep — "Running", which is what proves the sweep join is wired |
| 726 | // rather than every word coming from the registry alone. |
| 727 | fx.addRoute(t, registry.Route{Host: "run.go.tito.io", Kind: "instance", Status: "ready", |
| 728 | App: "titogo", Account: fx.sqid, Channel: "stable", Provisioned: true}) |
| 729 | at := time.Now().UTC().Add(-time.Minute) |
| 730 | if err := metrics.WriteRaw(context.Background(), fx.rel, metrics.RawChunk{ |
| 731 | V: 1, Box: "box-health", Hour: metrics.HourOf(at), |
| 732 | Samples: []metrics.Sample{{At: at, Units: map[string]metrics.UnitSample{ |
| 733 | activator.UnitName("run.go.tito.io"): {}, |
| 734 | "neighbour-canary@b.service": {}, |
| 735 | }}}, |
| 736 | }); err != nil { |
| 737 | t.Fatal(err) |
| 738 | } |
| 739 | |
| 740 | type listBody struct { |
| 741 | Instances []struct { |
| 742 | Host string `json:"host"` |
| 743 | Health string `json:"health"` |
| 744 | StagedVersion string `json:"staged_version"` |
| 745 | } `json:"instances"` |
| 746 | Truncated bool `json:"truncated"` |
| 747 | Total int `json:"total"` |
| 748 | Counts map[string]int `json:"counts"` |
| 749 | NextCursor string `json:"next_cursor"` |
| 750 | } |
| 751 | get := func(t *testing.T, query string) (listBody, string) { |
| 752 | t.Helper() |
| 753 | code, _, out := rawReq(t, http.MethodGet, u+query, token, "") |
| 754 | if code != http.StatusOK { |
| 755 | t.Fatalf("list%s = %d %s, want 200", query, code, out) |
| 756 | } |
| 757 | var body listBody |
| 758 | if err := json.Unmarshal([]byte(out), &body); err != nil { |
| 759 | t.Fatalf("list%s body %s: %v", query, out, err) |
| 760 | } |
| 761 | return body, out |
| 762 | } |
| 763 | hostsOf := func(b listBody) []string { |
| 764 | var out []string |
| 765 | for _, r := range b.Instances { |
| 766 | out = append(out, r.Host) |
| 767 | } |
| 768 | return out |
| 769 | } |
| 770 | |
| 771 | order := []string{"a.go.tito.io", "ab.go.tito.io", "b.go.tito.io", "run.go.tito.io"} |
| 772 | wantCounts := map[string]int{"No readings": 1, "Setting up": 2, "Running": 1} |
| 773 | |
| 774 | // 1) No params: every record, each carrying its health word, plus the |
| 775 | // whole-app totals the header states. |
| 776 | all, _ := get(t, "") |
| 777 | if got := hostsOf(all); !slices.Equal(got, order) { |
| 778 | t.Fatalf("unfiltered list = %v, want all four in host order", got) |
| 779 | } |
| 780 | wantHealth := map[string]string{ |
| 781 | "a.go.tito.io": "No readings", |
| 782 | "ab.go.tito.io": "Setting up", |
| 783 | "b.go.tito.io": "Setting up", // a neighbour's route answers for nobody |
| 784 | "run.go.tito.io": "Running", |
| 785 | } |
| 786 | for _, r := range all.Instances { |
| 787 | if r.Health != wantHealth[r.Host] { |
| 788 | t.Errorf("%s health = %q, want %q", r.Host, r.Health, wantHealth[r.Host]) |
| 789 | } |
| 790 | } |
| 791 | if all.Total != 4 { |
| 792 | t.Errorf("total = %d, want 4", all.Total) |
| 793 | } |
| 794 | if !maps.Equal(all.Counts, wantCounts) { |
| 795 | t.Errorf("counts = %v, want %v", all.Counts, wantCounts) |
| 796 | } |
| 797 | if all.NextCursor != "" { |
| 798 | t.Errorf("next_cursor = %q on a single-page list, want empty", all.NextCursor) |
| 799 | } |
| 800 | |
| 801 | // 2) q is a substring match on the host. |
| 802 | q, _ := get(t, "?q=a") |
| 803 | if got := hostsOf(q); !slices.Equal(got, []string{"a.go.tito.io", "ab.go.tito.io"}) { |
| 804 | t.Errorf("q=a = %v, want a and ab", got) |
| 805 | } |
| 806 | if q.Total != 2 { |
| 807 | t.Errorf("q=a total = %d, want 2", q.Total) |
| 808 | } |
| 809 | // Counts are the app's WHOLE health story regardless of the filter — |
| 810 | // they are the header's chips, and a filtered count would make the chips |
| 811 | // unable to lead anywhere. |
| 812 | if !maps.Equal(q.Counts, wantCounts) { |
| 813 | t.Errorf("q=a counts = %v, want the unfiltered %v", q.Counts, wantCounts) |
| 814 | } |
| 815 | |
| 816 | // 3) limit + cursor: one record a page, in host order, no overlap, and |
| 817 | // the cursor ends on the last page. |
| 818 | cursor := "" |
| 819 | for i, want := range order { |
| 820 | page, _ := get(t, "?limit=1&cursor="+url.QueryEscape(cursor)) |
| 821 | if got := hostsOf(page); !slices.Equal(got, []string{want}) { |
| 822 | t.Fatalf("limit=1 page %d = %v, want %q alone", i+1, got, want) |
| 823 | } |
| 824 | if page.Total != 4 { |
| 825 | t.Errorf("limit=1 page %d total = %d, want 4 (the match count, not the page length)", i+1, page.Total) |
| 826 | } |
| 827 | last := i == len(order)-1 |
| 828 | if last && page.NextCursor != "" { |
| 829 | t.Errorf("next_cursor = %q on the last page, want empty", page.NextCursor) |
| 830 | } |
| 831 | if !last && page.NextCursor == "" { |
| 832 | t.Fatalf("limit=1 page %d has no next_cursor with records left", i+1) |
| 833 | } |
| 834 | cursor = page.NextCursor |
| 835 | } |
| 836 | |
| 837 | // 4) health filters on the computed word — and a record with no route of |
| 838 | // its own reads as "Setting up". |
| 839 | h, _ := get(t, "?health=setting-up") |
| 840 | if got := hostsOf(h); !slices.Equal(got, []string{"ab.go.tito.io", "b.go.tito.io"}) { |
| 841 | t.Errorf("health=setting-up = %v, want the two records with no route of their own", got) |
| 842 | } |
| 843 | if h.Total != 2 { |
| 844 | t.Errorf("health=setting-up total = %d, want 2", h.Total) |
| 845 | } |
| 846 | running, _ := get(t, "?health=running") |
| 847 | if got := hostsOf(running); !slices.Equal(got, []string{"run.go.tito.io"}) { |
| 848 | t.Errorf("health=running = %v, want only the reporting instance", got) |
| 849 | } |
| 850 | // A mistyped word is no filter at all (healthFromSlug), never an empty |
| 851 | // list a caller would read as "you have no instances". |
| 852 | unknown, _ := get(t, "?health=nonsense") |
| 853 | if got := hostsOf(unknown); !slices.Equal(got, order) { |
| 854 | t.Errorf("health=nonsense = %v, want everything — an unknown word is no filter", got) |
| 855 | } |
| 856 | stable, _ := get(t, "?channel=stable") |
| 857 | if got := hostsOf(stable); !slices.Equal(got, order) || stable.Total != len(order) { |
| 858 | t.Errorf("channel=stable = %v total %d, want all stable records", got, stable.Total) |
| 859 | } |
| 860 | missing, _ := get(t, "?channel=retired") |
| 861 | if got := hostsOf(missing); len(got) != 0 || missing.Total != 0 { |
| 862 | t.Errorf("channel=retired widened to %v total %d", got, missing.Total) |
| 863 | } |
| 864 | |
| 865 | // 5) ADDITIVE: the OLD client's shape still parses, and every field a |
| 866 | // record already had is byte-identical. The keys the wire gained are |
| 867 | // "health", "staged_version" and "gated". |
| 868 | _, raw := get(t, "") |
| 869 | var old struct { |
| 870 | Instances []provision.Record `json:"instances"` |
| 871 | Truncated bool `json:"truncated"` |
| 872 | } |
| 873 | if err := json.Unmarshal([]byte(raw), &old); err != nil { |
| 874 | t.Fatalf("old client shape no longer parses %s: %v", raw, err) |
| 875 | } |
| 876 | if len(old.Instances) != len(order) || old.Truncated { |
| 877 | t.Fatalf("old shape = %d records truncated=%v, want %d and false", |
| 878 | len(old.Instances), old.Truncated, len(order)) |
| 879 | } |
| 880 | var wire struct { |
| 881 | Instances []map[string]json.RawMessage `json:"instances"` |
| 882 | } |
| 883 | if err := json.Unmarshal([]byte(raw), &wire); err != nil { |
| 884 | t.Fatal(err) |
| 885 | } |
| 886 | for i, rec := range old.Instances { |
| 887 | b, err := json.Marshal(rec) |
| 888 | if err != nil { |
| 889 | t.Fatal(err) |
| 890 | } |
| 891 | var want map[string]json.RawMessage |
| 892 | if err := json.Unmarshal(b, &want); err != nil { |
| 893 | t.Fatal(err) |
| 894 | } |
| 895 | for k, v := range want { |
| 896 | if !bytes.Equal(wire.Instances[i][k], v) { |
| 897 | t.Errorf("%s field %q = %s, want the record's own %s", |
| 898 | rec.Host, k, wire.Instances[i][k], v) |
| 899 | } |
| 900 | } |
| 901 | var extra []string |
| 902 | for k := range wire.Instances[i] { |
| 903 | if _, ok := want[k]; !ok { |
| 904 | extra = append(extra, k) |
| 905 | } |
| 906 | } |
| 907 | sort.Strings(extra) |
| 908 | // "gated" joined health and staged_version when the gate landed: the |
| 909 | // hash itself is shadowed out of the wire (recordView), so what a |
| 910 | // list gains is the one bit a member may know. |
| 911 | // "settings_default" joined them with the update settings (plan 5): |
| 912 | // the one bit the CLI needs to tell an instance on a box that ignores |
| 913 | // settings from one that never asked for any. |
| 914 | if !slices.Equal(extra, []string{"gated", "health", "settings_default", "staged_version"}) { |
| 915 | t.Errorf("%s carries extra keys %v, want exactly gated, health, settings_default and staged_version", rec.Host, extra) |
| 916 | } |
| 917 | } |
| 918 | } |
| 919 | |
| 920 | // TestAPIInstanceCreateCarriesRouteShape (multi-upstream design, issue |
| 921 | // #152): the create door now carries the record's ROUTE SHAPE — kind, the |
| 922 | // upstream set and the probe config — so an app fronting more than one |
| 923 | // process can be provisioned through the same one write every other instance |
| 924 | // is. Policy is unchanged and stays server-side: pool and channel still come |
| 925 | // from the manifest, and the host still has to be one label under the app's |
| 926 | // own domain (a shape field must never become a way to claim a hostname the |
| 927 | // suffix rule refuses). |
| 928 | func TestAPIInstanceCreateCarriesRouteShape(t *testing.T) { |
| 929 | fx, token := instanceFixture(t, "code-inst-shape") |
| 930 | letStateUpstreams(t, fx, "alice@keymail.dev") |
| 931 | enableInstances(t, fx, "titogo", titoManifest()) |
| 932 | u := instancesURL(fx, "bab", "titogo") |
| 933 | |
| 934 | status, _, out := rawReq(t, http.MethodPost, u, token, |
| 935 | `{"host":"ha.go.tito.io","region":"ie","kind":"service",`+ |
| 936 | `"addrs":["127.0.0.1:7001","127.0.0.1:7002"],"probe":{"path":"/healthz"}}`) |
| 937 | if status != http.StatusCreated { |
| 938 | t.Fatalf("create with a route shape = %d %s, want 201", status, out) |
| 939 | } |
| 940 | var rec provision.Record |
| 941 | if err := json.Unmarshal([]byte(out), &rec); err != nil { |
| 942 | t.Fatalf("create body %s: %v", out, err) |
| 943 | } |
| 944 | if rec.Kind != "service" || !slices.Equal(rec.Addrs, []string{"127.0.0.1:7001", "127.0.0.1:7002"}) { |
| 945 | t.Fatalf("echoed record = %+v, want the requested kind and upstreams in order", rec) |
| 946 | } |
| 947 | if rec.Health == nil || rec.Health.Path != "/healthz" { |
| 948 | t.Fatalf("echoed probe = %+v, want the requested path", rec.Health) |
| 949 | } |
| 950 | // Manifest policy still wins over anything the body could say. |
| 951 | if rec.Pool != "tito" || rec.Channel != "stable" { |
| 952 | t.Fatalf("record = %+v, want pool and channel from the manifest", rec) |
| 953 | } |
| 954 | |
| 955 | // The bucket copy is what the reconcile pass mints from, so it — not just |
| 956 | // the echo — has to carry the shape, under the record's own wire keys. |
| 957 | stored, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "ha.go.tito.io") |
| 958 | if err != nil { |
| 959 | t.Fatal(err) |
| 960 | } |
| 961 | if stored.Kind != "service" || !slices.Equal(stored.Addrs, rec.Addrs) || |
| 962 | stored.Health == nil || stored.Health.Path != "/healthz" { |
| 963 | t.Fatalf("stored record = %+v", stored) |
| 964 | } |
| 965 | raw, _, err := fx.rel.Get(context.Background(), provision.RecordKey("bab", "titogo", "ha.go.tito.io")) |
| 966 | if err != nil { |
| 967 | t.Fatal(err) |
| 968 | } |
| 969 | if !strings.Contains(string(raw), `"probe":`) { |
| 970 | t.Fatalf("stored record does not use the probe wire key: %s", raw) |
| 971 | } |
| 972 | |
| 973 | // A record shape ValidateShape refuses never reaches the bucket, and the |
| 974 | // refusal is ValidateShape's own sentence — the caller has to know WHICH |
| 975 | // upstream was wrong. |
| 976 | for _, c := range []struct{ what, body, want string }{ |
| 977 | {"bad upstream", `{"host":"bad.go.tito.io","region":"ie","kind":"service",` + |
| 978 | `"addrs":["127.0.0.1:7001","not-an-addr"]}`, "host:port"}, |
| 979 | {"duplicate upstream", `{"host":"dup.go.tito.io","region":"ie","kind":"service",` + |
| 980 | `"addrs":["127.0.0.1:7001","127.0.0.1:7001"]}`, "duplicate upstream"}, |
| 981 | {"unknown kind", `{"host":"weird.go.tito.io","region":"ie","kind":"daemon"}`, "instance, service or static"}, |
| 982 | {"service with no upstream", `{"host":"svc.go.tito.io","region":"ie","kind":"service"}`, "must name the upstream"}, |
| 983 | {"static with upstreams", `{"host":"site.go.tito.io","region":"ie","kind":"static",` + |
| 984 | `"addrs":["127.0.0.1:7001"]}`, "proxies to nothing"}, |
| 985 | } { |
| 986 | status, _, out := rawReq(t, http.MethodPost, u, token, c.body) |
| 987 | if status != http.StatusBadRequest { |
| 988 | t.Errorf("%s = %d %s, want 400", c.what, status, out) |
| 989 | continue |
| 990 | } |
| 991 | if !strings.Contains(out, c.want) { |
| 992 | t.Errorf("%s refusal = %s, want it to name %q", c.what, out, c.want) |
| 993 | } |
| 994 | } |
| 995 | keys, err := fx.rel.List(context.Background(), provision.RecordsPrefix("bab", "titogo")) |
| 996 | if err != nil { |
| 997 | t.Fatal(err) |
| 998 | } |
| 999 | if len(keys) != 1 { |
| 1000 | t.Fatalf("refused creates left records in the bucket: %v", keys) |
| 1001 | } |
| 1002 | |
| 1003 | // A record carrying none of the new fields is byte-identical to what the |
| 1004 | // door wrote before they existed — the additivity rule, at the door. |
| 1005 | if status, _, out := rawReq(t, http.MethodPost, u, token, |
| 1006 | `{"host":"plain.go.tito.io","region":"ie"}`); status != http.StatusCreated { |
| 1007 | t.Fatalf("plain create = %d %s, want 201", status, out) |
| 1008 | } |
| 1009 | plain, _, err := fx.rel.Get(context.Background(), provision.RecordKey("bab", "titogo", "plain.go.tito.io")) |
| 1010 | if err != nil { |
| 1011 | t.Fatal(err) |
| 1012 | } |
| 1013 | for _, k := range []string{`"kind"`, `"addrs"`, `"probe"`} { |
| 1014 | if strings.Contains(string(plain), k) { |
| 1015 | t.Errorf("a shapeless create wrote %s into the record: %s", k, plain) |
| 1016 | } |
| 1017 | } |
| 1018 | } |
| 1019 | |
| 1020 | // updateURL is the PATCH door: one instance's mutable route shape. |
| 1021 | func updateURL(fx *consoleFixture, acct, app, host string) string { |
| 1022 | return instancesURL(fx, acct, app) + "/" + host |
| 1023 | } |
| 1024 | |
| 1025 | // TestAPIInstanceUpdateUpstreams: the PATCH door rewrites a live record's |
| 1026 | // upstream set and probe config in place — the whole point of the |
| 1027 | // multi-upstream feature being operable from the CLI — and announces the |
| 1028 | // write so boxes converge in seconds. The record is rewritten through the |
| 1029 | // canonical marshaler (so upstream ORDER, which shapeMismatch compares |
| 1030 | // verbatim, lands stable), everything else about the record is left exactly |
| 1031 | // as it was, and a body that asks for nothing is refused rather than costing |
| 1032 | // a bucket write and a fleet-wide wake. |
| 1033 | func TestAPIInstanceUpdateUpstreams(t *testing.T) { |
| 1034 | fx, token := instanceFixture(t, "code-inst-patch") |
| 1035 | letStateUpstreams(t, fx, "alice@keymail.dev") |
| 1036 | enableInstances(t, fx, "titogo", titoManifest()) |
| 1037 | u := instancesURL(fx, "bab", "titogo") |
| 1038 | if status, _, out := rawReq(t, http.MethodPost, u, token, |
| 1039 | `{"host":"ha.go.tito.io","region":"ie","kind":"service","addrs":["127.0.0.1:7001"]}`); status != http.StatusCreated { |
| 1040 | t.Fatalf("create = %d %s", status, out) |
| 1041 | } |
| 1042 | created, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "ha.go.tito.io") |
| 1043 | if err != nil { |
| 1044 | t.Fatal(err) |
| 1045 | } |
| 1046 | afterCreate := serial(t, fx) |
| 1047 | if afterCreate == 0 { |
| 1048 | t.Fatalf("create did not bump the serial") |
| 1049 | } |
| 1050 | |
| 1051 | pu := updateURL(fx, "bab", "titogo", "ha.go.tito.io") |
| 1052 | status, _, out := rawReq(t, http.MethodPatch, pu, token, |
| 1053 | `{"addrs":["127.0.0.1:7002","127.0.0.1:7001"],"probe":{"path":"/healthz","fail_after":5}}`) |
| 1054 | if status != http.StatusOK { |
| 1055 | t.Fatalf("patch = %d %s, want 200", status, out) |
| 1056 | } |
| 1057 | var echoed provision.Record |
| 1058 | if err := json.Unmarshal([]byte(out), &echoed); err != nil { |
| 1059 | t.Fatalf("patch body %s: %v", out, err) |
| 1060 | } |
| 1061 | if !slices.Equal(echoed.Addrs, []string{"127.0.0.1:7002", "127.0.0.1:7001"}) { |
| 1062 | t.Fatalf("echoed addrs = %v, want the requested pair IN THE REQUESTED ORDER", echoed.Addrs) |
| 1063 | } |
| 1064 | |
| 1065 | stored, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "ha.go.tito.io") |
| 1066 | if err != nil { |
| 1067 | t.Fatal(err) |
| 1068 | } |
| 1069 | if !slices.Equal(stored.Addrs, []string{"127.0.0.1:7002", "127.0.0.1:7001"}) { |
| 1070 | t.Fatalf("stored addrs = %v, want the requested pair in order", stored.Addrs) |
| 1071 | } |
| 1072 | if stored.Health == nil || stored.Health.Path != "/healthz" || stored.Health.FailAfter != 5 { |
| 1073 | t.Fatalf("stored probe = %+v", stored.Health) |
| 1074 | } |
| 1075 | // Identity, provenance and everything policy owns are untouched: this |
| 1076 | // door speaks for two fields and no others. |
| 1077 | if stored.Host != created.Host || stored.App != created.App || stored.Account != created.Account || |
| 1078 | stored.Slug != created.Slug || stored.Kind != created.Kind || stored.Pool != created.Pool || |
| 1079 | stored.Region != created.Region || stored.Channel != created.Channel || |
| 1080 | stored.CreatedBy != created.CreatedBy || !stored.CreatedAt.Equal(created.CreatedAt) { |
| 1081 | t.Fatalf("patch moved a field it does not own:\n got %+v\nwant %+v", stored, created) |
| 1082 | } |
| 1083 | // The record is what a box's reconcile leg reads, so the write is |
| 1084 | // announced exactly as create's is — once. |
| 1085 | if got := serial(t, fx); got != afterCreate+1 { |
| 1086 | t.Fatalf("serial after patch = %d, want exactly one bump past %d", got, afterCreate) |
| 1087 | } |
| 1088 | |
| 1089 | // addrs alone leaves the probe standing; probe alone leaves the addrs. |
| 1090 | if status, _, out := rawReq(t, http.MethodPatch, pu, token, `{"addrs":["127.0.0.1:7003"]}`); status != http.StatusOK { |
| 1091 | t.Fatalf("patch addrs only = %d %s, want 200", status, out) |
| 1092 | } |
| 1093 | stored, err = provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "ha.go.tito.io") |
| 1094 | if err != nil { |
| 1095 | t.Fatal(err) |
| 1096 | } |
| 1097 | if !slices.Equal(stored.Addrs, []string{"127.0.0.1:7003"}) || stored.Health == nil || stored.Health.Path != "/healthz" { |
| 1098 | t.Fatalf("addrs-only patch = %+v, want the probe left standing", stored) |
| 1099 | } |
| 1100 | if status, _, out := rawReq(t, http.MethodPatch, pu, token, `{"probe":{"path":"/up"}}`); status != http.StatusOK { |
| 1101 | t.Fatalf("patch probe only = %d %s, want 200", status, out) |
| 1102 | } |
| 1103 | stored, err = provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "ha.go.tito.io") |
| 1104 | if err != nil { |
| 1105 | t.Fatal(err) |
| 1106 | } |
| 1107 | if !slices.Equal(stored.Addrs, []string{"127.0.0.1:7003"}) || stored.Health.Path != "/up" { |
| 1108 | t.Fatalf("probe-only patch = %+v, want the addrs left standing", stored) |
| 1109 | } |
| 1110 | |
| 1111 | // Explicit null clears the probe — absent and null are different requests |
| 1112 | // on the wire (the apiConfigApply precedent). |
| 1113 | if status, _, out := rawReq(t, http.MethodPatch, pu, token, `{"probe":null}`); status != http.StatusOK { |
| 1114 | t.Fatalf("patch clearing the probe = %d %s, want 200", status, out) |
| 1115 | } |
| 1116 | stored, err = provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "ha.go.tito.io") |
| 1117 | if err != nil { |
| 1118 | t.Fatal(err) |
| 1119 | } |
| 1120 | if stored.Health != nil { |
| 1121 | t.Fatalf("probe:null did not clear the probe: %+v", stored.Health) |
| 1122 | } |
| 1123 | |
| 1124 | before := serial(t, fx) |
| 1125 | for _, c := range []struct{ what, body, want string }{ |
| 1126 | {"neither field", `{}`, "addrs"}, |
| 1127 | {"bad upstream", `{"addrs":["nope"]}`, "host:port"}, |
| 1128 | {"duplicate upstream", `{"addrs":["127.0.0.1:7003","127.0.0.1:7003"]}`, "duplicate upstream"}, |
| 1129 | } { |
| 1130 | status, _, out := rawReq(t, http.MethodPatch, pu, token, c.body) |
| 1131 | if status != http.StatusBadRequest { |
| 1132 | t.Errorf("patch with %s = %d %s, want 400", c.what, status, out) |
| 1133 | continue |
| 1134 | } |
| 1135 | if !strings.Contains(out, c.want) { |
| 1136 | t.Errorf("patch with %s refused with %s, want it to name %q", c.what, out, c.want) |
| 1137 | } |
| 1138 | } |
| 1139 | // A refused patch wrote nothing and announced nothing. |
| 1140 | unchanged, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "ha.go.tito.io") |
| 1141 | if err != nil { |
| 1142 | t.Fatal(err) |
| 1143 | } |
| 1144 | if !slices.Equal(unchanged.Addrs, []string{"127.0.0.1:7003"}) { |
| 1145 | t.Errorf("a refused patch changed the record: %+v", unchanged) |
| 1146 | } |
| 1147 | if got := serial(t, fx); got != before { |
| 1148 | t.Errorf("serial after refused patches = %d, want it unmoved from %d", got, before) |
| 1149 | } |
| 1150 | |
| 1151 | // A host with no record is the family's byte-identical 404 — a record's |
| 1152 | // existence is not a thing to probe for through this door either — and so |
| 1153 | // is a host that could never name one. |
| 1154 | want := canonicalNotFound(t) |
| 1155 | status, h, out := rawReq(t, http.MethodPatch, updateURL(fx, "bab", "titogo", "missing.go.tito.io"), |
| 1156 | token, `{"addrs":["127.0.0.1:7001"]}`) |
| 1157 | assertNotFound(t, "patch of a host with no record", want, status, h, out) |
| 1158 | status, h, out = rawReq(t, http.MethodPatch, instancesURL(fx, "bab", "titogo")+"/..%2f..%2fchannels", |
| 1159 | token, `{"addrs":["127.0.0.1:7001"]}`) |
| 1160 | assertNotFound(t, "patch of a traversal host", want, status, h, out) |
| 1161 | } |
| 1162 | |
| 1163 | // TestAPIInstanceUpdateRefusesShapeTransitions: the patch door repoints |
| 1164 | // upstreams; it does not convert one KIND of route into another. Both |
| 1165 | // directions across provision.Record.DerivesPaths are refused with a 400 that |
| 1166 | // names the way through (delete + create), because a box cannot make either |
| 1167 | // change in place: the socket-backed route would go on serving its socket |
| 1168 | // while the reconcile pass logged the same shape complaint every tick, and the |
| 1169 | // upstream-backed one emptied to nothing would become a route that serves |
| 1170 | // nothing at all. A 200 for a change that can never take effect is the worst |
| 1171 | // of the three answers. |
| 1172 | func TestAPIInstanceUpdateRefusesShapeTransitions(t *testing.T) { |
| 1173 | fx, token := instanceFixture(t, "code-inst-patch-shape") |
| 1174 | letStateUpstreams(t, fx, "alice@keymail.dev") |
| 1175 | enableInstances(t, fx, "titogo", titoManifest()) |
| 1176 | u := instancesURL(fx, "bab", "titogo") |
| 1177 | // A socket-backed instance (the ordinary provisioned tenant) and an |
| 1178 | // upstream-backed service, side by side. |
| 1179 | if code, _, out := rawReq(t, http.MethodPost, u, token, |
| 1180 | `{"host":"sock.go.tito.io","region":"ie"}`); code != http.StatusCreated { |
| 1181 | t.Fatalf("create socket-backed = %d %s", code, out) |
| 1182 | } |
| 1183 | if code, _, out := rawReq(t, http.MethodPost, u, token, |
| 1184 | `{"host":"svc.go.tito.io","region":"ie","kind":"service","addrs":["127.0.0.1:7001"]}`); code != http.StatusCreated { |
| 1185 | t.Fatalf("create upstream-backed = %d %s", code, out) |
| 1186 | } |
| 1187 | before := serial(t, fx) |
| 1188 | |
| 1189 | // Socket-backed -> upstream-backed. |
| 1190 | code, _, out := rawReq(t, http.MethodPatch, updateURL(fx, "bab", "titogo", "sock.go.tito.io"), |
| 1191 | token, `{"addrs":["127.0.0.1:7001"]}`) |
| 1192 | if code != http.StatusBadRequest { |
| 1193 | t.Fatalf("giving a socket-backed instance upstreams = %d %s, want 400", code, out) |
| 1194 | } |
| 1195 | if !strings.Contains(out, "delete") { |
| 1196 | t.Errorf("refusal = %s, want it to name the way through (delete and create)", out) |
| 1197 | } |
| 1198 | // Upstream-backed -> socket-backed (an emptied set on an instance-kind |
| 1199 | // record; kind "" derives paths again the moment it has no upstream). |
| 1200 | if code, _, out := rawReq(t, http.MethodPost, u, token, |
| 1201 | `{"host":"back.go.tito.io","region":"ie","addrs":["127.0.0.1:7002"]}`); code != http.StatusCreated { |
| 1202 | t.Fatalf("create instance-kind upstream-backed = %d %s", code, out) |
| 1203 | } |
| 1204 | code, _, out = rawReq(t, http.MethodPatch, updateURL(fx, "bab", "titogo", "back.go.tito.io"), |
| 1205 | token, `{"addrs":[]}`) |
| 1206 | if code != http.StatusBadRequest { |
| 1207 | t.Fatalf("emptying an upstream-backed instance's set = %d %s, want 400", code, out) |
| 1208 | } |
| 1209 | |
| 1210 | // Neither refusal wrote or announced anything. |
| 1211 | sock, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "sock.go.tito.io") |
| 1212 | if err != nil { |
| 1213 | t.Fatal(err) |
| 1214 | } |
| 1215 | if len(sock.Addrs) != 0 { |
| 1216 | t.Errorf("a refused shape transition reached the record: %+v", sock) |
| 1217 | } |
| 1218 | back, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "back.go.tito.io") |
| 1219 | if err != nil { |
| 1220 | t.Fatal(err) |
| 1221 | } |
| 1222 | if !slices.Equal(back.Addrs, []string{"127.0.0.1:7002"}) { |
| 1223 | t.Errorf("a refused clear reached the record: %+v", back) |
| 1224 | } |
| 1225 | // One create happened in between, so the serial moved exactly once. |
| 1226 | if got := serial(t, fx); got != before+1 { |
| 1227 | t.Errorf("serial = %d, want %d — a refused patch must not announce", got, before+1) |
| 1228 | } |
| 1229 | |
| 1230 | // The same-shape patch these guards must NOT catch: an upstream-backed |
| 1231 | // route swapping its upstreams stays upstream-backed and still works. |
| 1232 | if code, _, out := rawReq(t, http.MethodPatch, updateURL(fx, "bab", "titogo", "svc.go.tito.io"), |
| 1233 | token, `{"addrs":["127.0.0.1:7003","127.0.0.1:7004"]}`); code != http.StatusOK { |
| 1234 | t.Fatalf("same-shape repoint = %d %s, want 200", code, out) |
| 1235 | } |
| 1236 | } |
| 1237 | |
| 1238 | // TestAPIInstanceUpdateRefusalMatrix: the NEW verb joins the family's refusal |
| 1239 | // discipline — every way of not being allowed to touch this record answers |
| 1240 | // with the bytes http.NotFound writes, so a probe cannot learn that an |
| 1241 | // account, an app or an instance exists. This is the gate rule this repo has |
| 1242 | // paid for; a new route is exactly where it gets forgotten. |
| 1243 | func TestAPIInstanceUpdateRefusalMatrix(t *testing.T) { |
| 1244 | fx, member := instanceFixture(t, "code-inst-patch-refuse") |
| 1245 | letStateUpstreams(t, fx, "alice@keymail.dev") |
| 1246 | enableInstances(t, fx, "titogo", titoManifest()) |
| 1247 | if status, _, out := rawReq(t, http.MethodPost, instancesURL(fx, "bab", "titogo"), member, |
| 1248 | `{"host":"ha.go.tito.io","region":"ie","kind":"service","addrs":["127.0.0.1:7001"]}`); status != http.StatusCreated { |
| 1249 | t.Fatalf("create = %d %s", status, out) |
| 1250 | } |
| 1251 | now := time.Now() |
| 1252 | if err := fx.st.CreateAccount(42, "cdef", "widgets", "bob@keymail.dev", now); err != nil { |
| 1253 | t.Fatal(err) |
| 1254 | } |
| 1255 | if _, err := fx.st.CreateAppIn(42, "titogo", "bob@keymail.dev", now); err != nil { |
| 1256 | t.Fatal(err) |
| 1257 | } |
| 1258 | appTok, err := fx.st.NewAppToken(OpsAccountID, "titogo", "alice@keymail.dev", "signup", now) |
| 1259 | if err != nil { |
| 1260 | t.Fatal(err) |
| 1261 | } |
| 1262 | eve := cliToken(t, fx, "eve@keymail.dev", "code-inst-patch-eve") |
| 1263 | |
| 1264 | body := `{"addrs":["10.0.0.1:9999"]}` |
| 1265 | want := canonicalNotFound(t) |
| 1266 | for _, c := range []struct{ what, url, token string }{ |
| 1267 | {"non-member", updateURL(fx, "bab", "titogo", "ha.go.tito.io"), eve}, |
| 1268 | {"app token, foreign account", updateURL(fx, "cdef", "titogo", "ha.go.tito.io"), appTok}, |
| 1269 | {"app token, foreign app", updateURL(fx, "bab", "other", "ha.go.tito.io"), appTok}, |
| 1270 | {"app token, own account by name", updateURL(fx, "ops", "titogo", "ha.go.tito.io"), appTok}, |
| 1271 | {"unknown account", updateURL(fx, "zzzz", "titogo", "ha.go.tito.io"), member}, |
| 1272 | {"unclaimed app", updateURL(fx, "bab", "ghost-app", "ha.go.tito.io"), member}, |
| 1273 | } { |
| 1274 | status, h, out := rawReq(t, http.MethodPatch, c.url, c.token, body) |
| 1275 | assertNotFound(t, "patch as "+c.what, want, status, h, out) |
| 1276 | } |
| 1277 | // No credential at all is the 401 every CLI door gives. |
| 1278 | if status, _, out := rawReq(t, http.MethodPatch, updateURL(fx, "bab", "titogo", "ha.go.tito.io"), "", body); status != http.StatusUnauthorized { |
| 1279 | t.Errorf("patch with no token = %d %s, want 401", status, out) |
| 1280 | } |
| 1281 | // Nothing above touched the record. |
| 1282 | rec, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "ha.go.tito.io") |
| 1283 | if err != nil { |
| 1284 | t.Fatal(err) |
| 1285 | } |
| 1286 | if !slices.Equal(rec.Addrs, []string{"127.0.0.1:7001"}) { |
| 1287 | t.Fatalf("a refused patch reached the record: %+v", rec) |
| 1288 | } |
| 1289 | // An app token MAY patch its own app's instance. Stating a NEW upstream |
| 1290 | // is an operator's alone (refuseMemberUpstreams), so it patches the probe. |
| 1291 | if status, _, out := rawReq(t, http.MethodPatch, updateURL(fx, "bab", "titogo", "ha.go.tito.io"), |
| 1292 | appTok, `{"probe":{"path":"/up"}}`); status != http.StatusOK { |
| 1293 | t.Fatalf("app token patching its own app = %d %s, want 200", status, out) |
| 1294 | } |
| 1295 | // And a console with no deployment bucket answers 503, not a panic. |
| 1296 | fx.srv.cfg.Releases = nil |
| 1297 | if status, _, out := rawReq(t, http.MethodPatch, updateURL(fx, "bab", "titogo", "ha.go.tito.io"), |
| 1298 | member, body); status != http.StatusServiceUnavailable { |
| 1299 | t.Errorf("patch with no bucket = %d %s, want 503", status, out) |
| 1300 | } |
| 1301 | } |
| 1302 | |
| 1303 | // staleETagStore hands every reader an etag no stored object carries — what a |
| 1304 | // concurrent writer looks like from inside one request: the record was read, |
| 1305 | // and by the time the write-back went out the bytes had moved on. |
| 1306 | type staleETagStore struct{ release.Store } |
| 1307 | |
| 1308 | func (s staleETagStore) Get(ctx context.Context, key string) ([]byte, string, error) { |
| 1309 | body, _, err := s.Store.Get(ctx, key) |
| 1310 | return body, "moved-on", err |
| 1311 | } |
| 1312 | |
| 1313 | // TestAPIInstanceUpdateLostRace: the guarded write-back surfaces a lost race |
| 1314 | // as a 409 rather than overwriting whatever landed in between. The read and |
| 1315 | // the write are one decision — a member sent the upstream set they wanted |
| 1316 | // GIVEN the record they saw — so silently re-applying it to a body nobody |
| 1317 | // read would be exactly the clobber the etag exists to prevent. Nothing is |
| 1318 | // announced: no write landed, so there is no news. |
| 1319 | func TestAPIInstanceUpdateLostRace(t *testing.T) { |
| 1320 | fx, token := instanceFixture(t, "code-inst-patch-race") |
| 1321 | letStateUpstreams(t, fx, "alice@keymail.dev") |
| 1322 | enableInstances(t, fx, "titogo", titoManifest()) |
| 1323 | if status, _, out := rawReq(t, http.MethodPost, instancesURL(fx, "bab", "titogo"), token, |
| 1324 | `{"host":"ha.go.tito.io","region":"ie","kind":"service","addrs":["127.0.0.1:7001"]}`); status != http.StatusCreated { |
| 1325 | t.Fatalf("create = %d %s", status, out) |
| 1326 | } |
| 1327 | before := serial(t, fx) |
| 1328 | |
| 1329 | fx.srv.cfg.Releases = staleETagStore{fx.rel} |
| 1330 | status, _, out := rawReq(t, http.MethodPatch, updateURL(fx, "bab", "titogo", "ha.go.tito.io"), |
| 1331 | token, `{"addrs":["127.0.0.1:7002"]}`) |
| 1332 | if status != http.StatusConflict { |
| 1333 | t.Fatalf("patch losing the race = %d %s, want 409", status, out) |
| 1334 | } |
| 1335 | fx.srv.cfg.Releases = fx.rel |
| 1336 | rec, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "ha.go.tito.io") |
| 1337 | if err != nil { |
| 1338 | t.Fatal(err) |
| 1339 | } |
| 1340 | if !slices.Equal(rec.Addrs, []string{"127.0.0.1:7001"}) { |
| 1341 | t.Fatalf("the lost race still overwrote the record: %+v", rec) |
| 1342 | } |
| 1343 | if got := serial(t, fx); got != before { |
| 1344 | t.Fatalf("serial after a failed write = %d, want it unmoved from %d", got, before) |
| 1345 | } |
| 1346 | } |
| 1347 | |
| 1348 | // sharedPoolManifest is titoManifest with Pool cleared — the shared-pool |
| 1349 | // shape steering requires (v1's pool guardrail), so a test needing a record |
| 1350 | // that MAY steer starts here rather than mutating titoManifest in place. |
| 1351 | func sharedPoolManifest() provision.Manifest { |
| 1352 | m := titoManifest() |
| 1353 | m.Pool = "" |
| 1354 | return m |
| 1355 | } |
| 1356 | |
| 1357 | // TestAPIInstanceUpdateSteeringRoundTrip: the PATCH door's third PATCHable |
| 1358 | // field (Task 3) — a shared-pool record may opt into "latency" and back out |
| 1359 | // to "", each write bumping the serial exactly once, and neither write |
| 1360 | // disturbs the addrs/probe fields this same door already owns. |
| 1361 | func TestAPIInstanceUpdateSteeringRoundTrip(t *testing.T) { |
| 1362 | fx, token := instanceFixture(t, "code-inst-steer-round") |
| 1363 | letStateUpstreams(t, fx, "alice@keymail.dev") |
| 1364 | enableInstances(t, fx, "titogo", sharedPoolManifest()) |
| 1365 | u := instancesURL(fx, "bab", "titogo") |
| 1366 | if status, _, out := rawReq(t, http.MethodPost, u, token, |
| 1367 | `{"host":"steer.go.tito.io","region":"ie","addrs":["127.0.0.1:7001"]}`); status != http.StatusCreated { |
| 1368 | t.Fatalf("create = %d %s", status, out) |
| 1369 | } |
| 1370 | afterCreate := serial(t, fx) |
| 1371 | |
| 1372 | pu := updateURL(fx, "bab", "titogo", "steer.go.tito.io") |
| 1373 | status, _, out := rawReq(t, http.MethodPatch, pu, token, `{"steering":"latency"}`) |
| 1374 | if status != http.StatusOK { |
| 1375 | t.Fatalf("patch steering on = %d %s, want 200", status, out) |
| 1376 | } |
| 1377 | var echoed provision.Record |
| 1378 | if err := json.Unmarshal([]byte(out), &echoed); err != nil { |
| 1379 | t.Fatalf("patch body %s: %v", out, err) |
| 1380 | } |
| 1381 | if echoed.Steering != "latency" { |
| 1382 | t.Fatalf("echoed steering = %q, want latency", echoed.Steering) |
| 1383 | } |
| 1384 | if !slices.Equal(echoed.Addrs, []string{"127.0.0.1:7001"}) { |
| 1385 | t.Fatalf("steering-only patch moved addrs: %v", echoed.Addrs) |
| 1386 | } |
| 1387 | stored, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "steer.go.tito.io") |
| 1388 | if err != nil { |
| 1389 | t.Fatal(err) |
| 1390 | } |
| 1391 | if stored.Steering != "latency" { |
| 1392 | t.Fatalf("stored steering = %q, want latency", stored.Steering) |
| 1393 | } |
| 1394 | if got := serial(t, fx); got != afterCreate+1 { |
| 1395 | t.Fatalf("serial after patch on = %d, want exactly one bump past %d", got, afterCreate) |
| 1396 | } |
| 1397 | afterOn := serial(t, fx) |
| 1398 | |
| 1399 | status, _, out = rawReq(t, http.MethodPatch, pu, token, `{"steering":""}`) |
| 1400 | if status != http.StatusOK { |
| 1401 | t.Fatalf("patch steering off = %d %s, want 200", status, out) |
| 1402 | } |
| 1403 | stored, err = provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "steer.go.tito.io") |
| 1404 | if err != nil { |
| 1405 | t.Fatal(err) |
| 1406 | } |
| 1407 | if stored.Steering != "" { |
| 1408 | t.Fatalf("stored steering after clearing = %q, want empty", stored.Steering) |
| 1409 | } |
| 1410 | if !slices.Equal(stored.Addrs, []string{"127.0.0.1:7001"}) { |
| 1411 | t.Fatalf("steering-only patch moved addrs: %v", stored.Addrs) |
| 1412 | } |
| 1413 | if got := serial(t, fx); got != afterOn+1 { |
| 1414 | t.Fatalf("serial after patch off = %d, want exactly one bump past %d", got, afterOn) |
| 1415 | } |
| 1416 | |
| 1417 | // A body naming neither addrs, probe nor steering is still refused — the |
| 1418 | // third field joins the existing "nothing to update" rail rather than |
| 1419 | // making it optional. |
| 1420 | before := serial(t, fx) |
| 1421 | status, _, out = rawReq(t, http.MethodPatch, pu, token, `{}`) |
| 1422 | if status != http.StatusBadRequest { |
| 1423 | t.Fatalf("patch with neither field = %d %s, want 400", status, out) |
| 1424 | } |
| 1425 | if !strings.Contains(out, "steering") { |
| 1426 | t.Errorf("empty-body refusal = %s, want it to name steering among the patchable fields", out) |
| 1427 | } |
| 1428 | if got := serial(t, fx); got != before { |
| 1429 | t.Errorf("serial after a refused empty patch = %d, want it unmoved from %d", got, before) |
| 1430 | } |
| 1431 | } |
| 1432 | |
| 1433 | // lastEventDetail returns the most recent instance-update audit row's |
| 1434 | // Detail — the field this test cares about — or fails the test if none |
| 1435 | // exists. Events is newest-first (the activity tab's own assumption, and |
| 1436 | // every other audit test in this file's). |
| 1437 | func lastEventDetail(t *testing.T, fx *consoleFixture, app string) string { |
| 1438 | t.Helper() |
| 1439 | events, err := fx.st.Events(OpsAccountID, app, 20) |
| 1440 | if err != nil { |
| 1441 | t.Fatal(err) |
| 1442 | } |
| 1443 | for _, e := range events { |
| 1444 | if e.Action == "instance-update" { |
| 1445 | return e.Detail |
| 1446 | } |
| 1447 | } |
| 1448 | t.Fatalf("events = %v, want an instance-update row", events) |
| 1449 | return "" |
| 1450 | } |
| 1451 | |
| 1452 | // TestAPIInstanceUpdateAuditNamesChangedFields: review finding (Task 3 |
| 1453 | // follow-up) — a field-agnostic "updated instance <host> (cli)" audit line |
| 1454 | // collapses three distinct member-facing changes (addrs/probe/steering) |
| 1455 | // into one indistinguishable string on a security surface: "did anyone turn |
| 1456 | // on steering for this host last week?" must stay answerable from the audit |
| 1457 | // log alone. The detail names exactly the fields THIS request carried, built |
| 1458 | // from the same req.* nil checks the mutation itself uses, so it can never |
| 1459 | // claim a change the request didn't make (or omit one it did). |
| 1460 | func TestAPIInstanceUpdateAuditNamesChangedFields(t *testing.T) { |
| 1461 | fx, token := instanceFixture(t, "code-inst-audit-fields") |
| 1462 | letStateUpstreams(t, fx, "alice@keymail.dev") |
| 1463 | enableInstances(t, fx, "titogo", sharedPoolManifest()) |
| 1464 | u := instancesURL(fx, "bab", "titogo") |
| 1465 | if status, _, out := rawReq(t, http.MethodPost, u, token, |
| 1466 | `{"host":"audit.go.tito.io","region":"ie","addrs":["127.0.0.1:7001"]}`); status != http.StatusCreated { |
| 1467 | t.Fatalf("create = %d %s", status, out) |
| 1468 | } |
| 1469 | pu := updateURL(fx, "bab", "titogo", "audit.go.tito.io") |
| 1470 | |
| 1471 | // Single-field case: a steering-only patch names ONLY steering — not |
| 1472 | // "upstreams", not addrs, not probe. |
| 1473 | if status, _, out := rawReq(t, http.MethodPatch, pu, token, `{"steering":"latency"}`); status != http.StatusOK { |
| 1474 | t.Fatalf("patch steering = %d %s, want 200", status, out) |
| 1475 | } |
| 1476 | if detail := lastEventDetail(t, fx, "titogo"); detail != "updated instance audit.go.tito.io (steering) (cli)" { |
| 1477 | t.Fatalf("audit detail = %q, want it to name exactly steering", detail) |
| 1478 | } |
| 1479 | |
| 1480 | // Multi-field case: a patch touching addrs AND steering together names |
| 1481 | // both, in the order the request struct declares them (addrs, probe, |
| 1482 | // steering, environment, gate) — not just one of the two, and not a |
| 1483 | // generic "updated". |
| 1484 | if status, _, out := rawReq(t, http.MethodPatch, pu, token, |
| 1485 | `{"addrs":["127.0.0.1:7002"],"steering":""}`); status != http.StatusOK { |
| 1486 | t.Fatalf("patch addrs+steering = %d %s, want 200", status, out) |
| 1487 | } |
| 1488 | if detail := lastEventDetail(t, fx, "titogo"); detail != "updated instance audit.go.tito.io (addrs, steering) (cli)" { |
| 1489 | t.Fatalf("audit detail = %q, want it to name both addrs and steering", detail) |
| 1490 | } |
| 1491 | |
| 1492 | // environment-only case: an environment patch names ONLY environment — |
| 1493 | // the mutation applies req.Environment, so the audit line must too. |
| 1494 | if status, _, out := rawReq(t, http.MethodPatch, pu, token, `{"environment":"demo"}`); status != http.StatusOK { |
| 1495 | t.Fatalf("patch environment = %d %s, want 200", status, out) |
| 1496 | } |
| 1497 | if detail := lastEventDetail(t, fx, "titogo"); detail != "updated instance audit.go.tito.io (environment) (cli)" { |
| 1498 | t.Fatalf("audit detail = %q, want it to name exactly environment", detail) |
| 1499 | } |
| 1500 | |
| 1501 | // Combined case: steering, environment and gate together name all |
| 1502 | // three, in request-struct order. |
| 1503 | if status, _, out := rawReq(t, http.MethodPatch, pu, token, |
| 1504 | `{"steering":"","environment":"","gate":""}`); status != http.StatusOK { |
| 1505 | t.Fatalf("patch steering+environment+gate = %d %s, want 200", status, out) |
| 1506 | } |
| 1507 | if detail := lastEventDetail(t, fx, "titogo"); detail != "updated instance audit.go.tito.io (steering, environment, gate) (cli)" { |
| 1508 | t.Fatalf("audit detail = %q, want it to name steering, environment and gate", detail) |
| 1509 | } |
| 1510 | } |
| 1511 | |
| 1512 | // TestAPIInstanceUpdateSteeringInvalidValue: the door's own vocabulary rail |
| 1513 | // (provision.ValidateShape -> ValidSteering), surfaced as a 400 with the |
| 1514 | // door's own sentence — a caller sending anything but "" or "latency" is |
| 1515 | // told exactly that, and nothing is written or announced. |
| 1516 | func TestAPIInstanceUpdateSteeringInvalidValue(t *testing.T) { |
| 1517 | fx, token := instanceFixture(t, "code-inst-steer-invalid") |
| 1518 | enableInstances(t, fx, "titogo", sharedPoolManifest()) |
| 1519 | u := instancesURL(fx, "bab", "titogo") |
| 1520 | if status, _, out := rawReq(t, http.MethodPost, u, token, |
| 1521 | `{"host":"badsteer.go.tito.io","region":"ie"}`); status != http.StatusCreated { |
| 1522 | t.Fatalf("create = %d %s", status, out) |
| 1523 | } |
| 1524 | before := serial(t, fx) |
| 1525 | |
| 1526 | status, _, out := rawReq(t, http.MethodPatch, updateURL(fx, "bab", "titogo", "badsteer.go.tito.io"), |
| 1527 | token, `{"steering":"geo"}`) |
| 1528 | if status != http.StatusBadRequest { |
| 1529 | t.Fatalf("patch with invalid steering = %d %s, want 400", status, out) |
| 1530 | } |
| 1531 | var errBody struct { |
| 1532 | Error string `json:"error"` |
| 1533 | } |
| 1534 | if err := json.Unmarshal([]byte(out), &errBody); err != nil { |
| 1535 | t.Fatalf("refusal body %s: %v", out, err) |
| 1536 | } |
| 1537 | if want := `steering "geo": want "" or "latency"`; errBody.Error != want { |
| 1538 | t.Errorf("refusal = %q, want ValidateShape's own sentence %q", errBody.Error, want) |
| 1539 | } |
| 1540 | stored, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "badsteer.go.tito.io") |
| 1541 | if err != nil { |
| 1542 | t.Fatal(err) |
| 1543 | } |
| 1544 | if stored.Steering != "" { |
| 1545 | t.Fatalf("a refused patch reached the record: steering = %q", stored.Steering) |
| 1546 | } |
| 1547 | if got := serial(t, fx); got != before { |
| 1548 | t.Fatalf("serial after a refused patch = %d, want it unmoved from %d", got, before) |
| 1549 | } |
| 1550 | } |
| 1551 | |
| 1552 | // TestAPIInstanceUpdateSteeringPoolConflict: v1's pool guardrail (plan-2 |
| 1553 | // ledger) — a route whose Pool != "" cannot opt into latency steering, |
| 1554 | // because the latency records would point at edges that structurally |
| 1555 | // cannot serve a pool-scoped route. The refusal is a 409 naming the v1 |
| 1556 | // constraint verbatim, not a 400: the request is well-formed, the record |
| 1557 | // merely cannot honour it TODAY. |
| 1558 | // |
| 1559 | // The self-review point this test is built to settle: the rail reads the |
| 1560 | // STORED RECORD's Pool — the field the door itself stamped at create and |
| 1561 | // the only copy this handler ever reads — not the app's manifest, which a |
| 1562 | // later `instances enable` may have since repointed to a different pool. |
| 1563 | // The second half of this test proves that: the record keeps the pool it |
| 1564 | // was created with even after the manifest changes underneath it, and the |
| 1565 | // 409 keeps firing on that account. |
| 1566 | func TestAPIInstanceUpdateSteeringPoolConflict(t *testing.T) { |
| 1567 | fx, token := instanceFixture(t, "code-inst-steer-pool") |
| 1568 | enableInstances(t, fx, "titogo", titoManifest()) // Pool: "tito" |
| 1569 | u := instancesURL(fx, "bab", "titogo") |
| 1570 | if status, _, out := rawReq(t, http.MethodPost, u, token, |
| 1571 | `{"host":"pooled.go.tito.io","region":"ie"}`); status != http.StatusCreated { |
| 1572 | t.Fatalf("create = %d %s", status, out) |
| 1573 | } |
| 1574 | created, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "pooled.go.tito.io") |
| 1575 | if err != nil { |
| 1576 | t.Fatal(err) |
| 1577 | } |
| 1578 | if created.Pool != "tito" { |
| 1579 | t.Fatalf("created record pool = %q, want tito (test setup)", created.Pool) |
| 1580 | } |
| 1581 | before := serial(t, fx) |
| 1582 | |
| 1583 | pu := updateURL(fx, "bab", "titogo", "pooled.go.tito.io") |
| 1584 | status, _, out := rawReq(t, http.MethodPatch, pu, token, `{"steering":"latency"}`) |
| 1585 | if status != http.StatusConflict { |
| 1586 | t.Fatalf("patch steering on a pooled route = %d %s, want 409", status, out) |
| 1587 | } |
| 1588 | const wantMsg = "steering requires the shared pool — pool-scoped steering is a recorded future" |
| 1589 | if out != `{"error":"`+wantMsg+`"}`+"\n" && !strings.Contains(out, wantMsg) { |
| 1590 | t.Errorf("refusal = %s, want the v1 constraint named verbatim: %q", out, wantMsg) |
| 1591 | } |
| 1592 | stored, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "pooled.go.tito.io") |
| 1593 | if err != nil { |
| 1594 | t.Fatal(err) |
| 1595 | } |
| 1596 | if stored.Steering != "" { |
| 1597 | t.Fatalf("a refused patch reached the record: steering = %q", stored.Steering) |
| 1598 | } |
| 1599 | if got := serial(t, fx); got != before { |
| 1600 | t.Fatalf("serial after a refused patch = %d, want it unmoved from %d", got, before) |
| 1601 | } |
| 1602 | |
| 1603 | // Clearing steering (turning it off) is never blocked by the pool rail — |
| 1604 | // only an attempt to TURN latency ON is. |
| 1605 | if status, _, out := rawReq(t, http.MethodPatch, pu, token, `{"steering":""}`); status != http.StatusOK { |
| 1606 | t.Fatalf("patch steering off on a pooled route = %d %s, want 200", status, out) |
| 1607 | } |
| 1608 | |
| 1609 | // The manifest changes to the shared pool AFTER the record was created — |
| 1610 | // the record itself is immutable on this axis (Pool is stamped at |
| 1611 | // create and this door never touches it), so the rail must keep reading |
| 1612 | // the record it fetched, not re-derive the pool from the manifest that |
| 1613 | // happens to be live right now. |
| 1614 | enableInstances(t, fx, "titogo", sharedPoolManifest()) |
| 1615 | status, _, out = rawReq(t, http.MethodPatch, pu, token, `{"steering":"latency"}`) |
| 1616 | if status != http.StatusConflict { |
| 1617 | t.Fatalf("patch steering after the MANIFEST moved to the shared pool = %d %s, want 409 still "+ |
| 1618 | "(the RECORD's own pool, stamped at create, is what the rail reads)", status, out) |
| 1619 | } |
| 1620 | } |
| 1621 | |
| 1622 | // TestAPIInstanceUpdateSteeringLostRace: the steering field goes through |
| 1623 | // the exact same CAS write-back as addrs/probe — a concurrent writer means |
| 1624 | // the decision to steer was made about bytes that no longer exist, and the |
| 1625 | // honest answer is 409, not a silent clobber. |
| 1626 | func TestAPIInstanceUpdateSteeringLostRace(t *testing.T) { |
| 1627 | fx, token := instanceFixture(t, "code-inst-steer-race") |
| 1628 | enableInstances(t, fx, "titogo", sharedPoolManifest()) |
| 1629 | if status, _, out := rawReq(t, http.MethodPost, instancesURL(fx, "bab", "titogo"), token, |
| 1630 | `{"host":"steerrace.go.tito.io","region":"ie"}`); status != http.StatusCreated { |
| 1631 | t.Fatalf("create = %d %s", status, out) |
| 1632 | } |
| 1633 | before := serial(t, fx) |
| 1634 | |
| 1635 | fx.srv.cfg.Releases = staleETagStore{fx.rel} |
| 1636 | status, _, out := rawReq(t, http.MethodPatch, updateURL(fx, "bab", "titogo", "steerrace.go.tito.io"), |
| 1637 | token, `{"steering":"latency"}`) |
| 1638 | if status != http.StatusConflict { |
| 1639 | t.Fatalf("patch steering losing the race = %d %s, want 409", status, out) |
| 1640 | } |
| 1641 | fx.srv.cfg.Releases = fx.rel |
| 1642 | rec, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "steerrace.go.tito.io") |
| 1643 | if err != nil { |
| 1644 | t.Fatal(err) |
| 1645 | } |
| 1646 | if rec.Steering != "" { |
| 1647 | t.Fatalf("the lost race still overwrote the record: steering = %q", rec.Steering) |
| 1648 | } |
| 1649 | if got := serial(t, fx); got != before { |
| 1650 | t.Fatalf("serial after a failed write = %d, want it unmoved from %d", got, before) |
| 1651 | } |
| 1652 | } |
| 1653 | |
| 1654 | // TestAPIInstanceListJoinsUpstreams: the list's third join (beside |
| 1655 | // staged_version and health) — per-upstream probe state, read from the box |
| 1656 | // snapshots the same way and with the same staleness rule, so `carlos |
| 1657 | // instances` can say WHICH upstream is down without a box session. Additive: |
| 1658 | // the key is absent entirely for a record no box is probing, which is every |
| 1659 | // single-upstream record in production today. |
| 1660 | func TestAPIInstanceListJoinsUpstreams(t *testing.T) { |
| 1661 | fx, token := instanceFixture(t, "code-inst-upstreams") |
| 1662 | enableInstances(t, fx, "titogo", titoManifest()) |
| 1663 | u := instancesURL(fx, "bab", "titogo") |
| 1664 | for _, h := range []string{"ha.go.tito.io", "plain.go.tito.io", "stale.go.tito.io"} { |
| 1665 | if code, _, out := rawReq(t, http.MethodPost, u, token, |
| 1666 | fmt.Sprintf(`{"host":%q,"region":"ie"}`, h)); code != http.StatusCreated { |
| 1667 | t.Fatalf("create %s = %d %s", h, code, out) |
| 1668 | } |
| 1669 | } |
| 1670 | since := time.Now().UTC().Add(-2 * time.Minute).Truncate(time.Second) |
| 1671 | fresh := status.CollectBox(nil, status.Probes{MachineID: "box1"}, time.Now()) |
| 1672 | fresh.Metrics = &status.BoxMetrics{Upstreams: map[string][]status.UpstreamState{ |
| 1673 | "ha.go.tito.io": { |
| 1674 | {Addr: "127.0.0.1:7001", Up: true, Since: since}, |
| 1675 | {Addr: "127.0.0.1:7002", Up: false, Since: since, Fails: 3, LastError: "dial refused"}, |
| 1676 | }, |
| 1677 | }} |
| 1678 | if err := status.WriteBox(context.Background(), fx.rel, fresh); err != nil { |
| 1679 | t.Fatal(err) |
| 1680 | } |
| 1681 | stale := status.CollectBox(nil, status.Probes{MachineID: "box2"}, time.Now().Add(-time.Hour)) |
| 1682 | stale.Metrics = &status.BoxMetrics{Upstreams: map[string][]status.UpstreamState{ |
| 1683 | "stale.go.tito.io": {{Addr: "127.0.0.1:7009", Up: true, Since: since}}, |
| 1684 | }} |
| 1685 | if err := status.WriteBox(context.Background(), fx.rel, stale); err != nil { |
| 1686 | t.Fatal(err) |
| 1687 | } |
| 1688 | |
| 1689 | code, _, out := rawReq(t, http.MethodGet, u, token, "") |
| 1690 | if code != http.StatusOK { |
| 1691 | t.Fatalf("list = %d %s, want 200", code, out) |
| 1692 | } |
| 1693 | var list struct { |
| 1694 | Instances []struct { |
| 1695 | Host string `json:"host"` |
| 1696 | Upstreams []status.UpstreamState `json:"upstreams"` |
| 1697 | } `json:"instances"` |
| 1698 | } |
| 1699 | if err := json.Unmarshal([]byte(out), &list); err != nil { |
| 1700 | t.Fatalf("list body %s: %v", out, err) |
| 1701 | } |
| 1702 | got := map[string][]status.UpstreamState{} |
| 1703 | for _, r := range list.Instances { |
| 1704 | got[r.Host] = r.Upstreams |
| 1705 | } |
| 1706 | ha := got["ha.go.tito.io"] |
| 1707 | if len(ha) != 2 { |
| 1708 | t.Fatalf("ha upstreams = %+v, want both probed addrs", ha) |
| 1709 | } |
| 1710 | if ha[0].Addr != "127.0.0.1:7001" || !ha[0].Up { |
| 1711 | t.Errorf("first upstream = %+v, want 7001 up", ha[0]) |
| 1712 | } |
| 1713 | if ha[1].Addr != "127.0.0.1:7002" || ha[1].Up || ha[1].Fails != 3 || ha[1].LastError != "dial refused" { |
| 1714 | t.Errorf("second upstream = %+v, want 7002 down with its fail count and reason", ha[1]) |
| 1715 | } |
| 1716 | if len(got["plain.go.tito.io"]) != 0 { |
| 1717 | t.Errorf("an unprobed record carries upstreams: %+v", got["plain.go.tito.io"]) |
| 1718 | } |
| 1719 | // A box that stopped reporting drops OUT of the join rather than serving |
| 1720 | // its last-known belief as current — status.BoxStale, the same rule |
| 1721 | // staged_version votes with. |
| 1722 | if len(got["stale.go.tito.io"]) != 0 { |
| 1723 | t.Errorf("a stale box's probe table was joined: %+v", got["stale.go.tito.io"]) |
| 1724 | } |
| 1725 | // Absent, never present-and-empty: a client that never heard of this key |
| 1726 | // parses the record exactly as it always did. |
| 1727 | var wire struct { |
| 1728 | Instances []map[string]json.RawMessage `json:"instances"` |
| 1729 | } |
| 1730 | if err := json.Unmarshal([]byte(out), &wire); err != nil { |
| 1731 | t.Fatal(err) |
| 1732 | } |
| 1733 | for i, r := range list.Instances { |
| 1734 | _, present := wire.Instances[i]["upstreams"] |
| 1735 | if present != (r.Host == "ha.go.tito.io") { |
| 1736 | t.Errorf("%s upstreams key present = %v, want it only on the probed record", r.Host, present) |
| 1737 | } |
| 1738 | } |
| 1739 | } |
| 1740 | |
| 1741 | // TestAPIInstanceDelete: deletion is idempotent (the caller's goal is "not |
| 1742 | // there"), survives the manifest being retired, and still refuses a host |
| 1743 | // that isn't one. |
| 1744 | func TestAPIInstanceDelete(t *testing.T) { |
| 1745 | fx, token := instanceFixture(t, "code-inst-delete") |
| 1746 | enableInstances(t, fx, "titogo", titoManifest()) |
| 1747 | u := instancesURL(fx, "bab", "titogo") |
| 1748 | if status, _, out := rawReq(t, http.MethodPost, u, token, |
| 1749 | `{"host":"acme.go.tito.io","region":"ie"}`); status != http.StatusCreated { |
| 1750 | t.Fatalf("create = %d %s", status, out) |
| 1751 | } |
| 1752 | |
| 1753 | // A hostile host is refused before the key builder, and deletes nothing. |
| 1754 | if status, _, out := rawReq(t, http.MethodDelete, u+"/..%2f..%2finstances.json", token, ""); status != http.StatusBadRequest { |
| 1755 | t.Fatalf("delete of a traversal host = %d %s, want 400", status, out) |
| 1756 | } |
| 1757 | if _, _, err := fx.rel.Get(context.Background(), provision.ManifestKey("bab", "titogo")); err != nil { |
| 1758 | t.Fatalf("the manifest was destroyed by a traversal delete: %v", err) |
| 1759 | } |
| 1760 | |
| 1761 | req, err := http.NewRequest(http.MethodDelete, u+"/acme.go.tito.io", nil) |
| 1762 | if err != nil { |
| 1763 | t.Fatal(err) |
| 1764 | } |
| 1765 | req.Header.Set("Authorization", "Bearer "+token) |
| 1766 | req.Header.Set(activityMessageHeader, "retire the old web copy") |
| 1767 | res, err := http.DefaultClient.Do(req) |
| 1768 | if err != nil { |
| 1769 | t.Fatal(err) |
| 1770 | } |
| 1771 | outBytes, err := io.ReadAll(res.Body) |
| 1772 | res.Body.Close() |
| 1773 | if err != nil { |
| 1774 | t.Fatal(err) |
| 1775 | } |
| 1776 | status, out := res.StatusCode, string(outBytes) |
| 1777 | if status != http.StatusOK || !strings.Contains(out, "deleted") { |
| 1778 | t.Fatalf("delete = %d %s, want 200 deleted", status, out) |
| 1779 | } |
| 1780 | events, err := fx.st.Events(OpsAccountID, "titogo", 1) |
| 1781 | if err != nil || len(events) != 1 || events[0].Message != "retire the old web copy" { |
| 1782 | t.Fatalf("delete activity = %+v, %v", events, err) |
| 1783 | } |
| 1784 | if _, _, err := fx.rel.Get(context.Background(), |
| 1785 | provision.RecordKey("bab", "titogo", "acme.go.tito.io")); err == nil { |
| 1786 | t.Fatalf("the record survived its delete") |
| 1787 | } |
| 1788 | // Again: still 200. "Already gone" is the goal, reached. |
| 1789 | if status, _, out := rawReq(t, http.MethodDelete, u+"/acme.go.tito.io", token, ""); status != http.StatusOK { |
| 1790 | t.Fatalf("second delete = %d %s, want an idempotent 200", status, out) |
| 1791 | } |
| 1792 | |
| 1793 | // Deletion outlives the manifest: an app that retired instances must |
| 1794 | // still be able to clean up the records it left behind. |
| 1795 | if status, _, out := rawReq(t, http.MethodPost, u, token, |
| 1796 | `{"host":"beta.go.tito.io","region":"ie"}`); status != http.StatusCreated { |
| 1797 | t.Fatalf("create = %d %s", status, out) |
| 1798 | } |
| 1799 | if err := fx.rel.Delete(context.Background(), provision.ManifestKey("bab", "titogo")); err != nil { |
| 1800 | t.Fatal(err) |
| 1801 | } |
| 1802 | if status, _, out := rawReq(t, http.MethodDelete, u+"/beta.go.tito.io", token, ""); status != http.StatusOK { |
| 1803 | t.Fatalf("delete with the manifest retired = %d %s, want 200", status, out) |
| 1804 | } |
| 1805 | // And the shape check still applies with no domain to check against. |
| 1806 | if status, _, out := rawReq(t, http.MethodDelete, u+"/..%2f..%2fchannels", token, ""); status != http.StatusBadRequest { |
| 1807 | t.Fatalf("traversal delete with no manifest = %d %s, want 400", status, out) |
| 1808 | } |
| 1809 | } |
| 1810 | |
| 1811 | // TestAPIInstanceDeleteAfterDomainChange: the suffix check is CREATE-time |
| 1812 | // policy — "which hosts may this app claim" — and deletion must outlive it. |
| 1813 | // An app that moves domains still has records under the old one, and they |
| 1814 | // are exactly the records that must be deletable: refusing them would strand |
| 1815 | // provisioned routes with nothing left that can retire them. Delete |
| 1816 | // validates SHAPE only, always, which is the whole traversal defence it |
| 1817 | // needs. |
| 1818 | func TestAPIInstanceDeleteAfterDomainChange(t *testing.T) { |
| 1819 | fx, token := instanceFixture(t, "code-inst-redomain") |
| 1820 | enableInstances(t, fx, "titogo", titoManifest()) |
| 1821 | u := instancesURL(fx, "bab", "titogo") |
| 1822 | if status, _, out := rawReq(t, http.MethodPost, u, token, |
| 1823 | `{"host":"acme.go.tito.io","region":"ie"}`); status != http.StatusCreated { |
| 1824 | t.Fatalf("create = %d %s", status, out) |
| 1825 | } |
| 1826 | |
| 1827 | // The app moves to a new domain: new instances land under it, and the |
| 1828 | // old host is now a host this manifest would never have allowed. |
| 1829 | moved := titoManifest() |
| 1830 | moved.Domain = "go.tito.dev" |
| 1831 | enableInstances(t, fx, "titogo", moved) |
| 1832 | if status, _, out := rawReq(t, http.MethodPost, u, token, |
| 1833 | `{"host":"acme.go.tito.io","region":"ie"}`); status != http.StatusBadRequest { |
| 1834 | t.Fatalf("create under the OLD domain = %d %s, want 400 — the suffix rule still governs creates", status, out) |
| 1835 | } |
| 1836 | |
| 1837 | if status, _, out := rawReq(t, http.MethodDelete, u+"/acme.go.tito.io", token, ""); status != http.StatusOK { |
| 1838 | t.Fatalf("delete of a record created under the old domain = %d %s, want 200", status, out) |
| 1839 | } |
| 1840 | if _, _, err := fx.rel.Get(context.Background(), |
| 1841 | provision.RecordKey("bab", "titogo", "acme.go.tito.io")); err == nil { |
| 1842 | t.Fatalf("the record survived its delete") |
| 1843 | } |
| 1844 | // Shape is still the door: a traversal host is refused whatever the |
| 1845 | // manifest says. |
| 1846 | if status, _, out := rawReq(t, http.MethodDelete, u+"/..%2f..%2finstances.json", token, ""); status != http.StatusBadRequest { |
| 1847 | t.Fatalf("traversal delete after a domain change = %d %s, want 400", status, out) |
| 1848 | } |
| 1849 | if _, _, err := fx.rel.Get(context.Background(), provision.ManifestKey("bab", "titogo")); err != nil { |
| 1850 | t.Fatalf("the manifest was destroyed by a traversal delete: %v", err) |
| 1851 | } |
| 1852 | } |
| 1853 | |
| 1854 | // TestAPIInstanceTokenMintEndToEnd is the whole app-token story: a member |
| 1855 | // mints one, the raw token is shown exactly once, and it then provisions on |
| 1856 | // its own — and every mutation left an audit row naming its principal. |
| 1857 | func TestAPIInstanceTokenMintEndToEnd(t *testing.T) { |
| 1858 | fx, token := instanceFixture(t, "code-inst-mint") |
| 1859 | enableInstances(t, fx, "titogo", titoManifest()) |
| 1860 | |
| 1861 | status, _, out := rawReq(t, http.MethodPost, fx.ts.URL+"/api/cli/apps/bab/titogo/instance-token", |
| 1862 | token, `{"label":"tito signup"}`) |
| 1863 | if status != http.StatusCreated { |
| 1864 | t.Fatalf("mint = %d %s, want 201", status, out) |
| 1865 | } |
| 1866 | var minted struct { |
| 1867 | Token string `json:"token"` |
| 1868 | Account string `json:"account"` |
| 1869 | App string `json:"app"` |
| 1870 | } |
| 1871 | if err := json.Unmarshal([]byte(out), &minted); err != nil { |
| 1872 | t.Fatalf("mint body %s: %v", out, err) |
| 1873 | } |
| 1874 | if minted.Token == "" || minted.Account != "bab" || minted.App != "titogo" { |
| 1875 | t.Fatalf("mint = %+v, want a token bound to bab/titogo", minted) |
| 1876 | } |
| 1877 | |
| 1878 | // The minted token provisions, and nothing else. |
| 1879 | u := instancesURL(fx, "bab", "titogo") |
| 1880 | if status, _, out := rawReq(t, http.MethodPost, u, minted.Token, |
| 1881 | `{"host":"acme.go.tito.io","region":"ie"}`); status != http.StatusCreated { |
| 1882 | t.Fatalf("create with the minted token = %d %s, want 201", status, out) |
| 1883 | } |
| 1884 | if status, _, out := rawReq(t, http.MethodDelete, u+"/acme.go.tito.io", minted.Token, ""); status != http.StatusOK { |
| 1885 | t.Fatalf("delete with the minted token = %d %s, want 200", status, out) |
| 1886 | } |
| 1887 | |
| 1888 | // The audit trail: three actions, each suffixed " (cli)", the create and |
| 1889 | // the delete attributed to the APP principal and the mint to the human. |
| 1890 | events, err := fx.st.Events(OpsAccountID, "titogo", 50) |
| 1891 | if err != nil { |
| 1892 | t.Fatal(err) |
| 1893 | } |
| 1894 | seen := map[string]string{} |
| 1895 | for _, e := range events { |
| 1896 | seen[e.Action] = e.Address |
| 1897 | if !strings.HasSuffix(e.Detail, " (cli)") { |
| 1898 | t.Errorf("audit %s detail = %q, want the family's \" (cli)\" suffix", e.Action, e.Detail) |
| 1899 | } |
| 1900 | } |
| 1901 | if seen["instance-token"] != "alice@keymail.dev" { |
| 1902 | t.Errorf("instance-token actor = %q, want the minting member", seen["instance-token"]) |
| 1903 | } |
| 1904 | if seen["instance-create"] != "app:bab/titogo" { |
| 1905 | t.Errorf("instance-create actor = %q, want app:bab/titogo", seen["instance-create"]) |
| 1906 | } |
| 1907 | if seen["instance-delete"] != "app:bab/titogo" { |
| 1908 | t.Errorf("instance-delete actor = %q, want app:bab/titogo", seen["instance-delete"]) |
| 1909 | } |
| 1910 | |
| 1911 | // The minted credential lists under the human who minted it — which is |
| 1912 | // what makes revoking it possible at all. |
| 1913 | toks, err := fx.st.CLITokens("alice@keymail.dev") |
| 1914 | if err != nil { |
| 1915 | t.Fatal(err) |
| 1916 | } |
| 1917 | found := false |
| 1918 | for _, tk := range toks { |
| 1919 | if tk.Kind == "app" { |
| 1920 | found = true |
| 1921 | if tk.App != "titogo" { |
| 1922 | t.Errorf("app token row = %+v, want it bound to titogo", tk) |
| 1923 | } |
| 1924 | } |
| 1925 | } |
| 1926 | if !found { |
| 1927 | t.Fatalf("the minted app token does not list under its minter: %+v", toks) |
| 1928 | } |
| 1929 | } |
| 1930 | |
| 1931 | // TestAPIInstanceRateLimit: the instance budget is its own, keyed by token, |
| 1932 | // and — the property that matters — it is consulted only AFTER the |
| 1933 | // principal gate, so a 429 can never confirm what the 404 hides. |
| 1934 | func TestAPIInstanceRateLimit(t *testing.T) { |
| 1935 | fx, token := instanceFixture(t, "code-inst-ratelimit") |
| 1936 | enableInstances(t, fx, "titogo", titoManifest()) |
| 1937 | u := instancesURL(fx, "bab", "titogo") |
| 1938 | |
| 1939 | for i := 0; i < cliInstanceBurst; i++ { |
| 1940 | body := fmt.Sprintf(`{"host":"host%d.go.tito.io","region":"ie"}`, i) |
| 1941 | if status, _, out := rawReq(t, http.MethodPost, u, token, body); status != http.StatusCreated { |
| 1942 | t.Fatalf("create %d = %d %s, want 201 inside the budget", i, status, out) |
| 1943 | } |
| 1944 | } |
| 1945 | if status, _, out := rawReq(t, http.MethodPost, u, token, |
| 1946 | `{"host":"over.go.tito.io","region":"ie"}`); status != http.StatusTooManyRequests { |
| 1947 | t.Fatalf("create past the budget = %d %s, want 429", status, out) |
| 1948 | } |
| 1949 | // A stranger's refusal never changes shape, however many times they |
| 1950 | // ask. This is the gate-before-budget ordering asserted from outside: |
| 1951 | // past the burst, a limiter consulted FIRST would start answering 429 |
| 1952 | // where the 404 belongs — a different answer on the same door, which is |
| 1953 | // a difference a prober can measure and a refusal that has stopped |
| 1954 | // being byte-identical to every other refusal here. |
| 1955 | eve := cliToken(t, fx, "eve@keymail.dev", "code-inst-rl-eve") |
| 1956 | want := canonicalNotFound(t) |
| 1957 | for i := 0; i < cliInstanceBurst+2; i++ { |
| 1958 | status, h, out := rawReq(t, http.MethodPost, u, eve, `{"host":"x.go.tito.io","region":"ie"}`) |
| 1959 | assertNotFound(t, fmt.Sprintf("non-member probe %d", i), want, status, h, out) |
| 1960 | if t.Failed() { |
| 1961 | break |
| 1962 | } |
| 1963 | } |
| 1964 | // The write budget minting uses is a different one: the member above |
| 1965 | // spent the instance budget, not this. |
| 1966 | if status, _, out := rawReq(t, http.MethodPost, fx.ts.URL+"/api/cli/apps/bab/titogo/instance-token", |
| 1967 | token, `{}`); status != http.StatusCreated { |
| 1968 | t.Fatalf("mint after spending the instance budget = %d %s, want 201", status, out) |
| 1969 | } |
| 1970 | } |
| 1971 | |
| 1972 | // TestAPIInstanceNoStore: a console running without a deployment bucket |
| 1973 | // answers 503 — the shape promote's key check has — rather than panicking |
| 1974 | // on a nil store. |
| 1975 | func TestAPIInstanceNoStore(t *testing.T) { |
| 1976 | fx, token := instanceFixture(t, "code-inst-nostore") |
| 1977 | fx.srv.cfg.Releases = nil |
| 1978 | u := instancesURL(fx, "bab", "titogo") |
| 1979 | for _, c := range []struct{ what, method, url, body string }{ |
| 1980 | {"list", http.MethodGet, u, ""}, |
| 1981 | {"create", http.MethodPost, u, `{"host":"acme.go.tito.io","region":"ie"}`}, |
| 1982 | {"get", http.MethodGet, u + "/acme.go.tito.io", ""}, |
| 1983 | {"delete", http.MethodDelete, u + "/acme.go.tito.io", ""}, |
| 1984 | } { |
| 1985 | if status, _, out := rawReq(t, c.method, c.url, token, c.body); status != http.StatusServiceUnavailable { |
| 1986 | t.Errorf("%s with no bucket = %d %s, want 503", c.what, status, out) |
| 1987 | } |
| 1988 | } |
| 1989 | } |
| 1990 | |
| 1991 | // unreadableStore is a bucket whose reads fail — distinct from a MISSING |
| 1992 | // manifest, which is an answer ("instances are not enabled") rather than a |
| 1993 | // failure. Conflating the two would tell an app its feature was switched |
| 1994 | // off every time the bucket hiccuped. |
| 1995 | type unreadableStore struct{ release.Store } |
| 1996 | |
| 1997 | func (u unreadableStore) Get(context.Context, string) ([]byte, string, error) { |
| 1998 | return nil, "", errors.New(brokenStoreErr) |
| 1999 | } |
| 2000 | |
| 2001 | // undeletableStore is a bucket whose object deletes fail. Delete's own store |
| 2002 | // call is the only one it makes — it reads no manifest (the suffix rule is |
| 2003 | // create-time policy) — so this is what a broken bucket looks like from |
| 2004 | // there. |
| 2005 | type undeletableStore struct{ release.Store } |
| 2006 | |
| 2007 | func (u undeletableStore) Delete(context.Context, string) error { |
| 2008 | return errors.New(brokenStoreErr) |
| 2009 | } |
| 2010 | |
| 2011 | // DeleteIfMatch fails the same way: the delete door deletes a record |
| 2012 | // conditionally on the etag its import check read (final review F7). |
| 2013 | func (u undeletableStore) DeleteIfMatch(context.Context, string, string) error { |
| 2014 | return errors.New(brokenStoreErr) |
| 2015 | } |
| 2016 | |
| 2017 | // TestAPIInstanceStoreFailure: a bucket that cannot be read or written is a |
| 2018 | // 502, and its error text — bucket names, request ids — never reaches the |
| 2019 | // caller. A read failure is NOT reported as "instances are not enabled". |
| 2020 | func TestAPIInstanceStoreFailure(t *testing.T) { |
| 2021 | fx, token := instanceFixture(t, "code-inst-broken") |
| 2022 | enableInstances(t, fx, "titogo", titoManifest()) |
| 2023 | u := instancesURL(fx, "bab", "titogo") |
| 2024 | |
| 2025 | fx.srv.cfg.Releases = brokenStore{fx.rel} |
| 2026 | status, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"acme.go.tito.io","region":"ie"}`) |
| 2027 | if status != http.StatusBadGateway { |
| 2028 | t.Fatalf("create against an unwritable bucket = %d %s, want 502", status, out) |
| 2029 | } |
| 2030 | if strings.Contains(out, "carlos-prod-secret-bucket") || strings.Contains(out, "ABC123") { |
| 2031 | t.Fatalf("refusal leaks the store's error text: %s", out) |
| 2032 | } |
| 2033 | |
| 2034 | fx.srv.cfg.Releases = unreadableStore{fx.rel} |
| 2035 | status, _, out = rawReq(t, http.MethodPost, u, token, `{"host":"acme.go.tito.io","region":"ie"}`) |
| 2036 | if status != http.StatusBadGateway { |
| 2037 | t.Fatalf("create against an unreadable bucket = %d %s, want 502", status, out) |
| 2038 | } |
| 2039 | if strings.Contains(out, "instances enable") { |
| 2040 | t.Fatalf("a read failure was reported as the feature being disabled: %s", out) |
| 2041 | } |
| 2042 | // The record must exist for a delete to be attempted: a host with no |
| 2043 | // record is the delete's idempotent 200 with nothing deleted (final |
| 2044 | // review F7). |
| 2045 | fx.srv.cfg.Releases = fx.rel |
| 2046 | if status, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"acme.go.tito.io","region":"ie"}`); status != http.StatusCreated { |
| 2047 | t.Fatalf("create = %d %s", status, out) |
| 2048 | } |
| 2049 | fx.srv.cfg.Releases = undeletableStore{fx.rel} |
| 2050 | status, _, out = rawReq(t, http.MethodDelete, u+"/acme.go.tito.io", token, "") |
| 2051 | if status != http.StatusBadGateway { |
| 2052 | t.Fatalf("delete against a bucket that cannot delete = %d %s, want 502", status, out) |
| 2053 | } |
| 2054 | if strings.Contains(out, "carlos-prod-secret-bucket") { |
| 2055 | t.Fatalf("delete refusal leaks the store's error text: %s", out) |
| 2056 | } |
| 2057 | } |
| 2058 | |
| 2059 | // TestAPIInstanceCreateStaticAtOwnApex pins the shape `carlos deploy -kind |
| 2060 | // static` mints for an app with no route: a kind:static record at the app's |
| 2061 | // OWN platform apex, carrying an explicit channel. |
| 2062 | // |
| 2063 | // The channel is the half worth a test of its own. An omitted channel is |
| 2064 | // defaulted to the manifest's, then to "stable" — and a fresh app's pipeline |
| 2065 | // entry is "edge", so a mint that let the default stand would write a route |
| 2066 | // that follows a channel nothing ever promotes to, with no door able to |
| 2067 | // change it afterwards (the hello.bdn incident, 2026-08-22). |
| 2068 | func TestAPIInstanceCreateStaticAtOwnApex(t *testing.T) { |
| 2069 | fx, member := instanceFixture(t, "code-inst-static-apex") |
| 2070 | fx.srv.cfg.AppsDomain = "oncarlos.test" |
| 2071 | if status, _, out := rawReq(t, http.MethodPut, enableURL(fx, "bab", "titogo"), member, `{}`); status != http.StatusOK { |
| 2072 | t.Fatalf("enable = %d %s", status, out) |
| 2073 | } |
| 2074 | host := "titogo." + fx.sqid + ".oncarlos.test" |
| 2075 | |
| 2076 | status, _, out := rawReq(t, http.MethodPost, instancesURL(fx, "bab", "titogo"), member, |
| 2077 | `{"host":"`+host+`","kind":"static","channel":"edge"}`) |
| 2078 | if status != http.StatusCreated { |
| 2079 | t.Fatalf("static apex create = %d %s, want 201", status, out) |
| 2080 | } |
| 2081 | var rec provision.Record |
| 2082 | if err := json.Unmarshal([]byte(out), &rec); err != nil { |
| 2083 | t.Fatalf("create body %s: %v", out, err) |
| 2084 | } |
| 2085 | if rec.Kind != "static" { |
| 2086 | t.Fatalf("echoed Kind = %q, want static", rec.Kind) |
| 2087 | } |
| 2088 | if rec.Channel != "edge" { |
| 2089 | t.Fatalf("echoed Channel = %q, want the requested edge — an omitted channel falls to stable", rec.Channel) |
| 2090 | } |
| 2091 | |
| 2092 | // The bucket copy is what the reconcile pass mints the route from, so it |
| 2093 | // is the one that has to carry both. |
| 2094 | stored, err := provision.GetRecord(context.Background(), fx.rel, fx.sqid, "titogo", host) |
| 2095 | if err != nil { |
| 2096 | t.Fatal(err) |
| 2097 | } |
| 2098 | if stored.Kind != "static" || stored.Channel != "edge" { |
| 2099 | t.Fatalf("stored record = %+v, want kind static on channel edge", stored) |
| 2100 | } |
| 2101 | // A static record with no upstream and no unit is the shape ValidateShape |
| 2102 | // requires; anything the door let through that carried one would wedge |
| 2103 | // the host. |
| 2104 | if len(stored.Addrs) != 0 || stored.Unit != "" { |
| 2105 | t.Fatalf("stored record = %+v, want no upstream and no unit", stored) |
| 2106 | } |
| 2107 | } |
| 2108 | |
| 2109 | // TestAPIInstanceListServedBy (issue #337): each instance carries where it |
| 2110 | // is OBSERVED to be served — every box whose control/edges/<box>.json names |
| 2111 | // the host, with that box's AWS region — as an additive `served_by` list on |
| 2112 | // both the list and the get. The record's own `region` (the REQUESTED |
| 2113 | // placement code) is untouched. A host no edges file names carries no key |
| 2114 | // at all; a garbage or unversioned object under the prefix is skipped, never |
| 2115 | // fatal; an everywhere-class host named by two boxes lists both, box-sorted. |
| 2116 | func TestAPIInstanceListServedBy(t *testing.T) { |
| 2117 | fx, token := instanceFixture(t, "code-inst-served") |
| 2118 | enableInstances(t, fx, "titogo", titoManifest()) |
| 2119 | u := instancesURL(fx, "bab", "titogo") |
| 2120 | for _, h := range []string{"acme.go.tito.io", "beta.go.tito.io", "gamma.go.tito.io"} { |
| 2121 | if code, _, out := rawReq(t, http.MethodPost, u, token, |
| 2122 | fmt.Sprintf(`{"host":%q,"region":"ie"}`, h)); code != http.StatusCreated { |
| 2123 | t.Fatalf("create %s = %d %s", h, code, out) |
| 2124 | } |
| 2125 | } |
| 2126 | type served struct { |
| 2127 | Box string `json:"box"` |
| 2128 | Region string `json:"region"` |
| 2129 | } |
| 2130 | type row struct { |
| 2131 | Host string `json:"host"` |
| 2132 | Region string `json:"region"` |
| 2133 | ServedBy json.RawMessage `json:"served_by"` |
| 2134 | } |
| 2135 | decodeList := func(out string) map[string]row { |
| 2136 | var list struct { |
| 2137 | Instances []row `json:"instances"` |
| 2138 | } |
| 2139 | if err := json.Unmarshal([]byte(out), &list); err != nil { |
| 2140 | t.Fatalf("list body %s: %v", out, err) |
| 2141 | } |
| 2142 | got := map[string]row{} |
| 2143 | for _, r := range list.Instances { |
| 2144 | got[r.Host] = r |
| 2145 | } |
| 2146 | return got |
| 2147 | } |
| 2148 | |
| 2149 | // No edges file at all: the list serves, no record carries the key. |
| 2150 | code, _, out := rawReq(t, http.MethodGet, u, token, "") |
| 2151 | if code != http.StatusOK { |
| 2152 | t.Fatalf("list with no edges files = %d %s, want 200", code, out) |
| 2153 | } |
| 2154 | for h, r := range decodeList(out) { |
| 2155 | if r.ServedBy != nil { |
| 2156 | t.Errorf("%s served_by = %s before any edges file, want the key absent", h, r.ServedBy) |
| 2157 | } |
| 2158 | } |
| 2159 | |
| 2160 | put := func(key, body string) { |
| 2161 | t.Helper() |
| 2162 | if _, err := fx.rel.Put(context.Background(), key, []byte(body), ""); err != nil { |
| 2163 | t.Fatal(err) |
| 2164 | } |
| 2165 | } |
| 2166 | put("control/edges/zed-box.json", `{"v":1,"box":"zed-box","eip":"1.2.3.4","region":"eu-west-2","generation":"g","hosts":{"acme.go.tito.io":{"role":"home"}}}`) |
| 2167 | put("control/edges/annie.json", `{"v":1,"box":"annie","eip":"5.6.7.8","region":"eu-west-1","generation":"g","hosts":{"acme.go.tito.io":{"role":"home"},"gamma.go.tito.io":{"role":"home"}}}`) |
| 2168 | put("control/edges/broken.json", `{not json`) |
| 2169 | put("control/edges/old.json", `{"box":"old","hosts":{"beta.go.tito.io":{"role":"home"}}}`) |
| 2170 | put("control/edges/nested/x.json", `{"v":1,"box":"nested","hosts":{"beta.go.tito.io":{"role":"home"}}}`) |
| 2171 | |
| 2172 | code, _, out = rawReq(t, http.MethodGet, u, token, "") |
| 2173 | if code != http.StatusOK { |
| 2174 | t.Fatalf("list with seeded edges files = %d %s, want 200", code, out) |
| 2175 | } |
| 2176 | got := decodeList(out) |
| 2177 | want := map[string][]served{ |
| 2178 | "acme.go.tito.io": {{"annie", "eu-west-1"}, {"zed-box", "eu-west-2"}}, |
| 2179 | "gamma.go.tito.io": {{"annie", "eu-west-1"}}, |
| 2180 | } |
| 2181 | for h, w := range want { |
| 2182 | var s []served |
| 2183 | if err := json.Unmarshal(got[h].ServedBy, &s); err != nil { |
| 2184 | t.Fatalf("%s served_by %s: %v", h, got[h].ServedBy, err) |
| 2185 | } |
| 2186 | if fmt.Sprint(s) != fmt.Sprint(w) { |
| 2187 | t.Errorf("%s served_by = %v, want %v", h, s, w) |
| 2188 | } |
| 2189 | if got[h].Region != "ie" { |
| 2190 | t.Errorf("%s region = %q, want the requested placement code ie untouched", h, got[h].Region) |
| 2191 | } |
| 2192 | } |
| 2193 | if got["beta.go.tito.io"].ServedBy != nil { |
| 2194 | t.Errorf("beta served_by = %s, want absent — only an unversioned and a mis-keyed file name it", got["beta.go.tito.io"].ServedBy) |
| 2195 | } |
| 2196 | |
| 2197 | // The single-record get carries the same join. |
| 2198 | code, _, out = rawReq(t, http.MethodGet, u+"/acme.go.tito.io", token, "") |
| 2199 | if code != http.StatusOK { |
| 2200 | t.Fatalf("get = %d %s", code, out) |
| 2201 | } |
| 2202 | var one row |
| 2203 | if err := json.Unmarshal([]byte(out), &one); err != nil { |
| 2204 | t.Fatalf("get body %s: %v", out, err) |
| 2205 | } |
| 2206 | var s []served |
| 2207 | if err := json.Unmarshal(one.ServedBy, &s); err != nil || fmt.Sprint(s) != fmt.Sprint(want["acme.go.tito.io"]) { |
| 2208 | t.Errorf("get served_by = %s (%v), want %v", one.ServedBy, err, want["acme.go.tito.io"]) |
| 2209 | } |
| 2210 | code, _, out = rawReq(t, http.MethodGet, u+"/beta.go.tito.io", token, "") |
| 2211 | if code != http.StatusOK || strings.Contains(out, "served_by") { |
| 2212 | t.Errorf("get beta = %d %s, want 200 with no served_by key", code, out) |
| 2213 | } |
| 2214 | } |
| 2215 | |
| 2216 | // TestAPIInstanceCreateArmedPlacementGate (issue #329): once any box |
| 2217 | // publishes a placement code in its edges file, a create naming a region no |
| 2218 | // armed box serves is refused with the real choices — a record nothing will |
| 2219 | // ever mint is a dark instance, worse than a 400. Dormant before that |
| 2220 | // (legacy fleet: behaviour unchanged); a placement without an EIP is not a |
| 2221 | // landing target; the unconstrained "" region always passes. |
| 2222 | func TestAPIInstanceCreateArmedPlacementGate(t *testing.T) { |
| 2223 | fx, token := instanceFixture(t, "code-inst-armed") |
| 2224 | m := titoManifest() |
| 2225 | m.Regions = nil // the manifest allow-list is a separate rail; keep it out |
| 2226 | enableInstances(t, fx, "titogo", m) |
| 2227 | u := instancesURL(fx, "bab", "titogo") |
| 2228 | put := func(key, body string) { |
| 2229 | t.Helper() |
| 2230 | if _, err := fx.rel.Put(context.Background(), key, []byte(body), ""); err != nil { |
| 2231 | t.Fatal(err) |
| 2232 | } |
| 2233 | } |
| 2234 | |
| 2235 | // DORMANT: no edges file carries a placement — any shape-valid region |
| 2236 | // is accepted, exactly as before the gate existed. |
| 2237 | put("control/edges/legacy.json", `{"v":1,"box":"legacy","eip":"9.9.9.9","hosts":{}}`) |
| 2238 | if code, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"acme.go.tito.io","region":"zz-typo"}`); code != http.StatusCreated { |
| 2239 | t.Fatalf("dormant create = %d %s, want 201", code, out) |
| 2240 | } |
| 2241 | |
| 2242 | // ARMED: uk has an EIP, se does not, legacy still publishes none. |
| 2243 | put("control/edges/uk-box.json", `{"v":1,"box":"uk-box","eip":"1.2.3.4","region":"eu-west-2","placement":"uk","hosts":{}}`) |
| 2244 | put("control/edges/se-box.json", `{"v":1,"box":"se-box","eip":"","region":"eu-north-1","placement":"se","hosts":{}}`) |
| 2245 | |
| 2246 | if code, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"beta.go.tito.io","region":"uk"}`); code != http.StatusCreated { |
| 2247 | t.Fatalf("create region uk (armed) = %d %s, want 201", code, out) |
| 2248 | } |
| 2249 | if code, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"gamma.go.tito.io"}`); code != http.StatusCreated { |
| 2250 | t.Fatalf("create with unconstrained region = %d %s, want 201", code, out) |
| 2251 | } |
| 2252 | code, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"delta.go.tito.io","region":"zz-typo"}`) |
| 2253 | if code != http.StatusBadRequest { |
| 2254 | t.Fatalf("create region zz-typo (armed) = %d %s, want 400", code, out) |
| 2255 | } |
| 2256 | if !strings.Contains(out, "armed: uk") { |
| 2257 | t.Fatalf("refusal %q does not name the armed set", out) |
| 2258 | } |
| 2259 | // se published a placement but no EIP: not a landing target. |
| 2260 | if code, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"eps.go.tito.io","region":"se"}`); code != http.StatusBadRequest { |
| 2261 | t.Fatalf("create region se (no EIP) = %d %s, want 400", code, out) |
| 2262 | } |
| 2263 | } |
| 2264 | |
| 2265 | // A static site is HA by default. Static is the one serving class every |
| 2266 | // platform box materializes unconditionally (provision.everywhereClass): |
| 2267 | // no unit, no process, content read from the bucket channel pointer, so a |
| 2268 | // far edge serves it with no home box and no backhaul. The content half of |
| 2269 | // that was already true and measured — every flagship edge served every |
| 2270 | // static host with a valid cert. What was missing was DNS: without an |
| 2271 | // opt-in the host resolves to one edge, so four healthy boxes held the site |
| 2272 | // and every visitor went to one of them. |
| 2273 | // |
| 2274 | // The opt-in is inert until a deployment sets CARLOS_STEERING_ZONE_ID, so a |
| 2275 | // single-box deployment states the intent harmlessly and becomes HA the day |
| 2276 | // it arms a second edge. `carlos steering --app x --host y off` is the |
| 2277 | // escape hatch, and nothing here ever converges an existing record. |
| 2278 | // shipStaticRelease ships a STATIC release: a "site" artifact and no build |
| 2279 | // for any box's platform. That absence is the whole bug — a box stages |
| 2280 | // nothing, writes no live symlink, and every wake of an instance route on |
| 2281 | // this channel fails forever (skills#29). |
| 2282 | func shipStaticRelease(t *testing.T, fx *consoleFixture, app, version string) { |
| 2283 | t.Helper() |
| 2284 | m := release.Manifest{ |
| 2285 | Schema: 1, App: app, Version: version, Kind: "static", |
| 2286 | BuiltAt: "2026-09-01T23:16:21Z", |
| 2287 | Artifacts: map[string]release.Artifact{"site": {File: "site.tar"}}, |
| 2288 | } |
| 2289 | if err := release.Ship(context.Background(), fx.rel, release.OpsAccount, m, |
| 2290 | map[string][]byte{"site": []byte("<h1>" + version + "</h1>")}); err != nil { |
| 2291 | t.Fatal(err) |
| 2292 | } |
| 2293 | } |
| 2294 | |
| 2295 | // skills#29, at the door that could have prevented it. A member shipped a |
| 2296 | // static site, promoted it, and then declared an INSTANCE for it — the kind |
| 2297 | // this door defaults to. The record was created happily and the route it |
| 2298 | // provisioned could never wake, because a static release carries no binary to |
| 2299 | // run. What they saw was "nothing is promoted for this instance's channel" |
| 2300 | // against a `carlos channels` showing the promote plainly there. |
| 2301 | // |
| 2302 | // The refusal has to do more than refuse: an agent that reads it must be able |
| 2303 | // to act on it without another round of discovery, so it names both ways out. |
| 2304 | func TestAPIInstanceCreateRefusesAnInstanceOnAStaticHead(t *testing.T) { |
| 2305 | fx, member := instanceFixture(t, "code-inst-static-head") |
| 2306 | fx.srv.cfg.AppsDomain = "oncarlos.test" |
| 2307 | if status, _, out := rawReq(t, http.MethodPut, enableURL(fx, "bab", "titogo"), member, `{}`); status != http.StatusOK { |
| 2308 | t.Fatalf("enable = %d %s", status, out) |
| 2309 | } |
| 2310 | shipStaticRelease(t, fx, "titogo", "0.1.0") |
| 2311 | seedChannel(t, fx, "titogo", "edge", "0.1.0") |
| 2312 | host := "titogo." + fx.sqid + ".oncarlos.test" |
| 2313 | |
| 2314 | // No "kind" at all — the CLI's default, and exactly the request that was |
| 2315 | // made in the reported session. |
| 2316 | status, _, out := rawReq(t, http.MethodPost, instancesURL(fx, "bab", "titogo"), member, |
| 2317 | `{"host":"`+host+`","channel":"edge"}`) |
| 2318 | if status != http.StatusUnprocessableEntity { |
| 2319 | t.Fatalf("create = %d %s, want 422 — an instance on a static head can never wake", status, out) |
| 2320 | } |
| 2321 | for _, want := range []string{"static release", "0.1.0", "--kind static", "carlos deploy"} { |
| 2322 | if !strings.Contains(out, want) { |
| 2323 | t.Errorf("refusal %q does not name %q — the member is refused without being told the way out", out, want) |
| 2324 | } |
| 2325 | } |
| 2326 | if _, err := provision.GetRecord(context.Background(), fx.rel, fx.sqid, "titogo", host); err == nil { |
| 2327 | t.Error("the refused create wrote a record anyway") |
| 2328 | } |
| 2329 | |
| 2330 | // The way out the refusal names actually works, in the same fixture. |
| 2331 | status, _, out = rawReq(t, http.MethodPost, instancesURL(fx, "bab", "titogo"), member, |
| 2332 | `{"host":"`+host+`","channel":"edge","kind":"static"}`) |
| 2333 | if status != http.StatusCreated { |
| 2334 | t.Fatalf("static create = %d %s, want 201 — the refusal named a door that is shut", status, out) |
| 2335 | } |
| 2336 | } |
| 2337 | |
| 2338 | // The guard is about the SHAPE mismatch, not about static apps: an app whose |
| 2339 | // channel head is an ordinary binary release still gets an ordinary instance |
| 2340 | // with no kind named, which is most creates on the platform. |
| 2341 | func TestAPIInstanceCreateAllowsAnInstanceOnABinaryHead(t *testing.T) { |
| 2342 | fx, member := instanceFixture(t, "code-inst-binary-head") |
| 2343 | fx.srv.cfg.AppsDomain = "oncarlos.test" |
| 2344 | if status, _, out := rawReq(t, http.MethodPut, enableURL(fx, "bab", "titogo"), member, `{}`); status != http.StatusOK { |
| 2345 | t.Fatalf("enable = %d %s", status, out) |
| 2346 | } |
| 2347 | shipVersion(t, fx, "titogo", "v1") |
| 2348 | seedChannel(t, fx, "titogo", "edge", "v1") |
| 2349 | host := "titogo." + fx.sqid + ".oncarlos.test" |
| 2350 | |
| 2351 | status, _, out := rawReq(t, http.MethodPost, instancesURL(fx, "bab", "titogo"), member, |
| 2352 | `{"host":"`+host+`","channel":"edge"}`) |
| 2353 | if status != http.StatusCreated { |
| 2354 | t.Fatalf("create = %d %s, want 201", status, out) |
| 2355 | } |
| 2356 | } |
| 2357 | |
| 2358 | // A brand-new app with nothing promoted anywhere is the FIRST-instance case, |
| 2359 | // and it must stay exactly as it was: nothing to read, nothing to refuse. |
| 2360 | func TestAPIInstanceCreateUnaffectedWithNothingPromoted(t *testing.T) { |
| 2361 | fx, member := instanceFixture(t, "code-inst-no-head") |
| 2362 | fx.srv.cfg.AppsDomain = "oncarlos.test" |
| 2363 | if status, _, out := rawReq(t, http.MethodPut, enableURL(fx, "bab", "titogo"), member, `{}`); status != http.StatusOK { |
| 2364 | t.Fatalf("enable = %d %s", status, out) |
| 2365 | } |
| 2366 | host := "titogo." + fx.sqid + ".oncarlos.test" |
| 2367 | |
| 2368 | status, _, out := rawReq(t, http.MethodPost, instancesURL(fx, "bab", "titogo"), member, |
| 2369 | `{"host":"`+host+`","channel":"edge"}`) |
| 2370 | if status != http.StatusCreated { |
| 2371 | t.Fatalf("create = %d %s, want 201 — the first instance of an app that has shipped nothing", status, out) |
| 2372 | } |
| 2373 | } |
| 2374 | |
| 2375 | func TestAPIInstanceCreateStaticSteersByDefault(t *testing.T) { |
| 2376 | fx, member := instanceFixture(t, "code-inst-static-steer") |
| 2377 | fx.srv.cfg.AppsDomain = "oncarlos.test" |
| 2378 | if status, _, out := rawReq(t, http.MethodPut, enableURL(fx, "bab", "titogo"), member, `{}`); status != http.StatusOK { |
| 2379 | t.Fatalf("enable = %d %s", status, out) |
| 2380 | } |
| 2381 | host := "titogo." + fx.sqid + ".oncarlos.test" |
| 2382 | |
| 2383 | status, _, out := rawReq(t, http.MethodPost, instancesURL(fx, "bab", "titogo"), member, |
| 2384 | `{"host":"`+host+`","kind":"static","channel":"edge"}`) |
| 2385 | if status != http.StatusCreated { |
| 2386 | t.Fatalf("static apex create = %d %s, want 201", status, out) |
| 2387 | } |
| 2388 | stored, err := provision.GetRecord(context.Background(), fx.rel, fx.sqid, "titogo", host) |
| 2389 | if err != nil { |
| 2390 | t.Fatal(err) |
| 2391 | } |
| 2392 | if stored.Steering != "latency" { |
| 2393 | t.Fatalf("stored Steering = %q, want latency — a static platform host is HA by default", stored.Steering) |
| 2394 | } |
| 2395 | // Whatever the door writes has to survive ValidateShape, or the record |
| 2396 | // it just stored is one no box will ever materialize. |
| 2397 | if err := provision.ValidateShape(*stored); err != nil { |
| 2398 | t.Fatalf("the defaulted record does not validate: %v", err) |
| 2399 | } |
| 2400 | } |
| 2401 | |
| 2402 | // The default is scoped to the platform namespace, and deliberately so. The |
| 2403 | // steering converger only ever writes latency records for hosts under the |
| 2404 | // apps domain (internal/steering's steeredHosts), so defaulting a custom |
| 2405 | // domain's record to "latency" would write an opt-in that does nothing — |
| 2406 | // a claim in the record that no pass will honour, which is #306's shape. |
| 2407 | // A custom domain reaches the edges by its owner's own DNS, and the console |
| 2408 | // already tells them to CNAME the steering hostname. |
| 2409 | func TestAPIInstanceCreateStaticCustomDomainDoesNotSteer(t *testing.T) { |
| 2410 | fx, member := instanceFixture(t, "code-inst-static-custom") |
| 2411 | fx.srv.cfg.AppsDomain = "oncarlos.test" |
| 2412 | // The operator-written manifest, pinned to the app's OWN domain rather |
| 2413 | // than the account's platform subdomain — the custom-domain shape. |
| 2414 | enableInstances(t, fx, "titogo", titoManifest()) |
| 2415 | host := "site.go.tito.io" |
| 2416 | |
| 2417 | status, _, out := rawReq(t, http.MethodPost, instancesURL(fx, "bab", "titogo"), member, |
| 2418 | `{"host":"`+host+`","region":"ie","kind":"static","channel":"edge"}`) |
| 2419 | if status != http.StatusCreated { |
| 2420 | t.Fatalf("custom-domain static create = %d %s, want 201", status, out) |
| 2421 | } |
| 2422 | stored, err := provision.GetRecord(context.Background(), fx.rel, fx.sqid, "titogo", host) |
| 2423 | if err != nil { |
| 2424 | t.Fatal(err) |
| 2425 | } |
| 2426 | if stored.Steering != "" { |
| 2427 | t.Fatalf("stored Steering = %q, want empty — the converger never steers a host outside the apps domain", stored.Steering) |
| 2428 | } |
| 2429 | } |
| 2430 | |
| 2431 | // Only static. A home-boxed instance route reaches a far edge by backhaul, |
| 2432 | // which is a different trade with a different cost, and widening this |
| 2433 | // default to every kind would make that trade silently for every app on the |
| 2434 | // deployment. `carlos steering` remains the door for those, per host. |
| 2435 | func TestAPIInstanceCreateNonStaticDoesNotSteer(t *testing.T) { |
| 2436 | fx, member := instanceFixture(t, "code-inst-nonstatic-steer") |
| 2437 | fx.srv.cfg.AppsDomain = "oncarlos.test" |
| 2438 | if status, _, out := rawReq(t, http.MethodPut, enableURL(fx, "bab", "titogo"), member, `{}`); status != http.StatusOK { |
| 2439 | t.Fatalf("enable = %d %s", status, out) |
| 2440 | } |
| 2441 | host := "titogo." + fx.sqid + ".oncarlos.test" |
| 2442 | |
| 2443 | status, _, out := rawReq(t, http.MethodPost, instancesURL(fx, "bab", "titogo"), member, |
| 2444 | `{"host":"`+host+`","channel":"edge"}`) // no kind: the instance default |
| 2445 | if status != http.StatusCreated { |
| 2446 | t.Fatalf("instance create = %d %s, want 201", status, out) |
| 2447 | } |
| 2448 | stored, err := provision.GetRecord(context.Background(), fx.rel, fx.sqid, "titogo", host) |
| 2449 | if err != nil { |
| 2450 | t.Fatal(err) |
| 2451 | } |
| 2452 | if stored.Steering != "" { |
| 2453 | t.Fatalf("stored Steering = %q, want empty — only static is steered by default", stored.Steering) |
| 2454 | } |
| 2455 | } |
| 2456 | |
| 2457 | // v1's pool rail applies to the default exactly as it applies to the PATCH |
| 2458 | // door, which answers 409 for the same request. A default that wrote what |
| 2459 | // the update door refuses would put an opt-in in the record that nobody |
| 2460 | // could have asked for through the front door — and that the converger |
| 2461 | // skips regardless, since a pool's own edges are not the deployment's. |
| 2462 | func TestAPIInstanceCreateStaticInAPoolDoesNotSteer(t *testing.T) { |
| 2463 | fx, member := instanceFixture(t, "code-inst-static-pooled") |
| 2464 | fx.srv.cfg.AppsDomain = "oncarlos.test" |
| 2465 | // An operator-written manifest can pin BOTH the account's platform |
| 2466 | // subdomain and a pool; the member-facing enable door cannot. |
| 2467 | enableInstances(t, fx, "titogo", provision.Manifest{ |
| 2468 | Schema: 1, App: "titogo", Domain: fx.sqid + ".oncarlos.test", |
| 2469 | Pool: "tito", Channel: "stable", Seed: []string{"seed"}, |
| 2470 | }) |
| 2471 | host := "titogo." + fx.sqid + ".oncarlos.test" |
| 2472 | |
| 2473 | status, _, out := rawReq(t, http.MethodPost, instancesURL(fx, "bab", "titogo"), member, |
| 2474 | `{"host":"`+host+`","kind":"static","channel":"edge"}`) |
| 2475 | if status != http.StatusCreated { |
| 2476 | t.Fatalf("pooled static create = %d %s, want 201", status, out) |
| 2477 | } |
| 2478 | stored, err := provision.GetRecord(context.Background(), fx.rel, fx.sqid, "titogo", host) |
| 2479 | if err != nil { |
| 2480 | t.Fatal(err) |
| 2481 | } |
| 2482 | if stored.Steering != "" { |
| 2483 | t.Fatalf("stored Steering = %q, want empty — steering requires the shared pool", stored.Steering) |
| 2484 | } |
| 2485 | } |
| 2486 | |
| 2487 | // TestAPIInstanceUpdateGateRoundTrip: the PATCH takes a PLAINTEXT password, |
| 2488 | // hashes it here (spec §5.2) and answers with a record carrying "gated":true |
| 2489 | // and no "gate" key at all — the bcrypt hash is written to the bucket and |
| 2490 | // never leaves the console again, not on the write reply, not on the list, |
| 2491 | // not on the GET. Clearing is `{"gate":""}`, which opens the host and flips |
| 2492 | // the same flag back. |
| 2493 | func TestAPIInstanceUpdateGateRoundTrip(t *testing.T) { |
| 2494 | fx, token := instanceFixture(t, "code-inst-gate-round") |
| 2495 | letStateUpstreams(t, fx, "alice@keymail.dev") |
| 2496 | enableInstances(t, fx, "titogo", sharedPoolManifest()) |
| 2497 | u := instancesURL(fx, "bab", "titogo") |
| 2498 | if status, _, out := rawReq(t, http.MethodPost, u, token, |
| 2499 | `{"host":"gate.go.tito.io","region":"ie","addrs":["127.0.0.1:7001"]}`); status != http.StatusCreated { |
| 2500 | t.Fatalf("create = %d %s", status, out) |
| 2501 | } |
| 2502 | pu := updateURL(fx, "bab", "titogo", "gate.go.tito.io") |
| 2503 | |
| 2504 | status, _, out := rawReq(t, http.MethodPatch, pu, token, `{"gate":"correct horse"}`) |
| 2505 | if status != http.StatusOK { |
| 2506 | t.Fatalf("patch gate = %d %s, want 200", status, out) |
| 2507 | } |
| 2508 | var echoed map[string]any |
| 2509 | if err := json.Unmarshal([]byte(out), &echoed); err != nil { |
| 2510 | t.Fatal(err) |
| 2511 | } |
| 2512 | if _, leaked := echoed["gate"]; leaked { |
| 2513 | t.Fatalf("patch reply carries the gate hash: %s", out) |
| 2514 | } |
| 2515 | if echoed["gated"] != true { |
| 2516 | t.Fatalf("patch reply gated = %v, want true", echoed["gated"]) |
| 2517 | } |
| 2518 | stored, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "gate.go.tito.io") |
| 2519 | if err != nil { |
| 2520 | t.Fatal(err) |
| 2521 | } |
| 2522 | if !strings.HasPrefix(stored.Gate, "$2a$10$") { |
| 2523 | t.Fatalf("stored gate = %q, want a bcrypt hash", stored.Gate) |
| 2524 | } |
| 2525 | if strings.Contains(stored.Gate, "correct horse") { |
| 2526 | t.Fatal("stored gate carries the plaintext") |
| 2527 | } |
| 2528 | if detail := lastEventDetail(t, fx, "titogo"); detail != "updated instance gate.go.tito.io (gate) (cli)" { |
| 2529 | t.Fatalf("audit detail = %q", detail) |
| 2530 | } |
| 2531 | |
| 2532 | // The list and the GET shadow the hash the same way. |
| 2533 | for _, path := range []string{u, pu} { |
| 2534 | _, _, body := rawReq(t, http.MethodGet, path, token, "") |
| 2535 | if strings.Contains(body, "$2a$") { |
| 2536 | t.Fatalf("GET %s leaks the hash: %s", path, body) |
| 2537 | } |
| 2538 | if !strings.Contains(body, `"gated":true`) { |
| 2539 | t.Fatalf("GET %s lacks gated:true: %s", path, body) |
| 2540 | } |
| 2541 | } |
| 2542 | |
| 2543 | status, _, out = rawReq(t, http.MethodPatch, pu, token, `{"gate":""}`) |
| 2544 | if status != http.StatusOK { |
| 2545 | t.Fatalf("patch clear = %d %s", status, out) |
| 2546 | } |
| 2547 | stored, _ = provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "gate.go.tito.io") |
| 2548 | if stored.Gate != "" { |
| 2549 | t.Fatalf("stored gate after clear = %q", stored.Gate) |
| 2550 | } |
| 2551 | _, _, body := rawReq(t, http.MethodGet, pu, token, "") |
| 2552 | if !strings.Contains(body, `"gated":false`) { |
| 2553 | t.Fatalf("GET after clear: %s", body) |
| 2554 | } |
| 2555 | } |
| 2556 | |
| 2557 | // TestAPIInstanceUpdateGateRefusals: gate.CheckPassword's two bounds are this |
| 2558 | // door's own 400, in the caller's terms — and the refusal never echoes the |
| 2559 | // password back, because a 400 body is exactly where a rejected password |
| 2560 | // would otherwise land in a log or a terminal's scrollback. |
| 2561 | func TestAPIInstanceUpdateGateRefusals(t *testing.T) { |
| 2562 | fx, token := instanceFixture(t, "code-inst-gate-refuse") |
| 2563 | letStateUpstreams(t, fx, "alice@keymail.dev") |
| 2564 | enableInstances(t, fx, "titogo", sharedPoolManifest()) |
| 2565 | u := instancesURL(fx, "bab", "titogo") |
| 2566 | if status, _, out := rawReq(t, http.MethodPost, u, token, |
| 2567 | `{"host":"gr.go.tito.io","region":"ie","addrs":["127.0.0.1:7001"]}`); status != http.StatusCreated { |
| 2568 | t.Fatalf("create = %d %s", status, out) |
| 2569 | } |
| 2570 | pu := updateURL(fx, "bab", "titogo", "gr.go.tito.io") |
| 2571 | for _, body := range []string{`{"gate":"short7!"}`, `{"gate":"` + strings.Repeat("x", 73) + `"}`} { |
| 2572 | status, _, out := rawReq(t, http.MethodPatch, pu, token, body) |
| 2573 | if status != http.StatusBadRequest || !strings.Contains(out, "password") { |
| 2574 | t.Fatalf("%s -> %d %s, want 400 naming the password rule", body, status, out) |
| 2575 | } |
| 2576 | if strings.Contains(out, "short7!") || strings.Contains(out, "xxxxxxxx") { |
| 2577 | t.Fatalf("refusal echoes the password: %s", out) |
| 2578 | } |
| 2579 | } |
| 2580 | // The empty-body rail names gate among the patchable fields. |
| 2581 | if _, _, out := rawReq(t, http.MethodPatch, pu, token, `{}`); !strings.Contains(out, "gate") { |
| 2582 | t.Fatalf("empty-body refusal should name gate: %s", out) |
| 2583 | } |
| 2584 | } |
| 2585 | |
| 2586 | // TestAPIInstanceListDBBytes: db_bytes joins from a FRESH box's |
| 2587 | // Metrics.RouteDBBytes, is absent (not 0) for a host no box sized, and is |
| 2588 | // dropped for a host whose only reporting box has gone stale — the same |
| 2589 | // rules as staged_version. |
| 2590 | func TestAPIInstanceListDBBytes(t *testing.T) { |
| 2591 | fx, token := instanceFixture(t, "code-inst-dbbytes") |
| 2592 | enableInstances(t, fx, "titogo", titoManifest()) |
| 2593 | u := instancesURL(fx, "bab", "titogo") |
| 2594 | for _, h := range []string{"acme.go.tito.io", "beta.go.tito.io", "gamma.go.tito.io"} { |
| 2595 | if code, _, out := rawReq(t, http.MethodPost, u, token, |
| 2596 | fmt.Sprintf(`{"host":%q,"region":"ie"}`, h)); code != http.StatusCreated { |
| 2597 | t.Fatalf("create %s = %d %s", h, code, out) |
| 2598 | } |
| 2599 | } |
| 2600 | fresh := status.CollectBox(nil, status.Probes{MachineID: "box1"}, time.Now()) |
| 2601 | fresh.Metrics = &status.BoxMetrics{RouteDBBytes: map[string]int64{"acme.go.tito.io": 123456}} |
| 2602 | if err := status.WriteBox(context.Background(), fx.rel, fresh); err != nil { |
| 2603 | t.Fatal(err) |
| 2604 | } |
| 2605 | stale := status.CollectBox(nil, status.Probes{MachineID: "box2"}, time.Now().Add(-time.Hour)) |
| 2606 | stale.Metrics = &status.BoxMetrics{RouteDBBytes: map[string]int64{"gamma.go.tito.io": 999}} |
| 2607 | if err := status.WriteBox(context.Background(), fx.rel, stale); err != nil { |
| 2608 | t.Fatal(err) |
| 2609 | } |
| 2610 | |
| 2611 | code, _, out := rawReq(t, http.MethodGet, u, token, "") |
| 2612 | if code != http.StatusOK { |
| 2613 | t.Fatalf("list = %d %s", code, out) |
| 2614 | } |
| 2615 | var list struct { |
| 2616 | Instances []map[string]any `json:"instances"` |
| 2617 | } |
| 2618 | if err := json.Unmarshal([]byte(out), &list); err != nil { |
| 2619 | t.Fatalf("list body %s: %v", out, err) |
| 2620 | } |
| 2621 | got := map[string]map[string]any{} |
| 2622 | for _, r := range list.Instances { |
| 2623 | got[r["host"].(string)] = r |
| 2624 | } |
| 2625 | if v := got["acme.go.tito.io"]["db_bytes"]; v != float64(123456) { |
| 2626 | t.Errorf("acme db_bytes = %v, want 123456", v) |
| 2627 | } |
| 2628 | for _, h := range []string{"beta.go.tito.io", "gamma.go.tito.io"} { |
| 2629 | if v, ok := got[h]["db_bytes"]; ok { |
| 2630 | t.Errorf("%s db_bytes = %v, want the key absent", h, v) |
| 2631 | } |
| 2632 | } |
| 2633 | } |
| 2634 | |
| 2635 | // TestAPIInstanceCreateReplicaFields pins the create door's replica rails |
| 2636 | // (#162): role and placement travel together or not at all; a valid replica |
| 2637 | // create lands role on the record, in the bucket, canonicalized; the home's |
| 2638 | // placement set is deduped and sorted by the door, not by the writer's |
| 2639 | // typing; and a record carrying none of the fields writes no replica keys — |
| 2640 | // the additivity rule, at the door. |
| 2641 | func TestAPIInstanceCreateReplicaFields(t *testing.T) { |
| 2642 | fx, token := instanceFixture(t, "code-inst-replica") |
| 2643 | m := titoManifest() |
| 2644 | m.Regions = []string{"ie", "uk"} // the replica placement must be armed |
| 2645 | enableInstances(t, fx, "titogo", m) |
| 2646 | u := instancesURL(fx, "bab", "titogo") |
| 2647 | |
| 2648 | // A replica create: role and region together. |
| 2649 | status, _, out := rawReq(t, http.MethodPost, u, token, |
| 2650 | `{"host":"acme-uk.go.tito.io","region":"uk","role":"replica"}`) |
| 2651 | if status != http.StatusCreated { |
| 2652 | t.Fatalf("replica create = %d %s, want 201", status, out) |
| 2653 | } |
| 2654 | stored, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "acme-uk.go.tito.io") |
| 2655 | if err != nil { |
| 2656 | t.Fatal(err) |
| 2657 | } |
| 2658 | if stored.Role != "replica" || stored.Region != "uk" { |
| 2659 | t.Fatalf("stored replica record = %+v, want role replica in region uk", stored) |
| 2660 | } |
| 2661 | |
| 2662 | // The home record's placement set: canonicalized by the door. The set |
| 2663 | // is the HOME's opt-in, carried on a PLAIN record — role stays empty. |
| 2664 | status, _, out = rawReq(t, http.MethodPost, u, token, |
| 2665 | `{"host":"acme.go.tito.io","region":"ie","replicas":["uk"," ie ","uk"]}`) |
| 2666 | if status != http.StatusCreated { |
| 2667 | t.Fatalf("home create = %d %s, want 201", status, out) |
| 2668 | } |
| 2669 | home, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "acme.go.tito.io") |
| 2670 | if err != nil { |
| 2671 | t.Fatal(err) |
| 2672 | } |
| 2673 | if got := strings.Join(home.Replicas, ","); got != "ie,uk" { |
| 2674 | t.Fatalf("stored replica set = %q, want canonical ie,uk", got) |
| 2675 | } |
| 2676 | |
| 2677 | // Half a decision is refused, each way. |
| 2678 | for _, c := range []struct{ what, body, want string }{ |
| 2679 | {"role without region", `{"host":"half1.go.tito.io","role":"replica"}`, "needs a placement"}, |
| 2680 | {"replica with a set", `{"host":"half2.go.tito.io","role":"replica","region":"uk","replicas":["uk"]}`, "HOME record's opt-in"}, |
| 2681 | {"bad placement", `{"host":"half3.go.tito.io","replicas":["fleet/pi"]}`, "placement code"}, |
| 2682 | } { |
| 2683 | status, _, out := rawReq(t, http.MethodPost, u, token, c.body) |
| 2684 | if status != http.StatusBadRequest { |
| 2685 | t.Errorf("%s = %d %s, want 400", c.what, status, out) |
| 2686 | continue |
| 2687 | } |
| 2688 | if !strings.Contains(out, c.want) { |
| 2689 | t.Errorf("%s refusal = %s, want it to name %q", c.what, out, c.want) |
| 2690 | } |
| 2691 | } |
| 2692 | keys, _ := fx.rel.List(context.Background(), provision.RecordsPrefix("bab", "titogo")) |
| 2693 | if len(keys) != 2 { |
| 2694 | t.Fatalf("refused creates left records: %v", keys) |
| 2695 | } |
| 2696 | |
| 2697 | // A shapeless create writes no replica keys. |
| 2698 | if status, _, out := rawReq(t, http.MethodPost, u, token, |
| 2699 | `{"host":"plain.go.tito.io","region":"ie"}`); status != http.StatusCreated { |
| 2700 | t.Fatalf("plain create = %d %s, want 201", status, out) |
| 2701 | } |
| 2702 | plain, _, err := fx.rel.Get(context.Background(), provision.RecordKey("bab", "titogo", "plain.go.tito.io")) |
| 2703 | if err != nil { |
| 2704 | t.Fatal(err) |
| 2705 | } |
| 2706 | for _, k := range []string{`"role"`, `"replicas"`, `"replica"`} { |
| 2707 | if strings.Contains(string(plain), k) { |
| 2708 | t.Errorf("a shapeless create wrote %s into the record: %s", k, plain) |
| 2709 | } |
| 2710 | } |
| 2711 | } |
| 2712 | |
| 2713 | // letStateUpstreams gives addr the operator bit. Only an operator may point |
| 2714 | // an instance at its own backends (refuseMemberUpstreams), so tests about |
| 2715 | // something else that need an addr-backed record use this to make one. |
| 2716 | func letStateUpstreams(t *testing.T, fx *consoleFixture, addr string) { |
| 2717 | t.Helper() |
| 2718 | if err := fx.st.SetOperator(addr, true); err != nil { |
| 2719 | t.Fatal(err) |
| 2720 | } |
| 2721 | } |
| 2722 | |
| 2723 | // TestAPIInstanceUpstreamsAreOperatorOnly: a member may not point an |
| 2724 | // instance at its own backends, at any address, public included — the edge |
| 2725 | // dials these from inside the platform, and a member's could reach the box's |
| 2726 | // own services or another tenant's private upstream (discussion 214, item 4; |
| 2727 | // Paul's ruling on discussion 50, option a). Operators keep the old rail, |
| 2728 | // which is where the platform's own addr-backed routes point. |
| 2729 | // |
| 2730 | // Nothing a member already has is taken away: a record an operator pointed |
| 2731 | // somewhere keeps its upstreams through a member's unrelated PATCH, and the |
| 2732 | // member may restate or drop the addresses already on it. |
| 2733 | func TestAPIInstanceUpstreamsAreOperatorOnly(t *testing.T) { |
| 2734 | fx, member := instanceFixture(t, "code-inst-internal") |
| 2735 | enableInstances(t, fx, "titogo", titoManifest()) |
| 2736 | if err := fx.st.SetOperator("paul@keymail.dev", true); err != nil { |
| 2737 | t.Fatal(err) |
| 2738 | } |
| 2739 | operator := cliToken(t, fx, "paul@keymail.dev", "code-inst-internal-op") |
| 2740 | u := instancesURL(fx, "bab", "titogo") |
| 2741 | const refusal = "only a platform operator can point an instance at its own backends" |
| 2742 | |
| 2743 | for _, addr := range []string{"203.0.113.7:443", "127.0.0.1:8090", "10.0.0.5:3000", "[fd00::5]:3000"} { |
| 2744 | body := `{"host":"in.go.tito.io","region":"ie","kind":"service","addrs":["` + addr + `"]}` |
| 2745 | status, _, out := rawReq(t, http.MethodPost, u, member, body) |
| 2746 | if status != http.StatusForbidden || !strings.Contains(out, refusal) { |
| 2747 | t.Errorf("member create with %s = %d %s, want 403 %q", addr, status, out, refusal) |
| 2748 | } |
| 2749 | } |
| 2750 | if _, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "in.go.tito.io"); !errors.Is(err, os.ErrNotExist) { |
| 2751 | t.Fatalf("a refused create left a record behind: %v", err) |
| 2752 | } |
| 2753 | // A member's create that states no upstream is untouched by the rail. |
| 2754 | if status, _, out := rawReq(t, http.MethodPost, u, member, `{"host":"plain.go.tito.io","region":"ie"}`); status != http.StatusCreated { |
| 2755 | t.Fatalf("member create without addrs = %d %s, want 201", status, out) |
| 2756 | } |
| 2757 | |
| 2758 | // The operator's create is the platform's own path, unchanged. |
| 2759 | if status, _, out := rawReq(t, http.MethodPost, u, operator, |
| 2760 | `{"host":"in.go.tito.io","region":"ie","kind":"service","addrs":["127.0.0.1:7001","127.0.0.1:7002"]}`); status != http.StatusCreated { |
| 2761 | t.Fatalf("operator create with loopback = %d %s, want 201", status, out) |
| 2762 | } |
| 2763 | pu := updateURL(fx, "bab", "titogo", "in.go.tito.io") |
| 2764 | |
| 2765 | // A member's PATCH of other fields leaves the upstreams standing. |
| 2766 | for _, body := range []string{`{"probe":{"path":"/up"}}`, `{"environment":"acme"}`, `{"steering":""}`} { |
| 2767 | if status, _, out := rawReq(t, http.MethodPatch, pu, member, body); status != http.StatusOK { |
| 2768 | t.Fatalf("member patch %s on an operator-made route = %d %s, want 200", body, status, out) |
| 2769 | } |
| 2770 | } |
| 2771 | stored, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "in.go.tito.io") |
| 2772 | if err != nil { |
| 2773 | t.Fatal(err) |
| 2774 | } |
| 2775 | if !slices.Equal(stored.Addrs, []string{"127.0.0.1:7001", "127.0.0.1:7002"}) { |
| 2776 | t.Fatalf("stored addrs after member patches = %v, want the operator's pair untouched", stored.Addrs) |
| 2777 | } |
| 2778 | // It may not add one, public or not... |
| 2779 | for _, body := range []string{`{"addrs":["127.0.0.1:7001","192.168.1.1:7003"]}`, `{"addrs":["127.0.0.1:7001","203.0.113.7:443"]}`} { |
| 2780 | if status, _, out := rawReq(t, http.MethodPatch, pu, member, body); status != http.StatusForbidden || !strings.Contains(out, refusal) { |
| 2781 | t.Fatalf("member patch %s = %d %s, want 403 %q", body, status, out, refusal) |
| 2782 | } |
| 2783 | } |
| 2784 | // ...but may restate what is there, or drop a dead one. |
| 2785 | for _, body := range []string{`{"addrs":["127.0.0.1:7002","127.0.0.1:7001"]}`, `{"addrs":["127.0.0.1:7001"]}`} { |
| 2786 | if status, _, out := rawReq(t, http.MethodPatch, pu, member, body); status != http.StatusOK { |
| 2787 | t.Fatalf("member patch %s = %d %s, want 200", body, status, out) |
| 2788 | } |
| 2789 | } |
| 2790 | // The operator's PATCH keeps the old rail. |
| 2791 | if status, _, out := rawReq(t, http.MethodPatch, pu, operator, `{"addrs":["10.0.0.5:7001"]}`); status != http.StatusOK { |
| 2792 | t.Fatalf("operator patch to a private addr = %d %s, want 200", status, out) |
| 2793 | } |
| 2794 | stored, err = provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "in.go.tito.io") |
| 2795 | if err != nil { |
| 2796 | t.Fatal(err) |
| 2797 | } |
| 2798 | if !slices.Equal(stored.Addrs, []string{"10.0.0.5:7001"}) { |
| 2799 | t.Fatalf("stored addrs = %v, want the operator's", stored.Addrs) |
| 2800 | } |
| 2801 | |
| 2802 | // A service credential is not a person, operator or not. |
| 2803 | appTok, err := fx.st.NewAppToken(OpsAccountID, "titogo", "paul@keymail.dev", "ci", time.Now()) |
| 2804 | if err != nil { |
| 2805 | t.Fatal(err) |
| 2806 | } |
| 2807 | if status, _, out := rawReq(t, http.MethodPatch, pu, appTok, `{"addrs":["10.0.0.6:7001"]}`); status != http.StatusForbidden { |
| 2808 | t.Fatalf("app token patch to a new addr = %d %s, want 403", status, out) |
| 2809 | } |
| 2810 | } |
| 2811 | |
| 2812 | // TestExistingUpstreamRecordsKeepServing: records already in the bucket with |
| 2813 | // upstreams — including the legacy single Addr, written before this rail by |
| 2814 | // whoever could then — still pass the shape rail the box reconciles with |
| 2815 | // (which this branch does not touch; reconcile_test.go's addr-backed |
| 2816 | // records cover the minting) and survive a member's PATCH of unrelated |
| 2817 | // fields. The rail governs only what a member newly states. |
| 2818 | func TestExistingUpstreamRecordsKeepServing(t *testing.T) { |
| 2819 | fx, member := instanceFixture(t, "code-inst-legacy") |
| 2820 | enableInstances(t, fx, "titogo", titoManifest()) |
| 2821 | rec := provision.Record{Schema: 1, Host: "legacy.go.tito.io", Slug: "legacy", App: "titogo", |
| 2822 | Account: "bab", Pool: "tito", Region: "ie", Channel: "stable", Kind: "service", |
| 2823 | Addr: "10.0.0.1:8090", CreatedAt: time.Now().UTC(), CreatedBy: "alice@keymail.dev"} |
| 2824 | if err := provision.ValidateShape(rec); err != nil { |
| 2825 | t.Fatalf("an existing private-addr record no longer validates: %v", err) |
| 2826 | } |
| 2827 | if err := provision.CreateRecord(context.Background(), fx.rel, rec); err != nil { |
| 2828 | t.Fatal(err) |
| 2829 | } |
| 2830 | pu := updateURL(fx, "bab", "titogo", "legacy.go.tito.io") |
| 2831 | if status, _, out := rawReq(t, http.MethodPatch, pu, member, `{"probe":{"path":"/up"}}`); status != http.StatusOK { |
| 2832 | t.Fatalf("member probe patch on a legacy addr record = %d %s, want 200", status, out) |
| 2833 | } |
| 2834 | got, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "legacy.go.tito.io") |
| 2835 | if err != nil { |
| 2836 | t.Fatal(err) |
| 2837 | } |
| 2838 | if !slices.Equal(got.AddrsOrDefault(), []string{"10.0.0.1:8090"}) { |
| 2839 | t.Fatalf("upstreams after a member probe patch = %v, want the record's own", got.AddrsOrDefault()) |
| 2840 | } |
| 2841 | // Restating the legacy addr through the addrs door moves it into the set, |
| 2842 | // as an operator's PATCH would; that is not a new upstream. |
| 2843 | if status, _, out := rawReq(t, http.MethodPatch, pu, member, `{"addrs":["10.0.0.1:8090"]}`); status != http.StatusOK { |
| 2844 | t.Fatalf("member restating the legacy addr = %d %s, want 200", status, out) |
| 2845 | } |
| 2846 | } |
| 2847 | |
amadan